344 	struct intel_ringbuffer *ringbuf0 = to0->engine[ring->id].ringbuf;  in execlists_submit_contexts()
355 		ringbuf1 = to1->engine[ring->id].ringbuf;  in execlists_submit_contexts()
408 			struct intel_ringbuffer *ringbuf;  in execlists_context_unqueue()  local
410 			ringbuf = req0->ctx->engine[ring->id].ringbuf;  in execlists_context_unqueue()
412 			req0->tail &= ringbuf->size - 1;  in execlists_context_unqueue()
582 static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf,  in logical_ring_invalidate_all_caches()  argument
585 	struct intel_engine_cs *ring = ringbuf->ring;  in logical_ring_invalidate_all_caches()
593 	ret = ring->emit_flush(ringbuf, ctx,  in logical_ring_invalidate_all_caches()
602 static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf,  in execlists_move_to_gpu()  argument
606 	struct intel_engine_cs *ring = ringbuf->ring;  in execlists_move_to_gpu()
631 	return logical_ring_invalidate_all_caches(ringbuf, ctx);  in execlists_move_to_gpu()
660 	struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;  in intel_execlists_submission()  local
711 	ret = execlists_move_to_gpu(ringbuf, ctx, vmas);  in intel_execlists_submission()
717 		ret = intel_logical_ring_begin(ringbuf, ctx, 4);  in intel_execlists_submission()
721 		intel_logical_ring_emit(ringbuf, MI_NOOP);  in intel_execlists_submission()
722 		intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));  in intel_execlists_submission()
723 		intel_logical_ring_emit(ringbuf, INSTPM);  in intel_execlists_submission()
724 		intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);  in intel_execlists_submission()
725 		intel_logical_ring_advance(ringbuf);  in intel_execlists_submission()
730 	ret = ring->emit_bb_start(ringbuf, ctx, exec_start, dispatch_flags);  in intel_execlists_submission()
793 int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf,  in logical_ring_flush_all_caches()  argument
796 	struct intel_engine_cs *ring = ringbuf->ring;  in logical_ring_flush_all_caches()
802 	ret = ring->emit_flush(ringbuf, ctx, 0, I915_GEM_GPU_DOMAINS);  in logical_ring_flush_all_caches()
820 intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf,  in intel_logical_ring_advance_and_submit()  argument
824 	struct intel_engine_cs *ring = ringbuf->ring;  in intel_logical_ring_advance_and_submit()
826 	intel_logical_ring_advance(ringbuf);  in intel_logical_ring_advance_and_submit()
831 	execlists_context_queue(ring, ctx, ringbuf->tail, request);  in intel_logical_ring_advance_and_submit()
838 	struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;  in intel_lr_context_pin()  local
848 		ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);  in intel_lr_context_pin()
869 	struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;  in intel_lr_context_unpin()  local
874 			intel_unpin_ringbuffer_obj(ringbuf);  in intel_lr_context_unpin()
915 	request->ringbuf = ctx->engine[ring->id].ringbuf;  in logical_ring_alloc_request()
921 static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf,  in logical_ring_wait_request()  argument
924 	struct intel_engine_cs *ring = ringbuf->ring;  in logical_ring_wait_request()
928 	if (intel_ring_space(ringbuf) >= bytes)  in logical_ring_wait_request()
938 		if (ctx->engine[ring->id].ringbuf != ringbuf)  in logical_ring_wait_request()
942 		if (__intel_ring_space(request->tail, ringbuf->tail,  in logical_ring_wait_request()
943 				       ringbuf->size) >= bytes) {  in logical_ring_wait_request()
957 	return intel_ring_space(ringbuf) >= bytes ? 0 : -ENOSPC;  in logical_ring_wait_request()
960 static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,  in logical_ring_wait_for_space()  argument
964 	struct intel_engine_cs *ring = ringbuf->ring;  in logical_ring_wait_for_space()
970 	ret = logical_ring_wait_request(ringbuf, bytes);  in logical_ring_wait_for_space()
975 	intel_logical_ring_advance_and_submit(ringbuf, ctx, NULL);  in logical_ring_wait_for_space()
986 		if (intel_ring_space(ringbuf) >= bytes)  in logical_ring_wait_for_space()
1010 static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf,  in logical_ring_wrap_buffer()  argument
1014 	int rem = ringbuf->size - ringbuf->tail;  in logical_ring_wrap_buffer()
1016 	if (ringbuf->space < rem) {  in logical_ring_wrap_buffer()
1017 		int ret = logical_ring_wait_for_space(ringbuf, ctx, rem);  in logical_ring_wrap_buffer()
1023 	virt = ringbuf->virtual_start + ringbuf->tail;  in logical_ring_wrap_buffer()
1028 	ringbuf->tail = 0;  in logical_ring_wrap_buffer()
1029 	intel_ring_update_space(ringbuf);  in logical_ring_wrap_buffer()
1034 static int logical_ring_prepare(struct intel_ringbuffer *ringbuf,  in logical_ring_prepare()  argument
1039 	if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {  in logical_ring_prepare()
1040 		ret = logical_ring_wrap_buffer(ringbuf, ctx);  in logical_ring_prepare()
1045 	if (unlikely(ringbuf->space < bytes)) {  in logical_ring_prepare()
1046 		ret = logical_ring_wait_for_space(ringbuf, ctx, bytes);  in logical_ring_prepare()
1067 int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf,  in intel_logical_ring_begin()  argument
1070 	struct intel_engine_cs *ring = ringbuf->ring;  in intel_logical_ring_begin()
1080 	ret = logical_ring_prepare(ringbuf, ctx, num_dwords * sizeof(uint32_t));  in intel_logical_ring_begin()
1089 	ringbuf->space -= num_dwords * sizeof(uint32_t);  in intel_logical_ring_begin()
1097 	struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;  in intel_logical_ring_workarounds_emit()  local
1106 	ret = logical_ring_flush_all_caches(ringbuf, ctx);  in intel_logical_ring_workarounds_emit()
1110 	ret = intel_logical_ring_begin(ringbuf, ctx, w->count * 2 + 2);  in intel_logical_ring_workarounds_emit()
1114 	intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count));  in intel_logical_ring_workarounds_emit()
1116 		intel_logical_ring_emit(ringbuf, w->reg[i].addr);  in intel_logical_ring_workarounds_emit()
1117 		intel_logical_ring_emit(ringbuf, w->reg[i].value);  in intel_logical_ring_workarounds_emit()
1119 	intel_logical_ring_emit(ringbuf, MI_NOOP);  in intel_logical_ring_workarounds_emit()
1121 	intel_logical_ring_advance(ringbuf);  in intel_logical_ring_workarounds_emit()
1124 	ret = logical_ring_flush_all_caches(ringbuf, ctx);  in intel_logical_ring_workarounds_emit()
1191 static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf,  in gen8_emit_bb_start()  argument
1198 	ret = intel_logical_ring_begin(ringbuf, ctx, 4);  in gen8_emit_bb_start()
1203 	intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));  in gen8_emit_bb_start()
1204 	intel_logical_ring_emit(ringbuf, lower_32_bits(offset));  in gen8_emit_bb_start()
1205 	intel_logical_ring_emit(ringbuf, upper_32_bits(offset));  in gen8_emit_bb_start()
1206 	intel_logical_ring_emit(ringbuf, MI_NOOP);  in gen8_emit_bb_start()
1207 	intel_logical_ring_advance(ringbuf);  in gen8_emit_bb_start()
1245 static int gen8_emit_flush(struct intel_ringbuffer *ringbuf,  in gen8_emit_flush()  argument
1250 	struct intel_engine_cs *ring = ringbuf->ring;  in gen8_emit_flush()
1256 	ret = intel_logical_ring_begin(ringbuf, ctx, 4);  in gen8_emit_flush()
1275 	intel_logical_ring_emit(ringbuf, cmd);  in gen8_emit_flush()
1276 	intel_logical_ring_emit(ringbuf,  in gen8_emit_flush()
1279 	intel_logical_ring_emit(ringbuf, 0); /* upper addr */  in gen8_emit_flush()
1280 	intel_logical_ring_emit(ringbuf, 0); /* value */  in gen8_emit_flush()
1281 	intel_logical_ring_advance(ringbuf);  in gen8_emit_flush()
1286 static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf,  in gen8_emit_flush_render()  argument
1291 	struct intel_engine_cs *ring = ringbuf->ring;  in gen8_emit_flush_render()
1315 	ret = intel_logical_ring_begin(ringbuf, ctx, 6);  in gen8_emit_flush_render()
1319 	intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));  in gen8_emit_flush_render()
1320 	intel_logical_ring_emit(ringbuf, flags);  in gen8_emit_flush_render()
1321 	intel_logical_ring_emit(ringbuf, scratch_addr);  in gen8_emit_flush_render()
1322 	intel_logical_ring_emit(ringbuf, 0);  in gen8_emit_flush_render()
1323 	intel_logical_ring_emit(ringbuf, 0);  in gen8_emit_flush_render()
1324 	intel_logical_ring_emit(ringbuf, 0);  in gen8_emit_flush_render()
1325 	intel_logical_ring_advance(ringbuf);  in gen8_emit_flush_render()
1340 static int gen8_emit_request(struct intel_ringbuffer *ringbuf,  in gen8_emit_request()  argument
1343 	struct intel_engine_cs *ring = ringbuf->ring;  in gen8_emit_request()
1352 	ret = intel_logical_ring_begin(ringbuf, request->ctx, 8);  in gen8_emit_request()
1359 	intel_logical_ring_emit(ringbuf, cmd);  in gen8_emit_request()
1360 	intel_logical_ring_emit(ringbuf,  in gen8_emit_request()
1363 	intel_logical_ring_emit(ringbuf, 0);  in gen8_emit_request()
1364 	intel_logical_ring_emit(ringbuf,  in gen8_emit_request()
1366 	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);  in gen8_emit_request()
1367 	intel_logical_ring_emit(ringbuf, MI_NOOP);  in gen8_emit_request()
1368 	intel_logical_ring_advance_and_submit(ringbuf, request->ctx, request);  in gen8_emit_request()
1374 	intel_logical_ring_emit(ringbuf, MI_NOOP);  in gen8_emit_request()
1375 	intel_logical_ring_emit(ringbuf, MI_NOOP);  in gen8_emit_request()
1376 	intel_logical_ring_advance(ringbuf);  in gen8_emit_request()
1384 	struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;  in intel_lr_context_render_state_init()  local
1397 	ret = ring->emit_bb_start(ringbuf,  in intel_lr_context_render_state_init()
1727 		    struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)  in populate_lr_context()  argument
1782 			((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;  in populate_lr_context()
1857 			struct intel_ringbuffer *ringbuf =  in intel_lr_context_free()  local
1858 					ctx->engine[i].ringbuf;  in intel_lr_context_free()
1859 			struct intel_engine_cs *ring = ringbuf->ring;  in intel_lr_context_free()
1862 				intel_unpin_ringbuffer_obj(ringbuf);  in intel_lr_context_free()
1866 			intel_destroy_ringbuffer_obj(ringbuf);  in intel_lr_context_free()
1867 			kfree(ringbuf);  in intel_lr_context_free()
1934 	struct intel_ringbuffer *ringbuf;  in intel_lr_context_deferred_create()  local
1959 	ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);  in intel_lr_context_deferred_create()
1960 	if (!ringbuf) {  in intel_lr_context_deferred_create()
1967 	ringbuf->ring = ring;  in intel_lr_context_deferred_create()
1969 	ringbuf->size = 32 * PAGE_SIZE;  in intel_lr_context_deferred_create()
1970 	ringbuf->effective_size = ringbuf->size;  in intel_lr_context_deferred_create()
1971 	ringbuf->head = 0;  in intel_lr_context_deferred_create()
1972 	ringbuf->tail = 0;  in intel_lr_context_deferred_create()
1973 	ringbuf->last_retired_head = -1;  in intel_lr_context_deferred_create()
1974 	intel_ring_update_space(ringbuf);  in intel_lr_context_deferred_create()
1976 	if (ringbuf->obj == NULL) {  in intel_lr_context_deferred_create()
1977 		ret = intel_alloc_ringbuffer_obj(dev, ringbuf);  in intel_lr_context_deferred_create()
1986 			ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);  in intel_lr_context_deferred_create()
1997 	ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);  in intel_lr_context_deferred_create()
2003 	ctx->engine[ring->id].ringbuf = ringbuf;  in intel_lr_context_deferred_create()
2013 				ctx->engine[ring->id].ringbuf = NULL;  in intel_lr_context_deferred_create()
2026 		intel_unpin_ringbuffer_obj(ringbuf);  in intel_lr_context_deferred_create()
2028 	intel_destroy_ringbuffer_obj(ringbuf);  in intel_lr_context_deferred_create()
2030 	kfree(ringbuf);  in intel_lr_context_deferred_create()
2048 		struct intel_ringbuffer *ringbuf =  in intel_lr_context_reset()  local
2049 				ctx->engine[ring->id].ringbuf;  in intel_lr_context_reset()
2068 		ringbuf->head = 0;  in intel_lr_context_reset()
2069 		ringbuf->tail = 0;  in intel_lr_context_reset()