Lines Matching refs:ring
37 intel_ring_initialized(struct intel_engine_cs *ring) in intel_ring_initialized() argument
39 struct drm_device *dev = ring->dev; in intel_ring_initialized()
45 struct intel_context *dctx = ring->default_context; in intel_ring_initialized()
46 struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf; in intel_ring_initialized()
50 return ring->buffer && ring->buffer->obj; in intel_ring_initialized()
78 bool intel_ring_stopped(struct intel_engine_cs *ring) in intel_ring_stopped() argument
80 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_stopped()
81 return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring); in intel_ring_stopped()
84 void __intel_ring_advance(struct intel_engine_cs *ring) in __intel_ring_advance() argument
86 struct intel_ringbuffer *ringbuf = ring->buffer; in __intel_ring_advance()
88 if (intel_ring_stopped(ring)) in __intel_ring_advance()
90 ring->write_tail(ring, ringbuf->tail); in __intel_ring_advance()
94 gen2_render_ring_flush(struct intel_engine_cs *ring, in gen2_render_ring_flush() argument
108 ret = intel_ring_begin(ring, 2); in gen2_render_ring_flush()
112 intel_ring_emit(ring, cmd); in gen2_render_ring_flush()
113 intel_ring_emit(ring, MI_NOOP); in gen2_render_ring_flush()
114 intel_ring_advance(ring); in gen2_render_ring_flush()
120 gen4_render_ring_flush(struct intel_engine_cs *ring, in gen4_render_ring_flush() argument
124 struct drm_device *dev = ring->dev; in gen4_render_ring_flush()
166 ret = intel_ring_begin(ring, 2); in gen4_render_ring_flush()
170 intel_ring_emit(ring, cmd); in gen4_render_ring_flush()
171 intel_ring_emit(ring, MI_NOOP); in gen4_render_ring_flush()
172 intel_ring_advance(ring); in gen4_render_ring_flush()
215 intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) in intel_emit_post_sync_nonzero_flush() argument
217 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in intel_emit_post_sync_nonzero_flush()
221 ret = intel_ring_begin(ring, 6); in intel_emit_post_sync_nonzero_flush()
225 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
226 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in intel_emit_post_sync_nonzero_flush()
228 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
229 intel_ring_emit(ring, 0); /* low dword */ in intel_emit_post_sync_nonzero_flush()
230 intel_ring_emit(ring, 0); /* high dword */ in intel_emit_post_sync_nonzero_flush()
231 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
232 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
234 ret = intel_ring_begin(ring, 6); in intel_emit_post_sync_nonzero_flush()
238 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
239 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); in intel_emit_post_sync_nonzero_flush()
240 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
241 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
242 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
243 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
244 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
250 gen6_render_ring_flush(struct intel_engine_cs *ring, in gen6_render_ring_flush() argument
254 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen6_render_ring_flush()
258 ret = intel_emit_post_sync_nonzero_flush(ring); in gen6_render_ring_flush()
288 ret = intel_ring_begin(ring, 4); in gen6_render_ring_flush()
292 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen6_render_ring_flush()
293 intel_ring_emit(ring, flags); in gen6_render_ring_flush()
294 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); in gen6_render_ring_flush()
295 intel_ring_emit(ring, 0); in gen6_render_ring_flush()
296 intel_ring_advance(ring); in gen6_render_ring_flush()
302 gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) in gen7_render_ring_cs_stall_wa() argument
306 ret = intel_ring_begin(ring, 4); in gen7_render_ring_cs_stall_wa()
310 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_cs_stall_wa()
311 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in gen7_render_ring_cs_stall_wa()
313 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
314 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
315 intel_ring_advance(ring); in gen7_render_ring_cs_stall_wa()
321 gen7_render_ring_flush(struct intel_engine_cs *ring, in gen7_render_ring_flush() argument
325 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen7_render_ring_flush()
366 gen7_render_ring_cs_stall_wa(ring); in gen7_render_ring_flush()
369 ret = intel_ring_begin(ring, 4); in gen7_render_ring_flush()
373 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_flush()
374 intel_ring_emit(ring, flags); in gen7_render_ring_flush()
375 intel_ring_emit(ring, scratch_addr); in gen7_render_ring_flush()
376 intel_ring_emit(ring, 0); in gen7_render_ring_flush()
377 intel_ring_advance(ring); in gen7_render_ring_flush()
383 gen8_emit_pipe_control(struct intel_engine_cs *ring, in gen8_emit_pipe_control() argument
388 ret = intel_ring_begin(ring, 6); in gen8_emit_pipe_control()
392 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); in gen8_emit_pipe_control()
393 intel_ring_emit(ring, flags); in gen8_emit_pipe_control()
394 intel_ring_emit(ring, scratch_addr); in gen8_emit_pipe_control()
395 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
396 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
397 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
398 intel_ring_advance(ring); in gen8_emit_pipe_control()
404 gen8_render_ring_flush(struct intel_engine_cs *ring, in gen8_render_ring_flush() argument
408 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen8_render_ring_flush()
429 ret = gen8_emit_pipe_control(ring, in gen8_render_ring_flush()
437 return gen8_emit_pipe_control(ring, flags, scratch_addr); in gen8_render_ring_flush()
440 static void ring_write_tail(struct intel_engine_cs *ring, in ring_write_tail() argument
443 struct drm_i915_private *dev_priv = ring->dev->dev_private; in ring_write_tail()
444 I915_WRITE_TAIL(ring, value); in ring_write_tail()
447 u64 intel_ring_get_active_head(struct intel_engine_cs *ring) in intel_ring_get_active_head() argument
449 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_get_active_head()
452 if (INTEL_INFO(ring->dev)->gen >= 8) in intel_ring_get_active_head()
453 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base), in intel_ring_get_active_head()
454 RING_ACTHD_UDW(ring->mmio_base)); in intel_ring_get_active_head()
455 else if (INTEL_INFO(ring->dev)->gen >= 4) in intel_ring_get_active_head()
456 acthd = I915_READ(RING_ACTHD(ring->mmio_base)); in intel_ring_get_active_head()
463 static void ring_setup_phys_status_page(struct intel_engine_cs *ring) in ring_setup_phys_status_page() argument
465 struct drm_i915_private *dev_priv = ring->dev->dev_private; in ring_setup_phys_status_page()
469 if (INTEL_INFO(ring->dev)->gen >= 4) in ring_setup_phys_status_page()
474 static void intel_ring_setup_status_page(struct intel_engine_cs *ring) in intel_ring_setup_status_page() argument
476 struct drm_device *dev = ring->dev; in intel_ring_setup_status_page()
477 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_setup_status_page()
484 switch (ring->id) { in intel_ring_setup_status_page()
503 } else if (IS_GEN6(ring->dev)) { in intel_ring_setup_status_page()
504 mmio = RING_HWS_PGA_GEN6(ring->mmio_base); in intel_ring_setup_status_page()
507 mmio = RING_HWS_PGA(ring->mmio_base); in intel_ring_setup_status_page()
510 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); in intel_ring_setup_status_page()
521 u32 reg = RING_INSTPM(ring->mmio_base); in intel_ring_setup_status_page()
524 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); in intel_ring_setup_status_page()
532 ring->name); in intel_ring_setup_status_page()
536 static bool stop_ring(struct intel_engine_cs *ring) in stop_ring() argument
538 struct drm_i915_private *dev_priv = to_i915(ring->dev); in stop_ring()
540 if (!IS_GEN2(ring->dev)) { in stop_ring()
541 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING)); in stop_ring()
542 if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { in stop_ring()
543 DRM_ERROR("%s : timed out trying to stop ring\n", ring->name); in stop_ring()
548 if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring)) in stop_ring()
553 I915_WRITE_CTL(ring, 0); in stop_ring()
554 I915_WRITE_HEAD(ring, 0); in stop_ring()
555 ring->write_tail(ring, 0); in stop_ring()
557 if (!IS_GEN2(ring->dev)) { in stop_ring()
558 (void)I915_READ_CTL(ring); in stop_ring()
559 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); in stop_ring()
562 return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0; in stop_ring()
565 static int init_ring_common(struct intel_engine_cs *ring) in init_ring_common() argument
567 struct drm_device *dev = ring->dev; in init_ring_common()
569 struct intel_ringbuffer *ringbuf = ring->buffer; in init_ring_common()
575 if (!stop_ring(ring)) { in init_ring_common()
579 ring->name, in init_ring_common()
580 I915_READ_CTL(ring), in init_ring_common()
581 I915_READ_HEAD(ring), in init_ring_common()
582 I915_READ_TAIL(ring), in init_ring_common()
583 I915_READ_START(ring)); in init_ring_common()
585 if (!stop_ring(ring)) { in init_ring_common()
588 ring->name, in init_ring_common()
589 I915_READ_CTL(ring), in init_ring_common()
590 I915_READ_HEAD(ring), in init_ring_common()
591 I915_READ_TAIL(ring), in init_ring_common()
592 I915_READ_START(ring)); in init_ring_common()
599 intel_ring_setup_status_page(ring); in init_ring_common()
601 ring_setup_phys_status_page(ring); in init_ring_common()
604 I915_READ_HEAD(ring); in init_ring_common()
610 I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj)); in init_ring_common()
613 if (I915_READ_HEAD(ring)) in init_ring_common()
615 ring->name, I915_READ_HEAD(ring)); in init_ring_common()
616 I915_WRITE_HEAD(ring, 0); in init_ring_common()
617 (void)I915_READ_HEAD(ring); in init_ring_common()
619 I915_WRITE_CTL(ring, in init_ring_common()
624 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 && in init_ring_common()
625 I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) && in init_ring_common()
626 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { in init_ring_common()
629 ring->name, in init_ring_common()
630 I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID, in init_ring_common()
631 I915_READ_HEAD(ring), I915_READ_TAIL(ring), in init_ring_common()
632 I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj)); in init_ring_common()
638 ringbuf->head = I915_READ_HEAD(ring); in init_ring_common()
639 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; in init_ring_common()
642 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); in init_ring_common()
651 intel_fini_pipe_control(struct intel_engine_cs *ring) in intel_fini_pipe_control() argument
653 struct drm_device *dev = ring->dev; in intel_fini_pipe_control()
655 if (ring->scratch.obj == NULL) in intel_fini_pipe_control()
659 kunmap(sg_page(ring->scratch.obj->pages->sgl)); in intel_fini_pipe_control()
660 i915_gem_object_ggtt_unpin(ring->scratch.obj); in intel_fini_pipe_control()
663 drm_gem_object_unreference(&ring->scratch.obj->base); in intel_fini_pipe_control()
664 ring->scratch.obj = NULL; in intel_fini_pipe_control()
668 intel_init_pipe_control(struct intel_engine_cs *ring) in intel_init_pipe_control() argument
672 WARN_ON(ring->scratch.obj); in intel_init_pipe_control()
674 ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096); in intel_init_pipe_control()
675 if (ring->scratch.obj == NULL) { in intel_init_pipe_control()
681 ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); in intel_init_pipe_control()
685 ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0); in intel_init_pipe_control()
689 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj); in intel_init_pipe_control()
690 ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl)); in intel_init_pipe_control()
691 if (ring->scratch.cpu_page == NULL) { in intel_init_pipe_control()
697 ring->name, ring->scratch.gtt_offset); in intel_init_pipe_control()
701 i915_gem_object_ggtt_unpin(ring->scratch.obj); in intel_init_pipe_control()
703 drm_gem_object_unreference(&ring->scratch.obj->base); in intel_init_pipe_control()
708 static int intel_ring_workarounds_emit(struct intel_engine_cs *ring, in intel_ring_workarounds_emit() argument
712 struct drm_device *dev = ring->dev; in intel_ring_workarounds_emit()
719 ring->gpu_caches_dirty = true; in intel_ring_workarounds_emit()
720 ret = intel_ring_flush_all_caches(ring); in intel_ring_workarounds_emit()
724 ret = intel_ring_begin(ring, (w->count * 2 + 2)); in intel_ring_workarounds_emit()
728 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count)); in intel_ring_workarounds_emit()
730 intel_ring_emit(ring, w->reg[i].addr); in intel_ring_workarounds_emit()
731 intel_ring_emit(ring, w->reg[i].value); in intel_ring_workarounds_emit()
733 intel_ring_emit(ring, MI_NOOP); in intel_ring_workarounds_emit()
735 intel_ring_advance(ring); in intel_ring_workarounds_emit()
737 ring->gpu_caches_dirty = true; in intel_ring_workarounds_emit()
738 ret = intel_ring_flush_all_caches(ring); in intel_ring_workarounds_emit()
747 static int intel_rcs_ctx_init(struct intel_engine_cs *ring, in intel_rcs_ctx_init() argument
752 ret = intel_ring_workarounds_emit(ring, ctx); in intel_rcs_ctx_init()
756 ret = i915_gem_render_state_init(ring); in intel_rcs_ctx_init()
800 static int bdw_init_workarounds(struct intel_engine_cs *ring) in bdw_init_workarounds() argument
802 struct drm_device *dev = ring->dev; in bdw_init_workarounds()
861 static int chv_init_workarounds(struct intel_engine_cs *ring) in chv_init_workarounds() argument
863 struct drm_device *dev = ring->dev; in chv_init_workarounds()
909 static int gen9_init_workarounds(struct intel_engine_cs *ring) in gen9_init_workarounds() argument
911 struct drm_device *dev = ring->dev; in gen9_init_workarounds()
967 static int skl_tune_iz_hashing(struct intel_engine_cs *ring) in skl_tune_iz_hashing() argument
969 struct drm_device *dev = ring->dev; in skl_tune_iz_hashing()
1010 static int skl_init_workarounds(struct intel_engine_cs *ring) in skl_init_workarounds() argument
1012 struct drm_device *dev = ring->dev; in skl_init_workarounds()
1015 gen9_init_workarounds(ring); in skl_init_workarounds()
1029 return skl_tune_iz_hashing(ring); in skl_init_workarounds()
1032 int init_workarounds_ring(struct intel_engine_cs *ring) in init_workarounds_ring() argument
1034 struct drm_device *dev = ring->dev; in init_workarounds_ring()
1037 WARN_ON(ring->id != RCS); in init_workarounds_ring()
1042 return bdw_init_workarounds(ring); in init_workarounds_ring()
1045 return chv_init_workarounds(ring); in init_workarounds_ring()
1048 return skl_init_workarounds(ring); in init_workarounds_ring()
1050 return gen9_init_workarounds(ring); in init_workarounds_ring()
1055 static int init_render_ring(struct intel_engine_cs *ring) in init_render_ring() argument
1057 struct drm_device *dev = ring->dev; in init_render_ring()
1059 int ret = init_ring_common(ring); in init_render_ring()
1102 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); in init_render_ring()
1104 return init_workarounds_ring(ring); in init_render_ring()
1107 static void render_ring_cleanup(struct intel_engine_cs *ring) in render_ring_cleanup() argument
1109 struct drm_device *dev = ring->dev; in render_ring_cleanup()
1118 intel_fini_pipe_control(ring); in render_ring_cleanup()
1246 gen6_add_request(struct intel_engine_cs *ring) in gen6_add_request() argument
1250 if (ring->semaphore.signal) in gen6_add_request()
1251 ret = ring->semaphore.signal(ring, 4); in gen6_add_request()
1253 ret = intel_ring_begin(ring, 4); in gen6_add_request()
1258 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in gen6_add_request()
1259 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in gen6_add_request()
1260 intel_ring_emit(ring, in gen6_add_request()
1261 i915_gem_request_get_seqno(ring->outstanding_lazy_request)); in gen6_add_request()
1262 intel_ring_emit(ring, MI_USER_INTERRUPT); in gen6_add_request()
1263 __intel_ring_advance(ring); in gen6_add_request()
1358 pc_render_add_request(struct intel_engine_cs *ring) in pc_render_add_request() argument
1360 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in pc_render_add_request()
1371 ret = intel_ring_begin(ring, 32); in pc_render_add_request()
1375 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
1378 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
1379 intel_ring_emit(ring, in pc_render_add_request()
1380 i915_gem_request_get_seqno(ring->outstanding_lazy_request)); in pc_render_add_request()
1381 intel_ring_emit(ring, 0); in pc_render_add_request()
1382 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1384 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1386 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1388 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1390 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1392 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1394 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
1398 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
1399 intel_ring_emit(ring, in pc_render_add_request()
1400 i915_gem_request_get_seqno(ring->outstanding_lazy_request)); in pc_render_add_request()
1401 intel_ring_emit(ring, 0); in pc_render_add_request()
1402 __intel_ring_advance(ring); in pc_render_add_request()
1408 gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in gen6_ring_get_seqno() argument
1414 struct drm_i915_private *dev_priv = ring->dev->dev_private; in gen6_ring_get_seqno()
1415 POSTING_READ(RING_ACTHD(ring->mmio_base)); in gen6_ring_get_seqno()
1418 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in gen6_ring_get_seqno()
1422 ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in ring_get_seqno() argument
1424 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in ring_get_seqno()
1428 ring_set_seqno(struct intel_engine_cs *ring, u32 seqno) in ring_set_seqno() argument
1430 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); in ring_set_seqno()
1434 pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in pc_render_get_seqno() argument
1436 return ring->scratch.cpu_page[0]; in pc_render_get_seqno()
1440 pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno) in pc_render_set_seqno() argument
1442 ring->scratch.cpu_page[0] = seqno; in pc_render_set_seqno()
1446 gen5_ring_get_irq(struct intel_engine_cs *ring) in gen5_ring_get_irq() argument
1448 struct drm_device *dev = ring->dev; in gen5_ring_get_irq()
1456 if (ring->irq_refcount++ == 0) in gen5_ring_get_irq()
1457 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); in gen5_ring_get_irq()
1464 gen5_ring_put_irq(struct intel_engine_cs *ring) in gen5_ring_put_irq() argument
1466 struct drm_device *dev = ring->dev; in gen5_ring_put_irq()
1471 if (--ring->irq_refcount == 0) in gen5_ring_put_irq()
1472 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); in gen5_ring_put_irq()
1477 i9xx_ring_get_irq(struct intel_engine_cs *ring) in i9xx_ring_get_irq() argument
1479 struct drm_device *dev = ring->dev; in i9xx_ring_get_irq()
1487 if (ring->irq_refcount++ == 0) { in i9xx_ring_get_irq()
1488 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i9xx_ring_get_irq()
1498 i9xx_ring_put_irq(struct intel_engine_cs *ring) in i9xx_ring_put_irq() argument
1500 struct drm_device *dev = ring->dev; in i9xx_ring_put_irq()
1505 if (--ring->irq_refcount == 0) { in i9xx_ring_put_irq()
1506 dev_priv->irq_mask |= ring->irq_enable_mask; in i9xx_ring_put_irq()
1514 i8xx_ring_get_irq(struct intel_engine_cs *ring) in i8xx_ring_get_irq() argument
1516 struct drm_device *dev = ring->dev; in i8xx_ring_get_irq()
1524 if (ring->irq_refcount++ == 0) { in i8xx_ring_get_irq()
1525 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i8xx_ring_get_irq()
1535 i8xx_ring_put_irq(struct intel_engine_cs *ring) in i8xx_ring_put_irq() argument
1537 struct drm_device *dev = ring->dev; in i8xx_ring_put_irq()
1542 if (--ring->irq_refcount == 0) { in i8xx_ring_put_irq()
1543 dev_priv->irq_mask |= ring->irq_enable_mask; in i8xx_ring_put_irq()
1551 bsd_ring_flush(struct intel_engine_cs *ring, in bsd_ring_flush() argument
1557 ret = intel_ring_begin(ring, 2); in bsd_ring_flush()
1561 intel_ring_emit(ring, MI_FLUSH); in bsd_ring_flush()
1562 intel_ring_emit(ring, MI_NOOP); in bsd_ring_flush()
1563 intel_ring_advance(ring); in bsd_ring_flush()
1568 i9xx_add_request(struct intel_engine_cs *ring) in i9xx_add_request() argument
1572 ret = intel_ring_begin(ring, 4); in i9xx_add_request()
1576 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in i9xx_add_request()
1577 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in i9xx_add_request()
1578 intel_ring_emit(ring, in i9xx_add_request()
1579 i915_gem_request_get_seqno(ring->outstanding_lazy_request)); in i9xx_add_request()
1580 intel_ring_emit(ring, MI_USER_INTERRUPT); in i9xx_add_request()
1581 __intel_ring_advance(ring); in i9xx_add_request()
1587 gen6_ring_get_irq(struct intel_engine_cs *ring) in gen6_ring_get_irq() argument
1589 struct drm_device *dev = ring->dev; in gen6_ring_get_irq()
1597 if (ring->irq_refcount++ == 0) { in gen6_ring_get_irq()
1598 if (HAS_L3_DPF(dev) && ring->id == RCS) in gen6_ring_get_irq()
1599 I915_WRITE_IMR(ring, in gen6_ring_get_irq()
1600 ~(ring->irq_enable_mask | in gen6_ring_get_irq()
1603 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen6_ring_get_irq()
1604 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); in gen6_ring_get_irq()
1612 gen6_ring_put_irq(struct intel_engine_cs *ring) in gen6_ring_put_irq() argument
1614 struct drm_device *dev = ring->dev; in gen6_ring_put_irq()
1619 if (--ring->irq_refcount == 0) { in gen6_ring_put_irq()
1620 if (HAS_L3_DPF(dev) && ring->id == RCS) in gen6_ring_put_irq()
1621 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); in gen6_ring_put_irq()
1623 I915_WRITE_IMR(ring, ~0); in gen6_ring_put_irq()
1624 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); in gen6_ring_put_irq()
1630 hsw_vebox_get_irq(struct intel_engine_cs *ring) in hsw_vebox_get_irq() argument
1632 struct drm_device *dev = ring->dev; in hsw_vebox_get_irq()
1640 if (ring->irq_refcount++ == 0) { in hsw_vebox_get_irq()
1641 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in hsw_vebox_get_irq()
1642 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask); in hsw_vebox_get_irq()
1650 hsw_vebox_put_irq(struct intel_engine_cs *ring) in hsw_vebox_put_irq() argument
1652 struct drm_device *dev = ring->dev; in hsw_vebox_put_irq()
1657 if (--ring->irq_refcount == 0) { in hsw_vebox_put_irq()
1658 I915_WRITE_IMR(ring, ~0); in hsw_vebox_put_irq()
1659 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask); in hsw_vebox_put_irq()
1665 gen8_ring_get_irq(struct intel_engine_cs *ring) in gen8_ring_get_irq() argument
1667 struct drm_device *dev = ring->dev; in gen8_ring_get_irq()
1675 if (ring->irq_refcount++ == 0) { in gen8_ring_get_irq()
1676 if (HAS_L3_DPF(dev) && ring->id == RCS) { in gen8_ring_get_irq()
1677 I915_WRITE_IMR(ring, in gen8_ring_get_irq()
1678 ~(ring->irq_enable_mask | in gen8_ring_get_irq()
1681 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen8_ring_get_irq()
1683 POSTING_READ(RING_IMR(ring->mmio_base)); in gen8_ring_get_irq()
1691 gen8_ring_put_irq(struct intel_engine_cs *ring) in gen8_ring_put_irq() argument
1693 struct drm_device *dev = ring->dev; in gen8_ring_put_irq()
1698 if (--ring->irq_refcount == 0) { in gen8_ring_put_irq()
1699 if (HAS_L3_DPF(dev) && ring->id == RCS) { in gen8_ring_put_irq()
1700 I915_WRITE_IMR(ring, in gen8_ring_put_irq()
1703 I915_WRITE_IMR(ring, ~0); in gen8_ring_put_irq()
1705 POSTING_READ(RING_IMR(ring->mmio_base)); in gen8_ring_put_irq()
1711 i965_dispatch_execbuffer(struct intel_engine_cs *ring, in i965_dispatch_execbuffer() argument
1717 ret = intel_ring_begin(ring, 2); in i965_dispatch_execbuffer()
1721 intel_ring_emit(ring, in i965_dispatch_execbuffer()
1726 intel_ring_emit(ring, offset); in i965_dispatch_execbuffer()
1727 intel_ring_advance(ring); in i965_dispatch_execbuffer()
1737 i830_dispatch_execbuffer(struct intel_engine_cs *ring, in i830_dispatch_execbuffer() argument
1741 u32 cs_offset = ring->scratch.gtt_offset; in i830_dispatch_execbuffer()
1744 ret = intel_ring_begin(ring, 6); in i830_dispatch_execbuffer()
1749 intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA); in i830_dispatch_execbuffer()
1750 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096); in i830_dispatch_execbuffer()
1751 intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */ in i830_dispatch_execbuffer()
1752 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1753 intel_ring_emit(ring, 0xdeadbeef); in i830_dispatch_execbuffer()
1754 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1755 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1761 ret = intel_ring_begin(ring, 6 + 2); in i830_dispatch_execbuffer()
1769 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA); in i830_dispatch_execbuffer()
1770 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096); in i830_dispatch_execbuffer()
1771 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096); in i830_dispatch_execbuffer()
1772 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1773 intel_ring_emit(ring, 4096); in i830_dispatch_execbuffer()
1774 intel_ring_emit(ring, offset); in i830_dispatch_execbuffer()
1776 intel_ring_emit(ring, MI_FLUSH); in i830_dispatch_execbuffer()
1777 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1778 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1784 ret = intel_ring_begin(ring, 4); in i830_dispatch_execbuffer()
1788 intel_ring_emit(ring, MI_BATCH_BUFFER); in i830_dispatch_execbuffer()
1789 intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ? in i830_dispatch_execbuffer()
1791 intel_ring_emit(ring, offset + len - 8); in i830_dispatch_execbuffer()
1792 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1793 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1799 i915_dispatch_execbuffer(struct intel_engine_cs *ring, in i915_dispatch_execbuffer() argument
1805 ret = intel_ring_begin(ring, 2); in i915_dispatch_execbuffer()
1809 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT); in i915_dispatch_execbuffer()
1810 intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ? in i915_dispatch_execbuffer()
1812 intel_ring_advance(ring); in i915_dispatch_execbuffer()
1817 static void cleanup_status_page(struct intel_engine_cs *ring) in cleanup_status_page() argument
1821 obj = ring->status_page.obj; in cleanup_status_page()
1828 ring->status_page.obj = NULL; in cleanup_status_page()
1831 static int init_status_page(struct intel_engine_cs *ring) in init_status_page() argument
1835 if ((obj = ring->status_page.obj) == NULL) { in init_status_page()
1839 obj = i915_gem_alloc_object(ring->dev, 4096); in init_status_page()
1850 if (!HAS_LLC(ring->dev)) in init_status_page()
1869 ring->status_page.obj = obj; in init_status_page()
1872 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); in init_status_page()
1873 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl)); in init_status_page()
1874 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_status_page()
1877 ring->name, ring->status_page.gfx_addr); in init_status_page()
1882 static int init_phys_status_page(struct intel_engine_cs *ring) in init_phys_status_page() argument
1884 struct drm_i915_private *dev_priv = ring->dev->dev_private; in init_phys_status_page()
1888 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE); in init_phys_status_page()
1893 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; in init_phys_status_page()
1894 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_phys_status_page()
1961 struct intel_engine_cs *ring) in intel_init_ring_buffer() argument
1966 WARN_ON(ring->buffer); in intel_init_ring_buffer()
1971 ring->buffer = ringbuf; in intel_init_ring_buffer()
1973 ring->dev = dev; in intel_init_ring_buffer()
1974 INIT_LIST_HEAD(&ring->active_list); in intel_init_ring_buffer()
1975 INIT_LIST_HEAD(&ring->request_list); in intel_init_ring_buffer()
1976 INIT_LIST_HEAD(&ring->execlist_queue); in intel_init_ring_buffer()
1978 ringbuf->ring = ring; in intel_init_ring_buffer()
1979 memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno)); in intel_init_ring_buffer()
1981 init_waitqueue_head(&ring->irq_queue); in intel_init_ring_buffer()
1984 ret = init_status_page(ring); in intel_init_ring_buffer()
1988 BUG_ON(ring->id != RCS); in intel_init_ring_buffer()
1989 ret = init_phys_status_page(ring); in intel_init_ring_buffer()
1999 ring->name, ret); in intel_init_ring_buffer()
2006 ring->name, ret); in intel_init_ring_buffer()
2019 ret = i915_cmd_parser_init_ring(ring); in intel_init_ring_buffer()
2027 ring->buffer = NULL; in intel_init_ring_buffer()
2031 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) in intel_cleanup_ring_buffer() argument
2036 if (!intel_ring_initialized(ring)) in intel_cleanup_ring_buffer()
2039 dev_priv = to_i915(ring->dev); in intel_cleanup_ring_buffer()
2040 ringbuf = ring->buffer; in intel_cleanup_ring_buffer()
2042 intel_stop_ring_buffer(ring); in intel_cleanup_ring_buffer()
2043 WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); in intel_cleanup_ring_buffer()
2047 i915_gem_request_assign(&ring->outstanding_lazy_request, NULL); in intel_cleanup_ring_buffer()
2049 if (ring->cleanup) in intel_cleanup_ring_buffer()
2050 ring->cleanup(ring); in intel_cleanup_ring_buffer()
2052 cleanup_status_page(ring); in intel_cleanup_ring_buffer()
2054 i915_cmd_parser_fini_ring(ring); in intel_cleanup_ring_buffer()
2057 ring->buffer = NULL; in intel_cleanup_ring_buffer()
2060 static int intel_ring_wait_request(struct intel_engine_cs *ring, int n) in intel_ring_wait_request() argument
2062 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_ring_wait_request()
2069 list_for_each_entry(request, &ring->request_list, list) { in intel_ring_wait_request()
2076 if (&request->list == &ring->request_list) in intel_ring_wait_request()
2083 i915_gem_retire_requests_ring(ring); in intel_ring_wait_request()
2088 static int ring_wait_for_space(struct intel_engine_cs *ring, int n) in ring_wait_for_space() argument
2090 struct drm_device *dev = ring->dev; in ring_wait_for_space()
2092 struct intel_ringbuffer *ringbuf = ring->buffer; in ring_wait_for_space()
2096 ret = intel_ring_wait_request(ring, n); in ring_wait_for_space()
2101 __intel_ring_advance(ring); in ring_wait_for_space()
2111 trace_i915_ring_wait_begin(ring); in ring_wait_for_space()
2115 ringbuf->head = I915_READ_HEAD(ring); in ring_wait_for_space()
2136 trace_i915_ring_wait_end(ring); in ring_wait_for_space()
2140 static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) in intel_wrap_ring_buffer() argument
2143 struct intel_ringbuffer *ringbuf = ring->buffer; in intel_wrap_ring_buffer()
2147 int ret = ring_wait_for_space(ring, rem); in intel_wrap_ring_buffer()
2163 int intel_ring_idle(struct intel_engine_cs *ring) in intel_ring_idle() argument
2169 if (ring->outstanding_lazy_request) { in intel_ring_idle()
2170 ret = i915_add_request(ring); in intel_ring_idle()
2176 if (list_empty(&ring->request_list)) in intel_ring_idle()
2179 req = list_entry(ring->request_list.prev, in intel_ring_idle()
2187 intel_ring_alloc_request(struct intel_engine_cs *ring) in intel_ring_alloc_request() argument
2191 struct drm_i915_private *dev_private = ring->dev->dev_private; in intel_ring_alloc_request()
2193 if (ring->outstanding_lazy_request) in intel_ring_alloc_request()
2201 request->ring = ring; in intel_ring_alloc_request()
2202 request->ringbuf = ring->buffer; in intel_ring_alloc_request()
2205 ret = i915_gem_get_seqno(ring->dev, &request->seqno); in intel_ring_alloc_request()
2211 ring->outstanding_lazy_request = request; in intel_ring_alloc_request()
2215 static int __intel_ring_prepare(struct intel_engine_cs *ring, in __intel_ring_prepare() argument
2218 struct intel_ringbuffer *ringbuf = ring->buffer; in __intel_ring_prepare()
2222 ret = intel_wrap_ring_buffer(ring); in __intel_ring_prepare()
2228 ret = ring_wait_for_space(ring, bytes); in __intel_ring_prepare()
2236 int intel_ring_begin(struct intel_engine_cs *ring, in intel_ring_begin() argument
2239 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_begin()
2247 ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t)); in intel_ring_begin()
2252 ret = intel_ring_alloc_request(ring); in intel_ring_begin()
2256 ring->buffer->space -= num_dwords * sizeof(uint32_t); in intel_ring_begin()
2261 int intel_ring_cacheline_align(struct intel_engine_cs *ring) in intel_ring_cacheline_align() argument
2263 int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); in intel_ring_cacheline_align()
2270 ret = intel_ring_begin(ring, num_dwords); in intel_ring_cacheline_align()
2275 intel_ring_emit(ring, MI_NOOP); in intel_ring_cacheline_align()
2277 intel_ring_advance(ring); in intel_ring_cacheline_align()
2282 void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno) in intel_ring_init_seqno() argument
2284 struct drm_device *dev = ring->dev; in intel_ring_init_seqno()
2287 BUG_ON(ring->outstanding_lazy_request); in intel_ring_init_seqno()
2290 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); in intel_ring_init_seqno()
2291 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0); in intel_ring_init_seqno()
2293 I915_WRITE(RING_SYNC_2(ring->mmio_base), 0); in intel_ring_init_seqno()
2296 ring->set_seqno(ring, seqno); in intel_ring_init_seqno()
2297 ring->hangcheck.seqno = seqno; in intel_ring_init_seqno()
2300 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, in gen6_bsd_ring_write_tail() argument
2303 struct drm_i915_private *dev_priv = ring->dev->dev_private; in gen6_bsd_ring_write_tail()
2323 I915_WRITE_TAIL(ring, value); in gen6_bsd_ring_write_tail()
2324 POSTING_READ(RING_TAIL(ring->mmio_base)); in gen6_bsd_ring_write_tail()
2333 static int gen6_bsd_ring_flush(struct intel_engine_cs *ring, in gen6_bsd_ring_flush() argument
2339 ret = intel_ring_begin(ring, 4); in gen6_bsd_ring_flush()
2344 if (INTEL_INFO(ring->dev)->gen >= 8) in gen6_bsd_ring_flush()
2363 intel_ring_emit(ring, cmd); in gen6_bsd_ring_flush()
2364 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_bsd_ring_flush()
2365 if (INTEL_INFO(ring->dev)->gen >= 8) { in gen6_bsd_ring_flush()
2366 intel_ring_emit(ring, 0); /* upper addr */ in gen6_bsd_ring_flush()
2367 intel_ring_emit(ring, 0); /* value */ in gen6_bsd_ring_flush()
2369 intel_ring_emit(ring, 0); in gen6_bsd_ring_flush()
2370 intel_ring_emit(ring, MI_NOOP); in gen6_bsd_ring_flush()
2372 intel_ring_advance(ring); in gen6_bsd_ring_flush()
2377 gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring, in gen8_ring_dispatch_execbuffer() argument
2381 bool ppgtt = USES_PPGTT(ring->dev) && in gen8_ring_dispatch_execbuffer()
2385 ret = intel_ring_begin(ring, 4); in gen8_ring_dispatch_execbuffer()
2390 intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); in gen8_ring_dispatch_execbuffer()
2391 intel_ring_emit(ring, lower_32_bits(offset)); in gen8_ring_dispatch_execbuffer()
2392 intel_ring_emit(ring, upper_32_bits(offset)); in gen8_ring_dispatch_execbuffer()
2393 intel_ring_emit(ring, MI_NOOP); in gen8_ring_dispatch_execbuffer()
2394 intel_ring_advance(ring); in gen8_ring_dispatch_execbuffer()
2400 hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, in hsw_ring_dispatch_execbuffer() argument
2406 ret = intel_ring_begin(ring, 2); in hsw_ring_dispatch_execbuffer()
2410 intel_ring_emit(ring, in hsw_ring_dispatch_execbuffer()
2415 intel_ring_emit(ring, offset); in hsw_ring_dispatch_execbuffer()
2416 intel_ring_advance(ring); in hsw_ring_dispatch_execbuffer()
2422 gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, in gen6_ring_dispatch_execbuffer() argument
2428 ret = intel_ring_begin(ring, 2); in gen6_ring_dispatch_execbuffer()
2432 intel_ring_emit(ring, in gen6_ring_dispatch_execbuffer()
2437 intel_ring_emit(ring, offset); in gen6_ring_dispatch_execbuffer()
2438 intel_ring_advance(ring); in gen6_ring_dispatch_execbuffer()
2445 static int gen6_ring_flush(struct intel_engine_cs *ring, in gen6_ring_flush() argument
2448 struct drm_device *dev = ring->dev; in gen6_ring_flush()
2452 ret = intel_ring_begin(ring, 4); in gen6_ring_flush()
2475 intel_ring_emit(ring, cmd); in gen6_ring_flush()
2476 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_ring_flush()
2478 intel_ring_emit(ring, 0); /* upper addr */ in gen6_ring_flush()
2479 intel_ring_emit(ring, 0); /* value */ in gen6_ring_flush()
2481 intel_ring_emit(ring, 0); in gen6_ring_flush()
2482 intel_ring_emit(ring, MI_NOOP); in gen6_ring_flush()
2484 intel_ring_advance(ring); in gen6_ring_flush()
2492 struct intel_engine_cs *ring = &dev_priv->ring[RCS]; in intel_init_render_ring_buffer() local
2496 ring->name = "render ring"; in intel_init_render_ring_buffer()
2497 ring->id = RCS; in intel_init_render_ring_buffer()
2498 ring->mmio_base = RENDER_RING_BASE; in intel_init_render_ring_buffer()
2518 ring->init_context = intel_rcs_ctx_init; in intel_init_render_ring_buffer()
2519 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
2520 ring->flush = gen8_render_ring_flush; in intel_init_render_ring_buffer()
2521 ring->irq_get = gen8_ring_get_irq; in intel_init_render_ring_buffer()
2522 ring->irq_put = gen8_ring_put_irq; in intel_init_render_ring_buffer()
2523 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; in intel_init_render_ring_buffer()
2524 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
2525 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2528 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_render_ring_buffer()
2529 ring->semaphore.signal = gen8_rcs_signal; in intel_init_render_ring_buffer()
2533 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
2534 ring->flush = gen7_render_ring_flush; in intel_init_render_ring_buffer()
2536 ring->flush = gen6_render_ring_flush; in intel_init_render_ring_buffer()
2537 ring->irq_get = gen6_ring_get_irq; in intel_init_render_ring_buffer()
2538 ring->irq_put = gen6_ring_put_irq; in intel_init_render_ring_buffer()
2539 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; in intel_init_render_ring_buffer()
2540 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
2541 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2543 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_render_ring_buffer()
2544 ring->semaphore.signal = gen6_signal; in intel_init_render_ring_buffer()
2552 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
2553 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; in intel_init_render_ring_buffer()
2554 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; in intel_init_render_ring_buffer()
2555 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; in intel_init_render_ring_buffer()
2556 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
2557 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; in intel_init_render_ring_buffer()
2558 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; in intel_init_render_ring_buffer()
2559 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; in intel_init_render_ring_buffer()
2560 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; in intel_init_render_ring_buffer()
2561 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_render_ring_buffer()
2564 ring->add_request = pc_render_add_request; in intel_init_render_ring_buffer()
2565 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
2566 ring->get_seqno = pc_render_get_seqno; in intel_init_render_ring_buffer()
2567 ring->set_seqno = pc_render_set_seqno; in intel_init_render_ring_buffer()
2568 ring->irq_get = gen5_ring_get_irq; in intel_init_render_ring_buffer()
2569 ring->irq_put = gen5_ring_put_irq; in intel_init_render_ring_buffer()
2570 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT | in intel_init_render_ring_buffer()
2573 ring->add_request = i9xx_add_request; in intel_init_render_ring_buffer()
2575 ring->flush = gen2_render_ring_flush; in intel_init_render_ring_buffer()
2577 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
2578 ring->get_seqno = ring_get_seqno; in intel_init_render_ring_buffer()
2579 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2581 ring->irq_get = i8xx_ring_get_irq; in intel_init_render_ring_buffer()
2582 ring->irq_put = i8xx_ring_put_irq; in intel_init_render_ring_buffer()
2584 ring->irq_get = i9xx_ring_get_irq; in intel_init_render_ring_buffer()
2585 ring->irq_put = i9xx_ring_put_irq; in intel_init_render_ring_buffer()
2587 ring->irq_enable_mask = I915_USER_INTERRUPT; in intel_init_render_ring_buffer()
2589 ring->write_tail = ring_write_tail; in intel_init_render_ring_buffer()
2592 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2594 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2596 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2598 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_render_ring_buffer()
2600 ring->dispatch_execbuffer = i830_dispatch_execbuffer; in intel_init_render_ring_buffer()
2602 ring->dispatch_execbuffer = i915_dispatch_execbuffer; in intel_init_render_ring_buffer()
2603 ring->init_hw = init_render_ring; in intel_init_render_ring_buffer()
2604 ring->cleanup = render_ring_cleanup; in intel_init_render_ring_buffer()
2621 ring->scratch.obj = obj; in intel_init_render_ring_buffer()
2622 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); in intel_init_render_ring_buffer()
2625 ret = intel_init_ring_buffer(dev, ring); in intel_init_render_ring_buffer()
2630 ret = intel_init_pipe_control(ring); in intel_init_render_ring_buffer()
2641 struct intel_engine_cs *ring = &dev_priv->ring[VCS]; in intel_init_bsd_ring_buffer() local
2643 ring->name = "bsd ring"; in intel_init_bsd_ring_buffer()
2644 ring->id = VCS; in intel_init_bsd_ring_buffer()
2646 ring->write_tail = ring_write_tail; in intel_init_bsd_ring_buffer()
2648 ring->mmio_base = GEN6_BSD_RING_BASE; in intel_init_bsd_ring_buffer()
2651 ring->write_tail = gen6_bsd_ring_write_tail; in intel_init_bsd_ring_buffer()
2652 ring->flush = gen6_bsd_ring_flush; in intel_init_bsd_ring_buffer()
2653 ring->add_request = gen6_add_request; in intel_init_bsd_ring_buffer()
2654 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd_ring_buffer()
2655 ring->set_seqno = ring_set_seqno; in intel_init_bsd_ring_buffer()
2657 ring->irq_enable_mask = in intel_init_bsd_ring_buffer()
2659 ring->irq_get = gen8_ring_get_irq; in intel_init_bsd_ring_buffer()
2660 ring->irq_put = gen8_ring_put_irq; in intel_init_bsd_ring_buffer()
2661 ring->dispatch_execbuffer = in intel_init_bsd_ring_buffer()
2664 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_bsd_ring_buffer()
2665 ring->semaphore.signal = gen8_xcs_signal; in intel_init_bsd_ring_buffer()
2669 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2670 ring->irq_get = gen6_ring_get_irq; in intel_init_bsd_ring_buffer()
2671 ring->irq_put = gen6_ring_put_irq; in intel_init_bsd_ring_buffer()
2672 ring->dispatch_execbuffer = in intel_init_bsd_ring_buffer()
2675 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_bsd_ring_buffer()
2676 ring->semaphore.signal = gen6_signal; in intel_init_bsd_ring_buffer()
2677 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; in intel_init_bsd_ring_buffer()
2678 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
2679 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; in intel_init_bsd_ring_buffer()
2680 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; in intel_init_bsd_ring_buffer()
2681 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
2682 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; in intel_init_bsd_ring_buffer()
2683 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; in intel_init_bsd_ring_buffer()
2684 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; in intel_init_bsd_ring_buffer()
2685 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; in intel_init_bsd_ring_buffer()
2686 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_bsd_ring_buffer()
2690 ring->mmio_base = BSD_RING_BASE; in intel_init_bsd_ring_buffer()
2691 ring->flush = bsd_ring_flush; in intel_init_bsd_ring_buffer()
2692 ring->add_request = i9xx_add_request; in intel_init_bsd_ring_buffer()
2693 ring->get_seqno = ring_get_seqno; in intel_init_bsd_ring_buffer()
2694 ring->set_seqno = ring_set_seqno; in intel_init_bsd_ring_buffer()
2696 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2697 ring->irq_get = gen5_ring_get_irq; in intel_init_bsd_ring_buffer()
2698 ring->irq_put = gen5_ring_put_irq; in intel_init_bsd_ring_buffer()
2700 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2701 ring->irq_get = i9xx_ring_get_irq; in intel_init_bsd_ring_buffer()
2702 ring->irq_put = i9xx_ring_put_irq; in intel_init_bsd_ring_buffer()
2704 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_bsd_ring_buffer()
2706 ring->init_hw = init_ring_common; in intel_init_bsd_ring_buffer()
2708 return intel_init_ring_buffer(dev, ring); in intel_init_bsd_ring_buffer()
2717 struct intel_engine_cs *ring = &dev_priv->ring[VCS2]; in intel_init_bsd2_ring_buffer() local
2719 ring->name = "bsd2 ring"; in intel_init_bsd2_ring_buffer()
2720 ring->id = VCS2; in intel_init_bsd2_ring_buffer()
2722 ring->write_tail = ring_write_tail; in intel_init_bsd2_ring_buffer()
2723 ring->mmio_base = GEN8_BSD2_RING_BASE; in intel_init_bsd2_ring_buffer()
2724 ring->flush = gen6_bsd_ring_flush; in intel_init_bsd2_ring_buffer()
2725 ring->add_request = gen6_add_request; in intel_init_bsd2_ring_buffer()
2726 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd2_ring_buffer()
2727 ring->set_seqno = ring_set_seqno; in intel_init_bsd2_ring_buffer()
2728 ring->irq_enable_mask = in intel_init_bsd2_ring_buffer()
2730 ring->irq_get = gen8_ring_get_irq; in intel_init_bsd2_ring_buffer()
2731 ring->irq_put = gen8_ring_put_irq; in intel_init_bsd2_ring_buffer()
2732 ring->dispatch_execbuffer = in intel_init_bsd2_ring_buffer()
2735 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_bsd2_ring_buffer()
2736 ring->semaphore.signal = gen8_xcs_signal; in intel_init_bsd2_ring_buffer()
2739 ring->init_hw = init_ring_common; in intel_init_bsd2_ring_buffer()
2741 return intel_init_ring_buffer(dev, ring); in intel_init_bsd2_ring_buffer()
2747 struct intel_engine_cs *ring = &dev_priv->ring[BCS]; in intel_init_blt_ring_buffer() local
2749 ring->name = "blitter ring"; in intel_init_blt_ring_buffer()
2750 ring->id = BCS; in intel_init_blt_ring_buffer()
2752 ring->mmio_base = BLT_RING_BASE; in intel_init_blt_ring_buffer()
2753 ring->write_tail = ring_write_tail; in intel_init_blt_ring_buffer()
2754 ring->flush = gen6_ring_flush; in intel_init_blt_ring_buffer()
2755 ring->add_request = gen6_add_request; in intel_init_blt_ring_buffer()
2756 ring->get_seqno = gen6_ring_get_seqno; in intel_init_blt_ring_buffer()
2757 ring->set_seqno = ring_set_seqno; in intel_init_blt_ring_buffer()
2759 ring->irq_enable_mask = in intel_init_blt_ring_buffer()
2761 ring->irq_get = gen8_ring_get_irq; in intel_init_blt_ring_buffer()
2762 ring->irq_put = gen8_ring_put_irq; in intel_init_blt_ring_buffer()
2763 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
2765 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_blt_ring_buffer()
2766 ring->semaphore.signal = gen8_xcs_signal; in intel_init_blt_ring_buffer()
2770 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; in intel_init_blt_ring_buffer()
2771 ring->irq_get = gen6_ring_get_irq; in intel_init_blt_ring_buffer()
2772 ring->irq_put = gen6_ring_put_irq; in intel_init_blt_ring_buffer()
2773 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
2775 ring->semaphore.signal = gen6_signal; in intel_init_blt_ring_buffer()
2776 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_blt_ring_buffer()
2784 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; in intel_init_blt_ring_buffer()
2785 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; in intel_init_blt_ring_buffer()
2786 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
2787 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; in intel_init_blt_ring_buffer()
2788 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
2789 ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; in intel_init_blt_ring_buffer()
2790 ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; in intel_init_blt_ring_buffer()
2791 ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; in intel_init_blt_ring_buffer()
2792 ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; in intel_init_blt_ring_buffer()
2793 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_blt_ring_buffer()
2796 ring->init_hw = init_ring_common; in intel_init_blt_ring_buffer()
2798 return intel_init_ring_buffer(dev, ring); in intel_init_blt_ring_buffer()
2804 struct intel_engine_cs *ring = &dev_priv->ring[VECS]; in intel_init_vebox_ring_buffer() local
2806 ring->name = "video enhancement ring"; in intel_init_vebox_ring_buffer()
2807 ring->id = VECS; in intel_init_vebox_ring_buffer()
2809 ring->mmio_base = VEBOX_RING_BASE; in intel_init_vebox_ring_buffer()
2810 ring->write_tail = ring_write_tail; in intel_init_vebox_ring_buffer()
2811 ring->flush = gen6_ring_flush; in intel_init_vebox_ring_buffer()
2812 ring->add_request = gen6_add_request; in intel_init_vebox_ring_buffer()
2813 ring->get_seqno = gen6_ring_get_seqno; in intel_init_vebox_ring_buffer()
2814 ring->set_seqno = ring_set_seqno; in intel_init_vebox_ring_buffer()
2817 ring->irq_enable_mask = in intel_init_vebox_ring_buffer()
2819 ring->irq_get = gen8_ring_get_irq; in intel_init_vebox_ring_buffer()
2820 ring->irq_put = gen8_ring_put_irq; in intel_init_vebox_ring_buffer()
2821 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_vebox_ring_buffer()
2823 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_vebox_ring_buffer()
2824 ring->semaphore.signal = gen8_xcs_signal; in intel_init_vebox_ring_buffer()
2828 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; in intel_init_vebox_ring_buffer()
2829 ring->irq_get = hsw_vebox_get_irq; in intel_init_vebox_ring_buffer()
2830 ring->irq_put = hsw_vebox_put_irq; in intel_init_vebox_ring_buffer()
2831 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_vebox_ring_buffer()
2833 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_vebox_ring_buffer()
2834 ring->semaphore.signal = gen6_signal; in intel_init_vebox_ring_buffer()
2835 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; in intel_init_vebox_ring_buffer()
2836 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; in intel_init_vebox_ring_buffer()
2837 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; in intel_init_vebox_ring_buffer()
2838 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_vebox_ring_buffer()
2839 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_vebox_ring_buffer()
2840 ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; in intel_init_vebox_ring_buffer()
2841 ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; in intel_init_vebox_ring_buffer()
2842 ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; in intel_init_vebox_ring_buffer()
2843 ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; in intel_init_vebox_ring_buffer()
2844 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_vebox_ring_buffer()
2847 ring->init_hw = init_ring_common; in intel_init_vebox_ring_buffer()
2849 return intel_init_ring_buffer(dev, ring); in intel_init_vebox_ring_buffer()
2853 intel_ring_flush_all_caches(struct intel_engine_cs *ring) in intel_ring_flush_all_caches() argument
2857 if (!ring->gpu_caches_dirty) in intel_ring_flush_all_caches()
2860 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS); in intel_ring_flush_all_caches()
2864 trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS); in intel_ring_flush_all_caches()
2866 ring->gpu_caches_dirty = false; in intel_ring_flush_all_caches()
2871 intel_ring_invalidate_all_caches(struct intel_engine_cs *ring) in intel_ring_invalidate_all_caches() argument
2877 if (ring->gpu_caches_dirty) in intel_ring_invalidate_all_caches()
2880 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); in intel_ring_invalidate_all_caches()
2884 trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); in intel_ring_invalidate_all_caches()
2886 ring->gpu_caches_dirty = false; in intel_ring_invalidate_all_caches()
2891 intel_stop_ring_buffer(struct intel_engine_cs *ring) in intel_stop_ring_buffer() argument
2895 if (!intel_ring_initialized(ring)) in intel_stop_ring_buffer()
2898 ret = intel_ring_idle(ring); in intel_stop_ring_buffer()
2899 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error)) in intel_stop_ring_buffer()
2901 ring->name, ret); in intel_stop_ring_buffer()
2903 stop_ring(ring); in intel_stop_ring_buffer()