Lines Matching refs:ring
37 intel_ring_initialized(struct intel_engine_cs *ring) in intel_ring_initialized() argument
39 struct drm_device *dev = ring->dev; in intel_ring_initialized()
45 struct intel_context *dctx = ring->default_context; in intel_ring_initialized()
46 struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf; in intel_ring_initialized()
50 return ring->buffer && ring->buffer->obj; in intel_ring_initialized()
78 bool intel_ring_stopped(struct intel_engine_cs *ring) in intel_ring_stopped() argument
80 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_stopped()
81 return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring); in intel_ring_stopped()
84 static void __intel_ring_advance(struct intel_engine_cs *ring) in __intel_ring_advance() argument
86 struct intel_ringbuffer *ringbuf = ring->buffer; in __intel_ring_advance()
88 if (intel_ring_stopped(ring)) in __intel_ring_advance()
90 ring->write_tail(ring, ringbuf->tail); in __intel_ring_advance()
98 struct intel_engine_cs *ring = req->ring; in gen2_render_ring_flush() local
113 intel_ring_emit(ring, cmd); in gen2_render_ring_flush()
114 intel_ring_emit(ring, MI_NOOP); in gen2_render_ring_flush()
115 intel_ring_advance(ring); in gen2_render_ring_flush()
125 struct intel_engine_cs *ring = req->ring; in gen4_render_ring_flush() local
126 struct drm_device *dev = ring->dev; in gen4_render_ring_flush()
172 intel_ring_emit(ring, cmd); in gen4_render_ring_flush()
173 intel_ring_emit(ring, MI_NOOP); in gen4_render_ring_flush()
174 intel_ring_advance(ring); in gen4_render_ring_flush()
219 struct intel_engine_cs *ring = req->ring; in intel_emit_post_sync_nonzero_flush() local
220 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in intel_emit_post_sync_nonzero_flush()
227 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
228 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in intel_emit_post_sync_nonzero_flush()
230 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
231 intel_ring_emit(ring, 0); /* low dword */ in intel_emit_post_sync_nonzero_flush()
232 intel_ring_emit(ring, 0); /* high dword */ in intel_emit_post_sync_nonzero_flush()
233 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
234 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
240 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5)); in intel_emit_post_sync_nonzero_flush()
241 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE); in intel_emit_post_sync_nonzero_flush()
242 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */ in intel_emit_post_sync_nonzero_flush()
243 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
244 intel_ring_emit(ring, 0); in intel_emit_post_sync_nonzero_flush()
245 intel_ring_emit(ring, MI_NOOP); in intel_emit_post_sync_nonzero_flush()
246 intel_ring_advance(ring); in intel_emit_post_sync_nonzero_flush()
255 struct intel_engine_cs *ring = req->ring; in gen6_render_ring_flush() local
257 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen6_render_ring_flush()
295 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen6_render_ring_flush()
296 intel_ring_emit(ring, flags); in gen6_render_ring_flush()
297 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); in gen6_render_ring_flush()
298 intel_ring_emit(ring, 0); in gen6_render_ring_flush()
299 intel_ring_advance(ring); in gen6_render_ring_flush()
307 struct intel_engine_cs *ring = req->ring; in gen7_render_ring_cs_stall_wa() local
314 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_cs_stall_wa()
315 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL | in gen7_render_ring_cs_stall_wa()
317 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
318 intel_ring_emit(ring, 0); in gen7_render_ring_cs_stall_wa()
319 intel_ring_advance(ring); in gen7_render_ring_cs_stall_wa()
328 struct intel_engine_cs *ring = req->ring; in gen7_render_ring_flush() local
330 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen7_render_ring_flush()
379 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4)); in gen7_render_ring_flush()
380 intel_ring_emit(ring, flags); in gen7_render_ring_flush()
381 intel_ring_emit(ring, scratch_addr); in gen7_render_ring_flush()
382 intel_ring_emit(ring, 0); in gen7_render_ring_flush()
383 intel_ring_advance(ring); in gen7_render_ring_flush()
392 struct intel_engine_cs *ring = req->ring; in gen8_emit_pipe_control() local
399 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); in gen8_emit_pipe_control()
400 intel_ring_emit(ring, flags); in gen8_emit_pipe_control()
401 intel_ring_emit(ring, scratch_addr); in gen8_emit_pipe_control()
402 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
403 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
404 intel_ring_emit(ring, 0); in gen8_emit_pipe_control()
405 intel_ring_advance(ring); in gen8_emit_pipe_control()
415 u32 scratch_addr = req->ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in gen8_render_ring_flush()
448 static void ring_write_tail(struct intel_engine_cs *ring, in ring_write_tail() argument
451 struct drm_i915_private *dev_priv = ring->dev->dev_private; in ring_write_tail()
452 I915_WRITE_TAIL(ring, value); in ring_write_tail()
455 u64 intel_ring_get_active_head(struct intel_engine_cs *ring) in intel_ring_get_active_head() argument
457 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_get_active_head()
460 if (INTEL_INFO(ring->dev)->gen >= 8) in intel_ring_get_active_head()
461 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base), in intel_ring_get_active_head()
462 RING_ACTHD_UDW(ring->mmio_base)); in intel_ring_get_active_head()
463 else if (INTEL_INFO(ring->dev)->gen >= 4) in intel_ring_get_active_head()
464 acthd = I915_READ(RING_ACTHD(ring->mmio_base)); in intel_ring_get_active_head()
471 static void ring_setup_phys_status_page(struct intel_engine_cs *ring) in ring_setup_phys_status_page() argument
473 struct drm_i915_private *dev_priv = ring->dev->dev_private; in ring_setup_phys_status_page()
477 if (INTEL_INFO(ring->dev)->gen >= 4) in ring_setup_phys_status_page()
482 static void intel_ring_setup_status_page(struct intel_engine_cs *ring) in intel_ring_setup_status_page() argument
484 struct drm_device *dev = ring->dev; in intel_ring_setup_status_page()
485 struct drm_i915_private *dev_priv = ring->dev->dev_private; in intel_ring_setup_status_page()
492 switch (ring->id) { in intel_ring_setup_status_page()
511 } else if (IS_GEN6(ring->dev)) { in intel_ring_setup_status_page()
512 mmio = RING_HWS_PGA_GEN6(ring->mmio_base); in intel_ring_setup_status_page()
515 mmio = RING_HWS_PGA(ring->mmio_base); in intel_ring_setup_status_page()
518 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); in intel_ring_setup_status_page()
529 u32 reg = RING_INSTPM(ring->mmio_base); in intel_ring_setup_status_page()
532 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); in intel_ring_setup_status_page()
540 ring->name); in intel_ring_setup_status_page()
544 static bool stop_ring(struct intel_engine_cs *ring) in stop_ring() argument
546 struct drm_i915_private *dev_priv = to_i915(ring->dev); in stop_ring()
548 if (!IS_GEN2(ring->dev)) { in stop_ring()
549 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING)); in stop_ring()
550 if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { in stop_ring()
551 DRM_ERROR("%s : timed out trying to stop ring\n", ring->name); in stop_ring()
556 if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring)) in stop_ring()
561 I915_WRITE_CTL(ring, 0); in stop_ring()
562 I915_WRITE_HEAD(ring, 0); in stop_ring()
563 ring->write_tail(ring, 0); in stop_ring()
565 if (!IS_GEN2(ring->dev)) { in stop_ring()
566 (void)I915_READ_CTL(ring); in stop_ring()
567 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); in stop_ring()
570 return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0; in stop_ring()
573 static int init_ring_common(struct intel_engine_cs *ring) in init_ring_common() argument
575 struct drm_device *dev = ring->dev; in init_ring_common()
577 struct intel_ringbuffer *ringbuf = ring->buffer; in init_ring_common()
583 if (!stop_ring(ring)) { in init_ring_common()
587 ring->name, in init_ring_common()
588 I915_READ_CTL(ring), in init_ring_common()
589 I915_READ_HEAD(ring), in init_ring_common()
590 I915_READ_TAIL(ring), in init_ring_common()
591 I915_READ_START(ring)); in init_ring_common()
593 if (!stop_ring(ring)) { in init_ring_common()
596 ring->name, in init_ring_common()
597 I915_READ_CTL(ring), in init_ring_common()
598 I915_READ_HEAD(ring), in init_ring_common()
599 I915_READ_TAIL(ring), in init_ring_common()
600 I915_READ_START(ring)); in init_ring_common()
607 intel_ring_setup_status_page(ring); in init_ring_common()
609 ring_setup_phys_status_page(ring); in init_ring_common()
612 I915_READ_HEAD(ring); in init_ring_common()
618 I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj)); in init_ring_common()
621 if (I915_READ_HEAD(ring)) in init_ring_common()
623 ring->name, I915_READ_HEAD(ring)); in init_ring_common()
624 I915_WRITE_HEAD(ring, 0); in init_ring_common()
625 (void)I915_READ_HEAD(ring); in init_ring_common()
627 I915_WRITE_CTL(ring, in init_ring_common()
632 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 && in init_ring_common()
633 I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) && in init_ring_common()
634 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { in init_ring_common()
637 ring->name, in init_ring_common()
638 I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID, in init_ring_common()
639 I915_READ_HEAD(ring), I915_READ_TAIL(ring), in init_ring_common()
640 I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj)); in init_ring_common()
646 ringbuf->head = I915_READ_HEAD(ring); in init_ring_common()
647 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; in init_ring_common()
650 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); in init_ring_common()
659 intel_fini_pipe_control(struct intel_engine_cs *ring) in intel_fini_pipe_control() argument
661 struct drm_device *dev = ring->dev; in intel_fini_pipe_control()
663 if (ring->scratch.obj == NULL) in intel_fini_pipe_control()
667 kunmap(sg_page(ring->scratch.obj->pages->sgl)); in intel_fini_pipe_control()
668 i915_gem_object_ggtt_unpin(ring->scratch.obj); in intel_fini_pipe_control()
671 drm_gem_object_unreference(&ring->scratch.obj->base); in intel_fini_pipe_control()
672 ring->scratch.obj = NULL; in intel_fini_pipe_control()
676 intel_init_pipe_control(struct intel_engine_cs *ring) in intel_init_pipe_control() argument
680 WARN_ON(ring->scratch.obj); in intel_init_pipe_control()
682 ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096); in intel_init_pipe_control()
683 if (ring->scratch.obj == NULL) { in intel_init_pipe_control()
689 ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); in intel_init_pipe_control()
693 ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0); in intel_init_pipe_control()
697 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj); in intel_init_pipe_control()
698 ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl)); in intel_init_pipe_control()
699 if (ring->scratch.cpu_page == NULL) { in intel_init_pipe_control()
705 ring->name, ring->scratch.gtt_offset); in intel_init_pipe_control()
709 i915_gem_object_ggtt_unpin(ring->scratch.obj); in intel_init_pipe_control()
711 drm_gem_object_unreference(&ring->scratch.obj->base); in intel_init_pipe_control()
719 struct intel_engine_cs *ring = req->ring; in intel_ring_workarounds_emit() local
720 struct drm_device *dev = ring->dev; in intel_ring_workarounds_emit()
727 ring->gpu_caches_dirty = true; in intel_ring_workarounds_emit()
736 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count)); in intel_ring_workarounds_emit()
738 intel_ring_emit(ring, w->reg[i].addr); in intel_ring_workarounds_emit()
739 intel_ring_emit(ring, w->reg[i].value); in intel_ring_workarounds_emit()
741 intel_ring_emit(ring, MI_NOOP); in intel_ring_workarounds_emit()
743 intel_ring_advance(ring); in intel_ring_workarounds_emit()
745 ring->gpu_caches_dirty = true; in intel_ring_workarounds_emit()
807 static int gen8_init_workarounds(struct intel_engine_cs *ring) in gen8_init_workarounds() argument
809 struct drm_device *dev = ring->dev; in gen8_init_workarounds()
859 static int bdw_init_workarounds(struct intel_engine_cs *ring) in bdw_init_workarounds() argument
862 struct drm_device *dev = ring->dev; in bdw_init_workarounds()
865 ret = gen8_init_workarounds(ring); in bdw_init_workarounds()
888 static int chv_init_workarounds(struct intel_engine_cs *ring) in chv_init_workarounds() argument
891 struct drm_device *dev = ring->dev; in chv_init_workarounds()
894 ret = gen8_init_workarounds(ring); in chv_init_workarounds()
907 static int gen9_init_workarounds(struct intel_engine_cs *ring) in gen9_init_workarounds() argument
909 struct drm_device *dev = ring->dev; in gen9_init_workarounds()
991 static int skl_tune_iz_hashing(struct intel_engine_cs *ring) in skl_tune_iz_hashing() argument
993 struct drm_device *dev = ring->dev; in skl_tune_iz_hashing()
1033 static int skl_init_workarounds(struct intel_engine_cs *ring) in skl_init_workarounds() argument
1036 struct drm_device *dev = ring->dev; in skl_init_workarounds()
1039 ret = gen9_init_workarounds(ring); in skl_init_workarounds()
1097 return skl_tune_iz_hashing(ring); in skl_init_workarounds()
1100 static int bxt_init_workarounds(struct intel_engine_cs *ring) in bxt_init_workarounds() argument
1103 struct drm_device *dev = ring->dev; in bxt_init_workarounds()
1106 ret = gen9_init_workarounds(ring); in bxt_init_workarounds()
1135 int init_workarounds_ring(struct intel_engine_cs *ring) in init_workarounds_ring() argument
1137 struct drm_device *dev = ring->dev; in init_workarounds_ring()
1140 WARN_ON(ring->id != RCS); in init_workarounds_ring()
1145 return bdw_init_workarounds(ring); in init_workarounds_ring()
1148 return chv_init_workarounds(ring); in init_workarounds_ring()
1151 return skl_init_workarounds(ring); in init_workarounds_ring()
1154 return bxt_init_workarounds(ring); in init_workarounds_ring()
1159 static int init_render_ring(struct intel_engine_cs *ring) in init_render_ring() argument
1161 struct drm_device *dev = ring->dev; in init_render_ring()
1163 int ret = init_ring_common(ring); in init_render_ring()
1206 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); in init_render_ring()
1208 return init_workarounds_ring(ring); in init_render_ring()
1211 static void render_ring_cleanup(struct intel_engine_cs *ring) in render_ring_cleanup() argument
1213 struct drm_device *dev = ring->dev; in render_ring_cleanup()
1222 intel_fini_pipe_control(ring); in render_ring_cleanup()
1229 struct intel_engine_cs *signaller = signaller_req->ring; in gen8_rcs_signal()
1270 struct intel_engine_cs *signaller = signaller_req->ring; in gen8_xcs_signal()
1308 struct intel_engine_cs *signaller = signaller_req->ring; in gen6_signal()
1351 struct intel_engine_cs *ring = req->ring; in gen6_add_request() local
1354 if (ring->semaphore.signal) in gen6_add_request()
1355 ret = ring->semaphore.signal(req, 4); in gen6_add_request()
1362 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in gen6_add_request()
1363 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in gen6_add_request()
1364 intel_ring_emit(ring, i915_gem_request_get_seqno(req)); in gen6_add_request()
1365 intel_ring_emit(ring, MI_USER_INTERRUPT); in gen6_add_request()
1366 __intel_ring_advance(ring); in gen6_add_request()
1391 struct intel_engine_cs *waiter = waiter_req->ring; in gen8_ring_sync()
1417 struct intel_engine_cs *waiter = waiter_req->ring; in gen6_ring_sync()
1465 struct intel_engine_cs *ring = req->ring; in pc_render_add_request() local
1466 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; in pc_render_add_request()
1481 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
1484 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
1485 intel_ring_emit(ring, i915_gem_request_get_seqno(req)); in pc_render_add_request()
1486 intel_ring_emit(ring, 0); in pc_render_add_request()
1487 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1489 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1491 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1493 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1495 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1497 PIPE_CONTROL_FLUSH(ring, scratch_addr); in pc_render_add_request()
1499 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | in pc_render_add_request()
1503 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); in pc_render_add_request()
1504 intel_ring_emit(ring, i915_gem_request_get_seqno(req)); in pc_render_add_request()
1505 intel_ring_emit(ring, 0); in pc_render_add_request()
1506 __intel_ring_advance(ring); in pc_render_add_request()
1512 gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in gen6_ring_get_seqno() argument
1518 struct drm_i915_private *dev_priv = ring->dev->dev_private; in gen6_ring_get_seqno()
1519 POSTING_READ(RING_ACTHD(ring->mmio_base)); in gen6_ring_get_seqno()
1522 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in gen6_ring_get_seqno()
1526 ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in ring_get_seqno() argument
1528 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); in ring_get_seqno()
1532 ring_set_seqno(struct intel_engine_cs *ring, u32 seqno) in ring_set_seqno() argument
1534 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); in ring_set_seqno()
1538 pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) in pc_render_get_seqno() argument
1540 return ring->scratch.cpu_page[0]; in pc_render_get_seqno()
1544 pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno) in pc_render_set_seqno() argument
1546 ring->scratch.cpu_page[0] = seqno; in pc_render_set_seqno()
1550 gen5_ring_get_irq(struct intel_engine_cs *ring) in gen5_ring_get_irq() argument
1552 struct drm_device *dev = ring->dev; in gen5_ring_get_irq()
1560 if (ring->irq_refcount++ == 0) in gen5_ring_get_irq()
1561 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); in gen5_ring_get_irq()
1568 gen5_ring_put_irq(struct intel_engine_cs *ring) in gen5_ring_put_irq() argument
1570 struct drm_device *dev = ring->dev; in gen5_ring_put_irq()
1575 if (--ring->irq_refcount == 0) in gen5_ring_put_irq()
1576 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); in gen5_ring_put_irq()
1581 i9xx_ring_get_irq(struct intel_engine_cs *ring) in i9xx_ring_get_irq() argument
1583 struct drm_device *dev = ring->dev; in i9xx_ring_get_irq()
1591 if (ring->irq_refcount++ == 0) { in i9xx_ring_get_irq()
1592 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i9xx_ring_get_irq()
1602 i9xx_ring_put_irq(struct intel_engine_cs *ring) in i9xx_ring_put_irq() argument
1604 struct drm_device *dev = ring->dev; in i9xx_ring_put_irq()
1609 if (--ring->irq_refcount == 0) { in i9xx_ring_put_irq()
1610 dev_priv->irq_mask |= ring->irq_enable_mask; in i9xx_ring_put_irq()
1618 i8xx_ring_get_irq(struct intel_engine_cs *ring) in i8xx_ring_get_irq() argument
1620 struct drm_device *dev = ring->dev; in i8xx_ring_get_irq()
1628 if (ring->irq_refcount++ == 0) { in i8xx_ring_get_irq()
1629 dev_priv->irq_mask &= ~ring->irq_enable_mask; in i8xx_ring_get_irq()
1639 i8xx_ring_put_irq(struct intel_engine_cs *ring) in i8xx_ring_put_irq() argument
1641 struct drm_device *dev = ring->dev; in i8xx_ring_put_irq()
1646 if (--ring->irq_refcount == 0) { in i8xx_ring_put_irq()
1647 dev_priv->irq_mask |= ring->irq_enable_mask; in i8xx_ring_put_irq()
1659 struct intel_engine_cs *ring = req->ring; in bsd_ring_flush() local
1666 intel_ring_emit(ring, MI_FLUSH); in bsd_ring_flush()
1667 intel_ring_emit(ring, MI_NOOP); in bsd_ring_flush()
1668 intel_ring_advance(ring); in bsd_ring_flush()
1675 struct intel_engine_cs *ring = req->ring; in i9xx_add_request() local
1682 intel_ring_emit(ring, MI_STORE_DWORD_INDEX); in i9xx_add_request()
1683 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); in i9xx_add_request()
1684 intel_ring_emit(ring, i915_gem_request_get_seqno(req)); in i9xx_add_request()
1685 intel_ring_emit(ring, MI_USER_INTERRUPT); in i9xx_add_request()
1686 __intel_ring_advance(ring); in i9xx_add_request()
1692 gen6_ring_get_irq(struct intel_engine_cs *ring) in gen6_ring_get_irq() argument
1694 struct drm_device *dev = ring->dev; in gen6_ring_get_irq()
1702 if (ring->irq_refcount++ == 0) { in gen6_ring_get_irq()
1703 if (HAS_L3_DPF(dev) && ring->id == RCS) in gen6_ring_get_irq()
1704 I915_WRITE_IMR(ring, in gen6_ring_get_irq()
1705 ~(ring->irq_enable_mask | in gen6_ring_get_irq()
1708 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen6_ring_get_irq()
1709 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask); in gen6_ring_get_irq()
1717 gen6_ring_put_irq(struct intel_engine_cs *ring) in gen6_ring_put_irq() argument
1719 struct drm_device *dev = ring->dev; in gen6_ring_put_irq()
1724 if (--ring->irq_refcount == 0) { in gen6_ring_put_irq()
1725 if (HAS_L3_DPF(dev) && ring->id == RCS) in gen6_ring_put_irq()
1726 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev)); in gen6_ring_put_irq()
1728 I915_WRITE_IMR(ring, ~0); in gen6_ring_put_irq()
1729 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask); in gen6_ring_put_irq()
1735 hsw_vebox_get_irq(struct intel_engine_cs *ring) in hsw_vebox_get_irq() argument
1737 struct drm_device *dev = ring->dev; in hsw_vebox_get_irq()
1745 if (ring->irq_refcount++ == 0) { in hsw_vebox_get_irq()
1746 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in hsw_vebox_get_irq()
1747 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask); in hsw_vebox_get_irq()
1755 hsw_vebox_put_irq(struct intel_engine_cs *ring) in hsw_vebox_put_irq() argument
1757 struct drm_device *dev = ring->dev; in hsw_vebox_put_irq()
1762 if (--ring->irq_refcount == 0) { in hsw_vebox_put_irq()
1763 I915_WRITE_IMR(ring, ~0); in hsw_vebox_put_irq()
1764 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask); in hsw_vebox_put_irq()
1770 gen8_ring_get_irq(struct intel_engine_cs *ring) in gen8_ring_get_irq() argument
1772 struct drm_device *dev = ring->dev; in gen8_ring_get_irq()
1780 if (ring->irq_refcount++ == 0) { in gen8_ring_get_irq()
1781 if (HAS_L3_DPF(dev) && ring->id == RCS) { in gen8_ring_get_irq()
1782 I915_WRITE_IMR(ring, in gen8_ring_get_irq()
1783 ~(ring->irq_enable_mask | in gen8_ring_get_irq()
1786 I915_WRITE_IMR(ring, ~ring->irq_enable_mask); in gen8_ring_get_irq()
1788 POSTING_READ(RING_IMR(ring->mmio_base)); in gen8_ring_get_irq()
1796 gen8_ring_put_irq(struct intel_engine_cs *ring) in gen8_ring_put_irq() argument
1798 struct drm_device *dev = ring->dev; in gen8_ring_put_irq()
1803 if (--ring->irq_refcount == 0) { in gen8_ring_put_irq()
1804 if (HAS_L3_DPF(dev) && ring->id == RCS) { in gen8_ring_put_irq()
1805 I915_WRITE_IMR(ring, in gen8_ring_put_irq()
1808 I915_WRITE_IMR(ring, ~0); in gen8_ring_put_irq()
1810 POSTING_READ(RING_IMR(ring->mmio_base)); in gen8_ring_put_irq()
1820 struct intel_engine_cs *ring = req->ring; in i965_dispatch_execbuffer() local
1827 intel_ring_emit(ring, in i965_dispatch_execbuffer()
1832 intel_ring_emit(ring, offset); in i965_dispatch_execbuffer()
1833 intel_ring_advance(ring); in i965_dispatch_execbuffer()
1847 struct intel_engine_cs *ring = req->ring; in i830_dispatch_execbuffer() local
1848 u32 cs_offset = ring->scratch.gtt_offset; in i830_dispatch_execbuffer()
1856 intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA); in i830_dispatch_execbuffer()
1857 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096); in i830_dispatch_execbuffer()
1858 intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */ in i830_dispatch_execbuffer()
1859 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1860 intel_ring_emit(ring, 0xdeadbeef); in i830_dispatch_execbuffer()
1861 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1862 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1876 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA); in i830_dispatch_execbuffer()
1877 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096); in i830_dispatch_execbuffer()
1878 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096); in i830_dispatch_execbuffer()
1879 intel_ring_emit(ring, cs_offset); in i830_dispatch_execbuffer()
1880 intel_ring_emit(ring, 4096); in i830_dispatch_execbuffer()
1881 intel_ring_emit(ring, offset); in i830_dispatch_execbuffer()
1883 intel_ring_emit(ring, MI_FLUSH); in i830_dispatch_execbuffer()
1884 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1885 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1895 intel_ring_emit(ring, MI_BATCH_BUFFER); in i830_dispatch_execbuffer()
1896 intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ? in i830_dispatch_execbuffer()
1898 intel_ring_emit(ring, offset + len - 8); in i830_dispatch_execbuffer()
1899 intel_ring_emit(ring, MI_NOOP); in i830_dispatch_execbuffer()
1900 intel_ring_advance(ring); in i830_dispatch_execbuffer()
1910 struct intel_engine_cs *ring = req->ring; in i915_dispatch_execbuffer() local
1917 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT); in i915_dispatch_execbuffer()
1918 intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ? in i915_dispatch_execbuffer()
1920 intel_ring_advance(ring); in i915_dispatch_execbuffer()
1925 static void cleanup_phys_status_page(struct intel_engine_cs *ring) in cleanup_phys_status_page() argument
1927 struct drm_i915_private *dev_priv = to_i915(ring->dev); in cleanup_phys_status_page()
1932 drm_pci_free(ring->dev, dev_priv->status_page_dmah); in cleanup_phys_status_page()
1933 ring->status_page.page_addr = NULL; in cleanup_phys_status_page()
1936 static void cleanup_status_page(struct intel_engine_cs *ring) in cleanup_status_page() argument
1940 obj = ring->status_page.obj; in cleanup_status_page()
1947 ring->status_page.obj = NULL; in cleanup_status_page()
1950 static int init_status_page(struct intel_engine_cs *ring) in init_status_page() argument
1952 struct drm_i915_gem_object *obj = ring->status_page.obj; in init_status_page()
1958 obj = i915_gem_alloc_object(ring->dev, 4096); in init_status_page()
1969 if (!HAS_LLC(ring->dev)) in init_status_page()
1988 ring->status_page.obj = obj; in init_status_page()
1991 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); in init_status_page()
1992 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl)); in init_status_page()
1993 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_status_page()
1996 ring->name, ring->status_page.gfx_addr); in init_status_page()
2001 static int init_phys_status_page(struct intel_engine_cs *ring) in init_phys_status_page() argument
2003 struct drm_i915_private *dev_priv = ring->dev->dev_private; in init_phys_status_page()
2007 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE); in init_phys_status_page()
2012 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; in init_phys_status_page()
2013 memset(ring->status_page.page_addr, 0, PAGE_SIZE); in init_phys_status_page()
2082 struct intel_ringbuffer *ring; in intel_engine_create_ringbuffer() local
2085 ring = kzalloc(sizeof(*ring), GFP_KERNEL); in intel_engine_create_ringbuffer()
2086 if (ring == NULL) in intel_engine_create_ringbuffer()
2089 ring->ring = engine; in intel_engine_create_ringbuffer()
2091 ring->size = size; in intel_engine_create_ringbuffer()
2096 ring->effective_size = size; in intel_engine_create_ringbuffer()
2098 ring->effective_size -= 2 * CACHELINE_BYTES; in intel_engine_create_ringbuffer()
2100 ring->last_retired_head = -1; in intel_engine_create_ringbuffer()
2101 intel_ring_update_space(ring); in intel_engine_create_ringbuffer()
2103 ret = intel_alloc_ringbuffer_obj(engine->dev, ring); in intel_engine_create_ringbuffer()
2107 kfree(ring); in intel_engine_create_ringbuffer()
2111 return ring; in intel_engine_create_ringbuffer()
2115 intel_ringbuffer_free(struct intel_ringbuffer *ring) in intel_ringbuffer_free() argument
2117 intel_destroy_ringbuffer_obj(ring); in intel_ringbuffer_free()
2118 kfree(ring); in intel_ringbuffer_free()
2122 struct intel_engine_cs *ring) in intel_init_ring_buffer() argument
2127 WARN_ON(ring->buffer); in intel_init_ring_buffer()
2129 ring->dev = dev; in intel_init_ring_buffer()
2130 INIT_LIST_HEAD(&ring->active_list); in intel_init_ring_buffer()
2131 INIT_LIST_HEAD(&ring->request_list); in intel_init_ring_buffer()
2132 INIT_LIST_HEAD(&ring->execlist_queue); in intel_init_ring_buffer()
2133 i915_gem_batch_pool_init(dev, &ring->batch_pool); in intel_init_ring_buffer()
2134 memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno)); in intel_init_ring_buffer()
2136 init_waitqueue_head(&ring->irq_queue); in intel_init_ring_buffer()
2138 ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE); in intel_init_ring_buffer()
2141 ring->buffer = ringbuf; in intel_init_ring_buffer()
2144 ret = init_status_page(ring); in intel_init_ring_buffer()
2148 WARN_ON(ring->id != RCS); in intel_init_ring_buffer()
2149 ret = init_phys_status_page(ring); in intel_init_ring_buffer()
2157 ring->name, ret); in intel_init_ring_buffer()
2162 ret = i915_cmd_parser_init_ring(ring); in intel_init_ring_buffer()
2170 ring->buffer = NULL; in intel_init_ring_buffer()
2174 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) in intel_cleanup_ring_buffer() argument
2178 if (!intel_ring_initialized(ring)) in intel_cleanup_ring_buffer()
2181 dev_priv = to_i915(ring->dev); in intel_cleanup_ring_buffer()
2183 intel_stop_ring_buffer(ring); in intel_cleanup_ring_buffer()
2184 WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); in intel_cleanup_ring_buffer()
2186 intel_unpin_ringbuffer_obj(ring->buffer); in intel_cleanup_ring_buffer()
2187 intel_ringbuffer_free(ring->buffer); in intel_cleanup_ring_buffer()
2188 ring->buffer = NULL; in intel_cleanup_ring_buffer()
2190 if (ring->cleanup) in intel_cleanup_ring_buffer()
2191 ring->cleanup(ring); in intel_cleanup_ring_buffer()
2193 if (I915_NEED_GFX_HWS(ring->dev)) { in intel_cleanup_ring_buffer()
2194 cleanup_status_page(ring); in intel_cleanup_ring_buffer()
2196 WARN_ON(ring->id != RCS); in intel_cleanup_ring_buffer()
2197 cleanup_phys_status_page(ring); in intel_cleanup_ring_buffer()
2200 i915_cmd_parser_fini_ring(ring); in intel_cleanup_ring_buffer()
2201 i915_gem_batch_pool_fini(&ring->batch_pool); in intel_cleanup_ring_buffer()
2204 static int ring_wait_for_space(struct intel_engine_cs *ring, int n) in ring_wait_for_space() argument
2206 struct intel_ringbuffer *ringbuf = ring->buffer; in ring_wait_for_space()
2217 list_for_each_entry(request, &ring->request_list, list) { in ring_wait_for_space()
2224 if (WARN_ON(&request->list == &ring->request_list)) in ring_wait_for_space()
2249 int intel_ring_idle(struct intel_engine_cs *ring) in intel_ring_idle() argument
2254 if (list_empty(&ring->request_list)) in intel_ring_idle()
2257 req = list_entry(ring->request_list.prev, in intel_ring_idle()
2263 atomic_read(&to_i915(ring->dev)->gpu_error.reset_counter), in intel_ring_idle()
2264 to_i915(ring->dev)->mm.interruptible, in intel_ring_idle()
2270 request->ringbuf = request->ring->buffer; in intel_ring_alloc_request_extras()
2336 static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes) in __intel_ring_prepare() argument
2338 struct intel_ringbuffer *ringbuf = ring->buffer; in __intel_ring_prepare()
2372 ret = ring_wait_for_space(ring, wait_bytes); in __intel_ring_prepare()
2386 struct intel_engine_cs *ring; in intel_ring_begin() local
2391 ring = req->ring; in intel_ring_begin()
2392 dev_priv = ring->dev->dev_private; in intel_ring_begin()
2399 ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t)); in intel_ring_begin()
2403 ring->buffer->space -= num_dwords * sizeof(uint32_t); in intel_ring_begin()
2410 struct intel_engine_cs *ring = req->ring; in intel_ring_cacheline_align() local
2411 int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); in intel_ring_cacheline_align()
2423 intel_ring_emit(ring, MI_NOOP); in intel_ring_cacheline_align()
2425 intel_ring_advance(ring); in intel_ring_cacheline_align()
2430 void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno) in intel_ring_init_seqno() argument
2432 struct drm_device *dev = ring->dev; in intel_ring_init_seqno()
2436 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); in intel_ring_init_seqno()
2437 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0); in intel_ring_init_seqno()
2439 I915_WRITE(RING_SYNC_2(ring->mmio_base), 0); in intel_ring_init_seqno()
2442 ring->set_seqno(ring, seqno); in intel_ring_init_seqno()
2443 ring->hangcheck.seqno = seqno; in intel_ring_init_seqno()
2446 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, in gen6_bsd_ring_write_tail() argument
2449 struct drm_i915_private *dev_priv = ring->dev->dev_private; in gen6_bsd_ring_write_tail()
2469 I915_WRITE_TAIL(ring, value); in gen6_bsd_ring_write_tail()
2470 POSTING_READ(RING_TAIL(ring->mmio_base)); in gen6_bsd_ring_write_tail()
2482 struct intel_engine_cs *ring = req->ring; in gen6_bsd_ring_flush() local
2491 if (INTEL_INFO(ring->dev)->gen >= 8) in gen6_bsd_ring_flush()
2510 intel_ring_emit(ring, cmd); in gen6_bsd_ring_flush()
2511 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_bsd_ring_flush()
2512 if (INTEL_INFO(ring->dev)->gen >= 8) { in gen6_bsd_ring_flush()
2513 intel_ring_emit(ring, 0); /* upper addr */ in gen6_bsd_ring_flush()
2514 intel_ring_emit(ring, 0); /* value */ in gen6_bsd_ring_flush()
2516 intel_ring_emit(ring, 0); in gen6_bsd_ring_flush()
2517 intel_ring_emit(ring, MI_NOOP); in gen6_bsd_ring_flush()
2519 intel_ring_advance(ring); in gen6_bsd_ring_flush()
2528 struct intel_engine_cs *ring = req->ring; in gen8_ring_dispatch_execbuffer() local
2529 bool ppgtt = USES_PPGTT(ring->dev) && in gen8_ring_dispatch_execbuffer()
2538 intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) | in gen8_ring_dispatch_execbuffer()
2541 intel_ring_emit(ring, lower_32_bits(offset)); in gen8_ring_dispatch_execbuffer()
2542 intel_ring_emit(ring, upper_32_bits(offset)); in gen8_ring_dispatch_execbuffer()
2543 intel_ring_emit(ring, MI_NOOP); in gen8_ring_dispatch_execbuffer()
2544 intel_ring_advance(ring); in gen8_ring_dispatch_execbuffer()
2554 struct intel_engine_cs *ring = req->ring; in hsw_ring_dispatch_execbuffer() local
2561 intel_ring_emit(ring, in hsw_ring_dispatch_execbuffer()
2568 intel_ring_emit(ring, offset); in hsw_ring_dispatch_execbuffer()
2569 intel_ring_advance(ring); in hsw_ring_dispatch_execbuffer()
2579 struct intel_engine_cs *ring = req->ring; in gen6_ring_dispatch_execbuffer() local
2586 intel_ring_emit(ring, in gen6_ring_dispatch_execbuffer()
2591 intel_ring_emit(ring, offset); in gen6_ring_dispatch_execbuffer()
2592 intel_ring_advance(ring); in gen6_ring_dispatch_execbuffer()
2602 struct intel_engine_cs *ring = req->ring; in gen6_ring_flush() local
2603 struct drm_device *dev = ring->dev; in gen6_ring_flush()
2630 intel_ring_emit(ring, cmd); in gen6_ring_flush()
2631 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); in gen6_ring_flush()
2633 intel_ring_emit(ring, 0); /* upper addr */ in gen6_ring_flush()
2634 intel_ring_emit(ring, 0); /* value */ in gen6_ring_flush()
2636 intel_ring_emit(ring, 0); in gen6_ring_flush()
2637 intel_ring_emit(ring, MI_NOOP); in gen6_ring_flush()
2639 intel_ring_advance(ring); in gen6_ring_flush()
2647 struct intel_engine_cs *ring = &dev_priv->ring[RCS]; in intel_init_render_ring_buffer() local
2651 ring->name = "render ring"; in intel_init_render_ring_buffer()
2652 ring->id = RCS; in intel_init_render_ring_buffer()
2653 ring->mmio_base = RENDER_RING_BASE; in intel_init_render_ring_buffer()
2673 ring->init_context = intel_rcs_ctx_init; in intel_init_render_ring_buffer()
2674 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
2675 ring->flush = gen8_render_ring_flush; in intel_init_render_ring_buffer()
2676 ring->irq_get = gen8_ring_get_irq; in intel_init_render_ring_buffer()
2677 ring->irq_put = gen8_ring_put_irq; in intel_init_render_ring_buffer()
2678 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; in intel_init_render_ring_buffer()
2679 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
2680 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2683 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_render_ring_buffer()
2684 ring->semaphore.signal = gen8_rcs_signal; in intel_init_render_ring_buffer()
2688 ring->init_context = intel_rcs_ctx_init; in intel_init_render_ring_buffer()
2689 ring->add_request = gen6_add_request; in intel_init_render_ring_buffer()
2690 ring->flush = gen7_render_ring_flush; in intel_init_render_ring_buffer()
2692 ring->flush = gen6_render_ring_flush; in intel_init_render_ring_buffer()
2693 ring->irq_get = gen6_ring_get_irq; in intel_init_render_ring_buffer()
2694 ring->irq_put = gen6_ring_put_irq; in intel_init_render_ring_buffer()
2695 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; in intel_init_render_ring_buffer()
2696 ring->get_seqno = gen6_ring_get_seqno; in intel_init_render_ring_buffer()
2697 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2699 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_render_ring_buffer()
2700 ring->semaphore.signal = gen6_signal; in intel_init_render_ring_buffer()
2708 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
2709 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; in intel_init_render_ring_buffer()
2710 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; in intel_init_render_ring_buffer()
2711 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; in intel_init_render_ring_buffer()
2712 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_render_ring_buffer()
2713 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; in intel_init_render_ring_buffer()
2714 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; in intel_init_render_ring_buffer()
2715 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; in intel_init_render_ring_buffer()
2716 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; in intel_init_render_ring_buffer()
2717 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_render_ring_buffer()
2720 ring->add_request = pc_render_add_request; in intel_init_render_ring_buffer()
2721 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
2722 ring->get_seqno = pc_render_get_seqno; in intel_init_render_ring_buffer()
2723 ring->set_seqno = pc_render_set_seqno; in intel_init_render_ring_buffer()
2724 ring->irq_get = gen5_ring_get_irq; in intel_init_render_ring_buffer()
2725 ring->irq_put = gen5_ring_put_irq; in intel_init_render_ring_buffer()
2726 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT | in intel_init_render_ring_buffer()
2729 ring->add_request = i9xx_add_request; in intel_init_render_ring_buffer()
2731 ring->flush = gen2_render_ring_flush; in intel_init_render_ring_buffer()
2733 ring->flush = gen4_render_ring_flush; in intel_init_render_ring_buffer()
2734 ring->get_seqno = ring_get_seqno; in intel_init_render_ring_buffer()
2735 ring->set_seqno = ring_set_seqno; in intel_init_render_ring_buffer()
2737 ring->irq_get = i8xx_ring_get_irq; in intel_init_render_ring_buffer()
2738 ring->irq_put = i8xx_ring_put_irq; in intel_init_render_ring_buffer()
2740 ring->irq_get = i9xx_ring_get_irq; in intel_init_render_ring_buffer()
2741 ring->irq_put = i9xx_ring_put_irq; in intel_init_render_ring_buffer()
2743 ring->irq_enable_mask = I915_USER_INTERRUPT; in intel_init_render_ring_buffer()
2745 ring->write_tail = ring_write_tail; in intel_init_render_ring_buffer()
2748 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2750 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2752 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_render_ring_buffer()
2754 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_render_ring_buffer()
2756 ring->dispatch_execbuffer = i830_dispatch_execbuffer; in intel_init_render_ring_buffer()
2758 ring->dispatch_execbuffer = i915_dispatch_execbuffer; in intel_init_render_ring_buffer()
2759 ring->init_hw = init_render_ring; in intel_init_render_ring_buffer()
2760 ring->cleanup = render_ring_cleanup; in intel_init_render_ring_buffer()
2777 ring->scratch.obj = obj; in intel_init_render_ring_buffer()
2778 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); in intel_init_render_ring_buffer()
2781 ret = intel_init_ring_buffer(dev, ring); in intel_init_render_ring_buffer()
2786 ret = intel_init_pipe_control(ring); in intel_init_render_ring_buffer()
2797 struct intel_engine_cs *ring = &dev_priv->ring[VCS]; in intel_init_bsd_ring_buffer() local
2799 ring->name = "bsd ring"; in intel_init_bsd_ring_buffer()
2800 ring->id = VCS; in intel_init_bsd_ring_buffer()
2802 ring->write_tail = ring_write_tail; in intel_init_bsd_ring_buffer()
2804 ring->mmio_base = GEN6_BSD_RING_BASE; in intel_init_bsd_ring_buffer()
2807 ring->write_tail = gen6_bsd_ring_write_tail; in intel_init_bsd_ring_buffer()
2808 ring->flush = gen6_bsd_ring_flush; in intel_init_bsd_ring_buffer()
2809 ring->add_request = gen6_add_request; in intel_init_bsd_ring_buffer()
2810 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd_ring_buffer()
2811 ring->set_seqno = ring_set_seqno; in intel_init_bsd_ring_buffer()
2813 ring->irq_enable_mask = in intel_init_bsd_ring_buffer()
2815 ring->irq_get = gen8_ring_get_irq; in intel_init_bsd_ring_buffer()
2816 ring->irq_put = gen8_ring_put_irq; in intel_init_bsd_ring_buffer()
2817 ring->dispatch_execbuffer = in intel_init_bsd_ring_buffer()
2820 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_bsd_ring_buffer()
2821 ring->semaphore.signal = gen8_xcs_signal; in intel_init_bsd_ring_buffer()
2825 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2826 ring->irq_get = gen6_ring_get_irq; in intel_init_bsd_ring_buffer()
2827 ring->irq_put = gen6_ring_put_irq; in intel_init_bsd_ring_buffer()
2828 ring->dispatch_execbuffer = in intel_init_bsd_ring_buffer()
2831 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_bsd_ring_buffer()
2832 ring->semaphore.signal = gen6_signal; in intel_init_bsd_ring_buffer()
2833 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; in intel_init_bsd_ring_buffer()
2834 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
2835 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; in intel_init_bsd_ring_buffer()
2836 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; in intel_init_bsd_ring_buffer()
2837 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_bsd_ring_buffer()
2838 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; in intel_init_bsd_ring_buffer()
2839 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; in intel_init_bsd_ring_buffer()
2840 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; in intel_init_bsd_ring_buffer()
2841 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; in intel_init_bsd_ring_buffer()
2842 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_bsd_ring_buffer()
2846 ring->mmio_base = BSD_RING_BASE; in intel_init_bsd_ring_buffer()
2847 ring->flush = bsd_ring_flush; in intel_init_bsd_ring_buffer()
2848 ring->add_request = i9xx_add_request; in intel_init_bsd_ring_buffer()
2849 ring->get_seqno = ring_get_seqno; in intel_init_bsd_ring_buffer()
2850 ring->set_seqno = ring_set_seqno; in intel_init_bsd_ring_buffer()
2852 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2853 ring->irq_get = gen5_ring_get_irq; in intel_init_bsd_ring_buffer()
2854 ring->irq_put = gen5_ring_put_irq; in intel_init_bsd_ring_buffer()
2856 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT; in intel_init_bsd_ring_buffer()
2857 ring->irq_get = i9xx_ring_get_irq; in intel_init_bsd_ring_buffer()
2858 ring->irq_put = i9xx_ring_put_irq; in intel_init_bsd_ring_buffer()
2860 ring->dispatch_execbuffer = i965_dispatch_execbuffer; in intel_init_bsd_ring_buffer()
2862 ring->init_hw = init_ring_common; in intel_init_bsd_ring_buffer()
2864 return intel_init_ring_buffer(dev, ring); in intel_init_bsd_ring_buffer()
2873 struct intel_engine_cs *ring = &dev_priv->ring[VCS2]; in intel_init_bsd2_ring_buffer() local
2875 ring->name = "bsd2 ring"; in intel_init_bsd2_ring_buffer()
2876 ring->id = VCS2; in intel_init_bsd2_ring_buffer()
2878 ring->write_tail = ring_write_tail; in intel_init_bsd2_ring_buffer()
2879 ring->mmio_base = GEN8_BSD2_RING_BASE; in intel_init_bsd2_ring_buffer()
2880 ring->flush = gen6_bsd_ring_flush; in intel_init_bsd2_ring_buffer()
2881 ring->add_request = gen6_add_request; in intel_init_bsd2_ring_buffer()
2882 ring->get_seqno = gen6_ring_get_seqno; in intel_init_bsd2_ring_buffer()
2883 ring->set_seqno = ring_set_seqno; in intel_init_bsd2_ring_buffer()
2884 ring->irq_enable_mask = in intel_init_bsd2_ring_buffer()
2886 ring->irq_get = gen8_ring_get_irq; in intel_init_bsd2_ring_buffer()
2887 ring->irq_put = gen8_ring_put_irq; in intel_init_bsd2_ring_buffer()
2888 ring->dispatch_execbuffer = in intel_init_bsd2_ring_buffer()
2891 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_bsd2_ring_buffer()
2892 ring->semaphore.signal = gen8_xcs_signal; in intel_init_bsd2_ring_buffer()
2895 ring->init_hw = init_ring_common; in intel_init_bsd2_ring_buffer()
2897 return intel_init_ring_buffer(dev, ring); in intel_init_bsd2_ring_buffer()
2903 struct intel_engine_cs *ring = &dev_priv->ring[BCS]; in intel_init_blt_ring_buffer() local
2905 ring->name = "blitter ring"; in intel_init_blt_ring_buffer()
2906 ring->id = BCS; in intel_init_blt_ring_buffer()
2908 ring->mmio_base = BLT_RING_BASE; in intel_init_blt_ring_buffer()
2909 ring->write_tail = ring_write_tail; in intel_init_blt_ring_buffer()
2910 ring->flush = gen6_ring_flush; in intel_init_blt_ring_buffer()
2911 ring->add_request = gen6_add_request; in intel_init_blt_ring_buffer()
2912 ring->get_seqno = gen6_ring_get_seqno; in intel_init_blt_ring_buffer()
2913 ring->set_seqno = ring_set_seqno; in intel_init_blt_ring_buffer()
2915 ring->irq_enable_mask = in intel_init_blt_ring_buffer()
2917 ring->irq_get = gen8_ring_get_irq; in intel_init_blt_ring_buffer()
2918 ring->irq_put = gen8_ring_put_irq; in intel_init_blt_ring_buffer()
2919 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
2921 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_blt_ring_buffer()
2922 ring->semaphore.signal = gen8_xcs_signal; in intel_init_blt_ring_buffer()
2926 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT; in intel_init_blt_ring_buffer()
2927 ring->irq_get = gen6_ring_get_irq; in intel_init_blt_ring_buffer()
2928 ring->irq_put = gen6_ring_put_irq; in intel_init_blt_ring_buffer()
2929 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_blt_ring_buffer()
2931 ring->semaphore.signal = gen6_signal; in intel_init_blt_ring_buffer()
2932 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_blt_ring_buffer()
2940 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; in intel_init_blt_ring_buffer()
2941 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; in intel_init_blt_ring_buffer()
2942 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
2943 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; in intel_init_blt_ring_buffer()
2944 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_blt_ring_buffer()
2945 ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; in intel_init_blt_ring_buffer()
2946 ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; in intel_init_blt_ring_buffer()
2947 ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; in intel_init_blt_ring_buffer()
2948 ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; in intel_init_blt_ring_buffer()
2949 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_blt_ring_buffer()
2952 ring->init_hw = init_ring_common; in intel_init_blt_ring_buffer()
2954 return intel_init_ring_buffer(dev, ring); in intel_init_blt_ring_buffer()
2960 struct intel_engine_cs *ring = &dev_priv->ring[VECS]; in intel_init_vebox_ring_buffer() local
2962 ring->name = "video enhancement ring"; in intel_init_vebox_ring_buffer()
2963 ring->id = VECS; in intel_init_vebox_ring_buffer()
2965 ring->mmio_base = VEBOX_RING_BASE; in intel_init_vebox_ring_buffer()
2966 ring->write_tail = ring_write_tail; in intel_init_vebox_ring_buffer()
2967 ring->flush = gen6_ring_flush; in intel_init_vebox_ring_buffer()
2968 ring->add_request = gen6_add_request; in intel_init_vebox_ring_buffer()
2969 ring->get_seqno = gen6_ring_get_seqno; in intel_init_vebox_ring_buffer()
2970 ring->set_seqno = ring_set_seqno; in intel_init_vebox_ring_buffer()
2973 ring->irq_enable_mask = in intel_init_vebox_ring_buffer()
2975 ring->irq_get = gen8_ring_get_irq; in intel_init_vebox_ring_buffer()
2976 ring->irq_put = gen8_ring_put_irq; in intel_init_vebox_ring_buffer()
2977 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; in intel_init_vebox_ring_buffer()
2979 ring->semaphore.sync_to = gen8_ring_sync; in intel_init_vebox_ring_buffer()
2980 ring->semaphore.signal = gen8_xcs_signal; in intel_init_vebox_ring_buffer()
2984 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; in intel_init_vebox_ring_buffer()
2985 ring->irq_get = hsw_vebox_get_irq; in intel_init_vebox_ring_buffer()
2986 ring->irq_put = hsw_vebox_put_irq; in intel_init_vebox_ring_buffer()
2987 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; in intel_init_vebox_ring_buffer()
2989 ring->semaphore.sync_to = gen6_ring_sync; in intel_init_vebox_ring_buffer()
2990 ring->semaphore.signal = gen6_signal; in intel_init_vebox_ring_buffer()
2991 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; in intel_init_vebox_ring_buffer()
2992 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; in intel_init_vebox_ring_buffer()
2993 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; in intel_init_vebox_ring_buffer()
2994 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_vebox_ring_buffer()
2995 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; in intel_init_vebox_ring_buffer()
2996 ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; in intel_init_vebox_ring_buffer()
2997 ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; in intel_init_vebox_ring_buffer()
2998 ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; in intel_init_vebox_ring_buffer()
2999 ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; in intel_init_vebox_ring_buffer()
3000 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; in intel_init_vebox_ring_buffer()
3003 ring->init_hw = init_ring_common; in intel_init_vebox_ring_buffer()
3005 return intel_init_ring_buffer(dev, ring); in intel_init_vebox_ring_buffer()
3011 struct intel_engine_cs *ring = req->ring; in intel_ring_flush_all_caches() local
3014 if (!ring->gpu_caches_dirty) in intel_ring_flush_all_caches()
3017 ret = ring->flush(req, 0, I915_GEM_GPU_DOMAINS); in intel_ring_flush_all_caches()
3023 ring->gpu_caches_dirty = false; in intel_ring_flush_all_caches()
3030 struct intel_engine_cs *ring = req->ring; in intel_ring_invalidate_all_caches() local
3035 if (ring->gpu_caches_dirty) in intel_ring_invalidate_all_caches()
3038 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, flush_domains); in intel_ring_invalidate_all_caches()
3044 ring->gpu_caches_dirty = false; in intel_ring_invalidate_all_caches()
3049 intel_stop_ring_buffer(struct intel_engine_cs *ring) in intel_stop_ring_buffer() argument
3053 if (!intel_ring_initialized(ring)) in intel_stop_ring_buffer()
3056 ret = intel_ring_idle(ring); in intel_stop_ring_buffer()
3057 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error)) in intel_stop_ring_buffer()
3059 ring->name, ret); in intel_stop_ring_buffer()
3061 stop_ring(ring); in intel_stop_ring_buffer()