80 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_stopped()  local
81 	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);  in intel_ring_stopped()
443 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in ring_write_tail()  local
449 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_get_active_head()  local
465 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in ring_setup_phys_status_page()  local
468 	addr = dev_priv->status_page_dmah->busaddr;  in ring_setup_phys_status_page()
470 		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;  in ring_setup_phys_status_page()
477 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_setup_status_page()  local
538 	struct drm_i915_private *dev_priv = to_i915(ring->dev);  in stop_ring()  local
568 	struct drm_i915_private *dev_priv = dev->dev_private;  in init_ring_common()  local
573 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);  in init_ring_common()
645 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);  in init_ring_common()
713 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_ring_workarounds_emit()  local
714 	struct i915_workarounds *w = &dev_priv->workarounds;  in intel_ring_workarounds_emit()
763 static int wa_add(struct drm_i915_private *dev_priv,  in wa_add()  argument
766 	const u32 idx = dev_priv->workarounds.count;  in wa_add()
771 	dev_priv->workarounds.reg[idx].addr = addr;  in wa_add()
772 	dev_priv->workarounds.reg[idx].value = val;  in wa_add()
773 	dev_priv->workarounds.reg[idx].mask = mask;  in wa_add()
775 	dev_priv->workarounds.count++;  in wa_add()
781 		const int r = wa_add(dev_priv, (addr), (mask), (val)); \
803 	struct drm_i915_private *dev_priv = dev->dev_private;  in bdw_init_workarounds()  local
864 	struct drm_i915_private *dev_priv = dev->dev_private;  in chv_init_workarounds()  local
912 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen9_init_workarounds()  local
970 	struct drm_i915_private *dev_priv = dev->dev_private;  in skl_tune_iz_hashing()  local
981 		if (hweight8(dev_priv->info.subslice_7eu[i]) != 1)  in skl_tune_iz_hashing()
990 		ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;  in skl_tune_iz_hashing()
1013 	struct drm_i915_private *dev_priv = dev->dev_private;  in skl_init_workarounds()  local
1035 	struct drm_i915_private *dev_priv = dev->dev_private;  in init_workarounds_ring()  local
1039 	dev_priv->workarounds.count = 0;  in init_workarounds_ring()
1058 	struct drm_i915_private *dev_priv = dev->dev_private;  in init_render_ring()  local
1110 	struct drm_i915_private *dev_priv = dev->dev_private;  in render_ring_cleanup()  local
1112 	if (dev_priv->semaphore_obj) {  in render_ring_cleanup()
1113 		i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);  in render_ring_cleanup()
1114 		drm_gem_object_unreference(&dev_priv->semaphore_obj->base);  in render_ring_cleanup()
1115 		dev_priv->semaphore_obj = NULL;  in render_ring_cleanup()
1126 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_rcs_signal()  local
1138 	for_each_ring(waiter, dev_priv, i) {  in gen8_rcs_signal()
1167 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_xcs_signal()  local
1179 	for_each_ring(waiter, dev_priv, i) {  in gen8_xcs_signal()
1205 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_signal()  local
1218 	for_each_ring(useless, dev_priv, i) {  in gen6_signal()
1271 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_has_seqno_wrapped()  local
1272 	return dev_priv->last_seqno < seqno;  in i915_gem_has_seqno_wrapped()
1288 	struct drm_i915_private *dev_priv = waiter->dev->dev_private;  in gen8_ring_sync()  local
1414 		struct drm_i915_private *dev_priv = ring->dev->dev_private;  in gen6_ring_get_seqno()  local
1449 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen5_ring_get_irq()  local
1452 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in gen5_ring_get_irq()
1455 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen5_ring_get_irq()
1457 		gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen5_ring_get_irq()
1458 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen5_ring_get_irq()
1467 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen5_ring_put_irq()  local
1470 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen5_ring_put_irq()
1472 		gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen5_ring_put_irq()
1473 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen5_ring_put_irq()
1480 	struct drm_i915_private *dev_priv = dev->dev_private;  in i9xx_ring_get_irq()  local
1483 	if (!intel_irqs_enabled(dev_priv))  in i9xx_ring_get_irq()
1486 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i9xx_ring_get_irq()
1488 		dev_priv->irq_mask &= ~ring->irq_enable_mask;  in i9xx_ring_get_irq()
1489 		I915_WRITE(IMR, dev_priv->irq_mask);  in i9xx_ring_get_irq()
1492 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i9xx_ring_get_irq()
1501 	struct drm_i915_private *dev_priv = dev->dev_private;  in i9xx_ring_put_irq()  local
1504 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i9xx_ring_put_irq()
1506 		dev_priv->irq_mask |= ring->irq_enable_mask;  in i9xx_ring_put_irq()
1507 		I915_WRITE(IMR, dev_priv->irq_mask);  in i9xx_ring_put_irq()
1510 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i9xx_ring_put_irq()
1517 	struct drm_i915_private *dev_priv = dev->dev_private;  in i8xx_ring_get_irq()  local
1520 	if (!intel_irqs_enabled(dev_priv))  in i8xx_ring_get_irq()
1523 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i8xx_ring_get_irq()
1525 		dev_priv->irq_mask &= ~ring->irq_enable_mask;  in i8xx_ring_get_irq()
1526 		I915_WRITE16(IMR, dev_priv->irq_mask);  in i8xx_ring_get_irq()
1529 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i8xx_ring_get_irq()
1538 	struct drm_i915_private *dev_priv = dev->dev_private;  in i8xx_ring_put_irq()  local
1541 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i8xx_ring_put_irq()
1543 		dev_priv->irq_mask |= ring->irq_enable_mask;  in i8xx_ring_put_irq()
1544 		I915_WRITE16(IMR, dev_priv->irq_mask);  in i8xx_ring_put_irq()
1547 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i8xx_ring_put_irq()
1590 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ring_get_irq()  local
1593 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in gen6_ring_get_irq()
1596 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen6_ring_get_irq()
1604 		gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen6_ring_get_irq()
1606 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen6_ring_get_irq()
1615 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ring_put_irq()  local
1618 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen6_ring_put_irq()
1624 		gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen6_ring_put_irq()
1626 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen6_ring_put_irq()
1633 	struct drm_i915_private *dev_priv = dev->dev_private;  in hsw_vebox_get_irq()  local
1636 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in hsw_vebox_get_irq()
1639 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in hsw_vebox_get_irq()
1642 		gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);  in hsw_vebox_get_irq()
1644 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in hsw_vebox_get_irq()
1653 	struct drm_i915_private *dev_priv = dev->dev_private;  in hsw_vebox_put_irq()  local
1656 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in hsw_vebox_put_irq()
1659 		gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);  in hsw_vebox_put_irq()
1661 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in hsw_vebox_put_irq()
1668 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_ring_get_irq()  local
1671 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in gen8_ring_get_irq()
1674 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen8_ring_get_irq()
1685 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen8_ring_get_irq()
1694 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_ring_put_irq()  local
1697 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen8_ring_put_irq()
1707 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen8_ring_put_irq()
1884 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in init_phys_status_page()  local
1886 	if (!dev_priv->status_page_dmah) {  in init_phys_status_page()
1887 		dev_priv->status_page_dmah =  in init_phys_status_page()
1889 		if (!dev_priv->status_page_dmah)  in init_phys_status_page()
1893 	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;  in init_phys_status_page()
1909 	struct drm_i915_private *dev_priv = to_i915(dev);  in intel_pin_and_map_ringbuffer_obj()  local
1923 	ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +  in intel_pin_and_map_ringbuffer_obj()
2033 	struct drm_i915_private *dev_priv;  in intel_cleanup_ring_buffer()  local
2039 	dev_priv = to_i915(ring->dev);  in intel_cleanup_ring_buffer()
2091 	struct drm_i915_private *dev_priv = dev->dev_private;  in ring_wait_for_space()  local
2121 		if (dev_priv->mm.interruptible && signal_pending(current)) {  in ring_wait_for_space()
2126 		ret = i915_gem_check_wedge(&dev_priv->gpu_error,  in ring_wait_for_space()
2127 					   dev_priv->mm.interruptible);  in ring_wait_for_space()
2239 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_begin()  local
2242 	ret = i915_gem_check_wedge(&dev_priv->gpu_error,  in intel_ring_begin()
2243 				   dev_priv->mm.interruptible);  in intel_ring_begin()
2285 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_ring_init_seqno()  local
2303 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in gen6_bsd_ring_write_tail()  local
2491 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_render_ring_buffer()  local
2492 	struct intel_engine_cs *ring = &dev_priv->ring[RCS];  in intel_init_render_ring_buffer()
2514 					dev_priv->semaphore_obj = obj;  in intel_init_render_ring_buffer()
2527 			WARN_ON(!dev_priv->semaphore_obj);  in intel_init_render_ring_buffer()
2640 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_bsd_ring_buffer()  local
2641 	struct intel_engine_cs *ring = &dev_priv->ring[VCS];  in intel_init_bsd_ring_buffer()
2716 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_bsd2_ring_buffer()  local
2717 	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];  in intel_init_bsd2_ring_buffer()
2746 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_blt_ring_buffer()  local
2747 	struct intel_engine_cs *ring = &dev_priv->ring[BCS];  in intel_init_blt_ring_buffer()
2803 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_vebox_ring_buffer()  local
2804 	struct intel_engine_cs *ring = &dev_priv->ring[VECS];  in intel_init_vebox_ring_buffer()