80 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_stopped()  local
81 	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);  in intel_ring_stopped()
451 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in ring_write_tail()  local
457 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_get_active_head()  local
473 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in ring_setup_phys_status_page()  local
476 	addr = dev_priv->status_page_dmah->busaddr;  in ring_setup_phys_status_page()
478 		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;  in ring_setup_phys_status_page()
485 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in intel_ring_setup_status_page()  local
546 	struct drm_i915_private *dev_priv = to_i915(ring->dev);  in stop_ring()  local
576 	struct drm_i915_private *dev_priv = dev->dev_private;  in init_ring_common()  local
581 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);  in init_ring_common()
653 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);  in init_ring_common()
721 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_ring_workarounds_emit()  local
722 	struct i915_workarounds *w = &dev_priv->workarounds;  in intel_ring_workarounds_emit()
770 static int wa_add(struct drm_i915_private *dev_priv,  in wa_add()  argument
773 	const u32 idx = dev_priv->workarounds.count;  in wa_add()
778 	dev_priv->workarounds.reg[idx].addr = addr;  in wa_add()
779 	dev_priv->workarounds.reg[idx].value = val;  in wa_add()
780 	dev_priv->workarounds.reg[idx].mask = mask;  in wa_add()
782 	dev_priv->workarounds.count++;  in wa_add()
788 		const int r = wa_add(dev_priv, (addr), (mask), (val)); \
810 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_init_workarounds()  local
863 	struct drm_i915_private *dev_priv = dev->dev_private;  in bdw_init_workarounds()  local
892 	struct drm_i915_private *dev_priv = dev->dev_private;  in chv_init_workarounds()  local
910 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen9_init_workarounds()  local
994 	struct drm_i915_private *dev_priv = dev->dev_private;  in skl_tune_iz_hashing()  local
1005 		if (hweight8(dev_priv->info.subslice_7eu[i]) != 1)  in skl_tune_iz_hashing()
1014 		ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;  in skl_tune_iz_hashing()
1037 	struct drm_i915_private *dev_priv = dev->dev_private;  in skl_init_workarounds()  local
1104 	struct drm_i915_private *dev_priv = dev->dev_private;  in bxt_init_workarounds()  local
1138 	struct drm_i915_private *dev_priv = dev->dev_private;  in init_workarounds_ring()  local
1142 	dev_priv->workarounds.count = 0;  in init_workarounds_ring()
1162 	struct drm_i915_private *dev_priv = dev->dev_private;  in init_render_ring()  local
1214 	struct drm_i915_private *dev_priv = dev->dev_private;  in render_ring_cleanup()  local
1216 	if (dev_priv->semaphore_obj) {  in render_ring_cleanup()
1217 		i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);  in render_ring_cleanup()
1218 		drm_gem_object_unreference(&dev_priv->semaphore_obj->base);  in render_ring_cleanup()
1219 		dev_priv->semaphore_obj = NULL;  in render_ring_cleanup()
1231 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_rcs_signal()  local
1243 	for_each_ring(waiter, dev_priv, i) {  in gen8_rcs_signal()
1272 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_xcs_signal()  local
1284 	for_each_ring(waiter, dev_priv, i) {  in gen8_xcs_signal()
1310 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_signal()  local
1323 	for_each_ring(useless, dev_priv, i) {  in gen6_signal()
1374 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_has_seqno_wrapped()  local
1375 	return dev_priv->last_seqno < seqno;  in i915_gem_has_seqno_wrapped()
1392 	struct drm_i915_private *dev_priv = waiter->dev->dev_private;  in gen8_ring_sync()  local
1518 		struct drm_i915_private *dev_priv = ring->dev->dev_private;  in gen6_ring_get_seqno()  local
1553 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen5_ring_get_irq()  local
1556 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in gen5_ring_get_irq()
1559 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen5_ring_get_irq()
1561 		gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen5_ring_get_irq()
1562 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen5_ring_get_irq()
1571 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen5_ring_put_irq()  local
1574 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen5_ring_put_irq()
1576 		gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen5_ring_put_irq()
1577 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen5_ring_put_irq()
1584 	struct drm_i915_private *dev_priv = dev->dev_private;  in i9xx_ring_get_irq()  local
1587 	if (!intel_irqs_enabled(dev_priv))  in i9xx_ring_get_irq()
1590 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i9xx_ring_get_irq()
1592 		dev_priv->irq_mask &= ~ring->irq_enable_mask;  in i9xx_ring_get_irq()
1593 		I915_WRITE(IMR, dev_priv->irq_mask);  in i9xx_ring_get_irq()
1596 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i9xx_ring_get_irq()
1605 	struct drm_i915_private *dev_priv = dev->dev_private;  in i9xx_ring_put_irq()  local
1608 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i9xx_ring_put_irq()
1610 		dev_priv->irq_mask |= ring->irq_enable_mask;  in i9xx_ring_put_irq()
1611 		I915_WRITE(IMR, dev_priv->irq_mask);  in i9xx_ring_put_irq()
1614 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i9xx_ring_put_irq()
1621 	struct drm_i915_private *dev_priv = dev->dev_private;  in i8xx_ring_get_irq()  local
1624 	if (!intel_irqs_enabled(dev_priv))  in i8xx_ring_get_irq()
1627 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i8xx_ring_get_irq()
1629 		dev_priv->irq_mask &= ~ring->irq_enable_mask;  in i8xx_ring_get_irq()
1630 		I915_WRITE16(IMR, dev_priv->irq_mask);  in i8xx_ring_get_irq()
1633 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i8xx_ring_get_irq()
1642 	struct drm_i915_private *dev_priv = dev->dev_private;  in i8xx_ring_put_irq()  local
1645 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in i8xx_ring_put_irq()
1647 		dev_priv->irq_mask |= ring->irq_enable_mask;  in i8xx_ring_put_irq()
1648 		I915_WRITE16(IMR, dev_priv->irq_mask);  in i8xx_ring_put_irq()
1651 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in i8xx_ring_put_irq()
1695 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ring_get_irq()  local
1698 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in gen6_ring_get_irq()
1701 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen6_ring_get_irq()
1709 		gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen6_ring_get_irq()
1711 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen6_ring_get_irq()
1720 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ring_put_irq()  local
1723 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen6_ring_put_irq()
1729 		gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);  in gen6_ring_put_irq()
1731 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen6_ring_put_irq()
1738 	struct drm_i915_private *dev_priv = dev->dev_private;  in hsw_vebox_get_irq()  local
1741 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in hsw_vebox_get_irq()
1744 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in hsw_vebox_get_irq()
1747 		gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);  in hsw_vebox_get_irq()
1749 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in hsw_vebox_get_irq()
1758 	struct drm_i915_private *dev_priv = dev->dev_private;  in hsw_vebox_put_irq()  local
1761 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in hsw_vebox_put_irq()
1764 		gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);  in hsw_vebox_put_irq()
1766 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in hsw_vebox_put_irq()
1773 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_ring_get_irq()  local
1776 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))  in gen8_ring_get_irq()
1779 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen8_ring_get_irq()
1790 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen8_ring_get_irq()
1799 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_ring_put_irq()  local
1802 	spin_lock_irqsave(&dev_priv->irq_lock, flags);  in gen8_ring_put_irq()
1812 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);  in gen8_ring_put_irq()
1927 	struct drm_i915_private *dev_priv = to_i915(ring->dev);  in cleanup_phys_status_page()  local
1929 	if (!dev_priv->status_page_dmah)  in cleanup_phys_status_page()
1932 	drm_pci_free(ring->dev, dev_priv->status_page_dmah);  in cleanup_phys_status_page()
2003 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in init_phys_status_page()  local
2005 	if (!dev_priv->status_page_dmah) {  in init_phys_status_page()
2006 		dev_priv->status_page_dmah =  in init_phys_status_page()
2008 		if (!dev_priv->status_page_dmah)  in init_phys_status_page()
2012 	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;  in init_phys_status_page()
2028 	struct drm_i915_private *dev_priv = to_i915(dev);  in intel_pin_and_map_ringbuffer_obj()  local
2042 	ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +  in intel_pin_and_map_ringbuffer_obj()
2176 	struct drm_i915_private *dev_priv;  in intel_cleanup_ring_buffer()  local
2181 	dev_priv = to_i915(ring->dev);  in intel_cleanup_ring_buffer()
2387 	struct drm_i915_private *dev_priv;  in intel_ring_begin()  local
2392 	dev_priv = ring->dev->dev_private;  in intel_ring_begin()
2394 	ret = i915_gem_check_wedge(&dev_priv->gpu_error,  in intel_ring_begin()
2395 				   dev_priv->mm.interruptible);  in intel_ring_begin()
2433 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_ring_init_seqno()  local
2449 	struct drm_i915_private *dev_priv = ring->dev->dev_private;  in gen6_bsd_ring_write_tail()  local
2646 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_render_ring_buffer()  local
2647 	struct intel_engine_cs *ring = &dev_priv->ring[RCS];  in intel_init_render_ring_buffer()
2669 					dev_priv->semaphore_obj = obj;  in intel_init_render_ring_buffer()
2682 			WARN_ON(!dev_priv->semaphore_obj);  in intel_init_render_ring_buffer()
2796 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_bsd_ring_buffer()  local
2797 	struct intel_engine_cs *ring = &dev_priv->ring[VCS];  in intel_init_bsd_ring_buffer()
2872 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_bsd2_ring_buffer()  local
2873 	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];  in intel_init_bsd2_ring_buffer()
2902 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_blt_ring_buffer()  local
2903 	struct intel_engine_cs *ring = &dev_priv->ring[BCS];  in intel_init_blt_ring_buffer()
2959 	struct drm_i915_private *dev_priv = dev->dev_private;  in intel_init_vebox_ring_buffer()  local
2960 	struct intel_engine_cs *ring = &dev_priv->ring[VECS];  in intel_init_vebox_ring_buffer()