906 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_ppgtt_notify_vgt()  local
1627 static void gen6_write_page_range(struct drm_i915_private *dev_priv,  in gen6_write_page_range()  argument
1639 	readl(dev_priv->gtt.gsm);  in gen6_write_page_range()
1679 	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);  in vgpu_mm_switch()  local
1724 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_mm_switch()  local
1737 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_ppgtt_enable()  local
1741 	for_each_ring(ring, dev_priv, j) {  in gen8_ppgtt_enable()
1750 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen7_ppgtt_enable()  local
1767 	for_each_ring(ring, dev_priv, i) {  in gen7_ppgtt_enable()
1776 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ppgtt_enable()  local
1866 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_alloc_va_range()  local
1934 	readl(dev_priv->gtt.gsm);  in gen6_alloc_va_range()
1999 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ppgtt_allocate_page_directories()  local
2007 	BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));  in gen6_ppgtt_allocate_page_directories()
2014 	ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,  in gen6_ppgtt_allocate_page_directories()
2017 						  0, dev_priv->gtt.base.total,  in gen6_ppgtt_allocate_page_directories()
2020 		ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,  in gen6_ppgtt_allocate_page_directories()
2023 					       0, dev_priv->gtt.base.total,  in gen6_ppgtt_allocate_page_directories()
2036 	if (ppgtt->node.start < dev_priv->gtt.mappable_end)  in gen6_ppgtt_allocate_page_directories()
2064 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_ppgtt_init()  local
2067 	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;  in gen6_ppgtt_init()
2097 	ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +  in gen6_ppgtt_init()
2102 	gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);  in gen6_ppgtt_init()
2125 				    struct drm_i915_private *dev_priv)  in i915_address_space_init()  argument
2128 	vm->dev = dev_priv->dev;  in i915_address_space_init()
2131 	list_add_tail(&vm->global_link, &dev_priv->vm_list);  in i915_address_space_init()
2136 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_ppgtt_init()  local
2142 		i915_address_space_init(&ppgtt->base, dev_priv);  in i915_ppgtt_init()
2173 	struct drm_i915_private *dev_priv = req->ring->dev->dev_private;  in i915_ppgtt_init_ring()  local
2174 	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;  in i915_ppgtt_init_ring()
2242 static bool do_idling(struct drm_i915_private *dev_priv)  in do_idling()  argument
2244 	bool ret = dev_priv->mm.interruptible;  in do_idling()
2246 	if (unlikely(dev_priv->gtt.do_idle_maps)) {  in do_idling()
2247 		dev_priv->mm.interruptible = false;  in do_idling()
2248 		if (i915_gpu_idle(dev_priv->dev)) {  in do_idling()
2258 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)  in undo_idling()  argument
2260 	if (unlikely(dev_priv->gtt.do_idle_maps))  in undo_idling()
2261 		dev_priv->mm.interruptible = interruptible;  in undo_idling()
2266 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_check_and_clear_faults()  local
2273 	for_each_ring(ring, dev_priv, i) {  in i915_check_and_clear_faults()
2290 	POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));  in i915_check_and_clear_faults()
2293 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)  in i915_ggtt_flush()  argument
2295 	if (INTEL_INFO(dev_priv->dev)->gen < 6) {  in i915_ggtt_flush()
2305 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_suspend_gtt_mappings()  local
2315 	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,  in i915_gem_suspend_gtt_mappings()
2316 				       dev_priv->gtt.base.start,  in i915_gem_suspend_gtt_mappings()
2317 				       dev_priv->gtt.base.total,  in i915_gem_suspend_gtt_mappings()
2320 	i915_ggtt_flush(dev_priv);  in i915_gem_suspend_gtt_mappings()
2348 	struct drm_i915_private *dev_priv = vm->dev->dev_private;  in gen8_ggtt_insert_entries()  local
2351 		(gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;  in gen8_ggtt_insert_entries()
2394 	struct drm_i915_private *dev_priv = vm->dev->dev_private;  in gen6_ggtt_insert_entries()  local
2397 		(gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;  in gen6_ggtt_insert_entries()
2432 	struct drm_i915_private *dev_priv = vm->dev->dev_private;  in gen8_ggtt_clear_range()  local
2436 		(gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;  in gen8_ggtt_clear_range()
2437 	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;  in gen8_ggtt_clear_range()
2458 	struct drm_i915_private *dev_priv = vm->dev->dev_private;  in gen6_ggtt_clear_range()  local
2462 		(gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;  in gen6_ggtt_clear_range()
2463 	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;  in gen6_ggtt_clear_range()
2536 	struct drm_i915_private *dev_priv = dev->dev_private;  in aliasing_gtt_bind_vma()  local
2559 		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;  in aliasing_gtt_bind_vma()
2571 	struct drm_i915_private *dev_priv = dev->dev_private;  in ggtt_unbind_vma()  local
2584 	if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) {  in ggtt_unbind_vma()
2585 		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;  in ggtt_unbind_vma()
2597 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_gtt_finish_object()  local
2600 	interruptible = do_idling(dev_priv);  in i915_gem_gtt_finish_object()
2605 	undo_idling(dev_priv, interruptible);  in i915_gem_gtt_finish_object()
2639 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_setup_global_gtt()  local
2640 	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;  in i915_gem_setup_global_gtt()
2653 	i915_address_space_init(ggtt_vm, dev_priv);  in i915_gem_setup_global_gtt()
2666 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {  in i915_gem_setup_global_gtt()
2722 		dev_priv->mm.aliasing_ppgtt = ppgtt;  in i915_gem_setup_global_gtt()
2723 		WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma);  in i915_gem_setup_global_gtt()
2724 		dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma;  in i915_gem_setup_global_gtt()
2732 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_init_global_gtt()  local
2735 	gtt_size = dev_priv->gtt.base.total;  in i915_gem_init_global_gtt()
2736 	mappable_size = dev_priv->gtt.mappable_end;  in i915_gem_init_global_gtt()
2743 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_global_gtt_cleanup()  local
2744 	struct i915_address_space *vm = &dev_priv->gtt.base;  in i915_global_gtt_cleanup()
2746 	if (dev_priv->mm.aliasing_ppgtt) {  in i915_global_gtt_cleanup()
2747 		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;  in i915_global_gtt_cleanup()
2844 	struct drm_i915_private *dev_priv = dev->dev_private;  in ggtt_probe_common()  local
2860 		dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size);  in ggtt_probe_common()
2862 		dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);  in ggtt_probe_common()
2863 	if (!dev_priv->gtt.gsm) {  in ggtt_probe_common()
2872 		iounmap(dev_priv->gtt.gsm);  in ggtt_probe_common()
2876 	dev_priv->gtt.base.scratch_page = scratch_page;  in ggtt_probe_common()
2884 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)  in bdw_setup_private_ppat()  argument
2897 	if (!USES_PPGTT(dev_priv->dev))  in bdw_setup_private_ppat()
2919 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)  in chv_setup_private_ppat()  argument
2960 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen8_gmch_probe()  local
2988 		chv_setup_private_ppat(dev_priv);  in gen8_gmch_probe()
2990 		bdw_setup_private_ppat(dev_priv);  in gen8_gmch_probe()
2994 	dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;  in gen8_gmch_probe()
2995 	dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;  in gen8_gmch_probe()
2996 	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;  in gen8_gmch_probe()
2997 	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;  in gen8_gmch_probe()
3008 	struct drm_i915_private *dev_priv = dev->dev_private;  in gen6_gmch_probe()  local
3021 			  dev_priv->gtt.mappable_end);  in gen6_gmch_probe()
3036 	dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;  in gen6_gmch_probe()
3037 	dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;  in gen6_gmch_probe()
3038 	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;  in gen6_gmch_probe()
3039 	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;  in gen6_gmch_probe()
3059 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gmch_probe()  local
3062 	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);  in i915_gmch_probe()
3070 	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);  in i915_gmch_probe()
3071 	dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;  in i915_gmch_probe()
3072 	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;  in i915_gmch_probe()
3073 	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;  in i915_gmch_probe()
3074 	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;  in i915_gmch_probe()
3076 	if (unlikely(dev_priv->gtt.do_idle_maps))  in i915_gmch_probe()
3089 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_gtt_init()  local
3090 	struct i915_gtt *gtt = &dev_priv->gtt;  in i915_gem_gtt_init()
3099 		if (IS_HASWELL(dev) && dev_priv->ellc_size)  in i915_gem_gtt_init()
3110 		dev_priv->gtt.gtt_probe = gen8_gmch_probe;  in i915_gem_gtt_init()
3111 		dev_priv->gtt.base.cleanup = gen6_gmch_remove;  in i915_gem_gtt_init()
3144 	struct drm_i915_private *dev_priv = dev->dev_private;  in i915_gem_restore_gtt_mappings()  local
3153 	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,  in i915_gem_restore_gtt_mappings()
3154 				       dev_priv->gtt.base.start,  in i915_gem_restore_gtt_mappings()
3155 				       dev_priv->gtt.base.total,  in i915_gem_restore_gtt_mappings()
3159 	vm = &dev_priv->gtt.base;  in i915_gem_restore_gtt_mappings()
3160 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {  in i915_gem_restore_gtt_mappings()
3178 			chv_setup_private_ppat(dev_priv);  in i915_gem_restore_gtt_mappings()
3180 			bdw_setup_private_ppat(dev_priv);  in i915_gem_restore_gtt_mappings()
3186 		list_for_each_entry(vm, &dev_priv->vm_list, global_link) {  in i915_gem_restore_gtt_mappings()
3194 				ppgtt = dev_priv->mm.aliasing_ppgtt;  in i915_gem_restore_gtt_mappings()
3196 			gen6_write_page_range(dev_priv, &ppgtt->pd,  in i915_gem_restore_gtt_mappings()
3201 	i915_ggtt_flush(dev_priv);  in i915_gem_restore_gtt_mappings()