464 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,  in gen8_mm_switch()  argument
470 	int used_pd = ppgtt->num_pd_entries / I915_PDES;  in gen8_mm_switch()
473 		dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr;  in gen8_mm_switch()
487 	struct i915_hw_ppgtt *ppgtt =  in gen8_ppgtt_clear_range()  local
496 	scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,  in gen8_ppgtt_clear_range()
504 		if (WARN_ON(!ppgtt->pdp.page_directory[pdpe]))  in gen8_ppgtt_clear_range()
507 		pd = ppgtt->pdp.page_directory[pdpe];  in gen8_ppgtt_clear_range()
530 		if (!HAS_LLC(ppgtt->base.dev))  in gen8_ppgtt_clear_range()
547 	struct i915_hw_ppgtt *ppgtt =  in gen8_ppgtt_insert_entries()  local
562 			struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe];  in gen8_ppgtt_insert_entries()
573 			if (!HAS_LLC(ppgtt->base.dev))  in gen8_ppgtt_insert_entries()
585 		if (!HAS_LLC(ppgtt->base.dev))  in gen8_ppgtt_insert_entries()
607 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)  in gen8_ppgtt_free()  argument
611 	for (i = 0; i < ppgtt->num_pd_pages; i++) {  in gen8_ppgtt_free()
612 		if (WARN_ON(!ppgtt->pdp.page_directory[i]))  in gen8_ppgtt_free()
615 		gen8_free_page_tables(ppgtt->pdp.page_directory[i], ppgtt->base.dev);  in gen8_ppgtt_free()
616 		unmap_and_free_pd(ppgtt->pdp.page_directory[i]);  in gen8_ppgtt_free()
620 static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)  in gen8_ppgtt_unmap_pages()  argument
622 	struct pci_dev *hwdev = ppgtt->base.dev->pdev;  in gen8_ppgtt_unmap_pages()
625 	for (i = 0; i < ppgtt->num_pd_pages; i++) {  in gen8_ppgtt_unmap_pages()
628 		if (!ppgtt->pdp.page_directory[i]->daddr)  in gen8_ppgtt_unmap_pages()
631 		pci_unmap_page(hwdev, ppgtt->pdp.page_directory[i]->daddr, PAGE_SIZE,  in gen8_ppgtt_unmap_pages()
635 			struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];  in gen8_ppgtt_unmap_pages()
654 	struct i915_hw_ppgtt *ppgtt =  in gen8_ppgtt_cleanup()  local
657 	gen8_ppgtt_unmap_pages(ppgtt);  in gen8_ppgtt_cleanup()
658 	gen8_ppgtt_free(ppgtt);  in gen8_ppgtt_cleanup()
661 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)  in gen8_ppgtt_allocate_page_tables()  argument
665 	for (i = 0; i < ppgtt->num_pd_pages; i++) {  in gen8_ppgtt_allocate_page_tables()
666 		ret = alloc_pt_range(ppgtt->pdp.page_directory[i],  in gen8_ppgtt_allocate_page_tables()
667 				     0, I915_PDES, ppgtt->base.dev);  in gen8_ppgtt_allocate_page_tables()
676 		gen8_free_page_tables(ppgtt->pdp.page_directory[i], ppgtt->base.dev);  in gen8_ppgtt_allocate_page_tables()
681 static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,  in gen8_ppgtt_allocate_page_directories()  argument
687 		ppgtt->pdp.page_directory[i] = alloc_pd_single();  in gen8_ppgtt_allocate_page_directories()
688 		if (IS_ERR(ppgtt->pdp.page_directory[i]))  in gen8_ppgtt_allocate_page_directories()
692 	ppgtt->num_pd_pages = max_pdp;  in gen8_ppgtt_allocate_page_directories()
693 	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPES);  in gen8_ppgtt_allocate_page_directories()
699 		unmap_and_free_pd(ppgtt->pdp.page_directory[i]);  in gen8_ppgtt_allocate_page_directories()
704 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,  in gen8_ppgtt_alloc()  argument
709 	ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);  in gen8_ppgtt_alloc()
713 	ret = gen8_ppgtt_allocate_page_tables(ppgtt);  in gen8_ppgtt_alloc()
717 	ppgtt->num_pd_entries = max_pdp * I915_PDES;  in gen8_ppgtt_alloc()
722 	gen8_ppgtt_free(ppgtt);  in gen8_ppgtt_alloc()
726 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,  in gen8_ppgtt_setup_page_directories()  argument
732 	pd_addr = pci_map_page(ppgtt->base.dev->pdev,  in gen8_ppgtt_setup_page_directories()
733 			       ppgtt->pdp.page_directory[pd]->page, 0,  in gen8_ppgtt_setup_page_directories()
736 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);  in gen8_ppgtt_setup_page_directories()
740 	ppgtt->pdp.page_directory[pd]->daddr = pd_addr;  in gen8_ppgtt_setup_page_directories()
745 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,  in gen8_ppgtt_setup_page_tables()  argument
750 	struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd];  in gen8_ppgtt_setup_page_tables()
755 	pt_addr = pci_map_page(ppgtt->base.dev->pdev,  in gen8_ppgtt_setup_page_tables()
757 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);  in gen8_ppgtt_setup_page_tables()
776 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)  in gen8_ppgtt_init()  argument
790 	ret = gen8_ppgtt_alloc(ppgtt, GEN8_LEGACY_PDPES);  in gen8_ppgtt_init()
798 		ret = gen8_ppgtt_setup_page_directories(ppgtt, i);  in gen8_ppgtt_init()
803 			ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);  in gen8_ppgtt_init()
818 		struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];  in gen8_ppgtt_init()
820 		pd_vaddr = kmap_atomic(ppgtt->pdp.page_directory[i]->page);  in gen8_ppgtt_init()
824 			pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,  in gen8_ppgtt_init()
827 		if (!HAS_LLC(ppgtt->base.dev))  in gen8_ppgtt_init()
832 	ppgtt->switch_mm = gen8_mm_switch;  in gen8_ppgtt_init()
833 	ppgtt->base.clear_range = gen8_ppgtt_clear_range;  in gen8_ppgtt_init()
834 	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;  in gen8_ppgtt_init()
835 	ppgtt->base.cleanup = gen8_ppgtt_cleanup;  in gen8_ppgtt_init()
836 	ppgtt->base.start = 0;  in gen8_ppgtt_init()
839 	ppgtt->base.total = max_pdp * I915_PDES * GEN8_PTES * PAGE_SIZE;  in gen8_ppgtt_init()
842 	ppgtt->base.clear_range(&ppgtt->base, 0,  in gen8_ppgtt_init()
843 				ppgtt->num_pd_pages * GEN8_PTES * PAGE_SIZE,  in gen8_ppgtt_init()
847 			 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);  in gen8_ppgtt_init()
849 			 ppgtt->num_pd_entries,  in gen8_ppgtt_init()
850 			 (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));  in gen8_ppgtt_init()
854 	gen8_ppgtt_unmap_pages(ppgtt);  in gen8_ppgtt_init()
855 	gen8_ppgtt_free(ppgtt);  in gen8_ppgtt_init()
859 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)  in gen6_dump_ppgtt()  argument
861 	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;  in gen6_dump_ppgtt()
862 	struct i915_address_space *vm = &ppgtt->base;  in gen6_dump_ppgtt()
871 		ppgtt->pd.pd_offset / sizeof(gen6_pte_t);  in gen6_dump_ppgtt()
874 		   ppgtt->pd.pd_offset,  in gen6_dump_ppgtt()
875 		   ppgtt->pd.pd_offset + ppgtt->num_pd_entries);  in gen6_dump_ppgtt()
876 	for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {  in gen6_dump_ppgtt()
879 		dma_addr_t pt_addr = ppgtt->pd.page_table[pde]->daddr;  in gen6_dump_ppgtt()
890 		pt_vaddr = kmap_atomic(ppgtt->pd.page_table[pde]->page);  in gen6_dump_ppgtt()
921 	struct i915_hw_ppgtt *ppgtt =  in gen6_write_pde()  local
928 	writel(pd_entry, ppgtt->pd_addr + pde);  in gen6_write_pde()
948 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)  in get_pd_offset()  argument
950 	BUG_ON(ppgtt->pd.pd_offset & 0x3f);  in get_pd_offset()
952 	return (ppgtt->pd.pd_offset / 64) << 16;  in get_pd_offset()
955 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,  in hsw_mm_switch()  argument
973 	intel_ring_emit(ring, get_pd_offset(ppgtt));  in hsw_mm_switch()
980 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,  in vgpu_mm_switch()  argument
983 	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);  in vgpu_mm_switch()
986 	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));  in vgpu_mm_switch()
990 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,  in gen7_mm_switch()  argument
1008 	intel_ring_emit(ring, get_pd_offset(ppgtt));  in gen7_mm_switch()
1022 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,  in gen6_mm_switch()  argument
1025 	struct drm_device *dev = ppgtt->base.dev;  in gen6_mm_switch()
1030 	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));  in gen6_mm_switch()
1099 	struct i915_hw_ppgtt *ppgtt =  in gen6_ppgtt_clear_range()  local
1115 		pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);  in gen6_ppgtt_clear_range()
1133 	struct i915_hw_ppgtt *ppgtt =  in gen6_ppgtt_insert_entries()  local
1144 			pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);  in gen6_ppgtt_insert_entries()
1165 static inline void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)  in mark_tlbs_dirty()  argument
1168 	ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;  in mark_tlbs_dirty()
1196 	struct i915_hw_ppgtt *ppgtt =  in gen6_alloc_va_range()  local
1212 	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {  in gen6_alloc_va_range()
1213 		if (pt != ppgtt->scratch_pt) {  in gen6_alloc_va_range()
1229 		ppgtt->pd.page_table[pde] = pt;  in gen6_alloc_va_range()
1237 	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {  in gen6_alloc_va_range()
1245 			gen6_write_pde(&ppgtt->pd, pde, pt);  in gen6_alloc_va_range()
1261 	mark_tlbs_dirty(ppgtt);  in gen6_alloc_va_range()
1266 		struct i915_page_table_entry *pt = ppgtt->pd.page_table[pde];  in gen6_alloc_va_range()
1268 		ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;  in gen6_alloc_va_range()
1272 	mark_tlbs_dirty(ppgtt);  in gen6_alloc_va_range()
1276 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)  in gen6_ppgtt_free()  argument
1280 	for (i = 0; i < ppgtt->num_pd_entries; i++) {  in gen6_ppgtt_free()
1281 		struct i915_page_table_entry *pt = ppgtt->pd.page_table[i];  in gen6_ppgtt_free()
1283 		if (pt != ppgtt->scratch_pt)  in gen6_ppgtt_free()
1284 			unmap_and_free_pt(ppgtt->pd.page_table[i], ppgtt->base.dev);  in gen6_ppgtt_free()
1287 	unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);  in gen6_ppgtt_free()
1288 	unmap_and_free_pd(&ppgtt->pd);  in gen6_ppgtt_free()
1293 	struct i915_hw_ppgtt *ppgtt =  in gen6_ppgtt_cleanup()  local
1296 	drm_mm_remove_node(&ppgtt->node);  in gen6_ppgtt_cleanup()
1298 	gen6_ppgtt_free(ppgtt);  in gen6_ppgtt_cleanup()
1301 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)  in gen6_ppgtt_allocate_page_directories()  argument
1303 	struct drm_device *dev = ppgtt->base.dev;  in gen6_ppgtt_allocate_page_directories()
1313 	ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev);  in gen6_ppgtt_allocate_page_directories()
1314 	if (IS_ERR(ppgtt->scratch_pt))  in gen6_ppgtt_allocate_page_directories()
1315 		return PTR_ERR(ppgtt->scratch_pt);  in gen6_ppgtt_allocate_page_directories()
1317 	gen6_initialize_pt(&ppgtt->base, ppgtt->scratch_pt);  in gen6_ppgtt_allocate_page_directories()
1321 						  &ppgtt->node, GEN6_PD_SIZE,  in gen6_ppgtt_allocate_page_directories()
1342 	if (ppgtt->node.start < dev_priv->gtt.mappable_end)  in gen6_ppgtt_allocate_page_directories()
1345 	ppgtt->num_pd_entries = I915_PDES;  in gen6_ppgtt_allocate_page_directories()
1349 	unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);  in gen6_ppgtt_allocate_page_directories()
1353 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)  in gen6_ppgtt_alloc()  argument
1355 	return gen6_ppgtt_allocate_page_directories(ppgtt);  in gen6_ppgtt_alloc()
1358 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,  in gen6_scratch_va_range()  argument
1364 	gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)  in gen6_scratch_va_range()
1365 		ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;  in gen6_scratch_va_range()
1368 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)  in gen6_ppgtt_init()  argument
1370 	struct drm_device *dev = ppgtt->base.dev;  in gen6_ppgtt_init()
1374 	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;  in gen6_ppgtt_init()
1376 		ppgtt->switch_mm = gen6_mm_switch;  in gen6_ppgtt_init()
1378 		ppgtt->switch_mm = hsw_mm_switch;  in gen6_ppgtt_init()
1380 		ppgtt->switch_mm = gen7_mm_switch;  in gen6_ppgtt_init()
1385 		ppgtt->switch_mm = vgpu_mm_switch;  in gen6_ppgtt_init()
1387 	ret = gen6_ppgtt_alloc(ppgtt);  in gen6_ppgtt_init()
1393 		ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries,  in gen6_ppgtt_init()
1394 				ppgtt->base.dev);  in gen6_ppgtt_init()
1397 			gen6_ppgtt_cleanup(&ppgtt->base);  in gen6_ppgtt_init()
1402 	ppgtt->base.allocate_va_range = gen6_alloc_va_range;  in gen6_ppgtt_init()
1403 	ppgtt->base.clear_range = gen6_ppgtt_clear_range;  in gen6_ppgtt_init()
1404 	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;  in gen6_ppgtt_init()
1405 	ppgtt->base.cleanup = gen6_ppgtt_cleanup;  in gen6_ppgtt_init()
1406 	ppgtt->base.start = 0;  in gen6_ppgtt_init()
1407 	ppgtt->base.total = ppgtt->num_pd_entries * GEN6_PTES * PAGE_SIZE;  in gen6_ppgtt_init()
1408 	ppgtt->debug_dump = gen6_dump_ppgtt;  in gen6_ppgtt_init()
1410 	ppgtt->pd.pd_offset =  in gen6_ppgtt_init()
1411 		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);  in gen6_ppgtt_init()
1413 	ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +  in gen6_ppgtt_init()
1414 		ppgtt->pd.pd_offset / sizeof(gen6_pte_t);  in gen6_ppgtt_init()
1417 		ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);  in gen6_ppgtt_init()
1419 		gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);  in gen6_ppgtt_init()
1421 	gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);  in gen6_ppgtt_init()
1424 			 ppgtt->node.size >> 20,  in gen6_ppgtt_init()
1425 			 ppgtt->node.start / PAGE_SIZE);  in gen6_ppgtt_init()
1428 		  ppgtt->pd.pd_offset << 10);  in gen6_ppgtt_init()
1433 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,  in __hw_ppgtt_init()  argument
1438 	ppgtt->base.dev = dev;  in __hw_ppgtt_init()
1439 	ppgtt->base.scratch = dev_priv->gtt.base.scratch;  in __hw_ppgtt_init()
1442 		return gen6_ppgtt_init(ppgtt, aliasing);  in __hw_ppgtt_init()
1444 		return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);  in __hw_ppgtt_init()
1446 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)  in i915_ppgtt_init()  argument
1451 	ret = __hw_ppgtt_init(dev, ppgtt, false);  in i915_ppgtt_init()
1453 		kref_init(&ppgtt->ref);  in i915_ppgtt_init()
1454 		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,  in i915_ppgtt_init()
1455 			    ppgtt->base.total);  in i915_ppgtt_init()
1456 		i915_init_vm(dev_priv, &ppgtt->base);  in i915_ppgtt_init()
1466 	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;  in i915_ppgtt_init_hw()  local
1487 	if (ppgtt) {  in i915_ppgtt_init_hw()
1489 			ret = ppgtt->switch_mm(ppgtt, ring);  in i915_ppgtt_init_hw()
1500 	struct i915_hw_ppgtt *ppgtt;  in i915_ppgtt_create()  local
1503 	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);  in i915_ppgtt_create()
1504 	if (!ppgtt)  in i915_ppgtt_create()
1507 	ret = i915_ppgtt_init(dev, ppgtt);  in i915_ppgtt_create()
1509 		kfree(ppgtt);  in i915_ppgtt_create()
1513 	ppgtt->file_priv = fpriv;  in i915_ppgtt_create()
1515 	trace_i915_ppgtt_create(&ppgtt->base);  in i915_ppgtt_create()
1517 	return ppgtt;  in i915_ppgtt_create()
1522 	struct i915_hw_ppgtt *ppgtt =  in i915_ppgtt_release()  local
1525 	trace_i915_ppgtt_release(&ppgtt->base);  in i915_ppgtt_release()
1528 	WARN_ON(!list_empty(&ppgtt->base.active_list));  in i915_ppgtt_release()
1529 	WARN_ON(!list_empty(&ppgtt->base.inactive_list));  in i915_ppgtt_release()
1531 	list_del(&ppgtt->base.global_link);  in i915_ppgtt_release()
1532 	drm_mm_takedown(&ppgtt->base.mm);  in i915_ppgtt_release()
1534 	ppgtt->base.cleanup(&ppgtt->base);  in i915_ppgtt_release()
1535 	kfree(ppgtt);  in i915_ppgtt_release()
1702 			struct i915_hw_ppgtt *ppgtt =  in i915_gem_restore_gtt_mappings()  local
1707 				ppgtt = dev_priv->mm.aliasing_ppgtt;  in i915_gem_restore_gtt_mappings()
1709 			gen6_write_page_range(dev_priv, &ppgtt->pd,  in i915_gem_restore_gtt_mappings()
1710 					      0, ppgtt->base.total);  in i915_gem_restore_gtt_mappings()
2080 		struct i915_hw_ppgtt *ppgtt;  in i915_gem_setup_global_gtt()  local
2082 		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);  in i915_gem_setup_global_gtt()
2083 		if (!ppgtt)  in i915_gem_setup_global_gtt()
2086 		ret = __hw_ppgtt_init(dev, ppgtt, true);  in i915_gem_setup_global_gtt()
2088 			kfree(ppgtt);  in i915_gem_setup_global_gtt()
2092 		dev_priv->mm.aliasing_ppgtt = ppgtt;  in i915_gem_setup_global_gtt()
2115 		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;  in i915_global_gtt_cleanup()  local
2117 		ppgtt->base.cleanup(&ppgtt->base);  in i915_global_gtt_cleanup()