529 		      struct i915_page_directory_pointer *pdp)  in __pdp_init()  argument
533 	pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),  in __pdp_init()
536 	if (!pdp->used_pdpes)  in __pdp_init()
539 	pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),  in __pdp_init()
541 	if (!pdp->page_directory) {  in __pdp_init()
542 		kfree(pdp->used_pdpes);  in __pdp_init()
545 		pdp->used_pdpes = NULL;  in __pdp_init()
552 static void __pdp_fini(struct i915_page_directory_pointer *pdp)  in __pdp_fini()  argument
554 	kfree(pdp->used_pdpes);  in __pdp_fini()
555 	kfree(pdp->page_directory);  in __pdp_fini()
556 	pdp->page_directory = NULL;  in __pdp_fini()
562 	struct i915_page_directory_pointer *pdp;  in alloc_pdp()  local
567 	pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);  in alloc_pdp()
568 	if (!pdp)  in alloc_pdp()
571 	ret = __pdp_init(dev, pdp);  in alloc_pdp()
575 	ret = setup_px(dev, pdp);  in alloc_pdp()
579 	return pdp;  in alloc_pdp()
582 	__pdp_fini(pdp);  in alloc_pdp()
584 	kfree(pdp);  in alloc_pdp()
590 		     struct i915_page_directory_pointer *pdp)  in free_pdp()  argument
592 	__pdp_fini(pdp);  in free_pdp()
594 		cleanup_px(dev, pdp);  in free_pdp()
595 		kfree(pdp);  in free_pdp()
600 				struct i915_page_directory_pointer *pdp)  in gen8_initialize_pdp()  argument
606 	fill_px(vm->dev, pdp, scratch_pdpe);  in gen8_initialize_pdp()
622 			  struct i915_page_directory_pointer *pdp,  in gen8_setup_page_directory()  argument
631 	page_directorypo = kmap_px(pdp);  in gen8_setup_page_directory()
639 				  struct i915_page_directory_pointer *pdp,  in gen8_setup_page_directory_pointer()  argument
645 	pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);  in gen8_setup_page_directory_pointer()
697 				       struct i915_page_directory_pointer *pdp,  in gen8_ppgtt_clear_pte_range()  argument
711 	if (WARN_ON(!pdp))  in gen8_ppgtt_clear_pte_range()
718 		if (WARN_ON(!pdp->page_directory[pdpe]))  in gen8_ppgtt_clear_pte_range()
721 		pd = pdp->page_directory[pdpe];  in gen8_ppgtt_clear_pte_range()
764 		gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,  in gen8_ppgtt_clear_range()
768 		struct i915_page_directory_pointer *pdp;  in gen8_ppgtt_clear_range()  local
770 		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {  in gen8_ppgtt_clear_range()
771 			gen8_ppgtt_clear_pte_range(vm, pdp, start, length,  in gen8_ppgtt_clear_range()
779 			      struct i915_page_directory_pointer *pdp,  in gen8_ppgtt_insert_pte_entries()  argument
795 			struct i915_page_directory *pd = pdp->page_directory[pdpe];  in gen8_ppgtt_insert_pte_entries()
832 		gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,  in gen8_ppgtt_insert_entries()
835 		struct i915_page_directory_pointer *pdp;  in gen8_ppgtt_insert_entries()  local
839 		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {  in gen8_ppgtt_insert_entries()
840 			gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,  in gen8_ppgtt_insert_entries()
949 				    struct i915_page_directory_pointer *pdp)  in gen8_ppgtt_cleanup_3lvl()  argument
953 	for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) {  in gen8_ppgtt_cleanup_3lvl()
954 		if (WARN_ON(!pdp->page_directory[i]))  in gen8_ppgtt_cleanup_3lvl()
957 		gen8_free_page_tables(dev, pdp->page_directory[i]);  in gen8_ppgtt_cleanup_3lvl()
958 		free_pd(dev, pdp->page_directory[i]);  in gen8_ppgtt_cleanup_3lvl()
961 	free_pdp(dev, pdp);  in gen8_ppgtt_cleanup_3lvl()
987 		gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp);  in gen8_ppgtt_cleanup()
1075 				  struct i915_page_directory_pointer *pdp,  in gen8_ppgtt_alloc_page_directories()  argument
1088 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_ppgtt_alloc_page_directories()
1089 		if (test_bit(pdpe, pdp->used_pdpes))  in gen8_ppgtt_alloc_page_directories()
1097 		pdp->page_directory[pdpe] = pd;  in gen8_ppgtt_alloc_page_directories()
1106 		free_pd(dev, pdp->page_directory[pdpe]);  in gen8_ppgtt_alloc_page_directories()
1135 	struct i915_page_directory_pointer *pdp;  in gen8_ppgtt_alloc_page_dirpointers()  local
1141 	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {  in gen8_ppgtt_alloc_page_dirpointers()
1143 			pdp = alloc_pdp(dev);  in gen8_ppgtt_alloc_page_dirpointers()
1144 			if (IS_ERR(pdp))  in gen8_ppgtt_alloc_page_dirpointers()
1147 			gen8_initialize_pdp(vm, pdp);  in gen8_ppgtt_alloc_page_dirpointers()
1148 			pml4->pdps[pml4e] = pdp;  in gen8_ppgtt_alloc_page_dirpointers()
1214 				    struct i915_page_directory_pointer *pdp,  in gen8_alloc_va_range_3lvl()  argument
1244 	ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,  in gen8_alloc_va_range_3lvl()
1252 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_alloc_va_range_3lvl()
1264 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_alloc_va_range_3lvl()
1301 		__set_bit(pdpe, pdp->used_pdpes);  in gen8_alloc_va_range_3lvl()
1302 		gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);  in gen8_alloc_va_range_3lvl()
1313 			free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);  in gen8_alloc_va_range_3lvl()
1317 		free_pd(dev, pdp->page_directory[pdpe]);  in gen8_alloc_va_range_3lvl()
1332 	struct i915_page_directory_pointer *pdp;  in gen8_alloc_va_range_4lvl()  local
1352 	gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {  in gen8_alloc_va_range_4lvl()
1353 		WARN_ON(!pdp);  in gen8_alloc_va_range_4lvl()
1355 		ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);  in gen8_alloc_va_range_4lvl()
1359 		gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);  in gen8_alloc_va_range_4lvl()
1383 		return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);  in gen8_alloc_va_range()
1386 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,  in gen8_dump_pdp()  argument
1395 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_dump_pdp()
1401 		if (!test_bit(pdpe, pdp->used_pdpes))  in gen8_dump_pdp()
1453 		gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);  in gen8_dump_ppgtt()
1457 		struct i915_page_directory_pointer *pdp;  in gen8_dump_ppgtt()  local
1459 		gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) {  in gen8_dump_ppgtt()
1464 			gen8_dump_pdp(pdp, start, length, scratch_pte, m);  in gen8_dump_ppgtt()
1485 	ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,  in gen8_preallocate_top_level_pdps()
1489 		*ppgtt->pdp.used_pdpes = *new_page_dirs;  in gen8_preallocate_top_level_pdps()
1530 		ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp);  in gen8_ppgtt_init()