483 	struct i915_page_directory *pd;  in alloc_pd()  local
486 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);  in alloc_pd()
487 	if (!pd)  in alloc_pd()
490 	pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),  in alloc_pd()
491 				sizeof(*pd->used_pdes), GFP_KERNEL);  in alloc_pd()
492 	if (!pd->used_pdes)  in alloc_pd()
495 	ret = setup_px(dev, pd);  in alloc_pd()
499 	return pd;  in alloc_pd()
502 	kfree(pd->used_pdes);  in alloc_pd()
504 	kfree(pd);  in alloc_pd()
509 static void free_pd(struct drm_device *dev, struct i915_page_directory *pd)  in free_pd()  argument
511 	if (px_page(pd)) {  in free_pd()
512 		cleanup_px(dev, pd);  in free_pd()
513 		kfree(pd->used_pdes);  in free_pd()
514 		kfree(pd);  in free_pd()
519 			       struct i915_page_directory *pd)  in gen8_initialize_pd()  argument
525 	fill_px(vm->dev, pd, scratch_pde);  in gen8_initialize_pd()
623 			  struct i915_page_directory *pd,  in gen8_setup_page_directory()  argument
632 	page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);  in gen8_setup_page_directory()
715 		struct i915_page_directory *pd;  in gen8_ppgtt_clear_pte_range()  local
721 		pd = pdp->page_directory[pdpe];  in gen8_ppgtt_clear_pte_range()
723 		if (WARN_ON(!pd->page_table[pde]))  in gen8_ppgtt_clear_pte_range()
726 		pt = pd->page_table[pde];  in gen8_ppgtt_clear_pte_range()
795 			struct i915_page_directory *pd = pdp->page_directory[pdpe];  in gen8_ppgtt_insert_pte_entries()  local
796 			struct i915_page_table *pt = pd->page_table[pde];  in gen8_ppgtt_insert_pte_entries()
847 				  struct i915_page_directory *pd)  in gen8_free_page_tables()  argument
851 	if (!px_page(pd))  in gen8_free_page_tables()
854 	for_each_set_bit(i, pd->used_pdes, I915_PDES) {  in gen8_free_page_tables()
855 		if (WARN_ON(!pd->page_table[i]))  in gen8_free_page_tables()
858 		free_pt(dev, pd->page_table[i]);  in gen8_free_page_tables()
859 		pd->page_table[i] = NULL;  in gen8_free_page_tables()
1013 				     struct i915_page_directory *pd,  in gen8_ppgtt_alloc_pagetabs()  argument
1023 	gen8_for_each_pde(pt, pd, start, length, temp, pde) {  in gen8_ppgtt_alloc_pagetabs()
1025 		if (test_bit(pde, pd->used_pdes)) {  in gen8_ppgtt_alloc_pagetabs()
1036 		pd->page_table[pde] = pt;  in gen8_ppgtt_alloc_pagetabs()
1045 		free_pt(dev, pd->page_table[pde]);  in gen8_ppgtt_alloc_pagetabs()
1081 	struct i915_page_directory *pd;  in gen8_ppgtt_alloc_page_directories()  local
1088 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_ppgtt_alloc_page_directories()
1092 		pd = alloc_pd(dev);  in gen8_ppgtt_alloc_page_directories()
1093 		if (IS_ERR(pd))  in gen8_ppgtt_alloc_page_directories()
1096 		gen8_initialize_pd(vm, pd);  in gen8_ppgtt_alloc_page_directories()
1097 		pdp->page_directory[pdpe] = pd;  in gen8_ppgtt_alloc_page_directories()
1222 	struct i915_page_directory *pd;  in gen8_alloc_va_range_3lvl()  local
1252 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_alloc_va_range_3lvl()
1253 		ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,  in gen8_alloc_va_range_3lvl()
1264 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_alloc_va_range_3lvl()
1265 		gen8_pde_t *const page_directory = kmap_px(pd);  in gen8_alloc_va_range_3lvl()
1272 		WARN_ON(!pd);  in gen8_alloc_va_range_3lvl()
1274 		gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {  in gen8_alloc_va_range_3lvl()
1286 			__set_bit(pde, pd->used_pdes);  in gen8_alloc_va_range_3lvl()
1302 		gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);  in gen8_alloc_va_range_3lvl()
1391 	struct i915_page_directory *pd;  in gen8_dump_pdp()  local
1395 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {  in gen8_dump_pdp()
1405 		gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {  in gen8_dump_pdp()
1409 			if (!test_bit(pde, pd->used_pdes))  in gen8_dump_pdp()
1569 	gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {  in gen6_dump_ppgtt()
1572 		const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);  in gen6_dump_ppgtt()
1583 		pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);  in gen6_dump_ppgtt()
1611 static void gen6_write_pde(struct i915_page_directory *pd,  in gen6_write_pde()  argument
1616 		container_of(pd, struct i915_hw_ppgtt, pd);  in gen6_write_pde()
1628 				  struct i915_page_directory *pd,  in gen6_write_page_range()  argument
1634 	gen6_for_each_pde(pt, pd, start, length, temp, pde)  in gen6_write_page_range()
1635 		gen6_write_pde(pd, pde, pt);  in gen6_write_page_range()
1644 	BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);  in get_pd_offset()
1646 	return (ppgtt->pd.base.ggtt_offset / 64) << 16;  in get_pd_offset()
1815 		pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);  in gen6_ppgtt_clear_range()
1844 			pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);  in gen6_ppgtt_insert_entries()
1887 	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {  in gen6_alloc_va_range()
1904 		ppgtt->pd.page_table[pde] = pt;  in gen6_alloc_va_range()
1912 	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {  in gen6_alloc_va_range()
1920 			gen6_write_pde(&ppgtt->pd, pde, pt);  in gen6_alloc_va_range()
1941 		struct i915_page_table *pt = ppgtt->pd.page_table[pde];  in gen6_alloc_va_range()
1943 		ppgtt->pd.page_table[pde] = vm->scratch_pt;  in gen6_alloc_va_range()
2057 	gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)  in gen6_scratch_va_range()
2058 		ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;  in gen6_scratch_va_range()
2094 	ppgtt->pd.base.ggtt_offset =  in gen6_ppgtt_init()
2098 		ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);  in gen6_ppgtt_init()
2102 	gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);  in gen6_ppgtt_init()
2109 		  ppgtt->pd.base.ggtt_offset << 10);  in gen6_ppgtt_init()
3196 			gen6_write_page_range(dev_priv, &ppgtt->pd,  in i915_gem_restore_gtt_mappings()