386 static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, size_t count,  in alloc_pt_range()  argument
402 		WARN(pd->page_table[i],  in alloc_pt_range()
404 		     i, pd->page_table[i]);  in alloc_pt_range()
405 		pd->page_table[i] = pt;  in alloc_pt_range()
412 		unmap_and_free_pt(pd->page_table[i], dev);  in alloc_pt_range()
416 static void unmap_and_free_pd(struct i915_page_directory_entry *pd)  in unmap_and_free_pd()  argument
418 	if (pd->page) {  in unmap_and_free_pd()
419 		__free_page(pd->page);  in unmap_and_free_pd()
420 		kfree(pd);  in unmap_and_free_pd()
426 	struct i915_page_directory_entry *pd;  in alloc_pd_single()  local
428 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);  in alloc_pd_single()
429 	if (!pd)  in alloc_pd_single()
432 	pd->page = alloc_page(GFP_KERNEL | __GFP_ZERO);  in alloc_pd_single()
433 	if (!pd->page) {  in alloc_pd_single()
434 		kfree(pd);  in alloc_pd_single()
438 	return pd;  in alloc_pd_single()
500 		struct i915_page_directory_entry *pd;  in gen8_ppgtt_clear_range()  local
507 		pd = ppgtt->pdp.page_directory[pdpe];  in gen8_ppgtt_clear_range()
509 		if (WARN_ON(!pd->page_table[pde]))  in gen8_ppgtt_clear_range()
512 		pt = pd->page_table[pde];  in gen8_ppgtt_clear_range()
562 			struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe];  in gen8_ppgtt_insert_entries()  local
563 			struct i915_page_table_entry *pt = pd->page_table[pde];  in gen8_ppgtt_insert_entries()
591 static void gen8_free_page_tables(struct i915_page_directory_entry *pd, struct drm_device *dev)  in gen8_free_page_tables()  argument
595 	if (!pd->page)  in gen8_free_page_tables()
599 		if (WARN_ON(!pd->page_table[i]))  in gen8_free_page_tables()
602 		unmap_and_free_pt(pd->page_table[i], dev);  in gen8_free_page_tables()
603 		pd->page_table[i] = NULL;  in gen8_free_page_tables()
635 			struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];  in gen8_ppgtt_unmap_pages()  local
639 			if (WARN_ON(!pd->page_table[j]))  in gen8_ppgtt_unmap_pages()
642 			pt = pd->page_table[j];  in gen8_ppgtt_unmap_pages()
727 					     const int pd)  in gen8_ppgtt_setup_page_directories()  argument
733 			       ppgtt->pdp.page_directory[pd]->page, 0,  in gen8_ppgtt_setup_page_directories()
740 	ppgtt->pdp.page_directory[pd]->daddr = pd_addr;  in gen8_ppgtt_setup_page_directories()
746 					const int pd,  in gen8_ppgtt_setup_page_tables()  argument
750 	struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd];  in gen8_ppgtt_setup_page_tables()
818 		struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];  in gen8_ppgtt_init()  local
822 			struct i915_page_table_entry *pt = pd->page_table[j];  in gen8_ppgtt_init()
871 		ppgtt->pd.pd_offset / sizeof(gen6_pte_t);  in gen6_dump_ppgtt()
874 		   ppgtt->pd.pd_offset,  in gen6_dump_ppgtt()
875 		   ppgtt->pd.pd_offset + ppgtt->num_pd_entries);  in gen6_dump_ppgtt()
879 		dma_addr_t pt_addr = ppgtt->pd.page_table[pde]->daddr;  in gen6_dump_ppgtt()
890 		pt_vaddr = kmap_atomic(ppgtt->pd.page_table[pde]->page);  in gen6_dump_ppgtt()
917 static void gen6_write_pde(struct i915_page_directory_entry *pd,  in gen6_write_pde()  argument
922 		container_of(pd, struct i915_hw_ppgtt, pd);  in gen6_write_pde()
934 				  struct i915_page_directory_entry *pd,  in gen6_write_page_range()  argument
940 	gen6_for_each_pde(pt, pd, start, length, temp, pde)  in gen6_write_page_range()
941 		gen6_write_pde(pd, pde, pt);  in gen6_write_page_range()
950 	BUG_ON(ppgtt->pd.pd_offset & 0x3f);  in get_pd_offset()
952 	return (ppgtt->pd.pd_offset / 64) << 16;  in get_pd_offset()
1115 		pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);  in gen6_ppgtt_clear_range()
1144 			pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);  in gen6_ppgtt_insert_entries()
1212 	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {  in gen6_alloc_va_range()
1229 		ppgtt->pd.page_table[pde] = pt;  in gen6_alloc_va_range()
1237 	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {  in gen6_alloc_va_range()
1245 			gen6_write_pde(&ppgtt->pd, pde, pt);  in gen6_alloc_va_range()
1266 		struct i915_page_table_entry *pt = ppgtt->pd.page_table[pde];  in gen6_alloc_va_range()
1268 		ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;  in gen6_alloc_va_range()
1281 		struct i915_page_table_entry *pt = ppgtt->pd.page_table[i];  in gen6_ppgtt_free()
1284 			unmap_and_free_pt(ppgtt->pd.page_table[i], ppgtt->base.dev);  in gen6_ppgtt_free()
1288 	unmap_and_free_pd(&ppgtt->pd);  in gen6_ppgtt_free()
1364 	gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)  in gen6_scratch_va_range()
1365 		ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;  in gen6_scratch_va_range()
1393 		ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries,  in gen6_ppgtt_init()
1410 	ppgtt->pd.pd_offset =  in gen6_ppgtt_init()
1414 		ppgtt->pd.pd_offset / sizeof(gen6_pte_t);  in gen6_ppgtt_init()
1421 	gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);  in gen6_ppgtt_init()
1428 		  ppgtt->pd.pd_offset << 10);  in gen6_ppgtt_init()
1709 			gen6_write_page_range(dev_priv, &ppgtt->pd,  in i915_gem_restore_gtt_mappings()