96 static void kvm_flush_dcache_pud(pud_t pud)  in kvm_flush_dcache_pud()  argument
98 	__kvm_flush_dcache_pud(pud);  in kvm_flush_dcache_pud()
166 static void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)  in clear_pud_entry()  argument
168 	pmd_t *pmd_table = pmd_offset(pud, 0);  in clear_pud_entry()
169 	VM_BUG_ON(pud_huge(*pud));  in clear_pud_entry()
170 	pud_clear(pud);  in clear_pud_entry()
173 	put_page(virt_to_page(pud));  in clear_pud_entry()
232 static void unmap_pmds(struct kvm *kvm, pud_t *pud,  in unmap_pmds()  argument
238 	start_pmd = pmd = pmd_offset(pud, addr);  in unmap_pmds()
258 		clear_pud_entry(kvm, pud, start_addr);  in unmap_pmds()
265 	pud_t *pud, *start_pud;  in unmap_puds()  local
267 	start_pud = pud = pud_offset(pgd, addr);  in unmap_puds()
270 		if (!pud_none(*pud)) {  in unmap_puds()
271 			if (pud_huge(*pud)) {  in unmap_puds()
272 				pud_t old_pud = *pud;  in unmap_puds()
274 				pud_clear(pud);  in unmap_puds()
279 				put_page(virt_to_page(pud));  in unmap_puds()
281 				unmap_pmds(kvm, pud, addr, next);  in unmap_puds()
284 	} while (pud++, addr = next, addr != end);  in unmap_puds()
318 static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,  in stage2_flush_pmds()  argument
324 	pmd = pmd_offset(pud, addr);  in stage2_flush_pmds()
339 	pud_t *pud;  in stage2_flush_puds()  local
342 	pud = pud_offset(pgd, addr);  in stage2_flush_puds()
345 		if (!pud_none(*pud)) {  in stage2_flush_puds()
346 			if (pud_huge(*pud))  in stage2_flush_puds()
347 				kvm_flush_dcache_pud(*pud);  in stage2_flush_puds()
349 				stage2_flush_pmds(kvm, pud, addr, next);  in stage2_flush_puds()
351 	} while (pud++, addr = next, addr != end);  in stage2_flush_puds()
468 static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,  in create_hyp_pmd_mappings()  argument
478 		pmd = pmd_offset(pud, addr);  in create_hyp_pmd_mappings()
506 	pud_t *pud;  in create_hyp_pud_mappings()  local
513 		pud = pud_offset(pgd, addr);  in create_hyp_pud_mappings()
515 		if (pud_none_or_clear_bad(pud)) {  in create_hyp_pud_mappings()
521 			pud_populate(NULL, pud, pmd);  in create_hyp_pud_mappings()
522 			get_page(virt_to_page(pud));  in create_hyp_pud_mappings()
523 			kvm_flush_dcache_to_poc(pud, sizeof(*pud));  in create_hyp_pud_mappings()
527 		ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);  in create_hyp_pud_mappings()
541 	pud_t *pud;  in __create_hyp_mappings()  local
552 			pud = pud_alloc_one(NULL, addr);  in __create_hyp_mappings()
553 			if (!pud) {  in __create_hyp_mappings()
558 			pgd_populate(NULL, pgd, pud);  in __create_hyp_mappings()
837 	pud_t *pud;  in stage2_get_pud()  local
843 		pud = mmu_memory_cache_alloc(cache);  in stage2_get_pud()
844 		pgd_populate(NULL, pgd, pud);  in stage2_get_pud()
854 	pud_t *pud;  in stage2_get_pmd()  local
857 	pud = stage2_get_pud(kvm, cache, addr);  in stage2_get_pmd()
858 	if (pud_none(*pud)) {  in stage2_get_pmd()
862 		pud_populate(NULL, pud, pmd);  in stage2_get_pmd()
863 		get_page(virt_to_page(pud));  in stage2_get_pmd()
866 	return pmd_offset(pud, addr);  in stage2_get_pmd()
1074 static void stage2_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)  in stage2_wp_pmds()  argument
1079 	pmd = pmd_offset(pud, addr);  in stage2_wp_pmds()
1104 	pud_t *pud;  in stage2_wp_puds()  local
1107 	pud = pud_offset(pgd, addr);  in stage2_wp_puds()
1110 		if (!pud_none(*pud)) {  in stage2_wp_puds()
1112 			BUG_ON(kvm_pud_huge(*pud));  in stage2_wp_puds()
1113 			stage2_wp_pmds(pud, addr, next);  in stage2_wp_puds()
1115 	} while (pud++, addr = next, addr != end);  in stage2_wp_puds()