91 static void kvm_flush_dcache_pmd(pmd_t pmd)  in kvm_flush_dcache_pmd()  argument
93 	__kvm_flush_dcache_pmd(pmd);  in kvm_flush_dcache_pmd()
115 static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)  in stage2_dissolve_pmd()  argument
117 	if (!kvm_pmd_huge(*pmd))  in stage2_dissolve_pmd()
120 	pmd_clear(pmd);  in stage2_dissolve_pmd()
122 	put_page(virt_to_page(pmd));  in stage2_dissolve_pmd()
176 static void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)  in clear_pmd_entry()  argument
178 	pte_t *pte_table = pte_offset_kernel(pmd, 0);  in clear_pmd_entry()
179 	VM_BUG_ON(kvm_pmd_huge(*pmd));  in clear_pmd_entry()
180 	pmd_clear(pmd);  in clear_pmd_entry()
183 	put_page(virt_to_page(pmd));  in clear_pmd_entry()
206 static void unmap_ptes(struct kvm *kvm, pmd_t *pmd,  in unmap_ptes()  argument
212 	start_pte = pte = pte_offset_kernel(pmd, addr);  in unmap_ptes()
229 		clear_pmd_entry(kvm, pmd, start_addr);  in unmap_ptes()
236 	pmd_t *pmd, *start_pmd;  in unmap_pmds()  local
238 	start_pmd = pmd = pmd_offset(pud, addr);  in unmap_pmds()
241 		if (!pmd_none(*pmd)) {  in unmap_pmds()
242 			if (kvm_pmd_huge(*pmd)) {  in unmap_pmds()
243 				pmd_t old_pmd = *pmd;  in unmap_pmds()
245 				pmd_clear(pmd);  in unmap_pmds()
250 				put_page(virt_to_page(pmd));  in unmap_pmds()
252 				unmap_ptes(kvm, pmd, addr, next);  in unmap_pmds()
255 	} while (pmd++, addr = next, addr != end);  in unmap_pmds()
306 static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,  in stage2_flush_ptes()  argument
311 	pte = pte_offset_kernel(pmd, addr);  in stage2_flush_ptes()
321 	pmd_t *pmd;  in stage2_flush_pmds()  local
324 	pmd = pmd_offset(pud, addr);  in stage2_flush_pmds()
327 		if (!pmd_none(*pmd)) {  in stage2_flush_pmds()
328 			if (kvm_pmd_huge(*pmd))  in stage2_flush_pmds()
329 				kvm_flush_dcache_pmd(*pmd);  in stage2_flush_pmds()
331 				stage2_flush_ptes(kvm, pmd, addr, next);  in stage2_flush_pmds()
333 	} while (pmd++, addr = next, addr != end);  in stage2_flush_pmds()
451 static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,  in create_hyp_pte_mappings()  argument
460 		pte = pte_offset_kernel(pmd, addr);  in create_hyp_pte_mappings()
472 	pmd_t *pmd;  in create_hyp_pmd_mappings()  local
478 		pmd = pmd_offset(pud, addr);  in create_hyp_pmd_mappings()
480 		BUG_ON(pmd_sect(*pmd));  in create_hyp_pmd_mappings()
482 		if (pmd_none(*pmd)) {  in create_hyp_pmd_mappings()
488 			pmd_populate_kernel(NULL, pmd, pte);  in create_hyp_pmd_mappings()
489 			get_page(virt_to_page(pmd));  in create_hyp_pmd_mappings()
490 			kvm_flush_dcache_to_poc(pmd, sizeof(*pmd));  in create_hyp_pmd_mappings()
495 		create_hyp_pte_mappings(pmd, addr, next, pfn, prot);  in create_hyp_pmd_mappings()
507 	pmd_t *pmd;  in create_hyp_pud_mappings()  local
516 			pmd = pmd_alloc_one(NULL, addr);  in create_hyp_pud_mappings()
517 			if (!pmd) {  in create_hyp_pud_mappings()
521 			pud_populate(NULL, pud, pmd);  in create_hyp_pud_mappings()
855 	pmd_t *pmd;  in stage2_get_pmd()  local
861 		pmd = mmu_memory_cache_alloc(cache);  in stage2_get_pmd()
862 		pud_populate(NULL, pud, pmd);  in stage2_get_pmd()
872 	pmd_t *pmd, old_pmd;  in stage2_set_pmd_huge()  local
874 	pmd = stage2_get_pmd(kvm, cache, addr);  in stage2_set_pmd_huge()
875 	VM_BUG_ON(!pmd);  in stage2_set_pmd_huge()
886 	VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd));  in stage2_set_pmd_huge()
888 	old_pmd = *pmd;  in stage2_set_pmd_huge()
890 		pmd_clear(pmd);  in stage2_set_pmd_huge()
893 		get_page(virt_to_page(pmd));  in stage2_set_pmd_huge()
896 	kvm_set_pmd(pmd, *new_pmd);  in stage2_set_pmd_huge()
904 	pmd_t *pmd;  in stage2_set_pte()  local
912 	pmd = stage2_get_pmd(kvm, cache, addr);  in stage2_set_pte()
913 	if (!pmd) {  in stage2_set_pte()
926 		stage2_dissolve_pmd(kvm, addr, pmd);  in stage2_set_pte()
929 	if (pmd_none(*pmd)) {  in stage2_set_pte()
934 		pmd_populate_kernel(NULL, pmd, pte);  in stage2_set_pte()
935 		get_page(virt_to_page(pmd));  in stage2_set_pte()
938 	pte = pte_offset_kernel(pmd, addr);  in stage2_set_pte()
1055 static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)  in stage2_wp_ptes()  argument
1059 	pte = pte_offset_kernel(pmd, addr);  in stage2_wp_ptes()
1076 	pmd_t *pmd;  in stage2_wp_pmds()  local
1079 	pmd = pmd_offset(pud, addr);  in stage2_wp_pmds()
1083 		if (!pmd_none(*pmd)) {  in stage2_wp_pmds()
1084 			if (kvm_pmd_huge(*pmd)) {  in stage2_wp_pmds()
1085 				if (!kvm_s2pmd_readonly(pmd))  in stage2_wp_pmds()
1086 					kvm_set_s2pmd_readonly(pmd);  in stage2_wp_pmds()
1088 				stage2_wp_ptes(pmd, addr, next);  in stage2_wp_pmds()
1091 	} while (pmd++, addr = next, addr != end);  in stage2_wp_pmds()
1351 	pmd_t *pmd;  in handle_access_fault()  local
1360 	pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);  in handle_access_fault()
1361 	if (!pmd || pmd_none(*pmd))	/* Nothing there */  in handle_access_fault()
1364 	if (kvm_pmd_huge(*pmd)) {	/* THP, HugeTLB */  in handle_access_fault()
1365 		*pmd = pmd_mkyoung(*pmd);  in handle_access_fault()
1366 		pfn = pmd_pfn(*pmd);  in handle_access_fault()
1371 	pte = pte_offset_kernel(pmd, fault_ipa);  in handle_access_fault()
1565 	pmd_t *pmd;  in kvm_age_hva_handler()  local
1568 	pmd = stage2_get_pmd(kvm, NULL, gpa);  in kvm_age_hva_handler()
1569 	if (!pmd || pmd_none(*pmd))	/* Nothing there */  in kvm_age_hva_handler()
1572 	if (kvm_pmd_huge(*pmd)) {	/* THP, HugeTLB */  in kvm_age_hva_handler()
1573 		if (pmd_young(*pmd)) {  in kvm_age_hva_handler()
1574 			*pmd = pmd_mkold(*pmd);  in kvm_age_hva_handler()
1581 	pte = pte_offset_kernel(pmd, gpa);  in kvm_age_hva_handler()
1595 	pmd_t *pmd;  in kvm_test_age_hva_handler()  local
1598 	pmd = stage2_get_pmd(kvm, NULL, gpa);  in kvm_test_age_hva_handler()
1599 	if (!pmd || pmd_none(*pmd))	/* Nothing there */  in kvm_test_age_hva_handler()
1602 	if (kvm_pmd_huge(*pmd))		/* THP, HugeTLB */  in kvm_test_age_hva_handler()
1603 		return pmd_young(*pmd);  in kvm_test_age_hva_handler()
1605 	pte = pte_offset_kernel(pmd, gpa);  in kvm_test_age_hva_handler()