115 static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)  in stage2_dissolve_pmd()  argument
121 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in stage2_dissolve_pmd()
157 static void clear_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)  in clear_pgd_entry()  argument
161 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in clear_pgd_entry()
166 static void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)  in clear_pud_entry()  argument
171 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in clear_pud_entry()
176 static void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)  in clear_pmd_entry()  argument
181 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in clear_pmd_entry()
207 		       phys_addr_t addr, phys_addr_t end)  in unmap_ptes()  argument
209 	phys_addr_t start_addr = addr;  in unmap_ptes()
212 	start_pte = pte = pte_offset_kernel(pmd, addr);  in unmap_ptes()
218 			kvm_tlb_flush_vmid_ipa(kvm, addr);  in unmap_ptes()
226 	} while (pte++, addr += PAGE_SIZE, addr != end);  in unmap_ptes()
233 		       phys_addr_t addr, phys_addr_t end)  in unmap_pmds()  argument
235 	phys_addr_t next, start_addr = addr;  in unmap_pmds()
238 	start_pmd = pmd = pmd_offset(pud, addr);  in unmap_pmds()
240 		next = kvm_pmd_addr_end(addr, end);  in unmap_pmds()
246 				kvm_tlb_flush_vmid_ipa(kvm, addr);  in unmap_pmds()
252 				unmap_ptes(kvm, pmd, addr, next);  in unmap_pmds()
255 	} while (pmd++, addr = next, addr != end);  in unmap_pmds()
262 		       phys_addr_t addr, phys_addr_t end)  in unmap_puds()  argument
264 	phys_addr_t next, start_addr = addr;  in unmap_puds()
267 	start_pud = pud = pud_offset(pgd, addr);  in unmap_puds()
269 		next = kvm_pud_addr_end(addr, end);  in unmap_puds()
275 				kvm_tlb_flush_vmid_ipa(kvm, addr);  in unmap_puds()
281 				unmap_pmds(kvm, pud, addr, next);  in unmap_puds()
284 	} while (pud++, addr = next, addr != end);  in unmap_puds()
295 	phys_addr_t addr = start, end = start + size;  in unmap_range()  local
298 	pgd = pgdp + kvm_pgd_index(addr);  in unmap_range()
300 		next = kvm_pgd_addr_end(addr, end);  in unmap_range()
302 			unmap_puds(kvm, pgd, addr, next);  in unmap_range()
303 	} while (pgd++, addr = next, addr != end);  in unmap_range()
307 			      phys_addr_t addr, phys_addr_t end)  in stage2_flush_ptes()  argument
311 	pte = pte_offset_kernel(pmd, addr);  in stage2_flush_ptes()
315 	} while (pte++, addr += PAGE_SIZE, addr != end);  in stage2_flush_ptes()
319 			      phys_addr_t addr, phys_addr_t end)  in stage2_flush_pmds()  argument
324 	pmd = pmd_offset(pud, addr);  in stage2_flush_pmds()
326 		next = kvm_pmd_addr_end(addr, end);  in stage2_flush_pmds()
331 				stage2_flush_ptes(kvm, pmd, addr, next);  in stage2_flush_pmds()
333 	} while (pmd++, addr = next, addr != end);  in stage2_flush_pmds()
337 			      phys_addr_t addr, phys_addr_t end)  in stage2_flush_puds()  argument
342 	pud = pud_offset(pgd, addr);  in stage2_flush_puds()
344 		next = kvm_pud_addr_end(addr, end);  in stage2_flush_puds()
349 				stage2_flush_pmds(kvm, pud, addr, next);  in stage2_flush_puds()
351 	} while (pud++, addr = next, addr != end);  in stage2_flush_puds()
357 	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;  in stage2_flush_memslot()  local
358 	phys_addr_t end = addr + PAGE_SIZE * memslot->npages;  in stage2_flush_memslot()
362 	pgd = kvm->arch.pgd + kvm_pgd_index(addr);  in stage2_flush_memslot()
364 		next = kvm_pgd_addr_end(addr, end);  in stage2_flush_memslot()
365 		stage2_flush_puds(kvm, pgd, addr, next);  in stage2_flush_memslot()
366 	} while (pgd++, addr = next, addr != end);  in stage2_flush_memslot()
427 	unsigned long addr;  in free_hyp_pgds()  local
434 		for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE)  in free_hyp_pgds()
435 			unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);  in free_hyp_pgds()
436 		for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE)  in free_hyp_pgds()
437 			unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);  in free_hyp_pgds()
456 	unsigned long addr;  in create_hyp_pte_mappings()  local
458 	addr = start;  in create_hyp_pte_mappings()
460 		pte = pte_offset_kernel(pmd, addr);  in create_hyp_pte_mappings()
465 	} while (addr += PAGE_SIZE, addr != end);  in create_hyp_pte_mappings()
474 	unsigned long addr, next;  in create_hyp_pmd_mappings()  local
476 	addr = start;  in create_hyp_pmd_mappings()
478 		pmd = pmd_offset(pud, addr);  in create_hyp_pmd_mappings()
483 			pte = pte_alloc_one_kernel(NULL, addr);  in create_hyp_pmd_mappings()
493 		next = pmd_addr_end(addr, end);  in create_hyp_pmd_mappings()
495 		create_hyp_pte_mappings(pmd, addr, next, pfn, prot);  in create_hyp_pmd_mappings()
496 		pfn += (next - addr) >> PAGE_SHIFT;  in create_hyp_pmd_mappings()
497 	} while (addr = next, addr != end);  in create_hyp_pmd_mappings()
508 	unsigned long addr, next;  in create_hyp_pud_mappings()  local
511 	addr = start;  in create_hyp_pud_mappings()
513 		pud = pud_offset(pgd, addr);  in create_hyp_pud_mappings()
516 			pmd = pmd_alloc_one(NULL, addr);  in create_hyp_pud_mappings()
526 		next = pud_addr_end(addr, end);  in create_hyp_pud_mappings()
527 		ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);  in create_hyp_pud_mappings()
530 		pfn += (next - addr) >> PAGE_SHIFT;  in create_hyp_pud_mappings()
531 	} while (addr = next, addr != end);  in create_hyp_pud_mappings()
542 	unsigned long addr, next;  in __create_hyp_mappings()  local
546 	addr = start & PAGE_MASK;  in __create_hyp_mappings()
549 		pgd = pgdp + pgd_index(addr);  in __create_hyp_mappings()
552 			pud = pud_alloc_one(NULL, addr);  in __create_hyp_mappings()
563 		next = pgd_addr_end(addr, end);  in __create_hyp_mappings()
564 		err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);  in __create_hyp_mappings()
567 		pfn += (next - addr) >> PAGE_SHIFT;  in __create_hyp_mappings()
568 	} while (addr = next, addr != end);  in __create_hyp_mappings()
748 	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;  in stage2_unmap_memslot()  local
778 			gpa_t gpa = addr + (vm_start - memslot->userspace_addr);  in stage2_unmap_memslot()
834 			     phys_addr_t addr)  in stage2_get_pud()  argument
839 	pgd = kvm->arch.pgd + kvm_pgd_index(addr);  in stage2_get_pud()
848 	return pud_offset(pgd, addr);  in stage2_get_pud()
852 			     phys_addr_t addr)  in stage2_get_pmd()  argument
857 	pud = stage2_get_pud(kvm, cache, addr);  in stage2_get_pmd()
866 	return pmd_offset(pud, addr);  in stage2_get_pmd()
870 			       *cache, phys_addr_t addr, const pmd_t *new_pmd)  in stage2_set_pmd_huge()  argument
874 	pmd = stage2_get_pmd(kvm, cache, addr);  in stage2_set_pmd_huge()
891 		kvm_tlb_flush_vmid_ipa(kvm, addr);  in stage2_set_pmd_huge()
901 			  phys_addr_t addr, const pte_t *new_pte,  in stage2_set_pte()  argument
912 	pmd = stage2_get_pmd(kvm, cache, addr);  in stage2_set_pte()
926 		stage2_dissolve_pmd(kvm, addr, pmd);  in stage2_set_pte()
938 	pte = pte_offset_kernel(pmd, addr);  in stage2_set_pte()
947 		kvm_tlb_flush_vmid_ipa(kvm, addr);  in stage2_set_pte()
967 	phys_addr_t addr, end;  in kvm_phys_addr_ioremap()  local
975 	for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {  in kvm_phys_addr_ioremap()
986 		ret = stage2_set_pte(kvm, &cache, addr, &pte,  in kvm_phys_addr_ioremap()
1055 static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)  in stage2_wp_ptes()  argument
1059 	pte = pte_offset_kernel(pmd, addr);  in stage2_wp_ptes()
1065 	} while (pte++, addr += PAGE_SIZE, addr != end);  in stage2_wp_ptes()
1074 static void stage2_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)  in stage2_wp_pmds()  argument
1079 	pmd = pmd_offset(pud, addr);  in stage2_wp_pmds()
1082 		next = kvm_pmd_addr_end(addr, end);  in stage2_wp_pmds()
1088 				stage2_wp_ptes(pmd, addr, next);  in stage2_wp_pmds()
1091 	} while (pmd++, addr = next, addr != end);  in stage2_wp_pmds()
1102 static void  stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)  in stage2_wp_puds()  argument
1107 	pud = pud_offset(pgd, addr);  in stage2_wp_puds()
1109 		next = kvm_pud_addr_end(addr, end);  in stage2_wp_puds()
1113 			stage2_wp_pmds(pud, addr, next);  in stage2_wp_puds()
1115 	} while (pud++, addr = next, addr != end);  in stage2_wp_puds()
1124 static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)  in stage2_wp_range()  argument
1129 	pgd = kvm->arch.pgd + kvm_pgd_index(addr);  in stage2_wp_range()
1141 		next = kvm_pgd_addr_end(addr, end);  in stage2_wp_range()
1143 			stage2_wp_puds(pgd, addr, next);  in stage2_wp_range()
1144 	} while (pgd++, addr = next, addr != end);  in stage2_wp_range()