64 void kvm_flush_remote_tlbs(struct kvm *kvm)  in kvm_flush_remote_tlbs()  argument
66 	kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);  in kvm_flush_remote_tlbs()
69 static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)  in kvm_tlb_flush_vmid_ipa()  argument
77 	if (kvm)  in kvm_tlb_flush_vmid_ipa()
78 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);  in kvm_tlb_flush_vmid_ipa()
115 static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)  in stage2_dissolve_pmd()  argument
121 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in stage2_dissolve_pmd()
157 static void clear_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)  in clear_pgd_entry()  argument
161 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in clear_pgd_entry()
166 static void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)  in clear_pud_entry()  argument
171 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in clear_pud_entry()
176 static void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)  in clear_pmd_entry()  argument
181 	kvm_tlb_flush_vmid_ipa(kvm, addr);  in clear_pmd_entry()
206 static void unmap_ptes(struct kvm *kvm, pmd_t *pmd,  in unmap_ptes()  argument
218 			kvm_tlb_flush_vmid_ipa(kvm, addr);  in unmap_ptes()
228 	if (kvm_pte_table_empty(kvm, start_pte))  in unmap_ptes()
229 		clear_pmd_entry(kvm, pmd, start_addr);  in unmap_ptes()
232 static void unmap_pmds(struct kvm *kvm, pud_t *pud,  in unmap_pmds()  argument
246 				kvm_tlb_flush_vmid_ipa(kvm, addr);  in unmap_pmds()
252 				unmap_ptes(kvm, pmd, addr, next);  in unmap_pmds()
257 	if (kvm_pmd_table_empty(kvm, start_pmd))  in unmap_pmds()
258 		clear_pud_entry(kvm, pud, start_addr);  in unmap_pmds()
261 static void unmap_puds(struct kvm *kvm, pgd_t *pgd,  in unmap_puds()  argument
275 				kvm_tlb_flush_vmid_ipa(kvm, addr);  in unmap_puds()
281 				unmap_pmds(kvm, pud, addr, next);  in unmap_puds()
286 	if (kvm_pud_table_empty(kvm, start_pud))  in unmap_puds()
287 		clear_pgd_entry(kvm, pgd, start_addr);  in unmap_puds()
291 static void unmap_range(struct kvm *kvm, pgd_t *pgdp,  in unmap_range()  argument
302 			unmap_puds(kvm, pgd, addr, next);  in unmap_range()
306 static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,  in stage2_flush_ptes()  argument
318 static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,  in stage2_flush_pmds()  argument
331 				stage2_flush_ptes(kvm, pmd, addr, next);  in stage2_flush_pmds()
336 static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,  in stage2_flush_puds()  argument
349 				stage2_flush_pmds(kvm, pud, addr, next);  in stage2_flush_puds()
354 static void stage2_flush_memslot(struct kvm *kvm,  in stage2_flush_memslot()  argument
362 	pgd = kvm->arch.pgd + kvm_pgd_index(addr);  in stage2_flush_memslot()
365 		stage2_flush_puds(kvm, pgd, addr, next);  in stage2_flush_memslot()
376 static void stage2_flush_vm(struct kvm *kvm)  in stage2_flush_vm()  argument
382 	idx = srcu_read_lock(&kvm->srcu);  in stage2_flush_vm()
383 	spin_lock(&kvm->mmu_lock);  in stage2_flush_vm()
385 	slots = kvm_memslots(kvm);  in stage2_flush_vm()
387 		stage2_flush_memslot(kvm, memslot);  in stage2_flush_vm()
389 	spin_unlock(&kvm->mmu_lock);  in stage2_flush_vm()
390 	srcu_read_unlock(&kvm->srcu, idx);  in stage2_flush_vm()
666 int kvm_alloc_stage2_pgd(struct kvm *kvm)  in kvm_alloc_stage2_pgd()  argument
671 	if (kvm->arch.pgd != NULL) {  in kvm_alloc_stage2_pgd()
724 	kvm->arch.pgd = pgd;  in kvm_alloc_stage2_pgd()
739 static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)  in unmap_stage2_range()  argument
741 	unmap_range(kvm, kvm->arch.pgd, start, size);  in unmap_stage2_range()
744 static void stage2_unmap_memslot(struct kvm *kvm,  in stage2_unmap_memslot()  argument
779 			unmap_stage2_range(kvm, gpa, vm_end - vm_start);  in stage2_unmap_memslot()
792 void stage2_unmap_vm(struct kvm *kvm)  in stage2_unmap_vm()  argument
798 	idx = srcu_read_lock(&kvm->srcu);  in stage2_unmap_vm()
799 	spin_lock(&kvm->mmu_lock);  in stage2_unmap_vm()
801 	slots = kvm_memslots(kvm);  in stage2_unmap_vm()
803 		stage2_unmap_memslot(kvm, memslot);  in stage2_unmap_vm()
805 	spin_unlock(&kvm->mmu_lock);  in stage2_unmap_vm()
806 	srcu_read_unlock(&kvm->srcu, idx);  in stage2_unmap_vm()
820 void kvm_free_stage2_pgd(struct kvm *kvm)  in kvm_free_stage2_pgd()  argument
822 	if (kvm->arch.pgd == NULL)  in kvm_free_stage2_pgd()
825 	unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);  in kvm_free_stage2_pgd()
826 	kvm_free_hwpgd(kvm_get_hwpgd(kvm));  in kvm_free_stage2_pgd()
828 		kfree(kvm->arch.pgd);  in kvm_free_stage2_pgd()
830 	kvm->arch.pgd = NULL;  in kvm_free_stage2_pgd()
833 static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,  in stage2_get_pud()  argument
839 	pgd = kvm->arch.pgd + kvm_pgd_index(addr);  in stage2_get_pud()
851 static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,  in stage2_get_pmd()  argument
857 	pud = stage2_get_pud(kvm, cache, addr);  in stage2_get_pmd()
869 static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache  in stage2_set_pmd_huge()  argument
874 	pmd = stage2_get_pmd(kvm, cache, addr);  in stage2_set_pmd_huge()
891 		kvm_tlb_flush_vmid_ipa(kvm, addr);  in stage2_set_pmd_huge()
900 static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,  in stage2_set_pte()  argument
912 	pmd = stage2_get_pmd(kvm, cache, addr);  in stage2_set_pte()
926 		stage2_dissolve_pmd(kvm, addr, pmd);  in stage2_set_pte()
947 		kvm_tlb_flush_vmid_ipa(kvm, addr);  in stage2_set_pte()
964 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,  in kvm_phys_addr_ioremap()  argument
985 		spin_lock(&kvm->mmu_lock);  in kvm_phys_addr_ioremap()
986 		ret = stage2_set_pte(kvm, &cache, addr, &pte,  in kvm_phys_addr_ioremap()
988 		spin_unlock(&kvm->mmu_lock);  in kvm_phys_addr_ioremap()
1124 static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)  in stage2_wp_range()  argument
1129 	pgd = kvm->arch.pgd + kvm_pgd_index(addr);  in stage2_wp_range()
1138 		if (need_resched() || spin_needbreak(&kvm->mmu_lock))  in stage2_wp_range()
1139 			cond_resched_lock(&kvm->mmu_lock);  in stage2_wp_range()
1160 void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)  in kvm_mmu_wp_memory_region()  argument
1162 	struct kvm_memslots *slots = kvm_memslots(kvm);  in kvm_mmu_wp_memory_region()
1167 	spin_lock(&kvm->mmu_lock);  in kvm_mmu_wp_memory_region()
1168 	stage2_wp_range(kvm, start, end);  in kvm_mmu_wp_memory_region()
1169 	spin_unlock(&kvm->mmu_lock);  in kvm_mmu_wp_memory_region()
1170 	kvm_flush_remote_tlbs(kvm);  in kvm_mmu_wp_memory_region()
1184 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,  in kvm_mmu_write_protect_pt_masked()  argument
1192 	stage2_wp_range(kvm, start, end);  in kvm_mmu_write_protect_pt_masked()
1202 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,  in kvm_arch_mmu_enable_log_dirty_pt_masked()  argument
1206 	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1223 	struct kvm *kvm = vcpu->kvm;  in user_mem_abort()  local
1272 	mmu_seq = vcpu->kvm->mmu_notifier_seq;  in user_mem_abort()
1284 	pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);  in user_mem_abort()
1308 	spin_lock(&kvm->mmu_lock);  in user_mem_abort()
1309 	if (mmu_notifier_retry(kvm, mmu_seq))  in user_mem_abort()
1325 		ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);  in user_mem_abort()
1332 			mark_page_dirty(kvm, gfn);  in user_mem_abort()
1335 		ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);  in user_mem_abort()
1339 	spin_unlock(&kvm->mmu_lock);  in user_mem_abort()
1359 	spin_lock(&vcpu->kvm->mmu_lock);  in handle_access_fault()
1361 	pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);  in handle_access_fault()
1380 	spin_unlock(&vcpu->kvm->mmu_lock);  in handle_access_fault()
1424 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_handle_guest_abort()
1427 	memslot = gfn_to_memslot(vcpu->kvm, gfn);  in kvm_handle_guest_abort()
1462 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_handle_guest_abort()
1466 static int handle_hva_to_gpa(struct kvm *kvm,  in handle_hva_to_gpa()  argument
1469 			     int (*handler)(struct kvm *kvm,  in handle_hva_to_gpa()  argument
1477 	slots = kvm_memslots(kvm);  in handle_hva_to_gpa()
1499 			ret |= handler(kvm, gpa, data);  in handle_hva_to_gpa()
1506 static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)  in kvm_unmap_hva_handler()  argument
1508 	unmap_stage2_range(kvm, gpa, PAGE_SIZE);  in kvm_unmap_hva_handler()
1512 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)  in kvm_unmap_hva()  argument
1516 	if (!kvm->arch.pgd)  in kvm_unmap_hva()
1520 	handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);  in kvm_unmap_hva()
1524 int kvm_unmap_hva_range(struct kvm *kvm,  in kvm_unmap_hva_range()  argument
1527 	if (!kvm->arch.pgd)  in kvm_unmap_hva_range()
1531 	handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);  in kvm_unmap_hva_range()
1535 static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)  in kvm_set_spte_handler()  argument
1546 	stage2_set_pte(kvm, NULL, gpa, pte, 0);  in kvm_set_spte_handler()
1551 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)  in kvm_set_spte_hva()  argument
1556 	if (!kvm->arch.pgd)  in kvm_set_spte_hva()
1561 	handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);  in kvm_set_spte_hva()
1564 static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)  in kvm_age_hva_handler()  argument
1569 	pmd = stage2_get_pmd(kvm, NULL, gpa);  in kvm_age_hva_handler()
1594 static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)  in kvm_test_age_hva_handler()  argument
1599 	pmd = stage2_get_pmd(kvm, NULL, gpa);  in kvm_test_age_hva_handler()
1613 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)  in kvm_age_hva()  argument
1616 	return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);  in kvm_age_hva()
1619 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)  in kvm_test_age_hva()  argument
1622 	return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);  in kvm_test_age_hva()
1725 void kvm_arch_commit_memory_region(struct kvm *kvm,  in kvm_arch_commit_memory_region()  argument
1737 		kvm_mmu_wp_memory_region(kvm, mem->slot);  in kvm_arch_commit_memory_region()
1740 int kvm_arch_prepare_memory_region(struct kvm *kvm,  in kvm_arch_prepare_memory_region()  argument
1808 			ret = kvm_phys_addr_ioremap(kvm, gpa, pa,  in kvm_arch_prepare_memory_region()
1820 	spin_lock(&kvm->mmu_lock);  in kvm_arch_prepare_memory_region()
1822 		unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);  in kvm_arch_prepare_memory_region()
1824 		stage2_flush_memslot(kvm, memslot);  in kvm_arch_prepare_memory_region()
1825 	spin_unlock(&kvm->mmu_lock);  in kvm_arch_prepare_memory_region()
1829 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,  in kvm_arch_free_memslot()  argument
1834 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,  in kvm_arch_create_memslot()  argument
1849 void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots)  in kvm_arch_memslots_updated()  argument
1853 void kvm_arch_flush_shadow_all(struct kvm *kvm)  in kvm_arch_flush_shadow_all()  argument
1857 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,  in kvm_arch_flush_shadow_memslot()  argument
1863 	spin_lock(&kvm->mmu_lock);  in kvm_arch_flush_shadow_memslot()
1864 	unmap_stage2_range(kvm, gpa, size);  in kvm_arch_flush_shadow_memslot()
1865 	spin_unlock(&kvm->mmu_lock);  in kvm_arch_flush_shadow_memslot()
1912 		stage2_flush_vm(vcpu->kvm);  in kvm_set_way_flush()
1927 		stage2_flush_vm(vcpu->kvm);  in kvm_toggle_cache()