46 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
49 static void kvmppc_rmap_reset(struct kvm *kvm);
51 long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp)  in kvmppc_alloc_hpt()  argument
64 	kvm->arch.hpt_cma_alloc = 0;  in kvmppc_alloc_hpt()
69 		kvm->arch.hpt_cma_alloc = 1;  in kvmppc_alloc_hpt()
83 	kvm->arch.hpt_virt = hpt;  in kvmppc_alloc_hpt()
84 	kvm->arch.hpt_order = order;  in kvmppc_alloc_hpt()
86 	kvm->arch.hpt_npte = 1ul << (order - 4);  in kvmppc_alloc_hpt()
88 	kvm->arch.hpt_mask = (1ul << (order - 7)) - 1;  in kvmppc_alloc_hpt()
91 	rev = vmalloc(sizeof(struct revmap_entry) * kvm->arch.hpt_npte);  in kvmppc_alloc_hpt()
96 	kvm->arch.revmap = rev;  in kvmppc_alloc_hpt()
97 	kvm->arch.sdr1 = __pa(hpt) | (order - 18);  in kvmppc_alloc_hpt()
100 		hpt, order, kvm->arch.lpid);  in kvmppc_alloc_hpt()
107 	if (kvm->arch.hpt_cma_alloc)  in kvmppc_alloc_hpt()
114 long kvmppc_alloc_reset_hpt(struct kvm *kvm, u32 *htab_orderp)  in kvmppc_alloc_reset_hpt()  argument
119 	mutex_lock(&kvm->lock);  in kvmppc_alloc_reset_hpt()
120 	if (kvm->arch.hpte_setup_done) {  in kvmppc_alloc_reset_hpt()
121 		kvm->arch.hpte_setup_done = 0;  in kvmppc_alloc_reset_hpt()
124 		if (atomic_read(&kvm->arch.vcpus_running)) {  in kvmppc_alloc_reset_hpt()
125 			kvm->arch.hpte_setup_done = 1;  in kvmppc_alloc_reset_hpt()
129 	if (kvm->arch.hpt_virt) {  in kvmppc_alloc_reset_hpt()
130 		order = kvm->arch.hpt_order;  in kvmppc_alloc_reset_hpt()
132 		memset((void *)kvm->arch.hpt_virt, 0, 1ul << order);  in kvmppc_alloc_reset_hpt()
136 		kvmppc_rmap_reset(kvm);  in kvmppc_alloc_reset_hpt()
138 		cpumask_setall(&kvm->arch.need_tlb_flush);  in kvmppc_alloc_reset_hpt()
142 		err = kvmppc_alloc_hpt(kvm, htab_orderp);  in kvmppc_alloc_reset_hpt()
146 	mutex_unlock(&kvm->lock);  in kvmppc_alloc_reset_hpt()
150 void kvmppc_free_hpt(struct kvm *kvm)  in kvmppc_free_hpt()  argument
152 	kvmppc_free_lpid(kvm->arch.lpid);  in kvmppc_free_hpt()
153 	vfree(kvm->arch.revmap);  in kvmppc_free_hpt()
154 	if (kvm->arch.hpt_cma_alloc)  in kvmppc_free_hpt()
155 		kvm_release_hpt(virt_to_page(kvm->arch.hpt_virt),  in kvmppc_free_hpt()
156 				1 << (kvm->arch.hpt_order - PAGE_SHIFT));  in kvmppc_free_hpt()
158 		free_pages(kvm->arch.hpt_virt,  in kvmppc_free_hpt()
159 			   kvm->arch.hpt_order - PAGE_SHIFT);  in kvmppc_free_hpt()
185 	struct kvm *kvm = vcpu->kvm;  in kvmppc_map_vrma()  local
194 	if (npages > kvm->arch.hpt_mask + 1)  in kvmppc_map_vrma()
195 		npages = kvm->arch.hpt_mask + 1;  in kvmppc_map_vrma()
205 		hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25))) & kvm->arch.hpt_mask;  in kvmppc_map_vrma()
215 		ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,  in kvmppc_map_vrma()
257 long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,  in kvmppc_virtmode_do_h_enter()  argument
265 	ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,  in kvmppc_virtmode_do_h_enter()
310 	struct kvm *kvm = vcpu->kvm;  in kvmppc_mmu_book3s_64_hv_xlate()  local
327 		slb_v = vcpu->kvm->arch.vrma_slb_v;  in kvmppc_mmu_book3s_64_hv_xlate()
332 	index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,  in kvmppc_mmu_book3s_64_hv_xlate()
338 	hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));  in kvmppc_mmu_book3s_64_hv_xlate()
340 	gr = kvm->arch.revmap[index].guest_rpte;  in kvmppc_mmu_book3s_64_hv_xlate()
438 	struct kvm *kvm = vcpu->kvm;  in kvmppc_book3s_hv_page_fault()  local
463 	hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));  in kvmppc_book3s_hv_page_fault()
464 	rev = &kvm->arch.revmap[index];  in kvmppc_book3s_hv_page_fault()
484 	memslot = gfn_to_memslot(kvm, gfn);  in kvmppc_book3s_hv_page_fault()
501 	mmu_seq = kvm->mmu_notifier_seq;  in kvmppc_book3s_hv_page_fault()
598 	if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {  in kvmppc_book3s_hv_page_fault()
611 		kvmppc_invalidate_hpte(kvm, hptep, index);  in kvmppc_book3s_hv_page_fault()
615 		kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);  in kvmppc_book3s_hv_page_fault()
646 static void kvmppc_rmap_reset(struct kvm *kvm)  in kvmppc_rmap_reset()  argument
652 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_rmap_reset()
653 	slots = kvm->memslots;  in kvmppc_rmap_reset()
662 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_rmap_reset()
665 static int kvm_handle_hva_range(struct kvm *kvm,  in kvm_handle_hva_range()  argument
668 				int (*handler)(struct kvm *kvm,  in kvm_handle_hva_range()  argument
677 	slots = kvm_memslots(kvm);  in kvm_handle_hva_range()
697 			ret = handler(kvm, &memslot->arch.rmap[gfn_offset], gfn);  in kvm_handle_hva_range()
705 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,  in kvm_handle_hva()  argument
706 			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,  in kvm_handle_hva()  argument
709 	return kvm_handle_hva_range(kvm, hva, hva + 1, handler);  in kvm_handle_hva()
712 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,  in kvm_unmap_rmapp()  argument
715 	struct revmap_entry *rev = kvm->arch.revmap;  in kvm_unmap_rmapp()
733 		hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));  in kvm_unmap_rmapp()
760 			kvmppc_invalidate_hpte(kvm, hptep, i);  in kvm_unmap_rmapp()
766 				note_hpte_modification(kvm, &rev[i]);  in kvm_unmap_rmapp()
775 int kvm_unmap_hva_hv(struct kvm *kvm, unsigned long hva)  in kvm_unmap_hva_hv()  argument
777 	kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);  in kvm_unmap_hva_hv()
781 int kvm_unmap_hva_range_hv(struct kvm *kvm, unsigned long start, unsigned long end)  in kvm_unmap_hva_range_hv()  argument
783 	kvm_handle_hva_range(kvm, start, end, kvm_unmap_rmapp);  in kvm_unmap_hva_range_hv()
787 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,  in kvmppc_core_flush_memslot_hv()  argument
804 			kvm_unmap_rmapp(kvm, rmapp, gfn);  in kvmppc_core_flush_memslot_hv()
810 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,  in kvm_age_rmapp()  argument
813 	struct revmap_entry *rev = kvm->arch.revmap;  in kvm_age_rmapp()
831 		hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));  in kvm_age_rmapp()
849 			kvmppc_clear_ref_hpte(kvm, hptep, i);  in kvm_age_rmapp()
852 				note_hpte_modification(kvm, &rev[i]);  in kvm_age_rmapp()
863 int kvm_age_hva_hv(struct kvm *kvm, unsigned long start, unsigned long end)  in kvm_age_hva_hv()  argument
865 	return kvm_handle_hva_range(kvm, start, end, kvm_age_rmapp);  in kvm_age_hva_hv()
868 static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,  in kvm_test_age_rmapp()  argument
871 	struct revmap_entry *rev = kvm->arch.revmap;  in kvm_test_age_rmapp()
886 			hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4));  in kvm_test_age_rmapp()
899 int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva)  in kvm_test_age_hva_hv()  argument
901 	return kvm_handle_hva(kvm, hva, kvm_test_age_rmapp);  in kvm_test_age_hva_hv()
904 void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte)  in kvm_set_spte_hva_hv()  argument
906 	kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);  in kvm_set_spte_hva_hv()
909 static int vcpus_running(struct kvm *kvm)  in vcpus_running()  argument
911 	return atomic_read(&kvm->arch.vcpus_running) != 0;  in vcpus_running()
918 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)  in kvm_test_clear_dirty_npages()  argument
920 	struct revmap_entry *rev = kvm->arch.revmap;  in kvm_test_clear_dirty_npages()
941 		hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));  in kvm_test_clear_dirty_npages()
960 		    (!hpte_is_writable(hptep1) || vcpus_running(kvm)))  in kvm_test_clear_dirty_npages()
979 		kvmppc_invalidate_hpte(kvm, hptep, i);  in kvm_test_clear_dirty_npages()
986 				note_hpte_modification(kvm, &rev[i]);  in kvm_test_clear_dirty_npages()
1021 long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot,  in kvmppc_hv_get_dirty_log()  argument
1031 		int npages = kvm_test_clear_dirty_npages(kvm, rmapp);  in kvmppc_hv_get_dirty_log()
1045 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmppc_hv_get_dirty_log()
1055 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,  in kvmppc_pin_guest_page()  argument
1065 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_pin_guest_page()
1066 	memslot = gfn_to_memslot(kvm, gfn);  in kvmppc_pin_guest_page()
1074 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_pin_guest_page()
1082 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_pin_guest_page()
1086 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,  in kvmppc_unpin_guest_page()  argument
1102 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_unpin_guest_page()
1103 	memslot = gfn_to_memslot(kvm, gfn);  in kvmppc_unpin_guest_page()
1110 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_unpin_guest_page()
1132 	struct kvm	*kvm;  member
1230 	struct kvm *kvm = ctx->kvm;  in kvm_htab_read()  local
1248 	hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));  in kvm_htab_read()
1249 	revp = kvm->arch.revmap + i;  in kvm_htab_read()
1264 			while (i < kvm->arch.hpt_npte &&  in kvm_htab_read()
1274 		while (i < kvm->arch.hpt_npte &&  in kvm_htab_read()
1290 		while (i < kvm->arch.hpt_npte &&  in kvm_htab_read()
1311 		if (i >= kvm->arch.hpt_npte) {  in kvm_htab_read()
1327 	struct kvm *kvm = ctx->kvm;  in kvm_htab_write()  local
1342 	mutex_lock(&kvm->lock);  in kvm_htab_write()
1343 	hpte_setup = kvm->arch.hpte_setup_done;  in kvm_htab_write()
1345 		kvm->arch.hpte_setup_done = 0;	/* temporarily */  in kvm_htab_write()
1348 		if (atomic_read(&kvm->arch.vcpus_running)) {  in kvm_htab_write()
1349 			kvm->arch.hpte_setup_done = 1;  in kvm_htab_write()
1350 			mutex_unlock(&kvm->lock);  in kvm_htab_write()
1370 		if (i >= kvm->arch.hpt_npte ||  in kvm_htab_write()
1371 		    i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte)  in kvm_htab_write()
1374 		hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));  in kvm_htab_write()
1393 				kvmppc_do_h_remove(kvm, 0, i, 0, tmp);  in kvm_htab_write()
1395 			ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,  in kvm_htab_write()
1407 				kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |  in kvm_htab_write()
1410 				kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);  in kvm_htab_write()
1419 				kvmppc_do_h_remove(kvm, 0, i, 0, tmp);  in kvm_htab_write()
1429 	kvm->arch.hpte_setup_done = hpte_setup;  in kvm_htab_write()
1430 	mutex_unlock(&kvm->lock);  in kvm_htab_write()
1443 		atomic_dec(&ctx->kvm->arch.hpte_mod_interest);  in kvm_htab_release()
1444 	kvm_put_kvm(ctx->kvm);  in kvm_htab_release()
1456 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)  in kvm_vm_ioctl_get_htab_fd()  argument
1468 	kvm_get_kvm(kvm);  in kvm_vm_ioctl_get_htab_fd()
1469 	ctx->kvm = kvm;  in kvm_vm_ioctl_get_htab_fd()
1477 		kvm_put_kvm(kvm);  in kvm_vm_ioctl_get_htab_fd()
1482 		mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_get_htab_fd()
1483 		atomic_inc(&kvm->arch.hpte_mod_interest);  in kvm_vm_ioctl_get_htab_fd()
1485 		synchronize_srcu_expedited(&kvm->srcu);  in kvm_vm_ioctl_get_htab_fd()
1486 		mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_get_htab_fd()
1493 	struct kvm	*kvm;  member
1503 	struct kvm *kvm = inode->i_private;  in debugfs_htab_open()  local
1510 	kvm_get_kvm(kvm);  in debugfs_htab_open()
1511 	p->kvm = kvm;  in debugfs_htab_open()
1522 	kvm_put_kvm(p->kvm);  in debugfs_htab_release()
1534 	struct kvm *kvm;  in debugfs_htab_read()  local
1559 	kvm = p->kvm;  in debugfs_htab_read()
1561 	hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));  in debugfs_htab_read()
1562 	for (; len != 0 && i < kvm->arch.hpt_npte; ++i, hptp += 2) {  in debugfs_htab_read()
1572 		gr = kvm->arch.revmap[i].guest_rpte;  in debugfs_htab_read()
1620 void kvmppc_mmu_debugfs_init(struct kvm *kvm)  in kvmppc_mmu_debugfs_init()  argument
1622 	kvm->arch.htab_dentry = debugfs_create_file("htab", 0400,  in kvmppc_mmu_debugfs_init()
1623 						    kvm->arch.debugfs_dir, kvm,  in kvmppc_mmu_debugfs_init()