42 static int global_invalidates(struct kvm *kvm, unsigned long flags)  in global_invalidates()  argument
53 	if (kvm->arch.online_vcores == 1 && local_paca->kvm_hstate.kvm_vcpu)  in global_invalidates()
61 		cpumask_setall(&kvm->arch.need_tlb_flush);  in global_invalidates()
63 				  &kvm->arch.need_tlb_flush);  in global_invalidates()
73 void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev,  in kvmppc_add_revmap_chain()  argument
81 		head = &kvm->arch.revmap[i];  in kvmppc_add_revmap_chain()
84 		tail = &kvm->arch.revmap[head->back];  in kvmppc_add_revmap_chain()
101 static void remove_revmap_chain(struct kvm *kvm, long pte_index,  in remove_revmap_chain()  argument
114 	memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);  in remove_revmap_chain()
122 	next = real_vmalloc_addr(&kvm->arch.revmap[rev->forw]);  in remove_revmap_chain()
123 	prev = real_vmalloc_addr(&kvm->arch.revmap[rev->back]);  in remove_revmap_chain()
137 long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,  in kvmppc_do_h_enter()  argument
164 	mmu_seq = kvm->mmu_notifier_seq;  in kvmppc_do_h_enter()
170 	memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);  in kvmppc_do_h_enter()
252 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_do_h_enter()
256 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_do_h_enter()
287 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_do_h_enter()
304 	rev = &kvm->arch.revmap[pte_index];  in kvmppc_do_h_enter()
309 		note_hpte_modification(kvm, rev);  in kvmppc_do_h_enter()
318 		if (mmu_notifier_retry(kvm, mmu_seq)) {  in kvmppc_do_h_enter()
324 			kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index,  in kvmppc_do_h_enter()
347 	return kvmppc_do_h_enter(vcpu->kvm, flags, pte_index, pteh, ptel,  in kvmppc_h_enter()
375 static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,  in do_tlbies()  argument
381 		while (!try_lock_tlbie(&kvm->arch.tlbie_lock))  in do_tlbies()
387 				     "r" (rbvalues[i]), "r" (kvm->arch.lpid));  in do_tlbies()
389 		kvm->arch.tlbie_lock = 0;  in do_tlbies()
399 long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,  in kvmppc_do_h_remove()  argument
408 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_do_h_remove()
410 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_do_h_remove()
421 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_do_h_remove()
426 		do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);  in kvmppc_do_h_remove()
436 		remove_revmap_chain(kvm, pte_index, rev, v,  in kvmppc_do_h_remove()
440 	note_hpte_modification(kvm, rev);  in kvmppc_do_h_remove()
452 	return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn,  in kvmppc_h_remove()
458 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_bulk_remove()  local
469 	global = global_invalidates(kvm, 0);  in kvmppc_h_bulk_remove()
484 			    pte_index >= kvm->arch.hpt_npte) {  in kvmppc_h_bulk_remove()
490 			hp = (__be64 *) (kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_bulk_remove()
522 			rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_bulk_remove()
523 			note_hpte_modification(kvm, rev);  in kvmppc_h_bulk_remove()
547 		do_tlbies(kvm, tlbrb, n, global, true);  in kvmppc_h_bulk_remove()
555 			remove_revmap_chain(kvm, pte_index, rev,  in kvmppc_h_bulk_remove()
570 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_protect()  local
576 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_h_protect()
579 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_protect()
597 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_protect()
601 		note_hpte_modification(kvm, rev);  in kvmppc_h_protect()
620 			do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags),  in kvmppc_h_protect()
633 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_read()  local
639 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_h_read()
645 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_read()
647 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_read()
664 void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,  in kvmppc_invalidate_hpte()  argument
672 	do_tlbies(kvm, &rb, 1, 1, true);  in kvmppc_invalidate_hpte()
676 void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,  in kvmppc_clear_ref_hpte()  argument
687 	do_tlbies(kvm, &rb, 1, 1, false);  in kvmppc_clear_ref_hpte()
702 long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,  in kvmppc_hv_find_lock_hpte()  argument
731 	hash = (vsid ^ ((eaddr & somask) >> pshift)) & kvm->arch.hpt_mask;  in kvmppc_hv_find_lock_hpte()
742 		hpte = (__be64 *)(kvm->arch.hpt_virt + (hash << 7));  in kvmppc_hv_find_lock_hpte()
772 		hash = hash ^ kvm->arch.hpt_mask;  in kvmppc_hv_find_lock_hpte()
792 	struct kvm *kvm = vcpu->kvm;  in kvmppc_hpte_hv_fault()  local
805 	index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid);  in kvmppc_hpte_hv_fault()
811 	hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4));  in kvmppc_hpte_hv_fault()
814 	rev = real_vmalloc_addr(&kvm->arch.revmap[index]);  in kvmppc_hpte_hv_fault()