43 static int global_invalidates(struct kvm *kvm, unsigned long flags)  in global_invalidates()  argument
54 	if (kvm->arch.online_vcores == 1 && local_paca->kvm_hstate.kvm_vcpu)  in global_invalidates()
62 		cpumask_setall(&kvm->arch.need_tlb_flush);  in global_invalidates()
64 				  &kvm->arch.need_tlb_flush);  in global_invalidates()
74 void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev,  in kvmppc_add_revmap_chain()  argument
82 		head = &kvm->arch.revmap[i];  in kvmppc_add_revmap_chain()
85 		tail = &kvm->arch.revmap[head->back];  in kvmppc_add_revmap_chain()
116 static unsigned long *revmap_for_hpte(struct kvm *kvm, unsigned long hpte_v,  in revmap_for_hpte()  argument
124 	memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);  in revmap_for_hpte()
133 static void remove_revmap_chain(struct kvm *kvm, long pte_index,  in remove_revmap_chain()  argument
144 	rmap = revmap_for_hpte(kvm, hpte_v, ptel);  in remove_revmap_chain()
150 	next = real_vmalloc_addr(&kvm->arch.revmap[rev->forw]);  in remove_revmap_chain()
151 	prev = real_vmalloc_addr(&kvm->arch.revmap[rev->back]);  in remove_revmap_chain()
167 long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,  in kvmppc_do_h_enter()  argument
194 	mmu_seq = kvm->mmu_notifier_seq;  in kvmppc_do_h_enter()
200 	memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn);  in kvmppc_do_h_enter()
284 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_do_h_enter()
288 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_do_h_enter()
319 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_do_h_enter()
336 	rev = &kvm->arch.revmap[pte_index];  in kvmppc_do_h_enter()
341 		note_hpte_modification(kvm, rev);  in kvmppc_do_h_enter()
350 		if (mmu_notifier_retry(kvm, mmu_seq)) {  in kvmppc_do_h_enter()
356 			kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index,  in kvmppc_do_h_enter()
379 	return kvmppc_do_h_enter(vcpu->kvm, flags, pte_index, pteh, ptel,  in kvmppc_h_enter()
407 static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,  in do_tlbies()  argument
413 		while (!try_lock_tlbie(&kvm->arch.tlbie_lock))  in do_tlbies()
419 				     "r" (rbvalues[i]), "r" (kvm->arch.lpid));  in do_tlbies()
421 		kvm->arch.tlbie_lock = 0;  in do_tlbies()
431 long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,  in kvmppc_do_h_remove()  argument
440 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_do_h_remove()
442 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_do_h_remove()
453 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_do_h_remove()
458 		do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);  in kvmppc_do_h_remove()
468 		remove_revmap_chain(kvm, pte_index, rev, v,  in kvmppc_do_h_remove()
472 	note_hpte_modification(kvm, rev);  in kvmppc_do_h_remove()
486 	return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn,  in kvmppc_h_remove()
492 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_bulk_remove()  local
503 	global = global_invalidates(kvm, 0);  in kvmppc_h_bulk_remove()
518 			    pte_index >= kvm->arch.hpt_npte) {  in kvmppc_h_bulk_remove()
524 			hp = (__be64 *) (kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_bulk_remove()
556 			rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_bulk_remove()
557 			note_hpte_modification(kvm, rev);  in kvmppc_h_bulk_remove()
581 		do_tlbies(kvm, tlbrb, n, global, true);  in kvmppc_h_bulk_remove()
589 			remove_revmap_chain(kvm, pte_index, rev,  in kvmppc_h_bulk_remove()
604 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_protect()  local
610 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_h_protect()
613 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_protect()
631 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_protect()
635 		note_hpte_modification(kvm, rev);  in kvmppc_h_protect()
654 			do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags),  in kvmppc_h_protect()
667 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_read()  local
673 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_h_read()
679 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_read()
681 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_read()
701 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_clear_ref()  local
708 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_h_clear_ref()
711 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_clear_ref()
712 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_clear_ref()
723 		note_hpte_modification(kvm, rev);  in kvmppc_h_clear_ref()
728 			kvmppc_clear_ref_hpte(kvm, hpte, pte_index);  in kvmppc_h_clear_ref()
729 			rmap = revmap_for_hpte(kvm, v, gr);  in kvmppc_h_clear_ref()
747 	struct kvm *kvm = vcpu->kvm;  in kvmppc_h_clear_mod()  local
754 	if (pte_index >= kvm->arch.hpt_npte)  in kvmppc_h_clear_mod()
757 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);  in kvmppc_h_clear_mod()
758 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));  in kvmppc_h_clear_mod()
769 		note_hpte_modification(kvm, rev);  in kvmppc_h_clear_mod()
774 		kvmppc_invalidate_hpte(kvm, hpte, pte_index);  in kvmppc_h_clear_mod()
781 			rmap = revmap_for_hpte(kvm, v, gr);  in kvmppc_h_clear_mod()
797 void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,  in kvmppc_invalidate_hpte()  argument
805 	do_tlbies(kvm, &rb, 1, 1, true);  in kvmppc_invalidate_hpte()
809 void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,  in kvmppc_clear_ref_hpte()  argument
820 	do_tlbies(kvm, &rb, 1, 1, false);  in kvmppc_clear_ref_hpte()
835 long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,  in kvmppc_hv_find_lock_hpte()  argument
864 	hash = (vsid ^ ((eaddr & somask) >> pshift)) & kvm->arch.hpt_mask;  in kvmppc_hv_find_lock_hpte()
875 		hpte = (__be64 *)(kvm->arch.hpt_virt + (hash << 7));  in kvmppc_hv_find_lock_hpte()
905 		hash = hash ^ kvm->arch.hpt_mask;  in kvmppc_hv_find_lock_hpte()
925 	struct kvm *kvm = vcpu->kvm;  in kvmppc_hpte_hv_fault()  local
938 	index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid);  in kvmppc_hpte_hv_fault()
944 	hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4));  in kvmppc_hpte_hv_fault()
947 	rev = real_vmalloc_addr(&kvm->arch.revmap[index]);  in kvmppc_hpte_hv_fault()