311 	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,  in kvmppc_dump_regs()
315 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)  in kvmppc_find_vcpu()  argument
320 	mutex_lock(&kvm->lock);  in kvmppc_find_vcpu()
321 	kvm_for_each_vcpu(r, v, kvm) {  in kvmppc_find_vcpu()
327 	mutex_unlock(&kvm->lock);  in kvmppc_find_vcpu()
373 	struct kvm *kvm = vcpu->kvm;  in do_h_register_vpa()  local
381 	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);  in do_h_register_vpa()
393 		va = kvmppc_pin_guest_page(kvm, vpa, &nb);  in do_h_register_vpa()
400 		kvmppc_unpin_guest_page(kvm, va, vpa, false);  in do_h_register_vpa()
481 	struct kvm *kvm = vcpu->kvm;  in kvmppc_update_vpa()  local
500 			va = kvmppc_pin_guest_page(kvm, gpa, &nb);  in kvmppc_update_vpa()
506 			kvmppc_unpin_guest_page(kvm, va, gpa, false);  in kvmppc_update_vpa()
516 		kvmppc_unpin_guest_page(kvm, va, gpa, false);  in kvmppc_update_vpa()
520 		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,  in kvmppc_update_vpa()
694 	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))  in kvmppc_pseries_do_hcall()
702 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);  in kvmppc_pseries_do_hcall()
720 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);  in kvmppc_pseries_do_hcall()
736 		if (list_empty(&vcpu->kvm->arch.rtas_tokens))  in kvmppc_pseries_do_hcall()
739 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvmppc_pseries_do_hcall()
741 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvmppc_pseries_do_hcall()
998 	struct kvm *kvm = vcpu->kvm;  in kvmppc_set_lpcr()  local
1002 	mutex_lock(&kvm->lock);  in kvmppc_set_lpcr()
1012 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmppc_set_lpcr()
1036 	mutex_unlock(&kvm->lock);  in kvmppc_set_lpcr()
1438 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)  in kvmppc_vcore_create()  argument
1452 	vcore->lpcr = kvm->arch.lpcr;  in kvmppc_vcore_create()
1454 	vcore->kvm = kvm;  in kvmppc_vcore_create()
1489 	kvm_get_kvm(vcpu->kvm);  in debugfs_timings_open()
1500 	kvm_put_kvm(p->vcpu->kvm);  in debugfs_timings_release()
1589 	struct kvm *kvm = vcpu->kvm;  in debugfs_vcpu_init()  local
1592 	if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))  in debugfs_vcpu_init()
1594 	vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);  in debugfs_vcpu_init()
1608 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,  in kvmppc_core_vcpu_create_hv()  argument
1625 	err = kvm_vcpu_init(vcpu, kvm, id);  in kvmppc_core_vcpu_create_hv()
1656 	mutex_lock(&kvm->lock);  in kvmppc_core_vcpu_create_hv()
1657 	vcore = kvm->arch.vcores[core];  in kvmppc_core_vcpu_create_hv()
1659 		vcore = kvmppc_vcore_create(kvm, core);  in kvmppc_core_vcpu_create_hv()
1660 		kvm->arch.vcores[core] = vcore;  in kvmppc_core_vcpu_create_hv()
1661 		kvm->arch.online_vcores++;  in kvmppc_core_vcpu_create_hv()
1663 	mutex_unlock(&kvm->lock);  in kvmppc_core_vcpu_create_hv()
1688 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)  in unpin_vpa()  argument
1691 		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,  in unpin_vpa()
1698 	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);  in kvmppc_core_vcpu_free_hv()
1699 	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);  in kvmppc_core_vcpu_free_hv()
1700 	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);  in kvmppc_core_vcpu_free_hv()
1951 	struct kvm	*subcore_vm[MAX_SUBCORES];
1970 	cip->subcore_vm[0] = vc->kvm;  in init_core_info()
2045 				cip->subcore_vm[new_sub] = vc->kvm;  in can_split_piggybacked_subcores()
2094 	cip->subcore_vm[sub] = vc->kvm;  in can_dynamic_split()
2112 	if (pvc->kvm != vc->kvm ||  in can_piggyback_subcore()
2437 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);  in kvmppc_run_core()
2441 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);  in kvmppc_run_core()
2698 	atomic_inc(&vcpu->kvm->arch.vcpus_running);  in kvmppc_vcpu_run_hv()
2703 	if (!vcpu->kvm->arch.hpte_setup_done) {  in kvmppc_vcpu_run_hv()
2726 			srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvmppc_vcpu_run_hv()
2729 			srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);  in kvmppc_vcpu_run_hv()
2735 	atomic_dec(&vcpu->kvm->arch.vcpus_running);  in kvmppc_vcpu_run_hv()
2760 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,  in kvm_vm_ioctl_get_smmu_info_hv()  argument
2782 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,  in kvm_vm_ioctl_get_dirty_log_hv()  argument
2790 	mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_get_dirty_log_hv()
2796 	slots = kvm_memslots(kvm);  in kvm_vm_ioctl_get_dirty_log_hv()
2805 	r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);  in kvm_vm_ioctl_get_dirty_log_hv()
2815 	mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_get_dirty_log_hv()
2838 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,  in kvmppc_core_prepare_memory_region_hv()  argument
2845 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,  in kvmppc_core_commit_memory_region_hv()  argument
2861 		slots = kvm_memslots(kvm);  in kvmppc_core_commit_memory_region_hv()
2863 		kvmppc_hv_get_dirty_log(kvm, memslot, NULL);  in kvmppc_core_commit_memory_region_hv()
2871 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)  in kvmppc_update_lpcr()  argument
2876 	if ((kvm->arch.lpcr & mask) == lpcr)  in kvmppc_update_lpcr()
2879 	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;  in kvmppc_update_lpcr()
2882 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];  in kvmppc_update_lpcr()
2888 		if (++cores_done >= kvm->arch.online_vcores)  in kvmppc_update_lpcr()
2901 	struct kvm *kvm = vcpu->kvm;  in kvmppc_hv_setup_htab_rma()  local
2909 	mutex_lock(&kvm->lock);  in kvmppc_hv_setup_htab_rma()
2910 	if (kvm->arch.hpte_setup_done)  in kvmppc_hv_setup_htab_rma()
2914 	if (!kvm->arch.hpt_virt) {  in kvmppc_hv_setup_htab_rma()
2915 		err = kvmppc_alloc_hpt(kvm, NULL);  in kvmppc_hv_setup_htab_rma()
2923 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_hv_setup_htab_rma()
2924 	memslot = gfn_to_memslot(kvm, 0);  in kvmppc_hv_setup_htab_rma()
2951 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |  in kvmppc_hv_setup_htab_rma()
2959 	kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);  in kvmppc_hv_setup_htab_rma()
2963 	kvm->arch.hpte_setup_done = 1;  in kvmppc_hv_setup_htab_rma()
2966 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_hv_setup_htab_rma()
2968 	mutex_unlock(&kvm->lock);  in kvmppc_hv_setup_htab_rma()
2976 static int kvmppc_core_init_vm_hv(struct kvm *kvm)  in kvmppc_core_init_vm_hv()  argument
2986 	kvm->arch.lpid = lpid;  in kvmppc_core_init_vm_hv()
2993 	cpumask_setall(&kvm->arch.need_tlb_flush);  in kvmppc_core_init_vm_hv()
2996 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,  in kvmppc_core_init_vm_hv()
2997 	       sizeof(kvm->arch.enabled_hcalls));  in kvmppc_core_init_vm_hv()
2999 	kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);  in kvmppc_core_init_vm_hv()
3002 	kvm->arch.host_lpid = mfspr(SPRN_LPID);  in kvmppc_core_init_vm_hv()
3003 	kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);  in kvmppc_core_init_vm_hv()
3007 	kvm->arch.vrma_slb_v = SLB_VSID_B_1T |  in kvmppc_core_init_vm_hv()
3012 	kvm->arch.lpcr = lpcr;  in kvmppc_core_init_vm_hv()
3024 	kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);  in kvmppc_core_init_vm_hv()
3025 	if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir))  in kvmppc_core_init_vm_hv()
3026 		kvmppc_mmu_debugfs_init(kvm);  in kvmppc_core_init_vm_hv()
3031 static void kvmppc_free_vcores(struct kvm *kvm)  in kvmppc_free_vcores()  argument
3036 		kfree(kvm->arch.vcores[i]);  in kvmppc_free_vcores()
3037 	kvm->arch.online_vcores = 0;  in kvmppc_free_vcores()
3040 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)  in kvmppc_core_destroy_vm_hv()  argument
3042 	debugfs_remove_recursive(kvm->arch.debugfs_dir);  in kvmppc_core_destroy_vm_hv()
3046 	kvmppc_free_vcores(kvm);  in kvmppc_core_destroy_vm_hv()
3048 	kvmppc_free_hpt(kvm);  in kvmppc_core_destroy_vm_hv()
3081 	struct kvm *kvm __maybe_unused = filp->private_data;  in kvm_arch_vm_ioctl_hv()
3093 		r = kvmppc_alloc_reset_hpt(kvm, &htab_order);  in kvm_arch_vm_ioctl_hv()
3109 		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);  in kvm_arch_vm_ioctl_hv()