297 	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,  in kvmppc_dump_regs()
301 struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)  in kvmppc_find_vcpu()  argument
306 	mutex_lock(&kvm->lock);  in kvmppc_find_vcpu()
307 	kvm_for_each_vcpu(r, v, kvm) {  in kvmppc_find_vcpu()
313 	mutex_unlock(&kvm->lock);  in kvmppc_find_vcpu()
359 	struct kvm *kvm = vcpu->kvm;  in do_h_register_vpa()  local
367 	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);  in do_h_register_vpa()
379 		va = kvmppc_pin_guest_page(kvm, vpa, &nb);  in do_h_register_vpa()
386 		kvmppc_unpin_guest_page(kvm, va, vpa, false);  in do_h_register_vpa()
467 	struct kvm *kvm = vcpu->kvm;  in kvmppc_update_vpa()  local
486 			va = kvmppc_pin_guest_page(kvm, gpa, &nb);  in kvmppc_update_vpa()
492 			kvmppc_unpin_guest_page(kvm, va, gpa, false);  in kvmppc_update_vpa()
502 		kvmppc_unpin_guest_page(kvm, va, gpa, false);  in kvmppc_update_vpa()
506 		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,  in kvmppc_update_vpa()
679 	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))  in kvmppc_pseries_do_hcall()
687 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);  in kvmppc_pseries_do_hcall()
705 		tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);  in kvmppc_pseries_do_hcall()
721 		if (list_empty(&vcpu->kvm->arch.rtas_tokens))  in kvmppc_pseries_do_hcall()
724 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvmppc_pseries_do_hcall()
726 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvmppc_pseries_do_hcall()
983 	struct kvm *kvm = vcpu->kvm;  in kvmppc_set_lpcr()  local
987 	mutex_lock(&kvm->lock);  in kvmppc_set_lpcr()
997 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmppc_set_lpcr()
1021 	mutex_unlock(&kvm->lock);  in kvmppc_set_lpcr()
1423 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)  in kvmppc_vcore_create()  argument
1437 	vcore->lpcr = kvm->arch.lpcr;  in kvmppc_vcore_create()
1439 	vcore->kvm = kvm;  in kvmppc_vcore_create()
1480 	kvm_get_kvm(vcpu->kvm);  in debugfs_timings_open()
1491 	kvm_put_kvm(p->vcpu->kvm);  in debugfs_timings_release()
1580 	struct kvm *kvm = vcpu->kvm;  in debugfs_vcpu_init()  local
1583 	if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir))  in debugfs_vcpu_init()
1585 	vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);  in debugfs_vcpu_init()
1599 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,  in kvmppc_core_vcpu_create_hv()  argument
1616 	err = kvm_vcpu_init(vcpu, kvm, id);  in kvmppc_core_vcpu_create_hv()
1647 	mutex_lock(&kvm->lock);  in kvmppc_core_vcpu_create_hv()
1648 	vcore = kvm->arch.vcores[core];  in kvmppc_core_vcpu_create_hv()
1650 		vcore = kvmppc_vcore_create(kvm, core);  in kvmppc_core_vcpu_create_hv()
1651 		kvm->arch.vcores[core] = vcore;  in kvmppc_core_vcpu_create_hv()
1652 		kvm->arch.online_vcores++;  in kvmppc_core_vcpu_create_hv()
1654 	mutex_unlock(&kvm->lock);  in kvmppc_core_vcpu_create_hv()
1678 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)  in unpin_vpa()  argument
1681 		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,  in unpin_vpa()
1688 	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);  in kvmppc_core_vcpu_free_hv()
1689 	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);  in kvmppc_core_vcpu_free_hv()
1690 	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);  in kvmppc_core_vcpu_free_hv()
2021 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);  in kvmppc_run_core()
2044 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);  in kvmppc_run_core()
2243 	atomic_inc(&vcpu->kvm->arch.vcpus_running);  in kvmppc_vcpu_run_hv()
2248 	if (!vcpu->kvm->arch.hpte_setup_done) {  in kvmppc_vcpu_run_hv()
2271 			srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvmppc_vcpu_run_hv()
2274 			srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);  in kvmppc_vcpu_run_hv()
2280 	atomic_dec(&vcpu->kvm->arch.vcpus_running);  in kvmppc_vcpu_run_hv()
2305 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,  in kvm_vm_ioctl_get_smmu_info_hv()  argument
2327 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,  in kvm_vm_ioctl_get_dirty_log_hv()  argument
2334 	mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_get_dirty_log_hv()
2340 	memslot = id_to_memslot(kvm->memslots, log->slot);  in kvm_vm_ioctl_get_dirty_log_hv()
2348 	r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);  in kvm_vm_ioctl_get_dirty_log_hv()
2358 	mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_get_dirty_log_hv()
2381 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,  in kvmppc_core_prepare_memory_region_hv()  argument
2388 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,  in kvmppc_core_commit_memory_region_hv()  argument
2402 		memslot = id_to_memslot(kvm->memslots, mem->slot);  in kvmppc_core_commit_memory_region_hv()
2403 		kvmppc_hv_get_dirty_log(kvm, memslot, NULL);  in kvmppc_core_commit_memory_region_hv()
2411 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)  in kvmppc_update_lpcr()  argument
2416 	if ((kvm->arch.lpcr & mask) == lpcr)  in kvmppc_update_lpcr()
2419 	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;  in kvmppc_update_lpcr()
2422 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];  in kvmppc_update_lpcr()
2428 		if (++cores_done >= kvm->arch.online_vcores)  in kvmppc_update_lpcr()
2441 	struct kvm *kvm = vcpu->kvm;  in kvmppc_hv_setup_htab_rma()  local
2449 	mutex_lock(&kvm->lock);  in kvmppc_hv_setup_htab_rma()
2450 	if (kvm->arch.hpte_setup_done)  in kvmppc_hv_setup_htab_rma()
2454 	if (!kvm->arch.hpt_virt) {  in kvmppc_hv_setup_htab_rma()
2455 		err = kvmppc_alloc_hpt(kvm, NULL);  in kvmppc_hv_setup_htab_rma()
2463 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_hv_setup_htab_rma()
2464 	memslot = gfn_to_memslot(kvm, 0);  in kvmppc_hv_setup_htab_rma()
2491 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |  in kvmppc_hv_setup_htab_rma()
2499 	kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);  in kvmppc_hv_setup_htab_rma()
2503 	kvm->arch.hpte_setup_done = 1;  in kvmppc_hv_setup_htab_rma()
2506 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_hv_setup_htab_rma()
2508 	mutex_unlock(&kvm->lock);  in kvmppc_hv_setup_htab_rma()
2516 static int kvmppc_core_init_vm_hv(struct kvm *kvm)  in kvmppc_core_init_vm_hv()  argument
2526 	kvm->arch.lpid = lpid;  in kvmppc_core_init_vm_hv()
2533 	cpumask_setall(&kvm->arch.need_tlb_flush);  in kvmppc_core_init_vm_hv()
2536 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,  in kvmppc_core_init_vm_hv()
2537 	       sizeof(kvm->arch.enabled_hcalls));  in kvmppc_core_init_vm_hv()
2539 	kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);  in kvmppc_core_init_vm_hv()
2542 	kvm->arch.host_lpid = mfspr(SPRN_LPID);  in kvmppc_core_init_vm_hv()
2543 	kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);  in kvmppc_core_init_vm_hv()
2547 	kvm->arch.vrma_slb_v = SLB_VSID_B_1T |  in kvmppc_core_init_vm_hv()
2552 	kvm->arch.lpcr = lpcr;  in kvmppc_core_init_vm_hv()
2564 	kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);  in kvmppc_core_init_vm_hv()
2565 	if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir))  in kvmppc_core_init_vm_hv()
2566 		kvmppc_mmu_debugfs_init(kvm);  in kvmppc_core_init_vm_hv()
2571 static void kvmppc_free_vcores(struct kvm *kvm)  in kvmppc_free_vcores()  argument
2576 		if (kvm->arch.vcores[i] && kvm->arch.vcores[i]->mpp_buffer) {  in kvmppc_free_vcores()
2577 			struct kvmppc_vcore *vc = kvm->arch.vcores[i];  in kvmppc_free_vcores()
2581 		kfree(kvm->arch.vcores[i]);  in kvmppc_free_vcores()
2583 	kvm->arch.online_vcores = 0;  in kvmppc_free_vcores()
2586 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)  in kvmppc_core_destroy_vm_hv()  argument
2588 	debugfs_remove_recursive(kvm->arch.debugfs_dir);  in kvmppc_core_destroy_vm_hv()
2592 	kvmppc_free_vcores(kvm);  in kvmppc_core_destroy_vm_hv()
2594 	kvmppc_free_hpt(kvm);  in kvmppc_core_destroy_vm_hv()
2627 	struct kvm *kvm __maybe_unused = filp->private_data;  in kvm_arch_vm_ioctl_hv()
2639 		r = kvmppc_alloc_reset_hpt(kvm, &htab_order);  in kvm_arch_vm_ioctl_hv()
2655 		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);  in kvm_arch_vm_ioctl_hv()