67 static unsigned int get_tlb_esel(struct kvm_vcpu *vcpu, int tlbsel)  in get_tlb_esel()  argument
69 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in get_tlb_esel()
70 	int esel = get_tlb_esel_bit(vcpu);  in get_tlb_esel()
74 		esel += gtlb0_set_base(vcpu_e500, vcpu->arch.shared->mas2);  in get_tlb_esel()
129 static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,  in kvmppc_e500_deliver_tlb_miss()  argument
132 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_deliver_tlb_miss()
137 	tlbsel = (vcpu->arch.shared->mas4 >> 28) & 0x1;  in kvmppc_e500_deliver_tlb_miss()
139 	tsized = (vcpu->arch.shared->mas4 >> 7) & 0x1f;  in kvmppc_e500_deliver_tlb_miss()
141 	vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim)  in kvmppc_e500_deliver_tlb_miss()
143 	vcpu->arch.shared->mas1 = MAS1_VALID | (as ? MAS1_TS : 0)  in kvmppc_e500_deliver_tlb_miss()
144 		| MAS1_TID(get_tlbmiss_tid(vcpu))  in kvmppc_e500_deliver_tlb_miss()
146 	vcpu->arch.shared->mas2 = (eaddr & MAS2_EPN)  in kvmppc_e500_deliver_tlb_miss()
147 		| (vcpu->arch.shared->mas4 & MAS2_ATTRIB_MASK);  in kvmppc_e500_deliver_tlb_miss()
148 	vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3;  in kvmppc_e500_deliver_tlb_miss()
149 	vcpu->arch.shared->mas6 = (vcpu->arch.shared->mas6 & MAS6_SPID1)  in kvmppc_e500_deliver_tlb_miss()
150 		| (get_cur_pid(vcpu) << 16)  in kvmppc_e500_deliver_tlb_miss()
196 static void kvmppc_set_tlb1map_range(struct kvm_vcpu *vcpu,  in kvmppc_set_tlb1map_range()  argument
200 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_set_tlb1map_range()
243 	kvmppc_core_flush_tlb(&vcpu_e500->vcpu);  in kvmppc_e500_emul_mt_mmucsr0()
248 int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, gva_t ea)  in kvmppc_e500_emul_tlbivax()  argument
250 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_emul_tlbivax()
267 				get_cur_pid(vcpu), -1);  in kvmppc_e500_emul_tlbivax()
273 	kvmppc_core_flush_tlb(&vcpu_e500->vcpu);  in kvmppc_e500_emul_tlbivax()
310 int kvmppc_e500_emul_tlbilx(struct kvm_vcpu *vcpu, int type, gva_t ea)  in kvmppc_e500_emul_tlbilx()  argument
312 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_emul_tlbilx()
313 	int pid = get_cur_spid(vcpu);  in kvmppc_e500_emul_tlbilx()
325 int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu)  in kvmppc_e500_emul_tlbre()  argument
327 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_emul_tlbre()
331 	tlbsel = get_tlb_tlbsel(vcpu);  in kvmppc_e500_emul_tlbre()
332 	esel = get_tlb_esel(vcpu, tlbsel);  in kvmppc_e500_emul_tlbre()
335 	vcpu->arch.shared->mas0 &= ~MAS0_NV(~0);  in kvmppc_e500_emul_tlbre()
336 	vcpu->arch.shared->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);  in kvmppc_e500_emul_tlbre()
337 	vcpu->arch.shared->mas1 = gtlbe->mas1;  in kvmppc_e500_emul_tlbre()
338 	vcpu->arch.shared->mas2 = gtlbe->mas2;  in kvmppc_e500_emul_tlbre()
339 	vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;  in kvmppc_e500_emul_tlbre()
344 int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea)  in kvmppc_e500_emul_tlbsx()  argument
346 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_emul_tlbsx()
347 	int as = !!get_cur_sas(vcpu);  in kvmppc_e500_emul_tlbsx()
348 	unsigned int pid = get_cur_spid(vcpu);  in kvmppc_e500_emul_tlbsx()
363 		vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel)  in kvmppc_e500_emul_tlbsx()
365 		vcpu->arch.shared->mas1 = gtlbe->mas1;  in kvmppc_e500_emul_tlbsx()
366 		vcpu->arch.shared->mas2 = gtlbe->mas2;  in kvmppc_e500_emul_tlbsx()
367 		vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;  in kvmppc_e500_emul_tlbsx()
372 		tlbsel = vcpu->arch.shared->mas4 >> 28 & 0x1;  in kvmppc_e500_emul_tlbsx()
375 		vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel)  in kvmppc_e500_emul_tlbsx()
378 		vcpu->arch.shared->mas1 =  in kvmppc_e500_emul_tlbsx()
379 			  (vcpu->arch.shared->mas6 & MAS6_SPID0)  in kvmppc_e500_emul_tlbsx()
380 			| ((vcpu->arch.shared->mas6 & MAS6_SAS) ? MAS1_TS : 0)  in kvmppc_e500_emul_tlbsx()
381 			| (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0));  in kvmppc_e500_emul_tlbsx()
382 		vcpu->arch.shared->mas2 &= MAS2_EPN;  in kvmppc_e500_emul_tlbsx()
383 		vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 &  in kvmppc_e500_emul_tlbsx()
385 		vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 |  in kvmppc_e500_emul_tlbsx()
389 	kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS);  in kvmppc_e500_emul_tlbsx()
393 int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)  in kvmppc_e500_emul_tlbwe()  argument
395 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_emul_tlbwe()
401 	tlbsel = get_tlb_tlbsel(vcpu);  in kvmppc_e500_emul_tlbwe()
402 	esel = get_tlb_esel(vcpu, tlbsel);  in kvmppc_e500_emul_tlbwe()
413 	gtlbe->mas1 = vcpu->arch.shared->mas1;  in kvmppc_e500_emul_tlbwe()
414 	gtlbe->mas2 = vcpu->arch.shared->mas2;  in kvmppc_e500_emul_tlbwe()
415 	if (!(vcpu->arch.shared->msr & MSR_CM))  in kvmppc_e500_emul_tlbwe()
417 	gtlbe->mas7_3 = vcpu->arch.shared->mas7_3;  in kvmppc_e500_emul_tlbwe()
419 	trace_kvm_booke206_gtlb_write(vcpu->arch.shared->mas0, gtlbe->mas1,  in kvmppc_e500_emul_tlbwe()
431 			kvmppc_set_tlb1map_range(vcpu, gtlbe);  in kvmppc_e500_emul_tlbwe()
434 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvmppc_e500_emul_tlbwe()
437 	if (tlbe_is_host_safe(vcpu, gtlbe)) {  in kvmppc_e500_emul_tlbwe()
447 		kvmppc_mmu_map(vcpu, eaddr, raddr, index_of(tlbsel, esel));  in kvmppc_e500_emul_tlbwe()
450 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvmppc_e500_emul_tlbwe()
452 	kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS);  in kvmppc_e500_emul_tlbwe()
456 static int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu,  in kvmppc_e500_tlb_search()  argument
459 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_e500_tlb_search()
472 int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu,  in kvmppc_core_vcpu_translate()  argument
484 	index = kvmppc_e500_tlb_search(vcpu, eaddr, pid, as);  in kvmppc_core_vcpu_translate()
490 	tr->physical_address = kvmppc_mmu_xlate(vcpu, index, eaddr);  in kvmppc_core_vcpu_translate()
498 int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)  in kvmppc_mmu_itlb_index()  argument
500 	unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);  in kvmppc_mmu_itlb_index()
502 	return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);  in kvmppc_mmu_itlb_index()
505 int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)  in kvmppc_mmu_dtlb_index()  argument
507 	unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);  in kvmppc_mmu_dtlb_index()
509 	return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);  in kvmppc_mmu_dtlb_index()
512 void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu)  in kvmppc_mmu_itlb_miss()  argument
514 	unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);  in kvmppc_mmu_itlb_miss()
516 	kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as);  in kvmppc_mmu_itlb_miss()
519 void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu)  in kvmppc_mmu_dtlb_miss()  argument
521 	unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);  in kvmppc_mmu_dtlb_miss()
523 	kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as);  in kvmppc_mmu_dtlb_miss()
526 gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index,  in kvmppc_mmu_xlate()  argument
529 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvmppc_mmu_xlate()
539 void kvmppc_mmu_destroy_e500(struct kvm_vcpu *vcpu)  in kvmppc_mmu_destroy_e500()  argument
549 	kvmppc_core_flush_tlb(&vcpu_e500->vcpu);  in free_gtlb()
574 void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvmppc_get_sregs_e500_tlb()  argument
576 	sregs->u.e.mas0 = vcpu->arch.shared->mas0;  in kvmppc_get_sregs_e500_tlb()
577 	sregs->u.e.mas1 = vcpu->arch.shared->mas1;  in kvmppc_get_sregs_e500_tlb()
578 	sregs->u.e.mas2 = vcpu->arch.shared->mas2;  in kvmppc_get_sregs_e500_tlb()
579 	sregs->u.e.mas7_3 = vcpu->arch.shared->mas7_3;  in kvmppc_get_sregs_e500_tlb()
580 	sregs->u.e.mas4 = vcpu->arch.shared->mas4;  in kvmppc_get_sregs_e500_tlb()
581 	sregs->u.e.mas6 = vcpu->arch.shared->mas6;  in kvmppc_get_sregs_e500_tlb()
583 	sregs->u.e.mmucfg = vcpu->arch.mmucfg;  in kvmppc_get_sregs_e500_tlb()
584 	sregs->u.e.tlbcfg[0] = vcpu->arch.tlbcfg[0];  in kvmppc_get_sregs_e500_tlb()
585 	sregs->u.e.tlbcfg[1] = vcpu->arch.tlbcfg[1];  in kvmppc_get_sregs_e500_tlb()
590 int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvmppc_set_sregs_e500_tlb()  argument
593 		vcpu->arch.shared->mas0 = sregs->u.e.mas0;  in kvmppc_set_sregs_e500_tlb()
594 		vcpu->arch.shared->mas1 = sregs->u.e.mas1;  in kvmppc_set_sregs_e500_tlb()
595 		vcpu->arch.shared->mas2 = sregs->u.e.mas2;  in kvmppc_set_sregs_e500_tlb()
596 		vcpu->arch.shared->mas7_3 = sregs->u.e.mas7_3;  in kvmppc_set_sregs_e500_tlb()
597 		vcpu->arch.shared->mas4 = sregs->u.e.mas4;  in kvmppc_set_sregs_e500_tlb()
598 		vcpu->arch.shared->mas6 = sregs->u.e.mas6;  in kvmppc_set_sregs_e500_tlb()
604 int kvmppc_get_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id,  in kvmppc_get_one_reg_e500_tlb()  argument
612 		*val = get_reg_val(id, vcpu->arch.shared->mas0);  in kvmppc_get_one_reg_e500_tlb()
615 		*val = get_reg_val(id, vcpu->arch.shared->mas1);  in kvmppc_get_one_reg_e500_tlb()
618 		*val = get_reg_val(id, vcpu->arch.shared->mas2);  in kvmppc_get_one_reg_e500_tlb()
621 		*val = get_reg_val(id, vcpu->arch.shared->mas7_3);  in kvmppc_get_one_reg_e500_tlb()
624 		*val = get_reg_val(id, vcpu->arch.shared->mas4);  in kvmppc_get_one_reg_e500_tlb()
627 		*val = get_reg_val(id, vcpu->arch.shared->mas6);  in kvmppc_get_one_reg_e500_tlb()
630 		*val = get_reg_val(id, vcpu->arch.mmucfg);  in kvmppc_get_one_reg_e500_tlb()
633 		*val = get_reg_val(id, vcpu->arch.eptcfg);  in kvmppc_get_one_reg_e500_tlb()
640 		*val = get_reg_val(id, vcpu->arch.tlbcfg[i]);  in kvmppc_get_one_reg_e500_tlb()
647 		*val = get_reg_val(id, vcpu->arch.tlbps[i]);  in kvmppc_get_one_reg_e500_tlb()
657 int kvmppc_set_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id,  in kvmppc_set_one_reg_e500_tlb()  argument
665 		vcpu->arch.shared->mas0 = set_reg_val(id, *val);  in kvmppc_set_one_reg_e500_tlb()
668 		vcpu->arch.shared->mas1 = set_reg_val(id, *val);  in kvmppc_set_one_reg_e500_tlb()
671 		vcpu->arch.shared->mas2 = set_reg_val(id, *val);  in kvmppc_set_one_reg_e500_tlb()
674 		vcpu->arch.shared->mas7_3 = set_reg_val(id, *val);  in kvmppc_set_one_reg_e500_tlb()
677 		vcpu->arch.shared->mas4 = set_reg_val(id, *val);  in kvmppc_set_one_reg_e500_tlb()
680 		vcpu->arch.shared->mas6 = set_reg_val(id, *val);  in kvmppc_set_one_reg_e500_tlb()
685 		if (reg != vcpu->arch.mmucfg)  in kvmppc_set_one_reg_e500_tlb()
691 		if (reg != vcpu->arch.eptcfg)  in kvmppc_set_one_reg_e500_tlb()
702 		if (reg != vcpu->arch.tlbcfg[i])  in kvmppc_set_one_reg_e500_tlb()
712 		if (reg != vcpu->arch.tlbps[i])  in kvmppc_set_one_reg_e500_tlb()
724 static int vcpu_mmu_geometry_update(struct kvm_vcpu *vcpu,  in vcpu_mmu_geometry_update()  argument
727 	vcpu->arch.tlbcfg[0] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);  in vcpu_mmu_geometry_update()
729 		vcpu->arch.tlbcfg[0] |= params->tlb_sizes[0];  in vcpu_mmu_geometry_update()
730 	vcpu->arch.tlbcfg[0] |= params->tlb_ways[0] << TLBnCFG_ASSOC_SHIFT;  in vcpu_mmu_geometry_update()
732 	vcpu->arch.tlbcfg[1] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);  in vcpu_mmu_geometry_update()
733 	vcpu->arch.tlbcfg[1] |= params->tlb_sizes[1];  in vcpu_mmu_geometry_update()
734 	vcpu->arch.tlbcfg[1] |= params->tlb_ways[1] << TLBnCFG_ASSOC_SHIFT;  in vcpu_mmu_geometry_update()
738 int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_config_tlb()  argument
741 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvm_vcpu_ioctl_config_tlb()
835 	vcpu_mmu_geometry_update(vcpu, &params);  in kvm_vcpu_ioctl_config_tlb()
862 int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_dirty_tlb()  argument
865 	struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);  in kvm_vcpu_ioctl_dirty_tlb()
867 	kvmppc_core_flush_tlb(vcpu);  in kvm_vcpu_ioctl_dirty_tlb()
872 static int vcpu_mmu_init(struct kvm_vcpu *vcpu,  in vcpu_mmu_init()  argument
876 	vcpu->arch.mmucfg = mfspr(SPRN_MMUCFG) & ~MMUCFG_LPIDSIZE;  in vcpu_mmu_init()
879 	vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) &  in vcpu_mmu_init()
881 	vcpu->arch.tlbcfg[0] |= params[0].entries;  in vcpu_mmu_init()
882 	vcpu->arch.tlbcfg[0] |= params[0].ways << TLBnCFG_ASSOC_SHIFT;  in vcpu_mmu_init()
884 	vcpu->arch.tlbcfg[1] = mfspr(SPRN_TLB1CFG) &  in vcpu_mmu_init()
886 	vcpu->arch.tlbcfg[1] |= params[1].entries;  in vcpu_mmu_init()
887 	vcpu->arch.tlbcfg[1] |= params[1].ways << TLBnCFG_ASSOC_SHIFT;  in vcpu_mmu_init()
889 	if (has_feature(vcpu, VCPU_FTR_MMU_V2)) {  in vcpu_mmu_init()
890 		vcpu->arch.tlbps[0] = mfspr(SPRN_TLB0PS);  in vcpu_mmu_init()
891 		vcpu->arch.tlbps[1] = mfspr(SPRN_TLB1PS);  in vcpu_mmu_init()
893 		vcpu->arch.mmucfg &= ~MMUCFG_LRAT;  in vcpu_mmu_init()
896 		vcpu->arch.eptcfg = 0;  in vcpu_mmu_init()
897 		vcpu->arch.tlbcfg[0] &= ~TLBnCFG_PT;  in vcpu_mmu_init()
898 		vcpu->arch.tlbcfg[1] &= ~TLBnCFG_IND;  in vcpu_mmu_init()
906 	struct kvm_vcpu *vcpu = &vcpu_e500->vcpu;  in kvmppc_e500_tlb_init()  local
948 	vcpu_mmu_init(vcpu, vcpu_e500->gtlb_params);  in kvmppc_e500_tlb_init()