80 static bool access_dcsw(struct kvm_vcpu *vcpu,  in access_dcsw()  argument
85 		return read_from_write_only(vcpu, p);  in access_dcsw()
87 	kvm_set_way_flush(vcpu);  in access_dcsw()
96 static bool access_vm_reg(struct kvm_vcpu *vcpu,  in access_vm_reg()  argument
100 	bool was_enabled = vcpu_has_cache_enabled(vcpu);  in access_vm_reg()
105 		vcpu_sys_reg(vcpu, r->reg) = p->regval;  in access_vm_reg()
108 			vcpu_cp15_64_high(vcpu, r->reg) = upper_32_bits(p->regval);  in access_vm_reg()
109 		vcpu_cp15_64_low(vcpu, r->reg) = lower_32_bits(p->regval);  in access_vm_reg()
112 	kvm_toggle_cache(vcpu, was_enabled);  in access_vm_reg()
122 static bool access_gic_sgi(struct kvm_vcpu *vcpu,  in access_gic_sgi()  argument
127 		return read_from_write_only(vcpu, p);  in access_gic_sgi()
129 	vgic_v3_dispatch_sgi(vcpu, p->regval);  in access_gic_sgi()
134 static bool trap_raz_wi(struct kvm_vcpu *vcpu,  in trap_raz_wi()  argument
139 		return ignore_write(vcpu, p);  in trap_raz_wi()
141 		return read_zero(vcpu, p);  in trap_raz_wi()
144 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,  in trap_oslsr_el1()  argument
149 		return ignore_write(vcpu, p);  in trap_oslsr_el1()
156 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,  in trap_dbgauthstatus_el1()  argument
161 		return ignore_write(vcpu, p);  in trap_dbgauthstatus_el1()
197 static bool trap_debug_regs(struct kvm_vcpu *vcpu,  in trap_debug_regs()  argument
202 		vcpu_sys_reg(vcpu, r->reg) = p->regval;  in trap_debug_regs()
203 		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;  in trap_debug_regs()
205 		p->regval = vcpu_sys_reg(vcpu, r->reg);  in trap_debug_regs()
222 static inline void reg_to_dbg(struct kvm_vcpu *vcpu,  in reg_to_dbg()  argument
234 	vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;  in reg_to_dbg()
237 static inline void dbg_to_reg(struct kvm_vcpu *vcpu,  in dbg_to_reg()  argument
246 static inline bool trap_bvr(struct kvm_vcpu *vcpu,  in trap_bvr()  argument
250 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];  in trap_bvr()
253 		reg_to_dbg(vcpu, p, dbg_reg);  in trap_bvr()
255 		dbg_to_reg(vcpu, p, dbg_reg);  in trap_bvr()
262 static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in set_bvr()  argument
265 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];  in set_bvr()
272 static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in get_bvr()  argument
275 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];  in get_bvr()
282 static inline void reset_bvr(struct kvm_vcpu *vcpu,  in reset_bvr()  argument
285 	vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;  in reset_bvr()
288 static inline bool trap_bcr(struct kvm_vcpu *vcpu,  in trap_bcr()  argument
292 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];  in trap_bcr()
295 		reg_to_dbg(vcpu, p, dbg_reg);  in trap_bcr()
297 		dbg_to_reg(vcpu, p, dbg_reg);  in trap_bcr()
304 static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in set_bcr()  argument
307 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];  in set_bcr()
315 static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in get_bcr()  argument
318 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];  in get_bcr()
325 static inline void reset_bcr(struct kvm_vcpu *vcpu,  in reset_bcr()  argument
328 	vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;  in reset_bcr()
331 static inline bool trap_wvr(struct kvm_vcpu *vcpu,  in trap_wvr()  argument
335 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];  in trap_wvr()
338 		reg_to_dbg(vcpu, p, dbg_reg);  in trap_wvr()
340 		dbg_to_reg(vcpu, p, dbg_reg);  in trap_wvr()
343 		vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);  in trap_wvr()
348 static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in set_wvr()  argument
351 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];  in set_wvr()
358 static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in get_wvr()  argument
361 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];  in get_wvr()
368 static inline void reset_wvr(struct kvm_vcpu *vcpu,  in reset_wvr()  argument
371 	vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;  in reset_wvr()
374 static inline bool trap_wcr(struct kvm_vcpu *vcpu,  in trap_wcr()  argument
378 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];  in trap_wcr()
381 		reg_to_dbg(vcpu, p, dbg_reg);  in trap_wcr()
383 		dbg_to_reg(vcpu, p, dbg_reg);  in trap_wcr()
390 static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in set_wcr()  argument
393 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];  in set_wcr()
400 static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,  in get_wcr()  argument
403 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];  in get_wcr()
410 static inline void reset_wcr(struct kvm_vcpu *vcpu,  in reset_wcr()  argument
413 	vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;  in reset_wcr()
416 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)  in reset_amair_el1()  argument
421 	vcpu_sys_reg(vcpu, AMAIR_EL1) = amair;  in reset_amair_el1()
424 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)  in reset_mpidr()  argument
435 	mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);  in reset_mpidr()
436 	mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);  in reset_mpidr()
437 	mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);  in reset_mpidr()
438 	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;  in reset_mpidr()
681 static bool trap_dbgidr(struct kvm_vcpu *vcpu,  in trap_dbgidr()  argument
686 		return ignore_write(vcpu, p);  in trap_dbgidr()
700 static bool trap_debug32(struct kvm_vcpu *vcpu,  in trap_debug32()  argument
705 		vcpu_cp14(vcpu, r->reg) = p->regval;  in trap_debug32()
706 		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;  in trap_debug32()
708 		p->regval = vcpu_cp14(vcpu, r->reg);  in trap_debug32()
725 static inline bool trap_xvr(struct kvm_vcpu *vcpu,  in trap_xvr()  argument
729 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];  in trap_xvr()
738 		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;  in trap_xvr()
969 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)  in kvm_handle_cp14_load_store()  argument
971 	kvm_inject_undefined(vcpu);  in kvm_handle_cp14_load_store()
985 static int emulate_cp(struct kvm_vcpu *vcpu,  in emulate_cp()  argument
1006 		if (likely(r->access(vcpu, params, r))) {  in emulate_cp()
1008 			kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));  in emulate_cp()
1019 static void unhandled_cp_access(struct kvm_vcpu *vcpu,  in unhandled_cp_access()  argument
1022 	u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);  in unhandled_cp_access()
1039 		cp, *vcpu_pc(vcpu));  in unhandled_cp_access()
1041 	kvm_inject_undefined(vcpu);  in unhandled_cp_access()
1049 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,  in kvm_handle_cp_64()  argument
1056 	u32 hsr = kvm_vcpu_get_hsr(vcpu);  in kvm_handle_cp_64()
1075 		params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;  in kvm_handle_cp_64()
1076 		params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;  in kvm_handle_cp_64()
1079 	if (!emulate_cp(vcpu, &params, target_specific, nr_specific))  in kvm_handle_cp_64()
1081 	if (!emulate_cp(vcpu, &params, global, nr_global))  in kvm_handle_cp_64()
1084 	unhandled_cp_access(vcpu, &params);  in kvm_handle_cp_64()
1089 		vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));  in kvm_handle_cp_64()
1090 		vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));  in kvm_handle_cp_64()
1101 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,  in kvm_handle_cp_32()  argument
1108 	u32 hsr = kvm_vcpu_get_hsr(vcpu);  in kvm_handle_cp_32()
1114 	params.regval = vcpu_get_reg(vcpu, Rt);  in kvm_handle_cp_32()
1121 	if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||  in kvm_handle_cp_32()
1122 	    !emulate_cp(vcpu, &params, global, nr_global)) {  in kvm_handle_cp_32()
1124 			vcpu_set_reg(vcpu, Rt, params.regval);  in kvm_handle_cp_32()
1128 	unhandled_cp_access(vcpu, &params);  in kvm_handle_cp_32()
1132 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)  in kvm_handle_cp15_64()  argument
1137 	target_specific = get_target_table(vcpu->arch.target, false, &num);  in kvm_handle_cp15_64()
1138 	return kvm_handle_cp_64(vcpu,  in kvm_handle_cp15_64()
1143 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)  in kvm_handle_cp15_32()  argument
1148 	target_specific = get_target_table(vcpu->arch.target, false, &num);  in kvm_handle_cp15_32()
1149 	return kvm_handle_cp_32(vcpu,  in kvm_handle_cp15_32()
1154 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)  in kvm_handle_cp14_64()  argument
1156 	return kvm_handle_cp_64(vcpu,  in kvm_handle_cp14_64()
1161 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)  in kvm_handle_cp14_32()  argument
1163 	return kvm_handle_cp_32(vcpu,  in kvm_handle_cp14_32()
1168 static int emulate_sys_reg(struct kvm_vcpu *vcpu,  in emulate_sys_reg()  argument
1174 	table = get_target_table(vcpu->arch.target, true, &num);  in emulate_sys_reg()
1190 		if (likely(r->access(vcpu, params, r))) {  in emulate_sys_reg()
1192 			kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));  in emulate_sys_reg()
1198 			*vcpu_pc(vcpu));  in emulate_sys_reg()
1201 	kvm_inject_undefined(vcpu);  in emulate_sys_reg()
1205 static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,  in reset_sys_reg_descs()  argument
1212 			table[i].reset(vcpu, &table[i]);  in reset_sys_reg_descs()
1220 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)  in kvm_handle_sys_reg()  argument
1223 	unsigned long esr = kvm_vcpu_get_hsr(vcpu);  in kvm_handle_sys_reg()
1236 	params.regval = vcpu_get_reg(vcpu, Rt);  in kvm_handle_sys_reg()
1239 	ret = emulate_sys_reg(vcpu, &params);  in kvm_handle_sys_reg()
1242 		vcpu_set_reg(vcpu, Rt, params.regval);  in kvm_handle_sys_reg()
1280 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,  in index_to_sys_reg_desc()  argument
1294 	table = get_target_table(vcpu->arch.target, true, &num);  in index_to_sys_reg_desc()
1522 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)  in kvm_arm_sys_reg_get_reg()  argument
1533 	r = index_to_sys_reg_desc(vcpu, reg->id);  in kvm_arm_sys_reg_get_reg()
1538 		return (r->get_user)(vcpu, r, reg, uaddr);  in kvm_arm_sys_reg_get_reg()
1540 	return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id);  in kvm_arm_sys_reg_get_reg()
1543 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)  in kvm_arm_sys_reg_set_reg()  argument
1554 	r = index_to_sys_reg_desc(vcpu, reg->id);  in kvm_arm_sys_reg_set_reg()
1559 		return (r->set_user)(vcpu, r, reg, uaddr);  in kvm_arm_sys_reg_set_reg()
1561 	return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);  in kvm_arm_sys_reg_set_reg()
1615 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)  in walk_sys_regs()  argument
1622 	i1 = get_target_table(vcpu->arch.target, true, &num);  in walk_sys_regs()
1657 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)  in kvm_arm_num_sys_reg_descs()  argument
1661 		+ walk_sys_regs(vcpu, (u64 __user *)NULL);  in kvm_arm_num_sys_reg_descs()
1664 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)  in kvm_arm_copy_sys_reg_indices()  argument
1676 	err = walk_sys_regs(vcpu, uindices);  in kvm_arm_copy_sys_reg_indices()
1741 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)  in kvm_reset_sys_regs()  argument
1747 	memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));  in kvm_reset_sys_regs()
1750 	reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));  in kvm_reset_sys_regs()
1752 	table = get_target_table(vcpu->arch.target, true, &num);  in kvm_reset_sys_regs()
1753 	reset_sys_reg_descs(vcpu, table, num);  in kvm_reset_sys_regs()
1756 		if (vcpu_sys_reg(vcpu, num) == 0x4242424242424242)  in kvm_reset_sys_regs()