1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * KVM/MIPS: Instruction/Exception emulation
7  *
8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
10  */
11 
12 #include <linux/errno.h>
13 #include <linux/err.h>
14 #include <linux/ktime.h>
15 #include <linux/kvm_host.h>
16 #include <linux/module.h>
17 #include <linux/vmalloc.h>
18 #include <linux/fs.h>
19 #include <linux/bootmem.h>
20 #include <linux/random.h>
21 #include <asm/page.h>
22 #include <asm/cacheflush.h>
23 #include <asm/cpu-info.h>
24 #include <asm/mmu_context.h>
25 #include <asm/tlbflush.h>
26 #include <asm/inst.h>
27 
28 #undef CONFIG_MIPS_MT
29 #include <asm/r4kcache.h>
30 #define CONFIG_MIPS_MT
31 
32 #include "opcode.h"
33 #include "interrupt.h"
34 #include "commpage.h"
35 
36 #include "trace.h"
37 
38 /*
39  * Compute the return address and do emulate branch simulation, if required.
40  * This function should be called only in branch delay slot active.
41  */
kvm_compute_return_epc(struct kvm_vcpu * vcpu,unsigned long instpc)42 unsigned long kvm_compute_return_epc(struct kvm_vcpu *vcpu,
43 	unsigned long instpc)
44 {
45 	unsigned int dspcontrol;
46 	union mips_instruction insn;
47 	struct kvm_vcpu_arch *arch = &vcpu->arch;
48 	long epc = instpc;
49 	long nextpc = KVM_INVALID_INST;
50 
51 	if (epc & 3)
52 		goto unaligned;
53 
54 	/* Read the instruction */
55 	insn.word = kvm_get_inst((uint32_t *) epc, vcpu);
56 
57 	if (insn.word == KVM_INVALID_INST)
58 		return KVM_INVALID_INST;
59 
60 	switch (insn.i_format.opcode) {
61 		/* jr and jalr are in r_format format. */
62 	case spec_op:
63 		switch (insn.r_format.func) {
64 		case jalr_op:
65 			arch->gprs[insn.r_format.rd] = epc + 8;
66 			/* Fall through */
67 		case jr_op:
68 			nextpc = arch->gprs[insn.r_format.rs];
69 			break;
70 		}
71 		break;
72 
73 		/*
74 		 * This group contains:
75 		 * bltz_op, bgez_op, bltzl_op, bgezl_op,
76 		 * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
77 		 */
78 	case bcond_op:
79 		switch (insn.i_format.rt) {
80 		case bltz_op:
81 		case bltzl_op:
82 			if ((long)arch->gprs[insn.i_format.rs] < 0)
83 				epc = epc + 4 + (insn.i_format.simmediate << 2);
84 			else
85 				epc += 8;
86 			nextpc = epc;
87 			break;
88 
89 		case bgez_op:
90 		case bgezl_op:
91 			if ((long)arch->gprs[insn.i_format.rs] >= 0)
92 				epc = epc + 4 + (insn.i_format.simmediate << 2);
93 			else
94 				epc += 8;
95 			nextpc = epc;
96 			break;
97 
98 		case bltzal_op:
99 		case bltzall_op:
100 			arch->gprs[31] = epc + 8;
101 			if ((long)arch->gprs[insn.i_format.rs] < 0)
102 				epc = epc + 4 + (insn.i_format.simmediate << 2);
103 			else
104 				epc += 8;
105 			nextpc = epc;
106 			break;
107 
108 		case bgezal_op:
109 		case bgezall_op:
110 			arch->gprs[31] = epc + 8;
111 			if ((long)arch->gprs[insn.i_format.rs] >= 0)
112 				epc = epc + 4 + (insn.i_format.simmediate << 2);
113 			else
114 				epc += 8;
115 			nextpc = epc;
116 			break;
117 		case bposge32_op:
118 			if (!cpu_has_dsp)
119 				goto sigill;
120 
121 			dspcontrol = rddsp(0x01);
122 
123 			if (dspcontrol >= 32)
124 				epc = epc + 4 + (insn.i_format.simmediate << 2);
125 			else
126 				epc += 8;
127 			nextpc = epc;
128 			break;
129 		}
130 		break;
131 
132 		/* These are unconditional and in j_format. */
133 	case jal_op:
134 		arch->gprs[31] = instpc + 8;
135 	case j_op:
136 		epc += 4;
137 		epc >>= 28;
138 		epc <<= 28;
139 		epc |= (insn.j_format.target << 2);
140 		nextpc = epc;
141 		break;
142 
143 		/* These are conditional and in i_format. */
144 	case beq_op:
145 	case beql_op:
146 		if (arch->gprs[insn.i_format.rs] ==
147 		    arch->gprs[insn.i_format.rt])
148 			epc = epc + 4 + (insn.i_format.simmediate << 2);
149 		else
150 			epc += 8;
151 		nextpc = epc;
152 		break;
153 
154 	case bne_op:
155 	case bnel_op:
156 		if (arch->gprs[insn.i_format.rs] !=
157 		    arch->gprs[insn.i_format.rt])
158 			epc = epc + 4 + (insn.i_format.simmediate << 2);
159 		else
160 			epc += 8;
161 		nextpc = epc;
162 		break;
163 
164 	case blez_op:		/* not really i_format */
165 	case blezl_op:
166 		/* rt field assumed to be zero */
167 		if ((long)arch->gprs[insn.i_format.rs] <= 0)
168 			epc = epc + 4 + (insn.i_format.simmediate << 2);
169 		else
170 			epc += 8;
171 		nextpc = epc;
172 		break;
173 
174 	case bgtz_op:
175 	case bgtzl_op:
176 		/* rt field assumed to be zero */
177 		if ((long)arch->gprs[insn.i_format.rs] > 0)
178 			epc = epc + 4 + (insn.i_format.simmediate << 2);
179 		else
180 			epc += 8;
181 		nextpc = epc;
182 		break;
183 
184 		/* And now the FPA/cp1 branch instructions. */
185 	case cop1_op:
186 		kvm_err("%s: unsupported cop1_op\n", __func__);
187 		break;
188 	}
189 
190 	return nextpc;
191 
192 unaligned:
193 	kvm_err("%s: unaligned epc\n", __func__);
194 	return nextpc;
195 
196 sigill:
197 	kvm_err("%s: DSP branch but not DSP ASE\n", __func__);
198 	return nextpc;
199 }
200 
update_pc(struct kvm_vcpu * vcpu,uint32_t cause)201 enum emulation_result update_pc(struct kvm_vcpu *vcpu, uint32_t cause)
202 {
203 	unsigned long branch_pc;
204 	enum emulation_result er = EMULATE_DONE;
205 
206 	if (cause & CAUSEF_BD) {
207 		branch_pc = kvm_compute_return_epc(vcpu, vcpu->arch.pc);
208 		if (branch_pc == KVM_INVALID_INST) {
209 			er = EMULATE_FAIL;
210 		} else {
211 			vcpu->arch.pc = branch_pc;
212 			kvm_debug("BD update_pc(): New PC: %#lx\n",
213 				  vcpu->arch.pc);
214 		}
215 	} else
216 		vcpu->arch.pc += 4;
217 
218 	kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
219 
220 	return er;
221 }
222 
223 /**
224  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
225  * @vcpu:	Virtual CPU.
226  *
227  * Returns:	1 if the CP0_Count timer is disabled by either the guest
228  *		CP0_Cause.DC bit or the count_ctl.DC bit.
229  *		0 otherwise (in which case CP0_Count timer is running).
230  */
kvm_mips_count_disabled(struct kvm_vcpu * vcpu)231 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
232 {
233 	struct mips_coproc *cop0 = vcpu->arch.cop0;
234 
235 	return	(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
236 		(kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
237 }
238 
239 /**
240  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
241  *
242  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
243  *
244  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
245  */
kvm_mips_ktime_to_count(struct kvm_vcpu * vcpu,ktime_t now)246 static uint32_t kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
247 {
248 	s64 now_ns, periods;
249 	u64 delta;
250 
251 	now_ns = ktime_to_ns(now);
252 	delta = now_ns + vcpu->arch.count_dyn_bias;
253 
254 	if (delta >= vcpu->arch.count_period) {
255 		/* If delta is out of safe range the bias needs adjusting */
256 		periods = div64_s64(now_ns, vcpu->arch.count_period);
257 		vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
258 		/* Recalculate delta with new bias */
259 		delta = now_ns + vcpu->arch.count_dyn_bias;
260 	}
261 
262 	/*
263 	 * We've ensured that:
264 	 *   delta < count_period
265 	 *
266 	 * Therefore the intermediate delta*count_hz will never overflow since
267 	 * at the boundary condition:
268 	 *   delta = count_period
269 	 *   delta = NSEC_PER_SEC * 2^32 / count_hz
270 	 *   delta * count_hz = NSEC_PER_SEC * 2^32
271 	 */
272 	return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
273 }
274 
275 /**
276  * kvm_mips_count_time() - Get effective current time.
277  * @vcpu:	Virtual CPU.
278  *
279  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
280  * except when the master disable bit is set in count_ctl, in which case it is
281  * count_resume, i.e. the time that the count was disabled.
282  *
283  * Returns:	Effective monotonic ktime for CP0_Count.
284  */
kvm_mips_count_time(struct kvm_vcpu * vcpu)285 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
286 {
287 	if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
288 		return vcpu->arch.count_resume;
289 
290 	return ktime_get();
291 }
292 
293 /**
294  * kvm_mips_read_count_running() - Read the current count value as if running.
295  * @vcpu:	Virtual CPU.
296  * @now:	Kernel time to read CP0_Count at.
297  *
298  * Returns the current guest CP0_Count register at time @now and handles if the
299  * timer interrupt is pending and hasn't been handled yet.
300  *
301  * Returns:	The current value of the guest CP0_Count register.
302  */
kvm_mips_read_count_running(struct kvm_vcpu * vcpu,ktime_t now)303 static uint32_t kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
304 {
305 	struct mips_coproc *cop0 = vcpu->arch.cop0;
306 	ktime_t expires, threshold;
307 	uint32_t count, compare;
308 	int running;
309 
310 	/* Calculate the biased and scaled guest CP0_Count */
311 	count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
312 	compare = kvm_read_c0_guest_compare(cop0);
313 
314 	/*
315 	 * Find whether CP0_Count has reached the closest timer interrupt. If
316 	 * not, we shouldn't inject it.
317 	 */
318 	if ((int32_t)(count - compare) < 0)
319 		return count;
320 
321 	/*
322 	 * The CP0_Count we're going to return has already reached the closest
323 	 * timer interrupt. Quickly check if it really is a new interrupt by
324 	 * looking at whether the interval until the hrtimer expiry time is
325 	 * less than 1/4 of the timer period.
326 	 */
327 	expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
328 	threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
329 	if (ktime_before(expires, threshold)) {
330 		/*
331 		 * Cancel it while we handle it so there's no chance of
332 		 * interference with the timeout handler.
333 		 */
334 		running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
335 
336 		/* Nothing should be waiting on the timeout */
337 		kvm_mips_callbacks->queue_timer_int(vcpu);
338 
339 		/*
340 		 * Restart the timer if it was running based on the expiry time
341 		 * we read, so that we don't push it back 2 periods.
342 		 */
343 		if (running) {
344 			expires = ktime_add_ns(expires,
345 					       vcpu->arch.count_period);
346 			hrtimer_start(&vcpu->arch.comparecount_timer, expires,
347 				      HRTIMER_MODE_ABS);
348 		}
349 	}
350 
351 	return count;
352 }
353 
354 /**
355  * kvm_mips_read_count() - Read the current count value.
356  * @vcpu:	Virtual CPU.
357  *
358  * Read the current guest CP0_Count value, taking into account whether the timer
359  * is stopped.
360  *
361  * Returns:	The current guest CP0_Count value.
362  */
kvm_mips_read_count(struct kvm_vcpu * vcpu)363 uint32_t kvm_mips_read_count(struct kvm_vcpu *vcpu)
364 {
365 	struct mips_coproc *cop0 = vcpu->arch.cop0;
366 
367 	/* If count disabled just read static copy of count */
368 	if (kvm_mips_count_disabled(vcpu))
369 		return kvm_read_c0_guest_count(cop0);
370 
371 	return kvm_mips_read_count_running(vcpu, ktime_get());
372 }
373 
374 /**
375  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
376  * @vcpu:	Virtual CPU.
377  * @count:	Output pointer for CP0_Count value at point of freeze.
378  *
379  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
380  * at the point it was frozen. It is guaranteed that any pending interrupts at
381  * the point it was frozen are handled, and none after that point.
382  *
383  * This is useful where the time/CP0_Count is needed in the calculation of the
384  * new parameters.
385  *
386  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
387  *
388  * Returns:	The ktime at the point of freeze.
389  */
kvm_mips_freeze_hrtimer(struct kvm_vcpu * vcpu,uint32_t * count)390 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu,
391 				       uint32_t *count)
392 {
393 	ktime_t now;
394 
395 	/* stop hrtimer before finding time */
396 	hrtimer_cancel(&vcpu->arch.comparecount_timer);
397 	now = ktime_get();
398 
399 	/* find count at this point and handle pending hrtimer */
400 	*count = kvm_mips_read_count_running(vcpu, now);
401 
402 	return now;
403 }
404 
405 /**
406  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
407  * @vcpu:	Virtual CPU.
408  * @now:	ktime at point of resume.
409  * @count:	CP0_Count at point of resume.
410  *
411  * Resumes the timer and updates the timer expiry based on @now and @count.
412  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
413  * parameters need to be changed.
414  *
415  * It is guaranteed that a timer interrupt immediately after resume will be
416  * handled, but not if CP_Compare is exactly at @count. That case is already
417  * handled by kvm_mips_freeze_timer().
418  *
419  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
420  */
kvm_mips_resume_hrtimer(struct kvm_vcpu * vcpu,ktime_t now,uint32_t count)421 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
422 				    ktime_t now, uint32_t count)
423 {
424 	struct mips_coproc *cop0 = vcpu->arch.cop0;
425 	uint32_t compare;
426 	u64 delta;
427 	ktime_t expire;
428 
429 	/* Calculate timeout (wrap 0 to 2^32) */
430 	compare = kvm_read_c0_guest_compare(cop0);
431 	delta = (u64)(uint32_t)(compare - count - 1) + 1;
432 	delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
433 	expire = ktime_add_ns(now, delta);
434 
435 	/* Update hrtimer to use new timeout */
436 	hrtimer_cancel(&vcpu->arch.comparecount_timer);
437 	hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
438 }
439 
440 /**
441  * kvm_mips_write_count() - Modify the count and update timer.
442  * @vcpu:	Virtual CPU.
443  * @count:	Guest CP0_Count value to set.
444  *
445  * Sets the CP0_Count value and updates the timer accordingly.
446  */
kvm_mips_write_count(struct kvm_vcpu * vcpu,uint32_t count)447 void kvm_mips_write_count(struct kvm_vcpu *vcpu, uint32_t count)
448 {
449 	struct mips_coproc *cop0 = vcpu->arch.cop0;
450 	ktime_t now;
451 
452 	/* Calculate bias */
453 	now = kvm_mips_count_time(vcpu);
454 	vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
455 
456 	if (kvm_mips_count_disabled(vcpu))
457 		/* The timer's disabled, adjust the static count */
458 		kvm_write_c0_guest_count(cop0, count);
459 	else
460 		/* Update timeout */
461 		kvm_mips_resume_hrtimer(vcpu, now, count);
462 }
463 
464 /**
465  * kvm_mips_init_count() - Initialise timer.
466  * @vcpu:	Virtual CPU.
467  *
468  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
469  * it going if it's enabled.
470  */
kvm_mips_init_count(struct kvm_vcpu * vcpu)471 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
472 {
473 	/* 100 MHz */
474 	vcpu->arch.count_hz = 100*1000*1000;
475 	vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
476 					  vcpu->arch.count_hz);
477 	vcpu->arch.count_dyn_bias = 0;
478 
479 	/* Starting at 0 */
480 	kvm_mips_write_count(vcpu, 0);
481 }
482 
483 /**
484  * kvm_mips_set_count_hz() - Update the frequency of the timer.
485  * @vcpu:	Virtual CPU.
486  * @count_hz:	Frequency of CP0_Count timer in Hz.
487  *
488  * Change the frequency of the CP0_Count timer. This is done atomically so that
489  * CP0_Count is continuous and no timer interrupt is lost.
490  *
491  * Returns:	-EINVAL if @count_hz is out of range.
492  *		0 on success.
493  */
kvm_mips_set_count_hz(struct kvm_vcpu * vcpu,s64 count_hz)494 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
495 {
496 	struct mips_coproc *cop0 = vcpu->arch.cop0;
497 	int dc;
498 	ktime_t now;
499 	u32 count;
500 
501 	/* ensure the frequency is in a sensible range... */
502 	if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
503 		return -EINVAL;
504 	/* ... and has actually changed */
505 	if (vcpu->arch.count_hz == count_hz)
506 		return 0;
507 
508 	/* Safely freeze timer so we can keep it continuous */
509 	dc = kvm_mips_count_disabled(vcpu);
510 	if (dc) {
511 		now = kvm_mips_count_time(vcpu);
512 		count = kvm_read_c0_guest_count(cop0);
513 	} else {
514 		now = kvm_mips_freeze_hrtimer(vcpu, &count);
515 	}
516 
517 	/* Update the frequency */
518 	vcpu->arch.count_hz = count_hz;
519 	vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
520 	vcpu->arch.count_dyn_bias = 0;
521 
522 	/* Calculate adjusted bias so dynamic count is unchanged */
523 	vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
524 
525 	/* Update and resume hrtimer */
526 	if (!dc)
527 		kvm_mips_resume_hrtimer(vcpu, now, count);
528 	return 0;
529 }
530 
531 /**
532  * kvm_mips_write_compare() - Modify compare and update timer.
533  * @vcpu:	Virtual CPU.
534  * @compare:	New CP0_Compare value.
535  * @ack:	Whether to acknowledge timer interrupt.
536  *
537  * Update CP0_Compare to a new value and update the timeout.
538  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
539  * any pending timer interrupt is preserved.
540  */
kvm_mips_write_compare(struct kvm_vcpu * vcpu,uint32_t compare,bool ack)541 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, uint32_t compare, bool ack)
542 {
543 	struct mips_coproc *cop0 = vcpu->arch.cop0;
544 	int dc;
545 	u32 old_compare = kvm_read_c0_guest_compare(cop0);
546 	ktime_t now;
547 	uint32_t count;
548 
549 	/* if unchanged, must just be an ack */
550 	if (old_compare == compare) {
551 		if (!ack)
552 			return;
553 		kvm_mips_callbacks->dequeue_timer_int(vcpu);
554 		kvm_write_c0_guest_compare(cop0, compare);
555 		return;
556 	}
557 
558 	/* freeze_hrtimer() takes care of timer interrupts <= count */
559 	dc = kvm_mips_count_disabled(vcpu);
560 	if (!dc)
561 		now = kvm_mips_freeze_hrtimer(vcpu, &count);
562 
563 	if (ack)
564 		kvm_mips_callbacks->dequeue_timer_int(vcpu);
565 
566 	kvm_write_c0_guest_compare(cop0, compare);
567 
568 	/* resume_hrtimer() takes care of timer interrupts > count */
569 	if (!dc)
570 		kvm_mips_resume_hrtimer(vcpu, now, count);
571 }
572 
573 /**
574  * kvm_mips_count_disable() - Disable count.
575  * @vcpu:	Virtual CPU.
576  *
577  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
578  * time will be handled but not after.
579  *
580  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
581  * count_ctl.DC has been set (count disabled).
582  *
583  * Returns:	The time that the timer was stopped.
584  */
kvm_mips_count_disable(struct kvm_vcpu * vcpu)585 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
586 {
587 	struct mips_coproc *cop0 = vcpu->arch.cop0;
588 	uint32_t count;
589 	ktime_t now;
590 
591 	/* Stop hrtimer */
592 	hrtimer_cancel(&vcpu->arch.comparecount_timer);
593 
594 	/* Set the static count from the dynamic count, handling pending TI */
595 	now = ktime_get();
596 	count = kvm_mips_read_count_running(vcpu, now);
597 	kvm_write_c0_guest_count(cop0, count);
598 
599 	return now;
600 }
601 
602 /**
603  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
604  * @vcpu:	Virtual CPU.
605  *
606  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
607  * before the final stop time will be handled if the timer isn't disabled by
608  * count_ctl.DC, but not after.
609  *
610  * Assumes CP0_Cause.DC is clear (count enabled).
611  */
kvm_mips_count_disable_cause(struct kvm_vcpu * vcpu)612 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
613 {
614 	struct mips_coproc *cop0 = vcpu->arch.cop0;
615 
616 	kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
617 	if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
618 		kvm_mips_count_disable(vcpu);
619 }
620 
621 /**
622  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
623  * @vcpu:	Virtual CPU.
624  *
625  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
626  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
627  * potentially before even returning, so the caller should be careful with
628  * ordering of CP0_Cause modifications so as not to lose it.
629  *
630  * Assumes CP0_Cause.DC is set (count disabled).
631  */
kvm_mips_count_enable_cause(struct kvm_vcpu * vcpu)632 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
633 {
634 	struct mips_coproc *cop0 = vcpu->arch.cop0;
635 	uint32_t count;
636 
637 	kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
638 
639 	/*
640 	 * Set the dynamic count to match the static count.
641 	 * This starts the hrtimer if count_ctl.DC allows it.
642 	 * Otherwise it conveniently updates the biases.
643 	 */
644 	count = kvm_read_c0_guest_count(cop0);
645 	kvm_mips_write_count(vcpu, count);
646 }
647 
648 /**
649  * kvm_mips_set_count_ctl() - Update the count control KVM register.
650  * @vcpu:	Virtual CPU.
651  * @count_ctl:	Count control register new value.
652  *
653  * Set the count control KVM register. The timer is updated accordingly.
654  *
655  * Returns:	-EINVAL if reserved bits are set.
656  *		0 on success.
657  */
kvm_mips_set_count_ctl(struct kvm_vcpu * vcpu,s64 count_ctl)658 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
659 {
660 	struct mips_coproc *cop0 = vcpu->arch.cop0;
661 	s64 changed = count_ctl ^ vcpu->arch.count_ctl;
662 	s64 delta;
663 	ktime_t expire, now;
664 	uint32_t count, compare;
665 
666 	/* Only allow defined bits to be changed */
667 	if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
668 		return -EINVAL;
669 
670 	/* Apply new value */
671 	vcpu->arch.count_ctl = count_ctl;
672 
673 	/* Master CP0_Count disable */
674 	if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
675 		/* Is CP0_Cause.DC already disabling CP0_Count? */
676 		if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
677 			if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
678 				/* Just record the current time */
679 				vcpu->arch.count_resume = ktime_get();
680 		} else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
681 			/* disable timer and record current time */
682 			vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
683 		} else {
684 			/*
685 			 * Calculate timeout relative to static count at resume
686 			 * time (wrap 0 to 2^32).
687 			 */
688 			count = kvm_read_c0_guest_count(cop0);
689 			compare = kvm_read_c0_guest_compare(cop0);
690 			delta = (u64)(uint32_t)(compare - count - 1) + 1;
691 			delta = div_u64(delta * NSEC_PER_SEC,
692 					vcpu->arch.count_hz);
693 			expire = ktime_add_ns(vcpu->arch.count_resume, delta);
694 
695 			/* Handle pending interrupt */
696 			now = ktime_get();
697 			if (ktime_compare(now, expire) >= 0)
698 				/* Nothing should be waiting on the timeout */
699 				kvm_mips_callbacks->queue_timer_int(vcpu);
700 
701 			/* Resume hrtimer without changing bias */
702 			count = kvm_mips_read_count_running(vcpu, now);
703 			kvm_mips_resume_hrtimer(vcpu, now, count);
704 		}
705 	}
706 
707 	return 0;
708 }
709 
710 /**
711  * kvm_mips_set_count_resume() - Update the count resume KVM register.
712  * @vcpu:		Virtual CPU.
713  * @count_resume:	Count resume register new value.
714  *
715  * Set the count resume KVM register.
716  *
717  * Returns:	-EINVAL if out of valid range (0..now).
718  *		0 on success.
719  */
kvm_mips_set_count_resume(struct kvm_vcpu * vcpu,s64 count_resume)720 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
721 {
722 	/*
723 	 * It doesn't make sense for the resume time to be in the future, as it
724 	 * would be possible for the next interrupt to be more than a full
725 	 * period in the future.
726 	 */
727 	if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
728 		return -EINVAL;
729 
730 	vcpu->arch.count_resume = ns_to_ktime(count_resume);
731 	return 0;
732 }
733 
734 /**
735  * kvm_mips_count_timeout() - Push timer forward on timeout.
736  * @vcpu:	Virtual CPU.
737  *
738  * Handle an hrtimer event by push the hrtimer forward a period.
739  *
740  * Returns:	The hrtimer_restart value to return to the hrtimer subsystem.
741  */
kvm_mips_count_timeout(struct kvm_vcpu * vcpu)742 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
743 {
744 	/* Add the Count period to the current expiry time */
745 	hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
746 			       vcpu->arch.count_period);
747 	return HRTIMER_RESTART;
748 }
749 
kvm_mips_emul_eret(struct kvm_vcpu * vcpu)750 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
751 {
752 	struct mips_coproc *cop0 = vcpu->arch.cop0;
753 	enum emulation_result er = EMULATE_DONE;
754 
755 	if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
756 		kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
757 			  kvm_read_c0_guest_epc(cop0));
758 		kvm_clear_c0_guest_status(cop0, ST0_EXL);
759 		vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
760 
761 	} else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
762 		kvm_clear_c0_guest_status(cop0, ST0_ERL);
763 		vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
764 	} else {
765 		kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
766 			vcpu->arch.pc);
767 		er = EMULATE_FAIL;
768 	}
769 
770 	return er;
771 }
772 
kvm_mips_emul_wait(struct kvm_vcpu * vcpu)773 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
774 {
775 	kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
776 		  vcpu->arch.pending_exceptions);
777 
778 	++vcpu->stat.wait_exits;
779 	trace_kvm_exit(vcpu, WAIT_EXITS);
780 	if (!vcpu->arch.pending_exceptions) {
781 		vcpu->arch.wait = 1;
782 		kvm_vcpu_block(vcpu);
783 
784 		/*
785 		 * We we are runnable, then definitely go off to user space to
786 		 * check if any I/O interrupts are pending.
787 		 */
788 		if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
789 			clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
790 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
791 		}
792 	}
793 
794 	return EMULATE_DONE;
795 }
796 
797 /*
798  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
799  * we can catch this, if things ever change
800  */
kvm_mips_emul_tlbr(struct kvm_vcpu * vcpu)801 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
802 {
803 	struct mips_coproc *cop0 = vcpu->arch.cop0;
804 	uint32_t pc = vcpu->arch.pc;
805 
806 	kvm_err("[%#x] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
807 	return EMULATE_FAIL;
808 }
809 
810 /* Write Guest TLB Entry @ Index */
kvm_mips_emul_tlbwi(struct kvm_vcpu * vcpu)811 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
812 {
813 	struct mips_coproc *cop0 = vcpu->arch.cop0;
814 	int index = kvm_read_c0_guest_index(cop0);
815 	struct kvm_mips_tlb *tlb = NULL;
816 	uint32_t pc = vcpu->arch.pc;
817 
818 	if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
819 		kvm_debug("%s: illegal index: %d\n", __func__, index);
820 		kvm_debug("[%#x] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
821 			  pc, index, kvm_read_c0_guest_entryhi(cop0),
822 			  kvm_read_c0_guest_entrylo0(cop0),
823 			  kvm_read_c0_guest_entrylo1(cop0),
824 			  kvm_read_c0_guest_pagemask(cop0));
825 		index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
826 	}
827 
828 	tlb = &vcpu->arch.guest_tlb[index];
829 	/*
830 	 * Probe the shadow host TLB for the entry being overwritten, if one
831 	 * matches, invalidate it
832 	 */
833 	kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
834 
835 	tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
836 	tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
837 	tlb->tlb_lo0 = kvm_read_c0_guest_entrylo0(cop0);
838 	tlb->tlb_lo1 = kvm_read_c0_guest_entrylo1(cop0);
839 
840 	kvm_debug("[%#x] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
841 		  pc, index, kvm_read_c0_guest_entryhi(cop0),
842 		  kvm_read_c0_guest_entrylo0(cop0),
843 		  kvm_read_c0_guest_entrylo1(cop0),
844 		  kvm_read_c0_guest_pagemask(cop0));
845 
846 	return EMULATE_DONE;
847 }
848 
849 /* Write Guest TLB Entry @ Random Index */
kvm_mips_emul_tlbwr(struct kvm_vcpu * vcpu)850 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
851 {
852 	struct mips_coproc *cop0 = vcpu->arch.cop0;
853 	struct kvm_mips_tlb *tlb = NULL;
854 	uint32_t pc = vcpu->arch.pc;
855 	int index;
856 
857 	get_random_bytes(&index, sizeof(index));
858 	index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
859 
860 	tlb = &vcpu->arch.guest_tlb[index];
861 
862 	/*
863 	 * Probe the shadow host TLB for the entry being overwritten, if one
864 	 * matches, invalidate it
865 	 */
866 	kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
867 
868 	tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
869 	tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
870 	tlb->tlb_lo0 = kvm_read_c0_guest_entrylo0(cop0);
871 	tlb->tlb_lo1 = kvm_read_c0_guest_entrylo1(cop0);
872 
873 	kvm_debug("[%#x] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
874 		  pc, index, kvm_read_c0_guest_entryhi(cop0),
875 		  kvm_read_c0_guest_entrylo0(cop0),
876 		  kvm_read_c0_guest_entrylo1(cop0));
877 
878 	return EMULATE_DONE;
879 }
880 
kvm_mips_emul_tlbp(struct kvm_vcpu * vcpu)881 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
882 {
883 	struct mips_coproc *cop0 = vcpu->arch.cop0;
884 	long entryhi = kvm_read_c0_guest_entryhi(cop0);
885 	uint32_t pc = vcpu->arch.pc;
886 	int index = -1;
887 
888 	index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
889 
890 	kvm_write_c0_guest_index(cop0, index);
891 
892 	kvm_debug("[%#x] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
893 		  index);
894 
895 	return EMULATE_DONE;
896 }
897 
898 /**
899  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
900  * @vcpu:	Virtual CPU.
901  *
902  * Finds the mask of bits which are writable in the guest's Config1 CP0
903  * register, by userland (currently read-only to the guest).
904  */
kvm_mips_config1_wrmask(struct kvm_vcpu * vcpu)905 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
906 {
907 	unsigned int mask = 0;
908 
909 	/* Permit FPU to be present if FPU is supported */
910 	if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
911 		mask |= MIPS_CONF1_FP;
912 
913 	return mask;
914 }
915 
916 /**
917  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
918  * @vcpu:	Virtual CPU.
919  *
920  * Finds the mask of bits which are writable in the guest's Config3 CP0
921  * register, by userland (currently read-only to the guest).
922  */
kvm_mips_config3_wrmask(struct kvm_vcpu * vcpu)923 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
924 {
925 	/* Config4 is optional */
926 	unsigned int mask = MIPS_CONF_M;
927 
928 	/* Permit MSA to be present if MSA is supported */
929 	if (kvm_mips_guest_can_have_msa(&vcpu->arch))
930 		mask |= MIPS_CONF3_MSA;
931 
932 	return mask;
933 }
934 
935 /**
936  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
937  * @vcpu:	Virtual CPU.
938  *
939  * Finds the mask of bits which are writable in the guest's Config4 CP0
940  * register, by userland (currently read-only to the guest).
941  */
kvm_mips_config4_wrmask(struct kvm_vcpu * vcpu)942 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
943 {
944 	/* Config5 is optional */
945 	return MIPS_CONF_M;
946 }
947 
948 /**
949  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
950  * @vcpu:	Virtual CPU.
951  *
952  * Finds the mask of bits which are writable in the guest's Config5 CP0
953  * register, by the guest itself.
954  */
kvm_mips_config5_wrmask(struct kvm_vcpu * vcpu)955 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
956 {
957 	unsigned int mask = 0;
958 
959 	/* Permit MSAEn changes if MSA supported and enabled */
960 	if (kvm_mips_guest_has_msa(&vcpu->arch))
961 		mask |= MIPS_CONF5_MSAEN;
962 
963 	/*
964 	 * Permit guest FPU mode changes if FPU is enabled and the relevant
965 	 * feature exists according to FIR register.
966 	 */
967 	if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
968 		if (cpu_has_fre)
969 			mask |= MIPS_CONF5_FRE;
970 		/* We don't support UFR or UFE */
971 	}
972 
973 	return mask;
974 }
975 
kvm_mips_emulate_CP0(uint32_t inst,uint32_t * opc,uint32_t cause,struct kvm_run * run,struct kvm_vcpu * vcpu)976 enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
977 					   uint32_t cause, struct kvm_run *run,
978 					   struct kvm_vcpu *vcpu)
979 {
980 	struct mips_coproc *cop0 = vcpu->arch.cop0;
981 	enum emulation_result er = EMULATE_DONE;
982 	int32_t rt, rd, copz, sel, co_bit, op;
983 	uint32_t pc = vcpu->arch.pc;
984 	unsigned long curr_pc;
985 
986 	/*
987 	 * Update PC and hold onto current PC in case there is
988 	 * an error and we want to rollback the PC
989 	 */
990 	curr_pc = vcpu->arch.pc;
991 	er = update_pc(vcpu, cause);
992 	if (er == EMULATE_FAIL)
993 		return er;
994 
995 	copz = (inst >> 21) & 0x1f;
996 	rt = (inst >> 16) & 0x1f;
997 	rd = (inst >> 11) & 0x1f;
998 	sel = inst & 0x7;
999 	co_bit = (inst >> 25) & 1;
1000 
1001 	if (co_bit) {
1002 		op = (inst) & 0xff;
1003 
1004 		switch (op) {
1005 		case tlbr_op:	/*  Read indexed TLB entry  */
1006 			er = kvm_mips_emul_tlbr(vcpu);
1007 			break;
1008 		case tlbwi_op:	/*  Write indexed  */
1009 			er = kvm_mips_emul_tlbwi(vcpu);
1010 			break;
1011 		case tlbwr_op:	/*  Write random  */
1012 			er = kvm_mips_emul_tlbwr(vcpu);
1013 			break;
1014 		case tlbp_op:	/* TLB Probe */
1015 			er = kvm_mips_emul_tlbp(vcpu);
1016 			break;
1017 		case rfe_op:
1018 			kvm_err("!!!COP0_RFE!!!\n");
1019 			break;
1020 		case eret_op:
1021 			er = kvm_mips_emul_eret(vcpu);
1022 			goto dont_update_pc;
1023 			break;
1024 		case wait_op:
1025 			er = kvm_mips_emul_wait(vcpu);
1026 			break;
1027 		}
1028 	} else {
1029 		switch (copz) {
1030 		case mfc_op:
1031 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1032 			cop0->stat[rd][sel]++;
1033 #endif
1034 			/* Get reg */
1035 			if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1036 				vcpu->arch.gprs[rt] = kvm_mips_read_count(vcpu);
1037 			} else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1038 				vcpu->arch.gprs[rt] = 0x0;
1039 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1040 				kvm_mips_trans_mfc0(inst, opc, vcpu);
1041 #endif
1042 			} else {
1043 				vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1044 
1045 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1046 				kvm_mips_trans_mfc0(inst, opc, vcpu);
1047 #endif
1048 			}
1049 
1050 			kvm_debug
1051 			    ("[%#x] MFCz[%d][%d], vcpu->arch.gprs[%d]: %#lx\n",
1052 			     pc, rd, sel, rt, vcpu->arch.gprs[rt]);
1053 
1054 			break;
1055 
1056 		case dmfc_op:
1057 			vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1058 			break;
1059 
1060 		case mtc_op:
1061 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1062 			cop0->stat[rd][sel]++;
1063 #endif
1064 			if ((rd == MIPS_CP0_TLB_INDEX)
1065 			    && (vcpu->arch.gprs[rt] >=
1066 				KVM_MIPS_GUEST_TLB_SIZE)) {
1067 				kvm_err("Invalid TLB Index: %ld",
1068 					vcpu->arch.gprs[rt]);
1069 				er = EMULATE_FAIL;
1070 				break;
1071 			}
1072 #define C0_EBASE_CORE_MASK 0xff
1073 			if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1074 				/* Preserve CORE number */
1075 				kvm_change_c0_guest_ebase(cop0,
1076 							  ~(C0_EBASE_CORE_MASK),
1077 							  vcpu->arch.gprs[rt]);
1078 				kvm_err("MTCz, cop0->reg[EBASE]: %#lx\n",
1079 					kvm_read_c0_guest_ebase(cop0));
1080 			} else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1081 				uint32_t nasid =
1082 					vcpu->arch.gprs[rt] & ASID_MASK;
1083 				if ((KSEGX(vcpu->arch.gprs[rt]) != CKSEG0) &&
1084 				    ((kvm_read_c0_guest_entryhi(cop0) &
1085 				      ASID_MASK) != nasid)) {
1086 					kvm_debug("MTCz, change ASID from %#lx to %#lx\n",
1087 						kvm_read_c0_guest_entryhi(cop0)
1088 						& ASID_MASK,
1089 						vcpu->arch.gprs[rt]
1090 						& ASID_MASK);
1091 
1092 					/* Blow away the shadow host TLBs */
1093 					kvm_mips_flush_host_tlb(1);
1094 				}
1095 				kvm_write_c0_guest_entryhi(cop0,
1096 							   vcpu->arch.gprs[rt]);
1097 			}
1098 			/* Are we writing to COUNT */
1099 			else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1100 				kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1101 				goto done;
1102 			} else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1103 				kvm_debug("[%#x] MTCz, COMPARE %#lx <- %#lx\n",
1104 					  pc, kvm_read_c0_guest_compare(cop0),
1105 					  vcpu->arch.gprs[rt]);
1106 
1107 				/* If we are writing to COMPARE */
1108 				/* Clear pending timer interrupt, if any */
1109 				kvm_mips_write_compare(vcpu,
1110 						       vcpu->arch.gprs[rt],
1111 						       true);
1112 			} else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1113 				unsigned int old_val, val, change;
1114 
1115 				old_val = kvm_read_c0_guest_status(cop0);
1116 				val = vcpu->arch.gprs[rt];
1117 				change = val ^ old_val;
1118 
1119 				/* Make sure that the NMI bit is never set */
1120 				val &= ~ST0_NMI;
1121 
1122 				/*
1123 				 * Don't allow CU1 or FR to be set unless FPU
1124 				 * capability enabled and exists in guest
1125 				 * configuration.
1126 				 */
1127 				if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1128 					val &= ~(ST0_CU1 | ST0_FR);
1129 
1130 				/*
1131 				 * Also don't allow FR to be set if host doesn't
1132 				 * support it.
1133 				 */
1134 				if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1135 					val &= ~ST0_FR;
1136 
1137 
1138 				/* Handle changes in FPU mode */
1139 				preempt_disable();
1140 
1141 				/*
1142 				 * FPU and Vector register state is made
1143 				 * UNPREDICTABLE by a change of FR, so don't
1144 				 * even bother saving it.
1145 				 */
1146 				if (change & ST0_FR)
1147 					kvm_drop_fpu(vcpu);
1148 
1149 				/*
1150 				 * If MSA state is already live, it is undefined
1151 				 * how it interacts with FR=0 FPU state, and we
1152 				 * don't want to hit reserved instruction
1153 				 * exceptions trying to save the MSA state later
1154 				 * when CU=1 && FR=1, so play it safe and save
1155 				 * it first.
1156 				 */
1157 				if (change & ST0_CU1 && !(val & ST0_FR) &&
1158 				    vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1159 					kvm_lose_fpu(vcpu);
1160 
1161 				/*
1162 				 * Propagate CU1 (FPU enable) changes
1163 				 * immediately if the FPU context is already
1164 				 * loaded. When disabling we leave the context
1165 				 * loaded so it can be quickly enabled again in
1166 				 * the near future.
1167 				 */
1168 				if (change & ST0_CU1 &&
1169 				    vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1170 					change_c0_status(ST0_CU1, val);
1171 
1172 				preempt_enable();
1173 
1174 				kvm_write_c0_guest_status(cop0, val);
1175 
1176 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1177 				/*
1178 				 * If FPU present, we need CU1/FR bits to take
1179 				 * effect fairly soon.
1180 				 */
1181 				if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1182 					kvm_mips_trans_mtc0(inst, opc, vcpu);
1183 #endif
1184 			} else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1185 				unsigned int old_val, val, change, wrmask;
1186 
1187 				old_val = kvm_read_c0_guest_config5(cop0);
1188 				val = vcpu->arch.gprs[rt];
1189 
1190 				/* Only a few bits are writable in Config5 */
1191 				wrmask = kvm_mips_config5_wrmask(vcpu);
1192 				change = (val ^ old_val) & wrmask;
1193 				val = old_val ^ change;
1194 
1195 
1196 				/* Handle changes in FPU/MSA modes */
1197 				preempt_disable();
1198 
1199 				/*
1200 				 * Propagate FRE changes immediately if the FPU
1201 				 * context is already loaded.
1202 				 */
1203 				if (change & MIPS_CONF5_FRE &&
1204 				    vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
1205 					change_c0_config5(MIPS_CONF5_FRE, val);
1206 
1207 				/*
1208 				 * Propagate MSAEn changes immediately if the
1209 				 * MSA context is already loaded. When disabling
1210 				 * we leave the context loaded so it can be
1211 				 * quickly enabled again in the near future.
1212 				 */
1213 				if (change & MIPS_CONF5_MSAEN &&
1214 				    vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
1215 					change_c0_config5(MIPS_CONF5_MSAEN,
1216 							  val);
1217 
1218 				preempt_enable();
1219 
1220 				kvm_write_c0_guest_config5(cop0, val);
1221 			} else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1222 				uint32_t old_cause, new_cause;
1223 
1224 				old_cause = kvm_read_c0_guest_cause(cop0);
1225 				new_cause = vcpu->arch.gprs[rt];
1226 				/* Update R/W bits */
1227 				kvm_change_c0_guest_cause(cop0, 0x08800300,
1228 							  new_cause);
1229 				/* DC bit enabling/disabling timer? */
1230 				if ((old_cause ^ new_cause) & CAUSEF_DC) {
1231 					if (new_cause & CAUSEF_DC)
1232 						kvm_mips_count_disable_cause(vcpu);
1233 					else
1234 						kvm_mips_count_enable_cause(vcpu);
1235 				}
1236 			} else {
1237 				cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1238 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1239 				kvm_mips_trans_mtc0(inst, opc, vcpu);
1240 #endif
1241 			}
1242 
1243 			kvm_debug("[%#x] MTCz, cop0->reg[%d][%d]: %#lx\n", pc,
1244 				  rd, sel, cop0->reg[rd][sel]);
1245 			break;
1246 
1247 		case dmtc_op:
1248 			kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1249 				vcpu->arch.pc, rt, rd, sel);
1250 			er = EMULATE_FAIL;
1251 			break;
1252 
1253 		case mfmcz_op:
1254 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1255 			cop0->stat[MIPS_CP0_STATUS][0]++;
1256 #endif
1257 			if (rt != 0) {
1258 				vcpu->arch.gprs[rt] =
1259 				    kvm_read_c0_guest_status(cop0);
1260 			}
1261 			/* EI */
1262 			if (inst & 0x20) {
1263 				kvm_debug("[%#lx] mfmcz_op: EI\n",
1264 					  vcpu->arch.pc);
1265 				kvm_set_c0_guest_status(cop0, ST0_IE);
1266 			} else {
1267 				kvm_debug("[%#lx] mfmcz_op: DI\n",
1268 					  vcpu->arch.pc);
1269 				kvm_clear_c0_guest_status(cop0, ST0_IE);
1270 			}
1271 
1272 			break;
1273 
1274 		case wrpgpr_op:
1275 			{
1276 				uint32_t css =
1277 				    cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1278 				uint32_t pss =
1279 				    (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1280 				/*
1281 				 * We don't support any shadow register sets, so
1282 				 * SRSCtl[PSS] == SRSCtl[CSS] = 0
1283 				 */
1284 				if (css || pss) {
1285 					er = EMULATE_FAIL;
1286 					break;
1287 				}
1288 				kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1289 					  vcpu->arch.gprs[rt]);
1290 				vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1291 			}
1292 			break;
1293 		default:
1294 			kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1295 				vcpu->arch.pc, copz);
1296 			er = EMULATE_FAIL;
1297 			break;
1298 		}
1299 	}
1300 
1301 done:
1302 	/* Rollback PC only if emulation was unsuccessful */
1303 	if (er == EMULATE_FAIL)
1304 		vcpu->arch.pc = curr_pc;
1305 
1306 dont_update_pc:
1307 	/*
1308 	 * This is for special instructions whose emulation
1309 	 * updates the PC, so do not overwrite the PC under
1310 	 * any circumstances
1311 	 */
1312 
1313 	return er;
1314 }
1315 
kvm_mips_emulate_store(uint32_t inst,uint32_t cause,struct kvm_run * run,struct kvm_vcpu * vcpu)1316 enum emulation_result kvm_mips_emulate_store(uint32_t inst, uint32_t cause,
1317 					     struct kvm_run *run,
1318 					     struct kvm_vcpu *vcpu)
1319 {
1320 	enum emulation_result er = EMULATE_DO_MMIO;
1321 	int32_t op, base, rt, offset;
1322 	uint32_t bytes;
1323 	void *data = run->mmio.data;
1324 	unsigned long curr_pc;
1325 
1326 	/*
1327 	 * Update PC and hold onto current PC in case there is
1328 	 * an error and we want to rollback the PC
1329 	 */
1330 	curr_pc = vcpu->arch.pc;
1331 	er = update_pc(vcpu, cause);
1332 	if (er == EMULATE_FAIL)
1333 		return er;
1334 
1335 	rt = (inst >> 16) & 0x1f;
1336 	base = (inst >> 21) & 0x1f;
1337 	offset = inst & 0xffff;
1338 	op = (inst >> 26) & 0x3f;
1339 
1340 	switch (op) {
1341 	case sb_op:
1342 		bytes = 1;
1343 		if (bytes > sizeof(run->mmio.data)) {
1344 			kvm_err("%s: bad MMIO length: %d\n", __func__,
1345 			       run->mmio.len);
1346 		}
1347 		run->mmio.phys_addr =
1348 		    kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1349 						   host_cp0_badvaddr);
1350 		if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1351 			er = EMULATE_FAIL;
1352 			break;
1353 		}
1354 		run->mmio.len = bytes;
1355 		run->mmio.is_write = 1;
1356 		vcpu->mmio_needed = 1;
1357 		vcpu->mmio_is_write = 1;
1358 		*(u8 *) data = vcpu->arch.gprs[rt];
1359 		kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1360 			  vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt],
1361 			  *(uint8_t *) data);
1362 
1363 		break;
1364 
1365 	case sw_op:
1366 		bytes = 4;
1367 		if (bytes > sizeof(run->mmio.data)) {
1368 			kvm_err("%s: bad MMIO length: %d\n", __func__,
1369 			       run->mmio.len);
1370 		}
1371 		run->mmio.phys_addr =
1372 		    kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1373 						   host_cp0_badvaddr);
1374 		if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1375 			er = EMULATE_FAIL;
1376 			break;
1377 		}
1378 
1379 		run->mmio.len = bytes;
1380 		run->mmio.is_write = 1;
1381 		vcpu->mmio_needed = 1;
1382 		vcpu->mmio_is_write = 1;
1383 		*(uint32_t *) data = vcpu->arch.gprs[rt];
1384 
1385 		kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1386 			  vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1387 			  vcpu->arch.gprs[rt], *(uint32_t *) data);
1388 		break;
1389 
1390 	case sh_op:
1391 		bytes = 2;
1392 		if (bytes > sizeof(run->mmio.data)) {
1393 			kvm_err("%s: bad MMIO length: %d\n", __func__,
1394 			       run->mmio.len);
1395 		}
1396 		run->mmio.phys_addr =
1397 		    kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1398 						   host_cp0_badvaddr);
1399 		if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1400 			er = EMULATE_FAIL;
1401 			break;
1402 		}
1403 
1404 		run->mmio.len = bytes;
1405 		run->mmio.is_write = 1;
1406 		vcpu->mmio_needed = 1;
1407 		vcpu->mmio_is_write = 1;
1408 		*(uint16_t *) data = vcpu->arch.gprs[rt];
1409 
1410 		kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1411 			  vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1412 			  vcpu->arch.gprs[rt], *(uint32_t *) data);
1413 		break;
1414 
1415 	default:
1416 		kvm_err("Store not yet supported");
1417 		er = EMULATE_FAIL;
1418 		break;
1419 	}
1420 
1421 	/* Rollback PC if emulation was unsuccessful */
1422 	if (er == EMULATE_FAIL)
1423 		vcpu->arch.pc = curr_pc;
1424 
1425 	return er;
1426 }
1427 
kvm_mips_emulate_load(uint32_t inst,uint32_t cause,struct kvm_run * run,struct kvm_vcpu * vcpu)1428 enum emulation_result kvm_mips_emulate_load(uint32_t inst, uint32_t cause,
1429 					    struct kvm_run *run,
1430 					    struct kvm_vcpu *vcpu)
1431 {
1432 	enum emulation_result er = EMULATE_DO_MMIO;
1433 	int32_t op, base, rt, offset;
1434 	uint32_t bytes;
1435 
1436 	rt = (inst >> 16) & 0x1f;
1437 	base = (inst >> 21) & 0x1f;
1438 	offset = inst & 0xffff;
1439 	op = (inst >> 26) & 0x3f;
1440 
1441 	vcpu->arch.pending_load_cause = cause;
1442 	vcpu->arch.io_gpr = rt;
1443 
1444 	switch (op) {
1445 	case lw_op:
1446 		bytes = 4;
1447 		if (bytes > sizeof(run->mmio.data)) {
1448 			kvm_err("%s: bad MMIO length: %d\n", __func__,
1449 			       run->mmio.len);
1450 			er = EMULATE_FAIL;
1451 			break;
1452 		}
1453 		run->mmio.phys_addr =
1454 		    kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1455 						   host_cp0_badvaddr);
1456 		if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1457 			er = EMULATE_FAIL;
1458 			break;
1459 		}
1460 
1461 		run->mmio.len = bytes;
1462 		run->mmio.is_write = 0;
1463 		vcpu->mmio_needed = 1;
1464 		vcpu->mmio_is_write = 0;
1465 		break;
1466 
1467 	case lh_op:
1468 	case lhu_op:
1469 		bytes = 2;
1470 		if (bytes > sizeof(run->mmio.data)) {
1471 			kvm_err("%s: bad MMIO length: %d\n", __func__,
1472 			       run->mmio.len);
1473 			er = EMULATE_FAIL;
1474 			break;
1475 		}
1476 		run->mmio.phys_addr =
1477 		    kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1478 						   host_cp0_badvaddr);
1479 		if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1480 			er = EMULATE_FAIL;
1481 			break;
1482 		}
1483 
1484 		run->mmio.len = bytes;
1485 		run->mmio.is_write = 0;
1486 		vcpu->mmio_needed = 1;
1487 		vcpu->mmio_is_write = 0;
1488 
1489 		if (op == lh_op)
1490 			vcpu->mmio_needed = 2;
1491 		else
1492 			vcpu->mmio_needed = 1;
1493 
1494 		break;
1495 
1496 	case lbu_op:
1497 	case lb_op:
1498 		bytes = 1;
1499 		if (bytes > sizeof(run->mmio.data)) {
1500 			kvm_err("%s: bad MMIO length: %d\n", __func__,
1501 			       run->mmio.len);
1502 			er = EMULATE_FAIL;
1503 			break;
1504 		}
1505 		run->mmio.phys_addr =
1506 		    kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1507 						   host_cp0_badvaddr);
1508 		if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1509 			er = EMULATE_FAIL;
1510 			break;
1511 		}
1512 
1513 		run->mmio.len = bytes;
1514 		run->mmio.is_write = 0;
1515 		vcpu->mmio_is_write = 0;
1516 
1517 		if (op == lb_op)
1518 			vcpu->mmio_needed = 2;
1519 		else
1520 			vcpu->mmio_needed = 1;
1521 
1522 		break;
1523 
1524 	default:
1525 		kvm_err("Load not yet supported");
1526 		er = EMULATE_FAIL;
1527 		break;
1528 	}
1529 
1530 	return er;
1531 }
1532 
kvm_mips_sync_icache(unsigned long va,struct kvm_vcpu * vcpu)1533 int kvm_mips_sync_icache(unsigned long va, struct kvm_vcpu *vcpu)
1534 {
1535 	unsigned long offset = (va & ~PAGE_MASK);
1536 	struct kvm *kvm = vcpu->kvm;
1537 	unsigned long pa;
1538 	gfn_t gfn;
1539 	pfn_t pfn;
1540 
1541 	gfn = va >> PAGE_SHIFT;
1542 
1543 	if (gfn >= kvm->arch.guest_pmap_npages) {
1544 		kvm_err("%s: Invalid gfn: %#llx\n", __func__, gfn);
1545 		kvm_mips_dump_host_tlbs();
1546 		kvm_arch_vcpu_dump_regs(vcpu);
1547 		return -1;
1548 	}
1549 	pfn = kvm->arch.guest_pmap[gfn];
1550 	pa = (pfn << PAGE_SHIFT) | offset;
1551 
1552 	kvm_debug("%s: va: %#lx, unmapped: %#x\n", __func__, va,
1553 		  CKSEG0ADDR(pa));
1554 
1555 	local_flush_icache_range(CKSEG0ADDR(pa), 32);
1556 	return 0;
1557 }
1558 
1559 #define MIPS_CACHE_OP_INDEX_INV         0x0
1560 #define MIPS_CACHE_OP_INDEX_LD_TAG      0x1
1561 #define MIPS_CACHE_OP_INDEX_ST_TAG      0x2
1562 #define MIPS_CACHE_OP_IMP               0x3
1563 #define MIPS_CACHE_OP_HIT_INV           0x4
1564 #define MIPS_CACHE_OP_FILL_WB_INV       0x5
1565 #define MIPS_CACHE_OP_HIT_HB            0x6
1566 #define MIPS_CACHE_OP_FETCH_LOCK        0x7
1567 
1568 #define MIPS_CACHE_ICACHE               0x0
1569 #define MIPS_CACHE_DCACHE               0x1
1570 #define MIPS_CACHE_SEC                  0x3
1571 
kvm_mips_emulate_cache(uint32_t inst,uint32_t * opc,uint32_t cause,struct kvm_run * run,struct kvm_vcpu * vcpu)1572 enum emulation_result kvm_mips_emulate_cache(uint32_t inst, uint32_t *opc,
1573 					     uint32_t cause,
1574 					     struct kvm_run *run,
1575 					     struct kvm_vcpu *vcpu)
1576 {
1577 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1578 	enum emulation_result er = EMULATE_DONE;
1579 	int32_t offset, cache, op_inst, op, base;
1580 	struct kvm_vcpu_arch *arch = &vcpu->arch;
1581 	unsigned long va;
1582 	unsigned long curr_pc;
1583 
1584 	/*
1585 	 * Update PC and hold onto current PC in case there is
1586 	 * an error and we want to rollback the PC
1587 	 */
1588 	curr_pc = vcpu->arch.pc;
1589 	er = update_pc(vcpu, cause);
1590 	if (er == EMULATE_FAIL)
1591 		return er;
1592 
1593 	base = (inst >> 21) & 0x1f;
1594 	op_inst = (inst >> 16) & 0x1f;
1595 	offset = (int16_t)inst;
1596 	cache = (inst >> 16) & 0x3;
1597 	op = (inst >> 18) & 0x7;
1598 
1599 	va = arch->gprs[base] + offset;
1600 
1601 	kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1602 		  cache, op, base, arch->gprs[base], offset);
1603 
1604 	/*
1605 	 * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1606 	 * invalidate the caches entirely by stepping through all the
1607 	 * ways/indexes
1608 	 */
1609 	if (op == MIPS_CACHE_OP_INDEX_INV) {
1610 		kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1611 			  vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1612 			  arch->gprs[base], offset);
1613 
1614 		if (cache == MIPS_CACHE_DCACHE)
1615 			r4k_blast_dcache();
1616 		else if (cache == MIPS_CACHE_ICACHE)
1617 			r4k_blast_icache();
1618 		else {
1619 			kvm_err("%s: unsupported CACHE INDEX operation\n",
1620 				__func__);
1621 			return EMULATE_FAIL;
1622 		}
1623 
1624 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1625 		kvm_mips_trans_cache_index(inst, opc, vcpu);
1626 #endif
1627 		goto done;
1628 	}
1629 
1630 	preempt_disable();
1631 	if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) {
1632 		if (kvm_mips_host_tlb_lookup(vcpu, va) < 0)
1633 			kvm_mips_handle_kseg0_tlb_fault(va, vcpu);
1634 	} else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) ||
1635 		   KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) {
1636 		int index;
1637 
1638 		/* If an entry already exists then skip */
1639 		if (kvm_mips_host_tlb_lookup(vcpu, va) >= 0)
1640 			goto skip_fault;
1641 
1642 		/*
1643 		 * If address not in the guest TLB, then give the guest a fault,
1644 		 * the resulting handler will do the right thing
1645 		 */
1646 		index = kvm_mips_guest_tlb_lookup(vcpu, (va & VPN2_MASK) |
1647 						  (kvm_read_c0_guest_entryhi
1648 						   (cop0) & ASID_MASK));
1649 
1650 		if (index < 0) {
1651 			vcpu->arch.host_cp0_entryhi = (va & VPN2_MASK);
1652 			vcpu->arch.host_cp0_badvaddr = va;
1653 			er = kvm_mips_emulate_tlbmiss_ld(cause, NULL, run,
1654 							 vcpu);
1655 			preempt_enable();
1656 			goto dont_update_pc;
1657 		} else {
1658 			struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
1659 			/*
1660 			 * Check if the entry is valid, if not then setup a TLB
1661 			 * invalid exception to the guest
1662 			 */
1663 			if (!TLB_IS_VALID(*tlb, va)) {
1664 				er = kvm_mips_emulate_tlbinv_ld(cause, NULL,
1665 								run, vcpu);
1666 				preempt_enable();
1667 				goto dont_update_pc;
1668 			} else {
1669 				/*
1670 				 * We fault an entry from the guest tlb to the
1671 				 * shadow host TLB
1672 				 */
1673 				kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb,
1674 								     NULL,
1675 								     NULL);
1676 			}
1677 		}
1678 	} else {
1679 		kvm_err("INVALID CACHE INDEX/ADDRESS (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1680 			cache, op, base, arch->gprs[base], offset);
1681 		er = EMULATE_FAIL;
1682 		preempt_enable();
1683 		goto dont_update_pc;
1684 
1685 	}
1686 
1687 skip_fault:
1688 	/* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1689 	if (cache == MIPS_CACHE_DCACHE
1690 	    && (op == MIPS_CACHE_OP_FILL_WB_INV
1691 		|| op == MIPS_CACHE_OP_HIT_INV)) {
1692 		flush_dcache_line(va);
1693 
1694 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1695 		/*
1696 		 * Replace the CACHE instruction, with a SYNCI, not the same,
1697 		 * but avoids a trap
1698 		 */
1699 		kvm_mips_trans_cache_va(inst, opc, vcpu);
1700 #endif
1701 	} else if (op == MIPS_CACHE_OP_HIT_INV && cache == MIPS_CACHE_ICACHE) {
1702 		flush_dcache_line(va);
1703 		flush_icache_line(va);
1704 
1705 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1706 		/* Replace the CACHE instruction, with a SYNCI */
1707 		kvm_mips_trans_cache_va(inst, opc, vcpu);
1708 #endif
1709 	} else {
1710 		kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1711 			cache, op, base, arch->gprs[base], offset);
1712 		er = EMULATE_FAIL;
1713 		preempt_enable();
1714 		goto dont_update_pc;
1715 	}
1716 
1717 	preempt_enable();
1718 
1719 dont_update_pc:
1720 	/* Rollback PC */
1721 	vcpu->arch.pc = curr_pc;
1722 done:
1723 	return er;
1724 }
1725 
kvm_mips_emulate_inst(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1726 enum emulation_result kvm_mips_emulate_inst(unsigned long cause, uint32_t *opc,
1727 					    struct kvm_run *run,
1728 					    struct kvm_vcpu *vcpu)
1729 {
1730 	enum emulation_result er = EMULATE_DONE;
1731 	uint32_t inst;
1732 
1733 	/* Fetch the instruction. */
1734 	if (cause & CAUSEF_BD)
1735 		opc += 1;
1736 
1737 	inst = kvm_get_inst(opc, vcpu);
1738 
1739 	switch (((union mips_instruction)inst).r_format.opcode) {
1740 	case cop0_op:
1741 		er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1742 		break;
1743 	case sb_op:
1744 	case sh_op:
1745 	case sw_op:
1746 		er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1747 		break;
1748 	case lb_op:
1749 	case lbu_op:
1750 	case lhu_op:
1751 	case lh_op:
1752 	case lw_op:
1753 		er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1754 		break;
1755 
1756 	case cache_op:
1757 		++vcpu->stat.cache_exits;
1758 		trace_kvm_exit(vcpu, CACHE_EXITS);
1759 		er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1760 		break;
1761 
1762 	default:
1763 		kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1764 			inst);
1765 		kvm_arch_vcpu_dump_regs(vcpu);
1766 		er = EMULATE_FAIL;
1767 		break;
1768 	}
1769 
1770 	return er;
1771 }
1772 
kvm_mips_emulate_syscall(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1773 enum emulation_result kvm_mips_emulate_syscall(unsigned long cause,
1774 					       uint32_t *opc,
1775 					       struct kvm_run *run,
1776 					       struct kvm_vcpu *vcpu)
1777 {
1778 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1779 	struct kvm_vcpu_arch *arch = &vcpu->arch;
1780 	enum emulation_result er = EMULATE_DONE;
1781 
1782 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1783 		/* save old pc */
1784 		kvm_write_c0_guest_epc(cop0, arch->pc);
1785 		kvm_set_c0_guest_status(cop0, ST0_EXL);
1786 
1787 		if (cause & CAUSEF_BD)
1788 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1789 		else
1790 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1791 
1792 		kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1793 
1794 		kvm_change_c0_guest_cause(cop0, (0xff),
1795 					  (T_SYSCALL << CAUSEB_EXCCODE));
1796 
1797 		/* Set PC to the exception entry point */
1798 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1799 
1800 	} else {
1801 		kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1802 		er = EMULATE_FAIL;
1803 	}
1804 
1805 	return er;
1806 }
1807 
kvm_mips_emulate_tlbmiss_ld(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1808 enum emulation_result kvm_mips_emulate_tlbmiss_ld(unsigned long cause,
1809 						  uint32_t *opc,
1810 						  struct kvm_run *run,
1811 						  struct kvm_vcpu *vcpu)
1812 {
1813 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1814 	struct kvm_vcpu_arch *arch = &vcpu->arch;
1815 	unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1816 				(kvm_read_c0_guest_entryhi(cop0) & ASID_MASK);
1817 
1818 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1819 		/* save old pc */
1820 		kvm_write_c0_guest_epc(cop0, arch->pc);
1821 		kvm_set_c0_guest_status(cop0, ST0_EXL);
1822 
1823 		if (cause & CAUSEF_BD)
1824 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1825 		else
1826 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1827 
1828 		kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1829 			  arch->pc);
1830 
1831 		/* set pc to the exception entry point */
1832 		arch->pc = KVM_GUEST_KSEG0 + 0x0;
1833 
1834 	} else {
1835 		kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1836 			  arch->pc);
1837 
1838 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1839 	}
1840 
1841 	kvm_change_c0_guest_cause(cop0, (0xff),
1842 				  (T_TLB_LD_MISS << CAUSEB_EXCCODE));
1843 
1844 	/* setup badvaddr, context and entryhi registers for the guest */
1845 	kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1846 	/* XXXKYMA: is the context register used by linux??? */
1847 	kvm_write_c0_guest_entryhi(cop0, entryhi);
1848 	/* Blow away the shadow host TLBs */
1849 	kvm_mips_flush_host_tlb(1);
1850 
1851 	return EMULATE_DONE;
1852 }
1853 
kvm_mips_emulate_tlbinv_ld(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1854 enum emulation_result kvm_mips_emulate_tlbinv_ld(unsigned long cause,
1855 						 uint32_t *opc,
1856 						 struct kvm_run *run,
1857 						 struct kvm_vcpu *vcpu)
1858 {
1859 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1860 	struct kvm_vcpu_arch *arch = &vcpu->arch;
1861 	unsigned long entryhi =
1862 		(vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1863 		(kvm_read_c0_guest_entryhi(cop0) & ASID_MASK);
1864 
1865 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1866 		/* save old pc */
1867 		kvm_write_c0_guest_epc(cop0, arch->pc);
1868 		kvm_set_c0_guest_status(cop0, ST0_EXL);
1869 
1870 		if (cause & CAUSEF_BD)
1871 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1872 		else
1873 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1874 
1875 		kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
1876 			  arch->pc);
1877 
1878 		/* set pc to the exception entry point */
1879 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1880 
1881 	} else {
1882 		kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1883 			  arch->pc);
1884 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1885 	}
1886 
1887 	kvm_change_c0_guest_cause(cop0, (0xff),
1888 				  (T_TLB_LD_MISS << CAUSEB_EXCCODE));
1889 
1890 	/* setup badvaddr, context and entryhi registers for the guest */
1891 	kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1892 	/* XXXKYMA: is the context register used by linux??? */
1893 	kvm_write_c0_guest_entryhi(cop0, entryhi);
1894 	/* Blow away the shadow host TLBs */
1895 	kvm_mips_flush_host_tlb(1);
1896 
1897 	return EMULATE_DONE;
1898 }
1899 
kvm_mips_emulate_tlbmiss_st(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1900 enum emulation_result kvm_mips_emulate_tlbmiss_st(unsigned long cause,
1901 						  uint32_t *opc,
1902 						  struct kvm_run *run,
1903 						  struct kvm_vcpu *vcpu)
1904 {
1905 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1906 	struct kvm_vcpu_arch *arch = &vcpu->arch;
1907 	unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1908 				(kvm_read_c0_guest_entryhi(cop0) & ASID_MASK);
1909 
1910 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1911 		/* save old pc */
1912 		kvm_write_c0_guest_epc(cop0, arch->pc);
1913 		kvm_set_c0_guest_status(cop0, ST0_EXL);
1914 
1915 		if (cause & CAUSEF_BD)
1916 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1917 		else
1918 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1919 
1920 		kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1921 			  arch->pc);
1922 
1923 		/* Set PC to the exception entry point */
1924 		arch->pc = KVM_GUEST_KSEG0 + 0x0;
1925 	} else {
1926 		kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1927 			  arch->pc);
1928 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1929 	}
1930 
1931 	kvm_change_c0_guest_cause(cop0, (0xff),
1932 				  (T_TLB_ST_MISS << CAUSEB_EXCCODE));
1933 
1934 	/* setup badvaddr, context and entryhi registers for the guest */
1935 	kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1936 	/* XXXKYMA: is the context register used by linux??? */
1937 	kvm_write_c0_guest_entryhi(cop0, entryhi);
1938 	/* Blow away the shadow host TLBs */
1939 	kvm_mips_flush_host_tlb(1);
1940 
1941 	return EMULATE_DONE;
1942 }
1943 
kvm_mips_emulate_tlbinv_st(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1944 enum emulation_result kvm_mips_emulate_tlbinv_st(unsigned long cause,
1945 						 uint32_t *opc,
1946 						 struct kvm_run *run,
1947 						 struct kvm_vcpu *vcpu)
1948 {
1949 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1950 	struct kvm_vcpu_arch *arch = &vcpu->arch;
1951 	unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1952 		(kvm_read_c0_guest_entryhi(cop0) & ASID_MASK);
1953 
1954 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1955 		/* save old pc */
1956 		kvm_write_c0_guest_epc(cop0, arch->pc);
1957 		kvm_set_c0_guest_status(cop0, ST0_EXL);
1958 
1959 		if (cause & CAUSEF_BD)
1960 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1961 		else
1962 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1963 
1964 		kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1965 			  arch->pc);
1966 
1967 		/* Set PC to the exception entry point */
1968 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1969 	} else {
1970 		kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1971 			  arch->pc);
1972 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
1973 	}
1974 
1975 	kvm_change_c0_guest_cause(cop0, (0xff),
1976 				  (T_TLB_ST_MISS << CAUSEB_EXCCODE));
1977 
1978 	/* setup badvaddr, context and entryhi registers for the guest */
1979 	kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1980 	/* XXXKYMA: is the context register used by linux??? */
1981 	kvm_write_c0_guest_entryhi(cop0, entryhi);
1982 	/* Blow away the shadow host TLBs */
1983 	kvm_mips_flush_host_tlb(1);
1984 
1985 	return EMULATE_DONE;
1986 }
1987 
1988 /* TLBMOD: store into address matching TLB with Dirty bit off */
kvm_mips_handle_tlbmod(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)1989 enum emulation_result kvm_mips_handle_tlbmod(unsigned long cause, uint32_t *opc,
1990 					     struct kvm_run *run,
1991 					     struct kvm_vcpu *vcpu)
1992 {
1993 	enum emulation_result er = EMULATE_DONE;
1994 #ifdef DEBUG
1995 	struct mips_coproc *cop0 = vcpu->arch.cop0;
1996 	unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1997 				(kvm_read_c0_guest_entryhi(cop0) & ASID_MASK);
1998 	int index;
1999 
2000 	/* If address not in the guest TLB, then we are in trouble */
2001 	index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
2002 	if (index < 0) {
2003 		/* XXXKYMA Invalidate and retry */
2004 		kvm_mips_host_tlb_inv(vcpu, vcpu->arch.host_cp0_badvaddr);
2005 		kvm_err("%s: host got TLBMOD for %#lx but entry not present in Guest TLB\n",
2006 		     __func__, entryhi);
2007 		kvm_mips_dump_guest_tlbs(vcpu);
2008 		kvm_mips_dump_host_tlbs();
2009 		return EMULATE_FAIL;
2010 	}
2011 #endif
2012 
2013 	er = kvm_mips_emulate_tlbmod(cause, opc, run, vcpu);
2014 	return er;
2015 }
2016 
kvm_mips_emulate_tlbmod(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2017 enum emulation_result kvm_mips_emulate_tlbmod(unsigned long cause,
2018 					      uint32_t *opc,
2019 					      struct kvm_run *run,
2020 					      struct kvm_vcpu *vcpu)
2021 {
2022 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2023 	unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2024 				(kvm_read_c0_guest_entryhi(cop0) & ASID_MASK);
2025 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2026 
2027 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2028 		/* save old pc */
2029 		kvm_write_c0_guest_epc(cop0, arch->pc);
2030 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2031 
2032 		if (cause & CAUSEF_BD)
2033 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2034 		else
2035 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2036 
2037 		kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2038 			  arch->pc);
2039 
2040 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2041 	} else {
2042 		kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2043 			  arch->pc);
2044 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2045 	}
2046 
2047 	kvm_change_c0_guest_cause(cop0, (0xff), (T_TLB_MOD << CAUSEB_EXCCODE));
2048 
2049 	/* setup badvaddr, context and entryhi registers for the guest */
2050 	kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2051 	/* XXXKYMA: is the context register used by linux??? */
2052 	kvm_write_c0_guest_entryhi(cop0, entryhi);
2053 	/* Blow away the shadow host TLBs */
2054 	kvm_mips_flush_host_tlb(1);
2055 
2056 	return EMULATE_DONE;
2057 }
2058 
kvm_mips_emulate_fpu_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2059 enum emulation_result kvm_mips_emulate_fpu_exc(unsigned long cause,
2060 					       uint32_t *opc,
2061 					       struct kvm_run *run,
2062 					       struct kvm_vcpu *vcpu)
2063 {
2064 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2065 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2066 
2067 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2068 		/* save old pc */
2069 		kvm_write_c0_guest_epc(cop0, arch->pc);
2070 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2071 
2072 		if (cause & CAUSEF_BD)
2073 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2074 		else
2075 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2076 
2077 	}
2078 
2079 	arch->pc = KVM_GUEST_KSEG0 + 0x180;
2080 
2081 	kvm_change_c0_guest_cause(cop0, (0xff),
2082 				  (T_COP_UNUSABLE << CAUSEB_EXCCODE));
2083 	kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2084 
2085 	return EMULATE_DONE;
2086 }
2087 
kvm_mips_emulate_ri_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2088 enum emulation_result kvm_mips_emulate_ri_exc(unsigned long cause,
2089 					      uint32_t *opc,
2090 					      struct kvm_run *run,
2091 					      struct kvm_vcpu *vcpu)
2092 {
2093 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2094 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2095 	enum emulation_result er = EMULATE_DONE;
2096 
2097 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2098 		/* save old pc */
2099 		kvm_write_c0_guest_epc(cop0, arch->pc);
2100 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2101 
2102 		if (cause & CAUSEF_BD)
2103 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2104 		else
2105 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2106 
2107 		kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2108 
2109 		kvm_change_c0_guest_cause(cop0, (0xff),
2110 					  (T_RES_INST << CAUSEB_EXCCODE));
2111 
2112 		/* Set PC to the exception entry point */
2113 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2114 
2115 	} else {
2116 		kvm_err("Trying to deliver RI when EXL is already set\n");
2117 		er = EMULATE_FAIL;
2118 	}
2119 
2120 	return er;
2121 }
2122 
kvm_mips_emulate_bp_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2123 enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
2124 					      uint32_t *opc,
2125 					      struct kvm_run *run,
2126 					      struct kvm_vcpu *vcpu)
2127 {
2128 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2129 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2130 	enum emulation_result er = EMULATE_DONE;
2131 
2132 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2133 		/* save old pc */
2134 		kvm_write_c0_guest_epc(cop0, arch->pc);
2135 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2136 
2137 		if (cause & CAUSEF_BD)
2138 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2139 		else
2140 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2141 
2142 		kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2143 
2144 		kvm_change_c0_guest_cause(cop0, (0xff),
2145 					  (T_BREAK << CAUSEB_EXCCODE));
2146 
2147 		/* Set PC to the exception entry point */
2148 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2149 
2150 	} else {
2151 		kvm_err("Trying to deliver BP when EXL is already set\n");
2152 		er = EMULATE_FAIL;
2153 	}
2154 
2155 	return er;
2156 }
2157 
kvm_mips_emulate_trap_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2158 enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
2159 						uint32_t *opc,
2160 						struct kvm_run *run,
2161 						struct kvm_vcpu *vcpu)
2162 {
2163 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2164 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2165 	enum emulation_result er = EMULATE_DONE;
2166 
2167 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2168 		/* save old pc */
2169 		kvm_write_c0_guest_epc(cop0, arch->pc);
2170 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2171 
2172 		if (cause & CAUSEF_BD)
2173 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2174 		else
2175 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2176 
2177 		kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2178 
2179 		kvm_change_c0_guest_cause(cop0, (0xff),
2180 					  (T_TRAP << CAUSEB_EXCCODE));
2181 
2182 		/* Set PC to the exception entry point */
2183 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2184 
2185 	} else {
2186 		kvm_err("Trying to deliver TRAP when EXL is already set\n");
2187 		er = EMULATE_FAIL;
2188 	}
2189 
2190 	return er;
2191 }
2192 
kvm_mips_emulate_msafpe_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2193 enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
2194 						  uint32_t *opc,
2195 						  struct kvm_run *run,
2196 						  struct kvm_vcpu *vcpu)
2197 {
2198 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2199 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2200 	enum emulation_result er = EMULATE_DONE;
2201 
2202 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2203 		/* save old pc */
2204 		kvm_write_c0_guest_epc(cop0, arch->pc);
2205 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2206 
2207 		if (cause & CAUSEF_BD)
2208 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2209 		else
2210 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2211 
2212 		kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2213 
2214 		kvm_change_c0_guest_cause(cop0, (0xff),
2215 					  (T_MSAFPE << CAUSEB_EXCCODE));
2216 
2217 		/* Set PC to the exception entry point */
2218 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2219 
2220 	} else {
2221 		kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2222 		er = EMULATE_FAIL;
2223 	}
2224 
2225 	return er;
2226 }
2227 
kvm_mips_emulate_fpe_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2228 enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
2229 					       uint32_t *opc,
2230 					       struct kvm_run *run,
2231 					       struct kvm_vcpu *vcpu)
2232 {
2233 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2234 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2235 	enum emulation_result er = EMULATE_DONE;
2236 
2237 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2238 		/* save old pc */
2239 		kvm_write_c0_guest_epc(cop0, arch->pc);
2240 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2241 
2242 		if (cause & CAUSEF_BD)
2243 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2244 		else
2245 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2246 
2247 		kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2248 
2249 		kvm_change_c0_guest_cause(cop0, (0xff),
2250 					  (T_FPE << CAUSEB_EXCCODE));
2251 
2252 		/* Set PC to the exception entry point */
2253 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2254 
2255 	} else {
2256 		kvm_err("Trying to deliver FPE when EXL is already set\n");
2257 		er = EMULATE_FAIL;
2258 	}
2259 
2260 	return er;
2261 }
2262 
kvm_mips_emulate_msadis_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2263 enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
2264 						  uint32_t *opc,
2265 						  struct kvm_run *run,
2266 						  struct kvm_vcpu *vcpu)
2267 {
2268 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2269 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2270 	enum emulation_result er = EMULATE_DONE;
2271 
2272 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2273 		/* save old pc */
2274 		kvm_write_c0_guest_epc(cop0, arch->pc);
2275 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2276 
2277 		if (cause & CAUSEF_BD)
2278 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2279 		else
2280 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2281 
2282 		kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2283 
2284 		kvm_change_c0_guest_cause(cop0, (0xff),
2285 					  (T_MSADIS << CAUSEB_EXCCODE));
2286 
2287 		/* Set PC to the exception entry point */
2288 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2289 
2290 	} else {
2291 		kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2292 		er = EMULATE_FAIL;
2293 	}
2294 
2295 	return er;
2296 }
2297 
2298 /* ll/sc, rdhwr, sync emulation */
2299 
2300 #define OPCODE 0xfc000000
2301 #define BASE   0x03e00000
2302 #define RT     0x001f0000
2303 #define OFFSET 0x0000ffff
2304 #define LL     0xc0000000
2305 #define SC     0xe0000000
2306 #define SPEC0  0x00000000
2307 #define SPEC3  0x7c000000
2308 #define RD     0x0000f800
2309 #define FUNC   0x0000003f
2310 #define SYNC   0x0000000f
2311 #define RDHWR  0x0000003b
2312 
kvm_mips_handle_ri(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2313 enum emulation_result kvm_mips_handle_ri(unsigned long cause, uint32_t *opc,
2314 					 struct kvm_run *run,
2315 					 struct kvm_vcpu *vcpu)
2316 {
2317 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2318 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2319 	enum emulation_result er = EMULATE_DONE;
2320 	unsigned long curr_pc;
2321 	uint32_t inst;
2322 
2323 	/*
2324 	 * Update PC and hold onto current PC in case there is
2325 	 * an error and we want to rollback the PC
2326 	 */
2327 	curr_pc = vcpu->arch.pc;
2328 	er = update_pc(vcpu, cause);
2329 	if (er == EMULATE_FAIL)
2330 		return er;
2331 
2332 	/* Fetch the instruction. */
2333 	if (cause & CAUSEF_BD)
2334 		opc += 1;
2335 
2336 	inst = kvm_get_inst(opc, vcpu);
2337 
2338 	if (inst == KVM_INVALID_INST) {
2339 		kvm_err("%s: Cannot get inst @ %p\n", __func__, opc);
2340 		return EMULATE_FAIL;
2341 	}
2342 
2343 	if ((inst & OPCODE) == SPEC3 && (inst & FUNC) == RDHWR) {
2344 		int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2345 		int rd = (inst & RD) >> 11;
2346 		int rt = (inst & RT) >> 16;
2347 		/* If usermode, check RDHWR rd is allowed by guest HWREna */
2348 		if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2349 			kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2350 				  rd, opc);
2351 			goto emulate_ri;
2352 		}
2353 		switch (rd) {
2354 		case 0:	/* CPU number */
2355 			arch->gprs[rt] = 0;
2356 			break;
2357 		case 1:	/* SYNCI length */
2358 			arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2359 					     current_cpu_data.icache.linesz);
2360 			break;
2361 		case 2:	/* Read count register */
2362 			arch->gprs[rt] = kvm_mips_read_count(vcpu);
2363 			break;
2364 		case 3:	/* Count register resolution */
2365 			switch (current_cpu_data.cputype) {
2366 			case CPU_20KC:
2367 			case CPU_25KF:
2368 				arch->gprs[rt] = 1;
2369 				break;
2370 			default:
2371 				arch->gprs[rt] = 2;
2372 			}
2373 			break;
2374 		case 29:
2375 			arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2376 			break;
2377 
2378 		default:
2379 			kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2380 			goto emulate_ri;
2381 		}
2382 	} else {
2383 		kvm_debug("Emulate RI not supported @ %p: %#x\n", opc, inst);
2384 		goto emulate_ri;
2385 	}
2386 
2387 	return EMULATE_DONE;
2388 
2389 emulate_ri:
2390 	/*
2391 	 * Rollback PC (if in branch delay slot then the PC already points to
2392 	 * branch target), and pass the RI exception to the guest OS.
2393 	 */
2394 	vcpu->arch.pc = curr_pc;
2395 	return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2396 }
2397 
kvm_mips_complete_mmio_load(struct kvm_vcpu * vcpu,struct kvm_run * run)2398 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2399 						  struct kvm_run *run)
2400 {
2401 	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2402 	enum emulation_result er = EMULATE_DONE;
2403 
2404 	if (run->mmio.len > sizeof(*gpr)) {
2405 		kvm_err("Bad MMIO length: %d", run->mmio.len);
2406 		er = EMULATE_FAIL;
2407 		goto done;
2408 	}
2409 
2410 	er = update_pc(vcpu, vcpu->arch.pending_load_cause);
2411 	if (er == EMULATE_FAIL)
2412 		return er;
2413 
2414 	switch (run->mmio.len) {
2415 	case 4:
2416 		*gpr = *(int32_t *) run->mmio.data;
2417 		break;
2418 
2419 	case 2:
2420 		if (vcpu->mmio_needed == 2)
2421 			*gpr = *(int16_t *) run->mmio.data;
2422 		else
2423 			*gpr = *(uint16_t *)run->mmio.data;
2424 
2425 		break;
2426 	case 1:
2427 		if (vcpu->mmio_needed == 2)
2428 			*gpr = *(int8_t *) run->mmio.data;
2429 		else
2430 			*gpr = *(u8 *) run->mmio.data;
2431 		break;
2432 	}
2433 
2434 	if (vcpu->arch.pending_load_cause & CAUSEF_BD)
2435 		kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n",
2436 			  vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr,
2437 			  vcpu->mmio_needed);
2438 
2439 done:
2440 	return er;
2441 }
2442 
kvm_mips_emulate_exc(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2443 static enum emulation_result kvm_mips_emulate_exc(unsigned long cause,
2444 						  uint32_t *opc,
2445 						  struct kvm_run *run,
2446 						  struct kvm_vcpu *vcpu)
2447 {
2448 	uint32_t exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2449 	struct mips_coproc *cop0 = vcpu->arch.cop0;
2450 	struct kvm_vcpu_arch *arch = &vcpu->arch;
2451 	enum emulation_result er = EMULATE_DONE;
2452 
2453 	if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2454 		/* save old pc */
2455 		kvm_write_c0_guest_epc(cop0, arch->pc);
2456 		kvm_set_c0_guest_status(cop0, ST0_EXL);
2457 
2458 		if (cause & CAUSEF_BD)
2459 			kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2460 		else
2461 			kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2462 
2463 		kvm_change_c0_guest_cause(cop0, (0xff),
2464 					  (exccode << CAUSEB_EXCCODE));
2465 
2466 		/* Set PC to the exception entry point */
2467 		arch->pc = KVM_GUEST_KSEG0 + 0x180;
2468 		kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2469 
2470 		kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2471 			  exccode, kvm_read_c0_guest_epc(cop0),
2472 			  kvm_read_c0_guest_badvaddr(cop0));
2473 	} else {
2474 		kvm_err("Trying to deliver EXC when EXL is already set\n");
2475 		er = EMULATE_FAIL;
2476 	}
2477 
2478 	return er;
2479 }
2480 
kvm_mips_check_privilege(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2481 enum emulation_result kvm_mips_check_privilege(unsigned long cause,
2482 					       uint32_t *opc,
2483 					       struct kvm_run *run,
2484 					       struct kvm_vcpu *vcpu)
2485 {
2486 	enum emulation_result er = EMULATE_DONE;
2487 	uint32_t exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2488 	unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2489 
2490 	int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2491 
2492 	if (usermode) {
2493 		switch (exccode) {
2494 		case T_INT:
2495 		case T_SYSCALL:
2496 		case T_BREAK:
2497 		case T_RES_INST:
2498 		case T_TRAP:
2499 		case T_MSAFPE:
2500 		case T_FPE:
2501 		case T_MSADIS:
2502 			break;
2503 
2504 		case T_COP_UNUSABLE:
2505 			if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2506 				er = EMULATE_PRIV_FAIL;
2507 			break;
2508 
2509 		case T_TLB_MOD:
2510 			break;
2511 
2512 		case T_TLB_LD_MISS:
2513 			/*
2514 			 * We we are accessing Guest kernel space, then send an
2515 			 * address error exception to the guest
2516 			 */
2517 			if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2518 				kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2519 					  badvaddr);
2520 				cause &= ~0xff;
2521 				cause |= (T_ADDR_ERR_LD << CAUSEB_EXCCODE);
2522 				er = EMULATE_PRIV_FAIL;
2523 			}
2524 			break;
2525 
2526 		case T_TLB_ST_MISS:
2527 			/*
2528 			 * We we are accessing Guest kernel space, then send an
2529 			 * address error exception to the guest
2530 			 */
2531 			if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2532 				kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2533 					  badvaddr);
2534 				cause &= ~0xff;
2535 				cause |= (T_ADDR_ERR_ST << CAUSEB_EXCCODE);
2536 				er = EMULATE_PRIV_FAIL;
2537 			}
2538 			break;
2539 
2540 		case T_ADDR_ERR_ST:
2541 			kvm_debug("%s: address error ST @ %#lx\n", __func__,
2542 				  badvaddr);
2543 			if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2544 				cause &= ~0xff;
2545 				cause |= (T_TLB_ST_MISS << CAUSEB_EXCCODE);
2546 			}
2547 			er = EMULATE_PRIV_FAIL;
2548 			break;
2549 		case T_ADDR_ERR_LD:
2550 			kvm_debug("%s: address error LD @ %#lx\n", __func__,
2551 				  badvaddr);
2552 			if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2553 				cause &= ~0xff;
2554 				cause |= (T_TLB_LD_MISS << CAUSEB_EXCCODE);
2555 			}
2556 			er = EMULATE_PRIV_FAIL;
2557 			break;
2558 		default:
2559 			er = EMULATE_PRIV_FAIL;
2560 			break;
2561 		}
2562 	}
2563 
2564 	if (er == EMULATE_PRIV_FAIL)
2565 		kvm_mips_emulate_exc(cause, opc, run, vcpu);
2566 
2567 	return er;
2568 }
2569 
2570 /*
2571  * User Address (UA) fault, this could happen if
2572  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2573  *     case we pass on the fault to the guest kernel and let it handle it.
2574  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2575  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2576  */
kvm_mips_handle_tlbmiss(unsigned long cause,uint32_t * opc,struct kvm_run * run,struct kvm_vcpu * vcpu)2577 enum emulation_result kvm_mips_handle_tlbmiss(unsigned long cause,
2578 					      uint32_t *opc,
2579 					      struct kvm_run *run,
2580 					      struct kvm_vcpu *vcpu)
2581 {
2582 	enum emulation_result er = EMULATE_DONE;
2583 	uint32_t exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2584 	unsigned long va = vcpu->arch.host_cp0_badvaddr;
2585 	int index;
2586 
2587 	kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx, entryhi: %#lx\n",
2588 		  vcpu->arch.host_cp0_badvaddr, vcpu->arch.host_cp0_entryhi);
2589 
2590 	/*
2591 	 * KVM would not have got the exception if this entry was valid in the
2592 	 * shadow host TLB. Check the Guest TLB, if the entry is not there then
2593 	 * send the guest an exception. The guest exc handler should then inject
2594 	 * an entry into the guest TLB.
2595 	 */
2596 	index = kvm_mips_guest_tlb_lookup(vcpu,
2597 					  (va & VPN2_MASK) |
2598 					  (kvm_read_c0_guest_entryhi
2599 					   (vcpu->arch.cop0) & ASID_MASK));
2600 	if (index < 0) {
2601 		if (exccode == T_TLB_LD_MISS) {
2602 			er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2603 		} else if (exccode == T_TLB_ST_MISS) {
2604 			er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2605 		} else {
2606 			kvm_err("%s: invalid exc code: %d\n", __func__,
2607 				exccode);
2608 			er = EMULATE_FAIL;
2609 		}
2610 	} else {
2611 		struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2612 
2613 		/*
2614 		 * Check if the entry is valid, if not then setup a TLB invalid
2615 		 * exception to the guest
2616 		 */
2617 		if (!TLB_IS_VALID(*tlb, va)) {
2618 			if (exccode == T_TLB_LD_MISS) {
2619 				er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2620 								vcpu);
2621 			} else if (exccode == T_TLB_ST_MISS) {
2622 				er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2623 								vcpu);
2624 			} else {
2625 				kvm_err("%s: invalid exc code: %d\n", __func__,
2626 					exccode);
2627 				er = EMULATE_FAIL;
2628 			}
2629 		} else {
2630 			kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2631 				  tlb->tlb_hi, tlb->tlb_lo0, tlb->tlb_lo1);
2632 			/*
2633 			 * OK we have a Guest TLB entry, now inject it into the
2634 			 * shadow host TLB
2635 			 */
2636 			kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, NULL,
2637 							     NULL);
2638 		}
2639 	}
2640 
2641 	return er;
2642 }
2643