This source file includes following definitions.
- early_parse_kvm_cma_resv
- kvm_alloc_hpt_cma
- kvm_free_hpt_cma
- kvm_cma_reserve
- kvmppc_rm_h_confer
- kvm_hv_vm_activated
- kvm_hv_vm_deactivated
- kvm_hv_mode_active
- kvmppc_hcall_impl_hv_realmode
- kvmppc_hwrng_present
- kvmppc_h_random
- kvmhv_rm_send_ipi
- kvmhv_interrupt_vcore
- kvmhv_commence_exit
- get_irqmap
- kvmppc_check_passthru
- kvmppc_check_passthru
- kvmppc_read_intr
- kvmppc_read_one_intr
- is_rm
- kvmppc_rm_h_xirr
- kvmppc_rm_h_xirr_x
- kvmppc_rm_h_ipoll
- kvmppc_rm_h_ipi
- kvmppc_rm_h_cppr
- kvmppc_rm_h_eoi
- kvmppc_bad_interrupt
- wait_for_sync
- kvmhv_p9_set_lpcr
- kvmhv_p9_restore_lpcr
- kvmppc_guest_entry_inject_int
- flush_guest_tlb
- kvmppc_check_need_tlb_flush
1
2
3
4
5
6 #include <linux/cpu.h>
7 #include <linux/kvm_host.h>
8 #include <linux/preempt.h>
9 #include <linux/export.h>
10 #include <linux/sched.h>
11 #include <linux/spinlock.h>
12 #include <linux/init.h>
13 #include <linux/memblock.h>
14 #include <linux/sizes.h>
15 #include <linux/cma.h>
16 #include <linux/bitops.h>
17
18 #include <asm/asm-prototypes.h>
19 #include <asm/cputable.h>
20 #include <asm/kvm_ppc.h>
21 #include <asm/kvm_book3s.h>
22 #include <asm/archrandom.h>
23 #include <asm/xics.h>
24 #include <asm/xive.h>
25 #include <asm/dbell.h>
26 #include <asm/cputhreads.h>
27 #include <asm/io.h>
28 #include <asm/opal.h>
29 #include <asm/smp.h>
30
31 #define KVM_CMA_CHUNK_ORDER 18
32
33 #include "book3s_xics.h"
34 #include "book3s_xive.h"
35
36
37
38
39 unsigned long (*__xive_vm_h_xirr)(struct kvm_vcpu *vcpu);
40 unsigned long (*__xive_vm_h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server);
41 int (*__xive_vm_h_ipi)(struct kvm_vcpu *vcpu, unsigned long server,
42 unsigned long mfrr);
43 int (*__xive_vm_h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr);
44 int (*__xive_vm_h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr);
45 EXPORT_SYMBOL_GPL(__xive_vm_h_xirr);
46 EXPORT_SYMBOL_GPL(__xive_vm_h_ipoll);
47 EXPORT_SYMBOL_GPL(__xive_vm_h_ipi);
48 EXPORT_SYMBOL_GPL(__xive_vm_h_cppr);
49 EXPORT_SYMBOL_GPL(__xive_vm_h_eoi);
50
51
52
53
54
55 #define HPT_ALIGN_PAGES ((1 << 18) >> PAGE_SHIFT)
56
57
58
59 static unsigned long kvm_cma_resv_ratio = 5;
60
61 static struct cma *kvm_cma;
62
63 static int __init early_parse_kvm_cma_resv(char *p)
64 {
65 pr_debug("%s(%s)\n", __func__, p);
66 if (!p)
67 return -EINVAL;
68 return kstrtoul(p, 0, &kvm_cma_resv_ratio);
69 }
70 early_param("kvm_cma_resv_ratio", early_parse_kvm_cma_resv);
71
72 struct page *kvm_alloc_hpt_cma(unsigned long nr_pages)
73 {
74 VM_BUG_ON(order_base_2(nr_pages) < KVM_CMA_CHUNK_ORDER - PAGE_SHIFT);
75
76 return cma_alloc(kvm_cma, nr_pages, order_base_2(HPT_ALIGN_PAGES),
77 false);
78 }
79 EXPORT_SYMBOL_GPL(kvm_alloc_hpt_cma);
80
81 void kvm_free_hpt_cma(struct page *page, unsigned long nr_pages)
82 {
83 cma_release(kvm_cma, page, nr_pages);
84 }
85 EXPORT_SYMBOL_GPL(kvm_free_hpt_cma);
86
87
88
89
90
91
92
93
94
95 void __init kvm_cma_reserve(void)
96 {
97 unsigned long align_size;
98 struct memblock_region *reg;
99 phys_addr_t selected_size = 0;
100
101
102
103
104 if (!cpu_has_feature(CPU_FTR_HVMODE))
105 return;
106
107
108
109
110 for_each_memblock(memory, reg)
111 selected_size += memblock_region_memory_end_pfn(reg) -
112 memblock_region_memory_base_pfn(reg);
113
114 selected_size = (selected_size * kvm_cma_resv_ratio / 100) << PAGE_SHIFT;
115 if (selected_size) {
116 pr_debug("%s: reserving %ld MiB for global area\n", __func__,
117 (unsigned long)selected_size / SZ_1M);
118 align_size = HPT_ALIGN_PAGES << PAGE_SHIFT;
119 cma_declare_contiguous(0, selected_size, 0, align_size,
120 KVM_CMA_CHUNK_ORDER - PAGE_SHIFT, false, "kvm_cma",
121 &kvm_cma);
122 }
123 }
124
125
126
127
128
129
130
131
132 long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target,
133 unsigned int yield_count)
134 {
135 struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
136 int ptid = local_paca->kvm_hstate.ptid;
137 int threads_running;
138 int threads_ceded;
139 int threads_conferring;
140 u64 stop = get_tb() + 10 * tb_ticks_per_usec;
141 int rv = H_SUCCESS;
142
143 set_bit(ptid, &vc->conferring_threads);
144 while ((get_tb() < stop) && !VCORE_IS_EXITING(vc)) {
145 threads_running = VCORE_ENTRY_MAP(vc);
146 threads_ceded = vc->napping_threads;
147 threads_conferring = vc->conferring_threads;
148 if ((threads_ceded | threads_conferring) == threads_running) {
149 rv = H_TOO_HARD;
150 break;
151 }
152 }
153 clear_bit(ptid, &vc->conferring_threads);
154 return rv;
155 }
156
157
158
159
160
161
162
163
164 static atomic_t hv_vm_count;
165
166 void kvm_hv_vm_activated(void)
167 {
168 get_online_cpus();
169 atomic_inc(&hv_vm_count);
170 put_online_cpus();
171 }
172 EXPORT_SYMBOL_GPL(kvm_hv_vm_activated);
173
174 void kvm_hv_vm_deactivated(void)
175 {
176 get_online_cpus();
177 atomic_dec(&hv_vm_count);
178 put_online_cpus();
179 }
180 EXPORT_SYMBOL_GPL(kvm_hv_vm_deactivated);
181
182 bool kvm_hv_mode_active(void)
183 {
184 return atomic_read(&hv_vm_count) != 0;
185 }
186
187 extern int hcall_real_table[], hcall_real_table_end[];
188
189 int kvmppc_hcall_impl_hv_realmode(unsigned long cmd)
190 {
191 cmd /= 4;
192 if (cmd < hcall_real_table_end - hcall_real_table &&
193 hcall_real_table[cmd])
194 return 1;
195
196 return 0;
197 }
198 EXPORT_SYMBOL_GPL(kvmppc_hcall_impl_hv_realmode);
199
200 int kvmppc_hwrng_present(void)
201 {
202 return powernv_hwrng_present();
203 }
204 EXPORT_SYMBOL_GPL(kvmppc_hwrng_present);
205
206 long kvmppc_h_random(struct kvm_vcpu *vcpu)
207 {
208 int r;
209
210
211 if (kvm_is_radix(vcpu->kvm) && (mfmsr() & MSR_IR))
212 r = powernv_get_random_long(&vcpu->arch.regs.gpr[4]);
213 else
214 r = powernv_get_random_real_mode(&vcpu->arch.regs.gpr[4]);
215 if (r)
216 return H_SUCCESS;
217
218 return H_HARDWARE;
219 }
220
221
222
223
224
225
226 void kvmhv_rm_send_ipi(int cpu)
227 {
228 void __iomem *xics_phys;
229 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
230
231
232 if (kvmhv_on_pseries()) {
233 unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
234
235 plpar_hcall_raw(H_IPI, retbuf, get_hard_smp_processor_id(cpu),
236 IPI_PRIORITY);
237 return;
238 }
239
240
241 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
242 msg |= get_hard_smp_processor_id(cpu);
243 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
244 return;
245 }
246
247
248 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
249 cpu_first_thread_sibling(cpu) ==
250 cpu_first_thread_sibling(raw_smp_processor_id())) {
251 msg |= cpu_thread_in_core(cpu);
252 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
253 return;
254 }
255
256
257 if (WARN_ON_ONCE(xics_on_xive()))
258 return;
259
260
261 xics_phys = paca_ptrs[cpu]->kvm_hstate.xics_phys;
262 if (xics_phys)
263 __raw_rm_writeb(IPI_PRIORITY, xics_phys + XICS_MFRR);
264 else
265 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
266 }
267
268
269
270
271
272 static void kvmhv_interrupt_vcore(struct kvmppc_vcore *vc, int active)
273 {
274 int cpu = vc->pcpu;
275
276
277 smp_mb();
278 for (; active; active >>= 1, ++cpu)
279 if (active & 1)
280 kvmhv_rm_send_ipi(cpu);
281 }
282
283 void kvmhv_commence_exit(int trap)
284 {
285 struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
286 int ptid = local_paca->kvm_hstate.ptid;
287 struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode;
288 int me, ee, i, t;
289 int cpu0;
290
291
292
293 me = 0x100 << ptid;
294 do {
295 ee = vc->entry_exit_map;
296 } while (cmpxchg(&vc->entry_exit_map, ee, ee | me) != ee);
297
298
299 if ((ee >> 8) != 0)
300 return;
301
302
303
304
305
306
307 if (trap != BOOK3S_INTERRUPT_HV_DECREMENTER)
308 kvmhv_interrupt_vcore(vc, ee & ~(1 << ptid));
309
310
311
312
313
314 if (!sip)
315 return;
316
317 for (i = 0; i < MAX_SUBCORES; ++i) {
318 vc = sip->vc[i];
319 if (!vc)
320 break;
321 do {
322 ee = vc->entry_exit_map;
323
324 if ((ee >> 8) != 0)
325 break;
326 } while (cmpxchg(&vc->entry_exit_map, ee,
327 ee | VCORE_EXIT_REQ) != ee);
328 if ((ee >> 8) == 0)
329 kvmhv_interrupt_vcore(vc, ee);
330 }
331
332
333
334
335
336
337 if (sip->lpcr_req) {
338 if (cmpxchg(&sip->do_restore, 0, 1) == 0) {
339 vc = local_paca->kvm_hstate.kvm_vcore;
340 cpu0 = vc->pcpu + ptid - local_paca->kvm_hstate.tid;
341 for (t = 1; t < threads_per_core; ++t) {
342 if (sip->napped[t])
343 kvmhv_rm_send_ipi(cpu0 + t);
344 }
345 }
346 }
347 }
348
349 struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv;
350 EXPORT_SYMBOL_GPL(kvmppc_host_rm_ops_hv);
351
352 #ifdef CONFIG_KVM_XICS
353 static struct kvmppc_irq_map *get_irqmap(struct kvmppc_passthru_irqmap *pimap,
354 u32 xisr)
355 {
356 int i;
357
358
359
360
361
362
363
364
365
366
367
368 for (i = 0; i < pimap->n_mapped; i++) {
369 if (xisr == pimap->mapped[i].r_hwirq) {
370
371
372
373
374 smp_rmb();
375 return &pimap->mapped[i];
376 }
377 }
378 return NULL;
379 }
380
381
382
383
384
385
386
387
388
389
390
391
392
393 static int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
394 {
395 struct kvmppc_passthru_irqmap *pimap;
396 struct kvmppc_irq_map *irq_map;
397 struct kvm_vcpu *vcpu;
398
399 vcpu = local_paca->kvm_hstate.kvm_vcpu;
400 if (!vcpu)
401 return 1;
402 pimap = kvmppc_get_passthru_irqmap(vcpu->kvm);
403 if (!pimap)
404 return 1;
405 irq_map = get_irqmap(pimap, xisr);
406 if (!irq_map)
407 return 1;
408
409
410 local_paca->kvm_hstate.saved_xirr = 0;
411
412 return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap, again);
413 }
414
415 #else
416 static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
417 {
418 return 1;
419 }
420 #endif
421
422
423
424
425
426
427
428
429
430
431 static long kvmppc_read_one_intr(bool *again);
432
433 long kvmppc_read_intr(void)
434 {
435 long ret = 0;
436 long rc;
437 bool again;
438
439 if (xive_enabled())
440 return 1;
441
442 do {
443 again = false;
444 rc = kvmppc_read_one_intr(&again);
445 if (rc && (ret == 0 || rc > ret))
446 ret = rc;
447 } while (again);
448 return ret;
449 }
450
451 static long kvmppc_read_one_intr(bool *again)
452 {
453 void __iomem *xics_phys;
454 u32 h_xirr;
455 __be32 xirr;
456 u32 xisr;
457 u8 host_ipi;
458 int64_t rc;
459
460 if (xive_enabled())
461 return 1;
462
463
464 host_ipi = local_paca->kvm_hstate.host_ipi;
465 if (host_ipi)
466 return 1;
467
468
469 if (kvmhv_on_pseries()) {
470 unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
471
472 rc = plpar_hcall_raw(H_XIRR, retbuf, 0xFF);
473 xirr = cpu_to_be32(retbuf[0]);
474 } else {
475 xics_phys = local_paca->kvm_hstate.xics_phys;
476 rc = 0;
477 if (!xics_phys)
478 rc = opal_int_get_xirr(&xirr, false);
479 else
480 xirr = __raw_rm_readl(xics_phys + XICS_XIRR);
481 }
482 if (rc < 0)
483 return 1;
484
485
486
487
488
489
490
491 h_xirr = be32_to_cpu(xirr);
492 local_paca->kvm_hstate.saved_xirr = h_xirr;
493 xisr = h_xirr & 0xffffff;
494
495
496
497
498 smp_mb();
499
500
501 if (!xisr)
502 return 0;
503
504
505
506
507
508 if (xisr == XICS_IPI) {
509 rc = 0;
510 if (kvmhv_on_pseries()) {
511 unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
512
513 plpar_hcall_raw(H_IPI, retbuf,
514 hard_smp_processor_id(), 0xff);
515 plpar_hcall_raw(H_EOI, retbuf, h_xirr);
516 } else if (xics_phys) {
517 __raw_rm_writeb(0xff, xics_phys + XICS_MFRR);
518 __raw_rm_writel(xirr, xics_phys + XICS_XIRR);
519 } else {
520 opal_int_set_mfrr(hard_smp_processor_id(), 0xff);
521 rc = opal_int_eoi(h_xirr);
522 }
523
524 *again = rc > 0;
525
526
527
528
529
530 smp_mb();
531
532
533
534
535
536
537 host_ipi = local_paca->kvm_hstate.host_ipi;
538 if (unlikely(host_ipi != 0)) {
539
540
541
542 if (kvmhv_on_pseries()) {
543 unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
544
545 plpar_hcall_raw(H_IPI, retbuf,
546 hard_smp_processor_id(),
547 IPI_PRIORITY);
548 } else if (xics_phys)
549 __raw_rm_writeb(IPI_PRIORITY,
550 xics_phys + XICS_MFRR);
551 else
552 opal_int_set_mfrr(hard_smp_processor_id(),
553 IPI_PRIORITY);
554
555 smp_mb();
556 return 1;
557 }
558
559
560 local_paca->kvm_hstate.saved_xirr = 0;
561 return -1;
562 }
563
564 return kvmppc_check_passthru(xisr, xirr, again);
565 }
566
567 #ifdef CONFIG_KVM_XICS
568 static inline bool is_rm(void)
569 {
570 return !(mfmsr() & MSR_DR);
571 }
572
573 unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu)
574 {
575 if (!kvmppc_xics_enabled(vcpu))
576 return H_TOO_HARD;
577 if (xics_on_xive()) {
578 if (is_rm())
579 return xive_rm_h_xirr(vcpu);
580 if (unlikely(!__xive_vm_h_xirr))
581 return H_NOT_AVAILABLE;
582 return __xive_vm_h_xirr(vcpu);
583 } else
584 return xics_rm_h_xirr(vcpu);
585 }
586
587 unsigned long kvmppc_rm_h_xirr_x(struct kvm_vcpu *vcpu)
588 {
589 if (!kvmppc_xics_enabled(vcpu))
590 return H_TOO_HARD;
591 vcpu->arch.regs.gpr[5] = get_tb();
592 if (xics_on_xive()) {
593 if (is_rm())
594 return xive_rm_h_xirr(vcpu);
595 if (unlikely(!__xive_vm_h_xirr))
596 return H_NOT_AVAILABLE;
597 return __xive_vm_h_xirr(vcpu);
598 } else
599 return xics_rm_h_xirr(vcpu);
600 }
601
602 unsigned long kvmppc_rm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
603 {
604 if (!kvmppc_xics_enabled(vcpu))
605 return H_TOO_HARD;
606 if (xics_on_xive()) {
607 if (is_rm())
608 return xive_rm_h_ipoll(vcpu, server);
609 if (unlikely(!__xive_vm_h_ipoll))
610 return H_NOT_AVAILABLE;
611 return __xive_vm_h_ipoll(vcpu, server);
612 } else
613 return H_TOO_HARD;
614 }
615
616 int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
617 unsigned long mfrr)
618 {
619 if (!kvmppc_xics_enabled(vcpu))
620 return H_TOO_HARD;
621 if (xics_on_xive()) {
622 if (is_rm())
623 return xive_rm_h_ipi(vcpu, server, mfrr);
624 if (unlikely(!__xive_vm_h_ipi))
625 return H_NOT_AVAILABLE;
626 return __xive_vm_h_ipi(vcpu, server, mfrr);
627 } else
628 return xics_rm_h_ipi(vcpu, server, mfrr);
629 }
630
631 int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
632 {
633 if (!kvmppc_xics_enabled(vcpu))
634 return H_TOO_HARD;
635 if (xics_on_xive()) {
636 if (is_rm())
637 return xive_rm_h_cppr(vcpu, cppr);
638 if (unlikely(!__xive_vm_h_cppr))
639 return H_NOT_AVAILABLE;
640 return __xive_vm_h_cppr(vcpu, cppr);
641 } else
642 return xics_rm_h_cppr(vcpu, cppr);
643 }
644
645 int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
646 {
647 if (!kvmppc_xics_enabled(vcpu))
648 return H_TOO_HARD;
649 if (xics_on_xive()) {
650 if (is_rm())
651 return xive_rm_h_eoi(vcpu, xirr);
652 if (unlikely(!__xive_vm_h_eoi))
653 return H_NOT_AVAILABLE;
654 return __xive_vm_h_eoi(vcpu, xirr);
655 } else
656 return xics_rm_h_eoi(vcpu, xirr);
657 }
658 #endif
659
660 void kvmppc_bad_interrupt(struct pt_regs *regs)
661 {
662
663
664
665
666 if (TRAP(regs) == 0x100) {
667 get_paca()->in_nmi++;
668 system_reset_exception(regs);
669 get_paca()->in_nmi--;
670 } else if (TRAP(regs) == 0x200) {
671 machine_check_exception(regs);
672 } else {
673 die("Bad interrupt in KVM entry/exit code", regs, SIGABRT);
674 }
675 panic("Bad KVM trap");
676 }
677
678
679
680
681
682
683 #define PHASE_REALMODE 1
684 #define PHASE_SET_LPCR 2
685 #define PHASE_OUT_OF_GUEST 4
686 #define PHASE_RESET_LPCR 8
687
688 #define ALL(p) (((p) << 24) | ((p) << 16) | ((p) << 8) | (p))
689
690 static void wait_for_sync(struct kvm_split_mode *sip, int phase)
691 {
692 int thr = local_paca->kvm_hstate.tid;
693
694 sip->lpcr_sync.phase[thr] |= phase;
695 phase = ALL(phase);
696 while ((sip->lpcr_sync.allphases & phase) != phase) {
697 HMT_low();
698 barrier();
699 }
700 HMT_medium();
701 }
702
703 void kvmhv_p9_set_lpcr(struct kvm_split_mode *sip)
704 {
705 unsigned long rb, set;
706
707
708 wait_for_sync(sip, PHASE_REALMODE);
709
710
711 mtspr(SPRN_LPCR, sip->lpcr_req);
712 mtspr(SPRN_LPID, sip->lpidr_req);
713 isync();
714
715
716 if (local_paca->kvm_hstate.tid == 0) {
717 sip->do_set = 0;
718 asm volatile("ptesync" : : : "memory");
719 for (set = 0; set < POWER9_TLB_SETS_RADIX; ++set) {
720 rb = TLBIEL_INVAL_SET_LPID +
721 (set << TLBIEL_INVAL_SET_SHIFT);
722 asm volatile(PPC_TLBIEL(%0, %1, 0, 0, 0) : :
723 "r" (rb), "r" (0));
724 }
725 asm volatile("ptesync" : : : "memory");
726 }
727
728
729 wait_for_sync(sip, PHASE_SET_LPCR);
730
731 smp_rmb();
732 }
733
734
735
736
737
738
739 void kvmhv_p9_restore_lpcr(struct kvm_split_mode *sip)
740 {
741
742 wait_for_sync(sip, PHASE_OUT_OF_GUEST);
743
744 mtspr(SPRN_LPID, 0);
745 mtspr(SPRN_LPCR, sip->host_lpcr);
746 isync();
747
748 if (local_paca->kvm_hstate.tid == 0) {
749 sip->do_restore = 0;
750 smp_wmb();
751 }
752
753 wait_for_sync(sip, PHASE_RESET_LPCR);
754 smp_mb();
755 local_paca->kvm_hstate.kvm_split_mode = NULL;
756 }
757
758
759
760
761
762 void kvmppc_guest_entry_inject_int(struct kvm_vcpu *vcpu)
763 {
764 int ext;
765 unsigned long vec = 0;
766 unsigned long lpcr;
767
768
769 ext = (vcpu->arch.pending_exceptions >> BOOK3S_IRQPRIO_EXTERNAL) & 1;
770 lpcr = mfspr(SPRN_LPCR);
771 lpcr |= ext << LPCR_MER_SH;
772 mtspr(SPRN_LPCR, lpcr);
773 isync();
774
775 if (vcpu->arch.shregs.msr & MSR_EE) {
776 if (ext) {
777 vec = BOOK3S_INTERRUPT_EXTERNAL;
778 } else {
779 long int dec = mfspr(SPRN_DEC);
780 if (!(lpcr & LPCR_LD))
781 dec = (int) dec;
782 if (dec < 0)
783 vec = BOOK3S_INTERRUPT_DECREMENTER;
784 }
785 }
786 if (vec) {
787 unsigned long msr, old_msr = vcpu->arch.shregs.msr;
788
789 kvmppc_set_srr0(vcpu, kvmppc_get_pc(vcpu));
790 kvmppc_set_srr1(vcpu, old_msr);
791 kvmppc_set_pc(vcpu, vec);
792 msr = vcpu->arch.intr_msr;
793 if (MSR_TM_ACTIVE(old_msr))
794 msr |= MSR_TS_S;
795 vcpu->arch.shregs.msr = msr;
796 }
797
798 if (vcpu->arch.doorbell_request) {
799 mtspr(SPRN_DPDES, 1);
800 vcpu->arch.vcore->dpdes = 1;
801 smp_wmb();
802 vcpu->arch.doorbell_request = 0;
803 }
804 }
805
806 static void flush_guest_tlb(struct kvm *kvm)
807 {
808 unsigned long rb, set;
809
810 rb = PPC_BIT(52);
811 if (kvm_is_radix(kvm)) {
812
813 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
814 : : "r" (rb), "i" (1), "i" (1), "i" (2),
815 "r" (0) : "memory");
816 for (set = 1; set < kvm->arch.tlb_sets; ++set) {
817 rb += PPC_BIT(51);
818
819 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
820 : : "r" (rb), "i" (1), "i" (1), "i" (0),
821 "r" (0) : "memory");
822 }
823 asm volatile("ptesync": : :"memory");
824 asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
825 } else {
826 for (set = 0; set < kvm->arch.tlb_sets; ++set) {
827
828 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
829 : : "r" (rb), "i" (0), "i" (0), "i" (0),
830 "r" (0) : "memory");
831 rb += PPC_BIT(51);
832 }
833 asm volatile("ptesync": : :"memory");
834 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
835 }
836 }
837
838 void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu,
839 struct kvm_nested_guest *nested)
840 {
841 cpumask_t *need_tlb_flush;
842
843
844
845
846
847
848
849 if (cpu_has_feature(CPU_FTR_ARCH_300))
850 pcpu = cpu_first_thread_sibling(pcpu);
851
852 if (nested)
853 need_tlb_flush = &nested->need_tlb_flush;
854 else
855 need_tlb_flush = &kvm->arch.need_tlb_flush;
856
857 if (cpumask_test_cpu(pcpu, need_tlb_flush)) {
858 flush_guest_tlb(kvm);
859
860
861 cpumask_clear_cpu(pcpu, need_tlb_flush);
862 }
863 }
864 EXPORT_SYMBOL_GPL(kvmppc_check_need_tlb_flush);