This source file includes following definitions.
- ics_rm_check_resend
- icp_send_hcore_msg
- icp_send_hcore_msg
- grab_next_hostcore
- find_available_hostcore
- icp_rm_set_vcpu_irq
- icp_rm_clr_vcpu_irq
- icp_rm_try_update
- check_too_hard
- icp_rm_check_resend
- icp_rm_try_to_deliver
- icp_rm_deliver_irq
- icp_rm_down_cppr
- xics_rm_h_xirr
- xics_rm_h_ipi
- xics_rm_h_cppr
- ics_rm_eoi
- xics_rm_h_eoi
- icp_eoi
- xics_opal_set_server
- this_cpu_inc_rm
- kvmppc_rm_handle_irq_desc
- kvmppc_deliver_irq_passthru
- rm_host_ipi_action
- kvmppc_xics_ipi_action
1
2
3
4
5
6
7 #include <linux/kernel.h>
8 #include <linux/kvm_host.h>
9 #include <linux/err.h>
10 #include <linux/kernel_stat.h>
11
12 #include <asm/kvm_book3s.h>
13 #include <asm/kvm_ppc.h>
14 #include <asm/hvcall.h>
15 #include <asm/xics.h>
16 #include <asm/synch.h>
17 #include <asm/cputhreads.h>
18 #include <asm/pgtable.h>
19 #include <asm/ppc-opcode.h>
20 #include <asm/pnv-pci.h>
21 #include <asm/opal.h>
22 #include <asm/smp.h>
23
24 #include "book3s_xics.h"
25
26 #define DEBUG_PASSUP
27
28 int h_ipi_redirect = 1;
29 EXPORT_SYMBOL(h_ipi_redirect);
30 int kvm_irq_bypass = 1;
31 EXPORT_SYMBOL(kvm_irq_bypass);
32
33 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
34 u32 new_irq, bool check_resend);
35 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu);
36
37
38 static void ics_rm_check_resend(struct kvmppc_xics *xics,
39 struct kvmppc_ics *ics, struct kvmppc_icp *icp)
40 {
41 int i;
42
43 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
44 struct ics_irq_state *state = &ics->irq_state[i];
45 if (state->resend)
46 icp_rm_deliver_irq(xics, icp, state->number, true);
47 }
48
49 }
50
51
52
53 #ifdef CONFIG_SMP
54 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu)
55 {
56 int hcpu;
57
58 hcpu = hcore << threads_shift;
59 kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu;
60 smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION);
61 kvmppc_set_host_ipi(hcpu);
62 smp_mb();
63 kvmhv_rm_send_ipi(hcpu);
64 }
65 #else
66 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { }
67 #endif
68
69
70
71
72
73
74
75
76
77
78
79
80
81 static inline int grab_next_hostcore(int start,
82 struct kvmppc_host_rm_core *rm_core, int max, int action)
83 {
84 bool success;
85 int core;
86 union kvmppc_rm_state old, new;
87
88 for (core = start + 1; core < max; core++) {
89 old = new = READ_ONCE(rm_core[core].rm_state);
90
91 if (!old.in_host || old.rm_action)
92 continue;
93
94
95 new.rm_action = action;
96
97 success = cmpxchg64(&rm_core[core].rm_state.raw,
98 old.raw, new.raw) == old.raw;
99 if (success) {
100
101
102
103
104
105
106 smp_wmb();
107 return core;
108 }
109 }
110
111 return -1;
112 }
113
114 static inline int find_available_hostcore(int action)
115 {
116 int core;
117 int my_core = smp_processor_id() >> threads_shift;
118 struct kvmppc_host_rm_core *rm_core = kvmppc_host_rm_ops_hv->rm_core;
119
120 core = grab_next_hostcore(my_core, rm_core, cpu_nr_cores(), action);
121 if (core == -1)
122 core = grab_next_hostcore(core, rm_core, my_core, action);
123
124 return core;
125 }
126
127 static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu,
128 struct kvm_vcpu *this_vcpu)
129 {
130 struct kvmppc_icp *this_icp = this_vcpu->arch.icp;
131 int cpu;
132 int hcore;
133
134
135 vcpu->stat.queue_intr++;
136 set_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
137
138
139 if (vcpu == this_vcpu) {
140 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER);
141 return;
142 }
143
144 if (xive_enabled() && kvmhv_on_pseries()) {
145
146 this_icp->rm_action |= XICS_RM_KICK_VCPU;
147 this_icp->rm_kick_target = vcpu;
148 return;
149 }
150
151
152
153
154
155
156 cpu = vcpu->arch.thread_cpu;
157 if (cpu < 0 || cpu >= nr_cpu_ids) {
158 hcore = -1;
159 if (kvmppc_host_rm_ops_hv && h_ipi_redirect)
160 hcore = find_available_hostcore(XICS_RM_KICK_VCPU);
161 if (hcore != -1) {
162 icp_send_hcore_msg(hcore, vcpu);
163 } else {
164 this_icp->rm_action |= XICS_RM_KICK_VCPU;
165 this_icp->rm_kick_target = vcpu;
166 }
167 return;
168 }
169
170 smp_mb();
171 kvmhv_rm_send_ipi(cpu);
172 }
173
174 static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu)
175 {
176
177 clear_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
178 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER);
179 }
180
181 static inline bool icp_rm_try_update(struct kvmppc_icp *icp,
182 union kvmppc_icp_state old,
183 union kvmppc_icp_state new)
184 {
185 struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu;
186 bool success;
187
188
189 new.out_ee = (new.xisr && (new.pending_pri < new.cppr));
190
191
192 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;
193 if (!success)
194 goto bail;
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211 if (new.out_ee)
212 icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu);
213
214
215 this_vcpu->arch.icp->rm_dbgstate = new;
216 this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu;
217
218 bail:
219 return success;
220 }
221
222 static inline int check_too_hard(struct kvmppc_xics *xics,
223 struct kvmppc_icp *icp)
224 {
225 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS;
226 }
227
228 static void icp_rm_check_resend(struct kvmppc_xics *xics,
229 struct kvmppc_icp *icp)
230 {
231 u32 icsid;
232
233
234 smp_rmb();
235 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {
236 struct kvmppc_ics *ics = xics->ics[icsid];
237
238 if (!test_and_clear_bit(icsid, icp->resend_map))
239 continue;
240 if (!ics)
241 continue;
242 ics_rm_check_resend(xics, ics, icp);
243 }
244 }
245
246 static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,
247 u32 *reject)
248 {
249 union kvmppc_icp_state old_state, new_state;
250 bool success;
251
252 do {
253 old_state = new_state = READ_ONCE(icp->state);
254
255 *reject = 0;
256
257
258 success = new_state.cppr > priority &&
259 new_state.mfrr > priority &&
260 new_state.pending_pri > priority;
261
262
263
264
265
266 if (success) {
267 *reject = new_state.xisr;
268 new_state.xisr = irq;
269 new_state.pending_pri = priority;
270 } else {
271
272
273
274
275
276 new_state.need_resend = true;
277 }
278
279 } while (!icp_rm_try_update(icp, old_state, new_state));
280
281 return success;
282 }
283
284 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
285 u32 new_irq, bool check_resend)
286 {
287 struct ics_irq_state *state;
288 struct kvmppc_ics *ics;
289 u32 reject;
290 u16 src;
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307 again:
308
309 ics = kvmppc_xics_find_ics(xics, new_irq, &src);
310 if (!ics) {
311
312 xics->err_noics++;
313 return;
314 }
315 state = &ics->irq_state[src];
316
317
318 arch_spin_lock(&ics->lock);
319
320
321 if (!icp || state->server != icp->server_num) {
322 icp = kvmppc_xics_find_server(xics->kvm, state->server);
323 if (!icp) {
324
325 xics->err_noicp++;
326 goto out;
327 }
328 }
329
330 if (check_resend)
331 if (!state->resend)
332 goto out;
333
334
335 state->resend = 0;
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352 if (state->priority == MASKED) {
353 state->masked_pending = 1;
354 goto out;
355 }
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373 if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) {
374
375
376
377 if (reject && reject != XICS_IPI) {
378 arch_spin_unlock(&ics->lock);
379 icp->n_reject++;
380 new_irq = reject;
381 check_resend = 0;
382 goto again;
383 }
384 } else {
385
386
387
388
389 state->resend = 1;
390
391
392
393
394
395 smp_wmb();
396 set_bit(ics->icsid, icp->resend_map);
397
398
399
400
401
402
403
404 smp_mb();
405 if (!icp->state.need_resend) {
406 state->resend = 0;
407 arch_spin_unlock(&ics->lock);
408 check_resend = 0;
409 goto again;
410 }
411 }
412 out:
413 arch_spin_unlock(&ics->lock);
414 }
415
416 static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
417 u8 new_cppr)
418 {
419 union kvmppc_icp_state old_state, new_state;
420 bool resend;
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451 do {
452 old_state = new_state = READ_ONCE(icp->state);
453
454
455 new_state.cppr = new_cppr;
456
457
458
459
460
461
462
463
464
465
466 if (new_state.mfrr < new_cppr &&
467 new_state.mfrr <= new_state.pending_pri) {
468 new_state.pending_pri = new_state.mfrr;
469 new_state.xisr = XICS_IPI;
470 }
471
472
473 resend = new_state.need_resend;
474 new_state.need_resend = 0;
475
476 } while (!icp_rm_try_update(icp, old_state, new_state));
477
478
479
480
481
482
483 if (resend) {
484 icp->n_check_resend++;
485 icp_rm_check_resend(xics, icp);
486 }
487 }
488
489
490 unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu)
491 {
492 union kvmppc_icp_state old_state, new_state;
493 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
494 struct kvmppc_icp *icp = vcpu->arch.icp;
495 u32 xirr;
496
497 if (!xics || !xics->real_mode)
498 return H_TOO_HARD;
499
500
501 icp_rm_clr_vcpu_irq(icp->vcpu);
502
503
504
505
506
507
508
509
510 do {
511 old_state = new_state = READ_ONCE(icp->state);
512
513 xirr = old_state.xisr | (((u32)old_state.cppr) << 24);
514 if (!old_state.xisr)
515 break;
516 new_state.cppr = new_state.pending_pri;
517 new_state.pending_pri = 0xff;
518 new_state.xisr = 0;
519
520 } while (!icp_rm_try_update(icp, old_state, new_state));
521
522
523 vcpu->arch.regs.gpr[4] = xirr;
524
525 return check_too_hard(xics, icp);
526 }
527
528 int xics_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
529 unsigned long mfrr)
530 {
531 union kvmppc_icp_state old_state, new_state;
532 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
533 struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp;
534 u32 reject;
535 bool resend;
536 bool local;
537
538 if (!xics || !xics->real_mode)
539 return H_TOO_HARD;
540
541 local = this_icp->server_num == server;
542 if (local)
543 icp = this_icp;
544 else
545 icp = kvmppc_xics_find_server(vcpu->kvm, server);
546 if (!icp)
547 return H_PARAMETER;
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576 do {
577 old_state = new_state = READ_ONCE(icp->state);
578
579
580 new_state.mfrr = mfrr;
581
582
583 reject = 0;
584 resend = false;
585 if (mfrr < new_state.cppr) {
586
587 if (mfrr <= new_state.pending_pri) {
588 reject = new_state.xisr;
589 new_state.pending_pri = mfrr;
590 new_state.xisr = XICS_IPI;
591 }
592 }
593
594 if (mfrr > old_state.mfrr) {
595 resend = new_state.need_resend;
596 new_state.need_resend = 0;
597 }
598 } while (!icp_rm_try_update(icp, old_state, new_state));
599
600
601 if (reject && reject != XICS_IPI) {
602 this_icp->n_reject++;
603 icp_rm_deliver_irq(xics, icp, reject, false);
604 }
605
606
607 if (resend) {
608 this_icp->n_check_resend++;
609 icp_rm_check_resend(xics, icp);
610 }
611
612 return check_too_hard(xics, this_icp);
613 }
614
615 int xics_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
616 {
617 union kvmppc_icp_state old_state, new_state;
618 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
619 struct kvmppc_icp *icp = vcpu->arch.icp;
620 u32 reject;
621
622 if (!xics || !xics->real_mode)
623 return H_TOO_HARD;
624
625
626
627
628
629
630
631
632 if (cppr > icp->state.cppr) {
633 icp_rm_down_cppr(xics, icp, cppr);
634 goto bail;
635 } else if (cppr == icp->state.cppr)
636 return H_SUCCESS;
637
638
639
640
641
642
643
644
645
646
647
648
649 icp_rm_clr_vcpu_irq(icp->vcpu);
650
651 do {
652 old_state = new_state = READ_ONCE(icp->state);
653
654 reject = 0;
655 new_state.cppr = cppr;
656
657 if (cppr <= new_state.pending_pri) {
658 reject = new_state.xisr;
659 new_state.xisr = 0;
660 new_state.pending_pri = 0xff;
661 }
662
663 } while (!icp_rm_try_update(icp, old_state, new_state));
664
665
666
667
668
669 if (reject && reject != XICS_IPI) {
670 icp->n_reject++;
671 icp_rm_deliver_irq(xics, icp, reject, false);
672 }
673 bail:
674 return check_too_hard(xics, icp);
675 }
676
677 static int ics_rm_eoi(struct kvm_vcpu *vcpu, u32 irq)
678 {
679 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
680 struct kvmppc_icp *icp = vcpu->arch.icp;
681 struct kvmppc_ics *ics;
682 struct ics_irq_state *state;
683 u16 src;
684 u32 pq_old, pq_new;
685
686
687
688
689
690
691
692
693
694 ics = kvmppc_xics_find_ics(xics, irq, &src);
695 if (!ics)
696 goto bail;
697
698 state = &ics->irq_state[src];
699
700 if (state->lsi)
701 pq_new = state->pq_state;
702 else
703 do {
704 pq_old = state->pq_state;
705 pq_new = pq_old >> 1;
706 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);
707
708 if (pq_new & PQ_PRESENTED)
709 icp_rm_deliver_irq(xics, NULL, irq, false);
710
711 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) {
712 icp->rm_action |= XICS_RM_NOTIFY_EOI;
713 icp->rm_eoied_irq = irq;
714 }
715
716 if (state->host_irq) {
717 ++vcpu->stat.pthru_all;
718 if (state->intr_cpu != -1) {
719 int pcpu = raw_smp_processor_id();
720
721 pcpu = cpu_first_thread_sibling(pcpu);
722 ++vcpu->stat.pthru_host;
723 if (state->intr_cpu != pcpu) {
724 ++vcpu->stat.pthru_bad_aff;
725 xics_opal_set_server(state->host_irq, pcpu);
726 }
727 state->intr_cpu = -1;
728 }
729 }
730
731 bail:
732 return check_too_hard(xics, icp);
733 }
734
735 int xics_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
736 {
737 struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
738 struct kvmppc_icp *icp = vcpu->arch.icp;
739 u32 irq = xirr & 0x00ffffff;
740
741 if (!xics || !xics->real_mode)
742 return H_TOO_HARD;
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758 icp_rm_down_cppr(xics, icp, xirr >> 24);
759
760
761 if (irq == XICS_IPI)
762 return check_too_hard(xics, icp);
763
764 return ics_rm_eoi(vcpu, irq);
765 }
766
767 unsigned long eoi_rc;
768
769 static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again)
770 {
771 void __iomem *xics_phys;
772 int64_t rc;
773
774 if (kvmhv_on_pseries()) {
775 unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
776
777 iosync();
778 plpar_hcall_raw(H_EOI, retbuf, hwirq);
779 return;
780 }
781
782 rc = pnv_opal_pci_msi_eoi(c, hwirq);
783
784 if (rc)
785 eoi_rc = rc;
786
787 iosync();
788
789
790 xics_phys = local_paca->kvm_hstate.xics_phys;
791 if (xics_phys) {
792 __raw_rm_writel(xirr, xics_phys + XICS_XIRR);
793 } else {
794 rc = opal_int_eoi(be32_to_cpu(xirr));
795 *again = rc > 0;
796 }
797 }
798
799 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu)
800 {
801 unsigned int mangle_cpu = get_hard_smp_processor_id(server_cpu) << 2;
802
803 return opal_set_xive(hw_irq, mangle_cpu, DEFAULT_PRIORITY);
804 }
805
806
807
808
809
810
811
812
813 static inline void this_cpu_inc_rm(unsigned int __percpu *addr)
814 {
815 unsigned long l;
816 unsigned int *raddr;
817 int cpu = smp_processor_id();
818
819 raddr = per_cpu_ptr(addr, cpu);
820 l = (unsigned long)raddr;
821
822 if (get_region_id(l) == VMALLOC_REGION_ID) {
823 l = vmalloc_to_phys(raddr);
824 raddr = (unsigned int *)l;
825 }
826 ++*raddr;
827 }
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847 static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc)
848 {
849 this_cpu_inc_rm(desc->kstat_irqs);
850 __this_cpu_inc(kstat.irqs_sum);
851 }
852
853 long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu,
854 __be32 xirr,
855 struct kvmppc_irq_map *irq_map,
856 struct kvmppc_passthru_irqmap *pimap,
857 bool *again)
858 {
859 struct kvmppc_xics *xics;
860 struct kvmppc_icp *icp;
861 struct kvmppc_ics *ics;
862 struct ics_irq_state *state;
863 u32 irq;
864 u16 src;
865 u32 pq_old, pq_new;
866
867 irq = irq_map->v_hwirq;
868 xics = vcpu->kvm->arch.xics;
869 icp = vcpu->arch.icp;
870
871 kvmppc_rm_handle_irq_desc(irq_map->desc);
872
873 ics = kvmppc_xics_find_ics(xics, irq, &src);
874 if (!ics)
875 return 2;
876
877 state = &ics->irq_state[src];
878
879
880 do {
881 pq_old = state->pq_state;
882 pq_new = ((pq_old << 1) & 3) | PQ_PRESENTED;
883 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);
884
885
886 if (pq_new == PQ_PRESENTED)
887 icp_rm_deliver_irq(xics, icp, irq, false);
888
889
890 icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr,
891 again);
892
893 if (check_too_hard(xics, icp) == H_TOO_HARD)
894 return 2;
895 else
896 return -2;
897 }
898
899
900
901
902
903
904 static void rm_host_ipi_action(int action, void *data)
905 {
906 switch (action) {
907 case XICS_RM_KICK_VCPU:
908 kvmppc_host_rm_ops_hv->vcpu_kick(data);
909 break;
910 default:
911 WARN(1, "Unexpected rm_action=%d data=%p\n", action, data);
912 break;
913 }
914
915 }
916
917 void kvmppc_xics_ipi_action(void)
918 {
919 int core;
920 unsigned int cpu = smp_processor_id();
921 struct kvmppc_host_rm_core *rm_corep;
922
923 core = cpu >> threads_shift;
924 rm_corep = &kvmppc_host_rm_ops_hv->rm_core[core];
925
926 if (rm_corep->rm_data) {
927 rm_host_ipi_action(rm_corep->rm_state.rm_action,
928 rm_corep->rm_data);
929
930 rm_corep->rm_data = NULL;
931 smp_wmb();
932 rm_corep->rm_state.rm_action = 0;
933 }
934 }