This source file includes following definitions.
- clear_evtchn_to_irq_row
- clear_evtchn_to_irq_all
- set_evtchn_to_irq
- get_evtchn_to_irq
- info_for_irq
- xen_irq_info_common_setup
- xen_irq_info_evtchn_setup
- xen_irq_info_ipi_setup
- xen_irq_info_virq_setup
- xen_irq_info_pirq_setup
- xen_irq_info_cleanup
- evtchn_from_irq
- irq_from_evtchn
- irq_from_virq
- ipi_from_irq
- virq_from_irq
- pirq_from_irq
- type_from_irq
- cpu_from_irq
- cpu_from_evtchn
- pirq_check_eoi_map
- pirq_needs_eoi_flag
- bind_evtchn_to_cpu
- notify_remote_via_irq
- xen_irq_init
- xen_allocate_irqs_dynamic
- xen_allocate_irq_dynamic
- xen_allocate_irq_gsi
- xen_free_irq
- xen_evtchn_close
- pirq_query_unmask
- eoi_pirq
- mask_ack_pirq
- __startup_pirq
- startup_pirq
- shutdown_pirq
- enable_pirq
- disable_pirq
- xen_irq_from_gsi
- __unbind_from_irq
- xen_bind_pirq_gsi_to_irq
- xen_allocate_pirq_msi
- xen_bind_pirq_msi_to_irq
- xen_destroy_irq
- xen_irq_from_pirq
- xen_pirq_from_irq
- bind_evtchn_to_irq
- bind_ipi_to_irq
- bind_interdomain_evtchn_to_irq
- find_virq
- xen_evtchn_nr_channels
- bind_virq_to_irq
- unbind_from_irq
- bind_evtchn_to_irqhandler
- bind_interdomain_evtchn_to_irqhandler
- bind_virq_to_irqhandler
- bind_ipi_to_irqhandler
- unbind_from_irqhandler
- xen_set_irq_priority
- evtchn_make_refcounted
- evtchn_get
- evtchn_put
- xen_send_IPI_one
- __xen_evtchn_do_upcall
- xen_evtchn_do_upcall
- xen_hvm_evtchn_do_upcall
- rebind_evtchn_irq
- xen_rebind_evtchn_to_cpu
- set_affinity_irq
- xen_set_affinity_evtchn
- enable_dynirq
- disable_dynirq
- ack_dynirq
- mask_ack_dynirq
- retrigger_dynirq
- restore_pirqs
- restore_cpu_virqs
- restore_cpu_ipis
- xen_clear_irq_pending
- xen_set_irq_pending
- xen_test_irq_pending
- xen_poll_irq_timeout
- xen_poll_irq
- xen_test_irq_shared
- xen_irq_resume
- xen_set_callback_via
- xen_callback_vector
- xen_callback_vector
- xen_init_IRQ
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
26
27 #include <linux/linkage.h>
28 #include <linux/interrupt.h>
29 #include <linux/irq.h>
30 #include <linux/moduleparam.h>
31 #include <linux/string.h>
32 #include <linux/memblock.h>
33 #include <linux/slab.h>
34 #include <linux/irqnr.h>
35 #include <linux/pci.h>
36
37 #ifdef CONFIG_X86
38 #include <asm/desc.h>
39 #include <asm/ptrace.h>
40 #include <asm/irq.h>
41 #include <asm/io_apic.h>
42 #include <asm/i8259.h>
43 #include <asm/xen/pci.h>
44 #endif
45 #include <asm/sync_bitops.h>
46 #include <asm/xen/hypercall.h>
47 #include <asm/xen/hypervisor.h>
48 #include <xen/page.h>
49
50 #include <xen/xen.h>
51 #include <xen/hvm.h>
52 #include <xen/xen-ops.h>
53 #include <xen/events.h>
54 #include <xen/interface/xen.h>
55 #include <xen/interface/event_channel.h>
56 #include <xen/interface/hvm/hvm_op.h>
57 #include <xen/interface/hvm/params.h>
58 #include <xen/interface/physdev.h>
59 #include <xen/interface/sched.h>
60 #include <xen/interface/vcpu.h>
61 #include <asm/hw_irq.h>
62
63 #include "events_internal.h"
64
65 const struct evtchn_ops *evtchn_ops;
66
67
68
69
70
71 static DEFINE_MUTEX(irq_mapping_update_lock);
72
73 static LIST_HEAD(xen_irq_list_head);
74
75
76 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
77
78
79 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
80
81 int **evtchn_to_irq;
82 #ifdef CONFIG_X86
83 static unsigned long *pirq_eoi_map;
84 #endif
85 static bool (*pirq_needs_eoi)(unsigned irq);
86
87 #define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
88 #define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
89 #define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
90
91
92 #define VALID_EVTCHN(chn) ((chn) != 0)
93
94 static struct irq_chip xen_dynamic_chip;
95 static struct irq_chip xen_percpu_chip;
96 static struct irq_chip xen_pirq_chip;
97 static void enable_dynirq(struct irq_data *data);
98 static void disable_dynirq(struct irq_data *data);
99
100 static void clear_evtchn_to_irq_row(unsigned row)
101 {
102 unsigned col;
103
104 for (col = 0; col < EVTCHN_PER_ROW; col++)
105 evtchn_to_irq[row][col] = -1;
106 }
107
108 static void clear_evtchn_to_irq_all(void)
109 {
110 unsigned row;
111
112 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
113 if (evtchn_to_irq[row] == NULL)
114 continue;
115 clear_evtchn_to_irq_row(row);
116 }
117 }
118
119 static int set_evtchn_to_irq(unsigned evtchn, unsigned irq)
120 {
121 unsigned row;
122 unsigned col;
123
124 if (evtchn >= xen_evtchn_max_channels())
125 return -EINVAL;
126
127 row = EVTCHN_ROW(evtchn);
128 col = EVTCHN_COL(evtchn);
129
130 if (evtchn_to_irq[row] == NULL) {
131
132 if (irq == -1)
133 return 0;
134
135 evtchn_to_irq[row] = (int *)get_zeroed_page(GFP_KERNEL);
136 if (evtchn_to_irq[row] == NULL)
137 return -ENOMEM;
138
139 clear_evtchn_to_irq_row(row);
140 }
141
142 evtchn_to_irq[row][col] = irq;
143 return 0;
144 }
145
146 int get_evtchn_to_irq(unsigned evtchn)
147 {
148 if (evtchn >= xen_evtchn_max_channels())
149 return -1;
150 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
151 return -1;
152 return evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)];
153 }
154
155
156 struct irq_info *info_for_irq(unsigned irq)
157 {
158 return irq_get_handler_data(irq);
159 }
160
161
162 static int xen_irq_info_common_setup(struct irq_info *info,
163 unsigned irq,
164 enum xen_irq_type type,
165 unsigned evtchn,
166 unsigned short cpu)
167 {
168 int ret;
169
170 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
171
172 info->type = type;
173 info->irq = irq;
174 info->evtchn = evtchn;
175 info->cpu = cpu;
176
177 ret = set_evtchn_to_irq(evtchn, irq);
178 if (ret < 0)
179 return ret;
180
181 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
182
183 return xen_evtchn_port_setup(info);
184 }
185
186 static int xen_irq_info_evtchn_setup(unsigned irq,
187 unsigned evtchn)
188 {
189 struct irq_info *info = info_for_irq(irq);
190
191 return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0);
192 }
193
194 static int xen_irq_info_ipi_setup(unsigned cpu,
195 unsigned irq,
196 unsigned evtchn,
197 enum ipi_vector ipi)
198 {
199 struct irq_info *info = info_for_irq(irq);
200
201 info->u.ipi = ipi;
202
203 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
204
205 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0);
206 }
207
208 static int xen_irq_info_virq_setup(unsigned cpu,
209 unsigned irq,
210 unsigned evtchn,
211 unsigned virq)
212 {
213 struct irq_info *info = info_for_irq(irq);
214
215 info->u.virq = virq;
216
217 per_cpu(virq_to_irq, cpu)[virq] = irq;
218
219 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0);
220 }
221
222 static int xen_irq_info_pirq_setup(unsigned irq,
223 unsigned evtchn,
224 unsigned pirq,
225 unsigned gsi,
226 uint16_t domid,
227 unsigned char flags)
228 {
229 struct irq_info *info = info_for_irq(irq);
230
231 info->u.pirq.pirq = pirq;
232 info->u.pirq.gsi = gsi;
233 info->u.pirq.domid = domid;
234 info->u.pirq.flags = flags;
235
236 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0);
237 }
238
239 static void xen_irq_info_cleanup(struct irq_info *info)
240 {
241 set_evtchn_to_irq(info->evtchn, -1);
242 info->evtchn = 0;
243 }
244
245
246
247
248 unsigned int evtchn_from_irq(unsigned irq)
249 {
250 if (WARN(irq >= nr_irqs, "Invalid irq %d!\n", irq))
251 return 0;
252
253 return info_for_irq(irq)->evtchn;
254 }
255
256 unsigned irq_from_evtchn(unsigned int evtchn)
257 {
258 return get_evtchn_to_irq(evtchn);
259 }
260 EXPORT_SYMBOL_GPL(irq_from_evtchn);
261
262 int irq_from_virq(unsigned int cpu, unsigned int virq)
263 {
264 return per_cpu(virq_to_irq, cpu)[virq];
265 }
266
267 static enum ipi_vector ipi_from_irq(unsigned irq)
268 {
269 struct irq_info *info = info_for_irq(irq);
270
271 BUG_ON(info == NULL);
272 BUG_ON(info->type != IRQT_IPI);
273
274 return info->u.ipi;
275 }
276
277 static unsigned virq_from_irq(unsigned irq)
278 {
279 struct irq_info *info = info_for_irq(irq);
280
281 BUG_ON(info == NULL);
282 BUG_ON(info->type != IRQT_VIRQ);
283
284 return info->u.virq;
285 }
286
287 static unsigned pirq_from_irq(unsigned irq)
288 {
289 struct irq_info *info = info_for_irq(irq);
290
291 BUG_ON(info == NULL);
292 BUG_ON(info->type != IRQT_PIRQ);
293
294 return info->u.pirq.pirq;
295 }
296
297 static enum xen_irq_type type_from_irq(unsigned irq)
298 {
299 return info_for_irq(irq)->type;
300 }
301
302 unsigned cpu_from_irq(unsigned irq)
303 {
304 return info_for_irq(irq)->cpu;
305 }
306
307 unsigned int cpu_from_evtchn(unsigned int evtchn)
308 {
309 int irq = get_evtchn_to_irq(evtchn);
310 unsigned ret = 0;
311
312 if (irq != -1)
313 ret = cpu_from_irq(irq);
314
315 return ret;
316 }
317
318 #ifdef CONFIG_X86
319 static bool pirq_check_eoi_map(unsigned irq)
320 {
321 return test_bit(pirq_from_irq(irq), pirq_eoi_map);
322 }
323 #endif
324
325 static bool pirq_needs_eoi_flag(unsigned irq)
326 {
327 struct irq_info *info = info_for_irq(irq);
328 BUG_ON(info->type != IRQT_PIRQ);
329
330 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
331 }
332
333 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
334 {
335 int irq = get_evtchn_to_irq(chn);
336 struct irq_info *info = info_for_irq(irq);
337
338 BUG_ON(irq == -1);
339 #ifdef CONFIG_SMP
340 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu));
341 #endif
342 xen_evtchn_port_bind_to_cpu(info, cpu);
343
344 info->cpu = cpu;
345 }
346
347
348
349
350
351
352
353
354
355 void notify_remote_via_irq(int irq)
356 {
357 int evtchn = evtchn_from_irq(irq);
358
359 if (VALID_EVTCHN(evtchn))
360 notify_remote_via_evtchn(evtchn);
361 }
362 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
363
364 static void xen_irq_init(unsigned irq)
365 {
366 struct irq_info *info;
367 #ifdef CONFIG_SMP
368
369 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0));
370 #endif
371
372 info = kzalloc(sizeof(*info), GFP_KERNEL);
373 if (info == NULL)
374 panic("Unable to allocate metadata for IRQ%d\n", irq);
375
376 info->type = IRQT_UNBOUND;
377 info->refcnt = -1;
378
379 irq_set_handler_data(irq, info);
380
381 list_add_tail(&info->list, &xen_irq_list_head);
382 }
383
384 static int __must_check xen_allocate_irqs_dynamic(int nvec)
385 {
386 int i, irq = irq_alloc_descs(-1, 0, nvec, -1);
387
388 if (irq >= 0) {
389 for (i = 0; i < nvec; i++)
390 xen_irq_init(irq + i);
391 }
392
393 return irq;
394 }
395
396 static inline int __must_check xen_allocate_irq_dynamic(void)
397 {
398
399 return xen_allocate_irqs_dynamic(1);
400 }
401
402 static int __must_check xen_allocate_irq_gsi(unsigned gsi)
403 {
404 int irq;
405
406
407
408
409
410
411
412 if (xen_pv_domain() && !xen_initial_domain())
413 return xen_allocate_irq_dynamic();
414
415
416 if (gsi < nr_legacy_irqs())
417 irq = gsi;
418 else
419 irq = irq_alloc_desc_at(gsi, -1);
420
421 xen_irq_init(irq);
422
423 return irq;
424 }
425
426 static void xen_free_irq(unsigned irq)
427 {
428 struct irq_info *info = irq_get_handler_data(irq);
429
430 if (WARN_ON(!info))
431 return;
432
433 list_del(&info->list);
434
435 irq_set_handler_data(irq, NULL);
436
437 WARN_ON(info->refcnt > 0);
438
439 kfree(info);
440
441
442 if (irq < nr_legacy_irqs())
443 return;
444
445 irq_free_desc(irq);
446 }
447
448 static void xen_evtchn_close(unsigned int port)
449 {
450 struct evtchn_close close;
451
452 close.port = port;
453 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
454 BUG();
455 }
456
457 static void pirq_query_unmask(int irq)
458 {
459 struct physdev_irq_status_query irq_status;
460 struct irq_info *info = info_for_irq(irq);
461
462 BUG_ON(info->type != IRQT_PIRQ);
463
464 irq_status.irq = pirq_from_irq(irq);
465 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
466 irq_status.flags = 0;
467
468 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
469 if (irq_status.flags & XENIRQSTAT_needs_eoi)
470 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
471 }
472
473 static void eoi_pirq(struct irq_data *data)
474 {
475 int evtchn = evtchn_from_irq(data->irq);
476 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
477 int rc = 0;
478
479 if (!VALID_EVTCHN(evtchn))
480 return;
481
482 if (unlikely(irqd_is_setaffinity_pending(data)) &&
483 likely(!irqd_irq_disabled(data))) {
484 int masked = test_and_set_mask(evtchn);
485
486 clear_evtchn(evtchn);
487
488 irq_move_masked_irq(data);
489
490 if (!masked)
491 unmask_evtchn(evtchn);
492 } else
493 clear_evtchn(evtchn);
494
495 if (pirq_needs_eoi(data->irq)) {
496 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
497 WARN_ON(rc);
498 }
499 }
500
501 static void mask_ack_pirq(struct irq_data *data)
502 {
503 disable_dynirq(data);
504 eoi_pirq(data);
505 }
506
507 static unsigned int __startup_pirq(unsigned int irq)
508 {
509 struct evtchn_bind_pirq bind_pirq;
510 struct irq_info *info = info_for_irq(irq);
511 int evtchn = evtchn_from_irq(irq);
512 int rc;
513
514 BUG_ON(info->type != IRQT_PIRQ);
515
516 if (VALID_EVTCHN(evtchn))
517 goto out;
518
519 bind_pirq.pirq = pirq_from_irq(irq);
520
521 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
522 BIND_PIRQ__WILL_SHARE : 0;
523 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
524 if (rc != 0) {
525 pr_warn("Failed to obtain physical IRQ %d\n", irq);
526 return 0;
527 }
528 evtchn = bind_pirq.port;
529
530 pirq_query_unmask(irq);
531
532 rc = set_evtchn_to_irq(evtchn, irq);
533 if (rc)
534 goto err;
535
536 info->evtchn = evtchn;
537 bind_evtchn_to_cpu(evtchn, 0);
538
539 rc = xen_evtchn_port_setup(info);
540 if (rc)
541 goto err;
542
543 out:
544 unmask_evtchn(evtchn);
545 eoi_pirq(irq_get_irq_data(irq));
546
547 return 0;
548
549 err:
550 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
551 xen_evtchn_close(evtchn);
552 return 0;
553 }
554
555 static unsigned int startup_pirq(struct irq_data *data)
556 {
557 return __startup_pirq(data->irq);
558 }
559
560 static void shutdown_pirq(struct irq_data *data)
561 {
562 unsigned int irq = data->irq;
563 struct irq_info *info = info_for_irq(irq);
564 unsigned evtchn = evtchn_from_irq(irq);
565
566 BUG_ON(info->type != IRQT_PIRQ);
567
568 if (!VALID_EVTCHN(evtchn))
569 return;
570
571 mask_evtchn(evtchn);
572 xen_evtchn_close(evtchn);
573 xen_irq_info_cleanup(info);
574 }
575
576 static void enable_pirq(struct irq_data *data)
577 {
578 enable_dynirq(data);
579 }
580
581 static void disable_pirq(struct irq_data *data)
582 {
583 disable_dynirq(data);
584 }
585
586 int xen_irq_from_gsi(unsigned gsi)
587 {
588 struct irq_info *info;
589
590 list_for_each_entry(info, &xen_irq_list_head, list) {
591 if (info->type != IRQT_PIRQ)
592 continue;
593
594 if (info->u.pirq.gsi == gsi)
595 return info->irq;
596 }
597
598 return -1;
599 }
600 EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
601
602 static void __unbind_from_irq(unsigned int irq)
603 {
604 int evtchn = evtchn_from_irq(irq);
605 struct irq_info *info = irq_get_handler_data(irq);
606
607 if (info->refcnt > 0) {
608 info->refcnt--;
609 if (info->refcnt != 0)
610 return;
611 }
612
613 if (VALID_EVTCHN(evtchn)) {
614 unsigned int cpu = cpu_from_irq(irq);
615
616 xen_evtchn_close(evtchn);
617
618 switch (type_from_irq(irq)) {
619 case IRQT_VIRQ:
620 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1;
621 break;
622 case IRQT_IPI:
623 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1;
624 break;
625 default:
626 break;
627 }
628
629 xen_irq_info_cleanup(info);
630 }
631
632 xen_free_irq(irq);
633 }
634
635
636
637
638
639
640
641
642
643
644
645 int xen_bind_pirq_gsi_to_irq(unsigned gsi,
646 unsigned pirq, int shareable, char *name)
647 {
648 int irq = -1;
649 struct physdev_irq irq_op;
650 int ret;
651
652 mutex_lock(&irq_mapping_update_lock);
653
654 irq = xen_irq_from_gsi(gsi);
655 if (irq != -1) {
656 pr_info("%s: returning irq %d for gsi %u\n",
657 __func__, irq, gsi);
658 goto out;
659 }
660
661 irq = xen_allocate_irq_gsi(gsi);
662 if (irq < 0)
663 goto out;
664
665 irq_op.irq = irq;
666 irq_op.vector = 0;
667
668
669
670
671 if (xen_initial_domain() &&
672 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
673 xen_free_irq(irq);
674 irq = -ENOSPC;
675 goto out;
676 }
677
678 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF,
679 shareable ? PIRQ_SHAREABLE : 0);
680 if (ret < 0) {
681 __unbind_from_irq(irq);
682 irq = ret;
683 goto out;
684 }
685
686 pirq_query_unmask(irq);
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702 if (shareable)
703 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
704 handle_fasteoi_irq, name);
705 else
706 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
707 handle_edge_irq, name);
708
709 out:
710 mutex_unlock(&irq_mapping_update_lock);
711
712 return irq;
713 }
714
715 #ifdef CONFIG_PCI_MSI
716 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
717 {
718 int rc;
719 struct physdev_get_free_pirq op_get_free_pirq;
720
721 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
722 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
723
724 WARN_ONCE(rc == -ENOSYS,
725 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
726
727 return rc ? -1 : op_get_free_pirq.pirq;
728 }
729
730 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
731 int pirq, int nvec, const char *name, domid_t domid)
732 {
733 int i, irq, ret;
734
735 mutex_lock(&irq_mapping_update_lock);
736
737 irq = xen_allocate_irqs_dynamic(nvec);
738 if (irq < 0)
739 goto out;
740
741 for (i = 0; i < nvec; i++) {
742 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
743
744 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid,
745 i == 0 ? 0 : PIRQ_MSI_GROUP);
746 if (ret < 0)
747 goto error_irq;
748 }
749
750 ret = irq_set_msi_desc(irq, msidesc);
751 if (ret < 0)
752 goto error_irq;
753 out:
754 mutex_unlock(&irq_mapping_update_lock);
755 return irq;
756 error_irq:
757 while (nvec--)
758 __unbind_from_irq(irq + nvec);
759 mutex_unlock(&irq_mapping_update_lock);
760 return ret;
761 }
762 #endif
763
764 int xen_destroy_irq(int irq)
765 {
766 struct physdev_unmap_pirq unmap_irq;
767 struct irq_info *info = info_for_irq(irq);
768 int rc = -ENOENT;
769
770 mutex_lock(&irq_mapping_update_lock);
771
772
773
774
775
776
777 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
778 unmap_irq.pirq = info->u.pirq.pirq;
779 unmap_irq.domid = info->u.pirq.domid;
780 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
781
782
783
784
785 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
786 pr_info("domain %d does not have %d anymore\n",
787 info->u.pirq.domid, info->u.pirq.pirq);
788 else if (rc) {
789 pr_warn("unmap irq failed %d\n", rc);
790 goto out;
791 }
792 }
793
794 xen_free_irq(irq);
795
796 out:
797 mutex_unlock(&irq_mapping_update_lock);
798 return rc;
799 }
800
801 int xen_irq_from_pirq(unsigned pirq)
802 {
803 int irq;
804
805 struct irq_info *info;
806
807 mutex_lock(&irq_mapping_update_lock);
808
809 list_for_each_entry(info, &xen_irq_list_head, list) {
810 if (info->type != IRQT_PIRQ)
811 continue;
812 irq = info->irq;
813 if (info->u.pirq.pirq == pirq)
814 goto out;
815 }
816 irq = -1;
817 out:
818 mutex_unlock(&irq_mapping_update_lock);
819
820 return irq;
821 }
822
823
824 int xen_pirq_from_irq(unsigned irq)
825 {
826 return pirq_from_irq(irq);
827 }
828 EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
829
830 int bind_evtchn_to_irq(unsigned int evtchn)
831 {
832 int irq;
833 int ret;
834
835 if (evtchn >= xen_evtchn_max_channels())
836 return -ENOMEM;
837
838 mutex_lock(&irq_mapping_update_lock);
839
840 irq = get_evtchn_to_irq(evtchn);
841
842 if (irq == -1) {
843 irq = xen_allocate_irq_dynamic();
844 if (irq < 0)
845 goto out;
846
847 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
848 handle_edge_irq, "event");
849
850 ret = xen_irq_info_evtchn_setup(irq, evtchn);
851 if (ret < 0) {
852 __unbind_from_irq(irq);
853 irq = ret;
854 goto out;
855 }
856
857 bind_evtchn_to_cpu(evtchn, 0);
858 } else {
859 struct irq_info *info = info_for_irq(irq);
860 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
861 }
862
863 out:
864 mutex_unlock(&irq_mapping_update_lock);
865
866 return irq;
867 }
868 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
869
870 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
871 {
872 struct evtchn_bind_ipi bind_ipi;
873 int evtchn, irq;
874 int ret;
875
876 mutex_lock(&irq_mapping_update_lock);
877
878 irq = per_cpu(ipi_to_irq, cpu)[ipi];
879
880 if (irq == -1) {
881 irq = xen_allocate_irq_dynamic();
882 if (irq < 0)
883 goto out;
884
885 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
886 handle_percpu_irq, "ipi");
887
888 bind_ipi.vcpu = xen_vcpu_nr(cpu);
889 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
890 &bind_ipi) != 0)
891 BUG();
892 evtchn = bind_ipi.port;
893
894 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
895 if (ret < 0) {
896 __unbind_from_irq(irq);
897 irq = ret;
898 goto out;
899 }
900 bind_evtchn_to_cpu(evtchn, cpu);
901 } else {
902 struct irq_info *info = info_for_irq(irq);
903 WARN_ON(info == NULL || info->type != IRQT_IPI);
904 }
905
906 out:
907 mutex_unlock(&irq_mapping_update_lock);
908 return irq;
909 }
910
911 int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
912 unsigned int remote_port)
913 {
914 struct evtchn_bind_interdomain bind_interdomain;
915 int err;
916
917 bind_interdomain.remote_dom = remote_domain;
918 bind_interdomain.remote_port = remote_port;
919
920 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
921 &bind_interdomain);
922
923 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port);
924 }
925 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq);
926
927 static int find_virq(unsigned int virq, unsigned int cpu)
928 {
929 struct evtchn_status status;
930 int port, rc = -ENOENT;
931
932 memset(&status, 0, sizeof(status));
933 for (port = 0; port < xen_evtchn_max_channels(); port++) {
934 status.dom = DOMID_SELF;
935 status.port = port;
936 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
937 if (rc < 0)
938 continue;
939 if (status.status != EVTCHNSTAT_virq)
940 continue;
941 if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) {
942 rc = port;
943 break;
944 }
945 }
946 return rc;
947 }
948
949
950
951
952
953
954
955
956 unsigned xen_evtchn_nr_channels(void)
957 {
958 return evtchn_ops->nr_channels();
959 }
960 EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
961
962 int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
963 {
964 struct evtchn_bind_virq bind_virq;
965 int evtchn, irq, ret;
966
967 mutex_lock(&irq_mapping_update_lock);
968
969 irq = per_cpu(virq_to_irq, cpu)[virq];
970
971 if (irq == -1) {
972 irq = xen_allocate_irq_dynamic();
973 if (irq < 0)
974 goto out;
975
976 if (percpu)
977 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
978 handle_percpu_irq, "virq");
979 else
980 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
981 handle_edge_irq, "virq");
982
983 bind_virq.virq = virq;
984 bind_virq.vcpu = xen_vcpu_nr(cpu);
985 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
986 &bind_virq);
987 if (ret == 0)
988 evtchn = bind_virq.port;
989 else {
990 if (ret == -EEXIST)
991 ret = find_virq(virq, cpu);
992 BUG_ON(ret < 0);
993 evtchn = ret;
994 }
995
996 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
997 if (ret < 0) {
998 __unbind_from_irq(irq);
999 irq = ret;
1000 goto out;
1001 }
1002
1003 bind_evtchn_to_cpu(evtchn, cpu);
1004 } else {
1005 struct irq_info *info = info_for_irq(irq);
1006 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1007 }
1008
1009 out:
1010 mutex_unlock(&irq_mapping_update_lock);
1011
1012 return irq;
1013 }
1014
1015 static void unbind_from_irq(unsigned int irq)
1016 {
1017 mutex_lock(&irq_mapping_update_lock);
1018 __unbind_from_irq(irq);
1019 mutex_unlock(&irq_mapping_update_lock);
1020 }
1021
1022 int bind_evtchn_to_irqhandler(unsigned int evtchn,
1023 irq_handler_t handler,
1024 unsigned long irqflags,
1025 const char *devname, void *dev_id)
1026 {
1027 int irq, retval;
1028
1029 irq = bind_evtchn_to_irq(evtchn);
1030 if (irq < 0)
1031 return irq;
1032 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1033 if (retval != 0) {
1034 unbind_from_irq(irq);
1035 return retval;
1036 }
1037
1038 return irq;
1039 }
1040 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1041
1042 int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1043 unsigned int remote_port,
1044 irq_handler_t handler,
1045 unsigned long irqflags,
1046 const char *devname,
1047 void *dev_id)
1048 {
1049 int irq, retval;
1050
1051 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
1052 if (irq < 0)
1053 return irq;
1054
1055 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1056 if (retval != 0) {
1057 unbind_from_irq(irq);
1058 return retval;
1059 }
1060
1061 return irq;
1062 }
1063 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1064
1065 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1066 irq_handler_t handler,
1067 unsigned long irqflags, const char *devname, void *dev_id)
1068 {
1069 int irq, retval;
1070
1071 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
1072 if (irq < 0)
1073 return irq;
1074 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1075 if (retval != 0) {
1076 unbind_from_irq(irq);
1077 return retval;
1078 }
1079
1080 return irq;
1081 }
1082 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1083
1084 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1085 unsigned int cpu,
1086 irq_handler_t handler,
1087 unsigned long irqflags,
1088 const char *devname,
1089 void *dev_id)
1090 {
1091 int irq, retval;
1092
1093 irq = bind_ipi_to_irq(ipi, cpu);
1094 if (irq < 0)
1095 return irq;
1096
1097 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1098 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1099 if (retval != 0) {
1100 unbind_from_irq(irq);
1101 return retval;
1102 }
1103
1104 return irq;
1105 }
1106
1107 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1108 {
1109 struct irq_info *info = irq_get_handler_data(irq);
1110
1111 if (WARN_ON(!info))
1112 return;
1113 free_irq(irq, dev_id);
1114 unbind_from_irq(irq);
1115 }
1116 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1117
1118
1119
1120
1121
1122
1123 int xen_set_irq_priority(unsigned irq, unsigned priority)
1124 {
1125 struct evtchn_set_priority set_priority;
1126
1127 set_priority.port = evtchn_from_irq(irq);
1128 set_priority.priority = priority;
1129
1130 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1131 &set_priority);
1132 }
1133 EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1134
1135 int evtchn_make_refcounted(unsigned int evtchn)
1136 {
1137 int irq = get_evtchn_to_irq(evtchn);
1138 struct irq_info *info;
1139
1140 if (irq == -1)
1141 return -ENOENT;
1142
1143 info = irq_get_handler_data(irq);
1144
1145 if (!info)
1146 return -ENOENT;
1147
1148 WARN_ON(info->refcnt != -1);
1149
1150 info->refcnt = 1;
1151
1152 return 0;
1153 }
1154 EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1155
1156 int evtchn_get(unsigned int evtchn)
1157 {
1158 int irq;
1159 struct irq_info *info;
1160 int err = -ENOENT;
1161
1162 if (evtchn >= xen_evtchn_max_channels())
1163 return -EINVAL;
1164
1165 mutex_lock(&irq_mapping_update_lock);
1166
1167 irq = get_evtchn_to_irq(evtchn);
1168 if (irq == -1)
1169 goto done;
1170
1171 info = irq_get_handler_data(irq);
1172
1173 if (!info)
1174 goto done;
1175
1176 err = -EINVAL;
1177 if (info->refcnt <= 0)
1178 goto done;
1179
1180 info->refcnt++;
1181 err = 0;
1182 done:
1183 mutex_unlock(&irq_mapping_update_lock);
1184
1185 return err;
1186 }
1187 EXPORT_SYMBOL_GPL(evtchn_get);
1188
1189 void evtchn_put(unsigned int evtchn)
1190 {
1191 int irq = get_evtchn_to_irq(evtchn);
1192 if (WARN_ON(irq == -1))
1193 return;
1194 unbind_from_irq(irq);
1195 }
1196 EXPORT_SYMBOL_GPL(evtchn_put);
1197
1198 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1199 {
1200 int irq;
1201
1202 #ifdef CONFIG_X86
1203 if (unlikely(vector == XEN_NMI_VECTOR)) {
1204 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu),
1205 NULL);
1206 if (rc < 0)
1207 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1208 return;
1209 }
1210 #endif
1211 irq = per_cpu(ipi_to_irq, cpu)[vector];
1212 BUG_ON(irq < 0);
1213 notify_remote_via_irq(irq);
1214 }
1215
1216 static DEFINE_PER_CPU(unsigned, xed_nesting_count);
1217
1218 static void __xen_evtchn_do_upcall(void)
1219 {
1220 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1221 int cpu = get_cpu();
1222 unsigned count;
1223
1224 do {
1225 vcpu_info->evtchn_upcall_pending = 0;
1226
1227 if (__this_cpu_inc_return(xed_nesting_count) - 1)
1228 goto out;
1229
1230 xen_evtchn_handle_events(cpu);
1231
1232 BUG_ON(!irqs_disabled());
1233
1234 count = __this_cpu_read(xed_nesting_count);
1235 __this_cpu_write(xed_nesting_count, 0);
1236 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
1237
1238 out:
1239
1240 put_cpu();
1241 }
1242
1243 void xen_evtchn_do_upcall(struct pt_regs *regs)
1244 {
1245 struct pt_regs *old_regs = set_irq_regs(regs);
1246
1247 irq_enter();
1248 #ifdef CONFIG_X86
1249 inc_irq_stat(irq_hv_callback_count);
1250 #endif
1251
1252 __xen_evtchn_do_upcall();
1253
1254 irq_exit();
1255 set_irq_regs(old_regs);
1256 }
1257
1258 void xen_hvm_evtchn_do_upcall(void)
1259 {
1260 __xen_evtchn_do_upcall();
1261 }
1262 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1263
1264
1265 void rebind_evtchn_irq(int evtchn, int irq)
1266 {
1267 struct irq_info *info = info_for_irq(irq);
1268
1269 if (WARN_ON(!info))
1270 return;
1271
1272
1273
1274 disable_irq(irq);
1275
1276 mutex_lock(&irq_mapping_update_lock);
1277
1278
1279 BUG_ON(get_evtchn_to_irq(evtchn) != -1);
1280
1281
1282 BUG_ON(info->type == IRQT_UNBOUND);
1283
1284 (void)xen_irq_info_evtchn_setup(irq, evtchn);
1285
1286 mutex_unlock(&irq_mapping_update_lock);
1287
1288 bind_evtchn_to_cpu(evtchn, info->cpu);
1289
1290 irq_set_affinity(irq, cpumask_of(info->cpu));
1291
1292
1293 enable_irq(irq);
1294 }
1295
1296
1297 static int xen_rebind_evtchn_to_cpu(int evtchn, unsigned int tcpu)
1298 {
1299 struct evtchn_bind_vcpu bind_vcpu;
1300 int masked;
1301
1302 if (!VALID_EVTCHN(evtchn))
1303 return -1;
1304
1305 if (!xen_support_evtchn_rebind())
1306 return -1;
1307
1308
1309 bind_vcpu.port = evtchn;
1310 bind_vcpu.vcpu = xen_vcpu_nr(tcpu);
1311
1312
1313
1314
1315
1316 masked = test_and_set_mask(evtchn);
1317
1318
1319
1320
1321
1322
1323 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1324 bind_evtchn_to_cpu(evtchn, tcpu);
1325
1326 if (!masked)
1327 unmask_evtchn(evtchn);
1328
1329 return 0;
1330 }
1331
1332 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1333 bool force)
1334 {
1335 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask);
1336 int ret = xen_rebind_evtchn_to_cpu(evtchn_from_irq(data->irq), tcpu);
1337
1338 if (!ret)
1339 irq_data_update_effective_affinity(data, cpumask_of(tcpu));
1340
1341 return ret;
1342 }
1343
1344
1345 int xen_set_affinity_evtchn(struct irq_desc *desc, unsigned int tcpu)
1346 {
1347 struct irq_data *d = irq_desc_get_irq_data(desc);
1348
1349 return set_affinity_irq(d, cpumask_of(tcpu), false);
1350 }
1351 EXPORT_SYMBOL_GPL(xen_set_affinity_evtchn);
1352
1353 static void enable_dynirq(struct irq_data *data)
1354 {
1355 int evtchn = evtchn_from_irq(data->irq);
1356
1357 if (VALID_EVTCHN(evtchn))
1358 unmask_evtchn(evtchn);
1359 }
1360
1361 static void disable_dynirq(struct irq_data *data)
1362 {
1363 int evtchn = evtchn_from_irq(data->irq);
1364
1365 if (VALID_EVTCHN(evtchn))
1366 mask_evtchn(evtchn);
1367 }
1368
1369 static void ack_dynirq(struct irq_data *data)
1370 {
1371 int evtchn = evtchn_from_irq(data->irq);
1372
1373 if (!VALID_EVTCHN(evtchn))
1374 return;
1375
1376 if (unlikely(irqd_is_setaffinity_pending(data)) &&
1377 likely(!irqd_irq_disabled(data))) {
1378 int masked = test_and_set_mask(evtchn);
1379
1380 clear_evtchn(evtchn);
1381
1382 irq_move_masked_irq(data);
1383
1384 if (!masked)
1385 unmask_evtchn(evtchn);
1386 } else
1387 clear_evtchn(evtchn);
1388 }
1389
1390 static void mask_ack_dynirq(struct irq_data *data)
1391 {
1392 disable_dynirq(data);
1393 ack_dynirq(data);
1394 }
1395
1396 static int retrigger_dynirq(struct irq_data *data)
1397 {
1398 unsigned int evtchn = evtchn_from_irq(data->irq);
1399 int masked;
1400
1401 if (!VALID_EVTCHN(evtchn))
1402 return 0;
1403
1404 masked = test_and_set_mask(evtchn);
1405 set_evtchn(evtchn);
1406 if (!masked)
1407 unmask_evtchn(evtchn);
1408
1409 return 1;
1410 }
1411
1412 static void restore_pirqs(void)
1413 {
1414 int pirq, rc, irq, gsi;
1415 struct physdev_map_pirq map_irq;
1416 struct irq_info *info;
1417
1418 list_for_each_entry(info, &xen_irq_list_head, list) {
1419 if (info->type != IRQT_PIRQ)
1420 continue;
1421
1422 pirq = info->u.pirq.pirq;
1423 gsi = info->u.pirq.gsi;
1424 irq = info->irq;
1425
1426
1427
1428 if (!gsi)
1429 continue;
1430
1431 map_irq.domid = DOMID_SELF;
1432 map_irq.type = MAP_PIRQ_TYPE_GSI;
1433 map_irq.index = gsi;
1434 map_irq.pirq = pirq;
1435
1436 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1437 if (rc) {
1438 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1439 gsi, irq, pirq, rc);
1440 xen_free_irq(irq);
1441 continue;
1442 }
1443
1444 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1445
1446 __startup_pirq(irq);
1447 }
1448 }
1449
1450 static void restore_cpu_virqs(unsigned int cpu)
1451 {
1452 struct evtchn_bind_virq bind_virq;
1453 int virq, irq, evtchn;
1454
1455 for (virq = 0; virq < NR_VIRQS; virq++) {
1456 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1457 continue;
1458
1459 BUG_ON(virq_from_irq(irq) != virq);
1460
1461
1462 bind_virq.virq = virq;
1463 bind_virq.vcpu = xen_vcpu_nr(cpu);
1464 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1465 &bind_virq) != 0)
1466 BUG();
1467 evtchn = bind_virq.port;
1468
1469
1470 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1471 bind_evtchn_to_cpu(evtchn, cpu);
1472 }
1473 }
1474
1475 static void restore_cpu_ipis(unsigned int cpu)
1476 {
1477 struct evtchn_bind_ipi bind_ipi;
1478 int ipi, irq, evtchn;
1479
1480 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1481 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1482 continue;
1483
1484 BUG_ON(ipi_from_irq(irq) != ipi);
1485
1486
1487 bind_ipi.vcpu = xen_vcpu_nr(cpu);
1488 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1489 &bind_ipi) != 0)
1490 BUG();
1491 evtchn = bind_ipi.port;
1492
1493
1494 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
1495 bind_evtchn_to_cpu(evtchn, cpu);
1496 }
1497 }
1498
1499
1500 void xen_clear_irq_pending(int irq)
1501 {
1502 int evtchn = evtchn_from_irq(irq);
1503
1504 if (VALID_EVTCHN(evtchn))
1505 clear_evtchn(evtchn);
1506 }
1507 EXPORT_SYMBOL(xen_clear_irq_pending);
1508 void xen_set_irq_pending(int irq)
1509 {
1510 int evtchn = evtchn_from_irq(irq);
1511
1512 if (VALID_EVTCHN(evtchn))
1513 set_evtchn(evtchn);
1514 }
1515
1516 bool xen_test_irq_pending(int irq)
1517 {
1518 int evtchn = evtchn_from_irq(irq);
1519 bool ret = false;
1520
1521 if (VALID_EVTCHN(evtchn))
1522 ret = test_evtchn(evtchn);
1523
1524 return ret;
1525 }
1526
1527
1528
1529 void xen_poll_irq_timeout(int irq, u64 timeout)
1530 {
1531 evtchn_port_t evtchn = evtchn_from_irq(irq);
1532
1533 if (VALID_EVTCHN(evtchn)) {
1534 struct sched_poll poll;
1535
1536 poll.nr_ports = 1;
1537 poll.timeout = timeout;
1538 set_xen_guest_handle(poll.ports, &evtchn);
1539
1540 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1541 BUG();
1542 }
1543 }
1544 EXPORT_SYMBOL(xen_poll_irq_timeout);
1545
1546
1547 void xen_poll_irq(int irq)
1548 {
1549 xen_poll_irq_timeout(irq, 0 );
1550 }
1551
1552
1553 int xen_test_irq_shared(int irq)
1554 {
1555 struct irq_info *info = info_for_irq(irq);
1556 struct physdev_irq_status_query irq_status;
1557
1558 if (WARN_ON(!info))
1559 return -ENOENT;
1560
1561 irq_status.irq = info->u.pirq.pirq;
1562
1563 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1564 return 0;
1565 return !(irq_status.flags & XENIRQSTAT_shared);
1566 }
1567 EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1568
1569 void xen_irq_resume(void)
1570 {
1571 unsigned int cpu;
1572 struct irq_info *info;
1573
1574
1575 xen_evtchn_resume();
1576
1577
1578 list_for_each_entry(info, &xen_irq_list_head, list)
1579 info->evtchn = 0;
1580
1581 clear_evtchn_to_irq_all();
1582
1583 for_each_possible_cpu(cpu) {
1584 restore_cpu_virqs(cpu);
1585 restore_cpu_ipis(cpu);
1586 }
1587
1588 restore_pirqs();
1589 }
1590
1591 static struct irq_chip xen_dynamic_chip __read_mostly = {
1592 .name = "xen-dyn",
1593
1594 .irq_disable = disable_dynirq,
1595 .irq_mask = disable_dynirq,
1596 .irq_unmask = enable_dynirq,
1597
1598 .irq_ack = ack_dynirq,
1599 .irq_mask_ack = mask_ack_dynirq,
1600
1601 .irq_set_affinity = set_affinity_irq,
1602 .irq_retrigger = retrigger_dynirq,
1603 };
1604
1605 static struct irq_chip xen_pirq_chip __read_mostly = {
1606 .name = "xen-pirq",
1607
1608 .irq_startup = startup_pirq,
1609 .irq_shutdown = shutdown_pirq,
1610 .irq_enable = enable_pirq,
1611 .irq_disable = disable_pirq,
1612
1613 .irq_mask = disable_dynirq,
1614 .irq_unmask = enable_dynirq,
1615
1616 .irq_ack = eoi_pirq,
1617 .irq_eoi = eoi_pirq,
1618 .irq_mask_ack = mask_ack_pirq,
1619
1620 .irq_set_affinity = set_affinity_irq,
1621
1622 .irq_retrigger = retrigger_dynirq,
1623 };
1624
1625 static struct irq_chip xen_percpu_chip __read_mostly = {
1626 .name = "xen-percpu",
1627
1628 .irq_disable = disable_dynirq,
1629 .irq_mask = disable_dynirq,
1630 .irq_unmask = enable_dynirq,
1631
1632 .irq_ack = ack_dynirq,
1633 };
1634
1635 int xen_set_callback_via(uint64_t via)
1636 {
1637 struct xen_hvm_param a;
1638 a.domid = DOMID_SELF;
1639 a.index = HVM_PARAM_CALLBACK_IRQ;
1640 a.value = via;
1641 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1642 }
1643 EXPORT_SYMBOL_GPL(xen_set_callback_via);
1644
1645 #ifdef CONFIG_XEN_PVHVM
1646
1647
1648
1649 void xen_callback_vector(void)
1650 {
1651 int rc;
1652 uint64_t callback_via;
1653
1654 if (xen_have_vector_callback) {
1655 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
1656 rc = xen_set_callback_via(callback_via);
1657 if (rc) {
1658 pr_err("Request for Xen HVM callback vector failed\n");
1659 xen_have_vector_callback = 0;
1660 return;
1661 }
1662 pr_info_once("Xen HVM callback vector for event delivery is enabled\n");
1663 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
1664 xen_hvm_callback_vector);
1665 }
1666 }
1667 #else
1668 void xen_callback_vector(void) {}
1669 #endif
1670
1671 #undef MODULE_PARAM_PREFIX
1672 #define MODULE_PARAM_PREFIX "xen."
1673
1674 static bool fifo_events = true;
1675 module_param(fifo_events, bool, 0);
1676
1677 void __init xen_init_IRQ(void)
1678 {
1679 int ret = -EINVAL;
1680 unsigned int evtchn;
1681
1682 if (fifo_events)
1683 ret = xen_evtchn_fifo_init();
1684 if (ret < 0)
1685 xen_evtchn_2l_init();
1686
1687 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
1688 sizeof(*evtchn_to_irq), GFP_KERNEL);
1689 BUG_ON(!evtchn_to_irq);
1690
1691
1692 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
1693 mask_evtchn(evtchn);
1694
1695 pirq_needs_eoi = pirq_needs_eoi_flag;
1696
1697 #ifdef CONFIG_X86
1698 if (xen_pv_domain()) {
1699 if (xen_initial_domain())
1700 pci_xen_initial_domain();
1701 }
1702 if (xen_feature(XENFEAT_hvm_callback_vector))
1703 xen_callback_vector();
1704
1705 if (xen_hvm_domain()) {
1706 native_init_IRQ();
1707
1708
1709 pci_xen_hvm_init();
1710 } else {
1711 int rc;
1712 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1713
1714 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
1715 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
1716 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
1717 if (rc != 0) {
1718 free_page((unsigned long) pirq_eoi_map);
1719 pirq_eoi_map = NULL;
1720 } else
1721 pirq_needs_eoi = pirq_check_eoi_map;
1722 }
1723 #endif
1724 }