1/* 2 * Xen event channels 3 * 4 * Xen models interrupts with abstract event channels. Because each 5 * domain gets 1024 event channels, but NR_IRQ is not that large, we 6 * must dynamically map irqs<->event channels. The event channels 7 * interface with the rest of the kernel by defining a xen interrupt 8 * chip. When an event is received, it is mapped to an irq and sent 9 * through the normal interrupt processing path. 10 * 11 * There are four kinds of events which can be mapped to an event 12 * channel: 13 * 14 * 1. Inter-domain notifications. This includes all the virtual 15 * device events, since they're driven by front-ends in another domain 16 * (typically dom0). 17 * 2. VIRQs, typically used for timers. These are per-cpu events. 18 * 3. IPIs. 19 * 4. PIRQs - Hardware interrupts. 20 * 21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 22 */ 23 24#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 25 26#include <linux/linkage.h> 27#include <linux/interrupt.h> 28#include <linux/irq.h> 29#include <linux/module.h> 30#include <linux/string.h> 31#include <linux/bootmem.h> 32#include <linux/slab.h> 33#include <linux/irqnr.h> 34#include <linux/pci.h> 35 36#ifdef CONFIG_X86 37#include <asm/desc.h> 38#include <asm/ptrace.h> 39#include <asm/irq.h> 40#include <asm/idle.h> 41#include <asm/io_apic.h> 42#include <asm/xen/page.h> 43#include <asm/xen/pci.h> 44#endif 45#include <asm/sync_bitops.h> 46#include <asm/xen/hypercall.h> 47#include <asm/xen/hypervisor.h> 48 49#include <xen/xen.h> 50#include <xen/hvm.h> 51#include <xen/xen-ops.h> 52#include <xen/events.h> 53#include <xen/interface/xen.h> 54#include <xen/interface/event_channel.h> 55#include <xen/interface/hvm/hvm_op.h> 56#include <xen/interface/hvm/params.h> 57#include <xen/interface/physdev.h> 58#include <xen/interface/sched.h> 59#include <xen/interface/vcpu.h> 60#include <asm/hw_irq.h> 61 62#include "events_internal.h" 63 64const struct evtchn_ops *evtchn_ops; 65 66/* 67 * This lock protects updates to the following mapping and reference-count 68 * arrays. The lock does not need to be acquired to read the mapping tables. 69 */ 70static DEFINE_MUTEX(irq_mapping_update_lock); 71 72static LIST_HEAD(xen_irq_list_head); 73 74/* IRQ <-> VIRQ mapping. */ 75static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1}; 76 77/* IRQ <-> IPI mapping */ 78static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1}; 79 80int **evtchn_to_irq; 81#ifdef CONFIG_X86 82static unsigned long *pirq_eoi_map; 83#endif 84static bool (*pirq_needs_eoi)(unsigned irq); 85 86#define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq))) 87#define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq))) 88#define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq)) 89 90/* Xen will never allocate port zero for any purpose. */ 91#define VALID_EVTCHN(chn) ((chn) != 0) 92 93static struct irq_chip xen_dynamic_chip; 94static struct irq_chip xen_percpu_chip; 95static struct irq_chip xen_pirq_chip; 96static void enable_dynirq(struct irq_data *data); 97static void disable_dynirq(struct irq_data *data); 98 99static void clear_evtchn_to_irq_row(unsigned row) 100{ 101 unsigned col; 102 103 for (col = 0; col < EVTCHN_PER_ROW; col++) 104 evtchn_to_irq[row][col] = -1; 105} 106 107static void clear_evtchn_to_irq_all(void) 108{ 109 unsigned row; 110 111 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) { 112 if (evtchn_to_irq[row] == NULL) 113 continue; 114 clear_evtchn_to_irq_row(row); 115 } 116} 117 118static int set_evtchn_to_irq(unsigned evtchn, unsigned irq) 119{ 120 unsigned row; 121 unsigned col; 122 123 if (evtchn >= xen_evtchn_max_channels()) 124 return -EINVAL; 125 126 row = EVTCHN_ROW(evtchn); 127 col = EVTCHN_COL(evtchn); 128 129 if (evtchn_to_irq[row] == NULL) { 130 /* Unallocated irq entries return -1 anyway */ 131 if (irq == -1) 132 return 0; 133 134 evtchn_to_irq[row] = (int *)get_zeroed_page(GFP_KERNEL); 135 if (evtchn_to_irq[row] == NULL) 136 return -ENOMEM; 137 138 clear_evtchn_to_irq_row(row); 139 } 140 141 evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)] = irq; 142 return 0; 143} 144 145int get_evtchn_to_irq(unsigned evtchn) 146{ 147 if (evtchn >= xen_evtchn_max_channels()) 148 return -1; 149 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL) 150 return -1; 151 return evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]; 152} 153 154/* Get info for IRQ */ 155struct irq_info *info_for_irq(unsigned irq) 156{ 157 return irq_get_handler_data(irq); 158} 159 160/* Constructors for packed IRQ information. */ 161static int xen_irq_info_common_setup(struct irq_info *info, 162 unsigned irq, 163 enum xen_irq_type type, 164 unsigned evtchn, 165 unsigned short cpu) 166{ 167 int ret; 168 169 BUG_ON(info->type != IRQT_UNBOUND && info->type != type); 170 171 info->type = type; 172 info->irq = irq; 173 info->evtchn = evtchn; 174 info->cpu = cpu; 175 176 ret = set_evtchn_to_irq(evtchn, irq); 177 if (ret < 0) 178 return ret; 179 180 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN); 181 182 return xen_evtchn_port_setup(info); 183} 184 185static int xen_irq_info_evtchn_setup(unsigned irq, 186 unsigned evtchn) 187{ 188 struct irq_info *info = info_for_irq(irq); 189 190 return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0); 191} 192 193static int xen_irq_info_ipi_setup(unsigned cpu, 194 unsigned irq, 195 unsigned evtchn, 196 enum ipi_vector ipi) 197{ 198 struct irq_info *info = info_for_irq(irq); 199 200 info->u.ipi = ipi; 201 202 per_cpu(ipi_to_irq, cpu)[ipi] = irq; 203 204 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0); 205} 206 207static int xen_irq_info_virq_setup(unsigned cpu, 208 unsigned irq, 209 unsigned evtchn, 210 unsigned virq) 211{ 212 struct irq_info *info = info_for_irq(irq); 213 214 info->u.virq = virq; 215 216 per_cpu(virq_to_irq, cpu)[virq] = irq; 217 218 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0); 219} 220 221static int xen_irq_info_pirq_setup(unsigned irq, 222 unsigned evtchn, 223 unsigned pirq, 224 unsigned gsi, 225 uint16_t domid, 226 unsigned char flags) 227{ 228 struct irq_info *info = info_for_irq(irq); 229 230 info->u.pirq.pirq = pirq; 231 info->u.pirq.gsi = gsi; 232 info->u.pirq.domid = domid; 233 info->u.pirq.flags = flags; 234 235 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0); 236} 237 238static void xen_irq_info_cleanup(struct irq_info *info) 239{ 240 set_evtchn_to_irq(info->evtchn, -1); 241 info->evtchn = 0; 242} 243 244/* 245 * Accessors for packed IRQ information. 246 */ 247unsigned int evtchn_from_irq(unsigned irq) 248{ 249 if (unlikely(WARN(irq >= nr_irqs, "Invalid irq %d!\n", irq))) 250 return 0; 251 252 return info_for_irq(irq)->evtchn; 253} 254 255unsigned irq_from_evtchn(unsigned int evtchn) 256{ 257 return get_evtchn_to_irq(evtchn); 258} 259EXPORT_SYMBOL_GPL(irq_from_evtchn); 260 261int irq_from_virq(unsigned int cpu, unsigned int virq) 262{ 263 return per_cpu(virq_to_irq, cpu)[virq]; 264} 265 266static enum ipi_vector ipi_from_irq(unsigned irq) 267{ 268 struct irq_info *info = info_for_irq(irq); 269 270 BUG_ON(info == NULL); 271 BUG_ON(info->type != IRQT_IPI); 272 273 return info->u.ipi; 274} 275 276static unsigned virq_from_irq(unsigned irq) 277{ 278 struct irq_info *info = info_for_irq(irq); 279 280 BUG_ON(info == NULL); 281 BUG_ON(info->type != IRQT_VIRQ); 282 283 return info->u.virq; 284} 285 286static unsigned pirq_from_irq(unsigned irq) 287{ 288 struct irq_info *info = info_for_irq(irq); 289 290 BUG_ON(info == NULL); 291 BUG_ON(info->type != IRQT_PIRQ); 292 293 return info->u.pirq.pirq; 294} 295 296static enum xen_irq_type type_from_irq(unsigned irq) 297{ 298 return info_for_irq(irq)->type; 299} 300 301unsigned cpu_from_irq(unsigned irq) 302{ 303 return info_for_irq(irq)->cpu; 304} 305 306unsigned int cpu_from_evtchn(unsigned int evtchn) 307{ 308 int irq = get_evtchn_to_irq(evtchn); 309 unsigned ret = 0; 310 311 if (irq != -1) 312 ret = cpu_from_irq(irq); 313 314 return ret; 315} 316 317#ifdef CONFIG_X86 318static bool pirq_check_eoi_map(unsigned irq) 319{ 320 return test_bit(pirq_from_irq(irq), pirq_eoi_map); 321} 322#endif 323 324static bool pirq_needs_eoi_flag(unsigned irq) 325{ 326 struct irq_info *info = info_for_irq(irq); 327 BUG_ON(info->type != IRQT_PIRQ); 328 329 return info->u.pirq.flags & PIRQ_NEEDS_EOI; 330} 331 332static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu) 333{ 334 int irq = get_evtchn_to_irq(chn); 335 struct irq_info *info = info_for_irq(irq); 336 337 BUG_ON(irq == -1); 338#ifdef CONFIG_SMP 339 cpumask_copy(irq_get_irq_data(irq)->affinity, cpumask_of(cpu)); 340#endif 341 xen_evtchn_port_bind_to_cpu(info, cpu); 342 343 info->cpu = cpu; 344} 345 346static void xen_evtchn_mask_all(void) 347{ 348 unsigned int evtchn; 349 350 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++) 351 mask_evtchn(evtchn); 352} 353 354/** 355 * notify_remote_via_irq - send event to remote end of event channel via irq 356 * @irq: irq of event channel to send event to 357 * 358 * Unlike notify_remote_via_evtchn(), this is safe to use across 359 * save/restore. Notifications on a broken connection are silently 360 * dropped. 361 */ 362void notify_remote_via_irq(int irq) 363{ 364 int evtchn = evtchn_from_irq(irq); 365 366 if (VALID_EVTCHN(evtchn)) 367 notify_remote_via_evtchn(evtchn); 368} 369EXPORT_SYMBOL_GPL(notify_remote_via_irq); 370 371static void xen_irq_init(unsigned irq) 372{ 373 struct irq_info *info; 374#ifdef CONFIG_SMP 375 /* By default all event channels notify CPU#0. */ 376 cpumask_copy(irq_get_irq_data(irq)->affinity, cpumask_of(0)); 377#endif 378 379 info = kzalloc(sizeof(*info), GFP_KERNEL); 380 if (info == NULL) 381 panic("Unable to allocate metadata for IRQ%d\n", irq); 382 383 info->type = IRQT_UNBOUND; 384 info->refcnt = -1; 385 386 irq_set_handler_data(irq, info); 387 388 list_add_tail(&info->list, &xen_irq_list_head); 389} 390 391static int __must_check xen_allocate_irqs_dynamic(int nvec) 392{ 393 int i, irq = irq_alloc_descs(-1, 0, nvec, -1); 394 395 if (irq >= 0) { 396 for (i = 0; i < nvec; i++) 397 xen_irq_init(irq + i); 398 } 399 400 return irq; 401} 402 403static inline int __must_check xen_allocate_irq_dynamic(void) 404{ 405 406 return xen_allocate_irqs_dynamic(1); 407} 408 409static int __must_check xen_allocate_irq_gsi(unsigned gsi) 410{ 411 int irq; 412 413 /* 414 * A PV guest has no concept of a GSI (since it has no ACPI 415 * nor access to/knowledge of the physical APICs). Therefore 416 * all IRQs are dynamically allocated from the entire IRQ 417 * space. 418 */ 419 if (xen_pv_domain() && !xen_initial_domain()) 420 return xen_allocate_irq_dynamic(); 421 422 /* Legacy IRQ descriptors are already allocated by the arch. */ 423 if (gsi < NR_IRQS_LEGACY) 424 irq = gsi; 425 else 426 irq = irq_alloc_desc_at(gsi, -1); 427 428 xen_irq_init(irq); 429 430 return irq; 431} 432 433static void xen_free_irq(unsigned irq) 434{ 435 struct irq_info *info = irq_get_handler_data(irq); 436 437 if (WARN_ON(!info)) 438 return; 439 440 list_del(&info->list); 441 442 irq_set_handler_data(irq, NULL); 443 444 WARN_ON(info->refcnt > 0); 445 446 kfree(info); 447 448 /* Legacy IRQ descriptors are managed by the arch. */ 449 if (irq < NR_IRQS_LEGACY) 450 return; 451 452 irq_free_desc(irq); 453} 454 455static void xen_evtchn_close(unsigned int port) 456{ 457 struct evtchn_close close; 458 459 close.port = port; 460 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0) 461 BUG(); 462} 463 464static void pirq_query_unmask(int irq) 465{ 466 struct physdev_irq_status_query irq_status; 467 struct irq_info *info = info_for_irq(irq); 468 469 BUG_ON(info->type != IRQT_PIRQ); 470 471 irq_status.irq = pirq_from_irq(irq); 472 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 473 irq_status.flags = 0; 474 475 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI; 476 if (irq_status.flags & XENIRQSTAT_needs_eoi) 477 info->u.pirq.flags |= PIRQ_NEEDS_EOI; 478} 479 480static void eoi_pirq(struct irq_data *data) 481{ 482 int evtchn = evtchn_from_irq(data->irq); 483 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) }; 484 int rc = 0; 485 486 if (!VALID_EVTCHN(evtchn)) 487 return; 488 489 if (unlikely(irqd_is_setaffinity_pending(data)) && 490 likely(!irqd_irq_disabled(data))) { 491 int masked = test_and_set_mask(evtchn); 492 493 clear_evtchn(evtchn); 494 495 irq_move_masked_irq(data); 496 497 if (!masked) 498 unmask_evtchn(evtchn); 499 } else 500 clear_evtchn(evtchn); 501 502 if (pirq_needs_eoi(data->irq)) { 503 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); 504 WARN_ON(rc); 505 } 506} 507 508static void mask_ack_pirq(struct irq_data *data) 509{ 510 disable_dynirq(data); 511 eoi_pirq(data); 512} 513 514static unsigned int __startup_pirq(unsigned int irq) 515{ 516 struct evtchn_bind_pirq bind_pirq; 517 struct irq_info *info = info_for_irq(irq); 518 int evtchn = evtchn_from_irq(irq); 519 int rc; 520 521 BUG_ON(info->type != IRQT_PIRQ); 522 523 if (VALID_EVTCHN(evtchn)) 524 goto out; 525 526 bind_pirq.pirq = pirq_from_irq(irq); 527 /* NB. We are happy to share unless we are probing. */ 528 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ? 529 BIND_PIRQ__WILL_SHARE : 0; 530 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq); 531 if (rc != 0) { 532 pr_warn("Failed to obtain physical IRQ %d\n", irq); 533 return 0; 534 } 535 evtchn = bind_pirq.port; 536 537 pirq_query_unmask(irq); 538 539 rc = set_evtchn_to_irq(evtchn, irq); 540 if (rc) 541 goto err; 542 543 info->evtchn = evtchn; 544 bind_evtchn_to_cpu(evtchn, 0); 545 546 rc = xen_evtchn_port_setup(info); 547 if (rc) 548 goto err; 549 550out: 551 unmask_evtchn(evtchn); 552 eoi_pirq(irq_get_irq_data(irq)); 553 554 return 0; 555 556err: 557 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc); 558 xen_evtchn_close(evtchn); 559 return 0; 560} 561 562static unsigned int startup_pirq(struct irq_data *data) 563{ 564 return __startup_pirq(data->irq); 565} 566 567static void shutdown_pirq(struct irq_data *data) 568{ 569 unsigned int irq = data->irq; 570 struct irq_info *info = info_for_irq(irq); 571 unsigned evtchn = evtchn_from_irq(irq); 572 573 BUG_ON(info->type != IRQT_PIRQ); 574 575 if (!VALID_EVTCHN(evtchn)) 576 return; 577 578 mask_evtchn(evtchn); 579 xen_evtchn_close(evtchn); 580 xen_irq_info_cleanup(info); 581} 582 583static void enable_pirq(struct irq_data *data) 584{ 585 startup_pirq(data); 586} 587 588static void disable_pirq(struct irq_data *data) 589{ 590 disable_dynirq(data); 591} 592 593int xen_irq_from_gsi(unsigned gsi) 594{ 595 struct irq_info *info; 596 597 list_for_each_entry(info, &xen_irq_list_head, list) { 598 if (info->type != IRQT_PIRQ) 599 continue; 600 601 if (info->u.pirq.gsi == gsi) 602 return info->irq; 603 } 604 605 return -1; 606} 607EXPORT_SYMBOL_GPL(xen_irq_from_gsi); 608 609static void __unbind_from_irq(unsigned int irq) 610{ 611 int evtchn = evtchn_from_irq(irq); 612 struct irq_info *info = irq_get_handler_data(irq); 613 614 if (info->refcnt > 0) { 615 info->refcnt--; 616 if (info->refcnt != 0) 617 return; 618 } 619 620 if (VALID_EVTCHN(evtchn)) { 621 unsigned int cpu = cpu_from_irq(irq); 622 623 xen_evtchn_close(evtchn); 624 625 switch (type_from_irq(irq)) { 626 case IRQT_VIRQ: 627 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1; 628 break; 629 case IRQT_IPI: 630 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1; 631 break; 632 default: 633 break; 634 } 635 636 xen_irq_info_cleanup(info); 637 } 638 639 BUG_ON(info_for_irq(irq)->type == IRQT_UNBOUND); 640 641 xen_free_irq(irq); 642} 643 644/* 645 * Do not make any assumptions regarding the relationship between the 646 * IRQ number returned here and the Xen pirq argument. 647 * 648 * Note: We don't assign an event channel until the irq actually started 649 * up. Return an existing irq if we've already got one for the gsi. 650 * 651 * Shareable implies level triggered, not shareable implies edge 652 * triggered here. 653 */ 654int xen_bind_pirq_gsi_to_irq(unsigned gsi, 655 unsigned pirq, int shareable, char *name) 656{ 657 int irq = -1; 658 struct physdev_irq irq_op; 659 int ret; 660 661 mutex_lock(&irq_mapping_update_lock); 662 663 irq = xen_irq_from_gsi(gsi); 664 if (irq != -1) { 665 pr_info("%s: returning irq %d for gsi %u\n", 666 __func__, irq, gsi); 667 goto out; 668 } 669 670 irq = xen_allocate_irq_gsi(gsi); 671 if (irq < 0) 672 goto out; 673 674 irq_op.irq = irq; 675 irq_op.vector = 0; 676 677 /* Only the privileged domain can do this. For non-priv, the pcifront 678 * driver provides a PCI bus that does the call to do exactly 679 * this in the priv domain. */ 680 if (xen_initial_domain() && 681 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) { 682 xen_free_irq(irq); 683 irq = -ENOSPC; 684 goto out; 685 } 686 687 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF, 688 shareable ? PIRQ_SHAREABLE : 0); 689 if (ret < 0) { 690 __unbind_from_irq(irq); 691 irq = ret; 692 goto out; 693 } 694 695 pirq_query_unmask(irq); 696 /* We try to use the handler with the appropriate semantic for the 697 * type of interrupt: if the interrupt is an edge triggered 698 * interrupt we use handle_edge_irq. 699 * 700 * On the other hand if the interrupt is level triggered we use 701 * handle_fasteoi_irq like the native code does for this kind of 702 * interrupts. 703 * 704 * Depending on the Xen version, pirq_needs_eoi might return true 705 * not only for level triggered interrupts but for edge triggered 706 * interrupts too. In any case Xen always honors the eoi mechanism, 707 * not injecting any more pirqs of the same kind if the first one 708 * hasn't received an eoi yet. Therefore using the fasteoi handler 709 * is the right choice either way. 710 */ 711 if (shareable) 712 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, 713 handle_fasteoi_irq, name); 714 else 715 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, 716 handle_edge_irq, name); 717 718out: 719 mutex_unlock(&irq_mapping_update_lock); 720 721 return irq; 722} 723 724#ifdef CONFIG_PCI_MSI 725int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc) 726{ 727 int rc; 728 struct physdev_get_free_pirq op_get_free_pirq; 729 730 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI; 731 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq); 732 733 WARN_ONCE(rc == -ENOSYS, 734 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n"); 735 736 return rc ? -1 : op_get_free_pirq.pirq; 737} 738 739int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc, 740 int pirq, int nvec, const char *name, domid_t domid) 741{ 742 int i, irq, ret; 743 744 mutex_lock(&irq_mapping_update_lock); 745 746 irq = xen_allocate_irqs_dynamic(nvec); 747 if (irq < 0) 748 goto out; 749 750 for (i = 0; i < nvec; i++) { 751 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name); 752 753 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid, 754 i == 0 ? 0 : PIRQ_MSI_GROUP); 755 if (ret < 0) 756 goto error_irq; 757 } 758 759 ret = irq_set_msi_desc(irq, msidesc); 760 if (ret < 0) 761 goto error_irq; 762out: 763 mutex_unlock(&irq_mapping_update_lock); 764 return irq; 765error_irq: 766 for (; i >= 0; i--) 767 __unbind_from_irq(irq + i); 768 mutex_unlock(&irq_mapping_update_lock); 769 return ret; 770} 771#endif 772 773int xen_destroy_irq(int irq) 774{ 775 struct physdev_unmap_pirq unmap_irq; 776 struct irq_info *info = info_for_irq(irq); 777 int rc = -ENOENT; 778 779 mutex_lock(&irq_mapping_update_lock); 780 781 /* 782 * If trying to remove a vector in a MSI group different 783 * than the first one skip the PIRQ unmap unless this vector 784 * is the first one in the group. 785 */ 786 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) { 787 unmap_irq.pirq = info->u.pirq.pirq; 788 unmap_irq.domid = info->u.pirq.domid; 789 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq); 790 /* If another domain quits without making the pci_disable_msix 791 * call, the Xen hypervisor takes care of freeing the PIRQs 792 * (free_domain_pirqs). 793 */ 794 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF)) 795 pr_info("domain %d does not have %d anymore\n", 796 info->u.pirq.domid, info->u.pirq.pirq); 797 else if (rc) { 798 pr_warn("unmap irq failed %d\n", rc); 799 goto out; 800 } 801 } 802 803 xen_free_irq(irq); 804 805out: 806 mutex_unlock(&irq_mapping_update_lock); 807 return rc; 808} 809 810int xen_irq_from_pirq(unsigned pirq) 811{ 812 int irq; 813 814 struct irq_info *info; 815 816 mutex_lock(&irq_mapping_update_lock); 817 818 list_for_each_entry(info, &xen_irq_list_head, list) { 819 if (info->type != IRQT_PIRQ) 820 continue; 821 irq = info->irq; 822 if (info->u.pirq.pirq == pirq) 823 goto out; 824 } 825 irq = -1; 826out: 827 mutex_unlock(&irq_mapping_update_lock); 828 829 return irq; 830} 831 832 833int xen_pirq_from_irq(unsigned irq) 834{ 835 return pirq_from_irq(irq); 836} 837EXPORT_SYMBOL_GPL(xen_pirq_from_irq); 838 839int bind_evtchn_to_irq(unsigned int evtchn) 840{ 841 int irq; 842 int ret; 843 844 if (evtchn >= xen_evtchn_max_channels()) 845 return -ENOMEM; 846 847 mutex_lock(&irq_mapping_update_lock); 848 849 irq = get_evtchn_to_irq(evtchn); 850 851 if (irq == -1) { 852 irq = xen_allocate_irq_dynamic(); 853 if (irq < 0) 854 goto out; 855 856 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip, 857 handle_edge_irq, "event"); 858 859 ret = xen_irq_info_evtchn_setup(irq, evtchn); 860 if (ret < 0) { 861 __unbind_from_irq(irq); 862 irq = ret; 863 goto out; 864 } 865 /* New interdomain events are bound to VCPU 0. */ 866 bind_evtchn_to_cpu(evtchn, 0); 867 } else { 868 struct irq_info *info = info_for_irq(irq); 869 WARN_ON(info == NULL || info->type != IRQT_EVTCHN); 870 } 871 872out: 873 mutex_unlock(&irq_mapping_update_lock); 874 875 return irq; 876} 877EXPORT_SYMBOL_GPL(bind_evtchn_to_irq); 878 879static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu) 880{ 881 struct evtchn_bind_ipi bind_ipi; 882 int evtchn, irq; 883 int ret; 884 885 mutex_lock(&irq_mapping_update_lock); 886 887 irq = per_cpu(ipi_to_irq, cpu)[ipi]; 888 889 if (irq == -1) { 890 irq = xen_allocate_irq_dynamic(); 891 if (irq < 0) 892 goto out; 893 894 irq_set_chip_and_handler_name(irq, &xen_percpu_chip, 895 handle_percpu_irq, "ipi"); 896 897 bind_ipi.vcpu = cpu; 898 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 899 &bind_ipi) != 0) 900 BUG(); 901 evtchn = bind_ipi.port; 902 903 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi); 904 if (ret < 0) { 905 __unbind_from_irq(irq); 906 irq = ret; 907 goto out; 908 } 909 bind_evtchn_to_cpu(evtchn, cpu); 910 } else { 911 struct irq_info *info = info_for_irq(irq); 912 WARN_ON(info == NULL || info->type != IRQT_IPI); 913 } 914 915 out: 916 mutex_unlock(&irq_mapping_update_lock); 917 return irq; 918} 919 920int bind_interdomain_evtchn_to_irq(unsigned int remote_domain, 921 unsigned int remote_port) 922{ 923 struct evtchn_bind_interdomain bind_interdomain; 924 int err; 925 926 bind_interdomain.remote_dom = remote_domain; 927 bind_interdomain.remote_port = remote_port; 928 929 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 930 &bind_interdomain); 931 932 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port); 933} 934EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq); 935 936static int find_virq(unsigned int virq, unsigned int cpu) 937{ 938 struct evtchn_status status; 939 int port, rc = -ENOENT; 940 941 memset(&status, 0, sizeof(status)); 942 for (port = 0; port < xen_evtchn_max_channels(); port++) { 943 status.dom = DOMID_SELF; 944 status.port = port; 945 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status); 946 if (rc < 0) 947 continue; 948 if (status.status != EVTCHNSTAT_virq) 949 continue; 950 if (status.u.virq == virq && status.vcpu == cpu) { 951 rc = port; 952 break; 953 } 954 } 955 return rc; 956} 957 958/** 959 * xen_evtchn_nr_channels - number of usable event channel ports 960 * 961 * This may be less than the maximum supported by the current 962 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum 963 * supported. 964 */ 965unsigned xen_evtchn_nr_channels(void) 966{ 967 return evtchn_ops->nr_channels(); 968} 969EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels); 970 971int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu) 972{ 973 struct evtchn_bind_virq bind_virq; 974 int evtchn, irq, ret; 975 976 mutex_lock(&irq_mapping_update_lock); 977 978 irq = per_cpu(virq_to_irq, cpu)[virq]; 979 980 if (irq == -1) { 981 irq = xen_allocate_irq_dynamic(); 982 if (irq < 0) 983 goto out; 984 985 if (percpu) 986 irq_set_chip_and_handler_name(irq, &xen_percpu_chip, 987 handle_percpu_irq, "virq"); 988 else 989 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip, 990 handle_edge_irq, "virq"); 991 992 bind_virq.virq = virq; 993 bind_virq.vcpu = cpu; 994 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 995 &bind_virq); 996 if (ret == 0) 997 evtchn = bind_virq.port; 998 else { 999 if (ret == -EEXIST) 1000 ret = find_virq(virq, cpu); 1001 BUG_ON(ret < 0); 1002 evtchn = ret; 1003 } 1004 1005 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq); 1006 if (ret < 0) { 1007 __unbind_from_irq(irq); 1008 irq = ret; 1009 goto out; 1010 } 1011 1012 bind_evtchn_to_cpu(evtchn, cpu); 1013 } else { 1014 struct irq_info *info = info_for_irq(irq); 1015 WARN_ON(info == NULL || info->type != IRQT_VIRQ); 1016 } 1017 1018out: 1019 mutex_unlock(&irq_mapping_update_lock); 1020 1021 return irq; 1022} 1023 1024static void unbind_from_irq(unsigned int irq) 1025{ 1026 mutex_lock(&irq_mapping_update_lock); 1027 __unbind_from_irq(irq); 1028 mutex_unlock(&irq_mapping_update_lock); 1029} 1030 1031int bind_evtchn_to_irqhandler(unsigned int evtchn, 1032 irq_handler_t handler, 1033 unsigned long irqflags, 1034 const char *devname, void *dev_id) 1035{ 1036 int irq, retval; 1037 1038 irq = bind_evtchn_to_irq(evtchn); 1039 if (irq < 0) 1040 return irq; 1041 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1042 if (retval != 0) { 1043 unbind_from_irq(irq); 1044 return retval; 1045 } 1046 1047 return irq; 1048} 1049EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler); 1050 1051int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, 1052 unsigned int remote_port, 1053 irq_handler_t handler, 1054 unsigned long irqflags, 1055 const char *devname, 1056 void *dev_id) 1057{ 1058 int irq, retval; 1059 1060 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port); 1061 if (irq < 0) 1062 return irq; 1063 1064 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1065 if (retval != 0) { 1066 unbind_from_irq(irq); 1067 return retval; 1068 } 1069 1070 return irq; 1071} 1072EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler); 1073 1074int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, 1075 irq_handler_t handler, 1076 unsigned long irqflags, const char *devname, void *dev_id) 1077{ 1078 int irq, retval; 1079 1080 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU); 1081 if (irq < 0) 1082 return irq; 1083 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1084 if (retval != 0) { 1085 unbind_from_irq(irq); 1086 return retval; 1087 } 1088 1089 return irq; 1090} 1091EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler); 1092 1093int bind_ipi_to_irqhandler(enum ipi_vector ipi, 1094 unsigned int cpu, 1095 irq_handler_t handler, 1096 unsigned long irqflags, 1097 const char *devname, 1098 void *dev_id) 1099{ 1100 int irq, retval; 1101 1102 irq = bind_ipi_to_irq(ipi, cpu); 1103 if (irq < 0) 1104 return irq; 1105 1106 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME; 1107 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1108 if (retval != 0) { 1109 unbind_from_irq(irq); 1110 return retval; 1111 } 1112 1113 return irq; 1114} 1115 1116void unbind_from_irqhandler(unsigned int irq, void *dev_id) 1117{ 1118 struct irq_info *info = irq_get_handler_data(irq); 1119 1120 if (WARN_ON(!info)) 1121 return; 1122 free_irq(irq, dev_id); 1123 unbind_from_irq(irq); 1124} 1125EXPORT_SYMBOL_GPL(unbind_from_irqhandler); 1126 1127/** 1128 * xen_set_irq_priority() - set an event channel priority. 1129 * @irq:irq bound to an event channel. 1130 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN. 1131 */ 1132int xen_set_irq_priority(unsigned irq, unsigned priority) 1133{ 1134 struct evtchn_set_priority set_priority; 1135 1136 set_priority.port = evtchn_from_irq(irq); 1137 set_priority.priority = priority; 1138 1139 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority, 1140 &set_priority); 1141} 1142EXPORT_SYMBOL_GPL(xen_set_irq_priority); 1143 1144int evtchn_make_refcounted(unsigned int evtchn) 1145{ 1146 int irq = get_evtchn_to_irq(evtchn); 1147 struct irq_info *info; 1148 1149 if (irq == -1) 1150 return -ENOENT; 1151 1152 info = irq_get_handler_data(irq); 1153 1154 if (!info) 1155 return -ENOENT; 1156 1157 WARN_ON(info->refcnt != -1); 1158 1159 info->refcnt = 1; 1160 1161 return 0; 1162} 1163EXPORT_SYMBOL_GPL(evtchn_make_refcounted); 1164 1165int evtchn_get(unsigned int evtchn) 1166{ 1167 int irq; 1168 struct irq_info *info; 1169 int err = -ENOENT; 1170 1171 if (evtchn >= xen_evtchn_max_channels()) 1172 return -EINVAL; 1173 1174 mutex_lock(&irq_mapping_update_lock); 1175 1176 irq = get_evtchn_to_irq(evtchn); 1177 if (irq == -1) 1178 goto done; 1179 1180 info = irq_get_handler_data(irq); 1181 1182 if (!info) 1183 goto done; 1184 1185 err = -EINVAL; 1186 if (info->refcnt <= 0) 1187 goto done; 1188 1189 info->refcnt++; 1190 err = 0; 1191 done: 1192 mutex_unlock(&irq_mapping_update_lock); 1193 1194 return err; 1195} 1196EXPORT_SYMBOL_GPL(evtchn_get); 1197 1198void evtchn_put(unsigned int evtchn) 1199{ 1200 int irq = get_evtchn_to_irq(evtchn); 1201 if (WARN_ON(irq == -1)) 1202 return; 1203 unbind_from_irq(irq); 1204} 1205EXPORT_SYMBOL_GPL(evtchn_put); 1206 1207void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector) 1208{ 1209 int irq; 1210 1211#ifdef CONFIG_X86 1212 if (unlikely(vector == XEN_NMI_VECTOR)) { 1213 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, cpu, NULL); 1214 if (rc < 0) 1215 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc); 1216 return; 1217 } 1218#endif 1219 irq = per_cpu(ipi_to_irq, cpu)[vector]; 1220 BUG_ON(irq < 0); 1221 notify_remote_via_irq(irq); 1222} 1223 1224static DEFINE_PER_CPU(unsigned, xed_nesting_count); 1225 1226static void __xen_evtchn_do_upcall(void) 1227{ 1228 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu); 1229 int cpu = get_cpu(); 1230 unsigned count; 1231 1232 do { 1233 vcpu_info->evtchn_upcall_pending = 0; 1234 1235 if (__this_cpu_inc_return(xed_nesting_count) - 1) 1236 goto out; 1237 1238 xen_evtchn_handle_events(cpu); 1239 1240 BUG_ON(!irqs_disabled()); 1241 1242 count = __this_cpu_read(xed_nesting_count); 1243 __this_cpu_write(xed_nesting_count, 0); 1244 } while (count != 1 || vcpu_info->evtchn_upcall_pending); 1245 1246out: 1247 1248 put_cpu(); 1249} 1250 1251void xen_evtchn_do_upcall(struct pt_regs *regs) 1252{ 1253 struct pt_regs *old_regs = set_irq_regs(regs); 1254 1255 irq_enter(); 1256#ifdef CONFIG_X86 1257 exit_idle(); 1258 inc_irq_stat(irq_hv_callback_count); 1259#endif 1260 1261 __xen_evtchn_do_upcall(); 1262 1263 irq_exit(); 1264 set_irq_regs(old_regs); 1265} 1266 1267void xen_hvm_evtchn_do_upcall(void) 1268{ 1269 __xen_evtchn_do_upcall(); 1270} 1271EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall); 1272 1273/* Rebind a new event channel to an existing irq. */ 1274void rebind_evtchn_irq(int evtchn, int irq) 1275{ 1276 struct irq_info *info = info_for_irq(irq); 1277 1278 if (WARN_ON(!info)) 1279 return; 1280 1281 /* Make sure the irq is masked, since the new event channel 1282 will also be masked. */ 1283 disable_irq(irq); 1284 1285 mutex_lock(&irq_mapping_update_lock); 1286 1287 /* After resume the irq<->evtchn mappings are all cleared out */ 1288 BUG_ON(get_evtchn_to_irq(evtchn) != -1); 1289 /* Expect irq to have been bound before, 1290 so there should be a proper type */ 1291 BUG_ON(info->type == IRQT_UNBOUND); 1292 1293 (void)xen_irq_info_evtchn_setup(irq, evtchn); 1294 1295 mutex_unlock(&irq_mapping_update_lock); 1296 1297 bind_evtchn_to_cpu(evtchn, info->cpu); 1298 /* This will be deferred until interrupt is processed */ 1299 irq_set_affinity(irq, cpumask_of(info->cpu)); 1300 1301 /* Unmask the event channel. */ 1302 enable_irq(irq); 1303} 1304 1305/* Rebind an evtchn so that it gets delivered to a specific cpu */ 1306static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu) 1307{ 1308 struct evtchn_bind_vcpu bind_vcpu; 1309 int evtchn = evtchn_from_irq(irq); 1310 int masked; 1311 1312 if (!VALID_EVTCHN(evtchn)) 1313 return -1; 1314 1315 /* 1316 * Events delivered via platform PCI interrupts are always 1317 * routed to vcpu 0 and hence cannot be rebound. 1318 */ 1319 if (xen_hvm_domain() && !xen_have_vector_callback) 1320 return -1; 1321 1322 /* Send future instances of this interrupt to other vcpu. */ 1323 bind_vcpu.port = evtchn; 1324 bind_vcpu.vcpu = tcpu; 1325 1326 /* 1327 * Mask the event while changing the VCPU binding to prevent 1328 * it being delivered on an unexpected VCPU. 1329 */ 1330 masked = test_and_set_mask(evtchn); 1331 1332 /* 1333 * If this fails, it usually just indicates that we're dealing with a 1334 * virq or IPI channel, which don't actually need to be rebound. Ignore 1335 * it, but don't do the xenlinux-level rebind in that case. 1336 */ 1337 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0) 1338 bind_evtchn_to_cpu(evtchn, tcpu); 1339 1340 if (!masked) 1341 unmask_evtchn(evtchn); 1342 1343 return 0; 1344} 1345 1346static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest, 1347 bool force) 1348{ 1349 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask); 1350 1351 return rebind_irq_to_cpu(data->irq, tcpu); 1352} 1353 1354static void enable_dynirq(struct irq_data *data) 1355{ 1356 int evtchn = evtchn_from_irq(data->irq); 1357 1358 if (VALID_EVTCHN(evtchn)) 1359 unmask_evtchn(evtchn); 1360} 1361 1362static void disable_dynirq(struct irq_data *data) 1363{ 1364 int evtchn = evtchn_from_irq(data->irq); 1365 1366 if (VALID_EVTCHN(evtchn)) 1367 mask_evtchn(evtchn); 1368} 1369 1370static void ack_dynirq(struct irq_data *data) 1371{ 1372 int evtchn = evtchn_from_irq(data->irq); 1373 1374 if (!VALID_EVTCHN(evtchn)) 1375 return; 1376 1377 if (unlikely(irqd_is_setaffinity_pending(data)) && 1378 likely(!irqd_irq_disabled(data))) { 1379 int masked = test_and_set_mask(evtchn); 1380 1381 clear_evtchn(evtchn); 1382 1383 irq_move_masked_irq(data); 1384 1385 if (!masked) 1386 unmask_evtchn(evtchn); 1387 } else 1388 clear_evtchn(evtchn); 1389} 1390 1391static void mask_ack_dynirq(struct irq_data *data) 1392{ 1393 disable_dynirq(data); 1394 ack_dynirq(data); 1395} 1396 1397static int retrigger_dynirq(struct irq_data *data) 1398{ 1399 unsigned int evtchn = evtchn_from_irq(data->irq); 1400 int masked; 1401 1402 if (!VALID_EVTCHN(evtchn)) 1403 return 0; 1404 1405 masked = test_and_set_mask(evtchn); 1406 set_evtchn(evtchn); 1407 if (!masked) 1408 unmask_evtchn(evtchn); 1409 1410 return 1; 1411} 1412 1413static void restore_pirqs(void) 1414{ 1415 int pirq, rc, irq, gsi; 1416 struct physdev_map_pirq map_irq; 1417 struct irq_info *info; 1418 1419 list_for_each_entry(info, &xen_irq_list_head, list) { 1420 if (info->type != IRQT_PIRQ) 1421 continue; 1422 1423 pirq = info->u.pirq.pirq; 1424 gsi = info->u.pirq.gsi; 1425 irq = info->irq; 1426 1427 /* save/restore of PT devices doesn't work, so at this point the 1428 * only devices present are GSI based emulated devices */ 1429 if (!gsi) 1430 continue; 1431 1432 map_irq.domid = DOMID_SELF; 1433 map_irq.type = MAP_PIRQ_TYPE_GSI; 1434 map_irq.index = gsi; 1435 map_irq.pirq = pirq; 1436 1437 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq); 1438 if (rc) { 1439 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n", 1440 gsi, irq, pirq, rc); 1441 xen_free_irq(irq); 1442 continue; 1443 } 1444 1445 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq); 1446 1447 __startup_pirq(irq); 1448 } 1449} 1450 1451static void restore_cpu_virqs(unsigned int cpu) 1452{ 1453 struct evtchn_bind_virq bind_virq; 1454 int virq, irq, evtchn; 1455 1456 for (virq = 0; virq < NR_VIRQS; virq++) { 1457 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) 1458 continue; 1459 1460 BUG_ON(virq_from_irq(irq) != virq); 1461 1462 /* Get a new binding from Xen. */ 1463 bind_virq.virq = virq; 1464 bind_virq.vcpu = cpu; 1465 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1466 &bind_virq) != 0) 1467 BUG(); 1468 evtchn = bind_virq.port; 1469 1470 /* Record the new mapping. */ 1471 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq); 1472 bind_evtchn_to_cpu(evtchn, cpu); 1473 } 1474} 1475 1476static void restore_cpu_ipis(unsigned int cpu) 1477{ 1478 struct evtchn_bind_ipi bind_ipi; 1479 int ipi, irq, evtchn; 1480 1481 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) { 1482 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) 1483 continue; 1484 1485 BUG_ON(ipi_from_irq(irq) != ipi); 1486 1487 /* Get a new binding from Xen. */ 1488 bind_ipi.vcpu = cpu; 1489 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 1490 &bind_ipi) != 0) 1491 BUG(); 1492 evtchn = bind_ipi.port; 1493 1494 /* Record the new mapping. */ 1495 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi); 1496 bind_evtchn_to_cpu(evtchn, cpu); 1497 } 1498} 1499 1500/* Clear an irq's pending state, in preparation for polling on it */ 1501void xen_clear_irq_pending(int irq) 1502{ 1503 int evtchn = evtchn_from_irq(irq); 1504 1505 if (VALID_EVTCHN(evtchn)) 1506 clear_evtchn(evtchn); 1507} 1508EXPORT_SYMBOL(xen_clear_irq_pending); 1509void xen_set_irq_pending(int irq) 1510{ 1511 int evtchn = evtchn_from_irq(irq); 1512 1513 if (VALID_EVTCHN(evtchn)) 1514 set_evtchn(evtchn); 1515} 1516 1517bool xen_test_irq_pending(int irq) 1518{ 1519 int evtchn = evtchn_from_irq(irq); 1520 bool ret = false; 1521 1522 if (VALID_EVTCHN(evtchn)) 1523 ret = test_evtchn(evtchn); 1524 1525 return ret; 1526} 1527 1528/* Poll waiting for an irq to become pending with timeout. In the usual case, 1529 * the irq will be disabled so it won't deliver an interrupt. */ 1530void xen_poll_irq_timeout(int irq, u64 timeout) 1531{ 1532 evtchn_port_t evtchn = evtchn_from_irq(irq); 1533 1534 if (VALID_EVTCHN(evtchn)) { 1535 struct sched_poll poll; 1536 1537 poll.nr_ports = 1; 1538 poll.timeout = timeout; 1539 set_xen_guest_handle(poll.ports, &evtchn); 1540 1541 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0) 1542 BUG(); 1543 } 1544} 1545EXPORT_SYMBOL(xen_poll_irq_timeout); 1546/* Poll waiting for an irq to become pending. In the usual case, the 1547 * irq will be disabled so it won't deliver an interrupt. */ 1548void xen_poll_irq(int irq) 1549{ 1550 xen_poll_irq_timeout(irq, 0 /* no timeout */); 1551} 1552 1553/* Check whether the IRQ line is shared with other guests. */ 1554int xen_test_irq_shared(int irq) 1555{ 1556 struct irq_info *info = info_for_irq(irq); 1557 struct physdev_irq_status_query irq_status; 1558 1559 if (WARN_ON(!info)) 1560 return -ENOENT; 1561 1562 irq_status.irq = info->u.pirq.pirq; 1563 1564 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 1565 return 0; 1566 return !(irq_status.flags & XENIRQSTAT_shared); 1567} 1568EXPORT_SYMBOL_GPL(xen_test_irq_shared); 1569 1570void xen_irq_resume(void) 1571{ 1572 unsigned int cpu; 1573 struct irq_info *info; 1574 1575 /* New event-channel space is not 'live' yet. */ 1576 xen_evtchn_mask_all(); 1577 xen_evtchn_resume(); 1578 1579 /* No IRQ <-> event-channel mappings. */ 1580 list_for_each_entry(info, &xen_irq_list_head, list) 1581 info->evtchn = 0; /* zap event-channel binding */ 1582 1583 clear_evtchn_to_irq_all(); 1584 1585 for_each_possible_cpu(cpu) { 1586 restore_cpu_virqs(cpu); 1587 restore_cpu_ipis(cpu); 1588 } 1589 1590 restore_pirqs(); 1591} 1592 1593static struct irq_chip xen_dynamic_chip __read_mostly = { 1594 .name = "xen-dyn", 1595 1596 .irq_disable = disable_dynirq, 1597 .irq_mask = disable_dynirq, 1598 .irq_unmask = enable_dynirq, 1599 1600 .irq_ack = ack_dynirq, 1601 .irq_mask_ack = mask_ack_dynirq, 1602 1603 .irq_set_affinity = set_affinity_irq, 1604 .irq_retrigger = retrigger_dynirq, 1605}; 1606 1607static struct irq_chip xen_pirq_chip __read_mostly = { 1608 .name = "xen-pirq", 1609 1610 .irq_startup = startup_pirq, 1611 .irq_shutdown = shutdown_pirq, 1612 .irq_enable = enable_pirq, 1613 .irq_disable = disable_pirq, 1614 1615 .irq_mask = disable_dynirq, 1616 .irq_unmask = enable_dynirq, 1617 1618 .irq_ack = eoi_pirq, 1619 .irq_eoi = eoi_pirq, 1620 .irq_mask_ack = mask_ack_pirq, 1621 1622 .irq_set_affinity = set_affinity_irq, 1623 1624 .irq_retrigger = retrigger_dynirq, 1625}; 1626 1627static struct irq_chip xen_percpu_chip __read_mostly = { 1628 .name = "xen-percpu", 1629 1630 .irq_disable = disable_dynirq, 1631 .irq_mask = disable_dynirq, 1632 .irq_unmask = enable_dynirq, 1633 1634 .irq_ack = ack_dynirq, 1635}; 1636 1637int xen_set_callback_via(uint64_t via) 1638{ 1639 struct xen_hvm_param a; 1640 a.domid = DOMID_SELF; 1641 a.index = HVM_PARAM_CALLBACK_IRQ; 1642 a.value = via; 1643 return HYPERVISOR_hvm_op(HVMOP_set_param, &a); 1644} 1645EXPORT_SYMBOL_GPL(xen_set_callback_via); 1646 1647#ifdef CONFIG_XEN_PVHVM 1648/* Vector callbacks are better than PCI interrupts to receive event 1649 * channel notifications because we can receive vector callbacks on any 1650 * vcpu and we don't need PCI support or APIC interactions. */ 1651void xen_callback_vector(void) 1652{ 1653 int rc; 1654 uint64_t callback_via; 1655 if (xen_have_vector_callback) { 1656 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR); 1657 rc = xen_set_callback_via(callback_via); 1658 if (rc) { 1659 pr_err("Request for Xen HVM callback vector failed\n"); 1660 xen_have_vector_callback = 0; 1661 return; 1662 } 1663 pr_info("Xen HVM callback vector for event delivery is enabled\n"); 1664 /* in the restore case the vector has already been allocated */ 1665 if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors)) 1666 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, 1667 xen_hvm_callback_vector); 1668 } 1669} 1670#else 1671void xen_callback_vector(void) {} 1672#endif 1673 1674#undef MODULE_PARAM_PREFIX 1675#define MODULE_PARAM_PREFIX "xen." 1676 1677static bool fifo_events = true; 1678module_param(fifo_events, bool, 0); 1679 1680void __init xen_init_IRQ(void) 1681{ 1682 int ret = -EINVAL; 1683 1684 if (fifo_events) 1685 ret = xen_evtchn_fifo_init(); 1686 if (ret < 0) 1687 xen_evtchn_2l_init(); 1688 1689 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()), 1690 sizeof(*evtchn_to_irq), GFP_KERNEL); 1691 BUG_ON(!evtchn_to_irq); 1692 1693 /* No event channels are 'live' right now. */ 1694 xen_evtchn_mask_all(); 1695 1696 pirq_needs_eoi = pirq_needs_eoi_flag; 1697 1698#ifdef CONFIG_X86 1699 if (xen_pv_domain()) { 1700 irq_ctx_init(smp_processor_id()); 1701 if (xen_initial_domain()) 1702 pci_xen_initial_domain(); 1703 } 1704 if (xen_feature(XENFEAT_hvm_callback_vector)) 1705 xen_callback_vector(); 1706 1707 if (xen_hvm_domain()) { 1708 native_init_IRQ(); 1709 /* pci_xen_hvm_init must be called after native_init_IRQ so that 1710 * __acpi_register_gsi can point at the right function */ 1711 pci_xen_hvm_init(); 1712 } else { 1713 int rc; 1714 struct physdev_pirq_eoi_gmfn eoi_gmfn; 1715 1716 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO); 1717 eoi_gmfn.gmfn = virt_to_mfn(pirq_eoi_map); 1718 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn); 1719 /* TODO: No PVH support for PIRQ EOI */ 1720 if (rc != 0) { 1721 free_page((unsigned long) pirq_eoi_map); 1722 pirq_eoi_map = NULL; 1723 } else 1724 pirq_needs_eoi = pirq_check_eoi_map; 1725 } 1726#endif 1727} 1728