This source file includes following definitions.
- pci_msi_setup_msi_irqs
- pci_msi_teardown_msi_irqs
- arch_setup_msi_irq
- arch_teardown_msi_irq
- arch_setup_msi_irqs
- default_teardown_msi_irqs
- arch_teardown_msi_irqs
- default_restore_msi_irq
- arch_restore_msi_irqs
- msi_mask
- __pci_msi_desc_mask_irq
- msi_mask_irq
- pci_msix_desc_addr
- __pci_msix_desc_mask_irq
- msix_mask_irq
- msi_set_mask_bit
- pci_msi_mask_irq
- pci_msi_unmask_irq
- default_restore_msi_irqs
- __pci_read_msi_msg
- __pci_write_msi_msg
- pci_write_msi_msg
- free_msi_irqs
- pci_intx_for_msi
- __pci_restore_msi_state
- __pci_restore_msix_state
- pci_restore_msi_state
- msi_mode_show
- populate_msi_sysfs
- msi_setup_entry
- msi_verify_entries
- msi_capability_init
- msix_map_region
- msix_setup_entries
- msix_program_entries
- msix_capability_init
- pci_msi_supported
- pci_msi_vec_count
- pci_msi_shutdown
- pci_disable_msi
- pci_msix_vec_count
- __pci_enable_msix
- pci_msix_shutdown
- pci_disable_msix
- pci_no_msi
- pci_msi_enabled
- __pci_enable_msi_range
- pci_enable_msi
- __pci_enable_msix_range
- pci_enable_msix_range
- pci_alloc_irq_vectors_affinity
- pci_free_irq_vectors
- pci_irq_vector
- pci_irq_get_affinity
- pci_irq_get_node
- msi_desc_to_pci_dev
- msi_desc_to_pci_sysdata
- pci_msi_domain_write_msg
- pci_msi_domain_calc_hwirq
- pci_msi_desc_is_multi_msi
- pci_msi_domain_check_cap
- pci_msi_domain_handle_error
- pci_msi_domain_set_desc
- pci_msi_domain_update_dom_ops
- pci_msi_domain_update_chip_ops
- pci_msi_create_irq_domain
- get_msi_id_cb
- pci_msi_domain_get_msi_rid
- pci_msi_get_device_domain
1
2
3
4
5
6
7
8
9
10 #include <linux/err.h>
11 #include <linux/mm.h>
12 #include <linux/irq.h>
13 #include <linux/interrupt.h>
14 #include <linux/export.h>
15 #include <linux/ioport.h>
16 #include <linux/pci.h>
17 #include <linux/proc_fs.h>
18 #include <linux/msi.h>
19 #include <linux/smp.h>
20 #include <linux/errno.h>
21 #include <linux/io.h>
22 #include <linux/acpi_iort.h>
23 #include <linux/slab.h>
24 #include <linux/irqdomain.h>
25 #include <linux/of_irq.h>
26
27 #include "pci.h"
28
29 static int pci_msi_enable = 1;
30 int pci_msi_ignore_mask;
31
32 #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
33
34 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
35 static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
36 {
37 struct irq_domain *domain;
38
39 domain = dev_get_msi_domain(&dev->dev);
40 if (domain && irq_domain_is_hierarchy(domain))
41 return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
42
43 return arch_setup_msi_irqs(dev, nvec, type);
44 }
45
46 static void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
47 {
48 struct irq_domain *domain;
49
50 domain = dev_get_msi_domain(&dev->dev);
51 if (domain && irq_domain_is_hierarchy(domain))
52 msi_domain_free_irqs(domain, &dev->dev);
53 else
54 arch_teardown_msi_irqs(dev);
55 }
56 #else
57 #define pci_msi_setup_msi_irqs arch_setup_msi_irqs
58 #define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
59 #endif
60
61
62
63 int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
64 {
65 struct msi_controller *chip = dev->bus->msi;
66 int err;
67
68 if (!chip || !chip->setup_irq)
69 return -EINVAL;
70
71 err = chip->setup_irq(chip, dev, desc);
72 if (err < 0)
73 return err;
74
75 irq_set_chip_data(desc->irq, chip);
76
77 return 0;
78 }
79
80 void __weak arch_teardown_msi_irq(unsigned int irq)
81 {
82 struct msi_controller *chip = irq_get_chip_data(irq);
83
84 if (!chip || !chip->teardown_irq)
85 return;
86
87 chip->teardown_irq(chip, irq);
88 }
89
90 int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
91 {
92 struct msi_controller *chip = dev->bus->msi;
93 struct msi_desc *entry;
94 int ret;
95
96 if (chip && chip->setup_irqs)
97 return chip->setup_irqs(chip, dev, nvec, type);
98
99
100
101
102 if (type == PCI_CAP_ID_MSI && nvec > 1)
103 return 1;
104
105 for_each_pci_msi_entry(entry, dev) {
106 ret = arch_setup_msi_irq(dev, entry);
107 if (ret < 0)
108 return ret;
109 if (ret > 0)
110 return -ENOSPC;
111 }
112
113 return 0;
114 }
115
116
117
118
119
120 void default_teardown_msi_irqs(struct pci_dev *dev)
121 {
122 int i;
123 struct msi_desc *entry;
124
125 for_each_pci_msi_entry(entry, dev)
126 if (entry->irq)
127 for (i = 0; i < entry->nvec_used; i++)
128 arch_teardown_msi_irq(entry->irq + i);
129 }
130
131 void __weak arch_teardown_msi_irqs(struct pci_dev *dev)
132 {
133 return default_teardown_msi_irqs(dev);
134 }
135
136 static void default_restore_msi_irq(struct pci_dev *dev, int irq)
137 {
138 struct msi_desc *entry;
139
140 entry = NULL;
141 if (dev->msix_enabled) {
142 for_each_pci_msi_entry(entry, dev) {
143 if (irq == entry->irq)
144 break;
145 }
146 } else if (dev->msi_enabled) {
147 entry = irq_get_msi_desc(irq);
148 }
149
150 if (entry)
151 __pci_write_msi_msg(entry, &entry->msg);
152 }
153
154 void __weak arch_restore_msi_irqs(struct pci_dev *dev)
155 {
156 return default_restore_msi_irqs(dev);
157 }
158
159 static inline __attribute_const__ u32 msi_mask(unsigned x)
160 {
161
162 if (x >= 5)
163 return 0xffffffff;
164 return (1 << (1 << x)) - 1;
165 }
166
167
168
169
170
171
172
173 u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
174 {
175 u32 mask_bits = desc->masked;
176
177 if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit)
178 return 0;
179
180 mask_bits &= ~mask;
181 mask_bits |= flag;
182 pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos,
183 mask_bits);
184
185 return mask_bits;
186 }
187
188 static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
189 {
190 desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag);
191 }
192
193 static void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
194 {
195 if (desc->msi_attrib.is_virtual)
196 return NULL;
197
198 return desc->mask_base +
199 desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
200 }
201
202
203
204
205
206
207
208
209 u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
210 {
211 u32 mask_bits = desc->masked;
212 void __iomem *desc_addr;
213
214 if (pci_msi_ignore_mask)
215 return 0;
216
217 desc_addr = pci_msix_desc_addr(desc);
218 if (!desc_addr)
219 return 0;
220
221 mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
222 if (flag & PCI_MSIX_ENTRY_CTRL_MASKBIT)
223 mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
224
225 writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
226
227 return mask_bits;
228 }
229
230 static void msix_mask_irq(struct msi_desc *desc, u32 flag)
231 {
232 desc->masked = __pci_msix_desc_mask_irq(desc, flag);
233 }
234
235 static void msi_set_mask_bit(struct irq_data *data, u32 flag)
236 {
237 struct msi_desc *desc = irq_data_get_msi_desc(data);
238
239 if (desc->msi_attrib.is_msix) {
240 msix_mask_irq(desc, flag);
241 readl(desc->mask_base);
242 } else {
243 unsigned offset = data->irq - desc->irq;
244 msi_mask_irq(desc, 1 << offset, flag << offset);
245 }
246 }
247
248
249
250
251
252 void pci_msi_mask_irq(struct irq_data *data)
253 {
254 msi_set_mask_bit(data, 1);
255 }
256 EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
257
258
259
260
261
262 void pci_msi_unmask_irq(struct irq_data *data)
263 {
264 msi_set_mask_bit(data, 0);
265 }
266 EXPORT_SYMBOL_GPL(pci_msi_unmask_irq);
267
268 void default_restore_msi_irqs(struct pci_dev *dev)
269 {
270 struct msi_desc *entry;
271
272 for_each_pci_msi_entry(entry, dev)
273 default_restore_msi_irq(dev, entry->irq);
274 }
275
276 void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
277 {
278 struct pci_dev *dev = msi_desc_to_pci_dev(entry);
279
280 BUG_ON(dev->current_state != PCI_D0);
281
282 if (entry->msi_attrib.is_msix) {
283 void __iomem *base = pci_msix_desc_addr(entry);
284
285 if (!base) {
286 WARN_ON(1);
287 return;
288 }
289
290 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
291 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
292 msg->data = readl(base + PCI_MSIX_ENTRY_DATA);
293 } else {
294 int pos = dev->msi_cap;
295 u16 data;
296
297 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
298 &msg->address_lo);
299 if (entry->msi_attrib.is_64) {
300 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
301 &msg->address_hi);
302 pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data);
303 } else {
304 msg->address_hi = 0;
305 pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data);
306 }
307 msg->data = data;
308 }
309 }
310
311 void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
312 {
313 struct pci_dev *dev = msi_desc_to_pci_dev(entry);
314
315 if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) {
316
317 } else if (entry->msi_attrib.is_msix) {
318 void __iomem *base = pci_msix_desc_addr(entry);
319
320 if (!base)
321 goto skip;
322
323 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
324 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
325 writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
326 } else {
327 int pos = dev->msi_cap;
328 u16 msgctl;
329
330 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
331 msgctl &= ~PCI_MSI_FLAGS_QSIZE;
332 msgctl |= entry->msi_attrib.multiple << 4;
333 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
334
335 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
336 msg->address_lo);
337 if (entry->msi_attrib.is_64) {
338 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
339 msg->address_hi);
340 pci_write_config_word(dev, pos + PCI_MSI_DATA_64,
341 msg->data);
342 } else {
343 pci_write_config_word(dev, pos + PCI_MSI_DATA_32,
344 msg->data);
345 }
346 }
347
348 skip:
349 entry->msg = *msg;
350
351 if (entry->write_msi_msg)
352 entry->write_msi_msg(entry, entry->write_msi_msg_data);
353
354 }
355
356 void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
357 {
358 struct msi_desc *entry = irq_get_msi_desc(irq);
359
360 __pci_write_msi_msg(entry, msg);
361 }
362 EXPORT_SYMBOL_GPL(pci_write_msi_msg);
363
364 static void free_msi_irqs(struct pci_dev *dev)
365 {
366 struct list_head *msi_list = dev_to_msi_list(&dev->dev);
367 struct msi_desc *entry, *tmp;
368 struct attribute **msi_attrs;
369 struct device_attribute *dev_attr;
370 int i, count = 0;
371
372 for_each_pci_msi_entry(entry, dev)
373 if (entry->irq)
374 for (i = 0; i < entry->nvec_used; i++)
375 BUG_ON(irq_has_action(entry->irq + i));
376
377 pci_msi_teardown_msi_irqs(dev);
378
379 list_for_each_entry_safe(entry, tmp, msi_list, list) {
380 if (entry->msi_attrib.is_msix) {
381 if (list_is_last(&entry->list, msi_list))
382 iounmap(entry->mask_base);
383 }
384
385 list_del(&entry->list);
386 free_msi_entry(entry);
387 }
388
389 if (dev->msi_irq_groups) {
390 sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
391 msi_attrs = dev->msi_irq_groups[0]->attrs;
392 while (msi_attrs[count]) {
393 dev_attr = container_of(msi_attrs[count],
394 struct device_attribute, attr);
395 kfree(dev_attr->attr.name);
396 kfree(dev_attr);
397 ++count;
398 }
399 kfree(msi_attrs);
400 kfree(dev->msi_irq_groups[0]);
401 kfree(dev->msi_irq_groups);
402 dev->msi_irq_groups = NULL;
403 }
404 }
405
406 static void pci_intx_for_msi(struct pci_dev *dev, int enable)
407 {
408 if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))
409 pci_intx(dev, enable);
410 }
411
412 static void __pci_restore_msi_state(struct pci_dev *dev)
413 {
414 u16 control;
415 struct msi_desc *entry;
416
417 if (!dev->msi_enabled)
418 return;
419
420 entry = irq_get_msi_desc(dev->irq);
421
422 pci_intx_for_msi(dev, 0);
423 pci_msi_set_enable(dev, 0);
424 arch_restore_msi_irqs(dev);
425
426 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
427 msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap),
428 entry->masked);
429 control &= ~PCI_MSI_FLAGS_QSIZE;
430 control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
431 pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
432 }
433
434 static void __pci_restore_msix_state(struct pci_dev *dev)
435 {
436 struct msi_desc *entry;
437
438 if (!dev->msix_enabled)
439 return;
440 BUG_ON(list_empty(dev_to_msi_list(&dev->dev)));
441
442
443 pci_intx_for_msi(dev, 0);
444 pci_msix_clear_and_set_ctrl(dev, 0,
445 PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
446
447 arch_restore_msi_irqs(dev);
448 for_each_pci_msi_entry(entry, dev)
449 msix_mask_irq(entry, entry->masked);
450
451 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
452 }
453
454 void pci_restore_msi_state(struct pci_dev *dev)
455 {
456 __pci_restore_msi_state(dev);
457 __pci_restore_msix_state(dev);
458 }
459 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
460
461 static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
462 char *buf)
463 {
464 struct msi_desc *entry;
465 unsigned long irq;
466 int retval;
467
468 retval = kstrtoul(attr->attr.name, 10, &irq);
469 if (retval)
470 return retval;
471
472 entry = irq_get_msi_desc(irq);
473 if (entry)
474 return sprintf(buf, "%s\n",
475 entry->msi_attrib.is_msix ? "msix" : "msi");
476
477 return -ENODEV;
478 }
479
480 static int populate_msi_sysfs(struct pci_dev *pdev)
481 {
482 struct attribute **msi_attrs;
483 struct attribute *msi_attr;
484 struct device_attribute *msi_dev_attr;
485 struct attribute_group *msi_irq_group;
486 const struct attribute_group **msi_irq_groups;
487 struct msi_desc *entry;
488 int ret = -ENOMEM;
489 int num_msi = 0;
490 int count = 0;
491 int i;
492
493
494 for_each_pci_msi_entry(entry, pdev)
495 num_msi += entry->nvec_used;
496 if (!num_msi)
497 return 0;
498
499
500 msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
501 if (!msi_attrs)
502 return -ENOMEM;
503 for_each_pci_msi_entry(entry, pdev) {
504 for (i = 0; i < entry->nvec_used; i++) {
505 msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
506 if (!msi_dev_attr)
507 goto error_attrs;
508 msi_attrs[count] = &msi_dev_attr->attr;
509
510 sysfs_attr_init(&msi_dev_attr->attr);
511 msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
512 entry->irq + i);
513 if (!msi_dev_attr->attr.name)
514 goto error_attrs;
515 msi_dev_attr->attr.mode = S_IRUGO;
516 msi_dev_attr->show = msi_mode_show;
517 ++count;
518 }
519 }
520
521 msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
522 if (!msi_irq_group)
523 goto error_attrs;
524 msi_irq_group->name = "msi_irqs";
525 msi_irq_group->attrs = msi_attrs;
526
527 msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
528 if (!msi_irq_groups)
529 goto error_irq_group;
530 msi_irq_groups[0] = msi_irq_group;
531
532 ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
533 if (ret)
534 goto error_irq_groups;
535 pdev->msi_irq_groups = msi_irq_groups;
536
537 return 0;
538
539 error_irq_groups:
540 kfree(msi_irq_groups);
541 error_irq_group:
542 kfree(msi_irq_group);
543 error_attrs:
544 count = 0;
545 msi_attr = msi_attrs[count];
546 while (msi_attr) {
547 msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
548 kfree(msi_attr->name);
549 kfree(msi_dev_attr);
550 ++count;
551 msi_attr = msi_attrs[count];
552 }
553 kfree(msi_attrs);
554 return ret;
555 }
556
557 static struct msi_desc *
558 msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
559 {
560 struct irq_affinity_desc *masks = NULL;
561 struct msi_desc *entry;
562 u16 control;
563
564 if (affd)
565 masks = irq_create_affinity_masks(nvec, affd);
566
567
568 entry = alloc_msi_entry(&dev->dev, nvec, masks);
569 if (!entry)
570 goto out;
571
572 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
573
574 entry->msi_attrib.is_msix = 0;
575 entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT);
576 entry->msi_attrib.is_virtual = 0;
577 entry->msi_attrib.entry_nr = 0;
578 entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT);
579 entry->msi_attrib.default_irq = dev->irq;
580 entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1;
581 entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
582
583 if (control & PCI_MSI_FLAGS_64BIT)
584 entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
585 else
586 entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32;
587
588
589 if (entry->msi_attrib.maskbit)
590 pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
591
592 out:
593 kfree(masks);
594 return entry;
595 }
596
597 static int msi_verify_entries(struct pci_dev *dev)
598 {
599 struct msi_desc *entry;
600
601 for_each_pci_msi_entry(entry, dev) {
602 if (!dev->no_64bit_msi || !entry->msg.address_hi)
603 continue;
604 pci_err(dev, "Device has broken 64-bit MSI but arch"
605 " tried to assign one above 4G\n");
606 return -EIO;
607 }
608 return 0;
609 }
610
611
612
613
614
615
616
617
618
619
620
621
622
623 static int msi_capability_init(struct pci_dev *dev, int nvec,
624 struct irq_affinity *affd)
625 {
626 struct msi_desc *entry;
627 int ret;
628 unsigned mask;
629
630 pci_msi_set_enable(dev, 0);
631
632 entry = msi_setup_entry(dev, nvec, affd);
633 if (!entry)
634 return -ENOMEM;
635
636
637 mask = msi_mask(entry->msi_attrib.multi_cap);
638 msi_mask_irq(entry, mask, mask);
639
640 list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
641
642
643 ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
644 if (ret) {
645 msi_mask_irq(entry, mask, ~mask);
646 free_msi_irqs(dev);
647 return ret;
648 }
649
650 ret = msi_verify_entries(dev);
651 if (ret) {
652 msi_mask_irq(entry, mask, ~mask);
653 free_msi_irqs(dev);
654 return ret;
655 }
656
657 ret = populate_msi_sysfs(dev);
658 if (ret) {
659 msi_mask_irq(entry, mask, ~mask);
660 free_msi_irqs(dev);
661 return ret;
662 }
663
664
665 pci_intx_for_msi(dev, 0);
666 pci_msi_set_enable(dev, 1);
667 dev->msi_enabled = 1;
668
669 pcibios_free_irq(dev);
670 dev->irq = entry->irq;
671 return 0;
672 }
673
674 static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)
675 {
676 resource_size_t phys_addr;
677 u32 table_offset;
678 unsigned long flags;
679 u8 bir;
680
681 pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE,
682 &table_offset);
683 bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR);
684 flags = pci_resource_flags(dev, bir);
685 if (!flags || (flags & IORESOURCE_UNSET))
686 return NULL;
687
688 table_offset &= PCI_MSIX_TABLE_OFFSET;
689 phys_addr = pci_resource_start(dev, bir) + table_offset;
690
691 return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
692 }
693
694 static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
695 struct msix_entry *entries, int nvec,
696 struct irq_affinity *affd)
697 {
698 struct irq_affinity_desc *curmsk, *masks = NULL;
699 struct msi_desc *entry;
700 int ret, i;
701 int vec_count = pci_msix_vec_count(dev);
702
703 if (affd)
704 masks = irq_create_affinity_masks(nvec, affd);
705
706 for (i = 0, curmsk = masks; i < nvec; i++) {
707 entry = alloc_msi_entry(&dev->dev, 1, curmsk);
708 if (!entry) {
709 if (!i)
710 iounmap(base);
711 else
712 free_msi_irqs(dev);
713
714 ret = -ENOMEM;
715 goto out;
716 }
717
718 entry->msi_attrib.is_msix = 1;
719 entry->msi_attrib.is_64 = 1;
720 if (entries)
721 entry->msi_attrib.entry_nr = entries[i].entry;
722 else
723 entry->msi_attrib.entry_nr = i;
724
725 entry->msi_attrib.is_virtual =
726 entry->msi_attrib.entry_nr >= vec_count;
727
728 entry->msi_attrib.default_irq = dev->irq;
729 entry->mask_base = base;
730
731 list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
732 if (masks)
733 curmsk++;
734 }
735 ret = 0;
736 out:
737 kfree(masks);
738 return ret;
739 }
740
741 static void msix_program_entries(struct pci_dev *dev,
742 struct msix_entry *entries)
743 {
744 struct msi_desc *entry;
745 int i = 0;
746 void __iomem *desc_addr;
747
748 for_each_pci_msi_entry(entry, dev) {
749 if (entries)
750 entries[i++].vector = entry->irq;
751
752 desc_addr = pci_msix_desc_addr(entry);
753 if (desc_addr)
754 entry->masked = readl(desc_addr +
755 PCI_MSIX_ENTRY_VECTOR_CTRL);
756 else
757 entry->masked = 0;
758
759 msix_mask_irq(entry, 1);
760 }
761 }
762
763
764
765
766
767
768
769
770
771
772
773
774 static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
775 int nvec, struct irq_affinity *affd)
776 {
777 int ret;
778 u16 control;
779 void __iomem *base;
780
781
782 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
783
784 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
785
786 base = msix_map_region(dev, msix_table_size(control));
787 if (!base)
788 return -ENOMEM;
789
790 ret = msix_setup_entries(dev, base, entries, nvec, affd);
791 if (ret)
792 return ret;
793
794 ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
795 if (ret)
796 goto out_avail;
797
798
799 ret = msi_verify_entries(dev);
800 if (ret)
801 goto out_free;
802
803
804
805
806
807
808 pci_msix_clear_and_set_ctrl(dev, 0,
809 PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE);
810
811 msix_program_entries(dev, entries);
812
813 ret = populate_msi_sysfs(dev);
814 if (ret)
815 goto out_free;
816
817
818 pci_intx_for_msi(dev, 0);
819 dev->msix_enabled = 1;
820 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
821
822 pcibios_free_irq(dev);
823 return 0;
824
825 out_avail:
826 if (ret < 0) {
827
828
829
830
831 struct msi_desc *entry;
832 int avail = 0;
833
834 for_each_pci_msi_entry(entry, dev) {
835 if (entry->irq != 0)
836 avail++;
837 }
838 if (avail != 0)
839 ret = avail;
840 }
841
842 out_free:
843 free_msi_irqs(dev);
844
845 return ret;
846 }
847
848
849
850
851
852
853
854
855
856
857 static int pci_msi_supported(struct pci_dev *dev, int nvec)
858 {
859 struct pci_bus *bus;
860
861
862 if (!pci_msi_enable)
863 return 0;
864
865 if (!dev || dev->no_msi || dev->current_state != PCI_D0)
866 return 0;
867
868
869
870
871
872
873 if (nvec < 1)
874 return 0;
875
876
877
878
879
880
881
882
883 for (bus = dev->bus; bus; bus = bus->parent)
884 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
885 return 0;
886
887 return 1;
888 }
889
890
891
892
893
894
895
896
897
898
899
900 int pci_msi_vec_count(struct pci_dev *dev)
901 {
902 int ret;
903 u16 msgctl;
904
905 if (!dev->msi_cap)
906 return -EINVAL;
907
908 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl);
909 ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1);
910
911 return ret;
912 }
913 EXPORT_SYMBOL(pci_msi_vec_count);
914
915 static void pci_msi_shutdown(struct pci_dev *dev)
916 {
917 struct msi_desc *desc;
918 u32 mask;
919
920 if (!pci_msi_enable || !dev || !dev->msi_enabled)
921 return;
922
923 BUG_ON(list_empty(dev_to_msi_list(&dev->dev)));
924 desc = first_pci_msi_entry(dev);
925
926 pci_msi_set_enable(dev, 0);
927 pci_intx_for_msi(dev, 1);
928 dev->msi_enabled = 0;
929
930
931 mask = msi_mask(desc->msi_attrib.multi_cap);
932
933 __pci_msi_desc_mask_irq(desc, mask, ~mask);
934
935
936 dev->irq = desc->msi_attrib.default_irq;
937 pcibios_alloc_irq(dev);
938 }
939
940 void pci_disable_msi(struct pci_dev *dev)
941 {
942 if (!pci_msi_enable || !dev || !dev->msi_enabled)
943 return;
944
945 pci_msi_shutdown(dev);
946 free_msi_irqs(dev);
947 }
948 EXPORT_SYMBOL(pci_disable_msi);
949
950
951
952
953
954
955
956
957
958 int pci_msix_vec_count(struct pci_dev *dev)
959 {
960 u16 control;
961
962 if (!dev->msix_cap)
963 return -EINVAL;
964
965 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
966 return msix_table_size(control);
967 }
968 EXPORT_SYMBOL(pci_msix_vec_count);
969
970 static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
971 int nvec, struct irq_affinity *affd, int flags)
972 {
973 int nr_entries;
974 int i, j;
975
976 if (!pci_msi_supported(dev, nvec))
977 return -EINVAL;
978
979 nr_entries = pci_msix_vec_count(dev);
980 if (nr_entries < 0)
981 return nr_entries;
982 if (nvec > nr_entries && !(flags & PCI_IRQ_VIRTUAL))
983 return nr_entries;
984
985 if (entries) {
986
987 for (i = 0; i < nvec; i++) {
988 if (entries[i].entry >= nr_entries)
989 return -EINVAL;
990 for (j = i + 1; j < nvec; j++) {
991 if (entries[i].entry == entries[j].entry)
992 return -EINVAL;
993 }
994 }
995 }
996
997
998 if (dev->msi_enabled) {
999 pci_info(dev, "can't enable MSI-X (MSI IRQ already assigned)\n");
1000 return -EINVAL;
1001 }
1002 return msix_capability_init(dev, entries, nvec, affd);
1003 }
1004
1005 static void pci_msix_shutdown(struct pci_dev *dev)
1006 {
1007 struct msi_desc *entry;
1008
1009 if (!pci_msi_enable || !dev || !dev->msix_enabled)
1010 return;
1011
1012 if (pci_dev_is_disconnected(dev)) {
1013 dev->msix_enabled = 0;
1014 return;
1015 }
1016
1017
1018 for_each_pci_msi_entry(entry, dev) {
1019
1020 __pci_msix_desc_mask_irq(entry, 1);
1021 }
1022
1023 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
1024 pci_intx_for_msi(dev, 1);
1025 dev->msix_enabled = 0;
1026 pcibios_alloc_irq(dev);
1027 }
1028
1029 void pci_disable_msix(struct pci_dev *dev)
1030 {
1031 if (!pci_msi_enable || !dev || !dev->msix_enabled)
1032 return;
1033
1034 pci_msix_shutdown(dev);
1035 free_msi_irqs(dev);
1036 }
1037 EXPORT_SYMBOL(pci_disable_msix);
1038
1039 void pci_no_msi(void)
1040 {
1041 pci_msi_enable = 0;
1042 }
1043
1044
1045
1046
1047
1048
1049
1050 int pci_msi_enabled(void)
1051 {
1052 return pci_msi_enable;
1053 }
1054 EXPORT_SYMBOL(pci_msi_enabled);
1055
1056 static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
1057 struct irq_affinity *affd)
1058 {
1059 int nvec;
1060 int rc;
1061
1062 if (!pci_msi_supported(dev, minvec))
1063 return -EINVAL;
1064
1065
1066 if (dev->msix_enabled) {
1067 pci_info(dev, "can't enable MSI (MSI-X already enabled)\n");
1068 return -EINVAL;
1069 }
1070
1071 if (maxvec < minvec)
1072 return -ERANGE;
1073
1074 if (WARN_ON_ONCE(dev->msi_enabled))
1075 return -EINVAL;
1076
1077 nvec = pci_msi_vec_count(dev);
1078 if (nvec < 0)
1079 return nvec;
1080 if (nvec < minvec)
1081 return -ENOSPC;
1082
1083 if (nvec > maxvec)
1084 nvec = maxvec;
1085
1086 for (;;) {
1087 if (affd) {
1088 nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
1089 if (nvec < minvec)
1090 return -ENOSPC;
1091 }
1092
1093 rc = msi_capability_init(dev, nvec, affd);
1094 if (rc == 0)
1095 return nvec;
1096
1097 if (rc < 0)
1098 return rc;
1099 if (rc < minvec)
1100 return -ENOSPC;
1101
1102 nvec = rc;
1103 }
1104 }
1105
1106
1107 int pci_enable_msi(struct pci_dev *dev)
1108 {
1109 int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
1110 if (rc < 0)
1111 return rc;
1112 return 0;
1113 }
1114 EXPORT_SYMBOL(pci_enable_msi);
1115
1116 static int __pci_enable_msix_range(struct pci_dev *dev,
1117 struct msix_entry *entries, int minvec,
1118 int maxvec, struct irq_affinity *affd,
1119 int flags)
1120 {
1121 int rc, nvec = maxvec;
1122
1123 if (maxvec < minvec)
1124 return -ERANGE;
1125
1126 if (WARN_ON_ONCE(dev->msix_enabled))
1127 return -EINVAL;
1128
1129 for (;;) {
1130 if (affd) {
1131 nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
1132 if (nvec < minvec)
1133 return -ENOSPC;
1134 }
1135
1136 rc = __pci_enable_msix(dev, entries, nvec, affd, flags);
1137 if (rc == 0)
1138 return nvec;
1139
1140 if (rc < 0)
1141 return rc;
1142 if (rc < minvec)
1143 return -ENOSPC;
1144
1145 nvec = rc;
1146 }
1147 }
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164 int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
1165 int minvec, int maxvec)
1166 {
1167 return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
1168 }
1169 EXPORT_SYMBOL(pci_enable_msix_range);
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189 int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
1190 unsigned int max_vecs, unsigned int flags,
1191 struct irq_affinity *affd)
1192 {
1193 struct irq_affinity msi_default_affd = {0};
1194 int msix_vecs = -ENOSPC;
1195 int msi_vecs = -ENOSPC;
1196
1197 if (flags & PCI_IRQ_AFFINITY) {
1198 if (!affd)
1199 affd = &msi_default_affd;
1200 } else {
1201 if (WARN_ON(affd))
1202 affd = NULL;
1203 }
1204
1205 if (flags & PCI_IRQ_MSIX) {
1206 msix_vecs = __pci_enable_msix_range(dev, NULL, min_vecs,
1207 max_vecs, affd, flags);
1208 if (msix_vecs > 0)
1209 return msix_vecs;
1210 }
1211
1212 if (flags & PCI_IRQ_MSI) {
1213 msi_vecs = __pci_enable_msi_range(dev, min_vecs, max_vecs,
1214 affd);
1215 if (msi_vecs > 0)
1216 return msi_vecs;
1217 }
1218
1219
1220 if (flags & PCI_IRQ_LEGACY) {
1221 if (min_vecs == 1 && dev->irq) {
1222
1223
1224
1225
1226
1227 if (affd)
1228 irq_create_affinity_masks(1, affd);
1229 pci_intx(dev, 1);
1230 return 1;
1231 }
1232 }
1233
1234 if (msix_vecs == -ENOSPC)
1235 return -ENOSPC;
1236 return msi_vecs;
1237 }
1238 EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
1239
1240
1241
1242
1243
1244
1245
1246 void pci_free_irq_vectors(struct pci_dev *dev)
1247 {
1248 pci_disable_msix(dev);
1249 pci_disable_msi(dev);
1250 }
1251 EXPORT_SYMBOL(pci_free_irq_vectors);
1252
1253
1254
1255
1256
1257
1258 int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
1259 {
1260 if (dev->msix_enabled) {
1261 struct msi_desc *entry;
1262 int i = 0;
1263
1264 for_each_pci_msi_entry(entry, dev) {
1265 if (i == nr)
1266 return entry->irq;
1267 i++;
1268 }
1269 WARN_ON_ONCE(1);
1270 return -EINVAL;
1271 }
1272
1273 if (dev->msi_enabled) {
1274 struct msi_desc *entry = first_pci_msi_entry(dev);
1275
1276 if (WARN_ON_ONCE(nr >= entry->nvec_used))
1277 return -EINVAL;
1278 } else {
1279 if (WARN_ON_ONCE(nr > 0))
1280 return -EINVAL;
1281 }
1282
1283 return dev->irq + nr;
1284 }
1285 EXPORT_SYMBOL(pci_irq_vector);
1286
1287
1288
1289
1290
1291
1292 const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
1293 {
1294 if (dev->msix_enabled) {
1295 struct msi_desc *entry;
1296 int i = 0;
1297
1298 for_each_pci_msi_entry(entry, dev) {
1299 if (i == nr)
1300 return &entry->affinity->mask;
1301 i++;
1302 }
1303 WARN_ON_ONCE(1);
1304 return NULL;
1305 } else if (dev->msi_enabled) {
1306 struct msi_desc *entry = first_pci_msi_entry(dev);
1307
1308 if (WARN_ON_ONCE(!entry || !entry->affinity ||
1309 nr >= entry->nvec_used))
1310 return NULL;
1311
1312 return &entry->affinity[nr].mask;
1313 } else {
1314 return cpu_possible_mask;
1315 }
1316 }
1317 EXPORT_SYMBOL(pci_irq_get_affinity);
1318
1319
1320
1321
1322
1323
1324 int pci_irq_get_node(struct pci_dev *pdev, int vec)
1325 {
1326 const struct cpumask *mask;
1327
1328 mask = pci_irq_get_affinity(pdev, vec);
1329 if (mask)
1330 return local_memory_node(cpu_to_node(cpumask_first(mask)));
1331 return dev_to_node(&pdev->dev);
1332 }
1333 EXPORT_SYMBOL(pci_irq_get_node);
1334
1335 struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
1336 {
1337 return to_pci_dev(desc->dev);
1338 }
1339 EXPORT_SYMBOL(msi_desc_to_pci_dev);
1340
1341 void *msi_desc_to_pci_sysdata(struct msi_desc *desc)
1342 {
1343 struct pci_dev *dev = msi_desc_to_pci_dev(desc);
1344
1345 return dev->bus->sysdata;
1346 }
1347 EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata);
1348
1349 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
1350
1351
1352
1353
1354
1355 void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
1356 {
1357 struct msi_desc *desc = irq_data_get_msi_desc(irq_data);
1358
1359
1360
1361
1362
1363 if (desc->irq == irq_data->irq)
1364 __pci_write_msi_msg(desc, msg);
1365 }
1366
1367
1368
1369
1370
1371
1372
1373
1374 irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,
1375 struct msi_desc *desc)
1376 {
1377 return (irq_hw_number_t)desc->msi_attrib.entry_nr |
1378 pci_dev_id(dev) << 11 |
1379 (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
1380 }
1381
1382 static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc)
1383 {
1384 return !desc->msi_attrib.is_msix && desc->nvec_used > 1;
1385 }
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399 int pci_msi_domain_check_cap(struct irq_domain *domain,
1400 struct msi_domain_info *info, struct device *dev)
1401 {
1402 struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev));
1403
1404
1405 if (pci_msi_desc_is_multi_msi(desc) &&
1406 !(info->flags & MSI_FLAG_MULTI_PCI_MSI))
1407 return 1;
1408 else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX))
1409 return -ENOTSUPP;
1410
1411 return 0;
1412 }
1413
1414 static int pci_msi_domain_handle_error(struct irq_domain *domain,
1415 struct msi_desc *desc, int error)
1416 {
1417
1418 if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC)
1419 return 1;
1420
1421 return error;
1422 }
1423
1424 #ifdef GENERIC_MSI_DOMAIN_OPS
1425 static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
1426 struct msi_desc *desc)
1427 {
1428 arg->desc = desc;
1429 arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc),
1430 desc);
1431 }
1432 #else
1433 #define pci_msi_domain_set_desc NULL
1434 #endif
1435
1436 static struct msi_domain_ops pci_msi_domain_ops_default = {
1437 .set_desc = pci_msi_domain_set_desc,
1438 .msi_check = pci_msi_domain_check_cap,
1439 .handle_error = pci_msi_domain_handle_error,
1440 };
1441
1442 static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)
1443 {
1444 struct msi_domain_ops *ops = info->ops;
1445
1446 if (ops == NULL) {
1447 info->ops = &pci_msi_domain_ops_default;
1448 } else {
1449 if (ops->set_desc == NULL)
1450 ops->set_desc = pci_msi_domain_set_desc;
1451 if (ops->msi_check == NULL)
1452 ops->msi_check = pci_msi_domain_check_cap;
1453 if (ops->handle_error == NULL)
1454 ops->handle_error = pci_msi_domain_handle_error;
1455 }
1456 }
1457
1458 static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)
1459 {
1460 struct irq_chip *chip = info->chip;
1461
1462 BUG_ON(!chip);
1463 if (!chip->irq_write_msi_msg)
1464 chip->irq_write_msi_msg = pci_msi_domain_write_msg;
1465 if (!chip->irq_mask)
1466 chip->irq_mask = pci_msi_mask_irq;
1467 if (!chip->irq_unmask)
1468 chip->irq_unmask = pci_msi_unmask_irq;
1469 }
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482 struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
1483 struct msi_domain_info *info,
1484 struct irq_domain *parent)
1485 {
1486 struct irq_domain *domain;
1487
1488 if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE))
1489 info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
1490
1491 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
1492 pci_msi_domain_update_dom_ops(info);
1493 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
1494 pci_msi_domain_update_chip_ops(info);
1495
1496 info->flags |= MSI_FLAG_ACTIVATE_EARLY;
1497 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
1498 info->flags |= MSI_FLAG_MUST_REACTIVATE;
1499
1500
1501 info->chip->flags |= IRQCHIP_ONESHOT_SAFE;
1502
1503 domain = msi_create_irq_domain(fwnode, info, parent);
1504 if (!domain)
1505 return NULL;
1506
1507 irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI);
1508 return domain;
1509 }
1510 EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525 static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
1526 {
1527 u32 *pa = data;
1528 u8 bus = PCI_BUS_NUM(*pa);
1529
1530 if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)
1531 *pa = alias;
1532
1533 return 0;
1534 }
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546 u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
1547 {
1548 struct device_node *of_node;
1549 u32 rid = pci_dev_id(pdev);
1550
1551 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
1552
1553 of_node = irq_domain_get_of_node(domain);
1554 rid = of_node ? of_msi_map_rid(&pdev->dev, of_node, rid) :
1555 iort_msi_map_rid(&pdev->dev, rid);
1556
1557 return rid;
1558 }
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569 struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
1570 {
1571 struct irq_domain *dom;
1572 u32 rid = pci_dev_id(pdev);
1573
1574 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
1575 dom = of_msi_map_get_device_domain(&pdev->dev, rid);
1576 if (!dom)
1577 dom = iort_get_device_domain(&pdev->dev, rid);
1578 return dom;
1579 }
1580 #endif