35 struct irq_domain * __weak arch_get_pci_msi_domain(struct pci_dev *dev)  in arch_get_pci_msi_domain()  argument
40 static struct irq_domain *pci_msi_get_domain(struct pci_dev *dev)  in pci_msi_get_domain()  argument
44 	if (dev->bus->msi)  in pci_msi_get_domain()
45 		domain = dev->bus->msi->domain;  in pci_msi_get_domain()
47 		domain = arch_get_pci_msi_domain(dev);  in pci_msi_get_domain()
52 static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)  in pci_msi_setup_msi_irqs()  argument
56 	domain = pci_msi_get_domain(dev);  in pci_msi_setup_msi_irqs()
58 		return pci_msi_domain_alloc_irqs(domain, dev, nvec, type);  in pci_msi_setup_msi_irqs()
60 	return arch_setup_msi_irqs(dev, nvec, type);  in pci_msi_setup_msi_irqs()
63 static void pci_msi_teardown_msi_irqs(struct pci_dev *dev)  in pci_msi_teardown_msi_irqs()  argument
67 	domain = pci_msi_get_domain(dev);  in pci_msi_teardown_msi_irqs()
69 		pci_msi_domain_free_irqs(domain, dev);  in pci_msi_teardown_msi_irqs()
71 		arch_teardown_msi_irqs(dev);  in pci_msi_teardown_msi_irqs()
80 struct msi_controller * __weak pcibios_msi_controller(struct pci_dev *dev)  in pcibios_msi_controller()  argument
85 static struct msi_controller *pci_msi_controller(struct pci_dev *dev)  in pci_msi_controller()  argument
87 	struct msi_controller *msi_ctrl = dev->bus->msi;  in pci_msi_controller()
92 	return pcibios_msi_controller(dev);  in pci_msi_controller()
95 int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)  in arch_setup_msi_irq()  argument
97 	struct msi_controller *chip = pci_msi_controller(dev);  in arch_setup_msi_irq()
103 	err = chip->setup_irq(chip, dev, desc);  in arch_setup_msi_irq()
122 int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)  in arch_setup_msi_irqs()  argument
134 	list_for_each_entry(entry, &dev->msi_list, list) {  in arch_setup_msi_irqs()
135 		ret = arch_setup_msi_irq(dev, entry);  in arch_setup_msi_irqs()
149 void default_teardown_msi_irqs(struct pci_dev *dev)  in default_teardown_msi_irqs()  argument
154 	list_for_each_entry(entry, &dev->msi_list, list)  in default_teardown_msi_irqs()
160 void __weak arch_teardown_msi_irqs(struct pci_dev *dev)  in arch_teardown_msi_irqs()  argument
162 	return default_teardown_msi_irqs(dev);  in arch_teardown_msi_irqs()
165 static void default_restore_msi_irq(struct pci_dev *dev, int irq)  in default_restore_msi_irq()  argument
170 	if (dev->msix_enabled) {  in default_restore_msi_irq()
171 		list_for_each_entry(entry, &dev->msi_list, list) {  in default_restore_msi_irq()
175 	} else if (dev->msi_enabled)  {  in default_restore_msi_irq()
183 void __weak arch_restore_msi_irqs(struct pci_dev *dev)  in arch_restore_msi_irqs()  argument
185 	return default_restore_msi_irqs(dev);  in arch_restore_msi_irqs()
188 static void msi_set_enable(struct pci_dev *dev, int enable)  in msi_set_enable()  argument
192 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);  in msi_set_enable()
196 	pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);  in msi_set_enable()
199 static void msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set)  in msix_clear_and_set_ctrl()  argument
203 	pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl);  in msix_clear_and_set_ctrl()
206 	pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in msix_clear_and_set_ctrl()
232 	pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits);  in __pci_msi_desc_mask_irq()
302 void default_restore_msi_irqs(struct pci_dev *dev)  in default_restore_msi_irqs()  argument
306 	list_for_each_entry(entry, &dev->msi_list, list)  in default_restore_msi_irqs()
307 		default_restore_msi_irq(dev, entry->irq);  in default_restore_msi_irqs()
312 	BUG_ON(entry->dev->current_state != PCI_D0);  in __pci_read_msi_msg()
322 		struct pci_dev *dev = entry->dev;  in __pci_read_msi_msg()  local
323 		int pos = dev->msi_cap;  in __pci_read_msi_msg()
326 		pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,  in __pci_read_msi_msg()
329 			pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,  in __pci_read_msi_msg()
331 			pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data);  in __pci_read_msi_msg()
334 			pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data);  in __pci_read_msi_msg()
342 	if (entry->dev->current_state != PCI_D0) {  in __pci_write_msi_msg()
353 		struct pci_dev *dev = entry->dev;  in __pci_write_msi_msg()  local
354 		int pos = dev->msi_cap;  in __pci_write_msi_msg()
357 		pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);  in __pci_write_msi_msg()
360 		pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);  in __pci_write_msi_msg()
362 		pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,  in __pci_write_msi_msg()
365 			pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,  in __pci_write_msi_msg()
367 			pci_write_config_word(dev, pos + PCI_MSI_DATA_64,  in __pci_write_msi_msg()
370 			pci_write_config_word(dev, pos + PCI_MSI_DATA_32,  in __pci_write_msi_msg()
385 static void free_msi_irqs(struct pci_dev *dev)  in free_msi_irqs()  argument
392 	list_for_each_entry(entry, &dev->msi_list, list)  in free_msi_irqs()
397 	pci_msi_teardown_msi_irqs(dev);  in free_msi_irqs()
399 	list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) {  in free_msi_irqs()
401 			if (list_is_last(&entry->list, &dev->msi_list))  in free_msi_irqs()
409 	if (dev->msi_irq_groups) {  in free_msi_irqs()
410 		sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);  in free_msi_irqs()
411 		msi_attrs = dev->msi_irq_groups[0]->attrs;  in free_msi_irqs()
420 		kfree(dev->msi_irq_groups[0]);  in free_msi_irqs()
421 		kfree(dev->msi_irq_groups);  in free_msi_irqs()
422 		dev->msi_irq_groups = NULL;  in free_msi_irqs()
426 static struct msi_desc *alloc_msi_entry(struct pci_dev *dev)  in alloc_msi_entry()  argument
433 	desc->dev = dev;  in alloc_msi_entry()
438 static void pci_intx_for_msi(struct pci_dev *dev, int enable)  in pci_intx_for_msi()  argument
440 	if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))  in pci_intx_for_msi()
441 		pci_intx(dev, enable);  in pci_intx_for_msi()
444 static void __pci_restore_msi_state(struct pci_dev *dev)  in __pci_restore_msi_state()  argument
449 	if (!dev->msi_enabled)  in __pci_restore_msi_state()
452 	entry = irq_get_msi_desc(dev->irq);  in __pci_restore_msi_state()
454 	pci_intx_for_msi(dev, 0);  in __pci_restore_msi_state()
455 	msi_set_enable(dev, 0);  in __pci_restore_msi_state()
456 	arch_restore_msi_irqs(dev);  in __pci_restore_msi_state()
458 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);  in __pci_restore_msi_state()
463 	pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);  in __pci_restore_msi_state()
466 static void __pci_restore_msix_state(struct pci_dev *dev)  in __pci_restore_msix_state()  argument
470 	if (!dev->msix_enabled)  in __pci_restore_msix_state()
472 	BUG_ON(list_empty(&dev->msi_list));  in __pci_restore_msix_state()
475 	pci_intx_for_msi(dev, 0);  in __pci_restore_msix_state()
476 	msix_clear_and_set_ctrl(dev, 0,  in __pci_restore_msix_state()
479 	arch_restore_msi_irqs(dev);  in __pci_restore_msix_state()
480 	list_for_each_entry(entry, &dev->msi_list, list)  in __pci_restore_msix_state()
483 	msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);  in __pci_restore_msix_state()
486 void pci_restore_msi_state(struct pci_dev *dev)  in pci_restore_msi_state()  argument
488 	__pci_restore_msi_state(dev);  in pci_restore_msi_state()
489 	__pci_restore_msix_state(dev);  in pci_restore_msi_state()
493 static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,  in msi_mode_show()  argument
561 	ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);  in populate_msi_sysfs()
586 static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)  in msi_setup_entry()  argument
592 	entry = alloc_msi_entry(dev);  in msi_setup_entry()
596 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);  in msi_setup_entry()
602 	entry->msi_attrib.default_irq	= dev->irq;	/* Save IOAPIC IRQ */  in msi_setup_entry()
608 		entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;  in msi_setup_entry()
610 		entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32;  in msi_setup_entry()
614 		pci_read_config_dword(dev, entry->mask_pos, &entry->masked);  in msi_setup_entry()
619 static int msi_verify_entries(struct pci_dev *dev)  in msi_verify_entries()  argument
623 	list_for_each_entry(entry, &dev->msi_list, list) {  in msi_verify_entries()
624 		if (!dev->no_64bit_msi || !entry->msg.address_hi)  in msi_verify_entries()
626 		dev_err(&dev->dev, "Device has broken 64-bit MSI but arch"  in msi_verify_entries()
644 static int msi_capability_init(struct pci_dev *dev, int nvec)  in msi_capability_init()  argument
650 	msi_set_enable(dev, 0);	/* Disable MSI during set up */  in msi_capability_init()
652 	entry = msi_setup_entry(dev, nvec);  in msi_capability_init()
660 	list_add_tail(&entry->list, &dev->msi_list);  in msi_capability_init()
663 	ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);  in msi_capability_init()
666 		free_msi_irqs(dev);  in msi_capability_init()
670 	ret = msi_verify_entries(dev);  in msi_capability_init()
673 		free_msi_irqs(dev);  in msi_capability_init()
677 	ret = populate_msi_sysfs(dev);  in msi_capability_init()
680 		free_msi_irqs(dev);  in msi_capability_init()
685 	pci_intx_for_msi(dev, 0);  in msi_capability_init()
686 	msi_set_enable(dev, 1);  in msi_capability_init()
687 	dev->msi_enabled = 1;  in msi_capability_init()
689 	dev->irq = entry->irq;  in msi_capability_init()
693 static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)  in msix_map_region()  argument
700 	pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE,  in msix_map_region()
703 	flags = pci_resource_flags(dev, bir);  in msix_map_region()
708 	phys_addr = pci_resource_start(dev, bir) + table_offset;  in msix_map_region()
713 static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,  in msix_setup_entries()  argument
720 		entry = alloc_msi_entry(dev);  in msix_setup_entries()
725 				free_msi_irqs(dev);  in msix_setup_entries()
733 		entry->msi_attrib.default_irq	= dev->irq;  in msix_setup_entries()
737 		list_add_tail(&entry->list, &dev->msi_list);  in msix_setup_entries()
743 static void msix_program_entries(struct pci_dev *dev,  in msix_program_entries()  argument
749 	list_for_each_entry(entry, &dev->msi_list, list) {  in msix_program_entries()
770 static int msix_capability_init(struct pci_dev *dev,  in msix_capability_init()  argument
778 	msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);  in msix_capability_init()
780 	pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);  in msix_capability_init()
782 	base = msix_map_region(dev, msix_table_size(control));  in msix_capability_init()
786 	ret = msix_setup_entries(dev, base, entries, nvec);  in msix_capability_init()
790 	ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);  in msix_capability_init()
795 	ret = msi_verify_entries(dev);  in msix_capability_init()
804 	msix_clear_and_set_ctrl(dev, 0,  in msix_capability_init()
807 	msix_program_entries(dev, entries);  in msix_capability_init()
809 	ret = populate_msi_sysfs(dev);  in msix_capability_init()
814 	pci_intx_for_msi(dev, 0);  in msix_capability_init()
815 	dev->msix_enabled = 1;  in msix_capability_init()
817 	msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);  in msix_capability_init()
830 		list_for_each_entry(entry, &dev->msi_list, list) {  in msix_capability_init()
839 	free_msi_irqs(dev);  in msix_capability_init()
853 static int pci_msi_supported(struct pci_dev *dev, int nvec)  in pci_msi_supported()  argument
861 	if (!dev || dev->no_msi || dev->current_state != PCI_D0)  in pci_msi_supported()
879 	for (bus = dev->bus; bus; bus = bus->parent)  in pci_msi_supported()
896 int pci_msi_vec_count(struct pci_dev *dev)  in pci_msi_vec_count()  argument
901 	if (!dev->msi_cap)  in pci_msi_vec_count()
904 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl);  in pci_msi_vec_count()
911 void pci_msi_shutdown(struct pci_dev *dev)  in pci_msi_shutdown()  argument
916 	if (!pci_msi_enable || !dev || !dev->msi_enabled)  in pci_msi_shutdown()
919 	BUG_ON(list_empty(&dev->msi_list));  in pci_msi_shutdown()
920 	desc = list_first_entry(&dev->msi_list, struct msi_desc, list);  in pci_msi_shutdown()
922 	msi_set_enable(dev, 0);  in pci_msi_shutdown()
923 	pci_intx_for_msi(dev, 1);  in pci_msi_shutdown()
924 	dev->msi_enabled = 0;  in pci_msi_shutdown()
932 	dev->irq = desc->msi_attrib.default_irq;  in pci_msi_shutdown()
935 void pci_disable_msi(struct pci_dev *dev)  in pci_disable_msi()  argument
937 	if (!pci_msi_enable || !dev || !dev->msi_enabled)  in pci_disable_msi()
940 	pci_msi_shutdown(dev);  in pci_disable_msi()
941 	free_msi_irqs(dev);  in pci_disable_msi()
953 int pci_msix_vec_count(struct pci_dev *dev)  in pci_msix_vec_count()  argument
957 	if (!dev->msix_cap)  in pci_msix_vec_count()
960 	pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);  in pci_msix_vec_count()
980 int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)  in pci_enable_msix()  argument
985 	if (!pci_msi_supported(dev, nvec))  in pci_enable_msix()
991 	nr_entries = pci_msix_vec_count(dev);  in pci_enable_msix()
1006 	WARN_ON(!!dev->msix_enabled);  in pci_enable_msix()
1009 	if (dev->msi_enabled) {  in pci_enable_msix()
1010 		dev_info(&dev->dev, "can't enable MSI-X (MSI IRQ already assigned)\n");  in pci_enable_msix()
1013 	return msix_capability_init(dev, entries, nvec);  in pci_enable_msix()
1017 void pci_msix_shutdown(struct pci_dev *dev)  in pci_msix_shutdown()  argument
1021 	if (!pci_msi_enable || !dev || !dev->msix_enabled)  in pci_msix_shutdown()
1025 	list_for_each_entry(entry, &dev->msi_list, list) {  in pci_msix_shutdown()
1030 	msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);  in pci_msix_shutdown()
1031 	pci_intx_for_msi(dev, 1);  in pci_msix_shutdown()
1032 	dev->msix_enabled = 0;  in pci_msix_shutdown()
1035 void pci_disable_msix(struct pci_dev *dev)  in pci_disable_msix()  argument
1037 	if (!pci_msi_enable || !dev || !dev->msix_enabled)  in pci_disable_msix()
1040 	pci_msix_shutdown(dev);  in pci_disable_msix()
1041 	free_msi_irqs(dev);  in pci_disable_msix()
1062 void pci_msi_init_pci_dev(struct pci_dev *dev)  in pci_msi_init_pci_dev()  argument
1064 	INIT_LIST_HEAD(&dev->msi_list);  in pci_msi_init_pci_dev()
1070 	dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI);  in pci_msi_init_pci_dev()
1071 	if (dev->msi_cap)  in pci_msi_init_pci_dev()
1072 		msi_set_enable(dev, 0);  in pci_msi_init_pci_dev()
1074 	dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX);  in pci_msi_init_pci_dev()
1075 	if (dev->msix_cap)  in pci_msi_init_pci_dev()
1076 		msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);  in pci_msi_init_pci_dev()
1091 int pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec)  in pci_enable_msi_range()  argument
1096 	if (!pci_msi_supported(dev, minvec))  in pci_enable_msi_range()
1099 	WARN_ON(!!dev->msi_enabled);  in pci_enable_msi_range()
1102 	if (dev->msix_enabled) {  in pci_enable_msi_range()
1103 		dev_info(&dev->dev,  in pci_enable_msi_range()
1111 	nvec = pci_msi_vec_count(dev);  in pci_enable_msi_range()
1120 		rc = msi_capability_init(dev, nvec);  in pci_enable_msi_range()
1149 int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,  in pci_enable_msix_range()  argument
1159 		rc = pci_enable_msix(dev, entries, nvec);  in pci_enable_msix_range()
1198 irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,  in pci_msi_domain_calc_hwirq()  argument
1202 		PCI_DEVID(dev->bus->number, dev->devfn) << 11 |  in pci_msi_domain_calc_hwirq()
1203 		(pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;  in pci_msi_domain_calc_hwirq()
1223 			     struct msi_domain_info *info, struct device *dev)  in pci_msi_domain_check_cap()  argument
1225 	struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev));  in pci_msi_domain_check_cap()
1323 int pci_msi_domain_alloc_irqs(struct irq_domain *domain, struct pci_dev *dev,  in pci_msi_domain_alloc_irqs()  argument
1326 	return msi_domain_alloc_irqs(domain, &dev->dev, nvec);  in pci_msi_domain_alloc_irqs()
1334 void pci_msi_domain_free_irqs(struct irq_domain *domain, struct pci_dev *dev)  in pci_msi_domain_free_irqs()  argument
1336 	msi_domain_free_irqs(domain, &dev->dev);  in pci_msi_domain_free_irqs()