22 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id)  in pci_iov_virtfn_bus()  argument
24 	if (!dev->is_physfn)  in pci_iov_virtfn_bus()
26 	return dev->bus->number + ((dev->devfn + dev->sriov->offset +  in pci_iov_virtfn_bus()
27 				    dev->sriov->stride * vf_id) >> 8);  in pci_iov_virtfn_bus()
30 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id)  in pci_iov_virtfn_devfn()  argument
32 	if (!dev->is_physfn)  in pci_iov_virtfn_devfn()
34 	return (dev->devfn + dev->sriov->offset +  in pci_iov_virtfn_devfn()
35 		dev->sriov->stride * vf_id) & 0xff;  in pci_iov_virtfn_devfn()
44 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn)  in pci_iov_set_numvfs()  argument
46 	struct pci_sriov *iov = dev->sriov;  in pci_iov_set_numvfs()
48 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);  in pci_iov_set_numvfs()
49 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset);  in pci_iov_set_numvfs()
50 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride);  in pci_iov_set_numvfs()
60 static inline u8 virtfn_max_buses(struct pci_dev *dev)  in virtfn_max_buses()  argument
62 	struct pci_sriov *iov = dev->sriov;  in virtfn_max_buses()
68 		pci_iov_set_numvfs(dev, nr_virtfn);  in virtfn_max_buses()
69 		busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1);  in virtfn_max_buses()
103 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)  in pci_iov_resource_size()  argument
105 	if (!dev->is_physfn)  in pci_iov_resource_size()
108 	return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];  in pci_iov_resource_size()
111 static int virtfn_add(struct pci_dev *dev, int id, int reset)  in virtfn_add()  argument
119 	struct pci_sriov *iov = dev->sriov;  in virtfn_add()
122 	mutex_lock(&iov->dev->sriov->lock);  in virtfn_add()
123 	bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));  in virtfn_add()
131 	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);  in virtfn_add()
132 	virtfn->vendor = dev->vendor;  in virtfn_add()
133 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);  in virtfn_add()
135 	virtfn->dev.parent = dev->dev.parent;  in virtfn_add()
136 	virtfn->physfn = pci_dev_get(dev);  in virtfn_add()
141 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in virtfn_add()
146 		size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);  in virtfn_add()
157 	mutex_unlock(&iov->dev->sriov->lock);  in virtfn_add()
161 	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);  in virtfn_add()
164 	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");  in virtfn_add()
168 	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);  in virtfn_add()
173 	sysfs_remove_link(&dev->dev.kobj, buf);  in virtfn_add()
175 	pci_dev_put(dev);  in virtfn_add()
176 	mutex_lock(&iov->dev->sriov->lock);  in virtfn_add()
179 	virtfn_remove_bus(dev->bus, bus);  in virtfn_add()
181 	mutex_unlock(&iov->dev->sriov->lock);  in virtfn_add()
186 static void virtfn_remove(struct pci_dev *dev, int id, int reset)  in virtfn_remove()  argument
190 	struct pci_sriov *iov = dev->sriov;  in virtfn_remove()
192 	virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),  in virtfn_remove()
193 					     pci_iov_virtfn_bus(dev, id),  in virtfn_remove()
194 					     pci_iov_virtfn_devfn(dev, id));  in virtfn_remove()
199 		device_release_driver(&virtfn->dev);  in virtfn_remove()
204 	sysfs_remove_link(&dev->dev.kobj, buf);  in virtfn_remove()
210 	if (virtfn->dev.kobj.sd)  in virtfn_remove()
211 		sysfs_remove_link(&virtfn->dev.kobj, "physfn");  in virtfn_remove()
213 	mutex_lock(&iov->dev->sriov->lock);  in virtfn_remove()
215 	virtfn_remove_bus(dev->bus, virtfn->bus);  in virtfn_remove()
216 	mutex_unlock(&iov->dev->sriov->lock);  in virtfn_remove()
220 	pci_dev_put(dev);  in virtfn_remove()
228 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)  in sriov_enable()  argument
236 	struct pci_sriov *iov = dev->sriov;  in sriov_enable()
247 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);  in sriov_enable()
256 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &offset);  in sriov_enable()
257 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &stride);  in sriov_enable()
264 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in sriov_enable()
269 		dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");  in sriov_enable()
276 	bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);  in sriov_enable()
277 	if (bus > dev->bus->busn_res.end) {  in sriov_enable()
278 		dev_err(&dev->dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",  in sriov_enable()
279 			nr_virtfn, bus, &dev->bus->busn_res);  in sriov_enable()
283 	if (pci_enable_resources(dev, bars)) {  in sriov_enable()
284 		dev_err(&dev->dev, "SR-IOV: IOV BARS not allocated\n");  in sriov_enable()
288 	if (iov->link != dev->devfn) {  in sriov_enable()
289 		pdev = pci_get_slot(dev->bus, iov->link);  in sriov_enable()
298 		rc = sysfs_create_link(&dev->dev.kobj,  in sriov_enable()
299 					&pdev->dev.kobj, "dep_link");  in sriov_enable()
305 	pci_iov_set_numvfs(dev, nr_virtfn);  in sriov_enable()
307 	pci_cfg_access_lock(dev);  in sriov_enable()
308 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_enable()
310 	pci_cfg_access_unlock(dev);  in sriov_enable()
316 	if ((retval = pcibios_sriov_enable(dev, initial))) {  in sriov_enable()
317 		dev_err(&dev->dev, "failure %d from pcibios_sriov_enable()\n",  in sriov_enable()
323 		rc = virtfn_add(dev, i, 0);  in sriov_enable()
328 	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);  in sriov_enable()
335 		virtfn_remove(dev, j, 0);  in sriov_enable()
338 	pci_cfg_access_lock(dev);  in sriov_enable()
339 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_enable()
340 	pci_iov_set_numvfs(dev, 0);  in sriov_enable()
342 	pci_cfg_access_unlock(dev);  in sriov_enable()
344 	if (iov->link != dev->devfn)  in sriov_enable()
345 		sysfs_remove_link(&dev->dev.kobj, "dep_link");  in sriov_enable()
355 static void sriov_disable(struct pci_dev *dev)  in sriov_disable()  argument
358 	struct pci_sriov *iov = dev->sriov;  in sriov_disable()
364 		virtfn_remove(dev, i, 0);  in sriov_disable()
366 	pcibios_sriov_disable(dev);  in sriov_disable()
369 	pci_cfg_access_lock(dev);  in sriov_disable()
370 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_disable()
372 	pci_cfg_access_unlock(dev);  in sriov_disable()
374 	if (iov->link != dev->devfn)  in sriov_disable()
375 		sysfs_remove_link(&dev->dev.kobj, "dep_link");  in sriov_disable()
378 	pci_iov_set_numvfs(dev, 0);  in sriov_disable()
381 static int sriov_init(struct pci_dev *dev, int pos)  in sriov_init()  argument
392 	if (pci_pcie_type(dev) != PCI_EXP_TYPE_RC_END &&  in sriov_init()
393 	    pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT)  in sriov_init()
396 	pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);  in sriov_init()
398 		pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);  in sriov_init()
402 	pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);  in sriov_init()
407 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)  in sriov_init()
412 	if (pci_ari_enabled(dev->bus))  in sriov_init()
416 	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);  in sriov_init()
417 	pci_write_config_word(dev, pos + PCI_SRIOV_NUM_VF, 0);  in sriov_init()
418 	pci_read_config_word(dev, pos + PCI_SRIOV_VF_OFFSET, &offset);  in sriov_init()
419 	pci_read_config_word(dev, pos + PCI_SRIOV_VF_STRIDE, &stride);  in sriov_init()
423 	pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);  in sriov_init()
430 	pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);  in sriov_init()
438 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in sriov_init()
439 		bar64 = __pci_read_base(dev, pci_bar_unknown, res,  in sriov_init()
449 		dev_info(&dev->dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n",  in sriov_init()
462 	iov->self = dev;  in sriov_init()
463 	pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);  in sriov_init()
464 	pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);  in sriov_init()
465 	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)  in sriov_init()
466 		iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);  in sriov_init()
469 		iov->dev = pci_dev_get(pdev);  in sriov_init()
471 		iov->dev = dev;  in sriov_init()
475 	dev->sriov = iov;  in sriov_init()
476 	dev->is_physfn = 1;  in sriov_init()
477 	iov->max_VF_buses = virtfn_max_buses(dev);  in sriov_init()
483 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in sriov_init()
491 static void sriov_release(struct pci_dev *dev)  in sriov_release()  argument
493 	BUG_ON(dev->sriov->num_VFs);  in sriov_release()
495 	if (dev != dev->sriov->dev)  in sriov_release()
496 		pci_dev_put(dev->sriov->dev);  in sriov_release()
498 	mutex_destroy(&dev->sriov->lock);  in sriov_release()
500 	kfree(dev->sriov);  in sriov_release()
501 	dev->sriov = NULL;  in sriov_release()
504 static void sriov_restore_state(struct pci_dev *dev)  in sriov_restore_state()  argument
508 	struct pci_sriov *iov = dev->sriov;  in sriov_restore_state()
510 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);  in sriov_restore_state()
515 		pci_update_resource(dev, i);  in sriov_restore_state()
517 	pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);  in sriov_restore_state()
518 	pci_iov_set_numvfs(dev, iov->num_VFs);  in sriov_restore_state()
519 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_restore_state()
530 int pci_iov_init(struct pci_dev *dev)  in pci_iov_init()  argument
534 	if (!pci_is_pcie(dev))  in pci_iov_init()
537 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);  in pci_iov_init()
539 		return sriov_init(dev, pos);  in pci_iov_init()
548 void pci_iov_release(struct pci_dev *dev)  in pci_iov_release()  argument
550 	if (dev->is_physfn)  in pci_iov_release()
551 		sriov_release(dev);  in pci_iov_release()
561 int pci_iov_resource_bar(struct pci_dev *dev, int resno)  in pci_iov_resource_bar()  argument
566 	BUG_ON(!dev->is_physfn);  in pci_iov_resource_bar()
568 	return dev->sriov->pos + PCI_SRIOV_BAR +  in pci_iov_resource_bar()
572 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,  in pcibios_iov_resource_alignment()  argument
575 	return pci_iov_resource_size(dev, resno);  in pcibios_iov_resource_alignment()
588 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)  in pci_sriov_resource_alignment()  argument
590 	return pcibios_iov_resource_alignment(dev, resno);  in pci_sriov_resource_alignment()
597 void pci_restore_iov_state(struct pci_dev *dev)  in pci_restore_iov_state()  argument
599 	if (dev->is_physfn)  in pci_restore_iov_state()
600 		sriov_restore_state(dev);  in pci_restore_iov_state()
613 	struct pci_dev *dev;  in pci_iov_bus_range()  local
615 	list_for_each_entry(dev, &bus->devices, bus_list) {  in pci_iov_bus_range()
616 		if (!dev->is_physfn)  in pci_iov_bus_range()
618 		if (dev->sriov->max_VF_buses > max)  in pci_iov_bus_range()
619 			max = dev->sriov->max_VF_buses;  in pci_iov_bus_range()
632 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)  in pci_enable_sriov()  argument
636 	if (!dev->is_physfn)  in pci_enable_sriov()
639 	return sriov_enable(dev, nr_virtfn);  in pci_enable_sriov()
647 void pci_disable_sriov(struct pci_dev *dev)  in pci_disable_sriov()  argument
651 	if (!dev->is_physfn)  in pci_disable_sriov()
654 	sriov_disable(dev);  in pci_disable_sriov()
664 int pci_num_vf(struct pci_dev *dev)  in pci_num_vf()  argument
666 	if (!dev->is_physfn)  in pci_num_vf()
669 	return dev->sriov->num_VFs;  in pci_num_vf()
680 int pci_vfs_assigned(struct pci_dev *dev)  in pci_vfs_assigned()  argument
687 	if (!dev->is_physfn)  in pci_vfs_assigned()
694 	pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);  in pci_vfs_assigned()
697 	vfdev = pci_get_device(dev->vendor, dev_id, NULL);  in pci_vfs_assigned()
703 		if (vfdev->is_virtfn && (vfdev->physfn == dev) &&  in pci_vfs_assigned()
707 		vfdev = pci_get_device(dev->vendor, dev_id, vfdev);  in pci_vfs_assigned()
727 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)  in pci_sriov_set_totalvfs()  argument
729 	if (!dev->is_physfn)  in pci_sriov_set_totalvfs()
731 	if (numvfs > dev->sriov->total_VFs)  in pci_sriov_set_totalvfs()
735 	if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE)  in pci_sriov_set_totalvfs()
738 		dev->sriov->driver_max_VFs = numvfs;  in pci_sriov_set_totalvfs()
752 int pci_sriov_get_totalvfs(struct pci_dev *dev)  in pci_sriov_get_totalvfs()  argument
754 	if (!dev->is_physfn)  in pci_sriov_get_totalvfs()
757 	if (dev->sriov->driver_max_VFs)  in pci_sriov_get_totalvfs()
758 		return dev->sriov->driver_max_VFs;  in pci_sriov_get_totalvfs()
760 	return dev->sriov->total_VFs;  in pci_sriov_get_totalvfs()