22 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id)  in pci_iov_virtfn_bus()  argument
24 	if (!dev->is_physfn)  in pci_iov_virtfn_bus()
26 	return dev->bus->number + ((dev->devfn + dev->sriov->offset +  in pci_iov_virtfn_bus()
27 				    dev->sriov->stride * vf_id) >> 8);  in pci_iov_virtfn_bus()
30 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id)  in pci_iov_virtfn_devfn()  argument
32 	if (!dev->is_physfn)  in pci_iov_virtfn_devfn()
34 	return (dev->devfn + dev->sriov->offset +  in pci_iov_virtfn_devfn()
35 		dev->sriov->stride * vf_id) & 0xff;  in pci_iov_virtfn_devfn()
44 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn)  in pci_iov_set_numvfs()  argument
46 	struct pci_sriov *iov = dev->sriov;  in pci_iov_set_numvfs()
48 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);  in pci_iov_set_numvfs()
49 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset);  in pci_iov_set_numvfs()
50 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride);  in pci_iov_set_numvfs()
60 static int compute_max_vf_buses(struct pci_dev *dev)  in compute_max_vf_buses()  argument
62 	struct pci_sriov *iov = dev->sriov;  in compute_max_vf_buses()
66 		pci_iov_set_numvfs(dev, nr_virtfn);  in compute_max_vf_buses()
72 		busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1);  in compute_max_vf_buses()
78 	pci_iov_set_numvfs(dev, 0);  in compute_max_vf_buses()
108 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)  in pci_iov_resource_size()  argument
110 	if (!dev->is_physfn)  in pci_iov_resource_size()
113 	return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];  in pci_iov_resource_size()
116 static int virtfn_add(struct pci_dev *dev, int id, int reset)  in virtfn_add()  argument
124 	struct pci_sriov *iov = dev->sriov;  in virtfn_add()
127 	mutex_lock(&iov->dev->sriov->lock);  in virtfn_add()
128 	bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));  in virtfn_add()
136 	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);  in virtfn_add()
137 	virtfn->vendor = dev->vendor;  in virtfn_add()
138 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);  in virtfn_add()
140 	virtfn->dev.parent = dev->dev.parent;  in virtfn_add()
141 	virtfn->physfn = pci_dev_get(dev);  in virtfn_add()
146 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in virtfn_add()
151 		size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);  in virtfn_add()
162 	mutex_unlock(&iov->dev->sriov->lock);  in virtfn_add()
166 	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);  in virtfn_add()
169 	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");  in virtfn_add()
173 	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);  in virtfn_add()
178 	sysfs_remove_link(&dev->dev.kobj, buf);  in virtfn_add()
180 	pci_dev_put(dev);  in virtfn_add()
181 	mutex_lock(&iov->dev->sriov->lock);  in virtfn_add()
184 	virtfn_remove_bus(dev->bus, bus);  in virtfn_add()
186 	mutex_unlock(&iov->dev->sriov->lock);  in virtfn_add()
191 static void virtfn_remove(struct pci_dev *dev, int id, int reset)  in virtfn_remove()  argument
195 	struct pci_sriov *iov = dev->sriov;  in virtfn_remove()
197 	virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),  in virtfn_remove()
198 					     pci_iov_virtfn_bus(dev, id),  in virtfn_remove()
199 					     pci_iov_virtfn_devfn(dev, id));  in virtfn_remove()
204 		device_release_driver(&virtfn->dev);  in virtfn_remove()
209 	sysfs_remove_link(&dev->dev.kobj, buf);  in virtfn_remove()
215 	if (virtfn->dev.kobj.sd)  in virtfn_remove()
216 		sysfs_remove_link(&virtfn->dev.kobj, "physfn");  in virtfn_remove()
218 	mutex_lock(&iov->dev->sriov->lock);  in virtfn_remove()
220 	virtfn_remove_bus(dev->bus, virtfn->bus);  in virtfn_remove()
221 	mutex_unlock(&iov->dev->sriov->lock);  in virtfn_remove()
225 	pci_dev_put(dev);  in virtfn_remove()
238 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)  in sriov_enable()  argument
246 	struct pci_sriov *iov = dev->sriov;  in sriov_enable()
256 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);  in sriov_enable()
268 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in sriov_enable()
273 		dev_err(&dev->dev, "not enough MMIO resources for SR-IOV\n");  in sriov_enable()
277 	bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);  in sriov_enable()
278 	if (bus > dev->bus->busn_res.end) {  in sriov_enable()
279 		dev_err(&dev->dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",  in sriov_enable()
280 			nr_virtfn, bus, &dev->bus->busn_res);  in sriov_enable()
284 	if (pci_enable_resources(dev, bars)) {  in sriov_enable()
285 		dev_err(&dev->dev, "SR-IOV: IOV BARS not allocated\n");  in sriov_enable()
289 	if (iov->link != dev->devfn) {  in sriov_enable()
290 		pdev = pci_get_slot(dev->bus, iov->link);  in sriov_enable()
299 		rc = sysfs_create_link(&dev->dev.kobj,  in sriov_enable()
300 					&pdev->dev.kobj, "dep_link");  in sriov_enable()
306 	pci_iov_set_numvfs(dev, nr_virtfn);  in sriov_enable()
308 	pci_cfg_access_lock(dev);  in sriov_enable()
309 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_enable()
311 	pci_cfg_access_unlock(dev);  in sriov_enable()
317 	rc = pcibios_sriov_enable(dev, initial);  in sriov_enable()
319 		dev_err(&dev->dev, "failure %d from pcibios_sriov_enable()\n", rc);  in sriov_enable()
324 		rc = virtfn_add(dev, i, 0);  in sriov_enable()
329 	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);  in sriov_enable()
336 		virtfn_remove(dev, i, 0);  in sriov_enable()
338 	pcibios_sriov_disable(dev);  in sriov_enable()
341 	pci_cfg_access_lock(dev);  in sriov_enable()
342 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_enable()
344 	pci_cfg_access_unlock(dev);  in sriov_enable()
346 	if (iov->link != dev->devfn)  in sriov_enable()
347 		sysfs_remove_link(&dev->dev.kobj, "dep_link");  in sriov_enable()
349 	pci_iov_set_numvfs(dev, 0);  in sriov_enable()
353 static void sriov_disable(struct pci_dev *dev)  in sriov_disable()  argument
356 	struct pci_sriov *iov = dev->sriov;  in sriov_disable()
362 		virtfn_remove(dev, i, 0);  in sriov_disable()
364 	pcibios_sriov_disable(dev);  in sriov_disable()
367 	pci_cfg_access_lock(dev);  in sriov_disable()
368 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_disable()
370 	pci_cfg_access_unlock(dev);  in sriov_disable()
372 	if (iov->link != dev->devfn)  in sriov_disable()
373 		sysfs_remove_link(&dev->dev.kobj, "dep_link");  in sriov_disable()
376 	pci_iov_set_numvfs(dev, 0);  in sriov_disable()
379 static int sriov_init(struct pci_dev *dev, int pos)  in sriov_init()  argument
390 	if (pci_pcie_type(dev) != PCI_EXP_TYPE_RC_END &&  in sriov_init()
391 	    pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT)  in sriov_init()
394 	pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);  in sriov_init()
396 		pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);  in sriov_init()
401 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)  in sriov_init()
406 	if (pci_ari_enabled(dev->bus))  in sriov_init()
410 	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);  in sriov_init()
412 	pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);  in sriov_init()
416 	pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);  in sriov_init()
423 	pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);  in sriov_init()
431 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in sriov_init()
439 			bar64 = __pci_read_base(dev, pci_bar_unknown, res,  in sriov_init()
449 		dev_info(&dev->dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n",  in sriov_init()
460 	iov->self = dev;  in sriov_init()
461 	pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);  in sriov_init()
462 	pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);  in sriov_init()
463 	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)  in sriov_init()
464 		iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);  in sriov_init()
467 		iov->dev = pci_dev_get(pdev);  in sriov_init()
469 		iov->dev = dev;  in sriov_init()
473 	dev->sriov = iov;  in sriov_init()
474 	dev->is_physfn = 1;  in sriov_init()
475 	rc = compute_max_vf_buses(dev);  in sriov_init()
482 	dev->sriov = NULL;  in sriov_init()
483 	dev->is_physfn = 0;  in sriov_init()
486 		res = &dev->resource[i + PCI_IOV_RESOURCES];  in sriov_init()
494 static void sriov_release(struct pci_dev *dev)  in sriov_release()  argument
496 	BUG_ON(dev->sriov->num_VFs);  in sriov_release()
498 	if (dev != dev->sriov->dev)  in sriov_release()
499 		pci_dev_put(dev->sriov->dev);  in sriov_release()
501 	mutex_destroy(&dev->sriov->lock);  in sriov_release()
503 	kfree(dev->sriov);  in sriov_release()
504 	dev->sriov = NULL;  in sriov_release()
507 static void sriov_restore_state(struct pci_dev *dev)  in sriov_restore_state()  argument
511 	struct pci_sriov *iov = dev->sriov;  in sriov_restore_state()
513 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);  in sriov_restore_state()
518 		pci_update_resource(dev, i);  in sriov_restore_state()
520 	pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);  in sriov_restore_state()
521 	pci_iov_set_numvfs(dev, iov->num_VFs);  in sriov_restore_state()
522 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);  in sriov_restore_state()
533 int pci_iov_init(struct pci_dev *dev)  in pci_iov_init()  argument
537 	if (!pci_is_pcie(dev))  in pci_iov_init()
540 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);  in pci_iov_init()
542 		return sriov_init(dev, pos);  in pci_iov_init()
551 void pci_iov_release(struct pci_dev *dev)  in pci_iov_release()  argument
553 	if (dev->is_physfn)  in pci_iov_release()
554 		sriov_release(dev);  in pci_iov_release()
564 int pci_iov_resource_bar(struct pci_dev *dev, int resno)  in pci_iov_resource_bar()  argument
569 	BUG_ON(!dev->is_physfn);  in pci_iov_resource_bar()
571 	return dev->sriov->pos + PCI_SRIOV_BAR +  in pci_iov_resource_bar()
575 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,  in pcibios_iov_resource_alignment()  argument
578 	return pci_iov_resource_size(dev, resno);  in pcibios_iov_resource_alignment()
591 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)  in pci_sriov_resource_alignment()  argument
593 	return pcibios_iov_resource_alignment(dev, resno);  in pci_sriov_resource_alignment()
600 void pci_restore_iov_state(struct pci_dev *dev)  in pci_restore_iov_state()  argument
602 	if (dev->is_physfn)  in pci_restore_iov_state()
603 		sriov_restore_state(dev);  in pci_restore_iov_state()
616 	struct pci_dev *dev;  in pci_iov_bus_range()  local
618 	list_for_each_entry(dev, &bus->devices, bus_list) {  in pci_iov_bus_range()
619 		if (!dev->is_physfn)  in pci_iov_bus_range()
621 		if (dev->sriov->max_VF_buses > max)  in pci_iov_bus_range()
622 			max = dev->sriov->max_VF_buses;  in pci_iov_bus_range()
635 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)  in pci_enable_sriov()  argument
639 	if (!dev->is_physfn)  in pci_enable_sriov()
642 	return sriov_enable(dev, nr_virtfn);  in pci_enable_sriov()
650 void pci_disable_sriov(struct pci_dev *dev)  in pci_disable_sriov()  argument
654 	if (!dev->is_physfn)  in pci_disable_sriov()
657 	sriov_disable(dev);  in pci_disable_sriov()
667 int pci_num_vf(struct pci_dev *dev)  in pci_num_vf()  argument
669 	if (!dev->is_physfn)  in pci_num_vf()
672 	return dev->sriov->num_VFs;  in pci_num_vf()
683 int pci_vfs_assigned(struct pci_dev *dev)  in pci_vfs_assigned()  argument
690 	if (!dev->is_physfn)  in pci_vfs_assigned()
697 	pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);  in pci_vfs_assigned()
700 	vfdev = pci_get_device(dev->vendor, dev_id, NULL);  in pci_vfs_assigned()
706 		if (vfdev->is_virtfn && (vfdev->physfn == dev) &&  in pci_vfs_assigned()
710 		vfdev = pci_get_device(dev->vendor, dev_id, vfdev);  in pci_vfs_assigned()
730 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)  in pci_sriov_set_totalvfs()  argument
732 	if (!dev->is_physfn)  in pci_sriov_set_totalvfs()
734 	if (numvfs > dev->sriov->total_VFs)  in pci_sriov_set_totalvfs()
738 	if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE)  in pci_sriov_set_totalvfs()
741 		dev->sriov->driver_max_VFs = numvfs;  in pci_sriov_set_totalvfs()
755 int pci_sriov_get_totalvfs(struct pci_dev *dev)  in pci_sriov_get_totalvfs()  argument
757 	if (!dev->is_physfn)  in pci_sriov_get_totalvfs()
760 	if (dev->sriov->driver_max_VFs)  in pci_sriov_get_totalvfs()
761 		return dev->sriov->driver_max_VFs;  in pci_sriov_get_totalvfs()
763 	return dev->sriov->total_VFs;  in pci_sriov_get_totalvfs()