117 	int	(*readfn)(struct vfio_pci_device *vdev, int pos, int count,
119 	int	(*writefn)(struct vfio_pci_device *vdev, int pos, int count,
180 static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_default_config_read()  argument
186 	memcpy(val, vdev->vconfig + pos, count);  in vfio_default_config_read()
192 		struct pci_dev *pdev = vdev->pdev;  in vfio_default_config_read()
206 static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_default_config_write()  argument
223 		memcpy(&virt_val, vdev->vconfig + pos, count);  in vfio_default_config_write()
228 		memcpy(vdev->vconfig + pos, &virt_val, count);  in vfio_default_config_write()
233 		struct pci_dev *pdev = vdev->pdev;  in vfio_default_config_write()
253 static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_direct_config_read()  argument
259 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);  in vfio_direct_config_read()
265 			memcpy(val, vdev->vconfig + pos, count);  in vfio_direct_config_read()
268 			memcpy(val, vdev->vconfig + pos,  in vfio_direct_config_read()
271 			memcpy(val, vdev->vconfig + pos, 1);  in vfio_direct_config_read()
278 static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_raw_config_write()  argument
284 	ret = vfio_user_config_write(vdev->pdev, pos, val, count);  in vfio_raw_config_write()
291 static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_raw_config_read()  argument
297 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);  in vfio_raw_config_read()
386 static void vfio_bar_restore(struct vfio_pci_device *vdev)  in vfio_bar_restore()  argument
388 	struct pci_dev *pdev = vdev->pdev;  in vfio_bar_restore()
389 	u32 *rbar = vdev->rbar;  in vfio_bar_restore()
427 static void vfio_bar_fixup(struct vfio_pci_device *vdev)  in vfio_bar_fixup()  argument
429 	struct pci_dev *pdev = vdev->pdev;  in vfio_bar_fixup()
434 	bar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0];  in vfio_bar_fixup()
454 	bar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS];  in vfio_bar_fixup()
468 	vdev->bardirty = false;  in vfio_bar_fixup()
471 static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_basic_config_read()  argument
476 		vfio_bar_fixup(vdev);  in vfio_basic_config_read()
478 	count = vfio_default_config_read(vdev, pos, count, perm, offset, val);  in vfio_basic_config_read()
481 	if (offset == PCI_COMMAND && vdev->pdev->is_virtfn) {  in vfio_basic_config_read()
482 		u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);  in vfio_basic_config_read()
492 static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_basic_config_write()  argument
496 	struct pci_dev *pdev = vdev->pdev;  in vfio_basic_config_write()
501 	virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND];  in vfio_basic_config_write()
531 			vfio_bar_restore(vdev);  in vfio_basic_config_write()
534 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_basic_config_write()
556 		if (virt_intx_disable && !vdev->virq_disabled) {  in vfio_basic_config_write()
557 			vdev->virq_disabled = true;  in vfio_basic_config_write()
558 			vfio_pci_intx_mask(vdev);  in vfio_basic_config_write()
559 		} else if (!virt_intx_disable && vdev->virq_disabled) {  in vfio_basic_config_write()
560 			vdev->virq_disabled = false;  in vfio_basic_config_write()
561 			vfio_pci_intx_unmask(vdev);  in vfio_basic_config_write()
566 		vdev->bardirty = true;  in vfio_basic_config_write()
619 static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_pm_config_write()  argument
623 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_pm_config_write()
645 		pci_set_power_state(vdev->pdev, state);  in vfio_pm_config_write()
674 static int vfio_vpd_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_vpd_config_write()  argument
678 	struct pci_dev *pdev = vdev->pdev;  in vfio_vpd_config_write()
679 	__le16 *paddr = (__le16 *)(vdev->vconfig + pos - offset + PCI_VPD_ADDR);  in vfio_vpd_config_write()
680 	__le32 *pdata = (__le32 *)(vdev->vconfig + pos - offset + PCI_VPD_DATA);  in vfio_vpd_config_write()
689 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_vpd_config_write()
895 static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)  in vfio_find_cap_start()  argument
900 	cap = vdev->pci_config_map[pos];  in vfio_find_cap_start()
906 	while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap)  in vfio_find_cap_start()
912 static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_msi_config_read()  argument
921 		start = vfio_find_cap_start(vdev, pos);  in vfio_msi_config_read()
923 		flags = (__le16 *)&vdev->vconfig[start];  in vfio_msi_config_read()
926 		*flags |= cpu_to_le16(vdev->msi_qmax << 1);  in vfio_msi_config_read()
929 	return vfio_default_config_read(vdev, pos, count, perm, offset, val);  in vfio_msi_config_read()
932 static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_msi_config_write()  argument
936 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_msi_config_write()
946 		start = vfio_find_cap_start(vdev, pos);  in vfio_msi_config_write()
948 		pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS];  in vfio_msi_config_write()
953 		if  (!is_msi(vdev))  in vfio_msi_config_write()
957 		if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) {  in vfio_msi_config_write()
959 			flags |= vdev->msi_qmax << 4;  in vfio_msi_config_write()
964 		ret = pci_user_write_config_word(vdev->pdev,  in vfio_msi_config_write()
1012 static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)  in vfio_msi_cap_len()  argument
1014 	struct pci_dev *pdev = vdev->pdev;  in vfio_msi_cap_len()
1028 	if (vdev->msi_perm)  in vfio_msi_cap_len()
1031 	vdev->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL);  in vfio_msi_cap_len()
1032 	if (!vdev->msi_perm)  in vfio_msi_cap_len()
1035 	ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags);  in vfio_msi_cap_len()
1043 static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos)  in vfio_vc_cap_len()  argument
1045 	struct pci_dev *pdev = vdev->pdev;  in vfio_vc_cap_len()
1084 static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)  in vfio_cap_len()  argument
1086 	struct pci_dev *pdev = vdev->pdev;  in vfio_cap_len()
1094 		return vfio_msi_cap_len(vdev, pos);  in vfio_cap_len()
1103 			vdev->extended_caps = (dword != 0);  in vfio_cap_len()
1117 		vdev->extended_caps = (dword != 0);  in vfio_cap_len()
1149 static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos)  in vfio_ext_cap_len()  argument
1151 	struct pci_dev *pdev = vdev->pdev;  in vfio_ext_cap_len()
1166 		return vfio_vc_cap_len(vdev, epos);  in vfio_ext_cap_len()
1223 static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,  in vfio_fill_vconfig_bytes()  argument
1226 	struct pci_dev *pdev = vdev->pdev;  in vfio_fill_vconfig_bytes()
1238 			__le32 *dwordp = (__le32 *)&vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1247 			__le16 *wordp = (__le16 *)&vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1256 			u8 *byte = &vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1270 static int vfio_cap_init(struct vfio_pci_device *vdev)  in vfio_cap_init()  argument
1272 	struct pci_dev *pdev = vdev->pdev;  in vfio_cap_init()
1273 	u8 *map = vdev->pci_config_map;  in vfio_cap_init()
1291 	prev = &vdev->vconfig[PCI_CAPABILITY_LIST];  in vfio_cap_init()
1311 				len = vfio_cap_len(vdev, cap, pos);  in vfio_cap_init()
1336 		ret = vfio_fill_vconfig_bytes(vdev, pos, len);  in vfio_cap_init()
1340 		prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT];  in vfio_cap_init()
1347 		__le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS];  in vfio_cap_init()
1354 static int vfio_ecap_init(struct vfio_pci_device *vdev)  in vfio_ecap_init()  argument
1356 	struct pci_dev *pdev = vdev->pdev;  in vfio_ecap_init()
1357 	u8 *map = vdev->pci_config_map;  in vfio_ecap_init()
1362 	if (!vdev->extended_caps)  in vfio_ecap_init()
1384 				len = vfio_ext_cap_len(vdev, ecap, epos);  in vfio_ecap_init()
1427 		ret = vfio_fill_vconfig_bytes(vdev, epos, len);  in vfio_ecap_init()
1438 			*(__le32 *)&vdev->vconfig[epos] &=  in vfio_ecap_init()
1443 		prev = (__le32 *)&vdev->vconfig[epos];  in vfio_ecap_init()
1448 		*(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0;  in vfio_ecap_init()
1466 int vfio_config_init(struct vfio_pci_device *vdev)  in vfio_config_init()  argument
1468 	struct pci_dev *pdev = vdev->pdev;  in vfio_config_init()
1488 	vdev->pci_config_map = map;  in vfio_config_init()
1489 	vdev->vconfig = vconfig;  in vfio_config_init()
1495 	ret = vfio_fill_vconfig_bytes(vdev, 0, PCI_STD_HEADER_SIZEOF);  in vfio_config_init()
1499 	vdev->bardirty = true;  in vfio_config_init()
1507 	vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]);  in vfio_config_init()
1508 	vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]);  in vfio_config_init()
1509 	vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]);  in vfio_config_init()
1510 	vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]);  in vfio_config_init()
1511 	vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]);  in vfio_config_init()
1512 	vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]);  in vfio_config_init()
1513 	vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]);  in vfio_config_init()
1523 	ret = vfio_cap_init(vdev);  in vfio_config_init()
1527 	ret = vfio_ecap_init(vdev);  in vfio_config_init()
1535 	vdev->pci_config_map = NULL;  in vfio_config_init()
1537 	vdev->vconfig = NULL;  in vfio_config_init()
1541 void vfio_config_free(struct vfio_pci_device *vdev)  in vfio_config_free()  argument
1543 	kfree(vdev->vconfig);  in vfio_config_free()
1544 	vdev->vconfig = NULL;  in vfio_config_free()
1545 	kfree(vdev->pci_config_map);  in vfio_config_free()
1546 	vdev->pci_config_map = NULL;  in vfio_config_free()
1547 	kfree(vdev->msi_perm);  in vfio_config_free()
1548 	vdev->msi_perm = NULL;  in vfio_config_free()
1555 static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,  in vfio_pci_cap_remaining_dword()  argument
1558 	u8 cap = vdev->pci_config_map[pos];  in vfio_pci_cap_remaining_dword()
1561 	for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++)  in vfio_pci_cap_remaining_dword()
1567 static ssize_t vfio_config_do_rw(struct vfio_pci_device *vdev, char __user *buf,  in vfio_config_do_rw()  argument
1570 	struct pci_dev *pdev = vdev->pdev;  in vfio_config_do_rw()
1585 	count = min(count, vfio_pci_cap_remaining_dword(vdev, *ppos));  in vfio_config_do_rw()
1595 	cap_id = vdev->pci_config_map[*ppos];  in vfio_config_do_rw()
1605 			cap_start = vfio_find_cap_start(vdev, *ppos);  in vfio_config_do_rw()
1612 				perm = vdev->msi_perm;  in vfio_config_do_rw()
1615 				cap_start = vfio_find_cap_start(vdev, *ppos);  in vfio_config_do_rw()
1631 		ret = perm->writefn(vdev, *ppos, count, perm, offset, val);  in vfio_config_do_rw()
1634 			ret = perm->readfn(vdev, *ppos, count,  in vfio_config_do_rw()
1647 ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev, char __user *buf,  in vfio_pci_config_rw()  argument
1657 		ret = vfio_config_do_rw(vdev, buf, count, &pos, iswrite);  in vfio_pci_config_rw()