117 	int	(*readfn)(struct vfio_pci_device *vdev, int pos, int count,
119 	int	(*writefn)(struct vfio_pci_device *vdev, int pos, int count,
180 static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_default_config_read()  argument
186 	memcpy(val, vdev->vconfig + pos, count);  in vfio_default_config_read()
192 		struct pci_dev *pdev = vdev->pdev;  in vfio_default_config_read()
206 static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_default_config_write()  argument
223 		memcpy(&virt_val, vdev->vconfig + pos, count);  in vfio_default_config_write()
228 		memcpy(vdev->vconfig + pos, &virt_val, count);  in vfio_default_config_write()
233 		struct pci_dev *pdev = vdev->pdev;  in vfio_default_config_write()
253 static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_direct_config_read()  argument
259 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);  in vfio_direct_config_read()
265 			memcpy(val, vdev->vconfig + pos, count);  in vfio_direct_config_read()
268 			memcpy(val, vdev->vconfig + pos,  in vfio_direct_config_read()
271 			memcpy(val, vdev->vconfig + pos, 1);  in vfio_direct_config_read()
278 static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_raw_config_write()  argument
284 	ret = vfio_user_config_write(vdev->pdev, pos, val, count);  in vfio_raw_config_write()
291 static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_raw_config_read()  argument
297 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);  in vfio_raw_config_read()
386 static void vfio_bar_restore(struct vfio_pci_device *vdev)  in vfio_bar_restore()  argument
388 	struct pci_dev *pdev = vdev->pdev;  in vfio_bar_restore()
389 	u32 *rbar = vdev->rbar;  in vfio_bar_restore()
427 static void vfio_bar_fixup(struct vfio_pci_device *vdev)  in vfio_bar_fixup()  argument
429 	struct pci_dev *pdev = vdev->pdev;  in vfio_bar_fixup()
434 	bar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0];  in vfio_bar_fixup()
454 	bar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS];  in vfio_bar_fixup()
468 	vdev->bardirty = false;  in vfio_bar_fixup()
471 static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_basic_config_read()  argument
476 		vfio_bar_fixup(vdev);  in vfio_basic_config_read()
478 	count = vfio_default_config_read(vdev, pos, count, perm, offset, val);  in vfio_basic_config_read()
481 	if (offset == PCI_COMMAND && vdev->pdev->is_virtfn) {  in vfio_basic_config_read()
482 		u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);  in vfio_basic_config_read()
492 static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_basic_config_write()  argument
496 	struct pci_dev *pdev = vdev->pdev;  in vfio_basic_config_write()
501 	virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND];  in vfio_basic_config_write()
531 			vfio_bar_restore(vdev);  in vfio_basic_config_write()
534 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_basic_config_write()
556 		if (virt_intx_disable && !vdev->virq_disabled) {  in vfio_basic_config_write()
557 			vdev->virq_disabled = true;  in vfio_basic_config_write()
558 			vfio_pci_intx_mask(vdev);  in vfio_basic_config_write()
559 		} else if (!virt_intx_disable && vdev->virq_disabled) {  in vfio_basic_config_write()
560 			vdev->virq_disabled = false;  in vfio_basic_config_write()
561 			vfio_pci_intx_unmask(vdev);  in vfio_basic_config_write()
566 		vdev->bardirty = true;  in vfio_basic_config_write()
619 static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_pm_config_write()  argument
623 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_pm_config_write()
645 		pci_set_power_state(vdev->pdev, state);  in vfio_pm_config_write()
827 static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)  in vfio_find_cap_start()  argument
832 	cap = vdev->pci_config_map[pos];  in vfio_find_cap_start()
838 	while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap)  in vfio_find_cap_start()
844 static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_msi_config_read()  argument
853 		start = vfio_find_cap_start(vdev, pos);  in vfio_msi_config_read()
855 		flags = (__le16 *)&vdev->vconfig[start];  in vfio_msi_config_read()
858 		*flags |= cpu_to_le16(vdev->msi_qmax << 1);  in vfio_msi_config_read()
861 	return vfio_default_config_read(vdev, pos, count, perm, offset, val);  in vfio_msi_config_read()
864 static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_msi_config_write()  argument
868 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_msi_config_write()
878 		start = vfio_find_cap_start(vdev, pos);  in vfio_msi_config_write()
880 		pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS];  in vfio_msi_config_write()
885 		if  (!is_msi(vdev))  in vfio_msi_config_write()
889 		if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) {  in vfio_msi_config_write()
891 			flags |= vdev->msi_qmax << 4;  in vfio_msi_config_write()
896 		ret = pci_user_write_config_word(vdev->pdev,  in vfio_msi_config_write()
944 static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)  in vfio_msi_cap_len()  argument
946 	struct pci_dev *pdev = vdev->pdev;  in vfio_msi_cap_len()
960 	if (vdev->msi_perm)  in vfio_msi_cap_len()
963 	vdev->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL);  in vfio_msi_cap_len()
964 	if (!vdev->msi_perm)  in vfio_msi_cap_len()
967 	ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags);  in vfio_msi_cap_len()
975 static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos)  in vfio_vc_cap_len()  argument
977 	struct pci_dev *pdev = vdev->pdev;  in vfio_vc_cap_len()
1016 static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)  in vfio_cap_len()  argument
1018 	struct pci_dev *pdev = vdev->pdev;  in vfio_cap_len()
1026 		return vfio_msi_cap_len(vdev, pos);  in vfio_cap_len()
1035 			vdev->extended_caps = (dword != 0);  in vfio_cap_len()
1049 		vdev->extended_caps = (dword != 0);  in vfio_cap_len()
1081 static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos)  in vfio_ext_cap_len()  argument
1083 	struct pci_dev *pdev = vdev->pdev;  in vfio_ext_cap_len()
1098 		return vfio_vc_cap_len(vdev, epos);  in vfio_ext_cap_len()
1155 static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,  in vfio_fill_vconfig_bytes()  argument
1158 	struct pci_dev *pdev = vdev->pdev;  in vfio_fill_vconfig_bytes()
1170 			__le32 *dwordp = (__le32 *)&vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1179 			__le16 *wordp = (__le16 *)&vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1188 			u8 *byte = &vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1202 static int vfio_cap_init(struct vfio_pci_device *vdev)  in vfio_cap_init()  argument
1204 	struct pci_dev *pdev = vdev->pdev;  in vfio_cap_init()
1205 	u8 *map = vdev->pci_config_map;  in vfio_cap_init()
1223 	prev = &vdev->vconfig[PCI_CAPABILITY_LIST];  in vfio_cap_init()
1243 				len = vfio_cap_len(vdev, cap, pos);  in vfio_cap_init()
1268 		ret = vfio_fill_vconfig_bytes(vdev, pos, len);  in vfio_cap_init()
1272 		prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT];  in vfio_cap_init()
1279 		__le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS];  in vfio_cap_init()
1286 static int vfio_ecap_init(struct vfio_pci_device *vdev)  in vfio_ecap_init()  argument
1288 	struct pci_dev *pdev = vdev->pdev;  in vfio_ecap_init()
1289 	u8 *map = vdev->pci_config_map;  in vfio_ecap_init()
1294 	if (!vdev->extended_caps)  in vfio_ecap_init()
1316 				len = vfio_ext_cap_len(vdev, ecap, epos);  in vfio_ecap_init()
1359 		ret = vfio_fill_vconfig_bytes(vdev, epos, len);  in vfio_ecap_init()
1370 			*(__le32 *)&vdev->vconfig[epos] &=  in vfio_ecap_init()
1375 		prev = (__le32 *)&vdev->vconfig[epos];  in vfio_ecap_init()
1380 		*(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0;  in vfio_ecap_init()
1398 int vfio_config_init(struct vfio_pci_device *vdev)  in vfio_config_init()  argument
1400 	struct pci_dev *pdev = vdev->pdev;  in vfio_config_init()
1420 	vdev->pci_config_map = map;  in vfio_config_init()
1421 	vdev->vconfig = vconfig;  in vfio_config_init()
1427 	ret = vfio_fill_vconfig_bytes(vdev, 0, PCI_STD_HEADER_SIZEOF);  in vfio_config_init()
1431 	vdev->bardirty = true;  in vfio_config_init()
1439 	vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]);  in vfio_config_init()
1440 	vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]);  in vfio_config_init()
1441 	vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]);  in vfio_config_init()
1442 	vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]);  in vfio_config_init()
1443 	vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]);  in vfio_config_init()
1444 	vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]);  in vfio_config_init()
1445 	vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]);  in vfio_config_init()
1455 	ret = vfio_cap_init(vdev);  in vfio_config_init()
1459 	ret = vfio_ecap_init(vdev);  in vfio_config_init()
1467 	vdev->pci_config_map = NULL;  in vfio_config_init()
1469 	vdev->vconfig = NULL;  in vfio_config_init()
1473 void vfio_config_free(struct vfio_pci_device *vdev)  in vfio_config_free()  argument
1475 	kfree(vdev->vconfig);  in vfio_config_free()
1476 	vdev->vconfig = NULL;  in vfio_config_free()
1477 	kfree(vdev->pci_config_map);  in vfio_config_free()
1478 	vdev->pci_config_map = NULL;  in vfio_config_free()
1479 	kfree(vdev->msi_perm);  in vfio_config_free()
1480 	vdev->msi_perm = NULL;  in vfio_config_free()
1487 static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,  in vfio_pci_cap_remaining_dword()  argument
1490 	u8 cap = vdev->pci_config_map[pos];  in vfio_pci_cap_remaining_dword()
1493 	for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++)  in vfio_pci_cap_remaining_dword()
1499 static ssize_t vfio_config_do_rw(struct vfio_pci_device *vdev, char __user *buf,  in vfio_config_do_rw()  argument
1502 	struct pci_dev *pdev = vdev->pdev;  in vfio_config_do_rw()
1517 	count = min(count, vfio_pci_cap_remaining_dword(vdev, *ppos));  in vfio_config_do_rw()
1527 	cap_id = vdev->pci_config_map[*ppos];  in vfio_config_do_rw()
1537 			cap_start = vfio_find_cap_start(vdev, *ppos);  in vfio_config_do_rw()
1544 				perm = vdev->msi_perm;  in vfio_config_do_rw()
1547 				cap_start = vfio_find_cap_start(vdev, *ppos);  in vfio_config_do_rw()
1563 		ret = perm->writefn(vdev, *ppos, count, perm, offset, val);  in vfio_config_do_rw()
1566 			ret = perm->readfn(vdev, *ppos, count,  in vfio_config_do_rw()
1579 ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev, char __user *buf,  in vfio_pci_config_rw()  argument
1589 		ret = vfio_config_do_rw(vdev, buf, count, &pos, iswrite);  in vfio_pci_config_rw()