145 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_get_features()  local
148 	vp_iowrite32(0, &vp_dev->common->device_feature_select);  in vp_get_features()
149 	features = vp_ioread32(&vp_dev->common->device_feature);  in vp_get_features()
150 	vp_iowrite32(1, &vp_dev->common->device_feature_select);  in vp_get_features()
151 	features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32);  in vp_get_features()
159 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_finalize_features()  local
170 	vp_iowrite32(0, &vp_dev->common->guest_feature_select);  in vp_finalize_features()
171 	vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature);  in vp_finalize_features()
172 	vp_iowrite32(1, &vp_dev->common->guest_feature_select);  in vp_finalize_features()
173 	vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature);  in vp_finalize_features()
182 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_get()  local
187 	BUG_ON(offset + len > vp_dev->device_len);  in vp_get()
191 		b = ioread8(vp_dev->device + offset);  in vp_get()
195 		w = cpu_to_le16(ioread16(vp_dev->device + offset));  in vp_get()
199 		l = cpu_to_le32(ioread32(vp_dev->device + offset));  in vp_get()
203 		l = cpu_to_le32(ioread32(vp_dev->device + offset));  in vp_get()
205 		l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l));  in vp_get()
218 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_set()  local
223 	BUG_ON(offset + len > vp_dev->device_len);  in vp_set()
228 		iowrite8(b, vp_dev->device + offset);  in vp_set()
232 		iowrite16(le16_to_cpu(w), vp_dev->device + offset);  in vp_set()
236 		iowrite32(le32_to_cpu(l), vp_dev->device + offset);  in vp_set()
240 		iowrite32(le32_to_cpu(l), vp_dev->device + offset);  in vp_set()
242 		iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l);  in vp_set()
251 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_generation()  local
252 	return vp_ioread8(&vp_dev->common->config_generation);  in vp_generation()
258 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_get_status()  local
259 	return vp_ioread8(&vp_dev->common->device_status);  in vp_get_status()
264 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_set_status()  local
267 	vp_iowrite8(status, &vp_dev->common->device_status);  in vp_set_status()
272 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_reset()  local
274 	vp_iowrite8(0, &vp_dev->common->device_status);  in vp_reset()
280 	while (vp_ioread8(&vp_dev->common->device_status))  in vp_reset()
286 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)  in vp_config_vector()  argument
289 	vp_iowrite16(vector, &vp_dev->common->msix_config);  in vp_config_vector()
292 	return vp_ioread16(&vp_dev->common->msix_config);  in vp_config_vector()
320 static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,  in setup_vq()  argument
327 	struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common;  in setup_vq()
344 		dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);  in setup_vq()
360 				 SMP_CACHE_BYTES, &vp_dev->vdev,  in setup_vq()
376 	if (vp_dev->notify_base) {  in setup_vq()
378 		if ((u64)off * vp_dev->notify_offset_multiplier + 2  in setup_vq()
379 		    > vp_dev->notify_len) {  in setup_vq()
380 			dev_warn(&vp_dev->pci_dev->dev,  in setup_vq()
383 				 off, vp_dev->notify_offset_multiplier,  in setup_vq()
384 				 index, vp_dev->notify_len);  in setup_vq()
388 		vq->priv = (void __force *)vp_dev->notify_base +  in setup_vq()
389 			off * vp_dev->notify_offset_multiplier;  in setup_vq()
391 		vq->priv = (void __force *)map_capability(vp_dev->pci_dev,  in setup_vq()
392 					  vp_dev->notify_map_cap, 2, 2,  in setup_vq()
393 					  off * vp_dev->notify_offset_multiplier, 2,  in setup_vq()
414 	if (!vp_dev->notify_base)  in setup_vq()
415 		pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv);  in setup_vq()
428 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);  in vp_modern_find_vqs()  local
439 		vp_iowrite16(vq->index, &vp_dev->common->queue_select);  in vp_modern_find_vqs()
440 		vp_iowrite16(1, &vp_dev->common->queue_enable);  in vp_modern_find_vqs()
449 	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);  in del_vq()  local
451 	vp_iowrite16(vq->index, &vp_dev->common->queue_select);  in del_vq()
453 	if (vp_dev->msix_enabled) {  in del_vq()
455 			     &vp_dev->common->queue_msix_vector);  in del_vq()
457 		vp_ioread16(&vp_dev->common->queue_msix_vector);  in del_vq()
460 	if (!vp_dev->notify_base)  in del_vq()
461 		pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv);  in del_vq()
599 int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev)  in virtio_pci_modern_probe()  argument
601 	struct pci_dev *pci_dev = vp_dev->pci_dev;  in virtio_pci_modern_probe()
616 		vp_dev->vdev.id.device = pci_dev->subsystem_device;  in virtio_pci_modern_probe()
619 		vp_dev->vdev.id.device = pci_dev->device - 0x1040;  in virtio_pci_modern_probe()
621 	vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;  in virtio_pci_modern_probe()
651 	vp_dev->common = map_capability(pci_dev, common,  in virtio_pci_modern_probe()
655 	if (!vp_dev->common)  in virtio_pci_modern_probe()
657 	vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1,  in virtio_pci_modern_probe()
660 	if (!vp_dev->isr)  in virtio_pci_modern_probe()
667 			      &vp_dev->notify_offset_multiplier);  in virtio_pci_modern_probe()
684 		vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2,  in virtio_pci_modern_probe()
686 						     &vp_dev->notify_len);  in virtio_pci_modern_probe()
687 		if (!vp_dev->notify_base)  in virtio_pci_modern_probe()
690 		vp_dev->notify_map_cap = notify;  in virtio_pci_modern_probe()
697 		vp_dev->device = map_capability(pci_dev, device, 0, 4,  in virtio_pci_modern_probe()
699 						&vp_dev->device_len);  in virtio_pci_modern_probe()
700 		if (!vp_dev->device)  in virtio_pci_modern_probe()
703 		vp_dev->vdev.config = &virtio_pci_config_ops;  in virtio_pci_modern_probe()
705 		vp_dev->vdev.config = &virtio_pci_config_nodev_ops;  in virtio_pci_modern_probe()
708 	vp_dev->config_vector = vp_config_vector;  in virtio_pci_modern_probe()
709 	vp_dev->setup_vq = setup_vq;  in virtio_pci_modern_probe()
710 	vp_dev->del_vq = del_vq;  in virtio_pci_modern_probe()
715 	if (vp_dev->notify_base)  in virtio_pci_modern_probe()
716 		pci_iounmap(pci_dev, vp_dev->notify_base);  in virtio_pci_modern_probe()
718 	pci_iounmap(pci_dev, vp_dev->isr);  in virtio_pci_modern_probe()
720 	pci_iounmap(pci_dev, vp_dev->common);  in virtio_pci_modern_probe()
725 void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev)  in virtio_pci_modern_remove()  argument
727 	struct pci_dev *pci_dev = vp_dev->pci_dev;  in virtio_pci_modern_remove()
729 	if (vp_dev->device)  in virtio_pci_modern_remove()
730 		pci_iounmap(pci_dev, vp_dev->device);  in virtio_pci_modern_remove()
731 	if (vp_dev->notify_base)  in virtio_pci_modern_remove()
732 		pci_iounmap(pci_dev, vp_dev->notify_base);  in virtio_pci_modern_remove()
733 	pci_iounmap(pci_dev, vp_dev->isr);  in virtio_pci_modern_remove()
734 	pci_iounmap(pci_dev, vp_dev->common);  in virtio_pci_modern_remove()