30 #define BAD_RING(_vq, fmt, args...)				\  argument
32 		dev_err(&(_vq)->vq.vdev->dev,			\
33 			"%s:"fmt, (_vq)->vq.name, ##args);	\
37 #define START_USE(_vq)						\  argument
39 		if ((_vq)->in_use)				\
41 			      (_vq)->vq.name, (_vq)->in_use);	\
42 		(_vq)->in_use = __LINE__;			\
44 #define END_USE(_vq) \  argument
45 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
47 #define BAD_RING(_vq, fmt, args...)				\  argument
49 		dev_err(&_vq->vq.vdev->dev,			\
50 			"%s:"fmt, (_vq)->vq.name, ##args);	\
51 		(_vq)->broken = true;				\
99 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)  argument
101 static struct vring_desc *alloc_indirect(struct virtqueue *_vq,  in alloc_indirect()  argument
119 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect()
123 static inline int virtqueue_add(struct virtqueue *_vq,  in virtqueue_add()  argument
131 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()
168 		desc = alloc_indirect(_vq, total_sg, gfp);  in virtqueue_add()
174 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
175 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc));  in virtqueue_add()
178 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
208 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);  in virtqueue_add()
209 			desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));  in virtqueue_add()
210 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
212 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
217 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);  in virtqueue_add()
218 			desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));  in virtqueue_add()
219 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
221 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
225 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add()
229 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
238 	avail = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) & (vq->vring.num - 1);  in virtqueue_add()
239 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
244 …vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx)…  in virtqueue_add()
253 		virtqueue_kick(_vq);  in virtqueue_add()
272 int virtqueue_add_sgs(struct virtqueue *_vq,  in virtqueue_add_sgs()  argument
287 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs, data, gfp);  in virtqueue_add_sgs()
346 bool virtqueue_kick_prepare(struct virtqueue *_vq)  in virtqueue_kick_prepare()  argument
348 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()
357 	old = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - vq->num_added;  in virtqueue_kick_prepare()
358 	new = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx);  in virtqueue_kick_prepare()
370 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
373 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
388 bool virtqueue_notify(struct virtqueue *_vq)  in virtqueue_notify()  argument
390 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()
396 	if (!vq->notify(_vq)) {  in virtqueue_notify()
470 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)  in virtqueue_get_buf()  argument
472 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf()
494 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf()
495 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf()
513 	if (!(vq->vring.avail->flags & cpu_to_virtio16(_vq->vdev, VRING_AVAIL_F_NO_INTERRUPT))) {  in virtqueue_get_buf()
514 		vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);  in virtqueue_get_buf()
536 void virtqueue_disable_cb(struct virtqueue *_vq)  in virtqueue_disable_cb()  argument
538 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()
540 	vq->vring.avail->flags |= cpu_to_virtio16(_vq->vdev, VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_disable_cb()
556 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare()  argument
558 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()
568 	vq->vring.avail->flags &= cpu_to_virtio16(_vq->vdev, ~VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_enable_cb_prepare()
569 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
584 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)  in virtqueue_poll()  argument
586 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()
589 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
604 bool virtqueue_enable_cb(struct virtqueue *_vq)  in virtqueue_enable_cb()  argument
606 	unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);  in virtqueue_enable_cb()
607 	return !virtqueue_poll(_vq, last_used_idx);  in virtqueue_enable_cb()
624 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed()  argument
626 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()
636 	vq->vring.avail->flags &= cpu_to_virtio16(_vq->vdev, ~VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_enable_cb_delayed()
638 	bufs = (u16)(virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - vq->last_used_idx) * 3 / 4;  in virtqueue_enable_cb_delayed()
639 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);  in virtqueue_enable_cb_delayed()
641 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
659 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)  in virtqueue_detach_unused_buf()  argument
661 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()
673 …vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx)…  in virtqueue_detach_unused_buf()
685 irqreturn_t vring_interrupt(int irq, void *_vq)  in vring_interrupt()  argument
687 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()
799 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)  in virtqueue_get_vring_size()  argument
802 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()
808 bool virtqueue_is_broken(struct virtqueue *_vq)  in virtqueue_is_broken()  argument
810 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()
822 	struct virtqueue *_vq;  in virtio_break_device()  local
824 	list_for_each_entry(_vq, &dev->vqs, list) {  in virtio_break_device()
825 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()
831 void *virtqueue_get_avail(struct virtqueue *_vq)  in virtqueue_get_avail()  argument
833 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail()
839 void *virtqueue_get_used(struct virtqueue *_vq)  in virtqueue_get_used()  argument
841 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used()