30 #define BAD_RING(_vq, fmt, args...)				\  argument
32 		dev_err(&(_vq)->vq.vdev->dev,			\
33 			"%s:"fmt, (_vq)->vq.name, ##args);	\
37 #define START_USE(_vq)						\  argument
39 		if ((_vq)->in_use)				\
41 			      (_vq)->vq.name, (_vq)->in_use);	\
42 		(_vq)->in_use = __LINE__;			\
44 #define END_USE(_vq) \  argument
45 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
47 #define BAD_RING(_vq, fmt, args...)				\  argument
49 		dev_err(&_vq->vq.vdev->dev,			\
50 			"%s:"fmt, (_vq)->vq.name, ##args);	\
51 		(_vq)->broken = true;				\
105 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)  argument
107 static struct vring_desc *alloc_indirect(struct virtqueue *_vq,  in alloc_indirect()  argument
125 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect()
129 static inline int virtqueue_add(struct virtqueue *_vq,  in virtqueue_add()  argument
137 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()
174 		desc = alloc_indirect(_vq, total_sg, gfp);  in virtqueue_add()
180 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
181 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc));  in virtqueue_add()
184 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
214 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);  in virtqueue_add()
215 			desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));  in virtqueue_add()
216 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
218 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
223 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);  in virtqueue_add()
224 			desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));  in virtqueue_add()
225 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
227 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
231 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add()
235 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
245 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
251 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_add()
260 		virtqueue_kick(_vq);  in virtqueue_add()
279 int virtqueue_add_sgs(struct virtqueue *_vq,  in virtqueue_add_sgs()  argument
294 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs, data, gfp);  in virtqueue_add_sgs()
353 bool virtqueue_kick_prepare(struct virtqueue *_vq)  in virtqueue_kick_prepare()  argument
355 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()
377 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
380 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
395 bool virtqueue_notify(struct virtqueue *_vq)  in virtqueue_notify()  argument
397 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()
403 	if (!vq->notify(_vq)) {  in virtqueue_notify()
477 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)  in virtqueue_get_buf()  argument
479 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf()
501 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf()
502 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf()
521 		vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);  in virtqueue_get_buf()
543 void virtqueue_disable_cb(struct virtqueue *_vq)  in virtqueue_disable_cb()  argument
545 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()
549 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_disable_cb()
567 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare()  argument
569 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()
581 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_prepare()
583 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
598 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)  in virtqueue_poll()  argument
600 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()
603 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
618 bool virtqueue_enable_cb(struct virtqueue *_vq)  in virtqueue_enable_cb()  argument
620 	unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);  in virtqueue_enable_cb()
621 	return !virtqueue_poll(_vq, last_used_idx);  in virtqueue_enable_cb()
638 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed()  argument
640 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()
652 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_delayed()
656 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);  in virtqueue_enable_cb_delayed()
658 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
676 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)  in virtqueue_detach_unused_buf()  argument
678 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()
691 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_detach_unused_buf()
703 irqreturn_t vring_interrupt(int irq, void *_vq)  in vring_interrupt()  argument
705 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()
821 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)  in virtqueue_get_vring_size()  argument
824 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()
830 bool virtqueue_is_broken(struct virtqueue *_vq)  in virtqueue_is_broken()  argument
832 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()
844 	struct virtqueue *_vq;  in virtio_break_device()  local
846 	list_for_each_entry(_vq, &dev->vqs, list) {  in virtio_break_device()
847 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()
853 void *virtqueue_get_avail(struct virtqueue *_vq)  in virtqueue_get_avail()  argument
855 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail()
861 void *virtqueue_get_used(struct virtqueue *_vq)  in virtqueue_get_used()  argument
863 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used()