32 		dev_err(&(_vq)->vq.vdev->dev,			\
33 			"%s:"fmt, (_vq)->vq.name, ##args);	\
41 			      (_vq)->vq.name, (_vq)->in_use);	\
49 		dev_err(&_vq->vq.vdev->dev,			\
50 			"%s:"fmt, (_vq)->vq.name, ##args);	\
53 #define START_USE(vq)  argument
54 #define END_USE(vq)  argument
58 	struct virtqueue vq;  member
84 	bool (*notify)(struct virtqueue *vq);
99 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
131 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()  local
138 	START_USE(vq);  in virtqueue_add()
142 	if (unlikely(vq->broken)) {  in virtqueue_add()
143 		END_USE(vq);  in virtqueue_add()
152 		if (vq->last_add_time_valid)  in virtqueue_add()
153 			WARN_ON(ktime_to_ms(ktime_sub(now, vq->last_add_time))  in virtqueue_add()
155 		vq->last_add_time = now;  in virtqueue_add()
156 		vq->last_add_time_valid = true;  in virtqueue_add()
160 	BUG_ON(total_sg > vq->vring.num);  in virtqueue_add()
163 	head = vq->free_head;  in virtqueue_add()
167 	if (vq->indirect && total_sg > 1 && vq->vq.num_free)  in virtqueue_add()
174 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
175 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc));  in virtqueue_add()
178 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
185 		desc = vq->vring.desc;  in virtqueue_add()
191 	if (vq->vq.num_free < descs_used) {  in virtqueue_add()
193 			 descs_used, vq->vq.num_free);  in virtqueue_add()
198 			vq->notify(&vq->vq);  in virtqueue_add()
199 		END_USE(vq);  in virtqueue_add()
204 	vq->vq.num_free -= descs_used;  in virtqueue_add()
229 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
231 		vq->free_head = i;  in virtqueue_add()
234 	vq->data[head] = data;  in virtqueue_add()
238 	avail = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) & (vq->vring.num - 1);  in virtqueue_add()
239 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
243 	virtio_wmb(vq->weak_barriers);  in virtqueue_add()
244 …vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx)…  in virtqueue_add()
245 	vq->num_added++;  in virtqueue_add()
247 	pr_debug("Added buffer head %i to %p\n", head, vq);  in virtqueue_add()
248 	END_USE(vq);  in virtqueue_add()
252 	if (unlikely(vq->num_added == (1 << 16) - 1))  in virtqueue_add()
304 int virtqueue_add_outbuf(struct virtqueue *vq,  in virtqueue_add_outbuf()  argument
309 	return virtqueue_add(vq, &sg, num, 1, 0, data, gfp);  in virtqueue_add_outbuf()
326 int virtqueue_add_inbuf(struct virtqueue *vq,  in virtqueue_add_inbuf()  argument
331 	return virtqueue_add(vq, &sg, num, 0, 1, data, gfp);  in virtqueue_add_inbuf()
348 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()  local
352 	START_USE(vq);  in virtqueue_kick_prepare()
355 	virtio_mb(vq->weak_barriers);  in virtqueue_kick_prepare()
357 	old = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - vq->num_added;  in virtqueue_kick_prepare()
358 	new = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx);  in virtqueue_kick_prepare()
359 	vq->num_added = 0;  in virtqueue_kick_prepare()
362 	if (vq->last_add_time_valid) {  in virtqueue_kick_prepare()
364 					      vq->last_add_time)) > 100);  in virtqueue_kick_prepare()
366 	vq->last_add_time_valid = false;  in virtqueue_kick_prepare()
369 	if (vq->event) {  in virtqueue_kick_prepare()
370 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
373 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
375 	END_USE(vq);  in virtqueue_kick_prepare()
390 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()  local
392 	if (unlikely(vq->broken))  in virtqueue_notify()
396 	if (!vq->notify(_vq)) {  in virtqueue_notify()
397 		vq->broken = true;  in virtqueue_notify()
416 bool virtqueue_kick(struct virtqueue *vq)  in virtqueue_kick()  argument
418 	if (virtqueue_kick_prepare(vq))  in virtqueue_kick()
419 		return virtqueue_notify(vq);  in virtqueue_kick()
424 static void detach_buf(struct vring_virtqueue *vq, unsigned int head)  in detach_buf()  argument
429 	vq->data[head] = NULL;  in detach_buf()
435 	if (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))  in detach_buf()
436 		kfree(phys_to_virt(virtio64_to_cpu(vq->vq.vdev, vq->vring.desc[i].addr)));  in detach_buf()
438 	while (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT)) {  in detach_buf()
439 		i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);  in detach_buf()
440 		vq->vq.num_free++;  in detach_buf()
443 	vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);  in detach_buf()
444 	vq->free_head = head;  in detach_buf()
446 	vq->vq.num_free++;  in detach_buf()
449 static inline bool more_used(const struct vring_virtqueue *vq)  in more_used()  argument
451 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);  in more_used()
472 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf()  local
477 	START_USE(vq);  in virtqueue_get_buf()
479 	if (unlikely(vq->broken)) {  in virtqueue_get_buf()
480 		END_USE(vq);  in virtqueue_get_buf()
484 	if (!more_used(vq)) {  in virtqueue_get_buf()
486 		END_USE(vq);  in virtqueue_get_buf()
491 	virtio_rmb(vq->weak_barriers);  in virtqueue_get_buf()
493 	last_used = (vq->last_used_idx & (vq->vring.num - 1));  in virtqueue_get_buf()
494 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf()
495 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf()
497 	if (unlikely(i >= vq->vring.num)) {  in virtqueue_get_buf()
498 		BAD_RING(vq, "id %u out of range\n", i);  in virtqueue_get_buf()
501 	if (unlikely(!vq->data[i])) {  in virtqueue_get_buf()
502 		BAD_RING(vq, "id %u is not a head!\n", i);  in virtqueue_get_buf()
507 	ret = vq->data[i];  in virtqueue_get_buf()
508 	detach_buf(vq, i);  in virtqueue_get_buf()
509 	vq->last_used_idx++;  in virtqueue_get_buf()
513 	if (!(vq->vring.avail->flags & cpu_to_virtio16(_vq->vdev, VRING_AVAIL_F_NO_INTERRUPT))) {  in virtqueue_get_buf()
514 		vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);  in virtqueue_get_buf()
515 		virtio_mb(vq->weak_barriers);  in virtqueue_get_buf()
519 	vq->last_add_time_valid = false;  in virtqueue_get_buf()
522 	END_USE(vq);  in virtqueue_get_buf()
538 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()  local
540 	vq->vring.avail->flags |= cpu_to_virtio16(_vq->vdev, VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_disable_cb()
558 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()  local
561 	START_USE(vq);  in virtqueue_enable_cb_prepare()
568 	vq->vring.avail->flags &= cpu_to_virtio16(_vq->vdev, ~VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_enable_cb_prepare()
569 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
570 	END_USE(vq);  in virtqueue_enable_cb_prepare()
586 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()  local
588 	virtio_mb(vq->weak_barriers);  in virtqueue_poll()
589 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
626 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()  local
629 	START_USE(vq);  in virtqueue_enable_cb_delayed()
636 	vq->vring.avail->flags &= cpu_to_virtio16(_vq->vdev, ~VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_enable_cb_delayed()
638 	bufs = (u16)(virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - vq->last_used_idx) * 3 / 4;  in virtqueue_enable_cb_delayed()
639 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);  in virtqueue_enable_cb_delayed()
640 	virtio_mb(vq->weak_barriers);  in virtqueue_enable_cb_delayed()
641 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
642 		END_USE(vq);  in virtqueue_enable_cb_delayed()
646 	END_USE(vq);  in virtqueue_enable_cb_delayed()
661 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()  local
665 	START_USE(vq);  in virtqueue_detach_unused_buf()
667 	for (i = 0; i < vq->vring.num; i++) {  in virtqueue_detach_unused_buf()
668 		if (!vq->data[i])  in virtqueue_detach_unused_buf()
671 		buf = vq->data[i];  in virtqueue_detach_unused_buf()
672 		detach_buf(vq, i);  in virtqueue_detach_unused_buf()
673 …vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx)…  in virtqueue_detach_unused_buf()
674 		END_USE(vq);  in virtqueue_detach_unused_buf()
678 	BUG_ON(vq->vq.num_free != vq->vring.num);  in virtqueue_detach_unused_buf()
680 	END_USE(vq);  in virtqueue_detach_unused_buf()
687 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()  local
689 	if (!more_used(vq)) {  in vring_interrupt()
690 		pr_debug("virtqueue interrupt with no work for %p\n", vq);  in vring_interrupt()
694 	if (unlikely(vq->broken))  in vring_interrupt()
697 	pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);  in vring_interrupt()
698 	if (vq->vq.callback)  in vring_interrupt()
699 		vq->vq.callback(&vq->vq);  in vring_interrupt()
715 	struct vring_virtqueue *vq;  in vring_new_virtqueue()  local
724 	vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);  in vring_new_virtqueue()
725 	if (!vq)  in vring_new_virtqueue()
728 	vring_init(&vq->vring, num, pages, vring_align);  in vring_new_virtqueue()
729 	vq->vq.callback = callback;  in vring_new_virtqueue()
730 	vq->vq.vdev = vdev;  in vring_new_virtqueue()
731 	vq->vq.name = name;  in vring_new_virtqueue()
732 	vq->vq.num_free = num;  in vring_new_virtqueue()
733 	vq->vq.index = index;  in vring_new_virtqueue()
734 	vq->notify = notify;  in vring_new_virtqueue()
735 	vq->weak_barriers = weak_barriers;  in vring_new_virtqueue()
736 	vq->broken = false;  in vring_new_virtqueue()
737 	vq->last_used_idx = 0;  in vring_new_virtqueue()
738 	vq->num_added = 0;  in vring_new_virtqueue()
739 	list_add_tail(&vq->vq.list, &vdev->vqs);  in vring_new_virtqueue()
741 	vq->in_use = false;  in vring_new_virtqueue()
742 	vq->last_add_time_valid = false;  in vring_new_virtqueue()
745 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);  in vring_new_virtqueue()
746 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);  in vring_new_virtqueue()
750 		vq->vring.avail->flags |= cpu_to_virtio16(vdev, VRING_AVAIL_F_NO_INTERRUPT);  in vring_new_virtqueue()
753 	vq->free_head = 0;  in vring_new_virtqueue()
755 		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);  in vring_new_virtqueue()
756 		vq->data[i] = NULL;  in vring_new_virtqueue()
758 	vq->data[i] = NULL;  in vring_new_virtqueue()
760 	return &vq->vq;  in vring_new_virtqueue()
764 void vring_del_virtqueue(struct virtqueue *vq)  in vring_del_virtqueue()  argument
766 	list_del(&vq->list);  in vring_del_virtqueue()
767 	kfree(to_vvq(vq));  in vring_del_virtqueue()
802 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()  local
804 	return vq->vring.num;  in virtqueue_get_vring_size()
810 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()  local
812 	return vq->broken;  in virtqueue_is_broken()
825 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()  local
826 		vq->broken = true;  in virtio_break_device()
833 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail()  local
835 	return vq->vring.avail;  in virtqueue_get_avail()
841 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used()  local
843 	return vq->vring.used;  in virtqueue_get_used()