32 		dev_err(&(_vq)->vq.vdev->dev,			\
49 		dev_err(&_vq->vq.vdev->dev,			\
125 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect()
180 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
181 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc));  in virtqueue_add()
184 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
214 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);  in virtqueue_add()
215 			desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));  in virtqueue_add()
216 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
218 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
223 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);  in virtqueue_add()
224 			desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));  in virtqueue_add()
225 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);  in virtqueue_add()
227 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add()
231 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add()
235 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
245 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
251 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_add()
377 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
380 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
442 	if (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))  in detach_buf()
443 		kfree(phys_to_virt(virtio64_to_cpu(vq->vq.vdev, vq->vring.desc[i].addr)));  in detach_buf()
445 	while (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT)) {  in detach_buf()
446 		i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);  in detach_buf()
450 	vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);  in detach_buf()
458 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);  in more_used()
501 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf()
502 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf()
521 		vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);  in virtqueue_get_buf()
549 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_disable_cb()
581 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_prepare()
583 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
603 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
652 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_delayed()
656 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);  in virtqueue_enable_cb_delayed()
658 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
691 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_detach_unused_buf()
726 				      struct virtio_device *vdev,  in vring_new_virtqueue()  argument
738 		dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);  in vring_new_virtqueue()
748 	vq->vq.vdev = vdev;  in vring_new_virtqueue()
759 	list_add_tail(&vq->vq.list, &vdev->vqs);  in vring_new_virtqueue()
765 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);  in vring_new_virtqueue()
766 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);  in vring_new_virtqueue()
771 		vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);  in vring_new_virtqueue()
777 		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);  in vring_new_virtqueue()
794 void vring_transport_features(struct virtio_device *vdev)  in vring_transport_features()  argument
808 			__virtio_clear_bit(vdev, i);  in vring_transport_features()