173 	struct virtqueue *vq;  member
227 	void (*service)(struct virtqueue *vq);
247 #define lg_last_avail(vq)	((vq)->last_avail_idx)  argument
266 #define bad_driver_vq(vq, fmt, ...)			       \  argument
267 	errx(1, "%s vq %s: bad driver: " fmt, (vq)->dev->name, \
268 	     vq->name, ## __VA_ARGS__)
686 static void trigger_irq(struct virtqueue *vq)  in trigger_irq()  argument
688 	unsigned long buf[] = { LHREQ_IRQ, vq->dev->config.irq_line };  in trigger_irq()
691 	if (!vq->pending_used)  in trigger_irq()
693 	vq->pending_used = 0;  in trigger_irq()
701 	if (vq->vring.avail->flags > 1)  in trigger_irq()
702 		bad_driver_vq(vq, "avail->flags = %u\n", vq->vring.avail->flags);  in trigger_irq()
714 	if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {  in trigger_irq()
725 	vq->dev->mmio->isr = 0x1;  in trigger_irq()
729 		err(1, "Triggering irq %i", vq->dev->config.irq_line);  in trigger_irq()
740 static unsigned wait_for_vq_desc(struct virtqueue *vq,  in wait_for_vq_desc()  argument
746 	u16 last_avail = lg_last_avail(vq);  in wait_for_vq_desc()
757 	while (last_avail == vq->vring.avail->idx) {  in wait_for_vq_desc()
764 		trigger_irq(vq);  in wait_for_vq_desc()
767 		vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
774 		if (last_avail != vq->vring.avail->idx) {  in wait_for_vq_desc()
775 			vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
780 		if (read(vq->eventfd, &event, sizeof(event)) != sizeof(event))  in wait_for_vq_desc()
784 		vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
788 	if ((u16)(vq->vring.avail->idx - last_avail) > vq->vring.num)  in wait_for_vq_desc()
789 		bad_driver_vq(vq, "Guest moved used index from %u to %u",  in wait_for_vq_desc()
790 			      last_avail, vq->vring.avail->idx);  in wait_for_vq_desc()
802 	head = vq->vring.avail->ring[last_avail % vq->vring.num];  in wait_for_vq_desc()
803 	lg_last_avail(vq)++;  in wait_for_vq_desc()
806 	if (head >= vq->vring.num)  in wait_for_vq_desc()
807 		bad_driver_vq(vq, "Guest says index %u is available", head);  in wait_for_vq_desc()
812 	max = vq->vring.num;  in wait_for_vq_desc()
813 	desc = vq->vring.desc;  in wait_for_vq_desc()
835 			if (!(vq->dev->features_accepted &  in wait_for_vq_desc()
837 				bad_driver_vq(vq, "vq indirect not negotiated");  in wait_for_vq_desc()
846 			if (desc != vq->vring.desc)  in wait_for_vq_desc()
847 				bad_driver_vq(vq, "Indirect within indirect");  in wait_for_vq_desc()
856 				bad_driver_vq(vq, "indirect and next together");  in wait_for_vq_desc()
859 				bad_driver_vq(vq,  in wait_for_vq_desc()
872 			desc = check_pointer(vq->dev, desc[i].addr, desc[i].len);  in wait_for_vq_desc()
880 			if (max > vq->pci_config.queue_size)  in wait_for_vq_desc()
881 				bad_driver_vq(vq,  in wait_for_vq_desc()
888 			= check_pointer(vq->dev, desc[i].addr, desc[i].len);  in wait_for_vq_desc()
898 				bad_driver_vq(vq,  in wait_for_vq_desc()
905 			bad_driver_vq(vq, "Looped descriptor");  in wait_for_vq_desc()
906 	} while ((i = next_desc(vq->dev, desc, i, max)) != max);  in wait_for_vq_desc()
916 static void add_used(struct virtqueue *vq, unsigned int head, int len)  in add_used()  argument
924 	used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];  in add_used()
929 	vq->vring.used->idx++;  in add_used()
930 	vq->pending_used++;  in add_used()
934 static void add_used_and_trigger(struct virtqueue *vq, unsigned head, int len)  in add_used_and_trigger()  argument
936 	add_used(vq, head, len);  in add_used_and_trigger()
937 	trigger_irq(vq);  in add_used_and_trigger()
953 static void console_input(struct virtqueue *vq)  in console_input()  argument
957 	struct console_abort *abort = vq->dev->priv;  in console_input()
958 	struct iovec iov[vq->vring.num];  in console_input()
961 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in console_input()
963 		bad_driver_vq(vq, "Output buffers in console in queue?");  in console_input()
979 	add_used_and_trigger(vq, head, len);  in console_input()
1008 static void console_output(struct virtqueue *vq)  in console_output()  argument
1011 	struct iovec iov[vq->vring.num];  in console_output()
1014 	head = wait_for_vq_desc(vq, iov, &out, &in);  in console_output()
1016 		bad_driver_vq(vq, "Input buffers in console output queue?");  in console_output()
1025 		iov_consume(vq->dev, iov, out, NULL, len);  in console_output()
1032 	add_used(vq, head, 0);  in console_output()
1045 static void net_output(struct virtqueue *vq)  in net_output()  argument
1047 	struct net_info *net_info = vq->dev->priv;  in net_output()
1049 	struct iovec iov[vq->vring.num];  in net_output()
1052 	head = wait_for_vq_desc(vq, iov, &out, &in);  in net_output()
1054 		bad_driver_vq(vq, "Input buffers in net output queue?");  in net_output()
1066 	add_used(vq, head, 0);  in net_output()
1089 static void net_input(struct virtqueue *vq)  in net_input()  argument
1093 	struct iovec iov[vq->vring.num];  in net_input()
1094 	struct net_info *net_info = vq->dev->priv;  in net_input()
1100 	head = wait_for_vq_desc(vq, iov, &out, &in);  in net_input()
1102 		bad_driver_vq(vq, "Output buffers in net input queue?");  in net_input()
1108 	if (vq->pending_used && will_block(net_info->tunfd))  in net_input()
1109 		trigger_irq(vq);  in net_input()
1123 	add_used(vq, head, len);  in net_input()
1130 	struct virtqueue *vq = _vq;  in do_thread()  local
1133 		vq->service(vq);  in do_thread()
1146 static void reset_vq_pci_config(struct virtqueue *vq)  in reset_vq_pci_config()  argument
1148 	vq->pci_config.queue_size = VIRTQUEUE_NUM;  in reset_vq_pci_config()
1149 	vq->pci_config.queue_enable = 0;  in reset_vq_pci_config()
1154 	struct virtqueue *vq;  in reset_device()  local
1174 	for (vq = dev->vq; vq; vq = vq->next) {  in reset_device()
1175 		vq->last_avail_idx = 0;  in reset_device()
1176 		reset_vq_pci_config(vq);  in reset_device()
1177 		if (vq->thread != (pid_t)-1) {  in reset_device()
1178 			kill(vq->thread, SIGTERM);  in reset_device()
1179 			waitpid(vq->thread, NULL, 0);  in reset_device()
1180 			vq->thread = (pid_t)-1;  in reset_device()
1767 	struct virtqueue *vq = d->vq;  in vq_by_num()  local
1769 	while (num-- && vq)  in vq_by_num()
1770 		vq = vq->next;  in vq_by_num()
1772 	return vq;  in vq_by_num()
1776 			   struct virtqueue *vq)  in save_vq_config()  argument
1778 	vq->pci_config = *cfg;  in save_vq_config()
1782 			      struct virtqueue *vq)  in restore_vq_config()  argument
1787 	memcpy((void *)cfg + off, (void *)&vq->pci_config + off,  in restore_vq_config()
1799 static void check_virtqueue(struct device *d, struct virtqueue *vq)  in check_virtqueue()  argument
1802 	if (vq->pci_config.queue_desc_hi  in check_virtqueue()
1803 	    || vq->pci_config.queue_avail_hi  in check_virtqueue()
1804 	    || vq->pci_config.queue_used_hi)  in check_virtqueue()
1805 		bad_driver_vq(vq, "invalid 64-bit queue address");  in check_virtqueue()
1814 	if (vq->pci_config.queue_desc_lo % 16  in check_virtqueue()
1815 	    || vq->pci_config.queue_avail_lo % 2  in check_virtqueue()
1816 	    || vq->pci_config.queue_used_lo % 4)  in check_virtqueue()
1817 		bad_driver_vq(vq, "invalid alignment in queue addresses");  in check_virtqueue()
1820 	vq->vring.num = vq->pci_config.queue_size;  in check_virtqueue()
1821 	vq->vring.desc = check_pointer(vq->dev,  in check_virtqueue()
1822 				       vq->pci_config.queue_desc_lo,  in check_virtqueue()
1823 				       sizeof(*vq->vring.desc) * vq->vring.num);  in check_virtqueue()
1824 	vq->vring.avail = check_pointer(vq->dev,  in check_virtqueue()
1825 					vq->pci_config.queue_avail_lo,  in check_virtqueue()
1826 					sizeof(*vq->vring.avail)  in check_virtqueue()
1827 					+ (sizeof(vq->vring.avail->ring[0])  in check_virtqueue()
1828 					   * vq->vring.num));  in check_virtqueue()
1829 	vq->vring.used = check_pointer(vq->dev,  in check_virtqueue()
1830 				       vq->pci_config.queue_used_lo,  in check_virtqueue()
1831 				       sizeof(*vq->vring.used)  in check_virtqueue()
1832 				       + (sizeof(vq->vring.used->ring[0])  in check_virtqueue()
1833 					  * vq->vring.num));  in check_virtqueue()
1841 	if (vq->vring.used->flags != 0)  in check_virtqueue()
1842 		bad_driver_vq(vq, "invalid initial used.flags %#x",  in check_virtqueue()
1843 			      vq->vring.used->flags);  in check_virtqueue()
1846 static void start_virtqueue(struct virtqueue *vq)  in start_virtqueue()  argument
1855 	vq->eventfd = eventfd(0, 0);  in start_virtqueue()
1856 	if (vq->eventfd < 0)  in start_virtqueue()
1863 	vq->thread = clone(do_thread, stack + 32768, CLONE_VM | SIGCHLD, vq);  in start_virtqueue()
1864 	if (vq->thread == (pid_t)-1)  in start_virtqueue()
1870 	struct virtqueue *vq;  in start_virtqueues()  local
1872 	for (vq = d->vq; vq; vq = vq->next) {  in start_virtqueues()
1873 		if (vq->pci_config.queue_enable)  in start_virtqueues()
1874 			start_virtqueue(vq);  in start_virtqueues()
1880 	struct virtqueue *vq;  in emulate_mmio_write()  local
2012 		vq = vq_by_num(d, val);  in emulate_mmio_write()
2019 		if (!vq) {  in emulate_mmio_write()
2028 		restore_vq_config(&d->mmio->cfg, vq);  in emulate_mmio_write()
2045 		struct virtqueue *vq = vq_by_num(d, d->mmio->cfg.queue_select);  in emulate_mmio_write()  local
2071 		save_vq_config(&d->mmio->cfg, vq);  in emulate_mmio_write()
2072 		check_virtqueue(d, vq);  in emulate_mmio_write()
2112 		vq = vq_by_num(d, val);  in emulate_mmio_write()
2113 		if (!vq)  in emulate_mmio_write()
2116 		write(vq->eventfd, "\1\0\0\0\0\0\0\0", 8);  in emulate_mmio_write()
2384 	struct virtqueue **i, *vq = malloc(sizeof(*vq));  in add_pci_virtqueue()  local
2387 	vq->next = NULL;  in add_pci_virtqueue()
2388 	vq->last_avail_idx = 0;  in add_pci_virtqueue()
2389 	vq->dev = dev;  in add_pci_virtqueue()
2390 	vq->name = name;  in add_pci_virtqueue()
2396 	vq->service = service;  in add_pci_virtqueue()
2397 	vq->thread = (pid_t)-1;  in add_pci_virtqueue()
2400 	reset_vq_pci_config(vq);  in add_pci_virtqueue()
2401 	vq->pci_config.queue_notify_off = 0;  in add_pci_virtqueue()
2404 	vq->dev->mmio->cfg.num_queues++;  in add_pci_virtqueue()
2410 	for (i = &dev->vq; *i; i = &(*i)->next);  in add_pci_virtqueue()
2411 	*i = vq;  in add_pci_virtqueue()
2633 	dev->vq = NULL;  in new_pci_device()
2938 static void blk_request(struct virtqueue *vq)  in blk_request()  argument
2940 	struct vblk_info *vblk = vq->dev->priv;  in blk_request()
2945 	struct iovec iov[vq->vring.num];  in blk_request()
2952 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in blk_request()
2955 	iov_consume(vq->dev, iov, out_num, &out, sizeof(out));  in blk_request()
2967 		bad_driver_vq(vq, "Bad virtblk cmd with no room for status");  in blk_request()
2997 			bad_driver_vq(vq, "Write past end %llu+%u", off, ret);  in blk_request()
3029 	add_used(vq, head, wlen);  in blk_request()
3080 static void rng_input(struct virtqueue *vq)  in rng_input()  argument
3084 	struct rng_info *rng_info = vq->dev->priv;  in rng_input()
3085 	struct iovec iov[vq->vring.num];  in rng_input()
3088 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in rng_input()
3090 		bad_driver_vq(vq, "Output buffers in rng?");  in rng_input()
3100 		iov_consume(vq->dev, iov, in_num, NULL, len);  in rng_input()
3105 	add_used(vq, head, totlen);  in rng_input()