169 	struct virtqueue *vq;  member
223 	void (*service)(struct virtqueue *vq);
243 #define lg_last_avail(vq)	((vq)->last_avail_idx)  argument
262 #define bad_driver_vq(vq, fmt, ...)			       \  argument
263 	errx(1, "%s vq %s: bad driver: " fmt, (vq)->dev->name, \
264 	     vq->name, ## __VA_ARGS__)
682 static void trigger_irq(struct virtqueue *vq)  in trigger_irq()  argument
684 	unsigned long buf[] = { LHREQ_IRQ, vq->dev->config.irq_line };  in trigger_irq()
687 	if (!vq->pending_used)  in trigger_irq()
689 	vq->pending_used = 0;  in trigger_irq()
697 	if (vq->vring.avail->flags > 1)  in trigger_irq()
698 		bad_driver_vq(vq, "avail->flags = %u\n", vq->vring.avail->flags);  in trigger_irq()
710 	if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {  in trigger_irq()
721 	vq->dev->mmio->isr = 0x1;  in trigger_irq()
725 		err(1, "Triggering irq %i", vq->dev->config.irq_line);  in trigger_irq()
736 static unsigned wait_for_vq_desc(struct virtqueue *vq,  in wait_for_vq_desc()  argument
742 	u16 last_avail = lg_last_avail(vq);  in wait_for_vq_desc()
753 	while (last_avail == vq->vring.avail->idx) {  in wait_for_vq_desc()
760 		trigger_irq(vq);  in wait_for_vq_desc()
763 		vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
770 		if (last_avail != vq->vring.avail->idx) {  in wait_for_vq_desc()
771 			vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
776 		if (read(vq->eventfd, &event, sizeof(event)) != sizeof(event))  in wait_for_vq_desc()
780 		vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
784 	if ((u16)(vq->vring.avail->idx - last_avail) > vq->vring.num)  in wait_for_vq_desc()
785 		bad_driver_vq(vq, "Guest moved used index from %u to %u",  in wait_for_vq_desc()
786 			      last_avail, vq->vring.avail->idx);  in wait_for_vq_desc()
798 	head = vq->vring.avail->ring[last_avail % vq->vring.num];  in wait_for_vq_desc()
799 	lg_last_avail(vq)++;  in wait_for_vq_desc()
802 	if (head >= vq->vring.num)  in wait_for_vq_desc()
803 		bad_driver_vq(vq, "Guest says index %u is available", head);  in wait_for_vq_desc()
808 	max = vq->vring.num;  in wait_for_vq_desc()
809 	desc = vq->vring.desc;  in wait_for_vq_desc()
831 			if (!(vq->dev->features_accepted &  in wait_for_vq_desc()
833 				bad_driver_vq(vq, "vq indirect not negotiated");  in wait_for_vq_desc()
842 			if (desc != vq->vring.desc)  in wait_for_vq_desc()
843 				bad_driver_vq(vq, "Indirect within indirect");  in wait_for_vq_desc()
852 				bad_driver_vq(vq, "indirect and next together");  in wait_for_vq_desc()
855 				bad_driver_vq(vq,  in wait_for_vq_desc()
868 			desc = check_pointer(vq->dev, desc[i].addr, desc[i].len);  in wait_for_vq_desc()
876 			if (max > vq->pci_config.queue_size)  in wait_for_vq_desc()
877 				bad_driver_vq(vq,  in wait_for_vq_desc()
884 			= check_pointer(vq->dev, desc[i].addr, desc[i].len);  in wait_for_vq_desc()
894 				bad_driver_vq(vq,  in wait_for_vq_desc()
901 			bad_driver_vq(vq, "Looped descriptor");  in wait_for_vq_desc()
902 	} while ((i = next_desc(vq->dev, desc, i, max)) != max);  in wait_for_vq_desc()
912 static void add_used(struct virtqueue *vq, unsigned int head, int len)  in add_used()  argument
920 	used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];  in add_used()
925 	vq->vring.used->idx++;  in add_used()
926 	vq->pending_used++;  in add_used()
930 static void add_used_and_trigger(struct virtqueue *vq, unsigned head, int len)  in add_used_and_trigger()  argument
932 	add_used(vq, head, len);  in add_used_and_trigger()
933 	trigger_irq(vq);  in add_used_and_trigger()
949 static void console_input(struct virtqueue *vq)  in console_input()  argument
953 	struct console_abort *abort = vq->dev->priv;  in console_input()
954 	struct iovec iov[vq->vring.num];  in console_input()
957 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in console_input()
959 		bad_driver_vq(vq, "Output buffers in console in queue?");  in console_input()
975 	add_used_and_trigger(vq, head, len);  in console_input()
1004 static void console_output(struct virtqueue *vq)  in console_output()  argument
1007 	struct iovec iov[vq->vring.num];  in console_output()
1010 	head = wait_for_vq_desc(vq, iov, &out, &in);  in console_output()
1012 		bad_driver_vq(vq, "Input buffers in console output queue?");  in console_output()
1021 		iov_consume(vq->dev, iov, out, NULL, len);  in console_output()
1028 	add_used(vq, head, 0);  in console_output()
1041 static void net_output(struct virtqueue *vq)  in net_output()  argument
1043 	struct net_info *net_info = vq->dev->priv;  in net_output()
1045 	struct iovec iov[vq->vring.num];  in net_output()
1048 	head = wait_for_vq_desc(vq, iov, &out, &in);  in net_output()
1050 		bad_driver_vq(vq, "Input buffers in net output queue?");  in net_output()
1062 	add_used(vq, head, 0);  in net_output()
1085 static void net_input(struct virtqueue *vq)  in net_input()  argument
1089 	struct iovec iov[vq->vring.num];  in net_input()
1090 	struct net_info *net_info = vq->dev->priv;  in net_input()
1096 	head = wait_for_vq_desc(vq, iov, &out, &in);  in net_input()
1098 		bad_driver_vq(vq, "Output buffers in net input queue?");  in net_input()
1104 	if (vq->pending_used && will_block(net_info->tunfd))  in net_input()
1105 		trigger_irq(vq);  in net_input()
1119 	add_used(vq, head, len);  in net_input()
1126 	struct virtqueue *vq = _vq;  in do_thread()  local
1129 		vq->service(vq);  in do_thread()
1142 static void reset_vq_pci_config(struct virtqueue *vq)  in reset_vq_pci_config()  argument
1144 	vq->pci_config.queue_size = VIRTQUEUE_NUM;  in reset_vq_pci_config()
1145 	vq->pci_config.queue_enable = 0;  in reset_vq_pci_config()
1150 	struct virtqueue *vq;  in reset_device()  local
1170 	for (vq = dev->vq; vq; vq = vq->next) {  in reset_device()
1171 		vq->last_avail_idx = 0;  in reset_device()
1172 		reset_vq_pci_config(vq);  in reset_device()
1173 		if (vq->thread != (pid_t)-1) {  in reset_device()
1174 			kill(vq->thread, SIGTERM);  in reset_device()
1175 			waitpid(vq->thread, NULL, 0);  in reset_device()
1176 			vq->thread = (pid_t)-1;  in reset_device()
1763 	struct virtqueue *vq = d->vq;  in vq_by_num()  local
1765 	while (num-- && vq)  in vq_by_num()
1766 		vq = vq->next;  in vq_by_num()
1768 	return vq;  in vq_by_num()
1772 			   struct virtqueue *vq)  in save_vq_config()  argument
1774 	vq->pci_config = *cfg;  in save_vq_config()
1778 			      struct virtqueue *vq)  in restore_vq_config()  argument
1783 	memcpy((void *)cfg + off, (void *)&vq->pci_config + off,  in restore_vq_config()
1795 static void check_virtqueue(struct device *d, struct virtqueue *vq)  in check_virtqueue()  argument
1798 	if (vq->pci_config.queue_desc_hi  in check_virtqueue()
1799 	    || vq->pci_config.queue_avail_hi  in check_virtqueue()
1800 	    || vq->pci_config.queue_used_hi)  in check_virtqueue()
1801 		bad_driver_vq(vq, "invalid 64-bit queue address");  in check_virtqueue()
1810 	if (vq->pci_config.queue_desc_lo % 16  in check_virtqueue()
1811 	    || vq->pci_config.queue_avail_lo % 2  in check_virtqueue()
1812 	    || vq->pci_config.queue_used_lo % 4)  in check_virtqueue()
1813 		bad_driver_vq(vq, "invalid alignment in queue addresses");  in check_virtqueue()
1816 	vq->vring.num = vq->pci_config.queue_size;  in check_virtqueue()
1817 	vq->vring.desc = check_pointer(vq->dev,  in check_virtqueue()
1818 				       vq->pci_config.queue_desc_lo,  in check_virtqueue()
1819 				       sizeof(*vq->vring.desc) * vq->vring.num);  in check_virtqueue()
1820 	vq->vring.avail = check_pointer(vq->dev,  in check_virtqueue()
1821 					vq->pci_config.queue_avail_lo,  in check_virtqueue()
1822 					sizeof(*vq->vring.avail)  in check_virtqueue()
1823 					+ (sizeof(vq->vring.avail->ring[0])  in check_virtqueue()
1824 					   * vq->vring.num));  in check_virtqueue()
1825 	vq->vring.used = check_pointer(vq->dev,  in check_virtqueue()
1826 				       vq->pci_config.queue_used_lo,  in check_virtqueue()
1827 				       sizeof(*vq->vring.used)  in check_virtqueue()
1828 				       + (sizeof(vq->vring.used->ring[0])  in check_virtqueue()
1829 					  * vq->vring.num));  in check_virtqueue()
1837 	if (vq->vring.used->flags != 0)  in check_virtqueue()
1838 		bad_driver_vq(vq, "invalid initial used.flags %#x",  in check_virtqueue()
1839 			      vq->vring.used->flags);  in check_virtqueue()
1842 static void start_virtqueue(struct virtqueue *vq)  in start_virtqueue()  argument
1851 	vq->eventfd = eventfd(0, 0);  in start_virtqueue()
1852 	if (vq->eventfd < 0)  in start_virtqueue()
1859 	vq->thread = clone(do_thread, stack + 32768, CLONE_VM | SIGCHLD, vq);  in start_virtqueue()
1860 	if (vq->thread == (pid_t)-1)  in start_virtqueue()
1866 	struct virtqueue *vq;  in start_virtqueues()  local
1868 	for (vq = d->vq; vq; vq = vq->next) {  in start_virtqueues()
1869 		if (vq->pci_config.queue_enable)  in start_virtqueues()
1870 			start_virtqueue(vq);  in start_virtqueues()
1876 	struct virtqueue *vq;  in emulate_mmio_write()  local
2008 		vq = vq_by_num(d, val);  in emulate_mmio_write()
2015 		if (!vq) {  in emulate_mmio_write()
2024 		restore_vq_config(&d->mmio->cfg, vq);  in emulate_mmio_write()
2041 		struct virtqueue *vq = vq_by_num(d, d->mmio->cfg.queue_select);  in emulate_mmio_write()  local
2067 		save_vq_config(&d->mmio->cfg, vq);  in emulate_mmio_write()
2068 		check_virtqueue(d, vq);  in emulate_mmio_write()
2108 		vq = vq_by_num(d, val);  in emulate_mmio_write()
2109 		if (!vq)  in emulate_mmio_write()
2112 		write(vq->eventfd, "\1\0\0\0\0\0\0\0", 8);  in emulate_mmio_write()
2380 	struct virtqueue **i, *vq = malloc(sizeof(*vq));  in add_pci_virtqueue()  local
2383 	vq->next = NULL;  in add_pci_virtqueue()
2384 	vq->last_avail_idx = 0;  in add_pci_virtqueue()
2385 	vq->dev = dev;  in add_pci_virtqueue()
2386 	vq->name = name;  in add_pci_virtqueue()
2392 	vq->service = service;  in add_pci_virtqueue()
2393 	vq->thread = (pid_t)-1;  in add_pci_virtqueue()
2396 	reset_vq_pci_config(vq);  in add_pci_virtqueue()
2397 	vq->pci_config.queue_notify_off = 0;  in add_pci_virtqueue()
2400 	vq->dev->mmio->cfg.num_queues++;  in add_pci_virtqueue()
2406 	for (i = &dev->vq; *i; i = &(*i)->next);  in add_pci_virtqueue()
2407 	*i = vq;  in add_pci_virtqueue()
2629 	dev->vq = NULL;  in new_pci_device()
2934 static void blk_request(struct virtqueue *vq)  in blk_request()  argument
2936 	struct vblk_info *vblk = vq->dev->priv;  in blk_request()
2941 	struct iovec iov[vq->vring.num];  in blk_request()
2948 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in blk_request()
2951 	iov_consume(vq->dev, iov, out_num, &out, sizeof(out));  in blk_request()
2963 		bad_driver_vq(vq, "Bad virtblk cmd with no room for status");  in blk_request()
2993 			bad_driver_vq(vq, "Write past end %llu+%u", off, ret);  in blk_request()
3025 	add_used(vq, head, wlen);  in blk_request()
3076 static void rng_input(struct virtqueue *vq)  in rng_input()  argument
3080 	struct rng_info *rng_info = vq->dev->priv;  in rng_input()
3081 	struct iovec iov[vq->vring.num];  in rng_input()
3084 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in rng_input()
3086 		bad_driver_vq(vq, "Output buffers in rng?");  in rng_input()
3096 		iov_consume(vq->dev, iov, in_num, NULL, len);  in rng_input()
3101 	add_used(vq, head, totlen);  in rng_input()