31 static inline int __vringh_get_head(const struct vringh *vrh,  in __vringh_get_head()  argument
32 				    int (*getu16)(const struct vringh *vrh,  in __vringh_get_head()  argument
39 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
42 			   &vrh->vring.avail->idx);  in __vringh_get_head()
47 		return vrh->vring.num;  in __vringh_get_head()
50 	virtio_rmb(vrh->weak_barriers);  in __vringh_get_head()
52 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
54 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
57 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
61 	if (head >= vrh->vring.num) {  in __vringh_get_head()
63 			   head, vrh->vring.num);  in __vringh_get_head()
106 static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,  in range_check()  argument
112 		if (!getrange(vrh, addr, range))  in range_check()
140 static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,  in no_range_check()  argument
149 static int move_to_indirect(const struct vringh *vrh,  in move_to_indirect()  argument
162 	len = vringh32_to_cpu(vrh, desc->len);  in move_to_indirect()
169 	if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))  in move_to_indirect()
170 		*up_next = vringh16_to_cpu(vrh, desc->next);  in move_to_indirect()
207 static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next,  in return_from_indirect()  argument
213 	*descs = vrh->vring.desc;  in return_from_indirect()
214 	*desc_max = vrh->vring.num;  in return_from_indirect()
218 static int slow_copy(struct vringh *vrh, void *dst, const void *src,  in slow_copy()  argument
219 		     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,  in slow_copy()  argument
221 				    bool (*getrange)(struct vringh *vrh,  in slow_copy()  argument
224 		     bool (*getrange)(struct vringh *vrh,  in slow_copy()
239 		if (!rcheck(vrh, addr, &part, range, getrange))  in slow_copy()
254 __vringh_iov(struct vringh *vrh, u16 i,  in __vringh_iov()  argument
257 	     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,  in __vringh_iov()  argument
271 	descs = vrh->vring.desc;  in __vringh_iov()
272 	desc_max = vrh->vring.num;  in __vringh_iov()
289 			err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange,  in __vringh_iov()
297 			     cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {  in __vringh_iov()
298 			u64 a = vringh64_to_cpu(vrh, desc.addr);  in __vringh_iov()
301 			len = vringh32_to_cpu(vrh, desc.len);  in __vringh_iov()
302 			if (!rcheck(vrh, a, &len, &range, getrange)) {  in __vringh_iov()
307 			if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {  in __vringh_iov()
314 			err = move_to_indirect(vrh, &up_next, &i, addr, &desc,  in __vringh_iov()
321 		if (count++ == vrh->vring.num) {  in __vringh_iov()
327 		if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))  in __vringh_iov()
348 		len = vringh32_to_cpu(vrh, desc.len);  in __vringh_iov()
349 		if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,  in __vringh_iov()
354 		addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +  in __vringh_iov()
367 		if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {  in __vringh_iov()
368 			desc.len = cpu_to_vringh32(vrh,  in __vringh_iov()
369 				   vringh32_to_cpu(vrh, desc.len) - len);  in __vringh_iov()
370 			desc.addr = cpu_to_vringh64(vrh,  in __vringh_iov()
371 				    vringh64_to_cpu(vrh, desc.addr) + len);  in __vringh_iov()
375 		if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {  in __vringh_iov()
376 			i = vringh16_to_cpu(vrh, desc.next);  in __vringh_iov()
380 				i = return_from_indirect(vrh, &up_next,  in __vringh_iov()
400 static inline int __vringh_complete(struct vringh *vrh,  in __vringh_complete()  argument
403 				    int (*putu16)(const struct vringh *vrh,  in __vringh_complete()  argument
413 	used_ring = vrh->vring.used;  in __vringh_complete()
414 	used_idx = vrh->last_used_idx + vrh->completed;  in __vringh_complete()
416 	off = used_idx % vrh->vring.num;  in __vringh_complete()
419 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
420 		u16 part = vrh->vring.num - off;  in __vringh_complete()
435 	virtio_wmb(vrh->weak_barriers);  in __vringh_complete()
437 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
440 			   &vrh->vring.used->idx);  in __vringh_complete()
444 	vrh->completed += num_used;  in __vringh_complete()
449 static inline int __vringh_need_notify(struct vringh *vrh,  in __vringh_need_notify()  argument
450 				       int (*getu16)(const struct vringh *vrh,  in __vringh_need_notify()  argument
461 	virtio_mb(vrh->weak_barriers);  in __vringh_need_notify()
464 	if (!vrh->event_indices) {  in __vringh_need_notify()
466 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
469 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
476 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
479 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
484 	if (unlikely(vrh->completed > 0xffff))  in __vringh_need_notify()
488 					  vrh->last_used_idx + vrh->completed,  in __vringh_need_notify()
489 					  vrh->last_used_idx);  in __vringh_need_notify()
491 	vrh->last_used_idx += vrh->completed;  in __vringh_need_notify()
492 	vrh->completed = 0;  in __vringh_need_notify()
496 static inline bool __vringh_notify_enable(struct vringh *vrh,  in __vringh_notify_enable()  argument
497 					  int (*getu16)(const struct vringh *vrh,  in __vringh_notify_enable()  argument
499 					  int (*putu16)(const struct vringh *vrh,  in __vringh_notify_enable()  argument
504 	if (!vrh->event_indices) {  in __vringh_notify_enable()
506 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
508 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
512 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
513 			   vrh->last_avail_idx) != 0) {  in __vringh_notify_enable()
515 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
522 	virtio_mb(vrh->weak_barriers);  in __vringh_notify_enable()
524 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
526 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
533 	return avail == vrh->last_avail_idx;  in __vringh_notify_enable()
536 static inline void __vringh_notify_disable(struct vringh *vrh,  in __vringh_notify_disable()  argument
537 					   int (*putu16)(const struct vringh *vrh,  in __vringh_notify_disable()  argument
540 	if (!vrh->event_indices) {  in __vringh_notify_disable()
542 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
545 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
551 static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)  in getu16_user()  argument
555 	*val = vringh16_to_cpu(vrh, v);  in getu16_user()
559 static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)  in putu16_user()  argument
561 	__virtio16 v = cpu_to_vringh16(vrh, val);  in putu16_user()
604 int vringh_init_user(struct vringh *vrh, u64 features,  in vringh_init_user()  argument
616 	vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));  in vringh_init_user()
617 	vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));  in vringh_init_user()
618 	vrh->weak_barriers = weak_barriers;  in vringh_init_user()
619 	vrh->completed = 0;  in vringh_init_user()
620 	vrh->last_avail_idx = 0;  in vringh_init_user()
621 	vrh->last_used_idx = 0;  in vringh_init_user()
622 	vrh->vring.num = num;  in vringh_init_user()
624 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
625 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
626 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
648 int vringh_getdesc_user(struct vringh *vrh,  in vringh_getdesc_user()  argument
651 			bool (*getrange)(struct vringh *vrh,  in vringh_getdesc_user()  argument
657 	*head = vrh->vring.num;  in vringh_getdesc_user()
658 	err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx);  in vringh_getdesc_user()
663 	if (err == vrh->vring.num)  in vringh_getdesc_user()
687 	err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov,  in vringh_getdesc_user()
736 void vringh_abandon_user(struct vringh *vrh, unsigned int num)  in vringh_abandon_user()  argument
740 	vrh->last_avail_idx -= num;  in vringh_abandon_user()
753 int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_user()  argument
757 	used.id = cpu_to_vringh32(vrh, head);  in vringh_complete_user()
758 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_user()
759 	return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);  in vringh_complete_user()
772 int vringh_complete_multi_user(struct vringh *vrh,  in vringh_complete_multi_user()  argument
776 	return __vringh_complete(vrh, used, num_used,  in vringh_complete_multi_user()
788 bool vringh_notify_enable_user(struct vringh *vrh)  in vringh_notify_enable_user()  argument
790 	return __vringh_notify_enable(vrh, getu16_user, putu16_user);  in vringh_notify_enable_user()
801 void vringh_notify_disable_user(struct vringh *vrh)  in vringh_notify_disable_user()  argument
803 	__vringh_notify_disable(vrh, putu16_user);  in vringh_notify_disable_user()
813 int vringh_need_notify_user(struct vringh *vrh)  in vringh_need_notify_user()  argument
815 	return __vringh_need_notify(vrh, getu16_user);  in vringh_need_notify_user()
820 static inline int getu16_kern(const struct vringh *vrh,  in getu16_kern()  argument
823 	*val = vringh16_to_cpu(vrh, ACCESS_ONCE(*p));  in getu16_kern()
827 static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)  in putu16_kern()  argument
829 	ACCESS_ONCE(*p) = cpu_to_vringh16(vrh, val);  in putu16_kern()
865 int vringh_init_kern(struct vringh *vrh, u64 features,  in vringh_init_kern()  argument
877 	vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));  in vringh_init_kern()
878 	vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));  in vringh_init_kern()
879 	vrh->weak_barriers = weak_barriers;  in vringh_init_kern()
880 	vrh->completed = 0;  in vringh_init_kern()
881 	vrh->last_avail_idx = 0;  in vringh_init_kern()
882 	vrh->last_used_idx = 0;  in vringh_init_kern()
883 	vrh->vring.num = num;  in vringh_init_kern()
884 	vrh->vring.desc = desc;  in vringh_init_kern()
885 	vrh->vring.avail = avail;  in vringh_init_kern()
886 	vrh->vring.used = used;  in vringh_init_kern()
908 int vringh_getdesc_kern(struct vringh *vrh,  in vringh_getdesc_kern()  argument
916 	err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx);  in vringh_getdesc_kern()
921 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
925 	err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,  in vringh_getdesc_kern()
971 void vringh_abandon_kern(struct vringh *vrh, unsigned int num)  in vringh_abandon_kern()  argument
975 	vrh->last_avail_idx -= num;  in vringh_abandon_kern()
988 int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_kern()  argument
992 	used.id = cpu_to_vringh32(vrh, head);  in vringh_complete_kern()
993 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_kern()
995 	return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);  in vringh_complete_kern()
1006 bool vringh_notify_enable_kern(struct vringh *vrh)  in vringh_notify_enable_kern()  argument
1008 	return __vringh_notify_enable(vrh, getu16_kern, putu16_kern);  in vringh_notify_enable_kern()
1019 void vringh_notify_disable_kern(struct vringh *vrh)  in vringh_notify_disable_kern()  argument
1021 	__vringh_notify_disable(vrh, putu16_kern);  in vringh_notify_disable_kern()
1031 int vringh_need_notify_kern(struct vringh *vrh)  in vringh_need_notify_kern()  argument
1033 	return __vringh_need_notify(vrh, getu16_kern);  in vringh_need_notify_kern()