73 				      void *ptr, size_t len,  in vringh_iov_xfer()  argument
75 						  size_t len))  in vringh_iov_xfer()
79 	while (len && iov->i < iov->used) {  in vringh_iov_xfer()
82 		partlen = min(iov->iov[iov->i].iov_len, len);  in vringh_iov_xfer()
87 		len -= partlen;  in vringh_iov_xfer()
106 static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,  in range_check()  argument
118 	if (unlikely(addr + *len == 0)) {  in range_check()
125 	if (addr + *len < addr) {  in range_check()
127 			   *len, (unsigned long long)addr);  in range_check()
131 	if (unlikely(addr + *len - 1 > range->end_incl))  in range_check()
136 	*len = range->end_incl + 1 - addr;  in range_check()
140 static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,  in no_range_check()  argument
154 	u32 len;  in move_to_indirect()  local
162 	len = vringh32_to_cpu(vrh, desc->len);  in move_to_indirect()
163 	if (unlikely(len % sizeof(struct vring_desc))) {  in move_to_indirect()
164 		vringh_bad("Strange indirect len %u", desc->len);  in move_to_indirect()
174 	*desc_max = len / sizeof(struct vring_desc);  in move_to_indirect()
219 		     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,  in slow_copy()  argument
228 		     int (*copy)(void *dst, const void *src, size_t len))  in slow_copy()  argument
230 	size_t part, len = sizeof(struct vring_desc);  in slow_copy()  local
236 		part = len;  in slow_copy()
248 		len -= part;  in slow_copy()
249 	} while (len);  in slow_copy()
257 	     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,  in __vringh_iov()  argument
263 	     int (*copy)(void *dst, const void *src, size_t len))  in __vringh_iov()  argument
286 		size_t len;  in __vringh_iov()  local
301 			len = vringh32_to_cpu(vrh, desc.len);  in __vringh_iov()
302 			if (!rcheck(vrh, a, &len, &range, getrange)) {  in __vringh_iov()
307 			if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {  in __vringh_iov()
348 		len = vringh32_to_cpu(vrh, desc.len);  in __vringh_iov()
349 		if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,  in __vringh_iov()
364 		iov->iov[iov->used].iov_len = len;  in __vringh_iov()
367 		if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {  in __vringh_iov()
368 			desc.len = cpu_to_vringh32(vrh,  in __vringh_iov()
369 				   vringh32_to_cpu(vrh, desc.len) - len);  in __vringh_iov()
371 				    vringh64_to_cpu(vrh, desc.addr) + len);  in __vringh_iov()
565 static inline int copydesc_user(void *dst, const void *src, size_t len)  in copydesc_user()  argument
567 	return copy_from_user(dst, (__force void __user *)src, len) ?  in copydesc_user()
579 static inline int xfer_from_user(void *src, void *dst, size_t len)  in xfer_from_user()  argument
581 	return copy_from_user(dst, (__force void __user *)src, len) ?  in xfer_from_user()
585 static inline int xfer_to_user(void *dst, void *src, size_t len)  in xfer_to_user()  argument
587 	return copy_to_user((__force void __user *)dst, src, len) ?  in xfer_to_user()
705 ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len)  in vringh_iov_pull_user()  argument
708 			       dst, len, xfer_from_user);  in vringh_iov_pull_user()
721 			     const void *src, size_t len)  in vringh_iov_push_user()  argument
724 			       (void *)src, len, xfer_to_user);  in vringh_iov_push_user()
753 int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_user()  argument
758 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_user()
833 static inline int copydesc_kern(void *dst, const void *src, size_t len)  in copydesc_kern()  argument
835 	memcpy(dst, src, len);  in copydesc_kern()
847 static inline int xfer_kern(void *src, void *dst, size_t len)  in xfer_kern()  argument
849 	memcpy(dst, src, len);  in xfer_kern()
942 ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len)  in vringh_iov_pull_kern()  argument
944 	return vringh_iov_xfer(riov, dst, len, xfer_kern);  in vringh_iov_pull_kern()
957 			     const void *src, size_t len)  in vringh_iov_push_kern()  argument
959 	return vringh_iov_xfer(wiov, (void *)src, len, xfer_kern);  in vringh_iov_push_kern()
988 int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)  in vringh_complete_kern()  argument
993 	used.len = cpu_to_vringh32(vrh, len);  in vringh_complete_kern()