8 #define iterate_iovec(i, n, __v, __p, skip, STEP) {	\  argument
11 	__p = i->iov;					\
36 #define iterate_kvec(i, n, __v, __p, skip, STEP) {	\  argument
38 	__p = i->kvec;					\
59 #define iterate_bvec(i, n, __v, __p, skip, STEP) {	\  argument
61 	__p = i->bvec;					\
84 #define iterate_all_kinds(i, n, v, I, B, K) {			\  argument
85 	size_t skip = i->iov_offset;				\
86 	if (unlikely(i->type & ITER_BVEC)) {			\
89 		iterate_bvec(i, n, v, bvec, skip, (B))		\
90 	} else if (unlikely(i->type & ITER_KVEC)) {		\
93 		iterate_kvec(i, n, v, kvec, skip, (K))		\
97 		iterate_iovec(i, n, v, iov, skip, (I))		\
101 #define iterate_and_advance(i, n, v, I, B, K) {			\  argument
102 	size_t skip = i->iov_offset;				\
103 	if (unlikely(i->type & ITER_BVEC)) {			\
106 		iterate_bvec(i, n, v, bvec, skip, (B))		\
111 		i->nr_segs -= bvec - i->bvec;			\
112 		i->bvec = bvec;					\
113 	} else if (unlikely(i->type & ITER_KVEC)) {		\
116 		iterate_kvec(i, n, v, kvec, skip, (K))		\
121 		i->nr_segs -= kvec - i->kvec;			\
122 		i->kvec = kvec;					\
126 		iterate_iovec(i, n, v, iov, skip, (I))		\
131 		i->nr_segs -= iov - i->iov;			\
132 		i->iov = iov;					\
134 	i->count -= n;						\
135 	i->iov_offset = skip;					\
139 			 struct iov_iter *i)  in copy_page_to_iter_iovec()  argument
146 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
147 		bytes = i->count;  in copy_page_to_iter_iovec()
153 	iov = i->iov;  in copy_page_to_iter_iovec()
154 	skip = i->iov_offset;  in copy_page_to_iter_iovec()
212 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
213 	i->nr_segs -= iov - i->iov;  in copy_page_to_iter_iovec()
214 	i->iov = iov;  in copy_page_to_iter_iovec()
215 	i->iov_offset = skip;  in copy_page_to_iter_iovec()
220 			 struct iov_iter *i)  in copy_page_from_iter_iovec()  argument
227 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
228 		bytes = i->count;  in copy_page_from_iter_iovec()
234 	iov = i->iov;  in copy_page_from_iter_iovec()
235 	skip = i->iov_offset;  in copy_page_from_iter_iovec()
293 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
294 	i->nr_segs -= iov - i->iov;  in copy_page_from_iter_iovec()
295 	i->iov = iov;  in copy_page_from_iter_iovec()
296 	i->iov_offset = skip;  in copy_page_from_iter_iovec()
309 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
311 	if (!(i->type & (ITER_BVEC|ITER_KVEC))) {  in iov_iter_fault_in_readable()
312 		char __user *buf = i->iov->iov_base + i->iov_offset;  in iov_iter_fault_in_readable()
313 		bytes = min(bytes, i->iov->iov_len - i->iov_offset);  in iov_iter_fault_in_readable()
327 int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_multipages_readable()  argument
329 	size_t skip = i->iov_offset;  in iov_iter_fault_in_multipages_readable()
334 	if (!(i->type & (ITER_BVEC|ITER_KVEC))) {  in iov_iter_fault_in_multipages_readable()
335 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_multipages_readable()
346 void iov_iter_init(struct iov_iter *i, int direction,  in iov_iter_init()  argument
353 		i->type = direction;  in iov_iter_init()
354 		i->kvec = (struct kvec *)iov;  in iov_iter_init()
356 		i->type = direction;  in iov_iter_init()
357 		i->iov = iov;  in iov_iter_init()
359 	i->nr_segs = nr_segs;  in iov_iter_init()
360 	i->iov_offset = 0;  in iov_iter_init()
361 	i->count = count;  in iov_iter_init()
386 size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i)  in copy_to_iter()  argument
389 	if (unlikely(bytes > i->count))  in copy_to_iter()
390 		bytes = i->count;  in copy_to_iter()
395 	iterate_and_advance(i, bytes, v,  in copy_to_iter()
407 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in copy_from_iter()  argument
410 	if (unlikely(bytes > i->count))  in copy_from_iter()
411 		bytes = i->count;  in copy_from_iter()
416 	iterate_and_advance(i, bytes, v,  in copy_from_iter()
428 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in copy_from_iter_nocache()  argument
431 	if (unlikely(bytes > i->count))  in copy_from_iter_nocache()
432 		bytes = i->count;  in copy_from_iter_nocache()
437 	iterate_and_advance(i, bytes, v,  in copy_from_iter_nocache()
450 			 struct iov_iter *i)  in copy_page_to_iter()  argument
452 	if (i->type & (ITER_BVEC|ITER_KVEC)) {  in copy_page_to_iter()
454 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
458 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
463 			 struct iov_iter *i)  in copy_page_from_iter()  argument
465 	if (i->type & (ITER_BVEC|ITER_KVEC)) {  in copy_page_from_iter()
467 		size_t wanted = copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
471 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
475 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
477 	if (unlikely(bytes > i->count))  in iov_iter_zero()
478 		bytes = i->count;  in iov_iter_zero()
483 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
494 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
497 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
509 void iov_iter_advance(struct iov_iter *i, size_t size)  in iov_iter_advance()  argument
511 	iterate_and_advance(i, size, v, 0, 0, 0)  in iov_iter_advance()
518 size_t iov_iter_single_seg_count(const struct iov_iter *i)  in iov_iter_single_seg_count()  argument
520 	if (i->nr_segs == 1)  in iov_iter_single_seg_count()
521 		return i->count;  in iov_iter_single_seg_count()
522 	else if (i->type & ITER_BVEC)  in iov_iter_single_seg_count()
523 		return min(i->count, i->bvec->bv_len - i->iov_offset);  in iov_iter_single_seg_count()
525 		return min(i->count, i->iov->iov_len - i->iov_offset);  in iov_iter_single_seg_count()
529 void iov_iter_kvec(struct iov_iter *i, int direction,  in iov_iter_kvec()  argument
534 	i->type = direction;  in iov_iter_kvec()
535 	i->kvec = kvec;  in iov_iter_kvec()
536 	i->nr_segs = nr_segs;  in iov_iter_kvec()
537 	i->iov_offset = 0;  in iov_iter_kvec()
538 	i->count = count;  in iov_iter_kvec()
542 void iov_iter_bvec(struct iov_iter *i, int direction,  in iov_iter_bvec()  argument
547 	i->type = direction;  in iov_iter_bvec()
548 	i->bvec = bvec;  in iov_iter_bvec()
549 	i->nr_segs = nr_segs;  in iov_iter_bvec()
550 	i->iov_offset = 0;  in iov_iter_bvec()
551 	i->count = count;  in iov_iter_bvec()
555 unsigned long iov_iter_alignment(const struct iov_iter *i)  in iov_iter_alignment()  argument
558 	size_t size = i->count;  in iov_iter_alignment()
563 	iterate_all_kinds(i, size, v,  in iov_iter_alignment()
572 ssize_t iov_iter_get_pages(struct iov_iter *i,  in iov_iter_get_pages()  argument
576 	if (maxsize > i->count)  in iov_iter_get_pages()
577 		maxsize = i->count;  in iov_iter_get_pages()
582 	iterate_all_kinds(i, maxsize, v, ({  in iov_iter_get_pages()
592 		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);  in iov_iter_get_pages()
617 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,  in iov_iter_get_pages_alloc()  argument
623 	if (maxsize > i->count)  in iov_iter_get_pages_alloc()
624 		maxsize = i->count;  in iov_iter_get_pages_alloc()
629 	iterate_all_kinds(i, maxsize, v, ({  in iov_iter_get_pages_alloc()
640 		res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);  in iov_iter_get_pages_alloc()
664 			       struct iov_iter *i)  in csum_and_copy_from_iter()  argument
669 	if (unlikely(bytes > i->count))  in csum_and_copy_from_iter()
670 		bytes = i->count;  in csum_and_copy_from_iter()
676 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
708 			     struct iov_iter *i)  in csum_and_copy_to_iter()  argument
713 	if (unlikely(bytes > i->count))  in csum_and_copy_to_iter()
714 		bytes = i->count;  in csum_and_copy_to_iter()
720 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
751 int iov_iter_npages(const struct iov_iter *i, int maxpages)  in iov_iter_npages()  argument
753 	size_t size = i->count;  in iov_iter_npages()
759 	iterate_all_kinds(i, size, v, ({  in iov_iter_npages()
798 		 struct iovec **iov, struct iov_iter *i)  in import_iovec()  argument
810 	iov_iter_init(i, type, p, nr_segs, n);  in import_iovec()
821 		 struct iovec **iov, struct iov_iter *i)  in compat_import_iovec()  argument
833 	iov_iter_init(i, type, p, nr_segs, n);  in compat_import_iovec()
840 		 struct iovec *iov, struct iov_iter *i)  in import_single_range()  argument
849 	iov_iter_init(i, rw, iov, 1, len);  in import_single_range()