709 			    struct pipe_buffer *buf, struct splice_desc *sd)  in pipe_to_sendpage()  argument
711 	struct file *file = sd->u.file;  in pipe_to_sendpage()
712 	loff_t pos = sd->pos;  in pipe_to_sendpage()
718 	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;  in pipe_to_sendpage()
720 	if (sd->len < sd->total_len && pipe->nrbufs > 1)  in pipe_to_sendpage()
724 				    sd->len, &pos, more);  in pipe_to_sendpage()
755 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
764 		sd->len = buf->len;  in splice_from_pipe_feed()
765 		if (sd->len > sd->total_len)  in splice_from_pipe_feed()
766 			sd->len = sd->total_len;  in splice_from_pipe_feed()
775 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
782 		sd->num_spliced += ret;  in splice_from_pipe_feed()
783 		sd->len -= ret;  in splice_from_pipe_feed()
784 		sd->pos += ret;  in splice_from_pipe_feed()
785 		sd->total_len -= ret;  in splice_from_pipe_feed()
793 				sd->need_wakeup = true;  in splice_from_pipe_feed()
796 		if (!sd->total_len)  in splice_from_pipe_feed()
813 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
826 		if (!pipe->waiting_writers && sd->num_spliced)  in splice_from_pipe_next()
829 		if (sd->flags & SPLICE_F_NONBLOCK)  in splice_from_pipe_next()
835 		if (sd->need_wakeup) {  in splice_from_pipe_next()
837 			sd->need_wakeup = false;  in splice_from_pipe_next()
855 static void splice_from_pipe_begin(struct splice_desc *sd)  in splice_from_pipe_begin()  argument
857 	sd->num_spliced = 0;  in splice_from_pipe_begin()
858 	sd->need_wakeup = false;  in splice_from_pipe_begin()
871 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
873 	if (sd->need_wakeup)  in splice_from_pipe_end()
890 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
895 	splice_from_pipe_begin(sd);  in __splice_from_pipe()
898 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
900 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
902 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
904 	return sd->num_spliced ? sd->num_spliced : ret;  in __splice_from_pipe()
927 	struct splice_desc sd = {  in splice_from_pipe()  local
935 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
959 	struct splice_desc sd = {  in iter_file_splice_write()  local
975 	splice_from_pipe_begin(&sd);  in iter_file_splice_write()
976 	while (sd.total_len) {  in iter_file_splice_write()
981 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
997 		left = sd.total_len;  in iter_file_splice_write()
1022 			      sd.total_len - left);  in iter_file_splice_write()
1023 		ret = vfs_iter_write(out, &from, &sd.pos);  in iter_file_splice_write()
1027 		sd.num_spliced += ret;  in iter_file_splice_write()
1028 		sd.total_len -= ret;  in iter_file_splice_write()
1029 		*ppos = sd.pos;  in iter_file_splice_write()
1043 					sd.need_wakeup = true;  in iter_file_splice_write()
1053 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
1057 	if (sd.num_spliced)  in iter_file_splice_write()
1058 		ret = sd.num_spliced;  in iter_file_splice_write()
1066 			  struct splice_desc *sd)  in write_pipe_buf()  argument
1070 	loff_t tmp = sd->pos;  in write_pipe_buf()
1073 	ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);  in write_pipe_buf()
1169 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,  in splice_direct_to_actor()  argument
1212 	len = sd->total_len;  in splice_direct_to_actor()
1213 	flags = sd->flags;  in splice_direct_to_actor()
1218 	sd->flags &= ~SPLICE_F_NONBLOCK;  in splice_direct_to_actor()
1219 	more = sd->flags & SPLICE_F_MORE;  in splice_direct_to_actor()
1223 		loff_t pos = sd->pos, prev_pos = pos;  in splice_direct_to_actor()
1230 		sd->total_len = read_len;  in splice_direct_to_actor()
1238 			sd->flags |= SPLICE_F_MORE;  in splice_direct_to_actor()
1240 			sd->flags &= ~SPLICE_F_MORE;  in splice_direct_to_actor()
1246 		ret = actor(pipe, sd);  in splice_direct_to_actor()
1248 			sd->pos = prev_pos;  in splice_direct_to_actor()
1254 		sd->pos = pos;  in splice_direct_to_actor()
1257 			sd->pos = prev_pos + ret;  in splice_direct_to_actor()
1289 			       struct splice_desc *sd)  in direct_splice_actor()  argument
1291 	struct file *file = sd->u.file;  in direct_splice_actor()
1293 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
1294 			      sd->flags);  in direct_splice_actor()
1316 	struct splice_desc sd = {  in do_splice_direct()  local
1336 	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);  in do_splice_direct()
1338 		*ppos = sd.pos;  in do_splice_direct()
1542 			struct splice_desc *sd)  in pipe_to_user()  argument
1544 	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);  in pipe_to_user()
1545 	return n == sd->len ? n : -EFAULT;  in pipe_to_user()
1556 	struct splice_desc sd;  in vmsplice_to_user()  local
1571 	sd.total_len = iov_iter_count(&iter);  in vmsplice_to_user()
1572 	sd.len = 0;  in vmsplice_to_user()
1573 	sd.flags = flags;  in vmsplice_to_user()
1574 	sd.u.data = &iter;  in vmsplice_to_user()
1575 	sd.pos = 0;  in vmsplice_to_user()
1577 	if (sd.total_len) {  in vmsplice_to_user()
1579 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()