708 			    struct pipe_buffer *buf, struct splice_desc *sd)  in pipe_to_sendpage()  argument
710 	struct file *file = sd->u.file;  in pipe_to_sendpage()
711 	loff_t pos = sd->pos;  in pipe_to_sendpage()
717 	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;  in pipe_to_sendpage()
719 	if (sd->len < sd->total_len && pipe->nrbufs > 1)  in pipe_to_sendpage()
723 				    sd->len, &pos, more);  in pipe_to_sendpage()
754 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
763 		sd->len = buf->len;  in splice_from_pipe_feed()
764 		if (sd->len > sd->total_len)  in splice_from_pipe_feed()
765 			sd->len = sd->total_len;  in splice_from_pipe_feed()
774 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
781 		sd->num_spliced += ret;  in splice_from_pipe_feed()
782 		sd->len -= ret;  in splice_from_pipe_feed()
783 		sd->pos += ret;  in splice_from_pipe_feed()
784 		sd->total_len -= ret;  in splice_from_pipe_feed()
792 				sd->need_wakeup = true;  in splice_from_pipe_feed()
795 		if (!sd->total_len)  in splice_from_pipe_feed()
812 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
818 		if (!pipe->waiting_writers && sd->num_spliced)  in splice_from_pipe_next()
821 		if (sd->flags & SPLICE_F_NONBLOCK)  in splice_from_pipe_next()
827 		if (sd->need_wakeup) {  in splice_from_pipe_next()
829 			sd->need_wakeup = false;  in splice_from_pipe_next()
847 static void splice_from_pipe_begin(struct splice_desc *sd)  in splice_from_pipe_begin()  argument
849 	sd->num_spliced = 0;  in splice_from_pipe_begin()
850 	sd->need_wakeup = false;  in splice_from_pipe_begin()
863 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
865 	if (sd->need_wakeup)  in splice_from_pipe_end()
882 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
887 	splice_from_pipe_begin(sd);  in __splice_from_pipe()
889 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
891 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
893 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
895 	return sd->num_spliced ? sd->num_spliced : ret;  in __splice_from_pipe()
918 	struct splice_desc sd = {  in splice_from_pipe()  local
926 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
950 	struct splice_desc sd = {  in iter_file_splice_write()  local
966 	splice_from_pipe_begin(&sd);  in iter_file_splice_write()
967 	while (sd.total_len) {  in iter_file_splice_write()
972 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
988 		left = sd.total_len;  in iter_file_splice_write()
1013 			      sd.total_len - left);  in iter_file_splice_write()
1014 		ret = vfs_iter_write(out, &from, &sd.pos);  in iter_file_splice_write()
1018 		sd.num_spliced += ret;  in iter_file_splice_write()
1019 		sd.total_len -= ret;  in iter_file_splice_write()
1020 		*ppos = sd.pos;  in iter_file_splice_write()
1034 					sd.need_wakeup = true;  in iter_file_splice_write()
1044 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
1048 	if (sd.num_spliced)  in iter_file_splice_write()
1049 		ret = sd.num_spliced;  in iter_file_splice_write()
1057 			  struct splice_desc *sd)  in write_pipe_buf()  argument
1061 	loff_t tmp = sd->pos;  in write_pipe_buf()
1064 	ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);  in write_pipe_buf()
1160 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,  in splice_direct_to_actor()  argument
1203 	len = sd->total_len;  in splice_direct_to_actor()
1204 	flags = sd->flags;  in splice_direct_to_actor()
1209 	sd->flags &= ~SPLICE_F_NONBLOCK;  in splice_direct_to_actor()
1210 	more = sd->flags & SPLICE_F_MORE;  in splice_direct_to_actor()
1214 		loff_t pos = sd->pos, prev_pos = pos;  in splice_direct_to_actor()
1221 		sd->total_len = read_len;  in splice_direct_to_actor()
1229 			sd->flags |= SPLICE_F_MORE;  in splice_direct_to_actor()
1231 			sd->flags &= ~SPLICE_F_MORE;  in splice_direct_to_actor()
1237 		ret = actor(pipe, sd);  in splice_direct_to_actor()
1239 			sd->pos = prev_pos;  in splice_direct_to_actor()
1245 		sd->pos = pos;  in splice_direct_to_actor()
1248 			sd->pos = prev_pos + ret;  in splice_direct_to_actor()
1280 			       struct splice_desc *sd)  in direct_splice_actor()  argument
1282 	struct file *file = sd->u.file;  in direct_splice_actor()
1284 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
1285 			      sd->flags);  in direct_splice_actor()
1307 	struct splice_desc sd = {  in do_splice_direct()  local
1327 	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);  in do_splice_direct()
1329 		*ppos = sd.pos;  in do_splice_direct()
1533 			struct splice_desc *sd)  in pipe_to_user()  argument
1535 	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);  in pipe_to_user()
1536 	return n == sd->len ? n : -EFAULT;  in pipe_to_user()
1547 	struct splice_desc sd;  in vmsplice_to_user()  local
1562 	sd.total_len = iov_iter_count(&iter);  in vmsplice_to_user()
1563 	sd.len = 0;  in vmsplice_to_user()
1564 	sd.flags = flags;  in vmsplice_to_user()
1565 	sd.u.data = &iter;  in vmsplice_to_user()
1566 	sd.pos = 0;  in vmsplice_to_user()
1568 	if (sd.total_len) {  in vmsplice_to_user()
1570 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()