43 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_steal()  argument
88 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_release()  argument
99 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_confirm()  argument
145 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,  in user_page_pipe_buf_steal()  argument
152 	return generic_pipe_buf_steal(pipe, buf);  in user_page_pipe_buf_steal()
163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)  in wakeup_pipe_readers()  argument
166 	if (waitqueue_active(&pipe->wait))  in wakeup_pipe_readers()
167 		wake_up_interruptible(&pipe->wait);  in wakeup_pipe_readers()
168 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in wakeup_pipe_readers()
182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,  in splice_to_pipe()  argument
195 	pipe_lock(pipe);  in splice_to_pipe()
198 		if (!pipe->readers) {  in splice_to_pipe()
205 		if (pipe->nrbufs < pipe->buffers) {  in splice_to_pipe()
206 			int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);  in splice_to_pipe()
207 			struct pipe_buffer *buf = pipe->bufs + newbuf;  in splice_to_pipe()
217 			pipe->nrbufs++;  in splice_to_pipe()
221 			if (pipe->files)  in splice_to_pipe()
226 			if (pipe->nrbufs < pipe->buffers)  in splice_to_pipe()
246 			if (waitqueue_active(&pipe->wait))  in splice_to_pipe()
247 				wake_up_interruptible_sync(&pipe->wait);  in splice_to_pipe()
248 			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in splice_to_pipe()
252 		pipe->waiting_writers++;  in splice_to_pipe()
253 		pipe_wait(pipe);  in splice_to_pipe()
254 		pipe->waiting_writers--;  in splice_to_pipe()
257 	pipe_unlock(pipe);  in splice_to_pipe()
260 		wakeup_pipe_readers(pipe);  in splice_to_pipe()
277 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)  in splice_grow_spd()  argument
279 	unsigned int buffers = ACCESS_ONCE(pipe->buffers);  in splice_grow_spd()
307 			   struct pipe_inode_info *pipe, size_t len,  in __generic_file_splice_read()  argument
327 	if (splice_grow_spd(pipe, &spd))  in __generic_file_splice_read()
502 		error = splice_to_pipe(pipe, &spd);  in __generic_file_splice_read()
523 				 struct pipe_inode_info *pipe, size_t len,  in generic_file_splice_read()  argument
530 		return default_file_splice_read(in, ppos, pipe, len, flags);  in generic_file_splice_read()
540 	ret = __generic_file_splice_read(in, ppos, pipe, len, flags);  in generic_file_splice_read()
558 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,  in generic_pipe_buf_nosteal()  argument
607 				 struct pipe_inode_info *pipe, size_t len,  in default_file_splice_read()  argument
629 	if (splice_grow_spd(pipe, &spd))  in default_file_splice_read()
684 	res = splice_to_pipe(pipe, &spd);  in default_file_splice_read()
707 static int pipe_to_sendpage(struct pipe_inode_info *pipe,  in pipe_to_sendpage()  argument
719 	if (sd->len < sd->total_len && pipe->nrbufs > 1)  in pipe_to_sendpage()
726 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)  in wakeup_pipe_writers()  argument
729 	if (waitqueue_active(&pipe->wait))  in wakeup_pipe_writers()
730 		wake_up_interruptible(&pipe->wait);  in wakeup_pipe_writers()
731 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in wakeup_pipe_writers()
754 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
759 	while (pipe->nrbufs) {  in splice_from_pipe_feed()
760 		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;  in splice_from_pipe_feed()
767 		ret = buf->ops->confirm(pipe, buf);  in splice_from_pipe_feed()
774 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
788 			ops->release(pipe, buf);  in splice_from_pipe_feed()
789 			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);  in splice_from_pipe_feed()
790 			pipe->nrbufs--;  in splice_from_pipe_feed()
791 			if (pipe->files)  in splice_from_pipe_feed()
812 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
814 	while (!pipe->nrbufs) {  in splice_from_pipe_next()
815 		if (!pipe->writers)  in splice_from_pipe_next()
818 		if (!pipe->waiting_writers && sd->num_spliced)  in splice_from_pipe_next()
828 			wakeup_pipe_writers(pipe);  in splice_from_pipe_next()
832 		pipe_wait(pipe);  in splice_from_pipe_next()
863 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
866 		wakeup_pipe_writers(pipe);  in splice_from_pipe_end()
882 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
889 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
891 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
893 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
913 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,  in splice_from_pipe()  argument
925 	pipe_lock(pipe);  in splice_from_pipe()
926 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
927 	pipe_unlock(pipe);  in splice_from_pipe()
947 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,  in iter_file_splice_write()  argument
956 	int nbufs = pipe->buffers;  in iter_file_splice_write()
964 	pipe_lock(pipe);  in iter_file_splice_write()
972 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
976 		if (unlikely(nbufs < pipe->buffers)) {  in iter_file_splice_write()
978 			nbufs = pipe->buffers;  in iter_file_splice_write()
989 		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {  in iter_file_splice_write()
990 			struct pipe_buffer *buf = pipe->bufs + idx;  in iter_file_splice_write()
996 			if (idx == pipe->buffers - 1)  in iter_file_splice_write()
999 			ret = buf->ops->confirm(pipe, buf);  in iter_file_splice_write()
1024 			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;  in iter_file_splice_write()
1030 				ops->release(pipe, buf);  in iter_file_splice_write()
1031 				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);  in iter_file_splice_write()
1032 				pipe->nrbufs--;  in iter_file_splice_write()
1033 				if (pipe->files)  in iter_file_splice_write()
1044 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
1046 	pipe_unlock(pipe);  in iter_file_splice_write()
1056 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in write_pipe_buf()  argument
1070 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,  in default_file_splice_write()  argument
1076 	ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);  in default_file_splice_write()
1096 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,  in generic_splice_sendpage()  argument
1099 	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);  in generic_splice_sendpage()
1107 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,  in do_splice_from()  argument
1118 	return splice_write(pipe, out, ppos, len, flags);  in do_splice_from()
1125 			 struct pipe_inode_info *pipe, size_t len,  in do_splice_to()  argument
1144 	return splice_read(in, ppos, pipe, len, flags);  in do_splice_to()
1163 	struct pipe_inode_info *pipe;  in splice_direct_to_actor()  local
1182 	pipe = current->splice_pipe;  in splice_direct_to_actor()
1183 	if (unlikely(!pipe)) {  in splice_direct_to_actor()
1184 		pipe = alloc_pipe_info();  in splice_direct_to_actor()
1185 		if (!pipe)  in splice_direct_to_actor()
1193 		pipe->readers = 1;  in splice_direct_to_actor()
1195 		current->splice_pipe = pipe;  in splice_direct_to_actor()
1216 		ret = do_splice_to(in, &pos, pipe, len, flags);  in splice_direct_to_actor()
1237 		ret = actor(pipe, sd);  in splice_direct_to_actor()
1254 	pipe->nrbufs = pipe->curbuf = 0;  in splice_direct_to_actor()
1263 	for (i = 0; i < pipe->buffers; i++) {  in splice_direct_to_actor()
1264 		struct pipe_buffer *buf = pipe->bufs + i;  in splice_direct_to_actor()
1267 			buf->ops->release(pipe, buf);  in splice_direct_to_actor()
1279 static int direct_splice_actor(struct pipe_inode_info *pipe,  in direct_splice_actor()  argument
1284 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
1532 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in pipe_to_user()  argument
1546 	struct pipe_inode_info *pipe;  in vmsplice_to_user()  local
1553 	pipe = get_pipe_info(file);  in vmsplice_to_user()
1554 	if (!pipe)  in vmsplice_to_user()
1569 		pipe_lock(pipe);  in vmsplice_to_user()
1570 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()
1571 		pipe_unlock(pipe);  in vmsplice_to_user()
1586 	struct pipe_inode_info *pipe;  in vmsplice_to_pipe()  local
1599 	pipe = get_pipe_info(file);  in vmsplice_to_pipe()
1600 	if (!pipe)  in vmsplice_to_pipe()
1603 	if (splice_grow_spd(pipe, &spd))  in vmsplice_to_pipe()
1612 		ret = splice_to_pipe(pipe, &spd);  in vmsplice_to_pipe()
1712 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in ipipe_prep()  argument
1720 	if (pipe->nrbufs)  in ipipe_prep()
1724 	pipe_lock(pipe);  in ipipe_prep()
1726 	while (!pipe->nrbufs) {  in ipipe_prep()
1731 		if (!pipe->writers)  in ipipe_prep()
1733 		if (!pipe->waiting_writers) {  in ipipe_prep()
1739 		pipe_wait(pipe);  in ipipe_prep()
1742 	pipe_unlock(pipe);  in ipipe_prep()
1750 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in opipe_prep()  argument
1758 	if (pipe->nrbufs < pipe->buffers)  in opipe_prep()
1762 	pipe_lock(pipe);  in opipe_prep()
1764 	while (pipe->nrbufs >= pipe->buffers) {  in opipe_prep()
1765 		if (!pipe->readers) {  in opipe_prep()
1778 		pipe->waiting_writers++;  in opipe_prep()
1779 		pipe_wait(pipe);  in opipe_prep()
1780 		pipe->waiting_writers--;  in opipe_prep()
1783 	pipe_unlock(pipe);  in opipe_prep()