43 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_steal()  argument
88 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_release()  argument
99 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_confirm()  argument
145 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,  in user_page_pipe_buf_steal()  argument
152 	return generic_pipe_buf_steal(pipe, buf);  in user_page_pipe_buf_steal()
163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)  in wakeup_pipe_readers()  argument
166 	if (waitqueue_active(&pipe->wait))  in wakeup_pipe_readers()
167 		wake_up_interruptible(&pipe->wait);  in wakeup_pipe_readers()
168 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in wakeup_pipe_readers()
182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,  in splice_to_pipe()  argument
195 	pipe_lock(pipe);  in splice_to_pipe()
198 		if (!pipe->readers) {  in splice_to_pipe()
205 		if (pipe->nrbufs < pipe->buffers) {  in splice_to_pipe()
206 			int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);  in splice_to_pipe()
207 			struct pipe_buffer *buf = pipe->bufs + newbuf;  in splice_to_pipe()
217 			pipe->nrbufs++;  in splice_to_pipe()
221 			if (pipe->files)  in splice_to_pipe()
226 			if (pipe->nrbufs < pipe->buffers)  in splice_to_pipe()
246 			if (waitqueue_active(&pipe->wait))  in splice_to_pipe()
247 				wake_up_interruptible_sync(&pipe->wait);  in splice_to_pipe()
248 			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in splice_to_pipe()
252 		pipe->waiting_writers++;  in splice_to_pipe()
253 		pipe_wait(pipe);  in splice_to_pipe()
254 		pipe->waiting_writers--;  in splice_to_pipe()
257 	pipe_unlock(pipe);  in splice_to_pipe()
260 		wakeup_pipe_readers(pipe);  in splice_to_pipe()
278 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)  in splice_grow_spd()  argument
280 	unsigned int buffers = ACCESS_ONCE(pipe->buffers);  in splice_grow_spd()
308 			   struct pipe_inode_info *pipe, size_t len,  in __generic_file_splice_read()  argument
328 	if (splice_grow_spd(pipe, &spd))  in __generic_file_splice_read()
503 		error = splice_to_pipe(pipe, &spd);  in __generic_file_splice_read()
524 				 struct pipe_inode_info *pipe, size_t len,  in generic_file_splice_read()  argument
531 		return default_file_splice_read(in, ppos, pipe, len, flags);  in generic_file_splice_read()
541 	ret = __generic_file_splice_read(in, ppos, pipe, len, flags);  in generic_file_splice_read()
559 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,  in generic_pipe_buf_nosteal()  argument
608 				 struct pipe_inode_info *pipe, size_t len,  in default_file_splice_read()  argument
630 	if (splice_grow_spd(pipe, &spd))  in default_file_splice_read()
685 	res = splice_to_pipe(pipe, &spd);  in default_file_splice_read()
708 static int pipe_to_sendpage(struct pipe_inode_info *pipe,  in pipe_to_sendpage()  argument
720 	if (sd->len < sd->total_len && pipe->nrbufs > 1)  in pipe_to_sendpage()
727 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)  in wakeup_pipe_writers()  argument
730 	if (waitqueue_active(&pipe->wait))  in wakeup_pipe_writers()
731 		wake_up_interruptible(&pipe->wait);  in wakeup_pipe_writers()
732 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in wakeup_pipe_writers()
755 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
760 	while (pipe->nrbufs) {  in splice_from_pipe_feed()
761 		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;  in splice_from_pipe_feed()
768 		ret = buf->ops->confirm(pipe, buf);  in splice_from_pipe_feed()
775 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
789 			ops->release(pipe, buf);  in splice_from_pipe_feed()
790 			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);  in splice_from_pipe_feed()
791 			pipe->nrbufs--;  in splice_from_pipe_feed()
792 			if (pipe->files)  in splice_from_pipe_feed()
813 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
822 	while (!pipe->nrbufs) {  in splice_from_pipe_next()
823 		if (!pipe->writers)  in splice_from_pipe_next()
826 		if (!pipe->waiting_writers && sd->num_spliced)  in splice_from_pipe_next()
836 			wakeup_pipe_writers(pipe);  in splice_from_pipe_next()
840 		pipe_wait(pipe);  in splice_from_pipe_next()
871 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
874 		wakeup_pipe_writers(pipe);  in splice_from_pipe_end()
890 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
898 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
900 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
902 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
922 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,  in splice_from_pipe()  argument
934 	pipe_lock(pipe);  in splice_from_pipe()
935 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
936 	pipe_unlock(pipe);  in splice_from_pipe()
956 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,  in iter_file_splice_write()  argument
965 	int nbufs = pipe->buffers;  in iter_file_splice_write()
973 	pipe_lock(pipe);  in iter_file_splice_write()
981 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
985 		if (unlikely(nbufs < pipe->buffers)) {  in iter_file_splice_write()
987 			nbufs = pipe->buffers;  in iter_file_splice_write()
998 		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {  in iter_file_splice_write()
999 			struct pipe_buffer *buf = pipe->bufs + idx;  in iter_file_splice_write()
1005 			if (idx == pipe->buffers - 1)  in iter_file_splice_write()
1008 			ret = buf->ops->confirm(pipe, buf);  in iter_file_splice_write()
1033 			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;  in iter_file_splice_write()
1039 				ops->release(pipe, buf);  in iter_file_splice_write()
1040 				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);  in iter_file_splice_write()
1041 				pipe->nrbufs--;  in iter_file_splice_write()
1042 				if (pipe->files)  in iter_file_splice_write()
1053 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
1055 	pipe_unlock(pipe);  in iter_file_splice_write()
1065 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in write_pipe_buf()  argument
1079 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,  in default_file_splice_write()  argument
1085 	ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);  in default_file_splice_write()
1105 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,  in generic_splice_sendpage()  argument
1108 	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);  in generic_splice_sendpage()
1116 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,  in do_splice_from()  argument
1127 	return splice_write(pipe, out, ppos, len, flags);  in do_splice_from()
1134 			 struct pipe_inode_info *pipe, size_t len,  in do_splice_to()  argument
1153 	return splice_read(in, ppos, pipe, len, flags);  in do_splice_to()
1172 	struct pipe_inode_info *pipe;  in splice_direct_to_actor()  local
1191 	pipe = current->splice_pipe;  in splice_direct_to_actor()
1192 	if (unlikely(!pipe)) {  in splice_direct_to_actor()
1193 		pipe = alloc_pipe_info();  in splice_direct_to_actor()
1194 		if (!pipe)  in splice_direct_to_actor()
1202 		pipe->readers = 1;  in splice_direct_to_actor()
1204 		current->splice_pipe = pipe;  in splice_direct_to_actor()
1225 		ret = do_splice_to(in, &pos, pipe, len, flags);  in splice_direct_to_actor()
1246 		ret = actor(pipe, sd);  in splice_direct_to_actor()
1263 	pipe->nrbufs = pipe->curbuf = 0;  in splice_direct_to_actor()
1272 	for (i = 0; i < pipe->buffers; i++) {  in splice_direct_to_actor()
1273 		struct pipe_buffer *buf = pipe->bufs + i;  in splice_direct_to_actor()
1276 			buf->ops->release(pipe, buf);  in splice_direct_to_actor()
1288 static int direct_splice_actor(struct pipe_inode_info *pipe,  in direct_splice_actor()  argument
1293 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
1541 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in pipe_to_user()  argument
1555 	struct pipe_inode_info *pipe;  in vmsplice_to_user()  local
1562 	pipe = get_pipe_info(file);  in vmsplice_to_user()
1563 	if (!pipe)  in vmsplice_to_user()
1578 		pipe_lock(pipe);  in vmsplice_to_user()
1579 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()
1580 		pipe_unlock(pipe);  in vmsplice_to_user()
1595 	struct pipe_inode_info *pipe;  in vmsplice_to_pipe()  local
1608 	pipe = get_pipe_info(file);  in vmsplice_to_pipe()
1609 	if (!pipe)  in vmsplice_to_pipe()
1612 	if (splice_grow_spd(pipe, &spd))  in vmsplice_to_pipe()
1621 		ret = splice_to_pipe(pipe, &spd);  in vmsplice_to_pipe()
1721 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in ipipe_prep()  argument
1729 	if (pipe->nrbufs)  in ipipe_prep()
1733 	pipe_lock(pipe);  in ipipe_prep()
1735 	while (!pipe->nrbufs) {  in ipipe_prep()
1740 		if (!pipe->writers)  in ipipe_prep()
1742 		if (!pipe->waiting_writers) {  in ipipe_prep()
1748 		pipe_wait(pipe);  in ipipe_prep()
1751 	pipe_unlock(pipe);  in ipipe_prep()
1759 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in opipe_prep()  argument
1767 	if (pipe->nrbufs < pipe->buffers)  in opipe_prep()
1771 	pipe_lock(pipe);  in opipe_prep()
1773 	while (pipe->nrbufs >= pipe->buffers) {  in opipe_prep()
1774 		if (!pipe->readers) {  in opipe_prep()
1787 		pipe->waiting_writers++;  in opipe_prep()
1788 		pipe_wait(pipe);  in opipe_prep()
1789 		pipe->waiting_writers--;  in opipe_prep()
1792 	pipe_unlock(pipe);  in opipe_prep()