536 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)  in fuse_get_res_by_io()  argument
538 	if (io->err)  in fuse_get_res_by_io()
539 		return io->err;  in fuse_get_res_by_io()
541 	if (io->bytes >= 0 && io->write)  in fuse_get_res_by_io()
544 	return io->bytes < 0 ? io->size : io->bytes;  in fuse_get_res_by_io()
563 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)  in fuse_aio_complete()  argument
565 	bool is_sync = is_sync_kiocb(io->iocb);  in fuse_aio_complete()
568 	spin_lock(&io->lock);  in fuse_aio_complete()
570 		io->err = io->err ? : err;  in fuse_aio_complete()
571 	else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))  in fuse_aio_complete()
572 		io->bytes = pos;  in fuse_aio_complete()
574 	left = --io->reqs;  in fuse_aio_complete()
576 		complete(io->done);  in fuse_aio_complete()
577 	spin_unlock(&io->lock);  in fuse_aio_complete()
580 		ssize_t res = fuse_get_res_by_io(io);  in fuse_aio_complete()
583 			struct inode *inode = file_inode(io->iocb->ki_filp);  in fuse_aio_complete()
592 		io->iocb->ki_complete(io->iocb, res, 0);  in fuse_aio_complete()
595 	kref_put(&io->refcnt, fuse_io_release);  in fuse_aio_complete()
600 	struct fuse_io_priv *io = req->io;  in fuse_aio_complete_req()  local
603 	fuse_release_user_pages(req, !io->write);  in fuse_aio_complete_req()
605 	if (io->write) {  in fuse_aio_complete_req()
607 			pos = req->misc.write.in.offset - io->offset +  in fuse_aio_complete_req()
611 			pos = req->misc.read.in.offset - io->offset +  in fuse_aio_complete_req()
615 	fuse_aio_complete(io, req->out.h.error, pos);  in fuse_aio_complete_req()
619 		size_t num_bytes, struct fuse_io_priv *io)  in fuse_async_req_send()  argument
621 	spin_lock(&io->lock);  in fuse_async_req_send()
622 	kref_get(&io->refcnt);  in fuse_async_req_send()
623 	io->size += num_bytes;  in fuse_async_req_send()
624 	io->reqs++;  in fuse_async_req_send()
625 	spin_unlock(&io->lock);  in fuse_async_req_send()
627 	req->io = io;  in fuse_async_req_send()
636 static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io,  in fuse_send_read()  argument
639 	struct file *file = io->file;  in fuse_send_read()
651 	if (io->async)  in fuse_send_read()
652 		return fuse_async_req_send(fc, req, count, io);  in fuse_send_read()
701 	struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(file);  in fuse_do_readpage()  local
729 	num_read = fuse_send_read(req, &io, pos, count, NULL);  in fuse_do_readpage()
948 static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io,  in fuse_send_write()  argument
951 	struct file *file = io->file;  in fuse_send_write()
963 	if (io->async)  in fuse_send_write()
964 		return fuse_async_req_send(fc, req, count, io);  in fuse_send_write()
994 	struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(file);  in fuse_send_write_pages()  local
999 	res = fuse_send_write(req, &io, pos, count, NULL);  in fuse_send_write_pages()
1306 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,  in fuse_direct_io()  argument
1311 	struct file *file = io->file;  in fuse_direct_io()
1323 	if (io->async)  in fuse_direct_io()
1349 			nres = fuse_send_write(req, io, pos, nbytes, owner);  in fuse_direct_io()
1351 			nres = fuse_send_read(req, io, pos, nbytes, owner);  in fuse_direct_io()
1353 		if (!io->async)  in fuse_direct_io()
1370 			if (io->async)  in fuse_direct_io()
1388 static ssize_t __fuse_direct_read(struct fuse_io_priv *io,  in __fuse_direct_read()  argument
1393 	struct file *file = io->file;  in __fuse_direct_read()
1399 	res = fuse_direct_io(io, iter, ppos, 0);  in __fuse_direct_read()
1408 	struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb->ki_filp);  in fuse_direct_read_iter()  local
1409 	return __fuse_direct_read(&io, to, &iocb->ki_pos);  in fuse_direct_read_iter()
1416 	struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(file);  in fuse_direct_write_iter()  local
1426 		res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);  in fuse_direct_write_iter()
2795 	struct fuse_io_priv *io;  in fuse_direct_IO()  local
2813 	io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);  in fuse_direct_IO()
2814 	if (!io)  in fuse_direct_IO()
2816 	spin_lock_init(&io->lock);  in fuse_direct_IO()
2817 	kref_init(&io->refcnt);  in fuse_direct_IO()
2818 	io->reqs = 1;  in fuse_direct_IO()
2819 	io->bytes = -1;  in fuse_direct_IO()
2820 	io->size = 0;  in fuse_direct_IO()
2821 	io->offset = offset;  in fuse_direct_IO()
2822 	io->write = (iov_iter_rw(iter) == WRITE);  in fuse_direct_IO()
2823 	io->err = 0;  in fuse_direct_IO()
2824 	io->file = file;  in fuse_direct_IO()
2829 	io->async = async_dio;  in fuse_direct_IO()
2830 	io->iocb = iocb;  in fuse_direct_IO()
2839 		io->async = false;  in fuse_direct_IO()
2841 	if (io->async && is_sync) {  in fuse_direct_IO()
2846 		kref_get(&io->refcnt);  in fuse_direct_IO()
2847 		io->done = &wait;  in fuse_direct_IO()
2851 		ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);  in fuse_direct_IO()
2854 		ret = __fuse_direct_read(io, iter, &pos);  in fuse_direct_IO()
2857 	if (io->async) {  in fuse_direct_IO()
2858 		fuse_aio_complete(io, ret < 0 ? ret : 0, -1);  in fuse_direct_IO()
2865 		ret = fuse_get_res_by_io(io);  in fuse_direct_IO()
2868 	kref_put(&io->refcnt, fuse_io_release);  in fuse_direct_IO()