1805 			 size_t size, struct bio *bio,  in btrfs_merge_bio_hook()  argument
1809 	u64 logical = (u64)bio->bi_iter.bi_sector << 9;  in btrfs_merge_bio_hook()
1817 	length = bio->bi_iter.bi_size;  in btrfs_merge_bio_hook()
1837 				    struct bio *bio, int mirror_num,  in __btrfs_submit_bio_start()  argument
1844 	ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);  in __btrfs_submit_bio_start()
1857 static int __btrfs_submit_bio_done(struct inode *inode, int rw, struct bio *bio,  in __btrfs_submit_bio_done()  argument
1864 	ret = btrfs_map_bio(root, rw, bio, mirror_num, 1);  in __btrfs_submit_bio_done()
1866 		bio->bi_error = ret;  in __btrfs_submit_bio_done()
1867 		bio_endio(bio);  in __btrfs_submit_bio_done()
1876 static int btrfs_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,  in btrfs_submit_bio_hook()  argument
1892 		ret = btrfs_bio_wq_end_io(root->fs_info, bio, metadata);  in btrfs_submit_bio_hook()
1897 			ret = btrfs_submit_compressed_read(inode, bio,  in btrfs_submit_bio_hook()
1902 			ret = btrfs_lookup_bio_sums(root, inode, bio, NULL);  in btrfs_submit_bio_hook()
1913 				   inode, rw, bio, mirror_num,  in btrfs_submit_bio_hook()
1919 		ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);  in btrfs_submit_bio_hook()
1925 	ret = btrfs_map_bio(root, rw, bio, mirror_num, 0);  in btrfs_submit_bio_hook()
1929 		bio->bi_error = ret;  in btrfs_submit_bio_hook()
1930 		bio_endio(bio);  in btrfs_submit_bio_hook()
7720 static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio,  in submit_dio_repair_bio()  argument
7728 	bio_get(bio);  in submit_dio_repair_bio()
7730 	ret = btrfs_bio_wq_end_io(root->fs_info, bio,  in submit_dio_repair_bio()
7735 	ret = btrfs_map_bio(root, rw, bio, mirror_num, 0);  in submit_dio_repair_bio()
7737 	bio_put(bio);  in submit_dio_repair_bio()
7742 				      struct bio *failed_bio,  in btrfs_check_dio_repairable()
7775 static int dio_read_error(struct inode *inode, struct bio *failed_bio,  in dio_read_error()
7781 	struct bio *bio;  in dio_read_error()  local
7806 	bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,  in dio_read_error()
7808 	if (!bio) {  in dio_read_error()
7817 	ret = submit_dio_repair_bio(inode, bio, read_mode,  in dio_read_error()
7821 		bio_put(bio);  in dio_read_error()
7834 static void btrfs_retry_endio_nocsum(struct bio *bio)  in btrfs_retry_endio_nocsum()  argument
7836 	struct btrfs_retry_complete *done = bio->bi_private;  in btrfs_retry_endio_nocsum()
7840 	if (bio->bi_error)  in btrfs_retry_endio_nocsum()
7844 	bio_for_each_segment_all(bvec, bio, i)  in btrfs_retry_endio_nocsum()
7848 	bio_put(bio);  in btrfs_retry_endio_nocsum()
7863 	bio_for_each_segment_all(bvec, &io_bio->bio, i) {  in __btrfs_correct_data_nocsum()
7869 		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, start,  in __btrfs_correct_data_nocsum()
7889 static void btrfs_retry_endio(struct bio *bio)  in btrfs_retry_endio()  argument
7891 	struct btrfs_retry_complete *done = bio->bi_private;  in btrfs_retry_endio()
7892 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);  in btrfs_retry_endio()
7898 	if (bio->bi_error)  in btrfs_retry_endio()
7902 	bio_for_each_segment_all(bvec, bio, i) {  in btrfs_retry_endio()
7916 	bio_put(bio);  in btrfs_retry_endio()
7933 	bio_for_each_segment_all(bvec, &io_bio->bio, i) {  in __btrfs_subio_endio_read()
7943 		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page, start,  in __btrfs_subio_endio_read()
7981 static void btrfs_endio_direct_read(struct bio *bio)  in btrfs_endio_direct_read()  argument
7983 	struct btrfs_dio_private *dip = bio->bi_private;  in btrfs_endio_direct_read()
7985 	struct bio *dio_bio;  in btrfs_endio_direct_read()
7986 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);  in btrfs_endio_direct_read()
7987 	int err = bio->bi_error;  in btrfs_endio_direct_read()
7998 	dio_bio->bi_error = bio->bi_error;  in btrfs_endio_direct_read()
7999 	dio_end_io(dio_bio, bio->bi_error);  in btrfs_endio_direct_read()
8003 	bio_put(bio);  in btrfs_endio_direct_read()
8006 static void btrfs_endio_direct_write(struct bio *bio)  in btrfs_endio_direct_write()  argument
8008 	struct btrfs_dio_private *dip = bio->bi_private;  in btrfs_endio_direct_write()
8014 	struct bio *dio_bio;  in btrfs_endio_direct_write()
8021 						   !bio->bi_error);  in btrfs_endio_direct_write()
8044 	dio_bio->bi_error = bio->bi_error;  in btrfs_endio_direct_write()
8045 	dio_end_io(dio_bio, bio->bi_error);  in btrfs_endio_direct_write()
8046 	bio_put(bio);  in btrfs_endio_direct_write()
8050 				    struct bio *bio, int mirror_num,  in __btrfs_submit_bio_start_direct_io()  argument
8055 	ret = btrfs_csum_one_bio(root, inode, bio, offset, 1);  in __btrfs_submit_bio_start_direct_io()
8060 static void btrfs_end_dio_bio(struct bio *bio)  in btrfs_end_dio_bio()  argument
8062 	struct btrfs_dio_private *dip = bio->bi_private;  in btrfs_end_dio_bio()
8063 	int err = bio->bi_error;  in btrfs_end_dio_bio()
8068 			   btrfs_ino(dip->inode), bio->bi_rw,  in btrfs_end_dio_bio()
8069 			   (unsigned long long)bio->bi_iter.bi_sector,  in btrfs_end_dio_bio()
8070 			   bio->bi_iter.bi_size, err);  in btrfs_end_dio_bio()
8073 		err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);  in btrfs_end_dio_bio()
8096 	bio_put(bio);  in btrfs_end_dio_bio()
8099 static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev,  in btrfs_dio_bio_alloc()
8102 	struct bio *bio;  in btrfs_dio_bio_alloc()  local
8103 	bio = btrfs_bio_alloc(bdev, first_sector, BIO_MAX_PAGES, gfp_flags);  in btrfs_dio_bio_alloc()
8104 	if (bio)  in btrfs_dio_bio_alloc()
8105 		bio_associate_current(bio);  in btrfs_dio_bio_alloc()
8106 	return bio;  in btrfs_dio_bio_alloc()
8112 						 struct bio *bio,  in btrfs_lookup_and_bind_dio_csum()  argument
8115 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);  in btrfs_lookup_and_bind_dio_csum()
8131 	if (bio == dip->orig_bio)  in btrfs_lookup_and_bind_dio_csum()
8141 static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode,  in __btrfs_submit_dio_bio()  argument
8145 	struct btrfs_dio_private *dip = bio->bi_private;  in __btrfs_submit_dio_bio()
8153 	bio_get(bio);  in __btrfs_submit_dio_bio()
8156 		ret = btrfs_bio_wq_end_io(root->fs_info, bio,  in __btrfs_submit_dio_bio()
8167 				   inode, rw, bio, 0, 0,  in __btrfs_submit_dio_bio()
8177 		ret = btrfs_csum_one_bio(root, inode, bio, file_offset, 1);  in __btrfs_submit_dio_bio()
8181 		ret = btrfs_lookup_and_bind_dio_csum(root, inode, dip, bio,  in __btrfs_submit_dio_bio()
8187 	ret = btrfs_map_bio(root, rw, bio, 0, async_submit);  in __btrfs_submit_dio_bio()
8189 	bio_put(bio);  in __btrfs_submit_dio_bio()
8198 	struct bio *bio;  in btrfs_submit_direct_hook()  local
8199 	struct bio *orig_bio = dip->orig_bio;  in btrfs_submit_direct_hook()
8216 		bio = orig_bio;  in btrfs_submit_direct_hook()
8227 	bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, start_sector, GFP_NOFS);  in btrfs_submit_direct_hook()
8228 	if (!bio)  in btrfs_submit_direct_hook()
8231 	bio->bi_private = dip;  in btrfs_submit_direct_hook()
8232 	bio->bi_end_io = btrfs_end_dio_bio;  in btrfs_submit_direct_hook()
8233 	btrfs_io_bio(bio)->logical = file_offset;  in btrfs_submit_direct_hook()
8238 		    bio_add_page(bio, bvec->bv_page, bvec->bv_len,  in btrfs_submit_direct_hook()
8247 			ret = __btrfs_submit_dio_bio(bio, inode, rw,  in btrfs_submit_direct_hook()
8251 				bio_put(bio);  in btrfs_submit_direct_hook()
8262 			bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev,  in btrfs_submit_direct_hook()
8264 			if (!bio)  in btrfs_submit_direct_hook()
8266 			bio->bi_private = dip;  in btrfs_submit_direct_hook()
8267 			bio->bi_end_io = btrfs_end_dio_bio;  in btrfs_submit_direct_hook()
8268 			btrfs_io_bio(bio)->logical = file_offset;  in btrfs_submit_direct_hook()
8275 				bio_put(bio);  in btrfs_submit_direct_hook()
8286 	ret = __btrfs_submit_dio_bio(bio, inode, rw, file_offset, skip_sum,  in btrfs_submit_direct_hook()
8291 	bio_put(bio);  in btrfs_submit_direct_hook()
8306 static void btrfs_submit_direct(int rw, struct bio *dio_bio,  in btrfs_submit_direct()
8310 	struct bio *io_bio = NULL;  in btrfs_submit_direct()