4879 static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)  in sort_parity_stripes()  argument
4889 			if (parity_smaller(bbio->raid_map[i],  in sort_parity_stripes()
4890 					   bbio->raid_map[i+1])) {  in sort_parity_stripes()
4891 				s = bbio->stripes[i];  in sort_parity_stripes()
4892 				l = bbio->raid_map[i];  in sort_parity_stripes()
4893 				bbio->stripes[i] = bbio->stripes[i+1];  in sort_parity_stripes()
4894 				bbio->raid_map[i] = bbio->raid_map[i+1];  in sort_parity_stripes()
4895 				bbio->stripes[i+1] = s;  in sort_parity_stripes()
4896 				bbio->raid_map[i+1] = l;  in sort_parity_stripes()
4906 	struct btrfs_bio *bbio = kzalloc(  in alloc_btrfs_bio()  local
4919 	if (!bbio)  in alloc_btrfs_bio()
4922 	atomic_set(&bbio->error, 0);  in alloc_btrfs_bio()
4923 	atomic_set(&bbio->refs, 1);  in alloc_btrfs_bio()
4925 	return bbio;  in alloc_btrfs_bio()
4928 void btrfs_get_bbio(struct btrfs_bio *bbio)  in btrfs_get_bbio()  argument
4930 	WARN_ON(!atomic_read(&bbio->refs));  in btrfs_get_bbio()
4931 	atomic_inc(&bbio->refs);  in btrfs_get_bbio()
4934 void btrfs_put_bbio(struct btrfs_bio *bbio)  in btrfs_put_bbio()  argument
4936 	if (!bbio)  in btrfs_put_bbio()
4938 	if (atomic_dec_and_test(&bbio->refs))  in btrfs_put_bbio()
4939 		kfree(bbio);  in btrfs_put_bbio()
4964 	struct btrfs_bio *bbio = NULL;  in __btrfs_map_block()  local
5254 	bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);  in __btrfs_map_block()
5255 	if (!bbio) {  in __btrfs_map_block()
5260 		bbio->tgtdev_map = (int *)(bbio->stripes + num_alloc_stripes);  in __btrfs_map_block()
5269 		bbio->raid_map = (u64 *)((void *)bbio->stripes +  in __btrfs_map_block()
5280 			bbio->raid_map[(i+rot) % num_stripes] =  in __btrfs_map_block()
5283 		bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;  in __btrfs_map_block()
5285 			bbio->raid_map[(i+rot+1) % num_stripes] =  in __btrfs_map_block()
5313 			bbio->stripes[i].physical =  in __btrfs_map_block()
5316 			bbio->stripes[i].dev = map->stripes[stripe_index].dev;  in __btrfs_map_block()
5320 				bbio->stripes[i].length = stripes_per_dev *  in __btrfs_map_block()
5324 					bbio->stripes[i].length +=  in __btrfs_map_block()
5336 					bbio->stripes[i].length -=  in __btrfs_map_block()
5342 					bbio->stripes[i].length -=  in __btrfs_map_block()
5348 				bbio->stripes[i].length = *length;  in __btrfs_map_block()
5359 			bbio->stripes[i].physical =  in __btrfs_map_block()
5363 			bbio->stripes[i].dev =  in __btrfs_map_block()
5372 	if (bbio->raid_map)  in __btrfs_map_block()
5373 		sort_parity_stripes(bbio, num_stripes);  in __btrfs_map_block()
5394 			if (bbio->stripes[i].dev->devid == srcdev_devid) {  in __btrfs_map_block()
5397 					bbio->stripes + index_where_to_add;  in __btrfs_map_block()
5399 					bbio->stripes + i;  in __btrfs_map_block()
5404 				bbio->tgtdev_map[i] = index_where_to_add;  in __btrfs_map_block()
5426 			if (bbio->stripes[i].dev->devid == srcdev_devid) {  in __btrfs_map_block()
5434 				     bbio->stripes[i].physical)  in __btrfs_map_block()
5438 				physical_of_found = bbio->stripes[i].physical;  in __btrfs_map_block()
5445 					bbio->stripes + num_stripes;  in __btrfs_map_block()
5449 					bbio->stripes[index_srcdev].length;  in __btrfs_map_block()
5451 				bbio->tgtdev_map[index_srcdev] = num_stripes;  in __btrfs_map_block()
5459 	*bbio_ret = bbio;  in __btrfs_map_block()
5460 	bbio->map_type = map->type;  in __btrfs_map_block()
5461 	bbio->num_stripes = num_stripes;  in __btrfs_map_block()
5462 	bbio->max_errors = max_errors;  in __btrfs_map_block()
5463 	bbio->mirror_num = mirror_num;  in __btrfs_map_block()
5464 	bbio->num_tgtdevs = tgtdev_indexes;  in __btrfs_map_block()
5473 		bbio->stripes[0].dev = dev_replace->tgtdev;  in __btrfs_map_block()
5474 		bbio->stripes[0].physical = physical_to_patch_in_first_stripe;  in __btrfs_map_block()
5475 		bbio->mirror_num = map->num_stripes + 1;  in __btrfs_map_block()
5588 static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio, int err)  in btrfs_end_bbio()  argument
5590 	if (likely(bbio->flags & BTRFS_BIO_ORIG_BIO_SUBMITTED))  in btrfs_end_bbio()
5594 	btrfs_put_bbio(bbio);  in btrfs_end_bbio()
5599 	struct btrfs_bio *bbio = bio->bi_private;  in btrfs_end_bio()  local
5600 	struct btrfs_device *dev = bbio->stripes[0].dev;  in btrfs_end_bio()
5604 		atomic_inc(&bbio->error);  in btrfs_end_bio()
5609 			BUG_ON(stripe_index >= bbio->num_stripes);  in btrfs_end_bio()
5610 			dev = bbio->stripes[stripe_index].dev;  in btrfs_end_bio()
5626 	if (bio == bbio->orig_bio)  in btrfs_end_bio()
5629 	btrfs_bio_counter_dec(bbio->fs_info);  in btrfs_end_bio()
5631 	if (atomic_dec_and_test(&bbio->stripes_pending)) {  in btrfs_end_bio()
5634 			bio = bbio->orig_bio;  in btrfs_end_bio()
5637 		bio->bi_private = bbio->private;  in btrfs_end_bio()
5638 		bio->bi_end_io = bbio->end_io;  in btrfs_end_bio()
5639 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;  in btrfs_end_bio()
5643 		if (atomic_read(&bbio->error) > bbio->max_errors) {  in btrfs_end_bio()
5654 		btrfs_end_bbio(bbio, bio, err);  in btrfs_end_bio()
5748 static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,  in submit_stripe_bio()  argument
5752 	struct btrfs_device *dev = bbio->stripes[dev_nr].dev;  in submit_stripe_bio()
5754 	bio->bi_private = bbio;  in submit_stripe_bio()
5781 static int breakup_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,  in breakup_stripe_bio()  argument
5788 	u64 physical = bbio->stripes[dev_nr].physical;  in breakup_stripe_bio()
5800 			atomic_inc(&bbio->stripes_pending);  in breakup_stripe_bio()
5801 			submit_stripe_bio(root, bbio, bio, physical, dev_nr,  in breakup_stripe_bio()
5809 	submit_stripe_bio(root, bbio, bio, physical, dev_nr, rw, async);  in breakup_stripe_bio()
5813 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)  in bbio_error()  argument
5815 	atomic_inc(&bbio->error);  in bbio_error()
5816 	if (atomic_dec_and_test(&bbio->stripes_pending)) {  in bbio_error()
5818 		WARN_ON(bio != bbio->orig_bio);  in bbio_error()
5820 		bio->bi_private = bbio->private;  in bbio_error()
5821 		bio->bi_end_io = bbio->end_io;  in bbio_error()
5822 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;  in bbio_error()
5825 		btrfs_end_bbio(bbio, bio, -EIO);  in bbio_error()
5840 	struct btrfs_bio *bbio = NULL;  in btrfs_map_bio()  local
5846 	ret = __btrfs_map_block(root->fs_info, rw, logical, &map_length, &bbio,  in btrfs_map_bio()
5853 	total_devs = bbio->num_stripes;  in btrfs_map_bio()
5854 	bbio->orig_bio = first_bio;  in btrfs_map_bio()
5855 	bbio->private = first_bio->bi_private;  in btrfs_map_bio()
5856 	bbio->end_io = first_bio->bi_end_io;  in btrfs_map_bio()
5857 	bbio->fs_info = root->fs_info;  in btrfs_map_bio()
5858 	atomic_set(&bbio->stripes_pending, bbio->num_stripes);  in btrfs_map_bio()
5860 	if (bbio->raid_map) {  in btrfs_map_bio()
5864 			ret = raid56_parity_write(root, bio, bbio, map_length);  in btrfs_map_bio()
5866 			ret = raid56_parity_recover(root, bio, bbio, map_length,  in btrfs_map_bio()
5881 		dev = bbio->stripes[dev_nr].dev;  in btrfs_map_bio()
5883 			bbio_error(bbio, first_bio, logical);  in btrfs_map_bio()
5892 				 bbio->stripes[dev_nr].physical >> 9)) {  in btrfs_map_bio()
5893 			ret = breakup_stripe_bio(root, bbio, first_bio, dev,  in btrfs_map_bio()
5904 			bbio->flags |= BTRFS_BIO_ORIG_BIO_SUBMITTED;  in btrfs_map_bio()
5907 		submit_stripe_bio(root, bbio, bio,  in btrfs_map_bio()
5908 				  bbio->stripes[dev_nr].physical, dev_nr, rw,  in btrfs_map_bio()