5147 static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)  in sort_parity_stripes()  argument
5157 			if (parity_smaller(bbio->raid_map[i],  in sort_parity_stripes()
5158 					   bbio->raid_map[i+1])) {  in sort_parity_stripes()
5159 				s = bbio->stripes[i];  in sort_parity_stripes()
5160 				l = bbio->raid_map[i];  in sort_parity_stripes()
5161 				bbio->stripes[i] = bbio->stripes[i+1];  in sort_parity_stripes()
5162 				bbio->raid_map[i] = bbio->raid_map[i+1];  in sort_parity_stripes()
5163 				bbio->stripes[i+1] = s;  in sort_parity_stripes()
5164 				bbio->raid_map[i+1] = l;  in sort_parity_stripes()
5174 	struct btrfs_bio *bbio = kzalloc(  in alloc_btrfs_bio()  local
5188 	atomic_set(&bbio->error, 0);  in alloc_btrfs_bio()
5189 	atomic_set(&bbio->refs, 1);  in alloc_btrfs_bio()
5191 	return bbio;  in alloc_btrfs_bio()
5194 void btrfs_get_bbio(struct btrfs_bio *bbio)  in btrfs_get_bbio()  argument
5196 	WARN_ON(!atomic_read(&bbio->refs));  in btrfs_get_bbio()
5197 	atomic_inc(&bbio->refs);  in btrfs_get_bbio()
5200 void btrfs_put_bbio(struct btrfs_bio *bbio)  in btrfs_put_bbio()  argument
5202 	if (!bbio)  in btrfs_put_bbio()
5204 	if (atomic_dec_and_test(&bbio->refs))  in btrfs_put_bbio()
5205 		kfree(bbio);  in btrfs_put_bbio()
5230 	struct btrfs_bio *bbio = NULL;  in __btrfs_map_block()  local
5520 	bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);  in __btrfs_map_block()
5521 	if (!bbio) {  in __btrfs_map_block()
5526 		bbio->tgtdev_map = (int *)(bbio->stripes + num_alloc_stripes);  in __btrfs_map_block()
5535 		bbio->raid_map = (u64 *)((void *)bbio->stripes +  in __btrfs_map_block()
5546 			bbio->raid_map[(i+rot) % num_stripes] =  in __btrfs_map_block()
5549 		bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;  in __btrfs_map_block()
5551 			bbio->raid_map[(i+rot+1) % num_stripes] =  in __btrfs_map_block()
5579 			bbio->stripes[i].physical =  in __btrfs_map_block()
5582 			bbio->stripes[i].dev = map->stripes[stripe_index].dev;  in __btrfs_map_block()
5586 				bbio->stripes[i].length = stripes_per_dev *  in __btrfs_map_block()
5590 					bbio->stripes[i].length +=  in __btrfs_map_block()
5602 					bbio->stripes[i].length -=  in __btrfs_map_block()
5608 					bbio->stripes[i].length -=  in __btrfs_map_block()
5614 				bbio->stripes[i].length = *length;  in __btrfs_map_block()
5625 			bbio->stripes[i].physical =  in __btrfs_map_block()
5629 			bbio->stripes[i].dev =  in __btrfs_map_block()
5638 	if (bbio->raid_map)  in __btrfs_map_block()
5639 		sort_parity_stripes(bbio, num_stripes);  in __btrfs_map_block()
5660 			if (bbio->stripes[i].dev->devid == srcdev_devid) {  in __btrfs_map_block()
5663 					bbio->stripes + index_where_to_add;  in __btrfs_map_block()
5665 					bbio->stripes + i;  in __btrfs_map_block()
5670 				bbio->tgtdev_map[i] = index_where_to_add;  in __btrfs_map_block()
5692 			if (bbio->stripes[i].dev->devid == srcdev_devid) {  in __btrfs_map_block()
5700 				     bbio->stripes[i].physical)  in __btrfs_map_block()
5704 				physical_of_found = bbio->stripes[i].physical;  in __btrfs_map_block()
5711 					bbio->stripes + num_stripes;  in __btrfs_map_block()
5715 					bbio->stripes[index_srcdev].length;  in __btrfs_map_block()
5717 				bbio->tgtdev_map[index_srcdev] = num_stripes;  in __btrfs_map_block()
5725 	*bbio_ret = bbio;  in __btrfs_map_block()
5726 	bbio->map_type = map->type;  in __btrfs_map_block()
5727 	bbio->num_stripes = num_stripes;  in __btrfs_map_block()
5728 	bbio->max_errors = max_errors;  in __btrfs_map_block()
5729 	bbio->mirror_num = mirror_num;  in __btrfs_map_block()
5730 	bbio->num_tgtdevs = tgtdev_indexes;  in __btrfs_map_block()
5739 		bbio->stripes[0].dev = dev_replace->tgtdev;  in __btrfs_map_block()
5740 		bbio->stripes[0].physical = physical_to_patch_in_first_stripe;  in __btrfs_map_block()
5741 		bbio->mirror_num = map->num_stripes + 1;  in __btrfs_map_block()
5854 static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio)  in btrfs_end_bbio()  argument
5856 	bio->bi_private = bbio->private;  in btrfs_end_bbio()
5857 	bio->bi_end_io = bbio->end_io;  in btrfs_end_bbio()
5860 	btrfs_put_bbio(bbio);  in btrfs_end_bbio()
5865 	struct btrfs_bio *bbio = bio->bi_private;  in btrfs_end_bio()  local
5869 		atomic_inc(&bbio->error);  in btrfs_end_bio()
5875 			BUG_ON(stripe_index >= bbio->num_stripes);  in btrfs_end_bio()
5876 			dev = bbio->stripes[stripe_index].dev;  in btrfs_end_bio()
5892 	if (bio == bbio->orig_bio)  in btrfs_end_bio()
5895 	btrfs_bio_counter_dec(bbio->fs_info);  in btrfs_end_bio()
5897 	if (atomic_dec_and_test(&bbio->stripes_pending)) {  in btrfs_end_bio()
5900 			bio = bbio->orig_bio;  in btrfs_end_bio()
5903 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;  in btrfs_end_bio()
5907 		if (atomic_read(&bbio->error) > bbio->max_errors) {  in btrfs_end_bio()
5917 		btrfs_end_bbio(bbio, bio);  in btrfs_end_bio()
5983 static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,  in submit_stripe_bio()  argument
5987 	struct btrfs_device *dev = bbio->stripes[dev_nr].dev;  in submit_stripe_bio()
5989 	bio->bi_private = bbio;  in submit_stripe_bio()
6016 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)  in bbio_error()  argument
6018 	atomic_inc(&bbio->error);  in bbio_error()
6019 	if (atomic_dec_and_test(&bbio->stripes_pending)) {  in bbio_error()
6021 		WARN_ON(bio != bbio->orig_bio);  in bbio_error()
6023 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;  in bbio_error()
6026 		btrfs_end_bbio(bbio, bio);  in bbio_error()
6041 	struct btrfs_bio *bbio = NULL;  in btrfs_map_bio()  local
6047 	ret = __btrfs_map_block(root->fs_info, rw, logical, &map_length, &bbio,  in btrfs_map_bio()
6054 	total_devs = bbio->num_stripes;  in btrfs_map_bio()
6055 	bbio->orig_bio = first_bio;  in btrfs_map_bio()
6056 	bbio->private = first_bio->bi_private;  in btrfs_map_bio()
6057 	bbio->end_io = first_bio->bi_end_io;  in btrfs_map_bio()
6058 	bbio->fs_info = root->fs_info;  in btrfs_map_bio()
6059 	atomic_set(&bbio->stripes_pending, bbio->num_stripes);  in btrfs_map_bio()
6061 	if (bbio->raid_map) {  in btrfs_map_bio()
6065 			ret = raid56_parity_write(root, bio, bbio, map_length);  in btrfs_map_bio()
6067 			ret = raid56_parity_recover(root, bio, bbio, map_length,  in btrfs_map_bio()
6082 		dev = bbio->stripes[dev_nr].dev;  in btrfs_map_bio()
6084 			bbio_error(bbio, first_bio, logical);  in btrfs_map_bio()
6094 		submit_stripe_bio(root, bbio, bio,  in btrfs_map_bio()
6095 				  bbio->stripes[dev_nr].physical, dev_nr, rw,  in btrfs_map_bio()