228 	struct bio *bi;  in return_io()  local
229 	while ((bi = bio_list_pop(return_bi)) != NULL) {  in return_io()
230 		bi->bi_iter.bi_size = 0;  in return_io()
231 		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),  in return_io()
232 					 bi, 0);  in return_io()
233 		bio_endio(bi);  in return_io()
881 raid5_end_read_request(struct bio *bi);
883 raid5_end_write_request(struct bio *bi);
898 		struct bio *bi, *rbi;  in ops_run_io()  local
921 		bi = &sh->dev[i].req;  in ops_run_io()
998 			bio_reset(bi);  in ops_run_io()
999 			bi->bi_bdev = rdev->bdev;  in ops_run_io()
1000 			bi->bi_rw = rw;  in ops_run_io()
1001 			bi->bi_end_io = (rw & WRITE)  in ops_run_io()
1004 			bi->bi_private = sh;  in ops_run_io()
1008 				bi->bi_rw, i);  in ops_run_io()
1013 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1016 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1019 				bi->bi_rw |= REQ_NOMERGE;  in ops_run_io()
1024 			bi->bi_vcnt = 1;  in ops_run_io()
1025 			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;  in ops_run_io()
1026 			bi->bi_io_vec[0].bv_offset = 0;  in ops_run_io()
1027 			bi->bi_iter.bi_size = STRIPE_SIZE;  in ops_run_io()
1033 				bi->bi_vcnt = 0;  in ops_run_io()
1038 				trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),  in ops_run_io()
1039 						      bi, disk_devt(conf->mddev->gendisk),  in ops_run_io()
1041 			generic_make_request(bi);  in ops_run_io()
1093 				bi->bi_rw, i, (unsigned long long)sh->sector);  in ops_run_io()
2287 static void raid5_end_read_request(struct bio * bi)  in raid5_end_read_request()  argument
2289 	struct stripe_head *sh = bi->bi_private;  in raid5_end_read_request()
2297 		if (bi == &sh->dev[i].req)  in raid5_end_read_request()
2302 		bi->bi_error);  in raid5_end_read_request()
2321 	if (!bi->bi_error) {  in raid5_end_read_request()
2409 static void raid5_end_write_request(struct bio *bi)  in raid5_end_write_request()  argument
2411 	struct stripe_head *sh = bi->bi_private;  in raid5_end_write_request()
2420 		if (bi == &sh->dev[i].req) {  in raid5_end_write_request()
2424 		if (bi == &sh->dev[i].rreq) {  in raid5_end_write_request()
2439 		bi->bi_error);  in raid5_end_write_request()
2446 		if (bi->bi_error)  in raid5_end_write_request()
2453 		if (bi->bi_error) {  in raid5_end_write_request()
2474 	if (sh->batch_head && bi->bi_error && !replacement)  in raid5_end_write_request()
2951 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,  in add_stripe_bio()  argument
2959 		(unsigned long long)bi->bi_iter.bi_sector,  in add_stripe_bio()
2980 	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {  in add_stripe_bio()
2981 		if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)  in add_stripe_bio()
2985 	if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))  in add_stripe_bio()
2991 	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);  in add_stripe_bio()
2993 		bi->bi_next = *bip;  in add_stripe_bio()
2994 	*bip = bi;  in add_stripe_bio()
2995 	raid5_inc_bi_active_stripes(bi);  in add_stripe_bio()
3000 		for (bi=sh->dev[dd_idx].towrite;  in add_stripe_bio()
3002 			     bi && bi->bi_iter.bi_sector <= sector;  in add_stripe_bio()
3003 		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {  in add_stripe_bio()
3004 			if (bio_end_sector(bi) >= sector)  in add_stripe_bio()
3005 				sector = bio_end_sector(bi);  in add_stripe_bio()
3078 		struct bio *bi;  in handle_failed_stripe()  local
3101 		bi = sh->dev[i].towrite;  in handle_failed_stripe()
3105 		if (bi)  in handle_failed_stripe()
3113 		while (bi && bi->bi_iter.bi_sector <  in handle_failed_stripe()
3115 			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);  in handle_failed_stripe()
3117 			bi->bi_error = -EIO;  in handle_failed_stripe()
3118 			if (!raid5_dec_bi_active_stripes(bi)) {  in handle_failed_stripe()
3120 				bio_list_add(return_bi, bi);  in handle_failed_stripe()
3122 			bi = nextbi;  in handle_failed_stripe()
3129 		bi = sh->dev[i].written;  in handle_failed_stripe()
3136 		if (bi) bitmap_end = 1;  in handle_failed_stripe()
3137 		while (bi && bi->bi_iter.bi_sector <  in handle_failed_stripe()
3139 			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);  in handle_failed_stripe()
3141 			bi->bi_error = -EIO;  in handle_failed_stripe()
3142 			if (!raid5_dec_bi_active_stripes(bi)) {  in handle_failed_stripe()
3144 				bio_list_add(return_bi, bi);  in handle_failed_stripe()
3146 			bi = bi2;  in handle_failed_stripe()
3157 			bi = sh->dev[i].toread;  in handle_failed_stripe()
3162 			if (bi)  in handle_failed_stripe()
3164 			while (bi && bi->bi_iter.bi_sector <  in handle_failed_stripe()
3167 					r5_next_bio(bi, sh->dev[i].sector);  in handle_failed_stripe()
3169 				bi->bi_error = -EIO;  in handle_failed_stripe()
3170 				if (!raid5_dec_bi_active_stripes(bi))  in handle_failed_stripe()
3171 					bio_list_add(return_bi, bi);  in handle_failed_stripe()
3172 				bi = nextbi;  in handle_failed_stripe()
4709 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)  in add_bio_to_retry()  argument
4715 	bi->bi_next = conf->retry_read_aligned_list;  in add_bio_to_retry()
4716 	conf->retry_read_aligned_list = bi;  in add_bio_to_retry()
4724 	struct bio *bi;  in remove_bio_from_retry()  local
4726 	bi = conf->retry_read_aligned;  in remove_bio_from_retry()
4727 	if (bi) {  in remove_bio_from_retry()
4729 		return bi;  in remove_bio_from_retry()
4731 	bi = conf->retry_read_aligned_list;  in remove_bio_from_retry()
4732 	if(bi) {  in remove_bio_from_retry()
4733 		conf->retry_read_aligned_list = bi->bi_next;  in remove_bio_from_retry()
4734 		bi->bi_next = NULL;  in remove_bio_from_retry()
4739 		raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */  in remove_bio_from_retry()
4742 	return bi;  in remove_bio_from_retry()
4751 static void raid5_align_endio(struct bio *bi)  in raid5_align_endio()  argument
4753 	struct bio* raid_bi  = bi->bi_private;  in raid5_align_endio()
4757 	int error = bi->bi_error;  in raid5_align_endio()
4759 	bio_put(bi);  in raid5_align_endio()
5051 static void make_discard_request(struct mddev *mddev, struct bio *bi)  in make_discard_request()  argument
5063 	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);  in make_discard_request()
5064 	last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);  in make_discard_request()
5066 	bi->bi_next = NULL;  in make_discard_request()
5067 	bi->bi_phys_segments = 1; /* over-loaded to count active stripes */  in make_discard_request()
5111 			sh->dev[d].towrite = bi;  in make_discard_request()
5113 			raid5_inc_bi_active_stripes(bi);  in make_discard_request()
5136 	remaining = raid5_dec_bi_active_stripes(bi);  in make_discard_request()
5139 		bio_endio(bi);  in make_discard_request()
5143 static void make_request(struct mddev *mddev, struct bio * bi)  in make_request()  argument
5150 	const int rw = bio_data_dir(bi);  in make_request()
5155 	if (unlikely(bi->bi_rw & REQ_FLUSH)) {  in make_request()
5156 		int ret = r5l_handle_flush_request(conf->log, bi);  in make_request()
5161 			md_flush_request(mddev, bi);  in make_request()
5167 	md_write_start(mddev, bi);  in make_request()
5176 		bi = chunk_aligned_read(mddev, bi);  in make_request()
5177 		if (!bi)  in make_request()
5181 	if (unlikely(bi->bi_rw & REQ_DISCARD)) {  in make_request()
5182 		make_discard_request(mddev, bi);  in make_request()
5186 	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);  in make_request()
5187 	last_sector = bio_end_sector(bi);  in make_request()
5188 	bi->bi_next = NULL;  in make_request()
5189 	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */  in make_request()
5238 				       (bi->bi_rw&RWA_MASK), 0);  in make_request()
5292 			    !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {  in make_request()
5306 			    (bi->bi_rw & REQ_SYNC) &&  in make_request()
5312 			bi->bi_error = -EIO;  in make_request()
5318 	remaining = raid5_dec_bi_active_stripes(bi);  in make_request()
5324 		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),  in make_request()
5325 					 bi, 0);  in make_request()
5326 		bio_endio(bi);  in make_request()