323 static void throtl_qnode_add_bio(struct bio *bio, struct throtl_qnode *qn,  in throtl_qnode_add_bio()  argument
326 	bio_list_add(&qn->bios, bio);  in throtl_qnode_add_bio()
337 static struct bio *throtl_peek_queued(struct list_head *queued)  in throtl_peek_queued()
340 	struct bio *bio;  in throtl_peek_queued()  local
345 	bio = bio_list_peek(&qn->bios);  in throtl_peek_queued()
346 	WARN_ON_ONCE(!bio);  in throtl_peek_queued()
347 	return bio;  in throtl_peek_queued()
364 static struct bio *throtl_pop_queued(struct list_head *queued,  in throtl_pop_queued()
368 	struct bio *bio;  in throtl_pop_queued()  local
373 	bio = bio_list_pop(&qn->bios);  in throtl_pop_queued()
374 	WARN_ON_ONCE(!bio);  in throtl_pop_queued()
386 	return bio;  in throtl_pop_queued()
811 static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,  in tg_with_in_iops_limit()  argument
814 	bool rw = bio_data_dir(bio);  in tg_with_in_iops_limit()
861 static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,  in tg_with_in_bps_limit()  argument
864 	bool rw = bio_data_dir(bio);  in tg_with_in_bps_limit()
880 	if (tg->bytes_disp[rw] + bio->bi_iter.bi_size <= bytes_allowed) {  in tg_with_in_bps_limit()
887 	extra_bytes = tg->bytes_disp[rw] + bio->bi_iter.bi_size - bytes_allowed;  in tg_with_in_bps_limit()
907 static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,  in tg_may_dispatch()  argument
910 	bool rw = bio_data_dir(bio);  in tg_may_dispatch()
920 	       bio != throtl_peek_queued(&tg->service_queue.queued[rw]));  in tg_may_dispatch()
941 	if (tg_with_in_bps_limit(tg, bio, &bps_wait) &&  in tg_may_dispatch()
942 	    tg_with_in_iops_limit(tg, bio, &iops_wait)) {  in tg_may_dispatch()
985 static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)  in throtl_charge_bio()  argument
987 	bool rw = bio_data_dir(bio);  in throtl_charge_bio()
990 	tg->bytes_disp[rw] += bio->bi_iter.bi_size;  in throtl_charge_bio()
1004 	if (!(bio->bi_rw & REQ_THROTTLED)) {  in throtl_charge_bio()
1005 		bio->bi_rw |= REQ_THROTTLED;  in throtl_charge_bio()
1007 					     bio->bi_iter.bi_size, bio->bi_rw);  in throtl_charge_bio()
1020 static void throtl_add_bio_tg(struct bio *bio, struct throtl_qnode *qn,  in throtl_add_bio_tg()  argument
1024 	bool rw = bio_data_dir(bio);  in throtl_add_bio_tg()
1038 	throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);  in throtl_add_bio_tg()
1048 	struct bio *bio;  in tg_update_disptime()  local
1050 	if ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_update_disptime()
1051 		tg_may_dispatch(tg, bio, &read_wait);  in tg_update_disptime()
1053 	if ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_update_disptime()
1054 		tg_may_dispatch(tg, bio, &write_wait);  in tg_update_disptime()
1084 	struct bio *bio;  in tg_dispatch_one_bio()  local
1092 	bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);  in tg_dispatch_one_bio()
1095 	throtl_charge_bio(tg, bio);  in tg_dispatch_one_bio()
1105 		throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg);  in tg_dispatch_one_bio()
1108 		throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw],  in tg_dispatch_one_bio()
1126 	struct bio *bio;  in throtl_dispatch_tg()  local
1130 	while ((bio = throtl_peek_queued(&sq->queued[READ])) &&  in throtl_dispatch_tg()
1131 	       tg_may_dispatch(tg, bio, NULL)) {  in throtl_dispatch_tg()
1133 		tg_dispatch_one_bio(tg, bio_data_dir(bio));  in throtl_dispatch_tg()
1140 	while ((bio = throtl_peek_queued(&sq->queued[WRITE])) &&  in throtl_dispatch_tg()
1141 	       tg_may_dispatch(tg, bio, NULL)) {  in throtl_dispatch_tg()
1143 		tg_dispatch_one_bio(tg, bio_data_dir(bio));  in throtl_dispatch_tg()
1268 	struct bio *bio;  in blk_throtl_dispatch_work_fn()  local
1276 		while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL)))  in blk_throtl_dispatch_work_fn()
1277 			bio_list_add(&bio_list_on_stack, bio);  in blk_throtl_dispatch_work_fn()
1282 		while((bio = bio_list_pop(&bio_list_on_stack)))  in blk_throtl_dispatch_work_fn()
1283 			generic_make_request(bio);  in blk_throtl_dispatch_work_fn()
1480 bool blk_throtl_bio(struct request_queue *q, struct bio *bio)  in blk_throtl_bio()  argument
1486 	bool rw = bio_data_dir(bio);  in blk_throtl_bio()
1491 	if (bio->bi_rw & REQ_THROTTLED)  in blk_throtl_bio()
1500 	blkcg = bio_blkcg(bio);  in blk_throtl_bio()
1505 					bio->bi_iter.bi_size, bio->bi_rw);  in blk_throtl_bio()
1527 		if (!tg_may_dispatch(tg, bio, NULL))  in blk_throtl_bio()
1531 		throtl_charge_bio(tg, bio);  in blk_throtl_bio()
1561 		   tg->bytes_disp[rw], bio->bi_iter.bi_size, tg->bps[rw],  in blk_throtl_bio()
1565 	bio_associate_current(bio);  in blk_throtl_bio()
1567 	throtl_add_bio_tg(bio, qn, tg);  in blk_throtl_bio()
1592 		bio->bi_rw &= ~REQ_THROTTLED;  in blk_throtl_bio()
1607 		struct bio *bio;  in tg_drain_bios()  local
1611 		while ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_drain_bios()
1612 			tg_dispatch_one_bio(tg, bio_data_dir(bio));  in tg_drain_bios()
1613 		while ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_drain_bios()
1614 			tg_dispatch_one_bio(tg, bio_data_dir(bio));  in tg_drain_bios()
1630 	struct bio *bio;  in blk_throtl_drain()  local
1653 		while ((bio = throtl_pop_queued(&td->service_queue.queued[rw],  in blk_throtl_drain()
1655 			generic_make_request(bio);  in blk_throtl_drain()