207 static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)  in sq_to_tg()  argument
209 	if (sq && sq->parent_sq)  in sq_to_tg()
210 		return container_of(sq, struct throtl_grp, service_queue);  in sq_to_tg()
222 static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)  in sq_to_td()  argument
224 	struct throtl_grp *tg = sq_to_tg(sq);  in sq_to_td()
229 		return container_of(sq, struct throtl_data, service_queue);  in sq_to_td()
244 #define throtl_log(sq, fmt, args...)	do {				\  argument
245 	struct throtl_grp *__tg = sq_to_tg((sq));			\
246 	struct throtl_data *__td = sq_to_td((sq));			\
390 static void throtl_service_queue_init(struct throtl_service_queue *sq,  in throtl_service_queue_init()  argument
393 	INIT_LIST_HEAD(&sq->queued[0]);  in throtl_service_queue_init()
394 	INIT_LIST_HEAD(&sq->queued[1]);  in throtl_service_queue_init()
395 	sq->pending_tree = RB_ROOT;  in throtl_service_queue_init()
396 	sq->parent_sq = parent_sq;  in throtl_service_queue_init()
397 	setup_timer(&sq->pending_timer, throtl_pending_timer_fn,  in throtl_service_queue_init()
398 		    (unsigned long)sq);  in throtl_service_queue_init()
401 static void throtl_service_queue_exit(struct throtl_service_queue *sq)  in throtl_service_queue_exit()  argument
403 	del_timer_sync(&sq->pending_timer);  in throtl_service_queue_exit()
648 static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,  in throtl_schedule_pending_timer()  argument
651 	mod_timer(&sq->pending_timer, expires);  in throtl_schedule_pending_timer()
652 	throtl_log(sq, "schedule timer. delay=%lu jiffies=%lu",  in throtl_schedule_pending_timer()
674 static bool throtl_schedule_next_dispatch(struct throtl_service_queue *sq,  in throtl_schedule_next_dispatch()  argument
678 	if (!sq->nr_pending)  in throtl_schedule_next_dispatch()
681 	update_min_dispatch_time(sq);  in throtl_schedule_next_dispatch()
684 	if (force || time_after(sq->first_pending_disptime, jiffies)) {  in throtl_schedule_next_dispatch()
685 		throtl_schedule_pending_timer(sq, sq->first_pending_disptime);  in throtl_schedule_next_dispatch()
1023 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_add_bio_tg()  local
1035 	if (!sq->nr_queued[rw])  in throtl_add_bio_tg()
1038 	throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);  in throtl_add_bio_tg()
1040 	sq->nr_queued[rw]++;  in throtl_add_bio_tg()
1046 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_update_disptime()  local
1050 	if ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_update_disptime()
1053 	if ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_update_disptime()
1080 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_dispatch_one_bio()  local
1081 	struct throtl_service_queue *parent_sq = sq->parent_sq;  in tg_dispatch_one_bio()
1092 	bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);  in tg_dispatch_one_bio()
1093 	sq->nr_queued[rw]--;  in tg_dispatch_one_bio()
1122 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_dispatch_tg()  local
1130 	while ((bio = throtl_peek_queued(&sq->queued[READ])) &&  in throtl_dispatch_tg()
1140 	while ((bio = throtl_peek_queued(&sq->queued[WRITE])) &&  in throtl_dispatch_tg()
1159 		struct throtl_service_queue *sq = &tg->service_queue;  in throtl_select_dispatch()  local
1171 		if (sq->nr_queued[0] || sq->nr_queued[1])  in throtl_select_dispatch()
1198 	struct throtl_service_queue *sq = (void *)arg;  in throtl_pending_timer_fn()  local
1199 	struct throtl_grp *tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1200 	struct throtl_data *td = sq_to_td(sq);  in throtl_pending_timer_fn()
1208 	parent_sq = sq->parent_sq;  in throtl_pending_timer_fn()
1212 		throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",  in throtl_pending_timer_fn()
1213 			   sq->nr_queued[READ] + sq->nr_queued[WRITE],  in throtl_pending_timer_fn()
1214 			   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in throtl_pending_timer_fn()
1216 		ret = throtl_select_dispatch(sq);  in throtl_pending_timer_fn()
1218 			throtl_log(sq, "bios disp=%u", ret);  in throtl_pending_timer_fn()
1222 		if (throtl_schedule_next_dispatch(sq, false))  in throtl_pending_timer_fn()
1240 				sq = parent_sq;  in throtl_pending_timer_fn()
1241 				tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1358 	struct throtl_service_queue *sq;  in tg_set_conf()  local
1368 	sq = &tg->service_queue;  in tg_set_conf()
1406 		throtl_schedule_next_dispatch(sq->parent_sq, true);  in tg_set_conf()
1485 	struct throtl_service_queue *sq;  in blk_throtl_bio()  local
1519 	sq = &tg->service_queue;  in blk_throtl_bio()
1523 		if (sq->nr_queued[rw])  in blk_throtl_bio()
1552 		sq = sq->parent_sq;  in blk_throtl_bio()
1553 		tg = sq_to_tg(sq);  in blk_throtl_bio()
1559 	throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",  in blk_throtl_bio()
1563 		   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in blk_throtl_bio()
1606 		struct throtl_service_queue *sq = &tg->service_queue;  in tg_drain_bios()  local
1611 		while ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_drain_bios()
1613 		while ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_drain_bios()