Lines Matching refs:rw
412 int rw; in throtl_pd_init() local
434 for (rw = READ; rw <= WRITE; rw++) { in throtl_pd_init()
435 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_init()
436 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_init()
466 int rw; in tg_update_has_rules() local
468 for (rw = READ; rw <= WRITE; rw++) in tg_update_has_rules()
469 tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) || in tg_update_has_rules()
470 (tg->bps[rw] != -1 || tg->iops[rw] != -1); in tg_update_has_rules()
694 bool rw, unsigned long start) in throtl_start_new_slice_with_credit() argument
696 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
697 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
705 if (time_after_eq(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
706 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
708 tg->slice_end[rw] = jiffies + throtl_slice; in throtl_start_new_slice_with_credit()
711 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
712 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
715 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw) in throtl_start_new_slice() argument
717 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
718 tg->io_disp[rw] = 0; in throtl_start_new_slice()
719 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
720 tg->slice_end[rw] = jiffies + throtl_slice; in throtl_start_new_slice()
723 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
724 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
727 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, in throtl_set_slice_end() argument
730 tg->slice_end[rw] = roundup(jiffy_end, throtl_slice); in throtl_set_slice_end()
733 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, in throtl_extend_slice() argument
736 tg->slice_end[rw] = roundup(jiffy_end, throtl_slice); in throtl_extend_slice()
739 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
740 tg->slice_end[rw], jiffies); in throtl_extend_slice()
744 static bool throtl_slice_used(struct throtl_grp *tg, bool rw) in throtl_slice_used() argument
746 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
753 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) in throtl_trim_slice() argument
758 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
765 if (throtl_slice_used(tg, rw)) in throtl_trim_slice()
776 throtl_set_slice_end(tg, rw, jiffies + throtl_slice); in throtl_trim_slice()
778 time_elapsed = jiffies - tg->slice_start[rw]; in throtl_trim_slice()
784 tmp = tg->bps[rw] * throtl_slice * nr_slices; in throtl_trim_slice()
788 io_trim = (tg->iops[rw] * throtl_slice * nr_slices)/HZ; in throtl_trim_slice()
793 if (tg->bytes_disp[rw] >= bytes_trim) in throtl_trim_slice()
794 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_slice()
796 tg->bytes_disp[rw] = 0; in throtl_trim_slice()
798 if (tg->io_disp[rw] >= io_trim) in throtl_trim_slice()
799 tg->io_disp[rw] -= io_trim; in throtl_trim_slice()
801 tg->io_disp[rw] = 0; in throtl_trim_slice()
803 tg->slice_start[rw] += nr_slices * throtl_slice; in throtl_trim_slice()
807 rw == READ ? 'R' : 'W', nr_slices, bytes_trim, io_trim, in throtl_trim_slice()
808 tg->slice_start[rw], tg->slice_end[rw], jiffies); in throtl_trim_slice()
814 bool rw = bio_data_dir(bio); in tg_with_in_iops_limit() local
819 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_with_in_iops_limit()
834 tmp = (u64)tg->iops[rw] * jiffy_elapsed_rnd; in tg_with_in_iops_limit()
842 if (tg->io_disp[rw] + 1 <= io_allowed) { in tg_with_in_iops_limit()
849 jiffy_wait = ((tg->io_disp[rw] + 1) * HZ)/tg->iops[rw] + 1; in tg_with_in_iops_limit()
864 bool rw = bio_data_dir(bio); in tg_with_in_bps_limit() local
868 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_with_in_bps_limit()
876 tmp = tg->bps[rw] * jiffy_elapsed_rnd; in tg_with_in_bps_limit()
880 if (tg->bytes_disp[rw] + bio->bi_iter.bi_size <= bytes_allowed) { in tg_with_in_bps_limit()
887 extra_bytes = tg->bytes_disp[rw] + bio->bi_iter.bi_size - bytes_allowed; in tg_with_in_bps_limit()
888 jiffy_wait = div64_u64(extra_bytes * HZ, tg->bps[rw]); in tg_with_in_bps_limit()
910 bool rw = bio_data_dir(bio); in tg_may_dispatch() local
919 BUG_ON(tg->service_queue.nr_queued[rw] && in tg_may_dispatch()
920 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_may_dispatch()
923 if (tg->bps[rw] == -1 && tg->iops[rw] == -1) { in tg_may_dispatch()
934 if (throtl_slice_used(tg, rw)) in tg_may_dispatch()
935 throtl_start_new_slice(tg, rw); in tg_may_dispatch()
937 if (time_before(tg->slice_end[rw], jiffies + throtl_slice)) in tg_may_dispatch()
938 throtl_extend_slice(tg, rw, jiffies + throtl_slice); in tg_may_dispatch()
953 if (time_before(tg->slice_end[rw], jiffies + max_wait)) in tg_may_dispatch()
954 throtl_extend_slice(tg, rw, jiffies + max_wait); in tg_may_dispatch()
960 int rw) in throtl_update_dispatch_stats() argument
979 blkg_rwstat_add(&stats_cpu->serviced, rw, 1); in throtl_update_dispatch_stats()
980 blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes); in throtl_update_dispatch_stats()
987 bool rw = bio_data_dir(bio); in throtl_charge_bio() local
990 tg->bytes_disp[rw] += bio->bi_iter.bi_size; in throtl_charge_bio()
991 tg->io_disp[rw]++; in throtl_charge_bio()
1024 bool rw = bio_data_dir(bio); in throtl_add_bio_tg() local
1027 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
1035 if (!sq->nr_queued[rw]) in throtl_add_bio_tg()
1038 throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); in throtl_add_bio_tg()
1040 sq->nr_queued[rw]++; in throtl_add_bio_tg()
1069 struct throtl_grp *parent_tg, bool rw) in start_parent_slice_with_credit() argument
1071 if (throtl_slice_used(parent_tg, rw)) { in start_parent_slice_with_credit()
1072 throtl_start_new_slice_with_credit(parent_tg, rw, in start_parent_slice_with_credit()
1073 child_tg->slice_start[rw]); in start_parent_slice_with_credit()
1078 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) in tg_dispatch_one_bio() argument
1092 bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); in tg_dispatch_one_bio()
1093 sq->nr_queued[rw]--; in tg_dispatch_one_bio()
1105 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1106 start_parent_slice_with_credit(tg, parent_tg, rw); in tg_dispatch_one_bio()
1108 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1109 &parent_sq->queued[rw]); in tg_dispatch_one_bio()
1110 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1111 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1114 throtl_trim_slice(tg, rw); in tg_dispatch_one_bio()
1270 int rw; in blk_throtl_dispatch_work_fn() local
1275 for (rw = READ; rw <= WRITE; rw++) in blk_throtl_dispatch_work_fn()
1276 while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL))) in blk_throtl_dispatch_work_fn()
1486 bool rw = bio_data_dir(bio); in blk_throtl_bio() local
1503 if (!tg->has_rules[rw]) { in blk_throtl_bio()
1523 if (sq->nr_queued[rw]) in blk_throtl_bio()
1544 throtl_trim_slice(tg, rw); in blk_throtl_bio()
1551 qn = &tg->qnode_on_parent[rw]; in blk_throtl_bio()
1560 rw == READ ? 'R' : 'W', in blk_throtl_bio()
1561 tg->bytes_disp[rw], bio->bi_iter.bi_size, tg->bps[rw], in blk_throtl_bio()
1562 tg->io_disp[rw], tg->iops[rw], in blk_throtl_bio()
1566 tg->td->nr_queued[rw]++; in blk_throtl_bio()
1631 int rw; in blk_throtl_drain() local
1652 for (rw = READ; rw <= WRITE; rw++) in blk_throtl_drain()
1653 while ((bio = throtl_pop_queued(&td->service_queue.queued[rw], in blk_throtl_drain()