iolat 187 block/blk-iolatency.c static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat) iolat 189 block/blk-iolatency.c return pd_to_blkg(&iolat->pd); iolat 192 block/blk-iolatency.c static inline void latency_stat_init(struct iolatency_grp *iolat, iolat 195 block/blk-iolatency.c if (iolat->ssd) { iolat 202 block/blk-iolatency.c static inline void latency_stat_sum(struct iolatency_grp *iolat, iolat 206 block/blk-iolatency.c if (iolat->ssd) { iolat 213 block/blk-iolatency.c static inline void latency_stat_record_time(struct iolatency_grp *iolat, iolat 216 block/blk-iolatency.c struct latency_stat *stat = get_cpu_ptr(iolat->stats); iolat 217 block/blk-iolatency.c if (iolat->ssd) { iolat 218 block/blk-iolatency.c if (req_time >= iolat->min_lat_nsec) iolat 226 block/blk-iolatency.c static inline bool latency_sum_ok(struct iolatency_grp *iolat, iolat 229 block/blk-iolatency.c if (iolat->ssd) { iolat 234 block/blk-iolatency.c return stat->rqs.mean <= iolat->min_lat_nsec; iolat 237 block/blk-iolatency.c static inline u64 latency_stat_samples(struct iolatency_grp *iolat, iolat 240 block/blk-iolatency.c if (iolat->ssd) iolat 245 block/blk-iolatency.c static inline void iolat_update_total_lat_avg(struct iolatency_grp *iolat, iolat 250 block/blk-iolatency.c if (iolat->ssd) iolat 261 block/blk-iolatency.c div64_u64(iolat->cur_win_nsec, iolat 263 block/blk-iolatency.c iolat->lat_avg = calc_load(iolat->lat_avg, iolat 276 block/blk-iolatency.c struct iolatency_grp *iolat = private_data; iolat 277 block/blk-iolatency.c return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth); iolat 281 block/blk-iolatency.c struct iolatency_grp *iolat, iolat 285 block/blk-iolatency.c struct rq_wait *rqw = &iolat->rq_wait; iolat 286 block/blk-iolatency.c unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay); iolat 303 block/blk-iolatency.c rq_qos_wait(rqw, iolat, iolat_acquire_inflight, iolat_cleanup_cb); iolat 365 block/blk-iolatency.c static void scale_change(struct iolatency_grp *iolat, bool up) iolat 367 block/blk-iolatency.c unsigned long qd = iolat->blkiolat->rqos.q->nr_requests; iolat 369 block/blk-iolatency.c unsigned long old = iolat->rq_depth.max_depth; iolat 375 block/blk-iolatency.c if (old == 1 && blkcg_unuse_delay(lat_to_blkg(iolat))) iolat 381 block/blk-iolatency.c iolat->rq_depth.max_depth = old; iolat 382 block/blk-iolatency.c wake_up_all(&iolat->rq_wait.wait); iolat 386 block/blk-iolatency.c iolat->rq_depth.max_depth = max(old, 1UL); iolat 391 block/blk-iolatency.c static void check_scale_change(struct iolatency_grp *iolat) iolat 396 block/blk-iolatency.c unsigned int our_cookie = atomic_read(&iolat->scale_cookie); iolat 401 block/blk-iolatency.c if (lat_to_blkg(iolat)->parent == NULL) iolat 404 block/blk-iolatency.c parent = blkg_to_lat(lat_to_blkg(iolat)->parent); iolat 419 block/blk-iolatency.c old = atomic_cmpxchg(&iolat->scale_cookie, our_cookie, cur_cookie); iolat 425 block/blk-iolatency.c if (direction < 0 && iolat->min_lat_nsec) { iolat 428 block/blk-iolatency.c if (!scale_lat || iolat->min_lat_nsec <= scale_lat) iolat 439 block/blk-iolatency.c if (iolat->nr_samples <= samples_thresh) iolat 444 block/blk-iolatency.c if (iolat->rq_depth.max_depth == 1 && direction < 0) { iolat 445 block/blk-iolatency.c blkcg_use_delay(lat_to_blkg(iolat)); iolat 451 block/blk-iolatency.c blkcg_clear_delay(lat_to_blkg(iolat)); iolat 452 block/blk-iolatency.c iolat->rq_depth.max_depth = UINT_MAX; iolat 453 block/blk-iolatency.c wake_up_all(&iolat->rq_wait.wait); iolat 457 block/blk-iolatency.c scale_change(iolat, direction > 0); iolat 470 block/blk-iolatency.c struct iolatency_grp *iolat = blkg_to_lat(blkg); iolat 471 block/blk-iolatency.c if (!iolat) { iolat 476 block/blk-iolatency.c check_scale_change(iolat); iolat 477 block/blk-iolatency.c __blkcg_iolatency_throttle(rqos, iolat, issue_as_root, iolat 485 block/blk-iolatency.c static void iolatency_record_time(struct iolatency_grp *iolat, iolat 507 block/blk-iolatency.c if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) { iolat 508 block/blk-iolatency.c u64 sub = iolat->min_lat_nsec; iolat 510 block/blk-iolatency.c blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time); iolat 514 block/blk-iolatency.c latency_stat_record_time(iolat, req_time); iolat 520 block/blk-iolatency.c static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) iolat 522 block/blk-iolatency.c struct blkcg_gq *blkg = lat_to_blkg(iolat); iolat 529 block/blk-iolatency.c latency_stat_init(iolat, &stat); iolat 533 block/blk-iolatency.c s = per_cpu_ptr(iolat->stats, cpu); iolat 534 block/blk-iolatency.c latency_stat_sum(iolat, &stat, s); iolat 535 block/blk-iolatency.c latency_stat_init(iolat, s); iolat 545 block/blk-iolatency.c iolat_update_total_lat_avg(iolat, &stat); iolat 548 block/blk-iolatency.c if (latency_sum_ok(iolat, &stat) && iolat 555 block/blk-iolatency.c latency_stat_sum(iolat, &iolat->cur_stat, &stat); iolat 556 block/blk-iolatency.c lat_info->nr_samples -= iolat->nr_samples; iolat 557 block/blk-iolatency.c lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat); iolat 558 block/blk-iolatency.c iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat); iolat 564 block/blk-iolatency.c if (latency_sum_ok(iolat, &iolat->cur_stat) && iolat 565 block/blk-iolatency.c latency_sum_ok(iolat, &stat)) { iolat 566 block/blk-iolatency.c if (latency_stat_samples(iolat, &iolat->cur_stat) < iolat 569 block/blk-iolatency.c if (lat_info->scale_grp == iolat) { iolat 571 block/blk-iolatency.c scale_cookie_change(iolat->blkiolat, lat_info, true); iolat 574 block/blk-iolatency.c lat_info->scale_lat >= iolat->min_lat_nsec) { iolat 577 block/blk-iolatency.c lat_info->scale_lat > iolat->min_lat_nsec) { iolat 578 block/blk-iolatency.c WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec); iolat 579 block/blk-iolatency.c lat_info->scale_grp = iolat; iolat 581 block/blk-iolatency.c scale_cookie_change(iolat->blkiolat, lat_info, false); iolat 583 block/blk-iolatency.c latency_stat_init(iolat, &iolat->cur_stat); iolat 592 block/blk-iolatency.c struct iolatency_grp *iolat; iolat 603 block/blk-iolatency.c iolat = blkg_to_lat(bio->bi_blkg); iolat 604 block/blk-iolatency.c if (!iolat) iolat 607 block/blk-iolatency.c enabled = blk_iolatency_enabled(iolat->blkiolat); iolat 612 block/blk-iolatency.c iolat = blkg_to_lat(blkg); iolat 613 block/blk-iolatency.c if (!iolat) { iolat 617 block/blk-iolatency.c rqw = &iolat->rq_wait; iolat 625 block/blk-iolatency.c if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) { iolat 626 block/blk-iolatency.c iolatency_record_time(iolat, &bio->bi_issue, now, iolat 628 block/blk-iolatency.c window_start = atomic64_read(&iolat->window_start); iolat 630 block/blk-iolatency.c (now - window_start) >= iolat->cur_win_nsec) { iolat 631 block/blk-iolatency.c if (atomic64_cmpxchg(&iolat->window_start, iolat 633 block/blk-iolatency.c iolatency_check_latencies(iolat, now); iolat 666 block/blk-iolatency.c struct iolatency_grp *iolat; iolat 678 block/blk-iolatency.c iolat = blkg_to_lat(blkg); iolat 679 block/blk-iolatency.c if (!iolat) iolat 682 block/blk-iolatency.c lat_info = &iolat->child_lat; iolat 697 block/blk-iolatency.c scale_cookie_change(iolat->blkiolat, lat_info, true); iolat 752 block/blk-iolatency.c struct iolatency_grp *iolat = blkg_to_lat(blkg); iolat 753 block/blk-iolatency.c u64 oldval = iolat->min_lat_nsec; iolat 755 block/blk-iolatency.c iolat->min_lat_nsec = val; iolat 756 block/blk-iolatency.c iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE); iolat 757 block/blk-iolatency.c iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec, iolat 772 block/blk-iolatency.c struct iolatency_grp *iolat = blkg_to_lat(blkg->parent); iolat 774 block/blk-iolatency.c if (!iolat) iolat 777 block/blk-iolatency.c lat_info = &iolat->child_lat; iolat 793 block/blk-iolatency.c struct iolatency_grp *iolat; iolat 804 block/blk-iolatency.c iolat = blkg_to_lat(ctx.blkg); iolat 831 block/blk-iolatency.c oldval = iolat->min_lat_nsec; iolat 839 block/blk-iolatency.c if (oldval != iolat->min_lat_nsec) { iolat 870 block/blk-iolatency.c struct iolatency_grp *iolat = pd_to_lat(pd); iolat 873 block/blk-iolatency.c if (!dname || !iolat->min_lat_nsec) iolat 876 block/blk-iolatency.c dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC)); iolat 888 block/blk-iolatency.c static size_t iolatency_ssd_stat(struct iolatency_grp *iolat, char *buf, iolat 894 block/blk-iolatency.c latency_stat_init(iolat, &stat); iolat 898 block/blk-iolatency.c s = per_cpu_ptr(iolat->stats, cpu); iolat 899 block/blk-iolatency.c latency_stat_sum(iolat, &stat, s); iolat 903 block/blk-iolatency.c if (iolat->rq_depth.max_depth == UINT_MAX) iolat 910 block/blk-iolatency.c iolat->rq_depth.max_depth); iolat 916 block/blk-iolatency.c struct iolatency_grp *iolat = pd_to_lat(pd); iolat 923 block/blk-iolatency.c if (iolat->ssd) iolat 924 block/blk-iolatency.c return iolatency_ssd_stat(iolat, buf, size); iolat 926 block/blk-iolatency.c avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC); iolat 927 block/blk-iolatency.c cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC); iolat 928 block/blk-iolatency.c if (iolat->rq_depth.max_depth == UINT_MAX) iolat 933 block/blk-iolatency.c iolat->rq_depth.max_depth, avg_lat, cur_win); iolat 941 block/blk-iolatency.c struct iolatency_grp *iolat; iolat 943 block/blk-iolatency.c iolat = kzalloc_node(sizeof(*iolat), gfp, q->node); iolat 944 block/blk-iolatency.c if (!iolat) iolat 946 block/blk-iolatency.c iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat), iolat 948 block/blk-iolatency.c if (!iolat->stats) { iolat 949 block/blk-iolatency.c kfree(iolat); iolat 952 block/blk-iolatency.c return &iolat->pd; iolat 957 block/blk-iolatency.c struct iolatency_grp *iolat = pd_to_lat(pd); iolat 958 block/blk-iolatency.c struct blkcg_gq *blkg = lat_to_blkg(iolat); iolat 965 block/blk-iolatency.c iolat->ssd = true; iolat 967 block/blk-iolatency.c iolat->ssd = false; iolat 971 block/blk-iolatency.c stat = per_cpu_ptr(iolat->stats, cpu); iolat 972 block/blk-iolatency.c latency_stat_init(iolat, stat); iolat 975 block/blk-iolatency.c latency_stat_init(iolat, &iolat->cur_stat); iolat 976 block/blk-iolatency.c rq_wait_init(&iolat->rq_wait); iolat 977 block/blk-iolatency.c spin_lock_init(&iolat->child_lat.lock); iolat 978 block/blk-iolatency.c iolat->rq_depth.queue_depth = blkg->q->nr_requests; iolat 979 block/blk-iolatency.c iolat->rq_depth.max_depth = UINT_MAX; iolat 980 block/blk-iolatency.c iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth; iolat 981 block/blk-iolatency.c iolat->blkiolat = blkiolat; iolat 982 block/blk-iolatency.c iolat->cur_win_nsec = 100 * NSEC_PER_MSEC; iolat 983 block/blk-iolatency.c atomic64_set(&iolat->window_start, now); iolat 991 block/blk-iolatency.c atomic_set(&iolat->scale_cookie, iolat 994 block/blk-iolatency.c atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE); iolat 997 block/blk-iolatency.c atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE); iolat 1002 block/blk-iolatency.c struct iolatency_grp *iolat = pd_to_lat(pd); iolat 1003 block/blk-iolatency.c struct blkcg_gq *blkg = lat_to_blkg(iolat); iolat 1004 block/blk-iolatency.c struct blk_iolatency *blkiolat = iolat->blkiolat; iolat 1017 block/blk-iolatency.c struct iolatency_grp *iolat = pd_to_lat(pd); iolat 1018 block/blk-iolatency.c free_percpu(iolat->stats); iolat 1019 block/blk-iolatency.c kfree(iolat);