iolat             187 block/blk-iolatency.c static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat)
iolat             189 block/blk-iolatency.c 	return pd_to_blkg(&iolat->pd);
iolat             192 block/blk-iolatency.c static inline void latency_stat_init(struct iolatency_grp *iolat,
iolat             195 block/blk-iolatency.c 	if (iolat->ssd) {
iolat             202 block/blk-iolatency.c static inline void latency_stat_sum(struct iolatency_grp *iolat,
iolat             206 block/blk-iolatency.c 	if (iolat->ssd) {
iolat             213 block/blk-iolatency.c static inline void latency_stat_record_time(struct iolatency_grp *iolat,
iolat             216 block/blk-iolatency.c 	struct latency_stat *stat = get_cpu_ptr(iolat->stats);
iolat             217 block/blk-iolatency.c 	if (iolat->ssd) {
iolat             218 block/blk-iolatency.c 		if (req_time >= iolat->min_lat_nsec)
iolat             226 block/blk-iolatency.c static inline bool latency_sum_ok(struct iolatency_grp *iolat,
iolat             229 block/blk-iolatency.c 	if (iolat->ssd) {
iolat             234 block/blk-iolatency.c 	return stat->rqs.mean <= iolat->min_lat_nsec;
iolat             237 block/blk-iolatency.c static inline u64 latency_stat_samples(struct iolatency_grp *iolat,
iolat             240 block/blk-iolatency.c 	if (iolat->ssd)
iolat             245 block/blk-iolatency.c static inline void iolat_update_total_lat_avg(struct iolatency_grp *iolat,
iolat             250 block/blk-iolatency.c 	if (iolat->ssd)
iolat             261 block/blk-iolatency.c 			div64_u64(iolat->cur_win_nsec,
iolat             263 block/blk-iolatency.c 	iolat->lat_avg = calc_load(iolat->lat_avg,
iolat             276 block/blk-iolatency.c 	struct iolatency_grp *iolat = private_data;
iolat             277 block/blk-iolatency.c 	return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth);
iolat             281 block/blk-iolatency.c 				       struct iolatency_grp *iolat,
iolat             285 block/blk-iolatency.c 	struct rq_wait *rqw = &iolat->rq_wait;
iolat             286 block/blk-iolatency.c 	unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay);
iolat             303 block/blk-iolatency.c 	rq_qos_wait(rqw, iolat, iolat_acquire_inflight, iolat_cleanup_cb);
iolat             365 block/blk-iolatency.c static void scale_change(struct iolatency_grp *iolat, bool up)
iolat             367 block/blk-iolatency.c 	unsigned long qd = iolat->blkiolat->rqos.q->nr_requests;
iolat             369 block/blk-iolatency.c 	unsigned long old = iolat->rq_depth.max_depth;
iolat             375 block/blk-iolatency.c 		if (old == 1 && blkcg_unuse_delay(lat_to_blkg(iolat)))
iolat             381 block/blk-iolatency.c 			iolat->rq_depth.max_depth = old;
iolat             382 block/blk-iolatency.c 			wake_up_all(&iolat->rq_wait.wait);
iolat             386 block/blk-iolatency.c 		iolat->rq_depth.max_depth = max(old, 1UL);
iolat             391 block/blk-iolatency.c static void check_scale_change(struct iolatency_grp *iolat)
iolat             396 block/blk-iolatency.c 	unsigned int our_cookie = atomic_read(&iolat->scale_cookie);
iolat             401 block/blk-iolatency.c 	if (lat_to_blkg(iolat)->parent == NULL)
iolat             404 block/blk-iolatency.c 	parent = blkg_to_lat(lat_to_blkg(iolat)->parent);
iolat             419 block/blk-iolatency.c 	old = atomic_cmpxchg(&iolat->scale_cookie, our_cookie, cur_cookie);
iolat             425 block/blk-iolatency.c 	if (direction < 0 && iolat->min_lat_nsec) {
iolat             428 block/blk-iolatency.c 		if (!scale_lat || iolat->min_lat_nsec <= scale_lat)
iolat             439 block/blk-iolatency.c 		if (iolat->nr_samples <= samples_thresh)
iolat             444 block/blk-iolatency.c 	if (iolat->rq_depth.max_depth == 1 && direction < 0) {
iolat             445 block/blk-iolatency.c 		blkcg_use_delay(lat_to_blkg(iolat));
iolat             451 block/blk-iolatency.c 		blkcg_clear_delay(lat_to_blkg(iolat));
iolat             452 block/blk-iolatency.c 		iolat->rq_depth.max_depth = UINT_MAX;
iolat             453 block/blk-iolatency.c 		wake_up_all(&iolat->rq_wait.wait);
iolat             457 block/blk-iolatency.c 	scale_change(iolat, direction > 0);
iolat             470 block/blk-iolatency.c 		struct iolatency_grp *iolat = blkg_to_lat(blkg);
iolat             471 block/blk-iolatency.c 		if (!iolat) {
iolat             476 block/blk-iolatency.c 		check_scale_change(iolat);
iolat             477 block/blk-iolatency.c 		__blkcg_iolatency_throttle(rqos, iolat, issue_as_root,
iolat             485 block/blk-iolatency.c static void iolatency_record_time(struct iolatency_grp *iolat,
iolat             507 block/blk-iolatency.c 	if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) {
iolat             508 block/blk-iolatency.c 		u64 sub = iolat->min_lat_nsec;
iolat             510 block/blk-iolatency.c 			blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time);
iolat             514 block/blk-iolatency.c 	latency_stat_record_time(iolat, req_time);
iolat             520 block/blk-iolatency.c static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now)
iolat             522 block/blk-iolatency.c 	struct blkcg_gq *blkg = lat_to_blkg(iolat);
iolat             529 block/blk-iolatency.c 	latency_stat_init(iolat, &stat);
iolat             533 block/blk-iolatency.c 		s = per_cpu_ptr(iolat->stats, cpu);
iolat             534 block/blk-iolatency.c 		latency_stat_sum(iolat, &stat, s);
iolat             535 block/blk-iolatency.c 		latency_stat_init(iolat, s);
iolat             545 block/blk-iolatency.c 	iolat_update_total_lat_avg(iolat, &stat);
iolat             548 block/blk-iolatency.c 	if (latency_sum_ok(iolat, &stat) &&
iolat             555 block/blk-iolatency.c 	latency_stat_sum(iolat, &iolat->cur_stat, &stat);
iolat             556 block/blk-iolatency.c 	lat_info->nr_samples -= iolat->nr_samples;
iolat             557 block/blk-iolatency.c 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);
iolat             558 block/blk-iolatency.c 	iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat);
iolat             564 block/blk-iolatency.c 	if (latency_sum_ok(iolat, &iolat->cur_stat) &&
iolat             565 block/blk-iolatency.c 	    latency_sum_ok(iolat, &stat)) {
iolat             566 block/blk-iolatency.c 		if (latency_stat_samples(iolat, &iolat->cur_stat) <
iolat             569 block/blk-iolatency.c 		if (lat_info->scale_grp == iolat) {
iolat             571 block/blk-iolatency.c 			scale_cookie_change(iolat->blkiolat, lat_info, true);
iolat             574 block/blk-iolatency.c 		   lat_info->scale_lat >= iolat->min_lat_nsec) {
iolat             577 block/blk-iolatency.c 		    lat_info->scale_lat > iolat->min_lat_nsec) {
iolat             578 block/blk-iolatency.c 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);
iolat             579 block/blk-iolatency.c 			lat_info->scale_grp = iolat;
iolat             581 block/blk-iolatency.c 		scale_cookie_change(iolat->blkiolat, lat_info, false);
iolat             583 block/blk-iolatency.c 	latency_stat_init(iolat, &iolat->cur_stat);
iolat             592 block/blk-iolatency.c 	struct iolatency_grp *iolat;
iolat             603 block/blk-iolatency.c 	iolat = blkg_to_lat(bio->bi_blkg);
iolat             604 block/blk-iolatency.c 	if (!iolat)
iolat             607 block/blk-iolatency.c 	enabled = blk_iolatency_enabled(iolat->blkiolat);
iolat             612 block/blk-iolatency.c 		iolat = blkg_to_lat(blkg);
iolat             613 block/blk-iolatency.c 		if (!iolat) {
iolat             617 block/blk-iolatency.c 		rqw = &iolat->rq_wait;
iolat             625 block/blk-iolatency.c 		if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) {
iolat             626 block/blk-iolatency.c 			iolatency_record_time(iolat, &bio->bi_issue, now,
iolat             628 block/blk-iolatency.c 			window_start = atomic64_read(&iolat->window_start);
iolat             630 block/blk-iolatency.c 			    (now - window_start) >= iolat->cur_win_nsec) {
iolat             631 block/blk-iolatency.c 				if (atomic64_cmpxchg(&iolat->window_start,
iolat             633 block/blk-iolatency.c 					iolatency_check_latencies(iolat, now);
iolat             666 block/blk-iolatency.c 		struct iolatency_grp *iolat;
iolat             678 block/blk-iolatency.c 		iolat = blkg_to_lat(blkg);
iolat             679 block/blk-iolatency.c 		if (!iolat)
iolat             682 block/blk-iolatency.c 		lat_info = &iolat->child_lat;
iolat             697 block/blk-iolatency.c 			scale_cookie_change(iolat->blkiolat, lat_info, true);
iolat             752 block/blk-iolatency.c 	struct iolatency_grp *iolat = blkg_to_lat(blkg);
iolat             753 block/blk-iolatency.c 	u64 oldval = iolat->min_lat_nsec;
iolat             755 block/blk-iolatency.c 	iolat->min_lat_nsec = val;
iolat             756 block/blk-iolatency.c 	iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE);
iolat             757 block/blk-iolatency.c 	iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec,
iolat             772 block/blk-iolatency.c 		struct iolatency_grp *iolat = blkg_to_lat(blkg->parent);
iolat             774 block/blk-iolatency.c 		if (!iolat)
iolat             777 block/blk-iolatency.c 		lat_info = &iolat->child_lat;
iolat             793 block/blk-iolatency.c 	struct iolatency_grp *iolat;
iolat             804 block/blk-iolatency.c 	iolat = blkg_to_lat(ctx.blkg);
iolat             831 block/blk-iolatency.c 	oldval = iolat->min_lat_nsec;
iolat             839 block/blk-iolatency.c 	if (oldval != iolat->min_lat_nsec) {
iolat             870 block/blk-iolatency.c 	struct iolatency_grp *iolat = pd_to_lat(pd);
iolat             873 block/blk-iolatency.c 	if (!dname || !iolat->min_lat_nsec)
iolat             876 block/blk-iolatency.c 		   dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC));
iolat             888 block/blk-iolatency.c static size_t iolatency_ssd_stat(struct iolatency_grp *iolat, char *buf,
iolat             894 block/blk-iolatency.c 	latency_stat_init(iolat, &stat);
iolat             898 block/blk-iolatency.c 		s = per_cpu_ptr(iolat->stats, cpu);
iolat             899 block/blk-iolatency.c 		latency_stat_sum(iolat, &stat, s);
iolat             903 block/blk-iolatency.c 	if (iolat->rq_depth.max_depth == UINT_MAX)
iolat             910 block/blk-iolatency.c 			 iolat->rq_depth.max_depth);
iolat             916 block/blk-iolatency.c 	struct iolatency_grp *iolat = pd_to_lat(pd);
iolat             923 block/blk-iolatency.c 	if (iolat->ssd)
iolat             924 block/blk-iolatency.c 		return iolatency_ssd_stat(iolat, buf, size);
iolat             926 block/blk-iolatency.c 	avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC);
iolat             927 block/blk-iolatency.c 	cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC);
iolat             928 block/blk-iolatency.c 	if (iolat->rq_depth.max_depth == UINT_MAX)
iolat             933 block/blk-iolatency.c 			 iolat->rq_depth.max_depth, avg_lat, cur_win);
iolat             941 block/blk-iolatency.c 	struct iolatency_grp *iolat;
iolat             943 block/blk-iolatency.c 	iolat = kzalloc_node(sizeof(*iolat), gfp, q->node);
iolat             944 block/blk-iolatency.c 	if (!iolat)
iolat             946 block/blk-iolatency.c 	iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat),
iolat             948 block/blk-iolatency.c 	if (!iolat->stats) {
iolat             949 block/blk-iolatency.c 		kfree(iolat);
iolat             952 block/blk-iolatency.c 	return &iolat->pd;
iolat             957 block/blk-iolatency.c 	struct iolatency_grp *iolat = pd_to_lat(pd);
iolat             958 block/blk-iolatency.c 	struct blkcg_gq *blkg = lat_to_blkg(iolat);
iolat             965 block/blk-iolatency.c 		iolat->ssd = true;
iolat             967 block/blk-iolatency.c 		iolat->ssd = false;
iolat             971 block/blk-iolatency.c 		stat = per_cpu_ptr(iolat->stats, cpu);
iolat             972 block/blk-iolatency.c 		latency_stat_init(iolat, stat);
iolat             975 block/blk-iolatency.c 	latency_stat_init(iolat, &iolat->cur_stat);
iolat             976 block/blk-iolatency.c 	rq_wait_init(&iolat->rq_wait);
iolat             977 block/blk-iolatency.c 	spin_lock_init(&iolat->child_lat.lock);
iolat             978 block/blk-iolatency.c 	iolat->rq_depth.queue_depth = blkg->q->nr_requests;
iolat             979 block/blk-iolatency.c 	iolat->rq_depth.max_depth = UINT_MAX;
iolat             980 block/blk-iolatency.c 	iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth;
iolat             981 block/blk-iolatency.c 	iolat->blkiolat = blkiolat;
iolat             982 block/blk-iolatency.c 	iolat->cur_win_nsec = 100 * NSEC_PER_MSEC;
iolat             983 block/blk-iolatency.c 	atomic64_set(&iolat->window_start, now);
iolat             991 block/blk-iolatency.c 		atomic_set(&iolat->scale_cookie,
iolat             994 block/blk-iolatency.c 		atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE);
iolat             997 block/blk-iolatency.c 	atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE);
iolat            1002 block/blk-iolatency.c 	struct iolatency_grp *iolat = pd_to_lat(pd);
iolat            1003 block/blk-iolatency.c 	struct blkcg_gq *blkg = lat_to_blkg(iolat);
iolat            1004 block/blk-iolatency.c 	struct blk_iolatency *blkiolat = iolat->blkiolat;
iolat            1017 block/blk-iolatency.c 	struct iolatency_grp *iolat = pd_to_lat(pd);
iolat            1018 block/blk-iolatency.c 	free_percpu(iolat->stats);
iolat            1019 block/blk-iolatency.c 	kfree(iolat);