rwb 78 block/blk-wbt.c static inline bool rwb_enabled(struct rq_wb *rwb) rwb 80 block/blk-wbt.c return rwb && rwb->wb_normal != 0; rwb 83 block/blk-wbt.c static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) rwb 85 block/blk-wbt.c if (rwb_enabled(rwb)) { rwb 97 block/blk-wbt.c static bool wb_recent_wait(struct rq_wb *rwb) rwb 99 block/blk-wbt.c struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb; rwb 104 block/blk-wbt.c static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, rwb 108 block/blk-wbt.c return &rwb->rq_wait[WBT_RWQ_KSWAPD]; rwb 110 block/blk-wbt.c return &rwb->rq_wait[WBT_RWQ_DISCARD]; rwb 112 block/blk-wbt.c return &rwb->rq_wait[WBT_RWQ_BG]; rwb 115 block/blk-wbt.c static void rwb_wake_all(struct rq_wb *rwb) rwb 120 block/blk-wbt.c struct rq_wait *rqw = &rwb->rq_wait[i]; rwb 127 block/blk-wbt.c static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, rwb 138 block/blk-wbt.c if (unlikely(!rwb_enabled(rwb))) { rwb 139 block/blk-wbt.c rwb_wake_all(rwb); rwb 149 block/blk-wbt.c limit = rwb->wb_background; rwb 150 block/blk-wbt.c else if (rwb->wc && !wb_recent_wait(rwb)) rwb 153 block/blk-wbt.c limit = rwb->wb_normal; rwb 164 block/blk-wbt.c if (!inflight || diff >= rwb->wb_background / 2) rwb 171 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 177 block/blk-wbt.c rqw = get_rq_wait(rwb, wb_acct); rwb 178 block/blk-wbt.c wbt_rqw_done(rwb, rqw, wb_acct); rwb 187 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 190 block/blk-wbt.c if (rwb->sync_cookie == rq) { rwb 191 block/blk-wbt.c rwb->sync_issue = 0; rwb 192 block/blk-wbt.c rwb->sync_cookie = NULL; rwb 196 block/blk-wbt.c wb_timestamp(rwb, &rwb->last_comp); rwb 198 block/blk-wbt.c WARN_ON_ONCE(rq == rwb->sync_cookie); rwb 216 block/blk-wbt.c static u64 rwb_sync_issue_lat(struct rq_wb *rwb) rwb 218 block/blk-wbt.c u64 now, issue = READ_ONCE(rwb->sync_issue); rwb 220 block/blk-wbt.c if (!issue || !rwb->sync_cookie) rwb 234 block/blk-wbt.c static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) rwb 236 block/blk-wbt.c struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info; rwb 237 block/blk-wbt.c struct rq_depth *rqd = &rwb->rq_depth; rwb 249 block/blk-wbt.c thislat = rwb_sync_issue_lat(rwb); rwb 250 block/blk-wbt.c if (thislat > rwb->cur_win_nsec || rwb 251 block/blk-wbt.c (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) { rwb 266 block/blk-wbt.c if (stat[WRITE].nr_samples || wb_recent_wait(rwb) || rwb 267 block/blk-wbt.c wbt_inflight(rwb)) rwb 275 block/blk-wbt.c if (stat[READ].min > rwb->min_lat_nsec) { rwb 287 block/blk-wbt.c static void rwb_trace_step(struct rq_wb *rwb, const char *msg) rwb 289 block/blk-wbt.c struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info; rwb 290 block/blk-wbt.c struct rq_depth *rqd = &rwb->rq_depth; rwb 292 block/blk-wbt.c trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec, rwb 293 block/blk-wbt.c rwb->wb_background, rwb->wb_normal, rqd->max_depth); rwb 296 block/blk-wbt.c static void calc_wb_limits(struct rq_wb *rwb) rwb 298 block/blk-wbt.c if (rwb->min_lat_nsec == 0) { rwb 299 block/blk-wbt.c rwb->wb_normal = rwb->wb_background = 0; rwb 300 block/blk-wbt.c } else if (rwb->rq_depth.max_depth <= 2) { rwb 301 block/blk-wbt.c rwb->wb_normal = rwb->rq_depth.max_depth; rwb 302 block/blk-wbt.c rwb->wb_background = 1; rwb 304 block/blk-wbt.c rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2; rwb 305 block/blk-wbt.c rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4; rwb 309 block/blk-wbt.c static void scale_up(struct rq_wb *rwb) rwb 311 block/blk-wbt.c if (!rq_depth_scale_up(&rwb->rq_depth)) rwb 313 block/blk-wbt.c calc_wb_limits(rwb); rwb 314 block/blk-wbt.c rwb->unknown_cnt = 0; rwb 315 block/blk-wbt.c rwb_wake_all(rwb); rwb 316 block/blk-wbt.c rwb_trace_step(rwb, "scale up"); rwb 319 block/blk-wbt.c static void scale_down(struct rq_wb *rwb, bool hard_throttle) rwb 321 block/blk-wbt.c if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle)) rwb 323 block/blk-wbt.c calc_wb_limits(rwb); rwb 324 block/blk-wbt.c rwb->unknown_cnt = 0; rwb 325 block/blk-wbt.c rwb_trace_step(rwb, "scale down"); rwb 328 block/blk-wbt.c static void rwb_arm_timer(struct rq_wb *rwb) rwb 330 block/blk-wbt.c struct rq_depth *rqd = &rwb->rq_depth; rwb 339 block/blk-wbt.c rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4, rwb 346 block/blk-wbt.c rwb->cur_win_nsec = rwb->win_nsec; rwb 349 block/blk-wbt.c blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec); rwb 354 block/blk-wbt.c struct rq_wb *rwb = cb->data; rwb 355 block/blk-wbt.c struct rq_depth *rqd = &rwb->rq_depth; rwb 356 block/blk-wbt.c unsigned int inflight = wbt_inflight(rwb); rwb 359 block/blk-wbt.c status = latency_exceeded(rwb, cb->stat); rwb 361 block/blk-wbt.c trace_wbt_timer(rwb->rqos.q->backing_dev_info, status, rqd->scale_step, rwb 371 block/blk-wbt.c scale_down(rwb, true); rwb 374 block/blk-wbt.c scale_up(rwb); rwb 382 block/blk-wbt.c scale_up(rwb); rwb 385 block/blk-wbt.c if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP) rwb 393 block/blk-wbt.c scale_up(rwb); rwb 395 block/blk-wbt.c scale_down(rwb, false); rwb 405 block/blk-wbt.c rwb_arm_timer(rwb); rwb 408 block/blk-wbt.c static void __wbt_update_limits(struct rq_wb *rwb) rwb 410 block/blk-wbt.c struct rq_depth *rqd = &rwb->rq_depth; rwb 416 block/blk-wbt.c calc_wb_limits(rwb); rwb 418 block/blk-wbt.c rwb_wake_all(rwb); rwb 448 block/blk-wbt.c static bool close_io(struct rq_wb *rwb) rwb 452 block/blk-wbt.c return time_before(now, rwb->last_issue + HZ / 10) || rwb 453 block/blk-wbt.c time_before(now, rwb->last_comp + HZ / 10); rwb 458 block/blk-wbt.c static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw) rwb 466 block/blk-wbt.c if (!rwb_enabled(rwb)) rwb 470 block/blk-wbt.c return rwb->wb_background; rwb 480 block/blk-wbt.c if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd()) rwb 481 block/blk-wbt.c limit = rwb->rq_depth.max_depth; rwb 482 block/blk-wbt.c else if ((rw & REQ_BACKGROUND) || close_io(rwb)) { rwb 487 block/blk-wbt.c limit = rwb->wb_background; rwb 489 block/blk-wbt.c limit = rwb->wb_normal; rwb 495 block/blk-wbt.c struct rq_wb *rwb; rwb 503 block/blk-wbt.c return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw)); rwb 509 block/blk-wbt.c wbt_rqw_done(data->rwb, rqw, data->wb_acct); rwb 516 block/blk-wbt.c static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, rwb 519 block/blk-wbt.c struct rq_wait *rqw = get_rq_wait(rwb, wb_acct); rwb 521 block/blk-wbt.c .rwb = rwb, rwb 529 block/blk-wbt.c static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio) rwb 547 block/blk-wbt.c static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) rwb 551 block/blk-wbt.c if (!rwb_enabled(rwb)) rwb 556 block/blk-wbt.c } else if (wbt_should_throttle(rwb, bio)) { rwb 568 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 569 block/blk-wbt.c enum wbt_flags flags = bio_to_wbt_flags(rwb, bio); rwb 581 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 584 block/blk-wbt.c flags = bio_to_wbt_flags(rwb, bio); rwb 587 block/blk-wbt.c wb_timestamp(rwb, &rwb->last_issue); rwb 591 block/blk-wbt.c __wbt_wait(rwb, flags, bio->bi_opf); rwb 593 block/blk-wbt.c if (!blk_stat_is_active(rwb->cb)) rwb 594 block/blk-wbt.c rwb_arm_timer(rwb); rwb 599 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 600 block/blk-wbt.c rq->wbt_flags |= bio_to_wbt_flags(rwb, bio); rwb 605 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 607 block/blk-wbt.c if (!rwb_enabled(rwb)) rwb 617 block/blk-wbt.c if (wbt_is_read(rq) && !rwb->sync_issue) { rwb 618 block/blk-wbt.c rwb->sync_cookie = rq; rwb 619 block/blk-wbt.c rwb->sync_issue = rq->io_start_time_ns; rwb 625 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 626 block/blk-wbt.c if (!rwb_enabled(rwb)) rwb 628 block/blk-wbt.c if (rq == rwb->sync_cookie) { rwb 629 block/blk-wbt.c rwb->sync_issue = 0; rwb 630 block/blk-wbt.c rwb->sync_cookie = NULL; rwb 693 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 696 block/blk-wbt.c blk_stat_remove_callback(q, rwb->cb); rwb 697 block/blk-wbt.c blk_stat_free_callback(rwb->cb); rwb 698 block/blk-wbt.c kfree(rwb); rwb 707 block/blk-wbt.c struct rq_wb *rwb; rwb 710 block/blk-wbt.c rwb = RQWB(rqos); rwb 711 block/blk-wbt.c if (rwb->enable_state == WBT_STATE_ON_DEFAULT) { rwb 712 block/blk-wbt.c blk_stat_deactivate(rwb->cb); rwb 713 block/blk-wbt.c rwb->wb_normal = 0; rwb 722 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 724 block/blk-wbt.c seq_printf(m, "%llu\n", rwb->cur_win_nsec); rwb 731 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 733 block/blk-wbt.c seq_printf(m, "%d\n", rwb->enable_state); rwb 748 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 753 block/blk-wbt.c atomic_read(&rwb->rq_wait[i].inflight)); rwb 760 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 762 block/blk-wbt.c seq_printf(m, "%lu\n", rwb->min_lat_nsec); rwb 769 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 771 block/blk-wbt.c seq_printf(m, "%u\n", rwb->unknown_cnt); rwb 778 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 780 block/blk-wbt.c seq_printf(m, "%u\n", rwb->wb_normal); rwb 787 block/blk-wbt.c struct rq_wb *rwb = RQWB(rqos); rwb 789 block/blk-wbt.c seq_printf(m, "%u\n", rwb->wb_background); rwb 822 block/blk-wbt.c struct rq_wb *rwb; rwb 825 block/blk-wbt.c rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); rwb 826 block/blk-wbt.c if (!rwb) rwb 829 block/blk-wbt.c rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); rwb 830 block/blk-wbt.c if (!rwb->cb) { rwb 831 block/blk-wbt.c kfree(rwb); rwb 836 block/blk-wbt.c rq_wait_init(&rwb->rq_wait[i]); rwb 838 block/blk-wbt.c rwb->rqos.id = RQ_QOS_WBT; rwb 839 block/blk-wbt.c rwb->rqos.ops = &wbt_rqos_ops; rwb 840 block/blk-wbt.c rwb->rqos.q = q; rwb 841 block/blk-wbt.c rwb->last_comp = rwb->last_issue = jiffies; rwb 842 block/blk-wbt.c rwb->win_nsec = RWB_WINDOW_NSEC; rwb 843 block/blk-wbt.c rwb->enable_state = WBT_STATE_ON_DEFAULT; rwb 844 block/blk-wbt.c rwb->wc = 1; rwb 845 block/blk-wbt.c rwb->rq_depth.default_depth = RWB_DEF_DEPTH; rwb 846 block/blk-wbt.c __wbt_update_limits(rwb); rwb 851 block/blk-wbt.c rq_qos_add(q, &rwb->rqos); rwb 852 block/blk-wbt.c blk_stat_add_callback(q, rwb->cb); rwb 854 block/blk-wbt.c rwb->min_lat_nsec = wbt_default_latency_nsec(q); rwb 856 block/blk-wbt.c wbt_queue_depth_changed(&rwb->rqos); rwb 77 block/blk-wbt.h static inline unsigned int wbt_inflight(struct rq_wb *rwb) rwb 82 block/blk-wbt.h ret += atomic_read(&rwb->rq_wait[i].inflight); rwb 40 drivers/lightnvm/pblk-cache.c ret = pblk_rb_may_write_user(&pblk->rwb, bio, nr_entries, &bpos); rwb 66 drivers/lightnvm/pblk-cache.c pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + i); rwb 67 drivers/lightnvm/pblk-cache.c pblk_rb_write_entry_user(&pblk->rwb, data, w_ctx, pos); rwb 105 drivers/lightnvm/pblk-cache.c if (!pblk_rb_may_write_gc(&pblk->rwb, gc_rq->secs_to_gc, &bpos)) { rwb 119 drivers/lightnvm/pblk-cache.c pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + valid_entries); rwb 120 drivers/lightnvm/pblk-cache.c pblk_rb_write_entry_gc(&pblk->rwb, data, w_ctx, gc_rq->line, rwb 379 drivers/lightnvm/pblk-core.c unsigned int secs_avail = pblk_rb_read_count(&pblk->rwb); rwb 397 drivers/lightnvm/pblk-core.c pblk_rb_flush(&pblk->rwb); rwb 399 drivers/lightnvm/pblk-core.c if (!pblk_rb_sync_count(&pblk->rwb)) rwb 1973 drivers/lightnvm/pblk-core.c BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa))); rwb 1988 drivers/lightnvm/pblk-core.c BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa_new))); rwb 171 drivers/lightnvm/pblk-init.c if (pblk_rb_tear_down_check(&pblk->rwb)) rwb 174 drivers/lightnvm/pblk-init.c pblk_rb_free(&pblk->rwb); rwb 193 drivers/lightnvm/pblk-init.c return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs); rwb 1090 drivers/lightnvm/pblk-init.c WARN(pblk_rb_read_count(&pblk->rwb), rwb 1093 drivers/lightnvm/pblk-init.c WARN(pblk_rb_sync_count(&pblk->rwb), rwb 1117 drivers/lightnvm/pblk-init.c pblk_rb_sync_l2p(&pblk->rwb); rwb 1255 drivers/lightnvm/pblk-init.c pblk->rwb.nr_entries); rwb 77 drivers/lightnvm/pblk-map.c w_ctx = pblk_rb_w_ctx(&pblk->rwb, sentry + i); rwb 75 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 239 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 340 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 364 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 454 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 476 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 479 drivers/lightnvm/pblk-rb.c if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem)) rwb 497 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 524 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 554 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 647 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 811 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); rwb 37 drivers/lightnvm/pblk-read.c return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa); rwb 110 drivers/lightnvm/pblk-sysfs.c return pblk_rb_sysfs(&pblk->rwb, page); rwb 26 drivers/lightnvm/pblk-write.c struct pblk_rb *rwb = &pblk->rwb; rwb 35 drivers/lightnvm/pblk-write.c w_ctx = pblk_rb_w_ctx(rwb, pos); rwb 44 drivers/lightnvm/pblk-write.c atomic_dec(&rwb->inflight_flush_point); rwb 60 drivers/lightnvm/pblk-write.c ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid); rwb 88 drivers/lightnvm/pblk-write.c pos = pblk_rb_sync_init(&pblk->rwb, &flags); rwb 104 drivers/lightnvm/pblk-write.c pblk_rb_sync_end(&pblk->rwb, &flags); rwb 150 drivers/lightnvm/pblk-write.c struct pblk_rb *rb = &pblk->rwb; rwb 593 drivers/lightnvm/pblk-write.c secs_avail = pblk_rb_read_count(&pblk->rwb); rwb 597 drivers/lightnvm/pblk-write.c secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb); rwb 610 drivers/lightnvm/pblk-write.c pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com); rwb 622 drivers/lightnvm/pblk-write.c if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync, rwb 608 drivers/lightnvm/pblk.h struct pblk_rb rwb;