bfqq 222 block/bfq-cgroup.c void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, bfqq 227 block/bfq-cgroup.c if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue)) bfqq 228 block/bfq-cgroup.c bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq)); bfqq 257 block/bfq-cgroup.c void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, bfqq 308 block/bfq-cgroup.c struct bfq_group *bfqq_group(struct bfq_queue *bfqq) bfqq 310 block/bfq-cgroup.c struct bfq_entity *group_entity = bfqq->entity.parent; bfqq 314 block/bfq-cgroup.c bfqq->bfqd->root_group; bfqq 415 block/bfq-cgroup.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 419 block/bfq-cgroup.c if (bfqq) { bfqq 420 block/bfq-cgroup.c bfqq->ioprio = bfqq->new_ioprio; bfqq 421 block/bfq-cgroup.c bfqq->ioprio_class = bfqq->new_ioprio_class; bfqq 623 block/bfq-cgroup.c void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 626 block/bfq-cgroup.c struct bfq_entity *entity = &bfqq->entity; bfqq 632 block/bfq-cgroup.c bfqq->ref++; bfqq 640 block/bfq-cgroup.c if (bfqq == bfqd->in_service_queue) bfqq 644 block/bfq-cgroup.c if (bfq_bfqq_busy(bfqq)) bfqq 645 block/bfq-cgroup.c bfq_deactivate_bfqq(bfqd, bfqq, false, false); bfqq 648 block/bfq-cgroup.c bfqg_and_blkg_put(bfqq_group(bfqq)); bfqq 655 block/bfq-cgroup.c if (bfq_bfqq_busy(bfqq)) { bfqq 657 block/bfq-cgroup.c bfq_pos_tree_add_move(bfqd, bfqq); bfqq 658 block/bfq-cgroup.c bfq_activate_bfqq(bfqd, bfqq); bfqq 664 block/bfq-cgroup.c bfq_put_queue(bfqq); bfqq 808 block/bfq-cgroup.c struct bfq_queue *bfqq; bfqq 823 block/bfq-cgroup.c bfqq = bfq_entity_to_bfqq(child_entity); bfqq 824 block/bfq-cgroup.c bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); bfqq 1376 block/bfq-cgroup.c void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1381 block/bfq-cgroup.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 1385 block/bfq-cgroup.c if (bfqq) { bfqq 1386 block/bfq-cgroup.c bfqq->ioprio = bfqq->new_ioprio; bfqq 1387 block/bfq-cgroup.c bfqq->ioprio_class = bfqq->new_ioprio_class; bfqq 1404 block/bfq-cgroup.c struct bfq_group *bfqq_group(struct bfq_queue *bfqq) bfqq 1406 block/bfq-cgroup.c return bfqq->bfqd->root_group; bfqq 135 block/bfq-iosched.c void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \ bfqq 137 block/bfq-iosched.c __set_bit(BFQQF_##name, &(bfqq)->flags); \ bfqq 139 block/bfq-iosched.c void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \ bfqq 141 block/bfq-iosched.c __clear_bit(BFQQF_##name, &(bfqq)->flags); \ bfqq 143 block/bfq-iosched.c int bfq_bfqq_##name(const struct bfq_queue *bfqq) \ bfqq 145 block/bfq-iosched.c return test_bit(BFQQF_##name, &(bfqq)->flags); \ bfqq 236 block/bfq-iosched.c #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) bfqq 244 block/bfq-iosched.c #define BFQQ_TOTALLY_SEEKY(bfqq) (bfqq->seek_history == -1) bfqq 370 block/bfq-iosched.c return bic->bfqq[is_sync]; bfqq 373 block/bfq-iosched.c void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync) bfqq 375 block/bfq-iosched.c bic->bfqq[is_sync] = bfqq; bfqq 429 block/bfq-iosched.c #define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) bfqq 430 block/bfq-iosched.c #define bfq_class_rt(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_RT) bfqq 556 block/bfq-iosched.c struct bfq_queue *bfqq = NULL; bfqq 564 block/bfq-iosched.c bfqq = rb_entry(parent, struct bfq_queue, pos_node); bfqq 570 block/bfq-iosched.c if (sector > blk_rq_pos(bfqq->next_rq)) bfqq 572 block/bfq-iosched.c else if (sector < blk_rq_pos(bfqq->next_rq)) bfqq 577 block/bfq-iosched.c bfqq = NULL; bfqq 586 block/bfq-iosched.c bfqq ? bfqq->pid : 0); bfqq 588 block/bfq-iosched.c return bfqq; bfqq 591 block/bfq-iosched.c static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) bfqq 593 block/bfq-iosched.c return bfqq->service_from_backlogged > 0 && bfqq 594 block/bfq-iosched.c time_is_before_jiffies(bfqq->first_IO_time + bfqq 607 block/bfq-iosched.c bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 612 block/bfq-iosched.c if (bfqq->pos_root) { bfqq 613 block/bfq-iosched.c rb_erase(&bfqq->pos_node, bfqq->pos_root); bfqq 614 block/bfq-iosched.c bfqq->pos_root = NULL; bfqq 618 block/bfq-iosched.c if (bfqq == &bfqd->oom_bfqq) bfqq 626 block/bfq-iosched.c if (bfq_too_late_for_merging(bfqq)) bfqq 629 block/bfq-iosched.c if (bfq_class_idle(bfqq)) bfqq 631 block/bfq-iosched.c if (!bfqq->next_rq) bfqq 634 block/bfq-iosched.c bfqq->pos_root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; bfqq 635 block/bfq-iosched.c __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, bfqq 636 block/bfq-iosched.c blk_rq_pos(bfqq->next_rq), &parent, &p); bfqq 638 block/bfq-iosched.c rb_link_node(&bfqq->pos_node, parent, p); bfqq 639 block/bfq-iosched.c rb_insert_color(&bfqq->pos_node, bfqq->pos_root); bfqq 641 block/bfq-iosched.c bfqq->pos_root = NULL; bfqq 678 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 680 block/bfq-iosched.c bool smallest_weight = bfqq && bfqq 681 block/bfq-iosched.c bfqq->weight_counter && bfqq 682 block/bfq-iosched.c bfqq->weight_counter == bfqq 722 block/bfq-iosched.c void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 725 block/bfq-iosched.c struct bfq_entity *entity = &bfqq->entity; bfqq 741 block/bfq-iosched.c if (bfqq->weight_counter) bfqq 751 block/bfq-iosched.c bfqq->weight_counter = __counter; bfqq 762 block/bfq-iosched.c bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), bfqq 777 block/bfq-iosched.c if (unlikely(!bfqq->weight_counter)) bfqq 780 block/bfq-iosched.c bfqq->weight_counter->weight = entity->weight; bfqq 781 block/bfq-iosched.c rb_link_node(&bfqq->weight_counter->weights_node, parent, new); bfqq 782 block/bfq-iosched.c rb_insert_color_cached(&bfqq->weight_counter->weights_node, root, bfqq 786 block/bfq-iosched.c bfqq->weight_counter->num_active++; bfqq 787 block/bfq-iosched.c bfqq->ref++; bfqq 797 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 800 block/bfq-iosched.c if (!bfqq->weight_counter) bfqq 803 block/bfq-iosched.c bfqq->weight_counter->num_active--; bfqq 804 block/bfq-iosched.c if (bfqq->weight_counter->num_active > 0) bfqq 807 block/bfq-iosched.c rb_erase_cached(&bfqq->weight_counter->weights_node, root); bfqq 808 block/bfq-iosched.c kfree(bfqq->weight_counter); bfqq 811 block/bfq-iosched.c bfqq->weight_counter = NULL; bfqq 812 block/bfq-iosched.c bfq_put_queue(bfqq); bfqq 820 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 822 block/bfq-iosched.c struct bfq_entity *entity = bfqq->entity.parent; bfqq 864 block/bfq-iosched.c __bfq_weights_tree_remove(bfqd, bfqq, bfqq 871 block/bfq-iosched.c static struct request *bfq_check_fifo(struct bfq_queue *bfqq, bfqq 876 block/bfq-iosched.c if (bfq_bfqq_fifo_expire(bfqq)) bfqq 879 block/bfq-iosched.c bfq_mark_bfqq_fifo_expire(bfqq); bfqq 881 block/bfq-iosched.c rq = rq_entry_fifo(bfqq->fifo.next); bfqq 886 block/bfq-iosched.c bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); bfqq 891 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 899 block/bfq-iosched.c next = bfq_check_fifo(bfqq, last); bfqq 909 block/bfq-iosched.c rbnext = rb_first(&bfqq->sort_list); bfqq 919 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 921 block/bfq-iosched.c if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 || bfqq 922 block/bfq-iosched.c bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) bfqq 940 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 942 block/bfq-iosched.c struct bfq_entity *entity = &bfqq->entity; bfqq 943 block/bfq-iosched.c struct request *next_rq = bfqq->next_rq; bfqq 949 block/bfq-iosched.c if (bfqq == bfqd->in_service_queue) bfqq 957 block/bfq-iosched.c max_t(unsigned long, bfqq->max_budget, bfqq 958 block/bfq-iosched.c bfq_serv_to_charge(next_rq, bfqq)), bfqq 962 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", bfqq 964 block/bfq-iosched.c bfq_requeue_bfqq(bfqd, bfqq, false); bfqq 1002 block/bfq-iosched.c static void switch_back_to_interactive_wr(struct bfq_queue *bfqq, bfqq 1005 block/bfq-iosched.c bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq 1006 block/bfq-iosched.c bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); bfqq 1007 block/bfq-iosched.c bfqq->last_wr_start_finish = bfqq->wr_start_at_switch_to_srt; bfqq 1011 block/bfq-iosched.c bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, bfqq 1014 block/bfq-iosched.c unsigned int old_wr_coeff = bfqq->wr_coeff; bfqq 1015 block/bfq-iosched.c bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq); bfqq 1018 block/bfq-iosched.c bfq_mark_bfqq_has_short_ttime(bfqq); bfqq 1020 block/bfq-iosched.c bfq_clear_bfqq_has_short_ttime(bfqq); bfqq 1023 block/bfq-iosched.c bfq_mark_bfqq_IO_bound(bfqq); bfqq 1025 block/bfq-iosched.c bfq_clear_bfqq_IO_bound(bfqq); bfqq 1027 block/bfq-iosched.c bfqq->entity.new_weight = bic->saved_weight; bfqq 1028 block/bfq-iosched.c bfqq->ttime = bic->saved_ttime; bfqq 1029 block/bfq-iosched.c bfqq->wr_coeff = bic->saved_wr_coeff; bfqq 1030 block/bfq-iosched.c bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; bfqq 1031 block/bfq-iosched.c bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; bfqq 1032 block/bfq-iosched.c bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time; bfqq 1034 block/bfq-iosched.c if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || bfqq 1035 block/bfq-iosched.c time_is_before_jiffies(bfqq->last_wr_start_finish + bfqq 1036 block/bfq-iosched.c bfqq->wr_cur_max_time))) { bfqq 1037 block/bfq-iosched.c if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && bfqq 1038 block/bfq-iosched.c !bfq_bfqq_in_large_burst(bfqq) && bfqq 1039 block/bfq-iosched.c time_is_after_eq_jiffies(bfqq->wr_start_at_switch_to_srt + bfqq 1041 block/bfq-iosched.c switch_back_to_interactive_wr(bfqq, bfqd); bfqq 1043 block/bfq-iosched.c bfqq->wr_coeff = 1; bfqq 1044 block/bfq-iosched.c bfq_log_bfqq(bfqq->bfqd, bfqq, bfqq 1050 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 1055 block/bfq-iosched.c if (old_wr_coeff == 1 && bfqq->wr_coeff > 1) bfqq 1057 block/bfq-iosched.c else if (old_wr_coeff > 1 && bfqq->wr_coeff == 1) bfqq 1061 block/bfq-iosched.c static int bfqq_process_refs(struct bfq_queue *bfqq) bfqq 1063 block/bfq-iosched.c return bfqq->ref - bfqq->allocated - bfqq->entity.on_st - bfqq 1064 block/bfq-iosched.c (bfqq->weight_counter != NULL); bfqq 1068 block/bfq-iosched.c static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 1082 block/bfq-iosched.c hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); bfqq 1087 block/bfq-iosched.c bfqd->burst_parent_entity = bfqq->entity.parent; bfqq 1091 block/bfq-iosched.c static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 1113 block/bfq-iosched.c bfq_mark_bfqq_in_large_burst(bfqq); bfqq 1131 block/bfq-iosched.c hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); bfqq 1243 block/bfq-iosched.c static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 1250 block/bfq-iosched.c if (!hlist_unhashed(&bfqq->burst_list_node) || bfqq 1251 block/bfq-iosched.c bfq_bfqq_in_large_burst(bfqq) || bfqq 1252 block/bfq-iosched.c time_is_after_eq_jiffies(bfqq->split_time + bfqq 1275 block/bfq-iosched.c bfqq->entity.parent != bfqd->burst_parent_entity) { bfqq 1277 block/bfq-iosched.c bfq_reset_burst_list(bfqd, bfqq); bfqq 1287 block/bfq-iosched.c bfq_mark_bfqq_in_large_burst(bfqq); bfqq 1296 block/bfq-iosched.c bfq_add_to_burst(bfqd, bfqq); bfqq 1309 block/bfq-iosched.c static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) bfqq 1311 block/bfq-iosched.c struct bfq_entity *entity = &bfqq->entity; bfqq 1446 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 1449 block/bfq-iosched.c struct bfq_entity *entity = &bfqq->entity; bfqq 1458 block/bfq-iosched.c if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time && bfqq 1459 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq) > 0) { bfqq 1478 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq), bfqq 1479 block/bfq-iosched.c bfqq->max_budget); bfqq 1499 block/bfq-iosched.c entity->budget = max_t(unsigned long, bfqq->max_budget, bfqq 1500 block/bfq-iosched.c bfq_serv_to_charge(bfqq->next_rq, bfqq)); bfqq 1501 block/bfq-iosched.c bfq_clear_bfqq_non_blocking_wait_rq(bfqq); bfqq 1515 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 1525 block/bfq-iosched.c bfqq->service_from_wr = 0; bfqq 1526 block/bfq-iosched.c bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq 1527 block/bfq-iosched.c bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); bfqq 1540 block/bfq-iosched.c bfqq->wr_start_at_switch_to_srt = bfqq 1542 block/bfq-iosched.c bfqq->wr_coeff = bfqd->bfq_wr_coeff * bfqq 1544 block/bfq-iosched.c bfqq->wr_cur_max_time = bfqq 1557 block/bfq-iosched.c bfqq->entity.budget = min_t(unsigned long, bfqq 1558 block/bfq-iosched.c bfqq->entity.budget, bfqq 1562 block/bfq-iosched.c bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq 1563 block/bfq-iosched.c bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); bfqq 1565 block/bfq-iosched.c bfqq->wr_coeff = 1; bfqq 1596 block/bfq-iosched.c if (bfqq->wr_cur_max_time != bfqq 1598 block/bfq-iosched.c bfqq->wr_start_at_switch_to_srt = bfqq 1599 block/bfq-iosched.c bfqq->last_wr_start_finish; bfqq 1601 block/bfq-iosched.c bfqq->wr_cur_max_time = bfqq 1603 block/bfq-iosched.c bfqq->wr_coeff = bfqd->bfq_wr_coeff * bfqq 1606 block/bfq-iosched.c bfqq->last_wr_start_finish = jiffies; bfqq 1612 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 1614 block/bfq-iosched.c return bfqq->dispatched == 0 && bfqq 1616 block/bfq-iosched.c bfqq->budget_timeout + bfqq 1625 block/bfq-iosched.c static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq, bfqq 1630 block/bfq-iosched.c if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class) bfqq 1633 block/bfq-iosched.c if (in_serv_bfqq->entity.parent == bfqq->entity.parent) { bfqq 1634 block/bfq-iosched.c bfqq_weight = bfqq->entity.weight; bfqq 1637 block/bfq-iosched.c if (bfqq->entity.parent) bfqq 1638 block/bfq-iosched.c bfqq_weight = bfqq->entity.parent->weight; bfqq 1640 block/bfq-iosched.c bfqq_weight = bfqq->entity.weight; bfqq 1651 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 1658 block/bfq-iosched.c idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), bfqq 1665 block/bfq-iosched.c bfqq->ttime.last_end_request + bfqq 1676 block/bfq-iosched.c in_burst = bfq_bfqq_in_large_burst(bfqq); bfqq 1678 block/bfq-iosched.c !BFQQ_TOTALLY_SEEKY(bfqq) && bfqq 1680 block/bfq-iosched.c time_is_before_jiffies(bfqq->soft_rt_next_start) && bfqq 1681 block/bfq-iosched.c bfqq->dispatched == 0; bfqq 1684 block/bfq-iosched.c (bfqq->wr_coeff > 1 || bfqq 1685 block/bfq-iosched.c (bfq_bfqq_sync(bfqq) && bfqq 1686 block/bfq-iosched.c bfqq->bic && (*interactive || soft_rt))); bfqq 1693 block/bfq-iosched.c bfq_bfqq_update_budg_for_activation(bfqd, bfqq, bfqq 1709 block/bfq-iosched.c if (likely(!bfq_bfqq_just_created(bfqq)) && bfqq 1712 block/bfq-iosched.c bfqq->budget_timeout + bfqq 1714 block/bfq-iosched.c hlist_del_init(&bfqq->burst_list_node); bfqq 1715 block/bfq-iosched.c bfq_clear_bfqq_in_large_burst(bfqq); bfqq 1718 block/bfq-iosched.c bfq_clear_bfqq_just_created(bfqq); bfqq 1721 block/bfq-iosched.c if (!bfq_bfqq_IO_bound(bfqq)) { bfqq 1723 block/bfq-iosched.c bfqq->requests_within_timer++; bfqq 1724 block/bfq-iosched.c if (bfqq->requests_within_timer >= bfqq 1726 block/bfq-iosched.c bfq_mark_bfqq_IO_bound(bfqq); bfqq 1728 block/bfq-iosched.c bfqq->requests_within_timer = 0; bfqq 1732 block/bfq-iosched.c if (unlikely(time_is_after_jiffies(bfqq->split_time))) bfqq 1734 block/bfq-iosched.c bfqq->split_time = bfqq 1737 block/bfq-iosched.c if (time_is_before_jiffies(bfqq->split_time + bfqq 1739 block/bfq-iosched.c bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, bfqq 1746 block/bfq-iosched.c if (old_wr_coeff != bfqq->wr_coeff) bfqq 1747 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 1751 block/bfq-iosched.c bfqq->last_idle_bklogged = jiffies; bfqq 1752 block/bfq-iosched.c bfqq->service_from_backlogged = 0; bfqq 1753 block/bfq-iosched.c bfq_clear_bfqq_softrt_update(bfqq); bfqq 1755 block/bfq-iosched.c bfq_add_bfqq_busy(bfqd, bfqq); bfqq 1791 block/bfq-iosched.c bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) || bfqq 1792 block/bfq-iosched.c bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) && bfqq 1799 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 1802 block/bfq-iosched.c bfqq->last_serv_time_ns = 0; bfqq 1856 block/bfq-iosched.c if (bfq_bfqq_has_short_ttime(bfqq)) bfqq 1857 block/bfq-iosched.c bfqq->inject_limit = 0; bfqq 1859 block/bfq-iosched.c bfqq->inject_limit = 1; bfqq 1861 block/bfq-iosched.c bfqq->decrease_time_jif = jiffies; bfqq 1866 block/bfq-iosched.c struct bfq_queue *bfqq = RQ_BFQQ(rq); bfqq 1867 block/bfq-iosched.c struct bfq_data *bfqd = bfqq->bfqd; bfqq 1869 block/bfq-iosched.c unsigned int old_wr_coeff = bfqq->wr_coeff; bfqq 1872 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); bfqq 1873 block/bfq-iosched.c bfqq->queued[rq_is_sync(rq)]++; bfqq 1876 block/bfq-iosched.c if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) { bfqq 1932 block/bfq-iosched.c !bfq_bfqq_has_short_ttime(bfqq) && bfqq 1935 block/bfq-iosched.c if (bfqd->last_completed_rq_bfqq != bfqq && bfqq 1937 block/bfq-iosched.c bfqq->waker_bfqq) { bfqq 1944 block/bfq-iosched.c bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq; bfqq 1966 block/bfq-iosched.c if (!hlist_unhashed(&bfqq->woken_list_node)) bfqq 1967 block/bfq-iosched.c hlist_del_init(&bfqq->woken_list_node); bfqq 1968 block/bfq-iosched.c hlist_add_head(&bfqq->woken_list_node, bfqq 1971 block/bfq-iosched.c bfq_clear_bfqq_has_waker(bfqq); bfqq 1973 block/bfq-iosched.c bfqq->waker_bfqq && bfqq 1974 block/bfq-iosched.c !bfq_bfqq_has_waker(bfqq)) { bfqq 1979 block/bfq-iosched.c bfq_mark_bfqq_has_waker(bfqq); bfqq 1989 block/bfq-iosched.c if (time_is_before_eq_jiffies(bfqq->decrease_time_jif + bfqq 1991 block/bfq-iosched.c bfq_reset_inject_limit(bfqd, bfqq); bfqq 2018 block/bfq-iosched.c if (bfqq == bfqd->in_service_queue && bfqq 2020 block/bfq-iosched.c (bfqq->last_serv_time_ns > 0 && bfqq 2022 block/bfq-iosched.c time_is_before_eq_jiffies(bfqq->decrease_time_jif + bfqq 2050 block/bfq-iosched.c elv_rb_add(&bfqq->sort_list, rq); bfqq 2055 block/bfq-iosched.c prev = bfqq->next_rq; bfqq 2056 block/bfq-iosched.c next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); bfqq 2057 block/bfq-iosched.c bfqq->next_rq = next_rq; bfqq 2063 block/bfq-iosched.c if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) bfqq 2064 block/bfq-iosched.c bfq_pos_tree_add_move(bfqd, bfqq); bfqq 2066 block/bfq-iosched.c if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ bfqq 2067 block/bfq-iosched.c bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, bfqq 2072 block/bfq-iosched.c bfqq->last_wr_start_finish + bfqq 2074 block/bfq-iosched.c bfqq->wr_coeff = bfqd->bfq_wr_coeff; bfqq 2075 block/bfq-iosched.c bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); bfqq 2078 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 2080 block/bfq-iosched.c if (prev != bfqq->next_rq) bfqq 2081 block/bfq-iosched.c bfq_updated_next_req(bfqd, bfqq); bfqq 2111 block/bfq-iosched.c (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) bfqq 2112 block/bfq-iosched.c bfqq->last_wr_start_finish = jiffies; bfqq 2119 block/bfq-iosched.c struct bfq_queue *bfqq = bfqd->bio_bfqq; bfqq 2122 block/bfq-iosched.c if (bfqq) bfqq 2123 block/bfq-iosched.c return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); bfqq 2155 block/bfq-iosched.c struct bfq_queue *bfqq = RQ_BFQQ(rq); bfqq 2156 block/bfq-iosched.c struct bfq_data *bfqd = bfqq->bfqd; bfqq 2159 block/bfq-iosched.c if (bfqq->next_rq == rq) { bfqq 2160 block/bfq-iosched.c bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); bfqq 2161 block/bfq-iosched.c bfq_updated_next_req(bfqd, bfqq); bfqq 2166 block/bfq-iosched.c bfqq->queued[sync]--; bfqq 2168 block/bfq-iosched.c elv_rb_del(&bfqq->sort_list, rq); bfqq 2174 block/bfq-iosched.c if (RB_EMPTY_ROOT(&bfqq->sort_list)) { bfqq 2175 block/bfq-iosched.c bfqq->next_rq = NULL; bfqq 2177 block/bfq-iosched.c if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { bfqq 2178 block/bfq-iosched.c bfq_del_bfqq_busy(bfqd, bfqq, false); bfqq 2192 block/bfq-iosched.c bfqq->entity.budget = bfqq->entity.service = 0; bfqq 2198 block/bfq-iosched.c if (bfqq->pos_root) { bfqq 2199 block/bfq-iosched.c rb_erase(&bfqq->pos_node, bfqq->pos_root); bfqq 2200 block/bfq-iosched.c bfqq->pos_root = NULL; bfqq 2205 block/bfq-iosched.c bfq_pos_tree_add_move(bfqd, bfqq); bfqq 2209 block/bfq-iosched.c bfqq->meta_pending--; bfqq 2271 block/bfq-iosched.c struct bfq_queue *bfqq = bfq_init_rq(req); bfqq 2275 block/bfq-iosched.c if (!bfqq) bfqq 2278 block/bfq-iosched.c bfqd = bfqq->bfqd; bfqq 2281 block/bfq-iosched.c elv_rb_del(&bfqq->sort_list, req); bfqq 2282 block/bfq-iosched.c elv_rb_add(&bfqq->sort_list, req); bfqq 2285 block/bfq-iosched.c prev = bfqq->next_rq; bfqq 2286 block/bfq-iosched.c next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, bfqq 2288 block/bfq-iosched.c bfqq->next_rq = next_rq; bfqq 2294 block/bfq-iosched.c if (prev != bfqq->next_rq) { bfqq 2295 block/bfq-iosched.c bfq_updated_next_req(bfqd, bfqq); bfqq 2301 block/bfq-iosched.c bfq_pos_tree_add_move(bfqd, bfqq); bfqq 2323 block/bfq-iosched.c struct bfq_queue *bfqq = bfq_init_rq(rq), bfqq 2326 block/bfq-iosched.c if (!bfqq) bfqq 2338 block/bfq-iosched.c if (bfqq == next_bfqq && bfqq 2346 block/bfq-iosched.c if (bfqq->next_rq == next) bfqq 2347 block/bfq-iosched.c bfqq->next_rq = rq; bfqq 2349 block/bfq-iosched.c bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags); bfqq 2353 block/bfq-iosched.c static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) bfqq 2355 block/bfq-iosched.c if (bfq_bfqq_busy(bfqq)) bfqq 2356 block/bfq-iosched.c bfqq->bfqd->wr_busy_queues--; bfqq 2357 block/bfq-iosched.c bfqq->wr_coeff = 1; bfqq 2358 block/bfq-iosched.c bfqq->wr_cur_max_time = 0; bfqq 2359 block/bfq-iosched.c bfqq->last_wr_start_finish = jiffies; bfqq 2364 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 2382 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 2386 block/bfq-iosched.c list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) bfqq 2387 block/bfq-iosched.c bfq_bfqq_end_wr(bfqq); bfqq 2388 block/bfq-iosched.c list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) bfqq 2389 block/bfq-iosched.c bfq_bfqq_end_wr(bfqq); bfqq 2411 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 2414 block/bfq-iosched.c struct rb_root *root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; bfqq 2456 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 2465 block/bfq-iosched.c bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); bfqq 2466 block/bfq-iosched.c if (!bfqq || bfqq == cur_bfqq) bfqq 2469 block/bfq-iosched.c return bfqq; bfqq 2473 block/bfq-iosched.c bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) bfqq 2489 block/bfq-iosched.c if (__bfqq == bfqq) bfqq 2494 block/bfq-iosched.c process_refs = bfqq_process_refs(bfqq); bfqq 2503 block/bfq-iosched.c bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", bfqq 2526 block/bfq-iosched.c bfqq->new_bfqq = new_bfqq; bfqq 2531 block/bfq-iosched.c static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, bfqq 2537 block/bfq-iosched.c if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || bfqq 2538 block/bfq-iosched.c (bfqq->ioprio_class != new_bfqq->ioprio_class)) bfqq 2546 block/bfq-iosched.c if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq)) bfqq 2554 block/bfq-iosched.c if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq)) bfqq 2581 block/bfq-iosched.c bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 2637 block/bfq-iosched.c if (bfq_too_late_for_merging(bfqq)) bfqq 2640 block/bfq-iosched.c if (bfqq->new_bfqq) bfqq 2641 block/bfq-iosched.c return bfqq->new_bfqq; bfqq 2643 block/bfq-iosched.c if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) bfqq 2652 block/bfq-iosched.c if (in_service_bfqq && in_service_bfqq != bfqq && bfqq 2656 block/bfq-iosched.c bfqq->entity.parent == in_service_bfqq->entity.parent && bfqq 2657 block/bfq-iosched.c bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { bfqq 2658 block/bfq-iosched.c new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); bfqq 2667 block/bfq-iosched.c new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, bfqq 2671 block/bfq-iosched.c bfq_may_be_close_cooperator(bfqq, new_bfqq)) bfqq 2672 block/bfq-iosched.c return bfq_setup_merge(bfqq, new_bfqq); bfqq 2677 block/bfq-iosched.c static void bfq_bfqq_save_state(struct bfq_queue *bfqq) bfqq 2679 block/bfq-iosched.c struct bfq_io_cq *bic = bfqq->bic; bfqq 2689 block/bfq-iosched.c bic->saved_weight = bfqq->entity.orig_weight; bfqq 2690 block/bfq-iosched.c bic->saved_ttime = bfqq->ttime; bfqq 2691 block/bfq-iosched.c bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq); bfqq 2692 block/bfq-iosched.c bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); bfqq 2693 block/bfq-iosched.c bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); bfqq 2694 block/bfq-iosched.c bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); bfqq 2695 block/bfq-iosched.c if (unlikely(bfq_bfqq_just_created(bfqq) && bfqq 2696 block/bfq-iosched.c !bfq_bfqq_in_large_burst(bfqq) && bfqq 2697 block/bfq-iosched.c bfqq->bfqd->low_latency)) { bfqq 2707 block/bfq-iosched.c bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; bfqq 2709 block/bfq-iosched.c bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd); bfqq 2712 block/bfq-iosched.c bic->saved_wr_coeff = bfqq->wr_coeff; bfqq 2714 block/bfq-iosched.c bfqq->wr_start_at_switch_to_srt; bfqq 2715 block/bfq-iosched.c bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish; bfqq 2716 block/bfq-iosched.c bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time; bfqq 2720 block/bfq-iosched.c void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 2733 block/bfq-iosched.c if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq 2734 block/bfq-iosched.c bfqq != bfqd->in_service_queue) bfqq 2735 block/bfq-iosched.c bfq_del_bfqq_busy(bfqd, bfqq, false); bfqq 2737 block/bfq-iosched.c bfq_put_queue(bfqq); bfqq 2742 block/bfq-iosched.c struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) bfqq 2744 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", bfqq 2747 block/bfq-iosched.c bfq_bfqq_save_state(bfqq); bfqq 2749 block/bfq-iosched.c if (bfq_bfqq_IO_bound(bfqq)) bfqq 2751 block/bfq-iosched.c bfq_clear_bfqq_IO_bound(bfqq); bfqq 2762 block/bfq-iosched.c if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) { bfqq 2763 block/bfq-iosched.c new_bfqq->wr_coeff = bfqq->wr_coeff; bfqq 2764 block/bfq-iosched.c new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; bfqq 2765 block/bfq-iosched.c new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; bfqq 2767 block/bfq-iosched.c bfqq->wr_start_at_switch_to_srt; bfqq 2773 block/bfq-iosched.c if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */ bfqq 2774 block/bfq-iosched.c bfqq->wr_coeff = 1; bfqq 2775 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 2776 block/bfq-iosched.c if (bfq_bfqq_busy(bfqq)) bfqq 2809 block/bfq-iosched.c bfqq->bic = NULL; bfqq 2810 block/bfq-iosched.c bfq_release_process_ref(bfqd, bfqq); bfqq 2818 block/bfq-iosched.c struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; bfqq 2830 block/bfq-iosched.c if (!bfqq) bfqq 2837 block/bfq-iosched.c new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); bfqq 2846 block/bfq-iosched.c bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq, bfqq 2853 block/bfq-iosched.c bfqq = new_bfqq; bfqq 2861 block/bfq-iosched.c bfqd->bio_bfqq = bfqq; bfqq 2864 block/bfq-iosched.c return bfqq == RQ_BFQQ(rq); bfqq 2874 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 2878 block/bfq-iosched.c if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) bfqq 2881 block/bfq-iosched.c timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; bfqq 2885 block/bfq-iosched.c bfqq->budget_timeout = jiffies + bfqq 2890 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 2892 block/bfq-iosched.c if (bfqq) { bfqq 2893 block/bfq-iosched.c bfq_clear_bfqq_fifo_expire(bfqq); bfqq 2897 block/bfq-iosched.c if (time_is_before_jiffies(bfqq->last_wr_start_finish) && bfqq 2898 block/bfq-iosched.c bfqq->wr_coeff > 1 && bfqq 2899 block/bfq-iosched.c bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && bfqq 2900 block/bfq-iosched.c time_is_before_jiffies(bfqq->budget_timeout)) { bfqq 2925 block/bfq-iosched.c if (time_after(bfqq->budget_timeout, bfqq 2926 block/bfq-iosched.c bfqq->last_wr_start_finish)) bfqq 2927 block/bfq-iosched.c bfqq->last_wr_start_finish += bfqq 2928 block/bfq-iosched.c jiffies - bfqq->budget_timeout; bfqq 2930 block/bfq-iosched.c bfqq->last_wr_start_finish = jiffies; bfqq 2933 block/bfq-iosched.c bfq_set_budget_timeout(bfqd, bfqq); bfqq 2934 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, bfqq 2936 block/bfq-iosched.c bfqq->entity.budget); bfqq 2939 block/bfq-iosched.c bfqd->in_service_queue = bfqq; bfqq 2947 block/bfq-iosched.c struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); bfqq 2949 block/bfq-iosched.c __bfq_set_in_service_queue(bfqd, bfqq); bfqq 2950 block/bfq-iosched.c return bfqq; bfqq 2955 block/bfq-iosched.c struct bfq_queue *bfqq = bfqd->in_service_queue; bfqq 2958 block/bfq-iosched.c bfq_mark_bfqq_wait_request(bfqq); bfqq 2976 block/bfq-iosched.c if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && bfqq 2977 block/bfq-iosched.c !bfq_asymmetric_scenario(bfqd, bfqq)) bfqq 2979 block/bfq-iosched.c else if (bfqq->wr_coeff > 1) bfqq 2987 block/bfq-iosched.c bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); bfqq 3240 block/bfq-iosched.c struct bfq_queue *bfqq = RQ_BFQQ(rq); bfqq 3254 block/bfq-iosched.c bfqq->dispatched++; bfqq 3447 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 3450 block/bfq-iosched.c if (unlikely(!bfqq_process_refs(bfqq))) bfqq 3453 block/bfq-iosched.c return (bfqq->wr_coeff > 1 && bfqq 3457 block/bfq-iosched.c bfqq->dispatched + 4)) || bfqq 3458 block/bfq-iosched.c bfq_asymmetric_scenario(bfqd, bfqq); bfqq 3461 block/bfq-iosched.c static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 3470 block/bfq-iosched.c if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq)) bfqq 3471 block/bfq-iosched.c bfq_mark_bfqq_split_coop(bfqq); bfqq 3486 block/bfq-iosched.c if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq 3488 block/bfq-iosched.c idling_needed_for_service_guarantees(bfqd, bfqq))) { bfqq 3489 block/bfq-iosched.c if (bfqq->dispatched == 0) bfqq 3496 block/bfq-iosched.c bfqq->budget_timeout = jiffies; bfqq 3498 block/bfq-iosched.c bfq_del_bfqq_busy(bfqd, bfqq, true); bfqq 3500 block/bfq-iosched.c bfq_requeue_bfqq(bfqd, bfqq, true); bfqq 3506 block/bfq-iosched.c !RB_EMPTY_ROOT(&bfqq->sort_list))) bfqq 3507 block/bfq-iosched.c bfq_pos_tree_add_move(bfqd, bfqq); bfqq 3530 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 3538 block/bfq-iosched.c if (bfqq->wr_coeff == 1) bfqq 3539 block/bfq-iosched.c budget = bfqq->max_budget; bfqq 3548 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", bfqq 3549 block/bfq-iosched.c bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); bfqq 3550 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", bfqq 3552 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", bfqq 3553 block/bfq-iosched.c bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); bfqq 3555 block/bfq-iosched.c if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { bfqq 3586 block/bfq-iosched.c if (bfqq->dispatched > 0) /* still outstanding reqs */ bfqq 3649 block/bfq-iosched.c budget = max_t(int, bfqq->entity.service, min_budget); bfqq 3654 block/bfq-iosched.c } else if (!bfq_bfqq_sync(bfqq)) { bfqq 3664 block/bfq-iosched.c bfqq->max_budget = budget; bfqq 3668 block/bfq-iosched.c bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); bfqq 3680 block/bfq-iosched.c next_rq = bfqq->next_rq; bfqq 3682 block/bfq-iosched.c bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget, bfqq 3683 block/bfq-iosched.c bfq_serv_to_charge(next_rq, bfqq)); bfqq 3685 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", bfqq 3687 block/bfq-iosched.c bfqq->entity.budget); bfqq 3721 block/bfq-iosched.c static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 3727 block/bfq-iosched.c bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */ bfqq 3729 block/bfq-iosched.c if (!bfq_bfqq_sync(bfqq)) bfqq 3770 block/bfq-iosched.c slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; bfqq 3773 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); bfqq 3872 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 3874 block/bfq-iosched.c return max3(bfqq->soft_rt_next_start, bfqq 3875 block/bfq-iosched.c bfqq->last_idle_bklogged + bfqq 3876 block/bfq-iosched.c HZ * bfqq->service_from_backlogged / bfqq 3878 block/bfq-iosched.c jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); bfqq 3908 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 3914 block/bfq-iosched.c struct bfq_entity *entity = &bfqq->entity; bfqq 3919 block/bfq-iosched.c slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); bfqq 3936 block/bfq-iosched.c if (bfqq->wr_coeff == 1 && bfqq 3939 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq) >= entity->budget / 3))) bfqq 3940 block/bfq-iosched.c bfq_bfqq_charge_time(bfqd, bfqq, delta); bfqq 3944 block/bfq-iosched.c bfq_clear_bfqq_IO_bound(bfqq); bfqq 3946 block/bfq-iosched.c if (bfqd->low_latency && bfqq->wr_coeff == 1) bfqq 3947 block/bfq-iosched.c bfqq->last_wr_start_finish = jiffies; bfqq 3950 block/bfq-iosched.c RB_EMPTY_ROOT(&bfqq->sort_list)) { bfqq 3977 block/bfq-iosched.c if (bfqq->dispatched == 0 && bfqq 3978 block/bfq-iosched.c bfqq->wr_coeff != bfqd->bfq_wr_coeff) bfqq 3979 block/bfq-iosched.c bfqq->soft_rt_next_start = bfqq 3980 block/bfq-iosched.c bfq_bfqq_softrt_next_start(bfqd, bfqq); bfqq 3981 block/bfq-iosched.c else if (bfqq->dispatched > 0) { bfqq 3986 block/bfq-iosched.c bfq_mark_bfqq_softrt_update(bfqq); bfqq 3990 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, bfqq 3992 block/bfq-iosched.c slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq)); bfqq 4006 block/bfq-iosched.c __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); bfqq 4007 block/bfq-iosched.c if (__bfq_bfqq_expire(bfqd, bfqq, reason)) bfqq 4012 block/bfq-iosched.c if (!bfq_bfqq_busy(bfqq) && bfqq 4015 block/bfq-iosched.c bfq_mark_bfqq_non_blocking_wait_rq(bfqq); bfqq 4051 block/bfq-iosched.c static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) bfqq 4053 block/bfq-iosched.c return time_is_before_eq_jiffies(bfqq->budget_timeout); bfqq 4064 block/bfq-iosched.c static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) bfqq 4066 block/bfq-iosched.c bfq_log_bfqq(bfqq->bfqd, bfqq, bfqq 4068 block/bfq-iosched.c bfq_bfqq_wait_request(bfqq), bfqq 4069 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, bfqq 4070 block/bfq-iosched.c bfq_bfqq_budget_timeout(bfqq)); bfqq 4072 block/bfq-iosched.c return (!bfq_bfqq_wait_request(bfqq) || bfqq 4073 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3) bfqq 4075 block/bfq-iosched.c bfq_bfqq_budget_timeout(bfqq); bfqq 4079 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 4087 block/bfq-iosched.c if (unlikely(!bfqq_process_refs(bfqq))) bfqq 4090 block/bfq-iosched.c bfqq_sequential_and_IO_bound = !BFQQ_SEEKY(bfqq) && bfqq 4091 block/bfq-iosched.c bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_has_short_ttime(bfqq); bfqq 4178 block/bfq-iosched.c static bool bfq_better_to_idle(struct bfq_queue *bfqq) bfqq 4180 block/bfq-iosched.c struct bfq_data *bfqd = bfqq->bfqd; bfqq 4184 block/bfq-iosched.c if (unlikely(!bfqq_process_refs(bfqq))) bfqq 4198 block/bfq-iosched.c if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) || bfqq 4199 block/bfq-iosched.c bfq_class_idle(bfqq)) bfqq 4203 block/bfq-iosched.c idling_boosts_thr_without_issues(bfqd, bfqq); bfqq 4206 block/bfq-iosched.c idling_needed_for_service_guarantees(bfqd, bfqq); bfqq 4229 block/bfq-iosched.c static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) bfqq 4231 block/bfq-iosched.c return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq); bfqq 4244 block/bfq-iosched.c struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; bfqq 4292 block/bfq-iosched.c list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) bfqq 4293 block/bfq-iosched.c if (!RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq 4294 block/bfq-iosched.c (in_serv_always_inject || bfqq->wr_coeff > 1) && bfqq 4295 block/bfq-iosched.c bfq_serv_to_charge(bfqq->next_rq, bfqq) <= bfqq 4296 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq)) { bfqq 4315 block/bfq-iosched.c blk_rq_sectors(bfqq->next_rq) >= bfqq 4323 block/bfq-iosched.c return bfqq; bfqq 4336 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 4340 block/bfq-iosched.c bfqq = bfqd->in_service_queue; bfqq 4341 block/bfq-iosched.c if (!bfqq) bfqq 4344 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); bfqq 4353 block/bfq-iosched.c if (bfq_may_expire_for_budg_timeout(bfqq) && bfqq 4354 block/bfq-iosched.c !bfq_bfqq_must_idle(bfqq)) bfqq 4364 block/bfq-iosched.c next_rq = bfqq->next_rq; bfqq 4370 block/bfq-iosched.c if (bfq_serv_to_charge(next_rq, bfqq) > bfqq 4371 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq)) { bfqq 4386 block/bfq-iosched.c if (bfq_bfqq_wait_request(bfqq)) { bfqq 4400 block/bfq-iosched.c bfq_clear_bfqq_wait_request(bfqq); bfqq 4415 block/bfq-iosched.c if (bfq_bfqq_wait_request(bfqq) || bfqq 4416 block/bfq-iosched.c (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) { bfqq 4418 block/bfq-iosched.c bfqq->bic && bfqq->bic->bfqq[0] && bfqq 4419 block/bfq-iosched.c bfq_bfqq_busy(bfqq->bic->bfqq[0]) && bfqq 4420 block/bfq-iosched.c bfqq->bic->bfqq[0]->next_rq ? bfqq 4421 block/bfq-iosched.c bfqq->bic->bfqq[0] : NULL; bfqq 4496 block/bfq-iosched.c icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic && bfqq 4499 block/bfq-iosched.c bfqq = bfqq->bic->bfqq[0]; bfqq 4500 block/bfq-iosched.c else if (bfq_bfqq_has_waker(bfqq) && bfqq 4501 block/bfq-iosched.c bfq_bfqq_busy(bfqq->waker_bfqq) && bfqq 4502 block/bfq-iosched.c bfqq->next_rq && bfqq 4503 block/bfq-iosched.c bfq_serv_to_charge(bfqq->waker_bfqq->next_rq, bfqq 4504 block/bfq-iosched.c bfqq->waker_bfqq) <= bfqq 4505 block/bfq-iosched.c bfq_bfqq_budget_left(bfqq->waker_bfqq) bfqq 4507 block/bfq-iosched.c bfqq = bfqq->waker_bfqq; bfqq 4508 block/bfq-iosched.c else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && bfqq 4509 block/bfq-iosched.c (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || bfqq 4510 block/bfq-iosched.c !bfq_bfqq_has_short_ttime(bfqq))) bfqq 4511 block/bfq-iosched.c bfqq = bfq_choose_bfqq_for_injection(bfqd); bfqq 4513 block/bfq-iosched.c bfqq = NULL; bfqq 4520 block/bfq-iosched.c bfq_bfqq_expire(bfqd, bfqq, false, reason); bfqq 4522 block/bfq-iosched.c bfqq = bfq_set_in_service_queue(bfqd); bfqq 4523 block/bfq-iosched.c if (bfqq) { bfqq 4524 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); bfqq 4528 block/bfq-iosched.c if (bfqq) bfqq 4529 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); bfqq 4533 block/bfq-iosched.c return bfqq; bfqq 4536 block/bfq-iosched.c static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 4538 block/bfq-iosched.c struct bfq_entity *entity = &bfqq->entity; bfqq 4540 block/bfq-iosched.c if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ bfqq 4541 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, bfqq 4543 block/bfq-iosched.c jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), bfqq 4544 block/bfq-iosched.c jiffies_to_msecs(bfqq->wr_cur_max_time), bfqq 4545 block/bfq-iosched.c bfqq->wr_coeff, bfqq 4546 block/bfq-iosched.c bfqq->entity.weight, bfqq->entity.orig_weight); bfqq 4549 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); bfqq 4556 block/bfq-iosched.c if (bfq_bfqq_in_large_burst(bfqq)) bfqq 4557 block/bfq-iosched.c bfq_bfqq_end_wr(bfqq); bfqq 4558 block/bfq-iosched.c else if (time_is_before_jiffies(bfqq->last_wr_start_finish + bfqq 4559 block/bfq-iosched.c bfqq->wr_cur_max_time)) { bfqq 4560 block/bfq-iosched.c if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || bfqq 4561 block/bfq-iosched.c time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt + bfqq 4563 block/bfq-iosched.c bfq_bfqq_end_wr(bfqq); bfqq 4565 block/bfq-iosched.c switch_back_to_interactive_wr(bfqq, bfqd); bfqq 4566 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 4569 block/bfq-iosched.c if (bfqq->wr_coeff > 1 && bfqq 4570 block/bfq-iosched.c bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && bfqq 4571 block/bfq-iosched.c bfqq->service_from_wr > max_service_from_wr) { bfqq 4573 block/bfq-iosched.c bfq_bfqq_end_wr(bfqq); bfqq 4584 block/bfq-iosched.c if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) bfqq 4593 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 4595 block/bfq-iosched.c struct request *rq = bfqq->next_rq; bfqq 4598 block/bfq-iosched.c service_to_charge = bfq_serv_to_charge(rq, bfqq); bfqq 4600 block/bfq-iosched.c bfq_bfqq_served(bfqq, service_to_charge); bfqq 4602 block/bfq-iosched.c if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) { bfqq 4609 block/bfq-iosched.c if (bfqq != bfqd->in_service_queue) bfqq 4623 block/bfq-iosched.c bfq_update_wr_data(bfqd, bfqq); bfqq 4630 block/bfq-iosched.c if (!(bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq))) bfqq 4633 block/bfq-iosched.c bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED); bfqq 4655 block/bfq-iosched.c struct bfq_queue *bfqq = NULL; bfqq 4662 block/bfq-iosched.c bfqq = RQ_BFQQ(rq); bfqq 4664 block/bfq-iosched.c if (bfqq) { bfqq 4671 block/bfq-iosched.c bfqq->dispatched++; bfqq 4723 block/bfq-iosched.c bfqq = bfq_select_queue(bfqd); bfqq 4724 block/bfq-iosched.c if (!bfqq) bfqq 4727 block/bfq-iosched.c rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq); bfqq 4745 block/bfq-iosched.c struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL; bfqq 4747 block/bfq-iosched.c if (!idle_timer_disabled && !bfqq) bfqq 4775 block/bfq-iosched.c if (bfqq) { bfqq 4776 block/bfq-iosched.c struct bfq_group *bfqg = bfqq_group(bfqq); bfqq 4823 block/bfq-iosched.c void bfq_put_queue(struct bfq_queue *bfqq) bfqq 4827 block/bfq-iosched.c struct bfq_group *bfqg = bfqq_group(bfqq); bfqq 4829 block/bfq-iosched.c if (bfqq->bfqd) bfqq 4830 block/bfq-iosched.c bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq 4831 block/bfq-iosched.c bfqq, bfqq->ref); bfqq 4833 block/bfq-iosched.c bfqq->ref--; bfqq 4834 block/bfq-iosched.c if (bfqq->ref) bfqq 4837 block/bfq-iosched.c if (!hlist_unhashed(&bfqq->burst_list_node)) { bfqq 4838 block/bfq-iosched.c hlist_del_init(&bfqq->burst_list_node); bfqq 4865 block/bfq-iosched.c if (bfqq->bic && bfqq->bfqd->burst_size > 0) bfqq 4866 block/bfq-iosched.c bfqq->bfqd->burst_size--; bfqq 4885 block/bfq-iosched.c if (!hlist_unhashed(&bfqq->woken_list_node)) bfqq 4886 block/bfq-iosched.c hlist_del_init(&bfqq->woken_list_node); bfqq 4889 block/bfq-iosched.c hlist_for_each_entry_safe(item, n, &bfqq->woken_list, bfqq 4896 block/bfq-iosched.c if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) bfqq 4897 block/bfq-iosched.c bfqq->bfqd->last_completed_rq_bfqq = NULL; bfqq 4899 block/bfq-iosched.c kmem_cache_free(bfq_pool, bfqq); bfqq 4903 block/bfq-iosched.c static void bfq_put_cooperator(struct bfq_queue *bfqq) bfqq 4912 block/bfq-iosched.c __bfqq = bfqq->new_bfqq; bfqq 4914 block/bfq-iosched.c if (__bfqq == bfqq) bfqq 4922 block/bfq-iosched.c static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 4924 block/bfq-iosched.c if (bfqq == bfqd->in_service_queue) { bfqq 4925 block/bfq-iosched.c __bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT); bfqq 4929 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); bfqq 4931 block/bfq-iosched.c bfq_put_cooperator(bfqq); bfqq 4933 block/bfq-iosched.c bfq_release_process_ref(bfqd, bfqq); bfqq 4938 block/bfq-iosched.c struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync); bfqq 4941 block/bfq-iosched.c if (bfqq) bfqq 4942 block/bfq-iosched.c bfqd = bfqq->bfqd; /* NULL if scheduler already exited */ bfqq 4944 block/bfq-iosched.c if (bfqq && bfqd) { bfqq 4948 block/bfq-iosched.c bfqq->bic = NULL; bfqq 4949 block/bfq-iosched.c bfq_exit_bfqq(bfqd, bfqq); bfqq 4968 block/bfq-iosched.c bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) bfqq 4972 block/bfq-iosched.c struct bfq_data *bfqd = bfqq->bfqd; bfqq 4980 block/bfq-iosched.c dev_err(bfqq->bfqd->queue->backing_dev_info->dev, bfqq 4987 block/bfq-iosched.c bfqq->new_ioprio = task_nice_ioprio(tsk); bfqq 4988 block/bfq-iosched.c bfqq->new_ioprio_class = task_nice_ioclass(tsk); bfqq 4991 block/bfq-iosched.c bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); bfqq 4992 block/bfq-iosched.c bfqq->new_ioprio_class = IOPRIO_CLASS_RT; bfqq 4995 block/bfq-iosched.c bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); bfqq 4996 block/bfq-iosched.c bfqq->new_ioprio_class = IOPRIO_CLASS_BE; bfqq 4999 block/bfq-iosched.c bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE; bfqq 5000 block/bfq-iosched.c bfqq->new_ioprio = 7; bfqq 5004 block/bfq-iosched.c if (bfqq->new_ioprio >= IOPRIO_BE_NR) { bfqq 5006 block/bfq-iosched.c bfqq->new_ioprio); bfqq 5007 block/bfq-iosched.c bfqq->new_ioprio = IOPRIO_BE_NR; bfqq 5010 block/bfq-iosched.c bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); bfqq 5011 block/bfq-iosched.c bfqq->entity.prio_changed = 1; bfqq 5021 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 5033 block/bfq-iosched.c bfqq = bic_to_bfqq(bic, false); bfqq 5034 block/bfq-iosched.c if (bfqq) { bfqq 5035 block/bfq-iosched.c bfq_release_process_ref(bfqd, bfqq); bfqq 5036 block/bfq-iosched.c bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); bfqq 5037 block/bfq-iosched.c bic_set_bfqq(bic, bfqq, false); bfqq 5040 block/bfq-iosched.c bfqq = bic_to_bfqq(bic, true); bfqq 5041 block/bfq-iosched.c if (bfqq) bfqq 5042 block/bfq-iosched.c bfq_set_next_ioprio_data(bfqq, bic); bfqq 5045 block/bfq-iosched.c static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 5048 block/bfq-iosched.c RB_CLEAR_NODE(&bfqq->entity.rb_node); bfqq 5049 block/bfq-iosched.c INIT_LIST_HEAD(&bfqq->fifo); bfqq 5050 block/bfq-iosched.c INIT_HLIST_NODE(&bfqq->burst_list_node); bfqq 5051 block/bfq-iosched.c INIT_HLIST_NODE(&bfqq->woken_list_node); bfqq 5052 block/bfq-iosched.c INIT_HLIST_HEAD(&bfqq->woken_list); bfqq 5054 block/bfq-iosched.c bfqq->ref = 0; bfqq 5055 block/bfq-iosched.c bfqq->bfqd = bfqd; bfqq 5058 block/bfq-iosched.c bfq_set_next_ioprio_data(bfqq, bic); bfqq 5066 block/bfq-iosched.c if (!bfq_class_idle(bfqq)) bfqq 5068 block/bfq-iosched.c bfq_mark_bfqq_has_short_ttime(bfqq); bfqq 5069 block/bfq-iosched.c bfq_mark_bfqq_sync(bfqq); bfqq 5070 block/bfq-iosched.c bfq_mark_bfqq_just_created(bfqq); bfqq 5072 block/bfq-iosched.c bfq_clear_bfqq_sync(bfqq); bfqq 5075 block/bfq-iosched.c bfqq->ttime.last_end_request = ktime_get_ns() + 1; bfqq 5077 block/bfq-iosched.c bfq_mark_bfqq_IO_bound(bfqq); bfqq 5079 block/bfq-iosched.c bfqq->pid = pid; bfqq 5082 block/bfq-iosched.c bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; bfqq 5083 block/bfq-iosched.c bfqq->budget_timeout = bfq_smallest_from_now(); bfqq 5085 block/bfq-iosched.c bfqq->wr_coeff = 1; bfqq 5086 block/bfq-iosched.c bfqq->last_wr_start_finish = jiffies; bfqq 5087 block/bfq-iosched.c bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now(); bfqq 5088 block/bfq-iosched.c bfqq->split_time = bfq_smallest_from_now(); bfqq 5099 block/bfq-iosched.c bfqq->soft_rt_next_start = jiffies; bfqq 5102 block/bfq-iosched.c bfqq->seek_history = 1; bfqq 5131 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 5138 block/bfq-iosched.c bfqq = &bfqd->oom_bfqq; bfqq 5145 block/bfq-iosched.c bfqq = *async_bfqq; bfqq 5146 block/bfq-iosched.c if (bfqq) bfqq 5150 block/bfq-iosched.c bfqq = kmem_cache_alloc_node(bfq_pool, bfqq 5154 block/bfq-iosched.c if (bfqq) { bfqq 5155 block/bfq-iosched.c bfq_init_bfqq(bfqd, bfqq, bic, current->pid, bfqq 5157 block/bfq-iosched.c bfq_init_entity(&bfqq->entity, bfqg); bfqq 5158 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "allocated"); bfqq 5160 block/bfq-iosched.c bfqq = &bfqd->oom_bfqq; bfqq 5161 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); bfqq 5170 block/bfq-iosched.c bfqq->ref++; /* bfqq 5177 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", bfqq 5178 block/bfq-iosched.c bfqq, bfqq->ref); bfqq 5179 block/bfq-iosched.c *async_bfqq = bfqq; bfqq 5183 block/bfq-iosched.c bfqq->ref++; /* get a process reference to this queue */ bfqq 5184 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); bfqq 5186 block/bfq-iosched.c return bfqq; bfqq 5190 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 5192 block/bfq-iosched.c struct bfq_ttime *ttime = &bfqq->ttime; bfqq 5193 block/bfq-iosched.c u64 elapsed = ktime_get_ns() - bfqq->ttime.last_end_request; bfqq 5197 block/bfq-iosched.c ttime->ttime_samples = (7*bfqq->ttime.ttime_samples + 256) / 8; bfqq 5204 block/bfq-iosched.c bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 5207 block/bfq-iosched.c bfqq->seek_history <<= 1; bfqq 5208 block/bfq-iosched.c bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq); bfqq 5210 block/bfq-iosched.c if (bfqq->wr_coeff > 1 && bfqq 5211 block/bfq-iosched.c bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && bfqq 5212 block/bfq-iosched.c BFQQ_TOTALLY_SEEKY(bfqq)) bfqq 5213 block/bfq-iosched.c bfq_bfqq_end_wr(bfqq); bfqq 5217 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 5227 block/bfq-iosched.c if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq) || bfqq 5232 block/bfq-iosched.c if (time_is_after_eq_jiffies(bfqq->split_time + bfqq 5241 block/bfq-iosched.c (bfq_sample_valid(bfqq->ttime.ttime_samples) && bfqq 5242 block/bfq-iosched.c bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle)) bfqq 5245 block/bfq-iosched.c state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq); bfqq 5248 block/bfq-iosched.c bfq_mark_bfqq_has_short_ttime(bfqq); bfqq 5250 block/bfq-iosched.c bfq_clear_bfqq_has_short_ttime(bfqq); bfqq 5336 block/bfq-iosched.c if (state_changed && bfqq->last_serv_time_ns == 0 && bfqq 5337 block/bfq-iosched.c (time_is_before_eq_jiffies(bfqq->decrease_time_jif + bfqq 5340 block/bfq-iosched.c bfq_reset_inject_limit(bfqd, bfqq); bfqq 5347 block/bfq-iosched.c static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 5351 block/bfq-iosched.c bfqq->meta_pending++; bfqq 5353 block/bfq-iosched.c bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); bfqq 5355 block/bfq-iosched.c if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { bfqq 5356 block/bfq-iosched.c bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 && bfqq 5358 block/bfq-iosched.c bool budget_timeout = bfq_bfqq_budget_timeout(bfqq); bfqq 5376 block/bfq-iosched.c if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) && bfqq 5387 block/bfq-iosched.c bfq_clear_bfqq_wait_request(bfqq); bfqq 5398 block/bfq-iosched.c bfq_bfqq_expire(bfqd, bfqq, false, bfqq 5406 block/bfq-iosched.c struct bfq_queue *bfqq = RQ_BFQQ(rq), bfqq 5407 block/bfq-iosched.c *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); bfqq 5416 block/bfq-iosched.c bfqq->allocated--; bfqq 5426 block/bfq-iosched.c if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq) bfqq 5428 block/bfq-iosched.c bfqq, new_bfqq); bfqq 5430 block/bfq-iosched.c bfq_clear_bfqq_just_created(bfqq); bfqq 5435 block/bfq-iosched.c bfq_put_queue(bfqq); bfqq 5437 block/bfq-iosched.c bfqq = new_bfqq; bfqq 5440 block/bfq-iosched.c bfq_update_io_thinktime(bfqd, bfqq); bfqq 5441 block/bfq-iosched.c bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq)); bfqq 5442 block/bfq-iosched.c bfq_update_io_seektime(bfqd, bfqq, rq); bfqq 5444 block/bfq-iosched.c waiting = bfqq && bfq_bfqq_wait_request(bfqq); bfqq 5446 block/bfq-iosched.c idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq); bfqq 5449 block/bfq-iosched.c list_add_tail(&rq->queuelist, &bfqq->fifo); bfqq 5451 block/bfq-iosched.c bfq_rq_enqueued(bfqd, bfqq, rq); bfqq 5458 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 5462 block/bfq-iosched.c if (!bfqq) bfqq 5476 block/bfq-iosched.c bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags); bfqq 5478 block/bfq-iosched.c bfqg_stats_update_idle_time(bfqq_group(bfqq)); bfqq 5483 block/bfq-iosched.c struct bfq_queue *bfqq, bfqq 5493 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 5508 block/bfq-iosched.c bfqq = bfq_init_rq(rq); bfqq 5509 block/bfq-iosched.c if (!bfqq || at_head || blk_rq_is_passthrough(rq)) { bfqq 5521 block/bfq-iosched.c bfqq = RQ_BFQQ(rq); bfqq 5539 block/bfq-iosched.c bfq_update_insert_stats(q, bfqq, idle_timer_disabled, bfqq 5557 block/bfq-iosched.c struct bfq_queue *bfqq = bfqd->in_service_queue; bfqq 5579 block/bfq-iosched.c if (bfqq && bfq_bfqq_has_short_ttime(bfqq) && bfqq 5580 block/bfq-iosched.c bfqq->dispatched + bfqq->queued[0] + bfqq->queued[1] < bfqq 5596 block/bfq-iosched.c static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) bfqq 5604 block/bfq-iosched.c bfqq->dispatched--; bfqq 5606 block/bfq-iosched.c if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { bfqq 5613 block/bfq-iosched.c bfqq->budget_timeout = jiffies; bfqq 5615 block/bfq-iosched.c bfq_weights_tree_remove(bfqd, bfqq); bfqq 5620 block/bfq-iosched.c bfqq->ttime.last_end_request = now_ns; bfqq 5649 block/bfq-iosched.c bfqd->last_completed_rq_bfqq = bfqq; bfqq 5662 block/bfq-iosched.c if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 && bfqq 5663 block/bfq-iosched.c RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq 5664 block/bfq-iosched.c bfqq->wr_coeff != bfqd->bfq_wr_coeff) bfqq 5665 block/bfq-iosched.c bfqq->soft_rt_next_start = bfqq 5666 block/bfq-iosched.c bfq_bfqq_softrt_next_start(bfqd, bfqq); bfqq 5672 block/bfq-iosched.c if (bfqd->in_service_queue == bfqq) { bfqq 5673 block/bfq-iosched.c if (bfq_bfqq_must_idle(bfqq)) { bfqq 5674 block/bfq-iosched.c if (bfqq->dispatched == 0) bfqq 5700 block/bfq-iosched.c } else if (bfq_may_expire_for_budg_timeout(bfqq)) bfqq 5701 block/bfq-iosched.c bfq_bfqq_expire(bfqd, bfqq, false, bfqq 5703 block/bfq-iosched.c else if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq 5704 block/bfq-iosched.c (bfqq->dispatched == 0 || bfqq 5705 block/bfq-iosched.c !bfq_better_to_idle(bfqq))) bfqq 5706 block/bfq-iosched.c bfq_bfqq_expire(bfqd, bfqq, false, bfqq 5714 block/bfq-iosched.c static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq) bfqq 5716 block/bfq-iosched.c bfqq->allocated--; bfqq 5718 block/bfq-iosched.c bfq_put_queue(bfqq); bfqq 5826 block/bfq-iosched.c struct bfq_queue *bfqq) bfqq 5829 block/bfq-iosched.c unsigned int old_limit = bfqq->inject_limit; bfqq 5831 block/bfq-iosched.c if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) { bfqq 5832 block/bfq-iosched.c u64 threshold = (bfqq->last_serv_time_ns * 3)>>1; bfqq 5835 block/bfq-iosched.c bfqq->inject_limit--; bfqq 5836 block/bfq-iosched.c bfqq->decrease_time_jif = jiffies; bfqq 5839 block/bfq-iosched.c bfqq->inject_limit++; bfqq 5854 block/bfq-iosched.c if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) || bfqq 5855 block/bfq-iosched.c tot_time_ns < bfqq->last_serv_time_ns) { bfqq 5856 block/bfq-iosched.c if (bfqq->last_serv_time_ns == 0) { bfqq 5861 block/bfq-iosched.c bfqq->inject_limit = max_t(unsigned int, 1, old_limit); bfqq 5863 block/bfq-iosched.c bfqq->last_serv_time_ns = tot_time_ns; bfqq 5874 block/bfq-iosched.c bfqq->last_serv_time_ns = tot_time_ns; bfqq 5890 block/bfq-iosched.c struct bfq_queue *bfqq = RQ_BFQQ(rq); bfqq 5910 block/bfq-iosched.c if (!rq->elv.icq || !bfqq) bfqq 5913 block/bfq-iosched.c bfqd = bfqq->bfqd; bfqq 5916 block/bfq-iosched.c bfqg_stats_update_completion(bfqq_group(bfqq), bfqq 5927 block/bfq-iosched.c bfq_update_inject_limit(bfqd, bfqq); bfqq 5929 block/bfq-iosched.c bfq_completed_request(bfqq, bfqd); bfqq 5930 block/bfq-iosched.c bfq_finish_requeue_request_body(bfqq); bfqq 5949 block/bfq-iosched.c bfqg_stats_update_io_remove(bfqq_group(bfqq), bfqq 5952 block/bfq-iosched.c bfq_finish_requeue_request_body(bfqq); bfqq 5981 block/bfq-iosched.c bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) bfqq 5983 block/bfq-iosched.c bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); bfqq 5985 block/bfq-iosched.c if (bfqq_process_refs(bfqq) == 1) { bfqq 5986 block/bfq-iosched.c bfqq->pid = current->pid; bfqq 5987 block/bfq-iosched.c bfq_clear_bfqq_coop(bfqq); bfqq 5988 block/bfq-iosched.c bfq_clear_bfqq_split_coop(bfqq); bfqq 5989 block/bfq-iosched.c return bfqq; bfqq 5994 block/bfq-iosched.c bfq_put_cooperator(bfqq); bfqq 5996 block/bfq-iosched.c bfq_release_process_ref(bfqq->bfqd, bfqq); bfqq 6006 block/bfq-iosched.c struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync); bfqq 6008 block/bfq-iosched.c if (likely(bfqq && bfqq != &bfqd->oom_bfqq)) bfqq 6009 block/bfq-iosched.c return bfqq; bfqq 6014 block/bfq-iosched.c if (bfqq) bfqq 6015 block/bfq-iosched.c bfq_put_queue(bfqq); bfqq 6016 block/bfq-iosched.c bfqq = bfq_get_queue(bfqd, bio, is_sync, bic); bfqq 6018 block/bfq-iosched.c bic_set_bfqq(bic, bfqq, is_sync); bfqq 6022 block/bfq-iosched.c bfq_mark_bfqq_in_large_burst(bfqq); bfqq 6024 block/bfq-iosched.c bfq_clear_bfqq_in_large_burst(bfqq); bfqq 6054 block/bfq-iosched.c hlist_add_head(&bfqq->burst_list_node, bfqq 6057 block/bfq-iosched.c bfqq->split_time = jiffies; bfqq 6060 block/bfq-iosched.c return bfqq; bfqq 6109 block/bfq-iosched.c struct bfq_queue *bfqq; bfqq 6132 block/bfq-iosched.c bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync, bfqq 6137 block/bfq-iosched.c if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { bfqq 6138 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); bfqq 6141 block/bfq-iosched.c if (bfq_bfqq_in_large_burst(bfqq)) bfqq 6144 block/bfq-iosched.c bfqq = bfq_split_bfqq(bic, bfqq); bfqq 6147 block/bfq-iosched.c if (!bfqq) bfqq 6148 block/bfq-iosched.c bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, bfqq 6156 block/bfq-iosched.c bfqq->allocated++; bfqq 6157 block/bfq-iosched.c bfqq->ref++; bfqq 6158 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d", bfqq 6159 block/bfq-iosched.c rq, bfqq, bfqq->ref); bfqq 6162 block/bfq-iosched.c rq->elv.priv[1] = bfqq; bfqq 6170 block/bfq-iosched.c if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { bfqq 6171 block/bfq-iosched.c bfqq->bic = bic; bfqq 6178 block/bfq-iosched.c bfq_bfqq_resume_state(bfqq, bfqd, bic, bfqq 6203 block/bfq-iosched.c if (unlikely(bfq_bfqq_just_created(bfqq) && bfqq 6206 block/bfq-iosched.c bfq_handle_burst(bfqd, bfqq); bfqq 6208 block/bfq-iosched.c return bfqq; bfqq 6212 block/bfq-iosched.c bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 6226 block/bfq-iosched.c if (bfqq != bfqd->in_service_queue) { bfqq 6231 block/bfq-iosched.c bfq_clear_bfqq_wait_request(bfqq); bfqq 6233 block/bfq-iosched.c if (bfq_bfqq_budget_timeout(bfqq)) bfqq 6240 block/bfq-iosched.c else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0) bfqq 6251 block/bfq-iosched.c bfq_bfqq_expire(bfqd, bfqq, true, reason); bfqq 6266 block/bfq-iosched.c struct bfq_queue *bfqq = bfqd->in_service_queue; bfqq 6276 block/bfq-iosched.c if (bfqq) bfqq 6277 block/bfq-iosched.c bfq_idle_slice_timer_body(bfqd, bfqq); bfqq 6285 block/bfq-iosched.c struct bfq_queue *bfqq = *bfqq_ptr; bfqq 6287 block/bfq-iosched.c bfq_log(bfqd, "put_async_bfqq: %p", bfqq); bfqq 6288 block/bfq-iosched.c if (bfqq) { bfqq 6289 block/bfq-iosched.c bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); bfqq 6291 block/bfq-iosched.c bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", bfqq 6292 block/bfq-iosched.c bfqq, bfqq->ref); bfqq 6293 block/bfq-iosched.c bfq_put_queue(bfqq); bfqq 6381 block/bfq-iosched.c struct bfq_queue *bfqq, *n; bfqq 6386 block/bfq-iosched.c list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) bfqq 6387 block/bfq-iosched.c bfq_deactivate_bfqq(bfqd, bfqq, false, false); bfqq 390 block/bfq-iosched.h struct bfq_queue *bfqq[2]; bfqq 775 block/bfq-iosched.h void bfq_mark_bfqq_##name(struct bfq_queue *bfqq); \ bfqq 776 block/bfq-iosched.h void bfq_clear_bfqq_##name(struct bfq_queue *bfqq); \ bfqq 777 block/bfq-iosched.h int bfq_bfqq_##name(const struct bfq_queue *bfqq); bfqq 939 block/bfq-iosched.h void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync); bfqq 941 block/bfq-iosched.h void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq); bfqq 942 block/bfq-iosched.h void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 945 block/bfq-iosched.h struct bfq_queue *bfqq, bfqq 948 block/bfq-iosched.h struct bfq_queue *bfqq); bfqq 949 block/bfq-iosched.h void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 951 block/bfq-iosched.h void bfq_put_queue(struct bfq_queue *bfqq); bfqq 953 block/bfq-iosched.h void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq); bfqq 961 block/bfq-iosched.h void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, bfqq 972 block/bfq-iosched.h void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 981 block/bfq-iosched.h struct bfq_group *bfqq_group(struct bfq_queue *bfqq); bfqq 1023 block/bfq-iosched.h struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq); bfqq 1035 block/bfq-iosched.h void bfq_bfqq_served(struct bfq_queue *bfqq, int served); bfqq 1036 block/bfq-iosched.h void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1043 block/bfq-iosched.h void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1045 block/bfq-iosched.h void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); bfqq 1046 block/bfq-iosched.h void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1048 block/bfq-iosched.h void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1050 block/bfq-iosched.h void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); bfqq 1064 block/bfq-iosched.h struct bfq_group *bfqq_group(struct bfq_queue *bfqq); bfqq 1066 block/bfq-iosched.h #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ bfqq 1068 block/bfq-iosched.h bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH); \ bfqq 1070 block/bfq-iosched.h bfqg_to_blkg(bfqq_group(bfqq))->blkcg, \ bfqq 1072 block/bfq-iosched.h bfq_bfqq_sync((bfqq)) ? 'S' : 'A', ##args); \ bfqq 1082 block/bfq-iosched.h #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ bfqq 1084 block/bfq-iosched.h bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH); \ bfqq 1086 block/bfq-iosched.h bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ bfqq 32 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 34 block/bfq-wf2q.c return bfqq ? bfqq->ioprio_class - 1 : bfqq 148 block/bfq-wf2q.c struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) bfqq 150 block/bfq-wf2q.c struct bfq_entity *group_entity = bfqq->entity.parent; bfqq 153 block/bfq-wf2q.c group_entity = &bfqq->bfqd->root_group->entity; bfqq 236 block/bfq-wf2q.c struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) bfqq 238 block/bfq-wf2q.c return bfqq->bfqd->root_group; bfqq 264 block/bfq-wf2q.c struct bfq_queue *bfqq = NULL; bfqq 267 block/bfq-wf2q.c bfqq = container_of(entity, struct bfq_queue, entity); bfqq 269 block/bfq-wf2q.c return bfqq; bfqq 293 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 298 block/bfq-wf2q.c if (bfqq) { bfqq 299 block/bfq-wf2q.c bfq_log_bfqq(bfqq->bfqd, bfqq, bfqq 302 block/bfq-wf2q.c bfq_log_bfqq(bfqq->bfqd, bfqq, bfqq 347 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 362 block/bfq-wf2q.c if (bfqq) bfqq 363 block/bfq-wf2q.c list_del(&bfqq->bfqq_list); bfqq 478 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 500 block/bfq-wf2q.c if (bfqq) bfqq 501 block/bfq-wf2q.c list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list); bfqq 533 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 535 block/bfq-wf2q.c if (bfqq) { bfqq 536 block/bfq-wf2q.c bfqq->ref++; bfqq 537 block/bfq-wf2q.c bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d", bfqq 538 block/bfq-wf2q.c bfqq, bfqq->ref); bfqq 582 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 601 block/bfq-wf2q.c if (bfqq) bfqq 602 block/bfq-wf2q.c list_del(&bfqq->bfqq_list); bfqq 617 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 628 block/bfq-wf2q.c if (bfqq) bfqq 629 block/bfq-wf2q.c list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list); bfqq 651 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 658 block/bfq-wf2q.c if (bfqq) bfqq 659 block/bfq-wf2q.c bfq_put_queue(bfqq); bfqq 736 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 745 block/bfq-wf2q.c if (bfqq) bfqq 746 block/bfq-wf2q.c bfqd = bfqq->bfqd; bfqq 770 block/bfq-wf2q.c if (bfqq) bfqq 771 block/bfq-wf2q.c bfqq->ioprio = bfqq 775 block/bfq-wf2q.c if (bfqq && update_class_too) bfqq 776 block/bfq-wf2q.c bfqq->ioprio_class = bfqq->new_ioprio_class; bfqq 782 block/bfq-wf2q.c if (!bfqq || bfqq->ioprio_class == bfqq->new_ioprio_class) bfqq 796 block/bfq-wf2q.c (bfqq ? bfqq->wr_coeff : 1); bfqq 802 block/bfq-wf2q.c if (prev_weight != new_weight && bfqq) { bfqq 804 block/bfq-wf2q.c __bfq_weights_tree_remove(bfqd, bfqq, root); bfqq 811 block/bfq-wf2q.c if (prev_weight != new_weight && bfqq && bfqq->wr_coeff == 1) { bfqq 813 block/bfq-wf2q.c bfq_weights_tree_add(bfqd, bfqq, root); bfqq 835 block/bfq-wf2q.c void bfq_bfqq_served(struct bfq_queue *bfqq, int served) bfqq 837 block/bfq-wf2q.c struct bfq_entity *entity = &bfqq->entity; bfqq 840 block/bfq-wf2q.c if (!bfqq->service_from_backlogged) bfqq 841 block/bfq-wf2q.c bfqq->first_IO_time = jiffies; bfqq 843 block/bfq-wf2q.c if (bfqq->wr_coeff > 1) bfqq 844 block/bfq-wf2q.c bfqq->service_from_wr += served; bfqq 846 block/bfq-wf2q.c bfqq->service_from_backlogged += served; bfqq 855 block/bfq-wf2q.c bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %d secs", served); bfqq 885 block/bfq-wf2q.c void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 888 block/bfq-wf2q.c struct bfq_entity *entity = &bfqq->entity; bfqq 899 block/bfq-wf2q.c bfq_bfqq_served(bfqq, bfqq 907 block/bfq-wf2q.c struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); bfqq 953 block/bfq-wf2q.c if (bfqq) bfqq 954 block/bfq-wf2q.c delta /= bfqq->wr_coeff; bfqq 1521 block/bfq-wf2q.c struct bfq_queue *bfqq; bfqq 1593 block/bfq-wf2q.c bfqq = bfq_entity_to_bfqq(entity); bfqq 1606 block/bfq-wf2q.c return bfqq; bfqq 1650 block/bfq-wf2q.c void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1653 block/bfq-wf2q.c struct bfq_entity *entity = &bfqq->entity; bfqq 1658 block/bfq-wf2q.c void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 1660 block/bfq-wf2q.c struct bfq_entity *entity = &bfqq->entity; bfqq 1662 block/bfq-wf2q.c bfq_activate_requeue_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq), bfqq 1664 block/bfq-wf2q.c bfq_clear_bfqq_non_blocking_wait_rq(bfqq); bfqq 1667 block/bfq-wf2q.c void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1670 block/bfq-wf2q.c struct bfq_entity *entity = &bfqq->entity; bfqq 1673 block/bfq-wf2q.c bfqq == bfqd->in_service_queue, expiration); bfqq 1681 block/bfq-wf2q.c void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq 1684 block/bfq-wf2q.c bfq_log_bfqq(bfqd, bfqq, "del from busy"); bfqq 1686 block/bfq-wf2q.c bfq_clear_bfqq_busy(bfqq); bfqq 1688 block/bfq-wf2q.c bfqd->busy_queues[bfqq->ioprio_class - 1]--; bfqq 1690 block/bfq-wf2q.c if (bfqq->wr_coeff > 1) bfqq 1693 block/bfq-wf2q.c bfqg_stats_update_dequeue(bfqq_group(bfqq)); bfqq 1695 block/bfq-wf2q.c bfq_deactivate_bfqq(bfqd, bfqq, true, expiration); bfqq 1697 block/bfq-wf2q.c if (!bfqq->dispatched) bfqq 1698 block/bfq-wf2q.c bfq_weights_tree_remove(bfqd, bfqq); bfqq 1704 block/bfq-wf2q.c void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfqq 1706 block/bfq-wf2q.c bfq_log_bfqq(bfqd, bfqq, "add to busy"); bfqq 1708 block/bfq-wf2q.c bfq_activate_bfqq(bfqd, bfqq); bfqq 1710 block/bfq-wf2q.c bfq_mark_bfqq_busy(bfqq); bfqq 1711 block/bfq-wf2q.c bfqd->busy_queues[bfqq->ioprio_class - 1]++; bfqq 1713 block/bfq-wf2q.c if (!bfqq->dispatched) bfqq 1714 block/bfq-wf2q.c if (bfqq->wr_coeff == 1) bfqq 1715 block/bfq-wf2q.c bfq_weights_tree_add(bfqd, bfqq, bfqq 1718 block/bfq-wf2q.c if (bfqq->wr_coeff > 1)