bfqq              222 block/bfq-cgroup.c void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
bfqq              227 block/bfq-cgroup.c 	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
bfqq              228 block/bfq-cgroup.c 		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
bfqq              257 block/bfq-cgroup.c void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
bfqq              308 block/bfq-cgroup.c struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
bfqq              310 block/bfq-cgroup.c 	struct bfq_entity *group_entity = bfqq->entity.parent;
bfqq              314 block/bfq-cgroup.c 			      bfqq->bfqd->root_group;
bfqq              415 block/bfq-cgroup.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              419 block/bfq-cgroup.c 	if (bfqq) {
bfqq              420 block/bfq-cgroup.c 		bfqq->ioprio = bfqq->new_ioprio;
bfqq              421 block/bfq-cgroup.c 		bfqq->ioprio_class = bfqq->new_ioprio_class;
bfqq              623 block/bfq-cgroup.c void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq              626 block/bfq-cgroup.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq              632 block/bfq-cgroup.c 	bfqq->ref++;
bfqq              640 block/bfq-cgroup.c 	if (bfqq == bfqd->in_service_queue)
bfqq              644 block/bfq-cgroup.c 	if (bfq_bfqq_busy(bfqq))
bfqq              645 block/bfq-cgroup.c 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
bfqq              648 block/bfq-cgroup.c 	bfqg_and_blkg_put(bfqq_group(bfqq));
bfqq              655 block/bfq-cgroup.c 	if (bfq_bfqq_busy(bfqq)) {
bfqq              657 block/bfq-cgroup.c 			bfq_pos_tree_add_move(bfqd, bfqq);
bfqq              658 block/bfq-cgroup.c 		bfq_activate_bfqq(bfqd, bfqq);
bfqq              664 block/bfq-cgroup.c 	bfq_put_queue(bfqq);
bfqq              808 block/bfq-cgroup.c 	struct bfq_queue *bfqq;
bfqq              823 block/bfq-cgroup.c 	bfqq = bfq_entity_to_bfqq(child_entity);
bfqq              824 block/bfq-cgroup.c 	bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);
bfqq             1376 block/bfq-cgroup.c void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1381 block/bfq-cgroup.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq             1385 block/bfq-cgroup.c 	if (bfqq) {
bfqq             1386 block/bfq-cgroup.c 		bfqq->ioprio = bfqq->new_ioprio;
bfqq             1387 block/bfq-cgroup.c 		bfqq->ioprio_class = bfqq->new_ioprio_class;
bfqq             1404 block/bfq-cgroup.c struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
bfqq             1406 block/bfq-cgroup.c 	return bfqq->bfqd->root_group;
bfqq              135 block/bfq-iosched.c void bfq_mark_bfqq_##name(struct bfq_queue *bfqq)			\
bfqq              137 block/bfq-iosched.c 	__set_bit(BFQQF_##name, &(bfqq)->flags);			\
bfqq              139 block/bfq-iosched.c void bfq_clear_bfqq_##name(struct bfq_queue *bfqq)			\
bfqq              141 block/bfq-iosched.c 	__clear_bit(BFQQF_##name, &(bfqq)->flags);		\
bfqq              143 block/bfq-iosched.c int bfq_bfqq_##name(const struct bfq_queue *bfqq)			\
bfqq              145 block/bfq-iosched.c 	return test_bit(BFQQF_##name, &(bfqq)->flags);		\
bfqq              236 block/bfq-iosched.c #define BFQQ_SEEKY(bfqq)	(hweight32(bfqq->seek_history) > 19)
bfqq              244 block/bfq-iosched.c #define BFQQ_TOTALLY_SEEKY(bfqq)	(bfqq->seek_history == -1)
bfqq              370 block/bfq-iosched.c 	return bic->bfqq[is_sync];
bfqq              373 block/bfq-iosched.c void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync)
bfqq              375 block/bfq-iosched.c 	bic->bfqq[is_sync] = bfqq;
bfqq              429 block/bfq-iosched.c #define bfq_class_idle(bfqq)	((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
bfqq              430 block/bfq-iosched.c #define bfq_class_rt(bfqq)	((bfqq)->ioprio_class == IOPRIO_CLASS_RT)
bfqq              556 block/bfq-iosched.c 	struct bfq_queue *bfqq = NULL;
bfqq              564 block/bfq-iosched.c 		bfqq = rb_entry(parent, struct bfq_queue, pos_node);
bfqq              570 block/bfq-iosched.c 		if (sector > blk_rq_pos(bfqq->next_rq))
bfqq              572 block/bfq-iosched.c 		else if (sector < blk_rq_pos(bfqq->next_rq))
bfqq              577 block/bfq-iosched.c 		bfqq = NULL;
bfqq              586 block/bfq-iosched.c 		bfqq ? bfqq->pid : 0);
bfqq              588 block/bfq-iosched.c 	return bfqq;
bfqq              591 block/bfq-iosched.c static bool bfq_too_late_for_merging(struct bfq_queue *bfqq)
bfqq              593 block/bfq-iosched.c 	return bfqq->service_from_backlogged > 0 &&
bfqq              594 block/bfq-iosched.c 		time_is_before_jiffies(bfqq->first_IO_time +
bfqq              607 block/bfq-iosched.c bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq              612 block/bfq-iosched.c 	if (bfqq->pos_root) {
bfqq              613 block/bfq-iosched.c 		rb_erase(&bfqq->pos_node, bfqq->pos_root);
bfqq              614 block/bfq-iosched.c 		bfqq->pos_root = NULL;
bfqq              618 block/bfq-iosched.c 	if (bfqq == &bfqd->oom_bfqq)
bfqq              626 block/bfq-iosched.c 	if (bfq_too_late_for_merging(bfqq))
bfqq              629 block/bfq-iosched.c 	if (bfq_class_idle(bfqq))
bfqq              631 block/bfq-iosched.c 	if (!bfqq->next_rq)
bfqq              634 block/bfq-iosched.c 	bfqq->pos_root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree;
bfqq              635 block/bfq-iosched.c 	__bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root,
bfqq              636 block/bfq-iosched.c 			blk_rq_pos(bfqq->next_rq), &parent, &p);
bfqq              638 block/bfq-iosched.c 		rb_link_node(&bfqq->pos_node, parent, p);
bfqq              639 block/bfq-iosched.c 		rb_insert_color(&bfqq->pos_node, bfqq->pos_root);
bfqq              641 block/bfq-iosched.c 		bfqq->pos_root = NULL;
bfqq              678 block/bfq-iosched.c 				   struct bfq_queue *bfqq)
bfqq              680 block/bfq-iosched.c 	bool smallest_weight = bfqq &&
bfqq              681 block/bfq-iosched.c 		bfqq->weight_counter &&
bfqq              682 block/bfq-iosched.c 		bfqq->weight_counter ==
bfqq              722 block/bfq-iosched.c void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq              725 block/bfq-iosched.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq              741 block/bfq-iosched.c 	if (bfqq->weight_counter)
bfqq              751 block/bfq-iosched.c 			bfqq->weight_counter = __counter;
bfqq              762 block/bfq-iosched.c 	bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter),
bfqq              777 block/bfq-iosched.c 	if (unlikely(!bfqq->weight_counter))
bfqq              780 block/bfq-iosched.c 	bfqq->weight_counter->weight = entity->weight;
bfqq              781 block/bfq-iosched.c 	rb_link_node(&bfqq->weight_counter->weights_node, parent, new);
bfqq              782 block/bfq-iosched.c 	rb_insert_color_cached(&bfqq->weight_counter->weights_node, root,
bfqq              786 block/bfq-iosched.c 	bfqq->weight_counter->num_active++;
bfqq              787 block/bfq-iosched.c 	bfqq->ref++;
bfqq              797 block/bfq-iosched.c 			       struct bfq_queue *bfqq,
bfqq              800 block/bfq-iosched.c 	if (!bfqq->weight_counter)
bfqq              803 block/bfq-iosched.c 	bfqq->weight_counter->num_active--;
bfqq              804 block/bfq-iosched.c 	if (bfqq->weight_counter->num_active > 0)
bfqq              807 block/bfq-iosched.c 	rb_erase_cached(&bfqq->weight_counter->weights_node, root);
bfqq              808 block/bfq-iosched.c 	kfree(bfqq->weight_counter);
bfqq              811 block/bfq-iosched.c 	bfqq->weight_counter = NULL;
bfqq              812 block/bfq-iosched.c 	bfq_put_queue(bfqq);
bfqq              820 block/bfq-iosched.c 			     struct bfq_queue *bfqq)
bfqq              822 block/bfq-iosched.c 	struct bfq_entity *entity = bfqq->entity.parent;
bfqq              864 block/bfq-iosched.c 	__bfq_weights_tree_remove(bfqd, bfqq,
bfqq              871 block/bfq-iosched.c static struct request *bfq_check_fifo(struct bfq_queue *bfqq,
bfqq              876 block/bfq-iosched.c 	if (bfq_bfqq_fifo_expire(bfqq))
bfqq              879 block/bfq-iosched.c 	bfq_mark_bfqq_fifo_expire(bfqq);
bfqq              881 block/bfq-iosched.c 	rq = rq_entry_fifo(bfqq->fifo.next);
bfqq              886 block/bfq-iosched.c 	bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq);
bfqq              891 block/bfq-iosched.c 					struct bfq_queue *bfqq,
bfqq              899 block/bfq-iosched.c 	next = bfq_check_fifo(bfqq, last);
bfqq              909 block/bfq-iosched.c 		rbnext = rb_first(&bfqq->sort_list);
bfqq              919 block/bfq-iosched.c 					struct bfq_queue *bfqq)
bfqq              921 block/bfq-iosched.c 	if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 ||
bfqq              922 block/bfq-iosched.c 	    bfq_asymmetric_scenario(bfqq->bfqd, bfqq))
bfqq              940 block/bfq-iosched.c 				 struct bfq_queue *bfqq)
bfqq              942 block/bfq-iosched.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq              943 block/bfq-iosched.c 	struct request *next_rq = bfqq->next_rq;
bfqq              949 block/bfq-iosched.c 	if (bfqq == bfqd->in_service_queue)
bfqq              957 block/bfq-iosched.c 			   max_t(unsigned long, bfqq->max_budget,
bfqq              958 block/bfq-iosched.c 				 bfq_serv_to_charge(next_rq, bfqq)),
bfqq              962 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu",
bfqq              964 block/bfq-iosched.c 		bfq_requeue_bfqq(bfqd, bfqq, false);
bfqq             1002 block/bfq-iosched.c static void switch_back_to_interactive_wr(struct bfq_queue *bfqq,
bfqq             1005 block/bfq-iosched.c 	bfqq->wr_coeff = bfqd->bfq_wr_coeff;
bfqq             1006 block/bfq-iosched.c 	bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
bfqq             1007 block/bfq-iosched.c 	bfqq->last_wr_start_finish = bfqq->wr_start_at_switch_to_srt;
bfqq             1011 block/bfq-iosched.c bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd,
bfqq             1014 block/bfq-iosched.c 	unsigned int old_wr_coeff = bfqq->wr_coeff;
bfqq             1015 block/bfq-iosched.c 	bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq);
bfqq             1018 block/bfq-iosched.c 		bfq_mark_bfqq_has_short_ttime(bfqq);
bfqq             1020 block/bfq-iosched.c 		bfq_clear_bfqq_has_short_ttime(bfqq);
bfqq             1023 block/bfq-iosched.c 		bfq_mark_bfqq_IO_bound(bfqq);
bfqq             1025 block/bfq-iosched.c 		bfq_clear_bfqq_IO_bound(bfqq);
bfqq             1027 block/bfq-iosched.c 	bfqq->entity.new_weight = bic->saved_weight;
bfqq             1028 block/bfq-iosched.c 	bfqq->ttime = bic->saved_ttime;
bfqq             1029 block/bfq-iosched.c 	bfqq->wr_coeff = bic->saved_wr_coeff;
bfqq             1030 block/bfq-iosched.c 	bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;
bfqq             1031 block/bfq-iosched.c 	bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;
bfqq             1032 block/bfq-iosched.c 	bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;
bfqq             1034 block/bfq-iosched.c 	if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) ||
bfqq             1035 block/bfq-iosched.c 	    time_is_before_jiffies(bfqq->last_wr_start_finish +
bfqq             1036 block/bfq-iosched.c 				   bfqq->wr_cur_max_time))) {
bfqq             1037 block/bfq-iosched.c 		if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
bfqq             1038 block/bfq-iosched.c 		    !bfq_bfqq_in_large_burst(bfqq) &&
bfqq             1039 block/bfq-iosched.c 		    time_is_after_eq_jiffies(bfqq->wr_start_at_switch_to_srt +
bfqq             1041 block/bfq-iosched.c 			switch_back_to_interactive_wr(bfqq, bfqd);
bfqq             1043 block/bfq-iosched.c 			bfqq->wr_coeff = 1;
bfqq             1044 block/bfq-iosched.c 			bfq_log_bfqq(bfqq->bfqd, bfqq,
bfqq             1050 block/bfq-iosched.c 	bfqq->entity.prio_changed = 1;
bfqq             1055 block/bfq-iosched.c 	if (old_wr_coeff == 1 && bfqq->wr_coeff > 1)
bfqq             1057 block/bfq-iosched.c 	else if (old_wr_coeff > 1 && bfqq->wr_coeff == 1)
bfqq             1061 block/bfq-iosched.c static int bfqq_process_refs(struct bfq_queue *bfqq)
bfqq             1063 block/bfq-iosched.c 	return bfqq->ref - bfqq->allocated - bfqq->entity.on_st -
bfqq             1064 block/bfq-iosched.c 		(bfqq->weight_counter != NULL);
bfqq             1068 block/bfq-iosched.c static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             1082 block/bfq-iosched.c 		hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);
bfqq             1087 block/bfq-iosched.c 	bfqd->burst_parent_entity = bfqq->entity.parent;
bfqq             1091 block/bfq-iosched.c static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             1113 block/bfq-iosched.c 		bfq_mark_bfqq_in_large_burst(bfqq);
bfqq             1131 block/bfq-iosched.c 		hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);
bfqq             1243 block/bfq-iosched.c static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             1250 block/bfq-iosched.c 	if (!hlist_unhashed(&bfqq->burst_list_node) ||
bfqq             1251 block/bfq-iosched.c 	    bfq_bfqq_in_large_burst(bfqq) ||
bfqq             1252 block/bfq-iosched.c 	    time_is_after_eq_jiffies(bfqq->split_time +
bfqq             1275 block/bfq-iosched.c 	    bfqq->entity.parent != bfqd->burst_parent_entity) {
bfqq             1277 block/bfq-iosched.c 		bfq_reset_burst_list(bfqd, bfqq);
bfqq             1287 block/bfq-iosched.c 		bfq_mark_bfqq_in_large_burst(bfqq);
bfqq             1296 block/bfq-iosched.c 	bfq_add_to_burst(bfqd, bfqq);
bfqq             1309 block/bfq-iosched.c static int bfq_bfqq_budget_left(struct bfq_queue *bfqq)
bfqq             1311 block/bfq-iosched.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             1446 block/bfq-iosched.c 						struct bfq_queue *bfqq,
bfqq             1449 block/bfq-iosched.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             1458 block/bfq-iosched.c 	if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time &&
bfqq             1459 block/bfq-iosched.c 	    bfq_bfqq_budget_left(bfqq) > 0) {
bfqq             1478 block/bfq-iosched.c 				       bfq_bfqq_budget_left(bfqq),
bfqq             1479 block/bfq-iosched.c 				       bfqq->max_budget);
bfqq             1499 block/bfq-iosched.c 	entity->budget = max_t(unsigned long, bfqq->max_budget,
bfqq             1500 block/bfq-iosched.c 			       bfq_serv_to_charge(bfqq->next_rq, bfqq));
bfqq             1501 block/bfq-iosched.c 	bfq_clear_bfqq_non_blocking_wait_rq(bfqq);
bfqq             1515 block/bfq-iosched.c 					     struct bfq_queue *bfqq,
bfqq             1525 block/bfq-iosched.c 			bfqq->service_from_wr = 0;
bfqq             1526 block/bfq-iosched.c 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
bfqq             1527 block/bfq-iosched.c 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
bfqq             1540 block/bfq-iosched.c 			bfqq->wr_start_at_switch_to_srt =
bfqq             1542 block/bfq-iosched.c 			bfqq->wr_coeff = bfqd->bfq_wr_coeff *
bfqq             1544 block/bfq-iosched.c 			bfqq->wr_cur_max_time =
bfqq             1557 block/bfq-iosched.c 		bfqq->entity.budget = min_t(unsigned long,
bfqq             1558 block/bfq-iosched.c 					    bfqq->entity.budget,
bfqq             1562 block/bfq-iosched.c 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
bfqq             1563 block/bfq-iosched.c 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
bfqq             1565 block/bfq-iosched.c 			bfqq->wr_coeff = 1;
bfqq             1596 block/bfq-iosched.c 			if (bfqq->wr_cur_max_time !=
bfqq             1598 block/bfq-iosched.c 				bfqq->wr_start_at_switch_to_srt =
bfqq             1599 block/bfq-iosched.c 					bfqq->last_wr_start_finish;
bfqq             1601 block/bfq-iosched.c 				bfqq->wr_cur_max_time =
bfqq             1603 block/bfq-iosched.c 				bfqq->wr_coeff = bfqd->bfq_wr_coeff *
bfqq             1606 block/bfq-iosched.c 			bfqq->last_wr_start_finish = jiffies;
bfqq             1612 block/bfq-iosched.c 					struct bfq_queue *bfqq)
bfqq             1614 block/bfq-iosched.c 	return bfqq->dispatched == 0 &&
bfqq             1616 block/bfq-iosched.c 			bfqq->budget_timeout +
bfqq             1625 block/bfq-iosched.c static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq,
bfqq             1630 block/bfq-iosched.c 	if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class)
bfqq             1633 block/bfq-iosched.c 	if (in_serv_bfqq->entity.parent == bfqq->entity.parent) {
bfqq             1634 block/bfq-iosched.c 		bfqq_weight = bfqq->entity.weight;
bfqq             1637 block/bfq-iosched.c 		if (bfqq->entity.parent)
bfqq             1638 block/bfq-iosched.c 			bfqq_weight = bfqq->entity.parent->weight;
bfqq             1640 block/bfq-iosched.c 			bfqq_weight = bfqq->entity.weight;
bfqq             1651 block/bfq-iosched.c 					     struct bfq_queue *bfqq,
bfqq             1658 block/bfq-iosched.c 		idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq),
bfqq             1665 block/bfq-iosched.c 			bfqq->ttime.last_end_request +
bfqq             1676 block/bfq-iosched.c 	in_burst = bfq_bfqq_in_large_burst(bfqq);
bfqq             1678 block/bfq-iosched.c 		!BFQQ_TOTALLY_SEEKY(bfqq) &&
bfqq             1680 block/bfq-iosched.c 		time_is_before_jiffies(bfqq->soft_rt_next_start) &&
bfqq             1681 block/bfq-iosched.c 		bfqq->dispatched == 0;
bfqq             1684 block/bfq-iosched.c 		(bfqq->wr_coeff > 1 ||
bfqq             1685 block/bfq-iosched.c 		 (bfq_bfqq_sync(bfqq) &&
bfqq             1686 block/bfq-iosched.c 		  bfqq->bic && (*interactive || soft_rt)));
bfqq             1693 block/bfq-iosched.c 		bfq_bfqq_update_budg_for_activation(bfqd, bfqq,
bfqq             1709 block/bfq-iosched.c 	if (likely(!bfq_bfqq_just_created(bfqq)) &&
bfqq             1712 block/bfq-iosched.c 		    bfqq->budget_timeout +
bfqq             1714 block/bfq-iosched.c 		hlist_del_init(&bfqq->burst_list_node);
bfqq             1715 block/bfq-iosched.c 		bfq_clear_bfqq_in_large_burst(bfqq);
bfqq             1718 block/bfq-iosched.c 	bfq_clear_bfqq_just_created(bfqq);
bfqq             1721 block/bfq-iosched.c 	if (!bfq_bfqq_IO_bound(bfqq)) {
bfqq             1723 block/bfq-iosched.c 			bfqq->requests_within_timer++;
bfqq             1724 block/bfq-iosched.c 			if (bfqq->requests_within_timer >=
bfqq             1726 block/bfq-iosched.c 				bfq_mark_bfqq_IO_bound(bfqq);
bfqq             1728 block/bfq-iosched.c 			bfqq->requests_within_timer = 0;
bfqq             1732 block/bfq-iosched.c 		if (unlikely(time_is_after_jiffies(bfqq->split_time)))
bfqq             1734 block/bfq-iosched.c 			bfqq->split_time =
bfqq             1737 block/bfq-iosched.c 		if (time_is_before_jiffies(bfqq->split_time +
bfqq             1739 block/bfq-iosched.c 			bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq,
bfqq             1746 block/bfq-iosched.c 			if (old_wr_coeff != bfqq->wr_coeff)
bfqq             1747 block/bfq-iosched.c 				bfqq->entity.prio_changed = 1;
bfqq             1751 block/bfq-iosched.c 	bfqq->last_idle_bklogged = jiffies;
bfqq             1752 block/bfq-iosched.c 	bfqq->service_from_backlogged = 0;
bfqq             1753 block/bfq-iosched.c 	bfq_clear_bfqq_softrt_update(bfqq);
bfqq             1755 block/bfq-iosched.c 	bfq_add_bfqq_busy(bfqd, bfqq);
bfqq             1791 block/bfq-iosched.c 	      bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) ||
bfqq             1792 block/bfq-iosched.c 	     bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) &&
bfqq             1799 block/bfq-iosched.c 				   struct bfq_queue *bfqq)
bfqq             1802 block/bfq-iosched.c 	bfqq->last_serv_time_ns = 0;
bfqq             1856 block/bfq-iosched.c 	if (bfq_bfqq_has_short_ttime(bfqq))
bfqq             1857 block/bfq-iosched.c 		bfqq->inject_limit = 0;
bfqq             1859 block/bfq-iosched.c 		bfqq->inject_limit = 1;
bfqq             1861 block/bfq-iosched.c 	bfqq->decrease_time_jif = jiffies;
bfqq             1866 block/bfq-iosched.c 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
bfqq             1867 block/bfq-iosched.c 	struct bfq_data *bfqd = bfqq->bfqd;
bfqq             1869 block/bfq-iosched.c 	unsigned int old_wr_coeff = bfqq->wr_coeff;
bfqq             1872 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq));
bfqq             1873 block/bfq-iosched.c 	bfqq->queued[rq_is_sync(rq)]++;
bfqq             1876 block/bfq-iosched.c 	if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) {
bfqq             1932 block/bfq-iosched.c 		    !bfq_bfqq_has_short_ttime(bfqq) &&
bfqq             1935 block/bfq-iosched.c 			if (bfqd->last_completed_rq_bfqq != bfqq &&
bfqq             1937 block/bfq-iosched.c 			    bfqq->waker_bfqq) {
bfqq             1944 block/bfq-iosched.c 				bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;
bfqq             1966 block/bfq-iosched.c 				if (!hlist_unhashed(&bfqq->woken_list_node))
bfqq             1967 block/bfq-iosched.c 					hlist_del_init(&bfqq->woken_list_node);
bfqq             1968 block/bfq-iosched.c 				hlist_add_head(&bfqq->woken_list_node,
bfqq             1971 block/bfq-iosched.c 				bfq_clear_bfqq_has_waker(bfqq);
bfqq             1973 block/bfq-iosched.c 				   bfqq->waker_bfqq &&
bfqq             1974 block/bfq-iosched.c 				   !bfq_bfqq_has_waker(bfqq)) {
bfqq             1979 block/bfq-iosched.c 				bfq_mark_bfqq_has_waker(bfqq);
bfqq             1989 block/bfq-iosched.c 		if (time_is_before_eq_jiffies(bfqq->decrease_time_jif +
bfqq             1991 block/bfq-iosched.c 			bfq_reset_inject_limit(bfqd, bfqq);
bfqq             2018 block/bfq-iosched.c 		if (bfqq == bfqd->in_service_queue &&
bfqq             2020 block/bfq-iosched.c 		     (bfqq->last_serv_time_ns > 0 &&
bfqq             2022 block/bfq-iosched.c 		    time_is_before_eq_jiffies(bfqq->decrease_time_jif +
bfqq             2050 block/bfq-iosched.c 	elv_rb_add(&bfqq->sort_list, rq);
bfqq             2055 block/bfq-iosched.c 	prev = bfqq->next_rq;
bfqq             2056 block/bfq-iosched.c 	next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position);
bfqq             2057 block/bfq-iosched.c 	bfqq->next_rq = next_rq;
bfqq             2063 block/bfq-iosched.c 	if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq))
bfqq             2064 block/bfq-iosched.c 		bfq_pos_tree_add_move(bfqd, bfqq);
bfqq             2066 block/bfq-iosched.c 	if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */
bfqq             2067 block/bfq-iosched.c 		bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff,
bfqq             2072 block/bfq-iosched.c 				bfqq->last_wr_start_finish +
bfqq             2074 block/bfq-iosched.c 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
bfqq             2075 block/bfq-iosched.c 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
bfqq             2078 block/bfq-iosched.c 			bfqq->entity.prio_changed = 1;
bfqq             2080 block/bfq-iosched.c 		if (prev != bfqq->next_rq)
bfqq             2081 block/bfq-iosched.c 			bfq_updated_next_req(bfqd, bfqq);
bfqq             2111 block/bfq-iosched.c 		(old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive))
bfqq             2112 block/bfq-iosched.c 		bfqq->last_wr_start_finish = jiffies;
bfqq             2119 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfqd->bio_bfqq;
bfqq             2122 block/bfq-iosched.c 	if (bfqq)
bfqq             2123 block/bfq-iosched.c 		return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio));
bfqq             2155 block/bfq-iosched.c 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
bfqq             2156 block/bfq-iosched.c 	struct bfq_data *bfqd = bfqq->bfqd;
bfqq             2159 block/bfq-iosched.c 	if (bfqq->next_rq == rq) {
bfqq             2160 block/bfq-iosched.c 		bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq);
bfqq             2161 block/bfq-iosched.c 		bfq_updated_next_req(bfqd, bfqq);
bfqq             2166 block/bfq-iosched.c 	bfqq->queued[sync]--;
bfqq             2168 block/bfq-iosched.c 	elv_rb_del(&bfqq->sort_list, rq);
bfqq             2174 block/bfq-iosched.c 	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
bfqq             2175 block/bfq-iosched.c 		bfqq->next_rq = NULL;
bfqq             2177 block/bfq-iosched.c 		if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) {
bfqq             2178 block/bfq-iosched.c 			bfq_del_bfqq_busy(bfqd, bfqq, false);
bfqq             2192 block/bfq-iosched.c 			bfqq->entity.budget = bfqq->entity.service = 0;
bfqq             2198 block/bfq-iosched.c 		if (bfqq->pos_root) {
bfqq             2199 block/bfq-iosched.c 			rb_erase(&bfqq->pos_node, bfqq->pos_root);
bfqq             2200 block/bfq-iosched.c 			bfqq->pos_root = NULL;
bfqq             2205 block/bfq-iosched.c 			bfq_pos_tree_add_move(bfqd, bfqq);
bfqq             2209 block/bfq-iosched.c 		bfqq->meta_pending--;
bfqq             2271 block/bfq-iosched.c 		struct bfq_queue *bfqq = bfq_init_rq(req);
bfqq             2275 block/bfq-iosched.c 		if (!bfqq)
bfqq             2278 block/bfq-iosched.c 		bfqd = bfqq->bfqd;
bfqq             2281 block/bfq-iosched.c 		elv_rb_del(&bfqq->sort_list, req);
bfqq             2282 block/bfq-iosched.c 		elv_rb_add(&bfqq->sort_list, req);
bfqq             2285 block/bfq-iosched.c 		prev = bfqq->next_rq;
bfqq             2286 block/bfq-iosched.c 		next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req,
bfqq             2288 block/bfq-iosched.c 		bfqq->next_rq = next_rq;
bfqq             2294 block/bfq-iosched.c 		if (prev != bfqq->next_rq) {
bfqq             2295 block/bfq-iosched.c 			bfq_updated_next_req(bfqd, bfqq);
bfqq             2301 block/bfq-iosched.c 				bfq_pos_tree_add_move(bfqd, bfqq);
bfqq             2323 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfq_init_rq(rq),
bfqq             2326 block/bfq-iosched.c 	if (!bfqq)
bfqq             2338 block/bfq-iosched.c 	if (bfqq == next_bfqq &&
bfqq             2346 block/bfq-iosched.c 	if (bfqq->next_rq == next)
bfqq             2347 block/bfq-iosched.c 		bfqq->next_rq = rq;
bfqq             2349 block/bfq-iosched.c 	bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags);
bfqq             2353 block/bfq-iosched.c static void bfq_bfqq_end_wr(struct bfq_queue *bfqq)
bfqq             2355 block/bfq-iosched.c 	if (bfq_bfqq_busy(bfqq))
bfqq             2356 block/bfq-iosched.c 		bfqq->bfqd->wr_busy_queues--;
bfqq             2357 block/bfq-iosched.c 	bfqq->wr_coeff = 1;
bfqq             2358 block/bfq-iosched.c 	bfqq->wr_cur_max_time = 0;
bfqq             2359 block/bfq-iosched.c 	bfqq->last_wr_start_finish = jiffies;
bfqq             2364 block/bfq-iosched.c 	bfqq->entity.prio_changed = 1;
bfqq             2382 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             2386 block/bfq-iosched.c 	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list)
bfqq             2387 block/bfq-iosched.c 		bfq_bfqq_end_wr(bfqq);
bfqq             2388 block/bfq-iosched.c 	list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list)
bfqq             2389 block/bfq-iosched.c 		bfq_bfqq_end_wr(bfqq);
bfqq             2411 block/bfq-iosched.c 					 struct bfq_queue *bfqq,
bfqq             2414 block/bfq-iosched.c 	struct rb_root *root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree;
bfqq             2456 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             2465 block/bfq-iosched.c 	bfqq = bfqq_find_close(bfqd, cur_bfqq, sector);
bfqq             2466 block/bfq-iosched.c 	if (!bfqq || bfqq == cur_bfqq)
bfqq             2469 block/bfq-iosched.c 	return bfqq;
bfqq             2473 block/bfq-iosched.c bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
bfqq             2489 block/bfq-iosched.c 		if (__bfqq == bfqq)
bfqq             2494 block/bfq-iosched.c 	process_refs = bfqq_process_refs(bfqq);
bfqq             2503 block/bfq-iosched.c 	bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d",
bfqq             2526 block/bfq-iosched.c 	bfqq->new_bfqq = new_bfqq;
bfqq             2531 block/bfq-iosched.c static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
bfqq             2537 block/bfq-iosched.c 	if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) ||
bfqq             2538 block/bfq-iosched.c 	    (bfqq->ioprio_class != new_bfqq->ioprio_class))
bfqq             2546 block/bfq-iosched.c 	if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq))
bfqq             2554 block/bfq-iosched.c 	if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq))
bfqq             2581 block/bfq-iosched.c bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             2637 block/bfq-iosched.c 	if (bfq_too_late_for_merging(bfqq))
bfqq             2640 block/bfq-iosched.c 	if (bfqq->new_bfqq)
bfqq             2641 block/bfq-iosched.c 		return bfqq->new_bfqq;
bfqq             2643 block/bfq-iosched.c 	if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq))
bfqq             2652 block/bfq-iosched.c 	if (in_service_bfqq && in_service_bfqq != bfqq &&
bfqq             2656 block/bfq-iosched.c 	    bfqq->entity.parent == in_service_bfqq->entity.parent &&
bfqq             2657 block/bfq-iosched.c 	    bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) {
bfqq             2658 block/bfq-iosched.c 		new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq);
bfqq             2667 block/bfq-iosched.c 	new_bfqq = bfq_find_close_cooperator(bfqd, bfqq,
bfqq             2671 block/bfq-iosched.c 	    bfq_may_be_close_cooperator(bfqq, new_bfqq))
bfqq             2672 block/bfq-iosched.c 		return bfq_setup_merge(bfqq, new_bfqq);
bfqq             2677 block/bfq-iosched.c static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
bfqq             2679 block/bfq-iosched.c 	struct bfq_io_cq *bic = bfqq->bic;
bfqq             2689 block/bfq-iosched.c 	bic->saved_weight = bfqq->entity.orig_weight;
bfqq             2690 block/bfq-iosched.c 	bic->saved_ttime = bfqq->ttime;
bfqq             2691 block/bfq-iosched.c 	bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);
bfqq             2692 block/bfq-iosched.c 	bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
bfqq             2693 block/bfq-iosched.c 	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
bfqq             2694 block/bfq-iosched.c 	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
bfqq             2695 block/bfq-iosched.c 	if (unlikely(bfq_bfqq_just_created(bfqq) &&
bfqq             2696 block/bfq-iosched.c 		     !bfq_bfqq_in_large_burst(bfqq) &&
bfqq             2697 block/bfq-iosched.c 		     bfqq->bfqd->low_latency)) {
bfqq             2707 block/bfq-iosched.c 		bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff;
bfqq             2709 block/bfq-iosched.c 		bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd);
bfqq             2712 block/bfq-iosched.c 		bic->saved_wr_coeff = bfqq->wr_coeff;
bfqq             2714 block/bfq-iosched.c 			bfqq->wr_start_at_switch_to_srt;
bfqq             2715 block/bfq-iosched.c 		bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;
bfqq             2716 block/bfq-iosched.c 		bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;
bfqq             2720 block/bfq-iosched.c void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             2733 block/bfq-iosched.c 	if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) &&
bfqq             2734 block/bfq-iosched.c 	    bfqq != bfqd->in_service_queue)
bfqq             2735 block/bfq-iosched.c 		bfq_del_bfqq_busy(bfqd, bfqq, false);
bfqq             2737 block/bfq-iosched.c 	bfq_put_queue(bfqq);
bfqq             2742 block/bfq-iosched.c 		struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
bfqq             2744 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu",
bfqq             2747 block/bfq-iosched.c 	bfq_bfqq_save_state(bfqq);
bfqq             2749 block/bfq-iosched.c 	if (bfq_bfqq_IO_bound(bfqq))
bfqq             2751 block/bfq-iosched.c 	bfq_clear_bfqq_IO_bound(bfqq);
bfqq             2762 block/bfq-iosched.c 	if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) {
bfqq             2763 block/bfq-iosched.c 		new_bfqq->wr_coeff = bfqq->wr_coeff;
bfqq             2764 block/bfq-iosched.c 		new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time;
bfqq             2765 block/bfq-iosched.c 		new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish;
bfqq             2767 block/bfq-iosched.c 			bfqq->wr_start_at_switch_to_srt;
bfqq             2773 block/bfq-iosched.c 	if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */
bfqq             2774 block/bfq-iosched.c 		bfqq->wr_coeff = 1;
bfqq             2775 block/bfq-iosched.c 		bfqq->entity.prio_changed = 1;
bfqq             2776 block/bfq-iosched.c 		if (bfq_bfqq_busy(bfqq))
bfqq             2809 block/bfq-iosched.c 	bfqq->bic = NULL;
bfqq             2810 block/bfq-iosched.c 	bfq_release_process_ref(bfqd, bfqq);
bfqq             2818 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq;
bfqq             2830 block/bfq-iosched.c 	if (!bfqq)
bfqq             2837 block/bfq-iosched.c 	new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false);
bfqq             2846 block/bfq-iosched.c 		bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq,
bfqq             2853 block/bfq-iosched.c 		bfqq = new_bfqq;
bfqq             2861 block/bfq-iosched.c 		bfqd->bio_bfqq = bfqq;
bfqq             2864 block/bfq-iosched.c 	return bfqq == RQ_BFQQ(rq);
bfqq             2874 block/bfq-iosched.c 				   struct bfq_queue *bfqq)
bfqq             2878 block/bfq-iosched.c 	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)
bfqq             2881 block/bfq-iosched.c 		timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight;
bfqq             2885 block/bfq-iosched.c 	bfqq->budget_timeout = jiffies +
bfqq             2890 block/bfq-iosched.c 				       struct bfq_queue *bfqq)
bfqq             2892 block/bfq-iosched.c 	if (bfqq) {
bfqq             2893 block/bfq-iosched.c 		bfq_clear_bfqq_fifo_expire(bfqq);
bfqq             2897 block/bfq-iosched.c 		if (time_is_before_jiffies(bfqq->last_wr_start_finish) &&
bfqq             2898 block/bfq-iosched.c 		    bfqq->wr_coeff > 1 &&
bfqq             2899 block/bfq-iosched.c 		    bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
bfqq             2900 block/bfq-iosched.c 		    time_is_before_jiffies(bfqq->budget_timeout)) {
bfqq             2925 block/bfq-iosched.c 			if (time_after(bfqq->budget_timeout,
bfqq             2926 block/bfq-iosched.c 				       bfqq->last_wr_start_finish))
bfqq             2927 block/bfq-iosched.c 				bfqq->last_wr_start_finish +=
bfqq             2928 block/bfq-iosched.c 					jiffies - bfqq->budget_timeout;
bfqq             2930 block/bfq-iosched.c 				bfqq->last_wr_start_finish = jiffies;
bfqq             2933 block/bfq-iosched.c 		bfq_set_budget_timeout(bfqd, bfqq);
bfqq             2934 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq,
bfqq             2936 block/bfq-iosched.c 			     bfqq->entity.budget);
bfqq             2939 block/bfq-iosched.c 	bfqd->in_service_queue = bfqq;
bfqq             2947 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfq_get_next_queue(bfqd);
bfqq             2949 block/bfq-iosched.c 	__bfq_set_in_service_queue(bfqd, bfqq);
bfqq             2950 block/bfq-iosched.c 	return bfqq;
bfqq             2955 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfqd->in_service_queue;
bfqq             2958 block/bfq-iosched.c 	bfq_mark_bfqq_wait_request(bfqq);
bfqq             2976 block/bfq-iosched.c 	if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 &&
bfqq             2977 block/bfq-iosched.c 	    !bfq_asymmetric_scenario(bfqd, bfqq))
bfqq             2979 block/bfq-iosched.c 	else if (bfqq->wr_coeff > 1)
bfqq             2987 block/bfq-iosched.c 	bfqg_stats_set_start_idle_time(bfqq_group(bfqq));
bfqq             3240 block/bfq-iosched.c 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
bfqq             3254 block/bfq-iosched.c 	bfqq->dispatched++;
bfqq             3447 block/bfq-iosched.c 						 struct bfq_queue *bfqq)
bfqq             3450 block/bfq-iosched.c 	if (unlikely(!bfqq_process_refs(bfqq)))
bfqq             3453 block/bfq-iosched.c 	return (bfqq->wr_coeff > 1 &&
bfqq             3457 block/bfq-iosched.c 		 bfqq->dispatched + 4)) ||
bfqq             3458 block/bfq-iosched.c 		bfq_asymmetric_scenario(bfqd, bfqq);
bfqq             3461 block/bfq-iosched.c static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             3470 block/bfq-iosched.c 	if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq))
bfqq             3471 block/bfq-iosched.c 		bfq_mark_bfqq_split_coop(bfqq);
bfqq             3486 block/bfq-iosched.c 	if (RB_EMPTY_ROOT(&bfqq->sort_list) &&
bfqq             3488 block/bfq-iosched.c 	      idling_needed_for_service_guarantees(bfqd, bfqq))) {
bfqq             3489 block/bfq-iosched.c 		if (bfqq->dispatched == 0)
bfqq             3496 block/bfq-iosched.c 			bfqq->budget_timeout = jiffies;
bfqq             3498 block/bfq-iosched.c 		bfq_del_bfqq_busy(bfqd, bfqq, true);
bfqq             3500 block/bfq-iosched.c 		bfq_requeue_bfqq(bfqd, bfqq, true);
bfqq             3506 block/bfq-iosched.c 			     !RB_EMPTY_ROOT(&bfqq->sort_list)))
bfqq             3507 block/bfq-iosched.c 			bfq_pos_tree_add_move(bfqd, bfqq);
bfqq             3530 block/bfq-iosched.c 				     struct bfq_queue *bfqq,
bfqq             3538 block/bfq-iosched.c 	if (bfqq->wr_coeff == 1)
bfqq             3539 block/bfq-iosched.c 		budget = bfqq->max_budget;
bfqq             3548 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d",
bfqq             3549 block/bfq-iosched.c 		bfqq->entity.budget, bfq_bfqq_budget_left(bfqq));
bfqq             3550 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d",
bfqq             3552 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d",
bfqq             3553 block/bfq-iosched.c 		bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue));
bfqq             3555 block/bfq-iosched.c 	if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) {
bfqq             3586 block/bfq-iosched.c 			if (bfqq->dispatched > 0) /* still outstanding reqs */
bfqq             3649 block/bfq-iosched.c 			budget = max_t(int, bfqq->entity.service, min_budget);
bfqq             3654 block/bfq-iosched.c 	} else if (!bfq_bfqq_sync(bfqq)) {
bfqq             3664 block/bfq-iosched.c 	bfqq->max_budget = budget;
bfqq             3668 block/bfq-iosched.c 		bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget);
bfqq             3680 block/bfq-iosched.c 	next_rq = bfqq->next_rq;
bfqq             3682 block/bfq-iosched.c 		bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget,
bfqq             3683 block/bfq-iosched.c 					    bfq_serv_to_charge(next_rq, bfqq));
bfqq             3685 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d",
bfqq             3687 block/bfq-iosched.c 			bfqq->entity.budget);
bfqq             3721 block/bfq-iosched.c static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             3727 block/bfq-iosched.c 	bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */
bfqq             3729 block/bfq-iosched.c 	if (!bfq_bfqq_sync(bfqq))
bfqq             3770 block/bfq-iosched.c 		slow = bfqq->entity.service < bfqd->bfq_max_budget / 2;
bfqq             3773 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow);
bfqq             3872 block/bfq-iosched.c 						struct bfq_queue *bfqq)
bfqq             3874 block/bfq-iosched.c 	return max3(bfqq->soft_rt_next_start,
bfqq             3875 block/bfq-iosched.c 		    bfqq->last_idle_bklogged +
bfqq             3876 block/bfq-iosched.c 		    HZ * bfqq->service_from_backlogged /
bfqq             3878 block/bfq-iosched.c 		    jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4);
bfqq             3908 block/bfq-iosched.c 		     struct bfq_queue *bfqq,
bfqq             3914 block/bfq-iosched.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             3919 block/bfq-iosched.c 	slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta);
bfqq             3936 block/bfq-iosched.c 	if (bfqq->wr_coeff == 1 &&
bfqq             3939 block/bfq-iosched.c 	      bfq_bfqq_budget_left(bfqq) >=  entity->budget / 3)))
bfqq             3940 block/bfq-iosched.c 		bfq_bfqq_charge_time(bfqd, bfqq, delta);
bfqq             3944 block/bfq-iosched.c 		bfq_clear_bfqq_IO_bound(bfqq);
bfqq             3946 block/bfq-iosched.c 	if (bfqd->low_latency && bfqq->wr_coeff == 1)
bfqq             3947 block/bfq-iosched.c 		bfqq->last_wr_start_finish = jiffies;
bfqq             3950 block/bfq-iosched.c 	    RB_EMPTY_ROOT(&bfqq->sort_list)) {
bfqq             3977 block/bfq-iosched.c 		if (bfqq->dispatched == 0 &&
bfqq             3978 block/bfq-iosched.c 		    bfqq->wr_coeff != bfqd->bfq_wr_coeff)
bfqq             3979 block/bfq-iosched.c 			bfqq->soft_rt_next_start =
bfqq             3980 block/bfq-iosched.c 				bfq_bfqq_softrt_next_start(bfqd, bfqq);
bfqq             3981 block/bfq-iosched.c 		else if (bfqq->dispatched > 0) {
bfqq             3986 block/bfq-iosched.c 			bfq_mark_bfqq_softrt_update(bfqq);
bfqq             3990 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq,
bfqq             3992 block/bfq-iosched.c 		slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq));
bfqq             4006 block/bfq-iosched.c 	__bfq_bfqq_recalc_budget(bfqd, bfqq, reason);
bfqq             4007 block/bfq-iosched.c 	if (__bfq_bfqq_expire(bfqd, bfqq, reason))
bfqq             4012 block/bfq-iosched.c 	if (!bfq_bfqq_busy(bfqq) &&
bfqq             4015 block/bfq-iosched.c 		bfq_mark_bfqq_non_blocking_wait_rq(bfqq);
bfqq             4051 block/bfq-iosched.c static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq)
bfqq             4053 block/bfq-iosched.c 	return time_is_before_eq_jiffies(bfqq->budget_timeout);
bfqq             4064 block/bfq-iosched.c static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq)
bfqq             4066 block/bfq-iosched.c 	bfq_log_bfqq(bfqq->bfqd, bfqq,
bfqq             4068 block/bfq-iosched.c 		bfq_bfqq_wait_request(bfqq),
bfqq             4069 block/bfq-iosched.c 			bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3,
bfqq             4070 block/bfq-iosched.c 		bfq_bfqq_budget_timeout(bfqq));
bfqq             4072 block/bfq-iosched.c 	return (!bfq_bfqq_wait_request(bfqq) ||
bfqq             4073 block/bfq-iosched.c 		bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3)
bfqq             4075 block/bfq-iosched.c 		bfq_bfqq_budget_timeout(bfqq);
bfqq             4079 block/bfq-iosched.c 					     struct bfq_queue *bfqq)
bfqq             4087 block/bfq-iosched.c 	if (unlikely(!bfqq_process_refs(bfqq)))
bfqq             4090 block/bfq-iosched.c 	bfqq_sequential_and_IO_bound = !BFQQ_SEEKY(bfqq) &&
bfqq             4091 block/bfq-iosched.c 		bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_has_short_ttime(bfqq);
bfqq             4178 block/bfq-iosched.c static bool bfq_better_to_idle(struct bfq_queue *bfqq)
bfqq             4180 block/bfq-iosched.c 	struct bfq_data *bfqd = bfqq->bfqd;
bfqq             4184 block/bfq-iosched.c 	if (unlikely(!bfqq_process_refs(bfqq)))
bfqq             4198 block/bfq-iosched.c 	if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) ||
bfqq             4199 block/bfq-iosched.c 	   bfq_class_idle(bfqq))
bfqq             4203 block/bfq-iosched.c 		idling_boosts_thr_without_issues(bfqd, bfqq);
bfqq             4206 block/bfq-iosched.c 		idling_needed_for_service_guarantees(bfqd, bfqq);
bfqq             4229 block/bfq-iosched.c static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq)
bfqq             4231 block/bfq-iosched.c 	return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq);
bfqq             4244 block/bfq-iosched.c 	struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue;
bfqq             4292 block/bfq-iosched.c 	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list)
bfqq             4293 block/bfq-iosched.c 		if (!RB_EMPTY_ROOT(&bfqq->sort_list) &&
bfqq             4294 block/bfq-iosched.c 		    (in_serv_always_inject || bfqq->wr_coeff > 1) &&
bfqq             4295 block/bfq-iosched.c 		    bfq_serv_to_charge(bfqq->next_rq, bfqq) <=
bfqq             4296 block/bfq-iosched.c 		    bfq_bfqq_budget_left(bfqq)) {
bfqq             4315 block/bfq-iosched.c 			    blk_rq_sectors(bfqq->next_rq) >=
bfqq             4323 block/bfq-iosched.c 				return bfqq;
bfqq             4336 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             4340 block/bfq-iosched.c 	bfqq = bfqd->in_service_queue;
bfqq             4341 block/bfq-iosched.c 	if (!bfqq)
bfqq             4344 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue");
bfqq             4353 block/bfq-iosched.c 	if (bfq_may_expire_for_budg_timeout(bfqq) &&
bfqq             4354 block/bfq-iosched.c 	    !bfq_bfqq_must_idle(bfqq))
bfqq             4364 block/bfq-iosched.c 	next_rq = bfqq->next_rq;
bfqq             4370 block/bfq-iosched.c 		if (bfq_serv_to_charge(next_rq, bfqq) >
bfqq             4371 block/bfq-iosched.c 			bfq_bfqq_budget_left(bfqq)) {
bfqq             4386 block/bfq-iosched.c 			if (bfq_bfqq_wait_request(bfqq)) {
bfqq             4400 block/bfq-iosched.c 				bfq_clear_bfqq_wait_request(bfqq);
bfqq             4415 block/bfq-iosched.c 	if (bfq_bfqq_wait_request(bfqq) ||
bfqq             4416 block/bfq-iosched.c 	    (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) {
bfqq             4418 block/bfq-iosched.c 			bfqq->bic && bfqq->bic->bfqq[0] &&
bfqq             4419 block/bfq-iosched.c 			bfq_bfqq_busy(bfqq->bic->bfqq[0]) &&
bfqq             4420 block/bfq-iosched.c 			bfqq->bic->bfqq[0]->next_rq ?
bfqq             4421 block/bfq-iosched.c 			bfqq->bic->bfqq[0] : NULL;
bfqq             4496 block/bfq-iosched.c 		    icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&
bfqq             4499 block/bfq-iosched.c 			bfqq = bfqq->bic->bfqq[0];
bfqq             4500 block/bfq-iosched.c 		else if (bfq_bfqq_has_waker(bfqq) &&
bfqq             4501 block/bfq-iosched.c 			   bfq_bfqq_busy(bfqq->waker_bfqq) &&
bfqq             4502 block/bfq-iosched.c 			   bfqq->next_rq &&
bfqq             4503 block/bfq-iosched.c 			   bfq_serv_to_charge(bfqq->waker_bfqq->next_rq,
bfqq             4504 block/bfq-iosched.c 					      bfqq->waker_bfqq) <=
bfqq             4505 block/bfq-iosched.c 			   bfq_bfqq_budget_left(bfqq->waker_bfqq)
bfqq             4507 block/bfq-iosched.c 			bfqq = bfqq->waker_bfqq;
bfqq             4508 block/bfq-iosched.c 		else if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&
bfqq             4509 block/bfq-iosched.c 			 (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 ||
bfqq             4510 block/bfq-iosched.c 			  !bfq_bfqq_has_short_ttime(bfqq)))
bfqq             4511 block/bfq-iosched.c 			bfqq = bfq_choose_bfqq_for_injection(bfqd);
bfqq             4513 block/bfq-iosched.c 			bfqq = NULL;
bfqq             4520 block/bfq-iosched.c 	bfq_bfqq_expire(bfqd, bfqq, false, reason);
bfqq             4522 block/bfq-iosched.c 	bfqq = bfq_set_in_service_queue(bfqd);
bfqq             4523 block/bfq-iosched.c 	if (bfqq) {
bfqq             4524 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue");
bfqq             4528 block/bfq-iosched.c 	if (bfqq)
bfqq             4529 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue");
bfqq             4533 block/bfq-iosched.c 	return bfqq;
bfqq             4536 block/bfq-iosched.c static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             4538 block/bfq-iosched.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             4540 block/bfq-iosched.c 	if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */
bfqq             4541 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq,
bfqq             4543 block/bfq-iosched.c 			jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish),
bfqq             4544 block/bfq-iosched.c 			jiffies_to_msecs(bfqq->wr_cur_max_time),
bfqq             4545 block/bfq-iosched.c 			bfqq->wr_coeff,
bfqq             4546 block/bfq-iosched.c 			bfqq->entity.weight, bfqq->entity.orig_weight);
bfqq             4549 block/bfq-iosched.c 			bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change");
bfqq             4556 block/bfq-iosched.c 		if (bfq_bfqq_in_large_burst(bfqq))
bfqq             4557 block/bfq-iosched.c 			bfq_bfqq_end_wr(bfqq);
bfqq             4558 block/bfq-iosched.c 		else if (time_is_before_jiffies(bfqq->last_wr_start_finish +
bfqq             4559 block/bfq-iosched.c 						bfqq->wr_cur_max_time)) {
bfqq             4560 block/bfq-iosched.c 			if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time ||
bfqq             4561 block/bfq-iosched.c 			time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +
bfqq             4563 block/bfq-iosched.c 				bfq_bfqq_end_wr(bfqq);
bfqq             4565 block/bfq-iosched.c 				switch_back_to_interactive_wr(bfqq, bfqd);
bfqq             4566 block/bfq-iosched.c 				bfqq->entity.prio_changed = 1;
bfqq             4569 block/bfq-iosched.c 		if (bfqq->wr_coeff > 1 &&
bfqq             4570 block/bfq-iosched.c 		    bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time &&
bfqq             4571 block/bfq-iosched.c 		    bfqq->service_from_wr > max_service_from_wr) {
bfqq             4573 block/bfq-iosched.c 			bfq_bfqq_end_wr(bfqq);
bfqq             4584 block/bfq-iosched.c 	if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1))
bfqq             4593 block/bfq-iosched.c 						 struct bfq_queue *bfqq)
bfqq             4595 block/bfq-iosched.c 	struct request *rq = bfqq->next_rq;
bfqq             4598 block/bfq-iosched.c 	service_to_charge = bfq_serv_to_charge(rq, bfqq);
bfqq             4600 block/bfq-iosched.c 	bfq_bfqq_served(bfqq, service_to_charge);
bfqq             4602 block/bfq-iosched.c 	if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) {
bfqq             4609 block/bfq-iosched.c 	if (bfqq != bfqd->in_service_queue)
bfqq             4623 block/bfq-iosched.c 	bfq_update_wr_data(bfqd, bfqq);
bfqq             4630 block/bfq-iosched.c 	if (!(bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq)))
bfqq             4633 block/bfq-iosched.c 	bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED);
bfqq             4655 block/bfq-iosched.c 	struct bfq_queue *bfqq = NULL;
bfqq             4662 block/bfq-iosched.c 		bfqq = RQ_BFQQ(rq);
bfqq             4664 block/bfq-iosched.c 		if (bfqq) {
bfqq             4671 block/bfq-iosched.c 			bfqq->dispatched++;
bfqq             4723 block/bfq-iosched.c 	bfqq = bfq_select_queue(bfqd);
bfqq             4724 block/bfq-iosched.c 	if (!bfqq)
bfqq             4727 block/bfq-iosched.c 	rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq);
bfqq             4745 block/bfq-iosched.c 	struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL;
bfqq             4747 block/bfq-iosched.c 	if (!idle_timer_disabled && !bfqq)
bfqq             4775 block/bfq-iosched.c 	if (bfqq) {
bfqq             4776 block/bfq-iosched.c 		struct bfq_group *bfqg = bfqq_group(bfqq);
bfqq             4823 block/bfq-iosched.c void bfq_put_queue(struct bfq_queue *bfqq)
bfqq             4827 block/bfq-iosched.c 	struct bfq_group *bfqg = bfqq_group(bfqq);
bfqq             4829 block/bfq-iosched.c 	if (bfqq->bfqd)
bfqq             4830 block/bfq-iosched.c 		bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d",
bfqq             4831 block/bfq-iosched.c 			     bfqq, bfqq->ref);
bfqq             4833 block/bfq-iosched.c 	bfqq->ref--;
bfqq             4834 block/bfq-iosched.c 	if (bfqq->ref)
bfqq             4837 block/bfq-iosched.c 	if (!hlist_unhashed(&bfqq->burst_list_node)) {
bfqq             4838 block/bfq-iosched.c 		hlist_del_init(&bfqq->burst_list_node);
bfqq             4865 block/bfq-iosched.c 		if (bfqq->bic && bfqq->bfqd->burst_size > 0)
bfqq             4866 block/bfq-iosched.c 			bfqq->bfqd->burst_size--;
bfqq             4885 block/bfq-iosched.c 	if (!hlist_unhashed(&bfqq->woken_list_node))
bfqq             4886 block/bfq-iosched.c 		hlist_del_init(&bfqq->woken_list_node);
bfqq             4889 block/bfq-iosched.c 	hlist_for_each_entry_safe(item, n, &bfqq->woken_list,
bfqq             4896 block/bfq-iosched.c 	if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq)
bfqq             4897 block/bfq-iosched.c 		bfqq->bfqd->last_completed_rq_bfqq = NULL;
bfqq             4899 block/bfq-iosched.c 	kmem_cache_free(bfq_pool, bfqq);
bfqq             4903 block/bfq-iosched.c static void bfq_put_cooperator(struct bfq_queue *bfqq)
bfqq             4912 block/bfq-iosched.c 	__bfqq = bfqq->new_bfqq;
bfqq             4914 block/bfq-iosched.c 		if (__bfqq == bfqq)
bfqq             4922 block/bfq-iosched.c static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             4924 block/bfq-iosched.c 	if (bfqq == bfqd->in_service_queue) {
bfqq             4925 block/bfq-iosched.c 		__bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT);
bfqq             4929 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref);
bfqq             4931 block/bfq-iosched.c 	bfq_put_cooperator(bfqq);
bfqq             4933 block/bfq-iosched.c 	bfq_release_process_ref(bfqd, bfqq);
bfqq             4938 block/bfq-iosched.c 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);
bfqq             4941 block/bfq-iosched.c 	if (bfqq)
bfqq             4942 block/bfq-iosched.c 		bfqd = bfqq->bfqd; /* NULL if scheduler already exited */
bfqq             4944 block/bfq-iosched.c 	if (bfqq && bfqd) {
bfqq             4948 block/bfq-iosched.c 		bfqq->bic = NULL;
bfqq             4949 block/bfq-iosched.c 		bfq_exit_bfqq(bfqd, bfqq);
bfqq             4968 block/bfq-iosched.c bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)
bfqq             4972 block/bfq-iosched.c 	struct bfq_data *bfqd = bfqq->bfqd;
bfqq             4980 block/bfq-iosched.c 		dev_err(bfqq->bfqd->queue->backing_dev_info->dev,
bfqq             4987 block/bfq-iosched.c 		bfqq->new_ioprio = task_nice_ioprio(tsk);
bfqq             4988 block/bfq-iosched.c 		bfqq->new_ioprio_class = task_nice_ioclass(tsk);
bfqq             4991 block/bfq-iosched.c 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);
bfqq             4992 block/bfq-iosched.c 		bfqq->new_ioprio_class = IOPRIO_CLASS_RT;
bfqq             4995 block/bfq-iosched.c 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);
bfqq             4996 block/bfq-iosched.c 		bfqq->new_ioprio_class = IOPRIO_CLASS_BE;
bfqq             4999 block/bfq-iosched.c 		bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE;
bfqq             5000 block/bfq-iosched.c 		bfqq->new_ioprio = 7;
bfqq             5004 block/bfq-iosched.c 	if (bfqq->new_ioprio >= IOPRIO_BE_NR) {
bfqq             5006 block/bfq-iosched.c 			bfqq->new_ioprio);
bfqq             5007 block/bfq-iosched.c 		bfqq->new_ioprio = IOPRIO_BE_NR;
bfqq             5010 block/bfq-iosched.c 	bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio);
bfqq             5011 block/bfq-iosched.c 	bfqq->entity.prio_changed = 1;
bfqq             5021 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             5033 block/bfq-iosched.c 	bfqq = bic_to_bfqq(bic, false);
bfqq             5034 block/bfq-iosched.c 	if (bfqq) {
bfqq             5035 block/bfq-iosched.c 		bfq_release_process_ref(bfqd, bfqq);
bfqq             5036 block/bfq-iosched.c 		bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic);
bfqq             5037 block/bfq-iosched.c 		bic_set_bfqq(bic, bfqq, false);
bfqq             5040 block/bfq-iosched.c 	bfqq = bic_to_bfqq(bic, true);
bfqq             5041 block/bfq-iosched.c 	if (bfqq)
bfqq             5042 block/bfq-iosched.c 		bfq_set_next_ioprio_data(bfqq, bic);
bfqq             5045 block/bfq-iosched.c static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             5048 block/bfq-iosched.c 	RB_CLEAR_NODE(&bfqq->entity.rb_node);
bfqq             5049 block/bfq-iosched.c 	INIT_LIST_HEAD(&bfqq->fifo);
bfqq             5050 block/bfq-iosched.c 	INIT_HLIST_NODE(&bfqq->burst_list_node);
bfqq             5051 block/bfq-iosched.c 	INIT_HLIST_NODE(&bfqq->woken_list_node);
bfqq             5052 block/bfq-iosched.c 	INIT_HLIST_HEAD(&bfqq->woken_list);
bfqq             5054 block/bfq-iosched.c 	bfqq->ref = 0;
bfqq             5055 block/bfq-iosched.c 	bfqq->bfqd = bfqd;
bfqq             5058 block/bfq-iosched.c 		bfq_set_next_ioprio_data(bfqq, bic);
bfqq             5066 block/bfq-iosched.c 		if (!bfq_class_idle(bfqq))
bfqq             5068 block/bfq-iosched.c 			bfq_mark_bfqq_has_short_ttime(bfqq);
bfqq             5069 block/bfq-iosched.c 		bfq_mark_bfqq_sync(bfqq);
bfqq             5070 block/bfq-iosched.c 		bfq_mark_bfqq_just_created(bfqq);
bfqq             5072 block/bfq-iosched.c 		bfq_clear_bfqq_sync(bfqq);
bfqq             5075 block/bfq-iosched.c 	bfqq->ttime.last_end_request = ktime_get_ns() + 1;
bfqq             5077 block/bfq-iosched.c 	bfq_mark_bfqq_IO_bound(bfqq);
bfqq             5079 block/bfq-iosched.c 	bfqq->pid = pid;
bfqq             5082 block/bfq-iosched.c 	bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3;
bfqq             5083 block/bfq-iosched.c 	bfqq->budget_timeout = bfq_smallest_from_now();
bfqq             5085 block/bfq-iosched.c 	bfqq->wr_coeff = 1;
bfqq             5086 block/bfq-iosched.c 	bfqq->last_wr_start_finish = jiffies;
bfqq             5087 block/bfq-iosched.c 	bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now();
bfqq             5088 block/bfq-iosched.c 	bfqq->split_time = bfq_smallest_from_now();
bfqq             5099 block/bfq-iosched.c 	bfqq->soft_rt_next_start = jiffies;
bfqq             5102 block/bfq-iosched.c 	bfqq->seek_history = 1;
bfqq             5131 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             5138 block/bfq-iosched.c 		bfqq = &bfqd->oom_bfqq;
bfqq             5145 block/bfq-iosched.c 		bfqq = *async_bfqq;
bfqq             5146 block/bfq-iosched.c 		if (bfqq)
bfqq             5150 block/bfq-iosched.c 	bfqq = kmem_cache_alloc_node(bfq_pool,
bfqq             5154 block/bfq-iosched.c 	if (bfqq) {
bfqq             5155 block/bfq-iosched.c 		bfq_init_bfqq(bfqd, bfqq, bic, current->pid,
bfqq             5157 block/bfq-iosched.c 		bfq_init_entity(&bfqq->entity, bfqg);
bfqq             5158 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "allocated");
bfqq             5160 block/bfq-iosched.c 		bfqq = &bfqd->oom_bfqq;
bfqq             5161 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "using oom bfqq");
bfqq             5170 block/bfq-iosched.c 		bfqq->ref++; /*
bfqq             5177 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d",
bfqq             5178 block/bfq-iosched.c 			     bfqq, bfqq->ref);
bfqq             5179 block/bfq-iosched.c 		*async_bfqq = bfqq;
bfqq             5183 block/bfq-iosched.c 	bfqq->ref++; /* get a process reference to this queue */
bfqq             5184 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref);
bfqq             5186 block/bfq-iosched.c 	return bfqq;
bfqq             5190 block/bfq-iosched.c 				    struct bfq_queue *bfqq)
bfqq             5192 block/bfq-iosched.c 	struct bfq_ttime *ttime = &bfqq->ttime;
bfqq             5193 block/bfq-iosched.c 	u64 elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;
bfqq             5197 block/bfq-iosched.c 	ttime->ttime_samples = (7*bfqq->ttime.ttime_samples + 256) / 8;
bfqq             5204 block/bfq-iosched.c bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             5207 block/bfq-iosched.c 	bfqq->seek_history <<= 1;
bfqq             5208 block/bfq-iosched.c 	bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq);
bfqq             5210 block/bfq-iosched.c 	if (bfqq->wr_coeff > 1 &&
bfqq             5211 block/bfq-iosched.c 	    bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
bfqq             5212 block/bfq-iosched.c 	    BFQQ_TOTALLY_SEEKY(bfqq))
bfqq             5213 block/bfq-iosched.c 		bfq_bfqq_end_wr(bfqq);
bfqq             5217 block/bfq-iosched.c 				       struct bfq_queue *bfqq,
bfqq             5227 block/bfq-iosched.c 	if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq) ||
bfqq             5232 block/bfq-iosched.c 	if (time_is_after_eq_jiffies(bfqq->split_time +
bfqq             5241 block/bfq-iosched.c 	    (bfq_sample_valid(bfqq->ttime.ttime_samples) &&
bfqq             5242 block/bfq-iosched.c 	     bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle))
bfqq             5245 block/bfq-iosched.c 	state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq);
bfqq             5248 block/bfq-iosched.c 		bfq_mark_bfqq_has_short_ttime(bfqq);
bfqq             5250 block/bfq-iosched.c 		bfq_clear_bfqq_has_short_ttime(bfqq);
bfqq             5336 block/bfq-iosched.c 	if (state_changed && bfqq->last_serv_time_ns == 0 &&
bfqq             5337 block/bfq-iosched.c 	    (time_is_before_eq_jiffies(bfqq->decrease_time_jif +
bfqq             5340 block/bfq-iosched.c 		bfq_reset_inject_limit(bfqd, bfqq);
bfqq             5347 block/bfq-iosched.c static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             5351 block/bfq-iosched.c 		bfqq->meta_pending++;
bfqq             5353 block/bfq-iosched.c 	bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
bfqq             5355 block/bfq-iosched.c 	if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) {
bfqq             5356 block/bfq-iosched.c 		bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 &&
bfqq             5358 block/bfq-iosched.c 		bool budget_timeout = bfq_bfqq_budget_timeout(bfqq);
bfqq             5376 block/bfq-iosched.c 		if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) &&
bfqq             5387 block/bfq-iosched.c 		bfq_clear_bfqq_wait_request(bfqq);
bfqq             5398 block/bfq-iosched.c 			bfq_bfqq_expire(bfqd, bfqq, false,
bfqq             5406 block/bfq-iosched.c 	struct bfq_queue *bfqq = RQ_BFQQ(rq),
bfqq             5407 block/bfq-iosched.c 		*new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true);
bfqq             5416 block/bfq-iosched.c 		bfqq->allocated--;
bfqq             5426 block/bfq-iosched.c 		if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq)
bfqq             5428 block/bfq-iosched.c 					bfqq, new_bfqq);
bfqq             5430 block/bfq-iosched.c 		bfq_clear_bfqq_just_created(bfqq);
bfqq             5435 block/bfq-iosched.c 		bfq_put_queue(bfqq);
bfqq             5437 block/bfq-iosched.c 		bfqq = new_bfqq;
bfqq             5440 block/bfq-iosched.c 	bfq_update_io_thinktime(bfqd, bfqq);
bfqq             5441 block/bfq-iosched.c 	bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq));
bfqq             5442 block/bfq-iosched.c 	bfq_update_io_seektime(bfqd, bfqq, rq);
bfqq             5444 block/bfq-iosched.c 	waiting = bfqq && bfq_bfqq_wait_request(bfqq);
bfqq             5446 block/bfq-iosched.c 	idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq);
bfqq             5449 block/bfq-iosched.c 	list_add_tail(&rq->queuelist, &bfqq->fifo);
bfqq             5451 block/bfq-iosched.c 	bfq_rq_enqueued(bfqd, bfqq, rq);
bfqq             5458 block/bfq-iosched.c 				    struct bfq_queue *bfqq,
bfqq             5462 block/bfq-iosched.c 	if (!bfqq)
bfqq             5476 block/bfq-iosched.c 	bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags);
bfqq             5478 block/bfq-iosched.c 		bfqg_stats_update_idle_time(bfqq_group(bfqq));
bfqq             5483 block/bfq-iosched.c 					   struct bfq_queue *bfqq,
bfqq             5493 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             5508 block/bfq-iosched.c 	bfqq = bfq_init_rq(rq);
bfqq             5509 block/bfq-iosched.c 	if (!bfqq || at_head || blk_rq_is_passthrough(rq)) {
bfqq             5521 block/bfq-iosched.c 		bfqq = RQ_BFQQ(rq);
bfqq             5539 block/bfq-iosched.c 	bfq_update_insert_stats(q, bfqq, idle_timer_disabled,
bfqq             5557 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfqd->in_service_queue;
bfqq             5579 block/bfq-iosched.c 	if (bfqq && bfq_bfqq_has_short_ttime(bfqq) &&
bfqq             5580 block/bfq-iosched.c 	    bfqq->dispatched + bfqq->queued[0] + bfqq->queued[1] <
bfqq             5596 block/bfq-iosched.c static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd)
bfqq             5604 block/bfq-iosched.c 	bfqq->dispatched--;
bfqq             5606 block/bfq-iosched.c 	if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) {
bfqq             5613 block/bfq-iosched.c 		bfqq->budget_timeout = jiffies;
bfqq             5615 block/bfq-iosched.c 		bfq_weights_tree_remove(bfqd, bfqq);
bfqq             5620 block/bfq-iosched.c 	bfqq->ttime.last_end_request = now_ns;
bfqq             5649 block/bfq-iosched.c 	bfqd->last_completed_rq_bfqq = bfqq;
bfqq             5662 block/bfq-iosched.c 	if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 &&
bfqq             5663 block/bfq-iosched.c 	    RB_EMPTY_ROOT(&bfqq->sort_list) &&
bfqq             5664 block/bfq-iosched.c 	    bfqq->wr_coeff != bfqd->bfq_wr_coeff)
bfqq             5665 block/bfq-iosched.c 		bfqq->soft_rt_next_start =
bfqq             5666 block/bfq-iosched.c 			bfq_bfqq_softrt_next_start(bfqd, bfqq);
bfqq             5672 block/bfq-iosched.c 	if (bfqd->in_service_queue == bfqq) {
bfqq             5673 block/bfq-iosched.c 		if (bfq_bfqq_must_idle(bfqq)) {
bfqq             5674 block/bfq-iosched.c 			if (bfqq->dispatched == 0)
bfqq             5700 block/bfq-iosched.c 		} else if (bfq_may_expire_for_budg_timeout(bfqq))
bfqq             5701 block/bfq-iosched.c 			bfq_bfqq_expire(bfqd, bfqq, false,
bfqq             5703 block/bfq-iosched.c 		else if (RB_EMPTY_ROOT(&bfqq->sort_list) &&
bfqq             5704 block/bfq-iosched.c 			 (bfqq->dispatched == 0 ||
bfqq             5705 block/bfq-iosched.c 			  !bfq_better_to_idle(bfqq)))
bfqq             5706 block/bfq-iosched.c 			bfq_bfqq_expire(bfqd, bfqq, false,
bfqq             5714 block/bfq-iosched.c static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq)
bfqq             5716 block/bfq-iosched.c 	bfqq->allocated--;
bfqq             5718 block/bfq-iosched.c 	bfq_put_queue(bfqq);
bfqq             5826 block/bfq-iosched.c 				    struct bfq_queue *bfqq)
bfqq             5829 block/bfq-iosched.c 	unsigned int old_limit = bfqq->inject_limit;
bfqq             5831 block/bfq-iosched.c 	if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) {
bfqq             5832 block/bfq-iosched.c 		u64 threshold = (bfqq->last_serv_time_ns * 3)>>1;
bfqq             5835 block/bfq-iosched.c 			bfqq->inject_limit--;
bfqq             5836 block/bfq-iosched.c 			bfqq->decrease_time_jif = jiffies;
bfqq             5839 block/bfq-iosched.c 			bfqq->inject_limit++;
bfqq             5854 block/bfq-iosched.c 	if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) ||
bfqq             5855 block/bfq-iosched.c 	    tot_time_ns < bfqq->last_serv_time_ns) {
bfqq             5856 block/bfq-iosched.c 		if (bfqq->last_serv_time_ns == 0) {
bfqq             5861 block/bfq-iosched.c 			bfqq->inject_limit = max_t(unsigned int, 1, old_limit);
bfqq             5863 block/bfq-iosched.c 		bfqq->last_serv_time_ns = tot_time_ns;
bfqq             5874 block/bfq-iosched.c 		bfqq->last_serv_time_ns = tot_time_ns;
bfqq             5890 block/bfq-iosched.c 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
bfqq             5910 block/bfq-iosched.c 	if (!rq->elv.icq || !bfqq)
bfqq             5913 block/bfq-iosched.c 	bfqd = bfqq->bfqd;
bfqq             5916 block/bfq-iosched.c 		bfqg_stats_update_completion(bfqq_group(bfqq),
bfqq             5927 block/bfq-iosched.c 			bfq_update_inject_limit(bfqd, bfqq);
bfqq             5929 block/bfq-iosched.c 		bfq_completed_request(bfqq, bfqd);
bfqq             5930 block/bfq-iosched.c 		bfq_finish_requeue_request_body(bfqq);
bfqq             5949 block/bfq-iosched.c 			bfqg_stats_update_io_remove(bfqq_group(bfqq),
bfqq             5952 block/bfq-iosched.c 		bfq_finish_requeue_request_body(bfqq);
bfqq             5981 block/bfq-iosched.c bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)
bfqq             5983 block/bfq-iosched.c 	bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue");
bfqq             5985 block/bfq-iosched.c 	if (bfqq_process_refs(bfqq) == 1) {
bfqq             5986 block/bfq-iosched.c 		bfqq->pid = current->pid;
bfqq             5987 block/bfq-iosched.c 		bfq_clear_bfqq_coop(bfqq);
bfqq             5988 block/bfq-iosched.c 		bfq_clear_bfqq_split_coop(bfqq);
bfqq             5989 block/bfq-iosched.c 		return bfqq;
bfqq             5994 block/bfq-iosched.c 	bfq_put_cooperator(bfqq);
bfqq             5996 block/bfq-iosched.c 	bfq_release_process_ref(bfqq->bfqd, bfqq);
bfqq             6006 block/bfq-iosched.c 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);
bfqq             6008 block/bfq-iosched.c 	if (likely(bfqq && bfqq != &bfqd->oom_bfqq))
bfqq             6009 block/bfq-iosched.c 		return bfqq;
bfqq             6014 block/bfq-iosched.c 	if (bfqq)
bfqq             6015 block/bfq-iosched.c 		bfq_put_queue(bfqq);
bfqq             6016 block/bfq-iosched.c 	bfqq = bfq_get_queue(bfqd, bio, is_sync, bic);
bfqq             6018 block/bfq-iosched.c 	bic_set_bfqq(bic, bfqq, is_sync);
bfqq             6022 block/bfq-iosched.c 			bfq_mark_bfqq_in_large_burst(bfqq);
bfqq             6024 block/bfq-iosched.c 			bfq_clear_bfqq_in_large_burst(bfqq);
bfqq             6054 block/bfq-iosched.c 				hlist_add_head(&bfqq->burst_list_node,
bfqq             6057 block/bfq-iosched.c 		bfqq->split_time = jiffies;
bfqq             6060 block/bfq-iosched.c 	return bfqq;
bfqq             6109 block/bfq-iosched.c 	struct bfq_queue *bfqq;
bfqq             6132 block/bfq-iosched.c 	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,
bfqq             6137 block/bfq-iosched.c 		if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) {
bfqq             6138 block/bfq-iosched.c 			bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq");
bfqq             6141 block/bfq-iosched.c 			if (bfq_bfqq_in_large_burst(bfqq))
bfqq             6144 block/bfq-iosched.c 			bfqq = bfq_split_bfqq(bic, bfqq);
bfqq             6147 block/bfq-iosched.c 			if (!bfqq)
bfqq             6148 block/bfq-iosched.c 				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,
bfqq             6156 block/bfq-iosched.c 	bfqq->allocated++;
bfqq             6157 block/bfq-iosched.c 	bfqq->ref++;
bfqq             6158 block/bfq-iosched.c 	bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d",
bfqq             6159 block/bfq-iosched.c 		     rq, bfqq, bfqq->ref);
bfqq             6162 block/bfq-iosched.c 	rq->elv.priv[1] = bfqq;
bfqq             6170 block/bfq-iosched.c 	if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) {
bfqq             6171 block/bfq-iosched.c 		bfqq->bic = bic;
bfqq             6178 block/bfq-iosched.c 			bfq_bfqq_resume_state(bfqq, bfqd, bic,
bfqq             6203 block/bfq-iosched.c 	if (unlikely(bfq_bfqq_just_created(bfqq) &&
bfqq             6206 block/bfq-iosched.c 		bfq_handle_burst(bfqd, bfqq);
bfqq             6208 block/bfq-iosched.c 	return bfqq;
bfqq             6212 block/bfq-iosched.c bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             6226 block/bfq-iosched.c 	if (bfqq != bfqd->in_service_queue) {
bfqq             6231 block/bfq-iosched.c 	bfq_clear_bfqq_wait_request(bfqq);
bfqq             6233 block/bfq-iosched.c 	if (bfq_bfqq_budget_timeout(bfqq))
bfqq             6240 block/bfq-iosched.c 	else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0)
bfqq             6251 block/bfq-iosched.c 	bfq_bfqq_expire(bfqd, bfqq, true, reason);
bfqq             6266 block/bfq-iosched.c 	struct bfq_queue *bfqq = bfqd->in_service_queue;
bfqq             6276 block/bfq-iosched.c 	if (bfqq)
bfqq             6277 block/bfq-iosched.c 		bfq_idle_slice_timer_body(bfqd, bfqq);
bfqq             6285 block/bfq-iosched.c 	struct bfq_queue *bfqq = *bfqq_ptr;
bfqq             6287 block/bfq-iosched.c 	bfq_log(bfqd, "put_async_bfqq: %p", bfqq);
bfqq             6288 block/bfq-iosched.c 	if (bfqq) {
bfqq             6289 block/bfq-iosched.c 		bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);
bfqq             6291 block/bfq-iosched.c 		bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d",
bfqq             6292 block/bfq-iosched.c 			     bfqq, bfqq->ref);
bfqq             6293 block/bfq-iosched.c 		bfq_put_queue(bfqq);
bfqq             6381 block/bfq-iosched.c 	struct bfq_queue *bfqq, *n;
bfqq             6386 block/bfq-iosched.c 	list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list)
bfqq             6387 block/bfq-iosched.c 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
bfqq              390 block/bfq-iosched.h 	struct bfq_queue *bfqq[2];
bfqq              775 block/bfq-iosched.h void bfq_mark_bfqq_##name(struct bfq_queue *bfqq);			\
bfqq              776 block/bfq-iosched.h void bfq_clear_bfqq_##name(struct bfq_queue *bfqq);			\
bfqq              777 block/bfq-iosched.h int bfq_bfqq_##name(const struct bfq_queue *bfqq);
bfqq              939 block/bfq-iosched.h void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync);
bfqq              941 block/bfq-iosched.h void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq);
bfqq              942 block/bfq-iosched.h void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq              945 block/bfq-iosched.h 			       struct bfq_queue *bfqq,
bfqq              948 block/bfq-iosched.h 			     struct bfq_queue *bfqq);
bfqq              949 block/bfq-iosched.h void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq              951 block/bfq-iosched.h void bfq_put_queue(struct bfq_queue *bfqq);
bfqq              953 block/bfq-iosched.h void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq);
bfqq              961 block/bfq-iosched.h void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
bfqq              972 block/bfq-iosched.h void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq              981 block/bfq-iosched.h struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
bfqq             1023 block/bfq-iosched.h struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq);
bfqq             1035 block/bfq-iosched.h void bfq_bfqq_served(struct bfq_queue *bfqq, int served);
bfqq             1036 block/bfq-iosched.h void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1043 block/bfq-iosched.h void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1045 block/bfq-iosched.h void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq);
bfqq             1046 block/bfq-iosched.h void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1048 block/bfq-iosched.h void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1050 block/bfq-iosched.h void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq);
bfqq             1064 block/bfq-iosched.h struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
bfqq             1066 block/bfq-iosched.h #define bfq_log_bfqq(bfqd, bfqq, fmt, args...)	do {			\
bfqq             1068 block/bfq-iosched.h 	bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH);	\
bfqq             1070 block/bfq-iosched.h 			bfqg_to_blkg(bfqq_group(bfqq))->blkcg,		\
bfqq             1072 block/bfq-iosched.h 			bfq_bfqq_sync((bfqq)) ? 'S' : 'A', ##args);	\
bfqq             1082 block/bfq-iosched.h #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do {	\
bfqq             1084 block/bfq-iosched.h 	bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH);	\
bfqq             1086 block/bfq-iosched.h 			bfq_bfqq_sync((bfqq)) ? 'S' : 'A',		\
bfqq               32 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq               34 block/bfq-wf2q.c 	return bfqq ? bfqq->ioprio_class - 1 :
bfqq              148 block/bfq-wf2q.c struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq)
bfqq              150 block/bfq-wf2q.c 	struct bfq_entity *group_entity = bfqq->entity.parent;
bfqq              153 block/bfq-wf2q.c 		group_entity = &bfqq->bfqd->root_group->entity;
bfqq              236 block/bfq-wf2q.c struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq)
bfqq              238 block/bfq-wf2q.c 	return bfqq->bfqd->root_group;
bfqq              264 block/bfq-wf2q.c 	struct bfq_queue *bfqq = NULL;
bfqq              267 block/bfq-wf2q.c 		bfqq = container_of(entity, struct bfq_queue, entity);
bfqq              269 block/bfq-wf2q.c 	return bfqq;
bfqq              293 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              298 block/bfq-wf2q.c 	if (bfqq) {
bfqq              299 block/bfq-wf2q.c 		bfq_log_bfqq(bfqq->bfqd, bfqq,
bfqq              302 block/bfq-wf2q.c 		bfq_log_bfqq(bfqq->bfqd, bfqq,
bfqq              347 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              362 block/bfq-wf2q.c 	if (bfqq)
bfqq              363 block/bfq-wf2q.c 		list_del(&bfqq->bfqq_list);
bfqq              478 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              500 block/bfq-wf2q.c 	if (bfqq)
bfqq              501 block/bfq-wf2q.c 		list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list);
bfqq              533 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              535 block/bfq-wf2q.c 	if (bfqq) {
bfqq              536 block/bfq-wf2q.c 		bfqq->ref++;
bfqq              537 block/bfq-wf2q.c 		bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d",
bfqq              538 block/bfq-wf2q.c 			     bfqq, bfqq->ref);
bfqq              582 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              601 block/bfq-wf2q.c 	if (bfqq)
bfqq              602 block/bfq-wf2q.c 		list_del(&bfqq->bfqq_list);
bfqq              617 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              628 block/bfq-wf2q.c 	if (bfqq)
bfqq              629 block/bfq-wf2q.c 		list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list);
bfqq              651 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              658 block/bfq-wf2q.c 	if (bfqq)
bfqq              659 block/bfq-wf2q.c 		bfq_put_queue(bfqq);
bfqq              736 block/bfq-wf2q.c 		struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              745 block/bfq-wf2q.c 		if (bfqq)
bfqq              746 block/bfq-wf2q.c 			bfqd = bfqq->bfqd;
bfqq              770 block/bfq-wf2q.c 			if (bfqq)
bfqq              771 block/bfq-wf2q.c 				bfqq->ioprio =
bfqq              775 block/bfq-wf2q.c 		if (bfqq && update_class_too)
bfqq              776 block/bfq-wf2q.c 			bfqq->ioprio_class = bfqq->new_ioprio_class;
bfqq              782 block/bfq-wf2q.c 		if (!bfqq || bfqq->ioprio_class == bfqq->new_ioprio_class)
bfqq              796 block/bfq-wf2q.c 			     (bfqq ? bfqq->wr_coeff : 1);
bfqq              802 block/bfq-wf2q.c 		if (prev_weight != new_weight && bfqq) {
bfqq              804 block/bfq-wf2q.c 			__bfq_weights_tree_remove(bfqd, bfqq, root);
bfqq              811 block/bfq-wf2q.c 		if (prev_weight != new_weight && bfqq && bfqq->wr_coeff == 1) {
bfqq              813 block/bfq-wf2q.c 			bfq_weights_tree_add(bfqd, bfqq, root);
bfqq              835 block/bfq-wf2q.c void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
bfqq              837 block/bfq-wf2q.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq              840 block/bfq-wf2q.c 	if (!bfqq->service_from_backlogged)
bfqq              841 block/bfq-wf2q.c 		bfqq->first_IO_time = jiffies;
bfqq              843 block/bfq-wf2q.c 	if (bfqq->wr_coeff > 1)
bfqq              844 block/bfq-wf2q.c 		bfqq->service_from_wr += served;
bfqq              846 block/bfq-wf2q.c 	bfqq->service_from_backlogged += served;
bfqq              855 block/bfq-wf2q.c 	bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %d secs", served);
bfqq              885 block/bfq-wf2q.c void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq              888 block/bfq-wf2q.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq              899 block/bfq-wf2q.c 	bfq_bfqq_served(bfqq,
bfqq              907 block/bfq-wf2q.c 	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
bfqq              953 block/bfq-wf2q.c 		if (bfqq)
bfqq              954 block/bfq-wf2q.c 			delta /= bfqq->wr_coeff;
bfqq             1521 block/bfq-wf2q.c 	struct bfq_queue *bfqq;
bfqq             1593 block/bfq-wf2q.c 	bfqq = bfq_entity_to_bfqq(entity);
bfqq             1606 block/bfq-wf2q.c 	return bfqq;
bfqq             1650 block/bfq-wf2q.c void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1653 block/bfq-wf2q.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             1658 block/bfq-wf2q.c void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             1660 block/bfq-wf2q.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             1662 block/bfq-wf2q.c 	bfq_activate_requeue_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq),
bfqq             1664 block/bfq-wf2q.c 	bfq_clear_bfqq_non_blocking_wait_rq(bfqq);
bfqq             1667 block/bfq-wf2q.c void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1670 block/bfq-wf2q.c 	struct bfq_entity *entity = &bfqq->entity;
bfqq             1673 block/bfq-wf2q.c 				    bfqq == bfqd->in_service_queue, expiration);
bfqq             1681 block/bfq-wf2q.c void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
bfqq             1684 block/bfq-wf2q.c 	bfq_log_bfqq(bfqd, bfqq, "del from busy");
bfqq             1686 block/bfq-wf2q.c 	bfq_clear_bfqq_busy(bfqq);
bfqq             1688 block/bfq-wf2q.c 	bfqd->busy_queues[bfqq->ioprio_class - 1]--;
bfqq             1690 block/bfq-wf2q.c 	if (bfqq->wr_coeff > 1)
bfqq             1693 block/bfq-wf2q.c 	bfqg_stats_update_dequeue(bfqq_group(bfqq));
bfqq             1695 block/bfq-wf2q.c 	bfq_deactivate_bfqq(bfqd, bfqq, true, expiration);
bfqq             1697 block/bfq-wf2q.c 	if (!bfqq->dispatched)
bfqq             1698 block/bfq-wf2q.c 		bfq_weights_tree_remove(bfqd, bfqq);
bfqq             1704 block/bfq-wf2q.c void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq)
bfqq             1706 block/bfq-wf2q.c 	bfq_log_bfqq(bfqd, bfqq, "add to busy");
bfqq             1708 block/bfq-wf2q.c 	bfq_activate_bfqq(bfqd, bfqq);
bfqq             1710 block/bfq-wf2q.c 	bfq_mark_bfqq_busy(bfqq);
bfqq             1711 block/bfq-wf2q.c 	bfqd->busy_queues[bfqq->ioprio_class - 1]++;
bfqq             1713 block/bfq-wf2q.c 	if (!bfqq->dispatched)
bfqq             1714 block/bfq-wf2q.c 		if (bfqq->wr_coeff == 1)
bfqq             1715 block/bfq-wf2q.c 			bfq_weights_tree_add(bfqd, bfqq,
bfqq             1718 block/bfq-wf2q.c 	if (bfqq->wr_coeff > 1)