863 				       struct cfq_io_cq *cic, struct bio *bio);
879 static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)  in cic_to_cfqq()  argument
881 	return cic->cfqq[is_sync];  in cic_to_cfqq()
884 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,  in cic_set_cfqq()  argument
887 	cic->cfqq[is_sync] = cfqq;  in cic_set_cfqq()
890 static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)  in cic_to_cfqd()  argument
892 	return cic->icq.q->elevator->elevator_data;  in cic_to_cfqd()
2462 	struct cfq_io_cq *cic;  in cfq_find_rq_fmerge()  local
2465 	cic = cfq_cic_lookup(cfqd, tsk->io_context);  in cfq_find_rq_fmerge()
2466 	if (!cic)  in cfq_find_rq_fmerge()
2469 	cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));  in cfq_find_rq_fmerge()
2583 	struct cfq_io_cq *cic;  in cfq_allow_merge()  local
2596 	cic = cfq_cic_lookup(cfqd, current->io_context);  in cfq_allow_merge()
2597 	if (!cic)  in cfq_allow_merge()
2600 	cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));  in cfq_allow_merge()
2900 	struct cfq_io_cq *cic;  in cfq_arm_slice_timer()  local
2934 	cic = cfqd->active_cic;  in cfq_arm_slice_timer()
2935 	if (!cic || !atomic_read(&cic->icq.ioc->active_ref))  in cfq_arm_slice_timer()
2943 	if (sample_valid(cic->ttime.ttime_samples) &&  in cfq_arm_slice_timer()
2944 	    (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) {  in cfq_arm_slice_timer()
2946 			     cic->ttime.ttime_mean);  in cfq_arm_slice_timer()
3487 		struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_dispatch_request()  local
3489 		atomic_long_inc(&cic->icq.ioc->refcount);  in cfq_dispatch_request()
3490 		cfqd->active_cic = cic;  in cfq_dispatch_request()
3607 	struct cfq_io_cq *cic = icq_to_cic(icq);  in cfq_init_icq()  local
3609 	cic->ttime.last_end_request = jiffies;  in cfq_init_icq()
3614 	struct cfq_io_cq *cic = icq_to_cic(icq);  in cfq_exit_icq()  local
3615 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in cfq_exit_icq()
3617 	if (cic_to_cfqq(cic, false)) {  in cfq_exit_icq()
3618 		cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, false));  in cfq_exit_icq()
3619 		cic_set_cfqq(cic, NULL, false);  in cfq_exit_icq()
3622 	if (cic_to_cfqq(cic, true)) {  in cfq_exit_icq()
3623 		cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, true));  in cfq_exit_icq()
3624 		cic_set_cfqq(cic, NULL, true);  in cfq_exit_icq()
3628 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)  in cfq_init_prio_data()  argument
3636 	ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);  in cfq_init_prio_data()
3648 		cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_init_prio_data()
3652 		cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_init_prio_data()
3670 static void check_ioprio_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_ioprio_changed()  argument
3672 	int ioprio = cic->icq.ioc->ioprio;  in check_ioprio_changed()
3673 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in check_ioprio_changed()
3680 	if (unlikely(!cfqd) || likely(cic->ioprio == ioprio))  in check_ioprio_changed()
3683 	cfqq = cic_to_cfqq(cic, false);  in check_ioprio_changed()
3686 		cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio);  in check_ioprio_changed()
3687 		cic_set_cfqq(cic, cfqq, false);  in check_ioprio_changed()
3690 	cfqq = cic_to_cfqq(cic, true);  in check_ioprio_changed()
3694 	cic->ioprio = ioprio;  in check_ioprio_changed()
3718 static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_blkcg_changed()  argument
3720 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in check_blkcg_changed()
3732 	if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr))  in check_blkcg_changed()
3739 	cfqq = cic_to_cfqq(cic, false);  in check_blkcg_changed()
3742 		cic_set_cfqq(cic, NULL, false);  in check_blkcg_changed()
3746 	cfqq = cic_to_cfqq(cic, true);  in check_blkcg_changed()
3749 		cic_set_cfqq(cic, NULL, true);  in check_blkcg_changed()
3753 	cic->blkcg_serial_nr = serial_nr;  in check_blkcg_changed()
3756 static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { }  in check_blkcg_changed()  argument
3778 cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,  in cfq_get_queue()  argument
3781 	int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);  in cfq_get_queue()
3782 	int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_get_queue()
3795 		if (!ioprio_valid(cic->ioprio)) {  in cfq_get_queue()
3814 	cfq_init_prio_data(cfqq, cic);  in cfq_get_queue()
3842 			struct cfq_io_cq *cic)  in cfq_update_io_thinktime()  argument
3845 		__cfq_update_io_thinktime(&cic->ttime, cfqd->cfq_slice_idle);  in cfq_update_io_thinktime()
3880 		       struct cfq_io_cq *cic)  in cfq_update_idle_window()  argument
3897 	else if (!atomic_read(&cic->icq.ioc->active_ref) ||  in cfq_update_idle_window()
3901 	else if (sample_valid(cic->ttime.ttime_samples)) {  in cfq_update_idle_window()
3902 		if (cic->ttime.ttime_mean > cfqd->cfq_slice_idle)  in cfq_update_idle_window()
4031 	struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_rq_enqueued()  local
4037 	cfq_update_io_thinktime(cfqd, cfqq, cic);  in cfq_rq_enqueued()
4039 	cfq_update_idle_window(cfqd, cfqq, cic);  in cfq_rq_enqueued()
4132 	struct cfq_io_cq *cic = cfqd->active_cic;  in cfq_should_wait_busy()  local
4150 	if (cic && sample_valid(cic->ttime.ttime_samples)  in cfq_should_wait_busy()
4151 	    && (cfqq->slice_end - jiffies < cic->ttime.ttime_mean))  in cfq_should_wait_busy()
4269 	struct cfq_io_cq *cic;  in cfq_may_queue()  local
4278 	cic = cfq_cic_lookup(cfqd, tsk->io_context);  in cfq_may_queue()
4279 	if (!cic)  in cfq_may_queue()
4282 	cfqq = cic_to_cfqq(cic, rw_is_sync(rw));  in cfq_may_queue()
4284 		cfq_init_prio_data(cfqq, cic);  in cfq_may_queue()
4315 cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_cq *cic,  in cfq_merge_cfqqs()  argument
4319 	cic_set_cfqq(cic, cfqq->new_cfqq, 1);  in cfq_merge_cfqqs()
4322 	return cic_to_cfqq(cic, 1);  in cfq_merge_cfqqs()
4330 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)  in split_cfqq()  argument
4339 	cic_set_cfqq(cic, NULL, 1);  in split_cfqq()
4354 	struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);  in cfq_set_request()  local
4361 	check_ioprio_changed(cic, bio);  in cfq_set_request()
4362 	check_blkcg_changed(cic, bio);  in cfq_set_request()
4364 	cfqq = cic_to_cfqq(cic, is_sync);  in cfq_set_request()
4368 		cfqq = cfq_get_queue(cfqd, is_sync, cic, bio);  in cfq_set_request()
4369 		cic_set_cfqq(cic, cfqq, is_sync);  in cfq_set_request()
4376 			cfqq = split_cfqq(cic, cfqq);  in cfq_set_request()
4388 			cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq);  in cfq_set_request()