58 #define RQ_CFQQ(rq)		(struct cfq_queue *) ((rq)->elv.priv[0])
97 struct cfq_queue {  struct
147 	struct cfq_queue *new_cfqq;  argument
298 	struct cfq_queue	*cfqq[2];
356 	struct cfq_queue *active_queue;
362 	struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR];
363 	struct cfq_queue *async_idle_cfqq;
384 	struct cfq_queue oom_cfqq;
421 static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq)		\
425 static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq)	\
429 static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq)		\
821 static inline enum wl_class_t cfqq_class(struct cfq_queue *cfqq)  in cfqq_class()
831 static enum wl_type_t cfqq_type(struct cfq_queue *cfqq)  in cfqq_type()
860 static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync,
878 static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)  in cic_to_cfqq()
883 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,  in cic_set_cfqq()
931 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_to_slice()
1016 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_scaled_cfqq_slice()
1046 cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_set_prio_slice()
1061 static inline bool cfq_slice_used(struct cfq_queue *cfqq)  in cfq_slice_used()
1164 static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root)  in cfq_rb_first()
1174 		return rb_entry(root->left, struct cfq_queue, rb_node);  in cfq_rb_first()
1208 cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_find_next_rq()
1232 				      struct cfq_queue *cfqq)  in cfq_slice_offset()
1425 static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq,  in cfq_cfqq_slice_usage()
1458 				struct cfq_queue *cfqq)  in cfq_group_served()
1627 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)  in cfq_link_cfqq_cfqg()
2011 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {  in cfq_link_cfqq_cfqg()
2022 static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_service_tree_add()
2026 	struct cfq_queue *__cfqq;  in cfq_service_tree_add()
2037 			__cfqq = rb_entry(parent, struct cfq_queue, rb_node);  in cfq_service_tree_add()
2075 		__cfqq = rb_entry(parent, struct cfq_queue, rb_node);  in cfq_service_tree_add()
2100 static struct cfq_queue *
2106 	struct cfq_queue *cfqq = NULL;  in cfq_prio_tree_lookup()
2114 		cfqq = rb_entry(parent, struct cfq_queue, p_node);  in cfq_prio_tree_lookup()
2136 static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_tree_add()
2139 	struct cfq_queue *__cfqq;  in cfq_prio_tree_add()
2164 static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_resort_rr_list()
2179 static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_add_cfqq_rr()
2195 static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_del_cfqq_rr()
2222 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_del_rq_rb()
2245 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_add_rq_rb()
2271 static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)  in cfq_reposition_rq_rb()
2286 	struct cfq_queue *cfqq;  in cfq_find_rq_fmerge()
2322 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_remove_request()
2357 		struct cfq_queue *cfqq = RQ_CFQQ(req);  in cfq_merged_request()
2373 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_merged_requests()
2407 	struct cfq_queue *cfqq;  in cfq_allow_merge()
2427 static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_del_timer()
2434 				   struct cfq_queue *cfqq)  in __cfq_set_active_queue()
2463 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in __cfq_slice_expired()
2512 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_slice_expired()
2522 static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)  in cfq_get_next_queue()
2538 static struct cfq_queue *cfq_get_next_queue_forced(struct cfq_data *cfqd)  in cfq_get_next_queue_forced()
2541 	struct cfq_queue *cfqq;  in cfq_get_next_queue_forced()
2561 static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd,  in cfq_set_active_queue()
2562 					      struct cfq_queue *cfqq)  in cfq_set_active_queue()
2580 static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_rq_close()
2586 static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,  in cfqq_close()
2587 				    struct cfq_queue *cur_cfqq)  in cfqq_close()
2591 	struct cfq_queue *__cfqq;  in cfqq_close()
2609 	__cfqq = rb_entry(parent, struct cfq_queue, p_node);  in cfqq_close()
2620 	__cfqq = rb_entry(node, struct cfq_queue, p_node);  in cfqq_close()
2637 static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,  in cfq_close_cooperator()
2638 					      struct cfq_queue *cur_cfqq)  in cfq_close_cooperator()
2640 	struct cfq_queue *cfqq;  in cfq_close_cooperator()
2689 static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_should_idle()
2722 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_arm_slice_timer()
2796 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_dispatch_insert()
2814 static struct request *cfq_check_fifo(struct cfq_queue *cfqq)  in cfq_check_fifo()
2835 cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_to_maxrq()
2847 static int cfqq_process_refs(struct cfq_queue *cfqq)  in cfqq_process_refs()
2857 static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)  in cfq_setup_merge()
2860 	struct cfq_queue *__cfqq;  in cfq_setup_merge()
2902 	struct cfq_queue *queue;  in cfq_choose_wl_type()
3038 static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)  in cfq_select_queue()
3040 	struct cfq_queue *cfqq, *new_cfqq = NULL;  in cfq_select_queue()
3149 static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq)  in __cfq_forced_dispatch_cfqq()
3171 	struct cfq_queue *cfqq;  in cfq_forced_dispatch()
3188 	struct cfq_queue *cfqq)  in cfq_slice_used_soon()
3200 static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_may_dispatch()
3289 static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_dispatch_request()
3327 	struct cfq_queue *cfqq;  in cfq_dispatch_requests()
3370 static void cfq_put_queue(struct cfq_queue *cfqq)  in cfq_put_queue()
3396 static void cfq_put_cooperator(struct cfq_queue *cfqq)  in cfq_put_cooperator()
3398 	struct cfq_queue *__cfqq, *next;  in cfq_put_cooperator()
3417 static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_exit_cfqq()
3452 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)  in cfq_init_prio_data()
3498 	struct cfq_queue *cfqq;  in check_ioprio_changed()
3509 		struct cfq_queue *new_cfqq;  in check_ioprio_changed()
3525 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_init_cfqq()
3549 	struct cfq_queue *sync_cfqq;  in check_blkcg_changed()
3580 static struct cfq_queue *
3585 	struct cfq_queue *cfqq, *new_cfqq = NULL;  in cfq_find_alloc_queue()
3642 static struct cfq_queue **
3660 static struct cfq_queue *
3666 	struct cfq_queue **async_cfqq = NULL;  in cfq_get_queue()
3667 	struct cfq_queue *cfqq = NULL;  in cfq_get_queue()
3706 cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_io_thinktime()
3720 cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_io_seektime()
3744 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_idle_window()
3787 cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq,  in cfq_should_preempt()
3790 	struct cfq_queue *cfqq;  in cfq_should_preempt()
3862 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_preempt_queue()
3893 cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_rq_enqueued()
3945 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_insert_request()
3964 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_update_hw_tag()
3995 static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_should_wait_busy()
4034 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_completed_request()
4120 static inline int __cfq_may_queue(struct cfq_queue *cfqq)  in __cfq_may_queue()
4135 	struct cfq_queue *cfqq;  in cfq_may_queue()
4162 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_put_request()
4179 static struct cfq_queue *
4181 		struct cfq_queue *cfqq)  in cfq_merge_cfqqs()
4194 static struct cfq_queue *
4195 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)  in split_cfqq()
4222 	struct cfq_queue *cfqq;  in cfq_set_request()
4283 	struct cfq_queue *cfqq;  in cfq_idle_slice_timer()
4638 	cfq_pool = KMEM_CACHE(cfq_queue, 0);  in cfq_init()