58 #define RQ_CFQQ(rq)		(struct cfq_queue *) ((rq)->elv.priv[0])
102 struct cfq_queue {  struct
152 	struct cfq_queue *new_cfqq;  argument
303 	struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR];
304 	struct cfq_queue *async_idle_cfqq;
310 	struct cfq_queue	*cfqq[2];
368 	struct cfq_queue *active_queue;
390 	struct cfq_queue oom_cfqq;
396 static void cfq_put_queue(struct cfq_queue *cfqq);
428 static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq)		\
432 static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq)	\
436 static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq)		\
823 static inline enum wl_class_t cfqq_class(struct cfq_queue *cfqq)  in cfqq_class()
833 static enum wl_type_t cfqq_type(struct cfq_queue *cfqq)  in cfqq_type()
862 static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync,
879 static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)  in cic_to_cfqq()
884 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,  in cic_set_cfqq()
932 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_to_slice()
1017 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_scaled_cfqq_slice()
1047 cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_set_prio_slice()
1062 static inline bool cfq_slice_used(struct cfq_queue *cfqq)  in cfq_slice_used()
1165 static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root)  in cfq_rb_first()
1175 		return rb_entry(root->left, struct cfq_queue, rb_node);  in cfq_rb_first()
1209 cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_find_next_rq()
1233 				      struct cfq_queue *cfqq)  in cfq_slice_offset()
1426 static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq,  in cfq_cfqq_slice_usage()
1459 				struct cfq_queue *cfqq)  in cfq_group_served()
1688 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)  in cfq_link_cfqq_cfqg()
2188 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {  in cfq_link_cfqq_cfqg()
2199 static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_service_tree_add()
2203 	struct cfq_queue *__cfqq;  in cfq_service_tree_add()
2214 			__cfqq = rb_entry(parent, struct cfq_queue, rb_node);  in cfq_service_tree_add()
2252 		__cfqq = rb_entry(parent, struct cfq_queue, rb_node);  in cfq_service_tree_add()
2277 static struct cfq_queue *
2283 	struct cfq_queue *cfqq = NULL;  in cfq_prio_tree_lookup()
2291 		cfqq = rb_entry(parent, struct cfq_queue, p_node);  in cfq_prio_tree_lookup()
2313 static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_tree_add()
2316 	struct cfq_queue *__cfqq;  in cfq_prio_tree_add()
2341 static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_resort_rr_list()
2356 static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_add_cfqq_rr()
2372 static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_del_cfqq_rr()
2399 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_del_rq_rb()
2422 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_add_rq_rb()
2448 static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)  in cfq_reposition_rq_rb()
2463 	struct cfq_queue *cfqq;  in cfq_find_rq_fmerge()
2499 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_remove_request()
2534 		struct cfq_queue *cfqq = RQ_CFQQ(req);  in cfq_merged_request()
2550 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_merged_requests()
2584 	struct cfq_queue *cfqq;  in cfq_allow_merge()
2604 static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_del_timer()
2611 				   struct cfq_queue *cfqq)  in __cfq_set_active_queue()
2640 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in __cfq_slice_expired()
2689 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_slice_expired()
2699 static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)  in cfq_get_next_queue()
2715 static struct cfq_queue *cfq_get_next_queue_forced(struct cfq_data *cfqd)  in cfq_get_next_queue_forced()
2718 	struct cfq_queue *cfqq;  in cfq_get_next_queue_forced()
2738 static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd,  in cfq_set_active_queue()
2739 					      struct cfq_queue *cfqq)  in cfq_set_active_queue()
2757 static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_rq_close()
2763 static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,  in cfqq_close()
2764 				    struct cfq_queue *cur_cfqq)  in cfqq_close()
2768 	struct cfq_queue *__cfqq;  in cfqq_close()
2786 	__cfqq = rb_entry(parent, struct cfq_queue, p_node);  in cfqq_close()
2797 	__cfqq = rb_entry(node, struct cfq_queue, p_node);  in cfqq_close()
2814 static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,  in cfq_close_cooperator()
2815 					      struct cfq_queue *cur_cfqq)  in cfq_close_cooperator()
2817 	struct cfq_queue *cfqq;  in cfq_close_cooperator()
2866 static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_should_idle()
2899 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_arm_slice_timer()
2973 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_dispatch_insert()
2990 static struct request *cfq_check_fifo(struct cfq_queue *cfqq)  in cfq_check_fifo()
3011 cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_to_maxrq()
3023 static int cfqq_process_refs(struct cfq_queue *cfqq)  in cfqq_process_refs()
3033 static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)  in cfq_setup_merge()
3036 	struct cfq_queue *__cfqq;  in cfq_setup_merge()
3078 	struct cfq_queue *queue;  in cfq_choose_wl_type()
3214 static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)  in cfq_select_queue()
3216 	struct cfq_queue *cfqq, *new_cfqq = NULL;  in cfq_select_queue()
3325 static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq)  in __cfq_forced_dispatch_cfqq()
3347 	struct cfq_queue *cfqq;  in cfq_forced_dispatch()
3364 	struct cfq_queue *cfqq)  in cfq_slice_used_soon()
3376 static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_may_dispatch()
3465 static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_dispatch_request()
3503 	struct cfq_queue *cfqq;  in cfq_dispatch_requests()
3546 static void cfq_put_queue(struct cfq_queue *cfqq)  in cfq_put_queue()
3572 static void cfq_put_cooperator(struct cfq_queue *cfqq)  in cfq_put_cooperator()
3574 	struct cfq_queue *__cfqq, *next;  in cfq_put_cooperator()
3593 static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_exit_cfqq()
3628 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)  in cfq_init_prio_data()
3674 	struct cfq_queue *cfqq;  in check_ioprio_changed()
3697 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_init_cfqq()
3721 	struct cfq_queue *cfqq;  in check_blkcg_changed()
3759 static struct cfq_queue **
3777 static struct cfq_queue *
3783 	struct cfq_queue **async_cfqq = NULL;  in cfq_get_queue()
3784 	struct cfq_queue *cfqq;  in cfq_get_queue()
3841 cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_io_thinktime()
3855 cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_io_seektime()
3879 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_idle_window()
3922 cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq,  in cfq_should_preempt()
3925 	struct cfq_queue *cfqq;  in cfq_should_preempt()
3997 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_preempt_queue()
4028 cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_rq_enqueued()
4080 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_insert_request()
4099 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_update_hw_tag()
4130 static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_should_wait_busy()
4169 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_completed_request()
4255 static inline int __cfq_may_queue(struct cfq_queue *cfqq)  in __cfq_may_queue()
4270 	struct cfq_queue *cfqq;  in cfq_may_queue()
4297 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_put_request()
4314 static struct cfq_queue *
4316 		struct cfq_queue *cfqq)  in cfq_merge_cfqqs()
4329 static struct cfq_queue *
4330 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)  in split_cfqq()
4357 	struct cfq_queue *cfqq;  in cfq_set_request()
4418 	struct cfq_queue *cfqq;  in cfq_idle_slice_timer()
4776 	cfq_pool = KMEM_CACHE(cfq_queue, 0);  in cfq_init()