59 #define RQ_CFQG(rq)		(struct cfq_group *) ((rq)->elv.priv[1])
68 #define rb_entry_cfqg(node)	rb_entry((node), struct cfq_group, rb_node)
153 	struct cfq_group *cfqg;
224 struct cfq_group {  struct
325 	struct cfq_group *root_group;  argument
333 	struct cfq_group *serving_group;
395 static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd);
398 static struct cfq_rb_root *st_for(struct cfq_group *cfqg,  in st_for()
500 static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,  in cfqg_stats_set_start_group_wait_time()
501 						 struct cfq_group *curr_cfqg)  in cfqg_stats_set_start_group_wait_time()
528 static void cfqg_stats_update_dequeue(struct cfq_group *cfqg)  in cfqg_stats_update_dequeue()
533 static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)  in cfqg_stats_set_start_empty_time()
552 static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)  in cfqg_stats_update_idle_time()
566 static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)  in cfqg_stats_set_start_idle_time()
576 static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)  in cfqg_stats_update_avg_queue_size()
588 static inline void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg, struct cfq_group *c…  in cfqg_stats_set_start_group_wait_time()
590 static inline void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }  in cfqg_stats_update_dequeue()
591 static inline void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }  in cfqg_stats_set_start_empty_time()
592 static inline void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }  in cfqg_stats_update_idle_time()
593 static inline void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }  in cfqg_stats_set_start_idle_time()
594 static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }  in cfqg_stats_update_avg_queue_size()
600 static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)  in pd_to_cfqg()
602 	return pd ? container_of(pd, struct cfq_group, pd) : NULL;  in pd_to_cfqg()
611 static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)  in cfqg_to_blkg()
618 static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)  in blkg_to_cfqg()
628 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)  in cfqg_parent()
635 static inline void cfqg_get(struct cfq_group *cfqg)  in cfqg_get()
640 static inline void cfqg_put(struct cfq_group *cfqg)  in cfqg_put()
662 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,  in cfqg_stats_update_io_add()
663 					    struct cfq_group *curr_cfqg, int rw)  in cfqg_stats_update_io_add()
670 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,  in cfqg_stats_update_timeslice_used()
679 static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw)  in cfqg_stats_update_io_remove()
684 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw)  in cfqg_stats_update_io_merged()
689 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,  in cfqg_stats_update_completion()
745 static void cfqg_stats_xfer_dead(struct cfq_group *cfqg)  in cfqg_stats_xfer_dead()
747 	struct cfq_group *parent = cfqg_parent(cfqg);  in cfqg_stats_xfer_dead()
760 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg) { return NULL; }  in cfqg_parent()
761 static inline void cfqg_get(struct cfq_group *cfqg) { }  in cfqg_get()
762 static inline void cfqg_put(struct cfq_group *cfqg) { }  in cfqg_put()
771 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,  in cfqg_stats_update_io_add()
772 			struct cfq_group *curr_cfqg, int rw) { }  in cfqg_stats_update_io_add()
773 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,  in cfqg_stats_update_timeslice_used()
775 static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { }  in cfqg_stats_update_io_remove()
776 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { }  in cfqg_stats_update_io_merged()
777 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,  in cfqg_stats_update_completion()
844 					struct cfq_group *cfqg)  in cfq_group_busy_queues_wl()
855 					struct cfq_group *cfqg)  in cfqg_busy_async_queues()
980 	struct cfq_group *cfqg;  in update_min_vdisktime()
996 					struct cfq_group *cfqg, bool rt)  in cfq_group_get_avg_queues()
1011 cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_slice()
1180 static struct cfq_group *cfq_rb_first_group(struct cfq_rb_root *root)  in cfq_rb_first_group()
1243 cfqg_key(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfqg_key()
1249 __cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)  in __cfq_group_service_tree_add()
1253 	struct cfq_group *__cfqg;  in __cfq_group_service_tree_add()
1280 cfq_update_group_weight(struct cfq_group *cfqg)  in cfq_update_group_weight()
1289 cfq_update_group_leaf_weight(struct cfq_group *cfqg)  in cfq_update_group_leaf_weight()
1300 cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfq_group_service_tree_add()
1303 	struct cfq_group *pos = cfqg;  in cfq_group_service_tree_add()
1304 	struct cfq_group *parent;  in cfq_group_service_tree_add()
1351 cfq_group_notify_queue_add(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_notify_queue_add()
1354 	struct cfq_group *__cfqg;  in cfq_group_notify_queue_add()
1376 cfq_group_service_tree_del(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfq_group_service_tree_del()
1378 	struct cfq_group *pos = cfqg;  in cfq_group_service_tree_del()
1389 		struct cfq_group *parent = cfqg_parent(pos);  in cfq_group_service_tree_del()
1409 cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_notify_queue_del()
1458 static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,  in cfq_group_served()
1512 static void cfq_init_cfqg_base(struct cfq_group *cfqg)  in cfq_init_cfqg_base()
1614 	struct cfq_group *cfqg;  in cfq_pd_alloc()
1631 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_init()
1640 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_offline()
1664 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_free()
1672 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_reset_stats()
1677 static struct cfq_group *cfq_lookup_cfqg(struct cfq_data *cfqd,  in cfq_lookup_cfqg()
1688 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)  in cfq_link_cfqq_cfqg()
1698 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_weight_device()
1716 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_leaf_weight_device()
1765 	struct cfq_group *cfqg;  in __cfqg_set_weight_device()
1843 		struct cfq_group *cfqg = blkg_to_cfqg(blkg);  in __cfq_set_weight()
1962 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_avg_queue_size()
2027 		.private = offsetof(struct cfq_group, stats.time),
2046 		.private = offsetof(struct cfq_group, stats.service_time),
2051 		.private = offsetof(struct cfq_group, stats.wait_time),
2056 		.private = offsetof(struct cfq_group, stats.merged),
2061 		.private = offsetof(struct cfq_group, stats.queued),
2068 		.private = offsetof(struct cfq_group, stats.time),
2087 		.private = offsetof(struct cfq_group, stats.service_time),
2092 		.private = offsetof(struct cfq_group, stats.wait_time),
2097 		.private = offsetof(struct cfq_group, stats.merged),
2102 		.private = offsetof(struct cfq_group, stats.queued),
2112 		.private = offsetof(struct cfq_group, stats.group_wait_time),
2117 		.private = offsetof(struct cfq_group, stats.idle_time),
2122 		.private = offsetof(struct cfq_group, stats.empty_time),
2127 		.private = offsetof(struct cfq_group, stats.dequeue),
2132 		.private = offsetof(struct cfq_group, stats.unaccounted_time),
2181 static struct cfq_group *cfq_lookup_cfqg(struct cfq_data *cfqd,  in cfq_lookup_cfqg()
2188 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {  in cfq_link_cfqq_cfqg()
2717 	struct cfq_group *cfqg;  in cfq_get_next_queue_forced()
3076 			struct cfq_group *cfqg, enum wl_class_t wl_class)  in cfq_choose_wl_type()
3099 choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)  in choose_wl_class_and_type()
3181 static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd)  in cfq_get_next_cfqg()
3184 	struct cfq_group *cfqg;  in cfq_get_next_cfqg()
3195 	struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);  in cfq_choose_cfqg()
3549 	struct cfq_group *cfqg;  in cfq_put_queue()
3760 cfq_async_queue_prio(struct cfq_group *cfqg, int ioprio_class, int ioprio)  in cfq_async_queue_prio()
3785 	struct cfq_group *cfqg;  in cfq_get_queue()