Lines Matching refs:cfqq
55 #define CFQQ_SEEKY(cfqq) (hweight32(cfqq->seek_history) > 32/8) argument
64 #define cfq_class_idle(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE) argument
65 #define cfq_class_rt(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_RT) argument
310 struct cfq_queue *cfqq[2]; member
396 static void cfq_put_queue(struct cfq_queue *cfqq);
428 static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq) \
430 (cfqq)->flags |= (1 << CFQ_CFQQ_FLAG_##name); \
432 static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq) \
434 (cfqq)->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \
436 static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq) \
438 return ((cfqq)->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \
645 #define cfq_log_cfqq(cfqd, cfqq, fmt, args...) do { \ argument
648 blkg_path(cfqg_to_blkg((cfqq)->cfqg), __pbuf, sizeof(__pbuf)); \
649 blk_add_trace_msg((cfqd)->queue, "cfq%d%c%c %s " fmt, (cfqq)->pid, \
650 cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
651 cfqq_type((cfqq)) == SYNC_NOIDLE_WORKLOAD ? 'N' : ' ',\
764 #define cfq_log_cfqq(cfqd, cfqq, fmt, args...) \ argument
765 blk_add_trace_msg((cfqd)->queue, "cfq%d%c%c " fmt, (cfqq)->pid, \
766 cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
767 cfqq_type((cfqq)) == SYNC_NOIDLE_WORKLOAD ? 'N' : ' ',\
823 static inline enum wl_class_t cfqq_class(struct cfq_queue *cfqq) in cfqq_class() argument
825 if (cfq_class_idle(cfqq)) in cfqq_class()
827 if (cfq_class_rt(cfqq)) in cfqq_class()
833 static enum wl_type_t cfqq_type(struct cfq_queue *cfqq) in cfqq_type() argument
835 if (!cfq_cfqq_sync(cfqq)) in cfqq_type()
837 if (!cfq_cfqq_idle_window(cfqq)) in cfqq_type()
881 return cic->cfqq[is_sync]; in cic_to_cfqq()
884 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq, in cic_set_cfqq() argument
887 cic->cfqq[is_sync] = cfqq; in cic_set_cfqq()
932 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_prio_to_slice() argument
934 return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio); in cfq_prio_to_slice()
1017 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_scaled_cfqq_slice() argument
1019 unsigned slice = cfq_prio_to_slice(cfqd, cfqq); in cfq_scaled_cfqq_slice()
1025 unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg, in cfq_scaled_cfqq_slice()
1026 cfq_class_rt(cfqq)); in cfq_scaled_cfqq_slice()
1029 unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg); in cfq_scaled_cfqq_slice()
1047 cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_set_prio_slice() argument
1049 unsigned slice = cfq_scaled_cfqq_slice(cfqd, cfqq); in cfq_set_prio_slice()
1051 cfqq->slice_start = jiffies; in cfq_set_prio_slice()
1052 cfqq->slice_end = jiffies + slice; in cfq_set_prio_slice()
1053 cfqq->allocated_slice = slice; in cfq_set_prio_slice()
1054 cfq_log_cfqq(cfqd, cfqq, "set_slice=%lu", cfqq->slice_end - jiffies); in cfq_set_prio_slice()
1062 static inline bool cfq_slice_used(struct cfq_queue *cfqq) in cfq_slice_used() argument
1064 if (cfq_cfqq_slice_new(cfqq)) in cfq_slice_used()
1066 if (time_before(jiffies, cfqq->slice_end)) in cfq_slice_used()
1209 cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_find_next_rq() argument
1224 rbnext = rb_first(&cfqq->sort_list); in cfq_find_next_rq()
1233 struct cfq_queue *cfqq) in cfq_slice_offset() argument
1238 return (cfqq->cfqg->nr_cfqq - 1) * (cfq_prio_slice(cfqd, 1, 0) - in cfq_slice_offset()
1239 cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio)); in cfq_slice_offset()
1426 static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq, in cfq_cfqq_slice_usage() argument
1435 if (!cfqq->slice_start || cfqq->slice_start == jiffies) { in cfq_cfqq_slice_usage()
1442 slice_used = max_t(unsigned, (jiffies - cfqq->dispatch_start), in cfq_cfqq_slice_usage()
1445 slice_used = jiffies - cfqq->slice_start; in cfq_cfqq_slice_usage()
1446 if (slice_used > cfqq->allocated_slice) { in cfq_cfqq_slice_usage()
1447 *unaccounted_time = slice_used - cfqq->allocated_slice; in cfq_cfqq_slice_usage()
1448 slice_used = cfqq->allocated_slice; in cfq_cfqq_slice_usage()
1450 if (time_after(cfqq->slice_start, cfqq->dispatch_start)) in cfq_cfqq_slice_usage()
1451 *unaccounted_time += cfqq->slice_start - in cfq_cfqq_slice_usage()
1452 cfqq->dispatch_start; in cfq_cfqq_slice_usage()
1459 struct cfq_queue *cfqq) in cfq_group_served() argument
1468 used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl); in cfq_group_served()
1471 charge = cfqq->slice_dispatch; in cfq_group_served()
1472 else if (!cfq_cfqq_sync(cfqq) && !nr_sync) in cfq_group_served()
1473 charge = cfqq->allocated_slice; in cfq_group_served()
1497 cfq_log_cfqq(cfqq->cfqd, cfqq, in cfq_group_served()
1499 used_sl, cfqq->slice_dispatch, charge, in cfq_group_served()
1500 iops_mode(cfqd), cfqq->nr_sectors); in cfq_group_served()
1688 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) in cfq_link_cfqq_cfqg() argument
1690 cfqq->cfqg = cfqg; in cfq_link_cfqq_cfqg()
2188 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) { in cfq_link_cfqq_cfqg() argument
2189 cfqq->cfqg = cfqg; in cfq_link_cfqq_cfqg()
2199 static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_service_tree_add() argument
2209 st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq)); in cfq_service_tree_add()
2210 if (cfq_class_idle(cfqq)) { in cfq_service_tree_add()
2213 if (parent && parent != &cfqq->rb_node) { in cfq_service_tree_add()
2225 rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies; in cfq_service_tree_add()
2226 rb_key -= cfqq->slice_resid; in cfq_service_tree_add()
2227 cfqq->slice_resid = 0; in cfq_service_tree_add()
2234 if (!RB_EMPTY_NODE(&cfqq->rb_node)) { in cfq_service_tree_add()
2239 if (rb_key == cfqq->rb_key && cfqq->service_tree == st) in cfq_service_tree_add()
2242 cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree); in cfq_service_tree_add()
2243 cfqq->service_tree = NULL; in cfq_service_tree_add()
2248 cfqq->service_tree = st; in cfq_service_tree_add()
2266 st->left = &cfqq->rb_node; in cfq_service_tree_add()
2268 cfqq->rb_key = rb_key; in cfq_service_tree_add()
2269 rb_link_node(&cfqq->rb_node, parent, p); in cfq_service_tree_add()
2270 rb_insert_color(&cfqq->rb_node, &st->rb); in cfq_service_tree_add()
2274 cfq_group_notify_queue_add(cfqd, cfqq->cfqg); in cfq_service_tree_add()
2283 struct cfq_queue *cfqq = NULL; in cfq_prio_tree_lookup() local
2291 cfqq = rb_entry(parent, struct cfq_queue, p_node); in cfq_prio_tree_lookup()
2297 if (sector > blk_rq_pos(cfqq->next_rq)) in cfq_prio_tree_lookup()
2299 else if (sector < blk_rq_pos(cfqq->next_rq)) in cfq_prio_tree_lookup()
2304 cfqq = NULL; in cfq_prio_tree_lookup()
2310 return cfqq; in cfq_prio_tree_lookup()
2313 static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_prio_tree_add() argument
2318 if (cfqq->p_root) { in cfq_prio_tree_add()
2319 rb_erase(&cfqq->p_node, cfqq->p_root); in cfq_prio_tree_add()
2320 cfqq->p_root = NULL; in cfq_prio_tree_add()
2323 if (cfq_class_idle(cfqq)) in cfq_prio_tree_add()
2325 if (!cfqq->next_rq) in cfq_prio_tree_add()
2328 cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio]; in cfq_prio_tree_add()
2329 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root, in cfq_prio_tree_add()
2330 blk_rq_pos(cfqq->next_rq), &parent, &p); in cfq_prio_tree_add()
2332 rb_link_node(&cfqq->p_node, parent, p); in cfq_prio_tree_add()
2333 rb_insert_color(&cfqq->p_node, cfqq->p_root); in cfq_prio_tree_add()
2335 cfqq->p_root = NULL; in cfq_prio_tree_add()
2341 static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_resort_rr_list() argument
2346 if (cfq_cfqq_on_rr(cfqq)) { in cfq_resort_rr_list()
2347 cfq_service_tree_add(cfqd, cfqq, 0); in cfq_resort_rr_list()
2348 cfq_prio_tree_add(cfqd, cfqq); in cfq_resort_rr_list()
2356 static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_add_cfqq_rr() argument
2358 cfq_log_cfqq(cfqd, cfqq, "add_to_rr"); in cfq_add_cfqq_rr()
2359 BUG_ON(cfq_cfqq_on_rr(cfqq)); in cfq_add_cfqq_rr()
2360 cfq_mark_cfqq_on_rr(cfqq); in cfq_add_cfqq_rr()
2362 if (cfq_cfqq_sync(cfqq)) in cfq_add_cfqq_rr()
2365 cfq_resort_rr_list(cfqd, cfqq); in cfq_add_cfqq_rr()
2372 static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_del_cfqq_rr() argument
2374 cfq_log_cfqq(cfqd, cfqq, "del_from_rr"); in cfq_del_cfqq_rr()
2375 BUG_ON(!cfq_cfqq_on_rr(cfqq)); in cfq_del_cfqq_rr()
2376 cfq_clear_cfqq_on_rr(cfqq); in cfq_del_cfqq_rr()
2378 if (!RB_EMPTY_NODE(&cfqq->rb_node)) { in cfq_del_cfqq_rr()
2379 cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree); in cfq_del_cfqq_rr()
2380 cfqq->service_tree = NULL; in cfq_del_cfqq_rr()
2382 if (cfqq->p_root) { in cfq_del_cfqq_rr()
2383 rb_erase(&cfqq->p_node, cfqq->p_root); in cfq_del_cfqq_rr()
2384 cfqq->p_root = NULL; in cfq_del_cfqq_rr()
2387 cfq_group_notify_queue_del(cfqd, cfqq->cfqg); in cfq_del_cfqq_rr()
2390 if (cfq_cfqq_sync(cfqq)) in cfq_del_cfqq_rr()
2399 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_del_rq_rb() local
2402 BUG_ON(!cfqq->queued[sync]); in cfq_del_rq_rb()
2403 cfqq->queued[sync]--; in cfq_del_rq_rb()
2405 elv_rb_del(&cfqq->sort_list, rq); in cfq_del_rq_rb()
2407 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list)) { in cfq_del_rq_rb()
2413 if (cfqq->p_root) { in cfq_del_rq_rb()
2414 rb_erase(&cfqq->p_node, cfqq->p_root); in cfq_del_rq_rb()
2415 cfqq->p_root = NULL; in cfq_del_rq_rb()
2422 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_add_rq_rb() local
2423 struct cfq_data *cfqd = cfqq->cfqd; in cfq_add_rq_rb()
2426 cfqq->queued[rq_is_sync(rq)]++; in cfq_add_rq_rb()
2428 elv_rb_add(&cfqq->sort_list, rq); in cfq_add_rq_rb()
2430 if (!cfq_cfqq_on_rr(cfqq)) in cfq_add_rq_rb()
2431 cfq_add_cfqq_rr(cfqd, cfqq); in cfq_add_rq_rb()
2436 prev = cfqq->next_rq; in cfq_add_rq_rb()
2437 cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq, cfqd->last_position); in cfq_add_rq_rb()
2442 if (prev != cfqq->next_rq) in cfq_add_rq_rb()
2443 cfq_prio_tree_add(cfqd, cfqq); in cfq_add_rq_rb()
2445 BUG_ON(!cfqq->next_rq); in cfq_add_rq_rb()
2448 static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) in cfq_reposition_rq_rb() argument
2450 elv_rb_del(&cfqq->sort_list, rq); in cfq_reposition_rq_rb()
2451 cfqq->queued[rq_is_sync(rq)]--; in cfq_reposition_rq_rb()
2454 cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group, in cfq_reposition_rq_rb()
2463 struct cfq_queue *cfqq; in cfq_find_rq_fmerge() local
2469 cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio)); in cfq_find_rq_fmerge()
2470 if (cfqq) in cfq_find_rq_fmerge()
2471 return elv_rb_find(&cfqq->sort_list, bio_end_sector(bio)); in cfq_find_rq_fmerge()
2499 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_remove_request() local
2501 if (cfqq->next_rq == rq) in cfq_remove_request()
2502 cfqq->next_rq = cfq_find_next_rq(cfqq->cfqd, cfqq, rq); in cfq_remove_request()
2507 cfqq->cfqd->rq_queued--; in cfq_remove_request()
2510 WARN_ON(!cfqq->prio_pending); in cfq_remove_request()
2511 cfqq->prio_pending--; in cfq_remove_request()
2534 struct cfq_queue *cfqq = RQ_CFQQ(req); in cfq_merged_request() local
2536 cfq_reposition_rq_rb(cfqq, req); in cfq_merged_request()
2550 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_merged_requests() local
2558 cfqq == RQ_CFQQ(next)) { in cfq_merged_requests()
2563 if (cfqq->next_rq == next) in cfq_merged_requests()
2564 cfqq->next_rq = rq; in cfq_merged_requests()
2568 cfqq = RQ_CFQQ(next); in cfq_merged_requests()
2574 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list) && in cfq_merged_requests()
2575 cfqq != cfqd->active_queue) in cfq_merged_requests()
2576 cfq_del_cfqq_rr(cfqd, cfqq); in cfq_merged_requests()
2584 struct cfq_queue *cfqq; in cfq_allow_merge() local
2600 cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio)); in cfq_allow_merge()
2601 return cfqq == RQ_CFQQ(rq); in cfq_allow_merge()
2604 static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_del_timer() argument
2607 cfqg_stats_update_idle_time(cfqq->cfqg); in cfq_del_timer()
2611 struct cfq_queue *cfqq) in __cfq_set_active_queue() argument
2613 if (cfqq) { in __cfq_set_active_queue()
2614 cfq_log_cfqq(cfqd, cfqq, "set_active wl_class:%d wl_type:%d", in __cfq_set_active_queue()
2616 cfqg_stats_update_avg_queue_size(cfqq->cfqg); in __cfq_set_active_queue()
2617 cfqq->slice_start = 0; in __cfq_set_active_queue()
2618 cfqq->dispatch_start = jiffies; in __cfq_set_active_queue()
2619 cfqq->allocated_slice = 0; in __cfq_set_active_queue()
2620 cfqq->slice_end = 0; in __cfq_set_active_queue()
2621 cfqq->slice_dispatch = 0; in __cfq_set_active_queue()
2622 cfqq->nr_sectors = 0; in __cfq_set_active_queue()
2624 cfq_clear_cfqq_wait_request(cfqq); in __cfq_set_active_queue()
2625 cfq_clear_cfqq_must_dispatch(cfqq); in __cfq_set_active_queue()
2626 cfq_clear_cfqq_must_alloc_slice(cfqq); in __cfq_set_active_queue()
2627 cfq_clear_cfqq_fifo_expire(cfqq); in __cfq_set_active_queue()
2628 cfq_mark_cfqq_slice_new(cfqq); in __cfq_set_active_queue()
2630 cfq_del_timer(cfqd, cfqq); in __cfq_set_active_queue()
2633 cfqd->active_queue = cfqq; in __cfq_set_active_queue()
2640 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq, in __cfq_slice_expired() argument
2643 cfq_log_cfqq(cfqd, cfqq, "slice expired t=%d", timed_out); in __cfq_slice_expired()
2645 if (cfq_cfqq_wait_request(cfqq)) in __cfq_slice_expired()
2646 cfq_del_timer(cfqd, cfqq); in __cfq_slice_expired()
2648 cfq_clear_cfqq_wait_request(cfqq); in __cfq_slice_expired()
2649 cfq_clear_cfqq_wait_busy(cfqq); in __cfq_slice_expired()
2657 if (cfq_cfqq_coop(cfqq) && CFQQ_SEEKY(cfqq)) in __cfq_slice_expired()
2658 cfq_mark_cfqq_split_coop(cfqq); in __cfq_slice_expired()
2664 if (cfq_cfqq_slice_new(cfqq)) in __cfq_slice_expired()
2665 cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq); in __cfq_slice_expired()
2667 cfqq->slice_resid = cfqq->slice_end - jiffies; in __cfq_slice_expired()
2668 cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid); in __cfq_slice_expired()
2671 cfq_group_served(cfqd, cfqq->cfqg, cfqq); in __cfq_slice_expired()
2673 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list)) in __cfq_slice_expired()
2674 cfq_del_cfqq_rr(cfqd, cfqq); in __cfq_slice_expired()
2676 cfq_resort_rr_list(cfqd, cfqq); in __cfq_slice_expired()
2678 if (cfqq == cfqd->active_queue) in __cfq_slice_expired()
2689 struct cfq_queue *cfqq = cfqd->active_queue; in cfq_slice_expired() local
2691 if (cfqq) in cfq_slice_expired()
2692 __cfq_slice_expired(cfqd, cfqq, timed_out); in cfq_slice_expired()
2718 struct cfq_queue *cfqq; in cfq_get_next_queue_forced() local
2730 if ((cfqq = cfq_rb_first(st)) != NULL) in cfq_get_next_queue_forced()
2731 return cfqq; in cfq_get_next_queue_forced()
2739 struct cfq_queue *cfqq) in cfq_set_active_queue() argument
2741 if (!cfqq) in cfq_set_active_queue()
2742 cfqq = cfq_get_next_queue(cfqd); in cfq_set_active_queue()
2744 __cfq_set_active_queue(cfqd, cfqq); in cfq_set_active_queue()
2745 return cfqq; in cfq_set_active_queue()
2757 static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_rq_close() argument
2817 struct cfq_queue *cfqq; in cfq_close_cooperator() local
2837 cfqq = cfqq_close(cfqd, cur_cfqq); in cfq_close_cooperator()
2838 if (!cfqq) in cfq_close_cooperator()
2842 if (cur_cfqq->cfqg != cfqq->cfqg) in cfq_close_cooperator()
2848 if (!cfq_cfqq_sync(cfqq)) in cfq_close_cooperator()
2850 if (CFQQ_SEEKY(cfqq)) in cfq_close_cooperator()
2856 if (cfq_class_rt(cfqq) != cfq_class_rt(cur_cfqq)) in cfq_close_cooperator()
2859 return cfqq; in cfq_close_cooperator()
2866 static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_should_idle() argument
2868 enum wl_class_t wl_class = cfqq_class(cfqq); in cfq_should_idle()
2869 struct cfq_rb_root *st = cfqq->service_tree; in cfq_should_idle()
2882 if (cfq_cfqq_idle_window(cfqq) && in cfq_should_idle()
2890 if (st->count == 1 && cfq_cfqq_sync(cfqq) && in cfq_should_idle()
2893 cfq_log_cfqq(cfqd, cfqq, "Not idling. st->count:%d", st->count); in cfq_should_idle()
2899 struct cfq_queue *cfqq = cfqd->active_queue; in cfq_arm_slice_timer() local
2911 WARN_ON(!RB_EMPTY_ROOT(&cfqq->sort_list)); in cfq_arm_slice_timer()
2912 WARN_ON(cfq_cfqq_slice_new(cfqq)); in cfq_arm_slice_timer()
2917 if (!cfq_should_idle(cfqd, cfqq)) { in cfq_arm_slice_timer()
2928 if (cfqq->dispatched) in cfq_arm_slice_timer()
2944 (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) { in cfq_arm_slice_timer()
2945 cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%lu", in cfq_arm_slice_timer()
2951 if (group_idle && cfqq->cfqg->nr_cfqq > 1) in cfq_arm_slice_timer()
2954 cfq_mark_cfqq_wait_request(cfqq); in cfq_arm_slice_timer()
2962 cfqg_stats_set_start_idle_time(cfqq->cfqg); in cfq_arm_slice_timer()
2963 cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl, in cfq_arm_slice_timer()
2973 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_dispatch_insert() local
2975 cfq_log_cfqq(cfqd, cfqq, "dispatch_insert"); in cfq_dispatch_insert()
2977 cfqq->next_rq = cfq_find_next_rq(cfqd, cfqq, rq); in cfq_dispatch_insert()
2979 cfqq->dispatched++; in cfq_dispatch_insert()
2983 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++; in cfq_dispatch_insert()
2984 cfqq->nr_sectors += blk_rq_sectors(rq); in cfq_dispatch_insert()
2990 static struct request *cfq_check_fifo(struct cfq_queue *cfqq) in cfq_check_fifo() argument
2994 if (cfq_cfqq_fifo_expire(cfqq)) in cfq_check_fifo()
2997 cfq_mark_cfqq_fifo_expire(cfqq); in cfq_check_fifo()
2999 if (list_empty(&cfqq->fifo)) in cfq_check_fifo()
3002 rq = rq_entry_fifo(cfqq->fifo.next); in cfq_check_fifo()
3006 cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); in cfq_check_fifo()
3011 cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_prio_to_maxrq() argument
3015 WARN_ON(cfqq->ioprio >= IOPRIO_BE_NR); in cfq_prio_to_maxrq()
3017 return 2 * base_rq * (IOPRIO_BE_NR - cfqq->ioprio); in cfq_prio_to_maxrq()
3023 static int cfqq_process_refs(struct cfq_queue *cfqq) in cfqq_process_refs() argument
3027 io_refs = cfqq->allocated[READ] + cfqq->allocated[WRITE]; in cfqq_process_refs()
3028 process_refs = cfqq->ref - io_refs; in cfqq_process_refs()
3033 static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq) in cfq_setup_merge() argument
3049 if (__cfqq == cfqq) in cfq_setup_merge()
3054 process_refs = cfqq_process_refs(cfqq); in cfq_setup_merge()
3067 cfqq->new_cfqq = new_cfqq; in cfq_setup_merge()
3070 new_cfqq->new_cfqq = cfqq; in cfq_setup_merge()
3071 cfqq->ref += new_process_refs; in cfq_setup_merge()
3216 struct cfq_queue *cfqq, *new_cfqq = NULL; in cfq_select_queue() local
3218 cfqq = cfqd->active_queue; in cfq_select_queue()
3219 if (!cfqq) in cfq_select_queue()
3228 if (cfq_cfqq_wait_busy(cfqq) && !RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_select_queue()
3234 if (cfq_slice_used(cfqq) && !cfq_cfqq_must_dispatch(cfqq)) { in cfq_select_queue()
3244 if (cfqq->cfqg->nr_cfqq == 1 && RB_EMPTY_ROOT(&cfqq->sort_list) in cfq_select_queue()
3245 && cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) { in cfq_select_queue()
3246 cfqq = NULL; in cfq_select_queue()
3256 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_select_queue()
3265 new_cfqq = cfq_close_cooperator(cfqd, cfqq); in cfq_select_queue()
3267 if (!cfqq->new_cfqq) in cfq_select_queue()
3268 cfq_setup_merge(cfqq, new_cfqq); in cfq_select_queue()
3278 cfqq = NULL; in cfq_select_queue()
3286 if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) && in cfq_select_queue()
3287 (cfq_cfqq_slice_new(cfqq) || in cfq_select_queue()
3288 (cfqq->slice_end - jiffies > jiffies - cfqq->slice_start))) { in cfq_select_queue()
3289 cfq_clear_cfqq_deep(cfqq); in cfq_select_queue()
3290 cfq_clear_cfqq_idle_window(cfqq); in cfq_select_queue()
3293 if (cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) { in cfq_select_queue()
3294 cfqq = NULL; in cfq_select_queue()
3303 if (cfqd->cfq_group_idle && cfqq->cfqg->nr_cfqq == 1 && in cfq_select_queue()
3304 cfqq->cfqg->dispatched && in cfq_select_queue()
3305 !cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) { in cfq_select_queue()
3306 cfqq = NULL; in cfq_select_queue()
3320 cfqq = cfq_set_active_queue(cfqd, new_cfqq); in cfq_select_queue()
3322 return cfqq; in cfq_select_queue()
3325 static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq) in __cfq_forced_dispatch_cfqq() argument
3329 while (cfqq->next_rq) { in __cfq_forced_dispatch_cfqq()
3330 cfq_dispatch_insert(cfqq->cfqd->queue, cfqq->next_rq); in __cfq_forced_dispatch_cfqq()
3334 BUG_ON(!list_empty(&cfqq->fifo)); in __cfq_forced_dispatch_cfqq()
3337 __cfq_slice_expired(cfqq->cfqd, cfqq, 0); in __cfq_forced_dispatch_cfqq()
3347 struct cfq_queue *cfqq; in cfq_forced_dispatch() local
3352 while ((cfqq = cfq_get_next_queue_forced(cfqd)) != NULL) { in cfq_forced_dispatch()
3353 __cfq_set_active_queue(cfqd, cfqq); in cfq_forced_dispatch()
3354 dispatched += __cfq_forced_dispatch_cfqq(cfqq); in cfq_forced_dispatch()
3364 struct cfq_queue *cfqq) in cfq_slice_used_soon() argument
3367 if (cfq_cfqq_slice_new(cfqq)) in cfq_slice_used_soon()
3369 if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched, in cfq_slice_used_soon()
3370 cfqq->slice_end)) in cfq_slice_used_soon()
3376 static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_may_dispatch() argument
3383 if (cfq_should_idle(cfqd, cfqq) && cfqd->rq_in_flight[BLK_RW_ASYNC]) in cfq_may_dispatch()
3389 if (cfqd->rq_in_flight[BLK_RW_SYNC] && !cfq_cfqq_sync(cfqq)) in cfq_may_dispatch()
3393 if (cfq_class_idle(cfqq)) in cfq_may_dispatch()
3399 if (cfqq->dispatched >= max_dispatch) { in cfq_may_dispatch()
3404 if (cfq_class_idle(cfqq)) in cfq_may_dispatch()
3414 if (cfq_cfqq_sync(cfqq) && cfqd->busy_sync_queues == 1) in cfq_may_dispatch()
3420 if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq) && in cfq_may_dispatch()
3444 if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) { in cfq_may_dispatch()
3449 if (!depth && !cfqq->dispatched) in cfq_may_dispatch()
3458 return cfqq->dispatched < max_dispatch; in cfq_may_dispatch()
3465 static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_dispatch_request() argument
3469 BUG_ON(RB_EMPTY_ROOT(&cfqq->sort_list)); in cfq_dispatch_request()
3471 if (!cfq_may_dispatch(cfqd, cfqq)) in cfq_dispatch_request()
3477 rq = cfq_check_fifo(cfqq); in cfq_dispatch_request()
3479 rq = cfqq->next_rq; in cfq_dispatch_request()
3503 struct cfq_queue *cfqq; in cfq_dispatch_requests() local
3511 cfqq = cfq_select_queue(cfqd); in cfq_dispatch_requests()
3512 if (!cfqq) in cfq_dispatch_requests()
3518 if (!cfq_dispatch_request(cfqd, cfqq)) in cfq_dispatch_requests()
3521 cfqq->slice_dispatch++; in cfq_dispatch_requests()
3522 cfq_clear_cfqq_must_dispatch(cfqq); in cfq_dispatch_requests()
3528 if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) && in cfq_dispatch_requests()
3529 cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) || in cfq_dispatch_requests()
3530 cfq_class_idle(cfqq))) { in cfq_dispatch_requests()
3531 cfqq->slice_end = jiffies + 1; in cfq_dispatch_requests()
3535 cfq_log_cfqq(cfqd, cfqq, "dispatched a request"); in cfq_dispatch_requests()
3546 static void cfq_put_queue(struct cfq_queue *cfqq) in cfq_put_queue() argument
3548 struct cfq_data *cfqd = cfqq->cfqd; in cfq_put_queue()
3551 BUG_ON(cfqq->ref <= 0); in cfq_put_queue()
3553 cfqq->ref--; in cfq_put_queue()
3554 if (cfqq->ref) in cfq_put_queue()
3557 cfq_log_cfqq(cfqd, cfqq, "put_queue"); in cfq_put_queue()
3558 BUG_ON(rb_first(&cfqq->sort_list)); in cfq_put_queue()
3559 BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]); in cfq_put_queue()
3560 cfqg = cfqq->cfqg; in cfq_put_queue()
3562 if (unlikely(cfqd->active_queue == cfqq)) { in cfq_put_queue()
3563 __cfq_slice_expired(cfqd, cfqq, 0); in cfq_put_queue()
3567 BUG_ON(cfq_cfqq_on_rr(cfqq)); in cfq_put_queue()
3568 kmem_cache_free(cfq_pool, cfqq); in cfq_put_queue()
3572 static void cfq_put_cooperator(struct cfq_queue *cfqq) in cfq_put_cooperator() argument
3581 __cfqq = cfqq->new_cfqq; in cfq_put_cooperator()
3583 if (__cfqq == cfqq) { in cfq_put_cooperator()
3593 static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_exit_cfqq() argument
3595 if (unlikely(cfqq == cfqd->active_queue)) { in cfq_exit_cfqq()
3596 __cfq_slice_expired(cfqd, cfqq, 0); in cfq_exit_cfqq()
3600 cfq_put_cooperator(cfqq); in cfq_exit_cfqq()
3602 cfq_put_queue(cfqq); in cfq_exit_cfqq()
3628 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic) in cfq_init_prio_data() argument
3633 if (!cfq_cfqq_prio_changed(cfqq)) in cfq_init_prio_data()
3644 cfqq->ioprio = task_nice_ioprio(tsk); in cfq_init_prio_data()
3645 cfqq->ioprio_class = task_nice_ioclass(tsk); in cfq_init_prio_data()
3648 cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio); in cfq_init_prio_data()
3649 cfqq->ioprio_class = IOPRIO_CLASS_RT; in cfq_init_prio_data()
3652 cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio); in cfq_init_prio_data()
3653 cfqq->ioprio_class = IOPRIO_CLASS_BE; in cfq_init_prio_data()
3656 cfqq->ioprio_class = IOPRIO_CLASS_IDLE; in cfq_init_prio_data()
3657 cfqq->ioprio = 7; in cfq_init_prio_data()
3658 cfq_clear_cfqq_idle_window(cfqq); in cfq_init_prio_data()
3666 cfqq->org_ioprio = cfqq->ioprio; in cfq_init_prio_data()
3667 cfq_clear_cfqq_prio_changed(cfqq); in cfq_init_prio_data()
3674 struct cfq_queue *cfqq; in check_ioprio_changed() local
3683 cfqq = cic_to_cfqq(cic, false); in check_ioprio_changed()
3684 if (cfqq) { in check_ioprio_changed()
3685 cfq_put_queue(cfqq); in check_ioprio_changed()
3686 cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio); in check_ioprio_changed()
3687 cic_set_cfqq(cic, cfqq, false); in check_ioprio_changed()
3690 cfqq = cic_to_cfqq(cic, true); in check_ioprio_changed()
3691 if (cfqq) in check_ioprio_changed()
3692 cfq_mark_cfqq_prio_changed(cfqq); in check_ioprio_changed()
3697 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_init_cfqq() argument
3700 RB_CLEAR_NODE(&cfqq->rb_node); in cfq_init_cfqq()
3701 RB_CLEAR_NODE(&cfqq->p_node); in cfq_init_cfqq()
3702 INIT_LIST_HEAD(&cfqq->fifo); in cfq_init_cfqq()
3704 cfqq->ref = 0; in cfq_init_cfqq()
3705 cfqq->cfqd = cfqd; in cfq_init_cfqq()
3707 cfq_mark_cfqq_prio_changed(cfqq); in cfq_init_cfqq()
3710 if (!cfq_class_idle(cfqq)) in cfq_init_cfqq()
3711 cfq_mark_cfqq_idle_window(cfqq); in cfq_init_cfqq()
3712 cfq_mark_cfqq_sync(cfqq); in cfq_init_cfqq()
3714 cfqq->pid = pid; in cfq_init_cfqq()
3721 struct cfq_queue *cfqq; in check_blkcg_changed() local
3739 cfqq = cic_to_cfqq(cic, false); in check_blkcg_changed()
3740 if (cfqq) { in check_blkcg_changed()
3741 cfq_log_cfqq(cfqd, cfqq, "changed cgroup"); in check_blkcg_changed()
3743 cfq_put_queue(cfqq); in check_blkcg_changed()
3746 cfqq = cic_to_cfqq(cic, true); in check_blkcg_changed()
3747 if (cfqq) { in check_blkcg_changed()
3748 cfq_log_cfqq(cfqd, cfqq, "changed cgroup"); in check_blkcg_changed()
3750 cfq_put_queue(cfqq); in check_blkcg_changed()
3784 struct cfq_queue *cfqq; in cfq_get_queue() local
3790 cfqq = &cfqd->oom_cfqq; in cfq_get_queue()
3801 cfqq = *async_cfqq; in cfq_get_queue()
3802 if (cfqq) in cfq_get_queue()
3806 cfqq = kmem_cache_alloc_node(cfq_pool, GFP_NOWAIT | __GFP_ZERO, in cfq_get_queue()
3808 if (!cfqq) { in cfq_get_queue()
3809 cfqq = &cfqd->oom_cfqq; in cfq_get_queue()
3813 cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync); in cfq_get_queue()
3814 cfq_init_prio_data(cfqq, cic); in cfq_get_queue()
3815 cfq_link_cfqq_cfqg(cfqq, cfqg); in cfq_get_queue()
3816 cfq_log_cfqq(cfqd, cfqq, "alloced"); in cfq_get_queue()
3820 cfqq->ref++; in cfq_get_queue()
3821 *async_cfqq = cfqq; in cfq_get_queue()
3824 cfqq->ref++; in cfq_get_queue()
3826 return cfqq; in cfq_get_queue()
3841 cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_update_io_thinktime() argument
3844 if (cfq_cfqq_sync(cfqq)) { in cfq_update_io_thinktime()
3846 __cfq_update_io_thinktime(&cfqq->service_tree->ttime, in cfq_update_io_thinktime()
3850 __cfq_update_io_thinktime(&cfqq->cfqg->ttime, cfqd->cfq_group_idle); in cfq_update_io_thinktime()
3855 cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_update_io_seektime() argument
3860 if (cfqq->last_request_pos) { in cfq_update_io_seektime()
3861 if (cfqq->last_request_pos < blk_rq_pos(rq)) in cfq_update_io_seektime()
3862 sdist = blk_rq_pos(rq) - cfqq->last_request_pos; in cfq_update_io_seektime()
3864 sdist = cfqq->last_request_pos - blk_rq_pos(rq); in cfq_update_io_seektime()
3867 cfqq->seek_history <<= 1; in cfq_update_io_seektime()
3869 cfqq->seek_history |= (n_sec < CFQQ_SECT_THR_NONROT); in cfq_update_io_seektime()
3871 cfqq->seek_history |= (sdist > CFQQ_SEEK_THR); in cfq_update_io_seektime()
3879 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_update_idle_window() argument
3887 if (!cfq_cfqq_sync(cfqq) || cfq_class_idle(cfqq)) in cfq_update_idle_window()
3890 enable_idle = old_idle = cfq_cfqq_idle_window(cfqq); in cfq_update_idle_window()
3892 if (cfqq->queued[0] + cfqq->queued[1] >= 4) in cfq_update_idle_window()
3893 cfq_mark_cfqq_deep(cfqq); in cfq_update_idle_window()
3895 if (cfqq->next_rq && (cfqq->next_rq->cmd_flags & REQ_NOIDLE)) in cfq_update_idle_window()
3899 (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq))) in cfq_update_idle_window()
3909 cfq_log_cfqq(cfqd, cfqq, "idle=%d", enable_idle); in cfq_update_idle_window()
3911 cfq_mark_cfqq_idle_window(cfqq); in cfq_update_idle_window()
3913 cfq_clear_cfqq_idle_window(cfqq); in cfq_update_idle_window()
3925 struct cfq_queue *cfqq; in cfq_should_preempt() local
3927 cfqq = cfqd->active_queue; in cfq_should_preempt()
3928 if (!cfqq) in cfq_should_preempt()
3934 if (cfq_class_idle(cfqq)) in cfq_should_preempt()
3940 if (cfq_class_rt(cfqq) && !cfq_class_rt(new_cfqq)) in cfq_should_preempt()
3947 if (rq_is_sync(rq) && !cfq_cfqq_sync(cfqq)) in cfq_should_preempt()
3950 if (new_cfqq->cfqg != cfqq->cfqg) in cfq_should_preempt()
3953 if (cfq_slice_used(cfqq)) in cfq_should_preempt()
3960 RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_should_preempt()
3967 if ((rq->cmd_flags & REQ_PRIO) && !cfqq->prio_pending) in cfq_should_preempt()
3973 if (cfq_class_rt(new_cfqq) && !cfq_class_rt(cfqq)) in cfq_should_preempt()
3977 if (RB_EMPTY_ROOT(&cfqq->sort_list) && !cfq_should_idle(cfqd, cfqq)) in cfq_should_preempt()
3980 if (!cfqd->active_cic || !cfq_cfqq_wait_request(cfqq)) in cfq_should_preempt()
3987 if (cfq_rq_close(cfqd, cfqq, rq)) in cfq_should_preempt()
3997 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_preempt_queue() argument
4001 cfq_log_cfqq(cfqd, cfqq, "preempt"); in cfq_preempt_queue()
4008 if (old_type != cfqq_type(cfqq)) in cfq_preempt_queue()
4009 cfqq->cfqg->saved_wl_slice = 0; in cfq_preempt_queue()
4015 BUG_ON(!cfq_cfqq_on_rr(cfqq)); in cfq_preempt_queue()
4017 cfq_service_tree_add(cfqd, cfqq, 1); in cfq_preempt_queue()
4019 cfqq->slice_end = 0; in cfq_preempt_queue()
4020 cfq_mark_cfqq_slice_new(cfqq); in cfq_preempt_queue()
4028 cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_rq_enqueued() argument
4035 cfqq->prio_pending++; in cfq_rq_enqueued()
4037 cfq_update_io_thinktime(cfqd, cfqq, cic); in cfq_rq_enqueued()
4038 cfq_update_io_seektime(cfqd, cfqq, rq); in cfq_rq_enqueued()
4039 cfq_update_idle_window(cfqd, cfqq, cic); in cfq_rq_enqueued()
4041 cfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); in cfq_rq_enqueued()
4043 if (cfqq == cfqd->active_queue) { in cfq_rq_enqueued()
4054 if (cfq_cfqq_wait_request(cfqq)) { in cfq_rq_enqueued()
4057 cfq_del_timer(cfqd, cfqq); in cfq_rq_enqueued()
4058 cfq_clear_cfqq_wait_request(cfqq); in cfq_rq_enqueued()
4061 cfqg_stats_update_idle_time(cfqq->cfqg); in cfq_rq_enqueued()
4062 cfq_mark_cfqq_must_dispatch(cfqq); in cfq_rq_enqueued()
4065 } else if (cfq_should_preempt(cfqd, cfqq, rq)) { in cfq_rq_enqueued()
4072 cfq_preempt_queue(cfqd, cfqq); in cfq_rq_enqueued()
4080 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_insert_request() local
4082 cfq_log_cfqq(cfqd, cfqq, "insert_request"); in cfq_insert_request()
4083 cfq_init_prio_data(cfqq, RQ_CIC(rq)); in cfq_insert_request()
4086 list_add_tail(&rq->queuelist, &cfqq->fifo); in cfq_insert_request()
4090 cfq_rq_enqueued(cfqd, cfqq, rq); in cfq_insert_request()
4099 struct cfq_queue *cfqq = cfqd->active_queue; in cfq_update_hw_tag() local
4116 if (cfqq && cfq_cfqq_idle_window(cfqq) && in cfq_update_hw_tag()
4117 cfqq->dispatched + cfqq->queued[0] + cfqq->queued[1] < in cfq_update_hw_tag()
4130 static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_should_wait_busy() argument
4135 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_should_wait_busy()
4139 if (cfqq->cfqg->nr_cfqq > 1) in cfq_should_wait_busy()
4143 if (cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) in cfq_should_wait_busy()
4146 if (cfq_slice_used(cfqq)) in cfq_should_wait_busy()
4151 && (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) in cfq_should_wait_busy()
4161 if (cfqq->slice_end - jiffies == 1) in cfq_should_wait_busy()
4169 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_completed_request() local
4170 struct cfq_data *cfqd = cfqq->cfqd; in cfq_completed_request()
4175 cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d", in cfq_completed_request()
4181 WARN_ON(!cfqq->dispatched); in cfq_completed_request()
4183 cfqq->dispatched--; in cfq_completed_request()
4185 cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq), in cfq_completed_request()
4188 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--; in cfq_completed_request()
4195 if (cfq_cfqq_on_rr(cfqq)) in cfq_completed_request()
4196 st = cfqq->service_tree; in cfq_completed_request()
4198 st = st_for(cfqq->cfqg, cfqq_class(cfqq), in cfq_completed_request()
4199 cfqq_type(cfqq)); in cfq_completed_request()
4207 cfqq->cfqg->ttime.last_end_request = now; in cfq_completed_request()
4214 if (cfqd->active_queue == cfqq) { in cfq_completed_request()
4215 const bool cfqq_empty = RB_EMPTY_ROOT(&cfqq->sort_list); in cfq_completed_request()
4217 if (cfq_cfqq_slice_new(cfqq)) { in cfq_completed_request()
4218 cfq_set_prio_slice(cfqd, cfqq); in cfq_completed_request()
4219 cfq_clear_cfqq_slice_new(cfqq); in cfq_completed_request()
4226 if (cfq_should_wait_busy(cfqd, cfqq)) { in cfq_completed_request()
4230 cfqq->slice_end = jiffies + extend_sl; in cfq_completed_request()
4231 cfq_mark_cfqq_wait_busy(cfqq); in cfq_completed_request()
4232 cfq_log_cfqq(cfqd, cfqq, "will busy wait"); in cfq_completed_request()
4243 if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq)) in cfq_completed_request()
4246 !cfq_close_cooperator(cfqd, cfqq)) { in cfq_completed_request()
4255 static inline int __cfq_may_queue(struct cfq_queue *cfqq) in __cfq_may_queue() argument
4257 if (cfq_cfqq_wait_request(cfqq) && !cfq_cfqq_must_alloc_slice(cfqq)) { in __cfq_may_queue()
4258 cfq_mark_cfqq_must_alloc_slice(cfqq); in __cfq_may_queue()
4270 struct cfq_queue *cfqq; in cfq_may_queue() local
4282 cfqq = cic_to_cfqq(cic, rw_is_sync(rw)); in cfq_may_queue()
4283 if (cfqq) { in cfq_may_queue()
4284 cfq_init_prio_data(cfqq, cic); in cfq_may_queue()
4286 return __cfq_may_queue(cfqq); in cfq_may_queue()
4297 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_put_request() local
4299 if (cfqq) { in cfq_put_request()
4302 BUG_ON(!cfqq->allocated[rw]); in cfq_put_request()
4303 cfqq->allocated[rw]--; in cfq_put_request()
4310 cfq_put_queue(cfqq); in cfq_put_request()
4316 struct cfq_queue *cfqq) in cfq_merge_cfqqs() argument
4318 cfq_log_cfqq(cfqd, cfqq, "merging with queue %p", cfqq->new_cfqq); in cfq_merge_cfqqs()
4319 cic_set_cfqq(cic, cfqq->new_cfqq, 1); in cfq_merge_cfqqs()
4320 cfq_mark_cfqq_coop(cfqq->new_cfqq); in cfq_merge_cfqqs()
4321 cfq_put_queue(cfqq); in cfq_merge_cfqqs()
4330 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq) in split_cfqq() argument
4332 if (cfqq_process_refs(cfqq) == 1) { in split_cfqq()
4333 cfqq->pid = current->pid; in split_cfqq()
4334 cfq_clear_cfqq_coop(cfqq); in split_cfqq()
4335 cfq_clear_cfqq_split_coop(cfqq); in split_cfqq()
4336 return cfqq; in split_cfqq()
4341 cfq_put_cooperator(cfqq); in split_cfqq()
4343 cfq_put_queue(cfqq); in split_cfqq()
4357 struct cfq_queue *cfqq; in cfq_set_request() local
4364 cfqq = cic_to_cfqq(cic, is_sync); in cfq_set_request()
4365 if (!cfqq || cfqq == &cfqd->oom_cfqq) { in cfq_set_request()
4366 if (cfqq) in cfq_set_request()
4367 cfq_put_queue(cfqq); in cfq_set_request()
4368 cfqq = cfq_get_queue(cfqd, is_sync, cic, bio); in cfq_set_request()
4369 cic_set_cfqq(cic, cfqq, is_sync); in cfq_set_request()
4374 if (cfq_cfqq_coop(cfqq) && cfq_cfqq_split_coop(cfqq)) { in cfq_set_request()
4375 cfq_log_cfqq(cfqd, cfqq, "breaking apart cfqq"); in cfq_set_request()
4376 cfqq = split_cfqq(cic, cfqq); in cfq_set_request()
4377 if (!cfqq) in cfq_set_request()
4387 if (cfqq->new_cfqq) in cfq_set_request()
4388 cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq); in cfq_set_request()
4391 cfqq->allocated[rw]++; in cfq_set_request()
4393 cfqq->ref++; in cfq_set_request()
4394 cfqg_get(cfqq->cfqg); in cfq_set_request()
4395 rq->elv.priv[0] = cfqq; in cfq_set_request()
4396 rq->elv.priv[1] = cfqq->cfqg; in cfq_set_request()
4418 struct cfq_queue *cfqq; in cfq_idle_slice_timer() local
4426 cfqq = cfqd->active_queue; in cfq_idle_slice_timer()
4427 if (cfqq) { in cfq_idle_slice_timer()
4433 if (cfq_cfqq_must_dispatch(cfqq)) in cfq_idle_slice_timer()
4439 if (cfq_slice_used(cfqq)) in cfq_idle_slice_timer()
4452 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_idle_slice_timer()
4458 cfq_clear_cfqq_deep(cfqq); in cfq_idle_slice_timer()