57 #define RQ_CIC(rq)		icq_to_cic((rq)->elv.icq)  argument
58 #define RQ_CFQQ(rq)		(struct cfq_queue *) ((rq)->elv.priv[0])  argument
59 #define RQ_CFQG(rq)		(struct cfq_group *) ((rq)->elv.priv[1])  argument
2220 static void cfq_del_rq_rb(struct request *rq)  in cfq_del_rq_rb()  argument
2222 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_del_rq_rb()
2223 	const int sync = rq_is_sync(rq);  in cfq_del_rq_rb()
2228 	elv_rb_del(&cfqq->sort_list, rq);  in cfq_del_rq_rb()
2243 static void cfq_add_rq_rb(struct request *rq)  in cfq_add_rq_rb()  argument
2245 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_add_rq_rb()
2249 	cfqq->queued[rq_is_sync(rq)]++;  in cfq_add_rq_rb()
2251 	elv_rb_add(&cfqq->sort_list, rq);  in cfq_add_rq_rb()
2260 	cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq, cfqd->last_position);  in cfq_add_rq_rb()
2271 static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)  in cfq_reposition_rq_rb()  argument
2273 	elv_rb_del(&cfqq->sort_list, rq);  in cfq_reposition_rq_rb()
2274 	cfqq->queued[rq_is_sync(rq)]--;  in cfq_reposition_rq_rb()
2275 	cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);  in cfq_reposition_rq_rb()
2276 	cfq_add_rq_rb(rq);  in cfq_reposition_rq_rb()
2277 	cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group,  in cfq_reposition_rq_rb()
2278 				 rq->cmd_flags);  in cfq_reposition_rq_rb()
2299 static void cfq_activate_request(struct request_queue *q, struct request *rq)  in cfq_activate_request()  argument
2304 	cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "activate rq, drv=%d",  in cfq_activate_request()
2307 	cfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);  in cfq_activate_request()
2310 static void cfq_deactivate_request(struct request_queue *q, struct request *rq)  in cfq_deactivate_request()  argument
2316 	cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "deactivate rq, drv=%d",  in cfq_deactivate_request()
2320 static void cfq_remove_request(struct request *rq)  in cfq_remove_request()  argument
2322 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_remove_request()
2324 	if (cfqq->next_rq == rq)  in cfq_remove_request()
2325 		cfqq->next_rq = cfq_find_next_rq(cfqq->cfqd, cfqq, rq);  in cfq_remove_request()
2327 	list_del_init(&rq->queuelist);  in cfq_remove_request()
2328 	cfq_del_rq_rb(rq);  in cfq_remove_request()
2331 	cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);  in cfq_remove_request()
2332 	if (rq->cmd_flags & REQ_PRIO) {  in cfq_remove_request()
2370 cfq_merged_requests(struct request_queue *q, struct request *rq,  in cfq_merged_requests()  argument
2373 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_merged_requests()
2379 	if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&  in cfq_merged_requests()
2380 	    time_before(next->fifo_time, rq->fifo_time) &&  in cfq_merged_requests()
2382 		list_move(&rq->queuelist, &next->queuelist);  in cfq_merged_requests()
2383 		rq->fifo_time = next->fifo_time;  in cfq_merged_requests()
2387 		cfqq->next_rq = rq;  in cfq_merged_requests()
2389 	cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags);  in cfq_merged_requests()
2402 static int cfq_allow_merge(struct request_queue *q, struct request *rq,  in cfq_allow_merge()  argument
2412 	if (cfq_bio_sync(bio) && !rq_is_sync(rq))  in cfq_allow_merge()
2424 	return cfqq == RQ_CFQQ(rq);  in cfq_allow_merge()
2572 					  struct request *rq)  in cfq_dist_from_last()  argument
2574 	if (blk_rq_pos(rq) >= cfqd->last_position)  in cfq_dist_from_last()
2575 		return blk_rq_pos(rq) - cfqd->last_position;  in cfq_dist_from_last()
2577 		return cfqd->last_position - blk_rq_pos(rq);  in cfq_dist_from_last()
2581 			       struct request *rq)  in cfq_rq_close()  argument
2583 	return cfq_dist_from_last(cfqd, rq) <= CFQQ_CLOSE_THR;  in cfq_rq_close()
2793 static void cfq_dispatch_insert(struct request_queue *q, struct request *rq)  in cfq_dispatch_insert()  argument
2796 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_dispatch_insert()
2800 	cfqq->next_rq = cfq_find_next_rq(cfqd, cfqq, rq);  in cfq_dispatch_insert()
2801 	cfq_remove_request(rq);  in cfq_dispatch_insert()
2803 	(RQ_CFQG(rq))->dispatched++;  in cfq_dispatch_insert()
2804 	elv_dispatch_sort(q, rq);  in cfq_dispatch_insert()
2807 	cfqq->nr_sectors += blk_rq_sectors(rq);  in cfq_dispatch_insert()
2808 	cfqg_stats_update_dispatch(cfqq->cfqg, blk_rq_bytes(rq), rq->cmd_flags);  in cfq_dispatch_insert()
2816 	struct request *rq = NULL;  in cfq_check_fifo()  local
2826 	rq = rq_entry_fifo(cfqq->fifo.next);  in cfq_check_fifo()
2827 	if (time_before(jiffies, rq->fifo_time))  in cfq_check_fifo()
2828 		rq = NULL;  in cfq_check_fifo()
2830 	cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq);  in cfq_check_fifo()
2831 	return rq;  in cfq_check_fifo()
3291 	struct request *rq;  in cfq_dispatch_request()  local
3301 	rq = cfq_check_fifo(cfqq);  in cfq_dispatch_request()
3302 	if (!rq)  in cfq_dispatch_request()
3303 		rq = cfqq->next_rq;  in cfq_dispatch_request()
3308 	cfq_dispatch_insert(cfqd->queue, rq);  in cfq_dispatch_request()
3311 		struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_dispatch_request()
3721 		       struct request *rq)  in cfq_update_io_seektime()  argument
3724 	sector_t n_sec = blk_rq_sectors(rq);  in cfq_update_io_seektime()
3726 		if (cfqq->last_request_pos < blk_rq_pos(rq))  in cfq_update_io_seektime()
3727 			sdist = blk_rq_pos(rq) - cfqq->last_request_pos;  in cfq_update_io_seektime()
3729 			sdist = cfqq->last_request_pos - blk_rq_pos(rq);  in cfq_update_io_seektime()
3788 		   struct request *rq)  in cfq_should_preempt()  argument
3812 	if (rq_is_sync(rq) && !cfq_cfqq_sync(cfqq))  in cfq_should_preempt()
3832 	if ((rq->cmd_flags & REQ_PRIO) && !cfqq->prio_pending)  in cfq_should_preempt()
3852 	if (cfq_rq_close(cfqd, cfqq, rq))  in cfq_should_preempt()
3894 		struct request *rq)  in cfq_rq_enqueued()  argument
3896 	struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_rq_enqueued()
3899 	if (rq->cmd_flags & REQ_PRIO)  in cfq_rq_enqueued()
3903 	cfq_update_io_seektime(cfqd, cfqq, rq);  in cfq_rq_enqueued()
3906 	cfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);  in cfq_rq_enqueued()
3920 			if (blk_rq_bytes(rq) > PAGE_CACHE_SIZE ||  in cfq_rq_enqueued()
3930 	} else if (cfq_should_preempt(cfqd, cfqq, rq)) {  in cfq_rq_enqueued()
3942 static void cfq_insert_request(struct request_queue *q, struct request *rq)  in cfq_insert_request()  argument
3945 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_insert_request()
3948 	cfq_init_prio_data(cfqq, RQ_CIC(rq));  in cfq_insert_request()
3950 	rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)];  in cfq_insert_request()
3951 	list_add_tail(&rq->queuelist, &cfqq->fifo);  in cfq_insert_request()
3952 	cfq_add_rq_rb(rq);  in cfq_insert_request()
3953 	cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,  in cfq_insert_request()
3954 				 rq->cmd_flags);  in cfq_insert_request()
3955 	cfq_rq_enqueued(cfqd, cfqq, rq);  in cfq_insert_request()
4032 static void cfq_completed_request(struct request_queue *q, struct request *rq)  in cfq_completed_request()  argument
4034 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_completed_request()
4036 	const int sync = rq_is_sync(rq);  in cfq_completed_request()
4041 		     !!(rq->cmd_flags & REQ_NOIDLE));  in cfq_completed_request()
4049 	(RQ_CFQG(rq))->dispatched--;  in cfq_completed_request()
4050 	cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq),  in cfq_completed_request()
4051 				     rq_io_start_time_ns(rq), rq->cmd_flags);  in cfq_completed_request()
4058 		RQ_CIC(rq)->ttime.last_end_request = now;  in cfq_completed_request()
4067 		if (!time_after(rq->start_time + cfqd->cfq_fifo_expire[1], now))  in cfq_completed_request()
4160 static void cfq_put_request(struct request *rq)  in cfq_put_request()  argument
4162 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_put_request()
4165 		const int rw = rq_data_dir(rq);  in cfq_put_request()
4171 		cfqg_put(RQ_CFQG(rq));  in cfq_put_request()
4172 		rq->elv.priv[0] = NULL;  in cfq_put_request()
4173 		rq->elv.priv[1] = NULL;  in cfq_put_request()
4215 cfq_set_request(struct request_queue *q, struct request *rq, struct bio *bio,  in cfq_set_request()  argument
4219 	struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);  in cfq_set_request()
4220 	const int rw = rq_data_dir(rq);  in cfq_set_request()
4221 	const bool is_sync = rq_is_sync(rq);  in cfq_set_request()
4260 	rq->elv.priv[0] = cfqq;  in cfq_set_request()
4261 	rq->elv.priv[1] = cfqq->cfqg;  in cfq_set_request()