58 	struct request_queue *q = rq->q;  in elv_iosched_allow_merge()  local
59 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_merge()
62 		return e->type->ops.elevator_allow_merge_fn(q, rq, bio);  in elv_iosched_allow_merge()
153 struct elevator_queue *elevator_alloc(struct request_queue *q,  in elevator_alloc()  argument
158 	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);  in elevator_alloc()
180 int elevator_init(struct request_queue *q, char *name)  in elevator_init()  argument
189 	lockdep_assert_held(&q->sysfs_lock);  in elevator_init()
191 	if (unlikely(q->elevator))  in elevator_init()
194 	INIT_LIST_HEAD(&q->queue_head);  in elevator_init()
195 	q->last_merge = NULL;  in elevator_init()
196 	q->end_sector = 0;  in elevator_init()
197 	q->boundary_rq = NULL;  in elevator_init()
227 	err = e->ops.elevator_init_fn(q, e);  in elevator_init()
251 static void elv_rqhash_del(struct request_queue *q, struct request *rq)  in elv_rqhash_del()  argument
257 static void elv_rqhash_add(struct request_queue *q, struct request *rq)  in elv_rqhash_add()  argument
259 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()
266 static void elv_rqhash_reposition(struct request_queue *q, struct request *rq)  in elv_rqhash_reposition()  argument
269 	elv_rqhash_add(q, rq);  in elv_rqhash_reposition()
272 static struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)  in elv_rqhash_find()  argument
274 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()
351 void elv_dispatch_sort(struct request_queue *q, struct request *rq)  in elv_dispatch_sort()  argument
357 	if (q->last_merge == rq)  in elv_dispatch_sort()
358 		q->last_merge = NULL;  in elv_dispatch_sort()
360 	elv_rqhash_del(q, rq);  in elv_dispatch_sort()
362 	q->nr_sorted--;  in elv_dispatch_sort()
364 	boundary = q->end_sector;  in elv_dispatch_sort()
366 	list_for_each_prev(entry, &q->queue_head) {  in elv_dispatch_sort()
396 void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)  in elv_dispatch_add_tail()  argument
398 	if (q->last_merge == rq)  in elv_dispatch_add_tail()
399 		q->last_merge = NULL;  in elv_dispatch_add_tail()
401 	elv_rqhash_del(q, rq);  in elv_dispatch_add_tail()
403 	q->nr_sorted--;  in elv_dispatch_add_tail()
405 	q->end_sector = rq_end_sector(rq);  in elv_dispatch_add_tail()
406 	q->boundary_rq = rq;  in elv_dispatch_add_tail()
407 	list_add_tail(&rq->queuelist, &q->queue_head);  in elv_dispatch_add_tail()
411 int elv_merge(struct request_queue *q, struct request **req, struct bio *bio)  in elv_merge()  argument
413 	struct elevator_queue *e = q->elevator;  in elv_merge()
423 	if (blk_queue_nomerges(q))  in elv_merge()
429 	if (q->last_merge && elv_rq_merge_ok(q->last_merge, bio)) {  in elv_merge()
430 		ret = blk_try_merge(q->last_merge, bio);  in elv_merge()
432 			*req = q->last_merge;  in elv_merge()
437 	if (blk_queue_noxmerges(q))  in elv_merge()
443 	__rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);  in elv_merge()
450 		return e->type->ops.elevator_merge_fn(q, req, bio);  in elv_merge()
462 static bool elv_attempt_insert_merge(struct request_queue *q,  in elv_attempt_insert_merge()  argument
468 	if (blk_queue_nomerges(q))  in elv_attempt_insert_merge()
474 	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))  in elv_attempt_insert_merge()
477 	if (blk_queue_noxmerges(q))  in elv_attempt_insert_merge()
485 		__rq = elv_rqhash_find(q, blk_rq_pos(rq));  in elv_attempt_insert_merge()
486 		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))  in elv_attempt_insert_merge()
497 void elv_merged_request(struct request_queue *q, struct request *rq, int type)  in elv_merged_request()  argument
499 	struct elevator_queue *e = q->elevator;  in elv_merged_request()
502 		e->type->ops.elevator_merged_fn(q, rq, type);  in elv_merged_request()
505 		elv_rqhash_reposition(q, rq);  in elv_merged_request()
507 	q->last_merge = rq;  in elv_merged_request()
510 void elv_merge_requests(struct request_queue *q, struct request *rq,  in elv_merge_requests()  argument
513 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()
517 		e->type->ops.elevator_merge_req_fn(q, rq, next);  in elv_merge_requests()
519 	elv_rqhash_reposition(q, rq);  in elv_merge_requests()
522 		elv_rqhash_del(q, next);  in elv_merge_requests()
523 		q->nr_sorted--;  in elv_merge_requests()
526 	q->last_merge = rq;  in elv_merge_requests()
529 void elv_bio_merged(struct request_queue *q, struct request *rq,  in elv_bio_merged()  argument
532 	struct elevator_queue *e = q->elevator;  in elv_bio_merged()
535 		e->type->ops.elevator_bio_merged_fn(q, rq, bio);  in elv_bio_merged()
541 	if (rq->q->dev && !(rq->cmd_flags & REQ_PM))  in blk_pm_requeue_request()
542 		rq->q->nr_pending--;  in blk_pm_requeue_request()
545 static void blk_pm_add_request(struct request_queue *q, struct request *rq)  in blk_pm_add_request()  argument
547 	if (q->dev && !(rq->cmd_flags & REQ_PM) && q->nr_pending++ == 0 &&  in blk_pm_add_request()
548 	    (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING))  in blk_pm_add_request()
549 		pm_request_resume(q->dev);  in blk_pm_add_request()
553 static inline void blk_pm_add_request(struct request_queue *q,  in blk_pm_add_request()  argument
559 void elv_requeue_request(struct request_queue *q, struct request *rq)  in elv_requeue_request()  argument
566 		q->in_flight[rq_is_sync(rq)]--;  in elv_requeue_request()
568 			elv_deactivate_rq(q, rq);  in elv_requeue_request()
575 	__elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE);  in elv_requeue_request()
578 void elv_drain_elevator(struct request_queue *q)  in elv_drain_elevator()  argument
582 	lockdep_assert_held(q->queue_lock);  in elv_drain_elevator()
584 	while (q->elevator->type->ops.elevator_dispatch_fn(q, 1))  in elv_drain_elevator()
586 	if (q->nr_sorted && printed++ < 10) {  in elv_drain_elevator()
589 		       q->elevator->type->elevator_name, q->nr_sorted);  in elv_drain_elevator()
593 void __elv_add_request(struct request_queue *q, struct request *rq, int where)  in __elv_add_request()  argument
595 	trace_block_rq_insert(q, rq);  in __elv_add_request()
597 	blk_pm_add_request(q, rq);  in __elv_add_request()
599 	rq->q = q;  in __elv_add_request()
604 			q->end_sector = rq_end_sector(rq);  in __elv_add_request()
605 			q->boundary_rq = rq;  in __elv_add_request()
616 		list_add(&rq->queuelist, &q->queue_head);  in __elv_add_request()
621 		elv_drain_elevator(q);  in __elv_add_request()
622 		list_add_tail(&rq->queuelist, &q->queue_head);  in __elv_add_request()
633 		__blk_run_queue(q);  in __elv_add_request()
642 		if (elv_attempt_insert_merge(q, rq))  in __elv_add_request()
647 		q->nr_sorted++;  in __elv_add_request()
649 			elv_rqhash_add(q, rq);  in __elv_add_request()
650 			if (!q->last_merge)  in __elv_add_request()
651 				q->last_merge = rq;  in __elv_add_request()
659 		q->elevator->type->ops.elevator_add_req_fn(q, rq);  in __elv_add_request()
674 void elv_add_request(struct request_queue *q, struct request *rq, int where)  in elv_add_request()  argument
678 	spin_lock_irqsave(q->queue_lock, flags);  in elv_add_request()
679 	__elv_add_request(q, rq, where);  in elv_add_request()
680 	spin_unlock_irqrestore(q->queue_lock, flags);  in elv_add_request()
684 struct request *elv_latter_request(struct request_queue *q, struct request *rq)  in elv_latter_request()  argument
686 	struct elevator_queue *e = q->elevator;  in elv_latter_request()
689 		return e->type->ops.elevator_latter_req_fn(q, rq);  in elv_latter_request()
693 struct request *elv_former_request(struct request_queue *q, struct request *rq)  in elv_former_request()  argument
695 	struct elevator_queue *e = q->elevator;  in elv_former_request()
698 		return e->type->ops.elevator_former_req_fn(q, rq);  in elv_former_request()
702 int elv_set_request(struct request_queue *q, struct request *rq,  in elv_set_request()  argument
705 	struct elevator_queue *e = q->elevator;  in elv_set_request()
708 		return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask);  in elv_set_request()
712 void elv_put_request(struct request_queue *q, struct request *rq)  in elv_put_request()  argument
714 	struct elevator_queue *e = q->elevator;  in elv_put_request()
720 int elv_may_queue(struct request_queue *q, int rw)  in elv_may_queue()  argument
722 	struct elevator_queue *e = q->elevator;  in elv_may_queue()
725 		return e->type->ops.elevator_may_queue_fn(q, rw);  in elv_may_queue()
730 void elv_completed_request(struct request_queue *q, struct request *rq)  in elv_completed_request()  argument
732 	struct elevator_queue *e = q->elevator;  in elv_completed_request()
738 		q->in_flight[rq_is_sync(rq)]--;  in elv_completed_request()
741 			e->type->ops.elevator_completed_req_fn(q, rq);  in elv_completed_request()
792 int elv_register_queue(struct request_queue *q)  in elv_register_queue()  argument
794 	struct elevator_queue *e = q->elevator;  in elv_register_queue()
797 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");  in elv_register_queue()
814 void elv_unregister_queue(struct request_queue *q)  in elv_unregister_queue()  argument
816 	if (q) {  in elv_unregister_queue()
817 		struct elevator_queue *e = q->elevator;  in elv_unregister_queue()
892 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)  in elevator_switch()  argument
894 	struct elevator_queue *old = q->elevator;  in elevator_switch()
905 	blk_queue_bypass_start(q);  in elevator_switch()
909 		elv_unregister_queue(q);  in elevator_switch()
911 	spin_lock_irq(q->queue_lock);  in elevator_switch()
912 	ioc_clear_queue(q);  in elevator_switch()
913 	spin_unlock_irq(q->queue_lock);  in elevator_switch()
916 	err = new_e->ops.elevator_init_fn(q, new_e);  in elevator_switch()
921 		err = elv_register_queue(q);  in elevator_switch()
928 	blk_queue_bypass_end(q);  in elevator_switch()
930 	blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);  in elevator_switch()
935 	elevator_exit(q->elevator);  in elevator_switch()
938 	q->elevator = old;  in elevator_switch()
939 	elv_register_queue(q);  in elevator_switch()
940 	blk_queue_bypass_end(q);  in elevator_switch()
948 static int __elevator_change(struct request_queue *q, const char *name)  in __elevator_change()  argument
953 	if (!q->elevator)  in __elevator_change()
963 	if (!strcmp(elevator_name, q->elevator->type->elevator_name)) {  in __elevator_change()
968 	return elevator_switch(q, e);  in __elevator_change()
971 int elevator_change(struct request_queue *q, const char *name)  in elevator_change()  argument
976 	mutex_lock(&q->sysfs_lock);  in elevator_change()
977 	ret = __elevator_change(q, name);  in elevator_change()
978 	mutex_unlock(&q->sysfs_lock);  in elevator_change()
984 ssize_t elv_iosched_store(struct request_queue *q, const char *name,  in elv_iosched_store()  argument
989 	if (!q->elevator)  in elv_iosched_store()
992 	ret = __elevator_change(q, name);  in elv_iosched_store()
1000 ssize_t elv_iosched_show(struct request_queue *q, char *name)  in elv_iosched_show()  argument
1002 	struct elevator_queue *e = q->elevator;  in elv_iosched_show()
1007 	if (!q->elevator || !blk_queue_stackable(q))  in elv_iosched_show()
1025 struct request *elv_rb_former_request(struct request_queue *q,  in elv_rb_former_request()  argument
1037 struct request *elv_rb_latter_request(struct request_queue *q,  in elv_rb_latter_request()  argument