root/block/elevator.c

DEFINITIONS

1. elv_iosched_allow_bio_merge
2. elv_bio_merge_ok
3. elv_support_features
4. elevator_match
5. elevator_find
6. elevator_put
7. elevator_get
8. elevator_alloc
9. elevator_release
10. __elevator_exit
11. __elv_rqhash_del
12. elv_rqhash_del
13. elv_rqhash_add
14. elv_rqhash_reposition
15. elv_rqhash_find
16. elv_rb_add
17. elv_rb_del
18. elv_rb_find
19. elv_attempt_insert_merge
20. elv_merged_request
21. elv_merge_requests
22. elv_latter_request
23. elv_former_request
24. elv_attr_show
25. elv_attr_store
26. elv_register_queue
27. elv_unregister_queue
28. elv_register
29. elv_unregister
30. elevator_switch_mq
31. elv_support_iosched
32. elevator_get_default
33. elevator_get_by_features
34. elevator_init_mq
35. elevator_switch
36. __elevator_change
37. elv_iosched_store
38. elv_iosched_show
39. elv_rb_former_request
40. elv_rb_latter_request

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Block device elevator/IO-scheduler.
   4  *
   5  *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
   6  *
   7  * 30042000 Jens Axboe <axboe@kernel.dk> :
   8  *
   9  * Split the elevator a bit so that it is possible to choose a different
  10  * one or even write a new "plug in". There are three pieces:
  11  * - elevator_fn, inserts a new request in the queue list
  12  * - elevator_merge_fn, decides whether a new buffer can be merged with
  13  *   an existing request
  14  * - elevator_dequeue_fn, called when a request is taken off the active list
  15  *
  16  * 20082000 Dave Jones <davej@suse.de> :
  17  * Removed tests for max-bomb-segments, which was breaking elvtune
  18  *  when run without -bN
  19  *
  20  * Jens:
  21  * - Rework again to work with bio instead of buffer_heads
  22  * - loose bi_dev comparisons, partition handling is right now
  23  * - completely modularize elevator setup and teardown
  24  *
  25  */
  26 #include <linux/kernel.h>
  27 #include <linux/fs.h>
  28 #include <linux/blkdev.h>
  29 #include <linux/elevator.h>
  30 #include <linux/bio.h>
  31 #include <linux/module.h>
  32 #include <linux/slab.h>
  33 #include <linux/init.h>
  34 #include <linux/compiler.h>
  35 #include <linux/blktrace_api.h>
  36 #include <linux/hash.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/pm_runtime.h>
  39 #include <linux/blk-cgroup.h>
  40 
  41 #include <trace/events/block.h>
  42 
  43 #include "blk.h"
  44 #include "blk-mq-sched.h"
  45 #include "blk-pm.h"
  46 #include "blk-wbt.h"
  47 
  48 static DEFINE_SPINLOCK(elv_list_lock);
  49 static LIST_HEAD(elv_list);
  50 
  51 /*
  52  * Merge hash stuff.
  53  */
  54 #define rq_hash_key(rq)         (blk_rq_pos(rq) + blk_rq_sectors(rq))
  55 
  56 /*
  57  * Query io scheduler to see if the current process issuing bio may be
  58  * merged with rq.
  59  */
  60 static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)
  61 {
  62         struct request_queue *q = rq->q;
  63         struct elevator_queue *e = q->elevator;
  64 
  65         if (e->type->ops.allow_merge)
  66                 return e->type->ops.allow_merge(q, rq, bio);
  67 
  68         return 1;
  69 }
  70 
  71 /*
  72  * can we safely merge with this request?
  73  */
  74 bool elv_bio_merge_ok(struct request *rq, struct bio *bio)
  75 {
  76         if (!blk_rq_merge_ok(rq, bio))
  77                 return false;
  78 
  79         if (!elv_iosched_allow_bio_merge(rq, bio))
  80                 return false;
  81 
  82         return true;
  83 }
  84 EXPORT_SYMBOL(elv_bio_merge_ok);
  85 
  86 static inline bool elv_support_features(unsigned int elv_features,
  87                                         unsigned int required_features)
  88 {
  89         return (required_features & elv_features) == required_features;
  90 }
  91 
  92 /**
  93  * elevator_match - Test an elevator name and features
  94  * @e: Scheduler to test
  95  * @name: Elevator name to test
  96  * @required_features: Features that the elevator must provide
  97  *
  98  * Return true is the elevator @e name matches @name and if @e provides all the
  99  * the feratures spcified by @required_features.
 100  */
 101 static bool elevator_match(const struct elevator_type *e, const char *name,
 102                            unsigned int required_features)
 103 {
 104         if (!elv_support_features(e->elevator_features, required_features))
 105                 return false;
 106         if (!strcmp(e->elevator_name, name))
 107                 return true;
 108         if (e->elevator_alias && !strcmp(e->elevator_alias, name))
 109                 return true;
 110 
 111         return false;
 112 }
 113 
 114 /**
 115  * elevator_find - Find an elevator
 116  * @name: Name of the elevator to find
 117  * @required_features: Features that the elevator must provide
 118  *
 119  * Return the first registered scheduler with name @name and supporting the
 120  * features @required_features and NULL otherwise.
 121  */
 122 static struct elevator_type *elevator_find(const char *name,
 123                                            unsigned int required_features)
 124 {
 125         struct elevator_type *e;
 126 
 127         list_for_each_entry(e, &elv_list, list) {
 128                 if (elevator_match(e, name, required_features))
 129                         return e;
 130         }
 131 
 132         return NULL;
 133 }
 134 
 135 static void elevator_put(struct elevator_type *e)
 136 {
 137         module_put(e->elevator_owner);
 138 }
 139 
 140 static struct elevator_type *elevator_get(struct request_queue *q,
 141                                           const char *name, bool try_loading)
 142 {
 143         struct elevator_type *e;
 144 
 145         spin_lock(&elv_list_lock);
 146 
 147         e = elevator_find(name, q->required_elevator_features);
 148         if (!e && try_loading) {
 149                 spin_unlock(&elv_list_lock);
 150                 request_module("%s-iosched", name);
 151                 spin_lock(&elv_list_lock);
 152                 e = elevator_find(name, q->required_elevator_features);
 153         }
 154 
 155         if (e && !try_module_get(e->elevator_owner))
 156                 e = NULL;
 157 
 158         spin_unlock(&elv_list_lock);
 159         return e;
 160 }
 161 
 162 static struct kobj_type elv_ktype;
 163 
 164 struct elevator_queue *elevator_alloc(struct request_queue *q,
 165                                   struct elevator_type *e)
 166 {
 167         struct elevator_queue *eq;
 168 
 169         eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
 170         if (unlikely(!eq))
 171                 return NULL;
 172 
 173         eq->type = e;
 174         kobject_init(&eq->kobj, &elv_ktype);
 175         mutex_init(&eq->sysfs_lock);
 176         hash_init(eq->hash);
 177 
 178         return eq;
 179 }
 180 EXPORT_SYMBOL(elevator_alloc);
 181 
 182 static void elevator_release(struct kobject *kobj)
 183 {
 184         struct elevator_queue *e;
 185 
 186         e = container_of(kobj, struct elevator_queue, kobj);
 187         elevator_put(e->type);
 188         kfree(e);
 189 }
 190 
 191 void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
 192 {
 193         mutex_lock(&e->sysfs_lock);
 194         if (e->type->ops.exit_sched)
 195                 blk_mq_exit_sched(q, e);
 196         mutex_unlock(&e->sysfs_lock);
 197 
 198         kobject_put(&e->kobj);
 199 }
 200 
 201 static inline void __elv_rqhash_del(struct request *rq)
 202 {
 203         hash_del(&rq->hash);
 204         rq->rq_flags &= ~RQF_HASHED;
 205 }
 206 
 207 void elv_rqhash_del(struct request_queue *q, struct request *rq)
 208 {
 209         if (ELV_ON_HASH(rq))
 210                 __elv_rqhash_del(rq);
 211 }
 212 EXPORT_SYMBOL_GPL(elv_rqhash_del);
 213 
 214 void elv_rqhash_add(struct request_queue *q, struct request *rq)
 215 {
 216         struct elevator_queue *e = q->elevator;
 217 
 218         BUG_ON(ELV_ON_HASH(rq));
 219         hash_add(e->hash, &rq->hash, rq_hash_key(rq));
 220         rq->rq_flags |= RQF_HASHED;
 221 }
 222 EXPORT_SYMBOL_GPL(elv_rqhash_add);
 223 
 224 void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
 225 {
 226         __elv_rqhash_del(rq);
 227         elv_rqhash_add(q, rq);
 228 }
 229 
 230 struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
 231 {
 232         struct elevator_queue *e = q->elevator;
 233         struct hlist_node *next;
 234         struct request *rq;
 235 
 236         hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
 237                 BUG_ON(!ELV_ON_HASH(rq));
 238 
 239                 if (unlikely(!rq_mergeable(rq))) {
 240                         __elv_rqhash_del(rq);
 241                         continue;
 242                 }
 243 
 244                 if (rq_hash_key(rq) == offset)
 245                         return rq;
 246         }
 247 
 248         return NULL;
 249 }
 250 
 251 /*
 252  * RB-tree support functions for inserting/lookup/removal of requests
 253  * in a sorted RB tree.
 254  */
 255 void elv_rb_add(struct rb_root *root, struct request *rq)
 256 {
 257         struct rb_node **p = &root->rb_node;
 258         struct rb_node *parent = NULL;
 259         struct request *__rq;
 260 
 261         while (*p) {
 262                 parent = *p;
 263                 __rq = rb_entry(parent, struct request, rb_node);
 264 
 265                 if (blk_rq_pos(rq) < blk_rq_pos(__rq))
 266                         p = &(*p)->rb_left;
 267                 else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
 268                         p = &(*p)->rb_right;
 269         }
 270 
 271         rb_link_node(&rq->rb_node, parent, p);
 272         rb_insert_color(&rq->rb_node, root);
 273 }
 274 EXPORT_SYMBOL(elv_rb_add);
 275 
 276 void elv_rb_del(struct rb_root *root, struct request *rq)
 277 {
 278         BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
 279         rb_erase(&rq->rb_node, root);
 280         RB_CLEAR_NODE(&rq->rb_node);
 281 }
 282 EXPORT_SYMBOL(elv_rb_del);
 283 
 284 struct request *elv_rb_find(struct rb_root *root, sector_t sector)
 285 {
 286         struct rb_node *n = root->rb_node;
 287         struct request *rq;
 288 
 289         while (n) {
 290                 rq = rb_entry(n, struct request, rb_node);
 291 
 292                 if (sector < blk_rq_pos(rq))
 293                         n = n->rb_left;
 294                 else if (sector > blk_rq_pos(rq))
 295                         n = n->rb_right;
 296                 else
 297                         return rq;
 298         }
 299 
 300         return NULL;
 301 }
 302 EXPORT_SYMBOL(elv_rb_find);
 303 
 304 enum elv_merge elv_merge(struct request_queue *q, struct request **req,
 305                 struct bio *bio)
 306 {
 307         struct elevator_queue *e = q->elevator;
 308         struct request *__rq;
 309 
 310         /*
 311          * Levels of merges:
 312          *      nomerges:  No merges at all attempted
 313          *      noxmerges: Only simple one-hit cache try
 314          *      merges:    All merge tries attempted
 315          */
 316         if (blk_queue_nomerges(q) || !bio_mergeable(bio))
 317                 return ELEVATOR_NO_MERGE;
 318 
 319         /*
 320          * First try one-hit cache.
 321          */
 322         if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {
 323                 enum elv_merge ret = blk_try_merge(q->last_merge, bio);
 324 
 325                 if (ret != ELEVATOR_NO_MERGE) {
 326                         *req = q->last_merge;
 327                         return ret;
 328                 }
 329         }
 330 
 331         if (blk_queue_noxmerges(q))
 332                 return ELEVATOR_NO_MERGE;
 333 
 334         /*
 335          * See if our hash lookup can find a potential backmerge.
 336          */
 337         __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);
 338         if (__rq && elv_bio_merge_ok(__rq, bio)) {
 339                 *req = __rq;
 340                 return ELEVATOR_BACK_MERGE;
 341         }
 342 
 343         if (e->type->ops.request_merge)
 344                 return e->type->ops.request_merge(q, req, bio);
 345 
 346         return ELEVATOR_NO_MERGE;
 347 }
 348 
 349 /*
 350  * Attempt to do an insertion back merge. Only check for the case where
 351  * we can append 'rq' to an existing request, so we can throw 'rq' away
 352  * afterwards.
 353  *
 354  * Returns true if we merged, false otherwise
 355  */
 356 bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)
 357 {
 358         struct request *__rq;
 359         bool ret;
 360 
 361         if (blk_queue_nomerges(q))
 362                 return false;
 363 
 364         /*
 365          * First try one-hit cache.
 366          */
 367         if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))
 368                 return true;
 369 
 370         if (blk_queue_noxmerges(q))
 371                 return false;
 372 
 373         ret = false;
 374         /*
 375          * See if our hash lookup can find a potential backmerge.
 376          */
 377         while (1) {
 378                 __rq = elv_rqhash_find(q, blk_rq_pos(rq));
 379                 if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
 380                         break;
 381 
 382                 /* The merged request could be merged with others, try again */
 383                 ret = true;
 384                 rq = __rq;
 385         }
 386 
 387         return ret;
 388 }
 389 
 390 void elv_merged_request(struct request_queue *q, struct request *rq,
 391                 enum elv_merge type)
 392 {
 393         struct elevator_queue *e = q->elevator;
 394 
 395         if (e->type->ops.request_merged)
 396                 e->type->ops.request_merged(q, rq, type);
 397 
 398         if (type == ELEVATOR_BACK_MERGE)
 399                 elv_rqhash_reposition(q, rq);
 400 
 401         q->last_merge = rq;
 402 }
 403 
 404 void elv_merge_requests(struct request_queue *q, struct request *rq,
 405                              struct request *next)
 406 {
 407         struct elevator_queue *e = q->elevator;
 408 
 409         if (e->type->ops.requests_merged)
 410                 e->type->ops.requests_merged(q, rq, next);
 411 
 412         elv_rqhash_reposition(q, rq);
 413         q->last_merge = rq;
 414 }
 415 
 416 struct request *elv_latter_request(struct request_queue *q, struct request *rq)
 417 {
 418         struct elevator_queue *e = q->elevator;
 419 
 420         if (e->type->ops.next_request)
 421                 return e->type->ops.next_request(q, rq);
 422 
 423         return NULL;
 424 }
 425 
 426 struct request *elv_former_request(struct request_queue *q, struct request *rq)
 427 {
 428         struct elevator_queue *e = q->elevator;
 429 
 430         if (e->type->ops.former_request)
 431                 return e->type->ops.former_request(q, rq);
 432 
 433         return NULL;
 434 }
 435 
 436 #define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
 437 
 438 static ssize_t
 439 elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 440 {
 441         struct elv_fs_entry *entry = to_elv(attr);
 442         struct elevator_queue *e;
 443         ssize_t error;
 444 
 445         if (!entry->show)
 446                 return -EIO;
 447 
 448         e = container_of(kobj, struct elevator_queue, kobj);
 449         mutex_lock(&e->sysfs_lock);
 450         error = e->type ? entry->show(e, page) : -ENOENT;
 451         mutex_unlock(&e->sysfs_lock);
 452         return error;
 453 }
 454 
 455 static ssize_t
 456 elv_attr_store(struct kobject *kobj, struct attribute *attr,
 457                const char *page, size_t length)
 458 {
 459         struct elv_fs_entry *entry = to_elv(attr);
 460         struct elevator_queue *e;
 461         ssize_t error;
 462 
 463         if (!entry->store)
 464                 return -EIO;
 465 
 466         e = container_of(kobj, struct elevator_queue, kobj);
 467         mutex_lock(&e->sysfs_lock);
 468         error = e->type ? entry->store(e, page, length) : -ENOENT;
 469         mutex_unlock(&e->sysfs_lock);
 470         return error;
 471 }
 472 
 473 static const struct sysfs_ops elv_sysfs_ops = {
 474         .show   = elv_attr_show,
 475         .store  = elv_attr_store,
 476 };
 477 
 478 static struct kobj_type elv_ktype = {
 479         .sysfs_ops      = &elv_sysfs_ops,
 480         .release        = elevator_release,
 481 };
 482 
 483 /*
 484  * elv_register_queue is called from either blk_register_queue or
 485  * elevator_switch, elevator switch is prevented from being happen
 486  * in the two paths, so it is safe to not hold q->sysfs_lock.
 487  */
 488 int elv_register_queue(struct request_queue *q, bool uevent)
 489 {
 490         struct elevator_queue *e = q->elevator;
 491         int error;
 492 
 493         error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
 494         if (!error) {
 495                 struct elv_fs_entry *attr = e->type->elevator_attrs;
 496                 if (attr) {
 497                         while (attr->attr.name) {
 498                                 if (sysfs_create_file(&e->kobj, &attr->attr))
 499                                         break;
 500                                 attr++;
 501                         }
 502                 }
 503                 if (uevent)
 504                         kobject_uevent(&e->kobj, KOBJ_ADD);
 505 
 506                 e->registered = 1;
 507         }
 508         return error;
 509 }
 510 
 511 /*
 512  * elv_unregister_queue is called from either blk_unregister_queue or
 513  * elevator_switch, elevator switch is prevented from being happen
 514  * in the two paths, so it is safe to not hold q->sysfs_lock.
 515  */
 516 void elv_unregister_queue(struct request_queue *q)
 517 {
 518         if (q) {
 519                 struct elevator_queue *e = q->elevator;
 520 
 521                 kobject_uevent(&e->kobj, KOBJ_REMOVE);
 522                 kobject_del(&e->kobj);
 523 
 524                 e->registered = 0;
 525                 /* Re-enable throttling in case elevator disabled it */
 526                 wbt_enable_default(q);
 527         }
 528 }
 529 
 530 int elv_register(struct elevator_type *e)
 531 {
 532         /* create icq_cache if requested */
 533         if (e->icq_size) {
 534                 if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
 535                     WARN_ON(e->icq_align < __alignof__(struct io_cq)))
 536                         return -EINVAL;
 537 
 538                 snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
 539                          "%s_io_cq", e->elevator_name);
 540                 e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
 541                                                  e->icq_align, 0, NULL);
 542                 if (!e->icq_cache)
 543                         return -ENOMEM;
 544         }
 545 
 546         /* register, don't allow duplicate names */
 547         spin_lock(&elv_list_lock);
 548         if (elevator_find(e->elevator_name, 0)) {
 549                 spin_unlock(&elv_list_lock);
 550                 kmem_cache_destroy(e->icq_cache);
 551                 return -EBUSY;
 552         }
 553         list_add_tail(&e->list, &elv_list);
 554         spin_unlock(&elv_list_lock);
 555 
 556         printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);
 557 
 558         return 0;
 559 }
 560 EXPORT_SYMBOL_GPL(elv_register);
 561 
 562 void elv_unregister(struct elevator_type *e)
 563 {
 564         /* unregister */
 565         spin_lock(&elv_list_lock);
 566         list_del_init(&e->list);
 567         spin_unlock(&elv_list_lock);
 568 
 569         /*
 570          * Destroy icq_cache if it exists.  icq's are RCU managed.  Make
 571          * sure all RCU operations are complete before proceeding.
 572          */
 573         if (e->icq_cache) {
 574                 rcu_barrier();
 575                 kmem_cache_destroy(e->icq_cache);
 576                 e->icq_cache = NULL;
 577         }
 578 }
 579 EXPORT_SYMBOL_GPL(elv_unregister);
 580 
 581 int elevator_switch_mq(struct request_queue *q,
 582                               struct elevator_type *new_e)
 583 {
 584         int ret;
 585 
 586         lockdep_assert_held(&q->sysfs_lock);
 587 
 588         if (q->elevator) {
 589                 if (q->elevator->registered)
 590                         elv_unregister_queue(q);
 591 
 592                 ioc_clear_queue(q);
 593                 elevator_exit(q, q->elevator);
 594         }
 595 
 596         ret = blk_mq_init_sched(q, new_e);
 597         if (ret)
 598                 goto out;
 599 
 600         if (new_e) {
 601                 ret = elv_register_queue(q, true);
 602                 if (ret) {
 603                         elevator_exit(q, q->elevator);
 604                         goto out;
 605                 }
 606         }
 607 
 608         if (new_e)
 609                 blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
 610         else
 611                 blk_add_trace_msg(q, "elv switch: none");
 612 
 613 out:
 614         return ret;
 615 }
 616 
 617 static inline bool elv_support_iosched(struct request_queue *q)
 618 {
 619         if (!q->mq_ops ||
 620             (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
 621                 return false;
 622         return true;
 623 }
 624 
 625 /*
 626  * For single queue devices, default to using mq-deadline. If we have multiple
 627  * queues or mq-deadline is not available, default to "none".
 628  */
 629 static struct elevator_type *elevator_get_default(struct request_queue *q)
 630 {
 631         if (q->nr_hw_queues != 1)
 632                 return NULL;
 633 
 634         return elevator_get(q, "mq-deadline", false);
 635 }
 636 
 637 /*
 638  * Get the first elevator providing the features required by the request queue.
 639  * Default to "none" if no matching elevator is found.
 640  */
 641 static struct elevator_type *elevator_get_by_features(struct request_queue *q)
 642 {
 643         struct elevator_type *e, *found = NULL;
 644 
 645         spin_lock(&elv_list_lock);
 646 
 647         list_for_each_entry(e, &elv_list, list) {
 648                 if (elv_support_features(e->elevator_features,
 649                                          q->required_elevator_features)) {
 650                         found = e;
 651                         break;
 652                 }
 653         }
 654 
 655         if (found && !try_module_get(found->elevator_owner))
 656                 found = NULL;
 657 
 658         spin_unlock(&elv_list_lock);
 659         return found;
 660 }
 661 
 662 /*
 663  * For a device queue that has no required features, use the default elevator
 664  * settings. Otherwise, use the first elevator available matching the required
 665  * features. If no suitable elevator is find or if the chosen elevator
 666  * initialization fails, fall back to the "none" elevator (no elevator).
 667  */
 668 void elevator_init_mq(struct request_queue *q)
 669 {
 670         struct elevator_type *e;
 671         int err;
 672 
 673         if (!elv_support_iosched(q))
 674                 return;
 675 
 676         WARN_ON_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags));
 677 
 678         if (unlikely(q->elevator))
 679                 return;
 680 
 681         if (!q->required_elevator_features)
 682                 e = elevator_get_default(q);
 683         else
 684                 e = elevator_get_by_features(q);
 685         if (!e)
 686                 return;
 687 
 688         blk_mq_freeze_queue(q);
 689         blk_mq_quiesce_queue(q);
 690 
 691         err = blk_mq_init_sched(q, e);
 692 
 693         blk_mq_unquiesce_queue(q);
 694         blk_mq_unfreeze_queue(q);
 695 
 696         if (err) {
 697                 pr_warn("\"%s\" elevator initialization failed, "
 698                         "falling back to \"none\"\n", e->elevator_name);
 699                 elevator_put(e);
 700         }
 701 }
 702 
 703 
 704 /*
 705  * switch to new_e io scheduler. be careful not to introduce deadlocks -
 706  * we don't free the old io scheduler, before we have allocated what we
 707  * need for the new one. this way we have a chance of going back to the old
 708  * one, if the new one fails init for some reason.
 709  */
 710 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
 711 {
 712         int err;
 713 
 714         lockdep_assert_held(&q->sysfs_lock);
 715 
 716         blk_mq_freeze_queue(q);
 717         blk_mq_quiesce_queue(q);
 718 
 719         err = elevator_switch_mq(q, new_e);
 720 
 721         blk_mq_unquiesce_queue(q);
 722         blk_mq_unfreeze_queue(q);
 723 
 724         return err;
 725 }
 726 
 727 /*
 728  * Switch this queue to the given IO scheduler.
 729  */
 730 static int __elevator_change(struct request_queue *q, const char *name)
 731 {
 732         char elevator_name[ELV_NAME_MAX];
 733         struct elevator_type *e;
 734 
 735         /* Make sure queue is not in the middle of being removed */
 736         if (!blk_queue_registered(q))
 737                 return -ENOENT;
 738 
 739         /*
 740          * Special case for mq, turn off scheduling
 741          */
 742         if (!strncmp(name, "none", 4)) {
 743                 if (!q->elevator)
 744                         return 0;
 745                 return elevator_switch(q, NULL);
 746         }
 747 
 748         strlcpy(elevator_name, name, sizeof(elevator_name));
 749         e = elevator_get(q, strstrip(elevator_name), true);
 750         if (!e)
 751                 return -EINVAL;
 752 
 753         if (q->elevator &&
 754             elevator_match(q->elevator->type, elevator_name, 0)) {
 755                 elevator_put(e);
 756                 return 0;
 757         }
 758 
 759         return elevator_switch(q, e);
 760 }
 761 
 762 ssize_t elv_iosched_store(struct request_queue *q, const char *name,
 763                           size_t count)
 764 {
 765         int ret;
 766 
 767         if (!queue_is_mq(q) || !elv_support_iosched(q))
 768                 return count;
 769 
 770         ret = __elevator_change(q, name);
 771         if (!ret)
 772                 return count;
 773 
 774         return ret;
 775 }
 776 
 777 ssize_t elv_iosched_show(struct request_queue *q, char *name)
 778 {
 779         struct elevator_queue *e = q->elevator;
 780         struct elevator_type *elv = NULL;
 781         struct elevator_type *__e;
 782         int len = 0;
 783 
 784         if (!queue_is_mq(q))
 785                 return sprintf(name, "none\n");
 786 
 787         if (!q->elevator)
 788                 len += sprintf(name+len, "[none] ");
 789         else
 790                 elv = e->type;
 791 
 792         spin_lock(&elv_list_lock);
 793         list_for_each_entry(__e, &elv_list, list) {
 794                 if (elv && elevator_match(elv, __e->elevator_name, 0)) {
 795                         len += sprintf(name+len, "[%s] ", elv->elevator_name);
 796                         continue;
 797                 }
 798                 if (elv_support_iosched(q) &&
 799                     elevator_match(__e, __e->elevator_name,
 800                                    q->required_elevator_features))
 801                         len += sprintf(name+len, "%s ", __e->elevator_name);
 802         }
 803         spin_unlock(&elv_list_lock);
 804 
 805         if (q->elevator)
 806                 len += sprintf(name+len, "none");
 807 
 808         len += sprintf(len+name, "\n");
 809         return len;
 810 }
 811 
 812 struct request *elv_rb_former_request(struct request_queue *q,
 813                                       struct request *rq)
 814 {
 815         struct rb_node *rbprev = rb_prev(&rq->rb_node);
 816 
 817         if (rbprev)
 818                 return rb_entry_rq(rbprev);
 819 
 820         return NULL;
 821 }
 822 EXPORT_SYMBOL(elv_rb_former_request);
 823 
 824 struct request *elv_rb_latter_request(struct request_queue *q,
 825                                       struct request *rq)
 826 {
 827         struct rb_node *rbnext = rb_next(&rq->rb_node);
 828 
 829         if (rbnext)
 830                 return rb_entry_rq(rbnext);
 831 
 832         return NULL;
 833 }
 834 EXPORT_SYMBOL(elv_rb_latter_request);