1/* 2 * Common Block IO controller cgroup interface 3 * 4 * Based on ideas and code from CFQ, CFS and BFQ: 5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> 6 * 7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> 8 * Paolo Valente <paolo.valente@unimore.it> 9 * 10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> 11 * Nauman Rafique <nauman@google.com> 12 */ 13#include <linux/ioprio.h> 14#include <linux/kdev_t.h> 15#include <linux/module.h> 16#include <linux/err.h> 17#include <linux/blkdev.h> 18#include <linux/slab.h> 19#include <linux/genhd.h> 20#include <linux/delay.h> 21#include <linux/atomic.h> 22#include "blk-cgroup.h" 23#include "blk.h" 24 25#define MAX_KEY_LEN 100 26 27static DEFINE_MUTEX(blkcg_pol_mutex); 28 29struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT, 30 .cfq_leaf_weight = 2 * CFQ_WEIGHT_DEFAULT, }; 31EXPORT_SYMBOL_GPL(blkcg_root); 32 33static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; 34 35static bool blkcg_policy_enabled(struct request_queue *q, 36 const struct blkcg_policy *pol) 37{ 38 return pol && test_bit(pol->plid, q->blkcg_pols); 39} 40 41/** 42 * blkg_free - free a blkg 43 * @blkg: blkg to free 44 * 45 * Free @blkg which may be partially allocated. 46 */ 47static void blkg_free(struct blkcg_gq *blkg) 48{ 49 int i; 50 51 if (!blkg) 52 return; 53 54 for (i = 0; i < BLKCG_MAX_POLS; i++) 55 kfree(blkg->pd[i]); 56 57 blk_exit_rl(&blkg->rl); 58 kfree(blkg); 59} 60 61/** 62 * blkg_alloc - allocate a blkg 63 * @blkcg: block cgroup the new blkg is associated with 64 * @q: request_queue the new blkg is associated with 65 * @gfp_mask: allocation mask to use 66 * 67 * Allocate a new blkg assocating @blkcg and @q. 68 */ 69static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, 70 gfp_t gfp_mask) 71{ 72 struct blkcg_gq *blkg; 73 int i; 74 75 /* alloc and init base part */ 76 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); 77 if (!blkg) 78 return NULL; 79 80 blkg->q = q; 81 INIT_LIST_HEAD(&blkg->q_node); 82 blkg->blkcg = blkcg; 83 atomic_set(&blkg->refcnt, 1); 84 85 /* root blkg uses @q->root_rl, init rl only for !root blkgs */ 86 if (blkcg != &blkcg_root) { 87 if (blk_init_rl(&blkg->rl, q, gfp_mask)) 88 goto err_free; 89 blkg->rl.blkg = blkg; 90 } 91 92 for (i = 0; i < BLKCG_MAX_POLS; i++) { 93 struct blkcg_policy *pol = blkcg_policy[i]; 94 struct blkg_policy_data *pd; 95 96 if (!blkcg_policy_enabled(q, pol)) 97 continue; 98 99 /* alloc per-policy data and attach it to blkg */ 100 pd = kzalloc_node(pol->pd_size, gfp_mask, q->node); 101 if (!pd) 102 goto err_free; 103 104 blkg->pd[i] = pd; 105 pd->blkg = blkg; 106 pd->plid = i; 107 } 108 109 return blkg; 110 111err_free: 112 blkg_free(blkg); 113 return NULL; 114} 115 116/** 117 * __blkg_lookup - internal version of blkg_lookup() 118 * @blkcg: blkcg of interest 119 * @q: request_queue of interest 120 * @update_hint: whether to update lookup hint with the result or not 121 * 122 * This is internal version and shouldn't be used by policy 123 * implementations. Looks up blkgs for the @blkcg - @q pair regardless of 124 * @q's bypass state. If @update_hint is %true, the caller should be 125 * holding @q->queue_lock and lookup hint is updated on success. 126 */ 127struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, struct request_queue *q, 128 bool update_hint) 129{ 130 struct blkcg_gq *blkg; 131 132 blkg = rcu_dereference(blkcg->blkg_hint); 133 if (blkg && blkg->q == q) 134 return blkg; 135 136 /* 137 * Hint didn't match. Look up from the radix tree. Note that the 138 * hint can only be updated under queue_lock as otherwise @blkg 139 * could have already been removed from blkg_tree. The caller is 140 * responsible for grabbing queue_lock if @update_hint. 141 */ 142 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); 143 if (blkg && blkg->q == q) { 144 if (update_hint) { 145 lockdep_assert_held(q->queue_lock); 146 rcu_assign_pointer(blkcg->blkg_hint, blkg); 147 } 148 return blkg; 149 } 150 151 return NULL; 152} 153 154/** 155 * blkg_lookup - lookup blkg for the specified blkcg - q pair 156 * @blkcg: blkcg of interest 157 * @q: request_queue of interest 158 * 159 * Lookup blkg for the @blkcg - @q pair. This function should be called 160 * under RCU read lock and is guaranteed to return %NULL if @q is bypassing 161 * - see blk_queue_bypass_start() for details. 162 */ 163struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q) 164{ 165 WARN_ON_ONCE(!rcu_read_lock_held()); 166 167 if (unlikely(blk_queue_bypass(q))) 168 return NULL; 169 return __blkg_lookup(blkcg, q, false); 170} 171EXPORT_SYMBOL_GPL(blkg_lookup); 172 173/* 174 * If @new_blkg is %NULL, this function tries to allocate a new one as 175 * necessary using %GFP_ATOMIC. @new_blkg is always consumed on return. 176 */ 177static struct blkcg_gq *blkg_create(struct blkcg *blkcg, 178 struct request_queue *q, 179 struct blkcg_gq *new_blkg) 180{ 181 struct blkcg_gq *blkg; 182 int i, ret; 183 184 WARN_ON_ONCE(!rcu_read_lock_held()); 185 lockdep_assert_held(q->queue_lock); 186 187 /* blkg holds a reference to blkcg */ 188 if (!css_tryget_online(&blkcg->css)) { 189 ret = -EINVAL; 190 goto err_free_blkg; 191 } 192 193 /* allocate */ 194 if (!new_blkg) { 195 new_blkg = blkg_alloc(blkcg, q, GFP_ATOMIC); 196 if (unlikely(!new_blkg)) { 197 ret = -ENOMEM; 198 goto err_put_css; 199 } 200 } 201 blkg = new_blkg; 202 203 /* link parent */ 204 if (blkcg_parent(blkcg)) { 205 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); 206 if (WARN_ON_ONCE(!blkg->parent)) { 207 ret = -EINVAL; 208 goto err_put_css; 209 } 210 blkg_get(blkg->parent); 211 } 212 213 /* invoke per-policy init */ 214 for (i = 0; i < BLKCG_MAX_POLS; i++) { 215 struct blkcg_policy *pol = blkcg_policy[i]; 216 217 if (blkg->pd[i] && pol->pd_init_fn) 218 pol->pd_init_fn(blkg); 219 } 220 221 /* insert */ 222 spin_lock(&blkcg->lock); 223 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); 224 if (likely(!ret)) { 225 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); 226 list_add(&blkg->q_node, &q->blkg_list); 227 228 for (i = 0; i < BLKCG_MAX_POLS; i++) { 229 struct blkcg_policy *pol = blkcg_policy[i]; 230 231 if (blkg->pd[i] && pol->pd_online_fn) 232 pol->pd_online_fn(blkg); 233 } 234 } 235 blkg->online = true; 236 spin_unlock(&blkcg->lock); 237 238 if (!ret) { 239 if (blkcg == &blkcg_root) { 240 q->root_blkg = blkg; 241 q->root_rl.blkg = blkg; 242 } 243 return blkg; 244 } 245 246 /* @blkg failed fully initialized, use the usual release path */ 247 blkg_put(blkg); 248 return ERR_PTR(ret); 249 250err_put_css: 251 css_put(&blkcg->css); 252err_free_blkg: 253 blkg_free(new_blkg); 254 return ERR_PTR(ret); 255} 256 257/** 258 * blkg_lookup_create - lookup blkg, try to create one if not there 259 * @blkcg: blkcg of interest 260 * @q: request_queue of interest 261 * 262 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to 263 * create one. blkg creation is performed recursively from blkcg_root such 264 * that all non-root blkg's have access to the parent blkg. This function 265 * should be called under RCU read lock and @q->queue_lock. 266 * 267 * Returns pointer to the looked up or created blkg on success, ERR_PTR() 268 * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not 269 * dead and bypassing, returns ERR_PTR(-EBUSY). 270 */ 271struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, 272 struct request_queue *q) 273{ 274 struct blkcg_gq *blkg; 275 276 WARN_ON_ONCE(!rcu_read_lock_held()); 277 lockdep_assert_held(q->queue_lock); 278 279 /* 280 * This could be the first entry point of blkcg implementation and 281 * we shouldn't allow anything to go through for a bypassing queue. 282 */ 283 if (unlikely(blk_queue_bypass(q))) 284 return ERR_PTR(blk_queue_dying(q) ? -EINVAL : -EBUSY); 285 286 blkg = __blkg_lookup(blkcg, q, true); 287 if (blkg) 288 return blkg; 289 290 /* 291 * Create blkgs walking down from blkcg_root to @blkcg, so that all 292 * non-root blkgs have access to their parents. 293 */ 294 while (true) { 295 struct blkcg *pos = blkcg; 296 struct blkcg *parent = blkcg_parent(blkcg); 297 298 while (parent && !__blkg_lookup(parent, q, false)) { 299 pos = parent; 300 parent = blkcg_parent(parent); 301 } 302 303 blkg = blkg_create(pos, q, NULL); 304 if (pos == blkcg || IS_ERR(blkg)) 305 return blkg; 306 } 307} 308EXPORT_SYMBOL_GPL(blkg_lookup_create); 309 310static void blkg_destroy(struct blkcg_gq *blkg) 311{ 312 struct blkcg *blkcg = blkg->blkcg; 313 int i; 314 315 lockdep_assert_held(blkg->q->queue_lock); 316 lockdep_assert_held(&blkcg->lock); 317 318 /* Something wrong if we are trying to remove same group twice */ 319 WARN_ON_ONCE(list_empty(&blkg->q_node)); 320 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); 321 322 for (i = 0; i < BLKCG_MAX_POLS; i++) { 323 struct blkcg_policy *pol = blkcg_policy[i]; 324 325 if (blkg->pd[i] && pol->pd_offline_fn) 326 pol->pd_offline_fn(blkg); 327 } 328 blkg->online = false; 329 330 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); 331 list_del_init(&blkg->q_node); 332 hlist_del_init_rcu(&blkg->blkcg_node); 333 334 /* 335 * Both setting lookup hint to and clearing it from @blkg are done 336 * under queue_lock. If it's not pointing to @blkg now, it never 337 * will. Hint assignment itself can race safely. 338 */ 339 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) 340 rcu_assign_pointer(blkcg->blkg_hint, NULL); 341 342 /* 343 * If root blkg is destroyed. Just clear the pointer since root_rl 344 * does not take reference on root blkg. 345 */ 346 if (blkcg == &blkcg_root) { 347 blkg->q->root_blkg = NULL; 348 blkg->q->root_rl.blkg = NULL; 349 } 350 351 /* 352 * Put the reference taken at the time of creation so that when all 353 * queues are gone, group can be destroyed. 354 */ 355 blkg_put(blkg); 356} 357 358/** 359 * blkg_destroy_all - destroy all blkgs associated with a request_queue 360 * @q: request_queue of interest 361 * 362 * Destroy all blkgs associated with @q. 363 */ 364static void blkg_destroy_all(struct request_queue *q) 365{ 366 struct blkcg_gq *blkg, *n; 367 368 lockdep_assert_held(q->queue_lock); 369 370 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { 371 struct blkcg *blkcg = blkg->blkcg; 372 373 spin_lock(&blkcg->lock); 374 blkg_destroy(blkg); 375 spin_unlock(&blkcg->lock); 376 } 377} 378 379/* 380 * A group is RCU protected, but having an rcu lock does not mean that one 381 * can access all the fields of blkg and assume these are valid. For 382 * example, don't try to follow throtl_data and request queue links. 383 * 384 * Having a reference to blkg under an rcu allows accesses to only values 385 * local to groups like group stats and group rate limits. 386 */ 387void __blkg_release_rcu(struct rcu_head *rcu_head) 388{ 389 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head); 390 int i; 391 392 /* tell policies that this one is being freed */ 393 for (i = 0; i < BLKCG_MAX_POLS; i++) { 394 struct blkcg_policy *pol = blkcg_policy[i]; 395 396 if (blkg->pd[i] && pol->pd_exit_fn) 397 pol->pd_exit_fn(blkg); 398 } 399 400 /* release the blkcg and parent blkg refs this blkg has been holding */ 401 css_put(&blkg->blkcg->css); 402 if (blkg->parent) 403 blkg_put(blkg->parent); 404 405 blkg_free(blkg); 406} 407EXPORT_SYMBOL_GPL(__blkg_release_rcu); 408 409/* 410 * The next function used by blk_queue_for_each_rl(). It's a bit tricky 411 * because the root blkg uses @q->root_rl instead of its own rl. 412 */ 413struct request_list *__blk_queue_next_rl(struct request_list *rl, 414 struct request_queue *q) 415{ 416 struct list_head *ent; 417 struct blkcg_gq *blkg; 418 419 /* 420 * Determine the current blkg list_head. The first entry is 421 * root_rl which is off @q->blkg_list and mapped to the head. 422 */ 423 if (rl == &q->root_rl) { 424 ent = &q->blkg_list; 425 /* There are no more block groups, hence no request lists */ 426 if (list_empty(ent)) 427 return NULL; 428 } else { 429 blkg = container_of(rl, struct blkcg_gq, rl); 430 ent = &blkg->q_node; 431 } 432 433 /* walk to the next list_head, skip root blkcg */ 434 ent = ent->next; 435 if (ent == &q->root_blkg->q_node) 436 ent = ent->next; 437 if (ent == &q->blkg_list) 438 return NULL; 439 440 blkg = container_of(ent, struct blkcg_gq, q_node); 441 return &blkg->rl; 442} 443 444static int blkcg_reset_stats(struct cgroup_subsys_state *css, 445 struct cftype *cftype, u64 val) 446{ 447 struct blkcg *blkcg = css_to_blkcg(css); 448 struct blkcg_gq *blkg; 449 int i; 450 451 /* 452 * XXX: We invoke cgroup_add/rm_cftypes() under blkcg_pol_mutex 453 * which ends up putting cgroup's internal cgroup_tree_mutex under 454 * it; however, cgroup_tree_mutex is nested above cgroup file 455 * active protection and grabbing blkcg_pol_mutex from a cgroup 456 * file operation creates a possible circular dependency. cgroup 457 * internal locking is planned to go through further simplification 458 * and this issue should go away soon. For now, let's trylock 459 * blkcg_pol_mutex and restart the write on failure. 460 * 461 * http://lkml.kernel.org/g/5363C04B.4010400@oracle.com 462 */ 463 if (!mutex_trylock(&blkcg_pol_mutex)) 464 return restart_syscall(); 465 spin_lock_irq(&blkcg->lock); 466 467 /* 468 * Note that stat reset is racy - it doesn't synchronize against 469 * stat updates. This is a debug feature which shouldn't exist 470 * anyway. If you get hit by a race, retry. 471 */ 472 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { 473 for (i = 0; i < BLKCG_MAX_POLS; i++) { 474 struct blkcg_policy *pol = blkcg_policy[i]; 475 476 if (blkcg_policy_enabled(blkg->q, pol) && 477 pol->pd_reset_stats_fn) 478 pol->pd_reset_stats_fn(blkg); 479 } 480 } 481 482 spin_unlock_irq(&blkcg->lock); 483 mutex_unlock(&blkcg_pol_mutex); 484 return 0; 485} 486 487static const char *blkg_dev_name(struct blkcg_gq *blkg) 488{ 489 /* some drivers (floppy) instantiate a queue w/o disk registered */ 490 if (blkg->q->backing_dev_info.dev) 491 return dev_name(blkg->q->backing_dev_info.dev); 492 return NULL; 493} 494 495/** 496 * blkcg_print_blkgs - helper for printing per-blkg data 497 * @sf: seq_file to print to 498 * @blkcg: blkcg of interest 499 * @prfill: fill function to print out a blkg 500 * @pol: policy in question 501 * @data: data to be passed to @prfill 502 * @show_total: to print out sum of prfill return values or not 503 * 504 * This function invokes @prfill on each blkg of @blkcg if pd for the 505 * policy specified by @pol exists. @prfill is invoked with @sf, the 506 * policy data and @data and the matching queue lock held. If @show_total 507 * is %true, the sum of the return values from @prfill is printed with 508 * "Total" label at the end. 509 * 510 * This is to be used to construct print functions for 511 * cftype->read_seq_string method. 512 */ 513void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, 514 u64 (*prfill)(struct seq_file *, 515 struct blkg_policy_data *, int), 516 const struct blkcg_policy *pol, int data, 517 bool show_total) 518{ 519 struct blkcg_gq *blkg; 520 u64 total = 0; 521 522 rcu_read_lock(); 523 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { 524 spin_lock_irq(blkg->q->queue_lock); 525 if (blkcg_policy_enabled(blkg->q, pol)) 526 total += prfill(sf, blkg->pd[pol->plid], data); 527 spin_unlock_irq(blkg->q->queue_lock); 528 } 529 rcu_read_unlock(); 530 531 if (show_total) 532 seq_printf(sf, "Total %llu\n", (unsigned long long)total); 533} 534EXPORT_SYMBOL_GPL(blkcg_print_blkgs); 535 536/** 537 * __blkg_prfill_u64 - prfill helper for a single u64 value 538 * @sf: seq_file to print to 539 * @pd: policy private data of interest 540 * @v: value to print 541 * 542 * Print @v to @sf for the device assocaited with @pd. 543 */ 544u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v) 545{ 546 const char *dname = blkg_dev_name(pd->blkg); 547 548 if (!dname) 549 return 0; 550 551 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v); 552 return v; 553} 554EXPORT_SYMBOL_GPL(__blkg_prfill_u64); 555 556/** 557 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat 558 * @sf: seq_file to print to 559 * @pd: policy private data of interest 560 * @rwstat: rwstat to print 561 * 562 * Print @rwstat to @sf for the device assocaited with @pd. 563 */ 564u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 565 const struct blkg_rwstat *rwstat) 566{ 567 static const char *rwstr[] = { 568 [BLKG_RWSTAT_READ] = "Read", 569 [BLKG_RWSTAT_WRITE] = "Write", 570 [BLKG_RWSTAT_SYNC] = "Sync", 571 [BLKG_RWSTAT_ASYNC] = "Async", 572 }; 573 const char *dname = blkg_dev_name(pd->blkg); 574 u64 v; 575 int i; 576 577 if (!dname) 578 return 0; 579 580 for (i = 0; i < BLKG_RWSTAT_NR; i++) 581 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i], 582 (unsigned long long)rwstat->cnt[i]); 583 584 v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE]; 585 seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v); 586 return v; 587} 588EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat); 589 590/** 591 * blkg_prfill_stat - prfill callback for blkg_stat 592 * @sf: seq_file to print to 593 * @pd: policy private data of interest 594 * @off: offset to the blkg_stat in @pd 595 * 596 * prfill callback for printing a blkg_stat. 597 */ 598u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off) 599{ 600 return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off)); 601} 602EXPORT_SYMBOL_GPL(blkg_prfill_stat); 603 604/** 605 * blkg_prfill_rwstat - prfill callback for blkg_rwstat 606 * @sf: seq_file to print to 607 * @pd: policy private data of interest 608 * @off: offset to the blkg_rwstat in @pd 609 * 610 * prfill callback for printing a blkg_rwstat. 611 */ 612u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 613 int off) 614{ 615 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off); 616 617 return __blkg_prfill_rwstat(sf, pd, &rwstat); 618} 619EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); 620 621/** 622 * blkg_stat_recursive_sum - collect hierarchical blkg_stat 623 * @pd: policy private data of interest 624 * @off: offset to the blkg_stat in @pd 625 * 626 * Collect the blkg_stat specified by @off from @pd and all its online 627 * descendants and return the sum. The caller must be holding the queue 628 * lock for online tests. 629 */ 630u64 blkg_stat_recursive_sum(struct blkg_policy_data *pd, int off) 631{ 632 struct blkcg_policy *pol = blkcg_policy[pd->plid]; 633 struct blkcg_gq *pos_blkg; 634 struct cgroup_subsys_state *pos_css; 635 u64 sum = 0; 636 637 lockdep_assert_held(pd->blkg->q->queue_lock); 638 639 rcu_read_lock(); 640 blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) { 641 struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol); 642 struct blkg_stat *stat = (void *)pos_pd + off; 643 644 if (pos_blkg->online) 645 sum += blkg_stat_read(stat); 646 } 647 rcu_read_unlock(); 648 649 return sum; 650} 651EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum); 652 653/** 654 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat 655 * @pd: policy private data of interest 656 * @off: offset to the blkg_stat in @pd 657 * 658 * Collect the blkg_rwstat specified by @off from @pd and all its online 659 * descendants and return the sum. The caller must be holding the queue 660 * lock for online tests. 661 */ 662struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkg_policy_data *pd, 663 int off) 664{ 665 struct blkcg_policy *pol = blkcg_policy[pd->plid]; 666 struct blkcg_gq *pos_blkg; 667 struct cgroup_subsys_state *pos_css; 668 struct blkg_rwstat sum = { }; 669 int i; 670 671 lockdep_assert_held(pd->blkg->q->queue_lock); 672 673 rcu_read_lock(); 674 blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) { 675 struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol); 676 struct blkg_rwstat *rwstat = (void *)pos_pd + off; 677 struct blkg_rwstat tmp; 678 679 if (!pos_blkg->online) 680 continue; 681 682 tmp = blkg_rwstat_read(rwstat); 683 684 for (i = 0; i < BLKG_RWSTAT_NR; i++) 685 sum.cnt[i] += tmp.cnt[i]; 686 } 687 rcu_read_unlock(); 688 689 return sum; 690} 691EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum); 692 693/** 694 * blkg_conf_prep - parse and prepare for per-blkg config update 695 * @blkcg: target block cgroup 696 * @pol: target policy 697 * @input: input string 698 * @ctx: blkg_conf_ctx to be filled 699 * 700 * Parse per-blkg config update from @input and initialize @ctx with the 701 * result. @ctx->blkg points to the blkg to be updated and @ctx->v the new 702 * value. This function returns with RCU read lock and queue lock held and 703 * must be paired with blkg_conf_finish(). 704 */ 705int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, 706 const char *input, struct blkg_conf_ctx *ctx) 707 __acquires(rcu) __acquires(disk->queue->queue_lock) 708{ 709 struct gendisk *disk; 710 struct blkcg_gq *blkg; 711 unsigned int major, minor; 712 unsigned long long v; 713 int part, ret; 714 715 if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3) 716 return -EINVAL; 717 718 disk = get_gendisk(MKDEV(major, minor), &part); 719 if (!disk) 720 return -EINVAL; 721 if (part) { 722 put_disk(disk); 723 return -EINVAL; 724 } 725 726 rcu_read_lock(); 727 spin_lock_irq(disk->queue->queue_lock); 728 729 if (blkcg_policy_enabled(disk->queue, pol)) 730 blkg = blkg_lookup_create(blkcg, disk->queue); 731 else 732 blkg = ERR_PTR(-EINVAL); 733 734 if (IS_ERR(blkg)) { 735 ret = PTR_ERR(blkg); 736 rcu_read_unlock(); 737 spin_unlock_irq(disk->queue->queue_lock); 738 put_disk(disk); 739 /* 740 * If queue was bypassing, we should retry. Do so after a 741 * short msleep(). It isn't strictly necessary but queue 742 * can be bypassing for some time and it's always nice to 743 * avoid busy looping. 744 */ 745 if (ret == -EBUSY) { 746 msleep(10); 747 ret = restart_syscall(); 748 } 749 return ret; 750 } 751 752 ctx->disk = disk; 753 ctx->blkg = blkg; 754 ctx->v = v; 755 return 0; 756} 757EXPORT_SYMBOL_GPL(blkg_conf_prep); 758 759/** 760 * blkg_conf_finish - finish up per-blkg config update 761 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep() 762 * 763 * Finish up after per-blkg config update. This function must be paired 764 * with blkg_conf_prep(). 765 */ 766void blkg_conf_finish(struct blkg_conf_ctx *ctx) 767 __releases(ctx->disk->queue->queue_lock) __releases(rcu) 768{ 769 spin_unlock_irq(ctx->disk->queue->queue_lock); 770 rcu_read_unlock(); 771 put_disk(ctx->disk); 772} 773EXPORT_SYMBOL_GPL(blkg_conf_finish); 774 775struct cftype blkcg_files[] = { 776 { 777 .name = "reset_stats", 778 .write_u64 = blkcg_reset_stats, 779 }, 780 { } /* terminate */ 781}; 782 783/** 784 * blkcg_css_offline - cgroup css_offline callback 785 * @css: css of interest 786 * 787 * This function is called when @css is about to go away and responsible 788 * for shooting down all blkgs associated with @css. blkgs should be 789 * removed while holding both q and blkcg locks. As blkcg lock is nested 790 * inside q lock, this function performs reverse double lock dancing. 791 * 792 * This is the blkcg counterpart of ioc_release_fn(). 793 */ 794static void blkcg_css_offline(struct cgroup_subsys_state *css) 795{ 796 struct blkcg *blkcg = css_to_blkcg(css); 797 798 spin_lock_irq(&blkcg->lock); 799 800 while (!hlist_empty(&blkcg->blkg_list)) { 801 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, 802 struct blkcg_gq, blkcg_node); 803 struct request_queue *q = blkg->q; 804 805 if (spin_trylock(q->queue_lock)) { 806 blkg_destroy(blkg); 807 spin_unlock(q->queue_lock); 808 } else { 809 spin_unlock_irq(&blkcg->lock); 810 cpu_relax(); 811 spin_lock_irq(&blkcg->lock); 812 } 813 } 814 815 spin_unlock_irq(&blkcg->lock); 816} 817 818static void blkcg_css_free(struct cgroup_subsys_state *css) 819{ 820 struct blkcg *blkcg = css_to_blkcg(css); 821 822 if (blkcg != &blkcg_root) 823 kfree(blkcg); 824} 825 826static struct cgroup_subsys_state * 827blkcg_css_alloc(struct cgroup_subsys_state *parent_css) 828{ 829 struct blkcg *blkcg; 830 831 if (!parent_css) { 832 blkcg = &blkcg_root; 833 goto done; 834 } 835 836 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); 837 if (!blkcg) 838 return ERR_PTR(-ENOMEM); 839 840 blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT; 841 blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT; 842done: 843 spin_lock_init(&blkcg->lock); 844 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC); 845 INIT_HLIST_HEAD(&blkcg->blkg_list); 846 847 return &blkcg->css; 848} 849 850/** 851 * blkcg_init_queue - initialize blkcg part of request queue 852 * @q: request_queue to initialize 853 * 854 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg 855 * part of new request_queue @q. 856 * 857 * RETURNS: 858 * 0 on success, -errno on failure. 859 */ 860int blkcg_init_queue(struct request_queue *q) 861{ 862 might_sleep(); 863 864 return blk_throtl_init(q); 865} 866 867/** 868 * blkcg_drain_queue - drain blkcg part of request_queue 869 * @q: request_queue to drain 870 * 871 * Called from blk_drain_queue(). Responsible for draining blkcg part. 872 */ 873void blkcg_drain_queue(struct request_queue *q) 874{ 875 lockdep_assert_held(q->queue_lock); 876 877 /* 878 * @q could be exiting and already have destroyed all blkgs as 879 * indicated by NULL root_blkg. If so, don't confuse policies. 880 */ 881 if (!q->root_blkg) 882 return; 883 884 blk_throtl_drain(q); 885} 886 887/** 888 * blkcg_exit_queue - exit and release blkcg part of request_queue 889 * @q: request_queue being released 890 * 891 * Called from blk_release_queue(). Responsible for exiting blkcg part. 892 */ 893void blkcg_exit_queue(struct request_queue *q) 894{ 895 spin_lock_irq(q->queue_lock); 896 blkg_destroy_all(q); 897 spin_unlock_irq(q->queue_lock); 898 899 blk_throtl_exit(q); 900} 901 902/* 903 * We cannot support shared io contexts, as we have no mean to support 904 * two tasks with the same ioc in two different groups without major rework 905 * of the main cic data structures. For now we allow a task to change 906 * its cgroup only if it's the only owner of its ioc. 907 */ 908static int blkcg_can_attach(struct cgroup_subsys_state *css, 909 struct cgroup_taskset *tset) 910{ 911 struct task_struct *task; 912 struct io_context *ioc; 913 int ret = 0; 914 915 /* task_lock() is needed to avoid races with exit_io_context() */ 916 cgroup_taskset_for_each(task, tset) { 917 task_lock(task); 918 ioc = task->io_context; 919 if (ioc && atomic_read(&ioc->nr_tasks) > 1) 920 ret = -EINVAL; 921 task_unlock(task); 922 if (ret) 923 break; 924 } 925 return ret; 926} 927 928struct cgroup_subsys blkio_cgrp_subsys = { 929 .css_alloc = blkcg_css_alloc, 930 .css_offline = blkcg_css_offline, 931 .css_free = blkcg_css_free, 932 .can_attach = blkcg_can_attach, 933 .legacy_cftypes = blkcg_files, 934#ifdef CONFIG_MEMCG 935 /* 936 * This ensures that, if available, memcg is automatically enabled 937 * together on the default hierarchy so that the owner cgroup can 938 * be retrieved from writeback pages. 939 */ 940 .depends_on = 1 << memory_cgrp_id, 941#endif 942}; 943EXPORT_SYMBOL_GPL(blkio_cgrp_subsys); 944 945/** 946 * blkcg_activate_policy - activate a blkcg policy on a request_queue 947 * @q: request_queue of interest 948 * @pol: blkcg policy to activate 949 * 950 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through 951 * bypass mode to populate its blkgs with policy_data for @pol. 952 * 953 * Activation happens with @q bypassed, so nobody would be accessing blkgs 954 * from IO path. Update of each blkg is protected by both queue and blkcg 955 * locks so that holding either lock and testing blkcg_policy_enabled() is 956 * always enough for dereferencing policy data. 957 * 958 * The caller is responsible for synchronizing [de]activations and policy 959 * [un]registerations. Returns 0 on success, -errno on failure. 960 */ 961int blkcg_activate_policy(struct request_queue *q, 962 const struct blkcg_policy *pol) 963{ 964 LIST_HEAD(pds); 965 struct blkcg_gq *blkg, *new_blkg; 966 struct blkg_policy_data *pd, *n; 967 int cnt = 0, ret; 968 bool preloaded; 969 970 if (blkcg_policy_enabled(q, pol)) 971 return 0; 972 973 /* preallocations for root blkg */ 974 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL); 975 if (!new_blkg) 976 return -ENOMEM; 977 978 blk_queue_bypass_start(q); 979 980 preloaded = !radix_tree_preload(GFP_KERNEL); 981 982 /* 983 * Make sure the root blkg exists and count the existing blkgs. As 984 * @q is bypassing at this point, blkg_lookup_create() can't be 985 * used. Open code it. 986 */ 987 spin_lock_irq(q->queue_lock); 988 989 rcu_read_lock(); 990 blkg = __blkg_lookup(&blkcg_root, q, false); 991 if (blkg) 992 blkg_free(new_blkg); 993 else 994 blkg = blkg_create(&blkcg_root, q, new_blkg); 995 rcu_read_unlock(); 996 997 if (preloaded) 998 radix_tree_preload_end(); 999 1000 if (IS_ERR(blkg)) { 1001 ret = PTR_ERR(blkg); 1002 goto out_unlock; 1003 } 1004 1005 list_for_each_entry(blkg, &q->blkg_list, q_node) 1006 cnt++; 1007 1008 spin_unlock_irq(q->queue_lock); 1009 1010 /* allocate policy_data for all existing blkgs */ 1011 while (cnt--) { 1012 pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node); 1013 if (!pd) { 1014 ret = -ENOMEM; 1015 goto out_free; 1016 } 1017 list_add_tail(&pd->alloc_node, &pds); 1018 } 1019 1020 /* 1021 * Install the allocated pds. With @q bypassing, no new blkg 1022 * should have been created while the queue lock was dropped. 1023 */ 1024 spin_lock_irq(q->queue_lock); 1025 1026 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1027 if (WARN_ON(list_empty(&pds))) { 1028 /* umm... this shouldn't happen, just abort */ 1029 ret = -ENOMEM; 1030 goto out_unlock; 1031 } 1032 pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node); 1033 list_del_init(&pd->alloc_node); 1034 1035 /* grab blkcg lock too while installing @pd on @blkg */ 1036 spin_lock(&blkg->blkcg->lock); 1037 1038 blkg->pd[pol->plid] = pd; 1039 pd->blkg = blkg; 1040 pd->plid = pol->plid; 1041 pol->pd_init_fn(blkg); 1042 1043 spin_unlock(&blkg->blkcg->lock); 1044 } 1045 1046 __set_bit(pol->plid, q->blkcg_pols); 1047 ret = 0; 1048out_unlock: 1049 spin_unlock_irq(q->queue_lock); 1050out_free: 1051 blk_queue_bypass_end(q); 1052 list_for_each_entry_safe(pd, n, &pds, alloc_node) 1053 kfree(pd); 1054 return ret; 1055} 1056EXPORT_SYMBOL_GPL(blkcg_activate_policy); 1057 1058/** 1059 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue 1060 * @q: request_queue of interest 1061 * @pol: blkcg policy to deactivate 1062 * 1063 * Deactivate @pol on @q. Follows the same synchronization rules as 1064 * blkcg_activate_policy(). 1065 */ 1066void blkcg_deactivate_policy(struct request_queue *q, 1067 const struct blkcg_policy *pol) 1068{ 1069 struct blkcg_gq *blkg; 1070 1071 if (!blkcg_policy_enabled(q, pol)) 1072 return; 1073 1074 blk_queue_bypass_start(q); 1075 spin_lock_irq(q->queue_lock); 1076 1077 __clear_bit(pol->plid, q->blkcg_pols); 1078 1079 /* if no policy is left, no need for blkgs - shoot them down */ 1080 if (bitmap_empty(q->blkcg_pols, BLKCG_MAX_POLS)) 1081 blkg_destroy_all(q); 1082 1083 list_for_each_entry(blkg, &q->blkg_list, q_node) { 1084 /* grab blkcg lock too while removing @pd from @blkg */ 1085 spin_lock(&blkg->blkcg->lock); 1086 1087 if (pol->pd_offline_fn) 1088 pol->pd_offline_fn(blkg); 1089 if (pol->pd_exit_fn) 1090 pol->pd_exit_fn(blkg); 1091 1092 kfree(blkg->pd[pol->plid]); 1093 blkg->pd[pol->plid] = NULL; 1094 1095 spin_unlock(&blkg->blkcg->lock); 1096 } 1097 1098 spin_unlock_irq(q->queue_lock); 1099 blk_queue_bypass_end(q); 1100} 1101EXPORT_SYMBOL_GPL(blkcg_deactivate_policy); 1102 1103/** 1104 * blkcg_policy_register - register a blkcg policy 1105 * @pol: blkcg policy to register 1106 * 1107 * Register @pol with blkcg core. Might sleep and @pol may be modified on 1108 * successful registration. Returns 0 on success and -errno on failure. 1109 */ 1110int blkcg_policy_register(struct blkcg_policy *pol) 1111{ 1112 int i, ret; 1113 1114 if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data))) 1115 return -EINVAL; 1116 1117 mutex_lock(&blkcg_pol_mutex); 1118 1119 /* find an empty slot */ 1120 ret = -ENOSPC; 1121 for (i = 0; i < BLKCG_MAX_POLS; i++) 1122 if (!blkcg_policy[i]) 1123 break; 1124 if (i >= BLKCG_MAX_POLS) 1125 goto out_unlock; 1126 1127 /* register and update blkgs */ 1128 pol->plid = i; 1129 blkcg_policy[i] = pol; 1130 1131 /* everything is in place, add intf files for the new policy */ 1132 if (pol->cftypes) 1133 WARN_ON(cgroup_add_legacy_cftypes(&blkio_cgrp_subsys, 1134 pol->cftypes)); 1135 ret = 0; 1136out_unlock: 1137 mutex_unlock(&blkcg_pol_mutex); 1138 return ret; 1139} 1140EXPORT_SYMBOL_GPL(blkcg_policy_register); 1141 1142/** 1143 * blkcg_policy_unregister - unregister a blkcg policy 1144 * @pol: blkcg policy to unregister 1145 * 1146 * Undo blkcg_policy_register(@pol). Might sleep. 1147 */ 1148void blkcg_policy_unregister(struct blkcg_policy *pol) 1149{ 1150 mutex_lock(&blkcg_pol_mutex); 1151 1152 if (WARN_ON(blkcg_policy[pol->plid] != pol)) 1153 goto out_unlock; 1154 1155 /* kill the intf files first */ 1156 if (pol->cftypes) 1157 cgroup_rm_cftypes(pol->cftypes); 1158 1159 /* unregister and update blkgs */ 1160 blkcg_policy[pol->plid] = NULL; 1161out_unlock: 1162 mutex_unlock(&blkcg_pol_mutex); 1163} 1164EXPORT_SYMBOL_GPL(blkcg_policy_unregister); 1165