1/* 2 * Copyright (C) 2001 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8#include "dm.h" 9 10#include <linux/module.h> 11#include <linux/vmalloc.h> 12#include <linux/blkdev.h> 13#include <linux/namei.h> 14#include <linux/ctype.h> 15#include <linux/string.h> 16#include <linux/slab.h> 17#include <linux/interrupt.h> 18#include <linux/mutex.h> 19#include <linux/delay.h> 20#include <linux/atomic.h> 21#include <linux/blk-mq.h> 22#include <linux/mount.h> 23 24#define DM_MSG_PREFIX "table" 25 26#define MAX_DEPTH 16 27#define NODE_SIZE L1_CACHE_BYTES 28#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t)) 29#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1) 30 31struct dm_table { 32 struct mapped_device *md; 33 unsigned type; 34 35 /* btree table */ 36 unsigned int depth; 37 unsigned int counts[MAX_DEPTH]; /* in nodes */ 38 sector_t *index[MAX_DEPTH]; 39 40 unsigned int num_targets; 41 unsigned int num_allocated; 42 sector_t *highs; 43 struct dm_target *targets; 44 45 struct target_type *immutable_target_type; 46 unsigned integrity_supported:1; 47 unsigned singleton:1; 48 49 /* 50 * Indicates the rw permissions for the new logical 51 * device. This should be a combination of FMODE_READ 52 * and FMODE_WRITE. 53 */ 54 fmode_t mode; 55 56 /* a list of devices used by this table */ 57 struct list_head devices; 58 59 /* events get handed up using this callback */ 60 void (*event_fn)(void *); 61 void *event_context; 62 63 struct dm_md_mempools *mempools; 64 65 struct list_head target_callbacks; 66}; 67 68/* 69 * Similar to ceiling(log_size(n)) 70 */ 71static unsigned int int_log(unsigned int n, unsigned int base) 72{ 73 int result = 0; 74 75 while (n > 1) { 76 n = dm_div_up(n, base); 77 result++; 78 } 79 80 return result; 81} 82 83/* 84 * Calculate the index of the child node of the n'th node k'th key. 85 */ 86static inline unsigned int get_child(unsigned int n, unsigned int k) 87{ 88 return (n * CHILDREN_PER_NODE) + k; 89} 90 91/* 92 * Return the n'th node of level l from table t. 93 */ 94static inline sector_t *get_node(struct dm_table *t, 95 unsigned int l, unsigned int n) 96{ 97 return t->index[l] + (n * KEYS_PER_NODE); 98} 99 100/* 101 * Return the highest key that you could lookup from the n'th 102 * node on level l of the btree. 103 */ 104static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) 105{ 106 for (; l < t->depth - 1; l++) 107 n = get_child(n, CHILDREN_PER_NODE - 1); 108 109 if (n >= t->counts[l]) 110 return (sector_t) - 1; 111 112 return get_node(t, l, n)[KEYS_PER_NODE - 1]; 113} 114 115/* 116 * Fills in a level of the btree based on the highs of the level 117 * below it. 118 */ 119static int setup_btree_index(unsigned int l, struct dm_table *t) 120{ 121 unsigned int n, k; 122 sector_t *node; 123 124 for (n = 0U; n < t->counts[l]; n++) { 125 node = get_node(t, l, n); 126 127 for (k = 0U; k < KEYS_PER_NODE; k++) 128 node[k] = high(t, l + 1, get_child(n, k)); 129 } 130 131 return 0; 132} 133 134void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size) 135{ 136 unsigned long size; 137 void *addr; 138 139 /* 140 * Check that we're not going to overflow. 141 */ 142 if (nmemb > (ULONG_MAX / elem_size)) 143 return NULL; 144 145 size = nmemb * elem_size; 146 addr = vzalloc(size); 147 148 return addr; 149} 150EXPORT_SYMBOL(dm_vcalloc); 151 152/* 153 * highs, and targets are managed as dynamic arrays during a 154 * table load. 155 */ 156static int alloc_targets(struct dm_table *t, unsigned int num) 157{ 158 sector_t *n_highs; 159 struct dm_target *n_targets; 160 161 /* 162 * Allocate both the target array and offset array at once. 163 * Append an empty entry to catch sectors beyond the end of 164 * the device. 165 */ 166 n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) + 167 sizeof(sector_t)); 168 if (!n_highs) 169 return -ENOMEM; 170 171 n_targets = (struct dm_target *) (n_highs + num); 172 173 memset(n_highs, -1, sizeof(*n_highs) * num); 174 vfree(t->highs); 175 176 t->num_allocated = num; 177 t->highs = n_highs; 178 t->targets = n_targets; 179 180 return 0; 181} 182 183int dm_table_create(struct dm_table **result, fmode_t mode, 184 unsigned num_targets, struct mapped_device *md) 185{ 186 struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL); 187 188 if (!t) 189 return -ENOMEM; 190 191 INIT_LIST_HEAD(&t->devices); 192 INIT_LIST_HEAD(&t->target_callbacks); 193 194 if (!num_targets) 195 num_targets = KEYS_PER_NODE; 196 197 num_targets = dm_round_up(num_targets, KEYS_PER_NODE); 198 199 if (!num_targets) { 200 kfree(t); 201 return -ENOMEM; 202 } 203 204 if (alloc_targets(t, num_targets)) { 205 kfree(t); 206 return -ENOMEM; 207 } 208 209 t->mode = mode; 210 t->md = md; 211 *result = t; 212 return 0; 213} 214 215static void free_devices(struct list_head *devices, struct mapped_device *md) 216{ 217 struct list_head *tmp, *next; 218 219 list_for_each_safe(tmp, next, devices) { 220 struct dm_dev_internal *dd = 221 list_entry(tmp, struct dm_dev_internal, list); 222 DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s", 223 dm_device_name(md), dd->dm_dev->name); 224 dm_put_table_device(md, dd->dm_dev); 225 kfree(dd); 226 } 227} 228 229void dm_table_destroy(struct dm_table *t) 230{ 231 unsigned int i; 232 233 if (!t) 234 return; 235 236 /* free the indexes */ 237 if (t->depth >= 2) 238 vfree(t->index[t->depth - 2]); 239 240 /* free the targets */ 241 for (i = 0; i < t->num_targets; i++) { 242 struct dm_target *tgt = t->targets + i; 243 244 if (tgt->type->dtr) 245 tgt->type->dtr(tgt); 246 247 dm_put_target_type(tgt->type); 248 } 249 250 vfree(t->highs); 251 252 /* free the device list */ 253 free_devices(&t->devices, t->md); 254 255 dm_free_md_mempools(t->mempools); 256 257 kfree(t); 258} 259 260/* 261 * See if we've already got a device in the list. 262 */ 263static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev) 264{ 265 struct dm_dev_internal *dd; 266 267 list_for_each_entry (dd, l, list) 268 if (dd->dm_dev->bdev->bd_dev == dev) 269 return dd; 270 271 return NULL; 272} 273 274/* 275 * If possible, this checks an area of a destination device is invalid. 276 */ 277static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, 278 sector_t start, sector_t len, void *data) 279{ 280 struct request_queue *q; 281 struct queue_limits *limits = data; 282 struct block_device *bdev = dev->bdev; 283 sector_t dev_size = 284 i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; 285 unsigned short logical_block_size_sectors = 286 limits->logical_block_size >> SECTOR_SHIFT; 287 char b[BDEVNAME_SIZE]; 288 289 /* 290 * Some devices exist without request functions, 291 * such as loop devices not yet bound to backing files. 292 * Forbid the use of such devices. 293 */ 294 q = bdev_get_queue(bdev); 295 if (!q || !q->make_request_fn) { 296 DMWARN("%s: %s is not yet initialised: " 297 "start=%llu, len=%llu, dev_size=%llu", 298 dm_device_name(ti->table->md), bdevname(bdev, b), 299 (unsigned long long)start, 300 (unsigned long long)len, 301 (unsigned long long)dev_size); 302 return 1; 303 } 304 305 if (!dev_size) 306 return 0; 307 308 if ((start >= dev_size) || (start + len > dev_size)) { 309 DMWARN("%s: %s too small for target: " 310 "start=%llu, len=%llu, dev_size=%llu", 311 dm_device_name(ti->table->md), bdevname(bdev, b), 312 (unsigned long long)start, 313 (unsigned long long)len, 314 (unsigned long long)dev_size); 315 return 1; 316 } 317 318 if (logical_block_size_sectors <= 1) 319 return 0; 320 321 if (start & (logical_block_size_sectors - 1)) { 322 DMWARN("%s: start=%llu not aligned to h/w " 323 "logical block size %u of %s", 324 dm_device_name(ti->table->md), 325 (unsigned long long)start, 326 limits->logical_block_size, bdevname(bdev, b)); 327 return 1; 328 } 329 330 if (len & (logical_block_size_sectors - 1)) { 331 DMWARN("%s: len=%llu not aligned to h/w " 332 "logical block size %u of %s", 333 dm_device_name(ti->table->md), 334 (unsigned long long)len, 335 limits->logical_block_size, bdevname(bdev, b)); 336 return 1; 337 } 338 339 return 0; 340} 341 342/* 343 * This upgrades the mode on an already open dm_dev, being 344 * careful to leave things as they were if we fail to reopen the 345 * device and not to touch the existing bdev field in case 346 * it is accessed concurrently inside dm_table_any_congested(). 347 */ 348static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode, 349 struct mapped_device *md) 350{ 351 int r; 352 struct dm_dev *old_dev, *new_dev; 353 354 old_dev = dd->dm_dev; 355 356 r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev, 357 dd->dm_dev->mode | new_mode, &new_dev); 358 if (r) 359 return r; 360 361 dd->dm_dev = new_dev; 362 dm_put_table_device(md, old_dev); 363 364 return 0; 365} 366 367/* 368 * Add a device to the list, or just increment the usage count if 369 * it's already present. 370 */ 371int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode, 372 struct dm_dev **result) 373{ 374 int r; 375 dev_t uninitialized_var(dev); 376 struct dm_dev_internal *dd; 377 struct dm_table *t = ti->table; 378 struct block_device *bdev; 379 380 BUG_ON(!t); 381 382 /* convert the path to a device */ 383 bdev = lookup_bdev(path); 384 if (IS_ERR(bdev)) { 385 dev = name_to_dev_t(path); 386 if (!dev) 387 return -ENODEV; 388 } else { 389 dev = bdev->bd_dev; 390 bdput(bdev); 391 } 392 393 dd = find_device(&t->devices, dev); 394 if (!dd) { 395 dd = kmalloc(sizeof(*dd), GFP_KERNEL); 396 if (!dd) 397 return -ENOMEM; 398 399 if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) { 400 kfree(dd); 401 return r; 402 } 403 404 atomic_set(&dd->count, 0); 405 list_add(&dd->list, &t->devices); 406 407 } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) { 408 r = upgrade_mode(dd, mode, t->md); 409 if (r) 410 return r; 411 } 412 atomic_inc(&dd->count); 413 414 *result = dd->dm_dev; 415 return 0; 416} 417EXPORT_SYMBOL(dm_get_device); 418 419static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, 420 sector_t start, sector_t len, void *data) 421{ 422 struct queue_limits *limits = data; 423 struct block_device *bdev = dev->bdev; 424 struct request_queue *q = bdev_get_queue(bdev); 425 char b[BDEVNAME_SIZE]; 426 427 if (unlikely(!q)) { 428 DMWARN("%s: Cannot set limits for nonexistent device %s", 429 dm_device_name(ti->table->md), bdevname(bdev, b)); 430 return 0; 431 } 432 433 if (bdev_stack_limits(limits, bdev, start) < 0) 434 DMWARN("%s: adding target device %s caused an alignment inconsistency: " 435 "physical_block_size=%u, logical_block_size=%u, " 436 "alignment_offset=%u, start=%llu", 437 dm_device_name(ti->table->md), bdevname(bdev, b), 438 q->limits.physical_block_size, 439 q->limits.logical_block_size, 440 q->limits.alignment_offset, 441 (unsigned long long) start << SECTOR_SHIFT); 442 443 /* 444 * Check if merge fn is supported. 445 * If not we'll force DM to use PAGE_SIZE or 446 * smaller I/O, just to be safe. 447 */ 448 if (dm_queue_merge_is_compulsory(q) && !ti->type->merge) 449 blk_limits_max_hw_sectors(limits, 450 (unsigned int) (PAGE_SIZE >> 9)); 451 return 0; 452} 453 454/* 455 * Decrement a device's use count and remove it if necessary. 456 */ 457void dm_put_device(struct dm_target *ti, struct dm_dev *d) 458{ 459 int found = 0; 460 struct list_head *devices = &ti->table->devices; 461 struct dm_dev_internal *dd; 462 463 list_for_each_entry(dd, devices, list) { 464 if (dd->dm_dev == d) { 465 found = 1; 466 break; 467 } 468 } 469 if (!found) { 470 DMWARN("%s: device %s not in table devices list", 471 dm_device_name(ti->table->md), d->name); 472 return; 473 } 474 if (atomic_dec_and_test(&dd->count)) { 475 dm_put_table_device(ti->table->md, d); 476 list_del(&dd->list); 477 kfree(dd); 478 } 479} 480EXPORT_SYMBOL(dm_put_device); 481 482/* 483 * Checks to see if the target joins onto the end of the table. 484 */ 485static int adjoin(struct dm_table *table, struct dm_target *ti) 486{ 487 struct dm_target *prev; 488 489 if (!table->num_targets) 490 return !ti->begin; 491 492 prev = &table->targets[table->num_targets - 1]; 493 return (ti->begin == (prev->begin + prev->len)); 494} 495 496/* 497 * Used to dynamically allocate the arg array. 498 * 499 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must 500 * process messages even if some device is suspended. These messages have a 501 * small fixed number of arguments. 502 * 503 * On the other hand, dm-switch needs to process bulk data using messages and 504 * excessive use of GFP_NOIO could cause trouble. 505 */ 506static char **realloc_argv(unsigned *array_size, char **old_argv) 507{ 508 char **argv; 509 unsigned new_size; 510 gfp_t gfp; 511 512 if (*array_size) { 513 new_size = *array_size * 2; 514 gfp = GFP_KERNEL; 515 } else { 516 new_size = 8; 517 gfp = GFP_NOIO; 518 } 519 argv = kmalloc(new_size * sizeof(*argv), gfp); 520 if (argv) { 521 memcpy(argv, old_argv, *array_size * sizeof(*argv)); 522 *array_size = new_size; 523 } 524 525 kfree(old_argv); 526 return argv; 527} 528 529/* 530 * Destructively splits up the argument list to pass to ctr. 531 */ 532int dm_split_args(int *argc, char ***argvp, char *input) 533{ 534 char *start, *end = input, *out, **argv = NULL; 535 unsigned array_size = 0; 536 537 *argc = 0; 538 539 if (!input) { 540 *argvp = NULL; 541 return 0; 542 } 543 544 argv = realloc_argv(&array_size, argv); 545 if (!argv) 546 return -ENOMEM; 547 548 while (1) { 549 /* Skip whitespace */ 550 start = skip_spaces(end); 551 552 if (!*start) 553 break; /* success, we hit the end */ 554 555 /* 'out' is used to remove any back-quotes */ 556 end = out = start; 557 while (*end) { 558 /* Everything apart from '\0' can be quoted */ 559 if (*end == '\\' && *(end + 1)) { 560 *out++ = *(end + 1); 561 end += 2; 562 continue; 563 } 564 565 if (isspace(*end)) 566 break; /* end of token */ 567 568 *out++ = *end++; 569 } 570 571 /* have we already filled the array ? */ 572 if ((*argc + 1) > array_size) { 573 argv = realloc_argv(&array_size, argv); 574 if (!argv) 575 return -ENOMEM; 576 } 577 578 /* we know this is whitespace */ 579 if (*end) 580 end++; 581 582 /* terminate the string and put it in the array */ 583 *out = '\0'; 584 argv[*argc] = start; 585 (*argc)++; 586 } 587 588 *argvp = argv; 589 return 0; 590} 591 592/* 593 * Impose necessary and sufficient conditions on a devices's table such 594 * that any incoming bio which respects its logical_block_size can be 595 * processed successfully. If it falls across the boundary between 596 * two or more targets, the size of each piece it gets split into must 597 * be compatible with the logical_block_size of the target processing it. 598 */ 599static int validate_hardware_logical_block_alignment(struct dm_table *table, 600 struct queue_limits *limits) 601{ 602 /* 603 * This function uses arithmetic modulo the logical_block_size 604 * (in units of 512-byte sectors). 605 */ 606 unsigned short device_logical_block_size_sects = 607 limits->logical_block_size >> SECTOR_SHIFT; 608 609 /* 610 * Offset of the start of the next table entry, mod logical_block_size. 611 */ 612 unsigned short next_target_start = 0; 613 614 /* 615 * Given an aligned bio that extends beyond the end of a 616 * target, how many sectors must the next target handle? 617 */ 618 unsigned short remaining = 0; 619 620 struct dm_target *uninitialized_var(ti); 621 struct queue_limits ti_limits; 622 unsigned i = 0; 623 624 /* 625 * Check each entry in the table in turn. 626 */ 627 while (i < dm_table_get_num_targets(table)) { 628 ti = dm_table_get_target(table, i++); 629 630 blk_set_stacking_limits(&ti_limits); 631 632 /* combine all target devices' limits */ 633 if (ti->type->iterate_devices) 634 ti->type->iterate_devices(ti, dm_set_device_limits, 635 &ti_limits); 636 637 /* 638 * If the remaining sectors fall entirely within this 639 * table entry are they compatible with its logical_block_size? 640 */ 641 if (remaining < ti->len && 642 remaining & ((ti_limits.logical_block_size >> 643 SECTOR_SHIFT) - 1)) 644 break; /* Error */ 645 646 next_target_start = 647 (unsigned short) ((next_target_start + ti->len) & 648 (device_logical_block_size_sects - 1)); 649 remaining = next_target_start ? 650 device_logical_block_size_sects - next_target_start : 0; 651 } 652 653 if (remaining) { 654 DMWARN("%s: table line %u (start sect %llu len %llu) " 655 "not aligned to h/w logical block size %u", 656 dm_device_name(table->md), i, 657 (unsigned long long) ti->begin, 658 (unsigned long long) ti->len, 659 limits->logical_block_size); 660 return -EINVAL; 661 } 662 663 return 0; 664} 665 666int dm_table_add_target(struct dm_table *t, const char *type, 667 sector_t start, sector_t len, char *params) 668{ 669 int r = -EINVAL, argc; 670 char **argv; 671 struct dm_target *tgt; 672 673 if (t->singleton) { 674 DMERR("%s: target type %s must appear alone in table", 675 dm_device_name(t->md), t->targets->type->name); 676 return -EINVAL; 677 } 678 679 BUG_ON(t->num_targets >= t->num_allocated); 680 681 tgt = t->targets + t->num_targets; 682 memset(tgt, 0, sizeof(*tgt)); 683 684 if (!len) { 685 DMERR("%s: zero-length target", dm_device_name(t->md)); 686 return -EINVAL; 687 } 688 689 tgt->type = dm_get_target_type(type); 690 if (!tgt->type) { 691 DMERR("%s: %s: unknown target type", dm_device_name(t->md), 692 type); 693 return -EINVAL; 694 } 695 696 if (dm_target_needs_singleton(tgt->type)) { 697 if (t->num_targets) { 698 DMERR("%s: target type %s must appear alone in table", 699 dm_device_name(t->md), type); 700 return -EINVAL; 701 } 702 t->singleton = 1; 703 } 704 705 if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) { 706 DMERR("%s: target type %s may not be included in read-only tables", 707 dm_device_name(t->md), type); 708 return -EINVAL; 709 } 710 711 if (t->immutable_target_type) { 712 if (t->immutable_target_type != tgt->type) { 713 DMERR("%s: immutable target type %s cannot be mixed with other target types", 714 dm_device_name(t->md), t->immutable_target_type->name); 715 return -EINVAL; 716 } 717 } else if (dm_target_is_immutable(tgt->type)) { 718 if (t->num_targets) { 719 DMERR("%s: immutable target type %s cannot be mixed with other target types", 720 dm_device_name(t->md), tgt->type->name); 721 return -EINVAL; 722 } 723 t->immutable_target_type = tgt->type; 724 } 725 726 tgt->table = t; 727 tgt->begin = start; 728 tgt->len = len; 729 tgt->error = "Unknown error"; 730 731 /* 732 * Does this target adjoin the previous one ? 733 */ 734 if (!adjoin(t, tgt)) { 735 tgt->error = "Gap in table"; 736 r = -EINVAL; 737 goto bad; 738 } 739 740 r = dm_split_args(&argc, &argv, params); 741 if (r) { 742 tgt->error = "couldn't split parameters (insufficient memory)"; 743 goto bad; 744 } 745 746 r = tgt->type->ctr(tgt, argc, argv); 747 kfree(argv); 748 if (r) 749 goto bad; 750 751 t->highs[t->num_targets++] = tgt->begin + tgt->len - 1; 752 753 if (!tgt->num_discard_bios && tgt->discards_supported) 754 DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.", 755 dm_device_name(t->md), type); 756 757 return 0; 758 759 bad: 760 DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error); 761 dm_put_target_type(tgt->type); 762 return r; 763} 764 765/* 766 * Target argument parsing helpers. 767 */ 768static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, 769 unsigned *value, char **error, unsigned grouped) 770{ 771 const char *arg_str = dm_shift_arg(arg_set); 772 char dummy; 773 774 if (!arg_str || 775 (sscanf(arg_str, "%u%c", value, &dummy) != 1) || 776 (*value < arg->min) || 777 (*value > arg->max) || 778 (grouped && arg_set->argc < *value)) { 779 *error = arg->error; 780 return -EINVAL; 781 } 782 783 return 0; 784} 785 786int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, 787 unsigned *value, char **error) 788{ 789 return validate_next_arg(arg, arg_set, value, error, 0); 790} 791EXPORT_SYMBOL(dm_read_arg); 792 793int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set, 794 unsigned *value, char **error) 795{ 796 return validate_next_arg(arg, arg_set, value, error, 1); 797} 798EXPORT_SYMBOL(dm_read_arg_group); 799 800const char *dm_shift_arg(struct dm_arg_set *as) 801{ 802 char *r; 803 804 if (as->argc) { 805 as->argc--; 806 r = *as->argv; 807 as->argv++; 808 return r; 809 } 810 811 return NULL; 812} 813EXPORT_SYMBOL(dm_shift_arg); 814 815void dm_consume_args(struct dm_arg_set *as, unsigned num_args) 816{ 817 BUG_ON(as->argc < num_args); 818 as->argc -= num_args; 819 as->argv += num_args; 820} 821EXPORT_SYMBOL(dm_consume_args); 822 823static bool __table_type_request_based(unsigned table_type) 824{ 825 return (table_type == DM_TYPE_REQUEST_BASED || 826 table_type == DM_TYPE_MQ_REQUEST_BASED); 827} 828 829static int dm_table_set_type(struct dm_table *t) 830{ 831 unsigned i; 832 unsigned bio_based = 0, request_based = 0, hybrid = 0; 833 bool use_blk_mq = false; 834 struct dm_target *tgt; 835 struct dm_dev_internal *dd; 836 struct list_head *devices; 837 unsigned live_md_type = dm_get_md_type(t->md); 838 839 for (i = 0; i < t->num_targets; i++) { 840 tgt = t->targets + i; 841 if (dm_target_hybrid(tgt)) 842 hybrid = 1; 843 else if (dm_target_request_based(tgt)) 844 request_based = 1; 845 else 846 bio_based = 1; 847 848 if (bio_based && request_based) { 849 DMWARN("Inconsistent table: different target types" 850 " can't be mixed up"); 851 return -EINVAL; 852 } 853 } 854 855 if (hybrid && !bio_based && !request_based) { 856 /* 857 * The targets can work either way. 858 * Determine the type from the live device. 859 * Default to bio-based if device is new. 860 */ 861 if (__table_type_request_based(live_md_type)) 862 request_based = 1; 863 else 864 bio_based = 1; 865 } 866 867 if (bio_based) { 868 /* We must use this table as bio-based */ 869 t->type = DM_TYPE_BIO_BASED; 870 return 0; 871 } 872 873 BUG_ON(!request_based); /* No targets in this table */ 874 875 /* 876 * Request-based dm supports only tables that have a single target now. 877 * To support multiple targets, request splitting support is needed, 878 * and that needs lots of changes in the block-layer. 879 * (e.g. request completion process for partial completion.) 880 */ 881 if (t->num_targets > 1) { 882 DMWARN("Request-based dm doesn't support multiple targets yet"); 883 return -EINVAL; 884 } 885 886 /* Non-request-stackable devices can't be used for request-based dm */ 887 devices = dm_table_get_devices(t); 888 list_for_each_entry(dd, devices, list) { 889 struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev); 890 891 if (!blk_queue_stackable(q)) { 892 DMERR("table load rejected: including" 893 " non-request-stackable devices"); 894 return -EINVAL; 895 } 896 897 if (q->mq_ops) 898 use_blk_mq = true; 899 } 900 901 if (use_blk_mq) { 902 /* verify _all_ devices in the table are blk-mq devices */ 903 list_for_each_entry(dd, devices, list) 904 if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) { 905 DMERR("table load rejected: not all devices" 906 " are blk-mq request-stackable"); 907 return -EINVAL; 908 } 909 t->type = DM_TYPE_MQ_REQUEST_BASED; 910 911 } else if (list_empty(devices) && __table_type_request_based(live_md_type)) { 912 /* inherit live MD type */ 913 t->type = live_md_type; 914 915 } else 916 t->type = DM_TYPE_REQUEST_BASED; 917 918 return 0; 919} 920 921unsigned dm_table_get_type(struct dm_table *t) 922{ 923 return t->type; 924} 925 926struct target_type *dm_table_get_immutable_target_type(struct dm_table *t) 927{ 928 return t->immutable_target_type; 929} 930 931bool dm_table_request_based(struct dm_table *t) 932{ 933 return __table_type_request_based(dm_table_get_type(t)); 934} 935 936bool dm_table_mq_request_based(struct dm_table *t) 937{ 938 return dm_table_get_type(t) == DM_TYPE_MQ_REQUEST_BASED; 939} 940 941static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md) 942{ 943 unsigned type = dm_table_get_type(t); 944 unsigned per_bio_data_size = 0; 945 struct dm_target *tgt; 946 unsigned i; 947 948 if (unlikely(type == DM_TYPE_NONE)) { 949 DMWARN("no table type is set, can't allocate mempools"); 950 return -EINVAL; 951 } 952 953 if (type == DM_TYPE_BIO_BASED) 954 for (i = 0; i < t->num_targets; i++) { 955 tgt = t->targets + i; 956 per_bio_data_size = max(per_bio_data_size, tgt->per_bio_data_size); 957 } 958 959 t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported, per_bio_data_size); 960 if (!t->mempools) 961 return -ENOMEM; 962 963 return 0; 964} 965 966void dm_table_free_md_mempools(struct dm_table *t) 967{ 968 dm_free_md_mempools(t->mempools); 969 t->mempools = NULL; 970} 971 972struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t) 973{ 974 return t->mempools; 975} 976 977static int setup_indexes(struct dm_table *t) 978{ 979 int i; 980 unsigned int total = 0; 981 sector_t *indexes; 982 983 /* allocate the space for *all* the indexes */ 984 for (i = t->depth - 2; i >= 0; i--) { 985 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE); 986 total += t->counts[i]; 987 } 988 989 indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE); 990 if (!indexes) 991 return -ENOMEM; 992 993 /* set up internal nodes, bottom-up */ 994 for (i = t->depth - 2; i >= 0; i--) { 995 t->index[i] = indexes; 996 indexes += (KEYS_PER_NODE * t->counts[i]); 997 setup_btree_index(i, t); 998 } 999 1000 return 0; 1001} 1002 1003/* 1004 * Builds the btree to index the map. 1005 */ 1006static int dm_table_build_index(struct dm_table *t) 1007{ 1008 int r = 0; 1009 unsigned int leaf_nodes; 1010 1011 /* how many indexes will the btree have ? */ 1012 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); 1013 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); 1014 1015 /* leaf layer has already been set up */ 1016 t->counts[t->depth - 1] = leaf_nodes; 1017 t->index[t->depth - 1] = t->highs; 1018 1019 if (t->depth >= 2) 1020 r = setup_indexes(t); 1021 1022 return r; 1023} 1024 1025/* 1026 * Get a disk whose integrity profile reflects the table's profile. 1027 * If %match_all is true, all devices' profiles must match. 1028 * If %match_all is false, all devices must at least have an 1029 * allocated integrity profile; but uninitialized is ok. 1030 * Returns NULL if integrity support was inconsistent or unavailable. 1031 */ 1032static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t, 1033 bool match_all) 1034{ 1035 struct list_head *devices = dm_table_get_devices(t); 1036 struct dm_dev_internal *dd = NULL; 1037 struct gendisk *prev_disk = NULL, *template_disk = NULL; 1038 1039 list_for_each_entry(dd, devices, list) { 1040 template_disk = dd->dm_dev->bdev->bd_disk; 1041 if (!blk_get_integrity(template_disk)) 1042 goto no_integrity; 1043 if (!match_all && !blk_integrity_is_initialized(template_disk)) 1044 continue; /* skip uninitialized profiles */ 1045 else if (prev_disk && 1046 blk_integrity_compare(prev_disk, template_disk) < 0) 1047 goto no_integrity; 1048 prev_disk = template_disk; 1049 } 1050 1051 return template_disk; 1052 1053no_integrity: 1054 if (prev_disk) 1055 DMWARN("%s: integrity not set: %s and %s profile mismatch", 1056 dm_device_name(t->md), 1057 prev_disk->disk_name, 1058 template_disk->disk_name); 1059 return NULL; 1060} 1061 1062/* 1063 * Register the mapped device for blk_integrity support if 1064 * the underlying devices have an integrity profile. But all devices 1065 * may not have matching profiles (checking all devices isn't reliable 1066 * during table load because this table may use other DM device(s) which 1067 * must be resumed before they will have an initialized integity profile). 1068 * Stacked DM devices force a 2 stage integrity profile validation: 1069 * 1 - during load, validate all initialized integrity profiles match 1070 * 2 - during resume, validate all integrity profiles match 1071 */ 1072static int dm_table_prealloc_integrity(struct dm_table *t, struct mapped_device *md) 1073{ 1074 struct gendisk *template_disk = NULL; 1075 1076 template_disk = dm_table_get_integrity_disk(t, false); 1077 if (!template_disk) 1078 return 0; 1079 1080 if (!blk_integrity_is_initialized(dm_disk(md))) { 1081 t->integrity_supported = 1; 1082 return blk_integrity_register(dm_disk(md), NULL); 1083 } 1084 1085 /* 1086 * If DM device already has an initalized integrity 1087 * profile the new profile should not conflict. 1088 */ 1089 if (blk_integrity_is_initialized(template_disk) && 1090 blk_integrity_compare(dm_disk(md), template_disk) < 0) { 1091 DMWARN("%s: conflict with existing integrity profile: " 1092 "%s profile mismatch", 1093 dm_device_name(t->md), 1094 template_disk->disk_name); 1095 return 1; 1096 } 1097 1098 /* Preserve existing initialized integrity profile */ 1099 t->integrity_supported = 1; 1100 return 0; 1101} 1102 1103/* 1104 * Prepares the table for use by building the indices, 1105 * setting the type, and allocating mempools. 1106 */ 1107int dm_table_complete(struct dm_table *t) 1108{ 1109 int r; 1110 1111 r = dm_table_set_type(t); 1112 if (r) { 1113 DMERR("unable to set table type"); 1114 return r; 1115 } 1116 1117 r = dm_table_build_index(t); 1118 if (r) { 1119 DMERR("unable to build btrees"); 1120 return r; 1121 } 1122 1123 r = dm_table_prealloc_integrity(t, t->md); 1124 if (r) { 1125 DMERR("could not register integrity profile."); 1126 return r; 1127 } 1128 1129 r = dm_table_alloc_md_mempools(t, t->md); 1130 if (r) 1131 DMERR("unable to allocate mempools"); 1132 1133 return r; 1134} 1135 1136static DEFINE_MUTEX(_event_lock); 1137void dm_table_event_callback(struct dm_table *t, 1138 void (*fn)(void *), void *context) 1139{ 1140 mutex_lock(&_event_lock); 1141 t->event_fn = fn; 1142 t->event_context = context; 1143 mutex_unlock(&_event_lock); 1144} 1145 1146void dm_table_event(struct dm_table *t) 1147{ 1148 /* 1149 * You can no longer call dm_table_event() from interrupt 1150 * context, use a bottom half instead. 1151 */ 1152 BUG_ON(in_interrupt()); 1153 1154 mutex_lock(&_event_lock); 1155 if (t->event_fn) 1156 t->event_fn(t->event_context); 1157 mutex_unlock(&_event_lock); 1158} 1159EXPORT_SYMBOL(dm_table_event); 1160 1161sector_t dm_table_get_size(struct dm_table *t) 1162{ 1163 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0; 1164} 1165EXPORT_SYMBOL(dm_table_get_size); 1166 1167struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index) 1168{ 1169 if (index >= t->num_targets) 1170 return NULL; 1171 1172 return t->targets + index; 1173} 1174 1175/* 1176 * Search the btree for the correct target. 1177 * 1178 * Caller should check returned pointer with dm_target_is_valid() 1179 * to trap I/O beyond end of device. 1180 */ 1181struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) 1182{ 1183 unsigned int l, n = 0, k = 0; 1184 sector_t *node; 1185 1186 for (l = 0; l < t->depth; l++) { 1187 n = get_child(n, k); 1188 node = get_node(t, l, n); 1189 1190 for (k = 0; k < KEYS_PER_NODE; k++) 1191 if (node[k] >= sector) 1192 break; 1193 } 1194 1195 return &t->targets[(KEYS_PER_NODE * n) + k]; 1196} 1197 1198static int count_device(struct dm_target *ti, struct dm_dev *dev, 1199 sector_t start, sector_t len, void *data) 1200{ 1201 unsigned *num_devices = data; 1202 1203 (*num_devices)++; 1204 1205 return 0; 1206} 1207 1208/* 1209 * Check whether a table has no data devices attached using each 1210 * target's iterate_devices method. 1211 * Returns false if the result is unknown because a target doesn't 1212 * support iterate_devices. 1213 */ 1214bool dm_table_has_no_data_devices(struct dm_table *table) 1215{ 1216 struct dm_target *uninitialized_var(ti); 1217 unsigned i = 0, num_devices = 0; 1218 1219 while (i < dm_table_get_num_targets(table)) { 1220 ti = dm_table_get_target(table, i++); 1221 1222 if (!ti->type->iterate_devices) 1223 return false; 1224 1225 ti->type->iterate_devices(ti, count_device, &num_devices); 1226 if (num_devices) 1227 return false; 1228 } 1229 1230 return true; 1231} 1232 1233/* 1234 * Establish the new table's queue_limits and validate them. 1235 */ 1236int dm_calculate_queue_limits(struct dm_table *table, 1237 struct queue_limits *limits) 1238{ 1239 struct dm_target *uninitialized_var(ti); 1240 struct queue_limits ti_limits; 1241 unsigned i = 0; 1242 1243 blk_set_stacking_limits(limits); 1244 1245 while (i < dm_table_get_num_targets(table)) { 1246 blk_set_stacking_limits(&ti_limits); 1247 1248 ti = dm_table_get_target(table, i++); 1249 1250 if (!ti->type->iterate_devices) 1251 goto combine_limits; 1252 1253 /* 1254 * Combine queue limits of all the devices this target uses. 1255 */ 1256 ti->type->iterate_devices(ti, dm_set_device_limits, 1257 &ti_limits); 1258 1259 /* Set I/O hints portion of queue limits */ 1260 if (ti->type->io_hints) 1261 ti->type->io_hints(ti, &ti_limits); 1262 1263 /* 1264 * Check each device area is consistent with the target's 1265 * overall queue limits. 1266 */ 1267 if (ti->type->iterate_devices(ti, device_area_is_invalid, 1268 &ti_limits)) 1269 return -EINVAL; 1270 1271combine_limits: 1272 /* 1273 * Merge this target's queue limits into the overall limits 1274 * for the table. 1275 */ 1276 if (blk_stack_limits(limits, &ti_limits, 0) < 0) 1277 DMWARN("%s: adding target device " 1278 "(start sect %llu len %llu) " 1279 "caused an alignment inconsistency", 1280 dm_device_name(table->md), 1281 (unsigned long long) ti->begin, 1282 (unsigned long long) ti->len); 1283 } 1284 1285 return validate_hardware_logical_block_alignment(table, limits); 1286} 1287 1288/* 1289 * Set the integrity profile for this device if all devices used have 1290 * matching profiles. We're quite deep in the resume path but still 1291 * don't know if all devices (particularly DM devices this device 1292 * may be stacked on) have matching profiles. Even if the profiles 1293 * don't match we have no way to fail (to resume) at this point. 1294 */ 1295static void dm_table_set_integrity(struct dm_table *t) 1296{ 1297 struct gendisk *template_disk = NULL; 1298 1299 if (!blk_get_integrity(dm_disk(t->md))) 1300 return; 1301 1302 template_disk = dm_table_get_integrity_disk(t, true); 1303 if (template_disk) 1304 blk_integrity_register(dm_disk(t->md), 1305 blk_get_integrity(template_disk)); 1306 else if (blk_integrity_is_initialized(dm_disk(t->md))) 1307 DMWARN("%s: device no longer has a valid integrity profile", 1308 dm_device_name(t->md)); 1309 else 1310 DMWARN("%s: unable to establish an integrity profile", 1311 dm_device_name(t->md)); 1312} 1313 1314static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev, 1315 sector_t start, sector_t len, void *data) 1316{ 1317 unsigned flush = (*(unsigned *)data); 1318 struct request_queue *q = bdev_get_queue(dev->bdev); 1319 1320 return q && (q->flush_flags & flush); 1321} 1322 1323static bool dm_table_supports_flush(struct dm_table *t, unsigned flush) 1324{ 1325 struct dm_target *ti; 1326 unsigned i = 0; 1327 1328 /* 1329 * Require at least one underlying device to support flushes. 1330 * t->devices includes internal dm devices such as mirror logs 1331 * so we need to use iterate_devices here, which targets 1332 * supporting flushes must provide. 1333 */ 1334 while (i < dm_table_get_num_targets(t)) { 1335 ti = dm_table_get_target(t, i++); 1336 1337 if (!ti->num_flush_bios) 1338 continue; 1339 1340 if (ti->flush_supported) 1341 return true; 1342 1343 if (ti->type->iterate_devices && 1344 ti->type->iterate_devices(ti, device_flush_capable, &flush)) 1345 return true; 1346 } 1347 1348 return false; 1349} 1350 1351static bool dm_table_discard_zeroes_data(struct dm_table *t) 1352{ 1353 struct dm_target *ti; 1354 unsigned i = 0; 1355 1356 /* Ensure that all targets supports discard_zeroes_data. */ 1357 while (i < dm_table_get_num_targets(t)) { 1358 ti = dm_table_get_target(t, i++); 1359 1360 if (ti->discard_zeroes_data_unsupported) 1361 return false; 1362 } 1363 1364 return true; 1365} 1366 1367static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev, 1368 sector_t start, sector_t len, void *data) 1369{ 1370 struct request_queue *q = bdev_get_queue(dev->bdev); 1371 1372 return q && blk_queue_nonrot(q); 1373} 1374 1375static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev, 1376 sector_t start, sector_t len, void *data) 1377{ 1378 struct request_queue *q = bdev_get_queue(dev->bdev); 1379 1380 return q && !blk_queue_add_random(q); 1381} 1382 1383static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev, 1384 sector_t start, sector_t len, void *data) 1385{ 1386 struct request_queue *q = bdev_get_queue(dev->bdev); 1387 1388 return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags); 1389} 1390 1391static int queue_supports_sg_gaps(struct dm_target *ti, struct dm_dev *dev, 1392 sector_t start, sector_t len, void *data) 1393{ 1394 struct request_queue *q = bdev_get_queue(dev->bdev); 1395 1396 return q && !test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags); 1397} 1398 1399static bool dm_table_all_devices_attribute(struct dm_table *t, 1400 iterate_devices_callout_fn func) 1401{ 1402 struct dm_target *ti; 1403 unsigned i = 0; 1404 1405 while (i < dm_table_get_num_targets(t)) { 1406 ti = dm_table_get_target(t, i++); 1407 1408 if (!ti->type->iterate_devices || 1409 !ti->type->iterate_devices(ti, func, NULL)) 1410 return false; 1411 } 1412 1413 return true; 1414} 1415 1416static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev, 1417 sector_t start, sector_t len, void *data) 1418{ 1419 struct request_queue *q = bdev_get_queue(dev->bdev); 1420 1421 return q && !q->limits.max_write_same_sectors; 1422} 1423 1424static bool dm_table_supports_write_same(struct dm_table *t) 1425{ 1426 struct dm_target *ti; 1427 unsigned i = 0; 1428 1429 while (i < dm_table_get_num_targets(t)) { 1430 ti = dm_table_get_target(t, i++); 1431 1432 if (!ti->num_write_same_bios) 1433 return false; 1434 1435 if (!ti->type->iterate_devices || 1436 ti->type->iterate_devices(ti, device_not_write_same_capable, NULL)) 1437 return false; 1438 } 1439 1440 return true; 1441} 1442 1443static int device_discard_capable(struct dm_target *ti, struct dm_dev *dev, 1444 sector_t start, sector_t len, void *data) 1445{ 1446 struct request_queue *q = bdev_get_queue(dev->bdev); 1447 1448 return q && blk_queue_discard(q); 1449} 1450 1451static bool dm_table_supports_discards(struct dm_table *t) 1452{ 1453 struct dm_target *ti; 1454 unsigned i = 0; 1455 1456 /* 1457 * Unless any target used by the table set discards_supported, 1458 * require at least one underlying device to support discards. 1459 * t->devices includes internal dm devices such as mirror logs 1460 * so we need to use iterate_devices here, which targets 1461 * supporting discard selectively must provide. 1462 */ 1463 while (i < dm_table_get_num_targets(t)) { 1464 ti = dm_table_get_target(t, i++); 1465 1466 if (!ti->num_discard_bios) 1467 continue; 1468 1469 if (ti->discards_supported) 1470 return true; 1471 1472 if (ti->type->iterate_devices && 1473 ti->type->iterate_devices(ti, device_discard_capable, NULL)) 1474 return true; 1475 } 1476 1477 return false; 1478} 1479 1480void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, 1481 struct queue_limits *limits) 1482{ 1483 unsigned flush = 0; 1484 1485 /* 1486 * Copy table's limits to the DM device's request_queue 1487 */ 1488 q->limits = *limits; 1489 1490 if (!dm_table_supports_discards(t)) 1491 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); 1492 else 1493 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 1494 1495 if (dm_table_supports_flush(t, REQ_FLUSH)) { 1496 flush |= REQ_FLUSH; 1497 if (dm_table_supports_flush(t, REQ_FUA)) 1498 flush |= REQ_FUA; 1499 } 1500 blk_queue_flush(q, flush); 1501 1502 if (!dm_table_discard_zeroes_data(t)) 1503 q->limits.discard_zeroes_data = 0; 1504 1505 /* Ensure that all underlying devices are non-rotational. */ 1506 if (dm_table_all_devices_attribute(t, device_is_nonrot)) 1507 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); 1508 else 1509 queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, q); 1510 1511 if (!dm_table_supports_write_same(t)) 1512 q->limits.max_write_same_sectors = 0; 1513 1514 if (dm_table_all_devices_attribute(t, queue_supports_sg_merge)) 1515 queue_flag_clear_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 1516 else 1517 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 1518 1519 if (dm_table_all_devices_attribute(t, queue_supports_sg_gaps)) 1520 queue_flag_clear_unlocked(QUEUE_FLAG_SG_GAPS, q); 1521 else 1522 queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, q); 1523 1524 dm_table_set_integrity(t); 1525 1526 /* 1527 * Determine whether or not this queue's I/O timings contribute 1528 * to the entropy pool, Only request-based targets use this. 1529 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not 1530 * have it set. 1531 */ 1532 if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random)) 1533 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q); 1534 1535 /* 1536 * QUEUE_FLAG_STACKABLE must be set after all queue settings are 1537 * visible to other CPUs because, once the flag is set, incoming bios 1538 * are processed by request-based dm, which refers to the queue 1539 * settings. 1540 * Until the flag set, bios are passed to bio-based dm and queued to 1541 * md->deferred where queue settings are not needed yet. 1542 * Those bios are passed to request-based dm at the resume time. 1543 */ 1544 smp_mb(); 1545 if (dm_table_request_based(t)) 1546 queue_flag_set_unlocked(QUEUE_FLAG_STACKABLE, q); 1547} 1548 1549unsigned int dm_table_get_num_targets(struct dm_table *t) 1550{ 1551 return t->num_targets; 1552} 1553 1554struct list_head *dm_table_get_devices(struct dm_table *t) 1555{ 1556 return &t->devices; 1557} 1558 1559fmode_t dm_table_get_mode(struct dm_table *t) 1560{ 1561 return t->mode; 1562} 1563EXPORT_SYMBOL(dm_table_get_mode); 1564 1565enum suspend_mode { 1566 PRESUSPEND, 1567 PRESUSPEND_UNDO, 1568 POSTSUSPEND, 1569}; 1570 1571static void suspend_targets(struct dm_table *t, enum suspend_mode mode) 1572{ 1573 int i = t->num_targets; 1574 struct dm_target *ti = t->targets; 1575 1576 while (i--) { 1577 switch (mode) { 1578 case PRESUSPEND: 1579 if (ti->type->presuspend) 1580 ti->type->presuspend(ti); 1581 break; 1582 case PRESUSPEND_UNDO: 1583 if (ti->type->presuspend_undo) 1584 ti->type->presuspend_undo(ti); 1585 break; 1586 case POSTSUSPEND: 1587 if (ti->type->postsuspend) 1588 ti->type->postsuspend(ti); 1589 break; 1590 } 1591 ti++; 1592 } 1593} 1594 1595void dm_table_presuspend_targets(struct dm_table *t) 1596{ 1597 if (!t) 1598 return; 1599 1600 suspend_targets(t, PRESUSPEND); 1601} 1602 1603void dm_table_presuspend_undo_targets(struct dm_table *t) 1604{ 1605 if (!t) 1606 return; 1607 1608 suspend_targets(t, PRESUSPEND_UNDO); 1609} 1610 1611void dm_table_postsuspend_targets(struct dm_table *t) 1612{ 1613 if (!t) 1614 return; 1615 1616 suspend_targets(t, POSTSUSPEND); 1617} 1618 1619int dm_table_resume_targets(struct dm_table *t) 1620{ 1621 int i, r = 0; 1622 1623 for (i = 0; i < t->num_targets; i++) { 1624 struct dm_target *ti = t->targets + i; 1625 1626 if (!ti->type->preresume) 1627 continue; 1628 1629 r = ti->type->preresume(ti); 1630 if (r) { 1631 DMERR("%s: %s: preresume failed, error = %d", 1632 dm_device_name(t->md), ti->type->name, r); 1633 return r; 1634 } 1635 } 1636 1637 for (i = 0; i < t->num_targets; i++) { 1638 struct dm_target *ti = t->targets + i; 1639 1640 if (ti->type->resume) 1641 ti->type->resume(ti); 1642 } 1643 1644 return 0; 1645} 1646 1647void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb) 1648{ 1649 list_add(&cb->list, &t->target_callbacks); 1650} 1651EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks); 1652 1653int dm_table_any_congested(struct dm_table *t, int bdi_bits) 1654{ 1655 struct dm_dev_internal *dd; 1656 struct list_head *devices = dm_table_get_devices(t); 1657 struct dm_target_callbacks *cb; 1658 int r = 0; 1659 1660 list_for_each_entry(dd, devices, list) { 1661 struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev); 1662 char b[BDEVNAME_SIZE]; 1663 1664 if (likely(q)) 1665 r |= bdi_congested(&q->backing_dev_info, bdi_bits); 1666 else 1667 DMWARN_LIMIT("%s: any_congested: nonexistent device %s", 1668 dm_device_name(t->md), 1669 bdevname(dd->dm_dev->bdev, b)); 1670 } 1671 1672 list_for_each_entry(cb, &t->target_callbacks, list) 1673 if (cb->congested_fn) 1674 r |= cb->congested_fn(cb, bdi_bits); 1675 1676 return r; 1677} 1678 1679struct mapped_device *dm_table_get_md(struct dm_table *t) 1680{ 1681 return t->md; 1682} 1683EXPORT_SYMBOL(dm_table_get_md); 1684 1685void dm_table_run_md_queue_async(struct dm_table *t) 1686{ 1687 struct mapped_device *md; 1688 struct request_queue *queue; 1689 unsigned long flags; 1690 1691 if (!dm_table_request_based(t)) 1692 return; 1693 1694 md = dm_table_get_md(t); 1695 queue = dm_get_md_queue(md); 1696 if (queue) { 1697 if (queue->mq_ops) 1698 blk_mq_run_hw_queues(queue, true); 1699 else { 1700 spin_lock_irqsave(queue->queue_lock, flags); 1701 blk_run_queue_async(queue); 1702 spin_unlock_irqrestore(queue->queue_lock, flags); 1703 } 1704 } 1705} 1706EXPORT_SYMBOL(dm_table_run_md_queue_async); 1707 1708