drivers/md/dm-thin.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
build_key
build_data_key
build_virtual_key
throttle_init
throttle_work_start
throttle_work_update
throttle_work_complete
throttle_lock
throttle_unlock
get_pool_mode
notify_of_pool_mode_change
block_size_is_power_of_two
block_to_sectors
begin_discard
issue_discard
end_discard
wake_worker
bio_detain
cell_release
cell_visit_release
cell_release_no_holder
cell_error_with_code
get_pool_io_error_code
cell_error
cell_success
cell_requeue
pool_table_init
pool_table_exit
__pool_table_insert
__pool_table_remove
__pool_table_lookup
__pool_table_lookup_metadata_dev
__merge_bio_list
error_bio_list
error_thin_bio_list
requeue_deferred_cells
requeue_io
error_retry_list_with_code
error_retry_list
get_bio_block
get_bio_block_range
remap
remap_to_origin
bio_triggers_commit
inc_all_io_entry
issue
remap_to_origin_and_issue
remap_and_issue
__complete_mapping_preparation
complete_mapping_preparation
copy_complete
overwrite_endio
cell_defer_no_holder
__inc_remap_and_issue_cell
inc_remap_and_issue_cell
process_prepared_mapping_fail
complete_overwrite_bio
process_prepared_mapping
free_discard_mapping
process_prepared_discard_fail
process_prepared_discard_success
process_prepared_discard_no_passdown
passdown_double_checking_shared_status
queue_passdown_pt2
passdown_endio
process_prepared_discard_passdown_pt1
process_prepared_discard_passdown_pt2
process_prepared
io_overlaps_block
io_overwrites_block
save_and_set_endio
ensure_next_mapping
get_next_mapping
ll_zero
remap_and_issue_overwrite
schedule_copy
schedule_internal_copy
schedule_zero
schedule_external_copy
is_read_only_pool_mode
is_read_only
check_for_metadata_space
check_for_data_space
commit
check_low_water_mark
alloc_data_block
retry_on_resume
should_error_unserviceable_bio
handle_unserviceable_bio
retry_bios_on_resume
process_discard_cell_no_passdown
break_up_discard_bio
process_discard_cell_passdown
process_discard_bio
break_sharing
__remap_and_issue_shared_cell
remap_and_issue_shared_cell
process_shared_bio
provision_block
process_cell
process_bio
__process_bio_read_only
process_bio_read_only
process_cell_read_only
process_bio_success
process_bio_fail
process_cell_success
process_cell_fail
need_commit_due_to_time
__extract_sorted_bios
__sort_thin_deferred_bios
process_thin_deferred_bios
cmp_cells
sort_cells
process_thin_deferred_cells
get_first_thin
get_next_thin
process_deferred_bios
do_worker
do_waker
do_no_space_timeout
to_pool_work
pool_work_complete
pool_work_wait
to_noflush
do_noflush_start
do_noflush_stop
noflush_work
passdown_enabled
set_discard_callbacks
set_pool_mode
abort_transaction
metadata_operation_failed
thin_defer_bio
thin_defer_bio_with_throttle
thin_defer_cell
thin_hook_bio
thin_bio_map
pool_is_congested
requeue_bios
data_dev_supports_discard
is_factor
disable_passdown_if_not_supported
bind_control_target
unbind_control_target
pool_features_init
__pool_destroy
pool_create
__pool_inc
__pool_dec
__pool_find
pool_dtr
parse_pool_features
metadata_low_callback
metadata_pre_commit_callback
get_dev_size
warn_if_metadata_device_too_big
get_metadata_dev_size
get_metadata_dev_size_in_blocks
calc_metadata_threshold
pool_ctr
pool_map
maybe_resize_data_dev
maybe_resize_metadata_dev
pool_preresume
pool_suspend_active_thins
pool_resume_active_thins
pool_resume
pool_presuspend
pool_presuspend_undo
pool_postsuspend
check_arg_count
read_dev_id
process_create_thin_mesg
process_create_snap_mesg
process_delete_mesg
process_set_transaction_id_mesg
process_reserve_metadata_snap_mesg
process_release_metadata_snap_mesg
pool_message
emit_flags
pool_status
pool_iterate_devices
pool_io_hints
thin_get
thin_put
thin_dtr
thin_ctr
thin_map
thin_endio
thin_presuspend
thin_postsuspend
thin_preresume
thin_status
thin_iterate_devices
thin_io_hints
dm_thin_init
dm_thin_exit
   1 /*
   2  * Copyright (C) 2011-2012 Red Hat UK.
   3  *
   4  * This file is released under the GPL.
   5  */
   6 
   7 #include "dm-thin-metadata.h"
   8 #include "dm-bio-prison-v1.h"
   9 #include "dm.h"
  10 
  11 #include <linux/device-mapper.h>
  12 #include <linux/dm-io.h>
  13 #include <linux/dm-kcopyd.h>
  14 #include <linux/jiffies.h>
  15 #include <linux/log2.h>
  16 #include <linux/list.h>
  17 #include <linux/rculist.h>
  18 #include <linux/init.h>
  19 #include <linux/module.h>
  20 #include <linux/slab.h>
  21 #include <linux/vmalloc.h>
  22 #include <linux/sort.h>
  23 #include <linux/rbtree.h>
  24 
  25 #define DM_MSG_PREFIX   "thin"
  26 
  27 /*
  28  * Tunable constants
  29  */
  30 #define ENDIO_HOOK_POOL_SIZE 1024
  31 #define MAPPING_POOL_SIZE 1024
  32 #define COMMIT_PERIOD HZ
  33 #define NO_SPACE_TIMEOUT_SECS 60
  34 
  35 static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
  36 
  37 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
  38                 "A percentage of time allocated for copy on write");
  39 
  40 /*
  41  * The block size of the device holding pool data must be
  42  * between 64KB and 1GB.
  43  */
  44 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
  45 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
  46 
  47 /*
  48  * Device id is restricted to 24 bits.
  49  */
  50 #define MAX_DEV_ID ((1 << 24) - 1)
  51 
  52 /*
  53  * How do we handle breaking sharing of data blocks?
  54  * =================================================
  55  *
  56  * We use a standard copy-on-write btree to store the mappings for the
  57  * devices (note I'm talking about copy-on-write of the metadata here, not
  58  * the data).  When you take an internal snapshot you clone the root node
  59  * of the origin btree.  After this there is no concept of an origin or a
  60  * snapshot.  They are just two device trees that happen to point to the
  61  * same data blocks.
  62  *
  63  * When we get a write in we decide if it's to a shared data block using
  64  * some timestamp magic.  If it is, we have to break sharing.
  65  *
  66  * Let's say we write to a shared block in what was the origin.  The
  67  * steps are:
  68  *
  69  * i) plug io further to this physical block. (see bio_prison code).
  70  *
  71  * ii) quiesce any read io to that shared data block.  Obviously
  72  * including all devices that share this block.  (see dm_deferred_set code)
  73  *
  74  * iii) copy the data block to a newly allocate block.  This step can be
  75  * missed out if the io covers the block. (schedule_copy).
  76  *
  77  * iv) insert the new mapping into the origin's btree
  78  * (process_prepared_mapping).  This act of inserting breaks some
  79  * sharing of btree nodes between the two devices.  Breaking sharing only
  80  * effects the btree of that specific device.  Btrees for the other
  81  * devices that share the block never change.  The btree for the origin
  82  * device as it was after the last commit is untouched, ie. we're using
  83  * persistent data structures in the functional programming sense.
  84  *
  85  * v) unplug io to this physical block, including the io that triggered
  86  * the breaking of sharing.
  87  *
  88  * Steps (ii) and (iii) occur in parallel.
  89  *
  90  * The metadata _doesn't_ need to be committed before the io continues.  We
  91  * get away with this because the io is always written to a _new_ block.
  92  * If there's a crash, then:
  93  *
  94  * - The origin mapping will point to the old origin block (the shared
  95  * one).  This will contain the data as it was before the io that triggered
  96  * the breaking of sharing came in.
  97  *
  98  * - The snap mapping still points to the old block.  As it would after
  99  * the commit.
 100  *
 101  * The downside of this scheme is the timestamp magic isn't perfect, and
 102  * will continue to think that data block in the snapshot device is shared
 103  * even after the write to the origin has broken sharing.  I suspect data
 104  * blocks will typically be shared by many different devices, so we're
 105  * breaking sharing n + 1 times, rather than n, where n is the number of
 106  * devices that reference this data block.  At the moment I think the
 107  * benefits far, far outweigh the disadvantages.
 108  */
 109 
 110 /*----------------------------------------------------------------*/
 111 
 112 /*
 113  * Key building.
 114  */
 115 enum lock_space {
 116         VIRTUAL,
 117         PHYSICAL
 118 };
 119 
 120 static void build_key(struct dm_thin_device *td, enum lock_space ls,
 121                       dm_block_t b, dm_block_t e, struct dm_cell_key *key)
 122 {
 123         key->virtual = (ls == VIRTUAL);
 124         key->dev = dm_thin_dev_id(td);
 125         key->block_begin = b;
 126         key->block_end = e;
 127 }
 128 
 129 static void build_data_key(struct dm_thin_device *td, dm_block_t b,
 130                            struct dm_cell_key *key)
 131 {
 132         build_key(td, PHYSICAL, b, b + 1llu, key);
 133 }
 134 
 135 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
 136                               struct dm_cell_key *key)
 137 {
 138         build_key(td, VIRTUAL, b, b + 1llu, key);
 139 }
 140 
 141 /*----------------------------------------------------------------*/
 142 
 143 #define THROTTLE_THRESHOLD (1 * HZ)
 144 
 145 struct throttle {
 146         struct rw_semaphore lock;
 147         unsigned long threshold;
 148         bool throttle_applied;
 149 };
 150 
 151 static void throttle_init(struct throttle *t)
 152 {
 153         init_rwsem(&t->lock);
 154         t->throttle_applied = false;
 155 }
 156 
 157 static void throttle_work_start(struct throttle *t)
 158 {
 159         t->threshold = jiffies + THROTTLE_THRESHOLD;
 160 }
 161 
 162 static void throttle_work_update(struct throttle *t)
 163 {
 164         if (!t->throttle_applied && jiffies > t->threshold) {
 165                 down_write(&t->lock);
 166                 t->throttle_applied = true;
 167         }
 168 }
 169 
 170 static void throttle_work_complete(struct throttle *t)
 171 {
 172         if (t->throttle_applied) {
 173                 t->throttle_applied = false;
 174                 up_write(&t->lock);
 175         }
 176 }
 177 
 178 static void throttle_lock(struct throttle *t)
 179 {
 180         down_read(&t->lock);
 181 }
 182 
 183 static void throttle_unlock(struct throttle *t)
 184 {
 185         up_read(&t->lock);
 186 }
 187 
 188 /*----------------------------------------------------------------*/
 189 
 190 /*
 191  * A pool device ties together a metadata device and a data device.  It
 192  * also provides the interface for creating and destroying internal
 193  * devices.
 194  */
 195 struct dm_thin_new_mapping;
 196 
 197 /*
 198  * The pool runs in various modes.  Ordered in degraded order for comparisons.
 199  */
 200 enum pool_mode {
 201         PM_WRITE,               /* metadata may be changed */
 202         PM_OUT_OF_DATA_SPACE,   /* metadata may be changed, though data may not be allocated */
 203 
 204         /*
 205          * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.
 206          */
 207         PM_OUT_OF_METADATA_SPACE,
 208         PM_READ_ONLY,           /* metadata may not be changed */
 209 
 210         PM_FAIL,                /* all I/O fails */
 211 };
 212 
 213 struct pool_features {
 214         enum pool_mode mode;
 215 
 216         bool zero_new_blocks:1;
 217         bool discard_enabled:1;
 218         bool discard_passdown:1;
 219         bool error_if_no_space:1;
 220 };
 221 
 222 struct thin_c;
 223 typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
 224 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
 225 typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
 226 
 227 #define CELL_SORT_ARRAY_SIZE 8192
 228 
 229 struct pool {
 230         struct list_head list;
 231         struct dm_target *ti;   /* Only set if a pool target is bound */
 232 
 233         struct mapped_device *pool_md;
 234         struct block_device *data_dev;
 235         struct block_device *md_dev;
 236         struct dm_pool_metadata *pmd;
 237 
 238         dm_block_t low_water_blocks;
 239         uint32_t sectors_per_block;
 240         int sectors_per_block_shift;
 241 
 242         struct pool_features pf;
 243         bool low_water_triggered:1;     /* A dm event has been sent */
 244         bool suspended:1;
 245         bool out_of_data_space:1;
 246 
 247         struct dm_bio_prison *prison;
 248         struct dm_kcopyd_client *copier;
 249 
 250         struct work_struct worker;
 251         struct workqueue_struct *wq;
 252         struct throttle throttle;
 253         struct delayed_work waker;
 254         struct delayed_work no_space_timeout;
 255 
 256         unsigned long last_commit_jiffies;
 257         unsigned ref_count;
 258 
 259         spinlock_t lock;
 260         struct bio_list deferred_flush_bios;
 261         struct bio_list deferred_flush_completions;
 262         struct list_head prepared_mappings;
 263         struct list_head prepared_discards;
 264         struct list_head prepared_discards_pt2;
 265         struct list_head active_thins;
 266 
 267         struct dm_deferred_set *shared_read_ds;
 268         struct dm_deferred_set *all_io_ds;
 269 
 270         struct dm_thin_new_mapping *next_mapping;
 271 
 272         process_bio_fn process_bio;
 273         process_bio_fn process_discard;
 274 
 275         process_cell_fn process_cell;
 276         process_cell_fn process_discard_cell;
 277 
 278         process_mapping_fn process_prepared_mapping;
 279         process_mapping_fn process_prepared_discard;
 280         process_mapping_fn process_prepared_discard_pt2;
 281 
 282         struct dm_bio_prison_cell **cell_sort_array;
 283 
 284         mempool_t mapping_pool;
 285 };
 286 
 287 static void metadata_operation_failed(struct pool *pool, const char *op, int r);
 288 
 289 static enum pool_mode get_pool_mode(struct pool *pool)
 290 {
 291         return pool->pf.mode;
 292 }
 293 
 294 static void notify_of_pool_mode_change(struct pool *pool)
 295 {
 296         const char *descs[] = {
 297                 "write",
 298                 "out-of-data-space",
 299                 "read-only",
 300                 "read-only",
 301                 "fail"
 302         };
 303         const char *extra_desc = NULL;
 304         enum pool_mode mode = get_pool_mode(pool);
 305 
 306         if (mode == PM_OUT_OF_DATA_SPACE) {
 307                 if (!pool->pf.error_if_no_space)
 308                         extra_desc = " (queue IO)";
 309                 else
 310                         extra_desc = " (error IO)";
 311         }
 312 
 313         dm_table_event(pool->ti->table);
 314         DMINFO("%s: switching pool to %s%s mode",
 315                dm_device_name(pool->pool_md),
 316                descs[(int)mode], extra_desc ? : "");
 317 }
 318 
 319 /*
 320  * Target context for a pool.
 321  */
 322 struct pool_c {
 323         struct dm_target *ti;
 324         struct pool *pool;
 325         struct dm_dev *data_dev;
 326         struct dm_dev *metadata_dev;
 327         struct dm_target_callbacks callbacks;
 328 
 329         dm_block_t low_water_blocks;
 330         struct pool_features requested_pf; /* Features requested during table load */
 331         struct pool_features adjusted_pf;  /* Features used after adjusting for constituent devices */
 332         struct bio flush_bio;
 333 };
 334 
 335 /*
 336  * Target context for a thin.
 337  */
 338 struct thin_c {
 339         struct list_head list;
 340         struct dm_dev *pool_dev;
 341         struct dm_dev *origin_dev;
 342         sector_t origin_size;
 343         dm_thin_id dev_id;
 344 
 345         struct pool *pool;
 346         struct dm_thin_device *td;
 347         struct mapped_device *thin_md;
 348 
 349         bool requeue_mode:1;
 350         spinlock_t lock;
 351         struct list_head deferred_cells;
 352         struct bio_list deferred_bio_list;
 353         struct bio_list retry_on_resume_list;
 354         struct rb_root sort_bio_list; /* sorted list of deferred bios */
 355 
 356         /*
 357          * Ensures the thin is not destroyed until the worker has finished
 358          * iterating the active_thins list.
 359          */
 360         refcount_t refcount;
 361         struct completion can_destroy;
 362 };
 363 
 364 /*----------------------------------------------------------------*/
 365 
 366 static bool block_size_is_power_of_two(struct pool *pool)
 367 {
 368         return pool->sectors_per_block_shift >= 0;
 369 }
 370 
 371 static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
 372 {
 373         return block_size_is_power_of_two(pool) ?
 374                 (b << pool->sectors_per_block_shift) :
 375                 (b * pool->sectors_per_block);
 376 }
 377 
 378 /*----------------------------------------------------------------*/
 379 
 380 struct discard_op {
 381         struct thin_c *tc;
 382         struct blk_plug plug;
 383         struct bio *parent_bio;
 384         struct bio *bio;
 385 };
 386 
 387 static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)
 388 {
 389         BUG_ON(!parent);
 390 
 391         op->tc = tc;
 392         blk_start_plug(&op->plug);
 393         op->parent_bio = parent;
 394         op->bio = NULL;
 395 }
 396 
 397 static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)
 398 {
 399         struct thin_c *tc = op->tc;
 400         sector_t s = block_to_sectors(tc->pool, data_b);
 401         sector_t len = block_to_sectors(tc->pool, data_e - data_b);
 402 
 403         return __blkdev_issue_discard(tc->pool_dev->bdev, s, len,
 404                                       GFP_NOWAIT, 0, &op->bio);
 405 }
 406 
 407 static void end_discard(struct discard_op *op, int r)
 408 {
 409         if (op->bio) {
 410                 /*
 411                  * Even if one of the calls to issue_discard failed, we
 412                  * need to wait for the chain to complete.
 413                  */
 414                 bio_chain(op->bio, op->parent_bio);
 415                 bio_set_op_attrs(op->bio, REQ_OP_DISCARD, 0);
 416                 submit_bio(op->bio);
 417         }
 418 
 419         blk_finish_plug(&op->plug);
 420 
 421         /*
 422          * Even if r is set, there could be sub discards in flight that we
 423          * need to wait for.
 424          */
 425         if (r && !op->parent_bio->bi_status)
 426                 op->parent_bio->bi_status = errno_to_blk_status(r);
 427         bio_endio(op->parent_bio);
 428 }
 429 
 430 /*----------------------------------------------------------------*/
 431 
 432 /*
 433  * wake_worker() is used when new work is queued and when pool_resume is
 434  * ready to continue deferred IO processing.
 435  */
 436 static void wake_worker(struct pool *pool)
 437 {
 438         queue_work(pool->wq, &pool->worker);
 439 }
 440 
 441 /*----------------------------------------------------------------*/
 442 
 443 static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
 444                       struct dm_bio_prison_cell **cell_result)
 445 {
 446         int r;
 447         struct dm_bio_prison_cell *cell_prealloc;
 448 
 449         /*
 450          * Allocate a cell from the prison's mempool.
 451          * This might block but it can't fail.
 452          */
 453         cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
 454 
 455         r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
 456         if (r)
 457                 /*
 458                  * We reused an old cell; we can get rid of
 459                  * the new one.
 460                  */
 461                 dm_bio_prison_free_cell(pool->prison, cell_prealloc);
 462 
 463         return r;
 464 }
 465 
 466 static void cell_release(struct pool *pool,
 467                          struct dm_bio_prison_cell *cell,
 468                          struct bio_list *bios)
 469 {
 470         dm_cell_release(pool->prison, cell, bios);
 471         dm_bio_prison_free_cell(pool->prison, cell);
 472 }
 473 
 474 static void cell_visit_release(struct pool *pool,
 475                                void (*fn)(void *, struct dm_bio_prison_cell *),
 476                                void *context,
 477                                struct dm_bio_prison_cell *cell)
 478 {
 479         dm_cell_visit_release(pool->prison, fn, context, cell);
 480         dm_bio_prison_free_cell(pool->prison, cell);
 481 }
 482 
 483 static void cell_release_no_holder(struct pool *pool,
 484                                    struct dm_bio_prison_cell *cell,
 485                                    struct bio_list *bios)
 486 {
 487         dm_cell_release_no_holder(pool->prison, cell, bios);
 488         dm_bio_prison_free_cell(pool->prison, cell);
 489 }
 490 
 491 static void cell_error_with_code(struct pool *pool,
 492                 struct dm_bio_prison_cell *cell, blk_status_t error_code)
 493 {
 494         dm_cell_error(pool->prison, cell, error_code);
 495         dm_bio_prison_free_cell(pool->prison, cell);
 496 }
 497 
 498 static blk_status_t get_pool_io_error_code(struct pool *pool)
 499 {
 500         return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR;
 501 }
 502 
 503 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
 504 {
 505         cell_error_with_code(pool, cell, get_pool_io_error_code(pool));
 506 }
 507 
 508 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
 509 {
 510         cell_error_with_code(pool, cell, 0);
 511 }
 512 
 513 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
 514 {
 515         cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);
 516 }
 517 
 518 /*----------------------------------------------------------------*/
 519 
 520 /*
 521  * A global list of pools that uses a struct mapped_device as a key.
 522  */
 523 static struct dm_thin_pool_table {
 524         struct mutex mutex;
 525         struct list_head pools;
 526 } dm_thin_pool_table;
 527 
 528 static void pool_table_init(void)
 529 {
 530         mutex_init(&dm_thin_pool_table.mutex);
 531         INIT_LIST_HEAD(&dm_thin_pool_table.pools);
 532 }
 533 
 534 static void pool_table_exit(void)
 535 {
 536         mutex_destroy(&dm_thin_pool_table.mutex);
 537 }
 538 
 539 static void __pool_table_insert(struct pool *pool)
 540 {
 541         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 542         list_add(&pool->list, &dm_thin_pool_table.pools);
 543 }
 544 
 545 static void __pool_table_remove(struct pool *pool)
 546 {
 547         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 548         list_del(&pool->list);
 549 }
 550 
 551 static struct pool *__pool_table_lookup(struct mapped_device *md)
 552 {
 553         struct pool *pool = NULL, *tmp;
 554 
 555         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 556 
 557         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
 558                 if (tmp->pool_md == md) {
 559                         pool = tmp;
 560                         break;
 561                 }
 562         }
 563 
 564         return pool;
 565 }
 566 
 567 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
 568 {
 569         struct pool *pool = NULL, *tmp;
 570 
 571         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 572 
 573         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
 574                 if (tmp->md_dev == md_dev) {
 575                         pool = tmp;
 576                         break;
 577                 }
 578         }
 579 
 580         return pool;
 581 }
 582 
 583 /*----------------------------------------------------------------*/
 584 
 585 struct dm_thin_endio_hook {
 586         struct thin_c *tc;
 587         struct dm_deferred_entry *shared_read_entry;
 588         struct dm_deferred_entry *all_io_entry;
 589         struct dm_thin_new_mapping *overwrite_mapping;
 590         struct rb_node rb_node;
 591         struct dm_bio_prison_cell *cell;
 592 };
 593 
 594 static void __merge_bio_list(struct bio_list *bios, struct bio_list *master)
 595 {
 596         bio_list_merge(bios, master);
 597         bio_list_init(master);
 598 }
 599 
 600 static void error_bio_list(struct bio_list *bios, blk_status_t error)
 601 {
 602         struct bio *bio;
 603 
 604         while ((bio = bio_list_pop(bios))) {
 605                 bio->bi_status = error;
 606                 bio_endio(bio);
 607         }
 608 }
 609 
 610 static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master,
 611                 blk_status_t error)
 612 {
 613         struct bio_list bios;
 614         unsigned long flags;
 615 
 616         bio_list_init(&bios);
 617 
 618         spin_lock_irqsave(&tc->lock, flags);
 619         __merge_bio_list(&bios, master);
 620         spin_unlock_irqrestore(&tc->lock, flags);
 621 
 622         error_bio_list(&bios, error);
 623 }
 624 
 625 static void requeue_deferred_cells(struct thin_c *tc)
 626 {
 627         struct pool *pool = tc->pool;
 628         unsigned long flags;
 629         struct list_head cells;
 630         struct dm_bio_prison_cell *cell, *tmp;
 631 
 632         INIT_LIST_HEAD(&cells);
 633 
 634         spin_lock_irqsave(&tc->lock, flags);
 635         list_splice_init(&tc->deferred_cells, &cells);
 636         spin_unlock_irqrestore(&tc->lock, flags);
 637 
 638         list_for_each_entry_safe(cell, tmp, &cells, user_list)
 639                 cell_requeue(pool, cell);
 640 }
 641 
 642 static void requeue_io(struct thin_c *tc)
 643 {
 644         struct bio_list bios;
 645         unsigned long flags;
 646 
 647         bio_list_init(&bios);
 648 
 649         spin_lock_irqsave(&tc->lock, flags);
 650         __merge_bio_list(&bios, &tc->deferred_bio_list);
 651         __merge_bio_list(&bios, &tc->retry_on_resume_list);
 652         spin_unlock_irqrestore(&tc->lock, flags);
 653 
 654         error_bio_list(&bios, BLK_STS_DM_REQUEUE);
 655         requeue_deferred_cells(tc);
 656 }
 657 
 658 static void error_retry_list_with_code(struct pool *pool, blk_status_t error)
 659 {
 660         struct thin_c *tc;
 661 
 662         rcu_read_lock();
 663         list_for_each_entry_rcu(tc, &pool->active_thins, list)
 664                 error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
 665         rcu_read_unlock();
 666 }
 667 
 668 static void error_retry_list(struct pool *pool)
 669 {
 670         error_retry_list_with_code(pool, get_pool_io_error_code(pool));
 671 }
 672 
 673 /*
 674  * This section of code contains the logic for processing a thin device's IO.
 675  * Much of the code depends on pool object resources (lists, workqueues, etc)
 676  * but most is exclusively called from the thin target rather than the thin-pool
 677  * target.
 678  */
 679 
 680 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
 681 {
 682         struct pool *pool = tc->pool;
 683         sector_t block_nr = bio->bi_iter.bi_sector;
 684 
 685         if (block_size_is_power_of_two(pool))
 686                 block_nr >>= pool->sectors_per_block_shift;
 687         else
 688                 (void) sector_div(block_nr, pool->sectors_per_block);
 689 
 690         return block_nr;
 691 }
 692 
 693 /*
 694  * Returns the _complete_ blocks that this bio covers.
 695  */
 696 static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
 697                                 dm_block_t *begin, dm_block_t *end)
 698 {
 699         struct pool *pool = tc->pool;
 700         sector_t b = bio->bi_iter.bi_sector;
 701         sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
 702 
 703         b += pool->sectors_per_block - 1ull; /* so we round up */
 704 
 705         if (block_size_is_power_of_two(pool)) {
 706                 b >>= pool->sectors_per_block_shift;
 707                 e >>= pool->sectors_per_block_shift;
 708         } else {
 709                 (void) sector_div(b, pool->sectors_per_block);
 710                 (void) sector_div(e, pool->sectors_per_block);
 711         }
 712 
 713         if (e < b)
 714                 /* Can happen if the bio is within a single block. */
 715                 e = b;
 716 
 717         *begin = b;
 718         *end = e;
 719 }
 720 
 721 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
 722 {
 723         struct pool *pool = tc->pool;
 724         sector_t bi_sector = bio->bi_iter.bi_sector;
 725 
 726         bio_set_dev(bio, tc->pool_dev->bdev);
 727         if (block_size_is_power_of_two(pool))
 728                 bio->bi_iter.bi_sector =
 729                         (block << pool->sectors_per_block_shift) |
 730                         (bi_sector & (pool->sectors_per_block - 1));
 731         else
 732                 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
 733                                  sector_div(bi_sector, pool->sectors_per_block);
 734 }
 735 
 736 static void remap_to_origin(struct thin_c *tc, struct bio *bio)
 737 {
 738         bio_set_dev(bio, tc->origin_dev->bdev);
 739 }
 740 
 741 static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
 742 {
 743         return op_is_flush(bio->bi_opf) &&
 744                 dm_thin_changed_this_transaction(tc->td);
 745 }
 746 
 747 static void inc_all_io_entry(struct pool *pool, struct bio *bio)
 748 {
 749         struct dm_thin_endio_hook *h;
 750 
 751         if (bio_op(bio) == REQ_OP_DISCARD)
 752                 return;
 753 
 754         h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
 755         h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
 756 }
 757 
 758 static void issue(struct thin_c *tc, struct bio *bio)
 759 {
 760         struct pool *pool = tc->pool;
 761         unsigned long flags;
 762 
 763         if (!bio_triggers_commit(tc, bio)) {
 764                 generic_make_request(bio);
 765                 return;
 766         }
 767 
 768         /*
 769          * Complete bio with an error if earlier I/O caused changes to
 770          * the metadata that can't be committed e.g, due to I/O errors
 771          * on the metadata device.
 772          */
 773         if (dm_thin_aborted_changes(tc->td)) {
 774                 bio_io_error(bio);
 775                 return;
 776         }
 777 
 778         /*
 779          * Batch together any bios that trigger commits and then issue a
 780          * single commit for them in process_deferred_bios().
 781          */
 782         spin_lock_irqsave(&pool->lock, flags);
 783         bio_list_add(&pool->deferred_flush_bios, bio);
 784         spin_unlock_irqrestore(&pool->lock, flags);
 785 }
 786 
 787 static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
 788 {
 789         remap_to_origin(tc, bio);
 790         issue(tc, bio);
 791 }
 792 
 793 static void remap_and_issue(struct thin_c *tc, struct bio *bio,
 794                             dm_block_t block)
 795 {
 796         remap(tc, bio, block);
 797         issue(tc, bio);
 798 }
 799 
 800 /*----------------------------------------------------------------*/
 801 
 802 /*
 803  * Bio endio functions.
 804  */
 805 struct dm_thin_new_mapping {
 806         struct list_head list;
 807 
 808         bool pass_discard:1;
 809         bool maybe_shared:1;
 810 
 811         /*
 812          * Track quiescing, copying and zeroing preparation actions.  When this
 813          * counter hits zero the block is prepared and can be inserted into the
 814          * btree.
 815          */
 816         atomic_t prepare_actions;
 817 
 818         blk_status_t status;
 819         struct thin_c *tc;
 820         dm_block_t virt_begin, virt_end;
 821         dm_block_t data_block;
 822         struct dm_bio_prison_cell *cell;
 823 
 824         /*
 825          * If the bio covers the whole area of a block then we can avoid
 826          * zeroing or copying.  Instead this bio is hooked.  The bio will
 827          * still be in the cell, so care has to be taken to avoid issuing
 828          * the bio twice.
 829          */
 830         struct bio *bio;
 831         bio_end_io_t *saved_bi_end_io;
 832 };
 833 
 834 static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
 835 {
 836         struct pool *pool = m->tc->pool;
 837 
 838         if (atomic_dec_and_test(&m->prepare_actions)) {
 839                 list_add_tail(&m->list, &pool->prepared_mappings);
 840                 wake_worker(pool);
 841         }
 842 }
 843 
 844 static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
 845 {
 846         unsigned long flags;
 847         struct pool *pool = m->tc->pool;
 848 
 849         spin_lock_irqsave(&pool->lock, flags);
 850         __complete_mapping_preparation(m);
 851         spin_unlock_irqrestore(&pool->lock, flags);
 852 }
 853 
 854 static void copy_complete(int read_err, unsigned long write_err, void *context)
 855 {
 856         struct dm_thin_new_mapping *m = context;
 857 
 858         m->status = read_err || write_err ? BLK_STS_IOERR : 0;
 859         complete_mapping_preparation(m);
 860 }
 861 
 862 static void overwrite_endio(struct bio *bio)
 863 {
 864         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
 865         struct dm_thin_new_mapping *m = h->overwrite_mapping;
 866 
 867         bio->bi_end_io = m->saved_bi_end_io;
 868 
 869         m->status = bio->bi_status;
 870         complete_mapping_preparation(m);
 871 }
 872 
 873 /*----------------------------------------------------------------*/
 874 
 875 /*
 876  * Workqueue.
 877  */
 878 
 879 /*
 880  * Prepared mapping jobs.
 881  */
 882 
 883 /*
 884  * This sends the bios in the cell, except the original holder, back
 885  * to the deferred_bios list.
 886  */
 887 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
 888 {
 889         struct pool *pool = tc->pool;
 890         unsigned long flags;
 891 
 892         spin_lock_irqsave(&tc->lock, flags);
 893         cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
 894         spin_unlock_irqrestore(&tc->lock, flags);
 895 
 896         wake_worker(pool);
 897 }
 898 
 899 static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
 900 
 901 struct remap_info {
 902         struct thin_c *tc;
 903         struct bio_list defer_bios;
 904         struct bio_list issue_bios;
 905 };
 906 
 907 static void __inc_remap_and_issue_cell(void *context,
 908                                        struct dm_bio_prison_cell *cell)
 909 {
 910         struct remap_info *info = context;
 911         struct bio *bio;
 912 
 913         while ((bio = bio_list_pop(&cell->bios))) {
 914                 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)
 915                         bio_list_add(&info->defer_bios, bio);
 916                 else {
 917                         inc_all_io_entry(info->tc->pool, bio);
 918 
 919                         /*
 920                          * We can't issue the bios with the bio prison lock
 921                          * held, so we add them to a list to issue on
 922                          * return from this function.
 923                          */
 924                         bio_list_add(&info->issue_bios, bio);
 925                 }
 926         }
 927 }
 928 
 929 static void inc_remap_and_issue_cell(struct thin_c *tc,
 930                                      struct dm_bio_prison_cell *cell,
 931                                      dm_block_t block)
 932 {
 933         struct bio *bio;
 934         struct remap_info info;
 935 
 936         info.tc = tc;
 937         bio_list_init(&info.defer_bios);
 938         bio_list_init(&info.issue_bios);
 939 
 940         /*
 941          * We have to be careful to inc any bios we're about to issue
 942          * before the cell is released, and avoid a race with new bios
 943          * being added to the cell.
 944          */
 945         cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
 946                            &info, cell);
 947 
 948         while ((bio = bio_list_pop(&info.defer_bios)))
 949                 thin_defer_bio(tc, bio);
 950 
 951         while ((bio = bio_list_pop(&info.issue_bios)))
 952                 remap_and_issue(info.tc, bio, block);
 953 }
 954 
 955 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
 956 {
 957         cell_error(m->tc->pool, m->cell);
 958         list_del(&m->list);
 959         mempool_free(m, &m->tc->pool->mapping_pool);
 960 }
 961 
 962 static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio)
 963 {
 964         struct pool *pool = tc->pool;
 965         unsigned long flags;
 966 
 967         /*
 968          * If the bio has the REQ_FUA flag set we must commit the metadata
 969          * before signaling its completion.
 970          */
 971         if (!bio_triggers_commit(tc, bio)) {
 972                 bio_endio(bio);
 973                 return;
 974         }
 975 
 976         /*
 977          * Complete bio with an error if earlier I/O caused changes to the
 978          * metadata that can't be committed, e.g, due to I/O errors on the
 979          * metadata device.
 980          */
 981         if (dm_thin_aborted_changes(tc->td)) {
 982                 bio_io_error(bio);
 983                 return;
 984         }
 985 
 986         /*
 987          * Batch together any bios that trigger commits and then issue a
 988          * single commit for them in process_deferred_bios().
 989          */
 990         spin_lock_irqsave(&pool->lock, flags);
 991         bio_list_add(&pool->deferred_flush_completions, bio);
 992         spin_unlock_irqrestore(&pool->lock, flags);
 993 }
 994 
 995 static void process_prepared_mapping(struct dm_thin_new_mapping *m)
 996 {
 997         struct thin_c *tc = m->tc;
 998         struct pool *pool = tc->pool;
 999         struct bio *bio = m->bio;
1000         int r;
1001 
1002         if (m->status) {
1003                 cell_error(pool, m->cell);
1004                 goto out;
1005         }
1006 
1007         /*
1008          * Commit the prepared block into the mapping btree.
1009          * Any I/O for this block arriving after this point will get
1010          * remapped to it directly.
1011          */
1012         r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
1013         if (r) {
1014                 metadata_operation_failed(pool, "dm_thin_insert_block", r);
1015                 cell_error(pool, m->cell);
1016                 goto out;
1017         }
1018 
1019         /*
1020          * Release any bios held while the block was being provisioned.
1021          * If we are processing a write bio that completely covers the block,
1022          * we already processed it so can ignore it now when processing
1023          * the bios in the cell.
1024          */
1025         if (bio) {
1026                 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
1027                 complete_overwrite_bio(tc, bio);
1028         } else {
1029                 inc_all_io_entry(tc->pool, m->cell->holder);
1030                 remap_and_issue(tc, m->cell->holder, m->data_block);
1031                 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
1032         }
1033 
1034 out:
1035         list_del(&m->list);
1036         mempool_free(m, &pool->mapping_pool);
1037 }
1038 
1039 /*----------------------------------------------------------------*/
1040 
1041 static void free_discard_mapping(struct dm_thin_new_mapping *m)
1042 {
1043         struct thin_c *tc = m->tc;
1044         if (m->cell)
1045                 cell_defer_no_holder(tc, m->cell);
1046         mempool_free(m, &tc->pool->mapping_pool);
1047 }
1048 
1049 static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
1050 {
1051         bio_io_error(m->bio);
1052         free_discard_mapping(m);
1053 }
1054 
1055 static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
1056 {
1057         bio_endio(m->bio);
1058         free_discard_mapping(m);
1059 }
1060 
1061 static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
1062 {
1063         int r;
1064         struct thin_c *tc = m->tc;
1065 
1066         r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
1067         if (r) {
1068                 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
1069                 bio_io_error(m->bio);
1070         } else
1071                 bio_endio(m->bio);
1072 
1073         cell_defer_no_holder(tc, m->cell);
1074         mempool_free(m, &tc->pool->mapping_pool);
1075 }
1076 
1077 /*----------------------------------------------------------------*/
1078 
1079 static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,
1080                                                    struct bio *discard_parent)
1081 {
1082         /*
1083          * We've already unmapped this range of blocks, but before we
1084          * passdown we have to check that these blocks are now unused.
1085          */
1086         int r = 0;
1087         bool shared = true;
1088         struct thin_c *tc = m->tc;
1089         struct pool *pool = tc->pool;
1090         dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
1091         struct discard_op op;
1092 
1093         begin_discard(&op, tc, discard_parent);
1094         while (b != end) {
1095                 /* find start of unmapped run */
1096                 for (; b < end; b++) {
1097                         r = dm_pool_block_is_shared(pool->pmd, b, &shared);
1098                         if (r)
1099                                 goto out;
1100 
1101                         if (!shared)
1102                                 break;
1103                 }
1104 
1105                 if (b == end)
1106                         break;
1107 
1108                 /* find end of run */
1109                 for (e = b + 1; e != end; e++) {
1110                         r = dm_pool_block_is_shared(pool->pmd, e, &shared);
1111                         if (r)
1112                                 goto out;
1113 
1114                         if (shared)
1115                                 break;
1116                 }
1117 
1118                 r = issue_discard(&op, b, e);
1119                 if (r)
1120                         goto out;
1121 
1122                 b = e;
1123         }
1124 out:
1125         end_discard(&op, r);
1126 }
1127 
1128 static void queue_passdown_pt2(struct dm_thin_new_mapping *m)
1129 {
1130         unsigned long flags;
1131         struct pool *pool = m->tc->pool;
1132 
1133         spin_lock_irqsave(&pool->lock, flags);
1134         list_add_tail(&m->list, &pool->prepared_discards_pt2);
1135         spin_unlock_irqrestore(&pool->lock, flags);
1136         wake_worker(pool);
1137 }
1138 
1139 static void passdown_endio(struct bio *bio)
1140 {
1141         /*
1142          * It doesn't matter if the passdown discard failed, we still want
1143          * to unmap (we ignore err).
1144          */
1145         queue_passdown_pt2(bio->bi_private);
1146         bio_put(bio);
1147 }
1148 
1149 static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
1150 {
1151         int r;
1152         struct thin_c *tc = m->tc;
1153         struct pool *pool = tc->pool;
1154         struct bio *discard_parent;
1155         dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);
1156 
1157         /*
1158          * Only this thread allocates blocks, so we can be sure that the
1159          * newly unmapped blocks will not be allocated before the end of
1160          * the function.
1161          */
1162         r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
1163         if (r) {
1164                 metadata_operation_failed(pool, "dm_thin_remove_range", r);
1165                 bio_io_error(m->bio);
1166                 cell_defer_no_holder(tc, m->cell);
1167                 mempool_free(m, &pool->mapping_pool);
1168                 return;
1169         }
1170 
1171         /*
1172          * Increment the unmapped blocks.  This prevents a race between the
1173          * passdown io and reallocation of freed blocks.
1174          */
1175         r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end);
1176         if (r) {
1177                 metadata_operation_failed(pool, "dm_pool_inc_data_range", r);
1178                 bio_io_error(m->bio);
1179                 cell_defer_no_holder(tc, m->cell);
1180                 mempool_free(m, &pool->mapping_pool);
1181                 return;
1182         }
1183 
1184         discard_parent = bio_alloc(GFP_NOIO, 1);
1185         if (!discard_parent) {
1186                 DMWARN("%s: unable to allocate top level discard bio for passdown. Skipping passdown.",
1187                        dm_device_name(tc->pool->pool_md));
1188                 queue_passdown_pt2(m);
1189 
1190         } else {
1191                 discard_parent->bi_end_io = passdown_endio;
1192                 discard_parent->bi_private = m;
1193 
1194                 if (m->maybe_shared)
1195                         passdown_double_checking_shared_status(m, discard_parent);
1196                 else {
1197                         struct discard_op op;
1198 
1199                         begin_discard(&op, tc, discard_parent);
1200                         r = issue_discard(&op, m->data_block, data_end);
1201                         end_discard(&op, r);
1202                 }
1203         }
1204 }
1205 
1206 static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
1207 {
1208         int r;
1209         struct thin_c *tc = m->tc;
1210         struct pool *pool = tc->pool;
1211 
1212         /*
1213          * The passdown has completed, so now we can decrement all those
1214          * unmapped blocks.
1215          */
1216         r = dm_pool_dec_data_range(pool->pmd, m->data_block,
1217                                    m->data_block + (m->virt_end - m->virt_begin));
1218         if (r) {
1219                 metadata_operation_failed(pool, "dm_pool_dec_data_range", r);
1220                 bio_io_error(m->bio);
1221         } else
1222                 bio_endio(m->bio);
1223 
1224         cell_defer_no_holder(tc, m->cell);
1225         mempool_free(m, &pool->mapping_pool);
1226 }
1227 
1228 static void process_prepared(struct pool *pool, struct list_head *head,
1229                              process_mapping_fn *fn)
1230 {
1231         unsigned long flags;
1232         struct list_head maps;
1233         struct dm_thin_new_mapping *m, *tmp;
1234 
1235         INIT_LIST_HEAD(&maps);
1236         spin_lock_irqsave(&pool->lock, flags);
1237         list_splice_init(head, &maps);
1238         spin_unlock_irqrestore(&pool->lock, flags);
1239 
1240         list_for_each_entry_safe(m, tmp, &maps, list)
1241                 (*fn)(m);
1242 }
1243 
1244 /*
1245  * Deferred bio jobs.
1246  */
1247 static int io_overlaps_block(struct pool *pool, struct bio *bio)
1248 {
1249         return bio->bi_iter.bi_size ==
1250                 (pool->sectors_per_block << SECTOR_SHIFT);
1251 }
1252 
1253 static int io_overwrites_block(struct pool *pool, struct bio *bio)
1254 {
1255         return (bio_data_dir(bio) == WRITE) &&
1256                 io_overlaps_block(pool, bio);
1257 }
1258 
1259 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
1260                                bio_end_io_t *fn)
1261 {
1262         *save = bio->bi_end_io;
1263         bio->bi_end_io = fn;
1264 }
1265 
1266 static int ensure_next_mapping(struct pool *pool)
1267 {
1268         if (pool->next_mapping)
1269                 return 0;
1270 
1271         pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);
1272 
1273         return pool->next_mapping ? 0 : -ENOMEM;
1274 }
1275 
1276 static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
1277 {
1278         struct dm_thin_new_mapping *m = pool->next_mapping;
1279 
1280         BUG_ON(!pool->next_mapping);
1281 
1282         memset(m, 0, sizeof(struct dm_thin_new_mapping));
1283         INIT_LIST_HEAD(&m->list);
1284         m->bio = NULL;
1285 
1286         pool->next_mapping = NULL;
1287 
1288         return m;
1289 }
1290 
1291 static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
1292                     sector_t begin, sector_t end)
1293 {
1294         struct dm_io_region to;
1295 
1296         to.bdev = tc->pool_dev->bdev;
1297         to.sector = begin;
1298         to.count = end - begin;
1299 
1300         dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
1301 }
1302 
1303 static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
1304                                       dm_block_t data_begin,
1305                                       struct dm_thin_new_mapping *m)
1306 {
1307         struct pool *pool = tc->pool;
1308         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1309 
1310         h->overwrite_mapping = m;
1311         m->bio = bio;
1312         save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1313         inc_all_io_entry(pool, bio);
1314         remap_and_issue(tc, bio, data_begin);
1315 }
1316 
1317 /*
1318  * A partial copy also needs to zero the uncopied region.
1319  */
1320 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
1321                           struct dm_dev *origin, dm_block_t data_origin,
1322                           dm_block_t data_dest,
1323                           struct dm_bio_prison_cell *cell, struct bio *bio,
1324                           sector_t len)
1325 {
1326         struct pool *pool = tc->pool;
1327         struct dm_thin_new_mapping *m = get_next_mapping(pool);
1328 
1329         m->tc = tc;
1330         m->virt_begin = virt_block;
1331         m->virt_end = virt_block + 1u;
1332         m->data_block = data_dest;
1333         m->cell = cell;
1334 
1335         /*
1336          * quiesce action + copy action + an extra reference held for the
1337          * duration of this function (we may need to inc later for a
1338          * partial zero).
1339          */
1340         atomic_set(&m->prepare_actions, 3);
1341 
1342         if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
1343                 complete_mapping_preparation(m); /* already quiesced */
1344 
1345         /*
1346          * IO to pool_dev remaps to the pool target's data_dev.
1347          *
1348          * If the whole block of data is being overwritten, we can issue the
1349          * bio immediately. Otherwise we use kcopyd to clone the data first.
1350          */
1351         if (io_overwrites_block(pool, bio))
1352                 remap_and_issue_overwrite(tc, bio, data_dest, m);
1353         else {
1354                 struct dm_io_region from, to;
1355 
1356                 from.bdev = origin->bdev;
1357                 from.sector = data_origin * pool->sectors_per_block;
1358                 from.count = len;
1359 
1360                 to.bdev = tc->pool_dev->bdev;
1361                 to.sector = data_dest * pool->sectors_per_block;
1362                 to.count = len;
1363 
1364                 dm_kcopyd_copy(pool->copier, &from, 1, &to,
1365                                0, copy_complete, m);
1366 
1367                 /*
1368                  * Do we need to zero a tail region?
1369                  */
1370                 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
1371                         atomic_inc(&m->prepare_actions);
1372                         ll_zero(tc, m,
1373                                 data_dest * pool->sectors_per_block + len,
1374                                 (data_dest + 1) * pool->sectors_per_block);
1375                 }
1376         }
1377 
1378         complete_mapping_preparation(m); /* drop our ref */
1379 }
1380 
1381 static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1382                                    dm_block_t data_origin, dm_block_t data_dest,
1383                                    struct dm_bio_prison_cell *cell, struct bio *bio)
1384 {
1385         schedule_copy(tc, virt_block, tc->pool_dev,
1386                       data_origin, data_dest, cell, bio,
1387                       tc->pool->sectors_per_block);
1388 }
1389 
1390 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
1391                           dm_block_t data_block, struct dm_bio_prison_cell *cell,
1392                           struct bio *bio)
1393 {
1394         struct pool *pool = tc->pool;
1395         struct dm_thin_new_mapping *m = get_next_mapping(pool);
1396 
1397         atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
1398         m->tc = tc;
1399         m->virt_begin = virt_block;
1400         m->virt_end = virt_block + 1u;
1401         m->data_block = data_block;
1402         m->cell = cell;
1403 
1404         /*
1405          * If the whole block of data is being overwritten or we are not
1406          * zeroing pre-existing data, we can issue the bio immediately.
1407          * Otherwise we use kcopyd to zero the data first.
1408          */
1409         if (pool->pf.zero_new_blocks) {
1410                 if (io_overwrites_block(pool, bio))
1411                         remap_and_issue_overwrite(tc, bio, data_block, m);
1412                 else
1413                         ll_zero(tc, m, data_block * pool->sectors_per_block,
1414                                 (data_block + 1) * pool->sectors_per_block);
1415         } else
1416                 process_prepared_mapping(m);
1417 }
1418 
1419 static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1420                                    dm_block_t data_dest,
1421                                    struct dm_bio_prison_cell *cell, struct bio *bio)
1422 {
1423         struct pool *pool = tc->pool;
1424         sector_t virt_block_begin = virt_block * pool->sectors_per_block;
1425         sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
1426 
1427         if (virt_block_end <= tc->origin_size)
1428                 schedule_copy(tc, virt_block, tc->origin_dev,
1429                               virt_block, data_dest, cell, bio,
1430                               pool->sectors_per_block);
1431 
1432         else if (virt_block_begin < tc->origin_size)
1433                 schedule_copy(tc, virt_block, tc->origin_dev,
1434                               virt_block, data_dest, cell, bio,
1435                               tc->origin_size - virt_block_begin);
1436 
1437         else
1438                 schedule_zero(tc, virt_block, data_dest, cell, bio);
1439 }
1440 
1441 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1442 
1443 static void requeue_bios(struct pool *pool);
1444 
1445 static bool is_read_only_pool_mode(enum pool_mode mode)
1446 {
1447         return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY);
1448 }
1449 
1450 static bool is_read_only(struct pool *pool)
1451 {
1452         return is_read_only_pool_mode(get_pool_mode(pool));
1453 }
1454 
1455 static void check_for_metadata_space(struct pool *pool)
1456 {
1457         int r;
1458         const char *ooms_reason = NULL;
1459         dm_block_t nr_free;
1460 
1461         r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free);
1462         if (r)
1463                 ooms_reason = "Could not get free metadata blocks";
1464         else if (!nr_free)
1465                 ooms_reason = "No free metadata blocks";
1466 
1467         if (ooms_reason && !is_read_only(pool)) {
1468                 DMERR("%s", ooms_reason);
1469                 set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE);
1470         }
1471 }
1472 
1473 static void check_for_data_space(struct pool *pool)
1474 {
1475         int r;
1476         dm_block_t nr_free;
1477 
1478         if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
1479                 return;
1480 
1481         r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
1482         if (r)
1483                 return;
1484 
1485         if (nr_free) {
1486                 set_pool_mode(pool, PM_WRITE);
1487                 requeue_bios(pool);
1488         }
1489 }
1490 
1491 /*
1492  * A non-zero return indicates read_only or fail_io mode.
1493  * Many callers don't care about the return value.
1494  */
1495 static int commit(struct pool *pool)
1496 {
1497         int r;
1498 
1499         if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE)
1500                 return -EINVAL;
1501 
1502         r = dm_pool_commit_metadata(pool->pmd);
1503         if (r)
1504                 metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
1505         else {
1506                 check_for_metadata_space(pool);
1507                 check_for_data_space(pool);
1508         }
1509 
1510         return r;
1511 }
1512 
1513 static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1514 {
1515         unsigned long flags;
1516 
1517         if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1518                 DMWARN("%s: reached low water mark for data device: sending event.",
1519                        dm_device_name(pool->pool_md));
1520                 spin_lock_irqsave(&pool->lock, flags);
1521                 pool->low_water_triggered = true;
1522                 spin_unlock_irqrestore(&pool->lock, flags);
1523                 dm_table_event(pool->ti->table);
1524         }
1525 }
1526 
1527 static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1528 {
1529         int r;
1530         dm_block_t free_blocks;
1531         struct pool *pool = tc->pool;
1532 
1533         if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
1534                 return -EINVAL;
1535 
1536         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1537         if (r) {
1538                 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
1539                 return r;
1540         }
1541 
1542         check_low_water_mark(pool, free_blocks);
1543 
1544         if (!free_blocks) {
1545                 /*
1546                  * Try to commit to see if that will free up some
1547                  * more space.
1548                  */
1549                 r = commit(pool);
1550                 if (r)
1551                         return r;
1552 
1553                 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1554                 if (r) {
1555                         metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
1556                         return r;
1557                 }
1558 
1559                 if (!free_blocks) {
1560                         set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
1561                         return -ENOSPC;
1562                 }
1563         }
1564 
1565         r = dm_pool_alloc_data_block(pool->pmd, result);
1566         if (r) {
1567                 if (r == -ENOSPC)
1568                         set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
1569                 else
1570                         metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
1571                 return r;
1572         }
1573 
1574         r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks);
1575         if (r) {
1576                 metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r);
1577                 return r;
1578         }
1579 
1580         if (!free_blocks) {
1581                 /* Let's commit before we use up the metadata reserve. */
1582                 r = commit(pool);
1583                 if (r)
1584                         return r;
1585         }
1586 
1587         return 0;
1588 }
1589 
1590 /*
1591  * If we have run out of space, queue bios until the device is
1592  * resumed, presumably after having been reloaded with more space.
1593  */
1594 static void retry_on_resume(struct bio *bio)
1595 {
1596         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1597         struct thin_c *tc = h->tc;
1598         unsigned long flags;
1599 
1600         spin_lock_irqsave(&tc->lock, flags);
1601         bio_list_add(&tc->retry_on_resume_list, bio);
1602         spin_unlock_irqrestore(&tc->lock, flags);
1603 }
1604 
1605 static blk_status_t should_error_unserviceable_bio(struct pool *pool)
1606 {
1607         enum pool_mode m = get_pool_mode(pool);
1608 
1609         switch (m) {
1610         case PM_WRITE:
1611                 /* Shouldn't get here */
1612                 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
1613                 return BLK_STS_IOERR;
1614 
1615         case PM_OUT_OF_DATA_SPACE:
1616                 return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0;
1617 
1618         case PM_OUT_OF_METADATA_SPACE:
1619         case PM_READ_ONLY:
1620         case PM_FAIL:
1621                 return BLK_STS_IOERR;
1622         default:
1623                 /* Shouldn't get here */
1624                 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
1625                 return BLK_STS_IOERR;
1626         }
1627 }
1628 
1629 static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1630 {
1631         blk_status_t error = should_error_unserviceable_bio(pool);
1632 
1633         if (error) {
1634                 bio->bi_status = error;
1635                 bio_endio(bio);
1636         } else
1637                 retry_on_resume(bio);
1638 }
1639 
1640 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
1641 {
1642         struct bio *bio;
1643         struct bio_list bios;
1644         blk_status_t error;
1645 
1646         error = should_error_unserviceable_bio(pool);
1647         if (error) {
1648                 cell_error_with_code(pool, cell, error);
1649                 return;
1650         }
1651 
1652         bio_list_init(&bios);
1653         cell_release(pool, cell, &bios);
1654 
1655         while ((bio = bio_list_pop(&bios)))
1656                 retry_on_resume(bio);
1657 }
1658 
1659 static void process_discard_cell_no_passdown(struct thin_c *tc,
1660                                              struct dm_bio_prison_cell *virt_cell)
1661 {
1662         struct pool *pool = tc->pool;
1663         struct dm_thin_new_mapping *m = get_next_mapping(pool);
1664 
1665         /*
1666          * We don't need to lock the data blocks, since there's no
1667          * passdown.  We only lock data blocks for allocation and breaking sharing.
1668          */
1669         m->tc = tc;
1670         m->virt_begin = virt_cell->key.block_begin;
1671         m->virt_end = virt_cell->key.block_end;
1672         m->cell = virt_cell;
1673         m->bio = virt_cell->holder;
1674 
1675         if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1676                 pool->process_prepared_discard(m);
1677 }
1678 
1679 static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
1680                                  struct bio *bio)
1681 {
1682         struct pool *pool = tc->pool;
1683 
1684         int r;
1685         bool maybe_shared;
1686         struct dm_cell_key data_key;
1687         struct dm_bio_prison_cell *data_cell;
1688         struct dm_thin_new_mapping *m;
1689         dm_block_t virt_begin, virt_end, data_begin;
1690 
1691         while (begin != end) {
1692                 r = ensure_next_mapping(pool);
1693                 if (r)
1694                         /* we did our best */
1695                         return;
1696 
1697                 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
1698                                               &data_begin, &maybe_shared);
1699                 if (r)
1700                         /*
1701                          * Silently fail, letting any mappings we've
1702                          * created complete.
1703                          */
1704                         break;
1705 
1706                 build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
1707                 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
1708                         /* contention, we'll give up with this range */
1709                         begin = virt_end;
1710                         continue;
1711                 }
1712 
1713                 /*
1714                  * IO may still be going to the destination block.  We must
1715                  * quiesce before we can do the removal.
1716                  */
1717                 m = get_next_mapping(pool);
1718                 m->tc = tc;
1719                 m->maybe_shared = maybe_shared;
1720                 m->virt_begin = virt_begin;
1721                 m->virt_end = virt_end;
1722                 m->data_block = data_begin;
1723                 m->cell = data_cell;
1724                 m->bio = bio;
1725 
1726                 /*
1727                  * The parent bio must not complete before sub discard bios are
1728                  * chained to it (see end_discard's bio_chain)!
1729                  *
1730                  * This per-mapping bi_remaining increment is paired with
1731                  * the implicit decrement that occurs via bio_endio() in
1732                  * end_discard().
1733                  */
1734                 bio_inc_remaining(bio);
1735                 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1736                         pool->process_prepared_discard(m);
1737 
1738                 begin = virt_end;
1739         }
1740 }
1741 
1742 static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
1743 {
1744         struct bio *bio = virt_cell->holder;
1745         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1746 
1747         /*
1748          * The virt_cell will only get freed once the origin bio completes.
1749          * This means it will remain locked while all the individual
1750          * passdown bios are in flight.
1751          */
1752         h->cell = virt_cell;
1753         break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
1754 
1755         /*
1756          * We complete the bio now, knowing that the bi_remaining field
1757          * will prevent completion until the sub range discards have
1758          * completed.
1759          */
1760         bio_endio(bio);
1761 }
1762 
1763 static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1764 {
1765         dm_block_t begin, end;
1766         struct dm_cell_key virt_key;
1767         struct dm_bio_prison_cell *virt_cell;
1768 
1769         get_bio_block_range(tc, bio, &begin, &end);
1770         if (begin == end) {
1771                 /*
1772                  * The discard covers less than a block.
1773                  */
1774                 bio_endio(bio);
1775                 return;
1776         }
1777 
1778         build_key(tc->td, VIRTUAL, begin, end, &virt_key);
1779         if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
1780                 /*
1781                  * Potential starvation issue: We're relying on the
1782                  * fs/application being well behaved, and not trying to
1783                  * send IO to a region at the same time as discarding it.
1784                  * If they do this persistently then it's possible this
1785                  * cell will never be granted.
1786                  */
1787                 return;
1788 
1789         tc->pool->process_discard_cell(tc, virt_cell);
1790 }
1791 
1792 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1793                           struct dm_cell_key *key,
1794                           struct dm_thin_lookup_result *lookup_result,
1795                           struct dm_bio_prison_cell *cell)
1796 {
1797         int r;
1798         dm_block_t data_block;
1799         struct pool *pool = tc->pool;
1800 
1801         r = alloc_data_block(tc, &data_block);
1802         switch (r) {
1803         case 0:
1804                 schedule_internal_copy(tc, block, lookup_result->block,
1805                                        data_block, cell, bio);
1806                 break;
1807 
1808         case -ENOSPC:
1809                 retry_bios_on_resume(pool, cell);
1810                 break;
1811 
1812         default:
1813                 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1814                             __func__, r);
1815                 cell_error(pool, cell);
1816                 break;
1817         }
1818 }
1819 
1820 static void __remap_and_issue_shared_cell(void *context,
1821                                           struct dm_bio_prison_cell *cell)
1822 {
1823         struct remap_info *info = context;
1824         struct bio *bio;
1825 
1826         while ((bio = bio_list_pop(&cell->bios))) {
1827                 if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) ||
1828                     bio_op(bio) == REQ_OP_DISCARD)
1829                         bio_list_add(&info->defer_bios, bio);
1830                 else {
1831                         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1832 
1833                         h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
1834                         inc_all_io_entry(info->tc->pool, bio);
1835                         bio_list_add(&info->issue_bios, bio);
1836                 }
1837         }
1838 }
1839 
1840 static void remap_and_issue_shared_cell(struct thin_c *tc,
1841                                         struct dm_bio_prison_cell *cell,
1842                                         dm_block_t block)
1843 {
1844         struct bio *bio;
1845         struct remap_info info;
1846 
1847         info.tc = tc;
1848         bio_list_init(&info.defer_bios);
1849         bio_list_init(&info.issue_bios);
1850 
1851         cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
1852                            &info, cell);
1853 
1854         while ((bio = bio_list_pop(&info.defer_bios)))
1855                 thin_defer_bio(tc, bio);
1856 
1857         while ((bio = bio_list_pop(&info.issue_bios)))
1858                 remap_and_issue(tc, bio, block);
1859 }
1860 
1861 static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1862                                dm_block_t block,
1863                                struct dm_thin_lookup_result *lookup_result,
1864                                struct dm_bio_prison_cell *virt_cell)
1865 {
1866         struct dm_bio_prison_cell *data_cell;
1867         struct pool *pool = tc->pool;
1868         struct dm_cell_key key;
1869 
1870         /*
1871          * If cell is already occupied, then sharing is already in the process
1872          * of being broken so we have nothing further to do here.
1873          */
1874         build_data_key(tc->td, lookup_result->block, &key);
1875         if (bio_detain(pool, &key, bio, &data_cell)) {
1876                 cell_defer_no_holder(tc, virt_cell);
1877                 return;
1878         }
1879 
1880         if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
1881                 break_sharing(tc, bio, block, &key, lookup_result, data_cell);
1882                 cell_defer_no_holder(tc, virt_cell);
1883         } else {
1884                 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1885 
1886                 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
1887                 inc_all_io_entry(pool, bio);
1888                 remap_and_issue(tc, bio, lookup_result->block);
1889 
1890                 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
1891                 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
1892         }
1893 }
1894 
1895 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1896                             struct dm_bio_prison_cell *cell)
1897 {
1898         int r;
1899         dm_block_t data_block;
1900         struct pool *pool = tc->pool;
1901 
1902         /*
1903          * Remap empty bios (flushes) immediately, without provisioning.
1904          */
1905         if (!bio->bi_iter.bi_size) {
1906                 inc_all_io_entry(pool, bio);
1907                 cell_defer_no_holder(tc, cell);
1908 
1909                 remap_and_issue(tc, bio, 0);
1910                 return;
1911         }
1912 
1913         /*
1914          * Fill read bios with zeroes and complete them immediately.
1915          */
1916         if (bio_data_dir(bio) == READ) {
1917                 zero_fill_bio(bio);
1918                 cell_defer_no_holder(tc, cell);
1919                 bio_endio(bio);
1920                 return;
1921         }
1922 
1923         r = alloc_data_block(tc, &data_block);
1924         switch (r) {
1925         case 0:
1926                 if (tc->origin_dev)
1927                         schedule_external_copy(tc, block, data_block, cell, bio);
1928                 else
1929                         schedule_zero(tc, block, data_block, cell, bio);
1930                 break;
1931 
1932         case -ENOSPC:
1933                 retry_bios_on_resume(pool, cell);
1934                 break;
1935 
1936         default:
1937                 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1938                             __func__, r);
1939                 cell_error(pool, cell);
1940                 break;
1941         }
1942 }
1943 
1944 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1945 {
1946         int r;
1947         struct pool *pool = tc->pool;
1948         struct bio *bio = cell->holder;
1949         dm_block_t block = get_bio_block(tc, bio);
1950         struct dm_thin_lookup_result lookup_result;
1951 
1952         if (tc->requeue_mode) {
1953                 cell_requeue(pool, cell);
1954                 return;
1955         }
1956 
1957         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1958         switch (r) {
1959         case 0:
1960                 if (lookup_result.shared)
1961                         process_shared_bio(tc, bio, block, &lookup_result, cell);
1962                 else {
1963                         inc_all_io_entry(pool, bio);
1964                         remap_and_issue(tc, bio, lookup_result.block);
1965                         inc_remap_and_issue_cell(tc, cell, lookup_result.block);
1966                 }
1967                 break;
1968 
1969         case -ENODATA:
1970                 if (bio_data_dir(bio) == READ && tc->origin_dev) {
1971                         inc_all_io_entry(pool, bio);
1972                         cell_defer_no_holder(tc, cell);
1973 
1974                         if (bio_end_sector(bio) <= tc->origin_size)
1975                                 remap_to_origin_and_issue(tc, bio);
1976 
1977                         else if (bio->bi_iter.bi_sector < tc->origin_size) {
1978                                 zero_fill_bio(bio);
1979                                 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1980                                 remap_to_origin_and_issue(tc, bio);
1981 
1982                         } else {
1983                                 zero_fill_bio(bio);
1984                                 bio_endio(bio);
1985                         }
1986                 } else
1987                         provision_block(tc, bio, block, cell);
1988                 break;
1989 
1990         default:
1991                 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1992                             __func__, r);
1993                 cell_defer_no_holder(tc, cell);
1994                 bio_io_error(bio);
1995                 break;
1996         }
1997 }
1998 
1999 static void process_bio(struct thin_c *tc, struct bio *bio)
2000 {
2001         struct pool *pool = tc->pool;
2002         dm_block_t block = get_bio_block(tc, bio);
2003         struct dm_bio_prison_cell *cell;
2004         struct dm_cell_key key;
2005 
2006         /*
2007          * If cell is already occupied, then the block is already
2008          * being provisioned so we have nothing further to do here.
2009          */
2010         build_virtual_key(tc->td, block, &key);
2011         if (bio_detain(pool, &key, bio, &cell))
2012                 return;
2013 
2014         process_cell(tc, cell);
2015 }
2016 
2017 static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
2018                                     struct dm_bio_prison_cell *cell)
2019 {
2020         int r;
2021         int rw = bio_data_dir(bio);
2022         dm_block_t block = get_bio_block(tc, bio);
2023         struct dm_thin_lookup_result lookup_result;
2024 
2025         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
2026         switch (r) {
2027         case 0:
2028                 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
2029                         handle_unserviceable_bio(tc->pool, bio);
2030                         if (cell)
2031                                 cell_defer_no_holder(tc, cell);
2032                 } else {
2033                         inc_all_io_entry(tc->pool, bio);
2034                         remap_and_issue(tc, bio, lookup_result.block);
2035                         if (cell)
2036                                 inc_remap_and_issue_cell(tc, cell, lookup_result.block);
2037                 }
2038                 break;
2039 
2040         case -ENODATA:
2041                 if (cell)
2042                         cell_defer_no_holder(tc, cell);
2043                 if (rw != READ) {
2044                         handle_unserviceable_bio(tc->pool, bio);
2045                         break;
2046                 }
2047 
2048                 if (tc->origin_dev) {
2049                         inc_all_io_entry(tc->pool, bio);
2050                         remap_to_origin_and_issue(tc, bio);
2051                         break;
2052                 }
2053 
2054                 zero_fill_bio(bio);
2055                 bio_endio(bio);
2056                 break;
2057 
2058         default:
2059                 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
2060                             __func__, r);
2061                 if (cell)
2062                         cell_defer_no_holder(tc, cell);
2063                 bio_io_error(bio);
2064                 break;
2065         }
2066 }
2067 
2068 static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
2069 {
2070         __process_bio_read_only(tc, bio, NULL);
2071 }
2072 
2073 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2074 {
2075         __process_bio_read_only(tc, cell->holder, cell);
2076 }
2077 
2078 static void process_bio_success(struct thin_c *tc, struct bio *bio)
2079 {
2080         bio_endio(bio);
2081 }
2082 
2083 static void process_bio_fail(struct thin_c *tc, struct bio *bio)
2084 {
2085         bio_io_error(bio);
2086 }
2087 
2088 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2089 {
2090         cell_success(tc->pool, cell);
2091 }
2092 
2093 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2094 {
2095         cell_error(tc->pool, cell);
2096 }
2097 
2098 /*
2099  * FIXME: should we also commit due to size of transaction, measured in
2100  * metadata blocks?
2101  */
2102 static int need_commit_due_to_time(struct pool *pool)
2103 {
2104         return !time_in_range(jiffies, pool->last_commit_jiffies,
2105                               pool->last_commit_jiffies + COMMIT_PERIOD);
2106 }
2107 
2108 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
2109 #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
2110 
2111 static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
2112 {
2113         struct rb_node **rbp, *parent;
2114         struct dm_thin_endio_hook *pbd;
2115         sector_t bi_sector = bio->bi_iter.bi_sector;
2116 
2117         rbp = &tc->sort_bio_list.rb_node;
2118         parent = NULL;
2119         while (*rbp) {
2120                 parent = *rbp;
2121                 pbd = thin_pbd(parent);
2122 
2123                 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
2124                         rbp = &(*rbp)->rb_left;
2125                 else
2126                         rbp = &(*rbp)->rb_right;
2127         }
2128 
2129         pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2130         rb_link_node(&pbd->rb_node, parent, rbp);
2131         rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
2132 }
2133 
2134 static void __extract_sorted_bios(struct thin_c *tc)
2135 {
2136         struct rb_node *node;
2137         struct dm_thin_endio_hook *pbd;
2138         struct bio *bio;
2139 
2140         for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
2141                 pbd = thin_pbd(node);
2142                 bio = thin_bio(pbd);
2143 
2144                 bio_list_add(&tc->deferred_bio_list, bio);
2145                 rb_erase(&pbd->rb_node, &tc->sort_bio_list);
2146         }
2147 
2148         WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
2149 }
2150 
2151 static void __sort_thin_deferred_bios(struct thin_c *tc)
2152 {
2153         struct bio *bio;
2154         struct bio_list bios;
2155 
2156         bio_list_init(&bios);
2157         bio_list_merge(&bios, &tc->deferred_bio_list);
2158         bio_list_init(&tc->deferred_bio_list);
2159 
2160         /* Sort deferred_bio_list using rb-tree */
2161         while ((bio = bio_list_pop(&bios)))
2162                 __thin_bio_rb_add(tc, bio);
2163 
2164         /*
2165          * Transfer the sorted bios in sort_bio_list back to
2166          * deferred_bio_list to allow lockless submission of
2167          * all bios.
2168          */
2169         __extract_sorted_bios(tc);
2170 }
2171 
2172 static void process_thin_deferred_bios(struct thin_c *tc)
2173 {
2174         struct pool *pool = tc->pool;
2175         unsigned long flags;
2176         struct bio *bio;
2177         struct bio_list bios;
2178         struct blk_plug plug;
2179         unsigned count = 0;
2180 
2181         if (tc->requeue_mode) {
2182                 error_thin_bio_list(tc, &tc->deferred_bio_list,
2183                                 BLK_STS_DM_REQUEUE);
2184                 return;
2185         }
2186 
2187         bio_list_init(&bios);
2188 
2189         spin_lock_irqsave(&tc->lock, flags);
2190 
2191         if (bio_list_empty(&tc->deferred_bio_list)) {
2192                 spin_unlock_irqrestore(&tc->lock, flags);
2193                 return;
2194         }
2195 
2196         __sort_thin_deferred_bios(tc);
2197 
2198         bio_list_merge(&bios, &tc->deferred_bio_list);
2199         bio_list_init(&tc->deferred_bio_list);
2200 
2201         spin_unlock_irqrestore(&tc->lock, flags);
2202 
2203         blk_start_plug(&plug);
2204         while ((bio = bio_list_pop(&bios))) {
2205                 /*
2206                  * If we've got no free new_mapping structs, and processing
2207                  * this bio might require one, we pause until there are some
2208                  * prepared mappings to process.
2209                  */
2210                 if (ensure_next_mapping(pool)) {
2211                         spin_lock_irqsave(&tc->lock, flags);
2212                         bio_list_add(&tc->deferred_bio_list, bio);
2213                         bio_list_merge(&tc->deferred_bio_list, &bios);
2214                         spin_unlock_irqrestore(&tc->lock, flags);
2215                         break;
2216                 }
2217 
2218                 if (bio_op(bio) == REQ_OP_DISCARD)
2219                         pool->process_discard(tc, bio);
2220                 else
2221                         pool->process_bio(tc, bio);
2222 
2223                 if ((count++ & 127) == 0) {
2224                         throttle_work_update(&pool->throttle);
2225                         dm_pool_issue_prefetches(pool->pmd);
2226                 }
2227         }
2228         blk_finish_plug(&plug);
2229 }
2230 
2231 static int cmp_cells(const void *lhs, const void *rhs)
2232 {
2233         struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
2234         struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
2235 
2236         BUG_ON(!lhs_cell->holder);
2237         BUG_ON(!rhs_cell->holder);
2238 
2239         if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
2240                 return -1;
2241 
2242         if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
2243                 return 1;
2244 
2245         return 0;
2246 }
2247 
2248 static unsigned sort_cells(struct pool *pool, struct list_head *cells)
2249 {
2250         unsigned count = 0;
2251         struct dm_bio_prison_cell *cell, *tmp;
2252 
2253         list_for_each_entry_safe(cell, tmp, cells, user_list) {
2254                 if (count >= CELL_SORT_ARRAY_SIZE)
2255                         break;
2256 
2257                 pool->cell_sort_array[count++] = cell;
2258                 list_del(&cell->user_list);
2259         }
2260 
2261         sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
2262 
2263         return count;
2264 }
2265 
2266 static void process_thin_deferred_cells(struct thin_c *tc)
2267 {
2268         struct pool *pool = tc->pool;
2269         unsigned long flags;
2270         struct list_head cells;
2271         struct dm_bio_prison_cell *cell;
2272         unsigned i, j, count;
2273 
2274         INIT_LIST_HEAD(&cells);
2275 
2276         spin_lock_irqsave(&tc->lock, flags);
2277         list_splice_init(&tc->deferred_cells, &cells);
2278         spin_unlock_irqrestore(&tc->lock, flags);
2279 
2280         if (list_empty(&cells))
2281                 return;
2282 
2283         do {
2284                 count = sort_cells(tc->pool, &cells);
2285 
2286                 for (i = 0; i < count; i++) {
2287                         cell = pool->cell_sort_array[i];
2288                         BUG_ON(!cell->holder);
2289 
2290                         /*
2291                          * If we've got no free new_mapping structs, and processing
2292                          * this bio might require one, we pause until there are some
2293                          * prepared mappings to process.
2294                          */
2295                         if (ensure_next_mapping(pool)) {
2296                                 for (j = i; j < count; j++)
2297                                         list_add(&pool->cell_sort_array[j]->user_list, &cells);
2298 
2299                                 spin_lock_irqsave(&tc->lock, flags);
2300                                 list_splice(&cells, &tc->deferred_cells);
2301                                 spin_unlock_irqrestore(&tc->lock, flags);
2302                                 return;
2303                         }
2304 
2305                         if (bio_op(cell->holder) == REQ_OP_DISCARD)
2306                                 pool->process_discard_cell(tc, cell);
2307                         else
2308                                 pool->process_cell(tc, cell);
2309                 }
2310         } while (!list_empty(&cells));
2311 }
2312 
2313 static void thin_get(struct thin_c *tc);
2314 static void thin_put(struct thin_c *tc);
2315 
2316 /*
2317  * We can't hold rcu_read_lock() around code that can block.  So we
2318  * find a thin with the rcu lock held; bump a refcount; then drop
2319  * the lock.
2320  */
2321 static struct thin_c *get_first_thin(struct pool *pool)
2322 {
2323         struct thin_c *tc = NULL;
2324 
2325         rcu_read_lock();
2326         if (!list_empty(&pool->active_thins)) {
2327                 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
2328                 thin_get(tc);
2329         }
2330         rcu_read_unlock();
2331 
2332         return tc;
2333 }
2334 
2335 static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
2336 {
2337         struct thin_c *old_tc = tc;
2338 
2339         rcu_read_lock();
2340         list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
2341                 thin_get(tc);
2342                 thin_put(old_tc);
2343                 rcu_read_unlock();
2344                 return tc;
2345         }
2346         thin_put(old_tc);
2347         rcu_read_unlock();
2348 
2349         return NULL;
2350 }
2351 
2352 static void process_deferred_bios(struct pool *pool)
2353 {
2354         unsigned long flags;
2355         struct bio *bio;
2356         struct bio_list bios, bio_completions;
2357         struct thin_c *tc;
2358 
2359         tc = get_first_thin(pool);
2360         while (tc) {
2361                 process_thin_deferred_cells(tc);
2362                 process_thin_deferred_bios(tc);
2363                 tc = get_next_thin(pool, tc);
2364         }
2365 
2366         /*
2367          * If there are any deferred flush bios, we must commit the metadata
2368          * before issuing them or signaling their completion.
2369          */
2370         bio_list_init(&bios);
2371         bio_list_init(&bio_completions);
2372 
2373         spin_lock_irqsave(&pool->lock, flags);
2374         bio_list_merge(&bios, &pool->deferred_flush_bios);
2375         bio_list_init(&pool->deferred_flush_bios);
2376 
2377         bio_list_merge(&bio_completions, &pool->deferred_flush_completions);
2378         bio_list_init(&pool->deferred_flush_completions);
2379         spin_unlock_irqrestore(&pool->lock, flags);
2380 
2381         if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
2382             !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
2383                 return;
2384 
2385         if (commit(pool)) {
2386                 bio_list_merge(&bios, &bio_completions);
2387 
2388                 while ((bio = bio_list_pop(&bios)))
2389                         bio_io_error(bio);
2390                 return;
2391         }
2392         pool->last_commit_jiffies = jiffies;
2393 
2394         while ((bio = bio_list_pop(&bio_completions)))
2395                 bio_endio(bio);
2396 
2397         while ((bio = bio_list_pop(&bios))) {
2398                 /*
2399                  * The data device was flushed as part of metadata commit,
2400                  * so complete redundant flushes immediately.
2401                  */
2402                 if (bio->bi_opf & REQ_PREFLUSH)
2403                         bio_endio(bio);
2404                 else
2405                         generic_make_request(bio);
2406         }
2407 }
2408 
2409 static void do_worker(struct work_struct *ws)
2410 {
2411         struct pool *pool = container_of(ws, struct pool, worker);
2412 
2413         throttle_work_start(&pool->throttle);
2414         dm_pool_issue_prefetches(pool->pmd);
2415         throttle_work_update(&pool->throttle);
2416         process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
2417         throttle_work_update(&pool->throttle);
2418         process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
2419         throttle_work_update(&pool->throttle);
2420         process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2);
2421         throttle_work_update(&pool->throttle);
2422         process_deferred_bios(pool);
2423         throttle_work_complete(&pool->throttle);
2424 }
2425 
2426 /*
2427  * We want to commit periodically so that not too much
2428  * unwritten data builds up.
2429  */
2430 static void do_waker(struct work_struct *ws)
2431 {
2432         struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
2433         wake_worker(pool);
2434         queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
2435 }
2436 
2437 /*
2438  * We're holding onto IO to allow userland time to react.  After the
2439  * timeout either the pool will have been resized (and thus back in
2440  * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
2441  */
2442 static void do_no_space_timeout(struct work_struct *ws)
2443 {
2444         struct pool *pool = container_of(to_delayed_work(ws), struct pool,
2445                                          no_space_timeout);
2446 
2447         if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
2448                 pool->pf.error_if_no_space = true;
2449                 notify_of_pool_mode_change(pool);
2450                 error_retry_list_with_code(pool, BLK_STS_NOSPC);
2451         }
2452 }
2453 
2454 /*----------------------------------------------------------------*/
2455 
2456 struct pool_work {
2457         struct work_struct worker;
2458         struct completion complete;
2459 };
2460 
2461 static struct pool_work *to_pool_work(struct work_struct *ws)
2462 {
2463         return container_of(ws, struct pool_work, worker);
2464 }
2465 
2466 static void pool_work_complete(struct pool_work *pw)
2467 {
2468         complete(&pw->complete);
2469 }
2470 
2471 static void pool_work_wait(struct pool_work *pw, struct pool *pool,
2472                            void (*fn)(struct work_struct *))
2473 {
2474         INIT_WORK_ONSTACK(&pw->worker, fn);
2475         init_completion(&pw->complete);
2476         queue_work(pool->wq, &pw->worker);
2477         wait_for_completion(&pw->complete);
2478 }
2479 
2480 /*----------------------------------------------------------------*/
2481 
2482 struct noflush_work {
2483         struct pool_work pw;
2484         struct thin_c *tc;
2485 };
2486 
2487 static struct noflush_work *to_noflush(struct work_struct *ws)
2488 {
2489         return container_of(to_pool_work(ws), struct noflush_work, pw);
2490 }
2491 
2492 static void do_noflush_start(struct work_struct *ws)
2493 {
2494         struct noflush_work *w = to_noflush(ws);
2495         w->tc->requeue_mode = true;
2496         requeue_io(w->tc);
2497         pool_work_complete(&w->pw);
2498 }
2499 
2500 static void do_noflush_stop(struct work_struct *ws)
2501 {
2502         struct noflush_work *w = to_noflush(ws);
2503         w->tc->requeue_mode = false;
2504         pool_work_complete(&w->pw);
2505 }
2506 
2507 static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
2508 {
2509         struct noflush_work w;
2510 
2511         w.tc = tc;
2512         pool_work_wait(&w.pw, tc->pool, fn);
2513 }
2514 
2515 /*----------------------------------------------------------------*/
2516 
2517 static bool passdown_enabled(struct pool_c *pt)
2518 {
2519         return pt->adjusted_pf.discard_passdown;
2520 }
2521 
2522 static void set_discard_callbacks(struct pool *pool)
2523 {
2524         struct pool_c *pt = pool->ti->private;
2525 
2526         if (passdown_enabled(pt)) {
2527                 pool->process_discard_cell = process_discard_cell_passdown;
2528                 pool->process_prepared_discard = process_prepared_discard_passdown_pt1;
2529                 pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2;
2530         } else {
2531                 pool->process_discard_cell = process_discard_cell_no_passdown;
2532                 pool->process_prepared_discard = process_prepared_discard_no_passdown;
2533         }
2534 }
2535 
2536 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
2537 {
2538         struct pool_c *pt = pool->ti->private;
2539         bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
2540         enum pool_mode old_mode = get_pool_mode(pool);
2541         unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ;
2542 
2543         /*
2544          * Never allow the pool to transition to PM_WRITE mode if user
2545          * intervention is required to verify metadata and data consistency.
2546          */
2547         if (new_mode == PM_WRITE && needs_check) {
2548                 DMERR("%s: unable to switch pool to write mode until repaired.",
2549                       dm_device_name(pool->pool_md));
2550                 if (old_mode != new_mode)
2551                         new_mode = old_mode;
2552                 else
2553                         new_mode = PM_READ_ONLY;
2554         }
2555         /*
2556          * If we were in PM_FAIL mode, rollback of metadata failed.  We're
2557          * not going to recover without a thin_repair.  So we never let the
2558          * pool move out of the old mode.
2559          */
2560         if (old_mode == PM_FAIL)
2561                 new_mode = old_mode;
2562 
2563         switch (new_mode) {
2564         case PM_FAIL:
2565                 dm_pool_metadata_read_only(pool->pmd);
2566                 pool->process_bio = process_bio_fail;
2567                 pool->process_discard = process_bio_fail;
2568                 pool->process_cell = process_cell_fail;
2569                 pool->process_discard_cell = process_cell_fail;
2570                 pool->process_prepared_mapping = process_prepared_mapping_fail;
2571                 pool->process_prepared_discard = process_prepared_discard_fail;
2572 
2573                 error_retry_list(pool);
2574                 break;
2575 
2576         case PM_OUT_OF_METADATA_SPACE:
2577         case PM_READ_ONLY:
2578                 dm_pool_metadata_read_only(pool->pmd);
2579                 pool->process_bio = process_bio_read_only;
2580                 pool->process_discard = process_bio_success;
2581                 pool->process_cell = process_cell_read_only;
2582                 pool->process_discard_cell = process_cell_success;
2583                 pool->process_prepared_mapping = process_prepared_mapping_fail;
2584                 pool->process_prepared_discard = process_prepared_discard_success;
2585 
2586                 error_retry_list(pool);
2587                 break;
2588 
2589         case PM_OUT_OF_DATA_SPACE:
2590                 /*
2591                  * Ideally we'd never hit this state; the low water mark
2592                  * would trigger userland to extend the pool before we
2593                  * completely run out of data space.  However, many small
2594                  * IOs to unprovisioned space can consume data space at an
2595                  * alarming rate.  Adjust your low water mark if you're
2596                  * frequently seeing this mode.
2597                  */
2598                 pool->out_of_data_space = true;
2599                 pool->process_bio = process_bio_read_only;
2600                 pool->process_discard = process_discard_bio;
2601                 pool->process_cell = process_cell_read_only;
2602                 pool->process_prepared_mapping = process_prepared_mapping;
2603                 set_discard_callbacks(pool);
2604 
2605                 if (!pool->pf.error_if_no_space && no_space_timeout)
2606                         queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
2607                 break;
2608 
2609         case PM_WRITE:
2610                 if (old_mode == PM_OUT_OF_DATA_SPACE)
2611                         cancel_delayed_work_sync(&pool->no_space_timeout);
2612                 pool->out_of_data_space = false;
2613                 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;
2614                 dm_pool_metadata_read_write(pool->pmd);
2615                 pool->process_bio = process_bio;
2616                 pool->process_discard = process_discard_bio;
2617                 pool->process_cell = process_cell;
2618                 pool->process_prepared_mapping = process_prepared_mapping;
2619                 set_discard_callbacks(pool);
2620                 break;
2621         }
2622 
2623         pool->pf.mode = new_mode;
2624         /*
2625          * The pool mode may have changed, sync it so bind_control_target()
2626          * doesn't cause an unexpected mode transition on resume.
2627          */
2628         pt->adjusted_pf.mode = new_mode;
2629 
2630         if (old_mode != new_mode)
2631                 notify_of_pool_mode_change(pool);
2632 }
2633 
2634 static void abort_transaction(struct pool *pool)
2635 {
2636         const char *dev_name = dm_device_name(pool->pool_md);
2637 
2638         DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
2639         if (dm_pool_abort_metadata(pool->pmd)) {
2640                 DMERR("%s: failed to abort metadata transaction", dev_name);
2641                 set_pool_mode(pool, PM_FAIL);
2642         }
2643 
2644         if (dm_pool_metadata_set_needs_check(pool->pmd)) {
2645                 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
2646                 set_pool_mode(pool, PM_FAIL);
2647         }
2648 }
2649 
2650 static void metadata_operation_failed(struct pool *pool, const char *op, int r)
2651 {
2652         DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2653                     dm_device_name(pool->pool_md), op, r);
2654 
2655         abort_transaction(pool);
2656         set_pool_mode(pool, PM_READ_ONLY);
2657 }
2658 
2659 /*----------------------------------------------------------------*/
2660 
2661 /*
2662  * Mapping functions.
2663  */
2664 
2665 /*
2666  * Called only while mapping a thin bio to hand it over to the workqueue.
2667  */
2668 static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
2669 {
2670         unsigned long flags;
2671         struct pool *pool = tc->pool;
2672 
2673         spin_lock_irqsave(&tc->lock, flags);
2674         bio_list_add(&tc->deferred_bio_list, bio);
2675         spin_unlock_irqrestore(&tc->lock, flags);
2676 
2677         wake_worker(pool);
2678 }
2679 
2680 static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
2681 {
2682         struct pool *pool = tc->pool;
2683 
2684         throttle_lock(&pool->throttle);
2685         thin_defer_bio(tc, bio);
2686         throttle_unlock(&pool->throttle);
2687 }
2688 
2689 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2690 {
2691         unsigned long flags;
2692         struct pool *pool = tc->pool;
2693 
2694         throttle_lock(&pool->throttle);
2695         spin_lock_irqsave(&tc->lock, flags);
2696         list_add_tail(&cell->user_list, &tc->deferred_cells);
2697         spin_unlock_irqrestore(&tc->lock, flags);
2698         throttle_unlock(&pool->throttle);
2699 
2700         wake_worker(pool);
2701 }
2702 
2703 static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
2704 {
2705         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2706 
2707         h->tc = tc;
2708         h->shared_read_entry = NULL;
2709         h->all_io_entry = NULL;
2710         h->overwrite_mapping = NULL;
2711         h->cell = NULL;
2712 }
2713 
2714 /*
2715  * Non-blocking function called from the thin target's map function.
2716  */
2717 static int thin_bio_map(struct dm_target *ti, struct bio *bio)
2718 {
2719         int r;
2720         struct thin_c *tc = ti->private;
2721         dm_block_t block = get_bio_block(tc, bio);
2722         struct dm_thin_device *td = tc->td;
2723         struct dm_thin_lookup_result result;
2724         struct dm_bio_prison_cell *virt_cell, *data_cell;
2725         struct dm_cell_key key;
2726 
2727         thin_hook_bio(tc, bio);
2728 
2729         if (tc->requeue_mode) {
2730                 bio->bi_status = BLK_STS_DM_REQUEUE;
2731                 bio_endio(bio);
2732                 return DM_MAPIO_SUBMITTED;
2733         }
2734 
2735         if (get_pool_mode(tc->pool) == PM_FAIL) {
2736                 bio_io_error(bio);
2737                 return DM_MAPIO_SUBMITTED;
2738         }
2739 
2740         if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) {
2741                 thin_defer_bio_with_throttle(tc, bio);
2742                 return DM_MAPIO_SUBMITTED;
2743         }
2744 
2745         /*
2746          * We must hold the virtual cell before doing the lookup, otherwise
2747          * there's a race with discard.
2748          */
2749         build_virtual_key(tc->td, block, &key);
2750         if (bio_detain(tc->pool, &key, bio, &virt_cell))
2751                 return DM_MAPIO_SUBMITTED;
2752 
2753         r = dm_thin_find_block(td, block, 0, &result);
2754 
2755         /*
2756          * Note that we defer readahead too.
2757          */
2758         switch (r) {
2759         case 0:
2760                 if (unlikely(result.shared)) {
2761                         /*
2762                          * We have a race condition here between the
2763                          * result.shared value returned by the lookup and
2764                          * snapshot creation, which may cause new
2765                          * sharing.
2766                          *
2767                          * To avoid this always quiesce the origin before
2768                          * taking the snap.  You want to do this anyway to
2769                          * ensure a consistent application view
2770                          * (i.e. lockfs).
2771                          *
2772                          * More distant ancestors are irrelevant. The
2773                          * shared flag will be set in their case.
2774                          */
2775                         thin_defer_cell(tc, virt_cell);
2776                         return DM_MAPIO_SUBMITTED;
2777                 }
2778 
2779                 build_data_key(tc->td, result.block, &key);
2780                 if (bio_detain(tc->pool, &key, bio, &data_cell)) {
2781                         cell_defer_no_holder(tc, virt_cell);
2782                         return DM_MAPIO_SUBMITTED;
2783                 }
2784 
2785                 inc_all_io_entry(tc->pool, bio);
2786                 cell_defer_no_holder(tc, data_cell);
2787                 cell_defer_no_holder(tc, virt_cell);
2788 
2789                 remap(tc, bio, result.block);
2790                 return DM_MAPIO_REMAPPED;
2791 
2792         case -ENODATA:
2793         case -EWOULDBLOCK:
2794                 thin_defer_cell(tc, virt_cell);
2795                 return DM_MAPIO_SUBMITTED;
2796 
2797         default:
2798                 /*
2799                  * Must always call bio_io_error on failure.
2800                  * dm_thin_find_block can fail with -EINVAL if the
2801                  * pool is switched to fail-io mode.
2802                  */
2803                 bio_io_error(bio);
2804                 cell_defer_no_holder(tc, virt_cell);
2805                 return DM_MAPIO_SUBMITTED;
2806         }
2807 }
2808 
2809 static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2810 {
2811         struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
2812         struct request_queue *q;
2813 
2814         if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE)
2815                 return 1;
2816 
2817         q = bdev_get_queue(pt->data_dev->bdev);
2818         return bdi_congested(q->backing_dev_info, bdi_bits);
2819 }
2820 
2821 static void requeue_bios(struct pool *pool)
2822 {
2823         unsigned long flags;
2824         struct thin_c *tc;
2825 
2826         rcu_read_lock();
2827         list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2828                 spin_lock_irqsave(&tc->lock, flags);
2829                 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
2830                 bio_list_init(&tc->retry_on_resume_list);
2831                 spin_unlock_irqrestore(&tc->lock, flags);
2832         }
2833         rcu_read_unlock();
2834 }
2835 
2836 /*----------------------------------------------------------------
2837  * Binding of control targets to a pool object
2838  *--------------------------------------------------------------*/
2839 static bool data_dev_supports_discard(struct pool_c *pt)
2840 {
2841         struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2842 
2843         return q && blk_queue_discard(q);
2844 }
2845 
2846 static bool is_factor(sector_t block_size, uint32_t n)
2847 {
2848         return !sector_div(block_size, n);
2849 }
2850 
2851 /*
2852  * If discard_passdown was enabled verify that the data device
2853  * supports discards.  Disable discard_passdown if not.
2854  */
2855 static void disable_passdown_if_not_supported(struct pool_c *pt)
2856 {
2857         struct pool *pool = pt->pool;
2858         struct block_device *data_bdev = pt->data_dev->bdev;
2859         struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;
2860         const char *reason = NULL;
2861         char buf[BDEVNAME_SIZE];
2862 
2863         if (!pt->adjusted_pf.discard_passdown)
2864                 return;
2865 
2866         if (!data_dev_supports_discard(pt))
2867                 reason = "discard unsupported";
2868 
2869         else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2870                 reason = "max discard sectors smaller than a block";
2871 
2872         if (reason) {
2873                 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason);
2874                 pt->adjusted_pf.discard_passdown = false;
2875         }
2876 }
2877 
2878 static int bind_control_target(struct pool *pool, struct dm_target *ti)
2879 {
2880         struct pool_c *pt = ti->private;
2881 
2882         /*
2883          * We want to make sure that a pool in PM_FAIL mode is never upgraded.
2884          */
2885         enum pool_mode old_mode = get_pool_mode(pool);
2886         enum pool_mode new_mode = pt->adjusted_pf.mode;
2887 
2888         /*
2889          * Don't change the pool's mode until set_pool_mode() below.
2890          * Otherwise the pool's process_* function pointers may
2891          * not match the desired pool mode.
2892          */
2893         pt->adjusted_pf.mode = old_mode;
2894 
2895         pool->ti = ti;
2896         pool->pf = pt->adjusted_pf;
2897         pool->low_water_blocks = pt->low_water_blocks;
2898 
2899         set_pool_mode(pool, new_mode);
2900 
2901         return 0;
2902 }
2903 
2904 static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2905 {
2906         if (pool->ti == ti)
2907                 pool->ti = NULL;
2908 }
2909 
2910 /*----------------------------------------------------------------
2911  * Pool creation
2912  *--------------------------------------------------------------*/
2913 /* Initialize pool features. */
2914 static void pool_features_init(struct pool_features *pf)
2915 {
2916         pf->mode = PM_WRITE;
2917         pf->zero_new_blocks = true;
2918         pf->discard_enabled = true;
2919         pf->discard_passdown = true;
2920         pf->error_if_no_space = false;
2921 }
2922 
2923 static void __pool_destroy(struct pool *pool)
2924 {
2925         __pool_table_remove(pool);
2926 
2927         vfree(pool->cell_sort_array);
2928         if (dm_pool_metadata_close(pool->pmd) < 0)
2929                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2930 
2931         dm_bio_prison_destroy(pool->prison);
2932         dm_kcopyd_client_destroy(pool->copier);
2933 
2934         if (pool->wq)
2935                 destroy_workqueue(pool->wq);
2936 
2937         if (pool->next_mapping)
2938                 mempool_free(pool->next_mapping, &pool->mapping_pool);
2939         mempool_exit(&pool->mapping_pool);
2940         dm_deferred_set_destroy(pool->shared_read_ds);
2941         dm_deferred_set_destroy(pool->all_io_ds);
2942         kfree(pool);
2943 }
2944 
2945 static struct kmem_cache *_new_mapping_cache;
2946 
2947 static struct pool *pool_create(struct mapped_device *pool_md,
2948                                 struct block_device *metadata_dev,
2949                                 struct block_device *data_dev,
2950                                 unsigned long block_size,
2951                                 int read_only, char **error)
2952 {
2953         int r;
2954         void *err_p;
2955         struct pool *pool;
2956         struct dm_pool_metadata *pmd;
2957         bool format_device = read_only ? false : true;
2958 
2959         pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
2960         if (IS_ERR(pmd)) {
2961                 *error = "Error creating metadata object";
2962                 return (struct pool *)pmd;
2963         }
2964 
2965         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2966         if (!pool) {
2967                 *error = "Error allocating memory for pool";
2968                 err_p = ERR_PTR(-ENOMEM);
2969                 goto bad_pool;
2970         }
2971 
2972         pool->pmd = pmd;
2973         pool->sectors_per_block = block_size;
2974         if (block_size & (block_size - 1))
2975                 pool->sectors_per_block_shift = -1;
2976         else
2977                 pool->sectors_per_block_shift = __ffs(block_size);
2978         pool->low_water_blocks = 0;
2979         pool_features_init(&pool->pf);
2980         pool->prison = dm_bio_prison_create();
2981         if (!pool->prison) {
2982                 *error = "Error creating pool's bio prison";
2983                 err_p = ERR_PTR(-ENOMEM);
2984                 goto bad_prison;
2985         }
2986 
2987         pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2988         if (IS_ERR(pool->copier)) {
2989                 r = PTR_ERR(pool->copier);
2990                 *error = "Error creating pool's kcopyd client";
2991                 err_p = ERR_PTR(r);
2992                 goto bad_kcopyd_client;
2993         }
2994 
2995         /*
2996          * Create singlethreaded workqueue that will service all devices
2997          * that use this metadata.
2998          */
2999         pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
3000         if (!pool->wq) {
3001                 *error = "Error creating pool's workqueue";
3002                 err_p = ERR_PTR(-ENOMEM);
3003                 goto bad_wq;
3004         }
3005 
3006         throttle_init(&pool->throttle);
3007         INIT_WORK(&pool->worker, do_worker);
3008         INIT_DELAYED_WORK(&pool->waker, do_waker);
3009         INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
3010         spin_lock_init(&pool->lock);
3011         bio_list_init(&pool->deferred_flush_bios);
3012         bio_list_init(&pool->deferred_flush_completions);
3013         INIT_LIST_HEAD(&pool->prepared_mappings);
3014         INIT_LIST_HEAD(&pool->prepared_discards);
3015         INIT_LIST_HEAD(&pool->prepared_discards_pt2);
3016         INIT_LIST_HEAD(&pool->active_thins);
3017         pool->low_water_triggered = false;
3018         pool->suspended = true;
3019         pool->out_of_data_space = false;
3020 
3021         pool->shared_read_ds = dm_deferred_set_create();
3022         if (!pool->shared_read_ds) {
3023                 *error = "Error creating pool's shared read deferred set";
3024                 err_p = ERR_PTR(-ENOMEM);
3025                 goto bad_shared_read_ds;
3026         }
3027 
3028         pool->all_io_ds = dm_deferred_set_create();
3029         if (!pool->all_io_ds) {
3030                 *error = "Error creating pool's all io deferred set";
3031                 err_p = ERR_PTR(-ENOMEM);
3032                 goto bad_all_io_ds;
3033         }
3034 
3035         pool->next_mapping = NULL;
3036         r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE,
3037                                    _new_mapping_cache);
3038         if (r) {
3039                 *error = "Error creating pool's mapping mempool";
3040                 err_p = ERR_PTR(r);
3041                 goto bad_mapping_pool;
3042         }
3043 
3044         pool->cell_sort_array =
3045                 vmalloc(array_size(CELL_SORT_ARRAY_SIZE,
3046                                    sizeof(*pool->cell_sort_array)));
3047         if (!pool->cell_sort_array) {
3048                 *error = "Error allocating cell sort array";
3049                 err_p = ERR_PTR(-ENOMEM);
3050                 goto bad_sort_array;
3051         }
3052 
3053         pool->ref_count = 1;
3054         pool->last_commit_jiffies = jiffies;
3055         pool->pool_md = pool_md;
3056         pool->md_dev = metadata_dev;
3057         pool->data_dev = data_dev;
3058         __pool_table_insert(pool);
3059 
3060         return pool;
3061 
3062 bad_sort_array:
3063         mempool_exit(&pool->mapping_pool);
3064 bad_mapping_pool:
3065         dm_deferred_set_destroy(pool->all_io_ds);
3066 bad_all_io_ds:
3067         dm_deferred_set_destroy(pool->shared_read_ds);
3068 bad_shared_read_ds:
3069         destroy_workqueue(pool->wq);
3070 bad_wq:
3071         dm_kcopyd_client_destroy(pool->copier);
3072 bad_kcopyd_client:
3073         dm_bio_prison_destroy(pool->prison);
3074 bad_prison:
3075         kfree(pool);
3076 bad_pool:
3077         if (dm_pool_metadata_close(pmd))
3078                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
3079 
3080         return err_p;
3081 }
3082 
3083 static void __pool_inc(struct pool *pool)
3084 {
3085         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
3086         pool->ref_count++;
3087 }
3088 
3089 static void __pool_dec(struct pool *pool)
3090 {
3091         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
3092         BUG_ON(!pool->ref_count);
3093         if (!--pool->ref_count)
3094                 __pool_destroy(pool);
3095 }
3096 
3097 static struct pool *__pool_find(struct mapped_device *pool_md,
3098                                 struct block_device *metadata_dev,
3099                                 struct block_device *data_dev,
3100                                 unsigned long block_size, int read_only,
3101                                 char **error, int *created)
3102 {
3103         struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
3104 
3105         if (pool) {
3106                 if (pool->pool_md != pool_md) {
3107                         *error = "metadata device already in use by a pool";
3108                         return ERR_PTR(-EBUSY);
3109                 }
3110                 if (pool->data_dev != data_dev) {
3111                         *error = "data device already in use by a pool";
3112                         return ERR_PTR(-EBUSY);
3113                 }
3114                 __pool_inc(pool);
3115 
3116         } else {
3117                 pool = __pool_table_lookup(pool_md);
3118                 if (pool) {
3119                         if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) {
3120                                 *error = "different pool cannot replace a pool";
3121                                 return ERR_PTR(-EINVAL);
3122                         }
3123                         __pool_inc(pool);
3124 
3125                 } else {
3126                         pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error);
3127                         *created = 1;
3128                 }
3129         }
3130 
3131         return pool;
3132 }
3133 
3134 /*----------------------------------------------------------------
3135  * Pool target methods
3136  *--------------------------------------------------------------*/
3137 static void pool_dtr(struct dm_target *ti)
3138 {
3139         struct pool_c *pt = ti->private;
3140 
3141         mutex_lock(&dm_thin_pool_table.mutex);
3142 
3143         unbind_control_target(pt->pool, ti);
3144         __pool_dec(pt->pool);
3145         dm_put_device(ti, pt->metadata_dev);
3146         dm_put_device(ti, pt->data_dev);
3147         bio_uninit(&pt->flush_bio);
3148         kfree(pt);
3149 
3150         mutex_unlock(&dm_thin_pool_table.mutex);
3151 }
3152 
3153 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
3154                                struct dm_target *ti)
3155 {
3156         int r;
3157         unsigned argc;
3158         const char *arg_name;
3159 
3160         static const struct dm_arg _args[] = {
3161                 {0, 4, "Invalid number of pool feature arguments"},
3162         };
3163 
3164         /*
3165          * No feature arguments supplied.
3166          */
3167         if (!as->argc)
3168                 return 0;
3169 
3170         r = dm_read_arg_group(_args, as, &argc, &ti->error);
3171         if (r)
3172                 return -EINVAL;
3173 
3174         while (argc && !r) {
3175                 arg_name = dm_shift_arg(as);
3176                 argc--;
3177 
3178                 if (!strcasecmp(arg_name, "skip_block_zeroing"))
3179                         pf->zero_new_blocks = false;
3180 
3181                 else if (!strcasecmp(arg_name, "ignore_discard"))
3182                         pf->discard_enabled = false;
3183 
3184                 else if (!strcasecmp(arg_name, "no_discard_passdown"))
3185                         pf->discard_passdown = false;
3186 
3187                 else if (!strcasecmp(arg_name, "read_only"))
3188                         pf->mode = PM_READ_ONLY;
3189 
3190                 else if (!strcasecmp(arg_name, "error_if_no_space"))
3191                         pf->error_if_no_space = true;
3192 
3193                 else {
3194                         ti->error = "Unrecognised pool feature requested";
3195                         r = -EINVAL;
3196                         break;
3197                 }
3198         }
3199 
3200         return r;
3201 }
3202 
3203 static void metadata_low_callback(void *context)
3204 {
3205         struct pool *pool = context;
3206 
3207         DMWARN("%s: reached low water mark for metadata device: sending event.",
3208                dm_device_name(pool->pool_md));
3209 
3210         dm_table_event(pool->ti->table);
3211 }
3212 
3213 /*
3214  * We need to flush the data device **before** committing the metadata.
3215  *
3216  * This ensures that the data blocks of any newly inserted mappings are
3217  * properly written to non-volatile storage and won't be lost in case of a
3218  * crash.
3219  *
3220  * Failure to do so can result in data corruption in the case of internal or
3221  * external snapshots and in the case of newly provisioned blocks, when block
3222  * zeroing is enabled.
3223  */
3224 static int metadata_pre_commit_callback(void *context)
3225 {
3226         struct pool_c *pt = context;
3227         struct bio *flush_bio = &pt->flush_bio;
3228 
3229         bio_reset(flush_bio);
3230         bio_set_dev(flush_bio, pt->data_dev->bdev);
3231         flush_bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
3232 
3233         return submit_bio_wait(flush_bio);
3234 }
3235 
3236 static sector_t get_dev_size(struct block_device *bdev)
3237 {
3238         return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
3239 }
3240 
3241 static void warn_if_metadata_device_too_big(struct block_device *bdev)
3242 {
3243         sector_t metadata_dev_size = get_dev_size(bdev);
3244         char buffer[BDEVNAME_SIZE];
3245 
3246         if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
3247                 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
3248                        bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS);
3249 }
3250 
3251 static sector_t get_metadata_dev_size(struct block_device *bdev)
3252 {
3253         sector_t metadata_dev_size = get_dev_size(bdev);
3254 
3255         if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
3256                 metadata_dev_size = THIN_METADATA_MAX_SECTORS;
3257 
3258         return metadata_dev_size;
3259 }
3260 
3261 static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
3262 {
3263         sector_t metadata_dev_size = get_metadata_dev_size(bdev);
3264 
3265         sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
3266 
3267         return metadata_dev_size;
3268 }
3269 
3270 /*
3271  * When a metadata threshold is crossed a dm event is triggered, and
3272  * userland should respond by growing the metadata device.  We could let
3273  * userland set the threshold, like we do with the data threshold, but I'm
3274  * not sure they know enough to do this well.
3275  */
3276 static dm_block_t calc_metadata_threshold(struct pool_c *pt)
3277 {
3278         /*
3279          * 4M is ample for all ops with the possible exception of thin
3280          * device deletion which is harmless if it fails (just retry the
3281          * delete after you've grown the device).
3282          */
3283         dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
3284         return min((dm_block_t)1024ULL /* 4M */, quarter);
3285 }
3286 
3287 /*
3288  * thin-pool <metadata dev> <data dev>
3289  *           <data block size (sectors)>
3290  *           <low water mark (blocks)>
3291  *           [<#feature args> [<arg>]*]
3292  *
3293  * Optional feature arguments are:
3294  *           skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
3295  *           ignore_discard: disable discard
3296  *           no_discard_passdown: don't pass discards down to the data device
3297  *           read_only: Don't allow any changes to be made to the pool metadata.
3298  *           error_if_no_space: error IOs, instead of queueing, if no space.
3299  */
3300 static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
3301 {
3302         int r, pool_created = 0;
3303         struct pool_c *pt;
3304         struct pool *pool;
3305         struct pool_features pf;
3306         struct dm_arg_set as;
3307         struct dm_dev *data_dev;
3308         unsigned long block_size;
3309         dm_block_t low_water_blocks;
3310         struct dm_dev *metadata_dev;
3311         fmode_t metadata_mode;
3312 
3313         /*
3314          * FIXME Remove validation from scope of lock.
3315          */
3316         mutex_lock(&dm_thin_pool_table.mutex);
3317 
3318         if (argc < 4) {
3319                 ti->error = "Invalid argument count";
3320                 r = -EINVAL;
3321                 goto out_unlock;
3322         }
3323 
3324         as.argc = argc;
3325         as.argv = argv;
3326 
3327         /* make sure metadata and data are different devices */
3328         if (!strcmp(argv[0], argv[1])) {
3329                 ti->error = "Error setting metadata or data device";
3330                 r = -EINVAL;
3331                 goto out_unlock;
3332         }
3333 
3334         /*
3335          * Set default pool features.
3336          */
3337         pool_features_init(&pf);
3338 
3339         dm_consume_args(&as, 4);
3340         r = parse_pool_features(&as, &pf, ti);
3341         if (r)
3342                 goto out_unlock;
3343 
3344         metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE);
3345         r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
3346         if (r) {
3347                 ti->error = "Error opening metadata block device";
3348                 goto out_unlock;
3349         }
3350         warn_if_metadata_device_too_big(metadata_dev->bdev);
3351 
3352         r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
3353         if (r) {
3354                 ti->error = "Error getting data device";
3355                 goto out_metadata;
3356         }
3357 
3358         if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
3359             block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
3360             block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
3361             block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
3362                 ti->error = "Invalid block size";
3363                 r = -EINVAL;
3364                 goto out;
3365         }
3366 
3367         if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
3368                 ti->error = "Invalid low water mark";
3369                 r = -EINVAL;
3370                 goto out;
3371         }
3372 
3373         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
3374         if (!pt) {
3375                 r = -ENOMEM;
3376                 goto out;
3377         }
3378 
3379         pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, data_dev->bdev,
3380                            block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
3381         if (IS_ERR(pool)) {
3382                 r = PTR_ERR(pool);
3383                 goto out_free_pt;
3384         }
3385 
3386         /*
3387          * 'pool_created' reflects whether this is the first table load.
3388          * Top level discard support is not allowed to be changed after
3389          * initial load.  This would require a pool reload to trigger thin
3390          * device changes.
3391          */
3392         if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
3393                 ti->error = "Discard support cannot be disabled once enabled";
3394                 r = -EINVAL;
3395                 goto out_flags_changed;
3396         }
3397 
3398         pt->pool = pool;
3399         pt->ti = ti;
3400         pt->metadata_dev = metadata_dev;
3401         pt->data_dev = data_dev;
3402         pt->low_water_blocks = low_water_blocks;
3403         pt->adjusted_pf = pt->requested_pf = pf;
3404         bio_init(&pt->flush_bio, NULL, 0);
3405         ti->num_flush_bios = 1;
3406 
3407         /*
3408          * Only need to enable discards if the pool should pass
3409          * them down to the data device.  The thin device's discard
3410          * processing will cause mappings to be removed from the btree.
3411          */
3412         if (pf.discard_enabled && pf.discard_passdown) {
3413                 ti->num_discard_bios = 1;
3414 
3415                 /*
3416                  * Setting 'discards_supported' circumvents the normal
3417                  * stacking of discard limits (this keeps the pool and
3418                  * thin devices' discard limits consistent).
3419                  */
3420                 ti->discards_supported = true;
3421         }
3422         ti->private = pt;
3423 
3424         r = dm_pool_register_metadata_threshold(pt->pool->pmd,
3425                                                 calc_metadata_threshold(pt),
3426                                                 metadata_low_callback,
3427                                                 pool);
3428         if (r)
3429                 goto out_flags_changed;
3430 
3431         pt->callbacks.congested_fn = pool_is_congested;
3432         dm_table_add_target_callbacks(ti->table, &pt->callbacks);
3433 
3434         mutex_unlock(&dm_thin_pool_table.mutex);
3435 
3436         return 0;
3437 
3438 out_flags_changed:
3439         __pool_dec(pool);
3440 out_free_pt:
3441         kfree(pt);
3442 out:
3443         dm_put_device(ti, data_dev);
3444 out_metadata:
3445         dm_put_device(ti, metadata_dev);
3446 out_unlock:
3447         mutex_unlock(&dm_thin_pool_table.mutex);
3448 
3449         return r;
3450 }
3451 
3452 static int pool_map(struct dm_target *ti, struct bio *bio)
3453 {
3454         int r;
3455         struct pool_c *pt = ti->private;
3456         struct pool *pool = pt->pool;
3457         unsigned long flags;
3458 
3459         /*
3460          * As this is a singleton target, ti->begin is always zero.
3461          */
3462         spin_lock_irqsave(&pool->lock, flags);
3463         bio_set_dev(bio, pt->data_dev->bdev);
3464         r = DM_MAPIO_REMAPPED;
3465         spin_unlock_irqrestore(&pool->lock, flags);
3466 
3467         return r;
3468 }
3469 
3470 static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
3471 {
3472         int r;
3473         struct pool_c *pt = ti->private;
3474         struct pool *pool = pt->pool;
3475         sector_t data_size = ti->len;
3476         dm_block_t sb_data_size;
3477 
3478         *need_commit = false;
3479 
3480         (void) sector_div(data_size, pool->sectors_per_block);
3481 
3482         r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
3483         if (r) {
3484                 DMERR("%s: failed to retrieve data device size",
3485                       dm_device_name(pool->pool_md));
3486                 return r;
3487         }
3488 
3489         if (data_size < sb_data_size) {
3490                 DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3491                       dm_device_name(pool->pool_md),
3492                       (unsigned long long)data_size, sb_data_size);
3493                 return -EINVAL;
3494 
3495         } else if (data_size > sb_data_size) {
3496                 if (dm_pool_metadata_needs_check(pool->pmd)) {
3497                         DMERR("%s: unable to grow the data device until repaired.",
3498                               dm_device_name(pool->pool_md));
3499                         return 0;
3500                 }
3501 
3502                 if (sb_data_size)
3503                         DMINFO("%s: growing the data device from %llu to %llu blocks",
3504                                dm_device_name(pool->pool_md),
3505                                sb_data_size, (unsigned long long)data_size);
3506                 r = dm_pool_resize_data_dev(pool->pmd, data_size);
3507                 if (r) {
3508                         metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
3509                         return r;
3510                 }
3511 
3512                 *need_commit = true;
3513         }
3514 
3515         return 0;
3516 }
3517 
3518 static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
3519 {
3520         int r;
3521         struct pool_c *pt = ti->private;
3522         struct pool *pool = pt->pool;
3523         dm_block_t metadata_dev_size, sb_metadata_dev_size;
3524 
3525         *need_commit = false;
3526 
3527         metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
3528 
3529         r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
3530         if (r) {
3531                 DMERR("%s: failed to retrieve metadata device size",
3532                       dm_device_name(pool->pool_md));
3533                 return r;
3534         }
3535 
3536         if (metadata_dev_size < sb_metadata_dev_size) {
3537                 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3538                       dm_device_name(pool->pool_md),
3539                       metadata_dev_size, sb_metadata_dev_size);
3540                 return -EINVAL;
3541 
3542         } else if (metadata_dev_size > sb_metadata_dev_size) {
3543                 if (dm_pool_metadata_needs_check(pool->pmd)) {
3544                         DMERR("%s: unable to grow the metadata device until repaired.",
3545                               dm_device_name(pool->pool_md));
3546                         return 0;
3547                 }
3548 
3549                 warn_if_metadata_device_too_big(pool->md_dev);
3550                 DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3551                        dm_device_name(pool->pool_md),
3552                        sb_metadata_dev_size, metadata_dev_size);
3553 
3554                 if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE)
3555                         set_pool_mode(pool, PM_WRITE);
3556 
3557                 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
3558                 if (r) {
3559                         metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
3560                         return r;
3561                 }
3562 
3563                 *need_commit = true;
3564         }
3565 
3566         return 0;
3567 }
3568 
3569 /*
3570  * Retrieves the number of blocks of the data device from
3571  * the superblock and compares it to the actual device size,
3572  * thus resizing the data device in case it has grown.
3573  *
3574  * This both copes with opening preallocated data devices in the ctr
3575  * being followed by a resume
3576  * -and-
3577  * calling the resume method individually after userspace has
3578  * grown the data device in reaction to a table event.
3579  */
3580 static int pool_preresume(struct dm_target *ti)
3581 {
3582         int r;
3583         bool need_commit1, need_commit2;
3584         struct pool_c *pt = ti->private;
3585         struct pool *pool = pt->pool;
3586 
3587         /*
3588          * Take control of the pool object.
3589          */
3590         r = bind_control_target(pool, ti);
3591         if (r)
3592                 return r;
3593 
3594         dm_pool_register_pre_commit_callback(pool->pmd,
3595                                              metadata_pre_commit_callback, pt);
3596 
3597         r = maybe_resize_data_dev(ti, &need_commit1);
3598         if (r)
3599                 return r;
3600 
3601         r = maybe_resize_metadata_dev(ti, &need_commit2);
3602         if (r)
3603                 return r;
3604 
3605         if (need_commit1 || need_commit2)
3606                 (void) commit(pool);
3607 
3608         return 0;
3609 }
3610 
3611 static void pool_suspend_active_thins(struct pool *pool)
3612 {
3613         struct thin_c *tc;
3614 
3615         /* Suspend all active thin devices */
3616         tc = get_first_thin(pool);
3617         while (tc) {
3618                 dm_internal_suspend_noflush(tc->thin_md);
3619                 tc = get_next_thin(pool, tc);
3620         }
3621 }
3622 
3623 static void pool_resume_active_thins(struct pool *pool)
3624 {
3625         struct thin_c *tc;
3626 
3627         /* Resume all active thin devices */
3628         tc = get_first_thin(pool);
3629         while (tc) {
3630                 dm_internal_resume(tc->thin_md);
3631                 tc = get_next_thin(pool, tc);
3632         }
3633 }
3634 
3635 static void pool_resume(struct dm_target *ti)
3636 {
3637         struct pool_c *pt = ti->private;
3638         struct pool *pool = pt->pool;
3639         unsigned long flags;
3640 
3641         /*
3642          * Must requeue active_thins' bios and then resume
3643          * active_thins _before_ clearing 'suspend' flag.
3644          */
3645         requeue_bios(pool);
3646         pool_resume_active_thins(pool);
3647 
3648         spin_lock_irqsave(&pool->lock, flags);
3649         pool->low_water_triggered = false;
3650         pool->suspended = false;
3651         spin_unlock_irqrestore(&pool->lock, flags);
3652 
3653         do_waker(&pool->waker.work);
3654 }
3655 
3656 static void pool_presuspend(struct dm_target *ti)
3657 {
3658         struct pool_c *pt = ti->private;
3659         struct pool *pool = pt->pool;
3660         unsigned long flags;
3661 
3662         spin_lock_irqsave(&pool->lock, flags);
3663         pool->suspended = true;
3664         spin_unlock_irqrestore(&pool->lock, flags);
3665 
3666         pool_suspend_active_thins(pool);
3667 }
3668 
3669 static void pool_presuspend_undo(struct dm_target *ti)
3670 {
3671         struct pool_c *pt = ti->private;
3672         struct pool *pool = pt->pool;
3673         unsigned long flags;
3674 
3675         pool_resume_active_thins(pool);
3676 
3677         spin_lock_irqsave(&pool->lock, flags);
3678         pool->suspended = false;
3679         spin_unlock_irqrestore(&pool->lock, flags);
3680 }
3681 
3682 static void pool_postsuspend(struct dm_target *ti)
3683 {
3684         struct pool_c *pt = ti->private;
3685         struct pool *pool = pt->pool;
3686 
3687         cancel_delayed_work_sync(&pool->waker);
3688         cancel_delayed_work_sync(&pool->no_space_timeout);
3689         flush_workqueue(pool->wq);
3690         (void) commit(pool);
3691 }
3692 
3693 static int check_arg_count(unsigned argc, unsigned args_required)
3694 {
3695         if (argc != args_required) {
3696                 DMWARN("Message received with %u arguments instead of %u.",
3697                        argc, args_required);
3698                 return -EINVAL;
3699         }
3700 
3701         return 0;
3702 }
3703 
3704 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
3705 {
3706         if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
3707             *dev_id <= MAX_DEV_ID)
3708                 return 0;
3709 
3710         if (warning)
3711                 DMWARN("Message received with invalid device id: %s", arg);
3712 
3713         return -EINVAL;
3714 }
3715 
3716 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
3717 {
3718         dm_thin_id dev_id;
3719         int r;
3720 
3721         r = check_arg_count(argc, 2);
3722         if (r)
3723                 return r;
3724 
3725         r = read_dev_id(argv[1], &dev_id, 1);
3726         if (r)
3727                 return r;
3728 
3729         r = dm_pool_create_thin(pool->pmd, dev_id);
3730         if (r) {
3731                 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3732                        argv[1]);
3733                 return r;
3734         }
3735 
3736         return 0;
3737 }
3738 
3739 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3740 {
3741         dm_thin_id dev_id;
3742         dm_thin_id origin_dev_id;
3743         int r;
3744 
3745         r = check_arg_count(argc, 3);
3746         if (r)
3747                 return r;
3748 
3749         r = read_dev_id(argv[1], &dev_id, 1);
3750         if (r)
3751                 return r;
3752 
3753         r = read_dev_id(argv[2], &origin_dev_id, 1);
3754         if (r)
3755                 return r;
3756 
3757         r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
3758         if (r) {
3759                 DMWARN("Creation of new snapshot %s of device %s failed.",
3760                        argv[1], argv[2]);
3761                 return r;
3762         }
3763 
3764         return 0;
3765 }
3766 
3767 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
3768 {
3769         dm_thin_id dev_id;
3770         int r;
3771 
3772         r = check_arg_count(argc, 2);
3773         if (r)
3774                 return r;
3775 
3776         r = read_dev_id(argv[1], &dev_id, 1);
3777         if (r)
3778                 return r;
3779 
3780         r = dm_pool_delete_thin_device(pool->pmd, dev_id);
3781         if (r)
3782                 DMWARN("Deletion of thin device %s failed.", argv[1]);
3783 
3784         return r;
3785 }
3786 
3787 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
3788 {
3789         dm_thin_id old_id, new_id;
3790         int r;
3791 
3792         r = check_arg_count(argc, 3);
3793         if (r)
3794                 return r;
3795 
3796         if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
3797                 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
3798                 return -EINVAL;
3799         }
3800 
3801         if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
3802                 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
3803                 return -EINVAL;
3804         }
3805 
3806         r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
3807         if (r) {
3808                 DMWARN("Failed to change transaction id from %s to %s.",
3809                        argv[1], argv[2]);
3810                 return r;
3811         }
3812 
3813         return 0;
3814 }
3815 
3816 static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3817 {
3818         int r;
3819 
3820         r = check_arg_count(argc, 1);
3821         if (r)
3822                 return r;
3823 
3824         (void) commit(pool);
3825 
3826         r = dm_pool_reserve_metadata_snap(pool->pmd);
3827         if (r)
3828                 DMWARN("reserve_metadata_snap message failed.");
3829 
3830         return r;
3831 }
3832 
3833 static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3834 {
3835         int r;
3836 
3837         r = check_arg_count(argc, 1);
3838         if (r)
3839                 return r;
3840 
3841         r = dm_pool_release_metadata_snap(pool->pmd);
3842         if (r)
3843                 DMWARN("release_metadata_snap message failed.");
3844 
3845         return r;
3846 }
3847 
3848 /*
3849  * Messages supported:
3850  *   create_thin        <dev_id>
3851  *   create_snap        <dev_id> <origin_id>
3852  *   delete             <dev_id>
3853  *   set_transaction_id <current_trans_id> <new_trans_id>
3854  *   reserve_metadata_snap
3855  *   release_metadata_snap
3856  */
3857 static int pool_message(struct dm_target *ti, unsigned argc, char **argv,
3858                         char *result, unsigned maxlen)
3859 {
3860         int r = -EINVAL;
3861         struct pool_c *pt = ti->private;
3862         struct pool *pool = pt->pool;
3863 
3864         if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) {
3865                 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3866                       dm_device_name(pool->pool_md));
3867                 return -EOPNOTSUPP;
3868         }
3869 
3870         if (!strcasecmp(argv[0], "create_thin"))
3871                 r = process_create_thin_mesg(argc, argv, pool);
3872 
3873         else if (!strcasecmp(argv[0], "create_snap"))
3874                 r = process_create_snap_mesg(argc, argv, pool);
3875 
3876         else if (!strcasecmp(argv[0], "delete"))
3877                 r = process_delete_mesg(argc, argv, pool);
3878 
3879         else if (!strcasecmp(argv[0], "set_transaction_id"))
3880                 r = process_set_transaction_id_mesg(argc, argv, pool);
3881 
3882         else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
3883                 r = process_reserve_metadata_snap_mesg(argc, argv, pool);
3884 
3885         else if (!strcasecmp(argv[0], "release_metadata_snap"))
3886                 r = process_release_metadata_snap_mesg(argc, argv, pool);
3887 
3888         else
3889                 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
3890 
3891         if (!r)
3892                 (void) commit(pool);
3893 
3894         return r;
3895 }
3896 
3897 static void emit_flags(struct pool_features *pf, char *result,
3898                        unsigned sz, unsigned maxlen)
3899 {
3900         unsigned count = !pf->zero_new_blocks + !pf->discard_enabled +
3901                 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
3902                 pf->error_if_no_space;
3903         DMEMIT("%u ", count);
3904 
3905         if (!pf->zero_new_blocks)
3906                 DMEMIT("skip_block_zeroing ");
3907 
3908         if (!pf->discard_enabled)
3909                 DMEMIT("ignore_discard ");
3910 
3911         if (!pf->discard_passdown)
3912                 DMEMIT("no_discard_passdown ");
3913 
3914         if (pf->mode == PM_READ_ONLY)
3915                 DMEMIT("read_only ");
3916 
3917         if (pf->error_if_no_space)
3918                 DMEMIT("error_if_no_space ");
3919 }
3920 
3921 /*
3922  * Status line is:
3923  *    <transaction id> <used metadata sectors>/<total metadata sectors>
3924  *    <used data sectors>/<total data sectors> <held metadata root>
3925  *    <pool mode> <discard config> <no space config> <needs_check>
3926  */
3927 static void pool_status(struct dm_target *ti, status_type_t type,
3928                         unsigned status_flags, char *result, unsigned maxlen)
3929 {
3930         int r;
3931         unsigned sz = 0;
3932         uint64_t transaction_id;
3933         dm_block_t nr_free_blocks_data;
3934         dm_block_t nr_free_blocks_metadata;
3935         dm_block_t nr_blocks_data;
3936         dm_block_t nr_blocks_metadata;
3937         dm_block_t held_root;
3938         enum pool_mode mode;
3939         char buf[BDEVNAME_SIZE];
3940         char buf2[BDEVNAME_SIZE];
3941         struct pool_c *pt = ti->private;
3942         struct pool *pool = pt->pool;
3943 
3944         switch (type) {
3945         case STATUSTYPE_INFO:
3946                 if (get_pool_mode(pool) == PM_FAIL) {
3947                         DMEMIT("Fail");
3948                         break;
3949                 }
3950 
3951                 /* Commit to ensure statistics aren't out-of-date */
3952                 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3953                         (void) commit(pool);
3954 
3955                 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
3956                 if (r) {
3957                         DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3958                               dm_device_name(pool->pool_md), r);
3959                         goto err;
3960                 }
3961 
3962                 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
3963                 if (r) {
3964                         DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3965                               dm_device_name(pool->pool_md), r);
3966                         goto err;
3967                 }
3968 
3969                 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
3970                 if (r) {
3971                         DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3972                               dm_device_name(pool->pool_md), r);
3973                         goto err;
3974                 }
3975 
3976                 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
3977                 if (r) {
3978                         DMERR("%s: dm_pool_get_free_block_count returned %d",
3979                               dm_device_name(pool->pool_md), r);
3980                         goto err;
3981                 }
3982 
3983                 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
3984                 if (r) {
3985                         DMERR("%s: dm_pool_get_data_dev_size returned %d",
3986                               dm_device_name(pool->pool_md), r);
3987                         goto err;
3988                 }
3989 
3990                 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
3991                 if (r) {
3992                         DMERR("%s: dm_pool_get_metadata_snap returned %d",
3993                               dm_device_name(pool->pool_md), r);
3994                         goto err;
3995                 }
3996 
3997                 DMEMIT("%llu %llu/%llu %llu/%llu ",
3998                        (unsigned long long)transaction_id,
3999                        (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
4000                        (unsigned long long)nr_blocks_metadata,
4001                        (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
4002                        (unsigned long long)nr_blocks_data);
4003 
4004                 if (held_root)
4005                         DMEMIT("%llu ", held_root);
4006                 else
4007                         DMEMIT("- ");
4008 
4009                 mode = get_pool_mode(pool);
4010                 if (mode == PM_OUT_OF_DATA_SPACE)
4011                         DMEMIT("out_of_data_space ");
4012                 else if (is_read_only_pool_mode(mode))
4013                         DMEMIT("ro ");
4014                 else
4015                         DMEMIT("rw ");
4016 
4017                 if (!pool->pf.discard_enabled)
4018                         DMEMIT("ignore_discard ");
4019                 else if (pool->pf.discard_passdown)
4020                         DMEMIT("discard_passdown ");
4021                 else
4022                         DMEMIT("no_discard_passdown ");
4023 
4024                 if (pool->pf.error_if_no_space)
4025                         DMEMIT("error_if_no_space ");
4026                 else
4027                         DMEMIT("queue_if_no_space ");
4028 
4029                 if (dm_pool_metadata_needs_check(pool->pmd))
4030                         DMEMIT("needs_check ");
4031                 else
4032                         DMEMIT("- ");
4033 
4034                 DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt));
4035 
4036                 break;
4037 
4038         case STATUSTYPE_TABLE:
4039                 DMEMIT("%s %s %lu %llu ",
4040                        format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
4041                        format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
4042                        (unsigned long)pool->sectors_per_block,
4043                        (unsigned long long)pt->low_water_blocks);
4044                 emit_flags(&pt->requested_pf, result, sz, maxlen);
4045                 break;
4046         }
4047         return;
4048 
4049 err:
4050         DMEMIT("Error");
4051 }
4052 
4053 static int pool_iterate_devices(struct dm_target *ti,
4054                                 iterate_devices_callout_fn fn, void *data)
4055 {
4056         struct pool_c *pt = ti->private;
4057 
4058         return fn(ti, pt->data_dev, 0, ti->len, data);
4059 }
4060 
4061 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
4062 {
4063         struct pool_c *pt = ti->private;
4064         struct pool *pool = pt->pool;
4065         sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
4066 
4067         /*
4068          * If max_sectors is smaller than pool->sectors_per_block adjust it
4069          * to the highest possible power-of-2 factor of pool->sectors_per_block.
4070          * This is especially beneficial when the pool's data device is a RAID
4071          * device that has a full stripe width that matches pool->sectors_per_block
4072          * -- because even though partial RAID stripe-sized IOs will be issued to a
4073          *    single RAID stripe; when aggregated they will end on a full RAID stripe
4074          *    boundary.. which avoids additional partial RAID stripe writes cascading
4075          */
4076         if (limits->max_sectors < pool->sectors_per_block) {
4077                 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
4078                         if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
4079                                 limits->max_sectors--;
4080                         limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
4081                 }
4082         }
4083 
4084         /*
4085          * If the system-determined stacked limits are compatible with the
4086          * pool's blocksize (io_opt is a factor) do not override them.
4087          */
4088         if (io_opt_sectors < pool->sectors_per_block ||
4089             !is_factor(io_opt_sectors, pool->sectors_per_block)) {
4090                 if (is_factor(pool->sectors_per_block, limits->max_sectors))
4091                         blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT);
4092                 else
4093                         blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
4094                 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
4095         }
4096 
4097         /*
4098          * pt->adjusted_pf is a staging area for the actual features to use.
4099          * They get transferred to the live pool in bind_control_target()
4100          * called from pool_preresume().
4101          */
4102         if (!pt->adjusted_pf.discard_enabled) {
4103                 /*
4104                  * Must explicitly disallow stacking discard limits otherwise the
4105                  * block layer will stack them if pool's data device has support.
4106                  * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
4107                  * user to see that, so make sure to set all discard limits to 0.
4108                  */
4109                 limits->discard_granularity = 0;
4110                 return;
4111         }
4112 
4113         disable_passdown_if_not_supported(pt);
4114 
4115         /*
4116          * The pool uses the same discard limits as the underlying data
4117          * device.  DM core has already set this up.
4118          */
4119 }
4120 
4121 static struct target_type pool_target = {
4122         .name = "thin-pool",
4123         .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
4124                     DM_TARGET_IMMUTABLE,
4125         .version = {1, 22, 0},
4126         .module = THIS_MODULE,
4127         .ctr = pool_ctr,
4128         .dtr = pool_dtr,
4129         .map = pool_map,
4130         .presuspend = pool_presuspend,
4131         .presuspend_undo = pool_presuspend_undo,
4132         .postsuspend = pool_postsuspend,
4133         .preresume = pool_preresume,
4134         .resume = pool_resume,
4135         .message = pool_message,
4136         .status = pool_status,
4137         .iterate_devices = pool_iterate_devices,
4138         .io_hints = pool_io_hints,
4139 };
4140 
4141 /*----------------------------------------------------------------
4142  * Thin target methods
4143  *--------------------------------------------------------------*/
4144 static void thin_get(struct thin_c *tc)
4145 {
4146         refcount_inc(&tc->refcount);
4147 }
4148 
4149 static void thin_put(struct thin_c *tc)
4150 {
4151         if (refcount_dec_and_test(&tc->refcount))
4152                 complete(&tc->can_destroy);
4153 }
4154 
4155 static void thin_dtr(struct dm_target *ti)
4156 {
4157         struct thin_c *tc = ti->private;
4158         unsigned long flags;
4159 
4160         spin_lock_irqsave(&tc->pool->lock, flags);
4161         list_del_rcu(&tc->list);
4162         spin_unlock_irqrestore(&tc->pool->lock, flags);
4163         synchronize_rcu();
4164 
4165         thin_put(tc);
4166         wait_for_completion(&tc->can_destroy);
4167 
4168         mutex_lock(&dm_thin_pool_table.mutex);
4169 
4170         __pool_dec(tc->pool);
4171         dm_pool_close_thin_device(tc->td);
4172         dm_put_device(ti, tc->pool_dev);
4173         if (tc->origin_dev)
4174                 dm_put_device(ti, tc->origin_dev);
4175         kfree(tc);
4176 
4177         mutex_unlock(&dm_thin_pool_table.mutex);
4178 }
4179 
4180 /*
4181  * Thin target parameters:
4182  *
4183  * <pool_dev> <dev_id> [origin_dev]
4184  *
4185  * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
4186  * dev_id: the internal device identifier
4187  * origin_dev: a device external to the pool that should act as the origin
4188  *
4189  * If the pool device has discards disabled, they get disabled for the thin
4190  * device as well.
4191  */
4192 static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
4193 {
4194         int r;
4195         struct thin_c *tc;
4196         struct dm_dev *pool_dev, *origin_dev;
4197         struct mapped_device *pool_md;
4198         unsigned long flags;
4199 
4200         mutex_lock(&dm_thin_pool_table.mutex);
4201 
4202         if (argc != 2 && argc != 3) {
4203                 ti->error = "Invalid argument count";
4204                 r = -EINVAL;
4205                 goto out_unlock;
4206         }
4207 
4208         tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
4209         if (!tc) {
4210                 ti->error = "Out of memory";
4211                 r = -ENOMEM;
4212                 goto out_unlock;
4213         }
4214         tc->thin_md = dm_table_get_md(ti->table);
4215         spin_lock_init(&tc->lock);
4216         INIT_LIST_HEAD(&tc->deferred_cells);
4217         bio_list_init(&tc->deferred_bio_list);
4218         bio_list_init(&tc->retry_on_resume_list);
4219         tc->sort_bio_list = RB_ROOT;
4220 
4221         if (argc == 3) {
4222                 if (!strcmp(argv[0], argv[2])) {
4223                         ti->error = "Error setting origin device";
4224                         r = -EINVAL;
4225                         goto bad_origin_dev;
4226                 }
4227 
4228                 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
4229                 if (r) {
4230                         ti->error = "Error opening origin device";
4231                         goto bad_origin_dev;
4232                 }
4233                 tc->origin_dev = origin_dev;
4234         }
4235 
4236         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
4237         if (r) {
4238                 ti->error = "Error opening pool device";
4239                 goto bad_pool_dev;
4240         }
4241         tc->pool_dev = pool_dev;
4242 
4243         if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
4244                 ti->error = "Invalid device id";
4245                 r = -EINVAL;
4246                 goto bad_common;
4247         }
4248 
4249         pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
4250         if (!pool_md) {
4251                 ti->error = "Couldn't get pool mapped device";
4252                 r = -EINVAL;
4253                 goto bad_common;
4254         }
4255 
4256         tc->pool = __pool_table_lookup(pool_md);
4257         if (!tc->pool) {
4258                 ti->error = "Couldn't find pool object";
4259                 r = -EINVAL;
4260                 goto bad_pool_lookup;
4261         }
4262         __pool_inc(tc->pool);
4263 
4264         if (get_pool_mode(tc->pool) == PM_FAIL) {
4265                 ti->error = "Couldn't open thin device, Pool is in fail mode";
4266                 r = -EINVAL;
4267                 goto bad_pool;
4268         }
4269 
4270         r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
4271         if (r) {
4272                 ti->error = "Couldn't open thin internal device";
4273                 goto bad_pool;
4274         }
4275 
4276         r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
4277         if (r)
4278                 goto bad;
4279 
4280         ti->num_flush_bios = 1;
4281         ti->flush_supported = true;
4282         ti->per_io_data_size = sizeof(struct dm_thin_endio_hook);
4283 
4284         /* In case the pool supports discards, pass them on. */
4285         if (tc->pool->pf.discard_enabled) {
4286                 ti->discards_supported = true;
4287                 ti->num_discard_bios = 1;
4288         }
4289 
4290         mutex_unlock(&dm_thin_pool_table.mutex);
4291 
4292         spin_lock_irqsave(&tc->pool->lock, flags);
4293         if (tc->pool->suspended) {
4294                 spin_unlock_irqrestore(&tc->pool->lock, flags);
4295                 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
4296                 ti->error = "Unable to activate thin device while pool is suspended";
4297                 r = -EINVAL;
4298                 goto bad;
4299         }
4300         refcount_set(&tc->refcount, 1);
4301         init_completion(&tc->can_destroy);
4302         list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
4303         spin_unlock_irqrestore(&tc->pool->lock, flags);
4304         /*
4305          * This synchronize_rcu() call is needed here otherwise we risk a
4306          * wake_worker() call finding no bios to process (because the newly
4307          * added tc isn't yet visible).  So this reduces latency since we
4308          * aren't then dependent on the periodic commit to wake_worker().
4309          */
4310         synchronize_rcu();
4311 
4312         dm_put(pool_md);
4313 
4314         return 0;
4315 
4316 bad:
4317         dm_pool_close_thin_device(tc->td);
4318 bad_pool:
4319         __pool_dec(tc->pool);
4320 bad_pool_lookup:
4321         dm_put(pool_md);
4322 bad_common:
4323         dm_put_device(ti, tc->pool_dev);
4324 bad_pool_dev:
4325         if (tc->origin_dev)
4326                 dm_put_device(ti, tc->origin_dev);
4327 bad_origin_dev:
4328         kfree(tc);
4329 out_unlock:
4330         mutex_unlock(&dm_thin_pool_table.mutex);
4331 
4332         return r;
4333 }
4334 
4335 static int thin_map(struct dm_target *ti, struct bio *bio)
4336 {
4337         bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
4338 
4339         return thin_bio_map(ti, bio);
4340 }
4341 
4342 static int thin_endio(struct dm_target *ti, struct bio *bio,
4343                 blk_status_t *err)
4344 {
4345         unsigned long flags;
4346         struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
4347         struct list_head work;
4348         struct dm_thin_new_mapping *m, *tmp;
4349         struct pool *pool = h->tc->pool;
4350 
4351         if (h->shared_read_entry) {
4352                 INIT_LIST_HEAD(&work);
4353                 dm_deferred_entry_dec(h->shared_read_entry, &work);
4354 
4355                 spin_lock_irqsave(&pool->lock, flags);
4356                 list_for_each_entry_safe(m, tmp, &work, list) {
4357                         list_del(&m->list);
4358                         __complete_mapping_preparation(m);
4359                 }
4360                 spin_unlock_irqrestore(&pool->lock, flags);
4361         }
4362 
4363         if (h->all_io_entry) {
4364                 INIT_LIST_HEAD(&work);
4365                 dm_deferred_entry_dec(h->all_io_entry, &work);
4366                 if (!list_empty(&work)) {
4367                         spin_lock_irqsave(&pool->lock, flags);
4368                         list_for_each_entry_safe(m, tmp, &work, list)
4369                                 list_add_tail(&m->list, &pool->prepared_discards);
4370                         spin_unlock_irqrestore(&pool->lock, flags);
4371                         wake_worker(pool);
4372                 }
4373         }
4374 
4375         if (h->cell)
4376                 cell_defer_no_holder(h->tc, h->cell);
4377 
4378         return DM_ENDIO_DONE;
4379 }
4380 
4381 static void thin_presuspend(struct dm_target *ti)
4382 {
4383         struct thin_c *tc = ti->private;
4384 
4385         if (dm_noflush_suspending(ti))
4386                 noflush_work(tc, do_noflush_start);
4387 }
4388 
4389 static void thin_postsuspend(struct dm_target *ti)
4390 {
4391         struct thin_c *tc = ti->private;
4392 
4393         /*
4394          * The dm_noflush_suspending flag has been cleared by now, so
4395          * unfortunately we must always run this.
4396          */
4397         noflush_work(tc, do_noflush_stop);
4398 }
4399 
4400 static int thin_preresume(struct dm_target *ti)
4401 {
4402         struct thin_c *tc = ti->private;
4403 
4404         if (tc->origin_dev)
4405                 tc->origin_size = get_dev_size(tc->origin_dev->bdev);
4406 
4407         return 0;
4408 }
4409 
4410 /*
4411  * <nr mapped sectors> <highest mapped sector>
4412  */
4413 static void thin_status(struct dm_target *ti, status_type_t type,
4414                         unsigned status_flags, char *result, unsigned maxlen)
4415 {
4416         int r;
4417         ssize_t sz = 0;
4418         dm_block_t mapped, highest;
4419         char buf[BDEVNAME_SIZE];
4420         struct thin_c *tc = ti->private;
4421 
4422         if (get_pool_mode(tc->pool) == PM_FAIL) {
4423                 DMEMIT("Fail");
4424                 return;
4425         }
4426 
4427         if (!tc->td)
4428                 DMEMIT("-");
4429         else {
4430                 switch (type) {
4431                 case STATUSTYPE_INFO:
4432                         r = dm_thin_get_mapped_count(tc->td, &mapped);
4433                         if (r) {
4434                                 DMERR("dm_thin_get_mapped_count returned %d", r);
4435                                 goto err;
4436                         }
4437 
4438                         r = dm_thin_get_highest_mapped_block(tc->td, &highest);
4439                         if (r < 0) {
4440                                 DMERR("dm_thin_get_highest_mapped_block returned %d", r);
4441                                 goto err;
4442                         }
4443 
4444                         DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
4445                         if (r)
4446                                 DMEMIT("%llu", ((highest + 1) *
4447                                                 tc->pool->sectors_per_block) - 1);
4448                         else
4449                                 DMEMIT("-");
4450                         break;
4451 
4452                 case STATUSTYPE_TABLE:
4453                         DMEMIT("%s %lu",
4454                                format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
4455                                (unsigned long) tc->dev_id);
4456                         if (tc->origin_dev)
4457                                 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
4458                         break;
4459                 }
4460         }
4461 
4462         return;
4463 
4464 err:
4465         DMEMIT("Error");
4466 }
4467 
4468 static int thin_iterate_devices(struct dm_target *ti,
4469                                 iterate_devices_callout_fn fn, void *data)
4470 {
4471         sector_t blocks;
4472         struct thin_c *tc = ti->private;
4473         struct pool *pool = tc->pool;
4474 
4475         /*
4476          * We can't call dm_pool_get_data_dev_size() since that blocks.  So
4477          * we follow a more convoluted path through to the pool's target.
4478          */
4479         if (!pool->ti)
4480                 return 0;       /* nothing is bound */
4481 
4482         blocks = pool->ti->len;
4483         (void) sector_div(blocks, pool->sectors_per_block);
4484         if (blocks)
4485                 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
4486 
4487         return 0;
4488 }
4489 
4490 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
4491 {
4492         struct thin_c *tc = ti->private;
4493         struct pool *pool = tc->pool;
4494 
4495         if (!pool->pf.discard_enabled)
4496                 return;
4497 
4498         limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
4499         limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
4500 }
4501 
4502 static struct target_type thin_target = {
4503         .name = "thin",
4504         .version = {1, 22, 0},
4505         .module = THIS_MODULE,
4506         .ctr = thin_ctr,
4507         .dtr = thin_dtr,
4508         .map = thin_map,
4509         .end_io = thin_endio,
4510         .preresume = thin_preresume,
4511         .presuspend = thin_presuspend,
4512         .postsuspend = thin_postsuspend,
4513         .status = thin_status,
4514         .iterate_devices = thin_iterate_devices,
4515         .io_hints = thin_io_hints,
4516 };
4517 
4518 /*----------------------------------------------------------------*/
4519 
4520 static int __init dm_thin_init(void)
4521 {
4522         int r = -ENOMEM;
4523 
4524         pool_table_init();
4525 
4526         _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
4527         if (!_new_mapping_cache)
4528                 return r;
4529 
4530         r = dm_register_target(&thin_target);
4531         if (r)
4532                 goto bad_new_mapping_cache;
4533 
4534         r = dm_register_target(&pool_target);
4535         if (r)
4536                 goto bad_thin_target;
4537 
4538         return 0;
4539 
4540 bad_thin_target:
4541         dm_unregister_target(&thin_target);
4542 bad_new_mapping_cache:
4543         kmem_cache_destroy(_new_mapping_cache);
4544 
4545         return r;
4546 }
4547 
4548 static void dm_thin_exit(void)
4549 {
4550         dm_unregister_target(&thin_target);
4551         dm_unregister_target(&pool_target);
4552 
4553         kmem_cache_destroy(_new_mapping_cache);
4554 
4555         pool_table_exit();
4556 }
4557 
4558 module_init(dm_thin_init);
4559 module_exit(dm_thin_exit);
4560 
4561 module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
4562 MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
4563 
4564 MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
4565 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
4566 MODULE_LICENSE("GPL");
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/md/dm-thin.c

DEFINITIONS
