Lines Matching refs:s
151 struct dm_dev *dm_snap_origin(struct dm_snapshot *s) in dm_snap_origin() argument
153 return s->origin; in dm_snap_origin()
157 struct dm_dev *dm_snap_cow(struct dm_snapshot *s) in dm_snap_cow() argument
159 return s->cow; in dm_snap_cow()
237 static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk) in track_chunk() argument
243 spin_lock_irq(&s->tracked_chunk_lock); in track_chunk()
245 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); in track_chunk()
246 spin_unlock_irq(&s->tracked_chunk_lock); in track_chunk()
249 static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio) in stop_tracking_chunk() argument
254 spin_lock_irqsave(&s->tracked_chunk_lock, flags); in stop_tracking_chunk()
256 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); in stop_tracking_chunk()
259 static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk) in __chunk_is_tracked() argument
264 spin_lock_irq(&s->tracked_chunk_lock); in __chunk_is_tracked()
267 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { in __chunk_is_tracked()
274 spin_unlock_irq(&s->tracked_chunk_lock); in __chunk_is_tracked()
283 static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk) in __check_for_conflicting_io() argument
285 while (__chunk_is_tracked(s, chunk)) in __check_for_conflicting_io()
428 struct dm_snapshot *s; in __find_snapshots_sharing_cow() local
437 list_for_each_entry(s, &o->snapshots, list) { in __find_snapshots_sharing_cow()
438 if (dm_target_is_snapshot_merge(s->ti) && snap_merge) in __find_snapshots_sharing_cow()
439 *snap_merge = s; in __find_snapshots_sharing_cow()
440 if (!bdev_equal(s->cow->bdev, snap->cow->bdev)) in __find_snapshots_sharing_cow()
443 down_read(&s->lock); in __find_snapshots_sharing_cow()
444 active = s->active; in __find_snapshots_sharing_cow()
445 up_read(&s->lock); in __find_snapshots_sharing_cow()
449 *snap_src = s; in __find_snapshots_sharing_cow()
451 *snap_dest = s; in __find_snapshots_sharing_cow()
509 static void __insert_snapshot(struct origin *o, struct dm_snapshot *s) in __insert_snapshot() argument
515 if (l->store->chunk_size < s->store->chunk_size) in __insert_snapshot()
517 list_add_tail(&s->list, &l->list); in __insert_snapshot()
571 static void reregister_snapshot(struct dm_snapshot *s) in reregister_snapshot() argument
573 struct block_device *bdev = s->origin->bdev; in reregister_snapshot()
577 list_del(&s->list); in reregister_snapshot()
578 __insert_snapshot(__lookup_origin(bdev), s); in reregister_snapshot()
583 static void unregister_snapshot(struct dm_snapshot *s) in unregister_snapshot() argument
588 o = __lookup_origin(s->origin->bdev); in unregister_snapshot()
590 list_del(&s->list); in unregister_snapshot()
684 static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s) in alloc_pending_exception() argument
686 struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool, in alloc_pending_exception()
689 atomic_inc(&s->pending_exceptions_count); in alloc_pending_exception()
690 pe->snap = s; in alloc_pending_exception()
697 struct dm_snapshot *s = pe->snap; in free_pending_exception() local
699 mempool_free(pe, s->pending_pool); in free_pending_exception()
701 atomic_dec(&s->pending_exceptions_count); in free_pending_exception()
752 struct dm_snapshot *s = context; in dm_add_exception() local
764 dm_insert_exception(&s->complete, e); in dm_add_exception()
801 static int init_hash_tables(struct dm_snapshot *s) in init_hash_tables() argument
809 cow_dev_size = get_dev_size(s->cow->bdev); in init_hash_tables()
812 hash_size = cow_dev_size >> s->store->chunk_shift; in init_hash_tables()
818 if (dm_exception_table_init(&s->complete, hash_size, in init_hash_tables()
830 if (dm_exception_table_init(&s->pending, hash_size, 0)) { in init_hash_tables()
831 dm_exception_table_exit(&s->complete, exception_cache); in init_hash_tables()
838 static void merge_shutdown(struct dm_snapshot *s) in merge_shutdown() argument
840 clear_bit_unlock(RUNNING_MERGE, &s->state_bits); in merge_shutdown()
842 wake_up_bit(&s->state_bits, RUNNING_MERGE); in merge_shutdown()
845 static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s) in __release_queued_bios_after_merge() argument
847 s->first_merging_chunk = 0; in __release_queued_bios_after_merge()
848 s->num_merging_chunks = 0; in __release_queued_bios_after_merge()
850 return bio_list_get(&s->bios_queued_during_merge); in __release_queued_bios_after_merge()
856 static int __remove_single_exception_chunk(struct dm_snapshot *s, in __remove_single_exception_chunk() argument
861 e = dm_lookup_exception(&s->complete, old_chunk); in __remove_single_exception_chunk()
907 static int remove_single_exception_chunk(struct dm_snapshot *s) in remove_single_exception_chunk() argument
911 chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1; in remove_single_exception_chunk()
913 down_write(&s->lock); in remove_single_exception_chunk()
920 r = __remove_single_exception_chunk(s, old_chunk); in remove_single_exception_chunk()
923 } while (old_chunk-- > s->first_merging_chunk); in remove_single_exception_chunk()
925 b = __release_queued_bios_after_merge(s); in remove_single_exception_chunk()
928 up_write(&s->lock); in remove_single_exception_chunk()
961 static void snapshot_merge_next_chunks(struct dm_snapshot *s) in snapshot_merge_next_chunks() argument
969 BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits)); in snapshot_merge_next_chunks()
970 if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits))) in snapshot_merge_next_chunks()
976 if (!s->valid) { in snapshot_merge_next_chunks()
981 linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk, in snapshot_merge_next_chunks()
987 down_write(&s->lock); in snapshot_merge_next_chunks()
988 s->merge_failed = 1; in snapshot_merge_next_chunks()
989 up_write(&s->lock); in snapshot_merge_next_chunks()
1002 io_size = linear_chunks * s->store->chunk_size; in snapshot_merge_next_chunks()
1004 dest.bdev = s->origin->bdev; in snapshot_merge_next_chunks()
1005 dest.sector = chunk_to_sector(s->store, old_chunk); in snapshot_merge_next_chunks()
1008 src.bdev = s->cow->bdev; in snapshot_merge_next_chunks()
1009 src.sector = chunk_to_sector(s->store, new_chunk); in snapshot_merge_next_chunks()
1022 while (origin_write_extent(s, dest.sector, io_size)) { in snapshot_merge_next_chunks()
1030 down_write(&s->lock); in snapshot_merge_next_chunks()
1031 s->first_merging_chunk = old_chunk; in snapshot_merge_next_chunks()
1032 s->num_merging_chunks = linear_chunks; in snapshot_merge_next_chunks()
1033 up_write(&s->lock); in snapshot_merge_next_chunks()
1037 __check_for_conflicting_io(s, old_chunk + i); in snapshot_merge_next_chunks()
1039 dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); in snapshot_merge_next_chunks()
1043 merge_shutdown(s); in snapshot_merge_next_chunks()
1050 struct dm_snapshot *s = context; in merge_callback() local
1061 if (s->store->type->commit_merge(s->store, in merge_callback()
1062 s->num_merging_chunks) < 0) { in merge_callback()
1067 if (remove_single_exception_chunk(s) < 0) in merge_callback()
1070 snapshot_merge_next_chunks(s); in merge_callback()
1075 down_write(&s->lock); in merge_callback()
1076 s->merge_failed = 1; in merge_callback()
1077 b = __release_queued_bios_after_merge(s); in merge_callback()
1078 up_write(&s->lock); in merge_callback()
1081 merge_shutdown(s); in merge_callback()
1084 static void start_merge(struct dm_snapshot *s) in start_merge() argument
1086 if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits)) in start_merge()
1087 snapshot_merge_next_chunks(s); in start_merge()
1093 static void stop_merge(struct dm_snapshot *s) in stop_merge() argument
1095 set_bit(SHUTDOWN_MERGE, &s->state_bits); in stop_merge()
1096 wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE); in stop_merge()
1097 clear_bit(SHUTDOWN_MERGE, &s->state_bits); in stop_merge()
1105 struct dm_snapshot *s; in snapshot_ctr() local
1124 s = kmalloc(sizeof(*s), GFP_KERNEL); in snapshot_ctr()
1125 if (!s) { in snapshot_ctr()
1135 r = dm_get_device(ti, origin_path, origin_mode, &s->origin); in snapshot_ctr()
1140 origin_dev = s->origin->bdev->bd_dev; in snapshot_ctr()
1153 r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow); in snapshot_ctr()
1159 r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store); in snapshot_ctr()
1169 s->ti = ti; in snapshot_ctr()
1170 s->valid = 1; in snapshot_ctr()
1171 s->snapshot_overflowed = 0; in snapshot_ctr()
1172 s->active = 0; in snapshot_ctr()
1173 atomic_set(&s->pending_exceptions_count, 0); in snapshot_ctr()
1174 s->exception_start_sequence = 0; in snapshot_ctr()
1175 s->exception_complete_sequence = 0; in snapshot_ctr()
1176 INIT_LIST_HEAD(&s->out_of_order_list); in snapshot_ctr()
1177 init_rwsem(&s->lock); in snapshot_ctr()
1178 INIT_LIST_HEAD(&s->list); in snapshot_ctr()
1179 spin_lock_init(&s->pe_lock); in snapshot_ctr()
1180 s->state_bits = 0; in snapshot_ctr()
1181 s->merge_failed = 0; in snapshot_ctr()
1182 s->first_merging_chunk = 0; in snapshot_ctr()
1183 s->num_merging_chunks = 0; in snapshot_ctr()
1184 bio_list_init(&s->bios_queued_during_merge); in snapshot_ctr()
1187 if (init_hash_tables(s)) { in snapshot_ctr()
1193 s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); in snapshot_ctr()
1194 if (IS_ERR(s->kcopyd_client)) { in snapshot_ctr()
1195 r = PTR_ERR(s->kcopyd_client); in snapshot_ctr()
1200 s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache); in snapshot_ctr()
1201 if (!s->pending_pool) { in snapshot_ctr()
1208 INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]); in snapshot_ctr()
1210 spin_lock_init(&s->tracked_chunk_lock); in snapshot_ctr()
1212 ti->private = s; in snapshot_ctr()
1218 r = register_snapshot(s); in snapshot_ctr()
1234 s->store->chunk_size = 0; in snapshot_ctr()
1238 r = s->store->type->read_metadata(s->store, dm_add_exception, in snapshot_ctr()
1239 (void *)s); in snapshot_ctr()
1244 s->valid = 0; in snapshot_ctr()
1248 if (!s->store->chunk_size) { in snapshot_ctr()
1253 r = dm_set_target_max_io_len(ti, s->store->chunk_size); in snapshot_ctr()
1260 unregister_snapshot(s); in snapshot_ctr()
1263 mempool_destroy(s->pending_pool); in snapshot_ctr()
1266 dm_kcopyd_client_destroy(s->kcopyd_client); in snapshot_ctr()
1269 dm_exception_table_exit(&s->pending, pending_cache); in snapshot_ctr()
1270 dm_exception_table_exit(&s->complete, exception_cache); in snapshot_ctr()
1273 dm_exception_store_destroy(s->store); in snapshot_ctr()
1276 dm_put_device(ti, s->cow); in snapshot_ctr()
1279 dm_put_device(ti, s->origin); in snapshot_ctr()
1282 kfree(s); in snapshot_ctr()
1288 static void __free_exceptions(struct dm_snapshot *s) in __free_exceptions() argument
1290 dm_kcopyd_client_destroy(s->kcopyd_client); in __free_exceptions()
1291 s->kcopyd_client = NULL; in __free_exceptions()
1293 dm_exception_table_exit(&s->pending, pending_cache); in __free_exceptions()
1294 dm_exception_table_exit(&s->complete, exception_cache); in __free_exceptions()
1335 struct dm_snapshot *s = ti->private; in snapshot_dtr() local
1340 (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); in snapshot_dtr()
1341 if (snap_src && snap_dest && (s == snap_src)) { in snapshot_dtr()
1350 stop_merge(s); in snapshot_dtr()
1354 unregister_snapshot(s); in snapshot_dtr()
1356 while (atomic_read(&s->pending_exceptions_count)) in snapshot_dtr()
1366 BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i])); in snapshot_dtr()
1369 __free_exceptions(s); in snapshot_dtr()
1371 mempool_destroy(s->pending_pool); in snapshot_dtr()
1373 dm_exception_store_destroy(s->store); in snapshot_dtr()
1375 dm_put_device(ti, s->cow); in snapshot_dtr()
1377 dm_put_device(ti, s->origin); in snapshot_dtr()
1379 kfree(s); in snapshot_dtr()
1402 static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio) in retry_origin_bios() argument
1410 r = do_origin(s->origin, bio); in retry_origin_bios()
1432 static void __invalidate_snapshot(struct dm_snapshot *s, int err) in __invalidate_snapshot() argument
1434 if (!s->valid) in __invalidate_snapshot()
1442 if (s->store->type->drop_snapshot) in __invalidate_snapshot()
1443 s->store->type->drop_snapshot(s->store); in __invalidate_snapshot()
1445 s->valid = 0; in __invalidate_snapshot()
1447 dm_table_event(s->ti->table); in __invalidate_snapshot()
1454 struct dm_snapshot *s = pe->snap; in pending_complete() local
1462 down_write(&s->lock); in pending_complete()
1463 __invalidate_snapshot(s, -EIO); in pending_complete()
1470 down_write(&s->lock); in pending_complete()
1471 __invalidate_snapshot(s, -ENOMEM); in pending_complete()
1477 down_write(&s->lock); in pending_complete()
1478 if (!s->valid) { in pending_complete()
1485 __check_for_conflicting_io(s, pe->e.old_chunk); in pending_complete()
1491 dm_insert_exception(&s->complete, e); in pending_complete()
1504 up_write(&s->lock); in pending_complete()
1517 retry_origin_bios(s, origin_bios); in pending_complete()
1524 struct dm_snapshot *s = pe->snap; in complete_exception() local
1527 s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error, in complete_exception()
1538 struct dm_snapshot *s = pe->snap; in copy_callback() local
1542 if (pe->exception_sequence == s->exception_complete_sequence) { in copy_callback()
1543 s->exception_complete_sequence++; in copy_callback()
1546 while (!list_empty(&s->out_of_order_list)) { in copy_callback()
1547 pe = list_entry(s->out_of_order_list.next, in copy_callback()
1549 if (pe->exception_sequence != s->exception_complete_sequence) in copy_callback()
1551 s->exception_complete_sequence++; in copy_callback()
1559 list_for_each_prev(lh, &s->out_of_order_list) { in copy_callback()
1573 struct dm_snapshot *s = pe->snap; in start_copy() local
1575 struct block_device *bdev = s->origin->bdev; in start_copy()
1581 src.sector = chunk_to_sector(s->store, pe->e.old_chunk); in start_copy()
1582 src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector); in start_copy()
1584 dest.bdev = s->cow->bdev; in start_copy()
1585 dest.sector = chunk_to_sector(s->store, pe->e.new_chunk); in start_copy()
1589 dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); in start_copy()
1602 struct dm_snapshot *s = pe->snap; in start_full_bio() local
1609 callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client, in start_full_bio()
1619 __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk) in __lookup_pending_exception() argument
1621 struct dm_exception *e = dm_lookup_exception(&s->pending, chunk); in __lookup_pending_exception()
1638 __find_pending_exception(struct dm_snapshot *s, in __find_pending_exception() argument
1643 pe2 = __lookup_pending_exception(s, chunk); in __find_pending_exception()
1655 if (s->store->type->prepare_exception(s->store, &pe->e)) { in __find_pending_exception()
1660 pe->exception_sequence = s->exception_start_sequence++; in __find_pending_exception()
1662 dm_insert_exception(&s->pending, &pe->e); in __find_pending_exception()
1667 static void remap_exception(struct dm_snapshot *s, struct dm_exception *e, in remap_exception() argument
1670 bio->bi_bdev = s->cow->bdev; in remap_exception()
1672 chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) + in remap_exception()
1674 (bio->bi_iter.bi_sector & s->store->chunk_mask); in remap_exception()
1680 struct dm_snapshot *s = ti->private; in snapshot_map() local
1688 bio->bi_bdev = s->cow->bdev; in snapshot_map()
1692 chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); in snapshot_map()
1696 if (!s->valid) in snapshot_map()
1701 down_write(&s->lock); in snapshot_map()
1703 if (!s->valid || (unlikely(s->snapshot_overflowed) && bio_rw(bio) == WRITE)) { in snapshot_map()
1709 e = dm_lookup_exception(&s->complete, chunk); in snapshot_map()
1711 remap_exception(s, e, bio, chunk); in snapshot_map()
1721 pe = __lookup_pending_exception(s, chunk); in snapshot_map()
1723 up_write(&s->lock); in snapshot_map()
1724 pe = alloc_pending_exception(s); in snapshot_map()
1725 down_write(&s->lock); in snapshot_map()
1727 if (!s->valid || s->snapshot_overflowed) { in snapshot_map()
1733 e = dm_lookup_exception(&s->complete, chunk); in snapshot_map()
1736 remap_exception(s, e, bio, chunk); in snapshot_map()
1740 pe = __find_pending_exception(s, pe, chunk); in snapshot_map()
1742 if (s->store->userspace_supports_overflow) { in snapshot_map()
1743 s->snapshot_overflowed = 1; in snapshot_map()
1746 __invalidate_snapshot(s, -ENOMEM); in snapshot_map()
1752 remap_exception(s, &pe->e, bio, chunk); in snapshot_map()
1758 (s->store->chunk_size << SECTOR_SHIFT)) { in snapshot_map()
1760 up_write(&s->lock); in snapshot_map()
1770 up_write(&s->lock); in snapshot_map()
1775 bio->bi_bdev = s->origin->bdev; in snapshot_map()
1776 track_chunk(s, bio, chunk); in snapshot_map()
1780 up_write(&s->lock); in snapshot_map()
1800 struct dm_snapshot *s = ti->private; in snapshot_merge_map() local
1808 bio->bi_bdev = s->origin->bdev; in snapshot_merge_map()
1810 bio->bi_bdev = s->cow->bdev; in snapshot_merge_map()
1814 chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); in snapshot_merge_map()
1816 down_write(&s->lock); in snapshot_merge_map()
1819 if (!s->valid) in snapshot_merge_map()
1823 e = dm_lookup_exception(&s->complete, chunk); in snapshot_merge_map()
1827 chunk >= s->first_merging_chunk && in snapshot_merge_map()
1828 chunk < (s->first_merging_chunk + in snapshot_merge_map()
1829 s->num_merging_chunks)) { in snapshot_merge_map()
1830 bio->bi_bdev = s->origin->bdev; in snapshot_merge_map()
1831 bio_list_add(&s->bios_queued_during_merge, bio); in snapshot_merge_map()
1836 remap_exception(s, e, bio, chunk); in snapshot_merge_map()
1839 track_chunk(s, bio, chunk); in snapshot_merge_map()
1844 bio->bi_bdev = s->origin->bdev; in snapshot_merge_map()
1847 up_write(&s->lock); in snapshot_merge_map()
1848 return do_origin(s->origin, bio); in snapshot_merge_map()
1852 up_write(&s->lock); in snapshot_merge_map()
1859 struct dm_snapshot *s = ti->private; in snapshot_end_io() local
1862 stop_tracking_chunk(s, bio); in snapshot_end_io()
1869 struct dm_snapshot *s = ti->private; in snapshot_merge_presuspend() local
1871 stop_merge(s); in snapshot_merge_presuspend()
1877 struct dm_snapshot *s = ti->private; in snapshot_preresume() local
1881 (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); in snapshot_preresume()
1884 if (s == snap_src) { in snapshot_preresume()
1902 struct dm_snapshot *s = ti->private; in snapshot_resume() local
1910 o = __lookup_dm_origin(s->origin->bdev); in snapshot_resume()
1914 (void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging); in snapshot_resume()
1937 (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); in snapshot_resume()
1956 reregister_snapshot(s); in snapshot_resume()
1958 down_write(&s->lock); in snapshot_resume()
1959 s->active = 1; in snapshot_resume()
1960 up_write(&s->lock); in snapshot_resume()
1976 struct dm_snapshot *s = ti->private; in snapshot_merge_resume() local
1986 ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev); in snapshot_merge_resume()
1988 start_merge(s); in snapshot_merge_resume()