144 struct dm_dev *dm_snap_origin(struct dm_snapshot *s)  in dm_snap_origin()  argument
146 	return s->origin;  in dm_snap_origin()
150 struct dm_dev *dm_snap_cow(struct dm_snapshot *s)  in dm_snap_cow()  argument
152 	return s->cow;  in dm_snap_cow()
230 static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk)  in track_chunk()  argument
236 	spin_lock_irq(&s->tracked_chunk_lock);  in track_chunk()
238 		       &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);  in track_chunk()
239 	spin_unlock_irq(&s->tracked_chunk_lock);  in track_chunk()
242 static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio)  in stop_tracking_chunk()  argument
247 	spin_lock_irqsave(&s->tracked_chunk_lock, flags);  in stop_tracking_chunk()
249 	spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);  in stop_tracking_chunk()
252 static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)  in __chunk_is_tracked()  argument
257 	spin_lock_irq(&s->tracked_chunk_lock);  in __chunk_is_tracked()
260 	    &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {  in __chunk_is_tracked()
267 	spin_unlock_irq(&s->tracked_chunk_lock);  in __chunk_is_tracked()
276 static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk)  in __check_for_conflicting_io()  argument
278 	while (__chunk_is_tracked(s, chunk))  in __check_for_conflicting_io()
421 	struct dm_snapshot *s;  in __find_snapshots_sharing_cow()  local
430 	list_for_each_entry(s, &o->snapshots, list) {  in __find_snapshots_sharing_cow()
431 		if (dm_target_is_snapshot_merge(s->ti) && snap_merge)  in __find_snapshots_sharing_cow()
432 			*snap_merge = s;  in __find_snapshots_sharing_cow()
433 		if (!bdev_equal(s->cow->bdev, snap->cow->bdev))  in __find_snapshots_sharing_cow()
436 		down_read(&s->lock);  in __find_snapshots_sharing_cow()
437 		active = s->active;  in __find_snapshots_sharing_cow()
438 		up_read(&s->lock);  in __find_snapshots_sharing_cow()
442 				*snap_src = s;  in __find_snapshots_sharing_cow()
444 			*snap_dest = s;  in __find_snapshots_sharing_cow()
502 static void __insert_snapshot(struct origin *o, struct dm_snapshot *s)  in __insert_snapshot()  argument
508 		if (l->store->chunk_size < s->store->chunk_size)  in __insert_snapshot()
510 	list_add_tail(&s->list, &l->list);  in __insert_snapshot()
564 static void reregister_snapshot(struct dm_snapshot *s)  in reregister_snapshot()  argument
566 	struct block_device *bdev = s->origin->bdev;  in reregister_snapshot()
570 	list_del(&s->list);  in reregister_snapshot()
571 	__insert_snapshot(__lookup_origin(bdev), s);  in reregister_snapshot()
576 static void unregister_snapshot(struct dm_snapshot *s)  in unregister_snapshot()  argument
581 	o = __lookup_origin(s->origin->bdev);  in unregister_snapshot()
583 	list_del(&s->list);  in unregister_snapshot()
677 static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)  in alloc_pending_exception()  argument
679 	struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,  in alloc_pending_exception()
682 	atomic_inc(&s->pending_exceptions_count);  in alloc_pending_exception()
683 	pe->snap = s;  in alloc_pending_exception()
690 	struct dm_snapshot *s = pe->snap;  in free_pending_exception()  local
692 	mempool_free(pe, s->pending_pool);  in free_pending_exception()
694 	atomic_dec(&s->pending_exceptions_count);  in free_pending_exception()
745 	struct dm_snapshot *s = context;  in dm_add_exception()  local
757 	dm_insert_exception(&s->complete, e);  in dm_add_exception()
794 static int init_hash_tables(struct dm_snapshot *s)  in init_hash_tables()  argument
802 	cow_dev_size = get_dev_size(s->cow->bdev);  in init_hash_tables()
805 	hash_size = cow_dev_size >> s->store->chunk_shift;  in init_hash_tables()
811 	if (dm_exception_table_init(&s->complete, hash_size,  in init_hash_tables()
823 	if (dm_exception_table_init(&s->pending, hash_size, 0)) {  in init_hash_tables()
824 		dm_exception_table_exit(&s->complete, exception_cache);  in init_hash_tables()
831 static void merge_shutdown(struct dm_snapshot *s)  in merge_shutdown()  argument
833 	clear_bit_unlock(RUNNING_MERGE, &s->state_bits);  in merge_shutdown()
835 	wake_up_bit(&s->state_bits, RUNNING_MERGE);  in merge_shutdown()
838 static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)  in __release_queued_bios_after_merge()  argument
840 	s->first_merging_chunk = 0;  in __release_queued_bios_after_merge()
841 	s->num_merging_chunks = 0;  in __release_queued_bios_after_merge()
843 	return bio_list_get(&s->bios_queued_during_merge);  in __release_queued_bios_after_merge()
849 static int __remove_single_exception_chunk(struct dm_snapshot *s,  in __remove_single_exception_chunk()  argument
854 	e = dm_lookup_exception(&s->complete, old_chunk);  in __remove_single_exception_chunk()
900 static int remove_single_exception_chunk(struct dm_snapshot *s)  in remove_single_exception_chunk()  argument
904 	chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;  in remove_single_exception_chunk()
906 	down_write(&s->lock);  in remove_single_exception_chunk()
913 		r = __remove_single_exception_chunk(s, old_chunk);  in remove_single_exception_chunk()
916 	} while (old_chunk-- > s->first_merging_chunk);  in remove_single_exception_chunk()
918 	b = __release_queued_bios_after_merge(s);  in remove_single_exception_chunk()
921 	up_write(&s->lock);  in remove_single_exception_chunk()
954 static void snapshot_merge_next_chunks(struct dm_snapshot *s)  in snapshot_merge_next_chunks()  argument
962 	BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits));  in snapshot_merge_next_chunks()
963 	if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits)))  in snapshot_merge_next_chunks()
969 	if (!s->valid) {  in snapshot_merge_next_chunks()
974 	linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk,  in snapshot_merge_next_chunks()
980 			down_write(&s->lock);  in snapshot_merge_next_chunks()
981 			s->merge_failed = 1;  in snapshot_merge_next_chunks()
982 			up_write(&s->lock);  in snapshot_merge_next_chunks()
995 	io_size = linear_chunks * s->store->chunk_size;  in snapshot_merge_next_chunks()
997 	dest.bdev = s->origin->bdev;  in snapshot_merge_next_chunks()
998 	dest.sector = chunk_to_sector(s->store, old_chunk);  in snapshot_merge_next_chunks()
1001 	src.bdev = s->cow->bdev;  in snapshot_merge_next_chunks()
1002 	src.sector = chunk_to_sector(s->store, new_chunk);  in snapshot_merge_next_chunks()
1015 	while (origin_write_extent(s, dest.sector, io_size)) {  in snapshot_merge_next_chunks()
1023 	down_write(&s->lock);  in snapshot_merge_next_chunks()
1024 	s->first_merging_chunk = old_chunk;  in snapshot_merge_next_chunks()
1025 	s->num_merging_chunks = linear_chunks;  in snapshot_merge_next_chunks()
1026 	up_write(&s->lock);  in snapshot_merge_next_chunks()
1030 		__check_for_conflicting_io(s, old_chunk + i);  in snapshot_merge_next_chunks()
1032 	dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s);  in snapshot_merge_next_chunks()
1036 	merge_shutdown(s);  in snapshot_merge_next_chunks()
1043 	struct dm_snapshot *s = context;  in merge_callback()  local
1054 	if (s->store->type->commit_merge(s->store,  in merge_callback()
1055 					 s->num_merging_chunks) < 0) {  in merge_callback()
1060 	if (remove_single_exception_chunk(s) < 0)  in merge_callback()
1063 	snapshot_merge_next_chunks(s);  in merge_callback()
1068 	down_write(&s->lock);  in merge_callback()
1069 	s->merge_failed = 1;  in merge_callback()
1070 	b = __release_queued_bios_after_merge(s);  in merge_callback()
1071 	up_write(&s->lock);  in merge_callback()
1074 	merge_shutdown(s);  in merge_callback()
1077 static void start_merge(struct dm_snapshot *s)  in start_merge()  argument
1079 	if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits))  in start_merge()
1080 		snapshot_merge_next_chunks(s);  in start_merge()
1086 static void stop_merge(struct dm_snapshot *s)  in stop_merge()  argument
1088 	set_bit(SHUTDOWN_MERGE, &s->state_bits);  in stop_merge()
1089 	wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE);  in stop_merge()
1090 	clear_bit(SHUTDOWN_MERGE, &s->state_bits);  in stop_merge()
1098 	struct dm_snapshot *s;  in snapshot_ctr()  local
1116 	s = kmalloc(sizeof(*s), GFP_KERNEL);  in snapshot_ctr()
1117 	if (!s) {  in snapshot_ctr()
1127 	r = dm_get_device(ti, origin_path, origin_mode, &s->origin);  in snapshot_ctr()
1137 	r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow);  in snapshot_ctr()
1143 	r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store);  in snapshot_ctr()
1153 	s->ti = ti;  in snapshot_ctr()
1154 	s->valid = 1;  in snapshot_ctr()
1155 	s->active = 0;  in snapshot_ctr()
1156 	atomic_set(&s->pending_exceptions_count, 0);  in snapshot_ctr()
1157 	s->exception_start_sequence = 0;  in snapshot_ctr()
1158 	s->exception_complete_sequence = 0;  in snapshot_ctr()
1159 	INIT_LIST_HEAD(&s->out_of_order_list);  in snapshot_ctr()
1160 	init_rwsem(&s->lock);  in snapshot_ctr()
1161 	INIT_LIST_HEAD(&s->list);  in snapshot_ctr()
1162 	spin_lock_init(&s->pe_lock);  in snapshot_ctr()
1163 	s->state_bits = 0;  in snapshot_ctr()
1164 	s->merge_failed = 0;  in snapshot_ctr()
1165 	s->first_merging_chunk = 0;  in snapshot_ctr()
1166 	s->num_merging_chunks = 0;  in snapshot_ctr()
1167 	bio_list_init(&s->bios_queued_during_merge);  in snapshot_ctr()
1170 	if (init_hash_tables(s)) {  in snapshot_ctr()
1176 	s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);  in snapshot_ctr()
1177 	if (IS_ERR(s->kcopyd_client)) {  in snapshot_ctr()
1178 		r = PTR_ERR(s->kcopyd_client);  in snapshot_ctr()
1183 	s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);  in snapshot_ctr()
1184 	if (!s->pending_pool) {  in snapshot_ctr()
1191 		INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);  in snapshot_ctr()
1193 	spin_lock_init(&s->tracked_chunk_lock);  in snapshot_ctr()
1195 	ti->private = s;  in snapshot_ctr()
1201 	r = register_snapshot(s);  in snapshot_ctr()
1217 		s->store->chunk_size = 0;  in snapshot_ctr()
1221 	r = s->store->type->read_metadata(s->store, dm_add_exception,  in snapshot_ctr()
1222 					  (void *)s);  in snapshot_ctr()
1227 		s->valid = 0;  in snapshot_ctr()
1231 	if (!s->store->chunk_size) {  in snapshot_ctr()
1236 	r = dm_set_target_max_io_len(ti, s->store->chunk_size);  in snapshot_ctr()
1243 	unregister_snapshot(s);  in snapshot_ctr()
1246 	mempool_destroy(s->pending_pool);  in snapshot_ctr()
1249 	dm_kcopyd_client_destroy(s->kcopyd_client);  in snapshot_ctr()
1252 	dm_exception_table_exit(&s->pending, pending_cache);  in snapshot_ctr()
1253 	dm_exception_table_exit(&s->complete, exception_cache);  in snapshot_ctr()
1256 	dm_exception_store_destroy(s->store);  in snapshot_ctr()
1259 	dm_put_device(ti, s->cow);  in snapshot_ctr()
1262 	dm_put_device(ti, s->origin);  in snapshot_ctr()
1265 	kfree(s);  in snapshot_ctr()
1271 static void __free_exceptions(struct dm_snapshot *s)  in __free_exceptions()  argument
1273 	dm_kcopyd_client_destroy(s->kcopyd_client);  in __free_exceptions()
1274 	s->kcopyd_client = NULL;  in __free_exceptions()
1276 	dm_exception_table_exit(&s->pending, pending_cache);  in __free_exceptions()
1277 	dm_exception_table_exit(&s->complete, exception_cache);  in __free_exceptions()
1316 	struct dm_snapshot *s = ti->private;  in snapshot_dtr()  local
1321 	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);  in snapshot_dtr()
1322 	if (snap_src && snap_dest && (s == snap_src)) {  in snapshot_dtr()
1331 		stop_merge(s);  in snapshot_dtr()
1335 	unregister_snapshot(s);  in snapshot_dtr()
1337 	while (atomic_read(&s->pending_exceptions_count))  in snapshot_dtr()
1347 		BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));  in snapshot_dtr()
1350 	__free_exceptions(s);  in snapshot_dtr()
1352 	mempool_destroy(s->pending_pool);  in snapshot_dtr()
1354 	dm_exception_store_destroy(s->store);  in snapshot_dtr()
1356 	dm_put_device(ti, s->cow);  in snapshot_dtr()
1358 	dm_put_device(ti, s->origin);  in snapshot_dtr()
1360 	kfree(s);  in snapshot_dtr()
1383 static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)  in retry_origin_bios()  argument
1391 		r = do_origin(s->origin, bio);  in retry_origin_bios()
1413 static void __invalidate_snapshot(struct dm_snapshot *s, int err)  in __invalidate_snapshot()  argument
1415 	if (!s->valid)  in __invalidate_snapshot()
1423 	if (s->store->type->drop_snapshot)  in __invalidate_snapshot()
1424 		s->store->type->drop_snapshot(s->store);  in __invalidate_snapshot()
1426 	s->valid = 0;  in __invalidate_snapshot()
1428 	dm_table_event(s->ti->table);  in __invalidate_snapshot()
1435 	struct dm_snapshot *s = pe->snap;  in pending_complete()  local
1443 		down_write(&s->lock);  in pending_complete()
1444 		__invalidate_snapshot(s, -EIO);  in pending_complete()
1451 		down_write(&s->lock);  in pending_complete()
1452 		__invalidate_snapshot(s, -ENOMEM);  in pending_complete()
1458 	down_write(&s->lock);  in pending_complete()
1459 	if (!s->valid) {  in pending_complete()
1466 	__check_for_conflicting_io(s, pe->e.old_chunk);  in pending_complete()
1472 	dm_insert_exception(&s->complete, e);  in pending_complete()
1486 	up_write(&s->lock);  in pending_complete()
1499 	retry_origin_bios(s, origin_bios);  in pending_complete()
1506 	struct dm_snapshot *s = pe->snap;  in complete_exception()  local
1509 	s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error,  in complete_exception()
1520 	struct dm_snapshot *s = pe->snap;  in copy_callback()  local
1524 	if (pe->exception_sequence == s->exception_complete_sequence) {  in copy_callback()
1525 		s->exception_complete_sequence++;  in copy_callback()
1528 		while (!list_empty(&s->out_of_order_list)) {  in copy_callback()
1529 			pe = list_entry(s->out_of_order_list.next,  in copy_callback()
1531 			if (pe->exception_sequence != s->exception_complete_sequence)  in copy_callback()
1533 			s->exception_complete_sequence++;  in copy_callback()
1541 		list_for_each_prev(lh, &s->out_of_order_list) {  in copy_callback()
1555 	struct dm_snapshot *s = pe->snap;  in start_copy()  local
1557 	struct block_device *bdev = s->origin->bdev;  in start_copy()
1563 	src.sector = chunk_to_sector(s->store, pe->e.old_chunk);  in start_copy()
1564 	src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);  in start_copy()
1566 	dest.bdev = s->cow->bdev;  in start_copy()
1567 	dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);  in start_copy()
1571 	dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);  in start_copy()
1584 	struct dm_snapshot *s = pe->snap;  in start_full_bio()  local
1591 	callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,  in start_full_bio()
1601 __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)  in __lookup_pending_exception()  argument
1603 	struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);  in __lookup_pending_exception()
1620 __find_pending_exception(struct dm_snapshot *s,  in __find_pending_exception()  argument
1625 	pe2 = __lookup_pending_exception(s, chunk);  in __find_pending_exception()
1637 	if (s->store->type->prepare_exception(s->store, &pe->e)) {  in __find_pending_exception()
1642 	pe->exception_sequence = s->exception_start_sequence++;  in __find_pending_exception()
1644 	dm_insert_exception(&s->pending, &pe->e);  in __find_pending_exception()
1649 static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,  in remap_exception()  argument
1652 	bio->bi_bdev = s->cow->bdev;  in remap_exception()
1654 		chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) +  in remap_exception()
1656 		(bio->bi_iter.bi_sector & s->store->chunk_mask);  in remap_exception()
1662 	struct dm_snapshot *s = ti->private;  in snapshot_map()  local
1670 		bio->bi_bdev = s->cow->bdev;  in snapshot_map()
1674 	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);  in snapshot_map()
1678 	if (!s->valid)  in snapshot_map()
1683 	down_write(&s->lock);  in snapshot_map()
1685 	if (!s->valid) {  in snapshot_map()
1691 	e = dm_lookup_exception(&s->complete, chunk);  in snapshot_map()
1693 		remap_exception(s, e, bio, chunk);  in snapshot_map()
1703 		pe = __lookup_pending_exception(s, chunk);  in snapshot_map()
1705 			up_write(&s->lock);  in snapshot_map()
1706 			pe = alloc_pending_exception(s);  in snapshot_map()
1707 			down_write(&s->lock);  in snapshot_map()
1709 			if (!s->valid) {  in snapshot_map()
1715 			e = dm_lookup_exception(&s->complete, chunk);  in snapshot_map()
1718 				remap_exception(s, e, bio, chunk);  in snapshot_map()
1722 			pe = __find_pending_exception(s, pe, chunk);  in snapshot_map()
1724 				__invalidate_snapshot(s, -ENOMEM);  in snapshot_map()
1730 		remap_exception(s, &pe->e, bio, chunk);  in snapshot_map()
1736 		    (s->store->chunk_size << SECTOR_SHIFT)) {  in snapshot_map()
1738 			up_write(&s->lock);  in snapshot_map()
1748 			up_write(&s->lock);  in snapshot_map()
1753 		bio->bi_bdev = s->origin->bdev;  in snapshot_map()
1754 		track_chunk(s, bio, chunk);  in snapshot_map()
1758 	up_write(&s->lock);  in snapshot_map()
1778 	struct dm_snapshot *s = ti->private;  in snapshot_merge_map()  local
1786 			bio->bi_bdev = s->origin->bdev;  in snapshot_merge_map()
1788 			bio->bi_bdev = s->cow->bdev;  in snapshot_merge_map()
1792 	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);  in snapshot_merge_map()
1794 	down_write(&s->lock);  in snapshot_merge_map()
1797 	if (!s->valid)  in snapshot_merge_map()
1801 	e = dm_lookup_exception(&s->complete, chunk);  in snapshot_merge_map()
1805 		    chunk >= s->first_merging_chunk &&  in snapshot_merge_map()
1806 		    chunk < (s->first_merging_chunk +  in snapshot_merge_map()
1807 			     s->num_merging_chunks)) {  in snapshot_merge_map()
1808 			bio->bi_bdev = s->origin->bdev;  in snapshot_merge_map()
1809 			bio_list_add(&s->bios_queued_during_merge, bio);  in snapshot_merge_map()
1814 		remap_exception(s, e, bio, chunk);  in snapshot_merge_map()
1817 			track_chunk(s, bio, chunk);  in snapshot_merge_map()
1822 	bio->bi_bdev = s->origin->bdev;  in snapshot_merge_map()
1825 		up_write(&s->lock);  in snapshot_merge_map()
1826 		return do_origin(s->origin, bio);  in snapshot_merge_map()
1830 	up_write(&s->lock);  in snapshot_merge_map()
1837 	struct dm_snapshot *s = ti->private;  in snapshot_end_io()  local
1840 		stop_tracking_chunk(s, bio);  in snapshot_end_io()
1847 	struct dm_snapshot *s = ti->private;  in snapshot_merge_presuspend()  local
1849 	stop_merge(s);  in snapshot_merge_presuspend()
1855 	struct dm_snapshot *s = ti->private;  in snapshot_preresume()  local
1859 	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);  in snapshot_preresume()
1862 		if (s == snap_src) {  in snapshot_preresume()
1880 	struct dm_snapshot *s = ti->private;  in snapshot_resume()  local
1888 	o = __lookup_dm_origin(s->origin->bdev);  in snapshot_resume()
1892 		(void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging);  in snapshot_resume()
1915 	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);  in snapshot_resume()
1934 	reregister_snapshot(s);  in snapshot_resume()
1936 	down_write(&s->lock);  in snapshot_resume()
1937 	s->active = 1;  in snapshot_resume()
1938 	up_write(&s->lock);  in snapshot_resume()
1954 	struct dm_snapshot *s = ti->private;  in snapshot_merge_resume()  local
1964 	ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev);  in snapshot_merge_resume()
1966 	start_merge(s);  in snapshot_merge_resume()