201 	struct bio *full_bio;
218 static void init_tracked_chunk(struct bio *bio)  in init_tracked_chunk()  argument
220 	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));  in init_tracked_chunk()
224 static bool is_bio_tracked(struct bio *bio)  in is_bio_tracked()  argument
226 	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));  in is_bio_tracked()
230 static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk)  in track_chunk()  argument
232 	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));  in track_chunk()
242 static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio)  in stop_tracking_chunk()  argument
244 	struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));  in stop_tracking_chunk()
838 static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)  in __release_queued_bios_after_merge()
898 static void flush_bios(struct bio *bio);
902 	struct bio *b = NULL;  in remove_single_exception_chunk()
1039 static void error_bios(struct bio *bio);
1044 	struct bio *b = NULL;  in merge_callback()
1366 static void flush_bios(struct bio *bio)  in flush_bios()  argument
1368 	struct bio *n;  in flush_bios()
1370 	while (bio) {  in flush_bios()
1371 		n = bio->bi_next;  in flush_bios()
1372 		bio->bi_next = NULL;  in flush_bios()
1373 		generic_make_request(bio);  in flush_bios()
1374 		bio = n;  in flush_bios()
1378 static int do_origin(struct dm_dev *origin, struct bio *bio);
1383 static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)  in retry_origin_bios()  argument
1385 	struct bio *n;  in retry_origin_bios()
1388 	while (bio) {  in retry_origin_bios()
1389 		n = bio->bi_next;  in retry_origin_bios()
1390 		bio->bi_next = NULL;  in retry_origin_bios()
1391 		r = do_origin(s->origin, bio);  in retry_origin_bios()
1393 			generic_make_request(bio);  in retry_origin_bios()
1394 		bio = n;  in retry_origin_bios()
1401 static void error_bios(struct bio *bio)  in error_bios()  argument
1403 	struct bio *n;  in error_bios()
1405 	while (bio) {  in error_bios()
1406 		n = bio->bi_next;  in error_bios()
1407 		bio->bi_next = NULL;  in error_bios()
1408 		bio_io_error(bio);  in error_bios()
1409 		bio = n;  in error_bios()
1436 	struct bio *origin_bios = NULL;  in pending_complete()
1437 	struct bio *snapshot_bios = NULL;  in pending_complete()
1438 	struct bio *full_bio = NULL;  in pending_complete()
1574 static void full_bio_end_io(struct bio *bio, int error)  in full_bio_end_io()  argument
1576 	void *callback_data = bio->bi_private;  in full_bio_end_io()
1582 			   struct bio *bio)  in start_full_bio()  argument
1587 	pe->full_bio = bio;  in start_full_bio()
1588 	pe->full_bio_end_io = bio->bi_end_io;  in start_full_bio()
1589 	pe->full_bio_private = bio->bi_private;  in start_full_bio()
1594 	bio->bi_end_io = full_bio_end_io;  in start_full_bio()
1595 	bio->bi_private = callback_data;  in start_full_bio()
1597 	generic_make_request(bio);  in start_full_bio()
1650 			    struct bio *bio, chunk_t chunk)  in remap_exception()  argument
1652 	bio->bi_bdev = s->cow->bdev;  in remap_exception()
1653 	bio->bi_iter.bi_sector =  in remap_exception()
1656 		(bio->bi_iter.bi_sector & s->store->chunk_mask);  in remap_exception()
1659 static int snapshot_map(struct dm_target *ti, struct bio *bio)  in snapshot_map()  argument
1667 	init_tracked_chunk(bio);  in snapshot_map()
1669 	if (bio->bi_rw & REQ_FLUSH) {  in snapshot_map()
1670 		bio->bi_bdev = s->cow->bdev;  in snapshot_map()
1674 	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);  in snapshot_map()
1693 		remap_exception(s, e, bio, chunk);  in snapshot_map()
1702 	if (bio_rw(bio) == WRITE) {  in snapshot_map()
1718 				remap_exception(s, e, bio, chunk);  in snapshot_map()
1730 		remap_exception(s, &pe->e, bio, chunk);  in snapshot_map()
1735 		    bio->bi_iter.bi_size ==  in snapshot_map()
1739 			start_full_bio(pe, bio);  in snapshot_map()
1743 		bio_list_add(&pe->snapshot_bios, bio);  in snapshot_map()
1753 		bio->bi_bdev = s->origin->bdev;  in snapshot_map()
1754 		track_chunk(s, bio, chunk);  in snapshot_map()
1775 static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)  in snapshot_merge_map()  argument
1782 	init_tracked_chunk(bio);  in snapshot_merge_map()
1784 	if (bio->bi_rw & REQ_FLUSH) {  in snapshot_merge_map()
1785 		if (!dm_bio_get_target_bio_nr(bio))  in snapshot_merge_map()
1786 			bio->bi_bdev = s->origin->bdev;  in snapshot_merge_map()
1788 			bio->bi_bdev = s->cow->bdev;  in snapshot_merge_map()
1792 	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);  in snapshot_merge_map()
1804 		if (bio_rw(bio) == WRITE &&  in snapshot_merge_map()
1808 			bio->bi_bdev = s->origin->bdev;  in snapshot_merge_map()
1809 			bio_list_add(&s->bios_queued_during_merge, bio);  in snapshot_merge_map()
1814 		remap_exception(s, e, bio, chunk);  in snapshot_merge_map()
1816 		if (bio_rw(bio) == WRITE)  in snapshot_merge_map()
1817 			track_chunk(s, bio, chunk);  in snapshot_merge_map()
1822 	bio->bi_bdev = s->origin->bdev;  in snapshot_merge_map()
1824 	if (bio_rw(bio) == WRITE) {  in snapshot_merge_map()
1826 		return do_origin(s->origin, bio);  in snapshot_merge_map()
1835 static int snapshot_end_io(struct dm_target *ti, struct bio *bio, int error)  in snapshot_end_io()  argument
1839 	if (is_bio_tracked(bio))  in snapshot_end_io()
1840 		stop_tracking_chunk(s, bio);  in snapshot_end_io()
2048 			  struct bio *bio)  in __origin_write()  argument
2123 		if (bio) {  in __origin_write()
2124 			bio_list_add(&pe->origin_bios, bio);  in __origin_write()
2125 			bio = NULL;  in __origin_write()
2160 static int do_origin(struct dm_dev *origin, struct bio *bio)  in do_origin()  argument
2168 		r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio);  in do_origin()
2261 static int origin_map(struct dm_target *ti, struct bio *bio)  in origin_map()  argument
2266 	bio->bi_bdev = o->dev->bdev;  in origin_map()
2268 	if (unlikely(bio->bi_rw & REQ_FLUSH))  in origin_map()
2271 	if (bio_rw(bio) != WRITE)  in origin_map()
2275 		((unsigned)bio->bi_iter.bi_sector & (o->split_boundary - 1));  in origin_map()
2277 	if (bio_sectors(bio) > available_sectors)  in origin_map()
2278 		dm_accept_partial_bio(bio, available_sectors);  in origin_map()
2281 	return do_origin(o->dev, bio);  in origin_map()