686 	struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,  in alloc_pending_exception()  local
690 	pe->snap = s;  in alloc_pending_exception()
692 	return pe;  in alloc_pending_exception()
695 static void free_pending_exception(struct dm_snap_pending_exception *pe)  in free_pending_exception()  argument
697 	struct dm_snapshot *s = pe->snap;  in free_pending_exception()
699 	mempool_free(pe, s->pending_pool);  in free_pending_exception()
1452 	struct dm_snap_pending_exception *pe = context;  in pending_complete()  local
1454 	struct dm_snapshot *s = pe->snap;  in pending_complete()
1475 	*e = pe->e;  in pending_complete()
1485 	__check_for_conflicting_io(s, pe->e.old_chunk);  in pending_complete()
1494 	dm_remove_exception(&pe->e);  in pending_complete()
1495 	snapshot_bios = bio_list_get(&pe->snapshot_bios);  in pending_complete()
1496 	origin_bios = bio_list_get(&pe->origin_bios);  in pending_complete()
1497 	full_bio = pe->full_bio;  in pending_complete()
1499 		full_bio->bi_end_io = pe->full_bio_end_io;  in pending_complete()
1500 		full_bio->bi_private = pe->full_bio_private;  in pending_complete()
1519 	free_pending_exception(pe);  in pending_complete()
1522 static void complete_exception(struct dm_snap_pending_exception *pe)  in complete_exception()  argument
1524 	struct dm_snapshot *s = pe->snap;  in complete_exception()
1527 	s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error,  in complete_exception()
1528 					 pending_complete, pe);  in complete_exception()
1537 	struct dm_snap_pending_exception *pe = context;  in copy_callback()  local
1538 	struct dm_snapshot *s = pe->snap;  in copy_callback()
1540 	pe->copy_error = read_err || write_err;  in copy_callback()
1542 	if (pe->exception_sequence == s->exception_complete_sequence) {  in copy_callback()
1544 		complete_exception(pe);  in copy_callback()
1547 			pe = list_entry(s->out_of_order_list.next,  in copy_callback()
1549 			if (pe->exception_sequence != s->exception_complete_sequence)  in copy_callback()
1552 			list_del(&pe->out_of_order_entry);  in copy_callback()
1553 			complete_exception(pe);  in copy_callback()
1561 			if (pe2->exception_sequence < pe->exception_sequence)  in copy_callback()
1564 		list_add(&pe->out_of_order_entry, lh);  in copy_callback()
1571 static void start_copy(struct dm_snap_pending_exception *pe)  in start_copy()  argument
1573 	struct dm_snapshot *s = pe->snap;  in start_copy()
1581 	src.sector = chunk_to_sector(s->store, pe->e.old_chunk);  in start_copy()
1585 	dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);  in start_copy()
1589 	dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);  in start_copy()
1599 static void start_full_bio(struct dm_snap_pending_exception *pe,  in start_full_bio()  argument
1602 	struct dm_snapshot *s = pe->snap;  in start_full_bio()
1605 	pe->full_bio = bio;  in start_full_bio()
1606 	pe->full_bio_end_io = bio->bi_end_io;  in start_full_bio()
1607 	pe->full_bio_private = bio->bi_private;  in start_full_bio()
1610 						   copy_callback, pe);  in start_full_bio()
1639 			 struct dm_snap_pending_exception *pe, chunk_t chunk)  in __find_pending_exception()  argument
1645 		free_pending_exception(pe);  in __find_pending_exception()
1649 	pe->e.old_chunk = chunk;  in __find_pending_exception()
1650 	bio_list_init(&pe->origin_bios);  in __find_pending_exception()
1651 	bio_list_init(&pe->snapshot_bios);  in __find_pending_exception()
1652 	pe->started = 0;  in __find_pending_exception()
1653 	pe->full_bio = NULL;  in __find_pending_exception()
1655 	if (s->store->type->prepare_exception(s->store, &pe->e)) {  in __find_pending_exception()
1656 		free_pending_exception(pe);  in __find_pending_exception()
1660 	pe->exception_sequence = s->exception_start_sequence++;  in __find_pending_exception()
1662 	dm_insert_exception(&s->pending, &pe->e);  in __find_pending_exception()
1664 	return pe;  in __find_pending_exception()
1683 	struct dm_snap_pending_exception *pe = NULL;  in snapshot_map()  local
1721 		pe = __lookup_pending_exception(s, chunk);  in snapshot_map()
1722 		if (!pe) {  in snapshot_map()
1724 			pe = alloc_pending_exception(s);  in snapshot_map()
1728 				free_pending_exception(pe);  in snapshot_map()
1735 				free_pending_exception(pe);  in snapshot_map()
1740 			pe = __find_pending_exception(s, pe, chunk);  in snapshot_map()
1741 			if (!pe) {  in snapshot_map()
1752 		remap_exception(s, &pe->e, bio, chunk);  in snapshot_map()
1756 		if (!pe->started &&  in snapshot_map()
1759 			pe->started = 1;  in snapshot_map()
1761 			start_full_bio(pe, bio);  in snapshot_map()
1765 		bio_list_add(&pe->snapshot_bios, bio);  in snapshot_map()
1767 		if (!pe->started) {  in snapshot_map()
1769 			pe->started = 1;  in snapshot_map()
1771 			start_copy(pe);  in snapshot_map()
2077 	struct dm_snap_pending_exception *pe;  in __origin_write()  local
2116 		pe = __lookup_pending_exception(snap, chunk);  in __origin_write()
2117 		if (!pe) {  in __origin_write()
2119 			pe = alloc_pending_exception(snap);  in __origin_write()
2123 				free_pending_exception(pe);  in __origin_write()
2129 				free_pending_exception(pe);  in __origin_write()
2133 			pe = __find_pending_exception(snap, pe, chunk);  in __origin_write()
2134 			if (!pe) {  in __origin_write()
2148 			bio_list_add(&pe->origin_bios, bio);  in __origin_write()
2151 			if (!pe->started) {  in __origin_write()
2152 				pe->started = 1;  in __origin_write()
2153 				pe_to_start_last = pe;  in __origin_write()
2157 		if (!pe->started) {  in __origin_write()
2158 			pe->started = 1;  in __origin_write()
2159 			pe_to_start_now = pe;  in __origin_write()