210 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
351 			 struct dm_bio_prison_cell *cell,  in cell_release()  argument
354 	dm_cell_release(pool->prison, cell, bios);  in cell_release()
355 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_release()
361 			       struct dm_bio_prison_cell *cell)  in cell_visit_release()  argument
363 	dm_cell_visit_release(pool->prison, fn, context, cell);  in cell_visit_release()
364 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_visit_release()
368 				   struct dm_bio_prison_cell *cell,  in cell_release_no_holder()  argument
371 	dm_cell_release_no_holder(pool->prison, cell, bios);  in cell_release_no_holder()
372 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_release_no_holder()
376 				 struct dm_bio_prison_cell *cell, int error_code)  in cell_error_with_code()  argument
378 	dm_cell_error(pool->prison, cell, error_code);  in cell_error_with_code()
379 	dm_bio_prison_free_cell(pool->prison, cell);  in cell_error_with_code()
382 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)  in cell_error()  argument
384 	cell_error_with_code(pool, cell, -EIO);  in cell_error()
387 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)  in cell_success()  argument
389 	cell_error_with_code(pool, cell, 0);  in cell_success()
392 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)  in cell_requeue()  argument
394 	cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE);  in cell_requeue()
500 	struct dm_bio_prison_cell *cell, *tmp;  in requeue_deferred_cells()  local
508 	list_for_each_entry_safe(cell, tmp, &cells, user_list)  in requeue_deferred_cells()
509 		cell_requeue(pool, cell);  in requeue_deferred_cells()
664 	struct dm_bio_prison_cell *cell, *cell2;  member
727 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in cell_defer_no_holder()  argument
733 	cell_release_no_holder(pool, cell, &tc->deferred_bio_list);  in cell_defer_no_holder()
748 				       struct dm_bio_prison_cell *cell)  in __inc_remap_and_issue_cell()  argument
753 	while ((bio = bio_list_pop(&cell->bios))) {  in __inc_remap_and_issue_cell()
770 				     struct dm_bio_prison_cell *cell,  in inc_remap_and_issue_cell()  argument
786 			   &info, cell);  in inc_remap_and_issue_cell()
801 	cell_error(m->tc->pool, m->cell);  in process_prepared_mapping_fail()
820 		cell_error(pool, m->cell);  in process_prepared_mapping()
832 		cell_error(pool, m->cell);  in process_prepared_mapping()
843 		inc_remap_and_issue_cell(tc, m->cell, m->data_block);  in process_prepared_mapping()
846 		inc_all_io_entry(tc->pool, m->cell->holder);  in process_prepared_mapping()
847 		remap_and_issue(tc, m->cell->holder, m->data_block);  in process_prepared_mapping()
848 		inc_remap_and_issue_cell(tc, m->cell, m->data_block);  in process_prepared_mapping()
861 	cell_defer_no_holder(tc, m->cell);  in process_prepared_discard_fail()
871 	cell_defer_no_holder(tc, m->cell);  in process_prepared_discard_passdown()
1002 			  struct dm_bio_prison_cell *cell, struct bio *bio,  in schedule_copy()  argument
1012 	m->cell = cell;  in schedule_copy()
1073 				   struct dm_bio_prison_cell *cell, struct bio *bio)  in schedule_internal_copy()  argument
1076 		      data_origin, data_dest, cell, bio,  in schedule_internal_copy()
1081 			  dm_block_t data_block, struct dm_bio_prison_cell *cell,  in schedule_zero()  argument
1091 	m->cell = cell;  in schedule_zero()
1112 				   struct dm_bio_prison_cell *cell, struct bio *bio)  in schedule_external_copy()  argument
1120 			      virt_block, data_dest, cell, bio,  in schedule_external_copy()
1125 			      virt_block, data_dest, cell, bio,  in schedule_external_copy()
1129 		schedule_zero(tc, virt_block, data_dest, cell, bio);  in schedule_external_copy()
1279 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)  in retry_bios_on_resume()  argument
1287 		cell_error_with_code(pool, cell, error);  in retry_bios_on_resume()
1292 	cell_release(pool, cell, &bios);  in retry_bios_on_resume()
1298 static void process_discard_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_discard_cell()  argument
1301 	struct bio *bio = cell->holder;  in process_discard_cell()
1310 		cell_requeue(pool, cell);  in process_discard_cell()
1324 			cell_defer_no_holder(tc, cell);  in process_discard_cell()
1339 			m->cell = cell;  in process_discard_cell()
1348 			cell_defer_no_holder(tc, cell);  in process_discard_cell()
1367 		cell_defer_no_holder(tc, cell);  in process_discard_cell()
1374 		cell_defer_no_holder(tc, cell);  in process_discard_cell()
1382 	struct dm_bio_prison_cell *cell;  in process_discard_bio()  local
1387 	if (bio_detain(tc->pool, &key, bio, &cell))  in process_discard_bio()
1390 	process_discard_cell(tc, cell);  in process_discard_bio()
1396 			  struct dm_bio_prison_cell *cell)  in break_sharing()  argument
1406 				       data_block, cell, bio);  in break_sharing()
1410 		retry_bios_on_resume(pool, cell);  in break_sharing()
1416 		cell_error(pool, cell);  in break_sharing()
1422 					  struct dm_bio_prison_cell *cell)  in __remap_and_issue_shared_cell()  argument
1427 	while ((bio = bio_list_pop(&cell->bios))) {  in __remap_and_issue_shared_cell()
1442 					struct dm_bio_prison_cell *cell,  in remap_and_issue_shared_cell()  argument
1453 			   &info, cell);  in remap_and_issue_shared_cell()
1497 			    struct dm_bio_prison_cell *cell)  in provision_block()  argument
1508 		cell_defer_no_holder(tc, cell);  in provision_block()
1519 		cell_defer_no_holder(tc, cell);  in provision_block()
1528 			schedule_external_copy(tc, block, data_block, cell, bio);  in provision_block()
1530 			schedule_zero(tc, block, data_block, cell, bio);  in provision_block()
1534 		retry_bios_on_resume(pool, cell);  in provision_block()
1540 		cell_error(pool, cell);  in provision_block()
1545 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell()  argument
1549 	struct bio *bio = cell->holder;  in process_cell()
1554 		cell_requeue(pool, cell);  in process_cell()
1562 			process_shared_bio(tc, bio, block, &lookup_result, cell);  in process_cell()
1566 			inc_remap_and_issue_cell(tc, cell, lookup_result.block);  in process_cell()
1573 			cell_defer_no_holder(tc, cell);  in process_cell()
1588 			provision_block(tc, bio, block, cell);  in process_cell()
1594 		cell_defer_no_holder(tc, cell);  in process_cell()
1604 	struct dm_bio_prison_cell *cell;  in process_bio()  local
1612 	if (bio_detain(pool, &key, bio, &cell))  in process_bio()
1615 	process_cell(tc, cell);  in process_bio()
1619 				    struct dm_bio_prison_cell *cell)  in __process_bio_read_only()  argument
1631 			if (cell)  in __process_bio_read_only()
1632 				cell_defer_no_holder(tc, cell);  in __process_bio_read_only()
1636 			if (cell)  in __process_bio_read_only()
1637 				inc_remap_and_issue_cell(tc, cell, lookup_result.block);  in __process_bio_read_only()
1642 		if (cell)  in __process_bio_read_only()
1643 			cell_defer_no_holder(tc, cell);  in __process_bio_read_only()
1662 		if (cell)  in __process_bio_read_only()
1663 			cell_defer_no_holder(tc, cell);  in __process_bio_read_only()
1674 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell_read_only()  argument
1676 	__process_bio_read_only(tc, cell->holder, cell);  in process_cell_read_only()
1689 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell_success()  argument
1691 	cell_success(tc->pool, cell);  in process_cell_success()
1694 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in process_cell_fail()  argument
1696 	cell_error(tc->pool, cell);  in process_cell_fail()
1851 	struct dm_bio_prison_cell *cell, *tmp;  in sort_cells()  local
1853 	list_for_each_entry_safe(cell, tmp, cells, user_list) {  in sort_cells()
1857 		pool->cell_sort_array[count++] = cell;  in sort_cells()
1858 		list_del(&cell->user_list);  in sort_cells()
1861 	sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);  in sort_cells()
1871 	struct dm_bio_prison_cell *cell;  in process_thin_deferred_cells()  local
1887 			cell = pool->cell_sort_array[i];  in process_thin_deferred_cells()
1888 			BUG_ON(!cell->holder);  in process_thin_deferred_cells()
1905 			if (cell->holder->bi_rw & REQ_DISCARD)  in process_thin_deferred_cells()
1906 				pool->process_discard_cell(tc, cell);  in process_thin_deferred_cells()
1908 				pool->process_cell(tc, cell);  in process_thin_deferred_cells()
2260 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)  in thin_defer_cell()  argument
2267 	list_add_tail(&cell->user_list, &tc->deferred_cells);  in thin_defer_cell()