336 	struct dm_cache_migration *mg;  member
361 	struct dm_cache_migration *mg;  in alloc_migration()  local
363 	mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);  in alloc_migration()
364 	if (mg) {  in alloc_migration()
365 		mg->cache = cache;  in alloc_migration()
366 		atomic_inc(&mg->cache->nr_allocated_migrations);  in alloc_migration()
369 	return mg;  in alloc_migration()
372 static void free_migration(struct dm_cache_migration *mg)  in free_migration()  argument
374 	if (atomic_dec_and_test(&mg->cache->nr_allocated_migrations))  in free_migration()
375 		wake_up(&mg->cache->migration_wait);  in free_migration()
377 	mempool_free(mg, mg->cache->migration_pool);  in free_migration()
382 	if (!p->mg) {  in prealloc_data_structs()
383 		p->mg = alloc_migration(cache);  in prealloc_data_structs()
384 		if (!p->mg)  in prealloc_data_structs()
411 	if (p->mg)  in prealloc_free_structs()
412 		free_migration(p->mg);  in prealloc_free_structs()
417 	struct dm_cache_migration *mg = p->mg;  in prealloc_get_migration()  local
419 	BUG_ON(!mg);  in prealloc_get_migration()
420 	p->mg = NULL;  in prealloc_get_migration()
422 	return mg;  in prealloc_get_migration()
914 static void free_io_migration(struct dm_cache_migration *mg)  in free_io_migration()  argument
916 	dec_io_migrations(mg->cache);  in free_io_migration()
917 	free_migration(mg);  in free_io_migration()
920 static void migration_failure(struct dm_cache_migration *mg)  in migration_failure()  argument
922 	struct cache *cache = mg->cache;  in migration_failure()
924 	if (mg->writeback) {  in migration_failure()
926 		set_dirty(cache, mg->old_oblock, mg->cblock);  in migration_failure()
927 		cell_defer(cache, mg->old_ocell, false);  in migration_failure()
929 	} else if (mg->demote) {  in migration_failure()
931 		policy_force_mapping(cache->policy, mg->new_oblock, mg->old_oblock);  in migration_failure()
933 		cell_defer(cache, mg->old_ocell, mg->promote ? false : true);  in migration_failure()
934 		if (mg->promote)  in migration_failure()
935 			cell_defer(cache, mg->new_ocell, true);  in migration_failure()
938 		policy_remove_mapping(cache->policy, mg->new_oblock);  in migration_failure()
939 		cell_defer(cache, mg->new_ocell, true);  in migration_failure()
942 	free_io_migration(mg);  in migration_failure()
945 static void migration_success_pre_commit(struct dm_cache_migration *mg)  in migration_success_pre_commit()  argument
948 	struct cache *cache = mg->cache;  in migration_success_pre_commit()
950 	if (mg->writeback) {  in migration_success_pre_commit()
951 		clear_dirty(cache, mg->old_oblock, mg->cblock);  in migration_success_pre_commit()
952 		cell_defer(cache, mg->old_ocell, false);  in migration_success_pre_commit()
953 		free_io_migration(mg);  in migration_success_pre_commit()
956 	} else if (mg->demote) {  in migration_success_pre_commit()
957 		if (dm_cache_remove_mapping(cache->cmd, mg->cblock)) {  in migration_success_pre_commit()
959 			policy_force_mapping(cache->policy, mg->new_oblock,  in migration_success_pre_commit()
960 					     mg->old_oblock);  in migration_success_pre_commit()
961 			if (mg->promote)  in migration_success_pre_commit()
962 				cell_defer(cache, mg->new_ocell, true);  in migration_success_pre_commit()
963 			free_io_migration(mg);  in migration_success_pre_commit()
967 		if (dm_cache_insert_mapping(cache->cmd, mg->cblock, mg->new_oblock)) {  in migration_success_pre_commit()
969 			policy_remove_mapping(cache->policy, mg->new_oblock);  in migration_success_pre_commit()
970 			free_io_migration(mg);  in migration_success_pre_commit()
976 	list_add_tail(&mg->list, &cache->need_commit_migrations);  in migration_success_pre_commit()
981 static void migration_success_post_commit(struct dm_cache_migration *mg)  in migration_success_post_commit()  argument
984 	struct cache *cache = mg->cache;  in migration_success_post_commit()
986 	if (mg->writeback) {  in migration_success_post_commit()
990 	} else if (mg->demote) {  in migration_success_post_commit()
991 		cell_defer(cache, mg->old_ocell, mg->promote ? false : true);  in migration_success_post_commit()
993 		if (mg->promote) {  in migration_success_post_commit()
994 			mg->demote = false;  in migration_success_post_commit()
997 			list_add_tail(&mg->list, &cache->quiesced_migrations);  in migration_success_post_commit()
1001 			if (mg->invalidate)  in migration_success_post_commit()
1002 				policy_remove_mapping(cache->policy, mg->old_oblock);  in migration_success_post_commit()
1003 			free_io_migration(mg);  in migration_success_post_commit()
1007 		if (mg->requeue_holder) {  in migration_success_post_commit()
1008 			clear_dirty(cache, mg->new_oblock, mg->cblock);  in migration_success_post_commit()
1009 			cell_defer(cache, mg->new_ocell, true);  in migration_success_post_commit()
1014 			set_dirty(cache, mg->new_oblock, mg->cblock);  in migration_success_post_commit()
1015 			bio_endio(mg->new_ocell->holder, 0);  in migration_success_post_commit()
1016 			cell_defer(cache, mg->new_ocell, false);  in migration_success_post_commit()
1018 		free_io_migration(mg);  in migration_success_post_commit()
1025 	struct dm_cache_migration *mg = (struct dm_cache_migration *) context;  in copy_complete()  local
1026 	struct cache *cache = mg->cache;  in copy_complete()
1029 		mg->err = true;  in copy_complete()
1032 	list_add_tail(&mg->list, &cache->completed_migrations);  in copy_complete()
1038 static void issue_copy(struct dm_cache_migration *mg)  in issue_copy()  argument
1042 	struct cache *cache = mg->cache;  in issue_copy()
1043 	sector_t cblock = from_cblock(mg->cblock);  in issue_copy()
1052 	if (mg->writeback || mg->demote) {  in issue_copy()
1054 		o_region.sector = from_oblock(mg->old_oblock) * cache->sectors_per_block;  in issue_copy()
1055 		r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, mg);  in issue_copy()
1058 		o_region.sector = from_oblock(mg->new_oblock) * cache->sectors_per_block;  in issue_copy()
1059 		r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, mg);  in issue_copy()
1064 		migration_failure(mg);  in issue_copy()
1070 	struct dm_cache_migration *mg = bio->bi_private;  in overwrite_endio()  local
1071 	struct cache *cache = mg->cache;  in overwrite_endio()
1079 		mg->err = true;  in overwrite_endio()
1081 	mg->requeue_holder = false;  in overwrite_endio()
1084 	list_add_tail(&mg->list, &cache->completed_migrations);  in overwrite_endio()
1090 static void issue_overwrite(struct dm_cache_migration *mg, struct bio *bio)  in issue_overwrite()  argument
1092 	size_t pb_data_size = get_per_bio_data_size(mg->cache);  in issue_overwrite()
1095 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);  in issue_overwrite()
1096 	remap_to_cache_dirty(mg->cache, bio, mg->new_oblock, mg->cblock);  in issue_overwrite()
1111 static void avoid_copy(struct dm_cache_migration *mg)  in avoid_copy()  argument
1113 	atomic_inc(&mg->cache->stats.copies_avoided);  in avoid_copy()
1114 	migration_success_pre_commit(mg);  in avoid_copy()
1131 static void issue_discard(struct dm_cache_migration *mg)  in issue_discard()  argument
1134 	struct bio *bio = mg->new_ocell->holder;  in issue_discard()
1136 	calc_discard_block_range(mg->cache, bio, &b, &e);  in issue_discard()
1138 		set_discard(mg->cache, b);  in issue_discard()
1143 	cell_defer(mg->cache, mg->new_ocell, false);  in issue_discard()
1144 	free_migration(mg);  in issue_discard()
1147 static void issue_copy_or_discard(struct dm_cache_migration *mg)  in issue_copy_or_discard()  argument
1150 	struct cache *cache = mg->cache;  in issue_copy_or_discard()
1152 	if (mg->discard) {  in issue_copy_or_discard()
1153 		issue_discard(mg);  in issue_copy_or_discard()
1157 	if (mg->writeback || mg->demote)  in issue_copy_or_discard()
1158 		avoid = !is_dirty(cache, mg->cblock) ||  in issue_copy_or_discard()
1159 			is_discarded_oblock(cache, mg->old_oblock);  in issue_copy_or_discard()
1161 		struct bio *bio = mg->new_ocell->holder;  in issue_copy_or_discard()
1163 		avoid = is_discarded_oblock(cache, mg->new_oblock);  in issue_copy_or_discard()
1167 			issue_overwrite(mg, bio);  in issue_copy_or_discard()
1172 	avoid ? avoid_copy(mg) : issue_copy(mg);  in issue_copy_or_discard()
1175 static void complete_migration(struct dm_cache_migration *mg)  in complete_migration()  argument
1177 	if (mg->err)  in complete_migration()
1178 		migration_failure(mg);  in complete_migration()
1180 		migration_success_pre_commit(mg);  in complete_migration()
1188 	struct dm_cache_migration *mg, *tmp;  in process_migrations()  local
1195 	list_for_each_entry_safe(mg, tmp, &list, list)  in process_migrations()
1196 		fn(mg);  in process_migrations()
1199 static void __queue_quiesced_migration(struct dm_cache_migration *mg)  in __queue_quiesced_migration()  argument
1201 	list_add_tail(&mg->list, &mg->cache->quiesced_migrations);  in __queue_quiesced_migration()
1204 static void queue_quiesced_migration(struct dm_cache_migration *mg)  in queue_quiesced_migration()  argument
1207 	struct cache *cache = mg->cache;  in queue_quiesced_migration()
1210 	__queue_quiesced_migration(mg);  in queue_quiesced_migration()
1219 	struct dm_cache_migration *mg, *tmp;  in queue_quiesced_migrations()  local
1222 	list_for_each_entry_safe(mg, tmp, work, list)  in queue_quiesced_migrations()
1223 		__queue_quiesced_migration(mg);  in queue_quiesced_migrations()
1244 static void quiesce_migration(struct dm_cache_migration *mg)  in quiesce_migration()  argument
1246 	if (!dm_deferred_set_add_work(mg->cache->all_io_ds, &mg->list))  in quiesce_migration()
1247 		queue_quiesced_migration(mg);  in quiesce_migration()
1254 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in promote()  local
1256 	mg->err = false;  in promote()
1257 	mg->discard = false;  in promote()
1258 	mg->writeback = false;  in promote()
1259 	mg->demote = false;  in promote()
1260 	mg->promote = true;  in promote()
1261 	mg->requeue_holder = true;  in promote()
1262 	mg->invalidate = false;  in promote()
1263 	mg->cache = cache;  in promote()
1264 	mg->new_oblock = oblock;  in promote()
1265 	mg->cblock = cblock;  in promote()
1266 	mg->old_ocell = NULL;  in promote()
1267 	mg->new_ocell = cell;  in promote()
1268 	mg->start_jiffies = jiffies;  in promote()
1271 	quiesce_migration(mg);  in promote()
1278 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in writeback()  local
1280 	mg->err = false;  in writeback()
1281 	mg->discard = false;  in writeback()
1282 	mg->writeback = true;  in writeback()
1283 	mg->demote = false;  in writeback()
1284 	mg->promote = false;  in writeback()
1285 	mg->requeue_holder = true;  in writeback()
1286 	mg->invalidate = false;  in writeback()
1287 	mg->cache = cache;  in writeback()
1288 	mg->old_oblock = oblock;  in writeback()
1289 	mg->cblock = cblock;  in writeback()
1290 	mg->old_ocell = cell;  in writeback()
1291 	mg->new_ocell = NULL;  in writeback()
1292 	mg->start_jiffies = jiffies;  in writeback()
1295 	quiesce_migration(mg);  in writeback()
1304 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in demote_then_promote()  local
1306 	mg->err = false;  in demote_then_promote()
1307 	mg->discard = false;  in demote_then_promote()
1308 	mg->writeback = false;  in demote_then_promote()
1309 	mg->demote = true;  in demote_then_promote()
1310 	mg->promote = true;  in demote_then_promote()
1311 	mg->requeue_holder = true;  in demote_then_promote()
1312 	mg->invalidate = false;  in demote_then_promote()
1313 	mg->cache = cache;  in demote_then_promote()
1314 	mg->old_oblock = old_oblock;  in demote_then_promote()
1315 	mg->new_oblock = new_oblock;  in demote_then_promote()
1316 	mg->cblock = cblock;  in demote_then_promote()
1317 	mg->old_ocell = old_ocell;  in demote_then_promote()
1318 	mg->new_ocell = new_ocell;  in demote_then_promote()
1319 	mg->start_jiffies = jiffies;  in demote_then_promote()
1322 	quiesce_migration(mg);  in demote_then_promote()
1333 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in invalidate()  local
1335 	mg->err = false;  in invalidate()
1336 	mg->discard = false;  in invalidate()
1337 	mg->writeback = false;  in invalidate()
1338 	mg->demote = true;  in invalidate()
1339 	mg->promote = false;  in invalidate()
1340 	mg->requeue_holder = true;  in invalidate()
1341 	mg->invalidate = true;  in invalidate()
1342 	mg->cache = cache;  in invalidate()
1343 	mg->old_oblock = oblock;  in invalidate()
1344 	mg->cblock = cblock;  in invalidate()
1345 	mg->old_ocell = cell;  in invalidate()
1346 	mg->new_ocell = NULL;  in invalidate()
1347 	mg->start_jiffies = jiffies;  in invalidate()
1350 	quiesce_migration(mg);  in invalidate()
1356 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in discard()  local
1358 	mg->err = false;  in discard()
1359 	mg->discard = true;  in discard()
1360 	mg->writeback = false;  in discard()
1361 	mg->demote = false;  in discard()
1362 	mg->promote = false;  in discard()
1363 	mg->requeue_holder = false;  in discard()
1364 	mg->invalidate = false;  in discard()
1365 	mg->cache = cache;  in discard()
1366 	mg->old_ocell = NULL;  in discard()
1367 	mg->new_ocell = cell;  in discard()
1368 	mg->start_jiffies = jiffies;  in discard()
1370 	quiesce_migration(mg);  in discard()