206 	struct dm_pool_metadata *pmd;  member
362 static int superblock_lock_zero(struct dm_pool_metadata *pmd,  in superblock_lock_zero()  argument
365 	return dm_bm_write_lock_zero(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in superblock_lock_zero()
369 static int superblock_lock(struct dm_pool_metadata *pmd,  in superblock_lock()  argument
372 	return dm_bm_write_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in superblock_lock()
403 static void __setup_btree_details(struct dm_pool_metadata *pmd)  in __setup_btree_details()  argument
405 	pmd->info.tm = pmd->tm;  in __setup_btree_details()
406 	pmd->info.levels = 2;  in __setup_btree_details()
407 	pmd->info.value_type.context = pmd->data_sm;  in __setup_btree_details()
408 	pmd->info.value_type.size = sizeof(__le64);  in __setup_btree_details()
409 	pmd->info.value_type.inc = data_block_inc;  in __setup_btree_details()
410 	pmd->info.value_type.dec = data_block_dec;  in __setup_btree_details()
411 	pmd->info.value_type.equal = data_block_equal;  in __setup_btree_details()
413 	memcpy(&pmd->nb_info, &pmd->info, sizeof(pmd->nb_info));  in __setup_btree_details()
414 	pmd->nb_info.tm = pmd->nb_tm;  in __setup_btree_details()
416 	pmd->tl_info.tm = pmd->tm;  in __setup_btree_details()
417 	pmd->tl_info.levels = 1;  in __setup_btree_details()
418 	pmd->tl_info.value_type.context = &pmd->bl_info;  in __setup_btree_details()
419 	pmd->tl_info.value_type.size = sizeof(__le64);  in __setup_btree_details()
420 	pmd->tl_info.value_type.inc = subtree_inc;  in __setup_btree_details()
421 	pmd->tl_info.value_type.dec = subtree_dec;  in __setup_btree_details()
422 	pmd->tl_info.value_type.equal = subtree_equal;  in __setup_btree_details()
424 	pmd->bl_info.tm = pmd->tm;  in __setup_btree_details()
425 	pmd->bl_info.levels = 1;  in __setup_btree_details()
426 	pmd->bl_info.value_type.context = pmd->data_sm;  in __setup_btree_details()
427 	pmd->bl_info.value_type.size = sizeof(__le64);  in __setup_btree_details()
428 	pmd->bl_info.value_type.inc = data_block_inc;  in __setup_btree_details()
429 	pmd->bl_info.value_type.dec = data_block_dec;  in __setup_btree_details()
430 	pmd->bl_info.value_type.equal = data_block_equal;  in __setup_btree_details()
432 	pmd->details_info.tm = pmd->tm;  in __setup_btree_details()
433 	pmd->details_info.levels = 1;  in __setup_btree_details()
434 	pmd->details_info.value_type.context = NULL;  in __setup_btree_details()
435 	pmd->details_info.value_type.size = sizeof(struct disk_device_details);  in __setup_btree_details()
436 	pmd->details_info.value_type.inc = NULL;  in __setup_btree_details()
437 	pmd->details_info.value_type.dec = NULL;  in __setup_btree_details()
438 	pmd->details_info.value_type.equal = NULL;  in __setup_btree_details()
441 static int save_sm_roots(struct dm_pool_metadata *pmd)  in save_sm_roots()  argument
446 	r = dm_sm_root_size(pmd->metadata_sm, &len);  in save_sm_roots()
450 	r = dm_sm_copy_root(pmd->metadata_sm, &pmd->metadata_space_map_root, len);  in save_sm_roots()
454 	r = dm_sm_root_size(pmd->data_sm, &len);  in save_sm_roots()
458 	return dm_sm_copy_root(pmd->data_sm, &pmd->data_space_map_root, len);  in save_sm_roots()
461 static void copy_sm_roots(struct dm_pool_metadata *pmd,  in copy_sm_roots()  argument
465 	       &pmd->metadata_space_map_root,  in copy_sm_roots()
466 	       sizeof(pmd->metadata_space_map_root));  in copy_sm_roots()
469 	       &pmd->data_space_map_root,  in copy_sm_roots()
470 	       sizeof(pmd->data_space_map_root));  in copy_sm_roots()
473 static int __write_initial_superblock(struct dm_pool_metadata *pmd)  in __write_initial_superblock()  argument
478 	sector_t bdev_size = i_size_read(pmd->bdev->bd_inode) >> SECTOR_SHIFT;  in __write_initial_superblock()
483 	r = dm_sm_commit(pmd->data_sm);  in __write_initial_superblock()
487 	r = save_sm_roots(pmd);  in __write_initial_superblock()
491 	r = dm_tm_pre_commit(pmd->tm);  in __write_initial_superblock()
495 	r = superblock_lock_zero(pmd, &sblock);  in __write_initial_superblock()
508 	copy_sm_roots(pmd, disk_super);  in __write_initial_superblock()
510 	disk_super->data_mapping_root = cpu_to_le64(pmd->root);  in __write_initial_superblock()
511 	disk_super->device_details_root = cpu_to_le64(pmd->details_root);  in __write_initial_superblock()
514 	disk_super->data_block_size = cpu_to_le32(pmd->data_block_size);  in __write_initial_superblock()
516 	return dm_tm_commit(pmd->tm, sblock);  in __write_initial_superblock()
519 static int __format_metadata(struct dm_pool_metadata *pmd)  in __format_metadata()  argument
523 	r = dm_tm_create_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __format_metadata()
524 				 &pmd->tm, &pmd->metadata_sm);  in __format_metadata()
530 	pmd->data_sm = dm_sm_disk_create(pmd->tm, 0);  in __format_metadata()
531 	if (IS_ERR(pmd->data_sm)) {  in __format_metadata()
533 		r = PTR_ERR(pmd->data_sm);  in __format_metadata()
537 	pmd->nb_tm = dm_tm_create_non_blocking_clone(pmd->tm);  in __format_metadata()
538 	if (!pmd->nb_tm) {  in __format_metadata()
544 	__setup_btree_details(pmd);  in __format_metadata()
546 	r = dm_btree_empty(&pmd->info, &pmd->root);  in __format_metadata()
550 	r = dm_btree_empty(&pmd->details_info, &pmd->details_root);  in __format_metadata()
556 	r = __write_initial_superblock(pmd);  in __format_metadata()
563 	dm_tm_destroy(pmd->nb_tm);  in __format_metadata()
565 	dm_sm_destroy(pmd->data_sm);  in __format_metadata()
567 	dm_tm_destroy(pmd->tm);  in __format_metadata()
568 	dm_sm_destroy(pmd->metadata_sm);  in __format_metadata()
574 				     struct dm_pool_metadata *pmd)  in __check_incompat_features()  argument
588 	if (get_disk_ro(pmd->bdev->bd_disk))  in __check_incompat_features()
601 static int __open_metadata(struct dm_pool_metadata *pmd)  in __open_metadata()  argument
607 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
617 	if (le32_to_cpu(disk_super->data_block_size) != pmd->data_block_size) {  in __open_metadata()
620 		      (unsigned long long)pmd->data_block_size);  in __open_metadata()
625 	r = __check_incompat_features(disk_super, pmd);  in __open_metadata()
629 	r = dm_tm_open_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __open_metadata()
632 			       &pmd->tm, &pmd->metadata_sm);  in __open_metadata()
638 	pmd->data_sm = dm_sm_disk_open(pmd->tm, disk_super->data_space_map_root,  in __open_metadata()
640 	if (IS_ERR(pmd->data_sm)) {  in __open_metadata()
642 		r = PTR_ERR(pmd->data_sm);  in __open_metadata()
646 	pmd->nb_tm = dm_tm_create_non_blocking_clone(pmd->tm);  in __open_metadata()
647 	if (!pmd->nb_tm) {  in __open_metadata()
653 	__setup_btree_details(pmd);  in __open_metadata()
657 	dm_sm_destroy(pmd->data_sm);  in __open_metadata()
659 	dm_tm_destroy(pmd->tm);  in __open_metadata()
660 	dm_sm_destroy(pmd->metadata_sm);  in __open_metadata()
667 static int __open_or_format_metadata(struct dm_pool_metadata *pmd, bool format_device)  in __open_or_format_metadata()  argument
671 	r = __superblock_all_zeroes(pmd->bm, &unformatted);  in __open_or_format_metadata()
676 		return format_device ? __format_metadata(pmd) : -EPERM;  in __open_or_format_metadata()
678 	return __open_metadata(pmd);  in __open_or_format_metadata()
681 static int __create_persistent_data_objects(struct dm_pool_metadata *pmd, bool format_device)  in __create_persistent_data_objects()  argument
685 	pmd->bm = dm_block_manager_create(pmd->bdev, THIN_METADATA_BLOCK_SIZE << SECTOR_SHIFT,  in __create_persistent_data_objects()
688 	if (IS_ERR(pmd->bm)) {  in __create_persistent_data_objects()
690 		return PTR_ERR(pmd->bm);  in __create_persistent_data_objects()
693 	r = __open_or_format_metadata(pmd, format_device);  in __create_persistent_data_objects()
695 		dm_block_manager_destroy(pmd->bm);  in __create_persistent_data_objects()
700 static void __destroy_persistent_data_objects(struct dm_pool_metadata *pmd)  in __destroy_persistent_data_objects()  argument
702 	dm_sm_destroy(pmd->data_sm);  in __destroy_persistent_data_objects()
703 	dm_sm_destroy(pmd->metadata_sm);  in __destroy_persistent_data_objects()
704 	dm_tm_destroy(pmd->nb_tm);  in __destroy_persistent_data_objects()
705 	dm_tm_destroy(pmd->tm);  in __destroy_persistent_data_objects()
706 	dm_block_manager_destroy(pmd->bm);  in __destroy_persistent_data_objects()
709 static int __begin_transaction(struct dm_pool_metadata *pmd)  in __begin_transaction()  argument
719 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __begin_transaction()
725 	pmd->time = le32_to_cpu(disk_super->time);  in __begin_transaction()
726 	pmd->root = le64_to_cpu(disk_super->data_mapping_root);  in __begin_transaction()
727 	pmd->details_root = le64_to_cpu(disk_super->device_details_root);  in __begin_transaction()
728 	pmd->trans_id = le64_to_cpu(disk_super->trans_id);  in __begin_transaction()
729 	pmd->flags = le32_to_cpu(disk_super->flags);  in __begin_transaction()
730 	pmd->data_block_size = le32_to_cpu(disk_super->data_block_size);  in __begin_transaction()
736 static int __write_changed_details(struct dm_pool_metadata *pmd)  in __write_changed_details()  argument
743 	list_for_each_entry_safe(td, tmp, &pmd->thin_devices, list) {  in __write_changed_details()
755 		r = dm_btree_insert(&pmd->details_info, pmd->details_root,  in __write_changed_details()
756 				    &key, &details, &pmd->details_root);  in __write_changed_details()
771 static int __commit_transaction(struct dm_pool_metadata *pmd)  in __commit_transaction()  argument
783 	r = __write_changed_details(pmd);  in __commit_transaction()
787 	r = dm_sm_commit(pmd->data_sm);  in __commit_transaction()
791 	r = dm_tm_pre_commit(pmd->tm);  in __commit_transaction()
795 	r = dm_sm_root_size(pmd->metadata_sm, &metadata_len);  in __commit_transaction()
799 	r = dm_sm_root_size(pmd->data_sm, &data_len);  in __commit_transaction()
803 	r = save_sm_roots(pmd);  in __commit_transaction()
807 	r = superblock_lock(pmd, &sblock);  in __commit_transaction()
812 	disk_super->time = cpu_to_le32(pmd->time);  in __commit_transaction()
813 	disk_super->data_mapping_root = cpu_to_le64(pmd->root);  in __commit_transaction()
814 	disk_super->device_details_root = cpu_to_le64(pmd->details_root);  in __commit_transaction()
815 	disk_super->trans_id = cpu_to_le64(pmd->trans_id);  in __commit_transaction()
816 	disk_super->flags = cpu_to_le32(pmd->flags);  in __commit_transaction()
818 	copy_sm_roots(pmd, disk_super);  in __commit_transaction()
820 	return dm_tm_commit(pmd->tm, sblock);  in __commit_transaction()
828 	struct dm_pool_metadata *pmd;  in dm_pool_metadata_open()  local
830 	pmd = kmalloc(sizeof(*pmd), GFP_KERNEL);  in dm_pool_metadata_open()
831 	if (!pmd) {  in dm_pool_metadata_open()
836 	init_rwsem(&pmd->root_lock);  in dm_pool_metadata_open()
837 	pmd->time = 0;  in dm_pool_metadata_open()
838 	INIT_LIST_HEAD(&pmd->thin_devices);  in dm_pool_metadata_open()
839 	pmd->read_only = false;  in dm_pool_metadata_open()
840 	pmd->fail_io = false;  in dm_pool_metadata_open()
841 	pmd->bdev = bdev;  in dm_pool_metadata_open()
842 	pmd->data_block_size = data_block_size;  in dm_pool_metadata_open()
844 	r = __create_persistent_data_objects(pmd, format_device);  in dm_pool_metadata_open()
846 		kfree(pmd);  in dm_pool_metadata_open()
850 	r = __begin_transaction(pmd);  in dm_pool_metadata_open()
852 		if (dm_pool_metadata_close(pmd) < 0)  in dm_pool_metadata_open()
857 	return pmd;  in dm_pool_metadata_open()
860 int dm_pool_metadata_close(struct dm_pool_metadata *pmd)  in dm_pool_metadata_close()  argument
866 	down_read(&pmd->root_lock);  in dm_pool_metadata_close()
867 	list_for_each_entry_safe(td, tmp, &pmd->thin_devices, list) {  in dm_pool_metadata_close()
875 	up_read(&pmd->root_lock);  in dm_pool_metadata_close()
883 	if (!pmd->read_only && !pmd->fail_io) {  in dm_pool_metadata_close()
884 		r = __commit_transaction(pmd);  in dm_pool_metadata_close()
890 	if (!pmd->fail_io)  in dm_pool_metadata_close()
891 		__destroy_persistent_data_objects(pmd);  in dm_pool_metadata_close()
893 	kfree(pmd);  in dm_pool_metadata_close()
902 static int __open_device(struct dm_pool_metadata *pmd,  in __open_device()  argument
914 	list_for_each_entry(td2, &pmd->thin_devices, list)  in __open_device()
930 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __open_device()
941 		details_le.transaction_id = cpu_to_le64(pmd->trans_id);  in __open_device()
942 		details_le.creation_time = cpu_to_le32(pmd->time);  in __open_device()
943 		details_le.snapshotted_time = cpu_to_le32(pmd->time);  in __open_device()
950 	(*td)->pmd = pmd;  in __open_device()
960 	list_add(&(*td)->list, &pmd->thin_devices);  in __open_device()
970 static int __create_thin(struct dm_pool_metadata *pmd,  in __create_thin()  argument
980 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_thin()
988 	r = dm_btree_empty(&pmd->bl_info, &dev_root);  in __create_thin()
997 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_thin()
999 		dm_btree_del(&pmd->bl_info, dev_root);  in __create_thin()
1003 	r = __open_device(pmd, dev, 1, &td);  in __create_thin()
1005 		dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __create_thin()
1006 		dm_btree_del(&pmd->bl_info, dev_root);  in __create_thin()
1014 int dm_pool_create_thin(struct dm_pool_metadata *pmd, dm_thin_id dev)  in dm_pool_create_thin()  argument
1018 	down_write(&pmd->root_lock);  in dm_pool_create_thin()
1019 	if (!pmd->fail_io)  in dm_pool_create_thin()
1020 		r = __create_thin(pmd, dev);  in dm_pool_create_thin()
1021 	up_write(&pmd->root_lock);  in dm_pool_create_thin()
1026 static int __set_snapshot_details(struct dm_pool_metadata *pmd,  in __set_snapshot_details()  argument
1033 	r = __open_device(pmd, origin, 0, &td);  in __set_snapshot_details()
1047 static int __create_snap(struct dm_pool_metadata *pmd,  in __create_snap()  argument
1058 	r = dm_btree_lookup(&pmd->details_info, pmd->details_root,  in __create_snap()
1064 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &value);  in __create_snap()
1070 	dm_tm_inc(pmd->tm, origin_root);  in __create_snap()
1076 	r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);  in __create_snap()
1078 		dm_tm_dec(pmd->tm, origin_root);  in __create_snap()
1082 	pmd->time++;  in __create_snap()
1084 	r = __open_device(pmd, dev, 1, &td);  in __create_snap()
1088 	r = __set_snapshot_details(pmd, td, origin, pmd->time);  in __create_snap()
1097 	dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __create_snap()
1098 	dm_btree_remove(&pmd->details_info, pmd->details_root,  in __create_snap()
1099 			&key, &pmd->details_root);  in __create_snap()
1103 int dm_pool_create_snap(struct dm_pool_metadata *pmd,  in dm_pool_create_snap()  argument
1109 	down_write(&pmd->root_lock);  in dm_pool_create_snap()
1110 	if (!pmd->fail_io)  in dm_pool_create_snap()
1111 		r = __create_snap(pmd, dev, origin);  in dm_pool_create_snap()
1112 	up_write(&pmd->root_lock);  in dm_pool_create_snap()
1117 static int __delete_device(struct dm_pool_metadata *pmd, dm_thin_id dev)  in __delete_device()  argument
1124 	r = __open_device(pmd, dev, 0, &td);  in __delete_device()
1135 	r = dm_btree_remove(&pmd->details_info, pmd->details_root,  in __delete_device()
1136 			    &key, &pmd->details_root);  in __delete_device()
1140 	r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);  in __delete_device()
1147 int dm_pool_delete_thin_device(struct dm_pool_metadata *pmd,  in dm_pool_delete_thin_device()  argument
1152 	down_write(&pmd->root_lock);  in dm_pool_delete_thin_device()
1153 	if (!pmd->fail_io)  in dm_pool_delete_thin_device()
1154 		r = __delete_device(pmd, dev);  in dm_pool_delete_thin_device()
1155 	up_write(&pmd->root_lock);  in dm_pool_delete_thin_device()
1160 int dm_pool_set_metadata_transaction_id(struct dm_pool_metadata *pmd,  in dm_pool_set_metadata_transaction_id()  argument
1166 	down_write(&pmd->root_lock);  in dm_pool_set_metadata_transaction_id()
1168 	if (pmd->fail_io)  in dm_pool_set_metadata_transaction_id()
1171 	if (pmd->trans_id != current_id) {  in dm_pool_set_metadata_transaction_id()
1176 	pmd->trans_id = new_id;  in dm_pool_set_metadata_transaction_id()
1180 	up_write(&pmd->root_lock);  in dm_pool_set_metadata_transaction_id()
1185 int dm_pool_get_metadata_transaction_id(struct dm_pool_metadata *pmd,  in dm_pool_get_metadata_transaction_id()  argument
1190 	down_read(&pmd->root_lock);  in dm_pool_get_metadata_transaction_id()
1191 	if (!pmd->fail_io) {  in dm_pool_get_metadata_transaction_id()
1192 		*result = pmd->trans_id;  in dm_pool_get_metadata_transaction_id()
1195 	up_read(&pmd->root_lock);  in dm_pool_get_metadata_transaction_id()
1200 static int __reserve_metadata_snap(struct dm_pool_metadata *pmd)  in __reserve_metadata_snap()  argument
1210 	dm_sm_inc_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION);  in __reserve_metadata_snap()
1211 	r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION,  in __reserve_metadata_snap()
1224 		dm_tm_dec(pmd->tm, held_root);  in __reserve_metadata_snap()
1225 		dm_tm_unlock(pmd->tm, copy);  in __reserve_metadata_snap()
1240 	dm_tm_inc(pmd->tm, le64_to_cpu(disk_super->data_mapping_root));  in __reserve_metadata_snap()
1241 	dm_tm_inc(pmd->tm, le64_to_cpu(disk_super->device_details_root));  in __reserve_metadata_snap()
1242 	dm_tm_unlock(pmd->tm, copy);  in __reserve_metadata_snap()
1247 	r = superblock_lock(pmd, &sblock);  in __reserve_metadata_snap()
1249 		dm_tm_dec(pmd->tm, held_root);  in __reserve_metadata_snap()
1259 int dm_pool_reserve_metadata_snap(struct dm_pool_metadata *pmd)  in dm_pool_reserve_metadata_snap()  argument
1263 	down_write(&pmd->root_lock);  in dm_pool_reserve_metadata_snap()
1264 	if (!pmd->fail_io)  in dm_pool_reserve_metadata_snap()
1265 		r = __reserve_metadata_snap(pmd);  in dm_pool_reserve_metadata_snap()
1266 	up_write(&pmd->root_lock);  in dm_pool_reserve_metadata_snap()
1271 static int __release_metadata_snap(struct dm_pool_metadata *pmd)  in __release_metadata_snap()  argument
1278 	r = superblock_lock(pmd, &sblock);  in __release_metadata_snap()
1293 	r = dm_tm_read_lock(pmd->tm, held_root, &sb_validator, &copy);  in __release_metadata_snap()
1298 	dm_btree_del(&pmd->info, le64_to_cpu(disk_super->data_mapping_root));  in __release_metadata_snap()
1299 	dm_btree_del(&pmd->details_info, le64_to_cpu(disk_super->device_details_root));  in __release_metadata_snap()
1300 	dm_sm_dec_block(pmd->metadata_sm, held_root);  in __release_metadata_snap()
1302 	return dm_tm_unlock(pmd->tm, copy);  in __release_metadata_snap()
1305 int dm_pool_release_metadata_snap(struct dm_pool_metadata *pmd)  in dm_pool_release_metadata_snap()  argument
1309 	down_write(&pmd->root_lock);  in dm_pool_release_metadata_snap()
1310 	if (!pmd->fail_io)  in dm_pool_release_metadata_snap()
1311 		r = __release_metadata_snap(pmd);  in dm_pool_release_metadata_snap()
1312 	up_write(&pmd->root_lock);  in dm_pool_release_metadata_snap()
1317 static int __get_metadata_snap(struct dm_pool_metadata *pmd,  in __get_metadata_snap()  argument
1324 	r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,  in __get_metadata_snap()
1335 int dm_pool_get_metadata_snap(struct dm_pool_metadata *pmd,  in dm_pool_get_metadata_snap()  argument
1340 	down_read(&pmd->root_lock);  in dm_pool_get_metadata_snap()
1341 	if (!pmd->fail_io)  in dm_pool_get_metadata_snap()
1342 		r = __get_metadata_snap(pmd, result);  in dm_pool_get_metadata_snap()
1343 	up_read(&pmd->root_lock);  in dm_pool_get_metadata_snap()
1348 int dm_pool_open_thin_device(struct dm_pool_metadata *pmd, dm_thin_id dev,  in dm_pool_open_thin_device()  argument
1353 	down_write(&pmd->root_lock);  in dm_pool_open_thin_device()
1354 	if (!pmd->fail_io)  in dm_pool_open_thin_device()
1355 		r = __open_device(pmd, dev, 0, td);  in dm_pool_open_thin_device()
1356 	up_write(&pmd->root_lock);  in dm_pool_open_thin_device()
1363 	down_write(&td->pmd->root_lock);  in dm_pool_close_thin_device()
1365 	up_write(&td->pmd->root_lock);  in dm_pool_close_thin_device()
1391 	struct dm_pool_metadata *pmd = td->pmd;  in dm_thin_find_block()  local
1395 	if (pmd->fail_io)  in dm_thin_find_block()
1398 	down_read(&pmd->root_lock);  in dm_thin_find_block()
1401 		info = &pmd->info;  in dm_thin_find_block()
1403 		info = &pmd->nb_info;  in dm_thin_find_block()
1405 	r = dm_btree_lookup(info, pmd->root, keys, &value);  in dm_thin_find_block()
1418 	up_read(&pmd->root_lock);  in dm_thin_find_block()
1427 	struct dm_pool_metadata *pmd = td->pmd;  in __insert()  local
1430 	value = cpu_to_le64(pack_block_time(data_block, pmd->time));  in __insert()
1433 	r = dm_btree_insert_notify(&pmd->info, pmd->root, keys, &value,  in __insert()
1434 				   &pmd->root, &inserted);  in __insert()
1450 	down_write(&td->pmd->root_lock);  in dm_thin_insert_block()
1451 	if (!td->pmd->fail_io)  in dm_thin_insert_block()
1453 	up_write(&td->pmd->root_lock);  in dm_thin_insert_block()
1461 	struct dm_pool_metadata *pmd = td->pmd;  in __remove()  local
1464 	r = dm_btree_remove(&pmd->info, pmd->root, keys, &pmd->root);  in __remove()
1478 	down_write(&td->pmd->root_lock);  in dm_thin_remove_block()
1479 	if (!td->pmd->fail_io)  in dm_thin_remove_block()
1481 	up_write(&td->pmd->root_lock);  in dm_thin_remove_block()
1486 int dm_pool_block_is_used(struct dm_pool_metadata *pmd, dm_block_t b, bool *result)  in dm_pool_block_is_used()  argument
1491 	down_read(&pmd->root_lock);  in dm_pool_block_is_used()
1492 	r = dm_sm_get_count(pmd->data_sm, b, &ref_count);  in dm_pool_block_is_used()
1495 	up_read(&pmd->root_lock);  in dm_pool_block_is_used()
1504 	down_read(&td->pmd->root_lock);  in dm_thin_changed_this_transaction()
1506 	up_read(&td->pmd->root_lock);  in dm_thin_changed_this_transaction()
1511 bool dm_pool_changed_this_transaction(struct dm_pool_metadata *pmd)  in dm_pool_changed_this_transaction()  argument
1516 	down_read(&pmd->root_lock);  in dm_pool_changed_this_transaction()
1517 	list_for_each_entry_safe(td, tmp, &pmd->thin_devices, list) {  in dm_pool_changed_this_transaction()
1523 	up_read(&pmd->root_lock);  in dm_pool_changed_this_transaction()
1532 	down_read(&td->pmd->root_lock);  in dm_thin_aborted_changes()
1534 	up_read(&td->pmd->root_lock);  in dm_thin_aborted_changes()
1539 int dm_pool_alloc_data_block(struct dm_pool_metadata *pmd, dm_block_t *result)  in dm_pool_alloc_data_block()  argument
1543 	down_write(&pmd->root_lock);  in dm_pool_alloc_data_block()
1544 	if (!pmd->fail_io)  in dm_pool_alloc_data_block()
1545 		r = dm_sm_new_block(pmd->data_sm, result);  in dm_pool_alloc_data_block()
1546 	up_write(&pmd->root_lock);  in dm_pool_alloc_data_block()
1551 int dm_pool_commit_metadata(struct dm_pool_metadata *pmd)  in dm_pool_commit_metadata()  argument
1555 	down_write(&pmd->root_lock);  in dm_pool_commit_metadata()
1556 	if (pmd->fail_io)  in dm_pool_commit_metadata()
1559 	r = __commit_transaction(pmd);  in dm_pool_commit_metadata()
1566 	r = __begin_transaction(pmd);  in dm_pool_commit_metadata()
1568 	up_write(&pmd->root_lock);  in dm_pool_commit_metadata()
1572 static void __set_abort_with_changes_flags(struct dm_pool_metadata *pmd)  in __set_abort_with_changes_flags()  argument
1576 	list_for_each_entry(td, &pmd->thin_devices, list)  in __set_abort_with_changes_flags()
1580 int dm_pool_abort_metadata(struct dm_pool_metadata *pmd)  in dm_pool_abort_metadata()  argument
1584 	down_write(&pmd->root_lock);  in dm_pool_abort_metadata()
1585 	if (pmd->fail_io)  in dm_pool_abort_metadata()
1588 	__set_abort_with_changes_flags(pmd);  in dm_pool_abort_metadata()
1589 	__destroy_persistent_data_objects(pmd);  in dm_pool_abort_metadata()
1590 	r = __create_persistent_data_objects(pmd, false);  in dm_pool_abort_metadata()
1592 		pmd->fail_io = true;  in dm_pool_abort_metadata()
1595 	up_write(&pmd->root_lock);  in dm_pool_abort_metadata()
1600 int dm_pool_get_free_block_count(struct dm_pool_metadata *pmd, dm_block_t *result)  in dm_pool_get_free_block_count()  argument
1604 	down_read(&pmd->root_lock);  in dm_pool_get_free_block_count()
1605 	if (!pmd->fail_io)  in dm_pool_get_free_block_count()
1606 		r = dm_sm_get_nr_free(pmd->data_sm, result);  in dm_pool_get_free_block_count()
1607 	up_read(&pmd->root_lock);  in dm_pool_get_free_block_count()
1612 int dm_pool_get_free_metadata_block_count(struct dm_pool_metadata *pmd,  in dm_pool_get_free_metadata_block_count()  argument
1617 	down_read(&pmd->root_lock);  in dm_pool_get_free_metadata_block_count()
1618 	if (!pmd->fail_io)  in dm_pool_get_free_metadata_block_count()
1619 		r = dm_sm_get_nr_free(pmd->metadata_sm, result);  in dm_pool_get_free_metadata_block_count()
1620 	up_read(&pmd->root_lock);  in dm_pool_get_free_metadata_block_count()
1625 int dm_pool_get_metadata_dev_size(struct dm_pool_metadata *pmd,  in dm_pool_get_metadata_dev_size()  argument
1630 	down_read(&pmd->root_lock);  in dm_pool_get_metadata_dev_size()
1631 	if (!pmd->fail_io)  in dm_pool_get_metadata_dev_size()
1632 		r = dm_sm_get_nr_blocks(pmd->metadata_sm, result);  in dm_pool_get_metadata_dev_size()
1633 	up_read(&pmd->root_lock);  in dm_pool_get_metadata_dev_size()
1638 int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result)  in dm_pool_get_data_dev_size()  argument
1642 	down_read(&pmd->root_lock);  in dm_pool_get_data_dev_size()
1643 	if (!pmd->fail_io)  in dm_pool_get_data_dev_size()
1644 		r = dm_sm_get_nr_blocks(pmd->data_sm, result);  in dm_pool_get_data_dev_size()
1645 	up_read(&pmd->root_lock);  in dm_pool_get_data_dev_size()
1653 	struct dm_pool_metadata *pmd = td->pmd;  in dm_thin_get_mapped_count()  local
1655 	down_read(&pmd->root_lock);  in dm_thin_get_mapped_count()
1656 	if (!pmd->fail_io) {  in dm_thin_get_mapped_count()
1660 	up_read(&pmd->root_lock);  in dm_thin_get_mapped_count()
1670 	struct dm_pool_metadata *pmd = td->pmd;  in __highest_block()  local
1672 	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &td->id, &value_le);  in __highest_block()
1678 	return dm_btree_find_highest_key(&pmd->bl_info, thin_root, result);  in __highest_block()
1685 	struct dm_pool_metadata *pmd = td->pmd;  in dm_thin_get_highest_mapped_block()  local
1687 	down_read(&pmd->root_lock);  in dm_thin_get_highest_mapped_block()
1688 	if (!pmd->fail_io)  in dm_thin_get_highest_mapped_block()
1690 	up_read(&pmd->root_lock);  in dm_thin_get_highest_mapped_block()
1715 int dm_pool_resize_data_dev(struct dm_pool_metadata *pmd, dm_block_t new_count)  in dm_pool_resize_data_dev()  argument
1719 	down_write(&pmd->root_lock);  in dm_pool_resize_data_dev()
1720 	if (!pmd->fail_io)  in dm_pool_resize_data_dev()
1721 		r = __resize_space_map(pmd->data_sm, new_count);  in dm_pool_resize_data_dev()
1722 	up_write(&pmd->root_lock);  in dm_pool_resize_data_dev()
1727 int dm_pool_resize_metadata_dev(struct dm_pool_metadata *pmd, dm_block_t new_count)  in dm_pool_resize_metadata_dev()  argument
1731 	down_write(&pmd->root_lock);  in dm_pool_resize_metadata_dev()
1732 	if (!pmd->fail_io)  in dm_pool_resize_metadata_dev()
1733 		r = __resize_space_map(pmd->metadata_sm, new_count);  in dm_pool_resize_metadata_dev()
1734 	up_write(&pmd->root_lock);  in dm_pool_resize_metadata_dev()
1739 void dm_pool_metadata_read_only(struct dm_pool_metadata *pmd)  in dm_pool_metadata_read_only()  argument
1741 	down_write(&pmd->root_lock);  in dm_pool_metadata_read_only()
1742 	pmd->read_only = true;  in dm_pool_metadata_read_only()
1743 	dm_bm_set_read_only(pmd->bm);  in dm_pool_metadata_read_only()
1744 	up_write(&pmd->root_lock);  in dm_pool_metadata_read_only()
1747 void dm_pool_metadata_read_write(struct dm_pool_metadata *pmd)  in dm_pool_metadata_read_write()  argument
1749 	down_write(&pmd->root_lock);  in dm_pool_metadata_read_write()
1750 	pmd->read_only = false;  in dm_pool_metadata_read_write()
1751 	dm_bm_set_read_write(pmd->bm);  in dm_pool_metadata_read_write()
1752 	up_write(&pmd->root_lock);  in dm_pool_metadata_read_write()
1755 int dm_pool_register_metadata_threshold(struct dm_pool_metadata *pmd,  in dm_pool_register_metadata_threshold()  argument
1762 	down_write(&pmd->root_lock);  in dm_pool_register_metadata_threshold()
1763 	r = dm_sm_register_threshold_callback(pmd->metadata_sm, threshold, fn, context);  in dm_pool_register_metadata_threshold()
1764 	up_write(&pmd->root_lock);  in dm_pool_register_metadata_threshold()
1769 int dm_pool_metadata_set_needs_check(struct dm_pool_metadata *pmd)  in dm_pool_metadata_set_needs_check()  argument
1775 	down_write(&pmd->root_lock);  in dm_pool_metadata_set_needs_check()
1776 	pmd->flags |= THIN_METADATA_NEEDS_CHECK_FLAG;  in dm_pool_metadata_set_needs_check()
1778 	r = superblock_lock(pmd, &sblock);  in dm_pool_metadata_set_needs_check()
1785 	disk_super->flags = cpu_to_le32(pmd->flags);  in dm_pool_metadata_set_needs_check()
1789 	up_write(&pmd->root_lock);  in dm_pool_metadata_set_needs_check()
1793 bool dm_pool_metadata_needs_check(struct dm_pool_metadata *pmd)  in dm_pool_metadata_needs_check()  argument
1797 	down_read(&pmd->root_lock);  in dm_pool_metadata_needs_check()
1798 	needs_check = pmd->flags & THIN_METADATA_NEEDS_CHECK_FLAG;  in dm_pool_metadata_needs_check()
1799 	up_read(&pmd->root_lock);  in dm_pool_metadata_needs_check()
1804 void dm_pool_issue_prefetches(struct dm_pool_metadata *pmd)  in dm_pool_issue_prefetches()  argument
1806 	down_read(&pmd->root_lock);  in dm_pool_issue_prefetches()
1807 	if (!pmd->fail_io)  in dm_pool_issue_prefetches()
1808 		dm_tm_issue_prefetches(pmd->tm);  in dm_pool_issue_prefetches()
1809 	up_read(&pmd->root_lock);  in dm_pool_issue_prefetches()