41 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
44 						struct btrfs_fs_info *fs_info,
50 static u64 __btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info);
52 static int btrfs_dev_replace_continue_on_mount(struct btrfs_fs_info *fs_info);
55 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)  in btrfs_init_dev_replace()  argument
58 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_init_dev_replace()
59 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()
106 		btrfs_warn(fs_info,  in btrfs_init_dev_replace()
138 		dev_replace->srcdev = btrfs_find_device(fs_info, src_devid,  in btrfs_init_dev_replace()
140 		dev_replace->tgtdev = btrfs_find_device(fs_info,  in btrfs_init_dev_replace()
150 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
152 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
159 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
161 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
179 			btrfs_init_dev_replace_tgtdev_for_resume(fs_info,  in btrfs_init_dev_replace()
195 			  struct btrfs_fs_info *fs_info)  in btrfs_run_dev_replace()  argument
198 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_run_dev_replace()
203 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()
224 		btrfs_warn(fs_info, "error %d while searching for dev_replace item!",  in btrfs_run_dev_replace()
244 			btrfs_warn(fs_info, "delete too small dev_replace item failed %d!",  in btrfs_run_dev_replace()
257 			btrfs_warn(fs_info, "insert dev_replace item failed %d!",  in btrfs_run_dev_replace()
300 void btrfs_after_dev_replace_commit(struct btrfs_fs_info *fs_info)  in btrfs_after_dev_replace_commit()  argument
302 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_after_dev_replace_commit()
312 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_dev_replace_start()  local
313 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()
331 	mutex_lock(&fs_info->volume_mutex);  in btrfs_dev_replace_start()
336 		mutex_unlock(&fs_info->volume_mutex);  in btrfs_dev_replace_start()
342 	mutex_unlock(&fs_info->volume_mutex);  in btrfs_dev_replace_start()
378 	btrfs_info_in_rcu(root->fs_info,  in btrfs_dev_replace_start()
402 		btrfs_err(root->fs_info, "kobj add dev failed %d\n", ret);  in btrfs_dev_replace_start()
404 	btrfs_wait_ordered_roots(root->fs_info, -1);  in btrfs_dev_replace_start()
418 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,  in btrfs_dev_replace_start()
422 	ret = btrfs_dev_replace_finishing(root->fs_info, ret);  in btrfs_dev_replace_start()
437 	btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);  in btrfs_dev_replace_start()
444 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_blocked()  argument
446 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_blocked()
447 	wait_event(fs_info->replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
448 		   &fs_info->bio_counter));  in btrfs_rm_dev_replace_blocked()
454 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_unblocked()  argument
456 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_unblocked()
457 	wake_up(&fs_info->replace_wait);  in btrfs_rm_dev_replace_unblocked()
460 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_finishing()  argument
463 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()
466 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_finishing()
491 	ret = btrfs_start_delalloc_roots(root->fs_info, 0, -1);  in btrfs_dev_replace_finishing()
496 	btrfs_wait_ordered_roots(root->fs_info, -1);  in btrfs_dev_replace_finishing()
508 	mutex_lock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
509 	mutex_lock(&root->fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
521 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,  in btrfs_dev_replace_finishing()
525 		btrfs_err_in_rcu(root->fs_info,  in btrfs_dev_replace_finishing()
532 		mutex_unlock(&root->fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
533 		mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
536 			btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);  in btrfs_dev_replace_finishing()
542 	btrfs_info_in_rcu(root->fs_info,  in btrfs_dev_replace_finishing()
561 	if (fs_info->sb->s_bdev == src_device->bdev)  in btrfs_dev_replace_finishing()
562 		fs_info->sb->s_bdev = tgt_device->bdev;  in btrfs_dev_replace_finishing()
563 	if (fs_info->fs_devices->latest_bdev == src_device->bdev)  in btrfs_dev_replace_finishing()
564 		fs_info->fs_devices->latest_bdev = tgt_device->bdev;  in btrfs_dev_replace_finishing()
565 	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);  in btrfs_dev_replace_finishing()
566 	fs_info->fs_devices->rw_devices++;  in btrfs_dev_replace_finishing()
570 	btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
572 	btrfs_rm_dev_replace_remove_srcdev(fs_info, src_device);  in btrfs_dev_replace_finishing()
574 	btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
583 	mutex_unlock(&root->fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
584 	mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
588 	btrfs_sysfs_rm_device_link(fs_info->fs_devices, src_device);  in btrfs_dev_replace_finishing()
589 	btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);  in btrfs_dev_replace_finishing()
602 						struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_update_device_in_mapping_tree()  argument
606 	struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;  in btrfs_dev_replace_update_device_in_mapping_tree()
635 		*device = btrfs_find_device(root->fs_info, srcdevid, NULL,  in btrfs_dev_replace_find_srcdev()
646 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_status()  argument
649 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()
681 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_cancel()  argument
684 	args->result = __btrfs_dev_replace_cancel(fs_info);  in btrfs_dev_replace_cancel()
688 static u64 __btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)  in __btrfs_dev_replace_cancel()  argument
690 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in __btrfs_dev_replace_cancel()
693 	struct btrfs_root *root = fs_info->tree_root;  in __btrfs_dev_replace_cancel()
697 	if (fs_info->sb->s_flags & MS_RDONLY)  in __btrfs_dev_replace_cancel()
721 	btrfs_scrub_cancel(fs_info);  in __btrfs_dev_replace_cancel()
731 		btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);  in __btrfs_dev_replace_cancel()
738 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_suspend_for_unmount()  argument
740 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()
755 		btrfs_info(fs_info, "suspending dev_replace for unmount");  in btrfs_dev_replace_suspend_for_unmount()
764 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)  in btrfs_resume_dev_replace_async()  argument
767 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()
784 		btrfs_info(fs_info, "cannot continue dev_replace, tgtdev is missing");  in btrfs_resume_dev_replace_async()
785 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
793 		&fs_info->mutually_exclusive_operation_running, 1));  in btrfs_resume_dev_replace_async()
794 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");  in btrfs_resume_dev_replace_async()
800 	struct btrfs_fs_info *fs_info = data;  in btrfs_dev_replace_kthread()  local
801 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()
807 		btrfs_dev_replace_status(fs_info, status_args);  in btrfs_dev_replace_kthread()
811 		btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_kthread()
821 	btrfs_dev_replace_continue_on_mount(fs_info);  in btrfs_dev_replace_kthread()
822 	atomic_set(&fs_info->mutually_exclusive_operation_running, 0);  in btrfs_dev_replace_kthread()
827 static int btrfs_dev_replace_continue_on_mount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_continue_on_mount()  argument
829 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_continue_on_mount()
832 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_continue_on_mount()
836 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_continue_on_mount()
912 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_noblocked()  argument
914 	percpu_counter_inc(&fs_info->bio_counter);  in btrfs_bio_counter_inc_noblocked()
917 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)  in btrfs_bio_counter_sub()  argument
919 	percpu_counter_sub(&fs_info->bio_counter, amount);  in btrfs_bio_counter_sub()
921 	if (waitqueue_active(&fs_info->replace_wait))  in btrfs_bio_counter_sub()
922 		wake_up(&fs_info->replace_wait);  in btrfs_bio_counter_sub()
925 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_blocked()  argument
928 		percpu_counter_inc(&fs_info->bio_counter);  in btrfs_bio_counter_inc_blocked()
930 				     &fs_info->fs_state)))  in btrfs_bio_counter_inc_blocked()
933 		btrfs_bio_counter_dec(fs_info);  in btrfs_bio_counter_inc_blocked()
934 		wait_event(fs_info->replace_wait,  in btrfs_bio_counter_inc_blocked()
936 				     &fs_info->fs_state));  in btrfs_bio_counter_inc_blocked()