59 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()  local
81 		dev_replace->replace_state =  in btrfs_init_dev_replace()
83 		dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
85 		dev_replace->replace_state = 0;  in btrfs_init_dev_replace()
86 		dev_replace->time_started = 0;  in btrfs_init_dev_replace()
87 		dev_replace->time_stopped = 0;  in btrfs_init_dev_replace()
88 		atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_init_dev_replace()
89 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_init_dev_replace()
90 		dev_replace->cursor_left = 0;  in btrfs_init_dev_replace()
91 		dev_replace->committed_cursor_left = 0;  in btrfs_init_dev_replace()
92 		dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_init_dev_replace()
93 		dev_replace->cursor_right = 0;  in btrfs_init_dev_replace()
94 		dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
95 		dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
96 		dev_replace->is_valid = 0;  in btrfs_init_dev_replace()
97 		dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
112 	dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
114 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);  in btrfs_init_dev_replace()
115 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);  in btrfs_init_dev_replace()
116 	dev_replace->time_stopped =  in btrfs_init_dev_replace()
118 	atomic64_set(&dev_replace->num_write_errors,  in btrfs_init_dev_replace()
120 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,  in btrfs_init_dev_replace()
122 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);  in btrfs_init_dev_replace()
123 	dev_replace->committed_cursor_left = dev_replace->cursor_left;  in btrfs_init_dev_replace()
124 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;  in btrfs_init_dev_replace()
125 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);  in btrfs_init_dev_replace()
126 	dev_replace->is_valid = 1;  in btrfs_init_dev_replace()
128 	dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
129 	switch (dev_replace->replace_state) {  in btrfs_init_dev_replace()
133 		dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
134 		dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
138 		dev_replace->srcdev = btrfs_find_device(fs_info, src_devid,  in btrfs_init_dev_replace()
140 		dev_replace->tgtdev = btrfs_find_device(fs_info,  in btrfs_init_dev_replace()
147 		if (!dev_replace->srcdev &&  in btrfs_init_dev_replace()
156 		if (!dev_replace->tgtdev &&  in btrfs_init_dev_replace()
165 		if (dev_replace->tgtdev) {  in btrfs_init_dev_replace()
166 			if (dev_replace->srcdev) {  in btrfs_init_dev_replace()
167 				dev_replace->tgtdev->total_bytes =  in btrfs_init_dev_replace()
168 					dev_replace->srcdev->total_bytes;  in btrfs_init_dev_replace()
169 				dev_replace->tgtdev->disk_total_bytes =  in btrfs_init_dev_replace()
170 					dev_replace->srcdev->disk_total_bytes;  in btrfs_init_dev_replace()
171 				dev_replace->tgtdev->commit_total_bytes =  in btrfs_init_dev_replace()
172 					dev_replace->srcdev->commit_total_bytes;  in btrfs_init_dev_replace()
173 				dev_replace->tgtdev->bytes_used =  in btrfs_init_dev_replace()
174 					dev_replace->srcdev->bytes_used;  in btrfs_init_dev_replace()
175 				dev_replace->tgtdev->commit_bytes_used =  in btrfs_init_dev_replace()
176 					dev_replace->srcdev->commit_bytes_used;  in btrfs_init_dev_replace()
178 			dev_replace->tgtdev->is_tgtdev_for_dev_replace = 1;  in btrfs_init_dev_replace()
180 				dev_replace->tgtdev);  in btrfs_init_dev_replace()
203 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()  local
205 	btrfs_dev_replace_lock(dev_replace);  in btrfs_run_dev_replace()
206 	if (!dev_replace->is_valid ||  in btrfs_run_dev_replace()
207 	    !dev_replace->item_needs_writeback) {  in btrfs_run_dev_replace()
208 		btrfs_dev_replace_unlock(dev_replace);  in btrfs_run_dev_replace()
211 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_run_dev_replace()
267 	btrfs_dev_replace_lock(dev_replace);  in btrfs_run_dev_replace()
268 	if (dev_replace->srcdev)  in btrfs_run_dev_replace()
270 			dev_replace->srcdev->devid);  in btrfs_run_dev_replace()
274 		dev_replace->cont_reading_from_srcdev_mode);  in btrfs_run_dev_replace()
276 		dev_replace->replace_state);  in btrfs_run_dev_replace()
277 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);  in btrfs_run_dev_replace()
278 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);  in btrfs_run_dev_replace()
280 		atomic64_read(&dev_replace->num_write_errors));  in btrfs_run_dev_replace()
282 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));  in btrfs_run_dev_replace()
283 	dev_replace->cursor_left_last_write_of_item =  in btrfs_run_dev_replace()
284 		dev_replace->cursor_left;  in btrfs_run_dev_replace()
286 		dev_replace->cursor_left_last_write_of_item);  in btrfs_run_dev_replace()
288 		dev_replace->cursor_right);  in btrfs_run_dev_replace()
289 	dev_replace->item_needs_writeback = 0;  in btrfs_run_dev_replace()
290 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_run_dev_replace()
302 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_after_dev_replace_commit()  local
304 	dev_replace->committed_cursor_left =  in btrfs_after_dev_replace_commit()
305 		dev_replace->cursor_left_last_write_of_item;  in btrfs_after_dev_replace_commit()
313 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()  local
359 	btrfs_dev_replace_lock(dev_replace);  in btrfs_dev_replace_start()
360 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_start()
371 	dev_replace->cont_reading_from_srcdev_mode =  in btrfs_dev_replace_start()
374 	dev_replace->srcdev = src_device;  in btrfs_dev_replace_start()
376 	dev_replace->tgtdev = tgt_device;  in btrfs_dev_replace_start()
389 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;  in btrfs_dev_replace_start()
390 	dev_replace->time_started = get_seconds();  in btrfs_dev_replace_start()
391 	dev_replace->cursor_left = 0;  in btrfs_dev_replace_start()
392 	dev_replace->committed_cursor_left = 0;  in btrfs_dev_replace_start()
393 	dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_dev_replace_start()
394 	dev_replace->cursor_right = 0;  in btrfs_dev_replace_start()
395 	dev_replace->is_valid = 1;  in btrfs_dev_replace_start()
396 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_start()
398 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_start()
410 		btrfs_dev_replace_lock(dev_replace);  in btrfs_dev_replace_start()
420 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_start()
434 	dev_replace->srcdev = NULL;  in btrfs_dev_replace_start()
435 	dev_replace->tgtdev = NULL;  in btrfs_dev_replace_start()
436 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_start()
463 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()  local
472 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
474 	btrfs_dev_replace_lock(dev_replace);  in btrfs_dev_replace_finishing()
476 	if (dev_replace->replace_state !=  in btrfs_dev_replace_finishing()
478 		btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_finishing()
479 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
483 	tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_finishing()
484 	src_device = dev_replace->srcdev;  in btrfs_dev_replace_finishing()
485 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_finishing()
493 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
500 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
510 	btrfs_dev_replace_lock(dev_replace);  in btrfs_dev_replace_finishing()
511 	dev_replace->replace_state =  in btrfs_dev_replace_finishing()
514 	dev_replace->tgtdev = NULL;  in btrfs_dev_replace_finishing()
515 	dev_replace->srcdev = NULL;  in btrfs_dev_replace_finishing()
516 	dev_replace->time_stopped = get_seconds();  in btrfs_dev_replace_finishing()
517 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_finishing()
531 		btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_finishing()
537 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
568 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_finishing()
596 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
649 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()  local
652 	btrfs_dev_replace_lock(dev_replace);  in btrfs_dev_replace_status()
656 	args->status.replace_state = dev_replace->replace_state;  in btrfs_dev_replace_status()
657 	args->status.time_started = dev_replace->time_started;  in btrfs_dev_replace_status()
658 	args->status.time_stopped = dev_replace->time_stopped;  in btrfs_dev_replace_status()
660 		atomic64_read(&dev_replace->num_write_errors);  in btrfs_dev_replace_status()
662 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);  in btrfs_dev_replace_status()
663 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_status()
673 		srcdev = dev_replace->srcdev;  in btrfs_dev_replace_status()
674 		args->status.progress_1000 = div_u64(dev_replace->cursor_left,  in btrfs_dev_replace_status()
678 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_status()
690 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in __btrfs_dev_replace_cancel()  local
700 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in __btrfs_dev_replace_cancel()
701 	btrfs_dev_replace_lock(dev_replace);  in __btrfs_dev_replace_cancel()
702 	switch (dev_replace->replace_state) {  in __btrfs_dev_replace_cancel()
707 		btrfs_dev_replace_unlock(dev_replace);  in __btrfs_dev_replace_cancel()
712 		tgt_device = dev_replace->tgtdev;  in __btrfs_dev_replace_cancel()
713 		dev_replace->tgtdev = NULL;  in __btrfs_dev_replace_cancel()
714 		dev_replace->srcdev = NULL;  in __btrfs_dev_replace_cancel()
717 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;  in __btrfs_dev_replace_cancel()
718 	dev_replace->time_stopped = get_seconds();  in __btrfs_dev_replace_cancel()
719 	dev_replace->item_needs_writeback = 1;  in __btrfs_dev_replace_cancel()
720 	btrfs_dev_replace_unlock(dev_replace);  in __btrfs_dev_replace_cancel()
725 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in __btrfs_dev_replace_cancel()
734 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in __btrfs_dev_replace_cancel()
740 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()  local
742 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
743 	btrfs_dev_replace_lock(dev_replace);  in btrfs_dev_replace_suspend_for_unmount()
744 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_suspend_for_unmount()
751 		dev_replace->replace_state =  in btrfs_dev_replace_suspend_for_unmount()
753 		dev_replace->time_stopped = get_seconds();  in btrfs_dev_replace_suspend_for_unmount()
754 		dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_suspend_for_unmount()
759 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_dev_replace_suspend_for_unmount()
760 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
767 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()  local
769 	btrfs_dev_replace_lock(dev_replace);  in btrfs_resume_dev_replace_async()
770 	switch (dev_replace->replace_state) {  in btrfs_resume_dev_replace_async()
774 		btrfs_dev_replace_unlock(dev_replace);  in btrfs_resume_dev_replace_async()
779 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
783 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {  in btrfs_resume_dev_replace_async()
787 		btrfs_dev_replace_unlock(dev_replace);  in btrfs_resume_dev_replace_async()
790 	btrfs_dev_replace_unlock(dev_replace);  in btrfs_resume_dev_replace_async()
801 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()  local
813 			dev_replace->srcdev->missing ? "<missing disk>" :  in btrfs_dev_replace_kthread()
814 			rcu_str_deref(dev_replace->srcdev->name),  in btrfs_dev_replace_kthread()
815 			dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
816 			dev_replace->tgtdev ?  in btrfs_dev_replace_kthread()
817 			rcu_str_deref(dev_replace->tgtdev->name) :  in btrfs_dev_replace_kthread()
829 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_continue_on_mount()  local
832 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_continue_on_mount()
833 			      dev_replace->committed_cursor_left,  in btrfs_dev_replace_continue_on_mount()
834 			      btrfs_device_get_total_bytes(dev_replace->srcdev),  in btrfs_dev_replace_continue_on_mount()
835 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_continue_on_mount()
841 int btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_is_ongoing()  argument
843 	if (!dev_replace->is_valid)  in btrfs_dev_replace_is_ongoing()
846 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_is_ongoing()
868 void btrfs_dev_replace_lock(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_lock()  argument
871 	if (atomic_read(&dev_replace->nesting_level) == 0) {  in btrfs_dev_replace_lock()
875 		mutex_lock(&dev_replace->lock);  in btrfs_dev_replace_lock()
876 		mutex_lock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_lock()
877 		dev_replace->lock_owner = current->pid;  in btrfs_dev_replace_lock()
878 		atomic_inc(&dev_replace->nesting_level);  in btrfs_dev_replace_lock()
879 		mutex_unlock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_lock()
883 	mutex_lock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_lock()
884 	if (atomic_read(&dev_replace->nesting_level) > 0 &&  in btrfs_dev_replace_lock()
885 	    dev_replace->lock_owner == current->pid) {  in btrfs_dev_replace_lock()
886 		WARN_ON(!mutex_is_locked(&dev_replace->lock));  in btrfs_dev_replace_lock()
887 		atomic_inc(&dev_replace->nesting_level);  in btrfs_dev_replace_lock()
888 		mutex_unlock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_lock()
892 	mutex_unlock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_lock()
896 void btrfs_dev_replace_unlock(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_unlock()  argument
898 	WARN_ON(!mutex_is_locked(&dev_replace->lock));  in btrfs_dev_replace_unlock()
899 	mutex_lock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_unlock()
900 	WARN_ON(atomic_read(&dev_replace->nesting_level) < 1);  in btrfs_dev_replace_unlock()
901 	WARN_ON(dev_replace->lock_owner != current->pid);  in btrfs_dev_replace_unlock()
902 	atomic_dec(&dev_replace->nesting_level);  in btrfs_dev_replace_unlock()
903 	if (atomic_read(&dev_replace->nesting_level) == 0) {  in btrfs_dev_replace_unlock()
904 		dev_replace->lock_owner = 0;  in btrfs_dev_replace_unlock()
905 		mutex_unlock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_unlock()
906 		mutex_unlock(&dev_replace->lock);  in btrfs_dev_replace_unlock()
908 		mutex_unlock(&dev_replace->lock_management_lock);  in btrfs_dev_replace_unlock()