41 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
44 						struct btrfs_fs_info *fs_info,
50 static u64 __btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info);
52 static int btrfs_dev_replace_continue_on_mount(struct btrfs_fs_info *fs_info);
55 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)  in btrfs_init_dev_replace()  argument
58 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_init_dev_replace()
59 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()
106 		btrfs_warn(fs_info,  in btrfs_init_dev_replace()
138 		dev_replace->srcdev = btrfs_find_device(fs_info, src_devid,  in btrfs_init_dev_replace()
140 		dev_replace->tgtdev = btrfs_find_device(fs_info,  in btrfs_init_dev_replace()
150 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
152 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
159 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
161 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
179 			btrfs_init_dev_replace_tgtdev_for_resume(fs_info,  in btrfs_init_dev_replace()
196 			  struct btrfs_fs_info *fs_info)  in btrfs_run_dev_replace()  argument
199 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_run_dev_replace()
204 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()
225 		btrfs_warn(fs_info, "error %d while searching for dev_replace item!",  in btrfs_run_dev_replace()
245 			btrfs_warn(fs_info, "delete too small dev_replace item failed %d!",  in btrfs_run_dev_replace()
258 			btrfs_warn(fs_info, "insert dev_replace item failed %d!",  in btrfs_run_dev_replace()
301 void btrfs_after_dev_replace_commit(struct btrfs_fs_info *fs_info)  in btrfs_after_dev_replace_commit()  argument
303 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_after_dev_replace_commit()
313 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_dev_replace_start()  local
314 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()
345 	mutex_lock(&fs_info->volume_mutex);  in btrfs_dev_replace_start()
350 		mutex_unlock(&fs_info->volume_mutex);  in btrfs_dev_replace_start()
356 	mutex_unlock(&fs_info->volume_mutex);  in btrfs_dev_replace_start()
401 	btrfs_wait_ordered_roots(root->fs_info, -1);  in btrfs_dev_replace_start()
415 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,  in btrfs_dev_replace_start()
419 	ret = btrfs_dev_replace_finishing(root->fs_info, ret);  in btrfs_dev_replace_start()
434 	btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);  in btrfs_dev_replace_start()
441 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_blocked()  argument
443 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_blocked()
444 	wait_event(fs_info->replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
445 		   &fs_info->bio_counter));  in btrfs_rm_dev_replace_blocked()
451 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_unblocked()  argument
453 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_unblocked()
454 	if (waitqueue_active(&fs_info->replace_wait))  in btrfs_rm_dev_replace_unblocked()
455 		wake_up(&fs_info->replace_wait);  in btrfs_rm_dev_replace_unblocked()
458 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_finishing()  argument
461 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()
464 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_finishing()
489 	ret = btrfs_start_delalloc_roots(root->fs_info, 0, -1);  in btrfs_dev_replace_finishing()
494 	btrfs_wait_ordered_roots(root->fs_info, -1);  in btrfs_dev_replace_finishing()
506 	mutex_lock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
507 	mutex_lock(&root->fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
519 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,  in btrfs_dev_replace_finishing()
530 		mutex_unlock(&root->fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
531 		mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
534 			btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);  in btrfs_dev_replace_finishing()
559 	if (fs_info->sb->s_bdev == src_device->bdev)  in btrfs_dev_replace_finishing()
560 		fs_info->sb->s_bdev = tgt_device->bdev;  in btrfs_dev_replace_finishing()
561 	if (fs_info->fs_devices->latest_bdev == src_device->bdev)  in btrfs_dev_replace_finishing()
562 		fs_info->fs_devices->latest_bdev = tgt_device->bdev;  in btrfs_dev_replace_finishing()
563 	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);  in btrfs_dev_replace_finishing()
564 	fs_info->fs_devices->rw_devices++;  in btrfs_dev_replace_finishing()
568 	btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
570 	btrfs_rm_dev_replace_remove_srcdev(fs_info, src_device);  in btrfs_dev_replace_finishing()
572 	btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
581 	mutex_unlock(&root->fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
582 	mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
586 	btrfs_kobj_rm_device(fs_info, src_device);  in btrfs_dev_replace_finishing()
587 	btrfs_kobj_add_device(fs_info, tgt_device);  in btrfs_dev_replace_finishing()
588 	btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);  in btrfs_dev_replace_finishing()
601 						struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_update_device_in_mapping_tree()  argument
605 	struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;  in btrfs_dev_replace_update_device_in_mapping_tree()
634 		*device = btrfs_find_device(root->fs_info, srcdevid, NULL,  in btrfs_dev_replace_find_srcdev()
645 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_status()  argument
648 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()
680 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_cancel()  argument
683 	args->result = __btrfs_dev_replace_cancel(fs_info);  in btrfs_dev_replace_cancel()
687 static u64 __btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)  in __btrfs_dev_replace_cancel()  argument
689 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in __btrfs_dev_replace_cancel()
692 	struct btrfs_root *root = fs_info->tree_root;  in __btrfs_dev_replace_cancel()
696 	if (fs_info->sb->s_flags & MS_RDONLY)  in __btrfs_dev_replace_cancel()
720 	btrfs_scrub_cancel(fs_info);  in __btrfs_dev_replace_cancel()
730 		btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);  in __btrfs_dev_replace_cancel()
737 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_suspend_for_unmount()  argument
739 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()
754 		btrfs_info(fs_info, "suspending dev_replace for unmount");  in btrfs_dev_replace_suspend_for_unmount()
763 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)  in btrfs_resume_dev_replace_async()  argument
766 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()
783 		btrfs_info(fs_info, "cannot continue dev_replace, tgtdev is missing");  in btrfs_resume_dev_replace_async()
784 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
792 		&fs_info->mutually_exclusive_operation_running, 1));  in btrfs_resume_dev_replace_async()
793 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");  in btrfs_resume_dev_replace_async()
799 	struct btrfs_fs_info *fs_info = data;  in btrfs_dev_replace_kthread()  local
800 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()
806 		btrfs_dev_replace_status(fs_info, status_args);  in btrfs_dev_replace_kthread()
820 	btrfs_dev_replace_continue_on_mount(fs_info);  in btrfs_dev_replace_kthread()
821 	atomic_set(&fs_info->mutually_exclusive_operation_running, 0);  in btrfs_dev_replace_kthread()
826 static int btrfs_dev_replace_continue_on_mount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_continue_on_mount()  argument
828 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_continue_on_mount()
831 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_continue_on_mount()
835 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_continue_on_mount()
911 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_noblocked()  argument
913 	percpu_counter_inc(&fs_info->bio_counter);  in btrfs_bio_counter_inc_noblocked()
916 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)  in btrfs_bio_counter_sub()  argument
918 	percpu_counter_sub(&fs_info->bio_counter, amount);  in btrfs_bio_counter_sub()
920 	if (waitqueue_active(&fs_info->replace_wait))  in btrfs_bio_counter_sub()
921 		wake_up(&fs_info->replace_wait);  in btrfs_bio_counter_sub()
924 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_blocked()  argument
927 		percpu_counter_inc(&fs_info->bio_counter);  in btrfs_bio_counter_inc_blocked()
929 				     &fs_info->fs_state)))  in btrfs_bio_counter_inc_blocked()
932 		btrfs_bio_counter_dec(fs_info);  in btrfs_bio_counter_inc_blocked()
933 		wait_event(fs_info->replace_wait,  in btrfs_bio_counter_inc_blocked()
935 				     &fs_info->fs_state));  in btrfs_bio_counter_inc_blocked()