249 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
253 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
284 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
291 			      struct btrfs_fs_info *fs_info,
307 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
308 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
325 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)  in __scrub_blocked_if_needed()  argument
327 	while (atomic_read(&fs_info->scrub_pause_req)) {  in __scrub_blocked_if_needed()
328 		mutex_unlock(&fs_info->scrub_lock);  in __scrub_blocked_if_needed()
329 		wait_event(fs_info->scrub_pause_wait,  in __scrub_blocked_if_needed()
330 		   atomic_read(&fs_info->scrub_pause_req) == 0);  in __scrub_blocked_if_needed()
331 		mutex_lock(&fs_info->scrub_lock);  in __scrub_blocked_if_needed()
335 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)  in scrub_blocked_if_needed()  argument
337 	atomic_inc(&fs_info->scrubs_paused);  in scrub_blocked_if_needed()
338 	wake_up(&fs_info->scrub_pause_wait);  in scrub_blocked_if_needed()
340 	mutex_lock(&fs_info->scrub_lock);  in scrub_blocked_if_needed()
341 	__scrub_blocked_if_needed(fs_info);  in scrub_blocked_if_needed()
342 	atomic_dec(&fs_info->scrubs_paused);  in scrub_blocked_if_needed()
343 	mutex_unlock(&fs_info->scrub_lock);  in scrub_blocked_if_needed()
345 	wake_up(&fs_info->scrub_pause_wait);  in scrub_blocked_if_needed()
354 	struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;  in scrub_pending_trans_workers_inc()  local
366 	mutex_lock(&fs_info->scrub_lock);  in scrub_pending_trans_workers_inc()
367 	atomic_inc(&fs_info->scrubs_running);  in scrub_pending_trans_workers_inc()
368 	atomic_inc(&fs_info->scrubs_paused);  in scrub_pending_trans_workers_inc()
369 	mutex_unlock(&fs_info->scrub_lock);  in scrub_pending_trans_workers_inc()
378 	wake_up(&fs_info->scrub_pause_wait);  in scrub_pending_trans_workers_inc()
386 	struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;  in scrub_pending_trans_workers_dec()  local
392 	mutex_lock(&fs_info->scrub_lock);  in scrub_pending_trans_workers_dec()
393 	atomic_dec(&fs_info->scrubs_running);  in scrub_pending_trans_workers_dec()
394 	atomic_dec(&fs_info->scrubs_paused);  in scrub_pending_trans_workers_dec()
395 	mutex_unlock(&fs_info->scrub_lock);  in scrub_pending_trans_workers_dec()
397 	wake_up(&fs_info->scrub_pause_wait);  in scrub_pending_trans_workers_dec()
456 	struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;  in scrub_setup_ctx()  local
505 	sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);  in scrub_setup_ctx()
512 	ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,  in scrub_setup_ctx()
513 				 fs_info->dev_replace.tgtdev, is_dev_replace);  in scrub_setup_ctx()
535 	struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;  in scrub_print_warning_inode()  local
544 	local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);  in scrub_print_warning_inode()
611 	struct btrfs_fs_info *fs_info;  in scrub_print_warning()  local
627 	fs_info = sblock->sctx->dev_root->fs_info;  in scrub_print_warning()
638 	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,  in scrub_print_warning()
670 		iterate_extent_inodes(fs_info, found_key.objectid,  in scrub_print_warning()
688 	struct btrfs_fs_info *fs_info;  in scrub_fixup_readpage()  local
697 	fs_info = fixup->root->fs_info;  in scrub_fixup_readpage()
698 	srcu_index = srcu_read_lock(&fs_info->subvol_srcu);  in scrub_fixup_readpage()
700 	local_root = btrfs_read_fs_root_no_name(fs_info, &key);  in scrub_fixup_readpage()
702 		srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);  in scrub_fixup_readpage()
709 	inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);  in scrub_fixup_readpage()
710 	srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);  in scrub_fixup_readpage()
833 	ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,  in scrub_fixup_nodatasum()
854 			&sctx->dev_root->fs_info->dev_replace.  in scrub_fixup_nodatasum()
892 	struct btrfs_fs_info *fs_info;  in scrub_handle_errored_block()  local
910 	fs_info = sctx->dev_root->fs_info;  in scrub_handle_errored_block()
993 	scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,  in scrub_handle_errored_block()
1071 		fixup_nodatasum->root = fs_info->extent_root;  in scrub_handle_errored_block()
1076 		btrfs_queue_work(fs_info->scrub_workers,  in scrub_handle_errored_block()
1107 		scrub_recheck_block(fs_info, sblock_other, is_metadata,  in scrub_handle_errored_block()
1195 					fs_info->dev_replace.  in scrub_handle_errored_block()
1221 			scrub_recheck_block(fs_info, sblock_bad,  in scrub_handle_errored_block()
1321 	struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;  in scrub_setup_recheck_block()  local
1350 		ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,  in scrub_setup_recheck_block()
1446 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,  in scrub_submit_raid56_bio_wait()  argument
1459 	ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,  in scrub_submit_raid56_bio_wait()
1479 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,  in scrub_recheck_block()  argument
1511 			if (scrub_submit_raid56_bio_wait(fs_info, bio, page))  in scrub_recheck_block()
1524 		scrub_recheck_block_checksum(fs_info, sblock, is_metadata,  in scrub_recheck_block()
1541 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,  in scrub_recheck_block_checksum()  argument
1561 		    memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,  in scrub_recheck_block_checksum()
1654 				&sblock_bad->sctx->dev_root->fs_info->  in scrub_repair_page_from_good_copy()
1682 				&sblock->sctx->dev_root->fs_info->dev_replace.  in scrub_write_block_to_dev_replace()
1796 	struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;  in scrub_wr_bio_end_io()  local
1803 	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);  in scrub_wr_bio_end_io()
1815 			&sbio->sctx->dev_root->fs_info->dev_replace;  in scrub_wr_bio_end_io_worker()
1905 	struct btrfs_fs_info *fs_info = root->fs_info;  in scrub_checksum_tree_block()  local
1939 	if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,  in scrub_checksum_tree_block()
2266 	struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;  in scrub_bio_end_io()  local
2271 	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);  in scrub_bio_end_io()
2694 	ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,  in scrub_parity_check_and_repair()
2761 	struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;  in scrub_raid56_parity()  local
2762 	struct btrfs_root *root = fs_info->extent_root;  in scrub_raid56_parity()
2763 	struct btrfs_root *csum_root = fs_info->csum_root;  in scrub_raid56_parity()
2805 		if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))  in scrub_raid56_parity()
2874 				btrfs_err(fs_info,  in scrub_raid56_parity()
2896 			scrub_remap_extent(fs_info, extent_logical,  in scrub_raid56_parity()
2964 	struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;  in scrub_stripe()  local
2965 	struct btrfs_root *root = fs_info->extent_root;  in scrub_stripe()
2966 	struct btrfs_root *csum_root = fs_info->csum_root;  in scrub_stripe()
3059 	scrub_blocked_if_needed(fs_info);  in scrub_stripe()
3114 		if (atomic_read(&fs_info->scrub_cancel_req) ||  in scrub_stripe()
3122 		if (atomic_read(&fs_info->scrub_pause_req)) {  in scrub_stripe()
3132 			scrub_blocked_if_needed(fs_info);  in scrub_stripe()
3135 		if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))  in scrub_stripe()
3205 				btrfs_err(fs_info,  in scrub_stripe()
3233 				scrub_remap_extent(fs_info, extent_logical,  in scrub_stripe()
3329 		&sctx->dev_root->fs_info->mapping_tree;  in scrub_chunk()
3373 	struct btrfs_fs_info *fs_info = root->fs_info;  in scrub_enumerate_chunks()  local
3384 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in scrub_enumerate_chunks()
3442 		cache = btrfs_lookup_block_group(fs_info, chunk_offset);  in scrub_enumerate_chunks()
3474 		atomic_inc(&fs_info->scrubs_paused);  in scrub_enumerate_chunks()
3475 		wake_up(&fs_info->scrub_pause_wait);  in scrub_enumerate_chunks()
3486 		mutex_lock(&fs_info->scrub_lock);  in scrub_enumerate_chunks()
3487 		__scrub_blocked_if_needed(fs_info);  in scrub_enumerate_chunks()
3488 		atomic_dec(&fs_info->scrubs_paused);  in scrub_enumerate_chunks()
3489 		mutex_unlock(&fs_info->scrub_lock);  in scrub_enumerate_chunks()
3490 		wake_up(&fs_info->scrub_pause_wait);  in scrub_enumerate_chunks()
3530 	if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))  in scrub_supers()
3534 	if (scrub_dev->fs_devices != root->fs_info->fs_devices)  in scrub_supers()
3537 		gen = root->fs_info->last_trans_committed;  in scrub_supers()
3559 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,  in scrub_workers_get()  argument
3564 	int max_active = fs_info->thread_pool_size;  in scrub_workers_get()
3566 	if (fs_info->scrub_workers_refcnt == 0) {  in scrub_workers_get()
3568 			fs_info->scrub_workers =  in scrub_workers_get()
3572 			fs_info->scrub_workers =  in scrub_workers_get()
3575 		if (!fs_info->scrub_workers) {  in scrub_workers_get()
3579 		fs_info->scrub_wr_completion_workers =  in scrub_workers_get()
3582 		if (!fs_info->scrub_wr_completion_workers) {  in scrub_workers_get()
3586 		fs_info->scrub_nocow_workers =  in scrub_workers_get()
3588 		if (!fs_info->scrub_nocow_workers) {  in scrub_workers_get()
3593 	++fs_info->scrub_workers_refcnt;  in scrub_workers_get()
3598 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)  in scrub_workers_put()  argument
3600 	if (--fs_info->scrub_workers_refcnt == 0) {  in scrub_workers_put()
3601 		btrfs_destroy_workqueue(fs_info->scrub_workers);  in scrub_workers_put()
3602 		btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);  in scrub_workers_put()
3603 		btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);  in scrub_workers_put()
3605 	WARN_ON(fs_info->scrub_workers_refcnt < 0);  in scrub_workers_put()
3608 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,  in btrfs_scrub_dev()  argument
3617 	if (btrfs_fs_closing(fs_info))  in btrfs_scrub_dev()
3620 	if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {  in btrfs_scrub_dev()
3626 		btrfs_err(fs_info,  in btrfs_scrub_dev()
3628 		       fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);  in btrfs_scrub_dev()
3632 	if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {  in btrfs_scrub_dev()
3634 		btrfs_err(fs_info,  in btrfs_scrub_dev()
3637 		       fs_info->chunk_root->sectorsize, PAGE_SIZE);  in btrfs_scrub_dev()
3641 	if (fs_info->chunk_root->nodesize >  in btrfs_scrub_dev()
3643 	    fs_info->chunk_root->sectorsize >  in btrfs_scrub_dev()
3649 		btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "  in btrfs_scrub_dev()
3651 		       fs_info->chunk_root->nodesize,  in btrfs_scrub_dev()
3653 		       fs_info->chunk_root->sectorsize,  in btrfs_scrub_dev()
3659 	mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3660 	dev = btrfs_find_device(fs_info, devid, NULL, NULL);  in btrfs_scrub_dev()
3662 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3667 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3670 		btrfs_err(fs_info, "scrub: device %s is not writable",  in btrfs_scrub_dev()
3676 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3678 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3679 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3683 	btrfs_dev_replace_lock(&fs_info->dev_replace);  in btrfs_scrub_dev()
3686 	     btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {  in btrfs_scrub_dev()
3687 		btrfs_dev_replace_unlock(&fs_info->dev_replace);  in btrfs_scrub_dev()
3688 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3689 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3692 	btrfs_dev_replace_unlock(&fs_info->dev_replace);  in btrfs_scrub_dev()
3694 	ret = scrub_workers_get(fs_info, is_dev_replace);  in btrfs_scrub_dev()
3696 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3697 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3703 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3704 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3705 		scrub_workers_put(fs_info);  in btrfs_scrub_dev()
3710 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3716 	__scrub_blocked_if_needed(fs_info);  in btrfs_scrub_dev()
3717 	atomic_inc(&fs_info->scrubs_running);  in btrfs_scrub_dev()
3718 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3725 		mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3727 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_dev()
3735 	atomic_dec(&fs_info->scrubs_running);  in btrfs_scrub_dev()
3736 	wake_up(&fs_info->scrub_pause_wait);  in btrfs_scrub_dev()
3743 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3745 	scrub_workers_put(fs_info);  in btrfs_scrub_dev()
3746 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_dev()
3755 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_scrub_pause()  local
3757 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
3758 	atomic_inc(&fs_info->scrub_pause_req);  in btrfs_scrub_pause()
3759 	while (atomic_read(&fs_info->scrubs_paused) !=  in btrfs_scrub_pause()
3760 	       atomic_read(&fs_info->scrubs_running)) {  in btrfs_scrub_pause()
3761 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
3762 		wait_event(fs_info->scrub_pause_wait,  in btrfs_scrub_pause()
3763 			   atomic_read(&fs_info->scrubs_paused) ==  in btrfs_scrub_pause()
3764 			   atomic_read(&fs_info->scrubs_running));  in btrfs_scrub_pause()
3765 		mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
3767 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_pause()
3772 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_scrub_continue()  local
3774 	atomic_dec(&fs_info->scrub_pause_req);  in btrfs_scrub_continue()
3775 	wake_up(&fs_info->scrub_pause_wait);  in btrfs_scrub_continue()
3778 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)  in btrfs_scrub_cancel()  argument
3780 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
3781 	if (!atomic_read(&fs_info->scrubs_running)) {  in btrfs_scrub_cancel()
3782 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
3786 	atomic_inc(&fs_info->scrub_cancel_req);  in btrfs_scrub_cancel()
3787 	while (atomic_read(&fs_info->scrubs_running)) {  in btrfs_scrub_cancel()
3788 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
3789 		wait_event(fs_info->scrub_pause_wait,  in btrfs_scrub_cancel()
3790 			   atomic_read(&fs_info->scrubs_running) == 0);  in btrfs_scrub_cancel()
3791 		mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
3793 	atomic_dec(&fs_info->scrub_cancel_req);  in btrfs_scrub_cancel()
3794 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel()
3799 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,  in btrfs_scrub_cancel_dev()  argument
3804 	mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
3807 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
3812 		mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
3813 		wait_event(fs_info->scrub_pause_wait,  in btrfs_scrub_cancel_dev()
3815 		mutex_lock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
3817 	mutex_unlock(&fs_info->scrub_lock);  in btrfs_scrub_cancel_dev()
3828 	mutex_lock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_progress()
3829 	dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);  in btrfs_scrub_progress()
3834 	mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);  in btrfs_scrub_progress()
3839 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,  in scrub_remap_extent()  argument
3850 	ret = btrfs_map_block(fs_info, READ, extent_logical,  in scrub_remap_extent()
3866 			      struct btrfs_fs_info *fs_info,  in scrub_setup_wr_ctx()  argument
3897 	struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;  in copy_nocow_pages()  local
3917 	btrfs_queue_work(fs_info->scrub_nocow_workers,  in copy_nocow_pages()
3951 	struct btrfs_fs_info *fs_info;  in copy_nocow_pages_worker()  local
3956 	fs_info = sctx->dev_root->fs_info;  in copy_nocow_pages_worker()
3957 	root = fs_info->extent_root;  in copy_nocow_pages_worker()
3974 	ret = iterate_inodes_from_logical(logical, fs_info, path,  in copy_nocow_pages_worker()
3977 		btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "  in copy_nocow_pages_worker()
4015 		btrfs_dev_replace_stats_inc(&fs_info->dev_replace.  in copy_nocow_pages_worker()
4071 	struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;  in copy_nocow_pages_for_inode()  local
4089 	srcu_index = srcu_read_lock(&fs_info->subvol_srcu);  in copy_nocow_pages_for_inode()
4091 	local_root = btrfs_read_fs_root_no_name(fs_info, &key);  in copy_nocow_pages_for_inode()
4093 		srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);  in copy_nocow_pages_for_inode()
4100 	inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);  in copy_nocow_pages_for_inode()
4101 	srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);  in copy_nocow_pages_for_inode()
4124 			btrfs_err(fs_info, "find_or_create_page() failed");  in copy_nocow_pages_for_inode()