73 	struct scrub_block	*sblock;  member
250 				struct scrub_block *sblock, int is_metadata,
254 					 struct scrub_block *sblock,
263 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
264 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
266 static int scrub_checksum_data(struct scrub_block *sblock);
267 static int scrub_checksum_tree_block(struct scrub_block *sblock);
268 static int scrub_checksum_super(struct scrub_block *sblock);
269 static void scrub_block_get(struct scrub_block *sblock);
270 static void scrub_block_put(struct scrub_block *sblock);
283 static void scrub_block_complete(struct scrub_block *sblock);
428 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
608 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)  in scrub_print_warning()  argument
625 	WARN_ON(sblock->page_count < 1);  in scrub_print_warning()
626 	dev = sblock->pagev[0]->dev;  in scrub_print_warning()
627 	fs_info = sblock->sctx->dev_root->fs_info;  in scrub_print_warning()
633 	swarn.sector = (sblock->pagev[0]->physical) >> 9;  in scrub_print_warning()
634 	swarn.logical = sblock->pagev[0]->logical;  in scrub_print_warning()
1254 			struct scrub_block *sblock = sblocks_for_recheck +  in scrub_handle_errored_block()  local
1259 			for (page_index = 0; page_index < sblock->page_count;  in scrub_handle_errored_block()
1261 				sblock->pagev[page_index]->sblock = NULL;  in scrub_handle_errored_block()
1262 				recover = sblock->pagev[page_index]->recover;  in scrub_handle_errored_block()
1265 					sblock->pagev[page_index]->recover =  in scrub_handle_errored_block()
1268 				scrub_page_put(sblock->pagev[page_index]);  in scrub_handle_errored_block()
1373 			struct scrub_block *sblock;  in scrub_setup_recheck_block()  local
1376 			sblock = sblocks_for_recheck + mirror_index;  in scrub_setup_recheck_block()
1377 			sblock->sctx = sctx;  in scrub_setup_recheck_block()
1388 			sblock->pagev[page_index] = page;  in scrub_setup_recheck_block()
1410 			sblock->page_count++;  in scrub_setup_recheck_block()
1480 				struct scrub_block *sblock, int is_metadata,  in scrub_recheck_block()  argument
1486 	sblock->no_io_error_seen = 1;  in scrub_recheck_block()
1487 	sblock->header_error = 0;  in scrub_recheck_block()
1488 	sblock->checksum_error = 0;  in scrub_recheck_block()
1490 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block()
1492 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block()
1496 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1504 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1512 				sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1517 				sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1523 	if (sblock->no_io_error_seen)  in scrub_recheck_block()
1524 		scrub_recheck_block_checksum(fs_info, sblock, is_metadata,  in scrub_recheck_block()
1542 					 struct scrub_block *sblock,  in scrub_recheck_block_checksum()  argument
1552 	WARN_ON(!sblock->pagev[0]->page);  in scrub_recheck_block_checksum()
1556 		mapped_buffer = kmap_atomic(sblock->pagev[0]->page);  in scrub_recheck_block_checksum()
1559 		if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||  in scrub_recheck_block_checksum()
1560 		    !scrub_check_fsid(h->fsid, sblock->pagev[0]) ||  in scrub_recheck_block_checksum()
1563 			sblock->header_error = 1;  in scrub_recheck_block_checksum()
1565 			sblock->header_error = 1;  in scrub_recheck_block_checksum()
1566 			sblock->generation_error = 1;  in scrub_recheck_block_checksum()
1573 		mapped_buffer = kmap_atomic(sblock->pagev[0]->page);  in scrub_recheck_block_checksum()
1586 		if (page_num >= sblock->page_count)  in scrub_recheck_block_checksum()
1588 		WARN_ON(!sblock->pagev[page_num]->page);  in scrub_recheck_block_checksum()
1590 		mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);  in scrub_recheck_block_checksum()
1595 		sblock->checksum_error = 1;  in scrub_recheck_block_checksum()
1665 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)  in scrub_write_block_to_dev_replace()  argument
1673 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1676 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_write_block_to_dev_replace()
1679 		ret = scrub_write_page_to_dev_replace(sblock, page_num);  in scrub_write_block_to_dev_replace()
1682 				&sblock->sctx->dev_root->fs_info->dev_replace.  in scrub_write_block_to_dev_replace()
1687 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,  in scrub_write_page_to_dev_replace()  argument
1690 	struct scrub_page *spage = sblock->pagev[page_num];  in scrub_write_page_to_dev_replace()
1700 	return scrub_add_page_to_wr_bio(sblock->sctx, spage);  in scrub_write_page_to_dev_replace()
1834 static int scrub_checksum(struct scrub_block *sblock)  in scrub_checksum()  argument
1839 	WARN_ON(sblock->page_count < 1);  in scrub_checksum()
1840 	flags = sblock->pagev[0]->flags;  in scrub_checksum()
1843 		ret = scrub_checksum_data(sblock);  in scrub_checksum()
1845 		ret = scrub_checksum_tree_block(sblock);  in scrub_checksum()
1847 		(void)scrub_checksum_super(sblock);  in scrub_checksum()
1851 		scrub_handle_errored_block(sblock);  in scrub_checksum()
1856 static int scrub_checksum_data(struct scrub_block *sblock)  in scrub_checksum_data()  argument
1858 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_data()
1868 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_data()
1869 	if (!sblock->pagev[0]->have_csum)  in scrub_checksum_data()
1872 	on_disk_csum = sblock->pagev[0]->csum;  in scrub_checksum_data()
1873 	page = sblock->pagev[0]->page;  in scrub_checksum_data()
1887 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_data()
1888 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_data()
1889 		page = sblock->pagev[index]->page;  in scrub_checksum_data()
1900 static int scrub_checksum_tree_block(struct scrub_block *sblock)  in scrub_checksum_tree_block()  argument
1902 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_tree_block()
1918 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_tree_block()
1919 	page = sblock->pagev[0]->page;  in scrub_checksum_tree_block()
1930 	if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))  in scrub_checksum_tree_block()
1933 	if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))  in scrub_checksum_tree_block()
1936 	if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))  in scrub_checksum_tree_block()
1956 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_tree_block()
1957 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_tree_block()
1958 		page = sblock->pagev[index]->page;  in scrub_checksum_tree_block()
1971 static int scrub_checksum_super(struct scrub_block *sblock)  in scrub_checksum_super()  argument
1974 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_super()
1987 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_super()
1988 	page = sblock->pagev[0]->page;  in scrub_checksum_super()
1993 	if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))  in scrub_checksum_super()
1996 	if (sblock->pagev[0]->generation != btrfs_super_generation(s))  in scrub_checksum_super()
1999 	if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))  in scrub_checksum_super()
2015 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_super()
2016 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_super()
2017 		page = sblock->pagev[index]->page;  in scrub_checksum_super()
2037 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
2040 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
2047 static void scrub_block_get(struct scrub_block *sblock)  in scrub_block_get()  argument
2049 	atomic_inc(&sblock->refs);  in scrub_block_get()
2052 static void scrub_block_put(struct scrub_block *sblock)  in scrub_block_put()  argument
2054 	if (atomic_dec_and_test(&sblock->refs)) {  in scrub_block_put()
2057 		if (sblock->sparity)  in scrub_block_put()
2058 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
2060 		for (i = 0; i < sblock->page_count; i++)  in scrub_block_put()
2061 			scrub_page_put(sblock->pagev[i]);  in scrub_block_put()
2062 		kfree(sblock);  in scrub_block_put()
2110 	struct scrub_block *sblock = spage->sblock;  in scrub_add_page_to_rd_bio()  local
2172 	scrub_block_get(sblock); /* one for the page added to the bio */  in scrub_add_page_to_rd_bio()
2173 	atomic_inc(&sblock->outstanding_pages);  in scrub_add_page_to_rd_bio()
2186 	struct scrub_block *sblock;  in scrub_pages()  local
2189 	sblock = kzalloc(sizeof(*sblock), GFP_NOFS);  in scrub_pages()
2190 	if (!sblock) {  in scrub_pages()
2199 	atomic_set(&sblock->refs, 1);  in scrub_pages()
2200 	sblock->sctx = sctx;  in scrub_pages()
2201 	sblock->no_io_error_seen = 1;  in scrub_pages()
2213 			scrub_block_put(sblock);  in scrub_pages()
2218 		sblock->pagev[index] = spage;  in scrub_pages()
2219 		spage->sblock = sblock;  in scrub_pages()
2233 		sblock->page_count++;  in scrub_pages()
2243 	WARN_ON(sblock->page_count == 0);  in scrub_pages()
2244 	for (index = 0; index < sblock->page_count; index++) {  in scrub_pages()
2245 		struct scrub_page *spage = sblock->pagev[index];  in scrub_pages()
2250 			scrub_block_put(sblock);  in scrub_pages()
2259 	scrub_block_put(sblock);  in scrub_pages()
2286 			spage->sblock->no_io_error_seen = 0;  in scrub_bio_end_io_worker()
2293 		struct scrub_block *sblock = spage->sblock;  in scrub_bio_end_io_worker()  local
2295 		if (atomic_dec_and_test(&sblock->outstanding_pages))  in scrub_bio_end_io_worker()
2296 			scrub_block_complete(sblock);  in scrub_bio_end_io_worker()
2297 		scrub_block_put(sblock);  in scrub_bio_end_io_worker()
2356 static void scrub_block_complete(struct scrub_block *sblock)  in scrub_block_complete()  argument
2360 	if (!sblock->no_io_error_seen) {  in scrub_block_complete()
2362 		scrub_handle_errored_block(sblock);  in scrub_block_complete()
2369 		corrupted = scrub_checksum(sblock);  in scrub_block_complete()
2370 		if (!corrupted && sblock->sctx->is_dev_replace)  in scrub_block_complete()
2371 			scrub_write_block_to_dev_replace(sblock);  in scrub_block_complete()
2374 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2375 		u64 start = sblock->pagev[0]->logical;  in scrub_block_complete()
2376 		u64 end = sblock->pagev[sblock->page_count - 1]->logical +  in scrub_block_complete()
2379 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2479 	struct scrub_block *sblock;  in scrub_pages_for_parity()  local
2482 	sblock = kzalloc(sizeof(*sblock), GFP_NOFS);  in scrub_pages_for_parity()
2483 	if (!sblock) {  in scrub_pages_for_parity()
2492 	atomic_set(&sblock->refs, 1);  in scrub_pages_for_parity()
2493 	sblock->sctx = sctx;  in scrub_pages_for_parity()
2494 	sblock->no_io_error_seen = 1;  in scrub_pages_for_parity()
2495 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2508 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2514 		sblock->pagev[index] = spage;  in scrub_pages_for_parity()
2518 		spage->sblock = sblock;  in scrub_pages_for_parity()
2531 		sblock->page_count++;  in scrub_pages_for_parity()
2540 	WARN_ON(sblock->page_count == 0);  in scrub_pages_for_parity()
2541 	for (index = 0; index < sblock->page_count; index++) {  in scrub_pages_for_parity()
2542 		struct scrub_page *spage = sblock->pagev[index];  in scrub_pages_for_parity()
2547 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2553 	scrub_block_put(sblock);  in scrub_pages_for_parity()