73 	struct scrub_block	*sblock;  member
251 				struct scrub_block *sblock,
253 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
259 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
260 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
262 static int scrub_checksum_data(struct scrub_block *sblock);
263 static int scrub_checksum_tree_block(struct scrub_block *sblock);
264 static int scrub_checksum_super(struct scrub_block *sblock);
265 static void scrub_block_get(struct scrub_block *sblock);
266 static void scrub_block_put(struct scrub_block *sblock);
279 static void scrub_block_complete(struct scrub_block *sblock);
433 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
600 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)  in scrub_print_warning()  argument
617 	WARN_ON(sblock->page_count < 1);  in scrub_print_warning()
618 	dev = sblock->pagev[0]->dev;  in scrub_print_warning()
619 	fs_info = sblock->sctx->dev_root->fs_info;  in scrub_print_warning()
625 	swarn.sector = (sblock->pagev[0]->physical) >> 9;  in scrub_print_warning()
626 	swarn.logical = sblock->pagev[0]->logical;  in scrub_print_warning()
1237 			struct scrub_block *sblock = sblocks_for_recheck +  in scrub_handle_errored_block()  local
1242 			for (page_index = 0; page_index < sblock->page_count;  in scrub_handle_errored_block()
1244 				sblock->pagev[page_index]->sblock = NULL;  in scrub_handle_errored_block()
1245 				recover = sblock->pagev[page_index]->recover;  in scrub_handle_errored_block()
1248 					sblock->pagev[page_index]->recover =  in scrub_handle_errored_block()
1251 				scrub_page_put(sblock->pagev[page_index]);  in scrub_handle_errored_block()
1359 			struct scrub_block *sblock;  in scrub_setup_recheck_block()  local
1362 			sblock = sblocks_for_recheck + mirror_index;  in scrub_setup_recheck_block()
1363 			sblock->sctx = sctx;  in scrub_setup_recheck_block()
1375 			sblock->pagev[page_index] = page;  in scrub_setup_recheck_block()
1376 			page->sblock = sblock;  in scrub_setup_recheck_block()
1405 			sblock->page_count++;  in scrub_setup_recheck_block()
1475 				struct scrub_block *sblock,  in scrub_recheck_block()  argument
1480 	sblock->no_io_error_seen = 1;  in scrub_recheck_block()
1482 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block()
1484 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block()
1488 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1496 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1504 				sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1509 				sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1515 	if (sblock->no_io_error_seen)  in scrub_recheck_block()
1516 		scrub_recheck_block_checksum(sblock);  in scrub_recheck_block()
1531 static void scrub_recheck_block_checksum(struct scrub_block *sblock)  in scrub_recheck_block_checksum()  argument
1533 	sblock->header_error = 0;  in scrub_recheck_block_checksum()
1534 	sblock->checksum_error = 0;  in scrub_recheck_block_checksum()
1535 	sblock->generation_error = 0;  in scrub_recheck_block_checksum()
1537 	if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)  in scrub_recheck_block_checksum()
1538 		scrub_checksum_data(sblock);  in scrub_recheck_block_checksum()
1540 		scrub_checksum_tree_block(sblock);  in scrub_recheck_block_checksum()
1610 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)  in scrub_write_block_to_dev_replace()  argument
1618 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1621 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_write_block_to_dev_replace()
1624 		ret = scrub_write_page_to_dev_replace(sblock, page_num);  in scrub_write_block_to_dev_replace()
1627 				&sblock->sctx->dev_root->fs_info->dev_replace.  in scrub_write_block_to_dev_replace()
1632 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,  in scrub_write_page_to_dev_replace()  argument
1635 	struct scrub_page *spage = sblock->pagev[page_num];  in scrub_write_page_to_dev_replace()
1645 	return scrub_add_page_to_wr_bio(sblock->sctx, spage);  in scrub_write_page_to_dev_replace()
1779 static int scrub_checksum(struct scrub_block *sblock)  in scrub_checksum()  argument
1792 	sblock->header_error = 0;  in scrub_checksum()
1793 	sblock->generation_error = 0;  in scrub_checksum()
1794 	sblock->checksum_error = 0;  in scrub_checksum()
1796 	WARN_ON(sblock->page_count < 1);  in scrub_checksum()
1797 	flags = sblock->pagev[0]->flags;  in scrub_checksum()
1800 		ret = scrub_checksum_data(sblock);  in scrub_checksum()
1802 		ret = scrub_checksum_tree_block(sblock);  in scrub_checksum()
1804 		(void)scrub_checksum_super(sblock);  in scrub_checksum()
1808 		scrub_handle_errored_block(sblock);  in scrub_checksum()
1813 static int scrub_checksum_data(struct scrub_block *sblock)  in scrub_checksum_data()  argument
1815 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_data()
1824 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_data()
1825 	if (!sblock->pagev[0]->have_csum)  in scrub_checksum_data()
1828 	on_disk_csum = sblock->pagev[0]->csum;  in scrub_checksum_data()
1829 	page = sblock->pagev[0]->page;  in scrub_checksum_data()
1843 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_data()
1844 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_data()
1845 		page = sblock->pagev[index]->page;  in scrub_checksum_data()
1851 		sblock->checksum_error = 1;  in scrub_checksum_data()
1853 	return sblock->checksum_error;  in scrub_checksum_data()
1856 static int scrub_checksum_tree_block(struct scrub_block *sblock)  in scrub_checksum_tree_block()  argument
1858 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_tree_block()
1872 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_tree_block()
1873 	page = sblock->pagev[0]->page;  in scrub_checksum_tree_block()
1883 	if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))  in scrub_checksum_tree_block()
1884 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1886 	if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {  in scrub_checksum_tree_block()
1887 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1888 		sblock->generation_error = 1;  in scrub_checksum_tree_block()
1891 	if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))  in scrub_checksum_tree_block()
1892 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1896 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1911 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_tree_block()
1912 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_tree_block()
1913 		page = sblock->pagev[index]->page;  in scrub_checksum_tree_block()
1921 		sblock->checksum_error = 1;  in scrub_checksum_tree_block()
1923 	return sblock->header_error || sblock->checksum_error;  in scrub_checksum_tree_block()
1926 static int scrub_checksum_super(struct scrub_block *sblock)  in scrub_checksum_super()  argument
1929 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_super()
1942 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_super()
1943 	page = sblock->pagev[0]->page;  in scrub_checksum_super()
1948 	if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))  in scrub_checksum_super()
1951 	if (sblock->pagev[0]->generation != btrfs_super_generation(s))  in scrub_checksum_super()
1954 	if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))  in scrub_checksum_super()
1970 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_super()
1971 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_super()
1972 		page = sblock->pagev[index]->page;  in scrub_checksum_super()
1992 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
1995 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
2002 static void scrub_block_get(struct scrub_block *sblock)  in scrub_block_get()  argument
2004 	atomic_inc(&sblock->refs);  in scrub_block_get()
2007 static void scrub_block_put(struct scrub_block *sblock)  in scrub_block_put()  argument
2009 	if (atomic_dec_and_test(&sblock->refs)) {  in scrub_block_put()
2012 		if (sblock->sparity)  in scrub_block_put()
2013 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
2015 		for (i = 0; i < sblock->page_count; i++)  in scrub_block_put()
2016 			scrub_page_put(sblock->pagev[i]);  in scrub_block_put()
2017 		kfree(sblock);  in scrub_block_put()
2051 	struct scrub_block *sblock = spage->sblock;  in scrub_add_page_to_rd_bio()  local
2113 	scrub_block_get(sblock); /* one for the page added to the bio */  in scrub_add_page_to_rd_bio()
2114 	atomic_inc(&sblock->outstanding_pages);  in scrub_add_page_to_rd_bio()
2124 	struct scrub_block *sblock = bio->bi_private;  in scrub_missing_raid56_end_io()  local
2125 	struct btrfs_fs_info *fs_info = sblock->sctx->dev_root->fs_info;  in scrub_missing_raid56_end_io()
2128 		sblock->no_io_error_seen = 0;  in scrub_missing_raid56_end_io()
2130 	btrfs_queue_work(fs_info->scrub_workers, &sblock->work);  in scrub_missing_raid56_end_io()
2135 	struct scrub_block *sblock = container_of(work, struct scrub_block, work);  in scrub_missing_raid56_worker()  local
2136 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_worker()
2140 	logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_worker()
2141 	dev = sblock->pagev[0]->dev;  in scrub_missing_raid56_worker()
2143 	if (sblock->no_io_error_seen)  in scrub_missing_raid56_worker()
2144 		scrub_recheck_block_checksum(sblock);  in scrub_missing_raid56_worker()
2146 	if (!sblock->no_io_error_seen) {  in scrub_missing_raid56_worker()
2153 	} else if (sblock->header_error || sblock->checksum_error) {  in scrub_missing_raid56_worker()
2161 		scrub_write_block_to_dev_replace(sblock);  in scrub_missing_raid56_worker()
2164 	scrub_block_put(sblock);  in scrub_missing_raid56_worker()
2176 static void scrub_missing_raid56_pages(struct scrub_block *sblock)  in scrub_missing_raid56_pages()  argument
2178 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_pages()
2180 	u64 length = sblock->page_count * PAGE_SIZE;  in scrub_missing_raid56_pages()
2181 	u64 logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_pages()
2209 	bio->bi_private = sblock;  in scrub_missing_raid56_pages()
2216 	for (i = 0; i < sblock->page_count; i++) {  in scrub_missing_raid56_pages()
2217 		struct scrub_page *spage = sblock->pagev[i];  in scrub_missing_raid56_pages()
2222 	btrfs_init_work(&sblock->work, btrfs_scrub_helper,  in scrub_missing_raid56_pages()
2224 	scrub_block_get(sblock);  in scrub_missing_raid56_pages()
2243 	struct scrub_block *sblock;  in scrub_pages()  local
2246 	sblock = kzalloc(sizeof(*sblock), GFP_NOFS);  in scrub_pages()
2247 	if (!sblock) {  in scrub_pages()
2256 	atomic_set(&sblock->refs, 1);  in scrub_pages()
2257 	sblock->sctx = sctx;  in scrub_pages()
2258 	sblock->no_io_error_seen = 1;  in scrub_pages()
2270 			scrub_block_put(sblock);  in scrub_pages()
2275 		sblock->pagev[index] = spage;  in scrub_pages()
2276 		spage->sblock = sblock;  in scrub_pages()
2290 		sblock->page_count++;  in scrub_pages()
2300 	WARN_ON(sblock->page_count == 0);  in scrub_pages()
2306 		scrub_missing_raid56_pages(sblock);  in scrub_pages()
2308 		for (index = 0; index < sblock->page_count; index++) {  in scrub_pages()
2309 			struct scrub_page *spage = sblock->pagev[index];  in scrub_pages()
2314 				scrub_block_put(sblock);  in scrub_pages()
2324 	scrub_block_put(sblock);  in scrub_pages()
2351 			spage->sblock->no_io_error_seen = 0;  in scrub_bio_end_io_worker()
2358 		struct scrub_block *sblock = spage->sblock;  in scrub_bio_end_io_worker()  local
2360 		if (atomic_dec_and_test(&sblock->outstanding_pages))  in scrub_bio_end_io_worker()
2361 			scrub_block_complete(sblock);  in scrub_bio_end_io_worker()
2362 		scrub_block_put(sblock);  in scrub_bio_end_io_worker()
2421 static void scrub_block_complete(struct scrub_block *sblock)  in scrub_block_complete()  argument
2425 	if (!sblock->no_io_error_seen) {  in scrub_block_complete()
2427 		scrub_handle_errored_block(sblock);  in scrub_block_complete()
2434 		corrupted = scrub_checksum(sblock);  in scrub_block_complete()
2435 		if (!corrupted && sblock->sctx->is_dev_replace)  in scrub_block_complete()
2436 			scrub_write_block_to_dev_replace(sblock);  in scrub_block_complete()
2439 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2440 		u64 start = sblock->pagev[0]->logical;  in scrub_block_complete()
2441 		u64 end = sblock->pagev[sblock->page_count - 1]->logical +  in scrub_block_complete()
2444 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2543 	struct scrub_block *sblock;  in scrub_pages_for_parity()  local
2546 	sblock = kzalloc(sizeof(*sblock), GFP_NOFS);  in scrub_pages_for_parity()
2547 	if (!sblock) {  in scrub_pages_for_parity()
2556 	atomic_set(&sblock->refs, 1);  in scrub_pages_for_parity()
2557 	sblock->sctx = sctx;  in scrub_pages_for_parity()
2558 	sblock->no_io_error_seen = 1;  in scrub_pages_for_parity()
2559 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2572 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2578 		sblock->pagev[index] = spage;  in scrub_pages_for_parity()
2582 		spage->sblock = sblock;  in scrub_pages_for_parity()
2595 		sblock->page_count++;  in scrub_pages_for_parity()
2604 	WARN_ON(sblock->page_count == 0);  in scrub_pages_for_parity()
2605 	for (index = 0; index < sblock->page_count; index++) {  in scrub_pages_for_parity()
2606 		struct scrub_page *spage = sblock->pagev[index];  in scrub_pages_for_parity()
2611 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2617 	scrub_block_put(sblock);  in scrub_pages_for_parity()