429 		struct scrub_bio *sbio = sctx->bios[sctx->curr];  in scrub_free_ctx()  local
431 		for (i = 0; i < sbio->page_count; i++) {  in scrub_free_ctx()
432 			WARN_ON(!sbio->pagev[i]->page);  in scrub_free_ctx()
433 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
435 		bio_put(sbio->bio);  in scrub_free_ctx()
439 		struct scrub_bio *sbio = sctx->bios[i];  in scrub_free_ctx()  local
441 		if (!sbio)  in scrub_free_ctx()
443 		kfree(sbio);  in scrub_free_ctx()
473 		struct scrub_bio *sbio;  in scrub_setup_ctx()  local
475 		sbio = kzalloc(sizeof(*sbio), GFP_NOFS);  in scrub_setup_ctx()
476 		if (!sbio)  in scrub_setup_ctx()
478 		sctx->bios[i] = sbio;  in scrub_setup_ctx()
480 		sbio->index = i;  in scrub_setup_ctx()
481 		sbio->sctx = sctx;  in scrub_setup_ctx()
482 		sbio->page_count = 0;  in scrub_setup_ctx()
483 		btrfs_init_work(&sbio->work, btrfs_scrub_helper,  in scrub_setup_ctx()
1652 	struct scrub_bio *sbio;  in scrub_add_page_to_wr_bio()  local
1667 	sbio = wr_ctx->wr_curr_bio;  in scrub_add_page_to_wr_bio()
1668 	if (sbio->page_count == 0) {  in scrub_add_page_to_wr_bio()
1671 		sbio->physical = spage->physical_for_dev_replace;  in scrub_add_page_to_wr_bio()
1672 		sbio->logical = spage->logical;  in scrub_add_page_to_wr_bio()
1673 		sbio->dev = wr_ctx->tgtdev;  in scrub_add_page_to_wr_bio()
1674 		bio = sbio->bio;  in scrub_add_page_to_wr_bio()
1681 			sbio->bio = bio;  in scrub_add_page_to_wr_bio()
1684 		bio->bi_private = sbio;  in scrub_add_page_to_wr_bio()
1686 		bio->bi_bdev = sbio->dev->bdev;  in scrub_add_page_to_wr_bio()
1687 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_wr_bio()
1688 		sbio->err = 0;  in scrub_add_page_to_wr_bio()
1689 	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_wr_bio()
1691 		   sbio->logical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_wr_bio()
1697 	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);  in scrub_add_page_to_wr_bio()
1699 		if (sbio->page_count < 1) {  in scrub_add_page_to_wr_bio()
1700 			bio_put(sbio->bio);  in scrub_add_page_to_wr_bio()
1701 			sbio->bio = NULL;  in scrub_add_page_to_wr_bio()
1709 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_wr_bio()
1711 	sbio->page_count++;  in scrub_add_page_to_wr_bio()
1712 	if (sbio->page_count == wr_ctx->pages_per_wr_bio)  in scrub_add_page_to_wr_bio()
1722 	struct scrub_bio *sbio;  in scrub_wr_submit()  local
1727 	sbio = wr_ctx->wr_curr_bio;  in scrub_wr_submit()
1729 	WARN_ON(!sbio->bio->bi_bdev);  in scrub_wr_submit()
1735 	btrfsic_submit_bio(WRITE, sbio->bio);  in scrub_wr_submit()
1740 	struct scrub_bio *sbio = bio->bi_private;  in scrub_wr_bio_end_io()  local
1741 	struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;  in scrub_wr_bio_end_io()
1743 	sbio->err = bio->bi_error;  in scrub_wr_bio_end_io()
1744 	sbio->bio = bio;  in scrub_wr_bio_end_io()
1746 	btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,  in scrub_wr_bio_end_io()
1748 	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);  in scrub_wr_bio_end_io()
1753 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_wr_bio_end_io_worker()  local
1754 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_wr_bio_end_io_worker()
1757 	WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);  in scrub_wr_bio_end_io_worker()
1758 	if (sbio->err) {  in scrub_wr_bio_end_io_worker()
1760 			&sbio->sctx->dev_root->fs_info->dev_replace;  in scrub_wr_bio_end_io_worker()
1762 		for (i = 0; i < sbio->page_count; i++) {  in scrub_wr_bio_end_io_worker()
1763 			struct scrub_page *spage = sbio->pagev[i];  in scrub_wr_bio_end_io_worker()
1771 	for (i = 0; i < sbio->page_count; i++)  in scrub_wr_bio_end_io_worker()
1772 		scrub_page_put(sbio->pagev[i]);  in scrub_wr_bio_end_io_worker()
1774 	bio_put(sbio->bio);  in scrub_wr_bio_end_io_worker()
1775 	kfree(sbio);  in scrub_wr_bio_end_io_worker()
2037 	struct scrub_bio *sbio;  in scrub_submit()  local
2042 	sbio = sctx->bios[sctx->curr];  in scrub_submit()
2045 	btrfsic_submit_bio(READ, sbio->bio);  in scrub_submit()
2052 	struct scrub_bio *sbio;  in scrub_add_page_to_rd_bio()  local
2072 	sbio = sctx->bios[sctx->curr];  in scrub_add_page_to_rd_bio()
2073 	if (sbio->page_count == 0) {  in scrub_add_page_to_rd_bio()
2076 		sbio->physical = spage->physical;  in scrub_add_page_to_rd_bio()
2077 		sbio->logical = spage->logical;  in scrub_add_page_to_rd_bio()
2078 		sbio->dev = spage->dev;  in scrub_add_page_to_rd_bio()
2079 		bio = sbio->bio;  in scrub_add_page_to_rd_bio()
2084 			sbio->bio = bio;  in scrub_add_page_to_rd_bio()
2087 		bio->bi_private = sbio;  in scrub_add_page_to_rd_bio()
2089 		bio->bi_bdev = sbio->dev->bdev;  in scrub_add_page_to_rd_bio()
2090 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_rd_bio()
2091 		sbio->err = 0;  in scrub_add_page_to_rd_bio()
2092 	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_rd_bio()
2094 		   sbio->logical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_rd_bio()
2096 		   sbio->dev != spage->dev) {  in scrub_add_page_to_rd_bio()
2101 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_rd_bio()
2102 	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);  in scrub_add_page_to_rd_bio()
2104 		if (sbio->page_count < 1) {  in scrub_add_page_to_rd_bio()
2105 			bio_put(sbio->bio);  in scrub_add_page_to_rd_bio()
2106 			sbio->bio = NULL;  in scrub_add_page_to_rd_bio()
2115 	sbio->page_count++;  in scrub_add_page_to_rd_bio()
2116 	if (sbio->page_count == sctx->pages_per_rd_bio)  in scrub_add_page_to_rd_bio()
2330 	struct scrub_bio *sbio = bio->bi_private;  in scrub_bio_end_io()  local
2331 	struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;  in scrub_bio_end_io()
2333 	sbio->err = bio->bi_error;  in scrub_bio_end_io()
2334 	sbio->bio = bio;  in scrub_bio_end_io()
2336 	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);  in scrub_bio_end_io()
2341 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_bio_end_io_worker()  local
2342 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_bio_end_io_worker()
2345 	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);  in scrub_bio_end_io_worker()
2346 	if (sbio->err) {  in scrub_bio_end_io_worker()
2347 		for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2348 			struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2356 	for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2357 		struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2365 	bio_put(sbio->bio);  in scrub_bio_end_io_worker()
2366 	sbio->bio = NULL;  in scrub_bio_end_io_worker()
2368 	sbio->next_free = sctx->first_free;  in scrub_bio_end_io_worker()
2369 	sctx->first_free = sbio->index;  in scrub_bio_end_io_worker()