117 	struct scrub_parity	*sparity;  member
269 static void scrub_parity_get(struct scrub_parity *sparity);
270 static void scrub_parity_put(struct scrub_parity *sparity);
1618 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
2012 		if (sblock->sparity)  in scrub_block_put()
2013 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
2382 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,  in __scrub_mark_bitmap()  argument
2388 	int sectorsize = sparity->sctx->dev_root->sectorsize;  in __scrub_mark_bitmap()
2390 	if (len >= sparity->stripe_len) {  in __scrub_mark_bitmap()
2391 		bitmap_set(bitmap, 0, sparity->nsectors);  in __scrub_mark_bitmap()
2395 	start -= sparity->logic_start;  in __scrub_mark_bitmap()
2396 	start = div_u64_rem(start, sparity->stripe_len, &offset);  in __scrub_mark_bitmap()
2400 	if (offset + nsectors <= sparity->nsectors) {  in __scrub_mark_bitmap()
2405 	bitmap_set(bitmap, offset, sparity->nsectors - offset);  in __scrub_mark_bitmap()
2406 	bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));  in __scrub_mark_bitmap()
2409 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,  in scrub_parity_mark_sectors_error()  argument
2412 	__scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);  in scrub_parity_mark_sectors_error()
2415 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,  in scrub_parity_mark_sectors_data()  argument
2418 	__scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);  in scrub_parity_mark_sectors_data()
2439 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2444 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2537 static int scrub_pages_for_parity(struct scrub_parity *sparity,  in scrub_pages_for_parity()  argument
2542 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_pages_for_parity()
2559 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2560 	scrub_parity_get(sparity);  in scrub_pages_for_parity()
2581 		list_add_tail(&spage->list, &sparity->spages);  in scrub_pages_for_parity()
2621 static int scrub_extent_for_parity(struct scrub_parity *sparity,  in scrub_extent_for_parity()  argument
2626 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_extent_for_parity()
2632 		scrub_parity_mark_sectors_error(sparity, logical, len);  in scrub_extent_for_parity()
2655 		ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,  in scrub_extent_for_parity()
2712 static void scrub_free_parity(struct scrub_parity *sparity)  in scrub_free_parity()  argument
2714 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_free_parity()
2718 	nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);  in scrub_free_parity()
2726 	list_for_each_entry_safe(curr, next, &sparity->spages, list) {  in scrub_free_parity()
2731 	kfree(sparity);  in scrub_free_parity()
2736 	struct scrub_parity *sparity = container_of(work, struct scrub_parity,  in scrub_parity_bio_endio_worker()  local
2738 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_parity_bio_endio_worker()
2740 	scrub_free_parity(sparity);  in scrub_parity_bio_endio_worker()
2746 	struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;  in scrub_parity_bio_endio()  local
2749 		bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,  in scrub_parity_bio_endio()
2750 			  sparity->nsectors);  in scrub_parity_bio_endio()
2754 	btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,  in scrub_parity_bio_endio()
2756 	btrfs_queue_work(sparity->sctx->dev_root->fs_info->scrub_parity_workers,  in scrub_parity_bio_endio()
2757 			 &sparity->work);  in scrub_parity_bio_endio()
2760 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)  in scrub_parity_check_and_repair()  argument
2762 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_parity_check_and_repair()
2770 	if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,  in scrub_parity_check_and_repair()
2771 			   sparity->nsectors))  in scrub_parity_check_and_repair()
2774 	length = sparity->logic_end - sparity->logic_start;  in scrub_parity_check_and_repair()
2776 			       sparity->logic_start,  in scrub_parity_check_and_repair()
2785 	bio->bi_iter.bi_sector = sparity->logic_start >> 9;  in scrub_parity_check_and_repair()
2786 	bio->bi_private = sparity;  in scrub_parity_check_and_repair()
2790 					      length, sparity->scrub_dev,  in scrub_parity_check_and_repair()
2791 					      sparity->dbitmap,  in scrub_parity_check_and_repair()
2792 					      sparity->nsectors);  in scrub_parity_check_and_repair()
2796 	list_for_each_entry(spage, &sparity->spages, list)  in scrub_parity_check_and_repair()
2807 	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,  in scrub_parity_check_and_repair()
2808 		  sparity->nsectors);  in scrub_parity_check_and_repair()
2813 	scrub_free_parity(sparity);  in scrub_parity_check_and_repair()
2821 static void scrub_parity_get(struct scrub_parity *sparity)  in scrub_parity_get()  argument
2823 	atomic_inc(&sparity->refs);  in scrub_parity_get()
2826 static void scrub_parity_put(struct scrub_parity *sparity)  in scrub_parity_put()  argument
2828 	if (!atomic_dec_and_test(&sparity->refs))  in scrub_parity_put()
2831 	scrub_parity_check_and_repair(sparity);  in scrub_parity_put()
2857 	struct scrub_parity *sparity;  in scrub_raid56_parity()  local
2865 	sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,  in scrub_raid56_parity()
2867 	if (!sparity) {  in scrub_raid56_parity()
2874 	sparity->stripe_len = map->stripe_len;  in scrub_raid56_parity()
2875 	sparity->nsectors = nsectors;  in scrub_raid56_parity()
2876 	sparity->sctx = sctx;  in scrub_raid56_parity()
2877 	sparity->scrub_dev = sdev;  in scrub_raid56_parity()
2878 	sparity->logic_start = logic_start;  in scrub_raid56_parity()
2879 	sparity->logic_end = logic_end;  in scrub_raid56_parity()
2880 	atomic_set(&sparity->refs, 1);  in scrub_raid56_parity()
2881 	INIT_LIST_HEAD(&sparity->spages);  in scrub_raid56_parity()
2882 	sparity->dbitmap = sparity->bitmap;  in scrub_raid56_parity()
2883 	sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;  in scrub_raid56_parity()
2979 			scrub_parity_mark_sectors_data(sparity, extent_logical,  in scrub_raid56_parity()
3005 			ret = scrub_extent_for_parity(sparity, extent_logical,  in scrub_raid56_parity()
3044 		scrub_parity_mark_sectors_error(sparity, logic_start,  in scrub_raid56_parity()
3046 	scrub_parity_put(sparity);  in scrub_raid56_parity()