117 	struct scrub_parity	*sparity;  member
273 static void scrub_parity_get(struct scrub_parity *sparity);
274 static void scrub_parity_put(struct scrub_parity *sparity);
1673 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
2057 		if (sblock->sparity)  in scrub_block_put()
2058 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
2317 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,  in __scrub_mark_bitmap()  argument
2323 	int sectorsize = sparity->sctx->dev_root->sectorsize;  in __scrub_mark_bitmap()
2325 	if (len >= sparity->stripe_len) {  in __scrub_mark_bitmap()
2326 		bitmap_set(bitmap, 0, sparity->nsectors);  in __scrub_mark_bitmap()
2330 	start -= sparity->logic_start;  in __scrub_mark_bitmap()
2331 	start = div_u64_rem(start, sparity->stripe_len, &offset);  in __scrub_mark_bitmap()
2335 	if (offset + nsectors <= sparity->nsectors) {  in __scrub_mark_bitmap()
2340 	bitmap_set(bitmap, offset, sparity->nsectors - offset);  in __scrub_mark_bitmap()
2341 	bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));  in __scrub_mark_bitmap()
2344 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,  in scrub_parity_mark_sectors_error()  argument
2347 	__scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);  in scrub_parity_mark_sectors_error()
2350 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,  in scrub_parity_mark_sectors_data()  argument
2353 	__scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);  in scrub_parity_mark_sectors_data()
2374 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2379 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2473 static int scrub_pages_for_parity(struct scrub_parity *sparity,  in scrub_pages_for_parity()  argument
2478 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_pages_for_parity()
2495 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2496 	scrub_parity_get(sparity);  in scrub_pages_for_parity()
2517 		list_add_tail(&spage->list, &sparity->spages);  in scrub_pages_for_parity()
2557 static int scrub_extent_for_parity(struct scrub_parity *sparity,  in scrub_extent_for_parity()  argument
2562 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_extent_for_parity()
2586 		ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,  in scrub_extent_for_parity()
2643 static void scrub_free_parity(struct scrub_parity *sparity)  in scrub_free_parity()  argument
2645 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_free_parity()
2649 	nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);  in scrub_free_parity()
2657 	list_for_each_entry_safe(curr, next, &sparity->spages, list) {  in scrub_free_parity()
2662 	kfree(sparity);  in scrub_free_parity()
2667 	struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;  in scrub_parity_bio_endio()  local
2668 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_parity_bio_endio()
2671 		bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,  in scrub_parity_bio_endio()
2672 			  sparity->nsectors);  in scrub_parity_bio_endio()
2674 	scrub_free_parity(sparity);  in scrub_parity_bio_endio()
2679 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)  in scrub_parity_check_and_repair()  argument
2681 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_parity_check_and_repair()
2689 	if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,  in scrub_parity_check_and_repair()
2690 			   sparity->nsectors))  in scrub_parity_check_and_repair()
2693 	length = sparity->logic_end - sparity->logic_start + 1;  in scrub_parity_check_and_repair()
2695 			       sparity->logic_start,  in scrub_parity_check_and_repair()
2704 	bio->bi_iter.bi_sector = sparity->logic_start >> 9;  in scrub_parity_check_and_repair()
2705 	bio->bi_private = sparity;  in scrub_parity_check_and_repair()
2709 					      length, sparity->scrub_dev,  in scrub_parity_check_and_repair()
2710 					      sparity->dbitmap,  in scrub_parity_check_and_repair()
2711 					      sparity->nsectors);  in scrub_parity_check_and_repair()
2715 	list_for_each_entry(spage, &sparity->spages, list)  in scrub_parity_check_and_repair()
2727 	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,  in scrub_parity_check_and_repair()
2728 		  sparity->nsectors);  in scrub_parity_check_and_repair()
2733 	scrub_free_parity(sparity);  in scrub_parity_check_and_repair()
2741 static void scrub_parity_get(struct scrub_parity *sparity)  in scrub_parity_get()  argument
2743 	atomic_inc(&sparity->refs);  in scrub_parity_get()
2746 static void scrub_parity_put(struct scrub_parity *sparity)  in scrub_parity_put()  argument
2748 	if (!atomic_dec_and_test(&sparity->refs))  in scrub_parity_put()
2751 	scrub_parity_check_and_repair(sparity);  in scrub_parity_put()
2775 	struct scrub_parity *sparity;  in scrub_raid56_parity()  local
2783 	sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,  in scrub_raid56_parity()
2785 	if (!sparity) {  in scrub_raid56_parity()
2792 	sparity->stripe_len = map->stripe_len;  in scrub_raid56_parity()
2793 	sparity->nsectors = nsectors;  in scrub_raid56_parity()
2794 	sparity->sctx = sctx;  in scrub_raid56_parity()
2795 	sparity->scrub_dev = sdev;  in scrub_raid56_parity()
2796 	sparity->logic_start = logic_start;  in scrub_raid56_parity()
2797 	sparity->logic_end = logic_end;  in scrub_raid56_parity()
2798 	atomic_set(&sparity->refs, 1);  in scrub_raid56_parity()
2799 	INIT_LIST_HEAD(&sparity->spages);  in scrub_raid56_parity()
2800 	sparity->dbitmap = sparity->bitmap;  in scrub_raid56_parity()
2801 	sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;  in scrub_raid56_parity()
2893 			scrub_parity_mark_sectors_data(sparity, extent_logical,  in scrub_raid56_parity()
2908 			ret = scrub_extent_for_parity(sparity, extent_logical,  in scrub_raid56_parity()
2945 		scrub_parity_mark_sectors_error(sparity, logic_start,  in scrub_raid56_parity()
2947 	scrub_parity_put(sparity);  in scrub_raid56_parity()