45 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,  in bio_integrity_alloc()  argument
50 	struct bio_set *bs = bio->bi_pool;  in bio_integrity_alloc()
80 	bip->bip_bio = bio;  in bio_integrity_alloc()
81 	bio->bi_integrity = bip;  in bio_integrity_alloc()
82 	bio->bi_rw |= REQ_INTEGRITY;  in bio_integrity_alloc()
98 void bio_integrity_free(struct bio *bio)  in bio_integrity_free()  argument
100 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_free()
101 	struct bio_set *bs = bio->bi_pool;  in bio_integrity_free()
117 	bio->bi_integrity = NULL;  in bio_integrity_free()
130 int bio_integrity_add_page(struct bio *bio, struct page *page,  in bio_integrity_add_page()  argument
133 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_add_page()
161 bool bio_integrity_enabled(struct bio *bio)  in bio_integrity_enabled()  argument
163 	struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);  in bio_integrity_enabled()
165 	if (!bio_is_rw(bio))  in bio_integrity_enabled()
169 	if (bio_integrity(bio))  in bio_integrity_enabled()
175 	if (bio_data_dir(bio) == READ && bi->verify_fn != NULL &&  in bio_integrity_enabled()
179 	if (bio_data_dir(bio) == WRITE && bi->generate_fn != NULL &&  in bio_integrity_enabled()
214 static int bio_integrity_process(struct bio *bio,  in bio_integrity_process()  argument
217 	struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);  in bio_integrity_process()
221 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_process()
226 	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;  in bio_integrity_process()
231 	bio_for_each_segment(bv, bio, bviter) {  in bio_integrity_process()
259 int bio_integrity_prep(struct bio *bio)  in bio_integrity_prep()  argument
270 	bi = bdev_get_integrity(bio->bi_bdev);  in bio_integrity_prep()
271 	q = bdev_get_queue(bio->bi_bdev);  in bio_integrity_prep()
273 	BUG_ON(bio_integrity(bio));  in bio_integrity_prep()
275 	intervals = bio_integrity_intervals(bi, bio_sectors(bio));  in bio_integrity_prep()
290 	bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);  in bio_integrity_prep()
299 	bip_set_seed(bip, bio->bi_iter.bi_sector);  in bio_integrity_prep()
316 		ret = bio_integrity_add_page(bio, virt_to_page(buf),  in bio_integrity_prep()
331 	if (bio_data_dir(bio) == READ) {  in bio_integrity_prep()
332 		bip->bip_end_io = bio->bi_end_io;  in bio_integrity_prep()
333 		bio->bi_end_io = bio_integrity_endio;  in bio_integrity_prep()
337 	if (bio_data_dir(bio) == WRITE)  in bio_integrity_prep()
338 		bio_integrity_process(bio, bi->generate_fn);  in bio_integrity_prep()
356 	struct bio *bio = bip->bip_bio;  in bio_integrity_verify_fn()  local
357 	struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);  in bio_integrity_verify_fn()
360 	error = bio_integrity_process(bio, bi->verify_fn);  in bio_integrity_verify_fn()
363 	bio->bi_end_io = bip->bip_end_io;  in bio_integrity_verify_fn()
364 	bio_endio_nodec(bio, error);  in bio_integrity_verify_fn()
379 void bio_integrity_endio(struct bio *bio, int error)  in bio_integrity_endio()  argument
381 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_endio()
383 	BUG_ON(bip->bip_bio != bio);  in bio_integrity_endio()
390 		bio->bi_end_io = bip->bip_end_io;  in bio_integrity_endio()
391 		bio_endio_nodec(bio, error);  in bio_integrity_endio()
410 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)  in bio_integrity_advance()  argument
412 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_advance()
413 	struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);  in bio_integrity_advance()
431 void bio_integrity_trim(struct bio *bio, unsigned int offset,  in bio_integrity_trim()  argument
434 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_trim()
435 	struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);  in bio_integrity_trim()
437 	bio_integrity_advance(bio, offset << 9);  in bio_integrity_trim()
450 int bio_integrity_clone(struct bio *bio, struct bio *bio_src,  in bio_integrity_clone()  argument
458 	bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt);  in bio_integrity_clone()