115 static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)  in io_hash_desc()  argument
117 	return (struct shash_desc *)(io + 1);  in io_hash_desc()
120 static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)  in io_real_digest()  argument
122 	return (u8 *)(io + 1) + v->shash_descsize;  in io_real_digest()
125 static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)  in io_want_digest()  argument
127 	return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;  in io_want_digest()
256 static int verity_verify_level(struct dm_verity_io *io, sector_t block,  in verity_verify_level()  argument
259 	struct dm_verity *v = io->v;  in verity_verify_level()
284 		desc = io_hash_desc(v, io);  in verity_verify_level()
315 		result = io_real_digest(v, io);  in verity_verify_level()
321 		if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {  in verity_verify_level()
333 	memcpy(io_want_digest(v, io), data, v->digest_size);  in verity_verify_level()
347 static int verity_verify_io(struct dm_verity_io *io)  in verity_verify_io()  argument
349 	struct dm_verity *v = io->v;  in verity_verify_io()
350 	struct bio *bio = dm_bio_from_per_bio_data(io,  in verity_verify_io()
355 	for (b = 0; b < io->n_blocks; b++) {  in verity_verify_io()
369 			int r = verity_verify_level(io, io->block + b, 0, true);  in verity_verify_io()
376 		memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);  in verity_verify_io()
379 			int r = verity_verify_level(io, io->block + b, i, false);  in verity_verify_io()
385 		desc = io_hash_desc(v, io);  in verity_verify_io()
405 			struct bio_vec bv = bio_iter_iovec(bio, io->iter);  in verity_verify_io()
419 			bio_advance_iter(bio, &io->iter, len);  in verity_verify_io()
431 		result = io_real_digest(v, io);  in verity_verify_io()
437 		if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {  in verity_verify_io()
439 					      io->block + b))  in verity_verify_io()
450 static void verity_finish_io(struct dm_verity_io *io, int error)  in verity_finish_io()  argument
452 	struct dm_verity *v = io->v;  in verity_finish_io()
453 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);  in verity_finish_io()
455 	bio->bi_end_io = io->orig_bi_end_io;  in verity_finish_io()
456 	bio->bi_private = io->orig_bi_private;  in verity_finish_io()
464 	struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);  in verity_work()  local
466 	verity_finish_io(io, verity_verify_io(io));  in verity_work()
471 	struct dm_verity_io *io = bio->bi_private;  in verity_end_io()  local
474 		verity_finish_io(io, bio->bi_error);  in verity_end_io()
478 	INIT_WORK(&io->work, verity_work);  in verity_end_io()
479 	queue_work(io->v->verify_wq, &io->work);  in verity_end_io()
522 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)  in verity_submit_prefetch()  argument
534 	pw->block = io->block;  in verity_submit_prefetch()
535 	pw->n_blocks = io->n_blocks;  in verity_submit_prefetch()
546 	struct dm_verity_io *io;  in verity_map()  local
566 	io = dm_per_bio_data(bio, ti->per_bio_data_size);  in verity_map()
567 	io->v = v;  in verity_map()
568 	io->orig_bi_end_io = bio->bi_end_io;  in verity_map()
569 	io->orig_bi_private = bio->bi_private;  in verity_map()
570 	io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);  in verity_map()
571 	io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;  in verity_map()
574 	bio->bi_private = io;  in verity_map()
575 	io->iter = bio->bi_iter;  in verity_map()
577 	verity_submit_prefetch(v, io);  in verity_map()