119 static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)  in io_hash_desc()  argument
121 	return (struct shash_desc *)(io + 1);  in io_hash_desc()
124 static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)  in io_real_digest()  argument
126 	return (u8 *)(io + 1) + v->shash_descsize;  in io_real_digest()
129 static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)  in io_want_digest()  argument
131 	return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;  in io_want_digest()
260 static int verity_verify_level(struct dm_verity_io *io, sector_t block,  in verity_verify_level()  argument
263 	struct dm_verity *v = io->v;  in verity_verify_level()
288 		desc = io_hash_desc(v, io);  in verity_verify_level()
319 		result = io_real_digest(v, io);  in verity_verify_level()
325 		if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {  in verity_verify_level()
337 	memcpy(io_want_digest(v, io), data, v->digest_size);  in verity_verify_level()
351 static int verity_verify_io(struct dm_verity_io *io)  in verity_verify_io()  argument
353 	struct dm_verity *v = io->v;  in verity_verify_io()
354 	struct bio *bio = dm_bio_from_per_bio_data(io,  in verity_verify_io()
359 	for (b = 0; b < io->n_blocks; b++) {  in verity_verify_io()
373 			int r = verity_verify_level(io, io->block + b, 0, true);  in verity_verify_io()
380 		memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);  in verity_verify_io()
383 			int r = verity_verify_level(io, io->block + b, i, false);  in verity_verify_io()
389 		desc = io_hash_desc(v, io);  in verity_verify_io()
409 			struct bio_vec bv = bio_iter_iovec(bio, io->iter);  in verity_verify_io()
423 			bio_advance_iter(bio, &io->iter, len);  in verity_verify_io()
435 		result = io_real_digest(v, io);  in verity_verify_io()
441 		if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {  in verity_verify_io()
443 					      io->block + b))  in verity_verify_io()
454 static void verity_finish_io(struct dm_verity_io *io, int error)  in verity_finish_io()  argument
456 	struct dm_verity *v = io->v;  in verity_finish_io()
457 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);  in verity_finish_io()
459 	bio->bi_end_io = io->orig_bi_end_io;  in verity_finish_io()
460 	bio->bi_private = io->orig_bi_private;  in verity_finish_io()
467 	struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);  in verity_work()  local
469 	verity_finish_io(io, verity_verify_io(io));  in verity_work()
474 	struct dm_verity_io *io = bio->bi_private;  in verity_end_io()  local
477 		verity_finish_io(io, error);  in verity_end_io()
481 	INIT_WORK(&io->work, verity_work);  in verity_end_io()
482 	queue_work(io->v->verify_wq, &io->work);  in verity_end_io()
525 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)  in verity_submit_prefetch()  argument
537 	pw->block = io->block;  in verity_submit_prefetch()
538 	pw->n_blocks = io->n_blocks;  in verity_submit_prefetch()
549 	struct dm_verity_io *io;  in verity_map()  local
569 	io = dm_per_bio_data(bio, ti->per_bio_data_size);  in verity_map()
570 	io->v = v;  in verity_map()
571 	io->orig_bi_end_io = bio->bi_end_io;  in verity_map()
572 	io->orig_bi_private = bio->bi_private;  in verity_map()
573 	io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);  in verity_map()
574 	io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;  in verity_map()
577 	bio->bi_private = io;  in verity_map()
578 	io->iter = bio->bi_iter;  in verity_map()
580 	verity_submit_prefetch(v, io);  in verity_map()