86 	struct request *orig, *clone;  member
106 	struct bio clone;  member
642 	bio_put(&tio->clone);  in free_tio()
995 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);  in clone_endio()
1028 static void end_clone_bio(struct bio *clone)  in end_clone_bio()  argument
1031 		container_of(clone, struct dm_rq_clone_bio_info, clone);  in end_clone_bio()
1035 	int error = clone->bi_error;  in end_clone_bio()
1037 	bio_put(clone);  in end_clone_bio()
1121 static void free_rq_clone(struct request *clone)  in free_rq_clone()  argument
1123 	struct dm_rq_target_io *tio = clone->end_io_data;  in free_rq_clone()
1126 	blk_rq_unprep_clone(clone);  in free_rq_clone()
1130 		tio->ti->type->release_clone_rq(clone);  in free_rq_clone()
1133 		free_clone_request(md, clone);  in free_rq_clone()
1149 static void dm_end_request(struct request *clone, int error)  in dm_end_request()  argument
1151 	int rw = rq_data_dir(clone);  in dm_end_request()
1152 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_end_request()
1157 		rq->errors = clone->errors;  in dm_end_request()
1158 		rq->resid_len = clone->resid_len;  in dm_end_request()
1166 			rq->sense_len = clone->sense_len;  in dm_end_request()
1169 	free_rq_clone(clone);  in dm_end_request()
1181 	struct request *clone = tio->clone;  in dm_unprep_request()  local
1188 	if (clone)  in dm_unprep_request()
1189 		free_rq_clone(clone);  in dm_unprep_request()
1264 static void dm_done(struct request *clone, int error, bool mapped)  in dm_done()  argument
1267 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_done()
1274 			r = rq_end_io(tio->ti, clone, error, &tio->info);  in dm_done()
1277 	if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) &&  in dm_done()
1278 		     !clone->q->limits.max_write_same_sectors))  in dm_done()
1283 		dm_end_request(clone, r);  in dm_done()
1303 	struct request *clone = tio->clone;  in dm_softirq_done()  local
1306 	if (!clone) {  in dm_softirq_done()
1323 	dm_done(clone, tio->error, mapped);  in dm_softirq_done()
1356 static void end_clone_request(struct request *clone, int error)  in end_clone_request()  argument
1358 	struct dm_rq_target_io *tio = clone->end_io_data;  in end_clone_request()
1360 	if (!clone->q->mq_ops) {  in end_clone_request()
1367 		__blk_put_request(clone->q, clone);  in end_clone_request()
1460 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);  in dm_accept_partial_bio()
1475 	struct bio *clone = &tio->clone;  in __map_bio()  local
1478 	clone->bi_end_io = clone_endio;  in __map_bio()
1486 	sector = clone->bi_iter.bi_sector;  in __map_bio()
1487 	r = ti->type->map(ti, clone);  in __map_bio()
1491 		trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,  in __map_bio()
1494 		generic_make_request(clone);  in __map_bio()
1527 	struct bio *clone = &tio->clone;  in clone_bio()  local
1529 	__bio_clone_fast(clone, bio);  in clone_bio()
1532 		bio_integrity_clone(clone, bio, GFP_NOIO);  in clone_bio()
1534 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));  in clone_bio()
1535 	clone->bi_iter.bi_size = to_bytes(len);  in clone_bio()
1538 		bio_integrity_trim(clone, 0, len);  in clone_bio()
1546 	struct bio *clone;  in alloc_tio()  local
1548 	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);  in alloc_tio()
1549 	tio = container_of(clone, struct dm_target_io, clone);  in alloc_tio()
1563 	struct bio *clone = &tio->clone;  in __clone_and_map_simple_bio()  local
1567 	__bio_clone_fast(clone, ci->bio);  in __clone_and_map_simple_bio()
1569 		bio_setup_sector(clone, ci->sector, *len);  in __clone_and_map_simple_bio()
1792 static void dm_dispatch_clone_request(struct request *clone, struct request *rq)  in dm_dispatch_clone_request()  argument
1796 	if (blk_queue_io_stat(clone->q))  in dm_dispatch_clone_request()
1797 		clone->cmd_flags |= REQ_IO_STAT;  in dm_dispatch_clone_request()
1799 	clone->start_time = jiffies;  in dm_dispatch_clone_request()
1800 	r = blk_insert_cloned_request(clone->q, clone);  in dm_dispatch_clone_request()
1811 		container_of(bio, struct dm_rq_clone_bio_info, clone);  in dm_rq_bio_constructor()
1820 static int setup_clone(struct request *clone, struct request *rq,  in setup_clone()  argument
1825 	r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,  in setup_clone()
1830 	clone->cmd = rq->cmd;  in setup_clone()
1831 	clone->cmd_len = rq->cmd_len;  in setup_clone()
1832 	clone->sense = rq->sense;  in setup_clone()
1833 	clone->end_io = end_clone_request;  in setup_clone()
1834 	clone->end_io_data = tio;  in setup_clone()
1836 	tio->clone = clone;  in setup_clone()
1848 	bool alloc_clone = !tio->clone;  in clone_rq()
1849 	struct request *clone;  in clone_rq()  local
1852 		clone = alloc_clone_request(md, gfp_mask);  in clone_rq()
1853 		if (!clone)  in clone_rq()
1856 		clone = tio->clone;  in clone_rq()
1858 	blk_rq_init(NULL, clone);  in clone_rq()
1859 	if (setup_clone(clone, rq, tio, gfp_mask)) {  in clone_rq()
1862 			free_clone_request(md, clone);  in clone_rq()
1866 	return clone;  in clone_rq()
1876 	tio->clone = NULL;  in init_tio()
1944 	struct request *clone = NULL;  in map_request()  local
1946 	if (tio->clone) {  in map_request()
1947 		clone = tio->clone;  in map_request()
1948 		r = ti->type->map_rq(ti, clone, &tio->info);  in map_request()
1950 		r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
1958 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
1960 			ti->type->release_clone_rq(clone);  in map_request()
1971 		trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),  in map_request()
1973 		dm_dispatch_clone_request(clone, rq);  in map_request()
2693 		tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io);  in dm_mq_queue_rq()
3500 …oundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);  in dm_alloc_md_mempools()
3512 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);  in dm_alloc_md_mempools()