109 union map_info *dm_get_rq_mapinfo(struct request *rq)  in dm_get_rq_mapinfo()  argument
111 	if (rq && rq->end_io_data)  in dm_get_rq_mapinfo()
112 		return &((struct dm_rq_target_io *)rq->end_io_data)->info;  in dm_get_rq_mapinfo()
662 static void free_clone_request(struct mapped_device *md, struct request *rq)  in free_clone_request()  argument
664 	mempool_free(rq, md->rq_pool);  in free_clone_request()
1077 static struct dm_rq_target_io *tio_from_request(struct request *rq)  in tio_from_request()  argument
1079 	return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);  in tio_from_request()
1154 	struct request *rq = tio->orig;  in dm_end_request()  local
1156 	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {  in dm_end_request()
1157 		rq->errors = clone->errors;  in dm_end_request()
1158 		rq->resid_len = clone->resid_len;  in dm_end_request()
1160 		if (rq->sense)  in dm_end_request()
1166 			rq->sense_len = clone->sense_len;  in dm_end_request()
1170 	rq_end_stats(md, rq);  in dm_end_request()
1171 	if (!rq->q->mq_ops)  in dm_end_request()
1172 		blk_end_request_all(rq, error);  in dm_end_request()
1174 		blk_mq_end_request(rq, error);  in dm_end_request()
1178 static void dm_unprep_request(struct request *rq)  in dm_unprep_request()  argument
1180 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_unprep_request()
1183 	if (!rq->q->mq_ops) {  in dm_unprep_request()
1184 		rq->special = NULL;  in dm_unprep_request()
1185 		rq->cmd_flags &= ~REQ_DONTPREP;  in dm_unprep_request()
1197 static void old_requeue_request(struct request *rq)  in old_requeue_request()  argument
1199 	struct request_queue *q = rq->q;  in old_requeue_request()
1203 	blk_requeue_request(q, rq);  in old_requeue_request()
1209 					struct request *rq)  in dm_requeue_original_request()  argument
1211 	int rw = rq_data_dir(rq);  in dm_requeue_original_request()
1213 	rq_end_stats(md, rq);  in dm_requeue_original_request()
1214 	dm_unprep_request(rq);  in dm_requeue_original_request()
1216 	if (!rq->q->mq_ops)  in dm_requeue_original_request()
1217 		old_requeue_request(rq);  in dm_requeue_original_request()
1219 		blk_mq_requeue_request(rq);  in dm_requeue_original_request()
1220 		blk_mq_kick_requeue_list(rq->q);  in dm_requeue_original_request()
1299 static void dm_softirq_done(struct request *rq)  in dm_softirq_done()  argument
1302 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_softirq_done()
1307 		rq_end_stats(tio->md, rq);  in dm_softirq_done()
1308 		rw = rq_data_dir(rq);  in dm_softirq_done()
1309 		if (!rq->q->mq_ops) {  in dm_softirq_done()
1310 			blk_end_request_all(rq, tio->error);  in dm_softirq_done()
1314 			blk_mq_end_request(rq, tio->error);  in dm_softirq_done()
1320 	if (rq->cmd_flags & REQ_FAILED)  in dm_softirq_done()
1330 static void dm_complete_request(struct request *rq, int error)  in dm_complete_request()  argument
1332 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_complete_request()
1335 	if (!rq->q->mq_ops)  in dm_complete_request()
1336 		blk_complete_request(rq);  in dm_complete_request()
1338 		blk_mq_complete_request(rq, error);  in dm_complete_request()
1347 static void dm_kill_unmapped_request(struct request *rq, int error)  in dm_kill_unmapped_request()  argument
1349 	rq->cmd_flags |= REQ_FAILED;  in dm_kill_unmapped_request()
1350 	dm_complete_request(rq, error);  in dm_kill_unmapped_request()
1792 static void dm_dispatch_clone_request(struct request *clone, struct request *rq)  in dm_dispatch_clone_request()  argument
1803 		dm_complete_request(rq, r);  in dm_dispatch_clone_request()
1820 static int setup_clone(struct request *clone, struct request *rq,  in setup_clone()  argument
1825 	r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,  in setup_clone()
1830 	clone->cmd = rq->cmd;  in setup_clone()
1831 	clone->cmd_len = rq->cmd_len;  in setup_clone()
1832 	clone->sense = rq->sense;  in setup_clone()
1841 static struct request *clone_rq(struct request *rq, struct mapped_device *md,  in clone_rq()  argument
1859 	if (setup_clone(clone, rq, tio, gfp_mask)) {  in clone_rq()
1871 static void init_tio(struct dm_rq_target_io *tio, struct request *rq,  in init_tio()  argument
1877 	tio->orig = rq;  in init_tio()
1884 static struct dm_rq_target_io *prep_tio(struct request *rq,  in prep_tio()  argument
1895 	init_tio(tio, rq, md);  in prep_tio()
1899 		if (!clone_rq(rq, md, tio, gfp_mask)) {  in prep_tio()
1913 static int dm_prep_fn(struct request_queue *q, struct request *rq)  in dm_prep_fn()  argument
1918 	if (unlikely(rq->special)) {  in dm_prep_fn()
1923 	tio = prep_tio(rq, md, GFP_ATOMIC);  in dm_prep_fn()
1927 	rq->special = tio;  in dm_prep_fn()
1928 	rq->cmd_flags |= REQ_DONTPREP;  in dm_prep_fn()
1939 static int map_request(struct dm_rq_target_io *tio, struct request *rq,  in map_request()  argument
1950 		r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
1953 			dm_kill_unmapped_request(rq, r);  in map_request()
1958 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
1972 				     blk_rq_pos(rq));  in map_request()
1973 		dm_dispatch_clone_request(clone, rq);  in map_request()
1986 		dm_kill_unmapped_request(rq, r);  in map_request()
1996 	struct request *rq = tio->orig;  in map_tio_request()  local
1999 	if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)  in map_tio_request()
2000 		dm_requeue_original_request(md, rq);  in map_tio_request()
2084 	struct request *rq;  in dm_request_fn()  local
2095 		rq = blk_peek_request(q);  in dm_request_fn()
2096 		if (!rq)  in dm_request_fn()
2101 		if (!(rq->cmd_flags & REQ_FLUSH))  in dm_request_fn()
2102 			pos = blk_rq_pos(rq);  in dm_request_fn()
2111 			dm_start_request(md, rq);  in dm_request_fn()
2112 			dm_kill_unmapped_request(rq, -EIO);  in dm_request_fn()
2117 		    md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 &&  in dm_request_fn()
2118 		    md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq))  in dm_request_fn()
2124 		dm_start_request(md, rq);  in dm_request_fn()
2126 		tio = tio_from_request(rq);  in dm_request_fn()
2631 static int dm_mq_init_request(void *data, struct request *rq,  in dm_mq_init_request()  argument
2636 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_init_request()
2650 	struct request *rq = bd->rq;  in dm_mq_queue_rq()  local
2651 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_queue_rq()
2660 	if (!(rq->cmd_flags & REQ_FLUSH))  in dm_mq_queue_rq()
2661 		pos = blk_rq_pos(rq);  in dm_mq_queue_rq()
2671 		dm_start_request(md, rq);  in dm_mq_queue_rq()
2679 	dm_start_request(md, rq);  in dm_mq_queue_rq()
2682 	init_tio(tio, rq, md);  in dm_mq_queue_rq()
2693 		tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io);  in dm_mq_queue_rq()
2694 		(void) clone_rq(rq, md, tio, GFP_ATOMIC);  in dm_mq_queue_rq()
2698 		if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) {  in dm_mq_queue_rq()
2700 			rq_end_stats(md, rq);  in dm_mq_queue_rq()
2701 			rq_completed(md, rq_data_dir(rq), false);  in dm_mq_queue_rq()