635 static void free_io(struct mapped_device *md, struct dm_io *io)  in free_io()  argument
637 	mempool_free(io, md->io_pool);  in free_io()
673 static void start_io_acct(struct dm_io *io)  in start_io_acct()  argument
675 	struct mapped_device *md = io->md;  in start_io_acct()
676 	struct bio *bio = io->bio;  in start_io_acct()
680 	io->start_time = jiffies;  in start_io_acct()
690 				    bio_sectors(bio), false, 0, &io->stats_aux);  in start_io_acct()
693 static void end_io_acct(struct dm_io *io)  in end_io_acct()  argument
695 	struct mapped_device *md = io->md;  in end_io_acct()
696 	struct bio *bio = io->bio;  in end_io_acct()
697 	unsigned long duration = jiffies - io->start_time;  in end_io_acct()
701 	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);  in end_io_acct()
705 				    bio_sectors(bio), true, duration, &io->stats_aux);  in end_io_acct()
930 static void dec_pending(struct dm_io *io, int error)  in dec_pending()  argument
935 	struct mapped_device *md = io->md;  in dec_pending()
939 		spin_lock_irqsave(&io->endio_lock, flags);  in dec_pending()
940 		if (!(io->error > 0 && __noflush_suspending(md)))  in dec_pending()
941 			io->error = error;  in dec_pending()
942 		spin_unlock_irqrestore(&io->endio_lock, flags);  in dec_pending()
945 	if (atomic_dec_and_test(&io->io_count)) {  in dec_pending()
946 		if (io->error == DM_ENDIO_REQUEUE) {  in dec_pending()
952 				bio_list_add_head(&md->deferred, io->bio);  in dec_pending()
955 				io->error = -EIO;  in dec_pending()
959 		io_error = io->error;  in dec_pending()
960 		bio = io->bio;  in dec_pending()
961 		end_io_acct(io);  in dec_pending()
962 		free_io(md, io);  in dec_pending()
996 	struct dm_io *io = tio->io;  in clone_endio()  local
997 	struct mapped_device *md = tio->io->md;  in clone_endio()
1022 	dec_pending(io, error);  in clone_endio()
1485 	atomic_inc(&tio->io->io_count);  in __map_bio()
1492 				      tio->io->bio->bi_bdev->bd_dev, sector);  in __map_bio()
1497 		md = tio->io->md;  in __map_bio()
1498 		dec_pending(tio->io, r);  in __map_bio()
1510 	struct dm_io *io;  member
1551 	tio->io = ci->io;  in alloc_tio()
1727 	ci.io = alloc_io(md);  in __split_and_process_bio()
1728 	ci.io->error = 0;  in __split_and_process_bio()
1729 	atomic_set(&ci.io->io_count, 1);  in __split_and_process_bio()
1730 	ci.io->bio = bio;  in __split_and_process_bio()
1731 	ci.io->md = md;  in __split_and_process_bio()
1732 	spin_lock_init(&ci.io->endio_lock);  in __split_and_process_bio()
1735 	start_io_acct(ci.io);  in __split_and_process_bio()
1750 	dec_pending(ci.io, error);  in __split_and_process_bio()