602 static void free_io(struct mapped_device *md, struct dm_io *io)  in free_io()  argument
604 	mempool_free(io, md->io_pool);  in free_io()
640 static void start_io_acct(struct dm_io *io)  in start_io_acct()  argument
642 	struct mapped_device *md = io->md;  in start_io_acct()
643 	struct bio *bio = io->bio;  in start_io_acct()
647 	io->start_time = jiffies;  in start_io_acct()
657 				    bio_sectors(bio), false, 0, &io->stats_aux);  in start_io_acct()
660 static void end_io_acct(struct dm_io *io)  in end_io_acct()  argument
662 	struct mapped_device *md = io->md;  in end_io_acct()
663 	struct bio *bio = io->bio;  in end_io_acct()
664 	unsigned long duration = jiffies - io->start_time;  in end_io_acct()
668 	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);  in end_io_acct()
672 				    bio_sectors(bio), true, duration, &io->stats_aux);  in end_io_acct()
897 static void dec_pending(struct dm_io *io, int error)  in dec_pending()  argument
902 	struct mapped_device *md = io->md;  in dec_pending()
906 		spin_lock_irqsave(&io->endio_lock, flags);  in dec_pending()
907 		if (!(io->error > 0 && __noflush_suspending(md)))  in dec_pending()
908 			io->error = error;  in dec_pending()
909 		spin_unlock_irqrestore(&io->endio_lock, flags);  in dec_pending()
912 	if (atomic_dec_and_test(&io->io_count)) {  in dec_pending()
913 		if (io->error == DM_ENDIO_REQUEUE) {  in dec_pending()
919 				bio_list_add_head(&md->deferred, io->bio);  in dec_pending()
922 				io->error = -EIO;  in dec_pending()
926 		io_error = io->error;  in dec_pending()
927 		bio = io->bio;  in dec_pending()
928 		end_io_acct(io);  in dec_pending()
929 		free_io(md, io);  in dec_pending()
961 	struct dm_io *io = tio->io;  in clone_endio()  local
962 	struct mapped_device *md = tio->io->md;  in clone_endio()
990 	dec_pending(io, error);  in clone_endio()
1445 	atomic_inc(&tio->io->io_count);  in __map_bio()
1452 				      tio->io->bio->bi_bdev->bd_dev, sector);  in __map_bio()
1457 		md = tio->io->md;  in __map_bio()
1458 		dec_pending(tio->io, r);  in __map_bio()
1470 	struct dm_io *io;  member
1511 	tio->io = ci->io;  in alloc_tio()
1687 	ci.io = alloc_io(md);  in __split_and_process_bio()
1688 	ci.io->error = 0;  in __split_and_process_bio()
1689 	atomic_set(&ci.io->io_count, 1);  in __split_and_process_bio()
1690 	ci.io->bio = bio;  in __split_and_process_bio()
1691 	ci.io->md = md;  in __split_and_process_bio()
1692 	spin_lock_init(&ci.io->endio_lock);  in __split_and_process_bio()
1695 	start_io_acct(ci.io);  in __split_and_process_bio()
1710 	dec_pending(ci.io, error);  in __split_and_process_bio()