Lines Matching refs:bio
75 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio, in dm_hook_bio() argument
78 h->bi_end_io = bio->bi_end_io; in dm_hook_bio()
79 h->bi_private = bio->bi_private; in dm_hook_bio()
81 bio->bi_end_io = bi_end_io; in dm_hook_bio()
82 bio->bi_private = bi_private; in dm_hook_bio()
85 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) in dm_unhook_bio() argument
87 bio->bi_end_io = h->bi_end_io; in dm_unhook_bio()
88 bio->bi_private = h->bi_private; in dm_unhook_bio()
94 atomic_inc(&bio->bi_remaining); in dm_unhook_bio()
480 struct bio *bio, struct dm_bio_prison_cell *cell_prealloc, in bio_detain_range() argument
488 r = dm_bio_detain(cache->prison, &key, bio, cell_prealloc, cell_result); in bio_detain_range()
496 struct bio *bio, struct dm_bio_prison_cell *cell_prealloc, in bio_detain() argument
501 return bio_detain_range(cache, oblock, end, bio, in bio_detain()
690 static struct per_bio_data *get_per_bio_data(struct bio *bio, size_t data_size) in get_per_bio_data() argument
692 struct per_bio_data *pb = dm_per_bio_data(bio, data_size); in get_per_bio_data()
697 static struct per_bio_data *init_per_bio_data(struct bio *bio, size_t data_size) in init_per_bio_data() argument
699 struct per_bio_data *pb = get_per_bio_data(bio, data_size); in init_per_bio_data()
702 pb->req_nr = dm_bio_get_target_bio_nr(bio); in init_per_bio_data()
711 static void remap_to_origin(struct cache *cache, struct bio *bio) in remap_to_origin() argument
713 bio->bi_bdev = cache->origin_dev->bdev; in remap_to_origin()
716 static void remap_to_cache(struct cache *cache, struct bio *bio, in remap_to_cache() argument
719 sector_t bi_sector = bio->bi_iter.bi_sector; in remap_to_cache()
722 bio->bi_bdev = cache->cache_dev->bdev; in remap_to_cache()
724 bio->bi_iter.bi_sector = in remap_to_cache()
728 bio->bi_iter.bi_sector = in remap_to_cache()
733 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) in check_if_tick_bio_needed() argument
737 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); in check_if_tick_bio_needed()
741 !(bio->bi_rw & (REQ_FUA | REQ_FLUSH | REQ_DISCARD))) { in check_if_tick_bio_needed()
748 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, in remap_to_origin_clear_discard() argument
751 check_if_tick_bio_needed(cache, bio); in remap_to_origin_clear_discard()
752 remap_to_origin(cache, bio); in remap_to_origin_clear_discard()
753 if (bio_data_dir(bio) == WRITE) in remap_to_origin_clear_discard()
757 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio, in remap_to_cache_dirty() argument
760 check_if_tick_bio_needed(cache, bio); in remap_to_cache_dirty()
761 remap_to_cache(cache, bio, cblock); in remap_to_cache_dirty()
762 if (bio_data_dir(bio) == WRITE) { in remap_to_cache_dirty()
768 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) in get_bio_block() argument
770 sector_t block_nr = bio->bi_iter.bi_sector; in get_bio_block()
780 static int bio_triggers_commit(struct cache *cache, struct bio *bio) in bio_triggers_commit() argument
782 return bio->bi_rw & (REQ_FLUSH | REQ_FUA); in bio_triggers_commit()
789 static void inc_ds(struct cache *cache, struct bio *bio, in inc_ds() argument
793 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); in inc_ds()
801 static void issue(struct cache *cache, struct bio *bio) in issue() argument
805 if (!bio_triggers_commit(cache, bio)) { in issue()
806 generic_make_request(bio); in issue()
816 bio_list_add(&cache->deferred_flush_bios, bio); in issue()
820 static void inc_and_issue(struct cache *cache, struct bio *bio, struct dm_bio_prison_cell *cell) in inc_and_issue() argument
822 inc_ds(cache, bio, cell); in inc_and_issue()
823 issue(cache, bio); in inc_and_issue()
826 static void defer_writethrough_bio(struct cache *cache, struct bio *bio) in defer_writethrough_bio() argument
831 bio_list_add(&cache->deferred_writethrough_bios, bio); in defer_writethrough_bio()
837 static void writethrough_endio(struct bio *bio, int err) in writethrough_endio() argument
839 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); in writethrough_endio()
841 dm_unhook_bio(&pb->hook_info, bio); in writethrough_endio()
844 bio_endio(bio, err); in writethrough_endio()
848 dm_bio_restore(&pb->bio_details, bio); in writethrough_endio()
849 remap_to_cache(pb->cache, bio, pb->cblock); in writethrough_endio()
856 defer_writethrough_bio(pb->cache, bio); in writethrough_endio()
865 static void remap_to_origin_then_cache(struct cache *cache, struct bio *bio, in remap_to_origin_then_cache() argument
868 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); in remap_to_origin_then_cache()
872 dm_hook_bio(&pb->hook_info, bio, writethrough_endio, NULL); in remap_to_origin_then_cache()
873 dm_bio_record(&pb->bio_details, bio); in remap_to_origin_then_cache()
875 remap_to_origin_clear_discard(pb->cache, bio, oblock); in remap_to_origin_then_cache()
1068 static void overwrite_endio(struct bio *bio, int err) in overwrite_endio() argument
1070 struct dm_cache_migration *mg = bio->bi_private; in overwrite_endio()
1073 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); in overwrite_endio()
1076 dm_unhook_bio(&pb->hook_info, bio); in overwrite_endio()
1090 static void issue_overwrite(struct dm_cache_migration *mg, struct bio *bio) in issue_overwrite() argument
1093 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); in issue_overwrite()
1095 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); in issue_overwrite()
1096 remap_to_cache_dirty(mg->cache, bio, mg->new_oblock, mg->cblock); in issue_overwrite()
1102 generic_make_request(bio); in issue_overwrite()
1105 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio) in bio_writes_complete_block() argument
1107 return (bio_data_dir(bio) == WRITE) && in bio_writes_complete_block()
1108 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT)); in bio_writes_complete_block()
1117 static void calc_discard_block_range(struct cache *cache, struct bio *bio, in calc_discard_block_range() argument
1120 sector_t sb = bio->bi_iter.bi_sector; in calc_discard_block_range()
1121 sector_t se = bio_end_sector(bio); in calc_discard_block_range()
1134 struct bio *bio = mg->new_ocell->holder; in issue_discard() local
1136 calc_discard_block_range(mg->cache, bio, &b, &e); in issue_discard()
1142 bio_endio(bio, 0); in issue_discard()
1161 struct bio *bio = mg->new_ocell->holder; in issue_copy_or_discard() local
1166 !avoid && bio_writes_complete_block(cache, bio)) { in issue_copy_or_discard()
1167 issue_overwrite(mg, bio); in issue_copy_or_discard()
1376 static void defer_bio(struct cache *cache, struct bio *bio) in defer_bio() argument
1381 bio_list_add(&cache->deferred_bios, bio); in defer_bio()
1387 static void process_flush_bio(struct cache *cache, struct bio *bio) in process_flush_bio() argument
1390 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); in process_flush_bio()
1392 BUG_ON(bio->bi_iter.bi_size); in process_flush_bio()
1394 remap_to_origin(cache, bio); in process_flush_bio()
1396 remap_to_cache(cache, bio, 0); in process_flush_bio()
1403 issue(cache, bio); in process_flush_bio()
1407 struct bio *bio) in process_discard_bio() argument
1413 calc_discard_block_range(cache, bio, &b, &e); in process_discard_bio()
1415 bio_endio(bio, 0); in process_discard_bio()
1420 …r = bio_detain_range(cache, dblock_to_oblock(cache, b), dblock_to_oblock(cache, e), bio, cell_prea… in process_discard_bio()
1436 static void inc_hit_counter(struct cache *cache, struct bio *bio) in inc_hit_counter() argument
1438 atomic_inc(bio_data_dir(bio) == READ ? in inc_hit_counter()
1442 static void inc_miss_counter(struct cache *cache, struct bio *bio) in inc_miss_counter() argument
1444 atomic_inc(bio_data_dir(bio) == READ ? in inc_miss_counter()
1475 struct bio *bio) in process_bio() argument
1479 dm_oblock_t block = get_bio_block(cache, bio); in process_bio()
1490 r = bio_detain(cache, block, bio, cell_prealloc, in process_bio()
1504 bio, &ool.locker, &lookup_result); in process_bio()
1513 inc_miss_counter(cache, bio); in process_bio()
1521 if (bio_data_dir(bio) == WRITE) { in process_bio()
1528 remap_to_origin_clear_discard(cache, bio, block); in process_bio()
1529 inc_and_issue(cache, bio, new_ocell); in process_bio()
1532 inc_hit_counter(cache, bio); in process_bio()
1534 if (bio_data_dir(bio) == WRITE && in process_bio()
1537 remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock); in process_bio()
1538 inc_and_issue(cache, bio, new_ocell); in process_bio()
1541 remap_to_cache_dirty(cache, bio, block, lookup_result.cblock); in process_bio()
1542 inc_and_issue(cache, bio, new_ocell); in process_bio()
1549 inc_miss_counter(cache, bio); in process_bio()
1550 remap_to_origin_clear_discard(cache, bio, block); in process_bio()
1551 inc_and_issue(cache, bio, new_ocell); in process_bio()
1572 bio_io_error(bio); in process_bio()
1604 struct bio *bio; in process_deferred_bios() local
1628 bio = bio_list_pop(&bios); in process_deferred_bios()
1630 if (bio->bi_rw & REQ_FLUSH) in process_deferred_bios()
1631 process_flush_bio(cache, bio); in process_deferred_bios()
1632 else if (bio->bi_rw & REQ_DISCARD) in process_deferred_bios()
1633 process_discard_bio(cache, &structs, bio); in process_deferred_bios()
1635 process_bio(cache, &structs, bio); in process_deferred_bios()
1645 struct bio *bio; in process_deferred_flush_bios() local
1657 while ((bio = bio_list_pop(&bios))) in process_deferred_flush_bios()
1658 submit_bios ? generic_make_request(bio) : bio_io_error(bio); in process_deferred_flush_bios()
1665 struct bio *bio; in process_deferred_writethrough_bios() local
1677 while ((bio = bio_list_pop(&bios))) in process_deferred_writethrough_bios()
1678 generic_make_request(bio); in process_deferred_writethrough_bios()
1809 struct bio *bio; in requeue_deferred_io() local
1816 while ((bio = bio_list_pop(&bios))) in requeue_deferred_io()
1817 bio_endio(bio, DM_ENDIO_REQUEUE); in requeue_deferred_io()
2604 static int __cache_map(struct cache *cache, struct bio *bio, struct dm_bio_prison_cell **cell) in __cache_map() argument
2607 dm_oblock_t block = get_bio_block(cache, bio); in __cache_map()
2612 struct per_bio_data *pb = init_per_bio_data(bio, pb_data_size); in __cache_map()
2623 remap_to_origin(cache, bio); in __cache_map()
2627 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA | REQ_DISCARD)) { in __cache_map()
2628 defer_bio(cache, bio); in __cache_map()
2637 defer_bio(cache, bio); in __cache_map()
2641 r = bio_detain(cache, block, bio, *cell, in __cache_map()
2646 defer_bio(cache, bio); in __cache_map()
2654 bio, &ool.locker, &lookup_result); in __cache_map()
2662 bio_io_error(bio); in __cache_map()
2670 if (bio_data_dir(bio) == WRITE) { in __cache_map()
2679 inc_miss_counter(cache, bio); in __cache_map()
2680 remap_to_origin_clear_discard(cache, bio, block); in __cache_map()
2684 inc_hit_counter(cache, bio); in __cache_map()
2685 if (bio_data_dir(bio) == WRITE && writethrough_mode(&cache->features) && in __cache_map()
2687 remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock); in __cache_map()
2689 remap_to_cache_dirty(cache, bio, block, lookup_result.cblock); in __cache_map()
2694 inc_miss_counter(cache, bio); in __cache_map()
2700 bio_endio(bio, 0); in __cache_map()
2705 remap_to_origin_clear_discard(cache, bio, block); in __cache_map()
2713 bio_io_error(bio); in __cache_map()
2720 static int cache_map(struct dm_target *ti, struct bio *bio) in cache_map() argument
2726 r = __cache_map(cache, bio, &cell); in cache_map()
2728 inc_ds(cache, bio, cell); in cache_map()
2735 static int cache_end_io(struct dm_target *ti, struct bio *bio, int error) in cache_end_io() argument
2740 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); in cache_end_io()