29 static unsigned cache_mode(struct cached_dev *dc, struct bio *bio)  in cache_mode()  argument
34 static bool verify(struct cached_dev *dc, struct bio *bio)  in verify()  argument
39 static void bio_csum(struct bio *bio, struct bkey *k)  in bio_csum()  argument
45 	bio_for_each_segment(bv, bio, iter) {  in bio_csum()
121 	struct bio *bio = op->bio;  in bch_data_invalidate()  local
124 		 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);  in bch_data_invalidate()
126 	while (bio_sectors(bio)) {  in bch_data_invalidate()
127 		unsigned sectors = min(bio_sectors(bio),  in bch_data_invalidate()
133 		bio->bi_iter.bi_sector	+= sectors;  in bch_data_invalidate()
134 		bio->bi_iter.bi_size	-= sectors << 9;  in bch_data_invalidate()
137 				&KEY(op->inode, bio->bi_iter.bi_sector, sectors));  in bch_data_invalidate()
141 	bio_put(bio);  in bch_data_invalidate()
176 static void bch_data_insert_endio(struct bio *bio)  in bch_data_insert_endio()  argument
178 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()
181 	if (bio->bi_error) {  in bch_data_insert_endio()
184 			op->error = bio->bi_error;  in bch_data_insert_endio()
191 	bch_bbio_endio(op->c, bio, bio->bi_error, "writing data to cache");  in bch_data_insert_endio()
197 	struct bio *bio = op->bio, *n;  in bch_data_insert_start()  local
199 	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) {  in bch_data_insert_start()
211 	bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA);  in bch_data_insert_start()
229 		SET_KEY_OFFSET(k, bio->bi_iter.bi_sector);  in bch_data_insert_start()
231 		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),  in bch_data_insert_start()
236 		n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);  in bch_data_insert_start()
258 	} while (n != bio);  in bch_data_insert_start()
288 		bio_put(bio);  in bch_data_insert_start()
320 	trace_bcache_write(op->c, op->inode, op->bio,  in bch_data_insert()
324 	bio_get(op->bio);  in bch_data_insert()
371 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)  in check_should_bypass()  argument
374 	unsigned mode = cache_mode(dc, bio);  in check_should_bypass()
381 	    (bio->bi_rw & REQ_DISCARD))  in check_should_bypass()
386 	     (bio->bi_rw & REQ_WRITE)))  in check_should_bypass()
389 	if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||  in check_should_bypass()
390 	    bio_sectors(bio) & (c->sb.block_size - 1)) {  in check_should_bypass()
407 	    (bio->bi_rw & REQ_WRITE) &&  in check_should_bypass()
408 	    (bio->bi_rw & REQ_SYNC))  in check_should_bypass()
413 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)  in check_should_bypass()
414 		if (i->last == bio->bi_iter.bi_sector &&  in check_should_bypass()
423 	if (i->sequential + bio->bi_iter.bi_size > i->sequential)  in check_should_bypass()
424 		i->sequential	+= bio->bi_iter.bi_size;  in check_should_bypass()
426 	i->last			 = bio_end_sector(bio);  in check_should_bypass()
441 		trace_bcache_bypass_sequential(bio);  in check_should_bypass()
446 		trace_bcache_bypass_congested(bio);  in check_should_bypass()
451 	bch_rescale_priorities(c, bio_sectors(bio));  in check_should_bypass()
454 	bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));  in check_should_bypass()
464 	struct bbio		bio;  member
465 	struct bio		*orig_bio;
466 	struct bio		*cache_miss;
480 static void bch_cache_read_endio(struct bio *bio)  in bch_cache_read_endio()  argument
482 	struct bbio *b = container_of(bio, struct bbio, bio);  in bch_cache_read_endio()
483 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()
493 	if (bio->bi_error)  in bch_cache_read_endio()
494 		s->iop.error = bio->bi_error;  in bch_cache_read_endio()
501 	bch_bbio_endio(s->iop.c, bio, bio->bi_error, "reading from cache");  in bch_cache_read_endio()
511 	struct bio *n, *bio = &s->bio.bio;  in cache_lookup_fn()  local
515 	if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0)  in cache_lookup_fn()
519 	    KEY_START(k) > bio->bi_iter.bi_sector) {  in cache_lookup_fn()
520 		unsigned bio_sectors = bio_sectors(bio);  in cache_lookup_fn()
523 				KEY_START(k) - bio->bi_iter.bi_sector)  in cache_lookup_fn()
526 		int ret = s->d->cache_miss(b, s, bio, sectors);  in cache_lookup_fn()
545 	n = bio_next_split(bio, min_t(uint64_t, INT_MAX,  in cache_lookup_fn()
546 				      KEY_OFFSET(k) - bio->bi_iter.bi_sector),  in cache_lookup_fn()
549 	bio_key = &container_of(n, struct bbio, bio)->key;  in cache_lookup_fn()
570 	return n == bio ? MAP_DONE : MAP_CONTINUE;  in cache_lookup_fn()
576 	struct bio *bio = &s->bio.bio;  in cache_lookup()  local
582 				 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0),  in cache_lookup()
594 static void request_endio(struct bio *bio)  in request_endio()  argument
596 	struct closure *cl = bio->bi_private;  in request_endio()
598 	if (bio->bi_error) {  in request_endio()
600 		s->iop.error = bio->bi_error;  in request_endio()
605 	bio_put(bio);  in request_endio()
622 static void do_bio_hook(struct search *s, struct bio *orig_bio)  in do_bio_hook()
624 	struct bio *bio = &s->bio.bio;  in do_bio_hook()  local
626 	bio_init(bio);  in do_bio_hook()
627 	__bio_clone_fast(bio, orig_bio);  in do_bio_hook()
628 	bio->bi_end_io		= request_endio;  in do_bio_hook()
629 	bio->bi_private		= &s->cl;  in do_bio_hook()
631 	bio_cnt_set(bio, 3);  in do_bio_hook()
639 	if (s->iop.bio)  in search_free()
640 		bio_put(s->iop.bio);  in search_free()
646 static inline struct search *search_alloc(struct bio *bio,  in search_alloc()  argument
654 	do_bio_hook(s, bio);  in search_alloc()
656 	s->orig_bio		= bio;  in search_alloc()
660 	s->write		= (bio->bi_rw & REQ_WRITE) != 0;  in search_alloc()
665 	s->iop.bio		= NULL;  in search_alloc()
671 	s->iop.flush_journal	= (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0;  in search_alloc()
697 	if (s->iop.bio) {  in cached_dev_cache_miss_done()
701 		bio_for_each_segment_all(bv, s->iop.bio, i)  in cached_dev_cache_miss_done()
711 	struct bio *bio = &s->bio.bio;  in cached_dev_read_error()  local
722 		closure_bio_submit(bio, cl);  in cached_dev_read_error()
741 	if (s->iop.bio) {  in cached_dev_read_done()
742 		bio_reset(s->iop.bio);  in cached_dev_read_done()
743 		s->iop.bio->bi_iter.bi_sector = s->cache_miss->bi_iter.bi_sector;  in cached_dev_read_done()
744 		s->iop.bio->bi_bdev = s->cache_miss->bi_bdev;  in cached_dev_read_done()
745 		s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;  in cached_dev_read_done()
746 		bch_bio_map(s->iop.bio, NULL);  in cached_dev_read_done()
748 		bio_copy_data(s->cache_miss, s->iop.bio);  in cached_dev_read_done()
754 	if (verify(dc, &s->bio.bio) && s->recoverable && !s->read_dirty_data)  in cached_dev_read_done()
759 	if (s->iop.bio &&  in cached_dev_read_done()
779 	else if (s->iop.bio || verify(dc, &s->bio.bio))  in cached_dev_read_done_bh()
786 				 struct bio *bio, unsigned sectors)  in cached_dev_cache_miss()  argument
791 	struct bio *miss, *cache_bio;  in cached_dev_cache_miss()
794 		miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);  in cached_dev_cache_miss()
795 		ret = miss == bio ? MAP_DONE : MAP_CONTINUE;  in cached_dev_cache_miss()
799 	if (!(bio->bi_rw & REQ_RAHEAD) &&  in cached_dev_cache_miss()
800 	    !(bio->bi_rw & REQ_META) &&  in cached_dev_cache_miss()
803 			      bdev_sectors(bio->bi_bdev) - bio_end_sector(bio));  in cached_dev_cache_miss()
805 	s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);  in cached_dev_cache_miss()
808 				 bio->bi_iter.bi_sector + s->insert_bio_sectors,  in cached_dev_cache_miss()
817 	miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);  in cached_dev_cache_miss()
820 	ret = miss == bio ? MAP_DONE : -EINTR;  in cached_dev_cache_miss()
843 	s->iop.bio	= cache_bio;  in cached_dev_cache_miss()
879 	struct bio *bio = &s->bio.bio;  in cached_dev_write()  local
880 	struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);  in cached_dev_write()
881 	struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);  in cached_dev_write()
902 	if (bio->bi_rw & REQ_DISCARD)  in cached_dev_write()
906 			     cache_mode(dc, bio),  in cached_dev_write()
913 		s->iop.bio = s->orig_bio;  in cached_dev_write()
914 		bio_get(s->iop.bio);  in cached_dev_write()
916 		if (!(bio->bi_rw & REQ_DISCARD) ||  in cached_dev_write()
918 			closure_bio_submit(bio, cl);  in cached_dev_write()
921 		s->iop.bio = bio;  in cached_dev_write()
923 		if (bio->bi_rw & REQ_FLUSH) {  in cached_dev_write()
925 			struct bio *flush = bio_alloc_bioset(GFP_NOIO, 0,  in cached_dev_write()
929 			flush->bi_bdev	= bio->bi_bdev;  in cached_dev_write()
936 		s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split);  in cached_dev_write()
938 		closure_bio_submit(bio, cl);  in cached_dev_write()
948 	struct bio *bio = &s->bio.bio;  in cached_dev_nodata()  local
954 	closure_bio_submit(bio, cl);  in cached_dev_nodata()
962 					struct bio *bio)  in cached_dev_make_request()  argument
965 	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;  in cached_dev_make_request()
967 	int rw = bio_data_dir(bio);  in cached_dev_make_request()
969 	generic_start_io_acct(rw, bio_sectors(bio), &d->disk->part0);  in cached_dev_make_request()
971 	bio->bi_bdev = dc->bdev;  in cached_dev_make_request()
972 	bio->bi_iter.bi_sector += dc->sb.data_offset;  in cached_dev_make_request()
975 		s = search_alloc(bio, d);  in cached_dev_make_request()
976 		trace_bcache_request_start(s->d, bio);  in cached_dev_make_request()
978 		if (!bio->bi_iter.bi_size) {  in cached_dev_make_request()
987 			s->iop.bypass = check_should_bypass(dc, bio);  in cached_dev_make_request()
995 		if ((bio->bi_rw & REQ_DISCARD) &&  in cached_dev_make_request()
997 			bio_endio(bio);  in cached_dev_make_request()
999 			generic_make_request(bio);  in cached_dev_make_request()
1050 				struct bio *bio, unsigned sectors)  in flash_dev_cache_miss()  argument
1052 	unsigned bytes = min(sectors, bio_sectors(bio)) << 9;  in flash_dev_cache_miss()
1054 	swap(bio->bi_iter.bi_size, bytes);  in flash_dev_cache_miss()
1055 	zero_fill_bio(bio);  in flash_dev_cache_miss()
1056 	swap(bio->bi_iter.bi_size, bytes);  in flash_dev_cache_miss()
1058 	bio_advance(bio, bytes);  in flash_dev_cache_miss()
1060 	if (!bio->bi_iter.bi_size)  in flash_dev_cache_miss()
1077 					     struct bio *bio)  in flash_dev_make_request()  argument
1081 	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;  in flash_dev_make_request()
1082 	int rw = bio_data_dir(bio);  in flash_dev_make_request()
1084 	generic_start_io_acct(rw, bio_sectors(bio), &d->disk->part0);  in flash_dev_make_request()
1086 	s = search_alloc(bio, d);  in flash_dev_make_request()
1088 	bio = &s->bio.bio;  in flash_dev_make_request()
1090 	trace_bcache_request_start(s->d, bio);  in flash_dev_make_request()
1092 	if (!bio->bi_iter.bi_size) {  in flash_dev_make_request()
1103 					&KEY(d->id, bio->bi_iter.bi_sector, 0),  in flash_dev_make_request()
1104 					&KEY(d->id, bio_end_sector(bio), 0));  in flash_dev_make_request()
1106 		s->iop.bypass		= (bio->bi_rw & REQ_DISCARD) != 0;  in flash_dev_make_request()
1108 		s->iop.bio		= bio;  in flash_dev_make_request()