28 static unsigned cache_mode(struct cached_dev *dc, struct bio *bio)  in cache_mode()  argument
33 static bool verify(struct cached_dev *dc, struct bio *bio)  in verify()  argument
38 static void bio_csum(struct bio *bio, struct bkey *k)  in bio_csum()  argument
44 	bio_for_each_segment(bv, bio, iter) {  in bio_csum()
118 	struct bio *bio = op->bio;  in bch_data_invalidate()  local
121 		 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);  in bch_data_invalidate()
123 	while (bio_sectors(bio)) {  in bch_data_invalidate()
124 		unsigned sectors = min(bio_sectors(bio),  in bch_data_invalidate()
130 		bio->bi_iter.bi_sector	+= sectors;  in bch_data_invalidate()
131 		bio->bi_iter.bi_size	-= sectors << 9;  in bch_data_invalidate()
134 				&KEY(op->inode, bio->bi_iter.bi_sector, sectors));  in bch_data_invalidate()
138 	bio_put(bio);  in bch_data_invalidate()
173 static void bch_data_insert_endio(struct bio *bio, int error)  in bch_data_insert_endio()  argument
175 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()
188 	bch_bbio_endio(op->c, bio, error, "writing data to cache");  in bch_data_insert_endio()
194 	struct bio *bio = op->bio, *n;  in bch_data_insert_start()  local
196 	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) {  in bch_data_insert_start()
208 	bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA);  in bch_data_insert_start()
224 		SET_KEY_OFFSET(k, bio->bi_iter.bi_sector);  in bch_data_insert_start()
226 		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),  in bch_data_insert_start()
231 		n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);  in bch_data_insert_start()
253 	} while (n != bio);  in bch_data_insert_start()
282 		bio_put(bio);  in bch_data_insert_start()
314 	trace_bcache_write(op->c, op->inode, op->bio,  in bch_data_insert()
318 	bio_get(op->bio);  in bch_data_insert()
365 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)  in check_should_bypass()  argument
368 	unsigned mode = cache_mode(dc, bio);  in check_should_bypass()
375 	    (bio->bi_rw & REQ_DISCARD))  in check_should_bypass()
380 	     (bio->bi_rw & REQ_WRITE)))  in check_should_bypass()
383 	if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||  in check_should_bypass()
384 	    bio_sectors(bio) & (c->sb.block_size - 1)) {  in check_should_bypass()
401 	    (bio->bi_rw & REQ_WRITE) &&  in check_should_bypass()
402 	    (bio->bi_rw & REQ_SYNC))  in check_should_bypass()
407 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)  in check_should_bypass()
408 		if (i->last == bio->bi_iter.bi_sector &&  in check_should_bypass()
417 	if (i->sequential + bio->bi_iter.bi_size > i->sequential)  in check_should_bypass()
418 		i->sequential	+= bio->bi_iter.bi_size;  in check_should_bypass()
420 	i->last			 = bio_end_sector(bio);  in check_should_bypass()
435 		trace_bcache_bypass_sequential(bio);  in check_should_bypass()
440 		trace_bcache_bypass_congested(bio);  in check_should_bypass()
445 	bch_rescale_priorities(c, bio_sectors(bio));  in check_should_bypass()
448 	bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));  in check_should_bypass()
458 	struct bbio		bio;  member
459 	struct bio		*orig_bio;
460 	struct bio		*cache_miss;
474 static void bch_cache_read_endio(struct bio *bio, int error)  in bch_cache_read_endio()  argument
476 	struct bbio *b = container_of(bio, struct bbio, bio);  in bch_cache_read_endio()
477 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()
495 	bch_bbio_endio(s->iop.c, bio, error, "reading from cache");  in bch_cache_read_endio()
505 	struct bio *n, *bio = &s->bio.bio;  in cache_lookup_fn()  local
509 	if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0)  in cache_lookup_fn()
513 	    KEY_START(k) > bio->bi_iter.bi_sector) {  in cache_lookup_fn()
514 		unsigned bio_sectors = bio_sectors(bio);  in cache_lookup_fn()
517 				KEY_START(k) - bio->bi_iter.bi_sector)  in cache_lookup_fn()
520 		int ret = s->d->cache_miss(b, s, bio, sectors);  in cache_lookup_fn()
539 	n = bio_next_split(bio, min_t(uint64_t, INT_MAX,  in cache_lookup_fn()
540 				      KEY_OFFSET(k) - bio->bi_iter.bi_sector),  in cache_lookup_fn()
543 	bio_key = &container_of(n, struct bbio, bio)->key;  in cache_lookup_fn()
564 	return n == bio ? MAP_DONE : MAP_CONTINUE;  in cache_lookup_fn()
570 	struct bio *bio = &s->bio.bio;  in cache_lookup()  local
576 				 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0),  in cache_lookup()
586 static void request_endio(struct bio *bio, int error)  in request_endio()  argument
588 	struct closure *cl = bio->bi_private;  in request_endio()
597 	bio_put(bio);  in request_endio()
613 static void do_bio_hook(struct search *s, struct bio *orig_bio)  in do_bio_hook()
615 	struct bio *bio = &s->bio.bio;  in do_bio_hook()  local
617 	bio_init(bio);  in do_bio_hook()
618 	__bio_clone_fast(bio, orig_bio);  in do_bio_hook()
619 	bio->bi_end_io		= request_endio;  in do_bio_hook()
620 	bio->bi_private		= &s->cl;  in do_bio_hook()
622 	atomic_set(&bio->bi_cnt, 3);  in do_bio_hook()
630 	if (s->iop.bio)  in search_free()
631 		bio_put(s->iop.bio);  in search_free()
637 static inline struct search *search_alloc(struct bio *bio,  in search_alloc()  argument
645 	do_bio_hook(s, bio);  in search_alloc()
647 	s->orig_bio		= bio;  in search_alloc()
651 	s->write		= (bio->bi_rw & REQ_WRITE) != 0;  in search_alloc()
656 	s->iop.bio		= NULL;  in search_alloc()
662 	s->iop.flush_journal	= (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0;  in search_alloc()
688 	if (s->iop.bio) {  in cached_dev_cache_miss_done()
692 		bio_for_each_segment_all(bv, s->iop.bio, i)  in cached_dev_cache_miss_done()
702 	struct bio *bio = &s->bio.bio;  in cached_dev_read_error()  local
713 		closure_bio_submit(bio, cl, s->d);  in cached_dev_read_error()
732 	if (s->iop.bio) {  in cached_dev_read_done()
733 		bio_reset(s->iop.bio);  in cached_dev_read_done()
734 		s->iop.bio->bi_iter.bi_sector = s->cache_miss->bi_iter.bi_sector;  in cached_dev_read_done()
735 		s->iop.bio->bi_bdev = s->cache_miss->bi_bdev;  in cached_dev_read_done()
736 		s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;  in cached_dev_read_done()
737 		bch_bio_map(s->iop.bio, NULL);  in cached_dev_read_done()
739 		bio_copy_data(s->cache_miss, s->iop.bio);  in cached_dev_read_done()
745 	if (verify(dc, &s->bio.bio) && s->recoverable && !s->read_dirty_data)  in cached_dev_read_done()
750 	if (s->iop.bio &&  in cached_dev_read_done()
770 	else if (s->iop.bio || verify(dc, &s->bio.bio))  in cached_dev_read_done_bh()
777 				 struct bio *bio, unsigned sectors)  in cached_dev_cache_miss()  argument
782 	struct bio *miss, *cache_bio;  in cached_dev_cache_miss()
785 		miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);  in cached_dev_cache_miss()
786 		ret = miss == bio ? MAP_DONE : MAP_CONTINUE;  in cached_dev_cache_miss()
790 	if (!(bio->bi_rw & REQ_RAHEAD) &&  in cached_dev_cache_miss()
791 	    !(bio->bi_rw & REQ_META) &&  in cached_dev_cache_miss()
794 			      bdev_sectors(bio->bi_bdev) - bio_end_sector(bio));  in cached_dev_cache_miss()
796 	s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);  in cached_dev_cache_miss()
799 				 bio->bi_iter.bi_sector + s->insert_bio_sectors,  in cached_dev_cache_miss()
808 	miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);  in cached_dev_cache_miss()
811 	ret = miss == bio ? MAP_DONE : -EINTR;  in cached_dev_cache_miss()
834 	s->iop.bio	= cache_bio;  in cached_dev_cache_miss()
870 	struct bio *bio = &s->bio.bio;  in cached_dev_write()  local
871 	struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);  in cached_dev_write()
872 	struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);  in cached_dev_write()
893 	if (bio->bi_rw & REQ_DISCARD)  in cached_dev_write()
897 			     cache_mode(dc, bio),  in cached_dev_write()
904 		s->iop.bio = s->orig_bio;  in cached_dev_write()
905 		bio_get(s->iop.bio);  in cached_dev_write()
907 		if (!(bio->bi_rw & REQ_DISCARD) ||  in cached_dev_write()
909 			closure_bio_submit(bio, cl, s->d);  in cached_dev_write()
912 		s->iop.bio = bio;  in cached_dev_write()
914 		if (bio->bi_rw & REQ_FLUSH) {  in cached_dev_write()
916 			struct bio *flush = bio_alloc_bioset(GFP_NOIO, 0,  in cached_dev_write()
920 			flush->bi_bdev	= bio->bi_bdev;  in cached_dev_write()
927 		s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split);  in cached_dev_write()
929 		closure_bio_submit(bio, cl, s->d);  in cached_dev_write()
939 	struct bio *bio = &s->bio.bio;  in cached_dev_nodata()  local
945 	closure_bio_submit(bio, cl, s->d);  in cached_dev_nodata()
952 static void cached_dev_make_request(struct request_queue *q, struct bio *bio)  in cached_dev_make_request()  argument
955 	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;  in cached_dev_make_request()
957 	int rw = bio_data_dir(bio);  in cached_dev_make_request()
959 	generic_start_io_acct(rw, bio_sectors(bio), &d->disk->part0);  in cached_dev_make_request()
961 	bio->bi_bdev = dc->bdev;  in cached_dev_make_request()
962 	bio->bi_iter.bi_sector += dc->sb.data_offset;  in cached_dev_make_request()
965 		s = search_alloc(bio, d);  in cached_dev_make_request()
966 		trace_bcache_request_start(s->d, bio);  in cached_dev_make_request()
968 		if (!bio->bi_iter.bi_size) {  in cached_dev_make_request()
977 			s->iop.bypass = check_should_bypass(dc, bio);  in cached_dev_make_request()
985 		if ((bio->bi_rw & REQ_DISCARD) &&  in cached_dev_make_request()
987 			bio_endio(bio, 0);  in cached_dev_make_request()
989 			bch_generic_make_request(bio, &d->bio_split_hook);  in cached_dev_make_request()
1038 				struct bio *bio, unsigned sectors)  in flash_dev_cache_miss()  argument
1040 	unsigned bytes = min(sectors, bio_sectors(bio)) << 9;  in flash_dev_cache_miss()
1042 	swap(bio->bi_iter.bi_size, bytes);  in flash_dev_cache_miss()
1043 	zero_fill_bio(bio);  in flash_dev_cache_miss()
1044 	swap(bio->bi_iter.bi_size, bytes);  in flash_dev_cache_miss()
1046 	bio_advance(bio, bytes);  in flash_dev_cache_miss()
1048 	if (!bio->bi_iter.bi_size)  in flash_dev_cache_miss()
1064 static void flash_dev_make_request(struct request_queue *q, struct bio *bio)  in flash_dev_make_request()  argument
1068 	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;  in flash_dev_make_request()
1069 	int rw = bio_data_dir(bio);  in flash_dev_make_request()
1071 	generic_start_io_acct(rw, bio_sectors(bio), &d->disk->part0);  in flash_dev_make_request()
1073 	s = search_alloc(bio, d);  in flash_dev_make_request()
1075 	bio = &s->bio.bio;  in flash_dev_make_request()
1077 	trace_bcache_request_start(s->d, bio);  in flash_dev_make_request()
1079 	if (!bio->bi_iter.bi_size) {  in flash_dev_make_request()
1089 					&KEY(d->id, bio->bi_iter.bi_sector, 0),  in flash_dev_make_request()
1090 					&KEY(d->id, bio_end_sector(bio), 0));  in flash_dev_make_request()
1092 		s->iop.bypass		= (bio->bi_rw & REQ_DISCARD) != 0;  in flash_dev_make_request()
1094 		s->iop.bio		= bio;  in flash_dev_make_request()