Lines Matching refs:s
70 while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING) in bch_data_insert_keys()
71 closure_sync(&s->cl); in bch_data_insert_keys()
478 struct search *s = container_of(cl, struct search, cl); in bch_cache_read_endio() local
488 s->iop.error = error; in bch_cache_read_endio()
490 ptr_stale(s->iop.c, &b->key, 0)) { in bch_cache_read_endio()
491 atomic_long_inc(&s->iop.c->cache_read_races); in bch_cache_read_endio()
492 s->iop.error = -EINTR; in bch_cache_read_endio()
495 bch_bbio_endio(s->iop.c, bio, error, "reading from cache"); in bch_cache_read_endio()
504 struct search *s = container_of(op, struct search, op); in cache_lookup_fn() local
505 struct bio *n, *bio = &s->bio.bio; in cache_lookup_fn()
509 if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0) in cache_lookup_fn()
512 if (KEY_INODE(k) != s->iop.inode || in cache_lookup_fn()
515 unsigned sectors = KEY_INODE(k) == s->iop.inode in cache_lookup_fn()
520 int ret = s->d->cache_miss(b, s, bio, sectors); in cache_lookup_fn()
537 s->read_dirty_data = true; in cache_lookup_fn()
541 GFP_NOIO, s->d->bio_split); in cache_lookup_fn()
546 bch_cut_front(&KEY(s->iop.inode, n->bi_iter.bi_sector, 0), bio_key); in cache_lookup_fn()
547 bch_cut_back(&KEY(s->iop.inode, bio_end_sector(n), 0), bio_key); in cache_lookup_fn()
550 n->bi_private = &s->cl; in cache_lookup_fn()
569 struct search *s = container_of(cl, struct search, iop.cl); in cache_lookup() local
570 struct bio *bio = &s->bio.bio; in cache_lookup()
573 bch_btree_op_init(&s->op, -1); in cache_lookup()
575 ret = bch_btree_map_keys(&s->op, s->iop.c, in cache_lookup()
576 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0), in cache_lookup()
591 struct search *s = container_of(cl, struct search, cl); in request_endio() local
592 s->iop.error = error; in request_endio()
594 s->recoverable = false; in request_endio()
601 static void bio_complete(struct search *s) in bio_complete() argument
603 if (s->orig_bio) { in bio_complete()
604 generic_end_io_acct(bio_data_dir(s->orig_bio), in bio_complete()
605 &s->d->disk->part0, s->start_time); in bio_complete()
607 trace_bcache_request_end(s->d, s->orig_bio); in bio_complete()
608 bio_endio(s->orig_bio, s->iop.error); in bio_complete()
609 s->orig_bio = NULL; in bio_complete()
613 static void do_bio_hook(struct search *s, struct bio *orig_bio) in do_bio_hook() argument
615 struct bio *bio = &s->bio.bio; in do_bio_hook()
620 bio->bi_private = &s->cl; in do_bio_hook()
627 struct search *s = container_of(cl, struct search, cl); in search_free() local
628 bio_complete(s); in search_free()
630 if (s->iop.bio) in search_free()
631 bio_put(s->iop.bio); in search_free()
634 mempool_free(s, s->d->c->search); in search_free()
640 struct search *s; in search_alloc() local
642 s = mempool_alloc(d->c->search, GFP_NOIO); in search_alloc()
644 closure_init(&s->cl, NULL); in search_alloc()
645 do_bio_hook(s, bio); in search_alloc()
647 s->orig_bio = bio; in search_alloc()
648 s->cache_miss = NULL; in search_alloc()
649 s->d = d; in search_alloc()
650 s->recoverable = 1; in search_alloc()
651 s->write = (bio->bi_rw & REQ_WRITE) != 0; in search_alloc()
652 s->read_dirty_data = 0; in search_alloc()
653 s->start_time = jiffies; in search_alloc()
655 s->iop.c = d->c; in search_alloc()
656 s->iop.bio = NULL; in search_alloc()
657 s->iop.inode = d->id; in search_alloc()
658 s->iop.write_point = hash_long((unsigned long) current, 16); in search_alloc()
659 s->iop.write_prio = 0; in search_alloc()
660 s->iop.error = 0; in search_alloc()
661 s->iop.flags = 0; in search_alloc()
662 s->iop.flush_journal = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0; in search_alloc()
663 s->iop.wq = bcache_wq; in search_alloc()
665 return s; in search_alloc()
672 struct search *s = container_of(cl, struct search, cl); in cached_dev_bio_complete() local
673 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); in cached_dev_bio_complete()
683 struct search *s = container_of(cl, struct search, cl); in cached_dev_cache_miss_done() local
685 if (s->iop.replace_collision) in cached_dev_cache_miss_done()
686 bch_mark_cache_miss_collision(s->iop.c, s->d); in cached_dev_cache_miss_done()
688 if (s->iop.bio) { in cached_dev_cache_miss_done()
692 bio_for_each_segment_all(bv, s->iop.bio, i) in cached_dev_cache_miss_done()
701 struct search *s = container_of(cl, struct search, cl); in cached_dev_read_error() local
702 struct bio *bio = &s->bio.bio; in cached_dev_read_error()
704 if (s->recoverable) { in cached_dev_read_error()
706 trace_bcache_read_retry(s->orig_bio); in cached_dev_read_error()
708 s->iop.error = 0; in cached_dev_read_error()
709 do_bio_hook(s, s->orig_bio); in cached_dev_read_error()
713 closure_bio_submit(bio, cl, s->d); in cached_dev_read_error()
721 struct search *s = container_of(cl, struct search, cl); in cached_dev_read_done() local
722 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); in cached_dev_read_done()
732 if (s->iop.bio) { in cached_dev_read_done()
733 bio_reset(s->iop.bio); in cached_dev_read_done()
734 s->iop.bio->bi_iter.bi_sector = s->cache_miss->bi_iter.bi_sector; in cached_dev_read_done()
735 s->iop.bio->bi_bdev = s->cache_miss->bi_bdev; in cached_dev_read_done()
736 s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9; in cached_dev_read_done()
737 bch_bio_map(s->iop.bio, NULL); in cached_dev_read_done()
739 bio_copy_data(s->cache_miss, s->iop.bio); in cached_dev_read_done()
741 bio_put(s->cache_miss); in cached_dev_read_done()
742 s->cache_miss = NULL; in cached_dev_read_done()
745 if (verify(dc, &s->bio.bio) && s->recoverable && !s->read_dirty_data) in cached_dev_read_done()
746 bch_data_verify(dc, s->orig_bio); in cached_dev_read_done()
748 bio_complete(s); in cached_dev_read_done()
750 if (s->iop.bio && in cached_dev_read_done()
751 !test_bit(CACHE_SET_STOPPING, &s->iop.c->flags)) { in cached_dev_read_done()
752 BUG_ON(!s->iop.replace); in cached_dev_read_done()
753 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); in cached_dev_read_done()
761 struct search *s = container_of(cl, struct search, cl); in cached_dev_read_done_bh() local
762 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); in cached_dev_read_done_bh()
764 bch_mark_cache_accounting(s->iop.c, s->d, in cached_dev_read_done_bh()
765 !s->cache_miss, s->iop.bypass); in cached_dev_read_done_bh()
766 trace_bcache_read(s->orig_bio, !s->cache_miss, s->iop.bypass); in cached_dev_read_done_bh()
768 if (s->iop.error) in cached_dev_read_done_bh()
770 else if (s->iop.bio || verify(dc, &s->bio.bio)) in cached_dev_read_done_bh()
776 static int cached_dev_cache_miss(struct btree *b, struct search *s, in cached_dev_cache_miss() argument
781 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); in cached_dev_cache_miss()
784 if (s->cache_miss || s->iop.bypass) { in cached_dev_cache_miss()
785 miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split); in cached_dev_cache_miss()
792 s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA) in cached_dev_cache_miss()
796 s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada); in cached_dev_cache_miss()
798 s->iop.replace_key = KEY(s->iop.inode, in cached_dev_cache_miss()
799 bio->bi_iter.bi_sector + s->insert_bio_sectors, in cached_dev_cache_miss()
800 s->insert_bio_sectors); in cached_dev_cache_miss()
802 ret = bch_btree_insert_check_key(b, &s->op, &s->iop.replace_key); in cached_dev_cache_miss()
806 s->iop.replace = true; in cached_dev_cache_miss()
808 miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split); in cached_dev_cache_miss()
814 DIV_ROUND_UP(s->insert_bio_sectors, PAGE_SECTORS), in cached_dev_cache_miss()
821 cache_bio->bi_iter.bi_size = s->insert_bio_sectors << 9; in cached_dev_cache_miss()
824 cache_bio->bi_private = &s->cl; in cached_dev_cache_miss()
831 bch_mark_cache_readahead(s->iop.c, s->d); in cached_dev_cache_miss()
833 s->cache_miss = miss; in cached_dev_cache_miss()
834 s->iop.bio = cache_bio; in cached_dev_cache_miss()
836 closure_bio_submit(cache_bio, &s->cl, s->d); in cached_dev_cache_miss()
843 miss->bi_private = &s->cl; in cached_dev_cache_miss()
844 closure_bio_submit(miss, &s->cl, s->d); in cached_dev_cache_miss()
848 static void cached_dev_read(struct cached_dev *dc, struct search *s) in cached_dev_read() argument
850 struct closure *cl = &s->cl; in cached_dev_read()
852 closure_call(&s->iop.cl, cache_lookup, NULL, cl); in cached_dev_read()
860 struct search *s = container_of(cl, struct search, cl); in cached_dev_write_complete() local
861 struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); in cached_dev_write_complete()
867 static void cached_dev_write(struct cached_dev *dc, struct search *s) in cached_dev_write() argument
869 struct closure *cl = &s->cl; in cached_dev_write()
870 struct bio *bio = &s->bio.bio; in cached_dev_write()
874 bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, &start, &end); in cached_dev_write()
882 s->iop.bypass = false; in cached_dev_write()
883 s->iop.writeback = true; in cached_dev_write()
894 s->iop.bypass = true; in cached_dev_write()
896 if (should_writeback(dc, s->orig_bio, in cached_dev_write()
898 s->iop.bypass)) { in cached_dev_write()
899 s->iop.bypass = false; in cached_dev_write()
900 s->iop.writeback = true; in cached_dev_write()
903 if (s->iop.bypass) { in cached_dev_write()
904 s->iop.bio = s->orig_bio; in cached_dev_write()
905 bio_get(s->iop.bio); in cached_dev_write()
909 closure_bio_submit(bio, cl, s->d); in cached_dev_write()
910 } else if (s->iop.writeback) { in cached_dev_write()
912 s->iop.bio = bio; in cached_dev_write()
924 closure_bio_submit(flush, cl, s->d); in cached_dev_write()
927 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split); in cached_dev_write()
929 closure_bio_submit(bio, cl, s->d); in cached_dev_write()
932 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); in cached_dev_write()
938 struct search *s = container_of(cl, struct search, cl); in cached_dev_nodata() local
939 struct bio *bio = &s->bio.bio; in cached_dev_nodata()
941 if (s->iop.flush_journal) in cached_dev_nodata()
942 bch_journal_meta(s->iop.c, cl); in cached_dev_nodata()
945 closure_bio_submit(bio, cl, s->d); in cached_dev_nodata()
954 struct search *s; in cached_dev_make_request() local
965 s = search_alloc(bio, d); in cached_dev_make_request()
966 trace_bcache_request_start(s->d, bio); in cached_dev_make_request()
973 continue_at_nobarrier(&s->cl, in cached_dev_make_request()
977 s->iop.bypass = check_should_bypass(dc, bio); in cached_dev_make_request()
980 cached_dev_write(dc, s); in cached_dev_make_request()
982 cached_dev_read(dc, s); in cached_dev_make_request()
1037 static int flash_dev_cache_miss(struct btree *b, struct search *s, in flash_dev_cache_miss() argument
1056 struct search *s = container_of(cl, struct search, cl); in flash_dev_nodata() local
1058 if (s->iop.flush_journal) in flash_dev_nodata()
1059 bch_journal_meta(s->iop.c, cl); in flash_dev_nodata()
1066 struct search *s; in flash_dev_make_request() local
1073 s = search_alloc(bio, d); in flash_dev_make_request()
1074 cl = &s->cl; in flash_dev_make_request()
1075 bio = &s->bio.bio; in flash_dev_make_request()
1077 trace_bcache_request_start(s->d, bio); in flash_dev_make_request()
1084 continue_at_nobarrier(&s->cl, in flash_dev_make_request()
1088 bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, in flash_dev_make_request()
1092 s->iop.bypass = (bio->bi_rw & REQ_DISCARD) != 0; in flash_dev_make_request()
1093 s->iop.writeback = true; in flash_dev_make_request()
1094 s->iop.bio = bio; in flash_dev_make_request()
1096 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); in flash_dev_make_request()
1098 closure_call(&s->iop.cl, cache_lookup, NULL, cl); in flash_dev_make_request()