22 static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
93 static void rrpc_discard(struct rrpc *rrpc, struct bio *bio)  in rrpc_discard()  argument
95 	sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG;  in rrpc_discard()
96 	sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;  in rrpc_discard()
106 		bio_io_error(bio);  in rrpc_discard()
248 static void rrpc_end_sync_bio(struct bio *bio)  in rrpc_end_sync_bio()  argument
250 	struct completion *waiting = bio->bi_private;  in rrpc_end_sync_bio()
252 	if (bio->bi_error)  in rrpc_end_sync_bio()
253 		pr_err("nvm: gc request failed (%u).\n", bio->bi_error);  in rrpc_end_sync_bio()
273 	struct bio *bio;  in rrpc_move_valid_pages()  local
283 	bio = bio_alloc(GFP_NOIO, 1);  in rrpc_move_valid_pages()
284 	if (!bio) {  in rrpc_move_valid_pages()
317 		bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);  in rrpc_move_valid_pages()
318 		bio->bi_rw = READ;  in rrpc_move_valid_pages()
319 		bio->bi_private = &wait;  in rrpc_move_valid_pages()
320 		bio->bi_end_io = rrpc_end_sync_bio;  in rrpc_move_valid_pages()
323 		bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);  in rrpc_move_valid_pages()
325 		if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {  in rrpc_move_valid_pages()
332 		bio_reset(bio);  in rrpc_move_valid_pages()
335 		bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);  in rrpc_move_valid_pages()
336 		bio->bi_rw = WRITE;  in rrpc_move_valid_pages()
337 		bio->bi_private = &wait;  in rrpc_move_valid_pages()
338 		bio->bi_end_io = rrpc_end_sync_bio;  in rrpc_move_valid_pages()
340 		bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);  in rrpc_move_valid_pages()
345 		if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {  in rrpc_move_valid_pages()
354 		bio_reset(bio);  in rrpc_move_valid_pages()
359 	bio_put(bio);  in rrpc_move_valid_pages()
648 	sector_t laddr = rrpc_get_laddr(rqd->bio) - npages;  in rrpc_end_io()
650 	if (bio_data_dir(rqd->bio) == WRITE)  in rrpc_end_io()
657 	bio_put(rqd->bio);  in rrpc_end_io()
669 static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,  in rrpc_read_ppalist_rq()  argument
674 	sector_t laddr = rrpc_get_laddr(bio);  in rrpc_read_ppalist_rq()
678 	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {  in rrpc_read_ppalist_rq()
705 static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,  in rrpc_read_rq()  argument
710 	sector_t laddr = rrpc_get_laddr(bio);  in rrpc_read_rq()
713 	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))  in rrpc_read_rq()
733 static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,  in rrpc_write_ppalist_rq()  argument
738 	sector_t laddr = rrpc_get_laddr(bio);  in rrpc_write_ppalist_rq()
742 	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {  in rrpc_write_ppalist_rq()
768 static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,  in rrpc_write_rq()  argument
774 	sector_t laddr = rrpc_get_laddr(bio);  in rrpc_write_rq()
776 	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))  in rrpc_write_rq()
794 static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio,  in rrpc_setup_rq()  argument
805 		if (bio_rw(bio) == WRITE)  in rrpc_setup_rq()
806 			return rrpc_write_ppalist_rq(rrpc, bio, rqd, flags,  in rrpc_setup_rq()
809 		return rrpc_read_ppalist_rq(rrpc, bio, rqd, flags, npages);  in rrpc_setup_rq()
812 	if (bio_rw(bio) == WRITE)  in rrpc_setup_rq()
813 		return rrpc_write_rq(rrpc, bio, rqd, flags);  in rrpc_setup_rq()
815 	return rrpc_read_rq(rrpc, bio, rqd, flags);  in rrpc_setup_rq()
818 static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,  in rrpc_submit_io()  argument
823 	uint8_t nr_pages = rrpc_get_pages(bio);  in rrpc_submit_io()
824 	int bio_size = bio_sectors(bio) << 9;  in rrpc_submit_io()
831 	err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages);  in rrpc_submit_io()
835 	bio_get(bio);  in rrpc_submit_io()
836 	rqd->bio = bio;  in rrpc_submit_io()
850 static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio)  in rrpc_make_rq()  argument
856 	if (bio->bi_rw & REQ_DISCARD) {  in rrpc_make_rq()
857 		rrpc_discard(rrpc, bio);  in rrpc_make_rq()
864 		bio_io_error(bio);  in rrpc_make_rq()
869 	err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE);  in rrpc_make_rq()
874 		bio_io_error(bio);  in rrpc_make_rq()
877 		bio_endio(bio);  in rrpc_make_rq()
881 		bio_list_add(&rrpc->requeue_bios, bio);  in rrpc_make_rq()
895 	struct bio *bio;  in rrpc_requeue()  local
904 	while ((bio = bio_list_pop(&bios)))  in rrpc_requeue()
905 		rrpc_make_rq(rrpc->disk->queue, bio);  in rrpc_requeue()