pblk 218 arch/x86/mm/numa_emulation.c int nr_nodes, struct numa_memblk *pblk, pblk 225 arch/x86/mm/numa_emulation.c if ((!size && !nr_nodes) || (nr_nodes && !pblk)) pblk 242 arch/x86/mm/numa_emulation.c node_set(pblk->nid, physnode_mask); pblk 345 drivers/acpi/acpi_processor.c pr->pblk = object.processor.pblk_address; pblk 221 drivers/acpi/processor_idle.c if (!pr->pblk) pblk 239 drivers/acpi/processor_idle.c pr->power.states[ACPI_STATE_C2].address = pr->pblk + 4; pblk 240 drivers/acpi/processor_idle.c pr->power.states[ACPI_STATE_C3].address = pr->pblk + 5; pblk 21 drivers/lightnvm/pblk-cache.c void pblk_write_to_cache(struct pblk *pblk, struct bio *bio, pblk 24 drivers/lightnvm/pblk-cache.c struct request_queue *q = pblk->dev->q; pblk 33 drivers/lightnvm/pblk-cache.c &pblk->disk->part0); pblk 40 drivers/lightnvm/pblk-cache.c ret = pblk_rb_may_write_user(&pblk->rwb, bio, nr_entries, &bpos); pblk 46 drivers/lightnvm/pblk-cache.c pblk_pipeline_stop(pblk); pblk 55 drivers/lightnvm/pblk-cache.c pblk_write_kick(pblk); pblk 66 drivers/lightnvm/pblk-cache.c pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + i); pblk 67 drivers/lightnvm/pblk-cache.c pblk_rb_write_entry_user(&pblk->rwb, data, w_ctx, pos); pblk 72 drivers/lightnvm/pblk-cache.c atomic64_add(nr_entries, &pblk->user_wa); pblk 75 drivers/lightnvm/pblk-cache.c atomic_long_add(nr_entries, &pblk->inflight_writes); pblk 76 drivers/lightnvm/pblk-cache.c atomic_long_add(nr_entries, &pblk->req_writes); pblk 79 drivers/lightnvm/pblk-cache.c pblk_rl_inserted(&pblk->rl, nr_entries); pblk 82 drivers/lightnvm/pblk-cache.c generic_end_io_acct(q, REQ_OP_WRITE, &pblk->disk->part0, start_time); pblk 83 drivers/lightnvm/pblk-cache.c pblk_write_should_kick(pblk); pblk 93 drivers/lightnvm/pblk-cache.c int pblk_write_gc_to_cache(struct pblk *pblk, struct pblk_gc_rq *gc_rq) pblk 105 drivers/lightnvm/pblk-cache.c if (!pblk_rb_may_write_gc(&pblk->rwb, gc_rq->secs_to_gc, &bpos)) { pblk 119 drivers/lightnvm/pblk-cache.c pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + valid_entries); pblk 120 drivers/lightnvm/pblk-cache.c pblk_rb_write_entry_gc(&pblk->rwb, data, w_ctx, gc_rq->line, pblk 130 drivers/lightnvm/pblk-cache.c atomic64_add(valid_entries, &pblk->gc_wa); pblk 133 drivers/lightnvm/pblk-cache.c atomic_long_add(valid_entries, &pblk->inflight_writes); pblk 134 drivers/lightnvm/pblk-cache.c atomic_long_add(valid_entries, &pblk->recov_gc_writes); pblk 137 drivers/lightnvm/pblk-cache.c pblk_write_should_kick(pblk); pblk 29 drivers/lightnvm/pblk-core.c struct pblk *pblk = line_ws->pblk; pblk 30 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 39 drivers/lightnvm/pblk-core.c line = pblk_ppa_to_line(pblk, *ppa); pblk 42 drivers/lightnvm/pblk-core.c pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n", pblk 47 drivers/lightnvm/pblk-core.c mempool_free(line_ws, &pblk->gen_ws_pool); pblk 50 drivers/lightnvm/pblk-core.c static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line, pblk 53 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 58 drivers/lightnvm/pblk-core.c pblk_debug(pblk, "erase failed: line:%d, pos:%d\n", line->id, pos); pblk 59 drivers/lightnvm/pblk-core.c atomic_long_inc(&pblk->erase_failed); pblk 63 drivers/lightnvm/pblk-core.c pblk_err(pblk, "attempted to erase bb: line:%d, pos:%d\n", pblk 75 drivers/lightnvm/pblk-core.c pblk_gen_run_ws(pblk, NULL, ppa, pblk_line_mark_bb, pblk 76 drivers/lightnvm/pblk-core.c GFP_ATOMIC, pblk->bb_wq); pblk 79 drivers/lightnvm/pblk-core.c static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd) pblk 81 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 87 drivers/lightnvm/pblk-core.c line = pblk_ppa_to_line(pblk, rqd->ppa_addr); pblk 94 drivers/lightnvm/pblk-core.c trace_pblk_chunk_reset(pblk_disk_name(pblk), pblk 98 drivers/lightnvm/pblk-core.c pblk_mark_bb(pblk, line, rqd->ppa_addr); pblk 100 drivers/lightnvm/pblk-core.c trace_pblk_chunk_reset(pblk_disk_name(pblk), pblk 106 drivers/lightnvm/pblk-core.c trace_pblk_chunk_state(pblk_disk_name(pblk), &rqd->ppa_addr, pblk 109 drivers/lightnvm/pblk-core.c atomic_dec(&pblk->inflight_io); pblk 115 drivers/lightnvm/pblk-core.c struct pblk *pblk = rqd->private; pblk 117 drivers/lightnvm/pblk-core.c __pblk_end_io_erase(pblk, rqd); pblk 118 drivers/lightnvm/pblk-core.c mempool_free(rqd, &pblk->e_rq_pool); pblk 126 drivers/lightnvm/pblk-core.c struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk) pblk 128 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 151 drivers/lightnvm/pblk-core.c struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk, pblk 155 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 164 drivers/lightnvm/pblk-core.c void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, pblk 167 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 185 drivers/lightnvm/pblk-core.c move_list = pblk_line_gc_list(pblk, line); pblk 204 drivers/lightnvm/pblk-core.c void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa) pblk 215 drivers/lightnvm/pblk-core.c line = pblk_ppa_to_line(pblk, ppa); pblk 216 drivers/lightnvm/pblk-core.c paddr = pblk_dev_ppa_to_line_addr(pblk, ppa); pblk 218 drivers/lightnvm/pblk-core.c __pblk_map_invalidate(pblk, line, paddr); pblk 221 drivers/lightnvm/pblk-core.c static void pblk_invalidate_range(struct pblk *pblk, sector_t slba, pblk 226 drivers/lightnvm/pblk-core.c spin_lock(&pblk->trans_lock); pblk 230 drivers/lightnvm/pblk-core.c ppa = pblk_trans_map_get(pblk, lba); pblk 233 drivers/lightnvm/pblk-core.c pblk_map_invalidate(pblk, ppa); pblk 236 drivers/lightnvm/pblk-core.c pblk_trans_map_set(pblk, lba, ppa); pblk 238 drivers/lightnvm/pblk-core.c spin_unlock(&pblk->trans_lock); pblk 241 drivers/lightnvm/pblk-core.c int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd) pblk 243 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 253 drivers/lightnvm/pblk-core.c rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size(pblk); pblk 254 drivers/lightnvm/pblk-core.c rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size(pblk); pblk 259 drivers/lightnvm/pblk-core.c void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd) pblk 261 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 269 drivers/lightnvm/pblk-core.c struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type) pblk 278 drivers/lightnvm/pblk-core.c pool = &pblk->w_rq_pool; pblk 282 drivers/lightnvm/pblk-core.c pool = &pblk->r_rq_pool; pblk 286 drivers/lightnvm/pblk-core.c pool = &pblk->e_rq_pool; pblk 297 drivers/lightnvm/pblk-core.c void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type) pblk 306 drivers/lightnvm/pblk-core.c pool = &pblk->w_rq_pool; pblk 309 drivers/lightnvm/pblk-core.c pool = &pblk->r_rq_pool; pblk 312 drivers/lightnvm/pblk-core.c pool = &pblk->e_rq_pool; pblk 315 drivers/lightnvm/pblk-core.c pblk_err(pblk, "trying to free unknown rqd type\n"); pblk 319 drivers/lightnvm/pblk-core.c pblk_free_rqd_meta(pblk, rqd); pblk 323 drivers/lightnvm/pblk-core.c void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off, pblk 335 drivers/lightnvm/pblk-core.c mempool_free(page++, &pblk->page_bio_pool); pblk 339 drivers/lightnvm/pblk-core.c int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags, pblk 342 drivers/lightnvm/pblk-core.c struct request_queue *q = pblk->dev->q; pblk 347 drivers/lightnvm/pblk-core.c page = mempool_alloc(&pblk->page_bio_pool, flags); pblk 351 drivers/lightnvm/pblk-core.c pblk_err(pblk, "could not add page to bio\n"); pblk 352 drivers/lightnvm/pblk-core.c mempool_free(page, &pblk->page_bio_pool); pblk 359 drivers/lightnvm/pblk-core.c pblk_bio_free_pages(pblk, bio, (bio->bi_vcnt - i), i); pblk 363 drivers/lightnvm/pblk-core.c void pblk_write_kick(struct pblk *pblk) pblk 365 drivers/lightnvm/pblk-core.c wake_up_process(pblk->writer_ts); pblk 366 drivers/lightnvm/pblk-core.c mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(1000)); pblk 371 drivers/lightnvm/pblk-core.c struct pblk *pblk = from_timer(pblk, t, wtimer); pblk 374 drivers/lightnvm/pblk-core.c pblk_write_kick(pblk); pblk 377 drivers/lightnvm/pblk-core.c void pblk_write_should_kick(struct pblk *pblk) pblk 379 drivers/lightnvm/pblk-core.c unsigned int secs_avail = pblk_rb_read_count(&pblk->rwb); pblk 381 drivers/lightnvm/pblk-core.c if (secs_avail >= pblk->min_write_pgs_data) pblk 382 drivers/lightnvm/pblk-core.c pblk_write_kick(pblk); pblk 385 drivers/lightnvm/pblk-core.c static void pblk_wait_for_meta(struct pblk *pblk) pblk 388 drivers/lightnvm/pblk-core.c if (!atomic_read(&pblk->inflight_io)) pblk 395 drivers/lightnvm/pblk-core.c static void pblk_flush_writer(struct pblk *pblk) pblk 397 drivers/lightnvm/pblk-core.c pblk_rb_flush(&pblk->rwb); pblk 399 drivers/lightnvm/pblk-core.c if (!pblk_rb_sync_count(&pblk->rwb)) pblk 402 drivers/lightnvm/pblk-core.c pblk_write_kick(pblk); pblk 407 drivers/lightnvm/pblk-core.c struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line) pblk 409 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 410 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 412 drivers/lightnvm/pblk-core.c int packed_meta = (le32_to_cpu(*line->vsc) / pblk->min_write_pgs_data) pblk 413 drivers/lightnvm/pblk-core.c * (pblk->min_write_pgs - pblk->min_write_pgs_data); pblk 422 drivers/lightnvm/pblk-core.c pblk_rl_werr_line_in(&pblk->rl); pblk 451 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 456 drivers/lightnvm/pblk-core.c pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n", pblk 465 drivers/lightnvm/pblk-core.c void pblk_discard(struct pblk *pblk, struct bio *bio) pblk 470 drivers/lightnvm/pblk-core.c pblk_invalidate_range(pblk, slba, nr_secs); pblk 473 drivers/lightnvm/pblk-core.c void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd) pblk 475 drivers/lightnvm/pblk-core.c atomic_long_inc(&pblk->write_failed); pblk 477 drivers/lightnvm/pblk-core.c pblk_print_failed_rqd(pblk, rqd, rqd->error); pblk 481 drivers/lightnvm/pblk-core.c void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd) pblk 485 drivers/lightnvm/pblk-core.c atomic_long_inc(&pblk->read_empty); pblk 491 drivers/lightnvm/pblk-core.c atomic_long_inc(&pblk->read_high_ecc); pblk 495 drivers/lightnvm/pblk-core.c atomic_long_inc(&pblk->read_failed); pblk 498 drivers/lightnvm/pblk-core.c pblk_err(pblk, "unknown read error:%d\n", rqd->error); pblk 501 drivers/lightnvm/pblk-core.c pblk_print_failed_rqd(pblk, rqd, rqd->error); pblk 505 drivers/lightnvm/pblk-core.c void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write) pblk 507 drivers/lightnvm/pblk-core.c pblk->sec_per_write = sec_per_write; pblk 510 drivers/lightnvm/pblk-core.c int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd, void *buf) pblk 512 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 514 drivers/lightnvm/pblk-core.c atomic_inc(&pblk->inflight_io); pblk 517 drivers/lightnvm/pblk-core.c if (pblk_check_io(pblk, rqd)) pblk 524 drivers/lightnvm/pblk-core.c void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd) pblk 532 drivers/lightnvm/pblk-core.c struct nvm_chk_meta *chunk = pblk_dev_ppa_to_chunk(pblk, *ppa); pblk 533 drivers/lightnvm/pblk-core.c u64 caddr = pblk_dev_ppa_to_chunk_addr(pblk, *ppa); pblk 536 drivers/lightnvm/pblk-core.c trace_pblk_chunk_state(pblk_disk_name(pblk), pblk 539 drivers/lightnvm/pblk-core.c trace_pblk_chunk_state(pblk_disk_name(pblk), pblk 544 drivers/lightnvm/pblk-core.c int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd, void *buf) pblk 546 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 549 drivers/lightnvm/pblk-core.c atomic_inc(&pblk->inflight_io); pblk 552 drivers/lightnvm/pblk-core.c if (pblk_check_io(pblk, rqd)) pblk 560 drivers/lightnvm/pblk-core.c pblk_check_chunk_state_update(pblk, rqd); pblk 565 drivers/lightnvm/pblk-core.c static int pblk_submit_io_sync_sem(struct pblk *pblk, struct nvm_rq *rqd, pblk 571 drivers/lightnvm/pblk-core.c pblk_down_chunk(pblk, ppa_list[0]); pblk 572 drivers/lightnvm/pblk-core.c ret = pblk_submit_io_sync(pblk, rqd, buf); pblk 573 drivers/lightnvm/pblk-core.c pblk_up_chunk(pblk, ppa_list[0]); pblk 578 drivers/lightnvm/pblk-core.c int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail, pblk 581 drivers/lightnvm/pblk-core.c int max = pblk->sec_per_write; pblk 582 drivers/lightnvm/pblk-core.c int min = pblk->min_write_pgs; pblk 585 drivers/lightnvm/pblk-core.c if (skip_meta && pblk->min_write_pgs_data != pblk->min_write_pgs) pblk 586 drivers/lightnvm/pblk-core.c min = max = pblk->min_write_pgs_data; pblk 598 drivers/lightnvm/pblk-core.c void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) pblk 605 drivers/lightnvm/pblk-core.c pblk->lm.sec_per_line, line->cur_sec); pblk 613 drivers/lightnvm/pblk-core.c u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) pblk 621 drivers/lightnvm/pblk-core.c if (line->cur_sec + nr_secs > pblk->lm.sec_per_line) { pblk 623 drivers/lightnvm/pblk-core.c nr_secs = pblk->lm.sec_per_line - line->cur_sec; pblk 627 drivers/lightnvm/pblk-core.c pblk->lm.sec_per_line, line->cur_sec); pblk 634 drivers/lightnvm/pblk-core.c u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) pblk 642 drivers/lightnvm/pblk-core.c addr = __pblk_alloc_page(pblk, line, nr_secs); pblk 650 drivers/lightnvm/pblk-core.c u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line) pblk 656 drivers/lightnvm/pblk-core.c pblk->lm.sec_per_line, line->cur_sec); pblk 662 drivers/lightnvm/pblk-core.c u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) pblk 664 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 666 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 677 drivers/lightnvm/pblk-core.c int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line) pblk 679 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 682 drivers/lightnvm/pblk-core.c u64 paddr = pblk_line_smeta_start(pblk, line); pblk 687 drivers/lightnvm/pblk-core.c ret = pblk_alloc_rqd_meta(pblk, &rqd); pblk 697 drivers/lightnvm/pblk-core.c ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); pblk 699 drivers/lightnvm/pblk-core.c ret = pblk_submit_io_sync(pblk, &rqd, line->smeta); pblk 701 drivers/lightnvm/pblk-core.c pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); pblk 705 drivers/lightnvm/pblk-core.c atomic_dec(&pblk->inflight_io); pblk 708 drivers/lightnvm/pblk-core.c pblk_log_read_err(pblk, &rqd); pblk 713 drivers/lightnvm/pblk-core.c pblk_free_rqd_meta(pblk, &rqd); pblk 717 drivers/lightnvm/pblk-core.c static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line, pblk 720 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 723 drivers/lightnvm/pblk-core.c __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); pblk 729 drivers/lightnvm/pblk-core.c ret = pblk_alloc_rqd_meta(pblk, &rqd); pblk 739 drivers/lightnvm/pblk-core.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, pblk 742 drivers/lightnvm/pblk-core.c ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); pblk 746 drivers/lightnvm/pblk-core.c ret = pblk_submit_io_sync_sem(pblk, &rqd, line->smeta); pblk 748 drivers/lightnvm/pblk-core.c pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); pblk 752 drivers/lightnvm/pblk-core.c atomic_dec(&pblk->inflight_io); pblk 755 drivers/lightnvm/pblk-core.c pblk_log_write_err(pblk, &rqd); pblk 760 drivers/lightnvm/pblk-core.c pblk_free_rqd_meta(pblk, &rqd); pblk 764 drivers/lightnvm/pblk-core.c int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, pblk 767 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 769 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 775 drivers/lightnvm/pblk-core.c int min = pblk->min_write_pgs; pblk 787 drivers/lightnvm/pblk-core.c ppa_list_buf = meta_list + pblk_dma_meta_size(pblk); pblk 788 drivers/lightnvm/pblk-core.c dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk); pblk 793 drivers/lightnvm/pblk-core.c rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); pblk 805 drivers/lightnvm/pblk-core.c struct ppa_addr ppa = addr_to_gen_ppa(pblk, paddr, line_id); pblk 808 drivers/lightnvm/pblk-core.c if (pblk_io_aligned(pblk, rq_ppas)) pblk 813 drivers/lightnvm/pblk-core.c if (pblk_boundary_paddr_checks(pblk, paddr)) { pblk 818 drivers/lightnvm/pblk-core.c ppa = addr_to_gen_ppa(pblk, paddr, line_id); pblk 822 drivers/lightnvm/pblk-core.c if (pblk_boundary_paddr_checks(pblk, paddr + min)) { pblk 828 drivers/lightnvm/pblk-core.c ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line_id); pblk 831 drivers/lightnvm/pblk-core.c ret = pblk_submit_io_sync(pblk, &rqd, emeta_buf); pblk 833 drivers/lightnvm/pblk-core.c pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); pblk 837 drivers/lightnvm/pblk-core.c atomic_dec(&pblk->inflight_io); pblk 840 drivers/lightnvm/pblk-core.c pblk_log_read_err(pblk, &rqd); pblk 855 drivers/lightnvm/pblk-core.c static void pblk_setup_e_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 865 drivers/lightnvm/pblk-core.c static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa) pblk 870 drivers/lightnvm/pblk-core.c trace_pblk_chunk_reset(pblk_disk_name(pblk), &ppa, pblk 873 drivers/lightnvm/pblk-core.c pblk_setup_e_rq(pblk, &rqd, ppa); pblk 878 drivers/lightnvm/pblk-core.c ret = pblk_submit_io_sync(pblk, &rqd, NULL); pblk 879 drivers/lightnvm/pblk-core.c rqd.private = pblk; pblk 880 drivers/lightnvm/pblk-core.c __pblk_end_io_erase(pblk, &rqd); pblk 885 drivers/lightnvm/pblk-core.c int pblk_line_erase(struct pblk *pblk, struct pblk_line *line) pblk 887 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 901 drivers/lightnvm/pblk-core.c ppa = pblk->luns[bit].bppa; /* set ch and lun */ pblk 908 drivers/lightnvm/pblk-core.c ret = pblk_blk_erase_sync(pblk, ppa); pblk 910 drivers/lightnvm/pblk-core.c pblk_err(pblk, "failed to erase line %d\n", line->id); pblk 951 drivers/lightnvm/pblk-core.c static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, pblk 954 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 956 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 957 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 972 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 979 drivers/lightnvm/pblk-core.c pblk_debug(pblk, "line %d is bad\n", line->id); pblk 991 drivers/lightnvm/pblk-core.c guid_copy((guid_t *)&smeta_buf->header.uuid, &pblk->instance_uuid); pblk 1012 drivers/lightnvm/pblk-core.c pblk_calc_meta_header_crc(pblk, &smeta_buf->header)); pblk 1013 drivers/lightnvm/pblk-core.c smeta_buf->crc = cpu_to_le32(pblk_calc_smeta_crc(pblk, smeta_buf)); pblk 1022 drivers/lightnvm/pblk-core.c pblk_calc_meta_header_crc(pblk, &emeta_buf->header)); pblk 1034 drivers/lightnvm/pblk-core.c static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line) pblk 1036 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1037 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1059 drivers/lightnvm/pblk-core.c static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, pblk 1062 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1064 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1065 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1090 drivers/lightnvm/pblk-core.c if (init && pblk_line_smeta_write(pblk, line, off)) { pblk 1091 drivers/lightnvm/pblk-core.c pblk_debug(pblk, "line smeta I/O failed. Retry\n"); pblk 1120 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1125 drivers/lightnvm/pblk-core.c pblk_err(pblk, "unexpected line %d is bad\n", line->id); pblk 1133 drivers/lightnvm/pblk-core.c static int pblk_prepare_new_line(struct pblk *pblk, struct pblk_line *line) pblk 1135 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1136 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1142 drivers/lightnvm/pblk-core.c struct pblk_lun *rlun = &pblk->luns[i]; pblk 1157 drivers/lightnvm/pblk-core.c static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) pblk 1159 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1172 drivers/lightnvm/pblk-core.c blk_to_erase = pblk_prepare_new_line(pblk, line); pblk 1174 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1193 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1209 drivers/lightnvm/pblk-core.c int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line) pblk 1211 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1218 drivers/lightnvm/pblk-core.c ret = pblk_line_prepare(pblk, line); pblk 1226 drivers/lightnvm/pblk-core.c ret = pblk_line_alloc_bitmaps(pblk, line); pblk 1230 drivers/lightnvm/pblk-core.c if (!pblk_line_init_bb(pblk, line, 0)) { pblk 1235 drivers/lightnvm/pblk-core.c pblk_rl_free_lines_dec(&pblk->rl, line, true); pblk 1246 drivers/lightnvm/pblk-core.c void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line) pblk 1248 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1268 drivers/lightnvm/pblk-core.c struct pblk *pblk = line->pblk; pblk 1269 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1277 drivers/lightnvm/pblk-core.c struct pblk_line *pblk_line_get(struct pblk *pblk) pblk 1279 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1280 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1288 drivers/lightnvm/pblk-core.c pblk_err(pblk, "no free lines\n"); pblk 1300 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1306 drivers/lightnvm/pblk-core.c pblk_debug(pblk, "line %d is bad\n", line->id); pblk 1310 drivers/lightnvm/pblk-core.c ret = pblk_line_prepare(pblk, line); pblk 1320 drivers/lightnvm/pblk-core.c pblk_err(pblk, "failed to prepare line %d\n", line->id); pblk 1330 drivers/lightnvm/pblk-core.c static struct pblk_line *pblk_line_retry(struct pblk *pblk, pblk 1333 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1338 drivers/lightnvm/pblk-core.c retry_line = pblk_line_get(pblk); pblk 1356 drivers/lightnvm/pblk-core.c pblk_rl_free_lines_dec(&pblk->rl, line, false); pblk 1358 drivers/lightnvm/pblk-core.c if (pblk_line_erase(pblk, retry_line)) pblk 1364 drivers/lightnvm/pblk-core.c static void pblk_set_space_limit(struct pblk *pblk) pblk 1366 drivers/lightnvm/pblk-core.c struct pblk_rl *rl = &pblk->rl; pblk 1371 drivers/lightnvm/pblk-core.c struct pblk_line *pblk_line_get_first_data(struct pblk *pblk) pblk 1373 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1377 drivers/lightnvm/pblk-core.c line = pblk_line_get(pblk); pblk 1387 drivers/lightnvm/pblk-core.c pblk_line_setup_metadata(line, l_mg, &pblk->lm); pblk 1390 drivers/lightnvm/pblk-core.c l_mg->data_next = pblk_line_get(pblk); pblk 1396 drivers/lightnvm/pblk-core.c pblk_set_space_limit(pblk); pblk 1405 drivers/lightnvm/pblk-core.c if (pblk_line_alloc_bitmaps(pblk, line)) pblk 1408 drivers/lightnvm/pblk-core.c if (pblk_line_erase(pblk, line)) { pblk 1409 drivers/lightnvm/pblk-core.c line = pblk_line_retry(pblk, line); pblk 1415 drivers/lightnvm/pblk-core.c if (!pblk_line_init_metadata(pblk, line, NULL)) { pblk 1416 drivers/lightnvm/pblk-core.c line = pblk_line_retry(pblk, line); pblk 1423 drivers/lightnvm/pblk-core.c if (!pblk_line_init_bb(pblk, line, 1)) { pblk 1424 drivers/lightnvm/pblk-core.c line = pblk_line_retry(pblk, line); pblk 1431 drivers/lightnvm/pblk-core.c pblk_rl_free_lines_dec(&pblk->rl, line, true); pblk 1436 drivers/lightnvm/pblk-core.c void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa) pblk 1440 drivers/lightnvm/pblk-core.c line = pblk_ppa_to_line(pblk, ppa); pblk 1444 drivers/lightnvm/pblk-core.c void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd) pblk 1450 drivers/lightnvm/pblk-core.c pblk_ppa_to_line_put(pblk, ppa_list[i]); pblk 1453 drivers/lightnvm/pblk-core.c static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line) pblk 1455 drivers/lightnvm/pblk-core.c lockdep_assert_held(&pblk->l_mg.free_lock); pblk 1457 drivers/lightnvm/pblk-core.c pblk_set_space_limit(pblk); pblk 1458 drivers/lightnvm/pblk-core.c pblk->state = PBLK_STATE_STOPPING; pblk 1459 drivers/lightnvm/pblk-core.c trace_pblk_state(pblk_disk_name(pblk), pblk->state); pblk 1462 drivers/lightnvm/pblk-core.c static void pblk_line_close_meta_sync(struct pblk *pblk) pblk 1464 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1465 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1484 drivers/lightnvm/pblk-core.c ret = pblk_submit_meta_io(pblk, line); pblk 1486 drivers/lightnvm/pblk-core.c pblk_err(pblk, "sync meta line %d failed (%d)\n", pblk 1493 drivers/lightnvm/pblk-core.c pblk_wait_for_meta(pblk); pblk 1494 drivers/lightnvm/pblk-core.c flush_workqueue(pblk->close_wq); pblk 1497 drivers/lightnvm/pblk-core.c void __pblk_pipeline_flush(struct pblk *pblk) pblk 1499 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1503 drivers/lightnvm/pblk-core.c if (pblk->state == PBLK_STATE_RECOVERING || pblk 1504 drivers/lightnvm/pblk-core.c pblk->state == PBLK_STATE_STOPPED) { pblk 1508 drivers/lightnvm/pblk-core.c pblk->state = PBLK_STATE_RECOVERING; pblk 1509 drivers/lightnvm/pblk-core.c trace_pblk_state(pblk_disk_name(pblk), pblk->state); pblk 1512 drivers/lightnvm/pblk-core.c pblk_flush_writer(pblk); pblk 1513 drivers/lightnvm/pblk-core.c pblk_wait_for_meta(pblk); pblk 1515 drivers/lightnvm/pblk-core.c ret = pblk_recov_pad(pblk); pblk 1517 drivers/lightnvm/pblk-core.c pblk_err(pblk, "could not close data on teardown(%d)\n", ret); pblk 1521 drivers/lightnvm/pblk-core.c flush_workqueue(pblk->bb_wq); pblk 1522 drivers/lightnvm/pblk-core.c pblk_line_close_meta_sync(pblk); pblk 1525 drivers/lightnvm/pblk-core.c void __pblk_pipeline_stop(struct pblk *pblk) pblk 1527 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1530 drivers/lightnvm/pblk-core.c pblk->state = PBLK_STATE_STOPPED; pblk 1531 drivers/lightnvm/pblk-core.c trace_pblk_state(pblk_disk_name(pblk), pblk->state); pblk 1537 drivers/lightnvm/pblk-core.c void pblk_pipeline_stop(struct pblk *pblk) pblk 1539 drivers/lightnvm/pblk-core.c __pblk_pipeline_flush(pblk); pblk 1540 drivers/lightnvm/pblk-core.c __pblk_pipeline_stop(pblk); pblk 1543 drivers/lightnvm/pblk-core.c struct pblk_line *pblk_line_replace_data(struct pblk *pblk) pblk 1545 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1557 drivers/lightnvm/pblk-core.c pblk_line_setup_metadata(new, l_mg, &pblk->lm); pblk 1565 drivers/lightnvm/pblk-core.c if (pblk_line_erase(pblk, new)) pblk 1573 drivers/lightnvm/pblk-core.c if (pblk_line_alloc_bitmaps(pblk, new)) pblk 1577 drivers/lightnvm/pblk-core.c if (!pblk_line_init_metadata(pblk, new, cur)) { pblk 1578 drivers/lightnvm/pblk-core.c new = pblk_line_retry(pblk, new); pblk 1585 drivers/lightnvm/pblk-core.c if (!pblk_line_init_bb(pblk, new, 1)) { pblk 1586 drivers/lightnvm/pblk-core.c new = pblk_line_retry(pblk, new); pblk 1593 drivers/lightnvm/pblk-core.c pblk_rl_free_lines_dec(&pblk->rl, new, true); pblk 1597 drivers/lightnvm/pblk-core.c l_mg->data_next = pblk_line_get(pblk); pblk 1603 drivers/lightnvm/pblk-core.c pblk_stop_writes(pblk, new); pblk 1615 drivers/lightnvm/pblk-core.c static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line) pblk 1617 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1618 drivers/lightnvm/pblk-core.c struct pblk_gc *gc = &pblk->gc; pblk 1624 drivers/lightnvm/pblk-core.c pblk_err(pblk, "line %d had errors during GC\n", line->id); pblk 1625 drivers/lightnvm/pblk-core.c pblk_put_line_back(pblk, line); pblk 1631 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1637 drivers/lightnvm/pblk-core.c pblk_rl_werr_line_out(&pblk->rl); pblk 1649 drivers/lightnvm/pblk-core.c pblk_rl_free_lines_inc(&pblk->rl, line); pblk 1656 drivers/lightnvm/pblk-core.c struct pblk *pblk = line_put_ws->pblk; pblk 1659 drivers/lightnvm/pblk-core.c __pblk_line_put(pblk, line); pblk 1660 drivers/lightnvm/pblk-core.c mempool_free(line_put_ws, &pblk->gen_ws_pool); pblk 1666 drivers/lightnvm/pblk-core.c struct pblk *pblk = line->pblk; pblk 1668 drivers/lightnvm/pblk-core.c __pblk_line_put(pblk, line); pblk 1674 drivers/lightnvm/pblk-core.c struct pblk *pblk = line->pblk; pblk 1677 drivers/lightnvm/pblk-core.c line_put_ws = mempool_alloc(&pblk->gen_ws_pool, GFP_ATOMIC); pblk 1681 drivers/lightnvm/pblk-core.c line_put_ws->pblk = pblk; pblk 1686 drivers/lightnvm/pblk-core.c queue_work(pblk->r_end_wq, &line_put_ws->ws); pblk 1689 drivers/lightnvm/pblk-core.c int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa) pblk 1694 drivers/lightnvm/pblk-core.c rqd = pblk_alloc_rqd(pblk, PBLK_ERASE); pblk 1696 drivers/lightnvm/pblk-core.c pblk_setup_e_rq(pblk, rqd, ppa); pblk 1699 drivers/lightnvm/pblk-core.c rqd->private = pblk; pblk 1701 drivers/lightnvm/pblk-core.c trace_pblk_chunk_reset(pblk_disk_name(pblk), pblk 1707 drivers/lightnvm/pblk-core.c err = pblk_submit_io(pblk, rqd, NULL); pblk 1709 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1712 drivers/lightnvm/pblk-core.c pblk_err(pblk, "could not async erase line:%d,blk:%d\n", pblk 1720 drivers/lightnvm/pblk-core.c struct pblk_line *pblk_line_get_data(struct pblk *pblk) pblk 1722 drivers/lightnvm/pblk-core.c return pblk->l_mg.data_line; pblk 1726 drivers/lightnvm/pblk-core.c struct pblk_line *pblk_line_get_erase(struct pblk *pblk) pblk 1728 drivers/lightnvm/pblk-core.c return pblk->l_mg.data_next; pblk 1736 drivers/lightnvm/pblk-core.c static void pblk_line_should_sync_meta(struct pblk *pblk) pblk 1738 drivers/lightnvm/pblk-core.c if (pblk_rl_is_limit(&pblk->rl)) pblk 1739 drivers/lightnvm/pblk-core.c pblk_line_close_meta_sync(pblk); pblk 1742 drivers/lightnvm/pblk-core.c void pblk_line_close(struct pblk *pblk, struct pblk_line *line) pblk 1744 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1746 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1747 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1764 drivers/lightnvm/pblk-core.c move_list = pblk_line_gc_list(pblk, line); pblk 1773 drivers/lightnvm/pblk-core.c struct pblk_lun *rlun = &pblk->luns[i]; pblk 1784 drivers/lightnvm/pblk-core.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1788 drivers/lightnvm/pblk-core.c void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) pblk 1790 drivers/lightnvm/pblk-core.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1791 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1797 drivers/lightnvm/pblk-core.c memcpy(emeta_to_vsc(pblk, emeta_buf), l_mg->vsc_list, lm->vsc_list_len); pblk 1800 drivers/lightnvm/pblk-core.c wa->user = cpu_to_le64(atomic64_read(&pblk->user_wa)); pblk 1801 drivers/lightnvm/pblk-core.c wa->pad = cpu_to_le64(atomic64_read(&pblk->pad_wa)); pblk 1802 drivers/lightnvm/pblk-core.c wa->gc = cpu_to_le64(atomic64_read(&pblk->gc_wa)); pblk 1807 drivers/lightnvm/pblk-core.c &pblk->instance_uuid); pblk 1813 drivers/lightnvm/pblk-core.c pblk_calc_meta_header_crc(pblk, &emeta_buf->header)); pblk 1817 drivers/lightnvm/pblk-core.c emeta_buf->crc = cpu_to_le32(pblk_calc_emeta_crc(pblk, emeta_buf)); pblk 1832 drivers/lightnvm/pblk-core.c pblk_line_should_sync_meta(pblk); pblk 1835 drivers/lightnvm/pblk-core.c static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line) pblk 1837 drivers/lightnvm/pblk-core.c struct pblk_line_meta *lm = &pblk->lm; pblk 1843 drivers/lightnvm/pblk-core.c memcpy(w_err_gc->lba_list, emeta_to_lbas(pblk, emeta->buf), pblk 1851 drivers/lightnvm/pblk-core.c struct pblk *pblk = line_ws->pblk; pblk 1859 drivers/lightnvm/pblk-core.c pblk_save_lba_list(pblk, line); pblk 1861 drivers/lightnvm/pblk-core.c pblk_line_close(pblk, line); pblk 1862 drivers/lightnvm/pblk-core.c mempool_free(line_ws, &pblk->gen_ws_pool); pblk 1865 drivers/lightnvm/pblk-core.c void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, pblk 1871 drivers/lightnvm/pblk-core.c line_ws = mempool_alloc(&pblk->gen_ws_pool, gfp_mask); pblk 1873 drivers/lightnvm/pblk-core.c line_ws->pblk = pblk; pblk 1881 drivers/lightnvm/pblk-core.c static void __pblk_down_chunk(struct pblk *pblk, int pos) pblk 1883 drivers/lightnvm/pblk-core.c struct pblk_lun *rlun = &pblk->luns[pos]; pblk 1893 drivers/lightnvm/pblk-core.c pblk_err(pblk, "taking lun semaphore timed out: err %d\n", pblk 1897 drivers/lightnvm/pblk-core.c void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa) pblk 1899 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1903 drivers/lightnvm/pblk-core.c __pblk_down_chunk(pblk, pos); pblk 1906 drivers/lightnvm/pblk-core.c void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa, pblk 1909 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1919 drivers/lightnvm/pblk-core.c __pblk_down_chunk(pblk, pos); pblk 1922 drivers/lightnvm/pblk-core.c void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa) pblk 1924 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1929 drivers/lightnvm/pblk-core.c rlun = &pblk->luns[pos]; pblk 1933 drivers/lightnvm/pblk-core.c void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap) pblk 1935 drivers/lightnvm/pblk-core.c struct nvm_tgt_dev *dev = pblk->dev; pblk 1942 drivers/lightnvm/pblk-core.c rlun = &pblk->luns[bit]; pblk 1947 drivers/lightnvm/pblk-core.c void pblk_update_map(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) pblk 1952 drivers/lightnvm/pblk-core.c if (!(lba < pblk->capacity)) { pblk 1957 drivers/lightnvm/pblk-core.c spin_lock(&pblk->trans_lock); pblk 1958 drivers/lightnvm/pblk-core.c ppa_l2p = pblk_trans_map_get(pblk, lba); pblk 1961 drivers/lightnvm/pblk-core.c pblk_map_invalidate(pblk, ppa_l2p); pblk 1963 drivers/lightnvm/pblk-core.c pblk_trans_map_set(pblk, lba, ppa); pblk 1964 drivers/lightnvm/pblk-core.c spin_unlock(&pblk->trans_lock); pblk 1967 drivers/lightnvm/pblk-core.c void pblk_update_map_cache(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) pblk 1973 drivers/lightnvm/pblk-core.c BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa))); pblk 1976 drivers/lightnvm/pblk-core.c pblk_update_map(pblk, lba, ppa); pblk 1979 drivers/lightnvm/pblk-core.c int pblk_update_map_gc(struct pblk *pblk, sector_t lba, struct ppa_addr ppa_new, pblk 1988 drivers/lightnvm/pblk-core.c BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa_new))); pblk 1992 drivers/lightnvm/pblk-core.c if (!(lba < pblk->capacity)) { pblk 1997 drivers/lightnvm/pblk-core.c spin_lock(&pblk->trans_lock); pblk 1998 drivers/lightnvm/pblk-core.c ppa_l2p = pblk_trans_map_get(pblk, lba); pblk 1999 drivers/lightnvm/pblk-core.c ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, gc_line->id); pblk 2011 drivers/lightnvm/pblk-core.c pblk_trans_map_set(pblk, lba, ppa_new); pblk 2013 drivers/lightnvm/pblk-core.c spin_unlock(&pblk->trans_lock); pblk 2017 drivers/lightnvm/pblk-core.c void pblk_update_map_dev(struct pblk *pblk, sector_t lba, pblk 2028 drivers/lightnvm/pblk-core.c atomic64_inc(&pblk->pad_wa); pblk 2030 drivers/lightnvm/pblk-core.c atomic_long_inc(&pblk->padded_wb); pblk 2033 drivers/lightnvm/pblk-core.c pblk_map_invalidate(pblk, ppa_mapped); pblk 2038 drivers/lightnvm/pblk-core.c if (!(lba < pblk->capacity)) { pblk 2043 drivers/lightnvm/pblk-core.c spin_lock(&pblk->trans_lock); pblk 2044 drivers/lightnvm/pblk-core.c ppa_l2p = pblk_trans_map_get(pblk, lba); pblk 2051 drivers/lightnvm/pblk-core.c pblk_map_invalidate(pblk, ppa_mapped); pblk 2059 drivers/lightnvm/pblk-core.c pblk_trans_map_set(pblk, lba, ppa_mapped); pblk 2061 drivers/lightnvm/pblk-core.c spin_unlock(&pblk->trans_lock); pblk 2064 drivers/lightnvm/pblk-core.c int pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas, pblk 2069 drivers/lightnvm/pblk-core.c spin_lock(&pblk->trans_lock); pblk 2073 drivers/lightnvm/pblk-core.c ppa = ppas[i] = pblk_trans_map_get(pblk, blba + i); pblk 2077 drivers/lightnvm/pblk-core.c struct pblk_line *line = pblk_ppa_to_line(pblk, ppa); pblk 2090 drivers/lightnvm/pblk-core.c spin_unlock(&pblk->trans_lock); pblk 2094 drivers/lightnvm/pblk-core.c void pblk_lookup_l2p_rand(struct pblk *pblk, struct ppa_addr *ppas, pblk 2100 drivers/lightnvm/pblk-core.c spin_lock(&pblk->trans_lock); pblk 2105 drivers/lightnvm/pblk-core.c if (!(lba < pblk->capacity)) { pblk 2109 drivers/lightnvm/pblk-core.c ppas[i] = pblk_trans_map_get(pblk, lba); pblk 2112 drivers/lightnvm/pblk-core.c spin_unlock(&pblk->trans_lock); pblk 2115 drivers/lightnvm/pblk-core.c void *pblk_get_meta_for_writes(struct pblk *pblk, struct nvm_rq *rqd) pblk 2119 drivers/lightnvm/pblk-core.c if (pblk_is_oob_meta_supported(pblk)) { pblk 2133 drivers/lightnvm/pblk-core.c void pblk_get_packed_meta(struct pblk *pblk, struct nvm_rq *rqd) pblk 2139 drivers/lightnvm/pblk-core.c if (pblk_is_oob_meta_supported(pblk)) pblk 2145 drivers/lightnvm/pblk-core.c memcpy(pblk_get_meta(pblk, meta_list, i), pblk 31 drivers/lightnvm/pblk-gc.c static int pblk_gc_write(struct pblk *pblk) pblk 33 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 48 drivers/lightnvm/pblk-gc.c pblk_write_gc_to_cache(pblk, gc_rq); pblk 62 drivers/lightnvm/pblk-gc.c void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line) pblk 64 drivers/lightnvm/pblk-gc.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 71 drivers/lightnvm/pblk-gc.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 80 drivers/lightnvm/pblk-gc.c move_list = pblk_line_gc_list(pblk, line); pblk 90 drivers/lightnvm/pblk-gc.c struct pblk *pblk = gc_rq_ws->pblk; pblk 91 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 99 drivers/lightnvm/pblk-gc.c ret = pblk_submit_read_gc(pblk, gc_rq); pblk 112 drivers/lightnvm/pblk-gc.c pblk_gc_writer_kick(&pblk->gc); pblk 120 drivers/lightnvm/pblk-gc.c pblk_gc_writer_kick(&pblk->gc); pblk 131 drivers/lightnvm/pblk-gc.c static __le64 *get_lba_list_from_emeta(struct pblk *pblk, pblk 135 drivers/lightnvm/pblk-gc.c struct pblk_line_meta *lm = &pblk->lm; pblk 144 drivers/lightnvm/pblk-gc.c ret = pblk_line_emeta_read(pblk, line, emeta_buf); pblk 146 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "line %d read emeta failed (%d)\n", pblk 158 drivers/lightnvm/pblk-gc.c ret = pblk_recov_check_emeta(pblk, emeta_buf); pblk 160 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "inconsistent emeta (line %d)\n", pblk 169 drivers/lightnvm/pblk-gc.c memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size); pblk 180 drivers/lightnvm/pblk-gc.c struct pblk *pblk = line_ws->pblk; pblk 182 drivers/lightnvm/pblk-gc.c struct pblk_line_meta *lm = &pblk->lm; pblk 183 drivers/lightnvm/pblk-gc.c struct nvm_tgt_dev *dev = pblk->dev; pblk 185 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 200 drivers/lightnvm/pblk-gc.c lba_list = get_lba_list_from_emeta(pblk, line); pblk 202 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "could not interpret emeta (line %d)\n", pblk 214 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "corrupted GC line (%d)\n", line->id); pblk 233 drivers/lightnvm/pblk-gc.c } while (nr_secs < pblk->max_write_pgs); pblk 251 drivers/lightnvm/pblk-gc.c gc_rq_ws->pblk = pblk; pblk 296 drivers/lightnvm/pblk-gc.c pblk_put_line_back(pblk, line); pblk 299 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "failed to GC line %d\n", line->id); pblk 302 drivers/lightnvm/pblk-gc.c static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line) pblk 304 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 307 drivers/lightnvm/pblk-gc.c pblk_debug(pblk, "line '%d' being reclaimed for GC\n", line->id); pblk 313 drivers/lightnvm/pblk-gc.c line_ws->pblk = pblk; pblk 328 drivers/lightnvm/pblk-gc.c static void pblk_gc_kick(struct pblk *pblk) pblk 330 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 343 drivers/lightnvm/pblk-gc.c static int pblk_gc_read(struct pblk *pblk) pblk 345 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 358 drivers/lightnvm/pblk-gc.c pblk_gc_kick(pblk); pblk 360 drivers/lightnvm/pblk-gc.c if (pblk_gc_line(pblk, line)) { pblk 361 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "failed to GC line %d\n", line->id); pblk 371 drivers/lightnvm/pblk-gc.c static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk, pblk 407 drivers/lightnvm/pblk-gc.c void pblk_gc_free_full_lines(struct pblk *pblk) pblk 409 drivers/lightnvm/pblk-gc.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 410 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 426 drivers/lightnvm/pblk-gc.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 444 drivers/lightnvm/pblk-gc.c static void pblk_gc_run(struct pblk *pblk) pblk 446 drivers/lightnvm/pblk-gc.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 447 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 453 drivers/lightnvm/pblk-gc.c pblk_gc_free_full_lines(pblk); pblk 455 drivers/lightnvm/pblk-gc.c run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl); pblk 465 drivers/lightnvm/pblk-gc.c line = pblk_gc_get_victim_line(pblk, group_list); pblk 474 drivers/lightnvm/pblk-gc.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 491 drivers/lightnvm/pblk-gc.c run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl); pblk 496 drivers/lightnvm/pblk-gc.c if (!prev_group && pblk->rl.rb_state > gc_group && pblk 503 drivers/lightnvm/pblk-gc.c struct pblk *pblk = from_timer(pblk, t, gc.gc_timer); pblk 505 drivers/lightnvm/pblk-gc.c pblk_gc_kick(pblk); pblk 510 drivers/lightnvm/pblk-gc.c struct pblk *pblk = data; pblk 513 drivers/lightnvm/pblk-gc.c pblk_gc_run(pblk); pblk 523 drivers/lightnvm/pblk-gc.c struct pblk *pblk = data; pblk 526 drivers/lightnvm/pblk-gc.c if (!pblk_gc_write(pblk)) pblk 537 drivers/lightnvm/pblk-gc.c struct pblk *pblk = data; pblk 538 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 541 drivers/lightnvm/pblk-gc.c if (!pblk_gc_read(pblk)) pblk 548 drivers/lightnvm/pblk-gc.c pblk_info(pblk, "flushing gc pipeline, %d lines left\n", pblk 562 drivers/lightnvm/pblk-gc.c static void pblk_gc_start(struct pblk *pblk) pblk 564 drivers/lightnvm/pblk-gc.c pblk->gc.gc_active = 1; pblk 565 drivers/lightnvm/pblk-gc.c pblk_debug(pblk, "gc start\n"); pblk 568 drivers/lightnvm/pblk-gc.c void pblk_gc_should_start(struct pblk *pblk) pblk 570 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 573 drivers/lightnvm/pblk-gc.c pblk_gc_start(pblk); pblk 574 drivers/lightnvm/pblk-gc.c pblk_gc_kick(pblk); pblk 578 drivers/lightnvm/pblk-gc.c void pblk_gc_should_stop(struct pblk *pblk) pblk 580 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 586 drivers/lightnvm/pblk-gc.c void pblk_gc_should_kick(struct pblk *pblk) pblk 588 drivers/lightnvm/pblk-gc.c pblk_rl_update_rates(&pblk->rl); pblk 591 drivers/lightnvm/pblk-gc.c void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled, pblk 594 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 602 drivers/lightnvm/pblk-gc.c int pblk_gc_sysfs_force(struct pblk *pblk, int force) pblk 604 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 618 drivers/lightnvm/pblk-gc.c pblk_gc_should_start(pblk); pblk 623 drivers/lightnvm/pblk-gc.c int pblk_gc_init(struct pblk *pblk) pblk 625 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 628 drivers/lightnvm/pblk-gc.c gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts"); pblk 630 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "could not allocate GC main kthread\n"); pblk 634 drivers/lightnvm/pblk-gc.c gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk, pblk 637 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "could not allocate GC writer kthread\n"); pblk 642 drivers/lightnvm/pblk-gc.c gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk, pblk 645 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "could not allocate GC reader kthread\n"); pblk 666 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "could not allocate GC line reader workqueue\n"); pblk 675 drivers/lightnvm/pblk-gc.c pblk_err(pblk, "could not allocate GC reader workqueue\n"); pblk 703 drivers/lightnvm/pblk-gc.c void pblk_gc_exit(struct pblk *pblk, bool graceful) pblk 705 drivers/lightnvm/pblk-gc.c struct pblk_gc *gc = &pblk->gc; pblk 52 drivers/lightnvm/pblk-init.c struct pblk *pblk = q->queuedata; pblk 55 drivers/lightnvm/pblk-init.c pblk_discard(pblk, bio); pblk 67 drivers/lightnvm/pblk-init.c pblk_submit_read(pblk, bio); pblk 73 drivers/lightnvm/pblk-init.c if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl)) pblk 76 drivers/lightnvm/pblk-init.c pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER); pblk 82 drivers/lightnvm/pblk-init.c static size_t pblk_trans_map_size(struct pblk *pblk) pblk 86 drivers/lightnvm/pblk-init.c if (pblk->addrf_len < 32) pblk 89 drivers/lightnvm/pblk-init.c return entry_size * pblk->capacity; pblk 93 drivers/lightnvm/pblk-init.c static u32 pblk_l2p_crc(struct pblk *pblk) pblk 98 drivers/lightnvm/pblk-init.c map_size = pblk_trans_map_size(pblk); pblk 99 drivers/lightnvm/pblk-init.c crc = crc32_le(crc, pblk->trans_map, map_size); pblk 104 drivers/lightnvm/pblk-init.c static void pblk_l2p_free(struct pblk *pblk) pblk 106 drivers/lightnvm/pblk-init.c vfree(pblk->trans_map); pblk 109 drivers/lightnvm/pblk-init.c static int pblk_l2p_recover(struct pblk *pblk, bool factory_init) pblk 114 drivers/lightnvm/pblk-init.c guid_gen(&pblk->instance_uuid); pblk 116 drivers/lightnvm/pblk-init.c line = pblk_recov_l2p(pblk); pblk 118 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not recover l2p table\n"); pblk 124 drivers/lightnvm/pblk-init.c pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk)); pblk 128 drivers/lightnvm/pblk-init.c pblk_gc_free_full_lines(pblk); pblk 132 drivers/lightnvm/pblk-init.c line = pblk_line_get_first_data(pblk); pblk 140 drivers/lightnvm/pblk-init.c static int pblk_l2p_init(struct pblk *pblk, bool factory_init) pblk 147 drivers/lightnvm/pblk-init.c map_size = pblk_trans_map_size(pblk); pblk 148 drivers/lightnvm/pblk-init.c pblk->trans_map = __vmalloc(map_size, GFP_KERNEL | __GFP_NOWARN pblk 151 drivers/lightnvm/pblk-init.c if (!pblk->trans_map) { pblk 152 drivers/lightnvm/pblk-init.c pblk_err(pblk, "failed to allocate L2P (need %zu of memory)\n", pblk 159 drivers/lightnvm/pblk-init.c for (i = 0; i < pblk->capacity; i++) pblk 160 drivers/lightnvm/pblk-init.c pblk_trans_map_set(pblk, i, ppa); pblk 162 drivers/lightnvm/pblk-init.c ret = pblk_l2p_recover(pblk, factory_init); pblk 164 drivers/lightnvm/pblk-init.c vfree(pblk->trans_map); pblk 169 drivers/lightnvm/pblk-init.c static void pblk_rwb_free(struct pblk *pblk) pblk 171 drivers/lightnvm/pblk-init.c if (pblk_rb_tear_down_check(&pblk->rwb)) pblk 172 drivers/lightnvm/pblk-init.c pblk_err(pblk, "write buffer error on tear down\n"); pblk 174 drivers/lightnvm/pblk-init.c pblk_rb_free(&pblk->rwb); pblk 177 drivers/lightnvm/pblk-init.c static int pblk_rwb_init(struct pblk *pblk) pblk 179 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 193 drivers/lightnvm/pblk-init.c return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs); pblk 196 drivers/lightnvm/pblk-init.c static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo, pblk 205 drivers/lightnvm/pblk-init.c pblk_err(pblk, "supports only power-of-two channel config.\n"); pblk 212 drivers/lightnvm/pblk-init.c pblk_err(pblk, "supports only power-of-two LUN config.\n"); pblk 269 drivers/lightnvm/pblk-init.c static int pblk_set_addrf(struct pblk *pblk) pblk 271 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 277 drivers/lightnvm/pblk-init.c div_u64_rem(geo->clba, pblk->min_write_pgs, &mod); pblk 279 drivers/lightnvm/pblk-init.c pblk_err(pblk, "bad configuration of sectors/pages\n"); pblk 283 drivers/lightnvm/pblk-init.c pblk->addrf_len = pblk_set_addrf_12(pblk, geo, pblk 284 drivers/lightnvm/pblk-init.c (void *)&pblk->addrf); pblk 287 drivers/lightnvm/pblk-init.c pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf, pblk 288 drivers/lightnvm/pblk-init.c &pblk->uaddrf); pblk 291 drivers/lightnvm/pblk-init.c pblk_err(pblk, "OCSSD revision not supported (%d)\n", pblk 371 drivers/lightnvm/pblk-init.c static int pblk_core_init(struct pblk *pblk) pblk 373 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 377 drivers/lightnvm/pblk-init.c atomic64_set(&pblk->user_wa, 0); pblk 378 drivers/lightnvm/pblk-init.c atomic64_set(&pblk->pad_wa, 0); pblk 379 drivers/lightnvm/pblk-init.c atomic64_set(&pblk->gc_wa, 0); pblk 380 drivers/lightnvm/pblk-init.c pblk->user_rst_wa = 0; pblk 381 drivers/lightnvm/pblk-init.c pblk->pad_rst_wa = 0; pblk 382 drivers/lightnvm/pblk-init.c pblk->gc_rst_wa = 0; pblk 384 drivers/lightnvm/pblk-init.c atomic64_set(&pblk->nr_flush, 0); pblk 385 drivers/lightnvm/pblk-init.c pblk->nr_flush_rst = 0; pblk 387 drivers/lightnvm/pblk-init.c pblk->min_write_pgs = geo->ws_opt; pblk 388 drivers/lightnvm/pblk-init.c pblk->min_write_pgs_data = pblk->min_write_pgs; pblk 389 drivers/lightnvm/pblk-init.c max_write_ppas = pblk->min_write_pgs * geo->all_luns; pblk 390 drivers/lightnvm/pblk-init.c pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA); pblk 391 drivers/lightnvm/pblk-init.c pblk->max_write_pgs = min_t(int, pblk->max_write_pgs, pblk 393 drivers/lightnvm/pblk-init.c pblk_set_sec_per_write(pblk, pblk->min_write_pgs); pblk 395 drivers/lightnvm/pblk-init.c pblk->oob_meta_size = geo->sos; pblk 396 drivers/lightnvm/pblk-init.c if (!pblk_is_oob_meta_supported(pblk)) { pblk 404 drivers/lightnvm/pblk-init.c if (pblk->min_write_pgs pblk 414 drivers/lightnvm/pblk-init.c pblk_err(pblk, "Not supported min write size\n"); pblk 423 drivers/lightnvm/pblk-init.c pblk->max_write_pgs = pblk->min_write_pgs; pblk 424 drivers/lightnvm/pblk-init.c pblk->min_write_pgs_data = pblk->min_write_pgs - 1; pblk 427 drivers/lightnvm/pblk-init.c pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t), pblk 429 drivers/lightnvm/pblk-init.c if (!pblk->pad_dist) pblk 436 drivers/lightnvm/pblk-init.c ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0); pblk 440 drivers/lightnvm/pblk-init.c ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE, pblk 445 drivers/lightnvm/pblk-init.c ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns, pblk 450 drivers/lightnvm/pblk-init.c ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns, pblk 455 drivers/lightnvm/pblk-init.c ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns, pblk 460 drivers/lightnvm/pblk-init.c ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns, pblk 465 drivers/lightnvm/pblk-init.c pblk->close_wq = alloc_workqueue("pblk-close-wq", pblk 467 drivers/lightnvm/pblk-init.c if (!pblk->close_wq) pblk 470 drivers/lightnvm/pblk-init.c pblk->bb_wq = alloc_workqueue("pblk-bb-wq", pblk 472 drivers/lightnvm/pblk-init.c if (!pblk->bb_wq) pblk 475 drivers/lightnvm/pblk-init.c pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq", pblk 477 drivers/lightnvm/pblk-init.c if (!pblk->r_end_wq) pblk 480 drivers/lightnvm/pblk-init.c if (pblk_set_addrf(pblk)) pblk 483 drivers/lightnvm/pblk-init.c INIT_LIST_HEAD(&pblk->compl_list); pblk 484 drivers/lightnvm/pblk-init.c INIT_LIST_HEAD(&pblk->resubmit_list); pblk 489 drivers/lightnvm/pblk-init.c destroy_workqueue(pblk->r_end_wq); pblk 491 drivers/lightnvm/pblk-init.c destroy_workqueue(pblk->bb_wq); pblk 493 drivers/lightnvm/pblk-init.c destroy_workqueue(pblk->close_wq); pblk 495 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->w_rq_pool); pblk 497 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->e_rq_pool); pblk 499 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->r_rq_pool); pblk 501 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->rec_pool); pblk 503 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->gen_ws_pool); pblk 505 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->page_bio_pool); pblk 509 drivers/lightnvm/pblk-init.c kfree(pblk->pad_dist); pblk 513 drivers/lightnvm/pblk-init.c static void pblk_core_free(struct pblk *pblk) pblk 515 drivers/lightnvm/pblk-init.c if (pblk->close_wq) pblk 516 drivers/lightnvm/pblk-init.c destroy_workqueue(pblk->close_wq); pblk 518 drivers/lightnvm/pblk-init.c if (pblk->r_end_wq) pblk 519 drivers/lightnvm/pblk-init.c destroy_workqueue(pblk->r_end_wq); pblk 521 drivers/lightnvm/pblk-init.c if (pblk->bb_wq) pblk 522 drivers/lightnvm/pblk-init.c destroy_workqueue(pblk->bb_wq); pblk 524 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->page_bio_pool); pblk 525 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->gen_ws_pool); pblk 526 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->rec_pool); pblk 527 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->r_rq_pool); pblk 528 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->e_rq_pool); pblk 529 drivers/lightnvm/pblk-init.c mempool_exit(&pblk->w_rq_pool); pblk 532 drivers/lightnvm/pblk-init.c kfree(pblk->pad_dist); pblk 535 drivers/lightnvm/pblk-init.c static void pblk_line_mg_free(struct pblk *pblk) pblk 537 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 567 drivers/lightnvm/pblk-init.c static void pblk_lines_free(struct pblk *pblk) pblk 569 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 574 drivers/lightnvm/pblk-init.c line = &pblk->lines[i]; pblk 580 drivers/lightnvm/pblk-init.c pblk_line_mg_free(pblk); pblk 582 drivers/lightnvm/pblk-init.c kfree(pblk->luns); pblk 583 drivers/lightnvm/pblk-init.c kfree(pblk->lines); pblk 586 drivers/lightnvm/pblk-init.c static int pblk_luns_init(struct pblk *pblk) pblk 588 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 595 drivers/lightnvm/pblk-init.c pblk_err(pblk, "unbalanced LUN config.\n"); pblk 599 drivers/lightnvm/pblk-init.c pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun), pblk 601 drivers/lightnvm/pblk-init.c if (!pblk->luns) pblk 610 drivers/lightnvm/pblk-init.c rlun = &pblk->luns[i]; pblk 620 drivers/lightnvm/pblk-init.c static unsigned int calc_emeta_len(struct pblk *pblk) pblk 622 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 623 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 624 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 648 drivers/lightnvm/pblk-init.c static int pblk_set_provision(struct pblk *pblk, int nr_free_chks) pblk 650 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 651 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 652 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 659 drivers/lightnvm/pblk-init.c pblk->op = PBLK_DEFAULT_OP; pblk 661 drivers/lightnvm/pblk-init.c pblk->op = geo->op; pblk 663 drivers/lightnvm/pblk-init.c minimum = pblk_get_min_chks(pblk); pblk 665 drivers/lightnvm/pblk-init.c provisioned *= (100 - pblk->op); pblk 670 drivers/lightnvm/pblk-init.c pblk_err(pblk, "OP too small to create a sane instance\n"); pblk 679 drivers/lightnvm/pblk-init.c pblk->op = (100 * minimum) / nr_free_chks; pblk 680 drivers/lightnvm/pblk-init.c pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n", pblk 681 drivers/lightnvm/pblk-init.c pblk->op); pblk 684 drivers/lightnvm/pblk-init.c pblk->op_blks = nr_free_chks - provisioned; pblk 689 drivers/lightnvm/pblk-init.c pblk->rl.total_blocks = nr_free_chks; pblk 695 drivers/lightnvm/pblk-init.c clba = (geo->clba / pblk->min_write_pgs) * pblk->min_write_pgs_data; pblk 696 drivers/lightnvm/pblk-init.c pblk->capacity = (provisioned - blk_meta) * clba; pblk 698 drivers/lightnvm/pblk-init.c atomic_set(&pblk->rl.free_blocks, nr_free_chks); pblk 699 drivers/lightnvm/pblk-init.c atomic_set(&pblk->rl.free_user_blocks, nr_free_chks); pblk 704 drivers/lightnvm/pblk-init.c static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line, pblk 707 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 709 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 713 drivers/lightnvm/pblk-init.c struct pblk_lun *rlun = &pblk->luns[i]; pblk 724 drivers/lightnvm/pblk-init.c chunk_meta = pblk_chunk_get_off(pblk, meta, ppa); pblk 733 drivers/lightnvm/pblk-init.c trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa, pblk 751 drivers/lightnvm/pblk-init.c static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line, pblk 754 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 755 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 758 drivers/lightnvm/pblk-init.c line->pblk = pblk; pblk 766 drivers/lightnvm/pblk-init.c nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta); pblk 783 drivers/lightnvm/pblk-init.c static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line) pblk 785 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 816 drivers/lightnvm/pblk-init.c static int pblk_line_mg_init(struct pblk *pblk) pblk 818 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 820 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 821 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 933 drivers/lightnvm/pblk-init.c static int pblk_line_meta_init(struct pblk *pblk) pblk 935 drivers/lightnvm/pblk-init.c struct nvm_tgt_dev *dev = pblk->dev; pblk 937 drivers/lightnvm/pblk-init.c struct pblk_line_meta *lm = &pblk->lm; pblk 948 drivers/lightnvm/pblk-init.c lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs; pblk 972 drivers/lightnvm/pblk-init.c emeta_len = calc_emeta_len(pblk); pblk 986 drivers/lightnvm/pblk-init.c pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n", pblk 994 drivers/lightnvm/pblk-init.c static int pblk_lines_init(struct pblk *pblk) pblk 996 drivers/lightnvm/pblk-init.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 1002 drivers/lightnvm/pblk-init.c ret = pblk_line_meta_init(pblk); pblk 1006 drivers/lightnvm/pblk-init.c ret = pblk_line_mg_init(pblk); pblk 1010 drivers/lightnvm/pblk-init.c ret = pblk_luns_init(pblk); pblk 1014 drivers/lightnvm/pblk-init.c chunk_meta = pblk_get_chunk_meta(pblk); pblk 1020 drivers/lightnvm/pblk-init.c pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line), pblk 1022 drivers/lightnvm/pblk-init.c if (!pblk->lines) { pblk 1028 drivers/lightnvm/pblk-init.c line = &pblk->lines[i]; pblk 1030 drivers/lightnvm/pblk-init.c ret = pblk_alloc_line_meta(pblk, line); pblk 1034 drivers/lightnvm/pblk-init.c nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i); pblk 1036 drivers/lightnvm/pblk-init.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 1041 drivers/lightnvm/pblk-init.c pblk_err(pblk, "too many bad blocks prevent for sane instance\n"); pblk 1046 drivers/lightnvm/pblk-init.c ret = pblk_set_provision(pblk, nr_free_chks); pblk 1055 drivers/lightnvm/pblk-init.c pblk_line_meta_free(l_mg, &pblk->lines[i]); pblk 1056 drivers/lightnvm/pblk-init.c kfree(pblk->lines); pblk 1060 drivers/lightnvm/pblk-init.c kfree(pblk->luns); pblk 1062 drivers/lightnvm/pblk-init.c pblk_line_mg_free(pblk); pblk 1067 drivers/lightnvm/pblk-init.c static int pblk_writer_init(struct pblk *pblk) pblk 1069 drivers/lightnvm/pblk-init.c pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t"); pblk 1070 drivers/lightnvm/pblk-init.c if (IS_ERR(pblk->writer_ts)) { pblk 1071 drivers/lightnvm/pblk-init.c int err = PTR_ERR(pblk->writer_ts); pblk 1074 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not allocate writer kthread (%d)\n", pblk 1079 drivers/lightnvm/pblk-init.c timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0); pblk 1080 drivers/lightnvm/pblk-init.c mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100)); pblk 1085 drivers/lightnvm/pblk-init.c static void pblk_writer_stop(struct pblk *pblk) pblk 1090 drivers/lightnvm/pblk-init.c WARN(pblk_rb_read_count(&pblk->rwb), pblk 1093 drivers/lightnvm/pblk-init.c WARN(pblk_rb_sync_count(&pblk->rwb), pblk 1096 drivers/lightnvm/pblk-init.c del_timer_sync(&pblk->wtimer); pblk 1097 drivers/lightnvm/pblk-init.c if (pblk->writer_ts) pblk 1098 drivers/lightnvm/pblk-init.c kthread_stop(pblk->writer_ts); pblk 1101 drivers/lightnvm/pblk-init.c static void pblk_free(struct pblk *pblk) pblk 1103 drivers/lightnvm/pblk-init.c pblk_lines_free(pblk); pblk 1104 drivers/lightnvm/pblk-init.c pblk_l2p_free(pblk); pblk 1105 drivers/lightnvm/pblk-init.c pblk_rwb_free(pblk); pblk 1106 drivers/lightnvm/pblk-init.c pblk_core_free(pblk); pblk 1108 drivers/lightnvm/pblk-init.c kfree(pblk); pblk 1111 drivers/lightnvm/pblk-init.c static void pblk_tear_down(struct pblk *pblk, bool graceful) pblk 1114 drivers/lightnvm/pblk-init.c __pblk_pipeline_flush(pblk); pblk 1115 drivers/lightnvm/pblk-init.c __pblk_pipeline_stop(pblk); pblk 1116 drivers/lightnvm/pblk-init.c pblk_writer_stop(pblk); pblk 1117 drivers/lightnvm/pblk-init.c pblk_rb_sync_l2p(&pblk->rwb); pblk 1118 drivers/lightnvm/pblk-init.c pblk_rl_free(&pblk->rl); pblk 1120 drivers/lightnvm/pblk-init.c pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful); pblk 1125 drivers/lightnvm/pblk-init.c struct pblk *pblk = private; pblk 1127 drivers/lightnvm/pblk-init.c pblk_gc_exit(pblk, graceful); pblk 1128 drivers/lightnvm/pblk-init.c pblk_tear_down(pblk, graceful); pblk 1131 drivers/lightnvm/pblk-init.c pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk)); pblk 1134 drivers/lightnvm/pblk-init.c pblk_free(pblk); pblk 1139 drivers/lightnvm/pblk-init.c struct pblk *pblk = private; pblk 1141 drivers/lightnvm/pblk-init.c return pblk->capacity * NR_PHY_IN_LOG; pblk 1150 drivers/lightnvm/pblk-init.c struct pblk *pblk; pblk 1153 drivers/lightnvm/pblk-init.c pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL); pblk 1154 drivers/lightnvm/pblk-init.c if (!pblk) pblk 1157 drivers/lightnvm/pblk-init.c pblk->dev = dev; pblk 1158 drivers/lightnvm/pblk-init.c pblk->disk = tdisk; pblk 1159 drivers/lightnvm/pblk-init.c pblk->state = PBLK_STATE_RUNNING; pblk 1160 drivers/lightnvm/pblk-init.c trace_pblk_state(pblk_disk_name(pblk), pblk->state); pblk 1161 drivers/lightnvm/pblk-init.c pblk->gc.gc_enabled = 0; pblk 1165 drivers/lightnvm/pblk-init.c pblk_err(pblk, "OCSSD version not supported (%u)\n", pblk 1167 drivers/lightnvm/pblk-init.c kfree(pblk); pblk 1172 drivers/lightnvm/pblk-init.c pblk_err(pblk, "extended metadata not supported\n"); pblk 1173 drivers/lightnvm/pblk-init.c kfree(pblk); pblk 1177 drivers/lightnvm/pblk-init.c spin_lock_init(&pblk->resubmit_lock); pblk 1178 drivers/lightnvm/pblk-init.c spin_lock_init(&pblk->trans_lock); pblk 1179 drivers/lightnvm/pblk-init.c spin_lock_init(&pblk->lock); pblk 1182 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->inflight_writes, 0); pblk 1183 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->padded_writes, 0); pblk 1184 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->padded_wb, 0); pblk 1185 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->req_writes, 0); pblk 1186 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->sub_writes, 0); pblk 1187 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->sync_writes, 0); pblk 1188 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->inflight_reads, 0); pblk 1189 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->cache_reads, 0); pblk 1190 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->sync_reads, 0); pblk 1191 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->recov_writes, 0); pblk 1192 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->recov_writes, 0); pblk 1193 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->recov_gc_writes, 0); pblk 1194 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->recov_gc_reads, 0); pblk 1197 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->read_failed, 0); pblk 1198 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->read_empty, 0); pblk 1199 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->read_high_ecc, 0); pblk 1200 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->read_failed_gc, 0); pblk 1201 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->write_failed, 0); pblk 1202 drivers/lightnvm/pblk-init.c atomic_long_set(&pblk->erase_failed, 0); pblk 1204 drivers/lightnvm/pblk-init.c ret = pblk_core_init(pblk); pblk 1206 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not initialize core\n"); pblk 1210 drivers/lightnvm/pblk-init.c ret = pblk_lines_init(pblk); pblk 1212 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not initialize lines\n"); pblk 1216 drivers/lightnvm/pblk-init.c ret = pblk_rwb_init(pblk); pblk 1218 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not initialize write buffer\n"); pblk 1222 drivers/lightnvm/pblk-init.c ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY); pblk 1224 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not initialize maps\n"); pblk 1228 drivers/lightnvm/pblk-init.c ret = pblk_writer_init(pblk); pblk 1231 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not initialize write thread\n"); pblk 1235 drivers/lightnvm/pblk-init.c ret = pblk_gc_init(pblk); pblk 1237 drivers/lightnvm/pblk-init.c pblk_err(pblk, "could not initialize gc\n"); pblk 1252 drivers/lightnvm/pblk-init.c pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n", pblk 1253 drivers/lightnvm/pblk-init.c geo->all_luns, pblk->l_mg.nr_lines, pblk 1254 drivers/lightnvm/pblk-init.c (unsigned long long)pblk->capacity, pblk 1255 drivers/lightnvm/pblk-init.c pblk->rwb.nr_entries); pblk 1257 drivers/lightnvm/pblk-init.c wake_up_process(pblk->writer_ts); pblk 1260 drivers/lightnvm/pblk-init.c pblk_gc_should_kick(pblk); pblk 1262 drivers/lightnvm/pblk-init.c return pblk; pblk 1265 drivers/lightnvm/pblk-init.c pblk_writer_stop(pblk); pblk 1267 drivers/lightnvm/pblk-init.c pblk_l2p_free(pblk); pblk 1269 drivers/lightnvm/pblk-init.c pblk_rwb_free(pblk); pblk 1271 drivers/lightnvm/pblk-init.c pblk_lines_free(pblk); pblk 1273 drivers/lightnvm/pblk-init.c pblk_core_free(pblk); pblk 1275 drivers/lightnvm/pblk-init.c kfree(pblk); pblk 22 drivers/lightnvm/pblk-map.c static int pblk_map_page_data(struct pblk *pblk, unsigned int sentry, pblk 28 drivers/lightnvm/pblk-map.c struct pblk_line *line = pblk_line_get_data(pblk); pblk 33 drivers/lightnvm/pblk-map.c int nr_secs = pblk->min_write_pgs; pblk 45 drivers/lightnvm/pblk-map.c line = pblk_line_replace_data(pblk); pblk 46 drivers/lightnvm/pblk-map.c pblk_line_close_meta(pblk, prev_line); pblk 49 drivers/lightnvm/pblk-map.c pblk_pipeline_stop(pblk); pblk 56 drivers/lightnvm/pblk-map.c lba_list = emeta_to_lbas(pblk, emeta->buf); pblk 58 drivers/lightnvm/pblk-map.c paddr = pblk_alloc_page(pblk, line, nr_secs); pblk 61 drivers/lightnvm/pblk-map.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); pblk 65 drivers/lightnvm/pblk-map.c ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); pblk 77 drivers/lightnvm/pblk-map.c w_ctx = pblk_rb_w_ctx(&pblk->rwb, sentry + i); pblk 84 drivers/lightnvm/pblk-map.c atomic64_inc(&pblk->pad_wa); pblk 88 drivers/lightnvm/pblk-map.c __pblk_map_invalidate(pblk, line, paddr); pblk 92 drivers/lightnvm/pblk-map.c pblk_down_rq(pblk, ppa_list[0], lun_bitmap); pblk 96 drivers/lightnvm/pblk-map.c int pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry, pblk 100 drivers/lightnvm/pblk-map.c void *meta_list = pblk_get_meta_for_writes(pblk, rqd); pblk 104 drivers/lightnvm/pblk-map.c int min = pblk->min_write_pgs; pblk 110 drivers/lightnvm/pblk-map.c meta_buffer = pblk_get_meta(pblk, meta_list, i); pblk 112 drivers/lightnvm/pblk-map.c ret = pblk_map_page_data(pblk, sentry + i, &ppa_list[i], pblk 122 drivers/lightnvm/pblk-map.c int pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 126 drivers/lightnvm/pblk-map.c struct nvm_tgt_dev *dev = pblk->dev; pblk 128 drivers/lightnvm/pblk-map.c struct pblk_line_meta *lm = &pblk->lm; pblk 129 drivers/lightnvm/pblk-map.c void *meta_list = pblk_get_meta_for_writes(pblk, rqd); pblk 134 drivers/lightnvm/pblk-map.c int min = pblk->min_write_pgs; pblk 141 drivers/lightnvm/pblk-map.c meta_buffer = pblk_get_meta(pblk, meta_list, i); pblk 143 drivers/lightnvm/pblk-map.c ret = pblk_map_page_data(pblk, sentry + i, &ppa_list[i], pblk 153 drivers/lightnvm/pblk-map.c e_line = pblk_line_get_erase(pblk); pblk 155 drivers/lightnvm/pblk-map.c return pblk_map_rq(pblk, rqd, sentry, lun_bitmap, pblk 170 drivers/lightnvm/pblk-map.c return pblk_map_rq(pblk, rqd, sentry, lun_bitmap, pblk 176 drivers/lightnvm/pblk-map.c d_line = pblk_line_get_data(pblk); pblk 181 drivers/lightnvm/pblk-map.c e_line = pblk_line_get_erase(pblk); pblk 205 drivers/lightnvm/pblk-map.c *erase_ppa = pblk->luns[bit].bppa; /* set ch and lun */ pblk 75 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 165 drivers/lightnvm/pblk-rb.c pblk_rl_init(&pblk->rl, rb->nr_entries, threshold); pblk 239 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 259 drivers/lightnvm/pblk-rb.c pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa, pblk 262 drivers/lightnvm/pblk-rb.c line = pblk_ppa_to_line(pblk, w_ctx->ppa); pblk 269 drivers/lightnvm/pblk-rb.c pblk_rl_out(&pblk->rl, user_io, gc_io); pblk 340 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 353 drivers/lightnvm/pblk-rb.c pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline); pblk 364 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 377 drivers/lightnvm/pblk-rb.c if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr)) pblk 454 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 460 drivers/lightnvm/pblk-rb.c pblk_write_kick(pblk); pblk 476 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 478 drivers/lightnvm/pblk-rb.c atomic64_inc(&pblk->nr_flush); pblk 479 drivers/lightnvm/pblk-rb.c if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem)) pblk 497 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 501 drivers/lightnvm/pblk-rb.c io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries); pblk 512 drivers/lightnvm/pblk-rb.c pblk_rl_user_in(&pblk->rl, nr_entries); pblk 524 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 527 drivers/lightnvm/pblk-rb.c if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) { pblk 537 drivers/lightnvm/pblk-rb.c pblk_rl_gc_in(&pblk->rl, nr_entries); pblk 554 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 555 drivers/lightnvm/pblk-rb.c struct request_queue *q = pblk->dev->q; pblk 570 drivers/lightnvm/pblk-rb.c pad += (pblk->min_write_pgs - pblk->min_write_pgs_data); pblk 591 drivers/lightnvm/pblk-rb.c pblk_err(pblk, "could not allocate write bio page\n"); pblk 601 drivers/lightnvm/pblk-rb.c pblk_err(pblk, "could not add page to write bio\n"); pblk 619 drivers/lightnvm/pblk-rb.c if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) { pblk 620 drivers/lightnvm/pblk-rb.c pblk_err(pblk, "could not pad page in write bio\n"); pblk 624 drivers/lightnvm/pblk-rb.c if (pad < pblk->min_write_pgs) pblk 625 drivers/lightnvm/pblk-rb.c atomic64_inc(&pblk->pad_dist[pad - 1]); pblk 627 drivers/lightnvm/pblk-rb.c pblk_warn(pblk, "padding more than min. sectors\n"); pblk 629 drivers/lightnvm/pblk-rb.c atomic64_add(pad, &pblk->pad_wa); pblk 633 drivers/lightnvm/pblk-rb.c atomic_long_add(pad, &pblk->padded_writes); pblk 647 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 666 drivers/lightnvm/pblk-rb.c spin_lock(&pblk->trans_lock); pblk 667 drivers/lightnvm/pblk-rb.c l2p_ppa = pblk_trans_map_get(pblk, lba); pblk 668 drivers/lightnvm/pblk-rb.c spin_unlock(&pblk->trans_lock); pblk 811 drivers/lightnvm/pblk-rb.c struct pblk *pblk = container_of(rb, struct pblk, rwb); pblk 817 drivers/lightnvm/pblk-rb.c list_for_each_entry(c, &pblk->compl_list, list) pblk 28 drivers/lightnvm/pblk-read.c static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio, pblk 37 drivers/lightnvm/pblk-read.c return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa); pblk 40 drivers/lightnvm/pblk-read.c static int pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 48 drivers/lightnvm/pblk-read.c nr_secs = pblk_lookup_l2p_seq(pblk, rqd->ppa_list, blba, rqd->nr_ppas, pblk 55 drivers/lightnvm/pblk-read.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); pblk 68 drivers/lightnvm/pblk-read.c if (!pblk_read_from_cache(pblk, bio, lba, pblk 91 drivers/lightnvm/pblk-read.c atomic_long_inc(&pblk->cache_reads); pblk 98 drivers/lightnvm/pblk-read.c if (pblk_io_aligned(pblk, nr_secs)) pblk 102 drivers/lightnvm/pblk-read.c atomic_long_add(nr_secs, &pblk->inflight_reads); pblk 109 drivers/lightnvm/pblk-read.c static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd, pblk 116 drivers/lightnvm/pblk-read.c if (!pblk_is_oob_meta_supported(pblk)) pblk 120 drivers/lightnvm/pblk-read.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); pblk 130 drivers/lightnvm/pblk-read.c print_ppa(pblk, &ppa_list[i], "seq", i); pblk 132 drivers/lightnvm/pblk-read.c pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", pblk 142 drivers/lightnvm/pblk-read.c static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, pblk 148 drivers/lightnvm/pblk-read.c if (!pblk_is_oob_meta_supported(pblk)) pblk 152 drivers/lightnvm/pblk-read.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, pblk 166 drivers/lightnvm/pblk-read.c print_ppa(pblk, &ppa_list[j], "rnd", j); pblk 168 drivers/lightnvm/pblk-read.c pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", pblk 187 drivers/lightnvm/pblk-read.c static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd, pblk 190 drivers/lightnvm/pblk-read.c struct nvm_tgt_dev *dev = pblk->dev; pblk 195 drivers/lightnvm/pblk-read.c generic_end_io_acct(dev->q, REQ_OP_READ, &pblk->disk->part0, start_time); pblk 198 drivers/lightnvm/pblk-read.c pblk_log_read_err(pblk, rqd); pblk 200 drivers/lightnvm/pblk-read.c pblk_read_check_seq(pblk, rqd, r_ctx->lba); pblk 204 drivers/lightnvm/pblk-read.c pblk_rq_to_line_put(pblk, rqd); pblk 207 drivers/lightnvm/pblk-read.c atomic_long_add(rqd->nr_ppas, &pblk->sync_reads); pblk 208 drivers/lightnvm/pblk-read.c atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads); pblk 211 drivers/lightnvm/pblk-read.c pblk_free_rqd(pblk, rqd, PBLK_READ); pblk 212 drivers/lightnvm/pblk-read.c atomic_dec(&pblk->inflight_io); pblk 217 drivers/lightnvm/pblk-read.c struct pblk *pblk = rqd->private; pblk 222 drivers/lightnvm/pblk-read.c __pblk_end_io_read(pblk, rqd, true); pblk 225 drivers/lightnvm/pblk-read.c static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio, pblk 228 drivers/lightnvm/pblk-read.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0); pblk 231 drivers/lightnvm/pblk-read.c pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache); pblk 234 drivers/lightnvm/pblk-read.c atomic_long_inc(&pblk->inflight_reads); pblk 249 drivers/lightnvm/pblk-read.c if (!pblk_read_from_cache(pblk, bio, lba, ppa)) { pblk 250 drivers/lightnvm/pblk-read.c pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache); pblk 257 drivers/lightnvm/pblk-read.c atomic_long_inc(&pblk->cache_reads); pblk 264 drivers/lightnvm/pblk-read.c void pblk_submit_read(struct pblk *pblk, struct bio *bio) pblk 266 drivers/lightnvm/pblk-read.c struct nvm_tgt_dev *dev = pblk->dev; pblk 276 drivers/lightnvm/pblk-read.c &pblk->disk->part0); pblk 278 drivers/lightnvm/pblk-read.c rqd = pblk_alloc_rqd(pblk, PBLK_READ); pblk 282 drivers/lightnvm/pblk-read.c rqd->private = pblk; pblk 289 drivers/lightnvm/pblk-read.c if (pblk_alloc_rqd_meta(pblk, rqd)) { pblk 291 drivers/lightnvm/pblk-read.c pblk_free_rqd(pblk, rqd, PBLK_READ); pblk 302 drivers/lightnvm/pblk-read.c nr_secs = pblk_read_ppalist_rq(pblk, rqd, int_bio, blba, pblk 305 drivers/lightnvm/pblk-read.c pblk_read_rq(pblk, rqd, int_bio, blba, &from_cache); pblk 314 drivers/lightnvm/pblk-read.c atomic_inc(&pblk->inflight_io); pblk 315 drivers/lightnvm/pblk-read.c __pblk_end_io_read(pblk, rqd, false); pblk 345 drivers/lightnvm/pblk-read.c } else if (pblk_submit_io(pblk, rqd, NULL)) { pblk 352 drivers/lightnvm/pblk-read.c static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, pblk 361 drivers/lightnvm/pblk-read.c pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs); pblk 367 drivers/lightnvm/pblk-read.c ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id); pblk 377 drivers/lightnvm/pblk-read.c atomic_long_add(valid_secs, &pblk->inflight_reads); pblk 383 drivers/lightnvm/pblk-read.c static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, pblk 394 drivers/lightnvm/pblk-read.c if (lba >= pblk->capacity) { pblk 399 drivers/lightnvm/pblk-read.c spin_lock(&pblk->trans_lock); pblk 400 drivers/lightnvm/pblk-read.c ppa_l2p = pblk_trans_map_get(pblk, lba); pblk 401 drivers/lightnvm/pblk-read.c spin_unlock(&pblk->trans_lock); pblk 403 drivers/lightnvm/pblk-read.c ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id); pblk 411 drivers/lightnvm/pblk-read.c atomic_long_inc(&pblk->inflight_reads); pblk 418 drivers/lightnvm/pblk-read.c int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) pblk 425 drivers/lightnvm/pblk-read.c ret = pblk_alloc_rqd_meta(pblk, &rqd); pblk 430 drivers/lightnvm/pblk-read.c gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line, pblk 437 drivers/lightnvm/pblk-read.c gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line, pblk 448 drivers/lightnvm/pblk-read.c if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) { pblk 453 drivers/lightnvm/pblk-read.c pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs); pblk 455 drivers/lightnvm/pblk-read.c atomic_dec(&pblk->inflight_io); pblk 458 drivers/lightnvm/pblk-read.c atomic_long_inc(&pblk->read_failed_gc); pblk 460 drivers/lightnvm/pblk-read.c pblk_print_failed_rqd(pblk, &rqd, rqd.error); pblk 465 drivers/lightnvm/pblk-read.c atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads); pblk 466 drivers/lightnvm/pblk-read.c atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads); pblk 467 drivers/lightnvm/pblk-read.c atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads); pblk 471 drivers/lightnvm/pblk-read.c pblk_free_rqd_meta(pblk, &rqd); pblk 475 drivers/lightnvm/pblk-read.c pblk_free_rqd_meta(pblk, &rqd); pblk 24 drivers/lightnvm/pblk-recovery.c int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta_buf) pblk 28 drivers/lightnvm/pblk-recovery.c crc = pblk_calc_emeta_crc(pblk, emeta_buf); pblk 38 drivers/lightnvm/pblk-recovery.c static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line) pblk 40 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 42 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 50 drivers/lightnvm/pblk-recovery.c lba_list = emeta_to_lbas(pblk, emeta_buf); pblk 54 drivers/lightnvm/pblk-recovery.c data_start = pblk_line_smeta_start(pblk, line) + lm->smeta_sec; pblk 62 drivers/lightnvm/pblk-recovery.c ppa = addr_to_gen_ppa(pblk, i, line->id); pblk 80 drivers/lightnvm/pblk-recovery.c pblk_update_map(pblk, le64_to_cpu(lba_list[i]), ppa); pblk 85 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "line %d - inconsistent lba list(%llu/%llu)\n", pblk 93 drivers/lightnvm/pblk-recovery.c static void pblk_update_line_wp(struct pblk *pblk, struct pblk_line *line, pblk 96 drivers/lightnvm/pblk-recovery.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 99 drivers/lightnvm/pblk-recovery.c for (i = 0; i < written_secs; i += pblk->min_write_pgs) pblk 100 drivers/lightnvm/pblk-recovery.c __pblk_alloc_page(pblk, line, pblk->min_write_pgs); pblk 116 drivers/lightnvm/pblk-recovery.c static u64 pblk_sec_in_open_line(struct pblk *pblk, struct pblk_line *line) pblk 118 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 135 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "recovery line %d is bad\n", line->id); pblk 137 drivers/lightnvm/pblk-recovery.c pblk_update_line_wp(pblk, line, written_secs - lm->smeta_sec); pblk 162 drivers/lightnvm/pblk-recovery.c struct pblk *pblk = pad_rq->pblk; pblk 164 drivers/lightnvm/pblk-recovery.c pblk_up_chunk(pblk, ppa_list[0]); pblk 166 drivers/lightnvm/pblk-recovery.c pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); pblk 168 drivers/lightnvm/pblk-recovery.c atomic_dec(&pblk->inflight_io); pblk 173 drivers/lightnvm/pblk-recovery.c static int pblk_recov_pad_line(struct pblk *pblk, struct pblk_line *line, pblk 176 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 183 drivers/lightnvm/pblk-recovery.c __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); pblk 197 drivers/lightnvm/pblk-recovery.c data = vzalloc(array_size(pblk->max_write_pgs, geo->csecs)); pblk 203 drivers/lightnvm/pblk-recovery.c pad_rq->pblk = pblk; pblk 208 drivers/lightnvm/pblk-recovery.c rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); pblk 209 drivers/lightnvm/pblk-recovery.c if (rq_ppas < pblk->min_write_pgs) { pblk 210 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "corrupted pad line %d\n", line->id); pblk 214 drivers/lightnvm/pblk-recovery.c rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT); pblk 216 drivers/lightnvm/pblk-recovery.c ret = pblk_alloc_rqd_meta(pblk, rqd); pblk 218 drivers/lightnvm/pblk-recovery.c pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); pblk 236 drivers/lightnvm/pblk-recovery.c w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs); pblk 237 drivers/lightnvm/pblk-recovery.c ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); pblk 241 drivers/lightnvm/pblk-recovery.c w_ptr += pblk->min_write_pgs; pblk 242 drivers/lightnvm/pblk-recovery.c ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); pblk 246 drivers/lightnvm/pblk-recovery.c for (j = 0; j < pblk->min_write_pgs; j++, i++, w_ptr++) { pblk 251 drivers/lightnvm/pblk-recovery.c dev_ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); pblk 253 drivers/lightnvm/pblk-recovery.c pblk_map_invalidate(pblk, dev_ppa); pblk 255 drivers/lightnvm/pblk-recovery.c meta = pblk_get_meta(pblk, meta_list, i); pblk 262 drivers/lightnvm/pblk-recovery.c pblk_down_chunk(pblk, ppa_list[0]); pblk 264 drivers/lightnvm/pblk-recovery.c ret = pblk_submit_io(pblk, rqd, data); pblk 266 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "I/O submission failed: %d\n", ret); pblk 267 drivers/lightnvm/pblk-recovery.c pblk_up_chunk(pblk, ppa_list[0]); pblk 269 drivers/lightnvm/pblk-recovery.c pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); pblk 283 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "corrupted padded line: %d\n", line->id); pblk 291 drivers/lightnvm/pblk-recovery.c static int pblk_pad_distance(struct pblk *pblk, struct pblk_line *line) pblk 293 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 301 drivers/lightnvm/pblk-recovery.c static struct nvm_chk_meta *pblk_get_stripe_chunk(struct pblk *pblk, pblk 305 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 311 drivers/lightnvm/pblk-recovery.c rlun = &pblk->luns[index]; pblk 318 drivers/lightnvm/pblk-recovery.c static int pblk_line_wps_are_unbalanced(struct pblk *pblk, pblk 321 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 335 drivers/lightnvm/pblk-recovery.c chunk = pblk_get_stripe_chunk(pblk, line, i); pblk 337 drivers/lightnvm/pblk-recovery.c if (max_wp > pblk->max_write_pgs) pblk 338 drivers/lightnvm/pblk-recovery.c min_wp = max_wp - pblk->max_write_pgs; pblk 344 drivers/lightnvm/pblk-recovery.c chunk = pblk_get_stripe_chunk(pblk, line, i); pblk 354 drivers/lightnvm/pblk-recovery.c static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line, pblk 357 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 358 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 366 drivers/lightnvm/pblk-recovery.c u64 paddr = pblk_line_smeta_start(pblk, line) + lm->smeta_sec; pblk 371 drivers/lightnvm/pblk-recovery.c u64 left_ppas = pblk_sec_in_open_line(pblk, line) - lm->smeta_sec; pblk 373 drivers/lightnvm/pblk-recovery.c if (pblk_line_wps_are_unbalanced(pblk, line)) pblk 374 drivers/lightnvm/pblk-recovery.c pblk_warn(pblk, "recovering unbalanced line (%d)\n", line->id); pblk 383 drivers/lightnvm/pblk-recovery.c lba_list = emeta_to_lbas(pblk, line->emeta->buf); pblk 388 drivers/lightnvm/pblk-recovery.c rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); pblk 390 drivers/lightnvm/pblk-recovery.c rq_ppas = pblk->min_write_pgs; pblk 402 drivers/lightnvm/pblk-recovery.c if (pblk_io_aligned(pblk, rq_ppas)) pblk 409 drivers/lightnvm/pblk-recovery.c ppa = addr_to_gen_ppa(pblk, paddr, line->id); pblk 413 drivers/lightnvm/pblk-recovery.c paddr += pblk->min_write_pgs; pblk 414 drivers/lightnvm/pblk-recovery.c ppa = addr_to_gen_ppa(pblk, paddr, line->id); pblk 418 drivers/lightnvm/pblk-recovery.c for (j = 0; j < pblk->min_write_pgs; j++, i++) pblk 420 drivers/lightnvm/pblk-recovery.c addr_to_gen_ppa(pblk, paddr + j, line->id); pblk 423 drivers/lightnvm/pblk-recovery.c ret = pblk_submit_io_sync(pblk, rqd, data); pblk 425 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "I/O submission failed: %d\n", ret); pblk 429 drivers/lightnvm/pblk-recovery.c atomic_dec(&pblk->inflight_io); pblk 436 drivers/lightnvm/pblk-recovery.c pblk_log_read_err(pblk, rqd); pblk 440 drivers/lightnvm/pblk-recovery.c pad_distance = pblk_pad_distance(pblk, line); pblk 441 drivers/lightnvm/pblk-recovery.c ret = pblk_recov_pad_line(pblk, line, pad_distance); pblk 450 drivers/lightnvm/pblk-recovery.c pblk_get_packed_meta(pblk, rqd); pblk 453 drivers/lightnvm/pblk-recovery.c struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); pblk 458 drivers/lightnvm/pblk-recovery.c if (lba == ADDR_EMPTY || lba >= pblk->capacity) pblk 462 drivers/lightnvm/pblk-recovery.c pblk_update_map(pblk, lba, ppa_list[i]); pblk 477 drivers/lightnvm/pblk-recovery.c static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line) pblk 479 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 493 drivers/lightnvm/pblk-recovery.c ppa_list = (void *)(meta_list) + pblk_dma_meta_size(pblk); pblk 494 drivers/lightnvm/pblk-recovery.c dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk); pblk 496 drivers/lightnvm/pblk-recovery.c data = kcalloc(pblk->max_write_pgs, geo->csecs, GFP_KERNEL); pblk 502 drivers/lightnvm/pblk-recovery.c rqd = mempool_alloc(&pblk->r_rq_pool, GFP_KERNEL); pblk 512 drivers/lightnvm/pblk-recovery.c ret = pblk_recov_scan_oob(pblk, line, p); pblk 514 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "could not recover L2P form OOB\n"); pblk 519 drivers/lightnvm/pblk-recovery.c pblk_line_recov_close(pblk, line); pblk 522 drivers/lightnvm/pblk-recovery.c mempool_free(rqd, &pblk->r_rq_pool); pblk 543 drivers/lightnvm/pblk-recovery.c static u64 pblk_line_emeta_start(struct pblk *pblk, struct pblk_line *line) pblk 545 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 547 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 558 drivers/lightnvm/pblk-recovery.c ppa = addr_to_gen_ppa(pblk, emeta_start, line->id); pblk 567 drivers/lightnvm/pblk-recovery.c static int pblk_recov_check_line_version(struct pblk *pblk, pblk 573 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "line major version mismatch: %d, expected: %d\n", pblk 580 drivers/lightnvm/pblk-recovery.c pblk_info(pblk, "newer line minor version found: %d\n", pblk 587 drivers/lightnvm/pblk-recovery.c static void pblk_recov_wa_counters(struct pblk *pblk, pblk 590 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 600 drivers/lightnvm/pblk-recovery.c atomic64_set(&pblk->user_wa, user); pblk 601 drivers/lightnvm/pblk-recovery.c atomic64_set(&pblk->pad_wa, pad); pblk 602 drivers/lightnvm/pblk-recovery.c atomic64_set(&pblk->gc_wa, gc); pblk 604 drivers/lightnvm/pblk-recovery.c pblk->user_rst_wa = user; pblk 605 drivers/lightnvm/pblk-recovery.c pblk->pad_rst_wa = pad; pblk 606 drivers/lightnvm/pblk-recovery.c pblk->gc_rst_wa = gc; pblk 611 drivers/lightnvm/pblk-recovery.c struct pblk *pblk) pblk 614 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 615 drivers/lightnvm/pblk-recovery.c struct nvm_tgt_dev *dev = pblk->dev; pblk 628 drivers/lightnvm/pblk-recovery.c bppa = pblk->luns[smeta_blk].bppa; pblk 639 drivers/lightnvm/pblk-recovery.c static bool pblk_line_is_open(struct pblk *pblk, struct pblk_line *line) pblk 641 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 651 drivers/lightnvm/pblk-recovery.c struct pblk_line *pblk_recov_l2p(struct pblk *pblk) pblk 653 drivers/lightnvm/pblk-recovery.c struct pblk_line_meta *lm = &pblk->lm; pblk 654 drivers/lightnvm/pblk-recovery.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 680 drivers/lightnvm/pblk-recovery.c line = &pblk->lines[i]; pblk 687 drivers/lightnvm/pblk-recovery.c if (!pblk_line_was_written(line, pblk)) pblk 691 drivers/lightnvm/pblk-recovery.c if (pblk_line_smeta_read(pblk, line)) pblk 694 drivers/lightnvm/pblk-recovery.c crc = pblk_calc_smeta_crc(pblk, smeta_buf); pblk 702 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "found incompatible line version %u\n", pblk 709 drivers/lightnvm/pblk-recovery.c guid_copy(&pblk->instance_uuid, pblk 714 drivers/lightnvm/pblk-recovery.c if (!guid_equal(&pblk->instance_uuid, pblk 716 drivers/lightnvm/pblk-recovery.c pblk_debug(pblk, "ignore line %u due to uuid mismatch\n", pblk 735 drivers/lightnvm/pblk-recovery.c if (pblk_line_recov_alloc(pblk, line)) pblk 740 drivers/lightnvm/pblk-recovery.c pblk_debug(pblk, "recovering data line %d, seq:%llu\n", pblk 745 drivers/lightnvm/pblk-recovery.c guid_gen(&pblk->instance_uuid); pblk 759 drivers/lightnvm/pblk-recovery.c line->emeta_ssec = pblk_line_emeta_start(pblk, line); pblk 763 drivers/lightnvm/pblk-recovery.c if (pblk_line_is_open(pblk, line)) { pblk 764 drivers/lightnvm/pblk-recovery.c pblk_recov_l2p_from_oob(pblk, line); pblk 768 drivers/lightnvm/pblk-recovery.c if (pblk_line_emeta_read(pblk, line, line->emeta->buf)) { pblk 769 drivers/lightnvm/pblk-recovery.c pblk_recov_l2p_from_oob(pblk, line); pblk 773 drivers/lightnvm/pblk-recovery.c if (pblk_recov_check_emeta(pblk, line->emeta->buf)) { pblk 774 drivers/lightnvm/pblk-recovery.c pblk_recov_l2p_from_oob(pblk, line); pblk 778 drivers/lightnvm/pblk-recovery.c if (pblk_recov_check_line_version(pblk, line->emeta->buf)) pblk 781 drivers/lightnvm/pblk-recovery.c pblk_recov_wa_counters(pblk, line->emeta->buf); pblk 783 drivers/lightnvm/pblk-recovery.c if (pblk_recov_l2p_from_emeta(pblk, line)) pblk 784 drivers/lightnvm/pblk-recovery.c pblk_recov_l2p_from_oob(pblk, line); pblk 792 drivers/lightnvm/pblk-recovery.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 794 drivers/lightnvm/pblk-recovery.c move_list = pblk_line_gc_list(pblk, line); pblk 813 drivers/lightnvm/pblk-recovery.c trace_pblk_line_state(pblk_disk_name(pblk), line->id, pblk 832 drivers/lightnvm/pblk-recovery.c l_mg->data_next = pblk_line_get(pblk); pblk 842 drivers/lightnvm/pblk-recovery.c pblk_line_erase(pblk, l_mg->data_next); pblk 846 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "failed to recover all found lines %d/%d\n", pblk 855 drivers/lightnvm/pblk-recovery.c int pblk_recov_pad(struct pblk *pblk) pblk 858 drivers/lightnvm/pblk-recovery.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 867 drivers/lightnvm/pblk-recovery.c ret = pblk_recov_pad_line(pblk, line, left_msecs); pblk 869 drivers/lightnvm/pblk-recovery.c pblk_err(pblk, "tear down padding failed (%d)\n", ret); pblk 873 drivers/lightnvm/pblk-recovery.c pblk_line_close_meta(pblk, line); pblk 111 drivers/lightnvm/pblk-rl.c struct pblk *pblk = container_of(rl, struct pblk, rl); pblk 149 drivers/lightnvm/pblk-rl.c pblk_gc_should_start(pblk); pblk 151 drivers/lightnvm/pblk-rl.c pblk_gc_should_stop(pblk); pblk 212 drivers/lightnvm/pblk-rl.c struct pblk *pblk = container_of(rl, struct pblk, rl); pblk 213 drivers/lightnvm/pblk-rl.c struct nvm_tgt_dev *dev = pblk->dev; pblk 215 drivers/lightnvm/pblk-rl.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 216 drivers/lightnvm/pblk-rl.c struct pblk_line_meta *lm = &pblk->lm; pblk 224 drivers/lightnvm/pblk-rl.c rl->high = pblk->op_blks - blk_meta - lm->blk_per_line; pblk 227 drivers/lightnvm/pblk-rl.c rl->rsv_blocks = pblk_get_min_chks(pblk); pblk 241 drivers/lightnvm/pblk-rl.c rl->rb_max_io = budget - pblk->min_write_pgs_data - threshold; pblk 243 drivers/lightnvm/pblk-rl.c rl->rb_max_io = budget - pblk->min_write_pgs_data - 1; pblk 24 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_luns_show(struct pblk *pblk, char *page) pblk 26 drivers/lightnvm/pblk-sysfs.c struct nvm_tgt_dev *dev = pblk->dev; pblk 35 drivers/lightnvm/pblk-sysfs.c rlun = &pblk->luns[i]; pblk 51 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_rate_limiter(struct pblk *pblk, char *page) pblk 57 drivers/lightnvm/pblk-sysfs.c free_blocks = pblk_rl_nr_free_blks(&pblk->rl); pblk 58 drivers/lightnvm/pblk-sysfs.c free_user_blocks = pblk_rl_nr_user_free_blks(&pblk->rl); pblk 59 drivers/lightnvm/pblk-sysfs.c rb_user_max = pblk->rl.rb_user_max; pblk 60 drivers/lightnvm/pblk-sysfs.c rb_user_cnt = atomic_read(&pblk->rl.rb_user_cnt); pblk 61 drivers/lightnvm/pblk-sysfs.c rb_gc_max = pblk->rl.rb_gc_max; pblk 62 drivers/lightnvm/pblk-sysfs.c rb_gc_cnt = atomic_read(&pblk->rl.rb_gc_cnt); pblk 63 drivers/lightnvm/pblk-sysfs.c rb_budget = pblk->rl.rb_budget; pblk 64 drivers/lightnvm/pblk-sysfs.c rb_state = pblk->rl.rb_state; pblk 66 drivers/lightnvm/pblk-sysfs.c total_blocks = pblk->rl.total_blocks; pblk 76 drivers/lightnvm/pblk-sysfs.c pblk->rl.high, pblk 80 drivers/lightnvm/pblk-sysfs.c READ_ONCE(pblk->rl.rb_user_active)); pblk 83 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_gc_state_show(struct pblk *pblk, char *page) pblk 87 drivers/lightnvm/pblk-sysfs.c pblk_gc_sysfs_state_show(pblk, &gc_enabled, &gc_active); pblk 92 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_stats(struct pblk *pblk, char *page) pblk 98 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->read_failed), pblk 99 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->read_high_ecc), pblk 100 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->read_empty), pblk 101 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->read_failed_gc), pblk 102 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->write_failed), pblk 103 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->erase_failed)); pblk 108 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_write_buffer(struct pblk *pblk, char *page) pblk 110 drivers/lightnvm/pblk-sysfs.c return pblk_rb_sysfs(&pblk->rwb, page); pblk 113 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_ppaf(struct pblk *pblk, char *page) pblk 115 drivers/lightnvm/pblk-sysfs.c struct nvm_tgt_dev *dev = pblk->dev; pblk 120 drivers/lightnvm/pblk-sysfs.c struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf; pblk 125 drivers/lightnvm/pblk-sysfs.c pblk->addrf_len, pblk 142 drivers/lightnvm/pblk-sysfs.c struct nvm_addrf *ppaf = &pblk->addrf; pblk 147 drivers/lightnvm/pblk-sysfs.c pblk->addrf_len, pblk 164 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_lines(struct pblk *pblk, char *page) pblk 166 drivers/lightnvm/pblk-sysfs.c struct nvm_tgt_dev *dev = pblk->dev; pblk 168 drivers/lightnvm/pblk-sysfs.c struct pblk_line_meta *lm = &pblk->lm; pblk 169 drivers/lightnvm/pblk-sysfs.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 278 drivers/lightnvm/pblk-sysfs.c pblk_err(pblk, "corrupted free line list:%d/%d\n", pblk 298 drivers/lightnvm/pblk-sysfs.c atomic_read(&pblk->gc.read_inflight_gc)); pblk 304 drivers/lightnvm/pblk-sysfs.c atomic_read(&pblk->inflight_io)); pblk 309 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_lines_info(struct pblk *pblk, char *page) pblk 311 drivers/lightnvm/pblk-sysfs.c struct nvm_tgt_dev *dev = pblk->dev; pblk 313 drivers/lightnvm/pblk-sysfs.c struct pblk_line_meta *lm = &pblk->lm; pblk 337 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_get_sec_per_write(struct pblk *pblk, char *page) pblk 339 drivers/lightnvm/pblk-sysfs.c return snprintf(page, PAGE_SIZE, "%d\n", pblk->sec_per_write); pblk 368 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_get_write_amp_mileage(struct pblk *pblk, char *page) pblk 370 drivers/lightnvm/pblk-sysfs.c return pblk_get_write_amp(atomic64_read(&pblk->user_wa), pblk 371 drivers/lightnvm/pblk-sysfs.c atomic64_read(&pblk->gc_wa), atomic64_read(&pblk->pad_wa), pblk 375 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_get_write_amp_trip(struct pblk *pblk, char *page) pblk 378 drivers/lightnvm/pblk-sysfs.c atomic64_read(&pblk->user_wa) - pblk->user_rst_wa, pblk 379 drivers/lightnvm/pblk-sysfs.c atomic64_read(&pblk->gc_wa) - pblk->gc_rst_wa, pblk 380 drivers/lightnvm/pblk-sysfs.c atomic64_read(&pblk->pad_wa) - pblk->pad_rst_wa, page); pblk 393 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_get_padding_dist(struct pblk *pblk, char *page) pblk 398 drivers/lightnvm/pblk-sysfs.c int buckets = pblk->min_write_pgs - 1; pblk 401 drivers/lightnvm/pblk-sysfs.c total = atomic64_read(&pblk->nr_flush) - pblk->nr_flush_rst; pblk 412 drivers/lightnvm/pblk-sysfs.c total_buckets += atomic64_read(&pblk->pad_dist[i]); pblk 420 drivers/lightnvm/pblk-sysfs.c p = bucket_percentage(atomic64_read(&pblk->pad_dist[i]), pblk 431 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_stats_debug(struct pblk *pblk, char *page) pblk 435 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->inflight_writes), pblk 436 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->inflight_reads), pblk 437 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->req_writes), pblk 438 drivers/lightnvm/pblk-sysfs.c (u64)atomic64_read(&pblk->nr_flush), pblk 439 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->padded_writes), pblk 440 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->padded_wb), pblk 441 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->sub_writes), pblk 442 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->sync_writes), pblk 443 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->recov_writes), pblk 444 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->recov_gc_writes), pblk 445 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->recov_gc_reads), pblk 446 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->cache_reads), pblk 447 drivers/lightnvm/pblk-sysfs.c atomic_long_read(&pblk->sync_reads)); pblk 451 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_gc_force(struct pblk *pblk, const char *page, pblk 464 drivers/lightnvm/pblk-sysfs.c pblk_gc_sysfs_force(pblk, force); pblk 469 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_set_sec_per_write(struct pblk *pblk, pblk 482 drivers/lightnvm/pblk-sysfs.c if (!pblk_is_oob_meta_supported(pblk)) { pblk 489 drivers/lightnvm/pblk-sysfs.c if (sec_per_write < pblk->min_write_pgs pblk 490 drivers/lightnvm/pblk-sysfs.c || sec_per_write > pblk->max_write_pgs pblk 491 drivers/lightnvm/pblk-sysfs.c || sec_per_write % pblk->min_write_pgs != 0) pblk 494 drivers/lightnvm/pblk-sysfs.c pblk_set_sec_per_write(pblk, sec_per_write); pblk 499 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_set_write_amp_trip(struct pblk *pblk, pblk 515 drivers/lightnvm/pblk-sysfs.c pblk->user_rst_wa = atomic64_read(&pblk->user_wa); pblk 516 drivers/lightnvm/pblk-sysfs.c pblk->pad_rst_wa = atomic64_read(&pblk->pad_wa); pblk 517 drivers/lightnvm/pblk-sysfs.c pblk->gc_rst_wa = atomic64_read(&pblk->gc_wa); pblk 523 drivers/lightnvm/pblk-sysfs.c static ssize_t pblk_sysfs_set_padding_dist(struct pblk *pblk, pblk 528 drivers/lightnvm/pblk-sysfs.c int buckets = pblk->min_write_pgs - 1; pblk 542 drivers/lightnvm/pblk-sysfs.c atomic64_set(&pblk->pad_dist[i], 0); pblk 544 drivers/lightnvm/pblk-sysfs.c pblk->nr_flush_rst = atomic64_read(&pblk->nr_flush); pblk 644 drivers/lightnvm/pblk-sysfs.c struct pblk *pblk = container_of(kobj, struct pblk, kobj); pblk 647 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_rate_limiter(pblk, buf); pblk 649 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_luns_show(pblk, buf); pblk 651 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_gc_state_show(pblk, buf); pblk 653 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_stats(pblk, buf); pblk 655 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_write_buffer(pblk, buf); pblk 657 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_ppaf(pblk, buf); pblk 659 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_lines(pblk, buf); pblk 661 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_lines_info(pblk, buf); pblk 663 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_get_sec_per_write(pblk, buf); pblk 665 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_get_write_amp_mileage(pblk, buf); pblk 667 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_get_write_amp_trip(pblk, buf); pblk 669 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_get_padding_dist(pblk, buf); pblk 672 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_stats_debug(pblk, buf); pblk 680 drivers/lightnvm/pblk-sysfs.c struct pblk *pblk = container_of(kobj, struct pblk, kobj); pblk 683 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_gc_force(pblk, buf, len); pblk 685 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_set_sec_per_write(pblk, buf, len); pblk 687 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_set_write_amp_trip(pblk, buf, len); pblk 689 drivers/lightnvm/pblk-sysfs.c return pblk_sysfs_set_padding_dist(pblk, buf, len); pblk 705 drivers/lightnvm/pblk-sysfs.c struct pblk *pblk = tdisk->private_data; pblk 706 drivers/lightnvm/pblk-sysfs.c struct device *parent_dev = disk_to_dev(pblk->disk); pblk 709 drivers/lightnvm/pblk-sysfs.c ret = kobject_init_and_add(&pblk->kobj, &pblk_ktype, pblk 713 drivers/lightnvm/pblk-sysfs.c pblk_err(pblk, "could not register\n"); pblk 717 drivers/lightnvm/pblk-sysfs.c kobject_uevent(&pblk->kobj, KOBJ_ADD); pblk 723 drivers/lightnvm/pblk-sysfs.c struct pblk *pblk = tdisk->private_data; pblk 725 drivers/lightnvm/pblk-sysfs.c kobject_uevent(&pblk->kobj, KOBJ_REMOVE); pblk 726 drivers/lightnvm/pblk-sysfs.c kobject_del(&pblk->kobj); pblk 727 drivers/lightnvm/pblk-sysfs.c kobject_put(&pblk->kobj); pblk 3 drivers/lightnvm/pblk-trace.h #define TRACE_SYSTEM pblk pblk 144 drivers/lightnvm/pblk-trace.h #define TRACE_INCLUDE_FILE pblk-trace pblk 22 drivers/lightnvm/pblk-write.c static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd, pblk 26 drivers/lightnvm/pblk-write.c struct pblk_rb *rwb = &pblk->rwb; pblk 53 drivers/lightnvm/pblk-write.c pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid, pblk 57 drivers/lightnvm/pblk-write.c atomic_long_add(rqd->nr_ppas, &pblk->sync_writes); pblk 60 drivers/lightnvm/pblk-write.c ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid); pblk 63 drivers/lightnvm/pblk-write.c pblk_free_rqd(pblk, rqd, PBLK_WRITE); pblk 68 drivers/lightnvm/pblk-write.c static unsigned long pblk_end_queued_w_bio(struct pblk *pblk, pblk 73 drivers/lightnvm/pblk-write.c return pblk_end_w_bio(pblk, rqd, c_ctx); pblk 76 drivers/lightnvm/pblk-write.c static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd, pblk 84 drivers/lightnvm/pblk-write.c atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes); pblk 86 drivers/lightnvm/pblk-write.c pblk_up_rq(pblk, c_ctx->lun_bitmap); pblk 88 drivers/lightnvm/pblk-write.c pos = pblk_rb_sync_init(&pblk->rwb, &flags); pblk 90 drivers/lightnvm/pblk-write.c pos = pblk_end_w_bio(pblk, rqd, c_ctx); pblk 93 drivers/lightnvm/pblk-write.c list_for_each_entry_safe(c, r, &pblk->compl_list, list) { pblk 96 drivers/lightnvm/pblk-write.c pos = pblk_end_queued_w_bio(pblk, rqd, c); pblk 102 drivers/lightnvm/pblk-write.c list_add_tail(&c_ctx->list, &pblk->compl_list); pblk 104 drivers/lightnvm/pblk-write.c pblk_rb_sync_end(&pblk->rwb, &flags); pblk 108 drivers/lightnvm/pblk-write.c static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa, pblk 119 drivers/lightnvm/pblk-write.c line = pblk_ppa_to_line(pblk, *ppa); pblk 120 drivers/lightnvm/pblk-write.c lba_list = emeta_to_lbas(pblk, line->emeta->buf); pblk 125 drivers/lightnvm/pblk-write.c paddr = pblk_dev_ppa_to_line_addr(pblk, map_ppa); pblk 138 drivers/lightnvm/pblk-write.c done = nvm_next_ppa_in_chk(pblk->dev, &map_ppa); pblk 147 drivers/lightnvm/pblk-write.c static void pblk_prepare_resubmit(struct pblk *pblk, unsigned int sentry, pblk 150 drivers/lightnvm/pblk-write.c struct pblk_rb *rb = &pblk->rwb; pblk 158 drivers/lightnvm/pblk-write.c spin_lock(&pblk->trans_lock); pblk 165 drivers/lightnvm/pblk-write.c ppa_l2p = pblk_trans_map_get(pblk, w_ctx->lba); pblk 179 drivers/lightnvm/pblk-write.c line = pblk_ppa_to_line(pblk, w_ctx->ppa); pblk 183 drivers/lightnvm/pblk-write.c spin_unlock(&pblk->trans_lock); pblk 186 drivers/lightnvm/pblk-write.c static void pblk_queue_resubmit(struct pblk *pblk, struct pblk_c_ctx *c_ctx) pblk 199 drivers/lightnvm/pblk-write.c spin_lock(&pblk->resubmit_lock); pblk 200 drivers/lightnvm/pblk-write.c list_add_tail(&r_ctx->list, &pblk->resubmit_list); pblk 201 drivers/lightnvm/pblk-write.c spin_unlock(&pblk->resubmit_lock); pblk 204 drivers/lightnvm/pblk-write.c atomic_long_add(c_ctx->nr_valid, &pblk->recov_writes); pblk 212 drivers/lightnvm/pblk-write.c struct pblk *pblk = recovery->pblk; pblk 217 drivers/lightnvm/pblk-write.c pblk_log_write_err(pblk, rqd); pblk 219 drivers/lightnvm/pblk-write.c pblk_map_remaining(pblk, ppa_list, rqd->nr_ppas); pblk 220 drivers/lightnvm/pblk-write.c pblk_queue_resubmit(pblk, c_ctx); pblk 222 drivers/lightnvm/pblk-write.c pblk_up_rq(pblk, c_ctx->lun_bitmap); pblk 224 drivers/lightnvm/pblk-write.c pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid, pblk 227 drivers/lightnvm/pblk-write.c pblk_free_rqd(pblk, rqd, PBLK_WRITE); pblk 228 drivers/lightnvm/pblk-write.c mempool_free(recovery, &pblk->rec_pool); pblk 230 drivers/lightnvm/pblk-write.c atomic_dec(&pblk->inflight_io); pblk 231 drivers/lightnvm/pblk-write.c pblk_write_kick(pblk); pblk 235 drivers/lightnvm/pblk-write.c static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd) pblk 239 drivers/lightnvm/pblk-write.c recovery = mempool_alloc(&pblk->rec_pool, GFP_ATOMIC); pblk 241 drivers/lightnvm/pblk-write.c pblk_err(pblk, "could not allocate recovery work\n"); pblk 245 drivers/lightnvm/pblk-write.c recovery->pblk = pblk; pblk 249 drivers/lightnvm/pblk-write.c queue_work(pblk->close_wq, &recovery->ws_rec); pblk 254 drivers/lightnvm/pblk-write.c struct pblk *pblk = rqd->private; pblk 258 drivers/lightnvm/pblk-write.c pblk_end_w_fail(pblk, rqd); pblk 262 drivers/lightnvm/pblk-write.c pblk_check_chunk_state_update(pblk, rqd); pblk 268 drivers/lightnvm/pblk-write.c pblk_complete_write(pblk, rqd, c_ctx); pblk 269 drivers/lightnvm/pblk-write.c atomic_dec(&pblk->inflight_io); pblk 274 drivers/lightnvm/pblk-write.c struct pblk *pblk = rqd->private; pblk 281 drivers/lightnvm/pblk-write.c pblk_up_chunk(pblk, ppa_list[0]); pblk 284 drivers/lightnvm/pblk-write.c pblk_log_write_err(pblk, rqd); pblk 285 drivers/lightnvm/pblk-write.c pblk_err(pblk, "metadata I/O failed. Line %d\n", line->id); pblk 289 drivers/lightnvm/pblk-write.c pblk_check_chunk_state_update(pblk, rqd); pblk 294 drivers/lightnvm/pblk-write.c pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws, pblk 295 drivers/lightnvm/pblk-write.c GFP_ATOMIC, pblk->close_wq); pblk 297 drivers/lightnvm/pblk-write.c pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); pblk 299 drivers/lightnvm/pblk-write.c atomic_dec(&pblk->inflight_io); pblk 302 drivers/lightnvm/pblk-write.c static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 309 drivers/lightnvm/pblk-write.c rqd->private = pblk; pblk 312 drivers/lightnvm/pblk-write.c return pblk_alloc_rqd_meta(pblk, rqd); pblk 315 drivers/lightnvm/pblk-write.c static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 318 drivers/lightnvm/pblk-write.c struct pblk_line_meta *lm = &pblk->lm; pblk 319 drivers/lightnvm/pblk-write.c struct pblk_line *e_line = pblk_line_get_erase(pblk); pblk 332 drivers/lightnvm/pblk-write.c ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write); pblk 339 drivers/lightnvm/pblk-write.c ret = pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, pblk 342 drivers/lightnvm/pblk-write.c ret = pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, pblk 348 drivers/lightnvm/pblk-write.c static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail, pblk 353 drivers/lightnvm/pblk-write.c secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush, true); pblk 359 drivers/lightnvm/pblk-write.c pblk_err(pblk, "bad sector calculation (a:%d,s:%d,f:%d)\n", pblk 367 drivers/lightnvm/pblk-write.c int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line) pblk 369 drivers/lightnvm/pblk-write.c struct nvm_tgt_dev *dev = pblk->dev; pblk 371 drivers/lightnvm/pblk-write.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 372 drivers/lightnvm/pblk-write.c struct pblk_line_meta *lm = &pblk->lm; pblk 379 drivers/lightnvm/pblk-write.c int rq_ppas = pblk->min_write_pgs; pblk 385 drivers/lightnvm/pblk-write.c rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT); pblk 393 drivers/lightnvm/pblk-write.c ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta); pblk 400 drivers/lightnvm/pblk-write.c paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas); pblk 403 drivers/lightnvm/pblk-write.c ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id); pblk 412 drivers/lightnvm/pblk-write.c pblk_down_chunk(pblk, ppa_list[0]); pblk 414 drivers/lightnvm/pblk-write.c ret = pblk_submit_io(pblk, rqd, data); pblk 416 drivers/lightnvm/pblk-write.c pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); pblk 423 drivers/lightnvm/pblk-write.c pblk_up_chunk(pblk, ppa_list[0]); pblk 425 drivers/lightnvm/pblk-write.c pblk_dealloc_page(pblk, meta_line, rq_ppas); pblk 429 drivers/lightnvm/pblk-write.c pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); pblk 433 drivers/lightnvm/pblk-write.c static inline bool pblk_valid_meta_ppa(struct pblk *pblk, pblk 437 drivers/lightnvm/pblk-write.c struct nvm_tgt_dev *dev = pblk->dev; pblk 440 drivers/lightnvm/pblk-write.c struct pblk_line *data_line = pblk_line_get_data(pblk); pblk 454 drivers/lightnvm/pblk-write.c paddr = pblk_lookup_page(pblk, meta_line); pblk 455 drivers/lightnvm/pblk-write.c ppa = addr_to_gen_ppa(pblk, paddr, 0); pblk 456 drivers/lightnvm/pblk-write.c ppa_opt = addr_to_gen_ppa(pblk, paddr + data_line->meta_distance, 0); pblk 469 drivers/lightnvm/pblk-write.c static struct pblk_line *pblk_should_submit_meta_io(struct pblk *pblk, pblk 472 drivers/lightnvm/pblk-write.c struct pblk_line_meta *lm = &pblk->lm; pblk 473 drivers/lightnvm/pblk-write.c struct pblk_line_mgmt *l_mg = &pblk->l_mg; pblk 488 drivers/lightnvm/pblk-write.c if (!pblk_valid_meta_ppa(pblk, meta_line, data_rqd)) pblk 494 drivers/lightnvm/pblk-write.c static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd) pblk 503 drivers/lightnvm/pblk-write.c err = pblk_setup_w_rq(pblk, rqd, &erase_ppa); pblk 505 drivers/lightnvm/pblk-write.c pblk_err(pblk, "could not setup write request: %d\n", err); pblk 509 drivers/lightnvm/pblk-write.c meta_line = pblk_should_submit_meta_io(pblk, rqd); pblk 512 drivers/lightnvm/pblk-write.c err = pblk_submit_io(pblk, rqd, NULL); pblk 514 drivers/lightnvm/pblk-write.c pblk_err(pblk, "data I/O submission failed: %d\n", err); pblk 520 drivers/lightnvm/pblk-write.c if (pblk_blk_erase_async(pblk, erase_ppa)) { pblk 521 drivers/lightnvm/pblk-write.c struct pblk_line *e_line = pblk_line_get_erase(pblk); pblk 522 drivers/lightnvm/pblk-write.c struct nvm_tgt_dev *dev = pblk->dev; pblk 534 drivers/lightnvm/pblk-write.c err = pblk_submit_meta_io(pblk, meta_line); pblk 536 drivers/lightnvm/pblk-write.c pblk_err(pblk, "metadata I/O submission failed: %d", pblk 545 drivers/lightnvm/pblk-write.c static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd) pblk 551 drivers/lightnvm/pblk-write.c pblk_bio_free_pages(pblk, bio, c_ctx->nr_valid, pblk 555 drivers/lightnvm/pblk-write.c static int pblk_submit_write(struct pblk *pblk, int *secs_left) pblk 566 drivers/lightnvm/pblk-write.c spin_lock(&pblk->resubmit_lock); pblk 567 drivers/lightnvm/pblk-write.c resubmit = !list_empty(&pblk->resubmit_list); pblk 568 drivers/lightnvm/pblk-write.c spin_unlock(&pblk->resubmit_lock); pblk 574 drivers/lightnvm/pblk-write.c spin_lock(&pblk->resubmit_lock); pblk 575 drivers/lightnvm/pblk-write.c r_ctx = list_first_entry(&pblk->resubmit_list, pblk 578 drivers/lightnvm/pblk-write.c spin_unlock(&pblk->resubmit_lock); pblk 583 drivers/lightnvm/pblk-write.c pblk_prepare_resubmit(pblk, pos, secs_avail); pblk 584 drivers/lightnvm/pblk-write.c secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail, pblk 593 drivers/lightnvm/pblk-write.c secs_avail = pblk_rb_read_count(&pblk->rwb); pblk 597 drivers/lightnvm/pblk-write.c secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb); pblk 598 drivers/lightnvm/pblk-write.c if (!secs_to_flush && secs_avail < pblk->min_write_pgs_data) pblk 601 drivers/lightnvm/pblk-write.c secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail, pblk 603 drivers/lightnvm/pblk-write.c if (secs_to_sync > pblk->max_write_pgs) { pblk 604 drivers/lightnvm/pblk-write.c pblk_err(pblk, "bad buffer sync calculation\n"); pblk 610 drivers/lightnvm/pblk-write.c pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com); pblk 613 drivers/lightnvm/pblk-write.c packed_meta_pgs = (pblk->min_write_pgs - pblk->min_write_pgs_data); pblk 619 drivers/lightnvm/pblk-write.c rqd = pblk_alloc_rqd(pblk, PBLK_WRITE); pblk 622 drivers/lightnvm/pblk-write.c if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync, pblk 624 drivers/lightnvm/pblk-write.c pblk_err(pblk, "corrupted write bio\n"); pblk 628 drivers/lightnvm/pblk-write.c if (pblk_submit_io_set(pblk, rqd)) pblk 632 drivers/lightnvm/pblk-write.c atomic_long_add(secs_to_sync, &pblk->sub_writes); pblk 639 drivers/lightnvm/pblk-write.c pblk_free_write_rqd(pblk, rqd); pblk 642 drivers/lightnvm/pblk-write.c pblk_free_rqd(pblk, rqd, PBLK_WRITE); pblk 649 drivers/lightnvm/pblk-write.c struct pblk *pblk = data; pblk 655 drivers/lightnvm/pblk-write.c write_failure = pblk_submit_write(pblk, &secs_left); pblk 126 drivers/lightnvm/pblk.h struct pblk *pblk; pblk 133 drivers/lightnvm/pblk.h struct pblk *pblk; pblk 433 drivers/lightnvm/pblk.h struct pblk *pblk; pblk 702 drivers/lightnvm/pblk.h struct pblk *pblk; pblk 711 drivers/lightnvm/pblk.h #define pblk_err(pblk, fmt, ...) \ pblk 712 drivers/lightnvm/pblk.h pr_err("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) pblk 713 drivers/lightnvm/pblk.h #define pblk_info(pblk, fmt, ...) \ pblk 714 drivers/lightnvm/pblk.h pr_info("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) pblk 715 drivers/lightnvm/pblk.h #define pblk_warn(pblk, fmt, ...) \ pblk 716 drivers/lightnvm/pblk.h pr_warn("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) pblk 717 drivers/lightnvm/pblk.h #define pblk_debug(pblk, fmt, ...) \ pblk 718 drivers/lightnvm/pblk.h pr_debug("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) pblk 764 drivers/lightnvm/pblk.h struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type); pblk 765 drivers/lightnvm/pblk.h void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type); pblk 766 drivers/lightnvm/pblk.h int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd); pblk 767 drivers/lightnvm/pblk.h void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd); pblk 768 drivers/lightnvm/pblk.h void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write); pblk 769 drivers/lightnvm/pblk.h int pblk_setup_w_rec_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 771 drivers/lightnvm/pblk.h void pblk_discard(struct pblk *pblk, struct bio *bio); pblk 772 drivers/lightnvm/pblk.h struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk); pblk 773 drivers/lightnvm/pblk.h struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk, pblk 776 drivers/lightnvm/pblk.h void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd); pblk 777 drivers/lightnvm/pblk.h void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd); pblk 778 drivers/lightnvm/pblk.h int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd, void *buf); pblk 779 drivers/lightnvm/pblk.h int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd, void *buf); pblk 780 drivers/lightnvm/pblk.h int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line); pblk 781 drivers/lightnvm/pblk.h void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd); pblk 782 drivers/lightnvm/pblk.h struct pblk_line *pblk_line_get(struct pblk *pblk); pblk 783 drivers/lightnvm/pblk.h struct pblk_line *pblk_line_get_first_data(struct pblk *pblk); pblk 784 drivers/lightnvm/pblk.h struct pblk_line *pblk_line_replace_data(struct pblk *pblk); pblk 785 drivers/lightnvm/pblk.h void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa); pblk 786 drivers/lightnvm/pblk.h void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd); pblk 787 drivers/lightnvm/pblk.h int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line); pblk 788 drivers/lightnvm/pblk.h void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line); pblk 789 drivers/lightnvm/pblk.h struct pblk_line *pblk_line_get_data(struct pblk *pblk); pblk 790 drivers/lightnvm/pblk.h struct pblk_line *pblk_line_get_erase(struct pblk *pblk); pblk 791 drivers/lightnvm/pblk.h int pblk_line_erase(struct pblk *pblk, struct pblk_line *line); pblk 794 drivers/lightnvm/pblk.h void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line); pblk 795 drivers/lightnvm/pblk.h void pblk_line_close(struct pblk *pblk, struct pblk_line *line); pblk 797 drivers/lightnvm/pblk.h void pblk_pipeline_stop(struct pblk *pblk); pblk 798 drivers/lightnvm/pblk.h void __pblk_pipeline_stop(struct pblk *pblk); pblk 799 drivers/lightnvm/pblk.h void __pblk_pipeline_flush(struct pblk *pblk); pblk 800 drivers/lightnvm/pblk.h void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, pblk 803 drivers/lightnvm/pblk.h u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line); pblk 804 drivers/lightnvm/pblk.h int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line); pblk 805 drivers/lightnvm/pblk.h int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, pblk 807 drivers/lightnvm/pblk.h int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr erase_ppa); pblk 810 drivers/lightnvm/pblk.h struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line); pblk 811 drivers/lightnvm/pblk.h u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line); pblk 812 drivers/lightnvm/pblk.h void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); pblk 813 drivers/lightnvm/pblk.h u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); pblk 814 drivers/lightnvm/pblk.h u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); pblk 815 drivers/lightnvm/pblk.h int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail, pblk 817 drivers/lightnvm/pblk.h void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa, pblk 819 drivers/lightnvm/pblk.h void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa); pblk 820 drivers/lightnvm/pblk.h void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa); pblk 821 drivers/lightnvm/pblk.h void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap); pblk 822 drivers/lightnvm/pblk.h int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags, pblk 824 drivers/lightnvm/pblk.h void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off, pblk 826 drivers/lightnvm/pblk.h void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa); pblk 827 drivers/lightnvm/pblk.h void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, pblk 829 drivers/lightnvm/pblk.h void pblk_update_map(struct pblk *pblk, sector_t lba, struct ppa_addr ppa); pblk 830 drivers/lightnvm/pblk.h void pblk_update_map_cache(struct pblk *pblk, sector_t lba, pblk 832 drivers/lightnvm/pblk.h void pblk_update_map_dev(struct pblk *pblk, sector_t lba, pblk 834 drivers/lightnvm/pblk.h int pblk_update_map_gc(struct pblk *pblk, sector_t lba, struct ppa_addr ppa, pblk 836 drivers/lightnvm/pblk.h void pblk_lookup_l2p_rand(struct pblk *pblk, struct ppa_addr *ppas, pblk 838 drivers/lightnvm/pblk.h int pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas, pblk 840 drivers/lightnvm/pblk.h void *pblk_get_meta_for_writes(struct pblk *pblk, struct nvm_rq *rqd); pblk 841 drivers/lightnvm/pblk.h void pblk_get_packed_meta(struct pblk *pblk, struct nvm_rq *rqd); pblk 846 drivers/lightnvm/pblk.h void pblk_write_to_cache(struct pblk *pblk, struct bio *bio, pblk 848 drivers/lightnvm/pblk.h int pblk_write_gc_to_cache(struct pblk *pblk, struct pblk_gc_rq *gc_rq); pblk 853 drivers/lightnvm/pblk.h int pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, pblk 856 drivers/lightnvm/pblk.h int pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry, pblk 865 drivers/lightnvm/pblk.h void pblk_write_should_kick(struct pblk *pblk); pblk 866 drivers/lightnvm/pblk.h void pblk_write_kick(struct pblk *pblk); pblk 872 drivers/lightnvm/pblk.h void pblk_submit_read(struct pblk *pblk, struct bio *bio); pblk 873 drivers/lightnvm/pblk.h int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq); pblk 877 drivers/lightnvm/pblk.h struct pblk_line *pblk_recov_l2p(struct pblk *pblk); pblk 878 drivers/lightnvm/pblk.h int pblk_recov_pad(struct pblk *pblk); pblk 879 drivers/lightnvm/pblk.h int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta); pblk 888 drivers/lightnvm/pblk.h int pblk_gc_init(struct pblk *pblk); pblk 889 drivers/lightnvm/pblk.h void pblk_gc_exit(struct pblk *pblk, bool graceful); pblk 890 drivers/lightnvm/pblk.h void pblk_gc_should_start(struct pblk *pblk); pblk 891 drivers/lightnvm/pblk.h void pblk_gc_should_stop(struct pblk *pblk); pblk 892 drivers/lightnvm/pblk.h void pblk_gc_should_kick(struct pblk *pblk); pblk 893 drivers/lightnvm/pblk.h void pblk_gc_free_full_lines(struct pblk *pblk); pblk 894 drivers/lightnvm/pblk.h void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled, pblk 896 drivers/lightnvm/pblk.h int pblk_gc_sysfs_force(struct pblk *pblk, int force); pblk 897 drivers/lightnvm/pblk.h void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line); pblk 945 drivers/lightnvm/pblk.h static inline void *emeta_to_lbas(struct pblk *pblk, struct line_emeta *emeta) pblk 947 drivers/lightnvm/pblk.h return ((void *)emeta + pblk->lm.emeta_len[1]); pblk 950 drivers/lightnvm/pblk.h static inline void *emeta_to_vsc(struct pblk *pblk, struct line_emeta *emeta) pblk 952 drivers/lightnvm/pblk.h return (emeta_to_lbas(pblk, emeta) + pblk->lm.emeta_len[2]); pblk 965 drivers/lightnvm/pblk.h static inline struct pblk_line *pblk_ppa_to_line(struct pblk *pblk, pblk 968 drivers/lightnvm/pblk.h return &pblk->lines[pblk_ppa_to_line_id(p)]; pblk 976 drivers/lightnvm/pblk.h static inline struct ppa_addr addr_to_gen_ppa(struct pblk *pblk, u64 paddr, pblk 979 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 984 drivers/lightnvm/pblk.h struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf; pblk 994 drivers/lightnvm/pblk.h struct pblk_addrf *uaddrf = &pblk->uaddrf; pblk 1016 drivers/lightnvm/pblk.h static inline struct nvm_chk_meta *pblk_dev_ppa_to_chunk(struct pblk *pblk, pblk 1019 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 1021 drivers/lightnvm/pblk.h struct pblk_line *line = pblk_ppa_to_line(pblk, p); pblk 1027 drivers/lightnvm/pblk.h static inline u64 pblk_dev_ppa_to_chunk_addr(struct pblk *pblk, pblk 1030 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 1032 drivers/lightnvm/pblk.h return dev_to_chunk_addr(dev->parent, &pblk->addrf, p); pblk 1035 drivers/lightnvm/pblk.h static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk, pblk 1038 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 1043 drivers/lightnvm/pblk.h struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf; pblk 1051 drivers/lightnvm/pblk.h struct pblk_addrf *uaddrf = &pblk->uaddrf; pblk 1066 drivers/lightnvm/pblk.h static inline struct ppa_addr pblk_ppa32_to_ppa64(struct pblk *pblk, u32 ppa32) pblk 1068 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 1070 drivers/lightnvm/pblk.h return nvm_ppa32_to_ppa64(dev->parent, &pblk->addrf, ppa32); pblk 1073 drivers/lightnvm/pblk.h static inline u32 pblk_ppa64_to_ppa32(struct pblk *pblk, struct ppa_addr ppa64) pblk 1075 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 1077 drivers/lightnvm/pblk.h return nvm_ppa64_to_ppa32(dev->parent, &pblk->addrf, ppa64); pblk 1080 drivers/lightnvm/pblk.h static inline struct ppa_addr pblk_trans_map_get(struct pblk *pblk, pblk 1085 drivers/lightnvm/pblk.h if (pblk->addrf_len < 32) { pblk 1086 drivers/lightnvm/pblk.h u32 *map = (u32 *)pblk->trans_map; pblk 1088 drivers/lightnvm/pblk.h ppa = pblk_ppa32_to_ppa64(pblk, map[lba]); pblk 1090 drivers/lightnvm/pblk.h struct ppa_addr *map = (struct ppa_addr *)pblk->trans_map; pblk 1098 drivers/lightnvm/pblk.h static inline void pblk_trans_map_set(struct pblk *pblk, sector_t lba, pblk 1101 drivers/lightnvm/pblk.h if (pblk->addrf_len < 32) { pblk 1102 drivers/lightnvm/pblk.h u32 *map = (u32 *)pblk->trans_map; pblk 1104 drivers/lightnvm/pblk.h map[lba] = pblk_ppa64_to_ppa32(pblk, ppa); pblk 1106 drivers/lightnvm/pblk.h u64 *map = (u64 *)pblk->trans_map; pblk 1147 drivers/lightnvm/pblk.h static inline u32 pblk_calc_meta_header_crc(struct pblk *pblk, pblk 1158 drivers/lightnvm/pblk.h static inline u32 pblk_calc_smeta_crc(struct pblk *pblk, pblk 1161 drivers/lightnvm/pblk.h struct pblk_line_meta *lm = &pblk->lm; pblk 1172 drivers/lightnvm/pblk.h static inline u32 pblk_calc_emeta_crc(struct pblk *pblk, pblk 1175 drivers/lightnvm/pblk.h struct pblk_line_meta *lm = &pblk->lm; pblk 1186 drivers/lightnvm/pblk.h static inline int pblk_io_aligned(struct pblk *pblk, int nr_secs) pblk 1188 drivers/lightnvm/pblk.h return !(nr_secs % pblk->min_write_pgs); pblk 1192 drivers/lightnvm/pblk.h static inline void print_ppa(struct pblk *pblk, struct ppa_addr *p, pblk 1195 drivers/lightnvm/pblk.h struct nvm_geo *geo = &pblk->dev->geo; pblk 1198 drivers/lightnvm/pblk.h pblk_err(pblk, "ppa: (%s: %x) cache line: %llu\n", pblk 1201 drivers/lightnvm/pblk.h pblk_err(pblk, "ppa: (%s: %x):ch:%d,lun:%d,blk:%d,pg:%d,pl:%d,sec:%d\n", pblk 1206 drivers/lightnvm/pblk.h pblk_err(pblk, "ppa: (%s: %x):ch:%d,lun:%d,chk:%d,sec:%d\n", pblk 1212 drivers/lightnvm/pblk.h static inline void pblk_print_failed_rqd(struct pblk *pblk, struct nvm_rq *rqd, pblk 1218 drivers/lightnvm/pblk.h print_ppa(pblk, &rqd->ppa_addr, "rqd", error); pblk 1224 drivers/lightnvm/pblk.h print_ppa(pblk, &rqd->ppa_list[bit], "rqd", error); pblk 1227 drivers/lightnvm/pblk.h pblk_err(pblk, "error:%d, ppa_status:%llx\n", error, rqd->ppa_status); pblk 1265 drivers/lightnvm/pblk.h static inline int pblk_check_io(struct pblk *pblk, struct nvm_rq *rqd) pblk 1267 drivers/lightnvm/pblk.h struct nvm_tgt_dev *dev = pblk->dev; pblk 1280 drivers/lightnvm/pblk.h line = pblk_ppa_to_line(pblk, ppa_list[i]); pblk 1284 drivers/lightnvm/pblk.h pblk_err(pblk, "bad ppa: line:%d,state:%d\n", pblk 1298 drivers/lightnvm/pblk.h static inline int pblk_boundary_paddr_checks(struct pblk *pblk, u64 paddr) pblk 1300 drivers/lightnvm/pblk.h struct pblk_line_meta *lm = &pblk->lm; pblk 1323 drivers/lightnvm/pblk.h static inline char *pblk_disk_name(struct pblk *pblk) pblk 1325 drivers/lightnvm/pblk.h struct gendisk *disk = pblk->disk; pblk 1330 drivers/lightnvm/pblk.h static inline unsigned int pblk_get_min_chks(struct pblk *pblk) pblk 1332 drivers/lightnvm/pblk.h struct pblk_line_meta *lm = &pblk->lm; pblk 1337 drivers/lightnvm/pblk.h return DIV_ROUND_UP(100, pblk->op) * lm->blk_per_line; pblk 1340 drivers/lightnvm/pblk.h static inline struct pblk_sec_meta *pblk_get_meta(struct pblk *pblk, pblk 1344 drivers/lightnvm/pblk.h max_t(int, sizeof(struct pblk_sec_meta), pblk->oob_meta_size) pblk 1348 drivers/lightnvm/pblk.h static inline int pblk_dma_meta_size(struct pblk *pblk) pblk 1350 drivers/lightnvm/pblk.h return max_t(int, sizeof(struct pblk_sec_meta), pblk->oob_meta_size) pblk 1354 drivers/lightnvm/pblk.h static inline int pblk_is_oob_meta_supported(struct pblk *pblk) pblk 1356 drivers/lightnvm/pblk.h return pblk->oob_meta_size >= sizeof(struct pblk_sec_meta); pblk 866 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c int blk, const u8 *pblk, const u8 *fw) pblk 898 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c if (pblk >= (fw + size - len)) { pblk 899 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c len = fw + size - pblk; pblk 904 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c memcpy(pdata, pblk, len); pblk 917 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c const u8 *pblk = fw; pblk 930 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c len = qtnf_ep_fw_send(ps->base.pdev, fw_size, blk, pblk, fw); pblk 954 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c pblk -= ((last_round - 1) * pblk 958 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c pblk -= QTN_PCIE_FW_DLMASK * blk_size; pblk 971 drivers/net/wireless/quantenna/qtnfmac/pcie/pearl_pcie.c pblk += len; pblk 333 drivers/xen/xen-acpi-processor.c acpi_io_address pblk = 0; pblk 345 drivers/xen/xen-acpi-processor.c pblk = object.processor.pblk_address; pblk 372 drivers/xen/xen-acpi-processor.c pr_debug("ACPI CPU%u w/ PBLK:0x%lx\n", acpi_id, (unsigned long)pblk); pblk 383 drivers/xen/xen-acpi-processor.c if (!pblk) pblk 72 fs/crypto/bio.c sector_t pblk, unsigned int len) pblk 97 fs/crypto/bio.c bio->bi_iter.bi_sector = pblk << (blockbits - 9); pblk 113 fs/crypto/bio.c pblk++; pblk 106 fs/erofs/zmap.c erofs_blk_t pblk; pblk 174 fs/erofs/zmap.c m->pblk = le32_to_cpu(di->di_u.blkaddr); pblk 255 fs/erofs/zmap.c m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; pblk 453 fs/erofs/zmap.c map->m_pa = blknr_to_addr(m.pblk); pblk 292 fs/ext4/ext4.h #define EXT4_PBLK_CMASK(s, pblk) ((pblk) & \ pblk 300 fs/ext4/ext4.h #define EXT4_PBLK_COFF(s, pblk) ((pblk) & \ pblk 2646 fs/ext4/ext4.h ext4_fsblk_t pblk, ext4_lblk_t len); pblk 439 fs/ext4/extents.c int depth, ext4_fsblk_t pblk) pblk 486 fs/ext4/extents.c (unsigned long long) pblk, error_msg, pblk 493 fs/ext4/extents.c #define ext4_ext_check(inode, eh, depth, pblk) \ pblk 494 fs/ext4/extents.c __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk)) pblk 527 fs/ext4/extents.c struct inode *inode, ext4_fsblk_t pblk, int depth, pblk 533 fs/ext4/extents.c bh = sb_getblk_gfp(inode->i_sb, pblk, __GFP_MOVABLE | GFP_NOFS); pblk 538 fs/ext4/extents.c trace_ext4_ext_load_extent(inode, pblk, _RET_IP_); pblk 549 fs/ext4/extents.c ext_block_hdr(bh), depth, pblk); pblk 568 fs/ext4/extents.c #define read_extent_tree_block(inode, pblk, depth, flags) \ pblk 569 fs/ext4/extents.c __read_extent_tree_block(__func__, __LINE__, (inode), (pblk), \ pblk 2601 fs/ext4/extents.c ext4_fsblk_t last_pblk, pblk; pblk 2651 fs/ext4/extents.c pblk = ext4_ext_pblock(ex) + ee_len - num; pblk 2685 fs/ext4/extents.c ext4_free_blocks(handle, inode, NULL, pblk, num, flags); pblk 2688 fs/ext4/extents.c if (partial->state != initial && partial->pclu != EXT4_B2C(sbi, pblk)) pblk 2703 fs/ext4/extents.c partial->pclu = EXT4_B2C(sbi, pblk); pblk 2745 fs/ext4/extents.c ext4_fsblk_t pblk; pblk 2794 fs/ext4/extents.c pblk = ext4_ext_pblock(ex); pblk 2795 fs/ext4/extents.c partial->pclu = EXT4_B2C(sbi, pblk); pblk 2896 fs/ext4/extents.c pblk = ext4_ext_pblock(ex) + ex_ee_len - 1; pblk 2897 fs/ext4/extents.c if (partial->pclu != EXT4_B2C(sbi, pblk)) { pblk 2975 fs/ext4/extents.c ext4_fsblk_t pblk; pblk 3013 fs/ext4/extents.c pblk = ext4_ext_pblock(ex) + end - ee_block + 2; pblk 3014 fs/ext4/extents.c partial.pclu = EXT4_B2C(sbi, pblk); pblk 3042 fs/ext4/extents.c err = ext4_ext_search_right(inode, path, &lblk, &pblk, pblk 3046 fs/ext4/extents.c if (pblk) { pblk 3047 fs/ext4/extents.c partial.pclu = EXT4_B2C(sbi, pblk); pblk 446 fs/ext4/extents_status.c ext4_fsblk_t pblk) pblk 454 fs/ext4/extents_status.c es->es_pblk = pblk; pblk 807 fs/ext4/extents_status.c ext4_lblk_t len, ext4_fsblk_t pblk, pblk 816 fs/ext4/extents_status.c lblk, len, pblk, status, inode->i_ino); pblk 833 fs/ext4/extents_status.c ext4_es_store_pblock_status(&newes, pblk, status); pblk 869 fs/ext4/extents_status.c ext4_lblk_t len, ext4_fsblk_t pblk, pblk 878 fs/ext4/extents_status.c ext4_es_store_pblock_status(&newes, pblk, status); pblk 131 fs/ext4/extents_status.h ext4_lblk_t len, ext4_fsblk_t pblk, pblk 134 fs/ext4/extents_status.h ext4_lblk_t len, ext4_fsblk_t pblk, pblk 426 fs/ext4/inode.c int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, pblk 432 fs/ext4/inode.c return fscrypt_zeroout_range(inode, lblk, pblk, len); pblk 434 fs/ext4/inode.c ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); pblk 3050 fs/xfs/xfs_trace.h __field(xfs_fsblock_t, pblk) pblk 3058 fs/xfs/xfs_trace.h __entry->pblk = irec->br_startblock; pblk 3066 fs/xfs/xfs_trace.h __entry->pblk, pblk 223 include/acpi/processor.h u32 pblk; pblk 493 include/linux/fscrypt.h sector_t pblk, unsigned int len) pblk 1662 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 1673 include/trace/events/ext4.h __entry->pblk = map->m_pblk; pblk 1684 include/trace/events/ext4.h show_map_flags(__entry->flags), __entry->lblk, __entry->pblk, pblk 1703 include/trace/events/ext4.h TP_PROTO(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk), pblk 1705 include/trace/events/ext4.h TP_ARGS(inode, lblk, pblk), pblk 1710 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 1717 include/trace/events/ext4.h __entry->pblk = pblk; pblk 1724 include/trace/events/ext4.h __entry->lblk, __entry->pblk) pblk 1854 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 1865 include/trace/events/ext4.h __entry->pblk = map->m_pblk; pblk 1875 include/trace/events/ext4.h (unsigned) __entry->lblk, (unsigned long long) __entry->pblk, pblk 1890 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 1899 include/trace/events/ext4.h __entry->pblk = map->m_pblk; pblk 1906 include/trace/events/ext4.h __entry->lblk, (unsigned long long) __entry->pblk, pblk 2029 include/trace/events/ext4.h TP_PROTO(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, pblk 2032 include/trace/events/ext4.h TP_ARGS(inode, lblk, pblk, len), pblk 2037 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 2045 include/trace/events/ext4.h __entry->pblk = pblk; pblk 2054 include/trace/events/ext4.h (unsigned long long) __entry->pblk, pblk 2150 include/trace/events/ext4.h TP_PROTO(struct inode *inode, ext4_fsblk_t pblk), pblk 2152 include/trace/events/ext4.h TP_ARGS(inode, pblk), pblk 2157 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 2163 include/trace/events/ext4.h __entry->pblk = pblk; pblk 2169 include/trace/events/ext4.h (unsigned long long) __entry->pblk) pblk 2256 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 2265 include/trace/events/ext4.h __entry->pblk = ext4_es_pblock(es); pblk 2273 include/trace/events/ext4.h __entry->pblk, show_extent_status(__entry->status)) pblk 2345 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 2354 include/trace/events/ext4.h __entry->pblk = ext4_es_pblock(es); pblk 2362 include/trace/events/ext4.h __entry->pblk, show_extent_status(__entry->status)) pblk 2398 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 2408 include/trace/events/ext4.h __entry->pblk = ext4_es_pblock(es); pblk 2417 include/trace/events/ext4.h __entry->found ? __entry->pblk : 0, pblk 2566 include/trace/events/ext4.h __field( ext4_fsblk_t, pblk ) pblk 2576 include/trace/events/ext4.h __entry->pblk = ext4_es_pblock(es); pblk 2586 include/trace/events/ext4.h __entry->pblk, show_extent_status(__entry->status),