1/*
2 * Copyright (C) 2015 Matias Bjorling <m@bjorling.me>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; see the file COPYING.  If not, write to
15 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
16 * USA.
17 *
18 * Implementation of a generic nvm manager for Open-Channel SSDs.
19 */
20
21#include "gennvm.h"
22
23static void gennvm_blocks_free(struct nvm_dev *dev)
24{
25	struct gen_nvm *gn = dev->mp;
26	struct gen_lun *lun;
27	int i;
28
29	gennvm_for_each_lun(gn, lun, i) {
30		if (!lun->vlun.blocks)
31			break;
32		vfree(lun->vlun.blocks);
33	}
34}
35
36static void gennvm_luns_free(struct nvm_dev *dev)
37{
38	struct gen_nvm *gn = dev->mp;
39
40	kfree(gn->luns);
41}
42
43static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn)
44{
45	struct gen_lun *lun;
46	int i;
47
48	gn->luns = kcalloc(dev->nr_luns, sizeof(struct gen_lun), GFP_KERNEL);
49	if (!gn->luns)
50		return -ENOMEM;
51
52	gennvm_for_each_lun(gn, lun, i) {
53		spin_lock_init(&lun->vlun.lock);
54		INIT_LIST_HEAD(&lun->free_list);
55		INIT_LIST_HEAD(&lun->used_list);
56		INIT_LIST_HEAD(&lun->bb_list);
57
58		lun->reserved_blocks = 2; /* for GC only */
59		lun->vlun.id = i;
60		lun->vlun.lun_id = i % dev->luns_per_chnl;
61		lun->vlun.chnl_id = i / dev->luns_per_chnl;
62		lun->vlun.nr_free_blocks = dev->blks_per_lun;
63		lun->vlun.nr_inuse_blocks = 0;
64		lun->vlun.nr_bad_blocks = 0;
65	}
66	return 0;
67}
68
69static int gennvm_block_bb(struct ppa_addr ppa, int nr_blocks, u8 *blks,
70								void *private)
71{
72	struct gen_nvm *gn = private;
73	struct nvm_dev *dev = gn->dev;
74	struct gen_lun *lun;
75	struct nvm_block *blk;
76	int i;
77
78	lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun];
79
80	for (i = 0; i < nr_blocks; i++) {
81		if (blks[i] == 0)
82			continue;
83
84		blk = &lun->vlun.blocks[i];
85		if (!blk) {
86			pr_err("gennvm: BB data is out of bounds.\n");
87			return -EINVAL;
88		}
89
90		list_move_tail(&blk->list, &lun->bb_list);
91		lun->vlun.nr_bad_blocks++;
92	}
93
94	return 0;
95}
96
97static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private)
98{
99	struct nvm_dev *dev = private;
100	struct gen_nvm *gn = dev->mp;
101	sector_t max_pages = dev->total_pages * (dev->sec_size >> 9);
102	u64 elba = slba + nlb;
103	struct gen_lun *lun;
104	struct nvm_block *blk;
105	u64 i;
106	int lun_id;
107
108	if (unlikely(elba > dev->total_pages)) {
109		pr_err("gennvm: L2P data from device is out of bounds!\n");
110		return -EINVAL;
111	}
112
113	for (i = 0; i < nlb; i++) {
114		u64 pba = le64_to_cpu(entries[i]);
115
116		if (unlikely(pba >= max_pages && pba != U64_MAX)) {
117			pr_err("gennvm: L2P data entry is out of bounds!\n");
118			return -EINVAL;
119		}
120
121		/* Address zero is a special one. The first page on a disk is
122		 * protected. It often holds internal device boot
123		 * information.
124		 */
125		if (!pba)
126			continue;
127
128		/* resolve block from physical address */
129		lun_id = div_u64(pba, dev->sec_per_lun);
130		lun = &gn->luns[lun_id];
131
132		/* Calculate block offset into lun */
133		pba = pba - (dev->sec_per_lun * lun_id);
134		blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)];
135
136		if (!blk->type) {
137			/* at this point, we don't know anything about the
138			 * block. It's up to the FTL on top to re-etablish the
139			 * block state
140			 */
141			list_move_tail(&blk->list, &lun->used_list);
142			blk->type = 1;
143			lun->vlun.nr_free_blocks--;
144			lun->vlun.nr_inuse_blocks++;
145		}
146	}
147
148	return 0;
149}
150
151static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn)
152{
153	struct gen_lun *lun;
154	struct nvm_block *block;
155	sector_t lun_iter, blk_iter, cur_block_id = 0;
156	int ret;
157
158	gennvm_for_each_lun(gn, lun, lun_iter) {
159		lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) *
160							dev->blks_per_lun);
161		if (!lun->vlun.blocks)
162			return -ENOMEM;
163
164		for (blk_iter = 0; blk_iter < dev->blks_per_lun; blk_iter++) {
165			block = &lun->vlun.blocks[blk_iter];
166
167			INIT_LIST_HEAD(&block->list);
168
169			block->lun = &lun->vlun;
170			block->id = cur_block_id++;
171
172			/* First block is reserved for device */
173			if (unlikely(lun_iter == 0 && blk_iter == 0)) {
174				lun->vlun.nr_free_blocks--;
175				continue;
176			}
177
178			list_add_tail(&block->list, &lun->free_list);
179		}
180
181		if (dev->ops->get_bb_tbl) {
182			struct ppa_addr ppa;
183
184			ppa.ppa = 0;
185			ppa.g.ch = lun->vlun.chnl_id;
186			ppa.g.lun = lun->vlun.id;
187			ppa = generic_to_dev_addr(dev, ppa);
188
189			ret = dev->ops->get_bb_tbl(dev, ppa,
190						dev->blks_per_lun,
191						gennvm_block_bb, gn);
192			if (ret)
193				pr_err("gennvm: could not read BB table\n");
194		}
195	}
196
197	if (dev->ops->get_l2p_tbl) {
198		ret = dev->ops->get_l2p_tbl(dev, 0, dev->total_pages,
199							gennvm_block_map, dev);
200		if (ret) {
201			pr_err("gennvm: could not read L2P table.\n");
202			pr_warn("gennvm: default block initialization");
203		}
204	}
205
206	return 0;
207}
208
209static void gennvm_free(struct nvm_dev *dev)
210{
211	gennvm_blocks_free(dev);
212	gennvm_luns_free(dev);
213	kfree(dev->mp);
214	dev->mp = NULL;
215}
216
217static int gennvm_register(struct nvm_dev *dev)
218{
219	struct gen_nvm *gn;
220	int ret;
221
222	if (!try_module_get(THIS_MODULE))
223		return -ENODEV;
224
225	gn = kzalloc(sizeof(struct gen_nvm), GFP_KERNEL);
226	if (!gn)
227		return -ENOMEM;
228
229	gn->dev = dev;
230	gn->nr_luns = dev->nr_luns;
231	dev->mp = gn;
232
233	ret = gennvm_luns_init(dev, gn);
234	if (ret) {
235		pr_err("gennvm: could not initialize luns\n");
236		goto err;
237	}
238
239	ret = gennvm_blocks_init(dev, gn);
240	if (ret) {
241		pr_err("gennvm: could not initialize blocks\n");
242		goto err;
243	}
244
245	return 1;
246err:
247	gennvm_free(dev);
248	module_put(THIS_MODULE);
249	return ret;
250}
251
252static void gennvm_unregister(struct nvm_dev *dev)
253{
254	gennvm_free(dev);
255	module_put(THIS_MODULE);
256}
257
258static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
259				struct nvm_lun *vlun, unsigned long flags)
260{
261	struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
262	struct nvm_block *blk = NULL;
263	int is_gc = flags & NVM_IOTYPE_GC;
264
265	spin_lock(&vlun->lock);
266
267	if (list_empty(&lun->free_list)) {
268		pr_err_ratelimited("gennvm: lun %u have no free pages available",
269								lun->vlun.id);
270		goto out;
271	}
272
273	if (!is_gc && lun->vlun.nr_free_blocks < lun->reserved_blocks)
274		goto out;
275
276	blk = list_first_entry(&lun->free_list, struct nvm_block, list);
277	list_move_tail(&blk->list, &lun->used_list);
278	blk->type = 1;
279
280	lun->vlun.nr_free_blocks--;
281	lun->vlun.nr_inuse_blocks++;
282
283out:
284	spin_unlock(&vlun->lock);
285	return blk;
286}
287
288static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
289{
290	struct nvm_lun *vlun = blk->lun;
291	struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
292
293	spin_lock(&vlun->lock);
294
295	switch (blk->type) {
296	case 1:
297		list_move_tail(&blk->list, &lun->free_list);
298		lun->vlun.nr_free_blocks++;
299		lun->vlun.nr_inuse_blocks--;
300		blk->type = 0;
301		break;
302	case 2:
303		list_move_tail(&blk->list, &lun->bb_list);
304		lun->vlun.nr_bad_blocks++;
305		lun->vlun.nr_inuse_blocks--;
306		break;
307	default:
308		WARN_ON_ONCE(1);
309		pr_err("gennvm: erroneous block type (%lu -> %u)\n",
310							blk->id, blk->type);
311		list_move_tail(&blk->list, &lun->bb_list);
312		lun->vlun.nr_bad_blocks++;
313		lun->vlun.nr_inuse_blocks--;
314	}
315
316	spin_unlock(&vlun->lock);
317}
318
319static void gennvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
320{
321	int i;
322
323	if (rqd->nr_pages > 1) {
324		for (i = 0; i < rqd->nr_pages; i++)
325			rqd->ppa_list[i] = dev_to_generic_addr(dev,
326							rqd->ppa_list[i]);
327	} else {
328		rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
329	}
330}
331
332static void gennvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
333{
334	int i;
335
336	if (rqd->nr_pages > 1) {
337		for (i = 0; i < rqd->nr_pages; i++)
338			rqd->ppa_list[i] = generic_to_dev_addr(dev,
339							rqd->ppa_list[i]);
340	} else {
341		rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
342	}
343}
344
345static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
346{
347	if (!dev->ops->submit_io)
348		return 0;
349
350	/* Convert address space */
351	gennvm_generic_to_addr_mode(dev, rqd);
352
353	rqd->dev = dev;
354	return dev->ops->submit_io(dev, rqd);
355}
356
357static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa,
358								int type)
359{
360	struct gen_nvm *gn = dev->mp;
361	struct gen_lun *lun;
362	struct nvm_block *blk;
363
364	if (unlikely(ppa->g.ch > dev->nr_chnls ||
365					ppa->g.lun > dev->luns_per_chnl ||
366					ppa->g.blk > dev->blks_per_lun)) {
367		WARN_ON_ONCE(1);
368		pr_err("gennvm: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u",
369				ppa->g.ch, dev->nr_chnls,
370				ppa->g.lun, dev->luns_per_chnl,
371				ppa->g.blk, dev->blks_per_lun);
372		return;
373	}
374
375	lun = &gn->luns[ppa->g.lun * ppa->g.ch];
376	blk = &lun->vlun.blocks[ppa->g.blk];
377
378	/* will be moved to bb list on put_blk from target */
379	blk->type = type;
380}
381
382/* mark block bad. It is expected the target recover from the error. */
383static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd)
384{
385	int i;
386
387	if (!dev->ops->set_bb_tbl)
388		return;
389
390	if (dev->ops->set_bb_tbl(dev, rqd, 1))
391		return;
392
393	gennvm_addr_to_generic_mode(dev, rqd);
394
395	/* look up blocks and mark them as bad */
396	if (rqd->nr_pages > 1)
397		for (i = 0; i < rqd->nr_pages; i++)
398			gennvm_blk_set_type(dev, &rqd->ppa_list[i], 2);
399	else
400		gennvm_blk_set_type(dev, &rqd->ppa_addr, 2);
401}
402
403static int gennvm_end_io(struct nvm_rq *rqd, int error)
404{
405	struct nvm_tgt_instance *ins = rqd->ins;
406	int ret = 0;
407
408	switch (error) {
409	case NVM_RSP_SUCCESS:
410		break;
411	case NVM_RSP_ERR_EMPTYPAGE:
412		break;
413	case NVM_RSP_ERR_FAILWRITE:
414		gennvm_mark_blk_bad(rqd->dev, rqd);
415	default:
416		ret++;
417	}
418
419	ret += ins->tt->end_io(rqd, error);
420
421	return ret;
422}
423
424static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk,
425							unsigned long flags)
426{
427	int plane_cnt = 0, pl_idx, ret;
428	struct ppa_addr addr;
429	struct nvm_rq rqd;
430
431	if (!dev->ops->erase_block)
432		return 0;
433
434	addr = block_to_ppa(dev, blk);
435
436	if (dev->plane_mode == NVM_PLANE_SINGLE) {
437		rqd.nr_pages = 1;
438		rqd.ppa_addr = addr;
439	} else {
440		plane_cnt = (1 << dev->plane_mode);
441		rqd.nr_pages = plane_cnt;
442
443		rqd.ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL,
444							&rqd.dma_ppa_list);
445		if (!rqd.ppa_list) {
446			pr_err("gennvm: failed to allocate dma memory\n");
447			return -ENOMEM;
448		}
449
450		for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
451			addr.g.pl = pl_idx;
452			rqd.ppa_list[pl_idx] = addr;
453		}
454	}
455
456	gennvm_generic_to_addr_mode(dev, &rqd);
457
458	ret = dev->ops->erase_block(dev, &rqd);
459
460	if (plane_cnt)
461		nvm_dev_dma_free(dev, rqd.ppa_list, rqd.dma_ppa_list);
462
463	return ret;
464}
465
466static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid)
467{
468	struct gen_nvm *gn = dev->mp;
469
470	return &gn->luns[lunid].vlun;
471}
472
473static void gennvm_lun_info_print(struct nvm_dev *dev)
474{
475	struct gen_nvm *gn = dev->mp;
476	struct gen_lun *lun;
477	unsigned int i;
478
479
480	gennvm_for_each_lun(gn, lun, i) {
481		spin_lock(&lun->vlun.lock);
482
483		pr_info("%s: lun%8u\t%u\t%u\t%u\n",
484				dev->name, i,
485				lun->vlun.nr_free_blocks,
486				lun->vlun.nr_inuse_blocks,
487				lun->vlun.nr_bad_blocks);
488
489		spin_unlock(&lun->vlun.lock);
490	}
491}
492
493static struct nvmm_type gennvm = {
494	.name		= "gennvm",
495	.version	= {0, 1, 0},
496
497	.register_mgr	= gennvm_register,
498	.unregister_mgr	= gennvm_unregister,
499
500	.get_blk	= gennvm_get_blk,
501	.put_blk	= gennvm_put_blk,
502
503	.submit_io	= gennvm_submit_io,
504	.end_io		= gennvm_end_io,
505	.erase_blk	= gennvm_erase_blk,
506
507	.get_lun	= gennvm_get_lun,
508	.lun_info_print = gennvm_lun_info_print,
509};
510
511static int __init gennvm_module_init(void)
512{
513	return nvm_register_mgr(&gennvm);
514}
515
516static void gennvm_module_exit(void)
517{
518	nvm_unregister_mgr(&gennvm);
519}
520
521module_init(gennvm_module_init);
522module_exit(gennvm_module_exit);
523MODULE_LICENSE("GPL v2");
524MODULE_DESCRIPTION("Generic media manager for Open-Channel SSDs");
525