36 static int arena_read_bytes(struct arena_info *arena, resource_size_t offset,  in arena_read_bytes()  argument
39 	struct nd_btt *nd_btt = arena->nd_btt;  in arena_read_bytes()
47 static int arena_write_bytes(struct arena_info *arena, resource_size_t offset,  in arena_write_bytes()  argument
50 	struct nd_btt *nd_btt = arena->nd_btt;  in arena_write_bytes()
58 static int btt_info_write(struct arena_info *arena, struct btt_sb *super)  in btt_info_write()  argument
62 	ret = arena_write_bytes(arena, arena->info2off, super,  in btt_info_write()
67 	return arena_write_bytes(arena, arena->infooff, super,  in btt_info_write()
71 static int btt_info_read(struct arena_info *arena, struct btt_sb *super)  in btt_info_read()  argument
74 	return arena_read_bytes(arena, arena->infooff, super,  in btt_info_read()
84 static int __btt_map_write(struct arena_info *arena, u32 lba, __le32 mapping)  in __btt_map_write()  argument
86 	u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE);  in __btt_map_write()
88 	WARN_ON(lba >= arena->external_nlba);  in __btt_map_write()
89 	return arena_write_bytes(arena, ns_off, &mapping, MAP_ENT_SIZE);  in __btt_map_write()
92 static int btt_map_write(struct arena_info *arena, u32 lba, u32 mapping,  in btt_map_write()  argument
132 	return __btt_map_write(arena, lba, mapping_le);  in btt_map_write()
135 static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping,  in btt_map_read()  argument
141 	u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE);  in btt_map_read()
143 	WARN_ON(lba >= arena->external_nlba);  in btt_map_read()
145 	ret = arena_read_bytes(arena, ns_off, &in, MAP_ENT_SIZE);  in btt_map_read()
188 static int btt_log_read_pair(struct arena_info *arena, u32 lane,  in btt_log_read_pair()  argument
192 	return arena_read_bytes(arena,  in btt_log_read_pair()
193 			arena->logoff + (2 * lane * LOG_ENT_SIZE), ent,  in btt_log_read_pair()
239 	struct arena_info *arena;  in btt_debugfs_init()  local
246 	list_for_each_entry(arena, &btt->arena_list, list) {  in btt_debugfs_init()
247 		arena_debugfs_init(arena, btt->debugfs_dir, i);  in btt_debugfs_init()
295 static struct device *to_dev(struct arena_info *arena)  in to_dev()  argument
297 	return &arena->nd_btt->dev;  in to_dev()
306 static int btt_log_read(struct arena_info *arena, u32 lane,  in btt_log_read()  argument
313 	ret = btt_log_read_pair(arena, lane, log);  in btt_log_read()
319 		dev_info(to_dev(arena),  in btt_log_read()
339 static int __btt_log_write(struct arena_info *arena, u32 lane,  in __btt_log_write()  argument
351 	u64 ns_off = arena->logoff + (((2 * lane) + sub) * LOG_ENT_SIZE);  in __btt_log_write()
355 	ret = arena_write_bytes(arena, ns_off, src, log_half);  in __btt_log_write()
361 	return arena_write_bytes(arena, ns_off, src, log_half);  in __btt_log_write()
364 static int btt_flog_write(struct arena_info *arena, u32 lane, u32 sub,  in btt_flog_write()  argument
369 	ret = __btt_log_write(arena, lane, sub, ent);  in btt_flog_write()
374 	arena->freelist[lane].sub = 1 - arena->freelist[lane].sub;  in btt_flog_write()
375 	if (++(arena->freelist[lane].seq) == 4)  in btt_flog_write()
376 		arena->freelist[lane].seq = 1;  in btt_flog_write()
377 	arena->freelist[lane].block = le32_to_cpu(ent->old_map);  in btt_flog_write()
386 static int btt_map_init(struct arena_info *arena)  in btt_map_init()  argument
392 	size_t mapsize = arena->logoff - arena->mapoff;  in btt_map_init()
401 		ret = arena_write_bytes(arena, arena->mapoff + offset, zerobuf,  in btt_map_init()
420 static int btt_log_init(struct arena_info *arena)  in btt_log_init()  argument
428 	for (i = 0; i < arena->nfree; i++) {  in btt_log_init()
430 		log.old_map = cpu_to_le32(arena->external_nlba + i);  in btt_log_init()
431 		log.new_map = cpu_to_le32(arena->external_nlba + i);  in btt_log_init()
433 		ret = __btt_log_write(arena, i, 0, &log);  in btt_log_init()
436 		ret = __btt_log_write(arena, i, 1, &zerolog);  in btt_log_init()
444 static int btt_freelist_init(struct arena_info *arena)  in btt_freelist_init()  argument
450 	arena->freelist = kcalloc(arena->nfree, sizeof(struct free_entry),  in btt_freelist_init()
452 	if (!arena->freelist)  in btt_freelist_init()
455 	for (i = 0; i < arena->nfree; i++) {  in btt_freelist_init()
456 		old = btt_log_read(arena, i, &log_old, LOG_OLD_ENT);  in btt_freelist_init()
460 		new = btt_log_read(arena, i, &log_new, LOG_NEW_ENT);  in btt_freelist_init()
465 		arena->freelist[i].sub = 1 - new;  in btt_freelist_init()
466 		arena->freelist[i].seq = nd_inc_seq(le32_to_cpu(log_new.seq));  in btt_freelist_init()
467 		arena->freelist[i].block = le32_to_cpu(log_new.old_map);  in btt_freelist_init()
474 		ret = btt_map_read(arena, le32_to_cpu(log_new.lba), &map_entry,  in btt_freelist_init()
484 			ret = btt_map_write(arena, le32_to_cpu(log_new.lba),  in btt_freelist_init()
495 static int btt_rtt_init(struct arena_info *arena)  in btt_rtt_init()  argument
497 	arena->rtt = kcalloc(arena->nfree, sizeof(u32), GFP_KERNEL);  in btt_rtt_init()
498 	if (arena->rtt == NULL)  in btt_rtt_init()
504 static int btt_maplocks_init(struct arena_info *arena)  in btt_maplocks_init()  argument
508 	arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),  in btt_maplocks_init()
510 	if (!arena->map_locks)  in btt_maplocks_init()
513 	for (i = 0; i < arena->nfree; i++)  in btt_maplocks_init()
514 		spin_lock_init(&arena->map_locks[i].lock);  in btt_maplocks_init()
522 	struct arena_info *arena;  in alloc_arena()  local
526 	arena = kzalloc(sizeof(struct arena_info), GFP_KERNEL);  in alloc_arena()
527 	if (!arena)  in alloc_arena()
529 	arena->nd_btt = btt->nd_btt;  in alloc_arena()
532 		return arena;  in alloc_arena()
534 	arena->size = size;  in alloc_arena()
535 	arena->external_lba_start = start;  in alloc_arena()
536 	arena->external_lbasize = btt->lbasize;  in alloc_arena()
537 	arena->internal_lbasize = roundup(arena->external_lbasize,  in alloc_arena()
539 	arena->nfree = BTT_DEFAULT_NFREE;  in alloc_arena()
540 	arena->version_major = 1;  in alloc_arena()
541 	arena->version_minor = 1;  in alloc_arena()
550 	logsize = roundup(2 * arena->nfree * sizeof(struct log_entry),  in alloc_arena()
555 	arena->internal_nlba = div_u64(available - BTT_PG_SIZE,  in alloc_arena()
556 			arena->internal_lbasize + MAP_ENT_SIZE);  in alloc_arena()
557 	arena->external_nlba = arena->internal_nlba - arena->nfree;  in alloc_arena()
559 	mapsize = roundup((arena->external_nlba * MAP_ENT_SIZE), BTT_PG_SIZE);  in alloc_arena()
563 	arena->infooff = arena_off;  in alloc_arena()
564 	arena->dataoff = arena->infooff + BTT_PG_SIZE;  in alloc_arena()
565 	arena->mapoff = arena->dataoff + datasize;  in alloc_arena()
566 	arena->logoff = arena->mapoff + mapsize;  in alloc_arena()
567 	arena->info2off = arena->logoff + logsize;  in alloc_arena()
568 	return arena;  in alloc_arena()
573 	struct arena_info *arena, *next;  in free_arenas()  local
575 	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {  in free_arenas()
576 		list_del(&arena->list);  in free_arenas()
577 		kfree(arena->rtt);  in free_arenas()
578 		kfree(arena->map_locks);  in free_arenas()
579 		kfree(arena->freelist);  in free_arenas()
580 		debugfs_remove_recursive(arena->debugfs_dir);  in free_arenas()
581 		kfree(arena);  in free_arenas()
589 static void parse_arena_meta(struct arena_info *arena, struct btt_sb *super,  in parse_arena_meta()  argument
592 	arena->internal_nlba = le32_to_cpu(super->internal_nlba);  in parse_arena_meta()
593 	arena->internal_lbasize = le32_to_cpu(super->internal_lbasize);  in parse_arena_meta()
594 	arena->external_nlba = le32_to_cpu(super->external_nlba);  in parse_arena_meta()
595 	arena->external_lbasize = le32_to_cpu(super->external_lbasize);  in parse_arena_meta()
596 	arena->nfree = le32_to_cpu(super->nfree);  in parse_arena_meta()
597 	arena->version_major = le16_to_cpu(super->version_major);  in parse_arena_meta()
598 	arena->version_minor = le16_to_cpu(super->version_minor);  in parse_arena_meta()
600 	arena->nextoff = (super->nextoff == 0) ? 0 : (arena_off +  in parse_arena_meta()
602 	arena->infooff = arena_off;  in parse_arena_meta()
603 	arena->dataoff = arena_off + le64_to_cpu(super->dataoff);  in parse_arena_meta()
604 	arena->mapoff = arena_off + le64_to_cpu(super->mapoff);  in parse_arena_meta()
605 	arena->logoff = arena_off + le64_to_cpu(super->logoff);  in parse_arena_meta()
606 	arena->info2off = arena_off + le64_to_cpu(super->info2off);  in parse_arena_meta()
608 	arena->size = (le64_to_cpu(super->nextoff) > 0)  in parse_arena_meta()
610 		: (arena->info2off - arena->infooff + BTT_PG_SIZE);  in parse_arena_meta()
612 	arena->flags = le32_to_cpu(super->flags);  in parse_arena_meta()
618 	struct arena_info *arena;  in discover_arenas()  local
631 		arena = alloc_arena(btt, 0, 0, 0);  in discover_arenas()
632 		if (!arena) {  in discover_arenas()
637 		arena->infooff = cur_off;  in discover_arenas()
638 		ret = btt_info_read(arena, super);  in discover_arenas()
645 				dev_info(to_dev(arena), "No existing arenas\n");  in discover_arenas()
648 				dev_info(to_dev(arena),  in discover_arenas()
655 		arena->external_lba_start = cur_nlba;  in discover_arenas()
656 		parse_arena_meta(arena, super, cur_off);  in discover_arenas()
658 		ret = btt_freelist_init(arena);  in discover_arenas()
662 		ret = btt_rtt_init(arena);  in discover_arenas()
666 		ret = btt_maplocks_init(arena);  in discover_arenas()
670 		list_add_tail(&arena->list, &btt->arena_list);  in discover_arenas()
672 		remaining -= arena->size;  in discover_arenas()
673 		cur_off += arena->size;  in discover_arenas()
674 		cur_nlba += arena->external_nlba;  in discover_arenas()
677 		if (arena->nextoff == 0)  in discover_arenas()
688 	kfree(arena);  in discover_arenas()
701 		struct arena_info *arena;  in create_arenas()  local
708 		arena = alloc_arena(btt, arena_size, btt->nlba, cur_off);  in create_arenas()
709 		if (!arena) {  in create_arenas()
713 		btt->nlba += arena->external_nlba;  in create_arenas()
715 			arena->nextoff = arena->size;  in create_arenas()
717 			arena->nextoff = 0;  in create_arenas()
719 		list_add_tail(&arena->list, &btt->arena_list);  in create_arenas()
731 static int btt_arena_write_layout(struct arena_info *arena)  in btt_arena_write_layout()  argument
736 	struct nd_btt *nd_btt = arena->nd_btt;  in btt_arena_write_layout()
739 	ret = btt_map_init(arena);  in btt_arena_write_layout()
743 	ret = btt_log_init(arena);  in btt_arena_write_layout()
754 	super->flags = cpu_to_le32(arena->flags);  in btt_arena_write_layout()
755 	super->version_major = cpu_to_le16(arena->version_major);  in btt_arena_write_layout()
756 	super->version_minor = cpu_to_le16(arena->version_minor);  in btt_arena_write_layout()
757 	super->external_lbasize = cpu_to_le32(arena->external_lbasize);  in btt_arena_write_layout()
758 	super->external_nlba = cpu_to_le32(arena->external_nlba);  in btt_arena_write_layout()
759 	super->internal_lbasize = cpu_to_le32(arena->internal_lbasize);  in btt_arena_write_layout()
760 	super->internal_nlba = cpu_to_le32(arena->internal_nlba);  in btt_arena_write_layout()
761 	super->nfree = cpu_to_le32(arena->nfree);  in btt_arena_write_layout()
763 	super->nextoff = cpu_to_le64(arena->nextoff);  in btt_arena_write_layout()
768 	super->dataoff = cpu_to_le64(arena->dataoff - arena->infooff);  in btt_arena_write_layout()
769 	super->mapoff = cpu_to_le64(arena->mapoff - arena->infooff);  in btt_arena_write_layout()
770 	super->logoff = cpu_to_le64(arena->logoff - arena->infooff);  in btt_arena_write_layout()
771 	super->info2off = cpu_to_le64(arena->info2off - arena->infooff);  in btt_arena_write_layout()
777 	ret = btt_info_write(arena, super);  in btt_arena_write_layout()
790 	struct arena_info *arena;  in btt_meta_init()  local
793 	list_for_each_entry(arena, &btt->arena_list, list) {  in btt_meta_init()
794 		ret = btt_arena_write_layout(arena);  in btt_meta_init()
798 		ret = btt_freelist_init(arena);  in btt_meta_init()
802 		ret = btt_rtt_init(arena);  in btt_meta_init()
806 		ret = btt_maplocks_init(arena);  in btt_meta_init()
831 				struct arena_info **arena)  in lba_to_arena()  argument
838 			*arena = arena_list;  in lba_to_arena()
852 static void lock_map(struct arena_info *arena, u32 premap)  in lock_map()  argument
853 		__acquires(&arena->map_locks[idx].lock)  in lock_map()
855 	u32 idx = (premap * MAP_ENT_SIZE / L1_CACHE_BYTES) % arena->nfree;  in lock_map()
857 	spin_lock(&arena->map_locks[idx].lock);  in lock_map()
860 static void unlock_map(struct arena_info *arena, u32 premap)  in unlock_map()  argument
861 		__releases(&arena->map_locks[idx].lock)  in unlock_map()
863 	u32 idx = (premap * MAP_ENT_SIZE / L1_CACHE_BYTES) % arena->nfree;  in unlock_map()
865 	spin_unlock(&arena->map_locks[idx].lock);  in unlock_map()
868 static u64 to_namespace_offset(struct arena_info *arena, u64 lba)  in to_namespace_offset()  argument
870 	return arena->dataoff + ((u64)lba * arena->internal_lbasize);  in to_namespace_offset()
873 static int btt_data_read(struct arena_info *arena, struct page *page,  in btt_data_read()  argument
877 	u64 nsoff = to_namespace_offset(arena, lba);  in btt_data_read()
880 	ret = arena_read_bytes(arena, nsoff, mem + off, len);  in btt_data_read()
886 static int btt_data_write(struct arena_info *arena, u32 lba,  in btt_data_write()  argument
890 	u64 nsoff = to_namespace_offset(arena, lba);  in btt_data_write()
893 	ret = arena_write_bytes(arena, nsoff, mem + off, len);  in btt_data_write()
909 			struct arena_info *arena, u32 postmap, int rw)  in btt_rw_integrity()  argument
918 	meta_nsoff = to_namespace_offset(arena, postmap) + btt->sector_size;  in btt_rw_integrity()
935 			ret = arena_write_bytes(arena, meta_nsoff,  in btt_rw_integrity()
938 			ret = arena_read_bytes(arena, meta_nsoff,  in btt_rw_integrity()
955 			struct arena_info *arena, u32 postmap, int rw)  in btt_rw_integrity()  argument
967 	struct arena_info *arena = NULL;  in btt_read_pg()  local
975 		ret = lba_to_arena(btt, sector, &premap, &arena);  in btt_read_pg()
981 		ret = btt_map_read(arena, premap, &postmap, &t_flag, &e_flag);  in btt_read_pg()
1003 			arena->rtt[lane] = RTT_VALID | postmap;  in btt_read_pg()
1010 			ret = btt_map_read(arena, premap, &new_map, &t_flag,  in btt_read_pg()
1021 		ret = btt_data_read(arena, page, off, postmap, cur_len);  in btt_read_pg()
1026 			ret = btt_rw_integrity(btt, bip, arena, postmap, READ);  in btt_read_pg()
1031 		arena->rtt[lane] = RTT_INVALID;  in btt_read_pg()
1042 	arena->rtt[lane] = RTT_INVALID;  in btt_read_pg()
1053 	struct arena_info *arena = NULL;  in btt_write_pg()  local
1063 		ret = lba_to_arena(btt, sector, &premap, &arena);  in btt_write_pg()
1068 		if ((arena->flags & IB_FLAG_ERROR_MASK) != 0) {  in btt_write_pg()
1073 		new_postmap = arena->freelist[lane].block;  in btt_write_pg()
1076 		for (i = 0; i < arena->nfree; i++)  in btt_write_pg()
1077 			while (arena->rtt[i] == (RTT_VALID | new_postmap))  in btt_write_pg()
1081 		if (new_postmap >= arena->internal_nlba) {  in btt_write_pg()
1086 		ret = btt_data_write(arena, new_postmap, page, off, cur_len);  in btt_write_pg()
1091 			ret = btt_rw_integrity(btt, bip, arena, new_postmap,  in btt_write_pg()
1097 		lock_map(arena, premap);  in btt_write_pg()
1098 		ret = btt_map_read(arena, premap, &old_postmap, NULL, NULL);  in btt_write_pg()
1101 		if (old_postmap >= arena->internal_nlba) {  in btt_write_pg()
1109 		log.seq = cpu_to_le32(arena->freelist[lane].seq);  in btt_write_pg()
1110 		sub = arena->freelist[lane].sub;  in btt_write_pg()
1111 		ret = btt_flog_write(arena, lane, sub, &log);  in btt_write_pg()
1115 		ret = btt_map_write(arena, premap, new_postmap, 0, 0);  in btt_write_pg()
1119 		unlock_map(arena, premap);  in btt_write_pg()
1130 	unlock_map(arena, premap);  in btt_write_pg()