236 static void btt_debugfs_init(struct btt *btt)  in btt_debugfs_init()  argument
241 	btt->debugfs_dir = debugfs_create_dir(dev_name(&btt->nd_btt->dev),  in btt_debugfs_init()
243 	if (IS_ERR_OR_NULL(btt->debugfs_dir))  in btt_debugfs_init()
246 	list_for_each_entry(arena, &btt->arena_list, list) {  in btt_debugfs_init()
247 		arena_debugfs_init(arena, btt->debugfs_dir, i);  in btt_debugfs_init()
519 static struct arena_info *alloc_arena(struct btt *btt, size_t size,  in alloc_arena()  argument
529 	arena->nd_btt = btt->nd_btt;  in alloc_arena()
536 	arena->external_lbasize = btt->lbasize;  in alloc_arena()
571 static void free_arenas(struct btt *btt)  in free_arenas()  argument
575 	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {  in free_arenas()
615 static int discover_arenas(struct btt *btt)  in discover_arenas()  argument
620 	size_t remaining = btt->rawsize;  in discover_arenas()
631 		arena = alloc_arena(btt, 0, 0, 0);  in discover_arenas()
642 		if (!nd_btt_arena_is_valid(btt->nd_btt, super)) {  in discover_arenas()
643 			if (remaining == btt->rawsize) {  in discover_arenas()
644 				btt->init_state = INIT_NOTFOUND;  in discover_arenas()
670 		list_add_tail(&arena->list, &btt->arena_list);  in discover_arenas()
680 	btt->num_arenas = num_arenas;  in discover_arenas()
681 	btt->nlba = cur_nlba;  in discover_arenas()
682 	btt->init_state = INIT_READY;  in discover_arenas()
689 	free_arenas(btt);  in discover_arenas()
695 static int create_arenas(struct btt *btt)  in create_arenas()  argument
697 	size_t remaining = btt->rawsize;  in create_arenas()
708 		arena = alloc_arena(btt, arena_size, btt->nlba, cur_off);  in create_arenas()
710 			free_arenas(btt);  in create_arenas()
713 		btt->nlba += arena->external_nlba;  in create_arenas()
719 		list_add_tail(&arena->list, &btt->arena_list);  in create_arenas()
787 static int btt_meta_init(struct btt *btt)  in btt_meta_init()  argument
792 	mutex_lock(&btt->init_lock);  in btt_meta_init()
793 	list_for_each_entry(arena, &btt->arena_list, list) {  in btt_meta_init()
811 	btt->init_state = INIT_READY;  in btt_meta_init()
814 	mutex_unlock(&btt->init_lock);  in btt_meta_init()
818 static u32 btt_meta_size(struct btt *btt)  in btt_meta_size()  argument
820 	return btt->lbasize - btt->sector_size;  in btt_meta_size()
830 static int lba_to_arena(struct btt *btt, sector_t sector, __u32 *premap,  in lba_to_arena()  argument
834 	__u64 lba = div_u64(sector << SECTOR_SHIFT, btt->sector_size);  in lba_to_arena()
836 	list_for_each_entry(arena_list, &btt->arena_list, list) {  in lba_to_arena()
908 static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip,  in btt_rw_integrity()  argument
911 	unsigned int len = btt_meta_size(btt);  in btt_rw_integrity()
918 	meta_nsoff = to_namespace_offset(arena, postmap) + btt->sector_size;  in btt_rw_integrity()
954 static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip,  in btt_rw_integrity()  argument
961 static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip,  in btt_read_pg()  argument
973 		lane = nd_region_acquire_lane(btt->nd_region);  in btt_read_pg()
975 		ret = lba_to_arena(btt, sector, &premap, &arena);  in btt_read_pg()
979 		cur_len = min(btt->sector_size, len);  in btt_read_pg()
1026 			ret = btt_rw_integrity(btt, bip, arena, postmap, READ);  in btt_read_pg()
1032 		nd_region_release_lane(btt->nd_region, lane);  in btt_read_pg()
1036 		sector += btt->sector_size >> SECTOR_SHIFT;  in btt_read_pg()
1044 	nd_region_release_lane(btt->nd_region, lane);  in btt_read_pg()
1048 static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,  in btt_write_pg()  argument
1061 		lane = nd_region_acquire_lane(btt->nd_region);  in btt_write_pg()
1063 		ret = lba_to_arena(btt, sector, &premap, &arena);  in btt_write_pg()
1066 		cur_len = min(btt->sector_size, len);  in btt_write_pg()
1091 			ret = btt_rw_integrity(btt, bip, arena, new_postmap,  in btt_write_pg()
1120 		nd_region_release_lane(btt->nd_region, lane);  in btt_write_pg()
1124 		sector += btt->sector_size >> SECTOR_SHIFT;  in btt_write_pg()
1132 	nd_region_release_lane(btt->nd_region, lane);  in btt_write_pg()
1136 static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip,  in btt_do_bvec()  argument
1143 		ret = btt_read_pg(btt, bip, page, off, sector, len);  in btt_do_bvec()
1147 		ret = btt_write_pg(btt, bip, sector, page, off, len);  in btt_do_bvec()
1156 	struct btt *btt = q->queuedata;  in btt_make_request()  local
1182 		BUG_ON(len < btt->sector_size);  in btt_make_request()
1183 		BUG_ON(len % btt->sector_size);  in btt_make_request()
1185 		err = btt_do_bvec(btt, bip, bvec.bv_page, len, bvec.bv_offset,  in btt_make_request()
1188 			dev_info(&btt->nd_btt->dev,  in btt_make_request()
1207 	struct btt *btt = bdev->bd_disk->private_data;  in btt_rw_page()  local
1209 	btt_do_bvec(btt, NULL, page, PAGE_CACHE_SIZE, 0, rw, sector);  in btt_rw_page()
1231 static int btt_blk_init(struct btt *btt)  in btt_blk_init()  argument
1233 	struct nd_btt *nd_btt = btt->nd_btt;  in btt_blk_init()
1237 	btt->btt_queue = blk_alloc_queue(GFP_KERNEL);  in btt_blk_init()
1238 	if (!btt->btt_queue)  in btt_blk_init()
1241 	btt->btt_disk = alloc_disk(0);  in btt_blk_init()
1242 	if (!btt->btt_disk) {  in btt_blk_init()
1243 		blk_cleanup_queue(btt->btt_queue);  in btt_blk_init()
1247 	nvdimm_namespace_disk_name(ndns, btt->btt_disk->disk_name);  in btt_blk_init()
1248 	btt->btt_disk->driverfs_dev = &btt->nd_btt->dev;  in btt_blk_init()
1249 	btt->btt_disk->major = btt_major;  in btt_blk_init()
1250 	btt->btt_disk->first_minor = 0;  in btt_blk_init()
1251 	btt->btt_disk->fops = &btt_fops;  in btt_blk_init()
1252 	btt->btt_disk->private_data = btt;  in btt_blk_init()
1253 	btt->btt_disk->queue = btt->btt_queue;  in btt_blk_init()
1254 	btt->btt_disk->flags = GENHD_FL_EXT_DEVT;  in btt_blk_init()
1256 	blk_queue_make_request(btt->btt_queue, btt_make_request);  in btt_blk_init()
1257 	blk_queue_logical_block_size(btt->btt_queue, btt->sector_size);  in btt_blk_init()
1258 	blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX);  in btt_blk_init()
1259 	blk_queue_bounce_limit(btt->btt_queue, BLK_BOUNCE_ANY);  in btt_blk_init()
1260 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, btt->btt_queue);  in btt_blk_init()
1261 	btt->btt_queue->queuedata = btt;  in btt_blk_init()
1263 	set_capacity(btt->btt_disk, 0);  in btt_blk_init()
1264 	add_disk(btt->btt_disk);  in btt_blk_init()
1265 	if (btt_meta_size(btt)) {  in btt_blk_init()
1266 		int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt));  in btt_blk_init()
1269 			del_gendisk(btt->btt_disk);  in btt_blk_init()
1270 			put_disk(btt->btt_disk);  in btt_blk_init()
1271 			blk_cleanup_queue(btt->btt_queue);  in btt_blk_init()
1275 	set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);  in btt_blk_init()
1276 	revalidate_disk(btt->btt_disk);  in btt_blk_init()
1281 static void btt_blk_cleanup(struct btt *btt)  in btt_blk_cleanup()  argument
1283 	del_gendisk(btt->btt_disk);  in btt_blk_cleanup()
1284 	put_disk(btt->btt_disk);  in btt_blk_cleanup()
1285 	blk_cleanup_queue(btt->btt_queue);  in btt_blk_cleanup()
1305 static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,  in btt_init()
1309 	struct btt *btt;  in btt_init()  local
1312 	btt = kzalloc(sizeof(struct btt), GFP_KERNEL);  in btt_init()
1313 	if (!btt)  in btt_init()
1316 	btt->nd_btt = nd_btt;  in btt_init()
1317 	btt->rawsize = rawsize;  in btt_init()
1318 	btt->lbasize = lbasize;  in btt_init()
1319 	btt->sector_size = ((lbasize >= 4096) ? 4096 : 512);  in btt_init()
1320 	INIT_LIST_HEAD(&btt->arena_list);  in btt_init()
1321 	mutex_init(&btt->init_lock);  in btt_init()
1322 	btt->nd_region = nd_region;  in btt_init()
1324 	ret = discover_arenas(btt);  in btt_init()
1330 	if (btt->init_state != INIT_READY && nd_region->ro) {  in btt_init()
1334 	} else if (btt->init_state != INIT_READY) {  in btt_init()
1335 		btt->num_arenas = (rawsize / ARENA_MAX_SIZE) +  in btt_init()
1338 				btt->num_arenas, rawsize);  in btt_init()
1340 		ret = create_arenas(btt);  in btt_init()
1346 		ret = btt_meta_init(btt);  in btt_init()
1353 	ret = btt_blk_init(btt);  in btt_init()
1359 	btt_debugfs_init(btt);  in btt_init()
1361 	return btt;  in btt_init()
1364 	kfree(btt);  in btt_init()
1377 static void btt_fini(struct btt *btt)  in btt_fini()  argument
1379 	if (btt) {  in btt_fini()
1380 		btt_blk_cleanup(btt);  in btt_fini()
1381 		free_arenas(btt);  in btt_fini()
1382 		debugfs_remove_recursive(btt->debugfs_dir);  in btt_fini()
1383 		kfree(btt);  in btt_fini()
1391 	struct btt *btt;  in nvdimm_namespace_attach_btt()  local
1402 	btt = btt_init(nd_btt, rawsize, nd_btt->lbasize, nd_btt->uuid,  in nvdimm_namespace_attach_btt()
1404 	if (!btt)  in nvdimm_namespace_attach_btt()
1406 	nd_btt->btt = btt;  in nvdimm_namespace_attach_btt()
1415 	struct btt *btt = nd_btt->btt;  in nvdimm_namespace_detach_btt()  local
1417 	btt_fini(btt);  in nvdimm_namespace_detach_btt()
1418 	nd_btt->btt = NULL;  in nvdimm_namespace_detach_btt()