191 static int sm_ll_init(struct ll_disk *ll, struct dm_transaction_manager *tm)  in sm_ll_init()  argument
193 	ll->tm = tm;  in sm_ll_init()
195 	ll->bitmap_info.tm = tm;  in sm_ll_init()
196 	ll->bitmap_info.levels = 1;  in sm_ll_init()
203 	ll->bitmap_info.value_type.size = sizeof(struct disk_index_entry);  in sm_ll_init()
204 	ll->bitmap_info.value_type.inc = NULL;  in sm_ll_init()
205 	ll->bitmap_info.value_type.dec = NULL;  in sm_ll_init()
206 	ll->bitmap_info.value_type.equal = NULL;  in sm_ll_init()
208 	ll->ref_count_info.tm = tm;  in sm_ll_init()
209 	ll->ref_count_info.levels = 1;  in sm_ll_init()
210 	ll->ref_count_info.value_type.size = sizeof(uint32_t);  in sm_ll_init()
211 	ll->ref_count_info.value_type.inc = NULL;  in sm_ll_init()
212 	ll->ref_count_info.value_type.dec = NULL;  in sm_ll_init()
213 	ll->ref_count_info.value_type.equal = NULL;  in sm_ll_init()
215 	ll->block_size = dm_bm_block_size(dm_tm_get_bm(tm));  in sm_ll_init()
217 	if (ll->block_size > (1 << 30)) {  in sm_ll_init()
222 	ll->entries_per_block = (ll->block_size - sizeof(struct disk_bitmap_header)) *  in sm_ll_init()
224 	ll->nr_blocks = 0;  in sm_ll_init()
225 	ll->bitmap_root = 0;  in sm_ll_init()
226 	ll->ref_count_root = 0;  in sm_ll_init()
227 	ll->bitmap_index_changed = false;  in sm_ll_init()
232 int sm_ll_extend(struct ll_disk *ll, dm_block_t extra_blocks)  in sm_ll_extend()  argument
238 	nr_blocks = ll->nr_blocks + extra_blocks;  in sm_ll_extend()
239 	old_blocks = dm_sector_div_up(ll->nr_blocks, ll->entries_per_block);  in sm_ll_extend()
240 	blocks = dm_sector_div_up(nr_blocks, ll->entries_per_block);  in sm_ll_extend()
242 	nr_indexes = dm_sector_div_up(nr_blocks, ll->entries_per_block);  in sm_ll_extend()
243 	if (nr_indexes > ll->max_entries(ll)) {  in sm_ll_extend()
251 	ll->nr_blocks = nr_blocks;  in sm_ll_extend()
256 		r = dm_tm_new_block(ll->tm, &dm_sm_bitmap_validator, &b);  in sm_ll_extend()
262 		r = dm_tm_unlock(ll->tm, b);  in sm_ll_extend()
266 		idx.nr_free = cpu_to_le32(ll->entries_per_block);  in sm_ll_extend()
269 		r = ll->save_ie(ll, i, &idx);  in sm_ll_extend()
277 int sm_ll_lookup_bitmap(struct ll_disk *ll, dm_block_t b, uint32_t *result)  in sm_ll_lookup_bitmap()  argument
284 	b = do_div(index, ll->entries_per_block);  in sm_ll_lookup_bitmap()
285 	r = ll->load_ie(ll, index, &ie_disk);  in sm_ll_lookup_bitmap()
289 	r = dm_tm_read_lock(ll->tm, le64_to_cpu(ie_disk.blocknr),  in sm_ll_lookup_bitmap()
296 	return dm_tm_unlock(ll->tm, blk);  in sm_ll_lookup_bitmap()
299 static int sm_ll_lookup_big_ref_count(struct ll_disk *ll, dm_block_t b,  in sm_ll_lookup_big_ref_count()  argument
305 	r = dm_btree_lookup(&ll->ref_count_info, ll->ref_count_root, &b, &le_rc);  in sm_ll_lookup_big_ref_count()
314 int sm_ll_lookup(struct ll_disk *ll, dm_block_t b, uint32_t *result)  in sm_ll_lookup()  argument
316 	int r = sm_ll_lookup_bitmap(ll, b, result);  in sm_ll_lookup()
324 	return sm_ll_lookup_big_ref_count(ll, b, result);  in sm_ll_lookup()
327 int sm_ll_find_free_block(struct ll_disk *ll, dm_block_t begin,  in sm_ll_find_free_block()  argument
333 	dm_block_t index_end = dm_sector_div_up(end, ll->entries_per_block);  in sm_ll_find_free_block()
338 	begin = do_div(index_begin, ll->entries_per_block);  in sm_ll_find_free_block()
339 	end = do_div(end, ll->entries_per_block);  in sm_ll_find_free_block()
346 		r = ll->load_ie(ll, i, &ie_disk);  in sm_ll_find_free_block()
353 		r = dm_tm_read_lock(ll->tm, le64_to_cpu(ie_disk.blocknr),  in sm_ll_find_free_block()
358 		bit_end = (i == index_end - 1) ?  end : ll->entries_per_block;  in sm_ll_find_free_block()
368 			dm_tm_unlock(ll->tm, blk);  in sm_ll_find_free_block()
372 			dm_tm_unlock(ll->tm, blk);  in sm_ll_find_free_block()
376 		r = dm_tm_unlock(ll->tm, blk);  in sm_ll_find_free_block()
380 		*result = i * ll->entries_per_block + (dm_block_t) position;  in sm_ll_find_free_block()
387 static int sm_ll_mutate(struct ll_disk *ll, dm_block_t b,  in sm_ll_mutate()  argument
399 	bit = do_div(index, ll->entries_per_block);  in sm_ll_mutate()
400 	r = ll->load_ie(ll, index, &ie_disk);  in sm_ll_mutate()
404 	r = dm_tm_shadow_block(ll->tm, le64_to_cpu(ie_disk.blocknr),  in sm_ll_mutate()
416 		r = sm_ll_lookup_big_ref_count(ll, b, &old);  in sm_ll_mutate()
418 			dm_tm_unlock(ll->tm, nb);  in sm_ll_mutate()
425 		dm_tm_unlock(ll->tm, nb);  in sm_ll_mutate()
432 		r = dm_tm_unlock(ll->tm, nb);  in sm_ll_mutate()
437 			r = dm_btree_remove(&ll->ref_count_info,  in sm_ll_mutate()
438 					    ll->ref_count_root,  in sm_ll_mutate()
439 					    &b, &ll->ref_count_root);  in sm_ll_mutate()
448 		r = dm_tm_unlock(ll->tm, nb);  in sm_ll_mutate()
453 		r = dm_btree_insert(&ll->ref_count_info, ll->ref_count_root,  in sm_ll_mutate()
454 				    &b, &le_rc, &ll->ref_count_root);  in sm_ll_mutate()
463 		ll->nr_allocated++;  in sm_ll_mutate()
470 		ll->nr_allocated--;  in sm_ll_mutate()
475 	return ll->save_ie(ll, index, &ie_disk);  in sm_ll_mutate()
484 int sm_ll_insert(struct ll_disk *ll, dm_block_t b,  in sm_ll_insert()  argument
487 	return sm_ll_mutate(ll, b, set_ref_count, &ref_count, ev);  in sm_ll_insert()
496 int sm_ll_inc(struct ll_disk *ll, dm_block_t b, enum allocation_event *ev)  in sm_ll_inc()  argument
498 	return sm_ll_mutate(ll, b, inc_ref_count, NULL, ev);  in sm_ll_inc()
512 int sm_ll_dec(struct ll_disk *ll, dm_block_t b, enum allocation_event *ev)  in sm_ll_dec()  argument
514 	return sm_ll_mutate(ll, b, dec_ref_count, NULL, ev);  in sm_ll_dec()
517 int sm_ll_commit(struct ll_disk *ll)  in sm_ll_commit()  argument
521 	if (ll->bitmap_index_changed) {  in sm_ll_commit()
522 		r = ll->commit(ll);  in sm_ll_commit()
524 			ll->bitmap_index_changed = false;  in sm_ll_commit()
532 static int metadata_ll_load_ie(struct ll_disk *ll, dm_block_t index,  in metadata_ll_load_ie()  argument
535 	memcpy(ie, ll->mi_le.index + index, sizeof(*ie));  in metadata_ll_load_ie()
539 static int metadata_ll_save_ie(struct ll_disk *ll, dm_block_t index,  in metadata_ll_save_ie()  argument
542 	ll->bitmap_index_changed = true;  in metadata_ll_save_ie()
543 	memcpy(ll->mi_le.index + index, ie, sizeof(*ie));  in metadata_ll_save_ie()
547 static int metadata_ll_init_index(struct ll_disk *ll)  in metadata_ll_init_index()  argument
552 	r = dm_tm_new_block(ll->tm, &index_validator, &b);  in metadata_ll_init_index()
556 	memcpy(dm_block_data(b), &ll->mi_le, sizeof(ll->mi_le));  in metadata_ll_init_index()
557 	ll->bitmap_root = dm_block_location(b);  in metadata_ll_init_index()
559 	return dm_tm_unlock(ll->tm, b);  in metadata_ll_init_index()
562 static int metadata_ll_open(struct ll_disk *ll)  in metadata_ll_open()  argument
567 	r = dm_tm_read_lock(ll->tm, ll->bitmap_root,  in metadata_ll_open()
572 	memcpy(&ll->mi_le, dm_block_data(block), sizeof(ll->mi_le));  in metadata_ll_open()
573 	return dm_tm_unlock(ll->tm, block);  in metadata_ll_open()
576 static dm_block_t metadata_ll_max_entries(struct ll_disk *ll)  in metadata_ll_max_entries()  argument
581 static int metadata_ll_commit(struct ll_disk *ll)  in metadata_ll_commit()  argument
586 	r = dm_tm_shadow_block(ll->tm, ll->bitmap_root, &index_validator, &b, &inc);  in metadata_ll_commit()
590 	memcpy(dm_block_data(b), &ll->mi_le, sizeof(ll->mi_le));  in metadata_ll_commit()
591 	ll->bitmap_root = dm_block_location(b);  in metadata_ll_commit()
593 	return dm_tm_unlock(ll->tm, b);  in metadata_ll_commit()
596 int sm_ll_new_metadata(struct ll_disk *ll, struct dm_transaction_manager *tm)  in sm_ll_new_metadata()  argument
600 	r = sm_ll_init(ll, tm);  in sm_ll_new_metadata()
604 	ll->load_ie = metadata_ll_load_ie;  in sm_ll_new_metadata()
605 	ll->save_ie = metadata_ll_save_ie;  in sm_ll_new_metadata()
606 	ll->init_index = metadata_ll_init_index;  in sm_ll_new_metadata()
607 	ll->open_index = metadata_ll_open;  in sm_ll_new_metadata()
608 	ll->max_entries = metadata_ll_max_entries;  in sm_ll_new_metadata()
609 	ll->commit = metadata_ll_commit;  in sm_ll_new_metadata()
611 	ll->nr_blocks = 0;  in sm_ll_new_metadata()
612 	ll->nr_allocated = 0;  in sm_ll_new_metadata()
614 	r = ll->init_index(ll);  in sm_ll_new_metadata()
618 	r = dm_btree_empty(&ll->ref_count_info, &ll->ref_count_root);  in sm_ll_new_metadata()
625 int sm_ll_open_metadata(struct ll_disk *ll, struct dm_transaction_manager *tm,  in sm_ll_open_metadata()  argument
636 	r = sm_ll_init(ll, tm);  in sm_ll_open_metadata()
640 	ll->load_ie = metadata_ll_load_ie;  in sm_ll_open_metadata()
641 	ll->save_ie = metadata_ll_save_ie;  in sm_ll_open_metadata()
642 	ll->init_index = metadata_ll_init_index;  in sm_ll_open_metadata()
643 	ll->open_index = metadata_ll_open;  in sm_ll_open_metadata()
644 	ll->max_entries = metadata_ll_max_entries;  in sm_ll_open_metadata()
645 	ll->commit = metadata_ll_commit;  in sm_ll_open_metadata()
647 	ll->nr_blocks = le64_to_cpu(smr->nr_blocks);  in sm_ll_open_metadata()
648 	ll->nr_allocated = le64_to_cpu(smr->nr_allocated);  in sm_ll_open_metadata()
649 	ll->bitmap_root = le64_to_cpu(smr->bitmap_root);  in sm_ll_open_metadata()
650 	ll->ref_count_root = le64_to_cpu(smr->ref_count_root);  in sm_ll_open_metadata()
652 	return ll->open_index(ll);  in sm_ll_open_metadata()
657 static int disk_ll_load_ie(struct ll_disk *ll, dm_block_t index,  in disk_ll_load_ie()  argument
660 	return dm_btree_lookup(&ll->bitmap_info, ll->bitmap_root, &index, ie);  in disk_ll_load_ie()
663 static int disk_ll_save_ie(struct ll_disk *ll, dm_block_t index,  in disk_ll_save_ie()  argument
667 	return dm_btree_insert(&ll->bitmap_info, ll->bitmap_root,  in disk_ll_save_ie()
668 			       &index, ie, &ll->bitmap_root);  in disk_ll_save_ie()
671 static int disk_ll_init_index(struct ll_disk *ll)  in disk_ll_init_index()  argument
673 	return dm_btree_empty(&ll->bitmap_info, &ll->bitmap_root);  in disk_ll_init_index()
676 static int disk_ll_open(struct ll_disk *ll)  in disk_ll_open()  argument
682 static dm_block_t disk_ll_max_entries(struct ll_disk *ll)  in disk_ll_max_entries()  argument
687 static int disk_ll_commit(struct ll_disk *ll)  in disk_ll_commit()  argument
692 int sm_ll_new_disk(struct ll_disk *ll, struct dm_transaction_manager *tm)  in sm_ll_new_disk()  argument
696 	r = sm_ll_init(ll, tm);  in sm_ll_new_disk()
700 	ll->load_ie = disk_ll_load_ie;  in sm_ll_new_disk()
701 	ll->save_ie = disk_ll_save_ie;  in sm_ll_new_disk()
702 	ll->init_index = disk_ll_init_index;  in sm_ll_new_disk()
703 	ll->open_index = disk_ll_open;  in sm_ll_new_disk()
704 	ll->max_entries = disk_ll_max_entries;  in sm_ll_new_disk()
705 	ll->commit = disk_ll_commit;  in sm_ll_new_disk()
707 	ll->nr_blocks = 0;  in sm_ll_new_disk()
708 	ll->nr_allocated = 0;  in sm_ll_new_disk()
710 	r = ll->init_index(ll);  in sm_ll_new_disk()
714 	r = dm_btree_empty(&ll->ref_count_info, &ll->ref_count_root);  in sm_ll_new_disk()
721 int sm_ll_open_disk(struct ll_disk *ll, struct dm_transaction_manager *tm,  in sm_ll_open_disk()  argument
732 	r = sm_ll_init(ll, tm);  in sm_ll_open_disk()
736 	ll->load_ie = disk_ll_load_ie;  in sm_ll_open_disk()
737 	ll->save_ie = disk_ll_save_ie;  in sm_ll_open_disk()
738 	ll->init_index = disk_ll_init_index;  in sm_ll_open_disk()
739 	ll->open_index = disk_ll_open;  in sm_ll_open_disk()
740 	ll->max_entries = disk_ll_max_entries;  in sm_ll_open_disk()
741 	ll->commit = disk_ll_commit;  in sm_ll_open_disk()
743 	ll->nr_blocks = le64_to_cpu(smr->nr_blocks);  in sm_ll_open_disk()
744 	ll->nr_allocated = le64_to_cpu(smr->nr_allocated);  in sm_ll_open_disk()
745 	ll->bitmap_root = le64_to_cpu(smr->bitmap_root);  in sm_ll_open_disk()
746 	ll->ref_count_root = le64_to_cpu(smr->ref_count_root);  in sm_ll_open_disk()
748 	return ll->open_index(ll);  in sm_ll_open_disk()