180 static int add_bop(struct sm_metadata *smm, enum block_op_type type, dm_block_t b)  in add_bop()  argument
182 	int r = brb_push(&smm->uncommitted, type, b);  in add_bop()
192 static int commit_bop(struct sm_metadata *smm, struct block_op *op)  in commit_bop()  argument
199 		r = sm_ll_inc(&smm->ll, op->block, &ev);  in commit_bop()
203 		r = sm_ll_dec(&smm->ll, op->block, &ev);  in commit_bop()
210 static void in(struct sm_metadata *smm)  in in()  argument
212 	smm->recursion_count++;  in in()
215 static int apply_bops(struct sm_metadata *smm)  in apply_bops()  argument
219 	while (!brb_empty(&smm->uncommitted)) {  in apply_bops()
222 		r = brb_peek(&smm->uncommitted, &bop);  in apply_bops()
228 		r = commit_bop(smm, &bop);  in apply_bops()
232 		brb_pop(&smm->uncommitted);  in apply_bops()
238 static int out(struct sm_metadata *smm)  in out()  argument
245 	if (!smm->recursion_count) {  in out()
250 	if (smm->recursion_count == 1)  in out()
251 		apply_bops(smm);  in out()
253 	smm->recursion_count--;  in out()
268 static int recursing(struct sm_metadata *smm)  in recursing()  argument
270 	return smm->recursion_count;  in recursing()
275 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_destroy()  local
277 	kfree(smm);  in sm_metadata_destroy()
282 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_blocks()  local
284 	*count = smm->ll.nr_blocks;  in sm_metadata_get_nr_blocks()
291 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_free()  local
293 	*count = smm->old_ll.nr_blocks - smm->old_ll.nr_allocated -  in sm_metadata_get_nr_free()
294 		 smm->allocated_this_transaction;  in sm_metadata_get_nr_free()
304 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_count()  local
311 	for (i = smm->uncommitted.begin;  in sm_metadata_get_count()
312 	     i != smm->uncommitted.end;  in sm_metadata_get_count()
313 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_get_count()
314 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_get_count()
330 	r = sm_ll_lookup(&smm->ll, b, result);  in sm_metadata_get_count()
344 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_count_is_more_than_one()  local
351 	for (i = smm->uncommitted.begin;  in sm_metadata_count_is_more_than_one()
352 	     i != smm->uncommitted.end;  in sm_metadata_count_is_more_than_one()
353 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_count_is_more_than_one()
355 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_count_is_more_than_one()
376 	r = sm_ll_lookup_bitmap(&smm->ll, b, &rc);  in sm_metadata_count_is_more_than_one()
396 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_set_count()  local
398 	if (smm->recursion_count) {  in sm_metadata_set_count()
403 	in(smm);  in sm_metadata_set_count()
404 	r = sm_ll_insert(&smm->ll, b, count, &ev);  in sm_metadata_set_count()
405 	r2 = out(smm);  in sm_metadata_set_count()
414 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_inc_block()  local
416 	if (recursing(smm))  in sm_metadata_inc_block()
417 		r = add_bop(smm, BOP_INC, b);  in sm_metadata_inc_block()
419 		in(smm);  in sm_metadata_inc_block()
420 		r = sm_ll_inc(&smm->ll, b, &ev);  in sm_metadata_inc_block()
421 		r2 = out(smm);  in sm_metadata_inc_block()
431 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_dec_block()  local
433 	if (recursing(smm))  in sm_metadata_dec_block()
434 		r = add_bop(smm, BOP_DEC, b);  in sm_metadata_dec_block()
436 		in(smm);  in sm_metadata_dec_block()
437 		r = sm_ll_dec(&smm->ll, b, &ev);  in sm_metadata_dec_block()
438 		r2 = out(smm);  in sm_metadata_dec_block()
448 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block_()  local
450 	r = sm_ll_find_free_block(&smm->old_ll, smm->begin, smm->old_ll.nr_blocks, b);  in sm_metadata_new_block_()
454 	smm->begin = *b + 1;  in sm_metadata_new_block_()
456 	if (recursing(smm))  in sm_metadata_new_block_()
457 		r = add_bop(smm, BOP_INC, *b);  in sm_metadata_new_block_()
459 		in(smm);  in sm_metadata_new_block_()
460 		r = sm_ll_inc(&smm->ll, *b, &ev);  in sm_metadata_new_block_()
461 		r2 = out(smm);  in sm_metadata_new_block_()
465 		smm->allocated_this_transaction++;  in sm_metadata_new_block_()
473 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block()  local
487 	check_threshold(&smm->threshold, count);  in sm_metadata_new_block()
495 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_commit()  local
497 	r = sm_ll_commit(&smm->ll);  in sm_metadata_commit()
501 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in sm_metadata_commit()
502 	smm->begin = 0;  in sm_metadata_commit()
503 	smm->allocated_this_transaction = 0;  in sm_metadata_commit()
513 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_register_threshold_callback()  local
515 	set_threshold(&smm->threshold, threshold, fn, context);  in sm_metadata_register_threshold_callback()
529 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_copy_root()  local
532 	root_le.nr_blocks = cpu_to_le64(smm->ll.nr_blocks);  in sm_metadata_copy_root()
533 	root_le.nr_allocated = cpu_to_le64(smm->ll.nr_allocated);  in sm_metadata_copy_root()
534 	root_le.bitmap_root = cpu_to_le64(smm->ll.bitmap_root);  in sm_metadata_copy_root()
535 	root_le.ref_count_root = cpu_to_le64(smm->ll.ref_count_root);  in sm_metadata_copy_root()
583 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_blocks()  local
585 	*count = smm->ll.nr_blocks;  in sm_bootstrap_get_nr_blocks()
592 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_free()  local
594 	*count = smm->ll.nr_blocks - smm->begin;  in sm_bootstrap_get_nr_free()
602 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_count()  local
604 	*result = (b < smm->begin) ? 1 : 0;  in sm_bootstrap_get_count()
627 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_new_block()  local
632 	if (smm->begin == smm->ll.nr_blocks)  in sm_bootstrap_new_block()
635 	*b = smm->begin++;  in sm_bootstrap_new_block()
642 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_inc_block()  local
644 	return add_bop(smm, BOP_INC, b);  in sm_bootstrap_inc_block()
649 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_dec_block()  local
651 	return add_bop(smm, BOP_DEC, b);  in sm_bootstrap_dec_block()
696 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_extend()  local
697 	dm_block_t old_len = smm->ll.nr_blocks;  in sm_metadata_extend()
702 	smm->begin = old_len;  in sm_metadata_extend()
708 	r = sm_ll_extend(&smm->ll, extra_blocks);  in sm_metadata_extend()
717 		for (i = old_len; !r && i < smm->begin; i++)  in sm_metadata_extend()
718 			r = add_bop(smm, BOP_INC, i);  in sm_metadata_extend()
723 		old_len = smm->begin;  in sm_metadata_extend()
725 		r = apply_bops(smm);  in sm_metadata_extend()
731 		r = sm_ll_commit(&smm->ll);  in sm_metadata_extend()
735 	} while (old_len != smm->begin);  in sm_metadata_extend()
749 	struct sm_metadata *smm;  in dm_sm_metadata_init()  local
751 	smm = kmalloc(sizeof(*smm), GFP_KERNEL);  in dm_sm_metadata_init()
752 	if (!smm)  in dm_sm_metadata_init()
755 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_init()
757 	return &smm->sm;  in dm_sm_metadata_init()
767 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_create()  local
769 	smm->begin = superblock + 1;  in dm_sm_metadata_create()
770 	smm->recursion_count = 0;  in dm_sm_metadata_create()
771 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_create()
772 	brb_init(&smm->uncommitted);  in dm_sm_metadata_create()
773 	threshold_init(&smm->threshold);  in dm_sm_metadata_create()
775 	memcpy(&smm->sm, &bootstrap_ops, sizeof(smm->sm));  in dm_sm_metadata_create()
777 	r = sm_ll_new_metadata(&smm->ll, tm);  in dm_sm_metadata_create()
783 	r = sm_ll_extend(&smm->ll, nr_blocks);  in dm_sm_metadata_create()
787 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_create()
793 	for (i = superblock; !r && i < smm->begin; i++)  in dm_sm_metadata_create()
794 		r = add_bop(smm, BOP_INC, i);  in dm_sm_metadata_create()
799 	r = apply_bops(smm);  in dm_sm_metadata_create()
813 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_open()  local
815 	r = sm_ll_open_metadata(&smm->ll, tm, root_le, len);  in dm_sm_metadata_open()
819 	smm->begin = 0;  in dm_sm_metadata_open()
820 	smm->recursion_count = 0;  in dm_sm_metadata_open()
821 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_open()
822 	brb_init(&smm->uncommitted);  in dm_sm_metadata_open()
823 	threshold_init(&smm->threshold);  in dm_sm_metadata_open()
825 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in dm_sm_metadata_open()