172 static int add_bop(struct sm_metadata *smm, enum block_op_type type, dm_block_t b)  in add_bop()  argument
174 	int r = brb_push(&smm->uncommitted, type, b);  in add_bop()
184 static int commit_bop(struct sm_metadata *smm, struct block_op *op)  in commit_bop()  argument
191 		r = sm_ll_inc(&smm->ll, op->block, &ev);  in commit_bop()
195 		r = sm_ll_dec(&smm->ll, op->block, &ev);  in commit_bop()
202 static void in(struct sm_metadata *smm)  in in()  argument
204 	smm->recursion_count++;  in in()
207 static int apply_bops(struct sm_metadata *smm)  in apply_bops()  argument
211 	while (!brb_empty(&smm->uncommitted)) {  in apply_bops()
214 		r = brb_pop(&smm->uncommitted, &bop);  in apply_bops()
220 		r = commit_bop(smm, &bop);  in apply_bops()
228 static int out(struct sm_metadata *smm)  in out()  argument
235 	if (!smm->recursion_count) {  in out()
240 	if (smm->recursion_count == 1)  in out()
241 		apply_bops(smm);  in out()
243 	smm->recursion_count--;  in out()
258 static int recursing(struct sm_metadata *smm)  in recursing()  argument
260 	return smm->recursion_count;  in recursing()
265 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_destroy()  local
267 	kfree(smm);  in sm_metadata_destroy()
272 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_blocks()  local
274 	*count = smm->ll.nr_blocks;  in sm_metadata_get_nr_blocks()
281 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_nr_free()  local
283 	*count = smm->old_ll.nr_blocks - smm->old_ll.nr_allocated -  in sm_metadata_get_nr_free()
284 		 smm->allocated_this_transaction;  in sm_metadata_get_nr_free()
294 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_get_count()  local
301 	for (i = smm->uncommitted.begin;  in sm_metadata_get_count()
302 	     i != smm->uncommitted.end;  in sm_metadata_get_count()
303 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_get_count()
304 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_get_count()
320 	r = sm_ll_lookup(&smm->ll, b, result);  in sm_metadata_get_count()
334 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_count_is_more_than_one()  local
341 	for (i = smm->uncommitted.begin;  in sm_metadata_count_is_more_than_one()
342 	     i != smm->uncommitted.end;  in sm_metadata_count_is_more_than_one()
343 	     i = brb_next(&smm->uncommitted, i)) {  in sm_metadata_count_is_more_than_one()
345 		struct block_op *op = smm->uncommitted.bops + i;  in sm_metadata_count_is_more_than_one()
366 	r = sm_ll_lookup_bitmap(&smm->ll, b, &rc);  in sm_metadata_count_is_more_than_one()
386 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_set_count()  local
388 	if (smm->recursion_count) {  in sm_metadata_set_count()
393 	in(smm);  in sm_metadata_set_count()
394 	r = sm_ll_insert(&smm->ll, b, count, &ev);  in sm_metadata_set_count()
395 	r2 = out(smm);  in sm_metadata_set_count()
404 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_inc_block()  local
406 	if (recursing(smm))  in sm_metadata_inc_block()
407 		r = add_bop(smm, BOP_INC, b);  in sm_metadata_inc_block()
409 		in(smm);  in sm_metadata_inc_block()
410 		r = sm_ll_inc(&smm->ll, b, &ev);  in sm_metadata_inc_block()
411 		r2 = out(smm);  in sm_metadata_inc_block()
421 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_dec_block()  local
423 	if (recursing(smm))  in sm_metadata_dec_block()
424 		r = add_bop(smm, BOP_DEC, b);  in sm_metadata_dec_block()
426 		in(smm);  in sm_metadata_dec_block()
427 		r = sm_ll_dec(&smm->ll, b, &ev);  in sm_metadata_dec_block()
428 		r2 = out(smm);  in sm_metadata_dec_block()
438 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block_()  local
440 	r = sm_ll_find_free_block(&smm->old_ll, smm->begin, smm->old_ll.nr_blocks, b);  in sm_metadata_new_block_()
444 	smm->begin = *b + 1;  in sm_metadata_new_block_()
446 	if (recursing(smm))  in sm_metadata_new_block_()
447 		r = add_bop(smm, BOP_INC, *b);  in sm_metadata_new_block_()
449 		in(smm);  in sm_metadata_new_block_()
450 		r = sm_ll_inc(&smm->ll, *b, &ev);  in sm_metadata_new_block_()
451 		r2 = out(smm);  in sm_metadata_new_block_()
455 		smm->allocated_this_transaction++;  in sm_metadata_new_block_()
463 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_new_block()  local
477 	check_threshold(&smm->threshold, count);  in sm_metadata_new_block()
485 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_commit()  local
487 	r = sm_ll_commit(&smm->ll);  in sm_metadata_commit()
491 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in sm_metadata_commit()
492 	smm->begin = 0;  in sm_metadata_commit()
493 	smm->allocated_this_transaction = 0;  in sm_metadata_commit()
503 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_register_threshold_callback()  local
505 	set_threshold(&smm->threshold, threshold, fn, context);  in sm_metadata_register_threshold_callback()
519 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_copy_root()  local
522 	root_le.nr_blocks = cpu_to_le64(smm->ll.nr_blocks);  in sm_metadata_copy_root()
523 	root_le.nr_allocated = cpu_to_le64(smm->ll.nr_allocated);  in sm_metadata_copy_root()
524 	root_le.bitmap_root = cpu_to_le64(smm->ll.bitmap_root);  in sm_metadata_copy_root()
525 	root_le.ref_count_root = cpu_to_le64(smm->ll.ref_count_root);  in sm_metadata_copy_root()
573 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_blocks()  local
575 	*count = smm->ll.nr_blocks;  in sm_bootstrap_get_nr_blocks()
582 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_nr_free()  local
584 	*count = smm->ll.nr_blocks - smm->begin;  in sm_bootstrap_get_nr_free()
592 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_get_count()  local
594 	*result = (b < smm->begin) ? 1 : 0;  in sm_bootstrap_get_count()
617 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_new_block()  local
622 	if (smm->begin == smm->ll.nr_blocks)  in sm_bootstrap_new_block()
625 	*b = smm->begin++;  in sm_bootstrap_new_block()
632 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_inc_block()  local
634 	return add_bop(smm, BOP_INC, b);  in sm_bootstrap_inc_block()
639 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_bootstrap_dec_block()  local
641 	return add_bop(smm, BOP_DEC, b);  in sm_bootstrap_dec_block()
687 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in sm_metadata_extend()  local
688 	dm_block_t old_len = smm->ll.nr_blocks;  in sm_metadata_extend()
693 	smm->begin = old_len;  in sm_metadata_extend()
699 	r = sm_ll_extend(&smm->ll, extra_blocks);  in sm_metadata_extend()
708 		for (i = old_len; !r && i < smm->begin; i++) {  in sm_metadata_extend()
709 			r = sm_ll_inc(&smm->ll, i, &ev);  in sm_metadata_extend()
713 		old_len = smm->begin;  in sm_metadata_extend()
715 		r = apply_bops(smm);  in sm_metadata_extend()
721 		r = sm_ll_commit(&smm->ll);  in sm_metadata_extend()
725 	} while (old_len != smm->begin);  in sm_metadata_extend()
739 	struct sm_metadata *smm;  in dm_sm_metadata_init()  local
741 	smm = kmalloc(sizeof(*smm), GFP_KERNEL);  in dm_sm_metadata_init()
742 	if (!smm)  in dm_sm_metadata_init()
745 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_init()
747 	return &smm->sm;  in dm_sm_metadata_init()
758 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_create()  local
760 	smm->begin = superblock + 1;  in dm_sm_metadata_create()
761 	smm->recursion_count = 0;  in dm_sm_metadata_create()
762 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_create()
763 	brb_init(&smm->uncommitted);  in dm_sm_metadata_create()
764 	threshold_init(&smm->threshold);  in dm_sm_metadata_create()
766 	memcpy(&smm->sm, &bootstrap_ops, sizeof(smm->sm));  in dm_sm_metadata_create()
768 	r = sm_ll_new_metadata(&smm->ll, tm);  in dm_sm_metadata_create()
774 	r = sm_ll_extend(&smm->ll, nr_blocks);  in dm_sm_metadata_create()
778 	memcpy(&smm->sm, &ops, sizeof(smm->sm));  in dm_sm_metadata_create()
784 	for (i = superblock; !r && i < smm->begin; i++)  in dm_sm_metadata_create()
785 		r = sm_ll_inc(&smm->ll, i, &ev);  in dm_sm_metadata_create()
790 	r = apply_bops(smm);  in dm_sm_metadata_create()
804 	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);  in dm_sm_metadata_open()  local
806 	r = sm_ll_open_metadata(&smm->ll, tm, root_le, len);  in dm_sm_metadata_open()
810 	smm->begin = 0;  in dm_sm_metadata_open()
811 	smm->recursion_count = 0;  in dm_sm_metadata_open()
812 	smm->allocated_this_transaction = 0;  in dm_sm_metadata_open()
813 	brb_init(&smm->uncommitted);  in dm_sm_metadata_open()
814 	threshold_init(&smm->threshold);  in dm_sm_metadata_open()
816 	memcpy(&smm->old_ll, &smm->ll, sizeof(smm->old_ll));  in dm_sm_metadata_open()