668 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_mark_free_simple()  local
693 				     buddy + sbi->s_mb_offsets[min]);  in ext4_mb_mark_free_simple()
726 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_generate_buddy()  local
765 			percpu_counter_sub(&sbi->s_freeclusters_counter,  in ext4_mb_generate_buddy()
1122 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_load_buddy()  local
1123 	struct inode *inode = sbi->s_buddy_cache;  in ext4_mb_load_buddy()
1453 		struct ext4_sb_info *sbi = EXT4_SB(sb);  in mb_free_blocks()  local
1465 			percpu_counter_sub(&sbi->s_freeclusters_counter,  in mb_free_blocks()
1637 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()  local
1668 		spin_lock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1669 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
1670 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
1671 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_use_best_found()
1683 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()  local
1694 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
1706 	if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
1823 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()  local
1846 	if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {  in ext4_mb_find_by_goal()
1852 		if (do_div(start, sbi->s_stripe) == 0) {  in ext4_mb_find_by_goal()
1992 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_scan_aligned()  local
2000 	BUG_ON(sbi->s_stripe == 0);  in ext4_mb_scan_aligned()
2005 	a = first_group_block + sbi->s_stripe - 1;  in ext4_mb_scan_aligned()
2006 	do_div(a, sbi->s_stripe);  in ext4_mb_scan_aligned()
2007 	i = (a * sbi->s_stripe) - first_group_block;  in ext4_mb_scan_aligned()
2011 			max = mb_find_extent(e4b, i, sbi->s_stripe, &ex);  in ext4_mb_scan_aligned()
2012 			if (max >= sbi->s_stripe) {  in ext4_mb_scan_aligned()
2020 		i += sbi->s_stripe;  in ext4_mb_scan_aligned()
2100 	struct ext4_sb_info *sbi;  in ext4_mb_regular_allocator()  local
2105 	sbi = EXT4_SB(sb);  in ext4_mb_regular_allocator()
2109 		ngroups = sbi->s_blockfile_groups;  in ext4_mb_regular_allocator()
2133 	if (i >= sbi->s_mb_order2_reqs) {  in ext4_mb_regular_allocator()
2144 		spin_lock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2145 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2146 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2147 		spin_unlock(&sbi->s_md_lock);  in ext4_mb_regular_allocator()
2205 			else if (cr == 1 && sbi->s_stripe &&  in ext4_mb_regular_allocator()
2206 					!(ac->ac_g_ex.fe_len % sbi->s_stripe))  in ext4_mb_regular_allocator()
2240 			atomic_inc(&sbi->s_mb_lost_chunks);  in ext4_mb_regular_allocator()
2368 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_alloc_groupinfo()  local
2374 	if (size <= sbi->s_group_info_size)  in ext4_mb_alloc_groupinfo()
2377 	size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size);  in ext4_mb_alloc_groupinfo()
2383 	if (sbi->s_group_info) {  in ext4_mb_alloc_groupinfo()
2384 		memcpy(new_groupinfo, sbi->s_group_info,  in ext4_mb_alloc_groupinfo()
2385 		       sbi->s_group_info_size * sizeof(*sbi->s_group_info));  in ext4_mb_alloc_groupinfo()
2386 		kvfree(sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2388 	sbi->s_group_info = new_groupinfo;  in ext4_mb_alloc_groupinfo()
2389 	sbi->s_group_info_size = size / sizeof(*sbi->s_group_info);  in ext4_mb_alloc_groupinfo()
2391 		   sbi->s_group_info_size);  in ext4_mb_alloc_groupinfo()
2401 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_add_groupinfo()  local
2419 		sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] =  in ext4_mb_add_groupinfo()
2424 		sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)];  in ext4_mb_add_groupinfo()
2471 		kfree(sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]);  in ext4_mb_add_groupinfo()
2472 		sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = NULL;  in ext4_mb_add_groupinfo()
2482 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_init_backend()  local
2491 	sbi->s_buddy_cache = new_inode(sb);  in ext4_mb_init_backend()
2492 	if (sbi->s_buddy_cache == NULL) {  in ext4_mb_init_backend()
2500 	sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;  in ext4_mb_init_backend()
2501 	EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;  in ext4_mb_init_backend()
2518 	i = sbi->s_group_info_size;  in ext4_mb_init_backend()
2520 		kfree(sbi->s_group_info[i]);  in ext4_mb_init_backend()
2521 	iput(sbi->s_buddy_cache);  in ext4_mb_init_backend()
2523 	kvfree(sbi->s_group_info);  in ext4_mb_init_backend()
2579 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_init()  local
2585 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets);  in ext4_mb_init()
2587 	sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2588 	if (sbi->s_mb_offsets == NULL) {  in ext4_mb_init()
2593 	i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs);  in ext4_mb_init()
2594 	sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);  in ext4_mb_init()
2595 	if (sbi->s_mb_maxs == NULL) {  in ext4_mb_init()
2605 	sbi->s_mb_maxs[0] = sb->s_blocksize << 3;  in ext4_mb_init()
2606 	sbi->s_mb_offsets[0] = 0;  in ext4_mb_init()
2613 		sbi->s_mb_offsets[i] = offset;  in ext4_mb_init()
2614 		sbi->s_mb_maxs[i] = max;  in ext4_mb_init()
2621 	spin_lock_init(&sbi->s_md_lock);  in ext4_mb_init()
2622 	spin_lock_init(&sbi->s_bal_lock);  in ext4_mb_init()
2624 	sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;  in ext4_mb_init()
2625 	sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;  in ext4_mb_init()
2626 	sbi->s_mb_stats = MB_DEFAULT_STATS;  in ext4_mb_init()
2627 	sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;  in ext4_mb_init()
2628 	sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;  in ext4_mb_init()
2641 	sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>  in ext4_mb_init()
2642 				       sbi->s_cluster_bits, 32);  in ext4_mb_init()
2651 	if (sbi->s_stripe > 1) {  in ext4_mb_init()
2652 		sbi->s_mb_group_prealloc = roundup(  in ext4_mb_init()
2653 			sbi->s_mb_group_prealloc, sbi->s_stripe);  in ext4_mb_init()
2656 	sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);  in ext4_mb_init()
2657 	if (sbi->s_locality_groups == NULL) {  in ext4_mb_init()
2663 		lg = per_cpu_ptr(sbi->s_locality_groups, i);  in ext4_mb_init()
2678 	free_percpu(sbi->s_locality_groups);  in ext4_mb_init()
2679 	sbi->s_locality_groups = NULL;  in ext4_mb_init()
2681 	kfree(sbi->s_mb_offsets);  in ext4_mb_init()
2682 	sbi->s_mb_offsets = NULL;  in ext4_mb_init()
2683 	kfree(sbi->s_mb_maxs);  in ext4_mb_init()
2684 	sbi->s_mb_maxs = NULL;  in ext4_mb_init()
2712 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_release()  local
2715 	if (sbi->s_group_info) {  in ext4_mb_release()
2730 			kfree(sbi->s_group_info[i]);  in ext4_mb_release()
2731 		kvfree(sbi->s_group_info);  in ext4_mb_release()
2733 	kfree(sbi->s_mb_offsets);  in ext4_mb_release()
2734 	kfree(sbi->s_mb_maxs);  in ext4_mb_release()
2735 	iput(sbi->s_buddy_cache);  in ext4_mb_release()
2736 	if (sbi->s_mb_stats) {  in ext4_mb_release()
2739 				atomic_read(&sbi->s_bal_allocated),  in ext4_mb_release()
2740 				atomic_read(&sbi->s_bal_reqs),  in ext4_mb_release()
2741 				atomic_read(&sbi->s_bal_success));  in ext4_mb_release()
2745 				atomic_read(&sbi->s_bal_ex_scanned),  in ext4_mb_release()
2746 				atomic_read(&sbi->s_bal_goals),  in ext4_mb_release()
2747 				atomic_read(&sbi->s_bal_2orders),  in ext4_mb_release()
2748 				atomic_read(&sbi->s_bal_breaks),  in ext4_mb_release()
2749 				atomic_read(&sbi->s_mb_lost_chunks));  in ext4_mb_release()
2752 				sbi->s_mb_buddies_generated,  in ext4_mb_release()
2753 				sbi->s_mb_generation_time);  in ext4_mb_release()
2756 				atomic_read(&sbi->s_mb_preallocated),  in ext4_mb_release()
2757 				atomic_read(&sbi->s_mb_discarded));  in ext4_mb_release()
2760 	free_percpu(sbi->s_locality_groups);  in ext4_mb_release()
2892 	struct ext4_sb_info *sbi;  in ext4_mb_mark_diskspace_used()  local
2901 	sbi = EXT4_SB(sb);  in ext4_mb_mark_diskspace_used()
2930 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2931 	if (!ext4_data_block_valid(sbi, block, len)) {  in ext4_mb_mark_diskspace_used()
2972 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2978 		percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_mark_diskspace_used()
2981 	if (sbi->s_log_groups_per_flex) {  in ext4_mb_mark_diskspace_used()
2982 		ext4_group_t flex_group = ext4_flex_group(sbi,  in ext4_mb_mark_diskspace_used()
2985 			     &sbi->s_flex_groups[flex_group].free_clusters);  in ext4_mb_mark_diskspace_used()
3026 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()  local
3058 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_normalize_request()
3188 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
3212 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()  local
3214 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {  in ext4_mb_collect_stats()
3215 		atomic_inc(&sbi->s_bal_reqs);  in ext4_mb_collect_stats()
3216 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
3218 			atomic_inc(&sbi->s_bal_success);  in ext4_mb_collect_stats()
3219 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
3222 			atomic_inc(&sbi->s_bal_goals);  in ext4_mb_collect_stats()
3223 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
3224 			atomic_inc(&sbi->s_bal_breaks);  in ext4_mb_collect_stats()
3275 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()  local
3282 	end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),  in ext4_mb_use_inode_pa()
3283 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
3284 	len = EXT4_NUM_B2C(sbi, end - start);  in ext4_mb_use_inode_pa()
3292 	BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));  in ext4_mb_use_inode_pa()
3358 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()  local
3377 					       EXT4_C2B(sbi, pa->pa_len)))  in ext4_mb_use_preallocated()
3382 		    (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) >  in ext4_mb_use_preallocated()
3583 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_new_inode_pa()  local
3615 		wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3621 			EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3626 			EXT4_NUM_B2C(sbi, win);  in ext4_mb_new_inode_pa()
3651 	atomic_add(pa->pa_free, &sbi->s_mb_preallocated);  in ext4_mb_new_inode_pa()
3755 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_release_inode_pa()  local
3766 	grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit);  in ext4_mb_release_inode_pa()
3782 						    EXT4_C2B(sbi, bit)),  in ext4_mb_release_inode_pa()
3800 	atomic_add(free, &sbi->s_mb_discarded);  in ext4_mb_release_inode_pa()
4121 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()  local
4131 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_group_or_file()
4136 	    !ext4_fs_is_busy(sbi) &&  in ext4_mb_group_or_file()
4142 	if (sbi->s_mb_group_prealloc <= 0) {  in ext4_mb_group_or_file()
4149 	if (size > sbi->s_mb_stream_request) {  in ext4_mb_group_or_file()
4160 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
4174 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_initialize_context()  local
4175 	struct ext4_super_block *es = sbi->s_es;  in ext4_mb_initialize_context()
4196 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
4356 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()  local
4362 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4363 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4420 	struct ext4_sb_info *sbi;  in ext4_mb_new_blocks()  local
4428 	sbi = EXT4_SB(sb);  in ext4_mb_new_blocks()
4442 			ext4_claim_free_clusters(sbi, ar->len, ar->flags)) {  in ext4_mb_new_blocks()
4455 						 EXT4_C2B(sbi, ar->len));  in ext4_mb_new_blocks()
4459 						  EXT4_C2B(sbi, ar->len))) {  in ext4_mb_new_blocks()
4545 		dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len));  in ext4_mb_new_blocks()
4549 			percpu_counter_sub(&sbi->s_dirtyclusters_counter,  in ext4_mb_new_blocks()
4582 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_mb_free_metadata()  local
4612 				EXT4_C2B(sbi, cluster),  in ext4_mb_free_metadata()
4669 	struct ext4_sb_info *sbi;  in ext4_free_blocks()  local
4683 	sbi = EXT4_SB(sb);  in ext4_free_blocks()
4685 	    !ext4_data_block_valid(sbi, block, count)) {  in ext4_free_blocks()
4718 	overflow = EXT4_PBLK_COFF(sbi, block);  in ext4_free_blocks()
4721 			overflow = sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
4732 	overflow = EXT4_LBLK_COFF(sbi, count);  in ext4_free_blocks()
4740 			count += sbi->s_cluster_ratio - overflow;  in ext4_free_blocks()
4768 	if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) {  in ext4_free_blocks()
4769 		overflow = EXT4_C2B(sbi, bit) + count -  in ext4_free_blocks()
4773 	count_clusters = EXT4_NUM_B2C(sbi, count);  in ext4_free_blocks()
4871 	if (sbi->s_log_groups_per_flex) {  in ext4_free_blocks()
4872 		ext4_group_t flex_group = ext4_flex_group(sbi, block_group);  in ext4_free_blocks()
4874 			     &sbi->s_flex_groups[flex_group].free_clusters);  in ext4_free_blocks()
4878 		dquot_free_block(inode, EXT4_C2B(sbi, count_clusters));  in ext4_free_blocks()
4879 	percpu_counter_add(&sbi->s_freeclusters_counter, count_clusters);  in ext4_free_blocks()
4923 	struct ext4_sb_info *sbi = EXT4_SB(sb);  in ext4_group_add_blocks()  local
4960 	    in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) ||  in ext4_group_add_blocks()
4962 		     sbi->s_itb_per_group)) {  in ext4_group_add_blocks()
5013 	percpu_counter_add(&sbi->s_freeclusters_counter,  in ext4_group_add_blocks()
5014 			   EXT4_NUM_B2C(sbi, blocks_freed));  in ext4_group_add_blocks()
5016 	if (sbi->s_log_groups_per_flex) {  in ext4_group_add_blocks()
5017 		ext4_group_t flex_group = ext4_flex_group(sbi, block_group);  in ext4_group_add_blocks()
5018 		atomic64_add(EXT4_NUM_B2C(sbi, blocks_freed),  in ext4_group_add_blocks()
5019 			     &sbi->s_flex_groups[flex_group].free_clusters);  in ext4_group_add_blocks()