1634 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,  in ext4_mb_use_best_found()  argument
1637 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()
1640 	BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);  in ext4_mb_use_best_found()
1641 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_use_best_found()
1643 	ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);  in ext4_mb_use_best_found()
1644 	ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_use_best_found()
1645 	ret = mb_mark_used(e4b, &ac->ac_b_ex);  in ext4_mb_use_best_found()
1649 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_use_best_found()
1651 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_best_found()
1652 	ac->ac_tail = ret & 0xffff;  in ext4_mb_use_best_found()
1653 	ac->ac_buddy = ret >> 16;  in ext4_mb_use_best_found()
1662 	ac->ac_bitmap_page = e4b->bd_bitmap_page;  in ext4_mb_use_best_found()
1663 	get_page(ac->ac_bitmap_page);  in ext4_mb_use_best_found()
1664 	ac->ac_buddy_page = e4b->bd_buddy_page;  in ext4_mb_use_best_found()
1665 	get_page(ac->ac_buddy_page);  in ext4_mb_use_best_found()
1667 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_use_best_found()
1669 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
1670 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
1679 static void ext4_mb_check_limits(struct ext4_allocation_context *ac,  in ext4_mb_check_limits()  argument
1683 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()
1684 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_check_limits()
1685 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_check_limits()
1689 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_check_limits()
1694 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
1695 			!(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_check_limits()
1696 		ac->ac_status = AC_STATUS_BREAK;  in ext4_mb_check_limits()
1706 	if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
1713 			ext4_mb_use_best_found(ac, e4b);  in ext4_mb_check_limits()
1729 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,  in ext4_mb_measure_extent()  argument
1733 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_measure_extent()
1734 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_measure_extent()
1737 	BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1738 	BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1739 	BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);  in ext4_mb_measure_extent()
1741 	ac->ac_found++;  in ext4_mb_measure_extent()
1746 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_measure_extent()
1748 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_measure_extent()
1757 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_measure_extent()
1785 	ext4_mb_check_limits(ac, e4b, 0);  in ext4_mb_measure_extent()
1789 int ext4_mb_try_best_found(struct ext4_allocation_context *ac,  in ext4_mb_try_best_found()  argument
1792 	struct ext4_free_extent ex = ac->ac_b_ex;  in ext4_mb_try_best_found()
1798 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_try_best_found()
1802 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1806 		ac->ac_b_ex = ex;  in ext4_mb_try_best_found()
1807 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_try_best_found()
1810 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1817 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,  in ext4_mb_find_by_goal()  argument
1820 	ext4_group_t group = ac->ac_g_ex.fe_group;  in ext4_mb_find_by_goal()
1823 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()
1824 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1827 	if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))  in ext4_mb_find_by_goal()
1832 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_find_by_goal()
1841 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1842 	max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,  in ext4_mb_find_by_goal()
1843 			     ac->ac_g_ex.fe_len, &ex);  in ext4_mb_find_by_goal()
1846 	if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {  in ext4_mb_find_by_goal()
1849 		start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +  in ext4_mb_find_by_goal()
1853 			ac->ac_found++;  in ext4_mb_find_by_goal()
1854 			ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1855 			ext4_mb_use_best_found(ac, e4b);  in ext4_mb_find_by_goal()
1857 	} else if (max >= ac->ac_g_ex.fe_len) {  in ext4_mb_find_by_goal()
1859 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1860 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1861 		ac->ac_found++;  in ext4_mb_find_by_goal()
1862 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1863 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_find_by_goal()
1864 	} else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {  in ext4_mb_find_by_goal()
1868 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1869 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1870 		ac->ac_found++;  in ext4_mb_find_by_goal()
1871 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1872 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_find_by_goal()
1874 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1885 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,  in ext4_mb_simple_scan_group()  argument
1888 	struct super_block *sb = ac->ac_sb;  in ext4_mb_simple_scan_group()
1895 	BUG_ON(ac->ac_2order <= 0);  in ext4_mb_simple_scan_group()
1896 	for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {  in ext4_mb_simple_scan_group()
1906 		ac->ac_found++;  in ext4_mb_simple_scan_group()
1908 		ac->ac_b_ex.fe_len = 1 << i;  in ext4_mb_simple_scan_group()
1909 		ac->ac_b_ex.fe_start = k << i;  in ext4_mb_simple_scan_group()
1910 		ac->ac_b_ex.fe_group = e4b->bd_group;  in ext4_mb_simple_scan_group()
1912 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_simple_scan_group()
1914 		BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);  in ext4_mb_simple_scan_group()
1929 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,  in ext4_mb_complex_scan_group()  argument
1932 	struct super_block *sb = ac->ac_sb;  in ext4_mb_complex_scan_group()
1943 	while (free && ac->ac_status == AC_STATUS_CONTINUE) {  in ext4_mb_complex_scan_group()
1959 		mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);  in ext4_mb_complex_scan_group()
1974 		ext4_mb_measure_extent(ac, &ex, e4b);  in ext4_mb_complex_scan_group()
1980 	ext4_mb_check_limits(ac, e4b, 1);  in ext4_mb_complex_scan_group()
1988 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,  in ext4_mb_scan_aligned()  argument
1991 	struct super_block *sb = ac->ac_sb;  in ext4_mb_scan_aligned()
2013 				ac->ac_found++;  in ext4_mb_scan_aligned()
2015 				ac->ac_b_ex = ex;  in ext4_mb_scan_aligned()
2016 				ext4_mb_use_best_found(ac, e4b);  in ext4_mb_scan_aligned()
2030 static int ext4_mb_good_group(struct ext4_allocation_context *ac,  in ext4_mb_good_group()  argument
2034 	int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));  in ext4_mb_good_group()
2035 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group()
2042 	if (cr <= 2 && free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2050 		int ret = ext4_mb_init_group(ac->ac_sb, group);  in ext4_mb_good_group()
2061 		BUG_ON(ac->ac_2order == 0);  in ext4_mb_good_group()
2064 		if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&  in ext4_mb_good_group()
2069 		if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) ||  in ext4_mb_good_group()
2070 		    (free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2073 		if (grp->bb_largest_free_order < ac->ac_2order)  in ext4_mb_good_group()
2078 		if ((free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2082 		if (free >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2095 ext4_mb_regular_allocator(struct ext4_allocation_context *ac)  in ext4_mb_regular_allocator()  argument
2104 	sb = ac->ac_sb;  in ext4_mb_regular_allocator()
2108 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))  in ext4_mb_regular_allocator()
2111 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_regular_allocator()
2114 	err = ext4_mb_find_by_goal(ac, &e4b);  in ext4_mb_regular_allocator()
2115 	if (err || ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2118 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_regular_allocator()
2126 	i = fls(ac->ac_g_ex.fe_len);  in ext4_mb_regular_allocator()
2127 	ac->ac_2order = 0;  in ext4_mb_regular_allocator()
2137 		if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)  in ext4_mb_regular_allocator()
2138 			ac->ac_2order = i - 1;  in ext4_mb_regular_allocator()
2142 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_regular_allocator()
2145 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2146 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2151 	cr = ac->ac_2order ? 0 : 1;  in ext4_mb_regular_allocator()
2157 	for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {  in ext4_mb_regular_allocator()
2158 		ac->ac_criteria = cr;  in ext4_mb_regular_allocator()
2163 		group = ac->ac_g_ex.fe_group;  in ext4_mb_regular_allocator()
2176 			ret = ext4_mb_good_group(ac, group, cr);  in ext4_mb_regular_allocator()
2193 			ret = ext4_mb_good_group(ac, group, cr);  in ext4_mb_regular_allocator()
2202 			ac->ac_groups_scanned++;  in ext4_mb_regular_allocator()
2203 			if (cr == 0 && ac->ac_2order < sb->s_blocksize_bits+2)  in ext4_mb_regular_allocator()
2204 				ext4_mb_simple_scan_group(ac, &e4b);  in ext4_mb_regular_allocator()
2206 					!(ac->ac_g_ex.fe_len % sbi->s_stripe))  in ext4_mb_regular_allocator()
2207 				ext4_mb_scan_aligned(ac, &e4b);  in ext4_mb_regular_allocator()
2209 				ext4_mb_complex_scan_group(ac, &e4b);  in ext4_mb_regular_allocator()
2214 			if (ac->ac_status != AC_STATUS_CONTINUE)  in ext4_mb_regular_allocator()
2219 	if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&  in ext4_mb_regular_allocator()
2220 	    !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_regular_allocator()
2226 		ext4_mb_try_best_found(ac, &e4b);  in ext4_mb_regular_allocator()
2227 		if (ac->ac_status != AC_STATUS_FOUND) {  in ext4_mb_regular_allocator()
2234 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_regular_allocator()
2235 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_regular_allocator()
2236 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_regular_allocator()
2237 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_regular_allocator()
2238 			ac->ac_flags |= EXT4_MB_HINT_FIRST;  in ext4_mb_regular_allocator()
2245 	if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)  in ext4_mb_regular_allocator()
2886 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,  in ext4_mb_mark_diskspace_used()  argument
2897 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_mark_diskspace_used()
2898 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_mark_diskspace_used()
2900 	sb = ac->ac_sb;  in ext4_mb_mark_diskspace_used()
2903 	bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2916 	gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);  in ext4_mb_mark_diskspace_used()
2920 	ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,  in ext4_mb_mark_diskspace_used()
2928 	block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_mark_diskspace_used()
2930 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2938 		ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2939 		ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
2940 			      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2941 		ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2948 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2952 		for (i = 0; i < ac->ac_b_ex.fe_len; i++) {  in ext4_mb_mark_diskspace_used()
2953 			BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,  in ext4_mb_mark_diskspace_used()
2958 	ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
2959 		      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2964 						ac->ac_b_ex.fe_group, gdp));  in ext4_mb_mark_diskspace_used()
2966 	len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;  in ext4_mb_mark_diskspace_used()
2968 	ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh);  in ext4_mb_mark_diskspace_used()
2969 	ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);  in ext4_mb_mark_diskspace_used()
2971 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2972 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2976 	if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))  in ext4_mb_mark_diskspace_used()
2983 							  ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2984 		atomic64_sub(ac->ac_b_ex.fe_len,  in ext4_mb_mark_diskspace_used()
3007 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)  in ext4_mb_normalize_group_request()  argument
3009 	struct super_block *sb = ac->ac_sb;  in ext4_mb_normalize_group_request()
3010 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_normalize_group_request()
3013 	ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;  in ext4_mb_normalize_group_request()
3015 		current->pid, ac->ac_g_ex.fe_len);  in ext4_mb_normalize_group_request()
3023 ext4_mb_normalize_request(struct ext4_allocation_context *ac,  in ext4_mb_normalize_request()  argument
3026 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()
3032 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_normalize_request()
3037 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_normalize_request()
3041 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_normalize_request()
3046 	if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)  in ext4_mb_normalize_request()
3049 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {  in ext4_mb_normalize_request()
3050 		ext4_mb_normalize_group_request(ac);  in ext4_mb_normalize_request()
3054 	bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_normalize_request()
3058 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_normalize_request()
3060 	if (size < i_size_read(ac->ac_inode))  in ext4_mb_normalize_request()
3061 		size = i_size_read(ac->ac_inode);  in ext4_mb_normalize_request()
3088 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3092 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3095 	} else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,  in ext4_mb_normalize_request()
3097 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3101 		start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;  in ext4_mb_normalize_request()
3102 		size	  = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3103 					      ac->ac_o_ex.fe_len) << bsbits;  in ext4_mb_normalize_request()
3131 		pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3135 		BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||  in ext4_mb_normalize_request()
3136 			ac->ac_o_ex.fe_logical < pa->pa_lstart));  in ext4_mb_normalize_request()
3146 		if (pa_end <= ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3149 		} else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3165 			pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3173 	if (start + size <= ac->ac_o_ex.fe_logical &&  in ext4_mb_normalize_request()
3174 			start > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3175 		ext4_msg(ac->ac_sb, KERN_ERR,  in ext4_mb_normalize_request()
3178 			 (unsigned long) ac->ac_o_ex.fe_logical);  in ext4_mb_normalize_request()
3181 	BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));  in ext4_mb_normalize_request()
3187 	ac->ac_g_ex.fe_logical = start;  in ext4_mb_normalize_request()
3188 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
3193 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,  in ext4_mb_normalize_request()
3194 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3195 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3196 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3200 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,  in ext4_mb_normalize_request()
3201 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3202 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3203 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3210 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)  in ext4_mb_collect_stats()  argument
3212 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()
3214 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {  in ext4_mb_collect_stats()
3216 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
3217 		if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)  in ext4_mb_collect_stats()
3219 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
3220 		if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&  in ext4_mb_collect_stats()
3221 				ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)  in ext4_mb_collect_stats()
3223 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
3227 	if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)  in ext4_mb_collect_stats()
3228 		trace_ext4_mballoc_alloc(ac);  in ext4_mb_collect_stats()
3230 		trace_ext4_mballoc_prealloc(ac);  in ext4_mb_collect_stats()
3239 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac)  in ext4_discard_allocated_blocks()  argument
3241 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_discard_allocated_blocks()
3246 		if (ac->ac_f_ex.fe_len == 0)  in ext4_discard_allocated_blocks()
3248 		err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);  in ext4_discard_allocated_blocks()
3258 		ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3259 		mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,  in ext4_discard_allocated_blocks()
3260 			       ac->ac_f_ex.fe_len);  in ext4_discard_allocated_blocks()
3261 		ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3266 		pa->pa_free += ac->ac_b_ex.fe_len;  in ext4_discard_allocated_blocks()
3272 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,  in ext4_mb_use_inode_pa()  argument
3275 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()
3281 	start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);  in ext4_mb_use_inode_pa()
3283 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
3285 	ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,  in ext4_mb_use_inode_pa()
3286 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_inode_pa()
3287 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_inode_pa()
3288 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_inode_pa()
3289 	ac->ac_pa = pa;  in ext4_mb_use_inode_pa()
3302 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,  in ext4_mb_use_group_pa()  argument
3305 	unsigned int len = ac->ac_o_ex.fe_len;  in ext4_mb_use_group_pa()
3307 	ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,  in ext4_mb_use_group_pa()
3308 					&ac->ac_b_ex.fe_group,  in ext4_mb_use_group_pa()
3309 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_group_pa()
3310 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_group_pa()
3311 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_group_pa()
3312 	ac->ac_pa = pa;  in ext4_mb_use_group_pa()
3356 ext4_mb_use_preallocated(struct ext4_allocation_context *ac)  in ext4_mb_use_preallocated()  argument
3358 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()
3360 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_use_preallocated()
3366 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_use_preallocated()
3375 		if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||  in ext4_mb_use_preallocated()
3376 		    ac->ac_o_ex.fe_logical >= (pa->pa_lstart +  in ext4_mb_use_preallocated()
3381 		if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&  in ext4_mb_use_preallocated()
3390 			ext4_mb_use_inode_pa(ac, pa);  in ext4_mb_use_preallocated()
3392 			ac->ac_criteria = 10;  in ext4_mb_use_preallocated()
3401 	if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))  in ext4_mb_use_preallocated()
3405 	lg = ac->ac_lg;  in ext4_mb_use_preallocated()
3408 	order  = fls(ac->ac_o_ex.fe_len) - 1;  in ext4_mb_use_preallocated()
3413 	goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);  in ext4_mb_use_preallocated()
3424 					pa->pa_free >= ac->ac_o_ex.fe_len) {  in ext4_mb_use_preallocated()
3434 		ext4_mb_use_group_pa(ac, cpa);  in ext4_mb_use_preallocated()
3435 		ac->ac_criteria = 20;  in ext4_mb_use_preallocated()
3520 static void ext4_mb_put_pa(struct ext4_allocation_context *ac,  in ext4_mb_put_pa()  argument
3580 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)  in ext4_mb_new_inode_pa()  argument
3582 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_inode_pa()
3589 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3590 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_inode_pa()
3591 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_inode_pa()
3597 	if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {  in ext4_mb_new_inode_pa()
3606 		BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);  in ext4_mb_new_inode_pa()
3607 		BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3612 		winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical;  in ext4_mb_new_inode_pa()
3615 		wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3620 		offs = ac->ac_o_ex.fe_logical %  in ext4_mb_new_inode_pa()
3621 			EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3625 		ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical -  in ext4_mb_new_inode_pa()
3627 		BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);  in ext4_mb_new_inode_pa()
3628 		BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3633 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_inode_pa()
3635 	pa->pa_lstart = ac->ac_b_ex.fe_logical;  in ext4_mb_new_inode_pa()
3636 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_inode_pa()
3637 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
3648 	trace_ext4_mb_new_inode_pa(ac, pa);  in ext4_mb_new_inode_pa()
3650 	ext4_mb_use_inode_pa(ac, pa);  in ext4_mb_new_inode_pa()
3653 	ei = EXT4_I(ac->ac_inode);  in ext4_mb_new_inode_pa()
3654 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3657 	pa->pa_inode = ac->ac_inode;  in ext4_mb_new_inode_pa()
3659 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3661 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3674 ext4_mb_new_group_pa(struct ext4_allocation_context *ac)  in ext4_mb_new_group_pa()  argument
3676 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_group_pa()
3682 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_group_pa()
3683 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_group_pa()
3684 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_group_pa()
3693 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_group_pa()
3695 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_group_pa()
3697 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_group_pa()
3708 	trace_ext4_mb_new_group_pa(ac, pa);  in ext4_mb_new_group_pa()
3710 	ext4_mb_use_group_pa(ac, pa);  in ext4_mb_new_group_pa()
3713 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3714 	lg = ac->ac_lg;  in ext4_mb_new_group_pa()
3720 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3722 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3731 static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac)  in ext4_mb_new_preallocation()  argument
3735 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_new_preallocation()
3736 		err = ext4_mb_new_group_pa(ac);  in ext4_mb_new_preallocation()
3738 		err = ext4_mb_new_inode_pa(ac);  in ext4_mb_new_preallocation()
4048 static void ext4_mb_show_ac(struct ext4_allocation_context *ac)  in ext4_mb_show_ac()  argument
4050 	struct super_block *sb = ac->ac_sb;  in ext4_mb_show_ac()
4057 	ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:"  in ext4_mb_show_ac()
4059 	ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d",  in ext4_mb_show_ac()
4060 			ac->ac_status, ac->ac_flags);  in ext4_mb_show_ac()
4061 	ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, "  in ext4_mb_show_ac()
4064 			(unsigned long)ac->ac_o_ex.fe_group,  in ext4_mb_show_ac()
4065 			(unsigned long)ac->ac_o_ex.fe_start,  in ext4_mb_show_ac()
4066 			(unsigned long)ac->ac_o_ex.fe_len,  in ext4_mb_show_ac()
4067 			(unsigned long)ac->ac_o_ex.fe_logical,  in ext4_mb_show_ac()
4068 			(unsigned long)ac->ac_g_ex.fe_group,  in ext4_mb_show_ac()
4069 			(unsigned long)ac->ac_g_ex.fe_start,  in ext4_mb_show_ac()
4070 			(unsigned long)ac->ac_g_ex.fe_len,  in ext4_mb_show_ac()
4071 			(unsigned long)ac->ac_g_ex.fe_logical,  in ext4_mb_show_ac()
4072 			(unsigned long)ac->ac_b_ex.fe_group,  in ext4_mb_show_ac()
4073 			(unsigned long)ac->ac_b_ex.fe_start,  in ext4_mb_show_ac()
4074 			(unsigned long)ac->ac_b_ex.fe_len,  in ext4_mb_show_ac()
4075 			(unsigned long)ac->ac_b_ex.fe_logical,  in ext4_mb_show_ac()
4076 			(int)ac->ac_criteria);  in ext4_mb_show_ac()
4077 	ext4_msg(ac->ac_sb, KERN_ERR, "%d found", ac->ac_found);  in ext4_mb_show_ac()
4078 	ext4_msg(ac->ac_sb, KERN_ERR, "groups: ");  in ext4_mb_show_ac()
4106 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac)  in ext4_mb_show_ac()  argument
4119 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac)  in ext4_mb_group_or_file()  argument
4121 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()
4122 	int bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_group_or_file()
4125 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_group_or_file()
4128 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_group_or_file()
4131 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_group_or_file()
4132 	isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)  in ext4_mb_group_or_file()
4137 	    (atomic_read(&ac->ac_inode->i_writecount) == 0)) {  in ext4_mb_group_or_file()
4138 		ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_group_or_file()
4143 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4150 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4154 	BUG_ON(ac->ac_lg != NULL);  in ext4_mb_group_or_file()
4160 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
4163 	ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;  in ext4_mb_group_or_file()
4166 	mutex_lock(&ac->ac_lg->lg_mutex);  in ext4_mb_group_or_file()
4170 ext4_mb_initialize_context(struct ext4_allocation_context *ac,  in ext4_mb_initialize_context()  argument
4196 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
4197 	ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_initialize_context()
4198 	ac->ac_sb = sb;  in ext4_mb_initialize_context()
4199 	ac->ac_inode = ar->inode;  in ext4_mb_initialize_context()
4200 	ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;  in ext4_mb_initialize_context()
4201 	ac->ac_o_ex.fe_group = group;  in ext4_mb_initialize_context()
4202 	ac->ac_o_ex.fe_start = block;  in ext4_mb_initialize_context()
4203 	ac->ac_o_ex.fe_len = len;  in ext4_mb_initialize_context()
4204 	ac->ac_g_ex = ac->ac_o_ex;  in ext4_mb_initialize_context()
4205 	ac->ac_flags = ar->flags;  in ext4_mb_initialize_context()
4209 	ext4_mb_group_or_file(ac);  in ext4_mb_initialize_context()
4214 			(unsigned) ar->goal, ac->ac_flags, ac->ac_2order,  in ext4_mb_initialize_context()
4304 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac)  in ext4_mb_add_n_trim()  argument
4307 	struct super_block *sb = ac->ac_sb;  in ext4_mb_add_n_trim()
4308 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_add_n_trim()
4309 	struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;  in ext4_mb_add_n_trim()
4354 static int ext4_mb_release_context(struct ext4_allocation_context *ac)  in ext4_mb_release_context()  argument
4356 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()
4357 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_release_context()
4362 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4363 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4364 			pa->pa_free -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4365 			pa->pa_len -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4380 			ext4_mb_add_n_trim(ac);  in ext4_mb_release_context()
4382 		ext4_mb_put_pa(ac, ac->ac_sb, pa);  in ext4_mb_release_context()
4384 	if (ac->ac_bitmap_page)  in ext4_mb_release_context()
4385 		page_cache_release(ac->ac_bitmap_page);  in ext4_mb_release_context()
4386 	if (ac->ac_buddy_page)  in ext4_mb_release_context()
4387 		page_cache_release(ac->ac_buddy_page);  in ext4_mb_release_context()
4388 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_release_context()
4389 		mutex_unlock(&ac->ac_lg->lg_mutex);  in ext4_mb_release_context()
4390 	ext4_mb_collect_stats(ac);  in ext4_mb_release_context()
4419 	struct ext4_allocation_context *ac = NULL;  in ext4_mb_new_blocks()  local
4472 	ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS);  in ext4_mb_new_blocks()
4473 	if (!ac) {  in ext4_mb_new_blocks()
4479 	*errp = ext4_mb_initialize_context(ac, ar);  in ext4_mb_new_blocks()
4485 	ac->ac_op = EXT4_MB_HISTORY_PREALLOC;  in ext4_mb_new_blocks()
4486 	if (!ext4_mb_use_preallocated(ac)) {  in ext4_mb_new_blocks()
4487 		ac->ac_op = EXT4_MB_HISTORY_ALLOC;  in ext4_mb_new_blocks()
4488 		ext4_mb_normalize_request(ac, ar);  in ext4_mb_new_blocks()
4491 		*errp = ext4_mb_regular_allocator(ac);  in ext4_mb_new_blocks()
4498 		if (ac->ac_status == AC_STATUS_FOUND &&  in ext4_mb_new_blocks()
4499 		    ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)  in ext4_mb_new_blocks()
4500 			*errp = ext4_mb_new_preallocation(ac);  in ext4_mb_new_blocks()
4503 			ext4_discard_allocated_blocks(ac);  in ext4_mb_new_blocks()
4507 	if (likely(ac->ac_status == AC_STATUS_FOUND)) {  in ext4_mb_new_blocks()
4508 		*errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs);  in ext4_mb_new_blocks()
4514 			ext4_mb_release_context(ac);  in ext4_mb_new_blocks()
4515 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_new_blocks()
4516 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_new_blocks()
4517 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
4518 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_new_blocks()
4521 			ext4_discard_allocated_blocks(ac);  in ext4_mb_new_blocks()
4524 			block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_blocks()
4525 			ar->len = ac->ac_b_ex.fe_len;  in ext4_mb_new_blocks()
4528 		freed  = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);  in ext4_mb_new_blocks()
4536 		ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
4538 		ext4_mb_show_ac(ac);  in ext4_mb_new_blocks()
4540 	ext4_mb_release_context(ac);  in ext4_mb_new_blocks()
4542 	if (ac)  in ext4_mb_new_blocks()
4543 		kmem_cache_free(ext4_ac_cachep, ac);  in ext4_mb_new_blocks()