96 				      *block_group, struct btrfs_path *path)  in lookup_free_space_inode()
101 	spin_lock(&block_group->lock);  in lookup_free_space_inode()
102 	if (block_group->inode)  in lookup_free_space_inode()
103 		inode = igrab(block_group->inode);  in lookup_free_space_inode()
104 	spin_unlock(&block_group->lock);  in lookup_free_space_inode()
109 					  block_group->key.objectid);  in lookup_free_space_inode()
113 	spin_lock(&block_group->lock);  in lookup_free_space_inode()
119 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in lookup_free_space_inode()
122 	if (!block_group->iref) {  in lookup_free_space_inode()
123 		block_group->inode = igrab(inode);  in lookup_free_space_inode()
124 		block_group->iref = 1;  in lookup_free_space_inode()
126 	spin_unlock(&block_group->lock);  in lookup_free_space_inode()
194 			    struct btrfs_block_group_cache *block_group,  in create_free_space_inode()  argument
205 					 block_group->key.objectid);  in create_free_space_inode()
229 				    struct btrfs_block_group_cache *block_group,  in btrfs_truncate_free_space_cache()  argument
241 	if (block_group) {  in btrfs_truncate_free_space_cache()
244 		if (!list_empty(&block_group->io_list)) {  in btrfs_truncate_free_space_cache()
245 			list_del_init(&block_group->io_list);  in btrfs_truncate_free_space_cache()
247 			btrfs_wait_cache_io(root, trans, block_group,  in btrfs_truncate_free_space_cache()
248 					    &block_group->io_ctl, path,  in btrfs_truncate_free_space_cache()
249 					    block_group->key.objectid);  in btrfs_truncate_free_space_cache()
250 			btrfs_put_block_group(block_group);  in btrfs_truncate_free_space_cache()
257 		spin_lock(&block_group->lock);  in btrfs_truncate_free_space_cache()
258 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in btrfs_truncate_free_space_cache()
259 		spin_unlock(&block_group->lock);  in btrfs_truncate_free_space_cache()
827 			  struct btrfs_block_group_cache *block_group)  in load_free_space_cache()  argument
829 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in load_free_space_cache()
835 	u64 used = btrfs_block_group_used(&block_group->item);  in load_free_space_cache()
841 	spin_lock(&block_group->lock);  in load_free_space_cache()
842 	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {  in load_free_space_cache()
843 		spin_unlock(&block_group->lock);  in load_free_space_cache()
846 	spin_unlock(&block_group->lock);  in load_free_space_cache()
854 	inode = lookup_free_space_inode(root, block_group, path);  in load_free_space_cache()
861 	spin_lock(&block_group->lock);  in load_free_space_cache()
862 	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {  in load_free_space_cache()
863 		spin_unlock(&block_group->lock);  in load_free_space_cache()
867 	spin_unlock(&block_group->lock);  in load_free_space_cache()
870 				      path, block_group->key.objectid);  in load_free_space_cache()
876 	matched = (ctl->free_space == (block_group->key.offset - used -  in load_free_space_cache()
877 				       block_group->bytes_super));  in load_free_space_cache()
883 			block_group->key.objectid);  in load_free_space_cache()
889 		spin_lock(&block_group->lock);  in load_free_space_cache()
890 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in load_free_space_cache()
891 		spin_unlock(&block_group->lock);  in load_free_space_cache()
895 			block_group->key.objectid);  in load_free_space_cache()
905 			      struct btrfs_block_group_cache *block_group,  in write_cache_extent_entries()  argument
916 	if (block_group && !list_empty(&block_group->cluster_list)) {  in write_cache_extent_entries()
917 		cluster = list_entry(block_group->cluster_list.next,  in write_cache_extent_entries()
1036 			    struct btrfs_block_group_cache *block_group,  in write_pinned_extent_entries()  argument
1044 	if (!block_group)  in write_pinned_extent_entries()
1056 	start = block_group->key.objectid;  in write_pinned_extent_entries()
1058 	while (start < block_group->key.objectid + block_group->key.offset) {  in write_pinned_extent_entries()
1066 		if (extent_start >= block_group->key.objectid +  in write_pinned_extent_entries()
1067 		    block_group->key.offset)  in write_pinned_extent_entries()
1071 		extent_end = min(block_group->key.objectid +  in write_pinned_extent_entries()
1072 				 block_group->key.offset, extent_end + 1);  in write_pinned_extent_entries()
1145 			struct btrfs_block_group_cache *block_group,  in btrfs_wait_cache_io()  argument
1155 	if (block_group)  in btrfs_wait_cache_io()
1171 		if (block_group) {  in btrfs_wait_cache_io()
1175 				block_group->key.objectid);  in btrfs_wait_cache_io()
1181 	if (block_group) {  in btrfs_wait_cache_io()
1186 		spin_lock(&block_group->lock);  in btrfs_wait_cache_io()
1193 		if (!ret && list_empty(&block_group->dirty_list))  in btrfs_wait_cache_io()
1194 			block_group->disk_cache_state = BTRFS_DC_WRITTEN;  in btrfs_wait_cache_io()
1196 			block_group->disk_cache_state = BTRFS_DC_ERROR;  in btrfs_wait_cache_io()
1198 		spin_unlock(&block_group->lock);  in btrfs_wait_cache_io()
1223 				   struct btrfs_block_group_cache *block_group,  in __btrfs_write_out_cache()  argument
1243 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)) {  in __btrfs_write_out_cache()
1244 		down_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1245 		spin_lock(&block_group->lock);  in __btrfs_write_out_cache()
1246 		if (block_group->delalloc_bytes) {  in __btrfs_write_out_cache()
1247 			block_group->disk_cache_state = BTRFS_DC_WRITTEN;  in __btrfs_write_out_cache()
1248 			spin_unlock(&block_group->lock);  in __btrfs_write_out_cache()
1249 			up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1255 		spin_unlock(&block_group->lock);  in __btrfs_write_out_cache()
1272 					 block_group, &entries, &bitmaps,  in __btrfs_write_out_cache()
1285 	ret = write_pinned_extent_entries(root, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1309 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))  in __btrfs_write_out_cache()
1310 		up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1354 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))  in __btrfs_write_out_cache()
1355 		up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1362 			  struct btrfs_block_group_cache *block_group,  in btrfs_write_out_cache()  argument
1365 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_write_out_cache()
1371 	spin_lock(&block_group->lock);  in btrfs_write_out_cache()
1372 	if (block_group->disk_cache_state < BTRFS_DC_SETUP) {  in btrfs_write_out_cache()
1373 		spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1376 	spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1378 	inode = lookup_free_space_inode(root, block_group, path);  in btrfs_write_out_cache()
1382 	ret = __btrfs_write_out_cache(root, inode, ctl, block_group,  in btrfs_write_out_cache()
1383 				      &block_group->io_ctl, trans,  in btrfs_write_out_cache()
1384 				      path, block_group->key.objectid);  in btrfs_write_out_cache()
1389 			block_group->key.objectid);  in btrfs_write_out_cache()
1391 		spin_lock(&block_group->lock);  in btrfs_write_out_cache()
1392 		block_group->disk_cache_state = BTRFS_DC_ERROR;  in btrfs_write_out_cache()
1393 		spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1395 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()
1642 	struct btrfs_block_group_cache *block_group = ctl->private;  in recalculate_thresholds()  local
1646 	u64 size = block_group->key.offset;  in recalculate_thresholds()
1976 	struct btrfs_block_group_cache *block_group = ctl->private;  in use_bitmap()  local
1980 	if (btrfs_should_fragment_free_space(block_group->fs_info->extent_root,  in use_bitmap()
1981 					     block_group))  in use_bitmap()
1997 		if (info->bytes <= block_group->sectorsize * 4) {  in use_bitmap()
2013 	if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->key.offset)  in use_bitmap()
2028 	struct btrfs_block_group_cache *block_group = NULL;  in insert_into_bitmap()  local
2040 		block_group = ctl->private;  in insert_into_bitmap()
2047 	if (block_group && !list_empty(&block_group->cluster_list)) {  in insert_into_bitmap()
2052 		cluster = list_entry(block_group->cluster_list.next,  in insert_into_bitmap()
2362 int btrfs_remove_free_space(struct btrfs_block_group_cache *block_group,  in btrfs_remove_free_space()  argument
2365 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_remove_free_space()
2434 			ret = btrfs_add_free_space(block_group, offset + bytes,  in btrfs_remove_free_space()
2452 void btrfs_dump_free_space(struct btrfs_block_group_cache *block_group,  in btrfs_dump_free_space()  argument
2455 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_dump_free_space()
2462 		if (info->bytes >= bytes && !block_group->ro)  in btrfs_dump_free_space()
2464 		btrfs_crit(block_group->fs_info,  in btrfs_dump_free_space()
2469 	btrfs_info(block_group->fs_info, "block group has cluster?: %s",  in btrfs_dump_free_space()
2470 	       list_empty(&block_group->cluster_list) ? "no" : "yes");  in btrfs_dump_free_space()
2471 	btrfs_info(block_group->fs_info,  in btrfs_dump_free_space()
2475 void btrfs_init_free_space_ctl(struct btrfs_block_group_cache *block_group)  in btrfs_init_free_space_ctl()  argument
2477 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_init_free_space_ctl()
2480 	ctl->unit = block_group->sectorsize;  in btrfs_init_free_space_ctl()
2481 	ctl->start = block_group->key.objectid;  in btrfs_init_free_space_ctl()
2482 	ctl->private = block_group;  in btrfs_init_free_space_ctl()
2504 			     struct btrfs_block_group_cache *block_group,  in __btrfs_return_cluster_to_free_space()  argument
2507 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in __btrfs_return_cluster_to_free_space()
2512 	if (cluster->block_group != block_group)  in __btrfs_return_cluster_to_free_space()
2515 	cluster->block_group = NULL;  in __btrfs_return_cluster_to_free_space()
2540 	btrfs_put_block_group(block_group);  in __btrfs_return_cluster_to_free_space()
2570 void btrfs_remove_free_space_cache(struct btrfs_block_group_cache *block_group)  in btrfs_remove_free_space_cache()  argument
2572 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_remove_free_space_cache()
2577 	while ((head = block_group->cluster_list.next) !=  in btrfs_remove_free_space_cache()
2578 	       &block_group->cluster_list) {  in btrfs_remove_free_space_cache()
2582 		WARN_ON(cluster->block_group != block_group);  in btrfs_remove_free_space_cache()
2583 		__btrfs_return_cluster_to_free_space(block_group, cluster);  in btrfs_remove_free_space_cache()
2592 u64 btrfs_find_space_for_alloc(struct btrfs_block_group_cache *block_group,  in btrfs_find_space_for_alloc()  argument
2596 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_find_space_for_alloc()
2605 				block_group->full_stripe_len, max_extent_size);  in btrfs_find_space_for_alloc()
2645 			       struct btrfs_block_group_cache *block_group,  in btrfs_return_cluster_to_free_space()  argument
2653 	if (!block_group) {  in btrfs_return_cluster_to_free_space()
2654 		block_group = cluster->block_group;  in btrfs_return_cluster_to_free_space()
2655 		if (!block_group) {  in btrfs_return_cluster_to_free_space()
2659 	} else if (cluster->block_group != block_group) {  in btrfs_return_cluster_to_free_space()
2664 	atomic_inc(&block_group->count);  in btrfs_return_cluster_to_free_space()
2667 	ctl = block_group->free_space_ctl;  in btrfs_return_cluster_to_free_space()
2671 	ret = __btrfs_return_cluster_to_free_space(block_group, cluster);  in btrfs_return_cluster_to_free_space()
2675 	btrfs_put_block_group(block_group);  in btrfs_return_cluster_to_free_space()
2679 static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group_cache *block_group,  in btrfs_alloc_from_bitmap()  argument
2685 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_alloc_from_bitmap()
2712 u64 btrfs_alloc_from_cluster(struct btrfs_block_group_cache *block_group,  in btrfs_alloc_from_cluster()  argument
2716 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_alloc_from_cluster()
2725 	if (cluster->block_group != block_group)  in btrfs_alloc_from_cluster()
2748 			ret = btrfs_alloc_from_bitmap(block_group,  in btrfs_alloc_from_cluster()
2796 static int btrfs_bitmap_cluster(struct btrfs_block_group_cache *block_group,  in btrfs_bitmap_cluster()  argument
2802 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_bitmap_cluster()
2867 	trace_btrfs_setup_cluster(block_group, cluster,  in btrfs_bitmap_cluster()
2878 setup_cluster_no_bitmap(struct btrfs_block_group_cache *block_group,  in setup_cluster_no_bitmap()  argument
2883 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in setup_cluster_no_bitmap()
2960 	trace_btrfs_setup_cluster(block_group, cluster, total_size, 0);  in setup_cluster_no_bitmap()
2969 setup_cluster_bitmap(struct btrfs_block_group_cache *block_group,  in setup_cluster_bitmap()  argument
2974 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in setup_cluster_bitmap()
2998 		ret = btrfs_bitmap_cluster(block_group, entry, cluster, offset,  in setup_cluster_bitmap()
3020 			     struct btrfs_block_group_cache *block_group,  in btrfs_find_space_cluster()  argument
3024 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_find_space_cluster()
3039 	} else if (block_group->flags & BTRFS_BLOCK_GROUP_METADATA) {  in btrfs_find_space_cluster()
3041 		min_bytes = block_group->sectorsize;  in btrfs_find_space_cluster()
3044 		min_bytes = block_group->sectorsize;  in btrfs_find_space_cluster()
3061 	if (cluster->block_group) {  in btrfs_find_space_cluster()
3066 	trace_btrfs_find_cluster(block_group, offset, bytes, empty_size,  in btrfs_find_space_cluster()
3069 	ret = setup_cluster_no_bitmap(block_group, cluster, &bitmaps, offset,  in btrfs_find_space_cluster()
3073 		ret = setup_cluster_bitmap(block_group, cluster, &bitmaps,  in btrfs_find_space_cluster()
3082 		atomic_inc(&block_group->count);  in btrfs_find_space_cluster()
3084 			      &block_group->cluster_list);  in btrfs_find_space_cluster()
3085 		cluster->block_group = block_group;  in btrfs_find_space_cluster()
3087 		trace_btrfs_failed_cluster_setup(block_group);  in btrfs_find_space_cluster()
3107 	cluster->block_group = NULL;  in btrfs_init_free_cluster()
3110 static int do_trimming(struct btrfs_block_group_cache *block_group,  in do_trimming()  argument
3115 	struct btrfs_space_info *space_info = block_group->space_info;  in do_trimming()
3116 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in do_trimming()
3117 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in do_trimming()
3123 	spin_lock(&block_group->lock);  in do_trimming()
3124 	if (!block_group->ro) {  in do_trimming()
3125 		block_group->reserved += reserved_bytes;  in do_trimming()
3129 	spin_unlock(&block_group->lock);  in do_trimming()
3138 	btrfs_add_free_space(block_group, reserved_start, reserved_bytes);  in do_trimming()
3144 		spin_lock(&block_group->lock);  in do_trimming()
3145 		if (block_group->ro)  in do_trimming()
3147 		block_group->reserved -= reserved_bytes;  in do_trimming()
3150 		spin_unlock(&block_group->lock);  in do_trimming()
3156 static int trim_no_bitmap(struct btrfs_block_group_cache *block_group,  in trim_no_bitmap()  argument
3159 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in trim_no_bitmap()
3223 		ret = do_trimming(block_group, total_trimmed, start, bytes,  in trim_no_bitmap()
3241 static int trim_bitmaps(struct btrfs_block_group_cache *block_group,  in trim_bitmaps()  argument
3244 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in trim_bitmaps()
3298 		ret = do_trimming(block_group, total_trimmed, start, bytes,  in trim_bitmaps()
3327 void btrfs_put_block_group_trimming(struct btrfs_block_group_cache *block_group)  in btrfs_put_block_group_trimming()  argument
3333 	spin_lock(&block_group->lock);  in btrfs_put_block_group_trimming()
3334 	cleanup = (atomic_dec_and_test(&block_group->trimming) &&  in btrfs_put_block_group_trimming()
3335 		   block_group->removed);  in btrfs_put_block_group_trimming()
3336 	spin_unlock(&block_group->lock);  in btrfs_put_block_group_trimming()
3339 		lock_chunks(block_group->fs_info->chunk_root);  in btrfs_put_block_group_trimming()
3340 		em_tree = &block_group->fs_info->mapping_tree.map_tree;  in btrfs_put_block_group_trimming()
3342 		em = lookup_extent_mapping(em_tree, block_group->key.objectid,  in btrfs_put_block_group_trimming()
3351 		unlock_chunks(block_group->fs_info->chunk_root);  in btrfs_put_block_group_trimming()
3361 		__btrfs_remove_free_space_cache(block_group->free_space_ctl);  in btrfs_put_block_group_trimming()
3365 int btrfs_trim_block_group(struct btrfs_block_group_cache *block_group,  in btrfs_trim_block_group()  argument
3372 	spin_lock(&block_group->lock);  in btrfs_trim_block_group()
3373 	if (block_group->removed) {  in btrfs_trim_block_group()
3374 		spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3377 	btrfs_get_block_group_trimming(block_group);  in btrfs_trim_block_group()
3378 	spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3380 	ret = trim_no_bitmap(block_group, trimmed, start, end, minlen);  in btrfs_trim_block_group()
3384 	ret = trim_bitmaps(block_group, trimmed, start, end, minlen);  in btrfs_trim_block_group()
3386 	btrfs_put_block_group_trimming(block_group);  in btrfs_trim_block_group()