96 				      *block_group, struct btrfs_path *path)  in lookup_free_space_inode()
101 	spin_lock(&block_group->lock);  in lookup_free_space_inode()
102 	if (block_group->inode)  in lookup_free_space_inode()
103 		inode = igrab(block_group->inode);  in lookup_free_space_inode()
104 	spin_unlock(&block_group->lock);  in lookup_free_space_inode()
109 					  block_group->key.objectid);  in lookup_free_space_inode()
113 	spin_lock(&block_group->lock);  in lookup_free_space_inode()
119 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in lookup_free_space_inode()
122 	if (!block_group->iref) {  in lookup_free_space_inode()
123 		block_group->inode = igrab(inode);  in lookup_free_space_inode()
124 		block_group->iref = 1;  in lookup_free_space_inode()
126 	spin_unlock(&block_group->lock);  in lookup_free_space_inode()
194 			    struct btrfs_block_group_cache *block_group,  in create_free_space_inode()  argument
205 					 block_group->key.objectid);  in create_free_space_inode()
229 				    struct btrfs_block_group_cache *block_group,  in btrfs_truncate_free_space_cache()  argument
240 	if (block_group) {  in btrfs_truncate_free_space_cache()
242 		if (!list_empty(&block_group->io_list)) {  in btrfs_truncate_free_space_cache()
243 			list_del_init(&block_group->io_list);  in btrfs_truncate_free_space_cache()
245 			btrfs_wait_cache_io(root, trans, block_group,  in btrfs_truncate_free_space_cache()
246 					    &block_group->io_ctl, path,  in btrfs_truncate_free_space_cache()
247 					    block_group->key.objectid);  in btrfs_truncate_free_space_cache()
248 			btrfs_put_block_group(block_group);  in btrfs_truncate_free_space_cache()
255 		spin_lock(&block_group->lock);  in btrfs_truncate_free_space_cache()
256 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in btrfs_truncate_free_space_cache()
257 		spin_unlock(&block_group->lock);  in btrfs_truncate_free_space_cache()
280 	if (block_group)  in btrfs_truncate_free_space_cache()
829 			  struct btrfs_block_group_cache *block_group)  in load_free_space_cache()  argument
831 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in load_free_space_cache()
837 	u64 used = btrfs_block_group_used(&block_group->item);  in load_free_space_cache()
843 	spin_lock(&block_group->lock);  in load_free_space_cache()
844 	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {  in load_free_space_cache()
845 		spin_unlock(&block_group->lock);  in load_free_space_cache()
848 	spin_unlock(&block_group->lock);  in load_free_space_cache()
856 	inode = lookup_free_space_inode(root, block_group, path);  in load_free_space_cache()
863 	spin_lock(&block_group->lock);  in load_free_space_cache()
864 	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {  in load_free_space_cache()
865 		spin_unlock(&block_group->lock);  in load_free_space_cache()
869 	spin_unlock(&block_group->lock);  in load_free_space_cache()
872 				      path, block_group->key.objectid);  in load_free_space_cache()
878 	matched = (ctl->free_space == (block_group->key.offset - used -  in load_free_space_cache()
879 				       block_group->bytes_super));  in load_free_space_cache()
885 			block_group->key.objectid);  in load_free_space_cache()
891 		spin_lock(&block_group->lock);  in load_free_space_cache()
892 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in load_free_space_cache()
893 		spin_unlock(&block_group->lock);  in load_free_space_cache()
897 			block_group->key.objectid);  in load_free_space_cache()
907 			      struct btrfs_block_group_cache *block_group,  in write_cache_extent_entries()  argument
918 	if (block_group && !list_empty(&block_group->cluster_list)) {  in write_cache_extent_entries()
919 		cluster = list_entry(block_group->cluster_list.next,  in write_cache_extent_entries()
1038 			    struct btrfs_block_group_cache *block_group,  in write_pinned_extent_entries()  argument
1046 	if (!block_group)  in write_pinned_extent_entries()
1058 	start = block_group->key.objectid;  in write_pinned_extent_entries()
1060 	while (start < block_group->key.objectid + block_group->key.offset) {  in write_pinned_extent_entries()
1068 		if (extent_start >= block_group->key.objectid +  in write_pinned_extent_entries()
1069 		    block_group->key.offset)  in write_pinned_extent_entries()
1073 		extent_end = min(block_group->key.objectid +  in write_pinned_extent_entries()
1074 				 block_group->key.offset, extent_end + 1);  in write_pinned_extent_entries()
1147 			struct btrfs_block_group_cache *block_group,  in btrfs_wait_cache_io()  argument
1157 	if (block_group)  in btrfs_wait_cache_io()
1173 		if (block_group) {  in btrfs_wait_cache_io()
1177 				block_group->key.objectid);  in btrfs_wait_cache_io()
1183 	if (block_group) {  in btrfs_wait_cache_io()
1188 		spin_lock(&block_group->lock);  in btrfs_wait_cache_io()
1195 		if (!ret && list_empty(&block_group->dirty_list))  in btrfs_wait_cache_io()
1196 			block_group->disk_cache_state = BTRFS_DC_WRITTEN;  in btrfs_wait_cache_io()
1198 			block_group->disk_cache_state = BTRFS_DC_ERROR;  in btrfs_wait_cache_io()
1200 		spin_unlock(&block_group->lock);  in btrfs_wait_cache_io()
1225 				   struct btrfs_block_group_cache *block_group,  in __btrfs_write_out_cache()  argument
1245 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)) {  in __btrfs_write_out_cache()
1246 		down_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1247 		spin_lock(&block_group->lock);  in __btrfs_write_out_cache()
1248 		if (block_group->delalloc_bytes) {  in __btrfs_write_out_cache()
1249 			block_group->disk_cache_state = BTRFS_DC_WRITTEN;  in __btrfs_write_out_cache()
1250 			spin_unlock(&block_group->lock);  in __btrfs_write_out_cache()
1251 			up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1257 		spin_unlock(&block_group->lock);  in __btrfs_write_out_cache()
1274 					 block_group, &entries, &bitmaps,  in __btrfs_write_out_cache()
1287 	ret = write_pinned_extent_entries(root, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1311 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))  in __btrfs_write_out_cache()
1312 		up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1356 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))  in __btrfs_write_out_cache()
1357 		up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1364 			  struct btrfs_block_group_cache *block_group,  in btrfs_write_out_cache()  argument
1367 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_write_out_cache()
1373 	spin_lock(&block_group->lock);  in btrfs_write_out_cache()
1374 	if (block_group->disk_cache_state < BTRFS_DC_SETUP) {  in btrfs_write_out_cache()
1375 		spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1378 	spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1380 	inode = lookup_free_space_inode(root, block_group, path);  in btrfs_write_out_cache()
1384 	ret = __btrfs_write_out_cache(root, inode, ctl, block_group,  in btrfs_write_out_cache()
1385 				      &block_group->io_ctl, trans,  in btrfs_write_out_cache()
1386 				      path, block_group->key.objectid);  in btrfs_write_out_cache()
1391 			block_group->key.objectid);  in btrfs_write_out_cache()
1393 		spin_lock(&block_group->lock);  in btrfs_write_out_cache()
1394 		block_group->disk_cache_state = BTRFS_DC_ERROR;  in btrfs_write_out_cache()
1395 		spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1397 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()
1644 	struct btrfs_block_group_cache *block_group = ctl->private;  in recalculate_thresholds()  local
1648 	u64 size = block_group->key.offset;  in recalculate_thresholds()
1955 	struct btrfs_block_group_cache *block_group = ctl->private;  in use_bitmap()  local
1969 		if (info->bytes <= block_group->sectorsize * 4) {  in use_bitmap()
1985 	if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->key.offset)  in use_bitmap()
2000 	struct btrfs_block_group_cache *block_group = NULL;  in insert_into_bitmap()  local
2012 		block_group = ctl->private;  in insert_into_bitmap()
2019 	if (block_group && !list_empty(&block_group->cluster_list)) {  in insert_into_bitmap()
2024 		cluster = list_entry(block_group->cluster_list.next,  in insert_into_bitmap()
2334 int btrfs_remove_free_space(struct btrfs_block_group_cache *block_group,  in btrfs_remove_free_space()  argument
2337 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_remove_free_space()
2406 			ret = btrfs_add_free_space(block_group, offset + bytes,  in btrfs_remove_free_space()
2424 void btrfs_dump_free_space(struct btrfs_block_group_cache *block_group,  in btrfs_dump_free_space()  argument
2427 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_dump_free_space()
2434 		if (info->bytes >= bytes && !block_group->ro)  in btrfs_dump_free_space()
2436 		btrfs_crit(block_group->fs_info,  in btrfs_dump_free_space()
2441 	btrfs_info(block_group->fs_info, "block group has cluster?: %s",  in btrfs_dump_free_space()
2442 	       list_empty(&block_group->cluster_list) ? "no" : "yes");  in btrfs_dump_free_space()
2443 	btrfs_info(block_group->fs_info,  in btrfs_dump_free_space()
2447 void btrfs_init_free_space_ctl(struct btrfs_block_group_cache *block_group)  in btrfs_init_free_space_ctl()  argument
2449 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_init_free_space_ctl()
2452 	ctl->unit = block_group->sectorsize;  in btrfs_init_free_space_ctl()
2453 	ctl->start = block_group->key.objectid;  in btrfs_init_free_space_ctl()
2454 	ctl->private = block_group;  in btrfs_init_free_space_ctl()
2476 			     struct btrfs_block_group_cache *block_group,  in __btrfs_return_cluster_to_free_space()  argument
2479 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in __btrfs_return_cluster_to_free_space()
2484 	if (cluster->block_group != block_group)  in __btrfs_return_cluster_to_free_space()
2487 	cluster->block_group = NULL;  in __btrfs_return_cluster_to_free_space()
2512 	btrfs_put_block_group(block_group);  in __btrfs_return_cluster_to_free_space()
2542 void btrfs_remove_free_space_cache(struct btrfs_block_group_cache *block_group)  in btrfs_remove_free_space_cache()  argument
2544 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_remove_free_space_cache()
2549 	while ((head = block_group->cluster_list.next) !=  in btrfs_remove_free_space_cache()
2550 	       &block_group->cluster_list) {  in btrfs_remove_free_space_cache()
2554 		WARN_ON(cluster->block_group != block_group);  in btrfs_remove_free_space_cache()
2555 		__btrfs_return_cluster_to_free_space(block_group, cluster);  in btrfs_remove_free_space_cache()
2564 u64 btrfs_find_space_for_alloc(struct btrfs_block_group_cache *block_group,  in btrfs_find_space_for_alloc()  argument
2568 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_find_space_for_alloc()
2577 				block_group->full_stripe_len, max_extent_size);  in btrfs_find_space_for_alloc()
2617 			       struct btrfs_block_group_cache *block_group,  in btrfs_return_cluster_to_free_space()  argument
2625 	if (!block_group) {  in btrfs_return_cluster_to_free_space()
2626 		block_group = cluster->block_group;  in btrfs_return_cluster_to_free_space()
2627 		if (!block_group) {  in btrfs_return_cluster_to_free_space()
2631 	} else if (cluster->block_group != block_group) {  in btrfs_return_cluster_to_free_space()
2636 	atomic_inc(&block_group->count);  in btrfs_return_cluster_to_free_space()
2639 	ctl = block_group->free_space_ctl;  in btrfs_return_cluster_to_free_space()
2643 	ret = __btrfs_return_cluster_to_free_space(block_group, cluster);  in btrfs_return_cluster_to_free_space()
2647 	btrfs_put_block_group(block_group);  in btrfs_return_cluster_to_free_space()
2651 static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group_cache *block_group,  in btrfs_alloc_from_bitmap()  argument
2657 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_alloc_from_bitmap()
2684 u64 btrfs_alloc_from_cluster(struct btrfs_block_group_cache *block_group,  in btrfs_alloc_from_cluster()  argument
2688 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_alloc_from_cluster()
2697 	if (cluster->block_group != block_group)  in btrfs_alloc_from_cluster()
2720 			ret = btrfs_alloc_from_bitmap(block_group,  in btrfs_alloc_from_cluster()
2768 static int btrfs_bitmap_cluster(struct btrfs_block_group_cache *block_group,  in btrfs_bitmap_cluster()  argument
2774 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_bitmap_cluster()
2825 	trace_btrfs_setup_cluster(block_group, cluster,  in btrfs_bitmap_cluster()
2836 setup_cluster_no_bitmap(struct btrfs_block_group_cache *block_group,  in setup_cluster_no_bitmap()  argument
2841 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in setup_cluster_no_bitmap()
2918 	trace_btrfs_setup_cluster(block_group, cluster, total_size, 0);  in setup_cluster_no_bitmap()
2927 setup_cluster_bitmap(struct btrfs_block_group_cache *block_group,  in setup_cluster_bitmap()  argument
2932 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in setup_cluster_bitmap()
2954 		ret = btrfs_bitmap_cluster(block_group, entry, cluster, offset,  in setup_cluster_bitmap()
2976 			     struct btrfs_block_group_cache *block_group,  in btrfs_find_space_cluster()  argument
2980 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_find_space_cluster()
2995 	} else if (block_group->flags & BTRFS_BLOCK_GROUP_METADATA) {  in btrfs_find_space_cluster()
2997 		min_bytes = block_group->sectorsize;  in btrfs_find_space_cluster()
3000 		min_bytes = block_group->sectorsize;  in btrfs_find_space_cluster()
3017 	if (cluster->block_group) {  in btrfs_find_space_cluster()
3022 	trace_btrfs_find_cluster(block_group, offset, bytes, empty_size,  in btrfs_find_space_cluster()
3025 	ret = setup_cluster_no_bitmap(block_group, cluster, &bitmaps, offset,  in btrfs_find_space_cluster()
3029 		ret = setup_cluster_bitmap(block_group, cluster, &bitmaps,  in btrfs_find_space_cluster()
3038 		atomic_inc(&block_group->count);  in btrfs_find_space_cluster()
3040 			      &block_group->cluster_list);  in btrfs_find_space_cluster()
3041 		cluster->block_group = block_group;  in btrfs_find_space_cluster()
3043 		trace_btrfs_failed_cluster_setup(block_group);  in btrfs_find_space_cluster()
3062 	cluster->block_group = NULL;  in btrfs_init_free_cluster()
3065 static int do_trimming(struct btrfs_block_group_cache *block_group,  in do_trimming()  argument
3070 	struct btrfs_space_info *space_info = block_group->space_info;  in do_trimming()
3071 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in do_trimming()
3072 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in do_trimming()
3078 	spin_lock(&block_group->lock);  in do_trimming()
3079 	if (!block_group->ro) {  in do_trimming()
3080 		block_group->reserved += reserved_bytes;  in do_trimming()
3084 	spin_unlock(&block_group->lock);  in do_trimming()
3093 	btrfs_add_free_space(block_group, reserved_start, reserved_bytes);  in do_trimming()
3099 		spin_lock(&block_group->lock);  in do_trimming()
3100 		if (block_group->ro)  in do_trimming()
3102 		block_group->reserved -= reserved_bytes;  in do_trimming()
3105 		spin_unlock(&block_group->lock);  in do_trimming()
3111 static int trim_no_bitmap(struct btrfs_block_group_cache *block_group,  in trim_no_bitmap()  argument
3114 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in trim_no_bitmap()
3178 		ret = do_trimming(block_group, total_trimmed, start, bytes,  in trim_no_bitmap()
3196 static int trim_bitmaps(struct btrfs_block_group_cache *block_group,  in trim_bitmaps()  argument
3199 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in trim_bitmaps()
3253 		ret = do_trimming(block_group, total_trimmed, start, bytes,  in trim_bitmaps()
3277 int btrfs_trim_block_group(struct btrfs_block_group_cache *block_group,  in btrfs_trim_block_group()  argument
3284 	spin_lock(&block_group->lock);  in btrfs_trim_block_group()
3285 	if (block_group->removed) {  in btrfs_trim_block_group()
3286 		spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3289 	atomic_inc(&block_group->trimming);  in btrfs_trim_block_group()
3290 	spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3292 	ret = trim_no_bitmap(block_group, trimmed, start, end, minlen);  in btrfs_trim_block_group()
3296 	ret = trim_bitmaps(block_group, trimmed, start, end, minlen);  in btrfs_trim_block_group()
3298 	spin_lock(&block_group->lock);  in btrfs_trim_block_group()
3299 	if (atomic_dec_and_test(&block_group->trimming) &&  in btrfs_trim_block_group()
3300 	    block_group->removed) {  in btrfs_trim_block_group()
3304 		spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3306 		lock_chunks(block_group->fs_info->chunk_root);  in btrfs_trim_block_group()
3307 		em_tree = &block_group->fs_info->mapping_tree.map_tree;  in btrfs_trim_block_group()
3309 		em = lookup_extent_mapping(em_tree, block_group->key.objectid,  in btrfs_trim_block_group()
3318 		unlock_chunks(block_group->fs_info->chunk_root);  in btrfs_trim_block_group()
3328 		__btrfs_remove_free_space_cache(block_group->free_space_ctl);  in btrfs_trim_block_group()
3330 		spin_unlock(&block_group->lock);  in btrfs_trim_block_group()