Lines Matching refs:ctl
41 static int link_free_space(struct btrfs_free_space_ctl *ctl,
43 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
641 static void merge_space_tree(struct btrfs_free_space_ctl *ctl) in merge_space_tree() argument
647 spin_lock(&ctl->tree_lock); in merge_space_tree()
648 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in merge_space_tree()
655 unlink_free_space(ctl, prev); in merge_space_tree()
656 unlink_free_space(ctl, e); in merge_space_tree()
659 link_free_space(ctl, prev); in merge_space_tree()
661 spin_unlock(&ctl->tree_lock); in merge_space_tree()
667 spin_unlock(&ctl->tree_lock); in merge_space_tree()
671 struct btrfs_free_space_ctl *ctl, in __load_free_space_cache() argument
768 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
769 ret = link_free_space(ctl, e); in __load_free_space_cache()
770 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
786 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
787 ret = link_free_space(ctl, e); in __load_free_space_cache()
788 ctl->total_bitmaps++; in __load_free_space_cache()
789 ctl->op->recalc_thresholds(ctl); in __load_free_space_cache()
790 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
817 merge_space_tree(ctl); in __load_free_space_cache()
824 __btrfs_remove_free_space_cache(ctl); in __load_free_space_cache()
831 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in load_free_space_cache() local
871 ret = __load_free_space_cache(fs_info->tree_root, inode, ctl, in load_free_space_cache()
877 spin_lock(&ctl->tree_lock); in load_free_space_cache()
878 matched = (ctl->free_space == (block_group->key.offset - used - in load_free_space_cache()
880 spin_unlock(&ctl->tree_lock); in load_free_space_cache()
883 __btrfs_remove_free_space_cache(ctl); in load_free_space_cache()
906 struct btrfs_free_space_ctl *ctl, in write_cache_extent_entries() argument
914 struct rb_node *node = rb_first(&ctl->free_space_offset); in write_cache_extent_entries()
966 list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { in write_cache_extent_entries()
1224 struct btrfs_free_space_ctl *ctl, in __btrfs_write_out_cache() argument
1270 mutex_lock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1272 spin_lock(&ctl->tree_lock); in __btrfs_write_out_cache()
1273 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1297 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1298 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1350 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1351 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1367 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_write_out_cache() local
1384 ret = __btrfs_write_out_cache(root, inode, ctl, block_group, in btrfs_write_out_cache()
1422 static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, in offset_to_bitmap() argument
1428 bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; in offset_to_bitmap()
1429 bitmap_start = offset - ctl->start; in offset_to_bitmap()
1432 bitmap_start += ctl->start; in offset_to_bitmap()
1496 tree_search_offset(struct btrfs_free_space_ctl *ctl, in tree_search_offset() argument
1499 struct rb_node *n = ctl->free_space_offset.rb_node; in tree_search_offset()
1585 if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) in tree_search_offset()
1596 ctl->unit > offset) in tree_search_offset()
1612 __unlink_free_space(struct btrfs_free_space_ctl *ctl, in __unlink_free_space() argument
1615 rb_erase(&info->offset_index, &ctl->free_space_offset); in __unlink_free_space()
1616 ctl->free_extents--; in __unlink_free_space()
1619 static void unlink_free_space(struct btrfs_free_space_ctl *ctl, in unlink_free_space() argument
1622 __unlink_free_space(ctl, info); in unlink_free_space()
1623 ctl->free_space -= info->bytes; in unlink_free_space()
1626 static int link_free_space(struct btrfs_free_space_ctl *ctl, in link_free_space() argument
1632 ret = tree_insert_offset(&ctl->free_space_offset, info->offset, in link_free_space()
1637 ctl->free_space += info->bytes; in link_free_space()
1638 ctl->free_extents++; in link_free_space()
1642 static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) in recalculate_thresholds() argument
1644 struct btrfs_block_group_cache *block_group = ctl->private; in recalculate_thresholds()
1649 u32 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; in recalculate_thresholds()
1654 ASSERT(ctl->total_bitmaps <= max_bitmaps); in recalculate_thresholds()
1672 bitmap_bytes = (ctl->total_bitmaps + 1) * PAGE_CACHE_SIZE; in recalculate_thresholds()
1675 ctl->extents_thresh = 0; in recalculate_thresholds()
1686 ctl->extents_thresh = in recalculate_thresholds()
1690 static inline void __bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in __bitmap_clear_bits() argument
1696 start = offset_to_bit(info->offset, ctl->unit, offset); in __bitmap_clear_bits()
1697 count = bytes_to_bits(bytes, ctl->unit); in __bitmap_clear_bits()
1705 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in bitmap_clear_bits() argument
1709 __bitmap_clear_bits(ctl, info, offset, bytes); in bitmap_clear_bits()
1710 ctl->free_space -= bytes; in bitmap_clear_bits()
1713 static void bitmap_set_bits(struct btrfs_free_space_ctl *ctl, in bitmap_set_bits() argument
1719 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_set_bits()
1720 count = bytes_to_bits(bytes, ctl->unit); in bitmap_set_bits()
1726 ctl->free_space += bytes; in bitmap_set_bits()
1733 static int search_bitmap(struct btrfs_free_space_ctl *ctl, in search_bitmap() argument
1743 i = offset_to_bit(bitmap_info->offset, ctl->unit, in search_bitmap()
1745 bits = bytes_to_bits(*bytes, ctl->unit); in search_bitmap()
1761 *offset = (u64)(i * ctl->unit) + bitmap_info->offset; in search_bitmap()
1762 *bytes = (u64)(found_bits) * ctl->unit; in search_bitmap()
1766 *bytes = (u64)(max_bits) * ctl->unit; in search_bitmap()
1772 find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, in find_free_space() argument
1781 if (!ctl->free_space_offset.rb_node) in find_free_space()
1784 entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), 0, 1); in find_free_space()
1800 tmp = entry->offset - ctl->start + align - 1; in find_free_space()
1802 tmp = tmp * align + ctl->start; in find_free_space()
1818 ret = search_bitmap(ctl, entry, &tmp, &size); in find_free_space()
1837 static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, in add_new_bitmap() argument
1840 info->offset = offset_to_bitmap(ctl, offset); in add_new_bitmap()
1843 link_free_space(ctl, info); in add_new_bitmap()
1844 ctl->total_bitmaps++; in add_new_bitmap()
1846 ctl->op->recalc_thresholds(ctl); in add_new_bitmap()
1849 static void free_bitmap(struct btrfs_free_space_ctl *ctl, in free_bitmap() argument
1852 unlink_free_space(ctl, bitmap_info); in free_bitmap()
1855 ctl->total_bitmaps--; in free_bitmap()
1856 ctl->op->recalc_thresholds(ctl); in free_bitmap()
1859 static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, in remove_from_bitmap() argument
1868 end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; in remove_from_bitmap()
1877 search_bytes = ctl->unit; in remove_from_bitmap()
1879 ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes); in remove_from_bitmap()
1889 bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes); in remove_from_bitmap()
1896 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
1922 search_bytes = ctl->unit; in remove_from_bitmap()
1923 ret = search_bitmap(ctl, bitmap_info, &search_start, in remove_from_bitmap()
1930 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
1935 static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, in add_bytes_to_bitmap() argument
1942 end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); in add_bytes_to_bitmap()
1946 bitmap_set_bits(ctl, info, offset, bytes_to_set); in add_bytes_to_bitmap()
1952 static bool use_bitmap(struct btrfs_free_space_ctl *ctl, in use_bitmap() argument
1955 struct btrfs_block_group_cache *block_group = ctl->private; in use_bitmap()
1961 if (ctl->free_extents < ctl->extents_thresh) { in use_bitmap()
1970 if (ctl->free_extents * 2 <= ctl->extents_thresh) in use_bitmap()
1985 if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->key.offset) in use_bitmap()
1996 static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, in insert_into_bitmap() argument
2008 if (!ctl->op->use_bitmap(ctl, info)) in insert_into_bitmap()
2011 if (ctl->op == &free_space_op) in insert_into_bitmap()
2012 block_group = ctl->private; in insert_into_bitmap()
2040 if (entry->offset == offset_to_bitmap(ctl, offset)) { in insert_into_bitmap()
2041 bytes_added = add_bytes_to_bitmap(ctl, entry, in insert_into_bitmap()
2054 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in insert_into_bitmap()
2061 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes); in insert_into_bitmap()
2074 add_new_bitmap(ctl, info, offset); in insert_into_bitmap()
2079 spin_unlock(&ctl->tree_lock); in insert_into_bitmap()
2086 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2094 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2112 static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, in try_merge_free_space() argument
2126 right_info = tree_search_offset(ctl, offset + bytes, 0, 0); in try_merge_free_space()
2131 left_info = tree_search_offset(ctl, offset - 1, 0, 0); in try_merge_free_space()
2135 unlink_free_space(ctl, right_info); in try_merge_free_space()
2137 __unlink_free_space(ctl, right_info); in try_merge_free_space()
2146 unlink_free_space(ctl, left_info); in try_merge_free_space()
2148 __unlink_free_space(ctl, left_info); in try_merge_free_space()
2158 static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_end() argument
2166 const u64 bitmap_offset = offset_to_bitmap(ctl, end); in steal_from_bitmap_to_end()
2169 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_end()
2173 i = offset_to_bit(bitmap->offset, ctl->unit, end); in steal_from_bitmap_to_end()
2177 bytes = (j - i) * ctl->unit; in steal_from_bitmap_to_end()
2181 bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2183 __bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2186 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_end()
2191 static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_front() argument
2202 bitmap_offset = offset_to_bitmap(ctl, info->offset); in steal_from_bitmap_to_front()
2207 bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); in steal_from_bitmap_to_front()
2210 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_front()
2214 i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; in steal_from_bitmap_to_front()
2226 bytes = (i + 1) * ctl->unit; in steal_from_bitmap_to_front()
2228 bytes = (i - prev_j) * ctl->unit; in steal_from_bitmap_to_front()
2234 bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2236 __bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2239 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_front()
2255 static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap() argument
2266 if (ctl->total_bitmaps > 0) { in steal_from_bitmap()
2270 stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); in steal_from_bitmap()
2271 if (ctl->total_bitmaps > 0) in steal_from_bitmap()
2272 stole_front = steal_from_bitmap_to_front(ctl, info, in steal_from_bitmap()
2276 try_merge_free_space(ctl, info, update_stat); in steal_from_bitmap()
2280 int __btrfs_add_free_space(struct btrfs_free_space_ctl *ctl, in __btrfs_add_free_space() argument
2294 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space()
2296 if (try_merge_free_space(ctl, info, true)) in __btrfs_add_free_space()
2304 ret = insert_into_bitmap(ctl, info); in __btrfs_add_free_space()
2318 steal_from_bitmap(ctl, info, true); in __btrfs_add_free_space()
2320 ret = link_free_space(ctl, info); in __btrfs_add_free_space()
2324 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space()
2337 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space() local
2342 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space()
2349 info = tree_search_offset(ctl, offset, 0, 0); in btrfs_remove_free_space()
2355 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in btrfs_remove_free_space()
2370 unlink_free_space(ctl, info); in btrfs_remove_free_space()
2377 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2390 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2404 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2413 ret = remove_from_bitmap(ctl, info, &offset, &bytes); in btrfs_remove_free_space()
2419 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2427 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_dump_free_space() local
2432 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in btrfs_dump_free_space()
2449 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_init_free_space_ctl() local
2451 spin_lock_init(&ctl->tree_lock); in btrfs_init_free_space_ctl()
2452 ctl->unit = block_group->sectorsize; in btrfs_init_free_space_ctl()
2453 ctl->start = block_group->key.objectid; in btrfs_init_free_space_ctl()
2454 ctl->private = block_group; in btrfs_init_free_space_ctl()
2455 ctl->op = &free_space_op; in btrfs_init_free_space_ctl()
2456 INIT_LIST_HEAD(&ctl->trimming_ranges); in btrfs_init_free_space_ctl()
2457 mutex_init(&ctl->cache_writeout_mutex); in btrfs_init_free_space_ctl()
2464 ctl->extents_thresh = ((1024 * 32) / 2) / in btrfs_init_free_space_ctl()
2479 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_return_cluster_to_free_space() local
2502 try_merge_free_space(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2503 steal_from_bitmap(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2505 tree_insert_offset(&ctl->free_space_offset, in __btrfs_return_cluster_to_free_space()
2517 struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache_locked() argument
2522 while ((node = rb_last(&ctl->free_space_offset)) != NULL) { in __btrfs_remove_free_space_cache_locked()
2525 unlink_free_space(ctl, info); in __btrfs_remove_free_space_cache_locked()
2528 free_bitmap(ctl, info); in __btrfs_remove_free_space_cache_locked()
2531 cond_resched_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache_locked()
2535 void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache() argument
2537 spin_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2538 __btrfs_remove_free_space_cache_locked(ctl); in __btrfs_remove_free_space_cache()
2539 spin_unlock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2544 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space_cache() local
2548 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2557 cond_resched_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2559 __btrfs_remove_free_space_cache_locked(ctl); in btrfs_remove_free_space_cache()
2560 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2568 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_for_alloc() local
2575 spin_lock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2576 entry = find_free_space(ctl, &offset, &bytes_search, in btrfs_find_space_for_alloc()
2583 bitmap_clear_bits(ctl, entry, offset, bytes); in btrfs_find_space_for_alloc()
2585 free_bitmap(ctl, entry); in btrfs_find_space_for_alloc()
2587 unlink_free_space(ctl, entry); in btrfs_find_space_for_alloc()
2598 link_free_space(ctl, entry); in btrfs_find_space_for_alloc()
2601 spin_unlock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2604 __btrfs_add_free_space(ctl, align_gap, align_gap_len); in btrfs_find_space_for_alloc()
2620 struct btrfs_free_space_ctl *ctl; in btrfs_return_cluster_to_free_space() local
2639 ctl = block_group->free_space_ctl; in btrfs_return_cluster_to_free_space()
2642 spin_lock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
2644 spin_unlock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
2657 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_bitmap() local
2666 err = search_bitmap(ctl, entry, &search_start, &search_bytes); in btrfs_alloc_from_bitmap()
2674 __bitmap_clear_bits(ctl, entry, ret, bytes); in btrfs_alloc_from_bitmap()
2688 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_cluster() local
2750 spin_lock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
2752 ctl->free_space -= bytes; in btrfs_alloc_from_cluster()
2754 ctl->free_extents--; in btrfs_alloc_from_cluster()
2757 ctl->total_bitmaps--; in btrfs_alloc_from_cluster()
2758 ctl->op->recalc_thresholds(ctl); in btrfs_alloc_from_cluster()
2763 spin_unlock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
2774 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_bitmap_cluster() local
2784 i = offset_to_bit(entry->offset, ctl->unit, in btrfs_bitmap_cluster()
2786 want_bits = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_cluster()
2787 min_bits = bytes_to_bits(min_bytes, ctl->unit); in btrfs_bitmap_cluster()
2811 if (cluster->max_size < found_bits * ctl->unit) in btrfs_bitmap_cluster()
2812 cluster->max_size = found_bits * ctl->unit; in btrfs_bitmap_cluster()
2819 cluster->window_start = start * ctl->unit + entry->offset; in btrfs_bitmap_cluster()
2820 rb_erase(&entry->offset_index, &ctl->free_space_offset); in btrfs_bitmap_cluster()
2826 total_found * ctl->unit, 1); in btrfs_bitmap_cluster()
2841 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_no_bitmap() local
2850 entry = tree_search_offset(ctl, offset, 0, 1); in setup_cluster_no_bitmap()
2910 rb_erase(&entry->offset_index, &ctl->free_space_offset); in setup_cluster_no_bitmap()
2932 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_bitmap() local
2935 u64 bitmap_offset = offset_to_bitmap(ctl, offset); in setup_cluster_bitmap()
2937 if (ctl->total_bitmaps == 0) in setup_cluster_bitmap()
2946 entry = tree_search_offset(ctl, bitmap_offset, 1, 0); in setup_cluster_bitmap()
2980 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_cluster() local
3003 spin_lock(&ctl->tree_lock); in btrfs_find_space_cluster()
3009 if (ctl->free_space < bytes) { in btrfs_find_space_cluster()
3010 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3047 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3072 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_trimming() local
3092 mutex_lock(&ctl->cache_writeout_mutex); in do_trimming()
3095 mutex_unlock(&ctl->cache_writeout_mutex); in do_trimming()
3114 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_no_bitmap() local
3125 mutex_lock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3126 spin_lock(&ctl->tree_lock); in trim_no_bitmap()
3128 if (ctl->free_space < minlen) { in trim_no_bitmap()
3129 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3130 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3134 entry = tree_search_offset(ctl, start, 0, 1); in trim_no_bitmap()
3136 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3137 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3145 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3146 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3154 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3155 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3164 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3165 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3169 unlink_free_space(ctl, entry); in trim_no_bitmap()
3172 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3175 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_no_bitmap()
3176 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3199 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_bitmaps() local
3204 u64 offset = offset_to_bitmap(ctl, start); in trim_bitmaps()
3210 mutex_lock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3211 spin_lock(&ctl->tree_lock); in trim_bitmaps()
3213 if (ctl->free_space < minlen) { in trim_bitmaps()
3214 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3215 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3219 entry = tree_search_offset(ctl, offset, 1, 0); in trim_bitmaps()
3221 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3222 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3228 ret2 = search_bitmap(ctl, entry, &start, &bytes); in trim_bitmaps()
3230 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3231 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3238 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3239 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3243 bitmap_clear_bits(ctl, entry, start, bytes); in trim_bitmaps()
3245 free_bitmap(ctl, entry); in trim_bitmaps()
3247 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3250 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_bitmaps()
3251 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3259 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3262 if (start >= offset + BITS_PER_BITMAP * ctl->unit) in trim_bitmaps()
3263 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3345 struct btrfs_free_space_ctl *ctl = fs_root->free_ino_ctl; in btrfs_find_ino_for_alloc() local
3349 spin_lock(&ctl->tree_lock); in btrfs_find_ino_for_alloc()
3351 if (RB_EMPTY_ROOT(&ctl->free_space_offset)) in btrfs_find_ino_for_alloc()
3354 entry = rb_entry(rb_first(&ctl->free_space_offset), in btrfs_find_ino_for_alloc()
3360 unlink_free_space(ctl, entry); in btrfs_find_ino_for_alloc()
3366 link_free_space(ctl, entry); in btrfs_find_ino_for_alloc()
3372 ret = search_bitmap(ctl, entry, &offset, &count); in btrfs_find_ino_for_alloc()
3377 bitmap_clear_bits(ctl, entry, offset, 1); in btrfs_find_ino_for_alloc()
3379 free_bitmap(ctl, entry); in btrfs_find_ino_for_alloc()
3382 spin_unlock(&ctl->tree_lock); in btrfs_find_ino_for_alloc()
3421 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl; in load_free_ino_cache() local
3448 ret = __load_free_space_cache(root, inode, ctl, path, 0); in load_free_ino_cache()
3466 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl; in btrfs_write_out_ino_cache() local
3475 ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, in btrfs_write_out_ino_cache()
3511 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_add_free_space_entry() local
3525 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
3528 ret = link_free_space(ctl, info); in test_add_free_space_entry()
3529 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
3543 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
3544 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_add_free_space_entry()
3549 add_new_bitmap(ctl, info, offset); in test_add_free_space_entry()
3554 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes); in test_add_free_space_entry()
3557 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
3577 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_check_exists() local
3581 spin_lock(&ctl->tree_lock); in test_check_exists()
3582 info = tree_search_offset(ctl, offset, 0, 0); in test_check_exists()
3584 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_check_exists()
3597 bit_bytes = ctl->unit; in test_check_exists()
3598 ret = search_bitmap(ctl, info, &bit_off, &bit_bytes); in test_check_exists()
3650 spin_unlock(&ctl->tree_lock); in test_check_exists()