25 #define on_build_free_nids(nmi) mutex_is_locked(&nm_i->build_lock)
33 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in available_free_memory()  local
48 		mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >>  in available_free_memory()
50 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);  in available_free_memory()
52 		mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>  in available_free_memory()
54 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);  in available_free_memory()
59 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);  in available_free_memory()
66 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);  in available_free_memory()
71 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);  in available_free_memory()
111 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in get_next_nat_page()  local
127 	set_to_next_nat(nm_i, nid);  in get_next_nat_page()
132 static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)  in __lookup_nat_cache()  argument
134 	return radix_tree_lookup(&nm_i->nat_root, n);  in __lookup_nat_cache()
137 static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,  in __gang_lookup_nat_cache()  argument
140 	return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);  in __gang_lookup_nat_cache()
143 static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)  in __del_from_nat_cache()  argument
146 	radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));  in __del_from_nat_cache()
147 	nm_i->nat_cnt--;  in __del_from_nat_cache()
151 static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,  in __set_nat_cache_dirty()  argument
160 	head = radix_tree_lookup(&nm_i->nat_set_root, set);  in __set_nat_cache_dirty()
168 		f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);  in __set_nat_cache_dirty()
171 	nm_i->dirty_nat_cnt++;  in __set_nat_cache_dirty()
176 static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,  in __clear_nat_cache_dirty()  argument
182 	head = radix_tree_lookup(&nm_i->nat_set_root, set);  in __clear_nat_cache_dirty()
184 		list_move_tail(&ne->list, &nm_i->nat_entries);  in __clear_nat_cache_dirty()
187 		nm_i->dirty_nat_cnt--;  in __clear_nat_cache_dirty()
191 static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,  in __gang_lookup_nat_set()  argument
194 	return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep,  in __gang_lookup_nat_set()
200 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in is_checkpointed_node()  local
204 	down_read(&nm_i->nat_tree_lock);  in is_checkpointed_node()
205 	e = __lookup_nat_cache(nm_i, nid);  in is_checkpointed_node()
208 	up_read(&nm_i->nat_tree_lock);  in is_checkpointed_node()
214 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in has_fsynced_inode()  local
218 	down_read(&nm_i->nat_tree_lock);  in has_fsynced_inode()
219 	e = __lookup_nat_cache(nm_i, ino);  in has_fsynced_inode()
222 	up_read(&nm_i->nat_tree_lock);  in has_fsynced_inode()
228 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in need_inode_block_update()  local
232 	down_read(&nm_i->nat_tree_lock);  in need_inode_block_update()
233 	e = __lookup_nat_cache(nm_i, ino);  in need_inode_block_update()
238 	up_read(&nm_i->nat_tree_lock);  in need_inode_block_update()
242 static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)  in grab_nat_entry()  argument
247 	f2fs_radix_tree_insert(&nm_i->nat_root, nid, new);  in grab_nat_entry()
251 	list_add_tail(&new->list, &nm_i->nat_entries);  in grab_nat_entry()
252 	nm_i->nat_cnt++;  in grab_nat_entry()
256 static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,  in cache_nat_entry()  argument
261 	down_write(&nm_i->nat_tree_lock);  in cache_nat_entry()
262 	e = __lookup_nat_cache(nm_i, nid);  in cache_nat_entry()
264 		e = grab_nat_entry(nm_i, nid);  in cache_nat_entry()
267 	up_write(&nm_i->nat_tree_lock);  in cache_nat_entry()
273 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in set_node_addr()  local
276 	down_write(&nm_i->nat_tree_lock);  in set_node_addr()
277 	e = __lookup_nat_cache(nm_i, ni->nid);  in set_node_addr()
279 		e = grab_nat_entry(nm_i, ni->nid);  in set_node_addr()
312 	__set_nat_cache_dirty(nm_i, e);  in set_node_addr()
315 	e = __lookup_nat_cache(nm_i, ni->ino);  in set_node_addr()
321 	up_write(&nm_i->nat_tree_lock);  in set_node_addr()
326 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in try_to_free_nats()  local
331 	down_write(&nm_i->nat_tree_lock);  in try_to_free_nats()
332 	while (nr_shrink && !list_empty(&nm_i->nat_entries)) {  in try_to_free_nats()
334 		ne = list_first_entry(&nm_i->nat_entries,  in try_to_free_nats()
336 		__del_from_nat_cache(nm_i, ne);  in try_to_free_nats()
339 	up_write(&nm_i->nat_tree_lock);  in try_to_free_nats()
348 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in get_node_info()  local
361 	down_read(&nm_i->nat_tree_lock);  in get_node_info()
362 	e = __lookup_nat_cache(nm_i, nid);  in get_node_info()
368 	up_read(&nm_i->nat_tree_lock);  in get_node_info()
1400 static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,  in __lookup_free_nid_list()  argument
1403 	return radix_tree_lookup(&nm_i->free_nid_root, n);  in __lookup_free_nid_list()
1406 static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i,  in __del_from_free_nid_list()  argument
1410 	radix_tree_delete(&nm_i->free_nid_root, i->nid);  in __del_from_free_nid_list()
1415 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in add_free_nid()  local
1429 		down_read(&nm_i->nat_tree_lock);  in add_free_nid()
1430 		ne = __lookup_nat_cache(nm_i, nid);  in add_free_nid()
1435 		up_read(&nm_i->nat_tree_lock);  in add_free_nid()
1449 	spin_lock(&nm_i->free_nid_list_lock);  in add_free_nid()
1450 	if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {  in add_free_nid()
1451 		spin_unlock(&nm_i->free_nid_list_lock);  in add_free_nid()
1456 	list_add_tail(&i->list, &nm_i->free_nid_list);  in add_free_nid()
1457 	nm_i->fcnt++;  in add_free_nid()
1458 	spin_unlock(&nm_i->free_nid_list_lock);  in add_free_nid()
1463 static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)  in remove_free_nid()  argument
1468 	spin_lock(&nm_i->free_nid_list_lock);  in remove_free_nid()
1469 	i = __lookup_free_nid_list(nm_i, nid);  in remove_free_nid()
1471 		__del_from_free_nid_list(nm_i, i);  in remove_free_nid()
1472 		nm_i->fcnt--;  in remove_free_nid()
1475 	spin_unlock(&nm_i->free_nid_list_lock);  in remove_free_nid()
1484 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in scan_nat_page()  local
1493 		if (unlikely(start_nid >= nm_i->max_nid))  in scan_nat_page()
1507 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in build_free_nids()  local
1511 	nid_t nid = nm_i->next_scan_nid;  in build_free_nids()
1514 	if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK)  in build_free_nids()
1527 		if (unlikely(nid >= nm_i->max_nid))  in build_free_nids()
1535 	nm_i->next_scan_nid = nid;  in build_free_nids()
1545 			remove_free_nid(nm_i, nid);  in build_free_nids()
1557 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in alloc_nid()  local
1560 	if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids))  in alloc_nid()
1563 	spin_lock(&nm_i->free_nid_list_lock);  in alloc_nid()
1566 	if (nm_i->fcnt && !on_build_free_nids(nm_i)) {  in alloc_nid()
1567 		f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list));  in alloc_nid()
1568 		list_for_each_entry(i, &nm_i->free_nid_list, list)  in alloc_nid()
1575 		nm_i->fcnt--;  in alloc_nid()
1576 		spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid()
1579 	spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid()
1582 	mutex_lock(&nm_i->build_lock);  in alloc_nid()
1584 	mutex_unlock(&nm_i->build_lock);  in alloc_nid()
1593 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in alloc_nid_done()  local
1596 	spin_lock(&nm_i->free_nid_list_lock);  in alloc_nid_done()
1597 	i = __lookup_free_nid_list(nm_i, nid);  in alloc_nid_done()
1599 	__del_from_free_nid_list(nm_i, i);  in alloc_nid_done()
1600 	spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid_done()
1610 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in alloc_nid_failed()  local
1617 	spin_lock(&nm_i->free_nid_list_lock);  in alloc_nid_failed()
1618 	i = __lookup_free_nid_list(nm_i, nid);  in alloc_nid_failed()
1621 		__del_from_free_nid_list(nm_i, i);  in alloc_nid_failed()
1625 		nm_i->fcnt++;  in alloc_nid_failed()
1627 	spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid_failed()
1779 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in remove_nats_in_journal()  local
1792 		down_write(&nm_i->nat_tree_lock);  in remove_nats_in_journal()
1793 		ne = __lookup_nat_cache(nm_i, nid);  in remove_nats_in_journal()
1795 			ne = grab_nat_entry(nm_i, nid);  in remove_nats_in_journal()
1798 		__set_nat_cache_dirty(nm_i, ne);  in remove_nats_in_journal()
1799 		up_write(&nm_i->nat_tree_lock);  in remove_nats_in_journal()
1833 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in __flush_nat_entry_set()  local
1887 	down_write(&nm_i->nat_tree_lock);  in __flush_nat_entry_set()
1889 	up_write(&nm_i->nat_tree_lock);  in __flush_nat_entry_set()
1898 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in flush_nat_entries()  local
1907 	if (!nm_i->dirty_nat_cnt)  in flush_nat_entries()
1914 	if (!__has_cursum_space(sum, nm_i->dirty_nat_cnt, NAT_JOURNAL))  in flush_nat_entries()
1917 	down_write(&nm_i->nat_tree_lock);  in flush_nat_entries()
1918 	while ((found = __gang_lookup_nat_set(nm_i,  in flush_nat_entries()
1926 	up_write(&nm_i->nat_tree_lock);  in flush_nat_entries()
1932 	f2fs_bug_on(sbi, nm_i->dirty_nat_cnt);  in flush_nat_entries()
1938 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in init_node_manager()  local
1942 	nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);  in init_node_manager()
1948 	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks;  in init_node_manager()
1951 	nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM;  in init_node_manager()
1952 	nm_i->fcnt = 0;  in init_node_manager()
1953 	nm_i->nat_cnt = 0;  in init_node_manager()
1954 	nm_i->ram_thresh = DEF_RAM_THRESHOLD;  in init_node_manager()
1956 	INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);  in init_node_manager()
1957 	INIT_LIST_HEAD(&nm_i->free_nid_list);  in init_node_manager()
1958 	INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);  in init_node_manager()
1959 	INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);  in init_node_manager()
1960 	INIT_LIST_HEAD(&nm_i->nat_entries);  in init_node_manager()
1962 	mutex_init(&nm_i->build_lock);  in init_node_manager()
1963 	spin_lock_init(&nm_i->free_nid_list_lock);  in init_node_manager()
1964 	init_rwsem(&nm_i->nat_tree_lock);  in init_node_manager()
1966 	nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);  in init_node_manager()
1967 	nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);  in init_node_manager()
1972 	nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,  in init_node_manager()
1974 	if (!nm_i->nat_bitmap)  in init_node_manager()
1997 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in destroy_node_manager()  local
2004 	if (!nm_i)  in destroy_node_manager()
2008 	spin_lock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2009 	list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {  in destroy_node_manager()
2011 		__del_from_free_nid_list(nm_i, i);  in destroy_node_manager()
2012 		nm_i->fcnt--;  in destroy_node_manager()
2013 		spin_unlock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2015 		spin_lock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2017 	f2fs_bug_on(sbi, nm_i->fcnt);  in destroy_node_manager()
2018 	spin_unlock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2021 	down_write(&nm_i->nat_tree_lock);  in destroy_node_manager()
2022 	while ((found = __gang_lookup_nat_cache(nm_i,  in destroy_node_manager()
2028 			__del_from_nat_cache(nm_i, natvec[idx]);  in destroy_node_manager()
2030 	f2fs_bug_on(sbi, nm_i->nat_cnt);  in destroy_node_manager()
2034 	while ((found = __gang_lookup_nat_set(nm_i,  in destroy_node_manager()
2042 			radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);  in destroy_node_manager()
2046 	up_write(&nm_i->nat_tree_lock);  in destroy_node_manager()
2048 	kfree(nm_i->nat_bitmap);  in destroy_node_manager()
2050 	kfree(nm_i);  in destroy_node_manager()