25 #define on_build_free_nids(nmi) mutex_is_locked(&nm_i->build_lock)
33 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in available_free_memory()  local
48 		mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >>  in available_free_memory()
50 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);  in available_free_memory()
52 		mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>  in available_free_memory()
54 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);  in available_free_memory()
59 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);  in available_free_memory()
66 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);  in available_free_memory()
71 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);  in available_free_memory()
111 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in get_next_nat_page()  local
127 	set_to_next_nat(nm_i, nid);  in get_next_nat_page()
132 static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)  in __lookup_nat_cache()  argument
134 	return radix_tree_lookup(&nm_i->nat_root, n);  in __lookup_nat_cache()
137 static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,  in __gang_lookup_nat_cache()  argument
140 	return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);  in __gang_lookup_nat_cache()
143 static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)  in __del_from_nat_cache()  argument
146 	radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));  in __del_from_nat_cache()
147 	nm_i->nat_cnt--;  in __del_from_nat_cache()
151 static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,  in __set_nat_cache_dirty()  argument
160 	head = radix_tree_lookup(&nm_i->nat_set_root, set);  in __set_nat_cache_dirty()
168 		f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);  in __set_nat_cache_dirty()
171 	nm_i->dirty_nat_cnt++;  in __set_nat_cache_dirty()
176 static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,  in __clear_nat_cache_dirty()  argument
182 	head = radix_tree_lookup(&nm_i->nat_set_root, set);  in __clear_nat_cache_dirty()
184 		list_move_tail(&ne->list, &nm_i->nat_entries);  in __clear_nat_cache_dirty()
187 		nm_i->dirty_nat_cnt--;  in __clear_nat_cache_dirty()
191 static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,  in __gang_lookup_nat_set()  argument
194 	return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep,  in __gang_lookup_nat_set()
200 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in need_dentry_mark()  local
204 	down_read(&nm_i->nat_tree_lock);  in need_dentry_mark()
205 	e = __lookup_nat_cache(nm_i, nid);  in need_dentry_mark()
211 	up_read(&nm_i->nat_tree_lock);  in need_dentry_mark()
217 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in is_checkpointed_node()  local
221 	down_read(&nm_i->nat_tree_lock);  in is_checkpointed_node()
222 	e = __lookup_nat_cache(nm_i, nid);  in is_checkpointed_node()
225 	up_read(&nm_i->nat_tree_lock);  in is_checkpointed_node()
231 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in need_inode_block_update()  local
235 	down_read(&nm_i->nat_tree_lock);  in need_inode_block_update()
236 	e = __lookup_nat_cache(nm_i, ino);  in need_inode_block_update()
241 	up_read(&nm_i->nat_tree_lock);  in need_inode_block_update()
245 static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)  in grab_nat_entry()  argument
250 	f2fs_radix_tree_insert(&nm_i->nat_root, nid, new);  in grab_nat_entry()
254 	list_add_tail(&new->list, &nm_i->nat_entries);  in grab_nat_entry()
255 	nm_i->nat_cnt++;  in grab_nat_entry()
259 static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,  in cache_nat_entry()  argument
264 	down_write(&nm_i->nat_tree_lock);  in cache_nat_entry()
265 	e = __lookup_nat_cache(nm_i, nid);  in cache_nat_entry()
267 		e = grab_nat_entry(nm_i, nid);  in cache_nat_entry()
270 	up_write(&nm_i->nat_tree_lock);  in cache_nat_entry()
276 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in set_node_addr()  local
279 	down_write(&nm_i->nat_tree_lock);  in set_node_addr()
280 	e = __lookup_nat_cache(nm_i, ni->nid);  in set_node_addr()
282 		e = grab_nat_entry(nm_i, ni->nid);  in set_node_addr()
311 		if (nm_i->next_scan_nid > ni->nid)  in set_node_addr()
312 			nm_i->next_scan_nid = ni->nid;  in set_node_addr()
319 	__set_nat_cache_dirty(nm_i, e);  in set_node_addr()
323 		e = __lookup_nat_cache(nm_i, ni->ino);  in set_node_addr()
329 	up_write(&nm_i->nat_tree_lock);  in set_node_addr()
334 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in try_to_free_nats()  local
337 	if (!down_write_trylock(&nm_i->nat_tree_lock))  in try_to_free_nats()
340 	while (nr_shrink && !list_empty(&nm_i->nat_entries)) {  in try_to_free_nats()
342 		ne = list_first_entry(&nm_i->nat_entries,  in try_to_free_nats()
344 		__del_from_nat_cache(nm_i, ne);  in try_to_free_nats()
347 	up_write(&nm_i->nat_tree_lock);  in try_to_free_nats()
356 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in get_node_info()  local
369 	down_read(&nm_i->nat_tree_lock);  in get_node_info()
370 	e = __lookup_nat_cache(nm_i, nid);  in get_node_info()
376 	up_read(&nm_i->nat_tree_lock);  in get_node_info()
1414 static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,  in __lookup_free_nid_list()  argument
1417 	return radix_tree_lookup(&nm_i->free_nid_root, n);  in __lookup_free_nid_list()
1420 static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i,  in __del_from_free_nid_list()  argument
1424 	radix_tree_delete(&nm_i->free_nid_root, i->nid);  in __del_from_free_nid_list()
1429 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in add_free_nid()  local
1443 		down_read(&nm_i->nat_tree_lock);  in add_free_nid()
1444 		ne = __lookup_nat_cache(nm_i, nid);  in add_free_nid()
1449 		up_read(&nm_i->nat_tree_lock);  in add_free_nid()
1463 	spin_lock(&nm_i->free_nid_list_lock);  in add_free_nid()
1464 	if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {  in add_free_nid()
1465 		spin_unlock(&nm_i->free_nid_list_lock);  in add_free_nid()
1470 	list_add_tail(&i->list, &nm_i->free_nid_list);  in add_free_nid()
1471 	nm_i->fcnt++;  in add_free_nid()
1472 	spin_unlock(&nm_i->free_nid_list_lock);  in add_free_nid()
1477 static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)  in remove_free_nid()  argument
1482 	spin_lock(&nm_i->free_nid_list_lock);  in remove_free_nid()
1483 	i = __lookup_free_nid_list(nm_i, nid);  in remove_free_nid()
1485 		__del_from_free_nid_list(nm_i, i);  in remove_free_nid()
1486 		nm_i->fcnt--;  in remove_free_nid()
1489 	spin_unlock(&nm_i->free_nid_list_lock);  in remove_free_nid()
1498 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in scan_nat_page()  local
1507 		if (unlikely(start_nid >= nm_i->max_nid))  in scan_nat_page()
1521 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in build_free_nids()  local
1525 	nid_t nid = nm_i->next_scan_nid;  in build_free_nids()
1528 	if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK)  in build_free_nids()
1542 		if (unlikely(nid >= nm_i->max_nid))  in build_free_nids()
1550 	nm_i->next_scan_nid = nid;  in build_free_nids()
1560 			remove_free_nid(nm_i, nid);  in build_free_nids()
1564 	ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),  in build_free_nids()
1565 					nm_i->ra_nid_pages, META_NAT, false);  in build_free_nids()
1575 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in alloc_nid()  local
1578 	if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids))  in alloc_nid()
1581 	spin_lock(&nm_i->free_nid_list_lock);  in alloc_nid()
1584 	if (nm_i->fcnt && !on_build_free_nids(nm_i)) {  in alloc_nid()
1587 		f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list));  in alloc_nid()
1588 		list_for_each_entry(i, &nm_i->free_nid_list, list)  in alloc_nid()
1595 		nm_i->fcnt--;  in alloc_nid()
1596 		spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid()
1606 	spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid()
1609 	mutex_lock(&nm_i->build_lock);  in alloc_nid()
1611 	mutex_unlock(&nm_i->build_lock);  in alloc_nid()
1620 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in alloc_nid_done()  local
1623 	spin_lock(&nm_i->free_nid_list_lock);  in alloc_nid_done()
1624 	i = __lookup_free_nid_list(nm_i, nid);  in alloc_nid_done()
1626 	__del_from_free_nid_list(nm_i, i);  in alloc_nid_done()
1627 	spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid_done()
1637 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in alloc_nid_failed()  local
1644 	spin_lock(&nm_i->free_nid_list_lock);  in alloc_nid_failed()
1645 	i = __lookup_free_nid_list(nm_i, nid);  in alloc_nid_failed()
1648 		__del_from_free_nid_list(nm_i, i);  in alloc_nid_failed()
1652 		nm_i->fcnt++;  in alloc_nid_failed()
1654 	spin_unlock(&nm_i->free_nid_list_lock);  in alloc_nid_failed()
1662 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in try_to_free_nids()  local
1666 	if (!mutex_trylock(&nm_i->build_lock))  in try_to_free_nids()
1669 	spin_lock(&nm_i->free_nid_list_lock);  in try_to_free_nids()
1670 	list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) {  in try_to_free_nids()
1671 		if (nr_shrink <= 0 || nm_i->fcnt <= NAT_ENTRY_PER_BLOCK)  in try_to_free_nids()
1675 		__del_from_free_nid_list(nm_i, i);  in try_to_free_nids()
1677 		nm_i->fcnt--;  in try_to_free_nids()
1680 	spin_unlock(&nm_i->free_nid_list_lock);  in try_to_free_nids()
1681 	mutex_unlock(&nm_i->build_lock);  in try_to_free_nids()
1832 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in remove_nats_in_journal()  local
1845 		down_write(&nm_i->nat_tree_lock);  in remove_nats_in_journal()
1846 		ne = __lookup_nat_cache(nm_i, nid);  in remove_nats_in_journal()
1848 			ne = grab_nat_entry(nm_i, nid);  in remove_nats_in_journal()
1851 		__set_nat_cache_dirty(nm_i, ne);  in remove_nats_in_journal()
1852 		up_write(&nm_i->nat_tree_lock);  in remove_nats_in_journal()
1886 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in __flush_nat_entry_set()  local
1940 	down_write(&nm_i->nat_tree_lock);  in __flush_nat_entry_set()
1942 	up_write(&nm_i->nat_tree_lock);  in __flush_nat_entry_set()
1951 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in flush_nat_entries()  local
1960 	if (!nm_i->dirty_nat_cnt)  in flush_nat_entries()
1967 	if (!__has_cursum_space(sum, nm_i->dirty_nat_cnt, NAT_JOURNAL))  in flush_nat_entries()
1970 	down_write(&nm_i->nat_tree_lock);  in flush_nat_entries()
1971 	while ((found = __gang_lookup_nat_set(nm_i,  in flush_nat_entries()
1979 	up_write(&nm_i->nat_tree_lock);  in flush_nat_entries()
1985 	f2fs_bug_on(sbi, nm_i->dirty_nat_cnt);  in flush_nat_entries()
1991 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in init_node_manager()  local
1995 	nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);  in init_node_manager()
2001 	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks;  in init_node_manager()
2004 	nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM;  in init_node_manager()
2005 	nm_i->fcnt = 0;  in init_node_manager()
2006 	nm_i->nat_cnt = 0;  in init_node_manager()
2007 	nm_i->ram_thresh = DEF_RAM_THRESHOLD;  in init_node_manager()
2008 	nm_i->ra_nid_pages = DEF_RA_NID_PAGES;  in init_node_manager()
2010 	INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);  in init_node_manager()
2011 	INIT_LIST_HEAD(&nm_i->free_nid_list);  in init_node_manager()
2012 	INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);  in init_node_manager()
2013 	INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);  in init_node_manager()
2014 	INIT_LIST_HEAD(&nm_i->nat_entries);  in init_node_manager()
2016 	mutex_init(&nm_i->build_lock);  in init_node_manager()
2017 	spin_lock_init(&nm_i->free_nid_list_lock);  in init_node_manager()
2018 	init_rwsem(&nm_i->nat_tree_lock);  in init_node_manager()
2020 	nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);  in init_node_manager()
2021 	nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);  in init_node_manager()
2026 	nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,  in init_node_manager()
2028 	if (!nm_i->nat_bitmap)  in init_node_manager()
2051 	struct f2fs_nm_info *nm_i = NM_I(sbi);  in destroy_node_manager()  local
2058 	if (!nm_i)  in destroy_node_manager()
2062 	spin_lock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2063 	list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {  in destroy_node_manager()
2065 		__del_from_free_nid_list(nm_i, i);  in destroy_node_manager()
2066 		nm_i->fcnt--;  in destroy_node_manager()
2067 		spin_unlock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2069 		spin_lock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2071 	f2fs_bug_on(sbi, nm_i->fcnt);  in destroy_node_manager()
2072 	spin_unlock(&nm_i->free_nid_list_lock);  in destroy_node_manager()
2075 	down_write(&nm_i->nat_tree_lock);  in destroy_node_manager()
2076 	while ((found = __gang_lookup_nat_cache(nm_i,  in destroy_node_manager()
2082 			__del_from_nat_cache(nm_i, natvec[idx]);  in destroy_node_manager()
2084 	f2fs_bug_on(sbi, nm_i->nat_cnt);  in destroy_node_manager()
2088 	while ((found = __gang_lookup_nat_set(nm_i,  in destroy_node_manager()
2096 			radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);  in destroy_node_manager()
2100 	up_write(&nm_i->nat_tree_lock);  in destroy_node_manager()
2102 	kfree(nm_i->nat_bitmap);  in destroy_node_manager()
2104 	kfree(nm_i);  in destroy_node_manager()