52 	struct f2fs_sb_info *sbi = bio->bi_private;  in f2fs_write_end_io()  local
62 			f2fs_stop_checkpoint(sbi);  in f2fs_write_end_io()
65 		dec_page_count(sbi, F2FS_WRITEBACK);  in f2fs_write_end_io()
68 	if (!get_pages(sbi, F2FS_WRITEBACK) &&  in f2fs_write_end_io()
69 			!list_empty(&sbi->cp_wait.task_list))  in f2fs_write_end_io()
70 		wake_up(&sbi->cp_wait);  in f2fs_write_end_io()
78 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,  in __bio_alloc()  argument
86 	bio->bi_bdev = sbi->sb->s_bdev;  in __bio_alloc()
89 	bio->bi_private = sbi;  in __bio_alloc()
102 		trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);  in __submit_merged_bio()
104 		trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);  in __submit_merged_bio()
110 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,  in f2fs_submit_merged_bio()  argument
116 	io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];  in f2fs_submit_merged_bio()
123 		if (test_opt(sbi, NOBARRIER))  in f2fs_submit_merged_bio()
136 int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_submit_page_bio()  argument
145 	bio = __bio_alloc(sbi, fio->blk_addr, 1, is_read_io(fio->rw));  in f2fs_submit_page_bio()
157 void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_submit_page_mbio()  argument
164 	io = is_read ? &sbi->read_io : &sbi->write_io[btype];  in f2fs_submit_page_mbio()
166 	verify_block_addr(sbi, fio->blk_addr);  in f2fs_submit_page_mbio()
171 		inc_page_count(sbi, F2FS_WRITEBACK);  in f2fs_submit_page_mbio()
178 		int bio_blocks = MAX_BIO_BLOCKS(sbi);  in f2fs_submit_page_mbio()
180 		io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read);  in f2fs_submit_page_mbio()
222 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);  in reserve_new_block()  local
226 	if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))  in reserve_new_block()
364 static struct extent_node *__attach_extent_node(struct f2fs_sb_info *sbi,  in __attach_extent_node()  argument
380 	atomic_inc(&sbi->total_ext_node);  in __attach_extent_node()
384 static void __detach_extent_node(struct f2fs_sb_info *sbi,  in __detach_extent_node()  argument
389 	atomic_dec(&sbi->total_ext_node);  in __detach_extent_node()
395 static struct extent_tree *__find_extent_tree(struct f2fs_sb_info *sbi,  in __find_extent_tree()  argument
400 	down_read(&sbi->extent_tree_lock);  in __find_extent_tree()
401 	et = radix_tree_lookup(&sbi->extent_tree_root, ino);  in __find_extent_tree()
403 		up_read(&sbi->extent_tree_lock);  in __find_extent_tree()
407 	up_read(&sbi->extent_tree_lock);  in __find_extent_tree()
414 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __grab_extent_tree()  local
418 	down_write(&sbi->extent_tree_lock);  in __grab_extent_tree()
419 	et = radix_tree_lookup(&sbi->extent_tree_root, ino);  in __grab_extent_tree()
422 		f2fs_radix_tree_insert(&sbi->extent_tree_root, ino, et);  in __grab_extent_tree()
430 		sbi->total_ext_tree++;  in __grab_extent_tree()
433 	up_write(&sbi->extent_tree_lock);  in __grab_extent_tree()
466 static struct extent_node *__try_back_merge(struct f2fs_sb_info *sbi,  in __try_back_merge()  argument
481 		__detach_extent_node(sbi, et, prev);  in __try_back_merge()
487 static struct extent_node *__try_front_merge(struct f2fs_sb_info *sbi,  in __try_front_merge()  argument
500 		__detach_extent_node(sbi, et, next);  in __try_front_merge()
506 static struct extent_node *__insert_extent_tree(struct f2fs_sb_info *sbi,  in __insert_extent_tree()  argument
520 				f2fs_bug_on(sbi, !den);  in __insert_extent_tree()
524 				*den = __try_back_merge(sbi, et, en);  in __insert_extent_tree()
530 				f2fs_bug_on(sbi, !den);  in __insert_extent_tree()
532 				*den = __try_front_merge(sbi, et, en);  in __insert_extent_tree()
537 			f2fs_bug_on(sbi, 1);  in __insert_extent_tree()
541 	return __attach_extent_node(sbi, et, ei, parent, p);  in __insert_extent_tree()
544 static unsigned int __free_extent_tree(struct f2fs_sb_info *sbi,  in __free_extent_tree()  argument
557 			spin_lock(&sbi->extent_lock);  in __free_extent_tree()
560 			spin_unlock(&sbi->extent_lock);  in __free_extent_tree()
564 			__detach_extent_node(sbi, et, en);  in __free_extent_tree()
576 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_init_extent_tree()  local
593 	en = __insert_extent_tree(sbi, et, &ei, NULL);  in f2fs_init_extent_tree()
597 		spin_lock(&sbi->extent_lock);  in f2fs_init_extent_tree()
598 		list_add_tail(&en->list, &sbi->extent_list);  in f2fs_init_extent_tree()
599 		spin_unlock(&sbi->extent_lock);  in f2fs_init_extent_tree()
609 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_lookup_extent_tree()  local
615 	et = __find_extent_tree(sbi, inode->i_ino);  in f2fs_lookup_extent_tree()
623 		spin_lock(&sbi->extent_lock);  in f2fs_lookup_extent_tree()
625 			list_move_tail(&en->list, &sbi->extent_list);  in f2fs_lookup_extent_tree()
626 		spin_unlock(&sbi->extent_lock);  in f2fs_lookup_extent_tree()
627 		stat_inc_read_hit(sbi->sb);  in f2fs_lookup_extent_tree()
629 	stat_inc_total_hit(sbi->sb);  in f2fs_lookup_extent_tree()
641 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_update_extent_tree()  local
660 	__detach_extent_node(sbi, et, en);  in f2fs_update_extent_tree()
668 			en1 = __insert_extent_tree(sbi, et, &ei, NULL);  in f2fs_update_extent_tree()
676 			en2 = __insert_extent_tree(sbi, et, &ei, NULL);  in f2fs_update_extent_tree()
684 		en3 = __insert_extent_tree(sbi, et, &ei, &den);  in f2fs_update_extent_tree()
688 	spin_lock(&sbi->extent_lock);  in f2fs_update_extent_tree()
697 		list_add_tail(&en1->list, &sbi->extent_list);  in f2fs_update_extent_tree()
699 		list_add_tail(&en2->list, &sbi->extent_list);  in f2fs_update_extent_tree()
702 			list_add_tail(&en3->list, &sbi->extent_list);  in f2fs_update_extent_tree()
704 			list_move_tail(&en3->list, &sbi->extent_list);  in f2fs_update_extent_tree()
708 	spin_unlock(&sbi->extent_lock);  in f2fs_update_extent_tree()
769 void f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink)  in f2fs_shrink_extent_tree()  argument
773 	unsigned long ino = F2FS_ROOT_INO(sbi);  in f2fs_shrink_extent_tree()
779 	if (!test_opt(sbi, EXTENT_CACHE))  in f2fs_shrink_extent_tree()
782 	if (available_free_memory(sbi, EXTENT_CACHE))  in f2fs_shrink_extent_tree()
785 	spin_lock(&sbi->extent_lock);  in f2fs_shrink_extent_tree()
786 	list_for_each_entry_safe(en, tmp, &sbi->extent_list, list) {  in f2fs_shrink_extent_tree()
791 	spin_unlock(&sbi->extent_lock);  in f2fs_shrink_extent_tree()
793 	down_read(&sbi->extent_tree_lock);  in f2fs_shrink_extent_tree()
794 	while ((found = radix_tree_gang_lookup(&sbi->extent_tree_root,  in f2fs_shrink_extent_tree()
804 			node_cnt += __free_extent_tree(sbi, et, false);  in f2fs_shrink_extent_tree()
809 	up_read(&sbi->extent_tree_lock);  in f2fs_shrink_extent_tree()
811 	down_write(&sbi->extent_tree_lock);  in f2fs_shrink_extent_tree()
812 	radix_tree_for_each_slot(slot, &sbi->extent_tree_root, &iter,  in f2fs_shrink_extent_tree()
813 							F2FS_ROOT_INO(sbi)) {  in f2fs_shrink_extent_tree()
817 			radix_tree_delete(&sbi->extent_tree_root, et->ino);  in f2fs_shrink_extent_tree()
819 			sbi->total_ext_tree--;  in f2fs_shrink_extent_tree()
823 	up_write(&sbi->extent_tree_lock);  in f2fs_shrink_extent_tree()
825 	trace_f2fs_shrink_extent_tree(sbi, node_cnt, tree_cnt);  in f2fs_shrink_extent_tree()
830 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_destroy_extent_tree()  local
834 	if (!test_opt(sbi, EXTENT_CACHE))  in f2fs_destroy_extent_tree()
837 	et = __find_extent_tree(sbi, inode->i_ino);  in f2fs_destroy_extent_tree()
843 	node_cnt = __free_extent_tree(sbi, et, true);  in f2fs_destroy_extent_tree()
849 	down_write(&sbi->extent_tree_lock);  in f2fs_destroy_extent_tree()
850 	et = radix_tree_lookup(&sbi->extent_tree_root, inode->i_ino);  in f2fs_destroy_extent_tree()
852 		up_write(&sbi->extent_tree_lock);  in f2fs_destroy_extent_tree()
855 	f2fs_bug_on(sbi, atomic_read(&et->refcount) || et->count);  in f2fs_destroy_extent_tree()
856 	radix_tree_delete(&sbi->extent_tree_root, inode->i_ino);  in f2fs_destroy_extent_tree()
858 	sbi->total_ext_tree--;  in f2fs_destroy_extent_tree()
859 	up_write(&sbi->extent_tree_lock);  in f2fs_destroy_extent_tree()
1118 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);  in __allocate_data_block()  local
1132 	if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))  in __allocate_data_block()
1136 	get_node_info(sbi, dn->nid, &ni);  in __allocate_data_block()
1142 	allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,  in __allocate_data_block()
1160 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __allocate_data_blocks()  local
1168 		f2fs_balance_fs(sbi);  in __allocate_data_blocks()
1169 		f2fs_lock_op(sbi);  in __allocate_data_blocks()
1197 		f2fs_unlock_op(sbi);  in __allocate_data_blocks()
1206 	f2fs_unlock_op(sbi);  in __allocate_data_blocks()
1420 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_write_data_page()  local
1447 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))  in f2fs_write_data_page()
1452 			available_free_memory(sbi, BASE_CHECK))  in f2fs_write_data_page()
1457 		if (unlikely(f2fs_cp_error(sbi)))  in f2fs_write_data_page()
1464 	if (unlikely(f2fs_cp_error(sbi))) {  in f2fs_write_data_page()
1471 	else if (has_not_enough_free_secs(sbi, 0))  in f2fs_write_data_page()
1475 	f2fs_lock_op(sbi);  in f2fs_write_data_page()
1480 	f2fs_unlock_op(sbi);  in f2fs_write_data_page()
1492 		f2fs_balance_fs(sbi);  in f2fs_write_data_page()
1494 		f2fs_submit_merged_bio(sbi, DATA, WRITE);  in f2fs_write_data_page()
1515 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_write_data_pages()  local
1527 			get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&  in f2fs_write_data_pages()
1528 			available_free_memory(sbi, DIRTY_DENTS))  in f2fs_write_data_pages()
1532 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))  in f2fs_write_data_pages()
1535 	diff = nr_pages_to_write(sbi, DATA, wbc);  in f2fs_write_data_pages()
1538 		mutex_lock(&sbi->writepages);  in f2fs_write_data_pages()
1543 		mutex_unlock(&sbi->writepages);  in f2fs_write_data_pages()
1545 	f2fs_submit_merged_bio(sbi, DATA, WRITE);  in f2fs_write_data_pages()
1572 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_write_begin()  local
1580 	f2fs_balance_fs(sbi);  in f2fs_write_begin()
1601 	f2fs_lock_op(sbi);  in f2fs_write_begin()
1604 	ipage = get_node_page(sbi, inode->i_ino);  in f2fs_write_begin()
1628 	f2fs_unlock_op(sbi);  in f2fs_write_begin()
1652 		err = f2fs_submit_page_bio(sbi, page, &fio);  in f2fs_write_begin()
1675 	f2fs_unlock_op(sbi);  in f2fs_write_begin()
1757 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_invalidate_page()  local
1759 	if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&  in f2fs_invalidate_page()
1764 		if (inode->i_ino == F2FS_META_INO(sbi))  in f2fs_invalidate_page()
1765 			dec_page_count(sbi, F2FS_DIRTY_META);  in f2fs_invalidate_page()
1766 		else if (inode->i_ino == F2FS_NODE_INO(sbi))  in f2fs_invalidate_page()
1767 			dec_page_count(sbi, F2FS_DIRTY_NODES);  in f2fs_invalidate_page()
1821 void init_extent_cache_info(struct f2fs_sb_info *sbi)  in init_extent_cache_info()  argument
1823 	INIT_RADIX_TREE(&sbi->extent_tree_root, GFP_NOIO);  in init_extent_cache_info()
1824 	init_rwsem(&sbi->extent_tree_lock);  in init_extent_cache_info()
1825 	INIT_LIST_HEAD(&sbi->extent_list);  in init_extent_cache_info()
1826 	spin_lock_init(&sbi->extent_lock);  in init_extent_cache_info()
1827 	sbi->total_ext_tree = 0;  in init_extent_cache_info()
1828 	atomic_set(&sbi->total_ext_node, 0);  in init_extent_cache_info()