Lines Matching refs:sbi
216 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in commit_inmem_pages() local
221 .sbi = sbi, in commit_inmem_pages()
236 f2fs_balance_fs(sbi); in commit_inmem_pages()
237 f2fs_lock_op(sbi); in commit_inmem_pages()
273 f2fs_unlock_op(sbi); in commit_inmem_pages()
275 f2fs_submit_merged_bio(sbi, DATA, WRITE); in commit_inmem_pages()
284 void f2fs_balance_fs(struct f2fs_sb_info *sbi) in f2fs_balance_fs() argument
290 if (has_not_enough_free_secs(sbi, 0)) { in f2fs_balance_fs()
291 mutex_lock(&sbi->gc_mutex); in f2fs_balance_fs()
292 f2fs_gc(sbi, false); in f2fs_balance_fs()
296 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) in f2fs_balance_fs_bg() argument
299 if (!available_free_memory(sbi, EXTENT_CACHE)) in f2fs_balance_fs_bg()
300 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); in f2fs_balance_fs_bg()
303 if (!available_free_memory(sbi, NAT_ENTRIES)) in f2fs_balance_fs_bg()
304 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); in f2fs_balance_fs_bg()
306 if (!available_free_memory(sbi, FREE_NIDS)) in f2fs_balance_fs_bg()
307 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES); in f2fs_balance_fs_bg()
310 if (!available_free_memory(sbi, NAT_ENTRIES) || in f2fs_balance_fs_bg()
311 excess_prefree_segs(sbi) || in f2fs_balance_fs_bg()
312 !available_free_memory(sbi, INO_ENTRIES) || in f2fs_balance_fs_bg()
313 jiffies > sbi->cp_expires) in f2fs_balance_fs_bg()
314 f2fs_sync_fs(sbi->sb, true); in f2fs_balance_fs_bg()
319 struct f2fs_sb_info *sbi = data; in issue_flush_thread() local
320 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; in issue_flush_thread()
336 bio->bi_bdev = sbi->sb->s_bdev; in issue_flush_thread()
353 int f2fs_issue_flush(struct f2fs_sb_info *sbi) in f2fs_issue_flush() argument
355 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; in f2fs_issue_flush()
358 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER), in f2fs_issue_flush()
359 test_opt(sbi, FLUSH_MERGE)); in f2fs_issue_flush()
361 if (test_opt(sbi, NOBARRIER)) in f2fs_issue_flush()
364 if (!test_opt(sbi, FLUSH_MERGE)) { in f2fs_issue_flush()
368 bio->bi_bdev = sbi->sb->s_bdev; in f2fs_issue_flush()
386 int create_flush_cmd_control(struct f2fs_sb_info *sbi) in create_flush_cmd_control() argument
388 dev_t dev = sbi->sb->s_bdev->bd_dev; in create_flush_cmd_control()
397 SM_I(sbi)->cmd_control_info = fcc; in create_flush_cmd_control()
398 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, in create_flush_cmd_control()
403 SM_I(sbi)->cmd_control_info = NULL; in create_flush_cmd_control()
410 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi) in destroy_flush_cmd_control() argument
412 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; in destroy_flush_cmd_control()
417 SM_I(sbi)->cmd_control_info = NULL; in destroy_flush_cmd_control()
420 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __locate_dirty_segment() argument
423 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __locate_dirty_segment()
426 if (IS_CURSEG(sbi, segno)) in __locate_dirty_segment()
433 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __locate_dirty_segment()
437 f2fs_bug_on(sbi, 1); in __locate_dirty_segment()
445 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __remove_dirty_segment() argument
448 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __remove_dirty_segment()
454 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __remove_dirty_segment()
460 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0) in __remove_dirty_segment()
461 clear_bit(GET_SECNO(sbi, segno), in __remove_dirty_segment()
471 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) in locate_dirty_segment() argument
473 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in locate_dirty_segment()
476 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) in locate_dirty_segment()
481 valid_blocks = get_valid_blocks(sbi, segno, 0); in locate_dirty_segment()
484 __locate_dirty_segment(sbi, segno, PRE); in locate_dirty_segment()
485 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
486 } else if (valid_blocks < sbi->blocks_per_seg) { in locate_dirty_segment()
487 __locate_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
490 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
496 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, in f2fs_issue_discard() argument
506 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); in f2fs_issue_discard()
507 offset = GET_BLKOFF_FROM_SEG0(sbi, i); in f2fs_issue_discard()
510 sbi->discard_blks--; in f2fs_issue_discard()
512 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen); in f2fs_issue_discard()
513 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0); in f2fs_issue_discard()
516 bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr) in discard_next_dnode() argument
520 if (test_opt(sbi, DISCARD)) { in discard_next_dnode()
521 struct seg_entry *se = get_seg_entry(sbi, in discard_next_dnode()
522 GET_SEGNO(sbi, blkaddr)); in discard_next_dnode()
523 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in discard_next_dnode()
528 err = f2fs_issue_discard(sbi, blkaddr, 1); in discard_next_dnode()
532 update_meta_page(sbi, NULL, blkaddr); in discard_next_dnode()
538 static void __add_discard_entry(struct f2fs_sb_info *sbi, in __add_discard_entry() argument
542 struct list_head *head = &SM_I(sbi)->discard_list; in __add_discard_entry()
547 if (START_BLOCK(sbi, cpc->trim_start) + start == in __add_discard_entry()
556 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start; in __add_discard_entry()
560 SM_I(sbi)->nr_discards += end - start; in __add_discard_entry()
563 static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc) in add_discard_addrs() argument
566 int max_blocks = sbi->blocks_per_seg; in add_discard_addrs()
567 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); in add_discard_addrs()
571 unsigned long *dmap = SIT_I(sbi)->tmp_map; in add_discard_addrs()
580 if (!test_opt(sbi, DISCARD) || !se->valid_blocks || in add_discard_addrs()
581 SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards) in add_discard_addrs()
590 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) { in add_discard_addrs()
596 __add_discard_entry(sbi, cpc, se, start, end); in add_discard_addrs()
600 void release_discard_addrs(struct f2fs_sb_info *sbi) in release_discard_addrs() argument
602 struct list_head *head = &(SM_I(sbi)->discard_list); in release_discard_addrs()
615 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) in set_prefree_as_free_segments() argument
617 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in set_prefree_as_free_segments()
621 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) in set_prefree_as_free_segments()
622 __set_test_and_free(sbi, segno); in set_prefree_as_free_segments()
626 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc) in clear_prefree_segments() argument
628 struct list_head *head = &(SM_I(sbi)->discard_list); in clear_prefree_segments()
630 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in clear_prefree_segments()
638 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); in clear_prefree_segments()
639 if (start >= MAIN_SEGS(sbi)) in clear_prefree_segments()
641 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), in clear_prefree_segments()
649 if (!test_opt(sbi, DISCARD)) in clear_prefree_segments()
652 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), in clear_prefree_segments()
653 (end - start) << sbi->log_blocks_per_seg); in clear_prefree_segments()
661 f2fs_issue_discard(sbi, entry->blkaddr, entry->len); in clear_prefree_segments()
665 SM_I(sbi)->nr_discards -= entry->len; in clear_prefree_segments()
670 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) in __mark_sit_entry_dirty() argument
672 struct sit_info *sit_i = SIT_I(sbi); in __mark_sit_entry_dirty()
682 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, in __set_sit_entry_type() argument
685 struct seg_entry *se = get_seg_entry(sbi, segno); in __set_sit_entry_type()
688 __mark_sit_entry_dirty(sbi, segno); in __set_sit_entry_type()
691 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) in update_sit_entry() argument
697 segno = GET_SEGNO(sbi, blkaddr); in update_sit_entry()
699 se = get_seg_entry(sbi, segno); in update_sit_entry()
701 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in update_sit_entry()
703 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || in update_sit_entry()
704 (new_vblocks > sbi->blocks_per_seg))); in update_sit_entry()
707 se->mtime = get_mtime(sbi); in update_sit_entry()
708 SIT_I(sbi)->max_mtime = se->mtime; in update_sit_entry()
713 f2fs_bug_on(sbi, 1); in update_sit_entry()
715 sbi->discard_blks--; in update_sit_entry()
718 f2fs_bug_on(sbi, 1); in update_sit_entry()
720 sbi->discard_blks++; in update_sit_entry()
725 __mark_sit_entry_dirty(sbi, segno); in update_sit_entry()
728 SIT_I(sbi)->written_valid_blocks += del; in update_sit_entry()
730 if (sbi->segs_per_sec > 1) in update_sit_entry()
731 get_sec_entry(sbi, segno)->valid_blocks += del; in update_sit_entry()
734 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new) in refresh_sit_entry() argument
736 update_sit_entry(sbi, new, 1); in refresh_sit_entry()
737 if (GET_SEGNO(sbi, old) != NULL_SEGNO) in refresh_sit_entry()
738 update_sit_entry(sbi, old, -1); in refresh_sit_entry()
740 locate_dirty_segment(sbi, GET_SEGNO(sbi, old)); in refresh_sit_entry()
741 locate_dirty_segment(sbi, GET_SEGNO(sbi, new)); in refresh_sit_entry()
744 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) in invalidate_blocks() argument
746 unsigned int segno = GET_SEGNO(sbi, addr); in invalidate_blocks()
747 struct sit_info *sit_i = SIT_I(sbi); in invalidate_blocks()
749 f2fs_bug_on(sbi, addr == NULL_ADDR); in invalidate_blocks()
756 update_sit_entry(sbi, addr, -1); in invalidate_blocks()
759 locate_dirty_segment(sbi, segno); in invalidate_blocks()
764 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) in is_checkpointed_data() argument
766 struct sit_info *sit_i = SIT_I(sbi); in is_checkpointed_data()
776 segno = GET_SEGNO(sbi, blkaddr); in is_checkpointed_data()
777 se = get_seg_entry(sbi, segno); in is_checkpointed_data()
778 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in is_checkpointed_data()
791 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, in __add_sum_entry() argument
794 struct curseg_info *curseg = CURSEG_I(sbi, type); in __add_sum_entry()
803 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) in npages_for_summary_flush() argument
809 if (sbi->ckpt->alloc_type[i] == SSR) in npages_for_summary_flush()
810 valid_sum_count += sbi->blocks_per_seg; in npages_for_summary_flush()
814 F2FS_CKPT(sbi)->cur_data_blkoff[i]); in npages_for_summary_flush()
816 valid_sum_count += curseg_blkoff(sbi, i); in npages_for_summary_flush()
833 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) in get_sum_page() argument
835 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); in get_sum_page()
838 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) in update_meta_page() argument
840 struct page *page = grab_meta_page(sbi, blk_addr); in update_meta_page()
851 static void write_sum_page(struct f2fs_sb_info *sbi, in write_sum_page() argument
854 update_meta_page(sbi, (void *)sum_blk, blk_addr); in write_sum_page()
857 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) in is_next_segment_free() argument
859 struct curseg_info *curseg = CURSEG_I(sbi, type); in is_next_segment_free()
861 struct free_segmap_info *free_i = FREE_I(sbi); in is_next_segment_free()
863 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) in is_next_segment_free()
872 static void get_new_segment(struct f2fs_sb_info *sbi, in get_new_segment() argument
875 struct free_segmap_info *free_i = FREE_I(sbi); in get_new_segment()
877 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; in get_new_segment()
878 unsigned int hint = *newseg / sbi->segs_per_sec; in get_new_segment()
879 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg); in get_new_segment()
887 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { in get_new_segment()
889 MAIN_SEGS(sbi), *newseg + 1); in get_new_segment()
890 if (segno - *newseg < sbi->segs_per_sec - in get_new_segment()
891 (*newseg % sbi->segs_per_sec)) in get_new_segment()
895 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); in get_new_segment()
896 if (secno >= MAIN_SECS(sbi)) { in get_new_segment()
899 MAIN_SECS(sbi), 0); in get_new_segment()
900 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); in get_new_segment()
915 MAIN_SECS(sbi), 0); in get_new_segment()
916 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); in get_new_segment()
922 segno = secno * sbi->segs_per_sec; in get_new_segment()
923 zoneno = secno / sbi->secs_per_zone; in get_new_segment()
928 if (sbi->secs_per_zone == 1) in get_new_segment()
939 if (CURSEG_I(sbi, i)->zone == zoneno) in get_new_segment()
945 hint = zoneno * sbi->secs_per_zone - 1; in get_new_segment()
949 hint = (zoneno + 1) * sbi->secs_per_zone; in get_new_segment()
955 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); in get_new_segment()
956 __set_inuse(sbi, segno); in get_new_segment()
961 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) in reset_curseg() argument
963 struct curseg_info *curseg = CURSEG_I(sbi, type); in reset_curseg()
967 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno); in reset_curseg()
977 __set_sit_entry_type(sbi, type, curseg->segno, modified); in reset_curseg()
984 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) in new_curseg() argument
986 struct curseg_info *curseg = CURSEG_I(sbi, type); in new_curseg()
990 write_sum_page(sbi, curseg->sum_blk, in new_curseg()
991 GET_SUM_BLOCK(sbi, segno)); in new_curseg()
995 if (test_opt(sbi, NOHEAP)) in new_curseg()
998 get_new_segment(sbi, &segno, new_sec, dir); in new_curseg()
1000 reset_curseg(sbi, type, 1); in new_curseg()
1004 static void __next_free_blkoff(struct f2fs_sb_info *sbi, in __next_free_blkoff() argument
1007 struct seg_entry *se = get_seg_entry(sbi, seg->segno); in __next_free_blkoff()
1009 unsigned long *target_map = SIT_I(sbi)->tmp_map; in __next_free_blkoff()
1017 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); in __next_free_blkoff()
1027 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, in __refresh_next_blkoff() argument
1031 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); in __refresh_next_blkoff()
1040 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse) in change_curseg() argument
1042 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in change_curseg()
1043 struct curseg_info *curseg = CURSEG_I(sbi, type); in change_curseg()
1048 write_sum_page(sbi, curseg->sum_blk, in change_curseg()
1049 GET_SUM_BLOCK(sbi, curseg->segno)); in change_curseg()
1050 __set_test_and_inuse(sbi, new_segno); in change_curseg()
1053 __remove_dirty_segment(sbi, new_segno, PRE); in change_curseg()
1054 __remove_dirty_segment(sbi, new_segno, DIRTY); in change_curseg()
1057 reset_curseg(sbi, type, 1); in change_curseg()
1059 __next_free_blkoff(sbi, curseg, 0); in change_curseg()
1062 sum_page = get_sum_page(sbi, new_segno); in change_curseg()
1069 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) in get_ssr_segment() argument
1071 struct curseg_info *curseg = CURSEG_I(sbi, type); in get_ssr_segment()
1072 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; in get_ssr_segment()
1074 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0)) in get_ssr_segment()
1075 return v_ops->get_victim(sbi, in get_ssr_segment()
1080 if (v_ops->get_victim(sbi, &(curseg)->next_segno, in get_ssr_segment()
1090 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, in allocate_segment_by_default() argument
1093 struct curseg_info *curseg = CURSEG_I(sbi, type); in allocate_segment_by_default()
1096 new_curseg(sbi, type, true); in allocate_segment_by_default()
1098 new_curseg(sbi, type, false); in allocate_segment_by_default()
1099 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) in allocate_segment_by_default()
1100 new_curseg(sbi, type, false); in allocate_segment_by_default()
1101 else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) in allocate_segment_by_default()
1102 change_curseg(sbi, type, true); in allocate_segment_by_default()
1104 new_curseg(sbi, type, false); in allocate_segment_by_default()
1106 stat_inc_seg_type(sbi, curseg); in allocate_segment_by_default()
1109 static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type) in __allocate_new_segments() argument
1111 struct curseg_info *curseg = CURSEG_I(sbi, type); in __allocate_new_segments()
1115 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true); in __allocate_new_segments()
1116 locate_dirty_segment(sbi, old_segno); in __allocate_new_segments()
1119 void allocate_new_segments(struct f2fs_sb_info *sbi) in allocate_new_segments() argument
1124 __allocate_new_segments(sbi, i); in allocate_new_segments()
1131 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) in f2fs_trim_fs() argument
1138 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) in f2fs_trim_fs()
1142 if (end <= MAIN_BLKADDR(sbi)) in f2fs_trim_fs()
1146 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); in f2fs_trim_fs()
1147 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : in f2fs_trim_fs()
1148 GET_SEGNO(sbi, end); in f2fs_trim_fs()
1156 if (sbi->discard_blks == 0) in f2fs_trim_fs()
1158 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi)) in f2fs_trim_fs()
1163 BATCHED_TRIM_SEGMENTS(sbi), in f2fs_trim_fs()
1164 sbi->segs_per_sec) - 1, end_segno); in f2fs_trim_fs()
1166 mutex_lock(&sbi->gc_mutex); in f2fs_trim_fs()
1167 write_checkpoint(sbi, &cpc); in f2fs_trim_fs()
1168 mutex_unlock(&sbi->gc_mutex); in f2fs_trim_fs()
1175 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) in __has_curseg_space() argument
1177 struct curseg_info *curseg = CURSEG_I(sbi, type); in __has_curseg_space()
1178 if (curseg->next_blkoff < sbi->blocks_per_seg) in __has_curseg_space()
1242 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, in allocate_data_block() argument
1246 struct sit_info *sit_i = SIT_I(sbi); in allocate_data_block()
1252 curseg = CURSEG_I(sbi, type); in allocate_data_block()
1259 !has_not_enough_free_secs(sbi, 0)) in allocate_data_block()
1260 __allocate_new_segments(sbi, type); in allocate_data_block()
1262 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); in allocate_data_block()
1269 __add_sum_entry(sbi, type, sum); in allocate_data_block()
1271 __refresh_next_blkoff(sbi, curseg); in allocate_data_block()
1273 stat_inc_block_count(sbi, curseg); in allocate_data_block()
1275 if (!__has_curseg_space(sbi, type)) in allocate_data_block()
1276 sit_i->s_ops->allocate_segment(sbi, type, false); in allocate_data_block()
1281 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr); in allocate_data_block()
1286 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); in allocate_data_block()
1295 allocate_data_block(fio->sbi, fio->page, fio->blk_addr, in do_write_page()
1302 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) in write_meta_page() argument
1305 .sbi = sbi, in write_meta_page()
1313 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) in write_meta_page()
1330 struct f2fs_sb_info *sbi = fio->sbi; in write_data_page() local
1334 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); in write_data_page()
1335 get_node_info(sbi, dn->nid, &ni); in write_data_page()
1343 stat_inc_inplace_blocks(fio->sbi); in rewrite_data_page()
1347 static void __f2fs_replace_block(struct f2fs_sb_info *sbi, in __f2fs_replace_block() argument
1352 struct sit_info *sit_i = SIT_I(sbi); in __f2fs_replace_block()
1359 segno = GET_SEGNO(sbi, new_blkaddr); in __f2fs_replace_block()
1360 se = get_seg_entry(sbi, segno); in __f2fs_replace_block()
1365 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { in __f2fs_replace_block()
1372 if (!IS_CURSEG(sbi, segno)) in __f2fs_replace_block()
1376 curseg = CURSEG_I(sbi, type); in __f2fs_replace_block()
1387 change_curseg(sbi, type, true); in __f2fs_replace_block()
1390 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); in __f2fs_replace_block()
1391 __add_sum_entry(sbi, type, sum); in __f2fs_replace_block()
1394 update_sit_entry(sbi, new_blkaddr, 1); in __f2fs_replace_block()
1395 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) in __f2fs_replace_block()
1396 update_sit_entry(sbi, old_blkaddr, -1); in __f2fs_replace_block()
1398 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in __f2fs_replace_block()
1399 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); in __f2fs_replace_block()
1401 locate_dirty_segment(sbi, old_cursegno); in __f2fs_replace_block()
1406 change_curseg(sbi, type, true); in __f2fs_replace_block()
1415 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, in f2fs_replace_block() argument
1423 __f2fs_replace_block(sbi, &sum, old_addr, new_addr, recover_curseg); in f2fs_replace_block()
1430 static inline bool is_merged_page(struct f2fs_sb_info *sbi, in is_merged_page() argument
1434 struct f2fs_bio_info *io = &sbi->write_io[btype]; in is_merged_page()
1472 struct f2fs_sb_info *sbi = F2FS_P_SB(page); in f2fs_wait_on_page_writeback() local
1474 if (is_merged_page(sbi, page, type)) in f2fs_wait_on_page_writeback()
1475 f2fs_submit_merged_bio(sbi, type, WRITE); in f2fs_wait_on_page_writeback()
1480 void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi, in f2fs_wait_on_encrypted_page_writeback() argument
1488 f2fs_bug_on(sbi, blkaddr == NULL_ADDR); in f2fs_wait_on_encrypted_page_writeback()
1490 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); in f2fs_wait_on_encrypted_page_writeback()
1497 static int read_compacted_summaries(struct f2fs_sb_info *sbi) in read_compacted_summaries() argument
1499 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_compacted_summaries()
1506 start = start_sum_block(sbi); in read_compacted_summaries()
1508 page = get_meta_page(sbi, start++); in read_compacted_summaries()
1512 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in read_compacted_summaries()
1516 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in read_compacted_summaries()
1526 seg_i = CURSEG_I(sbi, i); in read_compacted_summaries()
1530 reset_curseg(sbi, i, 0); in read_compacted_summaries()
1535 blk_off = sbi->blocks_per_seg; in read_compacted_summaries()
1549 page = get_meta_page(sbi, start++); in read_compacted_summaries()
1558 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) in read_normal_summaries() argument
1560 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_normal_summaries()
1573 if (__exist_node_summaries(sbi)) in read_normal_summaries()
1574 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); in read_normal_summaries()
1576 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); in read_normal_summaries()
1582 if (__exist_node_summaries(sbi)) in read_normal_summaries()
1583 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, in read_normal_summaries()
1586 blk_addr = GET_SUM_BLOCK(sbi, segno); in read_normal_summaries()
1589 new = get_meta_page(sbi, blk_addr); in read_normal_summaries()
1593 if (__exist_node_summaries(sbi)) { in read_normal_summaries()
1596 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { in read_normal_summaries()
1603 err = restore_node_summary(sbi, segno, sum); in read_normal_summaries()
1612 curseg = CURSEG_I(sbi, type); in read_normal_summaries()
1616 reset_curseg(sbi, type, 0); in read_normal_summaries()
1624 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) in restore_curseg_summaries() argument
1629 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) { in restore_curseg_summaries()
1630 int npages = npages_for_summary_flush(sbi, true); in restore_curseg_summaries()
1633 ra_meta_pages(sbi, start_sum_block(sbi), npages, in restore_curseg_summaries()
1637 if (read_compacted_summaries(sbi)) in restore_curseg_summaries()
1642 if (__exist_node_summaries(sbi)) in restore_curseg_summaries()
1643 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type), in restore_curseg_summaries()
1647 err = read_normal_summaries(sbi, type); in restore_curseg_summaries()
1655 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) in write_compacted_summaries() argument
1664 page = grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
1668 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in write_compacted_summaries()
1673 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in write_compacted_summaries()
1681 seg_i = CURSEG_I(sbi, i); in write_compacted_summaries()
1682 if (sbi->ckpt->alloc_type[i] == SSR) in write_compacted_summaries()
1683 blkoff = sbi->blocks_per_seg; in write_compacted_summaries()
1685 blkoff = curseg_blkoff(sbi, i); in write_compacted_summaries()
1689 page = grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
1712 static void write_normal_summaries(struct f2fs_sb_info *sbi, in write_normal_summaries() argument
1722 struct curseg_info *sum = CURSEG_I(sbi, i); in write_normal_summaries()
1724 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type)); in write_normal_summaries()
1729 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in write_data_summaries() argument
1731 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) in write_data_summaries()
1732 write_compacted_summaries(sbi, start_blk); in write_data_summaries()
1734 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); in write_data_summaries()
1737 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in write_node_summaries() argument
1739 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); in write_node_summaries()
1764 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, in get_current_sit_page() argument
1767 return get_meta_page(sbi, current_sit_addr(sbi, segno)); in get_current_sit_page()
1770 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, in get_next_sit_page() argument
1773 struct sit_info *sit_i = SIT_I(sbi); in get_next_sit_page()
1778 src_off = current_sit_addr(sbi, start); in get_next_sit_page()
1779 dst_off = next_sit_addr(sbi, src_off); in get_next_sit_page()
1782 src_page = get_meta_page(sbi, src_off); in get_next_sit_page()
1783 dst_page = grab_meta_page(sbi, dst_off); in get_next_sit_page()
1784 f2fs_bug_on(sbi, PageDirty(src_page)); in get_next_sit_page()
1849 static void add_sits_in_set(struct f2fs_sb_info *sbi) in add_sits_in_set() argument
1851 struct f2fs_sm_info *sm_info = SM_I(sbi); in add_sits_in_set()
1853 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; in add_sits_in_set()
1856 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) in add_sits_in_set()
1860 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) in remove_sits_in_journal() argument
1862 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in remove_sits_in_journal()
1871 dirtied = __mark_sit_entry_dirty(sbi, segno); in remove_sits_in_journal()
1874 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); in remove_sits_in_journal()
1883 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) in flush_sit_entries() argument
1885 struct sit_info *sit_i = SIT_I(sbi); in flush_sit_entries()
1887 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in flush_sit_entries()
1890 struct list_head *head = &SM_I(sbi)->sit_entry_set; in flush_sit_entries()
1904 add_sits_in_set(sbi); in flush_sit_entries()
1912 remove_sits_in_journal(sbi); in flush_sit_entries()
1924 (unsigned long)MAIN_SEGS(sbi)); in flush_sit_entries()
1932 page = get_next_sit_page(sbi, start_segno); in flush_sit_entries()
1940 se = get_seg_entry(sbi, segno); in flush_sit_entries()
1945 add_discard_addrs(sbi, cpc); in flush_sit_entries()
1951 f2fs_bug_on(sbi, offset < 0); in flush_sit_entries()
1970 f2fs_bug_on(sbi, ses->entry_cnt); in flush_sit_entries()
1974 f2fs_bug_on(sbi, !list_empty(head)); in flush_sit_entries()
1975 f2fs_bug_on(sbi, sit_i->dirty_sentries); in flush_sit_entries()
1979 add_discard_addrs(sbi, cpc); in flush_sit_entries()
1984 set_prefree_as_free_segments(sbi); in flush_sit_entries()
1987 static int build_sit_info(struct f2fs_sb_info *sbi) in build_sit_info() argument
1989 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in build_sit_info()
1990 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in build_sit_info()
2001 SM_I(sbi)->sit_info = sit_i; in build_sit_info()
2003 sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) * in build_sit_info()
2008 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_sit_info()
2013 for (start = 0; start < MAIN_SEGS(sbi); start++) { in build_sit_info()
2030 if (sbi->segs_per_sec > 1) { in build_sit_info()
2031 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) * in build_sit_info()
2041 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); in build_sit_info()
2042 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); in build_sit_info()
2052 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; in build_sit_info()
2058 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); in build_sit_info()
2064 static int build_free_segmap(struct f2fs_sb_info *sbi) in build_free_segmap() argument
2074 SM_I(sbi)->free_info = free_i; in build_free_segmap()
2076 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_free_segmap()
2081 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in build_free_segmap()
2091 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); in build_free_segmap()
2098 static int build_curseg(struct f2fs_sb_info *sbi) in build_curseg() argument
2107 SM_I(sbi)->curseg_array = array; in build_curseg()
2117 return restore_curseg_summaries(sbi); in build_curseg()
2120 static void build_sit_entries(struct f2fs_sb_info *sbi) in build_sit_entries() argument
2122 struct sit_info *sit_i = SIT_I(sbi); in build_sit_entries()
2123 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in build_sit_entries()
2125 int sit_blk_cnt = SIT_BLK_CNT(sbi); in build_sit_entries()
2128 int nrpages = MAX_BIO_BLOCKS(sbi); in build_sit_entries()
2131 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true); in build_sit_entries()
2136 for (; start < end && start < MAIN_SEGS(sbi); start++) { in build_sit_entries()
2153 page = get_current_sit_page(sbi, start); in build_sit_entries()
2158 check_block_count(sbi, start, &sit); in build_sit_entries()
2163 sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks; in build_sit_entries()
2165 if (sbi->segs_per_sec > 1) { in build_sit_entries()
2166 struct sec_entry *e = get_sec_entry(sbi, start); in build_sit_entries()
2174 static void init_free_segmap(struct f2fs_sb_info *sbi) in init_free_segmap() argument
2179 for (start = 0; start < MAIN_SEGS(sbi); start++) { in init_free_segmap()
2180 struct seg_entry *sentry = get_seg_entry(sbi, start); in init_free_segmap()
2182 __set_free(sbi, start); in init_free_segmap()
2187 struct curseg_info *curseg_t = CURSEG_I(sbi, type); in init_free_segmap()
2188 __set_test_and_inuse(sbi, curseg_t->segno); in init_free_segmap()
2192 static void init_dirty_segmap(struct f2fs_sb_info *sbi) in init_dirty_segmap() argument
2194 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_dirty_segmap()
2195 struct free_segmap_info *free_i = FREE_I(sbi); in init_dirty_segmap()
2201 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); in init_dirty_segmap()
2202 if (segno >= MAIN_SEGS(sbi)) in init_dirty_segmap()
2205 valid_blocks = get_valid_blocks(sbi, segno, 0); in init_dirty_segmap()
2206 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) in init_dirty_segmap()
2208 if (valid_blocks > sbi->blocks_per_seg) { in init_dirty_segmap()
2209 f2fs_bug_on(sbi, 1); in init_dirty_segmap()
2213 __locate_dirty_segment(sbi, segno, DIRTY); in init_dirty_segmap()
2218 static int init_victim_secmap(struct f2fs_sb_info *sbi) in init_victim_secmap() argument
2220 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_victim_secmap()
2221 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in init_victim_secmap()
2229 static int build_dirty_segmap(struct f2fs_sb_info *sbi) in build_dirty_segmap() argument
2239 SM_I(sbi)->dirty_info = dirty_i; in build_dirty_segmap()
2242 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_dirty_segmap()
2250 init_dirty_segmap(sbi); in build_dirty_segmap()
2251 return init_victim_secmap(sbi); in build_dirty_segmap()
2257 static void init_min_max_mtime(struct f2fs_sb_info *sbi) in init_min_max_mtime() argument
2259 struct sit_info *sit_i = SIT_I(sbi); in init_min_max_mtime()
2266 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { in init_min_max_mtime()
2270 for (i = 0; i < sbi->segs_per_sec; i++) in init_min_max_mtime()
2271 mtime += get_seg_entry(sbi, segno + i)->mtime; in init_min_max_mtime()
2273 mtime = div_u64(mtime, sbi->segs_per_sec); in init_min_max_mtime()
2278 sit_i->max_mtime = get_mtime(sbi); in init_min_max_mtime()
2282 int build_segment_manager(struct f2fs_sb_info *sbi) in build_segment_manager() argument
2284 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in build_segment_manager()
2285 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in build_segment_manager()
2294 sbi->sm_info = sm_info; in build_segment_manager()
2316 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) { in build_segment_manager()
2317 err = create_flush_cmd_control(sbi); in build_segment_manager()
2322 err = build_sit_info(sbi); in build_segment_manager()
2325 err = build_free_segmap(sbi); in build_segment_manager()
2328 err = build_curseg(sbi); in build_segment_manager()
2333 build_sit_entries(sbi); in build_segment_manager()
2335 init_free_segmap(sbi); in build_segment_manager()
2336 err = build_dirty_segmap(sbi); in build_segment_manager()
2340 init_min_max_mtime(sbi); in build_segment_manager()
2344 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, in discard_dirty_segmap() argument
2347 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in discard_dirty_segmap()
2355 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) in destroy_victim_secmap() argument
2357 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_victim_secmap()
2361 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) in destroy_dirty_segmap() argument
2363 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_dirty_segmap()
2371 discard_dirty_segmap(sbi, i); in destroy_dirty_segmap()
2373 destroy_victim_secmap(sbi); in destroy_dirty_segmap()
2374 SM_I(sbi)->dirty_info = NULL; in destroy_dirty_segmap()
2378 static void destroy_curseg(struct f2fs_sb_info *sbi) in destroy_curseg() argument
2380 struct curseg_info *array = SM_I(sbi)->curseg_array; in destroy_curseg()
2385 SM_I(sbi)->curseg_array = NULL; in destroy_curseg()
2391 static void destroy_free_segmap(struct f2fs_sb_info *sbi) in destroy_free_segmap() argument
2393 struct free_segmap_info *free_i = SM_I(sbi)->free_info; in destroy_free_segmap()
2396 SM_I(sbi)->free_info = NULL; in destroy_free_segmap()
2402 static void destroy_sit_info(struct f2fs_sb_info *sbi) in destroy_sit_info() argument
2404 struct sit_info *sit_i = SIT_I(sbi); in destroy_sit_info()
2411 for (start = 0; start < MAIN_SEGS(sbi); start++) { in destroy_sit_info()
2423 SM_I(sbi)->sit_info = NULL; in destroy_sit_info()
2428 void destroy_segment_manager(struct f2fs_sb_info *sbi) in destroy_segment_manager() argument
2430 struct f2fs_sm_info *sm_info = SM_I(sbi); in destroy_segment_manager()
2434 destroy_flush_cmd_control(sbi); in destroy_segment_manager()
2435 destroy_dirty_segmap(sbi); in destroy_segment_manager()
2436 destroy_curseg(sbi); in destroy_segment_manager()
2437 destroy_free_segmap(sbi); in destroy_segment_manager()
2438 destroy_sit_info(sbi); in destroy_segment_manager()
2439 sbi->sm_info = NULL; in destroy_segment_manager()