This source file includes following definitions.
- __reverse_ulong
- __reverse_ffs
- __find_rev_next_bit
- __find_rev_next_zero_bit
- f2fs_need_SSR
- f2fs_register_inmem_page
- __revoke_inmem_pages
- f2fs_drop_inmem_pages_all
- f2fs_drop_inmem_pages
- f2fs_drop_inmem_page
- __f2fs_commit_inmem_pages
- f2fs_commit_inmem_pages
- f2fs_balance_fs
- f2fs_balance_fs_bg
- __submit_flush_wait
- submit_flush_wait
- issue_flush_thread
- f2fs_issue_flush
- f2fs_create_flush_cmd_control
- f2fs_destroy_flush_cmd_control
- f2fs_flush_device_cache
- __locate_dirty_segment
- __remove_dirty_segment
- locate_dirty_segment
- f2fs_dirty_to_prefree
- f2fs_get_unusable_blocks
- f2fs_disable_cp_again
- get_free_segment
- __create_discard_cmd
- __attach_discard_cmd
- __detach_discard_cmd
- __remove_discard_cmd
- f2fs_submit_discard_endio
- __check_sit_bitmap
- __init_discard_policy
- __submit_discard_cmd
- __insert_discard_tree
- __relocate_discard_cmd
- __punch_discard_cmd
- __update_discard_tree_range
- __queue_discard_cmd
- __issue_discard_cmd_orderly
- __issue_discard_cmd
- __drop_discard_cmd
- f2fs_drop_discard_cmd
- __wait_one_discard_bio
- __wait_discard_cmd_range
- __wait_all_discard_cmd
- f2fs_wait_discard_bio
- f2fs_stop_discard_thread
- f2fs_issue_discard_timeout
- issue_discard_thread
- __f2fs_issue_discard_zone
- __issue_discard_async
- f2fs_issue_discard
- add_discard_addrs
- release_discard_addr
- f2fs_release_discard_addrs
- set_prefree_as_free_segments
- f2fs_clear_prefree_segments
- create_discard_cmd_control
- destroy_discard_cmd_control
- __mark_sit_entry_dirty
- __set_sit_entry_type
- update_sit_entry
- f2fs_invalidate_blocks
- f2fs_is_checkpointed_data
- __add_sum_entry
- f2fs_npages_for_summary_flush
- f2fs_get_sum_page
- f2fs_update_meta_page
- write_sum_page
- write_current_sum_page
- is_next_segment_free
- get_new_segment
- reset_curseg
- __get_next_segno
- new_curseg
- __next_free_blkoff
- __refresh_next_blkoff
- change_curseg
- get_ssr_segment
- allocate_segment_by_default
- allocate_segment_for_resize
- f2fs_allocate_new_segments
- f2fs_exist_trim_candidates
- __issue_discard_cmd_range
- f2fs_trim_fs
- __has_curseg_space
- f2fs_rw_hint_to_seg_type
- f2fs_io_type_to_rw_hint
- __get_segment_type_2
- __get_segment_type_4
- __get_segment_type_6
- __get_segment_type
- f2fs_allocate_data_block
- update_device_state
- do_write_page
- f2fs_do_write_meta_page
- f2fs_do_write_node_page
- f2fs_outplace_write_data
- f2fs_inplace_write_data
- __f2fs_get_curseg
- f2fs_do_replace_block
- f2fs_replace_block
- f2fs_wait_on_page_writeback
- f2fs_wait_on_block_writeback
- f2fs_wait_on_block_writeback_range
- read_compacted_summaries
- read_normal_summaries
- restore_curseg_summaries
- write_compacted_summaries
- write_normal_summaries
- f2fs_write_data_summaries
- f2fs_write_node_summaries
- f2fs_lookup_journal_in_cursum
- get_current_sit_page
- get_next_sit_page
- grab_sit_entry_set
- release_sit_entry_set
- adjust_sit_entry_set
- add_sit_entry
- add_sits_in_set
- remove_sits_in_journal
- f2fs_flush_sit_entries
- build_sit_info
- build_free_segmap
- build_curseg
- build_sit_entries
- init_free_segmap
- init_dirty_segmap
- init_victim_secmap
- build_dirty_segmap
- sanity_check_curseg
- init_min_max_mtime
- f2fs_build_segment_manager
- discard_dirty_segmap
- destroy_victim_secmap
- destroy_dirty_segmap
- destroy_curseg
- destroy_free_segmap
- destroy_sit_info
- f2fs_destroy_segment_manager
- f2fs_create_segment_manager_caches
- f2fs_destroy_segment_manager_caches
1
2
3
4
5
6
7
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/prefetch.h>
13 #include <linux/kthread.h>
14 #include <linux/swap.h>
15 #include <linux/timer.h>
16 #include <linux/freezer.h>
17 #include <linux/sched/signal.h>
18
19 #include "f2fs.h"
20 #include "segment.h"
21 #include "node.h"
22 #include "gc.h"
23 #include "trace.h"
24 #include <trace/events/f2fs.h>
25
26 #define __reverse_ffz(x) __reverse_ffs(~(x))
27
28 static struct kmem_cache *discard_entry_slab;
29 static struct kmem_cache *discard_cmd_slab;
30 static struct kmem_cache *sit_entry_set_slab;
31 static struct kmem_cache *inmem_entry_slab;
32
33 static unsigned long __reverse_ulong(unsigned char *str)
34 {
35 unsigned long tmp = 0;
36 int shift = 24, idx = 0;
37
38 #if BITS_PER_LONG == 64
39 shift = 56;
40 #endif
41 while (shift >= 0) {
42 tmp |= (unsigned long)str[idx++] << shift;
43 shift -= BITS_PER_BYTE;
44 }
45 return tmp;
46 }
47
48
49
50
51
52 static inline unsigned long __reverse_ffs(unsigned long word)
53 {
54 int num = 0;
55
56 #if BITS_PER_LONG == 64
57 if ((word & 0xffffffff00000000UL) == 0)
58 num += 32;
59 else
60 word >>= 32;
61 #endif
62 if ((word & 0xffff0000) == 0)
63 num += 16;
64 else
65 word >>= 16;
66
67 if ((word & 0xff00) == 0)
68 num += 8;
69 else
70 word >>= 8;
71
72 if ((word & 0xf0) == 0)
73 num += 4;
74 else
75 word >>= 4;
76
77 if ((word & 0xc) == 0)
78 num += 2;
79 else
80 word >>= 2;
81
82 if ((word & 0x2) == 0)
83 num += 1;
84 return num;
85 }
86
87
88
89
90
91
92
93
94
95
96 static unsigned long __find_rev_next_bit(const unsigned long *addr,
97 unsigned long size, unsigned long offset)
98 {
99 const unsigned long *p = addr + BIT_WORD(offset);
100 unsigned long result = size;
101 unsigned long tmp;
102
103 if (offset >= size)
104 return size;
105
106 size -= (offset & ~(BITS_PER_LONG - 1));
107 offset %= BITS_PER_LONG;
108
109 while (1) {
110 if (*p == 0)
111 goto pass;
112
113 tmp = __reverse_ulong((unsigned char *)p);
114
115 tmp &= ~0UL >> offset;
116 if (size < BITS_PER_LONG)
117 tmp &= (~0UL << (BITS_PER_LONG - size));
118 if (tmp)
119 goto found;
120 pass:
121 if (size <= BITS_PER_LONG)
122 break;
123 size -= BITS_PER_LONG;
124 offset = 0;
125 p++;
126 }
127 return result;
128 found:
129 return result - size + __reverse_ffs(tmp);
130 }
131
132 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
133 unsigned long size, unsigned long offset)
134 {
135 const unsigned long *p = addr + BIT_WORD(offset);
136 unsigned long result = size;
137 unsigned long tmp;
138
139 if (offset >= size)
140 return size;
141
142 size -= (offset & ~(BITS_PER_LONG - 1));
143 offset %= BITS_PER_LONG;
144
145 while (1) {
146 if (*p == ~0UL)
147 goto pass;
148
149 tmp = __reverse_ulong((unsigned char *)p);
150
151 if (offset)
152 tmp |= ~0UL << (BITS_PER_LONG - offset);
153 if (size < BITS_PER_LONG)
154 tmp |= ~0UL >> size;
155 if (tmp != ~0UL)
156 goto found;
157 pass:
158 if (size <= BITS_PER_LONG)
159 break;
160 size -= BITS_PER_LONG;
161 offset = 0;
162 p++;
163 }
164 return result;
165 found:
166 return result - size + __reverse_ffz(tmp);
167 }
168
169 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
170 {
171 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
172 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
173 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
174
175 if (test_opt(sbi, LFS))
176 return false;
177 if (sbi->gc_mode == GC_URGENT)
178 return true;
179 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
180 return true;
181
182 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
183 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
184 }
185
186 void f2fs_register_inmem_page(struct inode *inode, struct page *page)
187 {
188 struct inmem_pages *new;
189
190 f2fs_trace_pid(page);
191
192 f2fs_set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
193
194 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
195
196
197 new->page = page;
198 INIT_LIST_HEAD(&new->list);
199
200
201 get_page(page);
202 mutex_lock(&F2FS_I(inode)->inmem_lock);
203 list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
204 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
205 mutex_unlock(&F2FS_I(inode)->inmem_lock);
206
207 trace_f2fs_register_inmem_page(page, INMEM);
208 }
209
210 static int __revoke_inmem_pages(struct inode *inode,
211 struct list_head *head, bool drop, bool recover,
212 bool trylock)
213 {
214 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
215 struct inmem_pages *cur, *tmp;
216 int err = 0;
217
218 list_for_each_entry_safe(cur, tmp, head, list) {
219 struct page *page = cur->page;
220
221 if (drop)
222 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
223
224 if (trylock) {
225
226
227
228
229 if (!trylock_page(page))
230 continue;
231 } else {
232 lock_page(page);
233 }
234
235 f2fs_wait_on_page_writeback(page, DATA, true, true);
236
237 if (recover) {
238 struct dnode_of_data dn;
239 struct node_info ni;
240
241 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
242 retry:
243 set_new_dnode(&dn, inode, NULL, NULL, 0);
244 err = f2fs_get_dnode_of_data(&dn, page->index,
245 LOOKUP_NODE);
246 if (err) {
247 if (err == -ENOMEM) {
248 congestion_wait(BLK_RW_ASYNC, HZ/50);
249 cond_resched();
250 goto retry;
251 }
252 err = -EAGAIN;
253 goto next;
254 }
255
256 err = f2fs_get_node_info(sbi, dn.nid, &ni);
257 if (err) {
258 f2fs_put_dnode(&dn);
259 return err;
260 }
261
262 if (cur->old_addr == NEW_ADDR) {
263 f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
264 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
265 } else
266 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
267 cur->old_addr, ni.version, true, true);
268 f2fs_put_dnode(&dn);
269 }
270 next:
271
272 if (drop || recover) {
273 ClearPageUptodate(page);
274 clear_cold_data(page);
275 }
276 f2fs_clear_page_private(page);
277 f2fs_put_page(page, 1);
278
279 list_del(&cur->list);
280 kmem_cache_free(inmem_entry_slab, cur);
281 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
282 }
283 return err;
284 }
285
286 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
287 {
288 struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
289 struct inode *inode;
290 struct f2fs_inode_info *fi;
291 unsigned int count = sbi->atomic_files;
292 unsigned int looped = 0;
293 next:
294 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
295 if (list_empty(head)) {
296 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
297 return;
298 }
299 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
300 inode = igrab(&fi->vfs_inode);
301 if (inode)
302 list_move_tail(&fi->inmem_ilist, head);
303 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
304
305 if (inode) {
306 if (gc_failure) {
307 if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
308 goto skip;
309 }
310 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
311 f2fs_drop_inmem_pages(inode);
312 skip:
313 iput(inode);
314 }
315 congestion_wait(BLK_RW_ASYNC, HZ/50);
316 cond_resched();
317 if (gc_failure) {
318 if (++looped >= count)
319 return;
320 }
321 goto next;
322 }
323
324 void f2fs_drop_inmem_pages(struct inode *inode)
325 {
326 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
327 struct f2fs_inode_info *fi = F2FS_I(inode);
328
329 while (!list_empty(&fi->inmem_pages)) {
330 mutex_lock(&fi->inmem_lock);
331 __revoke_inmem_pages(inode, &fi->inmem_pages,
332 true, false, true);
333 mutex_unlock(&fi->inmem_lock);
334 }
335
336 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
337 stat_dec_atomic_write(inode);
338
339 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
340 if (!list_empty(&fi->inmem_ilist))
341 list_del_init(&fi->inmem_ilist);
342 if (f2fs_is_atomic_file(inode)) {
343 clear_inode_flag(inode, FI_ATOMIC_FILE);
344 sbi->atomic_files--;
345 }
346 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
347 }
348
349 void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
350 {
351 struct f2fs_inode_info *fi = F2FS_I(inode);
352 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
353 struct list_head *head = &fi->inmem_pages;
354 struct inmem_pages *cur = NULL;
355
356 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page));
357
358 mutex_lock(&fi->inmem_lock);
359 list_for_each_entry(cur, head, list) {
360 if (cur->page == page)
361 break;
362 }
363
364 f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
365 list_del(&cur->list);
366 mutex_unlock(&fi->inmem_lock);
367
368 dec_page_count(sbi, F2FS_INMEM_PAGES);
369 kmem_cache_free(inmem_entry_slab, cur);
370
371 ClearPageUptodate(page);
372 f2fs_clear_page_private(page);
373 f2fs_put_page(page, 0);
374
375 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
376 }
377
378 static int __f2fs_commit_inmem_pages(struct inode *inode)
379 {
380 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
381 struct f2fs_inode_info *fi = F2FS_I(inode);
382 struct inmem_pages *cur, *tmp;
383 struct f2fs_io_info fio = {
384 .sbi = sbi,
385 .ino = inode->i_ino,
386 .type = DATA,
387 .op = REQ_OP_WRITE,
388 .op_flags = REQ_SYNC | REQ_PRIO,
389 .io_type = FS_DATA_IO,
390 };
391 struct list_head revoke_list;
392 bool submit_bio = false;
393 int err = 0;
394
395 INIT_LIST_HEAD(&revoke_list);
396
397 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
398 struct page *page = cur->page;
399
400 lock_page(page);
401 if (page->mapping == inode->i_mapping) {
402 trace_f2fs_commit_inmem_page(page, INMEM);
403
404 f2fs_wait_on_page_writeback(page, DATA, true, true);
405
406 set_page_dirty(page);
407 if (clear_page_dirty_for_io(page)) {
408 inode_dec_dirty_pages(inode);
409 f2fs_remove_dirty_inode(inode);
410 }
411 retry:
412 fio.page = page;
413 fio.old_blkaddr = NULL_ADDR;
414 fio.encrypted_page = NULL;
415 fio.need_lock = LOCK_DONE;
416 err = f2fs_do_write_data_page(&fio);
417 if (err) {
418 if (err == -ENOMEM) {
419 congestion_wait(BLK_RW_ASYNC, HZ/50);
420 cond_resched();
421 goto retry;
422 }
423 unlock_page(page);
424 break;
425 }
426
427 cur->old_addr = fio.old_blkaddr;
428 submit_bio = true;
429 }
430 unlock_page(page);
431 list_move_tail(&cur->list, &revoke_list);
432 }
433
434 if (submit_bio)
435 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
436
437 if (err) {
438
439
440
441
442
443
444
445
446 err = __revoke_inmem_pages(inode, &revoke_list,
447 false, true, false);
448
449
450 __revoke_inmem_pages(inode, &fi->inmem_pages,
451 true, false, false);
452 } else {
453 __revoke_inmem_pages(inode, &revoke_list,
454 false, false, false);
455 }
456
457 return err;
458 }
459
460 int f2fs_commit_inmem_pages(struct inode *inode)
461 {
462 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
463 struct f2fs_inode_info *fi = F2FS_I(inode);
464 int err;
465
466 f2fs_balance_fs(sbi, true);
467
468 down_write(&fi->i_gc_rwsem[WRITE]);
469
470 f2fs_lock_op(sbi);
471 set_inode_flag(inode, FI_ATOMIC_COMMIT);
472
473 mutex_lock(&fi->inmem_lock);
474 err = __f2fs_commit_inmem_pages(inode);
475 mutex_unlock(&fi->inmem_lock);
476
477 clear_inode_flag(inode, FI_ATOMIC_COMMIT);
478
479 f2fs_unlock_op(sbi);
480 up_write(&fi->i_gc_rwsem[WRITE]);
481
482 return err;
483 }
484
485
486
487
488
489 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
490 {
491 if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
492 f2fs_show_injection_info(FAULT_CHECKPOINT);
493 f2fs_stop_checkpoint(sbi, false);
494 }
495
496
497 if (need && excess_cached_nats(sbi))
498 f2fs_balance_fs_bg(sbi);
499
500 if (!f2fs_is_checkpoint_ready(sbi))
501 return;
502
503
504
505
506
507 if (has_not_enough_free_secs(sbi, 0, 0)) {
508 mutex_lock(&sbi->gc_mutex);
509 f2fs_gc(sbi, false, false, NULL_SEGNO);
510 }
511 }
512
513 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
514 {
515 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
516 return;
517
518
519 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
520 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
521
522
523 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
524 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
525
526 if (!f2fs_available_free_memory(sbi, FREE_NIDS))
527 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
528 else
529 f2fs_build_free_nids(sbi, false, false);
530
531 if (!is_idle(sbi, REQ_TIME) &&
532 (!excess_dirty_nats(sbi) && !excess_dirty_nodes(sbi)))
533 return;
534
535
536 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
537 !f2fs_available_free_memory(sbi, INO_ENTRIES) ||
538 excess_prefree_segs(sbi) ||
539 excess_dirty_nats(sbi) ||
540 excess_dirty_nodes(sbi) ||
541 f2fs_time_over(sbi, CP_TIME)) {
542 if (test_opt(sbi, DATA_FLUSH)) {
543 struct blk_plug plug;
544
545 mutex_lock(&sbi->flush_lock);
546
547 blk_start_plug(&plug);
548 f2fs_sync_dirty_inodes(sbi, FILE_INODE);
549 blk_finish_plug(&plug);
550
551 mutex_unlock(&sbi->flush_lock);
552 }
553 f2fs_sync_fs(sbi->sb, true);
554 stat_inc_bg_cp_count(sbi->stat_info);
555 }
556 }
557
558 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
559 struct block_device *bdev)
560 {
561 struct bio *bio;
562 int ret;
563
564 bio = f2fs_bio_alloc(sbi, 0, false);
565 if (!bio)
566 return -ENOMEM;
567
568 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH;
569 bio_set_dev(bio, bdev);
570 ret = submit_bio_wait(bio);
571 bio_put(bio);
572
573 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
574 test_opt(sbi, FLUSH_MERGE), ret);
575 return ret;
576 }
577
578 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
579 {
580 int ret = 0;
581 int i;
582
583 if (!f2fs_is_multi_device(sbi))
584 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
585
586 for (i = 0; i < sbi->s_ndevs; i++) {
587 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
588 continue;
589 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
590 if (ret)
591 break;
592 }
593 return ret;
594 }
595
596 static int issue_flush_thread(void *data)
597 {
598 struct f2fs_sb_info *sbi = data;
599 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
600 wait_queue_head_t *q = &fcc->flush_wait_queue;
601 repeat:
602 if (kthread_should_stop())
603 return 0;
604
605 sb_start_intwrite(sbi->sb);
606
607 if (!llist_empty(&fcc->issue_list)) {
608 struct flush_cmd *cmd, *next;
609 int ret;
610
611 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
612 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
613
614 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
615
616 ret = submit_flush_wait(sbi, cmd->ino);
617 atomic_inc(&fcc->issued_flush);
618
619 llist_for_each_entry_safe(cmd, next,
620 fcc->dispatch_list, llnode) {
621 cmd->ret = ret;
622 complete(&cmd->wait);
623 }
624 fcc->dispatch_list = NULL;
625 }
626
627 sb_end_intwrite(sbi->sb);
628
629 wait_event_interruptible(*q,
630 kthread_should_stop() || !llist_empty(&fcc->issue_list));
631 goto repeat;
632 }
633
634 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
635 {
636 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
637 struct flush_cmd cmd;
638 int ret;
639
640 if (test_opt(sbi, NOBARRIER))
641 return 0;
642
643 if (!test_opt(sbi, FLUSH_MERGE)) {
644 atomic_inc(&fcc->queued_flush);
645 ret = submit_flush_wait(sbi, ino);
646 atomic_dec(&fcc->queued_flush);
647 atomic_inc(&fcc->issued_flush);
648 return ret;
649 }
650
651 if (atomic_inc_return(&fcc->queued_flush) == 1 ||
652 f2fs_is_multi_device(sbi)) {
653 ret = submit_flush_wait(sbi, ino);
654 atomic_dec(&fcc->queued_flush);
655
656 atomic_inc(&fcc->issued_flush);
657 return ret;
658 }
659
660 cmd.ino = ino;
661 init_completion(&cmd.wait);
662
663 llist_add(&cmd.llnode, &fcc->issue_list);
664
665
666 smp_mb();
667
668 if (waitqueue_active(&fcc->flush_wait_queue))
669 wake_up(&fcc->flush_wait_queue);
670
671 if (fcc->f2fs_issue_flush) {
672 wait_for_completion(&cmd.wait);
673 atomic_dec(&fcc->queued_flush);
674 } else {
675 struct llist_node *list;
676
677 list = llist_del_all(&fcc->issue_list);
678 if (!list) {
679 wait_for_completion(&cmd.wait);
680 atomic_dec(&fcc->queued_flush);
681 } else {
682 struct flush_cmd *tmp, *next;
683
684 ret = submit_flush_wait(sbi, ino);
685
686 llist_for_each_entry_safe(tmp, next, list, llnode) {
687 if (tmp == &cmd) {
688 cmd.ret = ret;
689 atomic_dec(&fcc->queued_flush);
690 continue;
691 }
692 tmp->ret = ret;
693 complete(&tmp->wait);
694 }
695 }
696 }
697
698 return cmd.ret;
699 }
700
701 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
702 {
703 dev_t dev = sbi->sb->s_bdev->bd_dev;
704 struct flush_cmd_control *fcc;
705 int err = 0;
706
707 if (SM_I(sbi)->fcc_info) {
708 fcc = SM_I(sbi)->fcc_info;
709 if (fcc->f2fs_issue_flush)
710 return err;
711 goto init_thread;
712 }
713
714 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
715 if (!fcc)
716 return -ENOMEM;
717 atomic_set(&fcc->issued_flush, 0);
718 atomic_set(&fcc->queued_flush, 0);
719 init_waitqueue_head(&fcc->flush_wait_queue);
720 init_llist_head(&fcc->issue_list);
721 SM_I(sbi)->fcc_info = fcc;
722 if (!test_opt(sbi, FLUSH_MERGE))
723 return err;
724
725 init_thread:
726 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
727 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
728 if (IS_ERR(fcc->f2fs_issue_flush)) {
729 err = PTR_ERR(fcc->f2fs_issue_flush);
730 kvfree(fcc);
731 SM_I(sbi)->fcc_info = NULL;
732 return err;
733 }
734
735 return err;
736 }
737
738 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
739 {
740 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
741
742 if (fcc && fcc->f2fs_issue_flush) {
743 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
744
745 fcc->f2fs_issue_flush = NULL;
746 kthread_stop(flush_thread);
747 }
748 if (free) {
749 kvfree(fcc);
750 SM_I(sbi)->fcc_info = NULL;
751 }
752 }
753
754 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
755 {
756 int ret = 0, i;
757
758 if (!f2fs_is_multi_device(sbi))
759 return 0;
760
761 for (i = 1; i < sbi->s_ndevs; i++) {
762 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
763 continue;
764 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
765 if (ret)
766 break;
767
768 spin_lock(&sbi->dev_lock);
769 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
770 spin_unlock(&sbi->dev_lock);
771 }
772
773 return ret;
774 }
775
776 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
777 enum dirty_type dirty_type)
778 {
779 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
780
781
782 if (IS_CURSEG(sbi, segno))
783 return;
784
785 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
786 dirty_i->nr_dirty[dirty_type]++;
787
788 if (dirty_type == DIRTY) {
789 struct seg_entry *sentry = get_seg_entry(sbi, segno);
790 enum dirty_type t = sentry->type;
791
792 if (unlikely(t >= DIRTY)) {
793 f2fs_bug_on(sbi, 1);
794 return;
795 }
796 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
797 dirty_i->nr_dirty[t]++;
798 }
799 }
800
801 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
802 enum dirty_type dirty_type)
803 {
804 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
805
806 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
807 dirty_i->nr_dirty[dirty_type]--;
808
809 if (dirty_type == DIRTY) {
810 struct seg_entry *sentry = get_seg_entry(sbi, segno);
811 enum dirty_type t = sentry->type;
812
813 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
814 dirty_i->nr_dirty[t]--;
815
816 if (get_valid_blocks(sbi, segno, true) == 0) {
817 clear_bit(GET_SEC_FROM_SEG(sbi, segno),
818 dirty_i->victim_secmap);
819 #ifdef CONFIG_F2FS_CHECK_FS
820 clear_bit(segno, SIT_I(sbi)->invalid_segmap);
821 #endif
822 }
823 }
824 }
825
826
827
828
829
830
831 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
832 {
833 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
834 unsigned short valid_blocks, ckpt_valid_blocks;
835
836 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
837 return;
838
839 mutex_lock(&dirty_i->seglist_lock);
840
841 valid_blocks = get_valid_blocks(sbi, segno, false);
842 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno);
843
844 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
845 ckpt_valid_blocks == sbi->blocks_per_seg)) {
846 __locate_dirty_segment(sbi, segno, PRE);
847 __remove_dirty_segment(sbi, segno, DIRTY);
848 } else if (valid_blocks < sbi->blocks_per_seg) {
849 __locate_dirty_segment(sbi, segno, DIRTY);
850 } else {
851
852 __remove_dirty_segment(sbi, segno, DIRTY);
853 }
854
855 mutex_unlock(&dirty_i->seglist_lock);
856 }
857
858
859 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
860 {
861 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
862 unsigned int segno;
863
864 mutex_lock(&dirty_i->seglist_lock);
865 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
866 if (get_valid_blocks(sbi, segno, false))
867 continue;
868 if (IS_CURSEG(sbi, segno))
869 continue;
870 __locate_dirty_segment(sbi, segno, PRE);
871 __remove_dirty_segment(sbi, segno, DIRTY);
872 }
873 mutex_unlock(&dirty_i->seglist_lock);
874 }
875
876 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
877 {
878 int ovp_hole_segs =
879 (overprovision_segments(sbi) - reserved_segments(sbi));
880 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
881 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
882 block_t holes[2] = {0, 0};
883 block_t unusable;
884 struct seg_entry *se;
885 unsigned int segno;
886
887 mutex_lock(&dirty_i->seglist_lock);
888 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
889 se = get_seg_entry(sbi, segno);
890 if (IS_NODESEG(se->type))
891 holes[NODE] += sbi->blocks_per_seg - se->valid_blocks;
892 else
893 holes[DATA] += sbi->blocks_per_seg - se->valid_blocks;
894 }
895 mutex_unlock(&dirty_i->seglist_lock);
896
897 unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
898 if (unusable > ovp_holes)
899 return unusable - ovp_holes;
900 return 0;
901 }
902
903 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
904 {
905 int ovp_hole_segs =
906 (overprovision_segments(sbi) - reserved_segments(sbi));
907 if (unusable > F2FS_OPTION(sbi).unusable_cap)
908 return -EAGAIN;
909 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
910 dirty_segments(sbi) > ovp_hole_segs)
911 return -EAGAIN;
912 return 0;
913 }
914
915
916 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
917 {
918 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
919 unsigned int segno = 0;
920
921 mutex_lock(&dirty_i->seglist_lock);
922 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
923 if (get_valid_blocks(sbi, segno, false))
924 continue;
925 if (get_ckpt_valid_blocks(sbi, segno))
926 continue;
927 mutex_unlock(&dirty_i->seglist_lock);
928 return segno;
929 }
930 mutex_unlock(&dirty_i->seglist_lock);
931 return NULL_SEGNO;
932 }
933
934 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
935 struct block_device *bdev, block_t lstart,
936 block_t start, block_t len)
937 {
938 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
939 struct list_head *pend_list;
940 struct discard_cmd *dc;
941
942 f2fs_bug_on(sbi, !len);
943
944 pend_list = &dcc->pend_list[plist_idx(len)];
945
946 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
947 INIT_LIST_HEAD(&dc->list);
948 dc->bdev = bdev;
949 dc->lstart = lstart;
950 dc->start = start;
951 dc->len = len;
952 dc->ref = 0;
953 dc->state = D_PREP;
954 dc->queued = 0;
955 dc->error = 0;
956 init_completion(&dc->wait);
957 list_add_tail(&dc->list, pend_list);
958 spin_lock_init(&dc->lock);
959 dc->bio_ref = 0;
960 atomic_inc(&dcc->discard_cmd_cnt);
961 dcc->undiscard_blks += len;
962
963 return dc;
964 }
965
966 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
967 struct block_device *bdev, block_t lstart,
968 block_t start, block_t len,
969 struct rb_node *parent, struct rb_node **p,
970 bool leftmost)
971 {
972 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
973 struct discard_cmd *dc;
974
975 dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
976
977 rb_link_node(&dc->rb_node, parent, p);
978 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
979
980 return dc;
981 }
982
983 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
984 struct discard_cmd *dc)
985 {
986 if (dc->state == D_DONE)
987 atomic_sub(dc->queued, &dcc->queued_discard);
988
989 list_del(&dc->list);
990 rb_erase_cached(&dc->rb_node, &dcc->root);
991 dcc->undiscard_blks -= dc->len;
992
993 kmem_cache_free(discard_cmd_slab, dc);
994
995 atomic_dec(&dcc->discard_cmd_cnt);
996 }
997
998 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
999 struct discard_cmd *dc)
1000 {
1001 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1002 unsigned long flags;
1003
1004 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
1005
1006 spin_lock_irqsave(&dc->lock, flags);
1007 if (dc->bio_ref) {
1008 spin_unlock_irqrestore(&dc->lock, flags);
1009 return;
1010 }
1011 spin_unlock_irqrestore(&dc->lock, flags);
1012
1013 f2fs_bug_on(sbi, dc->ref);
1014
1015 if (dc->error == -EOPNOTSUPP)
1016 dc->error = 0;
1017
1018 if (dc->error)
1019 printk_ratelimited(
1020 "%sF2FS-fs: Issue discard(%u, %u, %u) failed, ret: %d",
1021 KERN_INFO, dc->lstart, dc->start, dc->len, dc->error);
1022 __detach_discard_cmd(dcc, dc);
1023 }
1024
1025 static void f2fs_submit_discard_endio(struct bio *bio)
1026 {
1027 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1028 unsigned long flags;
1029
1030 dc->error = blk_status_to_errno(bio->bi_status);
1031
1032 spin_lock_irqsave(&dc->lock, flags);
1033 dc->bio_ref--;
1034 if (!dc->bio_ref && dc->state == D_SUBMIT) {
1035 dc->state = D_DONE;
1036 complete_all(&dc->wait);
1037 }
1038 spin_unlock_irqrestore(&dc->lock, flags);
1039 bio_put(bio);
1040 }
1041
1042 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1043 block_t start, block_t end)
1044 {
1045 #ifdef CONFIG_F2FS_CHECK_FS
1046 struct seg_entry *sentry;
1047 unsigned int segno;
1048 block_t blk = start;
1049 unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1050 unsigned long *map;
1051
1052 while (blk < end) {
1053 segno = GET_SEGNO(sbi, blk);
1054 sentry = get_seg_entry(sbi, segno);
1055 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1056
1057 if (end < START_BLOCK(sbi, segno + 1))
1058 size = GET_BLKOFF_FROM_SEG0(sbi, end);
1059 else
1060 size = max_blocks;
1061 map = (unsigned long *)(sentry->cur_valid_map);
1062 offset = __find_rev_next_bit(map, size, offset);
1063 f2fs_bug_on(sbi, offset != size);
1064 blk = START_BLOCK(sbi, segno + 1);
1065 }
1066 #endif
1067 }
1068
1069 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1070 struct discard_policy *dpolicy,
1071 int discard_type, unsigned int granularity)
1072 {
1073
1074 dpolicy->type = discard_type;
1075 dpolicy->sync = true;
1076 dpolicy->ordered = false;
1077 dpolicy->granularity = granularity;
1078
1079 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
1080 dpolicy->io_aware_gran = MAX_PLIST_NUM;
1081 dpolicy->timeout = 0;
1082
1083 if (discard_type == DPOLICY_BG) {
1084 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1085 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1086 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1087 dpolicy->io_aware = true;
1088 dpolicy->sync = false;
1089 dpolicy->ordered = true;
1090 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1091 dpolicy->granularity = 1;
1092 dpolicy->max_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1093 }
1094 } else if (discard_type == DPOLICY_FORCE) {
1095 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1096 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1097 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1098 dpolicy->io_aware = false;
1099 } else if (discard_type == DPOLICY_FSTRIM) {
1100 dpolicy->io_aware = false;
1101 } else if (discard_type == DPOLICY_UMOUNT) {
1102 dpolicy->max_requests = UINT_MAX;
1103 dpolicy->io_aware = false;
1104
1105 dpolicy->granularity = 1;
1106 }
1107 }
1108
1109 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1110 struct block_device *bdev, block_t lstart,
1111 block_t start, block_t len);
1112
1113 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1114 struct discard_policy *dpolicy,
1115 struct discard_cmd *dc,
1116 unsigned int *issued)
1117 {
1118 struct block_device *bdev = dc->bdev;
1119 struct request_queue *q = bdev_get_queue(bdev);
1120 unsigned int max_discard_blocks =
1121 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1122 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1123 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1124 &(dcc->fstrim_list) : &(dcc->wait_list);
1125 int flag = dpolicy->sync ? REQ_SYNC : 0;
1126 block_t lstart, start, len, total_len;
1127 int err = 0;
1128
1129 if (dc->state != D_PREP)
1130 return 0;
1131
1132 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1133 return 0;
1134
1135 trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1136
1137 lstart = dc->lstart;
1138 start = dc->start;
1139 len = dc->len;
1140 total_len = len;
1141
1142 dc->len = 0;
1143
1144 while (total_len && *issued < dpolicy->max_requests && !err) {
1145 struct bio *bio = NULL;
1146 unsigned long flags;
1147 bool last = true;
1148
1149 if (len > max_discard_blocks) {
1150 len = max_discard_blocks;
1151 last = false;
1152 }
1153
1154 (*issued)++;
1155 if (*issued == dpolicy->max_requests)
1156 last = true;
1157
1158 dc->len += len;
1159
1160 if (time_to_inject(sbi, FAULT_DISCARD)) {
1161 f2fs_show_injection_info(FAULT_DISCARD);
1162 err = -EIO;
1163 goto submit;
1164 }
1165 err = __blkdev_issue_discard(bdev,
1166 SECTOR_FROM_BLOCK(start),
1167 SECTOR_FROM_BLOCK(len),
1168 GFP_NOFS, 0, &bio);
1169 submit:
1170 if (err) {
1171 spin_lock_irqsave(&dc->lock, flags);
1172 if (dc->state == D_PARTIAL)
1173 dc->state = D_SUBMIT;
1174 spin_unlock_irqrestore(&dc->lock, flags);
1175
1176 break;
1177 }
1178
1179 f2fs_bug_on(sbi, !bio);
1180
1181
1182
1183
1184
1185 spin_lock_irqsave(&dc->lock, flags);
1186 if (last)
1187 dc->state = D_SUBMIT;
1188 else
1189 dc->state = D_PARTIAL;
1190 dc->bio_ref++;
1191 spin_unlock_irqrestore(&dc->lock, flags);
1192
1193 atomic_inc(&dcc->queued_discard);
1194 dc->queued++;
1195 list_move_tail(&dc->list, wait_list);
1196
1197
1198 __check_sit_bitmap(sbi, lstart, lstart + len);
1199
1200 bio->bi_private = dc;
1201 bio->bi_end_io = f2fs_submit_discard_endio;
1202 bio->bi_opf |= flag;
1203 submit_bio(bio);
1204
1205 atomic_inc(&dcc->issued_discard);
1206
1207 f2fs_update_iostat(sbi, FS_DISCARD, 1);
1208
1209 lstart += len;
1210 start += len;
1211 total_len -= len;
1212 len = total_len;
1213 }
1214
1215 if (!err && len)
1216 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1217 return err;
1218 }
1219
1220 static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi,
1221 struct block_device *bdev, block_t lstart,
1222 block_t start, block_t len,
1223 struct rb_node **insert_p,
1224 struct rb_node *insert_parent)
1225 {
1226 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1227 struct rb_node **p;
1228 struct rb_node *parent = NULL;
1229 struct discard_cmd *dc = NULL;
1230 bool leftmost = true;
1231
1232 if (insert_p && insert_parent) {
1233 parent = insert_parent;
1234 p = insert_p;
1235 goto do_insert;
1236 }
1237
1238 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1239 lstart, &leftmost);
1240 do_insert:
1241 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1242 p, leftmost);
1243 if (!dc)
1244 return NULL;
1245
1246 return dc;
1247 }
1248
1249 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1250 struct discard_cmd *dc)
1251 {
1252 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1253 }
1254
1255 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1256 struct discard_cmd *dc, block_t blkaddr)
1257 {
1258 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1259 struct discard_info di = dc->di;
1260 bool modified = false;
1261
1262 if (dc->state == D_DONE || dc->len == 1) {
1263 __remove_discard_cmd(sbi, dc);
1264 return;
1265 }
1266
1267 dcc->undiscard_blks -= di.len;
1268
1269 if (blkaddr > di.lstart) {
1270 dc->len = blkaddr - dc->lstart;
1271 dcc->undiscard_blks += dc->len;
1272 __relocate_discard_cmd(dcc, dc);
1273 modified = true;
1274 }
1275
1276 if (blkaddr < di.lstart + di.len - 1) {
1277 if (modified) {
1278 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1279 di.start + blkaddr + 1 - di.lstart,
1280 di.lstart + di.len - 1 - blkaddr,
1281 NULL, NULL);
1282 } else {
1283 dc->lstart++;
1284 dc->len--;
1285 dc->start++;
1286 dcc->undiscard_blks += dc->len;
1287 __relocate_discard_cmd(dcc, dc);
1288 }
1289 }
1290 }
1291
1292 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1293 struct block_device *bdev, block_t lstart,
1294 block_t start, block_t len)
1295 {
1296 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1297 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1298 struct discard_cmd *dc;
1299 struct discard_info di = {0};
1300 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1301 struct request_queue *q = bdev_get_queue(bdev);
1302 unsigned int max_discard_blocks =
1303 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1304 block_t end = lstart + len;
1305
1306 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1307 NULL, lstart,
1308 (struct rb_entry **)&prev_dc,
1309 (struct rb_entry **)&next_dc,
1310 &insert_p, &insert_parent, true, NULL);
1311 if (dc)
1312 prev_dc = dc;
1313
1314 if (!prev_dc) {
1315 di.lstart = lstart;
1316 di.len = next_dc ? next_dc->lstart - lstart : len;
1317 di.len = min(di.len, len);
1318 di.start = start;
1319 }
1320
1321 while (1) {
1322 struct rb_node *node;
1323 bool merged = false;
1324 struct discard_cmd *tdc = NULL;
1325
1326 if (prev_dc) {
1327 di.lstart = prev_dc->lstart + prev_dc->len;
1328 if (di.lstart < lstart)
1329 di.lstart = lstart;
1330 if (di.lstart >= end)
1331 break;
1332
1333 if (!next_dc || next_dc->lstart > end)
1334 di.len = end - di.lstart;
1335 else
1336 di.len = next_dc->lstart - di.lstart;
1337 di.start = start + di.lstart - lstart;
1338 }
1339
1340 if (!di.len)
1341 goto next;
1342
1343 if (prev_dc && prev_dc->state == D_PREP &&
1344 prev_dc->bdev == bdev &&
1345 __is_discard_back_mergeable(&di, &prev_dc->di,
1346 max_discard_blocks)) {
1347 prev_dc->di.len += di.len;
1348 dcc->undiscard_blks += di.len;
1349 __relocate_discard_cmd(dcc, prev_dc);
1350 di = prev_dc->di;
1351 tdc = prev_dc;
1352 merged = true;
1353 }
1354
1355 if (next_dc && next_dc->state == D_PREP &&
1356 next_dc->bdev == bdev &&
1357 __is_discard_front_mergeable(&di, &next_dc->di,
1358 max_discard_blocks)) {
1359 next_dc->di.lstart = di.lstart;
1360 next_dc->di.len += di.len;
1361 next_dc->di.start = di.start;
1362 dcc->undiscard_blks += di.len;
1363 __relocate_discard_cmd(dcc, next_dc);
1364 if (tdc)
1365 __remove_discard_cmd(sbi, tdc);
1366 merged = true;
1367 }
1368
1369 if (!merged) {
1370 __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1371 di.len, NULL, NULL);
1372 }
1373 next:
1374 prev_dc = next_dc;
1375 if (!prev_dc)
1376 break;
1377
1378 node = rb_next(&prev_dc->rb_node);
1379 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1380 }
1381 }
1382
1383 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1384 struct block_device *bdev, block_t blkstart, block_t blklen)
1385 {
1386 block_t lblkstart = blkstart;
1387
1388 if (!f2fs_bdev_support_discard(bdev))
1389 return 0;
1390
1391 trace_f2fs_queue_discard(bdev, blkstart, blklen);
1392
1393 if (f2fs_is_multi_device(sbi)) {
1394 int devi = f2fs_target_device_index(sbi, blkstart);
1395
1396 blkstart -= FDEV(devi).start_blk;
1397 }
1398 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1399 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1400 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1401 return 0;
1402 }
1403
1404 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1405 struct discard_policy *dpolicy)
1406 {
1407 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1408 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1409 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1410 struct discard_cmd *dc;
1411 struct blk_plug plug;
1412 unsigned int pos = dcc->next_pos;
1413 unsigned int issued = 0;
1414 bool io_interrupted = false;
1415
1416 mutex_lock(&dcc->cmd_lock);
1417 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1418 NULL, pos,
1419 (struct rb_entry **)&prev_dc,
1420 (struct rb_entry **)&next_dc,
1421 &insert_p, &insert_parent, true, NULL);
1422 if (!dc)
1423 dc = next_dc;
1424
1425 blk_start_plug(&plug);
1426
1427 while (dc) {
1428 struct rb_node *node;
1429 int err = 0;
1430
1431 if (dc->state != D_PREP)
1432 goto next;
1433
1434 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1435 io_interrupted = true;
1436 break;
1437 }
1438
1439 dcc->next_pos = dc->lstart + dc->len;
1440 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1441
1442 if (issued >= dpolicy->max_requests)
1443 break;
1444 next:
1445 node = rb_next(&dc->rb_node);
1446 if (err)
1447 __remove_discard_cmd(sbi, dc);
1448 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1449 }
1450
1451 blk_finish_plug(&plug);
1452
1453 if (!dc)
1454 dcc->next_pos = 0;
1455
1456 mutex_unlock(&dcc->cmd_lock);
1457
1458 if (!issued && io_interrupted)
1459 issued = -1;
1460
1461 return issued;
1462 }
1463
1464 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1465 struct discard_policy *dpolicy)
1466 {
1467 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1468 struct list_head *pend_list;
1469 struct discard_cmd *dc, *tmp;
1470 struct blk_plug plug;
1471 int i, issued = 0;
1472 bool io_interrupted = false;
1473
1474 if (dpolicy->timeout != 0)
1475 f2fs_update_time(sbi, dpolicy->timeout);
1476
1477 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1478 if (dpolicy->timeout != 0 &&
1479 f2fs_time_over(sbi, dpolicy->timeout))
1480 break;
1481
1482 if (i + 1 < dpolicy->granularity)
1483 break;
1484
1485 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1486 return __issue_discard_cmd_orderly(sbi, dpolicy);
1487
1488 pend_list = &dcc->pend_list[i];
1489
1490 mutex_lock(&dcc->cmd_lock);
1491 if (list_empty(pend_list))
1492 goto next;
1493 if (unlikely(dcc->rbtree_check))
1494 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1495 &dcc->root));
1496 blk_start_plug(&plug);
1497 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1498 f2fs_bug_on(sbi, dc->state != D_PREP);
1499
1500 if (dpolicy->timeout != 0 &&
1501 f2fs_time_over(sbi, dpolicy->timeout))
1502 break;
1503
1504 if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1505 !is_idle(sbi, DISCARD_TIME)) {
1506 io_interrupted = true;
1507 break;
1508 }
1509
1510 __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1511
1512 if (issued >= dpolicy->max_requests)
1513 break;
1514 }
1515 blk_finish_plug(&plug);
1516 next:
1517 mutex_unlock(&dcc->cmd_lock);
1518
1519 if (issued >= dpolicy->max_requests || io_interrupted)
1520 break;
1521 }
1522
1523 if (!issued && io_interrupted)
1524 issued = -1;
1525
1526 return issued;
1527 }
1528
1529 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1530 {
1531 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1532 struct list_head *pend_list;
1533 struct discard_cmd *dc, *tmp;
1534 int i;
1535 bool dropped = false;
1536
1537 mutex_lock(&dcc->cmd_lock);
1538 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1539 pend_list = &dcc->pend_list[i];
1540 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1541 f2fs_bug_on(sbi, dc->state != D_PREP);
1542 __remove_discard_cmd(sbi, dc);
1543 dropped = true;
1544 }
1545 }
1546 mutex_unlock(&dcc->cmd_lock);
1547
1548 return dropped;
1549 }
1550
1551 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1552 {
1553 __drop_discard_cmd(sbi);
1554 }
1555
1556 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1557 struct discard_cmd *dc)
1558 {
1559 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1560 unsigned int len = 0;
1561
1562 wait_for_completion_io(&dc->wait);
1563 mutex_lock(&dcc->cmd_lock);
1564 f2fs_bug_on(sbi, dc->state != D_DONE);
1565 dc->ref--;
1566 if (!dc->ref) {
1567 if (!dc->error)
1568 len = dc->len;
1569 __remove_discard_cmd(sbi, dc);
1570 }
1571 mutex_unlock(&dcc->cmd_lock);
1572
1573 return len;
1574 }
1575
1576 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1577 struct discard_policy *dpolicy,
1578 block_t start, block_t end)
1579 {
1580 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1581 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1582 &(dcc->fstrim_list) : &(dcc->wait_list);
1583 struct discard_cmd *dc, *tmp;
1584 bool need_wait;
1585 unsigned int trimmed = 0;
1586
1587 next:
1588 need_wait = false;
1589
1590 mutex_lock(&dcc->cmd_lock);
1591 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1592 if (dc->lstart + dc->len <= start || end <= dc->lstart)
1593 continue;
1594 if (dc->len < dpolicy->granularity)
1595 continue;
1596 if (dc->state == D_DONE && !dc->ref) {
1597 wait_for_completion_io(&dc->wait);
1598 if (!dc->error)
1599 trimmed += dc->len;
1600 __remove_discard_cmd(sbi, dc);
1601 } else {
1602 dc->ref++;
1603 need_wait = true;
1604 break;
1605 }
1606 }
1607 mutex_unlock(&dcc->cmd_lock);
1608
1609 if (need_wait) {
1610 trimmed += __wait_one_discard_bio(sbi, dc);
1611 goto next;
1612 }
1613
1614 return trimmed;
1615 }
1616
1617 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1618 struct discard_policy *dpolicy)
1619 {
1620 struct discard_policy dp;
1621 unsigned int discard_blks;
1622
1623 if (dpolicy)
1624 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1625
1626
1627 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1628 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1629 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1630 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1631
1632 return discard_blks;
1633 }
1634
1635
1636 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1637 {
1638 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1639 struct discard_cmd *dc;
1640 bool need_wait = false;
1641
1642 mutex_lock(&dcc->cmd_lock);
1643 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1644 NULL, blkaddr);
1645 if (dc) {
1646 if (dc->state == D_PREP) {
1647 __punch_discard_cmd(sbi, dc, blkaddr);
1648 } else {
1649 dc->ref++;
1650 need_wait = true;
1651 }
1652 }
1653 mutex_unlock(&dcc->cmd_lock);
1654
1655 if (need_wait)
1656 __wait_one_discard_bio(sbi, dc);
1657 }
1658
1659 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1660 {
1661 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1662
1663 if (dcc && dcc->f2fs_issue_discard) {
1664 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1665
1666 dcc->f2fs_issue_discard = NULL;
1667 kthread_stop(discard_thread);
1668 }
1669 }
1670
1671
1672 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1673 {
1674 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1675 struct discard_policy dpolicy;
1676 bool dropped;
1677
1678 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1679 dcc->discard_granularity);
1680 dpolicy.timeout = UMOUNT_DISCARD_TIMEOUT;
1681 __issue_discard_cmd(sbi, &dpolicy);
1682 dropped = __drop_discard_cmd(sbi);
1683
1684
1685 __wait_all_discard_cmd(sbi, NULL);
1686
1687 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1688 return dropped;
1689 }
1690
1691 static int issue_discard_thread(void *data)
1692 {
1693 struct f2fs_sb_info *sbi = data;
1694 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1695 wait_queue_head_t *q = &dcc->discard_wait_queue;
1696 struct discard_policy dpolicy;
1697 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
1698 int issued;
1699
1700 set_freezable();
1701
1702 do {
1703 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1704 dcc->discard_granularity);
1705
1706 wait_event_interruptible_timeout(*q,
1707 kthread_should_stop() || freezing(current) ||
1708 dcc->discard_wake,
1709 msecs_to_jiffies(wait_ms));
1710
1711 if (dcc->discard_wake)
1712 dcc->discard_wake = 0;
1713
1714
1715 if (atomic_read(&dcc->queued_discard))
1716 __wait_all_discard_cmd(sbi, NULL);
1717
1718 if (try_to_freeze())
1719 continue;
1720 if (f2fs_readonly(sbi->sb))
1721 continue;
1722 if (kthread_should_stop())
1723 return 0;
1724 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1725 wait_ms = dpolicy.max_interval;
1726 continue;
1727 }
1728
1729 if (sbi->gc_mode == GC_URGENT)
1730 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1731
1732 sb_start_intwrite(sbi->sb);
1733
1734 issued = __issue_discard_cmd(sbi, &dpolicy);
1735 if (issued > 0) {
1736 __wait_all_discard_cmd(sbi, &dpolicy);
1737 wait_ms = dpolicy.min_interval;
1738 } else if (issued == -1){
1739 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1740 if (!wait_ms)
1741 wait_ms = dpolicy.mid_interval;
1742 } else {
1743 wait_ms = dpolicy.max_interval;
1744 }
1745
1746 sb_end_intwrite(sbi->sb);
1747
1748 } while (!kthread_should_stop());
1749 return 0;
1750 }
1751
1752 #ifdef CONFIG_BLK_DEV_ZONED
1753 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1754 struct block_device *bdev, block_t blkstart, block_t blklen)
1755 {
1756 sector_t sector, nr_sects;
1757 block_t lblkstart = blkstart;
1758 int devi = 0;
1759
1760 if (f2fs_is_multi_device(sbi)) {
1761 devi = f2fs_target_device_index(sbi, blkstart);
1762 if (blkstart < FDEV(devi).start_blk ||
1763 blkstart > FDEV(devi).end_blk) {
1764 f2fs_err(sbi, "Invalid block %x", blkstart);
1765 return -EIO;
1766 }
1767 blkstart -= FDEV(devi).start_blk;
1768 }
1769
1770
1771 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1772 sector = SECTOR_FROM_BLOCK(blkstart);
1773 nr_sects = SECTOR_FROM_BLOCK(blklen);
1774
1775 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1776 nr_sects != bdev_zone_sectors(bdev)) {
1777 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1778 devi, sbi->s_ndevs ? FDEV(devi).path : "",
1779 blkstart, blklen);
1780 return -EIO;
1781 }
1782 trace_f2fs_issue_reset_zone(bdev, blkstart);
1783 return blkdev_reset_zones(bdev, sector, nr_sects, GFP_NOFS);
1784 }
1785
1786
1787 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1788 }
1789 #endif
1790
1791 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1792 struct block_device *bdev, block_t blkstart, block_t blklen)
1793 {
1794 #ifdef CONFIG_BLK_DEV_ZONED
1795 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1796 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1797 #endif
1798 return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1799 }
1800
1801 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1802 block_t blkstart, block_t blklen)
1803 {
1804 sector_t start = blkstart, len = 0;
1805 struct block_device *bdev;
1806 struct seg_entry *se;
1807 unsigned int offset;
1808 block_t i;
1809 int err = 0;
1810
1811 bdev = f2fs_target_device(sbi, blkstart, NULL);
1812
1813 for (i = blkstart; i < blkstart + blklen; i++, len++) {
1814 if (i != start) {
1815 struct block_device *bdev2 =
1816 f2fs_target_device(sbi, i, NULL);
1817
1818 if (bdev2 != bdev) {
1819 err = __issue_discard_async(sbi, bdev,
1820 start, len);
1821 if (err)
1822 return err;
1823 bdev = bdev2;
1824 start = i;
1825 len = 0;
1826 }
1827 }
1828
1829 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1830 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1831
1832 if (!f2fs_test_and_set_bit(offset, se->discard_map))
1833 sbi->discard_blks--;
1834 }
1835
1836 if (len)
1837 err = __issue_discard_async(sbi, bdev, start, len);
1838 return err;
1839 }
1840
1841 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1842 bool check_only)
1843 {
1844 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1845 int max_blocks = sbi->blocks_per_seg;
1846 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1847 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1848 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1849 unsigned long *discard_map = (unsigned long *)se->discard_map;
1850 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1851 unsigned int start = 0, end = -1;
1852 bool force = (cpc->reason & CP_DISCARD);
1853 struct discard_entry *de = NULL;
1854 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1855 int i;
1856
1857 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))
1858 return false;
1859
1860 if (!force) {
1861 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1862 SM_I(sbi)->dcc_info->nr_discards >=
1863 SM_I(sbi)->dcc_info->max_discards)
1864 return false;
1865 }
1866
1867
1868 for (i = 0; i < entries; i++)
1869 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1870 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1871
1872 while (force || SM_I(sbi)->dcc_info->nr_discards <=
1873 SM_I(sbi)->dcc_info->max_discards) {
1874 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1875 if (start >= max_blocks)
1876 break;
1877
1878 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1879 if (force && start && end != max_blocks
1880 && (end - start) < cpc->trim_minlen)
1881 continue;
1882
1883 if (check_only)
1884 return true;
1885
1886 if (!de) {
1887 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1888 GFP_F2FS_ZERO);
1889 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1890 list_add_tail(&de->list, head);
1891 }
1892
1893 for (i = start; i < end; i++)
1894 __set_bit_le(i, (void *)de->discard_map);
1895
1896 SM_I(sbi)->dcc_info->nr_discards += end - start;
1897 }
1898 return false;
1899 }
1900
1901 static void release_discard_addr(struct discard_entry *entry)
1902 {
1903 list_del(&entry->list);
1904 kmem_cache_free(discard_entry_slab, entry);
1905 }
1906
1907 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
1908 {
1909 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
1910 struct discard_entry *entry, *this;
1911
1912
1913 list_for_each_entry_safe(entry, this, head, list)
1914 release_discard_addr(entry);
1915 }
1916
1917
1918
1919
1920 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
1921 {
1922 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1923 unsigned int segno;
1924
1925 mutex_lock(&dirty_i->seglist_lock);
1926 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1927 __set_test_and_free(sbi, segno);
1928 mutex_unlock(&dirty_i->seglist_lock);
1929 }
1930
1931 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
1932 struct cp_control *cpc)
1933 {
1934 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1935 struct list_head *head = &dcc->entry_list;
1936 struct discard_entry *entry, *this;
1937 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1938 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
1939 unsigned int start = 0, end = -1;
1940 unsigned int secno, start_segno;
1941 bool force = (cpc->reason & CP_DISCARD);
1942 bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi);
1943
1944 mutex_lock(&dirty_i->seglist_lock);
1945
1946 while (1) {
1947 int i;
1948
1949 if (need_align && end != -1)
1950 end--;
1951 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
1952 if (start >= MAIN_SEGS(sbi))
1953 break;
1954 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
1955 start + 1);
1956
1957 if (need_align) {
1958 start = rounddown(start, sbi->segs_per_sec);
1959 end = roundup(end, sbi->segs_per_sec);
1960 }
1961
1962 for (i = start; i < end; i++) {
1963 if (test_and_clear_bit(i, prefree_map))
1964 dirty_i->nr_dirty[PRE]--;
1965 }
1966
1967 if (!f2fs_realtime_discard_enable(sbi))
1968 continue;
1969
1970 if (force && start >= cpc->trim_start &&
1971 (end - 1) <= cpc->trim_end)
1972 continue;
1973
1974 if (!test_opt(sbi, LFS) || !__is_large_section(sbi)) {
1975 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
1976 (end - start) << sbi->log_blocks_per_seg);
1977 continue;
1978 }
1979 next:
1980 secno = GET_SEC_FROM_SEG(sbi, start);
1981 start_segno = GET_SEG_FROM_SEC(sbi, secno);
1982 if (!IS_CURSEC(sbi, secno) &&
1983 !get_valid_blocks(sbi, start, true))
1984 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
1985 sbi->segs_per_sec << sbi->log_blocks_per_seg);
1986
1987 start = start_segno + sbi->segs_per_sec;
1988 if (start < end)
1989 goto next;
1990 else
1991 end = start - 1;
1992 }
1993 mutex_unlock(&dirty_i->seglist_lock);
1994
1995
1996 list_for_each_entry_safe(entry, this, head, list) {
1997 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
1998 bool is_valid = test_bit_le(0, entry->discard_map);
1999
2000 find_next:
2001 if (is_valid) {
2002 next_pos = find_next_zero_bit_le(entry->discard_map,
2003 sbi->blocks_per_seg, cur_pos);
2004 len = next_pos - cur_pos;
2005
2006 if (f2fs_sb_has_blkzoned(sbi) ||
2007 (force && len < cpc->trim_minlen))
2008 goto skip;
2009
2010 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2011 len);
2012 total_len += len;
2013 } else {
2014 next_pos = find_next_bit_le(entry->discard_map,
2015 sbi->blocks_per_seg, cur_pos);
2016 }
2017 skip:
2018 cur_pos = next_pos;
2019 is_valid = !is_valid;
2020
2021 if (cur_pos < sbi->blocks_per_seg)
2022 goto find_next;
2023
2024 release_discard_addr(entry);
2025 dcc->nr_discards -= total_len;
2026 }
2027
2028 wake_up_discard_thread(sbi, false);
2029 }
2030
2031 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2032 {
2033 dev_t dev = sbi->sb->s_bdev->bd_dev;
2034 struct discard_cmd_control *dcc;
2035 int err = 0, i;
2036
2037 if (SM_I(sbi)->dcc_info) {
2038 dcc = SM_I(sbi)->dcc_info;
2039 goto init_thread;
2040 }
2041
2042 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2043 if (!dcc)
2044 return -ENOMEM;
2045
2046 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2047 INIT_LIST_HEAD(&dcc->entry_list);
2048 for (i = 0; i < MAX_PLIST_NUM; i++)
2049 INIT_LIST_HEAD(&dcc->pend_list[i]);
2050 INIT_LIST_HEAD(&dcc->wait_list);
2051 INIT_LIST_HEAD(&dcc->fstrim_list);
2052 mutex_init(&dcc->cmd_lock);
2053 atomic_set(&dcc->issued_discard, 0);
2054 atomic_set(&dcc->queued_discard, 0);
2055 atomic_set(&dcc->discard_cmd_cnt, 0);
2056 dcc->nr_discards = 0;
2057 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2058 dcc->undiscard_blks = 0;
2059 dcc->next_pos = 0;
2060 dcc->root = RB_ROOT_CACHED;
2061 dcc->rbtree_check = false;
2062
2063 init_waitqueue_head(&dcc->discard_wait_queue);
2064 SM_I(sbi)->dcc_info = dcc;
2065 init_thread:
2066 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2067 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2068 if (IS_ERR(dcc->f2fs_issue_discard)) {
2069 err = PTR_ERR(dcc->f2fs_issue_discard);
2070 kvfree(dcc);
2071 SM_I(sbi)->dcc_info = NULL;
2072 return err;
2073 }
2074
2075 return err;
2076 }
2077
2078 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2079 {
2080 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2081
2082 if (!dcc)
2083 return;
2084
2085 f2fs_stop_discard_thread(sbi);
2086
2087
2088
2089
2090
2091 if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2092 f2fs_issue_discard_timeout(sbi);
2093
2094 kvfree(dcc);
2095 SM_I(sbi)->dcc_info = NULL;
2096 }
2097
2098 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2099 {
2100 struct sit_info *sit_i = SIT_I(sbi);
2101
2102 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2103 sit_i->dirty_sentries++;
2104 return false;
2105 }
2106
2107 return true;
2108 }
2109
2110 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2111 unsigned int segno, int modified)
2112 {
2113 struct seg_entry *se = get_seg_entry(sbi, segno);
2114 se->type = type;
2115 if (modified)
2116 __mark_sit_entry_dirty(sbi, segno);
2117 }
2118
2119 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2120 {
2121 struct seg_entry *se;
2122 unsigned int segno, offset;
2123 long int new_vblocks;
2124 bool exist;
2125 #ifdef CONFIG_F2FS_CHECK_FS
2126 bool mir_exist;
2127 #endif
2128
2129 segno = GET_SEGNO(sbi, blkaddr);
2130
2131 se = get_seg_entry(sbi, segno);
2132 new_vblocks = se->valid_blocks + del;
2133 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2134
2135 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
2136 (new_vblocks > sbi->blocks_per_seg)));
2137
2138 se->valid_blocks = new_vblocks;
2139 se->mtime = get_mtime(sbi, false);
2140 if (se->mtime > SIT_I(sbi)->max_mtime)
2141 SIT_I(sbi)->max_mtime = se->mtime;
2142
2143
2144 if (del > 0) {
2145 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2146 #ifdef CONFIG_F2FS_CHECK_FS
2147 mir_exist = f2fs_test_and_set_bit(offset,
2148 se->cur_valid_map_mir);
2149 if (unlikely(exist != mir_exist)) {
2150 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2151 blkaddr, exist);
2152 f2fs_bug_on(sbi, 1);
2153 }
2154 #endif
2155 if (unlikely(exist)) {
2156 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2157 blkaddr);
2158 f2fs_bug_on(sbi, 1);
2159 se->valid_blocks--;
2160 del = 0;
2161 }
2162
2163 if (!f2fs_test_and_set_bit(offset, se->discard_map))
2164 sbi->discard_blks--;
2165
2166
2167
2168
2169
2170 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2171 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2172 se->ckpt_valid_blocks++;
2173 }
2174 } else {
2175 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2176 #ifdef CONFIG_F2FS_CHECK_FS
2177 mir_exist = f2fs_test_and_clear_bit(offset,
2178 se->cur_valid_map_mir);
2179 if (unlikely(exist != mir_exist)) {
2180 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2181 blkaddr, exist);
2182 f2fs_bug_on(sbi, 1);
2183 }
2184 #endif
2185 if (unlikely(!exist)) {
2186 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2187 blkaddr);
2188 f2fs_bug_on(sbi, 1);
2189 se->valid_blocks++;
2190 del = 0;
2191 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2192
2193
2194
2195
2196
2197
2198 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2199 spin_lock(&sbi->stat_lock);
2200 sbi->unusable_block_count++;
2201 spin_unlock(&sbi->stat_lock);
2202 }
2203 }
2204
2205 if (f2fs_test_and_clear_bit(offset, se->discard_map))
2206 sbi->discard_blks++;
2207 }
2208 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2209 se->ckpt_valid_blocks += del;
2210
2211 __mark_sit_entry_dirty(sbi, segno);
2212
2213
2214 SIT_I(sbi)->written_valid_blocks += del;
2215
2216 if (__is_large_section(sbi))
2217 get_sec_entry(sbi, segno)->valid_blocks += del;
2218 }
2219
2220 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2221 {
2222 unsigned int segno = GET_SEGNO(sbi, addr);
2223 struct sit_info *sit_i = SIT_I(sbi);
2224
2225 f2fs_bug_on(sbi, addr == NULL_ADDR);
2226 if (addr == NEW_ADDR)
2227 return;
2228
2229 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2230
2231
2232 down_write(&sit_i->sentry_lock);
2233
2234 update_sit_entry(sbi, addr, -1);
2235
2236
2237 locate_dirty_segment(sbi, segno);
2238
2239 up_write(&sit_i->sentry_lock);
2240 }
2241
2242 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2243 {
2244 struct sit_info *sit_i = SIT_I(sbi);
2245 unsigned int segno, offset;
2246 struct seg_entry *se;
2247 bool is_cp = false;
2248
2249 if (!__is_valid_data_blkaddr(blkaddr))
2250 return true;
2251
2252 down_read(&sit_i->sentry_lock);
2253
2254 segno = GET_SEGNO(sbi, blkaddr);
2255 se = get_seg_entry(sbi, segno);
2256 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2257
2258 if (f2fs_test_bit(offset, se->ckpt_valid_map))
2259 is_cp = true;
2260
2261 up_read(&sit_i->sentry_lock);
2262
2263 return is_cp;
2264 }
2265
2266
2267
2268
2269 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2270 struct f2fs_summary *sum)
2271 {
2272 struct curseg_info *curseg = CURSEG_I(sbi, type);
2273 void *addr = curseg->sum_blk;
2274 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2275 memcpy(addr, sum, sizeof(struct f2fs_summary));
2276 }
2277
2278
2279
2280
2281 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2282 {
2283 int valid_sum_count = 0;
2284 int i, sum_in_page;
2285
2286 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2287 if (sbi->ckpt->alloc_type[i] == SSR)
2288 valid_sum_count += sbi->blocks_per_seg;
2289 else {
2290 if (for_ra)
2291 valid_sum_count += le16_to_cpu(
2292 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2293 else
2294 valid_sum_count += curseg_blkoff(sbi, i);
2295 }
2296 }
2297
2298 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2299 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2300 if (valid_sum_count <= sum_in_page)
2301 return 1;
2302 else if ((valid_sum_count - sum_in_page) <=
2303 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2304 return 2;
2305 return 3;
2306 }
2307
2308
2309
2310
2311 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2312 {
2313 return f2fs_get_meta_page_nofail(sbi, GET_SUM_BLOCK(sbi, segno));
2314 }
2315
2316 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2317 void *src, block_t blk_addr)
2318 {
2319 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2320
2321 memcpy(page_address(page), src, PAGE_SIZE);
2322 set_page_dirty(page);
2323 f2fs_put_page(page, 1);
2324 }
2325
2326 static void write_sum_page(struct f2fs_sb_info *sbi,
2327 struct f2fs_summary_block *sum_blk, block_t blk_addr)
2328 {
2329 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2330 }
2331
2332 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2333 int type, block_t blk_addr)
2334 {
2335 struct curseg_info *curseg = CURSEG_I(sbi, type);
2336 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2337 struct f2fs_summary_block *src = curseg->sum_blk;
2338 struct f2fs_summary_block *dst;
2339
2340 dst = (struct f2fs_summary_block *)page_address(page);
2341 memset(dst, 0, PAGE_SIZE);
2342
2343 mutex_lock(&curseg->curseg_mutex);
2344
2345 down_read(&curseg->journal_rwsem);
2346 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2347 up_read(&curseg->journal_rwsem);
2348
2349 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2350 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2351
2352 mutex_unlock(&curseg->curseg_mutex);
2353
2354 set_page_dirty(page);
2355 f2fs_put_page(page, 1);
2356 }
2357
2358 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
2359 {
2360 struct curseg_info *curseg = CURSEG_I(sbi, type);
2361 unsigned int segno = curseg->segno + 1;
2362 struct free_segmap_info *free_i = FREE_I(sbi);
2363
2364 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2365 return !test_bit(segno, free_i->free_segmap);
2366 return 0;
2367 }
2368
2369
2370
2371
2372
2373 static void get_new_segment(struct f2fs_sb_info *sbi,
2374 unsigned int *newseg, bool new_sec, int dir)
2375 {
2376 struct free_segmap_info *free_i = FREE_I(sbi);
2377 unsigned int segno, secno, zoneno;
2378 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2379 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2380 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2381 unsigned int left_start = hint;
2382 bool init = true;
2383 int go_left = 0;
2384 int i;
2385
2386 spin_lock(&free_i->segmap_lock);
2387
2388 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2389 segno = find_next_zero_bit(free_i->free_segmap,
2390 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2391 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2392 goto got_it;
2393 }
2394 find_other_zone:
2395 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2396 if (secno >= MAIN_SECS(sbi)) {
2397 if (dir == ALLOC_RIGHT) {
2398 secno = find_next_zero_bit(free_i->free_secmap,
2399 MAIN_SECS(sbi), 0);
2400 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2401 } else {
2402 go_left = 1;
2403 left_start = hint - 1;
2404 }
2405 }
2406 if (go_left == 0)
2407 goto skip_left;
2408
2409 while (test_bit(left_start, free_i->free_secmap)) {
2410 if (left_start > 0) {
2411 left_start--;
2412 continue;
2413 }
2414 left_start = find_next_zero_bit(free_i->free_secmap,
2415 MAIN_SECS(sbi), 0);
2416 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2417 break;
2418 }
2419 secno = left_start;
2420 skip_left:
2421 segno = GET_SEG_FROM_SEC(sbi, secno);
2422 zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2423
2424
2425 if (!init)
2426 goto got_it;
2427 if (sbi->secs_per_zone == 1)
2428 goto got_it;
2429 if (zoneno == old_zoneno)
2430 goto got_it;
2431 if (dir == ALLOC_LEFT) {
2432 if (!go_left && zoneno + 1 >= total_zones)
2433 goto got_it;
2434 if (go_left && zoneno == 0)
2435 goto got_it;
2436 }
2437 for (i = 0; i < NR_CURSEG_TYPE; i++)
2438 if (CURSEG_I(sbi, i)->zone == zoneno)
2439 break;
2440
2441 if (i < NR_CURSEG_TYPE) {
2442
2443 if (go_left)
2444 hint = zoneno * sbi->secs_per_zone - 1;
2445 else if (zoneno + 1 >= total_zones)
2446 hint = 0;
2447 else
2448 hint = (zoneno + 1) * sbi->secs_per_zone;
2449 init = false;
2450 goto find_other_zone;
2451 }
2452 got_it:
2453
2454 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2455 __set_inuse(sbi, segno);
2456 *newseg = segno;
2457 spin_unlock(&free_i->segmap_lock);
2458 }
2459
2460 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2461 {
2462 struct curseg_info *curseg = CURSEG_I(sbi, type);
2463 struct summary_footer *sum_footer;
2464
2465 curseg->segno = curseg->next_segno;
2466 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2467 curseg->next_blkoff = 0;
2468 curseg->next_segno = NULL_SEGNO;
2469
2470 sum_footer = &(curseg->sum_blk->footer);
2471 memset(sum_footer, 0, sizeof(struct summary_footer));
2472 if (IS_DATASEG(type))
2473 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2474 if (IS_NODESEG(type))
2475 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2476 __set_sit_entry_type(sbi, type, curseg->segno, modified);
2477 }
2478
2479 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2480 {
2481
2482 if (__is_large_section(sbi))
2483 return CURSEG_I(sbi, type)->segno;
2484
2485 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2486 return 0;
2487
2488 if (test_opt(sbi, NOHEAP) &&
2489 (type == CURSEG_HOT_DATA || IS_NODESEG(type)))
2490 return 0;
2491
2492 if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2493 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2494
2495
2496 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2497 return 0;
2498
2499 return CURSEG_I(sbi, type)->segno;
2500 }
2501
2502
2503
2504
2505
2506 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2507 {
2508 struct curseg_info *curseg = CURSEG_I(sbi, type);
2509 unsigned int segno = curseg->segno;
2510 int dir = ALLOC_LEFT;
2511
2512 write_sum_page(sbi, curseg->sum_blk,
2513 GET_SUM_BLOCK(sbi, segno));
2514 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
2515 dir = ALLOC_RIGHT;
2516
2517 if (test_opt(sbi, NOHEAP))
2518 dir = ALLOC_RIGHT;
2519
2520 segno = __get_next_segno(sbi, type);
2521 get_new_segment(sbi, &segno, new_sec, dir);
2522 curseg->next_segno = segno;
2523 reset_curseg(sbi, type, 1);
2524 curseg->alloc_type = LFS;
2525 }
2526
2527 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
2528 struct curseg_info *seg, block_t start)
2529 {
2530 struct seg_entry *se = get_seg_entry(sbi, seg->segno);
2531 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2532 unsigned long *target_map = SIT_I(sbi)->tmp_map;
2533 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2534 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2535 int i, pos;
2536
2537 for (i = 0; i < entries; i++)
2538 target_map[i] = ckpt_map[i] | cur_map[i];
2539
2540 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2541
2542 seg->next_blkoff = pos;
2543 }
2544
2545
2546
2547
2548
2549
2550 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2551 struct curseg_info *seg)
2552 {
2553 if (seg->alloc_type == SSR)
2554 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
2555 else
2556 seg->next_blkoff++;
2557 }
2558
2559
2560
2561
2562
2563 static void change_curseg(struct f2fs_sb_info *sbi, int type)
2564 {
2565 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2566 struct curseg_info *curseg = CURSEG_I(sbi, type);
2567 unsigned int new_segno = curseg->next_segno;
2568 struct f2fs_summary_block *sum_node;
2569 struct page *sum_page;
2570
2571 write_sum_page(sbi, curseg->sum_blk,
2572 GET_SUM_BLOCK(sbi, curseg->segno));
2573 __set_test_and_inuse(sbi, new_segno);
2574
2575 mutex_lock(&dirty_i->seglist_lock);
2576 __remove_dirty_segment(sbi, new_segno, PRE);
2577 __remove_dirty_segment(sbi, new_segno, DIRTY);
2578 mutex_unlock(&dirty_i->seglist_lock);
2579
2580 reset_curseg(sbi, type, 1);
2581 curseg->alloc_type = SSR;
2582 __next_free_blkoff(sbi, curseg, 0);
2583
2584 sum_page = f2fs_get_sum_page(sbi, new_segno);
2585 f2fs_bug_on(sbi, IS_ERR(sum_page));
2586 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2587 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2588 f2fs_put_page(sum_page, 1);
2589 }
2590
2591 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
2592 {
2593 struct curseg_info *curseg = CURSEG_I(sbi, type);
2594 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2595 unsigned segno = NULL_SEGNO;
2596 int i, cnt;
2597 bool reversed = false;
2598
2599
2600 if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) {
2601 curseg->next_segno = segno;
2602 return 1;
2603 }
2604
2605
2606 if (IS_NODESEG(type)) {
2607 if (type >= CURSEG_WARM_NODE) {
2608 reversed = true;
2609 i = CURSEG_COLD_NODE;
2610 } else {
2611 i = CURSEG_HOT_NODE;
2612 }
2613 cnt = NR_CURSEG_NODE_TYPE;
2614 } else {
2615 if (type >= CURSEG_WARM_DATA) {
2616 reversed = true;
2617 i = CURSEG_COLD_DATA;
2618 } else {
2619 i = CURSEG_HOT_DATA;
2620 }
2621 cnt = NR_CURSEG_DATA_TYPE;
2622 }
2623
2624 for (; cnt-- > 0; reversed ? i-- : i++) {
2625 if (i == type)
2626 continue;
2627 if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) {
2628 curseg->next_segno = segno;
2629 return 1;
2630 }
2631 }
2632
2633
2634 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2635 segno = get_free_segment(sbi);
2636 if (segno != NULL_SEGNO) {
2637 curseg->next_segno = segno;
2638 return 1;
2639 }
2640 }
2641 return 0;
2642 }
2643
2644
2645
2646
2647
2648 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2649 int type, bool force)
2650 {
2651 struct curseg_info *curseg = CURSEG_I(sbi, type);
2652
2653 if (force)
2654 new_curseg(sbi, type, true);
2655 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2656 type == CURSEG_WARM_NODE)
2657 new_curseg(sbi, type, false);
2658 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type) &&
2659 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2660 new_curseg(sbi, type, false);
2661 else if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type))
2662 change_curseg(sbi, type);
2663 else
2664 new_curseg(sbi, type, false);
2665
2666 stat_inc_seg_type(sbi, curseg);
2667 }
2668
2669 void allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2670 unsigned int start, unsigned int end)
2671 {
2672 struct curseg_info *curseg = CURSEG_I(sbi, type);
2673 unsigned int segno;
2674
2675 down_read(&SM_I(sbi)->curseg_lock);
2676 mutex_lock(&curseg->curseg_mutex);
2677 down_write(&SIT_I(sbi)->sentry_lock);
2678
2679 segno = CURSEG_I(sbi, type)->segno;
2680 if (segno < start || segno > end)
2681 goto unlock;
2682
2683 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type))
2684 change_curseg(sbi, type);
2685 else
2686 new_curseg(sbi, type, true);
2687
2688 stat_inc_seg_type(sbi, curseg);
2689
2690 locate_dirty_segment(sbi, segno);
2691 unlock:
2692 up_write(&SIT_I(sbi)->sentry_lock);
2693
2694 if (segno != curseg->segno)
2695 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
2696 type, segno, curseg->segno);
2697
2698 mutex_unlock(&curseg->curseg_mutex);
2699 up_read(&SM_I(sbi)->curseg_lock);
2700 }
2701
2702 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
2703 {
2704 struct curseg_info *curseg;
2705 unsigned int old_segno;
2706 int i;
2707
2708 down_write(&SIT_I(sbi)->sentry_lock);
2709
2710 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2711 curseg = CURSEG_I(sbi, i);
2712 old_segno = curseg->segno;
2713 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
2714 locate_dirty_segment(sbi, old_segno);
2715 }
2716
2717 up_write(&SIT_I(sbi)->sentry_lock);
2718 }
2719
2720 static const struct segment_allocation default_salloc_ops = {
2721 .allocate_segment = allocate_segment_by_default,
2722 };
2723
2724 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
2725 struct cp_control *cpc)
2726 {
2727 __u64 trim_start = cpc->trim_start;
2728 bool has_candidate = false;
2729
2730 down_write(&SIT_I(sbi)->sentry_lock);
2731 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
2732 if (add_discard_addrs(sbi, cpc, true)) {
2733 has_candidate = true;
2734 break;
2735 }
2736 }
2737 up_write(&SIT_I(sbi)->sentry_lock);
2738
2739 cpc->trim_start = trim_start;
2740 return has_candidate;
2741 }
2742
2743 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
2744 struct discard_policy *dpolicy,
2745 unsigned int start, unsigned int end)
2746 {
2747 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2748 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
2749 struct rb_node **insert_p = NULL, *insert_parent = NULL;
2750 struct discard_cmd *dc;
2751 struct blk_plug plug;
2752 int issued;
2753 unsigned int trimmed = 0;
2754
2755 next:
2756 issued = 0;
2757
2758 mutex_lock(&dcc->cmd_lock);
2759 if (unlikely(dcc->rbtree_check))
2760 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
2761 &dcc->root));
2762
2763 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
2764 NULL, start,
2765 (struct rb_entry **)&prev_dc,
2766 (struct rb_entry **)&next_dc,
2767 &insert_p, &insert_parent, true, NULL);
2768 if (!dc)
2769 dc = next_dc;
2770
2771 blk_start_plug(&plug);
2772
2773 while (dc && dc->lstart <= end) {
2774 struct rb_node *node;
2775 int err = 0;
2776
2777 if (dc->len < dpolicy->granularity)
2778 goto skip;
2779
2780 if (dc->state != D_PREP) {
2781 list_move_tail(&dc->list, &dcc->fstrim_list);
2782 goto skip;
2783 }
2784
2785 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
2786
2787 if (issued >= dpolicy->max_requests) {
2788 start = dc->lstart + dc->len;
2789
2790 if (err)
2791 __remove_discard_cmd(sbi, dc);
2792
2793 blk_finish_plug(&plug);
2794 mutex_unlock(&dcc->cmd_lock);
2795 trimmed += __wait_all_discard_cmd(sbi, NULL);
2796 congestion_wait(BLK_RW_ASYNC, HZ/50);
2797 goto next;
2798 }
2799 skip:
2800 node = rb_next(&dc->rb_node);
2801 if (err)
2802 __remove_discard_cmd(sbi, dc);
2803 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
2804
2805 if (fatal_signal_pending(current))
2806 break;
2807 }
2808
2809 blk_finish_plug(&plug);
2810 mutex_unlock(&dcc->cmd_lock);
2811
2812 return trimmed;
2813 }
2814
2815 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
2816 {
2817 __u64 start = F2FS_BYTES_TO_BLK(range->start);
2818 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
2819 unsigned int start_segno, end_segno;
2820 block_t start_block, end_block;
2821 struct cp_control cpc;
2822 struct discard_policy dpolicy;
2823 unsigned long long trimmed = 0;
2824 int err = 0;
2825 bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi);
2826
2827 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
2828 return -EINVAL;
2829
2830 if (end < MAIN_BLKADDR(sbi))
2831 goto out;
2832
2833 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
2834 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
2835 return -EFSCORRUPTED;
2836 }
2837
2838
2839 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
2840 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
2841 GET_SEGNO(sbi, end);
2842 if (need_align) {
2843 start_segno = rounddown(start_segno, sbi->segs_per_sec);
2844 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
2845 }
2846
2847 cpc.reason = CP_DISCARD;
2848 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
2849 cpc.trim_start = start_segno;
2850 cpc.trim_end = end_segno;
2851
2852 if (sbi->discard_blks == 0)
2853 goto out;
2854
2855 mutex_lock(&sbi->gc_mutex);
2856 err = f2fs_write_checkpoint(sbi, &cpc);
2857 mutex_unlock(&sbi->gc_mutex);
2858 if (err)
2859 goto out;
2860
2861
2862
2863
2864
2865
2866
2867 if (f2fs_realtime_discard_enable(sbi))
2868 goto out;
2869
2870 start_block = START_BLOCK(sbi, start_segno);
2871 end_block = START_BLOCK(sbi, end_segno + 1);
2872
2873 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
2874 trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
2875 start_block, end_block);
2876
2877 trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
2878 start_block, end_block);
2879 out:
2880 if (!err)
2881 range->len = F2FS_BLK_TO_BYTES(trimmed);
2882 return err;
2883 }
2884
2885 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
2886 {
2887 struct curseg_info *curseg = CURSEG_I(sbi, type);
2888 if (curseg->next_blkoff < sbi->blocks_per_seg)
2889 return true;
2890 return false;
2891 }
2892
2893 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
2894 {
2895 switch (hint) {
2896 case WRITE_LIFE_SHORT:
2897 return CURSEG_HOT_DATA;
2898 case WRITE_LIFE_EXTREME:
2899 return CURSEG_COLD_DATA;
2900 default:
2901 return CURSEG_WARM_DATA;
2902 }
2903 }
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
2967 enum page_type type, enum temp_type temp)
2968 {
2969 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
2970 if (type == DATA) {
2971 if (temp == WARM)
2972 return WRITE_LIFE_NOT_SET;
2973 else if (temp == HOT)
2974 return WRITE_LIFE_SHORT;
2975 else if (temp == COLD)
2976 return WRITE_LIFE_EXTREME;
2977 } else {
2978 return WRITE_LIFE_NOT_SET;
2979 }
2980 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
2981 if (type == DATA) {
2982 if (temp == WARM)
2983 return WRITE_LIFE_LONG;
2984 else if (temp == HOT)
2985 return WRITE_LIFE_SHORT;
2986 else if (temp == COLD)
2987 return WRITE_LIFE_EXTREME;
2988 } else if (type == NODE) {
2989 if (temp == WARM || temp == HOT)
2990 return WRITE_LIFE_NOT_SET;
2991 else if (temp == COLD)
2992 return WRITE_LIFE_NONE;
2993 } else if (type == META) {
2994 return WRITE_LIFE_MEDIUM;
2995 }
2996 }
2997 return WRITE_LIFE_NOT_SET;
2998 }
2999
3000 static int __get_segment_type_2(struct f2fs_io_info *fio)
3001 {
3002 if (fio->type == DATA)
3003 return CURSEG_HOT_DATA;
3004 else
3005 return CURSEG_HOT_NODE;
3006 }
3007
3008 static int __get_segment_type_4(struct f2fs_io_info *fio)
3009 {
3010 if (fio->type == DATA) {
3011 struct inode *inode = fio->page->mapping->host;
3012
3013 if (S_ISDIR(inode->i_mode))
3014 return CURSEG_HOT_DATA;
3015 else
3016 return CURSEG_COLD_DATA;
3017 } else {
3018 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3019 return CURSEG_WARM_NODE;
3020 else
3021 return CURSEG_COLD_NODE;
3022 }
3023 }
3024
3025 static int __get_segment_type_6(struct f2fs_io_info *fio)
3026 {
3027 if (fio->type == DATA) {
3028 struct inode *inode = fio->page->mapping->host;
3029
3030 if (is_cold_data(fio->page) || file_is_cold(inode))
3031 return CURSEG_COLD_DATA;
3032 if (file_is_hot(inode) ||
3033 is_inode_flag_set(inode, FI_HOT_DATA) ||
3034 f2fs_is_atomic_file(inode) ||
3035 f2fs_is_volatile_file(inode))
3036 return CURSEG_HOT_DATA;
3037 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3038 } else {
3039 if (IS_DNODE(fio->page))
3040 return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3041 CURSEG_HOT_NODE;
3042 return CURSEG_COLD_NODE;
3043 }
3044 }
3045
3046 static int __get_segment_type(struct f2fs_io_info *fio)
3047 {
3048 int type = 0;
3049
3050 switch (F2FS_OPTION(fio->sbi).active_logs) {
3051 case 2:
3052 type = __get_segment_type_2(fio);
3053 break;
3054 case 4:
3055 type = __get_segment_type_4(fio);
3056 break;
3057 case 6:
3058 type = __get_segment_type_6(fio);
3059 break;
3060 default:
3061 f2fs_bug_on(fio->sbi, true);
3062 }
3063
3064 if (IS_HOT(type))
3065 fio->temp = HOT;
3066 else if (IS_WARM(type))
3067 fio->temp = WARM;
3068 else
3069 fio->temp = COLD;
3070 return type;
3071 }
3072
3073 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3074 block_t old_blkaddr, block_t *new_blkaddr,
3075 struct f2fs_summary *sum, int type,
3076 struct f2fs_io_info *fio, bool add_list)
3077 {
3078 struct sit_info *sit_i = SIT_I(sbi);
3079 struct curseg_info *curseg = CURSEG_I(sbi, type);
3080
3081 down_read(&SM_I(sbi)->curseg_lock);
3082
3083 mutex_lock(&curseg->curseg_mutex);
3084 down_write(&sit_i->sentry_lock);
3085
3086 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3087
3088 f2fs_wait_discard_bio(sbi, *new_blkaddr);
3089
3090
3091
3092
3093
3094
3095 __add_sum_entry(sbi, type, sum);
3096
3097 __refresh_next_blkoff(sbi, curseg);
3098
3099 stat_inc_block_count(sbi, curseg);
3100
3101
3102
3103
3104
3105 update_sit_entry(sbi, *new_blkaddr, 1);
3106 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3107 update_sit_entry(sbi, old_blkaddr, -1);
3108
3109 if (!__has_curseg_space(sbi, type))
3110 sit_i->s_ops->allocate_segment(sbi, type, false);
3111
3112
3113
3114
3115
3116
3117 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3118 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3119
3120 up_write(&sit_i->sentry_lock);
3121
3122 if (page && IS_NODESEG(type)) {
3123 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3124
3125 f2fs_inode_chksum_set(sbi, page);
3126 }
3127
3128 if (F2FS_IO_ALIGNED(sbi))
3129 fio->retry = false;
3130
3131 if (add_list) {
3132 struct f2fs_bio_info *io;
3133
3134 INIT_LIST_HEAD(&fio->list);
3135 fio->in_list = true;
3136 io = sbi->write_io[fio->type] + fio->temp;
3137 spin_lock(&io->io_lock);
3138 list_add_tail(&fio->list, &io->io_list);
3139 spin_unlock(&io->io_lock);
3140 }
3141
3142 mutex_unlock(&curseg->curseg_mutex);
3143
3144 up_read(&SM_I(sbi)->curseg_lock);
3145 }
3146
3147 static void update_device_state(struct f2fs_io_info *fio)
3148 {
3149 struct f2fs_sb_info *sbi = fio->sbi;
3150 unsigned int devidx;
3151
3152 if (!f2fs_is_multi_device(sbi))
3153 return;
3154
3155 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);
3156
3157
3158 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);
3159
3160
3161 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3162 spin_lock(&sbi->dev_lock);
3163 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3164 spin_unlock(&sbi->dev_lock);
3165 }
3166 }
3167
3168 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3169 {
3170 int type = __get_segment_type(fio);
3171 bool keep_order = (test_opt(fio->sbi, LFS) && type == CURSEG_COLD_DATA);
3172
3173 if (keep_order)
3174 down_read(&fio->sbi->io_order_lock);
3175 reallocate:
3176 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3177 &fio->new_blkaddr, sum, type, fio, true);
3178 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
3179 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3180 fio->old_blkaddr, fio->old_blkaddr);
3181
3182
3183 f2fs_submit_page_write(fio);
3184 if (fio->retry) {
3185 fio->old_blkaddr = fio->new_blkaddr;
3186 goto reallocate;
3187 }
3188
3189 update_device_state(fio);
3190
3191 if (keep_order)
3192 up_read(&fio->sbi->io_order_lock);
3193 }
3194
3195 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3196 enum iostat_type io_type)
3197 {
3198 struct f2fs_io_info fio = {
3199 .sbi = sbi,
3200 .type = META,
3201 .temp = HOT,
3202 .op = REQ_OP_WRITE,
3203 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3204 .old_blkaddr = page->index,
3205 .new_blkaddr = page->index,
3206 .page = page,
3207 .encrypted_page = NULL,
3208 .in_list = false,
3209 };
3210
3211 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3212 fio.op_flags &= ~REQ_META;
3213
3214 set_page_writeback(page);
3215 ClearPageError(page);
3216 f2fs_submit_page_write(&fio);
3217
3218 stat_inc_meta_count(sbi, page->index);
3219 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3220 }
3221
3222 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3223 {
3224 struct f2fs_summary sum;
3225
3226 set_summary(&sum, nid, 0, 0);
3227 do_write_page(&sum, fio);
3228
3229 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3230 }
3231
3232 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3233 struct f2fs_io_info *fio)
3234 {
3235 struct f2fs_sb_info *sbi = fio->sbi;
3236 struct f2fs_summary sum;
3237
3238 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3239 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3240 do_write_page(&sum, fio);
3241 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3242
3243 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3244 }
3245
3246 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3247 {
3248 int err;
3249 struct f2fs_sb_info *sbi = fio->sbi;
3250 unsigned int segno;
3251
3252 fio->new_blkaddr = fio->old_blkaddr;
3253
3254 __get_segment_type(fio);
3255
3256 segno = GET_SEGNO(sbi, fio->new_blkaddr);
3257
3258 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3259 set_sbi_flag(sbi, SBI_NEED_FSCK);
3260 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3261 __func__, segno);
3262 return -EFSCORRUPTED;
3263 }
3264
3265 stat_inc_inplace_blocks(fio->sbi);
3266
3267 if (fio->bio)
3268 err = f2fs_merge_page_bio(fio);
3269 else
3270 err = f2fs_submit_page_bio(fio);
3271 if (!err) {
3272 update_device_state(fio);
3273 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3274 }
3275
3276 return err;
3277 }
3278
3279 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3280 unsigned int segno)
3281 {
3282 int i;
3283
3284 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3285 if (CURSEG_I(sbi, i)->segno == segno)
3286 break;
3287 }
3288 return i;
3289 }
3290
3291 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3292 block_t old_blkaddr, block_t new_blkaddr,
3293 bool recover_curseg, bool recover_newaddr)
3294 {
3295 struct sit_info *sit_i = SIT_I(sbi);
3296 struct curseg_info *curseg;
3297 unsigned int segno, old_cursegno;
3298 struct seg_entry *se;
3299 int type;
3300 unsigned short old_blkoff;
3301
3302 segno = GET_SEGNO(sbi, new_blkaddr);
3303 se = get_seg_entry(sbi, segno);
3304 type = se->type;
3305
3306 down_write(&SM_I(sbi)->curseg_lock);
3307
3308 if (!recover_curseg) {
3309
3310 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3311 if (old_blkaddr == NULL_ADDR)
3312 type = CURSEG_COLD_DATA;
3313 else
3314 type = CURSEG_WARM_DATA;
3315 }
3316 } else {
3317 if (IS_CURSEG(sbi, segno)) {
3318
3319 type = __f2fs_get_curseg(sbi, segno);
3320 f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3321 } else {
3322 type = CURSEG_WARM_DATA;
3323 }
3324 }
3325
3326 f2fs_bug_on(sbi, !IS_DATASEG(type));
3327 curseg = CURSEG_I(sbi, type);
3328
3329 mutex_lock(&curseg->curseg_mutex);
3330 down_write(&sit_i->sentry_lock);
3331
3332 old_cursegno = curseg->segno;
3333 old_blkoff = curseg->next_blkoff;
3334
3335
3336 if (segno != curseg->segno) {
3337 curseg->next_segno = segno;
3338 change_curseg(sbi, type);
3339 }
3340
3341 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3342 __add_sum_entry(sbi, type, sum);
3343
3344 if (!recover_curseg || recover_newaddr)
3345 update_sit_entry(sbi, new_blkaddr, 1);
3346 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3347 invalidate_mapping_pages(META_MAPPING(sbi),
3348 old_blkaddr, old_blkaddr);
3349 update_sit_entry(sbi, old_blkaddr, -1);
3350 }
3351
3352 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3353 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3354
3355 locate_dirty_segment(sbi, old_cursegno);
3356
3357 if (recover_curseg) {
3358 if (old_cursegno != curseg->segno) {
3359 curseg->next_segno = old_cursegno;
3360 change_curseg(sbi, type);
3361 }
3362 curseg->next_blkoff = old_blkoff;
3363 }
3364
3365 up_write(&sit_i->sentry_lock);
3366 mutex_unlock(&curseg->curseg_mutex);
3367 up_write(&SM_I(sbi)->curseg_lock);
3368 }
3369
3370 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3371 block_t old_addr, block_t new_addr,
3372 unsigned char version, bool recover_curseg,
3373 bool recover_newaddr)
3374 {
3375 struct f2fs_summary sum;
3376
3377 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3378
3379 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3380 recover_curseg, recover_newaddr);
3381
3382 f2fs_update_data_blkaddr(dn, new_addr);
3383 }
3384
3385 void f2fs_wait_on_page_writeback(struct page *page,
3386 enum page_type type, bool ordered, bool locked)
3387 {
3388 if (PageWriteback(page)) {
3389 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3390
3391 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3392 if (ordered) {
3393 wait_on_page_writeback(page);
3394 f2fs_bug_on(sbi, locked && PageWriteback(page));
3395 } else {
3396 wait_for_stable_page(page);
3397 }
3398 }
3399 }
3400
3401 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3402 {
3403 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3404 struct page *cpage;
3405
3406 if (!f2fs_post_read_required(inode))
3407 return;
3408
3409 if (!__is_valid_data_blkaddr(blkaddr))
3410 return;
3411
3412 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3413 if (cpage) {
3414 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3415 f2fs_put_page(cpage, 1);
3416 }
3417 }
3418
3419 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3420 block_t len)
3421 {
3422 block_t i;
3423
3424 for (i = 0; i < len; i++)
3425 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3426 }
3427
3428 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3429 {
3430 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3431 struct curseg_info *seg_i;
3432 unsigned char *kaddr;
3433 struct page *page;
3434 block_t start;
3435 int i, j, offset;
3436
3437 start = start_sum_block(sbi);
3438
3439 page = f2fs_get_meta_page(sbi, start++);
3440 if (IS_ERR(page))
3441 return PTR_ERR(page);
3442 kaddr = (unsigned char *)page_address(page);
3443
3444
3445 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3446 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3447
3448
3449 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3450 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3451 offset = 2 * SUM_JOURNAL_SIZE;
3452
3453
3454 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3455 unsigned short blk_off;
3456 unsigned int segno;
3457
3458 seg_i = CURSEG_I(sbi, i);
3459 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3460 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3461 seg_i->next_segno = segno;
3462 reset_curseg(sbi, i, 0);
3463 seg_i->alloc_type = ckpt->alloc_type[i];
3464 seg_i->next_blkoff = blk_off;
3465
3466 if (seg_i->alloc_type == SSR)
3467 blk_off = sbi->blocks_per_seg;
3468
3469 for (j = 0; j < blk_off; j++) {
3470 struct f2fs_summary *s;
3471 s = (struct f2fs_summary *)(kaddr + offset);
3472 seg_i->sum_blk->entries[j] = *s;
3473 offset += SUMMARY_SIZE;
3474 if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3475 SUM_FOOTER_SIZE)
3476 continue;
3477
3478 f2fs_put_page(page, 1);
3479 page = NULL;
3480
3481 page = f2fs_get_meta_page(sbi, start++);
3482 if (IS_ERR(page))
3483 return PTR_ERR(page);
3484 kaddr = (unsigned char *)page_address(page);
3485 offset = 0;
3486 }
3487 }
3488 f2fs_put_page(page, 1);
3489 return 0;
3490 }
3491
3492 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3493 {
3494 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3495 struct f2fs_summary_block *sum;
3496 struct curseg_info *curseg;
3497 struct page *new;
3498 unsigned short blk_off;
3499 unsigned int segno = 0;
3500 block_t blk_addr = 0;
3501 int err = 0;
3502
3503
3504 if (IS_DATASEG(type)) {
3505 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3506 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3507 CURSEG_HOT_DATA]);
3508 if (__exist_node_summaries(sbi))
3509 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
3510 else
3511 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3512 } else {
3513 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3514 CURSEG_HOT_NODE]);
3515 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3516 CURSEG_HOT_NODE]);
3517 if (__exist_node_summaries(sbi))
3518 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3519 type - CURSEG_HOT_NODE);
3520 else
3521 blk_addr = GET_SUM_BLOCK(sbi, segno);
3522 }
3523
3524 new = f2fs_get_meta_page(sbi, blk_addr);
3525 if (IS_ERR(new))
3526 return PTR_ERR(new);
3527 sum = (struct f2fs_summary_block *)page_address(new);
3528
3529 if (IS_NODESEG(type)) {
3530 if (__exist_node_summaries(sbi)) {
3531 struct f2fs_summary *ns = &sum->entries[0];
3532 int i;
3533 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3534 ns->version = 0;
3535 ns->ofs_in_node = 0;
3536 }
3537 } else {
3538 err = f2fs_restore_node_summary(sbi, segno, sum);
3539 if (err)
3540 goto out;
3541 }
3542 }
3543
3544
3545 curseg = CURSEG_I(sbi, type);
3546 mutex_lock(&curseg->curseg_mutex);
3547
3548
3549 down_write(&curseg->journal_rwsem);
3550 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3551 up_write(&curseg->journal_rwsem);
3552
3553 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3554 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3555 curseg->next_segno = segno;
3556 reset_curseg(sbi, type, 0);
3557 curseg->alloc_type = ckpt->alloc_type[type];
3558 curseg->next_blkoff = blk_off;
3559 mutex_unlock(&curseg->curseg_mutex);
3560 out:
3561 f2fs_put_page(new, 1);
3562 return err;
3563 }
3564
3565 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3566 {
3567 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3568 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3569 int type = CURSEG_HOT_DATA;
3570 int err;
3571
3572 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3573 int npages = f2fs_npages_for_summary_flush(sbi, true);
3574
3575 if (npages >= 2)
3576 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3577 META_CP, true);
3578
3579
3580 err = read_compacted_summaries(sbi);
3581 if (err)
3582 return err;
3583 type = CURSEG_HOT_NODE;
3584 }
3585
3586 if (__exist_node_summaries(sbi))
3587 f2fs_ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
3588 NR_CURSEG_TYPE - type, META_CP, true);
3589
3590 for (; type <= CURSEG_COLD_NODE; type++) {
3591 err = read_normal_summaries(sbi, type);
3592 if (err)
3593 return err;
3594 }
3595
3596
3597 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3598 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3599 f2fs_err(sbi, "invalid journal entries nats %u sits %u\n",
3600 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3601 return -EINVAL;
3602 }
3603
3604 return 0;
3605 }
3606
3607 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3608 {
3609 struct page *page;
3610 unsigned char *kaddr;
3611 struct f2fs_summary *summary;
3612 struct curseg_info *seg_i;
3613 int written_size = 0;
3614 int i, j;
3615
3616 page = f2fs_grab_meta_page(sbi, blkaddr++);
3617 kaddr = (unsigned char *)page_address(page);
3618 memset(kaddr, 0, PAGE_SIZE);
3619
3620
3621 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3622 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3623 written_size += SUM_JOURNAL_SIZE;
3624
3625
3626 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3627 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3628 written_size += SUM_JOURNAL_SIZE;
3629
3630
3631 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3632 unsigned short blkoff;
3633 seg_i = CURSEG_I(sbi, i);
3634 if (sbi->ckpt->alloc_type[i] == SSR)
3635 blkoff = sbi->blocks_per_seg;
3636 else
3637 blkoff = curseg_blkoff(sbi, i);
3638
3639 for (j = 0; j < blkoff; j++) {
3640 if (!page) {
3641 page = f2fs_grab_meta_page(sbi, blkaddr++);
3642 kaddr = (unsigned char *)page_address(page);
3643 memset(kaddr, 0, PAGE_SIZE);
3644 written_size = 0;
3645 }
3646 summary = (struct f2fs_summary *)(kaddr + written_size);
3647 *summary = seg_i->sum_blk->entries[j];
3648 written_size += SUMMARY_SIZE;
3649
3650 if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
3651 SUM_FOOTER_SIZE)
3652 continue;
3653
3654 set_page_dirty(page);
3655 f2fs_put_page(page, 1);
3656 page = NULL;
3657 }
3658 }
3659 if (page) {
3660 set_page_dirty(page);
3661 f2fs_put_page(page, 1);
3662 }
3663 }
3664
3665 static void write_normal_summaries(struct f2fs_sb_info *sbi,
3666 block_t blkaddr, int type)
3667 {
3668 int i, end;
3669 if (IS_DATASEG(type))
3670 end = type + NR_CURSEG_DATA_TYPE;
3671 else
3672 end = type + NR_CURSEG_NODE_TYPE;
3673
3674 for (i = type; i < end; i++)
3675 write_current_sum_page(sbi, i, blkaddr + (i - type));
3676 }
3677
3678 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3679 {
3680 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
3681 write_compacted_summaries(sbi, start_blk);
3682 else
3683 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
3684 }
3685
3686 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3687 {
3688 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
3689 }
3690
3691 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3692 unsigned int val, int alloc)
3693 {
3694 int i;
3695
3696 if (type == NAT_JOURNAL) {
3697 for (i = 0; i < nats_in_cursum(journal); i++) {
3698 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
3699 return i;
3700 }
3701 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
3702 return update_nats_in_cursum(journal, 1);
3703 } else if (type == SIT_JOURNAL) {
3704 for (i = 0; i < sits_in_cursum(journal); i++)
3705 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
3706 return i;
3707 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
3708 return update_sits_in_cursum(journal, 1);
3709 }
3710 return -1;
3711 }
3712
3713 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
3714 unsigned int segno)
3715 {
3716 return f2fs_get_meta_page_nofail(sbi, current_sit_addr(sbi, segno));
3717 }
3718
3719 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
3720 unsigned int start)
3721 {
3722 struct sit_info *sit_i = SIT_I(sbi);
3723 struct page *page;
3724 pgoff_t src_off, dst_off;
3725
3726 src_off = current_sit_addr(sbi, start);
3727 dst_off = next_sit_addr(sbi, src_off);
3728
3729 page = f2fs_grab_meta_page(sbi, dst_off);
3730 seg_info_to_sit_page(sbi, page, start);
3731
3732 set_page_dirty(page);
3733 set_to_next_sit(sit_i, start);
3734
3735 return page;
3736 }
3737
3738 static struct sit_entry_set *grab_sit_entry_set(void)
3739 {
3740 struct sit_entry_set *ses =
3741 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
3742
3743 ses->entry_cnt = 0;
3744 INIT_LIST_HEAD(&ses->set_list);
3745 return ses;
3746 }
3747
3748 static void release_sit_entry_set(struct sit_entry_set *ses)
3749 {
3750 list_del(&ses->set_list);
3751 kmem_cache_free(sit_entry_set_slab, ses);
3752 }
3753
3754 static void adjust_sit_entry_set(struct sit_entry_set *ses,
3755 struct list_head *head)
3756 {
3757 struct sit_entry_set *next = ses;
3758
3759 if (list_is_last(&ses->set_list, head))
3760 return;
3761
3762 list_for_each_entry_continue(next, head, set_list)
3763 if (ses->entry_cnt <= next->entry_cnt)
3764 break;
3765
3766 list_move_tail(&ses->set_list, &next->set_list);
3767 }
3768
3769 static void add_sit_entry(unsigned int segno, struct list_head *head)
3770 {
3771 struct sit_entry_set *ses;
3772 unsigned int start_segno = START_SEGNO(segno);
3773
3774 list_for_each_entry(ses, head, set_list) {
3775 if (ses->start_segno == start_segno) {
3776 ses->entry_cnt++;
3777 adjust_sit_entry_set(ses, head);
3778 return;
3779 }
3780 }
3781
3782 ses = grab_sit_entry_set();
3783
3784 ses->start_segno = start_segno;
3785 ses->entry_cnt++;
3786 list_add(&ses->set_list, head);
3787 }
3788
3789 static void add_sits_in_set(struct f2fs_sb_info *sbi)
3790 {
3791 struct f2fs_sm_info *sm_info = SM_I(sbi);
3792 struct list_head *set_list = &sm_info->sit_entry_set;
3793 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
3794 unsigned int segno;
3795
3796 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
3797 add_sit_entry(segno, set_list);
3798 }
3799
3800 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
3801 {
3802 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
3803 struct f2fs_journal *journal = curseg->journal;
3804 int i;
3805
3806 down_write(&curseg->journal_rwsem);
3807 for (i = 0; i < sits_in_cursum(journal); i++) {
3808 unsigned int segno;
3809 bool dirtied;
3810
3811 segno = le32_to_cpu(segno_in_journal(journal, i));
3812 dirtied = __mark_sit_entry_dirty(sbi, segno);
3813
3814 if (!dirtied)
3815 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
3816 }
3817 update_sits_in_cursum(journal, -i);
3818 up_write(&curseg->journal_rwsem);
3819 }
3820
3821
3822
3823
3824
3825 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
3826 {
3827 struct sit_info *sit_i = SIT_I(sbi);
3828 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
3829 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
3830 struct f2fs_journal *journal = curseg->journal;
3831 struct sit_entry_set *ses, *tmp;
3832 struct list_head *head = &SM_I(sbi)->sit_entry_set;
3833 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
3834 struct seg_entry *se;
3835
3836 down_write(&sit_i->sentry_lock);
3837
3838 if (!sit_i->dirty_sentries)
3839 goto out;
3840
3841
3842
3843
3844
3845 add_sits_in_set(sbi);
3846
3847
3848
3849
3850
3851
3852 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
3853 !to_journal)
3854 remove_sits_in_journal(sbi);
3855
3856
3857
3858
3859
3860
3861 list_for_each_entry_safe(ses, tmp, head, set_list) {
3862 struct page *page = NULL;
3863 struct f2fs_sit_block *raw_sit = NULL;
3864 unsigned int start_segno = ses->start_segno;
3865 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
3866 (unsigned long)MAIN_SEGS(sbi));
3867 unsigned int segno = start_segno;
3868
3869 if (to_journal &&
3870 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
3871 to_journal = false;
3872
3873 if (to_journal) {
3874 down_write(&curseg->journal_rwsem);
3875 } else {
3876 page = get_next_sit_page(sbi, start_segno);
3877 raw_sit = page_address(page);
3878 }
3879
3880
3881 for_each_set_bit_from(segno, bitmap, end) {
3882 int offset, sit_offset;
3883
3884 se = get_seg_entry(sbi, segno);
3885 #ifdef CONFIG_F2FS_CHECK_FS
3886 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
3887 SIT_VBLOCK_MAP_SIZE))
3888 f2fs_bug_on(sbi, 1);
3889 #endif
3890
3891
3892 if (!(cpc->reason & CP_DISCARD)) {
3893 cpc->trim_start = segno;
3894 add_discard_addrs(sbi, cpc, false);
3895 }
3896
3897 if (to_journal) {
3898 offset = f2fs_lookup_journal_in_cursum(journal,
3899 SIT_JOURNAL, segno, 1);
3900 f2fs_bug_on(sbi, offset < 0);
3901 segno_in_journal(journal, offset) =
3902 cpu_to_le32(segno);
3903 seg_info_to_raw_sit(se,
3904 &sit_in_journal(journal, offset));
3905 check_block_count(sbi, segno,
3906 &sit_in_journal(journal, offset));
3907 } else {
3908 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
3909 seg_info_to_raw_sit(se,
3910 &raw_sit->entries[sit_offset]);
3911 check_block_count(sbi, segno,
3912 &raw_sit->entries[sit_offset]);
3913 }
3914
3915 __clear_bit(segno, bitmap);
3916 sit_i->dirty_sentries--;
3917 ses->entry_cnt--;
3918 }
3919
3920 if (to_journal)
3921 up_write(&curseg->journal_rwsem);
3922 else
3923 f2fs_put_page(page, 1);
3924
3925 f2fs_bug_on(sbi, ses->entry_cnt);
3926 release_sit_entry_set(ses);
3927 }
3928
3929 f2fs_bug_on(sbi, !list_empty(head));
3930 f2fs_bug_on(sbi, sit_i->dirty_sentries);
3931 out:
3932 if (cpc->reason & CP_DISCARD) {
3933 __u64 trim_start = cpc->trim_start;
3934
3935 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
3936 add_discard_addrs(sbi, cpc, false);
3937
3938 cpc->trim_start = trim_start;
3939 }
3940 up_write(&sit_i->sentry_lock);
3941
3942 set_prefree_as_free_segments(sbi);
3943 }
3944
3945 static int build_sit_info(struct f2fs_sb_info *sbi)
3946 {
3947 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
3948 struct sit_info *sit_i;
3949 unsigned int sit_segs, start;
3950 char *src_bitmap, *bitmap;
3951 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
3952
3953
3954 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
3955 if (!sit_i)
3956 return -ENOMEM;
3957
3958 SM_I(sbi)->sit_info = sit_i;
3959
3960 sit_i->sentries =
3961 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
3962 MAIN_SEGS(sbi)),
3963 GFP_KERNEL);
3964 if (!sit_i->sentries)
3965 return -ENOMEM;
3966
3967 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
3968 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
3969 GFP_KERNEL);
3970 if (!sit_i->dirty_sentries_bitmap)
3971 return -ENOMEM;
3972
3973 #ifdef CONFIG_F2FS_CHECK_FS
3974 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;
3975 #else
3976 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;
3977 #endif
3978 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
3979 if (!sit_i->bitmap)
3980 return -ENOMEM;
3981
3982 bitmap = sit_i->bitmap;
3983
3984 for (start = 0; start < MAIN_SEGS(sbi); start++) {
3985 sit_i->sentries[start].cur_valid_map = bitmap;
3986 bitmap += SIT_VBLOCK_MAP_SIZE;
3987
3988 sit_i->sentries[start].ckpt_valid_map = bitmap;
3989 bitmap += SIT_VBLOCK_MAP_SIZE;
3990
3991 #ifdef CONFIG_F2FS_CHECK_FS
3992 sit_i->sentries[start].cur_valid_map_mir = bitmap;
3993 bitmap += SIT_VBLOCK_MAP_SIZE;
3994 #endif
3995
3996 sit_i->sentries[start].discard_map = bitmap;
3997 bitmap += SIT_VBLOCK_MAP_SIZE;
3998 }
3999
4000 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4001 if (!sit_i->tmp_map)
4002 return -ENOMEM;
4003
4004 if (__is_large_section(sbi)) {
4005 sit_i->sec_entries =
4006 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4007 MAIN_SECS(sbi)),
4008 GFP_KERNEL);
4009 if (!sit_i->sec_entries)
4010 return -ENOMEM;
4011 }
4012
4013
4014 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4015
4016
4017 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4018 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4019
4020 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4021 if (!sit_i->sit_bitmap)
4022 return -ENOMEM;
4023
4024 #ifdef CONFIG_F2FS_CHECK_FS
4025 sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4026 sit_bitmap_size, GFP_KERNEL);
4027 if (!sit_i->sit_bitmap_mir)
4028 return -ENOMEM;
4029
4030 sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4031 main_bitmap_size, GFP_KERNEL);
4032 if (!sit_i->invalid_segmap)
4033 return -ENOMEM;
4034 #endif
4035
4036
4037 sit_i->s_ops = &default_salloc_ops;
4038
4039 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4040 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4041 sit_i->written_valid_blocks = 0;
4042 sit_i->bitmap_size = sit_bitmap_size;
4043 sit_i->dirty_sentries = 0;
4044 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4045 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4046 sit_i->mounted_time = ktime_get_real_seconds();
4047 init_rwsem(&sit_i->sentry_lock);
4048 return 0;
4049 }
4050
4051 static int build_free_segmap(struct f2fs_sb_info *sbi)
4052 {
4053 struct free_segmap_info *free_i;
4054 unsigned int bitmap_size, sec_bitmap_size;
4055
4056
4057 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4058 if (!free_i)
4059 return -ENOMEM;
4060
4061 SM_I(sbi)->free_info = free_i;
4062
4063 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4064 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4065 if (!free_i->free_segmap)
4066 return -ENOMEM;
4067
4068 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4069 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4070 if (!free_i->free_secmap)
4071 return -ENOMEM;
4072
4073
4074 memset(free_i->free_segmap, 0xff, bitmap_size);
4075 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4076
4077
4078 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4079 free_i->free_segments = 0;
4080 free_i->free_sections = 0;
4081 spin_lock_init(&free_i->segmap_lock);
4082 return 0;
4083 }
4084
4085 static int build_curseg(struct f2fs_sb_info *sbi)
4086 {
4087 struct curseg_info *array;
4088 int i;
4089
4090 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, sizeof(*array)),
4091 GFP_KERNEL);
4092 if (!array)
4093 return -ENOMEM;
4094
4095 SM_I(sbi)->curseg_array = array;
4096
4097 for (i = 0; i < NR_CURSEG_TYPE; i++) {
4098 mutex_init(&array[i].curseg_mutex);
4099 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4100 if (!array[i].sum_blk)
4101 return -ENOMEM;
4102 init_rwsem(&array[i].journal_rwsem);
4103 array[i].journal = f2fs_kzalloc(sbi,
4104 sizeof(struct f2fs_journal), GFP_KERNEL);
4105 if (!array[i].journal)
4106 return -ENOMEM;
4107 array[i].segno = NULL_SEGNO;
4108 array[i].next_blkoff = 0;
4109 }
4110 return restore_curseg_summaries(sbi);
4111 }
4112
4113 static int build_sit_entries(struct f2fs_sb_info *sbi)
4114 {
4115 struct sit_info *sit_i = SIT_I(sbi);
4116 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4117 struct f2fs_journal *journal = curseg->journal;
4118 struct seg_entry *se;
4119 struct f2fs_sit_entry sit;
4120 int sit_blk_cnt = SIT_BLK_CNT(sbi);
4121 unsigned int i, start, end;
4122 unsigned int readed, start_blk = 0;
4123 int err = 0;
4124 block_t total_node_blocks = 0;
4125
4126 do {
4127 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
4128 META_SIT, true);
4129
4130 start = start_blk * sit_i->sents_per_block;
4131 end = (start_blk + readed) * sit_i->sents_per_block;
4132
4133 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4134 struct f2fs_sit_block *sit_blk;
4135 struct page *page;
4136
4137 se = &sit_i->sentries[start];
4138 page = get_current_sit_page(sbi, start);
4139 if (IS_ERR(page))
4140 return PTR_ERR(page);
4141 sit_blk = (struct f2fs_sit_block *)page_address(page);
4142 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4143 f2fs_put_page(page, 1);
4144
4145 err = check_block_count(sbi, start, &sit);
4146 if (err)
4147 return err;
4148 seg_info_from_raw_sit(se, &sit);
4149 if (IS_NODESEG(se->type))
4150 total_node_blocks += se->valid_blocks;
4151
4152
4153 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4154 memset(se->discard_map, 0xff,
4155 SIT_VBLOCK_MAP_SIZE);
4156 } else {
4157 memcpy(se->discard_map,
4158 se->cur_valid_map,
4159 SIT_VBLOCK_MAP_SIZE);
4160 sbi->discard_blks +=
4161 sbi->blocks_per_seg -
4162 se->valid_blocks;
4163 }
4164
4165 if (__is_large_section(sbi))
4166 get_sec_entry(sbi, start)->valid_blocks +=
4167 se->valid_blocks;
4168 }
4169 start_blk += readed;
4170 } while (start_blk < sit_blk_cnt);
4171
4172 down_read(&curseg->journal_rwsem);
4173 for (i = 0; i < sits_in_cursum(journal); i++) {
4174 unsigned int old_valid_blocks;
4175
4176 start = le32_to_cpu(segno_in_journal(journal, i));
4177 if (start >= MAIN_SEGS(sbi)) {
4178 f2fs_err(sbi, "Wrong journal entry on segno %u",
4179 start);
4180 err = -EFSCORRUPTED;
4181 break;
4182 }
4183
4184 se = &sit_i->sentries[start];
4185 sit = sit_in_journal(journal, i);
4186
4187 old_valid_blocks = se->valid_blocks;
4188 if (IS_NODESEG(se->type))
4189 total_node_blocks -= old_valid_blocks;
4190
4191 err = check_block_count(sbi, start, &sit);
4192 if (err)
4193 break;
4194 seg_info_from_raw_sit(se, &sit);
4195 if (IS_NODESEG(se->type))
4196 total_node_blocks += se->valid_blocks;
4197
4198 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4199 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4200 } else {
4201 memcpy(se->discard_map, se->cur_valid_map,
4202 SIT_VBLOCK_MAP_SIZE);
4203 sbi->discard_blks += old_valid_blocks;
4204 sbi->discard_blks -= se->valid_blocks;
4205 }
4206
4207 if (__is_large_section(sbi)) {
4208 get_sec_entry(sbi, start)->valid_blocks +=
4209 se->valid_blocks;
4210 get_sec_entry(sbi, start)->valid_blocks -=
4211 old_valid_blocks;
4212 }
4213 }
4214 up_read(&curseg->journal_rwsem);
4215
4216 if (!err && total_node_blocks != valid_node_count(sbi)) {
4217 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4218 total_node_blocks, valid_node_count(sbi));
4219 err = -EFSCORRUPTED;
4220 }
4221
4222 return err;
4223 }
4224
4225 static void init_free_segmap(struct f2fs_sb_info *sbi)
4226 {
4227 unsigned int start;
4228 int type;
4229
4230 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4231 struct seg_entry *sentry = get_seg_entry(sbi, start);
4232 if (!sentry->valid_blocks)
4233 __set_free(sbi, start);
4234 else
4235 SIT_I(sbi)->written_valid_blocks +=
4236 sentry->valid_blocks;
4237 }
4238
4239
4240 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4241 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4242 __set_test_and_inuse(sbi, curseg_t->segno);
4243 }
4244 }
4245
4246 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4247 {
4248 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4249 struct free_segmap_info *free_i = FREE_I(sbi);
4250 unsigned int segno = 0, offset = 0;
4251 unsigned short valid_blocks;
4252
4253 while (1) {
4254
4255 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4256 if (segno >= MAIN_SEGS(sbi))
4257 break;
4258 offset = segno + 1;
4259 valid_blocks = get_valid_blocks(sbi, segno, false);
4260 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
4261 continue;
4262 if (valid_blocks > sbi->blocks_per_seg) {
4263 f2fs_bug_on(sbi, 1);
4264 continue;
4265 }
4266 mutex_lock(&dirty_i->seglist_lock);
4267 __locate_dirty_segment(sbi, segno, DIRTY);
4268 mutex_unlock(&dirty_i->seglist_lock);
4269 }
4270 }
4271
4272 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4273 {
4274 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4275 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4276
4277 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4278 if (!dirty_i->victim_secmap)
4279 return -ENOMEM;
4280 return 0;
4281 }
4282
4283 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4284 {
4285 struct dirty_seglist_info *dirty_i;
4286 unsigned int bitmap_size, i;
4287
4288
4289 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4290 GFP_KERNEL);
4291 if (!dirty_i)
4292 return -ENOMEM;
4293
4294 SM_I(sbi)->dirty_info = dirty_i;
4295 mutex_init(&dirty_i->seglist_lock);
4296
4297 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4298
4299 for (i = 0; i < NR_DIRTY_TYPE; i++) {
4300 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4301 GFP_KERNEL);
4302 if (!dirty_i->dirty_segmap[i])
4303 return -ENOMEM;
4304 }
4305
4306 init_dirty_segmap(sbi);
4307 return init_victim_secmap(sbi);
4308 }
4309
4310 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4311 {
4312 int i;
4313
4314
4315
4316
4317
4318 for (i = 0; i < NO_CHECK_TYPE; i++) {
4319 struct curseg_info *curseg = CURSEG_I(sbi, i);
4320 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4321 unsigned int blkofs = curseg->next_blkoff;
4322
4323 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4324 goto out;
4325
4326 if (curseg->alloc_type == SSR)
4327 continue;
4328
4329 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4330 if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4331 continue;
4332 out:
4333 f2fs_err(sbi,
4334 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4335 i, curseg->segno, curseg->alloc_type,
4336 curseg->next_blkoff, blkofs);
4337 return -EFSCORRUPTED;
4338 }
4339 }
4340 return 0;
4341 }
4342
4343
4344
4345
4346 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
4347 {
4348 struct sit_info *sit_i = SIT_I(sbi);
4349 unsigned int segno;
4350
4351 down_write(&sit_i->sentry_lock);
4352
4353 sit_i->min_mtime = ULLONG_MAX;
4354
4355 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4356 unsigned int i;
4357 unsigned long long mtime = 0;
4358
4359 for (i = 0; i < sbi->segs_per_sec; i++)
4360 mtime += get_seg_entry(sbi, segno + i)->mtime;
4361
4362 mtime = div_u64(mtime, sbi->segs_per_sec);
4363
4364 if (sit_i->min_mtime > mtime)
4365 sit_i->min_mtime = mtime;
4366 }
4367 sit_i->max_mtime = get_mtime(sbi, false);
4368 up_write(&sit_i->sentry_lock);
4369 }
4370
4371 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
4372 {
4373 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4374 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
4375 struct f2fs_sm_info *sm_info;
4376 int err;
4377
4378 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
4379 if (!sm_info)
4380 return -ENOMEM;
4381
4382
4383 sbi->sm_info = sm_info;
4384 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
4385 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
4386 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
4387 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
4388 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
4389 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
4390 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
4391 sm_info->rec_prefree_segments = sm_info->main_segments *
4392 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
4393 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
4394 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
4395
4396 if (!test_opt(sbi, LFS))
4397 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
4398 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
4399 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
4400 sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;
4401 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
4402 sm_info->min_ssr_sections = reserved_sections(sbi);
4403
4404 INIT_LIST_HEAD(&sm_info->sit_entry_set);
4405
4406 init_rwsem(&sm_info->curseg_lock);
4407
4408 if (!f2fs_readonly(sbi->sb)) {
4409 err = f2fs_create_flush_cmd_control(sbi);
4410 if (err)
4411 return err;
4412 }
4413
4414 err = create_discard_cmd_control(sbi);
4415 if (err)
4416 return err;
4417
4418 err = build_sit_info(sbi);
4419 if (err)
4420 return err;
4421 err = build_free_segmap(sbi);
4422 if (err)
4423 return err;
4424 err = build_curseg(sbi);
4425 if (err)
4426 return err;
4427
4428
4429 err = build_sit_entries(sbi);
4430 if (err)
4431 return err;
4432
4433 init_free_segmap(sbi);
4434 err = build_dirty_segmap(sbi);
4435 if (err)
4436 return err;
4437
4438 err = sanity_check_curseg(sbi);
4439 if (err)
4440 return err;
4441
4442 init_min_max_mtime(sbi);
4443 return 0;
4444 }
4445
4446 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
4447 enum dirty_type dirty_type)
4448 {
4449 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4450
4451 mutex_lock(&dirty_i->seglist_lock);
4452 kvfree(dirty_i->dirty_segmap[dirty_type]);
4453 dirty_i->nr_dirty[dirty_type] = 0;
4454 mutex_unlock(&dirty_i->seglist_lock);
4455 }
4456
4457 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
4458 {
4459 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4460 kvfree(dirty_i->victim_secmap);
4461 }
4462
4463 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
4464 {
4465 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4466 int i;
4467
4468 if (!dirty_i)
4469 return;
4470
4471
4472 for (i = 0; i < NR_DIRTY_TYPE; i++)
4473 discard_dirty_segmap(sbi, i);
4474
4475 destroy_victim_secmap(sbi);
4476 SM_I(sbi)->dirty_info = NULL;
4477 kvfree(dirty_i);
4478 }
4479
4480 static void destroy_curseg(struct f2fs_sb_info *sbi)
4481 {
4482 struct curseg_info *array = SM_I(sbi)->curseg_array;
4483 int i;
4484
4485 if (!array)
4486 return;
4487 SM_I(sbi)->curseg_array = NULL;
4488 for (i = 0; i < NR_CURSEG_TYPE; i++) {
4489 kvfree(array[i].sum_blk);
4490 kvfree(array[i].journal);
4491 }
4492 kvfree(array);
4493 }
4494
4495 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
4496 {
4497 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
4498 if (!free_i)
4499 return;
4500 SM_I(sbi)->free_info = NULL;
4501 kvfree(free_i->free_segmap);
4502 kvfree(free_i->free_secmap);
4503 kvfree(free_i);
4504 }
4505
4506 static void destroy_sit_info(struct f2fs_sb_info *sbi)
4507 {
4508 struct sit_info *sit_i = SIT_I(sbi);
4509
4510 if (!sit_i)
4511 return;
4512
4513 if (sit_i->sentries)
4514 kvfree(sit_i->bitmap);
4515 kvfree(sit_i->tmp_map);
4516
4517 kvfree(sit_i->sentries);
4518 kvfree(sit_i->sec_entries);
4519 kvfree(sit_i->dirty_sentries_bitmap);
4520
4521 SM_I(sbi)->sit_info = NULL;
4522 kvfree(sit_i->sit_bitmap);
4523 #ifdef CONFIG_F2FS_CHECK_FS
4524 kvfree(sit_i->sit_bitmap_mir);
4525 kvfree(sit_i->invalid_segmap);
4526 #endif
4527 kvfree(sit_i);
4528 }
4529
4530 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
4531 {
4532 struct f2fs_sm_info *sm_info = SM_I(sbi);
4533
4534 if (!sm_info)
4535 return;
4536 f2fs_destroy_flush_cmd_control(sbi, true);
4537 destroy_discard_cmd_control(sbi);
4538 destroy_dirty_segmap(sbi);
4539 destroy_curseg(sbi);
4540 destroy_free_segmap(sbi);
4541 destroy_sit_info(sbi);
4542 sbi->sm_info = NULL;
4543 kvfree(sm_info);
4544 }
4545
4546 int __init f2fs_create_segment_manager_caches(void)
4547 {
4548 discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
4549 sizeof(struct discard_entry));
4550 if (!discard_entry_slab)
4551 goto fail;
4552
4553 discard_cmd_slab = f2fs_kmem_cache_create("discard_cmd",
4554 sizeof(struct discard_cmd));
4555 if (!discard_cmd_slab)
4556 goto destroy_discard_entry;
4557
4558 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
4559 sizeof(struct sit_entry_set));
4560 if (!sit_entry_set_slab)
4561 goto destroy_discard_cmd;
4562
4563 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
4564 sizeof(struct inmem_pages));
4565 if (!inmem_entry_slab)
4566 goto destroy_sit_entry_set;
4567 return 0;
4568
4569 destroy_sit_entry_set:
4570 kmem_cache_destroy(sit_entry_set_slab);
4571 destroy_discard_cmd:
4572 kmem_cache_destroy(discard_cmd_slab);
4573 destroy_discard_entry:
4574 kmem_cache_destroy(discard_entry_slab);
4575 fail:
4576 return -ENOMEM;
4577 }
4578
4579 void f2fs_destroy_segment_manager_caches(void)
4580 {
4581 kmem_cache_destroy(sit_entry_set_slab);
4582 kmem_cache_destroy(discard_cmd_slab);
4583 kmem_cache_destroy(discard_entry_slab);
4584 kmem_cache_destroy(inmem_entry_slab);
4585 }