1 2#include <linux/wait.h> 3#include <linux/backing-dev.h> 4#include <linux/kthread.h> 5#include <linux/freezer.h> 6#include <linux/fs.h> 7#include <linux/pagemap.h> 8#include <linux/mm.h> 9#include <linux/sched.h> 10#include <linux/module.h> 11#include <linux/writeback.h> 12#include <linux/device.h> 13#include <trace/events/writeback.h> 14 15static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); 16 17struct backing_dev_info noop_backing_dev_info = { 18 .name = "noop", 19 .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK, 20}; 21 22static struct class *bdi_class; 23 24/* 25 * bdi_lock protects updates to bdi_list. bdi_list has RCU reader side 26 * locking. 27 */ 28DEFINE_SPINLOCK(bdi_lock); 29LIST_HEAD(bdi_list); 30 31/* bdi_wq serves all asynchronous writeback tasks */ 32struct workqueue_struct *bdi_wq; 33 34#ifdef CONFIG_DEBUG_FS 35#include <linux/debugfs.h> 36#include <linux/seq_file.h> 37 38static struct dentry *bdi_debug_root; 39 40static void bdi_debug_init(void) 41{ 42 bdi_debug_root = debugfs_create_dir("bdi", NULL); 43} 44 45static int bdi_debug_stats_show(struct seq_file *m, void *v) 46{ 47 struct backing_dev_info *bdi = m->private; 48 struct bdi_writeback *wb = &bdi->wb; 49 unsigned long background_thresh; 50 unsigned long dirty_thresh; 51 unsigned long bdi_thresh; 52 unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time; 53 struct inode *inode; 54 55 nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0; 56 spin_lock(&wb->list_lock); 57 list_for_each_entry(inode, &wb->b_dirty, i_wb_list) 58 nr_dirty++; 59 list_for_each_entry(inode, &wb->b_io, i_wb_list) 60 nr_io++; 61 list_for_each_entry(inode, &wb->b_more_io, i_wb_list) 62 nr_more_io++; 63 list_for_each_entry(inode, &wb->b_dirty_time, i_wb_list) 64 if (inode->i_state & I_DIRTY_TIME) 65 nr_dirty_time++; 66 spin_unlock(&wb->list_lock); 67 68 global_dirty_limits(&background_thresh, &dirty_thresh); 69 bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh); 70 71#define K(x) ((x) << (PAGE_SHIFT - 10)) 72 seq_printf(m, 73 "BdiWriteback: %10lu kB\n" 74 "BdiReclaimable: %10lu kB\n" 75 "BdiDirtyThresh: %10lu kB\n" 76 "DirtyThresh: %10lu kB\n" 77 "BackgroundThresh: %10lu kB\n" 78 "BdiDirtied: %10lu kB\n" 79 "BdiWritten: %10lu kB\n" 80 "BdiWriteBandwidth: %10lu kBps\n" 81 "b_dirty: %10lu\n" 82 "b_io: %10lu\n" 83 "b_more_io: %10lu\n" 84 "b_dirty_time: %10lu\n" 85 "bdi_list: %10u\n" 86 "state: %10lx\n", 87 (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)), 88 (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)), 89 K(bdi_thresh), 90 K(dirty_thresh), 91 K(background_thresh), 92 (unsigned long) K(bdi_stat(bdi, BDI_DIRTIED)), 93 (unsigned long) K(bdi_stat(bdi, BDI_WRITTEN)), 94 (unsigned long) K(bdi->write_bandwidth), 95 nr_dirty, 96 nr_io, 97 nr_more_io, 98 nr_dirty_time, 99 !list_empty(&bdi->bdi_list), bdi->state); 100#undef K 101 102 return 0; 103} 104 105static int bdi_debug_stats_open(struct inode *inode, struct file *file) 106{ 107 return single_open(file, bdi_debug_stats_show, inode->i_private); 108} 109 110static const struct file_operations bdi_debug_stats_fops = { 111 .open = bdi_debug_stats_open, 112 .read = seq_read, 113 .llseek = seq_lseek, 114 .release = single_release, 115}; 116 117static void bdi_debug_register(struct backing_dev_info *bdi, const char *name) 118{ 119 bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root); 120 bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir, 121 bdi, &bdi_debug_stats_fops); 122} 123 124static void bdi_debug_unregister(struct backing_dev_info *bdi) 125{ 126 debugfs_remove(bdi->debug_stats); 127 debugfs_remove(bdi->debug_dir); 128} 129#else 130static inline void bdi_debug_init(void) 131{ 132} 133static inline void bdi_debug_register(struct backing_dev_info *bdi, 134 const char *name) 135{ 136} 137static inline void bdi_debug_unregister(struct backing_dev_info *bdi) 138{ 139} 140#endif 141 142static ssize_t read_ahead_kb_store(struct device *dev, 143 struct device_attribute *attr, 144 const char *buf, size_t count) 145{ 146 struct backing_dev_info *bdi = dev_get_drvdata(dev); 147 unsigned long read_ahead_kb; 148 ssize_t ret; 149 150 ret = kstrtoul(buf, 10, &read_ahead_kb); 151 if (ret < 0) 152 return ret; 153 154 bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10); 155 156 return count; 157} 158 159#define K(pages) ((pages) << (PAGE_SHIFT - 10)) 160 161#define BDI_SHOW(name, expr) \ 162static ssize_t name##_show(struct device *dev, \ 163 struct device_attribute *attr, char *page) \ 164{ \ 165 struct backing_dev_info *bdi = dev_get_drvdata(dev); \ 166 \ 167 return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \ 168} \ 169static DEVICE_ATTR_RW(name); 170 171BDI_SHOW(read_ahead_kb, K(bdi->ra_pages)) 172 173static ssize_t min_ratio_store(struct device *dev, 174 struct device_attribute *attr, const char *buf, size_t count) 175{ 176 struct backing_dev_info *bdi = dev_get_drvdata(dev); 177 unsigned int ratio; 178 ssize_t ret; 179 180 ret = kstrtouint(buf, 10, &ratio); 181 if (ret < 0) 182 return ret; 183 184 ret = bdi_set_min_ratio(bdi, ratio); 185 if (!ret) 186 ret = count; 187 188 return ret; 189} 190BDI_SHOW(min_ratio, bdi->min_ratio) 191 192static ssize_t max_ratio_store(struct device *dev, 193 struct device_attribute *attr, const char *buf, size_t count) 194{ 195 struct backing_dev_info *bdi = dev_get_drvdata(dev); 196 unsigned int ratio; 197 ssize_t ret; 198 199 ret = kstrtouint(buf, 10, &ratio); 200 if (ret < 0) 201 return ret; 202 203 ret = bdi_set_max_ratio(bdi, ratio); 204 if (!ret) 205 ret = count; 206 207 return ret; 208} 209BDI_SHOW(max_ratio, bdi->max_ratio) 210 211static ssize_t stable_pages_required_show(struct device *dev, 212 struct device_attribute *attr, 213 char *page) 214{ 215 struct backing_dev_info *bdi = dev_get_drvdata(dev); 216 217 return snprintf(page, PAGE_SIZE-1, "%d\n", 218 bdi_cap_stable_pages_required(bdi) ? 1 : 0); 219} 220static DEVICE_ATTR_RO(stable_pages_required); 221 222static struct attribute *bdi_dev_attrs[] = { 223 &dev_attr_read_ahead_kb.attr, 224 &dev_attr_min_ratio.attr, 225 &dev_attr_max_ratio.attr, 226 &dev_attr_stable_pages_required.attr, 227 NULL, 228}; 229ATTRIBUTE_GROUPS(bdi_dev); 230 231static __init int bdi_class_init(void) 232{ 233 bdi_class = class_create(THIS_MODULE, "bdi"); 234 if (IS_ERR(bdi_class)) 235 return PTR_ERR(bdi_class); 236 237 bdi_class->dev_groups = bdi_dev_groups; 238 bdi_debug_init(); 239 return 0; 240} 241postcore_initcall(bdi_class_init); 242 243static int __init default_bdi_init(void) 244{ 245 int err; 246 247 bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_FREEZABLE | 248 WQ_UNBOUND | WQ_SYSFS, 0); 249 if (!bdi_wq) 250 return -ENOMEM; 251 252 err = bdi_init(&noop_backing_dev_info); 253 254 return err; 255} 256subsys_initcall(default_bdi_init); 257 258int bdi_has_dirty_io(struct backing_dev_info *bdi) 259{ 260 return wb_has_dirty_io(&bdi->wb); 261} 262 263/* 264 * This function is used when the first inode for this bdi is marked dirty. It 265 * wakes-up the corresponding bdi thread which should then take care of the 266 * periodic background write-out of dirty inodes. Since the write-out would 267 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just 268 * set up a timer which wakes the bdi thread up later. 269 * 270 * Note, we wouldn't bother setting up the timer, but this function is on the 271 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches 272 * by delaying the wake-up. 273 * 274 * We have to be careful not to postpone flush work if it is scheduled for 275 * earlier. Thus we use queue_delayed_work(). 276 */ 277void bdi_wakeup_thread_delayed(struct backing_dev_info *bdi) 278{ 279 unsigned long timeout; 280 281 timeout = msecs_to_jiffies(dirty_writeback_interval * 10); 282 spin_lock_bh(&bdi->wb_lock); 283 if (test_bit(BDI_registered, &bdi->state)) 284 queue_delayed_work(bdi_wq, &bdi->wb.dwork, timeout); 285 spin_unlock_bh(&bdi->wb_lock); 286} 287 288/* 289 * Remove bdi from bdi_list, and ensure that it is no longer visible 290 */ 291static void bdi_remove_from_list(struct backing_dev_info *bdi) 292{ 293 spin_lock_bh(&bdi_lock); 294 list_del_rcu(&bdi->bdi_list); 295 spin_unlock_bh(&bdi_lock); 296 297 synchronize_rcu_expedited(); 298} 299 300int bdi_register(struct backing_dev_info *bdi, struct device *parent, 301 const char *fmt, ...) 302{ 303 va_list args; 304 struct device *dev; 305 306 if (bdi->dev) /* The driver needs to use separate queues per device */ 307 return 0; 308 309 va_start(args, fmt); 310 dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args); 311 va_end(args); 312 if (IS_ERR(dev)) 313 return PTR_ERR(dev); 314 315 bdi->dev = dev; 316 317 bdi_debug_register(bdi, dev_name(dev)); 318 set_bit(BDI_registered, &bdi->state); 319 320 spin_lock_bh(&bdi_lock); 321 list_add_tail_rcu(&bdi->bdi_list, &bdi_list); 322 spin_unlock_bh(&bdi_lock); 323 324 trace_writeback_bdi_register(bdi); 325 return 0; 326} 327EXPORT_SYMBOL(bdi_register); 328 329int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev) 330{ 331 return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev)); 332} 333EXPORT_SYMBOL(bdi_register_dev); 334 335/* 336 * Remove bdi from the global list and shutdown any threads we have running 337 */ 338static void bdi_wb_shutdown(struct backing_dev_info *bdi) 339{ 340 /* Make sure nobody queues further work */ 341 spin_lock_bh(&bdi->wb_lock); 342 if (!test_and_clear_bit(BDI_registered, &bdi->state)) { 343 spin_unlock_bh(&bdi->wb_lock); 344 return; 345 } 346 spin_unlock_bh(&bdi->wb_lock); 347 348 /* 349 * Make sure nobody finds us on the bdi_list anymore 350 */ 351 bdi_remove_from_list(bdi); 352 353 /* 354 * Drain work list and shutdown the delayed_work. At this point, 355 * @bdi->bdi_list is empty telling bdi_Writeback_workfn() that @bdi 356 * is dying and its work_list needs to be drained no matter what. 357 */ 358 mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0); 359 flush_delayed_work(&bdi->wb.dwork); 360} 361 362static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi) 363{ 364 memset(wb, 0, sizeof(*wb)); 365 366 wb->bdi = bdi; 367 wb->last_old_flush = jiffies; 368 INIT_LIST_HEAD(&wb->b_dirty); 369 INIT_LIST_HEAD(&wb->b_io); 370 INIT_LIST_HEAD(&wb->b_more_io); 371 INIT_LIST_HEAD(&wb->b_dirty_time); 372 spin_lock_init(&wb->list_lock); 373 INIT_DELAYED_WORK(&wb->dwork, bdi_writeback_workfn); 374} 375 376/* 377 * Initial write bandwidth: 100 MB/s 378 */ 379#define INIT_BW (100 << (20 - PAGE_SHIFT)) 380 381int bdi_init(struct backing_dev_info *bdi) 382{ 383 int i, err; 384 385 bdi->dev = NULL; 386 387 bdi->min_ratio = 0; 388 bdi->max_ratio = 100; 389 bdi->max_prop_frac = FPROP_FRAC_BASE; 390 spin_lock_init(&bdi->wb_lock); 391 INIT_LIST_HEAD(&bdi->bdi_list); 392 INIT_LIST_HEAD(&bdi->work_list); 393 394 bdi_wb_init(&bdi->wb, bdi); 395 396 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) { 397 err = percpu_counter_init(&bdi->bdi_stat[i], 0, GFP_KERNEL); 398 if (err) 399 goto err; 400 } 401 402 bdi->dirty_exceeded = 0; 403 404 bdi->bw_time_stamp = jiffies; 405 bdi->written_stamp = 0; 406 407 bdi->balanced_dirty_ratelimit = INIT_BW; 408 bdi->dirty_ratelimit = INIT_BW; 409 bdi->write_bandwidth = INIT_BW; 410 bdi->avg_write_bandwidth = INIT_BW; 411 412 err = fprop_local_init_percpu(&bdi->completions, GFP_KERNEL); 413 414 if (err) { 415err: 416 while (i--) 417 percpu_counter_destroy(&bdi->bdi_stat[i]); 418 } 419 420 return err; 421} 422EXPORT_SYMBOL(bdi_init); 423 424void bdi_destroy(struct backing_dev_info *bdi) 425{ 426 int i; 427 428 bdi_wb_shutdown(bdi); 429 bdi_set_min_ratio(bdi, 0); 430 431 WARN_ON(!list_empty(&bdi->work_list)); 432 WARN_ON(delayed_work_pending(&bdi->wb.dwork)); 433 434 if (bdi->dev) { 435 bdi_debug_unregister(bdi); 436 device_unregister(bdi->dev); 437 bdi->dev = NULL; 438 } 439 440 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) 441 percpu_counter_destroy(&bdi->bdi_stat[i]); 442 fprop_local_destroy_percpu(&bdi->completions); 443} 444EXPORT_SYMBOL(bdi_destroy); 445 446/* 447 * For use from filesystems to quickly init and register a bdi associated 448 * with dirty writeback 449 */ 450int bdi_setup_and_register(struct backing_dev_info *bdi, char *name) 451{ 452 int err; 453 454 bdi->name = name; 455 bdi->capabilities = 0; 456 err = bdi_init(bdi); 457 if (err) 458 return err; 459 460 err = bdi_register(bdi, NULL, "%.28s-%ld", name, 461 atomic_long_inc_return(&bdi_seq)); 462 if (err) { 463 bdi_destroy(bdi); 464 return err; 465 } 466 467 return 0; 468} 469EXPORT_SYMBOL(bdi_setup_and_register); 470 471static wait_queue_head_t congestion_wqh[2] = { 472 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]), 473 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1]) 474 }; 475static atomic_t nr_bdi_congested[2]; 476 477void clear_bdi_congested(struct backing_dev_info *bdi, int sync) 478{ 479 enum bdi_state bit; 480 wait_queue_head_t *wqh = &congestion_wqh[sync]; 481 482 bit = sync ? BDI_sync_congested : BDI_async_congested; 483 if (test_and_clear_bit(bit, &bdi->state)) 484 atomic_dec(&nr_bdi_congested[sync]); 485 smp_mb__after_atomic(); 486 if (waitqueue_active(wqh)) 487 wake_up(wqh); 488} 489EXPORT_SYMBOL(clear_bdi_congested); 490 491void set_bdi_congested(struct backing_dev_info *bdi, int sync) 492{ 493 enum bdi_state bit; 494 495 bit = sync ? BDI_sync_congested : BDI_async_congested; 496 if (!test_and_set_bit(bit, &bdi->state)) 497 atomic_inc(&nr_bdi_congested[sync]); 498} 499EXPORT_SYMBOL(set_bdi_congested); 500 501/** 502 * congestion_wait - wait for a backing_dev to become uncongested 503 * @sync: SYNC or ASYNC IO 504 * @timeout: timeout in jiffies 505 * 506 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit 507 * write congestion. If no backing_devs are congested then just wait for the 508 * next write to be completed. 509 */ 510long congestion_wait(int sync, long timeout) 511{ 512 long ret; 513 unsigned long start = jiffies; 514 DEFINE_WAIT(wait); 515 wait_queue_head_t *wqh = &congestion_wqh[sync]; 516 517 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 518 ret = io_schedule_timeout(timeout); 519 finish_wait(wqh, &wait); 520 521 trace_writeback_congestion_wait(jiffies_to_usecs(timeout), 522 jiffies_to_usecs(jiffies - start)); 523 524 return ret; 525} 526EXPORT_SYMBOL(congestion_wait); 527 528/** 529 * wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a zone to complete writes 530 * @zone: A zone to check if it is heavily congested 531 * @sync: SYNC or ASYNC IO 532 * @timeout: timeout in jiffies 533 * 534 * In the event of a congested backing_dev (any backing_dev) and the given 535 * @zone has experienced recent congestion, this waits for up to @timeout 536 * jiffies for either a BDI to exit congestion of the given @sync queue 537 * or a write to complete. 538 * 539 * In the absence of zone congestion, cond_resched() is called to yield 540 * the processor if necessary but otherwise does not sleep. 541 * 542 * The return value is 0 if the sleep is for the full timeout. Otherwise, 543 * it is the number of jiffies that were still remaining when the function 544 * returned. return_value == timeout implies the function did not sleep. 545 */ 546long wait_iff_congested(struct zone *zone, int sync, long timeout) 547{ 548 long ret; 549 unsigned long start = jiffies; 550 DEFINE_WAIT(wait); 551 wait_queue_head_t *wqh = &congestion_wqh[sync]; 552 553 /* 554 * If there is no congestion, or heavy congestion is not being 555 * encountered in the current zone, yield if necessary instead 556 * of sleeping on the congestion queue 557 */ 558 if (atomic_read(&nr_bdi_congested[sync]) == 0 || 559 !test_bit(ZONE_CONGESTED, &zone->flags)) { 560 cond_resched(); 561 562 /* In case we scheduled, work out time remaining */ 563 ret = timeout - (jiffies - start); 564 if (ret < 0) 565 ret = 0; 566 567 goto out; 568 } 569 570 /* Sleep until uncongested or a write happens */ 571 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 572 ret = io_schedule_timeout(timeout); 573 finish_wait(wqh, &wait); 574 575out: 576 trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout), 577 jiffies_to_usecs(jiffies - start)); 578 579 return ret; 580} 581EXPORT_SYMBOL(wait_iff_congested); 582 583int pdflush_proc_obsolete(struct ctl_table *table, int write, 584 void __user *buffer, size_t *lenp, loff_t *ppos) 585{ 586 char kbuf[] = "0\n"; 587 588 if (*ppos || *lenp < sizeof(kbuf)) { 589 *lenp = 0; 590 return 0; 591 } 592 593 if (copy_to_user(buffer, kbuf, sizeof(kbuf))) 594 return -EFAULT; 595 printk_once(KERN_WARNING "%s exported in /proc is scheduled for removal\n", 596 table->procname); 597 598 *lenp = 2; 599 *ppos += *lenp; 600 return 2; 601} 602