1/* 2 * fs/fs-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * 6 * Contains all the functions related to writing back and waiting 7 * upon dirty inodes against superblocks, and writing back dirty 8 * pages against inodes. ie: data writeback. Writeout of the 9 * inode itself is not handled here. 10 * 11 * 10Apr2002 Andrew Morton 12 * Split out of fs/inode.c 13 * Additions for address_space-based writeback 14 */ 15 16#include <linux/kernel.h> 17#include <linux/module.h> 18#include <linux/spinlock.h> 19#include <linux/slab.h> 20#include <linux/sched.h> 21#include <linux/fs.h> 22#include <linux/mm.h> 23#include <linux/kthread.h> 24#include <linux/freezer.h> 25#include <linux/writeback.h> 26#include <linux/blkdev.h> 27#include <linux/backing-dev.h> 28#include <linux/buffer_head.h> 29#include <linux/tracepoint.h> 30#include "internal.h" 31 32/* 33 * Passed into wb_writeback(), essentially a subset of writeback_control 34 */ 35struct wb_writeback_work { 36 long nr_pages; 37 struct super_block *sb; 38 enum writeback_sync_modes sync_mode; 39 unsigned int for_kupdate:1; 40 unsigned int range_cyclic:1; 41 unsigned int for_background:1; 42 43 struct list_head list; /* pending work list */ 44 struct completion *done; /* set if the caller waits */ 45}; 46 47/* 48 * Include the creation of the trace points after defining the 49 * wb_writeback_work structure so that the definition remains local to this 50 * file. 51 */ 52#define CREATE_TRACE_POINTS 53#include <trace/events/writeback.h> 54 55/* 56 * We don't actually have pdflush, but this one is exported though /proc... 57 */ 58int nr_pdflush_threads; 59 60/** 61 * writeback_in_progress - determine whether there is writeback in progress 62 * @bdi: the device's backing_dev_info structure. 63 * 64 * Determine whether there is writeback waiting to be handled against a 65 * backing device. 66 */ 67int writeback_in_progress(struct backing_dev_info *bdi) 68{ 69 return test_bit(BDI_writeback_running, &bdi->state); 70} 71 72static inline struct backing_dev_info *inode_to_bdi(struct inode *inode) 73{ 74 struct super_block *sb = inode->i_sb; 75 76 if (strcmp(sb->s_type->name, "bdev") == 0) 77 return inode->i_mapping->backing_dev_info; 78 79 return sb->s_bdi; 80} 81 82static void bdi_queue_work(struct backing_dev_info *bdi, 83 struct wb_writeback_work *work) 84{ 85 trace_writeback_queue(bdi, work); 86 87 spin_lock_bh(&bdi->wb_lock); 88 list_add_tail(&work->list, &bdi->work_list); 89 if (bdi->wb.task) { 90 wake_up_process(bdi->wb.task); 91 } else { 92 /* 93 * The bdi thread isn't there, wake up the forker thread which 94 * will create and run it. 95 */ 96 trace_writeback_nothread(bdi, work); 97 wake_up_process(default_backing_dev_info.wb.task); 98 } 99 spin_unlock_bh(&bdi->wb_lock); 100} 101 102static void 103__bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages, 104 bool range_cyclic, bool for_background) 105{ 106 struct wb_writeback_work *work; 107 108 /* 109 * This is WB_SYNC_NONE writeback, so if allocation fails just 110 * wakeup the thread for old dirty data writeback 111 */ 112 work = kzalloc(sizeof(*work), GFP_ATOMIC); 113 if (!work) { 114 if (bdi->wb.task) { 115 trace_writeback_nowork(bdi); 116 wake_up_process(bdi->wb.task); 117 } 118 return; 119 } 120 121 work->sync_mode = WB_SYNC_NONE; 122 work->nr_pages = nr_pages; 123 work->range_cyclic = range_cyclic; 124 work->for_background = for_background; 125 126 bdi_queue_work(bdi, work); 127} 128 129/** 130 * bdi_start_writeback - start writeback 131 * @bdi: the backing device to write from 132 * @nr_pages: the number of pages to write 133 * 134 * Description: 135 * This does WB_SYNC_NONE opportunistic writeback. The IO is only 136 * started when this function returns, we make no guarentees on 137 * completion. Caller need not hold sb s_umount semaphore. 138 * 139 */ 140void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages) 141{ 142 __bdi_start_writeback(bdi, nr_pages, true, false); 143} 144 145/** 146 * bdi_start_background_writeback - start background writeback 147 * @bdi: the backing device to write from 148 * 149 * Description: 150 * This does WB_SYNC_NONE background writeback. The IO is only 151 * started when this function returns, we make no guarentees on 152 * completion. Caller need not hold sb s_umount semaphore. 153 */ 154void bdi_start_background_writeback(struct backing_dev_info *bdi) 155{ 156 __bdi_start_writeback(bdi, LONG_MAX, true, true); 157} 158 159/* 160 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the 161 * furthest end of its superblock's dirty-inode list. 162 * 163 * Before stamping the inode's ->dirtied_when, we check to see whether it is 164 * already the most-recently-dirtied inode on the b_dirty list. If that is 165 * the case then the inode must have been redirtied while it was being written 166 * out and we don't reset its dirtied_when. 167 */ 168static void redirty_tail(struct inode *inode) 169{ 170 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb; 171 172 if (!list_empty(&wb->b_dirty)) { 173 struct inode *tail; 174 175 tail = list_entry(wb->b_dirty.next, struct inode, i_list); 176 if (time_before(inode->dirtied_when, tail->dirtied_when)) 177 inode->dirtied_when = jiffies; 178 } 179 list_move(&inode->i_list, &wb->b_dirty); 180} 181 182/* 183 * requeue inode for re-scanning after bdi->b_io list is exhausted. 184 */ 185static void requeue_io(struct inode *inode) 186{ 187 struct bdi_writeback *wb = &inode_to_bdi(inode)->wb; 188 189 list_move(&inode->i_list, &wb->b_more_io); 190} 191 192static void inode_sync_complete(struct inode *inode) 193{ 194 /* 195 * Prevent speculative execution through spin_unlock(&inode_lock); 196 */ 197 smp_mb(); 198 wake_up_bit(&inode->i_state, __I_SYNC); 199} 200 201static bool inode_dirtied_after(struct inode *inode, unsigned long t) 202{ 203 bool ret = time_after(inode->dirtied_when, t); 204#ifndef CONFIG_64BIT 205 /* 206 * For inodes being constantly redirtied, dirtied_when can get stuck. 207 * It _appears_ to be in the future, but is actually in distant past. 208 * This test is necessary to prevent such wrapped-around relative times 209 * from permanently stopping the whole bdi writeback. 210 */ 211 ret = ret && time_before_eq(inode->dirtied_when, jiffies); 212#endif 213 return ret; 214} 215 216/* 217 * Move expired dirty inodes from @delaying_queue to @dispatch_queue. 218 */ 219static void move_expired_inodes(struct list_head *delaying_queue, 220 struct list_head *dispatch_queue, 221 unsigned long *older_than_this) 222{ 223 LIST_HEAD(tmp); 224 struct list_head *pos, *node; 225 struct super_block *sb = NULL; 226 struct inode *inode; 227 int do_sb_sort = 0; 228 229 while (!list_empty(delaying_queue)) { 230 inode = list_entry(delaying_queue->prev, struct inode, i_list); 231 if (older_than_this && 232 inode_dirtied_after(inode, *older_than_this)) 233 break; 234 if (sb && sb != inode->i_sb) 235 do_sb_sort = 1; 236 sb = inode->i_sb; 237 list_move(&inode->i_list, &tmp); 238 } 239 240 /* just one sb in list, splice to dispatch_queue and we're done */ 241 if (!do_sb_sort) { 242 list_splice(&tmp, dispatch_queue); 243 return; 244 } 245 246 /* Move inodes from one superblock together */ 247 while (!list_empty(&tmp)) { 248 inode = list_entry(tmp.prev, struct inode, i_list); 249 sb = inode->i_sb; 250 list_for_each_prev_safe(pos, node, &tmp) { 251 inode = list_entry(pos, struct inode, i_list); 252 if (inode->i_sb == sb) 253 list_move(&inode->i_list, dispatch_queue); 254 } 255 } 256} 257 258/* 259 * Queue all expired dirty inodes for io, eldest first. 260 * Before 261 * newly dirtied b_dirty b_io b_more_io 262 * =============> gf edc BA 263 * After 264 * newly dirtied b_dirty b_io b_more_io 265 * =============> g fBAedc 266 * | 267 * +--> dequeue for IO 268 */ 269static void queue_io(struct bdi_writeback *wb, unsigned long *older_than_this) 270{ 271 list_splice_init(&wb->b_more_io, &wb->b_io); 272 move_expired_inodes(&wb->b_dirty, &wb->b_io, older_than_this); 273} 274 275static int write_inode(struct inode *inode, struct writeback_control *wbc) 276{ 277 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) 278 return inode->i_sb->s_op->write_inode(inode, wbc); 279 return 0; 280} 281 282/* 283 * Wait for writeback on an inode to complete. 284 */ 285static void inode_wait_for_writeback(struct inode *inode) 286{ 287 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC); 288 wait_queue_head_t *wqh; 289 290 wqh = bit_waitqueue(&inode->i_state, __I_SYNC); 291 while (inode->i_state & I_SYNC) { 292 spin_unlock(&inode_lock); 293 __wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE); 294 spin_lock(&inode_lock); 295 } 296} 297 298/* 299 * Write out an inode's dirty pages. Called under inode_lock. Either the 300 * caller has ref on the inode (either via __iget or via syscall against an fd) 301 * or the inode has I_WILL_FREE set (via generic_forget_inode) 302 * 303 * If `wait' is set, wait on the writeout. 304 * 305 * The whole writeout design is quite complex and fragile. We want to avoid 306 * starvation of particular inodes when others are being redirtied, prevent 307 * livelocks, etc. 308 * 309 * Called under inode_lock. 310 */ 311static int 312writeback_single_inode(struct inode *inode, struct writeback_control *wbc) 313{ 314 struct address_space *mapping = inode->i_mapping; 315 unsigned dirty; 316 int ret; 317 318 if (!atomic_read(&inode->i_count)) 319 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING))); 320 else 321 WARN_ON(inode->i_state & I_WILL_FREE); 322 323 if (inode->i_state & I_SYNC) { 324 /* 325 * If this inode is locked for writeback and we are not doing 326 * writeback-for-data-integrity, move it to b_more_io so that 327 * writeback can proceed with the other inodes on s_io. 328 * 329 * We'll have another go at writing back this inode when we 330 * completed a full scan of b_io. 331 */ 332 if (wbc->sync_mode != WB_SYNC_ALL) { 333 requeue_io(inode); 334 return 0; 335 } 336 337 /* 338 * It's a data-integrity sync. We must wait. 339 */ 340 inode_wait_for_writeback(inode); 341 } 342 343 BUG_ON(inode->i_state & I_SYNC); 344 345 /* Set I_SYNC, reset I_DIRTY_PAGES */ 346 inode->i_state |= I_SYNC; 347 inode->i_state &= ~I_DIRTY_PAGES; 348 spin_unlock(&inode_lock); 349 350 ret = do_writepages(mapping, wbc); 351 352 /* 353 * Make sure to wait on the data before writing out the metadata. 354 * This is important for filesystems that modify metadata on data 355 * I/O completion. 356 */ 357 if (wbc->sync_mode == WB_SYNC_ALL) { 358 int err = filemap_fdatawait(mapping); 359 if (ret == 0) 360 ret = err; 361 } 362 363 /* 364 * Some filesystems may redirty the inode during the writeback 365 * due to delalloc, clear dirty metadata flags right before 366 * write_inode() 367 */ 368 spin_lock(&inode_lock); 369 dirty = inode->i_state & I_DIRTY; 370 inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC); 371 spin_unlock(&inode_lock); 372 /* Don't write the inode if only I_DIRTY_PAGES was set */ 373 if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) { 374 int err = write_inode(inode, wbc); 375 if (ret == 0) 376 ret = err; 377 } 378 379 spin_lock(&inode_lock); 380 inode->i_state &= ~I_SYNC; 381 if (!(inode->i_state & I_FREEING)) { 382 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) { 383 /* 384 * We didn't write back all the pages. nfs_writepages() 385 * sometimes bales out without doing anything. 386 */ 387 inode->i_state |= I_DIRTY_PAGES; 388 if (wbc->nr_to_write <= 0) { 389 /* 390 * slice used up: queue for next turn 391 */ 392 requeue_io(inode); 393 } else { 394 /* 395 * Writeback blocked by something other than 396 * congestion. Delay the inode for some time to 397 * avoid spinning on the CPU (100% iowait) 398 * retrying writeback of the dirty page/inode 399 * that cannot be performed immediately. 400 */ 401 redirty_tail(inode); 402 } 403 } else if (inode->i_state & I_DIRTY) { 404 /* 405 * Filesystems can dirty the inode during writeback 406 * operations, such as delayed allocation during 407 * submission or metadata updates after data IO 408 * completion. 409 */ 410 redirty_tail(inode); 411 } else if (atomic_read(&inode->i_count)) { 412 /* 413 * The inode is clean, inuse 414 */ 415 list_move(&inode->i_list, &inode_in_use); 416 } else { 417 /* 418 * The inode is clean, unused 419 */ 420 list_move(&inode->i_list, &inode_unused); 421 } 422 } 423 inode_sync_complete(inode); 424 return ret; 425} 426 427/* 428 * For background writeback the caller does not have the sb pinned 429 * before calling writeback. So make sure that we do pin it, so it doesn't 430 * go away while we are writing inodes from it. 431 */ 432static bool pin_sb_for_writeback(struct super_block *sb) 433{ 434 spin_lock(&sb_lock); 435 if (list_empty(&sb->s_instances)) { 436 spin_unlock(&sb_lock); 437 return false; 438 } 439 440 sb->s_count++; 441 spin_unlock(&sb_lock); 442 443 if (down_read_trylock(&sb->s_umount)) { 444 if (sb->s_root) 445 return true; 446 up_read(&sb->s_umount); 447 } 448 449 put_super(sb); 450 return false; 451} 452 453/* 454 * Write a portion of b_io inodes which belong to @sb. 455 * 456 * If @only_this_sb is true, then find and write all such 457 * inodes. Otherwise write only ones which go sequentially 458 * in reverse order. 459 * 460 * Return 1, if the caller writeback routine should be 461 * interrupted. Otherwise return 0. 462 */ 463static int writeback_sb_inodes(struct super_block *sb, struct bdi_writeback *wb, 464 struct writeback_control *wbc, bool only_this_sb) 465{ 466 while (!list_empty(&wb->b_io)) { 467 long pages_skipped; 468 struct inode *inode = list_entry(wb->b_io.prev, 469 struct inode, i_list); 470 471 if (inode->i_sb != sb) { 472 if (only_this_sb) { 473 /* 474 * We only want to write back data for this 475 * superblock, move all inodes not belonging 476 * to it back onto the dirty list. 477 */ 478 redirty_tail(inode); 479 continue; 480 } 481 482 /* 483 * The inode belongs to a different superblock. 484 * Bounce back to the caller to unpin this and 485 * pin the next superblock. 486 */ 487 return 0; 488 } 489 490 if (inode->i_state & (I_NEW | I_WILL_FREE)) { 491 requeue_io(inode); 492 continue; 493 } 494 /* 495 * Was this inode dirtied after sync_sb_inodes was called? 496 * This keeps sync from extra jobs and livelock. 497 */ 498 if (inode_dirtied_after(inode, wbc->wb_start)) 499 return 1; 500 501 BUG_ON(inode->i_state & I_FREEING); 502 __iget(inode); 503 pages_skipped = wbc->pages_skipped; 504 writeback_single_inode(inode, wbc); 505 if (wbc->pages_skipped != pages_skipped) { 506 /* 507 * writeback is not making progress due to locked 508 * buffers. Skip this inode for now. 509 */ 510 redirty_tail(inode); 511 } 512 spin_unlock(&inode_lock); 513 iput(inode); 514 cond_resched(); 515 spin_lock(&inode_lock); 516 if (wbc->nr_to_write <= 0) { 517 wbc->more_io = 1; 518 return 1; 519 } 520 if (!list_empty(&wb->b_more_io)) 521 wbc->more_io = 1; 522 } 523 /* b_io is empty */ 524 return 1; 525} 526 527void writeback_inodes_wb(struct bdi_writeback *wb, 528 struct writeback_control *wbc) 529{ 530 int ret = 0; 531 532 if (!wbc->wb_start) 533 wbc->wb_start = jiffies; /* livelock avoidance */ 534 spin_lock(&inode_lock); 535 if (!wbc->for_kupdate || list_empty(&wb->b_io)) 536 queue_io(wb, wbc->older_than_this); 537 538 while (!list_empty(&wb->b_io)) { 539 struct inode *inode = list_entry(wb->b_io.prev, 540 struct inode, i_list); 541 struct super_block *sb = inode->i_sb; 542 543 if (!pin_sb_for_writeback(sb)) { 544 requeue_io(inode); 545 continue; 546 } 547 ret = writeback_sb_inodes(sb, wb, wbc, false); 548 drop_super(sb); 549 550 if (ret) 551 break; 552 } 553 spin_unlock(&inode_lock); 554 /* Leave any unwritten inodes on b_io */ 555} 556 557static void __writeback_inodes_sb(struct super_block *sb, 558 struct bdi_writeback *wb, struct writeback_control *wbc) 559{ 560 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 561 562 spin_lock(&inode_lock); 563 if (!wbc->for_kupdate || list_empty(&wb->b_io)) 564 queue_io(wb, wbc->older_than_this); 565 writeback_sb_inodes(sb, wb, wbc, true); 566 spin_unlock(&inode_lock); 567} 568 569/* 570 * The maximum number of pages to writeout in a single bdi flush/kupdate 571 * operation. We do this so we don't hold I_SYNC against an inode for 572 * enormous amounts of time, which would block a userspace task which has 573 * been forced to throttle against that inode. Also, the code reevaluates 574 * the dirty each time it has written this many pages. 575 */ 576#define MAX_WRITEBACK_PAGES 1024 577 578static inline bool over_bground_thresh(void) 579{ 580 unsigned long background_thresh, dirty_thresh; 581 582 global_dirty_limits(&background_thresh, &dirty_thresh); 583 584 return (global_page_state(NR_FILE_DIRTY) + 585 global_page_state(NR_UNSTABLE_NFS) >= background_thresh); 586} 587 588/* 589 * Explicit flushing or periodic writeback of "old" data. 590 * 591 * Define "old": the first time one of an inode's pages is dirtied, we mark the 592 * dirtying-time in the inode's address_space. So this periodic writeback code 593 * just walks the superblock inode list, writing back any inodes which are 594 * older than a specific point in time. 595 * 596 * Try to run once per dirty_writeback_interval. But if a writeback event 597 * takes longer than a dirty_writeback_interval interval, then leave a 598 * one-second gap. 599 * 600 * older_than_this takes precedence over nr_to_write. So we'll only write back 601 * all dirty pages if they are all attached to "old" mappings. 602 */ 603static long wb_writeback(struct bdi_writeback *wb, 604 struct wb_writeback_work *work) 605{ 606 struct writeback_control wbc = { 607 .sync_mode = work->sync_mode, 608 .older_than_this = NULL, 609 .for_kupdate = work->for_kupdate, 610 .for_background = work->for_background, 611 .range_cyclic = work->range_cyclic, 612 }; 613 unsigned long oldest_jif; 614 long wrote = 0; 615 struct inode *inode; 616 617 if (wbc.for_kupdate) { 618 wbc.older_than_this = &oldest_jif; 619 oldest_jif = jiffies - 620 msecs_to_jiffies(dirty_expire_interval * 10); 621 } 622 if (!wbc.range_cyclic) { 623 wbc.range_start = 0; 624 wbc.range_end = LLONG_MAX; 625 } 626 627 wbc.wb_start = jiffies; /* livelock avoidance */ 628 for (;;) { 629 /* 630 * Stop writeback when nr_pages has been consumed 631 */ 632 if (work->nr_pages <= 0) 633 break; 634 635 /* 636 * For background writeout, stop when we are below the 637 * background dirty threshold 638 */ 639 if (work->for_background && !over_bground_thresh()) 640 break; 641 642 wbc.more_io = 0; 643 wbc.nr_to_write = MAX_WRITEBACK_PAGES; 644 wbc.pages_skipped = 0; 645 646 trace_wbc_writeback_start(&wbc, wb->bdi); 647 if (work->sb) 648 __writeback_inodes_sb(work->sb, wb, &wbc); 649 else 650 writeback_inodes_wb(wb, &wbc); 651 trace_wbc_writeback_written(&wbc, wb->bdi); 652 653 work->nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; 654 wrote += MAX_WRITEBACK_PAGES - wbc.nr_to_write; 655 656 /* 657 * If we consumed everything, see if we have more 658 */ 659 if (wbc.nr_to_write <= 0) 660 continue; 661 /* 662 * Didn't write everything and we don't have more IO, bail 663 */ 664 if (!wbc.more_io) 665 break; 666 /* 667 * Did we write something? Try for more 668 */ 669 if (wbc.nr_to_write < MAX_WRITEBACK_PAGES) 670 continue; 671 /* 672 * Nothing written. Wait for some inode to 673 * become available for writeback. Otherwise 674 * we'll just busyloop. 675 */ 676 spin_lock(&inode_lock); 677 if (!list_empty(&wb->b_more_io)) { 678 inode = list_entry(wb->b_more_io.prev, 679 struct inode, i_list); 680 trace_wbc_writeback_wait(&wbc, wb->bdi); 681 inode_wait_for_writeback(inode); 682 } 683 spin_unlock(&inode_lock); 684 } 685 686 return wrote; 687} 688 689/* 690 * Return the next wb_writeback_work struct that hasn't been processed yet. 691 */ 692static struct wb_writeback_work * 693get_next_work_item(struct backing_dev_info *bdi) 694{ 695 struct wb_writeback_work *work = NULL; 696 697 spin_lock_bh(&bdi->wb_lock); 698 if (!list_empty(&bdi->work_list)) { 699 work = list_entry(bdi->work_list.next, 700 struct wb_writeback_work, list); 701 list_del_init(&work->list); 702 } 703 spin_unlock_bh(&bdi->wb_lock); 704 return work; 705} 706 707static long wb_check_old_data_flush(struct bdi_writeback *wb) 708{ 709 unsigned long expired; 710 long nr_pages; 711 712 /* 713 * When set to zero, disable periodic writeback 714 */ 715 if (!dirty_writeback_interval) 716 return 0; 717 718 expired = wb->last_old_flush + 719 msecs_to_jiffies(dirty_writeback_interval * 10); 720 if (time_before(jiffies, expired)) 721 return 0; 722 723 wb->last_old_flush = jiffies; 724 nr_pages = global_page_state(NR_FILE_DIRTY) + 725 global_page_state(NR_UNSTABLE_NFS) + 726 (inodes_stat.nr_inodes - inodes_stat.nr_unused); 727 728 if (nr_pages) { 729 struct wb_writeback_work work = { 730 .nr_pages = nr_pages, 731 .sync_mode = WB_SYNC_NONE, 732 .for_kupdate = 1, 733 .range_cyclic = 1, 734 }; 735 736 return wb_writeback(wb, &work); 737 } 738 739 return 0; 740} 741 742/* 743 * Retrieve work items and do the writeback they describe 744 */ 745long wb_do_writeback(struct bdi_writeback *wb, int force_wait) 746{ 747 struct backing_dev_info *bdi = wb->bdi; 748 struct wb_writeback_work *work; 749 long wrote = 0; 750 751 set_bit(BDI_writeback_running, &wb->bdi->state); 752 while ((work = get_next_work_item(bdi)) != NULL) { 753 /* 754 * Override sync mode, in case we must wait for completion 755 * because this thread is exiting now. 756 */ 757 if (force_wait) 758 work->sync_mode = WB_SYNC_ALL; 759 760 trace_writeback_exec(bdi, work); 761 762 wrote += wb_writeback(wb, work); 763 764 /* 765 * Notify the caller of completion if this is a synchronous 766 * work item, otherwise just free it. 767 */ 768 if (work->done) 769 complete(work->done); 770 else 771 kfree(work); 772 } 773 774 /* 775 * Check for periodic writeback, kupdated() style 776 */ 777 wrote += wb_check_old_data_flush(wb); 778 clear_bit(BDI_writeback_running, &wb->bdi->state); 779 780 return wrote; 781} 782 783/* 784 * Handle writeback of dirty data for the device backed by this bdi. Also 785 * wakes up periodically and does kupdated style flushing. 786 */ 787int bdi_writeback_thread(void *data) 788{ 789 struct bdi_writeback *wb = data; 790 struct backing_dev_info *bdi = wb->bdi; 791 long pages_written; 792 793 current->flags |= PF_FLUSHER | PF_SWAPWRITE; 794 set_freezable(); 795 wb->last_active = jiffies; 796 797 /* 798 * Our parent may run at a different priority, just set us to normal 799 */ 800 set_user_nice(current, 0); 801 802 trace_writeback_thread_start(bdi); 803 804 while (!kthread_should_stop()) { 805 /* 806 * Remove own delayed wake-up timer, since we are already awake 807 * and we'll take care of the preriodic write-back. 808 */ 809 del_timer(&wb->wakeup_timer); 810 811 pages_written = wb_do_writeback(wb, 0); 812 813 trace_writeback_pages_written(pages_written); 814 815 if (pages_written) 816 wb->last_active = jiffies; 817 818 set_current_state(TASK_INTERRUPTIBLE); 819 if (!list_empty(&bdi->work_list) || kthread_should_stop()) { 820 __set_current_state(TASK_RUNNING); 821 continue; 822 } 823 824 if (wb_has_dirty_io(wb) && dirty_writeback_interval) 825 schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10)); 826 else { 827 /* 828 * We have nothing to do, so can go sleep without any 829 * timeout and save power. When a work is queued or 830 * something is made dirty - we will be woken up. 831 */ 832 schedule(); 833 } 834 835 try_to_freeze(); 836 } 837 838 /* Flush any work that raced with us exiting */ 839 if (!list_empty(&bdi->work_list)) 840 wb_do_writeback(wb, 1); 841 842 trace_writeback_thread_stop(bdi); 843 return 0; 844} 845 846 847/* 848 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back 849 * the whole world. 850 */ 851void wakeup_flusher_threads(long nr_pages) 852{ 853 struct backing_dev_info *bdi; 854 855 if (!nr_pages) { 856 nr_pages = global_page_state(NR_FILE_DIRTY) + 857 global_page_state(NR_UNSTABLE_NFS); 858 } 859 860 rcu_read_lock(); 861 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) { 862 if (!bdi_has_dirty_io(bdi)) 863 continue; 864 __bdi_start_writeback(bdi, nr_pages, false, false); 865 } 866 rcu_read_unlock(); 867} 868 869static noinline void block_dump___mark_inode_dirty(struct inode *inode) 870{ 871 if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) { 872 struct dentry *dentry; 873 const char *name = "?"; 874 875 dentry = d_find_alias(inode); 876 if (dentry) { 877 spin_lock(&dentry->d_lock); 878 name = (const char *) dentry->d_name.name; 879 } 880 printk(KERN_DEBUG 881 "%s(%d): dirtied inode %lu (%s) on %s\n", 882 current->comm, task_pid_nr(current), inode->i_ino, 883 name, inode->i_sb->s_id); 884 if (dentry) { 885 spin_unlock(&dentry->d_lock); 886 dput(dentry); 887 } 888 } 889} 890 891/** 892 * __mark_inode_dirty - internal function 893 * @inode: inode to mark 894 * @flags: what kind of dirty (i.e. I_DIRTY_SYNC) 895 * Mark an inode as dirty. Callers should use mark_inode_dirty or 896 * mark_inode_dirty_sync. 897 * 898 * Put the inode on the super block's dirty list. 899 * 900 * CAREFUL! We mark it dirty unconditionally, but move it onto the 901 * dirty list only if it is hashed or if it refers to a blockdev. 902 * If it was not hashed, it will never be added to the dirty list 903 * even if it is later hashed, as it will have been marked dirty already. 904 * 905 * In short, make sure you hash any inodes _before_ you start marking 906 * them dirty. 907 * 908 * This function *must* be atomic for the I_DIRTY_PAGES case - 909 * set_page_dirty() is called under spinlock in several places. 910 * 911 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of 912 * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of 913 * the kernel-internal blockdev inode represents the dirtying time of the 914 * blockdev's pages. This is why for I_DIRTY_PAGES we always use 915 * page->mapping->host, so the page-dirtying time is recorded in the internal 916 * blockdev inode. 917 */ 918void __mark_inode_dirty(struct inode *inode, int flags) 919{ 920 struct super_block *sb = inode->i_sb; 921 struct backing_dev_info *bdi = NULL; 922 bool wakeup_bdi = false; 923 924 /* 925 * Don't do this for I_DIRTY_PAGES - that doesn't actually 926 * dirty the inode itself 927 */ 928 if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) { 929 if (sb->s_op->dirty_inode) 930 sb->s_op->dirty_inode(inode); 931 } 932 933 /* 934 * make sure that changes are seen by all cpus before we test i_state 935 * -- mikulas 936 */ 937 smp_mb(); 938 939 /* avoid the locking if we can */ 940 if ((inode->i_state & flags) == flags) 941 return; 942 943 if (unlikely(block_dump)) 944 block_dump___mark_inode_dirty(inode); 945 946 spin_lock(&inode_lock); 947 if ((inode->i_state & flags) != flags) { 948 const int was_dirty = inode->i_state & I_DIRTY; 949 950 inode->i_state |= flags; 951 952 /* 953 * If the inode is being synced, just update its dirty state. 954 * The unlocker will place the inode on the appropriate 955 * superblock list, based upon its state. 956 */ 957 if (inode->i_state & I_SYNC) 958 goto out; 959 960 /* 961 * Only add valid (hashed) inodes to the superblock's 962 * dirty list. Add blockdev inodes as well. 963 */ 964 if (!S_ISBLK(inode->i_mode)) { 965 if (hlist_unhashed(&inode->i_hash)) 966 goto out; 967 } 968 if (inode->i_state & I_FREEING) 969 goto out; 970 971 /* 972 * If the inode was already on b_dirty/b_io/b_more_io, don't 973 * reposition it (that would break b_dirty time-ordering). 974 */ 975 if (!was_dirty) { 976 bdi = inode_to_bdi(inode); 977 978 if (bdi_cap_writeback_dirty(bdi)) { 979 WARN(!test_bit(BDI_registered, &bdi->state), 980 "bdi-%s not registered\n", bdi->name); 981 982 /* 983 * If this is the first dirty inode for this 984 * bdi, we have to wake-up the corresponding 985 * bdi thread to make sure background 986 * write-back happens later. 987 */ 988 if (!wb_has_dirty_io(&bdi->wb)) 989 wakeup_bdi = true; 990 } 991 992 inode->dirtied_when = jiffies; 993 list_move(&inode->i_list, &bdi->wb.b_dirty); 994 } 995 } 996out: 997 spin_unlock(&inode_lock); 998 999 if (wakeup_bdi) 1000 bdi_wakeup_thread_delayed(bdi); 1001} 1002EXPORT_SYMBOL(__mark_inode_dirty); 1003 1004/* 1005 * Write out a superblock's list of dirty inodes. A wait will be performed 1006 * upon no inodes, all inodes or the final one, depending upon sync_mode. 1007 * 1008 * If older_than_this is non-NULL, then only write out inodes which 1009 * had their first dirtying at a time earlier than *older_than_this. 1010 * 1011 * If `bdi' is non-zero then we're being asked to writeback a specific queue. 1012 * This function assumes that the blockdev superblock's inodes are backed by 1013 * a variety of queues, so all inodes are searched. For other superblocks, 1014 * assume that all inodes are backed by the same queue. 1015 * 1016 * The inodes to be written are parked on bdi->b_io. They are moved back onto 1017 * bdi->b_dirty as they are selected for writing. This way, none can be missed 1018 * on the writer throttling path, and we get decent balancing between many 1019 * throttled threads: we don't want them all piling up on inode_sync_wait. 1020 */ 1021static void wait_sb_inodes(struct super_block *sb) 1022{ 1023 struct inode *inode, *old_inode = NULL; 1024 1025 /* 1026 * We need to be protected against the filesystem going from 1027 * r/o to r/w or vice versa. 1028 */ 1029 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 1030 1031 spin_lock(&inode_lock); 1032 1033 /* 1034 * Data integrity sync. Must wait for all pages under writeback, 1035 * because there may have been pages dirtied before our sync 1036 * call, but which had writeout started before we write it out. 1037 * In which case, the inode may not be on the dirty list, but 1038 * we still have to wait for that writeout. 1039 */ 1040 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { 1041 struct address_space *mapping; 1042 1043 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) 1044 continue; 1045 mapping = inode->i_mapping; 1046 if (mapping->nrpages == 0) 1047 continue; 1048 __iget(inode); 1049 spin_unlock(&inode_lock); 1050 /* 1051 * We hold a reference to 'inode' so it couldn't have 1052 * been removed from s_inodes list while we dropped the 1053 * inode_lock. We cannot iput the inode now as we can 1054 * be holding the last reference and we cannot iput it 1055 * under inode_lock. So we keep the reference and iput 1056 * it later. 1057 */ 1058 iput(old_inode); 1059 old_inode = inode; 1060 1061 filemap_fdatawait(mapping); 1062 1063 cond_resched(); 1064 1065 spin_lock(&inode_lock); 1066 } 1067 spin_unlock(&inode_lock); 1068 iput(old_inode); 1069} 1070 1071/** 1072 * writeback_inodes_sb - writeback dirty inodes from given super_block 1073 * @sb: the superblock 1074 * 1075 * Start writeback on some inodes on this super_block. No guarantees are made 1076 * on how many (if any) will be written, and this function does not wait 1077 * for IO completion of submitted IO. The number of pages submitted is 1078 * returned. 1079 */ 1080void writeback_inodes_sb(struct super_block *sb) 1081{ 1082 unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY); 1083 unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS); 1084 DECLARE_COMPLETION_ONSTACK(done); 1085 struct wb_writeback_work work = { 1086 .sb = sb, 1087 .sync_mode = WB_SYNC_NONE, 1088 .done = &done, 1089 }; 1090 1091 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 1092 1093 work.nr_pages = nr_dirty + nr_unstable + 1094 (inodes_stat.nr_inodes - inodes_stat.nr_unused); 1095 1096 bdi_queue_work(sb->s_bdi, &work); 1097 wait_for_completion(&done); 1098} 1099EXPORT_SYMBOL(writeback_inodes_sb); 1100 1101/** 1102 * writeback_inodes_sb_if_idle - start writeback if none underway 1103 * @sb: the superblock 1104 * 1105 * Invoke writeback_inodes_sb if no writeback is currently underway. 1106 * Returns 1 if writeback was started, 0 if not. 1107 */ 1108int writeback_inodes_sb_if_idle(struct super_block *sb) 1109{ 1110 if (!writeback_in_progress(sb->s_bdi)) { 1111 down_read(&sb->s_umount); 1112 writeback_inodes_sb(sb); 1113 up_read(&sb->s_umount); 1114 return 1; 1115 } else 1116 return 0; 1117} 1118EXPORT_SYMBOL(writeback_inodes_sb_if_idle); 1119 1120/** 1121 * sync_inodes_sb - sync sb inode pages 1122 * @sb: the superblock 1123 * 1124 * This function writes and waits on any dirty inode belonging to this 1125 * super_block. The number of pages synced is returned. 1126 */ 1127void sync_inodes_sb(struct super_block *sb) 1128{ 1129 DECLARE_COMPLETION_ONSTACK(done); 1130 struct wb_writeback_work work = { 1131 .sb = sb, 1132 .sync_mode = WB_SYNC_ALL, 1133 .nr_pages = LONG_MAX, 1134 .range_cyclic = 0, 1135 .done = &done, 1136 }; 1137 1138 WARN_ON(!rwsem_is_locked(&sb->s_umount)); 1139 1140 bdi_queue_work(sb->s_bdi, &work); 1141 wait_for_completion(&done); 1142 1143 wait_sb_inodes(sb); 1144} 1145EXPORT_SYMBOL(sync_inodes_sb); 1146 1147/** 1148 * write_inode_now - write an inode to disk 1149 * @inode: inode to write to disk 1150 * @sync: whether the write should be synchronous or not 1151 * 1152 * This function commits an inode to disk immediately if it is dirty. This is 1153 * primarily needed by knfsd. 1154 * 1155 * The caller must either have a ref on the inode or must have set I_WILL_FREE. 1156 */ 1157int write_inode_now(struct inode *inode, int sync) 1158{ 1159 int ret; 1160 struct writeback_control wbc = { 1161 .nr_to_write = LONG_MAX, 1162 .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE, 1163 .range_start = 0, 1164 .range_end = LLONG_MAX, 1165 }; 1166 1167 if (!mapping_cap_writeback_dirty(inode->i_mapping)) 1168 wbc.nr_to_write = 0; 1169 1170 might_sleep(); 1171 spin_lock(&inode_lock); 1172 ret = writeback_single_inode(inode, &wbc); 1173 spin_unlock(&inode_lock); 1174 if (sync) 1175 inode_sync_wait(inode); 1176 return ret; 1177} 1178EXPORT_SYMBOL(write_inode_now); 1179 1180/** 1181 * sync_inode - write an inode and its pages to disk. 1182 * @inode: the inode to sync 1183 * @wbc: controls the writeback mode 1184 * 1185 * sync_inode() will write an inode and its pages to disk. It will also 1186 * correctly update the inode on its superblock's dirty inode lists and will 1187 * update inode->i_state. 1188 * 1189 * The caller must have a ref on the inode. 1190 */ 1191int sync_inode(struct inode *inode, struct writeback_control *wbc) 1192{ 1193 int ret; 1194 1195 spin_lock(&inode_lock); 1196 ret = writeback_single_inode(inode, wbc); 1197 spin_unlock(&inode_lock); 1198 return ret; 1199} 1200EXPORT_SYMBOL(sync_inode); 1201