Branch data Line data Source code
1 : : /*
2 : : * fs/fs-writeback.c
3 : : *
4 : : * Copyright (C) 2002, Linus Torvalds.
5 : : *
6 : : * Contains all the functions related to writing back and waiting
7 : : * upon dirty inodes against superblocks, and writing back dirty
8 : : * pages against inodes. ie: data writeback. Writeout of the
9 : : * inode itself is not handled here.
10 : : *
11 : : * 10Apr2002 Andrew Morton
12 : : * Split out of fs/inode.c
13 : : * Additions for address_space-based writeback
14 : : */
15 : :
16 : : #include <linux/kernel.h>
17 : : #include <linux/export.h>
18 : : #include <linux/spinlock.h>
19 : : #include <linux/slab.h>
20 : : #include <linux/sched.h>
21 : : #include <linux/fs.h>
22 : : #include <linux/mm.h>
23 : : #include <linux/pagemap.h>
24 : : #include <linux/kthread.h>
25 : : #include <linux/writeback.h>
26 : : #include <linux/blkdev.h>
27 : : #include <linux/backing-dev.h>
28 : : #include <linux/tracepoint.h>
29 : : #include <linux/device.h>
30 : : #include "internal.h"
31 : :
32 : : /*
33 : : * 4MB minimal write chunk size
34 : : */
35 : : #define MIN_WRITEBACK_PAGES (4096UL >> (PAGE_CACHE_SHIFT - 10))
36 : :
37 : : /*
38 : : * Passed into wb_writeback(), essentially a subset of writeback_control
39 : : */
40 : : struct wb_writeback_work {
41 : : long nr_pages;
42 : : struct super_block *sb;
43 : : unsigned long *older_than_this;
44 : : enum writeback_sync_modes sync_mode;
45 : : unsigned int tagged_writepages:1;
46 : : unsigned int for_kupdate:1;
47 : : unsigned int range_cyclic:1;
48 : : unsigned int for_background:1;
49 : : unsigned int for_sync:1; /* sync(2) WB_SYNC_ALL writeback */
50 : : enum wb_reason reason; /* why was writeback initiated? */
51 : :
52 : : struct list_head list; /* pending work list */
53 : : struct completion *done; /* set if the caller waits */
54 : : };
55 : :
56 : : /**
57 : : * writeback_in_progress - determine whether there is writeback in progress
58 : : * @bdi: the device's backing_dev_info structure.
59 : : *
60 : : * Determine whether there is writeback waiting to be handled against a
61 : : * backing device.
62 : : */
63 : 0 : int writeback_in_progress(struct backing_dev_info *bdi)
64 : : {
65 : 26550 : return test_bit(BDI_writeback_running, &bdi->state);
66 : : }
67 : : EXPORT_SYMBOL(writeback_in_progress);
68 : :
69 : : static inline struct backing_dev_info *inode_to_bdi(struct inode *inode)
70 : : {
71 : : struct super_block *sb = inode->i_sb;
72 : :
73 [ - + ][ - + ]: 133066 : if (sb_is_blkdev_sb(sb))
[ + + ][ - + ]
[ # # ][ # #
# # # # ]
74 : 1397 : return inode->i_mapping->backing_dev_info;
75 : :
76 : 131627 : return sb->s_bdi;
77 : : }
78 : :
79 : : static inline struct inode *wb_inode(struct list_head *head)
80 : : {
81 : : return list_entry(head, struct inode, i_wb_list);
82 : : }
83 : :
84 : : /*
85 : : * Include the creation of the trace points after defining the
86 : : * wb_writeback_work structure and inline functions so that the definition
87 : : * remains local to this file.
88 : : */
89 : : #define CREATE_TRACE_POINTS
90 : : #include <trace/events/writeback.h>
91 : :
92 : 0 : static void bdi_queue_work(struct backing_dev_info *bdi,
93 : : struct wb_writeback_work *work)
94 : : {
95 : : trace_writeback_queue(bdi, work);
96 : :
97 : : spin_lock_bh(&bdi->wb_lock);
98 : 5976 : list_add_tail(&work->list, &bdi->work_list);
99 : : spin_unlock_bh(&bdi->wb_lock);
100 : :
101 : 5976 : mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
102 : 5976 : }
103 : :
104 : : static void
105 : 0 : __bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
106 : : bool range_cyclic, enum wb_reason reason)
107 : : {
108 : : struct wb_writeback_work *work;
109 : :
110 : : /*
111 : : * This is WB_SYNC_NONE writeback, so if allocation fails just
112 : : * wakeup the thread for old dirty data writeback
113 : : */
114 : : work = kzalloc(sizeof(*work), GFP_ATOMIC);
115 [ - + ]: 2952 : if (!work) {
116 : : trace_writeback_nowork(bdi);
117 : 0 : mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
118 : 2954 : return;
119 : : }
120 : :
121 : 2952 : work->sync_mode = WB_SYNC_NONE;
122 : 2952 : work->nr_pages = nr_pages;
123 : 2952 : work->range_cyclic = range_cyclic;
124 : 2952 : work->reason = reason;
125 : :
126 : 2952 : bdi_queue_work(bdi, work);
127 : : }
128 : :
129 : : /**
130 : : * bdi_start_writeback - start writeback
131 : : * @bdi: the backing device to write from
132 : : * @nr_pages: the number of pages to write
133 : : * @reason: reason why some writeback work was initiated
134 : : *
135 : : * Description:
136 : : * This does WB_SYNC_NONE opportunistic writeback. The IO is only
137 : : * started when this function returns, we make no guarantees on
138 : : * completion. Caller need not hold sb s_umount semaphore.
139 : : *
140 : : */
141 : 0 : void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
142 : : enum wb_reason reason)
143 : : {
144 : 0 : __bdi_start_writeback(bdi, nr_pages, true, reason);
145 : 0 : }
146 : :
147 : : /**
148 : : * bdi_start_background_writeback - start background writeback
149 : : * @bdi: the backing device to write from
150 : : *
151 : : * Description:
152 : : * This makes sure WB_SYNC_NONE background writeback happens. When
153 : : * this function returns, it is only guaranteed that for given BDI
154 : : * some IO is happening if we are over background dirty threshold.
155 : : * Caller need not hold sb s_umount semaphore.
156 : : */
157 : 0 : void bdi_start_background_writeback(struct backing_dev_info *bdi)
158 : : {
159 : : /*
160 : : * We just wake up the flusher thread. It will perform background
161 : : * writeback as soon as there is no other work to do.
162 : : */
163 : : trace_writeback_wake_background(bdi);
164 : 0 : mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
165 : 23 : }
166 : :
167 : : /*
168 : : * Remove the inode from the writeback list it is on.
169 : : */
170 : 0 : void inode_wb_list_del(struct inode *inode)
171 : : {
172 : : struct backing_dev_info *bdi = inode_to_bdi(inode);
173 : :
174 : : spin_lock(&bdi->wb.list_lock);
175 : 58525 : list_del_init(&inode->i_wb_list);
176 : : spin_unlock(&bdi->wb.list_lock);
177 : 58525 : }
178 : :
179 : : /*
180 : : * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
181 : : * furthest end of its superblock's dirty-inode list.
182 : : *
183 : : * Before stamping the inode's ->dirtied_when, we check to see whether it is
184 : : * already the most-recently-dirtied inode on the b_dirty list. If that is
185 : : * the case then the inode must have been redirtied while it was being written
186 : : * out and we don't reset its dirtied_when.
187 : : */
188 : 0 : static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
189 : : {
190 [ - + ]: 1265 : assert_spin_locked(&wb->list_lock);
191 [ + + ]: 1265 : if (!list_empty(&wb->b_dirty)) {
192 : : struct inode *tail;
193 : :
194 : : tail = wb_inode(wb->b_dirty.next);
195 [ + + ]: 289 : if (time_before(inode->dirtied_when, tail->dirtied_when))
196 : 142 : inode->dirtied_when = jiffies;
197 : : }
198 : 0 : list_move(&inode->i_wb_list, &wb->b_dirty);
199 : 0 : }
200 : :
201 : : /*
202 : : * requeue inode for re-scanning after bdi->b_io list is exhausted.
203 : : */
204 : 0 : static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
205 : : {
206 [ - + ]: 315 : assert_spin_locked(&wb->list_lock);
207 : 315 : list_move(&inode->i_wb_list, &wb->b_more_io);
208 : 315 : }
209 : :
210 : 0 : static void inode_sync_complete(struct inode *inode)
211 : : {
212 : 17446 : inode->i_state &= ~I_SYNC;
213 : : /* If inode is clean an unused, put it into LRU now... */
214 : 17446 : inode_add_lru(inode);
215 : : /* Waiters must see I_SYNC cleared before being woken up */
216 : 17446 : smp_mb();
217 : 17446 : wake_up_bit(&inode->i_state, __I_SYNC);
218 : 17446 : }
219 : :
220 : : static bool inode_dirtied_after(struct inode *inode, unsigned long t)
221 : : {
222 [ + + ]: 19482 : bool ret = time_after(inode->dirtied_when, t);
223 : : #ifndef CONFIG_64BIT
224 : : /*
225 : : * For inodes being constantly redirtied, dirtied_when can get stuck.
226 : : * It _appears_ to be in the future, but is actually in distant past.
227 : : * This test is necessary to prevent such wrapped-around relative times
228 : : * from permanently stopping the whole bdi writeback.
229 : : */
230 [ + + ][ - + ]: 19482 : ret = ret && time_before_eq(inode->dirtied_when, jiffies);
231 : : #endif
232 : : return ret;
233 : : }
234 : :
235 : : /*
236 : : * Move expired (dirtied before work->older_than_this) dirty inodes from
237 : : * @delaying_queue to @dispatch_queue.
238 : : */
239 : 10947 : static int move_expired_inodes(struct list_head *delaying_queue,
240 : : struct list_head *dispatch_queue,
241 : : struct wb_writeback_work *work)
242 : : {
243 : 10947 : LIST_HEAD(tmp);
244 : : struct list_head *pos, *node;
245 : : struct super_block *sb = NULL;
246 : 19482 : struct inode *inode;
247 : : int do_sb_sort = 0;
248 : : int moved = 0;
249 : :
250 [ + + ]: 28217 : while (!list_empty(delaying_queue)) {
251 : 19482 : inode = wb_inode(delaying_queue->prev);
252 [ + - ][ + + ]: 38964 : if (work->older_than_this &&
253 : 19482 : inode_dirtied_after(inode, *work->older_than_this))
254 : : break;
255 : 17270 : list_move(&inode->i_wb_list, &tmp);
256 : 17270 : moved++;
257 [ + + ]: 17270 : if (sb_is_blkdev_sb(inode->i_sb))
258 : 1512 : continue;
259 [ + + ][ - + ]: 15758 : if (sb && sb != inode->i_sb)
260 : : do_sb_sort = 1;
261 : 17270 : sb = inode->i_sb;
262 : : }
263 : :
264 : : /* just one sb in list, splice to dispatch_queue and we're done */
265 [ + - ]: 10947 : if (!do_sb_sort) {
266 : : list_splice(&tmp, dispatch_queue);
267 : : goto out;
268 : : }
269 : :
270 : : /* Move inodes from one superblock together */
271 [ # # ]: 0 : while (!list_empty(&tmp)) {
272 : 0 : sb = wb_inode(tmp.prev)->i_sb;
273 [ # # ]: 0 : list_for_each_prev_safe(pos, node, &tmp) {
274 : : inode = wb_inode(pos);
275 [ # # ]: 0 : if (inode->i_sb == sb)
276 : 0 : list_move(&inode->i_wb_list, dispatch_queue);
277 : : }
278 : : }
279 : : out:
280 : 0 : return moved;
281 : : }
282 : :
283 : : /*
284 : : * Queue all expired dirty inodes for io, eldest first.
285 : : * Before
286 : : * newly dirtied b_dirty b_io b_more_io
287 : : * =============> gf edc BA
288 : : * After
289 : : * newly dirtied b_dirty b_io b_more_io
290 : : * =============> g fBAedc
291 : : * |
292 : : * +--> dequeue for IO
293 : : */
294 : 0 : static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work)
295 : : {
296 : : int moved;
297 [ - + ]: 10947 : assert_spin_locked(&wb->list_lock);
298 : 10947 : list_splice_init(&wb->b_more_io, &wb->b_io);
299 : 10947 : moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, work);
300 : : trace_writeback_queue_io(wb, work, moved);
301 : 0 : }
302 : :
303 : 0 : static int write_inode(struct inode *inode, struct writeback_control *wbc)
304 : : {
305 : : int ret;
306 : :
307 [ + - ][ + - ]: 14810 : if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
308 : : trace_writeback_write_inode_start(inode, wbc);
309 : 14810 : ret = inode->i_sb->s_op->write_inode(inode, wbc);
310 : : trace_writeback_write_inode(inode, wbc);
311 : 14810 : return ret;
312 : : }
313 : : return 0;
314 : : }
315 : :
316 : : /*
317 : : * Wait for writeback on an inode to complete. Called with i_lock held.
318 : : * Caller must make sure inode cannot go away when we drop i_lock.
319 : : */
320 : 0 : static void __inode_wait_for_writeback(struct inode *inode)
321 : : __releases(inode->i_lock)
322 : : __acquires(inode->i_lock)
323 : : {
324 : 3332950 : DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
325 : : wait_queue_head_t *wqh;
326 : :
327 : 1666475 : wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
328 [ + + ]: 1668644 : while (inode->i_state & I_SYNC) {
329 : : spin_unlock(&inode->i_lock);
330 : 26 : __wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
331 : : spin_lock(&inode->i_lock);
332 : : }
333 : 1668618 : }
334 : :
335 : : /*
336 : : * Wait for writeback on an inode to complete. Caller must have inode pinned.
337 : : */
338 : 0 : void inode_wait_for_writeback(struct inode *inode)
339 : : {
340 : : spin_lock(&inode->i_lock);
341 : 1694735 : __inode_wait_for_writeback(inode);
342 : : spin_unlock(&inode->i_lock);
343 : 1719828 : }
344 : :
345 : : /*
346 : : * Sleep until I_SYNC is cleared. This function must be called with i_lock
347 : : * held and drops it. It is aimed for callers not holding any inode reference
348 : : * so once i_lock is dropped, inode can go away.
349 : : */
350 : 0 : static void inode_sleep_on_writeback(struct inode *inode)
351 : : __releases(inode->i_lock)
352 : : {
353 : 0 : DEFINE_WAIT(wait);
354 : 0 : wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
355 : : int sleep;
356 : :
357 : 0 : prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
358 : 0 : sleep = inode->i_state & I_SYNC;
359 : : spin_unlock(&inode->i_lock);
360 [ # # ]: 0 : if (sleep)
361 : 0 : schedule();
362 : 0 : finish_wait(wqh, &wait);
363 : 0 : }
364 : :
365 : : /*
366 : : * Find proper writeback list for the inode depending on its current state and
367 : : * possibly also change of its state while we were doing writeback. Here we
368 : : * handle things such as livelock prevention or fairness of writeback among
369 : : * inodes. This function can be called only by flusher thread - noone else
370 : : * processes all inodes in writeback lists and requeueing inodes behind flusher
371 : : * thread's back can have unexpected consequences.
372 : : */
373 : 0 : static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
374 : : struct writeback_control *wbc)
375 : : {
376 [ + + ]: 17444 : if (inode->i_state & I_FREEING)
377 : : return;
378 : :
379 : : /*
380 : : * Sync livelock prevention. Each inode is tagged and synced in one
381 : : * shot. If still dirty, it will be redirty_tail()'ed below. Update
382 : : * the dirty time to prevent enqueue and sync it again.
383 : : */
384 [ + + ][ + + ]: 17418 : if ((inode->i_state & I_DIRTY) &&
385 [ - + ]: 1405 : (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
386 : 75 : inode->dirtied_when = jiffies;
387 : :
388 [ + + ]: 17418 : if (wbc->pages_skipped) {
389 : : /*
390 : : * writeback is not making progress due to locked
391 : : * buffers. Skip this inode for now.
392 : : */
393 : 23 : redirty_tail(inode, wb);
394 : 23 : return;
395 : : }
396 : :
397 [ + + ]: 17395 : if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
398 : : /*
399 : : * We didn't write back all the pages. nfs_writepages()
400 : : * sometimes bales out without doing anything.
401 : : */
402 [ + + ]: 1446 : if (wbc->nr_to_write <= 0) {
403 : : /* Slice used up. Queue for next turn. */
404 : 315 : requeue_io(inode, wb);
405 : : } else {
406 : : /*
407 : : * Writeback blocked by something other than
408 : : * congestion. Delay the inode for some time to
409 : : * avoid spinning on the CPU (100% iowait)
410 : : * retrying writeback of the dirty page/inode
411 : : * that cannot be performed immediately.
412 : : */
413 : 1131 : redirty_tail(inode, wb);
414 : : }
415 [ + + ]: 15949 : } else if (inode->i_state & I_DIRTY) {
416 : : /*
417 : : * Filesystems can dirty the inode during writeback operations,
418 : : * such as delayed allocation during submission or metadata
419 : : * updates after data IO completion.
420 : : */
421 : 13 : redirty_tail(inode, wb);
422 : : } else {
423 : : /* The inode is clean. Remove from writeback lists. */
424 : 15936 : list_del_init(&inode->i_wb_list);
425 : : }
426 : : }
427 : :
428 : : /*
429 : : * Write out an inode and its dirty pages. Do not update the writeback list
430 : : * linkage. That is left to the caller. The caller is also responsible for
431 : : * setting I_SYNC flag and calling inode_sync_complete() to clear it.
432 : : */
433 : : static int
434 : 0 : __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
435 : : {
436 : 17446 : struct address_space *mapping = inode->i_mapping;
437 : 17446 : long nr_to_write = wbc->nr_to_write;
438 : : unsigned dirty;
439 : : int ret;
440 : :
441 [ - + ]: 17446 : WARN_ON(!(inode->i_state & I_SYNC));
442 : :
443 : 17446 : trace_writeback_single_inode_start(inode, wbc, nr_to_write);
444 : :
445 : 17446 : ret = do_writepages(mapping, wbc);
446 : :
447 : : /*
448 : : * Make sure to wait on the data before writing out the metadata.
449 : : * This is important for filesystems that modify metadata on data
450 : : * I/O completion. We don't do it for sync(2) writeback because it has a
451 : : * separate, external IO completion path and ->sync_fs for guaranteeing
452 : : * inode metadata is written back correctly.
453 : : */
454 [ + + ][ + + ]: 17446 : if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {
455 : 2 : int err = filemap_fdatawait(mapping);
456 [ + - ]: 2 : if (ret == 0)
457 : : ret = err;
458 : : }
459 : :
460 : : /*
461 : : * Some filesystems may redirty the inode during the writeback
462 : : * due to delalloc, clear dirty metadata flags right before
463 : : * write_inode()
464 : : */
465 : : spin_lock(&inode->i_lock);
466 : : /* Clear I_DIRTY_PAGES if we've written out all dirty pages */
467 [ + + ]: 17446 : if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
468 : 15987 : inode->i_state &= ~I_DIRTY_PAGES;
469 : 17446 : dirty = inode->i_state & I_DIRTY;
470 : 17446 : inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC);
471 : : spin_unlock(&inode->i_lock);
472 : : /* Don't write the inode if only I_DIRTY_PAGES was set */
473 [ + + ]: 17446 : if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
474 : 14810 : int err = write_inode(inode, wbc);
475 [ + - ]: 14810 : if (ret == 0)
476 : : ret = err;
477 : : }
478 : : trace_writeback_single_inode(inode, wbc, nr_to_write);
479 : 0 : return ret;
480 : : }
481 : :
482 : : /*
483 : : * Write out an inode's dirty pages. Either the caller has an active reference
484 : : * on the inode or the inode has I_WILL_FREE set.
485 : : *
486 : : * This function is designed to be called for writing back one inode which
487 : : * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode()
488 : : * and does more profound writeback list handling in writeback_sb_inodes().
489 : : */
490 : : static int
491 : 0 : writeback_single_inode(struct inode *inode, struct bdi_writeback *wb,
492 : : struct writeback_control *wbc)
493 : : {
494 : : int ret = 0;
495 : :
496 : : spin_lock(&inode->i_lock);
497 [ + + ]: 5 : if (!atomic_read(&inode->i_count))
498 [ - + ]: 3 : WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
499 : : else
500 [ - + ]: 2 : WARN_ON(inode->i_state & I_WILL_FREE);
501 : :
502 [ - + ]: 10 : if (inode->i_state & I_SYNC) {
503 [ # # ]: 0 : if (wbc->sync_mode != WB_SYNC_ALL)
504 : : goto out;
505 : : /*
506 : : * It's a data-integrity sync. We must wait. Since callers hold
507 : : * inode reference or inode has I_WILL_FREE set, it cannot go
508 : : * away under us.
509 : : */
510 : 0 : __inode_wait_for_writeback(inode);
511 : : }
512 [ - + ]: 5 : WARN_ON(inode->i_state & I_SYNC);
513 : : /*
514 : : * Skip inode if it is clean and we have no outstanding writeback in
515 : : * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this
516 : : * function since flusher thread may be doing for example sync in
517 : : * parallel and if we move the inode, it could get skipped. So here we
518 : : * make sure inode is on some writeback list and leave it there unless
519 : : * we have completely cleaned the inode.
520 : : */
521 [ + + ][ + - ]: 5 : if (!(inode->i_state & I_DIRTY) &&
522 [ - + ]: 3 : (wbc->sync_mode != WB_SYNC_ALL ||
523 : 3 : !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
524 : : goto out;
525 : 2 : inode->i_state |= I_SYNC;
526 : : spin_unlock(&inode->i_lock);
527 : :
528 : 2 : ret = __writeback_single_inode(inode, wbc);
529 : :
530 : : spin_lock(&wb->list_lock);
531 : : spin_lock(&inode->i_lock);
532 : : /*
533 : : * If inode is clean, remove it from writeback lists. Otherwise don't
534 : : * touch it. See comment above for explanation.
535 : : */
536 [ + - ]: 2 : if (!(inode->i_state & I_DIRTY))
537 : 2 : list_del_init(&inode->i_wb_list);
538 : : spin_unlock(&wb->list_lock);
539 : 2 : inode_sync_complete(inode);
540 : : out:
541 : : spin_unlock(&inode->i_lock);
542 : 5 : return ret;
543 : : }
544 : :
545 : : static long writeback_chunk_size(struct backing_dev_info *bdi,
546 : : struct wb_writeback_work *work)
547 : : {
548 : : long pages;
549 : :
550 : : /*
551 : : * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
552 : : * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
553 : : * here avoids calling into writeback_inodes_wb() more than once.
554 : : *
555 : : * The intended call sequence for WB_SYNC_ALL writeback is:
556 : : *
557 : : * wb_writeback()
558 : : * writeback_sb_inodes() <== called only once
559 : : * write_cache_pages() <== called once for each inode
560 : : * (quickly) tag currently dirty pages
561 : : * (maybe slowly) sync all tagged pages
562 : : */
563 [ + + ][ + + ]: 17444 : if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
564 : : pages = LONG_MAX;
565 : : else {
566 : 16975 : pages = min(bdi->avg_write_bandwidth / 2,
567 : : global_dirty_limit / DIRTY_SCOPE);
568 : 16975 : pages = min(pages, work->nr_pages);
569 : 16975 : pages = round_down(pages + MIN_WRITEBACK_PAGES,
570 : : MIN_WRITEBACK_PAGES);
571 : : }
572 : :
573 : : return pages;
574 : : }
575 : :
576 : : /*
577 : : * Write a portion of b_io inodes which belong to @sb.
578 : : *
579 : : * Return the number of pages and/or inodes written.
580 : : */
581 : 0 : static long writeback_sb_inodes(struct super_block *sb,
582 : : struct bdi_writeback *wb,
583 : : struct wb_writeback_work *work)
584 : : {
585 : 55713 : struct writeback_control wbc = {
586 : 7959 : .sync_mode = work->sync_mode,
587 : 7959 : .tagged_writepages = work->tagged_writepages,
588 : 7959 : .for_kupdate = work->for_kupdate,
589 : 7959 : .for_background = work->for_background,
590 : 7959 : .for_sync = work->for_sync,
591 : 7959 : .range_cyclic = work->range_cyclic,
592 : : .range_start = 0,
593 : : .range_end = LLONG_MAX,
594 : : };
595 : 7959 : unsigned long start_time = jiffies;
596 : : long write_chunk;
597 : : long wrote = 0; /* count both pages and inodes */
598 : :
599 [ + + ]: 24978 : while (!list_empty(&wb->b_io)) {
600 : 18833 : struct inode *inode = wb_inode(wb->b_io.prev);
601 : :
602 [ + + ]: 18833 : if (inode->i_sb != sb) {
603 [ + + ]: 1384 : if (work->sb) {
604 : : /*
605 : : * We only want to write back data for this
606 : : * superblock, move all inodes not belonging
607 : : * to it back onto the dirty list.
608 : : */
609 : 93 : redirty_tail(inode, wb);
610 : 93 : continue;
611 : : }
612 : :
613 : : /*
614 : : * The inode belongs to a different superblock.
615 : : * Bounce back to the caller to unpin this and
616 : : * pin the next superblock.
617 : : */
618 : : break;
619 : : }
620 : :
621 : : /*
622 : : * Don't bother with new inodes or inodes being freed, first
623 : : * kind does not need periodic writeout yet, and for the latter
624 : : * kind writeout is handled by the freer.
625 : : */
626 : : spin_lock(&inode->i_lock);
627 [ + + ]: 17449 : if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
628 : : spin_unlock(&inode->i_lock);
629 : 5 : redirty_tail(inode, wb);
630 : 5 : continue;
631 : : }
632 [ - + ][ # # ]: 17444 : if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
633 : : /*
634 : : * If this inode is locked for writeback and we are not
635 : : * doing writeback-for-data-integrity, move it to
636 : : * b_more_io so that writeback can proceed with the
637 : : * other inodes on s_io.
638 : : *
639 : : * We'll have another go at writing back this inode
640 : : * when we completed a full scan of b_io.
641 : : */
642 : : spin_unlock(&inode->i_lock);
643 : 0 : requeue_io(inode, wb);
644 : : trace_writeback_sb_inodes_requeue(inode);
645 : 0 : continue;
646 : : }
647 : : spin_unlock(&wb->list_lock);
648 : :
649 : : /*
650 : : * We already requeued the inode if it had I_SYNC set and we
651 : : * are doing WB_SYNC_NONE writeback. So this catches only the
652 : : * WB_SYNC_ALL case.
653 : : */
654 [ - + ]: 17444 : if (inode->i_state & I_SYNC) {
655 : : /* Wait for I_SYNC. This function drops i_lock... */
656 : 0 : inode_sleep_on_writeback(inode);
657 : : /* Inode may be gone, start again */
658 : : spin_lock(&wb->list_lock);
659 : 0 : continue;
660 : : }
661 : 17444 : inode->i_state |= I_SYNC;
662 : : spin_unlock(&inode->i_lock);
663 : :
664 : 17444 : write_chunk = writeback_chunk_size(wb->bdi, work);
665 : 17444 : wbc.nr_to_write = write_chunk;
666 : 17444 : wbc.pages_skipped = 0;
667 : :
668 : : /*
669 : : * We use I_SYNC to pin the inode in memory. While it is set
670 : : * evict_inode() will wait so the inode cannot be freed.
671 : : */
672 : 17444 : __writeback_single_inode(inode, &wbc);
673 : :
674 : 17444 : work->nr_pages -= write_chunk - wbc.nr_to_write;
675 : 17444 : wrote += write_chunk - wbc.nr_to_write;
676 : : spin_lock(&wb->list_lock);
677 : : spin_lock(&inode->i_lock);
678 [ + + ]: 17444 : if (!(inode->i_state & I_DIRTY))
679 : 15956 : wrote++;
680 : 17444 : requeue_inode(inode, wb, &wbc);
681 : 17444 : inode_sync_complete(inode);
682 : : spin_unlock(&inode->i_lock);
683 : 17444 : cond_resched_lock(&wb->list_lock);
684 : : /*
685 : : * bail out to wb_writeback() often enough to check
686 : : * background threshold and other termination conditions.
687 : : */
688 [ + + ]: 17444 : if (wrote) {
689 [ + + ]: 17286 : if (time_is_before_jiffies(start_time + HZ / 10UL))
690 : : break;
691 [ + - ]: 17019 : if (work->nr_pages <= 0)
692 : : break;
693 : : }
694 : : }
695 : 0 : return wrote;
696 : : }
697 : :
698 : 0 : static long __writeback_inodes_wb(struct bdi_writeback *wb,
699 : : struct wb_writeback_work *work)
700 : : {
701 : 7733 : unsigned long start_time = jiffies;
702 : : long wrote = 0;
703 : :
704 [ + + ]: 11939 : while (!list_empty(&wb->b_io)) {
705 : 4561 : struct inode *inode = wb_inode(wb->b_io.prev);
706 : 4561 : struct super_block *sb = inode->i_sb;
707 : :
708 [ - + ]: 4561 : if (!grab_super_passive(sb)) {
709 : : /*
710 : : * grab_super_passive() may fail consistently due to
711 : : * s_umount being grabbed by someone else. Don't use
712 : : * requeue_io() to avoid busy retrying the inode/sb.
713 : : */
714 : 0 : redirty_tail(inode, wb);
715 : 0 : continue;
716 : : }
717 : 4561 : wrote += writeback_sb_inodes(sb, wb, work);
718 : 4561 : drop_super(sb);
719 : :
720 : : /* refer to the same tests at the end of writeback_sb_inodes */
721 [ + + ]: 4561 : if (wrote) {
722 [ + + ]: 4422 : if (time_is_before_jiffies(start_time + HZ / 10UL))
723 : : break;
724 [ + - ]: 4206 : if (work->nr_pages <= 0)
725 : : break;
726 : : }
727 : : }
728 : : /* Leave any unwritten inodes on b_io */
729 : 0 : return wrote;
730 : : }
731 : :
732 : 0 : static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
733 : : enum wb_reason reason)
734 : : {
735 : 0 : struct wb_writeback_work work = {
736 : : .nr_pages = nr_pages,
737 : : .sync_mode = WB_SYNC_NONE,
738 : : .range_cyclic = 1,
739 : : .reason = reason,
740 : : };
741 : :
742 : : spin_lock(&wb->list_lock);
743 [ # # ]: 0 : if (list_empty(&wb->b_io))
744 : 0 : queue_io(wb, &work);
745 : 0 : __writeback_inodes_wb(wb, &work);
746 : : spin_unlock(&wb->list_lock);
747 : :
748 : 0 : return nr_pages - work.nr_pages;
749 : : }
750 : :
751 : 0 : static bool over_bground_thresh(struct backing_dev_info *bdi)
752 : : {
753 : : unsigned long background_thresh, dirty_thresh;
754 : :
755 : 4101 : global_dirty_limits(&background_thresh, &dirty_thresh);
756 : :
757 [ + + ]: 4101 : if (global_page_state(NR_FILE_DIRTY) +
758 : 4101 : global_page_state(NR_UNSTABLE_NFS) > background_thresh)
759 : : return true;
760 : :
761 [ + + ]: 3973 : if (bdi_stat(bdi, BDI_RECLAIMABLE) >
762 : 3973 : bdi_dirty_limit(bdi, background_thresh))
763 : : return true;
764 : :
765 : 3440 : return false;
766 : : }
767 : :
768 : : /*
769 : : * Called under wb->list_lock. If there are multiple wb per bdi,
770 : : * only the flusher working on the first wb should do it.
771 : : */
772 : : static void wb_update_bandwidth(struct bdi_writeback *wb,
773 : : unsigned long start_time)
774 : : {
775 : 11131 : __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, 0, start_time);
776 : : }
777 : :
778 : : /*
779 : : * Explicit flushing or periodic writeback of "old" data.
780 : : *
781 : : * Define "old": the first time one of an inode's pages is dirtied, we mark the
782 : : * dirtying-time in the inode's address_space. So this periodic writeback code
783 : : * just walks the superblock inode list, writing back any inodes which are
784 : : * older than a specific point in time.
785 : : *
786 : : * Try to run once per dirty_writeback_interval. But if a writeback event
787 : : * takes longer than a dirty_writeback_interval interval, then leave a
788 : : * one-second gap.
789 : : *
790 : : * older_than_this takes precedence over nr_to_write. So we'll only write back
791 : : * all dirty pages if they are all attached to "old" mappings.
792 : : */
793 : 0 : static long wb_writeback(struct bdi_writeback *wb,
794 : : struct wb_writeback_work *work)
795 : : {
796 : 7706 : unsigned long wb_start = jiffies;
797 : 7706 : long nr_pages = work->nr_pages;
798 : : unsigned long oldest_jif;
799 : : struct inode *inode;
800 : : long progress;
801 : :
802 : 7706 : oldest_jif = jiffies;
803 : 7706 : work->older_than_this = &oldest_jif;
804 : :
805 : : spin_lock(&wb->list_lock);
806 : : for (;;) {
807 : : /*
808 : : * Stop writeback when nr_pages has been consumed
809 : : */
810 [ + + ]: 11213 : if (work->nr_pages <= 0)
811 : : break;
812 : :
813 : : /*
814 : : * Background writeout and kupdate-style writeback may
815 : : * run forever. Stop them if there is other work to do
816 : : * so that e.g. sync can proceed. They'll be restarted
817 : : * after the other works are all done.
818 : : */
819 [ + + ][ + - ]: 11209 : if ((work->for_background || work->for_kupdate) &&
820 : 2623 : !list_empty(&wb->bdi->work_list))
821 : : break;
822 : :
823 : : /*
824 : : * For background writeout, stop when we are below the
825 : : * background dirty threshold
826 : : */
827 [ + + ][ + + ]: 11209 : if (work->for_background && !over_bground_thresh(wb->bdi))
828 : : break;
829 : :
830 : : /*
831 : : * Kupdate and background works are special and we want to
832 : : * include all inodes that need writing. Livelock avoidance is
833 : : * handled by these works yielding to any other work so we are
834 : : * safe.
835 : : */
836 [ + + ]: 11131 : if (work->for_kupdate) {
837 : 2100 : oldest_jif = jiffies -
838 : 2100 : msecs_to_jiffies(dirty_expire_interval * 10);
839 [ + + ]: 9031 : } else if (work->for_background)
840 : 445 : oldest_jif = jiffies;
841 : :
842 : 11131 : trace_writeback_start(wb->bdi, work);
843 [ + + ]: 11131 : if (list_empty(&wb->b_io))
844 : 10947 : queue_io(wb, work);
845 [ + + ]: 11131 : if (work->sb)
846 : 3398 : progress = writeback_sb_inodes(work->sb, wb, work);
847 : : else
848 : 7733 : progress = __writeback_inodes_wb(wb, work);
849 : 11131 : trace_writeback_written(wb->bdi, work);
850 : :
851 : : wb_update_bandwidth(wb, wb_start);
852 : :
853 : : /*
854 : : * Did we write something? Try for more
855 : : *
856 : : * Dirty inodes are moved to b_io for writeback in batches.
857 : : * The completion of the current batch does not necessarily
858 : : * mean the overall work is done. So we keep looping as long
859 : : * as made some progress on cleaning pages or inodes.
860 : : */
861 [ + + ]: 11131 : if (progress)
862 : 3507 : continue;
863 : : /*
864 : : * No more inodes for IO, bail
865 : : */
866 [ - + ]: 7624 : if (list_empty(&wb->b_more_io))
867 : : break;
868 : : /*
869 : : * Nothing written. Wait for some inode to
870 : : * become available for writeback. Otherwise
871 : : * we'll just busyloop.
872 : : */
873 [ # # ]: 0 : if (!list_empty(&wb->b_more_io)) {
874 : 0 : trace_writeback_wait(wb->bdi, work);
875 : 0 : inode = wb_inode(wb->b_more_io.prev);
876 : : spin_lock(&inode->i_lock);
877 : : spin_unlock(&wb->list_lock);
878 : : /* This function drops i_lock... */
879 : 0 : inode_sleep_on_writeback(inode);
880 : : spin_lock(&wb->list_lock);
881 : : }
882 : : }
883 : : spin_unlock(&wb->list_lock);
884 : :
885 : 7706 : return nr_pages - work->nr_pages;
886 : : }
887 : :
888 : : /*
889 : : * Return the next wb_writeback_work struct that hasn't been processed yet.
890 : : */
891 : : static struct wb_writeback_work *
892 : 0 : get_next_work_item(struct backing_dev_info *bdi)
893 : : {
894 : : struct wb_writeback_work *work = NULL;
895 : :
896 : : spin_lock_bh(&bdi->wb_lock);
897 [ + + ]: 9554 : if (!list_empty(&bdi->work_list)) {
898 : 5976 : work = list_entry(bdi->work_list.next,
899 : : struct wb_writeback_work, list);
900 : 5976 : list_del_init(&work->list);
901 : : }
902 : : spin_unlock_bh(&bdi->wb_lock);
903 : 9554 : return work;
904 : : }
905 : :
906 : : /*
907 : : * Add in the number of potentially dirty inodes, because each inode
908 : : * write can dirty pagecache in the underlying blockdev.
909 : : */
910 : 0 : static unsigned long get_nr_dirty_pages(void)
911 : : {
912 : 4516 : return global_page_state(NR_FILE_DIRTY) +
913 : 4516 : global_page_state(NR_UNSTABLE_NFS) +
914 : 4516 : get_nr_dirty_inodes();
915 : : }
916 : :
917 : 0 : static long wb_check_background_flush(struct bdi_writeback *wb)
918 : : {
919 [ + + ]: 3578 : if (over_bground_thresh(wb->bdi)) {
920 : :
921 : 216 : struct wb_writeback_work work = {
922 : : .nr_pages = LONG_MAX,
923 : : .sync_mode = WB_SYNC_NONE,
924 : : .for_background = 1,
925 : : .range_cyclic = 1,
926 : : .reason = WB_REASON_BACKGROUND,
927 : : };
928 : :
929 : 216 : return wb_writeback(wb, &work);
930 : : }
931 : :
932 : : return 0;
933 : : }
934 : :
935 : 0 : static long wb_check_old_data_flush(struct bdi_writeback *wb)
936 : : {
937 : : unsigned long expired;
938 : : long nr_pages;
939 : :
940 : : /*
941 : : * When set to zero, disable periodic writeback
942 : : */
943 [ + - ]: 3578 : if (!dirty_writeback_interval)
944 : : return 0;
945 : :
946 : 7156 : expired = wb->last_old_flush +
947 : 3578 : msecs_to_jiffies(dirty_writeback_interval * 10);
948 [ + + ]: 3578 : if (time_before(jiffies, expired))
949 : : return 0;
950 : :
951 : 1514 : wb->last_old_flush = jiffies;
952 : 1514 : nr_pages = get_nr_dirty_pages();
953 : :
954 [ + - ]: 1514 : if (nr_pages) {
955 : 1514 : struct wb_writeback_work work = {
956 : : .nr_pages = nr_pages,
957 : : .sync_mode = WB_SYNC_NONE,
958 : : .for_kupdate = 1,
959 : : .range_cyclic = 1,
960 : : .reason = WB_REASON_PERIODIC,
961 : : };
962 : :
963 : 1514 : return wb_writeback(wb, &work);
964 : : }
965 : :
966 : : return 0;
967 : : }
968 : :
969 : : /*
970 : : * Retrieve work items and do the writeback they describe
971 : : */
972 : 0 : static long wb_do_writeback(struct bdi_writeback *wb)
973 : : {
974 : 3578 : struct backing_dev_info *bdi = wb->bdi;
975 : : struct wb_writeback_work *work;
976 : : long wrote = 0;
977 : :
978 : 3578 : set_bit(BDI_writeback_running, &wb->bdi->state);
979 [ + + ]: 9554 : while ((work = get_next_work_item(bdi)) != NULL) {
980 : :
981 : : trace_writeback_exec(bdi, work);
982 : :
983 : 5976 : wrote += wb_writeback(wb, work);
984 : :
985 : : /*
986 : : * Notify the caller of completion if this is a synchronous
987 : : * work item, otherwise just free it.
988 : : */
989 [ + + ]: 5976 : if (work->done)
990 : 3022 : complete(work->done);
991 : : else
992 : 5976 : kfree(work);
993 : : }
994 : :
995 : : /*
996 : : * Check for periodic writeback, kupdated() style
997 : : */
998 : 3578 : wrote += wb_check_old_data_flush(wb);
999 : 3578 : wrote += wb_check_background_flush(wb);
1000 : 3578 : clear_bit(BDI_writeback_running, &wb->bdi->state);
1001 : :
1002 : 3578 : return wrote;
1003 : : }
1004 : :
1005 : : /*
1006 : : * Handle writeback of dirty data for the device backed by this bdi. Also
1007 : : * reschedules periodically and does kupdated style flushing.
1008 : : */
1009 : 0 : void bdi_writeback_workfn(struct work_struct *work)
1010 : : {
1011 : 3578 : struct bdi_writeback *wb = container_of(to_delayed_work(work),
1012 : : struct bdi_writeback, dwork);
1013 : 3578 : struct backing_dev_info *bdi = wb->bdi;
1014 : : long pages_written;
1015 : :
1016 : 3578 : set_worker_desc("flush-%s", dev_name(bdi->dev));
1017 : 3578 : current->flags |= PF_SWAPWRITE;
1018 : :
1019 [ - + ][ # # ]: 3578 : if (likely(!current_is_workqueue_rescuer() ||
1020 : : list_empty(&bdi->bdi_list))) {
1021 : : /*
1022 : : * The normal path. Keep writing back @bdi until its
1023 : : * work_list is empty. Note that this path is also taken
1024 : : * if @bdi is shutting down even when we're running off the
1025 : : * rescuer as work_list needs to be drained.
1026 : : */
1027 : : do {
1028 : 3578 : pages_written = wb_do_writeback(wb);
1029 : : trace_writeback_pages_written(pages_written);
1030 [ - + ]: 3578 : } while (!list_empty(&bdi->work_list));
1031 : : } else {
1032 : : /*
1033 : : * bdi_wq can't get enough workers and we're running off
1034 : : * the emergency worker. Don't hog it. Hopefully, 1024 is
1035 : : * enough for efficient IO.
1036 : : */
1037 : 0 : pages_written = writeback_inodes_wb(&bdi->wb, 1024,
1038 : : WB_REASON_FORKER_THREAD);
1039 : : trace_writeback_pages_written(pages_written);
1040 : : }
1041 : :
1042 [ + - ][ + + ]: 7156 : if (!list_empty(&bdi->work_list) ||
1043 [ + - ]: 1247 : (wb_has_dirty_io(wb) && dirty_writeback_interval))
1044 : 1247 : queue_delayed_work(bdi_wq, &wb->dwork,
1045 : : msecs_to_jiffies(dirty_writeback_interval * 10));
1046 : :
1047 : 3578 : current->flags &= ~PF_SWAPWRITE;
1048 : 3578 : }
1049 : :
1050 : : /*
1051 : : * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
1052 : : * the whole world.
1053 : : */
1054 : 0 : void wakeup_flusher_threads(long nr_pages, enum wb_reason reason)
1055 : : {
1056 : : struct backing_dev_info *bdi;
1057 : :
1058 [ + + ]: 2997 : if (!nr_pages)
1059 : 2976 : nr_pages = get_nr_dirty_pages();
1060 : :
1061 : : rcu_read_lock();
1062 [ + + ]: 92815 : list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
1063 [ + + ]: 86821 : if (!bdi_has_dirty_io(bdi))
1064 : 83867 : continue;
1065 : 2953 : __bdi_start_writeback(bdi, nr_pages, false, reason);
1066 : : }
1067 : : rcu_read_unlock();
1068 : 2997 : }
1069 : :
1070 : 0 : static noinline void block_dump___mark_inode_dirty(struct inode *inode)
1071 : : {
1072 [ # # ][ # # ]: 0 : if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
1073 : : struct dentry *dentry;
1074 : : const char *name = "?";
1075 : :
1076 : 0 : dentry = d_find_alias(inode);
1077 [ # # ]: 0 : if (dentry) {
1078 : : spin_lock(&dentry->d_lock);
1079 : 0 : name = (const char *) dentry->d_name.name;
1080 : : }
1081 : 0 : printk(KERN_DEBUG
1082 : : "%s(%d): dirtied inode %lu (%s) on %s\n",
1083 : 0 : current->comm, task_pid_nr(current), inode->i_ino,
1084 : 0 : name, inode->i_sb->s_id);
1085 [ # # ]: 0 : if (dentry) {
1086 : : spin_unlock(&dentry->d_lock);
1087 : 0 : dput(dentry);
1088 : : }
1089 : : }
1090 : 0 : }
1091 : :
1092 : : /**
1093 : : * __mark_inode_dirty - internal function
1094 : : * @inode: inode to mark
1095 : : * @flags: what kind of dirty (i.e. I_DIRTY_SYNC)
1096 : : * Mark an inode as dirty. Callers should use mark_inode_dirty or
1097 : : * mark_inode_dirty_sync.
1098 : : *
1099 : : * Put the inode on the super block's dirty list.
1100 : : *
1101 : : * CAREFUL! We mark it dirty unconditionally, but move it onto the
1102 : : * dirty list only if it is hashed or if it refers to a blockdev.
1103 : : * If it was not hashed, it will never be added to the dirty list
1104 : : * even if it is later hashed, as it will have been marked dirty already.
1105 : : *
1106 : : * In short, make sure you hash any inodes _before_ you start marking
1107 : : * them dirty.
1108 : : *
1109 : : * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
1110 : : * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of
1111 : : * the kernel-internal blockdev inode represents the dirtying time of the
1112 : : * blockdev's pages. This is why for I_DIRTY_PAGES we always use
1113 : : * page->mapping->host, so the page-dirtying time is recorded in the internal
1114 : : * blockdev inode.
1115 : : */
1116 : 0 : void __mark_inode_dirty(struct inode *inode, int flags)
1117 : : {
1118 : 6862725 : struct super_block *sb = inode->i_sb;
1119 : 74827 : struct backing_dev_info *bdi = NULL;
1120 : :
1121 : : /*
1122 : : * Don't do this for I_DIRTY_PAGES - that doesn't actually
1123 : : * dirty the inode itself
1124 : : */
1125 [ + + ]: 6862725 : if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
1126 : : trace_writeback_dirty_inode_start(inode, flags);
1127 : :
1128 [ + + ]: 4748673 : if (sb->s_op->dirty_inode)
1129 : 4710276 : sb->s_op->dirty_inode(inode, flags);
1130 : :
1131 : : trace_writeback_dirty_inode(inode, flags);
1132 : : }
1133 : :
1134 : : /*
1135 : : * make sure that changes are seen by all cpus before we test i_state
1136 : : * -- mikulas
1137 : : */
1138 : 6861642 : smp_mb();
1139 : :
1140 : : /* avoid the locking if we can */
1141 [ + + ]: 6863083 : if ((inode->i_state & flags) == flags)
1142 : : return;
1143 : :
1144 [ - + ]: 113321 : if (unlikely(block_dump > 1))
1145 : 0 : block_dump___mark_inode_dirty(inode);
1146 : :
1147 : : spin_lock(&inode->i_lock);
1148 [ + + ]: 113105 : if ((inode->i_state & flags) != flags) {
1149 : 113019 : const int was_dirty = inode->i_state & I_DIRTY;
1150 : :
1151 : 113019 : inode->i_state |= flags;
1152 : :
1153 : : /*
1154 : : * If the inode is being synced, just update its dirty state.
1155 : : * The unlocker will place the inode on the appropriate
1156 : : * superblock list, based upon its state.
1157 : : */
1158 [ + + ]: 113019 : if (inode->i_state & I_SYNC)
1159 : : goto out_unlock_inode;
1160 : :
1161 : : /*
1162 : : * Only add valid (hashed) inodes to the superblock's
1163 : : * dirty list. Add blockdev inodes as well.
1164 : : */
1165 [ + + ]: 112465 : if (!S_ISBLK(inode->i_mode)) {
1166 [ + + ]: 111001 : if (inode_unhashed(inode))
1167 : : goto out_unlock_inode;
1168 : : }
1169 [ + + ]: 109215 : if (inode->i_state & I_FREEING)
1170 : : goto out_unlock_inode;
1171 : :
1172 : : /*
1173 : : * If the inode was already on b_dirty/b_io/b_more_io, don't
1174 : : * reposition it (that would break b_dirty time-ordering).
1175 : : */
1176 [ + + ]: 105903 : if (!was_dirty) {
1177 : : bool wakeup_bdi = false;
1178 : : bdi = inode_to_bdi(inode);
1179 : :
1180 : : spin_unlock(&inode->i_lock);
1181 : : spin_lock(&bdi->wb.list_lock);
1182 [ + + ]: 74827 : if (bdi_cap_writeback_dirty(bdi)) {
1183 [ - + ]: 74801 : WARN(!test_bit(BDI_registered, &bdi->state),
1184 : : "bdi-%s not registered\n", bdi->name);
1185 : :
1186 : : /*
1187 : : * If this is the first dirty inode for this
1188 : : * bdi, we have to wake-up the corresponding
1189 : : * bdi thread to make sure background
1190 : : * write-back happens later.
1191 : : */
1192 [ + + ]: 74801 : if (!wb_has_dirty_io(&bdi->wb))
1193 : : wakeup_bdi = true;
1194 : : }
1195 : :
1196 : 74827 : inode->dirtied_when = jiffies;
1197 : 74827 : list_move(&inode->i_wb_list, &bdi->wb.b_dirty);
1198 : : spin_unlock(&bdi->wb.list_lock);
1199 : :
1200 [ + + ]: 74827 : if (wakeup_bdi)
1201 : 1649 : bdi_wakeup_thread_delayed(bdi);
1202 : : return;
1203 : : }
1204 : : }
1205 : : out_unlock_inode:
1206 : : spin_unlock(&inode->i_lock);
1207 : :
1208 : : }
1209 : : EXPORT_SYMBOL(__mark_inode_dirty);
1210 : :
1211 : 0 : static void wait_sb_inodes(struct super_block *sb)
1212 : : {
1213 : : struct inode *inode, *old_inode = NULL;
1214 : :
1215 : : /*
1216 : : * We need to be protected against the filesystem going from
1217 : : * r/o to r/w or vice versa.
1218 : : */
1219 [ - + ]: 2975 : WARN_ON(!rwsem_is_locked(&sb->s_umount));
1220 : :
1221 : : spin_lock(&inode_sb_list_lock);
1222 : :
1223 : : /*
1224 : : * Data integrity sync. Must wait for all pages under writeback,
1225 : : * because there may have been pages dirtied before our sync
1226 : : * call, but which had writeout started before we write it out.
1227 : : * In which case, the inode may not be on the dirty list, but
1228 : : * we still have to wait for that writeout.
1229 : : */
1230 [ + + ]: 6164237 : list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
1231 : 6158263 : struct address_space *mapping = inode->i_mapping;
1232 : :
1233 : : spin_lock(&inode->i_lock);
1234 [ + + ][ + + ]: 6158263 : if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
1235 : 6158256 : (mapping->nrpages == 0)) {
1236 : : spin_unlock(&inode->i_lock);
1237 : 4353218 : continue;
1238 : : }
1239 : 1805045 : __iget(inode);
1240 : : spin_unlock(&inode->i_lock);
1241 : : spin_unlock(&inode_sb_list_lock);
1242 : :
1243 : : /*
1244 : : * We hold a reference to 'inode' so it couldn't have been
1245 : : * removed from s_inodes list while we dropped the
1246 : : * inode_sb_list_lock. We cannot iput the inode now as we can
1247 : : * be holding the last reference and we cannot iput it under
1248 : : * inode_sb_list_lock. So we keep the reference and iput it
1249 : : * later.
1250 : : */
1251 : 1805031 : iput(old_inode);
1252 : : old_inode = inode;
1253 : :
1254 : 1804818 : filemap_fdatawait(mapping);
1255 : :
1256 : 1775930 : cond_resched();
1257 : :
1258 : : spin_lock(&inode_sb_list_lock);
1259 : : }
1260 : : spin_unlock(&inode_sb_list_lock);
1261 : 2999 : iput(old_inode);
1262 : 2999 : }
1263 : :
1264 : : /**
1265 : : * writeback_inodes_sb_nr - writeback dirty inodes from given super_block
1266 : : * @sb: the superblock
1267 : : * @nr: the number of pages to write
1268 : : * @reason: reason why some writeback work initiated
1269 : : *
1270 : : * Start writeback on some inodes on this super_block. No guarantees are made
1271 : : * on how many (if any) will be written, and this function does not wait
1272 : : * for IO completion of submitted IO.
1273 : : */
1274 : 0 : void writeback_inodes_sb_nr(struct super_block *sb,
1275 : : unsigned long nr,
1276 : : enum wb_reason reason)
1277 : : {
1278 : 26 : DECLARE_COMPLETION_ONSTACK(done);
1279 : 26 : struct wb_writeback_work work = {
1280 : : .sb = sb,
1281 : : .sync_mode = WB_SYNC_NONE,
1282 : : .tagged_writepages = 1,
1283 : : .done = &done,
1284 : : .nr_pages = nr,
1285 : : .reason = reason,
1286 : : };
1287 : :
1288 [ + + ]: 26 : if (sb->s_bdi == &noop_backing_dev_info)
1289 : 3 : return;
1290 [ - + ]: 23 : WARN_ON(!rwsem_is_locked(&sb->s_umount));
1291 : 23 : bdi_queue_work(sb->s_bdi, &work);
1292 : 23 : wait_for_completion(&done);
1293 : : }
1294 : : EXPORT_SYMBOL(writeback_inodes_sb_nr);
1295 : :
1296 : : /**
1297 : : * writeback_inodes_sb - writeback dirty inodes from given super_block
1298 : : * @sb: the superblock
1299 : : * @reason: reason why some writeback work was initiated
1300 : : *
1301 : : * Start writeback on some inodes on this super_block. No guarantees are made
1302 : : * on how many (if any) will be written, and this function does not wait
1303 : : * for IO completion of submitted IO.
1304 : : */
1305 : 0 : void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
1306 : : {
1307 : 26 : return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
1308 : : }
1309 : : EXPORT_SYMBOL(writeback_inodes_sb);
1310 : :
1311 : : /**
1312 : : * try_to_writeback_inodes_sb_nr - try to start writeback if none underway
1313 : : * @sb: the superblock
1314 : : * @nr: the number of pages to write
1315 : : * @reason: the reason of writeback
1316 : : *
1317 : : * Invoke writeback_inodes_sb_nr if no writeback is currently underway.
1318 : : * Returns 1 if writeback was started, 0 if not.
1319 : : */
1320 : 0 : int try_to_writeback_inodes_sb_nr(struct super_block *sb,
1321 : : unsigned long nr,
1322 : : enum wb_reason reason)
1323 : : {
1324 [ # # ]: 0 : if (writeback_in_progress(sb->s_bdi))
1325 : : return 1;
1326 : :
1327 [ # # ]: 0 : if (!down_read_trylock(&sb->s_umount))
1328 : : return 0;
1329 : :
1330 : 0 : writeback_inodes_sb_nr(sb, nr, reason);
1331 : 0 : up_read(&sb->s_umount);
1332 : 0 : return 1;
1333 : : }
1334 : : EXPORT_SYMBOL(try_to_writeback_inodes_sb_nr);
1335 : :
1336 : : /**
1337 : : * try_to_writeback_inodes_sb - try to start writeback if none underway
1338 : : * @sb: the superblock
1339 : : * @reason: reason why some writeback work was initiated
1340 : : *
1341 : : * Implement by try_to_writeback_inodes_sb_nr()
1342 : : * Returns 1 if writeback was started, 0 if not.
1343 : : */
1344 : 0 : int try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
1345 : : {
1346 : 0 : return try_to_writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
1347 : : }
1348 : : EXPORT_SYMBOL(try_to_writeback_inodes_sb);
1349 : :
1350 : : /**
1351 : : * sync_inodes_sb - sync sb inode pages
1352 : : * @sb: the superblock
1353 : : *
1354 : : * This function writes and waits on any dirty inode belonging to this
1355 : : * super_block.
1356 : : */
1357 : 0 : void sync_inodes_sb(struct super_block *sb)
1358 : : {
1359 : 62517 : DECLARE_COMPLETION_ONSTACK(done);
1360 : 62517 : struct wb_writeback_work work = {
1361 : : .sb = sb,
1362 : : .sync_mode = WB_SYNC_ALL,
1363 : : .nr_pages = LONG_MAX,
1364 : : .range_cyclic = 0,
1365 : : .done = &done,
1366 : : .reason = WB_REASON_SYNC,
1367 : : .for_sync = 1,
1368 : : };
1369 : :
1370 : : /* Nothing to do? */
1371 [ + + ]: 62517 : if (sb->s_bdi == &noop_backing_dev_info)
1372 : 59518 : return;
1373 [ - + ]: 2999 : WARN_ON(!rwsem_is_locked(&sb->s_umount));
1374 : :
1375 : 2999 : bdi_queue_work(sb->s_bdi, &work);
1376 : 2999 : wait_for_completion(&done);
1377 : :
1378 : 2975 : wait_sb_inodes(sb);
1379 : : }
1380 : : EXPORT_SYMBOL(sync_inodes_sb);
1381 : :
1382 : : /**
1383 : : * write_inode_now - write an inode to disk
1384 : : * @inode: inode to write to disk
1385 : : * @sync: whether the write should be synchronous or not
1386 : : *
1387 : : * This function commits an inode to disk immediately if it is dirty. This is
1388 : : * primarily needed by knfsd.
1389 : : *
1390 : : * The caller must either have a ref on the inode or must have set I_WILL_FREE.
1391 : : */
1392 : 0 : int write_inode_now(struct inode *inode, int sync)
1393 : : {
1394 : 3 : struct bdi_writeback *wb = &inode_to_bdi(inode)->wb;
1395 : 6 : struct writeback_control wbc = {
1396 : : .nr_to_write = LONG_MAX,
1397 : 3 : .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
1398 : : .range_start = 0,
1399 : : .range_end = LLONG_MAX,
1400 : : };
1401 : :
1402 [ - + ]: 3 : if (!mapping_cap_writeback_dirty(inode->i_mapping))
1403 : 0 : wbc.nr_to_write = 0;
1404 : :
1405 : : might_sleep();
1406 : 3 : return writeback_single_inode(inode, wb, &wbc);
1407 : : }
1408 : : EXPORT_SYMBOL(write_inode_now);
1409 : :
1410 : : /**
1411 : : * sync_inode - write an inode and its pages to disk.
1412 : : * @inode: the inode to sync
1413 : : * @wbc: controls the writeback mode
1414 : : *
1415 : : * sync_inode() will write an inode and its pages to disk. It will also
1416 : : * correctly update the inode on its superblock's dirty inode lists and will
1417 : : * update inode->i_state.
1418 : : *
1419 : : * The caller must have a ref on the inode.
1420 : : */
1421 : 0 : int sync_inode(struct inode *inode, struct writeback_control *wbc)
1422 : : {
1423 : 2 : return writeback_single_inode(inode, &inode_to_bdi(inode)->wb, wbc);
1424 : : }
1425 : : EXPORT_SYMBOL(sync_inode);
1426 : :
1427 : : /**
1428 : : * sync_inode_metadata - write an inode to disk
1429 : : * @inode: the inode to sync
1430 : : * @wait: wait for I/O to complete.
1431 : : *
1432 : : * Write an inode to disk and adjust its dirty state after completion.
1433 : : *
1434 : : * Note: only writes the actual inode, no associated data or other metadata.
1435 : : */
1436 : 0 : int sync_inode_metadata(struct inode *inode, int wait)
1437 : : {
1438 : 4 : struct writeback_control wbc = {
1439 : 2 : .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
1440 : : .nr_to_write = 0, /* metadata-only */
1441 : : };
1442 : :
1443 : 2 : return sync_inode(inode, &wbc);
1444 : : }
1445 : : EXPORT_SYMBOL(sync_inode_metadata);
|
