Branch data Line data Source code
1 : : /*
2 : : * mm/rmap.c - physical to virtual reverse mappings
3 : : *
4 : : * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 : : * Released under the General Public License (GPL).
6 : : *
7 : : * Simple, low overhead reverse mapping scheme.
8 : : * Please try to keep this thing as modular as possible.
9 : : *
10 : : * Provides methods for unmapping each kind of mapped page:
11 : : * the anon methods track anonymous pages, and
12 : : * the file methods track pages belonging to an inode.
13 : : *
14 : : * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15 : : * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16 : : * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17 : : * Contributions by Hugh Dickins 2003, 2004
18 : : */
19 : :
20 : : /*
21 : : * Lock ordering in mm:
22 : : *
23 : : * inode->i_mutex (while writing or truncating, not reading or faulting)
24 : : * mm->mmap_sem
25 : : * page->flags PG_locked (lock_page)
26 : : * mapping->i_mmap_mutex
27 : : * anon_vma->rwsem
28 : : * mm->page_table_lock or pte_lock
29 : : * zone->lru_lock (in mark_page_accessed, isolate_lru_page)
30 : : * swap_lock (in swap_duplicate, swap_info_get)
31 : : * mmlist_lock (in mmput, drain_mmlist and others)
32 : : * mapping->private_lock (in __set_page_dirty_buffers)
33 : : * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
34 : : * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
35 : : * sb_lock (within inode_lock in fs/fs-writeback.c)
36 : : * mapping->tree_lock (widely used, in set_page_dirty,
37 : : * in arch-dependent flush_dcache_mmap_lock,
38 : : * within bdi.wb->list_lock in __sync_single_inode)
39 : : *
40 : : * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon)
41 : : * ->tasklist_lock
42 : : * pte map lock
43 : : */
44 : :
45 : : #include <linux/mm.h>
46 : : #include <linux/pagemap.h>
47 : : #include <linux/swap.h>
48 : : #include <linux/swapops.h>
49 : : #include <linux/slab.h>
50 : : #include <linux/init.h>
51 : : #include <linux/ksm.h>
52 : : #include <linux/rmap.h>
53 : : #include <linux/rcupdate.h>
54 : : #include <linux/export.h>
55 : : #include <linux/memcontrol.h>
56 : : #include <linux/mmu_notifier.h>
57 : : #include <linux/migrate.h>
58 : : #include <linux/hugetlb.h>
59 : : #include <linux/backing-dev.h>
60 : :
61 : : #include <asm/tlbflush.h>
62 : :
63 : : #include "internal.h"
64 : :
65 : : static struct kmem_cache *anon_vma_cachep;
66 : : static struct kmem_cache *anon_vma_chain_cachep;
67 : :
68 : : static inline struct anon_vma *anon_vma_alloc(void)
69 : : {
70 : : struct anon_vma *anon_vma;
71 : :
72 : 12496201 : anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
73 [ + + + - ]: 12496188 : if (anon_vma) {
74 : 12496184 : atomic_set(&anon_vma->refcount, 1);
75 : : /*
76 : : * Initialise the anon_vma root to point to itself. If called
77 : : * from fork, the root will be reset to the parents anon_vma.
78 : : */
79 : 12496184 : anon_vma->root = anon_vma;
80 : : }
81 : :
82 : : return anon_vma;
83 : : }
84 : :
85 : 9380 : static inline void anon_vma_free(struct anon_vma *anon_vma)
86 : : {
87 : : VM_BUG_ON(atomic_read(&anon_vma->refcount));
88 : :
89 : : /*
90 : : * Synchronize against page_lock_anon_vma_read() such that
91 : : * we can safely hold the lock without the anon_vma getting
92 : : * freed.
93 : : *
94 : : * Relies on the full mb implied by the atomic_dec_and_test() from
95 : : * put_anon_vma() against the acquire barrier implied by
96 : : * down_read_trylock() from page_lock_anon_vma_read(). This orders:
97 : : *
98 : : * page_lock_anon_vma_read() VS put_anon_vma()
99 : : * down_read_trylock() atomic_dec_and_test()
100 : : * LOCK MB
101 : : * atomic_read() rwsem_is_locked()
102 : : *
103 : : * LOCK should suffice since the actual taking of the lock must
104 : : * happen _before_ what follows.
105 : : */
106 [ # # ][ + + ]: 12496035 : if (rwsem_is_locked(&anon_vma->root->rwsem)) {
107 : : anon_vma_lock_write(anon_vma);
108 : : anon_vma_unlock_write(anon_vma);
109 : : }
110 : :
111 : 12496181 : kmem_cache_free(anon_vma_cachep, anon_vma);
112 : : }
113 : :
114 : : static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
115 : : {
116 : 25297648 : return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
117 : : }
118 : :
119 : : static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
120 : : {
121 : 25297391 : kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
122 : : }
123 : :
124 : : static void anon_vma_chain_link(struct vm_area_struct *vma,
125 : : struct anon_vma_chain *avc,
126 : : struct anon_vma *anon_vma)
127 : : {
128 : 12586049 : avc->vma = vma;
129 : 12586049 : avc->anon_vma = anon_vma;
130 : 12586049 : list_add(&avc->same_vma, &vma->anon_vma_chain);
131 : 25297734 : anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
132 : : }
133 : :
134 : : /**
135 : : * anon_vma_prepare - attach an anon_vma to a memory region
136 : : * @vma: the memory region in question
137 : : *
138 : : * This makes sure the memory mapping described by 'vma' has
139 : : * an 'anon_vma' attached to it, so that we can associate the
140 : : * anonymous pages mapped into it with that anon_vma.
141 : : *
142 : : * The common case will be that we already have one, but if
143 : : * not we either need to find an adjacent mapping that we
144 : : * can re-use the anon_vma from (very common when the only
145 : : * reason for splitting a vma has been mprotect()), or we
146 : : * allocate a new one.
147 : : *
148 : : * Anon-vma allocations are very subtle, because we may have
149 : : * optimistically looked up an anon_vma in page_lock_anon_vma_read()
150 : : * and that may actually touch the spinlock even in the newly
151 : : * allocated vma (it depends on RCU to make sure that the
152 : : * anon_vma isn't actually destroyed).
153 : : *
154 : : * As a result, we need to do proper anon_vma locking even
155 : : * for the new allocation. At the same time, we do not want
156 : : * to do any locking for the common case of already having
157 : : * an anon_vma.
158 : : *
159 : : * This must be called with the mmap_sem held for reading.
160 : : */
161 : 0 : int anon_vma_prepare(struct vm_area_struct *vma)
162 : : {
163 : 604762 : struct anon_vma *anon_vma = vma->anon_vma;
164 : : struct anon_vma_chain *avc;
165 : :
166 : : might_sleep();
167 [ + + ]: 20047961 : if (unlikely(!anon_vma)) {
168 : 302387 : struct mm_struct *mm = vma->vm_mm;
169 : : struct anon_vma *allocated;
170 : :
171 : : avc = anon_vma_chain_alloc(GFP_KERNEL);
172 [ + - ]: 302384 : if (!avc)
173 : : goto out_enomem;
174 : :
175 : 302384 : anon_vma = find_mergeable_anon_vma(vma);
176 : : allocated = NULL;
177 [ + + ]: 302377 : if (!anon_vma) {
178 : : anon_vma = anon_vma_alloc();
179 [ + - ]: 300285 : if (unlikely(!anon_vma))
180 : : goto out_enomem_free_avc;
181 : : allocated = anon_vma;
182 : : }
183 : :
184 : : anon_vma_lock_write(anon_vma);
185 : : /* page_table_lock to protect against threads */
186 : : spin_lock(&mm->page_table_lock);
187 [ + + ]: 302386 : if (likely(!vma->anon_vma)) {
188 : 302377 : vma->anon_vma = anon_vma;
189 : : anon_vma_chain_link(vma, avc, anon_vma);
190 : : allocated = NULL;
191 : : avc = NULL;
192 : : }
193 : : spin_unlock(&mm->page_table_lock);
194 : : anon_vma_unlock_write(anon_vma);
195 : :
196 [ + + ]: 302392 : if (unlikely(allocated))
197 : : put_anon_vma(allocated);
198 [ + + ]: 302386 : if (unlikely(avc))
199 : : anon_vma_chain_free(avc);
200 : : }
201 : : return 0;
202 : :
203 : : out_enomem_free_avc:
204 : : anon_vma_chain_free(avc);
205 : : out_enomem:
206 : : return -ENOMEM;
207 : : }
208 : :
209 : : /*
210 : : * This is a useful helper function for locking the anon_vma root as
211 : : * we traverse the vma->anon_vma_chain, looping over anon_vma's that
212 : : * have the same vma.
213 : : *
214 : : * Such anon_vma's should have the same root, so you'd expect to see
215 : : * just a single mutex_lock for the whole traversal.
216 : : */
217 : : static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
218 : : {
219 : : struct anon_vma *new_root = anon_vma->root;
220 [ + + ][ + + ]: 38095789 : if (new_root != root) {
221 [ - + ][ # # ]: 25211029 : if (WARN_ON_ONCE(root))
[ - + ][ - + ]
[ - + ][ # # ]
[ - ][ - + ]
222 : 0 : up_write(&root->rwsem);
223 : : root = new_root;
224 : 25211081 : down_write(&root->rwsem);
225 : : }
226 : : return root;
227 : : }
228 : :
229 : : static inline void unlock_anon_vma_root(struct anon_vma *root)
230 : : {
231 [ + + ][ # # ]: 33019824 : if (root)
[ + + ]
232 : 25211484 : up_write(&root->rwsem);
233 : : }
234 : :
235 : : /*
236 : : * Attach the anon_vmas from src to dst.
237 : : * Returns 0 on success, -ENOMEM on failure.
238 : : */
239 : 0 : int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
240 : : {
241 : : struct anon_vma_chain *avc, *pavc;
242 : : struct anon_vma *root = NULL;
243 : :
244 [ + + ]: 25511059 : list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
245 : 12799439 : struct anon_vma *anon_vma;
246 : :
247 : : avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
248 [ - + ]: 12799439 : if (unlikely(!avc)) {
249 : : unlock_anon_vma_root(root);
250 : : root = NULL;
251 : : avc = anon_vma_chain_alloc(GFP_KERNEL);
252 [ # # ]: 0 : if (!avc)
253 : : goto enomem_failure;
254 : : }
255 : 12799439 : anon_vma = pavc->anon_vma;
256 : : root = lock_anon_vma_root(root, anon_vma);
257 : : anon_vma_chain_link(dst, avc, anon_vma);
258 : : }
259 : : unlock_anon_vma_root(root);
260 : : return 0;
261 : :
262 : : enomem_failure:
263 : 0 : unlink_anon_vmas(dst);
264 : 0 : return -ENOMEM;
265 : : }
266 : :
267 : : /*
268 : : * Attach vma to its own anon_vma, as well as to the anon_vmas that
269 : : * the corresponding VMA in the parent process is attached to.
270 : : * Returns 0 on success, non-zero on failure.
271 : : */
272 : 0 : int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
273 : : {
274 : : struct anon_vma_chain *avc;
275 : 24391804 : struct anon_vma *anon_vma;
276 : :
277 : : /* Don't bother if the parent process has no anon_vma here. */
278 [ + + ]: 18090179 : if (!pvma->anon_vma)
279 : : return 0;
280 : :
281 : : /*
282 : : * First, attach the new VMA to the parent VMA's anon_vmas,
283 : : * so rmap can find non-COWed pages in child processes.
284 : : */
285 [ + ]: 12195906 : if (anon_vma_clone(vma, pvma))
286 : : return -ENOMEM;
287 : :
288 : : /* Then add our own anon_vma. */
289 : : anon_vma = anon_vma_alloc();
290 [ + - ]: 12195903 : if (!anon_vma)
291 : : goto out_error;
292 : : avc = anon_vma_chain_alloc(GFP_KERNEL);
293 [ + - ]: 12195924 : if (!avc)
294 : : goto out_error_free_anon_vma;
295 : :
296 : : /*
297 : : * The root anon_vma's spinlock is the lock actually used when we
298 : : * lock any of the anon_vmas in this anon_vma tree.
299 : : */
300 : 12195924 : anon_vma->root = pvma->anon_vma->root;
301 : : /*
302 : : * With refcounts, an anon_vma can stay around longer than the
303 : : * process it belongs to. The root anon_vma needs to be pinned until
304 : : * this anon_vma is freed, because the lock lives in the root.
305 : : */
306 : : get_anon_vma(anon_vma->root);
307 : : /* Mark this anon_vma as the one where our new (COWed) pages go. */
308 : 12195922 : vma->anon_vma = anon_vma;
309 : : anon_vma_lock_write(anon_vma);
310 : : anon_vma_chain_link(vma, avc, anon_vma);
311 : : anon_vma_unlock_write(anon_vma);
312 : :
313 : 12195912 : return 0;
314 : :
315 : : out_error_free_anon_vma:
316 : : put_anon_vma(anon_vma);
317 : : out_error:
318 : 0 : unlink_anon_vmas(vma);
319 : 0 : return -ENOMEM;
320 : : }
321 : :
322 : 0 : void unlink_anon_vmas(struct vm_area_struct *vma)
323 : : {
324 : : struct anon_vma_chain *avc, *next;
325 : : struct anon_vma *root = NULL;
326 : :
327 : : /*
328 : : * Unlink each anon_vma chained to the VMA. This list is ordered
329 : : * from newest to oldest, ensuring the root anon_vma gets freed last.
330 : : */
331 [ + + ]: 45604473 : list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
332 : 25296350 : struct anon_vma *anon_vma = avc->anon_vma;
333 : :
334 : : root = lock_anon_vma_root(root, anon_vma);
335 : 25296844 : anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
336 : :
337 : : /*
338 : : * Leave empty anon_vmas on the list - we'll need
339 : : * to free them outside the lock.
340 : : */
341 [ + + ]: 25297423 : if (RB_EMPTY_ROOT(&anon_vma->rb_root))
342 : 12496143 : continue;
343 : :
344 : : list_del(&avc->same_vma);
345 : : anon_vma_chain_free(avc);
346 : : }
347 : : unlock_anon_vma_root(root);
348 : :
349 : : /*
350 : : * Iterate the list once more, it now only contains empty and unlinked
351 : : * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
352 : : * needing to write-acquire the anon_vma->root->rwsem.
353 : : */
354 [ + + ]: 53111917 : list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
355 : 12496134 : struct anon_vma *anon_vma = avc->anon_vma;
356 : :
357 : : put_anon_vma(anon_vma);
358 : :
359 : : list_del(&avc->same_vma);
360 : : anon_vma_chain_free(avc);
361 : : }
362 : 20308152 : }
363 : :
364 : 0 : static void anon_vma_ctor(void *data)
365 : : {
366 : : struct anon_vma *anon_vma = data;
367 : :
368 : 16159 : init_rwsem(&anon_vma->rwsem);
369 : 16159 : atomic_set(&anon_vma->refcount, 0);
370 : 16159 : anon_vma->rb_root = RB_ROOT;
371 : 16159 : }
372 : :
373 : 0 : void __init anon_vma_init(void)
374 : : {
375 : 0 : anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
376 : : 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor);
377 : 0 : anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC);
378 : 0 : }
379 : :
380 : : /*
381 : : * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
382 : : *
383 : : * Since there is no serialization what so ever against page_remove_rmap()
384 : : * the best this function can do is return a locked anon_vma that might
385 : : * have been relevant to this page.
386 : : *
387 : : * The page might have been remapped to a different anon_vma or the anon_vma
388 : : * returned may already be freed (and even reused).
389 : : *
390 : : * In case it was remapped to a different anon_vma, the new anon_vma will be a
391 : : * child of the old anon_vma, and the anon_vma lifetime rules will therefore
392 : : * ensure that any anon_vma obtained from the page will still be valid for as
393 : : * long as we observe page_mapped() [ hence all those page_mapped() tests ].
394 : : *
395 : : * All users of this function must be very careful when walking the anon_vma
396 : : * chain and verify that the page in question is indeed mapped in it
397 : : * [ something equivalent to page_mapped_in_vma() ].
398 : : *
399 : : * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap()
400 : : * that the anon_vma pointer from page->mapping is valid if there is a
401 : : * mapcount, we can dereference the anon_vma after observing those.
402 : : */
403 : 0 : struct anon_vma *page_get_anon_vma(struct page *page)
404 : : {
405 : : struct anon_vma *anon_vma = NULL;
406 : : unsigned long anon_mapping;
407 : :
408 : : rcu_read_lock();
409 : 0 : anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
410 [ # # ]: 0 : if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
411 : : goto out;
412 [ # # ]: 0 : if (!page_mapped(page))
413 : : goto out;
414 : :
415 : 0 : anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
416 [ # # ]: 0 : if (!atomic_inc_not_zero(&anon_vma->refcount)) {
417 : : anon_vma = NULL;
418 : : goto out;
419 : : }
420 : :
421 : : /*
422 : : * If this page is still mapped, then its anon_vma cannot have been
423 : : * freed. But if it has been unmapped, we have no security against the
424 : : * anon_vma structure being freed and reused (for another anon_vma:
425 : : * SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero()
426 : : * above cannot corrupt).
427 : : */
428 [ # # ]: 0 : if (!page_mapped(page)) {
429 : : put_anon_vma(anon_vma);
430 : : anon_vma = NULL;
431 : : }
432 : : out:
433 : : rcu_read_unlock();
434 : :
435 : 0 : return anon_vma;
436 : : }
437 : :
438 : : /*
439 : : * Similar to page_get_anon_vma() except it locks the anon_vma.
440 : : *
441 : : * Its a little more complex as it tries to keep the fast path to a single
442 : : * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
443 : : * reference like with page_get_anon_vma() and then block on the mutex.
444 : : */
445 : 0 : struct anon_vma *page_lock_anon_vma_read(struct page *page)
446 : : {
447 : 0 : struct anon_vma *anon_vma = NULL;
448 : : struct anon_vma *root_anon_vma;
449 : : unsigned long anon_mapping;
450 : :
451 : : rcu_read_lock();
452 : 1 : anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
453 [ + - ]: 1 : if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
454 : : goto out;
455 [ + - ]: 1 : if (!page_mapped(page))
456 : : goto out;
457 : :
458 : 1 : anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
459 : 1 : root_anon_vma = ACCESS_ONCE(anon_vma->root);
460 [ + - ]: 1 : if (down_read_trylock(&root_anon_vma->rwsem)) {
461 : : /*
462 : : * If the page is still mapped, then this anon_vma is still
463 : : * its anon_vma, and holding the mutex ensures that it will
464 : : * not go away, see anon_vma_free().
465 : : */
466 [ - + ]: 1 : if (!page_mapped(page)) {
467 : 0 : up_read(&root_anon_vma->rwsem);
468 : : anon_vma = NULL;
469 : : }
470 : : goto out;
471 : : }
472 : :
473 : : /* trylock failed, we got to sleep */
474 [ # # ]: 0 : if (!atomic_inc_not_zero(&anon_vma->refcount)) {
475 : : anon_vma = NULL;
476 : : goto out;
477 : : }
478 : :
479 [ # # ]: 0 : if (!page_mapped(page)) {
480 : : put_anon_vma(anon_vma);
481 : : anon_vma = NULL;
482 : : goto out;
483 : : }
484 : :
485 : : /* we pinned the anon_vma, its safe to sleep */
486 : : rcu_read_unlock();
487 : : anon_vma_lock_read(anon_vma);
488 : :
489 [ # # ]: 0 : if (atomic_dec_and_test(&anon_vma->refcount)) {
490 : : /*
491 : : * Oops, we held the last refcount, release the lock
492 : : * and bail -- can't simply use put_anon_vma() because
493 : : * we'll deadlock on the anon_vma_lock_write() recursion.
494 : : */
495 : : anon_vma_unlock_read(anon_vma);
496 : 0 : __put_anon_vma(anon_vma);
497 : : anon_vma = NULL;
498 : : }
499 : :
500 : 0 : return anon_vma;
501 : :
502 : : out:
503 : : rcu_read_unlock();
504 : 1 : return anon_vma;
505 : : }
506 : :
507 : 0 : void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
508 : : {
509 : : anon_vma_unlock_read(anon_vma);
510 : 0 : }
511 : :
512 : : /*
513 : : * At what user virtual address is page expected in @vma?
514 : : */
515 : : static inline unsigned long
516 : : __vma_address(struct page *page, struct vm_area_struct *vma)
517 : : {
518 : : pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
519 : :
520 : : if (unlikely(is_vm_hugetlb_page(vma)))
521 : : pgoff = page->index << huge_page_order(page_hstate(page));
522 : :
523 : 305119 : return vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
524 : : }
525 : :
526 : : inline unsigned long
527 : 0 : vma_address(struct page *page, struct vm_area_struct *vma)
528 : : {
529 : : unsigned long address = __vma_address(page, vma);
530 : :
531 : : /* page should be within @vma mapping range */
532 : : VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
533 : :
534 : 0 : return address;
535 : : }
536 : :
537 : : /*
538 : : * At what user virtual address is page expected in vma?
539 : : * Caller should check the page is actually part of the vma.
540 : : */
541 : 0 : unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
542 : : {
543 : : unsigned long address;
544 [ # # ]: 0 : if (PageAnon(page)) {
545 : : struct anon_vma *page__anon_vma = page_anon_vma(page);
546 : : /*
547 : : * Note: swapoff's unuse_vma() is more efficient with this
548 : : * check, and needs it to match anon_vma when KSM is active.
549 : : */
550 [ # # ][ # # ]: 0 : if (!vma->anon_vma || !page__anon_vma ||
[ # # ]
551 : 0 : vma->anon_vma->root != page__anon_vma->root)
552 : : return -EFAULT;
553 [ # # ][ # # ]: 0 : } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
554 [ # # ][ # # ]: 0 : if (!vma->vm_file ||
555 : 0 : vma->vm_file->f_mapping != page->mapping)
556 : : return -EFAULT;
557 : : } else
558 : : return -EFAULT;
559 : : address = __vma_address(page, vma);
560 [ # # ][ # # ]: 0 : if (unlikely(address < vma->vm_start || address >= vma->vm_end))
561 : : return -EFAULT;
562 : 0 : return address;
563 : : }
564 : :
565 : 0 : pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
566 : : {
567 : : pgd_t *pgd;
568 : : pud_t *pud;
569 : : pmd_t *pmd = NULL;
570 : :
571 : 265427 : pgd = pgd_offset(mm, address);
572 : : if (!pgd_present(*pgd))
573 : : goto out;
574 : :
575 : : pud = pud_offset(pgd, address);
576 : : if (!pud_present(*pud))
577 : : goto out;
578 : :
579 : : pmd = pmd_offset(pud, address);
580 [ # # ][ + + ]: 265427 : if (!pmd_present(*pmd))
[ # # ]
581 : : pmd = NULL;
582 : : out:
583 : 0 : return pmd;
584 : : }
585 : :
586 : : /*
587 : : * Check that @page is mapped at @address into @mm.
588 : : *
589 : : * If @sync is false, page_check_address may perform a racy check to avoid
590 : : * the page table lock when the pte is not present (helpful when reclaiming
591 : : * highly shared pages).
592 : : *
593 : : * On success returns with pte mapped and locked.
594 : : */
595 : 0 : pte_t *__page_check_address(struct page *page, struct mm_struct *mm,
596 : : unsigned long address, spinlock_t **ptlp, int sync)
597 : : {
598 : 172032 : pmd_t *pmd;
599 : : pte_t *pte;
600 : : spinlock_t *ptl;
601 : :
602 : : if (unlikely(PageHuge(page))) {
603 : : /* when pud is not present, pte will be NULL */
604 : : pte = huge_pte_offset(mm, address);
605 : : if (!pte)
606 : : return NULL;
607 : :
608 : : ptl = huge_pte_lockptr(page_hstate(page), mm, pte);
609 : : goto check;
610 : : }
611 : :
612 : : pmd = mm_find_pmd(mm, address);
613 [ + + ]: 265427 : if (!pmd)
614 : : return NULL;
615 : :
616 : : if (pmd_trans_huge(*pmd))
617 : : return NULL;
618 : :
619 : 221415 : pte = pte_offset_map(pmd, address);
620 : : /* Make a quick check before getting the lock */
621 [ + + ][ + + ]: 221415 : if (!sync && !pte_present(*pte)) {
622 : 49383 : pte_unmap(pte);
623 : 49383 : return NULL;
624 : : }
625 : :
626 : 172032 : ptl = pte_lockptr(mm, pmd);
627 : : check:
628 : : spin_lock(ptl);
629 [ + + ][ + + ]: 172032 : if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) {
630 : 158678 : *ptlp = ptl;
631 : 158678 : return pte;
632 : : }
633 : 13354 : pte_unmap_unlock(pte, ptl);
634 : 13354 : return NULL;
635 : : }
636 : :
637 : : /**
638 : : * page_mapped_in_vma - check whether a page is really mapped in a VMA
639 : : * @page: the page to test
640 : : * @vma: the VMA to test
641 : : *
642 : : * Returns 1 if the page is mapped into the page tables of the VMA, 0
643 : : * if the page is not mapped into the page tables of this VMA. Only
644 : : * valid for normal file or anonymous VMAs.
645 : : */
646 : 0 : int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
647 : : {
648 : : unsigned long address;
649 : : pte_t *pte;
650 : : spinlock_t *ptl;
651 : :
652 : : address = __vma_address(page, vma);
653 [ # # ][ # # ]: 0 : if (unlikely(address < vma->vm_start || address >= vma->vm_end))
654 : : return 0;
655 : 0 : pte = page_check_address(page, vma->vm_mm, address, &ptl, 1);
656 [ # # ]: 0 : if (!pte) /* the page is not in this mm */
657 : : return 0;
658 : 0 : pte_unmap_unlock(pte, ptl);
659 : :
660 : 0 : return 1;
661 : : }
662 : :
663 : : struct page_referenced_arg {
664 : : int mapcount;
665 : : int referenced;
666 : : unsigned long vm_flags;
667 : : struct mem_cgroup *memcg;
668 : : };
669 : : /*
670 : : * arg: page_referenced_arg will be passed
671 : : */
672 : 0 : int page_referenced_one(struct page *page, struct vm_area_struct *vma,
673 : : unsigned long address, void *arg)
674 : : {
675 : 40779 : struct mm_struct *mm = vma->vm_mm;
676 : : spinlock_t *ptl;
677 : : int referenced = 0;
678 : : struct page_referenced_arg *pra = arg;
679 : :
680 : : if (unlikely(PageTransHuge(page))) {
681 : : pmd_t *pmd;
682 : :
683 : : /*
684 : : * rmap might return false positives; we must filter
685 : : * these out using page_check_address_pmd().
686 : : */
687 : : pmd = page_check_address_pmd(page, mm, address,
688 : : PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl);
689 : : if (!pmd)
690 : : return SWAP_AGAIN;
691 : :
692 : : if (vma->vm_flags & VM_LOCKED) {
693 : : spin_unlock(ptl);
694 : : pra->vm_flags |= VM_LOCKED;
695 : : return SWAP_FAIL; /* To break the loop */
696 : : }
697 : :
698 : : /* go ahead even if the pmd is pmd_trans_splitting() */
699 : : if (pmdp_clear_flush_young_notify(vma, address, pmd))
700 : : referenced++;
701 : : spin_unlock(ptl);
702 : : } else {
703 : : pte_t *pte;
704 : :
705 : : /*
706 : : * rmap might return false positives; we must filter
707 : : * these out using page_check_address().
708 : : */
709 : : pte = page_check_address(page, mm, address, &ptl, 0);
710 [ + + ]: 40779 : if (!pte)
711 : : return SWAP_AGAIN;
712 : :
713 [ - + ]: 16936 : if (vma->vm_flags & VM_LOCKED) {
714 : 0 : pte_unmap_unlock(pte, ptl);
715 : 0 : pra->vm_flags |= VM_LOCKED;
716 : 0 : return SWAP_FAIL; /* To break the loop */
717 : : }
718 : :
719 [ + + ]: 16936 : if (ptep_clear_flush_young_notify(vma, address, pte)) {
720 : : /*
721 : : * Don't treat a reference through a sequentially read
722 : : * mapping as such. If the page has been used in
723 : : * another mapping, we will catch it; if this other
724 : : * mapping is already gone, the unmap path will have
725 : : * set PG_referenced or activated the page.
726 : : */
727 [ + - ]: 5394 : if (likely(!(vma->vm_flags & VM_SEQ_READ)))
728 : : referenced++;
729 : : }
730 : 16936 : pte_unmap_unlock(pte, ptl);
731 : : }
732 : :
733 [ + + ]: 16936 : if (referenced) {
734 : 5394 : pra->referenced++;
735 : 5394 : pra->vm_flags |= vma->vm_flags;
736 : : }
737 : :
738 : 16936 : pra->mapcount--;
739 [ + + ]: 16936 : if (!pra->mapcount)
740 : : return SWAP_SUCCESS; /* To break the loop */
741 : :
742 : 12232 : return SWAP_AGAIN;
743 : : }
744 : :
745 : 0 : static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
746 : : {
747 : : struct page_referenced_arg *pra = arg;
748 : : struct mem_cgroup *memcg = pra->memcg;
749 : :
750 : : if (!mm_match_cgroup(vma->vm_mm, memcg))
751 : : return true;
752 : :
753 : : return false;
754 : : }
755 : :
756 : : /**
757 : : * page_referenced - test if the page was referenced
758 : : * @page: the page to test
759 : : * @is_locked: caller holds lock on the page
760 : : * @memcg: target memory cgroup
761 : : * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
762 : : *
763 : : * Quick test_and_clear_referenced for all mappings to a page,
764 : : * returns the number of ptes which referenced the page.
765 : : */
766 : 0 : int page_referenced(struct page *page,
767 : : int is_locked,
768 : : struct mem_cgroup *memcg,
769 : : unsigned long *vm_flags)
770 : : {
771 : : int ret;
772 : : int we_locked = 0;
773 : 294558 : struct page_referenced_arg pra = {
774 : : .mapcount = page_mapcount(page),
775 : : .memcg = memcg,
776 : : };
777 : 147279 : struct rmap_walk_control rwc = {
778 : : .rmap_one = page_referenced_one,
779 : : .arg = (void *)&pra,
780 : : .anon_lock = page_lock_anon_vma_read,
781 : : };
782 : :
783 : 147279 : *vm_flags = 0;
784 [ + + ]: 147279 : if (!page_mapped(page))
785 : : return 0;
786 : :
787 [ + - ]: 4704 : if (!page_rmapping(page))
788 : : return 0;
789 : :
790 [ + + ][ + - ]: 4704 : if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
791 : : we_locked = trylock_page(page);
792 [ + - ]: 2431 : if (!we_locked)
793 : : return 1;
794 : : }
795 : :
796 : : /*
797 : : * If we are reclaiming on behalf of a cgroup, skip
798 : : * counting on behalf of references from different
799 : : * cgroups
800 : : */
801 [ - + ]: 4704 : if (memcg) {
802 : 0 : rwc.invalid_vma = invalid_page_referenced_vma;
803 : : }
804 : :
805 : 4704 : ret = rmap_walk(page, &rwc);
806 : 4704 : *vm_flags = pra.vm_flags;
807 : :
808 [ + + ]: 4704 : if (we_locked)
809 : 2431 : unlock_page(page);
810 : :
811 : 4704 : return pra.referenced;
812 : : }
813 : :
814 : 0 : static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
815 : : unsigned long address, void *arg)
816 : : {
817 : 165146 : struct mm_struct *mm = vma->vm_mm;
818 : : pte_t *pte;
819 : : spinlock_t *ptl;
820 : : int ret = 0;
821 : : int *cleaned = arg;
822 : :
823 : : pte = page_check_address(page, mm, address, &ptl, 1);
824 [ + + ]: 165146 : if (!pte)
825 : : goto out;
826 : :
827 [ + + ]: 108105 : if (pte_dirty(*pte) || pte_write(*pte)) {
828 : : pte_t entry;
829 : :
830 : 96258 : flush_cache_page(vma, address, pte_pfn(*pte));
831 : 96258 : entry = ptep_clear_flush(vma, address, pte);
832 : : entry = pte_wrprotect(entry);
833 : : entry = pte_mkclean(entry);
834 : : set_pte_at(mm, address, pte, entry);
835 : : ret = 1;
836 : : }
837 : :
838 : 108105 : pte_unmap_unlock(pte, ptl);
839 : :
840 [ + + ]: 108105 : if (ret) {
841 : : mmu_notifier_invalidate_page(mm, address);
842 : 96258 : (*cleaned)++;
843 : : }
844 : : out:
845 : 165146 : return SWAP_AGAIN;
846 : : }
847 : :
848 : 0 : static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
849 : : {
850 [ + + ]: 204839 : if (vma->vm_flags & VM_SHARED)
851 : : return false;
852 : :
853 : 39693 : return true;
854 : : }
855 : :
856 : 0 : int page_mkclean(struct page *page)
857 : : {
858 : 958620 : int cleaned = 0;
859 : : struct address_space *mapping;
860 : 958620 : struct rmap_walk_control rwc = {
861 : : .arg = (void *)&cleaned,
862 : : .rmap_one = page_mkclean_one,
863 : : .invalid_vma = invalid_mkclean_vma,
864 : : };
865 : :
866 [ - + ]: 958620 : BUG_ON(!PageLocked(page));
867 : :
868 [ + + ]: 958620 : if (!page_mapped(page))
869 : : return 0;
870 : :
871 : 110707 : mapping = page_mapping(page);
872 [ + - ]: 110707 : if (!mapping)
873 : : return 0;
874 : :
875 : 110707 : rmap_walk(page, &rwc);
876 : :
877 : 110707 : return cleaned;
878 : : }
879 : : EXPORT_SYMBOL_GPL(page_mkclean);
880 : :
881 : : /**
882 : : * page_move_anon_rmap - move a page to our anon_vma
883 : : * @page: the page to move to our anon_vma
884 : : * @vma: the vma the page belongs to
885 : : * @address: the user virtual address mapped
886 : : *
887 : : * When a page belongs exclusively to one process after a COW event,
888 : : * that page can be moved into the anon_vma that belongs to just that
889 : : * process, so the rmap code will not search the parent or sibling
890 : : * processes.
891 : : */
892 : 0 : void page_move_anon_rmap(struct page *page,
893 : : struct vm_area_struct *vma, unsigned long address)
894 : : {
895 : 6947285 : struct anon_vma *anon_vma = vma->anon_vma;
896 : :
897 : : VM_BUG_ON_PAGE(!PageLocked(page), page);
898 : : VM_BUG_ON(!anon_vma);
899 : : VM_BUG_ON_PAGE(page->index != linear_page_index(vma, address), page);
900 : :
901 : 6947285 : anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
902 : 6947285 : page->mapping = (struct address_space *) anon_vma;
903 : 6947285 : }
904 : :
905 : : /**
906 : : * __page_set_anon_rmap - set up new anonymous rmap
907 : : * @page: Page to add to rmap
908 : : * @vma: VM area to add page to.
909 : : * @address: User virtual address of the mapping
910 : : * @exclusive: the page is exclusively owned by the current process
911 : : */
912 : 0 : static void __page_set_anon_rmap(struct page *page,
913 : 18032676 : struct vm_area_struct *vma, unsigned long address, int exclusive)
914 : : {
915 : 18032284 : struct anon_vma *anon_vma = vma->anon_vma;
916 : :
917 [ - + ]: 18032284 : BUG_ON(!anon_vma);
918 : :
919 [ + ]: 18032284 : if (PageAnon(page))
920 : 0 : return;
921 : :
922 : : /*
923 : : * If the page isn't exclusively mapped into this vma,
924 : : * we must use the _oldest_ possible anon_vma for the
925 : : * page mapping!
926 : : */
927 [ - + ]: 36064960 : if (!exclusive)
928 : 0 : anon_vma = anon_vma->root;
929 : :
930 : 18032676 : anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
931 : 18032676 : page->mapping = (struct address_space *) anon_vma;
932 : 18032676 : page->index = linear_page_index(vma, address);
933 : : }
934 : :
935 : : /**
936 : : * __page_check_anon_rmap - sanity check anonymous rmap addition
937 : : * @page: the page to add the mapping to
938 : : * @vma: the vm area in which the mapping is added
939 : : * @address: the user virtual address mapped
940 : : */
941 : : static void __page_check_anon_rmap(struct page *page,
942 : : struct vm_area_struct *vma, unsigned long address)
943 : : {
944 : : #ifdef CONFIG_DEBUG_VM
945 : : /*
946 : : * The page's anon-rmap details (mapping and index) are guaranteed to
947 : : * be set up correctly at this point.
948 : : *
949 : : * We have exclusion against page_add_anon_rmap because the caller
950 : : * always holds the page locked, except if called from page_dup_rmap,
951 : : * in which case the page is already known to be setup.
952 : : *
953 : : * We have exclusion against page_add_new_anon_rmap because those pages
954 : : * are initially only visible via the pagetables, and the pte is locked
955 : : * over the call to page_add_new_anon_rmap.
956 : : */
957 : : BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
958 : : BUG_ON(page->index != linear_page_index(vma, address));
959 : : #endif
960 : : }
961 : :
962 : : /**
963 : : * page_add_anon_rmap - add pte mapping to an anonymous page
964 : : * @page: the page to add the mapping to
965 : : * @vma: the vm area in which the mapping is added
966 : : * @address: the user virtual address mapped
967 : : *
968 : : * The caller needs to hold the pte lock, and the page must be locked in
969 : : * the anon_vma case: to serialize mapping,index checking after setting,
970 : : * and to ensure that PageAnon is not being upgraded racily to PageKsm
971 : : * (but PageKsm is never downgraded to PageAnon).
972 : : */
973 : 0 : void page_add_anon_rmap(struct page *page,
974 : : struct vm_area_struct *vma, unsigned long address)
975 : : {
976 : 0 : do_page_add_anon_rmap(page, vma, address, 0);
977 : 0 : }
978 : :
979 : : /*
980 : : * Special version of the above for do_swap_page, which often runs
981 : : * into pages that are exclusively owned by the current process.
982 : : * Everybody else should continue to use page_add_anon_rmap above.
983 : : */
984 : 0 : void do_page_add_anon_rmap(struct page *page,
985 : : struct vm_area_struct *vma, unsigned long address, int exclusive)
986 : : {
987 : 0 : int first = atomic_inc_and_test(&page->_mapcount);
988 [ # # ]: 0 : if (first) {
989 : : if (PageTransHuge(page))
990 : : __inc_zone_page_state(page,
991 : : NR_ANON_TRANSPARENT_HUGEPAGES);
992 : 0 : __mod_zone_page_state(page_zone(page), NR_ANON_PAGES,
993 : : hpage_nr_pages(page));
994 : : }
995 : : if (unlikely(PageKsm(page)))
996 : 0 : return;
997 : :
998 : : VM_BUG_ON_PAGE(!PageLocked(page), page);
999 : : /* address might be in next vma when migration races vma_adjust */
1000 [ # # ]: 0 : if (first)
1001 : 0 : __page_set_anon_rmap(page, vma, address, exclusive);
1002 : : else
1003 : : __page_check_anon_rmap(page, vma, address);
1004 : : }
1005 : :
1006 : : /**
1007 : : * page_add_new_anon_rmap - add pte mapping to a new anonymous page
1008 : : * @page: the page to add the mapping to
1009 : : * @vma: the vm area in which the mapping is added
1010 : : * @address: the user virtual address mapped
1011 : : *
1012 : : * Same as page_add_anon_rmap but must only be called on *new* pages.
1013 : : * This means the inc-and-test can be bypassed.
1014 : : * Page does not have to be locked.
1015 : : */
1016 : 0 : void page_add_new_anon_rmap(struct page *page,
1017 : 18032732 : struct vm_area_struct *vma, unsigned long address)
1018 : : {
1019 : : VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
1020 : : SetPageSwapBacked(page);
1021 : 18032947 : atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */
1022 : : if (PageTransHuge(page))
1023 : : __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
1024 : 18032947 : __mod_zone_page_state(page_zone(page), NR_ANON_PAGES,
1025 : : hpage_nr_pages(page));
1026 : 18032906 : __page_set_anon_rmap(page, vma, address, 1);
1027 [ + + ]: 18032716 : if (!mlocked_vma_newpage(vma, page)) {
1028 : : SetPageActive(page);
1029 : 18026923 : lru_cache_add(page);
1030 : : } else
1031 : 6277 : add_page_to_unevictable_list(page);
1032 : 18032756 : }
1033 : :
1034 : : /**
1035 : : * page_add_file_rmap - add pte mapping to a file page
1036 : : * @page: the page to add the mapping to
1037 : : *
1038 : : * The caller needs to hold the pte lock.
1039 : : */
1040 : 0 : void page_add_file_rmap(struct page *page)
1041 : : {
1042 : : bool locked;
1043 : : unsigned long flags;
1044 : :
1045 : : mem_cgroup_begin_update_page_stat(page, &locked, &flags);
1046 [ + + ]: 30361127 : if (atomic_inc_and_test(&page->_mapcount)) {
1047 : 1136540 : __inc_zone_page_state(page, NR_FILE_MAPPED);
1048 : : mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
1049 : : }
1050 : : mem_cgroup_end_update_page_stat(page, &locked, &flags);
1051 : 0 : }
1052 : :
1053 : : /**
1054 : : * page_remove_rmap - take down pte mapping from a page
1055 : : * @page: page to remove mapping from
1056 : : *
1057 : : * The caller needs to hold the pte lock.
1058 : : */
1059 : 0 : void page_remove_rmap(struct page *page)
1060 : : {
1061 : : bool anon = PageAnon(page);
1062 : : bool locked;
1063 : : unsigned long flags;
1064 : :
1065 : : /*
1066 : : * The anon case has no mem_cgroup page_stat to update; but may
1067 : : * uncharge_page() below, where the lock ordering can deadlock if
1068 : : * we hold the lock against page_stat move: so avoid it on anon.
1069 : : */
1070 : : if (!anon)
1071 : : mem_cgroup_begin_update_page_stat(page, &locked, &flags);
1072 : :
1073 : : /* page still mapped by someone else? */
1074 [ + + ]: 71922676 : if (!atomic_add_negative(-1, &page->_mapcount))
1075 : : goto out;
1076 : :
1077 : : /*
1078 : : * Hugepages are not counted in NR_ANON_PAGES nor NR_FILE_MAPPED
1079 : : * and not charged by memcg for now.
1080 : : */
1081 : : if (unlikely(PageHuge(page)))
1082 : : goto out;
1083 [ + + ]: 19164409 : if (anon) {
1084 : : mem_cgroup_uncharge_page(page);
1085 : : if (PageTransHuge(page))
1086 : : __dec_zone_page_state(page,
1087 : : NR_ANON_TRANSPARENT_HUGEPAGES);
1088 : 18027587 : __mod_zone_page_state(page_zone(page), NR_ANON_PAGES,
1089 : : -hpage_nr_pages(page));
1090 : : } else {
1091 : 1136822 : __dec_zone_page_state(page, NR_FILE_MAPPED);
1092 : : mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
1093 : : mem_cgroup_end_update_page_stat(page, &locked, &flags);
1094 : : }
1095 [ + + ]: 19165237 : if (unlikely(PageMlocked(page)))
1096 : 1 : clear_page_mlock(page);
1097 : : /*
1098 : : * It would be tidy to reset the PageAnon mapping here,
1099 : : * but that might overwrite a racing page_add_anon_rmap
1100 : : * which increments mapcount after us but sets mapping
1101 : : * before us: so leave the reset to free_hot_cold_page,
1102 : : * and remember that it's only reliable while mapped.
1103 : : * Leaving it set also helps swapoff to reinstate ptes
1104 : : * faster for those pages still in swapcache.
1105 : : */
1106 : 0 : return;
1107 : : out:
1108 : : if (!anon)
1109 : : mem_cgroup_end_update_page_stat(page, &locked, &flags);
1110 : : }
1111 : :
1112 : : /*
1113 : : * @arg: enum ttu_flags will be passed to this argument
1114 : : */
1115 : 0 : int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
1116 : : unsigned long address, void *arg)
1117 : : {
1118 : 59502 : struct mm_struct *mm = vma->vm_mm;
1119 : : pte_t *pte;
1120 : : pte_t pteval;
1121 : : spinlock_t *ptl;
1122 : : int ret = SWAP_AGAIN;
1123 : 59502 : enum ttu_flags flags = (enum ttu_flags)arg;
1124 : :
1125 : : pte = page_check_address(page, mm, address, &ptl, 0);
1126 [ + + ]: 59502 : if (!pte)
1127 : : goto out;
1128 : :
1129 : : /*
1130 : : * If the page is mlock()d, we cannot swap it out.
1131 : : * If it's recently referenced (perhaps page_referenced
1132 : : * skipped over this mm) then we should reactivate it.
1133 : : */
1134 [ + - ]: 33637 : if (!(flags & TTU_IGNORE_MLOCK)) {
1135 [ + - ]: 33637 : if (vma->vm_flags & VM_LOCKED)
1136 : : goto out_mlock;
1137 : :
1138 [ + + ]: 33637 : if (TTU_ACTION(flags) == TTU_MUNLOCK)
1139 : : goto out_unmap;
1140 : : }
1141 [ + - ]: 5263 : if (!(flags & TTU_IGNORE_ACCESS)) {
1142 [ + - ]: 5263 : if (ptep_clear_flush_young_notify(vma, address, pte)) {
1143 : : ret = SWAP_FAIL;
1144 : : goto out_unmap;
1145 : : }
1146 : : }
1147 : :
1148 : : /* Nuke the page table entry. */
1149 : 5263 : flush_cache_page(vma, address, page_to_pfn(page));
1150 : 5263 : pteval = ptep_clear_flush(vma, address, pte);
1151 : :
1152 : : /* Move the dirty bit to the physical page now the pte is gone. */
1153 [ - + ]: 5263 : if (pte_dirty(pteval))
1154 : 0 : set_page_dirty(page);
1155 : :
1156 : : /* Update high watermark before we lower rss */
1157 : : update_hiwater_rss(mm);
1158 : :
1159 : : if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
1160 : : if (!PageHuge(page)) {
1161 : : if (PageAnon(page))
1162 : : dec_mm_counter(mm, MM_ANONPAGES);
1163 : : else
1164 : : dec_mm_counter(mm, MM_FILEPAGES);
1165 : : }
1166 : : set_pte_at(mm, address, pte,
1167 : : swp_entry_to_pte(make_hwpoison_entry(page)));
1168 [ - + ]: 64765 : } else if (PageAnon(page)) {
1169 : 0 : swp_entry_t entry = { .val = page_private(page) };
1170 : : pte_t swp_pte;
1171 : :
1172 [ # # ]: 0 : if (PageSwapCache(page)) {
1173 : : /*
1174 : : * Store the swap location in the pte.
1175 : : * See handle_pte_fault() ...
1176 : : */
1177 [ # # ]: 0 : if (swap_duplicate(entry) < 0) {
1178 : : set_pte_at(mm, address, pte, pteval);
1179 : : ret = SWAP_FAIL;
1180 : : goto out_unmap;
1181 : : }
1182 [ # # ]: 0 : if (list_empty(&mm->mmlist)) {
1183 : : spin_lock(&mmlist_lock);
1184 [ # # ]: 0 : if (list_empty(&mm->mmlist))
1185 : : list_add(&mm->mmlist, &init_mm.mmlist);
1186 : : spin_unlock(&mmlist_lock);
1187 : : }
1188 : : dec_mm_counter(mm, MM_ANONPAGES);
1189 : : inc_mm_counter(mm, MM_SWAPENTS);
1190 : : } else if (IS_ENABLED(CONFIG_MIGRATION)) {
1191 : : /*
1192 : : * Store the pfn of the page in a special migration
1193 : : * pte. do_swap_page() will wait until the migration
1194 : : * pte is removed and then restart fault handling.
1195 : : */
1196 [ # # ]: 0 : BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION);
1197 : 0 : entry = make_migration_entry(page, pte_write(pteval));
1198 : : }
1199 : : swp_pte = swp_entry_to_pte(entry);
1200 : : if (pte_soft_dirty(pteval))
1201 : : swp_pte = pte_swp_mksoft_dirty(swp_pte);
1202 : : set_pte_at(mm, address, pte, swp_pte);
1203 [ # # ]: 0 : BUG_ON(pte_file(*pte));
1204 [ - + ]: 5263 : } else if (IS_ENABLED(CONFIG_MIGRATION) &&
1205 : 5263 : (TTU_ACTION(flags) == TTU_MIGRATION)) {
1206 : : /* Establish migration entry for a file page */
1207 : : swp_entry_t entry;
1208 : 0 : entry = make_migration_entry(page, pte_write(pteval));
1209 : : set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
1210 : : } else
1211 : : dec_mm_counter(mm, MM_FILEPAGES);
1212 : :
1213 : 5263 : page_remove_rmap(page);
1214 : 5263 : page_cache_release(page);
1215 : :
1216 : : out_unmap:
1217 : 33637 : pte_unmap_unlock(pte, ptl);
1218 : : if (ret != SWAP_FAIL)
1219 : : mmu_notifier_invalidate_page(mm, address);
1220 : : out:
1221 : 59502 : return ret;
1222 : :
1223 : : out_mlock:
1224 : 0 : pte_unmap_unlock(pte, ptl);
1225 : :
1226 : :
1227 : : /*
1228 : : * We need mmap_sem locking, Otherwise VM_LOCKED check makes
1229 : : * unstable result and race. Plus, We can't wait here because
1230 : : * we now hold anon_vma->rwsem or mapping->i_mmap_mutex.
1231 : : * if trylock failed, the page remain in evictable lru and later
1232 : : * vmscan could retry to move the page to unevictable lru if the
1233 : : * page is actually mlocked.
1234 : : */
1235 [ # # ]: 0 : if (down_read_trylock(&vma->vm_mm->mmap_sem)) {
1236 [ # # ]: 0 : if (vma->vm_flags & VM_LOCKED) {
1237 : 0 : mlock_vma_page(page);
1238 : : ret = SWAP_MLOCK;
1239 : : }
1240 : 0 : up_read(&vma->vm_mm->mmap_sem);
1241 : : }
1242 : 0 : return ret;
1243 : : }
1244 : :
1245 : : /*
1246 : : * objrmap doesn't work for nonlinear VMAs because the assumption that
1247 : : * offset-into-file correlates with offset-into-virtual-addresses does not hold.
1248 : : * Consequently, given a particular page and its ->index, we cannot locate the
1249 : : * ptes which are mapping that page without an exhaustive linear search.
1250 : : *
1251 : : * So what this code does is a mini "virtual scan" of each nonlinear VMA which
1252 : : * maps the file to which the target page belongs. The ->vm_private_data field
1253 : : * holds the current cursor into that scan. Successive searches will circulate
1254 : : * around the vma's virtual address space.
1255 : : *
1256 : : * So as more replacement pressure is applied to the pages in a nonlinear VMA,
1257 : : * more scanning pressure is placed against them as well. Eventually pages
1258 : : * will become fully unmapped and are eligible for eviction.
1259 : : *
1260 : : * For very sparsely populated VMAs this is a little inefficient - chances are
1261 : : * there there won't be many ptes located within the scan cluster. In this case
1262 : : * maybe we could scan further - to the end of the pte page, perhaps.
1263 : : *
1264 : : * Mlocked pages: check VM_LOCKED under mmap_sem held for read, if we can
1265 : : * acquire it without blocking. If vma locked, mlock the pages in the cluster,
1266 : : * rather than unmapping them. If we encounter the "check_page" that vmscan is
1267 : : * trying to unmap, return SWAP_MLOCK, else default SWAP_AGAIN.
1268 : : */
1269 : : #define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE)
1270 : : #define CLUSTER_MASK (~(CLUSTER_SIZE - 1))
1271 : :
1272 : 0 : static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
1273 : 0 : struct vm_area_struct *vma, struct page *check_page)
1274 : : {
1275 : 0 : struct mm_struct *mm = vma->vm_mm;
1276 : 0 : pmd_t *pmd;
1277 : : pte_t *pte;
1278 : : pte_t pteval;
1279 : : spinlock_t *ptl;
1280 : 0 : struct page *page;
1281 : : unsigned long address;
1282 : : unsigned long mmun_start; /* For mmu_notifiers */
1283 : : unsigned long mmun_end; /* For mmu_notifiers */
1284 : : unsigned long end;
1285 : : int ret = SWAP_AGAIN;
1286 : : int locked_vma = 0;
1287 : :
1288 : 0 : address = (vma->vm_start + cursor) & CLUSTER_MASK;
1289 : 0 : end = address + CLUSTER_SIZE;
1290 [ # # ]: 0 : if (address < vma->vm_start)
1291 : : address = vma->vm_start;
1292 [ # # ]: 0 : if (end > vma->vm_end)
1293 : : end = vma->vm_end;
1294 : :
1295 : : pmd = mm_find_pmd(mm, address);
1296 [ # # ]: 0 : if (!pmd)
1297 : : return ret;
1298 : :
1299 : : mmun_start = address;
1300 : : mmun_end = end;
1301 : : mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1302 : :
1303 : : /*
1304 : : * If we can acquire the mmap_sem for read, and vma is VM_LOCKED,
1305 : : * keep the sem while scanning the cluster for mlocking pages.
1306 : : */
1307 [ # # ]: 0 : if (down_read_trylock(&vma->vm_mm->mmap_sem)) {
1308 : 0 : locked_vma = (vma->vm_flags & VM_LOCKED);
1309 [ # # ]: 0 : if (!locked_vma)
1310 : 0 : up_read(&vma->vm_mm->mmap_sem); /* don't need it */
1311 : : }
1312 : :
1313 : 0 : pte = pte_offset_map_lock(mm, pmd, address, &ptl);
1314 : :
1315 : : /* Update high watermark before we lower rss */
1316 : : update_hiwater_rss(mm);
1317 : :
1318 [ # # ]: 0 : for (; address < end; pte++, address += PAGE_SIZE) {
1319 [ # # ]: 0 : if (!pte_present(*pte))
1320 : 0 : continue;
1321 : 0 : page = vm_normal_page(vma, address, *pte);
1322 [ # # ][ # # ]: 0 : BUG_ON(!page || PageAnon(page));
1323 : :
1324 [ # # ]: 0 : if (locked_vma) {
1325 : 0 : mlock_vma_page(page); /* no-op if already mlocked */
1326 [ # # ]: 0 : if (page == check_page)
1327 : : ret = SWAP_MLOCK;
1328 : 0 : continue; /* don't unmap */
1329 : : }
1330 : :
1331 [ # # ]: 0 : if (ptep_clear_flush_young_notify(vma, address, pte))
1332 : 0 : continue;
1333 : :
1334 : : /* Nuke the page table entry. */
1335 : 0 : flush_cache_page(vma, address, pte_pfn(*pte));
1336 : 0 : pteval = ptep_clear_flush(vma, address, pte);
1337 : :
1338 : : /* If nonlinear, store the file page offset in the pte. */
1339 [ # # ]: 0 : if (page->index != linear_page_index(vma, address)) {
1340 : 0 : pte_t ptfile = pgoff_to_pte(page->index);
1341 : : if (pte_soft_dirty(pteval))
1342 : : pte_file_mksoft_dirty(ptfile);
1343 : : set_pte_at(mm, address, pte, ptfile);
1344 : : }
1345 : :
1346 : : /* Move the dirty bit to the physical page now the pte is gone. */
1347 [ # # ]: 0 : if (pte_dirty(pteval))
1348 : 0 : set_page_dirty(page);
1349 : :
1350 : 0 : page_remove_rmap(page);
1351 : 0 : page_cache_release(page);
1352 : : dec_mm_counter(mm, MM_FILEPAGES);
1353 : 0 : (*mapcount)--;
1354 : : }
1355 : 0 : pte_unmap_unlock(pte - 1, ptl);
1356 : : mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1357 [ # # ]: 0 : if (locked_vma)
1358 : 0 : up_read(&vma->vm_mm->mmap_sem);
1359 : 0 : return ret;
1360 : : }
1361 : :
1362 : 0 : static int try_to_unmap_nonlinear(struct page *page,
1363 : : struct address_space *mapping, void *arg)
1364 : : {
1365 : : struct vm_area_struct *vma;
1366 : : int ret = SWAP_AGAIN;
1367 : : unsigned long cursor;
1368 : : unsigned long max_nl_cursor = 0;
1369 : : unsigned long max_nl_size = 0;
1370 : : unsigned int mapcount;
1371 : :
1372 [ # # ]: 0 : list_for_each_entry(vma,
1373 : : &mapping->i_mmap_nonlinear, shared.nonlinear) {
1374 : :
1375 : 0 : cursor = (unsigned long) vma->vm_private_data;
1376 [ # # ]: 0 : if (cursor > max_nl_cursor)
1377 : : max_nl_cursor = cursor;
1378 : 0 : cursor = vma->vm_end - vma->vm_start;
1379 [ # # ]: 0 : if (cursor > max_nl_size)
1380 : : max_nl_size = cursor;
1381 : : }
1382 : :
1383 [ # # ]: 0 : if (max_nl_size == 0) { /* all nonlinears locked or reserved ? */
1384 : : return SWAP_FAIL;
1385 : : }
1386 : :
1387 : : /*
1388 : : * We don't try to search for this page in the nonlinear vmas,
1389 : : * and page_referenced wouldn't have found it anyway. Instead
1390 : : * just walk the nonlinear vmas trying to age and unmap some.
1391 : : * The mapcount of the page we came in with is irrelevant,
1392 : : * but even so use it as a guide to how hard we should try?
1393 : : */
1394 : 0 : mapcount = page_mapcount(page);
1395 [ # # ]: 0 : if (!mapcount)
1396 : : return ret;
1397 : :
1398 : 0 : cond_resched();
1399 : :
1400 : 0 : max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK;
1401 [ # # ]: 0 : if (max_nl_cursor == 0)
1402 : : max_nl_cursor = CLUSTER_SIZE;
1403 : :
1404 : : do {
1405 [ # # ]: 0 : list_for_each_entry(vma,
1406 : : &mapping->i_mmap_nonlinear, shared.nonlinear) {
1407 : :
1408 : 0 : cursor = (unsigned long) vma->vm_private_data;
1409 [ # # ][ # # ]: 0 : while (cursor < max_nl_cursor &&
1410 : 0 : cursor < vma->vm_end - vma->vm_start) {
1411 [ # # ]: 0 : if (try_to_unmap_cluster(cursor, &mapcount,
1412 : : vma, page) == SWAP_MLOCK)
1413 : : ret = SWAP_MLOCK;
1414 : 0 : cursor += CLUSTER_SIZE;
1415 : 0 : vma->vm_private_data = (void *) cursor;
1416 [ # # ]: 0 : if ((int)mapcount <= 0)
1417 : : return ret;
1418 : : }
1419 : 0 : vma->vm_private_data = (void *) max_nl_cursor;
1420 : : }
1421 : 0 : cond_resched();
1422 : 0 : max_nl_cursor += CLUSTER_SIZE;
1423 [ # # ]: 0 : } while (max_nl_cursor <= max_nl_size);
1424 : :
1425 : : /*
1426 : : * Don't loop forever (perhaps all the remaining pages are
1427 : : * in locked vmas). Reset cursor on all unreserved nonlinear
1428 : : * vmas, now forgetting on which ones it had fallen behind.
1429 : : */
1430 [ # # ]: 0 : list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.nonlinear)
1431 : 0 : vma->vm_private_data = NULL;
1432 : :
1433 : : return ret;
1434 : : }
1435 : :
1436 : 0 : bool is_vma_temporary_stack(struct vm_area_struct *vma)
1437 : : {
1438 : 0 : int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
1439 : :
1440 [ # # ][ # # ]: 0 : if (!maybe_stack)
1441 : : return false;
1442 : :
1443 [ # # ][ # # ]: 0 : if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
1444 : : VM_STACK_INCOMPLETE_SETUP)
1445 : : return true;
1446 : :
1447 : 0 : return false;
1448 : : }
1449 : :
1450 : 0 : static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
1451 : : {
1452 : 0 : return is_vma_temporary_stack(vma);
1453 : : }
1454 : :
1455 : 0 : static int page_not_mapped(struct page *page)
1456 : : {
1457 : 59502 : return !page_mapped(page);
1458 : : };
1459 : :
1460 : : /**
1461 : : * try_to_unmap - try to remove all page table mappings to a page
1462 : : * @page: the page to get unmapped
1463 : : * @flags: action and flags
1464 : : *
1465 : : * Tries to remove all the page table entries which are mapping this
1466 : : * page, used in the pageout path. Caller must hold the page lock.
1467 : : * Return values are:
1468 : : *
1469 : : * SWAP_SUCCESS - we succeeded in removing all mappings
1470 : : * SWAP_AGAIN - we missed a mapping, try again later
1471 : : * SWAP_FAIL - the page is unswappable
1472 : : * SWAP_MLOCK - page is mlocked.
1473 : : */
1474 : 0 : int try_to_unmap(struct page *page, enum ttu_flags flags)
1475 : : {
1476 : : int ret;
1477 : 3038 : struct rmap_walk_control rwc = {
1478 : : .rmap_one = try_to_unmap_one,
1479 : 1519 : .arg = (void *)flags,
1480 : : .done = page_not_mapped,
1481 : : .file_nonlinear = try_to_unmap_nonlinear,
1482 : : .anon_lock = page_lock_anon_vma_read,
1483 : : };
1484 : :
1485 : : VM_BUG_ON_PAGE(!PageHuge(page) && PageTransHuge(page), page);
1486 : :
1487 : : /*
1488 : : * During exec, a temporary VMA is setup and later moved.
1489 : : * The VMA is moved under the anon_vma lock but not the
1490 : : * page tables leading to a race where migration cannot
1491 : : * find the migration ptes. Rather than increasing the
1492 : : * locking requirements of exec(), migration skips
1493 : : * temporary VMAs until after exec() completes.
1494 : : */
1495 [ - + ][ # # ]: 1519 : if (flags & TTU_MIGRATION && !PageKsm(page) && PageAnon(page))
1496 : 0 : rwc.invalid_vma = invalid_migration_vma;
1497 : :
1498 : 1519 : ret = rmap_walk(page, &rwc);
1499 : :
1500 [ + - ][ + - ]: 1519 : if (ret != SWAP_MLOCK && !page_mapped(page))
1501 : : ret = SWAP_SUCCESS;
1502 : 1519 : return ret;
1503 : : }
1504 : :
1505 : : /**
1506 : : * try_to_munlock - try to munlock a page
1507 : : * @page: the page to be munlocked
1508 : : *
1509 : : * Called from munlock code. Checks all of the VMAs mapping the page
1510 : : * to make sure nobody else has this page mlocked. The page will be
1511 : : * returned with PG_mlocked cleared if no other vmas have it mlocked.
1512 : : *
1513 : : * Return values are:
1514 : : *
1515 : : * SWAP_AGAIN - no vma is holding page mlocked, or,
1516 : : * SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem
1517 : : * SWAP_FAIL - page cannot be located at present
1518 : : * SWAP_MLOCK - page is now mlocked.
1519 : : */
1520 : 0 : int try_to_munlock(struct page *page)
1521 : : {
1522 : : int ret;
1523 : 1870 : struct rmap_walk_control rwc = {
1524 : : .rmap_one = try_to_unmap_one,
1525 : : .arg = (void *)TTU_MUNLOCK,
1526 : : .done = page_not_mapped,
1527 : : /*
1528 : : * We don't bother to try to find the munlocked page in
1529 : : * nonlinears. It's costly. Instead, later, page reclaim logic
1530 : : * may call try_to_unmap() and recover PG_mlocked lazily.
1531 : : */
1532 : : .file_nonlinear = NULL,
1533 : : .anon_lock = page_lock_anon_vma_read,
1534 : :
1535 : : };
1536 : :
1537 : : VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
1538 : :
1539 : 1870 : ret = rmap_walk(page, &rwc);
1540 : 1870 : return ret;
1541 : : }
1542 : :
1543 : 0 : void __put_anon_vma(struct anon_vma *anon_vma)
1544 : : {
1545 : 12496088 : struct anon_vma *root = anon_vma->root;
1546 : :
1547 [ + + - + ]: 24691853 : if (root != anon_vma && atomic_dec_and_test(&root->refcount))
1548 : : anon_vma_free(root);
1549 : :
1550 : : anon_vma_free(anon_vma);
1551 : 12496181 : }
1552 : :
1553 : 1 : static struct anon_vma *rmap_walk_anon_lock(struct page *page,
1554 : : struct rmap_walk_control *rwc)
1555 : : {
1556 : 0 : struct anon_vma *anon_vma;
1557 : :
1558 [ + - ]: 1 : if (rwc->anon_lock)
1559 : 1 : return rwc->anon_lock(page);
1560 : :
1561 : : /*
1562 : : * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()
1563 : : * because that depends on page_mapped(); but not all its usages
1564 : : * are holding mmap_sem. Users without mmap_sem are required to
1565 : : * take a reference count to prevent the anon_vma disappearing
1566 : : */
1567 : : anon_vma = page_anon_vma(page);
1568 [ # # ]: 0 : if (!anon_vma)
1569 : : return NULL;
1570 : :
1571 : : anon_vma_lock_read(anon_vma);
1572 : : return anon_vma;
1573 : : }
1574 : :
1575 : : /*
1576 : : * rmap_walk_anon - do something to anonymous page using the object-based
1577 : : * rmap method
1578 : : * @page: the page to be handled
1579 : : * @rwc: control variable according to each walk type
1580 : : *
1581 : : * Find all the mappings of a page using the mapping pointer and the vma chains
1582 : : * contained in the anon_vma struct it points to.
1583 : : *
1584 : : * When called from try_to_munlock(), the mmap_sem of the mm containing the vma
1585 : : * where the page was found will be held for write. So, we won't recheck
1586 : : * vm_flags for that VMA. That should be OK, because that vma shouldn't be
1587 : : * LOCKED.
1588 : : */
1589 : 0 : static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc)
1590 : : {
1591 : 1 : struct anon_vma *anon_vma;
1592 : 1 : pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
1593 : : struct anon_vma_chain *avc;
1594 : : int ret = SWAP_AGAIN;
1595 : :
1596 : 1 : anon_vma = rmap_walk_anon_lock(page, rwc);
1597 [ + - ]: 1 : if (!anon_vma)
1598 : : return ret;
1599 : :
1600 [ + + ]: 3 : anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
1601 : 2 : struct vm_area_struct *vma = avc->vma;
1602 : : unsigned long address = vma_address(page, vma);
1603 : :
1604 [ - + ][ # # ]: 2 : if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1605 : 0 : continue;
1606 : :
1607 : 2 : ret = rwc->rmap_one(page, vma, address, rwc->arg);
1608 [ + - ]: 2 : if (ret != SWAP_AGAIN)
1609 : : break;
1610 [ + - ][ + - ]: 2 : if (rwc->done && rwc->done(page))
1611 : : break;
1612 : : }
1613 : : anon_vma_unlock_read(anon_vma);
1614 : 1 : return ret;
1615 : : }
1616 : :
1617 : : /*
1618 : : * rmap_walk_file - do something to file page using the object-based rmap method
1619 : : * @page: the page to be handled
1620 : : * @rwc: control variable according to each walk type
1621 : : *
1622 : : * Find all the mappings of a page using the mapping pointer and the vma chains
1623 : : * contained in the address_space struct it points to.
1624 : : *
1625 : : * When called from try_to_munlock(), the mmap_sem of the mm containing the vma
1626 : : * where the page was found will be held for write. So, we won't recheck
1627 : : * vm_flags for that VMA. That should be OK, because that vma shouldn't be
1628 : : * LOCKED.
1629 : : */
1630 : 0 : static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc)
1631 : : {
1632 : 118799 : struct address_space *mapping = page->mapping;
1633 : 237598 : pgoff_t pgoff = page->index << compound_order(page);
1634 : : struct vm_area_struct *vma;
1635 : : int ret = SWAP_AGAIN;
1636 : :
1637 : : /*
1638 : : * The page lock not only makes sure that page->mapping cannot
1639 : : * suddenly be NULLified by truncation, it makes sure that the
1640 : : * structure at mapping cannot be freed and reused yet,
1641 : : * so we can safely take mapping->i_mmap_mutex.
1642 : : */
1643 : : VM_BUG_ON(!PageLocked(page));
1644 : :
1645 [ + - ]: 118799 : if (!mapping)
1646 : : return ret;
1647 : 118799 : mutex_lock(&mapping->i_mmap_mutex);
1648 [ + + ]: 417694 : vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1649 : : unsigned long address = vma_address(page, vma);
1650 : :
1651 [ + + ][ + + ]: 305117 : if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1652 : 39693 : continue;
1653 : :
1654 : 265424 : ret = rwc->rmap_one(page, vma, address, rwc->arg);
1655 [ + + ]: 265424 : if (ret != SWAP_AGAIN)
1656 : : goto done;
1657 [ + + ][ + + ]: 260721 : if (rwc->done && rwc->done(page))
1658 : : goto done;
1659 : : }
1660 : :
1661 [ - + ]: 112576 : if (!rwc->file_nonlinear)
1662 : : goto done;
1663 : :
1664 [ # # ]: 0 : if (list_empty(&mapping->i_mmap_nonlinear))
1665 : : goto done;
1666 : :
1667 : 0 : ret = rwc->file_nonlinear(page, mapping, rwc->arg);
1668 : :
1669 : : done:
1670 : 118798 : mutex_unlock(&mapping->i_mmap_mutex);
1671 : 118799 : return ret;
1672 : : }
1673 : :
1674 : 0 : int rmap_walk(struct page *page, struct rmap_walk_control *rwc)
1675 : : {
1676 : : if (unlikely(PageKsm(page)))
1677 : : return rmap_walk_ksm(page, rwc);
1678 [ + + ]: 118800 : else if (PageAnon(page))
1679 : 1 : return rmap_walk_anon(page, rwc);
1680 : : else
1681 : 118799 : return rmap_walk_file(page, rwc);
1682 : : }
1683 : :
1684 : : #ifdef CONFIG_HUGETLB_PAGE
1685 : : /*
1686 : : * The following three functions are for anonymous (private mapped) hugepages.
1687 : : * Unlike common anonymous pages, anonymous hugepages have no accounting code
1688 : : * and no lru code, because we handle hugepages differently from common pages.
1689 : : */
1690 : : static void __hugepage_set_anon_rmap(struct page *page,
1691 : : struct vm_area_struct *vma, unsigned long address, int exclusive)
1692 : : {
1693 : : struct anon_vma *anon_vma = vma->anon_vma;
1694 : :
1695 : : BUG_ON(!anon_vma);
1696 : :
1697 : : if (PageAnon(page))
1698 : : return;
1699 : : if (!exclusive)
1700 : : anon_vma = anon_vma->root;
1701 : :
1702 : : anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1703 : : page->mapping = (struct address_space *) anon_vma;
1704 : : page->index = linear_page_index(vma, address);
1705 : : }
1706 : :
1707 : : void hugepage_add_anon_rmap(struct page *page,
1708 : : struct vm_area_struct *vma, unsigned long address)
1709 : : {
1710 : : struct anon_vma *anon_vma = vma->anon_vma;
1711 : : int first;
1712 : :
1713 : : BUG_ON(!PageLocked(page));
1714 : : BUG_ON(!anon_vma);
1715 : : /* address might be in next vma when migration races vma_adjust */
1716 : : first = atomic_inc_and_test(&page->_mapcount);
1717 : : if (first)
1718 : : __hugepage_set_anon_rmap(page, vma, address, 0);
1719 : : }
1720 : :
1721 : : void hugepage_add_new_anon_rmap(struct page *page,
1722 : : struct vm_area_struct *vma, unsigned long address)
1723 : : {
1724 : : BUG_ON(address < vma->vm_start || address >= vma->vm_end);
1725 : : atomic_set(&page->_mapcount, 0);
1726 : : __hugepage_set_anon_rmap(page, vma, address, 1);
1727 : : }
1728 : : #endif /* CONFIG_HUGETLB_PAGE */
|
