Branch data Line data Source code
1 : : /*
2 : : * mm/mmap.c
3 : : *
4 : : * Written by obz.
5 : : *
6 : : * Address space accounting code <alan@lxorguk.ukuu.org.uk>
7 : : */
8 : :
9 : : #include <linux/kernel.h>
10 : : #include <linux/slab.h>
11 : : #include <linux/backing-dev.h>
12 : : #include <linux/mm.h>
13 : : #include <linux/shm.h>
14 : : #include <linux/mman.h>
15 : : #include <linux/pagemap.h>
16 : : #include <linux/swap.h>
17 : : #include <linux/syscalls.h>
18 : : #include <linux/capability.h>
19 : : #include <linux/init.h>
20 : : #include <linux/file.h>
21 : : #include <linux/fs.h>
22 : : #include <linux/personality.h>
23 : : #include <linux/security.h>
24 : : #include <linux/hugetlb.h>
25 : : #include <linux/profile.h>
26 : : #include <linux/export.h>
27 : : #include <linux/mount.h>
28 : : #include <linux/mempolicy.h>
29 : : #include <linux/rmap.h>
30 : : #include <linux/mmu_notifier.h>
31 : : #include <linux/perf_event.h>
32 : : #include <linux/audit.h>
33 : : #include <linux/khugepaged.h>
34 : : #include <linux/uprobes.h>
35 : : #include <linux/rbtree_augmented.h>
36 : : #include <linux/sched/sysctl.h>
37 : : #include <linux/notifier.h>
38 : : #include <linux/memory.h>
39 : :
40 : : #include <asm/uaccess.h>
41 : : #include <asm/cacheflush.h>
42 : : #include <asm/tlb.h>
43 : : #include <asm/mmu_context.h>
44 : :
45 : : #include "internal.h"
46 : :
47 : : #ifndef arch_mmap_check
48 : : #define arch_mmap_check(addr, len, flags) (0)
49 : : #endif
50 : :
51 : : #ifndef arch_rebalance_pgtables
52 : : #define arch_rebalance_pgtables(addr, len) (addr)
53 : : #endif
54 : :
55 : : static void unmap_region(struct mm_struct *mm,
56 : : struct vm_area_struct *vma, struct vm_area_struct *prev,
57 : : unsigned long start, unsigned long end);
58 : :
59 : : /* description of effects of mapping type and prot in current implementation.
60 : : * this is due to the limited x86 page protection hardware. The expected
61 : : * behavior is in parens:
62 : : *
63 : : * map_type prot
64 : : * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
65 : : * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
66 : : * w: (no) no w: (no) no w: (yes) yes w: (no) no
67 : : * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
68 : : *
69 : : * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
70 : : * w: (no) no w: (no) no w: (copy) copy w: (no) no
71 : : * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
72 : : *
73 : : */
74 : : pgprot_t protection_map[16] = {
75 : : __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
76 : : __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
77 : : };
78 : :
79 : 0 : pgprot_t vm_get_page_prot(unsigned long vm_flags)
80 : : {
81 : 2025672 : return __pgprot(pgprot_val(protection_map[vm_flags &
82 : : (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
83 : : pgprot_val(arch_vm_get_page_prot(vm_flags)));
84 : : }
85 : : EXPORT_SYMBOL(vm_get_page_prot);
86 : :
87 : : int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS; /* heuristic overcommit */
88 : : int sysctl_overcommit_ratio __read_mostly = 50; /* default is 50% */
89 : : unsigned long sysctl_overcommit_kbytes __read_mostly;
90 : : int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
91 : : unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
92 : : unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
93 : : /*
94 : : * Make sure vm_committed_as in one cacheline and not cacheline shared with
95 : : * other variables. It can be updated by several CPUs frequently.
96 : : */
97 : : struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
98 : :
99 : : /*
100 : : * The global memory commitment made in the system can be a metric
101 : : * that can be used to drive ballooning decisions when Linux is hosted
102 : : * as a guest. On Hyper-V, the host implements a policy engine for dynamically
103 : : * balancing memory across competing virtual machines that are hosted.
104 : : * Several metrics drive this policy engine including the guest reported
105 : : * memory commitment.
106 : : */
107 : 0 : unsigned long vm_memory_committed(void)
108 : : {
109 : 0 : return percpu_counter_read_positive(&vm_committed_as);
110 : : }
111 : : EXPORT_SYMBOL_GPL(vm_memory_committed);
112 : :
113 : : /*
114 : : * Check that a process has enough memory to allocate a new virtual
115 : : * mapping. 0 means there is enough memory for the allocation to
116 : : * succeed and -ENOMEM implies there is not.
117 : : *
118 : : * We currently support three overcommit policies, which are set via the
119 : : * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
120 : : *
121 : : * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
122 : : * Additional code 2002 Jul 20 by Robert Love.
123 : : *
124 : : * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
125 : : *
126 : : * Note this is a helper function intended to be used by LSMs which
127 : : * wish to use this logic.
128 : : */
129 : 0 : int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
130 : : {
131 : : unsigned long free, allowed, reserve;
132 : :
133 : : vm_acct_memory(pages);
134 : :
135 : : /*
136 : : * Sometimes we want to use more memory than we have
137 : : */
138 [ + + ]: 15624450 : if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
139 : : return 0;
140 : :
141 [ + + ]: 15618666 : if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
142 : : free = global_page_state(NR_FREE_PAGES);
143 : 14742266 : free += global_page_state(NR_FILE_PAGES);
144 : :
145 : : /*
146 : : * shmem pages shouldn't be counted as free in this
147 : : * case, they can't be purged, only swapped out, and
148 : : * that won't affect the overall amount of available
149 : : * memory in the system.
150 : : */
151 : 14742266 : free -= global_page_state(NR_SHMEM);
152 : :
153 : 14742266 : free += get_nr_swap_pages();
154 : :
155 : : /*
156 : : * Any slabs which are created with the
157 : : * SLAB_RECLAIM_ACCOUNT flag claim to have contents
158 : : * which are reclaimable, under pressure. The dentry
159 : : * cache and most inode caches should fall into this
160 : : */
161 : 0 : free += global_page_state(NR_SLAB_RECLAIMABLE);
162 : :
163 : : /*
164 : : * Leave reserved pages. The pages are not for anonymous pages.
165 : : */
166 [ + ]: 14742266 : if (free <= totalreserve_pages)
167 : : goto error;
168 : : else
169 : 14742269 : free -= totalreserve_pages;
170 : :
171 : : /*
172 : : * Reserve some for root
173 : : */
174 [ + + ]: 14742269 : if (!cap_sys_admin)
175 : 5954 : free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
176 : :
177 [ + + ]: 14742269 : if (free > pages)
178 : : return 0;
179 : :
180 : : goto error;
181 : : }
182 : :
183 : 876400 : allowed = vm_commit_limit();
184 : : /*
185 : : * Reserve some for root
186 : : */
187 [ - + ]: 876400 : if (!cap_sys_admin)
188 : 0 : allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
189 : :
190 : : /*
191 : : * Don't let a single process grow so big a user can't recover
192 : : */
193 [ + - ]: 876400 : if (mm) {
194 : 876400 : reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
195 : 876400 : allowed -= min(mm->total_vm / 32, reserve);
196 : : }
197 : :
198 [ + + ]: 876400 : if (percpu_counter_read_positive(&vm_committed_as) < allowed)
199 : : return 0;
200 : : error:
201 : : vm_unacct_memory(pages);
202 : :
203 : 9 : return -ENOMEM;
204 : : }
205 : :
206 : : /*
207 : : * Requires inode->i_mapping->i_mmap_mutex
208 : : */
209 : 0 : static void __remove_shared_vm_struct(struct vm_area_struct *vma,
210 : 6798624 : struct file *file, struct address_space *mapping)
211 : : {
212 [ + + ]: 13416335 : if (vma->vm_flags & VM_DENYWRITE)
213 : 6798624 : atomic_inc(&file_inode(file)->i_writecount);
214 [ + + ]: 13416465 : if (vma->vm_flags & VM_SHARED)
215 : 956696 : mapping->i_mmap_writable--;
216 : :
217 : : flush_dcache_mmap_lock(mapping);
218 [ + + ]: 26832635 : if (unlikely(vma->vm_flags & VM_NONLINEAR))
219 : 1 : list_del_init(&vma->shared.nonlinear);
220 : : else
221 : 13416299 : vma_interval_tree_remove(vma, &mapping->i_mmap);
222 : : flush_dcache_mmap_unlock(mapping);
223 : 13416575 : }
224 : :
225 : : /*
226 : : * Unlink a file-based vm structure from its interval tree, to hide
227 : : * vma from rmap and vmtruncate before freeing its page tables.
228 : : */
229 : 0 : void unlink_file_vma(struct vm_area_struct *vma)
230 : : {
231 : 20305548 : struct file *file = vma->vm_file;
232 : :
233 [ + + ]: 20305548 : if (file) {
234 : 13416185 : struct address_space *mapping = file->f_mapping;
235 : 13416185 : mutex_lock(&mapping->i_mmap_mutex);
236 : 13416364 : __remove_shared_vm_struct(vma, file, mapping);
237 : 13416431 : mutex_unlock(&mapping->i_mmap_mutex);
238 : : }
239 : 390 : }
240 : :
241 : : /*
242 : : * Close a vm structure and free it, returning the next.
243 : : */
244 : 0 : static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
245 : : {
246 : 20305501 : struct vm_area_struct *next = vma->vm_next;
247 : :
248 : : might_sleep();
249 [ + + ][ + + ]: 20305501 : if (vma->vm_ops && vma->vm_ops->close)
250 : 1129662 : vma->vm_ops->close(vma);
251 [ + + ]: 20305841 : if (vma->vm_file)
252 : 13416436 : fput(vma->vm_file);
253 : : mpol_put(vma_policy(vma));
254 : 20305816 : kmem_cache_free(vm_area_cachep, vma);
255 : 20305586 : return next;
256 : : }
257 : :
258 : : static unsigned long do_brk(unsigned long addr, unsigned long len);
259 : :
260 : 0 : SYSCALL_DEFINE1(brk, unsigned long, brk)
261 : : {
262 : : unsigned long rlim, retval;
263 : : unsigned long newbrk, oldbrk;
264 : 93592 : struct mm_struct *mm = current->mm;
265 : : unsigned long min_brk;
266 : : bool populate;
267 : :
268 : 93592 : down_write(&mm->mmap_sem);
269 : :
270 : : #ifdef CONFIG_COMPAT_BRK
271 : : /*
272 : : * CONFIG_COMPAT_BRK can still be overridden by setting
273 : : * randomize_va_space to 2, which will still cause mm->start_brk
274 : : * to be arbitrarily shifted
275 : : */
276 : : if (current->brk_randomized)
277 : : min_brk = mm->start_brk;
278 : : else
279 : : min_brk = mm->end_data;
280 : : #else
281 : 93593 : min_brk = mm->start_brk;
282 : : #endif
283 [ + + ]: 93593 : if (brk < min_brk)
284 : : goto out;
285 : :
286 : : /*
287 : : * Check against rlimit here. If this check is done later after the test
288 : : * of oldbrk with newbrk then it can escape the test and let the data
289 : : * segment grow beyond its set limit the in case where the limit is
290 : : * not page aligned -Ram Gupta
291 : : */
292 : : rlim = rlimit(RLIMIT_DATA);
293 [ - + ][ # # ]: 40765 : if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
294 : 0 : (mm->end_data - mm->start_data) > rlim)
295 : : goto out;
296 : :
297 : 40766 : newbrk = PAGE_ALIGN(brk);
298 : 40766 : oldbrk = PAGE_ALIGN(mm->brk);
299 [ + - ]: 134358 : if (oldbrk == newbrk)
300 : : goto set_brk;
301 : :
302 : : /* Always allow shrinking brk. */
303 [ + + ]: 40766 : if (brk <= mm->brk) {
304 [ + - ]: 4703 : if (!do_munmap(mm, newbrk, oldbrk-newbrk))
305 : : goto set_brk;
306 : : goto out;
307 : : }
308 : :
309 : : /* Check against existing mmap mappings. */
310 [ + + ]: 36063 : if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
311 : : goto out;
312 : :
313 : : /* Ok, looks good - let it rip. */
314 [ + - ]: 30032 : if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
315 : : goto out;
316 : :
317 : : set_brk:
318 : 34735 : mm->brk = brk;
319 [ + + ][ + + ]: 34735 : populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
320 : 34735 : up_write(&mm->mmap_sem);
321 [ + + ]: 34735 : if (populate)
322 : 1 : mm_populate(oldbrk, newbrk - oldbrk);
323 : : return brk;
324 : :
325 : : out:
326 : 0 : retval = mm->brk;
327 : 0 : up_write(&mm->mmap_sem);
328 : 58859 : return retval;
329 : : }
330 : :
331 : 0 : static long vma_compute_subtree_gap(struct vm_area_struct *vma)
332 : : {
333 : : unsigned long max, subtree_gap;
334 : 106961246 : max = vma->vm_start;
335 [ + + ]: 106961246 : if (vma->vm_prev)
336 : 101474594 : max -= vma->vm_prev->vm_end;
337 [ + + ]: 106961246 : if (vma->vm_rb.rb_left) {
338 : 62676739 : subtree_gap = rb_entry(vma->vm_rb.rb_left,
339 : : struct vm_area_struct, vm_rb)->rb_subtree_gap;
340 [ + + ]: 62676739 : if (subtree_gap > max)
341 : : max = subtree_gap;
342 : : }
343 [ # # ]: 106961246 : if (vma->vm_rb.rb_right) {
344 : 71970149 : subtree_gap = rb_entry(vma->vm_rb.rb_right,
345 : : struct vm_area_struct, vm_rb)->rb_subtree_gap;
346 [ + + ]: 71970149 : if (subtree_gap > max)
347 : : max = subtree_gap;
348 : : }
349 : 0 : return max;
350 : : }
351 : :
352 : : #ifdef CONFIG_DEBUG_VM_RB
353 : : static int browse_rb(struct rb_root *root)
354 : : {
355 : : int i = 0, j, bug = 0;
356 : : struct rb_node *nd, *pn = NULL;
357 : : unsigned long prev = 0, pend = 0;
358 : :
359 : : for (nd = rb_first(root); nd; nd = rb_next(nd)) {
360 : : struct vm_area_struct *vma;
361 : : vma = rb_entry(nd, struct vm_area_struct, vm_rb);
362 : : if (vma->vm_start < prev) {
363 : : printk("vm_start %lx prev %lx\n", vma->vm_start, prev);
364 : : bug = 1;
365 : : }
366 : : if (vma->vm_start < pend) {
367 : : printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
368 : : bug = 1;
369 : : }
370 : : if (vma->vm_start > vma->vm_end) {
371 : : printk("vm_end %lx < vm_start %lx\n",
372 : : vma->vm_end, vma->vm_start);
373 : : bug = 1;
374 : : }
375 : : if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
376 : : printk("free gap %lx, correct %lx\n",
377 : : vma->rb_subtree_gap,
378 : : vma_compute_subtree_gap(vma));
379 : : bug = 1;
380 : : }
381 : : i++;
382 : : pn = nd;
383 : : prev = vma->vm_start;
384 : : pend = vma->vm_end;
385 : : }
386 : : j = 0;
387 : : for (nd = pn; nd; nd = rb_prev(nd))
388 : : j++;
389 : : if (i != j) {
390 : : printk("backwards %d, forwards %d\n", j, i);
391 : : bug = 1;
392 : : }
393 : : return bug ? -1 : i;
394 : : }
395 : :
396 : : static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
397 : : {
398 : : struct rb_node *nd;
399 : :
400 : : for (nd = rb_first(root); nd; nd = rb_next(nd)) {
401 : : struct vm_area_struct *vma;
402 : : vma = rb_entry(nd, struct vm_area_struct, vm_rb);
403 : : BUG_ON(vma != ignore &&
404 : : vma->rb_subtree_gap != vma_compute_subtree_gap(vma));
405 : : }
406 : : }
407 : :
408 : : void validate_mm(struct mm_struct *mm)
409 : : {
410 : : int bug = 0;
411 : : int i = 0;
412 : : unsigned long highest_address = 0;
413 : : struct vm_area_struct *vma = mm->mmap;
414 : : while (vma) {
415 : : struct anon_vma_chain *avc;
416 : : vma_lock_anon_vma(vma);
417 : : list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
418 : : anon_vma_interval_tree_verify(avc);
419 : : vma_unlock_anon_vma(vma);
420 : : highest_address = vma->vm_end;
421 : : vma = vma->vm_next;
422 : : i++;
423 : : }
424 : : if (i != mm->map_count) {
425 : : printk("map_count %d vm_next %d\n", mm->map_count, i);
426 : : bug = 1;
427 : : }
428 : : if (highest_address != mm->highest_vm_end) {
429 : : printk("mm->highest_vm_end %lx, found %lx\n",
430 : : mm->highest_vm_end, highest_address);
431 : : bug = 1;
432 : : }
433 : : i = browse_rb(&mm->mm_rb);
434 : : if (i != mm->map_count) {
435 : : printk("map_count %d rb %d\n", mm->map_count, i);
436 : : bug = 1;
437 : : }
438 : : BUG_ON(bug);
439 : : }
440 : : #else
441 : : #define validate_mm_rb(root, ignore) do { } while (0)
442 : : #define validate_mm(mm) do { } while (0)
443 : : #endif
444 : :
445 [ + + ][ + + ]: 115000630 : RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
[ + + ][ + + ]
446 : : unsigned long, rb_subtree_gap, vma_compute_subtree_gap)
447 : :
448 : : /*
449 : : * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
450 : : * vma->vm_prev->vm_end values changed, without modifying the vma's position
451 : : * in the rbtree.
452 : : */
453 : 0 : static void vma_gap_update(struct vm_area_struct *vma)
454 : : {
455 : : /*
456 : : * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
457 : : * function that does exacltly what we want.
458 : : */
459 : 25122815 : vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
460 : 151 : }
461 : :
462 : : static inline void vma_rb_insert(struct vm_area_struct *vma,
463 : : struct rb_root *root)
464 : : {
465 : : /* All rb_subtree_gap values must be consistent prior to insertion */
466 : : validate_mm_rb(root, NULL);
467 : :
468 : : rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
469 : : }
470 : :
471 : 0 : static void vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
472 : : {
473 : : /*
474 : : * All rb_subtree_gap values must be consistent prior to erase,
475 : : * with the possible exception of the vma being erased.
476 : : */
477 : : validate_mm_rb(root, vma);
478 : :
479 : : /*
480 : : * Note rb_erase_augmented is a fairly large inline function,
481 : : * so make sure we instantiate it only once with our desired
482 : : * augmented rbtree callbacks.
483 : : */
484 : 425082 : rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
485 : 424964 : }
486 : :
487 : : /*
488 : : * vma has some anon_vma assigned, and is already inserted on that
489 : : * anon_vma's interval trees.
490 : : *
491 : : * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
492 : : * vma must be removed from the anon_vma's interval trees using
493 : : * anon_vma_interval_tree_pre_update_vma().
494 : : *
495 : : * After the update, the vma will be reinserted using
496 : : * anon_vma_interval_tree_post_update_vma().
497 : : *
498 : : * The entire update must be protected by exclusive mmap_sem and by
499 : : * the root anon_vma's mutex.
500 : : */
501 : : static inline void
502 : : anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
503 : : {
504 : : struct anon_vma_chain *avc;
505 : :
506 [ + + ][ + + ]: 7019877 : list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
[ - + ]
507 : 2504093 : anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
508 : : }
509 : :
510 : : static inline void
511 : : anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
512 : : {
513 : : struct anon_vma_chain *avc;
514 : :
515 [ + + ][ + + ]: 5006363 : list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
[ - + ]
516 : 2504064 : anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
517 : : }
518 : :
519 : : static int find_vma_links(struct mm_struct *mm, unsigned long addr,
520 : : unsigned long end, struct vm_area_struct **pprev,
521 : : struct rb_node ***rb_link, struct rb_node **rb_parent)
522 : : {
523 : : struct rb_node **__rb_link, *__rb_parent, *rb_prev;
524 : :
525 : 2524781 : __rb_link = &mm->mm_rb.rb_node;
526 : : rb_prev = __rb_parent = NULL;
527 : :
528 [ + + ][ + + ]: 32645063 : while (*__rb_link) {
[ + + ][ + + ]
[ + + ]
529 : : struct vm_area_struct *vma_tmp;
530 : :
531 : : __rb_parent = *__rb_link;
532 : : vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
533 : :
534 [ + + ][ + + ]: 30251583 : if (vma_tmp->vm_end > addr) {
[ + + ][ + + ]
[ + + ]
535 : : /* Fail if an existing vma overlaps the area */
536 [ + - ][ + - ]: 25248389 : if (vma_tmp->vm_start < end)
[ + + ][ + + ]
[ + ]
537 : : return -ENOMEM;
538 : 25117088 : __rb_link = &__rb_parent->rb_left;
539 : : } else {
540 : : rb_prev = __rb_parent;
541 : 30120282 : __rb_link = &__rb_parent->rb_right;
542 : : }
543 : : }
544 : :
545 : : *pprev = NULL;
546 [ + - ][ + + ]: 2393480 : if (rb_prev)
[ + ][ + + ]
[ + + ]
547 : 2393480 : *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
548 : : *rb_link = __rb_link;
549 : : *rb_parent = __rb_parent;
550 : : return 0;
551 : : }
552 : :
553 : 0 : static unsigned long count_vma_pages_range(struct mm_struct *mm,
554 : : unsigned long addr, unsigned long end)
555 : : {
556 : : unsigned long nr_pages = 0;
557 : : struct vm_area_struct *vma;
558 : :
559 : : /* Find first overlaping mapping */
560 : : vma = find_vma_intersection(mm, addr, end);
561 [ # # ]: 0 : if (!vma)
562 : : return 0;
563 : :
564 : 0 : nr_pages = (min(end, vma->vm_end) -
565 : 0 : max(addr, vma->vm_start)) >> PAGE_SHIFT;
566 : :
567 : : /* Iterate over the rest of the overlaps */
568 [ # # ]: 0 : for (vma = vma->vm_next; vma; vma = vma->vm_next) {
569 : : unsigned long overlap_len;
570 : :
571 [ # # ]: 0 : if (vma->vm_start > end)
572 : : break;
573 : :
574 : 0 : overlap_len = min(end, vma->vm_end) - vma->vm_start;
575 : 0 : nr_pages += overlap_len >> PAGE_SHIFT;
576 : : }
577 : :
578 : : return nr_pages;
579 : : }
580 : :
581 : 0 : void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
582 : : struct rb_node **rb_link, struct rb_node *rb_parent)
583 : : {
584 : : /* Update tracking information for the gap following the new vma. */
585 [ + + ]: 20308640 : if (vma->vm_next)
586 : 2191215 : vma_gap_update(vma->vm_next);
587 : : else
588 : 18117425 : mm->highest_vm_end = vma->vm_end;
589 : :
590 : : /*
591 : : * vma->vm_prev wasn't known when we followed the rbtree to find the
592 : : * correct insertion point for that vma. As a result, we could not
593 : : * update the vma vm_rb parents rb_subtree_gap values on the way down.
594 : : * So, we first insert the vma with a zero rb_subtree_gap value
595 : : * (to be consistent with what we did on the way down), and then
596 : : * immediately update the gap to the correct value. Finally we
597 : : * rebalance the rbtree after all augmented values have been set.
598 : : */
599 : 20308657 : rb_link_node(&vma->vm_rb, rb_parent, rb_link);
600 : 20308657 : vma->rb_subtree_gap = 0;
601 : 20308657 : vma_gap_update(vma);
602 : 20308476 : vma_rb_insert(vma, &mm->mm_rb);
603 : 20308445 : }
604 : :
605 : 0 : static void __vma_link_file(struct vm_area_struct *vma)
606 : : {
607 : 190709 : struct file *file;
608 : :
609 : 2107688 : file = vma->vm_file;
610 [ + + ]: 2107688 : if (file) {
611 : 1924915 : struct address_space *mapping = file->f_mapping;
612 : :
613 [ + + ]: 1924915 : if (vma->vm_flags & VM_DENYWRITE)
614 : 190709 : atomic_dec(&file_inode(file)->i_writecount);
615 [ + + ]: 1924939 : if (vma->vm_flags & VM_SHARED)
616 : 946953 : mapping->i_mmap_writable++;
617 : :
618 : : flush_dcache_mmap_lock(mapping);
619 [ - + ]: 4032595 : if (unlikely(vma->vm_flags & VM_NONLINEAR))
620 : 0 : vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
621 : : else
622 : 1924907 : vma_interval_tree_insert(vma, &mapping->i_mmap);
623 : : flush_dcache_mmap_unlock(mapping);
624 : : }
625 : 2107736 : }
626 : :
627 : : static void
628 : 0 : __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
629 : : struct vm_area_struct *prev, struct rb_node **rb_link,
630 : : struct rb_node *rb_parent)
631 : : {
632 : 2218381 : __vma_link_list(mm, vma, prev, rb_parent);
633 : 2218447 : __vma_link_rb(mm, vma, rb_link, rb_parent);
634 : 2218483 : }
635 : :
636 : 0 : static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
637 : : struct vm_area_struct *prev, struct rb_node **rb_link,
638 : : struct rb_node *rb_parent)
639 : : {
640 : : struct address_space *mapping = NULL;
641 : :
642 [ + + ]: 1702618 : if (vma->vm_file)
643 : 1519843 : mapping = vma->vm_file->f_mapping;
644 : :
645 [ + + ]: 1702618 : if (mapping)
646 : 1519853 : mutex_lock(&mapping->i_mmap_mutex);
647 : :
648 : 1702616 : __vma_link(mm, vma, prev, rb_link, rb_parent);
649 : 1702670 : __vma_link_file(vma);
650 : :
651 [ + + ]: 1702677 : if (mapping)
652 : 1519902 : mutex_unlock(&mapping->i_mmap_mutex);
653 : :
654 : 1702645 : mm->map_count++;
655 : : validate_mm(mm);
656 : 1702645 : }
657 : :
658 : : /*
659 : : * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
660 : : * mm's list and rbtree. It has already been inserted into the interval tree.
661 : : */
662 : 0 : static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
663 : : {
664 : : struct vm_area_struct *prev;
665 : : struct rb_node **rb_link, *rb_parent;
666 : :
667 [ - + ]: 515855 : if (find_vma_links(mm, vma->vm_start, vma->vm_end,
668 : : &prev, &rb_link, &rb_parent))
669 : 0 : BUG();
670 : 515855 : __vma_link(mm, vma, prev, rb_link, rb_parent);
671 : 515856 : mm->map_count++;
672 : 515856 : }
673 : :
674 : : static inline void
675 : : __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
676 : : struct vm_area_struct *prev)
677 : : {
678 : : struct vm_area_struct *next;
679 : :
680 : 2722 : vma_rb_erase(vma, &mm->mm_rb);
681 : 2722 : prev->vm_next = next = vma->vm_next;
682 [ + + ]: 2722 : if (next)
683 : 2712 : next->vm_prev = prev;
684 [ + + ]: 2722 : if (mm->mmap_cache == vma)
685 : 22 : mm->mmap_cache = prev;
686 : : }
687 : :
688 : : /*
689 : : * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
690 : : * is already present in an i_mmap tree without adjusting the tree.
691 : : * The following helper function should be used when such adjustments
692 : : * are necessary. The "insert" vma (if any) is to be inserted
693 : : * before we drop the necessary locks.
694 : : */
695 : 0 : int vma_adjust(struct vm_area_struct *vma, unsigned long start,
696 : : unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
697 : : {
698 : 746394 : struct mm_struct *mm = vma->vm_mm;
699 : 746394 : struct vm_area_struct *next = vma->vm_next;
700 : : struct vm_area_struct *importer = NULL;
701 : : struct address_space *mapping = NULL;
702 : : struct rb_root *root = NULL;
703 : 975463 : struct anon_vma *anon_vma = NULL;
704 : 746394 : struct file *file = vma->vm_file;
705 : : bool start_changed = false, end_changed = false;
706 : : long adjust_next = 0;
707 : : int remove_next = 0;
708 : :
709 [ + + ]: 746394 : if (next && !insert) {
710 : : struct vm_area_struct *exporter = NULL;
711 : :
712 [ + + ]: 176077 : if (end >= next->vm_end) {
713 : : /*
714 : : * vma expands, overlapping all the next, and
715 : : * perhaps the one after too (mprotect case 6).
716 : : */
717 [ + - ]: 2722 : again: remove_next = 1 + (end > next->vm_end);
718 : : end = next->vm_end;
719 : : exporter = next;
720 : : importer = vma;
721 [ + + ]: 173355 : } else if (end > next->vm_start) {
722 : : /*
723 : : * vma expands, overlapping part of the next:
724 : : * mprotect case 5 shifting the boundary up.
725 : : */
726 : 1081 : adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
727 : : exporter = next;
728 : : importer = vma;
729 [ - + ]: 172274 : } else if (end < vma->vm_end) {
730 : : /*
731 : : * vma shrinks, and !insert tells it's not
732 : : * split_vma inserting another: so it must be
733 : : * mprotect case 4 shifting the boundary down.
734 : : */
735 : 0 : adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
736 : : exporter = vma;
737 : : importer = next;
738 : : }
739 : :
740 : : /*
741 : : * Easily overlooked: when mprotect shifts the boundary,
742 : : * make sure the expanding vma has anon_vma set if the
743 : : * shrinking vma had, to cover any anon pages imported.
744 : : */
745 [ + + ][ + + ]: 176077 : if (exporter && exporter->anon_vma && !importer->anon_vma) {
[ - + ]
746 [ # # ]: 0 : if (anon_vma_clone(importer, exporter))
747 : : return -ENOMEM;
748 : 0 : importer->anon_vma = exporter->anon_vma;
749 : : }
750 : : }
751 : :
752 [ + + ]: 746394 : if (file) {
753 : 405162 : mapping = file->f_mapping;
754 [ + ]: 405162 : if (!(vma->vm_flags & VM_NONLINEAR)) {
755 : 405164 : root = &mapping->i_mmap;
756 : 405164 : uprobe_munmap(vma, vma->vm_start, vma->vm_end);
757 : :
758 [ - + ]: 405157 : if (adjust_next)
759 : 0 : uprobe_munmap(next, next->vm_start,
760 : : next->vm_end);
761 : : }
762 : :
763 : 405155 : mutex_lock(&mapping->i_mmap_mutex);
764 [ + + ]: 405167 : if (insert) {
765 : : /*
766 : : * Put into interval tree now, so instantiated pages
767 : : * are visible to arm/parisc __flush_dcache_page
768 : : * throughout; but we cannot insert into address
769 : : * space until vma start or end is updated.
770 : : */
771 : 405086 : __vma_link_file(insert);
772 : : }
773 : : }
774 : :
775 : : vma_adjust_trans_huge(vma, start, end, adjust_next);
776 : :
777 : 1492793 : anon_vma = vma->anon_vma;
778 [ - + ]: 1492793 : if (!anon_vma && adjust_next)
779 : 0 : anon_vma = next->anon_vma;
780 [ + + ]: 1492793 : if (anon_vma) {
781 : : VM_BUG_ON(adjust_next && next->anon_vma &&
782 : : anon_vma != next->anon_vma);
783 : : anon_vma_lock_write(anon_vma);
784 : : anon_vma_interval_tree_pre_update_vma(vma);
785 [ + + ]: 487735 : if (adjust_next)
786 : : anon_vma_interval_tree_pre_update_vma(next);
787 : : }
788 : :
789 [ + + ]: 746403 : if (root) {
790 : : flush_dcache_mmap_lock(mapping);
791 : 405170 : vma_interval_tree_remove(vma, root);
792 [ - + ]: 405163 : if (adjust_next)
793 : 0 : vma_interval_tree_remove(next, root);
794 : : }
795 : :
796 [ + + ]: 746406 : if (start != vma->vm_start) {
797 : 405015 : vma->vm_start = start;
798 : : start_changed = true;
799 : : }
800 [ + + ]: 746406 : if (end != vma->vm_end) {
801 : 341394 : vma->vm_end = end;
802 : : end_changed = true;
803 : : }
804 : 746406 : vma->vm_pgoff = pgoff;
805 [ + + ]: 746406 : if (adjust_next) {
806 : 1081 : next->vm_start += adjust_next << PAGE_SHIFT;
807 : 1081 : next->vm_pgoff += adjust_next;
808 : : }
809 : :
810 [ + + ]: 746406 : if (root) {
811 [ - + ]: 405166 : if (adjust_next)
812 : 0 : vma_interval_tree_insert(next, root);
813 : 405166 : vma_interval_tree_insert(vma, root);
814 : : flush_dcache_mmap_unlock(mapping);
815 : : }
816 : :
817 [ + + ]: 746416 : if (remove_next) {
818 : : /*
819 : : * vma_merge has merged next into vma, and needs
820 : : * us to remove next before dropping the locks.
821 : : */
822 : : __vma_unlink(mm, next, vma);
823 [ + + ]: 2722 : if (file)
824 : 1 : __remove_shared_vm_struct(next, file, mapping);
825 [ + + ]: 743694 : } else if (insert) {
826 : : /*
827 : : * split_vma has split insert from vma, and needs
828 : : * us to insert it before dropping the locks
829 : : * (it may either follow vma or precede it).
830 : : */
831 : 515869 : __insert_vm_struct(mm, insert);
832 : : } else {
833 [ + + ]: 227825 : if (start_changed)
834 : 190972 : vma_gap_update(vma);
835 [ + + ]: 227828 : if (end_changed) {
836 [ + + ]: 36857 : if (!next)
837 : 27239 : mm->highest_vm_end = end;
838 [ + + ]: 9618 : else if (!adjust_next)
839 : 8537 : vma_gap_update(next);
840 : : }
841 : : }
842 : :
843 [ + + ]: 746399 : if (anon_vma) {
844 : : anon_vma_interval_tree_post_update_vma(vma);
845 [ + + ]: 487732 : if (adjust_next)
846 : : anon_vma_interval_tree_post_update_vma(next);
847 : : anon_vma_unlock_write(anon_vma);
848 : : }
849 [ + + ]: 746408 : if (mapping)
850 : 405166 : mutex_unlock(&mapping->i_mmap_mutex);
851 : :
852 [ + + ]: 746416 : if (root) {
853 : 405174 : uprobe_mmap(vma);
854 : :
855 [ - + ]: 405173 : if (adjust_next)
856 : 0 : uprobe_mmap(next);
857 : : }
858 : :
859 [ + + ]: 746415 : if (remove_next) {
860 [ + + ]: 2722 : if (file) {
861 : 1 : uprobe_munmap(next, next->vm_start, next->vm_end);
862 : 1 : fput(file);
863 : : }
864 [ + + ]: 2722 : if (next->anon_vma)
865 : : anon_vma_merge(vma, next);
866 : 2722 : mm->map_count--;
867 : : mpol_put(vma_policy(next));
868 : 2722 : kmem_cache_free(vm_area_cachep, next);
869 : : /*
870 : : * In mprotect's case 6 (see comments on vma_merge),
871 : : * we must remove another next too. It would clutter
872 : : * up the code too much to do both in one go.
873 : : */
874 : 2722 : next = vma->vm_next;
875 [ - + ]: 2722 : if (remove_next == 2)
876 : : goto again;
877 [ + + ]: 2722 : else if (next)
878 : 2712 : vma_gap_update(next);
879 : : else
880 : 10 : mm->highest_vm_end = end;
881 : : }
882 [ + + ]: 746415 : if (insert && file)
883 : 405085 : uprobe_mmap(insert);
884 : :
885 : : validate_mm(mm);
886 : :
887 : : return 0;
888 : : }
889 : :
890 : : /*
891 : : * If the vma has a ->close operation then the driver probably needs to release
892 : : * per-vma resources, so we don't attempt to merge those.
893 : : */
894 : 180447 : static inline int is_mergeable_vma(struct vm_area_struct *vma,
895 : : struct file *file, unsigned long vm_flags,
896 : : const char __user *anon_name)
897 : : {
898 : : /*
899 : : * VM_SOFTDIRTY should not prevent from VMA merging, if we
900 : : * match the flags but dirty bit -- the caller should mark
901 : : * merged VMA as dirty. If dirty bit won't be excluded from
902 : : * comparison, we increase pressue on the memory system forcing
903 : : * the kernel to generate new VMAs when old one could be
904 : : * extended instead.
905 : : */
906 [ + + ][ + + ]: 2059148 : if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
907 : : return 0;
908 [ + + ][ + + ]: 1316728 : if (vma->vm_file != file)
909 : : return 0;
910 [ + + ][ + - ]: 180445 : if (vma->vm_ops && vma->vm_ops->close)
[ + + ][ + ]
911 : : return 0;
912 [ + - ][ + - ]: 180447 : if (vma_get_anon_name(vma) != anon_name)
913 : : return 0;
914 : : return 1;
915 : : }
916 : :
917 : : static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
918 : : struct anon_vma *anon_vma2,
919 : : struct vm_area_struct *vma)
920 : : {
921 : : /*
922 : : * The list_is_singular() test is to avoid merging VMA cloned from
923 : : * parents. This can improve scalability caused by anon_vma lock.
924 : : */
925 [ + + ][ + + ]: 363872 : if ((!anon_vma1 || !anon_vma2) && (!vma ||
[ + - ][ + + ]
[ + + ][ + ]
[ + + ]
926 : 180411 : list_is_singular(&vma->anon_vma_chain)))
927 : : return 1;
928 : 5895 : return anon_vma1 == anon_vma2;
929 : : }
930 : :
931 : : /*
932 : : * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
933 : : * in front of (at a lower virtual address and file offset than) the vma.
934 : : *
935 : : * We cannot merge two vmas if they have differently assigned (non-NULL)
936 : : * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
937 : : *
938 : : * We don't check here for the merged mmap wrapping around the end of pagecache
939 : : * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
940 : : * wrap, nor mmaps which cover the final page at index -1UL.
941 : : */
942 : : static int
943 : 0 : can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
944 : : struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff,
945 : : const char __user *anon_name)
946 : : {
947 [ # # ][ + + ]: 1832103 : if (is_mergeable_vma(vma, file, vm_flags, anon_name) &&
948 : 168025 : is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
949 [ + ]: 166769 : if (vma->vm_pgoff == vm_pgoff)
950 : : return 1;
951 : : }
952 : 1497359 : return 0;
953 : : }
954 : :
955 : : /*
956 : : * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
957 : : * beyond (at a higher virtual address and file offset than) the vma.
958 : : *
959 : : * We cannot merge two vmas if they have differently assigned (non-NULL)
960 : : * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
961 : : */
962 : : static int
963 : 0 : can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
964 : : struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff,
965 : : const char __user *anon_name)
966 : : {
967 [ # # ][ + + ]: 407492 : if (is_mergeable_vma(vma, file, vm_flags, anon_name) &&
968 : 12422 : is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
969 : : pgoff_t vm_pglen;
970 : : vm_pglen = vma_pages(vma);
971 [ + ]: 12360 : if (vma->vm_pgoff + vm_pglen == vm_pgoff)
972 : : return 1;
973 : : }
974 : 382727 : return 0;
975 : : }
976 : :
977 : : /*
978 : : * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
979 : : * figure out whether that can be merged with its predecessor or its
980 : : * successor. Or both (it neatly fills a hole).
981 : : *
982 : : * In most cases - when called for mmap, brk or mremap - [addr,end) is
983 : : * certain not to be mapped by the time vma_merge is called; but when
984 : : * called for mprotect, it is certain to be already mapped (either at
985 : : * an offset within prev, or at the start of next), and the flags of
986 : : * this area are about to be changed to vm_flags - and the no-change
987 : : * case has already been eliminated.
988 : : *
989 : : * The following mprotect cases have to be considered, where AAAA is
990 : : * the area passed down from mprotect_fixup, never extending beyond one
991 : : * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
992 : : *
993 : : * AAAA AAAA AAAA AAAA
994 : : * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
995 : : * cannot merge might become might become might become
996 : : * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
997 : : * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
998 : : * mremap move: PPPPNNNNNNNN 8
999 : : * AAAA
1000 : : * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
1001 : : * might become case 1 below case 2 below case 3 below
1002 : : *
1003 : : * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
1004 : : * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
1005 : : */
1006 : 0 : struct vm_area_struct *vma_merge(struct mm_struct *mm,
1007 : : struct vm_area_struct *prev, unsigned long addr,
1008 : : unsigned long end, unsigned long vm_flags,
1009 : : struct anon_vma *anon_vma, struct file *file,
1010 : : pgoff_t pgoff, struct mempolicy *policy,
1011 : : const char __user *anon_name)
1012 : : {
1013 : 2074186 : pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
1014 : : struct vm_area_struct *area, *next;
1015 : : int err;
1016 : :
1017 : : /*
1018 : : * We later require that vma->vm_flags == vm_flags,
1019 : : * so this tests vma->vm_flags & VM_SPECIAL, too.
1020 : : */
1021 [ + ]: 2074186 : if (vm_flags & VM_SPECIAL)
1022 : : return NULL;
1023 : :
1024 [ + + ]: 2074190 : if (prev)
1025 : 2029637 : next = prev->vm_next;
1026 : : else
1027 : 44553 : next = mm->mmap;
1028 : : area = next;
1029 [ + ][ + + ]: 2074190 : if (next && next->vm_end == end) /* cases 6, 7, 8 */
1030 : 357 : next = next->vm_next;
1031 : :
1032 : : /*
1033 : : * Can it merge with the predecessor?
1034 : : */
1035 [ + + ][ + + ]: 2074190 : if (prev && prev->vm_end == addr &&
1036 [ + + ]: 395077 : mpol_equal(vma_policy(prev), policy) &&
1037 : 395070 : can_vma_merge_after(prev, vm_flags, anon_vma,
1038 : : file, pgoff, anon_name)) {
1039 : : /*
1040 : : * OK, it can. Can we now merge in the successor as well?
1041 : : */
1042 [ + + ][ + + ]: 12336 : if (next && end == next->vm_start &&
1043 [ + + ]: 4394 : mpol_equal(policy, vma_policy(next)) &&
1044 : 4394 : can_vma_merge_before(next, vm_flags, anon_vma,
1045 [ + + ]: 3014 : file, pgoff+pglen, anon_name) &&
1046 : 3014 : is_mergeable_anon_vma(prev->anon_vma,
1047 : : next->anon_vma, NULL)) {
1048 : : /* cases 1, 6 */
1049 : 2703 : err = vma_adjust(prev, prev->vm_start,
1050 : : next->vm_end, prev->vm_pgoff, NULL);
1051 : : } else /* cases 2, 5, 7 */
1052 : 9633 : err = vma_adjust(prev, prev->vm_start,
1053 : : end, prev->vm_pgoff, NULL);
1054 [ + - ]: 12336 : if (err)
1055 : : return NULL;
1056 : : khugepaged_enter_vma_merge(prev);
1057 : 12336 : return prev;
1058 : : }
1059 : :
1060 : : /*
1061 : : * Can this new request be merged in front of next?
1062 : : */
1063 [ + + ][ + + ]: 2061861 : if (next && end == next->vm_start &&
1064 [ + + ]: 1659648 : mpol_equal(policy, vma_policy(next)) &&
1065 : 1659689 : can_vma_merge_before(next, vm_flags, anon_vma,
1066 : : file, pgoff+pglen, anon_name)) {
1067 [ + + ][ - + ]: 163741 : if (prev && addr < prev->vm_end) /* case 4 */
1068 : 0 : err = vma_adjust(prev, prev->vm_start,
1069 : : addr, prev->vm_pgoff, NULL);
1070 : : else /* cases 3, 8 */
1071 : 163741 : err = vma_adjust(area, addr, next->vm_end,
1072 : 163741 : next->vm_pgoff - pglen, NULL);
1073 [ + - ]: 163742 : if (err)
1074 : : return NULL;
1075 : : khugepaged_enter_vma_merge(area);
1076 : 163742 : return area;
1077 : : }
1078 : :
1079 : : return NULL;
1080 : : }
1081 : :
1082 : : /*
1083 : : * Rough compatbility check to quickly see if it's even worth looking
1084 : : * at sharing an anon_vma.
1085 : : *
1086 : : * They need to have the same vm_file, and the flags can only differ
1087 : : * in things that mprotect may change.
1088 : : *
1089 : : * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1090 : : * we can merge the two vma's. For example, we refuse to merge a vma if
1091 : : * there is a vm_ops->close() function, because that indicates that the
1092 : : * driver is doing some kind of reference counting. But that doesn't
1093 : : * really matter for the anon_vma sharing case.
1094 : : */
1095 : 0 : static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
1096 : : {
1097 : 547352 : return a->vm_end == b->vm_start &&
1098 [ + + ]: 327341 : mpol_equal(vma_policy(a), vma_policy(b)) &&
1099 [ + + ]: 167568 : a->vm_file == b->vm_file &&
1100 [ + + ][ + + ]: 874693 : !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
1101 : 4191 : b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1102 : : }
1103 : :
1104 : : /*
1105 : : * Do some basic sanity checking to see if we can re-use the anon_vma
1106 : : * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1107 : : * the same as 'old', the other will be the new one that is trying
1108 : : * to share the anon_vma.
1109 : : *
1110 : : * NOTE! This runs with mm_sem held for reading, so it is possible that
1111 : : * the anon_vma of 'old' is concurrently in the process of being set up
1112 : : * by another page fault trying to merge _that_. But that's ok: if it
1113 : : * is being set up, that automatically means that it will be a singleton
1114 : : * acceptable for merging, so we can do all of this optimistically. But
1115 : : * we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
1116 : : *
1117 : : * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1118 : : * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1119 : : * is to return an anon_vma that is "complex" due to having gone through
1120 : : * a fork).
1121 : : *
1122 : : * We also make sure that the two vma's are compatible (adjacent,
1123 : : * and with the same memory policies). That's all stable, even with just
1124 : : * a read lock on the mm_sem.
1125 : : */
1126 : 0 : static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1127 : : {
1128 [ + + ]: 547359 : if (anon_vma_compatible(a, b)) {
1129 : 4191 : struct anon_vma *anon_vma = ACCESS_ONCE(old->anon_vma);
1130 : :
1131 [ + + ][ + ]: 7563 : if (anon_vma && list_is_singular(&old->anon_vma_chain))
1132 : 2097 : return anon_vma;
1133 : : }
1134 : : return NULL;
1135 : : }
1136 : :
1137 : : /*
1138 : : * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1139 : : * neighbouring vmas for a suitable anon_vma, before it goes off
1140 : : * to allocate a new anon_vma. It checks because a repetitive
1141 : : * sequence of mprotects and faults may otherwise lead to distinct
1142 : : * anon_vmas being allocated, preventing vma merge in subsequent
1143 : : * mprotect.
1144 : : */
1145 : 0 : struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
1146 : : {
1147 : : struct anon_vma *anon_vma;
1148 : : struct vm_area_struct *near;
1149 : :
1150 : 302388 : near = vma->vm_next;
1151 [ + + ]: 302388 : if (!near)
1152 : : goto try_prev;
1153 : :
1154 : 275135 : anon_vma = reusable_anon_vma(near, vma, near);
1155 [ + + ]: 275122 : if (anon_vma)
1156 : : return anon_vma;
1157 : : try_prev:
1158 : 300276 : near = vma->vm_prev;
1159 [ + + ]: 300276 : if (!near)
1160 : : goto none;
1161 : :
1162 : 272242 : anon_vma = reusable_anon_vma(near, near, vma);
1163 [ - + ]: 272257 : if (anon_vma)
1164 : 0 : return anon_vma;
1165 : : none:
1166 : : /*
1167 : : * There's no absolute need to look only at touching neighbours:
1168 : : * we could search further afield for "compatible" anon_vmas.
1169 : : * But it would probably just be a waste of time searching,
1170 : : * or lead to too many vmas hanging off the same anon_vma.
1171 : : * We're trying to allow mprotect remerging later on,
1172 : : * not trying to minimize memory used for anon_vmas.
1173 : : */
1174 : : return NULL;
1175 : : }
1176 : :
1177 : : #ifdef CONFIG_PROC_FS
1178 : 0 : void vm_stat_account(struct mm_struct *mm, unsigned long flags,
1179 : : struct file *file, long pages)
1180 : : {
1181 : : const unsigned long stack_flags
1182 : : = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
1183 : :
1184 : 4719228 : mm->total_vm += pages;
1185 : :
1186 [ + + ]: 4719228 : if (file) {
[ - + + + ]
[ + + ]
1187 : 2300613 : mm->shared_vm += pages;
1188 [ + + ][ # # ]: 2300613 : if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
[ + + ][ + + ]
1189 : 408741 : mm->exec_vm += pages;
1190 [ - + ][ + ]: 2418615 : } else if (flags & stack_flags)
[ - + ][ - + ]
1191 : 2013488 : mm->stack_vm += pages;
1192 : 0 : }
1193 : : #endif /* CONFIG_PROC_FS */
1194 : :
1195 : : /*
1196 : : * If a hint addr is less than mmap_min_addr change hint to be as
1197 : : * low as possible but still greater than mmap_min_addr
1198 : : */
1199 : : static inline unsigned long round_hint_to_min(unsigned long hint)
1200 : : {
1201 : 1592479 : hint &= PAGE_MASK;
1202 [ + + ][ - + ]: 1592479 : if (((void *)hint != NULL) &&
1203 : 134 : (hint < mmap_min_addr))
1204 : 1592479 : return PAGE_ALIGN(mmap_min_addr);
1205 : : return hint;
1206 : : }
1207 : :
1208 : : static inline int mlock_future_check(struct mm_struct *mm,
1209 : : unsigned long flags,
1210 : : unsigned long len)
1211 : : {
1212 : : unsigned long locked, lock_limit;
1213 : :
1214 : : /* mlock MCL_FUTURE? */
1215 [ + + ][ + + ]: 1891682 : if (flags & VM_LOCKED) {
1216 : 1 : locked = len >> PAGE_SHIFT;
1217 : 180 : locked += mm->locked_vm;
1218 : : lock_limit = rlimit(RLIMIT_MEMLOCK);
1219 : 180 : lock_limit >>= PAGE_SHIFT;
1220 [ + - ][ + - ]: 180 : if (locked > lock_limit && !capable(CAP_IPC_LOCK))
[ + - ][ + - ]
1221 : : return -EAGAIN;
1222 : : }
1223 : : return 0;
1224 : : }
1225 : :
1226 : : /*
1227 : : * The caller must hold down_write(¤t->mm->mmap_sem).
1228 : : */
1229 : :
1230 : 0 : unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1231 : : unsigned long len, unsigned long prot,
1232 : : unsigned long flags, unsigned long pgoff,
1233 : : unsigned long *populate)
1234 : : {
1235 : 1804959 : struct mm_struct * mm = current->mm;
1236 : : vm_flags_t vm_flags;
1237 : :
1238 : 1804959 : *populate = 0;
1239 : :
1240 : : /*
1241 : : * Does the application expect PROT_READ to imply PROT_EXEC?
1242 : : *
1243 : : * (the exception is when the underlying filesystem is noexec
1244 : : * mounted, in which case we dont add PROT_EXEC.)
1245 : : */
1246 [ + + ][ - + ]: 1804959 : if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1247 [ # # ][ # # ]: 0 : if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
1248 : 0 : prot |= PROT_EXEC;
1249 : :
1250 [ + ]: 1804959 : if (!len)
1251 : : return -EINVAL;
1252 : :
1253 [ + + ]: 1805004 : if (!(flags & MAP_FIXED))
1254 : : addr = round_hint_to_min(addr);
1255 : :
1256 : : /* Careful about overflows.. */
1257 : 1805004 : len = PAGE_ALIGN(len);
1258 [ + + ]: 1805004 : if (!len)
1259 : : return -ENOMEM;
1260 : :
1261 : : /* offset overflow? */
1262 [ + ]: 1804991 : if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1263 : : return -EOVERFLOW;
1264 : :
1265 : : /* Too many mappings? */
1266 [ + + ]: 1805009 : if (mm->map_count > sysctl_max_map_count)
1267 : : return -ENOMEM;
1268 : :
1269 : : /* Obtain the address to map to. we verify (or select) it and ensure
1270 : : * that it represents a valid section of the address space.
1271 : : */
1272 : 1804956 : addr = get_unmapped_area(file, addr, len, pgoff, flags);
1273 [ + + ]: 1804883 : if (addr & ~PAGE_MASK)
1274 : : return addr;
1275 : :
1276 : : /* Do simple checking here so the lower-level routines won't have
1277 : : * to. we assume access permissions have been handled by the open
1278 : : * of the memory object, so we don't do any here.
1279 : : */
1280 : 3563624 : vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
1281 : 1781812 : mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1282 : :
1283 [ + + ]: 1781812 : if (flags & MAP_LOCKED)
1284 [ + ]: 1 : if (!can_do_mlock())
1285 : : return -EPERM;
1286 : :
1287 [ + - ]: 1781868 : if (mlock_future_check(mm, vm_flags, len))
1288 : : return -EAGAIN;
1289 : :
1290 [ + + ]: 1781868 : if (file) {
1291 : : struct inode *inode = file_inode(file);
1292 : :
1293 [ + + + ]: 647332 : switch (flags & MAP_TYPE) {
1294 : : case MAP_SHARED:
1295 [ + + ][ + ]: 102522 : if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1296 : : return -EACCES;
1297 : :
1298 : : /*
1299 : : * Make sure we don't allow writing to an append-only
1300 : : * file..
1301 : : */
1302 [ - + ][ # # ]: 1907482 : if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1303 : : return -EACCES;
1304 : :
1305 : : /*
1306 : : * Make sure there are no mandatory locks on the file.
1307 : : */
1308 [ + - ]: 102523 : if (locks_verify_locked(inode))
1309 : : return -EAGAIN;
1310 : :
1311 : 102523 : vm_flags |= VM_SHARED | VM_MAYSHARE;
1312 [ + + ]: 102523 : if (!(file->f_mode & FMODE_WRITE))
1313 : 28187 : vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1314 : :
1315 : : /* fall through */
1316 : : case MAP_PRIVATE:
1317 [ + + ]: 647294 : if (!(file->f_mode & FMODE_READ))
1318 : : return -EACCES;
1319 [ - + ]: 647292 : if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
1320 [ # # ]: 0 : if (vm_flags & VM_EXEC)
1321 : : return -EPERM;
1322 : 0 : vm_flags &= ~VM_MAYEXEC;
1323 : : }
1324 : :
1325 [ + + ]: 647292 : if (!file->f_op->mmap)
1326 : : return -ENODEV;
1327 [ + ]: 647258 : if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1328 : : return -EINVAL;
1329 : : break;
1330 : :
1331 : : default:
1332 : : return -EINVAL;
1333 : : }
1334 : : } else {
1335 [ + + - ]: 1134536 : switch (flags & MAP_TYPE) {
1336 : : case MAP_SHARED:
1337 [ + - ]: 872599 : if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1338 : : return -EINVAL;
1339 : : /*
1340 : : * Ignore pgoff.
1341 : : */
1342 : : pgoff = 0;
1343 : 872599 : vm_flags |= VM_SHARED | VM_MAYSHARE;
1344 : 872599 : break;
1345 : : case MAP_PRIVATE:
1346 : : /*
1347 : : * Set pgoff according to addr for anon_vma.
1348 : : */
1349 : 261937 : pgoff = addr >> PAGE_SHIFT;
1350 : 261937 : break;
1351 : : default:
1352 : : return -EINVAL;
1353 : : }
1354 : : }
1355 : :
1356 : : /*
1357 : : * Set 'VM_NORESERVE' if we should not account for the
1358 : : * memory use of this mapping.
1359 : : */
1360 [ + + ]: 1781813 : if (flags & MAP_NORESERVE) {
1361 : : /* We honor MAP_NORESERVE if allowed to overcommit */
1362 [ + - ]: 702 : if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1363 : 702 : vm_flags |= VM_NORESERVE;
1364 : :
1365 : : /* hugetlb applies strict overcommit unless MAP_NORESERVE */
1366 : : if (file && is_file_hugepages(file))
1367 : : vm_flags |= VM_NORESERVE;
1368 : : }
1369 : :
1370 : 1781813 : addr = mmap_region(file, addr, len, vm_flags, pgoff);
1371 [ + + ][ + + ]: 1781858 : if (!IS_ERR_VALUE(addr) &&
1372 [ + + ]: 1781653 : ((vm_flags & VM_LOCKED) ||
1373 : 1781653 : (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
1374 : 188 : *populate = len;
1375 : 1781858 : return addr;
1376 : : }
1377 : :
1378 : 0 : SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1379 : : unsigned long, prot, unsigned long, flags,
1380 : : unsigned long, fd, unsigned long, pgoff)
1381 : : {
1382 : : struct file *file = NULL;
1383 : : unsigned long retval = -EBADF;
1384 : :
1385 [ + + ]: 1694915 : if (!(flags & MAP_ANONYMOUS)) {
1386 : : audit_mmap_fd(fd, flags);
1387 : 537276 : file = fget(fd);
1388 [ + + ]: 537263 : if (!file)
1389 : : goto out;
1390 : : if (is_file_hugepages(file))
1391 : : len = ALIGN(len, huge_page_size(hstate_file(file)));
1392 : : retval = -EINVAL;
1393 [ + + ]: 537250 : if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
1394 : : goto out_fput;
1395 [ + + ]: 1157639 : } else if (flags & MAP_HUGETLB) {
1396 : : struct user_struct *user = NULL;
1397 : : struct hstate *hs;
1398 : :
1399 : : hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & SHM_HUGE_MASK);
1400 : : if (!hs)
1401 : : return -EINVAL;
1402 : :
1403 : : len = ALIGN(len, huge_page_size(hs));
1404 : : /*
1405 : : * VM_NORESERVE is used because the reservations will be
1406 : : * taken when vm_ops->mmap() is called
1407 : : * A dummy user value is used because we are not locking
1408 : : * memory so no accounting is necessary
1409 : : */
1410 : : file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1411 : : VM_NORESERVE,
1412 : : &user, HUGETLB_ANONHUGE_INODE,
1413 : : (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1414 : : if (IS_ERR(file))
1415 : : return PTR_ERR(file);
1416 : : }
1417 : :
1418 : 1694874 : flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1419 : :
1420 : 1694874 : retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1421 : : out_fput:
1422 [ + + ]: 1694953 : if (file)
1423 : 537289 : fput(file);
1424 : : out:
1425 : 1694962 : return retval;
1426 : : }
1427 : :
1428 : : #ifdef __ARCH_WANT_SYS_OLD_MMAP
1429 : : struct mmap_arg_struct {
1430 : : unsigned long addr;
1431 : : unsigned long len;
1432 : : unsigned long prot;
1433 : : unsigned long flags;
1434 : : unsigned long fd;
1435 : : unsigned long offset;
1436 : : };
1437 : :
1438 : 0 : SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1439 : : {
1440 : : struct mmap_arg_struct a;
1441 : :
1442 [ # # ]: 0 : if (copy_from_user(&a, arg, sizeof(a)))
1443 : : return -EFAULT;
1444 [ # # ]: 0 : if (a.offset & ~PAGE_MASK)
1445 : : return -EINVAL;
1446 : :
1447 : 0 : return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1448 : : a.offset >> PAGE_SHIFT);
1449 : : }
1450 : : #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1451 : :
1452 : : /*
1453 : : * Some shared mappigns will want the pages marked read-only
1454 : : * to track write events. If so, we'll downgrade vm_page_prot
1455 : : * to the private version (using protection_map[] without the
1456 : : * VM_SHARED bit).
1457 : : */
1458 : 0 : int vma_wants_writenotify(struct vm_area_struct *vma)
1459 : : {
1460 : 1861973 : vm_flags_t vm_flags = vma->vm_flags;
1461 : :
1462 : : /* If it was private or non-writable, the write bit is already clear */
1463 [ + + ]: 1861973 : if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
1464 : : return 0;
1465 : :
1466 : : /* The backer wishes to know when pages are first written to? */
1467 [ + - ][ + + ]: 72342 : if (vma->vm_ops && vma->vm_ops->page_mkwrite)
1468 : : return 1;
1469 : :
1470 : : /* The open routine did something to the protections already? */
1471 [ + - ]: 1086 : if (pgprot_val(vma->vm_page_prot) !=
1472 : : pgprot_val(vm_get_page_prot(vm_flags)))
1473 : : return 0;
1474 : :
1475 : : /* Specialty mapping? */
1476 [ + ]: 1086 : if (vm_flags & VM_PFNMAP)
1477 : : return 0;
1478 : :
1479 : : /* Can the mapping track the dirty pages? */
1480 [ + + ][ + - ]: 1863067 : return vma->vm_file && vma->vm_file->f_mapping &&
[ + + ]
1481 : : mapping_cap_account_dirty(vma->vm_file->f_mapping);
1482 : : }
1483 : :
1484 : : /*
1485 : : * We account for memory if it's a private writeable mapping,
1486 : : * not hugepages and VM_NORESERVE wasn't set.
1487 : : */
1488 : : static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
1489 : : {
1490 : : /*
1491 : : * hugetlb has its own accounting separate from the core VM
1492 : : * VM_HUGETLB may not be set yet so we cannot check for that flag.
1493 : : */
1494 : : if (file && is_file_hugepages(file))
1495 : : return 0;
1496 : :
1497 : 1781853 : return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1498 : : }
1499 : :
1500 : 0 : unsigned long mmap_region(struct file *file, unsigned long addr,
1501 : : unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
1502 : : {
1503 : 1781818 : struct mm_struct *mm = current->mm;
1504 : : struct vm_area_struct *vma, *prev;
1505 : : int error;
1506 : : struct rb_node **rb_link, *rb_parent;
1507 : : unsigned long charged = 0;
1508 : :
1509 : : /* Check against address space limit. */
1510 [ - + ]: 1781818 : if (!may_expand_vm(mm, len >> PAGE_SHIFT)) {
1511 : : unsigned long nr_pages;
1512 : :
1513 : : /*
1514 : : * MAP_FIXED may remove pages of mappings that intersects with
1515 : : * requested mapping. Account for the pages it would unmap.
1516 : : */
1517 [ # # ]: 0 : if (!(vm_flags & MAP_FIXED))
1518 : : return -ENOMEM;
1519 : :
1520 : 0 : nr_pages = count_vma_pages_range(mm, addr, addr + len);
1521 : :
1522 [ # # ]: 1781818 : if (!may_expand_vm(mm, (len >> PAGE_SHIFT) - nr_pages))
1523 : : return -ENOMEM;
1524 : : }
1525 : :
1526 : : /* Clear old maps */
1527 : : error = -ENOMEM;
1528 : : munmap_back:
1529 [ + + ]: 1913141 : if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
1530 [ + ]: 131288 : if (do_munmap(mm, addr, len))
1531 : : return -ENOMEM;
1532 : : goto munmap_back;
1533 : : }
1534 : :
1535 : : /*
1536 : : * Private writable mapping: check memory availability
1537 : : */
1538 [ + + ]: 1781853 : if (accountable_mapping(file, vm_flags)) {
1539 : : charged = len >> PAGE_SHIFT;
1540 [ + + ]: 412042 : if (security_vm_enough_memory_mm(mm, charged))
1541 : : return -ENOMEM;
1542 : 412030 : vm_flags |= VM_ACCOUNT;
1543 : : }
1544 : :
1545 : : /*
1546 : : * Can we just expand an old mapping?
1547 : : */
1548 : 1781841 : vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff,
1549 : : NULL, NULL);
1550 [ + + ]: 1781812 : if (vma)
1551 : : goto out;
1552 : :
1553 : : /*
1554 : : * Determine the object being mapped and call the appropriate
1555 : : * specific mapper. the address has already been validated, but
1556 : : * not unmapped, but the maps are removed from the list.
1557 : : */
1558 : 1611200 : vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1559 [ + + ]: 1611259 : if (!vma) {
1560 : : error = -ENOMEM;
1561 : : goto unacct_error;
1562 : : }
1563 : :
1564 : 1611218 : vma->vm_mm = mm;
1565 : 1611218 : vma->vm_start = addr;
1566 : 1611218 : vma->vm_end = addr + len;
1567 : 1611218 : vma->vm_flags = vm_flags;
1568 : 1611218 : vma->vm_page_prot = vm_get_page_prot(vm_flags);
1569 : 1611218 : vma->vm_pgoff = pgoff;
1570 : 1611218 : INIT_LIST_HEAD(&vma->anon_vma_chain);
1571 : :
1572 [ + + ]: 1611218 : if (file) {
1573 [ + + ]: 647190 : if (vm_flags & VM_DENYWRITE) {
1574 : : error = deny_write_access(file);
1575 [ + ]: 108949 : if (error)
1576 : : goto free_vma;
1577 : : }
1578 : 647212 : vma->vm_file = get_file(file);
1579 : 647212 : error = file->f_op->mmap(file, vma);
1580 [ + - ]: 647216 : if (error)
1581 : : goto unmap_and_free_vma;
1582 : :
1583 : : /* Can addr have changed??
1584 : : *
1585 : : * Answer: Yes, several device drivers can do it in their
1586 : : * f_op->mmap method. -DaveM
1587 : : * Bug: If addr is changed, prev, rb_link, rb_parent should
1588 : : * be updated for vma_link()
1589 : : */
1590 [ - + ][ # # ]: 647216 : WARN_ON_ONCE(addr != vma->vm_start);
[ - ]
1591 : :
1592 : 647215 : addr = vma->vm_start;
1593 : 647215 : vm_flags = vma->vm_flags;
1594 [ + + ]: 964028 : } else if (vm_flags & VM_SHARED) {
1595 : 872599 : error = shmem_zero_setup(vma);
1596 [ + - ]: 872599 : if (error)
1597 : : goto free_vma;
1598 : : }
1599 : :
1600 [ + + ]: 1611243 : if (vma_wants_writenotify(vma)) {
1601 : 71233 : pgprot_t pprot = vma->vm_page_prot;
1602 : :
1603 : : /* Can vma->vm_page_prot have changed??
1604 : : *
1605 : : * Answer: Yes, drivers may have changed it in their
1606 : : * f_op->mmap method.
1607 : : *
1608 : : * Ensures that vmas marked as uncached stay that way.
1609 : : */
1610 : 71233 : vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
1611 [ - + ]: 71233 : if (pgprot_val(pprot) == pgprot_val(pgprot_noncached(pprot)))
1612 : 0 : vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1613 : : }
1614 : :
1615 : 1611223 : vma_link(mm, vma, prev, rb_link, rb_parent);
1616 : : /* Once vma denies write, undo our temporary denial count */
1617 [ + + ]: 1611284 : if (vm_flags & VM_DENYWRITE)
1618 : : allow_write_access(file);
1619 : 1611311 : file = vma->vm_file;
1620 : : out:
1621 : 1781923 : perf_event_mmap(vma);
1622 : :
1623 : : vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
1624 [ + + ]: 1781876 : if (vm_flags & VM_LOCKED) {
1625 [ + - + - ]: 358 : if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
1626 : 179 : vma == get_gate_vma(current->mm)))
1627 : 179 : mm->locked_vm += (len >> PAGE_SHIFT);
1628 : : else
1629 : 0 : vma->vm_flags &= ~VM_LOCKED;
1630 : : }
1631 : :
1632 [ + + ]: 1781876 : if (file)
1633 : 1519952 : uprobe_mmap(vma);
1634 : :
1635 : : /*
1636 : : * New (or expanded) vma always get soft dirty status.
1637 : : * Otherwise user-space soft-dirty page tracker won't
1638 : : * be able to distinguish situation when vma area unmapped,
1639 : : * then new mapped in-place (which must be aimed as
1640 : : * a completely new data area).
1641 : : */
1642 : : vma->vm_flags |= VM_SOFTDIRTY;
1643 : :
1644 : 1781862 : return addr;
1645 : :
1646 : : unmap_and_free_vma:
1647 [ # # ]: 0 : if (vm_flags & VM_DENYWRITE)
1648 : : allow_write_access(file);
1649 : 0 : vma->vm_file = NULL;
1650 : 0 : fput(file);
1651 : :
1652 : : /* Undo any partial mapping done by a device driver. */
1653 : 0 : unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1654 : : charged = 0;
1655 : : free_vma:
1656 : 0 : kmem_cache_free(vm_area_cachep, vma);
1657 : : unacct_error:
1658 [ # # ]: 0 : if (charged)
1659 : 0 : vm_unacct_memory(charged);
1660 : 0 : return error;
1661 : : }
1662 : :
1663 : 0 : unsigned long unmapped_area(struct vm_unmapped_area_info *info)
1664 : : {
1665 : : /*
1666 : : * We implement the search by looking for an rbtree node that
1667 : : * immediately follows a suitable gap. That is,
1668 : : * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1669 : : * - gap_end = vma->vm_start >= info->low_limit + length;
1670 : : * - gap_end - gap_start >= length
1671 : : */
1672 : :
1673 : 34445 : struct mm_struct *mm = current->mm;
1674 : : struct vm_area_struct *vma;
1675 : : unsigned long length, low_limit, high_limit, gap_start, gap_end;
1676 : :
1677 : : /* Adjust search length to account for worst case alignment overhead */
1678 : 34445 : length = info->length + info->align_mask;
1679 [ + ]: 34445 : if (length < info->length)
1680 : : return -ENOMEM;
1681 : :
1682 : : /* Adjust search limits by the desired length */
1683 [ + - ]: 34448 : if (info->high_limit < length)
1684 : : return -ENOMEM;
1685 : 34448 : high_limit = info->high_limit - length;
1686 : :
1687 [ + + ]: 34448 : if (info->low_limit > high_limit)
1688 : : return -ENOMEM;
1689 : 31208 : low_limit = info->low_limit + length;
1690 : :
1691 : : /* Check if rbtree root looks promising */
1692 [ + - ]: 31208 : if (RB_EMPTY_ROOT(&mm->mm_rb))
1693 : : goto check_highest;
1694 : 31208 : vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1695 [ + ]: 80302 : if (vma->rb_subtree_gap < length)
1696 : : goto check_highest;
1697 : :
1698 : : while (true) {
1699 : : /* Visit left subtree if it looks promising */
1700 : 60452 : gap_end = vma->vm_start;
1701 [ + + ][ + + ]: 60452 : if (gap_end >= low_limit && vma->vm_rb.rb_left) {
1702 : 32269 : struct vm_area_struct *left =
1703 : : rb_entry(vma->vm_rb.rb_left,
1704 : : struct vm_area_struct, vm_rb);
1705 [ + + ]: 32269 : if (left->rb_subtree_gap >= length) {
1706 : : vma = left;
1707 : 18256 : continue;
1708 : : }
1709 : : }
1710 : :
1711 [ + + ]: 42196 : gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1712 : : check_current:
1713 : : /* Check if current node has a suitable gap */
1714 [ + + ]: 56731 : if (gap_start > high_limit)
1715 : : return -ENOMEM;
1716 [ + + ][ + + ]: 56722 : if (gap_end >= low_limit && gap_end - gap_start >= length)
1717 : : goto found;
1718 : :
1719 : : /* Visit right subtree if it looks promising */
1720 [ + + ]: 45373 : if (vma->vm_rb.rb_right) {
1721 : 38558 : struct vm_area_struct *right =
1722 : : rb_entry(vma->vm_rb.rb_right,
1723 : : struct vm_area_struct, vm_rb);
1724 [ + + ]: 38558 : if (right->rb_subtree_gap >= length) {
1725 : : vma = right;
1726 : 45373 : continue;
1727 : : }
1728 : : }
1729 : :
1730 : : /* Go back up the rbtree to find next candidate node */
1731 : : while (true) {
1732 : 23227 : struct rb_node *prev = &vma->vm_rb;
1733 [ + - ]: 23227 : if (!rb_parent(prev))
1734 : : goto check_highest;
1735 : 23227 : vma = rb_entry(rb_parent(prev),
1736 : : struct vm_area_struct, vm_rb);
1737 [ + + ]: 23227 : if (prev == vma->vm_rb.rb_left) {
1738 : 14535 : gap_start = vma->vm_prev->vm_end;
1739 : 14535 : gap_end = vma->vm_start;
1740 : 14535 : goto check_current;
1741 : : }
1742 : : }
1743 : : }
1744 : :
1745 : : check_highest:
1746 : : /* Check highest gap, which does not precede any rbtree node */
1747 : 0 : gap_start = mm->highest_vm_end;
1748 : : gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */
1749 [ + ]: 0 : if (gap_start > high_limit)
1750 : : return -ENOMEM;
1751 : :
1752 : : found:
1753 : : /* We found a suitable gap. Clip it with the original low_limit. */
1754 [ + + ]: 11361 : if (gap_start < info->low_limit)
1755 : : gap_start = info->low_limit;
1756 : :
1757 : : /* Adjust gap address to the desired alignment */
1758 : 11361 : gap_start += (info->align_offset - gap_start) & info->align_mask;
1759 : :
1760 : : VM_BUG_ON(gap_start + info->length > info->high_limit);
1761 : : VM_BUG_ON(gap_start + info->length > gap_end);
1762 : 11361 : return gap_start;
1763 : : }
1764 : :
1765 : 0 : unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
1766 : : {
1767 : 1608317 : struct mm_struct *mm = current->mm;
1768 : : struct vm_area_struct *vma;
1769 : : unsigned long length, low_limit, high_limit, gap_start, gap_end;
1770 : :
1771 : : /* Adjust search length to account for worst case alignment overhead */
1772 : 1608317 : length = info->length + info->align_mask;
1773 [ + ]: 1608317 : if (length < info->length)
1774 : : return -ENOMEM;
1775 : :
1776 : : /*
1777 : : * Adjust search limits by the desired length.
1778 : : * See implementation comment at top of unmapped_area().
1779 : : */
1780 : 1608339 : gap_end = info->high_limit;
1781 [ + + ]: 1608339 : if (gap_end < length)
1782 : : return -ENOMEM;
1783 : 1608336 : high_limit = gap_end - length;
1784 : :
1785 [ + ]: 1608336 : if (info->low_limit > high_limit)
1786 : : return -ENOMEM;
1787 : 1608355 : low_limit = info->low_limit + length;
1788 : :
1789 : : /* Check highest gap, which does not precede any rbtree node */
1790 : 1608355 : gap_start = mm->highest_vm_end;
1791 [ + + ]: 1608355 : if (gap_start <= high_limit)
1792 : : goto found_highest;
1793 : :
1794 : : /* Check if rbtree root looks promising */
1795 [ + + ]: 1608335 : if (RB_EMPTY_ROOT(&mm->mm_rb))
1796 : : return -ENOMEM;
1797 : 1608322 : vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1798 [ + ]: 38096111 : if (vma->rb_subtree_gap < length)
1799 : : return -ENOMEM;
1800 : :
1801 : : while (true) {
1802 : : /* Visit right subtree if it looks promising */
1803 [ + + ]: 38072823 : gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1804 [ + + ][ + + ]: 38072823 : if (gap_start <= high_limit && vma->vm_rb.rb_right) {
1805 : 35484100 : struct vm_area_struct *right =
1806 : : rb_entry(vma->vm_rb.rb_right,
1807 : : struct vm_area_struct, vm_rb);
1808 [ + + ]: 38072823 : if (right->rb_subtree_gap >= length) {
1809 : : vma = right;
1810 : 28954623 : continue;
1811 : : }
1812 : : }
1813 : :
1814 : : check_current:
1815 : : /* Check if current node has a suitable gap */
1816 : 35682788 : gap_end = vma->vm_start;
1817 [ + ]: 35682788 : if (gap_end < low_limit)
1818 : : return -ENOMEM;
1819 [ + + ][ + + ]: 35683004 : if (gap_start <= high_limit && gap_end - gap_start >= length)
1820 : : goto found;
1821 : :
1822 : : /* Visit left subtree if it looks promising */
1823 [ + + ]: 34097858 : if (vma->vm_rb.rb_left) {
1824 : 32226397 : struct vm_area_struct *left =
1825 : : rb_entry(vma->vm_rb.rb_left,
1826 : : struct vm_area_struct, vm_rb);
1827 [ + + ]: 32226397 : if (left->rb_subtree_gap >= length) {
1828 : : vma = left;
1829 : 34097858 : continue;
1830 : : }
1831 : : }
1832 : :
1833 : : /* Go back up the rbtree to find next candidate node */
1834 : : while (true) {
1835 : 13283076 : struct rb_node *prev = &vma->vm_rb;
1836 [ + + ]: 13283076 : if (!rb_parent(prev))
1837 : : return -ENOMEM;
1838 : 13282972 : vma = rb_entry(rb_parent(prev),
1839 : : struct vm_area_struct, vm_rb);
1840 [ + + ]: 13282972 : if (prev == vma->vm_rb.rb_right) {
1841 : 13282294 : gap_start = vma->vm_prev ?
1842 [ + + ]: 13282294 : vma->vm_prev->vm_end : 0;
1843 : : goto check_current;
1844 : : }
1845 : : }
1846 : : }
1847 : :
1848 : : found:
1849 : : /* We found a suitable gap. Clip it with the original high_limit. */
1850 [ + + ]: 1585146 : if (gap_end > info->high_limit)
1851 : : gap_end = info->high_limit;
1852 : :
1853 : : found_highest:
1854 : : /* Compute highest gap address at the desired alignment */
1855 : 1585166 : gap_end -= info->length;
1856 : 1585166 : gap_end -= (gap_end - info->align_offset) & info->align_mask;
1857 : :
1858 : : VM_BUG_ON(gap_end < info->low_limit);
1859 : : VM_BUG_ON(gap_end < gap_start);
1860 : 1585166 : return gap_end;
1861 : : }
1862 : :
1863 : : /* Get an address range which is currently unmapped.
1864 : : * For shmat() with addr=0.
1865 : : *
1866 : : * Ugly calling convention alert:
1867 : : * Return value with the low bits set means error value,
1868 : : * ie
1869 : : * if (ret & ~PAGE_MASK)
1870 : : * error = ret;
1871 : : *
1872 : : * This function "knows" that -ENOMEM has the bits set.
1873 : : */
1874 : : #ifndef HAVE_ARCH_UNMAPPED_AREA
1875 : : unsigned long
1876 : : arch_get_unmapped_area(struct file *filp, unsigned long addr,
1877 : : unsigned long len, unsigned long pgoff, unsigned long flags)
1878 : : {
1879 : : struct mm_struct *mm = current->mm;
1880 : : struct vm_area_struct *vma;
1881 : : struct vm_unmapped_area_info info;
1882 : :
1883 : : if (len > TASK_SIZE - mmap_min_addr)
1884 : : return -ENOMEM;
1885 : :
1886 : : if (flags & MAP_FIXED)
1887 : : return addr;
1888 : :
1889 : : if (addr) {
1890 : : addr = PAGE_ALIGN(addr);
1891 : : vma = find_vma(mm, addr);
1892 : : if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1893 : : (!vma || addr + len <= vma->vm_start))
1894 : : return addr;
1895 : : }
1896 : :
1897 : : info.flags = 0;
1898 : : info.length = len;
1899 : : info.low_limit = mm->mmap_base;
1900 : : info.high_limit = TASK_SIZE;
1901 : : info.align_mask = 0;
1902 : : return vm_unmapped_area(&info);
1903 : : }
1904 : : #endif
1905 : :
1906 : : /*
1907 : : * This mmap-allocator allocates new areas top-down from below the
1908 : : * stack's low limit (the base):
1909 : : */
1910 : : #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1911 : : unsigned long
1912 : : arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1913 : : const unsigned long len, const unsigned long pgoff,
1914 : : const unsigned long flags)
1915 : : {
1916 : : struct vm_area_struct *vma;
1917 : : struct mm_struct *mm = current->mm;
1918 : : unsigned long addr = addr0;
1919 : : struct vm_unmapped_area_info info;
1920 : :
1921 : : /* requested length too big for entire address space */
1922 : : if (len > TASK_SIZE - mmap_min_addr)
1923 : : return -ENOMEM;
1924 : :
1925 : : if (flags & MAP_FIXED)
1926 : : return addr;
1927 : :
1928 : : /* requesting a specific address */
1929 : : if (addr) {
1930 : : addr = PAGE_ALIGN(addr);
1931 : : vma = find_vma(mm, addr);
1932 : : if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1933 : : (!vma || addr + len <= vma->vm_start))
1934 : : return addr;
1935 : : }
1936 : :
1937 : : info.flags = VM_UNMAPPED_AREA_TOPDOWN;
1938 : : info.length = len;
1939 : : info.low_limit = max(PAGE_SIZE, mmap_min_addr);
1940 : : info.high_limit = mm->mmap_base;
1941 : : info.align_mask = 0;
1942 : : addr = vm_unmapped_area(&info);
1943 : :
1944 : : /*
1945 : : * A failed mmap() very likely causes application failure,
1946 : : * so fall back to the bottom-up function here. This scenario
1947 : : * can happen with large stack limits and large mmap()
1948 : : * allocations.
1949 : : */
1950 : : if (addr & ~PAGE_MASK) {
1951 : : VM_BUG_ON(addr != -ENOMEM);
1952 : : info.flags = 0;
1953 : : info.low_limit = TASK_UNMAPPED_BASE;
1954 : : info.high_limit = TASK_SIZE;
1955 : : addr = vm_unmapped_area(&info);
1956 : : }
1957 : :
1958 : : return addr;
1959 : : }
1960 : : #endif
1961 : :
1962 : : unsigned long
1963 : 0 : get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
1964 : : unsigned long pgoff, unsigned long flags)
1965 : : {
1966 : : unsigned long (*get_area)(struct file *, unsigned long,
1967 : : unsigned long, unsigned long, unsigned long);
1968 : :
1969 [ + + ][ + + ]: 1873415 : unsigned long error = arch_mmap_check(addr, len, flags);
1970 [ + + ]: 1873415 : if (error)
1971 : : return error;
1972 : :
1973 : : /* Careful about overflows.. */
1974 [ + ]: 1873377 : if (len > TASK_SIZE)
1975 : : return -ENOMEM;
1976 : :
1977 : 1873386 : get_area = current->mm->get_unmapped_area;
1978 [ + + ][ - + ]: 1873386 : if (file && file->f_op->get_unmapped_area)
1979 : : get_area = file->f_op->get_unmapped_area;
1980 : 1873386 : addr = get_area(file, addr, len, pgoff, flags);
1981 [ + + ]: 1873392 : if (IS_ERR_VALUE(addr))
1982 : : return addr;
1983 : :
1984 [ + ]: 1850331 : if (addr > TASK_SIZE - len)
1985 : : return -ENOMEM;
1986 [ + ]: 1850359 : if (addr & ~PAGE_MASK)
1987 : : return -EINVAL;
1988 : :
1989 : : addr = arch_rebalance_pgtables(addr, len);
1990 : 1850373 : error = security_mmap_addr(addr);
1991 [ + + ]: 1850338 : return error ? error : addr;
1992 : : }
1993 : :
1994 : : EXPORT_SYMBOL(get_unmapped_area);
1995 : :
1996 : : /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
1997 : 0 : struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
1998 : : {
1999 : : struct vm_area_struct *vma = NULL;
2000 : :
2001 : : /* Check the cache first. */
2002 : : /* (Cache hit rate is typically around 35%.) */
2003 : 68551444 : vma = ACCESS_ONCE(mm->mmap_cache);
2004 [ + + ][ + + ]: 68551444 : if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
[ + + ]
2005 : : struct rb_node *rb_node;
2006 : :
2007 : 43134024 : rb_node = mm->mm_rb.rb_node;
2008 : : vma = NULL;
2009 : :
2010 [ + + ]: 166399709 : while (rb_node) {
2011 : : struct vm_area_struct *vma_tmp;
2012 : :
2013 : 166225985 : vma_tmp = rb_entry(rb_node,
2014 : : struct vm_area_struct, vm_rb);
2015 : :
2016 [ + + ]: 166225985 : if (vma_tmp->vm_end > addr) {
2017 : : vma = vma_tmp;
2018 [ + + ]: 89046376 : if (vma_tmp->vm_start <= addr)
2019 : : break;
2020 : 46086076 : rb_node = rb_node->rb_left;
2021 : : } else
2022 : 123265685 : rb_node = rb_node->rb_right;
2023 : : }
2024 [ + ]: 43134024 : if (vma)
2025 : 43134481 : mm->mmap_cache = vma;
2026 : : }
2027 : 0 : return vma;
2028 : : }
2029 : :
2030 : : EXPORT_SYMBOL(find_vma);
2031 : :
2032 : : /*
2033 : : * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2034 : : */
2035 : : struct vm_area_struct *
2036 : 0 : find_vma_prev(struct mm_struct *mm, unsigned long addr,
2037 : : struct vm_area_struct **pprev)
2038 : : {
2039 : : struct vm_area_struct *vma;
2040 : :
2041 : 3349 : vma = find_vma(mm, addr);
2042 [ + - ]: 3349 : if (vma) {
2043 : 3349 : *pprev = vma->vm_prev;
2044 : : } else {
2045 : 0 : struct rb_node *rb_node = mm->mm_rb.rb_node;
2046 : 0 : *pprev = NULL;
2047 [ # # ]: 0 : while (rb_node) {
2048 : 0 : *pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2049 : 0 : rb_node = rb_node->rb_right;
2050 : : }
2051 : : }
2052 : 0 : return vma;
2053 : : }
2054 : :
2055 : : /*
2056 : : * Verify that the stack growth is acceptable and
2057 : : * update accounting. This is shared with both the
2058 : : * grow-up and grow-down cases.
2059 : : */
2060 : 0 : static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
2061 : : {
2062 : 2013492 : struct mm_struct *mm = vma->vm_mm;
2063 : 2013492 : struct rlimit *rlim = current->signal->rlim;
2064 : : unsigned long new_start;
2065 : :
2066 : : /* address space limit tests */
2067 [ + + ]: 2013492 : if (!may_expand_vm(mm, grow))
2068 : : return -ENOMEM;
2069 : :
2070 : : /* Stack limit test */
2071 [ + + ]: 2013489 : if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
2072 : : return -ENOMEM;
2073 : :
2074 : : /* mlock limit tests */
2075 [ - + ]: 2013488 : if (vma->vm_flags & VM_LOCKED) {
2076 : : unsigned long locked;
2077 : : unsigned long limit;
2078 : 0 : locked = mm->locked_vm + grow;
2079 : 0 : limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
2080 : 0 : limit >>= PAGE_SHIFT;
2081 [ # # ][ # # ]: 0 : if (locked > limit && !capable(CAP_IPC_LOCK))
2082 : : return -ENOMEM;
2083 : : }
2084 : :
2085 : : /* Check to ensure the stack will not grow into a hugetlb-only region */
2086 : : new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
2087 : : vma->vm_end - size;
2088 : : if (is_hugepage_only_range(vma->vm_mm, new_start, size))
2089 : : return -EFAULT;
2090 : :
2091 : : /*
2092 : : * Overcommit.. This must be the final test, as it will
2093 : : * update security statistics.
2094 : : */
2095 [ + + ]: 2013488 : if (security_vm_enough_memory_mm(mm, grow))
2096 : : return -ENOMEM;
2097 : :
2098 : : /* Ok, everything looks good - let it rip */
2099 [ - + ]: 2013484 : if (vma->vm_flags & VM_LOCKED)
2100 : 0 : mm->locked_vm += grow;
2101 : 2013484 : vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
2102 : : return 0;
2103 : : }
2104 : :
2105 : : #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
2106 : : /*
2107 : : * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2108 : : * vma is the last one with address > vma->vm_end. Have to extend vma.
2109 : : */
2110 : : int expand_upwards(struct vm_area_struct *vma, unsigned long address)
2111 : : {
2112 : : int error;
2113 : :
2114 : : if (!(vma->vm_flags & VM_GROWSUP))
2115 : : return -EFAULT;
2116 : :
2117 : : /*
2118 : : * We must make sure the anon_vma is allocated
2119 : : * so that the anon_vma locking is not a noop.
2120 : : */
2121 : : if (unlikely(anon_vma_prepare(vma)))
2122 : : return -ENOMEM;
2123 : : vma_lock_anon_vma(vma);
2124 : :
2125 : : /*
2126 : : * vma->vm_start/vm_end cannot change under us because the caller
2127 : : * is required to hold the mmap_sem in read mode. We need the
2128 : : * anon_vma lock to serialize against concurrent expand_stacks.
2129 : : * Also guard against wrapping around to address 0.
2130 : : */
2131 : : if (address < PAGE_ALIGN(address+4))
2132 : : address = PAGE_ALIGN(address+4);
2133 : : else {
2134 : : vma_unlock_anon_vma(vma);
2135 : : return -ENOMEM;
2136 : : }
2137 : : error = 0;
2138 : :
2139 : : /* Somebody else might have raced and expanded it already */
2140 : : if (address > vma->vm_end) {
2141 : : unsigned long size, grow;
2142 : :
2143 : : size = address - vma->vm_start;
2144 : : grow = (address - vma->vm_end) >> PAGE_SHIFT;
2145 : :
2146 : : error = -ENOMEM;
2147 : : if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
2148 : : error = acct_stack_growth(vma, size, grow);
2149 : : if (!error) {
2150 : : /*
2151 : : * vma_gap_update() doesn't support concurrent
2152 : : * updates, but we only hold a shared mmap_sem
2153 : : * lock here, so we need to protect against
2154 : : * concurrent vma expansions.
2155 : : * vma_lock_anon_vma() doesn't help here, as
2156 : : * we don't guarantee that all growable vmas
2157 : : * in a mm share the same root anon vma.
2158 : : * So, we reuse mm->page_table_lock to guard
2159 : : * against concurrent vma expansions.
2160 : : */
2161 : : spin_lock(&vma->vm_mm->page_table_lock);
2162 : : anon_vma_interval_tree_pre_update_vma(vma);
2163 : : vma->vm_end = address;
2164 : : anon_vma_interval_tree_post_update_vma(vma);
2165 : : if (vma->vm_next)
2166 : : vma_gap_update(vma->vm_next);
2167 : : else
2168 : : vma->vm_mm->highest_vm_end = address;
2169 : : spin_unlock(&vma->vm_mm->page_table_lock);
2170 : :
2171 : : perf_event_mmap(vma);
2172 : : }
2173 : : }
2174 : : }
2175 : : vma_unlock_anon_vma(vma);
2176 : : khugepaged_enter_vma_merge(vma);
2177 : : validate_mm(vma->vm_mm);
2178 : : return error;
2179 : : }
2180 : : #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
2181 : :
2182 : : /*
2183 : : * vma is the first one with address < vma->vm_start. Have to extend vma.
2184 : : */
2185 : 0 : int expand_downwards(struct vm_area_struct *vma,
2186 : : unsigned long address)
2187 : : {
2188 : : int error;
2189 : :
2190 : : /*
2191 : : * We must make sure the anon_vma is allocated
2192 : : * so that the anon_vma locking is not a noop.
2193 : : */
2194 [ + + ]: 2013493 : if (unlikely(anon_vma_prepare(vma)))
2195 : : return -ENOMEM;
2196 : :
2197 : 2013490 : address &= PAGE_MASK;
2198 : 2013490 : error = security_mmap_addr(address);
2199 [ + + ]: 2013490 : if (error)
2200 : : return error;
2201 : :
2202 : : vma_lock_anon_vma(vma);
2203 : :
2204 : : /*
2205 : : * vma->vm_start/vm_end cannot change under us because the caller
2206 : : * is required to hold the mmap_sem in read mode. We need the
2207 : : * anon_vma lock to serialize against concurrent expand_stacks.
2208 : : */
2209 : :
2210 : : /* Somebody else might have raced and expanded it already */
2211 [ + ]: 2013490 : if (address < vma->vm_start) {
2212 : : unsigned long size, grow;
2213 : :
2214 : 2013494 : size = vma->vm_end - address;
2215 : 2013494 : grow = (vma->vm_start - address) >> PAGE_SHIFT;
2216 : :
2217 : : error = -ENOMEM;
2218 [ + + ]: 2013494 : if (grow <= vma->vm_pgoff) {
2219 : 2013490 : error = acct_stack_growth(vma, size, grow);
2220 [ + + ]: 2013493 : if (!error) {
2221 : : /*
2222 : : * vma_gap_update() doesn't support concurrent
2223 : : * updates, but we only hold a shared mmap_sem
2224 : : * lock here, so we need to protect against
2225 : : * concurrent vma expansions.
2226 : : * vma_lock_anon_vma() doesn't help here, as
2227 : : * we don't guarantee that all growable vmas
2228 : : * in a mm share the same root anon vma.
2229 : : * So, we reuse mm->page_table_lock to guard
2230 : : * against concurrent vma expansions.
2231 : : */
2232 : 2013489 : spin_lock(&vma->vm_mm->page_table_lock);
2233 : : anon_vma_interval_tree_pre_update_vma(vma);
2234 : 2013475 : vma->vm_start = address;
2235 : 2013475 : vma->vm_pgoff -= grow;
2236 : : anon_vma_interval_tree_post_update_vma(vma);
2237 : 2013486 : vma_gap_update(vma);
2238 : 2013490 : spin_unlock(&vma->vm_mm->page_table_lock);
2239 : :
2240 : 2013494 : perf_event_mmap(vma);
2241 : : }
2242 : : }
2243 : : }
2244 : : vma_unlock_anon_vma(vma);
2245 : : khugepaged_enter_vma_merge(vma);
2246 : : validate_mm(vma->vm_mm);
2247 : 2013492 : return error;
2248 : : }
2249 : :
2250 : : /*
2251 : : * Note how expand_stack() refuses to expand the stack all the way to
2252 : : * abut the next virtual mapping, *unless* that mapping itself is also
2253 : : * a stack mapping. We want to leave room for a guard page, after all
2254 : : * (the guard page itself is not added here, that is done by the
2255 : : * actual page faulting logic)
2256 : : *
2257 : : * This matches the behavior of the guard page logic (see mm/memory.c:
2258 : : * check_stack_guard_page()), which only allows the guard page to be
2259 : : * removed under these circumstances.
2260 : : */
2261 : : #ifdef CONFIG_STACK_GROWSUP
2262 : : int expand_stack(struct vm_area_struct *vma, unsigned long address)
2263 : : {
2264 : : struct vm_area_struct *next;
2265 : :
2266 : : address &= PAGE_MASK;
2267 : : next = vma->vm_next;
2268 : : if (next && next->vm_start == address + PAGE_SIZE) {
2269 : : if (!(next->vm_flags & VM_GROWSUP))
2270 : : return -ENOMEM;
2271 : : }
2272 : : return expand_upwards(vma, address);
2273 : : }
2274 : :
2275 : : struct vm_area_struct *
2276 : : find_extend_vma(struct mm_struct *mm, unsigned long addr)
2277 : : {
2278 : : struct vm_area_struct *vma, *prev;
2279 : :
2280 : : addr &= PAGE_MASK;
2281 : : vma = find_vma_prev(mm, addr, &prev);
2282 : : if (vma && (vma->vm_start <= addr))
2283 : : return vma;
2284 : : if (!prev || expand_stack(prev, addr))
2285 : : return NULL;
2286 : : if (prev->vm_flags & VM_LOCKED)
2287 : : __mlock_vma_pages_range(prev, addr, prev->vm_end, NULL);
2288 : : return prev;
2289 : : }
2290 : : #else
2291 : 0 : int expand_stack(struct vm_area_struct *vma, unsigned long address)
2292 : : {
2293 : : struct vm_area_struct *prev;
2294 : :
2295 : 27304 : address &= PAGE_MASK;
2296 : 27304 : prev = vma->vm_prev;
2297 [ + + ][ - + ]: 27304 : if (prev && prev->vm_end == address) {
2298 [ # # ]: 0 : if (!(prev->vm_flags & VM_GROWSDOWN))
2299 : : return -ENOMEM;
2300 : : }
2301 : 27304 : return expand_downwards(vma, address);
2302 : : }
2303 : :
2304 : : struct vm_area_struct *
2305 : 0 : find_extend_vma(struct mm_struct * mm, unsigned long addr)
2306 : : {
2307 : : struct vm_area_struct * vma;
2308 : : unsigned long start;
2309 : :
2310 : 2437946 : addr &= PAGE_MASK;
2311 : 2437946 : vma = find_vma(mm,addr);
2312 [ + + ]: 2437945 : if (!vma)
2313 : : return NULL;
2314 [ + + ]: 2437927 : if (vma->vm_start <= addr)
2315 : : return vma;
2316 [ - + ]: 6 : if (!(vma->vm_flags & VM_GROWSDOWN))
2317 : : return NULL;
2318 : : start = vma->vm_start;
2319 [ # # ]: 0 : if (expand_stack(vma, addr))
2320 : : return NULL;
2321 [ # # ]: 0 : if (vma->vm_flags & VM_LOCKED)
2322 : 0 : __mlock_vma_pages_range(vma, addr, start, NULL);
2323 : 0 : return vma;
2324 : : }
2325 : : #endif
2326 : :
2327 : : /*
2328 : : * Ok - we have the memory areas we should free on the vma list,
2329 : : * so release them, and do the vma updates.
2330 : : *
2331 : : * Called with the mm semaphore held.
2332 : : */
2333 : 0 : static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
2334 : : {
2335 : : unsigned long nr_accounted = 0;
2336 : :
2337 : : /* Update high watermark before we lower total_vm */
2338 : : update_hiwater_vm(mm);
2339 : : do {
2340 : 422355 : long nrpages = vma_pages(vma);
2341 : :
2342 [ + + ]: 422355 : if (vma->vm_flags & VM_ACCOUNT)
2343 : 140128 : nr_accounted += nrpages;
2344 : 422355 : vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
2345 : 422355 : vma = remove_vma(vma);
2346 [ + + ]: 830659 : } while (vma);
2347 : 408304 : vm_unacct_memory(nr_accounted);
2348 : : validate_mm(mm);
2349 : 408323 : }
2350 : :
2351 : : /*
2352 : : * Get rid of page table information in the indicated region.
2353 : : *
2354 : : * Called with the mm semaphore held.
2355 : : */
2356 : 0 : static void unmap_region(struct mm_struct *mm,
2357 : : struct vm_area_struct *vma, struct vm_area_struct *prev,
2358 : : unsigned long start, unsigned long end)
2359 : : {
2360 [ + + ]: 408338 : struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
2361 : : struct mmu_gather tlb;
2362 : :
2363 : 408338 : lru_add_drain();
2364 : : tlb_gather_mmu(&tlb, mm, start, end);
2365 : : update_hiwater_rss(mm);
2366 : 408292 : unmap_vmas(&tlb, vma, start, end);
2367 [ + + ][ + ]: 408333 : free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2368 : : next ? next->vm_start : USER_PGTABLES_CEILING);
2369 : : tlb_finish_mmu(&tlb, start, end);
2370 : 408284 : }
2371 : :
2372 : : /*
2373 : : * Create a list of vma's touched by the unmap, removing them from the mm's
2374 : : * vma list as we go..
2375 : : */
2376 : : static void
2377 : 0 : detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
2378 : : struct vm_area_struct *prev, unsigned long end)
2379 : : {
2380 : : struct vm_area_struct **insertion_point;
2381 : : struct vm_area_struct *tail_vma = NULL;
2382 : :
2383 [ + + ]: 408333 : insertion_point = (prev ? &prev->vm_next : &mm->mmap);
2384 : 408333 : vma->vm_prev = NULL;
2385 : : do {
2386 : 422355 : vma_rb_erase(vma, &mm->mm_rb);
2387 : 830679 : mm->map_count--;
2388 : : tail_vma = vma;
2389 : 830679 : vma = vma->vm_next;
2390 [ + + ][ + + ]: 830679 : } while (vma && vma->vm_start < end);
2391 : 408324 : *insertion_point = vma;
2392 [ + - ]: 816657 : if (vma) {
2393 : 408324 : vma->vm_prev = prev;
2394 : 408324 : vma_gap_update(vma);
2395 : : } else
2396 [ # # ]: 0 : mm->highest_vm_end = prev ? prev->vm_end : 0;
2397 : 408282 : tail_vma->vm_next = NULL;
2398 : 408282 : mm->mmap_cache = NULL; /* Kill the cache. */
2399 : 408282 : }
2400 : :
2401 : : /*
2402 : : * __split_vma() bypasses sysctl_max_map_count checking. We use this on the
2403 : : * munmap path where it doesn't make sense to fail.
2404 : : */
2405 : 515847 : static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
2406 : : unsigned long addr, int new_below)
2407 : : {
2408 : : struct vm_area_struct *new;
2409 : : int err = -ENOMEM;
2410 : :
2411 : : if (is_vm_hugetlb_page(vma) && (addr &
2412 : : ~(huge_page_mask(hstate_vma(vma)))))
2413 : : return -EINVAL;
2414 : :
2415 : 515847 : new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2416 [ + ]: 515840 : if (!new)
2417 : : goto out_err;
2418 : :
2419 : : /* most fields are the same, copy all, and then fixup */
2420 : 515851 : *new = *vma;
2421 : :
2422 : 515851 : INIT_LIST_HEAD(&new->anon_vma_chain);
2423 : :
2424 [ + + ]: 515851 : if (new_below)
2425 : 214040 : new->vm_end = addr;
2426 : : else {
2427 : 301811 : new->vm_start = addr;
2428 : 301811 : new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
2429 : : }
2430 : :
2431 : : err = vma_dup_policy(vma, new);
2432 : : if (err)
2433 : : goto out_free_vma;
2434 : :
2435 [ + + ]: 515851 : if (anon_vma_clone(new, vma))
2436 : : goto out_free_mpol;
2437 : :
2438 [ + + ]: 515840 : if (new->vm_file)
2439 : : get_file(new->vm_file);
2440 : :
2441 [ + + ][ - + ]: 1031713 : if (new->vm_ops && new->vm_ops->open)
2442 : 0 : new->vm_ops->open(new);
2443 : :
2444 [ + + ]: 515866 : if (new_below)
2445 : 214046 : err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
2446 : 214046 : ((addr - new->vm_start) >> PAGE_SHIFT), new);
2447 : : else
2448 : 301820 : err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
2449 : :
2450 : : /* Success. */
2451 [ - + ]: 515870 : if (!err)
2452 : : return 0;
2453 : :
2454 : : /* Clean everything up if vma_adjust failed. */
2455 [ # # ][ # # ]: 0 : if (new->vm_ops && new->vm_ops->close)
2456 : 0 : new->vm_ops->close(new);
2457 [ # # ]: 0 : if (new->vm_file)
2458 : 0 : fput(new->vm_file);
2459 : 0 : unlink_anon_vmas(new);
2460 : : out_free_mpol:
2461 : : mpol_put(vma_policy(new));
2462 : : out_free_vma:
2463 : 0 : kmem_cache_free(vm_area_cachep, new);
2464 : : out_err:
2465 : : return err;
2466 : : }
2467 : :
2468 : : /*
2469 : : * Split a vma into two pieces at address 'addr', a new vma is allocated
2470 : : * either for the first part or the tail.
2471 : : */
2472 : 0 : int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2473 : : unsigned long addr, int new_below)
2474 : : {
2475 [ + - ]: 345717 : if (mm->map_count >= sysctl_max_map_count)
2476 : : return -ENOMEM;
2477 : :
2478 : 345717 : return __split_vma(mm, vma, addr, new_below);
2479 : : }
2480 : :
2481 : : /* Munmap is split into 2 main parts -- this part which finds
2482 : : * what needs doing, and the areas themselves, which do the
2483 : : * work. This now handles partial unmappings.
2484 : : * Jeremy Fitzhardinge <jeremy@goop.org>
2485 : : */
2486 : 0 : int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
2487 : : {
2488 : : unsigned long end;
2489 : : struct vm_area_struct *vma, *prev, *last;
2490 : :
2491 [ + + ][ + ]: 408349 : if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
[ + + ]
2492 : : return -EINVAL;
2493 : :
2494 [ + + ]: 408345 : if ((len = PAGE_ALIGN(len)) == 0)
2495 : : return -EINVAL;
2496 : :
2497 : : /* Find the first overlapping VMA */
2498 : 408341 : vma = find_vma(mm, start);
2499 [ + ]: 408296 : if (!vma)
2500 : : return 0;
2501 : 408316 : prev = vma->vm_prev;
2502 : : /* we have start < vma->vm_end */
2503 : :
2504 : : /* if it doesn't overlap, we have nothing.. */
2505 : 408316 : end = start + len;
2506 [ + ]: 408316 : if (vma->vm_start >= end)
2507 : : return 0;
2508 : :
2509 : : /*
2510 : : * If we need to split any vma, do it now to save pain later.
2511 : : *
2512 : : * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2513 : : * unmapped vm_area_struct will remain in use: so lower split_vma
2514 : : * places tmp vma above, and higher split_vma places tmp vma below.
2515 : : */
2516 [ + + ]: 408317 : if (start > vma->vm_start) {
2517 : : int error;
2518 : :
2519 : : /*
2520 : : * Make sure that map_count on return from munmap() will
2521 : : * not exceed its limit; but let map_count go just above
2522 : : * its limit temporarily, to help free resources as expected.
2523 : : */
2524 [ + + ][ + - ]: 52410 : if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2525 : : return -ENOMEM;
2526 : :
2527 : 52410 : error = __split_vma(mm, vma, start, 0);
2528 [ + - ]: 52410 : if (error)
2529 : : return error;
2530 : : prev = vma;
2531 : : }
2532 : :
2533 : : /* Does it split the last one? */
2534 : 408317 : last = find_vma(mm, end);
2535 [ + ][ + + ]: 408335 : if (last && end > last->vm_start) {
2536 : 117733 : int error = __split_vma(mm, last, end, 1);
2537 [ + + ]: 117732 : if (error)
2538 : : return error;
2539 : : }
2540 [ + + ]: 408324 : vma = prev? prev->vm_next: mm->mmap;
2541 : :
2542 : : /*
2543 : : * unlock any mlock()ed ranges before detaching vmas
2544 : : */
2545 [ + + ]: 408324 : if (mm->locked_vm) {
2546 : 191 : struct vm_area_struct *tmp = vma;
2547 [ + - ][ + + ]: 384 : while (tmp && tmp->vm_start < end) {
2548 [ + + ]: 192 : if (tmp->vm_flags & VM_LOCKED) {
2549 : 382 : mm->locked_vm -= vma_pages(tmp);
2550 : : munlock_vma_pages_all(tmp);
2551 : : }
2552 : 192 : tmp = tmp->vm_next;
2553 : : }
2554 : : }
2555 : :
2556 : : /*
2557 : : * Remove the vma's, and unmap the actual pages
2558 : : */
2559 : 408324 : detach_vmas_to_be_unmapped(mm, vma, prev, end);
2560 : 408290 : unmap_region(mm, vma, prev, start, end);
2561 : :
2562 : : /* Fix up all other VM information */
2563 : 408336 : remove_vma_list(mm, vma);
2564 : :
2565 : 408323 : return 0;
2566 : : }
2567 : :
2568 : 0 : int vm_munmap(unsigned long start, size_t len)
2569 : : {
2570 : : int ret;
2571 : 271277 : struct mm_struct *mm = current->mm;
2572 : :
2573 : 271277 : down_write(&mm->mmap_sem);
2574 : 271286 : ret = do_munmap(mm, start, len);
2575 : 271269 : up_write(&mm->mmap_sem);
2576 : 271278 : return ret;
2577 : : }
2578 : : EXPORT_SYMBOL(vm_munmap);
2579 : :
2580 : 0 : SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
2581 : : {
2582 : 244038 : profile_munmap(addr);
2583 : 243846 : return vm_munmap(addr, len);
2584 : : }
2585 : :
2586 : : static inline void verify_mm_writelocked(struct mm_struct *mm)
2587 : : {
2588 : : #ifdef CONFIG_DEBUG_VM
2589 : : if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2590 : : WARN_ON(1);
2591 : : up_read(&mm->mmap_sem);
2592 : : }
2593 : : #endif
2594 : : }
2595 : :
2596 : : /*
2597 : : * this is really a simplified "do_mmap". it only handles
2598 : : * anonymous maps. eventually we may be able to do some
2599 : : * brk-specific accounting here.
2600 : : */
2601 : 0 : static unsigned long do_brk(unsigned long addr, unsigned long len)
2602 : : {
2603 : 68529 : struct mm_struct * mm = current->mm;
2604 : : struct vm_area_struct * vma, * prev;
2605 : : unsigned long flags;
2606 : : struct rb_node ** rb_link, * rb_parent;
2607 : 68529 : pgoff_t pgoff = addr >> PAGE_SHIFT;
2608 : : int error;
2609 : :
2610 : 68529 : len = PAGE_ALIGN(len);
2611 [ + + ]: 68529 : if (!len)
2612 : : return addr;
2613 : :
2614 [ + - ]: 41285 : flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2615 : :
2616 : 41285 : error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2617 [ + ]: 41285 : if (error & ~PAGE_MASK)
2618 : : return error;
2619 : :
2620 : 109814 : error = mlock_future_check(mm, mm->def_flags, len);
2621 [ - + ]: 109814 : if (error)
2622 : 109814 : return error;
2623 : :
2624 : : /*
2625 : : * mm->mmap_sem is required to protect against another thread
2626 : : * changing the mappings in case we sleep.
2627 : : */
2628 : : verify_mm_writelocked(mm);
2629 : :
2630 : : /*
2631 : : * Clear old maps. this also does some error checking for us
2632 : : */
2633 : : munmap_back:
2634 [ - + ]: 41285 : if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
2635 [ # # ]: 0 : if (do_munmap(mm, addr, len))
2636 : : return -ENOMEM;
2637 : : goto munmap_back;
2638 : : }
2639 : :
2640 : : /* Check against address space limits *after* clearing old maps... */
2641 [ + - ]: 41285 : if (!may_expand_vm(mm, len >> PAGE_SHIFT))
2642 : : return -ENOMEM;
2643 : :
2644 [ + + ]: 41285 : if (mm->map_count > sysctl_max_map_count)
2645 : : return -ENOMEM;
2646 : :
2647 [ + ]: 41284 : if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
2648 : : return -ENOMEM;
2649 : :
2650 : : /* Can we just expand an old private anonymous mapping? */
2651 : 41285 : vma = vma_merge(mm, prev, addr, addr + len, flags,
2652 : : NULL, NULL, pgoff, NULL, NULL);
2653 [ + + ]: 41283 : if (vma)
2654 : : goto out;
2655 : :
2656 : : /*
2657 : : * create a vma struct for an anonymous mapping
2658 : : */
2659 : 36886 : vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2660 [ - + ]: 36887 : if (!vma) {
2661 : : vm_unacct_memory(len >> PAGE_SHIFT);
2662 : 0 : return -ENOMEM;
2663 : : }
2664 : :
2665 : 36887 : INIT_LIST_HEAD(&vma->anon_vma_chain);
2666 : 36887 : vma->vm_mm = mm;
2667 : 36887 : vma->vm_start = addr;
2668 : 36887 : vma->vm_end = addr + len;
2669 : 36887 : vma->vm_pgoff = pgoff;
2670 : 36887 : vma->vm_flags = flags;
2671 : 36887 : vma->vm_page_prot = vm_get_page_prot(flags);
2672 : 36887 : vma_link(mm, vma, prev, rb_link, rb_parent);
2673 : : out:
2674 : 41284 : perf_event_mmap(vma);
2675 : 41285 : mm->total_vm += len >> PAGE_SHIFT;
2676 [ + + ]: 41285 : if (flags & VM_LOCKED)
2677 : 1 : mm->locked_vm += (len >> PAGE_SHIFT);
2678 : : vma->vm_flags |= VM_SOFTDIRTY;
2679 : 41285 : return addr;
2680 : : }
2681 : :
2682 : 0 : unsigned long vm_brk(unsigned long addr, unsigned long len)
2683 : : {
2684 : 38497 : struct mm_struct *mm = current->mm;
2685 : : unsigned long ret;
2686 : : bool populate;
2687 : :
2688 : 38497 : down_write(&mm->mmap_sem);
2689 : 38497 : ret = do_brk(addr, len);
2690 : 38497 : populate = ((mm->def_flags & VM_LOCKED) != 0);
2691 : 38497 : up_write(&mm->mmap_sem);
2692 [ - + ]: 38497 : if (populate)
2693 : : mm_populate(addr, len);
2694 : 0 : return ret;
2695 : : }
2696 : : EXPORT_SYMBOL(vm_brk);
2697 : :
2698 : : /* Release all mmaps. */
2699 : 0 : void exit_mmap(struct mm_struct *mm)
2700 : : {
2701 : : struct mmu_gather tlb;
2702 : 12674893 : struct vm_area_struct *vma;
2703 : : unsigned long nr_accounted = 0;
2704 : :
2705 : : /* mm's last user has gone, and its about to be pulled down */
2706 : : mmu_notifier_release(mm);
2707 : :
2708 [ + + ]: 1128568 : if (mm->locked_vm) {
2709 : 4 : vma = mm->mmap;
2710 [ + + ]: 78 : while (vma) {
2711 [ + + ]: 74 : if (vma->vm_flags & VM_LOCKED)
2712 : : munlock_vma_pages_all(vma);
2713 : 74 : vma = vma->vm_next;
2714 : : }
2715 : : }
2716 : :
2717 : : arch_exit_mmap(mm);
2718 : :
2719 : 1128568 : vma = mm->mmap;
2720 [ + - ]: 1128568 : if (!vma) /* Can happen if dup_mmap() received an OOM */
2721 : 0 : return;
2722 : :
2723 : 1128568 : lru_add_drain();
2724 : 1128570 : flush_cache_mm(mm);
2725 : : tlb_gather_mmu(&tlb, mm, 0, -1);
2726 : : /* update_hiwater_rss(mm) here? but nobody should be looking */
2727 : : /* Use -1 here to ensure all VMAs in the mm are unmapped */
2728 : 1128569 : unmap_vmas(&tlb, vma, 0, -1);
2729 : :
2730 : 1128577 : free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
2731 : : tlb_finish_mmu(&tlb, 0, -1);
2732 : :
2733 : : /*
2734 : : * Walk the list again, actually closing and freeing it,
2735 : : * with preemption enabled, without holding any MM locks.
2736 : : */
2737 [ + + ]: 21011863 : while (vma) {
2738 [ + + ]: 19883286 : if (vma->vm_flags & VM_ACCOUNT)
2739 : 12674893 : nr_accounted += vma_pages(vma);
2740 : 19883286 : vma = remove_vma(vma);
2741 : : }
2742 : 1128577 : vm_unacct_memory(nr_accounted);
2743 : :
2744 [ - + ]: 1128572 : WARN_ON(atomic_long_read(&mm->nr_ptes) >
2745 : : (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
2746 : : }
2747 : :
2748 : : /* Insert vm structure into process list sorted by address
2749 : : * and into the inode's i_mmap tree. If vm_file is non-NULL
2750 : : * then i_mmap_mutex is taken here.
2751 : : */
2752 : 0 : int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
2753 : : {
2754 : : struct vm_area_struct *prev;
2755 : : struct rb_node **rb_link, *rb_parent;
2756 : :
2757 : : /*
2758 : : * The vm_pgoff of a purely anonymous vma should be irrelevant
2759 : : * until its first write fault, when page's anon_vma and index
2760 : : * are set. But now set the vm_pgoff it will almost certainly
2761 : : * end up with (unless mremap moves it elsewhere before that
2762 : : * first wfault), so /proc/pid/maps tells a consistent story.
2763 : : *
2764 : : * By setting it to reflect the virtual start address of the
2765 : : * vma, merges and splits can happen in a seamless way, just
2766 : : * using the existing file pgoff checks and manipulations.
2767 : : * Similarly in do_mmap_pgoff and in do_brk.
2768 : : */
2769 [ + + ]: 54497 : if (!vma->vm_file) {
2770 [ - + ]: 54494 : BUG_ON(vma->anon_vma);
2771 : 54494 : vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2772 : : }
2773 [ + + ]: 54497 : if (find_vma_links(mm, vma->vm_start, vma->vm_end,
2774 : : &prev, &rb_link, &rb_parent))
2775 : : return -ENOMEM;
2776 [ + + + ]: 81734 : if ((vma->vm_flags & VM_ACCOUNT) &&
2777 : 27253 : security_vm_enough_memory_mm(mm, vma_pages(vma)))
2778 : : return -ENOMEM;
2779 : :
2780 : 54494 : vma_link(mm, vma, prev, rb_link, rb_parent);
2781 : 54491 : return 0;
2782 : : }
2783 : :
2784 : : /*
2785 : : * Copy the vma structure to a new location in the same mm,
2786 : : * prior to moving page table entries, to effect an mremap move.
2787 : : */
2788 : 0 : struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
2789 : : unsigned long addr, unsigned long len, pgoff_t pgoff,
2790 : : bool *need_rmap_locks)
2791 : : {
2792 : 6 : struct vm_area_struct *vma = *vmap;
2793 : 3 : unsigned long vma_start = vma->vm_start;
2794 : 3 : struct mm_struct *mm = vma->vm_mm;
2795 : : struct vm_area_struct *new_vma, *prev;
2796 : : struct rb_node **rb_link, *rb_parent;
2797 : : bool faulted_in_anon_vma = true;
2798 : :
2799 : : /*
2800 : : * If anonymous vma has not yet been faulted, update new pgoff
2801 : : * to match new location, to increase its chance of merging.
2802 : : */
2803 [ + + ][ - + ]: 3 : if (unlikely(!vma->vm_file && !vma->anon_vma)) {
2804 : 0 : pgoff = addr >> PAGE_SHIFT;
2805 : : faulted_in_anon_vma = false;
2806 : : }
2807 : :
2808 [ + - ]: 3 : if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
2809 : : return NULL; /* should never get here */
2810 : 3 : new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2811 : : vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
2812 : : vma_get_anon_name(vma));
2813 [ - + ]: 6 : if (new_vma) {
2814 : : /*
2815 : : * Source vma may have been merged into new_vma
2816 : : */
2817 [ # # ][ # # ]: 0 : if (unlikely(vma_start >= new_vma->vm_start &&
2818 : : vma_start < new_vma->vm_end)) {
2819 : : /*
2820 : : * The only way we can get a vma_merge with
2821 : : * self during an mremap is if the vma hasn't
2822 : : * been faulted in yet and we were allowed to
2823 : : * reset the dst vma->vm_pgoff to the
2824 : : * destination address of the mremap to allow
2825 : : * the merge to happen. mremap must change the
2826 : : * vm_pgoff linearity between src and dst vmas
2827 : : * (in turn preventing a vma_merge) to be
2828 : : * safe. It is only safe to keep the vm_pgoff
2829 : : * linear if there are no pages mapped yet.
2830 : : */
2831 : : VM_BUG_ON(faulted_in_anon_vma);
2832 : 0 : *vmap = vma = new_vma;
2833 : : }
2834 : 0 : *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
2835 : : } else {
2836 : 3 : new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2837 [ + - ]: 3 : if (new_vma) {
2838 : 3 : *new_vma = *vma;
2839 : 3 : new_vma->vm_start = addr;
2840 : 3 : new_vma->vm_end = addr + len;
2841 : 3 : new_vma->vm_pgoff = pgoff;
2842 : : if (vma_dup_policy(vma, new_vma))
2843 : : goto out_free_vma;
2844 : 3 : INIT_LIST_HEAD(&new_vma->anon_vma_chain);
2845 [ + - ]: 3 : if (anon_vma_clone(new_vma, vma))
2846 : : goto out_free_mempol;
2847 [ + + ]: 3 : if (new_vma->vm_file)
2848 : : get_file(new_vma->vm_file);
2849 [ + + ][ - + ]: 3 : if (new_vma->vm_ops && new_vma->vm_ops->open)
2850 : 0 : new_vma->vm_ops->open(new_vma);
2851 : 3 : vma_link(mm, new_vma, prev, rb_link, rb_parent);
2852 : 3 : *need_rmap_locks = false;
2853 : : }
2854 : : }
2855 : 3 : return new_vma;
2856 : :
2857 : : out_free_mempol:
2858 : : mpol_put(vma_policy(new_vma));
2859 : : out_free_vma:
2860 : 0 : kmem_cache_free(vm_area_cachep, new_vma);
2861 : 0 : return NULL;
2862 : : }
2863 : :
2864 : : /*
2865 : : * Return true if the calling process may expand its vm space by the passed
2866 : : * number of pages
2867 : : */
2868 : 0 : int may_expand_vm(struct mm_struct *mm, unsigned long npages)
2869 : : {
2870 : 3836599 : unsigned long cur = mm->total_vm; /* pages */
2871 : : unsigned long lim;
2872 : :
2873 : 3836599 : lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
2874 : :
2875 [ + - ][ + - ]: 3836599 : if (cur + npages > lim)
[ + + ][ + + ]
[ # # ]
2876 : : return 0;
2877 : 4 : return 1;
2878 : : }
2879 : :
2880 : :
2881 : 0 : static int special_mapping_fault(struct vm_area_struct *vma,
2882 : : struct vm_fault *vmf)
2883 : : {
2884 : : pgoff_t pgoff;
2885 : : struct page **pages;
2886 : :
2887 : : /*
2888 : : * special mappings have no vm_file, and in that case, the mm
2889 : : * uses vm_pgoff internally. So we have to subtract it from here.
2890 : : * We are allowed to do this because we are the mm; do not copy
2891 : : * this code into drivers!
2892 : : */
2893 : 24 : pgoff = vmf->pgoff - vma->vm_pgoff;
2894 : :
2895 [ - + ][ # # ]: 24 : for (pages = vma->vm_private_data; pgoff && *pages; ++pages)
2896 : 0 : pgoff--;
2897 : :
2898 [ + - ]: 24 : if (*pages) {
2899 : : struct page *page = *pages;
2900 : : get_page(page);
2901 : 24 : vmf->page = page;
2902 : 24 : return 0;
2903 : : }
2904 : :
2905 : : return VM_FAULT_SIGBUS;
2906 : : }
2907 : :
2908 : : /*
2909 : : * Having a close hook prevents vma merging regardless of flags.
2910 : : */
2911 : 0 : static void special_mapping_close(struct vm_area_struct *vma)
2912 : : {
2913 : 1128567 : }
2914 : :
2915 : : static const struct vm_operations_struct special_mapping_vmops = {
2916 : : .close = special_mapping_close,
2917 : : .fault = special_mapping_fault,
2918 : : };
2919 : :
2920 : : /*
2921 : : * Called with mm->mmap_sem held for writing.
2922 : : * Insert a new vma covering the given region, with the given flags.
2923 : : * Its pages are supplied by the given array of struct page *.
2924 : : * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
2925 : : * The region past the last page supplied will always produce SIGBUS.
2926 : : * The array pointer and the pages it points to are assumed to stay alive
2927 : : * for as long as this mapping might exist.
2928 : : */
2929 : 0 : int install_special_mapping(struct mm_struct *mm,
2930 : : unsigned long addr, unsigned long len,
2931 : : unsigned long vm_flags, struct page **pages)
2932 : : {
2933 : : int ret;
2934 : : struct vm_area_struct *vma;
2935 : :
2936 : 27246 : vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2937 [ + - ]: 27246 : if (unlikely(vma == NULL))
2938 : : return -ENOMEM;
2939 : :
2940 : 27246 : INIT_LIST_HEAD(&vma->anon_vma_chain);
2941 : 27246 : vma->vm_mm = mm;
2942 : 27246 : vma->vm_start = addr;
2943 : 27246 : vma->vm_end = addr + len;
2944 : :
2945 : 27246 : vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
2946 : 27246 : vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2947 : :
2948 : 27246 : vma->vm_ops = &special_mapping_vmops;
2949 : 27246 : vma->vm_private_data = pages;
2950 : :
2951 : 27246 : ret = insert_vm_struct(mm, vma);
2952 [ + - ]: 27246 : if (ret)
2953 : : goto out;
2954 : :
2955 : 27246 : mm->total_vm += len >> PAGE_SHIFT;
2956 : :
2957 : 27246 : perf_event_mmap(vma);
2958 : :
2959 : 27246 : return 0;
2960 : :
2961 : : out:
2962 : 0 : kmem_cache_free(vm_area_cachep, vma);
2963 : 0 : return ret;
2964 : : }
2965 : :
2966 : : static DEFINE_MUTEX(mm_all_locks_mutex);
2967 : :
2968 : 0 : static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
2969 : : {
2970 [ # # ]: 0 : if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
2971 : : /*
2972 : : * The LSB of head.next can't change from under us
2973 : : * because we hold the mm_all_locks_mutex.
2974 : : */
2975 : 0 : down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
2976 : : /*
2977 : : * We can safely modify head.next after taking the
2978 : : * anon_vma->root->rwsem. If some other vma in this mm shares
2979 : : * the same anon_vma we won't take it again.
2980 : : *
2981 : : * No need of atomic instructions here, head.next
2982 : : * can't change from under us thanks to the
2983 : : * anon_vma->root->rwsem.
2984 : : */
2985 [ # # ]: 0 : if (__test_and_set_bit(0, (unsigned long *)
2986 : 0 : &anon_vma->root->rb_root.rb_node))
2987 : 0 : BUG();
2988 : : }
2989 : 0 : }
2990 : :
2991 : 0 : static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
2992 : : {
2993 [ # # ]: 0 : if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2994 : : /*
2995 : : * AS_MM_ALL_LOCKS can't change from under us because
2996 : : * we hold the mm_all_locks_mutex.
2997 : : *
2998 : : * Operations on ->flags have to be atomic because
2999 : : * even if AS_MM_ALL_LOCKS is stable thanks to the
3000 : : * mm_all_locks_mutex, there may be other cpus
3001 : : * changing other bitflags in parallel to us.
3002 : : */
3003 [ # # ]: 0 : if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
3004 : 0 : BUG();
3005 : 0 : mutex_lock_nest_lock(&mapping->i_mmap_mutex, &mm->mmap_sem);
3006 : : }
3007 : 0 : }
3008 : :
3009 : : /*
3010 : : * This operation locks against the VM for all pte/vma/mm related
3011 : : * operations that could ever happen on a certain mm. This includes
3012 : : * vmtruncate, try_to_unmap, and all page faults.
3013 : : *
3014 : : * The caller must take the mmap_sem in write mode before calling
3015 : : * mm_take_all_locks(). The caller isn't allowed to release the
3016 : : * mmap_sem until mm_drop_all_locks() returns.
3017 : : *
3018 : : * mmap_sem in write mode is required in order to block all operations
3019 : : * that could modify pagetables and free pages without need of
3020 : : * altering the vma layout (for example populate_range() with
3021 : : * nonlinear vmas). It's also needed in write mode to avoid new
3022 : : * anon_vmas to be associated with existing vmas.
3023 : : *
3024 : : * A single task can't take more than one mm_take_all_locks() in a row
3025 : : * or it would deadlock.
3026 : : *
3027 : : * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3028 : : * mapping->flags avoid to take the same lock twice, if more than one
3029 : : * vma in this mm is backed by the same anon_vma or address_space.
3030 : : *
3031 : : * We can take all the locks in random order because the VM code
3032 : : * taking i_mmap_mutex or anon_vma->rwsem outside the mmap_sem never
3033 : : * takes more than one of them in a row. Secondly we're protected
3034 : : * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
3035 : : *
3036 : : * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3037 : : * that may have to take thousand of locks.
3038 : : *
3039 : : * mm_take_all_locks() can fail if it's interrupted by signals.
3040 : : */
3041 : 0 : int mm_take_all_locks(struct mm_struct *mm)
3042 : : {
3043 : : struct vm_area_struct *vma;
3044 : : struct anon_vma_chain *avc;
3045 : :
3046 [ # # ]: 0 : BUG_ON(down_read_trylock(&mm->mmap_sem));
3047 : :
3048 : 0 : mutex_lock(&mm_all_locks_mutex);
3049 : :
3050 [ # # ]: 0 : for (vma = mm->mmap; vma; vma = vma->vm_next) {
3051 [ # # ]: 0 : if (signal_pending(current))
3052 : : goto out_unlock;
3053 [ # # ][ # # ]: 0 : if (vma->vm_file && vma->vm_file->f_mapping)
3054 : 0 : vm_lock_mapping(mm, vma->vm_file->f_mapping);
3055 : : }
3056 : :
3057 [ # # ]: 0 : for (vma = mm->mmap; vma; vma = vma->vm_next) {
3058 [ # # ]: 0 : if (signal_pending(current))
3059 : : goto out_unlock;
3060 [ # # ]: 0 : if (vma->anon_vma)
3061 [ # # ]: 0 : list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3062 : 0 : vm_lock_anon_vma(mm, avc->anon_vma);
3063 : : }
3064 : :
3065 : : return 0;
3066 : :
3067 : : out_unlock:
3068 : 0 : mm_drop_all_locks(mm);
3069 : 0 : return -EINTR;
3070 : : }
3071 : :
3072 : 0 : static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
3073 : : {
3074 [ # # ]: 0 : if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3075 : : /*
3076 : : * The LSB of head.next can't change to 0 from under
3077 : : * us because we hold the mm_all_locks_mutex.
3078 : : *
3079 : : * We must however clear the bitflag before unlocking
3080 : : * the vma so the users using the anon_vma->rb_root will
3081 : : * never see our bitflag.
3082 : : *
3083 : : * No need of atomic instructions here, head.next
3084 : : * can't change from under us until we release the
3085 : : * anon_vma->root->rwsem.
3086 : : */
3087 [ # # ]: 0 : if (!__test_and_clear_bit(0, (unsigned long *)
3088 : : &anon_vma->root->rb_root.rb_node))
3089 : 0 : BUG();
3090 : : anon_vma_unlock_write(anon_vma);
3091 : : }
3092 : 0 : }
3093 : :
3094 : 0 : static void vm_unlock_mapping(struct address_space *mapping)
3095 : : {
3096 [ # # ]: 0 : if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3097 : : /*
3098 : : * AS_MM_ALL_LOCKS can't change to 0 from under us
3099 : : * because we hold the mm_all_locks_mutex.
3100 : : */
3101 : 0 : mutex_unlock(&mapping->i_mmap_mutex);
3102 [ # # ]: 0 : if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
3103 : : &mapping->flags))
3104 : 0 : BUG();
3105 : : }
3106 : 0 : }
3107 : :
3108 : : /*
3109 : : * The mmap_sem cannot be released by the caller until
3110 : : * mm_drop_all_locks() returns.
3111 : : */
3112 : 0 : void mm_drop_all_locks(struct mm_struct *mm)
3113 : : {
3114 : : struct vm_area_struct *vma;
3115 : : struct anon_vma_chain *avc;
3116 : :
3117 [ # # ]: 0 : BUG_ON(down_read_trylock(&mm->mmap_sem));
3118 [ # # ]: 0 : BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
3119 : :
3120 [ # # ]: 0 : for (vma = mm->mmap; vma; vma = vma->vm_next) {
3121 [ # # ]: 0 : if (vma->anon_vma)
3122 [ # # ]: 0 : list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3123 : 0 : vm_unlock_anon_vma(avc->anon_vma);
3124 [ # # ][ # # ]: 0 : if (vma->vm_file && vma->vm_file->f_mapping)
3125 : 0 : vm_unlock_mapping(vma->vm_file->f_mapping);
3126 : : }
3127 : :
3128 : 0 : mutex_unlock(&mm_all_locks_mutex);
3129 : 0 : }
3130 : :
3131 : : /*
3132 : : * initialise the VMA slab
3133 : : */
3134 : 0 : void __init mmap_init(void)
3135 : : {
3136 : : int ret;
3137 : :
3138 : 0 : ret = percpu_counter_init(&vm_committed_as, 0);
3139 : : VM_BUG_ON(ret);
3140 : 0 : }
3141 : :
3142 : : /*
3143 : : * Initialise sysctl_user_reserve_kbytes.
3144 : : *
3145 : : * This is intended to prevent a user from starting a single memory hogging
3146 : : * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3147 : : * mode.
3148 : : *
3149 : : * The default value is min(3% of free memory, 128MB)
3150 : : * 128MB is enough to recover with sshd/login, bash, and top/kill.
3151 : : */
3152 : 0 : static int init_user_reserve(void)
3153 : : {
3154 : : unsigned long free_kbytes;
3155 : :
3156 : 0 : free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3157 : :
3158 : 0 : sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
3159 : 0 : return 0;
3160 : : }
3161 : : subsys_initcall(init_user_reserve);
3162 : :
3163 : : /*
3164 : : * Initialise sysctl_admin_reserve_kbytes.
3165 : : *
3166 : : * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3167 : : * to log in and kill a memory hogging process.
3168 : : *
3169 : : * Systems with more than 256MB will reserve 8MB, enough to recover
3170 : : * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3171 : : * only reserve 3% of free pages by default.
3172 : : */
3173 : 0 : static int init_admin_reserve(void)
3174 : : {
3175 : : unsigned long free_kbytes;
3176 : :
3177 : 0 : free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3178 : :
3179 : 0 : sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
3180 : 0 : return 0;
3181 : : }
3182 : : subsys_initcall(init_admin_reserve);
3183 : :
3184 : : /*
3185 : : * Reinititalise user and admin reserves if memory is added or removed.
3186 : : *
3187 : : * The default user reserve max is 128MB, and the default max for the
3188 : : * admin reserve is 8MB. These are usually, but not always, enough to
3189 : : * enable recovery from a memory hogging process using login/sshd, a shell,
3190 : : * and tools like top. It may make sense to increase or even disable the
3191 : : * reserve depending on the existence of swap or variations in the recovery
3192 : : * tools. So, the admin may have changed them.
3193 : : *
3194 : : * If memory is added and the reserves have been eliminated or increased above
3195 : : * the default max, then we'll trust the admin.
3196 : : *
3197 : : * If memory is removed and there isn't enough free memory, then we
3198 : : * need to reset the reserves.
3199 : : *
3200 : : * Otherwise keep the reserve set by the admin.
3201 : : */
3202 : : static int reserve_mem_notifier(struct notifier_block *nb,
3203 : : unsigned long action, void *data)
3204 : : {
3205 : : unsigned long tmp, free_kbytes;
3206 : :
3207 : : switch (action) {
3208 : : case MEM_ONLINE:
3209 : : /* Default max is 128MB. Leave alone if modified by operator. */
3210 : : tmp = sysctl_user_reserve_kbytes;
3211 : : if (0 < tmp && tmp < (1UL << 17))
3212 : : init_user_reserve();
3213 : :
3214 : : /* Default max is 8MB. Leave alone if modified by operator. */
3215 : : tmp = sysctl_admin_reserve_kbytes;
3216 : : if (0 < tmp && tmp < (1UL << 13))
3217 : : init_admin_reserve();
3218 : :
3219 : : break;
3220 : : case MEM_OFFLINE:
3221 : : free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3222 : :
3223 : : if (sysctl_user_reserve_kbytes > free_kbytes) {
3224 : : init_user_reserve();
3225 : : pr_info("vm.user_reserve_kbytes reset to %lu\n",
3226 : : sysctl_user_reserve_kbytes);
3227 : : }
3228 : :
3229 : : if (sysctl_admin_reserve_kbytes > free_kbytes) {
3230 : : init_admin_reserve();
3231 : : pr_info("vm.admin_reserve_kbytes reset to %lu\n",
3232 : : sysctl_admin_reserve_kbytes);
3233 : : }
3234 : : break;
3235 : : default:
3236 : : break;
3237 : : }
3238 : : return NOTIFY_OK;
3239 : : }
3240 : :
3241 : : static struct notifier_block reserve_mem_nb = {
3242 : : .notifier_call = reserve_mem_notifier,
3243 : : };
3244 : :
3245 : 0 : static int __meminit init_reserve_notifier(void)
3246 : : {
3247 : : if (register_hotmemory_notifier(&reserve_mem_nb))
3248 : : printk("Failed registering memory add/remove notifier for admin reserve");
3249 : :
3250 : 0 : return 0;
3251 : : }
3252 : : subsys_initcall(init_reserve_notifier);
|
