OpenVZ kernel 2.6.32-feoktistov released
[linux-2.6.32-openvz] / mm / mmap.c
CommitLineData
1da177e4
LT
1/*
2 * mm/mmap.c
3 *
4 * Written by obz.
5 *
046c6884 6 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
1da177e4
LT
7 */
8
9#include <linux/slab.h>
4af3c9cc 10#include <linux/backing-dev.h>
1da177e4
LT
11#include <linux/mm.h>
12#include <linux/shm.h>
13#include <linux/mman.h>
14#include <linux/pagemap.h>
15#include <linux/swap.h>
16#include <linux/syscalls.h>
c59ede7b 17#include <linux/capability.h>
1da177e4
LT
18#include <linux/init.h>
19#include <linux/file.h>
20#include <linux/fs.h>
21#include <linux/personality.h>
22#include <linux/security.h>
6146f0d5 23#include <linux/ima.h>
1da177e4
LT
24#include <linux/hugetlb.h>
25#include <linux/profile.h>
26#include <linux/module.h>
27#include <linux/mount.h>
28#include <linux/mempolicy.h>
29#include <linux/rmap.h>
cddb8a5c 30#include <linux/mmu_notifier.h>
cdd6c482 31#include <linux/perf_event.h>
5198e6ea 32#include <linux/virtinfo.h>
1da177e4
LT
33
34#include <asm/uaccess.h>
35#include <asm/cacheflush.h>
36#include <asm/tlb.h>
d6dd61c8 37#include <asm/mmu_context.h>
1da177e4 38
42b77728
JB
39#include "internal.h"
40
3a459756
KK
41#ifndef arch_mmap_check
42#define arch_mmap_check(addr, len, flags) (0)
43#endif
44
5198e6ea
PE
45#include <bc/vmpages.h>
46
08e7d9b5
MS
47#ifndef arch_rebalance_pgtables
48#define arch_rebalance_pgtables(addr, len) (addr)
49#endif
50
5198e6ea 51static unsigned long __do_brk(unsigned long addr, unsigned long len, int soft);
e0da382c
HD
52static void unmap_region(struct mm_struct *mm,
53 struct vm_area_struct *vma, struct vm_area_struct *prev,
54 unsigned long start, unsigned long end);
55
1da177e4
LT
56/*
57 * WARNING: the debugging will use recursive algorithms so never enable this
58 * unless you know what you are doing.
59 */
60#undef DEBUG_MM_RB
61
62/* description of effects of mapping type and prot in current implementation.
63 * this is due to the limited x86 page protection hardware. The expected
64 * behavior is in parens:
65 *
66 * map_type prot
67 * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
68 * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
69 * w: (no) no w: (no) no w: (yes) yes w: (no) no
70 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
71 *
72 * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
73 * w: (no) no w: (no) no w: (copy) copy w: (no) no
74 * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
75 *
76 */
77pgprot_t protection_map[16] = {
78 __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
79 __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
80};
81
804af2cf
HD
82pgprot_t vm_get_page_prot(unsigned long vm_flags)
83{
b845f313
DK
84 return __pgprot(pgprot_val(protection_map[vm_flags &
85 (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
86 pgprot_val(arch_vm_get_page_prot(vm_flags)));
804af2cf
HD
87}
88EXPORT_SYMBOL(vm_get_page_prot);
89
1da177e4
LT
90int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
91int sysctl_overcommit_ratio = 50; /* default is 50% */
c3d8c141 92int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
00a62ce9 93struct percpu_counter vm_committed_as;
1da177e4
LT
94
95/*
96 * Check that a process has enough memory to allocate a new virtual
97 * mapping. 0 means there is enough memory for the allocation to
98 * succeed and -ENOMEM implies there is not.
99 *
100 * We currently support three overcommit policies, which are set via the
101 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
102 *
103 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
104 * Additional code 2002 Jul 20 by Robert Love.
105 *
106 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
107 *
108 * Note this is a helper function intended to be used by LSMs which
109 * wish to use this logic.
110 */
34b4e4aa 111int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
1da177e4
LT
112{
113 unsigned long free, allowed;
114
115 vm_acct_memory(pages);
116
5198e6ea
PE
117#ifdef CONFIG_BEANCOUNTERS
118 switch (virtinfo_notifier_call(VITYPE_GENERAL, VIRTINFO_ENOUGHMEM,
119 (void *)pages)
120 & (NOTIFY_OK | NOTIFY_FAIL)) {
121 case NOTIFY_OK:
122 return 0;
123 case NOTIFY_FAIL:
124 vm_unacct_memory(pages);
125 return -ENOMEM;
126 }
127#endif
128
1da177e4
LT
129 /*
130 * Sometimes we want to use more memory than we have
131 */
132 if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
133 return 0;
134
135 if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
136 unsigned long n;
137
347ce434 138 free = global_page_state(NR_FILE_PAGES);
1da177e4
LT
139 free += nr_swap_pages;
140
141 /*
142 * Any slabs which are created with the
143 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
144 * which are reclaimable, under pressure. The dentry
145 * cache and most inode caches should fall into this
146 */
972d1a7b 147 free += global_page_state(NR_SLAB_RECLAIMABLE);
1da177e4
LT
148
149 /*
150 * Leave the last 3% for root
151 */
152 if (!cap_sys_admin)
153 free -= free / 32;
154
155 if (free > pages)
156 return 0;
157
158 /*
159 * nr_free_pages() is very expensive on large systems,
160 * only call if we're about to fail.
161 */
162 n = nr_free_pages();
6d9f7839
HA
163
164 /*
165 * Leave reserved pages. The pages are not for anonymous pages.
166 */
167 if (n <= totalreserve_pages)
168 goto error;
169 else
170 n -= totalreserve_pages;
171
172 /*
173 * Leave the last 3% for root
174 */
1da177e4
LT
175 if (!cap_sys_admin)
176 n -= n / 32;
177 free += n;
178
179 if (free > pages)
180 return 0;
6d9f7839
HA
181
182 goto error;
1da177e4
LT
183 }
184
185 allowed = (totalram_pages - hugetlb_total_pages())
186 * sysctl_overcommit_ratio / 100;
187 /*
188 * Leave the last 3% for root
189 */
190 if (!cap_sys_admin)
191 allowed -= allowed / 32;
192 allowed += total_swap_pages;
193
194 /* Don't let a single process grow too big:
195 leave 3% of the size of this process for other processes */
731572d3
AC
196 if (mm)
197 allowed -= mm->total_vm / 32;
1da177e4 198
00a62ce9 199 if (percpu_counter_read_positive(&vm_committed_as) < allowed)
1da177e4 200 return 0;
6d9f7839 201error:
1da177e4
LT
202 vm_unacct_memory(pages);
203
204 return -ENOMEM;
205}
206
1da177e4
LT
207/*
208 * Requires inode->i_mapping->i_mmap_lock
209 */
210static void __remove_shared_vm_struct(struct vm_area_struct *vma,
211 struct file *file, struct address_space *mapping)
212{
213 if (vma->vm_flags & VM_DENYWRITE)
d3ac7f89 214 atomic_inc(&file->f_path.dentry->d_inode->i_writecount);
1da177e4
LT
215 if (vma->vm_flags & VM_SHARED)
216 mapping->i_mmap_writable--;
217
218 flush_dcache_mmap_lock(mapping);
219 if (unlikely(vma->vm_flags & VM_NONLINEAR))
220 list_del_init(&vma->shared.vm_set.list);
221 else
222 vma_prio_tree_remove(vma, &mapping->i_mmap);
223 flush_dcache_mmap_unlock(mapping);
224}
225
226/*
a8fb5618
HD
227 * Unlink a file-based vm structure from its prio_tree, to hide
228 * vma from rmap and vmtruncate before freeing its page tables.
1da177e4 229 */
a8fb5618 230void unlink_file_vma(struct vm_area_struct *vma)
1da177e4
LT
231{
232 struct file *file = vma->vm_file;
233
1da177e4
LT
234 if (file) {
235 struct address_space *mapping = file->f_mapping;
236 spin_lock(&mapping->i_mmap_lock);
237 __remove_shared_vm_struct(vma, file, mapping);
238 spin_unlock(&mapping->i_mmap_lock);
239 }
a8fb5618
HD
240}
241
242/*
243 * Close a vm structure and free it, returning the next.
244 */
245static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
246{
247 struct vm_area_struct *next = vma->vm_next;
248
a8fb5618 249 might_sleep();
5198e6ea
PE
250
251 ub_memory_uncharge(vma->vm_mm, vma->vm_end - vma->vm_start,
252 vma->vm_flags, vma->vm_file);
1da177e4
LT
253 if (vma->vm_ops && vma->vm_ops->close)
254 vma->vm_ops->close(vma);
925d1c40 255 if (vma->vm_file) {
a8fb5618 256 fput(vma->vm_file);
925d1c40
MH
257 if (vma->vm_flags & VM_EXECUTABLE)
258 removed_exe_file_vma(vma->vm_mm);
259 }
f0be3d32 260 mpol_put(vma_policy(vma));
1da177e4 261 kmem_cache_free(vm_area_cachep, vma);
a8fb5618 262 return next;
1da177e4
LT
263}
264
6a6160a7 265SYSCALL_DEFINE1(brk, unsigned long, brk)
1da177e4
LT
266{
267 unsigned long rlim, retval;
268 unsigned long newbrk, oldbrk;
269 struct mm_struct *mm = current->mm;
a5b4592c 270 unsigned long min_brk;
1da177e4
LT
271
272 down_write(&mm->mmap_sem);
273
a5b4592c
JK
274#ifdef CONFIG_COMPAT_BRK
275 min_brk = mm->end_code;
276#else
277 min_brk = mm->start_brk;
278#endif
279 if (brk < min_brk)
1da177e4 280 goto out;
1e624196
RG
281
282 /*
283 * Check against rlimit here. If this check is done later after the test
284 * of oldbrk with newbrk then it can escape the test and let the data
285 * segment grow beyond its set limit the in case where the limit is
286 * not page aligned -Ram Gupta
287 */
288 rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
c1d171a0
JK
289 if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
290 (mm->end_data - mm->start_data) > rlim)
1e624196
RG
291 goto out;
292
1da177e4
LT
293 newbrk = PAGE_ALIGN(brk);
294 oldbrk = PAGE_ALIGN(mm->brk);
295 if (oldbrk == newbrk)
296 goto set_brk;
297
298 /* Always allow shrinking brk. */
299 if (brk <= mm->brk) {
300 if (!do_munmap(mm, newbrk, oldbrk-newbrk))
301 goto set_brk;
302 goto out;
303 }
304
1da177e4
LT
305 /* Check against existing mmap mappings. */
306 if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
307 goto out;
308
309 /* Ok, looks good - let it rip. */
5198e6ea 310 if (__do_brk(oldbrk, newbrk-oldbrk, UB_HARD) != oldbrk)
1da177e4
LT
311 goto out;
312set_brk:
313 mm->brk = brk;
314out:
315 retval = mm->brk;
316 up_write(&mm->mmap_sem);
317 return retval;
318}
319
320#ifdef DEBUG_MM_RB
321static int browse_rb(struct rb_root *root)
322{
323 int i = 0, j;
324 struct rb_node *nd, *pn = NULL;
325 unsigned long prev = 0, pend = 0;
326
327 for (nd = rb_first(root); nd; nd = rb_next(nd)) {
328 struct vm_area_struct *vma;
329 vma = rb_entry(nd, struct vm_area_struct, vm_rb);
330 if (vma->vm_start < prev)
331 printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1;
332 if (vma->vm_start < pend)
333 printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
334 if (vma->vm_start > vma->vm_end)
335 printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start);
336 i++;
337 pn = nd;
d1af65d1
DM
338 prev = vma->vm_start;
339 pend = vma->vm_end;
1da177e4
LT
340 }
341 j = 0;
342 for (nd = pn; nd; nd = rb_prev(nd)) {
343 j++;
344 }
345 if (i != j)
346 printk("backwards %d, forwards %d\n", j, i), i = 0;
347 return i;
348}
349
350void validate_mm(struct mm_struct *mm)
351{
352 int bug = 0;
353 int i = 0;
354 struct vm_area_struct *tmp = mm->mmap;
355 while (tmp) {
356 tmp = tmp->vm_next;
357 i++;
358 }
359 if (i != mm->map_count)
360 printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1;
361 i = browse_rb(&mm->mm_rb);
362 if (i != mm->map_count)
363 printk("map_count %d rb %d\n", mm->map_count, i), bug = 1;
46a350ef 364 BUG_ON(bug);
1da177e4
LT
365}
366#else
367#define validate_mm(mm) do { } while (0)
368#endif
369
370static struct vm_area_struct *
371find_vma_prepare(struct mm_struct *mm, unsigned long addr,
372 struct vm_area_struct **pprev, struct rb_node ***rb_link,
373 struct rb_node ** rb_parent)
374{
375 struct vm_area_struct * vma;
376 struct rb_node ** __rb_link, * __rb_parent, * rb_prev;
377
378 __rb_link = &mm->mm_rb.rb_node;
379 rb_prev = __rb_parent = NULL;
380 vma = NULL;
381
382 while (*__rb_link) {
383 struct vm_area_struct *vma_tmp;
384
385 __rb_parent = *__rb_link;
386 vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
387
388 if (vma_tmp->vm_end > addr) {
389 vma = vma_tmp;
390 if (vma_tmp->vm_start <= addr)
dfe195fb 391 break;
1da177e4
LT
392 __rb_link = &__rb_parent->rb_left;
393 } else {
394 rb_prev = __rb_parent;
395 __rb_link = &__rb_parent->rb_right;
396 }
397 }
398
399 *pprev = NULL;
400 if (rb_prev)
401 *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
402 *rb_link = __rb_link;
403 *rb_parent = __rb_parent;
404 return vma;
405}
406
407static inline void
408__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
409 struct vm_area_struct *prev, struct rb_node *rb_parent)
410{
378776c2
LT
411 struct vm_area_struct *next;
412
413 vma->vm_prev = prev;
1da177e4 414 if (prev) {
378776c2 415 next = prev->vm_next;
1da177e4
LT
416 prev->vm_next = vma;
417 } else {
418 mm->mmap = vma;
419 if (rb_parent)
378776c2 420 next = rb_entry(rb_parent,
1da177e4
LT
421 struct vm_area_struct, vm_rb);
422 else
378776c2 423 next = NULL;
1da177e4 424 }
378776c2
LT
425 vma->vm_next = next;
426 if (next)
427 next->vm_prev = vma;
1da177e4
LT
428}
429
430void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
431 struct rb_node **rb_link, struct rb_node *rb_parent)
432{
433 rb_link_node(&vma->vm_rb, rb_parent, rb_link);
434 rb_insert_color(&vma->vm_rb, &mm->mm_rb);
435}
436
cb8f488c 437static void __vma_link_file(struct vm_area_struct *vma)
1da177e4 438{
48aae425 439 struct file *file;
1da177e4
LT
440
441 file = vma->vm_file;
442 if (file) {
443 struct address_space *mapping = file->f_mapping;
444
445 if (vma->vm_flags & VM_DENYWRITE)
d3ac7f89 446 atomic_dec(&file->f_path.dentry->d_inode->i_writecount);
1da177e4
LT
447 if (vma->vm_flags & VM_SHARED)
448 mapping->i_mmap_writable++;
449
450 flush_dcache_mmap_lock(mapping);
451 if (unlikely(vma->vm_flags & VM_NONLINEAR))
452 vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
453 else
454 vma_prio_tree_insert(vma, &mapping->i_mmap);
455 flush_dcache_mmap_unlock(mapping);
456 }
457}
458
459static void
460__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
461 struct vm_area_struct *prev, struct rb_node **rb_link,
462 struct rb_node *rb_parent)
463{
464 __vma_link_list(mm, vma, prev, rb_parent);
465 __vma_link_rb(mm, vma, rb_link, rb_parent);
466 __anon_vma_link(vma);
467}
468
469static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
470 struct vm_area_struct *prev, struct rb_node **rb_link,
471 struct rb_node *rb_parent)
472{
473 struct address_space *mapping = NULL;
474
475 if (vma->vm_file)
476 mapping = vma->vm_file->f_mapping;
477
478 if (mapping) {
479 spin_lock(&mapping->i_mmap_lock);
480 vma->vm_truncate_count = mapping->truncate_count;
481 }
482 anon_vma_lock(vma);
483
484 __vma_link(mm, vma, prev, rb_link, rb_parent);
485 __vma_link_file(vma);
486
487 anon_vma_unlock(vma);
488 if (mapping)
489 spin_unlock(&mapping->i_mmap_lock);
490
491 mm->map_count++;
492 validate_mm(mm);
493}
494
495/*
496 * Helper for vma_adjust in the split_vma insert case:
497 * insert vm structure into list and rbtree and anon_vma,
498 * but it has already been inserted into prio_tree earlier.
499 */
48aae425 500static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
1da177e4 501{
48aae425
Z
502 struct vm_area_struct *__vma, *prev;
503 struct rb_node **rb_link, *rb_parent;
1da177e4
LT
504
505 __vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent);
46a350ef 506 BUG_ON(__vma && __vma->vm_start < vma->vm_end);
1da177e4
LT
507 __vma_link(mm, vma, prev, rb_link, rb_parent);
508 mm->map_count++;
509}
510
511static inline void
512__vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
513 struct vm_area_struct *prev)
514{
378776c2
LT
515 struct vm_area_struct *next = vma->vm_next;
516
517 prev->vm_next = next;
518 if (next)
519 next->vm_prev = prev;
1da177e4
LT
520 rb_erase(&vma->vm_rb, &mm->mm_rb);
521 if (mm->mmap_cache == vma)
522 mm->mmap_cache = prev;
523}
524
525/*
526 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
527 * is already present in an i_mmap tree without adjusting the tree.
528 * The following helper function should be used when such adjustments
529 * are necessary. The "insert" vma (if any) is to be inserted
530 * before we drop the necessary locks.
531 */
532void vma_adjust(struct vm_area_struct *vma, unsigned long start,
533 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
534{
535 struct mm_struct *mm = vma->vm_mm;
536 struct vm_area_struct *next = vma->vm_next;
537 struct vm_area_struct *importer = NULL;
538 struct address_space *mapping = NULL;
539 struct prio_tree_root *root = NULL;
540 struct file *file = vma->vm_file;
541 struct anon_vma *anon_vma = NULL;
542 long adjust_next = 0;
543 int remove_next = 0;
544
545 if (next && !insert) {
546 if (end >= next->vm_end) {
547 /*
548 * vma expands, overlapping all the next, and
549 * perhaps the one after too (mprotect case 6).
550 */
551again: remove_next = 1 + (end > next->vm_end);
552 end = next->vm_end;
553 anon_vma = next->anon_vma;
554 importer = vma;
555 } else if (end > next->vm_start) {
556 /*
557 * vma expands, overlapping part of the next:
558 * mprotect case 5 shifting the boundary up.
559 */
560 adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
561 anon_vma = next->anon_vma;
562 importer = vma;
563 } else if (end < vma->vm_end) {
564 /*
565 * vma shrinks, and !insert tells it's not
566 * split_vma inserting another: so it must be
567 * mprotect case 4 shifting the boundary down.
568 */
569 adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
570 anon_vma = next->anon_vma;
571 importer = next;
572 }
573 }
574
575 if (file) {
576 mapping = file->f_mapping;
577 if (!(vma->vm_flags & VM_NONLINEAR))
578 root = &mapping->i_mmap;
579 spin_lock(&mapping->i_mmap_lock);
580 if (importer &&
581 vma->vm_truncate_count != next->vm_truncate_count) {
582 /*
583 * unmap_mapping_range might be in progress:
584 * ensure that the expanding vma is rescanned.
585 */
586 importer->vm_truncate_count = 0;
587 }
588 if (insert) {
589 insert->vm_truncate_count = vma->vm_truncate_count;
590 /*
591 * Put into prio_tree now, so instantiated pages
592 * are visible to arm/parisc __flush_dcache_page
593 * throughout; but we cannot insert into address
594 * space until vma start or end is updated.
595 */
596 __vma_link_file(insert);
597 }
598 }
599
600 /*
601 * When changing only vma->vm_end, we don't really need
252c5f94 602 * anon_vma lock.
1da177e4 603 */
252c5f94 604 if (vma->anon_vma && (insert || importer || start != vma->vm_start))
1da177e4
LT
605 anon_vma = vma->anon_vma;
606 if (anon_vma) {
607 spin_lock(&anon_vma->lock);
608 /*
609 * Easily overlooked: when mprotect shifts the boundary,
610 * make sure the expanding vma has anon_vma set if the
611 * shrinking vma had, to cover any anon pages imported.
612 */
613 if (importer && !importer->anon_vma) {
614 importer->anon_vma = anon_vma;
615 __anon_vma_link(importer);
616 }
617 }
618
619 if (root) {
620 flush_dcache_mmap_lock(mapping);
621 vma_prio_tree_remove(vma, root);
622 if (adjust_next)
623 vma_prio_tree_remove(next, root);
624 }
625
626 vma->vm_start = start;
627 vma->vm_end = end;
628 vma->vm_pgoff = pgoff;
629 if (adjust_next) {
630 next->vm_start += adjust_next << PAGE_SHIFT;
631 next->vm_pgoff += adjust_next;
632 }
633
634 if (root) {
635 if (adjust_next)
636 vma_prio_tree_insert(next, root);
637 vma_prio_tree_insert(vma, root);
638 flush_dcache_mmap_unlock(mapping);
639 }
640
641 if (remove_next) {
642 /*
643 * vma_merge has merged next into vma, and needs
644 * us to remove next before dropping the locks.
645 */
646 __vma_unlink(mm, next, vma);
647 if (file)
648 __remove_shared_vm_struct(next, file, mapping);
649 if (next->anon_vma)
650 __anon_vma_merge(vma, next);
651 } else if (insert) {
652 /*
653 * split_vma has split insert from vma, and needs
654 * us to insert it before dropping the locks
655 * (it may either follow vma or precede it).
656 */
657 __insert_vm_struct(mm, insert);
658 }
659
660 if (anon_vma)
661 spin_unlock(&anon_vma->lock);
662 if (mapping)
663 spin_unlock(&mapping->i_mmap_lock);
664
665 if (remove_next) {
925d1c40 666 if (file) {
1da177e4 667 fput(file);
925d1c40
MH
668 if (next->vm_flags & VM_EXECUTABLE)
669 removed_exe_file_vma(mm);
670 }
1da177e4 671 mm->map_count--;
f0be3d32 672 mpol_put(vma_policy(next));
1da177e4
LT
673 kmem_cache_free(vm_area_cachep, next);
674 /*
675 * In mprotect's case 6 (see comments on vma_merge),
676 * we must remove another next too. It would clutter
677 * up the code too much to do both in one go.
678 */
679 if (remove_next == 2) {
680 next = vma->vm_next;
681 goto again;
682 }
683 }
684
685 validate_mm(mm);
686}
687
688/*
689 * If the vma has a ->close operation then the driver probably needs to release
690 * per-vma resources, so we don't attempt to merge those.
691 */
1da177e4
LT
692static inline int is_mergeable_vma(struct vm_area_struct *vma,
693 struct file *file, unsigned long vm_flags)
694{
8314c4f2
HD
695 /* VM_CAN_NONLINEAR may get set later by f_op->mmap() */
696 if ((vma->vm_flags ^ vm_flags) & ~VM_CAN_NONLINEAR)
1da177e4
LT
697 return 0;
698 if (vma->vm_file != file)
699 return 0;
700 if (vma->vm_ops && vma->vm_ops->close)
701 return 0;
702 return 1;
703}
704
705static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
706 struct anon_vma *anon_vma2)
707{
708 return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);
709}
710
711/*
712 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
713 * in front of (at a lower virtual address and file offset than) the vma.
714 *
715 * We cannot merge two vmas if they have differently assigned (non-NULL)
716 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
717 *
718 * We don't check here for the merged mmap wrapping around the end of pagecache
719 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
720 * wrap, nor mmaps which cover the final page at index -1UL.
721 */
722static int
723can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
724 struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
725{
726 if (is_mergeable_vma(vma, file, vm_flags) &&
727 is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
728 if (vma->vm_pgoff == vm_pgoff)
729 return 1;
730 }
731 return 0;
732}
733
734/*
735 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
736 * beyond (at a higher virtual address and file offset than) the vma.
737 *
738 * We cannot merge two vmas if they have differently assigned (non-NULL)
739 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
740 */
741static int
742can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
743 struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
744{
745 if (is_mergeable_vma(vma, file, vm_flags) &&
746 is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
747 pgoff_t vm_pglen;
748 vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
749 if (vma->vm_pgoff + vm_pglen == vm_pgoff)
750 return 1;
751 }
752 return 0;
753}
754
755/*
756 * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
757 * whether that can be merged with its predecessor or its successor.
758 * Or both (it neatly fills a hole).
759 *
760 * In most cases - when called for mmap, brk or mremap - [addr,end) is
761 * certain not to be mapped by the time vma_merge is called; but when
762 * called for mprotect, it is certain to be already mapped (either at
763 * an offset within prev, or at the start of next), and the flags of
764 * this area are about to be changed to vm_flags - and the no-change
765 * case has already been eliminated.
766 *
767 * The following mprotect cases have to be considered, where AAAA is
768 * the area passed down from mprotect_fixup, never extending beyond one
769 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
770 *
771 * AAAA AAAA AAAA AAAA
772 * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
773 * cannot merge might become might become might become
774 * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
775 * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
776 * mremap move: PPPPNNNNNNNN 8
777 * AAAA
778 * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
779 * might become case 1 below case 2 below case 3 below
780 *
781 * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
782 * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
783 */
784struct vm_area_struct *vma_merge(struct mm_struct *mm,
785 struct vm_area_struct *prev, unsigned long addr,
786 unsigned long end, unsigned long vm_flags,
787 struct anon_vma *anon_vma, struct file *file,
788 pgoff_t pgoff, struct mempolicy *policy)
789{
790 pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
791 struct vm_area_struct *area, *next;
792
793 /*
794 * We later require that vma->vm_flags == vm_flags,
795 * so this tests vma->vm_flags & VM_SPECIAL, too.
796 */
797 if (vm_flags & VM_SPECIAL)
798 return NULL;
799
800 if (prev)
801 next = prev->vm_next;
802 else
803 next = mm->mmap;
804 area = next;
805 if (next && next->vm_end == end) /* cases 6, 7, 8 */
806 next = next->vm_next;
807
808 /*
809 * Can it merge with the predecessor?
810 */
811 if (prev && prev->vm_end == addr &&
812 mpol_equal(vma_policy(prev), policy) &&
813 can_vma_merge_after(prev, vm_flags,
814 anon_vma, file, pgoff)) {
815 /*
816 * OK, it can. Can we now merge in the successor as well?
817 */
818 if (next && end == next->vm_start &&
819 mpol_equal(policy, vma_policy(next)) &&
820 can_vma_merge_before(next, vm_flags,
821 anon_vma, file, pgoff+pglen) &&
822 is_mergeable_anon_vma(prev->anon_vma,
823 next->anon_vma)) {
824 /* cases 1, 6 */
825 vma_adjust(prev, prev->vm_start,
826 next->vm_end, prev->vm_pgoff, NULL);
827 } else /* cases 2, 5, 7 */
828 vma_adjust(prev, prev->vm_start,
829 end, prev->vm_pgoff, NULL);
830 return prev;
831 }
832
833 /*
834 * Can this new request be merged in front of next?
835 */
836 if (next && end == next->vm_start &&
837 mpol_equal(policy, vma_policy(next)) &&
838 can_vma_merge_before(next, vm_flags,
839 anon_vma, file, pgoff+pglen)) {
840 if (prev && addr < prev->vm_end) /* case 4 */
841 vma_adjust(prev, prev->vm_start,
842 addr, prev->vm_pgoff, NULL);
843 else /* cases 3, 8 */
844 vma_adjust(area, addr, next->vm_end,
845 next->vm_pgoff - pglen, NULL);
846 return area;
847 }
848
849 return NULL;
850}
851
852/*
853 * find_mergeable_anon_vma is used by anon_vma_prepare, to check
854 * neighbouring vmas for a suitable anon_vma, before it goes off
855 * to allocate a new anon_vma. It checks because a repetitive
856 * sequence of mprotects and faults may otherwise lead to distinct
857 * anon_vmas being allocated, preventing vma merge in subsequent
858 * mprotect.
859 */
860struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
861{
862 struct vm_area_struct *near;
863 unsigned long vm_flags;
864
865 near = vma->vm_next;
866 if (!near)
867 goto try_prev;
868
869 /*
870 * Since only mprotect tries to remerge vmas, match flags
871 * which might be mprotected into each other later on.
872 * Neither mlock nor madvise tries to remerge at present,
873 * so leave their flags as obstructing a merge.
874 */
875 vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
876 vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
877
878 if (near->anon_vma && vma->vm_end == near->vm_start &&
879 mpol_equal(vma_policy(vma), vma_policy(near)) &&
880 can_vma_merge_before(near, vm_flags,
881 NULL, vma->vm_file, vma->vm_pgoff +
882 ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))
883 return near->anon_vma;
884try_prev:
885 /*
886 * It is potentially slow to have to call find_vma_prev here.
887 * But it's only on the first write fault on the vma, not
888 * every time, and we could devise a way to avoid it later
889 * (e.g. stash info in next's anon_vma_node when assigning
890 * an anon_vma, or when trying vma_merge). Another time.
891 */
46a350ef 892 BUG_ON(find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma);
1da177e4
LT
893 if (!near)
894 goto none;
895
896 vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
897 vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
898
899 if (near->anon_vma && near->vm_end == vma->vm_start &&
900 mpol_equal(vma_policy(near), vma_policy(vma)) &&
901 can_vma_merge_after(near, vm_flags,
902 NULL, vma->vm_file, vma->vm_pgoff))
903 return near->anon_vma;
904none:
905 /*
906 * There's no absolute need to look only at touching neighbours:
907 * we could search further afield for "compatible" anon_vmas.
908 * But it would probably just be a waste of time searching,
909 * or lead to too many vmas hanging off the same anon_vma.
910 * We're trying to allow mprotect remerging later on,
911 * not trying to minimize memory used for anon_vmas.
912 */
913 return NULL;
914}
915
916#ifdef CONFIG_PROC_FS
ab50b8ed 917void vm_stat_account(struct mm_struct *mm, unsigned long flags,
1da177e4
LT
918 struct file *file, long pages)
919{
920 const unsigned long stack_flags
921 = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
922
1da177e4
LT
923 if (file) {
924 mm->shared_vm += pages;
925 if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
926 mm->exec_vm += pages;
927 } else if (flags & stack_flags)
928 mm->stack_vm += pages;
929 if (flags & (VM_RESERVED|VM_IO))
930 mm->reserved_vm += pages;
931}
932#endif /* CONFIG_PROC_FS */
933
934/*
27f5de79 935 * The caller must hold down_write(&current->mm->mmap_sem).
1da177e4
LT
936 */
937
48aae425 938unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1da177e4
LT
939 unsigned long len, unsigned long prot,
940 unsigned long flags, unsigned long pgoff)
941{
942 struct mm_struct * mm = current->mm;
1da177e4
LT
943 struct inode *inode;
944 unsigned int vm_flags;
1da177e4 945 int error;
0165ab44 946 unsigned long reqprot = prot;
1da177e4 947
1da177e4
LT
948 /*
949 * Does the application expect PROT_READ to imply PROT_EXEC?
950 *
951 * (the exception is when the underlying filesystem is noexec
952 * mounted, in which case we dont add PROT_EXEC.)
953 */
954 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
d3ac7f89 955 if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
1da177e4
LT
956 prot |= PROT_EXEC;
957
958 if (!len)
98447fa5 959 return -EINVAL;
1da177e4 960
7cd94146
EP
961 if (!(flags & MAP_FIXED))
962 addr = round_hint_to_min(addr);
963
1da177e4
LT
964 /* Careful about overflows.. */
965 len = PAGE_ALIGN(len);
1f51eb3a 966 if (!len)
1da177e4
LT
967 return -ENOMEM;
968
969 /* offset overflow? */
970 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
971 return -EOVERFLOW;
972
973 /* Too many mappings? */
974 if (mm->map_count > sysctl_max_map_count)
975 return -ENOMEM;
976
977 /* Obtain the address to map to. we verify (or select) it and ensure
978 * that it represents a valid section of the address space.
979 */
980 addr = get_unmapped_area(file, addr, len, pgoff, flags);
981 if (addr & ~PAGE_MASK)
982 return addr;
983
984 /* Do simple checking here so the lower-level routines won't have
985 * to. we assume access permissions have been handled by the open
986 * of the memory object, so we don't do any here.
987 */
988 vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
989 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
990
cdf7b341 991 if (flags & MAP_LOCKED)
1da177e4
LT
992 if (!can_do_mlock())
993 return -EPERM;
ba470de4 994
1da177e4
LT
995 /* mlock MCL_FUTURE? */
996 if (vm_flags & VM_LOCKED) {
997 unsigned long locked, lock_limit;
93ea1d0a
CW
998 locked = len >> PAGE_SHIFT;
999 locked += mm->locked_vm;
1da177e4 1000 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
93ea1d0a 1001 lock_limit >>= PAGE_SHIFT;
1da177e4
LT
1002 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
1003 return -EAGAIN;
1004 }
1005
d3ac7f89 1006 inode = file ? file->f_path.dentry->d_inode : NULL;
1da177e4
LT
1007
1008 if (file) {
1009 switch (flags & MAP_TYPE) {
1010 case MAP_SHARED:
1011 if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1012 return -EACCES;
1013
1014 /*
1015 * Make sure we don't allow writing to an append-only
1016 * file..
1017 */
1018 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1019 return -EACCES;
1020
1021 /*
1022 * Make sure there are no mandatory locks on the file.
1023 */
1024 if (locks_verify_locked(inode))
1025 return -EAGAIN;
1026
1027 vm_flags |= VM_SHARED | VM_MAYSHARE;
1028 if (!(file->f_mode & FMODE_WRITE))
1029 vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1030
1031 /* fall through */
1032 case MAP_PRIVATE:
1033 if (!(file->f_mode & FMODE_READ))
1034 return -EACCES;
d3ac7f89 1035 if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
80c5606c
LT
1036 if (vm_flags & VM_EXEC)
1037 return -EPERM;
1038 vm_flags &= ~VM_MAYEXEC;
1039 }
80c5606c
LT
1040
1041 if (!file->f_op || !file->f_op->mmap)
1042 return -ENODEV;
1da177e4
LT
1043 break;
1044
1045 default:
1046 return -EINVAL;
1047 }
1048 } else {
1049 switch (flags & MAP_TYPE) {
1050 case MAP_SHARED:
ce363942
TH
1051 /*
1052 * Ignore pgoff.
1053 */
1054 pgoff = 0;
1da177e4
LT
1055 vm_flags |= VM_SHARED | VM_MAYSHARE;
1056 break;
1057 case MAP_PRIVATE:
1058 /*
1059 * Set pgoff according to addr for anon_vma.
1060 */
1061 pgoff = addr >> PAGE_SHIFT;
1062 break;
1063 default:
1064 return -EINVAL;
1065 }
1066 }
1067
ed032189 1068 error = security_file_mmap(file, reqprot, prot, flags, addr, 0);
1da177e4
LT
1069 if (error)
1070 return error;
6146f0d5
MZ
1071 error = ima_file_mmap(file, prot);
1072 if (error)
1073 return error;
ed032189 1074
5a6fe125 1075 return mmap_region(file, addr, len, flags, vm_flags, pgoff);
0165ab44
MS
1076}
1077EXPORT_SYMBOL(do_mmap_pgoff);
1078
4e950f6f
AD
1079/*
1080 * Some shared mappigns will want the pages marked read-only
1081 * to track write events. If so, we'll downgrade vm_page_prot
1082 * to the private version (using protection_map[] without the
1083 * VM_SHARED bit).
1084 */
1085int vma_wants_writenotify(struct vm_area_struct *vma)
1086{
1087 unsigned int vm_flags = vma->vm_flags;
1088
1089 /* If it was private or non-writable, the write bit is already clear */
1090 if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
1091 return 0;
1092
1093 /* The backer wishes to know when pages are first written to? */
1094 if (vma->vm_ops && vma->vm_ops->page_mkwrite)
1095 return 1;
1096
1097 /* The open routine did something to the protections already? */
1098 if (pgprot_val(vma->vm_page_prot) !=
3ed75eb8 1099 pgprot_val(vm_get_page_prot(vm_flags)))
4e950f6f
AD
1100 return 0;
1101
1102 /* Specialty mapping? */
1103 if (vm_flags & (VM_PFNMAP|VM_INSERTPAGE))
1104 return 0;
1105
1106 /* Can the mapping track the dirty pages? */
1107 return vma->vm_file && vma->vm_file->f_mapping &&
1108 mapping_cap_account_dirty(vma->vm_file->f_mapping);
1109}
1110
fc8744ad
LT
1111/*
1112 * We account for memory if it's a private writeable mapping,
5a6fe125 1113 * not hugepages and VM_NORESERVE wasn't set.
fc8744ad 1114 */
5a6fe125 1115static inline int accountable_mapping(struct file *file, unsigned int vm_flags)
fc8744ad 1116{
5a6fe125
MG
1117 /*
1118 * hugetlb has its own accounting separate from the core VM
1119 * VM_HUGETLB may not be set yet so we cannot check for that flag.
1120 */
1121 if (file && is_file_hugepages(file))
1122 return 0;
1123
fc8744ad
LT
1124 return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1125}
1126
0165ab44
MS
1127unsigned long mmap_region(struct file *file, unsigned long addr,
1128 unsigned long len, unsigned long flags,
5a6fe125 1129 unsigned int vm_flags, unsigned long pgoff)
0165ab44
MS
1130{
1131 struct mm_struct *mm = current->mm;
1132 struct vm_area_struct *vma, *prev;
1133 int correct_wcount = 0;
1134 int error;
1135 struct rb_node **rb_link, *rb_parent;
1136 unsigned long charged = 0;
1137 struct inode *inode = file ? file->f_path.dentry->d_inode : NULL;
5198e6ea 1138 unsigned long ub_charged = 0;
0165ab44 1139
1da177e4
LT
1140 /* Clear old maps */
1141 error = -ENOMEM;
1142munmap_back:
1143 vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
1144 if (vma && vma->vm_start < addr + len) {
1145 if (do_munmap(mm, addr, len))
1146 return -ENOMEM;
1147 goto munmap_back;
1148 }
1149
1150 /* Check against address space limit. */
119f657c 1151 if (!may_expand_vm(mm, len >> PAGE_SHIFT))
1da177e4
LT
1152 return -ENOMEM;
1153
fc8744ad
LT
1154 /*
1155 * Set 'VM_NORESERVE' if we should not account for the
5a6fe125 1156 * memory use of this mapping.
fc8744ad 1157 */
5a6fe125
MG
1158 if ((flags & MAP_NORESERVE)) {
1159 /* We honor MAP_NORESERVE if allowed to overcommit */
1160 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1161 vm_flags |= VM_NORESERVE;
1162
1163 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
1164 if (file && is_file_hugepages(file))
1165 vm_flags |= VM_NORESERVE;
1166 }
cdfd4325 1167
fc8744ad
LT
1168 /*
1169 * Private writable mapping: check memory availability
1170 */
5a6fe125 1171 if (accountable_mapping(file, vm_flags)) {
fc8744ad
LT
1172 charged = len >> PAGE_SHIFT;
1173 if (security_vm_enough_memory(charged))
1174 return -ENOMEM;
1175 vm_flags |= VM_ACCOUNT;
1da177e4
LT
1176 }
1177
5198e6ea
PE
1178 if (ub_memory_charge(mm, len, vm_flags, file,
1179 (flags & MAP_EXECPRIO ? UB_SOFT : UB_HARD)))
1180 goto charge_error;
1181 ub_charged = 1;
1182
1da177e4 1183 /*
de33c8db 1184 * Can we just expand an old mapping?
1da177e4 1185 */
de33c8db
LT
1186 vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff, NULL);
1187 if (vma)
1188 goto out;
1da177e4
LT
1189
1190 /*
1191 * Determine the object being mapped and call the appropriate
1192 * specific mapper. the address has already been validated, but
1193 * not unmapped, but the maps are removed from the list.
1194 */
5198e6ea
PE
1195 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL |
1196 (flags & MAP_EXECPRIO ? __GFP_SOFT_UBC : 0));
1da177e4
LT
1197 if (!vma) {
1198 error = -ENOMEM;
1199 goto unacct_error;
1200 }
1da177e4
LT
1201
1202 vma->vm_mm = mm;
1203 vma->vm_start = addr;
1204 vma->vm_end = addr + len;
1205 vma->vm_flags = vm_flags;
3ed75eb8 1206 vma->vm_page_prot = vm_get_page_prot(vm_flags);
1da177e4
LT
1207 vma->vm_pgoff = pgoff;
1208
1209 if (file) {
1210 error = -EINVAL;
1211 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1212 goto free_vma;
1213 if (vm_flags & VM_DENYWRITE) {
1214 error = deny_write_access(file);
1215 if (error)
1216 goto free_vma;
1217 correct_wcount = 1;
1218 }
1219 vma->vm_file = file;
1220 get_file(file);
1221 error = file->f_op->mmap(file, vma);
1222 if (error)
1223 goto unmap_and_free_vma;
925d1c40
MH
1224 if (vm_flags & VM_EXECUTABLE)
1225 added_exe_file_vma(mm);
5198e6ea
PE
1226 if (vm_flags != vma->vm_flags) {
1227 /*
1228 * ->vm_flags has been changed in f_op->mmap method.
1229 * We have to recharge ub memory.
1230 */
1231 ub_memory_uncharge(mm, len, vm_flags, file);
1232 if (ub_memory_charge(mm, len, vma->vm_flags, file,
1233 (flags & MAP_EXECPRIO ? UB_SOFT : UB_HARD))) {
1234 ub_charged = 0;
1235 error = -ENOMEM;
1236 goto unmap_and_free_vma;
1237 }
1238 }
f8dbf0a7
HS
1239
1240 /* Can addr have changed??
1241 *
1242 * Answer: Yes, several device drivers can do it in their
1243 * f_op->mmap method. -DaveM
1244 */
1245 addr = vma->vm_start;
1246 pgoff = vma->vm_pgoff;
1247 vm_flags = vma->vm_flags;
1da177e4
LT
1248 } else if (vm_flags & VM_SHARED) {
1249 error = shmem_zero_setup(vma);
1250 if (error)
1251 goto free_vma;
1252 }
1253
d08b3851 1254 if (vma_wants_writenotify(vma))
1ddd439e 1255 vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
d08b3851 1256
de33c8db
LT
1257 vma_link(mm, vma, prev, rb_link, rb_parent);
1258 file = vma->vm_file;
4d3d5b41
ON
1259
1260 /* Once vma denies write, undo our temporary denial count */
1261 if (correct_wcount)
1262 atomic_inc(&inode->i_writecount);
1263out:
cdd6c482 1264 perf_event_mmap(vma);
0a4a9391 1265
1da177e4 1266 mm->total_vm += len >> PAGE_SHIFT;
ab50b8ed 1267 vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
1da177e4 1268 if (vm_flags & VM_LOCKED) {
ba470de4
RR
1269 /*
1270 * makes pages present; downgrades, drops, reacquires mmap_sem
1271 */
1272 long nr_pages = mlock_vma_pages_range(vma, addr, addr + len);
1273 if (nr_pages < 0)
1274 return nr_pages; /* vma gone! */
1275 mm->locked_vm += (len >> PAGE_SHIFT) - nr_pages;
1276 } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
54cb8821 1277 make_pages_present(addr, addr + len);
1da177e4
LT
1278 return addr;
1279
1280unmap_and_free_vma:
1281 if (correct_wcount)
1282 atomic_inc(&inode->i_writecount);
1283 vma->vm_file = NULL;
1284 fput(file);
1285
1286 /* Undo any partial mapping done by a device driver. */
e0da382c
HD
1287 unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1288 charged = 0;
1da177e4
LT
1289free_vma:
1290 kmem_cache_free(vm_area_cachep, vma);
1291unacct_error:
5198e6ea
PE
1292 if (ub_charged)
1293 ub_memory_uncharge(mm, len, vm_flags, file);
1294charge_error:
1da177e4
LT
1295 if (charged)
1296 vm_unacct_memory(charged);
1297 return error;
1298}
1299
1da177e4
LT
1300/* Get an address range which is currently unmapped.
1301 * For shmat() with addr=0.
1302 *
1303 * Ugly calling convention alert:
1304 * Return value with the low bits set means error value,
1305 * ie
1306 * if (ret & ~PAGE_MASK)
1307 * error = ret;
1308 *
1309 * This function "knows" that -ENOMEM has the bits set.
1310 */
1311#ifndef HAVE_ARCH_UNMAPPED_AREA
1312unsigned long
1313arch_get_unmapped_area(struct file *filp, unsigned long addr,
1314 unsigned long len, unsigned long pgoff, unsigned long flags)
1315{
1316 struct mm_struct *mm = current->mm;
1317 struct vm_area_struct *vma;
1318 unsigned long start_addr;
1319
1320 if (len > TASK_SIZE)
1321 return -ENOMEM;
1322
06abdfb4
BH
1323 if (flags & MAP_FIXED)
1324 return addr;
1325
1da177e4
LT
1326 if (addr) {
1327 addr = PAGE_ALIGN(addr);
1328 vma = find_vma(mm, addr);
1329 if (TASK_SIZE - len >= addr &&
1330 (!vma || addr + len <= vma->vm_start))
1331 return addr;
1332 }
1363c3cd
WW
1333 if (len > mm->cached_hole_size) {
1334 start_addr = addr = mm->free_area_cache;
1335 } else {
1336 start_addr = addr = TASK_UNMAPPED_BASE;
1337 mm->cached_hole_size = 0;
1338 }
1da177e4
LT
1339
1340full_search:
1341 for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
1342 /* At this point: (!vma || addr < vma->vm_end). */
1343 if (TASK_SIZE - len < addr) {
1344 /*
1345 * Start a new search - just in case we missed
1346 * some holes.
1347 */
1348 if (start_addr != TASK_UNMAPPED_BASE) {
1363c3cd
WW
1349 addr = TASK_UNMAPPED_BASE;
1350 start_addr = addr;
1351 mm->cached_hole_size = 0;
1da177e4
LT
1352 goto full_search;
1353 }
1354 return -ENOMEM;
1355 }
1356 if (!vma || addr + len <= vma->vm_start) {
1357 /*
1358 * Remember the place where we stopped the search:
1359 */
1360 mm->free_area_cache = addr + len;
1361 return addr;
1362 }
1363c3cd
WW
1363 if (addr + mm->cached_hole_size < vma->vm_start)
1364 mm->cached_hole_size = vma->vm_start - addr;
1da177e4
LT
1365 addr = vma->vm_end;
1366 }
1367}
1368#endif
1369
1363c3cd 1370void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
1da177e4
LT
1371{
1372 /*
1373 * Is this a new hole at the lowest possible address?
1374 */
1363c3cd
WW
1375 if (addr >= TASK_UNMAPPED_BASE && addr < mm->free_area_cache) {
1376 mm->free_area_cache = addr;
1377 mm->cached_hole_size = ~0UL;
1378 }
1da177e4
LT
1379}
1380
1381/*
1382 * This mmap-allocator allocates new areas top-down from below the
1383 * stack's low limit (the base):
1384 */
1385#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1386unsigned long
1387arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1388 const unsigned long len, const unsigned long pgoff,
1389 const unsigned long flags)
1390{
1391 struct vm_area_struct *vma;
1392 struct mm_struct *mm = current->mm;
1393 unsigned long addr = addr0;
1394
1395 /* requested length too big for entire address space */
1396 if (len > TASK_SIZE)
1397 return -ENOMEM;
1398
06abdfb4
BH
1399 if (flags & MAP_FIXED)
1400 return addr;
1401
1da177e4
LT
1402 /* requesting a specific address */
1403 if (addr) {
1404 addr = PAGE_ALIGN(addr);
1405 vma = find_vma(mm, addr);
1406 if (TASK_SIZE - len >= addr &&
1407 (!vma || addr + len <= vma->vm_start))
1408 return addr;
1409 }
1410
1363c3cd
WW
1411 /* check if free_area_cache is useful for us */
1412 if (len <= mm->cached_hole_size) {
1413 mm->cached_hole_size = 0;
1414 mm->free_area_cache = mm->mmap_base;
1415 }
1416
1da177e4
LT
1417 /* either no address requested or can't fit in requested address hole */
1418 addr = mm->free_area_cache;
1419
1420 /* make sure it can fit in the remaining address space */
49a43876 1421 if (addr > len) {
1da177e4
LT
1422 vma = find_vma(mm, addr-len);
1423 if (!vma || addr <= vma->vm_start)
1424 /* remember the address as a hint for next time */
1425 return (mm->free_area_cache = addr-len);
1426 }
1427
73219d17
CW
1428 if (mm->mmap_base < len)
1429 goto bottomup;
1430
1da177e4
LT
1431 addr = mm->mmap_base-len;
1432
1433 do {
1434 /*
1435 * Lookup failure means no vma is above this address,
1436 * else if new region fits below vma->vm_start,
1437 * return with success:
1438 */
1439 vma = find_vma(mm, addr);
1440 if (!vma || addr+len <= vma->vm_start)
1441 /* remember the address as a hint for next time */
1442 return (mm->free_area_cache = addr);
1443
1363c3cd
WW
1444 /* remember the largest hole we saw so far */
1445 if (addr + mm->cached_hole_size < vma->vm_start)
1446 mm->cached_hole_size = vma->vm_start - addr;
1447
1da177e4
LT
1448 /* try just below the current vma->vm_start */
1449 addr = vma->vm_start-len;
49a43876 1450 } while (len < vma->vm_start);
1da177e4 1451
73219d17 1452bottomup:
1da177e4
LT
1453 /*
1454 * A failed mmap() very likely causes application failure,
1455 * so fall back to the bottom-up function here. This scenario
1456 * can happen with large stack limits and large mmap()
1457 * allocations.
1458 */
1363c3cd
WW
1459 mm->cached_hole_size = ~0UL;
1460 mm->free_area_cache = TASK_UNMAPPED_BASE;
1da177e4
LT
1461 addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
1462 /*
1463 * Restore the topdown base:
1464 */
1465 mm->free_area_cache = mm->mmap_base;
1363c3cd 1466 mm->cached_hole_size = ~0UL;
1da177e4
LT
1467
1468 return addr;
1469}
1470#endif
1471
1363c3cd 1472void arch_unmap_area_topdown(struct mm_struct *mm, unsigned long addr)
1da177e4
LT
1473{
1474 /*
1475 * Is this a new hole at the highest possible address?
1476 */
1363c3cd
WW
1477 if (addr > mm->free_area_cache)
1478 mm->free_area_cache = addr;
1da177e4
LT
1479
1480 /* dont allow allocations above current base */
1363c3cd
WW
1481 if (mm->free_area_cache > mm->mmap_base)
1482 mm->free_area_cache = mm->mmap_base;
1da177e4
LT
1483}
1484
1485unsigned long
1486get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
1487 unsigned long pgoff, unsigned long flags)
1488{
06abdfb4
BH
1489 unsigned long (*get_area)(struct file *, unsigned long,
1490 unsigned long, unsigned long, unsigned long);
1491
1f51eb3a
AV
1492 unsigned long error = arch_mmap_check(addr, len, flags);
1493 if (error)
1494 return error;
1495
1496 /* Careful about overflows.. */
1497 if (len > TASK_SIZE)
1498 return -ENOMEM;
1499
06abdfb4
BH
1500 get_area = current->mm->get_unmapped_area;
1501 if (file && file->f_op && file->f_op->get_unmapped_area)
1502 get_area = file->f_op->get_unmapped_area;
1503 addr = get_area(file, addr, len, pgoff, flags);
1504 if (IS_ERR_VALUE(addr))
1505 return addr;
1da177e4 1506
07ab67c8
LT
1507 if (addr > TASK_SIZE - len)
1508 return -ENOMEM;
1509 if (addr & ~PAGE_MASK)
1510 return -EINVAL;
06abdfb4 1511
08e7d9b5 1512 return arch_rebalance_pgtables(addr, len);
1da177e4
LT
1513}
1514
1515EXPORT_SYMBOL(get_unmapped_area);
1516
1517/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
48aae425 1518struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
1da177e4
LT
1519{
1520 struct vm_area_struct *vma = NULL;
1521
1522 if (mm) {
1523 /* Check the cache first. */
1524 /* (Cache hit rate is typically around 35%.) */
1525 vma = mm->mmap_cache;
1526 if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
1527 struct rb_node * rb_node;
1528
1529 rb_node = mm->mm_rb.rb_node;
1530 vma = NULL;
1531
1532 while (rb_node) {
1533 struct vm_area_struct * vma_tmp;
1534
1535 vma_tmp = rb_entry(rb_node,
1536 struct vm_area_struct, vm_rb);
1537
1538 if (vma_tmp->vm_end > addr) {
1539 vma = vma_tmp;
1540 if (vma_tmp->vm_start <= addr)
1541 break;
1542 rb_node = rb_node->rb_left;
1543 } else
1544 rb_node = rb_node->rb_right;
1545 }
1546 if (vma)
1547 mm->mmap_cache = vma;
1548 }
1549 }
1550 return vma;
1551}
1552
1553EXPORT_SYMBOL(find_vma);
1554
1555/* Same as find_vma, but also return a pointer to the previous VMA in *pprev. */
1556struct vm_area_struct *
1557find_vma_prev(struct mm_struct *mm, unsigned long addr,
1558 struct vm_area_struct **pprev)
1559{
1560 struct vm_area_struct *vma = NULL, *prev = NULL;
48aae425 1561 struct rb_node *rb_node;
1da177e4
LT
1562 if (!mm)
1563 goto out;
1564
1565 /* Guard against addr being lower than the first VMA */
1566 vma = mm->mmap;
1567
1568 /* Go through the RB tree quickly. */
1569 rb_node = mm->mm_rb.rb_node;
1570
1571 while (rb_node) {
1572 struct vm_area_struct *vma_tmp;
1573 vma_tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
1574
1575 if (addr < vma_tmp->vm_end) {
1576 rb_node = rb_node->rb_left;
1577 } else {
1578 prev = vma_tmp;
1579 if (!prev->vm_next || (addr < prev->vm_next->vm_end))
1580 break;
1581 rb_node = rb_node->rb_right;
1582 }
1583 }
1584
1585out:
1586 *pprev = prev;
1587 return prev ? prev->vm_next : vma;
1588}
1589
1590/*
1591 * Verify that the stack growth is acceptable and
1592 * update accounting. This is shared with both the
1593 * grow-up and grow-down cases.
1594 */
48aae425 1595static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
1da177e4
LT
1596{
1597 struct mm_struct *mm = vma->vm_mm;
1598 struct rlimit *rlim = current->signal->rlim;
0d59a01b 1599 unsigned long new_start;
1da177e4
LT
1600
1601 /* address space limit tests */
119f657c 1602 if (!may_expand_vm(mm, grow))
1da177e4
LT
1603 return -ENOMEM;
1604
1605 /* Stack limit test */
1606 if (size > rlim[RLIMIT_STACK].rlim_cur)
1607 return -ENOMEM;
1608
1609 /* mlock limit tests */
1610 if (vma->vm_flags & VM_LOCKED) {
1611 unsigned long locked;
1612 unsigned long limit;
1613 locked = mm->locked_vm + grow;
1614 limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
1615 if (locked > limit && !capable(CAP_IPC_LOCK))
1616 return -ENOMEM;
1617 }
1618
0d59a01b
AL
1619 /* Check to ensure the stack will not grow into a hugetlb-only region */
1620 new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
1621 vma->vm_end - size;
1622 if (is_hugepage_only_range(vma->vm_mm, new_start, size))
1623 return -EFAULT;
1624
5198e6ea
PE
1625 if (ub_memory_charge(mm, grow << PAGE_SHIFT, vma->vm_flags,
1626 vma->vm_file, UB_SOFT))
1627 goto fail_charge;
1628
1da177e4
LT
1629 /*
1630 * Overcommit.. This must be the final test, as it will
1631 * update security statistics.
1632 */
05fa199d 1633 if (security_vm_enough_memory_mm(mm, grow))
5198e6ea 1634 goto fail_sec;
1da177e4
LT
1635
1636 /* Ok, everything looks good - let it rip */
1637 mm->total_vm += grow;
1638 if (vma->vm_flags & VM_LOCKED)
1639 mm->locked_vm += grow;
ab50b8ed 1640 vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
1da177e4 1641 return 0;
5198e6ea
PE
1642
1643fail_sec:
1644 ub_memory_uncharge(mm, grow << PAGE_SHIFT, vma->vm_flags, vma->vm_file);
1645fail_charge:
1646 return -ENOMEM;
1da177e4
LT
1647}
1648
46dea3d0 1649#if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
1da177e4 1650/*
46dea3d0
HD
1651 * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
1652 * vma is the last one with address > vma->vm_end. Have to extend vma.
1da177e4 1653 */
46dea3d0 1654int expand_upwards(struct vm_area_struct *vma, unsigned long address)
1da177e4
LT
1655{
1656 int error;
1657
1658 if (!(vma->vm_flags & VM_GROWSUP))
1659 return -EFAULT;
1660
1661 /*
1662 * We must make sure the anon_vma is allocated
1663 * so that the anon_vma locking is not a noop.
1664 */
1665 if (unlikely(anon_vma_prepare(vma)))
1666 return -ENOMEM;
1667 anon_vma_lock(vma);
1668
1669 /*
1670 * vma->vm_start/vm_end cannot change under us because the caller
1671 * is required to hold the mmap_sem in read mode. We need the
1672 * anon_vma lock to serialize against concurrent expand_stacks.
06b32f3a 1673 * Also guard against wrapping around to address 0.
1da177e4 1674 */
06b32f3a
HD
1675 if (address < PAGE_ALIGN(address+4))
1676 address = PAGE_ALIGN(address+4);
1677 else {
1678 anon_vma_unlock(vma);
1679 return -ENOMEM;
1680 }
1da177e4
LT
1681 error = 0;
1682
1683 /* Somebody else might have raced and expanded it already */
1684 if (address > vma->vm_end) {
1685 unsigned long size, grow;
1686
1687 size = address - vma->vm_start;
1688 grow = (address - vma->vm_end) >> PAGE_SHIFT;
1689
1690 error = acct_stack_growth(vma, size, grow);
1691 if (!error)
1692 vma->vm_end = address;
1693 }
1694 anon_vma_unlock(vma);
1695 return error;
1696}
46dea3d0
HD
1697#endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
1698
1da177e4
LT
1699/*
1700 * vma is the first one with address < vma->vm_start. Have to extend vma.
1701 */
cb8f488c 1702static int expand_downwards(struct vm_area_struct *vma,
b6a2fea3 1703 unsigned long address)
1da177e4
LT
1704{
1705 int error;
1706
1707 /*
1708 * We must make sure the anon_vma is allocated
1709 * so that the anon_vma locking is not a noop.
1710 */
1711 if (unlikely(anon_vma_prepare(vma)))
1712 return -ENOMEM;
8869477a
EP
1713
1714 address &= PAGE_MASK;
88c3f7a8 1715 error = security_file_mmap(NULL, 0, 0, 0, address, 1);
8869477a
EP
1716 if (error)
1717 return error;
1718
1da177e4
LT
1719 anon_vma_lock(vma);
1720
1721 /*
1722 * vma->vm_start/vm_end cannot change under us because the caller
1723 * is required to hold the mmap_sem in read mode. We need the
1724 * anon_vma lock to serialize against concurrent expand_stacks.
1725 */
1da177e4
LT
1726
1727 /* Somebody else might have raced and expanded it already */
1728 if (address < vma->vm_start) {
1729 unsigned long size, grow;
1730
1731 size = vma->vm_end - address;
1732 grow = (vma->vm_start - address) >> PAGE_SHIFT;
1733
1734 error = acct_stack_growth(vma, size, grow);
1735 if (!error) {
1736 vma->vm_start = address;
1737 vma->vm_pgoff -= grow;
1738 }
1739 }
1740 anon_vma_unlock(vma);
1741 return error;
1742}
1743
b6a2fea3
OW
1744int expand_stack_downwards(struct vm_area_struct *vma, unsigned long address)
1745{
1746 return expand_downwards(vma, address);
1747}
1748
1749#ifdef CONFIG_STACK_GROWSUP
1750int expand_stack(struct vm_area_struct *vma, unsigned long address)
1751{
1752 return expand_upwards(vma, address);
1753}
1754
1755struct vm_area_struct *
1756find_extend_vma(struct mm_struct *mm, unsigned long addr)
1757{
1758 struct vm_area_struct *vma, *prev;
1759
1760 addr &= PAGE_MASK;
1761 vma = find_vma_prev(mm, addr, &prev);
1762 if (vma && (vma->vm_start <= addr))
1763 return vma;
1c127185 1764 if (!prev || expand_stack(prev, addr))
b6a2fea3 1765 return NULL;
ba470de4
RR
1766 if (prev->vm_flags & VM_LOCKED) {
1767 if (mlock_vma_pages_range(prev, addr, prev->vm_end) < 0)
1768 return NULL; /* vma gone! */
1769 }
b6a2fea3
OW
1770 return prev;
1771}
1772#else
1773int expand_stack(struct vm_area_struct *vma, unsigned long address)
1774{
1775 return expand_downwards(vma, address);
1776}
1777
1da177e4
LT
1778struct vm_area_struct *
1779find_extend_vma(struct mm_struct * mm, unsigned long addr)
1780{
1781 struct vm_area_struct * vma;
1782 unsigned long start;
1783
1784 addr &= PAGE_MASK;
1785 vma = find_vma(mm,addr);
1786 if (!vma)
1787 return NULL;
1788 if (vma->vm_start <= addr)
1789 return vma;
1790 if (!(vma->vm_flags & VM_GROWSDOWN))
1791 return NULL;
1792 start = vma->vm_start;
1793 if (expand_stack(vma, addr))
1794 return NULL;
ba470de4
RR
1795 if (vma->vm_flags & VM_LOCKED) {
1796 if (mlock_vma_pages_range(vma, addr, start) < 0)
1797 return NULL; /* vma gone! */
1798 }
1da177e4
LT
1799 return vma;
1800}
1801#endif
1802
1da177e4 1803/*
2c0b3814 1804 * Ok - we have the memory areas we should free on the vma list,
1da177e4 1805 * so release them, and do the vma updates.
2c0b3814
HD
1806 *
1807 * Called with the mm semaphore held.
1da177e4 1808 */
2c0b3814 1809static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
1da177e4 1810{
365e9c87
HD
1811 /* Update high watermark before we lower total_vm */
1812 update_hiwater_vm(mm);
1da177e4 1813 do {
2c0b3814
HD
1814 long nrpages = vma_pages(vma);
1815
1816 mm->total_vm -= nrpages;
2c0b3814 1817 vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
a8fb5618 1818 vma = remove_vma(vma);
146425a3 1819 } while (vma);
1da177e4
LT
1820 validate_mm(mm);
1821}
1822
1823/*
1824 * Get rid of page table information in the indicated region.
1825 *
f10df686 1826 * Called with the mm semaphore held.
1da177e4
LT
1827 */
1828static void unmap_region(struct mm_struct *mm,
e0da382c
HD
1829 struct vm_area_struct *vma, struct vm_area_struct *prev,
1830 unsigned long start, unsigned long end)
1da177e4 1831{
e0da382c 1832 struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
1da177e4
LT
1833 struct mmu_gather *tlb;
1834 unsigned long nr_accounted = 0;
1835
1836 lru_add_drain();
1837 tlb = tlb_gather_mmu(mm, 0);
365e9c87 1838 update_hiwater_rss(mm);
508034a3 1839 unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
1da177e4 1840 vm_unacct_memory(nr_accounted);
42b77728 1841 free_pgtables(tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
e0da382c 1842 next? next->vm_start: 0);
1da177e4
LT
1843 tlb_finish_mmu(tlb, start, end);
1844}
1845
1846/*
1847 * Create a list of vma's touched by the unmap, removing them from the mm's
1848 * vma list as we go..
1849 */
1850static void
1851detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
1852 struct vm_area_struct *prev, unsigned long end)
1853{
1854 struct vm_area_struct **insertion_point;
1855 struct vm_area_struct *tail_vma = NULL;
1363c3cd 1856 unsigned long addr;
1da177e4
LT
1857
1858 insertion_point = (prev ? &prev->vm_next : &mm->mmap);
378776c2 1859 vma->vm_prev = NULL;
1da177e4
LT
1860 do {
1861 rb_erase(&vma->vm_rb, &mm->mm_rb);
1862 mm->map_count--;
1863 tail_vma = vma;
1864 vma = vma->vm_next;
1865 } while (vma && vma->vm_start < end);
1866 *insertion_point = vma;
378776c2
LT
1867 if (vma)
1868 vma->vm_prev = prev;
1da177e4 1869 tail_vma->vm_next = NULL;
1363c3cd
WW
1870 if (mm->unmap_area == arch_unmap_area)
1871 addr = prev ? prev->vm_end : mm->mmap_base;
1872 else
1873 addr = vma ? vma->vm_start : mm->mmap_base;
1874 mm->unmap_area(mm, addr);
1da177e4
LT
1875 mm->mmap_cache = NULL; /* Kill the cache. */
1876}
1877
1878/*
1879 * Split a vma into two pieces at address 'addr', a new vma is allocated
59c51591 1880 * either for the first part or the tail.
1da177e4
LT
1881 */
1882int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
1883 unsigned long addr, int new_below)
1884{
1885 struct mempolicy *pol;
1886 struct vm_area_struct *new;
1887
a5516438
AK
1888 if (is_vm_hugetlb_page(vma) && (addr &
1889 ~(huge_page_mask(hstate_vma(vma)))))
1da177e4
LT
1890 return -EINVAL;
1891
1892 if (mm->map_count >= sysctl_max_map_count)
1893 return -ENOMEM;
1894
e94b1766 1895 new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1da177e4
LT
1896 if (!new)
1897 return -ENOMEM;
1898
1899 /* most fields are the same, copy all, and then fixup */
1900 *new = *vma;
1901
1902 if (new_below)
1903 new->vm_end = addr;
1904 else {
1905 new->vm_start = addr;
1906 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
1907 }
1908
846a16bf 1909 pol = mpol_dup(vma_policy(vma));
1da177e4
LT
1910 if (IS_ERR(pol)) {
1911 kmem_cache_free(vm_area_cachep, new);
1912 return PTR_ERR(pol);
1913 }
1914 vma_set_policy(new, pol);
1915
925d1c40 1916 if (new->vm_file) {
1da177e4 1917 get_file(new->vm_file);
925d1c40
MH
1918 if (vma->vm_flags & VM_EXECUTABLE)
1919 added_exe_file_vma(mm);
1920 }
1da177e4
LT
1921
1922 if (new->vm_ops && new->vm_ops->open)
1923 new->vm_ops->open(new);
1924
1925 if (new_below)
1926 vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
1927 ((addr - new->vm_start) >> PAGE_SHIFT), new);
1928 else
1929 vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
1930
1931 return 0;
1932}
5198e6ea 1933EXPORT_SYMBOL_GPL(split_vma);
1da177e4
LT
1934
1935/* Munmap is split into 2 main parts -- this part which finds
1936 * what needs doing, and the areas themselves, which do the
1937 * work. This now handles partial unmappings.
1938 * Jeremy Fitzhardinge <jeremy@goop.org>
1939 */
1940int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
1941{
1942 unsigned long end;
146425a3 1943 struct vm_area_struct *vma, *prev, *last;
1da177e4
LT
1944
1945 if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
1946 return -EINVAL;
1947
1948 if ((len = PAGE_ALIGN(len)) == 0)
1949 return -EINVAL;
1950
1951 /* Find the first overlapping VMA */
146425a3
HD
1952 vma = find_vma_prev(mm, start, &prev);
1953 if (!vma)
1da177e4 1954 return 0;
146425a3 1955 /* we have start < vma->vm_end */
1da177e4
LT
1956
1957 /* if it doesn't overlap, we have nothing.. */
1958 end = start + len;
146425a3 1959 if (vma->vm_start >= end)
1da177e4
LT
1960 return 0;
1961
1962 /*
1963 * If we need to split any vma, do it now to save pain later.
1964 *
1965 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
1966 * unmapped vm_area_struct will remain in use: so lower split_vma
1967 * places tmp vma above, and higher split_vma places tmp vma below.
1968 */
146425a3
HD
1969 if (start > vma->vm_start) {
1970 int error = split_vma(mm, vma, start, 0);
1da177e4
LT
1971 if (error)
1972 return error;
146425a3 1973 prev = vma;
1da177e4
LT
1974 }
1975
1976 /* Does it split the last one? */
1977 last = find_vma(mm, end);
1978 if (last && end > last->vm_start) {
1979 int error = split_vma(mm, last, end, 1);
1980 if (error)
1981 return error;
1982 }
146425a3 1983 vma = prev? prev->vm_next: mm->mmap;
1da177e4
LT
1984
1985 /*
ba470de4
RR
1986 * unlock any mlock()ed ranges before detaching vmas
1987 */
1988 if (mm->locked_vm) {
1989 struct vm_area_struct *tmp = vma;
1990 while (tmp && tmp->vm_start < end) {
1991 if (tmp->vm_flags & VM_LOCKED) {
1992 mm->locked_vm -= vma_pages(tmp);
1993 munlock_vma_pages_all(tmp);
1994 }
1995 tmp = tmp->vm_next;
1996 }
1997 }
1998
1999 /*
1da177e4
LT
2000 * Remove the vma's, and unmap the actual pages
2001 */
146425a3
HD
2002 detach_vmas_to_be_unmapped(mm, vma, prev, end);
2003 unmap_region(mm, vma, prev, start, end);
1da177e4
LT
2004
2005 /* Fix up all other VM information */
2c0b3814 2006 remove_vma_list(mm, vma);
1da177e4
LT
2007
2008 return 0;
2009}
2010
2011EXPORT_SYMBOL(do_munmap);
2012
6a6160a7 2013SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1da177e4
LT
2014{
2015 int ret;
2016 struct mm_struct *mm = current->mm;
2017
2018 profile_munmap(addr);
2019
2020 down_write(&mm->mmap_sem);
2021 ret = do_munmap(mm, addr, len);
2022 up_write(&mm->mmap_sem);
2023 return ret;
2024}
2025
2026static inline void verify_mm_writelocked(struct mm_struct *mm)
2027{
a241ec65 2028#ifdef CONFIG_DEBUG_VM
1da177e4
LT
2029 if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2030 WARN_ON(1);
2031 up_read(&mm->mmap_sem);
2032 }
2033#endif
2034}
2035
2036/*
2037 * this is really a simplified "do_mmap". it only handles
2038 * anonymous maps. eventually we may be able to do some
2039 * brk-specific accounting here.
2040 */
5198e6ea 2041static unsigned long __do_brk(unsigned long addr, unsigned long len, int soft)
1da177e4
LT
2042{
2043 struct mm_struct * mm = current->mm;
2044 struct vm_area_struct * vma, * prev;
2045 unsigned long flags;
2046 struct rb_node ** rb_link, * rb_parent;
2047 pgoff_t pgoff = addr >> PAGE_SHIFT;
3a459756 2048 int error;
1da177e4
LT
2049
2050 len = PAGE_ALIGN(len);
2051 if (!len)
2052 return addr;
2053
88c3f7a8 2054 error = security_file_mmap(NULL, 0, 0, 0, addr, 1);
5a211a5d
EP
2055 if (error)
2056 return error;
2057
3a459756
KK
2058 flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2059
1f51eb3a
AV
2060 error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2061 if (error & ~PAGE_MASK)
3a459756
KK
2062 return error;
2063
1da177e4
LT
2064 /*
2065 * mlock MCL_FUTURE?
2066 */
2067 if (mm->def_flags & VM_LOCKED) {
2068 unsigned long locked, lock_limit;
93ea1d0a
CW
2069 locked = len >> PAGE_SHIFT;
2070 locked += mm->locked_vm;
1da177e4 2071 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
93ea1d0a 2072 lock_limit >>= PAGE_SHIFT;
1da177e4
LT
2073 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
2074 return -EAGAIN;
2075 }
2076
2077 /*
2078 * mm->mmap_sem is required to protect against another thread
2079 * changing the mappings in case we sleep.
2080 */
2081 verify_mm_writelocked(mm);
2082
2083 /*
2084 * Clear old maps. this also does some error checking for us
2085 */
2086 munmap_back:
2087 vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
2088 if (vma && vma->vm_start < addr + len) {
2089 if (do_munmap(mm, addr, len))
2090 return -ENOMEM;
2091 goto munmap_back;
2092 }
2093
2094 /* Check against address space limits *after* clearing old maps... */
119f657c 2095 if (!may_expand_vm(mm, len >> PAGE_SHIFT))
1da177e4
LT
2096 return -ENOMEM;
2097
2098 if (mm->map_count > sysctl_max_map_count)
2099 return -ENOMEM;
2100
5198e6ea
PE
2101 if (ub_memory_charge(mm, len, flags, NULL, soft))
2102 goto fail_charge;
2103
1da177e4 2104 if (security_vm_enough_memory(len >> PAGE_SHIFT))
5198e6ea 2105 goto fail_sec;
1da177e4 2106
1da177e4 2107 /* Can we just expand an old private anonymous mapping? */
ba470de4
RR
2108 vma = vma_merge(mm, prev, addr, addr + len, flags,
2109 NULL, NULL, pgoff, NULL);
2110 if (vma)
1da177e4
LT
2111 goto out;
2112
2113 /*
2114 * create a vma struct for an anonymous mapping
2115 */
5198e6ea
PE
2116 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL |
2117 (soft == UB_SOFT ? __GFP_SOFT_UBC : 0));
2118 if (!vma)
2119 goto fail_alloc;
1da177e4
LT
2120
2121 vma->vm_mm = mm;
2122 vma->vm_start = addr;
2123 vma->vm_end = addr + len;
2124 vma->vm_pgoff = pgoff;
2125 vma->vm_flags = flags;
3ed75eb8 2126 vma->vm_page_prot = vm_get_page_prot(flags);
1da177e4
LT
2127 vma_link(mm, vma, prev, rb_link, rb_parent);
2128out:
2129 mm->total_vm += len >> PAGE_SHIFT;
2130 if (flags & VM_LOCKED) {
ba470de4
RR
2131 if (!mlock_vma_pages_range(vma, addr, addr + len))
2132 mm->locked_vm += (len >> PAGE_SHIFT);
1da177e4
LT
2133 }
2134 return addr;
5198e6ea
PE
2135
2136fail_alloc:
2137 vm_unacct_memory(len >> PAGE_SHIFT);
2138fail_sec:
2139 ub_memory_uncharge(mm, len, flags, NULL);
2140fail_charge:
2141 return -ENOMEM;
1da177e4
LT
2142}
2143
5198e6ea
PE
2144unsigned long do_brk(unsigned long addr, unsigned long len)
2145{
2146 return __do_brk(addr, len, UB_SOFT);
2147}
1da177e4
LT
2148EXPORT_SYMBOL(do_brk);
2149
2150/* Release all mmaps. */
2151void exit_mmap(struct mm_struct *mm)
2152{
2153 struct mmu_gather *tlb;
ba470de4 2154 struct vm_area_struct *vma;
1da177e4 2155 unsigned long nr_accounted = 0;
ee39b37b 2156 unsigned long end;
1da177e4 2157
d6dd61c8 2158 /* mm's last user has gone, and its about to be pulled down */
cddb8a5c 2159 mmu_notifier_release(mm);
d6dd61c8 2160
ba470de4
RR
2161 if (mm->locked_vm) {
2162 vma = mm->mmap;
2163 while (vma) {
2164 if (vma->vm_flags & VM_LOCKED)
2165 munlock_vma_pages_all(vma);
2166 vma = vma->vm_next;
2167 }
2168 }
9480c53e
JF
2169
2170 arch_exit_mmap(mm);
2171
ba470de4 2172 vma = mm->mmap;
9480c53e
JF
2173 if (!vma) /* Can happen if dup_mmap() received an OOM */
2174 return;
2175
1da177e4 2176 lru_add_drain();
1da177e4 2177 flush_cache_mm(mm);
e0da382c 2178 tlb = tlb_gather_mmu(mm, 1);
901608d9 2179 /* update_hiwater_rss(mm) here? but nobody should be looking */
e0da382c 2180 /* Use -1 here to ensure all VMAs in the mm are unmapped */
508034a3 2181 end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
1da177e4 2182 vm_unacct_memory(nr_accounted);
9ba69294 2183
42b77728 2184 free_pgtables(tlb, vma, FIRST_USER_ADDRESS, 0);
ee39b37b 2185 tlb_finish_mmu(tlb, 0, end);
1da177e4 2186
1da177e4 2187 /*
8f4f8c16
HD
2188 * Walk the list again, actually closing and freeing it,
2189 * with preemption enabled, without holding any MM locks.
1da177e4 2190 */
a8fb5618
HD
2191 while (vma)
2192 vma = remove_vma(vma);
e0da382c 2193
e2cdef8c 2194 BUG_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
1da177e4
LT
2195}
2196
2197/* Insert vm structure into process list sorted by address
2198 * and into the inode's i_mmap tree. If vm_file is non-NULL
2199 * then i_mmap_lock is taken here.
2200 */
2201int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
2202{
2203 struct vm_area_struct * __vma, * prev;
2204 struct rb_node ** rb_link, * rb_parent;
2205
2206 /*
2207 * The vm_pgoff of a purely anonymous vma should be irrelevant
2208 * until its first write fault, when page's anon_vma and index
2209 * are set. But now set the vm_pgoff it will almost certainly
2210 * end up with (unless mremap moves it elsewhere before that
2211 * first wfault), so /proc/pid/maps tells a consistent story.
2212 *
2213 * By setting it to reflect the virtual start address of the
2214 * vma, merges and splits can happen in a seamless way, just
2215 * using the existing file pgoff checks and manipulations.
2216 * Similarly in do_mmap_pgoff and in do_brk.
2217 */
2218 if (!vma->vm_file) {
2219 BUG_ON(vma->anon_vma);
2220 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2221 }
2222 __vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent);
2223 if (__vma && __vma->vm_start < vma->vm_end)
2224 return -ENOMEM;
2fd4ef85 2225 if ((vma->vm_flags & VM_ACCOUNT) &&
34b4e4aa 2226 security_vm_enough_memory_mm(mm, vma_pages(vma)))
2fd4ef85 2227 return -ENOMEM;
1da177e4
LT
2228 vma_link(mm, vma, prev, rb_link, rb_parent);
2229 return 0;
2230}
2231
2232/*
2233 * Copy the vma structure to a new location in the same mm,
2234 * prior to moving page table entries, to effect an mremap move.
2235 */
2236struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
2237 unsigned long addr, unsigned long len, pgoff_t pgoff)
2238{
2239 struct vm_area_struct *vma = *vmap;
2240 unsigned long vma_start = vma->vm_start;
2241 struct mm_struct *mm = vma->vm_mm;
2242 struct vm_area_struct *new_vma, *prev;
2243 struct rb_node **rb_link, *rb_parent;
2244 struct mempolicy *pol;
2245
2246 /*
2247 * If anonymous vma has not yet been faulted, update new pgoff
2248 * to match new location, to increase its chance of merging.
2249 */
2250 if (!vma->vm_file && !vma->anon_vma)
2251 pgoff = addr >> PAGE_SHIFT;
2252
2253 find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
2254 new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2255 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
2256 if (new_vma) {
2257 /*
2258 * Source vma may have been merged into new_vma
2259 */
2260 if (vma_start >= new_vma->vm_start &&
2261 vma_start < new_vma->vm_end)
2262 *vmap = new_vma;
2263 } else {
e94b1766 2264 new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1da177e4
LT
2265 if (new_vma) {
2266 *new_vma = *vma;
846a16bf 2267 pol = mpol_dup(vma_policy(vma));
1da177e4
LT
2268 if (IS_ERR(pol)) {
2269 kmem_cache_free(vm_area_cachep, new_vma);
2270 return NULL;
2271 }
2272 vma_set_policy(new_vma, pol);
2273 new_vma->vm_start = addr;
2274 new_vma->vm_end = addr + len;
2275 new_vma->vm_pgoff = pgoff;
925d1c40 2276 if (new_vma->vm_file) {
1da177e4 2277 get_file(new_vma->vm_file);
925d1c40
MH
2278 if (vma->vm_flags & VM_EXECUTABLE)
2279 added_exe_file_vma(mm);
2280 }
1da177e4
LT
2281 if (new_vma->vm_ops && new_vma->vm_ops->open)
2282 new_vma->vm_ops->open(new_vma);
2283 vma_link(mm, new_vma, prev, rb_link, rb_parent);
2284 }
2285 }
2286 return new_vma;
2287}
119f657c 2288
2289/*
2290 * Return true if the calling process may expand its vm space by the passed
2291 * number of pages
2292 */
2293int may_expand_vm(struct mm_struct *mm, unsigned long npages)
2294{
2295 unsigned long cur = mm->total_vm; /* pages */
2296 unsigned long lim;
2297
2298 lim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
2299
2300 if (cur + npages > lim)
2301 return 0;
2302 return 1;
2303}
fa5dc22f
RM
2304
2305
b1d0e4f5
NP
2306static int special_mapping_fault(struct vm_area_struct *vma,
2307 struct vm_fault *vmf)
fa5dc22f 2308{
b1d0e4f5 2309 pgoff_t pgoff;
fa5dc22f
RM
2310 struct page **pages;
2311
b1d0e4f5
NP
2312 /*
2313 * special mappings have no vm_file, and in that case, the mm
2314 * uses vm_pgoff internally. So we have to subtract it from here.
2315 * We are allowed to do this because we are the mm; do not copy
2316 * this code into drivers!
2317 */
2318 pgoff = vmf->pgoff - vma->vm_pgoff;
fa5dc22f 2319
b1d0e4f5
NP
2320 for (pages = vma->vm_private_data; pgoff && *pages; ++pages)
2321 pgoff--;
fa5dc22f
RM
2322
2323 if (*pages) {
2324 struct page *page = *pages;
2325 get_page(page);
b1d0e4f5
NP
2326 vmf->page = page;
2327 return 0;
fa5dc22f
RM
2328 }
2329
b1d0e4f5 2330 return VM_FAULT_SIGBUS;
fa5dc22f
RM
2331}
2332
2333/*
2334 * Having a close hook prevents vma merging regardless of flags.
2335 */
2336static void special_mapping_close(struct vm_area_struct *vma)
2337{
2338}
2339
5198e6ea 2340const struct vm_operations_struct special_mapping_vmops = {
fa5dc22f 2341 .close = special_mapping_close,
b1d0e4f5 2342 .fault = special_mapping_fault,
fa5dc22f 2343};
5198e6ea 2344EXPORT_SYMBOL_GPL(special_mapping_vmops);
fa5dc22f
RM
2345
2346/*
2347 * Called with mm->mmap_sem held for writing.
2348 * Insert a new vma covering the given region, with the given flags.
2349 * Its pages are supplied by the given array of struct page *.
2350 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
2351 * The region past the last page supplied will always produce SIGBUS.
2352 * The array pointer and the pages it points to are assumed to stay alive
2353 * for as long as this mapping might exist.
2354 */
2355int install_special_mapping(struct mm_struct *mm,
2356 unsigned long addr, unsigned long len,
2357 unsigned long vm_flags, struct page **pages)
2358{
6c2cde9c 2359 int ret;
fa5dc22f
RM
2360 struct vm_area_struct *vma;
2361
2362 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2363 if (unlikely(vma == NULL))
2364 return -ENOMEM;
2365
2366 vma->vm_mm = mm;
2367 vma->vm_start = addr;
2368 vma->vm_end = addr + len;
2369
2f98735c 2370 vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND;
3ed75eb8 2371 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
fa5dc22f
RM
2372
2373 vma->vm_ops = &special_mapping_vmops;
2374 vma->vm_private_data = pages;
2375
6c2cde9c
TO
2376 ret = security_file_mmap(NULL, 0, 0, 0, vma->vm_start, 1);
2377 if (ret)
2378 goto out;
2379
2380 ret = insert_vm_struct(mm, vma);
2381 if (ret)
2382 goto out;
fa5dc22f
RM
2383
2384 mm->total_vm += len >> PAGE_SHIFT;
2385
cdd6c482 2386 perf_event_mmap(vma);
089dd79d 2387
fa5dc22f 2388 return 0;
6c2cde9c
TO
2389
2390out:
2391 kmem_cache_free(vm_area_cachep, vma);
2392 return ret;
fa5dc22f 2393}
7906d00c
AA
2394
2395static DEFINE_MUTEX(mm_all_locks_mutex);
2396
454ed842 2397static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
7906d00c
AA
2398{
2399 if (!test_bit(0, (unsigned long *) &anon_vma->head.next)) {
2400 /*
2401 * The LSB of head.next can't change from under us
2402 * because we hold the mm_all_locks_mutex.
2403 */
454ed842 2404 spin_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
7906d00c
AA
2405 /*
2406 * We can safely modify head.next after taking the
2407 * anon_vma->lock. If some other vma in this mm shares
2408 * the same anon_vma we won't take it again.
2409 *
2410 * No need of atomic instructions here, head.next
2411 * can't change from under us thanks to the
2412 * anon_vma->lock.
2413 */
2414 if (__test_and_set_bit(0, (unsigned long *)
2415 &anon_vma->head.next))
2416 BUG();
2417 }
2418}
2419
454ed842 2420static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
7906d00c
AA
2421{
2422 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2423 /*
2424 * AS_MM_ALL_LOCKS can't change from under us because
2425 * we hold the mm_all_locks_mutex.
2426 *
2427 * Operations on ->flags have to be atomic because
2428 * even if AS_MM_ALL_LOCKS is stable thanks to the
2429 * mm_all_locks_mutex, there may be other cpus
2430 * changing other bitflags in parallel to us.
2431 */
2432 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
2433 BUG();
454ed842 2434 spin_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
7906d00c
AA
2435 }
2436}
2437
2438/*
2439 * This operation locks against the VM for all pte/vma/mm related
2440 * operations that could ever happen on a certain mm. This includes
2441 * vmtruncate, try_to_unmap, and all page faults.
2442 *
2443 * The caller must take the mmap_sem in write mode before calling
2444 * mm_take_all_locks(). The caller isn't allowed to release the
2445 * mmap_sem until mm_drop_all_locks() returns.
2446 *
2447 * mmap_sem in write mode is required in order to block all operations
2448 * that could modify pagetables and free pages without need of
2449 * altering the vma layout (for example populate_range() with
2450 * nonlinear vmas). It's also needed in write mode to avoid new
2451 * anon_vmas to be associated with existing vmas.
2452 *
2453 * A single task can't take more than one mm_take_all_locks() in a row
2454 * or it would deadlock.
2455 *
2456 * The LSB in anon_vma->head.next and the AS_MM_ALL_LOCKS bitflag in
2457 * mapping->flags avoid to take the same lock twice, if more than one
2458 * vma in this mm is backed by the same anon_vma or address_space.
2459 *
2460 * We can take all the locks in random order because the VM code
2461 * taking i_mmap_lock or anon_vma->lock outside the mmap_sem never
2462 * takes more than one of them in a row. Secondly we're protected
2463 * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
2464 *
2465 * mm_take_all_locks() and mm_drop_all_locks are expensive operations
2466 * that may have to take thousand of locks.
2467 *
2468 * mm_take_all_locks() can fail if it's interrupted by signals.
2469 */
2470int mm_take_all_locks(struct mm_struct *mm)
2471{
2472 struct vm_area_struct *vma;
2473 int ret = -EINTR;
2474
2475 BUG_ON(down_read_trylock(&mm->mmap_sem));
2476
2477 mutex_lock(&mm_all_locks_mutex);
2478
2479 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2480 if (signal_pending(current))
2481 goto out_unlock;
7906d00c 2482 if (vma->vm_file && vma->vm_file->f_mapping)
454ed842 2483 vm_lock_mapping(mm, vma->vm_file->f_mapping);
7906d00c 2484 }
7cd5a02f
PZ
2485
2486 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2487 if (signal_pending(current))
2488 goto out_unlock;
2489 if (vma->anon_vma)
2490 vm_lock_anon_vma(mm, vma->anon_vma);
7906d00c 2491 }
7cd5a02f 2492
7906d00c
AA
2493 ret = 0;
2494
2495out_unlock:
2496 if (ret)
2497 mm_drop_all_locks(mm);
2498
2499 return ret;
2500}
2501
2502static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
2503{
2504 if (test_bit(0, (unsigned long *) &anon_vma->head.next)) {
2505 /*
2506 * The LSB of head.next can't change to 0 from under
2507 * us because we hold the mm_all_locks_mutex.
2508 *
2509 * We must however clear the bitflag before unlocking
2510 * the vma so the users using the anon_vma->head will
2511 * never see our bitflag.
2512 *
2513 * No need of atomic instructions here, head.next
2514 * can't change from under us until we release the
2515 * anon_vma->lock.
2516 */
2517 if (!__test_and_clear_bit(0, (unsigned long *)
2518 &anon_vma->head.next))
2519 BUG();
2520 spin_unlock(&anon_vma->lock);
2521 }
2522}
2523
2524static void vm_unlock_mapping(struct address_space *mapping)
2525{
2526 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2527 /*
2528 * AS_MM_ALL_LOCKS can't change to 0 from under us
2529 * because we hold the mm_all_locks_mutex.
2530 */
2531 spin_unlock(&mapping->i_mmap_lock);
2532 if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
2533 &mapping->flags))
2534 BUG();
2535 }
2536}
2537
2538/*
2539 * The mmap_sem cannot be released by the caller until
2540 * mm_drop_all_locks() returns.
2541 */
2542void mm_drop_all_locks(struct mm_struct *mm)
2543{
2544 struct vm_area_struct *vma;
2545
2546 BUG_ON(down_read_trylock(&mm->mmap_sem));
2547 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
2548
2549 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2550 if (vma->anon_vma)
2551 vm_unlock_anon_vma(vma->anon_vma);
2552 if (vma->vm_file && vma->vm_file->f_mapping)
2553 vm_unlock_mapping(vma->vm_file->f_mapping);
2554 }
2555
2556 mutex_unlock(&mm_all_locks_mutex);
2557}
8feae131
DH
2558
2559/*
2560 * initialise the VMA slab
2561 */
2562void __init mmap_init(void)
2563{
00a62ce9
KM
2564 int ret;
2565
2566 ret = percpu_counter_init(&vm_committed_as, 0);
2567 VM_BUG_ON(ret);
8feae131 2568}