mm/mremap.c

   1 /*
   2  *      mm/mremap.c
   3  *
   4  *      (C) Copyright 1996 Linus Torvalds
   5  *
   6  *      Address space accounting code   <alan@redhat.com>
   7  *      (C) Copyright 2002 Red Hat Inc, All Rights Reserved
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/hugetlb.h>
  12 #include <linux/slab.h>
  13 #include <linux/shm.h>
  14 #include <linux/mman.h>
  15 #include <linux/swap.h>
  16 #include <linux/fs.h>
  17 #include <linux/highmem.h>
  18 #include <linux/security.h>
  19 #include <linux/acct.h>
  20 #include <linux/syscalls.h>
  21 #include <linux/vs_memory.h>
  22
  23 #include <asm/uaccess.h>
  24 #include <asm/cacheflush.h>
  25 #include <asm/tlbflush.h>
  26
  27 static pte_t *get_one_pte_map_nested(struct mm_struct *mm, unsigned long addr)
  28 {
  29         pgd_t *pgd;
  30         pud_t *pud;
  31         pmd_t *pmd;
  32         pte_t *pte = NULL;
  33
  34         pgd = pgd_offset(mm, addr);
  35         if (pgd_none(*pgd))
  36                 goto end;
  37
  38         pud = pud_offset(pgd, addr);
  39         if (pud_none(*pud))
  40                 goto end;
  41         if (pud_bad(*pud)) {
  42                 pud_ERROR(*pud);
  43                 pud_clear(pud);
  44                 goto end;
  45         }
  46
  47         pmd = pmd_offset(pud, addr);
  48         if (pmd_none(*pmd))
  49                 goto end;
  50         if (pmd_bad(*pmd)) {
  51                 pmd_ERROR(*pmd);
  52                 pmd_clear(pmd);
  53                 goto end;
  54         }
  55
  56         pte = pte_offset_map_nested(pmd, addr);
  57         if (pte_none(*pte)) {
  58                 pte_unmap_nested(pte);
  59                 pte = NULL;
  60         }
  61 end:
  62         return pte;
  63 }
  64
  65 static pte_t *get_one_pte_map(struct mm_struct *mm, unsigned long addr)
  66 {
  67         pgd_t *pgd;
  68         pud_t *pud;
  69         pmd_t *pmd;
  70
  71         pgd = pgd_offset(mm, addr);
  72         if (pgd_none(*pgd))
  73                 return NULL;
  74
  75         pud = pud_offset(pgd, addr);
  76         if (pud_none(*pud))
  77                 return NULL;
  78         pmd = pmd_offset(pud, addr);
  79         if (!pmd_present(*pmd))
  80                 return NULL;
  81         return pte_offset_map(pmd, addr);
  82 }
  83
  84 static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
  85 {
  86         pgd_t *pgd;
  87         pud_t *pud;
  88         pmd_t *pmd;
  89         pte_t *pte = NULL;
  90
  91         pgd = pgd_offset(mm, addr);
  92
  93         pud = pud_alloc(mm, pgd, addr);
  94         if (!pud)
  95                 return NULL;
  96         pmd = pmd_alloc(mm, pud, addr);
  97         if (pmd)
  98                 pte = pte_alloc_map(mm, pmd, addr);
  99         return pte;
 100 }
 101
 102 static int
 103 move_one_page(struct vm_area_struct *vma, unsigned long old_addr,
 104                 struct vm_area_struct *new_vma, unsigned long new_addr)
 105 {
 106         struct address_space *mapping = NULL;
 107         struct mm_struct *mm = vma->vm_mm;
 108         int error = 0;
 109         pte_t *src, *dst;
 110
 111         if (vma->vm_file) {
 112                 /*
 113                  * Subtle point from Rajesh Venkatasubramanian: before
 114                  * moving file-based ptes, we must lock vmtruncate out,
 115                  * since it might clean the dst vma before the src vma,
 116                  * and we propagate stale pages into the dst afterward.
 117                  */
 118                 mapping = vma->vm_file->f_mapping;
 119                 spin_lock(&mapping->i_mmap_lock);
 120                 if (new_vma->vm_truncate_count &&
 121                     new_vma->vm_truncate_count != vma->vm_truncate_count)
 122                         new_vma->vm_truncate_count = 0;
 123         }
 124         spin_lock(&mm->page_table_lock);
 125
 126         src = get_one_pte_map_nested(mm, old_addr);
 127         if (src) {
 128                 /*
 129                  * Look to see whether alloc_one_pte_map needs to perform a
 130                  * memory allocation.  If it does then we need to drop the
 131                  * atomic kmap
 132                  */
 133                 dst = get_one_pte_map(mm, new_addr);
 134                 if (unlikely(!dst)) {
 135                         pte_unmap_nested(src);
 136                         if (mapping)
 137                                 spin_unlock(&mapping->i_mmap_lock);
 138                         dst = alloc_one_pte_map(mm, new_addr);
 139                         if (mapping && !spin_trylock(&mapping->i_mmap_lock)) {
 140                                 spin_unlock(&mm->page_table_lock);
 141                                 spin_lock(&mapping->i_mmap_lock);
 142                                 spin_lock(&mm->page_table_lock);
 143                         }
 144                         src = get_one_pte_map_nested(mm, old_addr);
 145                 }
 146                 /*
 147                  * Since alloc_one_pte_map can drop and re-acquire
 148                  * page_table_lock, we should re-check the src entry...
 149                  */
 150                 if (src) {
 151                         if (dst) {
 152                                 pte_t pte;
 153                                 pte = ptep_clear_flush(vma, old_addr, src);
 154                                 set_pte(dst, pte);
 155                         } else
 156                                 error = -ENOMEM;
 157                         pte_unmap_nested(src);
 158                 }
 159                 if (dst)
 160                         pte_unmap(dst);
 161         }
 162         spin_unlock(&mm->page_table_lock);
 163         if (mapping)
 164                 spin_unlock(&mapping->i_mmap_lock);
 165         return error;
 166 }
 167
 168 static unsigned long move_page_tables(struct vm_area_struct *vma,
 169                 unsigned long old_addr, struct vm_area_struct *new_vma,
 170                 unsigned long new_addr, unsigned long len)
 171 {
 172         unsigned long offset;
 173
 174         flush_cache_range(vma, old_addr, old_addr + len);
 175
 176         /*
 177          * This is not the clever way to do this, but we're taking the
 178          * easy way out on the assumption that most remappings will be
 179          * only a few pages.. This also makes error recovery easier.
 180          */
 181         for (offset = 0; offset < len; offset += PAGE_SIZE) {
 182                 if (move_one_page(vma, old_addr + offset,
 183                                 new_vma, new_addr + offset) < 0)
 184                         break;
 185                 cond_resched();
 186         }
 187         return offset;
 188 }
 189
 190 static unsigned long move_vma(struct vm_area_struct *vma,
 191                 unsigned long old_addr, unsigned long old_len,
 192                 unsigned long new_len, unsigned long new_addr)
 193 {
 194         struct mm_struct *mm = vma->vm_mm;
 195         struct vm_area_struct *new_vma;
 196         unsigned long vm_flags = vma->vm_flags;
 197         unsigned long new_pgoff;
 198         unsigned long moved_len;
 199         unsigned long excess = 0;
 200         int split = 0;
 201
 202         /*
 203          * We'd prefer to avoid failure later on in do_munmap:
 204          * which may split one vma into three before unmapping.
 205          */
 206         if (mm->map_count >= sysctl_max_map_count - 3)
 207                 return -ENOMEM;
 208
 209         new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
 210         new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
 211         if (!new_vma)
 212                 return -ENOMEM;
 213
 214         moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
 215         if (moved_len < old_len) {
 216                 /*
 217                  * On error, move entries back from new area to old,
 218                  * which will succeed since page tables still there,
 219                  * and then proceed to unmap new area instead of old.
 220                  */
 221                 move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
 222                 vma = new_vma;
 223                 old_len = new_len;
 224                 old_addr = new_addr;
 225                 new_addr = -ENOMEM;
 226         }
 227
 228         /* Conceal VM_ACCOUNT so old reservation is not undone */
 229         if (vm_flags & VM_ACCOUNT) {
 230                 vma->vm_flags &= ~VM_ACCOUNT;
 231                 excess = vma->vm_end - vma->vm_start - old_len;
 232                 if (old_addr > vma->vm_start &&
 233                     old_addr + old_len < vma->vm_end)
 234                         split = 1;
 235         }
 236
 237         if (do_munmap(mm, old_addr, old_len) < 0) {
 238                 /* OOM: unable to split vma, just get accounts right */
 239                 vm_unacct_memory(excess >> PAGE_SHIFT);
 240                 excess = 0;
 241         }
 242
 243         /* Restore VM_ACCOUNT if one or two pieces of vma left */
 244         if (excess) {
 245                 vma->vm_flags |= VM_ACCOUNT;
 246                 if (split)
 247                         vma->vm_next->vm_flags |= VM_ACCOUNT;
 248         }
 249
 250         vx_vmpages_add(mm, new_len >> PAGE_SHIFT);
 251         __vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
 252         if (vm_flags & VM_LOCKED) {
 253                 vx_vmlocked_add(mm, new_len >> PAGE_SHIFT);
 254                 if (new_len > old_len)
 255                         make_pages_present(new_addr + old_len,
 256                                            new_addr + new_len);
 257         }
 258
 259         acct_update_integrals();
 260         update_mem_hiwater();
 261
 262         return new_addr;
 263 }
 264
 265 /*
 266  * Expand (or shrink) an existing mapping, potentially moving it at the
 267  * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 268  *
 269  * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 270  * This option implies MREMAP_MAYMOVE.
 271  */
 272 unsigned long do_mremap(unsigned long addr,
 273         unsigned long old_len, unsigned long new_len,
 274         unsigned long flags, unsigned long new_addr)
 275 {
 276         struct vm_area_struct *vma;
 277         unsigned long ret = -EINVAL;
 278         unsigned long charged = 0;
 279
 280         if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
 281                 goto out;
 282
 283         if (addr & ~PAGE_MASK)
 284                 goto out;
 285
 286         old_len = PAGE_ALIGN(old_len);
 287         new_len = PAGE_ALIGN(new_len);
 288
 289         /*
 290          * We allow a zero old-len as a special case
 291          * for DOS-emu "duplicate shm area" thing. But
 292          * a zero new-len is nonsensical.
 293          */
 294         if (!new_len)
 295                 goto out;
 296
 297         /* new_addr is only valid if MREMAP_FIXED is specified */
 298         if (flags & MREMAP_FIXED) {
 299                 if (new_addr & ~PAGE_MASK)
 300                         goto out;
 301                 if (!(flags & MREMAP_MAYMOVE))
 302                         goto out;
 303
 304                 if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
 305                         goto out;
 306
 307                 /* Check if the location we're moving into overlaps the
 308                  * old location at all, and fail if it does.
 309                  */
 310                 if ((new_addr <= addr) && (new_addr+new_len) > addr)
 311                         goto out;
 312
 313                 if ((addr <= new_addr) && (addr+old_len) > new_addr)
 314                         goto out;
 315
 316                 ret = do_munmap(current->mm, new_addr, new_len);
 317                 if (ret)
 318                         goto out;
 319         }
 320
 321         /*
 322          * Always allow a shrinking remap: that just unmaps
 323          * the unnecessary pages..
 324          * do_munmap does all the needed commit accounting
 325          */
 326         if (old_len >= new_len) {
 327                 ret = do_munmap(current->mm, addr+new_len, old_len - new_len);
 328                 if (ret && old_len != new_len)
 329                         goto out;
 330                 ret = addr;
 331                 if (!(flags & MREMAP_FIXED) || (new_addr == addr))
 332                         goto out;
 333                 old_len = new_len;
 334         }
 335
 336         /*
 337          * Ok, we need to grow..  or relocate.
 338          */
 339         ret = -EFAULT;
 340         vma = find_vma(current->mm, addr);
 341         if (!vma || vma->vm_start > addr)
 342                 goto out;
 343         if (is_vm_hugetlb_page(vma)) {
 344                 ret = -EINVAL;
 345                 goto out;
 346         }
 347         /* We can't remap across vm area boundaries */
 348         if (old_len > vma->vm_end - addr)
 349                 goto out;
 350         if (vma->vm_flags & VM_DONTEXPAND) {
 351                 if (new_len > old_len)
 352                         goto out;
 353         }
 354         if (vma->vm_flags & VM_LOCKED) {
 355                 unsigned long locked, lock_limit;
 356                 locked = current->mm->locked_vm << PAGE_SHIFT;
 357                 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
 358                 locked += new_len - old_len;
 359                 ret = -EAGAIN;
 360                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
 361                         goto out;
 362                 ret = -ENOMEM;
 363                 if (!vx_vmlocked_avail(current->mm,
 364                         (new_len - old_len) >> PAGE_SHIFT))
 365                         goto out;
 366         }
 367         ret = -ENOMEM;
 368         if ((current->mm->total_vm << PAGE_SHIFT) + (new_len - old_len)
 369             > current->signal->rlim[RLIMIT_AS].rlim_cur)
 370                 goto out;
 371         /* check context space, maybe only Private writable mapping? */
 372         if (!vx_vmpages_avail(current->mm, (new_len - old_len) >> PAGE_SHIFT))
 373                 goto out;
 374
 375         if (vma->vm_flags & VM_ACCOUNT) {
 376                 charged = (new_len - old_len) >> PAGE_SHIFT;
 377                 if (security_vm_enough_memory(charged))
 378                         goto out_nc;
 379         }
 380
 381         /* old_len exactly to the end of the area..
 382          * And we're not relocating the area.
 383          */
 384         if (old_len == vma->vm_end - addr &&
 385             !((flags & MREMAP_FIXED) && (addr != new_addr)) &&
 386             (old_len != new_len || !(flags & MREMAP_MAYMOVE))) {
 387                 unsigned long max_addr = TASK_SIZE;
 388                 if (vma->vm_next)
 389                         max_addr = vma->vm_next->vm_start;
 390                 /* can we just expand the current mapping? */
 391                 if (max_addr - addr >= new_len) {
 392                         int pages = (new_len - old_len) >> PAGE_SHIFT;
 393
 394                         vma_adjust(vma, vma->vm_start,
 395                                 addr + new_len, vma->vm_pgoff, NULL);
 396
 397                         vx_vmpages_add(current->mm, pages);
 398                         __vm_stat_account(vma->vm_mm, vma->vm_flags,
 399                                                         vma->vm_file, pages);
 400                         if (vma->vm_flags & VM_LOCKED) {
 401                                 vx_vmlocked_add(vma->vm_mm, pages);
 402                                 make_pages_present(addr + old_len,
 403                                                    addr + new_len);
 404                         }
 405                         acct_update_integrals();
 406                         update_mem_hiwater();
 407                         ret = addr;
 408                         goto out;
 409                 }
 410         }
 411
 412         /*
 413          * We weren't able to just expand or shrink the area,
 414          * we need to create a new one and move it..
 415          */
 416         ret = -ENOMEM;
 417         if (flags & MREMAP_MAYMOVE) {
 418                 if (!(flags & MREMAP_FIXED)) {
 419                         unsigned long map_flags = 0;
 420                         if (vma->vm_flags & VM_MAYSHARE)
 421                                 map_flags |= MAP_SHARED;
 422
 423                         new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
 424                                                 vma->vm_pgoff, map_flags);
 425                         ret = new_addr;
 426                         if (new_addr & ~PAGE_MASK)
 427                                 goto out;
 428                 }
 429                 ret = move_vma(vma, addr, old_len, new_len, new_addr);
 430         }
 431 out:
 432         if (ret & ~PAGE_MASK)
 433                 vm_unacct_memory(charged);
 434 out_nc:
 435         return ret;
 436 }
 437
 438 asmlinkage unsigned long sys_mremap(unsigned long addr,
 439         unsigned long old_len, unsigned long new_len,
 440         unsigned long flags, unsigned long new_addr)
 441 {
 442         unsigned long ret;
 443
 444         down_write(&current->mm->mmap_sem);
 445         ret = do_mremap(addr, old_len, new_len, flags, new_addr);
 446         up_write(&current->mm->mmap_sem);
 447         return ret;
 448 }