mm/vmalloc.c

   1 /*
   2  *  linux/mm/vmalloc.c
   3  *
   4  *  Copyright (C) 1993  Linus Torvalds
   5  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
   6  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
   7  *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
   8  */
   9
  10 #include <linux/mm.h>
  11 #include <linux/module.h>
  12 #include <linux/highmem.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/interrupt.h>
  16
  17 #include <linux/vmalloc.h>
  18
  19 #include <asm/uaccess.h>
  20 #include <asm/pgalloc.h>
  21 #include <asm/tlbflush.h>
  22
  23
  24 rwlock_t vmlist_lock = RW_LOCK_UNLOCKED;
  25 struct vm_struct *vmlist;
  26
  27 static void unmap_area_pte(pmd_t *pmd, unsigned long address,
  28                                   unsigned long size)
  29 {
  30         unsigned long end;
  31         pte_t *pte;
  32
  33         if (pmd_none(*pmd))
  34                 return;
  35         if (pmd_bad(*pmd)) {
  36                 pmd_ERROR(*pmd);
  37                 pmd_clear(pmd);
  38                 return;
  39         }
  40
  41         pte = pte_offset_kernel(pmd, address);
  42         address &= ~PMD_MASK;
  43         end = address + size;
  44         if (end > PMD_SIZE)
  45                 end = PMD_SIZE;
  46
  47         do {
  48                 pte_t page;
  49                 page = ptep_get_and_clear(pte);
  50                 address += PAGE_SIZE;
  51                 pte++;
  52                 if (pte_none(page))
  53                         continue;
  54                 if (pte_present(page))
  55                         continue;
  56                 printk(KERN_CRIT "Whee.. Swapped out page in kernel page table\n");
  57         } while (address < end);
  58 }
  59
  60 static void unmap_area_pmd(pgd_t *dir, unsigned long address,
  61                                   unsigned long size)
  62 {
  63         unsigned long end;
  64         pmd_t *pmd;
  65
  66         if (pgd_none(*dir))
  67                 return;
  68         if (pgd_bad(*dir)) {
  69                 pgd_ERROR(*dir);
  70                 pgd_clear(dir);
  71                 return;
  72         }
  73
  74         pmd = pmd_offset(dir, address);
  75         address &= ~PGDIR_MASK;
  76         end = address + size;
  77         if (end > PGDIR_SIZE)
  78                 end = PGDIR_SIZE;
  79
  80         do {
  81                 unmap_area_pte(pmd, address, end - address);
  82                 address = (address + PMD_SIZE) & PMD_MASK;
  83                 pmd++;
  84         } while (address < end);
  85 }
  86
  87 static int map_area_pte(pte_t *pte, unsigned long address,
  88                                unsigned long size, pgprot_t prot,
  89                                struct page ***pages)
  90 {
  91         unsigned long end;
  92
  93         address &= ~PMD_MASK;
  94         end = address + size;
  95         if (end > PMD_SIZE)
  96                 end = PMD_SIZE;
  97
  98         do {
  99                 struct page *page = **pages;
 100
 101                 WARN_ON(!pte_none(*pte));
 102                 if (!page)
 103                         return -ENOMEM;
 104
 105                 set_pte(pte, mk_pte(page, prot));
 106                 address += PAGE_SIZE;
 107                 pte++;
 108                 (*pages)++;
 109         } while (address < end);
 110         return 0;
 111 }
 112
 113 static int map_area_pmd(pmd_t *pmd, unsigned long address,
 114                                unsigned long size, pgprot_t prot,
 115                                struct page ***pages)
 116 {
 117         unsigned long base, end;
 118
 119         base = address & PGDIR_MASK;
 120         address &= ~PGDIR_MASK;
 121         end = address + size;
 122         if (end > PGDIR_SIZE)
 123                 end = PGDIR_SIZE;
 124
 125         do {
 126                 pte_t * pte = pte_alloc_kernel(&init_mm, pmd, base + address);
 127                 if (!pte)
 128                         return -ENOMEM;
 129                 if (map_area_pte(pte, address, end - address, prot, pages))
 130                         return -ENOMEM;
 131                 address = (address + PMD_SIZE) & PMD_MASK;
 132                 pmd++;
 133         } while (address < end);
 134
 135         return 0;
 136 }
 137
 138 void unmap_vm_area(struct vm_struct *area)
 139 {
 140         unsigned long address = (unsigned long) area->addr;
 141         unsigned long end = (address + area->size);
 142         pgd_t *dir;
 143
 144         dir = pgd_offset_k(address);
 145         flush_cache_vunmap(address, end);
 146         do {
 147                 unmap_area_pmd(dir, address, end - address);
 148                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
 149                 dir++;
 150         } while (address && (address < end));
 151         flush_tlb_kernel_range((unsigned long) area->addr, end);
 152 }
 153
 154 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
 155 {
 156         unsigned long address = (unsigned long) area->addr;
 157         unsigned long end = address + (area->size-PAGE_SIZE);
 158         pgd_t *dir;
 159         int err = 0;
 160
 161         dir = pgd_offset_k(address);
 162         spin_lock(&init_mm.page_table_lock);
 163         do {
 164                 pmd_t *pmd = pmd_alloc(&init_mm, dir, address);
 165                 if (!pmd) {
 166                         err = -ENOMEM;
 167                         break;
 168                 }
 169                 if (map_area_pmd(pmd, address, end - address, prot, pages)) {
 170                         err = -ENOMEM;
 171                         break;
 172                 }
 173
 174                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
 175                 dir++;
 176         } while (address && (address < end));
 177
 178         spin_unlock(&init_mm.page_table_lock);
 179         flush_cache_vmap((unsigned long) area->addr, end);
 180         return err;
 181 }
 182
 183 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
 184                                 unsigned long start, unsigned long end)
 185 {
 186         struct vm_struct **p, *tmp, *area;
 187         unsigned long addr = start;
 188
 189         area = kmalloc(sizeof(*area), GFP_KERNEL);
 190         if (unlikely(!area))
 191                 return NULL;
 192
 193         /*
 194          * We always allocate a guard page.
 195          */
 196         size += PAGE_SIZE;
 197         if (unlikely(!size)) {
 198                 kfree (area);
 199                 return NULL;
 200         }
 201
 202         write_lock(&vmlist_lock);
 203         for (p = &vmlist; (tmp = *p) ;p = &tmp->next) {
 204                 if ((unsigned long)tmp->addr < addr)
 205                         continue;
 206                 if ((size + addr) < addr)
 207                         goto out;
 208                 if (size + addr <= (unsigned long)tmp->addr)
 209                         goto found;
 210                 addr = tmp->size + (unsigned long)tmp->addr;
 211                 if (addr > end - size)
 212                         goto out;
 213         }
 214
 215 found:
 216         area->next = *p;
 217         *p = area;
 218
 219         area->flags = flags;
 220         area->addr = (void *)addr;
 221         area->size = size;
 222         area->pages = NULL;
 223         area->nr_pages = 0;
 224         area->phys_addr = 0;
 225         write_unlock(&vmlist_lock);
 226
 227         return area;
 228
 229 out:
 230         write_unlock(&vmlist_lock);
 231         kfree(area);
 232         return NULL;
 233 }
 234
 235 /**
 236  *      get_vm_area  -  reserve a contingous kernel virtual area
 237  *
 238  *      @size:          size of the area
 239  *      @flags:         %VM_IOREMAP for I/O mappings or VM_ALLOC
 240  *
 241  *      Search an area of @size in the kernel virtual mapping area,
 242  *      and reserved it for out purposes.  Returns the area descriptor
 243  *      on success or %NULL on failure.
 244  */
 245 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
 246 {
 247         return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
 248 }
 249
 250 /**
 251  *      remove_vm_area  -  find and remove a contingous kernel virtual area
 252  *
 253  *      @addr:          base address
 254  *
 255  *      Search for the kernel VM area starting at @addr, and remove it.
 256  *      This function returns the found VM area, but using it is NOT safe
 257  *      on SMP machines.
 258  */
 259 struct vm_struct *remove_vm_area(void *addr)
 260 {
 261         struct vm_struct **p, *tmp;
 262
 263         write_lock(&vmlist_lock);
 264         for (p = &vmlist ; (tmp = *p) ;p = &tmp->next) {
 265                  if (tmp->addr == addr)
 266                          goto found;
 267         }
 268         write_unlock(&vmlist_lock);
 269         return NULL;
 270
 271 found:
 272         unmap_vm_area(tmp);
 273         *p = tmp->next;
 274         write_unlock(&vmlist_lock);
 275         return tmp;
 276 }
 277
 278 void __vunmap(void *addr, int deallocate_pages)
 279 {
 280         struct vm_struct *area;
 281
 282         if (!addr)
 283                 return;
 284
 285         if ((PAGE_SIZE-1) & (unsigned long)addr) {
 286                 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
 287                 return;
 288         }
 289
 290         area = remove_vm_area(addr);
 291         if (unlikely(!area)) {
 292                 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
 293                                 addr);
 294                 return;
 295         }
 296
 297         if (deallocate_pages) {
 298                 int i;
 299
 300                 for (i = 0; i < area->nr_pages; i++) {
 301                         if (unlikely(!area->pages[i]))
 302                                 BUG();
 303                         __free_page(area->pages[i]);
 304                 }
 305
 306                 kfree(area->pages);
 307         }
 308
 309         kfree(area);
 310         return;
 311 }
 312
 313 /**
 314  *      vfree  -  release memory allocated by vmalloc()
 315  *
 316  *      @addr:          memory base address
 317  *
 318  *      Free the virtually contiguous memory area starting at @addr, as
 319  *      obtained from vmalloc(), vmalloc_32() or __vmalloc().
 320  *
 321  *      May not be called in interrupt context.
 322  */
 323 void vfree(void *addr)
 324 {
 325         BUG_ON(in_interrupt());
 326         __vunmap(addr, 1);
 327 }
 328
 329 EXPORT_SYMBOL(vfree);
 330
 331 /**
 332  *      vunmap  -  release virtual mapping obtained by vmap()
 333  *
 334  *      @addr:          memory base address
 335  *
 336  *      Free the virtually contiguous memory area starting at @addr,
 337  *      which was created from the page array passed to vmap().
 338  *
 339  *      May not be called in interrupt context.
 340  */
 341 void vunmap(void *addr)
 342 {
 343         BUG_ON(in_interrupt());
 344         __vunmap(addr, 0);
 345 }
 346
 347 EXPORT_SYMBOL(vunmap);
 348
 349 /**
 350  *      vmap  -  map an array of pages into virtually contiguous space
 351  *
 352  *      @pages:         array of page pointers
 353  *      @count:         number of pages to map
 354  *      @flags:         vm_area->flags
 355  *      @prot:          page protection for the mapping
 356  *
 357  *      Maps @count pages from @pages into contiguous kernel virtual
 358  *      space.
 359  */
 360 void *vmap(struct page **pages, unsigned int count,
 361                 unsigned long flags, pgprot_t prot)
 362 {
 363         struct vm_struct *area;
 364
 365         if (count > num_physpages)
 366                 return NULL;
 367
 368         area = get_vm_area((count << PAGE_SHIFT), flags);
 369         if (!area)
 370                 return NULL;
 371         if (map_vm_area(area, prot, &pages)) {
 372                 vunmap(area->addr);
 373                 return NULL;
 374         }
 375
 376         return area->addr;
 377 }
 378
 379 EXPORT_SYMBOL(vmap);
 380
 381 /**
 382  *      __vmalloc  -  allocate virtually contiguous memory
 383  *
 384  *      @size:          allocation size
 385  *      @gfp_mask:      flags for the page level allocator
 386  *      @prot:          protection mask for the allocated pages
 387  *
 388  *      Allocate enough pages to cover @size from the page level
 389  *      allocator with @gfp_mask flags.  Map them into contiguous
 390  *      kernel virtual space, using a pagetable protection of @prot.
 391  */
 392 void *__vmalloc(unsigned long size, int gfp_mask, pgprot_t prot)
 393 {
 394         struct vm_struct *area;
 395         struct page **pages;
 396         unsigned int nr_pages, array_size, i;
 397
 398         size = PAGE_ALIGN(size);
 399         if (!size || (size >> PAGE_SHIFT) > num_physpages)
 400                 return NULL;
 401
 402         area = get_vm_area(size, VM_ALLOC);
 403         if (!area)
 404                 return NULL;
 405
 406         nr_pages = size >> PAGE_SHIFT;
 407         array_size = (nr_pages * sizeof(struct page *));
 408
 409         area->nr_pages = nr_pages;
 410         area->pages = pages = kmalloc(array_size, (gfp_mask & ~__GFP_HIGHMEM));
 411         if (!area->pages) {
 412                 remove_vm_area(area->addr);
 413                 kfree(area);
 414                 return NULL;
 415         }
 416         memset(area->pages, 0, array_size);
 417
 418         for (i = 0; i < area->nr_pages; i++) {
 419                 area->pages[i] = alloc_page(gfp_mask);
 420                 if (unlikely(!area->pages[i])) {
 421                         /* Successfully allocated i pages, free them in __vunmap() */
 422                         area->nr_pages = i;
 423                         goto fail;
 424                 }
 425         }
 426
 427         if (map_vm_area(area, prot, &pages))
 428                 goto fail;
 429         return area->addr;
 430
 431 fail:
 432         vfree(area->addr);
 433         return NULL;
 434 }
 435
 436 EXPORT_SYMBOL(__vmalloc);
 437
 438 /**
 439  *      vmalloc  -  allocate virtually contiguous memory
 440  *
 441  *      @size:          allocation size
 442  *
 443  *      Allocate enough pages to cover @size from the page level
 444  *      allocator and map them into contiguous kernel virtual space.
 445  *
 446  *      For tight cotrol over page level allocator and protection flags
 447  *      use __vmalloc() instead.
 448  */
 449 void *vmalloc(unsigned long size)
 450 {
 451        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
 452 }
 453
 454 EXPORT_SYMBOL(vmalloc);
 455
 456 /**
 457  *      vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
 458  *
 459  *      @size:          allocation size
 460  *
 461  *      Allocate enough 32bit PA addressable pages to cover @size from the
 462  *      page level allocator and map them into contiguous kernel virtual space.
 463  */
 464 void *vmalloc_32(unsigned long size)
 465 {
 466         return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
 467 }
 468
 469 EXPORT_SYMBOL(vmalloc_32);
 470
 471 long vread(char *buf, char *addr, unsigned long count)
 472 {
 473         struct vm_struct *tmp;
 474         char *vaddr, *buf_start = buf;
 475         unsigned long n;
 476
 477         /* Don't allow overflow */
 478         if ((unsigned long) addr + count < count)
 479                 count = -(unsigned long) addr;
 480
 481         read_lock(&vmlist_lock);
 482         for (tmp = vmlist; tmp; tmp = tmp->next) {
 483                 vaddr = (char *) tmp->addr;
 484                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
 485                         continue;
 486                 while (addr < vaddr) {
 487                         if (count == 0)
 488                                 goto finished;
 489                         *buf = '\0';
 490                         buf++;
 491                         addr++;
 492                         count--;
 493                 }
 494                 n = vaddr + tmp->size - PAGE_SIZE - addr;
 495                 do {
 496                         if (count == 0)
 497                                 goto finished;
 498                         *buf = *addr;
 499                         buf++;
 500                         addr++;
 501                         count--;
 502                 } while (--n > 0);
 503         }
 504 finished:
 505         read_unlock(&vmlist_lock);
 506         return buf - buf_start;
 507 }
 508
 509 long vwrite(char *buf, char *addr, unsigned long count)
 510 {
 511         struct vm_struct *tmp;
 512         char *vaddr, *buf_start = buf;
 513         unsigned long n;
 514
 515         /* Don't allow overflow */
 516         if ((unsigned long) addr + count < count)
 517                 count = -(unsigned long) addr;
 518
 519         read_lock(&vmlist_lock);
 520         for (tmp = vmlist; tmp; tmp = tmp->next) {
 521                 vaddr = (char *) tmp->addr;
 522                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
 523                         continue;
 524                 while (addr < vaddr) {
 525                         if (count == 0)
 526                                 goto finished;
 527                         buf++;
 528                         addr++;
 529                         count--;
 530                 }
 531                 n = vaddr + tmp->size - PAGE_SIZE - addr;
 532                 do {
 533                         if (count == 0)
 534                                 goto finished;
 535                         *addr = *buf;
 536                         buf++;
 537                         addr++;
 538                         count--;
 539                 } while (--n > 0);
 540         }
 541 finished:
 542         read_unlock(&vmlist_lock);
 543         return buf - buf_start;
 544 }