vserver 1.9.3

[linux-2.6.git] / mm / bootmem.c

diff --git a/mm/bootmem.c b/mm/bootmem.c
index 052dfea..092d851 100644 (file)
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -16,6 +16,7 @@
 #include <linux/init.h>
 #include <linux/bootmem.h>
 #include <linux/mmzone.h>
+#include <linux/module.h>
 #include <asm/dma.h>
 #include <asm/io.h>
 
@@ -27,6 +28,10 @@ unsigned long max_low_pfn;
 unsigned long min_low_pfn;
 unsigned long max_pfn;
 
+EXPORT_SYMBOL(max_pfn);                /* This is exported so
+                                * dma_get_required_mask(), which uses
+                                * it, can be an inline function */
+
 /* return the number of _pages_ that will be allocated for the boot bitmap */
 unsigned long __init bootmem_bootmap_pages (unsigned long pages)
 {
@@ -254,6 +259,7 @@ static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
        unsigned long i, count, total = 0;
        unsigned long idx;
        unsigned long *map; 
+       int gofast = 0;
 
        BUG_ON(!bdata->node_bootmem_map);
 
@@ -262,14 +268,32 @@ static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
        page = virt_to_page(phys_to_virt(bdata->node_boot_start));
        idx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
        map = bdata->node_bootmem_map;
+       /* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
+       if (bdata->node_boot_start == 0 ||
+           ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
+               gofast = 1;
        for (i = 0; i < idx; ) {
                unsigned long v = ~map[i / BITS_PER_LONG];
-               if (v) {
+               if (gofast && v == ~0UL) {
+                       int j;
+
+                       count += BITS_PER_LONG;
+                       __ClearPageReserved(page);
+                       set_page_count(page, 1);
+                       for (j = 1; j < BITS_PER_LONG; j++) {
+                               if (j + 16 < BITS_PER_LONG)
+                                       prefetchw(page + j + 16);
+                               __ClearPageReserved(page + j);
+                       }
+                       __free_pages(page, ffs(BITS_PER_LONG)-1);
+                       i += BITS_PER_LONG;
+                       page += BITS_PER_LONG;
+               } else if (v) {
                        unsigned long m;
                        for (m = 1; m && i < idx; m<<=1, page++, i++) {
                                if (v & m) {
                                        count++;
-                                       ClearPageReserved(page);
+                                       __ClearPageReserved(page);
                                        set_page_count(page, 1);
                                        __free_page(page);
                                }
@@ -289,7 +313,7 @@ static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
        count = 0;
        for (i = 0; i < ((bdata->node_low_pfn-(bdata->node_boot_start >> PAGE_SHIFT))/8 + PAGE_SIZE-1)/PAGE_SIZE; i++,page++) {
                count++;
-               ClearPageReserved(page);
+               __ClearPageReserved(page);
                set_page_count(page, 1);
                __free_page(page);
        }
@@ -371,11 +395,6 @@ void * __init __alloc_bootmem_node (pg_data_t *pgdat, unsigned long size, unsign
        if (ptr)
                return (ptr);
 
-       /*
-        * Whoops, we cannot satisfy the allocation request.
-        */
-       printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
-       panic("Out of memory");
-       return NULL;
+       return __alloc_bootmem(size, align, goal);
 }