X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=mm%2Fpage_alloc.c;h=2fd62e233e188ad02181abad3e039122f31bdc6a;hb=987b0145d94eecf292d8b301228356f44611ab7c;hp=5581dd1a489b501a6168e7a934e19ad081df0b23;hpb=f7ed79d23a47594e7834d66a8f14449796d4f3e6;p=linux-2.6.git diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 5581dd1a4..2fd62e233 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -40,7 +40,6 @@ #include #include -#include #include "internal.h" /* @@ -51,12 +50,13 @@ nodemask_t node_online_map __read_mostly = { { [0] = 1UL } }; EXPORT_SYMBOL(node_online_map); nodemask_t node_possible_map __read_mostly = NODE_MASK_ALL; EXPORT_SYMBOL(node_possible_map); +struct pglist_data *pgdat_list __read_mostly; unsigned long totalram_pages __read_mostly; unsigned long totalhigh_pages __read_mostly; -unsigned long totalreserve_pages __read_mostly; long nr_swap_pages; int percpu_pagelist_fraction; +static void fastcall free_hot_cold_page(struct page *page, int cold); static void __free_pages_ok(struct page *page, unsigned int order); /* @@ -139,12 +139,12 @@ static inline int bad_range(struct zone *zone, struct page *page) static void bad_page(struct page *page) { printk(KERN_EMERG "Bad page state in process '%s'\n" - KERN_EMERG "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d (%s)\n" + KERN_EMERG "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d\n" KERN_EMERG "Trying to fix it up, but a reboot is needed\n" KERN_EMERG "Backtrace:\n", current->comm, page, (int)(2*sizeof(unsigned long)), (unsigned long)page->flags, page->mapping, - page_mapcount(page), page_count(page), print_tainted()); + page_mapcount(page), page_count(page)); dump_stack(); page->flags &= ~(1 << PG_lru | 1 << PG_private | @@ -192,7 +192,7 @@ static void prep_compound_page(struct page *page, unsigned long order) for (i = 0; i < nr_pages; i++) { struct page *p = page + i; - __SetPageCompound(p); + SetPageCompound(p); set_page_private(p, (unsigned long)page); } } @@ -211,36 +211,20 @@ static void destroy_compound_page(struct page *page, unsigned long order) if (unlikely(!PageCompound(p) | (page_private(p) != (unsigned long)page))) bad_page(page); - __ClearPageCompound(p); + ClearPageCompound(p); } } -static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags) -{ - int i; - - BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM); - /* - * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO - * and __GFP_HIGHMEM from hard or soft interrupt context. - */ - BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt()); - for (i = 0; i < (1 << order); i++) - clear_highpage(page + i); -} - /* * function for dealing with page's order in buddy system. * zone->lock is already acquired when we use these. * So, we don't need atomic page->flags operations here. */ -static inline unsigned long page_order(struct page *page) -{ +static inline unsigned long page_order(struct page *page) { return page_private(page); } -static inline void set_page_order(struct page *page, int order) -{ +static inline void set_page_order(struct page *page, int order) { set_page_private(page, order); __SetPageBuddy(page); } @@ -303,9 +287,9 @@ static inline int page_is_buddy(struct page *page, int order) if (PageBuddy(page) && page_order(page) == order) { BUG_ON(page_count(page) != 0); - return 1; + return 1; } - return 0; + return 0; } /* @@ -439,12 +423,16 @@ static void __free_pages_ok(struct page *page, unsigned int order) int i; int reserved = 0; - if (arch_free_page(page, order)) - return; + arch_free_page(page, order); if (!PageHighMem(page)) mutex_debug_check_no_locks_freed(page_address(page), PAGE_SIZE<lru, &list); + kernel_map_pages(page, 1 << order, 0); + free_pages_bulk(page_zone(page), 1, &list, order); } } @@ -519,7 +512,7 @@ static inline void expand(struct zone *zone, struct page *page, /* * This page is about to be returned from the page allocator */ -static int prep_new_page(struct page *page, int order, gfp_t gfp_flags) +static int prep_new_page(struct page *page, int order) { if (unlikely(page_mapcount(page) | (page->mapping != NULL) | @@ -549,15 +542,8 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags) 1 << PG_referenced | 1 << PG_arch_1 | 1 << PG_checked | 1 << PG_mappedtodisk); set_page_private(page, 0); - set_page_refcounted(page); + set_page_refs(page, order); kernel_map_pages(page, 1 << order, 1); - - if (gfp_flags & __GFP_ZERO) - prep_zero_page(page, order, gfp_flags); - - if (order && (gfp_flags & __GFP_COMP)) - prep_compound_page(page, order); - return 0; } @@ -613,14 +599,13 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, /* * Called from the slab reaper to drain pagesets on a particular node that * belong to the currently executing processor. - * Note that this function must be called with the thread pinned to - * a single processor. */ void drain_node_pages(int nodeid) { int i, z; unsigned long flags; + local_irq_save(flags); for (z = 0; z < MAX_NR_ZONES; z++) { struct zone *zone = NODE_DATA(nodeid)->node_zones + z; struct per_cpu_pageset *pset; @@ -630,14 +615,11 @@ void drain_node_pages(int nodeid) struct per_cpu_pages *pcp; pcp = &pset->pcp[i]; - if (pcp->count) { - local_irq_save(flags); - free_pages_bulk(zone, pcp->count, &pcp->list, 0); - pcp->count = 0; - local_irq_restore(flags); - } + free_pages_bulk(zone, pcp->count, &pcp->list, 0); + pcp->count = 0; } } + local_irq_restore(flags); } #endif @@ -735,8 +717,7 @@ static void fastcall free_hot_cold_page(struct page *page, int cold) struct per_cpu_pages *pcp; unsigned long flags; - if (arch_free_page(page, 0)) - return; + arch_free_page(page, 0); if (PageAnon(page)) page->mapping = NULL; @@ -768,22 +749,13 @@ void fastcall free_cold_page(struct page *page) free_hot_cold_page(page, 1); } -/* - * split_page takes a non-compound higher-order page, and splits it into - * n (1<node_zones[ZONE_HIGHMEM].free_pages; return pages; @@ -1244,22 +1221,24 @@ DEFINE_PER_CPU(long, nr_pagecache_local) = 0; static void __get_page_state(struct page_state *ret, int nr, cpumask_t *cpumask) { - unsigned cpu; + int cpu = 0; memset(ret, 0, nr * sizeof(unsigned long)); cpus_and(*cpumask, *cpumask, cpu_online_map); - for_each_cpu_mask(cpu, *cpumask) { - unsigned long *in; - unsigned long *out; - unsigned off; - unsigned next_cpu; + cpu = first_cpu(*cpumask); + while (cpu < NR_CPUS) { + unsigned long *in, *out, off; + + if (!cpu_isset(cpu, *cpumask)) + continue; in = (unsigned long *)&per_cpu(page_states, cpu); - next_cpu = next_cpu(cpu, *cpumask); - if (likely(next_cpu < NR_CPUS)) - prefetch(&per_cpu(page_states, next_cpu)); + cpu = next_cpu(cpu, *cpumask); + + if (likely(cpu < NR_CPUS)) + prefetch(&per_cpu(page_states, cpu)); out = (unsigned long *)ret; for (off = 0; off < nr; off++) @@ -1355,7 +1334,7 @@ void get_zone_counts(unsigned long *active, *active = 0; *inactive = 0; *free = 0; - for_each_online_pgdat(pgdat) { + for_each_pgdat(pgdat) { unsigned long l, m, n; __get_zone_counts(&l, &m, &n, pgdat); *active += l; @@ -1796,7 +1775,7 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, continue; page = pfn_to_page(pfn); set_page_links(page, zone, nid, pfn); - init_page_count(page); + set_page_count(page, 1); reset_page_mapcount(page); SetPageReserved(page); INIT_LIST_HEAD(&page->lru); @@ -1971,7 +1950,7 @@ static inline void free_zone_pagesets(int cpu) } } -static int pageset_cpuup_callback(struct notifier_block *nfb, +static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -2045,9 +2024,8 @@ static __meminit void zone_pcp_init(struct zone *zone) setup_pageset(zone_pcp(zone,cpu), batch); #endif } - if (zone->present_pages) - printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", - zone->name, zone->present_pages, batch); + printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", + zone->name, zone->present_pages, batch); } static __meminit void init_currently_empty_zone(struct zone *zone, @@ -2058,6 +2036,7 @@ static __meminit void init_currently_empty_zone(struct zone *zone, zone_wait_table_init(zone, size); pgdat->nr_zones = zone_idx(zone) + 1; + zone->zone_mem_map = pfn_to_page(zone_start_pfn); zone->zone_start_pfn = zone_start_pfn; memmap_init(size, pgdat->node_id, zone_idx(zone), zone_start_pfn); @@ -2132,22 +2111,14 @@ static void __init alloc_node_mem_map(struct pglist_data *pgdat) #ifdef CONFIG_FLAT_NODE_MEM_MAP /* ia64 gets its own node_mem_map, before this, without bootmem */ if (!pgdat->node_mem_map) { - unsigned long size, start, end; + unsigned long size; struct page *map; - /* - * The zone's endpoints aren't required to be MAX_ORDER - * aligned but the node_mem_map endpoints must be in order - * for the buddy allocator to function correctly. - */ - start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); - end = pgdat->node_start_pfn + pgdat->node_spanned_pages; - end = ALIGN(end, MAX_ORDER_NR_PAGES); - size = (end - start) * sizeof(struct page); + size = (pgdat->node_spanned_pages + 1) * sizeof(struct page); map = alloc_remap(pgdat->node_id, size); if (!map) map = alloc_bootmem_node(pgdat, size); - pgdat->node_mem_map = map + (pgdat->node_start_pfn - start); + pgdat->node_mem_map = map; } #ifdef CONFIG_FLATMEM /* @@ -2193,9 +2164,8 @@ static void *frag_start(struct seq_file *m, loff_t *pos) { pg_data_t *pgdat; loff_t node = *pos; - for (pgdat = first_online_pgdat(); - pgdat && node; - pgdat = next_online_pgdat(pgdat)) + + for (pgdat = pgdat_list; pgdat && node; pgdat = pgdat->pgdat_next) --node; return pgdat; @@ -2206,7 +2176,7 @@ static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) pg_data_t *pgdat = (pg_data_t *)arg; (*pos)++; - return next_online_pgdat(pgdat); + return pgdat->pgdat_next; } static void frag_stop(struct seq_file *m, void *arg) @@ -2496,38 +2466,6 @@ void __init page_alloc_init(void) hotcpu_notifier(page_alloc_cpu_notify, 0); } -/* - * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio - * or min_free_kbytes changes. - */ -static void calculate_totalreserve_pages(void) -{ - struct pglist_data *pgdat; - unsigned long reserve_pages = 0; - int i, j; - - for_each_online_pgdat(pgdat) { - for (i = 0; i < MAX_NR_ZONES; i++) { - struct zone *zone = pgdat->node_zones + i; - unsigned long max = 0; - - /* Find valid and maximum lowmem_reserve in the zone */ - for (j = i; j < MAX_NR_ZONES; j++) { - if (zone->lowmem_reserve[j] > max) - max = zone->lowmem_reserve[j]; - } - - /* we treat pages_high as reserved pages. */ - max += zone->pages_high; - - if (max > zone->present_pages) - max = zone->present_pages; - reserve_pages += max; - } - } - totalreserve_pages = reserve_pages; -} - /* * setup_per_zone_lowmem_reserve - called whenever * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone @@ -2539,7 +2477,7 @@ static void setup_per_zone_lowmem_reserve(void) struct pglist_data *pgdat; int j, idx; - for_each_online_pgdat(pgdat) { + for_each_pgdat(pgdat) { for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = pgdat->node_zones + j; unsigned long present_pages = zone->present_pages; @@ -2559,9 +2497,6 @@ static void setup_per_zone_lowmem_reserve(void) } } } - - /* update totalreserve_pages */ - calculate_totalreserve_pages(); } /* @@ -2583,11 +2518,9 @@ void setup_per_zone_pages_min(void) } for_each_zone(zone) { - u64 tmp; - + unsigned long tmp; spin_lock_irqsave(&zone->lru_lock, flags); - tmp = (u64)pages_min * zone->present_pages; - do_div(tmp, lowmem_pages); + tmp = (pages_min * zone->present_pages) / lowmem_pages; if (is_highmem(zone)) { /* * __GFP_HIGH and PF_MEMALLOC allocations usually don't @@ -2614,13 +2547,10 @@ void setup_per_zone_pages_min(void) zone->pages_min = tmp; } - zone->pages_low = zone->pages_min + (tmp >> 2); - zone->pages_high = zone->pages_min + (tmp >> 1); + zone->pages_low = zone->pages_min + tmp / 4; + zone->pages_high = zone->pages_min + tmp / 2; spin_unlock_irqrestore(&zone->lru_lock, flags); } - - /* update totalreserve_pages */ - calculate_totalreserve_pages(); } /* @@ -2766,7 +2696,8 @@ void *__init alloc_large_system_hash(const char *tablename, else numentries <<= (PAGE_SHIFT - scale); } - numentries = roundup_pow_of_two(numentries); + /* rounded up to nearest power of 2 in size */ + numentries = 1UL << (long_log2(numentries) + 1); /* limit allocation size to 1/16 total memory by default */ if (max == 0) { @@ -2809,44 +2740,3 @@ void *__init alloc_large_system_hash(const char *tablename, return table; } - -#ifdef CONFIG_OUT_OF_LINE_PFN_TO_PAGE -/* - * pfn <-> page translation. out-of-line version. - * (see asm-generic/memory_model.h) - */ -#if defined(CONFIG_FLATMEM) -struct page *pfn_to_page(unsigned long pfn) -{ - return mem_map + (pfn - ARCH_PFN_OFFSET); -} -unsigned long page_to_pfn(struct page *page) -{ - return (page - mem_map) + ARCH_PFN_OFFSET; -} -#elif defined(CONFIG_DISCONTIGMEM) -struct page *pfn_to_page(unsigned long pfn) -{ - int nid = arch_pfn_to_nid(pfn); - return NODE_DATA(nid)->node_mem_map + arch_local_page_offset(pfn,nid); -} -unsigned long page_to_pfn(struct page *page) -{ - struct pglist_data *pgdat = NODE_DATA(page_to_nid(page)); - return (page - pgdat->node_mem_map) + pgdat->node_start_pfn; -} -#elif defined(CONFIG_SPARSEMEM) -struct page *pfn_to_page(unsigned long pfn) -{ - return __section_mem_map_addr(__pfn_to_section(pfn)) + pfn; -} - -unsigned long page_to_pfn(struct page *page) -{ - long section_id = page_to_section(page); - return page - __section_mem_map_addr(__nr_to_section(section_id)); -} -#endif /* CONFIG_FLATMEM/DISCONTIGMME/SPARSEMEM */ -EXPORT_SYMBOL(pfn_to_page); -EXPORT_SYMBOL(page_to_pfn); -#endif /* CONFIG_OUT_OF_LINE_PFN_TO_PAGE */