X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=include%2Fasm-ia64%2Fpgalloc.h;h=d80b29d99e17198f32fa297079a28bb0cbdd0188;hb=97bf2856c6014879bd04983a3e9dfcdac1e7fe85;hp=fda0dba0b1871dae65d8ab0dbd8c143fbdee64f7;hpb=6a77f38946aaee1cd85eeec6cf4229b204c15071;p=linux-2.6.git diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h index fda0dba0b..d80b29d99 100644 --- a/include/asm-ia64/pgalloc.h +++ b/include/asm-ia64/pgalloc.h @@ -1,6 +1,10 @@ #ifndef _ASM_IA64_PGALLOC_H #define _ASM_IA64_PGALLOC_H +#define arch_add_exec_range(mm, limit) do { ; } while (0) +#define arch_flush_exec_range(mm) do { ; } while (0) +#define arch_remove_exec_range(mm, limit) do { ; } while (0) + /* * This file contains the functions and defines necessary to allocate * page tables. @@ -13,7 +17,6 @@ * Copyright (C) 2000, Goutham Rao */ -#include #include #include @@ -21,148 +24,148 @@ #include #include -#include -/* - * Very stupidly, we used to get new pgd's and pmd's, init their contents - * to point to the NULL versions of the next level page table, later on - * completely re-init them the same way, then free them up. This wasted - * a lot of work and caused unnecessary memory traffic. How broken... - * We fix this by caching them. - */ -#define pgd_quicklist (local_cpu_data->pgd_quick) -#define pmd_quicklist (local_cpu_data->pmd_quick) -#define pgtable_cache_size (local_cpu_data->pgtable_cache_sz) +DECLARE_PER_CPU(unsigned long *, __pgtable_quicklist); +#define pgtable_quicklist __ia64_per_cpu_var(__pgtable_quicklist) +DECLARE_PER_CPU(long, __pgtable_quicklist_size); +#define pgtable_quicklist_size __ia64_per_cpu_var(__pgtable_quicklist_size) -static inline pgd_t* -pgd_alloc_one_fast (struct mm_struct *mm) +static inline long pgtable_quicklist_total_size(void) +{ + long ql_size = 0; + int cpuid; + + for_each_online_cpu(cpuid) { + ql_size += per_cpu(__pgtable_quicklist_size, cpuid); + } + return ql_size; +} + +static inline void *pgtable_quicklist_alloc(void) { unsigned long *ret = NULL; preempt_disable(); - ret = pgd_quicklist; + ret = pgtable_quicklist; if (likely(ret != NULL)) { - pgd_quicklist = (unsigned long *)(*ret); + pgtable_quicklist = (unsigned long *)(*ret); ret[0] = 0; - --pgtable_cache_size; - } else - ret = NULL; - - preempt_enable(); + --pgtable_quicklist_size; + preempt_enable(); + } else { + preempt_enable(); + ret = (unsigned long *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + } - return (pgd_t *) ret; + return ret; } -static inline pgd_t* -pgd_alloc (struct mm_struct *mm) +static inline void pgtable_quicklist_free(void *pgtable_entry) { - /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */ - pgd_t *pgd = pgd_alloc_one_fast(mm); +#ifdef CONFIG_NUMA + int nid = page_to_nid(virt_to_page(pgtable_entry)); - if (unlikely(pgd == NULL)) { - pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); + if (unlikely(nid != numa_node_id())) { + free_page((unsigned long)pgtable_entry); + return; } - return pgd; -} +#endif -static inline void -pgd_free (pgd_t *pgd) -{ preempt_disable(); - *(unsigned long *)pgd = (unsigned long) pgd_quicklist; - pgd_quicklist = (unsigned long *) pgd; - ++pgtable_cache_size; + *(unsigned long *)pgtable_entry = (unsigned long)pgtable_quicklist; + pgtable_quicklist = (unsigned long *)pgtable_entry; + ++pgtable_quicklist_size; preempt_enable(); } -static inline void -pud_populate (struct mm_struct *mm, pud_t *pud_entry, pmd_t *pmd) +static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - pud_val(*pud_entry) = __pa(pmd); + return pgtable_quicklist_alloc(); } -static inline pmd_t* -pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr) +static inline void pgd_free(pgd_t * pgd) { - unsigned long *ret = NULL; - - preempt_disable(); + pgtable_quicklist_free(pgd); +} - ret = (unsigned long *)pmd_quicklist; - if (likely(ret != NULL)) { - pmd_quicklist = (unsigned long *)(*ret); - ret[0] = 0; - --pgtable_cache_size; - } +#ifdef CONFIG_PGTABLE_4 +static inline void +pgd_populate(struct mm_struct *mm, pgd_t * pgd_entry, pud_t * pud) +{ + pgd_val(*pgd_entry) = __pa(pud); +} - preempt_enable(); +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return pgtable_quicklist_alloc(); +} - return (pmd_t *)ret; +static inline void pud_free(pud_t * pud) +{ + pgtable_quicklist_free(pud); } +#define __pud_free_tlb(tlb, pud) pud_free(pud) +#endif /* CONFIG_PGTABLE_4 */ -static inline pmd_t* -pmd_alloc_one (struct mm_struct *mm, unsigned long addr) +static inline void +pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd) { - pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pud_val(*pud_entry) = __pa(pmd); +} - return pmd; +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return pgtable_quicklist_alloc(); } -static inline void -pmd_free (pmd_t *pmd) +static inline void pmd_free(pmd_t * pmd) { - preempt_disable(); - *(unsigned long *)pmd = (unsigned long) pmd_quicklist; - pmd_quicklist = (unsigned long *) pmd; - ++pgtable_cache_size; - preempt_enable(); + pgtable_quicklist_free(pmd); } #define __pmd_free_tlb(tlb, pmd) pmd_free(pmd) static inline void -pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte) +pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, struct page *pte) { +#ifndef CONFIG_XEN pmd_val(*pmd_entry) = page_to_phys(pte); +#else + pmd_val(*pmd_entry) = page_to_pseudophys(pte); +#endif } static inline void -pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte) +pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte) { pmd_val(*pmd_entry) = __pa(pte); } -static inline struct page * -pte_alloc_one (struct mm_struct *mm, unsigned long addr) +static inline struct page *pte_alloc_one(struct mm_struct *mm, + unsigned long addr) { - struct page *pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0); - - return pte; + return virt_to_page(pgtable_quicklist_alloc()); } -static inline pte_t * -pte_alloc_one_kernel (struct mm_struct *mm, unsigned long addr) +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, + unsigned long addr) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); - - return pte; + return pgtable_quicklist_alloc(); } -static inline void -pte_free (struct page *pte) +static inline void pte_free(struct page *pte) { - __free_page(pte); + pgtable_quicklist_free(page_address(pte)); } -static inline void -pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel(pte_t * pte) { - free_page((unsigned long) pte); + pgtable_quicklist_free(pte); } -#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte)) +#define __pte_free_tlb(tlb, pte) pte_free(pte) -extern void check_pgt_cache (void); +extern void check_pgt_cache(void); -#endif /* _ASM_IA64_PGALLOC_H */ +#endif /* _ASM_IA64_PGALLOC_H */