#include <linux/threads.h>
#include <asm/mmu_context.h>
-#include <asm/processor.h>
-/*
- * Very stupidly, we used to get new pgd's and pmd's, init their contents
- * to point to the NULL versions of the next level page table, later on
- * completely re-init them the same way, then free them up. This wasted
- * a lot of work and caused unnecessary memory traffic. How broken...
- * We fix this by caching them.
- */
-#define pgd_quicklist (local_cpu_data->pgd_quick)
-#define pmd_quicklist (local_cpu_data->pmd_quick)
-#define pgtable_cache_size (local_cpu_data->pgtable_cache_sz)
+DECLARE_PER_CPU(unsigned long *, __pgtable_quicklist);
+#define pgtable_quicklist __ia64_per_cpu_var(__pgtable_quicklist)
+DECLARE_PER_CPU(long, __pgtable_quicklist_size);
+#define pgtable_quicklist_size __ia64_per_cpu_var(__pgtable_quicklist_size)
-static inline pgd_t*
-pgd_alloc_one_fast (struct mm_struct *mm)
+static inline long pgtable_quicklist_total_size(void)
+{
+ long ql_size = 0;
+ int cpuid;
+
+ for_each_online_cpu(cpuid) {
+ ql_size += per_cpu(__pgtable_quicklist_size, cpuid);
+ }
+ return ql_size;
+}
+
+static inline void *pgtable_quicklist_alloc(void)
{
unsigned long *ret = NULL;
preempt_disable();
- ret = pgd_quicklist;
+ ret = pgtable_quicklist;
if (likely(ret != NULL)) {
- pgd_quicklist = (unsigned long *)(*ret);
+ pgtable_quicklist = (unsigned long *)(*ret);
ret[0] = 0;
- --pgtable_cache_size;
- } else
- ret = NULL;
-
- preempt_enable();
+ --pgtable_quicklist_size;
+ preempt_enable();
+ } else {
+ preempt_enable();
+ ret = (unsigned long *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+ }
- return (pgd_t *) ret;
+ return ret;
}
-static inline pgd_t*
-pgd_alloc (struct mm_struct *mm)
+static inline void pgtable_quicklist_free(void *pgtable_entry)
{
- /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */
- pgd_t *pgd = pgd_alloc_one_fast(mm);
+#ifdef CONFIG_NUMA
+ unsigned long nid = page_to_nid(virt_to_page(pgtable_entry));
- if (unlikely(pgd == NULL)) {
- pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
+ if (unlikely(nid != numa_node_id())) {
+ free_page((unsigned long)pgtable_entry);
+ return;
}
- return pgd;
-}
+#endif
-static inline void
-pgd_free (pgd_t *pgd)
-{
preempt_disable();
- *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
- pgd_quicklist = (unsigned long *) pgd;
- ++pgtable_cache_size;
+ *(unsigned long *)pgtable_entry = (unsigned long)pgtable_quicklist;
+ pgtable_quicklist = (unsigned long *)pgtable_entry;
+ ++pgtable_quicklist_size;
preempt_enable();
}
-static inline void
-pud_populate (struct mm_struct *mm, pud_t *pud_entry, pmd_t *pmd)
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
- pud_val(*pud_entry) = __pa(pmd);
+ return pgtable_quicklist_alloc();
}
-static inline pmd_t*
-pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr)
+static inline void pgd_free(pgd_t * pgd)
{
- unsigned long *ret = NULL;
-
- preempt_disable();
-
- ret = (unsigned long *)pmd_quicklist;
- if (likely(ret != NULL)) {
- pmd_quicklist = (unsigned long *)(*ret);
- ret[0] = 0;
- --pgtable_cache_size;
- }
-
- preempt_enable();
-
- return (pmd_t *)ret;
+ pgtable_quicklist_free(pgd);
}
-static inline pmd_t*
-pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
+static inline void
+pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
{
- pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
+ pud_val(*pud_entry) = __pa(pmd);
+}
- return pmd;
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+ return pgtable_quicklist_alloc();
}
-static inline void
-pmd_free (pmd_t *pmd)
+static inline void pmd_free(pmd_t * pmd)
{
- preempt_disable();
- *(unsigned long *)pmd = (unsigned long) pmd_quicklist;
- pmd_quicklist = (unsigned long *) pmd;
- ++pgtable_cache_size;
- preempt_enable();
+ pgtable_quicklist_free(pmd);
}
#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
static inline void
-pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte)
+pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, struct page *pte)
{
pmd_val(*pmd_entry) = page_to_phys(pte);
}
static inline void
-pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte)
+pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte)
{
pmd_val(*pmd_entry) = __pa(pte);
}
-static inline struct page *
-pte_alloc_one (struct mm_struct *mm, unsigned long addr)
+static inline struct page *pte_alloc_one(struct mm_struct *mm,
+ unsigned long addr)
{
- struct page *pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
-
- return pte;
+ return virt_to_page(pgtable_quicklist_alloc());
}
-static inline pte_t *
-pte_alloc_one_kernel (struct mm_struct *mm, unsigned long addr)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+ unsigned long addr)
{
- pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
-
- return pte;
+ return pgtable_quicklist_alloc();
}
-static inline void
-pte_free (struct page *pte)
+static inline void pte_free(struct page *pte)
{
- __free_page(pte);
+ pgtable_quicklist_free(page_address(pte));
}
-static inline void
-pte_free_kernel (pte_t *pte)
+static inline void pte_free_kernel(pte_t * pte)
{
- free_page((unsigned long) pte);
+ pgtable_quicklist_free(pte);
}
-#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte))
+#define __pte_free_tlb(tlb, pte) pte_free(pte)
-extern void check_pgt_cache (void);
+extern void check_pgt_cache(void);
-#endif /* _ASM_IA64_PGALLOC_H */
+#endif /* _ASM_IA64_PGALLOC_H */
git://git.onelab.eu / linux-2.6.git / blobdiff