X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fi386%2Fmm%2Fpageattr.c;h=6f5efbde67ff801058ea5563c6f51b6e26dd67b8;hb=97bf2856c6014879bd04983a3e9dfcdac1e7fe85;hp=862326f5dc38e65e358f2230d1d36737d0f5602d;hpb=6a77f38946aaee1cd85eeec6cf4229b204c15071;p=linux-2.6.git diff --git a/arch/i386/mm/pageattr.c b/arch/i386/mm/pageattr.c index 862326f5d..6f5efbde6 100644 --- a/arch/i386/mm/pageattr.c +++ b/arch/i386/mm/pageattr.c @@ -3,15 +3,18 @@ * Thanks to Ben LaHaise for precious feedback. */ -#include #include #include #include #include #include +#include +#include #include #include #include +#include +#include static DEFINE_SPINLOCK(cpa_lock); static struct list_head df_list = LIST_HEAD_INIT(df_list); @@ -35,7 +38,8 @@ pte_t *lookup_address(unsigned long address) return pte_offset_kernel(pmd, address); } -static struct page *split_large_page(unsigned long address, pgprot_t prot) +static struct page *split_large_page(unsigned long address, pgprot_t prot, + pgprot_t ref_prot) { int i; unsigned long addr; @@ -48,21 +52,34 @@ static struct page *split_large_page(unsigned long address, pgprot_t prot) if (!base) return NULL; + /* + * page_private is used to track the number of entries in + * the page table page that have non standard attributes. + */ + SetPagePrivate(base); + page_private(base) = 0; + address = __pa(address); addr = address & LARGE_PAGE_MASK; pbase = (pte_t *)page_address(base); for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) { - pbase[i] = pfn_pte(addr >> PAGE_SHIFT, - addr == address ? prot : PAGE_KERNEL); + set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, + addr == address ? prot : ref_prot)); } return base; } -static void flush_kernel_map(void *dummy) +static void flush_kernel_map(void *arg) { - /* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */ - if (boot_cpu_data.x86_model >= 4) - asm volatile("wbinvd":::"memory"); + unsigned long adr = (unsigned long)arg; + + if (adr && cpu_has_clflush) { + int i; + for (i = 0; i < PAGE_SIZE; i += boot_cpu_data.x86_clflush_size) + asm volatile("clflush (%0)" :: "r" (adr + i)); + } else if (boot_cpu_data.x86_model >= 4) + wbinvd(); + /* Flush all to work around Errata in early athlons regarding * large page flushing. */ @@ -75,7 +92,7 @@ static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) unsigned long flags; set_pte_atomic(kpte, pte); /* change init_mm */ - if (PTRS_PER_PMD > 1) + if (HAVE_SHARED_KERNEL_PMD) return; spin_lock_irqsave(&pgd_lock, flags); @@ -97,11 +114,18 @@ static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) */ static inline void revert_page(struct page *kpte_page, unsigned long address) { - pte_t *linear = (pte_t *) + pgprot_t ref_prot; + pte_t *linear; + + ref_prot = + ((address & LARGE_PAGE_MASK) < (unsigned long)&_etext) + ? PAGE_KERNEL_LARGE_EXEC : PAGE_KERNEL_LARGE; + + linear = (pte_t *) pmd_offset(pud_offset(pgd_offset_k(address), address), address); set_pmd_pte(linear, address, pfn_pte((__pa(address) & LARGE_PAGE_MASK) >> PAGE_SHIFT, - PAGE_KERNEL_LARGE)); + ref_prot)); } static int @@ -114,6 +138,12 @@ __change_page_attr(struct page *page, pgprot_t prot) BUG_ON(PageHighMem(page)); address = (unsigned long)page_address(page); + if (address >= (unsigned long)__start_rodata && address <= (unsigned long)__end_rodata && + (pgprot_val(prot) & _PAGE_RW)) { + pgprot_val(prot) &= ~(_PAGE_RW); + add_taint(TAINT_MACHINE_CHECK); + } + kpte = lookup_address(address); if (!kpte) return -EINVAL; @@ -122,16 +152,23 @@ __change_page_attr(struct page *page, pgprot_t prot) if ((pte_val(*kpte) & _PAGE_PSE) == 0) { set_pte_atomic(kpte, mk_pte(page, prot)); } else { - struct page *split = split_large_page(address, prot); + pgprot_t ref_prot; + struct page *split; + + ref_prot = + ((address & LARGE_PAGE_MASK) < (unsigned long)&_etext) + ? PAGE_KERNEL_EXEC : PAGE_KERNEL; + split = split_large_page(address, prot, ref_prot); if (!split) return -ENOMEM; - set_pmd_pte(kpte,address,mk_pte(split, PAGE_KERNEL)); + set_pmd_pte(kpte,address,mk_pte(split, ref_prot)); kpte_page = split; - } - get_page(kpte_page); + } + page_private(kpte_page)++; } else if ((pte_val(*kpte) & _PAGE_PSE) == 0) { set_pte_atomic(kpte, mk_pte(page, PAGE_KERNEL)); - __put_page(kpte_page); + BUG_ON(page_private(kpte_page) == 0); + page_private(kpte_page)--; } else BUG(); @@ -141,10 +178,8 @@ __change_page_attr(struct page *page, pgprot_t prot) * replace it with a largepage. */ if (!PageReserved(kpte_page)) { - /* memleak and potential failed 2M page regeneration */ - BUG_ON(!page_count(kpte_page)); - - if (cpu_has_pse && (page_count(kpte_page) == 1)) { + if (cpu_has_pse && (page_private(kpte_page) == 0)) { + ClearPagePrivate(kpte_page); list_add(&kpte_page->lru, &df_list); revert_page(kpte_page, address); } @@ -152,9 +187,9 @@ __change_page_attr(struct page *page, pgprot_t prot) return 0; } -static inline void flush_map(void) +static inline void flush_map(void *adr) { - on_each_cpu(flush_kernel_map, NULL, 1, 1); + on_each_cpu(flush_kernel_map, adr, 1, 1); } /* @@ -187,29 +222,33 @@ int change_page_attr(struct page *page, int numpages, pgprot_t prot) } void global_flush_tlb(void) -{ - LIST_HEAD(l); - struct list_head* n; +{ + struct list_head l; + struct page *pg, *next; BUG_ON(irqs_disabled()); spin_lock_irq(&cpa_lock); - list_splice_init(&df_list, &l); + list_replace_init(&df_list, &l); spin_unlock_irq(&cpa_lock); - flush_map(); - n = l.next; - while (n != &l) { - struct page *pg = list_entry(n, struct page, lru); - n = n->next; + if (!cpu_has_clflush) + flush_map(0); + list_for_each_entry_safe(pg, next, &l, lru) { + if (cpu_has_clflush) + flush_map(page_address(pg)); __free_page(pg); } -} +} #ifdef CONFIG_DEBUG_PAGEALLOC void kernel_map_pages(struct page *page, int numpages, int enable) { if (PageHighMem(page)) return; + if (!enable) + debug_check_no_locks_freed(page_address(page), + numpages * PAGE_SIZE); + /* the return value is ignored - the calls cannot fail, * large pages are disabled at boot time. */