Merge to kernel-2.6.20-1.2949.fc6.vs2.2.0.1

[linux-2.6.git] / include / asm-i386 / tlbflush.h

diff --git a/include/asm-i386/tlbflush.h b/include/asm-i386/tlbflush.h
index 84975eb..4dd8284 100644 (file)
--- a/include/asm-i386/tlbflush.h
+++ b/include/asm-i386/tlbflush.h
@@ -1,11 +1,18 @@
 #ifndef _I386_TLBFLUSH_H
 #define _I386_TLBFLUSH_H
 
-#include <linux/config.h>
 #include <linux/mm.h>
 #include <asm/processor.h>
 
-#define __flush_tlb()                                                  \
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#else
+#define __flush_tlb() __native_flush_tlb()
+#define __flush_tlb_global() __native_flush_tlb_global()
+#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
+#endif
+
+#define __native_flush_tlb()                                           \
        do {                                                            \
                unsigned int tmpreg;                                    \
                                                                        \
@@ -20,22 +27,25 @@
  * Global pages have to be flushed a bit differently. Not a real
  * performance problem because this does not happen often.
  */
-#define __flush_tlb_global()                                           \
+#define __native_flush_tlb_global()                                    \
        do {                                                            \
-               unsigned int tmpreg;                                    \
+               unsigned int tmpreg, cr4, cr4_orig;                     \
                                                                        \
                __asm__ __volatile__(                                   \
-                       "movl %1, %%cr4;  # turn off PGE     \n"        \
+                       "movl %%cr4, %2;  # turn off PGE     \n"        \
+                       "movl %2, %1;                        \n"        \
+                       "andl %3, %1;                        \n"        \
+                       "movl %1, %%cr4;                     \n"        \
                        "movl %%cr3, %0;                     \n"        \
                        "movl %0, %%cr3;  # flush TLB        \n"        \
                        "movl %2, %%cr4;  # turn PGE back on \n"        \
-                       : "=&r" (tmpreg)                                \
-                       : "r" (mmu_cr4_features & ~X86_CR4_PGE),        \
-                         "r" (mmu_cr4_features)                        \
+                       : "=&r" (tmpreg), "=&r" (cr4), "=&r" (cr4_orig) \
+                       : "i" (~X86_CR4_PGE)                            \
                        : "memory");                                    \
        } while (0)
 
-extern unsigned long pgkern_mask;
+#define __native_flush_tlb_single(addr)                                \
+       __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory")
 
 # define __flush_tlb_all()                                             \
        do {                                                            \
@@ -47,9 +57,6 @@ extern unsigned long pgkern_mask;
 
 #define cpu_has_invlpg (boot_cpu_data.x86 > 3)
 
-#define __flush_tlb_single(addr) \
-       __asm__ __volatile__("invlpg %0": :"m" (*(char *) addr))
-
 #ifdef CONFIG_X86_INVLPG
 # define __flush_tlb_one(addr) __flush_tlb_single(addr)
 #else
@@ -85,28 +92,22 @@ extern unsigned long pgkern_mask;
 
 static inline void flush_tlb_mm(struct mm_struct *mm)
 {
-#ifndef CONFIG_X86_SWITCH_PAGETABLES
        if (mm == current->active_mm)
                __flush_tlb();
-#endif
 }
 
 static inline void flush_tlb_page(struct vm_area_struct *vma,
        unsigned long addr)
 {
-#ifndef CONFIG_X86_SWITCH_PAGETABLES
        if (vma->vm_mm == current->active_mm)
                __flush_tlb_one(addr);
-#endif
 }
 
 static inline void flush_tlb_range(struct vm_area_struct *vma,
        unsigned long start, unsigned long end)
 {
-#ifndef CONFIG_X86_SWITCH_PAGETABLES
        if (vma->vm_mm == current->active_mm)
                __flush_tlb();
-#endif
 }
 
 #else
@@ -117,10 +118,11 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
        __flush_tlb()
 
 extern void flush_tlb_all(void);
+extern void flush_tlb_current_task(void);
 extern void flush_tlb_mm(struct mm_struct *);
 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
 
-#define flush_tlb()    flush_tlb_all()
+#define flush_tlb()    flush_tlb_current_task()
 
 static inline void flush_tlb_range(struct vm_area_struct * vma, unsigned long start, unsigned long end)
 {
@@ -136,7 +138,7 @@ struct tlb_state
        int state;
        char __cacheline_padding[L1_CACHE_BYTES-8];
 };
-extern struct tlb_state cpu_tlbstate[NR_CPUS];
+DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
 
 
 #endif