linux 2.6.16.38 w/ vs2.0.3-rc1

[linux-2.6.git] / arch / i386 / kernel / smp.c

diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c
index 5c4f178..218d725 100644 (file)
--- a/arch/i386/kernel/smp.c
+++ b/arch/i386/kernel/smp.c
@@ -11,7 +11,6 @@
 #include <linux/init.h>
 
 #include <linux/mm.h>
-#include <linux/irq.h>
 #include <linux/delay.h>
 #include <linux/spinlock.h>
 #include <linux/smp_lock.h>
@@ -19,7 +18,8 @@
 #include <linux/mc146818rtc.h>
 #include <linux/cache.h>
 #include <linux/interrupt.h>
-#include <linux/dump.h>
+#include <linux/cpu.h>
+#include <linux/module.h>
 
 #include <asm/mtrr.h>
 #include <asm/tlbflush.h>
@@ -143,13 +143,6 @@ void __send_IPI_shortcut(unsigned int shortcut, int vector)
         */
        cfg = __prepare_ICR(shortcut, vector);
 
-       if (vector == DUMP_VECTOR) {
-               /*
-                * Setup DUMP IPI to be delivered as an NMI
-                */
-               cfg = (cfg&~APIC_VECTOR_MASK)|APIC_DM_NMI;
-       }
-
        /*
         * Send the IPI. The write to APIC_ICR fires this off.
         */
@@ -171,7 +164,7 @@ void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
        unsigned long flags;
 
        local_irq_save(flags);
-               
+       WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
        /*
         * Wait for idle.
         */
@@ -196,7 +189,7 @@ void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
        local_irq_restore(flags);
 }
 
-inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
+void send_IPI_mask_sequence(cpumask_t mask, int vector)
 {
        unsigned long cfg, flags;
        unsigned int query_cpu;
@@ -227,13 +220,7 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
                         * program the ICR 
                         */
                        cfg = __prepare_ICR(0, vector);
-               
-                       if (vector == DUMP_VECTOR) {
-                               /*
-                                * Setup DUMP IPI to be delivered as an NMI
-                                */
-                               cfg = (cfg&~APIC_VECTOR_MASK)|APIC_DM_NMI;
-                       }       
+                       
                        /*
                         * Send the IPI. The write to APIC_ICR fires this off.
                         */
@@ -258,7 +245,7 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
 static cpumask_t flush_cpumask;
 static struct mm_struct * flush_mm;
 static unsigned long flush_va;
-static spinlock_t tlbstate_lock = SPIN_LOCK_UNLOCKED;
+static DEFINE_SPINLOCK(tlbstate_lock);
 #define FLUSH_ALL      0xffffffff
 
 /*
@@ -322,7 +309,7 @@ static inline void leave_mm (unsigned long cpu)
  * 2) Leave the mm if we are in the lazy tlb mode.
  */
 
-asmlinkage void smp_invalidate_interrupt (void)
+fastcall void smp_invalidate_interrupt(struct pt_regs *regs)
 {
        unsigned long cpu;
 
@@ -359,21 +346,21 @@ out:
 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
                                                unsigned long va)
 {
-       cpumask_t tmp;
        /*
         * A couple of (to be removed) sanity checks:
         *
-        * - we do not send IPIs to not-yet booted CPUs.
         * - current CPU must not be in mask
         * - mask must exist :)
         */
        BUG_ON(cpus_empty(cpumask));
-
-       cpus_and(tmp, cpumask, cpu_online_map);
-       BUG_ON(!cpus_equal(cpumask, tmp));
        BUG_ON(cpu_isset(smp_processor_id(), cpumask));
        BUG_ON(!mm);
 
+       /* If a CPU which we ran on has gone down, OK. */
+       cpus_and(cpumask, cpumask, cpu_online_map);
+       if (cpus_empty(cpumask))
+               return;
+
        /*
         * i'm not happy about this global shared spinlock in the
         * MM hot path, but we'll see how contended it is.
@@ -466,6 +453,7 @@ void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
 
        preempt_enable();
 }
+EXPORT_SYMBOL(flush_tlb_page);
 
 static void do_flush_tlb_all(void* info)
 {
@@ -481,11 +469,6 @@ void flush_tlb_all(void)
        on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
 }
 
-void dump_send_ipi(void)
-{
-       send_IPI_allbutself(DUMP_VECTOR);
-}
-
 /*
  * this function sends a 'reschedule' IPI to another CPU.
  * it goes straight through and wastes no time serializing
@@ -493,6 +476,7 @@ void dump_send_ipi(void)
  */
 void smp_send_reschedule(int cpu)
 {
+       WARN_ON(cpu_is_offline(cpu));
        send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
 }
 
@@ -500,7 +484,7 @@ void smp_send_reschedule(int cpu)
  * Structure and data for smp_call_function(). This is designed to minimise
  * static memory requirements. It also looks cleaner.
  */
-static spinlock_t call_lock = SPIN_LOCK_UNLOCKED;
+static DEFINE_SPINLOCK(call_lock);
 
 struct call_data_struct {
        void (*func) (void *info);
@@ -510,6 +494,16 @@ struct call_data_struct {
        int wait;
 };
 
+void lock_ipi_call_lock(void)
+{
+       spin_lock_irq(&call_lock);
+}
+
+void unlock_ipi_call_lock(void)
+{
+       spin_unlock_irq(&call_lock);
+}
+
 static struct call_data_struct * call_data;
 
 /*
@@ -533,10 +527,15 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
  */
 {
        struct call_data_struct data;
-       int cpus = num_online_cpus()-1;
+       int cpus;
 
-       if (!cpus)
+       /* Holding any lock stops cpus from going down. */
+       spin_lock(&call_lock);
+       cpus = num_online_cpus() - 1;
+       if (!cpus) {
+               spin_unlock(&call_lock);
                return 0;
+       }
 
        /* Can deadlock when called with interrupts disabled */
        WARN_ON(irqs_disabled());
@@ -548,7 +547,6 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
        if (wait)
                atomic_set(&data.finished, 0);
 
-       spin_lock(&call_lock);
        call_data = &data;
        mb();
        
@@ -557,17 +555,18 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
 
        /* Wait for response */
        while (atomic_read(&data.started) != cpus)
-               barrier();
+               cpu_relax();
 
        if (wait)
                while (atomic_read(&data.finished) != cpus)
-                       barrier();
+                       cpu_relax();
        spin_unlock(&call_lock);
 
        return 0;
 }
+EXPORT_SYMBOL(smp_call_function);
 
-void stop_this_cpu (void * dummy)
+static void stop_this_cpu (void * dummy)
 {
        /*
         * Remove this CPU:
@@ -576,7 +575,7 @@ void stop_this_cpu (void * dummy)
        local_irq_disable();
        disable_local_APIC();
        if (cpu_data[smp_processor_id()].hlt_works_ok)
-               for(;;) __asm__("hlt");
+               for(;;) halt();
        for (;;);
 }
 
@@ -593,19 +592,17 @@ void smp_send_stop(void)
        local_irq_enable();
 }
 
-EXPORT_SYMBOL(smp_send_stop);
-
 /*
  * Reschedule call back. Nothing to do,
  * all the work is done automatically when
  * we return from the interrupt.
  */
-asmlinkage void smp_reschedule_interrupt(void)
+fastcall void smp_reschedule_interrupt(struct pt_regs *regs)
 {
        ack_APIC_irq();
 }
 
-asmlinkage void smp_call_function_interrupt(void)
+fastcall void smp_call_function_interrupt(struct pt_regs *regs)
 {
        void (*func) (void *info) = call_data->func;
        void *info = call_data->info;
@@ -630,3 +627,4 @@ asmlinkage void smp_call_function_interrupt(void)
                atomic_inc(&call_data->finished);
        }
 }
+