X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fi386%2Fkernel%2Fsmp.c;h=d2523c3ce0356cd2154e61e590e2cc0639ee554e;hb=16c70f8c1b54b61c3b951b6fb220df250fe09b32;hp=bb5960134f673a2f233e4f7735b960cd54fc0b3d;hpb=9bf4aaab3e101692164d49b7ca357651eb691cb6;p=linux-2.6.git diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c index bb5960134..d2523c3ce 100644 --- a/arch/i386/kernel/smp.c +++ b/arch/i386/kernel/smp.c @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -19,10 +18,12 @@ #include #include #include +#include +#include #include #include -#include +#include #include /* @@ -104,7 +105,7 @@ * about nothing of note with C stepping upwards. */ -struct tlb_state cpu_tlbstate[NR_CPUS] __cacheline_aligned = {[0 ... NR_CPUS-1] = { &init_mm, 0, }}; +DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0, }; /* * the following functions deal with sending IPIs between CPUs. @@ -114,7 +115,17 @@ struct tlb_state cpu_tlbstate[NR_CPUS] __cacheline_aligned = {[0 ... NR_CPUS-1] static inline int __prepare_ICR (unsigned int shortcut, int vector) { - return APIC_DM_FIXED | shortcut | vector | APIC_DEST_LOGICAL; + unsigned int icr = shortcut | APIC_DEST_LOGICAL; + + switch (vector) { + default: + icr |= APIC_DM_FIXED | vector; + break; + case NMI_VECTOR: + icr |= APIC_DM_NMI; + break; + } + return icr; } static inline int __prepare_ICR2 (unsigned int mask) @@ -122,7 +133,7 @@ static inline int __prepare_ICR2 (unsigned int mask) return SET_APIC_DEST_FIELD(mask); } -inline void __send_IPI_shortcut(unsigned int shortcut, int vector) +void __send_IPI_shortcut(unsigned int shortcut, int vector) { /* * Subtle. In the case of the 'never do double writes' workaround @@ -157,14 +168,14 @@ void fastcall send_IPI_self(int vector) /* * This is only used on smaller machines. */ -inline void send_IPI_mask_bitmask(cpumask_t cpumask, int vector) +void send_IPI_mask_bitmask(cpumask_t cpumask, int vector) { unsigned long mask = cpus_addr(cpumask)[0]; unsigned long cfg; unsigned long flags; local_irq_save(flags); - + WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]); /* * Wait for idle. */ @@ -189,7 +200,7 @@ inline void send_IPI_mask_bitmask(cpumask_t cpumask, int vector) local_irq_restore(flags); } -inline void send_IPI_mask_sequence(cpumask_t mask, int vector) +void send_IPI_mask_sequence(cpumask_t mask, int vector) { unsigned long cfg, flags; unsigned int query_cpu; @@ -230,6 +241,8 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector) local_irq_restore(flags); } +#include /* must come after the send_IPI functions above for inlining */ + /* * Smarter SMP flushing macros. * c/o Linus Torvalds. @@ -243,7 +256,7 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector) static cpumask_t flush_cpumask; static struct mm_struct * flush_mm; static unsigned long flush_va; -static spinlock_t tlbstate_lock = SPIN_LOCK_UNLOCKED; +static DEFINE_SPINLOCK(tlbstate_lock); #define FLUSH_ALL 0xffffffff /* @@ -255,9 +268,9 @@ static spinlock_t tlbstate_lock = SPIN_LOCK_UNLOCKED; */ static inline void leave_mm (unsigned long cpu) { - if (cpu_tlbstate[cpu].state == TLBSTATE_OK) + if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) BUG(); - cpu_clear(cpu, cpu_tlbstate[cpu].active_mm->cpu_vm_mask); + cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask); load_cr3(swapper_pg_dir); } @@ -307,11 +320,13 @@ static inline void leave_mm (unsigned long cpu) * 2) Leave the mm if we are in the lazy tlb mode. */ -asmlinkage void smp_invalidate_interrupt (void) +fastcall void smp_invalidate_interrupt(struct pt_regs *regs) { unsigned long cpu; cpu = get_cpu(); + if (current->active_mm) + load_user_cs_desc(cpu, current->active_mm); if (!cpu_isset(cpu, flush_cpumask)) goto out; @@ -324,8 +339,8 @@ asmlinkage void smp_invalidate_interrupt (void) * BUG(); */ - if (flush_mm == cpu_tlbstate[cpu].active_mm) { - if (cpu_tlbstate[cpu].state == TLBSTATE_OK) { + if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) { + if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) { if (flush_va == FLUSH_ALL) local_flush_tlb(); else @@ -344,21 +359,21 @@ out: static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, unsigned long va) { - cpumask_t tmp; /* * A couple of (to be removed) sanity checks: * - * - we do not send IPIs to not-yet booted CPUs. * - current CPU must not be in mask * - mask must exist :) */ BUG_ON(cpus_empty(cpumask)); - - cpus_and(tmp, cpumask, cpu_online_map); - BUG_ON(!cpus_equal(cpumask, tmp)); BUG_ON(cpu_isset(smp_processor_id(), cpumask)); BUG_ON(!mm); + /* If a CPU which we ran on has gone down, OK. */ + cpus_and(cpumask, cpumask, cpu_online_map); + if (cpus_empty(cpumask)) + return; + /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. @@ -451,13 +466,14 @@ void flush_tlb_page(struct vm_area_struct * vma, unsigned long va) preempt_enable(); } +EXPORT_SYMBOL(flush_tlb_page); static void do_flush_tlb_all(void* info) { unsigned long cpu = smp_processor_id(); __flush_tlb_all(); - if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY) + if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY) leave_mm(cpu); } @@ -473,6 +489,7 @@ void flush_tlb_all(void) */ void smp_send_reschedule(int cpu) { + WARN_ON(cpu_is_offline(cpu)); send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); } @@ -480,7 +497,7 @@ void smp_send_reschedule(int cpu) * Structure and data for smp_call_function(). This is designed to minimise * static memory requirements. It also looks cleaner. */ -static spinlock_t call_lock = SPIN_LOCK_UNLOCKED; +static DEFINE_SPINLOCK(call_lock); struct call_data_struct { void (*func) (void *info); @@ -490,33 +507,44 @@ struct call_data_struct { int wait; }; -static struct call_data_struct * call_data; +void lock_ipi_call_lock(void) +{ + spin_lock_irq(&call_lock); +} -/* - * this function sends a 'generic call function' IPI to all other CPUs - * in the system. - */ +void unlock_ipi_call_lock(void) +{ + spin_unlock_irq(&call_lock); +} -int smp_call_function (void (*func) (void *info), void *info, int nonatomic, - int wait) -/* - * [SUMMARY] Run a function on all other CPUs. - * The function to run. This must be fast and non-blocking. - * An arbitrary pointer to pass to the function. - * currently unused. - * If true, wait (atomically) until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. Does not return until +static struct call_data_struct *call_data; + +/** + * smp_call_function(): Run a function on all other CPUs. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @nonatomic: currently unused. + * @wait: If true, wait (atomically) until function has completed on other CPUs. + * + * Returns 0 on success, else a negative status code. Does not return until * remote CPUs are nearly ready to execute <> or are or have executed. * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. */ +int smp_call_function (void (*func) (void *info), void *info, int nonatomic, + int wait) { struct call_data_struct data; - int cpus = num_online_cpus()-1; + int cpus; - if (!cpus) + /* Holding any lock stops cpus from going down. */ + spin_lock(&call_lock); + cpus = num_online_cpus() - 1; + if (!cpus) { + spin_unlock(&call_lock); return 0; + } /* Can deadlock when called with interrupts disabled */ WARN_ON(irqs_disabled()); @@ -528,7 +556,6 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic, if (wait) atomic_set(&data.finished, 0); - spin_lock(&call_lock); call_data = &data; mb(); @@ -537,15 +564,16 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic, /* Wait for response */ while (atomic_read(&data.started) != cpus) - barrier(); + cpu_relax(); if (wait) while (atomic_read(&data.finished) != cpus) - barrier(); + cpu_relax(); spin_unlock(&call_lock); return 0; } +EXPORT_SYMBOL(smp_call_function); static void stop_this_cpu (void * dummy) { @@ -556,7 +584,7 @@ static void stop_this_cpu (void * dummy) local_irq_disable(); disable_local_APIC(); if (cpu_data[smp_processor_id()].hlt_works_ok) - for(;;) __asm__("hlt"); + for(;;) halt(); for (;;); } @@ -578,12 +606,12 @@ void smp_send_stop(void) * all the work is done automatically when * we return from the interrupt. */ -asmlinkage void smp_reschedule_interrupt(void) +fastcall void smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); } -asmlinkage void smp_call_function_interrupt(void) +fastcall void smp_call_function_interrupt(struct pt_regs *regs) { void (*func) (void *info) = call_data->func; void *info = call_data->info;