X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fi386%2Fkernel%2Fsmp.c;h=812b50a1fdab1dfe03aaca6275ebf75945101df1;hb=17a8f298a974b79549792030d6a79ab6b34065bd;hp=afd92719d30f632fce5cd5a46599eed4d8e467b7;hpb=bc77d24c47b89f1e0efed0b8e4be5f8aad102883;p=linux-2.6.git diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c index afd92719d..812b50a1f 100644 --- a/arch/i386/kernel/smp.c +++ b/arch/i386/kernel/smp.c @@ -19,11 +19,11 @@ #include #include #include +#include #include -#include #include -#include +#include #include /* @@ -105,7 +105,7 @@ * about nothing of note with C stepping upwards. */ -struct tlb_state cpu_tlbstate[NR_CPUS] __cacheline_aligned = {[0 ... NR_CPUS-1] = { &init_mm, 0, }}; +DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0, }; /* * the following functions deal with sending IPIs between CPUs. @@ -123,7 +123,7 @@ static inline int __prepare_ICR2 (unsigned int mask) return SET_APIC_DEST_FIELD(mask); } -inline void __send_IPI_shortcut(unsigned int shortcut, int vector) +void __send_IPI_shortcut(unsigned int shortcut, int vector) { /* * Subtle. In the case of the 'never do double writes' workaround @@ -144,6 +144,13 @@ inline void __send_IPI_shortcut(unsigned int shortcut, int vector) */ cfg = __prepare_ICR(shortcut, vector); + if (vector == DUMP_VECTOR) { + /* + * Setup DUMP IPI to be delivered as an NMI + */ + cfg = (cfg&~APIC_VECTOR_MASK)|APIC_DM_NMI; + } + /* * Send the IPI. The write to APIC_ICR fires this off. */ @@ -158,9 +165,9 @@ void fastcall send_IPI_self(int vector) /* * This is only used on smaller machines. */ -inline void send_IPI_mask_bitmask(cpumask_t cpumask, int vector) +void send_IPI_mask_bitmask(cpumask_t cpumask, int vector) { - unsigned long mask = cpus_coerce(cpumask); + unsigned long mask = cpus_addr(cpumask)[0]; unsigned long cfg; unsigned long flags; @@ -221,7 +228,13 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector) * program the ICR */ cfg = __prepare_ICR(0, vector); - + + if (vector == DUMP_VECTOR) { + /* + * Setup DUMP IPI to be delivered as an NMI + */ + cfg = (cfg&~APIC_VECTOR_MASK)|APIC_DM_NMI; + } /* * Send the IPI. The write to APIC_ICR fires this off. */ @@ -231,6 +244,8 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector) local_irq_restore(flags); } +#include /* must come after the send_IPI functions above for inlining */ + /* * Smarter SMP flushing macros. * c/o Linus Torvalds. @@ -256,9 +271,9 @@ static spinlock_t tlbstate_lock = SPIN_LOCK_UNLOCKED; */ static inline void leave_mm (unsigned long cpu) { - if (cpu_tlbstate[cpu].state == TLBSTATE_OK) + if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) BUG(); - cpu_clear(cpu, cpu_tlbstate[cpu].active_mm->cpu_vm_mask); + cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask); load_cr3(swapper_pg_dir); } @@ -308,11 +323,13 @@ static inline void leave_mm (unsigned long cpu) * 2) Leave the mm if we are in the lazy tlb mode. */ -asmlinkage void smp_invalidate_interrupt (void) +fastcall void smp_invalidate_interrupt(struct pt_regs *regs) { unsigned long cpu; cpu = get_cpu(); + if (current->active_mm) + load_user_cs_desc(cpu, current->active_mm); if (!cpu_isset(cpu, flush_cpumask)) goto out; @@ -325,14 +342,12 @@ asmlinkage void smp_invalidate_interrupt (void) * BUG(); */ - if (flush_mm == cpu_tlbstate[cpu].active_mm) { - if (cpu_tlbstate[cpu].state == TLBSTATE_OK) { -#ifndef CONFIG_X86_SWITCH_PAGETABLES + if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) { + if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) { if (flush_va == FLUSH_ALL) local_flush_tlb(); else __flush_tlb_one(flush_va); -#endif } else leave_mm(cpu); } @@ -398,6 +413,21 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, spin_unlock(&tlbstate_lock); } +void flush_tlb_current_task(void) +{ + struct mm_struct *mm = current->mm; + cpumask_t cpu_mask; + + preempt_disable(); + cpu_mask = mm->cpu_vm_mask; + cpu_clear(smp_processor_id(), cpu_mask); + + local_flush_tlb(); + if (!cpus_empty(cpu_mask)) + flush_tlb_others(cpu_mask, mm, FLUSH_ALL); + preempt_enable(); +} + void flush_tlb_mm (struct mm_struct * mm) { cpumask_t cpu_mask; @@ -429,10 +459,7 @@ void flush_tlb_page(struct vm_area_struct * vma, unsigned long va) if (current->active_mm == mm) { if(current->mm) -#ifndef CONFIG_X86_SWITCH_PAGETABLES - __flush_tlb_one(va) -#endif - ; + __flush_tlb_one(va); else leave_mm(smp_processor_id()); } @@ -448,13 +475,18 @@ static void do_flush_tlb_all(void* info) unsigned long cpu = smp_processor_id(); __flush_tlb_all(); - if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY) + if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY) leave_mm(cpu); } void flush_tlb_all(void) { - on_each_cpu(do_flush_tlb_all, 0, 1, 1); + on_each_cpu(do_flush_tlb_all, NULL, 1, 1); +} + +void dump_send_ipi(void) +{ + send_IPI_allbutself(DUMP_VECTOR); } /* @@ -495,7 +527,10 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic, * The function to run. This must be fast and non-blocking. * An arbitrary pointer to pass to the function. * currently unused. - * If true, wait (atomically) until function has completed on other CPUs. + * If 1, wait (atomically) until function has completed on other CPUs. + * If 0, wait for the IPI to be received by other CPUs, but do not wait + * for the completion of the function on each CPU. + * If -1, do not wait for other CPUs to receive IPI. * [RETURNS] 0 on success, else a negative status code. Does not return until * remote CPUs are nearly ready to execute <> or are or have executed. * @@ -503,42 +538,56 @@ int smp_call_function (void (*func) (void *info), void *info, int nonatomic, * hardware interrupt handler or from a bottom half handler. */ { - struct call_data_struct data; + static struct call_data_struct dumpdata; + struct call_data_struct normaldata; + struct call_data_struct *data; int cpus = num_online_cpus()-1; if (!cpus) return 0; /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); + /* Only if we are waiting for other CPU to ack */ + WARN_ON(irqs_disabled() && wait >= 0); spin_lock(&call_lock); - call_data = &data; + if (wait == -1) { + /* if another cpu beat us, they win! */ + if (dumpdata.func) { + spin_unlock(&call_lock); + return 0; + } + data = &dumpdata; + } else + data = &normaldata; + + data->func = func; + data->info = info; + atomic_set(&data->started, 0); + data->wait = wait > 0 ? wait : 0; + if (wait > 0) + atomic_set(&data->finished, 0); + + call_data = data; mb(); /* Send a message to all other CPUs and wait for them to respond */ send_IPI_allbutself(CALL_FUNCTION_VECTOR); /* Wait for response */ - while (atomic_read(&data.started) != cpus) - barrier(); + if (wait >= 0) + while (atomic_read(&data->started) != cpus) + cpu_relax(); - if (wait) - while (atomic_read(&data.finished) != cpus) - barrier(); + if (wait > 0) + while (atomic_read(&data->finished) != cpus) + cpu_relax(); spin_unlock(&call_lock); return 0; } -static void stop_this_cpu (void * dummy) +void stop_this_cpu (void * dummy) { /* * Remove this CPU: @@ -564,17 +613,19 @@ void smp_send_stop(void) local_irq_enable(); } +EXPORT_SYMBOL(smp_send_stop); + /* * Reschedule call back. Nothing to do, * all the work is done automatically when * we return from the interrupt. */ -asmlinkage void smp_reschedule_interrupt(void) +fastcall void smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); } -asmlinkage void smp_call_function_interrupt(void) +fastcall void smp_call_function_interrupt(struct pt_regs *regs) { void (*func) (void *info) = call_data->func; void *info = call_data->info; @@ -599,4 +650,3 @@ asmlinkage void smp_call_function_interrupt(void) atomic_inc(&call_data->finished); } } -