X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fsparc64%2Fkernel%2Fsmp.c;h=f03d52d0b88dd73803cee49b98d9b0aeb90e5d92;hb=43bc926fffd92024b46cafaf7350d669ba9ca884;hp=e5b9c7a27789853d13c6e350afabdfb123092373;hpb=cee37fe97739d85991964371c1f3a745c00dd236;p=linux-2.6.git diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index e5b9c7a27..f03d52d0b 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c @@ -38,15 +38,17 @@ #include #include #include +#include -extern int linux_num_cpus; extern void calibrate_delay(void); /* Please don't make this stuff initdata!!! --DaveM */ static unsigned char boot_cpu_id; -cpumask_t cpu_online_map = CPU_MASK_NONE; -cpumask_t phys_cpu_present_map = CPU_MASK_NONE; +cpumask_t cpu_online_map __read_mostly = CPU_MASK_NONE; +cpumask_t phys_cpu_present_map __read_mostly = CPU_MASK_NONE; +cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly = + { [0 ... NR_CPUS-1] = CPU_MASK_NONE }; static cpumask_t smp_commenced_mask; static cpumask_t cpu_callout_map; @@ -55,30 +57,26 @@ void smp_info(struct seq_file *m) int i; seq_printf(m, "State:\n"); - for (i = 0; i < NR_CPUS; i++) { - if (cpu_online(i)) - seq_printf(m, - "CPU%d:\t\tonline\n", i); - } + for_each_online_cpu(i) + seq_printf(m, "CPU%d:\t\tonline\n", i); } void smp_bogo(struct seq_file *m) { int i; - for (i = 0; i < NR_CPUS; i++) - if (cpu_online(i)) - seq_printf(m, - "Cpu%dBogo\t: %lu.%02lu\n" - "Cpu%dClkTck\t: %016lx\n", - i, cpu_data(i).udelay_val / (500000/HZ), - (cpu_data(i).udelay_val / (5000/HZ)) % 100, - i, cpu_data(i).clock_tick); + for_each_online_cpu(i) + seq_printf(m, + "Cpu%dBogo\t: %lu.%02lu\n" + "Cpu%dClkTck\t: %016lx\n", + i, cpu_data(i).udelay_val / (500000/HZ), + (cpu_data(i).udelay_val / (5000/HZ)) % 100, + i, cpu_data(i).clock_tick); } void __init smp_store_cpu_info(int id) { - int cpu_node; + int cpu_node, def; /* multiplier and counter set by smp_setup_percpu_timer() */ @@ -88,38 +86,56 @@ void __init smp_store_cpu_info(int id) cpu_data(id).clock_tick = prom_getintdefault(cpu_node, "clock-frequency", 0); - cpu_data(id).pgcache_size = 0; - cpu_data(id).pte_cache[0] = NULL; - cpu_data(id).pte_cache[1] = NULL; - cpu_data(id).pgd_cache = NULL; - cpu_data(id).idle_volume = 1; + def = ((tlb_type == hypervisor) ? (8 * 1024) : (16 * 1024)); + cpu_data(id).dcache_size = prom_getintdefault(cpu_node, "dcache-size", + def); + + def = 32; + cpu_data(id).dcache_line_size = + prom_getintdefault(cpu_node, "dcache-line-size", def); + + def = 16 * 1024; + cpu_data(id).icache_size = prom_getintdefault(cpu_node, "icache-size", + def); + + def = 32; + cpu_data(id).icache_line_size = + prom_getintdefault(cpu_node, "icache-line-size", def); + + def = ((tlb_type == hypervisor) ? + (3 * 1024 * 1024) : + (4 * 1024 * 1024)); + cpu_data(id).ecache_size = prom_getintdefault(cpu_node, "ecache-size", + def); + + def = 64; + cpu_data(id).ecache_line_size = + prom_getintdefault(cpu_node, "ecache-line-size", def); + + printk("CPU[%d]: Caches " + "D[sz(%d):line_sz(%d)] " + "I[sz(%d):line_sz(%d)] " + "E[sz(%d):line_sz(%d)]\n", + id, + cpu_data(id).dcache_size, cpu_data(id).dcache_line_size, + cpu_data(id).icache_size, cpu_data(id).icache_line_size, + cpu_data(id).ecache_size, cpu_data(id).ecache_line_size); } static void smp_setup_percpu_timer(void); static volatile unsigned long callin_flag = 0; -extern void inherit_locked_prom_mappings(int save_p); - -static inline void cpu_setup_percpu_base(unsigned long cpu_id) -{ - __asm__ __volatile__("mov %0, %%g5\n\t" - "stxa %0, [%1] %2\n\t" - "membar #Sync" - : /* no outputs */ - : "r" (__per_cpu_offset(cpu_id)), - "r" (TSB_REG), "i" (ASI_IMMU)); -} - void __init smp_callin(void) { int cpuid = hard_smp_processor_id(); - inherit_locked_prom_mappings(0); + __local_per_cpu_offset = __per_cpu_offset(cpuid); - __flush_tlb_all(); + if (tlb_type == hypervisor) + sun4v_ktsb_register(); - cpu_setup_percpu_base(cpuid); + __flush_tlb_all(); smp_setup_percpu_timer(); @@ -137,16 +153,19 @@ void __init smp_callin(void) /* Clear this or we will die instantly when we * schedule back to this idler... */ - clear_thread_flag(TIF_NEWCHILD); + current_thread_info()->new_child = 0; /* Attach to the address space of init_task. */ atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; while (!cpu_isset(cpuid, smp_commenced_mask)) - membar("#LoadLoad"); + rmb(); cpu_set(cpuid, cpu_online_map); + + /* idle thread is expected to have preempt disabled */ + preempt_disable(); } void cpu_panic(void) @@ -155,7 +174,7 @@ void cpu_panic(void) panic("SMP bolixed\n"); } -static unsigned long current_tick_offset; +static unsigned long current_tick_offset __read_mostly; /* This tick register synchronization scheme is taken entirely from * the ia64 port, see arch/ia64/kernel/smpboot.c for details and credit. @@ -184,11 +203,11 @@ static inline long get_delta (long *rt, long *master) for (i = 0; i < NUM_ITERS; i++) { t0 = tick_ops->get_tick(); go[MASTER] = 1; - membar("#StoreLoad"); + membar_storeload(); while (!(tm = go[SLAVE])) - membar("#LoadLoad"); + rmb(); go[SLAVE] = 0; - membar("#StoreStore"); + wmb(); t1 = tick_ops->get_tick(); if (t1 - t0 < best_t1 - best_t0) @@ -221,7 +240,7 @@ void smp_synchronize_tick_client(void) go[MASTER] = 1; while (go[MASTER]) - membar("#LoadLoad"); + rmb(); local_irq_save(flags); { @@ -273,26 +292,28 @@ static void smp_synchronize_one_tick(int cpu) /* wait for client to be ready */ while (!go[MASTER]) - membar("#LoadLoad"); + rmb(); /* now let the client proceed into his loop */ go[MASTER] = 0; - membar("#StoreLoad"); + membar_storeload(); spin_lock_irqsave(&itc_sync_lock, flags); { for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { while (!go[MASTER]) - membar("#LoadLoad"); + rmb(); go[MASTER] = 0; - membar("#StoreStore"); + wmb(); go[SLAVE] = tick_ops->get_tick(); - membar("#StoreLoad"); + membar_storeload(); } } spin_unlock_irqrestore(&itc_sync_lock, flags); } +extern void sun4v_init_mondo_queues(int use_bootmem, int cpu, int alloc, int load); + extern unsigned long sparc64_cpu_startup; /* The OBP cpu startup callback truncates the 3rd arg cookie to @@ -308,21 +329,31 @@ static int __devinit smp_boot_one_cpu(unsigned int cpu) unsigned long cookie = (unsigned long)(&cpu_new_thread); struct task_struct *p; - int timeout, ret, cpu_node; + int timeout, ret; p = fork_idle(cpu); callin_flag = 0; - cpu_new_thread = p->thread_info; + cpu_new_thread = task_thread_info(p); cpu_set(cpu, cpu_callout_map); - cpu_find_by_mid(cpu, &cpu_node); - prom_startcpu(cpu_node, entry, cookie); + if (tlb_type == hypervisor) { + /* Alloc the mondo queues, cpu will load them. */ + sun4v_init_mondo_queues(0, cpu, 1, 0); + + prom_startcpu_cpuid(cpu, entry, cookie); + } else { + int cpu_node; + + cpu_find_by_mid(cpu, &cpu_node); + prom_startcpu(cpu_node, entry, cookie); + } for (timeout = 0; timeout < 5000000; timeout++) { if (callin_flag) break; udelay(100); } + if (callin_flag) { ret = 0; } else { @@ -418,7 +449,7 @@ static __inline__ void spitfire_xcall_deliver(u64 data0, u64 data1, u64 data2, c static void cheetah_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask) { u64 pstate, ver; - int nack_busy_id, is_jalapeno; + int nack_busy_id, is_jbus; if (cpus_empty(mask)) return; @@ -428,7 +459,8 @@ static void cheetah_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mas * derivative processor. */ __asm__ ("rdpr %%ver, %0" : "=r" (ver)); - is_jalapeno = ((ver >> 32) == 0x003e0016); + is_jbus = ((ver >> 32) == __JALAPENO_ID || + (ver >> 32) == __SERRANO_ID); __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); @@ -453,7 +485,7 @@ retry: for_each_cpu_mask(i, mask) { u64 target = (i << 14) | 0x70; - if (!is_jalapeno) + if (!is_jbus) target |= (nack_busy_id << 24); __asm__ __volatile__( "stxa %%g0, [%0] %1\n\t" @@ -506,7 +538,7 @@ retry: for_each_cpu_mask(i, mask) { u64 check_mask; - if (is_jalapeno) + if (is_jbus) check_mask = (0x2UL << (2*i)); else check_mask = (0x2UL << @@ -521,6 +553,155 @@ retry: } } +/* Multi-cpu list version. */ +static void hypervisor_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask) +{ + struct trap_per_cpu *tb; + u16 *cpu_list; + u64 *mondo; + cpumask_t error_mask; + unsigned long flags, status; + int cnt, retries, this_cpu, prev_sent, i; + + /* We have to do this whole thing with interrupts fully disabled. + * Otherwise if we send an xcall from interrupt context it will + * corrupt both our mondo block and cpu list state. + * + * One consequence of this is that we cannot use timeout mechanisms + * that depend upon interrupts being delivered locally. So, for + * example, we cannot sample jiffies and expect it to advance. + * + * Fortunately, udelay() uses %stick/%tick so we can use that. + */ + local_irq_save(flags); + + this_cpu = smp_processor_id(); + tb = &trap_block[this_cpu]; + + mondo = __va(tb->cpu_mondo_block_pa); + mondo[0] = data0; + mondo[1] = data1; + mondo[2] = data2; + wmb(); + + cpu_list = __va(tb->cpu_list_pa); + + /* Setup the initial cpu list. */ + cnt = 0; + for_each_cpu_mask(i, mask) + cpu_list[cnt++] = i; + + cpus_clear(error_mask); + retries = 0; + prev_sent = 0; + do { + int forward_progress, n_sent; + + status = sun4v_cpu_mondo_send(cnt, + tb->cpu_list_pa, + tb->cpu_mondo_block_pa); + + /* HV_EOK means all cpus received the xcall, we're done. */ + if (likely(status == HV_EOK)) + break; + + /* First, see if we made any forward progress. + * + * The hypervisor indicates successful sends by setting + * cpu list entries to the value 0xffff. + */ + n_sent = 0; + for (i = 0; i < cnt; i++) { + if (likely(cpu_list[i] == 0xffff)) + n_sent++; + } + + forward_progress = 0; + if (n_sent > prev_sent) + forward_progress = 1; + + prev_sent = n_sent; + + /* If we get a HV_ECPUERROR, then one or more of the cpus + * in the list are in error state. Use the cpu_state() + * hypervisor call to find out which cpus are in error state. + */ + if (unlikely(status == HV_ECPUERROR)) { + for (i = 0; i < cnt; i++) { + long err; + u16 cpu; + + cpu = cpu_list[i]; + if (cpu == 0xffff) + continue; + + err = sun4v_cpu_state(cpu); + if (err >= 0 && + err == HV_CPU_STATE_ERROR) { + cpu_list[i] = 0xffff; + cpu_set(cpu, error_mask); + } + } + } else if (unlikely(status != HV_EWOULDBLOCK)) + goto fatal_mondo_error; + + /* Don't bother rewriting the CPU list, just leave the + * 0xffff and non-0xffff entries in there and the + * hypervisor will do the right thing. + * + * Only advance timeout state if we didn't make any + * forward progress. + */ + if (unlikely(!forward_progress)) { + if (unlikely(++retries > 10000)) + goto fatal_mondo_timeout; + + /* Delay a little bit to let other cpus catch up + * on their cpu mondo queue work. + */ + udelay(2 * cnt); + } + } while (1); + + local_irq_restore(flags); + + if (unlikely(!cpus_empty(error_mask))) + goto fatal_mondo_cpu_error; + + return; + +fatal_mondo_cpu_error: + printk(KERN_CRIT "CPU[%d]: SUN4V mondo cpu error, some target cpus " + "were in error state\n", + this_cpu); + printk(KERN_CRIT "CPU[%d]: Error mask [ ", this_cpu); + for_each_cpu_mask(i, error_mask) + printk("%d ", i); + printk("]\n"); + return; + +fatal_mondo_timeout: + local_irq_restore(flags); + printk(KERN_CRIT "CPU[%d]: SUN4V mondo timeout, no forward " + " progress after %d retries.\n", + this_cpu, retries); + goto dump_cpu_list_and_out; + +fatal_mondo_error: + local_irq_restore(flags); + printk(KERN_CRIT "CPU[%d]: Unexpected SUN4V mondo error %lu\n", + this_cpu, status); + printk(KERN_CRIT "CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) " + "mondo_block_pa(%lx)\n", + this_cpu, cnt, tb->cpu_list_pa, tb->cpu_mondo_block_pa); + +dump_cpu_list_and_out: + printk(KERN_CRIT "CPU[%d]: CPU list [ ", this_cpu); + for (i = 0; i < cnt; i++) + printk("%u ", cpu_list[i]); + printk("]\n"); +} + /* Send cross call to all processors mentioned in MASK * except self. */ @@ -534,8 +715,10 @@ static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 d if (tlb_type == spitfire) spitfire_xcall_deliver(data0, data1, data2, mask); - else + else if (tlb_type == cheetah || tlb_type == cheetah_plus) cheetah_xcall_deliver(data0, data1, data2, mask); + else + hypervisor_xcall_deliver(data0, data1, data2, mask); /* NOTE: Caller runs local copy on master. */ put_cpu(); @@ -562,24 +745,29 @@ struct call_data_struct { int wait; }; -static DEFINE_SPINLOCK(call_lock); +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock); static struct call_data_struct *call_data; extern unsigned long xcall_call_function; -/* +/** + * smp_call_function(): Run a function on all other CPUs. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @nonatomic: currently unused. + * @wait: If true, wait (atomically) until function has completed on other CPUs. + * + * Returns 0 on success, else a negative status code. Does not return until + * remote CPUs are nearly ready to execute <> or are or have executed. + * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. */ -int smp_call_function(void (*func)(void *info), void *info, - int nonatomic, int wait) +static int smp_call_function_mask(void (*func)(void *info), void *info, + int nonatomic, int wait, cpumask_t mask) { struct call_data_struct data; - int cpus = num_online_cpus() - 1; - long timeout; - - if (!cpus) - return 0; + int cpus; /* Can deadlock when called with interrupts disabled */ WARN_ON(irqs_disabled()); @@ -591,32 +779,31 @@ int smp_call_function(void (*func)(void *info), void *info, spin_lock(&call_lock); + cpu_clear(smp_processor_id(), mask); + cpus = cpus_weight(mask); + if (!cpus) + goto out_unlock; + call_data = &data; + mb(); - smp_cross_call(&xcall_call_function, 0, 0, 0); + smp_cross_call_masked(&xcall_call_function, 0, 0, 0, mask); - /* - * Wait for other cpus to complete function or at - * least snap the call data. - */ - timeout = 1000000; - while (atomic_read(&data.finished) != cpus) { - if (--timeout <= 0) - goto out_timeout; - barrier(); - udelay(1); - } + /* Wait for response */ + while (atomic_read(&data.finished) != cpus) + cpu_relax(); +out_unlock: spin_unlock(&call_lock); return 0; +} -out_timeout: - spin_unlock(&call_lock); - printk("XCALL: Remote cpus not responding, ncpus=%ld finished=%ld\n", - (long) num_online_cpus() - 1L, - (long) atomic_read(&data.finished)); - return 0; +int smp_call_function(void (*func)(void *info), void *info, + int nonatomic, int wait) +{ + return smp_call_function_mask(func, info, nonatomic, wait, + cpu_online_map); } void smp_call_function_client(int irq, struct pt_regs *regs) @@ -636,13 +823,32 @@ void smp_call_function_client(int irq, struct pt_regs *regs) } } +static void tsb_sync(void *info) +{ + struct trap_per_cpu *tp = &trap_block[raw_smp_processor_id()]; + struct mm_struct *mm = info; + + /* It is not valid to test "currrent->active_mm == mm" here. + * + * The value of "current" is not changed atomically with + * switch_mm(). But that's OK, we just need to check the + * current cpu's trap block PGD physical address. + */ + if (tp->pgd_paddr == __pa(mm->pgd)) + tsb_context_switch(mm); +} + +void smp_tsb_sync(struct mm_struct *mm) +{ + smp_call_function_mask(tsb_sync, mm, 0, 1, mm->cpu_vm_mask); +} + extern unsigned long xcall_flush_tlb_mm; extern unsigned long xcall_flush_tlb_pending; extern unsigned long xcall_flush_tlb_kernel_range; -extern unsigned long xcall_flush_tlb_all_spitfire; -extern unsigned long xcall_flush_tlb_all_cheetah; extern unsigned long xcall_report_regs; extern unsigned long xcall_receive_signal; +extern unsigned long xcall_new_mmu_context_version; #ifdef DCACHE_ALIASING_POSSIBLE extern unsigned long xcall_flush_dcache_page_cheetah; @@ -670,11 +876,17 @@ static __inline__ void __local_flush_dcache_page(struct page *page) void smp_flush_dcache_page_impl(struct page *page, int cpu) { cpumask_t mask = cpumask_of_cpu(cpu); - int this_cpu = get_cpu(); + int this_cpu; + + if (tlb_type == hypervisor) + return; #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes); #endif + + this_cpu = get_cpu(); + if (cpu == this_cpu) { __local_flush_dcache_page(page); } else if (cpu_online(cpu)) { @@ -690,7 +902,7 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu) __pa(pg_addr), (u64) pg_addr, mask); - } else { + } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { #ifdef DCACHE_ALIASING_POSSIBLE data0 = ((u64)&xcall_flush_dcache_page_cheetah); @@ -712,7 +924,12 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) void *pg_addr = page_address(page); cpumask_t mask = cpu_online_map; u64 data0; - int this_cpu = get_cpu(); + int this_cpu; + + if (tlb_type == hypervisor) + return; + + this_cpu = get_cpu(); cpu_clear(this_cpu, mask); @@ -729,7 +946,7 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) __pa(pg_addr), (u64) pg_addr, mask); - } else { + } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { #ifdef DCACHE_ALIASING_POSSIBLE data0 = ((u64)&xcall_flush_dcache_page_cheetah); cheetah_xcall_deliver(data0, @@ -746,38 +963,58 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) put_cpu(); } +static void __smp_receive_signal_mask(cpumask_t mask) +{ + smp_cross_call_masked(&xcall_receive_signal, 0, 0, 0, mask); +} + void smp_receive_signal(int cpu) { cpumask_t mask = cpumask_of_cpu(cpu); - if (cpu_online(cpu)) { - u64 data0 = (((u64)&xcall_receive_signal) & 0xffffffff); - - if (tlb_type == spitfire) - spitfire_xcall_deliver(data0, 0, 0, mask); - else - cheetah_xcall_deliver(data0, 0, 0, mask); - } + if (cpu_online(cpu)) + __smp_receive_signal_mask(mask); } void smp_receive_signal_client(int irq, struct pt_regs *regs) { - /* Just return, rtrap takes care of the rest. */ clear_softint(1 << irq); } -void smp_report_regs(void) +void smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) { - smp_cross_call(&xcall_report_regs, 0, 0, 0); + struct mm_struct *mm; + unsigned long flags; + + clear_softint(1 << irq); + + /* See if we need to allocate a new TLB context because + * the version of the one we are using is now out of date. + */ + mm = current->active_mm; + if (unlikely(!mm || (mm == &init_mm))) + return; + + spin_lock_irqsave(&mm->context.lock, flags); + + if (unlikely(!CTX_VALID(mm->context))) + get_new_mmu_context(mm); + + spin_unlock_irqrestore(&mm->context.lock, flags); + + load_secondary_context(mm); + __flush_tlb_mm(CTX_HWBITS(mm->context), + SECONDARY_CONTEXT); } -void smp_flush_tlb_all(void) +void smp_new_mmu_context_version(void) { - if (tlb_type == spitfire) - smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0); - else - smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0); - __flush_tlb_all(); + smp_cross_call(&xcall_new_mmu_context_version, 0, 0, 0); +} + +void smp_report_regs(void) +{ + smp_cross_call(&xcall_report_regs, 0, 0, 0); } /* We know that the window frames of the user have been flushed @@ -818,43 +1055,29 @@ void smp_flush_tlb_all(void) * questionable (in theory the big win for threads is the massive sharing of * address space state across processors). */ + +/* This currently is only used by the hugetlb arch pre-fault + * hook on UltraSPARC-III+ and later when changing the pagesize + * bits of the context register for an address space. + */ void smp_flush_tlb_mm(struct mm_struct *mm) { - /* - * This code is called from two places, dup_mmap and exit_mmap. In the - * former case, we really need a flush. In the later case, the callers - * are single threaded exec_mmap (really need a flush), multithreaded - * exec_mmap case (do not need to flush, since the caller gets a new - * context via activate_mm), and all other callers of mmput() whence - * the flush can be optimized since the associated threads are dead and - * the mm is being torn down (__exit_mm and other mmput callers) or the - * owning thread is dissociating itself from the mm. The - * (atomic_read(&mm->mm_users) == 0) check ensures real work is done - * for single thread exec and dup_mmap cases. An alternate check might - * have been (current->mm != mm). - * Kanoj Sarcar - */ - if (atomic_read(&mm->mm_users) == 0) - return; - - { - u32 ctx = CTX_HWBITS(mm->context); - int cpu = get_cpu(); + u32 ctx = CTX_HWBITS(mm->context); + int cpu = get_cpu(); - if (atomic_read(&mm->mm_users) == 1) { - mm->cpu_vm_mask = cpumask_of_cpu(cpu); - goto local_flush_and_out; - } + if (atomic_read(&mm->mm_users) == 1) { + mm->cpu_vm_mask = cpumask_of_cpu(cpu); + goto local_flush_and_out; + } - smp_cross_call_masked(&xcall_flush_tlb_mm, - ctx, 0, 0, - mm->cpu_vm_mask); + smp_cross_call_masked(&xcall_flush_tlb_mm, + ctx, 0, 0, + mm->cpu_vm_mask); - local_flush_and_out: - __flush_tlb_mm(ctx, SECONDARY_CONTEXT); +local_flush_and_out: + __flush_tlb_mm(ctx, SECONDARY_CONTEXT); - put_cpu(); - } + put_cpu(); } void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs) @@ -862,34 +1085,13 @@ void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long u32 ctx = CTX_HWBITS(mm->context); int cpu = get_cpu(); - if (mm == current->active_mm && atomic_read(&mm->mm_users) == 1) { + if (mm == current->active_mm && atomic_read(&mm->mm_users) == 1) mm->cpu_vm_mask = cpumask_of_cpu(cpu); - goto local_flush_and_out; - } else { - /* This optimization is not valid. Normally - * we will be holding the page_table_lock, but - * there is an exception which is copy_page_range() - * when forking. The lock is held during the individual - * page table updates in the parent, but not at the - * top level, which is where we are invoked. - */ - if (0) { - cpumask_t this_cpu_mask = cpumask_of_cpu(cpu); - - /* By virtue of running under the mm->page_table_lock, - * and mmu_context.h:switch_mm doing the same, the - * following operation is safe. - */ - if (cpus_equal(mm->cpu_vm_mask, this_cpu_mask)) - goto local_flush_and_out; - } - } - - smp_cross_call_masked(&xcall_flush_tlb_pending, - ctx, nr, (unsigned long) vaddrs, - mm->cpu_vm_mask); + else + smp_cross_call_masked(&xcall_flush_tlb_pending, + ctx, nr, (unsigned long) vaddrs, + mm->cpu_vm_mask); -local_flush_and_out: __flush_tlb_pending(ctx, nr, vaddrs); put_cpu(); @@ -927,11 +1129,11 @@ void smp_capture(void) smp_processor_id()); #endif penguins_are_doing_time = 1; - membar("#StoreStore | #LoadStore"); + membar_storestore_loadstore(); atomic_inc(&smp_capture_registry); smp_cross_call(&xcall_capture, 0, 0, 0); while (atomic_read(&smp_capture_registry) != ncpus) - membar("#LoadLoad"); + rmb(); #ifdef CAPTURE_DEBUG printk("done\n"); #endif @@ -947,7 +1149,7 @@ void smp_release(void) smp_processor_id()); #endif penguins_are_doing_time = 0; - membar("#StoreStore | #StoreLoad"); + membar_storeload_storestore(); atomic_dec(&smp_capture_registry); } } @@ -956,37 +1158,25 @@ void smp_release(void) * can service tlb flush xcalls... */ extern void prom_world(int); -extern void save_alternate_globals(unsigned long *); -extern void restore_alternate_globals(unsigned long *); + void smp_penguin_jailcell(int irq, struct pt_regs *regs) { - unsigned long global_save[24]; - clear_softint(1 << irq); preempt_disable(); __asm__ __volatile__("flushw"); - save_alternate_globals(global_save); prom_world(1); atomic_inc(&smp_capture_registry); - membar("#StoreLoad | #StoreStore"); + membar_storeload_storestore(); while (penguins_are_doing_time) - membar("#LoadLoad"); - restore_alternate_globals(global_save); + rmb(); atomic_dec(&smp_capture_registry); prom_world(0); preempt_enable(); } -extern unsigned long xcall_promstop; - -void smp_promstop_others(void) -{ - smp_cross_call(&xcall_promstop, 0, 0, 0); -} - #define prof_multiplier(__cpu) cpu_data(__cpu).multiplier #define prof_counter(__cpu) cpu_data(__cpu).counter @@ -1074,7 +1264,6 @@ void __init smp_tick_init(void) boot_cpu_id = hard_smp_processor_id(); current_tick_offset = timer_tick_offset; - cpu_set(boot_cpu_id, cpu_online_map); prof_counter(boot_cpu_id) = prof_multiplier(boot_cpu_id) = 1; } @@ -1090,7 +1279,7 @@ int setup_profiling_timer(unsigned int multiplier) return -EINVAL; spin_lock_irqsave(&prof_setup_lock, flags); - for (i = 0; i < NR_CPUS; i++) + for_each_possible_cpu(i) prof_multiplier(i) = multiplier; current_tick_offset = (timer_tick_offset / multiplier); spin_unlock_irqrestore(&prof_setup_lock, flags); @@ -1098,22 +1287,53 @@ int setup_profiling_timer(unsigned int multiplier) return 0; } -void __init smp_prepare_cpus(unsigned int max_cpus) +static void __init smp_tune_scheduling(void) { - int instance, mid; + int instance, node; + unsigned int def, smallest = ~0U; + + def = ((tlb_type == hypervisor) ? + (3 * 1024 * 1024) : + (4 * 1024 * 1024)); instance = 0; - while (!cpu_find_by_instance(instance, NULL, &mid)) { - if (mid < max_cpus) - cpu_set(mid, phys_cpu_present_map); + while (!cpu_find_by_instance(instance, &node, NULL)) { + unsigned int val; + + val = prom_getintdefault(node, "ecache-size", def); + if (val < smallest) + smallest = val; + instance++; } + /* Any value less than 256K is nonsense. */ + if (smallest < (256U * 1024U)) + smallest = 256 * 1024; + + max_cache_size = smallest; + + if (smallest < 1U * 1024U * 1024U) + printk(KERN_INFO "Using max_cache_size of %uKB\n", + smallest / 1024U); + else + printk(KERN_INFO "Using max_cache_size of %uMB\n", + smallest / 1024U / 1024U); +} + +/* Constrain the number of cpus to max_cpus. */ +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + int i; + if (num_possible_cpus() > max_cpus) { + int instance, mid; + instance = 0; while (!cpu_find_by_instance(instance, NULL, &mid)) { if (mid != boot_cpu_id) { cpu_clear(mid, phys_cpu_present_map); + cpu_clear(mid, cpu_present_map); if (num_possible_cpus() <= max_cpus) break; } @@ -1121,20 +1341,44 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } + for_each_possible_cpu(i) { + if (tlb_type == hypervisor) { + int j; + + /* XXX get this mapping from machine description */ + for_each_possible_cpu(j) { + if ((j >> 2) == (i >> 2)) + cpu_set(j, cpu_sibling_map[i]); + } + } else { + cpu_set(i, cpu_sibling_map[i]); + } + } + smp_store_cpu_info(boot_cpu_id); + smp_tune_scheduling(); } -void __devinit smp_prepare_boot_cpu(void) +/* Set this up early so that things like the scheduler can init + * properly. We use the same cpu mask for both the present and + * possible cpu map. + */ +void __init smp_setup_cpu_possible_map(void) { - if (hard_smp_processor_id() >= NR_CPUS) { - prom_printf("Serious problem, boot cpu id >= NR_CPUS\n"); - prom_halt(); - } + int instance, mid; - current_thread_info()->cpu = hard_smp_processor_id(); + instance = 0; + while (!cpu_find_by_instance(instance, NULL, &mid)) { + if (mid < NR_CPUS) { + cpu_set(mid, phys_cpu_present_map); + cpu_set(mid, cpu_present_map); + } + instance++; + } +} - cpu_set(smp_processor_id(), cpu_online_map); - cpu_set(smp_processor_id(), phys_cpu_present_map); +void __devinit smp_prepare_boot_cpu(void) +{ } int __devinit __cpu_up(unsigned int cpu) @@ -1148,7 +1392,11 @@ int __devinit __cpu_up(unsigned int cpu) if (!cpu_isset(cpu, cpu_online_map)) { ret = -ENODEV; } else { - smp_synchronize_one_tick(cpu); + /* On SUN4V, writes to %tick and %stick are + * not allowed. + */ + if (tlb_type != hypervisor) + smp_synchronize_one_tick(cpu); } } return ret; @@ -1159,10 +1407,8 @@ void __init smp_cpus_done(unsigned int max_cpus) unsigned long bogosum = 0; int i; - for (i = 0; i < NR_CPUS; i++) { - if (cpu_online(i)) - bogosum += cpu_data(i).udelay_val; - } + for_each_online_cpu(i) + bogosum += cpu_data(i).udelay_val; printk("Total of %ld processors activated " "(%lu.%02lu BogoMIPS).\n", (long) num_online_cpus(), @@ -1170,20 +1416,9 @@ void __init smp_cpus_done(unsigned int max_cpus) (bogosum/(5000/HZ))%100); } -/* This needn't do anything as we do not sleep the cpu - * inside of the idler task, so an interrupt is not needed - * to get a clean fast response. - * - * XXX Reverify this assumption... -DaveM - * - * Addendum: We do want it to do something for the signal - * delivery case, we detect that by just seeing - * if we are trying to send this to an idler or not. - */ void smp_send_reschedule(int cpu) { - if (cpu_data(cpu).idle_volume == 0) - smp_receive_signal(cpu); + smp_receive_signal(cpu); } /* This is a nop because we capture all other cpus @@ -1193,8 +1428,8 @@ void smp_send_stop(void) { } -unsigned long __per_cpu_base; -unsigned long __per_cpu_shift; +unsigned long __per_cpu_base __read_mostly; +unsigned long __per_cpu_shift __read_mostly; EXPORT_SYMBOL(__per_cpu_base); EXPORT_SYMBOL(__per_cpu_shift); @@ -1203,12 +1438,9 @@ void __init setup_per_cpu_areas(void) { unsigned long goal, size, i; char *ptr; - /* Created by linker magic */ - extern char __per_cpu_start[], __per_cpu_end[]; /* Copy section for each CPU (we discard the original) */ - goal = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE); - + goal = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES); #ifdef CONFIG_MODULES if (goal < PERCPU_ENOUGH_ROOM) goal = PERCPU_ENOUGH_ROOM; @@ -1217,31 +1449,13 @@ void __init setup_per_cpu_areas(void) for (size = 1UL; size < goal; size <<= 1UL) __per_cpu_shift++; - /* Make sure the resulting __per_cpu_base value - * will fit in the 43-bit sign extended IMMU - * TSB register. - */ - ptr = __alloc_bootmem(size * NR_CPUS, PAGE_SIZE, - (unsigned long) __per_cpu_start); + ptr = alloc_bootmem(size * NR_CPUS); __per_cpu_base = ptr - __per_cpu_start; - if ((__per_cpu_shift < PAGE_SHIFT) || - (__per_cpu_base & ~PAGE_MASK) || - (__per_cpu_base != (((long) __per_cpu_base << 20) >> 20))) { - prom_printf("PER_CPU: Invalid layout, " - "ptr[%p] shift[%lx] base[%lx]\n", - ptr, __per_cpu_shift, __per_cpu_base); - prom_halt(); - } - for (i = 0; i < NR_CPUS; i++, ptr += size) memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); - /* Finally, load in the boot cpu's base value. - * We abuse the IMMU TSB register for trap handler - * entry and exit loading of %g5. That is why it - * has to be page aligned. - */ - cpu_setup_percpu_base(hard_smp_processor_id()); + /* Setup %g5 for the boot cpu. */ + __local_per_cpu_offset = __per_cpu_offset(smp_processor_id()); }