arch/sparc/kernel/sun4d_smp.c

   1 /* sun4d_smp.c: Sparc SS1000/SC2000 SMP support.
   2  *
   3  * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   4  *
   5  * Based on sun4m's smp.c, which is:
   6  * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
   7  */
   8
   9 #include <asm/head.h>
  10
  11 #include <linux/kernel.h>
  12 #include <linux/sched.h>
  13 #include <linux/threads.h>
  14 #include <linux/smp.h>
  15 #include <linux/smp_lock.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/kernel_stat.h>
  18 #include <linux/init.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/mm.h>
  21 #include <linux/swap.h>
  22
  23 #include <asm/ptrace.h>
  24 #include <asm/atomic.h>
  25
  26 #include <asm/delay.h>
  27 #include <asm/irq.h>
  28 #include <asm/page.h>
  29 #include <asm/pgalloc.h>
  30 #include <asm/pgtable.h>
  31 #include <asm/oplib.h>
  32 #include <asm/hardirq.h>
  33 #include <asm/sbus.h>
  34 #include <asm/sbi.h>
  35 #include <asm/tlbflush.h>
  36 #include <asm/cacheflush.h>
  37 #include <asm/cpudata.h>
  38
  39 #define IRQ_CROSS_CALL          15
  40
  41 extern ctxd_t *srmmu_ctx_table_phys;
  42
  43 extern void calibrate_delay(void);
  44
  45 extern volatile int smp_processors_ready;
  46 extern unsigned long cpu_present_map;
  47 extern int smp_num_cpus;
  48 static int smp_highest_cpu;
  49 extern int smp_threads_ready;
  50 extern volatile unsigned long cpu_callin_map[NR_CPUS];
  51 extern struct cpuinfo_sparc cpu_data[NR_CPUS];
  52 extern unsigned char boot_cpu_id;
  53 extern int smp_activated;
  54 extern volatile int __cpu_number_map[NR_CPUS];
  55 extern volatile int __cpu_logical_map[NR_CPUS];
  56 extern volatile unsigned long ipi_count;
  57 extern volatile int smp_process_available;
  58 extern volatile int smp_commenced;
  59 extern int __smp4d_processor_id(void);
  60
  61 /* #define SMP_DEBUG */
  62
  63 #ifdef SMP_DEBUG
  64 #define SMP_PRINTK(x)   printk x
  65 #else
  66 #define SMP_PRINTK(x)
  67 #endif
  68
  69 static inline unsigned long swap(volatile unsigned long *ptr, unsigned long val)
  70 {
  71         __asm__ __volatile__("swap [%1], %0\n\t" :
  72                              "=&r" (val), "=&r" (ptr) :
  73                              "0" (val), "1" (ptr));
  74         return val;
  75 }
  76
  77 static void smp_setup_percpu_timer(void);
  78 extern void cpu_probe(void);
  79 extern void sun4d_distribute_irqs(void);
  80
  81 void __init smp4d_callin(void)
  82 {
  83         int cpuid = hard_smp4d_processor_id();
  84         extern spinlock_t sun4d_imsk_lock;
  85         unsigned long flags;
  86
  87         /* Show we are alive */
  88         cpu_leds[cpuid] = 0x6;
  89         show_leds(cpuid);
  90
  91         /* Enable level15 interrupt, disable level14 interrupt for now */
  92         cc_set_imsk((cc_get_imsk() & ~0x8000) | 0x4000);
  93
  94         local_flush_cache_all();
  95         local_flush_tlb_all();
  96
  97         /*
  98          * Unblock the master CPU _only_ when the scheduler state
  99          * of all secondary CPUs will be up-to-date, so after
 100          * the SMP initialization the master will be just allowed
 101          * to call the scheduler code.
 102          */
 103         init_idle();
 104
 105         /* Get our local ticker going. */
 106         smp_setup_percpu_timer();
 107
 108         calibrate_delay();
 109         smp_store_cpu_info(cpuid);
 110         local_flush_cache_all();
 111         local_flush_tlb_all();
 112
 113         /* Allow master to continue. */
 114         swap((unsigned long *)&cpu_callin_map[cpuid], 1);
 115         local_flush_cache_all();
 116         local_flush_tlb_all();
 117
 118         cpu_probe();
 119
 120         while((unsigned long)current_set[cpuid] < PAGE_OFFSET)
 121                 barrier();
 122
 123         while(current_set[cpuid]->cpu != cpuid)
 124                 barrier();
 125
 126         /* Fix idle thread fields. */
 127         __asm__ __volatile__("ld [%0], %%g6\n\t"
 128                              "sta %%g6, [%%g0] %1\n\t"
 129                              : : "r" (&current_set[cpuid]), "i" (ASI_M_VIKING_TMP2)
 130                              : "memory" /* paranoid */);
 131
 132         cpu_leds[cpuid] = 0x9;
 133         show_leds(cpuid);
 134
 135         /* Attach to the address space of init_task. */
 136         atomic_inc(&init_mm.mm_count);
 137         current->active_mm = &init_mm;
 138
 139         local_flush_cache_all();
 140         local_flush_tlb_all();
 141
 142         local_irq_enable();     /* We don't allow PIL 14 yet */
 143
 144         while(!smp_commenced)
 145                 barrier();
 146
 147         spin_lock_irqsave(&sun4d_imsk_lock, flags);
 148         cc_set_imsk(cc_get_imsk() & ~0x4000); /* Allow PIL 14 as well */
 149         spin_unlock_irqrestore(&sun4d_imsk_lock, flags);
 150 }
 151
 152 extern int cpu_idle(void *unused);
 153 extern void init_IRQ(void);
 154 extern void cpu_panic(void);
 155 extern int start_secondary(void *unused);
 156
 157 /*
 158  *      Cycle through the processors asking the PROM to start each one.
 159  */
 160
 161 extern struct linux_prom_registers smp_penguin_ctable;
 162 extern unsigned long trapbase_cpu1[];
 163 extern unsigned long trapbase_cpu2[];
 164 extern unsigned long trapbase_cpu3[];
 165
 166 void __init smp4d_boot_cpus(void)
 167 {
 168         int cpucount = 0;
 169         int i, mid;
 170
 171         printk("Entering SMP Mode...\n");
 172
 173         if (boot_cpu_id)
 174                 current_set[0] = NULL;
 175
 176         local_irq_enable();
 177         cpu_present_map = 0;
 178
 179         /* XXX This whole thing has to go.  See sparc64. */
 180         for (i = 0; !cpu_find_by_instance(i, NULL, &mid); i++)
 181                 cpu_present_map |= (1<<mid);
 182         SMP_PRINTK(("cpu_present_map %08lx\n", cpu_present_map));
 183         for(i=0; i < NR_CPUS; i++)
 184                 __cpu_number_map[i] = -1;
 185         for(i=0; i < NR_CPUS; i++)
 186                 __cpu_logical_map[i] = -1;
 187         __cpu_number_map[boot_cpu_id] = 0;
 188         __cpu_logical_map[0] = boot_cpu_id;
 189         current_thread_info()->cpu = boot_cpu_id;
 190         smp_store_cpu_info(boot_cpu_id);
 191         smp_setup_percpu_timer();
 192         local_flush_cache_all();
 193         if (cpu_find_by_instance(1, NULL, NULL))
 194                 return;  /* Not an MP box. */
 195         SMP_PRINTK(("Iterating over CPUs\n"));
 196         for(i = 0; i < NR_CPUS; i++) {
 197                 if(i == boot_cpu_id)
 198                         continue;
 199
 200                 if(cpu_present_map & (1 << i)) {
 201                         extern unsigned long sun4d_cpu_startup;
 202                         unsigned long *entry = &sun4d_cpu_startup;
 203                         struct task_struct *p;
 204                         int timeout;
 205                         int no;
 206
 207                         /* Cook up an idler for this guy. */
 208                         kernel_thread(start_secondary, NULL, CLONE_IDLETASK);
 209
 210                         cpucount++;
 211
 212                         p = prev_task(&init_task);
 213
 214                         init_idle(p, i);
 215
 216                         current_set[i] = p->thread_info;
 217
 218                         unhash_process(p);
 219
 220                         for (no = 0; !cpu_find_by_instance(no, NULL, &mid)
 221                                      && mid != i; no++) ;
 222
 223                         /*
 224                          * Initialize the contexts table
 225                          * Since the call to prom_startcpu() trashes the structure,
 226                          * we need to re-initialize it for each cpu
 227                          */
 228                         smp_penguin_ctable.which_io = 0;
 229                         smp_penguin_ctable.phys_addr = (unsigned int) srmmu_ctx_table_phys;
 230                         smp_penguin_ctable.reg_size = 0;
 231
 232                         /* whirrr, whirrr, whirrrrrrrrr... */
 233                         SMP_PRINTK(("Starting CPU %d at %p task %d node %08x\n", i, entry, cpucount, cpu_data(no).prom_node));
 234                         local_flush_cache_all();
 235                         prom_startcpu(cpu_data(no).prom_node,
 236                                       &smp_penguin_ctable, 0, (char *)entry);
 237
 238                         SMP_PRINTK(("prom_startcpu returned :)\n"));
 239
 240                         /* wheee... it's going... */
 241                         for(timeout = 0; timeout < 10000; timeout++) {
 242                                 if(cpu_callin_map[i])
 243                                         break;
 244                                 udelay(200);
 245                         }
 246
 247                         if(cpu_callin_map[i]) {
 248                                 /* Another "Red Snapper". */
 249                                 __cpu_number_map[i] = cpucount;
 250                                 __cpu_logical_map[cpucount] = i;
 251                         } else {
 252                                 cpucount--;
 253                                 printk("Processor %d is stuck.\n", i);
 254                         }
 255                 }
 256                 if(!(cpu_callin_map[i])) {
 257                         cpu_present_map &= ~(1 << i);
 258                         __cpu_number_map[i] = -1;
 259                 }
 260         }
 261         local_flush_cache_all();
 262         if(cpucount == 0) {
 263                 printk("Error: only one Processor found.\n");
 264                 cpu_present_map = (1 << hard_smp4d_processor_id());
 265         } else {
 266                 unsigned long bogosum = 0;
 267
 268                 for(i = 0; i < NR_CPUS; i++) {
 269                         if(cpu_present_map & (1 << i)) {
 270                                 bogosum += cpu_data(i).udelay_val;
 271                                 smp_highest_cpu = i;
 272                         }
 273                 }
 274                 SMP_PRINTK(("Total of %d Processors activated (%lu.%02lu BogoMIPS).\n", cpucount + 1, bogosum/(500000/HZ), (bogosum/(5000/HZ))%100));
 275                 printk("Total of %d Processors activated (%lu.%02lu BogoMIPS).\n",
 276                        cpucount + 1,
 277                        bogosum/(500000/HZ),
 278                        (bogosum/(5000/HZ))%100);
 279                 smp_activated = 1;
 280                 smp_num_cpus = cpucount + 1;
 281         }
 282
 283         /* Free unneeded trap tables */
 284         ClearPageReserved(virt_to_page(trapbase_cpu1));
 285         set_page_count(virt_to_page(trapbase_cpu1), 1);
 286         free_page((unsigned long)trapbase_cpu1);
 287         totalram_pages++;
 288         num_physpages++;
 289
 290         ClearPageReserved(virt_to_page(trapbase_cpu2));
 291         set_page_count(virt_to_page(trapbase_cpu2), 1);
 292         free_page((unsigned long)trapbase_cpu2);
 293         totalram_pages++;
 294         num_physpages++;
 295
 296         ClearPageReserved(virt_to_page(trapbase_cpu3));
 297         set_page_count(virt_to_page(trapbase_cpu3), 1);
 298         free_page((unsigned long)trapbase_cpu3);
 299         totalram_pages++;
 300         num_physpages++;
 301
 302         /* Ok, they are spinning and ready to go. */
 303         smp_processors_ready = 1;
 304         sun4d_distribute_irqs();
 305 }
 306
 307 static struct smp_funcall {
 308         smpfunc_t func;
 309         unsigned long arg1;
 310         unsigned long arg2;
 311         unsigned long arg3;
 312         unsigned long arg4;
 313         unsigned long arg5;
 314         unsigned char processors_in[NR_CPUS];  /* Set when ipi entered. */
 315         unsigned char processors_out[NR_CPUS]; /* Set when ipi exited. */
 316 } ccall_info __attribute__((aligned(8)));
 317
 318 static spinlock_t cross_call_lock = SPIN_LOCK_UNLOCKED;
 319
 320 /* Cross calls must be serialized, at least currently. */
 321 void smp4d_cross_call(smpfunc_t func, unsigned long arg1, unsigned long arg2,
 322                     unsigned long arg3, unsigned long arg4, unsigned long arg5)
 323 {
 324         if(smp_processors_ready) {
 325                 register int high = smp_highest_cpu;
 326                 unsigned long flags;
 327
 328                 spin_lock_irqsave(&cross_call_lock, flags);
 329
 330                 {
 331                         /* If you make changes here, make sure gcc generates proper code... */
 332                         register smpfunc_t f asm("i0") = func;
 333                         register unsigned long a1 asm("i1") = arg1;
 334                         register unsigned long a2 asm("i2") = arg2;
 335                         register unsigned long a3 asm("i3") = arg3;
 336                         register unsigned long a4 asm("i4") = arg4;
 337                         register unsigned long a5 asm("i5") = arg5;
 338
 339                         __asm__ __volatile__(
 340                                 "std %0, [%6]\n\t"
 341                                 "std %2, [%6 + 8]\n\t"
 342                                 "std %4, [%6 + 16]\n\t" : :
 343                                 "r"(f), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
 344                                 "r" (&ccall_info.func));
 345                 }
 346
 347                 /* Init receive/complete mapping, plus fire the IPI's off. */
 348                 {
 349                         register unsigned long mask;
 350                         register int i;
 351
 352                         mask = (cpu_present_map & ~(1 << hard_smp4d_processor_id()));
 353                         for(i = 0; i <= high; i++) {
 354                                 if(mask & (1 << i)) {
 355                                         ccall_info.processors_in[i] = 0;
 356                                         ccall_info.processors_out[i] = 0;
 357                                         sun4d_send_ipi(i, IRQ_CROSS_CALL);
 358                                 }
 359                         }
 360                 }
 361
 362                 {
 363                         register int i;
 364
 365                         i = 0;
 366                         do {
 367                                 while(!ccall_info.processors_in[i])
 368                                         barrier();
 369                         } while(++i <= high);
 370
 371                         i = 0;
 372                         do {
 373                                 while(!ccall_info.processors_out[i])
 374                                         barrier();
 375                         } while(++i <= high);
 376                 }
 377
 378                 spin_unlock_irqrestore(&cross_call_lock, flags);
 379         }
 380 }
 381
 382 /* Running cross calls. */
 383 void smp4d_cross_call_irq(void)
 384 {
 385         int i = hard_smp4d_processor_id();
 386
 387         ccall_info.processors_in[i] = 1;
 388         ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3,
 389                         ccall_info.arg4, ccall_info.arg5);
 390         ccall_info.processors_out[i] = 1;
 391 }
 392
 393 static int smp4d_stop_cpu_sender;
 394
 395 static void smp4d_stop_cpu(void)
 396 {
 397         int me = hard_smp4d_processor_id();
 398
 399         if (me != smp4d_stop_cpu_sender)
 400                 while(1) barrier();
 401 }
 402
 403 /* Cross calls, in order to work efficiently and atomically do all
 404  * the message passing work themselves, only stopcpu and reschedule
 405  * messages come through here.
 406  */
 407 void smp4d_message_pass(int target, int msg, unsigned long data, int wait)
 408 {
 409         int me = hard_smp4d_processor_id();
 410
 411         SMP_PRINTK(("smp4d_message_pass %d %d %08lx %d\n", target, msg, data, wait));
 412         if (msg == MSG_STOP_CPU && target == MSG_ALL_BUT_SELF) {
 413                 unsigned long flags;
 414                 static spinlock_t stop_cpu_lock = SPIN_LOCK_UNLOCKED;
 415                 spin_lock_irqsave(&stop_cpu_lock, flags);
 416                 smp4d_stop_cpu_sender = me;
 417                 smp4d_cross_call((smpfunc_t)smp4d_stop_cpu, 0, 0, 0, 0, 0);
 418                 spin_unlock_irqrestore(&stop_cpu_lock, flags);
 419         }
 420         printk("Yeeee, trying to send SMP msg(%d) to %d on cpu %d\n", msg, target, me);
 421         panic("Bogon SMP message pass.");
 422 }
 423
 424 extern void sparc_do_profile(unsigned long pc, unsigned long o7);
 425
 426 void smp4d_percpu_timer_interrupt(struct pt_regs *regs)
 427 {
 428         int cpu = hard_smp4d_processor_id();
 429         static int cpu_tick[NR_CPUS];
 430         static char led_mask[] = { 0xe, 0xd, 0xb, 0x7, 0xb, 0xd };
 431
 432         bw_get_prof_limit(cpu);
 433         bw_clear_intr_mask(0, 1);       /* INTR_TABLE[0] & 1 is Profile IRQ */
 434
 435         cpu_tick[cpu]++;
 436         if (!(cpu_tick[cpu] & 15)) {
 437                 if (cpu_tick[cpu] == 0x60)
 438                         cpu_tick[cpu] = 0;
 439                 cpu_leds[cpu] = led_mask[cpu_tick[cpu] >> 4];
 440                 show_leds(cpu);
 441         }
 442
 443         if(!user_mode(regs))
 444                 sparc_do_profile(regs->pc, regs->u_regs[UREG_RETPC]);
 445
 446         if(!--prof_counter(cpu)) {
 447                 int user = user_mode(regs);
 448
 449                 irq_enter();
 450                 update_process_times(user);
 451                 irq_exit();
 452
 453                 prof_counter(cpu) = prof_multiplier(cpu);
 454         }
 455 }
 456
 457 extern unsigned int lvl14_resolution;
 458
 459 static void __init smp_setup_percpu_timer(void)
 460 {
 461         int cpu = hard_smp4d_processor_id();
 462
 463         prof_counter(cpu) = prof_multiplier(cpu) = 1;
 464         load_profile_irq(cpu, lvl14_resolution);
 465 }
 466
 467 void __init smp4d_blackbox_id(unsigned *addr)
 468 {
 469         int rd = *addr & 0x3e000000;
 470
 471         addr[0] = 0xc0800800 | rd;              /* lda [%g0] ASI_M_VIKING_TMP1, reg */
 472         addr[1] = 0x01000000;                   /* nop */
 473         addr[2] = 0x01000000;                   /* nop */
 474 }
 475
 476 void __init smp4d_blackbox_current(unsigned *addr)
 477 {
 478         /* We have a nice Linux current register :) */
 479         int rd = addr[1] & 0x3e000000;
 480
 481         addr[0] = 0x10800006;                   /* b .+24 */
 482         addr[1] = 0xc0800820 | rd;              /* lda [%g0] ASI_M_VIKING_TMP2, reg */
 483 }
 484
 485 void __init sun4d_init_smp(void)
 486 {
 487         int i;
 488         extern unsigned int patchme_store_new_current[];
 489         extern unsigned int t_nmi[], linux_trap_ipi15_sun4d[], linux_trap_ipi15_sun4m[];
 490
 491         /* Store current into Linux current register :) */
 492         __asm__ __volatile__("sta %%g6, [%%g0] %0" : : "i"(ASI_M_VIKING_TMP2));
 493
 494         /* Patch switch_to */
 495         patchme_store_new_current[0] = (patchme_store_new_current[0] & 0x3e000000) | 0xc0a00820;
 496
 497         /* Patch ipi15 trap table */
 498         t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_sun4d - linux_trap_ipi15_sun4m);
 499
 500         /* And set btfixup... */
 501         BTFIXUPSET_BLACKBOX(smp_processor_id, smp4d_blackbox_id);
 502         BTFIXUPSET_BLACKBOX(load_current, smp4d_blackbox_current);
 503         BTFIXUPSET_CALL(smp_cross_call, smp4d_cross_call, BTFIXUPCALL_NORM);
 504         BTFIXUPSET_CALL(smp_message_pass, smp4d_message_pass, BTFIXUPCALL_NORM);
 505         BTFIXUPSET_CALL(__smp_processor_id, __smp4d_processor_id, BTFIXUPCALL_NORM);
 506
 507         for (i = 0; i < NR_CPUS; i++) {
 508                 ccall_info.processors_in[i] = 1;
 509                 ccall_info.processors_out[i] = 1;
 510         }
 511 }