arch/ppc64/kernel/smp.c

   1 /*
   2  * SMP support for ppc.
   3  *
   4  * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
   5  * deal of code from the sparc and intel versions.
   6  *
   7  * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
   8  *
   9  * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
  10  * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
  11  *
  12  *      This program is free software; you can redistribute it and/or
  13  *      modify it under the terms of the GNU General Public License
  14  *      as published by the Free Software Foundation; either version
  15  *      2 of the License, or (at your option) any later version.
  16  */
  17
  18 #include <linux/config.h>
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/sched.h>
  22 #include <linux/smp.h>
  23 #include <linux/smp_lock.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/delay.h>
  27 #include <linux/init.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/cache.h>
  30 #include <linux/err.h>
  31 #include <linux/sysdev.h>
  32 #include <linux/cpu.h>
  33
  34 #include <asm/ptrace.h>
  35 #include <asm/atomic.h>
  36 #include <asm/irq.h>
  37 #include <asm/page.h>
  38 #include <asm/pgtable.h>
  39 #include <asm/hardirq.h>
  40 #include <asm/io.h>
  41 #include <asm/prom.h>
  42 #include <asm/smp.h>
  43 #include <asm/naca.h>
  44 #include <asm/paca.h>
  45 #include <asm/iSeries/LparData.h>
  46 #include <asm/iSeries/HvCall.h>
  47 #include <asm/iSeries/HvCallCfg.h>
  48 #include <asm/time.h>
  49 #include <asm/ppcdebug.h>
  50 #include "open_pic.h"
  51 #include <asm/machdep.h>
  52 #include <asm/xics.h>
  53 #include <asm/cputable.h>
  54 #include <asm/system.h>
  55 #include <asm/rtas.h>
  56
  57 int smp_threads_ready;
  58 unsigned long cache_decay_ticks;
  59
  60 cpumask_t cpu_possible_map = CPU_MASK_NONE;
  61 cpumask_t cpu_online_map = CPU_MASK_NONE;
  62 cpumask_t cpu_available_map = CPU_MASK_NONE;
  63 cpumask_t cpu_present_at_boot = CPU_MASK_NONE;
  64
  65 EXPORT_SYMBOL(cpu_online_map);
  66 EXPORT_SYMBOL(cpu_possible_map);
  67
  68 struct smp_ops_t *smp_ops;
  69
  70 static volatile unsigned int cpu_callin_map[NR_CPUS];
  71
  72 extern unsigned char stab_array[];
  73
  74 extern int cpu_idle(void *unused);
  75 void smp_call_function_interrupt(void);
  76 extern long register_vpa(unsigned long flags, unsigned long proc,
  77                          unsigned long vpa);
  78
  79 /* Low level assembly function used to backup CPU 0 state */
  80 extern void __save_cpu_setup(void);
  81
  82 #ifdef CONFIG_PPC_ISERIES
  83 static unsigned long iSeries_smp_message[NR_CPUS];
  84
  85 void iSeries_smp_message_recv( struct pt_regs * regs )
  86 {
  87         int cpu = smp_processor_id();
  88         int msg;
  89
  90         if ( num_online_cpus() < 2 )
  91                 return;
  92
  93         for ( msg = 0; msg < 4; ++msg )
  94                 if ( test_and_clear_bit( msg, &iSeries_smp_message[cpu] ) )
  95                         smp_message_recv( msg, regs );
  96 }
  97
  98 static inline void smp_iSeries_do_message(int cpu, int msg)
  99 {
 100         set_bit(msg, &iSeries_smp_message[cpu]);
 101         HvCall_sendIPI(&(paca[cpu]));
 102 }
 103
 104 static void smp_iSeries_message_pass(int target, int msg)
 105 {
 106         int i;
 107
 108         if (target < NR_CPUS)
 109                 smp_iSeries_do_message(target, msg);
 110         else {
 111                 for_each_online_cpu(i) {
 112                         if (target == MSG_ALL_BUT_SELF
 113                             && i == smp_processor_id())
 114                                 continue;
 115                         smp_iSeries_do_message(i, msg);
 116                 }
 117         }
 118 }
 119
 120 static int smp_iSeries_numProcs(void)
 121 {
 122         unsigned np, i;
 123
 124         np = 0;
 125         for (i=0; i < NR_CPUS; ++i) {
 126                 if (paca[i].lppaca.xDynProcStatus < 2) {
 127                         cpu_set(i, cpu_available_map);
 128                         cpu_set(i, cpu_possible_map);
 129                         cpu_set(i, cpu_present_at_boot);
 130                         ++np;
 131                 }
 132         }
 133         return np;
 134 }
 135
 136 static int smp_iSeries_probe(void)
 137 {
 138         unsigned i;
 139         unsigned np = 0;
 140
 141         for (i=0; i < NR_CPUS; ++i) {
 142                 if (paca[i].lppaca.xDynProcStatus < 2) {
 143                         /*paca[i].active = 1;*/
 144                         ++np;
 145                 }
 146         }
 147
 148         return np;
 149 }
 150
 151 static void smp_iSeries_kick_cpu(int nr)
 152 {
 153         BUG_ON(nr < 0 || nr >= NR_CPUS);
 154
 155         /* Verify that our partition has a processor nr */
 156         if (paca[nr].lppaca.xDynProcStatus >= 2)
 157                 return;
 158
 159         /* The processor is currently spinning, waiting
 160          * for the cpu_start field to become non-zero
 161          * After we set cpu_start, the processor will
 162          * continue on to secondary_start in iSeries_head.S
 163          */
 164         paca[nr].cpu_start = 1;
 165 }
 166
 167 static void __devinit smp_iSeries_setup_cpu(int nr)
 168 {
 169 }
 170
 171 static struct smp_ops_t iSeries_smp_ops = {
 172         .message_pass = smp_iSeries_message_pass,
 173         .probe        = smp_iSeries_probe,
 174         .kick_cpu     = smp_iSeries_kick_cpu,
 175         .setup_cpu    = smp_iSeries_setup_cpu,
 176 };
 177
 178 /* This is called very early. */
 179 void __init smp_init_iSeries(void)
 180 {
 181         smp_ops = &iSeries_smp_ops;
 182         systemcfg->processorCount       = smp_iSeries_numProcs();
 183 }
 184 #endif
 185
 186 #ifdef CONFIG_PPC_PSERIES
 187 void smp_openpic_message_pass(int target, int msg)
 188 {
 189         /* make sure we're sending something that translates to an IPI */
 190         if ( msg > 0x3 ){
 191                 printk("SMP %d: smp_message_pass: unknown msg %d\n",
 192                        smp_processor_id(), msg);
 193                 return;
 194         }
 195         switch ( target )
 196         {
 197         case MSG_ALL:
 198                 openpic_cause_IPI(msg, 0xffffffff);
 199                 break;
 200         case MSG_ALL_BUT_SELF:
 201                 openpic_cause_IPI(msg,
 202                                   0xffffffff & ~(1 << smp_processor_id()));
 203                 break;
 204         default:
 205                 openpic_cause_IPI(msg, 1<<target);
 206                 break;
 207         }
 208 }
 209
 210 static int __init smp_openpic_probe(void)
 211 {
 212         int nr_cpus;
 213
 214         nr_cpus = cpus_weight(cpu_possible_map);
 215
 216         if (nr_cpus > 1)
 217                 openpic_request_IPIs();
 218
 219         return nr_cpus;
 220 }
 221
 222 static void __devinit smp_openpic_setup_cpu(int cpu)
 223 {
 224         do_openpic_setup_cpu();
 225 }
 226
 227 #ifdef CONFIG_HOTPLUG_CPU
 228 /* Get state of physical CPU.
 229  * Return codes:
 230  *      0       - The processor is in the RTAS stopped state
 231  *      1       - stop-self is in progress
 232  *      2       - The processor is not in the RTAS stopped state
 233  *      -1      - Hardware Error
 234  *      -2      - Hardware Busy, Try again later.
 235  */
 236 static int query_cpu_stopped(unsigned int pcpu)
 237 {
 238         int cpu_status;
 239         int status, qcss_tok;
 240
 241         qcss_tok = rtas_token("query-cpu-stopped-state");
 242         BUG_ON(qcss_tok == RTAS_UNKNOWN_SERVICE);
 243         status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu);
 244         if (status != 0) {
 245                 printk(KERN_ERR
 246                        "RTAS query-cpu-stopped-state failed: %i\n", status);
 247                 return status;
 248         }
 249
 250         return cpu_status;
 251 }
 252
 253 int __cpu_disable(void)
 254 {
 255         /* FIXME: go put this in a header somewhere */
 256         extern void xics_migrate_irqs_away(void);
 257
 258         systemcfg->processorCount--;
 259
 260         /*fix boot_cpuid here*/
 261         if (smp_processor_id() == boot_cpuid)
 262                 boot_cpuid = any_online_cpu(cpu_online_map);
 263
 264         /* FIXME: abstract this to not be platform specific later on */
 265         xics_migrate_irqs_away();
 266         return 0;
 267 }
 268
 269 void __cpu_die(unsigned int cpu)
 270 {
 271         int tries;
 272         int cpu_status;
 273         unsigned int pcpu = get_hard_smp_processor_id(cpu);
 274
 275         for (tries = 0; tries < 5; tries++) {
 276                 cpu_status = query_cpu_stopped(pcpu);
 277
 278                 if (cpu_status == 0)
 279                         break;
 280                 set_current_state(TASK_UNINTERRUPTIBLE);
 281                 schedule_timeout(HZ);
 282         }
 283         if (cpu_status != 0) {
 284                 printk("Querying DEAD? cpu %i (%i) shows %i\n",
 285                        cpu, pcpu, cpu_status);
 286         }
 287
 288         /* Isolation and deallocation are definatly done by
 289          * drslot_chrp_cpu.  If they were not they would be
 290          * done here.  Change isolate state to Isolate and
 291          * change allocation-state to Unusable.
 292          */
 293         paca[cpu].cpu_start = 0;
 294
 295         /* So we can recognize if it fails to come up next time. */
 296         cpu_callin_map[cpu] = 0;
 297 }
 298
 299 /* Kill this cpu */
 300 void cpu_die(void)
 301 {
 302         local_irq_disable();
 303         rtas_stop_self();
 304         /* Should never get here... */
 305         BUG();
 306         for(;;);
 307 }
 308
 309 /* Search all cpu device nodes for an offline logical cpu.  If a
 310  * device node has a "ibm,my-drc-index" property (meaning this is an
 311  * LPAR), paranoid-check whether we own the cpu.  For each "thread"
 312  * of a cpu, if it is offline and has the same hw index as before,
 313  * grab that in preference.
 314  */
 315 static unsigned int find_physical_cpu_to_start(unsigned int old_hwindex)
 316 {
 317         struct device_node *np = NULL;
 318         unsigned int best = -1U;
 319
 320         while ((np = of_find_node_by_type(np, "cpu"))) {
 321                 int nr_threads, len;
 322                 u32 *index = (u32 *)get_property(np, "ibm,my-drc-index", NULL);
 323                 u32 *tid = (u32 *)
 324                         get_property(np, "ibm,ppc-interrupt-server#s", &len);
 325
 326                 if (!tid)
 327                         tid = (u32 *)get_property(np, "reg", &len);
 328
 329                 if (!tid)
 330                         continue;
 331
 332                 /* If there is a drc-index, make sure that we own
 333                  * the cpu.
 334                  */
 335                 if (index) {
 336                         int state;
 337                         int rc = rtas_get_sensor(9003, *index, &state);
 338                         if (rc != 0 || state != 1)
 339                                 continue;
 340                 }
 341
 342                 nr_threads = len / sizeof(u32);
 343
 344                 while (nr_threads--) {
 345                         if (0 == query_cpu_stopped(tid[nr_threads])) {
 346                                 best = tid[nr_threads];
 347                                 if (best == old_hwindex)
 348                                         goto out;
 349                         }
 350                 }
 351         }
 352 out:
 353         of_node_put(np);
 354         return best;
 355 }
 356
 357 /**
 358  * smp_startup_cpu() - start the given cpu
 359  *
 360  * At boot time, there is nothing to do.  At run-time, call RTAS with
 361  * the appropriate start location, if the cpu is in the RTAS stopped
 362  * state.
 363  *
 364  * Returns:
 365  *      0       - failure
 366  *      1       - success
 367  */
 368 static inline int __devinit smp_startup_cpu(unsigned int lcpu)
 369 {
 370         int status;
 371         extern void (*pseries_secondary_smp_init)(unsigned int cpu);
 372         unsigned long start_here = __pa(pseries_secondary_smp_init);
 373         unsigned int pcpu;
 374
 375         /* At boot time the cpus are already spinning in hold
 376          * loops, so nothing to do. */
 377         if (system_state == SYSTEM_BOOTING)
 378                 return 1;
 379
 380         pcpu = find_physical_cpu_to_start(get_hard_smp_processor_id(lcpu));
 381         if (pcpu == -1U) {
 382                 printk(KERN_INFO "No more cpus available, failing\n");
 383                 return 0;
 384         }
 385
 386         /* Fixup atomic count: it exited inside IRQ handler. */
 387         paca[lcpu].__current->thread_info->preempt_count        = 0;
 388         /* Fixup SLB round-robin so next segment (kernel) goes in segment 0 */
 389         paca[lcpu].stab_next_rr = 0;
 390
 391         /* At boot this is done in prom.c. */
 392         paca[lcpu].hw_cpu_id = pcpu;
 393
 394         status = rtas_call(rtas_token("start-cpu"), 3, 1, NULL,
 395                            pcpu, start_here, lcpu);
 396         if (status != 0) {
 397                 printk(KERN_ERR "start-cpu failed: %i\n", status);
 398                 return 0;
 399         }
 400         return 1;
 401 }
 402
 403 static inline void look_for_more_cpus(void)
 404 {
 405         int num_addr_cell, num_size_cell, len, i, maxcpus;
 406         struct device_node *np;
 407         unsigned int *ireg;
 408
 409         /* Find the property which will tell us about how many CPUs
 410          * we're allowed to have. */
 411         if ((np = find_path_device("/rtas")) == NULL) {
 412                 printk(KERN_ERR "Could not find /rtas in device tree!");
 413                 return;
 414         }
 415         num_addr_cell = prom_n_addr_cells(np);
 416         num_size_cell = prom_n_size_cells(np);
 417
 418         ireg = (unsigned int *)get_property(np, "ibm,lrdr-capacity", &len);
 419         if (ireg == NULL) {
 420                 /* FIXME: make sure not marked as lrdr_capable() */
 421                 return;
 422         }
 423
 424         maxcpus = ireg[num_addr_cell + num_size_cell];
 425         /* DRENG need to account for threads here too */
 426
 427         if (maxcpus > NR_CPUS) {
 428                 printk(KERN_WARNING
 429                        "Partition configured for %d cpus, "
 430                        "operating system maximum is %d.\n", maxcpus, NR_CPUS);
 431                 maxcpus = NR_CPUS;
 432         } else
 433                 printk(KERN_INFO "Partition configured for %d cpus.\n",
 434                        maxcpus);
 435
 436         /* Make those cpus (which might appear later) possible too. */
 437         for (i = 0; i < maxcpus; i++)
 438                 cpu_set(i, cpu_possible_map);
 439 }
 440 #else /* ... CONFIG_HOTPLUG_CPU */
 441 static inline int __devinit smp_startup_cpu(unsigned int lcpu)
 442 {
 443         return 1;
 444 }
 445 static inline void look_for_more_cpus(void)
 446 {
 447 }
 448 #endif /* CONFIG_HOTPLUG_CPU */
 449
 450 static void smp_pSeries_kick_cpu(int nr)
 451 {
 452         BUG_ON(nr < 0 || nr >= NR_CPUS);
 453
 454         if (!smp_startup_cpu(nr))
 455                 return;
 456
 457         /*
 458          * The processor is currently spinning, waiting for the
 459          * cpu_start field to become non-zero After we set cpu_start,
 460          * the processor will continue on to secondary_start
 461          */
 462         paca[nr].cpu_start = 1;
 463 }
 464 #endif /* CONFIG_PPC_PSERIES */
 465
 466 static void __init smp_space_timers(unsigned int max_cpus)
 467 {
 468         int i;
 469         unsigned long offset = tb_ticks_per_jiffy / max_cpus;
 470         unsigned long previous_tb = paca[boot_cpuid].next_jiffy_update_tb;
 471
 472         for_each_cpu(i) {
 473                 if (i != boot_cpuid) {
 474                         paca[i].next_jiffy_update_tb =
 475                                 previous_tb + offset;
 476                         previous_tb = paca[i].next_jiffy_update_tb;
 477                 }
 478         }
 479 }
 480
 481 #ifdef CONFIG_PPC_PSERIES
 482 void vpa_init(int cpu)
 483 {
 484         unsigned long flags;
 485
 486         /* Register the Virtual Processor Area (VPA) */
 487         flags = 1UL << (63 - 18);
 488         register_vpa(flags, cpu, __pa((unsigned long)&(paca[cpu].lppaca)));
 489 }
 490
 491 static inline void smp_xics_do_message(int cpu, int msg)
 492 {
 493         set_bit(msg, &xics_ipi_message[cpu].value);
 494         mb();
 495         xics_cause_IPI(cpu);
 496 }
 497
 498 static void smp_xics_message_pass(int target, int msg)
 499 {
 500         unsigned int i;
 501
 502         if (target < NR_CPUS) {
 503                 smp_xics_do_message(target, msg);
 504         } else {
 505                 for_each_online_cpu(i) {
 506                         if (target == MSG_ALL_BUT_SELF
 507                             && i == smp_processor_id())
 508                                 continue;
 509                         smp_xics_do_message(i, msg);
 510                 }
 511         }
 512 }
 513
 514 extern void xics_request_IPIs(void);
 515
 516 static int __init smp_xics_probe(void)
 517 {
 518 #ifdef CONFIG_SMP
 519         xics_request_IPIs();
 520 #endif
 521
 522         return cpus_weight(cpu_possible_map);
 523 }
 524
 525 static void __devinit smp_xics_setup_cpu(int cpu)
 526 {
 527         if (cpu != boot_cpuid)
 528                 xics_setup_cpu();
 529 }
 530
 531 static spinlock_t timebase_lock = SPIN_LOCK_UNLOCKED;
 532 static unsigned long timebase = 0;
 533
 534 static void __devinit pSeries_give_timebase(void)
 535 {
 536         spin_lock(&timebase_lock);
 537         rtas_call(rtas_token("freeze-time-base"), 0, 1, NULL);
 538         timebase = get_tb();
 539         spin_unlock(&timebase_lock);
 540
 541         while (timebase)
 542                 barrier();
 543         rtas_call(rtas_token("thaw-time-base"), 0, 1, NULL);
 544 }
 545
 546 static void __devinit pSeries_take_timebase(void)
 547 {
 548         while (!timebase)
 549                 barrier();
 550         spin_lock(&timebase_lock);
 551         set_tb(timebase >> 32, timebase & 0xffffffff);
 552         timebase = 0;
 553         spin_unlock(&timebase_lock);
 554 }
 555
 556 static struct smp_ops_t pSeries_openpic_smp_ops = {
 557         .message_pass   = smp_openpic_message_pass,
 558         .probe          = smp_openpic_probe,
 559         .kick_cpu       = smp_pSeries_kick_cpu,
 560         .setup_cpu      = smp_openpic_setup_cpu,
 561 };
 562
 563 static struct smp_ops_t pSeries_xics_smp_ops = {
 564         .message_pass   = smp_xics_message_pass,
 565         .probe          = smp_xics_probe,
 566         .kick_cpu       = smp_pSeries_kick_cpu,
 567         .setup_cpu      = smp_xics_setup_cpu,
 568 };
 569
 570 /* This is called very early */
 571 void __init smp_init_pSeries(void)
 572 {
 573
 574         if (naca->interrupt_controller == IC_OPEN_PIC)
 575                 smp_ops = &pSeries_openpic_smp_ops;
 576         else
 577                 smp_ops = &pSeries_xics_smp_ops;
 578
 579         /* Non-lpar has additional take/give timebase */
 580         if (systemcfg->platform == PLATFORM_PSERIES) {
 581                 smp_ops->give_timebase = pSeries_give_timebase;
 582                 smp_ops->take_timebase = pSeries_take_timebase;
 583         }
 584 }
 585 #endif
 586
 587 void smp_local_timer_interrupt(struct pt_regs * regs)
 588 {
 589         if (!--(get_paca()->prof_counter)) {
 590                 update_process_times(user_mode(regs));
 591                 (get_paca()->prof_counter)=get_paca()->prof_multiplier;
 592         }
 593 }
 594
 595 void smp_message_recv(int msg, struct pt_regs *regs)
 596 {
 597         switch(msg) {
 598         case PPC_MSG_CALL_FUNCTION:
 599                 smp_call_function_interrupt();
 600                 break;
 601         case PPC_MSG_RESCHEDULE:
 602                 /* XXX Do we have to do this? */
 603                 set_need_resched();
 604                 break;
 605 #if 0
 606         case PPC_MSG_MIGRATE_TASK:
 607                 /* spare */
 608                 break;
 609 #endif
 610 #ifdef CONFIG_DEBUGGER
 611         case PPC_MSG_DEBUGGER_BREAK:
 612                 debugger_ipi(regs);
 613                 break;
 614 #endif
 615         default:
 616                 printk("SMP %d: smp_message_recv(): unknown msg %d\n",
 617                        smp_processor_id(), msg);
 618                 break;
 619         }
 620 }
 621
 622 void smp_send_reschedule(int cpu)
 623 {
 624         smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
 625 }
 626
 627 #ifdef CONFIG_DEBUGGER
 628 void smp_send_debugger_break(int cpu)
 629 {
 630         smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
 631 }
 632 #endif
 633
 634 static void stop_this_cpu(void *dummy)
 635 {
 636         local_irq_disable();
 637         while (1)
 638                 ;
 639 }
 640
 641 void smp_send_stop(void)
 642 {
 643         smp_call_function(stop_this_cpu, NULL, 1, 0);
 644 }
 645
 646 /*
 647  * Structure and data for smp_call_function(). This is designed to minimise
 648  * static memory requirements. It also looks cleaner.
 649  * Stolen from the i386 version.
 650  */
 651 static spinlock_t call_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 652
 653 static struct call_data_struct {
 654         void (*func) (void *info);
 655         void *info;
 656         atomic_t started;
 657         atomic_t finished;
 658         int wait;
 659 } *call_data;
 660
 661 /* delay of at least 8 seconds on 1GHz cpu */
 662 #define SMP_CALL_TIMEOUT (1UL << (30 + 3))
 663
 664 /*
 665  * This function sends a 'generic call function' IPI to all other CPUs
 666  * in the system.
 667  *
 668  * [SUMMARY] Run a function on all other CPUs.
 669  * <func> The function to run. This must be fast and non-blocking.
 670  * <info> An arbitrary pointer to pass to the function.
 671  * <nonatomic> currently unused.
 672  * <wait> If true, wait (atomically) until function has completed on other CPUs.
 673  * [RETURNS] 0 on success, else a negative status code. Does not return until
 674  * remote CPUs are nearly ready to execute <<func>> or are or have executed.
 675  *
 676  * You must not call this function with disabled interrupts or from a
 677  * hardware interrupt handler or from a bottom half handler.
 678  */
 679 int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
 680                        int wait)
 681 {
 682         struct call_data_struct data;
 683         int ret = -1, cpus;
 684         unsigned long timeout;
 685
 686         /* Can deadlock when called with interrupts disabled */
 687         WARN_ON(irqs_disabled());
 688
 689         data.func = func;
 690         data.info = info;
 691         atomic_set(&data.started, 0);
 692         data.wait = wait;
 693         if (wait)
 694                 atomic_set(&data.finished, 0);
 695
 696         spin_lock(&call_lock);
 697         /* Must grab online cpu count with preempt disabled, otherwise
 698          * it can change. */
 699         cpus = num_online_cpus() - 1;
 700         if (!cpus) {
 701                 ret = 0;
 702                 goto out;
 703         }
 704
 705         call_data = &data;
 706         wmb();
 707         /* Send a message to all other CPUs and wait for them to respond */
 708         smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_CALL_FUNCTION);
 709
 710         /* Wait for response */
 711         timeout = SMP_CALL_TIMEOUT;
 712         while (atomic_read(&data.started) != cpus) {
 713                 HMT_low();
 714                 if (--timeout == 0) {
 715                         printk("smp_call_function on cpu %d: other cpus not "
 716                                "responding (%d)\n", smp_processor_id(),
 717                                atomic_read(&data.started));
 718                         debugger(0);
 719                         goto out;
 720                 }
 721         }
 722
 723         if (wait) {
 724                 timeout = SMP_CALL_TIMEOUT;
 725                 while (atomic_read(&data.finished) != cpus) {
 726                         HMT_low();
 727                         if (--timeout == 0) {
 728                                 printk("smp_call_function on cpu %d: other "
 729                                        "cpus not finishing (%d/%d)\n",
 730                                        smp_processor_id(),
 731                                        atomic_read(&data.finished),
 732                                        atomic_read(&data.started));
 733                                 debugger(0);
 734                                 goto out;
 735                         }
 736                 }
 737         }
 738
 739         ret = 0;
 740
 741 out:
 742         call_data = NULL;
 743         HMT_medium();
 744         spin_unlock(&call_lock);
 745         return ret;
 746 }
 747
 748 EXPORT_SYMBOL_GPL(smp_call_function);
 749
 750 void smp_call_function_interrupt(void)
 751 {
 752         void (*func) (void *info);
 753         void *info;
 754         int wait;
 755
 756         /* call_data will be NULL if the sender timed out while
 757          * waiting on us to receive the call.
 758          */
 759         if (!call_data)
 760                 return;
 761
 762         func = call_data->func;
 763         info = call_data->info;
 764         wait = call_data->wait;
 765
 766         if (!wait)
 767                 smp_mb__before_atomic_inc();
 768
 769         /*
 770          * Notify initiating CPU that I've grabbed the data and am
 771          * about to execute the function
 772          */
 773         atomic_inc(&call_data->started);
 774         /*
 775          * At this point the info structure may be out of scope unless wait==1
 776          */
 777         (*func)(info);
 778         if (wait) {
 779                 smp_mb__before_atomic_inc();
 780                 atomic_inc(&call_data->finished);
 781         }
 782 }
 783
 784 extern unsigned long decr_overclock;
 785 extern struct gettimeofday_struct do_gtod;
 786
 787 struct thread_info *current_set[NR_CPUS];
 788
 789 DECLARE_PER_CPU(unsigned int, pvr);
 790
 791 static void __devinit smp_store_cpu_info(int id)
 792 {
 793         per_cpu(pvr, id) = _get_PVR();
 794 }
 795
 796 static void __init smp_create_idle(unsigned int cpu)
 797 {
 798         struct pt_regs regs;
 799         struct task_struct *p;
 800
 801         /* create a process for the processor */
 802         /* only regs.msr is actually used, and 0 is OK for it */
 803         memset(&regs, 0, sizeof(struct pt_regs));
 804         p = copy_process(CLONE_VM | CLONE_IDLETASK,
 805                          0, &regs, 0, NULL, NULL);
 806         if (IS_ERR(p))
 807                 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
 808
 809         wake_up_forked_process(p);
 810         init_idle(p, cpu);
 811         unhash_process(p);
 812
 813         paca[cpu].__current = p;
 814         current_set[cpu] = p->thread_info;
 815 }
 816
 817 void __init smp_prepare_cpus(unsigned int max_cpus)
 818 {
 819         unsigned int cpu;
 820
 821         /*
 822          * setup_cpu may need to be called on the boot cpu. We havent
 823          * spun any cpus up but lets be paranoid.
 824          */
 825         BUG_ON(boot_cpuid != smp_processor_id());
 826
 827         /* Fixup boot cpu */
 828         smp_store_cpu_info(boot_cpuid);
 829         cpu_callin_map[boot_cpuid] = 1;
 830         paca[boot_cpuid].prof_counter = 1;
 831         paca[boot_cpuid].prof_multiplier = 1;
 832
 833 #ifndef CONFIG_PPC_ISERIES
 834         paca[boot_cpuid].next_jiffy_update_tb = tb_last_stamp = get_tb();
 835
 836         /*
 837          * Should update do_gtod.stamp_xsec.
 838          * For now we leave it which means the time can be some
 839          * number of msecs off until someone does a settimeofday()
 840          */
 841         do_gtod.tb_orig_stamp = tb_last_stamp;
 842
 843         look_for_more_cpus();
 844 #endif
 845
 846         max_cpus = smp_ops->probe();
 847
 848         /* Backup CPU 0 state if necessary */
 849         __save_cpu_setup();
 850
 851         smp_space_timers(max_cpus);
 852
 853         for_each_cpu(cpu)
 854                 if (cpu != boot_cpuid)
 855                         smp_create_idle(cpu);
 856 }
 857
 858 void __devinit smp_prepare_boot_cpu(void)
 859 {
 860         BUG_ON(smp_processor_id() != boot_cpuid);
 861
 862         /* cpu_possible is set up in prom.c */
 863         cpu_set(boot_cpuid, cpu_online_map);
 864
 865         paca[boot_cpuid].__current = current;
 866         current_set[boot_cpuid] = current->thread_info;
 867 }
 868
 869 int __devinit __cpu_up(unsigned int cpu)
 870 {
 871         int c;
 872
 873         /* At boot, don't bother with non-present cpus -JSCHOPP */
 874         if (system_state == SYSTEM_BOOTING && !cpu_present_at_boot(cpu))
 875                 return -ENOENT;
 876
 877         paca[cpu].prof_counter = 1;
 878         paca[cpu].prof_multiplier = 1;
 879         paca[cpu].default_decr = tb_ticks_per_jiffy / decr_overclock;
 880
 881         if (!(cur_cpu_spec->cpu_features & CPU_FTR_SLB)) {
 882                 void *tmp;
 883
 884                 /* maximum of 48 CPUs on machines with a segment table */
 885                 if (cpu >= 48)
 886                         BUG();
 887
 888                 tmp = &stab_array[PAGE_SIZE * cpu];
 889                 memset(tmp, 0, PAGE_SIZE);
 890                 paca[cpu].stab_addr = (unsigned long)tmp;
 891                 paca[cpu].stab_real = virt_to_abs(tmp);
 892         }
 893
 894         /* The information for processor bringup must
 895          * be written out to main store before we release
 896          * the processor.
 897          */
 898         mb();
 899
 900         /* wake up cpus */
 901         smp_ops->kick_cpu(cpu);
 902
 903         /*
 904          * wait to see if the cpu made a callin (is actually up).
 905          * use this value that I found through experimentation.
 906          * -- Cort
 907          */
 908         if (system_state == SYSTEM_BOOTING)
 909                 for (c = 5000; c && !cpu_callin_map[cpu]; c--)
 910                         udelay(100);
 911 #ifdef CONFIG_HOTPLUG_CPU
 912         else
 913                 /*
 914                  * CPUs can take much longer to come up in the
 915                  * hotplug case.  Wait five seconds.
 916                  */
 917                 for (c = 25; c && !cpu_callin_map[cpu]; c--) {
 918                         set_current_state(TASK_UNINTERRUPTIBLE);
 919                         schedule_timeout(HZ/5);
 920                 }
 921 #endif
 922
 923         if (!cpu_callin_map[cpu]) {
 924                 printk("Processor %u is stuck.\n", cpu);
 925                 return -ENOENT;
 926         }
 927
 928         printk("Processor %u found.\n", cpu);
 929
 930         if (smp_ops->give_timebase)
 931                 smp_ops->give_timebase();
 932         cpu_set(cpu, cpu_online_map);
 933         return 0;
 934 }
 935
 936 extern unsigned int default_distrib_server;
 937 /* Activate a secondary processor. */
 938 int __devinit start_secondary(void *unused)
 939 {
 940         unsigned int cpu = smp_processor_id();
 941
 942         atomic_inc(&init_mm.mm_count);
 943         current->active_mm = &init_mm;
 944
 945         smp_store_cpu_info(cpu);
 946         set_dec(paca[cpu].default_decr);
 947         cpu_callin_map[cpu] = 1;
 948
 949         smp_ops->setup_cpu(cpu);
 950         if (smp_ops->take_timebase)
 951                 smp_ops->take_timebase();
 952
 953 #ifdef CONFIG_PPC_PSERIES
 954         if (cur_cpu_spec->firmware_features & FW_FEATURE_SPLPAR) {
 955                 vpa_init(cpu);
 956         }
 957
 958 #ifdef CONFIG_IRQ_ALL_CPUS
 959         /* Put the calling processor into the GIQ.  This is really only
 960          * necessary from a secondary thread as the OF start-cpu interface
 961          * performs this function for us on primary threads.
 962          */
 963         /* TODO: 9005 is #defined in rtas-proc.c -- move to a header */
 964         rtas_set_indicator(9005, default_distrib_server, 1);
 965 #endif
 966 #endif
 967
 968         local_irq_enable();
 969
 970         return cpu_idle(NULL);
 971 }
 972
 973 int setup_profiling_timer(unsigned int multiplier)
 974 {
 975         return 0;
 976 }
 977
 978 void __init smp_cpus_done(unsigned int max_cpus)
 979 {
 980         cpumask_t old_mask;
 981
 982         /* We want the setup_cpu() here to be called from CPU 0, but our
 983          * init thread may have been "borrowed" by another CPU in the meantime
 984          * se we pin us down to CPU 0 for a short while
 985          */
 986         old_mask = current->cpus_allowed;
 987         set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid));
 988
 989         smp_ops->setup_cpu(boot_cpuid);
 990
 991         /* XXX fix this, xics currently relies on it - Anton */
 992         smp_threads_ready = 1;
 993
 994         set_cpus_allowed(current, old_mask);
 995 }
 996
 997 #ifdef CONFIG_SCHED_SMT
 998 #ifdef CONFIG_NUMA
 999 static struct sched_group sched_group_cpus[NR_CPUS];
1000 static struct sched_group sched_group_phys[NR_CPUS];
1001 static struct sched_group sched_group_nodes[MAX_NUMNODES];
1002 static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
1003 static DEFINE_PER_CPU(struct sched_domain, phys_domains);
1004 static DEFINE_PER_CPU(struct sched_domain, node_domains);
1005 __init void arch_init_sched_domains(void)
1006 {
1007         int i;
1008         struct sched_group *first = NULL, *last = NULL;
1009
1010         /* Set up domains */
1011         for_each_cpu(i) {
1012                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1013                 struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
1014                 struct sched_domain *node_domain = &per_cpu(node_domains, i);
1015                 int node = cpu_to_node(i);
1016                 cpumask_t nodemask = node_to_cpumask(node);
1017                 cpumask_t my_cpumask = cpumask_of_cpu(i);
1018                 cpumask_t sibling_cpumask = cpumask_of_cpu(i ^ 0x1);
1019
1020                 *cpu_domain = SD_SIBLING_INIT;
1021                 if (cur_cpu_spec->cpu_features & CPU_FTR_SMT)
1022                         cpus_or(cpu_domain->span, my_cpumask, sibling_cpumask);
1023                 else
1024                         cpu_domain->span = my_cpumask;
1025                 cpu_domain->parent = phys_domain;
1026                 cpu_domain->groups = &sched_group_cpus[i];
1027
1028                 *phys_domain = SD_CPU_INIT;
1029                 phys_domain->span = nodemask;
1030                 phys_domain->parent = node_domain;
1031                 phys_domain->groups = &sched_group_phys[first_cpu(cpu_domain->span)];
1032
1033                 *node_domain = SD_NODE_INIT;
1034                 node_domain->span = cpu_possible_map;
1035                 node_domain->groups = &sched_group_nodes[node];
1036         }
1037
1038         /* Set up CPU (sibling) groups */
1039         for_each_cpu(i) {
1040                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1041                 int j;
1042                 first = last = NULL;
1043
1044                 if (i != first_cpu(cpu_domain->span))
1045                         continue;
1046
1047                 for_each_cpu_mask(j, cpu_domain->span) {
1048                         struct sched_group *cpu = &sched_group_cpus[j];
1049
1050                         cpus_clear(cpu->cpumask);
1051                         cpu_set(j, cpu->cpumask);
1052                         cpu->cpu_power = SCHED_LOAD_SCALE;
1053
1054                         if (!first)
1055                                 first = cpu;
1056                         if (last)
1057                                 last->next = cpu;
1058                         last = cpu;
1059                 }
1060                 last->next = first;
1061         }
1062
1063         for (i = 0; i < MAX_NUMNODES; i++) {
1064                 int j;
1065                 cpumask_t nodemask;
1066                 struct sched_group *node = &sched_group_nodes[i];
1067                 cpumask_t node_cpumask = node_to_cpumask(i);
1068                 cpus_and(nodemask, node_cpumask, cpu_possible_map);
1069
1070                 if (cpus_empty(nodemask))
1071                         continue;
1072
1073                 first = last = NULL;
1074                 /* Set up physical groups */
1075                 for_each_cpu_mask(j, nodemask) {
1076                         struct sched_domain *cpu_domain = &per_cpu(cpu_domains, j);
1077                         struct sched_group *cpu = &sched_group_phys[j];
1078
1079                         if (j != first_cpu(cpu_domain->span))
1080                                 continue;
1081
1082                         cpu->cpumask = cpu_domain->span;
1083                         /*
1084                          * Make each extra sibling increase power by 10% of
1085                          * the basic CPU. This is very arbitrary.
1086                          */
1087                         cpu->cpu_power = SCHED_LOAD_SCALE + SCHED_LOAD_SCALE*(cpus_weight(cpu->cpumask)-1) / 10;
1088                         node->cpu_power += cpu->cpu_power;
1089
1090                         if (!first)
1091                                 first = cpu;
1092                         if (last)
1093                                 last->next = cpu;
1094                         last = cpu;
1095                 }
1096                 last->next = first;
1097         }
1098
1099         /* Set up nodes */
1100         first = last = NULL;
1101         for (i = 0; i < MAX_NUMNODES; i++) {
1102                 struct sched_group *cpu = &sched_group_nodes[i];
1103                 cpumask_t nodemask;
1104                 cpumask_t node_cpumask = node_to_cpumask(i);
1105                 cpus_and(nodemask, node_cpumask, cpu_possible_map);
1106
1107                 if (cpus_empty(nodemask))
1108                         continue;
1109
1110                 cpu->cpumask = nodemask;
1111                 /* ->cpu_power already setup */
1112
1113                 if (!first)
1114                         first = cpu;
1115                 if (last)
1116                         last->next = cpu;
1117                 last = cpu;
1118         }
1119         last->next = first;
1120
1121         mb();
1122         for_each_cpu(i) {
1123                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1124                 cpu_attach_domain(cpu_domain, i);
1125         }
1126 }
1127 #else /* !CONFIG_NUMA */
1128 static struct sched_group sched_group_cpus[NR_CPUS];
1129 static struct sched_group sched_group_phys[NR_CPUS];
1130 static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
1131 static DEFINE_PER_CPU(struct sched_domain, phys_domains);
1132 __init void arch_init_sched_domains(void)
1133 {
1134         int i;
1135         struct sched_group *first = NULL, *last = NULL;
1136
1137         /* Set up domains */
1138         for_each_cpu(i) {
1139                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1140                 struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
1141                 cpumask_t my_cpumask = cpumask_of_cpu(i);
1142                 cpumask_t sibling_cpumask = cpumask_of_cpu(i ^ 0x1);
1143
1144                 *cpu_domain = SD_SIBLING_INIT;
1145                 if (cur_cpu_spec->cpu_features & CPU_FTR_SMT)
1146                         cpus_or(cpu_domain->span, my_cpumask, sibling_cpumask);
1147                 else
1148                         cpu_domain->span = my_cpumask;
1149                 cpu_domain->parent = phys_domain;
1150                 cpu_domain->groups = &sched_group_cpus[i];
1151
1152                 *phys_domain = SD_CPU_INIT;
1153                 phys_domain->span = cpu_possible_map;
1154                 phys_domain->groups = &sched_group_phys[first_cpu(cpu_domain->span)];
1155         }
1156
1157         /* Set up CPU (sibling) groups */
1158         for_each_cpu(i) {
1159                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1160                 int j;
1161                 first = last = NULL;
1162
1163                 if (i != first_cpu(cpu_domain->span))
1164                         continue;
1165
1166                 for_each_cpu_mask(j, cpu_domain->span) {
1167                         struct sched_group *cpu = &sched_group_cpus[j];
1168
1169                         cpus_clear(cpu->cpumask);
1170                         cpu_set(j, cpu->cpumask);
1171                         cpu->cpu_power = SCHED_LOAD_SCALE;
1172
1173                         if (!first)
1174                                 first = cpu;
1175                         if (last)
1176                                 last->next = cpu;
1177                         last = cpu;
1178                 }
1179                 last->next = first;
1180         }
1181
1182         first = last = NULL;
1183         /* Set up physical groups */
1184         for_each_cpu(i) {
1185                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1186                 struct sched_group *cpu = &sched_group_phys[i];
1187
1188                 if (i != first_cpu(cpu_domain->span))
1189                         continue;
1190
1191                 cpu->cpumask = cpu_domain->span;
1192                 /* See SMT+NUMA setup for comment */
1193                 cpu->cpu_power = SCHED_LOAD_SCALE + SCHED_LOAD_SCALE*(cpus_weight(cpu->cpumask)-1) / 10;
1194
1195                 if (!first)
1196                         first = cpu;
1197                 if (last)
1198                         last->next = cpu;
1199                 last = cpu;
1200         }
1201         last->next = first;
1202
1203         mb();
1204         for_each_cpu(i) {
1205                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1206                 cpu_attach_domain(cpu_domain, i);
1207         }
1208 }
1209 #endif /* CONFIG_NUMA */
1210 #endif /* CONFIG_SCHED_SMT */