arch/ppc64/kernel/smp.c

   1 /*
   2  * SMP support for ppc.
   3  *
   4  * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
   5  * deal of code from the sparc and intel versions.
   6  *
   7  * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
   8  *
   9  * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
  10  * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
  11  *
  12  *      This program is free software; you can redistribute it and/or
  13  *      modify it under the terms of the GNU General Public License
  14  *      as published by the Free Software Foundation; either version
  15  *      2 of the License, or (at your option) any later version.
  16  */
  17
  18 #include <linux/config.h>
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/sched.h>
  22 #include <linux/smp.h>
  23 #include <linux/smp_lock.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/delay.h>
  27 #include <linux/init.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/cache.h>
  30 #include <linux/err.h>
  31 #include <linux/sysdev.h>
  32 #include <linux/cpu.h>
  33
  34 #include <asm/ptrace.h>
  35 #include <asm/atomic.h>
  36 #include <asm/irq.h>
  37 #include <asm/page.h>
  38 #include <asm/pgtable.h>
  39 #include <asm/hardirq.h>
  40 #include <asm/io.h>
  41 #include <asm/prom.h>
  42 #include <asm/smp.h>
  43 #include <asm/naca.h>
  44 #include <asm/paca.h>
  45 #include <asm/iSeries/LparData.h>
  46 #include <asm/iSeries/HvCall.h>
  47 #include <asm/iSeries/HvCallCfg.h>
  48 #include <asm/time.h>
  49 #include <asm/ppcdebug.h>
  50 #include "open_pic.h"
  51 #include <asm/machdep.h>
  52 #include <asm/xics.h>
  53 #include <asm/cputable.h>
  54 #include <asm/system.h>
  55 #include <asm/rtas.h>
  56
  57 int smp_threads_ready;
  58 unsigned long cache_decay_ticks;
  59
  60 cpumask_t cpu_possible_map = CPU_MASK_NONE;
  61 cpumask_t cpu_online_map = CPU_MASK_NONE;
  62 cpumask_t cpu_available_map = CPU_MASK_NONE;
  63 cpumask_t cpu_present_at_boot = CPU_MASK_NONE;
  64
  65 EXPORT_SYMBOL(cpu_online_map);
  66 EXPORT_SYMBOL(cpu_possible_map);
  67
  68 struct smp_ops_t *smp_ops;
  69
  70 static volatile unsigned int cpu_callin_map[NR_CPUS];
  71
  72 extern unsigned char stab_array[];
  73
  74 extern int cpu_idle(void *unused);
  75 void smp_call_function_interrupt(void);
  76 extern long register_vpa(unsigned long flags, unsigned long proc,
  77                          unsigned long vpa);
  78
  79 /* Low level assembly function used to backup CPU 0 state */
  80 extern void __save_cpu_setup(void);
  81
  82 #ifdef CONFIG_PPC_ISERIES
  83 static unsigned long iSeries_smp_message[NR_CPUS];
  84
  85 void iSeries_smp_message_recv( struct pt_regs * regs )
  86 {
  87         int cpu = smp_processor_id();
  88         int msg;
  89
  90         if ( num_online_cpus() < 2 )
  91                 return;
  92
  93         for ( msg = 0; msg < 4; ++msg )
  94                 if ( test_and_clear_bit( msg, &iSeries_smp_message[cpu] ) )
  95                         smp_message_recv( msg, regs );
  96 }
  97
  98 static inline void smp_iSeries_do_message(int cpu, int msg)
  99 {
 100         set_bit(msg, &iSeries_smp_message[cpu]);
 101         HvCall_sendIPI(&(paca[cpu]));
 102 }
 103
 104 static void smp_iSeries_message_pass(int target, int msg)
 105 {
 106         int i;
 107
 108         if (target < NR_CPUS)
 109                 smp_iSeries_do_message(target, msg);
 110         else {
 111                 for_each_online_cpu(i) {
 112                         if (target == MSG_ALL_BUT_SELF
 113                             && i == smp_processor_id())
 114                                 continue;
 115                         smp_iSeries_do_message(i, msg);
 116                 }
 117         }
 118 }
 119
 120 static int smp_iSeries_numProcs(void)
 121 {
 122         unsigned np, i;
 123
 124         np = 0;
 125         for (i=0; i < NR_CPUS; ++i) {
 126                 if (paca[i].lppaca.xDynProcStatus < 2) {
 127                         cpu_set(i, cpu_available_map);
 128                         cpu_set(i, cpu_possible_map);
 129                         cpu_set(i, cpu_present_at_boot);
 130                         ++np;
 131                 }
 132         }
 133         return np;
 134 }
 135
 136 static int smp_iSeries_probe(void)
 137 {
 138         unsigned i;
 139         unsigned np = 0;
 140
 141         for (i=0; i < NR_CPUS; ++i) {
 142                 if (paca[i].lppaca.xDynProcStatus < 2) {
 143                         /*paca[i].active = 1;*/
 144                         ++np;
 145                 }
 146         }
 147
 148         return np;
 149 }
 150
 151 static void smp_iSeries_kick_cpu(int nr)
 152 {
 153         BUG_ON(nr < 0 || nr >= NR_CPUS);
 154
 155         /* Verify that our partition has a processor nr */
 156         if (paca[nr].lppaca.xDynProcStatus >= 2)
 157                 return;
 158
 159         /* The processor is currently spinning, waiting
 160          * for the cpu_start field to become non-zero
 161          * After we set cpu_start, the processor will
 162          * continue on to secondary_start in iSeries_head.S
 163          */
 164         paca[nr].cpu_start = 1;
 165 }
 166
 167 static void __devinit smp_iSeries_setup_cpu(int nr)
 168 {
 169 }
 170
 171 static struct smp_ops_t iSeries_smp_ops = {
 172         .message_pass = smp_iSeries_message_pass,
 173         .probe        = smp_iSeries_probe,
 174         .kick_cpu     = smp_iSeries_kick_cpu,
 175         .setup_cpu    = smp_iSeries_setup_cpu,
 176 };
 177
 178 /* This is called very early. */
 179 void __init smp_init_iSeries(void)
 180 {
 181         smp_ops = &iSeries_smp_ops;
 182         systemcfg->processorCount       = smp_iSeries_numProcs();
 183 }
 184 #endif
 185
 186 #ifdef CONFIG_PPC_PSERIES
 187 void smp_openpic_message_pass(int target, int msg)
 188 {
 189         /* make sure we're sending something that translates to an IPI */
 190         if ( msg > 0x3 ){
 191                 printk("SMP %d: smp_message_pass: unknown msg %d\n",
 192                        smp_processor_id(), msg);
 193                 return;
 194         }
 195         switch ( target )
 196         {
 197         case MSG_ALL:
 198                 openpic_cause_IPI(msg, 0xffffffff);
 199                 break;
 200         case MSG_ALL_BUT_SELF:
 201                 openpic_cause_IPI(msg,
 202                                   0xffffffff & ~(1 << smp_processor_id()));
 203                 break;
 204         default:
 205                 openpic_cause_IPI(msg, 1<<target);
 206                 break;
 207         }
 208 }
 209
 210 static int __init smp_openpic_probe(void)
 211 {
 212         int nr_cpus;
 213
 214         nr_cpus = cpus_weight(cpu_possible_map);
 215
 216         if (nr_cpus > 1)
 217                 openpic_request_IPIs();
 218
 219         return nr_cpus;
 220 }
 221
 222 static void __devinit smp_openpic_setup_cpu(int cpu)
 223 {
 224         do_openpic_setup_cpu();
 225 }
 226
 227 #ifdef CONFIG_HOTPLUG_CPU
 228 /* Get state of physical CPU.
 229  * Return codes:
 230  *      0       - The processor is in the RTAS stopped state
 231  *      1       - stop-self is in progress
 232  *      2       - The processor is not in the RTAS stopped state
 233  *      -1      - Hardware Error
 234  *      -2      - Hardware Busy, Try again later.
 235  */
 236 static int query_cpu_stopped(unsigned int pcpu)
 237 {
 238         int cpu_status;
 239         int status, qcss_tok;
 240
 241         qcss_tok = rtas_token("query-cpu-stopped-state");
 242         BUG_ON(qcss_tok == RTAS_UNKNOWN_SERVICE);
 243         status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu);
 244         if (status != 0) {
 245                 printk(KERN_ERR
 246                        "RTAS query-cpu-stopped-state failed: %i\n", status);
 247                 return status;
 248         }
 249
 250         return cpu_status;
 251 }
 252
 253 int __cpu_disable(void)
 254 {
 255         /* FIXME: go put this in a header somewhere */
 256         extern void xics_migrate_irqs_away(void);
 257
 258         systemcfg->processorCount--;
 259
 260         /*fix boot_cpuid here*/
 261         if (smp_processor_id() == boot_cpuid)
 262                 boot_cpuid = any_online_cpu(cpu_online_map);
 263
 264         /* FIXME: abstract this to not be platform specific later on */
 265         xics_migrate_irqs_away();
 266         return 0;
 267 }
 268
 269 void __cpu_die(unsigned int cpu)
 270 {
 271         int tries;
 272         int cpu_status;
 273         unsigned int pcpu = get_hard_smp_processor_id(cpu);
 274
 275         for (tries = 0; tries < 5; tries++) {
 276                 cpu_status = query_cpu_stopped(pcpu);
 277
 278                 if (cpu_status == 0)
 279                         break;
 280                 set_current_state(TASK_UNINTERRUPTIBLE);
 281                 schedule_timeout(HZ);
 282         }
 283         if (cpu_status != 0) {
 284                 printk("Querying DEAD? cpu %i (%i) shows %i\n",
 285                        cpu, pcpu, cpu_status);
 286         }
 287
 288         /* Isolation and deallocation are definatly done by
 289          * drslot_chrp_cpu.  If they were not they would be
 290          * done here.  Change isolate state to Isolate and
 291          * change allocation-state to Unusable.
 292          */
 293         paca[cpu].cpu_start = 0;
 294
 295         /* So we can recognize if it fails to come up next time. */
 296         cpu_callin_map[cpu] = 0;
 297 }
 298
 299 /* Kill this cpu */
 300 void cpu_die(void)
 301 {
 302         local_irq_disable();
 303         /* Some hardware requires clearing the CPPR, while other hardware does not
 304          * it is safe either way
 305          */
 306         pSeriesLP_cppr_info(0, 0);
 307         rtas_stop_self();
 308         /* Should never get here... */
 309         BUG();
 310         for(;;);
 311 }
 312
 313 /* Search all cpu device nodes for an offline logical cpu.  If a
 314  * device node has a "ibm,my-drc-index" property (meaning this is an
 315  * LPAR), paranoid-check whether we own the cpu.  For each "thread"
 316  * of a cpu, if it is offline and has the same hw index as before,
 317  * grab that in preference.
 318  */
 319 static unsigned int find_physical_cpu_to_start(unsigned int old_hwindex)
 320 {
 321         struct device_node *np = NULL;
 322         unsigned int best = -1U;
 323
 324         while ((np = of_find_node_by_type(np, "cpu"))) {
 325                 int nr_threads, len;
 326                 u32 *index = (u32 *)get_property(np, "ibm,my-drc-index", NULL);
 327                 u32 *tid = (u32 *)
 328                         get_property(np, "ibm,ppc-interrupt-server#s", &len);
 329
 330                 if (!tid)
 331                         tid = (u32 *)get_property(np, "reg", &len);
 332
 333                 if (!tid)
 334                         continue;
 335
 336                 /* If there is a drc-index, make sure that we own
 337                  * the cpu.
 338                  */
 339                 if (index) {
 340                         int state;
 341                         int rc = rtas_get_sensor(9003, *index, &state);
 342                         if (rc != 0 || state != 1)
 343                                 continue;
 344                 }
 345
 346                 nr_threads = len / sizeof(u32);
 347
 348                 while (nr_threads--) {
 349                         if (0 == query_cpu_stopped(tid[nr_threads])) {
 350                                 best = tid[nr_threads];
 351                                 if (best == old_hwindex)
 352                                         goto out;
 353                         }
 354                 }
 355         }
 356 out:
 357         of_node_put(np);
 358         return best;
 359 }
 360
 361 /**
 362  * smp_startup_cpu() - start the given cpu
 363  *
 364  * At boot time, there is nothing to do.  At run-time, call RTAS with
 365  * the appropriate start location, if the cpu is in the RTAS stopped
 366  * state.
 367  *
 368  * Returns:
 369  *      0       - failure
 370  *      1       - success
 371  */
 372 static inline int __devinit smp_startup_cpu(unsigned int lcpu)
 373 {
 374         int status;
 375         extern void (*pseries_secondary_smp_init)(unsigned int cpu);
 376         unsigned long start_here = __pa(pseries_secondary_smp_init);
 377         unsigned int pcpu;
 378
 379         /* At boot time the cpus are already spinning in hold
 380          * loops, so nothing to do. */
 381         if (system_state == SYSTEM_BOOTING)
 382                 return 1;
 383
 384         pcpu = find_physical_cpu_to_start(get_hard_smp_processor_id(lcpu));
 385         if (pcpu == -1U) {
 386                 printk(KERN_INFO "No more cpus available, failing\n");
 387                 return 0;
 388         }
 389
 390         /* Fixup atomic count: it exited inside IRQ handler. */
 391         paca[lcpu].__current->thread_info->preempt_count        = 0;
 392
 393         /* At boot this is done in prom.c. */
 394         paca[lcpu].hw_cpu_id = pcpu;
 395
 396         status = rtas_call(rtas_token("start-cpu"), 3, 1, NULL,
 397                            pcpu, start_here, lcpu);
 398         if (status != 0) {
 399                 printk(KERN_ERR "start-cpu failed: %i\n", status);
 400                 return 0;
 401         }
 402         return 1;
 403 }
 404
 405 static inline void look_for_more_cpus(void)
 406 {
 407         int num_addr_cell, num_size_cell, len, i, maxcpus;
 408         struct device_node *np;
 409         unsigned int *ireg;
 410
 411         /* Find the property which will tell us about how many CPUs
 412          * we're allowed to have. */
 413         if ((np = find_path_device("/rtas")) == NULL) {
 414                 printk(KERN_ERR "Could not find /rtas in device tree!");
 415                 return;
 416         }
 417         num_addr_cell = prom_n_addr_cells(np);
 418         num_size_cell = prom_n_size_cells(np);
 419
 420         ireg = (unsigned int *)get_property(np, "ibm,lrdr-capacity", &len);
 421         if (ireg == NULL) {
 422                 /* FIXME: make sure not marked as lrdr_capable() */
 423                 return;
 424         }
 425
 426         maxcpus = ireg[num_addr_cell + num_size_cell];
 427
 428         /* Double maxcpus for processors which have SMT capability */
 429         if (cur_cpu_spec->cpu_features & CPU_FTR_SMT)
 430                 maxcpus *= 2;
 431
 432
 433         if (maxcpus > NR_CPUS) {
 434                 printk(KERN_WARNING
 435                        "Partition configured for %d cpus, "
 436                        "operating system maximum is %d.\n", maxcpus, NR_CPUS);
 437                 maxcpus = NR_CPUS;
 438         } else
 439                 printk(KERN_INFO "Partition configured for %d cpus.\n",
 440                        maxcpus);
 441
 442         /* Make those cpus (which might appear later) possible too. */
 443         for (i = 0; i < maxcpus; i++)
 444                 cpu_set(i, cpu_possible_map);
 445 }
 446 #else /* ... CONFIG_HOTPLUG_CPU */
 447 static inline int __devinit smp_startup_cpu(unsigned int lcpu)
 448 {
 449         return 1;
 450 }
 451 static inline void look_for_more_cpus(void)
 452 {
 453 }
 454 #endif /* CONFIG_HOTPLUG_CPU */
 455
 456 static void smp_pSeries_kick_cpu(int nr)
 457 {
 458         BUG_ON(nr < 0 || nr >= NR_CPUS);
 459
 460         if (!smp_startup_cpu(nr))
 461                 return;
 462
 463         /*
 464          * The processor is currently spinning, waiting for the
 465          * cpu_start field to become non-zero After we set cpu_start,
 466          * the processor will continue on to secondary_start
 467          */
 468         paca[nr].cpu_start = 1;
 469 }
 470 #endif /* CONFIG_PPC_PSERIES */
 471
 472 static void __init smp_space_timers(unsigned int max_cpus)
 473 {
 474         int i;
 475         unsigned long offset = tb_ticks_per_jiffy / max_cpus;
 476         unsigned long previous_tb = paca[boot_cpuid].next_jiffy_update_tb;
 477
 478         for_each_cpu(i) {
 479                 if (i != boot_cpuid) {
 480                         paca[i].next_jiffy_update_tb =
 481                                 previous_tb + offset;
 482                         previous_tb = paca[i].next_jiffy_update_tb;
 483                 }
 484         }
 485 }
 486
 487 #ifdef CONFIG_PPC_PSERIES
 488 void vpa_init(int cpu)
 489 {
 490         unsigned long flags;
 491
 492         /* Register the Virtual Processor Area (VPA) */
 493         flags = 1UL << (63 - 18);
 494         register_vpa(flags, cpu, __pa((unsigned long)&(paca[cpu].lppaca)));
 495 }
 496
 497 static inline void smp_xics_do_message(int cpu, int msg)
 498 {
 499         set_bit(msg, &xics_ipi_message[cpu].value);
 500         mb();
 501         xics_cause_IPI(cpu);
 502 }
 503
 504 static void smp_xics_message_pass(int target, int msg)
 505 {
 506         unsigned int i;
 507
 508         if (target < NR_CPUS) {
 509                 smp_xics_do_message(target, msg);
 510         } else {
 511                 for_each_online_cpu(i) {
 512                         if (target == MSG_ALL_BUT_SELF
 513                             && i == smp_processor_id())
 514                                 continue;
 515                         smp_xics_do_message(i, msg);
 516                 }
 517         }
 518 }
 519
 520 extern void xics_request_IPIs(void);
 521
 522 static int __init smp_xics_probe(void)
 523 {
 524 #ifdef CONFIG_SMP
 525         xics_request_IPIs();
 526 #endif
 527
 528         return cpus_weight(cpu_possible_map);
 529 }
 530
 531 static void __devinit smp_xics_setup_cpu(int cpu)
 532 {
 533         if (cpu != boot_cpuid)
 534                 xics_setup_cpu();
 535 }
 536
 537 static spinlock_t timebase_lock = SPIN_LOCK_UNLOCKED;
 538 static unsigned long timebase = 0;
 539
 540 static void __devinit pSeries_give_timebase(void)
 541 {
 542         spin_lock(&timebase_lock);
 543         rtas_call(rtas_token("freeze-time-base"), 0, 1, NULL);
 544         timebase = get_tb();
 545         spin_unlock(&timebase_lock);
 546
 547         while (timebase)
 548                 barrier();
 549         rtas_call(rtas_token("thaw-time-base"), 0, 1, NULL);
 550 }
 551
 552 static void __devinit pSeries_take_timebase(void)
 553 {
 554         while (!timebase)
 555                 barrier();
 556         spin_lock(&timebase_lock);
 557         set_tb(timebase >> 32, timebase & 0xffffffff);
 558         timebase = 0;
 559         spin_unlock(&timebase_lock);
 560 }
 561
 562 static struct smp_ops_t pSeries_openpic_smp_ops = {
 563         .message_pass   = smp_openpic_message_pass,
 564         .probe          = smp_openpic_probe,
 565         .kick_cpu       = smp_pSeries_kick_cpu,
 566         .setup_cpu      = smp_openpic_setup_cpu,
 567 };
 568
 569 static struct smp_ops_t pSeries_xics_smp_ops = {
 570         .message_pass   = smp_xics_message_pass,
 571         .probe          = smp_xics_probe,
 572         .kick_cpu       = smp_pSeries_kick_cpu,
 573         .setup_cpu      = smp_xics_setup_cpu,
 574 };
 575
 576 /* This is called very early */
 577 void __init smp_init_pSeries(void)
 578 {
 579
 580         if (naca->interrupt_controller == IC_OPEN_PIC)
 581                 smp_ops = &pSeries_openpic_smp_ops;
 582         else
 583                 smp_ops = &pSeries_xics_smp_ops;
 584
 585         /* Non-lpar has additional take/give timebase */
 586         if (systemcfg->platform == PLATFORM_PSERIES) {
 587                 smp_ops->give_timebase = pSeries_give_timebase;
 588                 smp_ops->take_timebase = pSeries_take_timebase;
 589         }
 590 }
 591 #endif
 592
 593 void smp_local_timer_interrupt(struct pt_regs * regs)
 594 {
 595         if (!--(get_paca()->prof_counter)) {
 596                 update_process_times(user_mode(regs));
 597                 (get_paca()->prof_counter)=get_paca()->prof_multiplier;
 598         }
 599 }
 600
 601 void smp_message_recv(int msg, struct pt_regs *regs)
 602 {
 603         switch(msg) {
 604         case PPC_MSG_CALL_FUNCTION:
 605                 smp_call_function_interrupt();
 606                 break;
 607         case PPC_MSG_RESCHEDULE:
 608                 /* XXX Do we have to do this? */
 609                 set_need_resched();
 610                 break;
 611 #if 0
 612         case PPC_MSG_MIGRATE_TASK:
 613                 /* spare */
 614                 break;
 615 #endif
 616 #ifdef CONFIG_DEBUGGER
 617         case PPC_MSG_DEBUGGER_BREAK:
 618                 debugger_ipi(regs);
 619                 break;
 620 #endif
 621         default:
 622                 printk("SMP %d: smp_message_recv(): unknown msg %d\n",
 623                        smp_processor_id(), msg);
 624                 break;
 625         }
 626 }
 627
 628 void smp_send_reschedule(int cpu)
 629 {
 630         smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
 631 }
 632
 633 #ifdef CONFIG_DEBUGGER
 634 void smp_send_debugger_break(int cpu)
 635 {
 636         smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
 637 }
 638 #endif
 639
 640 static void stop_this_cpu(void *dummy)
 641 {
 642         local_irq_disable();
 643         while (1)
 644                 ;
 645 }
 646
 647 void smp_send_stop(void)
 648 {
 649         smp_call_function(stop_this_cpu, NULL, 1, 0);
 650 }
 651
 652 /*
 653  * Structure and data for smp_call_function(). This is designed to minimise
 654  * static memory requirements. It also looks cleaner.
 655  * Stolen from the i386 version.
 656  */
 657 static spinlock_t call_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 658
 659 static struct call_data_struct {
 660         void (*func) (void *info);
 661         void *info;
 662         atomic_t started;
 663         atomic_t finished;
 664         int wait;
 665 } *call_data;
 666
 667 /* delay of at least 8 seconds on 1GHz cpu */
 668 #define SMP_CALL_TIMEOUT (1UL << (30 + 3))
 669
 670 /*
 671  * This function sends a 'generic call function' IPI to all other CPUs
 672  * in the system.
 673  *
 674  * [SUMMARY] Run a function on all other CPUs.
 675  * <func> The function to run. This must be fast and non-blocking.
 676  * <info> An arbitrary pointer to pass to the function.
 677  * <nonatomic> currently unused.
 678  * <wait> If true, wait (atomically) until function has completed on other CPUs.
 679  * [RETURNS] 0 on success, else a negative status code. Does not return until
 680  * remote CPUs are nearly ready to execute <<func>> or are or have executed.
 681  *
 682  * You must not call this function with disabled interrupts or from a
 683  * hardware interrupt handler or from a bottom half handler.
 684  */
 685 int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
 686                        int wait)
 687 {
 688         struct call_data_struct data;
 689         int ret = -1, cpus;
 690         unsigned long timeout;
 691
 692         /* Can deadlock when called with interrupts disabled */
 693         WARN_ON(irqs_disabled());
 694
 695         data.func = func;
 696         data.info = info;
 697         atomic_set(&data.started, 0);
 698         data.wait = wait;
 699         if (wait)
 700                 atomic_set(&data.finished, 0);
 701
 702         spin_lock(&call_lock);
 703         /* Must grab online cpu count with preempt disabled, otherwise
 704          * it can change. */
 705         cpus = num_online_cpus() - 1;
 706         if (!cpus) {
 707                 ret = 0;
 708                 goto out;
 709         }
 710
 711         call_data = &data;
 712         wmb();
 713         /* Send a message to all other CPUs and wait for them to respond */
 714         smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_CALL_FUNCTION);
 715
 716         /* Wait for response */
 717         timeout = SMP_CALL_TIMEOUT;
 718         while (atomic_read(&data.started) != cpus) {
 719                 HMT_low();
 720                 if (--timeout == 0) {
 721                         printk("smp_call_function on cpu %d: other cpus not "
 722                                "responding (%d)\n", smp_processor_id(),
 723                                atomic_read(&data.started));
 724                         debugger(NULL);
 725                         goto out;
 726                 }
 727         }
 728
 729         if (wait) {
 730                 timeout = SMP_CALL_TIMEOUT;
 731                 while (atomic_read(&data.finished) != cpus) {
 732                         HMT_low();
 733                         if (--timeout == 0) {
 734                                 printk("smp_call_function on cpu %d: other "
 735                                        "cpus not finishing (%d/%d)\n",
 736                                        smp_processor_id(),
 737                                        atomic_read(&data.finished),
 738                                        atomic_read(&data.started));
 739                                 debugger(NULL);
 740                                 goto out;
 741                         }
 742                 }
 743         }
 744
 745         ret = 0;
 746
 747 out:
 748         call_data = NULL;
 749         HMT_medium();
 750         spin_unlock(&call_lock);
 751         return ret;
 752 }
 753
 754 EXPORT_SYMBOL_GPL(smp_call_function);
 755
 756 void smp_call_function_interrupt(void)
 757 {
 758         void (*func) (void *info);
 759         void *info;
 760         int wait;
 761
 762         /* call_data will be NULL if the sender timed out while
 763          * waiting on us to receive the call.
 764          */
 765         if (!call_data)
 766                 return;
 767
 768         func = call_data->func;
 769         info = call_data->info;
 770         wait = call_data->wait;
 771
 772         if (!wait)
 773                 smp_mb__before_atomic_inc();
 774
 775         /*
 776          * Notify initiating CPU that I've grabbed the data and am
 777          * about to execute the function
 778          */
 779         atomic_inc(&call_data->started);
 780         /*
 781          * At this point the info structure may be out of scope unless wait==1
 782          */
 783         (*func)(info);
 784         if (wait) {
 785                 smp_mb__before_atomic_inc();
 786                 atomic_inc(&call_data->finished);
 787         }
 788 }
 789
 790 extern unsigned long decr_overclock;
 791 extern struct gettimeofday_struct do_gtod;
 792
 793 struct thread_info *current_set[NR_CPUS];
 794
 795 DECLARE_PER_CPU(unsigned int, pvr);
 796
 797 static void __devinit smp_store_cpu_info(int id)
 798 {
 799         per_cpu(pvr, id) = _get_PVR();
 800 }
 801
 802 static void __init smp_create_idle(unsigned int cpu)
 803 {
 804         struct pt_regs regs;
 805         struct task_struct *p;
 806
 807         /* create a process for the processor */
 808         /* only regs.msr is actually used, and 0 is OK for it */
 809         memset(&regs, 0, sizeof(struct pt_regs));
 810         p = copy_process(CLONE_VM | CLONE_IDLETASK,
 811                          0, &regs, 0, NULL, NULL);
 812         if (IS_ERR(p))
 813                 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
 814
 815         wake_up_forked_process(p);
 816         init_idle(p, cpu);
 817         unhash_process(p);
 818
 819         paca[cpu].__current = p;
 820         current_set[cpu] = p->thread_info;
 821 }
 822
 823 void __init smp_prepare_cpus(unsigned int max_cpus)
 824 {
 825         unsigned int cpu;
 826
 827         /*
 828          * setup_cpu may need to be called on the boot cpu. We havent
 829          * spun any cpus up but lets be paranoid.
 830          */
 831         BUG_ON(boot_cpuid != smp_processor_id());
 832
 833         /* Fixup boot cpu */
 834         smp_store_cpu_info(boot_cpuid);
 835         cpu_callin_map[boot_cpuid] = 1;
 836         paca[boot_cpuid].prof_counter = 1;
 837         paca[boot_cpuid].prof_multiplier = 1;
 838
 839 #ifndef CONFIG_PPC_ISERIES
 840         paca[boot_cpuid].next_jiffy_update_tb = tb_last_stamp = get_tb();
 841
 842         /*
 843          * Should update do_gtod.stamp_xsec.
 844          * For now we leave it which means the time can be some
 845          * number of msecs off until someone does a settimeofday()
 846          */
 847         do_gtod.tb_orig_stamp = tb_last_stamp;
 848
 849         look_for_more_cpus();
 850 #endif
 851
 852         max_cpus = smp_ops->probe();
 853
 854         /* Backup CPU 0 state if necessary */
 855         __save_cpu_setup();
 856
 857         smp_space_timers(max_cpus);
 858
 859         for_each_cpu(cpu)
 860                 if (cpu != boot_cpuid)
 861                         smp_create_idle(cpu);
 862 }
 863
 864 void __devinit smp_prepare_boot_cpu(void)
 865 {
 866         BUG_ON(smp_processor_id() != boot_cpuid);
 867
 868         /* cpu_possible is set up in prom.c */
 869         cpu_set(boot_cpuid, cpu_online_map);
 870
 871         paca[boot_cpuid].__current = current;
 872         current_set[boot_cpuid] = current->thread_info;
 873 }
 874
 875 int __devinit __cpu_up(unsigned int cpu)
 876 {
 877         int c;
 878
 879         /* At boot, don't bother with non-present cpus -JSCHOPP */
 880         if (system_state == SYSTEM_BOOTING && !cpu_present_at_boot(cpu))
 881                 return -ENOENT;
 882
 883         paca[cpu].prof_counter = 1;
 884         paca[cpu].prof_multiplier = 1;
 885         paca[cpu].default_decr = tb_ticks_per_jiffy / decr_overclock;
 886
 887         if (!(cur_cpu_spec->cpu_features & CPU_FTR_SLB)) {
 888                 void *tmp;
 889
 890                 /* maximum of 48 CPUs on machines with a segment table */
 891                 if (cpu >= 48)
 892                         BUG();
 893
 894                 tmp = &stab_array[PAGE_SIZE * cpu];
 895                 memset(tmp, 0, PAGE_SIZE);
 896                 paca[cpu].stab_addr = (unsigned long)tmp;
 897                 paca[cpu].stab_real = virt_to_abs(tmp);
 898         }
 899
 900         /* The information for processor bringup must
 901          * be written out to main store before we release
 902          * the processor.
 903          */
 904         mb();
 905
 906         /* wake up cpus */
 907         smp_ops->kick_cpu(cpu);
 908
 909         /*
 910          * wait to see if the cpu made a callin (is actually up).
 911          * use this value that I found through experimentation.
 912          * -- Cort
 913          */
 914         if (system_state == SYSTEM_BOOTING)
 915                 for (c = 5000; c && !cpu_callin_map[cpu]; c--)
 916                         udelay(100);
 917 #ifdef CONFIG_HOTPLUG_CPU
 918         else
 919                 /*
 920                  * CPUs can take much longer to come up in the
 921                  * hotplug case.  Wait five seconds.
 922                  */
 923                 for (c = 25; c && !cpu_callin_map[cpu]; c--) {
 924                         set_current_state(TASK_UNINTERRUPTIBLE);
 925                         schedule_timeout(HZ/5);
 926                 }
 927 #endif
 928
 929         if (!cpu_callin_map[cpu]) {
 930                 printk("Processor %u is stuck.\n", cpu);
 931                 return -ENOENT;
 932         }
 933
 934         printk("Processor %u found.\n", cpu);
 935
 936         if (smp_ops->give_timebase)
 937                 smp_ops->give_timebase();
 938
 939         /* Wait until cpu puts itself in the online map */
 940         while (!cpu_online(cpu))
 941                 cpu_relax();
 942
 943         return 0;
 944 }
 945
 946 extern unsigned int default_distrib_server;
 947 /* Activate a secondary processor. */
 948 int __devinit start_secondary(void *unused)
 949 {
 950         unsigned int cpu = smp_processor_id();
 951
 952         atomic_inc(&init_mm.mm_count);
 953         current->active_mm = &init_mm;
 954
 955         smp_store_cpu_info(cpu);
 956         set_dec(paca[cpu].default_decr);
 957         cpu_callin_map[cpu] = 1;
 958
 959         smp_ops->setup_cpu(cpu);
 960         if (smp_ops->take_timebase)
 961                 smp_ops->take_timebase();
 962
 963 #ifdef CONFIG_PPC_PSERIES
 964         if (cur_cpu_spec->firmware_features & FW_FEATURE_SPLPAR) {
 965                 vpa_init(cpu);
 966         }
 967
 968 #ifdef CONFIG_IRQ_ALL_CPUS
 969         /* Put the calling processor into the GIQ.  This is really only
 970          * necessary from a secondary thread as the OF start-cpu interface
 971          * performs this function for us on primary threads.
 972          */
 973         /* TODO: 9005 is #defined in rtas-proc.c -- move to a header */
 974         rtas_set_indicator(9005, default_distrib_server, 1);
 975 #endif
 976 #endif
 977
 978         spin_lock(&call_lock);
 979         cpu_set(cpu, cpu_online_map);
 980         spin_unlock(&call_lock);
 981
 982         local_irq_enable();
 983
 984         return cpu_idle(NULL);
 985 }
 986
 987 int setup_profiling_timer(unsigned int multiplier)
 988 {
 989         return 0;
 990 }
 991
 992 void __init smp_cpus_done(unsigned int max_cpus)
 993 {
 994         cpumask_t old_mask;
 995
 996         /* We want the setup_cpu() here to be called from CPU 0, but our
 997          * init thread may have been "borrowed" by another CPU in the meantime
 998          * se we pin us down to CPU 0 for a short while
 999          */
1000         old_mask = current->cpus_allowed;
1001         set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid));
1002
1003         smp_ops->setup_cpu(boot_cpuid);
1004
1005         /* XXX fix this, xics currently relies on it - Anton */
1006         smp_threads_ready = 1;
1007
1008         set_cpus_allowed(current, old_mask);
1009 }
1010
1011 #ifdef CONFIG_SCHED_SMT
1012 #ifdef CONFIG_NUMA
1013 static struct sched_group sched_group_cpus[NR_CPUS];
1014 static struct sched_group sched_group_phys[NR_CPUS];
1015 static struct sched_group sched_group_nodes[MAX_NUMNODES];
1016 static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
1017 static DEFINE_PER_CPU(struct sched_domain, phys_domains);
1018 static DEFINE_PER_CPU(struct sched_domain, node_domains);
1019 __init void arch_init_sched_domains(void)
1020 {
1021         int i;
1022         struct sched_group *first = NULL, *last = NULL;
1023
1024         /* Set up domains */
1025         for_each_cpu(i) {
1026                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1027                 struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
1028                 struct sched_domain *node_domain = &per_cpu(node_domains, i);
1029                 int node = cpu_to_node(i);
1030                 cpumask_t nodemask = node_to_cpumask(node);
1031                 cpumask_t my_cpumask = cpumask_of_cpu(i);
1032                 cpumask_t sibling_cpumask = cpumask_of_cpu(i ^ 0x1);
1033
1034                 *cpu_domain = SD_SIBLING_INIT;
1035                 if (cur_cpu_spec->cpu_features & CPU_FTR_SMT)
1036                         cpus_or(cpu_domain->span, my_cpumask, sibling_cpumask);
1037                 else
1038                         cpu_domain->span = my_cpumask;
1039                 cpu_domain->parent = phys_domain;
1040                 cpu_domain->groups = &sched_group_cpus[i];
1041
1042                 *phys_domain = SD_CPU_INIT;
1043                 phys_domain->span = nodemask;
1044                 phys_domain->parent = node_domain;
1045                 phys_domain->groups = &sched_group_phys[first_cpu(cpu_domain->span)];
1046
1047                 *node_domain = SD_NODE_INIT;
1048                 node_domain->span = cpu_possible_map;
1049                 node_domain->groups = &sched_group_nodes[node];
1050         }
1051
1052         /* Set up CPU (sibling) groups */
1053         for_each_cpu(i) {
1054                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1055                 int j;
1056                 first = last = NULL;
1057
1058                 if (i != first_cpu(cpu_domain->span))
1059                         continue;
1060
1061                 for_each_cpu_mask(j, cpu_domain->span) {
1062                         struct sched_group *cpu = &sched_group_cpus[j];
1063
1064                         cpus_clear(cpu->cpumask);
1065                         cpu_set(j, cpu->cpumask);
1066                         cpu->cpu_power = SCHED_LOAD_SCALE;
1067
1068                         if (!first)
1069                                 first = cpu;
1070                         if (last)
1071                                 last->next = cpu;
1072                         last = cpu;
1073                 }
1074                 last->next = first;
1075         }
1076
1077         for (i = 0; i < MAX_NUMNODES; i++) {
1078                 int j;
1079                 cpumask_t nodemask;
1080                 struct sched_group *node = &sched_group_nodes[i];
1081                 cpumask_t node_cpumask = node_to_cpumask(i);
1082                 cpus_and(nodemask, node_cpumask, cpu_possible_map);
1083
1084                 if (cpus_empty(nodemask))
1085                         continue;
1086
1087                 first = last = NULL;
1088                 /* Set up physical groups */
1089                 for_each_cpu_mask(j, nodemask) {
1090                         struct sched_domain *cpu_domain = &per_cpu(cpu_domains, j);
1091                         struct sched_group *cpu = &sched_group_phys[j];
1092
1093                         if (j != first_cpu(cpu_domain->span))
1094                                 continue;
1095
1096                         cpu->cpumask = cpu_domain->span;
1097                         /*
1098                          * Make each extra sibling increase power by 10% of
1099                          * the basic CPU. This is very arbitrary.
1100                          */
1101                         cpu->cpu_power = SCHED_LOAD_SCALE + SCHED_LOAD_SCALE*(cpus_weight(cpu->cpumask)-1) / 10;
1102                         node->cpu_power += cpu->cpu_power;
1103
1104                         if (!first)
1105                                 first = cpu;
1106                         if (last)
1107                                 last->next = cpu;
1108                         last = cpu;
1109                 }
1110                 last->next = first;
1111         }
1112
1113         /* Set up nodes */
1114         first = last = NULL;
1115         for (i = 0; i < MAX_NUMNODES; i++) {
1116                 struct sched_group *cpu = &sched_group_nodes[i];
1117                 cpumask_t nodemask;
1118                 cpumask_t node_cpumask = node_to_cpumask(i);
1119                 cpus_and(nodemask, node_cpumask, cpu_possible_map);
1120
1121                 if (cpus_empty(nodemask))
1122                         continue;
1123
1124                 cpu->cpumask = nodemask;
1125                 /* ->cpu_power already setup */
1126
1127                 if (!first)
1128                         first = cpu;
1129                 if (last)
1130                         last->next = cpu;
1131                 last = cpu;
1132         }
1133         last->next = first;
1134
1135         mb();
1136         for_each_cpu(i) {
1137                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1138                 cpu_attach_domain(cpu_domain, i);
1139         }
1140 }
1141 #else /* !CONFIG_NUMA */
1142 static struct sched_group sched_group_cpus[NR_CPUS];
1143 static struct sched_group sched_group_phys[NR_CPUS];
1144 static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
1145 static DEFINE_PER_CPU(struct sched_domain, phys_domains);
1146 __init void arch_init_sched_domains(void)
1147 {
1148         int i;
1149         struct sched_group *first = NULL, *last = NULL;
1150
1151         /* Set up domains */
1152         for_each_cpu(i) {
1153                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1154                 struct sched_domain *phys_domain = &per_cpu(phys_domains, i);
1155                 cpumask_t my_cpumask = cpumask_of_cpu(i);
1156                 cpumask_t sibling_cpumask = cpumask_of_cpu(i ^ 0x1);
1157
1158                 *cpu_domain = SD_SIBLING_INIT;
1159                 if (cur_cpu_spec->cpu_features & CPU_FTR_SMT)
1160                         cpus_or(cpu_domain->span, my_cpumask, sibling_cpumask);
1161                 else
1162                         cpu_domain->span = my_cpumask;
1163                 cpu_domain->parent = phys_domain;
1164                 cpu_domain->groups = &sched_group_cpus[i];
1165
1166                 *phys_domain = SD_CPU_INIT;
1167                 phys_domain->span = cpu_possible_map;
1168                 phys_domain->groups = &sched_group_phys[first_cpu(cpu_domain->span)];
1169         }
1170
1171         /* Set up CPU (sibling) groups */
1172         for_each_cpu(i) {
1173                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1174                 int j;
1175                 first = last = NULL;
1176
1177                 if (i != first_cpu(cpu_domain->span))
1178                         continue;
1179
1180                 for_each_cpu_mask(j, cpu_domain->span) {
1181                         struct sched_group *cpu = &sched_group_cpus[j];
1182
1183                         cpus_clear(cpu->cpumask);
1184                         cpu_set(j, cpu->cpumask);
1185                         cpu->cpu_power = SCHED_LOAD_SCALE;
1186
1187                         if (!first)
1188                                 first = cpu;
1189                         if (last)
1190                                 last->next = cpu;
1191                         last = cpu;
1192                 }
1193                 last->next = first;
1194         }
1195
1196         first = last = NULL;
1197         /* Set up physical groups */
1198         for_each_cpu(i) {
1199                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1200                 struct sched_group *cpu = &sched_group_phys[i];
1201
1202                 if (i != first_cpu(cpu_domain->span))
1203                         continue;
1204
1205                 cpu->cpumask = cpu_domain->span;
1206                 /* See SMT+NUMA setup for comment */
1207                 cpu->cpu_power = SCHED_LOAD_SCALE + SCHED_LOAD_SCALE*(cpus_weight(cpu->cpumask)-1) / 10;
1208
1209                 if (!first)
1210                         first = cpu;
1211                 if (last)
1212                         last->next = cpu;
1213                 last = cpu;
1214         }
1215         last->next = first;
1216
1217         mb();
1218         for_each_cpu(i) {
1219                 struct sched_domain *cpu_domain = &per_cpu(cpu_domains, i);
1220                 cpu_attach_domain(cpu_domain, i);
1221         }
1222 }
1223 #endif /* CONFIG_NUMA */
1224 #endif /* CONFIG_SCHED_SMT */