drivers/xen/core/smpboot.c

   1 /*
   2  *      Xen SMP booting functions
   3  *
   4  *      See arch/i386/kernel/smpboot.c for copyright and credits for derived
   5  *      portions of this file.
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/kernel.h>
  11 #include <linux/mm.h>
  12 #include <linux/sched.h>
  13 #include <linux/kernel_stat.h>
  14 #include <linux/smp_lock.h>
  15 #include <linux/irq.h>
  16 #include <linux/bootmem.h>
  17 #include <linux/notifier.h>
  18 #include <linux/cpu.h>
  19 #include <linux/percpu.h>
  20 #include <asm/desc.h>
  21 #include <asm/arch_hooks.h>
  22 #include <asm/pgalloc.h>
  23 #include <xen/evtchn.h>
  24 #include <xen/interface/vcpu.h>
  25 #include <xen/cpu_hotplug.h>
  26 #include <xen/xenbus.h>
  27
  28 extern irqreturn_t smp_reschedule_interrupt(int, void *, struct pt_regs *);
  29 extern irqreturn_t smp_call_function_interrupt(int, void *, struct pt_regs *);
  30
  31 extern void local_setup_timer(unsigned int cpu);
  32 extern void local_teardown_timer(unsigned int cpu);
  33
  34 extern void hypervisor_callback(void);
  35 extern void failsafe_callback(void);
  36 extern void system_call(void);
  37 extern void smp_trap_init(trap_info_t *);
  38
  39 /* Number of siblings per CPU package */
  40 int smp_num_siblings = 1;
  41 EXPORT_SYMBOL(smp_num_siblings);
  42 #if defined(__i386__)
  43 int cpu_llc_id[NR_CPUS] __cpuinitdata = {[0 ... NR_CPUS-1] = BAD_APICID};
  44 #elif defined(__x86_64__)
  45 u8 cpu_llc_id[NR_CPUS] __cpuinitdata  = {[0 ... NR_CPUS-1] = BAD_APICID};
  46 #endif
  47 EXPORT_SYMBOL(cpu_llc_id);
  48
  49 cpumask_t cpu_online_map;
  50 EXPORT_SYMBOL(cpu_online_map);
  51 cpumask_t cpu_possible_map;
  52 EXPORT_SYMBOL(cpu_possible_map);
  53
  54 struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;
  55 EXPORT_SYMBOL(cpu_data);
  56
  57 #ifdef CONFIG_HOTPLUG_CPU
  58 DEFINE_PER_CPU(int, cpu_state) = { 0 };
  59 #endif
  60
  61 static DEFINE_PER_CPU(int, resched_irq);
  62 static DEFINE_PER_CPU(int, callfunc_irq);
  63 static char resched_name[NR_CPUS][15];
  64 static char callfunc_name[NR_CPUS][15];
  65
  66 u8 cpu_2_logical_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID };
  67
  68 void *xquad_portio;
  69
  70 cpumask_t cpu_sibling_map[NR_CPUS] __cacheline_aligned;
  71 EXPORT_SYMBOL(cpu_sibling_map);
  72 cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned;
  73 EXPORT_SYMBOL(cpu_core_map);
  74
  75 #if defined(__i386__)
  76 u8 x86_cpu_to_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = 0xff };
  77 EXPORT_SYMBOL(x86_cpu_to_apicid);
  78 #elif !defined(CONFIG_X86_IO_APIC)
  79 unsigned int maxcpus = NR_CPUS;
  80 #endif
  81
  82 void __init prefill_possible_map(void)
  83 {
  84         int i, rc;
  85
  86         for (i = 0; i < NR_CPUS; i++) {
  87                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
  88                 if (rc >= 0)
  89                         cpu_set(i, cpu_possible_map);
  90         }
  91 }
  92
  93 void __init smp_alloc_memory(void)
  94 {
  95 }
  96
  97 static inline void
  98 set_cpu_sibling_map(int cpu)
  99 {
 100         cpu_data[cpu].phys_proc_id = cpu;
 101         cpu_data[cpu].cpu_core_id = 0;
 102
 103         cpu_sibling_map[cpu] = cpumask_of_cpu(cpu);
 104         cpu_core_map[cpu]    = cpumask_of_cpu(cpu);
 105
 106         cpu_data[cpu].booted_cores = 1;
 107 }
 108
 109 static void xen_smp_intr_init(unsigned int cpu)
 110 {
 111         sprintf(resched_name[cpu], "resched%d", cpu);
 112         per_cpu(resched_irq, cpu) =
 113                 bind_ipi_to_irqhandler(
 114                         RESCHEDULE_VECTOR,
 115                         cpu,
 116                         smp_reschedule_interrupt,
 117                         SA_INTERRUPT,
 118                         resched_name[cpu],
 119                         NULL);
 120         BUG_ON(per_cpu(resched_irq, cpu) < 0);
 121
 122         sprintf(callfunc_name[cpu], "callfunc%d", cpu);
 123         per_cpu(callfunc_irq, cpu) =
 124                 bind_ipi_to_irqhandler(
 125                         CALL_FUNCTION_VECTOR,
 126                         cpu,
 127                         smp_call_function_interrupt,
 128                         SA_INTERRUPT,
 129                         callfunc_name[cpu],
 130                         NULL);
 131         BUG_ON(per_cpu(callfunc_irq, cpu) < 0);
 132
 133         if (cpu != 0)
 134                 local_setup_timer(cpu);
 135 }
 136
 137 #ifdef CONFIG_HOTPLUG_CPU
 138 static void xen_smp_intr_exit(unsigned int cpu)
 139 {
 140         if (cpu != 0)
 141                 local_teardown_timer(cpu);
 142
 143         unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
 144         unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
 145 }
 146 #endif
 147
 148 void cpu_bringup(void)
 149 {
 150         cpu_init();
 151         touch_softlockup_watchdog();
 152         preempt_disable();
 153         local_irq_enable();
 154 }
 155
 156 static void cpu_bringup_and_idle(void)
 157 {
 158         cpu_bringup();
 159         cpu_idle();
 160 }
 161
 162 void cpu_initialize_context(unsigned int cpu)
 163 {
 164         vcpu_guest_context_t ctxt;
 165         struct task_struct *idle = idle_task(cpu);
 166 #ifdef __x86_64__
 167         struct desc_ptr *gdt_descr = &cpu_gdt_descr[cpu];
 168 #else
 169         struct Xgt_desc_struct *gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
 170 #endif
 171
 172         if (cpu == 0)
 173                 return;
 174
 175         memset(&ctxt, 0, sizeof(ctxt));
 176
 177         ctxt.flags = VGCF_IN_KERNEL;
 178         ctxt.user_regs.ds = __USER_DS;
 179         ctxt.user_regs.es = __USER_DS;
 180         ctxt.user_regs.fs = 0;
 181         ctxt.user_regs.gs = 0;
 182         ctxt.user_regs.ss = __KERNEL_DS;
 183         ctxt.user_regs.eip = (unsigned long)cpu_bringup_and_idle;
 184         ctxt.user_regs.eflags = X86_EFLAGS_IF | 0x1000; /* IOPL_RING1 */
 185
 186         memset(&ctxt.fpu_ctxt, 0, sizeof(ctxt.fpu_ctxt));
 187
 188         smp_trap_init(ctxt.trap_ctxt);
 189
 190         ctxt.ldt_ents = 0;
 191
 192         ctxt.gdt_frames[0] = virt_to_mfn(gdt_descr->address);
 193         ctxt.gdt_ents      = gdt_descr->size / 8;
 194
 195 #ifdef __i386__
 196         ctxt.user_regs.cs = __KERNEL_CS;
 197         ctxt.user_regs.esp = idle->thread.esp0 - sizeof(struct pt_regs);
 198
 199         ctxt.kernel_ss = __KERNEL_DS;
 200         ctxt.kernel_sp = idle->thread.esp0;
 201
 202         ctxt.event_callback_cs     = __KERNEL_CS;
 203         ctxt.event_callback_eip    = (unsigned long)hypervisor_callback;
 204         ctxt.failsafe_callback_cs  = __KERNEL_CS;
 205         ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
 206
 207         ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
 208 #else /* __x86_64__ */
 209         ctxt.user_regs.cs = __KERNEL_CS;
 210         ctxt.user_regs.esp = idle->thread.rsp0 - sizeof(struct pt_regs);
 211
 212         ctxt.kernel_ss = __KERNEL_DS;
 213         ctxt.kernel_sp = idle->thread.rsp0;
 214
 215         ctxt.event_callback_eip    = (unsigned long)hypervisor_callback;
 216         ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
 217         ctxt.syscall_callback_eip  = (unsigned long)system_call;
 218
 219         ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
 220
 221         ctxt.gs_base_kernel = (unsigned long)(cpu_pda(cpu));
 222 #endif
 223
 224         BUG_ON(HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, &ctxt));
 225 }
 226
 227 void __init smp_prepare_cpus(unsigned int max_cpus)
 228 {
 229         int cpu;
 230         struct task_struct *idle;
 231 #ifdef __x86_64__
 232         struct desc_ptr *gdt_descr;
 233 #else
 234         struct Xgt_desc_struct *gdt_descr;
 235 #endif
 236
 237         boot_cpu_data.apicid = 0;
 238         cpu_data[0] = boot_cpu_data;
 239
 240         cpu_2_logical_apicid[0] = 0;
 241         x86_cpu_to_apicid[0] = 0;
 242
 243         current_thread_info()->cpu = 0;
 244
 245         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 246                 cpus_clear(cpu_sibling_map[cpu]);
 247                 cpus_clear(cpu_core_map[cpu]);
 248         }
 249
 250         set_cpu_sibling_map(0);
 251
 252         xen_smp_intr_init(0);
 253
 254         /* Restrict the possible_map according to max_cpus. */
 255         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
 256                 for (cpu = NR_CPUS-1; !cpu_isset(cpu, cpu_possible_map); cpu--)
 257                         continue;
 258                 cpu_clear(cpu, cpu_possible_map);
 259         }
 260
 261         for_each_possible_cpu (cpu) {
 262                 if (cpu == 0)
 263                         continue;
 264
 265 #ifdef __x86_64__
 266                 gdt_descr = &cpu_gdt_descr[cpu];
 267 #else
 268                 gdt_descr = &per_cpu(cpu_gdt_descr, cpu);
 269 #endif
 270                 gdt_descr->address = get_zeroed_page(GFP_KERNEL);
 271                 if (unlikely(!gdt_descr->address)) {
 272                         printk(KERN_CRIT "CPU%d failed to allocate GDT\n",
 273                                cpu);
 274                         continue;
 275                 }
 276                 gdt_descr->size = GDT_SIZE;
 277                 memcpy((void *)gdt_descr->address, cpu_gdt_table, GDT_SIZE);
 278                 make_page_readonly(
 279                         (void *)gdt_descr->address,
 280                         XENFEAT_writable_descriptor_tables);
 281
 282                 cpu_data[cpu] = boot_cpu_data;
 283                 cpu_data[cpu].apicid = cpu;
 284
 285                 cpu_2_logical_apicid[cpu] = cpu;
 286                 x86_cpu_to_apicid[cpu] = cpu;
 287
 288                 idle = fork_idle(cpu);
 289                 if (IS_ERR(idle))
 290                         panic("failed fork for CPU %d", cpu);
 291
 292 #ifdef __x86_64__
 293                 cpu_pda(cpu)->pcurrent = idle;
 294                 cpu_pda(cpu)->cpunumber = cpu;
 295                 clear_ti_thread_flag(idle->thread_info, TIF_FORK);
 296 #endif
 297
 298                 irq_ctx_init(cpu);
 299
 300 #ifdef CONFIG_HOTPLUG_CPU
 301                 if (is_initial_xendomain())
 302                         cpu_set(cpu, cpu_present_map);
 303 #else
 304                 cpu_set(cpu, cpu_present_map);
 305 #endif
 306
 307                 cpu_initialize_context(cpu);
 308         }
 309
 310         init_xenbus_allowed_cpumask();
 311
 312 #ifdef CONFIG_X86_IO_APIC
 313         /*
 314          * Here we can be sure that there is an IO-APIC in the system. Let's
 315          * go and set it up:
 316          */
 317         if (!skip_ioapic_setup && nr_ioapics)
 318                 setup_IO_APIC();
 319 #endif
 320 }
 321
 322 void __devinit smp_prepare_boot_cpu(void)
 323 {
 324 }
 325
 326 #ifdef CONFIG_HOTPLUG_CPU
 327
 328 /*
 329  * Initialize cpu_present_map late to skip SMP boot code in init/main.c.
 330  * But do it early enough to catch critical for_each_present_cpu() loops
 331  * in i386-specific code.
 332  */
 333 static int __init initialize_cpu_present_map(void)
 334 {
 335         cpu_present_map = cpu_possible_map;
 336         return 0;
 337 }
 338 core_initcall(initialize_cpu_present_map);
 339
 340 static void
 341 remove_siblinginfo(int cpu)
 342 {
 343         cpu_data[cpu].phys_proc_id = BAD_APICID;
 344         cpu_data[cpu].cpu_core_id = BAD_APICID;
 345
 346         cpus_clear(cpu_sibling_map[cpu]);
 347         cpus_clear(cpu_core_map[cpu]);
 348
 349         cpu_data[cpu].booted_cores = 0;
 350 }
 351
 352 int __cpu_disable(void)
 353 {
 354         cpumask_t map = cpu_online_map;
 355         int cpu = smp_processor_id();
 356
 357         if (cpu == 0)
 358                 return -EBUSY;
 359
 360         remove_siblinginfo(cpu);
 361
 362         cpu_clear(cpu, map);
 363         fixup_irqs(map);
 364         cpu_clear(cpu, cpu_online_map);
 365
 366         return 0;
 367 }
 368
 369 void __cpu_die(unsigned int cpu)
 370 {
 371         while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
 372                 current->state = TASK_UNINTERRUPTIBLE;
 373                 schedule_timeout(HZ/10);
 374         }
 375
 376         xen_smp_intr_exit(cpu);
 377
 378         if (num_online_cpus() == 1)
 379                 alternatives_smp_switch(0);
 380 }
 381
 382 #else /* !CONFIG_HOTPLUG_CPU */
 383
 384 int __cpu_disable(void)
 385 {
 386         return -ENOSYS;
 387 }
 388
 389 void __cpu_die(unsigned int cpu)
 390 {
 391         BUG();
 392 }
 393
 394 #endif /* CONFIG_HOTPLUG_CPU */
 395
 396 int __devinit __cpu_up(unsigned int cpu)
 397 {
 398         int rc;
 399
 400         rc = cpu_up_check(cpu);
 401         if (rc)
 402                 return rc;
 403
 404         if (num_online_cpus() == 1)
 405                 alternatives_smp_switch(1);
 406
 407         /* This must be done before setting cpu_online_map */
 408         set_cpu_sibling_map(cpu);
 409         wmb();
 410
 411         xen_smp_intr_init(cpu);
 412         cpu_set(cpu, cpu_online_map);
 413
 414         rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
 415         BUG_ON(rc);
 416
 417         return 0;
 418 }
 419
 420 void __init smp_cpus_done(unsigned int max_cpus)
 421 {
 422 }
 423
 424 #ifndef CONFIG_X86_LOCAL_APIC
 425 int setup_profiling_timer(unsigned int multiplier)
 426 {
 427         return -EINVAL;
 428 }
 429 #endif