2 * Local APIC handling, local APIC timers
4 * (c) 1999, 2000 Ingo Molnar <mingo@redhat.com>
7 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
8 * thanks to Eric Gilmore
10 * for testing these extensively.
11 * Maciej W. Rozycki : Various updates and fixes.
12 * Mikael Pettersson : Power Management for UP-APIC.
14 * Mikael Pettersson : PM converted to driver model.
17 #include <linux/init.h>
20 #include <linux/delay.h>
21 #include <linux/bootmem.h>
22 #include <linux/smp_lock.h>
23 #include <linux/interrupt.h>
24 #include <linux/mc146818rtc.h>
25 #include <linux/kernel_stat.h>
26 #include <linux/sysdev.h>
27 #include <linux/cpu.h>
28 #include <linux/module.h>
29 #include <linux/dmi.h>
31 #include <asm/atomic.h>
34 #include <asm/mpspec.h>
36 #include <asm/arch_hooks.h>
38 #include <asm/i8253.h>
41 #include <mach_apic.h>
42 #include <mach_apicdef.h>
48 * cpu_mask that denotes the CPUs that needs timer interrupt coming in as
49 * IPIs in place of local APIC timers
51 static cpumask_t timer_bcast_ipi;
54 * Knob to control our willingness to enable the local APIC.
56 int enable_local_apic __initdata = 0; /* -1=force-disable, +1=force-enable */
57 int prefer_apic __initdata = 0; /* when enable_local_apic == 0 prefer APIC but don't force against
59 int apic_disabled_by_dmi __initdata;
67 static void apic_pm_activate(void);
69 static int modern_apic(void)
71 unsigned int lvr, version;
72 /* AMD systems use old APIC versions, so check the CPU */
73 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
74 boot_cpu_data.x86 >= 0xf)
76 lvr = apic_read(APIC_LVR);
77 version = GET_APIC_VERSION(lvr);
78 return version >= 0x14;
82 * 'what should we do if we get a hw irq event on an illegal vector'.
83 * each architecture has to answer this themselves.
85 void ack_bad_irq(unsigned int irq)
87 printk("unexpected IRQ trap at vector %02x\n", irq);
89 * Currently unexpected vectors happen only on SMP and APIC.
90 * We _must_ ack these because every local APIC has only N
91 * irq slots per priority level, and a 'hanging, unacked' IRQ
92 * holds up an irq slot - in excessive cases (when multiple
93 * unexpected vectors occur) that might lock up the APIC
95 * But only ack when the APIC is enabled -AK
101 void __init apic_intr_init(void)
106 /* self generated IPI for local APIC timer */
107 set_intr_gate(LOCAL_TIMER_VECTOR, apic_timer_interrupt);
109 /* IPI vectors for APIC spurious and error interrupts */
110 set_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
111 set_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
113 /* thermal monitor LVT interrupt */
114 #ifdef CONFIG_X86_MCE_P4THERMAL
115 set_intr_gate(THERMAL_APIC_VECTOR, thermal_interrupt);
119 /* Using APIC to generate smp_local_timer_interrupt? */
120 int using_apic_timer __read_mostly = 0;
122 static int enabled_via_apicbase;
124 void enable_NMI_through_LVT0 (void * dummy)
128 ver = apic_read(APIC_LVR);
129 ver = GET_APIC_VERSION(ver);
130 v = APIC_DM_NMI; /* unmask and set to NMI */
131 if (!APIC_INTEGRATED(ver)) /* 82489DX */
132 v |= APIC_LVT_LEVEL_TRIGGER;
133 apic_write_around(APIC_LVT0, v);
136 int get_physical_broadcast(void)
146 unsigned int v, ver, maxlvt;
148 v = apic_read(APIC_LVR);
149 ver = GET_APIC_VERSION(v);
150 /* 82489DXs do not report # of LVT entries. */
151 maxlvt = APIC_INTEGRATED(ver) ? GET_APIC_MAXLVT(v) : 2;
155 void clear_local_APIC(void)
160 maxlvt = get_maxlvt();
163 * Masking an LVT entry can trigger a local APIC error
164 * if the vector is zero. Mask LVTERR first to prevent this.
167 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
168 apic_write_around(APIC_LVTERR, v | APIC_LVT_MASKED);
171 * Careful: we have to set masks only first to deassert
172 * any level-triggered sources.
174 v = apic_read(APIC_LVTT);
175 apic_write_around(APIC_LVTT, v | APIC_LVT_MASKED);
176 v = apic_read(APIC_LVT0);
177 apic_write_around(APIC_LVT0, v | APIC_LVT_MASKED);
178 v = apic_read(APIC_LVT1);
179 apic_write_around(APIC_LVT1, v | APIC_LVT_MASKED);
181 v = apic_read(APIC_LVTPC);
182 apic_write_around(APIC_LVTPC, v | APIC_LVT_MASKED);
185 /* lets not touch this if we didn't frob it */
186 #ifdef CONFIG_X86_MCE_P4THERMAL
188 v = apic_read(APIC_LVTTHMR);
189 apic_write_around(APIC_LVTTHMR, v | APIC_LVT_MASKED);
193 * Clean APIC state for other OSs:
195 apic_write_around(APIC_LVTT, APIC_LVT_MASKED);
196 apic_write_around(APIC_LVT0, APIC_LVT_MASKED);
197 apic_write_around(APIC_LVT1, APIC_LVT_MASKED);
199 apic_write_around(APIC_LVTERR, APIC_LVT_MASKED);
201 apic_write_around(APIC_LVTPC, APIC_LVT_MASKED);
203 #ifdef CONFIG_X86_MCE_P4THERMAL
205 apic_write_around(APIC_LVTTHMR, APIC_LVT_MASKED);
207 v = GET_APIC_VERSION(apic_read(APIC_LVR));
208 if (APIC_INTEGRATED(v)) { /* !82489DX */
209 if (maxlvt > 3) /* Due to Pentium errata 3AP and 11AP. */
210 apic_write(APIC_ESR, 0);
215 void __init connect_bsp_APIC(void)
219 * Do not trust the local APIC being empty at bootup.
223 * PIC mode, enable APIC mode in the IMCR, i.e.
224 * connect BSP's local APIC to INT and NMI lines.
226 apic_printk(APIC_VERBOSE, "leaving PIC mode, "
227 "enabling APIC mode.\n");
234 void disconnect_bsp_APIC(int virt_wire_setup)
238 * Put the board back into PIC mode (has an effect
239 * only on certain older boards). Note that APIC
240 * interrupts, including IPIs, won't work beyond
241 * this point! The only exception are INIT IPIs.
243 apic_printk(APIC_VERBOSE, "disabling APIC mode, "
244 "entering PIC mode.\n");
249 /* Go back to Virtual Wire compatibility mode */
252 /* For the spurious interrupt use vector F, and enable it */
253 value = apic_read(APIC_SPIV);
254 value &= ~APIC_VECTOR_MASK;
255 value |= APIC_SPIV_APIC_ENABLED;
257 apic_write_around(APIC_SPIV, value);
259 if (!virt_wire_setup) {
260 /* For LVT0 make it edge triggered, active high, external and enabled */
261 value = apic_read(APIC_LVT0);
262 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
263 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
264 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED );
265 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
266 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
267 apic_write_around(APIC_LVT0, value);
271 apic_write_around(APIC_LVT0, APIC_LVT_MASKED);
274 /* For LVT1 make it edge triggered, active high, nmi and enabled */
275 value = apic_read(APIC_LVT1);
277 APIC_MODE_MASK | APIC_SEND_PENDING |
278 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
279 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
280 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
281 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
282 apic_write_around(APIC_LVT1, value);
286 void disable_local_APIC(void)
293 * Disable APIC (implies clearing of registers
296 value = apic_read(APIC_SPIV);
297 value &= ~APIC_SPIV_APIC_ENABLED;
298 apic_write_around(APIC_SPIV, value);
300 if (enabled_via_apicbase) {
302 rdmsr(MSR_IA32_APICBASE, l, h);
303 l &= ~MSR_IA32_APICBASE_ENABLE;
304 wrmsr(MSR_IA32_APICBASE, l, h);
309 * This is to verify that we're looking at a real local APIC.
310 * Check these against your board if the CPUs aren't getting
311 * started for no apparent reason.
313 int __init verify_local_APIC(void)
315 unsigned int reg0, reg1;
318 * The version register is read-only in a real APIC.
320 reg0 = apic_read(APIC_LVR);
321 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
322 apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
323 reg1 = apic_read(APIC_LVR);
324 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
327 * The two version reads above should print the same
328 * numbers. If the second one is different, then we
329 * poke at a non-APIC.
335 * Check if the version looks reasonably.
337 reg1 = GET_APIC_VERSION(reg0);
338 if (reg1 == 0x00 || reg1 == 0xff)
341 if (reg1 < 0x02 || reg1 == 0xff)
345 * The ID register is read/write in a real APIC.
347 reg0 = apic_read(APIC_ID);
348 apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
351 * The next two are just to see if we have sane values.
352 * They're only really relevant if we're in Virtual Wire
353 * compatibility mode, but most boxes are anymore.
355 reg0 = apic_read(APIC_LVT0);
356 apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
357 reg1 = apic_read(APIC_LVT1);
358 apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
363 void __init sync_Arb_IDs(void)
365 /* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1
366 And not needed on AMD */
372 apic_wait_icr_idle();
374 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
375 apic_write_around(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG
379 extern void __error_in_apic_c (void);
382 * An initial setup of the virtual wire mode.
384 void __init init_bsp_APIC(void)
386 unsigned long value, ver;
389 * Don't do the setup now if we have a SMP BIOS as the
390 * through-I/O-APIC virtual wire mode might be active.
392 if (smp_found_config || !cpu_has_apic)
395 value = apic_read(APIC_LVR);
396 ver = GET_APIC_VERSION(value);
399 * Do not trust the local APIC being empty at bootup.
406 value = apic_read(APIC_SPIV);
407 value &= ~APIC_VECTOR_MASK;
408 value |= APIC_SPIV_APIC_ENABLED;
410 /* This bit is reserved on P4/Xeon and should be cleared */
411 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 15))
412 value &= ~APIC_SPIV_FOCUS_DISABLED;
414 value |= APIC_SPIV_FOCUS_DISABLED;
415 value |= SPURIOUS_APIC_VECTOR;
416 apic_write_around(APIC_SPIV, value);
419 * Set up the virtual wire mode.
421 apic_write_around(APIC_LVT0, APIC_DM_EXTINT);
423 if (!APIC_INTEGRATED(ver)) /* 82489DX */
424 value |= APIC_LVT_LEVEL_TRIGGER;
425 apic_write_around(APIC_LVT1, value);
428 void __devinit setup_local_APIC(void)
430 unsigned long oldvalue, value, ver, maxlvt;
433 /* Pound the ESR really hard over the head with a big hammer - mbligh */
435 apic_write(APIC_ESR, 0);
436 apic_write(APIC_ESR, 0);
437 apic_write(APIC_ESR, 0);
438 apic_write(APIC_ESR, 0);
441 value = apic_read(APIC_LVR);
442 ver = GET_APIC_VERSION(value);
444 if ((SPURIOUS_APIC_VECTOR & 0x0f) != 0x0f)
448 * Double-check whether this APIC is really registered.
450 if (!apic_id_registered())
454 * Intel recommends to set DFR, LDR and TPR before enabling
455 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
456 * document number 292116). So here it goes...
461 * Set Task Priority to 'accept all'. We never change this
464 value = apic_read(APIC_TASKPRI);
465 value &= ~APIC_TPRI_MASK;
466 apic_write_around(APIC_TASKPRI, value);
469 * After a crash, we no longer service the interrupts and a pending
470 * interrupt from previous kernel might still have ISR bit set.
472 * Most probably by now CPU has serviced that pending interrupt and
473 * it might not have done the ack_APIC_irq() because it thought,
474 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
475 * does not clear the ISR bit and cpu thinks it has already serivced
476 * the interrupt. Hence a vector might get locked. It was noticed
477 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
479 for (i = APIC_ISR_NR - 1; i >= 0; i--) {
480 value = apic_read(APIC_ISR + i*0x10);
481 for (j = 31; j >= 0; j--) {
488 * Now that we are all set up, enable the APIC
490 value = apic_read(APIC_SPIV);
491 value &= ~APIC_VECTOR_MASK;
495 value |= APIC_SPIV_APIC_ENABLED;
498 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
499 * certain networking cards. If high frequency interrupts are
500 * happening on a particular IOAPIC pin, plus the IOAPIC routing
501 * entry is masked/unmasked at a high rate as well then sooner or
502 * later IOAPIC line gets 'stuck', no more interrupts are received
503 * from the device. If focus CPU is disabled then the hang goes
506 * [ This bug can be reproduced easily with a level-triggered
507 * PCI Ne2000 networking cards and PII/PIII processors, dual
511 * Actually disabling the focus CPU check just makes the hang less
512 * frequent as it makes the interrupt distributon model be more
513 * like LRU than MRU (the short-term load is more even across CPUs).
514 * See also the comment in end_level_ioapic_irq(). --macro
517 /* Enable focus processor (bit==0) */
518 value &= ~APIC_SPIV_FOCUS_DISABLED;
520 /* Disable focus processor (bit==1) */
521 value |= APIC_SPIV_FOCUS_DISABLED;
524 * Set spurious IRQ vector
526 value |= SPURIOUS_APIC_VECTOR;
527 apic_write_around(APIC_SPIV, value);
532 * set up through-local-APIC on the BP's LINT0. This is not
533 * strictly necessery in pure symmetric-IO mode, but sometimes
534 * we delegate interrupts to the 8259A.
537 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
539 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
540 if (!smp_processor_id() && (pic_mode || !value)) {
541 value = APIC_DM_EXTINT;
542 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n",
545 value = APIC_DM_EXTINT | APIC_LVT_MASKED;
546 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n",
549 apic_write_around(APIC_LVT0, value);
552 * only the BP should see the LINT1 NMI signal, obviously.
554 if (!smp_processor_id())
557 value = APIC_DM_NMI | APIC_LVT_MASKED;
558 if (!APIC_INTEGRATED(ver)) /* 82489DX */
559 value |= APIC_LVT_LEVEL_TRIGGER;
560 apic_write_around(APIC_LVT1, value);
562 if (APIC_INTEGRATED(ver) && !esr_disable) { /* !82489DX */
563 maxlvt = get_maxlvt();
564 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
565 apic_write(APIC_ESR, 0);
566 oldvalue = apic_read(APIC_ESR);
568 value = ERROR_APIC_VECTOR; // enables sending errors
569 apic_write_around(APIC_LVTERR, value);
571 * spec says clear errors after enabling vector.
574 apic_write(APIC_ESR, 0);
575 value = apic_read(APIC_ESR);
576 if (value != oldvalue)
577 apic_printk(APIC_VERBOSE, "ESR value before enabling "
578 "vector: 0x%08lx after: 0x%08lx\n",
583 * Something untraceble is creating bad interrupts on
584 * secondary quads ... for the moment, just leave the
585 * ESR disabled - we can't do anything useful with the
586 * errors anyway - mbligh
588 printk("Leaving ESR disabled.\n");
590 printk("No ESR for 82489DX.\n");
593 if (nmi_watchdog == NMI_LOCAL_APIC)
594 setup_apic_nmi_watchdog();
599 * If Linux enabled the LAPIC against the BIOS default
600 * disable it down before re-entering the BIOS on shutdown.
601 * Otherwise the BIOS may get confused and not power-off.
602 * Additionally clear all LVT entries before disable_local_APIC
603 * for the case where Linux didn't enable the LAPIC.
605 void lapic_shutdown(void)
612 local_irq_save(flags);
615 if (enabled_via_apicbase)
616 disable_local_APIC();
618 local_irq_restore(flags);
625 /* r/w apic fields */
626 unsigned int apic_id;
627 unsigned int apic_taskpri;
628 unsigned int apic_ldr;
629 unsigned int apic_dfr;
630 unsigned int apic_spiv;
631 unsigned int apic_lvtt;
632 unsigned int apic_lvtpc;
633 unsigned int apic_lvt0;
634 unsigned int apic_lvt1;
635 unsigned int apic_lvterr;
636 unsigned int apic_tmict;
637 unsigned int apic_tdcr;
638 unsigned int apic_thmr;
641 static int lapic_suspend(struct sys_device *dev, pm_message_t state)
645 if (!apic_pm_state.active)
648 apic_pm_state.apic_id = apic_read(APIC_ID);
649 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
650 apic_pm_state.apic_ldr = apic_read(APIC_LDR);
651 apic_pm_state.apic_dfr = apic_read(APIC_DFR);
652 apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
653 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
654 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
655 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
656 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
657 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
658 apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
659 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
660 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
662 local_irq_save(flags);
663 disable_local_APIC();
664 local_irq_restore(flags);
668 static int lapic_resume(struct sys_device *dev)
673 if (!apic_pm_state.active)
676 local_irq_save(flags);
679 * Make sure the APICBASE points to the right address
681 * FIXME! This will be wrong if we ever support suspend on
682 * SMP! We'll need to do this as part of the CPU restore!
684 rdmsr(MSR_IA32_APICBASE, l, h);
685 l &= ~MSR_IA32_APICBASE_BASE;
686 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
687 wrmsr(MSR_IA32_APICBASE, l, h);
689 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
690 apic_write(APIC_ID, apic_pm_state.apic_id);
691 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
692 apic_write(APIC_LDR, apic_pm_state.apic_ldr);
693 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
694 apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
695 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
696 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
697 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
698 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
699 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
700 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
701 apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
702 apic_write(APIC_ESR, 0);
704 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
705 apic_write(APIC_ESR, 0);
707 local_irq_restore(flags);
712 * This device has no shutdown method - fully functioning local APICs
713 * are needed on every CPU up until machine_halt/restart/poweroff.
716 static struct sysdev_class lapic_sysclass = {
717 set_kset_name("lapic"),
718 .resume = lapic_resume,
719 .suspend = lapic_suspend,
722 static struct sys_device device_lapic = {
724 .cls = &lapic_sysclass,
727 static void __devinit apic_pm_activate(void)
729 apic_pm_state.active = 1;
732 static int __init init_lapic_sysfs(void)
738 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
740 error = sysdev_class_register(&lapic_sysclass);
742 error = sysdev_register(&device_lapic);
745 device_initcall(init_lapic_sysfs);
747 #else /* CONFIG_PM */
749 static void apic_pm_activate(void) { }
751 #endif /* CONFIG_PM */
754 * Detect and enable local APICs on non-SMP boards.
755 * Original code written by Keir Fraser.
758 static int __init apic_set_verbosity(char *str)
764 if (strcmp("debug", str) == 0)
765 apic_verbosity = APIC_DEBUG;
766 else if (strcmp("verbose", str) == 0)
767 apic_verbosity = APIC_VERBOSE;
771 __setup("apic", apic_set_verbosity);
773 static int __init detect_init_APIC (void)
777 /* Disabled by kernel option? */
778 if (enable_local_apic < 0)
781 switch (boot_cpu_data.x86_vendor) {
783 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
784 (boot_cpu_data.x86 == 15))
787 case X86_VENDOR_INTEL:
788 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
789 (boot_cpu_data.x86 == 5 && cpu_has_apic))
798 * Over-ride BIOS and try to enable the local
799 * APIC only if "lapic" specified.
801 if (enable_local_apic <= 0) {
802 if (!apic_disabled_by_dmi)
803 printk("Local APIC disabled by BIOS -- "
804 "you can enable it with \"lapic\"\n");
808 * Some BIOSes disable the local APIC in the
809 * APIC_BASE MSR. This can only be done in
810 * software for Intel P6 or later and AMD K7
811 * (Model > 1) or later.
813 rdmsr(MSR_IA32_APICBASE, l, h);
814 if (!(l & MSR_IA32_APICBASE_ENABLE)) {
815 printk("Local APIC disabled by BIOS -- reenabling.\n");
816 l &= ~MSR_IA32_APICBASE_BASE;
817 l |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE;
818 wrmsr(MSR_IA32_APICBASE, l, h);
819 enabled_via_apicbase = 1;
823 * The APIC feature bit should now be enabled
826 features = cpuid_edx(1);
827 if (!(features & (1 << X86_FEATURE_APIC))) {
828 printk("Could not enable APIC!\n");
831 set_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability);
832 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
834 /* The BIOS may have set up the APIC at some other address */
835 rdmsr(MSR_IA32_APICBASE, l, h);
836 if (l & MSR_IA32_APICBASE_ENABLE)
837 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
839 if (nmi_watchdog != NMI_NONE)
840 nmi_watchdog = NMI_LOCAL_APIC;
842 printk("Found and enabled local APIC!\n");
849 printk("No local APIC present or hardware disabled\n");
853 void __init init_apic_mappings(void)
855 unsigned long apic_phys;
858 * If no local APIC can be found then set up a fake all
859 * zeroes page to simulate the local APIC and another
860 * one for the IO-APIC.
862 if (!smp_found_config && detect_init_APIC()) {
863 apic_phys = (unsigned long) alloc_bootmem_pages(PAGE_SIZE);
864 apic_phys = __pa(apic_phys);
866 apic_phys = mp_lapic_addr;
868 set_fixmap_nocache(FIX_APIC_BASE, apic_phys);
869 printk(KERN_DEBUG "mapped APIC to %08lx (%08lx)\n", APIC_BASE,
873 * Fetch the APIC ID of the BSP in case we have a
874 * default configuration (or the MP table is broken).
876 if (boot_cpu_physical_apicid == -1U)
877 boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
879 #ifdef CONFIG_X86_IO_APIC
881 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
884 for (i = 0; i < nr_ioapics; i++) {
885 if (smp_found_config) {
886 ioapic_phys = mp_ioapics[i].mpc_apicaddr;
889 "WARNING: bogus zero IO-APIC "
890 "address found in MPTABLE, "
891 "disabling IO/APIC support!\n");
892 smp_found_config = 0;
893 skip_ioapic_setup = 1;
894 goto fake_ioapic_page;
898 ioapic_phys = (unsigned long)
899 alloc_bootmem_pages(PAGE_SIZE);
900 ioapic_phys = __pa(ioapic_phys);
902 set_fixmap_nocache(idx, ioapic_phys);
903 printk(KERN_DEBUG "mapped IOAPIC to %08lx (%08lx)\n",
904 __fix_to_virt(idx), ioapic_phys);
912 * This part sets up the APIC 32 bit clock in LVTT1, with HZ interrupts
913 * per second. We assume that the caller has already set up the local
916 * The APIC timer is not exactly sync with the external timer chip, it
917 * closely follows bus clocks.
921 * The timer chip is already set up at HZ interrupts per second here,
922 * but we do not accept timer interrupts yet. We only allow the BP
925 static unsigned int __devinit get_8254_timer_count(void)
931 spin_lock_irqsave(&i8253_lock, flags);
933 outb_p(0x00, PIT_MODE);
934 count = inb_p(PIT_CH0);
935 count |= inb_p(PIT_CH0) << 8;
937 spin_unlock_irqrestore(&i8253_lock, flags);
942 /* next tick in 8254 can be caught by catching timer wraparound */
943 static void __devinit wait_8254_wraparound(void)
945 unsigned int curr_count, prev_count;
947 curr_count = get_8254_timer_count();
949 prev_count = curr_count;
950 curr_count = get_8254_timer_count();
952 /* workaround for broken Mercury/Neptune */
953 if (prev_count >= curr_count + 0x100)
954 curr_count = get_8254_timer_count();
956 } while (prev_count >= curr_count);
960 * Default initialization for 8254 timers. If we use other timers like HPET,
961 * we override this later
963 void (*wait_timer_tick)(void) __devinitdata = wait_8254_wraparound;
966 * This function sets up the local APIC timer, with a timeout of
967 * 'clocks' APIC bus clock. During calibration we actually call
968 * this function twice on the boot CPU, once with a bogus timeout
969 * value, second time for real. The other (noncalibrating) CPUs
970 * call this function only once, with the real, calibrated value.
972 * We do reads before writes even if unnecessary, to get around the
973 * P5 APIC double write bug.
976 #define APIC_DIVISOR 16
978 static void __setup_APIC_LVTT(unsigned int clocks)
980 unsigned int lvtt_value, tmp_value, ver;
981 int cpu = smp_processor_id();
983 ver = GET_APIC_VERSION(apic_read(APIC_LVR));
984 lvtt_value = APIC_LVT_TIMER_PERIODIC | LOCAL_TIMER_VECTOR;
985 if (!APIC_INTEGRATED(ver))
986 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
988 if (cpu_isset(cpu, timer_bcast_ipi))
989 lvtt_value |= APIC_LVT_MASKED;
991 apic_write_around(APIC_LVTT, lvtt_value);
996 tmp_value = apic_read(APIC_TDCR);
997 apic_write_around(APIC_TDCR, (tmp_value
998 & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE))
1001 apic_write_around(APIC_TMICT, clocks/APIC_DIVISOR);
1004 static void __devinit setup_APIC_timer(unsigned int clocks)
1006 unsigned long flags;
1008 local_irq_save(flags);
1011 * Wait for IRQ0's slice:
1015 __setup_APIC_LVTT(clocks);
1017 local_irq_restore(flags);
1021 * In this function we calibrate APIC bus clocks to the external
1022 * timer. Unfortunately we cannot use jiffies and the timer irq
1023 * to calibrate, since some later bootup code depends on getting
1024 * the first irq? Ugh.
1026 * We want to do the calibration only once since we
1027 * want to have local timer irqs syncron. CPUs connected
1028 * by the same APIC bus have the very same bus frequency.
1029 * And we want to have irqs off anyways, no accidental
1030 * APIC irq that way.
1033 static int __init calibrate_APIC_clock(void)
1035 unsigned long long t1 = 0, t2 = 0;
1039 const int LOOPS = HZ/10;
1041 apic_printk(APIC_VERBOSE, "calibrating APIC timer ...\n");
1044 * Put whatever arbitrary (but long enough) timeout
1045 * value into the APIC clock, we just want to get the
1046 * counter running for calibration.
1048 __setup_APIC_LVTT(1000000000);
1051 * The timer chip counts down to zero. Let's wait
1052 * for a wraparound to start exact measurement:
1053 * (the current tick might have been already half done)
1059 * We wrapped around just now. Let's start:
1063 tt1 = apic_read(APIC_TMCCT);
1066 * Let's wait LOOPS wraprounds:
1068 for (i = 0; i < LOOPS; i++)
1071 tt2 = apic_read(APIC_TMCCT);
1076 * The APIC bus clock counter is 32 bits only, it
1077 * might have overflown, but note that we use signed
1078 * longs, thus no extra care needed.
1080 * underflown to be exact, as the timer counts down ;)
1083 result = (tt1-tt2)*APIC_DIVISOR/LOOPS;
1086 apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
1088 ((long)(t2-t1)/LOOPS)/(1000000/HZ),
1089 ((long)(t2-t1)/LOOPS)%(1000000/HZ));
1091 apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
1093 result/(1000000/HZ),
1094 result%(1000000/HZ));
1099 static unsigned int calibration_result;
1101 void __init setup_boot_APIC_clock(void)
1103 unsigned long flags;
1104 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n");
1105 using_apic_timer = 1;
1107 local_irq_save(flags);
1109 calibration_result = calibrate_APIC_clock();
1111 * Now set up the timer for real.
1113 setup_APIC_timer(calibration_result);
1115 local_irq_restore(flags);
1118 void __devinit setup_secondary_APIC_clock(void)
1120 setup_APIC_timer(calibration_result);
1123 void disable_APIC_timer(void)
1125 if (using_apic_timer) {
1128 v = apic_read(APIC_LVTT);
1130 * When an illegal vector value (0-15) is written to an LVT
1131 * entry and delivery mode is Fixed, the APIC may signal an
1132 * illegal vector error, with out regard to whether the mask
1133 * bit is set or whether an interrupt is actually seen on input.
1135 * Boot sequence might call this function when the LVTT has
1136 * '0' vector value. So make sure vector field is set to
1139 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1140 apic_write_around(APIC_LVTT, v);
1144 void enable_APIC_timer(void)
1146 int cpu = smp_processor_id();
1148 if (using_apic_timer &&
1149 !cpu_isset(cpu, timer_bcast_ipi)) {
1152 v = apic_read(APIC_LVTT);
1153 apic_write_around(APIC_LVTT, v & ~APIC_LVT_MASKED);
1157 void switch_APIC_timer_to_ipi(void *cpumask)
1159 cpumask_t mask = *(cpumask_t *)cpumask;
1160 int cpu = smp_processor_id();
1162 if (cpu_isset(cpu, mask) &&
1163 !cpu_isset(cpu, timer_bcast_ipi)) {
1164 disable_APIC_timer();
1165 cpu_set(cpu, timer_bcast_ipi);
1168 EXPORT_SYMBOL(switch_APIC_timer_to_ipi);
1170 void switch_ipi_to_APIC_timer(void *cpumask)
1172 cpumask_t mask = *(cpumask_t *)cpumask;
1173 int cpu = smp_processor_id();
1175 if (cpu_isset(cpu, mask) &&
1176 cpu_isset(cpu, timer_bcast_ipi)) {
1177 cpu_clear(cpu, timer_bcast_ipi);
1178 enable_APIC_timer();
1181 EXPORT_SYMBOL(switch_ipi_to_APIC_timer);
1186 * Local timer interrupt handler. It does both profiling and
1187 * process statistics/rescheduling.
1189 * We do profiling in every local tick, statistics/rescheduling
1190 * happen only every 'profiling multiplier' ticks. The default
1191 * multiplier is 1 and it can be changed by writing the new multiplier
1192 * value into /proc/profile.
1195 inline void smp_local_timer_interrupt(struct pt_regs * regs)
1197 profile_tick(CPU_PROFILING, regs);
1199 update_process_times(user_mode_vm(regs));
1203 * We take the 'long' return path, and there every subsystem
1204 * grabs the apropriate locks (kernel lock/ irq lock).
1206 * we might want to decouple profiling from the 'long path',
1207 * and do the profiling totally in assembly.
1209 * Currently this isn't too much of an issue (performance wise),
1210 * we can take more than 100K local irqs per second on a 100 MHz P5.
1215 * Local APIC timer interrupt. This is the most natural way for doing
1216 * local interrupts, but local timer interrupts can be emulated by
1217 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
1219 * [ if a single-CPU system runs an SMP kernel then we call the local
1220 * interrupt as well. Thus we cannot inline the local irq ... ]
1223 fastcall void smp_apic_timer_interrupt(struct pt_regs *regs)
1225 int cpu = smp_processor_id();
1228 * the NMI deadlock-detector uses this.
1230 per_cpu(irq_stat, cpu).apic_timer_irqs++;
1233 * NOTE! We'd better ACK the irq immediately,
1234 * because timer handling can be slow.
1238 * update_process_times() expects us to have done irq_enter().
1239 * Besides, if we don't timer interrupts ignore the global
1240 * interrupt lock, which is the WrongThing (tm) to do.
1243 smp_local_timer_interrupt(regs);
1248 static void up_apic_timer_interrupt_call(struct pt_regs *regs)
1250 int cpu = smp_processor_id();
1253 * the NMI deadlock-detector uses this.
1255 per_cpu(irq_stat, cpu).apic_timer_irqs++;
1257 smp_local_timer_interrupt(regs);
1261 void smp_send_timer_broadcast_ipi(struct pt_regs *regs)
1265 cpus_and(mask, cpu_online_map, timer_bcast_ipi);
1266 if (!cpus_empty(mask)) {
1268 send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
1271 * We can directly call the apic timer interrupt handler
1272 * in UP case. Minus all irq related functions
1274 up_apic_timer_interrupt_call(regs);
1279 int setup_profiling_timer(unsigned int multiplier)
1285 * This interrupt should _never_ happen with our APIC/SMP architecture
1287 fastcall void smp_spurious_interrupt(struct pt_regs *regs)
1293 * Check if this really is a spurious interrupt and ACK it
1294 * if it is a vectored one. Just in case...
1295 * Spurious interrupts should not be ACKed.
1297 v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
1298 if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
1301 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
1302 printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n",
1303 smp_processor_id());
1308 * This interrupt should never happen with our APIC/SMP architecture
1311 fastcall void smp_error_interrupt(struct pt_regs *regs)
1313 unsigned long v, v1;
1316 /* First tickle the hardware, only then report what went on. -- REW */
1317 v = apic_read(APIC_ESR);
1318 apic_write(APIC_ESR, 0);
1319 v1 = apic_read(APIC_ESR);
1321 atomic_inc(&irq_err_count);
1323 /* Here is what the APIC error bits mean:
1326 2: Send accept error
1327 3: Receive accept error
1329 5: Send illegal vector
1330 6: Received illegal vector
1331 7: Illegal register address
1333 printk (KERN_DEBUG "APIC error on CPU%d: %02lx(%02lx)\n",
1334 smp_processor_id(), v , v1);
1338 #ifdef CONFIG_X86_APIC_AUTO
1340 /* Some heuristics to decide when to enable the APICs */
1342 static __init int dmi_enable_apic(void)
1348 /* If the machine has more than one CPU try to use APIC because it'll
1349 be running the SMP kernel with APIC soon anyways.
1350 This won't cover dual core, but they are handled by the date check
1355 year = dmi_get_year(DMI_BIOS_DATE);
1356 vendor = dmi_get_system_info(DMI_BIOS_VENDOR);
1359 /* All Intel BIOS since 1998 assumed APIC on. Don't include 1998 itself
1360 because we're not sure for that. */
1361 if (vendor && !strncmp(vendor, "Intel", 5))
1363 /* Use APIC for anything since 2001 */
1364 else if (year >= 2001)
1368 /* When ACPI is disabled also default to APIC off on very new systems (>= 2004)
1369 which typically don't have working mptables anymore */
1370 if (acpi_noirq && year >= 2004)
1375 apic_disabled_by_dmi = 1;
1380 void __init dmi_check_apic(void)
1382 if (enable_local_apic != 0 || prefer_apic)
1384 if (!dmi_enable_apic()) {
1385 clear_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability);
1387 enable_local_apic = -1;
1388 printk(KERN_INFO "IO/L-APIC disabled because your old system seems to be old\n");
1389 printk(KERN_INFO "overwrite with \"apic\"\n");
1392 printk(KERN_INFO "IO/L-APIC allowed because system is MP or new enough\n");
1397 * This initializes the IO-APIC and APIC hardware if this is
1400 int __init APIC_init_uniprocessor (void)
1402 if (enable_local_apic < 0)
1403 clear_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability);
1405 if (!smp_found_config && !cpu_has_apic)
1409 * Complain if the BIOS pretends there is one.
1411 if (!cpu_has_apic && APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
1412 printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
1413 boot_cpu_physical_apicid);
1414 clear_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability);
1418 verify_local_APIC();
1423 * Hack: In case of kdump, after a crash, kernel might be booting
1424 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
1425 * might be zero if read from MP tables. Get it from LAPIC.
1427 #ifdef CONFIG_CRASH_DUMP
1428 boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
1430 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid);
1434 #ifdef CONFIG_X86_IO_APIC
1435 if (smp_found_config)
1436 if (!skip_ioapic_setup && nr_ioapics)
1439 setup_boot_APIC_clock();