2 * linux/arch/x86-64/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
8 #include <linux/config.h>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/smp.h>
19 #include <linux/smp_lock.h>
20 #include <linux/interrupt.h>
21 #include <linux/init.h>
22 #include <linux/tty.h>
23 #include <linux/vt_kern.h> /* For unblank_screen() */
24 #include <linux/compiler.h>
25 #include <linux/module.h>
27 #include <asm/system.h>
28 #include <asm/uaccess.h>
29 #include <asm/pgalloc.h>
30 #include <asm/hardirq.h>
32 #include <asm/tlbflush.h>
33 #include <asm/proto.h>
34 #include <asm/kdebug.h>
35 #include <asm-generic/sections.h>
37 void bust_spinlocks(int yes)
39 int loglevel_save = console_loglevel;
48 * OK, the message is on the console. Now we call printk()
49 * without oops_in_progress set so that printk will give klogd
50 * a poke. Hold onto your hats...
52 console_loglevel = 15; /* NMI oopser may have shut the console up */
54 console_loglevel = loglevel_save;
58 /* Sometimes the CPU reports invalid exceptions on prefetch.
59 Check that here and ignore.
60 Opcode checker based on code by Richard Brunner */
61 static int is_prefetch(struct pt_regs *regs, unsigned long addr)
63 unsigned char *instr = (unsigned char *)(regs->rip);
66 unsigned char *max_instr = instr + 15;
68 /* Avoid recursive faults for this common case */
69 if (regs->rip == addr)
72 /* Code segments in LDT could have a non zero base. Don't check
73 when that's possible */
74 if (regs->cs & (1<<2))
77 if ((regs->cs & 3) != 0 && regs->rip >= TASK_SIZE)
80 while (scan_more && instr < max_instr) {
82 unsigned char instr_hi;
83 unsigned char instr_lo;
85 if (__get_user(opcode, instr))
88 instr_hi = opcode & 0xf0;
89 instr_lo = opcode & 0x0f;
95 /* Values 0x26,0x2E,0x36,0x3E are valid x86
96 prefixes. In long mode, the CPU will signal
97 invalid opcode if some of these prefixes are
98 present so we will never get here anyway */
99 scan_more = ((instr_lo & 7) == 0x6);
103 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
104 Need to figure out under what instruction mode the
105 instruction was issued ... */
106 /* Could check the LDT for lm, but for now it's good
107 enough to assume that long mode only uses well known
108 segments or kernel. */
109 scan_more = ((regs->cs & 3) == 0) || (regs->cs == __USER_CS);
113 /* 0x64 thru 0x67 are valid prefixes in all modes. */
114 scan_more = (instr_lo & 0xC) == 0x4;
117 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
118 scan_more = !instr_lo || (instr_lo>>1) == 1;
121 /* Prefetch instruction is 0x0F0D or 0x0F18 */
123 if (__get_user(opcode, instr))
125 prefetch = (instr_lo == 0xF) &&
126 (opcode == 0x0D || opcode == 0x18);
136 static int bad_address(void *p)
139 return __get_user(dummy, (unsigned long *)p);
142 void dump_pagetable(unsigned long address)
149 asm("movq %%cr3,%0" : "=r" (pml4));
151 pml4 = __va((unsigned long)pml4 & PHYSICAL_PAGE_MASK);
152 pml4 += pml4_index(address);
153 printk("PML4 %lx ", pml4_val(*pml4));
154 if (bad_address(pml4)) goto bad;
155 if (!pml4_present(*pml4)) goto ret;
157 pgd = __pgd_offset_k((pgd_t *)pml4_page(*pml4), address);
158 if (bad_address(pgd)) goto bad;
159 printk("PGD %lx ", pgd_val(*pgd));
160 if (!pgd_present(*pgd)) goto ret;
162 pmd = pmd_offset(pgd, address);
163 if (bad_address(pmd)) goto bad;
164 printk("PMD %lx ", pmd_val(*pmd));
165 if (!pmd_present(*pmd)) goto ret;
167 pte = pte_offset_kernel(pmd, address);
168 if (bad_address(pte)) goto bad;
169 printk("PTE %lx", pte_val(*pte));
177 static const char errata93_warning[] =
178 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
179 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
180 KERN_ERR "******* Please consider a BIOS update.\n"
181 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
183 /* Workaround for K8 erratum #93 & buggy BIOS.
184 BIOS SMM functions are required to use a specific workaround
185 to avoid corruption of the 64bit RIP register on C stepping K8.
186 A lot of BIOS that didn't get tested properly miss this.
187 The OS sees this as a page fault with the upper 32bits of RIP cleared.
188 Try to work around it here.
189 Note we only handle faults in kernel here. */
191 static int is_errata93(struct pt_regs *regs, unsigned long address)
194 if (address != regs->rip)
196 if ((address >> 32) != 0)
198 address |= 0xffffffffUL << 32;
199 if ((address >= (u64)_stext && address <= (u64)_etext) ||
200 (address >= MODULES_VADDR && address <= MODULES_END)) {
202 printk(errata93_warning);
211 int unhandled_signal(struct task_struct *tsk, int sig)
213 /* Warn for strace, but not for gdb */
214 if (!test_ti_thread_flag(tsk->thread_info, TIF_SYSCALL_TRACE) &&
215 (tsk->ptrace & PT_PTRACED))
217 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
218 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
221 int page_fault_trace;
222 int exception_trace = 1;
225 * This routine handles page faults. It determines the address,
226 * and the problem, and then passes it off to one of the appropriate
230 * bit 0 == 0 means no page found, 1 means protection fault
231 * bit 1 == 0 means read, 1 means write
232 * bit 2 == 0 means kernel, 1 means user-mode
233 * bit 3 == 1 means fault was an instruction fetch
235 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
237 struct task_struct *tsk;
238 struct mm_struct *mm;
239 struct vm_area_struct * vma;
240 unsigned long address;
241 const struct exception_table_entry *fixup;
245 #ifdef CONFIG_CHECKING
248 struct x8664_pda *pda = cpu_pda + stack_smp_processor_id();
249 rdmsrl(MSR_GS_BASE, gs);
250 if (gs != (unsigned long)pda) {
251 wrmsrl(MSR_GS_BASE, pda);
252 printk("page_fault: wrong gs %lx expected %p\n", gs, pda);
257 /* get the address */
258 __asm__("movq %%cr2,%0":"=r" (address));
260 if (likely(regs->eflags & X86_EFLAGS_IF))
263 if (unlikely(page_fault_trace))
264 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
265 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
269 info.si_code = SEGV_MAPERR;
271 /* 5 => page not present and from supervisor mode */
272 if (unlikely(!(error_code & 5) &&
273 ((address >= VMALLOC_START && address <= VMALLOC_END) ||
274 (address >= MODULES_VADDR && address <= MODULES_END))))
278 * If we're in an interrupt or have no user
279 * context, we must not take the fault..
281 if (unlikely(in_atomic() || !mm))
282 goto bad_area_nosemaphore;
285 down_read(&mm->mmap_sem);
287 vma = find_vma(mm, address);
290 if (likely(vma->vm_start <= address))
292 if (!(vma->vm_flags & VM_GROWSDOWN))
294 if (error_code & 4) {
295 // XXX: align red zone size with ABI
296 if (address + 128 < regs->rsp)
299 if (expand_stack(vma, address))
302 * Ok, we have a good vm_area for this memory access, so
306 info.si_code = SEGV_ACCERR;
308 switch (error_code & 3) {
309 default: /* 3: write, present */
311 case 2: /* write, not present */
312 if (!(vma->vm_flags & VM_WRITE))
316 case 1: /* read, present */
318 case 0: /* read, not present */
319 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
324 * If for any reason at all we couldn't handle the fault,
325 * make sure we exit gracefully rather than endlessly redo
328 switch (handle_mm_fault(mm, vma, address, write)) {
341 up_read(&mm->mmap_sem);
345 * Something tried to access memory that isn't in our memory map..
346 * Fix it, but check if it's kernel or user first..
349 up_read(&mm->mmap_sem);
351 bad_area_nosemaphore:
353 #ifdef CONFIG_IA32_EMULATION
354 /* 32bit vsyscall. map on demand. */
355 if (test_thread_flag(TIF_IA32) &&
356 address >= 0xffffe000 && address < 0xffffe000 + PAGE_SIZE) {
357 if (map_syscall32(mm, address) < 0)
363 /* User mode accesses just cause a SIGSEGV */
364 if (error_code & 4) {
365 if (is_prefetch(regs, address))
368 /* Work around K8 erratum #100 K8 in compat mode
369 occasionally jumps to illegal addresses >4GB. We
370 catch this here in the page fault handler because
371 these addresses are not reachable. Just detect this
372 case and return. Any code segment in LDT is
373 compatibility mode. */
374 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
378 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
380 "%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
381 tsk->comm, tsk->pid, address, regs->rip,
382 regs->rsp, error_code);
385 tsk->thread.cr2 = address;
386 /* Kernel addresses are always protection faults */
387 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
388 tsk->thread.trap_no = 14;
389 info.si_signo = SIGSEGV;
391 /* info.si_code has been set above */
392 info.si_addr = (void *)address;
393 force_sig_info(SIGSEGV, &info, tsk);
399 /* Are we prepared to handle this kernel fault? */
400 fixup = search_exception_tables(regs->rip);
402 regs->rip = fixup->fixup;
407 * Hall of shame of CPU/BIOS bugs.
410 if (is_prefetch(regs, address))
413 if (is_errata93(regs, address))
417 * Oops. The kernel tried to access some bad page. We'll have to
418 * terminate things with extreme prejudice.
423 if (address < PAGE_SIZE)
424 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
426 printk(KERN_ALERT "Unable to handle kernel paging request");
427 printk(" at %016lx RIP: \n" KERN_ALERT,address);
428 printk_address(regs->rip);
430 dump_pagetable(address);
431 __die("Oops", regs, error_code);
432 /* Executive summary in case the body of the oops scrolled away */
433 printk(KERN_EMERG "CR2: %016lx\n", address);
438 * We ran out of memory, or some other thing happened to us that made
439 * us unable to handle the page fault gracefully.
442 up_read(&mm->mmap_sem);
444 if (current->pid == 1) {
448 printk("VM: killing process %s\n", tsk->comm);
454 up_read(&mm->mmap_sem);
456 /* Kernel mode? Handle exceptions or die */
457 if (!(error_code & 4))
460 tsk->thread.cr2 = address;
461 tsk->thread.error_code = error_code;
462 tsk->thread.trap_no = 14;
463 info.si_signo = SIGBUS;
465 info.si_code = BUS_ADRERR;
466 info.si_addr = (void *)address;
467 force_sig_info(SIGBUS, &info, tsk);
477 * x86-64 has the same kernel 3rd level pages for all CPUs.
478 * But for vmalloc/modules the TLB synchronization works lazily,
479 * so it can happen that we get a page fault for something
480 * that is really already in the page table. Just check if it
481 * is really there and when yes flush the local TLB.
483 pgd = pgd_offset_k(address);
484 if (pgd != current_pgd_offset_k(address))
486 if (!pgd_present(*pgd))
487 goto bad_area_nosemaphore;
488 pmd = pmd_offset(pgd, address);
489 if (!pmd_present(*pmd))
490 goto bad_area_nosemaphore;
491 pte = pte_offset_kernel(pmd, address);
492 if (!pte_present(*pte))
493 goto bad_area_nosemaphore;