2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
12 * entry.S contains the system-call and fault low-level handling routines.
14 * NOTE: This code handles signal-recognition, which happens every time
15 * after an interrupt and after each system call.
17 * Normal syscalls and interrupts don't save a full stack frame, this is
18 * only done for syscall tracing, signals or fork/exec et.al.
20 * A note on terminology:
21 * - top of stack: Architecture defined interrupt frame from SS to RIP
22 * at the top of the kernel process stack.
23 * - partial stack frame: partially saved registers upto R11.
24 * - full stack frame: Like partial stack frame, but all register saved.
27 * - schedule it carefully for the final hardware.
31 #include <linux/config.h>
32 #include <linux/linkage.h>
33 #include <asm/segment.h>
35 #include <asm/cache.h>
36 #include <asm/errno.h>
37 #include <asm/dwarf2.h>
38 #include <asm/calling.h>
39 #include <asm/offset.h>
41 #include <asm/unistd.h>
42 #include <asm/thread_info.h>
43 #include <asm/hw_irq.h>
44 #include <asm/errno.h>
49 #define preempt_stop cli
52 #define retint_kernel retint_restore_args
56 * C code is not supposed to know about undefined top of stack. Every time
57 * a C function with an pt_regs argument is called from the SYSCALL based
58 * fast path FIXUP_TOP_OF_STACK is needed.
59 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
63 /* %rsp:at FRAMEEND */
64 .macro FIXUP_TOP_OF_STACK tmp
65 movq %gs:pda_oldrsp,\tmp
67 movq $__USER_DS,SS(%rsp)
68 movq $__USER_CS,CS(%rsp)
70 movq R11(%rsp),\tmp /* get eflags */
71 movq \tmp,EFLAGS(%rsp)
74 .macro RESTORE_TOP_OF_STACK tmp,offset=0
75 movq RSP-\offset(%rsp),\tmp
76 movq \tmp,%gs:pda_oldrsp
77 movq EFLAGS-\offset(%rsp),\tmp
78 movq \tmp,R11-\offset(%rsp)
81 .macro FAKE_STACK_FRAME child_rip
82 /* push in order ss, rsp, eflags, cs, rip */
85 CFI_ADJUST_CFA_OFFSET 8
87 CFI_ADJUST_CFA_OFFSET 8
89 pushq $(1<<9) /* eflags - interrupts on */
90 CFI_ADJUST_CFA_OFFSET 8
91 pushq $__KERNEL_CS /* cs */
92 CFI_ADJUST_CFA_OFFSET 8
93 pushq \child_rip /* rip */
94 CFI_ADJUST_CFA_OFFSET 8
96 pushq %rax /* orig rax */
97 CFI_ADJUST_CFA_OFFSET 8
100 .macro UNFAKE_STACK_FRAME
102 CFI_ADJUST_CFA_OFFSET -(6*8)
105 .macro CFI_DEFAULT_STACK
106 CFI_ADJUST_CFA_OFFSET (SS)
107 CFI_OFFSET r15,R15-SS
108 CFI_OFFSET r14,R14-SS
109 CFI_OFFSET r13,R13-SS
110 CFI_OFFSET r12,R12-SS
111 CFI_OFFSET rbp,RBP-SS
112 CFI_OFFSET rbx,RBX-SS
113 CFI_OFFSET r11,R11-SS
114 CFI_OFFSET r10,R10-SS
117 CFI_OFFSET rax,RAX-SS
118 CFI_OFFSET rcx,RCX-SS
119 CFI_OFFSET rdx,RDX-SS
120 CFI_OFFSET rsi,RSI-SS
121 CFI_OFFSET rdi,RDI-SS
122 CFI_OFFSET rsp,RSP-SS
123 CFI_OFFSET rip,RIP-SS
126 * A newly forked process directly context switches into this.
133 GET_THREAD_INFO(%rcx)
134 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
138 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
139 je int_ret_from_sys_call
140 testl $_TIF_IA32,threadinfo_flags(%rcx)
141 jnz int_ret_from_sys_call
142 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
143 jmp ret_from_sys_call
146 call syscall_trace_leave
147 GET_THREAD_INFO(%rcx)
152 * System call entry. Upto 6 arguments in registers are supported.
154 * SYSCALL does not save anything on the stack and does not change the
160 * rax system call number
162 * rcx return address for syscall/sysret, C arg3
165 * r10 arg3 (--> moved to rcx for C)
168 * r11 eflags for syscall/sysret, temporary for C
169 * r12-r15,rbp,rbx saved by C code, not touched.
171 * Interrupts are off on entry.
172 * Only called from user space.
174 * XXX if we had a free scratch register we could save the RSP into the stack frame
175 * and report it properly in ps. Unfortunately we haven't.
181 movq %rsp,%gs:pda_oldrsp
182 movq %gs:pda_kernelstack,%rsp
185 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
186 movq %rcx,RIP-ARGOFFSET(%rsp)
187 GET_THREAD_INFO(%rcx)
188 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
190 cmpq $__NR_syscall_max,%rax
193 call *sys_call_table(,%rax,8) # XXX: rip relative
194 movq %rax,RAX-ARGOFFSET(%rsp)
196 * Syscall return path ending with SYSRET (fast path)
197 * Has incomplete stack frame and undefined top of stack.
199 .globl ret_from_sys_call
201 movl $_TIF_WORK_MASK,%edi
204 GET_THREAD_INFO(%rcx)
206 movl threadinfo_flags(%rcx),%edx
209 movq RIP-ARGOFFSET(%rsp),%rcx
210 RESTORE_ARGS 0,-ARG_SKIP,1
211 movq %gs:pda_oldrsp,%rsp
215 /* Handle reschedules */
216 /* edx: work, edi: workmask */
218 bt $TIF_NEED_RESCHED,%edx
226 /* Handle a signal */
229 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
232 /* Really a signal */
233 /* edx: work flags (arg3) */
234 leaq do_notify_resume(%rip),%rax
235 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
236 xorl %esi,%esi # oldset -> arg2
237 call ptregscall_common
238 1: movl $_TIF_NEED_RESCHED,%edi
241 /* Do syscall tracing */
244 movq $-ENOSYS,RAX(%rsp)
245 FIXUP_TOP_OF_STACK %rdi
247 call syscall_trace_enter
248 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
250 cmpq $__NR_syscall_max,%rax
252 movq %r10,%rcx /* fixup for C */
253 call *sys_call_table(,%rax,8)
254 movq %rax,RAX-ARGOFFSET(%rsp)
257 call syscall_trace_leave
258 RESTORE_TOP_OF_STACK %rbx
260 jmp ret_from_sys_call
263 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
264 jmp ret_from_sys_call
267 * Syscall return path ending with IRET.
268 * Has correct top of stack, but partial stack frame.
270 ENTRY(int_ret_from_sys_call)
272 testl $3,CS-ARGOFFSET(%rsp)
273 je retint_restore_args
274 movl $_TIF_ALLWORK_MASK,%edi
275 /* edi: mask to check */
277 GET_THREAD_INFO(%rcx)
278 movl threadinfo_flags(%rcx),%edx
283 /* Either reschedule or signal or syscall exit tracking needed. */
284 /* First do a reschedule test. */
285 /* edx: work, edi: workmask */
287 bt $TIF_NEED_RESCHED,%edx
295 /* handle signals and tracing -- both require a full stack frame */
299 /* Check for syscall exit trace */
300 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),%edx
303 leaq 8(%rsp),%rdi # &ptregs -> arg1
304 call syscall_trace_leave
306 btr $TIF_SYSCALL_TRACE,%edi
307 btr $TIF_SYSCALL_AUDIT,%edi
311 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
313 movq %rsp,%rdi # &ptregs -> arg1
314 xorl %esi,%esi # oldset -> arg2
315 call do_notify_resume
316 1: movl $_TIF_NEED_RESCHED,%edi
323 * Certain special system calls that need to save a complete full stack frame.
326 .macro PTREGSCALL label,func
329 leaq \func(%rip),%rax
330 jmp ptregscall_common
333 PTREGSCALL stub_clone, sys_clone
334 PTREGSCALL stub_fork, sys_fork
335 PTREGSCALL stub_vfork, sys_vfork
336 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend
337 PTREGSCALL stub_sigaltstack, sys_sigaltstack
338 PTREGSCALL stub_iopl, sys_iopl
340 ENTRY(ptregscall_common)
343 CFI_ADJUST_CFA_OFFSET -8
346 FIXUP_TOP_OF_STACK %r11
348 RESTORE_TOP_OF_STACK %r11
352 CFI_ADJUST_CFA_OFFSET 8
359 CFI_ADJUST_CFA_OFFSET -8
362 FIXUP_TOP_OF_STACK %r11
364 GET_THREAD_INFO(%rcx)
365 bt $TIF_IA32,threadinfo_flags(%rcx)
367 RESTORE_TOP_OF_STACK %r11
374 CFI_ADJUST_CFA_OFFSET REST_SKIP
377 jmp int_ret_from_sys_call
381 * sigreturn is special because it needs to restore all registers on return.
382 * This cannot be done with SYSRET, so use the IRET return path instead.
384 ENTRY(stub_rt_sigreturn)
388 FIXUP_TOP_OF_STACK %r11
389 call sys_rt_sigreturn
390 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
392 jmp int_ret_from_sys_call
396 * Interrupt entry/exit.
398 * Interrupt entry points save only callee clobbered registers in fast path.
400 * Entry runs with interrupts off.
403 /* 0(%rsp): interrupt number */
404 .macro interrupt func
406 CFI_DEF_CFA rsp,(SS-RDI)
407 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
408 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
410 #ifdef CONFIG_DEBUG_INFO
414 * Setup a stack frame pointer. This allows gdb to trace
415 * back to the original stack.
418 CFI_DEF_CFA_REGISTER rbp
421 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
426 1: addl $1,%gs:pda_irqcount # RED-PEN should check preempt count
427 movq %gs:pda_irqstackptr,%rax
429 pushq %rdi # save old stack
433 ENTRY(common_interrupt)
435 /* 0(%rsp): oldrsp-ARGOFFSET */
439 subl $1,%gs:pda_irqcount
440 #ifdef CONFIG_DEBUG_INFO
443 leaq ARGOFFSET(%rdi),%rsp
445 GET_THREAD_INFO(%rcx)
446 testl $3,CS-ARGOFFSET(%rsp)
449 /* Interrupt came from user space */
451 * Has a correct top of stack, but a partial stack frame
452 * %rcx: thread info. Interrupts off.
454 retint_with_reschedule:
455 movl $_TIF_WORK_MASK,%edi
457 movl threadinfo_flags(%rcx),%edx
469 .section __ex_table,"a"
470 .quad iret_label,bad_iret
473 /* force a signal here? this matches i386 behaviour */
474 /* running with kernel gs */
476 movq $-9999,%rdi /* better code? */
480 /* edi: workmask, edx: work */
482 bt $TIF_NEED_RESCHED,%edx
488 GET_THREAD_INFO(%rcx)
493 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
497 movq $-1,ORIG_RAX(%rsp)
498 xorq %rsi,%rsi # oldset
499 movq %rsp,%rdi # &pt_regs
500 call do_notify_resume
503 movl $_TIF_NEED_RESCHED,%edi
504 GET_THREAD_INFO(%rcx)
507 #ifdef CONFIG_PREEMPT
508 /* Returning to kernel space. Check if we need preemption */
509 /* rcx: threadinfo. interrupts off. */
512 cmpl $0,threadinfo_preempt_count(%rcx)
513 jnz retint_restore_args
514 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
515 jnc retint_restore_args
516 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
517 jc retint_restore_args
518 movl $PREEMPT_ACTIVE,threadinfo_preempt_count(%rcx)
522 GET_THREAD_INFO(%rcx)
523 movl $0,threadinfo_preempt_count(%rcx)
531 .macro apicinterrupt num,func
539 ENTRY(reschedule_interrupt)
540 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
542 ENTRY(invalidate_interrupt)
543 apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
545 ENTRY(call_function_interrupt)
546 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
549 #ifdef CONFIG_X86_LOCAL_APIC
550 ENTRY(apic_timer_interrupt)
551 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
553 ENTRY(error_interrupt)
554 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
556 ENTRY(spurious_interrupt)
557 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
561 * Exception entry points.
564 pushq $0 /* push error code/oldrax */
565 pushq %rax /* push real oldrax to the rdi slot */
570 .macro errorentry sym
576 /* error code is on the stack already */
577 /* handle NMI like exceptions that can happen everywhere */
578 .macro paranoidentry sym
582 movl $MSR_GS_BASE,%ecx
589 movq ORIG_RAX(%rsp),%rsi
590 movq $-1,ORIG_RAX(%rsp)
595 * Exception entry point. This expects an error code/orig_rax on the stack
596 * and the exception handler in %rax.
600 CFI_DEF_CFA rsp,(SS-RDI)
601 CFI_REL_OFFSET rsp,(RSP-RDI)
602 CFI_REL_OFFSET rip,(RIP-RDI)
603 /* rdi slot contains rax, oldrax contains error code */
606 CFI_ADJUST_CFA_OFFSET (14*8)
608 CFI_REL_OFFSET rsi,RSI
609 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
611 CFI_REL_OFFSET rdx,RDX
613 CFI_REL_OFFSET rcx,RCX
614 movq %rsi,10*8(%rsp) /* store rax */
615 CFI_REL_OFFSET rax,RAX
621 CFI_REL_OFFSET r10,R10
623 CFI_REL_OFFSET r11,R11
625 CFI_REL_OFFSET rbx,RBX
627 CFI_REL_OFFSET rbp,RBP
629 CFI_REL_OFFSET r12,R12
631 CFI_REL_OFFSET r13,R13
633 CFI_REL_OFFSET r14,R14
635 CFI_REL_OFFSET r15,R15
644 movq ORIG_RAX(%rsp),%rsi /* get error code */
645 movq $-1,ORIG_RAX(%rsp)
647 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
652 GET_THREAD_INFO(%rcx)
655 movl threadinfo_flags(%rcx),%edx
656 movl $_TIF_WORK_MASK,%edi
666 /* There are two places in the kernel that can potentially fault with
667 usergs. Handle them here. The exception handlers after
668 iret run with kernel gs again, so don't set the user space flag.
669 B stepping K8s sometimes report an truncated RIP for IRET
670 exceptions returning to compat mode. Check for these here too. */
671 leaq iret_label(%rip),%rbp
674 movl %ebp,%ebp /* zero extend */
677 cmpq $gs_change,RIP(%rsp)
681 /* Reload gs selector with exception handling */
682 /* edi: new selector */
689 2: mfence /* workaround */
694 .section __ex_table,"a"
696 .quad gs_change,bad_gs
699 /* running with kernelgs */
701 swapgs /* switch back to user gs */
708 * Create a kernel thread.
710 * C extern interface:
711 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
713 * asm input arguments:
714 * rdi: fn, rsi: arg, rdx: flags
718 FAKE_STACK_FRAME $child_rip
721 # rdi: flags, rsi: usp, rdx: will be &pt_regs
723 orq kernel_thread_flags(%rip),%rdi
736 * It isn't worth to check for reschedule here,
737 * so internally to the x86_64 port you can rely on kernel_thread()
738 * not to reschedule the child before returning, this avoids the need
739 * of hacks for example to fork off the per-CPU idle tasks.
740 * [Hopefully no generic code relies on the reschedule -AK]
750 * Here we are in the child and the registers are set as they were
751 * at kernel_thread() invocation in the parent.
761 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
763 * C extern interface:
764 * extern long execve(char *name, char **argv, char **envp)
766 * asm input arguments:
767 * rdi: name, rsi: argv, rdx: envp
769 * We want to fallback into:
770 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
772 * do_sys_execve asm fallback arguments:
773 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
783 je int_ret_from_sys_call
790 errorentry do_page_fault
792 ENTRY(coprocessor_error)
793 zeroentry do_coprocessor_error
795 ENTRY(simd_coprocessor_error)
796 zeroentry do_simd_coprocessor_error
798 ENTRY(device_not_available)
799 zeroentry math_state_restore
801 /* runs on exception stack */
805 CFI_ADJUST_CFA_OFFSET 8
806 paranoidentry do_debug
807 /* switch back to process stack to restore the state ptrace touched */
810 jnz paranoid_userspace
814 /* runs on exception stack */
818 CFI_ADJUST_CFA_OFFSET 8
820 /* ebx: no swapgs flag */
822 testl %ebx,%ebx /* swapgs needed? */
832 GET_THREAD_INFO(%rcx)
833 movl threadinfo_flags(%rcx),%edx
834 testl $_TIF_NEED_RESCHED,%edx
836 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
845 xorl %esi,%esi /* oldset */
846 movq %rsp,%rdi /* &pt_regs */
847 call do_notify_resume
855 zeroentry do_overflow
861 zeroentry do_invalid_op
863 ENTRY(coprocessor_segment_overrun)
864 zeroentry do_coprocessor_segment_overrun
867 zeroentry do_reserved
869 /* runs on exception stack */
872 paranoidentry do_double_fault
875 jnz paranoid_userspace
880 errorentry do_invalid_TSS
882 ENTRY(segment_not_present)
883 errorentry do_segment_not_present
885 /* runs on exception stack */
888 paranoidentry do_stack_segment
891 jnz paranoid_userspace
895 ENTRY(general_protection)
896 errorentry do_general_protection
898 ENTRY(alignment_check)
899 errorentry do_alignment_check
902 zeroentry do_divide_error
904 ENTRY(spurious_interrupt_bug)
905 zeroentry do_spurious_interrupt_bug
907 /* runs on exception stack */
911 CFI_ADJUST_CFA_OFFSET 8
912 paranoidentry do_machine_check
917 zeroentry do_call_debug