2 * linux/arch/i386/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
40 * "current" is in register %ebx during any slow entries.
43 #include <linux/linkage.h>
44 #include <asm/thread_info.h>
45 #include <asm/irqflags.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
51 #include <asm/dwarf2.h>
52 #include "irq_vectors.h"
54 #define nr_syscalls ((syscall_table_size)/4)
80 #define preempt_stop cli; TRACE_IRQS_OFF
83 #define resume_kernel restore_nocheck
86 .macro TRACE_IRQS_IRET
87 #ifdef CONFIG_TRACE_IRQFLAGS
88 testl $IF_MASK,EFLAGS(%esp) # interrupts off?
96 #define resume_userspace_sig check_userspace
98 #define resume_userspace_sig resume_userspace
104 CFI_ADJUST_CFA_OFFSET 4;\
105 /*CFI_REL_OFFSET es, 0;*/\
107 CFI_ADJUST_CFA_OFFSET 4;\
108 /*CFI_REL_OFFSET ds, 0;*/\
110 CFI_ADJUST_CFA_OFFSET 4;\
111 CFI_REL_OFFSET eax, 0;\
113 CFI_ADJUST_CFA_OFFSET 4;\
114 CFI_REL_OFFSET ebp, 0;\
116 CFI_ADJUST_CFA_OFFSET 4;\
117 CFI_REL_OFFSET edi, 0;\
119 CFI_ADJUST_CFA_OFFSET 4;\
120 CFI_REL_OFFSET esi, 0;\
122 CFI_ADJUST_CFA_OFFSET 4;\
123 CFI_REL_OFFSET edx, 0;\
125 CFI_ADJUST_CFA_OFFSET 4;\
126 CFI_REL_OFFSET ecx, 0;\
128 CFI_ADJUST_CFA_OFFSET 4;\
129 CFI_REL_OFFSET ebx, 0;\
130 movl $(__USER_DS), %edx; \
134 #define RESTORE_INT_REGS \
136 CFI_ADJUST_CFA_OFFSET -4;\
139 CFI_ADJUST_CFA_OFFSET -4;\
142 CFI_ADJUST_CFA_OFFSET -4;\
145 CFI_ADJUST_CFA_OFFSET -4;\
148 CFI_ADJUST_CFA_OFFSET -4;\
151 CFI_ADJUST_CFA_OFFSET -4;\
154 CFI_ADJUST_CFA_OFFSET -4;\
157 #define RESTORE_REGS \
160 CFI_ADJUST_CFA_OFFSET -4;\
163 CFI_ADJUST_CFA_OFFSET -4;\
165 .section .fixup,"ax"; \
171 .section __ex_table,"a";\
177 #define RING0_INT_FRAME \
178 CFI_STARTPROC simple;\
179 CFI_DEF_CFA esp, 3*4;\
180 /*CFI_OFFSET cs, -2*4;*/\
183 #define RING0_EC_FRAME \
184 CFI_STARTPROC simple;\
185 CFI_DEF_CFA esp, 4*4;\
186 /*CFI_OFFSET cs, -2*4;*/\
189 #define RING0_PTREGS_FRAME \
190 CFI_STARTPROC simple;\
191 CFI_DEF_CFA esp, OLDESP-EBX;\
192 /*CFI_OFFSET cs, CS-OLDESP;*/\
193 CFI_OFFSET eip, EIP-OLDESP;\
194 /*CFI_OFFSET es, ES-OLDESP;*/\
195 /*CFI_OFFSET ds, DS-OLDESP;*/\
196 CFI_OFFSET eax, EAX-OLDESP;\
197 CFI_OFFSET ebp, EBP-OLDESP;\
198 CFI_OFFSET edi, EDI-OLDESP;\
199 CFI_OFFSET esi, ESI-OLDESP;\
200 CFI_OFFSET edx, EDX-OLDESP;\
201 CFI_OFFSET ecx, ECX-OLDESP;\
202 CFI_OFFSET ebx, EBX-OLDESP
207 CFI_ADJUST_CFA_OFFSET 4
209 GET_THREAD_INFO(%ebp)
211 CFI_ADJUST_CFA_OFFSET -4
212 pushl $0x0202 # Reset kernel eflags
213 CFI_ADJUST_CFA_OFFSET 4
215 CFI_ADJUST_CFA_OFFSET -4
220 * Return to user mode is not as complex as all this looks,
221 * but we want the default path for a system call return to
222 * go as quickly as possible which is why some of this is
223 * less clear than it otherwise should be.
226 # userspace resumption stub bypassing syscall exit tracing
232 GET_THREAD_INFO(%ebp)
234 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
236 testl $(VM_MASK | 3), %eax
238 ENTRY(resume_userspace)
239 cli # make sure we don't miss an interrupt
240 # setting need_resched or sigpending
241 # between sampling and the iret
242 movl TI_flags(%ebp), %ecx
243 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
244 # int/exception return?
248 #ifdef CONFIG_PREEMPT
251 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
254 movl TI_flags(%ebp), %ecx # need_resched set ?
255 testb $_TIF_NEED_RESCHED, %cl
257 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
259 call preempt_schedule_irq
264 /* SYSENTER_RETURN points to after the "sysenter" instruction in
265 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
267 # sysenter call handler stub
268 ENTRY(sysenter_entry)
271 CFI_REGISTER esp, ebp
272 movl SYSENTER_stack_esp0(%esp),%esp
275 * No need to follow this irqs on/off section: the syscall
276 * disabled irqs and here we enable it straight after entry:
280 CFI_ADJUST_CFA_OFFSET 4
281 /*CFI_REL_OFFSET ss, 0*/
283 CFI_ADJUST_CFA_OFFSET 4
284 CFI_REL_OFFSET esp, 0
286 CFI_ADJUST_CFA_OFFSET 4
288 CFI_ADJUST_CFA_OFFSET 4
289 /*CFI_REL_OFFSET cs, 0*/
291 * Push current_thread_info()->sysenter_return to the stack.
292 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
293 * pushed above; +8 corresponds to copy_thread's esp0 setting.
295 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
296 CFI_ADJUST_CFA_OFFSET 4
297 CFI_REL_OFFSET eip, 0
300 * Load the potential sixth argument from user stack.
301 * Careful about security.
303 cmpl $__PAGE_OFFSET-3,%ebp
306 .section __ex_table,"a"
308 .long 1b,syscall_fault
312 CFI_ADJUST_CFA_OFFSET 4
314 GET_THREAD_INFO(%ebp)
316 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
317 testw $(_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
318 jnz syscall_trace_entry
319 cmpl $(nr_syscalls), %eax
321 call *sys_call_table(,%eax,4)
325 movl TI_flags(%ebp), %ecx
326 testw $_TIF_ALLWORK_MASK, %cx
327 jne syscall_exit_work
328 /* if something modifies registers it must also disable sysexit */
330 movl OLDESP(%esp), %ecx
338 # system call handler stub
340 RING0_INT_FRAME # can't unwind into user space anyway
341 pushl %eax # save orig_eax
342 CFI_ADJUST_CFA_OFFSET 4
344 GET_THREAD_INFO(%ebp)
345 testl $TF_MASK,EFLAGS(%esp)
347 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
349 # system call tracing in operation / emulation
350 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
351 testw $(_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
352 jnz syscall_trace_entry
353 cmpl $(nr_syscalls), %eax
356 call *sys_call_table(,%eax,4)
357 movl %eax,EAX(%esp) # store the return value
359 cli # make sure we don't miss an interrupt
360 # setting need_resched or sigpending
361 # between sampling and the iret
363 movl TI_flags(%ebp), %ecx
364 testw $_TIF_ALLWORK_MASK, %cx # current->work
365 jne syscall_exit_work
368 movl EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
369 # Warning: OLDSS(%esp) contains the wrong/random values if we
370 # are returning to the kernel.
371 # See comments in process.c:copy_thread() for details.
372 movb OLDSS(%esp), %ah
374 andl $(VM_MASK | (4 << 8) | 3), %eax
375 cmpl $((4 << 8) | 3), %eax
377 je ldt_ss # returning to user-space with LDT SS
380 restore_nocheck_notrace:
383 CFI_ADJUST_CFA_OFFSET -4
389 pushl $0 # no error code
393 .section __ex_table,"a"
400 larl OLDSS(%esp), %eax
402 testl $0x00400000, %eax # returning to 32bit stack?
403 jnz restore_nocheck # allright, normal return
404 /* If returning to userspace with 16bit stack,
405 * try to fix the higher word of ESP, as the CPU
407 * This is an "official" bug of all the x86-compatible
408 * CPUs, which we can try to work around to make
409 * dosemu and wine happy. */
410 subl $8, %esp # reserve space for switch16 pointer
411 CFI_ADJUST_CFA_OFFSET 8
415 /* Set up the 16bit stack frame with switch32 pointer on top,
416 * and a switch16 pointer on top of the current frame. */
417 call setup_x86_bogus_stack
418 CFI_ADJUST_CFA_OFFSET -8 # frame has moved
421 lss 20+4(%esp), %esp # switch to 16bit stack
423 .section __ex_table,"a"
429 # perform work that needs to be done immediately before resumption
431 RING0_PTREGS_FRAME # can't unwind into user space anyway
433 testb $_TIF_NEED_RESCHED, %cl
437 cli # make sure we don't miss an interrupt
438 # setting need_resched or sigpending
439 # between sampling and the iret
441 movl TI_flags(%ebp), %ecx
442 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
443 # than syscall tracing?
445 testb $_TIF_NEED_RESCHED, %cl
448 work_notifysig: # deal with pending signals and
449 # notify-resume requests
450 testl $VM_MASK, EFLAGS(%esp)
452 jne work_notifysig_v86 # returning to kernel-space or
455 call do_notify_resume
456 jmp resume_userspace_sig
461 pushl %ecx # save ti_flags for do_notify_resume
462 CFI_ADJUST_CFA_OFFSET 4
463 call save_v86_state # %eax contains pt_regs pointer
465 CFI_ADJUST_CFA_OFFSET -4
468 call do_notify_resume
469 jmp resume_userspace_sig
472 # perform syscall exit tracing
475 movl $-ENOSYS,EAX(%esp)
478 call do_syscall_trace
479 movl ORIG_EAX(%esp), %eax
480 cmpl $(nr_syscalls), %eax
484 # perform syscall exit tracing
487 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
490 sti # could let do_syscall_trace() call
494 call do_syscall_trace
498 RING0_INT_FRAME # can't unwind into user space anyway
500 pushl %eax # save orig_eax
501 CFI_ADJUST_CFA_OFFSET 4
503 GET_THREAD_INFO(%ebp)
504 movl $-EFAULT,EAX(%esp)
508 movl $-ENOSYS,EAX(%esp)
512 #define FIXUP_ESPFIX_STACK \
514 /* switch to 32bit stack using the pointer on top of 16bit stack */ \
515 lss %ss:CPU_16BIT_STACK_SIZE-8, %esp; \
516 /* copy data from 16bit stack to 32bit stack */ \
517 call fixup_x86_bogus_stack; \
518 /* put ESP to the proper location */ \
520 #define UNWIND_ESPFIX_STACK \
522 CFI_ADJUST_CFA_OFFSET 4; \
524 /* see if on 16bit stack */ \
525 cmpw $__ESPFIX_SS, %ax; \
528 CFI_ADJUST_CFA_OFFSET -4; \
529 .section .fixup,"ax"; \
530 28: movl $__KERNEL_DS, %eax; \
533 /* switch to 32bit stack */ \
534 FIXUP_ESPFIX_STACK; \
539 * Build the entry stubs and pointer table with
540 * some assembler magic.
547 ENTRY(irq_entries_start)
552 CFI_ADJUST_CFA_OFFSET -4
555 CFI_ADJUST_CFA_OFFSET 4
564 * the CPU automatically disables interrupts when executing an IRQ vector,
565 * so IRQ-flags tracing has to follow that:
576 #define BUILD_INTERRUPT(name, nr) \
580 CFI_ADJUST_CFA_OFFSET 4; \
588 /* The include is where all of the SMP etc. interrupts come from */
589 #include "entry_arch.h"
593 pushl $0 # no error code
594 CFI_ADJUST_CFA_OFFSET 4
595 pushl $do_divide_error
596 CFI_ADJUST_CFA_OFFSET 4
600 CFI_ADJUST_CFA_OFFSET 4
601 /*CFI_REL_OFFSET ds, 0*/
603 CFI_ADJUST_CFA_OFFSET 4
604 CFI_REL_OFFSET eax, 0
607 CFI_ADJUST_CFA_OFFSET 4
608 CFI_REL_OFFSET ebp, 0
610 CFI_ADJUST_CFA_OFFSET 4
611 CFI_REL_OFFSET edi, 0
613 CFI_ADJUST_CFA_OFFSET 4
614 CFI_REL_OFFSET esi, 0
616 CFI_ADJUST_CFA_OFFSET 4
617 CFI_REL_OFFSET edx, 0
620 CFI_ADJUST_CFA_OFFSET 4
621 CFI_REL_OFFSET ecx, 0
623 CFI_ADJUST_CFA_OFFSET 4
624 CFI_REL_OFFSET ebx, 0
627 CFI_ADJUST_CFA_OFFSET 4
628 /*CFI_REL_OFFSET es, 0*/
631 CFI_ADJUST_CFA_OFFSET -4
632 /*CFI_REGISTER es, ecx*/
633 movl ES(%esp), %edi # get the function address
634 movl ORIG_EAX(%esp), %edx # get the error code
635 movl %eax, ORIG_EAX(%esp)
637 /*CFI_REL_OFFSET es, ES*/
638 movl $(__USER_DS), %ecx
641 movl %esp,%eax # pt_regs pointer
643 jmp ret_from_exception
646 ENTRY(coprocessor_error)
649 CFI_ADJUST_CFA_OFFSET 4
650 pushl $do_coprocessor_error
651 CFI_ADJUST_CFA_OFFSET 4
655 ENTRY(simd_coprocessor_error)
658 CFI_ADJUST_CFA_OFFSET 4
659 pushl $do_simd_coprocessor_error
660 CFI_ADJUST_CFA_OFFSET 4
664 ENTRY(device_not_available)
666 pushl $-1 # mark this as an int
667 CFI_ADJUST_CFA_OFFSET 4
670 testl $0x4, %eax # EM (math emulation bit)
671 jne device_not_available_emulate
673 call math_state_restore
674 jmp ret_from_exception
675 device_not_available_emulate:
676 pushl $0 # temporary storage for ORIG_EIP
677 CFI_ADJUST_CFA_OFFSET 4
680 CFI_ADJUST_CFA_OFFSET -4
681 jmp ret_from_exception
685 * Debug traps and NMI can happen at the one SYSENTER instruction
686 * that sets up the real kernel stack. Check here, since we can't
687 * allow the wrong stack to be used.
689 * "SYSENTER_stack_esp0+12" is because the NMI/debug handler will have
690 * already pushed 3 words if it hits on the sysenter instruction:
691 * eflags, cs and eip.
693 * We just load the right stack, and push the three (known) values
694 * by hand onto the new stack - while updating the return eip past
695 * the instruction that would have done it for sysenter.
697 #define FIX_STACK(offset, ok, label) \
698 cmpw $__KERNEL_CS,4(%esp); \
701 movl SYSENTER_stack_esp0+offset(%esp),%esp; \
702 CFI_DEF_CFA esp, 0; \
705 CFI_ADJUST_CFA_OFFSET 4; \
706 pushl $__KERNEL_CS; \
707 CFI_ADJUST_CFA_OFFSET 4; \
708 pushl $sysenter_past_esp; \
709 CFI_ADJUST_CFA_OFFSET 4; \
710 CFI_REL_OFFSET eip, 0
714 cmpl $sysenter_entry,(%esp)
715 jne debug_stack_correct
716 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
718 pushl $-1 # mark this as an int
719 CFI_ADJUST_CFA_OFFSET 4
721 xorl %edx,%edx # error code 0
722 movl %esp,%eax # pt_regs pointer
724 jmp ret_from_exception
728 * NMI is doubly nasty. It can happen _while_ we're handling
729 * a debug fault, and the debug fault hasn't yet been able to
730 * clear up the stack. So we first check whether we got an
731 * NMI on the sysenter entry path, but after that we need to
732 * check whether we got an NMI on the debug path where the debug
733 * fault happened on the sysenter path.
738 CFI_ADJUST_CFA_OFFSET 4
740 cmpw $__ESPFIX_SS, %ax
742 CFI_ADJUST_CFA_OFFSET -4
744 cmpl $sysenter_entry,(%esp)
747 CFI_ADJUST_CFA_OFFSET 4
749 /* Do not access memory above the end of our stack page,
750 * it might not exist.
752 andl $(THREAD_SIZE-1),%eax
753 cmpl $(THREAD_SIZE-20),%eax
755 CFI_ADJUST_CFA_OFFSET -4
756 jae nmi_stack_correct
757 cmpl $sysenter_entry,12(%esp)
758 je nmi_debug_stack_check
760 /* We have a RING0_INT_FRAME here */
762 CFI_ADJUST_CFA_OFFSET 4
764 xorl %edx,%edx # zero error code
765 movl %esp,%eax # pt_regs pointer
767 jmp restore_nocheck_notrace
772 FIX_STACK(12,nmi_stack_correct, 1)
773 jmp nmi_stack_correct
775 nmi_debug_stack_check:
776 /* We have a RING0_INT_FRAME here */
777 cmpw $__KERNEL_CS,16(%esp)
778 jne nmi_stack_correct
781 cmpl $debug_esp_fix_insn,(%esp)
783 FIX_STACK(24,nmi_stack_correct, 1)
784 jmp nmi_stack_correct
787 /* We have a RING0_INT_FRAME here.
789 * create the pointer to lss back
792 CFI_ADJUST_CFA_OFFSET 4
794 CFI_ADJUST_CFA_OFFSET 4
797 /* copy the iret frame of 12 bytes */
800 CFI_ADJUST_CFA_OFFSET 4
803 CFI_ADJUST_CFA_OFFSET 4
805 FIXUP_ESPFIX_STACK # %eax == %esp
806 CFI_ADJUST_CFA_OFFSET -20 # the frame has now moved
807 xorl %edx,%edx # zero error code
810 lss 12+4(%esp), %esp # back to 16bit stack
813 .section __ex_table,"a"
820 pushl $-1 # mark this as an int
821 CFI_ADJUST_CFA_OFFSET 4
823 xorl %edx,%edx # zero error code
824 movl %esp,%eax # pt_regs pointer
826 jmp ret_from_exception
833 CFI_ADJUST_CFA_OFFSET 4
835 CFI_ADJUST_CFA_OFFSET 4
842 CFI_ADJUST_CFA_OFFSET 4
844 CFI_ADJUST_CFA_OFFSET 4
851 CFI_ADJUST_CFA_OFFSET 4
853 CFI_ADJUST_CFA_OFFSET 4
857 ENTRY(coprocessor_segment_overrun)
860 CFI_ADJUST_CFA_OFFSET 4
861 pushl $do_coprocessor_segment_overrun
862 CFI_ADJUST_CFA_OFFSET 4
868 pushl $do_invalid_TSS
869 CFI_ADJUST_CFA_OFFSET 4
873 ENTRY(segment_not_present)
875 pushl $do_segment_not_present
876 CFI_ADJUST_CFA_OFFSET 4
882 pushl $do_stack_segment
883 CFI_ADJUST_CFA_OFFSET 4
887 KPROBE_ENTRY(general_protection)
889 pushl $do_general_protection
890 CFI_ADJUST_CFA_OFFSET 4
895 ENTRY(alignment_check)
897 pushl $do_alignment_check
898 CFI_ADJUST_CFA_OFFSET 4
902 KPROBE_ENTRY(page_fault)
905 CFI_ADJUST_CFA_OFFSET 4
910 #ifdef CONFIG_X86_MCE
914 CFI_ADJUST_CFA_OFFSET 4
915 pushl machine_check_vector
916 CFI_ADJUST_CFA_OFFSET 4
921 ENTRY(spurious_interrupt_bug)
924 CFI_ADJUST_CFA_OFFSET 4
925 pushl $do_spurious_interrupt_bug
926 CFI_ADJUST_CFA_OFFSET 4
930 #ifdef CONFIG_STACK_UNWIND
931 ENTRY(arch_unwind_init_running)
944 movl $__USER_DS, DS(%edx)
945 movl $__USER_DS, ES(%edx)
946 movl %ebx, ORIG_EAX(%edx)
949 movl $__KERNEL_CS, CS(%edx)
950 movl %ebx, EFLAGS(%edx)
951 movl %eax, OLDESP(%edx)
955 movl $__KERNEL_DS, OLDSS(%edx)
958 ENDPROC(arch_unwind_init_running)
961 ENTRY(kernel_thread_helper)
962 pushl $0 # fake return address for unwinder
966 CFI_ADJUST_CFA_OFFSET 4
969 CFI_ADJUST_CFA_OFFSET 4
972 ENDPROC(kernel_thread_helper)
975 #include "syscall_table.S"
977 syscall_table_size=(.-sys_call_table)