2 * linux/arch/i386/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
40 * "current" is in register %ebx during any slow entries.
43 #include <linux/config.h>
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
51 #include "irq_vectors.h"
53 #define nr_syscalls ((syscall_table_size)/4)
79 #define preempt_stop cli
82 #define resume_kernel restore_nocheck
96 movl $(__USER_DS), %edx; \
100 #define RESTORE_INT_REGS \
109 #define RESTORE_REGS \
113 .section .fixup,"ax"; \
119 .section __ex_table,"a";\
129 GET_THREAD_INFO(%ebp)
134 * Return to user mode is not as complex as all this looks,
135 * but we want the default path for a system call return to
136 * go as quickly as possible which is why some of this is
137 * less clear than it otherwise should be.
140 # userspace resumption stub bypassing syscall exit tracing
145 GET_THREAD_INFO(%ebp)
146 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
148 testl $(VM_MASK | 3), %eax
150 ENTRY(resume_userspace)
151 cli # make sure we don't miss an interrupt
152 # setting need_resched or sigpending
153 # between sampling and the iret
154 movl TI_flags(%ebp), %ecx
155 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
156 # int/exception return?
160 #ifdef CONFIG_PREEMPT
163 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
166 movl TI_flags(%ebp), %ecx # need_resched set ?
167 testb $_TIF_NEED_RESCHED, %cl
169 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
171 call preempt_schedule_irq
175 /* SYSENTER_RETURN points to after the "sysenter" instruction in
176 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
178 # sysenter call handler stub
179 ENTRY(sysenter_entry)
180 movl SYSENTER_stack_esp0(%esp),%esp
188 * Push current_thread_info()->sysenter_return to the stack.
189 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
190 * pushed above; +8 corresponds to copy_thread's esp0 setting.
192 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
194 * Load the potential sixth argument from user stack.
195 * Careful about security.
197 cmpl $__PAGE_OFFSET-3,%ebp
200 .section __ex_table,"a"
202 .long 1b,syscall_fault
207 GET_THREAD_INFO(%ebp)
209 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
210 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
211 jnz syscall_trace_entry
212 cmpl $(nr_syscalls), %eax
214 call *sys_call_table(,%eax,4)
217 movl TI_flags(%ebp), %ecx
218 testw $_TIF_ALLWORK_MASK, %cx
219 jne syscall_exit_work
220 /* if something modifies registers it must also disable sysexit */
222 movl OLDESP(%esp), %ecx
228 # system call handler stub
230 pushl %eax # save orig_eax
232 GET_THREAD_INFO(%ebp)
233 testl $TF_MASK,EFLAGS(%esp)
235 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
237 # system call tracing in operation / emulation
238 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
239 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
240 jnz syscall_trace_entry
241 cmpl $(nr_syscalls), %eax
244 call *sys_call_table(,%eax,4)
245 movl %eax,EAX(%esp) # store the return value
247 cli # make sure we don't miss an interrupt
248 # setting need_resched or sigpending
249 # between sampling and the iret
250 movl TI_flags(%ebp), %ecx
251 testw $_TIF_ALLWORK_MASK, %cx # current->work
252 jne syscall_exit_work
255 movl EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
256 # Warning: OLDSS(%esp) contains the wrong/random values if we
257 # are returning to the kernel.
258 # See comments in process.c:copy_thread() for details.
259 movb OLDSS(%esp), %ah
261 andl $(VM_MASK | (4 << 8) | 3), %eax
262 cmpl $((4 << 8) | 3), %eax
263 je ldt_ss # returning to user-space with LDT SS
271 pushl $0 # no error code
275 .section __ex_table,"a"
281 larl OLDSS(%esp), %eax
283 testl $0x00400000, %eax # returning to 32bit stack?
284 jnz restore_nocheck # allright, normal return
285 /* If returning to userspace with 16bit stack,
286 * try to fix the higher word of ESP, as the CPU
288 * This is an "official" bug of all the x86-compatible
289 * CPUs, which we can try to work around to make
290 * dosemu and wine happy. */
291 subl $8, %esp # reserve space for switch16 pointer
294 /* Set up the 16bit stack frame with switch32 pointer on top,
295 * and a switch16 pointer on top of the current frame. */
296 call setup_x86_bogus_stack
298 lss 20+4(%esp), %esp # switch to 16bit stack
300 .section __ex_table,"a"
305 # perform work that needs to be done immediately before resumption
308 testb $_TIF_NEED_RESCHED, %cl
312 cli # make sure we don't miss an interrupt
313 # setting need_resched or sigpending
314 # between sampling and the iret
315 movl TI_flags(%ebp), %ecx
316 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
317 # than syscall tracing?
319 testb $_TIF_NEED_RESCHED, %cl
322 work_notifysig: # deal with pending signals and
323 # notify-resume requests
324 testl $VM_MASK, EFLAGS(%esp)
326 jne work_notifysig_v86 # returning to kernel-space or
329 call do_notify_resume
335 pushl %ecx # save ti_flags for do_notify_resume
336 call save_v86_state # %eax contains pt_regs pointer
340 call do_notify_resume
344 # perform syscall exit tracing
347 movl $-ENOSYS,EAX(%esp)
350 call do_syscall_trace
352 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
353 # so must skip actual syscall
354 movl ORIG_EAX(%esp), %eax
355 cmpl $(nr_syscalls), %eax
359 # perform syscall exit tracing
362 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
364 sti # could let do_syscall_trace() call
368 call do_syscall_trace
373 pushl %eax # save orig_eax
375 GET_THREAD_INFO(%ebp)
376 movl $-EFAULT,EAX(%esp)
381 movl $-ENOSYS,EAX(%esp)
384 #define FIXUP_ESPFIX_STACK \
386 /* switch to 32bit stack using the pointer on top of 16bit stack */ \
387 lss %ss:CPU_16BIT_STACK_SIZE-8, %esp; \
388 /* copy data from 16bit stack to 32bit stack */ \
389 call fixup_x86_bogus_stack; \
390 /* put ESP to the proper location */ \
392 #define UNWIND_ESPFIX_STACK \
395 /* see if on 16bit stack */ \
396 cmpw $__ESPFIX_SS, %ax; \
398 movl $__KERNEL_DS, %edx; \
401 /* switch to 32bit stack */ \
406 * Build the entry stubs and pointer table with
407 * some assembler magic.
414 ENTRY(irq_entries_start)
432 #define BUILD_INTERRUPT(name, nr) \
440 /* The include is where all of the SMP etc. interrupts come from */
441 #include "entry_arch.h"
444 pushl $0 # no error code
445 pushl $do_divide_error
462 movl ES(%esp), %edi # get the function address
463 movl ORIG_EAX(%esp), %edx # get the error code
464 movl %eax, ORIG_EAX(%esp)
466 movl $(__USER_DS), %ecx
469 movl %esp,%eax # pt_regs pointer
471 jmp ret_from_exception
473 ENTRY(coprocessor_error)
475 pushl $do_coprocessor_error
478 ENTRY(simd_coprocessor_error)
480 pushl $do_simd_coprocessor_error
483 ENTRY(device_not_available)
484 pushl $-1 # mark this as an int
487 testl $0x4, %eax # EM (math emulation bit)
488 jne device_not_available_emulate
490 call math_state_restore
491 jmp ret_from_exception
492 device_not_available_emulate:
493 pushl $0 # temporary storage for ORIG_EIP
496 jmp ret_from_exception
499 * Debug traps and NMI can happen at the one SYSENTER instruction
500 * that sets up the real kernel stack. Check here, since we can't
501 * allow the wrong stack to be used.
503 * "SYSENTER_stack_esp0+12" is because the NMI/debug handler will have
504 * already pushed 3 words if it hits on the sysenter instruction:
505 * eflags, cs and eip.
507 * We just load the right stack, and push the three (known) values
508 * by hand onto the new stack - while updating the return eip past
509 * the instruction that would have done it for sysenter.
511 #define FIX_STACK(offset, ok, label) \
512 cmpw $__KERNEL_CS,4(%esp); \
515 movl SYSENTER_stack_esp0+offset(%esp),%esp; \
517 pushl $__KERNEL_CS; \
518 pushl $sysenter_past_esp
521 cmpl $sysenter_entry,(%esp)
522 jne debug_stack_correct
523 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
525 pushl $-1 # mark this as an int
527 xorl %edx,%edx # error code 0
528 movl %esp,%eax # pt_regs pointer
530 jmp ret_from_exception
533 * NMI is doubly nasty. It can happen _while_ we're handling
534 * a debug fault, and the debug fault hasn't yet been able to
535 * clear up the stack. So we first check whether we got an
536 * NMI on the sysenter entry path, but after that we need to
537 * check whether we got an NMI on the debug path where the debug
538 * fault happened on the sysenter path.
543 cmpw $__ESPFIX_SS, %ax
546 cmpl $sysenter_entry,(%esp)
550 /* Do not access memory above the end of our stack page,
551 * it might not exist.
553 andl $(THREAD_SIZE-1),%eax
554 cmpl $(THREAD_SIZE-20),%eax
556 jae nmi_stack_correct
557 cmpl $sysenter_entry,12(%esp)
558 je nmi_debug_stack_check
562 xorl %edx,%edx # zero error code
563 movl %esp,%eax # pt_regs pointer
568 FIX_STACK(12,nmi_stack_correct, 1)
569 jmp nmi_stack_correct
570 nmi_debug_stack_check:
571 cmpw $__KERNEL_CS,16(%esp)
572 jne nmi_stack_correct
575 cmpl $debug_esp_fix_insn,(%esp)
577 FIX_STACK(24,nmi_stack_correct, 1)
578 jmp nmi_stack_correct
581 /* create the pointer to lss back */
586 /* copy the iret frame of 12 bytes */
592 FIXUP_ESPFIX_STACK # %eax == %esp
593 xorl %edx,%edx # zero error code
596 lss 12+4(%esp), %esp # back to 16bit stack
598 .section __ex_table,"a"
604 pushl $-1 # mark this as an int
606 xorl %edx,%edx # zero error code
607 movl %esp,%eax # pt_regs pointer
609 jmp ret_from_exception
627 ENTRY(coprocessor_segment_overrun)
629 pushl $do_coprocessor_segment_overrun
633 pushl $do_invalid_TSS
636 ENTRY(segment_not_present)
637 pushl $do_segment_not_present
641 pushl $do_stack_segment
644 KPROBE_ENTRY(general_protection)
645 pushl $do_general_protection
649 ENTRY(alignment_check)
650 pushl $do_alignment_check
653 KPROBE_ENTRY(page_fault)
658 #ifdef CONFIG_X86_MCE
661 pushl machine_check_vector
665 ENTRY(spurious_interrupt_bug)
667 pushl $do_spurious_interrupt_bug
671 #include "syscall_table.S"
673 syscall_table_size=(.-sys_call_table)