2 * linux/arch/i386/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
40 * "current" is in register %ebx during any slow entries.
43 #include <linux/config.h>
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
50 #include "irq_vectors.h"
52 #define nr_syscalls ((syscall_table_size)/4)
78 * ESP0 is at offset 4. 0x200 is the size of the TSS, and
79 * also thus the top-of-stack pointer offset of SYSENTER_ESP
81 TSS_ESP0_OFFSET = (4 - 0x200)
84 #define preempt_stop cli
87 #define resume_kernel restore_all
101 movl $(__USER_DS), %edx; \
105 #define RESTORE_INT_REGS \
114 #define RESTORE_REGS \
118 .section .fixup,"ax"; \
124 .section __ex_table,"a";\
131 #define RESTORE_ALL \
135 .section .fixup,"ax"; \
137 movl $(__USER_DS), %edx; \
143 .section __ex_table,"a";\
151 pushfl # We get a different stack layout with call
152 # gates, which has to be cleaned up later..
159 movl EIP(%ebp), %eax # due to call gates, this is eflags, not eip..
160 movl CS(%ebp), %edx # this is eip..
161 movl EFLAGS(%ebp), %ecx # and this is cs..
162 movl %eax,EFLAGS(%ebp) #
163 movl %edx,EIP(%ebp) # Now we move them to their "normal" places
165 GET_THREAD_INFO_WITH_ESP(%ebp) # GET_THREAD_INFO
166 movl TI_EXEC_DOMAIN(%ebp), %edx # Get the execution domain
167 call *4(%edx) # Call the lcall7 handler for the domain
173 pushfl # We get a different stack layout with call
174 # gates, which has to be cleaned up later..
186 GET_THREAD_INFO(%ebp)
191 * Return to user mode is not as complex as all this looks,
192 * but we want the default path for a system call return to
193 * go as quickly as possible which is why some of this is
194 * less clear than it otherwise should be.
197 # userspace resumption stub bypassing syscall exit tracing
202 GET_THREAD_INFO(%ebp)
203 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
205 testl $(VM_MASK | 3), %eax
206 jz resume_kernel # returning to kernel or vm86-space
207 ENTRY(resume_userspace)
208 cli # make sure we don't miss an interrupt
209 # setting need_resched or sigpending
210 # between sampling and the iret
211 movl TI_FLAGS(%ebp), %ecx
212 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
213 # int/exception return?
217 #ifdef CONFIG_PREEMPT
219 cmpl $0,TI_PRE_COUNT(%ebp) # non-zero preempt_count ?
222 movl TI_FLAGS(%ebp), %ecx # need_resched set ?
223 testb $_TIF_NEED_RESCHED, %cl
225 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
227 movl $PREEMPT_ACTIVE,TI_PRE_COUNT(%ebp)
230 movl $0,TI_PRE_COUNT(%ebp)
235 /* SYSENTER_RETURN points to after the "sysenter" instruction in
236 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
238 # sysenter call handler stub
239 ENTRY(sysenter_entry)
240 movl TSS_ESP0_OFFSET(%esp),%esp
247 pushl $SYSENTER_RETURN
250 * Load the potential sixth argument from user stack.
251 * Careful about security.
253 cmpl $__PAGE_OFFSET-3,%ebp
256 .section __ex_table,"a"
258 .long 1b,syscall_fault
263 GET_THREAD_INFO(%ebp)
264 cmpl $(nr_syscalls), %eax
267 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_FLAGS(%ebp)
268 jnz syscall_trace_entry
269 call *sys_call_table(,%eax,4)
272 movl TI_FLAGS(%ebp), %ecx
273 testw $_TIF_ALLWORK_MASK, %cx
274 jne syscall_exit_work
275 /* if something modifies registers it must also disable sysexit */
277 movl OLDESP(%esp), %ecx
282 # system call handler stub
284 pushl %eax # save orig_eax
286 GET_THREAD_INFO(%ebp)
287 cmpl $(nr_syscalls), %eax
289 # system call tracing in operation
290 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_FLAGS(%ebp)
291 jnz syscall_trace_entry
293 call *sys_call_table(,%eax,4)
294 movl %eax,EAX(%esp) # store the return value
296 cli # make sure we don't miss an interrupt
297 # setting need_resched or sigpending
298 # between sampling and the iret
299 movl TI_FLAGS(%ebp), %ecx
300 testw $_TIF_ALLWORK_MASK, %cx # current->work
301 jne syscall_exit_work
305 # perform work that needs to be done immediately before resumption
308 testb $_TIF_NEED_RESCHED, %cl
312 cli # make sure we don't miss an interrupt
313 # setting need_resched or sigpending
314 # between sampling and the iret
315 movl TI_FLAGS(%ebp), %ecx
316 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
317 # than syscall tracing?
319 testb $_TIF_NEED_RESCHED, %cl
322 work_notifysig: # deal with pending signals and
323 # notify-resume requests
324 testl $VM_MASK, EFLAGS(%esp)
326 jne work_notifysig_v86 # returning to kernel-space or
329 call do_notify_resume
339 call do_notify_resume
342 # perform syscall exit tracing
345 movl $-ENOSYS,EAX(%esp)
348 call do_syscall_trace
349 movl ORIG_EAX(%esp), %eax
350 cmpl $(nr_syscalls), %eax
354 # perform syscall exit tracing
357 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT), %cl
359 sti # could let do_syscall_trace() call
363 call do_syscall_trace
368 pushl %eax # save orig_eax
370 GET_THREAD_INFO(%ebp)
371 movl $-EFAULT,EAX(%esp)
376 movl $-ENOSYS,EAX(%esp)
380 * Build the entry stubs and pointer table with
381 * some assembler magic.
388 ENTRY(irq_entries_start)
405 #define BUILD_INTERRUPT(name, nr) \
412 /* The include is where all of the SMP etc. interrupts come from */
413 #include "entry_arch.h"
416 pushl $0 # no error code
417 pushl $do_divide_error
432 movl ORIG_EAX(%esp), %esi # get the error code
433 movl ES(%esp), %edi # get the function address
434 movl %eax, ORIG_EAX(%esp)
437 pushl %esi # push the error code
438 pushl %edx # push the pt_regs pointer
439 movl $(__USER_DS), %edx
444 jmp ret_from_exception
446 ENTRY(coprocessor_error)
448 pushl $do_coprocessor_error
451 ENTRY(simd_coprocessor_error)
453 pushl $do_simd_coprocessor_error
456 ENTRY(device_not_available)
457 pushl $-1 # mark this as an int
460 testl $0x4, %eax # EM (math emulation bit)
461 jne device_not_available_emulate
463 call math_state_restore
464 jmp ret_from_exception
465 device_not_available_emulate:
466 pushl $0 # temporary storage for ORIG_EIP
469 jmp ret_from_exception
472 * Debug traps and NMI can happen at the one SYSENTER instruction
473 * that sets up the real kernel stack. Check here, since we can't
474 * allow the wrong stack to be used.
476 * "TSS_ESP0_OFFSET+12" is because the NMI/debug handler will have
477 * already pushed 3 words if it hits on the sysenter instruction:
478 * eflags, cs and eip.
480 * We just load the right stack, and push the three (known) values
481 * by hand onto the new stack - while updating the return eip past
482 * the instruction that would have done it for sysenter.
484 #define FIX_STACK(offset, ok, label) \
485 cmpw $__KERNEL_CS,4(%esp); \
488 movl TSS_ESP0_OFFSET+offset(%esp),%esp; \
490 pushl $__KERNEL_CS; \
491 pushl $sysenter_past_esp
494 cmpl $sysenter_entry,(%esp)
495 jne debug_stack_correct
496 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
503 * NMI is doubly nasty. It can happen _while_ we're handling
504 * a debug fault, and the debug fault hasn't yet been able to
505 * clear up the stack. So we first check whether we got an
506 * NMI on the sysenter entry path, but after that we need to
507 * check whether we got an NMI on the debug path where the debug
508 * fault happened on the sysenter path.
511 cmpl $sysenter_entry,(%esp)
515 /* Do not access memory above the end of our stack page,
516 * it might not exist.
518 andl $(THREAD_SIZE-1),%eax
519 cmpl $(THREAD_SIZE-20),%eax
521 jae nmi_stack_correct
522 cmpl $sysenter_entry,12(%esp)
523 je nmi_debug_stack_check
535 FIX_STACK(12,nmi_stack_correct, 1)
536 jmp nmi_stack_correct
537 nmi_debug_stack_check:
538 cmpw $__KERNEL_CS,16(%esp)
539 jne nmi_stack_correct
540 cmpl $debug - 1,(%esp)
541 jle nmi_stack_correct
542 cmpl $debug_esp_fix_insn,(%esp)
543 jle nmi_debug_stack_fixup
544 nmi_debug_stack_fixup:
545 FIX_STACK(24,nmi_stack_correct, 1)
546 jmp nmi_stack_correct
568 ENTRY(coprocessor_segment_overrun)
570 pushl $do_coprocessor_segment_overrun
574 pushl $do_invalid_TSS
577 ENTRY(segment_not_present)
578 pushl $do_segment_not_present
582 pushl $do_stack_segment
585 ENTRY(general_protection)
586 pushl $do_general_protection
589 ENTRY(alignment_check)
590 pushl $do_alignment_check
597 #ifdef CONFIG_X86_MCE
600 pushl machine_check_vector
604 ENTRY(spurious_interrupt_bug)
606 pushl $do_spurious_interrupt_bug
610 ENTRY(sys_call_table)
611 .long sys_restart_syscall /* 0 - old "setup()" system call, used for restarting */
616 .long sys_open /* 5 */
621 .long sys_unlink /* 10 */
626 .long sys_chmod /* 15 */
628 .long sys_ni_syscall /* old break syscall holder */
631 .long sys_getpid /* 20 */
636 .long sys_stime /* 25 */
641 .long sys_utime /* 30 */
642 .long sys_ni_syscall /* old stty syscall holder */
643 .long sys_ni_syscall /* old gtty syscall holder */
646 .long sys_ni_syscall /* 35 - old ftime syscall holder */
651 .long sys_rmdir /* 40 */
655 .long sys_ni_syscall /* old prof syscall holder */
656 .long sys_brk /* 45 */
661 .long sys_getegid16 /* 50 */
663 .long sys_umount /* recycled never used phys() */
664 .long sys_ni_syscall /* old lock syscall holder */
666 .long sys_fcntl /* 55 */
667 .long sys_ni_syscall /* old mpx syscall holder */
669 .long sys_ni_syscall /* old ulimit syscall holder */
671 .long sys_umask /* 60 */
676 .long sys_getpgrp /* 65 */
681 .long sys_setreuid16 /* 70 */
685 .long sys_sethostname
686 .long sys_setrlimit /* 75 */
687 .long sys_old_getrlimit
689 .long sys_gettimeofday
690 .long sys_settimeofday
691 .long sys_getgroups16 /* 80 */
692 .long sys_setgroups16
696 .long sys_readlink /* 85 */
701 .long old_mmap /* 90 */
706 .long sys_fchown16 /* 95 */
707 .long sys_getpriority
708 .long sys_setpriority
709 .long sys_ni_syscall /* old profil syscall holder */
711 .long sys_fstatfs /* 100 */
716 .long sys_getitimer /* 105 */
721 .long sys_iopl /* 110 */
723 .long sys_ni_syscall /* old "idle" system call */
726 .long sys_swapoff /* 115 */
731 .long sys_clone /* 120 */
732 .long sys_setdomainname
736 .long sys_mprotect /* 125 */
737 .long sys_sigprocmask
738 .long sys_ni_syscall /* old "create_module" */
739 .long sys_init_module
740 .long sys_delete_module
741 .long sys_ni_syscall /* 130: old "get_kernel_syms" */
746 .long sys_sysfs /* 135 */
747 .long sys_personality
748 .long sys_ni_syscall /* reserved for afs_syscall */
751 .long sys_llseek /* 140 */
756 .long sys_readv /* 145 */
761 .long sys_mlock /* 150 */
765 .long sys_sched_setparam
766 .long sys_sched_getparam /* 155 */
767 .long sys_sched_setscheduler
768 .long sys_sched_getscheduler
769 .long sys_sched_yield
770 .long sys_sched_get_priority_max
771 .long sys_sched_get_priority_min /* 160 */
772 .long sys_sched_rr_get_interval
775 .long sys_setresuid16
776 .long sys_getresuid16 /* 165 */
778 .long sys_ni_syscall /* Old sys_query_module */
781 .long sys_setresgid16 /* 170 */
782 .long sys_getresgid16
784 .long sys_rt_sigreturn
785 .long sys_rt_sigaction
786 .long sys_rt_sigprocmask /* 175 */
787 .long sys_rt_sigpending
788 .long sys_rt_sigtimedwait
789 .long sys_rt_sigqueueinfo
790 .long sys_rt_sigsuspend
791 .long sys_pread64 /* 180 */
796 .long sys_capset /* 185 */
797 .long sys_sigaltstack
799 .long sys_ni_syscall /* reserved for streams1 */
800 .long sys_ni_syscall /* reserved for streams2 */
801 .long sys_vfork /* 190 */
805 .long sys_ftruncate64
806 .long sys_stat64 /* 195 */
811 .long sys_getgid /* 200 */
816 .long sys_getgroups /* 205 */
821 .long sys_setresgid /* 210 */
826 .long sys_setfsuid /* 215 */
831 .long sys_getdents64 /* 220 */
833 .long sys_ni_syscall /* reserved for TUX */
836 .long sys_readahead /* 225 */
841 .long sys_lgetxattr /* 230 */
846 .long sys_removexattr /* 235 */
847 .long sys_lremovexattr
848 .long sys_fremovexattr
851 .long sys_futex /* 240 */
852 .long sys_sched_setaffinity
853 .long sys_sched_getaffinity
854 .long sys_set_thread_area
855 .long sys_get_thread_area
856 .long sys_io_setup /* 245 */
858 .long sys_io_getevents
861 .long sys_fadvise64 /* 250 */
864 .long sys_lookup_dcookie
865 .long sys_epoll_create
866 .long sys_epoll_ctl /* 255 */
868 .long sys_remap_file_pages
869 .long sys_set_tid_address
870 .long sys_timer_create
871 .long sys_timer_settime /* 260 */
872 .long sys_timer_gettime
873 .long sys_timer_getoverrun
874 .long sys_timer_delete
875 .long sys_clock_settime
876 .long sys_clock_gettime /* 265 */
877 .long sys_clock_getres
878 .long sys_clock_nanosleep
881 .long sys_tgkill /* 270 */
883 .long sys_fadvise64_64
884 .long sys_ni_syscall /* sys_vserver */
885 .long sys_ni_syscall /* sys_mbind */
886 .long sys_ni_syscall /* 275 sys_get_mempolicy */
887 .long sys_ni_syscall /* sys_set_mempolicy */
890 .long sys_mq_timedsend
891 .long sys_mq_timedreceive /* 280 */
893 .long sys_mq_getsetattr
895 syscall_table_size=(.-sys_call_table)