2 * linux/arch/i386/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
40 * "current" is in register %ebx during any slow entries.
43 #include <linux/config.h>
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
50 #include "irq_vectors.h"
52 #define nr_syscalls ((syscall_table_size)/4)
78 #define preempt_stop cli
81 #define resume_kernel restore_all
95 movl $(__USER_DS), %edx; \
99 #define RESTORE_INT_REGS \
108 #define RESTORE_REGS \
112 .section .fixup,"ax"; \
118 .section __ex_table,"a";\
125 #define RESTORE_ALL \
129 .section .fixup,"ax"; \
131 movl $(__USER_DS), %edx; \
137 .section __ex_table,"a";\
145 pushfl # We get a different stack layout with call
146 # gates, which has to be cleaned up later..
153 movl EIP(%ebp), %eax # due to call gates, this is eflags, not eip..
154 movl CS(%ebp), %edx # this is eip..
155 movl EFLAGS(%ebp), %ecx # and this is cs..
156 movl %eax,EFLAGS(%ebp) #
157 movl %edx,EIP(%ebp) # Now we move them to their "normal" places
159 GET_THREAD_INFO_WITH_ESP(%ebp) # GET_THREAD_INFO
160 movl TI_exec_domain(%ebp), %edx # Get the execution domain
161 call *EXEC_DOMAIN_handler(%edx) # Call the handler for the domain
167 pushfl # We get a different stack layout with call
168 # gates, which has to be cleaned up later..
180 GET_THREAD_INFO(%ebp)
185 * Return to user mode is not as complex as all this looks,
186 * but we want the default path for a system call return to
187 * go as quickly as possible which is why some of this is
188 * less clear than it otherwise should be.
191 # userspace resumption stub bypassing syscall exit tracing
196 GET_THREAD_INFO(%ebp)
197 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
199 testl $(VM_MASK | 3), %eax
200 jz resume_kernel # returning to kernel or vm86-space
201 ENTRY(resume_userspace)
202 cli # make sure we don't miss an interrupt
203 # setting need_resched or sigpending
204 # between sampling and the iret
205 movl TI_flags(%ebp), %ecx
206 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
207 # int/exception return?
211 #ifdef CONFIG_PREEMPT
213 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
216 movl TI_flags(%ebp), %ecx # need_resched set ?
217 testb $_TIF_NEED_RESCHED, %cl
219 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
221 movl $PREEMPT_ACTIVE,TI_preempt_count(%ebp)
224 movl $0,TI_preempt_count(%ebp)
229 /* SYSENTER_RETURN points to after the "sysenter" instruction in
230 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
232 # sysenter call handler stub
233 ENTRY(sysenter_entry)
234 movl TSS_sysenter_esp0(%esp),%esp
242 * Push current_thread_info()->sysenter_return to the stack.
243 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
244 * pushed above, and the word being pushed now:
246 pushl (TI_sysenter_return-THREAD_SIZE+4*4)(%esp)
248 * Load the potential sixth argument from user stack.
249 * Careful about security.
251 cmpl $__PAGE_OFFSET-3,%ebp
254 .section __ex_table,"a"
256 .long 1b,syscall_fault
261 GET_THREAD_INFO(%ebp)
263 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
264 jnz syscall_trace_entry
265 cmpl $(nr_syscalls), %eax
267 call *sys_call_table(,%eax,4)
270 movl TI_flags(%ebp), %ecx
271 testw $_TIF_ALLWORK_MASK, %cx
272 jne syscall_exit_work
273 /* if something modifies registers it must also disable sysexit */
275 movl OLDESP(%esp), %ecx
281 # system call handler stub
283 pushl %eax # save orig_eax
285 GET_THREAD_INFO(%ebp)
286 # system call tracing in operation
287 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
288 jnz syscall_trace_entry
289 cmpl $(nr_syscalls), %eax
292 call *sys_call_table(,%eax,4)
293 movl %eax,EAX(%esp) # store the return value
295 cli # make sure we don't miss an interrupt
296 # setting need_resched or sigpending
297 # between sampling and the iret
298 movl TI_flags(%ebp), %ecx
299 testw $_TIF_ALLWORK_MASK, %cx # current->work
300 jne syscall_exit_work
304 # perform work that needs to be done immediately before resumption
307 testb $_TIF_NEED_RESCHED, %cl
311 cli # make sure we don't miss an interrupt
312 # setting need_resched or sigpending
313 # between sampling and the iret
314 movl TI_flags(%ebp), %ecx
315 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
316 # than syscall tracing?
318 testb $_TIF_NEED_RESCHED, %cl
321 work_notifysig: # deal with pending signals and
322 # notify-resume requests
323 testl $VM_MASK, EFLAGS(%esp)
325 jne work_notifysig_v86 # returning to kernel-space or
328 call do_notify_resume
338 call do_notify_resume
341 # perform syscall exit tracing
344 movl $-ENOSYS,EAX(%esp)
347 call do_syscall_trace
348 movl ORIG_EAX(%esp), %eax
349 cmpl $(nr_syscalls), %eax
353 # perform syscall exit tracing
356 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
358 sti # could let do_syscall_trace() call
362 call do_syscall_trace
367 pushl %eax # save orig_eax
369 GET_THREAD_INFO(%ebp)
370 movl $-EFAULT,EAX(%esp)
375 movl $-ENOSYS,EAX(%esp)
379 * Build the entry stubs and pointer table with
380 * some assembler magic.
387 ENTRY(irq_entries_start)
404 #define BUILD_INTERRUPT(name, nr) \
411 /* The include is where all of the SMP etc. interrupts come from */
412 #include "entry_arch.h"
415 pushl $0 # no error code
416 pushl $do_divide_error
431 movl ORIG_EAX(%esp), %esi # get the error code
432 movl ES(%esp), %edi # get the function address
433 movl %eax, ORIG_EAX(%esp)
436 pushl %esi # push the error code
437 pushl %edx # push the pt_regs pointer
438 movl $(__USER_DS), %edx
443 jmp ret_from_exception
445 ENTRY(coprocessor_error)
447 pushl $do_coprocessor_error
450 ENTRY(simd_coprocessor_error)
452 pushl $do_simd_coprocessor_error
455 ENTRY(device_not_available)
456 pushl $-1 # mark this as an int
459 testl $0x4, %eax # EM (math emulation bit)
460 jne device_not_available_emulate
462 call math_state_restore
463 jmp ret_from_exception
464 device_not_available_emulate:
465 pushl $0 # temporary storage for ORIG_EIP
468 jmp ret_from_exception
471 * Debug traps and NMI can happen at the one SYSENTER instruction
472 * that sets up the real kernel stack. Check here, since we can't
473 * allow the wrong stack to be used.
475 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
476 * already pushed 3 words if it hits on the sysenter instruction:
477 * eflags, cs and eip.
479 * We just load the right stack, and push the three (known) values
480 * by hand onto the new stack - while updating the return eip past
481 * the instruction that would have done it for sysenter.
483 #define FIX_STACK(offset, ok, label) \
484 cmpw $__KERNEL_CS,4(%esp); \
487 movl TSS_sysenter_esp0+offset(%esp),%esp; \
489 pushl $__KERNEL_CS; \
490 pushl $sysenter_past_esp
493 cmpl $sysenter_entry,(%esp)
494 jne debug_stack_correct
495 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
497 pushl $-1 # mark this as an int
506 jmp ret_from_exception
509 * NMI is doubly nasty. It can happen _while_ we're handling
510 * a debug fault, and the debug fault hasn't yet been able to
511 * clear up the stack. So we first check whether we got an
512 * NMI on the sysenter entry path, but after that we need to
513 * check whether we got an NMI on the debug path where the debug
514 * fault happened on the sysenter path.
517 cmpl $sysenter_entry,(%esp)
521 /* Do not access memory above the end of our stack page,
522 * it might not exist.
524 andl $(THREAD_SIZE-1),%eax
525 cmpl $(THREAD_SIZE-20),%eax
527 jae nmi_stack_correct
528 cmpl $sysenter_entry,12(%esp)
529 je nmi_debug_stack_check
541 FIX_STACK(12,nmi_stack_correct, 1)
542 jmp nmi_stack_correct
543 nmi_debug_stack_check:
544 cmpw $__KERNEL_CS,16(%esp)
545 jne nmi_stack_correct
546 cmpl $debug - 1,(%esp)
547 jle nmi_stack_correct
548 cmpl $debug_esp_fix_insn,(%esp)
549 jle nmi_debug_stack_fixup
550 nmi_debug_stack_fixup:
551 FIX_STACK(24,nmi_stack_correct, 1)
552 jmp nmi_stack_correct
555 pushl $-1 # mark this as an int
564 jmp ret_from_exception
581 ENTRY(coprocessor_segment_overrun)
583 pushl $do_coprocessor_segment_overrun
587 pushl $do_invalid_TSS
590 ENTRY(segment_not_present)
591 pushl $do_segment_not_present
595 pushl $do_stack_segment
598 ENTRY(general_protection)
599 pushl $do_general_protection
602 ENTRY(alignment_check)
603 pushl $do_alignment_check
610 #ifdef CONFIG_X86_MCE
613 pushl machine_check_vector
617 ENTRY(spurious_interrupt_bug)
619 pushl $do_spurious_interrupt_bug
623 ENTRY(sys_call_table)
624 .long sys_restart_syscall /* 0 - old "setup()" system call, used for restarting */
629 .long sys_open /* 5 */
634 .long sys_unlink /* 10 */
639 .long sys_chmod /* 15 */
641 .long sys_ni_syscall /* old break syscall holder */
644 .long sys_getpid /* 20 */
649 .long sys_stime /* 25 */
654 .long sys_utime /* 30 */
655 .long sys_ni_syscall /* old stty syscall holder */
656 .long sys_ni_syscall /* old gtty syscall holder */
659 .long sys_ni_syscall /* 35 - old ftime syscall holder */
664 .long sys_rmdir /* 40 */
668 .long sys_ni_syscall /* old prof syscall holder */
669 .long sys_brk /* 45 */
674 .long sys_getegid16 /* 50 */
676 .long sys_umount /* recycled never used phys() */
677 .long sys_ni_syscall /* old lock syscall holder */
679 .long sys_fcntl /* 55 */
680 .long sys_ni_syscall /* old mpx syscall holder */
682 .long sys_ni_syscall /* old ulimit syscall holder */
684 .long sys_umask /* 60 */
689 .long sys_getpgrp /* 65 */
694 .long sys_setreuid16 /* 70 */
698 .long sys_sethostname
699 .long sys_setrlimit /* 75 */
700 .long sys_old_getrlimit
702 .long sys_gettimeofday
703 .long sys_settimeofday
704 .long sys_getgroups16 /* 80 */
705 .long sys_setgroups16
709 .long sys_readlink /* 85 */
714 .long old_mmap /* 90 */
719 .long sys_fchown16 /* 95 */
720 .long sys_getpriority
721 .long sys_setpriority
722 .long sys_ni_syscall /* old profil syscall holder */
724 .long sys_fstatfs /* 100 */
729 .long sys_getitimer /* 105 */
734 .long sys_iopl /* 110 */
736 .long sys_ni_syscall /* old "idle" system call */
739 .long sys_swapoff /* 115 */
744 .long sys_clone /* 120 */
745 .long sys_setdomainname
749 .long sys_mprotect /* 125 */
750 .long sys_sigprocmask
751 .long sys_ni_syscall /* old "create_module" */
752 .long sys_init_module
753 .long sys_delete_module
754 .long sys_ni_syscall /* 130: old "get_kernel_syms" */
759 .long sys_sysfs /* 135 */
760 .long sys_personality
761 .long sys_ni_syscall /* reserved for afs_syscall */
764 .long sys_llseek /* 140 */
769 .long sys_readv /* 145 */
774 .long sys_mlock /* 150 */
778 .long sys_sched_setparam
779 .long sys_sched_getparam /* 155 */
780 .long sys_sched_setscheduler
781 .long sys_sched_getscheduler
782 .long sys_sched_yield
783 .long sys_sched_get_priority_max
784 .long sys_sched_get_priority_min /* 160 */
785 .long sys_sched_rr_get_interval
788 .long sys_setresuid16
789 .long sys_getresuid16 /* 165 */
791 .long sys_ni_syscall /* Old sys_query_module */
794 .long sys_setresgid16 /* 170 */
795 .long sys_getresgid16
797 .long sys_rt_sigreturn
798 .long sys_rt_sigaction
799 .long sys_rt_sigprocmask /* 175 */
800 .long sys_rt_sigpending
801 .long sys_rt_sigtimedwait
802 .long sys_rt_sigqueueinfo
803 .long sys_rt_sigsuspend
804 .long sys_pread64 /* 180 */
809 .long sys_capset /* 185 */
810 .long sys_sigaltstack
812 .long sys_ni_syscall /* reserved for streams1 */
813 .long sys_ni_syscall /* reserved for streams2 */
814 .long sys_vfork /* 190 */
818 .long sys_ftruncate64
819 .long sys_stat64 /* 195 */
824 .long sys_getgid /* 200 */
829 .long sys_getgroups /* 205 */
834 .long sys_setresgid /* 210 */
839 .long sys_setfsuid /* 215 */
844 .long sys_getdents64 /* 220 */
849 # ifdef CONFIG_TUX_MODULE
857 .long sys_readahead /* 225 */
862 .long sys_lgetxattr /* 230 */
867 .long sys_removexattr /* 235 */
868 .long sys_lremovexattr
869 .long sys_fremovexattr
872 .long sys_futex /* 240 */
873 .long sys_sched_setaffinity
874 .long sys_sched_getaffinity
875 .long sys_set_thread_area
876 .long sys_get_thread_area
877 .long sys_io_setup /* 245 */
879 .long sys_io_getevents
882 .long sys_fadvise64 /* 250 */
885 .long sys_lookup_dcookie
886 .long sys_epoll_create
887 .long sys_epoll_ctl /* 255 */
889 .long sys_remap_file_pages
890 .long sys_set_tid_address
891 .long sys_timer_create
892 .long sys_timer_settime /* 260 */
893 .long sys_timer_gettime
894 .long sys_timer_getoverrun
895 .long sys_timer_delete
896 .long sys_clock_settime
897 .long sys_clock_gettime /* 265 */
898 .long sys_clock_getres
899 .long sys_clock_nanosleep
902 .long sys_tgkill /* 270 */
904 .long sys_fadvise64_64
907 .long sys_get_mempolicy
908 .long sys_set_mempolicy
911 .long sys_mq_timedsend
912 .long sys_mq_timedreceive /* 280 */
914 .long sys_mq_getsetattr
917 #ifdef USE_IOPRIO_SYSCALLS
918 .long sys_ioprio_set /* 285 */
921 #warning MEF not includig sys_ioprio_{set,get} syscalls
924 syscall_table_size=(.-sys_call_table)
926 #ifdef CONFIG_X86_STACK_CHECK
928 .globl stack_overflowed
934 #warning stack check enabled
936 movl $(THREAD_SIZE - 1),%eax
938 cmpl $STACK_WARN,%eax
944 /* prevent infinite recursion from call to mcount from the
945 * stack_overflow function. Need to revisit this code for
948 lock; btsl $0,stack_overflowed
951 /* prepare to jmp to stack_overflow directly, as if it were
952 * called directly by the caller of mcount.
960 /* Note that stack_overflow() will clear the stack_overflowed