3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
22 #include <linux/config.h>
23 #include <linux/errno.h>
24 #include <linux/sys.h>
25 #include <linux/threads.h>
26 #include <asm/processor.h>
29 #include <asm/cputable.h>
30 #include <asm/thread_info.h>
31 #include <asm/ppc_asm.h>
32 #include <asm/offsets.h>
33 #include <asm/unistd.h>
36 #undef SHOW_SYSCALLS_TASK
39 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
41 #if MSR_KERNEL >= 0x10000
42 #define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l
44 #define LOAD_MSR_KERNEL(r, x) li r,(x)
48 #define COR r8 /* Critical Offset Register (COR) */
49 #define BOOKE_LOAD_COR lis COR,crit_save@ha
50 #define BOOKE_REST_COR mfspr COR,SPRG2
51 #define BOOKE_SAVE_COR mtspr SPRG2,COR
54 #define BOOKE_LOAD_COR
55 #define BOOKE_REST_COR
56 #define BOOKE_SAVE_COR
60 .globl mcheck_transfer_to_handler
61 mcheck_transfer_to_handler:
64 lwz r0,mcheck_r10@l(r8)
66 lwz r0,mcheck_r11@l(r8)
69 b transfer_to_handler_full
72 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
73 .globl crit_transfer_to_handler
74 crit_transfer_to_handler:
77 lwz r0,crit_r10@l(COR)
79 lwz r0,crit_r11@l(COR)
86 * This code finishes saving the registers to the exception frame
87 * and jumps to the appropriate handler for the exception, turning
88 * on address translation.
89 * Note that we rely on the caller having set cr0.eq iff the exception
90 * occurred in kernel mode (i.e. MSR:PR = 0).
92 .globl transfer_to_handler_full
93 transfer_to_handler_full:
97 .globl transfer_to_handler
109 tovirt(r2,r2) /* set r2 to current */
110 beq 2f /* if from user, fix up THREAD.regs */
111 addi r11,r1,STACK_FRAME_OVERHEAD
113 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
114 lwz r12,PTRACE-THREAD(r12)
115 andi. r12,r12,PT_PTRACED
117 /* From user and task is ptraced - load up global dbcr0 */
118 li r12,-1 /* clear all pending debug events */
120 lis r11,global_dbcr0@ha
122 addi r11,r11,global_dbcr0@l
130 2: /* if from kernel, check interrupted DOZE/NAP mode and
131 * check for stack overflow
137 bt- 8,power_save_6xx_restore /* Check DOZE */
138 END_FTR_SECTION_IFSET(CPU_FTR_CAN_DOZE)
140 bt- 9,power_save_6xx_restore /* Check NAP */
141 END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
142 #endif /* CONFIG_6xx */
143 .globl transfer_to_handler_cont
144 transfer_to_handler_cont:
145 lwz r11,THREAD_INFO-THREAD(r12)
146 cmplw r1,r11 /* if r1 <= current->thread_info */
147 ble- stack_ovf /* then the kernel stack overflowed */
150 lwz r11,0(r9) /* virtual address of handler */
151 lwz r9,4(r9) /* where to go when done */
157 RFI /* jump to handler, enable MMU */
160 * On kernel stack overflow, load up an initial stack pointer
161 * and call StackOverflow(regs), which should not return.
164 /* sometimes we use a statically-allocated stack, which is OK. */
168 ble 3b /* r1 <= &_end is OK */
170 addi r3,r1,STACK_FRAME_OVERHEAD
171 lis r1,init_thread_union@ha
172 addi r1,r1,init_thread_union@l
173 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
174 lis r9,StackOverflow@ha
175 addi r9,r9,StackOverflow@l
176 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
184 * Handle a system call.
186 .stabs "arch/ppc/kernel/",N_SO,0,0,0f
187 .stabs "entry.S",N_SO,0,0,0f
191 stw r0,THREAD+LAST_SYSCALL(r2)
195 lwz r11,_CCR(r1) /* Clear SO bit in CR */
200 #endif /* SHOW_SYSCALLS */
201 rlwinm r10,r1,0,0,18 /* current_thread_info() */
202 lwz r11,TI_LOCAL_FLAGS(r10)
203 rlwinm r11,r11,0,~_TIFL_FORCE_NOERROR
204 stw r11,TI_LOCAL_FLAGS(r10)
205 lwz r11,TI_FLAGS(r10)
206 andi. r11,r11,_TIF_SYSCALL_TRACE
208 syscall_dotrace_cont:
209 cmpli 0,r0,NR_syscalls
210 lis r10,sys_call_table@h
211 ori r10,r10,sys_call_table@l
214 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */
216 addi r9,r1,STACK_FRAME_OVERHEAD
217 blrl /* Call handler */
218 .globl ret_from_syscall
221 bl do_show_syscall_exit
226 rlwinm r12,r1,0,0,18 /* current_thread_info() */
228 lwz r11,TI_LOCAL_FLAGS(r12)
229 andi. r11,r11,_TIFL_FORCE_NOERROR
232 lwz r10,_CCR(r1) /* Set SO bit in CR */
236 /* disable interrupts so current_thread_info()->flags can't change */
237 30: LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
241 andi. r0,r9,(_TIF_SYSCALL_TRACE|_TIF_SIGPENDING|_TIF_NEED_RESCHED)
242 bne- syscall_exit_work
244 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
245 /* If the process has its own DBCR0 value, load it up */
247 andi. r0,r0,PT_PTRACED
250 stwcx. r0,0,r1 /* to clear the reservation */
275 /* Traced system call support */
281 lwz r0,GPR0(r1) /* Restore original registers */
289 b syscall_dotrace_cont
292 stw r6,RESULT(r1) /* Save result */
293 stw r3,GPR3(r1) /* Update return value */
294 andi. r0,r9,_TIF_SYSCALL_TRACE
298 MTMSRD(r10) /* re-enable interrupts */
310 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
312 MTMSRD(r10) /* disable interrupts again */
313 rlwinm r12,r1,0,0,18 /* current_thread_info() */
316 andi. r0,r9,_TIF_NEED_RESCHED
320 beq syscall_exit_cont
321 andi. r0,r9,_TIF_SIGPENDING
322 beq syscall_exit_cont
327 MTMSRD(r10) /* re-enable interrupts */
333 #ifdef SHOW_SYSCALLS_TASK
334 lis r11,show_syscalls_task@ha
335 lwz r11,show_syscalls_task@l(r11)
366 do_show_syscall_exit:
367 #ifdef SHOW_SYSCALLS_TASK
368 lis r11,show_syscalls_task@ha
369 lwz r11,show_syscalls_task@l(r11)
375 stw r3,RESULT(r1) /* Save result */
385 7: .string "syscall %d(%x, %x, %x, %x, %x, "
386 77: .string "%x), current=%p\n"
387 79: .string " -> %x\n"
390 #ifdef SHOW_SYSCALLS_TASK
392 .globl show_syscalls_task
397 #endif /* SHOW_SYSCALLS */
400 * The sigsuspend and rt_sigsuspend system calls can call do_signal
401 * and thus put the process into the stopped state where we might
402 * want to examine its user state with ptrace. Therefore we need
403 * to save all the nonvolatile registers (r13 - r31) before calling
406 .globl ppc_sigsuspend
410 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
411 stw r0,TRAP(r1) /* register set saved */
414 .globl ppc_rt_sigsuspend
426 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
427 stw r0,TRAP(r1) /* register set saved */
434 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
435 stw r0,TRAP(r1) /* register set saved */
442 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
443 stw r0,TRAP(r1) /* register set saved */
446 .globl ppc_swapcontext
450 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */
451 stw r0,TRAP(r1) /* register set saved */
455 * Top-level page fault handling.
456 * This is in assembler because if do_page_fault tells us that
457 * it is a bad kernel page fault, we want to save the non-volatile
458 * registers before calling bad_page_fault.
460 .globl handle_page_fault
463 addi r3,r1,STACK_FRAME_OVERHEAD
472 addi r3,r1,STACK_FRAME_OVERHEAD
475 b ret_from_except_full
478 * This routine switches between two different tasks. The process
479 * state of one is saved on its kernel stack. Then the state
480 * of the other is restored from its kernel stack. The memory
481 * management hardware is updated to the second process's state.
482 * Finally, we can return to the second process.
483 * On entry, r3 points to the THREAD for the current task, r4
484 * points to the THREAD for the new task.
486 * This routine is always called with interrupts disabled.
488 * Note: there are two ways to get to the "going out" portion
489 * of this code; either by coming in via the entry (_switch)
490 * or via "fork" which must set up an environment equivalent
491 * to the "_switch" path. If you change this , you'll have to
492 * change the fork code also.
494 * The code which creates the new task context is in 'copy_thread'
495 * in arch/ppc/kernel/process.c
498 stwu r1,-INT_FRAME_SIZE(r1)
500 stw r0,INT_FRAME_SIZE+4(r1)
501 /* r3-r12 are caller saved -- Cort */
503 stw r0,_NIP(r1) /* Return to switch caller */
505 li r0,MSR_FP /* Disable floating-point */
506 #ifdef CONFIG_ALTIVEC
508 oris r0,r0,MSR_VEC@h /* Disable altivec */
509 mfspr r12,SPRN_VRSAVE /* save vrsave register value */
510 stw r12,THREAD+THREAD_VRSAVE(r2)
511 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
512 #endif /* CONFIG_ALTIVEC */
514 oris r0,r0,MSR_SPE@h /* Disable SPE */
515 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */
516 stw r12,THREAD+THREAD_SPEFSCR(r2)
517 #endif /* CONFIG_SPE */
518 and. r0,r0,r11 /* FP or altivec or SPE enabled? */
526 stw r1,KSP(r3) /* Set old stack pointer */
529 /* We need a sync somewhere here to make sure that if the
530 * previous task gets rescheduled on another CPU, it sees all
531 * stores it has performed on this one.
534 #endif /* CONFIG_SMP */
538 mtspr SPRG3,r0 /* Update current THREAD phys addr */
539 lwz r1,KSP(r4) /* Load new stack pointer */
541 /* save the old current 'last' for return value */
543 addi r2,r4,-THREAD /* Update current */
545 #ifdef CONFIG_ALTIVEC
547 lwz r0,THREAD+THREAD_VRSAVE(r2)
548 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
549 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
550 #endif /* CONFIG_ALTIVEC */
552 lwz r0,THREAD+THREAD_SPEFSCR(r2)
553 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */
554 #endif /* CONFIG_SPE */
558 /* r3-r12 are destroyed -- Cort */
561 lwz r4,_NIP(r1) /* Return to _switch caller in new task */
563 addi r1,r1,INT_FRAME_SIZE
566 .globl sigreturn_exit
568 subi r1,r3,STACK_FRAME_OVERHEAD
569 rlwinm r12,r1,0,0,18 /* current_thread_info() */
571 andi. r0,r9,_TIF_SYSCALL_TRACE
572 bnel- do_syscall_trace
575 .globl ret_from_except_full
576 ret_from_except_full:
580 .globl ret_from_except
582 /* Hard-disable interrupts so that current_thread_info()->flags
583 * can't change between when we test it and when we return
584 * from the interrupt. */
585 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
586 SYNC /* Some chip revs have problems here... */
587 MTMSRD(r10) /* disable interrupts */
589 lwz r3,_MSR(r1) /* Returning to user mode? */
593 user_exc_return: /* r10 contains MSR_KERNEL here */
594 /* Check current_thread_info()->flags */
597 andi. r0,r9,(_TIF_SIGPENDING|_TIF_NEED_RESCHED)
601 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
602 /* Check whether this process has its own DBCR0 value */
604 andi. r0,r0,PT_PTRACED
608 #ifdef CONFIG_PREEMPT
611 /* N.B. the only way to get here is from the beq following ret_from_except. */
613 /* check current_thread_info->preempt_count */
615 lwz r0,TI_PREEMPT(r9)
616 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
619 andi. r0,r0,_TIF_NEED_RESCHED
621 andi. r0,r3,MSR_EE /* interrupts off? */
622 beq restore /* don't schedule if so */
623 1: lis r0,PREEMPT_ACTIVE@h
624 stw r0,TI_PREEMPT(r9)
627 MTMSRD(r10) /* hard-enable interrupts */
629 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
631 MTMSRD(r10) /* disable interrupts */
634 stw r0,TI_PREEMPT(r9)
636 andi. r0,r3,_TIF_NEED_RESCHED
640 #endif /* CONFIG_PREEMPT */
642 /* interrupts are hard-disabled at this point */
655 stwcx. r0,0,r1 /* to clear the reservation */
657 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
659 andi. r10,r9,MSR_RI /* check if this exception occurred */
660 beql nonrecoverable /* at a bad place (MSR:RI = 0) */
668 * Once we put values in SRR0 and SRR1, we are in a state
669 * where exceptions are not recoverable, since taking an
670 * exception will trash SRR0 and SRR1. Therefore we clear the
671 * MSR:RI bit to indicate this. If we do take an exception,
672 * we can't return to the point of the exception but we
673 * can restart the exception exit path at the label
674 * exc_exit_restart below. -- paulus
676 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
678 MTMSRD(r10) /* clear the RI bit */
679 .globl exc_exit_restart
688 .globl exc_exit_restart_end
689 exc_exit_restart_end:
693 #else /* !(CONFIG_4xx || CONFIG_BOOKE) */
695 * This is a bit different on 4xx/Book-E because it doesn't have
696 * the RI bit in the MSR.
697 * The TLB miss handler checks if we have interrupted
698 * the exception exit path and restarts it if so
699 * (well maybe one day it will... :).
706 .globl exc_exit_restart
715 .globl exc_exit_restart_end
716 exc_exit_restart_end:
719 b . /* prevent prefetch past rfi */
722 * Returning from a critical interrupt in user mode doesn't need
723 * to be any different from a normal exception. For a critical
724 * interrupt in the kernel, we just return (without checking for
725 * preemption) since the interrupt may have happened at some crucial
726 * place (e.g. inside the TLB miss handler), and because we will be
727 * running with r1 pointing into critical_stack, not the current
728 * process's kernel stack (and therefore current_thread_info() will
729 * give the wrong answer).
730 * We have to restore various SPRs that may have been in use at the
731 * time of the critical interrupt.
733 * Note that SPRG6 is used for machine check on CONFIG_BOOKE parts and
734 * thus not saved in the critical handler
736 .globl ret_from_crit_exc
741 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
755 stwcx. r0,0,r1 /* to clear the reservation */
762 /* avoid any possible TLB misses here by turning off MSR.DR, we
763 * assume the instructions here are mapped by a pinned TLB entry */
781 lwz r10,crit_sprg0@l(COR)
783 lwz r10,crit_sprg1@l(COR)
785 lwz r10,crit_sprg4@l(COR)
787 lwz r10,crit_sprg5@l(COR)
790 lwz r10,crit_sprg6@l(COR)
793 lwz r10,crit_sprg7@l(COR)
795 lwz r10,crit_srr0@l(COR)
797 lwz r10,crit_srr1@l(COR)
799 lwz r10,crit_pid@l(COR)
807 b . /* prevent prefetch past rfci */
811 * Return from a machine check interrupt, similar to a critical
814 .globl ret_from_mcheck_exc
819 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
832 stwcx. r0,0,r1 /* to clear the reservation */
849 lis r8,mcheck_save@ha
850 lwz r10,mcheck_sprg0@l(r8)
852 lwz r10,mcheck_sprg1@l(r8)
854 lwz r10,mcheck_sprg4@l(r8)
856 lwz r10,mcheck_sprg5@l(r8)
858 lwz r10,mcheck_sprg7@l(r8)
860 lwz r10,mcheck_srr0@l(r8)
862 lwz r10,mcheck_srr1@l(r8)
864 lwz r10,mcheck_csrr0@l(r8)
866 lwz r10,mcheck_csrr1@l(r8)
868 lwz r10,mcheck_pid@l(r8)
875 #endif /* CONFIG_BOOKE */
878 * Load the DBCR0 value for a task that is being ptraced,
879 * having first saved away the global DBCR0.
882 mfmsr r0 /* first disable debug exceptions */
883 rlwinm r0,r0,0,~MSR_DE
887 lis r11,global_dbcr0@ha
888 addi r11,r11,global_dbcr0@l
889 lwz r0,THREAD+THREAD_DBCR0(r2)
896 mtspr SPRN_DBSR,r11 /* clear all pending debug events */
900 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
902 do_work: /* r10 contains MSR_KERNEL here */
903 andi. r0,r9,_TIF_NEED_RESCHED
906 do_resched: /* r10 contains MSR_KERNEL here */
909 MTMSRD(r10) /* hard-enable interrupts */
912 LOAD_MSR_KERNEL(r10,MSR_KERNEL)
914 MTMSRD(r10) /* disable interrupts */
917 andi. r0,r9,_TIF_NEED_RESCHED
919 andi. r0,r9,_TIF_SIGPENDING
921 do_user_signal: /* r10 contains MSR_KERNEL here */
924 MTMSRD(r10) /* hard-enable interrupts */
925 /* save r13-r31 in the exception frame, if not already done */
933 addi r4,r1,STACK_FRAME_OVERHEAD
939 * We come here when we are at the end of handling an exception
940 * that occurred at a place where taking an exception will lose
941 * state information, such as the contents of SRR0 and SRR1.
944 lis r10,exc_exit_restart_end@ha
945 addi r10,r10,exc_exit_restart_end@l
948 lis r11,exc_exit_restart@ha
949 addi r11,r11,exc_exit_restart@l
952 lis r10,ee_restarts@ha
953 lwz r12,ee_restarts@l(r10)
955 stw r12,ee_restarts@l(r10)
956 mr r12,r11 /* restart at exc_exit_restart */
958 3: /* OK, we can't recover, kill this process */
959 /* but the 601 doesn't implement the RI bit, so assume it's OK */
962 END_FTR_SECTION_IFSET(CPU_FTR_601)
969 4: addi r3,r1,STACK_FRAME_OVERHEAD
970 bl nonrecoverable_exception
971 /* shouldn't return */
977 * PROM code for specific machines follows. Put it
978 * here so it's easy to add arch-specific sections later.
983 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
984 * called with the MMU off.
987 stwu r1,-INT_FRAME_SIZE(r1)
989 stw r0,INT_FRAME_SIZE+4(r1)
991 lwz r4,rtas_data@l(r4)
992 lis r6,1f@ha /* physical return address for rtas */
997 lwz r8,rtas_entry@l(r8)
1000 LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1001 SYNC /* disable interrupts so SRR0/1 */
1002 MTMSRD(r0) /* don't get trashed */
1003 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1011 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1012 lwz r9,8(r9) /* original msr value */
1014 addi r1,r1,INT_FRAME_SIZE
1019 RFI /* return to caller */
1021 .globl machine_check_in_rtas
1022 machine_check_in_rtas:
1024 /* XXX load up BATs and panic */
1026 #endif /* CONFIG_PPC_OF */