X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fsparc64%2Fkernel%2Fkprobes.c;h=ae221f0d4a6f2171ad9e43d83f08ca1cc8d3fa09;hb=refs%2Fheads%2Fvserver;hp=7066d7ba667a5e89f447f9231b0abfa2910e22e7;hpb=f7f1b0f1e2fbadeab12d24236000e778aa9b1ead;p=linux-2.6.git diff --git a/arch/sparc64/kernel/kprobes.c b/arch/sparc64/kernel/kprobes.c index 7066d7ba6..ae221f0d4 100644 --- a/arch/sparc64/kernel/kprobes.c +++ b/arch/sparc64/kernel/kprobes.c @@ -3,12 +3,13 @@ * Copyright (C) 2004 David S. Miller */ -#include #include #include - +#include #include #include +#include +#include /* We do not have hardware single-stepping on sparc64. * So we implement software single-stepping with breakpoint @@ -38,92 +39,124 @@ * - Mark that we are no longer actively in a kprobe. */ -int arch_prepare_kprobe(struct kprobe *p) +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +int __kprobes arch_prepare_kprobe(struct kprobe *p) { + p->ainsn.insn[0] = *p->addr; + flushi(&p->ainsn.insn[0]); + + p->ainsn.insn[1] = BREAKPOINT_INSTRUCTION_2; + flushi(&p->ainsn.insn[1]); + + p->opcode = *p->addr; return 0; } -void arch_copy_kprobe(struct kprobe *p) +void __kprobes arch_arm_kprobe(struct kprobe *p) { - p->ainsn.insn[0] = *p->addr; - p->ainsn.insn[1] = BREAKPOINT_INSTRUCTION_2; + *p->addr = BREAKPOINT_INSTRUCTION; + flushi(p->addr); } -void arch_remove_kprobe(struct kprobe *p) +void __kprobes arch_disarm_kprobe(struct kprobe *p) { + *p->addr = p->opcode; + flushi(p->addr); } -/* kprobe_status settings */ -#define KPROBE_HIT_ACTIVE 0x00000001 -#define KPROBE_HIT_SS 0x00000002 +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; + kcb->prev_kprobe.orig_tnpc = kcb->kprobe_orig_tnpc; + kcb->prev_kprobe.orig_tstate_pil = kcb->kprobe_orig_tstate_pil; +} -static struct kprobe *current_kprobe; -static unsigned long current_kprobe_orig_tnpc; -static unsigned long current_kprobe_orig_tstate_pil; -static unsigned int kprobe_status; +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; + kcb->kprobe_status = kcb->prev_kprobe.status; + kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc; + kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil; +} -static inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) +static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + __get_cpu_var(current_kprobe) = p; + kcb->kprobe_orig_tnpc = regs->tnpc; + kcb->kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL); +} + +static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) { - current_kprobe_orig_tnpc = regs->tnpc; - current_kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL); regs->tstate |= TSTATE_PIL; /*single step inline, if it a breakpoint instruction*/ if (p->opcode == BREAKPOINT_INSTRUCTION) { regs->tpc = (unsigned long) p->addr; - regs->tnpc = current_kprobe_orig_tnpc; + regs->tnpc = kcb->kprobe_orig_tnpc; } else { regs->tpc = (unsigned long) &p->ainsn.insn[0]; regs->tnpc = (unsigned long) &p->ainsn.insn[1]; } } -static inline void disarm_kprobe(struct kprobe *p, struct pt_regs *regs) -{ - *p->addr = p->opcode; - flushi(p->addr); - - regs->tpc = (unsigned long) p->addr; - regs->tnpc = current_kprobe_orig_tnpc; - regs->tstate = ((regs->tstate & ~TSTATE_PIL) | - current_kprobe_orig_tstate_pil); -} - -static int kprobe_handler(struct pt_regs *regs) +static int __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p; void *addr = (void *) regs->tpc; int ret = 0; + struct kprobe_ctlblk *kcb; + /* + * We don't want to be preempted for the entire + * duration of kprobe processing + */ preempt_disable(); + kcb = get_kprobe_ctlblk(); if (kprobe_running()) { - /* We *are* holding lock here, so this is safe. - * Disarm the probe we just hit, and ignore it. - */ p = get_kprobe(addr); if (p) { - if (kprobe_status == KPROBE_HIT_SS) { + if (kcb->kprobe_status == KPROBE_HIT_SS) { regs->tstate = ((regs->tstate & ~TSTATE_PIL) | - current_kprobe_orig_tstate_pil); - unlock_kprobes(); + kcb->kprobe_orig_tstate_pil); goto no_kprobe; } - disarm_kprobe(p, regs); - ret = 1; + /* We have reentered the kprobe_handler(), since + * another probe was hit while within the handler. + * We here save the original kprobes variables and + * just single step on the instruction of the new probe + * without calling any user handlers. + */ + save_previous_kprobe(kcb); + set_current_kprobe(p, regs, kcb); + kprobes_inc_nmissed_count(p); + kcb->kprobe_status = KPROBE_REENTER; + prepare_singlestep(p, regs, kcb); + return 1; } else { - p = current_kprobe; + if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { + /* The breakpoint instruction was removed by + * another cpu right after we hit, no further + * handling of this interrupt is appropriate + */ + ret = 1; + goto no_kprobe; + } + p = __get_cpu_var(current_kprobe); if (p->break_handler && p->break_handler(p, regs)) goto ss_probe; } - /* If it's not ours, can't be delete race, (we hold lock). */ goto no_kprobe; } - lock_kprobes(); p = get_kprobe(addr); if (!p) { - unlock_kprobes(); if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { /* * The breakpoint instruction was removed right @@ -138,14 +171,14 @@ static int kprobe_handler(struct pt_regs *regs) goto no_kprobe; } - kprobe_status = KPROBE_HIT_ACTIVE; - current_kprobe = p; + set_current_kprobe(p, regs, kcb); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (p->pre_handler && p->pre_handler(p, regs)) return 1; ss_probe: - prepare_singlestep(p, regs); - kprobe_status = KPROBE_HIT_SS; + prepare_singlestep(p, regs, kcb); + kcb->kprobe_status = KPROBE_HIT_SS; return 1; no_kprobe: @@ -156,15 +189,19 @@ no_kprobe: /* If INSN is a relative control transfer instruction, * return the corrected branch destination value. * - * The original INSN location was REAL_PC, it actually - * executed at PC and produced destination address NPC. + * regs->tpc and regs->tnpc still hold the values of the + * program counters at the time of trap due to the execution + * of the BREAKPOINT_INSTRUCTION_2 at p->ainsn.insn[1] + * */ -static unsigned long relbranch_fixup(u32 insn, unsigned long real_pc, - unsigned long pc, unsigned long npc) +static unsigned long __kprobes relbranch_fixup(u32 insn, struct kprobe *p, + struct pt_regs *regs) { + unsigned long real_pc = (unsigned long) p->addr; + /* Branch not taken, no mods necessary. */ - if (npc == pc + 0x4UL) - return real_pc + 0x4UL; + if (regs->tnpc == regs->tpc + 0x4UL) + return real_pc + 0x8UL; /* The three cases are call, branch w/prediction, * and traditional branch. @@ -172,29 +209,37 @@ static unsigned long relbranch_fixup(u32 insn, unsigned long real_pc, if ((insn & 0xc0000000) == 0x40000000 || (insn & 0xc1c00000) == 0x00400000 || (insn & 0xc1c00000) == 0x00800000) { + unsigned long ainsn_addr; + + ainsn_addr = (unsigned long) &p->ainsn.insn[0]; + /* The instruction did all the work for us * already, just apply the offset to the correct * instruction location. */ - return (real_pc + (npc - pc)); + return (real_pc + (regs->tnpc - ainsn_addr)); } - return real_pc + 0x4UL; + /* It is jmpl or some other absolute PC modification instruction, + * leave NPC as-is. + */ + return regs->tnpc; } /* If INSN is an instruction which writes it's PC location * into a destination register, fix that up. */ -static void retpc_fixup(struct pt_regs *regs, u32 insn, unsigned long real_pc) +static void __kprobes retpc_fixup(struct pt_regs *regs, u32 insn, + unsigned long real_pc) { unsigned long *slot = NULL; - /* Simplest cast is call, which always uses %o7 */ + /* Simplest case is 'call', which always uses %o7 */ if ((insn & 0xc0000000) == 0x40000000) { slot = ®s->u_regs[UREG_I7]; } - /* Jmpl encodes the register inside of the opcode */ + /* 'jmpl' encodes the register inside of the opcode */ if ((insn & 0xc1f80000) == 0x81c00000) { unsigned long rd = ((insn >> 25) & 0x1f); @@ -216,95 +261,165 @@ static void retpc_fixup(struct pt_regs *regs, u32 insn, unsigned long real_pc) /* * Called after single-stepping. p->addr is the address of the - * instruction whose first byte has been replaced by the breakpoint + * instruction which has been replaced by the breakpoint * instruction. To avoid the SMP problems that can occur when we * temporarily put back the original opcode to single-step, we * single-stepped a copy of the instruction. The address of this - * copy is p->ainsn.insn. + * copy is &p->ainsn.insn[0]. * * This function prepares to return from the post-single-step * breakpoint trap. */ -static void resume_execution(struct kprobe *p, struct pt_regs *regs) +static void __kprobes resume_execution(struct kprobe *p, + struct pt_regs *regs, struct kprobe_ctlblk *kcb) { u32 insn = p->ainsn.insn[0]; - regs->tpc = current_kprobe_orig_tnpc; - regs->tnpc = relbranch_fixup(insn, - (unsigned long) p->addr, - (unsigned long) &p->ainsn.insn[0], - regs->tnpc); + regs->tnpc = relbranch_fixup(insn, p, regs); + + /* This assignment must occur after relbranch_fixup() */ + regs->tpc = kcb->kprobe_orig_tnpc; + retpc_fixup(regs, insn, (unsigned long) p->addr); regs->tstate = ((regs->tstate & ~TSTATE_PIL) | - current_kprobe_orig_tstate_pil); + kcb->kprobe_orig_tstate_pil); } -static inline int post_kprobe_handler(struct pt_regs *regs) +static int __kprobes post_kprobe_handler(struct pt_regs *regs) { - if (!kprobe_running()) + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (!cur) return 0; - if (current_kprobe->post_handler) - current_kprobe->post_handler(current_kprobe, regs, 0); + if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + cur->post_handler(cur, regs, 0); + } - resume_execution(current_kprobe, regs); + resume_execution(cur, regs, kcb); - unlock_kprobes(); + /*Restore back the original saved kprobes variables and continue. */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + goto out; + } + reset_current_kprobe(); +out: preempt_enable_no_resched(); return 1; } -/* Interrupts disabled, kprobe_lock held. */ -static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr) +static int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) { - if (current_kprobe->fault_handler - && current_kprobe->fault_handler(current_kprobe, regs, trapnr)) - return 1; + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + const struct exception_table_entry *entry; + + switch(kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the tpc points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->tpc = (unsigned long)cur->addr; + regs->tnpc = kcb->kprobe_orig_tnpc; + regs->tstate = ((regs->tstate & ~TSTATE_PIL) | + kcb->kprobe_orig_tstate_pil); + if (kcb->kprobe_status == KPROBE_REENTER) + restore_previous_kprobe(kcb); + else + reset_current_kprobe(); + preempt_enable_no_resched(); + break; + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * We increment the nmissed count for accounting, + * we can also use npre/npostfault count for accouting + * these specific fault cases. + */ + kprobes_inc_nmissed_count(cur); + + /* + * We come here because instructions in the pre/post + * handler caused the page_fault, this could happen + * if handler tries to access user space by + * copy_from_user(), get_user() etc. Let the + * user-specified handler try to fix it first. + */ + if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) + return 1; - if (kprobe_status & KPROBE_HIT_SS) { - resume_execution(current_kprobe, regs); + /* + * In case the user-specified fault handler returned + * zero, try to fix up. + */ - unlock_kprobes(); - preempt_enable_no_resched(); + entry = search_exception_tables(regs->tpc); + if (entry) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return 1; + } + + /* + * fixup_exception() could not handle it, + * Let do_page_fault() fix it. + */ + break; + default: + break; } + return 0; } /* * Wrapper routine to for handling exceptions. */ -int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, - void *data) +int __kprobes kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data) { struct die_args *args = (struct die_args *)data; + int ret = NOTIFY_DONE; + + if (args->regs && user_mode(args->regs)) + return ret; + switch (val) { case DIE_DEBUG: if (kprobe_handler(args->regs)) - return NOTIFY_STOP; + ret = NOTIFY_STOP; break; case DIE_DEBUG_2: if (post_kprobe_handler(args->regs)) - return NOTIFY_STOP; + ret = NOTIFY_STOP; break; case DIE_GPF: - if (kprobe_running() && - kprobe_fault_handler(args->regs, args->trapnr)) - return NOTIFY_STOP; - break; case DIE_PAGE_FAULT: + /* kprobe_running() needs smp_processor_id() */ + preempt_disable(); if (kprobe_running() && kprobe_fault_handler(args->regs, args->trapnr)) - return NOTIFY_STOP; + ret = NOTIFY_STOP; + preempt_enable(); break; default: break; } - return NOTIFY_DONE; + return ret; } -asmlinkage void kprobe_trap(unsigned long trap_level, struct pt_regs *regs) +asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, + struct pt_regs *regs) { BUG_ON(trap_level != 0x170 && trap_level != 0x171); @@ -324,24 +439,12 @@ asmlinkage void kprobe_trap(unsigned long trap_level, struct pt_regs *regs) } /* Jprobes support. */ -static struct pt_regs jprobe_saved_regs; -static struct pt_regs *jprobe_saved_regs_location; -static struct sparc_stackf jprobe_saved_stack; - -int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) +int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) { struct jprobe *jp = container_of(p, struct jprobe, kp); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - jprobe_saved_regs_location = regs; - memcpy(&jprobe_saved_regs, regs, sizeof(*regs)); - - /* Save a whole stack frame, this gets arguments - * pushed onto the stack after using up all the - * arg registers. - */ - memcpy(&jprobe_saved_stack, - (char *) (regs->u_regs[UREG_FP] + STACK_BIAS), - sizeof(jprobe_saved_stack)); + memcpy(&(kcb->jprobe_saved_regs), regs, sizeof(*regs)); regs->tpc = (unsigned long) jp->entry; regs->tnpc = ((unsigned long) jp->entry) + 0x4UL; @@ -350,45 +453,42 @@ int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) return 1; } -void jprobe_return(void) +void __kprobes jprobe_return(void) { - preempt_enable_no_resched(); - __asm__ __volatile__( - ".globl jprobe_return_trap_instruction\n" + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + register unsigned long orig_fp asm("g1"); + + orig_fp = kcb->jprobe_saved_regs.u_regs[UREG_FP]; + __asm__ __volatile__("\n" +"1: cmp %%sp, %0\n\t" + "blu,a,pt %%xcc, 1b\n\t" + " restore\n\t" + ".globl jprobe_return_trap_instruction\n" "jprobe_return_trap_instruction:\n\t" - "ta 0x70"); + "ta 0x70" + : /* no outputs */ + : "r" (orig_fp)); } extern void jprobe_return_trap_instruction(void); extern void __show_regs(struct pt_regs * regs); -int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) +int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) { u32 *addr = (u32 *) regs->tpc; + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); if (addr == (u32 *) jprobe_return_trap_instruction) { - if (jprobe_saved_regs_location != regs) { - printk("JPROBE: Current regs (%p) does not match " - "saved regs (%p).\n", - regs, jprobe_saved_regs_location); - printk("JPROBE: Saved registers\n"); - __show_regs(jprobe_saved_regs_location); - printk("JPROBE: Current registers\n"); - __show_regs(regs); - BUG(); - } - /* Restore old register state. Do pt_regs - * first so that UREG_FP is the original one for - * the stack frame restore. - */ - memcpy(regs, &jprobe_saved_regs, sizeof(*regs)); - - memcpy((char *) (regs->u_regs[UREG_FP] + STACK_BIAS), - &jprobe_saved_stack, - sizeof(jprobe_saved_stack)); - + memcpy(regs, &(kcb->jprobe_saved_regs), sizeof(*regs)); + preempt_enable_no_resched(); return 1; } return 0; } + +/* architecture specific initialization */ +int arch_init_kprobes(void) +{ + return 0; +}