2 * Copyright (C) 1999-2004 Hewlett-Packard Co
3 * David Mosberger-Tang <davidm@hpl.hp.com>
4 * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
5 * - Change pt_regs_off() to make it less dependant on pt_regs structure.
8 * This file implements call frame unwind support for the Linux
9 * kernel. Parsing and processing the unwind information is
10 * time-consuming, so this implementation translates the unwind
11 * descriptors into unwind scripts. These scripts are very simple
12 * (basically a sequence of assignments) and efficient to execute.
13 * They are cached for later re-use. Each script is specific for a
14 * given instruction pointer address and the set of predicate values
15 * that the script depends on (most unwind descriptors are
16 * unconditional and scripts often do not depend on predicates at
17 * all). This code is based on the unwind conventions described in
18 * the "IA-64 Software Conventions and Runtime Architecture" manual.
21 * o updates to the global unwind data (in structure "unw") are serialized
22 * by the unw.lock spinlock
23 * o each unwind script has its own read-write lock; a thread must acquire
24 * a read lock before executing a script and must acquire a write lock
25 * before modifying a script
26 * o if both the unw.lock spinlock and a script's read-write lock must be
27 * acquired, then the read-write lock must be acquired first.
29 #include <linux/module.h>
30 #include <linux/bootmem.h>
31 #include <linux/elf.h>
32 #include <linux/kernel.h>
33 #include <linux/sched.h>
34 #include <linux/slab.h>
36 #include <asm/unwind.h>
38 #include <asm/delay.h>
40 #include <asm/ptrace.h>
41 #include <asm/ptrace_offsets.h>
43 #include <asm/sections.h>
44 #include <asm/system.h>
45 #include <asm/uaccess.h>
52 #define UNW_LOG_CACHE_SIZE 7 /* each unw_script is ~256 bytes in size */
53 #define UNW_CACHE_SIZE (1 << UNW_LOG_CACHE_SIZE)
55 #define UNW_LOG_HASH_SIZE (UNW_LOG_CACHE_SIZE + 1)
56 #define UNW_HASH_SIZE (1 << UNW_LOG_HASH_SIZE)
58 #define UNW_STATS 0 /* WARNING: this disabled interrupts for long time-spans!! */
61 static unsigned int unw_debug_level = UNW_DEBUG;
62 # define UNW_DEBUG_ON(n) unw_debug_level >= n
63 /* Do not code a printk level, not all debug lines end in newline */
64 # define UNW_DPRINT(n, ...) if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
66 #else /* !UNW_DEBUG */
67 # define UNW_DEBUG_ON(n) 0
68 # define UNW_DPRINT(n, ...)
69 #endif /* UNW_DEBUG */
77 #define alloc_reg_state() kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
78 #define free_reg_state(usr) kfree(usr)
79 #define alloc_labeled_state() kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
80 #define free_labeled_state(usr) kfree(usr)
82 typedef unsigned long unw_word;
83 typedef unsigned char unw_hash_index_t;
86 spinlock_t lock; /* spinlock for unwind data */
88 /* list of unwind tables (one per load-module) */
89 struct unw_table *tables;
91 unsigned long r0; /* constant 0 for r0 */
93 /* table of registers that prologues can save (and order in which they're saved): */
94 const unsigned char save_order[8];
96 /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
97 unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
99 unsigned short lru_head; /* index of lead-recently used script */
100 unsigned short lru_tail; /* index of most-recently used script */
102 /* index into unw_frame_info for preserved register i */
103 unsigned short preg_index[UNW_NUM_REGS];
105 short pt_regs_offsets[32];
107 /* unwind table for the kernel: */
108 struct unw_table kernel_table;
110 /* unwind table describing the gate page (kernel code that is mapped into user space): */
111 size_t gate_table_size;
112 unsigned long *gate_table;
114 /* hash table that maps instruction pointer to script index: */
115 unsigned short hash[UNW_HASH_SIZE];
118 struct unw_script cache[UNW_CACHE_SIZE];
121 const char *preg_name[UNW_NUM_REGS];
129 int collision_chain_traversals;
132 unsigned long build_time;
133 unsigned long run_time;
134 unsigned long parse_time;
141 unsigned long init_time;
142 unsigned long unwind_time;
149 .tables = &unw.kernel_table,
150 .lock = SPIN_LOCK_UNLOCKED,
152 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
153 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
156 offsetof(struct unw_frame_info, pri_unat_loc)/8, /* PRI_UNAT_GR */
157 offsetof(struct unw_frame_info, pri_unat_loc)/8, /* PRI_UNAT_MEM */
158 offsetof(struct unw_frame_info, bsp_loc)/8,
159 offsetof(struct unw_frame_info, bspstore_loc)/8,
160 offsetof(struct unw_frame_info, pfs_loc)/8,
161 offsetof(struct unw_frame_info, rnat_loc)/8,
162 offsetof(struct unw_frame_info, psp)/8,
163 offsetof(struct unw_frame_info, rp_loc)/8,
164 offsetof(struct unw_frame_info, r4)/8,
165 offsetof(struct unw_frame_info, r5)/8,
166 offsetof(struct unw_frame_info, r6)/8,
167 offsetof(struct unw_frame_info, r7)/8,
168 offsetof(struct unw_frame_info, unat_loc)/8,
169 offsetof(struct unw_frame_info, pr_loc)/8,
170 offsetof(struct unw_frame_info, lc_loc)/8,
171 offsetof(struct unw_frame_info, fpsr_loc)/8,
172 offsetof(struct unw_frame_info, b1_loc)/8,
173 offsetof(struct unw_frame_info, b2_loc)/8,
174 offsetof(struct unw_frame_info, b3_loc)/8,
175 offsetof(struct unw_frame_info, b4_loc)/8,
176 offsetof(struct unw_frame_info, b5_loc)/8,
177 offsetof(struct unw_frame_info, f2_loc)/8,
178 offsetof(struct unw_frame_info, f3_loc)/8,
179 offsetof(struct unw_frame_info, f4_loc)/8,
180 offsetof(struct unw_frame_info, f5_loc)/8,
181 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
182 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
183 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
184 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
185 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
186 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
187 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
188 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
189 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
190 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
191 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
192 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
193 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
194 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
195 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
196 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
200 offsetof(struct pt_regs, r1),
201 offsetof(struct pt_regs, r2),
202 offsetof(struct pt_regs, r3),
203 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
204 offsetof(struct pt_regs, r8),
205 offsetof(struct pt_regs, r9),
206 offsetof(struct pt_regs, r10),
207 offsetof(struct pt_regs, r11),
208 offsetof(struct pt_regs, r12),
209 offsetof(struct pt_regs, r13),
210 offsetof(struct pt_regs, r14),
211 offsetof(struct pt_regs, r15),
212 offsetof(struct pt_regs, r16),
213 offsetof(struct pt_regs, r17),
214 offsetof(struct pt_regs, r18),
215 offsetof(struct pt_regs, r19),
216 offsetof(struct pt_regs, r20),
217 offsetof(struct pt_regs, r21),
218 offsetof(struct pt_regs, r22),
219 offsetof(struct pt_regs, r23),
220 offsetof(struct pt_regs, r24),
221 offsetof(struct pt_regs, r25),
222 offsetof(struct pt_regs, r26),
223 offsetof(struct pt_regs, r27),
224 offsetof(struct pt_regs, r28),
225 offsetof(struct pt_regs, r29),
226 offsetof(struct pt_regs, r30),
227 offsetof(struct pt_regs, r31),
229 .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
232 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
233 "r4", "r5", "r6", "r7",
234 "ar.unat", "pr", "ar.lc", "ar.fpsr",
235 "b1", "b2", "b3", "b4", "b5",
236 "f2", "f3", "f4", "f5",
237 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
238 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
244 read_only (void *addr)
246 return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
250 * Returns offset of rREG in struct pt_regs.
252 static inline unsigned long
253 pt_regs_off (unsigned long reg)
257 if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
258 off = unw.pt_regs_offsets[reg];
261 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
264 return (unsigned long) off;
267 static inline struct pt_regs *
268 get_scratch_regs (struct unw_frame_info *info)
271 /* This should not happen with valid unwind info. */
272 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
273 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
274 info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
276 info->pt = info->sp - 16;
278 UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
279 return (struct pt_regs *) info->pt;
282 /* Unwind accessors. */
285 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
287 unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
288 struct unw_ireg *ireg;
291 if ((unsigned) regnum - 1 >= 127) {
292 if (regnum == 0 && !write) {
293 *val = 0; /* read r0 always returns 0 */
297 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
298 __FUNCTION__, regnum);
303 if (regnum >= 4 && regnum <= 7) {
304 /* access a preserved register */
305 ireg = &info->r4 + (regnum - 4);
308 nat_addr = addr + ireg->nat.off;
309 switch (ireg->nat.type) {
311 /* simulate getf.sig/setf.sig */
314 /* write NaTVal and be done with it */
321 if (addr[0] == 0 && addr[1] == 0x1ffe) {
322 /* return NaT and be done with it */
331 nat_addr = &dummy_nat;
335 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
339 nat_addr = ia64_rse_rnat_addr(addr);
340 if ((unsigned long) addr < info->regstk.limit
341 || (unsigned long) addr >= info->regstk.top)
343 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
345 __FUNCTION__, (void *) addr,
350 if ((unsigned long) nat_addr >= info->regstk.top)
351 nat_addr = &info->sw->ar_rnat;
352 nat_mask = (1UL << ia64_rse_slot_num(addr));
356 addr = &info->sw->r4 + (regnum - 4);
357 nat_addr = &info->sw->ar_unat;
358 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
361 /* access a scratch register */
362 pt = get_scratch_regs(info);
363 addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
364 if (info->pri_unat_loc)
365 nat_addr = info->pri_unat_loc;
367 nat_addr = &info->sw->ar_unat;
368 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
371 /* access a stacked register */
372 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
373 nat_addr = ia64_rse_rnat_addr(addr);
374 if ((unsigned long) addr < info->regstk.limit
375 || (unsigned long) addr >= info->regstk.top)
377 UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
378 "of rbs\n", __FUNCTION__);
381 if ((unsigned long) nat_addr >= info->regstk.top)
382 nat_addr = &info->sw->ar_rnat;
383 nat_mask = (1UL << ia64_rse_slot_num(addr));
387 if (read_only(addr)) {
388 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
393 *nat_addr |= nat_mask;
395 *nat_addr &= ~nat_mask;
398 if ((*nat_addr & nat_mask) == 0) {
402 *val = 0; /* if register is a NaT, *addr may contain kernel data! */
408 EXPORT_SYMBOL(unw_access_gr);
411 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
418 case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
419 case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
420 case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
423 case 1: case 2: case 3: case 4: case 5:
424 addr = *(&info->b1_loc + (regnum - 1));
426 addr = &info->sw->b1 + (regnum - 1);
430 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
431 __FUNCTION__, regnum);
435 if (read_only(addr)) {
436 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
444 EXPORT_SYMBOL(unw_access_br);
447 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
449 struct ia64_fpreg *addr = NULL;
452 if ((unsigned) (regnum - 2) >= 126) {
453 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
454 __FUNCTION__, regnum);
459 addr = *(&info->f2_loc + (regnum - 2));
461 addr = &info->sw->f2 + (regnum - 2);
462 } else if (regnum <= 15) {
464 pt = get_scratch_regs(info);
465 addr = &pt->f6 + (regnum - 6);
468 addr = &info->sw->f12 + (regnum - 12);
469 } else if (regnum <= 31) {
470 addr = info->fr_loc[regnum - 16];
472 addr = &info->sw->f16 + (regnum - 16);
474 struct task_struct *t = info->task;
480 addr = t->thread.fph + (regnum - 32);
484 if (read_only(addr)) {
485 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
493 EXPORT_SYMBOL(unw_access_fr);
496 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
503 addr = info->bsp_loc;
505 addr = &info->sw->ar_bspstore;
508 case UNW_AR_BSPSTORE:
509 addr = info->bspstore_loc;
511 addr = &info->sw->ar_bspstore;
515 addr = info->pfs_loc;
517 addr = &info->sw->ar_pfs;
521 addr = info->rnat_loc;
523 addr = &info->sw->ar_rnat;
527 addr = info->unat_loc;
529 addr = &info->sw->ar_unat;
535 addr = &info->sw->ar_lc;
543 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
545 *val = (*info->cfm_loc >> 52) & 0x3f;
549 addr = info->fpsr_loc;
551 addr = &info->sw->ar_fpsr;
555 pt = get_scratch_regs(info);
560 pt = get_scratch_regs(info);
565 pt = get_scratch_regs(info);
570 pt = get_scratch_regs(info);
575 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
576 __FUNCTION__, regnum);
581 if (read_only(addr)) {
582 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
590 EXPORT_SYMBOL(unw_access_ar);
593 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
599 addr = &info->sw->pr;
602 if (read_only(addr)) {
603 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
611 EXPORT_SYMBOL(unw_access_pr);
614 /* Routines to manipulate the state stack. */
617 push (struct unw_state_record *sr)
619 struct unw_reg_state *rs;
621 rs = alloc_reg_state();
623 printk(KERN_ERR "unwind: cannot stack reg state!\n");
626 memcpy(rs, &sr->curr, sizeof(*rs));
631 pop (struct unw_state_record *sr)
633 struct unw_reg_state *rs = sr->curr.next;
636 printk(KERN_ERR "unwind: stack underflow!\n");
639 memcpy(&sr->curr, rs, sizeof(*rs));
643 /* Make a copy of the state stack. Non-recursive to avoid stack overflows. */
644 static struct unw_reg_state *
645 dup_state_stack (struct unw_reg_state *rs)
647 struct unw_reg_state *copy, *prev = NULL, *first = NULL;
650 copy = alloc_reg_state();
652 printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
655 memcpy(copy, rs, sizeof(*copy));
666 /* Free all stacked register states (but not RS itself). */
668 free_state_stack (struct unw_reg_state *rs)
670 struct unw_reg_state *p, *next;
672 for (p = rs->next; p != NULL; p = next) {
679 /* Unwind decoder routines */
681 static enum unw_register_index __attribute_const__
682 decode_abreg (unsigned char abreg, int memory)
685 case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
686 case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
687 case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
688 case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
689 case 0x60: return UNW_REG_PR;
690 case 0x61: return UNW_REG_PSP;
691 case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
692 case 0x63: return UNW_REG_RP;
693 case 0x64: return UNW_REG_BSP;
694 case 0x65: return UNW_REG_BSPSTORE;
695 case 0x66: return UNW_REG_RNAT;
696 case 0x67: return UNW_REG_UNAT;
697 case 0x68: return UNW_REG_FPSR;
698 case 0x69: return UNW_REG_PFS;
699 case 0x6a: return UNW_REG_LC;
703 UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
708 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
712 if (reg->when == UNW_WHEN_NEVER)
717 alloc_spill_area (unsigned long *offp, unsigned long regsize,
718 struct unw_reg_info *lo, struct unw_reg_info *hi)
720 struct unw_reg_info *reg;
722 for (reg = hi; reg >= lo; --reg) {
723 if (reg->where == UNW_WHERE_SPILL_HOME) {
724 reg->where = UNW_WHERE_PSPREL;
732 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
734 struct unw_reg_info *reg;
736 for (reg = *regp; reg <= lim; ++reg) {
737 if (reg->where == UNW_WHERE_SPILL_HOME) {
743 UNW_DPRINT(0, "unwind.%s: excess spill!\n", __FUNCTION__);
747 finish_prologue (struct unw_state_record *sr)
749 struct unw_reg_info *reg;
754 * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
755 * for Using Unwind Descriptors", rule 3):
757 for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
758 reg = sr->curr.reg + unw.save_order[i];
759 if (reg->where == UNW_WHERE_GR_SAVE) {
760 reg->where = UNW_WHERE_GR;
761 reg->val = sr->gr_save_loc++;
766 * Next, compute when the fp, general, and branch registers get
767 * saved. This must come before alloc_spill_area() because
768 * we need to know which registers are spilled to their home
772 unsigned char kind, mask = 0, *cp = sr->imask;
774 static const unsigned char limit[3] = {
775 UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
777 struct unw_reg_info *(regs[3]);
779 regs[0] = sr->curr.reg + UNW_REG_F2;
780 regs[1] = sr->curr.reg + UNW_REG_R4;
781 regs[2] = sr->curr.reg + UNW_REG_B1;
783 for (t = 0; t < sr->region_len; ++t) {
786 kind = (mask >> 2*(3-(t & 3))) & 3;
788 spill_next_when(®s[kind - 1], sr->curr.reg + limit[kind - 1],
789 sr->region_start + t);
793 * Next, lay out the memory stack spill area:
795 if (sr->any_spills) {
796 off = sr->spill_offset;
797 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
798 alloc_spill_area(&off, 8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
799 alloc_spill_area(&off, 8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
804 * Region header descriptors.
808 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
809 struct unw_state_record *sr)
813 if (!(sr->in_body || sr->first_region))
815 sr->first_region = 0;
817 /* check if we're done: */
818 if (sr->when_target < sr->region_start + sr->region_len) {
823 region_start = sr->region_start + sr->region_len;
825 for (i = 0; i < sr->epilogue_count; ++i)
827 sr->epilogue_count = 0;
828 sr->epilogue_start = UNW_WHEN_NEVER;
830 sr->region_start = region_start;
831 sr->region_len = rlen;
837 for (i = 0; i < 4; ++i) {
839 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
840 sr->region_start + sr->region_len - 1, grsave++);
843 sr->gr_save_loc = grsave;
846 sr->spill_offset = 0x10; /* default to psp+16 */
851 * Prologue descriptors.
855 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
857 if (abi == 3 && context == 'i') {
858 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
859 UNW_DPRINT(3, "unwind.%s: interrupt frame\n", __FUNCTION__);
862 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
863 __FUNCTION__, abi, context);
867 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
871 for (i = 0; i < 5; ++i) {
873 set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
874 sr->region_start + sr->region_len - 1, gr++);
880 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
884 for (i = 0; i < 5; ++i) {
886 set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
887 sr->region_start + sr->region_len - 1, 0);
895 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
899 for (i = 0; i < 4; ++i) {
900 if ((grmask & 1) != 0) {
901 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
902 sr->region_start + sr->region_len - 1, 0);
907 for (i = 0; i < 20; ++i) {
908 if ((frmask & 1) != 0) {
909 int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
910 set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
911 sr->region_start + sr->region_len - 1, 0);
919 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
923 for (i = 0; i < 4; ++i) {
924 if ((frmask & 1) != 0) {
925 set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
926 sr->region_start + sr->region_len - 1, 0);
934 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
938 for (i = 0; i < 4; ++i) {
939 if ((grmask & 1) != 0)
940 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
941 sr->region_start + sr->region_len - 1, gr++);
947 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
951 for (i = 0; i < 4; ++i) {
952 if ((grmask & 1) != 0) {
953 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
954 sr->region_start + sr->region_len - 1, 0);
962 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
964 set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
965 sr->region_start + min_t(int, t, sr->region_len - 1), 16*size);
969 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
971 sr->curr.reg[UNW_REG_PSP].when = sr->region_start + min_t(int, t, sr->region_len - 1);
975 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
977 set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
981 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
983 set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
988 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
990 set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
995 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
997 sr->return_link_reg = dst;
1001 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
1003 struct unw_reg_info *reg = sr->curr.reg + regnum;
1005 if (reg->where == UNW_WHERE_NONE)
1006 reg->where = UNW_WHERE_GR_SAVE;
1007 reg->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1011 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1013 sr->spill_offset = 0x10 - 4*pspoff;
1016 static inline unsigned char *
1017 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1020 return imaskp + (2*sr->region_len + 7)/8;
1027 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1029 sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1030 sr->epilogue_count = ecount + 1;
1034 desc_copy_state (unw_word label, struct unw_state_record *sr)
1036 struct unw_labeled_state *ls;
1038 for (ls = sr->labeled_states; ls; ls = ls->next) {
1039 if (ls->label == label) {
1040 free_state_stack(&sr->curr);
1041 memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1042 sr->curr.next = dup_state_stack(ls->saved_state.next);
1046 printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1050 desc_label_state (unw_word label, struct unw_state_record *sr)
1052 struct unw_labeled_state *ls;
1054 ls = alloc_labeled_state();
1056 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1060 memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1061 ls->saved_state.next = dup_state_stack(sr->curr.next);
1063 /* insert into list of labeled states: */
1064 ls->next = sr->labeled_states;
1065 sr->labeled_states = ls;
1069 * General descriptors.
1073 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1075 if (sr->when_target <= sr->region_start + min_t(int, t, sr->region_len - 1))
1078 if ((sr->pr_val & (1UL << qp)) == 0)
1080 sr->pr_mask |= (1UL << qp);
1086 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1088 struct unw_reg_info *r;
1090 if (!desc_is_active(qp, t, sr))
1093 r = sr->curr.reg + decode_abreg(abreg, 0);
1094 r->where = UNW_WHERE_NONE;
1095 r->when = UNW_WHEN_NEVER;
1100 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1101 unsigned char ytreg, struct unw_state_record *sr)
1103 enum unw_where where = UNW_WHERE_GR;
1104 struct unw_reg_info *r;
1106 if (!desc_is_active(qp, t, sr))
1110 where = UNW_WHERE_BR;
1111 else if (ytreg & 0x80)
1112 where = UNW_WHERE_FR;
1114 r = sr->curr.reg + decode_abreg(abreg, 0);
1116 r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1117 r->val = (ytreg & 0x7f);
1121 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1122 struct unw_state_record *sr)
1124 struct unw_reg_info *r;
1126 if (!desc_is_active(qp, t, sr))
1129 r = sr->curr.reg + decode_abreg(abreg, 1);
1130 r->where = UNW_WHERE_PSPREL;
1131 r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1132 r->val = 0x10 - 4*pspoff;
1136 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1137 struct unw_state_record *sr)
1139 struct unw_reg_info *r;
1141 if (!desc_is_active(qp, t, sr))
1144 r = sr->curr.reg + decode_abreg(abreg, 1);
1145 r->where = UNW_WHERE_SPREL;
1146 r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1150 #define UNW_DEC_BAD_CODE(code) printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1156 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg) desc_prologue(0,r,m,gr,arg)
1157 #define UNW_DEC_PROLOGUE(fmt,b,r,arg) desc_prologue(b,r,0,32,arg)
1159 * prologue descriptors:
1161 #define UNW_DEC_ABI(fmt,a,c,arg) desc_abi(a,c,arg)
1162 #define UNW_DEC_BR_GR(fmt,b,g,arg) desc_br_gr(b,g,arg)
1163 #define UNW_DEC_BR_MEM(fmt,b,arg) desc_br_mem(b,arg)
1164 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg) desc_frgr_mem(g,f,arg)
1165 #define UNW_DEC_FR_MEM(fmt,f,arg) desc_fr_mem(f,arg)
1166 #define UNW_DEC_GR_GR(fmt,m,g,arg) desc_gr_gr(m,g,arg)
1167 #define UNW_DEC_GR_MEM(fmt,m,arg) desc_gr_mem(m,arg)
1168 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg) desc_mem_stack_f(t,s,arg)
1169 #define UNW_DEC_MEM_STACK_V(fmt,t,arg) desc_mem_stack_v(t,arg)
1170 #define UNW_DEC_REG_GR(fmt,r,d,arg) desc_reg_gr(r,d,arg)
1171 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg) desc_reg_psprel(r,o,arg)
1172 #define UNW_DEC_REG_SPREL(fmt,r,o,arg) desc_reg_sprel(r,o,arg)
1173 #define UNW_DEC_REG_WHEN(fmt,r,t,arg) desc_reg_when(r,t,arg)
1174 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg) desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1175 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg) desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1176 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg) desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1177 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg) desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1178 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg) desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1179 #define UNW_DEC_RP_BR(fmt,d,arg) desc_rp_br(d,arg)
1180 #define UNW_DEC_SPILL_BASE(fmt,o,arg) desc_spill_base(o,arg)
1181 #define UNW_DEC_SPILL_MASK(fmt,m,arg) (m = desc_spill_mask(m,arg))
1185 #define UNW_DEC_EPILOGUE(fmt,t,c,arg) desc_epilogue(t,c,arg)
1186 #define UNW_DEC_COPY_STATE(fmt,l,arg) desc_copy_state(l,arg)
1187 #define UNW_DEC_LABEL_STATE(fmt,l,arg) desc_label_state(l,arg)
1189 * general unwind descriptors:
1191 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg) desc_spill_reg_p(p,t,a,x,y,arg)
1192 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg) desc_spill_reg_p(0,t,a,x,y,arg)
1193 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg) desc_spill_psprel_p(p,t,a,o,arg)
1194 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg) desc_spill_psprel_p(0,t,a,o,arg)
1195 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg) desc_spill_sprel_p(p,t,a,o,arg)
1196 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg) desc_spill_sprel_p(0,t,a,o,arg)
1197 #define UNW_DEC_RESTORE_P(f,p,t,a,arg) desc_restore_p(p,t,a,arg)
1198 #define UNW_DEC_RESTORE(f,t,a,arg) desc_restore_p(0,t,a,arg)
1200 #include "unwind_decoder.c"
1203 /* Unwind scripts. */
1205 static inline unw_hash_index_t
1206 hash (unsigned long ip)
1208 # define hashmagic 0x9e3779b97f4a7c16UL /* based on (sqrt(5)/2-1)*2^64 */
1210 return (ip >> 4)*hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1215 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1217 read_lock(&script->lock);
1218 if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1219 /* keep the read lock... */
1221 read_unlock(&script->lock);
1225 static inline struct unw_script *
1226 script_lookup (struct unw_frame_info *info)
1228 struct unw_script *script = unw.cache + info->hint;
1229 unsigned short index;
1230 unsigned long ip, pr;
1232 if (UNW_DEBUG_ON(0))
1233 return NULL; /* Always regenerate scripts in debug mode */
1235 STAT(++unw.stat.cache.lookups);
1240 if (cache_match(script, ip, pr)) {
1241 STAT(++unw.stat.cache.hinted_hits);
1245 index = unw.hash[hash(ip)];
1246 if (index >= UNW_CACHE_SIZE)
1249 script = unw.cache + index;
1251 if (cache_match(script, ip, pr)) {
1252 /* update hint; no locking required as single-word writes are atomic */
1253 STAT(++unw.stat.cache.normal_hits);
1254 unw.cache[info->prev_script].hint = script - unw.cache;
1257 if (script->coll_chain >= UNW_HASH_SIZE)
1259 script = unw.cache + script->coll_chain;
1260 STAT(++unw.stat.cache.collision_chain_traversals);
1265 * On returning, a write lock for the SCRIPT is still being held.
1267 static inline struct unw_script *
1268 script_new (unsigned long ip)
1270 struct unw_script *script, *prev, *tmp;
1271 unw_hash_index_t index;
1272 unsigned long flags;
1273 unsigned short head;
1275 STAT(++unw.stat.script.news);
1278 * Can't (easily) use cmpxchg() here because of ABA problem
1279 * that is intrinsic in cmpxchg()...
1281 spin_lock_irqsave(&unw.lock, flags);
1283 head = unw.lru_head;
1284 script = unw.cache + head;
1285 unw.lru_head = script->lru_chain;
1287 spin_unlock(&unw.lock);
1290 * We'd deadlock here if we interrupted a thread that is holding a read lock on
1291 * script->lock. Thus, if the write_trylock() fails, we simply bail out. The
1292 * alternative would be to disable interrupts whenever we hold a read-lock, but
1295 if (!write_trylock(&script->lock))
1298 spin_lock(&unw.lock);
1300 /* re-insert script at the tail of the LRU chain: */
1301 unw.cache[unw.lru_tail].lru_chain = head;
1302 unw.lru_tail = head;
1304 /* remove the old script from the hash table (if it's there): */
1306 index = hash(script->ip);
1307 tmp = unw.cache + unw.hash[index];
1310 if (tmp == script) {
1312 prev->coll_chain = tmp->coll_chain;
1314 unw.hash[index] = tmp->coll_chain;
1318 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1319 /* old script wasn't in the hash-table */
1321 tmp = unw.cache + tmp->coll_chain;
1325 /* enter new script in the hash table */
1327 script->coll_chain = unw.hash[index];
1328 unw.hash[index] = script - unw.cache;
1330 script->ip = ip; /* set new IP while we're holding the locks */
1332 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1334 spin_unlock_irqrestore(&unw.lock, flags);
1343 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1345 script->pr_mask = sr->pr_mask;
1346 script->pr_val = sr->pr_val;
1348 * We could down-grade our write-lock on script->lock here but
1349 * the rwlock API doesn't offer atomic lock downgrading, so
1350 * we'll just keep the write-lock and release it later when
1351 * we're done using the script.
1356 script_emit (struct unw_script *script, struct unw_insn insn)
1358 if (script->count >= UNW_MAX_SCRIPT_LEN) {
1359 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1360 __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1363 script->insn[script->count++] = insn;
1367 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1369 struct unw_reg_info *r = sr->curr.reg + i;
1370 enum unw_insn_opcode opc;
1371 struct unw_insn insn;
1372 unsigned long val = 0;
1377 /* register got spilled to a stacked register */
1378 opc = UNW_INSN_SETNAT_TYPE;
1379 val = UNW_NAT_REGSTK;
1381 /* register got spilled to a scratch register */
1382 opc = UNW_INSN_SETNAT_MEMSTK;
1386 opc = UNW_INSN_SETNAT_TYPE;
1391 opc = UNW_INSN_SETNAT_TYPE;
1395 case UNW_WHERE_PSPREL:
1396 case UNW_WHERE_SPREL:
1397 opc = UNW_INSN_SETNAT_MEMSTK;
1401 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1402 __FUNCTION__, r->where);
1406 insn.dst = unw.preg_index[i];
1408 script_emit(script, insn);
1412 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1414 struct unw_reg_info *r = sr->curr.reg + i;
1415 enum unw_insn_opcode opc;
1416 unsigned long val, rval;
1417 struct unw_insn insn;
1420 if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1423 opc = UNW_INSN_MOVE;
1424 val = rval = r->val;
1425 need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1430 opc = UNW_INSN_MOVE_STACKED;
1432 } else if (rval >= 4 && rval <= 7) {
1433 if (need_nat_info) {
1434 opc = UNW_INSN_MOVE2;
1437 val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1438 } else if (rval == 0) {
1439 opc = UNW_INSN_MOVE_CONST;
1442 /* register got spilled to a scratch register */
1443 opc = UNW_INSN_MOVE_SCRATCH;
1444 val = pt_regs_off(rval);
1450 val = unw.preg_index[UNW_REG_F2 + (rval - 2)];
1451 else if (rval >= 16 && rval <= 31)
1452 val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1454 opc = UNW_INSN_MOVE_SCRATCH;
1456 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1458 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1459 __FUNCTION__, rval);
1464 if (rval >= 1 && rval <= 5)
1465 val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1467 opc = UNW_INSN_MOVE_SCRATCH;
1469 val = offsetof(struct pt_regs, b0);
1471 val = offsetof(struct pt_regs, b6);
1473 val = offsetof(struct pt_regs, b7);
1477 case UNW_WHERE_SPREL:
1478 opc = UNW_INSN_ADD_SP;
1481 case UNW_WHERE_PSPREL:
1482 opc = UNW_INSN_ADD_PSP;
1486 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1487 __FUNCTION__, i, r->where);
1491 insn.dst = unw.preg_index[i];
1493 script_emit(script, insn);
1495 emit_nat_info(sr, i, script);
1497 if (i == UNW_REG_PSP) {
1499 * info->psp must contain the _value_ of the previous
1500 * sp, not it's save location. We get this by
1501 * dereferencing the value we just stored in
1504 insn.opc = UNW_INSN_LOAD;
1505 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1506 script_emit(script, insn);
1510 static inline const struct unw_table_entry *
1511 lookup (struct unw_table *table, unsigned long rel_ip)
1513 const struct unw_table_entry *e = NULL;
1514 unsigned long lo, hi, mid;
1516 /* do a binary search for right entry: */
1517 for (lo = 0, hi = table->length; lo < hi; ) {
1518 mid = (lo + hi) / 2;
1519 e = &table->array[mid];
1520 if (rel_ip < e->start_offset)
1522 else if (rel_ip >= e->end_offset)
1527 if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1533 * Build an unwind script that unwinds from state OLD_STATE to the
1534 * entrypoint of the function that called OLD_STATE.
1536 static inline struct unw_script *
1537 build_script (struct unw_frame_info *info)
1539 const struct unw_table_entry *e = NULL;
1540 struct unw_script *script = NULL;
1541 struct unw_labeled_state *ls, *next;
1542 unsigned long ip = info->ip;
1543 struct unw_state_record sr;
1544 struct unw_table *table;
1545 struct unw_reg_info *r;
1546 struct unw_insn insn;
1550 STAT(unsigned long start, parse_start;)
1552 STAT(++unw.stat.script.builds; start = ia64_get_itc());
1554 /* build state record */
1555 memset(&sr, 0, sizeof(sr));
1556 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1557 r->when = UNW_WHEN_NEVER;
1558 sr.pr_val = info->pr;
1560 UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1561 script = script_new(ip);
1563 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n", __FUNCTION__);
1564 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1567 unw.cache[info->prev_script].hint = script - unw.cache;
1569 /* search the kernels and the modules' unwind tables for IP: */
1571 STAT(parse_start = ia64_get_itc());
1573 for (table = unw.tables; table; table = table->next) {
1574 if (ip >= table->start && ip < table->end) {
1575 e = lookup(table, ip - table->segment_base);
1580 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs) */
1581 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1582 __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1583 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1584 sr.curr.reg[UNW_REG_RP].when = -1;
1585 sr.curr.reg[UNW_REG_RP].val = 0;
1586 compile_reg(&sr, UNW_REG_RP, script);
1587 script_finalize(script, &sr);
1588 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1589 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1593 sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1595 hdr = *(u64 *) (table->segment_base + e->info_offset);
1596 dp = (u8 *) (table->segment_base + e->info_offset + 8);
1597 desc_end = dp + 8*UNW_LENGTH(hdr);
1599 while (!sr.done && dp < desc_end)
1600 dp = unw_decode(dp, sr.in_body, &sr);
1602 if (sr.when_target > sr.epilogue_start) {
1604 * sp has been restored and all values on the memory stack below
1605 * psp also have been restored.
1607 sr.curr.reg[UNW_REG_PSP].val = 0;
1608 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1609 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1610 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1611 if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1612 || r->where == UNW_WHERE_SPREL)
1615 r->where = UNW_WHERE_NONE;
1616 r->when = UNW_WHEN_NEVER;
1620 script->flags = sr.flags;
1623 * If RP did't get saved, generate entry for the return link
1626 if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1627 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1628 sr.curr.reg[UNW_REG_RP].when = -1;
1629 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1630 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1631 __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1632 sr.curr.reg[UNW_REG_RP].val);
1636 UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1637 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1638 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1639 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1640 UNW_DPRINT(1, " %s <- ", unw.preg_name[r - sr.curr.reg]);
1642 case UNW_WHERE_GR: UNW_DPRINT(1, "r%lu", r->val); break;
1643 case UNW_WHERE_FR: UNW_DPRINT(1, "f%lu", r->val); break;
1644 case UNW_WHERE_BR: UNW_DPRINT(1, "b%lu", r->val); break;
1645 case UNW_WHERE_SPREL: UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1646 case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1647 case UNW_WHERE_NONE:
1648 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1652 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1655 UNW_DPRINT(1, "\t\t%d\n", r->when);
1660 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1662 /* translate state record into unwinder instructions: */
1665 * First, set psp if we're dealing with a fixed-size frame;
1666 * subsequent instructions may depend on this value.
1668 if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1669 && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1670 && sr.curr.reg[UNW_REG_PSP].val != 0) {
1671 /* new psp is sp plus frame size */
1672 insn.opc = UNW_INSN_ADD;
1673 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1674 insn.val = sr.curr.reg[UNW_REG_PSP].val; /* frame size */
1675 script_emit(script, insn);
1678 /* determine where the primary UNaT is: */
1679 if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1680 i = UNW_REG_PRI_UNAT_MEM;
1681 else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1682 i = UNW_REG_PRI_UNAT_GR;
1683 else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1684 i = UNW_REG_PRI_UNAT_MEM;
1686 i = UNW_REG_PRI_UNAT_GR;
1688 compile_reg(&sr, i, script);
1690 for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1691 compile_reg(&sr, i, script);
1693 /* free labeled register states & stack: */
1695 STAT(parse_start = ia64_get_itc());
1696 for (ls = sr.labeled_states; ls; ls = next) {
1698 free_state_stack(&ls->saved_state);
1699 free_labeled_state(ls);
1701 free_state_stack(&sr.curr);
1702 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1704 script_finalize(script, &sr);
1705 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1710 * Apply the unwinding actions represented by OPS and update SR to
1711 * reflect the state that existed upon entry to the function that this
1712 * unwinder represents.
1715 run_script (struct unw_script *script, struct unw_frame_info *state)
1717 struct unw_insn *ip, *limit, next_insn;
1718 unsigned long opc, dst, val, off;
1719 unsigned long *s = (unsigned long *) state;
1720 STAT(unsigned long start;)
1722 STAT(++unw.stat.script.runs; start = ia64_get_itc());
1723 state->flags = script->flags;
1725 limit = script->insn + script->count;
1728 while (ip++ < limit) {
1729 opc = next_insn.opc;
1730 dst = next_insn.dst;
1731 val = next_insn.val;
1740 case UNW_INSN_MOVE2:
1743 s[dst+1] = s[val+1];
1753 case UNW_INSN_MOVE_SCRATCH:
1755 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1758 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1759 __FUNCTION__, dst, val);
1763 case UNW_INSN_MOVE_CONST:
1765 s[dst] = (unsigned long) &unw.r0;
1768 UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1774 case UNW_INSN_MOVE_STACKED:
1775 s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1779 case UNW_INSN_ADD_PSP:
1780 s[dst] = state->psp + val;
1783 case UNW_INSN_ADD_SP:
1784 s[dst] = state->sp + val;
1787 case UNW_INSN_SETNAT_MEMSTK:
1788 if (!state->pri_unat_loc)
1789 state->pri_unat_loc = &state->sw->ar_unat;
1790 /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1791 s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1794 case UNW_INSN_SETNAT_TYPE:
1800 if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1801 || s[val] < TASK_SIZE)
1803 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1804 __FUNCTION__, s[val]);
1808 s[dst] = *(unsigned long *) s[val];
1812 STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1816 off = unw.sw_off[val];
1817 s[val] = (unsigned long) state->sw + off;
1818 if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1820 * We're initializing a general register: init NaT info, too. Note that
1821 * the offset is a multiple of 8 which gives us the 3 bits needed for
1824 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1829 find_save_locs (struct unw_frame_info *info)
1831 int have_write_lock = 0;
1832 struct unw_script *scr;
1834 if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1835 /* don't let obviously bad addresses pollute the cache */
1836 /* FIXME: should really be level 0 but it occurs too often. KAO */
1837 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1838 info->rp_loc = NULL;
1842 scr = script_lookup(info);
1844 scr = build_script(info);
1847 "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1848 __FUNCTION__, info->ip);
1851 have_write_lock = 1;
1853 info->hint = scr->hint;
1854 info->prev_script = scr - unw.cache;
1856 run_script(scr, info);
1858 if (have_write_lock)
1859 write_unlock(&scr->lock);
1861 read_unlock(&scr->lock);
1866 unw_unwind (struct unw_frame_info *info)
1868 unsigned long prev_ip, prev_sp, prev_bsp;
1869 unsigned long ip, pr, num_regs;
1870 STAT(unsigned long start, flags;)
1873 STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1877 prev_bsp = info->bsp;
1879 /* restore the ip */
1880 if (!info->rp_loc) {
1881 /* FIXME: should really be level 0 but it occurs too often. KAO */
1882 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1883 __FUNCTION__, info->ip);
1884 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1887 ip = info->ip = *info->rp_loc;
1888 if (ip < GATE_ADDR) {
1889 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1890 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1894 /* restore the cfm: */
1895 if (!info->pfs_loc) {
1896 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1897 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1900 info->cfm_loc = info->pfs_loc;
1902 /* restore the bsp: */
1905 if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1906 info->pt = info->sp + 16;
1907 if ((pr & (1UL << pNonSys)) != 0)
1908 num_regs = *info->cfm_loc & 0x7f; /* size of frame */
1910 (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1911 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1913 num_regs = (*info->cfm_loc >> 7) & 0x7f; /* size of locals */
1914 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1915 if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1916 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1917 __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1918 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1922 /* restore the sp: */
1923 info->sp = info->psp;
1924 if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1925 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1926 __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1927 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1931 if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1932 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1934 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1938 /* as we unwind, the saved ar.unat becomes the primary unat: */
1939 info->pri_unat_loc = info->unat_loc;
1941 /* finally, restore the predicates: */
1942 unw_get_pr(info, &info->pr);
1944 retval = find_save_locs(info);
1945 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1948 EXPORT_SYMBOL(unw_unwind);
1951 unw_unwind_to_user (struct unw_frame_info *info)
1955 while (unw_unwind(info) >= 0) {
1956 if (unw_get_rp(info, &ip) < 0) {
1957 unw_get_ip(info, &ip);
1958 UNW_DPRINT(0, "unwind.%s: failed to read return pointer (ip=0x%lx)\n",
1962 if (ip < FIXADDR_USER_END)
1965 unw_get_ip(info, &ip);
1966 UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n", __FUNCTION__, ip);
1969 EXPORT_SYMBOL(unw_unwind_to_user);
1972 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1973 struct switch_stack *sw, unsigned long stktop)
1975 unsigned long rbslimit, rbstop, stklimit;
1976 STAT(unsigned long start, flags;)
1978 STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1981 * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1982 * don't want to do that because it would be slow as each preserved register would
1983 * have to be processed. Instead, what we do here is zero out the frame info and
1984 * start the unwind process at the function that created the switch_stack frame.
1985 * When a preserved value in switch_stack needs to be accessed, run_script() will
1986 * initialize the appropriate pointer on demand.
1988 memset(info, 0, sizeof(*info));
1990 rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1991 rbstop = sw->ar_bspstore;
1992 if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1995 stklimit = (unsigned long) t + IA64_STK_OFFSET;
1996 if (stktop <= rbstop)
1999 info->regstk.limit = rbslimit;
2000 info->regstk.top = rbstop;
2001 info->memstk.limit = stklimit;
2002 info->memstk.top = stktop;
2005 info->sp = info->psp = stktop;
2007 UNW_DPRINT(3, "unwind.%s:\n"
2009 " rbs = [0x%lx-0x%lx)\n"
2010 " stk = [0x%lx-0x%lx)\n"
2014 __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2015 info->pr, (unsigned long) info->sw, info->sp);
2016 STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2020 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
2021 struct pt_regs *pt, struct switch_stack *sw)
2025 init_frame_info(info, t, sw, pt->r12);
2026 info->cfm_loc = &pt->cr_ifs;
2027 info->unat_loc = &pt->ar_unat;
2028 info->pfs_loc = &pt->ar_pfs;
2029 sof = *info->cfm_loc & 0x7f;
2030 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
2031 info->ip = pt->cr_iip + ia64_psr(pt)->ri;
2032 info->pt = (unsigned long) pt;
2033 UNW_DPRINT(3, "unwind.%s:\n"
2037 __FUNCTION__, info->bsp, sof, info->ip);
2038 find_save_locs(info);
2042 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2046 init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2047 info->cfm_loc = &sw->ar_pfs;
2048 sol = (*info->cfm_loc >> 7) & 0x7f;
2049 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2051 UNW_DPRINT(3, "unwind.%s:\n"
2055 __FUNCTION__, info->bsp, sol, info->ip);
2056 find_save_locs(info);
2060 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2062 struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2064 UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
2065 unw_init_frame_info(info, t, sw);
2067 EXPORT_SYMBOL(unw_init_from_blocked_task);
2070 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2071 unsigned long gp, const void *table_start, const void *table_end)
2073 const struct unw_table_entry *start = table_start, *end = table_end;
2076 table->segment_base = segment_base;
2078 table->start = segment_base + start[0].start_offset;
2079 table->end = segment_base + end[-1].end_offset;
2080 table->array = start;
2081 table->length = end - start;
2085 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2086 const void *table_start, const void *table_end)
2088 const struct unw_table_entry *start = table_start, *end = table_end;
2089 struct unw_table *table;
2090 unsigned long flags;
2092 if (end - start <= 0) {
2093 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2098 table = kmalloc(sizeof(*table), GFP_USER);
2102 init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2104 spin_lock_irqsave(&unw.lock, flags);
2106 /* keep kernel unwind table at the front (it's searched most commonly): */
2107 table->next = unw.tables->next;
2108 unw.tables->next = table;
2110 spin_unlock_irqrestore(&unw.lock, flags);
2116 unw_remove_unwind_table (void *handle)
2118 struct unw_table *table, *prev;
2119 struct unw_script *tmp;
2120 unsigned long flags;
2124 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2130 if (table == &unw.kernel_table) {
2131 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2132 "no-can-do!\n", __FUNCTION__);
2136 spin_lock_irqsave(&unw.lock, flags);
2138 /* first, delete the table: */
2140 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2141 if (prev->next == table)
2144 UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2145 __FUNCTION__, (void *) table);
2146 spin_unlock_irqrestore(&unw.lock, flags);
2149 prev->next = table->next;
2151 spin_unlock_irqrestore(&unw.lock, flags);
2153 /* next, remove hash table entries for this table */
2155 for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2156 tmp = unw.cache + unw.hash[index];
2157 if (unw.hash[index] >= UNW_CACHE_SIZE
2158 || tmp->ip < table->start || tmp->ip >= table->end)
2161 write_lock(&tmp->lock);
2163 if (tmp->ip >= table->start && tmp->ip < table->end) {
2164 unw.hash[index] = tmp->coll_chain;
2168 write_unlock(&tmp->lock);
2175 create_gate_table (void)
2177 const struct unw_table_entry *entry, *start, *end;
2178 unsigned long *lp, segbase = GATE_ADDR;
2179 size_t info_size, size;
2181 Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2184 for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2185 if (phdr->p_type == PT_IA_64_UNWIND) {
2191 printk("%s: failed to find gate DSO's unwind table!\n", __FUNCTION__);
2195 start = (const struct unw_table_entry *) punw->p_vaddr;
2196 end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2199 unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2201 for (entry = start; entry < end; ++entry)
2202 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2203 size += 8; /* reserve space for "end of table" marker */
2205 unw.gate_table = kmalloc(size, GFP_KERNEL);
2206 if (!unw.gate_table) {
2207 unw.gate_table_size = 0;
2208 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __FUNCTION__);
2211 unw.gate_table_size = size;
2213 lp = unw.gate_table;
2214 info = (char *) unw.gate_table + size;
2216 for (entry = start; entry < end; ++entry, lp += 3) {
2217 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2219 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2221 lp[0] = segbase + entry->start_offset; /* start */
2222 lp[1] = segbase + entry->end_offset; /* end */
2223 lp[2] = info - (char *) unw.gate_table; /* info */
2225 *lp = 0; /* end-of-table marker */
2229 __initcall(create_gate_table);
2235 extern void unw_hash_index_t_is_too_narrow (void);
2238 if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2239 unw_hash_index_t_is_too_narrow();
2241 unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2242 unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2243 unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2244 unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2245 unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2246 unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2247 unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2248 unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2249 for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2250 unw.sw_off[unw.preg_index[i]] = off;
2251 for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2252 unw.sw_off[unw.preg_index[i]] = off;
2253 for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2254 unw.sw_off[unw.preg_index[i]] = off;
2255 for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2256 unw.sw_off[unw.preg_index[i]] = off;
2258 for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2260 unw.cache[i].lru_chain = (i - 1);
2261 unw.cache[i].coll_chain = -1;
2262 unw.cache[i].lock = RW_LOCK_UNLOCKED;
2264 unw.lru_head = UNW_CACHE_SIZE - 1;
2267 init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2268 __start_unwind, __end_unwind);
2272 * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2274 * This system call has been deprecated. The new and improved way to get
2275 * at the kernel's unwind info is via the gate DSO. The address of the
2276 * ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2278 * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2280 * This system call copies the unwind data into the buffer pointed to by BUF and returns
2281 * the size of the unwind data. If BUF_SIZE is smaller than the size of the unwind data
2282 * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2285 * The first portion of the unwind data contains an unwind table and rest contains the
2286 * associated unwind info (in no particular order). The unwind table consists of a table
2287 * of entries of the form:
2289 * u64 start; (64-bit address of start of function)
2290 * u64 end; (64-bit address of start of function)
2291 * u64 info; (BUF-relative offset to unwind info)
2293 * The end of the unwind table is indicated by an entry with a START address of zero.
2295 * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2296 * on the format of the unwind info.
2299 * EFAULT BUF points outside your accessible address space.
2302 sys_getunwind (void __user *buf, size_t buf_size)
2304 if (buf && buf_size >= unw.gate_table_size)
2305 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2307 return unw.gate_table_size;