arch/ia64/kernel/unwind.c

   1 /*
   2  * Copyright (C) 1999-2004 Hewlett-Packard Co
   3  *      David Mosberger-Tang <davidm@hpl.hp.com>
   4  * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
   5  *      - Change pt_regs_off() to make it less dependant on pt_regs structure.
   6  */
   7 /*
   8  * This file implements call frame unwind support for the Linux
   9  * kernel.  Parsing and processing the unwind information is
  10  * time-consuming, so this implementation translates the unwind
  11  * descriptors into unwind scripts.  These scripts are very simple
  12  * (basically a sequence of assignments) and efficient to execute.
  13  * They are cached for later re-use.  Each script is specific for a
  14  * given instruction pointer address and the set of predicate values
  15  * that the script depends on (most unwind descriptors are
  16  * unconditional and scripts often do not depend on predicates at
  17  * all).  This code is based on the unwind conventions described in
  18  * the "IA-64 Software Conventions and Runtime Architecture" manual.
  19  *
  20  * SMP conventions:
  21  *      o updates to the global unwind data (in structure "unw") are serialized
  22  *        by the unw.lock spinlock
  23  *      o each unwind script has its own read-write lock; a thread must acquire
  24  *        a read lock before executing a script and must acquire a write lock
  25  *        before modifying a script
  26  *      o if both the unw.lock spinlock and a script's read-write lock must be
  27  *        acquired, then the read-write lock must be acquired first.
  28  */
  29 #include <linux/module.h>
  30 #include <linux/bootmem.h>
  31 #include <linux/elf.h>
  32 #include <linux/kernel.h>
  33 #include <linux/sched.h>
  34 #include <linux/slab.h>
  35
  36 #include <asm/unwind.h>
  37
  38 #include <asm/delay.h>
  39 #include <asm/page.h>
  40 #include <asm/ptrace.h>
  41 #include <asm/ptrace_offsets.h>
  42 #include <asm/rse.h>
  43 #include <asm/sections.h>
  44 #include <asm/system.h>
  45 #include <asm/uaccess.h>
  46
  47 #include "entry.h"
  48 #include "unwind_i.h"
  49
  50 #define p5              5
  51
  52 #define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  53 #define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  54
  55 #define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  56 #define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  57
  58 #define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  59
  60 #ifdef UNW_DEBUG
  61   static unsigned int unw_debug_level = UNW_DEBUG;
  62 #  define UNW_DEBUG_ON(n)       unw_debug_level >= n
  63    /* Do not code a printk level, not all debug lines end in newline */
  64 #  define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  65 #  define inline
  66 #else /* !UNW_DEBUG */
  67 #  define UNW_DEBUG_ON(n)  0
  68 #  define UNW_DPRINT(n, ...)
  69 #endif /* UNW_DEBUG */
  70
  71 #if UNW_STATS
  72 # define STAT(x...)     x
  73 #else
  74 # define STAT(x...)
  75 #endif
  76
  77 #define alloc_reg_state()       kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
  78 #define free_reg_state(usr)     kfree(usr)
  79 #define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  80 #define free_labeled_state(usr) kfree(usr)
  81
  82 typedef unsigned long unw_word;
  83 typedef unsigned char unw_hash_index_t;
  84
  85 static struct {
  86         spinlock_t lock;                        /* spinlock for unwind data */
  87
  88         /* list of unwind tables (one per load-module) */
  89         struct unw_table *tables;
  90
  91         unsigned long r0;                       /* constant 0 for r0 */
  92
  93         /* table of registers that prologues can save (and order in which they're saved): */
  94         const unsigned char save_order[8];
  95
  96         /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
  97         unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
  98
  99         unsigned short lru_head;                /* index of lead-recently used script */
 100         unsigned short lru_tail;                /* index of most-recently used script */
 101
 102         /* index into unw_frame_info for preserved register i */
 103         unsigned short preg_index[UNW_NUM_REGS];
 104
 105         short pt_regs_offsets[32];
 106
 107         /* unwind table for the kernel: */
 108         struct unw_table kernel_table;
 109
 110         /* unwind table describing the gate page (kernel code that is mapped into user space): */
 111         size_t gate_table_size;
 112         unsigned long *gate_table;
 113
 114         /* hash table that maps instruction pointer to script index: */
 115         unsigned short hash[UNW_HASH_SIZE];
 116
 117         /* script cache: */
 118         struct unw_script cache[UNW_CACHE_SIZE];
 119
 120 # ifdef UNW_DEBUG
 121         const char *preg_name[UNW_NUM_REGS];
 122 # endif
 123 # if UNW_STATS
 124         struct {
 125                 struct {
 126                         int lookups;
 127                         int hinted_hits;
 128                         int normal_hits;
 129                         int collision_chain_traversals;
 130                 } cache;
 131                 struct {
 132                         unsigned long build_time;
 133                         unsigned long run_time;
 134                         unsigned long parse_time;
 135                         int builds;
 136                         int news;
 137                         int collisions;
 138                         int runs;
 139                 } script;
 140                 struct {
 141                         unsigned long init_time;
 142                         unsigned long unwind_time;
 143                         int inits;
 144                         int unwinds;
 145                 } api;
 146         } stat;
 147 # endif
 148 } unw = {
 149         .tables = &unw.kernel_table,
 150         .lock = SPIN_LOCK_UNLOCKED,
 151         .save_order = {
 152                 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 153                 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 154         },
 155         .preg_index = {
 156                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_GR */
 157                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_MEM */
 158                 offsetof(struct unw_frame_info, bsp_loc)/8,
 159                 offsetof(struct unw_frame_info, bspstore_loc)/8,
 160                 offsetof(struct unw_frame_info, pfs_loc)/8,
 161                 offsetof(struct unw_frame_info, rnat_loc)/8,
 162                 offsetof(struct unw_frame_info, psp)/8,
 163                 offsetof(struct unw_frame_info, rp_loc)/8,
 164                 offsetof(struct unw_frame_info, r4)/8,
 165                 offsetof(struct unw_frame_info, r5)/8,
 166                 offsetof(struct unw_frame_info, r6)/8,
 167                 offsetof(struct unw_frame_info, r7)/8,
 168                 offsetof(struct unw_frame_info, unat_loc)/8,
 169                 offsetof(struct unw_frame_info, pr_loc)/8,
 170                 offsetof(struct unw_frame_info, lc_loc)/8,
 171                 offsetof(struct unw_frame_info, fpsr_loc)/8,
 172                 offsetof(struct unw_frame_info, b1_loc)/8,
 173                 offsetof(struct unw_frame_info, b2_loc)/8,
 174                 offsetof(struct unw_frame_info, b3_loc)/8,
 175                 offsetof(struct unw_frame_info, b4_loc)/8,
 176                 offsetof(struct unw_frame_info, b5_loc)/8,
 177                 offsetof(struct unw_frame_info, f2_loc)/8,
 178                 offsetof(struct unw_frame_info, f3_loc)/8,
 179                 offsetof(struct unw_frame_info, f4_loc)/8,
 180                 offsetof(struct unw_frame_info, f5_loc)/8,
 181                 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
 182                 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
 183                 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
 184                 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
 185                 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
 186                 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
 187                 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
 188                 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
 189                 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
 190                 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
 191                 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
 192                 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
 193                 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
 194                 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
 195                 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
 196                 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
 197         },
 198         .pt_regs_offsets = {
 199                 [0] = -1,
 200                 offsetof(struct pt_regs,  r1),
 201                 offsetof(struct pt_regs,  r2),
 202                 offsetof(struct pt_regs,  r3),
 203                 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
 204                 offsetof(struct pt_regs,  r8),
 205                 offsetof(struct pt_regs,  r9),
 206                 offsetof(struct pt_regs, r10),
 207                 offsetof(struct pt_regs, r11),
 208                 offsetof(struct pt_regs, r12),
 209                 offsetof(struct pt_regs, r13),
 210                 offsetof(struct pt_regs, r14),
 211                 offsetof(struct pt_regs, r15),
 212                 offsetof(struct pt_regs, r16),
 213                 offsetof(struct pt_regs, r17),
 214                 offsetof(struct pt_regs, r18),
 215                 offsetof(struct pt_regs, r19),
 216                 offsetof(struct pt_regs, r20),
 217                 offsetof(struct pt_regs, r21),
 218                 offsetof(struct pt_regs, r22),
 219                 offsetof(struct pt_regs, r23),
 220                 offsetof(struct pt_regs, r24),
 221                 offsetof(struct pt_regs, r25),
 222                 offsetof(struct pt_regs, r26),
 223                 offsetof(struct pt_regs, r27),
 224                 offsetof(struct pt_regs, r28),
 225                 offsetof(struct pt_regs, r29),
 226                 offsetof(struct pt_regs, r30),
 227                 offsetof(struct pt_regs, r31),
 228         },
 229         .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 230 #ifdef UNW_DEBUG
 231         .preg_name = {
 232                 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 233                 "r4", "r5", "r6", "r7",
 234                 "ar.unat", "pr", "ar.lc", "ar.fpsr",
 235                 "b1", "b2", "b3", "b4", "b5",
 236                 "f2", "f3", "f4", "f5",
 237                 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 238                 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 239         }
 240 #endif
 241 };
 242
 243 static inline int
 244 read_only (void *addr)
 245 {
 246         return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
 247 }
 248
 249 /*
 250  * Returns offset of rREG in struct pt_regs.
 251  */
 252 static inline unsigned long
 253 pt_regs_off (unsigned long reg)
 254 {
 255         short off = -1;
 256
 257         if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
 258                 off = unw.pt_regs_offsets[reg];
 259
 260         if (off < 0) {
 261                 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
 262                 off = 0;
 263         }
 264         return (unsigned long) off;
 265 }
 266
 267 static inline struct pt_regs *
 268 get_scratch_regs (struct unw_frame_info *info)
 269 {
 270         if (!info->pt) {
 271                 /* This should not happen with valid unwind info.  */
 272                 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
 273                 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 274                         info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 275                 else
 276                         info->pt = info->sp - 16;
 277         }
 278         UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
 279         return (struct pt_regs *) info->pt;
 280 }
 281
 282 /* Unwind accessors.  */
 283
 284 int
 285 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 286 {
 287         unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 288         struct unw_ireg *ireg;
 289         struct pt_regs *pt;
 290
 291         if ((unsigned) regnum - 1 >= 127) {
 292                 if (regnum == 0 && !write) {
 293                         *val = 0;       /* read r0 always returns 0 */
 294                         *nat = 0;
 295                         return 0;
 296                 }
 297                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 298                            __FUNCTION__, regnum);
 299                 return -1;
 300         }
 301
 302         if (regnum < 32) {
 303                 if (regnum >= 4 && regnum <= 7) {
 304                         /* access a preserved register */
 305                         ireg = &info->r4 + (regnum - 4);
 306                         addr = ireg->loc;
 307                         if (addr) {
 308                                 nat_addr = addr + ireg->nat.off;
 309                                 switch (ireg->nat.type) {
 310                                       case UNW_NAT_VAL:
 311                                         /* simulate getf.sig/setf.sig */
 312                                         if (write) {
 313                                                 if (*nat) {
 314                                                         /* write NaTVal and be done with it */
 315                                                         addr[0] = 0;
 316                                                         addr[1] = 0x1fffe;
 317                                                         return 0;
 318                                                 }
 319                                                 addr[1] = 0x1003e;
 320                                         } else {
 321                                                 if (addr[0] == 0 && addr[1] == 0x1ffe) {
 322                                                         /* return NaT and be done with it */
 323                                                         *val = 0;
 324                                                         *nat = 1;
 325                                                         return 0;
 326                                                 }
 327                                         }
 328                                         /* fall through */
 329                                       case UNW_NAT_NONE:
 330                                         dummy_nat = 0;
 331                                         nat_addr = &dummy_nat;
 332                                         break;
 333
 334                                       case UNW_NAT_MEMSTK:
 335                                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 336                                         break;
 337
 338                                       case UNW_NAT_REGSTK:
 339                                         nat_addr = ia64_rse_rnat_addr(addr);
 340                                         if ((unsigned long) addr < info->regstk.limit
 341                                             || (unsigned long) addr >= info->regstk.top)
 342                                         {
 343                                                 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 344                                                         "[0x%lx-0x%lx)\n",
 345                                                         __FUNCTION__, (void *) addr,
 346                                                         info->regstk.limit,
 347                                                         info->regstk.top);
 348                                                 return -1;
 349                                         }
 350                                         if ((unsigned long) nat_addr >= info->regstk.top)
 351                                                 nat_addr = &info->sw->ar_rnat;
 352                                         nat_mask = (1UL << ia64_rse_slot_num(addr));
 353                                         break;
 354                                 }
 355                         } else {
 356                                 addr = &info->sw->r4 + (regnum - 4);
 357                                 nat_addr = &info->sw->ar_unat;
 358                                 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 359                         }
 360                 } else {
 361                         /* access a scratch register */
 362                         pt = get_scratch_regs(info);
 363                         addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 364                         if (info->pri_unat_loc)
 365                                 nat_addr = info->pri_unat_loc;
 366                         else
 367                                 nat_addr = &info->sw->ar_unat;
 368                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 369                 }
 370         } else {
 371                 /* access a stacked register */
 372                 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 373                 nat_addr = ia64_rse_rnat_addr(addr);
 374                 if ((unsigned long) addr < info->regstk.limit
 375                     || (unsigned long) addr >= info->regstk.top)
 376                 {
 377                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 378                                    "of rbs\n",  __FUNCTION__);
 379                         return -1;
 380                 }
 381                 if ((unsigned long) nat_addr >= info->regstk.top)
 382                         nat_addr = &info->sw->ar_rnat;
 383                 nat_mask = (1UL << ia64_rse_slot_num(addr));
 384         }
 385
 386         if (write) {
 387                 if (read_only(addr)) {
 388                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 389                                 __FUNCTION__);
 390                 } else {
 391                         *addr = *val;
 392                         if (*nat)
 393                                 *nat_addr |= nat_mask;
 394                         else
 395                                 *nat_addr &= ~nat_mask;
 396                 }
 397         } else {
 398                 if ((*nat_addr & nat_mask) == 0) {
 399                         *val = *addr;
 400                         *nat = 0;
 401                 } else {
 402                         *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 403                         *nat = 1;
 404                 }
 405         }
 406         return 0;
 407 }
 408 EXPORT_SYMBOL(unw_access_gr);
 409
 410 int
 411 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 412 {
 413         unsigned long *addr;
 414         struct pt_regs *pt;
 415
 416         switch (regnum) {
 417                 /* scratch: */
 418               case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
 419               case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
 420               case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
 421
 422                 /* preserved: */
 423               case 1: case 2: case 3: case 4: case 5:
 424                 addr = *(&info->b1_loc + (regnum - 1));
 425                 if (!addr)
 426                         addr = &info->sw->b1 + (regnum - 1);
 427                 break;
 428
 429               default:
 430                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 431                            __FUNCTION__, regnum);
 432                 return -1;
 433         }
 434         if (write)
 435                 if (read_only(addr)) {
 436                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 437                                 __FUNCTION__);
 438                 } else
 439                         *addr = *val;
 440         else
 441                 *val = *addr;
 442         return 0;
 443 }
 444 EXPORT_SYMBOL(unw_access_br);
 445
 446 int
 447 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 448 {
 449         struct ia64_fpreg *addr = NULL;
 450         struct pt_regs *pt;
 451
 452         if ((unsigned) (regnum - 2) >= 126) {
 453                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 454                            __FUNCTION__, regnum);
 455                 return -1;
 456         }
 457
 458         if (regnum <= 5) {
 459                 addr = *(&info->f2_loc + (regnum - 2));
 460                 if (!addr)
 461                         addr = &info->sw->f2 + (regnum - 2);
 462         } else if (regnum <= 15) {
 463                 if (regnum <= 11) {
 464                         pt = get_scratch_regs(info);
 465                         addr = &pt->f6  + (regnum - 6);
 466                 }
 467                 else
 468                         addr = &info->sw->f12 + (regnum - 12);
 469         } else if (regnum <= 31) {
 470                 addr = info->fr_loc[regnum - 16];
 471                 if (!addr)
 472                         addr = &info->sw->f16 + (regnum - 16);
 473         } else {
 474                 struct task_struct *t = info->task;
 475
 476                 if (write)
 477                         ia64_sync_fph(t);
 478                 else
 479                         ia64_flush_fph(t);
 480                 addr = t->thread.fph + (regnum - 32);
 481         }
 482
 483         if (write)
 484                 if (read_only(addr)) {
 485                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 486                                 __FUNCTION__);
 487                 } else
 488                         *addr = *val;
 489         else
 490                 *val = *addr;
 491         return 0;
 492 }
 493 EXPORT_SYMBOL(unw_access_fr);
 494
 495 int
 496 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 497 {
 498         unsigned long *addr;
 499         struct pt_regs *pt;
 500
 501         switch (regnum) {
 502               case UNW_AR_BSP:
 503                 addr = info->bsp_loc;
 504                 if (!addr)
 505                         addr = &info->sw->ar_bspstore;
 506                 break;
 507
 508               case UNW_AR_BSPSTORE:
 509                 addr = info->bspstore_loc;
 510                 if (!addr)
 511                         addr = &info->sw->ar_bspstore;
 512                 break;
 513
 514               case UNW_AR_PFS:
 515                 addr = info->pfs_loc;
 516                 if (!addr)
 517                         addr = &info->sw->ar_pfs;
 518                 break;
 519
 520               case UNW_AR_RNAT:
 521                 addr = info->rnat_loc;
 522                 if (!addr)
 523                         addr = &info->sw->ar_rnat;
 524                 break;
 525
 526               case UNW_AR_UNAT:
 527                 addr = info->unat_loc;
 528                 if (!addr)
 529                         addr = &info->sw->ar_unat;
 530                 break;
 531
 532               case UNW_AR_LC:
 533                 addr = info->lc_loc;
 534                 if (!addr)
 535                         addr = &info->sw->ar_lc;
 536                 break;
 537
 538               case UNW_AR_EC:
 539                 if (!info->cfm_loc)
 540                         return -1;
 541                 if (write)
 542                         *info->cfm_loc =
 543                                 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 544                 else
 545                         *val = (*info->cfm_loc >> 52) & 0x3f;
 546                 return 0;
 547
 548               case UNW_AR_FPSR:
 549                 addr = info->fpsr_loc;
 550                 if (!addr)
 551                         addr = &info->sw->ar_fpsr;
 552                 break;
 553
 554               case UNW_AR_RSC:
 555                 pt = get_scratch_regs(info);
 556                 addr = &pt->ar_rsc;
 557                 break;
 558
 559               case UNW_AR_CCV:
 560                 pt = get_scratch_regs(info);
 561                 addr = &pt->ar_ccv;
 562                 break;
 563
 564               case UNW_AR_CSD:
 565                 pt = get_scratch_regs(info);
 566                 addr = &pt->ar_csd;
 567                 break;
 568
 569               case UNW_AR_SSD:
 570                 pt = get_scratch_regs(info);
 571                 addr = &pt->ar_ssd;
 572                 break;
 573
 574               default:
 575                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 576                            __FUNCTION__, regnum);
 577                 return -1;
 578         }
 579
 580         if (write) {
 581                 if (read_only(addr)) {
 582                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 583                                 __FUNCTION__);
 584                 } else
 585                         *addr = *val;
 586         } else
 587                 *val = *addr;
 588         return 0;
 589 }
 590 EXPORT_SYMBOL(unw_access_ar);
 591
 592 int
 593 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 594 {
 595         unsigned long *addr;
 596
 597         addr = info->pr_loc;
 598         if (!addr)
 599                 addr = &info->sw->pr;
 600
 601         if (write) {
 602                 if (read_only(addr)) {
 603                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 604                                 __FUNCTION__);
 605                 } else
 606                         *addr = *val;
 607         } else
 608                 *val = *addr;
 609         return 0;
 610 }
 611 EXPORT_SYMBOL(unw_access_pr);
 612
 613 \f
 614 /* Routines to manipulate the state stack.  */
 615
 616 static inline void
 617 push (struct unw_state_record *sr)
 618 {
 619         struct unw_reg_state *rs;
 620
 621         rs = alloc_reg_state();
 622         if (!rs) {
 623                 printk(KERN_ERR "unwind: cannot stack reg state!\n");
 624                 return;
 625         }
 626         memcpy(rs, &sr->curr, sizeof(*rs));
 627         sr->curr.next = rs;
 628 }
 629
 630 static void
 631 pop (struct unw_state_record *sr)
 632 {
 633         struct unw_reg_state *rs = sr->curr.next;
 634
 635         if (!rs) {
 636                 printk(KERN_ERR "unwind: stack underflow!\n");
 637                 return;
 638         }
 639         memcpy(&sr->curr, rs, sizeof(*rs));
 640         free_reg_state(rs);
 641 }
 642
 643 /* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 644 static struct unw_reg_state *
 645 dup_state_stack (struct unw_reg_state *rs)
 646 {
 647         struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 648
 649         while (rs) {
 650                 copy = alloc_reg_state();
 651                 if (!copy) {
 652                         printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 653                         return NULL;
 654                 }
 655                 memcpy(copy, rs, sizeof(*copy));
 656                 if (first)
 657                         prev->next = copy;
 658                 else
 659                         first = copy;
 660                 rs = rs->next;
 661                 prev = copy;
 662         }
 663         return first;
 664 }
 665
 666 /* Free all stacked register states (but not RS itself).  */
 667 static void
 668 free_state_stack (struct unw_reg_state *rs)
 669 {
 670         struct unw_reg_state *p, *next;
 671
 672         for (p = rs->next; p != NULL; p = next) {
 673                 next = p->next;
 674                 free_reg_state(p);
 675         }
 676         rs->next = NULL;
 677 }
 678 \f
 679 /* Unwind decoder routines */
 680
 681 static enum unw_register_index __attribute_const__
 682 decode_abreg (unsigned char abreg, int memory)
 683 {
 684         switch (abreg) {
 685               case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 686               case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 687               case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 688               case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 689               case 0x60: return UNW_REG_PR;
 690               case 0x61: return UNW_REG_PSP;
 691               case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 692               case 0x63: return UNW_REG_RP;
 693               case 0x64: return UNW_REG_BSP;
 694               case 0x65: return UNW_REG_BSPSTORE;
 695               case 0x66: return UNW_REG_RNAT;
 696               case 0x67: return UNW_REG_UNAT;
 697               case 0x68: return UNW_REG_FPSR;
 698               case 0x69: return UNW_REG_PFS;
 699               case 0x6a: return UNW_REG_LC;
 700               default:
 701                 break;
 702         }
 703         UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
 704         return UNW_REG_LC;
 705 }
 706
 707 static void
 708 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 709 {
 710         reg->val = val;
 711         reg->where = where;
 712         if (reg->when == UNW_WHEN_NEVER)
 713                 reg->when = when;
 714 }
 715
 716 static void
 717 alloc_spill_area (unsigned long *offp, unsigned long regsize,
 718                   struct unw_reg_info *lo, struct unw_reg_info *hi)
 719 {
 720         struct unw_reg_info *reg;
 721
 722         for (reg = hi; reg >= lo; --reg) {
 723                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 724                         reg->where = UNW_WHERE_PSPREL;
 725                         *offp -= regsize;
 726                         reg->val = *offp;
 727                 }
 728         }
 729 }
 730
 731 static inline void
 732 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 733 {
 734         struct unw_reg_info *reg;
 735
 736         for (reg = *regp; reg <= lim; ++reg) {
 737                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 738                         reg->when = t;
 739                         *regp = reg + 1;
 740                         return;
 741                 }
 742         }
 743         UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __FUNCTION__);
 744 }
 745
 746 static inline void
 747 finish_prologue (struct unw_state_record *sr)
 748 {
 749         struct unw_reg_info *reg;
 750         unsigned long off;
 751         int i;
 752
 753         /*
 754          * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 755          * for Using Unwind Descriptors", rule 3):
 756          */
 757         for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 758                 reg = sr->curr.reg + unw.save_order[i];
 759                 if (reg->where == UNW_WHERE_GR_SAVE) {
 760                         reg->where = UNW_WHERE_GR;
 761                         reg->val = sr->gr_save_loc++;
 762                 }
 763         }
 764
 765         /*
 766          * Next, compute when the fp, general, and branch registers get
 767          * saved.  This must come before alloc_spill_area() because
 768          * we need to know which registers are spilled to their home
 769          * locations.
 770          */
 771         if (sr->imask) {
 772                 unsigned char kind, mask = 0, *cp = sr->imask;
 773                 int t;
 774                 static const unsigned char limit[3] = {
 775                         UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 776                 };
 777                 struct unw_reg_info *(regs[3]);
 778
 779                 regs[0] = sr->curr.reg + UNW_REG_F2;
 780                 regs[1] = sr->curr.reg + UNW_REG_R4;
 781                 regs[2] = sr->curr.reg + UNW_REG_B1;
 782
 783                 for (t = 0; t < sr->region_len; ++t) {
 784                         if ((t & 3) == 0)
 785                                 mask = *cp++;
 786                         kind = (mask >> 2*(3-(t & 3))) & 3;
 787                         if (kind > 0)
 788                                 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 789                                                 sr->region_start + t);
 790                 }
 791         }
 792         /*
 793          * Next, lay out the memory stack spill area:
 794          */
 795         if (sr->any_spills) {
 796                 off = sr->spill_offset;
 797                 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 798                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 799                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 800         }
 801 }
 802
 803 /*
 804  * Region header descriptors.
 805  */
 806
 807 static void
 808 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 809                struct unw_state_record *sr)
 810 {
 811         int i, region_start;
 812
 813         if (!(sr->in_body || sr->first_region))
 814                 finish_prologue(sr);
 815         sr->first_region = 0;
 816
 817         /* check if we're done: */
 818         if (sr->when_target < sr->region_start + sr->region_len) {
 819                 sr->done = 1;
 820                 return;
 821         }
 822
 823         region_start = sr->region_start + sr->region_len;
 824
 825         for (i = 0; i < sr->epilogue_count; ++i)
 826                 pop(sr);
 827         sr->epilogue_count = 0;
 828         sr->epilogue_start = UNW_WHEN_NEVER;
 829
 830         sr->region_start = region_start;
 831         sr->region_len = rlen;
 832         sr->in_body = body;
 833
 834         if (!body) {
 835                 push(sr);
 836
 837                 for (i = 0; i < 4; ++i) {
 838                         if (mask & 0x8)
 839                                 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 840                                         sr->region_start + sr->region_len - 1, grsave++);
 841                         mask <<= 1;
 842                 }
 843                 sr->gr_save_loc = grsave;
 844                 sr->any_spills = 0;
 845                 sr->imask = NULL;
 846                 sr->spill_offset = 0x10;        /* default to psp+16 */
 847         }
 848 }
 849
 850 /*
 851  * Prologue descriptors.
 852  */
 853
 854 static inline void
 855 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 856 {
 857         if (abi == 3 && context == 'i') {
 858                 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 859                 UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __FUNCTION__);
 860         }
 861         else
 862                 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 863                                 __FUNCTION__, abi, context);
 864 }
 865
 866 static inline void
 867 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 868 {
 869         int i;
 870
 871         for (i = 0; i < 5; ++i) {
 872                 if (brmask & 1)
 873                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 874                                 sr->region_start + sr->region_len - 1, gr++);
 875                 brmask >>= 1;
 876         }
 877 }
 878
 879 static inline void
 880 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 881 {
 882         int i;
 883
 884         for (i = 0; i < 5; ++i) {
 885                 if (brmask & 1) {
 886                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 887                                 sr->region_start + sr->region_len - 1, 0);
 888                         sr->any_spills = 1;
 889                 }
 890                 brmask >>= 1;
 891         }
 892 }
 893
 894 static inline void
 895 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 896 {
 897         int i;
 898
 899         for (i = 0; i < 4; ++i) {
 900                 if ((grmask & 1) != 0) {
 901                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 902                                 sr->region_start + sr->region_len - 1, 0);
 903                         sr->any_spills = 1;
 904                 }
 905                 grmask >>= 1;
 906         }
 907         for (i = 0; i < 20; ++i) {
 908                 if ((frmask & 1) != 0) {
 909                         int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 910                         set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 911                                 sr->region_start + sr->region_len - 1, 0);
 912                         sr->any_spills = 1;
 913                 }
 914                 frmask >>= 1;
 915         }
 916 }
 917
 918 static inline void
 919 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 920 {
 921         int i;
 922
 923         for (i = 0; i < 4; ++i) {
 924                 if ((frmask & 1) != 0) {
 925                         set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 926                                 sr->region_start + sr->region_len - 1, 0);
 927                         sr->any_spills = 1;
 928                 }
 929                 frmask >>= 1;
 930         }
 931 }
 932
 933 static inline void
 934 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 935 {
 936         int i;
 937
 938         for (i = 0; i < 4; ++i) {
 939                 if ((grmask & 1) != 0)
 940                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 941                                 sr->region_start + sr->region_len - 1, gr++);
 942                 grmask >>= 1;
 943         }
 944 }
 945
 946 static inline void
 947 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 948 {
 949         int i;
 950
 951         for (i = 0; i < 4; ++i) {
 952                 if ((grmask & 1) != 0) {
 953                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 954                                 sr->region_start + sr->region_len - 1, 0);
 955                         sr->any_spills = 1;
 956                 }
 957                 grmask >>= 1;
 958         }
 959 }
 960
 961 static inline void
 962 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 963 {
 964         set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 965                 sr->region_start + min_t(int, t, sr->region_len - 1), 16*size);
 966 }
 967
 968 static inline void
 969 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 970 {
 971         sr->curr.reg[UNW_REG_PSP].when = sr->region_start + min_t(int, t, sr->region_len - 1);
 972 }
 973
 974 static inline void
 975 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 976 {
 977         set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 978 }
 979
 980 static inline void
 981 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 982 {
 983         set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 984                 0x10 - 4*pspoff);
 985 }
 986
 987 static inline void
 988 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 989 {
 990         set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 991                 4*spoff);
 992 }
 993
 994 static inline void
 995 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 996 {
 997         sr->return_link_reg = dst;
 998 }
 999
1000 static inline void
1001 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
1002 {
1003         struct unw_reg_info *reg = sr->curr.reg + regnum;
1004
1005         if (reg->where == UNW_WHERE_NONE)
1006                 reg->where = UNW_WHERE_GR_SAVE;
1007         reg->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1008 }
1009
1010 static inline void
1011 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1012 {
1013         sr->spill_offset = 0x10 - 4*pspoff;
1014 }
1015
1016 static inline unsigned char *
1017 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1018 {
1019         sr->imask = imaskp;
1020         return imaskp + (2*sr->region_len + 7)/8;
1021 }
1022
1023 /*
1024  * Body descriptors.
1025  */
1026 static inline void
1027 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1028 {
1029         sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1030         sr->epilogue_count = ecount + 1;
1031 }
1032
1033 static inline void
1034 desc_copy_state (unw_word label, struct unw_state_record *sr)
1035 {
1036         struct unw_labeled_state *ls;
1037
1038         for (ls = sr->labeled_states; ls; ls = ls->next) {
1039                 if (ls->label == label) {
1040                         free_state_stack(&sr->curr);
1041                         memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1042                         sr->curr.next = dup_state_stack(ls->saved_state.next);
1043                         return;
1044                 }
1045         }
1046         printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1047 }
1048
1049 static inline void
1050 desc_label_state (unw_word label, struct unw_state_record *sr)
1051 {
1052         struct unw_labeled_state *ls;
1053
1054         ls = alloc_labeled_state();
1055         if (!ls) {
1056                 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1057                 return;
1058         }
1059         ls->label = label;
1060         memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1061         ls->saved_state.next = dup_state_stack(sr->curr.next);
1062
1063         /* insert into list of labeled states: */
1064         ls->next = sr->labeled_states;
1065         sr->labeled_states = ls;
1066 }
1067
1068 /*
1069  * General descriptors.
1070  */
1071
1072 static inline int
1073 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1074 {
1075         if (sr->when_target <= sr->region_start + min_t(int, t, sr->region_len - 1))
1076                 return 0;
1077         if (qp > 0) {
1078                 if ((sr->pr_val & (1UL << qp)) == 0)
1079                         return 0;
1080                 sr->pr_mask |= (1UL << qp);
1081         }
1082         return 1;
1083 }
1084
1085 static inline void
1086 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1087 {
1088         struct unw_reg_info *r;
1089
1090         if (!desc_is_active(qp, t, sr))
1091                 return;
1092
1093         r = sr->curr.reg + decode_abreg(abreg, 0);
1094         r->where = UNW_WHERE_NONE;
1095         r->when = UNW_WHEN_NEVER;
1096         r->val = 0;
1097 }
1098
1099 static inline void
1100 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1101                      unsigned char ytreg, struct unw_state_record *sr)
1102 {
1103         enum unw_where where = UNW_WHERE_GR;
1104         struct unw_reg_info *r;
1105
1106         if (!desc_is_active(qp, t, sr))
1107                 return;
1108
1109         if (x)
1110                 where = UNW_WHERE_BR;
1111         else if (ytreg & 0x80)
1112                 where = UNW_WHERE_FR;
1113
1114         r = sr->curr.reg + decode_abreg(abreg, 0);
1115         r->where = where;
1116         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1117         r->val = (ytreg & 0x7f);
1118 }
1119
1120 static inline void
1121 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1122                      struct unw_state_record *sr)
1123 {
1124         struct unw_reg_info *r;
1125
1126         if (!desc_is_active(qp, t, sr))
1127                 return;
1128
1129         r = sr->curr.reg + decode_abreg(abreg, 1);
1130         r->where = UNW_WHERE_PSPREL;
1131         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1132         r->val = 0x10 - 4*pspoff;
1133 }
1134
1135 static inline void
1136 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1137                        struct unw_state_record *sr)
1138 {
1139         struct unw_reg_info *r;
1140
1141         if (!desc_is_active(qp, t, sr))
1142                 return;
1143
1144         r = sr->curr.reg + decode_abreg(abreg, 1);
1145         r->where = UNW_WHERE_SPREL;
1146         r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1147         r->val = 4*spoff;
1148 }
1149
1150 #define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1151                                                        code);
1152
1153 /*
1154  * region headers:
1155  */
1156 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1157 #define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1158 /*
1159  * prologue descriptors:
1160  */
1161 #define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1162 #define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1163 #define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1164 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1165 #define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1166 #define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1167 #define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1168 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1169 #define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1170 #define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1171 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1172 #define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1173 #define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1174 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1175 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1176 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1177 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1178 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1179 #define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1180 #define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1181 #define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1182 /*
1183  * body descriptors:
1184  */
1185 #define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1186 #define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1187 #define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1188 /*
1189  * general unwind descriptors:
1190  */
1191 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1192 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1193 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1194 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1195 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1196 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1197 #define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1198 #define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1199
1200 #include "unwind_decoder.c"
1201
1202 \f
1203 /* Unwind scripts. */
1204
1205 static inline unw_hash_index_t
1206 hash (unsigned long ip)
1207 {
1208 #       define hashmagic        0x9e3779b97f4a7c16UL    /* based on (sqrt(5)/2-1)*2^64 */
1209
1210         return (ip >> 4)*hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1211 #undef hashmagic
1212 }
1213
1214 static inline long
1215 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1216 {
1217         read_lock(&script->lock);
1218         if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1219                 /* keep the read lock... */
1220                 return 1;
1221         read_unlock(&script->lock);
1222         return 0;
1223 }
1224
1225 static inline struct unw_script *
1226 script_lookup (struct unw_frame_info *info)
1227 {
1228         struct unw_script *script = unw.cache + info->hint;
1229         unsigned short index;
1230         unsigned long ip, pr;
1231
1232         if (UNW_DEBUG_ON(0))
1233                 return NULL;    /* Always regenerate scripts in debug mode */
1234
1235         STAT(++unw.stat.cache.lookups);
1236
1237         ip = info->ip;
1238         pr = info->pr;
1239
1240         if (cache_match(script, ip, pr)) {
1241                 STAT(++unw.stat.cache.hinted_hits);
1242                 return script;
1243         }
1244
1245         index = unw.hash[hash(ip)];
1246         if (index >= UNW_CACHE_SIZE)
1247                 return NULL;
1248
1249         script = unw.cache + index;
1250         while (1) {
1251                 if (cache_match(script, ip, pr)) {
1252                         /* update hint; no locking required as single-word writes are atomic */
1253                         STAT(++unw.stat.cache.normal_hits);
1254                         unw.cache[info->prev_script].hint = script - unw.cache;
1255                         return script;
1256                 }
1257                 if (script->coll_chain >= UNW_HASH_SIZE)
1258                         return NULL;
1259                 script = unw.cache + script->coll_chain;
1260                 STAT(++unw.stat.cache.collision_chain_traversals);
1261         }
1262 }
1263
1264 /*
1265  * On returning, a write lock for the SCRIPT is still being held.
1266  */
1267 static inline struct unw_script *
1268 script_new (unsigned long ip)
1269 {
1270         struct unw_script *script, *prev, *tmp;
1271         unw_hash_index_t index;
1272         unsigned long flags;
1273         unsigned short head;
1274
1275         STAT(++unw.stat.script.news);
1276
1277         /*
1278          * Can't (easily) use cmpxchg() here because of ABA problem
1279          * that is intrinsic in cmpxchg()...
1280          */
1281         spin_lock_irqsave(&unw.lock, flags);
1282         {
1283                 head = unw.lru_head;
1284                 script = unw.cache + head;
1285                 unw.lru_head = script->lru_chain;
1286         }
1287         spin_unlock(&unw.lock);
1288
1289         /*
1290          * We'd deadlock here if we interrupted a thread that is holding a read lock on
1291          * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1292          * alternative would be to disable interrupts whenever we hold a read-lock, but
1293          * that seems silly.
1294          */
1295         if (!write_trylock(&script->lock))
1296                 return NULL;
1297
1298         spin_lock(&unw.lock);
1299         {
1300                 /* re-insert script at the tail of the LRU chain: */
1301                 unw.cache[unw.lru_tail].lru_chain = head;
1302                 unw.lru_tail = head;
1303
1304                 /* remove the old script from the hash table (if it's there): */
1305                 if (script->ip) {
1306                         index = hash(script->ip);
1307                         tmp = unw.cache + unw.hash[index];
1308                         prev = NULL;
1309                         while (1) {
1310                                 if (tmp == script) {
1311                                         if (prev)
1312                                                 prev->coll_chain = tmp->coll_chain;
1313                                         else
1314                                                 unw.hash[index] = tmp->coll_chain;
1315                                         break;
1316                                 } else
1317                                         prev = tmp;
1318                                 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1319                                 /* old script wasn't in the hash-table */
1320                                         break;
1321                                 tmp = unw.cache + tmp->coll_chain;
1322                         }
1323                 }
1324
1325                 /* enter new script in the hash table */
1326                 index = hash(ip);
1327                 script->coll_chain = unw.hash[index];
1328                 unw.hash[index] = script - unw.cache;
1329
1330                 script->ip = ip;        /* set new IP while we're holding the locks */
1331
1332                 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1333         }
1334         spin_unlock_irqrestore(&unw.lock, flags);
1335
1336         script->flags = 0;
1337         script->hint = 0;
1338         script->count = 0;
1339         return script;
1340 }
1341
1342 static void
1343 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1344 {
1345         script->pr_mask = sr->pr_mask;
1346         script->pr_val = sr->pr_val;
1347         /*
1348          * We could down-grade our write-lock on script->lock here but
1349          * the rwlock API doesn't offer atomic lock downgrading, so
1350          * we'll just keep the write-lock and release it later when
1351          * we're done using the script.
1352          */
1353 }
1354
1355 static inline void
1356 script_emit (struct unw_script *script, struct unw_insn insn)
1357 {
1358         if (script->count >= UNW_MAX_SCRIPT_LEN) {
1359                 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1360                         __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1361                 return;
1362         }
1363         script->insn[script->count++] = insn;
1364 }
1365
1366 static inline void
1367 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1368 {
1369         struct unw_reg_info *r = sr->curr.reg + i;
1370         enum unw_insn_opcode opc;
1371         struct unw_insn insn;
1372         unsigned long val = 0;
1373
1374         switch (r->where) {
1375               case UNW_WHERE_GR:
1376                 if (r->val >= 32) {
1377                         /* register got spilled to a stacked register */
1378                         opc = UNW_INSN_SETNAT_TYPE;
1379                         val = UNW_NAT_REGSTK;
1380                 } else
1381                         /* register got spilled to a scratch register */
1382                         opc = UNW_INSN_SETNAT_MEMSTK;
1383                 break;
1384
1385               case UNW_WHERE_FR:
1386                 opc = UNW_INSN_SETNAT_TYPE;
1387                 val = UNW_NAT_VAL;
1388                 break;
1389
1390               case UNW_WHERE_BR:
1391                 opc = UNW_INSN_SETNAT_TYPE;
1392                 val = UNW_NAT_NONE;
1393                 break;
1394
1395               case UNW_WHERE_PSPREL:
1396               case UNW_WHERE_SPREL:
1397                 opc = UNW_INSN_SETNAT_MEMSTK;
1398                 break;
1399
1400               default:
1401                 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1402                            __FUNCTION__, r->where);
1403                 return;
1404         }
1405         insn.opc = opc;
1406         insn.dst = unw.preg_index[i];
1407         insn.val = val;
1408         script_emit(script, insn);
1409 }
1410
1411 static void
1412 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1413 {
1414         struct unw_reg_info *r = sr->curr.reg + i;
1415         enum unw_insn_opcode opc;
1416         unsigned long val, rval;
1417         struct unw_insn insn;
1418         long need_nat_info;
1419
1420         if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1421                 return;
1422
1423         opc = UNW_INSN_MOVE;
1424         val = rval = r->val;
1425         need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1426
1427         switch (r->where) {
1428               case UNW_WHERE_GR:
1429                 if (rval >= 32) {
1430                         opc = UNW_INSN_MOVE_STACKED;
1431                         val = rval - 32;
1432                 } else if (rval >= 4 && rval <= 7) {
1433                         if (need_nat_info) {
1434                                 opc = UNW_INSN_MOVE2;
1435                                 need_nat_info = 0;
1436                         }
1437                         val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1438                 } else if (rval == 0) {
1439                         opc = UNW_INSN_MOVE_CONST;
1440                         val = 0;
1441                 } else {
1442                         /* register got spilled to a scratch register */
1443                         opc = UNW_INSN_MOVE_SCRATCH;
1444                         val = pt_regs_off(rval);
1445                 }
1446                 break;
1447
1448               case UNW_WHERE_FR:
1449                 if (rval <= 5)
1450                         val = unw.preg_index[UNW_REG_F2  + (rval -  2)];
1451                 else if (rval >= 16 && rval <= 31)
1452                         val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1453                 else {
1454                         opc = UNW_INSN_MOVE_SCRATCH;
1455                         if (rval <= 11)
1456                                 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1457                         else
1458                                 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1459                                            __FUNCTION__, rval);
1460                 }
1461                 break;
1462
1463               case UNW_WHERE_BR:
1464                 if (rval >= 1 && rval <= 5)
1465                         val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1466                 else {
1467                         opc = UNW_INSN_MOVE_SCRATCH;
1468                         if (rval == 0)
1469                                 val = offsetof(struct pt_regs, b0);
1470                         else if (rval == 6)
1471                                 val = offsetof(struct pt_regs, b6);
1472                         else
1473                                 val = offsetof(struct pt_regs, b7);
1474                 }
1475                 break;
1476
1477               case UNW_WHERE_SPREL:
1478                 opc = UNW_INSN_ADD_SP;
1479                 break;
1480
1481               case UNW_WHERE_PSPREL:
1482                 opc = UNW_INSN_ADD_PSP;
1483                 break;
1484
1485               default:
1486                 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1487                            __FUNCTION__, i, r->where);
1488                 break;
1489         }
1490         insn.opc = opc;
1491         insn.dst = unw.preg_index[i];
1492         insn.val = val;
1493         script_emit(script, insn);
1494         if (need_nat_info)
1495                 emit_nat_info(sr, i, script);
1496
1497         if (i == UNW_REG_PSP) {
1498                 /*
1499                  * info->psp must contain the _value_ of the previous
1500                  * sp, not it's save location.  We get this by
1501                  * dereferencing the value we just stored in
1502                  * info->psp:
1503                  */
1504                 insn.opc = UNW_INSN_LOAD;
1505                 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1506                 script_emit(script, insn);
1507         }
1508 }
1509
1510 static inline const struct unw_table_entry *
1511 lookup (struct unw_table *table, unsigned long rel_ip)
1512 {
1513         const struct unw_table_entry *e = NULL;
1514         unsigned long lo, hi, mid;
1515
1516         /* do a binary search for right entry: */
1517         for (lo = 0, hi = table->length; lo < hi; ) {
1518                 mid = (lo + hi) / 2;
1519                 e = &table->array[mid];
1520                 if (rel_ip < e->start_offset)
1521                         hi = mid;
1522                 else if (rel_ip >= e->end_offset)
1523                         lo = mid + 1;
1524                 else
1525                         break;
1526         }
1527         if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1528                 return NULL;
1529         return e;
1530 }
1531
1532 /*
1533  * Build an unwind script that unwinds from state OLD_STATE to the
1534  * entrypoint of the function that called OLD_STATE.
1535  */
1536 static inline struct unw_script *
1537 build_script (struct unw_frame_info *info)
1538 {
1539         const struct unw_table_entry *e = NULL;
1540         struct unw_script *script = NULL;
1541         struct unw_labeled_state *ls, *next;
1542         unsigned long ip = info->ip;
1543         struct unw_state_record sr;
1544         struct unw_table *table;
1545         struct unw_reg_info *r;
1546         struct unw_insn insn;
1547         u8 *dp, *desc_end;
1548         u64 hdr;
1549         int i;
1550         STAT(unsigned long start, parse_start;)
1551
1552         STAT(++unw.stat.script.builds; start = ia64_get_itc());
1553
1554         /* build state record */
1555         memset(&sr, 0, sizeof(sr));
1556         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1557                 r->when = UNW_WHEN_NEVER;
1558         sr.pr_val = info->pr;
1559
1560         UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1561         script = script_new(ip);
1562         if (!script) {
1563                 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __FUNCTION__);
1564                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1565                 return NULL;
1566         }
1567         unw.cache[info->prev_script].hint = script - unw.cache;
1568
1569         /* search the kernels and the modules' unwind tables for IP: */
1570
1571         STAT(parse_start = ia64_get_itc());
1572
1573         for (table = unw.tables; table; table = table->next) {
1574                 if (ip >= table->start && ip < table->end) {
1575                         e = lookup(table, ip - table->segment_base);
1576                         break;
1577                 }
1578         }
1579         if (!e) {
1580                 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1581                 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1582                         __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1583                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1584                 sr.curr.reg[UNW_REG_RP].when = -1;
1585                 sr.curr.reg[UNW_REG_RP].val = 0;
1586                 compile_reg(&sr, UNW_REG_RP, script);
1587                 script_finalize(script, &sr);
1588                 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1589                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1590                 return script;
1591         }
1592
1593         sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1594                           + (ip & 0xfUL));
1595         hdr = *(u64 *) (table->segment_base + e->info_offset);
1596         dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1597         desc_end = dp + 8*UNW_LENGTH(hdr);
1598
1599         while (!sr.done && dp < desc_end)
1600                 dp = unw_decode(dp, sr.in_body, &sr);
1601
1602         if (sr.when_target > sr.epilogue_start) {
1603                 /*
1604                  * sp has been restored and all values on the memory stack below
1605                  * psp also have been restored.
1606                  */
1607                 sr.curr.reg[UNW_REG_PSP].val = 0;
1608                 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1609                 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1610                 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1611                         if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1612                             || r->where == UNW_WHERE_SPREL)
1613                         {
1614                                 r->val = 0;
1615                                 r->where = UNW_WHERE_NONE;
1616                                 r->when = UNW_WHEN_NEVER;
1617                         }
1618         }
1619
1620         script->flags = sr.flags;
1621
1622         /*
1623          * If RP did't get saved, generate entry for the return link
1624          * register.
1625          */
1626         if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1627                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1628                 sr.curr.reg[UNW_REG_RP].when = -1;
1629                 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1630                 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1631                            __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1632                            sr.curr.reg[UNW_REG_RP].val);
1633         }
1634
1635 #ifdef UNW_DEBUG
1636         UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1637                 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1638         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1639                 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1640                         UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1641                         switch (r->where) {
1642                               case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1643                               case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1644                               case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1645                               case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1646                               case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1647                               case UNW_WHERE_NONE:
1648                                 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1649                                 break;
1650
1651                               default:
1652                                 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1653                                 break;
1654                         }
1655                         UNW_DPRINT(1, "\t\t%d\n", r->when);
1656                 }
1657         }
1658 #endif
1659
1660         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1661
1662         /* translate state record into unwinder instructions: */
1663
1664         /*
1665          * First, set psp if we're dealing with a fixed-size frame;
1666          * subsequent instructions may depend on this value.
1667          */
1668         if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1669             && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1670             && sr.curr.reg[UNW_REG_PSP].val != 0) {
1671                 /* new psp is sp plus frame size */
1672                 insn.opc = UNW_INSN_ADD;
1673                 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1674                 insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1675                 script_emit(script, insn);
1676         }
1677
1678         /* determine where the primary UNaT is: */
1679         if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1680                 i = UNW_REG_PRI_UNAT_MEM;
1681         else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1682                 i = UNW_REG_PRI_UNAT_GR;
1683         else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1684                 i = UNW_REG_PRI_UNAT_MEM;
1685         else
1686                 i = UNW_REG_PRI_UNAT_GR;
1687
1688         compile_reg(&sr, i, script);
1689
1690         for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1691                 compile_reg(&sr, i, script);
1692
1693         /* free labeled register states & stack: */
1694
1695         STAT(parse_start = ia64_get_itc());
1696         for (ls = sr.labeled_states; ls; ls = next) {
1697                 next = ls->next;
1698                 free_state_stack(&ls->saved_state);
1699                 free_labeled_state(ls);
1700         }
1701         free_state_stack(&sr.curr);
1702         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1703
1704         script_finalize(script, &sr);
1705         STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1706         return script;
1707 }
1708
1709 /*
1710  * Apply the unwinding actions represented by OPS and update SR to
1711  * reflect the state that existed upon entry to the function that this
1712  * unwinder represents.
1713  */
1714 static inline void
1715 run_script (struct unw_script *script, struct unw_frame_info *state)
1716 {
1717         struct unw_insn *ip, *limit, next_insn;
1718         unsigned long opc, dst, val, off;
1719         unsigned long *s = (unsigned long *) state;
1720         STAT(unsigned long start;)
1721
1722         STAT(++unw.stat.script.runs; start = ia64_get_itc());
1723         state->flags = script->flags;
1724         ip = script->insn;
1725         limit = script->insn + script->count;
1726         next_insn = *ip;
1727
1728         while (ip++ < limit) {
1729                 opc = next_insn.opc;
1730                 dst = next_insn.dst;
1731                 val = next_insn.val;
1732                 next_insn = *ip;
1733
1734           redo:
1735                 switch (opc) {
1736                       case UNW_INSN_ADD:
1737                         s[dst] += val;
1738                         break;
1739
1740                       case UNW_INSN_MOVE2:
1741                         if (!s[val])
1742                                 goto lazy_init;
1743                         s[dst+1] = s[val+1];
1744                         s[dst] = s[val];
1745                         break;
1746
1747                       case UNW_INSN_MOVE:
1748                         if (!s[val])
1749                                 goto lazy_init;
1750                         s[dst] = s[val];
1751                         break;
1752
1753                       case UNW_INSN_MOVE_SCRATCH:
1754                         if (state->pt) {
1755                                 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1756                         } else {
1757                                 s[dst] = 0;
1758                                 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1759                                            __FUNCTION__, dst, val);
1760                         }
1761                         break;
1762
1763                       case UNW_INSN_MOVE_CONST:
1764                         if (val == 0)
1765                                 s[dst] = (unsigned long) &unw.r0;
1766                         else {
1767                                 s[dst] = 0;
1768                                 UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1769                                            __FUNCTION__, val);
1770                         }
1771                         break;
1772
1773
1774                       case UNW_INSN_MOVE_STACKED:
1775                         s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1776                                                                     val);
1777                         break;
1778
1779                       case UNW_INSN_ADD_PSP:
1780                         s[dst] = state->psp + val;
1781                         break;
1782
1783                       case UNW_INSN_ADD_SP:
1784                         s[dst] = state->sp + val;
1785                         break;
1786
1787                       case UNW_INSN_SETNAT_MEMSTK:
1788                         if (!state->pri_unat_loc)
1789                                 state->pri_unat_loc = &state->sw->ar_unat;
1790                         /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1791                         s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1792                         break;
1793
1794                       case UNW_INSN_SETNAT_TYPE:
1795                         s[dst+1] = val;
1796                         break;
1797
1798                       case UNW_INSN_LOAD:
1799 #ifdef UNW_DEBUG
1800                         if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1801                             || s[val] < TASK_SIZE)
1802                         {
1803                                 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1804                                            __FUNCTION__, s[val]);
1805                                 break;
1806                         }
1807 #endif
1808                         s[dst] = *(unsigned long *) s[val];
1809                         break;
1810                 }
1811         }
1812         STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1813         return;
1814
1815   lazy_init:
1816         off = unw.sw_off[val];
1817         s[val] = (unsigned long) state->sw + off;
1818         if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1819                 /*
1820                  * We're initializing a general register: init NaT info, too.  Note that
1821                  * the offset is a multiple of 8 which gives us the 3 bits needed for
1822                  * the type field.
1823                  */
1824                 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1825         goto redo;
1826 }
1827
1828 static int
1829 find_save_locs (struct unw_frame_info *info)
1830 {
1831         int have_write_lock = 0;
1832         struct unw_script *scr;
1833
1834         if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1835                 /* don't let obviously bad addresses pollute the cache */
1836                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1837                 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1838                 info->rp_loc = NULL;
1839                 return -1;
1840         }
1841
1842         scr = script_lookup(info);
1843         if (!scr) {
1844                 scr = build_script(info);
1845                 if (!scr) {
1846                         UNW_DPRINT(0,
1847                                    "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1848                                    __FUNCTION__, info->ip);
1849                         return -1;
1850                 }
1851                 have_write_lock = 1;
1852         }
1853         info->hint = scr->hint;
1854         info->prev_script = scr - unw.cache;
1855
1856         run_script(scr, info);
1857
1858         if (have_write_lock)
1859                 write_unlock(&scr->lock);
1860         else
1861                 read_unlock(&scr->lock);
1862         return 0;
1863 }
1864
1865 int
1866 unw_unwind (struct unw_frame_info *info)
1867 {
1868         unsigned long prev_ip, prev_sp, prev_bsp;
1869         unsigned long ip, pr, num_regs;
1870         STAT(unsigned long start, flags;)
1871         int retval;
1872
1873         STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1874
1875         prev_ip = info->ip;
1876         prev_sp = info->sp;
1877         prev_bsp = info->bsp;
1878
1879         /* restore the ip */
1880         if (!info->rp_loc) {
1881                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1882                 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1883                            __FUNCTION__, info->ip);
1884                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1885                 return -1;
1886         }
1887         ip = info->ip = *info->rp_loc;
1888         if (ip < GATE_ADDR) {
1889                 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1890                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1891                 return -1;
1892         }
1893
1894         /* restore the cfm: */
1895         if (!info->pfs_loc) {
1896                 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1897                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1898                 return -1;
1899         }
1900         info->cfm_loc = info->pfs_loc;
1901
1902         /* restore the bsp: */
1903         pr = info->pr;
1904         num_regs = 0;
1905         if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1906                 info->pt = info->sp + 16;
1907                 if ((pr & (1UL << pNonSys)) != 0)
1908                         num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1909                 info->pfs_loc =
1910                         (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1911                 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1912         } else
1913                 num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1914         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1915         if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1916                 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1917                         __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1918                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1919                 return -1;
1920         }
1921
1922         /* restore the sp: */
1923         info->sp = info->psp;
1924         if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1925                 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1926                         __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1927                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1928                 return -1;
1929         }
1930
1931         if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1932                 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1933                            __FUNCTION__, ip);
1934                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1935                 return -1;
1936         }
1937
1938         /* as we unwind, the saved ar.unat becomes the primary unat: */
1939         info->pri_unat_loc = info->unat_loc;
1940
1941         /* finally, restore the predicates: */
1942         unw_get_pr(info, &info->pr);
1943
1944         retval = find_save_locs(info);
1945         STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1946         return retval;
1947 }
1948 EXPORT_SYMBOL(unw_unwind);
1949
1950 int
1951 unw_unwind_to_user (struct unw_frame_info *info)
1952 {
1953         unsigned long ip;
1954
1955         while (unw_unwind(info) >= 0) {
1956                 if (unw_get_rp(info, &ip) < 0) {
1957                         unw_get_ip(info, &ip);
1958                         UNW_DPRINT(0, "unwind.%s: failed to read return pointer (ip=0x%lx)\n",
1959                                    __FUNCTION__, ip);
1960                         return -1;
1961                 }
1962                 if (ip < FIXADDR_USER_END)
1963                         return 0;
1964         }
1965         unw_get_ip(info, &ip);
1966         UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n", __FUNCTION__, ip);
1967         return -1;
1968 }
1969 EXPORT_SYMBOL(unw_unwind_to_user);
1970
1971 static void
1972 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1973                  struct switch_stack *sw, unsigned long stktop)
1974 {
1975         unsigned long rbslimit, rbstop, stklimit;
1976         STAT(unsigned long start, flags;)
1977
1978         STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1979
1980         /*
1981          * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1982          * don't want to do that because it would be slow as each preserved register would
1983          * have to be processed.  Instead, what we do here is zero out the frame info and
1984          * start the unwind process at the function that created the switch_stack frame.
1985          * When a preserved value in switch_stack needs to be accessed, run_script() will
1986          * initialize the appropriate pointer on demand.
1987          */
1988         memset(info, 0, sizeof(*info));
1989
1990         rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1991         rbstop   = sw->ar_bspstore;
1992         if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1993                 rbstop = rbslimit;
1994
1995         stklimit = (unsigned long) t + IA64_STK_OFFSET;
1996         if (stktop <= rbstop)
1997                 stktop = rbstop;
1998
1999         info->regstk.limit = rbslimit;
2000         info->regstk.top   = rbstop;
2001         info->memstk.limit = stklimit;
2002         info->memstk.top   = stktop;
2003         info->task = t;
2004         info->sw  = sw;
2005         info->sp = info->psp = stktop;
2006         info->pr = sw->pr;
2007         UNW_DPRINT(3, "unwind.%s:\n"
2008                    "  task   0x%lx\n"
2009                    "  rbs = [0x%lx-0x%lx)\n"
2010                    "  stk = [0x%lx-0x%lx)\n"
2011                    "  pr     0x%lx\n"
2012                    "  sw     0x%lx\n"
2013                    "  sp     0x%lx\n",
2014                    __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2015                    info->pr, (unsigned long) info->sw, info->sp);
2016         STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2017 }
2018
2019 void
2020 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
2021                             struct pt_regs *pt, struct switch_stack *sw)
2022 {
2023         unsigned long sof;
2024
2025         init_frame_info(info, t, sw, pt->r12);
2026         info->cfm_loc = &pt->cr_ifs;
2027         info->unat_loc = &pt->ar_unat;
2028         info->pfs_loc = &pt->ar_pfs;
2029         sof = *info->cfm_loc & 0x7f;
2030         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
2031         info->ip = pt->cr_iip + ia64_psr(pt)->ri;
2032         info->pt = (unsigned long) pt;
2033         UNW_DPRINT(3, "unwind.%s:\n"
2034                    "  bsp    0x%lx\n"
2035                    "  sof    0x%lx\n"
2036                    "  ip     0x%lx\n",
2037                    __FUNCTION__, info->bsp, sof, info->ip);
2038         find_save_locs(info);
2039 }
2040
2041 void
2042 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2043 {
2044         unsigned long sol;
2045
2046         init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2047         info->cfm_loc = &sw->ar_pfs;
2048         sol = (*info->cfm_loc >> 7) & 0x7f;
2049         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2050         info->ip = sw->b0;
2051         UNW_DPRINT(3, "unwind.%s:\n"
2052                    "  bsp    0x%lx\n"
2053                    "  sol    0x%lx\n"
2054                    "  ip     0x%lx\n",
2055                    __FUNCTION__, info->bsp, sol, info->ip);
2056         find_save_locs(info);
2057 }
2058
2059 void
2060 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2061 {
2062         struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2063
2064         UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
2065         unw_init_frame_info(info, t, sw);
2066 }
2067 EXPORT_SYMBOL(unw_init_from_blocked_task);
2068
2069 static void
2070 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2071                    unsigned long gp, const void *table_start, const void *table_end)
2072 {
2073         const struct unw_table_entry *start = table_start, *end = table_end;
2074
2075         table->name = name;
2076         table->segment_base = segment_base;
2077         table->gp = gp;
2078         table->start = segment_base + start[0].start_offset;
2079         table->end = segment_base + end[-1].end_offset;
2080         table->array = start;
2081         table->length = end - start;
2082 }
2083
2084 void *
2085 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2086                       const void *table_start, const void *table_end)
2087 {
2088         const struct unw_table_entry *start = table_start, *end = table_end;
2089         struct unw_table *table;
2090         unsigned long flags;
2091
2092         if (end - start <= 0) {
2093                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2094                            __FUNCTION__);
2095                 return NULL;
2096         }
2097
2098         table = kmalloc(sizeof(*table), GFP_USER);
2099         if (!table)
2100                 return NULL;
2101
2102         init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2103
2104         spin_lock_irqsave(&unw.lock, flags);
2105         {
2106                 /* keep kernel unwind table at the front (it's searched most commonly): */
2107                 table->next = unw.tables->next;
2108                 unw.tables->next = table;
2109         }
2110         spin_unlock_irqrestore(&unw.lock, flags);
2111
2112         return table;
2113 }
2114
2115 void
2116 unw_remove_unwind_table (void *handle)
2117 {
2118         struct unw_table *table, *prev;
2119         struct unw_script *tmp;
2120         unsigned long flags;
2121         long index;
2122
2123         if (!handle) {
2124                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2125                            __FUNCTION__);
2126                 return;
2127         }
2128
2129         table = handle;
2130         if (table == &unw.kernel_table) {
2131                 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2132                            "no-can-do!\n", __FUNCTION__);
2133                 return;
2134         }
2135
2136         spin_lock_irqsave(&unw.lock, flags);
2137         {
2138                 /* first, delete the table: */
2139
2140                 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2141                         if (prev->next == table)
2142                                 break;
2143                 if (!prev) {
2144                         UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2145                                    __FUNCTION__, (void *) table);
2146                         spin_unlock_irqrestore(&unw.lock, flags);
2147                         return;
2148                 }
2149                 prev->next = table->next;
2150         }
2151         spin_unlock_irqrestore(&unw.lock, flags);
2152
2153         /* next, remove hash table entries for this table */
2154
2155         for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2156                 tmp = unw.cache + unw.hash[index];
2157                 if (unw.hash[index] >= UNW_CACHE_SIZE
2158                     || tmp->ip < table->start || tmp->ip >= table->end)
2159                         continue;
2160
2161                 write_lock(&tmp->lock);
2162                 {
2163                         if (tmp->ip >= table->start && tmp->ip < table->end) {
2164                                 unw.hash[index] = tmp->coll_chain;
2165                                 tmp->ip = 0;
2166                         }
2167                 }
2168                 write_unlock(&tmp->lock);
2169         }
2170
2171         kfree(table);
2172 }
2173
2174 static int __init
2175 create_gate_table (void)
2176 {
2177         const struct unw_table_entry *entry, *start, *end;
2178         unsigned long *lp, segbase = GATE_ADDR;
2179         size_t info_size, size;
2180         char *info;
2181         Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2182         int i;
2183
2184         for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2185                 if (phdr->p_type == PT_IA_64_UNWIND) {
2186                         punw = phdr;
2187                         break;
2188                 }
2189
2190         if (!punw) {
2191                 printk("%s: failed to find gate DSO's unwind table!\n", __FUNCTION__);
2192                 return 0;
2193         }
2194
2195         start = (const struct unw_table_entry *) punw->p_vaddr;
2196         end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2197         size  = 0;
2198
2199         unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2200
2201         for (entry = start; entry < end; ++entry)
2202                 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2203         size += 8;      /* reserve space for "end of table" marker */
2204
2205         unw.gate_table = kmalloc(size, GFP_KERNEL);
2206         if (!unw.gate_table) {
2207                 unw.gate_table_size = 0;
2208                 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __FUNCTION__);
2209                 return 0;
2210         }
2211         unw.gate_table_size = size;
2212
2213         lp = unw.gate_table;
2214         info = (char *) unw.gate_table + size;
2215
2216         for (entry = start; entry < end; ++entry, lp += 3) {
2217                 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2218                 info -= info_size;
2219                 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2220
2221                 lp[0] = segbase + entry->start_offset;          /* start */
2222                 lp[1] = segbase + entry->end_offset;            /* end */
2223                 lp[2] = info - (char *) unw.gate_table;         /* info */
2224         }
2225         *lp = 0;        /* end-of-table marker */
2226         return 0;
2227 }
2228
2229 __initcall(create_gate_table);
2230
2231 void __init
2232 unw_init (void)
2233 {
2234         extern char __gp[];
2235         extern void unw_hash_index_t_is_too_narrow (void);
2236         long i, off;
2237
2238         if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2239                 unw_hash_index_t_is_too_narrow();
2240
2241         unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2242         unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2243         unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2244         unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2245         unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2246         unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2247         unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2248         unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2249         for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2250                 unw.sw_off[unw.preg_index[i]] = off;
2251         for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2252                 unw.sw_off[unw.preg_index[i]] = off;
2253         for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2254                 unw.sw_off[unw.preg_index[i]] = off;
2255         for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2256                 unw.sw_off[unw.preg_index[i]] = off;
2257
2258         for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2259                 if (i > 0)
2260                         unw.cache[i].lru_chain = (i - 1);
2261                 unw.cache[i].coll_chain = -1;
2262                 unw.cache[i].lock = RW_LOCK_UNLOCKED;
2263         }
2264         unw.lru_head = UNW_CACHE_SIZE - 1;
2265         unw.lru_tail = 0;
2266
2267         init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2268                           __start_unwind, __end_unwind);
2269 }
2270
2271 /*
2272  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2273  *
2274  *      This system call has been deprecated.  The new and improved way to get
2275  *      at the kernel's unwind info is via the gate DSO.  The address of the
2276  *      ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2277  *
2278  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2279  *
2280  * This system call copies the unwind data into the buffer pointed to by BUF and returns
2281  * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2282  * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2283  * unwind data.
2284  *
2285  * The first portion of the unwind data contains an unwind table and rest contains the
2286  * associated unwind info (in no particular order).  The unwind table consists of a table
2287  * of entries of the form:
2288  *
2289  *      u64 start;      (64-bit address of start of function)
2290  *      u64 end;        (64-bit address of start of function)
2291  *      u64 info;       (BUF-relative offset to unwind info)
2292  *
2293  * The end of the unwind table is indicated by an entry with a START address of zero.
2294  *
2295  * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2296  * on the format of the unwind info.
2297  *
2298  * ERRORS
2299  *      EFAULT  BUF points outside your accessible address space.
2300  */
2301 asmlinkage long
2302 sys_getunwind (void __user *buf, size_t buf_size)
2303 {
2304         if (buf && buf_size >= unw.gate_table_size)
2305                 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2306                         return -EFAULT;
2307         return unw.gate_table_size;
2308 }