arch/ia64/kernel/unwind.c

   1 /*
   2  * Copyright (C) 1999-2004 Hewlett-Packard Co
   3  *      David Mosberger-Tang <davidm@hpl.hp.com>
   4  * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
   5  *      - Change pt_regs_off() to make it less dependant on pt_regs structure.
   6  */
   7 /*
   8  * This file implements call frame unwind support for the Linux
   9  * kernel.  Parsing and processing the unwind information is
  10  * time-consuming, so this implementation translates the unwind
  11  * descriptors into unwind scripts.  These scripts are very simple
  12  * (basically a sequence of assignments) and efficient to execute.
  13  * They are cached for later re-use.  Each script is specific for a
  14  * given instruction pointer address and the set of predicate values
  15  * that the script depends on (most unwind descriptors are
  16  * unconditional and scripts often do not depend on predicates at
  17  * all).  This code is based on the unwind conventions described in
  18  * the "IA-64 Software Conventions and Runtime Architecture" manual.
  19  *
  20  * SMP conventions:
  21  *      o updates to the global unwind data (in structure "unw") are serialized
  22  *        by the unw.lock spinlock
  23  *      o each unwind script has its own read-write lock; a thread must acquire
  24  *        a read lock before executing a script and must acquire a write lock
  25  *        before modifying a script
  26  *      o if both the unw.lock spinlock and a script's read-write lock must be
  27  *        acquired, then the read-write lock must be acquired first.
  28  */
  29 #include <linux/module.h>
  30 #include <linux/bootmem.h>
  31 #include <linux/elf.h>
  32 #include <linux/kernel.h>
  33 #include <linux/sched.h>
  34 #include <linux/slab.h>
  35
  36 #include <asm/unwind.h>
  37
  38 #include <asm/delay.h>
  39 #include <asm/page.h>
  40 #include <asm/ptrace.h>
  41 #include <asm/ptrace_offsets.h>
  42 #include <asm/rse.h>
  43 #include <asm/sections.h>
  44 #include <asm/system.h>
  45 #include <asm/uaccess.h>
  46
  47 #include "entry.h"
  48 #include "unwind_i.h"
  49
  50 #define MIN(a,b)        ((a) < (b) ? (a) : (b))
  51 #define p5              5
  52
  53 #define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  54 #define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  55
  56 #define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  57 #define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  58
  59 #define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  60
  61 #ifdef UNW_DEBUG
  62   static unsigned int unw_debug_level = UNW_DEBUG;
  63 #  define UNW_DEBUG_ON(n)       unw_debug_level >= n
  64    /* Do not code a printk level, not all debug lines end in newline */
  65 #  define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  66 #  define inline
  67 #else /* !UNW_DEBUG */
  68 #  define UNW_DEBUG_ON(n)  0
  69 #  define UNW_DPRINT(n, ...)
  70 #endif /* UNW_DEBUG */
  71
  72 #if UNW_STATS
  73 # define STAT(x...)     x
  74 #else
  75 # define STAT(x...)
  76 #endif
  77
  78 #define alloc_reg_state()       kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
  79 #define free_reg_state(usr)     kfree(usr)
  80 #define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  81 #define free_labeled_state(usr) kfree(usr)
  82
  83 typedef unsigned long unw_word;
  84 typedef unsigned char unw_hash_index_t;
  85
  86 static struct {
  87         spinlock_t lock;                        /* spinlock for unwind data */
  88
  89         /* list of unwind tables (one per load-module) */
  90         struct unw_table *tables;
  91
  92         unsigned long r0;                       /* constant 0 for r0 */
  93
  94         /* table of registers that prologues can save (and order in which they're saved): */
  95         const unsigned char save_order[8];
  96
  97         /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
  98         unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
  99
 100         unsigned short lru_head;                /* index of lead-recently used script */
 101         unsigned short lru_tail;                /* index of most-recently used script */
 102
 103         /* index into unw_frame_info for preserved register i */
 104         unsigned short preg_index[UNW_NUM_REGS];
 105
 106         short pt_regs_offsets[32];
 107
 108         /* unwind table for the kernel: */
 109         struct unw_table kernel_table;
 110
 111         /* unwind table describing the gate page (kernel code that is mapped into user space): */
 112         size_t gate_table_size;
 113         unsigned long *gate_table;
 114
 115         /* hash table that maps instruction pointer to script index: */
 116         unsigned short hash[UNW_HASH_SIZE];
 117
 118         /* script cache: */
 119         struct unw_script cache[UNW_CACHE_SIZE];
 120
 121 # ifdef UNW_DEBUG
 122         const char *preg_name[UNW_NUM_REGS];
 123 # endif
 124 # if UNW_STATS
 125         struct {
 126                 struct {
 127                         int lookups;
 128                         int hinted_hits;
 129                         int normal_hits;
 130                         int collision_chain_traversals;
 131                 } cache;
 132                 struct {
 133                         unsigned long build_time;
 134                         unsigned long run_time;
 135                         unsigned long parse_time;
 136                         int builds;
 137                         int news;
 138                         int collisions;
 139                         int runs;
 140                 } script;
 141                 struct {
 142                         unsigned long init_time;
 143                         unsigned long unwind_time;
 144                         int inits;
 145                         int unwinds;
 146                 } api;
 147         } stat;
 148 # endif
 149 } unw = {
 150         .tables = &unw.kernel_table,
 151         .lock = SPIN_LOCK_UNLOCKED,
 152         .save_order = {
 153                 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 154                 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 155         },
 156         .preg_index = {
 157                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_GR */
 158                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_MEM */
 159                 offsetof(struct unw_frame_info, bsp_loc)/8,
 160                 offsetof(struct unw_frame_info, bspstore_loc)/8,
 161                 offsetof(struct unw_frame_info, pfs_loc)/8,
 162                 offsetof(struct unw_frame_info, rnat_loc)/8,
 163                 offsetof(struct unw_frame_info, psp)/8,
 164                 offsetof(struct unw_frame_info, rp_loc)/8,
 165                 offsetof(struct unw_frame_info, r4)/8,
 166                 offsetof(struct unw_frame_info, r5)/8,
 167                 offsetof(struct unw_frame_info, r6)/8,
 168                 offsetof(struct unw_frame_info, r7)/8,
 169                 offsetof(struct unw_frame_info, unat_loc)/8,
 170                 offsetof(struct unw_frame_info, pr_loc)/8,
 171                 offsetof(struct unw_frame_info, lc_loc)/8,
 172                 offsetof(struct unw_frame_info, fpsr_loc)/8,
 173                 offsetof(struct unw_frame_info, b1_loc)/8,
 174                 offsetof(struct unw_frame_info, b2_loc)/8,
 175                 offsetof(struct unw_frame_info, b3_loc)/8,
 176                 offsetof(struct unw_frame_info, b4_loc)/8,
 177                 offsetof(struct unw_frame_info, b5_loc)/8,
 178                 offsetof(struct unw_frame_info, f2_loc)/8,
 179                 offsetof(struct unw_frame_info, f3_loc)/8,
 180                 offsetof(struct unw_frame_info, f4_loc)/8,
 181                 offsetof(struct unw_frame_info, f5_loc)/8,
 182                 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
 183                 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
 184                 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
 185                 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
 186                 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
 187                 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
 188                 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
 189                 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
 190                 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
 191                 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
 192                 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
 193                 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
 194                 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
 195                 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
 196                 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
 197                 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
 198         },
 199         .pt_regs_offsets = {
 200                 [0] = -1,
 201                 offsetof(struct pt_regs,  r1),
 202                 offsetof(struct pt_regs,  r2),
 203                 offsetof(struct pt_regs,  r3),
 204                 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
 205                 offsetof(struct pt_regs,  r8),
 206                 offsetof(struct pt_regs,  r9),
 207                 offsetof(struct pt_regs, r10),
 208                 offsetof(struct pt_regs, r11),
 209                 offsetof(struct pt_regs, r12),
 210                 offsetof(struct pt_regs, r13),
 211                 offsetof(struct pt_regs, r14),
 212                 offsetof(struct pt_regs, r15),
 213                 offsetof(struct pt_regs, r16),
 214                 offsetof(struct pt_regs, r17),
 215                 offsetof(struct pt_regs, r18),
 216                 offsetof(struct pt_regs, r19),
 217                 offsetof(struct pt_regs, r20),
 218                 offsetof(struct pt_regs, r21),
 219                 offsetof(struct pt_regs, r22),
 220                 offsetof(struct pt_regs, r23),
 221                 offsetof(struct pt_regs, r24),
 222                 offsetof(struct pt_regs, r25),
 223                 offsetof(struct pt_regs, r26),
 224                 offsetof(struct pt_regs, r27),
 225                 offsetof(struct pt_regs, r28),
 226                 offsetof(struct pt_regs, r29),
 227                 offsetof(struct pt_regs, r30),
 228                 offsetof(struct pt_regs, r31),
 229         },
 230         .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 231 #ifdef UNW_DEBUG
 232         .preg_name = {
 233                 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 234                 "r4", "r5", "r6", "r7",
 235                 "ar.unat", "pr", "ar.lc", "ar.fpsr",
 236                 "b1", "b2", "b3", "b4", "b5",
 237                 "f2", "f3", "f4", "f5",
 238                 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 239                 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 240         }
 241 #endif
 242 };
 243
 244 static inline int
 245 read_only (void *addr)
 246 {
 247         return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
 248 }
 249
 250 /*
 251  * Returns offset of rREG in struct pt_regs.
 252  */
 253 static inline unsigned long
 254 pt_regs_off (unsigned long reg)
 255 {
 256         short off = -1;
 257
 258         if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
 259                 off = unw.pt_regs_offsets[reg];
 260
 261         if (off < 0) {
 262                 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
 263                 off = 0;
 264         }
 265         return (unsigned long) off;
 266 }
 267
 268 static inline struct pt_regs *
 269 get_scratch_regs (struct unw_frame_info *info)
 270 {
 271         if (!info->pt) {
 272                 /* This should not happen with valid unwind info.  */
 273                 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
 274                 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 275                         info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 276                 else
 277                         info->pt = info->sp - 16;
 278         }
 279         UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
 280         return (struct pt_regs *) info->pt;
 281 }
 282
 283 /* Unwind accessors.  */
 284
 285 int
 286 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 287 {
 288         unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 289         struct unw_ireg *ireg;
 290         struct pt_regs *pt;
 291
 292         if ((unsigned) regnum - 1 >= 127) {
 293                 if (regnum == 0 && !write) {
 294                         *val = 0;       /* read r0 always returns 0 */
 295                         *nat = 0;
 296                         return 0;
 297                 }
 298                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 299                            __FUNCTION__, regnum);
 300                 return -1;
 301         }
 302
 303         if (regnum < 32) {
 304                 if (regnum >= 4 && regnum <= 7) {
 305                         /* access a preserved register */
 306                         ireg = &info->r4 + (regnum - 4);
 307                         addr = ireg->loc;
 308                         if (addr) {
 309                                 nat_addr = addr + ireg->nat.off;
 310                                 switch (ireg->nat.type) {
 311                                       case UNW_NAT_VAL:
 312                                         /* simulate getf.sig/setf.sig */
 313                                         if (write) {
 314                                                 if (*nat) {
 315                                                         /* write NaTVal and be done with it */
 316                                                         addr[0] = 0;
 317                                                         addr[1] = 0x1fffe;
 318                                                         return 0;
 319                                                 }
 320                                                 addr[1] = 0x1003e;
 321                                         } else {
 322                                                 if (addr[0] == 0 && addr[1] == 0x1ffe) {
 323                                                         /* return NaT and be done with it */
 324                                                         *val = 0;
 325                                                         *nat = 1;
 326                                                         return 0;
 327                                                 }
 328                                         }
 329                                         /* fall through */
 330                                       case UNW_NAT_NONE:
 331                                         dummy_nat = 0;
 332                                         nat_addr = &dummy_nat;
 333                                         break;
 334
 335                                       case UNW_NAT_MEMSTK:
 336                                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 337                                         break;
 338
 339                                       case UNW_NAT_REGSTK:
 340                                         nat_addr = ia64_rse_rnat_addr(addr);
 341                                         if ((unsigned long) addr < info->regstk.limit
 342                                             || (unsigned long) addr >= info->regstk.top)
 343                                         {
 344                                                 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 345                                                         "[0x%lx-0x%lx)\n",
 346                                                         __FUNCTION__, (void *) addr,
 347                                                         info->regstk.limit,
 348                                                         info->regstk.top);
 349                                                 return -1;
 350                                         }
 351                                         if ((unsigned long) nat_addr >= info->regstk.top)
 352                                                 nat_addr = &info->sw->ar_rnat;
 353                                         nat_mask = (1UL << ia64_rse_slot_num(addr));
 354                                         break;
 355                                 }
 356                         } else {
 357                                 addr = &info->sw->r4 + (regnum - 4);
 358                                 nat_addr = &info->sw->ar_unat;
 359                                 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 360                         }
 361                 } else {
 362                         /* access a scratch register */
 363                         pt = get_scratch_regs(info);
 364                         addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 365                         if (info->pri_unat_loc)
 366                                 nat_addr = info->pri_unat_loc;
 367                         else
 368                                 nat_addr = &info->sw->ar_unat;
 369                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 370                 }
 371         } else {
 372                 /* access a stacked register */
 373                 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 374                 nat_addr = ia64_rse_rnat_addr(addr);
 375                 if ((unsigned long) addr < info->regstk.limit
 376                     || (unsigned long) addr >= info->regstk.top)
 377                 {
 378                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 379                                    "of rbs\n",  __FUNCTION__);
 380                         return -1;
 381                 }
 382                 if ((unsigned long) nat_addr >= info->regstk.top)
 383                         nat_addr = &info->sw->ar_rnat;
 384                 nat_mask = (1UL << ia64_rse_slot_num(addr));
 385         }
 386
 387         if (write) {
 388                 if (read_only(addr))
 389                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n");
 390                 else {
 391                         *addr = *val;
 392                         if (*nat)
 393                                 *nat_addr |= nat_mask;
 394                         else
 395                                 *nat_addr &= ~nat_mask;
 396                 }
 397         } else {
 398                 if ((*nat_addr & nat_mask) == 0) {
 399                         *val = *addr;
 400                         *nat = 0;
 401                 } else {
 402                         *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 403                         *nat = 1;
 404                 }
 405         }
 406         return 0;
 407 }
 408 EXPORT_SYMBOL(unw_access_gr);
 409
 410 int
 411 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 412 {
 413         unsigned long *addr;
 414         struct pt_regs *pt;
 415
 416         switch (regnum) {
 417                 /* scratch: */
 418               case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
 419               case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
 420               case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
 421
 422                 /* preserved: */
 423               case 1: case 2: case 3: case 4: case 5:
 424                 addr = *(&info->b1_loc + (regnum - 1));
 425                 if (!addr)
 426                         addr = &info->sw->b1 + (regnum - 1);
 427                 break;
 428
 429               default:
 430                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 431                            __FUNCTION__, regnum);
 432                 return -1;
 433         }
 434         if (write)
 435                 if (read_only(addr))
 436                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n");
 437                 else
 438                         *addr = *val;
 439         else
 440                 *val = *addr;
 441         return 0;
 442 }
 443 EXPORT_SYMBOL(unw_access_br);
 444
 445 int
 446 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 447 {
 448         struct ia64_fpreg *addr = 0;
 449         struct pt_regs *pt;
 450
 451         if ((unsigned) (regnum - 2) >= 126) {
 452                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 453                            __FUNCTION__, regnum);
 454                 return -1;
 455         }
 456
 457         if (regnum <= 5) {
 458                 addr = *(&info->f2_loc + (regnum - 2));
 459                 if (!addr)
 460                         addr = &info->sw->f2 + (regnum - 2);
 461         } else if (regnum <= 15) {
 462                 if (regnum <= 11) {
 463                         pt = get_scratch_regs(info);
 464                         addr = &pt->f6  + (regnum - 6);
 465                 }
 466                 else
 467                         addr = &info->sw->f12 + (regnum - 12);
 468         } else if (regnum <= 31) {
 469                 addr = info->fr_loc[regnum - 16];
 470                 if (!addr)
 471                         addr = &info->sw->f16 + (regnum - 16);
 472         } else {
 473                 struct task_struct *t = info->task;
 474
 475                 if (write)
 476                         ia64_sync_fph(t);
 477                 else
 478                         ia64_flush_fph(t);
 479                 addr = t->thread.fph + (regnum - 32);
 480         }
 481
 482         if (write)
 483                 if (read_only(addr))
 484                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n");
 485                 else
 486                         *addr = *val;
 487         else
 488                 *val = *addr;
 489         return 0;
 490 }
 491 EXPORT_SYMBOL(unw_access_fr);
 492
 493 int
 494 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 495 {
 496         unsigned long *addr;
 497         struct pt_regs *pt;
 498
 499         switch (regnum) {
 500               case UNW_AR_BSP:
 501                 addr = info->bsp_loc;
 502                 if (!addr)
 503                         addr = &info->sw->ar_bspstore;
 504                 break;
 505
 506               case UNW_AR_BSPSTORE:
 507                 addr = info->bspstore_loc;
 508                 if (!addr)
 509                         addr = &info->sw->ar_bspstore;
 510                 break;
 511
 512               case UNW_AR_PFS:
 513                 addr = info->pfs_loc;
 514                 if (!addr)
 515                         addr = &info->sw->ar_pfs;
 516                 break;
 517
 518               case UNW_AR_RNAT:
 519                 addr = info->rnat_loc;
 520                 if (!addr)
 521                         addr = &info->sw->ar_rnat;
 522                 break;
 523
 524               case UNW_AR_UNAT:
 525                 addr = info->unat_loc;
 526                 if (!addr)
 527                         addr = &info->sw->ar_unat;
 528                 break;
 529
 530               case UNW_AR_LC:
 531                 addr = info->lc_loc;
 532                 if (!addr)
 533                         addr = &info->sw->ar_lc;
 534                 break;
 535
 536               case UNW_AR_EC:
 537                 if (!info->cfm_loc)
 538                         return -1;
 539                 if (write)
 540                         *info->cfm_loc =
 541                                 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 542                 else
 543                         *val = (*info->cfm_loc >> 52) & 0x3f;
 544                 return 0;
 545
 546               case UNW_AR_FPSR:
 547                 addr = info->fpsr_loc;
 548                 if (!addr)
 549                         addr = &info->sw->ar_fpsr;
 550                 break;
 551
 552               case UNW_AR_RSC:
 553                 pt = get_scratch_regs(info);
 554                 addr = &pt->ar_rsc;
 555                 break;
 556
 557               case UNW_AR_CCV:
 558                 pt = get_scratch_regs(info);
 559                 addr = &pt->ar_ccv;
 560                 break;
 561
 562               case UNW_AR_CSD:
 563                 pt = get_scratch_regs(info);
 564                 addr = &pt->ar_csd;
 565                 break;
 566
 567               case UNW_AR_SSD:
 568                 pt = get_scratch_regs(info);
 569                 addr = &pt->ar_ssd;
 570                 break;
 571
 572               default:
 573                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 574                            __FUNCTION__, regnum);
 575                 return -1;
 576         }
 577
 578         if (write) {
 579                 if (read_only(addr))
 580                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n");
 581                 else
 582                         *addr = *val;
 583         } else
 584                 *val = *addr;
 585         return 0;
 586 }
 587 EXPORT_SYMBOL(unw_access_ar);
 588
 589 int
 590 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 591 {
 592         unsigned long *addr;
 593
 594         addr = info->pr_loc;
 595         if (!addr)
 596                 addr = &info->sw->pr;
 597
 598         if (write) {
 599                 if (read_only(addr))
 600                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n");
 601                 else
 602                         *addr = *val;
 603         } else
 604                 *val = *addr;
 605         return 0;
 606 }
 607 EXPORT_SYMBOL(unw_access_pr);
 608
 609 \f
 610 /* Routines to manipulate the state stack.  */
 611
 612 static inline void
 613 push (struct unw_state_record *sr)
 614 {
 615         struct unw_reg_state *rs;
 616
 617         rs = alloc_reg_state();
 618         if (!rs) {
 619                 printk(KERN_ERR "unwind: cannot stack reg state!\n");
 620                 return;
 621         }
 622         memcpy(rs, &sr->curr, sizeof(*rs));
 623         sr->curr.next = rs;
 624 }
 625
 626 static void
 627 pop (struct unw_state_record *sr)
 628 {
 629         struct unw_reg_state *rs = sr->curr.next;
 630
 631         if (!rs) {
 632                 printk(KERN_ERR "unwind: stack underflow!\n");
 633                 return;
 634         }
 635         memcpy(&sr->curr, rs, sizeof(*rs));
 636         free_reg_state(rs);
 637 }
 638
 639 /* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 640 static struct unw_reg_state *
 641 dup_state_stack (struct unw_reg_state *rs)
 642 {
 643         struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 644
 645         while (rs) {
 646                 copy = alloc_reg_state();
 647                 if (!copy) {
 648                         printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 649                         return NULL;
 650                 }
 651                 memcpy(copy, rs, sizeof(*copy));
 652                 if (first)
 653                         prev->next = copy;
 654                 else
 655                         first = copy;
 656                 rs = rs->next;
 657                 prev = copy;
 658         }
 659         return first;
 660 }
 661
 662 /* Free all stacked register states (but not RS itself).  */
 663 static void
 664 free_state_stack (struct unw_reg_state *rs)
 665 {
 666         struct unw_reg_state *p, *next;
 667
 668         for (p = rs->next; p != NULL; p = next) {
 669                 next = p->next;
 670                 free_reg_state(p);
 671         }
 672         rs->next = NULL;
 673 }
 674 \f
 675 /* Unwind decoder routines */
 676
 677 static enum unw_register_index __attribute_const__
 678 decode_abreg (unsigned char abreg, int memory)
 679 {
 680         switch (abreg) {
 681               case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 682               case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 683               case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 684               case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 685               case 0x60: return UNW_REG_PR;
 686               case 0x61: return UNW_REG_PSP;
 687               case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 688               case 0x63: return UNW_REG_RP;
 689               case 0x64: return UNW_REG_BSP;
 690               case 0x65: return UNW_REG_BSPSTORE;
 691               case 0x66: return UNW_REG_RNAT;
 692               case 0x67: return UNW_REG_UNAT;
 693               case 0x68: return UNW_REG_FPSR;
 694               case 0x69: return UNW_REG_PFS;
 695               case 0x6a: return UNW_REG_LC;
 696               default:
 697                 break;
 698         }
 699         UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
 700         return UNW_REG_LC;
 701 }
 702
 703 static void
 704 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 705 {
 706         reg->val = val;
 707         reg->where = where;
 708         if (reg->when == UNW_WHEN_NEVER)
 709                 reg->when = when;
 710 }
 711
 712 static void
 713 alloc_spill_area (unsigned long *offp, unsigned long regsize,
 714                   struct unw_reg_info *lo, struct unw_reg_info *hi)
 715 {
 716         struct unw_reg_info *reg;
 717
 718         for (reg = hi; reg >= lo; --reg) {
 719                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 720                         reg->where = UNW_WHERE_PSPREL;
 721                         *offp -= regsize;
 722                         reg->val = *offp;
 723                 }
 724         }
 725 }
 726
 727 static inline void
 728 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 729 {
 730         struct unw_reg_info *reg;
 731
 732         for (reg = *regp; reg <= lim; ++reg) {
 733                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 734                         reg->when = t;
 735                         *regp = reg + 1;
 736                         return;
 737                 }
 738         }
 739         UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __FUNCTION__);
 740 }
 741
 742 static inline void
 743 finish_prologue (struct unw_state_record *sr)
 744 {
 745         struct unw_reg_info *reg;
 746         unsigned long off;
 747         int i;
 748
 749         /*
 750          * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 751          * for Using Unwind Descriptors", rule 3):
 752          */
 753         for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 754                 reg = sr->curr.reg + unw.save_order[i];
 755                 if (reg->where == UNW_WHERE_GR_SAVE) {
 756                         reg->where = UNW_WHERE_GR;
 757                         reg->val = sr->gr_save_loc++;
 758                 }
 759         }
 760
 761         /*
 762          * Next, compute when the fp, general, and branch registers get
 763          * saved.  This must come before alloc_spill_area() because
 764          * we need to know which registers are spilled to their home
 765          * locations.
 766          */
 767         if (sr->imask) {
 768                 unsigned char kind, mask = 0, *cp = sr->imask;
 769                 int t;
 770                 static const unsigned char limit[3] = {
 771                         UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 772                 };
 773                 struct unw_reg_info *(regs[3]);
 774
 775                 regs[0] = sr->curr.reg + UNW_REG_F2;
 776                 regs[1] = sr->curr.reg + UNW_REG_R4;
 777                 regs[2] = sr->curr.reg + UNW_REG_B1;
 778
 779                 for (t = 0; t < sr->region_len; ++t) {
 780                         if ((t & 3) == 0)
 781                                 mask = *cp++;
 782                         kind = (mask >> 2*(3-(t & 3))) & 3;
 783                         if (kind > 0)
 784                                 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 785                                                 sr->region_start + t);
 786                 }
 787         }
 788         /*
 789          * Next, lay out the memory stack spill area:
 790          */
 791         if (sr->any_spills) {
 792                 off = sr->spill_offset;
 793                 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 794                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 795                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 796         }
 797 }
 798
 799 /*
 800  * Region header descriptors.
 801  */
 802
 803 static void
 804 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 805                struct unw_state_record *sr)
 806 {
 807         int i, region_start;
 808
 809         if (!(sr->in_body || sr->first_region))
 810                 finish_prologue(sr);
 811         sr->first_region = 0;
 812
 813         /* check if we're done: */
 814         if (sr->when_target < sr->region_start + sr->region_len) {
 815                 sr->done = 1;
 816                 return;
 817         }
 818
 819         region_start = sr->region_start + sr->region_len;
 820
 821         for (i = 0; i < sr->epilogue_count; ++i)
 822                 pop(sr);
 823         sr->epilogue_count = 0;
 824         sr->epilogue_start = UNW_WHEN_NEVER;
 825
 826         sr->region_start = region_start;
 827         sr->region_len = rlen;
 828         sr->in_body = body;
 829
 830         if (!body) {
 831                 push(sr);
 832
 833                 for (i = 0; i < 4; ++i) {
 834                         if (mask & 0x8)
 835                                 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 836                                         sr->region_start + sr->region_len - 1, grsave++);
 837                         mask <<= 1;
 838                 }
 839                 sr->gr_save_loc = grsave;
 840                 sr->any_spills = 0;
 841                 sr->imask = 0;
 842                 sr->spill_offset = 0x10;        /* default to psp+16 */
 843         }
 844 }
 845
 846 /*
 847  * Prologue descriptors.
 848  */
 849
 850 static inline void
 851 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 852 {
 853         if (abi == 3 && context == 'i') {
 854                 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 855                 UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __FUNCTION__);
 856         }
 857         else
 858                 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 859                                 __FUNCTION__, abi, context);
 860 }
 861
 862 static inline void
 863 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 864 {
 865         int i;
 866
 867         for (i = 0; i < 5; ++i) {
 868                 if (brmask & 1)
 869                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 870                                 sr->region_start + sr->region_len - 1, gr++);
 871                 brmask >>= 1;
 872         }
 873 }
 874
 875 static inline void
 876 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 877 {
 878         int i;
 879
 880         for (i = 0; i < 5; ++i) {
 881                 if (brmask & 1) {
 882                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 883                                 sr->region_start + sr->region_len - 1, 0);
 884                         sr->any_spills = 1;
 885                 }
 886                 brmask >>= 1;
 887         }
 888 }
 889
 890 static inline void
 891 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 892 {
 893         int i;
 894
 895         for (i = 0; i < 4; ++i) {
 896                 if ((grmask & 1) != 0) {
 897                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 898                                 sr->region_start + sr->region_len - 1, 0);
 899                         sr->any_spills = 1;
 900                 }
 901                 grmask >>= 1;
 902         }
 903         for (i = 0; i < 20; ++i) {
 904                 if ((frmask & 1) != 0) {
 905                         int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 906                         set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 907                                 sr->region_start + sr->region_len - 1, 0);
 908                         sr->any_spills = 1;
 909                 }
 910                 frmask >>= 1;
 911         }
 912 }
 913
 914 static inline void
 915 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 916 {
 917         int i;
 918
 919         for (i = 0; i < 4; ++i) {
 920                 if ((frmask & 1) != 0) {
 921                         set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 922                                 sr->region_start + sr->region_len - 1, 0);
 923                         sr->any_spills = 1;
 924                 }
 925                 frmask >>= 1;
 926         }
 927 }
 928
 929 static inline void
 930 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 931 {
 932         int i;
 933
 934         for (i = 0; i < 4; ++i) {
 935                 if ((grmask & 1) != 0)
 936                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 937                                 sr->region_start + sr->region_len - 1, gr++);
 938                 grmask >>= 1;
 939         }
 940 }
 941
 942 static inline void
 943 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 944 {
 945         int i;
 946
 947         for (i = 0; i < 4; ++i) {
 948                 if ((grmask & 1) != 0) {
 949                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 950                                 sr->region_start + sr->region_len - 1, 0);
 951                         sr->any_spills = 1;
 952                 }
 953                 grmask >>= 1;
 954         }
 955 }
 956
 957 static inline void
 958 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 959 {
 960         set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 961                 sr->region_start + MIN((int)t, sr->region_len - 1), 16*size);
 962 }
 963
 964 static inline void
 965 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 966 {
 967         sr->curr.reg[UNW_REG_PSP].when = sr->region_start + MIN((int)t, sr->region_len - 1);
 968 }
 969
 970 static inline void
 971 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 972 {
 973         set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 974 }
 975
 976 static inline void
 977 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 978 {
 979         set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 980                 0x10 - 4*pspoff);
 981 }
 982
 983 static inline void
 984 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 985 {
 986         set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 987                 4*spoff);
 988 }
 989
 990 static inline void
 991 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 992 {
 993         sr->return_link_reg = dst;
 994 }
 995
 996 static inline void
 997 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
 998 {
 999         struct unw_reg_info *reg = sr->curr.reg + regnum;
1000
1001         if (reg->where == UNW_WHERE_NONE)
1002                 reg->where = UNW_WHERE_GR_SAVE;
1003         reg->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1004 }
1005
1006 static inline void
1007 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1008 {
1009         sr->spill_offset = 0x10 - 4*pspoff;
1010 }
1011
1012 static inline unsigned char *
1013 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1014 {
1015         sr->imask = imaskp;
1016         return imaskp + (2*sr->region_len + 7)/8;
1017 }
1018
1019 /*
1020  * Body descriptors.
1021  */
1022 static inline void
1023 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1024 {
1025         sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1026         sr->epilogue_count = ecount + 1;
1027 }
1028
1029 static inline void
1030 desc_copy_state (unw_word label, struct unw_state_record *sr)
1031 {
1032         struct unw_labeled_state *ls;
1033
1034         for (ls = sr->labeled_states; ls; ls = ls->next) {
1035                 if (ls->label == label) {
1036                         free_state_stack(&sr->curr);
1037                         memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1038                         sr->curr.next = dup_state_stack(ls->saved_state.next);
1039                         return;
1040                 }
1041         }
1042         printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1043 }
1044
1045 static inline void
1046 desc_label_state (unw_word label, struct unw_state_record *sr)
1047 {
1048         struct unw_labeled_state *ls;
1049
1050         ls = alloc_labeled_state();
1051         if (!ls) {
1052                 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1053                 return;
1054         }
1055         ls->label = label;
1056         memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1057         ls->saved_state.next = dup_state_stack(sr->curr.next);
1058
1059         /* insert into list of labeled states: */
1060         ls->next = sr->labeled_states;
1061         sr->labeled_states = ls;
1062 }
1063
1064 /*
1065  * General descriptors.
1066  */
1067
1068 static inline int
1069 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1070 {
1071         if (sr->when_target <= sr->region_start + MIN((int)t, sr->region_len - 1))
1072                 return 0;
1073         if (qp > 0) {
1074                 if ((sr->pr_val & (1UL << qp)) == 0)
1075                         return 0;
1076                 sr->pr_mask |= (1UL << qp);
1077         }
1078         return 1;
1079 }
1080
1081 static inline void
1082 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1083 {
1084         struct unw_reg_info *r;
1085
1086         if (!desc_is_active(qp, t, sr))
1087                 return;
1088
1089         r = sr->curr.reg + decode_abreg(abreg, 0);
1090         r->where = UNW_WHERE_NONE;
1091         r->when = UNW_WHEN_NEVER;
1092         r->val = 0;
1093 }
1094
1095 static inline void
1096 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1097                      unsigned char ytreg, struct unw_state_record *sr)
1098 {
1099         enum unw_where where = UNW_WHERE_GR;
1100         struct unw_reg_info *r;
1101
1102         if (!desc_is_active(qp, t, sr))
1103                 return;
1104
1105         if (x)
1106                 where = UNW_WHERE_BR;
1107         else if (ytreg & 0x80)
1108                 where = UNW_WHERE_FR;
1109
1110         r = sr->curr.reg + decode_abreg(abreg, 0);
1111         r->where = where;
1112         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1113         r->val = (ytreg & 0x7f);
1114 }
1115
1116 static inline void
1117 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1118                      struct unw_state_record *sr)
1119 {
1120         struct unw_reg_info *r;
1121
1122         if (!desc_is_active(qp, t, sr))
1123                 return;
1124
1125         r = sr->curr.reg + decode_abreg(abreg, 1);
1126         r->where = UNW_WHERE_PSPREL;
1127         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1128         r->val = 0x10 - 4*pspoff;
1129 }
1130
1131 static inline void
1132 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1133                        struct unw_state_record *sr)
1134 {
1135         struct unw_reg_info *r;
1136
1137         if (!desc_is_active(qp, t, sr))
1138                 return;
1139
1140         r = sr->curr.reg + decode_abreg(abreg, 1);
1141         r->where = UNW_WHERE_SPREL;
1142         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1143         r->val = 4*spoff;
1144 }
1145
1146 #define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1147                                                        code);
1148
1149 /*
1150  * region headers:
1151  */
1152 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1153 #define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1154 /*
1155  * prologue descriptors:
1156  */
1157 #define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1158 #define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1159 #define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1160 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1161 #define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1162 #define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1163 #define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1164 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1165 #define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1166 #define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1167 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1168 #define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1169 #define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1170 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1171 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1172 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1173 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1174 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1175 #define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1176 #define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1177 #define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1178 /*
1179  * body descriptors:
1180  */
1181 #define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1182 #define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1183 #define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1184 /*
1185  * general unwind descriptors:
1186  */
1187 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1188 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1189 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1190 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1191 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1192 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1193 #define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1194 #define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1195
1196 #include "unwind_decoder.c"
1197
1198 \f
1199 /* Unwind scripts. */
1200
1201 static inline unw_hash_index_t
1202 hash (unsigned long ip)
1203 {
1204 #       define hashmagic        0x9e3779b97f4a7c16      /* based on (sqrt(5)/2-1)*2^64 */
1205
1206         return (ip >> 4)*hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1207 #undef hashmagic
1208 }
1209
1210 static inline long
1211 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1212 {
1213         read_lock(&script->lock);
1214         if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1215                 /* keep the read lock... */
1216                 return 1;
1217         read_unlock(&script->lock);
1218         return 0;
1219 }
1220
1221 static inline struct unw_script *
1222 script_lookup (struct unw_frame_info *info)
1223 {
1224         struct unw_script *script = unw.cache + info->hint;
1225         unsigned short index;
1226         unsigned long ip, pr;
1227
1228         if (UNW_DEBUG_ON(0))
1229                 return 0;       /* Always regenerate scripts in debug mode */
1230
1231         STAT(++unw.stat.cache.lookups);
1232
1233         ip = info->ip;
1234         pr = info->pr;
1235
1236         if (cache_match(script, ip, pr)) {
1237                 STAT(++unw.stat.cache.hinted_hits);
1238                 return script;
1239         }
1240
1241         index = unw.hash[hash(ip)];
1242         if (index >= UNW_CACHE_SIZE)
1243                 return 0;
1244
1245         script = unw.cache + index;
1246         while (1) {
1247                 if (cache_match(script, ip, pr)) {
1248                         /* update hint; no locking required as single-word writes are atomic */
1249                         STAT(++unw.stat.cache.normal_hits);
1250                         unw.cache[info->prev_script].hint = script - unw.cache;
1251                         return script;
1252                 }
1253                 if (script->coll_chain >= UNW_HASH_SIZE)
1254                         return 0;
1255                 script = unw.cache + script->coll_chain;
1256                 STAT(++unw.stat.cache.collision_chain_traversals);
1257         }
1258 }
1259
1260 /*
1261  * On returning, a write lock for the SCRIPT is still being held.
1262  */
1263 static inline struct unw_script *
1264 script_new (unsigned long ip)
1265 {
1266         struct unw_script *script, *prev, *tmp;
1267         unw_hash_index_t index;
1268         unsigned long flags;
1269         unsigned short head;
1270
1271         STAT(++unw.stat.script.news);
1272
1273         /*
1274          * Can't (easily) use cmpxchg() here because of ABA problem
1275          * that is intrinsic in cmpxchg()...
1276          */
1277         spin_lock_irqsave(&unw.lock, flags);
1278         {
1279                 head = unw.lru_head;
1280                 script = unw.cache + head;
1281                 unw.lru_head = script->lru_chain;
1282         }
1283         spin_unlock(&unw.lock);
1284
1285         /*
1286          * We'd deadlock here if we interrupted a thread that is holding a read lock on
1287          * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1288          * alternative would be to disable interrupts whenever we hold a read-lock, but
1289          * that seems silly.
1290          */
1291         if (!write_trylock(&script->lock))
1292                 return NULL;
1293
1294         spin_lock(&unw.lock);
1295         {
1296                 /* re-insert script at the tail of the LRU chain: */
1297                 unw.cache[unw.lru_tail].lru_chain = head;
1298                 unw.lru_tail = head;
1299
1300                 /* remove the old script from the hash table (if it's there): */
1301                 if (script->ip) {
1302                         index = hash(script->ip);
1303                         tmp = unw.cache + unw.hash[index];
1304                         prev = 0;
1305                         while (1) {
1306                                 if (tmp == script) {
1307                                         if (prev)
1308                                                 prev->coll_chain = tmp->coll_chain;
1309                                         else
1310                                                 unw.hash[index] = tmp->coll_chain;
1311                                         break;
1312                                 } else
1313                                         prev = tmp;
1314                                 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1315                                 /* old script wasn't in the hash-table */
1316                                         break;
1317                                 tmp = unw.cache + tmp->coll_chain;
1318                         }
1319                 }
1320
1321                 /* enter new script in the hash table */
1322                 index = hash(ip);
1323                 script->coll_chain = unw.hash[index];
1324                 unw.hash[index] = script - unw.cache;
1325
1326                 script->ip = ip;        /* set new IP while we're holding the locks */
1327
1328                 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1329         }
1330         spin_unlock_irqrestore(&unw.lock, flags);
1331
1332         script->flags = 0;
1333         script->hint = 0;
1334         script->count = 0;
1335         return script;
1336 }
1337
1338 static void
1339 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1340 {
1341         script->pr_mask = sr->pr_mask;
1342         script->pr_val = sr->pr_val;
1343         /*
1344          * We could down-grade our write-lock on script->lock here but
1345          * the rwlock API doesn't offer atomic lock downgrading, so
1346          * we'll just keep the write-lock and release it later when
1347          * we're done using the script.
1348          */
1349 }
1350
1351 static inline void
1352 script_emit (struct unw_script *script, struct unw_insn insn)
1353 {
1354         if (script->count >= UNW_MAX_SCRIPT_LEN) {
1355                 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1356                         __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1357                 return;
1358         }
1359         script->insn[script->count++] = insn;
1360 }
1361
1362 static inline void
1363 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1364 {
1365         struct unw_reg_info *r = sr->curr.reg + i;
1366         enum unw_insn_opcode opc;
1367         struct unw_insn insn;
1368         unsigned long val = 0;
1369
1370         switch (r->where) {
1371               case UNW_WHERE_GR:
1372                 if (r->val >= 32) {
1373                         /* register got spilled to a stacked register */
1374                         opc = UNW_INSN_SETNAT_TYPE;
1375                         val = UNW_NAT_REGSTK;
1376                 } else
1377                         /* register got spilled to a scratch register */
1378                         opc = UNW_INSN_SETNAT_MEMSTK;
1379                 break;
1380
1381               case UNW_WHERE_FR:
1382                 opc = UNW_INSN_SETNAT_TYPE;
1383                 val = UNW_NAT_VAL;
1384                 break;
1385
1386               case UNW_WHERE_BR:
1387                 opc = UNW_INSN_SETNAT_TYPE;
1388                 val = UNW_NAT_NONE;
1389                 break;
1390
1391               case UNW_WHERE_PSPREL:
1392               case UNW_WHERE_SPREL:
1393                 opc = UNW_INSN_SETNAT_MEMSTK;
1394                 break;
1395
1396               default:
1397                 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1398                            __FUNCTION__, r->where);
1399                 return;
1400         }
1401         insn.opc = opc;
1402         insn.dst = unw.preg_index[i];
1403         insn.val = val;
1404         script_emit(script, insn);
1405 }
1406
1407 static void
1408 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1409 {
1410         struct unw_reg_info *r = sr->curr.reg + i;
1411         enum unw_insn_opcode opc;
1412         unsigned long val, rval;
1413         struct unw_insn insn;
1414         long need_nat_info;
1415
1416         if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1417                 return;
1418
1419         opc = UNW_INSN_MOVE;
1420         val = rval = r->val;
1421         need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1422
1423         switch (r->where) {
1424               case UNW_WHERE_GR:
1425                 if (rval >= 32) {
1426                         opc = UNW_INSN_MOVE_STACKED;
1427                         val = rval - 32;
1428                 } else if (rval >= 4 && rval <= 7) {
1429                         if (need_nat_info) {
1430                                 opc = UNW_INSN_MOVE2;
1431                                 need_nat_info = 0;
1432                         }
1433                         val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1434                 } else if (rval == 0) {
1435                         opc = UNW_INSN_MOVE_CONST;
1436                         val = 0;
1437                 } else {
1438                         /* register got spilled to a scratch register */
1439                         opc = UNW_INSN_MOVE_SCRATCH;
1440                         val = pt_regs_off(rval);
1441                 }
1442                 break;
1443
1444               case UNW_WHERE_FR:
1445                 if (rval <= 5)
1446                         val = unw.preg_index[UNW_REG_F2  + (rval -  2)];
1447                 else if (rval >= 16 && rval <= 31)
1448                         val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1449                 else {
1450                         opc = UNW_INSN_MOVE_SCRATCH;
1451                         if (rval <= 11)
1452                                 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1453                         else
1454                                 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1455                                            __FUNCTION__, rval);
1456                 }
1457                 break;
1458
1459               case UNW_WHERE_BR:
1460                 if (rval >= 1 && rval <= 5)
1461                         val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1462                 else {
1463                         opc = UNW_INSN_MOVE_SCRATCH;
1464                         if (rval == 0)
1465                                 val = offsetof(struct pt_regs, b0);
1466                         else if (rval == 6)
1467                                 val = offsetof(struct pt_regs, b6);
1468                         else
1469                                 val = offsetof(struct pt_regs, b7);
1470                 }
1471                 break;
1472
1473               case UNW_WHERE_SPREL:
1474                 opc = UNW_INSN_ADD_SP;
1475                 break;
1476
1477               case UNW_WHERE_PSPREL:
1478                 opc = UNW_INSN_ADD_PSP;
1479                 break;
1480
1481               default:
1482                 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1483                            __FUNCTION__, i, r->where);
1484                 break;
1485         }
1486         insn.opc = opc;
1487         insn.dst = unw.preg_index[i];
1488         insn.val = val;
1489         script_emit(script, insn);
1490         if (need_nat_info)
1491                 emit_nat_info(sr, i, script);
1492
1493         if (i == UNW_REG_PSP) {
1494                 /*
1495                  * info->psp must contain the _value_ of the previous
1496                  * sp, not it's save location.  We get this by
1497                  * dereferencing the value we just stored in
1498                  * info->psp:
1499                  */
1500                 insn.opc = UNW_INSN_LOAD;
1501                 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1502                 script_emit(script, insn);
1503         }
1504 }
1505
1506 static inline const struct unw_table_entry *
1507 lookup (struct unw_table *table, unsigned long rel_ip)
1508 {
1509         const struct unw_table_entry *e = 0;
1510         unsigned long lo, hi, mid;
1511
1512         /* do a binary search for right entry: */
1513         for (lo = 0, hi = table->length; lo < hi; ) {
1514                 mid = (lo + hi) / 2;
1515                 e = &table->array[mid];
1516                 if (rel_ip < e->start_offset)
1517                         hi = mid;
1518                 else if (rel_ip >= e->end_offset)
1519                         lo = mid + 1;
1520                 else
1521                         break;
1522         }
1523         if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1524                 return NULL;
1525         return e;
1526 }
1527
1528 /*
1529  * Build an unwind script that unwinds from state OLD_STATE to the
1530  * entrypoint of the function that called OLD_STATE.
1531  */
1532 static inline struct unw_script *
1533 build_script (struct unw_frame_info *info)
1534 {
1535         const struct unw_table_entry *e = 0;
1536         struct unw_script *script = 0;
1537         struct unw_labeled_state *ls, *next;
1538         unsigned long ip = info->ip;
1539         struct unw_state_record sr;
1540         struct unw_table *table;
1541         struct unw_reg_info *r;
1542         struct unw_insn insn;
1543         u8 *dp, *desc_end;
1544         u64 hdr;
1545         int i;
1546         STAT(unsigned long start, parse_start;)
1547
1548         STAT(++unw.stat.script.builds; start = ia64_get_itc());
1549
1550         /* build state record */
1551         memset(&sr, 0, sizeof(sr));
1552         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1553                 r->when = UNW_WHEN_NEVER;
1554         sr.pr_val = info->pr;
1555
1556         UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1557         script = script_new(ip);
1558         if (!script) {
1559                 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __FUNCTION__);
1560                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1561                 return 0;
1562         }
1563         unw.cache[info->prev_script].hint = script - unw.cache;
1564
1565         /* search the kernels and the modules' unwind tables for IP: */
1566
1567         STAT(parse_start = ia64_get_itc());
1568
1569         for (table = unw.tables; table; table = table->next) {
1570                 if (ip >= table->start && ip < table->end) {
1571                         e = lookup(table, ip - table->segment_base);
1572                         break;
1573                 }
1574         }
1575         if (!e) {
1576                 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1577                 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1578                         __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1579                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1580                 sr.curr.reg[UNW_REG_RP].when = -1;
1581                 sr.curr.reg[UNW_REG_RP].val = 0;
1582                 compile_reg(&sr, UNW_REG_RP, script);
1583                 script_finalize(script, &sr);
1584                 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1585                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1586                 return script;
1587         }
1588
1589         sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1590                           + (ip & 0xfUL));
1591         hdr = *(u64 *) (table->segment_base + e->info_offset);
1592         dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1593         desc_end = dp + 8*UNW_LENGTH(hdr);
1594
1595         while (!sr.done && dp < desc_end)
1596                 dp = unw_decode(dp, sr.in_body, &sr);
1597
1598         if (sr.when_target > sr.epilogue_start) {
1599                 /*
1600                  * sp has been restored and all values on the memory stack below
1601                  * psp also have been restored.
1602                  */
1603                 sr.curr.reg[UNW_REG_PSP].val = 0;
1604                 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1605                 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1606                 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1607                         if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1608                             || r->where == UNW_WHERE_SPREL)
1609                         {
1610                                 r->val = 0;
1611                                 r->where = UNW_WHERE_NONE;
1612                                 r->when = UNW_WHEN_NEVER;
1613                         }
1614         }
1615
1616         script->flags = sr.flags;
1617
1618         /*
1619          * If RP did't get saved, generate entry for the return link
1620          * register.
1621          */
1622         if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1623                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1624                 sr.curr.reg[UNW_REG_RP].when = -1;
1625                 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1626                 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1627                            __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1628                            sr.curr.reg[UNW_REG_RP].val);
1629         }
1630
1631 #ifdef UNW_DEBUG
1632         UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1633                 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1634         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1635                 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1636                         UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1637                         switch (r->where) {
1638                               case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1639                               case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1640                               case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1641                               case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1642                               case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1643                               case UNW_WHERE_NONE:
1644                                 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1645                                 break;
1646
1647                               default:
1648                                 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1649                                 break;
1650                         }
1651                         UNW_DPRINT(1, "\t\t%d\n", r->when);
1652                 }
1653         }
1654 #endif
1655
1656         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1657
1658         /* translate state record into unwinder instructions: */
1659
1660         /*
1661          * First, set psp if we're dealing with a fixed-size frame;
1662          * subsequent instructions may depend on this value.
1663          */
1664         if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1665             && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1666             && sr.curr.reg[UNW_REG_PSP].val != 0) {
1667                 /* new psp is sp plus frame size */
1668                 insn.opc = UNW_INSN_ADD;
1669                 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1670                 insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1671                 script_emit(script, insn);
1672         }
1673
1674         /* determine where the primary UNaT is: */
1675         if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1676                 i = UNW_REG_PRI_UNAT_MEM;
1677         else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1678                 i = UNW_REG_PRI_UNAT_GR;
1679         else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1680                 i = UNW_REG_PRI_UNAT_MEM;
1681         else
1682                 i = UNW_REG_PRI_UNAT_GR;
1683
1684         compile_reg(&sr, i, script);
1685
1686         for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1687                 compile_reg(&sr, i, script);
1688
1689         /* free labeled register states & stack: */
1690
1691         STAT(parse_start = ia64_get_itc());
1692         for (ls = sr.labeled_states; ls; ls = next) {
1693                 next = ls->next;
1694                 free_state_stack(&ls->saved_state);
1695                 free_labeled_state(ls);
1696         }
1697         free_state_stack(&sr.curr);
1698         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1699
1700         script_finalize(script, &sr);
1701         STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1702         return script;
1703 }
1704
1705 /*
1706  * Apply the unwinding actions represented by OPS and update SR to
1707  * reflect the state that existed upon entry to the function that this
1708  * unwinder represents.
1709  */
1710 static inline void
1711 run_script (struct unw_script *script, struct unw_frame_info *state)
1712 {
1713         struct unw_insn *ip, *limit, next_insn;
1714         unsigned long opc, dst, val, off;
1715         unsigned long *s = (unsigned long *) state;
1716         STAT(unsigned long start;)
1717
1718         STAT(++unw.stat.script.runs; start = ia64_get_itc());
1719         state->flags = script->flags;
1720         ip = script->insn;
1721         limit = script->insn + script->count;
1722         next_insn = *ip;
1723
1724         while (ip++ < limit) {
1725                 opc = next_insn.opc;
1726                 dst = next_insn.dst;
1727                 val = next_insn.val;
1728                 next_insn = *ip;
1729
1730           redo:
1731                 switch (opc) {
1732                       case UNW_INSN_ADD:
1733                         s[dst] += val;
1734                         break;
1735
1736                       case UNW_INSN_MOVE2:
1737                         if (!s[val])
1738                                 goto lazy_init;
1739                         s[dst+1] = s[val+1];
1740                         s[dst] = s[val];
1741                         break;
1742
1743                       case UNW_INSN_MOVE:
1744                         if (!s[val])
1745                                 goto lazy_init;
1746                         s[dst] = s[val];
1747                         break;
1748
1749                       case UNW_INSN_MOVE_SCRATCH:
1750                         if (state->pt) {
1751                                 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1752                         } else {
1753                                 s[dst] = 0;
1754                                 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1755                                            __FUNCTION__, dst, val);
1756                         }
1757                         break;
1758
1759                       case UNW_INSN_MOVE_CONST:
1760                         if (val == 0)
1761                                 s[dst] = (unsigned long) &unw.r0;
1762                         else {
1763                                 s[dst] = 0;
1764                                 UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1765                                            __FUNCTION__, val);
1766                         }
1767                         break;
1768
1769
1770                       case UNW_INSN_MOVE_STACKED:
1771                         s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1772                                                                     val);
1773                         break;
1774
1775                       case UNW_INSN_ADD_PSP:
1776                         s[dst] = state->psp + val;
1777                         break;
1778
1779                       case UNW_INSN_ADD_SP:
1780                         s[dst] = state->sp + val;
1781                         break;
1782
1783                       case UNW_INSN_SETNAT_MEMSTK:
1784                         if (!state->pri_unat_loc)
1785                                 state->pri_unat_loc = &state->sw->ar_unat;
1786                         /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1787                         s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1788                         break;
1789
1790                       case UNW_INSN_SETNAT_TYPE:
1791                         s[dst+1] = val;
1792                         break;
1793
1794                       case UNW_INSN_LOAD:
1795 #ifdef UNW_DEBUG
1796                         if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1797                             || s[val] < TASK_SIZE)
1798                         {
1799                                 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1800                                            __FUNCTION__, s[val]);
1801                                 break;
1802                         }
1803 #endif
1804                         s[dst] = *(unsigned long *) s[val];
1805                         break;
1806                 }
1807         }
1808         STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1809         return;
1810
1811   lazy_init:
1812         off = unw.sw_off[val];
1813         s[val] = (unsigned long) state->sw + off;
1814         if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1815                 /*
1816                  * We're initializing a general register: init NaT info, too.  Note that
1817                  * the offset is a multiple of 8 which gives us the 3 bits needed for
1818                  * the type field.
1819                  */
1820                 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1821         goto redo;
1822 }
1823
1824 static int
1825 find_save_locs (struct unw_frame_info *info)
1826 {
1827         int have_write_lock = 0;
1828         struct unw_script *scr;
1829
1830         if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1831                 /* don't let obviously bad addresses pollute the cache */
1832                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1833                 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1834                 info->rp_loc = 0;
1835                 return -1;
1836         }
1837
1838         scr = script_lookup(info);
1839         if (!scr) {
1840                 scr = build_script(info);
1841                 if (!scr) {
1842                         UNW_DPRINT(0,
1843                                    "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1844                                    __FUNCTION__, info->ip);
1845                         return -1;
1846                 }
1847                 have_write_lock = 1;
1848         }
1849         info->hint = scr->hint;
1850         info->prev_script = scr - unw.cache;
1851
1852         run_script(scr, info);
1853
1854         if (have_write_lock)
1855                 write_unlock(&scr->lock);
1856         else
1857                 read_unlock(&scr->lock);
1858         return 0;
1859 }
1860
1861 int
1862 unw_unwind (struct unw_frame_info *info)
1863 {
1864         unsigned long prev_ip, prev_sp, prev_bsp;
1865         unsigned long ip, pr, num_regs;
1866         STAT(unsigned long start, flags;)
1867         int retval;
1868
1869         STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1870
1871         prev_ip = info->ip;
1872         prev_sp = info->sp;
1873         prev_bsp = info->bsp;
1874
1875         /* restore the ip */
1876         if (!info->rp_loc) {
1877                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1878                 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1879                            __FUNCTION__, info->ip);
1880                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1881                 return -1;
1882         }
1883         ip = info->ip = *info->rp_loc;
1884         if (ip < GATE_ADDR) {
1885                 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1886                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1887                 return -1;
1888         }
1889
1890         /* restore the cfm: */
1891         if (!info->pfs_loc) {
1892                 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1893                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1894                 return -1;
1895         }
1896         info->cfm_loc = info->pfs_loc;
1897
1898         /* restore the bsp: */
1899         pr = info->pr;
1900         num_regs = 0;
1901         if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1902                 info->pt = info->sp + 16;
1903                 if ((pr & (1UL << pNonSys)) != 0)
1904                         num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1905                 info->pfs_loc =
1906                         (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1907                 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1908         } else
1909                 num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1910         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1911         if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1912                 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1913                         __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1914                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1915                 return -1;
1916         }
1917
1918         /* restore the sp: */
1919         info->sp = info->psp;
1920         if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1921                 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1922                         __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1923                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1924                 return -1;
1925         }
1926
1927         if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1928                 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1929                            __FUNCTION__, ip);
1930                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1931                 return -1;
1932         }
1933
1934         /* as we unwind, the saved ar.unat becomes the primary unat: */
1935         info->pri_unat_loc = info->unat_loc;
1936
1937         /* finally, restore the predicates: */
1938         unw_get_pr(info, &info->pr);
1939
1940         retval = find_save_locs(info);
1941         STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1942         return retval;
1943 }
1944 EXPORT_SYMBOL(unw_unwind);
1945
1946 int
1947 unw_unwind_to_user (struct unw_frame_info *info)
1948 {
1949         unsigned long ip;
1950
1951         while (unw_unwind(info) >= 0) {
1952                 if (unw_get_rp(info, &ip) < 0) {
1953                         unw_get_ip(info, &ip);
1954                         UNW_DPRINT(0, "unwind.%s: failed to read return pointer (ip=0x%lx)\n",
1955                                    __FUNCTION__, ip);
1956                         return -1;
1957                 }
1958                 if (ip < FIXADDR_USER_END)
1959                         return 0;
1960         }
1961         unw_get_ip(info, &ip);
1962         UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n", __FUNCTION__, ip);
1963         return -1;
1964 }
1965 EXPORT_SYMBOL(unw_unwind_to_user);
1966
1967 static void
1968 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1969                  struct switch_stack *sw, unsigned long stktop)
1970 {
1971         unsigned long rbslimit, rbstop, stklimit;
1972         STAT(unsigned long start, flags;)
1973
1974         STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1975
1976         /*
1977          * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1978          * don't want to do that because it would be slow as each preserved register would
1979          * have to be processed.  Instead, what we do here is zero out the frame info and
1980          * start the unwind process at the function that created the switch_stack frame.
1981          * When a preserved value in switch_stack needs to be accessed, run_script() will
1982          * initialize the appropriate pointer on demand.
1983          */
1984         memset(info, 0, sizeof(*info));
1985
1986         rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1987         rbstop   = sw->ar_bspstore;
1988         if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1989                 rbstop = rbslimit;
1990
1991         stklimit = (unsigned long) t + IA64_STK_OFFSET;
1992         if (stktop <= rbstop)
1993                 stktop = rbstop;
1994
1995         info->regstk.limit = rbslimit;
1996         info->regstk.top   = rbstop;
1997         info->memstk.limit = stklimit;
1998         info->memstk.top   = stktop;
1999         info->task = t;
2000         info->sw  = sw;
2001         info->sp = info->psp = stktop;
2002         info->pr = sw->pr;
2003         UNW_DPRINT(3, "unwind.%s:\n"
2004                    "  task   0x%lx\n"
2005                    "  rbs = [0x%lx-0x%lx)\n"
2006                    "  stk = [0x%lx-0x%lx)\n"
2007                    "  pr     0x%lx\n"
2008                    "  sw     0x%lx\n"
2009                    "  sp     0x%lx\n",
2010                    __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2011                    info->pr, (unsigned long) info->sw, info->sp);
2012         STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2013 }
2014
2015 void
2016 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
2017                             struct pt_regs *pt, struct switch_stack *sw)
2018 {
2019         unsigned long sof;
2020
2021         init_frame_info(info, t, sw, pt->r12);
2022         info->cfm_loc = &pt->cr_ifs;
2023         info->unat_loc = &pt->ar_unat;
2024         info->pfs_loc = &pt->ar_pfs;
2025         sof = *info->cfm_loc & 0x7f;
2026         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
2027         info->ip = pt->cr_iip + ia64_psr(pt)->ri;
2028         info->pt = (unsigned long) pt;
2029         UNW_DPRINT(3, "unwind.%s:\n"
2030                    "  bsp    0x%lx\n"
2031                    "  sof    0x%lx\n"
2032                    "  ip     0x%lx\n",
2033                    __FUNCTION__, info->bsp, sof, info->ip);
2034         find_save_locs(info);
2035 }
2036
2037 void
2038 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2039 {
2040         unsigned long sol;
2041
2042         init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2043         info->cfm_loc = &sw->ar_pfs;
2044         sol = (*info->cfm_loc >> 7) & 0x7f;
2045         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2046         info->ip = sw->b0;
2047         UNW_DPRINT(3, "unwind.%s:\n"
2048                    "  bsp    0x%lx\n"
2049                    "  sol    0x%lx\n"
2050                    "  ip     0x%lx\n",
2051                    __FUNCTION__, info->bsp, sol, info->ip);
2052         find_save_locs(info);
2053 }
2054
2055 void
2056 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2057 {
2058         struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2059
2060         UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
2061         unw_init_frame_info(info, t, sw);
2062 }
2063 EXPORT_SYMBOL(unw_init_from_blocked_task);
2064
2065 static void
2066 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2067                    unsigned long gp, const void *table_start, const void *table_end)
2068 {
2069         const struct unw_table_entry *start = table_start, *end = table_end;
2070
2071         table->name = name;
2072         table->segment_base = segment_base;
2073         table->gp = gp;
2074         table->start = segment_base + start[0].start_offset;
2075         table->end = segment_base + end[-1].end_offset;
2076         table->array = start;
2077         table->length = end - start;
2078 }
2079
2080 void *
2081 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2082                       const void *table_start, const void *table_end)
2083 {
2084         const struct unw_table_entry *start = table_start, *end = table_end;
2085         struct unw_table *table;
2086         unsigned long flags;
2087
2088         if (end - start <= 0) {
2089                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2090                            __FUNCTION__);
2091                 return 0;
2092         }
2093
2094         table = kmalloc(sizeof(*table), GFP_USER);
2095         if (!table)
2096                 return 0;
2097
2098         init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2099
2100         spin_lock_irqsave(&unw.lock, flags);
2101         {
2102                 /* keep kernel unwind table at the front (it's searched most commonly): */
2103                 table->next = unw.tables->next;
2104                 unw.tables->next = table;
2105         }
2106         spin_unlock_irqrestore(&unw.lock, flags);
2107
2108         return table;
2109 }
2110
2111 void
2112 unw_remove_unwind_table (void *handle)
2113 {
2114         struct unw_table *table, *prev;
2115         struct unw_script *tmp;
2116         unsigned long flags;
2117         long index;
2118
2119         if (!handle) {
2120                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2121                            __FUNCTION__);
2122                 return;
2123         }
2124
2125         table = handle;
2126         if (table == &unw.kernel_table) {
2127                 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2128                            "no-can-do!\n", __FUNCTION__);
2129                 return;
2130         }
2131
2132         spin_lock_irqsave(&unw.lock, flags);
2133         {
2134                 /* first, delete the table: */
2135
2136                 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2137                         if (prev->next == table)
2138                                 break;
2139                 if (!prev) {
2140                         UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2141                                    __FUNCTION__, (void *) table);
2142                         spin_unlock_irqrestore(&unw.lock, flags);
2143                         return;
2144                 }
2145                 prev->next = table->next;
2146         }
2147         spin_unlock_irqrestore(&unw.lock, flags);
2148
2149         /* next, remove hash table entries for this table */
2150
2151         for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2152                 tmp = unw.cache + unw.hash[index];
2153                 if (unw.hash[index] >= UNW_CACHE_SIZE
2154                     || tmp->ip < table->start || tmp->ip >= table->end)
2155                         continue;
2156
2157                 write_lock(&tmp->lock);
2158                 {
2159                         if (tmp->ip >= table->start && tmp->ip < table->end) {
2160                                 unw.hash[index] = tmp->coll_chain;
2161                                 tmp->ip = 0;
2162                         }
2163                 }
2164                 write_unlock(&tmp->lock);
2165         }
2166
2167         kfree(table);
2168 }
2169
2170 static int __init
2171 create_gate_table (void)
2172 {
2173         const struct unw_table_entry *entry, *start, *end;
2174         unsigned long *lp, segbase = GATE_ADDR;
2175         size_t info_size, size;
2176         char *info;
2177         Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2178         int i;
2179
2180         for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2181                 if (phdr->p_type == PT_IA_64_UNWIND) {
2182                         punw = phdr;
2183                         break;
2184                 }
2185
2186         if (!punw) {
2187                 printk("%s: failed to find gate DSO's unwind table!\n", __FUNCTION__);
2188                 return 0;
2189         }
2190
2191         start = (const struct unw_table_entry *) punw->p_vaddr;
2192         end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2193         size  = 0;
2194
2195         unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2196
2197         for (entry = start; entry < end; ++entry)
2198                 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2199         size += 8;      /* reserve space for "end of table" marker */
2200
2201         unw.gate_table = kmalloc(size, GFP_KERNEL);
2202         if (!unw.gate_table) {
2203                 unw.gate_table_size = 0;
2204                 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __FUNCTION__);
2205                 return 0;
2206         }
2207         unw.gate_table_size = size;
2208
2209         lp = unw.gate_table;
2210         info = (char *) unw.gate_table + size;
2211
2212         for (entry = start; entry < end; ++entry, lp += 3) {
2213                 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2214                 info -= info_size;
2215                 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2216
2217                 lp[0] = segbase + entry->start_offset;          /* start */
2218                 lp[1] = segbase + entry->end_offset;            /* end */
2219                 lp[2] = info - (char *) unw.gate_table;         /* info */
2220         }
2221         *lp = 0;        /* end-of-table marker */
2222         return 0;
2223 }
2224
2225 __initcall(create_gate_table);
2226
2227 void __init
2228 unw_init (void)
2229 {
2230         extern char __gp[];
2231         extern void unw_hash_index_t_is_too_narrow (void);
2232         long i, off;
2233
2234         if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2235                 unw_hash_index_t_is_too_narrow();
2236
2237         unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2238         unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2239         unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2240         unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2241         unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2242         unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2243         unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2244         unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2245         for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2246                 unw.sw_off[unw.preg_index[i]] = off;
2247         for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2248                 unw.sw_off[unw.preg_index[i]] = off;
2249         for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2250                 unw.sw_off[unw.preg_index[i]] = off;
2251         for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2252                 unw.sw_off[unw.preg_index[i]] = off;
2253
2254         for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2255                 if (i > 0)
2256                         unw.cache[i].lru_chain = (i - 1);
2257                 unw.cache[i].coll_chain = -1;
2258                 unw.cache[i].lock = RW_LOCK_UNLOCKED;
2259         }
2260         unw.lru_head = UNW_CACHE_SIZE - 1;
2261         unw.lru_tail = 0;
2262
2263         init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2264                           __start_unwind, __end_unwind);
2265 }
2266
2267 /*
2268  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2269  *
2270  *      This system call has been deprecated.  The new and improved way to get
2271  *      at the kernel's unwind info is via the gate DSO.  The address of the
2272  *      ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2273  *
2274  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2275  *
2276  * This system call copies the unwind data into the buffer pointed to by BUF and returns
2277  * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2278  * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2279  * unwind data.
2280  *
2281  * The first portion of the unwind data contains an unwind table and rest contains the
2282  * associated unwind info (in no particular order).  The unwind table consists of a table
2283  * of entries of the form:
2284  *
2285  *      u64 start;      (64-bit address of start of function)
2286  *      u64 end;        (64-bit address of start of function)
2287  *      u64 info;       (BUF-relative offset to unwind info)
2288  *
2289  * The end of the unwind table is indicated by an entry with a START address of zero.
2290  *
2291  * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2292  * on the format of the unwind info.
2293  *
2294  * ERRORS
2295  *      EFAULT  BUF points outside your accessible address space.
2296  */
2297 asmlinkage long
2298 sys_getunwind (void *buf, size_t buf_size)
2299 {
2300         if (buf && buf_size >= unw.gate_table_size)
2301                 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2302                         return -EFAULT;
2303         return unw.gate_table_size;
2304 }