arch/sparc64/kernel/unaligned.c

   1 /* $Id: unaligned.c,v 1.24 2002/02/09 19:49:31 davem Exp $
   2  * unaligned.c: Unaligned load/store trap handling with special
   3  *              cases for the kernel to do them more quickly.
   4  *
   5  * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
   6  * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   7  */
   8
   9
  10 #include <linux/kernel.h>
  11 #include <linux/sched.h>
  12 #include <linux/mm.h>
  13 #include <linux/module.h>
  14 #include <asm/asi.h>
  15 #include <asm/ptrace.h>
  16 #include <asm/pstate.h>
  17 #include <asm/processor.h>
  18 #include <asm/system.h>
  19 #include <asm/uaccess.h>
  20 #include <linux/smp.h>
  21 #include <linux/smp_lock.h>
  22 #include <asm/fpumacro.h>
  23 #include <asm/bitops.h>
  24
  25 /* #define DEBUG_MNA */
  26
  27 enum direction {
  28         load,    /* ld, ldd, ldh, ldsh */
  29         store,   /* st, std, sth, stsh */
  30         both,    /* Swap, ldstub, cas, ... */
  31         fpld,
  32         fpst,
  33         invalid,
  34 };
  35
  36 #ifdef DEBUG_MNA
  37 static char *dirstrings[] = {
  38   "load", "store", "both", "fpload", "fpstore", "invalid"
  39 };
  40 #endif
  41
  42 static inline enum direction decode_direction(unsigned int insn)
  43 {
  44         unsigned long tmp = (insn >> 21) & 1;
  45
  46         if (!tmp)
  47                 return load;
  48         else {
  49                 switch ((insn>>19)&0xf) {
  50                 case 15: /* swap* */
  51                         return both;
  52                 default:
  53                         return store;
  54                 }
  55         }
  56 }
  57
  58 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
  59 static inline int decode_access_size(unsigned int insn)
  60 {
  61         unsigned int tmp;
  62
  63         tmp = ((insn >> 19) & 0xf);
  64         if (tmp == 11 || tmp == 14) /* ldx/stx */
  65                 return 8;
  66         tmp &= 3;
  67         if (!tmp)
  68                 return 4;
  69         else if (tmp == 3)
  70                 return 16;      /* ldd/std - Although it is actually 8 */
  71         else if (tmp == 2)
  72                 return 2;
  73         else {
  74                 printk("Impossible unaligned trap. insn=%08x\n", insn);
  75                 die_if_kernel("Byte sized unaligned access?!?!", current_thread_info()->kregs);
  76
  77                 /* GCC should never warn that control reaches the end
  78                  * of this function without returning a value because
  79                  * die_if_kernel() is marked with attribute 'noreturn'.
  80                  * Alas, some versions do...
  81                  */
  82
  83                 return 0;
  84         }
  85 }
  86
  87 static inline int decode_asi(unsigned int insn, struct pt_regs *regs)
  88 {
  89         if (insn & 0x800000) {
  90                 if (insn & 0x2000)
  91                         return (unsigned char)(regs->tstate >> 24);     /* %asi */
  92                 else
  93                         return (unsigned char)(insn >> 5);              /* imm_asi */
  94         } else
  95                 return ASI_P;
  96 }
  97
  98 /* 0x400000 = signed, 0 = unsigned */
  99 static inline int decode_signedness(unsigned int insn)
 100 {
 101         return (insn & 0x400000);
 102 }
 103
 104 static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2,
 105                                        unsigned int rd, int from_kernel)
 106 {
 107         if (rs2 >= 16 || rs1 >= 16 || rd >= 16) {
 108                 if (from_kernel != 0)
 109                         __asm__ __volatile__("flushw");
 110                 else
 111                         flushw_user();
 112         }
 113 }
 114
 115 static inline long sign_extend_imm13(long imm)
 116 {
 117         return imm << 51 >> 51;
 118 }
 119
 120 static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs)
 121 {
 122         unsigned long value;
 123
 124         if (reg < 16)
 125                 return (!reg ? 0 : regs->u_regs[reg]);
 126         if (regs->tstate & TSTATE_PRIV) {
 127                 struct reg_window *win;
 128                 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS);
 129                 value = win->locals[reg - 16];
 130         } else if (test_thread_flag(TIF_32BIT)) {
 131                 struct reg_window32 __user *win32;
 132                 win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP]));
 133                 get_user(value, &win32->locals[reg - 16]);
 134         } else {
 135                 struct reg_window __user *win;
 136                 win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS);
 137                 get_user(value, &win->locals[reg - 16]);
 138         }
 139         return value;
 140 }
 141
 142 static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs)
 143 {
 144         if (reg < 16)
 145                 return &regs->u_regs[reg];
 146         if (regs->tstate & TSTATE_PRIV) {
 147                 struct reg_window *win;
 148                 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS);
 149                 return &win->locals[reg - 16];
 150         } else if (test_thread_flag(TIF_32BIT)) {
 151                 struct reg_window32 *win32;
 152                 win32 = (struct reg_window32 *)((unsigned long)((u32)regs->u_regs[UREG_FP]));
 153                 return (unsigned long *)&win32->locals[reg - 16];
 154         } else {
 155                 struct reg_window *win;
 156                 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS);
 157                 return &win->locals[reg - 16];
 158         }
 159 }
 160
 161 unsigned long compute_effective_address(struct pt_regs *regs,
 162                                         unsigned int insn, unsigned int rd)
 163 {
 164         unsigned int rs1 = (insn >> 14) & 0x1f;
 165         unsigned int rs2 = insn & 0x1f;
 166         int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
 167
 168         if (insn & 0x2000) {
 169                 maybe_flush_windows(rs1, 0, rd, from_kernel);
 170                 return (fetch_reg(rs1, regs) + sign_extend_imm13(insn));
 171         } else {
 172                 maybe_flush_windows(rs1, rs2, rd, from_kernel);
 173                 return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs));
 174         }
 175 }
 176
 177 /* This is just to make gcc think die_if_kernel does return... */
 178 static void __attribute_used__ unaligned_panic(char *str, struct pt_regs *regs)
 179 {
 180         die_if_kernel(str, regs);
 181 }
 182
 183 #define do_integer_load(dest_reg, size, saddr, is_signed, asi, errh) ({         \
 184 __asm__ __volatile__ (                                                          \
 185         "wr     %4, 0, %%asi\n\t"                                               \
 186         "cmp    %1, 8\n\t"                                                      \
 187         "bge,pn %%icc, 9f\n\t"                                                  \
 188         " cmp   %1, 4\n\t"                                                      \
 189         "be,pt  %%icc, 6f\n"                                                    \
 190 "4:\t"  " lduba [%2] %%asi, %%l1\n"                                             \
 191 "5:\t"  "lduba  [%2 + 1] %%asi, %%l2\n\t"                                       \
 192         "sll    %%l1, 8, %%l1\n\t"                                              \
 193         "brz,pt %3, 3f\n\t"                                                     \
 194         " add   %%l1, %%l2, %%l1\n\t"                                           \
 195         "sllx   %%l1, 48, %%l1\n\t"                                             \
 196         "srax   %%l1, 48, %%l1\n"                                               \
 197 "3:\t"  "ba,pt  %%xcc, 0f\n\t"                                                  \
 198         " stx   %%l1, [%0]\n"                                                   \
 199 "6:\t"  "lduba  [%2 + 1] %%asi, %%l2\n\t"                                       \
 200         "sll    %%l1, 24, %%l1\n"                                               \
 201 "7:\t"  "lduba  [%2 + 2] %%asi, %%g7\n\t"                                       \
 202         "sll    %%l2, 16, %%l2\n"                                               \
 203 "8:\t"  "lduba  [%2 + 3] %%asi, %%g1\n\t"                                       \
 204         "sll    %%g7, 8, %%g7\n\t"                                              \
 205         "or     %%l1, %%l2, %%l1\n\t"                                           \
 206         "or     %%g7, %%g1, %%g7\n\t"                                           \
 207         "or     %%l1, %%g7, %%l1\n\t"                                           \
 208         "brnz,a,pt %3, 3f\n\t"                                                  \
 209         " sra   %%l1, 0, %%l1\n"                                                \
 210 "3:\t"  "ba,pt  %%xcc, 0f\n\t"                                                  \
 211         " stx   %%l1, [%0]\n"                                                   \
 212 "9:\t"  "lduba  [%2] %%asi, %%l1\n"                                             \
 213 "10:\t" "lduba  [%2 + 1] %%asi, %%l2\n\t"                                       \
 214         "sllx   %%l1, 56, %%l1\n"                                               \
 215 "11:\t" "lduba  [%2 + 2] %%asi, %%g7\n\t"                                       \
 216         "sllx   %%l2, 48, %%l2\n"                                               \
 217 "12:\t" "lduba  [%2 + 3] %%asi, %%g1\n\t"                                       \
 218         "sllx   %%g7, 40, %%g7\n\t"                                             \
 219         "sllx   %%g1, 32, %%g1\n\t"                                             \
 220         "or     %%l1, %%l2, %%l1\n\t"                                           \
 221         "or     %%g7, %%g1, %%g7\n"                                             \
 222 "13:\t" "lduba  [%2 + 4] %%asi, %%l2\n\t"                                       \
 223         "or     %%l1, %%g7, %%g7\n"                                             \
 224 "14:\t" "lduba  [%2 + 5] %%asi, %%g1\n\t"                                       \
 225         "sllx   %%l2, 24, %%l2\n"                                               \
 226 "15:\t" "lduba  [%2 + 6] %%asi, %%l1\n\t"                                       \
 227         "sllx   %%g1, 16, %%g1\n\t"                                             \
 228         "or     %%g7, %%l2, %%g7\n"                                             \
 229 "16:\t" "lduba  [%2 + 7] %%asi, %%l2\n\t"                                       \
 230         "sllx   %%l1, 8, %%l1\n\t"                                              \
 231         "or     %%g7, %%g1, %%g7\n\t"                                           \
 232         "or     %%l1, %%l2, %%l1\n\t"                                           \
 233         "or     %%g7, %%l1, %%g7\n\t"                                           \
 234         "cmp    %1, 8\n\t"                                                      \
 235         "be,a,pt %%icc, 0f\n\t"                                                 \
 236         " stx   %%g7, [%0]\n\t"                                                 \
 237         "srlx   %%g7, 32, %%l1\n\t"                                             \
 238         "sra    %%g7, 0, %%g7\n\t"                                              \
 239         "stx    %%l1, [%0]\n\t"                                                 \
 240         "stx    %%g7, [%0 + 8]\n"                                               \
 241 "0:\n\t"                                                                        \
 242         "wr     %%g0, %5, %%asi\n\n\t"                                          \
 243         ".section __ex_table\n\t"                                               \
 244         ".word  4b, " #errh "\n\t"                                              \
 245         ".word  5b, " #errh "\n\t"                                              \
 246         ".word  6b, " #errh "\n\t"                                              \
 247         ".word  7b, " #errh "\n\t"                                              \
 248         ".word  8b, " #errh "\n\t"                                              \
 249         ".word  9b, " #errh "\n\t"                                              \
 250         ".word  10b, " #errh "\n\t"                                             \
 251         ".word  11b, " #errh "\n\t"                                             \
 252         ".word  12b, " #errh "\n\t"                                             \
 253         ".word  13b, " #errh "\n\t"                                             \
 254         ".word  14b, " #errh "\n\t"                                             \
 255         ".word  15b, " #errh "\n\t"                                             \
 256         ".word  16b, " #errh "\n\n\t"                                           \
 257         ".previous\n\t"                                                         \
 258         : : "r" (dest_reg), "r" (size), "r" (saddr), "r" (is_signed),           \
 259           "r" (asi), "i" (ASI_AIUS)                                             \
 260         : "l1", "l2", "g7", "g1", "cc");                                        \
 261 })
 262
 263 #define store_common(dst_addr, size, src_val, asi, errh) ({                     \
 264 __asm__ __volatile__ (                                                          \
 265         "wr     %3, 0, %%asi\n\t"                                               \
 266         "ldx    [%2], %%l1\n"                                                   \
 267         "cmp    %1, 2\n\t"                                                      \
 268         "be,pn  %%icc, 2f\n\t"                                                  \
 269         " cmp   %1, 4\n\t"                                                      \
 270         "be,pt  %%icc, 1f\n\t"                                                  \
 271         " srlx  %%l1, 24, %%l2\n\t"                                             \
 272         "srlx   %%l1, 56, %%g1\n\t"                                             \
 273         "srlx   %%l1, 48, %%g7\n"                                               \
 274 "4:\t"  "stba   %%g1, [%0] %%asi\n\t"                                           \
 275         "srlx   %%l1, 40, %%g1\n"                                               \
 276 "5:\t"  "stba   %%g7, [%0 + 1] %%asi\n\t"                                       \
 277         "srlx   %%l1, 32, %%g7\n"                                               \
 278 "6:\t"  "stba   %%g1, [%0 + 2] %%asi\n"                                         \
 279 "7:\t"  "stba   %%g7, [%0 + 3] %%asi\n\t"                                       \
 280         "srlx   %%l1, 16, %%g1\n"                                               \
 281 "8:\t"  "stba   %%l2, [%0 + 4] %%asi\n\t"                                       \
 282         "srlx   %%l1, 8, %%g7\n"                                                \
 283 "9:\t"  "stba   %%g1, [%0 + 5] %%asi\n"                                         \
 284 "10:\t" "stba   %%g7, [%0 + 6] %%asi\n\t"                                       \
 285         "ba,pt  %%xcc, 0f\n"                                                    \
 286 "11:\t" " stba  %%l1, [%0 + 7] %%asi\n"                                         \
 287 "1:\t"  "srl    %%l1, 16, %%g7\n"                                               \
 288 "12:\t" "stba   %%l2, [%0] %%asi\n\t"                                           \
 289         "srl    %%l1, 8, %%l2\n"                                                \
 290 "13:\t" "stba   %%g7, [%0 + 1] %%asi\n"                                         \
 291 "14:\t" "stba   %%l2, [%0 + 2] %%asi\n\t"                                       \
 292         "ba,pt  %%xcc, 0f\n"                                                    \
 293 "15:\t" " stba  %%l1, [%0 + 3] %%asi\n"                                         \
 294 "2:\t"  "srl    %%l1, 8, %%l2\n"                                                \
 295 "16:\t" "stba   %%l2, [%0] %%asi\n"                                             \
 296 "17:\t" "stba   %%l1, [%0 + 1] %%asi\n"                                         \
 297 "0:\n\t"                                                                        \
 298         "wr     %%g0, %4, %%asi\n\n\t"                                          \
 299         ".section __ex_table\n\t"                                               \
 300         ".word  4b, " #errh "\n\t"                                              \
 301         ".word  5b, " #errh "\n\t"                                              \
 302         ".word  6b, " #errh "\n\t"                                              \
 303         ".word  7b, " #errh "\n\t"                                              \
 304         ".word  8b, " #errh "\n\t"                                              \
 305         ".word  9b, " #errh "\n\t"                                              \
 306         ".word  10b, " #errh "\n\t"                                             \
 307         ".word  11b, " #errh "\n\t"                                             \
 308         ".word  12b, " #errh "\n\t"                                             \
 309         ".word  13b, " #errh "\n\t"                                             \
 310         ".word  14b, " #errh "\n\t"                                             \
 311         ".word  15b, " #errh "\n\t"                                             \
 312         ".word  16b, " #errh "\n\t"                                             \
 313         ".word  17b, " #errh "\n\n\t"                                           \
 314         ".previous\n\t"                                                         \
 315         : : "r" (dst_addr), "r" (size), "r" (src_val), "r" (asi), "i" (ASI_AIUS)\
 316         : "l1", "l2", "g7", "g1", "cc");                                        \
 317 })
 318
 319 #define do_integer_store(reg_num, size, dst_addr, regs, asi, errh) ({           \
 320         unsigned long zero = 0;                                                 \
 321         unsigned long *src_val = &zero;                                         \
 322                                                                                 \
 323         if (size == 16) {                                                       \
 324                 size = 8;                                                       \
 325                 zero = (((long)(reg_num ?                                       \
 326                         (unsigned)fetch_reg(reg_num, regs) : 0)) << 32) |       \
 327                         (unsigned)fetch_reg(reg_num + 1, regs);                 \
 328         } else if (reg_num) src_val = fetch_reg_addr(reg_num, regs);            \
 329         store_common(dst_addr, size, src_val, asi, errh);                       \
 330 })
 331
 332 extern void smp_capture(void);
 333 extern void smp_release(void);
 334
 335 #define do_atomic(srcdest_reg, mem, errh) ({                                    \
 336         unsigned long flags, tmp;                                               \
 337                                                                                 \
 338         smp_capture();                                                          \
 339         local_irq_save(flags);                                                  \
 340         tmp = *srcdest_reg;                                                     \
 341         do_integer_load(srcdest_reg, 4, mem, 0, errh);                          \
 342         store_common(mem, 4, &tmp, errh);                                       \
 343         local_irq_restore(flags);                                               \
 344         smp_release();                                                          \
 345 })
 346
 347 static inline void advance(struct pt_regs *regs)
 348 {
 349         regs->tpc   = regs->tnpc;
 350         regs->tnpc += 4;
 351         if (test_thread_flag(TIF_32BIT)) {
 352                 regs->tpc &= 0xffffffff;
 353                 regs->tnpc &= 0xffffffff;
 354         }
 355 }
 356
 357 static inline int floating_point_load_or_store_p(unsigned int insn)
 358 {
 359         return (insn >> 24) & 1;
 360 }
 361
 362 static inline int ok_for_kernel(unsigned int insn)
 363 {
 364         return !floating_point_load_or_store_p(insn);
 365 }
 366
 367 void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) __asm__ ("kernel_mna_trap_fault");
 368
 369 void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
 370 {
 371         unsigned long g2 = regs->u_regs [UREG_G2];
 372         unsigned long fixup = search_extables_range(regs->tpc, &g2);
 373
 374         if (!fixup) {
 375                 unsigned long address = compute_effective_address(regs, insn, ((insn >> 25) & 0x1f));
 376                 if (address < PAGE_SIZE) {
 377                         printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler");
 378                 } else
 379                         printk(KERN_ALERT "Unable to handle kernel paging request in mna handler");
 380                 printk(KERN_ALERT " at virtual address %016lx\n",address);
 381                 printk(KERN_ALERT "current->{mm,active_mm}->context = %016lx\n",
 382                         (current->mm ? current->mm->context :
 383                         current->active_mm->context));
 384                 printk(KERN_ALERT "current->{mm,active_mm}->pgd = %016lx\n",
 385                         (current->mm ? (unsigned long) current->mm->pgd :
 386                         (unsigned long) current->active_mm->pgd));
 387                 die_if_kernel("Oops", regs);
 388                 /* Not reached */
 389         }
 390         regs->tpc = fixup;
 391         regs->tnpc = regs->tpc + 4;
 392         regs->u_regs [UREG_G2] = g2;
 393
 394         regs->tstate &= ~TSTATE_ASI;
 395         regs->tstate |= (ASI_AIUS << 24UL);
 396 }
 397
 398 asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn, unsigned long sfar, unsigned long sfsr)
 399 {
 400         enum direction dir = decode_direction(insn);
 401         int size = decode_access_size(insn);
 402
 403         if (!ok_for_kernel(insn) || dir == both) {
 404                 printk("Unsupported unaligned load/store trap for kernel at <%016lx>.\n",
 405                        regs->tpc);
 406                 unaligned_panic("Kernel does fpu/atomic unaligned load/store.", regs);
 407
 408                 __asm__ __volatile__ ("\n"
 409 "kernel_unaligned_trap_fault:\n\t"
 410                 "mov    %0, %%o0\n\t"
 411                 "call   kernel_mna_trap_fault\n\t"
 412                 " mov   %1, %%o1\n\t"
 413                 :
 414                 : "r" (regs), "r" (insn)
 415                 : "o0", "o1", "o2", "o3", "o4", "o5", "o7",
 416                   "g1", "g2", "g3", "g4", "g5", "g7", "cc");
 417         } else {
 418                 unsigned long addr = compute_effective_address(regs, insn, ((insn >> 25) & 0x1f));
 419
 420 #ifdef DEBUG_MNA
 421                 printk("KMNA: pc=%016lx [dir=%s addr=%016lx size=%d] retpc[%016lx]\n",
 422                        regs->tpc, dirstrings[dir], addr, size, regs->u_regs[UREG_RETPC]);
 423 #endif
 424                 switch (dir) {
 425                 case load:
 426                         do_integer_load(fetch_reg_addr(((insn>>25)&0x1f), regs),
 427                                         size, (unsigned long *) addr,
 428                                         decode_signedness(insn), decode_asi(insn, regs),
 429                                         kernel_unaligned_trap_fault);
 430                         break;
 431
 432                 case store:
 433                         do_integer_store(((insn>>25)&0x1f), size,
 434                                          (unsigned long *) addr, regs,
 435                                          decode_asi(insn, regs),
 436                                          kernel_unaligned_trap_fault);
 437                         break;
 438 #if 0 /* unsupported */
 439                 case both:
 440                         do_atomic(fetch_reg_addr(((insn>>25)&0x1f), regs),
 441                                   (unsigned long *) addr,
 442                                   kernel_unaligned_trap_fault);
 443                         break;
 444 #endif
 445                 default:
 446                         panic("Impossible kernel unaligned trap.");
 447                         /* Not reached... */
 448                 }
 449                 advance(regs);
 450         }
 451 }
 452
 453 static char popc_helper[] = {
 454 0, 1, 1, 2, 1, 2, 2, 3,
 455 1, 2, 2, 3, 2, 3, 3, 4,
 456 };
 457
 458 int handle_popc(u32 insn, struct pt_regs *regs)
 459 {
 460         u64 value;
 461         int ret, i, rd = ((insn >> 25) & 0x1f);
 462         int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
 463
 464         if (insn & 0x2000) {
 465                 maybe_flush_windows(0, 0, rd, from_kernel);
 466                 value = sign_extend_imm13(insn);
 467         } else {
 468                 maybe_flush_windows(0, insn & 0x1f, rd, from_kernel);
 469                 value = fetch_reg(insn & 0x1f, regs);
 470         }
 471         for (ret = 0, i = 0; i < 16; i++) {
 472                 ret += popc_helper[value & 0xf];
 473                 value >>= 4;
 474         }
 475         if (rd < 16) {
 476                 if (rd)
 477                         regs->u_regs[rd] = ret;
 478         } else {
 479                 if (test_thread_flag(TIF_32BIT)) {
 480                         struct reg_window32 __user *win32;
 481                         win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP]));
 482                         put_user(ret, &win32->locals[rd - 16]);
 483                 } else {
 484                         struct reg_window __user *win;
 485                         win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS);
 486                         put_user(ret, &win->locals[rd - 16]);
 487                 }
 488         }
 489         advance(regs);
 490         return 1;
 491 }
 492
 493 extern void do_fpother(struct pt_regs *regs);
 494 extern void do_privact(struct pt_regs *regs);
 495 extern void data_access_exception(struct pt_regs *regs,
 496                                   unsigned long sfsr,
 497                                   unsigned long sfar);
 498
 499 int handle_ldf_stq(u32 insn, struct pt_regs *regs)
 500 {
 501         unsigned long addr = compute_effective_address(regs, insn, 0);
 502         int freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
 503         struct fpustate *f = FPUSTATE;
 504         int asi = decode_asi(insn, regs);
 505         int flag = (freg < 32) ? FPRS_DL : FPRS_DU;
 506
 507         save_and_clear_fpu();
 508         current_thread_info()->xfsr[0] &= ~0x1c000;
 509         if (freg & 3) {
 510                 current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */;
 511                 do_fpother(regs);
 512                 return 0;
 513         }
 514         if (insn & 0x200000) {
 515                 /* STQ */
 516                 u64 first = 0, second = 0;
 517
 518                 if (current_thread_info()->fpsaved[0] & flag) {
 519                         first = *(u64 *)&f->regs[freg];
 520                         second = *(u64 *)&f->regs[freg+2];
 521                 }
 522                 if (asi < 0x80) {
 523                         do_privact(regs);
 524                         return 1;
 525                 }
 526                 switch (asi) {
 527                 case ASI_P:
 528                 case ASI_S: break;
 529                 case ASI_PL:
 530                 case ASI_SL:
 531                         {
 532                                 /* Need to convert endians */
 533                                 u64 tmp = __swab64p(&first);
 534
 535                                 first = __swab64p(&second);
 536                                 second = tmp;
 537                                 break;
 538                         }
 539                 default:
 540                         data_access_exception(regs, 0, addr);
 541                         return 1;
 542                 }
 543                 if (put_user (first >> 32, (u32 __user *)addr) ||
 544                     __put_user ((u32)first, (u32 __user *)(addr + 4)) ||
 545                     __put_user (second >> 32, (u32 __user *)(addr + 8)) ||
 546                     __put_user ((u32)second, (u32 __user *)(addr + 12))) {
 547                         data_access_exception(regs, 0, addr);
 548                         return 1;
 549                 }
 550         } else {
 551                 /* LDF, LDDF, LDQF */
 552                 u32 data[4] __attribute__ ((aligned(8)));
 553                 int size, i;
 554                 int err;
 555
 556                 if (asi < 0x80) {
 557                         do_privact(regs);
 558                         return 1;
 559                 } else if (asi > ASI_SNFL) {
 560                         data_access_exception(regs, 0, addr);
 561                         return 1;
 562                 }
 563                 switch (insn & 0x180000) {
 564                 case 0x000000: size = 1; break;
 565                 case 0x100000: size = 4; break;
 566                 default: size = 2; break;
 567                 }
 568                 for (i = 0; i < size; i++)
 569                         data[i] = 0;
 570
 571                 err = get_user (data[0], (u32 __user *) addr);
 572                 if (!err) {
 573                         for (i = 1; i < size; i++)
 574                                 err |= __get_user (data[i], (u32 __user *)(addr + 4*i));
 575                 }
 576                 if (err && !(asi & 0x2 /* NF */)) {
 577                         data_access_exception(regs, 0, addr);
 578                         return 1;
 579                 }
 580                 if (asi & 0x8) /* Little */ {
 581                         u64 tmp;
 582
 583                         switch (size) {
 584                         case 1: data[0] = le32_to_cpup(data + 0); break;
 585                         default:*(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 0));
 586                                 break;
 587                         case 4: tmp = le64_to_cpup((u64 *)(data + 0));
 588                                 *(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 2));
 589                                 *(u64 *)(data + 2) = tmp;
 590                                 break;
 591                         }
 592                 }
 593                 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) {
 594                         current_thread_info()->fpsaved[0] = FPRS_FEF;
 595                         current_thread_info()->gsr[0] = 0;
 596                 }
 597                 if (!(current_thread_info()->fpsaved[0] & flag)) {
 598                         if (freg < 32)
 599                                 memset(f->regs, 0, 32*sizeof(u32));
 600                         else
 601                                 memset(f->regs+32, 0, 32*sizeof(u32));
 602                 }
 603                 memcpy(f->regs + freg, data, size * 4);
 604                 current_thread_info()->fpsaved[0] |= flag;
 605         }
 606         advance(regs);
 607         return 1;
 608 }
 609
 610 void handle_ld_nf(u32 insn, struct pt_regs *regs)
 611 {
 612         int rd = ((insn >> 25) & 0x1f);
 613         int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
 614         unsigned long *reg;
 615
 616         maybe_flush_windows(0, 0, rd, from_kernel);
 617         reg = fetch_reg_addr(rd, regs);
 618         if (from_kernel || rd < 16) {
 619                 reg[0] = 0;
 620                 if ((insn & 0x780000) == 0x180000)
 621                         reg[1] = 0;
 622         } else if (test_thread_flag(TIF_32BIT)) {
 623                 put_user(0, (int __user *) reg);
 624                 if ((insn & 0x780000) == 0x180000)
 625                         put_user(0, ((int __user *) reg) + 1);
 626         } else {
 627                 put_user(0, (unsigned long __user *) reg);
 628                 if ((insn & 0x780000) == 0x180000)
 629                         put_user(0, (unsigned long __user *) reg + 1);
 630         }
 631         advance(regs);
 632 }
 633
 634 void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
 635 {
 636         unsigned long pc = regs->tpc;
 637         unsigned long tstate = regs->tstate;
 638         u32 insn;
 639         u32 first, second;
 640         u64 value;
 641         u8 asi, freg;
 642         int flag;
 643         struct fpustate *f = FPUSTATE;
 644
 645         if (tstate & TSTATE_PRIV)
 646                 die_if_kernel("lddfmna from kernel", regs);
 647         if (test_thread_flag(TIF_32BIT))
 648                 pc = (u32)pc;
 649         if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
 650                 asi = sfsr >> 16;
 651                 if ((asi > ASI_SNFL) ||
 652                     (asi < ASI_P))
 653                         goto daex;
 654                 if (get_user(first, (u32 __user *)sfar) ||
 655                      get_user(second, (u32 __user *)(sfar + 4))) {
 656                         if (asi & 0x2) /* NF */ {
 657                                 first = 0; second = 0;
 658                         } else
 659                                 goto daex;
 660                 }
 661                 save_and_clear_fpu();
 662                 freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
 663                 value = (((u64)first) << 32) | second;
 664                 if (asi & 0x8) /* Little */
 665                         value = __swab64p(&value);
 666                 flag = (freg < 32) ? FPRS_DL : FPRS_DU;
 667                 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) {
 668                         current_thread_info()->fpsaved[0] = FPRS_FEF;
 669                         current_thread_info()->gsr[0] = 0;
 670                 }
 671                 if (!(current_thread_info()->fpsaved[0] & flag)) {
 672                         if (freg < 32)
 673                                 memset(f->regs, 0, 32*sizeof(u32));
 674                         else
 675                                 memset(f->regs+32, 0, 32*sizeof(u32));
 676                 }
 677                 *(u64 *)(f->regs + freg) = value;
 678                 current_thread_info()->fpsaved[0] |= flag;
 679         } else {
 680 daex:           data_access_exception(regs, sfsr, sfar);
 681                 return;
 682         }
 683         advance(regs);
 684         return;
 685 }
 686
 687 void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
 688 {
 689         unsigned long pc = regs->tpc;
 690         unsigned long tstate = regs->tstate;
 691         u32 insn;
 692         u64 value;
 693         u8 asi, freg;
 694         int flag;
 695         struct fpustate *f = FPUSTATE;
 696
 697         if (tstate & TSTATE_PRIV)
 698                 die_if_kernel("stdfmna from kernel", regs);
 699         if (test_thread_flag(TIF_32BIT))
 700                 pc = (u32)pc;
 701         if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
 702                 freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
 703                 asi = sfsr >> 16;
 704                 value = 0;
 705                 flag = (freg < 32) ? FPRS_DL : FPRS_DU;
 706                 if ((asi > ASI_SNFL) ||
 707                     (asi < ASI_P))
 708                         goto daex;
 709                 save_and_clear_fpu();
 710                 if (current_thread_info()->fpsaved[0] & flag)
 711                         value = *(u64 *)&f->regs[freg];
 712                 switch (asi) {
 713                 case ASI_P:
 714                 case ASI_S: break;
 715                 case ASI_PL:
 716                 case ASI_SL:
 717                         value = __swab64p(&value); break;
 718                 default: goto daex;
 719                 }
 720                 if (put_user (value >> 32, (u32 __user *) sfar) ||
 721                     __put_user ((u32)value, (u32 __user *)(sfar + 4)))
 722                         goto daex;
 723         } else {
 724 daex:           data_access_exception(regs, sfsr, sfar);
 725                 return;
 726         }
 727         advance(regs);
 728         return;
 729 }