arch/ia64/kernel/fsys.S

   1 /*
   2  * This file contains the light-weight system call handlers (fsyscall-handlers).
   3  *
   4  * Copyright (C) 2003 Hewlett-Packard Co
   5  *      David Mosberger-Tang <davidm@hpl.hp.com>
   6  *
   7  * 25-Sep-03 davidm     Implement fsys_rt_sigprocmask().
   8  * 18-Feb-03 louisk     Implement fsys_gettimeofday().
   9  * 28-Feb-03 davidm     Fixed several bugs in fsys_gettimeofday().  Tuned it some more,
  10  *                      probably broke it along the way... ;-)
  11  */
  12
  13 #include <asm/asmmacro.h>
  14 #include <asm/errno.h>
  15 #include <asm/offsets.h>
  16 #include <asm/percpu.h>
  17 #include <asm/thread_info.h>
  18 #include <asm/sal.h>
  19 #include <asm/signal.h>
  20 #include <asm/system.h>
  21 #include <asm/unistd.h>
  22
  23 #include "entry.h"
  24
  25 /*
  26  * See Documentation/ia64/fsys.txt for details on fsyscalls.
  27  *
  28  * On entry to an fsyscall handler:
  29  *   r10        = 0 (i.e., defaults to "successful syscall return")
  30  *   r11        = saved ar.pfs (a user-level value)
  31  *   r15        = system call number
  32  *   r16        = "current" task pointer (in normal kernel-mode, this is in r13)
  33  *   r32-r39    = system call arguments
  34  *   b6         = return address (a user-level value)
  35  *   ar.pfs     = previous frame-state (a user-level value)
  36  *   PSR.be     = cleared to zero (i.e., little-endian byte order is in effect)
  37  *   all other registers may contain values passed in from user-mode
  38  *
  39  * On return from an fsyscall handler:
  40  *   r11        = saved ar.pfs (as passed into the fsyscall handler)
  41  *   r15        = system call number (as passed into the fsyscall handler)
  42  *   r32-r39    = system call arguments (as passed into the fsyscall handler)
  43  *   b6         = return address (as passed into the fsyscall handler)
  44  *   ar.pfs     = previous frame-state (as passed into the fsyscall handler)
  45  */
  46
  47 ENTRY(fsys_ni_syscall)
  48         .prologue
  49         .altrp b6
  50         .body
  51         mov r8=ENOSYS
  52         mov r10=-1
  53         FSYS_RETURN
  54 END(fsys_ni_syscall)
  55
  56 ENTRY(fsys_getpid)
  57         .prologue
  58         .altrp b6
  59         .body
  60         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
  61         ;;
  62         ld4 r9=[r9]
  63         add r8=IA64_TASK_TGID_OFFSET,r16
  64         ;;
  65         and r9=TIF_ALLWORK_MASK,r9
  66         ld4 r8=[r8]                             // r8 = current->tgid
  67         ;;
  68         cmp.ne p8,p0=0,r9
  69 (p8)    br.spnt.many fsys_fallback_syscall
  70         FSYS_RETURN
  71 END(fsys_getpid)
  72
  73 ENTRY(fsys_getppid)
  74         .prologue
  75         .altrp b6
  76         .body
  77         add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
  78         ;;
  79         ld8 r17=[r17]                           // r17 = current->group_leader
  80         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
  81         ;;
  82
  83         ld4 r9=[r9]
  84         add r17=IA64_TASK_REAL_PARENT_OFFSET,r17 // r17 = &current->group_leader->real_parent
  85         ;;
  86         and r9=TIF_ALLWORK_MASK,r9
  87
  88 1:      ld8 r18=[r17]                           // r18 = current->group_leader->real_parent
  89         ;;
  90         cmp.ne p8,p0=0,r9
  91         add r8=IA64_TASK_TGID_OFFSET,r18        // r8 = &current->group_leader->real_parent->tgid
  92         ;;
  93
  94         /*
  95          * The .acq is needed to ensure that the read of tgid has returned its data before
  96          * we re-check "real_parent".
  97          */
  98         ld4.acq r8=[r8]                         // r8 = current->group_leader->real_parent->tgid
  99 #ifdef CONFIG_SMP
 100         /*
 101          * Re-read current->group_leader->real_parent.
 102          */
 103         ld8 r19=[r17]                           // r19 = current->group_leader->real_parent
 104 (p8)    br.spnt.many fsys_fallback_syscall
 105         ;;
 106         cmp.ne p6,p0=r18,r19                    // did real_parent change?
 107         mov r19=0                       // i must not leak kernel bits...
 108 (p6)    br.cond.spnt.few 1b                     // yes -> redo the read of tgid and the check
 109         ;;
 110         mov r17=0                       // i must not leak kernel bits...
 111         mov r18=0                       // i must not leak kernel bits...
 112 #else
 113         mov r17=0                       // i must not leak kernel bits...
 114         mov r18=0                       // i must not leak kernel bits...
 115         mov r19=0                       // i must not leak kernel bits...
 116 #endif
 117         FSYS_RETURN
 118 END(fsys_getppid)
 119
 120 ENTRY(fsys_set_tid_address)
 121         .prologue
 122         .altrp b6
 123         .body
 124         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
 125         ;;
 126         ld4 r9=[r9]
 127         tnat.z p6,p7=r32                // check argument register for being NaT
 128         ;;
 129         and r9=TIF_ALLWORK_MASK,r9
 130         add r8=IA64_TASK_PID_OFFSET,r16
 131         add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
 132         ;;
 133         ld4 r8=[r8]
 134         cmp.ne p8,p0=0,r9
 135         mov r17=-1
 136         ;;
 137 (p6)    st8 [r18]=r32
 138 (p7)    st8 [r18]=r17
 139 (p8)    br.spnt.many fsys_fallback_syscall
 140         ;;
 141         mov r17=0                       // i must not leak kernel bits...
 142         mov r18=0                       // i must not leak kernel bits...
 143         FSYS_RETURN
 144 END(fsys_set_tid_address)
 145
 146 /*
 147  * Note 1: This routine uses floating-point registers, but only with registers that
 148  *         operate on integers.  Because of that, we don't need to set ar.fpsr to the
 149  *         kernel default value.
 150  *
 151  * Note 2: For now, we will assume that all CPUs run at the same clock-frequency.
 152  *         If that wasn't the case, we would have to disable preemption (e.g.,
 153  *         by disabling interrupts) between reading the ITC and reading
 154  *         local_cpu_data->nsec_per_cyc.
 155  *
 156  * Note 3: On platforms where the ITC-drift bit is set in the SAL feature vector,
 157  *         we ought to either skip the ITC-based interpolation or run an ntp-like
 158  *         daemon to keep the ITCs from drifting too far apart.
 159  */
 160
 161 ENTRY(fsys_gettimeofday)
 162         .prologue
 163         .altrp b6
 164         .body
 165         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
 166         addl r3=THIS_CPU(cpu_info),r0
 167
 168         mov.m r31=ar.itc                // put time stamp into r31 (ITC) == now         (35 cyc)
 169 #ifdef CONFIG_SMP
 170         movl r10=__per_cpu_offset
 171         movl r2=sal_platform_features
 172         ;;
 173
 174         ld8 r2=[r2]
 175         movl r19=xtime                  // xtime is a timespec struct
 176
 177         ld8 r10=[r10]                   // r10 <- __per_cpu_offset[0]
 178         addl r21=THIS_CPU(cpu_info),r0
 179         ;;
 180         add r10=r21, r10                // r10 <- &cpu_data(time_keeper_id)
 181         tbit.nz p8,p0 = r2, IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT_BIT
 182 (p8)    br.spnt.many fsys_fallback_syscall
 183 #else
 184         ;;
 185         mov r10=r3
 186         movl r19=xtime                  // xtime is a timespec struct
 187 #endif
 188         ld4 r9=[r9]
 189         movl r17=xtime_lock
 190         ;;
 191
 192         // r32, r33 should contain the 2 args of gettimeofday
 193         adds r21=IA64_CPUINFO_ITM_NEXT_OFFSET, r10
 194         mov r2=-1
 195         tnat.nz p6,p7=r32               // guard against NaT args
 196         ;;
 197
 198         adds r10=IA64_CPUINFO_ITM_DELTA_OFFSET, r10
 199 (p7)    tnat.nz p6,p0=r33
 200 (p6)    br.cond.spnt.few .fail_einval
 201
 202         adds r8=IA64_CPUINFO_NSEC_PER_CYC_OFFSET, r3
 203         movl r24=2361183241434822607    // for division hack (only for / 1000)
 204         ;;
 205
 206         ldf8 f7=[r10]                   // f7 now contains itm_delta
 207         setf.sig f11=r2
 208         adds r10=8, r32
 209
 210         adds r20=IA64_TIMESPEC_TV_NSEC_OFFSET, r19      // r20 = &xtime->tv_nsec
 211         movl r26=jiffies
 212
 213         setf.sig f9=r24                 // f9 is used for division hack
 214         movl r27=wall_jiffies
 215
 216         and r9=TIF_ALLWORK_MASK,r9
 217         movl r25=last_nsec_offset
 218         ;;
 219
 220         /*
 221          * Verify that we have permission to write to struct timeval.  Note:
 222          * Another thread might unmap the mapping before we actually get
 223          * to store the result.  That's OK as long as the stores are also
 224          * protect by EX().
 225          */
 226 EX(.fail_efault, probe.w.fault r32, 3)          // this must come _after_ NaT-check
 227 EX(.fail_efault, probe.w.fault r10, 3)          // this must come _after_ NaT-check
 228         nop 0
 229
 230         ldf8 f10=[r8]                   // f10 <- local_cpu_data->nsec_per_cyc value
 231         cmp.ne p8, p0=0, r9
 232 (p8)    br.spnt.many fsys_fallback_syscall
 233         ;;
 234 .retry: // *** seq = read_seqbegin(&xtime_lock); ***
 235         ld4.acq r23=[r17]               // since &xtime_lock == &xtime_lock->sequence
 236         ld8 r14=[r25]                   // r14 (old) = last_nsec_offset
 237
 238         ld8 r28=[r26]                   // r28 = jiffies
 239         ld8 r29=[r27]                   // r29 = wall_jiffies
 240         ;;
 241
 242         ldf8 f8=[r21]                   // f8 now contains itm_next
 243         sub r28=r29, r28, 1             // r28 now contains "-(lost + 1)"
 244         tbit.nz p9, p10=r23, 0          // p9 <- is_odd(r23), p10 <- is_even(r23)
 245         ;;
 246
 247         ld8 r2=[r19]                    // r2 = sec = xtime.tv_sec
 248         ld8 r29=[r20]                   // r29 = nsec = xtime.tv_nsec
 249
 250         setf.sig f6=r28                 // f6 <- -(lost + 1)                            (6 cyc)
 251         ;;
 252
 253         mf
 254         xma.l f8=f6, f7, f8     // f8 (last_tick) <- -(lost + 1)*itm_delta + itm_next   (5 cyc)
 255         nop 0
 256
 257         setf.sig f12=r31                // f12 <- ITC                                   (6 cyc)
 258         // *** if (unlikely(read_seqretry(&xtime_lock, seq))) continue; ***
 259         ld4 r24=[r17]                   // r24 = xtime_lock->sequence (re-read)
 260         nop 0
 261         ;;
 262
 263         mov r31=ar.itc                  // re-read ITC in case we .retry                (35 cyc)
 264         xma.l f8=f11, f8, f12   // f8 (elapsed_cycles) <- (-1*last_tick + now) = (now - last_tick)
 265         nop 0
 266         ;;
 267
 268         getf.sig r18=f8                 // r18 <- (now - last_tick)
 269         xmpy.l f8=f8, f10               // f8 <- elapsed_cycles*nsec_per_cyc (5 cyc)
 270         add r3=r29, r14                 // r3 = (nsec + old)
 271         ;;
 272
 273         cmp.lt p7, p8=r18, r0           // if now < last_tick, set p7 = 1, p8 = 0
 274         getf.sig r18=f8                 // r18 = elapsed_cycles*nsec_per_cyc            (6 cyc)
 275         nop 0
 276         ;;
 277
 278 (p10)   cmp.ne p9, p0=r23, r24          // if xtime_lock->sequence != seq, set p9
 279         shr.u r18=r18, IA64_NSEC_PER_CYC_SHIFT  // r18 <- offset
 280 (p9)    br.spnt.many .retry
 281         ;;
 282
 283         mov ar.ccv=r14                  // ar.ccv = old                                 (1 cyc)
 284         cmp.leu p7, p8=r18, r14         // if (offset <= old), set p7 = 1, p8 = 0
 285         ;;
 286
 287 (p8)    cmpxchg8.rel r24=[r25], r18, ar.ccv     // compare-and-exchange (atomic!)
 288 (p8)    add r3=r29, r18                 // r3 = (nsec + offset)
 289         ;;
 290         shr.u r3=r3, 3                  // initiate dividing r3 by 1000
 291         ;;
 292         setf.sig f8=r3                  //                                              (6 cyc)
 293         mov r10=1000000                 // r10 = 1000000
 294         ;;
 295 (p8)    cmp.ne.unc p9, p0=r24, r14
 296         xmpy.hu f6=f8, f9               //                                              (5 cyc)
 297 (p9)    br.spnt.many .retry
 298         ;;
 299
 300         getf.sig r3=f6                  //                                              (6 cyc)
 301         ;;
 302         shr.u r3=r3, 4                  // end of division, r3 is divided by 1000 (=usec)
 303         ;;
 304
 305 1:      cmp.geu p7, p0=r3, r10          // while (usec >= 1000000)
 306         ;;
 307 (p7)    sub r3=r3, r10                  // usec -= 1000000
 308 (p7)    adds r2=1, r2                   // ++sec
 309 (p7)    br.spnt.many 1b
 310
 311         // finally: r2 = sec, r3 = usec
 312 EX(.fail_efault, st8 [r32]=r2)
 313         adds r9=8, r32
 314         mov r8=r0                       // success
 315         ;;
 316 EX(.fail_efault, st8 [r9]=r3)           // store them in the timeval struct
 317         mov r10=0
 318         FSYS_RETURN
 319         /*
 320          * Note: We are NOT clearing the scratch registers here.  Since the only things
 321          *       in those registers are time-related variables and some addresses (which
 322          *       can be obtained from System.map), none of this should be security-sensitive
 323          *       and we should be fine.
 324          */
 325
 326 .fail_einval:
 327         mov r8=EINVAL                   // r8 = EINVAL
 328         mov r10=-1                      // r10 = -1
 329         FSYS_RETURN
 330
 331 .fail_efault:
 332         mov r8=EFAULT                   // r8 = EFAULT
 333         mov r10=-1                      // r10 = -1
 334         FSYS_RETURN
 335 END(fsys_gettimeofday)
 336
 337 /*
 338  * long fsys_rt_sigprocmask (int how, sigset_t *set, sigset_t *oset, size_t sigsetsize).
 339  */
 340 #if _NSIG_WORDS != 1
 341 # error Sorry, fsys_rt_sigprocmask() needs to be updated for _NSIG_WORDS != 1.
 342 #endif
 343 ENTRY(fsys_rt_sigprocmask)
 344         .prologue
 345         .altrp b6
 346         .body
 347
 348         add r2=IA64_TASK_BLOCKED_OFFSET,r16
 349         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
 350         cmp4.ltu p6,p0=SIG_SETMASK,r32
 351
 352         cmp.ne p15,p0=r0,r34                    // oset != NULL?
 353         tnat.nz p8,p0=r34
 354         add r31=IA64_TASK_SIGHAND_OFFSET,r16
 355         ;;
 356         ld8 r3=[r2]                             // read/prefetch current->blocked
 357         ld4 r9=[r9]
 358         tnat.nz.or p6,p0=r35
 359
 360         cmp.ne.or p6,p0=_NSIG_WORDS*8,r35
 361         tnat.nz.or p6,p0=r32
 362 (p6)    br.spnt.few .fail_einval                // fail with EINVAL
 363         ;;
 364 #ifdef CONFIG_SMP
 365         ld8 r31=[r31]                           // r31 <- current->sighand
 366 #endif
 367         and r9=TIF_ALLWORK_MASK,r9
 368         tnat.nz.or p8,p0=r33
 369         ;;
 370         cmp.ne p7,p0=0,r9
 371         cmp.eq p6,p0=r0,r33                     // set == NULL?
 372         add r31=IA64_SIGHAND_SIGLOCK_OFFSET,r31 // r31 <- current->sighand->siglock
 373 (p8)    br.spnt.few .fail_efault                // fail with EFAULT
 374 (p7)    br.spnt.many fsys_fallback_syscall      // got pending kernel work...
 375 (p6)    br.dpnt.many .store_mask                // -> short-circuit to just reading the signal mask
 376
 377         /* Argh, we actually have to do some work and _update_ the signal mask: */
 378
 379 EX(.fail_efault, probe.r.fault r33, 3)          // verify user has read-access to *set
 380 EX(.fail_efault, ld8 r14=[r33])                 // r14 <- *set
 381         mov r17=(1 << (SIGKILL - 1)) | (1 << (SIGSTOP - 1))
 382         ;;
 383
 384         rsm psr.i                               // mask interrupt delivery
 385         mov ar.ccv=0
 386         andcm r14=r14,r17                       // filter out SIGKILL & SIGSTOP
 387
 388 #ifdef CONFIG_SMP
 389         mov r17=1
 390         ;;
 391         cmpxchg4.acq r18=[r31],r17,ar.ccv       // try to acquire the lock
 392         mov r8=EINVAL                   // default to EINVAL
 393         ;;
 394         ld8 r3=[r2]                     // re-read current->blocked now that we hold the lock
 395         cmp4.ne p6,p0=r18,r0
 396 (p6)    br.cond.spnt.many .lock_contention
 397         ;;
 398 #else
 399         ld8 r3=[r2]                     // re-read current->blocked now that we hold the lock
 400         mov r8=EINVAL                   // default to EINVAL
 401 #endif
 402         add r18=IA64_TASK_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r16
 403         add r19=IA64_TASK_SIGNAL_OFFSET,r16
 404         cmp4.eq p6,p0=SIG_BLOCK,r32
 405         ;;
 406         ld8 r19=[r19]                   // r19 <- current->signal
 407         cmp4.eq p7,p0=SIG_UNBLOCK,r32
 408         cmp4.eq p8,p0=SIG_SETMASK,r32
 409         ;;
 410         ld8 r18=[r18]                   // r18 <- current->pending.signal
 411         .pred.rel.mutex p6,p7,p8
 412 (p6)    or r14=r3,r14                   // SIG_BLOCK
 413 (p7)    andcm r14=r3,r14                // SIG_UNBLOCK
 414
 415 (p8)    mov r14=r14                     // SIG_SETMASK
 416 (p6)    mov r8=0                        // clear error code
 417         // recalc_sigpending()
 418         add r17=IA64_SIGNAL_GROUP_STOP_COUNT_OFFSET,r19
 419
 420         add r19=IA64_SIGNAL_SHARED_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r19
 421         ;;
 422         ld4 r17=[r17]           // r17 <- current->signal->group_stop_count
 423 (p7)    mov r8=0                // clear error code
 424
 425         ld8 r19=[r19]           // r19 <- current->signal->shared_pending
 426         ;;
 427         cmp4.gt p6,p7=r17,r0    // p6/p7 <- (current->signal->group_stop_count > 0)?
 428 (p8)    mov r8=0                // clear error code
 429
 430         or r18=r18,r19          // r18 <- current->pending | current->signal->shared_pending
 431         ;;
 432         // r18 <- (current->pending | current->signal->shared_pending) & ~current->blocked:
 433         andcm r18=r18,r14
 434         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
 435         ;;
 436
 437 (p7)    cmp.ne.or.andcm p6,p7=r18,r0            // p6/p7 <- signal pending
 438         mov r19=0                                       // i must not leak kernel bits...
 439 (p6)    br.cond.dpnt.many .sig_pending
 440         ;;
 441
 442 1:      ld4 r17=[r9]                            // r17 <- current->thread_info->flags
 443         ;;
 444         mov ar.ccv=r17
 445         and r18=~_TIF_SIGPENDING,r17            // r18 <- r17 & ~(1 << TIF_SIGPENDING)
 446         ;;
 447
 448         st8 [r2]=r14                            // update current->blocked with new mask
 449         cmpxchg4.acq r14=[r9],r18,ar.ccv        // current->thread_info->flags <- r18
 450         ;;
 451         cmp.ne p6,p0=r17,r14                    // update failed?
 452 (p6)    br.cond.spnt.few 1b                     // yes -> retry
 453
 454 #ifdef CONFIG_SMP
 455         st4.rel [r31]=r0                        // release the lock
 456 #endif
 457         ssm psr.i
 458         ;;
 459
 460         srlz.d                                  // ensure psr.i is set again
 461         mov r18=0                                       // i must not leak kernel bits...
 462
 463 .store_mask:
 464 EX(.fail_efault, (p15) probe.w.fault r34, 3)    // verify user has write-access to *oset
 465 EX(.fail_efault, (p15) st8 [r34]=r3)
 466         mov r2=0                                        // i must not leak kernel bits...
 467         mov r3=0                                        // i must not leak kernel bits...
 468         mov r8=0                                // return 0
 469         mov r9=0                                        // i must not leak kernel bits...
 470         mov r14=0                                       // i must not leak kernel bits...
 471         mov r17=0                                       // i must not leak kernel bits...
 472         mov r31=0                                       // i must not leak kernel bits...
 473         FSYS_RETURN
 474
 475 .sig_pending:
 476 #ifdef CONFIG_SMP
 477         st4.rel [r31]=r0                        // release the lock
 478 #endif
 479         ssm psr.i
 480         ;;
 481         srlz.d
 482         br.sptk.many fsys_fallback_syscall      // with signal pending, do the heavy-weight syscall
 483
 484 #ifdef CONFIG_SMP
 485 .lock_contention:
 486         /* Rather than spinning here, fall back on doing a heavy-weight syscall.  */
 487         ssm psr.i
 488         ;;
 489         srlz.d
 490         br.sptk.many fsys_fallback_syscall
 491 #endif
 492 END(fsys_rt_sigprocmask)
 493
 494 ENTRY(fsys_fallback_syscall)
 495         .prologue
 496         .altrp b6
 497         .body
 498         /*
 499          * We only get here from light-weight syscall handlers.  Thus, we already
 500          * know that r15 contains a valid syscall number.  No need to re-check.
 501          */
 502         adds r17=-1024,r15
 503         movl r14=sys_call_table
 504         ;;
 505         rsm psr.i
 506         shladd r18=r17,3,r14
 507         ;;
 508         ld8 r18=[r18]                           // load normal (heavy-weight) syscall entry-point
 509         mov r29=psr                             // read psr (12 cyc load latency)
 510         mov r27=ar.rsc
 511         mov r21=ar.fpsr
 512         mov r26=ar.pfs
 513 END(fsys_fallback_syscall)
 514         /* FALL THROUGH */
 515 GLOBAL_ENTRY(fsys_bubble_down)
 516         .prologue
 517         .altrp b6
 518         .body
 519         /*
 520          * We get here for syscalls that don't have a lightweight handler.  For those, we
 521          * need to bubble down into the kernel and that requires setting up a minimal
 522          * pt_regs structure, and initializing the CPU state more or less as if an
 523          * interruption had occurred.  To make syscall-restarts work, we setup pt_regs
 524          * such that cr_iip points to the second instruction in syscall_via_break.
 525          * Decrementing the IP hence will restart the syscall via break and not
 526          * decrementing IP will return us to the caller, as usual.  Note that we preserve
 527          * the value of psr.pp rather than initializing it from dcr.pp.  This makes it
 528          * possible to distinguish fsyscall execution from other privileged execution.
 529          *
 530          * On entry:
 531          *      - normal fsyscall handler register usage, except that we also have:
 532          *      - r18: address of syscall entry point
 533          *      - r21: ar.fpsr
 534          *      - r26: ar.pfs
 535          *      - r27: ar.rsc
 536          *      - r29: psr
 537          */
 538 #       define PSR_PRESERVED_BITS       (IA64_PSR_UP | IA64_PSR_MFL | IA64_PSR_MFH | IA64_PSR_PK \
 539                                          | IA64_PSR_DT | IA64_PSR_PP | IA64_PSR_SP | IA64_PSR_RT \
 540                                          | IA64_PSR_IC)
 541         /*
 542          * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.  The rest we have
 543          * to synthesize.
 544          */
 545 #       define PSR_ONE_BITS             ((3 << IA64_PSR_CPL0_BIT) | (0x1 << IA64_PSR_RI_BIT) \
 546                                          | IA64_PSR_BN | IA64_PSR_I)
 547
 548         invala
 549         movl r8=PSR_ONE_BITS
 550
 551         mov r25=ar.unat                 // save ar.unat (5 cyc)
 552         movl r9=PSR_PRESERVED_BITS
 553
 554         mov ar.rsc=0                    // set enforced lazy mode, pl 0, little-endian, loadrs=0
 555         movl r28=__kernel_syscall_via_break
 556         ;;
 557         mov r23=ar.bspstore             // save ar.bspstore (12 cyc)
 558         mov r31=pr                      // save pr (2 cyc)
 559         mov r20=r1                      // save caller's gp in r20
 560         ;;
 561         mov r2=r16                      // copy current task addr to addl-addressable register
 562         and r9=r9,r29
 563         mov r19=b6                      // save b6 (2 cyc)
 564         ;;
 565         mov psr.l=r9                    // slam the door (17 cyc to srlz.i)
 566         or r29=r8,r29                   // construct cr.ipsr value to save
 567         addl r22=IA64_RBS_OFFSET,r2     // compute base of RBS
 568         ;;
 569         // GAS reports a spurious RAW hazard on the read of ar.rnat because it thinks
 570         // we may be reading ar.itc after writing to psr.l.  Avoid that message with
 571         // this directive:
 572         dv_serialize_data
 573         mov.m r24=ar.rnat               // read ar.rnat (5 cyc lat)
 574         lfetch.fault.excl.nt1 [r22]
 575         adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r2
 576
 577         // ensure previous insn group is issued before we stall for srlz.i:
 578         ;;
 579         srlz.i                          // ensure new psr.l has been established
 580         /////////////////////////////////////////////////////////////////////////////
 581         ////////// from this point on, execution is not interruptible anymore
 582         /////////////////////////////////////////////////////////////////////////////
 583         addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2    // compute base of memory stack
 584         cmp.ne pKStk,pUStk=r0,r0        // set pKStk <- 0, pUStk <- 1
 585         ;;
 586         st1 [r16]=r0                    // clear current->thread.on_ustack flag
 587         mov ar.bspstore=r22             // switch to kernel RBS
 588         mov b6=r18                      // copy syscall entry-point to b6 (7 cyc)
 589         add r3=TI_FLAGS+IA64_TASK_SIZE,r2
 590         ;;
 591         ld4 r3=[r3]                             // r2 = current_thread_info()->flags
 592         mov r18=ar.bsp                  // save (kernel) ar.bsp (12 cyc)
 593         mov ar.rsc=0x3                  // set eager mode, pl 0, little-endian, loadrs=0
 594         br.call.sptk.many b7=ia64_syscall_setup
 595         ;;
 596         ssm psr.i
 597         movl r2=ia64_ret_from_syscall
 598         ;;
 599         mov rp=r2                               // set the real return addr
 600         tbit.z p8,p0=r3,TIF_SYSCALL_TRACE
 601
 602 (p8)    br.call.sptk.many b6=b6                 // ignore this return addr
 603         br.cond.sptk ia64_trace_syscall
 604 END(fsys_bubble_down)
 605
 606         .rodata
 607         .align 8
 608         .globl fsyscall_table
 609
 610         data8 fsys_bubble_down
 611 fsyscall_table:
 612         data8 fsys_ni_syscall
 613         data8 0                         // exit                 // 1025
 614         data8 0                         // read
 615         data8 0                         // write
 616         data8 0                         // open
 617         data8 0                         // close
 618         data8 0                         // creat                // 1030
 619         data8 0                         // link
 620         data8 0                         // unlink
 621         data8 0                         // execve
 622         data8 0                         // chdir
 623         data8 0                         // fchdir               // 1035
 624         data8 0                         // utimes
 625         data8 0                         // mknod
 626         data8 0                         // chmod
 627         data8 0                         // chown
 628         data8 0                         // lseek                // 1040
 629         data8 fsys_getpid               // getpid
 630         data8 fsys_getppid              // getppid
 631         data8 0                         // mount
 632         data8 0                         // umount
 633         data8 0                         // setuid               // 1045
 634         data8 0                         // getuid
 635         data8 0                         // geteuid
 636         data8 0                         // ptrace
 637         data8 0                         // access
 638         data8 0                         // sync                 // 1050
 639         data8 0                         // fsync
 640         data8 0                         // fdatasync
 641         data8 0                         // kill
 642         data8 0                         // rename
 643         data8 0                         // mkdir                // 1055
 644         data8 0                         // rmdir
 645         data8 0                         // dup
 646         data8 0                         // pipe
 647         data8 0                         // times
 648         data8 0                         // brk                  // 1060
 649         data8 0                         // setgid
 650         data8 0                         // getgid
 651         data8 0                         // getegid
 652         data8 0                         // acct
 653         data8 0                         // ioctl                // 1065
 654         data8 0                         // fcntl
 655         data8 0                         // umask
 656         data8 0                         // chroot
 657         data8 0                         // ustat
 658         data8 0                         // dup2                 // 1070
 659         data8 0                         // setreuid
 660         data8 0                         // setregid
 661         data8 0                         // getresuid
 662         data8 0                         // setresuid
 663         data8 0                         // getresgid            // 1075
 664         data8 0                         // setresgid
 665         data8 0                         // getgroups
 666         data8 0                         // setgroups
 667         data8 0                         // getpgid
 668         data8 0                         // setpgid              // 1080
 669         data8 0                         // setsid
 670         data8 0                         // getsid
 671         data8 0                         // sethostname
 672         data8 0                         // setrlimit
 673         data8 0                         // getrlimit            // 1085
 674         data8 0                         // getrusage
 675         data8 fsys_gettimeofday         // gettimeofday
 676         data8 0                         // settimeofday
 677         data8 0                         // select
 678         data8 0                         // poll                 // 1090
 679         data8 0                         // symlink
 680         data8 0                         // readlink
 681         data8 0                         // uselib
 682         data8 0                         // swapon
 683         data8 0                         // swapoff              // 1095
 684         data8 0                         // reboot
 685         data8 0                         // truncate
 686         data8 0                         // ftruncate
 687         data8 0                         // fchmod
 688         data8 0                         // fchown               // 1100
 689         data8 0                         // getpriority
 690         data8 0                         // setpriority
 691         data8 0                         // statfs
 692         data8 0                         // fstatfs
 693         data8 0                         // gettid               // 1105
 694         data8 0                         // semget
 695         data8 0                         // semop
 696         data8 0                         // semctl
 697         data8 0                         // msgget
 698         data8 0                         // msgsnd               // 1110
 699         data8 0                         // msgrcv
 700         data8 0                         // msgctl
 701         data8 0                         // shmget
 702         data8 0                         // shmat
 703         data8 0                         // shmdt                // 1115
 704         data8 0                         // shmctl
 705         data8 0                         // syslog
 706         data8 0                         // setitimer
 707         data8 0                         // getitimer
 708         data8 0                                                 // 1120
 709         data8 0
 710         data8 0
 711         data8 0                         // vhangup
 712         data8 0                         // lchown
 713         data8 0                         // remap_file_pages     // 1125
 714         data8 0                         // wait4
 715         data8 0                         // sysinfo
 716         data8 0                         // clone
 717         data8 0                         // setdomainname
 718         data8 0                         // newuname             // 1130
 719         data8 0                         // adjtimex
 720         data8 0
 721         data8 0                         // init_module
 722         data8 0                         // delete_module
 723         data8 0                                                 // 1135
 724         data8 0
 725         data8 0                         // quotactl
 726         data8 0                         // bdflush
 727         data8 0                         // sysfs
 728         data8 0                         // personality          // 1140
 729         data8 0                         // afs_syscall
 730         data8 0                         // setfsuid
 731         data8 0                         // setfsgid
 732         data8 0                         // getdents
 733         data8 0                         // flock                // 1145
 734         data8 0                         // readv
 735         data8 0                         // writev
 736         data8 0                         // pread64
 737         data8 0                         // pwrite64
 738         data8 0                         // sysctl               // 1150
 739         data8 0                         // mmap
 740         data8 0                         // munmap
 741         data8 0                         // mlock
 742         data8 0                         // mlockall
 743         data8 0                         // mprotect             // 1155
 744         data8 0                         // mremap
 745         data8 0                         // msync
 746         data8 0                         // munlock
 747         data8 0                         // munlockall
 748         data8 0                         // sched_getparam       // 1160
 749         data8 0                         // sched_setparam
 750         data8 0                         // sched_getscheduler
 751         data8 0                         // sched_setscheduler
 752         data8 0                         // sched_yield
 753         data8 0                         // sched_get_priority_max       // 1165
 754         data8 0                         // sched_get_priority_min
 755         data8 0                         // sched_rr_get_interval
 756         data8 0                         // nanosleep
 757         data8 0                         // nfsservctl
 758         data8 0                         // prctl                // 1170
 759         data8 0                         // getpagesize
 760         data8 0                         // mmap2
 761         data8 0                         // pciconfig_read
 762         data8 0                         // pciconfig_write
 763         data8 0                         // perfmonctl           // 1175
 764         data8 0                         // sigaltstack
 765         data8 0                         // rt_sigaction
 766         data8 0                         // rt_sigpending
 767         data8 fsys_rt_sigprocmask       // rt_sigprocmask
 768         data8 0                         // rt_sigqueueinfo      // 1180
 769         data8 0                         // rt_sigreturn
 770         data8 0                         // rt_sigsuspend
 771         data8 0                         // rt_sigtimedwait
 772         data8 0                         // getcwd
 773         data8 0                         // capget               // 1185
 774         data8 0                         // capset
 775         data8 0                         // sendfile
 776         data8 0
 777         data8 0
 778         data8 0                         // socket               // 1190
 779         data8 0                         // bind
 780         data8 0                         // connect
 781         data8 0                         // listen
 782         data8 0                         // accept
 783         data8 0                         // getsockname          // 1195
 784         data8 0                         // getpeername
 785         data8 0                         // socketpair
 786         data8 0                         // send
 787         data8 0                         // sendto
 788         data8 0                         // recv                 // 1200
 789         data8 0                         // recvfrom
 790         data8 0                         // shutdown
 791         data8 0                         // setsockopt
 792         data8 0                         // getsockopt
 793         data8 0                         // sendmsg              // 1205
 794         data8 0                         // recvmsg
 795         data8 0                         // pivot_root
 796         data8 0                         // mincore
 797         data8 0                         // madvise
 798         data8 0                         // newstat              // 1210
 799         data8 0                         // newlstat
 800         data8 0                         // newfstat
 801         data8 0                         // clone2
 802         data8 0                         // getdents64
 803         data8 0                         // getunwind            // 1215
 804         data8 0                         // readahead
 805         data8 0                         // setxattr
 806         data8 0                         // lsetxattr
 807         data8 0                         // fsetxattr
 808         data8 0                         // getxattr             // 1220
 809         data8 0                         // lgetxattr
 810         data8 0                         // fgetxattr
 811         data8 0                         // listxattr
 812         data8 0                         // llistxattr
 813         data8 0                         // flistxattr           // 1225
 814         data8 0                         // removexattr
 815         data8 0                         // lremovexattr
 816         data8 0                         // fremovexattr
 817         data8 0                         // tkill
 818         data8 0                         // futex                // 1230
 819         data8 0                         // sched_setaffinity
 820         data8 0                         // sched_getaffinity
 821         data8 fsys_set_tid_address      // set_tid_address
 822         data8 0                         // fadvise64_64
 823         data8 0                         // tgkill               // 1235
 824         data8 0                         // exit_group
 825         data8 0                         // lookup_dcookie
 826         data8 0                         // io_setup
 827         data8 0                         // io_destroy
 828         data8 0                         // io_getevents         // 1240
 829         data8 0                         // io_submit
 830         data8 0                         // io_cancel
 831         data8 0                         // epoll_create
 832         data8 0                         // epoll_ctl
 833         data8 0                         // epoll_wait           // 1245
 834         data8 0                         // restart_syscall
 835         data8 0                         // semtimedop
 836         data8 0                         // timer_create
 837         data8 0                         // timer_settime
 838         data8 0                         // timer_gettime        // 1250
 839         data8 0                         // timer_getoverrun
 840         data8 0                         // timer_delete
 841         data8 0                         // clock_settime
 842         data8 0                         // clock_gettime
 843         data8 0                         // clock_getres         // 1255
 844         data8 0                         // clock_nanosleep
 845         data8 0                         // fstatfs64
 846         data8 0                         // statfs64
 847         data8 0
 848         data8 0                                                 // 1260
 849         data8 0
 850         data8 0                         // mq_open
 851         data8 0                         // mq_unlink
 852         data8 0                         // mq_timedsend
 853         data8 0                         // mq_timedreceive      // 1265
 854         data8 0                         // mq_notify
 855         data8 0                         // mq_getsetattr
 856         data8 0                         // kexec_load
 857         data8 0
 858         data8 0                                                 // 1270
 859         data8 0
 860         data8 0
 861         data8 0
 862         data8 0
 863         data8 0                                                 // 1275
 864         data8 0
 865         data8 0
 866         data8 0
 867         data8 0
 868
 869         .org fsyscall_table + 8*NR_syscalls     // guard against failures to increase NR_syscalls