patch-2_6_7-vs1_9_1_12

[linux-2.6.git] / arch / ia64 / kernel / head.S

diff --git a/arch/ia64/kernel/head.S b/arch/ia64/kernel/head.S
index b2f67f1..0a5eb48 100644 (file)
--- a/arch/ia64/kernel/head.S
+++ b/arch/ia64/kernel/head.S
@@ -56,8 +56,7 @@ halt_msg:
 GLOBAL_ENTRY(_start)
 start_ap:
        .prologue
-       .save rp, r4            // terminate unwind chain with a NULL rp
-       mov r4=r0
+       .save rp, r0            // terminate unwind chain with a NULL rp
        .body
 
        rsm psr.i | psr.ic
@@ -816,6 +815,42 @@ GLOBAL_ENTRY(ia64_delay_loop)
        br.ret.sptk.many rp
 END(ia64_delay_loop)
 
+/*
+ * Return a CPU-local timestamp in nano-seconds.  This timestamp is
+ * NOT synchronized across CPUs its return value must never be
+ * compared against the values returned on another CPU.  The usage in
+ * kernel/sched.c ensures that.
+ *
+ * The return-value of sched_clock() is NOT supposed to wrap-around.
+ * If it did, it would cause some scheduling hiccups (at the worst).
+ * Fortunately, with a 64-bit cycle-counter ticking at 100GHz, even
+ * that would happen only once every 5+ years.
+ *
+ * The code below basically calculates:
+ *
+ *   (ia64_get_itc() * local_cpu_data->nsec_per_cyc) >> IA64_NSEC_PER_CYC_SHIFT
+ *
+ * except that the multiplication and the shift are done with 128-bit
+ * intermediate precision so that we can produce a full 64-bit result.
+ */
+GLOBAL_ENTRY(sched_clock)
+       addl r8=THIS_CPU(cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0
+       mov.m r9=ar.itc         // fetch cycle-counter                          (35 cyc)
+       ;;
+       ldf8 f8=[r8]
+       ;;
+       setf.sig f9=r9          // certain to stall, so issue it _after_ ldf8...
+       ;;
+       xmpy.lu f10=f9,f8       // calculate low 64 bits of 128-bit product     (4 cyc)
+       xmpy.hu f11=f9,f8       // calculate high 64 bits of 128-bit product
+       ;;
+       getf.sig r8=f10         //                                              (5 cyc)
+       getf.sig r9=f11
+       ;;
+       shrp r8=r9,r8,IA64_NSEC_PER_CYC_SHIFT
+       br.ret.sptk.many rp
+END(sched_clock)
+
 GLOBAL_ENTRY(start_kernel_thread)
        .prologue
        .save rp, r0                            // this is the end of the call-chain
@@ -866,12 +901,14 @@ SET_REG(b5);
         * Inputs:
         *   ar.pfs - saved CFM of caller
         *   ar.ccv - 0 (and available for use)
+        *   r27    - flags from spin_lock_irqsave or 0.  Must be preserved.
         *   r28    - available for use.
         *   r29    - available for use.
         *   r30    - available for use.
         *   r31    - address of lock, available for use.
         *   b6     - return address
         *   p14    - available for use.
+        *   p15    - used to track flag status.
         *
         * If you patch this code to use more registers, do not forget to update
         * the clobber lists for spin_lock() in include/asm-ia64/spinlock.h.
@@ -885,22 +922,26 @@ GLOBAL_ENTRY(ia64_spinlock_contention_pre3_4)
        .save rp, r28
        .body
        nop 0
-       nop 0
+       tbit.nz p15,p0=r27,IA64_PSR_I_BIT
        .restore sp             // pop existing prologue after next insn
        mov b6 = r28
        .prologue
        .save ar.pfs, r0
        .altrp b6
        .body
+       ;;
+(p15)  ssm psr.i               // reenable interrupts if they were on
+                               // DavidM says that srlz.d is slow and is not required in this case
 .wait:
        // exponential backoff, kdb, lockmeter etc. go in here
        hint @pause
        ld4 r30=[r31]           // don't use ld4.bias; if it's contended, we won't write the word
        nop 0
        ;;
-       cmp4.eq p14,p0=r30,r0
-(p14)  br.cond.sptk.few b6     // lock is now free, try to acquire
-       br.cond.sptk.few .wait
+       cmp4.ne p14,p0=r30,r0
+(p14)  br.cond.sptk.few .wait
+(p15)  rsm psr.i               // disable interrupts if we reenabled them
+       br.cond.sptk.few b6     // lock is now free, try to acquire
 END(ia64_spinlock_contention_pre3_4)
 
 #else
@@ -909,14 +950,20 @@ GLOBAL_ENTRY(ia64_spinlock_contention)
        .prologue
        .altrp b6
        .body
+       tbit.nz p15,p0=r27,IA64_PSR_I_BIT
+       ;;
 .wait:
+(p15)  ssm psr.i               // reenable interrupts if they were on
+                               // DavidM says that srlz.d is slow and is not required in this case
+.wait2:
        // exponential backoff, kdb, lockmeter etc. go in here
        hint @pause
        ld4 r30=[r31]           // don't use ld4.bias; if it's contended, we won't write the word
        ;;
        cmp4.ne p14,p0=r30,r0
        mov r30 = 1
-(p14)  br.cond.sptk.few .wait
+(p14)  br.cond.sptk.few .wait2
+(p15)  rsm psr.i               // disable interrupts if we reenabled them
        ;;
        cmpxchg4.acq r30=[r31], r30, ar.ccv
        ;;