arch/sparc64/lib/rwsem.c

   1 /* rwsem.c: Don't inline expand these suckers all over the place.
   2  *
   3  * Written by David S. Miller (davem@redhat.com), 2001.
   4  * Derived from asm-i386/rwsem.h
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/rwsem.h>
   9 #include <linux/init.h>
  10 #include <linux/module.h>
  11
  12 extern struct rw_semaphore *FASTCALL(rwsem_down_read_failed(struct rw_semaphore *sem));
  13 extern struct rw_semaphore *FASTCALL(rwsem_down_write_failed(struct rw_semaphore *sem));
  14 extern struct rw_semaphore *FASTCALL(rwsem_wake(struct rw_semaphore *));
  15 extern struct rw_semaphore *FASTCALL(rwsem_downgrade_wake(struct rw_semaphore *));
  16
  17 void __sched __down_read(struct rw_semaphore *sem)
  18 {
  19         __asm__ __volatile__(
  20                 "! beginning __down_read\n"
  21                 "1:\tlduw       [%0], %%g5\n\t"
  22                 "add            %%g5, 1, %%g7\n\t"
  23                 "cas            [%0], %%g5, %%g7\n\t"
  24                 "cmp            %%g5, %%g7\n\t"
  25                 "bne,pn         %%icc, 1b\n\t"
  26                 " add           %%g7, 1, %%g7\n\t"
  27                 "cmp            %%g7, 0\n\t"
  28                 "bl,pn          %%icc, 3f\n\t"
  29                 " membar        #StoreLoad | #StoreStore\n"
  30                 "2:\n\t"
  31                 ".subsection    2\n"
  32                 "3:\tmov        %0, %%g5\n\t"
  33                 "save           %%sp, -160, %%sp\n\t"
  34                 "mov            %%g1, %%l1\n\t"
  35                 "mov            %%g2, %%l2\n\t"
  36                 "mov            %%g3, %%l3\n\t"
  37                 "call           %1\n\t"
  38                 " mov           %%g5, %%o0\n\t"
  39                 "mov            %%l1, %%g1\n\t"
  40                 "mov            %%l2, %%g2\n\t"
  41                 "ba,pt          %%xcc, 2b\n\t"
  42                 " restore       %%l3, %%g0, %%g3\n\t"
  43                 ".previous\n\t"
  44                 "! ending __down_read"
  45                 : : "r" (sem), "i" (rwsem_down_read_failed)
  46                 : "g5", "g7", "memory", "cc");
  47 }
  48 EXPORT_SYMBOL(__down_read);
  49
  50 int __down_read_trylock(struct rw_semaphore *sem)
  51 {
  52         int result;
  53
  54         __asm__ __volatile__(
  55                 "! beginning __down_read_trylock\n"
  56                 "1:\tlduw       [%1], %%g5\n\t"
  57                 "add            %%g5, 1, %%g7\n\t"
  58                 "cmp            %%g7, 0\n\t"
  59                 "bl,pn          %%icc, 2f\n\t"
  60                 " mov           0, %0\n\t"
  61                 "cas            [%1], %%g5, %%g7\n\t"
  62                 "cmp            %%g5, %%g7\n\t"
  63                 "bne,pn         %%icc, 1b\n\t"
  64                 " mov           1, %0\n\t"
  65                 "membar         #StoreLoad | #StoreStore\n"
  66                 "2:\n\t"
  67                 "! ending __down_read_trylock"
  68                 : "=&r" (result)
  69                 : "r" (sem)
  70                 : "g5", "g7", "memory", "cc");
  71
  72         return result;
  73 }
  74 EXPORT_SYMBOL(__down_read_trylock);
  75
  76 void __sched __down_write(struct rw_semaphore *sem)
  77 {
  78         __asm__ __volatile__(
  79                 "! beginning __down_write\n\t"
  80                 "sethi          %%hi(%2), %%g1\n\t"
  81                 "or             %%g1, %%lo(%2), %%g1\n"
  82                 "1:\tlduw       [%0], %%g5\n\t"
  83                 "add            %%g5, %%g1, %%g7\n\t"
  84                 "cas            [%0], %%g5, %%g7\n\t"
  85                 "cmp            %%g5, %%g7\n\t"
  86                 "bne,pn         %%icc, 1b\n\t"
  87                 " cmp           %%g7, 0\n\t"
  88                 "bne,pn         %%icc, 3f\n\t"
  89                 " membar        #StoreLoad | #StoreStore\n"
  90                 "2:\n\t"
  91                 ".subsection    2\n"
  92                 "3:\tmov        %0, %%g5\n\t"
  93                 "save           %%sp, -160, %%sp\n\t"
  94                 "mov            %%g2, %%l2\n\t"
  95                 "mov            %%g3, %%l3\n\t"
  96                 "call           %1\n\t"
  97                 " mov           %%g5, %%o0\n\t"
  98                 "mov            %%l2, %%g2\n\t"
  99                 "ba,pt          %%xcc, 2b\n\t"
 100                 " restore       %%l3, %%g0, %%g3\n\t"
 101                 ".previous\n\t"
 102                 "! ending __down_write"
 103                 : : "r" (sem), "i" (rwsem_down_write_failed),
 104                     "i" (RWSEM_ACTIVE_WRITE_BIAS)
 105                 : "g1", "g5", "g7", "memory", "cc");
 106 }
 107 EXPORT_SYMBOL(__down_write);
 108
 109 int __down_write_trylock(struct rw_semaphore *sem)
 110 {
 111         int result;
 112
 113         __asm__ __volatile__(
 114                 "! beginning __down_write_trylock\n\t"
 115                 "sethi          %%hi(%2), %%g1\n\t"
 116                 "or             %%g1, %%lo(%2), %%g1\n"
 117                 "1:\tlduw       [%1], %%g5\n\t"
 118                 "cmp            %%g5, 0\n\t"
 119                 "bne,pn         %%icc, 2f\n\t"
 120                 " mov           0, %0\n\t"
 121                 "add            %%g5, %%g1, %%g7\n\t"
 122                 "cas            [%1], %%g5, %%g7\n\t"
 123                 "cmp            %%g5, %%g7\n\t"
 124                 "bne,pn         %%icc, 1b\n\t"
 125                 " mov           1, %0\n\t"
 126                 "membar         #StoreLoad | #StoreStore\n"
 127                 "2:\n\t"
 128                 "! ending __down_write_trylock"
 129                 : "=&r" (result)
 130                 : "r" (sem), "i" (RWSEM_ACTIVE_WRITE_BIAS)
 131                 : "g1", "g5", "g7", "memory", "cc");
 132
 133         return result;
 134 }
 135 EXPORT_SYMBOL(__down_write_trylock);
 136
 137 void __up_read(struct rw_semaphore *sem)
 138 {
 139         __asm__ __volatile__(
 140                 "! beginning __up_read\n\t"
 141                 "1:\tlduw       [%0], %%g5\n\t"
 142                 "sub            %%g5, 1, %%g7\n\t"
 143                 "cas            [%0], %%g5, %%g7\n\t"
 144                 "cmp            %%g5, %%g7\n\t"
 145                 "bne,pn         %%icc, 1b\n\t"
 146                 " cmp           %%g7, 0\n\t"
 147                 "bl,pn          %%icc, 3f\n\t"
 148                 " membar        #StoreLoad | #StoreStore\n"
 149                 "2:\n\t"
 150                 ".subsection    2\n"
 151                 "3:\tsethi      %%hi(%2), %%g1\n\t"
 152                 "sub            %%g7, 1, %%g7\n\t"
 153                 "or             %%g1, %%lo(%2), %%g1\n\t"
 154                 "andcc          %%g7, %%g1, %%g0\n\t"
 155                 "bne,pn         %%icc, 2b\n\t"
 156                 " mov           %0, %%g5\n\t"
 157                 "save           %%sp, -160, %%sp\n\t"
 158                 "mov            %%g2, %%l2\n\t"
 159                 "mov            %%g3, %%l3\n\t"
 160                 "call           %1\n\t"
 161                 " mov           %%g5, %%o0\n\t"
 162                 "mov            %%l2, %%g2\n\t"
 163                 "ba,pt          %%xcc, 2b\n\t"
 164                 " restore       %%l3, %%g0, %%g3\n\t"
 165                 ".previous\n\t"
 166                 "! ending __up_read"
 167                 : : "r" (sem), "i" (rwsem_wake),
 168                     "i" (RWSEM_ACTIVE_MASK)
 169                 : "g1", "g5", "g7", "memory", "cc");
 170 }
 171 EXPORT_SYMBOL(__up_read);
 172
 173 void __up_write(struct rw_semaphore *sem)
 174 {
 175         __asm__ __volatile__(
 176                 "! beginning __up_write\n\t"
 177                 "sethi          %%hi(%2), %%g1\n\t"
 178                 "or             %%g1, %%lo(%2), %%g1\n"
 179                 "1:\tlduw       [%0], %%g5\n\t"
 180                 "sub            %%g5, %%g1, %%g7\n\t"
 181                 "cas            [%0], %%g5, %%g7\n\t"
 182                 "cmp            %%g5, %%g7\n\t"
 183                 "bne,pn         %%icc, 1b\n\t"
 184                 " sub           %%g7, %%g1, %%g7\n\t"
 185                 "cmp            %%g7, 0\n\t"
 186                 "bl,pn          %%icc, 3f\n\t"
 187                 " membar        #StoreLoad | #StoreStore\n"
 188                 "2:\n\t"
 189                 ".subsection 2\n"
 190                 "3:\tmov        %0, %%g5\n\t"
 191                 "save           %%sp, -160, %%sp\n\t"
 192                 "mov            %%g2, %%l2\n\t"
 193                 "mov            %%g3, %%l3\n\t"
 194                 "call           %1\n\t"
 195                 " mov           %%g5, %%o0\n\t"
 196                 "mov            %%l2, %%g2\n\t"
 197                 "ba,pt          %%xcc, 2b\n\t"
 198                 " restore       %%l3, %%g0, %%g3\n\t"
 199                 ".previous\n\t"
 200                 "! ending __up_write"
 201                 : : "r" (sem), "i" (rwsem_wake),
 202                     "i" (RWSEM_ACTIVE_WRITE_BIAS)
 203                 : "g1", "g5", "g7", "memory", "cc");
 204 }
 205 EXPORT_SYMBOL(__up_write);
 206
 207 void __downgrade_write(struct rw_semaphore *sem)
 208 {
 209         __asm__ __volatile__(
 210                 "! beginning __downgrade_write\n\t"
 211                 "sethi          %%hi(%2), %%g1\n\t"
 212                 "or             %%g1, %%lo(%2), %%g1\n"
 213                 "1:\tlduw       [%0], %%g5\n\t"
 214                 "sub            %%g5, %%g1, %%g7\n\t"
 215                 "cas            [%0], %%g5, %%g7\n\t"
 216                 "cmp            %%g5, %%g7\n\t"
 217                 "bne,pn         %%icc, 1b\n\t"
 218                 " sub           %%g7, %%g1, %%g7\n\t"
 219                 "cmp            %%g7, 0\n\t"
 220                 "bl,pn          %%icc, 3f\n\t"
 221                 " membar        #StoreLoad | #StoreStore\n"
 222                 "2:\n\t"
 223                 ".subsection 2\n"
 224                 "3:\tmov        %0, %%g5\n\t"
 225                 "save           %%sp, -160, %%sp\n\t"
 226                 "mov            %%g2, %%l2\n\t"
 227                 "mov            %%g3, %%l3\n\t"
 228                 "call           %1\n\t"
 229                 " mov           %%g5, %%o0\n\t"
 230                 "mov            %%l2, %%g2\n\t"
 231                 "ba,pt          %%xcc, 2b\n\t"
 232                 " restore       %%l3, %%g0, %%g3\n\t"
 233                 ".previous\n\t"
 234                 "! ending __up_write"
 235                 : : "r" (sem), "i" (rwsem_downgrade_wake),
 236                     "i" (RWSEM_WAITING_BIAS)
 237                 : "g1", "g5", "g7", "memory", "cc");
 238 }
 239 EXPORT_SYMBOL(__downgrade_write);