-#define write_can_lock(x) ((x)->lock == 0)
-
-#ifndef __s390x__
-#define _raw_read_lock(rw) \
- asm volatile(" l 2,0(%1)\n" \
- " j 1f\n" \
- "0: diag 0,0,68\n" \
- "1: la 2,0(2)\n" /* clear high (=write) bit */ \
- " la 3,1(2)\n" /* one more reader */ \
- " cs 2,3,0(%1)\n" /* try to write new value */ \
- " jl 0b" \
- : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#else /* __s390x__ */
-#define _raw_read_lock(rw) \
- asm volatile(" lg 2,0(%1)\n" \
- " j 1f\n" \
- "0: " __DIAG44_INSN " 0,%2\n" \
- "1: nihh 2,0x7fff\n" /* clear high (=write) bit */ \
- " la 3,1(2)\n" /* one more reader */ \
- " csg 2,3,0(%1)\n" /* try to write new value */ \
- " jl 0b" \
- : "=m" ((rw)->lock) \
- : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#endif /* __s390x__ */
-
-#ifndef __s390x__
-#define _raw_read_unlock(rw) \
- asm volatile(" l 2,0(%1)\n" \
- " j 1f\n" \
- "0: diag 0,0,68\n" \
- "1: lr 3,2\n" \
- " ahi 3,-1\n" /* one less reader */ \
- " cs 2,3,0(%1)\n" \
- " jl 0b" \
- : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#else /* __s390x__ */
-#define _raw_read_unlock(rw) \
- asm volatile(" lg 2,0(%1)\n" \
- " j 1f\n" \
- "0: " __DIAG44_INSN " 0,%2\n" \
- "1: lgr 3,2\n" \
- " bctgr 3,0\n" /* one less reader */ \
- " csg 2,3,0(%1)\n" \
- " jl 0b" \
- : "=m" ((rw)->lock) \
- : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#endif /* __s390x__ */
-
-#ifndef __s390x__
-#define _raw_write_lock(rw) \
- asm volatile(" lhi 3,1\n" \
- " sll 3,31\n" /* new lock value = 0x80000000 */ \
- " j 1f\n" \
- "0: diag 0,0,68\n" \
- "1: slr 2,2\n" /* old lock value must be 0 */ \
- " cs 2,3,0(%1)\n" \
- " jl 0b" \
- : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#else /* __s390x__ */
-#define _raw_write_lock(rw) \
- asm volatile(" llihh 3,0x8000\n" /* new lock value = 0x80...0 */ \
- " j 1f\n" \
- "0: " __DIAG44_INSN " 0,%2\n" \
- "1: slgr 2,2\n" /* old lock value must be 0 */ \
- " csg 2,3,0(%1)\n" \
- " jl 0b" \
- : "=m" ((rw)->lock) \
- : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#endif /* __s390x__ */
-
-#ifndef __s390x__
-#define _raw_write_unlock(rw) \
- asm volatile(" slr 3,3\n" /* new lock value = 0 */ \
- " j 1f\n" \
- "0: diag 0,0,68\n" \
- "1: lhi 2,1\n" \
- " sll 2,31\n" /* old lock value must be 0x80000000 */ \
- " cs 2,3,0(%1)\n" \
- " jl 0b" \
- : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#else /* __s390x__ */
-#define _raw_write_unlock(rw) \
- asm volatile(" slgr 3,3\n" /* new lock value = 0 */ \
- " j 1f\n" \
- "0: " __DIAG44_INSN " 0,%2\n" \
- "1: llihh 2,0x8000\n" /* old lock value must be 0x8..0 */\
- " csg 2,3,0(%1)\n" \
- " jl 0b" \
- : "=m" ((rw)->lock) \
- : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
- "m" ((rw)->lock) : "2", "3", "cc", "memory" )
-#endif /* __s390x__ */
-
-#define _raw_read_trylock(lock) generic_raw_read_trylock(lock)
-
-extern inline int _raw_write_trylock(rwlock_t *rw)
+#define __raw_write_can_lock(x) ((x)->lock == 0)
+
+extern void _raw_read_lock_wait(raw_rwlock_t *lp);
+extern int _raw_read_trylock_retry(raw_rwlock_t *lp);
+extern void _raw_write_lock_wait(raw_rwlock_t *lp);
+extern int _raw_write_trylock_retry(raw_rwlock_t *lp);
+
+static inline void __raw_read_lock(raw_rwlock_t *rw)
+{
+ unsigned int old;
+ old = rw->lock & 0x7fffffffU;
+ if (_raw_compare_and_swap(&rw->lock, old, old + 1) != old)
+ _raw_read_lock_wait(rw);
+}
+
+static inline void __raw_read_unlock(raw_rwlock_t *rw)
+{
+ unsigned int old, cmp;
+
+ old = rw->lock;
+ do {
+ cmp = old;
+ old = _raw_compare_and_swap(&rw->lock, old, old - 1);
+ } while (cmp != old);
+}
+
+static inline void __raw_write_lock(raw_rwlock_t *rw)
+{
+ if (unlikely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000) != 0))
+ _raw_write_lock_wait(rw);
+}
+
+static inline void __raw_write_unlock(raw_rwlock_t *rw)
+{
+ _raw_compare_and_swap(&rw->lock, 0x80000000, 0);
+}
+
+static inline int __raw_read_trylock(raw_rwlock_t *rw)
+{
+ unsigned int old;
+ old = rw->lock & 0x7fffffffU;
+ if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1) == old))
+ return 1;
+ return _raw_read_trylock_retry(rw);
+}
+
+static inline int __raw_write_trylock(raw_rwlock_t *rw)