X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=include%2Fasm-i386%2Fspinlock.h;h=d3bcebed60ca6ef4895b7a22ea139023f4a6d49f;hb=refs%2Fheads%2Fvserver;hp=23604350cdf45e25df90180afa2985f9b489dfd1;hpb=76828883507a47dae78837ab5dec5a5b4513c667;p=linux-2.6.git diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h index 23604350c..d3bcebed6 100644 --- a/include/asm-i386/spinlock.h +++ b/include/asm-i386/spinlock.h @@ -4,9 +4,18 @@ #include #include #include -#include +#include #include +#ifdef CONFIG_PARAVIRT +#include +#else +#define CLI_STRING "cli" +#define STI_STRING "sti" +#define CLI_STI_CLOBBERS +#define CLI_STI_INPUT_ARGS +#endif /* CONFIG_PARAVIRT */ + /* * Your basic SMP spinlocks, allowing only a single CPU anywhere * @@ -18,56 +27,69 @@ * (the type definitions are in asm/spinlock_types.h) */ -#define __raw_spin_is_locked(x) \ - (*(volatile signed char *)(&(x)->slock) <= 0) - -#define __raw_spin_lock_string \ - "\n1:\t" \ - "lock ; decb %0\n\t" \ - "jns 3f\n" \ - "2:\t" \ - "rep;nop\n\t" \ - "cmpb $0,%0\n\t" \ - "jle 2b\n\t" \ - "jmp 1b\n" \ - "3:\n\t" - -#define __raw_spin_lock_string_flags \ - "\n1:\t" \ - "lock ; decb %0\n\t" \ - "jns 4f\n\t" \ - "2:\t" \ - "testl $0x200, %1\n\t" \ - "jz 3f\n\t" \ - "sti\n\t" \ - "3:\t" \ - "rep;nop\n\t" \ - "cmpb $0, %0\n\t" \ - "jle 3b\n\t" \ - "cli\n\t" \ - "jmp 1b\n" \ - "4:\n\t" +static inline int __raw_spin_is_locked(raw_spinlock_t *x) +{ + return *(volatile signed char *)(&(x)->slock) <= 0; +} static inline void __raw_spin_lock(raw_spinlock_t *lock) { - __asm__ __volatile__( - __raw_spin_lock_string - :"=m" (lock->slock) : : "memory"); + asm volatile("\n1:\t" + LOCK_PREFIX " ; decb %0\n\t" + "jns 3f\n" + "2:\t" + "rep;nop\n\t" + "cmpb $0,%0\n\t" + "jle 2b\n\t" + "jmp 1b\n" + "3:\n\t" + : "+m" (lock->slock) : : "memory"); } +/* + * It is easier for the lock validator if interrupts are not re-enabled + * in the middle of a lock-acquire. This is a performance feature anyway + * so we turn it off: + * + * NOTE: there's an irqs-on section here, which normally would have to be + * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant. + */ +#ifndef CONFIG_PROVE_LOCKING static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) { - __asm__ __volatile__( - __raw_spin_lock_string_flags - :"=m" (lock->slock) : "r" (flags) : "memory"); + asm volatile( + "\n1:\t" + LOCK_PREFIX " ; decb %[slock]\n\t" + "jns 5f\n" + "2:\t" + "testl $0x200, %[flags]\n\t" + "jz 4f\n\t" + STI_STRING "\n" + "3:\t" + "rep;nop\n\t" + "cmpb $0, %[slock]\n\t" + "jle 3b\n\t" + CLI_STRING "\n\t" + "jmp 1b\n" + "4:\t" + "rep;nop\n\t" + "cmpb $0, %[slock]\n\t" + "jg 1b\n\t" + "jmp 4b\n" + "5:\n\t" + : [slock] "+m" (lock->slock) + : [flags] "r" (flags) + CLI_STI_INPUT_ARGS + : "memory" CLI_STI_CLOBBERS); } +#endif static inline int __raw_spin_trylock(raw_spinlock_t *lock) { char oldval; - __asm__ __volatile__( + asm volatile( "xchgb %b0,%1" - :"=q" (oldval), "=m" (lock->slock) + :"=q" (oldval), "+m" (lock->slock) :"0" (0) : "memory"); return oldval > 0; } @@ -81,38 +103,29 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) #if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) -#define __raw_spin_unlock_string \ - "movb $1,%0" \ - :"=m" (lock->slock) : : "memory" - - static inline void __raw_spin_unlock(raw_spinlock_t *lock) { - __asm__ __volatile__( - __raw_spin_unlock_string - ); + asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory"); } #else -#define __raw_spin_unlock_string \ - "xchgb %b0, %1" \ - :"=q" (oldval), "=m" (lock->slock) \ - :"0" (oldval) : "memory" - static inline void __raw_spin_unlock(raw_spinlock_t *lock) { char oldval = 1; - __asm__ __volatile__( - __raw_spin_unlock_string - ); + asm volatile("xchgb %b0, %1" + : "=q" (oldval), "+m" (lock->slock) + : "0" (oldval) : "memory"); } #endif -#define __raw_spin_unlock_wait(lock) \ - do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) +{ + while (__raw_spin_is_locked(lock)) + cpu_relax(); +} /* * Read-write spinlocks, allowing multiple readers @@ -139,22 +152,36 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) * read_can_lock - would read_trylock() succeed? * @lock: the rwlock in question. */ -#define __raw_read_can_lock(x) ((int)(x)->lock > 0) +static inline int __raw_read_can_lock(raw_rwlock_t *x) +{ + return (int)(x)->lock > 0; +} /** * write_can_lock - would write_trylock() succeed? * @lock: the rwlock in question. */ -#define __raw_write_can_lock(x) ((x)->lock == RW_LOCK_BIAS) +static inline int __raw_write_can_lock(raw_rwlock_t *x) +{ + return (x)->lock == RW_LOCK_BIAS; +} static inline void __raw_read_lock(raw_rwlock_t *rw) { - __build_read_lock(rw, "__read_lock_failed"); + asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" + "jns 1f\n" + "call __read_lock_failed\n\t" + "1:\n" + ::"a" (rw) : "memory"); } static inline void __raw_write_lock(raw_rwlock_t *rw) { - __build_write_lock(rw, "__write_lock_failed"); + asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" + "jz 1f\n" + "call __write_lock_failed\n\t" + "1:\n" + ::"a" (rw) : "memory"); } static inline int __raw_read_trylock(raw_rwlock_t *lock) @@ -178,13 +205,17 @@ static inline int __raw_write_trylock(raw_rwlock_t *lock) static inline void __raw_read_unlock(raw_rwlock_t *rw) { - asm volatile("lock ; incl %0" :"=m" (rw->lock) : : "memory"); + asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory"); } static inline void __raw_write_unlock(raw_rwlock_t *rw) { - asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ", %0" - : "=m" (rw->lock) : : "memory"); + asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0" + : "+m" (rw->lock) : : "memory"); } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */