X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=include%2Fasm-i386%2Fspinlock.h;h=d3bcebed60ca6ef4895b7a22ea139023f4a6d49f;hb=97bf2856c6014879bd04983a3e9dfcdac1e7fe85;hp=89ad2d7b58ad16ef5202767208d569b4cdacca23;hpb=c7b5ebbddf7bcd3651947760f423e3783bbe6573;p=linux-2.6.git diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h index 89ad2d7b5..d3bcebed6 100644 --- a/include/asm-i386/spinlock.h +++ b/include/asm-i386/spinlock.h @@ -4,151 +4,127 @@ #include #include #include -#include +#include #include -asmlinkage int printk(const char * fmt, ...) - __attribute__ ((format (printf, 1, 2))); +#ifdef CONFIG_PARAVIRT +#include +#else +#define CLI_STRING "cli" +#define STI_STRING "sti" +#define CLI_STI_CLOBBERS +#define CLI_STI_INPUT_ARGS +#endif /* CONFIG_PARAVIRT */ /* * Your basic SMP spinlocks, allowing only a single CPU anywhere + * + * Simple spin lock operations. There are two variants, one clears IRQ's + * on the local processor, one does not. + * + * We make no fairness assumptions. They have a cost. + * + * (the type definitions are in asm/spinlock_types.h) */ -typedef struct { - volatile unsigned int lock; -#ifdef CONFIG_DEBUG_SPINLOCK - unsigned magic; -#endif -} spinlock_t; - -#define SPINLOCK_MAGIC 0xdead4ead - -#ifdef CONFIG_DEBUG_SPINLOCK -#define SPINLOCK_MAGIC_INIT , SPINLOCK_MAGIC -#else -#define SPINLOCK_MAGIC_INIT /* */ -#endif - -#define SPIN_LOCK_UNLOCKED (spinlock_t) { 1 SPINLOCK_MAGIC_INIT } +static inline int __raw_spin_is_locked(raw_spinlock_t *x) +{ + return *(volatile signed char *)(&(x)->slock) <= 0; +} -#define spin_lock_init(x) do { *(x) = SPIN_LOCK_UNLOCKED; } while(0) +static inline void __raw_spin_lock(raw_spinlock_t *lock) +{ + asm volatile("\n1:\t" + LOCK_PREFIX " ; decb %0\n\t" + "jns 3f\n" + "2:\t" + "rep;nop\n\t" + "cmpb $0,%0\n\t" + "jle 2b\n\t" + "jmp 1b\n" + "3:\n\t" + : "+m" (lock->slock) : : "memory"); +} /* - * Simple spin lock operations. There are two variants, one clears IRQ's - * on the local processor, one does not. + * It is easier for the lock validator if interrupts are not re-enabled + * in the middle of a lock-acquire. This is a performance feature anyway + * so we turn it off: * - * We make no fairness assumptions. They have a cost. + * NOTE: there's an irqs-on section here, which normally would have to be + * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant. */ +#ifndef CONFIG_PROVE_LOCKING +static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) +{ + asm volatile( + "\n1:\t" + LOCK_PREFIX " ; decb %[slock]\n\t" + "jns 5f\n" + "2:\t" + "testl $0x200, %[flags]\n\t" + "jz 4f\n\t" + STI_STRING "\n" + "3:\t" + "rep;nop\n\t" + "cmpb $0, %[slock]\n\t" + "jle 3b\n\t" + CLI_STRING "\n\t" + "jmp 1b\n" + "4:\t" + "rep;nop\n\t" + "cmpb $0, %[slock]\n\t" + "jg 1b\n\t" + "jmp 4b\n" + "5:\n\t" + : [slock] "+m" (lock->slock) + : [flags] "r" (flags) + CLI_STI_INPUT_ARGS + : "memory" CLI_STI_CLOBBERS); +} +#endif -#define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0) -#define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x)) - -#define spin_lock_string \ - "\n1:\t" \ - "lock ; decb %0\n\t" \ - "jns 3f\n" \ - "2:\t" \ - "rep;nop\n\t" \ - "cmpb $0,%0\n\t" \ - "jle 2b\n\t" \ - "jmp 1b\n" \ - "3:\n\t" - -#define spin_lock_string_flags \ - "\n1:\t" \ - "lock ; decb %0\n\t" \ - "jns 4f\n\t" \ - "2:\t" \ - "testl $0x200, %1\n\t" \ - "jz 3f\n\t" \ - "sti\n\t" \ - "3:\t" \ - "rep;nop\n\t" \ - "cmpb $0, %0\n\t" \ - "jle 3b\n\t" \ - "cli\n\t" \ - "jmp 1b\n" \ - "4:\n\t" +static inline int __raw_spin_trylock(raw_spinlock_t *lock) +{ + char oldval; + asm volatile( + "xchgb %b0,%1" + :"=q" (oldval), "+m" (lock->slock) + :"0" (0) : "memory"); + return oldval > 0; +} /* - * This works. Despite all the confusion. - * (except on PPro SMP or if we are using OOSTORE) + * __raw_spin_unlock based on writing $1 to the low byte. + * This method works. Despite all the confusion. + * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there) * (PPro errata 66, 92) */ - -#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) - -#define spin_unlock_string \ - "movb $1,%0" \ - :"=m" (lock->lock) : : "memory" +#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) -static inline void _raw_spin_unlock(spinlock_t *lock) +static inline void __raw_spin_unlock(raw_spinlock_t *lock) { -#ifdef CONFIG_DEBUG_SPINLOCK - BUG_ON(lock->magic != SPINLOCK_MAGIC); - BUG_ON(!spin_is_locked(lock)); -#endif - __asm__ __volatile__( - spin_unlock_string - ); + asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory"); } #else -#define spin_unlock_string \ - "xchgb %b0, %1" \ - :"=q" (oldval), "=m" (lock->lock) \ - :"0" (oldval) : "memory" - -static inline void _raw_spin_unlock(spinlock_t *lock) +static inline void __raw_spin_unlock(raw_spinlock_t *lock) { char oldval = 1; -#ifdef CONFIG_DEBUG_SPINLOCK - BUG_ON(lock->magic != SPINLOCK_MAGIC); - BUG_ON(!spin_is_locked(lock)); -#endif - __asm__ __volatile__( - spin_unlock_string - ); -} -#endif - -static inline int _raw_spin_trylock(spinlock_t *lock) -{ - char oldval; - __asm__ __volatile__( - "xchgb %b0,%1" - :"=q" (oldval), "=m" (lock->lock) - :"0" (0) : "memory"); - return oldval > 0; + asm volatile("xchgb %b0, %1" + : "=q" (oldval), "+m" (lock->slock) + : "0" (oldval) : "memory"); } -static inline void _raw_spin_lock(spinlock_t *lock) -{ -#ifdef CONFIG_DEBUG_SPINLOCK - if (unlikely(lock->magic != SPINLOCK_MAGIC)) { - printk("eip: %p\n", __builtin_return_address(0)); - BUG(); - } #endif - __asm__ __volatile__( - spin_lock_string - :"=m" (lock->lock) : : "memory"); -} -static inline void _raw_spin_lock_flags (spinlock_t *lock, unsigned long flags) +static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) { -#ifdef CONFIG_DEBUG_SPINLOCK - if (unlikely(lock->magic != SPINLOCK_MAGIC)) { - printk("eip: %p\n", __builtin_return_address(0)); - BUG(); - } -#endif - __asm__ __volatile__( - spin_lock_string_flags - :"=m" (lock->lock) : "r" (flags) : "memory"); + while (__raw_spin_is_locked(lock)) + cpu_relax(); } /* @@ -160,29 +136,7 @@ static inline void _raw_spin_lock_flags (spinlock_t *lock, unsigned long flags) * can "mix" irq-safe locks - any writer needs to get a * irq-safe write-lock, but readers can get non-irqsafe * read-locks. - */ -typedef struct { - volatile unsigned int lock; -#ifdef CONFIG_DEBUG_SPINLOCK - unsigned magic; -#endif -} rwlock_t; - -#define RWLOCK_MAGIC 0xdeaf1eed - -#ifdef CONFIG_DEBUG_SPINLOCK -#define RWLOCK_MAGIC_INIT , RWLOCK_MAGIC -#else -#define RWLOCK_MAGIC_INIT /* */ -#endif - -#define RW_LOCK_UNLOCKED (rwlock_t) { RW_LOCK_BIAS RWLOCK_MAGIC_INIT } - -#define rwlock_init(x) do { *(x) = RW_LOCK_UNLOCKED; } while(0) - -#define rwlock_is_locked(x) ((x)->lock != RW_LOCK_BIAS) - -/* + * * On x86, we implement read-write locks as a 32-bit counter * with the high bit (sign) being the "contended" bit. * @@ -190,29 +144,57 @@ typedef struct { * * Changed to use the same technique as rw semaphores. See * semaphore.h for details. -ben + * + * the helpers are in arch/i386/kernel/semaphore.c */ -/* the spinlock helpers are in arch/i386/kernel/semaphore.c */ -static inline void _raw_read_lock(rwlock_t *rw) +/** + * read_can_lock - would read_trylock() succeed? + * @lock: the rwlock in question. + */ +static inline int __raw_read_can_lock(raw_rwlock_t *x) { -#ifdef CONFIG_DEBUG_SPINLOCK - BUG_ON(rw->magic != RWLOCK_MAGIC); -#endif - __build_read_lock(rw, "__read_lock_failed"); + return (int)(x)->lock > 0; } -static inline void _raw_write_lock(rwlock_t *rw) +/** + * write_can_lock - would write_trylock() succeed? + * @lock: the rwlock in question. + */ +static inline int __raw_write_can_lock(raw_rwlock_t *x) { -#ifdef CONFIG_DEBUG_SPINLOCK - BUG_ON(rw->magic != RWLOCK_MAGIC); -#endif - __build_write_lock(rw, "__write_lock_failed"); + return (x)->lock == RW_LOCK_BIAS; +} + +static inline void __raw_read_lock(raw_rwlock_t *rw) +{ + asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" + "jns 1f\n" + "call __read_lock_failed\n\t" + "1:\n" + ::"a" (rw) : "memory"); } -#define _raw_read_unlock(rw) asm volatile("lock ; incl %0" :"=m" ((rw)->lock) : : "memory") -#define _raw_write_unlock(rw) asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory") +static inline void __raw_write_lock(raw_rwlock_t *rw) +{ + asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" + "jz 1f\n" + "call __write_lock_failed\n\t" + "1:\n" + ::"a" (rw) : "memory"); +} + +static inline int __raw_read_trylock(raw_rwlock_t *lock) +{ + atomic_t *count = (atomic_t *)lock; + atomic_dec(count); + if (atomic_read(count) >= 0) + return 1; + atomic_inc(count); + return 0; +} -static inline int _raw_write_trylock(rwlock_t *lock) +static inline int __raw_write_trylock(raw_rwlock_t *lock) { atomic_t *count = (atomic_t *)lock; if (atomic_sub_and_test(RW_LOCK_BIAS, count)) @@ -221,4 +203,19 @@ static inline int _raw_write_trylock(rwlock_t *lock) return 0; } +static inline void __raw_read_unlock(raw_rwlock_t *rw) +{ + asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory"); +} + +static inline void __raw_write_unlock(raw_rwlock_t *rw) +{ + asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0" + : "+m" (rw->lock) : : "memory"); +} + +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */