X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=include%2Flinux%2Fspinlock.h;h=d6ba068719b61bf4a035e4d6e297694b65941442;hb=f7f1b0f1e2fbadeab12d24236000e778aa9b1ead;hp=148ea9d43873e28b0aa2fb1a912494e4d725c7e8;hpb=5273a3df6485dc2ad6aa7ddd441b9a21970f003b;p=linux-2.6.git diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 148ea9d43..d6ba06871 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -19,29 +19,72 @@ /* * Must define these before including other files, inline functions need them */ -#define LOCK_SECTION_NAME \ - ".text.lock." __stringify(KBUILD_BASENAME) +#define LOCK_SECTION_NAME \ + ".text.lock." __stringify(KBUILD_BASENAME) -#define LOCK_SECTION_START(extra) \ - ".subsection 1\n\t" \ - extra \ - ".ifndef " LOCK_SECTION_NAME "\n\t" \ - LOCK_SECTION_NAME ":\n\t" \ - ".endif\n\t" +#define LOCK_SECTION_START(extra) \ + ".subsection 1\n\t" \ + extra \ + ".ifndef " LOCK_SECTION_NAME "\n\t" \ + LOCK_SECTION_NAME ":\n\t" \ + ".endif\n" -#define LOCK_SECTION_END \ - ".previous\n\t" +#define LOCK_SECTION_END \ + ".previous\n\t" + +#define __lockfunc fastcall __attribute__((section(".spinlock.text"))) /* * If CONFIG_SMP is set, pull in the _raw_* definitions */ #ifdef CONFIG_SMP + +#define assert_spin_locked(x) BUG_ON(!spin_is_locked(x)) #include +int __lockfunc _spin_trylock(spinlock_t *lock); +int __lockfunc _read_trylock(rwlock_t *lock); +int __lockfunc _write_trylock(rwlock_t *lock); + +void __lockfunc _spin_lock(spinlock_t *lock) __acquires(spinlock_t); +void __lockfunc _read_lock(rwlock_t *lock) __acquires(rwlock_t); +void __lockfunc _write_lock(rwlock_t *lock) __acquires(rwlock_t); + +void __lockfunc _spin_unlock(spinlock_t *lock) __releases(spinlock_t); +void __lockfunc _read_unlock(rwlock_t *lock) __releases(rwlock_t); +void __lockfunc _write_unlock(rwlock_t *lock) __releases(rwlock_t); + +unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) __acquires(spinlock_t); +unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) __acquires(rwlock_t); +unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) __acquires(rwlock_t); + +void __lockfunc _spin_lock_irq(spinlock_t *lock) __acquires(spinlock_t); +void __lockfunc _spin_lock_bh(spinlock_t *lock) __acquires(spinlock_t); +void __lockfunc _read_lock_irq(rwlock_t *lock) __acquires(rwlock_t); +void __lockfunc _read_lock_bh(rwlock_t *lock) __acquires(rwlock_t); +void __lockfunc _write_lock_irq(rwlock_t *lock) __acquires(rwlock_t); +void __lockfunc _write_lock_bh(rwlock_t *lock) __acquires(rwlock_t); + +void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(spinlock_t); +void __lockfunc _spin_unlock_irq(spinlock_t *lock) __releases(spinlock_t); +void __lockfunc _spin_unlock_bh(spinlock_t *lock) __releases(spinlock_t); +void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) __releases(rwlock_t); +void __lockfunc _read_unlock_irq(rwlock_t *lock) __releases(rwlock_t); +void __lockfunc _read_unlock_bh(rwlock_t *lock) __releases(rwlock_t); +void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) __releases(rwlock_t); +void __lockfunc _write_unlock_irq(rwlock_t *lock) __releases(rwlock_t); +void __lockfunc _write_unlock_bh(rwlock_t *lock) __releases(rwlock_t); + +int __lockfunc _spin_trylock_bh(spinlock_t *lock); +int __lockfunc generic_raw_read_trylock(rwlock_t *lock); +int in_lock_functions(unsigned long addr); + #else +#define in_lock_functions(ADDR) 0 + #if !defined(CONFIG_PREEMPT) && !defined(CONFIG_DEBUG_SPINLOCK) -# define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic) +# define _atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic) # define ATOMIC_DEC_AND_LOCK #endif @@ -104,6 +147,14 @@ typedef struct { 0; \ }) +/* with debugging, assert_spin_locked() on UP does check + * the lock value properly */ +#define assert_spin_locked(x) \ + ({ \ + CHECK_LOCK(x); \ + BUG_ON(!(x)->lock); \ + }) + /* without debugging, spin_trylock on UP always says * TRUE. --> printk if already locked. */ #define _raw_spin_trylock(x) \ @@ -160,9 +211,10 @@ typedef struct { #define spin_lock_init(lock) do { (void)(lock); } while(0) #define _raw_spin_lock(lock) do { (void)(lock); } while(0) #define spin_is_locked(lock) ((void)(lock), 0) -#define _raw_spin_trylock(lock) ((void)(lock), 1) -#define spin_unlock_wait(lock) do { (void)(lock); } while(0) -#define _raw_spin_unlock(lock) do { (void)(lock); } while(0) +#define assert_spin_locked(lock) do { (void)(lock); } while(0) +#define _raw_spin_trylock(lock) (((void)(lock), 1)) +#define spin_unlock_wait(lock) (void)(lock) +#define _raw_spin_unlock(lock) do { (void)(lock); } while(0) #endif /* CONFIG_DEBUG_SPINLOCK */ /* RW spinlocks: No debug version */ @@ -180,221 +232,296 @@ typedef struct { #define _raw_read_unlock(lock) do { (void)(lock); } while(0) #define _raw_write_lock(lock) do { (void)(lock); } while(0) #define _raw_write_unlock(lock) do { (void)(lock); } while(0) +#define read_can_lock(lock) (((void)(lock), 1)) +#define write_can_lock(lock) (((void)(lock), 1)) +#define _raw_read_trylock(lock) ({ (void)(lock); (1); }) #define _raw_write_trylock(lock) ({ (void)(lock); (1); }) -#endif /* !SMP */ - -/* - * Define the various spin_lock and rw_lock methods. Note we define these - * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various - * methods are defined as nops in the case they are not required. - */ -#define spin_trylock(lock) ({preempt_disable(); _raw_spin_trylock(lock) ? \ +#define _spin_trylock(lock) ({preempt_disable(); _raw_spin_trylock(lock) ? \ 1 : ({preempt_enable(); 0;});}) -#define write_trylock(lock) ({preempt_disable();_raw_write_trylock(lock) ? \ +#define _read_trylock(lock) ({preempt_disable();_raw_read_trylock(lock) ? \ 1 : ({preempt_enable(); 0;});}) -/* Where's read_trylock? */ - -#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) -void __preempt_spin_lock(spinlock_t *lock); -void __preempt_write_lock(rwlock_t *lock); - -#define spin_lock(lock) \ -do { \ - preempt_disable(); \ - if (unlikely(!_raw_spin_trylock(lock))) \ - __preempt_spin_lock(lock); \ -} while (0) +#define _write_trylock(lock) ({preempt_disable(); _raw_write_trylock(lock) ? \ + 1 : ({preempt_enable(); 0;});}) -#define write_lock(lock) \ -do { \ - preempt_disable(); \ - if (unlikely(!_raw_write_trylock(lock))) \ - __preempt_write_lock(lock); \ -} while (0) +#define _spin_trylock_bh(lock) ({preempt_disable(); local_bh_disable(); \ + _raw_spin_trylock(lock) ? \ + 1 : ({preempt_enable_no_resched(); local_bh_enable(); 0;});}) -#else -#define spin_lock(lock) \ +#define _spin_lock(lock) \ do { \ preempt_disable(); \ _raw_spin_lock(lock); \ + __acquire(lock); \ } while(0) -#define write_lock(lock) \ +#define _write_lock(lock) \ do { \ preempt_disable(); \ _raw_write_lock(lock); \ + __acquire(lock); \ } while(0) -#endif - -#define read_lock(lock) \ + +#define _read_lock(lock) \ do { \ preempt_disable(); \ _raw_read_lock(lock); \ + __acquire(lock); \ } while(0) -#define spin_unlock(lock) \ +#define _spin_unlock(lock) \ do { \ _raw_spin_unlock(lock); \ preempt_enable(); \ + __release(lock); \ } while (0) -#define write_unlock(lock) \ +#define _write_unlock(lock) \ do { \ _raw_write_unlock(lock); \ preempt_enable(); \ + __release(lock); \ } while(0) -#define read_unlock(lock) \ +#define _read_unlock(lock) \ do { \ _raw_read_unlock(lock); \ preempt_enable(); \ + __release(lock); \ } while(0) -#define spin_lock_irqsave(lock, flags) \ -do { \ +#define _spin_lock_irqsave(lock, flags) \ +do { \ local_irq_save(flags); \ preempt_disable(); \ _raw_spin_lock(lock); \ + __acquire(lock); \ } while (0) -#define spin_lock_irq(lock) \ +#define _spin_lock_irq(lock) \ do { \ local_irq_disable(); \ preempt_disable(); \ _raw_spin_lock(lock); \ + __acquire(lock); \ } while (0) -#define spin_lock_bh(lock) \ +#define _spin_lock_bh(lock) \ do { \ local_bh_disable(); \ preempt_disable(); \ _raw_spin_lock(lock); \ + __acquire(lock); \ } while (0) -#define read_lock_irqsave(lock, flags) \ -do { \ +#define _read_lock_irqsave(lock, flags) \ +do { \ local_irq_save(flags); \ preempt_disable(); \ _raw_read_lock(lock); \ + __acquire(lock); \ } while (0) -#define read_lock_irq(lock) \ +#define _read_lock_irq(lock) \ do { \ local_irq_disable(); \ preempt_disable(); \ _raw_read_lock(lock); \ + __acquire(lock); \ } while (0) -#define read_lock_bh(lock) \ +#define _read_lock_bh(lock) \ do { \ local_bh_disable(); \ preempt_disable(); \ _raw_read_lock(lock); \ + __acquire(lock); \ } while (0) -#define write_lock_irqsave(lock, flags) \ -do { \ +#define _write_lock_irqsave(lock, flags) \ +do { \ local_irq_save(flags); \ preempt_disable(); \ _raw_write_lock(lock); \ + __acquire(lock); \ } while (0) -#define write_lock_irq(lock) \ +#define _write_lock_irq(lock) \ do { \ local_irq_disable(); \ preempt_disable(); \ _raw_write_lock(lock); \ + __acquire(lock); \ } while (0) -#define write_lock_bh(lock) \ +#define _write_lock_bh(lock) \ do { \ local_bh_disable(); \ preempt_disable(); \ _raw_write_lock(lock); \ + __acquire(lock); \ } while (0) -#define spin_unlock_irqrestore(lock, flags) \ +#define _spin_unlock_irqrestore(lock, flags) \ do { \ _raw_spin_unlock(lock); \ local_irq_restore(flags); \ preempt_enable(); \ + __release(lock); \ } while (0) -#define _raw_spin_unlock_irqrestore(lock, flags) \ -do { \ - _raw_spin_unlock(lock); \ - local_irq_restore(flags); \ -} while (0) - -#define spin_unlock_irq(lock) \ +#define _spin_unlock_irq(lock) \ do { \ _raw_spin_unlock(lock); \ local_irq_enable(); \ preempt_enable(); \ + __release(lock); \ } while (0) -#define spin_unlock_bh(lock) \ +#define _spin_unlock_bh(lock) \ do { \ _raw_spin_unlock(lock); \ - preempt_enable(); \ + preempt_enable_no_resched(); \ local_bh_enable(); \ + __release(lock); \ } while (0) -#define read_unlock_irqrestore(lock, flags) \ +#define _write_unlock_bh(lock) \ do { \ - _raw_read_unlock(lock); \ - local_irq_restore(flags); \ - preempt_enable(); \ + _raw_write_unlock(lock); \ + preempt_enable_no_resched(); \ + local_bh_enable(); \ + __release(lock); \ } while (0) -#define read_unlock_irq(lock) \ +#define _read_unlock_irqrestore(lock, flags) \ do { \ _raw_read_unlock(lock); \ - local_irq_enable(); \ + local_irq_restore(flags); \ preempt_enable(); \ + __release(lock); \ } while (0) -#define read_unlock_bh(lock) \ +#define _write_unlock_irqrestore(lock, flags) \ do { \ - _raw_read_unlock(lock); \ + _raw_write_unlock(lock); \ + local_irq_restore(flags); \ preempt_enable(); \ - local_bh_enable(); \ + __release(lock); \ } while (0) -#define write_unlock_irqrestore(lock, flags) \ +#define _read_unlock_irq(lock) \ do { \ - _raw_write_unlock(lock); \ - local_irq_restore(flags); \ - preempt_enable(); \ + _raw_read_unlock(lock); \ + local_irq_enable(); \ + preempt_enable(); \ + __release(lock); \ } while (0) -#define write_unlock_irq(lock) \ +#define _read_unlock_bh(lock) \ do { \ - _raw_write_unlock(lock); \ - local_irq_enable(); \ - preempt_enable(); \ + _raw_read_unlock(lock); \ + preempt_enable_no_resched(); \ + local_bh_enable(); \ + __release(lock); \ } while (0) -#define write_unlock_bh(lock) \ +#define _write_unlock_irq(lock) \ do { \ - _raw_write_unlock(lock); \ - preempt_enable(); \ - local_bh_enable(); \ + _raw_write_unlock(lock); \ + local_irq_enable(); \ + preempt_enable(); \ + __release(lock); \ } while (0) -#define spin_trylock_bh(lock) ({ local_bh_disable(); preempt_disable(); \ - _raw_spin_trylock(lock) ? 1 : \ - ({preempt_enable(); local_bh_enable(); 0;});}) +#endif /* !SMP */ + +/* + * Define the various spin_lock and rw_lock methods. Note we define these + * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various + * methods are defined as nops in the case they are not required. + */ +#define spin_trylock(lock) __cond_lock(_spin_trylock(lock)) +#define read_trylock(lock) __cond_lock(_read_trylock(lock)) +#define write_trylock(lock) __cond_lock(_write_trylock(lock)) + +#define spin_lock(lock) _spin_lock(lock) +#define write_lock(lock) _write_lock(lock) +#define read_lock(lock) _read_lock(lock) + +#ifdef CONFIG_SMP +#define spin_lock_irqsave(lock, flags) flags = _spin_lock_irqsave(lock) +#define read_lock_irqsave(lock, flags) flags = _read_lock_irqsave(lock) +#define write_lock_irqsave(lock, flags) flags = _write_lock_irqsave(lock) +#else +#define spin_lock_irqsave(lock, flags) _spin_lock_irqsave(lock, flags) +#define read_lock_irqsave(lock, flags) _read_lock_irqsave(lock, flags) +#define write_lock_irqsave(lock, flags) _write_lock_irqsave(lock, flags) +#endif + +#define spin_lock_irq(lock) _spin_lock_irq(lock) +#define spin_lock_bh(lock) _spin_lock_bh(lock) + +#define read_lock_irq(lock) _read_lock_irq(lock) +#define read_lock_bh(lock) _read_lock_bh(lock) + +#define write_lock_irq(lock) _write_lock_irq(lock) +#define write_lock_bh(lock) _write_lock_bh(lock) + +#define spin_unlock(lock) _spin_unlock(lock) +#define write_unlock(lock) _write_unlock(lock) +#define read_unlock(lock) _read_unlock(lock) + +#define spin_unlock_irqrestore(lock, flags) _spin_unlock_irqrestore(lock, flags) +#define spin_unlock_irq(lock) _spin_unlock_irq(lock) +#define spin_unlock_bh(lock) _spin_unlock_bh(lock) + +#define read_unlock_irqrestore(lock, flags) _read_unlock_irqrestore(lock, flags) +#define read_unlock_irq(lock) _read_unlock_irq(lock) +#define read_unlock_bh(lock) _read_unlock_bh(lock) + +#define write_unlock_irqrestore(lock, flags) _write_unlock_irqrestore(lock, flags) +#define write_unlock_irq(lock) _write_unlock_irq(lock) +#define write_unlock_bh(lock) _write_unlock_bh(lock) + +#define spin_trylock_bh(lock) __cond_lock(_spin_trylock_bh(lock)) + +#define spin_trylock_irq(lock) \ +({ \ + local_irq_disable(); \ + _spin_trylock(lock) ? \ + 1 : ({local_irq_enable(); 0; }); \ +}) + +#define spin_trylock_irqsave(lock, flags) \ +({ \ + local_irq_save(flags); \ + _spin_trylock(lock) ? \ + 1 : ({local_irq_restore(flags); 0;}); \ +}) + +#ifdef CONFIG_LOCKMETER +extern void _metered_spin_lock (spinlock_t *lock); +extern void _metered_spin_unlock (spinlock_t *lock); +extern int _metered_spin_trylock(spinlock_t *lock); +extern void _metered_read_lock (rwlock_t *lock); +extern void _metered_read_unlock (rwlock_t *lock); +extern void _metered_write_lock (rwlock_t *lock); +extern void _metered_write_unlock (rwlock_t *lock); +extern int _metered_read_trylock (rwlock_t *lock); +extern int _metered_write_trylock(rwlock_t *lock); +#endif /* "lock on reference count zero" */ #ifndef ATOMIC_DEC_AND_LOCK #include -extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock); +extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock); #endif +#define atomic_dec_and_lock(atomic,lock) __cond_lock(_atomic_dec_and_lock(atomic,lock)) + /* * bit-based spin_lock() * @@ -413,10 +540,14 @@ static inline void bit_spin_lock(int bitnum, unsigned long *addr) preempt_disable(); #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) while (test_and_set_bit(bitnum, addr)) { - while (test_bit(bitnum, addr)) + while (test_bit(bitnum, addr)) { + preempt_enable(); cpu_relax(); + preempt_disable(); + } } #endif + __acquire(bitlock); } /* @@ -424,18 +555,15 @@ static inline void bit_spin_lock(int bitnum, unsigned long *addr) */ static inline int bit_spin_trylock(int bitnum, unsigned long *addr) { + preempt_disable(); #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) - int ret; - - preempt_disable(); - ret = !test_and_set_bit(bitnum, addr); - if (!ret) + if (test_and_set_bit(bitnum, addr)) { preempt_enable(); - return ret; -#else - preempt_disable(); - return 1; + return 0; + } #endif + __acquire(bitlock); + return 1; } /* @@ -449,6 +577,7 @@ static inline void bit_spin_unlock(int bitnum, unsigned long *addr) clear_bit(bitnum, addr); #endif preempt_enable(); + __release(bitlock); } /* @@ -465,4 +594,13 @@ static inline int bit_spin_is_locked(int bitnum, unsigned long *addr) #endif } +#define DEFINE_SPINLOCK(x) spinlock_t x = SPIN_LOCK_UNLOCKED +#define DEFINE_RWLOCK(x) rwlock_t x = RW_LOCK_UNLOCKED + +/** + * spin_can_lock - would spin_trylock() succeed? + * @lock: the spinlock in question. + */ +#define spin_can_lock(lock) (!spin_is_locked(lock)) + #endif /* __LINUX_SPINLOCK_H */