include/linux/spinlock.h

   1 #ifndef __LINUX_SPINLOCK_H
   2 #define __LINUX_SPINLOCK_H
   3
   4 /*
   5  * include/linux/spinlock.h - generic locking declarations
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/preempt.h>
  10 #include <linux/linkage.h>
  11 #include <linux/compiler.h>
  12 #include <linux/thread_info.h>
  13 #include <linux/kernel.h>
  14 #include <linux/stringify.h>
  15
  16 #include <asm/processor.h>      /* for cpu relax */
  17 #include <asm/system.h>
  18
  19 /*
  20  * Must define these before including other files, inline functions need them
  21  */
  22 #define LOCK_SECTION_NAME                       \
  23         ".text.lock." __stringify(KBUILD_BASENAME)
  24
  25 #define LOCK_SECTION_START(extra)               \
  26         ".subsection 1\n\t"                     \
  27         extra                                   \
  28         ".ifndef " LOCK_SECTION_NAME "\n\t"     \
  29         LOCK_SECTION_NAME ":\n\t"               \
  30         ".endif\n\t"
  31
  32 #define LOCK_SECTION_END                        \
  33         ".previous\n\t"
  34
  35 /*
  36  * If CONFIG_SMP is set, pull in the _raw_* definitions
  37  */
  38 #ifdef CONFIG_SMP
  39 #include <asm/spinlock.h>
  40
  41 #else
  42
  43 #define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
  44
  45 #if !defined(CONFIG_PREEMPT) && !defined(CONFIG_DEBUG_SPINLOCK)
  46 # define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic)
  47 # define ATOMIC_DEC_AND_LOCK
  48 #endif
  49
  50 #ifdef CONFIG_DEBUG_SPINLOCK
  51
  52 #define SPINLOCK_MAGIC  0x1D244B3C
  53 typedef struct {
  54         unsigned long magic;
  55         volatile unsigned long lock;
  56         volatile unsigned int babble;
  57         const char *module;
  58         char *owner;
  59         int oline;
  60 } spinlock_t;
  61 #define SPIN_LOCK_UNLOCKED (spinlock_t) { SPINLOCK_MAGIC, 0, 10, __FILE__ , NULL, 0}
  62
  63 #define spin_lock_init(x) \
  64         do { \
  65                 (x)->magic = SPINLOCK_MAGIC; \
  66                 (x)->lock = 0; \
  67                 (x)->babble = 5; \
  68                 (x)->module = __FILE__; \
  69                 (x)->owner = NULL; \
  70                 (x)->oline = 0; \
  71         } while (0)
  72
  73 #define CHECK_LOCK(x) \
  74         do { \
  75                 if ((x)->magic != SPINLOCK_MAGIC) { \
  76                         printk(KERN_ERR "%s:%d: spin_is_locked on uninitialized spinlock %p.\n", \
  77                                         __FILE__, __LINE__, (x)); \
  78                 } \
  79         } while(0)
  80
  81 #define _raw_spin_lock(x)               \
  82         do { \
  83                 CHECK_LOCK(x); \
  84                 if ((x)->lock&&(x)->babble) { \
  85                         (x)->babble--; \
  86                         printk("%s:%d: spin_lock(%s:%p) already locked by %s/%d\n", \
  87                                         __FILE__,__LINE__, (x)->module, \
  88                                         (x), (x)->owner, (x)->oline); \
  89                 } \
  90                 (x)->lock = 1; \
  91                 (x)->owner = __FILE__; \
  92                 (x)->oline = __LINE__; \
  93         } while (0)
  94
  95 /* without debugging, spin_is_locked on UP always says
  96  * FALSE. --> printk if already locked. */
  97 #define spin_is_locked(x) \
  98         ({ \
  99                 CHECK_LOCK(x); \
 100                 if ((x)->lock&&(x)->babble) { \
 101                         (x)->babble--; \
 102                         printk("%s:%d: spin_is_locked(%s:%p) already locked by %s/%d\n", \
 103                                         __FILE__,__LINE__, (x)->module, \
 104                                         (x), (x)->owner, (x)->oline); \
 105                 } \
 106                 0; \
 107         })
 108
 109 /* without debugging, spin_trylock on UP always says
 110  * TRUE. --> printk if already locked. */
 111 #define _raw_spin_trylock(x) \
 112         ({ \
 113                 CHECK_LOCK(x); \
 114                 if ((x)->lock&&(x)->babble) { \
 115                         (x)->babble--; \
 116                         printk("%s:%d: spin_trylock(%s:%p) already locked by %s/%d\n", \
 117                                         __FILE__,__LINE__, (x)->module, \
 118                                         (x), (x)->owner, (x)->oline); \
 119                 } \
 120                 (x)->lock = 1; \
 121                 (x)->owner = __FILE__; \
 122                 (x)->oline = __LINE__; \
 123                 1; \
 124         })
 125
 126 #define spin_unlock_wait(x)     \
 127         do { \
 128                 CHECK_LOCK(x); \
 129                 if ((x)->lock&&(x)->babble) { \
 130                         (x)->babble--; \
 131                         printk("%s:%d: spin_unlock_wait(%s:%p) owned by %s/%d\n", \
 132                                         __FILE__,__LINE__, (x)->module, (x), \
 133                                         (x)->owner, (x)->oline); \
 134                 }\
 135         } while (0)
 136
 137 #define _raw_spin_unlock(x) \
 138         do { \
 139                 CHECK_LOCK(x); \
 140                 if (!(x)->lock&&(x)->babble) { \
 141                         (x)->babble--; \
 142                         printk("%s:%d: spin_unlock(%s:%p) not locked\n", \
 143                                         __FILE__,__LINE__, (x)->module, (x));\
 144                 } \
 145                 (x)->lock = 0; \
 146         } while (0)
 147 #else
 148 /*
 149  * gcc versions before ~2.95 have a nasty bug with empty initializers.
 150  */
 151 #if (__GNUC__ > 2)
 152   typedef struct { } spinlock_t;
 153   #define SPIN_LOCK_UNLOCKED (spinlock_t) { }
 154 #else
 155   typedef struct { int gcc_is_buggy; } spinlock_t;
 156   #define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
 157 #endif
 158
 159 /*
 160  * If CONFIG_SMP is unset, declare the _raw_* definitions as nops
 161  */
 162 #define spin_lock_init(lock)    do { (void)(lock); } while(0)
 163 #define _raw_spin_lock(lock)    do { (void)(lock); } while(0)
 164 #define spin_is_locked(lock)    ((void)(lock), 0)
 165 #define _raw_spin_trylock(lock) ((void)(lock), 1)
 166 #define spin_unlock_wait(lock)  do { (void)(lock); } while(0)
 167 #define _raw_spin_unlock(lock)  do { (void)(lock); } while(0)
 168 #endif /* CONFIG_DEBUG_SPINLOCK */
 169
 170 /* RW spinlocks: No debug version */
 171
 172 #if (__GNUC__ > 2)
 173   typedef struct { } rwlock_t;
 174   #define RW_LOCK_UNLOCKED (rwlock_t) { }
 175 #else
 176   typedef struct { int gcc_is_buggy; } rwlock_t;
 177   #define RW_LOCK_UNLOCKED (rwlock_t) { 0 }
 178 #endif
 179
 180 #define rwlock_init(lock)       do { (void)(lock); } while(0)
 181 #define _raw_read_lock(lock)    do { (void)(lock); } while(0)
 182 #define _raw_read_unlock(lock)  do { (void)(lock); } while(0)
 183 #define _raw_write_lock(lock)   do { (void)(lock); } while(0)
 184 #define _raw_write_unlock(lock) do { (void)(lock); } while(0)
 185 #define _raw_write_trylock(lock) ({ (void)(lock); (1); })
 186
 187 #endif /* !SMP */
 188
 189 /*
 190  * Define the various spin_lock and rw_lock methods.  Note we define these
 191  * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various
 192  * methods are defined as nops in the case they are not required.
 193  */
 194 #define spin_trylock(lock)      ({preempt_disable(); _raw_spin_trylock(lock) ? \
 195                                 1 : ({preempt_enable(); 0;});})
 196
 197 #define write_trylock(lock)     ({preempt_disable();_raw_write_trylock(lock) ? \
 198                                 1 : ({preempt_enable(); 0;});})
 199
 200 /* Where's read_trylock? */
 201
 202 #if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
 203 void __preempt_spin_lock(spinlock_t *lock);
 204 void __preempt_write_lock(rwlock_t *lock);
 205
 206 #define spin_lock(lock) \
 207 do { \
 208         preempt_disable(); \
 209         if (unlikely(!_raw_spin_trylock(lock))) \
 210                 __preempt_spin_lock(lock); \
 211 } while (0)
 212
 213 #define write_lock(lock) \
 214 do { \
 215         preempt_disable(); \
 216         if (unlikely(!_raw_write_trylock(lock))) \
 217                 __preempt_write_lock(lock); \
 218 } while (0)
 219
 220 #else
 221 #define spin_lock(lock) \
 222 do { \
 223         preempt_disable(); \
 224         _raw_spin_lock(lock); \
 225 } while(0)
 226
 227 #define write_lock(lock) \
 228 do { \
 229         preempt_disable(); \
 230         _raw_write_lock(lock); \
 231 } while(0)
 232 #endif
 233
 234 #define read_lock(lock) \
 235 do { \
 236         preempt_disable(); \
 237         _raw_read_lock(lock); \
 238 } while(0)
 239
 240 #define spin_unlock(lock) \
 241 do { \
 242         _raw_spin_unlock(lock); \
 243         preempt_enable(); \
 244 } while (0)
 245
 246 #define write_unlock(lock) \
 247 do { \
 248         _raw_write_unlock(lock); \
 249         preempt_enable(); \
 250 } while(0)
 251
 252 #define read_unlock(lock) \
 253 do { \
 254         _raw_read_unlock(lock); \
 255         preempt_enable(); \
 256 } while(0)
 257
 258 #define spin_lock_irqsave(lock, flags) \
 259 do { \
 260         local_irq_save(flags); \
 261         preempt_disable(); \
 262         _raw_spin_lock_flags(lock, flags); \
 263 } while (0)
 264
 265 #define spin_lock_irq(lock) \
 266 do { \
 267         local_irq_disable(); \
 268         preempt_disable(); \
 269         _raw_spin_lock(lock); \
 270 } while (0)
 271
 272 #define spin_lock_bh(lock) \
 273 do { \
 274         local_bh_disable(); \
 275         preempt_disable(); \
 276         _raw_spin_lock(lock); \
 277 } while (0)
 278
 279 #define read_lock_irqsave(lock, flags) \
 280 do { \
 281         local_irq_save(flags); \
 282         preempt_disable(); \
 283         _raw_read_lock(lock); \
 284 } while (0)
 285
 286 #define read_lock_irq(lock) \
 287 do { \
 288         local_irq_disable(); \
 289         preempt_disable(); \
 290         _raw_read_lock(lock); \
 291 } while (0)
 292
 293 #define read_lock_bh(lock) \
 294 do { \
 295         local_bh_disable(); \
 296         preempt_disable(); \
 297         _raw_read_lock(lock); \
 298 } while (0)
 299
 300 #define write_lock_irqsave(lock, flags) \
 301 do { \
 302         local_irq_save(flags); \
 303         preempt_disable(); \
 304         _raw_write_lock(lock); \
 305 } while (0)
 306
 307 #define write_lock_irq(lock) \
 308 do { \
 309         local_irq_disable(); \
 310         preempt_disable(); \
 311         _raw_write_lock(lock); \
 312 } while (0)
 313
 314 #define write_lock_bh(lock) \
 315 do { \
 316         local_bh_disable(); \
 317         preempt_disable(); \
 318         _raw_write_lock(lock); \
 319 } while (0)
 320
 321 #define spin_unlock_irqrestore(lock, flags) \
 322 do { \
 323         _raw_spin_unlock(lock); \
 324         local_irq_restore(flags); \
 325         preempt_enable(); \
 326 } while (0)
 327
 328 #define _raw_spin_unlock_irqrestore(lock, flags) \
 329 do { \
 330         _raw_spin_unlock(lock); \
 331         local_irq_restore(flags); \
 332 } while (0)
 333
 334 #define spin_unlock_irq(lock) \
 335 do { \
 336         _raw_spin_unlock(lock); \
 337         local_irq_enable(); \
 338         preempt_enable(); \
 339 } while (0)
 340
 341 #define spin_unlock_bh(lock) \
 342 do { \
 343         _raw_spin_unlock(lock); \
 344         preempt_enable(); \
 345         local_bh_enable(); \
 346 } while (0)
 347
 348 #define read_unlock_irqrestore(lock, flags) \
 349 do { \
 350         _raw_read_unlock(lock); \
 351         local_irq_restore(flags); \
 352         preempt_enable(); \
 353 } while (0)
 354
 355 #define read_unlock_irq(lock) \
 356 do { \
 357         _raw_read_unlock(lock); \
 358         local_irq_enable(); \
 359         preempt_enable(); \
 360 } while (0)
 361
 362 #define read_unlock_bh(lock) \
 363 do { \
 364         _raw_read_unlock(lock); \
 365         preempt_enable(); \
 366         local_bh_enable(); \
 367 } while (0)
 368
 369 #define write_unlock_irqrestore(lock, flags) \
 370 do { \
 371         _raw_write_unlock(lock); \
 372         local_irq_restore(flags); \
 373         preempt_enable(); \
 374 } while (0)
 375
 376 #define write_unlock_irq(lock) \
 377 do { \
 378         _raw_write_unlock(lock); \
 379         local_irq_enable(); \
 380         preempt_enable(); \
 381 } while (0)
 382
 383 #define write_unlock_bh(lock) \
 384 do { \
 385         _raw_write_unlock(lock); \
 386         preempt_enable(); \
 387         local_bh_enable(); \
 388 } while (0)
 389
 390 #define spin_trylock_bh(lock)   ({ local_bh_disable(); preempt_disable(); \
 391                                 _raw_spin_trylock(lock) ? 1 : \
 392                                 ({preempt_enable(); local_bh_enable(); 0;});})
 393
 394 /* "lock on reference count zero" */
 395 #ifndef ATOMIC_DEC_AND_LOCK
 396 #include <asm/atomic.h>
 397 extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
 398 #endif
 399
 400 /*
 401  *  bit-based spin_lock()
 402  *
 403  * Don't use this unless you really need to: spin_lock() and spin_unlock()
 404  * are significantly faster.
 405  */
 406 static inline void bit_spin_lock(int bitnum, unsigned long *addr)
 407 {
 408         /*
 409          * Assuming the lock is uncontended, this never enters
 410          * the body of the outer loop. If it is contended, then
 411          * within the inner loop a non-atomic test is used to
 412          * busywait with less bus contention for a good time to
 413          * attempt to acquire the lock bit.
 414          */
 415         preempt_disable();
 416 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 417         while (test_and_set_bit(bitnum, addr)) {
 418                 while (test_bit(bitnum, addr))
 419                         cpu_relax();
 420         }
 421 #endif
 422 }
 423
 424 /*
 425  * Return true if it was acquired
 426  */
 427 static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
 428 {
 429 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 430         int ret;
 431
 432         preempt_disable();
 433         ret = !test_and_set_bit(bitnum, addr);
 434         if (!ret)
 435                 preempt_enable();
 436         return ret;
 437 #else
 438         preempt_disable();
 439         return 1;
 440 #endif
 441 }
 442
 443 /*
 444  *  bit-based spin_unlock()
 445  */
 446 static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
 447 {
 448 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 449         BUG_ON(!test_bit(bitnum, addr));
 450         smp_mb__before_clear_bit();
 451         clear_bit(bitnum, addr);
 452 #endif
 453         preempt_enable();
 454 }
 455
 456 /*
 457  * Return true if the lock is held.
 458  */
 459 static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
 460 {
 461 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 462         return test_bit(bitnum, addr);
 463 #elif defined CONFIG_PREEMPT
 464         return preempt_count();
 465 #else
 466         return 1;
 467 #endif
 468 }
 469
 470 #endif /* __LINUX_SPINLOCK_H */