kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004) Ingo Molnar
   7  */
   8
   9 #include <linux/config.h>
  10 #include <linux/linkage.h>
  11 #include <linux/preempt.h>
  12 #include <linux/spinlock.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/module.h>
  15
  16 /*
  17  * Generic declaration of the raw read_trylock() function,
  18  * architectures are supposed to optimize this:
  19  */
  20 int __lockfunc generic_raw_read_trylock(rwlock_t *lock)
  21 {
  22         _raw_read_lock(lock);
  23         return 1;
  24 }
  25 EXPORT_SYMBOL(generic_raw_read_trylock);
  26
  27 int __lockfunc _spin_trylock(spinlock_t *lock)
  28 {
  29         preempt_disable();
  30         if (_raw_spin_trylock(lock))
  31                 return 1;
  32
  33         preempt_enable();
  34         return 0;
  35 }
  36 EXPORT_SYMBOL(_spin_trylock);
  37
  38 int __lockfunc _read_trylock(rwlock_t *lock)
  39 {
  40         preempt_disable();
  41         if (_raw_read_trylock(lock))
  42                 return 1;
  43
  44         preempt_enable();
  45         return 0;
  46 }
  47 EXPORT_SYMBOL(_read_trylock);
  48
  49 int __lockfunc _write_trylock(rwlock_t *lock)
  50 {
  51         preempt_disable();
  52         if (_raw_write_trylock(lock))
  53                 return 1;
  54
  55         preempt_enable();
  56         return 0;
  57 }
  58 EXPORT_SYMBOL(_write_trylock);
  59
  60 #ifndef CONFIG_PREEMPT
  61
  62 void __lockfunc _read_lock(rwlock_t *lock)
  63 {
  64         preempt_disable();
  65         _raw_read_lock(lock);
  66 }
  67 EXPORT_SYMBOL(_read_lock);
  68
  69 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  70 {
  71         unsigned long flags;
  72
  73         local_irq_save(flags);
  74         preempt_disable();
  75         _raw_spin_lock_flags(lock, flags);
  76         return flags;
  77 }
  78 EXPORT_SYMBOL(_spin_lock_irqsave);
  79
  80 void __lockfunc _spin_lock_irq(spinlock_t *lock)
  81 {
  82         local_irq_disable();
  83         preempt_disable();
  84         _raw_spin_lock(lock);
  85 }
  86 EXPORT_SYMBOL(_spin_lock_irq);
  87
  88 void __lockfunc _spin_lock_bh(spinlock_t *lock)
  89 {
  90         local_bh_disable();
  91         preempt_disable();
  92         _raw_spin_lock(lock);
  93 }
  94 EXPORT_SYMBOL(_spin_lock_bh);
  95
  96 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
  97 {
  98         unsigned long flags;
  99
 100         local_irq_save(flags);
 101         preempt_disable();
 102         _raw_read_lock(lock);
 103         return flags;
 104 }
 105 EXPORT_SYMBOL(_read_lock_irqsave);
 106
 107 void __lockfunc _read_lock_irq(rwlock_t *lock)
 108 {
 109         local_irq_disable();
 110         preempt_disable();
 111         _raw_read_lock(lock);
 112 }
 113 EXPORT_SYMBOL(_read_lock_irq);
 114
 115 void __lockfunc _read_lock_bh(rwlock_t *lock)
 116 {
 117         local_bh_disable();
 118         preempt_disable();
 119         _raw_read_lock(lock);
 120 }
 121 EXPORT_SYMBOL(_read_lock_bh);
 122
 123 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 124 {
 125         unsigned long flags;
 126
 127         local_irq_save(flags);
 128         preempt_disable();
 129         _raw_write_lock(lock);
 130         return flags;
 131 }
 132 EXPORT_SYMBOL(_write_lock_irqsave);
 133
 134 void __lockfunc _write_lock_irq(rwlock_t *lock)
 135 {
 136         local_irq_disable();
 137         preempt_disable();
 138         _raw_write_lock(lock);
 139 }
 140 EXPORT_SYMBOL(_write_lock_irq);
 141
 142 void __lockfunc _write_lock_bh(rwlock_t *lock)
 143 {
 144         local_bh_disable();
 145         preempt_disable();
 146         _raw_write_lock(lock);
 147 }
 148 EXPORT_SYMBOL(_write_lock_bh);
 149
 150 void __lockfunc _spin_lock(spinlock_t *lock)
 151 {
 152         preempt_disable();
 153         _raw_spin_lock(lock);
 154 }
 155
 156 EXPORT_SYMBOL(_spin_lock);
 157
 158 void __lockfunc _write_lock(rwlock_t *lock)
 159 {
 160         preempt_disable();
 161         _raw_write_lock(lock);
 162 }
 163
 164 EXPORT_SYMBOL(_write_lock);
 165
 166 #else /* CONFIG_PREEMPT: */
 167
 168 /*
 169  * This could be a long-held lock. We both prepare to spin for a long
 170  * time (making _this_ CPU preemptable if possible), and we also signal
 171  * towards that other CPU that it should break the lock ASAP.
 172  *
 173  * (We do this in a function because inlining it would be excessive.)
 174  */
 175
 176 #define BUILD_LOCK_OPS(op, locktype)                                    \
 177 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
 178 {                                                                       \
 179         preempt_disable();                                              \
 180         for (;;) {                                                      \
 181                 if (likely(_raw_##op##_trylock(lock)))                  \
 182                         break;                                          \
 183                 preempt_enable();                                       \
 184                 if (!(lock)->break_lock)                                \
 185                         (lock)->break_lock = 1;                         \
 186                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 187                         cpu_relax();                                    \
 188                 preempt_disable();                                      \
 189         }                                                               \
 190 }                                                                       \
 191                                                                         \
 192 EXPORT_SYMBOL(_##op##_lock);                                            \
 193                                                                         \
 194 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
 195 {                                                                       \
 196         unsigned long flags;                                            \
 197                                                                         \
 198         preempt_disable();                                              \
 199         for (;;) {                                                      \
 200                 local_irq_save(flags);                                  \
 201                 if (likely(_raw_##op##_trylock(lock)))                  \
 202                         break;                                          \
 203                 local_irq_restore(flags);                               \
 204                                                                         \
 205                 preempt_enable();                                       \
 206                 if (!(lock)->break_lock)                                \
 207                         (lock)->break_lock = 1;                         \
 208                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 209                         cpu_relax();                                    \
 210                 preempt_disable();                                      \
 211         }                                                               \
 212         return flags;                                                   \
 213 }                                                                       \
 214                                                                         \
 215 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
 216                                                                         \
 217 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
 218 {                                                                       \
 219         _##op##_lock_irqsave(lock);                                     \
 220 }                                                                       \
 221                                                                         \
 222 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
 223                                                                         \
 224 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
 225 {                                                                       \
 226         unsigned long flags;                                            \
 227                                                                         \
 228         /*                                                      */      \
 229         /* Careful: we must exclude softirqs too, hence the     */      \
 230         /* irq-disabling. We use the generic preemption-aware   */      \
 231         /* function:                                            */      \
 232         /**/                                                            \
 233         flags = _##op##_lock_irqsave(lock);                             \
 234         local_bh_disable();                                             \
 235         local_irq_restore(flags);                                       \
 236 }                                                                       \
 237                                                                         \
 238 EXPORT_SYMBOL(_##op##_lock_bh)
 239
 240 /*
 241  * Build preemption-friendly versions of the following
 242  * lock-spinning functions:
 243  *
 244  *         _[spin|read|write]_lock()
 245  *         _[spin|read|write]_lock_irq()
 246  *         _[spin|read|write]_lock_irqsave()
 247  *         _[spin|read|write]_lock_bh()
 248  */
 249 BUILD_LOCK_OPS(spin, spinlock);
 250 BUILD_LOCK_OPS(read, rwlock);
 251 BUILD_LOCK_OPS(write, rwlock);
 252
 253 #endif /* CONFIG_PREEMPT */
 254
 255 void __lockfunc _spin_unlock(spinlock_t *lock)
 256 {
 257         _raw_spin_unlock(lock);
 258         preempt_enable();
 259 }
 260 EXPORT_SYMBOL(_spin_unlock);
 261
 262 void __lockfunc _write_unlock(rwlock_t *lock)
 263 {
 264         _raw_write_unlock(lock);
 265         preempt_enable();
 266 }
 267 EXPORT_SYMBOL(_write_unlock);
 268
 269 void __lockfunc _read_unlock(rwlock_t *lock)
 270 {
 271         _raw_read_unlock(lock);
 272         preempt_enable();
 273 }
 274 EXPORT_SYMBOL(_read_unlock);
 275
 276 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 277 {
 278         _raw_spin_unlock(lock);
 279         local_irq_restore(flags);
 280         preempt_enable();
 281 }
 282 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 283
 284 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 285 {
 286         _raw_spin_unlock(lock);
 287         local_irq_enable();
 288         preempt_enable();
 289 }
 290 EXPORT_SYMBOL(_spin_unlock_irq);
 291
 292 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 293 {
 294         _raw_spin_unlock(lock);
 295         preempt_enable();
 296         local_bh_enable();
 297 }
 298 EXPORT_SYMBOL(_spin_unlock_bh);
 299
 300 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 301 {
 302         _raw_read_unlock(lock);
 303         local_irq_restore(flags);
 304         preempt_enable();
 305 }
 306 EXPORT_SYMBOL(_read_unlock_irqrestore);
 307
 308 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 309 {
 310         _raw_read_unlock(lock);
 311         local_irq_enable();
 312         preempt_enable();
 313 }
 314 EXPORT_SYMBOL(_read_unlock_irq);
 315
 316 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 317 {
 318         _raw_read_unlock(lock);
 319         preempt_enable();
 320         local_bh_enable();
 321 }
 322 EXPORT_SYMBOL(_read_unlock_bh);
 323
 324 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 325 {
 326         _raw_write_unlock(lock);
 327         local_irq_restore(flags);
 328         preempt_enable();
 329 }
 330 EXPORT_SYMBOL(_write_unlock_irqrestore);
 331
 332 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 333 {
 334         _raw_write_unlock(lock);
 335         local_irq_enable();
 336         preempt_enable();
 337 }
 338 EXPORT_SYMBOL(_write_unlock_irq);
 339
 340 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 341 {
 342         _raw_write_unlock(lock);
 343         preempt_enable();
 344         local_bh_enable();
 345 }
 346 EXPORT_SYMBOL(_write_unlock_bh);
 347
 348 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 349 {
 350         local_bh_disable();
 351         preempt_disable();
 352         if (_raw_spin_trylock(lock))
 353                 return 1;
 354
 355         preempt_enable();
 356         local_bh_enable();
 357         return 0;
 358 }
 359 EXPORT_SYMBOL(_spin_trylock_bh);
 360
 361 int in_lock_functions(unsigned long addr)
 362 {
 363         /* Linker adds these: start and end of __lockfunc functions */
 364         extern char __lock_text_start[], __lock_text_end[];
 365
 366         return addr >= (unsigned long)__lock_text_start
 367         && addr < (unsigned long)__lock_text_end;
 368 }
 369 EXPORT_SYMBOL(in_lock_functions);