1/* 2 * Copyright (2004) Linus Torvalds 3 * 4 * Author: Zwane Mwaikambo <zwane@fsmlabs.com> 5 * 6 * Copyright (2004, 2005) Ingo Molnar 7 * 8 * This file contains the spinlock/rwlock implementations for the 9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) 10 * 11 * Note that some architectures have special knowledge about the 12 * stack frames of these functions in their profile_pc. If you 13 * change anything significant here that could change the stack 14 * frame contact the architecture maintainers. 15 */ 16 17#include <linux/linkage.h> 18#include <linux/preempt.h> 19#include <linux/spinlock.h> 20#include <linux/interrupt.h> 21#include <linux/debug_locks.h> 22#include <linux/module.h> 23 24/* 25 * If lockdep is enabled then we use the non-preemption spin-ops 26 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are 27 * not re-enabled during lock-acquire (which the preempt-spin-ops do): 28 */ 29#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) 30/* 31 * The __lock_function inlines are taken from 32 * include/linux/spinlock_api_smp.h 33 */ 34#else 35#define raw_read_can_lock(l) read_can_lock(l) 36#define raw_write_can_lock(l) write_can_lock(l) 37/* 38 * We build the __lock_function inlines here. They are too large for 39 * inlining all over the place, but here is only one user per function 40 * which embedds them into the calling _lock_function below. 41 * 42 * This could be a long-held lock. We both prepare to spin for a long 43 * time (making _this_ CPU preemptable if possible), and we also signal 44 * towards that other CPU that it should break the lock ASAP. 45 */ 46#define BUILD_LOCK_OPS(op, locktype) \ 47void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ 48{ \ 49 for (;;) { \ 50 preempt_disable(); \ 51 if (likely(do_raw_##op##_trylock(lock))) \ 52 break; \ 53 preempt_enable(); \ 54 \ 55 if (!(lock)->break_lock) \ 56 (lock)->break_lock = 1; \ 57 while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ 58 arch_##op##_relax(&lock->raw_lock); \ 59 } \ 60 (lock)->break_lock = 0; \ 61} \ 62 \ 63unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ 64{ \ 65 unsigned long flags; \ 66 \ 67 for (;;) { \ 68 preempt_disable(); \ 69 local_irq_save(flags); \ 70 if (likely(do_raw_##op##_trylock(lock))) \ 71 break; \ 72 local_irq_restore(flags); \ 73 preempt_enable(); \ 74 \ 75 if (!(lock)->break_lock) \ 76 (lock)->break_lock = 1; \ 77 while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ 78 arch_##op##_relax(&lock->raw_lock); \ 79 } \ 80 (lock)->break_lock = 0; \ 81 return flags; \ 82} \ 83 \ 84void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ 85{ \ 86 _raw_##op##_lock_irqsave(lock); \ 87} \ 88 \ 89void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 90{ \ 91 unsigned long flags; \ 92 \ 93 /* */ \ 94 /* Careful: we must exclude softirqs too, hence the */ \ 95 /* irq-disabling. We use the generic preemption-aware */ \ 96 /* function: */ \ 97 /**/ \ 98 flags = _raw_##op##_lock_irqsave(lock); \ 99 local_bh_disable(); \ 100 local_irq_restore(flags); \ 101} \ 102 103/* 104 * Build preemption-friendly versions of the following 105 * lock-spinning functions: 106 * 107 * __[spin|read|write]_lock() 108 * __[spin|read|write]_lock_irq() 109 * __[spin|read|write]_lock_irqsave() 110 * __[spin|read|write]_lock_bh() 111 */ 112BUILD_LOCK_OPS(spin, raw_spinlock); 113BUILD_LOCK_OPS(read, rwlock); 114BUILD_LOCK_OPS(write, rwlock); 115 116#endif 117 118#ifndef CONFIG_INLINE_SPIN_TRYLOCK 119int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) 120{ 121 return __raw_spin_trylock(lock); 122} 123EXPORT_SYMBOL(_raw_spin_trylock); 124#endif 125 126#ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH 127int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) 128{ 129 return __raw_spin_trylock_bh(lock); 130} 131EXPORT_SYMBOL(_raw_spin_trylock_bh); 132#endif 133 134#ifndef CONFIG_INLINE_SPIN_LOCK 135void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) 136{ 137 __raw_spin_lock(lock); 138} 139EXPORT_SYMBOL(_raw_spin_lock); 140#endif 141 142#ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE 143unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) 144{ 145 return __raw_spin_lock_irqsave(lock); 146} 147EXPORT_SYMBOL(_raw_spin_lock_irqsave); 148#endif 149 150#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ 151void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) 152{ 153 __raw_spin_lock_irq(lock); 154} 155EXPORT_SYMBOL(_raw_spin_lock_irq); 156#endif 157 158#ifndef CONFIG_INLINE_SPIN_LOCK_BH 159void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) 160{ 161 __raw_spin_lock_bh(lock); 162} 163EXPORT_SYMBOL(_raw_spin_lock_bh); 164#endif 165 166#ifndef CONFIG_INLINE_SPIN_UNLOCK 167void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) 168{ 169 __raw_spin_unlock(lock); 170} 171EXPORT_SYMBOL(_raw_spin_unlock); 172#endif 173 174#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE 175void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) 176{ 177 __raw_spin_unlock_irqrestore(lock, flags); 178} 179EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); 180#endif 181 182#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ 183void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) 184{ 185 __raw_spin_unlock_irq(lock); 186} 187EXPORT_SYMBOL(_raw_spin_unlock_irq); 188#endif 189 190#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH 191void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) 192{ 193 __raw_spin_unlock_bh(lock); 194} 195EXPORT_SYMBOL(_raw_spin_unlock_bh); 196#endif 197 198#ifndef CONFIG_INLINE_READ_TRYLOCK 199int __lockfunc _raw_read_trylock(rwlock_t *lock) 200{ 201 return __raw_read_trylock(lock); 202} 203EXPORT_SYMBOL(_raw_read_trylock); 204#endif 205 206#ifndef CONFIG_INLINE_READ_LOCK 207void __lockfunc _raw_read_lock(rwlock_t *lock) 208{ 209 __raw_read_lock(lock); 210} 211EXPORT_SYMBOL(_raw_read_lock); 212#endif 213 214#ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE 215unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) 216{ 217 return __raw_read_lock_irqsave(lock); 218} 219EXPORT_SYMBOL(_raw_read_lock_irqsave); 220#endif 221 222#ifndef CONFIG_INLINE_READ_LOCK_IRQ 223void __lockfunc _raw_read_lock_irq(rwlock_t *lock) 224{ 225 __raw_read_lock_irq(lock); 226} 227EXPORT_SYMBOL(_raw_read_lock_irq); 228#endif 229 230#ifndef CONFIG_INLINE_READ_LOCK_BH 231void __lockfunc _raw_read_lock_bh(rwlock_t *lock) 232{ 233 __raw_read_lock_bh(lock); 234} 235EXPORT_SYMBOL(_raw_read_lock_bh); 236#endif 237 238#ifndef CONFIG_INLINE_READ_UNLOCK 239void __lockfunc _raw_read_unlock(rwlock_t *lock) 240{ 241 __raw_read_unlock(lock); 242} 243EXPORT_SYMBOL(_raw_read_unlock); 244#endif 245 246#ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE 247void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 248{ 249 __raw_read_unlock_irqrestore(lock, flags); 250} 251EXPORT_SYMBOL(_raw_read_unlock_irqrestore); 252#endif 253 254#ifndef CONFIG_INLINE_READ_UNLOCK_IRQ 255void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) 256{ 257 __raw_read_unlock_irq(lock); 258} 259EXPORT_SYMBOL(_raw_read_unlock_irq); 260#endif 261 262#ifndef CONFIG_INLINE_READ_UNLOCK_BH 263void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) 264{ 265 __raw_read_unlock_bh(lock); 266} 267EXPORT_SYMBOL(_raw_read_unlock_bh); 268#endif 269 270#ifndef CONFIG_INLINE_WRITE_TRYLOCK 271int __lockfunc _raw_write_trylock(rwlock_t *lock) 272{ 273 return __raw_write_trylock(lock); 274} 275EXPORT_SYMBOL(_raw_write_trylock); 276#endif 277 278#ifndef CONFIG_INLINE_WRITE_LOCK 279void __lockfunc _raw_write_lock(rwlock_t *lock) 280{ 281 __raw_write_lock(lock); 282} 283EXPORT_SYMBOL(_raw_write_lock); 284#endif 285 286#ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE 287unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) 288{ 289 return __raw_write_lock_irqsave(lock); 290} 291EXPORT_SYMBOL(_raw_write_lock_irqsave); 292#endif 293 294#ifndef CONFIG_INLINE_WRITE_LOCK_IRQ 295void __lockfunc _raw_write_lock_irq(rwlock_t *lock) 296{ 297 __raw_write_lock_irq(lock); 298} 299EXPORT_SYMBOL(_raw_write_lock_irq); 300#endif 301 302#ifndef CONFIG_INLINE_WRITE_LOCK_BH 303void __lockfunc _raw_write_lock_bh(rwlock_t *lock) 304{ 305 __raw_write_lock_bh(lock); 306} 307EXPORT_SYMBOL(_raw_write_lock_bh); 308#endif 309 310#ifndef CONFIG_INLINE_WRITE_UNLOCK 311void __lockfunc _raw_write_unlock(rwlock_t *lock) 312{ 313 __raw_write_unlock(lock); 314} 315EXPORT_SYMBOL(_raw_write_unlock); 316#endif 317 318#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE 319void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 320{ 321 __raw_write_unlock_irqrestore(lock, flags); 322} 323EXPORT_SYMBOL(_raw_write_unlock_irqrestore); 324#endif 325 326#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ 327void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) 328{ 329 __raw_write_unlock_irq(lock); 330} 331EXPORT_SYMBOL(_raw_write_unlock_irq); 332#endif 333 334#ifndef CONFIG_INLINE_WRITE_UNLOCK_BH 335void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) 336{ 337 __raw_write_unlock_bh(lock); 338} 339EXPORT_SYMBOL(_raw_write_unlock_bh); 340#endif 341 342#ifdef CONFIG_DEBUG_LOCK_ALLOC 343 344void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) 345{ 346 preempt_disable(); 347 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 348 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 349} 350EXPORT_SYMBOL(_raw_spin_lock_nested); 351 352unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, 353 int subclass) 354{ 355 unsigned long flags; 356 357 local_irq_save(flags); 358 preempt_disable(); 359 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 360 LOCK_CONTENDED_FLAGS(lock, do_raw_spin_trylock, do_raw_spin_lock, 361 do_raw_spin_lock_flags, &flags); 362 return flags; 363} 364EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested); 365 366void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock, 367 struct lockdep_map *nest_lock) 368{ 369 preempt_disable(); 370 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 371 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 372} 373EXPORT_SYMBOL(_raw_spin_lock_nest_lock); 374 375#endif 376 377notrace int in_lock_functions(unsigned long addr) 378{ 379 /* Linker adds these: start and end of __lockfunc functions */ 380 extern char __lock_text_start[], __lock_text_end[]; 381 382 return addr >= (unsigned long)__lock_text_start 383 && addr < (unsigned long)__lock_text_end; 384} 385EXPORT_SYMBOL(in_lock_functions); 386