1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * PREEMPT_RT substitution for spin/rw_locks 4 * 5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to 6 * resemble the non RT semantics: 7 * 8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state 9 * preserving. The task state is saved before blocking on the underlying 10 * rtmutex, and restored when the lock has been acquired. Regular wakeups 11 * during that time are redirected to the saved state so no wake up is 12 * missed. 13 * 14 * - Non RT spin/rwlocks disable preemption and eventually interrupts. 15 * Disabling preemption has the side effect of disabling migration and 16 * preventing RCU grace periods. 17 * 18 * The RT substitutions explicitly disable migration and take 19 * rcu_read_lock() across the lock held section. 20 */ 21#include <linux/spinlock.h> 22#include <linux/export.h> 23 24#define RT_MUTEX_BUILD_SPINLOCKS 25#include "rtmutex.c" 26 27/* 28 * __might_resched() skips the state check as rtlocks are state 29 * preserving. Take RCU nesting into account as spin/read/write_lock() can 30 * legitimately nest into an RCU read side critical section. 31 */ 32#define RTLOCK_RESCHED_OFFSETS \ 33 (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT) 34 35#define rtlock_might_resched() \ 36 __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS) 37 38static __always_inline void rtlock_lock(struct rt_mutex_base *rtm) 39{ 40 lockdep_assert(!current->pi_blocked_on); 41 42 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 43 rtlock_slowlock(rtm); 44} 45 46static __always_inline void __rt_spin_lock(spinlock_t *lock) 47{ 48 rtlock_might_resched(); 49 rtlock_lock(&lock->lock); 50 rcu_read_lock(); 51 migrate_disable(); 52} 53 54void __sched rt_spin_lock(spinlock_t *lock) 55{ 56 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 57 __rt_spin_lock(lock); 58} 59EXPORT_SYMBOL(rt_spin_lock); 60 61#ifdef CONFIG_DEBUG_LOCK_ALLOC 62void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass) 63{ 64 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 65 __rt_spin_lock(lock); 66} 67EXPORT_SYMBOL(rt_spin_lock_nested); 68 69void __sched rt_spin_lock_nest_lock(spinlock_t *lock, 70 struct lockdep_map *nest_lock) 71{ 72 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 73 __rt_spin_lock(lock); 74} 75EXPORT_SYMBOL(rt_spin_lock_nest_lock); 76#endif 77 78void __sched rt_spin_unlock(spinlock_t *lock) 79{ 80 spin_release(&lock->dep_map, _RET_IP_); 81 migrate_enable(); 82 rcu_read_unlock(); 83 84 if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) 85 rt_mutex_slowunlock(&lock->lock); 86} 87EXPORT_SYMBOL(rt_spin_unlock); 88 89/* 90 * Wait for the lock to get unlocked: instead of polling for an unlock 91 * (like raw spinlocks do), lock and unlock, to force the kernel to 92 * schedule if there's contention: 93 */ 94void __sched rt_spin_lock_unlock(spinlock_t *lock) 95{ 96 spin_lock(lock); 97 spin_unlock(lock); 98} 99EXPORT_SYMBOL(rt_spin_lock_unlock); 100 101static __always_inline int __rt_spin_trylock(spinlock_t *lock) 102{ 103 int ret = 1; 104 105 if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current))) 106 ret = rt_mutex_slowtrylock(&lock->lock); 107 108 if (ret) { 109 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); 110 rcu_read_lock(); 111 migrate_disable(); 112 } 113 return ret; 114} 115 116int __sched rt_spin_trylock(spinlock_t *lock) 117{ 118 return __rt_spin_trylock(lock); 119} 120EXPORT_SYMBOL(rt_spin_trylock); 121 122int __sched rt_spin_trylock_bh(spinlock_t *lock) 123{ 124 int ret; 125 126 local_bh_disable(); 127 ret = __rt_spin_trylock(lock); 128 if (!ret) 129 local_bh_enable(); 130 return ret; 131} 132EXPORT_SYMBOL(rt_spin_trylock_bh); 133 134#ifdef CONFIG_DEBUG_LOCK_ALLOC 135void __rt_spin_lock_init(spinlock_t *lock, const char *name, 136 struct lock_class_key *key, bool percpu) 137{ 138 u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL; 139 140 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 141 lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG, 142 LD_WAIT_INV, type); 143} 144EXPORT_SYMBOL(__rt_spin_lock_init); 145#endif 146 147/* 148 * RT-specific reader/writer locks 149 */ 150#define rwbase_set_and_save_current_state(state) \ 151 current_save_and_set_rtlock_wait_state() 152 153#define rwbase_restore_current_state() \ 154 current_restore_rtlock_saved_state() 155 156static __always_inline int 157rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state) 158{ 159 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 160 rtlock_slowlock(rtm); 161 return 0; 162} 163 164static __always_inline int 165rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state) 166{ 167 rtlock_slowlock_locked(rtm); 168 return 0; 169} 170 171static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm) 172{ 173 if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL))) 174 return; 175 176 rt_mutex_slowunlock(rtm); 177} 178 179static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm) 180{ 181 if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 182 return 1; 183 184 return rt_mutex_slowtrylock(rtm); 185} 186 187#define rwbase_signal_pending_state(state, current) (0) 188 189#define rwbase_pre_schedule() 190 191#define rwbase_schedule() \ 192 schedule_rtlock() 193 194#define rwbase_post_schedule() 195 196#include "rwbase_rt.c" 197/* 198 * The common functions which get wrapped into the rwlock API. 199 */ 200int __sched rt_read_trylock(rwlock_t *rwlock) 201{ 202 int ret; 203 204 ret = rwbase_read_trylock(&rwlock->rwbase); 205 if (ret) { 206 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_); 207 rcu_read_lock(); 208 migrate_disable(); 209 } 210 return ret; 211} 212EXPORT_SYMBOL(rt_read_trylock); 213 214int __sched rt_write_trylock(rwlock_t *rwlock) 215{ 216 int ret; 217 218 ret = rwbase_write_trylock(&rwlock->rwbase); 219 if (ret) { 220 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); 221 rcu_read_lock(); 222 migrate_disable(); 223 } 224 return ret; 225} 226EXPORT_SYMBOL(rt_write_trylock); 227 228void __sched rt_read_lock(rwlock_t *rwlock) 229{ 230 rtlock_might_resched(); 231 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); 232 rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 233 rcu_read_lock(); 234 migrate_disable(); 235} 236EXPORT_SYMBOL(rt_read_lock); 237 238void __sched rt_write_lock(rwlock_t *rwlock) 239{ 240 rtlock_might_resched(); 241 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); 242 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 243 rcu_read_lock(); 244 migrate_disable(); 245} 246EXPORT_SYMBOL(rt_write_lock); 247 248#ifdef CONFIG_DEBUG_LOCK_ALLOC 249void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) 250{ 251 rtlock_might_resched(); 252 rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_); 253 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 254 rcu_read_lock(); 255 migrate_disable(); 256} 257EXPORT_SYMBOL(rt_write_lock_nested); 258#endif 259 260void __sched rt_read_unlock(rwlock_t *rwlock) 261{ 262 rwlock_release(&rwlock->dep_map, _RET_IP_); 263 migrate_enable(); 264 rcu_read_unlock(); 265 rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 266} 267EXPORT_SYMBOL(rt_read_unlock); 268 269void __sched rt_write_unlock(rwlock_t *rwlock) 270{ 271 rwlock_release(&rwlock->dep_map, _RET_IP_); 272 rcu_read_unlock(); 273 migrate_enable(); 274 rwbase_write_unlock(&rwlock->rwbase); 275} 276EXPORT_SYMBOL(rt_write_unlock); 277 278#ifdef CONFIG_DEBUG_LOCK_ALLOC 279void __rt_rwlock_init(rwlock_t *rwlock, const char *name, 280 struct lock_class_key *key) 281{ 282 debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); 283 lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG); 284} 285EXPORT_SYMBOL(__rt_rwlock_init); 286#endif 287