kernel/locking/rwbase_rt.c

53 static __always_inline int rwbase_read_trylock(struct rwbase_rt *rwb)
61 	for (r = atomic_read(&rwb->readers); r < 0;) {
62 		if (likely(atomic_try_cmpxchg_acquire(&rwb->readers, &r, r + 1)))
68 static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
71 	struct rt_mutex_base *rtm = &rwb->rtmutex;
107 	trace_contention_begin(rwb, LCB_F_RT | LCB_F_READ);
123 		atomic_inc(&rwb->readers);
128 	trace_contention_end(rwb, ret);
133 static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb,
138 	if (rwbase_read_trylock(rwb))
141 	return __rwbase_read_lock(rwb, state);
144 static void __sched __rwbase_read_unlock(struct rwbase_rt *rwb,
147 	struct rt_mutex_base *rtm = &rwb->rtmutex;
155 	 * clean up rwb->readers it needs to acquire rtm->wait_lock. The
168 static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,
172 	 * rwb->readers can only hit 0 when a writer is waiting for the
177 	if (unlikely(atomic_dec_and_test(&rwb->readers)))
178 		__rwbase_read_unlock(rwb, state);
181 static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,
184 	struct rt_mutex_base *rtm = &rwb->rtmutex;
190 	(void)atomic_add_return_release(READER_BIAS - bias, &rwb->readers);
195 static inline void rwbase_write_unlock(struct rwbase_rt *rwb)
197 	struct rt_mutex_base *rtm = &rwb->rtmutex;
201 	__rwbase_write_unlock(rwb, WRITER_BIAS, flags);
204 static inline void rwbase_write_downgrade(struct rwbase_rt *rwb)
206 	struct rt_mutex_base *rtm = &rwb->rtmutex;
211 	__rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags);
214 static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb)
217 	lockdep_assert_held(&rwb->rtmutex.wait_lock);
223 	if (!atomic_read_acquire(&rwb->readers)) {
224 		atomic_set(&rwb->readers, WRITER_BIAS);
231 static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
234 	struct rt_mutex_base *rtm = &rwb->rtmutex;
242 	atomic_sub(READER_BIAS, &rwb->readers);
247 	if (__rwbase_write_trylock(rwb))
251 	trace_contention_begin(rwb, LCB_F_RT | LCB_F_WRITE);
256 			__rwbase_write_unlock(rwb, 0, flags);
258 			trace_contention_end(rwb, -EINTR);
262 		if (__rwbase_write_trylock(rwb))
272 	trace_contention_end(rwb, 0);
280 static inline int rwbase_write_trylock(struct rwbase_rt *rwb)
282 	struct rt_mutex_base *rtm = &rwb->rtmutex;
288 	atomic_sub(READER_BIAS, &rwb->readers);
291 	if (__rwbase_write_trylock(rwb)) {
295 	__rwbase_write_unlock(rwb, 0, flags);