Lines Matching refs:lock
27 #include <trace/events/lock.h>
36 struct rt_mutex *lock,
42 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock,
47 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock,
52 static inline int __ww_mutex_check_kill(struct rt_mutex *lock,
66 * lock->owner state tracking:
68 * lock->owner holds the task_struct pointer of the owner. Bit 0
69 * is used to keep track of the "lock has waiters" state.
72 * NULL 0 lock is free (fast acquire possible)
73 * NULL 1 lock is free and has waiters and the top waiter
74 * is going to take the lock*
75 * taskpointer 0 lock is held (fast release possible)
76 * taskpointer 1 lock is held and has waiters**
79 * possible when bit 0 of lock->owner is 0.
81 * (*) It also can be a transitional state when grabbing the lock
82 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
83 * we need to set the bit0 before looking at the lock, and the owner may be
87 * waiters. This can happen when grabbing the lock in the slow path.
88 * To prevent a cmpxchg of the owner releasing the lock, we need to
89 * set this bit before looking at the lock.
93 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)
97 if (rt_mutex_has_waiters(lock))
104 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
107 * lock->wait_lock is held but explicit acquire semantics are needed
108 * for a new lock owner so WRITE_ONCE is insufficient.
110 xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner));
113 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock)
115 /* lock->wait_lock is held so the unlock provides release semantics. */
116 WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));
119 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
121 lock->owner = (struct task_struct *)
122 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
126 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock)
128 unsigned long owner, *p = (unsigned long *) &lock->owner;
130 if (rt_mutex_has_waiters(lock))
135 * lock->owner still has the waiters bit set, otherwise the
141 * lock(l->lock)
145 * unlock(l->lock)
149 * lock(l->lock)
153 * unlock(l->lock)
156 * lock(l->lock)
159 * unlock(l->lock)
160 * lock(l->lock)
164 * unlock(l->lock)
165 * lock(l->lock)
172 * lock(l->lock)
185 * serialized by l->lock, so nothing else can modify the waiters
199 * in case that the lock acquisition failed it might
214 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
218 return try_cmpxchg_acquire(&lock->owner, &old, new);
221 static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
223 return rt_mutex_cmpxchg_acquire(lock, NULL, current);
226 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
230 return try_cmpxchg_release(&lock->owner, &old, new);
235 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
238 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
240 unsigned long *p = (unsigned long *) &lock->owner;
259 * 2) Drop lock->wait_lock
260 * 3) Try to unlock the lock with cmpxchg
262 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,
264 __releases(lock->wait_lock)
266 struct task_struct *owner = rt_mutex_owner(lock);
268 clear_rt_mutex_waiters(lock);
269 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
275 * lock(wait_lock);
277 * mark_rt_mutex_waiters(lock);
278 * acquire(lock);
282 * lock(wait_lock);
283 * mark_rt_mutex_waiters(lock);
288 * lock(wait_lock);
291 * lock(wait_lock);
292 * acquire(lock);
294 return rt_mutex_cmpxchg_release(lock, owner, NULL);
298 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
306 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock);
308 static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
317 return rt_mutex_slowtrylock(lock);
320 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
327 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
329 lock->owner = (struct task_struct *)
330 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
334 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
336 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,
338 __releases(lock->wait_lock)
340 lock->owner = NULL;
341 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
362 lockdep_assert_held(&waiter->lock->wait_lock);
375 lockdep_assert_held(&waiter->lock->wait_lock);
477 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
479 lockdep_assert_held(&lock->wait_lock);
481 rb_add_cached(&waiter->tree.entry, &lock->waiters, __waiter_less);
485 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
487 lockdep_assert_held(&lock->wait_lock);
492 rb_erase_cached(&waiter->tree.entry, &lock->waiters);
524 static __always_inline void rt_mutex_adjust_prio(struct rt_mutex_base *lock,
529 lockdep_assert_held(&lock->wait_lock);
530 lockdep_assert(rt_mutex_owner(lock) == p);
599 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
614 * comparison to detect lock chain changes.
648 * [1] lock(task->pi_lock); [R] acquire [P1]
651 * [4] lock = waiter->lock; [P1]
652 * [5] if (!try_lock(lock->wait_lock)) { [P1] try to acquire [L]
657 * [7] requeue_lock_waiter(lock, waiter); [P1] + [L]
661 * [10] task = owner(lock); [L]
663 * lock(task->pi_lock); [L] acquire [P2]
664 * [11] requeue_pi_waiter(tsk, waiters(lock));[P2] + [L]
667 * unlock(lock->wait_lock); release [L]
670 * Where P1 is the blocking task and P2 is the lock owner; going up one step
685 struct rt_mutex_base *lock;
699 * We limit the lock chain length for each invocation.
710 printk(KERN_WARNING "Maximum lock depth %d reached "
750 * the previous owner of the lock might have released the lock.
757 * the task might have moved on in the lock chain or even left
758 * the chain completely and blocks now on an unrelated lock or
761 * We stored the lock on which @task was blocked in @next_lock,
764 if (next_lock != waiter->lock)
768 * There could be 'spurious' loops in the lock graph due to ww_mutex,
829 * [4] Get the next lock; per holding task->pi_lock we can't unblock
830 * and guarantee @lock's existence.
832 lock = waiter->lock;
835 * which is the reverse lock order versus the other rtmutex
838 * Per the above, holding task->pi_lock guarantees lock exists, so
839 * inverting this lock order is infeasible from a life-time
842 if (!raw_spin_trylock(&lock->wait_lock)) {
850 * lock->wait_lock.
852 * Deadlock detection. If the lock is the same as the original
853 * lock which caused us to walk the lock chain or if the
854 * current lock is owned by the task which initiated the chain
857 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
872 raw_spin_unlock(&lock->wait_lock);
877 * If we just follow the lock chain for deadlock detection, no
890 * [9] check_exit_conditions_3 protected by lock->wait_lock.
891 * If there is no owner of the lock, end of chain.
893 if (!rt_mutex_owner(lock)) {
894 raw_spin_unlock_irq(&lock->wait_lock);
898 /* [10] Grab the next task, i.e. owner of @lock */
899 task = get_task_struct(rt_mutex_owner(lock));
912 top_waiter = rt_mutex_top_waiter(lock);
916 raw_spin_unlock_irq(&lock->wait_lock);
926 * operation on @lock. We need it for the boost/deboost
929 prerequeue_top_waiter = rt_mutex_top_waiter(lock);
931 /* [7] Requeue the waiter in the lock waiter tree. */
932 rt_mutex_dequeue(lock, waiter);
947 rt_mutex_enqueue(lock, waiter);
953 * Since we hold lock->waiter_lock, task cannot unblock, even if we
960 * [9] check_exit_conditions_3 protected by lock->wait_lock.
962 * We must abort the chain walk if there is no lock owner even
963 * in the dead lock detection case, as we have nothing to
966 if (!rt_mutex_owner(lock)) {
970 * to get the lock.
972 top_waiter = rt_mutex_top_waiter(lock);
975 raw_spin_unlock_irq(&lock->wait_lock);
980 * [10] Grab the next task, i.e. the owner of @lock
982 * Per holding lock->wait_lock and checking for !owner above, there
985 task = get_task_struct(rt_mutex_owner(lock));
989 if (waiter == rt_mutex_top_waiter(lock)) {
992 * waiter on the lock. Replace the previous top waiter
999 rt_mutex_adjust_prio(lock, task);
1003 * The waiter was the top waiter on the lock, but is
1013 waiter = rt_mutex_top_waiter(lock);
1016 rt_mutex_adjust_prio(lock, task);
1026 * and lock->wait_lock. The actual decisions are made after we
1029 * Check whether the task which owns the current lock is pi
1030 * blocked itself. If yes we store a pointer to the lock for
1031 * the lock chain change detection above. After we dropped
1036 * Store the top waiter of @lock for the end of chain walk
1039 top_waiter = rt_mutex_top_waiter(lock);
1043 raw_spin_unlock_irq(&lock->wait_lock);
1049 * We reached the end of the lock chain. Stop right here. No
1056 * If the current waiter is not the top waiter on the lock,
1076 * Must be called with lock->wait_lock held and interrupts disabled
1078 * @lock: The lock to be acquired.
1079 * @task: The task which wants to acquire the lock
1080 * @waiter: The waiter that is queued to the lock's wait tree if the
1084 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
1087 lockdep_assert_held(&lock->wait_lock);
1090 * Before testing whether we can acquire @lock, we set the
1091 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
1092 * other tasks which try to modify @lock into the slow path
1093 * and they serialize on @lock->wait_lock.
1098 * - There is a lock owner. The caller must fixup the
1099 * transient state if it does a trylock or leaves the lock
1102 * - @task acquires the lock and there are no other
1106 mark_rt_mutex_waiters(lock);
1109 * If @lock has an owner, give up.
1111 if (rt_mutex_owner(lock))
1116 * into @lock waiter tree. If @waiter == NULL then this is a
1120 struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock);
1123 * If waiter is the highest priority waiter of @lock,
1128 * We can acquire the lock. Remove the waiter from the
1129 * lock waiters tree.
1131 rt_mutex_dequeue(lock, waiter);
1137 * If the lock has waiters already we check whether @task is
1138 * eligible to take over the lock.
1141 * the lock. @task->pi_blocked_on is NULL, so it does
1144 if (rt_mutex_has_waiters(lock)) {
1147 rt_mutex_top_waiter(lock)))
1152 * don't have to change anything in the lock
1157 * No waiters. Take the lock without the
1175 * Finish the lock acquisition. @task is the new owner. If
1179 if (rt_mutex_has_waiters(lock))
1180 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
1188 rt_mutex_set_owner(lock, task);
1194 * Task blocks on lock.
1198 * This must be called with lock->wait_lock held and interrupts disabled
1200 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
1206 struct task_struct *owner = rt_mutex_owner(lock);
1211 lockdep_assert_held(&lock->wait_lock);
1230 waiter->lock = lock;
1234 /* Get the top priority waiter on the lock */
1235 if (rt_mutex_has_waiters(lock))
1236 top_waiter = rt_mutex_top_waiter(lock);
1237 rt_mutex_enqueue(lock, waiter);
1247 rtm = container_of(lock, struct rt_mutex, rtmutex);
1251 rt_mutex_dequeue(lock, waiter);
1262 if (waiter == rt_mutex_top_waiter(lock)) {
1266 rt_mutex_adjust_prio(lock, owner);
1273 /* Store the lock on which owner is blocked or NULL */
1286 * The owner can't disappear while holding a lock,
1292 raw_spin_unlock_irq(&lock->wait_lock);
1294 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
1297 raw_spin_lock_irq(&lock->wait_lock);
1306 * Called with lock->wait_lock held and interrupts disabled.
1309 struct rt_mutex_base *lock)
1313 lockdep_assert_held(&lock->wait_lock);
1317 waiter = rt_mutex_top_waiter(lock);
1327 rt_mutex_adjust_prio(lock, current);
1331 * queued on the lock until it gets the lock, this lock
1335 * the top waiter can steal this lock.
1337 lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
1354 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
1356 int ret = try_to_take_rt_mutex(lock, current, NULL);
1359 * try_to_take_rt_mutex() sets the lock waiters bit
1362 fixup_rt_mutex_waiters(lock, true);
1368 * Slow path try-lock function:
1370 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)
1376 * If the lock already has an owner we fail to get the lock.
1377 * This can be done without taking the @lock->wait_lock as
1380 if (rt_mutex_owner(lock))
1384 * The mutex has currently no owner. Lock the wait lock and try to
1385 * acquire the lock. We use irqsave here to support early boot calls.
1387 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1389 ret = __rt_mutex_slowtrylock(lock);
1391 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1396 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)
1398 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
1401 return rt_mutex_slowtrylock(lock);
1407 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)
1413 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1415 debug_rt_mutex_unlock(lock);
1422 * foo->lock->owner = NULL;
1423 * rtmutex_lock(foo->lock); <- fast path
1425 * rtmutex_unlock(foo->lock); <- fast path
1428 * raw_spin_unlock(foo->lock->wait_lock);
1433 * lock->wait_lock. So we do the following sequence:
1435 * owner = rt_mutex_owner(lock);
1436 * clear_rt_mutex_waiters(lock);
1437 * raw_spin_unlock(&lock->wait_lock);
1438 * if (cmpxchg(&lock->owner, owner, 0) == owner)
1443 * lock->owner is serialized by lock->wait_lock:
1445 * lock->owner = NULL;
1446 * raw_spin_unlock(&lock->wait_lock);
1448 while (!rt_mutex_has_waiters(lock)) {
1449 /* Drops lock->wait_lock ! */
1450 if (unlock_rt_mutex_safe(lock, flags) == true)
1453 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1462 mark_wakeup_next_waiter(&wqh, lock);
1463 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1468 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)
1470 if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
1473 rt_mutex_slowunlock(lock);
1477 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
1486 if (owner != rt_mutex_owner(lock))
1490 * the lock owner still matches @owner. If that fails,
1497 * - the lock owner has been scheduled out
1504 !rt_mutex_waiter_is_top_waiter(lock, waiter)) {
1514 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
1530 * Remove a waiter from a lock and give up
1532 * Must be called with lock->wait_lock held and interrupts disabled. It must
1535 static void __sched remove_waiter(struct rt_mutex_base *lock,
1538 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
1539 struct task_struct *owner = rt_mutex_owner(lock);
1542 lockdep_assert_held(&lock->wait_lock);
1545 rt_mutex_dequeue(lock, waiter);
1551 * waiter of the lock and there is an owner to update.
1560 if (rt_mutex_has_waiters(lock))
1561 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
1563 rt_mutex_adjust_prio(lock, owner);
1565 /* Store the lock on which owner is blocked or NULL */
1580 raw_spin_unlock_irq(&lock->wait_lock);
1582 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
1585 raw_spin_lock_irq(&lock->wait_lock);
1590 * @lock: the rt_mutex to take
1597 * Must be called with lock->wait_lock held and interrupts disabled
1599 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
1605 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);
1610 /* Try to acquire the lock: */
1611 if (try_to_take_rt_mutex(lock, current, waiter))
1629 if (waiter == rt_mutex_top_waiter(lock))
1630 owner = rt_mutex_owner(lock);
1633 raw_spin_unlock_irq(&lock->wait_lock);
1635 if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))
1638 raw_spin_lock_irq(&lock->wait_lock);
1670 * __rt_mutex_slowlock - Locking slowpath invoked with lock::wait_lock held
1671 * @lock: The rtmutex to block lock
1677 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
1683 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);
1687 lockdep_assert_held(&lock->wait_lock);
1689 /* Try to acquire the lock again: */
1690 if (try_to_take_rt_mutex(lock, current, NULL)) {
1700 trace_contention_begin(lock, LCB_F_RT);
1702 ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);
1704 ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);
1707 /* acquired the lock */
1715 remove_waiter(lock, waiter);
1723 fixup_rt_mutex_waiters(lock, true);
1725 trace_contention_end(lock, ret);
1730 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,
1740 ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK,
1749 * @lock: The rtmutex to block lock
1753 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
1773 * rtmutex with lock->wait_lock held. But we cannot unconditionally
1777 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1778 ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);
1779 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1785 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
1790 if (likely(rt_mutex_try_acquire(lock)))
1793 return rt_mutex_slowlock(lock, NULL, state);
1803 * rtlock_slowlock_locked - Slow path lock acquisition for RT locks
1804 * @lock: The underlying RT mutex
1806 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
1811 lockdep_assert_held(&lock->wait_lock);
1813 if (try_to_take_rt_mutex(lock, current, NULL))
1821 trace_contention_begin(lock, LCB_F_RT);
1823 task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);
1826 /* Try to acquire the lock again */
1827 if (try_to_take_rt_mutex(lock, current, &waiter))
1830 if (&waiter == rt_mutex_top_waiter(lock))
1831 owner = rt_mutex_owner(lock);
1834 raw_spin_unlock_irq(&lock->wait_lock);
1836 if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))
1839 raw_spin_lock_irq(&lock->wait_lock);
1850 fixup_rt_mutex_waiters(lock, true);
1853 trace_contention_end(lock, 0);
1856 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)
1860 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1861 rtlock_slowlock_locked(lock);
1862 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);