kernel/sched/wait.c

8 void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key)
10 	spin_lock_init(&wq_head->lock);
11 	lockdep_set_class_and_name(&wq_head->lock, key, name);
12 	INIT_LIST_HEAD(&wq_head->head);
17 void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
22 	spin_lock_irqsave(&wq_head->lock, flags);
23 	__add_wait_queue(wq_head, wq_entry);
24 	spin_unlock_irqrestore(&wq_head->lock, flags);
28 void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
33 	spin_lock_irqsave(&wq_head->lock, flags);
34 	__add_wait_queue_entry_tail(wq_head, wq_entry);
35 	spin_unlock_irqrestore(&wq_head->lock, flags);
39 void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
44 	spin_lock_irqsave(&wq_head->lock, flags);
45 	__add_wait_queue(wq_head, wq_entry);
46 	spin_unlock_irqrestore(&wq_head->lock, flags);
50 void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
54 	spin_lock_irqsave(&wq_head->lock, flags);
55 	__remove_wait_queue(wq_head, wq_entry);
56 	spin_unlock_irqrestore(&wq_head->lock, flags);
73 static int __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode,
78 	lockdep_assert_held(&wq_head->lock);
80 	curr = list_first_entry(&wq_head->head, wait_queue_entry_t, entry);
82 	if (&curr->entry == &wq_head->head)
85 	list_for_each_entry_safe_from(curr, next, &wq_head->head, entry) {
99 static int __wake_up_common_lock(struct wait_queue_head *wq_head, unsigned int mode,
105 	spin_lock_irqsave(&wq_head->lock, flags);
106 	remaining = __wake_up_common(wq_head, mode, nr_exclusive, wake_flags,
108 	spin_unlock_irqrestore(&wq_head->lock, flags);
115  * @wq_head: the waitqueue
124 int __wake_up(struct wait_queue_head *wq_head, unsigned int mode,
127 	return __wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key);
131 void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key)
133 	__wake_up_common_lock(wq_head, mode, 1, WF_CURRENT_CPU, key);
139 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr)
141 	__wake_up_common(wq_head, mode, nr, 0, NULL);
145 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key)
147 	__wake_up_common(wq_head, mode, 1, 0, key);
153  * @wq_head: the waitqueue
167 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode,
170 	if (unlikely(!wq_head))
173 	__wake_up_common_lock(wq_head, mode, 1, WF_SYNC, key);
179  * @wq_head: the waitqueue
193 void __wake_up_locked_sync_key(struct wait_queue_head *wq_head,
196         __wake_up_common(wq_head, mode, 1, WF_SYNC, key);
203 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode)
205 	__wake_up_sync_key(wq_head, mode, NULL);
209 void __wake_up_pollfree(struct wait_queue_head *wq_head)
211 	__wake_up(wq_head, TASK_NORMAL, 0, poll_to_key(EPOLLHUP | POLLFREE));
213 	WARN_ON_ONCE(waitqueue_active(wq_head));
229 prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
234 	spin_lock_irqsave(&wq_head->lock, flags);
236 		__add_wait_queue(wq_head, wq_entry);
238 	spin_unlock_irqrestore(&wq_head->lock, flags);
244 prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
250 	spin_lock_irqsave(&wq_head->lock, flags);
252 		was_empty = list_empty(&wq_head->head);
253 		__add_wait_queue_entry_tail(wq_head, wq_entry);
256 	spin_unlock_irqrestore(&wq_head->lock, flags);
270 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
275 	spin_lock_irqsave(&wq_head->lock, flags);
283 		 * wakeup locks/unlocks the same wq_head->lock.
294 				__add_wait_queue_entry_tail(wq_head, wq_entry);
296 				__add_wait_queue(wq_head, wq_entry);
300 	spin_unlock_irqrestore(&wq_head->lock, flags);
349  * @wq_head: waitqueue waited on
356 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
375 		spin_lock_irqsave(&wq_head->lock, flags);
377 		spin_unlock_irqrestore(&wq_head->lock, flags);
396  * add_wait_queue(&wq_head, &wait);
411  * remove_wait_queue(&wq_head, &wait);		wq_entry->flags |= WQ_FLAG_WOKEN;