1/*-
2 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: stable/11/sys/kern/kern_condvar.c 315386 2017-03-16 07:10:08Z mjg $");
29
30#include "opt_ktrace.h"
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/limits.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/proc.h>
38#include <sys/kernel.h>
39#include <sys/ktr.h>
40#include <sys/condvar.h>
41#include <sys/sched.h>
42#include <sys/signalvar.h>
43#include <sys/sleepqueue.h>
44#include <sys/resourcevar.h>
45#ifdef KTRACE
46#include <sys/uio.h>
47#include <sys/ktrace.h>
48#endif
49
50/*
51 * A bound below which cv_waiters is valid.  Once cv_waiters reaches this bound,
52 * cv_signal must manually check the wait queue for threads.
53 */
54#define	CV_WAITERS_BOUND	INT_MAX
55
56#define	CV_WAITERS_INC(cvp) do {					\
57	if ((cvp)->cv_waiters < CV_WAITERS_BOUND)			\
58		(cvp)->cv_waiters++;					\
59} while (0)
60
61/*
62 * Common sanity checks for cv_wait* functions.
63 */
64#define	CV_ASSERT(cvp, lock, td) do {					\
65	KASSERT((td) != NULL, ("%s: td NULL", __func__));		\
66	KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__));	\
67	KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__));		\
68	KASSERT((lock) != NULL, ("%s: lock NULL", __func__));		\
69} while (0)
70
71/*
72 * Initialize a condition variable.  Must be called before use.
73 */
74void
75cv_init(struct cv *cvp, const char *desc)
76{
77
78	cvp->cv_description = desc;
79	cvp->cv_waiters = 0;
80}
81
82/*
83 * Destroy a condition variable.  The condition variable must be re-initialized
84 * in order to be re-used.
85 */
86void
87cv_destroy(struct cv *cvp)
88{
89#ifdef INVARIANTS
90	struct sleepqueue *sq;
91
92	sleepq_lock(cvp);
93	sq = sleepq_lookup(cvp);
94	sleepq_release(cvp);
95	KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__));
96#endif
97}
98
99/*
100 * Wait on a condition variable.  The current thread is placed on the condition
101 * variable's wait queue and suspended.  A cv_signal or cv_broadcast on the same
102 * condition variable will resume the thread.  The mutex is released before
103 * sleeping and will be held on return.  It is recommended that the mutex be
104 * held when cv_signal or cv_broadcast are called.
105 */
106void
107_cv_wait(struct cv *cvp, struct lock_object *lock)
108{
109	WITNESS_SAVE_DECL(lock_witness);
110	struct lock_class *class;
111	struct thread *td;
112	uintptr_t lock_state;
113
114	td = curthread;
115	lock_state = 0;
116#ifdef KTRACE
117	if (KTRPOINT(td, KTR_CSW))
118		ktrcsw(1, 0, cv_wmesg(cvp));
119#endif
120	CV_ASSERT(cvp, lock, td);
121	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
122	    "Waiting on \"%s\"", cvp->cv_description);
123	class = LOCK_CLASS(lock);
124
125	if (SCHEDULER_STOPPED_TD(td))
126		return;
127
128	sleepq_lock(cvp);
129
130	CV_WAITERS_INC(cvp);
131	if (lock == &Giant.lock_object)
132		mtx_assert(&Giant, MA_OWNED);
133	DROP_GIANT();
134
135	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
136	if (lock != &Giant.lock_object) {
137		if (class->lc_flags & LC_SLEEPABLE)
138			sleepq_release(cvp);
139		WITNESS_SAVE(lock, lock_witness);
140		lock_state = class->lc_unlock(lock);
141		if (class->lc_flags & LC_SLEEPABLE)
142			sleepq_lock(cvp);
143	}
144	sleepq_wait(cvp, 0);
145
146#ifdef KTRACE
147	if (KTRPOINT(td, KTR_CSW))
148		ktrcsw(0, 0, cv_wmesg(cvp));
149#endif
150	PICKUP_GIANT();
151	if (lock != &Giant.lock_object) {
152		class->lc_lock(lock, lock_state);
153		WITNESS_RESTORE(lock, lock_witness);
154	}
155}
156
157/*
158 * Wait on a condition variable.  This function differs from cv_wait by
159 * not acquiring the mutex after condition variable was signaled.
160 */
161void
162_cv_wait_unlock(struct cv *cvp, struct lock_object *lock)
163{
164	struct lock_class *class;
165	struct thread *td;
166
167	td = curthread;
168#ifdef KTRACE
169	if (KTRPOINT(td, KTR_CSW))
170		ktrcsw(1, 0, cv_wmesg(cvp));
171#endif
172	CV_ASSERT(cvp, lock, td);
173	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
174	    "Waiting on \"%s\"", cvp->cv_description);
175	KASSERT(lock != &Giant.lock_object,
176	    ("cv_wait_unlock cannot be used with Giant"));
177	class = LOCK_CLASS(lock);
178
179	if (SCHEDULER_STOPPED_TD(td)) {
180		class->lc_unlock(lock);
181		return;
182	}
183
184	sleepq_lock(cvp);
185
186	CV_WAITERS_INC(cvp);
187	DROP_GIANT();
188
189	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
190	if (class->lc_flags & LC_SLEEPABLE)
191		sleepq_release(cvp);
192	class->lc_unlock(lock);
193	if (class->lc_flags & LC_SLEEPABLE)
194		sleepq_lock(cvp);
195	sleepq_wait(cvp, 0);
196
197#ifdef KTRACE
198	if (KTRPOINT(td, KTR_CSW))
199		ktrcsw(0, 0, cv_wmesg(cvp));
200#endif
201	PICKUP_GIANT();
202}
203
204/*
205 * Wait on a condition variable, allowing interruption by signals.  Return 0 if
206 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if
207 * a signal was caught.  If ERESTART is returned the system call should be
208 * restarted if possible.
209 */
210int
211_cv_wait_sig(struct cv *cvp, struct lock_object *lock)
212{
213	WITNESS_SAVE_DECL(lock_witness);
214	struct lock_class *class;
215	struct thread *td;
216	uintptr_t lock_state;
217	int rval;
218
219	td = curthread;
220	lock_state = 0;
221#ifdef KTRACE
222	if (KTRPOINT(td, KTR_CSW))
223		ktrcsw(1, 0, cv_wmesg(cvp));
224#endif
225	CV_ASSERT(cvp, lock, td);
226	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
227	    "Waiting on \"%s\"", cvp->cv_description);
228	class = LOCK_CLASS(lock);
229
230	if (SCHEDULER_STOPPED_TD(td))
231		return (0);
232
233	sleepq_lock(cvp);
234
235	CV_WAITERS_INC(cvp);
236	if (lock == &Giant.lock_object)
237		mtx_assert(&Giant, MA_OWNED);
238	DROP_GIANT();
239
240	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
241	    SLEEPQ_INTERRUPTIBLE, 0);
242	if (lock != &Giant.lock_object) {
243		if (class->lc_flags & LC_SLEEPABLE)
244			sleepq_release(cvp);
245		WITNESS_SAVE(lock, lock_witness);
246		lock_state = class->lc_unlock(lock);
247		if (class->lc_flags & LC_SLEEPABLE)
248			sleepq_lock(cvp);
249	}
250	rval = sleepq_wait_sig(cvp, 0);
251
252#ifdef KTRACE
253	if (KTRPOINT(td, KTR_CSW))
254		ktrcsw(0, 0, cv_wmesg(cvp));
255#endif
256	PICKUP_GIANT();
257	if (lock != &Giant.lock_object) {
258		class->lc_lock(lock, lock_state);
259		WITNESS_RESTORE(lock, lock_witness);
260	}
261
262	return (rval);
263}
264
265/*
266 * Wait on a condition variable for (at most) the value specified in sbt
267 * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast,
268 * EWOULDBLOCK if the timeout expires.
269 */
270int
271_cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
272    sbintime_t pr, int flags)
273{
274	WITNESS_SAVE_DECL(lock_witness);
275	struct lock_class *class;
276	struct thread *td;
277	int lock_state, rval;
278
279	td = curthread;
280	lock_state = 0;
281#ifdef KTRACE
282	if (KTRPOINT(td, KTR_CSW))
283		ktrcsw(1, 0, cv_wmesg(cvp));
284#endif
285	CV_ASSERT(cvp, lock, td);
286	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
287	    "Waiting on \"%s\"", cvp->cv_description);
288	class = LOCK_CLASS(lock);
289
290	if (SCHEDULER_STOPPED_TD(td))
291		return (0);
292
293	sleepq_lock(cvp);
294
295	CV_WAITERS_INC(cvp);
296	if (lock == &Giant.lock_object)
297		mtx_assert(&Giant, MA_OWNED);
298	DROP_GIANT();
299
300	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
301	sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
302	if (lock != &Giant.lock_object) {
303		if (class->lc_flags & LC_SLEEPABLE)
304			sleepq_release(cvp);
305		WITNESS_SAVE(lock, lock_witness);
306		lock_state = class->lc_unlock(lock);
307		if (class->lc_flags & LC_SLEEPABLE)
308			sleepq_lock(cvp);
309	}
310	rval = sleepq_timedwait(cvp, 0);
311
312#ifdef KTRACE
313	if (KTRPOINT(td, KTR_CSW))
314		ktrcsw(0, 0, cv_wmesg(cvp));
315#endif
316	PICKUP_GIANT();
317	if (lock != &Giant.lock_object) {
318		class->lc_lock(lock, lock_state);
319		WITNESS_RESTORE(lock, lock_witness);
320	}
321
322	return (rval);
323}
324
325/*
326 * Wait on a condition variable for (at most) the value specified in sbt
327 * argument, allowing interruption by signals.
328 * Returns 0 if the thread was resumed by cv_signal or cv_broadcast,
329 * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal
330 * was caught.
331 */
332int
333_cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
334    sbintime_t sbt, sbintime_t pr, int flags)
335{
336	WITNESS_SAVE_DECL(lock_witness);
337	struct lock_class *class;
338	struct thread *td;
339	int lock_state, rval;
340
341	td = curthread;
342	lock_state = 0;
343#ifdef KTRACE
344	if (KTRPOINT(td, KTR_CSW))
345		ktrcsw(1, 0, cv_wmesg(cvp));
346#endif
347	CV_ASSERT(cvp, lock, td);
348	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
349	    "Waiting on \"%s\"", cvp->cv_description);
350	class = LOCK_CLASS(lock);
351
352	if (SCHEDULER_STOPPED_TD(td))
353		return (0);
354
355	sleepq_lock(cvp);
356
357	CV_WAITERS_INC(cvp);
358	if (lock == &Giant.lock_object)
359		mtx_assert(&Giant, MA_OWNED);
360	DROP_GIANT();
361
362	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
363	    SLEEPQ_INTERRUPTIBLE, 0);
364	sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
365	if (lock != &Giant.lock_object) {
366		if (class->lc_flags & LC_SLEEPABLE)
367			sleepq_release(cvp);
368		WITNESS_SAVE(lock, lock_witness);
369		lock_state = class->lc_unlock(lock);
370		if (class->lc_flags & LC_SLEEPABLE)
371			sleepq_lock(cvp);
372	}
373	rval = sleepq_timedwait_sig(cvp, 0);
374
375#ifdef KTRACE
376	if (KTRPOINT(td, KTR_CSW))
377		ktrcsw(0, 0, cv_wmesg(cvp));
378#endif
379	PICKUP_GIANT();
380	if (lock != &Giant.lock_object) {
381		class->lc_lock(lock, lock_state);
382		WITNESS_RESTORE(lock, lock_witness);
383	}
384
385	return (rval);
386}
387
388/*
389 * Signal a condition variable, wakes up one waiting thread.  Will also wakeup
390 * the swapper if the process is not in memory, so that it can bring the
391 * sleeping process in.  Note that this may also result in additional threads
392 * being made runnable.  Should be called with the same mutex as was passed to
393 * cv_wait held.
394 */
395void
396cv_signal(struct cv *cvp)
397{
398	int wakeup_swapper;
399
400	if (cvp->cv_waiters == 0)
401		return;
402	wakeup_swapper = 0;
403	sleepq_lock(cvp);
404	if (cvp->cv_waiters > 0) {
405		if (cvp->cv_waiters == CV_WAITERS_BOUND &&
406		    sleepq_lookup(cvp) == NULL) {
407			cvp->cv_waiters = 0;
408		} else {
409			if (cvp->cv_waiters < CV_WAITERS_BOUND)
410				cvp->cv_waiters--;
411			wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0,
412			    0);
413		}
414	}
415	sleepq_release(cvp);
416	if (wakeup_swapper)
417		kick_proc0();
418}
419
420/*
421 * Broadcast a signal to a condition variable.  Wakes up all waiting threads.
422 * Should be called with the same mutex as was passed to cv_wait held.
423 */
424void
425cv_broadcastpri(struct cv *cvp, int pri)
426{
427	int wakeup_swapper;
428
429	if (cvp->cv_waiters == 0)
430		return;
431	/*
432	 * XXX sleepq_broadcast pri argument changed from -1 meaning
433	 * no pri to 0 meaning no pri.
434	 */
435	wakeup_swapper = 0;
436	if (pri == -1)
437		pri = 0;
438	sleepq_lock(cvp);
439	if (cvp->cv_waiters > 0) {
440		cvp->cv_waiters = 0;
441		wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
442	}
443	sleepq_release(cvp);
444	if (wakeup_swapper)
445		kick_proc0();
446}
447