171088Sjasone/*- 271088Sjasone * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 371088Sjasone * All rights reserved. 471088Sjasone * 571088Sjasone * Redistribution and use in source and binary forms, with or without 671088Sjasone * modification, are permitted provided that the following conditions 771088Sjasone * are met: 871088Sjasone * 1. Redistributions of source code must retain the above copyright 971088Sjasone * notice, this list of conditions and the following disclaimer. 1071088Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1171088Sjasone * notice, this list of conditions and the following disclaimer in the 1271088Sjasone * documentation and/or other materials provided with the distribution. 1371088Sjasone * 1471088Sjasone * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1571088Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1671088Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1771088Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1871088Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1971088Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2071088Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2171088Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2271088Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2371088Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2471088Sjasone * SUCH DAMAGE. 2571088Sjasone */ 2671088Sjasone 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD: stable/11/sys/kern/kern_condvar.c 315386 2017-03-16 07:10:08Z mjg $"); 29116182Sobrien 3071088Sjasone#include "opt_ktrace.h" 3171088Sjasone 3271088Sjasone#include <sys/param.h> 3371088Sjasone#include <sys/systm.h> 34293458Smarkj#include <sys/limits.h> 3576166Smarkm#include <sys/lock.h> 3676166Smarkm#include <sys/mutex.h> 3771088Sjasone#include <sys/proc.h> 3871088Sjasone#include <sys/kernel.h> 3971088Sjasone#include <sys/ktr.h> 4071088Sjasone#include <sys/condvar.h> 41109862Sjeff#include <sys/sched.h> 4271088Sjasone#include <sys/signalvar.h> 43126326Sjhb#include <sys/sleepqueue.h> 4471088Sjasone#include <sys/resourcevar.h> 4571088Sjasone#ifdef KTRACE 4671088Sjasone#include <sys/uio.h> 4771088Sjasone#include <sys/ktrace.h> 4871088Sjasone#endif 4971088Sjasone 5071088Sjasone/* 51293458Smarkj * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound, 52293458Smarkj * cv_signal must manually check the wait queue for threads. 53293458Smarkj */ 54293458Smarkj#define CV_WAITERS_BOUND INT_MAX 55293458Smarkj 56293458Smarkj#define CV_WAITERS_INC(cvp) do { \ 57293458Smarkj if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \ 58293458Smarkj (cvp)->cv_waiters++; \ 59293458Smarkj} while (0) 60293458Smarkj 61293458Smarkj/* 6271088Sjasone * Common sanity checks for cv_wait* functions. 6371088Sjasone */ 64167789Sjhb#define CV_ASSERT(cvp, lock, td) do { \ 65240475Sattilio KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 66103216Sjulian KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 6787594Sobrien KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 68167789Sjhb KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 6971088Sjasone} while (0) 7071088Sjasone 7171088Sjasone/* 7271088Sjasone * Initialize a condition variable. Must be called before use. 7371088Sjasone */ 7471088Sjasonevoid 7571088Sjasonecv_init(struct cv *cvp, const char *desc) 7671088Sjasone{ 7771088Sjasone 7871088Sjasone cvp->cv_description = desc; 79127954Sjhb cvp->cv_waiters = 0; 8071088Sjasone} 8171088Sjasone 8271088Sjasone/* 8371088Sjasone * Destroy a condition variable. The condition variable must be re-initialized 8471088Sjasone * in order to be re-used. 8571088Sjasone */ 8671088Sjasonevoid 8771088Sjasonecv_destroy(struct cv *cvp) 8871088Sjasone{ 89126326Sjhb#ifdef INVARIANTS 90136445Sjhb struct sleepqueue *sq; 9171088Sjasone 92136445Sjhb sleepq_lock(cvp); 93126326Sjhb sq = sleepq_lookup(cvp); 94126326Sjhb sleepq_release(cvp); 95126326Sjhb KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 96126326Sjhb#endif 9771088Sjasone} 9871088Sjasone 9971088Sjasone/* 10083366Sjulian * Wait on a condition variable. The current thread is placed on the condition 10171088Sjasone * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 10283366Sjulian * condition variable will resume the thread. The mutex is released before 10371088Sjasone * sleeping and will be held on return. It is recommended that the mutex be 10471088Sjasone * held when cv_signal or cv_broadcast are called. 10571088Sjasone */ 10671088Sjasonevoid 107167789Sjhb_cv_wait(struct cv *cvp, struct lock_object *lock) 10871088Sjasone{ 109167789Sjhb WITNESS_SAVE_DECL(lock_witness); 110167789Sjhb struct lock_class *class; 111167786Sjhb struct thread *td; 112255745Sdavide uintptr_t lock_state; 11371088Sjasone 114167786Sjhb td = curthread; 115181394Sjhb lock_state = 0; 116167786Sjhb#ifdef KTRACE 117167786Sjhb if (KTRPOINT(td, KTR_CSW)) 118234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 119167786Sjhb#endif 120167789Sjhb CV_ASSERT(cvp, lock, td); 121167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 122167786Sjhb "Waiting on \"%s\"", cvp->cv_description); 123167789Sjhb class = LOCK_CLASS(lock); 124153321Srodrigc 125315386Smjg if (SCHEDULER_STOPPED_TD(td)) 126153321Srodrigc return; 127153321Srodrigc 128167786Sjhb sleepq_lock(cvp); 129167786Sjhb 130293458Smarkj CV_WAITERS_INC(cvp); 131183352Sjhb if (lock == &Giant.lock_object) 132183352Sjhb mtx_assert(&Giant, MA_OWNED); 133167786Sjhb DROP_GIANT(); 134167786Sjhb 135167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 136181394Sjhb if (lock != &Giant.lock_object) { 137181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 138181394Sjhb sleepq_release(cvp); 139183352Sjhb WITNESS_SAVE(lock, lock_witness); 140181394Sjhb lock_state = class->lc_unlock(lock); 141181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 142181394Sjhb sleepq_lock(cvp); 143181394Sjhb } 144177085Sjeff sleepq_wait(cvp, 0); 145167786Sjhb 146167786Sjhb#ifdef KTRACE 147167786Sjhb if (KTRPOINT(td, KTR_CSW)) 148234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 149167786Sjhb#endif 150167786Sjhb PICKUP_GIANT(); 151181394Sjhb if (lock != &Giant.lock_object) { 152181394Sjhb class->lc_lock(lock, lock_state); 153181394Sjhb WITNESS_RESTORE(lock, lock_witness); 154181394Sjhb } 155153321Srodrigc} 156153321Srodrigc 157153321Srodrigc/* 158153321Srodrigc * Wait on a condition variable. This function differs from cv_wait by 159298819Spfg * not acquiring the mutex after condition variable was signaled. 160153321Srodrigc */ 161153321Srodrigcvoid 162167789Sjhb_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 163153321Srodrigc{ 164167789Sjhb struct lock_class *class; 165153321Srodrigc struct thread *td; 166153321Srodrigc 16783366Sjulian td = curthread; 16871088Sjasone#ifdef KTRACE 16997995Sjhb if (KTRPOINT(td, KTR_CSW)) 170234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 17171088Sjasone#endif 172167789Sjhb CV_ASSERT(cvp, lock, td); 173167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 174111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 175181394Sjhb KASSERT(lock != &Giant.lock_object, 176181394Sjhb ("cv_wait_unlock cannot be used with Giant")); 177167789Sjhb class = LOCK_CLASS(lock); 17871088Sjasone 179315386Smjg if (SCHEDULER_STOPPED_TD(td)) { 180167789Sjhb class->lc_unlock(lock); 18171088Sjasone return; 18271088Sjasone } 18395322Shsu 184136445Sjhb sleepq_lock(cvp); 18595322Shsu 186293458Smarkj CV_WAITERS_INC(cvp); 18788900Sjhb DROP_GIANT(); 18871088Sjasone 189167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 190169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 191169392Sjhb sleepq_release(cvp); 192169392Sjhb class->lc_unlock(lock); 193169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 194169392Sjhb sleepq_lock(cvp); 195177085Sjeff sleepq_wait(cvp, 0); 19671088Sjasone 197167786Sjhb#ifdef KTRACE 198167786Sjhb if (KTRPOINT(td, KTR_CSW)) 199234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 200167786Sjhb#endif 20171088Sjasone PICKUP_GIANT(); 20271088Sjasone} 20371088Sjasone 20471088Sjasone/* 20571088Sjasone * Wait on a condition variable, allowing interruption by signals. Return 0 if 20683366Sjulian * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 20771088Sjasone * a signal was caught. If ERESTART is returned the system call should be 20871088Sjasone * restarted if possible. 20971088Sjasone */ 21071088Sjasoneint 211167789Sjhb_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 21271088Sjasone{ 213167789Sjhb WITNESS_SAVE_DECL(lock_witness); 214167789Sjhb struct lock_class *class; 21583366Sjulian struct thread *td; 216255745Sdavide uintptr_t lock_state; 217255745Sdavide int rval; 21871088Sjasone 21983366Sjulian td = curthread; 220181394Sjhb lock_state = 0; 22171088Sjasone#ifdef KTRACE 22297995Sjhb if (KTRPOINT(td, KTR_CSW)) 223234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 22471088Sjasone#endif 225167789Sjhb CV_ASSERT(cvp, lock, td); 226167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 227111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 228167789Sjhb class = LOCK_CLASS(lock); 22971088Sjasone 230315386Smjg if (SCHEDULER_STOPPED_TD(td)) 231133440Sjhb return (0); 23295322Shsu 233136445Sjhb sleepq_lock(cvp); 23495322Shsu 235293458Smarkj CV_WAITERS_INC(cvp); 236183352Sjhb if (lock == &Giant.lock_object) 237183352Sjhb mtx_assert(&Giant, MA_OWNED); 23888900Sjhb DROP_GIANT(); 23971088Sjasone 240167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 241165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 242181394Sjhb if (lock != &Giant.lock_object) { 243181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 244181394Sjhb sleepq_release(cvp); 245183352Sjhb WITNESS_SAVE(lock, lock_witness); 246181394Sjhb lock_state = class->lc_unlock(lock); 247181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 248181394Sjhb sleepq_lock(cvp); 249181394Sjhb } 250177085Sjeff rval = sleepq_wait_sig(cvp, 0); 25171088Sjasone 25271088Sjasone#ifdef KTRACE 25397995Sjhb if (KTRPOINT(td, KTR_CSW)) 254234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 25571088Sjasone#endif 25697995Sjhb PICKUP_GIANT(); 257181394Sjhb if (lock != &Giant.lock_object) { 258181394Sjhb class->lc_lock(lock, lock_state); 259181394Sjhb WITNESS_RESTORE(lock, lock_witness); 260181394Sjhb } 26171088Sjasone 26271088Sjasone return (rval); 26371088Sjasone} 26471088Sjasone 26571088Sjasone/* 266247785Sdavide * Wait on a condition variable for (at most) the value specified in sbt 267247785Sdavide * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 268247785Sdavide * EWOULDBLOCK if the timeout expires. 26971088Sjasone */ 27071088Sjasoneint 271247785Sdavide_cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 272247785Sdavide sbintime_t pr, int flags) 27371088Sjasone{ 274167789Sjhb WITNESS_SAVE_DECL(lock_witness); 275167789Sjhb struct lock_class *class; 27683366Sjulian struct thread *td; 277167789Sjhb int lock_state, rval; 27871088Sjasone 27983366Sjulian td = curthread; 280181394Sjhb lock_state = 0; 28171088Sjasone#ifdef KTRACE 28297995Sjhb if (KTRPOINT(td, KTR_CSW)) 283234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 28471088Sjasone#endif 285167789Sjhb CV_ASSERT(cvp, lock, td); 286167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 287111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 288167789Sjhb class = LOCK_CLASS(lock); 28971088Sjasone 290315386Smjg if (SCHEDULER_STOPPED_TD(td)) 291297466Sjhb return (0); 29295322Shsu 293136445Sjhb sleepq_lock(cvp); 29495322Shsu 295293458Smarkj CV_WAITERS_INC(cvp); 296183352Sjhb if (lock == &Giant.lock_object) 297183352Sjhb mtx_assert(&Giant, MA_OWNED); 29888900Sjhb DROP_GIANT(); 29971088Sjasone 300167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 301247785Sdavide sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 302181394Sjhb if (lock != &Giant.lock_object) { 303277528Shselasky if (class->lc_flags & LC_SLEEPABLE) 304277528Shselasky sleepq_release(cvp); 305183352Sjhb WITNESS_SAVE(lock, lock_witness); 306181394Sjhb lock_state = class->lc_unlock(lock); 307277528Shselasky if (class->lc_flags & LC_SLEEPABLE) 308277528Shselasky sleepq_lock(cvp); 309181394Sjhb } 310177085Sjeff rval = sleepq_timedwait(cvp, 0); 31171088Sjasone 31271088Sjasone#ifdef KTRACE 31397995Sjhb if (KTRPOINT(td, KTR_CSW)) 314234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 31571088Sjasone#endif 31671088Sjasone PICKUP_GIANT(); 317181394Sjhb if (lock != &Giant.lock_object) { 318181394Sjhb class->lc_lock(lock, lock_state); 319181394Sjhb WITNESS_RESTORE(lock, lock_witness); 320181394Sjhb } 32171088Sjasone 32271088Sjasone return (rval); 32371088Sjasone} 32471088Sjasone 32571088Sjasone/* 326247785Sdavide * Wait on a condition variable for (at most) the value specified in sbt 327247785Sdavide * argument, allowing interruption by signals. 328247785Sdavide * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 329247785Sdavide * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 330247785Sdavide * was caught. 33171088Sjasone */ 33271088Sjasoneint 333247785Sdavide_cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 334247785Sdavide sbintime_t sbt, sbintime_t pr, int flags) 33571088Sjasone{ 336167789Sjhb WITNESS_SAVE_DECL(lock_witness); 337167789Sjhb struct lock_class *class; 33883366Sjulian struct thread *td; 339167789Sjhb int lock_state, rval; 34071088Sjasone 34183366Sjulian td = curthread; 342181394Sjhb lock_state = 0; 34371088Sjasone#ifdef KTRACE 34497995Sjhb if (KTRPOINT(td, KTR_CSW)) 345234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 34671088Sjasone#endif 347167789Sjhb CV_ASSERT(cvp, lock, td); 348167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 349111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 350167789Sjhb class = LOCK_CLASS(lock); 35171088Sjasone 352315386Smjg if (SCHEDULER_STOPPED_TD(td)) 353297466Sjhb return (0); 35495322Shsu 355136445Sjhb sleepq_lock(cvp); 35695322Shsu 357293458Smarkj CV_WAITERS_INC(cvp); 358183352Sjhb if (lock == &Giant.lock_object) 359183352Sjhb mtx_assert(&Giant, MA_OWNED); 36088900Sjhb DROP_GIANT(); 36171088Sjasone 362167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 363165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 364247785Sdavide sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 365181394Sjhb if (lock != &Giant.lock_object) { 366277528Shselasky if (class->lc_flags & LC_SLEEPABLE) 367277528Shselasky sleepq_release(cvp); 368183352Sjhb WITNESS_SAVE(lock, lock_witness); 369181394Sjhb lock_state = class->lc_unlock(lock); 370277528Shselasky if (class->lc_flags & LC_SLEEPABLE) 371277528Shselasky sleepq_lock(cvp); 372181394Sjhb } 373177085Sjeff rval = sleepq_timedwait_sig(cvp, 0); 37471088Sjasone 37571088Sjasone#ifdef KTRACE 37697995Sjhb if (KTRPOINT(td, KTR_CSW)) 377234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 37871088Sjasone#endif 37997995Sjhb PICKUP_GIANT(); 380181394Sjhb if (lock != &Giant.lock_object) { 381181394Sjhb class->lc_lock(lock, lock_state); 382181394Sjhb WITNESS_RESTORE(lock, lock_witness); 383181394Sjhb } 38471088Sjasone 38571088Sjasone return (rval); 38671088Sjasone} 38771088Sjasone 38871088Sjasone/* 38983366Sjulian * Signal a condition variable, wakes up one waiting thread. Will also wakeup 39071088Sjasone * the swapper if the process is not in memory, so that it can bring the 39183366Sjulian * sleeping process in. Note that this may also result in additional threads 39271088Sjasone * being made runnable. Should be called with the same mutex as was passed to 39371088Sjasone * cv_wait held. 39471088Sjasone */ 39571088Sjasonevoid 39671088Sjasonecv_signal(struct cv *cvp) 39771088Sjasone{ 398181334Sjhb int wakeup_swapper; 39971088Sjasone 400310962Smjg if (cvp->cv_waiters == 0) 401310962Smjg return; 402181334Sjhb wakeup_swapper = 0; 403136445Sjhb sleepq_lock(cvp); 404283250Sjhb if (cvp->cv_waiters > 0) { 405293458Smarkj if (cvp->cv_waiters == CV_WAITERS_BOUND && 406293458Smarkj sleepq_lookup(cvp) == NULL) { 407293458Smarkj cvp->cv_waiters = 0; 408293458Smarkj } else { 409293458Smarkj if (cvp->cv_waiters < CV_WAITERS_BOUND) 410293458Smarkj cvp->cv_waiters--; 411293458Smarkj wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 412293458Smarkj 0); 413293458Smarkj } 414283250Sjhb } 415170294Sjeff sleepq_release(cvp); 416181334Sjhb if (wakeup_swapper) 417181334Sjhb kick_proc0(); 41871088Sjasone} 41971088Sjasone 42071088Sjasone/* 42183366Sjulian * Broadcast a signal to a condition variable. Wakes up all waiting threads. 42271088Sjasone * Should be called with the same mutex as was passed to cv_wait held. 42371088Sjasone */ 42471088Sjasonevoid 425122352Stanimuracv_broadcastpri(struct cv *cvp, int pri) 42671088Sjasone{ 427181334Sjhb int wakeup_swapper; 428181334Sjhb 429310962Smjg if (cvp->cv_waiters == 0) 430310962Smjg return; 431177085Sjeff /* 432177085Sjeff * XXX sleepq_broadcast pri argument changed from -1 meaning 433177085Sjeff * no pri to 0 meaning no pri. 434177085Sjeff */ 435181334Sjhb wakeup_swapper = 0; 436177085Sjeff if (pri == -1) 437177085Sjeff pri = 0; 438136445Sjhb sleepq_lock(cvp); 439283250Sjhb if (cvp->cv_waiters > 0) { 440283250Sjhb cvp->cv_waiters = 0; 441181334Sjhb wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 442283250Sjhb } 443177085Sjeff sleepq_release(cvp); 444181334Sjhb if (wakeup_swapper) 445181334Sjhb kick_proc0(); 44671088Sjasone} 447