kern_condvar.c revision 170294
171088Sjasone/*- 271088Sjasone * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 371088Sjasone * All rights reserved. 471088Sjasone * 571088Sjasone * Redistribution and use in source and binary forms, with or without 671088Sjasone * modification, are permitted provided that the following conditions 771088Sjasone * are met: 871088Sjasone * 1. Redistributions of source code must retain the above copyright 971088Sjasone * notice, this list of conditions and the following disclaimer. 1071088Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1171088Sjasone * notice, this list of conditions and the following disclaimer in the 1271088Sjasone * documentation and/or other materials provided with the distribution. 1371088Sjasone * 1471088Sjasone * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1571088Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1671088Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1771088Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1871088Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1971088Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2071088Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2171088Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2271088Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2371088Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2471088Sjasone * SUCH DAMAGE. 2571088Sjasone */ 2671088Sjasone 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_condvar.c 170294 2007-06-04 23:50:56Z jeff $"); 29116182Sobrien 3071088Sjasone#include "opt_ktrace.h" 3171088Sjasone 3271088Sjasone#include <sys/param.h> 3371088Sjasone#include <sys/systm.h> 3476166Smarkm#include <sys/lock.h> 3576166Smarkm#include <sys/mutex.h> 3671088Sjasone#include <sys/proc.h> 3771088Sjasone#include <sys/kernel.h> 3871088Sjasone#include <sys/ktr.h> 3971088Sjasone#include <sys/condvar.h> 40109862Sjeff#include <sys/sched.h> 4171088Sjasone#include <sys/signalvar.h> 42126326Sjhb#include <sys/sleepqueue.h> 4371088Sjasone#include <sys/resourcevar.h> 4471088Sjasone#ifdef KTRACE 4571088Sjasone#include <sys/uio.h> 4671088Sjasone#include <sys/ktrace.h> 4771088Sjasone#endif 4871088Sjasone 4971088Sjasone/* 5071088Sjasone * Common sanity checks for cv_wait* functions. 5171088Sjasone */ 52167789Sjhb#define CV_ASSERT(cvp, lock, td) do { \ 5387594Sobrien KASSERT((td) != NULL, ("%s: curthread NULL", __func__)); \ 54103216Sjulian KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 5587594Sobrien KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 56167789Sjhb KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 5771088Sjasone} while (0) 5871088Sjasone 5971088Sjasone/* 6071088Sjasone * Initialize a condition variable. Must be called before use. 6171088Sjasone */ 6271088Sjasonevoid 6371088Sjasonecv_init(struct cv *cvp, const char *desc) 6471088Sjasone{ 6571088Sjasone 6671088Sjasone cvp->cv_description = desc; 67127954Sjhb cvp->cv_waiters = 0; 6871088Sjasone} 6971088Sjasone 7071088Sjasone/* 7171088Sjasone * Destroy a condition variable. The condition variable must be re-initialized 7271088Sjasone * in order to be re-used. 7371088Sjasone */ 7471088Sjasonevoid 7571088Sjasonecv_destroy(struct cv *cvp) 7671088Sjasone{ 77126326Sjhb#ifdef INVARIANTS 78136445Sjhb struct sleepqueue *sq; 7971088Sjasone 80136445Sjhb sleepq_lock(cvp); 81126326Sjhb sq = sleepq_lookup(cvp); 82126326Sjhb sleepq_release(cvp); 83126326Sjhb KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 84126326Sjhb#endif 8571088Sjasone} 8671088Sjasone 8771088Sjasone/* 8883366Sjulian * Wait on a condition variable. The current thread is placed on the condition 8971088Sjasone * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 9083366Sjulian * condition variable will resume the thread. The mutex is released before 9171088Sjasone * sleeping and will be held on return. It is recommended that the mutex be 9271088Sjasone * held when cv_signal or cv_broadcast are called. 9371088Sjasone */ 9471088Sjasonevoid 95167789Sjhb_cv_wait(struct cv *cvp, struct lock_object *lock) 9671088Sjasone{ 97167789Sjhb WITNESS_SAVE_DECL(lock_witness); 98167789Sjhb struct lock_class *class; 99167786Sjhb struct thread *td; 100167789Sjhb int lock_state; 10171088Sjasone 102167786Sjhb td = curthread; 103167786Sjhb#ifdef KTRACE 104167786Sjhb if (KTRPOINT(td, KTR_CSW)) 105167786Sjhb ktrcsw(1, 0); 106167786Sjhb#endif 107167789Sjhb CV_ASSERT(cvp, lock, td); 108167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 109167786Sjhb "Waiting on \"%s\"", cvp->cv_description); 110167789Sjhb WITNESS_SAVE(lock, lock_witness); 111167789Sjhb class = LOCK_CLASS(lock); 112153321Srodrigc 113153321Srodrigc if (cold || panicstr) { 114153321Srodrigc /* 115153321Srodrigc * During autoconfiguration, just give interrupts 116153321Srodrigc * a chance, then just return. Don't run any other 117153321Srodrigc * thread or panic below, in case this is the idle 118153321Srodrigc * process and already asleep. 119153321Srodrigc */ 120153321Srodrigc return; 121153321Srodrigc } 122153321Srodrigc 123167786Sjhb sleepq_lock(cvp); 124167786Sjhb 125167786Sjhb cvp->cv_waiters++; 126167786Sjhb DROP_GIANT(); 127167786Sjhb 128167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 129169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 130169392Sjhb sleepq_release(cvp); 131169392Sjhb lock_state = class->lc_unlock(lock); 132169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 133169392Sjhb sleepq_lock(cvp); 134167786Sjhb sleepq_wait(cvp); 135167786Sjhb 136167786Sjhb#ifdef KTRACE 137167786Sjhb if (KTRPOINT(td, KTR_CSW)) 138167786Sjhb ktrcsw(0, 0); 139167786Sjhb#endif 140167786Sjhb PICKUP_GIANT(); 141167789Sjhb class->lc_lock(lock, lock_state); 142167789Sjhb WITNESS_RESTORE(lock, lock_witness); 143153321Srodrigc} 144153321Srodrigc 145153321Srodrigc/* 146153321Srodrigc * Wait on a condition variable. This function differs from cv_wait by 147153321Srodrigc * not aquiring the mutex after condition variable was signaled. 148153321Srodrigc */ 149153321Srodrigcvoid 150167789Sjhb_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 151153321Srodrigc{ 152167789Sjhb struct lock_class *class; 153153321Srodrigc struct thread *td; 154153321Srodrigc 15583366Sjulian td = curthread; 15671088Sjasone#ifdef KTRACE 15797995Sjhb if (KTRPOINT(td, KTR_CSW)) 15897995Sjhb ktrcsw(1, 0); 15971088Sjasone#endif 160167789Sjhb CV_ASSERT(cvp, lock, td); 161167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 162111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 163167789Sjhb class = LOCK_CLASS(lock); 16471088Sjasone 165126326Sjhb if (cold || panicstr) { 16671088Sjasone /* 167100209Sgallatin * During autoconfiguration, just give interrupts 168100209Sgallatin * a chance, then just return. Don't run any other 169100209Sgallatin * thread or panic below, in case this is the idle 170100209Sgallatin * process and already asleep. 17171088Sjasone */ 172167789Sjhb class->lc_unlock(lock); 17371088Sjasone return; 17471088Sjasone } 17595322Shsu 176136445Sjhb sleepq_lock(cvp); 17795322Shsu 178127954Sjhb cvp->cv_waiters++; 17988900Sjhb DROP_GIANT(); 18071088Sjasone 181167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 182169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 183169392Sjhb sleepq_release(cvp); 184169392Sjhb class->lc_unlock(lock); 185169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 186169392Sjhb sleepq_lock(cvp); 187126326Sjhb sleepq_wait(cvp); 18871088Sjasone 189167786Sjhb#ifdef KTRACE 190167786Sjhb if (KTRPOINT(td, KTR_CSW)) 191167786Sjhb ktrcsw(0, 0); 192167786Sjhb#endif 19371088Sjasone PICKUP_GIANT(); 19471088Sjasone} 19571088Sjasone 19671088Sjasone/* 19771088Sjasone * Wait on a condition variable, allowing interruption by signals. Return 0 if 19883366Sjulian * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 19971088Sjasone * a signal was caught. If ERESTART is returned the system call should be 20071088Sjasone * restarted if possible. 20171088Sjasone */ 20271088Sjasoneint 203167789Sjhb_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 20471088Sjasone{ 205167789Sjhb WITNESS_SAVE_DECL(lock_witness); 206167789Sjhb struct lock_class *class; 20783366Sjulian struct thread *td; 20883658Speter struct proc *p; 209167789Sjhb int lock_state, rval; 21071088Sjasone 21183366Sjulian td = curthread; 21283650Sjhb p = td->td_proc; 21371088Sjasone#ifdef KTRACE 21497995Sjhb if (KTRPOINT(td, KTR_CSW)) 21597995Sjhb ktrcsw(1, 0); 21671088Sjasone#endif 217167789Sjhb CV_ASSERT(cvp, lock, td); 218167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 219111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 220167789Sjhb WITNESS_SAVE(lock, lock_witness); 221167789Sjhb class = LOCK_CLASS(lock); 22271088Sjasone 22371088Sjasone if (cold || panicstr) { 22471088Sjasone /* 22571088Sjasone * After a panic, or during autoconfiguration, just give 22671088Sjasone * interrupts a chance, then just return; don't run any other 22771088Sjasone * procs or panic below, in case this is the idle process and 22871088Sjasone * already asleep. 22971088Sjasone */ 230133440Sjhb return (0); 23171088Sjasone } 23295322Shsu 233136445Sjhb sleepq_lock(cvp); 23495322Shsu 235127954Sjhb cvp->cv_waiters++; 23688900Sjhb DROP_GIANT(); 23771088Sjasone 238167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 239165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 240169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 241169392Sjhb sleepq_release(cvp); 242169392Sjhb lock_state = class->lc_unlock(lock); 243169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 244169392Sjhb sleepq_lock(cvp); 245134013Sjhb rval = sleepq_wait_sig(cvp); 24671088Sjasone 24771088Sjasone#ifdef KTRACE 24897995Sjhb if (KTRPOINT(td, KTR_CSW)) 24997995Sjhb ktrcsw(0, 0); 25071088Sjasone#endif 25197995Sjhb PICKUP_GIANT(); 252167789Sjhb class->lc_lock(lock, lock_state); 253167789Sjhb WITNESS_RESTORE(lock, lock_witness); 25471088Sjasone 25571088Sjasone return (rval); 25671088Sjasone} 25771088Sjasone 25871088Sjasone/* 25971088Sjasone * Wait on a condition variable for at most timo/hz seconds. Returns 0 if the 26071088Sjasone * process was resumed by cv_signal or cv_broadcast, EWOULDBLOCK if the timeout 26171088Sjasone * expires. 26271088Sjasone */ 26371088Sjasoneint 264167789Sjhb_cv_timedwait(struct cv *cvp, struct lock_object *lock, int timo) 26571088Sjasone{ 266167789Sjhb WITNESS_SAVE_DECL(lock_witness); 267167789Sjhb struct lock_class *class; 26883366Sjulian struct thread *td; 269167789Sjhb int lock_state, rval; 27071088Sjasone 27183366Sjulian td = curthread; 27271088Sjasone rval = 0; 27371088Sjasone#ifdef KTRACE 27497995Sjhb if (KTRPOINT(td, KTR_CSW)) 27597995Sjhb ktrcsw(1, 0); 27671088Sjasone#endif 277167789Sjhb CV_ASSERT(cvp, lock, td); 278167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 279111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 280167789Sjhb WITNESS_SAVE(lock, lock_witness); 281167789Sjhb class = LOCK_CLASS(lock); 28271088Sjasone 28371088Sjasone if (cold || panicstr) { 28471088Sjasone /* 28571088Sjasone * After a panic, or during autoconfiguration, just give 28671088Sjasone * interrupts a chance, then just return; don't run any other 28783366Sjulian * thread or panic below, in case this is the idle process and 28871088Sjasone * already asleep. 28971088Sjasone */ 29071088Sjasone return 0; 29171088Sjasone } 29295322Shsu 293136445Sjhb sleepq_lock(cvp); 29495322Shsu 295127954Sjhb cvp->cv_waiters++; 29688900Sjhb DROP_GIANT(); 29771088Sjasone 298167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 299126885Sjhb sleepq_set_timeout(cvp, timo); 300169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 301169392Sjhb sleepq_release(cvp); 302169392Sjhb lock_state = class->lc_unlock(lock); 303169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 304169392Sjhb sleepq_lock(cvp); 305131249Sjhb rval = sleepq_timedwait(cvp); 30671088Sjasone 30771088Sjasone#ifdef KTRACE 30897995Sjhb if (KTRPOINT(td, KTR_CSW)) 30997995Sjhb ktrcsw(0, 0); 31071088Sjasone#endif 31171088Sjasone PICKUP_GIANT(); 312167789Sjhb class->lc_lock(lock, lock_state); 313167789Sjhb WITNESS_RESTORE(lock, lock_witness); 31471088Sjasone 31571088Sjasone return (rval); 31671088Sjasone} 31771088Sjasone 31871088Sjasone/* 31971088Sjasone * Wait on a condition variable for at most timo/hz seconds, allowing 32083366Sjulian * interruption by signals. Returns 0 if the thread was resumed by cv_signal 32171088Sjasone * or cv_broadcast, EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if 32271088Sjasone * a signal was caught. 32371088Sjasone */ 32471088Sjasoneint 325167789Sjhb_cv_timedwait_sig(struct cv *cvp, struct lock_object *lock, int timo) 32671088Sjasone{ 327167789Sjhb WITNESS_SAVE_DECL(lock_witness); 328167789Sjhb struct lock_class *class; 32983366Sjulian struct thread *td; 33083650Sjhb struct proc *p; 331167789Sjhb int lock_state, rval; 33271088Sjasone 33383366Sjulian td = curthread; 33483650Sjhb p = td->td_proc; 33571088Sjasone rval = 0; 33671088Sjasone#ifdef KTRACE 33797995Sjhb if (KTRPOINT(td, KTR_CSW)) 33897995Sjhb ktrcsw(1, 0); 33971088Sjasone#endif 340167789Sjhb CV_ASSERT(cvp, lock, td); 341167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 342111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 343167789Sjhb WITNESS_SAVE(lock, lock_witness); 344167789Sjhb class = LOCK_CLASS(lock); 34571088Sjasone 34671088Sjasone if (cold || panicstr) { 34771088Sjasone /* 34871088Sjasone * After a panic, or during autoconfiguration, just give 34971088Sjasone * interrupts a chance, then just return; don't run any other 35083366Sjulian * thread or panic below, in case this is the idle process and 35171088Sjasone * already asleep. 35271088Sjasone */ 35371088Sjasone return 0; 35471088Sjasone } 35595322Shsu 356136445Sjhb sleepq_lock(cvp); 35795322Shsu 358127954Sjhb cvp->cv_waiters++; 35988900Sjhb DROP_GIANT(); 36071088Sjasone 361167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 362165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 363126885Sjhb sleepq_set_timeout(cvp, timo); 364169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 365169392Sjhb sleepq_release(cvp); 366169392Sjhb lock_state = class->lc_unlock(lock); 367169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 368169392Sjhb sleepq_lock(cvp); 369155741Sdavidxu rval = sleepq_timedwait_sig(cvp); 37071088Sjasone 37171088Sjasone#ifdef KTRACE 37297995Sjhb if (KTRPOINT(td, KTR_CSW)) 37397995Sjhb ktrcsw(0, 0); 37471088Sjasone#endif 37597995Sjhb PICKUP_GIANT(); 376167789Sjhb class->lc_lock(lock, lock_state); 377167789Sjhb WITNESS_RESTORE(lock, lock_witness); 37871088Sjasone 37971088Sjasone return (rval); 38071088Sjasone} 38171088Sjasone 38271088Sjasone/* 38383366Sjulian * Signal a condition variable, wakes up one waiting thread. Will also wakeup 38471088Sjasone * the swapper if the process is not in memory, so that it can bring the 38583366Sjulian * sleeping process in. Note that this may also result in additional threads 38671088Sjasone * being made runnable. Should be called with the same mutex as was passed to 38771088Sjasone * cv_wait held. 38871088Sjasone */ 38971088Sjasonevoid 39071088Sjasonecv_signal(struct cv *cvp) 39171088Sjasone{ 39271088Sjasone 393136445Sjhb sleepq_lock(cvp); 394127954Sjhb if (cvp->cv_waiters > 0) { 395127954Sjhb cvp->cv_waiters--; 396165272Skmacy sleepq_signal(cvp, SLEEPQ_CONDVAR, -1, 0); 397170294Sjeff } 398170294Sjeff sleepq_release(cvp); 39971088Sjasone} 40071088Sjasone 40171088Sjasone/* 40283366Sjulian * Broadcast a signal to a condition variable. Wakes up all waiting threads. 40371088Sjasone * Should be called with the same mutex as was passed to cv_wait held. 40471088Sjasone */ 40571088Sjasonevoid 406122352Stanimuracv_broadcastpri(struct cv *cvp, int pri) 40771088Sjasone{ 40871088Sjasone 409136445Sjhb sleepq_lock(cvp); 410127954Sjhb if (cvp->cv_waiters > 0) { 411127954Sjhb cvp->cv_waiters = 0; 412165272Skmacy sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 413136445Sjhb } else 414136445Sjhb sleepq_release(cvp); 41571088Sjasone} 416