171088Sjasone/*- 271088Sjasone * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 371088Sjasone * All rights reserved. 471088Sjasone * 571088Sjasone * Redistribution and use in source and binary forms, with or without 671088Sjasone * modification, are permitted provided that the following conditions 771088Sjasone * are met: 871088Sjasone * 1. Redistributions of source code must retain the above copyright 971088Sjasone * notice, this list of conditions and the following disclaimer. 1071088Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1171088Sjasone * notice, this list of conditions and the following disclaimer in the 1271088Sjasone * documentation and/or other materials provided with the distribution. 1371088Sjasone * 1471088Sjasone * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1571088Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1671088Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1771088Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1871088Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1971088Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2071088Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2171088Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2271088Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2371088Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2471088Sjasone * SUCH DAMAGE. 2571088Sjasone */ 2671088Sjasone 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD$"); 29116182Sobrien 3071088Sjasone#include "opt_ktrace.h" 3171088Sjasone 3271088Sjasone#include <sys/param.h> 3371088Sjasone#include <sys/systm.h> 3476166Smarkm#include <sys/lock.h> 3576166Smarkm#include <sys/mutex.h> 3671088Sjasone#include <sys/proc.h> 3771088Sjasone#include <sys/kernel.h> 3871088Sjasone#include <sys/ktr.h> 3971088Sjasone#include <sys/condvar.h> 40109862Sjeff#include <sys/sched.h> 4171088Sjasone#include <sys/signalvar.h> 42126326Sjhb#include <sys/sleepqueue.h> 4371088Sjasone#include <sys/resourcevar.h> 4471088Sjasone#ifdef KTRACE 4571088Sjasone#include <sys/uio.h> 4671088Sjasone#include <sys/ktrace.h> 4771088Sjasone#endif 4871088Sjasone 4971088Sjasone/* 5071088Sjasone * Common sanity checks for cv_wait* functions. 5171088Sjasone */ 52167789Sjhb#define CV_ASSERT(cvp, lock, td) do { \ 53250581Shiren KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 54103216Sjulian KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 5587594Sobrien KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 56167789Sjhb KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 5771088Sjasone} while (0) 5871088Sjasone 5971088Sjasone/* 6071088Sjasone * Initialize a condition variable. Must be called before use. 6171088Sjasone */ 6271088Sjasonevoid 6371088Sjasonecv_init(struct cv *cvp, const char *desc) 6471088Sjasone{ 6571088Sjasone 6671088Sjasone cvp->cv_description = desc; 67127954Sjhb cvp->cv_waiters = 0; 6871088Sjasone} 6971088Sjasone 7071088Sjasone/* 7171088Sjasone * Destroy a condition variable. The condition variable must be re-initialized 7271088Sjasone * in order to be re-used. 7371088Sjasone */ 7471088Sjasonevoid 7571088Sjasonecv_destroy(struct cv *cvp) 7671088Sjasone{ 77126326Sjhb#ifdef INVARIANTS 78136445Sjhb struct sleepqueue *sq; 7971088Sjasone 80136445Sjhb sleepq_lock(cvp); 81126326Sjhb sq = sleepq_lookup(cvp); 82126326Sjhb sleepq_release(cvp); 83126326Sjhb KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 84126326Sjhb#endif 8571088Sjasone} 8671088Sjasone 8771088Sjasone/* 8883366Sjulian * Wait on a condition variable. The current thread is placed on the condition 8971088Sjasone * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 9083366Sjulian * condition variable will resume the thread. The mutex is released before 9171088Sjasone * sleeping and will be held on return. It is recommended that the mutex be 9271088Sjasone * held when cv_signal or cv_broadcast are called. 9371088Sjasone */ 9471088Sjasonevoid 95167789Sjhb_cv_wait(struct cv *cvp, struct lock_object *lock) 9671088Sjasone{ 97167789Sjhb WITNESS_SAVE_DECL(lock_witness); 98167789Sjhb struct lock_class *class; 99167786Sjhb struct thread *td; 100167789Sjhb int lock_state; 10171088Sjasone 102167786Sjhb td = curthread; 103181394Sjhb lock_state = 0; 104167786Sjhb#ifdef KTRACE 105167786Sjhb if (KTRPOINT(td, KTR_CSW)) 106237719Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 107167786Sjhb#endif 108167789Sjhb CV_ASSERT(cvp, lock, td); 109167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 110167786Sjhb "Waiting on \"%s\"", cvp->cv_description); 111167789Sjhb class = LOCK_CLASS(lock); 112153321Srodrigc 113153321Srodrigc if (cold || panicstr) { 114153321Srodrigc /* 115153321Srodrigc * During autoconfiguration, just give interrupts 116153321Srodrigc * a chance, then just return. Don't run any other 117153321Srodrigc * thread or panic below, in case this is the idle 118153321Srodrigc * process and already asleep. 119153321Srodrigc */ 120153321Srodrigc return; 121153321Srodrigc } 122153321Srodrigc 123167786Sjhb sleepq_lock(cvp); 124167786Sjhb 125167786Sjhb cvp->cv_waiters++; 126183352Sjhb if (lock == &Giant.lock_object) 127183352Sjhb mtx_assert(&Giant, MA_OWNED); 128167786Sjhb DROP_GIANT(); 129167786Sjhb 130167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 131181394Sjhb if (lock != &Giant.lock_object) { 132181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 133181394Sjhb sleepq_release(cvp); 134183352Sjhb WITNESS_SAVE(lock, lock_witness); 135181394Sjhb lock_state = class->lc_unlock(lock); 136181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 137181394Sjhb sleepq_lock(cvp); 138181394Sjhb } 139177085Sjeff sleepq_wait(cvp, 0); 140167786Sjhb 141167786Sjhb#ifdef KTRACE 142167786Sjhb if (KTRPOINT(td, KTR_CSW)) 143237719Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 144167786Sjhb#endif 145167786Sjhb PICKUP_GIANT(); 146181394Sjhb if (lock != &Giant.lock_object) { 147181394Sjhb class->lc_lock(lock, lock_state); 148181394Sjhb WITNESS_RESTORE(lock, lock_witness); 149181394Sjhb } 150153321Srodrigc} 151153321Srodrigc 152153321Srodrigc/* 153153321Srodrigc * Wait on a condition variable. This function differs from cv_wait by 154153321Srodrigc * not aquiring the mutex after condition variable was signaled. 155153321Srodrigc */ 156153321Srodrigcvoid 157167789Sjhb_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 158153321Srodrigc{ 159167789Sjhb struct lock_class *class; 160153321Srodrigc struct thread *td; 161153321Srodrigc 16283366Sjulian td = curthread; 16371088Sjasone#ifdef KTRACE 16497995Sjhb if (KTRPOINT(td, KTR_CSW)) 165237719Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 16671088Sjasone#endif 167167789Sjhb CV_ASSERT(cvp, lock, td); 168167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 169111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 170181394Sjhb KASSERT(lock != &Giant.lock_object, 171181394Sjhb ("cv_wait_unlock cannot be used with Giant")); 172167789Sjhb class = LOCK_CLASS(lock); 17371088Sjasone 174126326Sjhb if (cold || panicstr) { 17571088Sjasone /* 176100209Sgallatin * During autoconfiguration, just give interrupts 177100209Sgallatin * a chance, then just return. Don't run any other 178100209Sgallatin * thread or panic below, in case this is the idle 179100209Sgallatin * process and already asleep. 18071088Sjasone */ 181167789Sjhb class->lc_unlock(lock); 18271088Sjasone return; 18371088Sjasone } 18495322Shsu 185136445Sjhb sleepq_lock(cvp); 18695322Shsu 187127954Sjhb cvp->cv_waiters++; 18888900Sjhb DROP_GIANT(); 18971088Sjasone 190167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 191169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 192169392Sjhb sleepq_release(cvp); 193169392Sjhb class->lc_unlock(lock); 194169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 195169392Sjhb sleepq_lock(cvp); 196177085Sjeff sleepq_wait(cvp, 0); 19771088Sjasone 198167786Sjhb#ifdef KTRACE 199167786Sjhb if (KTRPOINT(td, KTR_CSW)) 200237719Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 201167786Sjhb#endif 20271088Sjasone PICKUP_GIANT(); 20371088Sjasone} 20471088Sjasone 20571088Sjasone/* 20671088Sjasone * Wait on a condition variable, allowing interruption by signals. Return 0 if 20783366Sjulian * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 20871088Sjasone * a signal was caught. If ERESTART is returned the system call should be 20971088Sjasone * restarted if possible. 21071088Sjasone */ 21171088Sjasoneint 212167789Sjhb_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 21371088Sjasone{ 214167789Sjhb WITNESS_SAVE_DECL(lock_witness); 215167789Sjhb struct lock_class *class; 21683366Sjulian struct thread *td; 217167789Sjhb int lock_state, rval; 21871088Sjasone 21983366Sjulian td = curthread; 220181394Sjhb lock_state = 0; 22171088Sjasone#ifdef KTRACE 22297995Sjhb if (KTRPOINT(td, KTR_CSW)) 223237719Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 22471088Sjasone#endif 225167789Sjhb CV_ASSERT(cvp, lock, td); 226167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 227111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 228167789Sjhb class = LOCK_CLASS(lock); 22971088Sjasone 23071088Sjasone if (cold || panicstr) { 23171088Sjasone /* 23271088Sjasone * After a panic, or during autoconfiguration, just give 23371088Sjasone * interrupts a chance, then just return; don't run any other 23471088Sjasone * procs or panic below, in case this is the idle process and 23571088Sjasone * already asleep. 23671088Sjasone */ 237133440Sjhb return (0); 23871088Sjasone } 23995322Shsu 240136445Sjhb sleepq_lock(cvp); 24195322Shsu 242127954Sjhb cvp->cv_waiters++; 243183352Sjhb if (lock == &Giant.lock_object) 244183352Sjhb mtx_assert(&Giant, MA_OWNED); 24588900Sjhb DROP_GIANT(); 24671088Sjasone 247167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 248165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 249181394Sjhb if (lock != &Giant.lock_object) { 250181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 251181394Sjhb sleepq_release(cvp); 252183352Sjhb WITNESS_SAVE(lock, lock_witness); 253181394Sjhb lock_state = class->lc_unlock(lock); 254181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 255181394Sjhb sleepq_lock(cvp); 256181394Sjhb } 257177085Sjeff rval = sleepq_wait_sig(cvp, 0); 25871088Sjasone 25971088Sjasone#ifdef KTRACE 26097995Sjhb if (KTRPOINT(td, KTR_CSW)) 261237719Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 26271088Sjasone#endif 26397995Sjhb PICKUP_GIANT(); 264181394Sjhb if (lock != &Giant.lock_object) { 265181394Sjhb class->lc_lock(lock, lock_state); 266181394Sjhb WITNESS_RESTORE(lock, lock_witness); 267181394Sjhb } 26871088Sjasone 26971088Sjasone return (rval); 27071088Sjasone} 27171088Sjasone 27271088Sjasone/* 27371088Sjasone * Wait on a condition variable for at most timo/hz seconds. Returns 0 if the 27471088Sjasone * process was resumed by cv_signal or cv_broadcast, EWOULDBLOCK if the timeout 27571088Sjasone * expires. 27671088Sjasone */ 27771088Sjasoneint 278167789Sjhb_cv_timedwait(struct cv *cvp, struct lock_object *lock, int timo) 27971088Sjasone{ 280167789Sjhb WITNESS_SAVE_DECL(lock_witness); 281167789Sjhb struct lock_class *class; 28283366Sjulian struct thread *td; 283167789Sjhb int lock_state, rval; 28471088Sjasone 28583366Sjulian td = curthread; 286181394Sjhb lock_state = 0; 28771088Sjasone#ifdef KTRACE 28897995Sjhb if (KTRPOINT(td, KTR_CSW)) 289237719Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 29071088Sjasone#endif 291167789Sjhb CV_ASSERT(cvp, lock, td); 292167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 293111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 294167789Sjhb class = LOCK_CLASS(lock); 29571088Sjasone 29671088Sjasone if (cold || panicstr) { 29771088Sjasone /* 29871088Sjasone * After a panic, or during autoconfiguration, just give 29971088Sjasone * interrupts a chance, then just return; don't run any other 30083366Sjulian * thread or panic below, in case this is the idle process and 30171088Sjasone * already asleep. 30271088Sjasone */ 30371088Sjasone return 0; 30471088Sjasone } 30595322Shsu 306136445Sjhb sleepq_lock(cvp); 30795322Shsu 308127954Sjhb cvp->cv_waiters++; 309183352Sjhb if (lock == &Giant.lock_object) 310183352Sjhb mtx_assert(&Giant, MA_OWNED); 31188900Sjhb DROP_GIANT(); 31271088Sjasone 313167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 314126885Sjhb sleepq_set_timeout(cvp, timo); 315181394Sjhb if (lock != &Giant.lock_object) { 316181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 317181394Sjhb sleepq_release(cvp); 318183352Sjhb WITNESS_SAVE(lock, lock_witness); 319181394Sjhb lock_state = class->lc_unlock(lock); 320181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 321181394Sjhb sleepq_lock(cvp); 322181394Sjhb } 323177085Sjeff rval = sleepq_timedwait(cvp, 0); 32471088Sjasone 32571088Sjasone#ifdef KTRACE 32697995Sjhb if (KTRPOINT(td, KTR_CSW)) 327237719Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 32871088Sjasone#endif 32971088Sjasone PICKUP_GIANT(); 330181394Sjhb if (lock != &Giant.lock_object) { 331181394Sjhb class->lc_lock(lock, lock_state); 332181394Sjhb WITNESS_RESTORE(lock, lock_witness); 333181394Sjhb } 33471088Sjasone 33571088Sjasone return (rval); 33671088Sjasone} 33771088Sjasone 33871088Sjasone/* 33971088Sjasone * Wait on a condition variable for at most timo/hz seconds, allowing 34083366Sjulian * interruption by signals. Returns 0 if the thread was resumed by cv_signal 34171088Sjasone * or cv_broadcast, EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if 34271088Sjasone * a signal was caught. 34371088Sjasone */ 34471088Sjasoneint 345167789Sjhb_cv_timedwait_sig(struct cv *cvp, struct lock_object *lock, int timo) 34671088Sjasone{ 347167789Sjhb WITNESS_SAVE_DECL(lock_witness); 348167789Sjhb struct lock_class *class; 34983366Sjulian struct thread *td; 350167789Sjhb int lock_state, rval; 35171088Sjasone 35283366Sjulian td = curthread; 353181394Sjhb lock_state = 0; 35471088Sjasone#ifdef KTRACE 35597995Sjhb if (KTRPOINT(td, KTR_CSW)) 356237719Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 35771088Sjasone#endif 358167789Sjhb CV_ASSERT(cvp, lock, td); 359167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 360111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 361167789Sjhb class = LOCK_CLASS(lock); 36271088Sjasone 36371088Sjasone if (cold || panicstr) { 36471088Sjasone /* 36571088Sjasone * After a panic, or during autoconfiguration, just give 36671088Sjasone * interrupts a chance, then just return; don't run any other 36783366Sjulian * thread or panic below, in case this is the idle process and 36871088Sjasone * already asleep. 36971088Sjasone */ 37071088Sjasone return 0; 37171088Sjasone } 37295322Shsu 373136445Sjhb sleepq_lock(cvp); 37495322Shsu 375127954Sjhb cvp->cv_waiters++; 376183352Sjhb if (lock == &Giant.lock_object) 377183352Sjhb mtx_assert(&Giant, MA_OWNED); 37888900Sjhb DROP_GIANT(); 37971088Sjasone 380167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 381165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 382126885Sjhb sleepq_set_timeout(cvp, timo); 383181394Sjhb if (lock != &Giant.lock_object) { 384181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 385181394Sjhb sleepq_release(cvp); 386183352Sjhb WITNESS_SAVE(lock, lock_witness); 387181394Sjhb lock_state = class->lc_unlock(lock); 388181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 389181394Sjhb sleepq_lock(cvp); 390181394Sjhb } 391177085Sjeff rval = sleepq_timedwait_sig(cvp, 0); 39271088Sjasone 39371088Sjasone#ifdef KTRACE 39497995Sjhb if (KTRPOINT(td, KTR_CSW)) 395237719Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 39671088Sjasone#endif 39797995Sjhb PICKUP_GIANT(); 398181394Sjhb if (lock != &Giant.lock_object) { 399181394Sjhb class->lc_lock(lock, lock_state); 400181394Sjhb WITNESS_RESTORE(lock, lock_witness); 401181394Sjhb } 40271088Sjasone 40371088Sjasone return (rval); 40471088Sjasone} 40571088Sjasone 40671088Sjasone/* 40783366Sjulian * Signal a condition variable, wakes up one waiting thread. Will also wakeup 40871088Sjasone * the swapper if the process is not in memory, so that it can bring the 40983366Sjulian * sleeping process in. Note that this may also result in additional threads 41071088Sjasone * being made runnable. Should be called with the same mutex as was passed to 41171088Sjasone * cv_wait held. 41271088Sjasone */ 41371088Sjasonevoid 41471088Sjasonecv_signal(struct cv *cvp) 41571088Sjasone{ 416181334Sjhb int wakeup_swapper; 41771088Sjasone 418181334Sjhb wakeup_swapper = 0; 419136445Sjhb sleepq_lock(cvp); 420127954Sjhb if (cvp->cv_waiters > 0) { 421127954Sjhb cvp->cv_waiters--; 422181334Sjhb wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0); 423170294Sjeff } 424170294Sjeff sleepq_release(cvp); 425181334Sjhb if (wakeup_swapper) 426181334Sjhb kick_proc0(); 42771088Sjasone} 42871088Sjasone 42971088Sjasone/* 43083366Sjulian * Broadcast a signal to a condition variable. Wakes up all waiting threads. 43171088Sjasone * Should be called with the same mutex as was passed to cv_wait held. 43271088Sjasone */ 43371088Sjasonevoid 434122352Stanimuracv_broadcastpri(struct cv *cvp, int pri) 43571088Sjasone{ 436181334Sjhb int wakeup_swapper; 437181334Sjhb 438177085Sjeff /* 439177085Sjeff * XXX sleepq_broadcast pri argument changed from -1 meaning 440177085Sjeff * no pri to 0 meaning no pri. 441177085Sjeff */ 442181334Sjhb wakeup_swapper = 0; 443177085Sjeff if (pri == -1) 444177085Sjeff pri = 0; 445136445Sjhb sleepq_lock(cvp); 446127954Sjhb if (cvp->cv_waiters > 0) { 447127954Sjhb cvp->cv_waiters = 0; 448181334Sjhb wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 449177085Sjeff } 450177085Sjeff sleepq_release(cvp); 451181334Sjhb if (wakeup_swapper) 452181334Sjhb kick_proc0(); 45371088Sjasone} 454