171088Sjasone/*- 271088Sjasone * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 371088Sjasone * All rights reserved. 471088Sjasone * 571088Sjasone * Redistribution and use in source and binary forms, with or without 671088Sjasone * modification, are permitted provided that the following conditions 771088Sjasone * are met: 871088Sjasone * 1. Redistributions of source code must retain the above copyright 971088Sjasone * notice, this list of conditions and the following disclaimer. 1071088Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1171088Sjasone * notice, this list of conditions and the following disclaimer in the 1271088Sjasone * documentation and/or other materials provided with the distribution. 1371088Sjasone * 1471088Sjasone * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1571088Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1671088Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1771088Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1871088Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1971088Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2071088Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2171088Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2271088Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2371088Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2471088Sjasone * SUCH DAMAGE. 2571088Sjasone */ 2671088Sjasone 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD$"); 29116182Sobrien 3071088Sjasone#include "opt_ktrace.h" 3171088Sjasone 3271088Sjasone#include <sys/param.h> 3371088Sjasone#include <sys/systm.h> 34294755Smarkj#include <sys/limits.h> 3576166Smarkm#include <sys/lock.h> 3676166Smarkm#include <sys/mutex.h> 3771088Sjasone#include <sys/proc.h> 3871088Sjasone#include <sys/kernel.h> 3971088Sjasone#include <sys/ktr.h> 4071088Sjasone#include <sys/condvar.h> 41109862Sjeff#include <sys/sched.h> 4271088Sjasone#include <sys/signalvar.h> 43126326Sjhb#include <sys/sleepqueue.h> 4471088Sjasone#include <sys/resourcevar.h> 4571088Sjasone#ifdef KTRACE 4671088Sjasone#include <sys/uio.h> 4771088Sjasone#include <sys/ktrace.h> 4871088Sjasone#endif 4971088Sjasone 5071088Sjasone/* 51294755Smarkj * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound, 52294755Smarkj * cv_signal must manually check the wait queue for threads. 53294755Smarkj */ 54294755Smarkj#define CV_WAITERS_BOUND INT_MAX 55294755Smarkj 56294755Smarkj#define CV_WAITERS_INC(cvp) do { \ 57294755Smarkj if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \ 58294755Smarkj (cvp)->cv_waiters++; \ 59294755Smarkj} while (0) 60294755Smarkj 61294755Smarkj/* 6271088Sjasone * Common sanity checks for cv_wait* functions. 6371088Sjasone */ 64167789Sjhb#define CV_ASSERT(cvp, lock, td) do { \ 65240475Sattilio KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 66103216Sjulian KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 6787594Sobrien KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 68167789Sjhb KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 6971088Sjasone} while (0) 7071088Sjasone 7171088Sjasone/* 7271088Sjasone * Initialize a condition variable. Must be called before use. 7371088Sjasone */ 7471088Sjasonevoid 7571088Sjasonecv_init(struct cv *cvp, const char *desc) 7671088Sjasone{ 7771088Sjasone 7871088Sjasone cvp->cv_description = desc; 79127954Sjhb cvp->cv_waiters = 0; 8071088Sjasone} 8171088Sjasone 8271088Sjasone/* 8371088Sjasone * Destroy a condition variable. The condition variable must be re-initialized 8471088Sjasone * in order to be re-used. 8571088Sjasone */ 8671088Sjasonevoid 8771088Sjasonecv_destroy(struct cv *cvp) 8871088Sjasone{ 89126326Sjhb#ifdef INVARIANTS 90136445Sjhb struct sleepqueue *sq; 9171088Sjasone 92136445Sjhb sleepq_lock(cvp); 93126326Sjhb sq = sleepq_lookup(cvp); 94126326Sjhb sleepq_release(cvp); 95126326Sjhb KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 96126326Sjhb#endif 9771088Sjasone} 9871088Sjasone 9971088Sjasone/* 10083366Sjulian * Wait on a condition variable. The current thread is placed on the condition 10171088Sjasone * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 10283366Sjulian * condition variable will resume the thread. The mutex is released before 10371088Sjasone * sleeping and will be held on return. It is recommended that the mutex be 10471088Sjasone * held when cv_signal or cv_broadcast are called. 10571088Sjasone */ 10671088Sjasonevoid 107167789Sjhb_cv_wait(struct cv *cvp, struct lock_object *lock) 10871088Sjasone{ 109167789Sjhb WITNESS_SAVE_DECL(lock_witness); 110167789Sjhb struct lock_class *class; 111167786Sjhb struct thread *td; 112255745Sdavide uintptr_t lock_state; 11371088Sjasone 114167786Sjhb td = curthread; 115181394Sjhb lock_state = 0; 116167786Sjhb#ifdef KTRACE 117167786Sjhb if (KTRPOINT(td, KTR_CSW)) 118234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 119167786Sjhb#endif 120167789Sjhb CV_ASSERT(cvp, lock, td); 121167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 122167786Sjhb "Waiting on \"%s\"", cvp->cv_description); 123167789Sjhb class = LOCK_CLASS(lock); 124153321Srodrigc 125153321Srodrigc if (cold || panicstr) { 126153321Srodrigc /* 127153321Srodrigc * During autoconfiguration, just give interrupts 128153321Srodrigc * a chance, then just return. Don't run any other 129153321Srodrigc * thread or panic below, in case this is the idle 130153321Srodrigc * process and already asleep. 131153321Srodrigc */ 132153321Srodrigc return; 133153321Srodrigc } 134153321Srodrigc 135167786Sjhb sleepq_lock(cvp); 136167786Sjhb 137294755Smarkj CV_WAITERS_INC(cvp); 138183352Sjhb if (lock == &Giant.lock_object) 139183352Sjhb mtx_assert(&Giant, MA_OWNED); 140167786Sjhb DROP_GIANT(); 141167786Sjhb 142167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 143181394Sjhb if (lock != &Giant.lock_object) { 144181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 145181394Sjhb sleepq_release(cvp); 146183352Sjhb WITNESS_SAVE(lock, lock_witness); 147181394Sjhb lock_state = class->lc_unlock(lock); 148181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 149181394Sjhb sleepq_lock(cvp); 150181394Sjhb } 151177085Sjeff sleepq_wait(cvp, 0); 152167786Sjhb 153167786Sjhb#ifdef KTRACE 154167786Sjhb if (KTRPOINT(td, KTR_CSW)) 155234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 156167786Sjhb#endif 157167786Sjhb PICKUP_GIANT(); 158181394Sjhb if (lock != &Giant.lock_object) { 159181394Sjhb class->lc_lock(lock, lock_state); 160181394Sjhb WITNESS_RESTORE(lock, lock_witness); 161181394Sjhb } 162153321Srodrigc} 163153321Srodrigc 164153321Srodrigc/* 165153321Srodrigc * Wait on a condition variable. This function differs from cv_wait by 166302234Sbdrewery * not acquiring the mutex after condition variable was signaled. 167153321Srodrigc */ 168153321Srodrigcvoid 169167789Sjhb_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 170153321Srodrigc{ 171167789Sjhb struct lock_class *class; 172153321Srodrigc struct thread *td; 173153321Srodrigc 17483366Sjulian td = curthread; 17571088Sjasone#ifdef KTRACE 17697995Sjhb if (KTRPOINT(td, KTR_CSW)) 177234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 17871088Sjasone#endif 179167789Sjhb CV_ASSERT(cvp, lock, td); 180167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 181111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 182181394Sjhb KASSERT(lock != &Giant.lock_object, 183181394Sjhb ("cv_wait_unlock cannot be used with Giant")); 184167789Sjhb class = LOCK_CLASS(lock); 18571088Sjasone 186126326Sjhb if (cold || panicstr) { 18771088Sjasone /* 188100209Sgallatin * During autoconfiguration, just give interrupts 189100209Sgallatin * a chance, then just return. Don't run any other 190100209Sgallatin * thread or panic below, in case this is the idle 191100209Sgallatin * process and already asleep. 19271088Sjasone */ 193167789Sjhb class->lc_unlock(lock); 19471088Sjasone return; 19571088Sjasone } 19695322Shsu 197136445Sjhb sleepq_lock(cvp); 19895322Shsu 199294755Smarkj CV_WAITERS_INC(cvp); 20088900Sjhb DROP_GIANT(); 20171088Sjasone 202167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 203169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 204169392Sjhb sleepq_release(cvp); 205169392Sjhb class->lc_unlock(lock); 206169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 207169392Sjhb sleepq_lock(cvp); 208177085Sjeff sleepq_wait(cvp, 0); 20971088Sjasone 210167786Sjhb#ifdef KTRACE 211167786Sjhb if (KTRPOINT(td, KTR_CSW)) 212234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 213167786Sjhb#endif 21471088Sjasone PICKUP_GIANT(); 21571088Sjasone} 21671088Sjasone 21771088Sjasone/* 21871088Sjasone * Wait on a condition variable, allowing interruption by signals. Return 0 if 21983366Sjulian * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 22071088Sjasone * a signal was caught. If ERESTART is returned the system call should be 22171088Sjasone * restarted if possible. 22271088Sjasone */ 22371088Sjasoneint 224167789Sjhb_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 22571088Sjasone{ 226167789Sjhb WITNESS_SAVE_DECL(lock_witness); 227167789Sjhb struct lock_class *class; 22883366Sjulian struct thread *td; 229255745Sdavide uintptr_t lock_state; 230255745Sdavide int rval; 23171088Sjasone 23283366Sjulian td = curthread; 233181394Sjhb lock_state = 0; 23471088Sjasone#ifdef KTRACE 23597995Sjhb if (KTRPOINT(td, KTR_CSW)) 236234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 23771088Sjasone#endif 238167789Sjhb CV_ASSERT(cvp, lock, td); 239167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 240111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 241167789Sjhb class = LOCK_CLASS(lock); 24271088Sjasone 24371088Sjasone if (cold || panicstr) { 24471088Sjasone /* 24571088Sjasone * After a panic, or during autoconfiguration, just give 24671088Sjasone * interrupts a chance, then just return; don't run any other 24771088Sjasone * procs or panic below, in case this is the idle process and 24871088Sjasone * already asleep. 24971088Sjasone */ 250133440Sjhb return (0); 25171088Sjasone } 25295322Shsu 253136445Sjhb sleepq_lock(cvp); 25495322Shsu 255294755Smarkj CV_WAITERS_INC(cvp); 256183352Sjhb if (lock == &Giant.lock_object) 257183352Sjhb mtx_assert(&Giant, MA_OWNED); 25888900Sjhb DROP_GIANT(); 25971088Sjasone 260167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 261165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 262181394Sjhb if (lock != &Giant.lock_object) { 263181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 264181394Sjhb sleepq_release(cvp); 265183352Sjhb WITNESS_SAVE(lock, lock_witness); 266181394Sjhb lock_state = class->lc_unlock(lock); 267181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 268181394Sjhb sleepq_lock(cvp); 269181394Sjhb } 270177085Sjeff rval = sleepq_wait_sig(cvp, 0); 27171088Sjasone 27271088Sjasone#ifdef KTRACE 27397995Sjhb if (KTRPOINT(td, KTR_CSW)) 274234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 27571088Sjasone#endif 27697995Sjhb PICKUP_GIANT(); 277181394Sjhb if (lock != &Giant.lock_object) { 278181394Sjhb class->lc_lock(lock, lock_state); 279181394Sjhb WITNESS_RESTORE(lock, lock_witness); 280181394Sjhb } 28171088Sjasone 28271088Sjasone return (rval); 28371088Sjasone} 28471088Sjasone 28571088Sjasone/* 286247785Sdavide * Wait on a condition variable for (at most) the value specified in sbt 287247785Sdavide * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 288247785Sdavide * EWOULDBLOCK if the timeout expires. 28971088Sjasone */ 29071088Sjasoneint 291247785Sdavide_cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 292247785Sdavide sbintime_t pr, int flags) 29371088Sjasone{ 294167789Sjhb WITNESS_SAVE_DECL(lock_witness); 295167789Sjhb struct lock_class *class; 29683366Sjulian struct thread *td; 297167789Sjhb int lock_state, rval; 29871088Sjasone 29983366Sjulian td = curthread; 300181394Sjhb lock_state = 0; 30171088Sjasone#ifdef KTRACE 30297995Sjhb if (KTRPOINT(td, KTR_CSW)) 303234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 30471088Sjasone#endif 305167789Sjhb CV_ASSERT(cvp, lock, td); 306167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 307111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 308167789Sjhb class = LOCK_CLASS(lock); 30971088Sjasone 31071088Sjasone if (cold || panicstr) { 31171088Sjasone /* 31271088Sjasone * After a panic, or during autoconfiguration, just give 31371088Sjasone * interrupts a chance, then just return; don't run any other 31483366Sjulian * thread or panic below, in case this is the idle process and 31571088Sjasone * already asleep. 31671088Sjasone */ 31771088Sjasone return 0; 31871088Sjasone } 31995322Shsu 320136445Sjhb sleepq_lock(cvp); 32195322Shsu 322294755Smarkj CV_WAITERS_INC(cvp); 323183352Sjhb if (lock == &Giant.lock_object) 324183352Sjhb mtx_assert(&Giant, MA_OWNED); 32588900Sjhb DROP_GIANT(); 32671088Sjasone 327167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 328247785Sdavide sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 329181394Sjhb if (lock != &Giant.lock_object) { 330181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 331181394Sjhb sleepq_release(cvp); 332183352Sjhb WITNESS_SAVE(lock, lock_witness); 333181394Sjhb lock_state = class->lc_unlock(lock); 334181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 335181394Sjhb sleepq_lock(cvp); 336181394Sjhb } 337177085Sjeff rval = sleepq_timedwait(cvp, 0); 33871088Sjasone 33971088Sjasone#ifdef KTRACE 34097995Sjhb if (KTRPOINT(td, KTR_CSW)) 341234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 34271088Sjasone#endif 34371088Sjasone PICKUP_GIANT(); 344181394Sjhb if (lock != &Giant.lock_object) { 345181394Sjhb class->lc_lock(lock, lock_state); 346181394Sjhb WITNESS_RESTORE(lock, lock_witness); 347181394Sjhb } 34871088Sjasone 34971088Sjasone return (rval); 35071088Sjasone} 35171088Sjasone 35271088Sjasone/* 353247785Sdavide * Wait on a condition variable for (at most) the value specified in sbt 354247785Sdavide * argument, allowing interruption by signals. 355247785Sdavide * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 356247785Sdavide * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 357247785Sdavide * was caught. 35871088Sjasone */ 35971088Sjasoneint 360247785Sdavide_cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 361247785Sdavide sbintime_t sbt, sbintime_t pr, int flags) 36271088Sjasone{ 363167789Sjhb WITNESS_SAVE_DECL(lock_witness); 364167789Sjhb struct lock_class *class; 36583366Sjulian struct thread *td; 366167789Sjhb int lock_state, rval; 36771088Sjasone 36883366Sjulian td = curthread; 369181394Sjhb lock_state = 0; 37071088Sjasone#ifdef KTRACE 37197995Sjhb if (KTRPOINT(td, KTR_CSW)) 372234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 37371088Sjasone#endif 374167789Sjhb CV_ASSERT(cvp, lock, td); 375167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 376111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 377167789Sjhb class = LOCK_CLASS(lock); 37871088Sjasone 37971088Sjasone if (cold || panicstr) { 38071088Sjasone /* 38171088Sjasone * After a panic, or during autoconfiguration, just give 38271088Sjasone * interrupts a chance, then just return; don't run any other 38383366Sjulian * thread or panic below, in case this is the idle process and 38471088Sjasone * already asleep. 38571088Sjasone */ 38671088Sjasone return 0; 38771088Sjasone } 38895322Shsu 389136445Sjhb sleepq_lock(cvp); 39095322Shsu 391294755Smarkj CV_WAITERS_INC(cvp); 392183352Sjhb if (lock == &Giant.lock_object) 393183352Sjhb mtx_assert(&Giant, MA_OWNED); 39488900Sjhb DROP_GIANT(); 39571088Sjasone 396167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 397165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 398247785Sdavide sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 399181394Sjhb if (lock != &Giant.lock_object) { 400181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 401181394Sjhb sleepq_release(cvp); 402183352Sjhb WITNESS_SAVE(lock, lock_witness); 403181394Sjhb lock_state = class->lc_unlock(lock); 404181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 405181394Sjhb sleepq_lock(cvp); 406181394Sjhb } 407177085Sjeff rval = sleepq_timedwait_sig(cvp, 0); 40871088Sjasone 40971088Sjasone#ifdef KTRACE 41097995Sjhb if (KTRPOINT(td, KTR_CSW)) 411234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 41271088Sjasone#endif 41397995Sjhb PICKUP_GIANT(); 414181394Sjhb if (lock != &Giant.lock_object) { 415181394Sjhb class->lc_lock(lock, lock_state); 416181394Sjhb WITNESS_RESTORE(lock, lock_witness); 417181394Sjhb } 41871088Sjasone 41971088Sjasone return (rval); 42071088Sjasone} 42171088Sjasone 42271088Sjasone/* 42383366Sjulian * Signal a condition variable, wakes up one waiting thread. Will also wakeup 42471088Sjasone * the swapper if the process is not in memory, so that it can bring the 42583366Sjulian * sleeping process in. Note that this may also result in additional threads 42671088Sjasone * being made runnable. Should be called with the same mutex as was passed to 42771088Sjasone * cv_wait held. 42871088Sjasone */ 42971088Sjasonevoid 43071088Sjasonecv_signal(struct cv *cvp) 43171088Sjasone{ 432181334Sjhb int wakeup_swapper; 43371088Sjasone 434181334Sjhb wakeup_swapper = 0; 435136445Sjhb sleepq_lock(cvp); 436127954Sjhb if (cvp->cv_waiters > 0) { 437294755Smarkj if (cvp->cv_waiters == CV_WAITERS_BOUND && 438294755Smarkj sleepq_lookup(cvp) == NULL) { 439294755Smarkj cvp->cv_waiters = 0; 440294755Smarkj } else { 441294755Smarkj if (cvp->cv_waiters < CV_WAITERS_BOUND) 442294755Smarkj cvp->cv_waiters--; 443294755Smarkj wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 444294755Smarkj 0); 445294755Smarkj } 446170294Sjeff } 447170294Sjeff sleepq_release(cvp); 448181334Sjhb if (wakeup_swapper) 449181334Sjhb kick_proc0(); 45071088Sjasone} 45171088Sjasone 45271088Sjasone/* 45383366Sjulian * Broadcast a signal to a condition variable. Wakes up all waiting threads. 45471088Sjasone * Should be called with the same mutex as was passed to cv_wait held. 45571088Sjasone */ 45671088Sjasonevoid 457122352Stanimuracv_broadcastpri(struct cv *cvp, int pri) 45871088Sjasone{ 459181334Sjhb int wakeup_swapper; 460181334Sjhb 461177085Sjeff /* 462177085Sjeff * XXX sleepq_broadcast pri argument changed from -1 meaning 463177085Sjeff * no pri to 0 meaning no pri. 464177085Sjeff */ 465181334Sjhb wakeup_swapper = 0; 466177085Sjeff if (pri == -1) 467177085Sjeff pri = 0; 468136445Sjhb sleepq_lock(cvp); 469127954Sjhb if (cvp->cv_waiters > 0) { 470127954Sjhb cvp->cv_waiters = 0; 471181334Sjhb wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 472177085Sjeff } 473177085Sjeff sleepq_release(cvp); 474181334Sjhb if (wakeup_swapper) 475181334Sjhb kick_proc0(); 47671088Sjasone} 477