kern_sx.c revision 82212
150477Speter/* 235388Smjacob * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>. All rights reserved. 335388Smjacob * 435388Smjacob * Redistribution and use in source and binary forms, with or without 566189Smjacob * modification, are permitted provided that the following conditions 635388Smjacob * are met: 752347Smjacob * 1. Redistributions of source code must retain the above copyright 835388Smjacob * notice(s), this list of conditions and the following disclaimer as 935388Smjacob * the first lines of this file unmodified other than the possible 1035388Smjacob * addition of one or more copyright notices. 1135388Smjacob * 2. Redistributions in binary form must reproduce the above copyright 1235388Smjacob * notice(s), this list of conditions and the following disclaimer in the 1335388Smjacob * documentation and/or other materials provided with the distribution. 1466189Smjacob * 1535388Smjacob * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 1635388Smjacob * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 1735388Smjacob * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 1835388Smjacob * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 1935388Smjacob * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2035388Smjacob * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 2135388Smjacob * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 2235388Smjacob * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2335388Smjacob * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2435388Smjacob * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 2535388Smjacob * DAMAGE. 2635388Smjacob * 2735388Smjacob * $FreeBSD: head/sys/kern/kern_sx.c 82212 2001-08-23 17:57:37Z jhb $ 2835388Smjacob */ 2935388Smjacob 3035388Smjacob/* 3135388Smjacob * Shared/exclusive locks. This implementation assures deterministic lock 3235388Smjacob * granting behavior, so that slocks and xlocks are interleaved. 3335388Smjacob * 3444819Smjacob * Priority propagation will not generally raise the priority of lock holders, 3535388Smjacob * so should not be relied upon in combination with sx locks. 3653487Smjacob */ 3753487Smjacob 3856004Smjacob#include <sys/param.h> 3935388Smjacob#include <sys/systm.h> 4053487Smjacob#include <sys/ktr.h> 4135388Smjacob#include <sys/condvar.h> 4235388Smjacob#include <sys/lock.h> 4353487Smjacob#include <sys/mutex.h> 4453487Smjacob#include <sys/sx.h> 4556004Smjacob 4635388Smjacobstruct lock_class lock_class_sx = { 4753487Smjacob "sx", 4835388Smjacob LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE 4942131Smjacob}; 5053487Smjacob 5153487Smjacobvoid 5256004Smjacobsx_init(struct sx *sx, const char *description) 5335388Smjacob{ 5453487Smjacob struct lock_object *lock; 5535388Smjacob 5664087Smjacob bzero(sx, sizeof(*sx)); 5787635Smjacob lock = &sx->sx_object; 5839235Sgibbs lock->lo_class = &lock_class_sx; 5935388Smjacob lock->lo_name = description; 6043420Smjacob lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE; 6135388Smjacob mtx_init(&sx->sx_lock, "sx backing lock", 6235388Smjacob MTX_DEF | MTX_NOWITNESS | MTX_QUIET); 6335388Smjacob sx->sx_cnt = 0; 6482689Smjacob cv_init(&sx->sx_shrd_cv, description); 6582689Smjacob sx->sx_shrd_wcnt = 0; 6674229Smjacob cv_init(&sx->sx_excl_cv, description); 6774229Smjacob sx->sx_excl_wcnt = 0; 6874229Smjacob sx->sx_xholder = NULL; 6974229Smjacob 7082689Smjacob LOCK_LOG_INIT(lock, 0); 7135388Smjacob 7282689Smjacob WITNESS_INIT(lock); 7374229Smjacob} 7474229Smjacob 7574229Smjacobvoid 7690224Smjacobsx_destroy(struct sx *sx) 7735388Smjacob{ 7835388Smjacob 7935388Smjacob LOCK_LOG_DESTROY(&sx->sx_object, 0); 8035388Smjacob 8164087Smjacob KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == 8264087Smjacob 0), ("%s (%s): holders or waiters\n", __FUNCTION__, 8364087Smjacob sx->sx_object.lo_name)); 8482689Smjacob 8582689Smjacob mtx_destroy(&sx->sx_lock); 8649909Smjacob cv_destroy(&sx->sx_shrd_cv); 8761772Smjacob cv_destroy(&sx->sx_excl_cv); 8849909Smjacob 8982689Smjacob WITNESS_DESTROY(&sx->sx_object); 9082689Smjacob} 9182689Smjacob 9282689Smjacobvoid 9382689Smjacob_sx_slock(struct sx *sx, const char *file, int line) 9482689Smjacob{ 9553487Smjacob 9653487Smjacob mtx_lock(&sx->sx_lock); 9764087Smjacob KASSERT(sx->sx_xholder != curproc, 9864087Smjacob ("%s (%s): slock while xlock is held @ %s:%d\n", __FUNCTION__, 9953487Smjacob sx->sx_object.lo_name, file, line)); 10082689Smjacob 10182689Smjacob /* 10253487Smjacob * Loop in case we lose the race for lock acquisition. 10353487Smjacob */ 10453487Smjacob while (sx->sx_cnt < 0) { 10553487Smjacob sx->sx_shrd_wcnt++; 10653487Smjacob cv_wait(&sx->sx_shrd_cv, &sx->sx_lock); 10753487Smjacob sx->sx_shrd_wcnt--; 10853487Smjacob } 10953487Smjacob 11053487Smjacob /* Acquire a shared lock. */ 11153487Smjacob sx->sx_cnt++; 11253487Smjacob 11353487Smjacob LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line); 11453487Smjacob WITNESS_LOCK(&sx->sx_object, 0, file, line); 11553487Smjacob 11653487Smjacob mtx_unlock(&sx->sx_lock); 11753487Smjacob} 11853487Smjacob 11953487Smjacobint 12053487Smjacob_sx_try_slock(struct sx *sx, const char *file, int line) 12153487Smjacob{ 12253487Smjacob 12364087Smjacob mtx_lock(&sx->sx_lock); 12464087Smjacob if (sx->sx_cnt >= 0) { 12553487Smjacob sx->sx_cnt++; 12653487Smjacob LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 1, file, line); 12753487Smjacob WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line); 12853487Smjacob mtx_unlock(&sx->sx_lock); 12953487Smjacob return (1); 13053487Smjacob } else { 13153487Smjacob LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 0, file, line); 13264087Smjacob mtx_unlock(&sx->sx_lock); 13364087Smjacob return (0); 13464087Smjacob } 13564087Smjacob} 13664087Smjacob 13764087Smjacobvoid 13864087Smjacob_sx_xlock(struct sx *sx, const char *file, int line) 13964087Smjacob{ 14064087Smjacob 14164087Smjacob mtx_lock(&sx->sx_lock); 14264087Smjacob 14364087Smjacob /* 14464087Smjacob * With sx locks, we're absolutely not permitted to recurse on 14564087Smjacob * xlocks, as it is fatal (deadlock). Normally, recursion is handled 14664087Smjacob * by WITNESS, but as it is not semantically correct to hold the 14764087Smjacob * xlock while in here, we consider it API abuse and put it under 14864087Smjacob * INVARIANTS. 14939235Sgibbs */ 15064087Smjacob KASSERT(sx->sx_xholder != curproc, 15164087Smjacob ("%s (%s): xlock already held @ %s:%d", __FUNCTION__, 15264087Smjacob sx->sx_object.lo_name, file, line)); 15364087Smjacob 15439235Sgibbs /* Loop in case we lose the race for lock acquisition. */ 15539235Sgibbs while (sx->sx_cnt != 0) { 15639235Sgibbs sx->sx_excl_wcnt++; 15765140Smjacob cv_wait(&sx->sx_excl_cv, &sx->sx_lock); 15839235Sgibbs sx->sx_excl_wcnt--; 15952347Smjacob } 16065140Smjacob 16165140Smjacob MPASS(sx->sx_cnt == 0); 16253487Smjacob 16387635Smjacob /* Acquire an exclusive lock. */ 16487635Smjacob sx->sx_cnt--; 16587635Smjacob sx->sx_xholder = curproc; 16687635Smjacob 16753487Smjacob LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line); 16835388Smjacob WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 16946968Smjacob 17035388Smjacob mtx_unlock(&sx->sx_lock); 17135388Smjacob} 17235388Smjacob 17346968Smjacobint 17452347Smjacob_sx_try_xlock(struct sx *sx, const char *file, int line) 17552347Smjacob{ 17635388Smjacob 17735388Smjacob mtx_lock(&sx->sx_lock); 17842461Smjacob if (sx->sx_cnt == 0) { 17943420Smjacob sx->sx_cnt--; 18035388Smjacob sx->sx_xholder = curproc; 18145040Smjacob LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 1, file, line); 18265140Smjacob WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, 18335388Smjacob line); 18452347Smjacob mtx_unlock(&sx->sx_lock); 18552347Smjacob return (1); 18652347Smjacob } else { 18735388Smjacob LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 0, file, line); 18852347Smjacob mtx_unlock(&sx->sx_lock); 18952347Smjacob return (0); 19052347Smjacob } 19135388Smjacob} 19280582Smjacob 19380582Smjacobvoid 19446968Smjacob_sx_sunlock(struct sx *sx, const char *file, int line) 19580582Smjacob{ 19643420Smjacob 19743420Smjacob mtx_lock(&sx->sx_lock); 19880582Smjacob _SX_ASSERT_SLOCKED(sx, file, line); 19980582Smjacob 20080582Smjacob WITNESS_UNLOCK(&sx->sx_object, 0, file, line); 20180582Smjacob 20280582Smjacob /* Release. */ 20380582Smjacob sx->sx_cnt--; 20480582Smjacob 20580582Smjacob /* 20680582Smjacob * If we just released the last shared lock, wake any waiters up, giving 20735388Smjacob * exclusive lockers precedence. In order to make sure that exclusive 20846968Smjacob * lockers won't be blocked forever, don't wake shared lock waiters if 20935388Smjacob * there are exclusive lock waiters. 21035388Smjacob */ 21135388Smjacob if (sx->sx_excl_wcnt > 0) { 21235388Smjacob if (sx->sx_cnt == 0) 21339235Sgibbs cv_signal(&sx->sx_excl_cv); 21439235Sgibbs } else if (sx->sx_shrd_wcnt > 0) 21539235Sgibbs cv_broadcast(&sx->sx_shrd_cv); 21639235Sgibbs 21739235Sgibbs LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line); 21839235Sgibbs 21939235Sgibbs mtx_unlock(&sx->sx_lock); 22039235Sgibbs} 22165140Smjacob 22265140Smjacobvoid 22365140Smjacob_sx_xunlock(struct sx *sx, const char *file, int line) 22443793Smjacob{ 22539235Sgibbs 22635388Smjacob mtx_lock(&sx->sx_lock); 22739235Sgibbs _SX_ASSERT_XLOCKED(sx, file, line); 22845040Smjacob MPASS(sx->sx_cnt == -1); 22965140Smjacob 23065140Smjacob WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 23165140Smjacob 23265140Smjacob /* Release. */ 23352347Smjacob sx->sx_cnt++; 23452347Smjacob sx->sx_xholder = NULL; 23552347Smjacob 23652347Smjacob /* 23735388Smjacob * Wake up waiters if there are any. Give precedence to slock waiters. 23835388Smjacob */ 23935388Smjacob if (sx->sx_shrd_wcnt > 0) 24035388Smjacob cv_broadcast(&sx->sx_shrd_cv); 24152347Smjacob else if (sx->sx_excl_wcnt > 0) 24252347Smjacob cv_signal(&sx->sx_excl_cv); 24352347Smjacob 24452347Smjacob LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line); 24535388Smjacob 24652347Smjacob mtx_unlock(&sx->sx_lock); 24784241Smjacob} 24872355Smjacob 24948484Smjacobint 25077776Smjacob_sx_try_upgrade(struct sx *sx, const char *file, int line) 25148484Smjacob{ 25248484Smjacob 25359454Smjacob mtx_lock(&sx->sx_lock); 25448484Smjacob _SX_ASSERT_SLOCKED(sx, file, line); 25572355Smjacob 25639235Sgibbs if (sx->sx_cnt == 1) { 25739235Sgibbs WITNESS_UNLOCK(&sx->sx_object, 0, file, line); 25872355Smjacob 25952347Smjacob sx->sx_cnt = -1; 26082841Smjacob sx->sx_xholder = curproc; 26139235Sgibbs 26252347Smjacob LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 1, file, line); 26352347Smjacob WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, 26452347Smjacob line); 26539235Sgibbs 26639235Sgibbs mtx_unlock(&sx->sx_lock); 26748484Smjacob return (1); 26848484Smjacob } else { 26935388Smjacob LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 0, file, line); 27048484Smjacob mtx_unlock(&sx->sx_lock); 27148484Smjacob return (0); 27248484Smjacob } 27348484Smjacob} 27448484Smjacob 27548484Smjacobvoid 27648484Smjacob_sx_downgrade(struct sx *sx, const char *file, int line) 27744819Smjacob{ 27848484Smjacob 27952347Smjacob mtx_lock(&sx->sx_lock); 28060221Smjacob _SX_ASSERT_XLOCKED(sx, file, line); 28177776Smjacob MPASS(sx->sx_cnt == -1); 28277776Smjacob 28371079Smjacob WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 28471079Smjacob 28560221Smjacob sx->sx_cnt = 1; 28660221Smjacob sx->sx_xholder = NULL; 28760221Smjacob if (sx->sx_shrd_wcnt > 0) 28852347Smjacob cv_broadcast(&sx->sx_shrd_cv); 28948484Smjacob 29048484Smjacob LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line); 29172355Smjacob WITNESS_LOCK(&sx->sx_object, 0, file, line); 29244819Smjacob 29344819Smjacob mtx_unlock(&sx->sx_lock); 29435388Smjacob} 29535388Smjacob