kern_sx.c revision 193025
1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice(s), this list of conditions and the following disclaimer as 11 * the first lines of this file unmodified other than the possible 12 * addition of one or more copyright notices. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice(s), this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27 * DAMAGE. 28 */ 29 30/* 31 * Shared/exclusive locks. This implementation attempts to ensure 32 * deterministic lock granting behavior, so that slocks and xlocks are 33 * interleaved. 34 * 35 * Priority propagation will not generally raise the priority of lock holders, 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39#include "opt_ddb.h" 40#include "opt_kdtrace.h" 41#include "opt_no_adaptive_sx.h" 42 43#include <sys/cdefs.h> 44__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 193025 2009-05-29 08:01:48Z attilio $"); 45 46#include <sys/param.h> 47#include <sys/ktr.h> 48#include <sys/linker_set.h> 49#include <sys/lock.h> 50#include <sys/mutex.h> 51#include <sys/proc.h> 52#include <sys/sleepqueue.h> 53#include <sys/sx.h> 54#include <sys/sysctl.h> 55#include <sys/systm.h> 56 57#if defined(SMP) && !defined(NO_ADAPTIVE_SX) 58#include <machine/cpu.h> 59#endif 60 61#ifdef DDB 62#include <ddb/ddb.h> 63#endif 64 65#if defined(SMP) && !defined(NO_ADAPTIVE_SX) 66#define ADAPTIVE_SX 67#endif 68 69CTASSERT(((SX_NOADAPTIVE | SX_RECURSE) & LO_CLASSFLAGS) == 70 (SX_NOADAPTIVE | SX_RECURSE)); 71 72/* Handy macros for sleep queues. */ 73#define SQ_EXCLUSIVE_QUEUE 0 74#define SQ_SHARED_QUEUE 1 75 76/* 77 * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We 78 * drop Giant anytime we have to sleep or if we adaptively spin. 79 */ 80#define GIANT_DECLARE \ 81 int _giantcnt = 0; \ 82 WITNESS_SAVE_DECL(Giant) \ 83 84#define GIANT_SAVE() do { \ 85 if (mtx_owned(&Giant)) { \ 86 WITNESS_SAVE(&Giant.lock_object, Giant); \ 87 while (mtx_owned(&Giant)) { \ 88 _giantcnt++; \ 89 mtx_unlock(&Giant); \ 90 } \ 91 } \ 92} while (0) 93 94#define GIANT_RESTORE() do { \ 95 if (_giantcnt > 0) { \ 96 mtx_assert(&Giant, MA_NOTOWNED); \ 97 while (_giantcnt--) \ 98 mtx_lock(&Giant); \ 99 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 100 } \ 101} while (0) 102 103/* 104 * Returns true if an exclusive lock is recursed. It assumes 105 * curthread currently has an exclusive lock. 106 */ 107#define sx_recurse lock_object.lo_data 108#define sx_recursed(sx) ((sx)->sx_recurse != 0) 109 110static void assert_sx(struct lock_object *lock, int what); 111#ifdef DDB 112static void db_show_sx(struct lock_object *lock); 113#endif 114static void lock_sx(struct lock_object *lock, int how); 115#ifdef KDTRACE_HOOKS 116static int owner_sx(struct lock_object *lock, struct thread **owner); 117#endif 118static int unlock_sx(struct lock_object *lock); 119 120struct lock_class lock_class_sx = { 121 .lc_name = "sx", 122 .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 123 .lc_assert = assert_sx, 124#ifdef DDB 125 .lc_ddb_show = db_show_sx, 126#endif 127 .lc_lock = lock_sx, 128 .lc_unlock = unlock_sx, 129#ifdef KDTRACE_HOOKS 130 .lc_owner = owner_sx, 131#endif 132}; 133 134#ifndef INVARIANTS 135#define _sx_assert(sx, what, file, line) 136#endif 137 138#ifdef ADAPTIVE_SX 139static u_int asx_retries = 10; 140static u_int asx_loops = 10000; 141SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging"); 142SYSCTL_INT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, ""); 143SYSCTL_INT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, ""); 144#endif 145 146void 147assert_sx(struct lock_object *lock, int what) 148{ 149 150 sx_assert((struct sx *)lock, what); 151} 152 153void 154lock_sx(struct lock_object *lock, int how) 155{ 156 struct sx *sx; 157 158 sx = (struct sx *)lock; 159 if (how) 160 sx_xlock(sx); 161 else 162 sx_slock(sx); 163} 164 165int 166unlock_sx(struct lock_object *lock) 167{ 168 struct sx *sx; 169 170 sx = (struct sx *)lock; 171 sx_assert(sx, SA_LOCKED | SA_NOTRECURSED); 172 if (sx_xlocked(sx)) { 173 sx_xunlock(sx); 174 return (1); 175 } else { 176 sx_sunlock(sx); 177 return (0); 178 } 179} 180 181#ifdef KDTRACE_HOOKS 182int 183owner_sx(struct lock_object *lock, struct thread **owner) 184{ 185 struct sx *sx = (struct sx *)lock; 186 uintptr_t x = sx->sx_lock; 187 188 *owner = (struct thread *)SX_OWNER(x); 189 return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) : 190 (*owner != NULL)); 191} 192#endif 193 194void 195sx_sysinit(void *arg) 196{ 197 struct sx_args *sargs = arg; 198 199 sx_init(sargs->sa_sx, sargs->sa_desc); 200} 201 202void 203sx_init_flags(struct sx *sx, const char *description, int opts) 204{ 205 int flags; 206 207 MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK | 208 SX_NOPROFILE | SX_NOADAPTIVE)) == 0); 209 210 flags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE; 211 if (opts & SX_DUPOK) 212 flags |= LO_DUPOK; 213 if (opts & SX_NOPROFILE) 214 flags |= LO_NOPROFILE; 215 if (!(opts & SX_NOWITNESS)) 216 flags |= LO_WITNESS; 217 if (opts & SX_QUIET) 218 flags |= LO_QUIET; 219 220 flags |= opts & (SX_NOADAPTIVE | SX_RECURSE); 221 sx->sx_lock = SX_LOCK_UNLOCKED; 222 sx->sx_recurse = 0; 223 lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags); 224} 225 226void 227sx_destroy(struct sx *sx) 228{ 229 230 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held")); 231 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed")); 232 sx->sx_lock = SX_LOCK_DESTROYED; 233 lock_destroy(&sx->lock_object); 234} 235 236int 237_sx_slock(struct sx *sx, int opts, const char *file, int line) 238{ 239 int error = 0; 240 241 MPASS(curthread != NULL); 242 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 243 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 244 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); 245 error = __sx_slock(sx, opts, file, line); 246 if (!error) { 247 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 248 WITNESS_LOCK(&sx->lock_object, 0, file, line); 249 curthread->td_locks++; 250 } 251 252 return (error); 253} 254 255int 256_sx_try_slock(struct sx *sx, const char *file, int line) 257{ 258 uintptr_t x; 259 260 for (;;) { 261 x = sx->sx_lock; 262 KASSERT(x != SX_LOCK_DESTROYED, 263 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 264 if (!(x & SX_LOCK_SHARED)) 265 break; 266 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 267 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 268 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); 269 curthread->td_locks++; 270 return (1); 271 } 272 } 273 274 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 275 return (0); 276} 277 278int 279_sx_xlock(struct sx *sx, int opts, const char *file, int line) 280{ 281 int error = 0; 282 283 MPASS(curthread != NULL); 284 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 285 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 286 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 287 line, NULL); 288 error = __sx_xlock(sx, curthread, opts, file, line); 289 if (!error) { 290 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, 291 file, line); 292 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 293 curthread->td_locks++; 294 } 295 296 return (error); 297} 298 299int 300_sx_try_xlock(struct sx *sx, const char *file, int line) 301{ 302 int rval; 303 304 MPASS(curthread != NULL); 305 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 306 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 307 308 if (sx_xlocked(sx) && (sx->lock_object.lo_flags & SX_RECURSE) != 0) { 309 sx->sx_recurse++; 310 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 311 rval = 1; 312 } else 313 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 314 (uintptr_t)curthread); 315 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 316 if (rval) { 317 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 318 file, line); 319 curthread->td_locks++; 320 } 321 322 return (rval); 323} 324 325void 326_sx_sunlock(struct sx *sx, const char *file, int line) 327{ 328 329 MPASS(curthread != NULL); 330 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 331 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 332 _sx_assert(sx, SA_SLOCKED, file, line); 333 curthread->td_locks--; 334 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 335 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 336 __sx_sunlock(sx, file, line); 337 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx); 338} 339 340void 341_sx_xunlock(struct sx *sx, const char *file, int line) 342{ 343 344 MPASS(curthread != NULL); 345 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 346 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 347 _sx_assert(sx, SA_XLOCKED, file, line); 348 curthread->td_locks--; 349 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 350 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 351 line); 352 if (!sx_recursed(sx)) 353 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_XUNLOCK_RELEASE, sx); 354 __sx_xunlock(sx, curthread, file, line); 355} 356 357/* 358 * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. 359 * This will only succeed if this thread holds a single shared lock. 360 * Return 1 if if the upgrade succeed, 0 otherwise. 361 */ 362int 363_sx_try_upgrade(struct sx *sx, const char *file, int line) 364{ 365 uintptr_t x; 366 int success; 367 368 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 369 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 370 _sx_assert(sx, SA_SLOCKED, file, line); 371 372 /* 373 * Try to switch from one shared lock to an exclusive lock. We need 374 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 375 * we will wake up the exclusive waiters when we drop the lock. 376 */ 377 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; 378 success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, 379 (uintptr_t)curthread | x); 380 LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); 381 if (success) { 382 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 383 file, line); 384 LOCKSTAT_RECORD0(LS_SX_TRYUPGRADE_UPGRADE, sx); 385 } 386 return (success); 387} 388 389/* 390 * Downgrade an unrecursed exclusive lock into a single shared lock. 391 */ 392void 393_sx_downgrade(struct sx *sx, const char *file, int line) 394{ 395 uintptr_t x; 396 int wakeup_swapper; 397 398 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 399 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 400 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); 401#ifndef INVARIANTS 402 if (sx_recursed(sx)) 403 panic("downgrade of a recursed lock"); 404#endif 405 406 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); 407 408 /* 409 * Try to switch from an exclusive lock with no shared waiters 410 * to one sharer with no shared waiters. If there are 411 * exclusive waiters, we don't need to lock the sleep queue so 412 * long as we preserve the flag. We do one quick try and if 413 * that fails we grab the sleepq lock to keep the flags from 414 * changing and do it the slow way. 415 * 416 * We have to lock the sleep queue if there are shared waiters 417 * so we can wake them up. 418 */ 419 x = sx->sx_lock; 420 if (!(x & SX_LOCK_SHARED_WAITERS) && 421 atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | 422 (x & SX_LOCK_EXCLUSIVE_WAITERS))) { 423 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 424 return; 425 } 426 427 /* 428 * Lock the sleep queue so we can read the waiters bits 429 * without any races and wakeup any shared waiters. 430 */ 431 sleepq_lock(&sx->lock_object); 432 433 /* 434 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single 435 * shared lock. If there are any shared waiters, wake them up. 436 */ 437 wakeup_swapper = 0; 438 x = sx->sx_lock; 439 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | 440 (x & SX_LOCK_EXCLUSIVE_WAITERS)); 441 if (x & SX_LOCK_SHARED_WAITERS) 442 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 443 0, SQ_SHARED_QUEUE); 444 sleepq_release(&sx->lock_object); 445 446 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 447 LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx); 448 449 if (wakeup_swapper) 450 kick_proc0(); 451} 452 453/* 454 * This function represents the so-called 'hard case' for sx_xlock 455 * operation. All 'easy case' failures are redirected to this. Note 456 * that ideally this would be a static function, but it needs to be 457 * accessible from at least sx.h. 458 */ 459int 460_sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file, 461 int line) 462{ 463 GIANT_DECLARE; 464#ifdef ADAPTIVE_SX 465 volatile struct thread *owner; 466 u_int i, spintries = 0; 467#endif 468 uintptr_t x; 469#ifdef LOCK_PROFILING 470 uint64_t waittime = 0; 471 int contested = 0; 472#endif 473 int error = 0; 474#ifdef KDTRACE_HOOKS 475 uint64_t spin_cnt = 0; 476 uint64_t sleep_cnt = 0; 477 int64_t sleep_time = 0; 478#endif 479 480 /* If we already hold an exclusive lock, then recurse. */ 481 if (sx_xlocked(sx)) { 482 KASSERT((sx->lock_object.lo_flags & SX_RECURSE) != 0, 483 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n", 484 sx->lock_object.lo_name, file, line)); 485 sx->sx_recurse++; 486 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 487 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 488 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 489 return (0); 490 } 491 492 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 493 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 494 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 495 496 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 497#ifdef KDTRACE_HOOKS 498 spin_cnt++; 499#endif 500 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 501 &waittime); 502#ifdef ADAPTIVE_SX 503 /* 504 * If the lock is write locked and the owner is 505 * running on another CPU, spin until the owner stops 506 * running or the state of the lock changes. 507 */ 508 x = sx->sx_lock; 509 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) != 0) { 510 if ((x & SX_LOCK_SHARED) == 0) { 511 x = SX_OWNER(x); 512 owner = (struct thread *)x; 513 if (TD_IS_RUNNING(owner)) { 514 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 515 CTR3(KTR_LOCK, 516 "%s: spinning on %p held by %p", 517 __func__, sx, owner); 518 GIANT_SAVE(); 519 while (SX_OWNER(sx->sx_lock) == x && 520 TD_IS_RUNNING(owner)) { 521 cpu_spinwait(); 522#ifdef KDTRACE_HOOKS 523 spin_cnt++; 524#endif 525 } 526 continue; 527 } 528 } else if (SX_SHARERS(x) && spintries < asx_retries) { 529 spintries++; 530 for (i = 0; i < asx_loops; i++) { 531 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 532 CTR4(KTR_LOCK, 533 "%s: shared spinning on %p with %u and %u", 534 __func__, sx, spintries, i); 535 GIANT_SAVE(); 536 x = sx->sx_lock; 537 if ((x & SX_LOCK_SHARED) == 0 || 538 SX_SHARERS(x) == 0) 539 break; 540 cpu_spinwait(); 541#ifdef KDTRACE_HOOKS 542 spin_cnt++; 543#endif 544 } 545 if (i != asx_loops) 546 continue; 547 } 548 } 549#endif 550 551 sleepq_lock(&sx->lock_object); 552 x = sx->sx_lock; 553 554 /* 555 * If the lock was released while spinning on the 556 * sleep queue chain lock, try again. 557 */ 558 if (x == SX_LOCK_UNLOCKED) { 559 sleepq_release(&sx->lock_object); 560 continue; 561 } 562 563#ifdef ADAPTIVE_SX 564 /* 565 * The current lock owner might have started executing 566 * on another CPU (or the lock could have changed 567 * owners) while we were waiting on the sleep queue 568 * chain lock. If so, drop the sleep queue lock and try 569 * again. 570 */ 571 if (!(x & SX_LOCK_SHARED) && 572 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 573 owner = (struct thread *)SX_OWNER(x); 574 if (TD_IS_RUNNING(owner)) { 575 sleepq_release(&sx->lock_object); 576 continue; 577 } 578 } 579#endif 580 581 /* 582 * If an exclusive lock was released with both shared 583 * and exclusive waiters and a shared waiter hasn't 584 * woken up and acquired the lock yet, sx_lock will be 585 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS. 586 * If we see that value, try to acquire it once. Note 587 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS 588 * as there are other exclusive waiters still. If we 589 * fail, restart the loop. 590 */ 591 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) { 592 if (atomic_cmpset_acq_ptr(&sx->sx_lock, 593 SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS, 594 tid | SX_LOCK_EXCLUSIVE_WAITERS)) { 595 sleepq_release(&sx->lock_object); 596 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 597 __func__, sx); 598 break; 599 } 600 sleepq_release(&sx->lock_object); 601 continue; 602 } 603 604 /* 605 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail, 606 * than loop back and retry. 607 */ 608 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 609 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 610 x | SX_LOCK_EXCLUSIVE_WAITERS)) { 611 sleepq_release(&sx->lock_object); 612 continue; 613 } 614 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 615 CTR2(KTR_LOCK, "%s: %p set excl waiters flag", 616 __func__, sx); 617 } 618 619 /* 620 * Since we have been unable to acquire the exclusive 621 * lock and the exclusive waiters flag is set, we have 622 * to sleep. 623 */ 624 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 625 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 626 __func__, sx); 627 628#ifdef KDTRACE_HOOKS 629 sleep_time -= lockstat_nsecs(); 630#endif 631 GIANT_SAVE(); 632 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 633 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 634 SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE); 635 if (!(opts & SX_INTERRUPTIBLE)) 636 sleepq_wait(&sx->lock_object, 0); 637 else 638 error = sleepq_wait_sig(&sx->lock_object, 0); 639#ifdef KDTRACE_HOOKS 640 sleep_time += lockstat_nsecs(); 641 sleep_cnt++; 642#endif 643 if (error) { 644 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 645 CTR2(KTR_LOCK, 646 "%s: interruptible sleep by %p suspended by signal", 647 __func__, sx); 648 break; 649 } 650 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 651 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 652 __func__, sx); 653 } 654 655 GIANT_RESTORE(); 656 if (!error) 657 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx, 658 contested, waittime, file, line); 659#ifdef KDTRACE_HOOKS 660 if (sleep_time) 661 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time); 662 if (spin_cnt > sleep_cnt) 663 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt)); 664#endif 665 return (error); 666} 667 668/* 669 * This function represents the so-called 'hard case' for sx_xunlock 670 * operation. All 'easy case' failures are redirected to this. Note 671 * that ideally this would be a static function, but it needs to be 672 * accessible from at least sx.h. 673 */ 674void 675_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 676{ 677 uintptr_t x; 678 int queue, wakeup_swapper; 679 680 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 681 682 /* If the lock is recursed, then unrecurse one level. */ 683 if (sx_xlocked(sx) && sx_recursed(sx)) { 684 if ((--sx->sx_recurse) == 0) 685 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 686 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 687 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); 688 return; 689 } 690 MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | 691 SX_LOCK_EXCLUSIVE_WAITERS)); 692 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 693 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); 694 695 sleepq_lock(&sx->lock_object); 696 x = SX_LOCK_UNLOCKED; 697 698 /* 699 * The wake up algorithm here is quite simple and probably not 700 * ideal. It gives precedence to shared waiters if they are 701 * present. For this condition, we have to preserve the 702 * state of the exclusive waiters flag. 703 */ 704 if (sx->sx_lock & SX_LOCK_SHARED_WAITERS) { 705 queue = SQ_SHARED_QUEUE; 706 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); 707 } else 708 queue = SQ_EXCLUSIVE_QUEUE; 709 710 /* Wake up all the waiters for the specific queue. */ 711 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 712 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", 713 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : 714 "exclusive"); 715 atomic_store_rel_ptr(&sx->sx_lock, x); 716 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, 717 queue); 718 sleepq_release(&sx->lock_object); 719 if (wakeup_swapper) 720 kick_proc0(); 721} 722 723/* 724 * This function represents the so-called 'hard case' for sx_slock 725 * operation. All 'easy case' failures are redirected to this. Note 726 * that ideally this would be a static function, but it needs to be 727 * accessible from at least sx.h. 728 */ 729int 730_sx_slock_hard(struct sx *sx, int opts, const char *file, int line) 731{ 732 GIANT_DECLARE; 733#ifdef ADAPTIVE_SX 734 volatile struct thread *owner; 735#endif 736#ifdef LOCK_PROFILING 737 uint64_t waittime = 0; 738 int contested = 0; 739#endif 740 uintptr_t x; 741 int error = 0; 742#ifdef KDTRACE_HOOKS 743 uint64_t spin_cnt = 0; 744 uint64_t sleep_cnt = 0; 745 int64_t sleep_time = 0; 746#endif 747 748 /* 749 * As with rwlocks, we don't make any attempt to try to block 750 * shared locks once there is an exclusive waiter. 751 */ 752 for (;;) { 753#ifdef KDTRACE_HOOKS 754 spin_cnt++; 755#endif 756 x = sx->sx_lock; 757 758 /* 759 * If no other thread has an exclusive lock then try to bump up 760 * the count of sharers. Since we have to preserve the state 761 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the 762 * shared lock loop back and retry. 763 */ 764 if (x & SX_LOCK_SHARED) { 765 MPASS(!(x & SX_LOCK_SHARED_WAITERS)); 766 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, 767 x + SX_ONE_SHARER)) { 768 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 769 CTR4(KTR_LOCK, 770 "%s: %p succeed %p -> %p", __func__, 771 sx, (void *)x, 772 (void *)(x + SX_ONE_SHARER)); 773 break; 774 } 775 continue; 776 } 777 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 778 &waittime); 779 780#ifdef ADAPTIVE_SX 781 /* 782 * If the owner is running on another CPU, spin until 783 * the owner stops running or the state of the lock 784 * changes. 785 */ 786 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 787 x = SX_OWNER(x); 788 owner = (struct thread *)x; 789 if (TD_IS_RUNNING(owner)) { 790 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 791 CTR3(KTR_LOCK, 792 "%s: spinning on %p held by %p", 793 __func__, sx, owner); 794 GIANT_SAVE(); 795 while (SX_OWNER(sx->sx_lock) == x && 796 TD_IS_RUNNING(owner)) { 797#ifdef KDTRACE_HOOKS 798 spin_cnt++; 799#endif 800 cpu_spinwait(); 801 } 802 continue; 803 } 804 } 805#endif 806 807 /* 808 * Some other thread already has an exclusive lock, so 809 * start the process of blocking. 810 */ 811 sleepq_lock(&sx->lock_object); 812 x = sx->sx_lock; 813 814 /* 815 * The lock could have been released while we spun. 816 * In this case loop back and retry. 817 */ 818 if (x & SX_LOCK_SHARED) { 819 sleepq_release(&sx->lock_object); 820 continue; 821 } 822 823#ifdef ADAPTIVE_SX 824 /* 825 * If the owner is running on another CPU, spin until 826 * the owner stops running or the state of the lock 827 * changes. 828 */ 829 if (!(x & SX_LOCK_SHARED) && 830 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 831 owner = (struct thread *)SX_OWNER(x); 832 if (TD_IS_RUNNING(owner)) { 833 sleepq_release(&sx->lock_object); 834 continue; 835 } 836 } 837#endif 838 839 /* 840 * Try to set the SX_LOCK_SHARED_WAITERS flag. If we 841 * fail to set it drop the sleep queue lock and loop 842 * back. 843 */ 844 if (!(x & SX_LOCK_SHARED_WAITERS)) { 845 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 846 x | SX_LOCK_SHARED_WAITERS)) { 847 sleepq_release(&sx->lock_object); 848 continue; 849 } 850 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 851 CTR2(KTR_LOCK, "%s: %p set shared waiters flag", 852 __func__, sx); 853 } 854 855 /* 856 * Since we have been unable to acquire the shared lock, 857 * we have to sleep. 858 */ 859 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 860 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 861 __func__, sx); 862 863#ifdef KDTRACE_HOOKS 864 sleep_time -= lockstat_nsecs(); 865#endif 866 GIANT_SAVE(); 867 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 868 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 869 SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE); 870 if (!(opts & SX_INTERRUPTIBLE)) 871 sleepq_wait(&sx->lock_object, 0); 872 else 873 error = sleepq_wait_sig(&sx->lock_object, 0); 874#ifdef KDTRACE_HOOKS 875 sleep_time += lockstat_nsecs(); 876 sleep_cnt++; 877#endif 878 if (error) { 879 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 880 CTR2(KTR_LOCK, 881 "%s: interruptible sleep by %p suspended by signal", 882 __func__, sx); 883 break; 884 } 885 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 886 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 887 __func__, sx); 888 } 889 if (error == 0) 890 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 891 contested, waittime, file, line); 892#ifdef KDTRACE_HOOKS 893 if (sleep_time) 894 LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time); 895 if (spin_cnt > sleep_cnt) 896 LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt)); 897#endif 898 GIANT_RESTORE(); 899 return (error); 900} 901 902/* 903 * This function represents the so-called 'hard case' for sx_sunlock 904 * operation. All 'easy case' failures are redirected to this. Note 905 * that ideally this would be a static function, but it needs to be 906 * accessible from at least sx.h. 907 */ 908void 909_sx_sunlock_hard(struct sx *sx, const char *file, int line) 910{ 911 uintptr_t x; 912 int wakeup_swapper; 913 914 for (;;) { 915 x = sx->sx_lock; 916 917 /* 918 * We should never have sharers while at least one thread 919 * holds a shared lock. 920 */ 921 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), 922 ("%s: waiting sharers", __func__)); 923 924 /* 925 * See if there is more than one shared lock held. If 926 * so, just drop one and return. 927 */ 928 if (SX_SHARERS(x) > 1) { 929 if (atomic_cmpset_ptr(&sx->sx_lock, x, 930 x - SX_ONE_SHARER)) { 931 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 932 CTR4(KTR_LOCK, 933 "%s: %p succeeded %p -> %p", 934 __func__, sx, (void *)x, 935 (void *)(x - SX_ONE_SHARER)); 936 break; 937 } 938 continue; 939 } 940 941 /* 942 * If there aren't any waiters for an exclusive lock, 943 * then try to drop it quickly. 944 */ 945 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 946 MPASS(x == SX_SHARERS_LOCK(1)); 947 if (atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1), 948 SX_LOCK_UNLOCKED)) { 949 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 950 CTR2(KTR_LOCK, "%s: %p last succeeded", 951 __func__, sx); 952 break; 953 } 954 continue; 955 } 956 957 /* 958 * At this point, there should just be one sharer with 959 * exclusive waiters. 960 */ 961 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS)); 962 963 sleepq_lock(&sx->lock_object); 964 965 /* 966 * Wake up semantic here is quite simple: 967 * Just wake up all the exclusive waiters. 968 * Note that the state of the lock could have changed, 969 * so if it fails loop back and retry. 970 */ 971 if (!atomic_cmpset_ptr(&sx->sx_lock, 972 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS, 973 SX_LOCK_UNLOCKED)) { 974 sleepq_release(&sx->lock_object); 975 continue; 976 } 977 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 978 CTR2(KTR_LOCK, "%s: %p waking up all thread on" 979 "exclusive queue", __func__, sx); 980 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 981 0, SQ_EXCLUSIVE_QUEUE); 982 sleepq_release(&sx->lock_object); 983 if (wakeup_swapper) 984 kick_proc0(); 985 break; 986 } 987} 988 989#ifdef INVARIANT_SUPPORT 990#ifndef INVARIANTS 991#undef _sx_assert 992#endif 993 994/* 995 * In the non-WITNESS case, sx_assert() can only detect that at least 996 * *some* thread owns an slock, but it cannot guarantee that *this* 997 * thread owns an slock. 998 */ 999void 1000_sx_assert(struct sx *sx, int what, const char *file, int line) 1001{ 1002#ifndef WITNESS 1003 int slocked = 0; 1004#endif 1005 1006 if (panicstr != NULL) 1007 return; 1008 switch (what) { 1009 case SA_SLOCKED: 1010 case SA_SLOCKED | SA_NOTRECURSED: 1011 case SA_SLOCKED | SA_RECURSED: 1012#ifndef WITNESS 1013 slocked = 1; 1014 /* FALLTHROUGH */ 1015#endif 1016 case SA_LOCKED: 1017 case SA_LOCKED | SA_NOTRECURSED: 1018 case SA_LOCKED | SA_RECURSED: 1019#ifdef WITNESS 1020 witness_assert(&sx->lock_object, what, file, line); 1021#else 1022 /* 1023 * If some other thread has an exclusive lock or we 1024 * have one and are asserting a shared lock, fail. 1025 * Also, if no one has a lock at all, fail. 1026 */ 1027 if (sx->sx_lock == SX_LOCK_UNLOCKED || 1028 (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked || 1029 sx_xholder(sx) != curthread))) 1030 panic("Lock %s not %slocked @ %s:%d\n", 1031 sx->lock_object.lo_name, slocked ? "share " : "", 1032 file, line); 1033 1034 if (!(sx->sx_lock & SX_LOCK_SHARED)) { 1035 if (sx_recursed(sx)) { 1036 if (what & SA_NOTRECURSED) 1037 panic("Lock %s recursed @ %s:%d\n", 1038 sx->lock_object.lo_name, file, 1039 line); 1040 } else if (what & SA_RECURSED) 1041 panic("Lock %s not recursed @ %s:%d\n", 1042 sx->lock_object.lo_name, file, line); 1043 } 1044#endif 1045 break; 1046 case SA_XLOCKED: 1047 case SA_XLOCKED | SA_NOTRECURSED: 1048 case SA_XLOCKED | SA_RECURSED: 1049 if (sx_xholder(sx) != curthread) 1050 panic("Lock %s not exclusively locked @ %s:%d\n", 1051 sx->lock_object.lo_name, file, line); 1052 if (sx_recursed(sx)) { 1053 if (what & SA_NOTRECURSED) 1054 panic("Lock %s recursed @ %s:%d\n", 1055 sx->lock_object.lo_name, file, line); 1056 } else if (what & SA_RECURSED) 1057 panic("Lock %s not recursed @ %s:%d\n", 1058 sx->lock_object.lo_name, file, line); 1059 break; 1060 case SA_UNLOCKED: 1061#ifdef WITNESS 1062 witness_assert(&sx->lock_object, what, file, line); 1063#else 1064 /* 1065 * If we hold an exclusve lock fail. We can't 1066 * reliably check to see if we hold a shared lock or 1067 * not. 1068 */ 1069 if (sx_xholder(sx) == curthread) 1070 panic("Lock %s exclusively locked @ %s:%d\n", 1071 sx->lock_object.lo_name, file, line); 1072#endif 1073 break; 1074 default: 1075 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 1076 line); 1077 } 1078} 1079#endif /* INVARIANT_SUPPORT */ 1080 1081#ifdef DDB 1082static void 1083db_show_sx(struct lock_object *lock) 1084{ 1085 struct thread *td; 1086 struct sx *sx; 1087 1088 sx = (struct sx *)lock; 1089 1090 db_printf(" state: "); 1091 if (sx->sx_lock == SX_LOCK_UNLOCKED) 1092 db_printf("UNLOCKED\n"); 1093 else if (sx->sx_lock == SX_LOCK_DESTROYED) { 1094 db_printf("DESTROYED\n"); 1095 return; 1096 } else if (sx->sx_lock & SX_LOCK_SHARED) 1097 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock)); 1098 else { 1099 td = sx_xholder(sx); 1100 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1101 td->td_tid, td->td_proc->p_pid, td->td_name); 1102 if (sx_recursed(sx)) 1103 db_printf(" recursed: %d\n", sx->sx_recurse); 1104 } 1105 1106 db_printf(" waiters: "); 1107 switch(sx->sx_lock & 1108 (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) { 1109 case SX_LOCK_SHARED_WAITERS: 1110 db_printf("shared\n"); 1111 break; 1112 case SX_LOCK_EXCLUSIVE_WAITERS: 1113 db_printf("exclusive\n"); 1114 break; 1115 case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS: 1116 db_printf("exclusive and shared\n"); 1117 break; 1118 default: 1119 db_printf("none\n"); 1120 } 1121} 1122 1123/* 1124 * Check to see if a thread that is blocked on a sleep queue is actually 1125 * blocked on an sx lock. If so, output some details and return true. 1126 * If the lock has an exclusive owner, return that in *ownerp. 1127 */ 1128int 1129sx_chain(struct thread *td, struct thread **ownerp) 1130{ 1131 struct sx *sx; 1132 1133 /* 1134 * Check to see if this thread is blocked on an sx lock. 1135 * First, we check the lock class. If that is ok, then we 1136 * compare the lock name against the wait message. 1137 */ 1138 sx = td->td_wchan; 1139 if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx || 1140 sx->lock_object.lo_name != td->td_wmesg) 1141 return (0); 1142 1143 /* We think we have an sx lock, so output some details. */ 1144 db_printf("blocked on sx \"%s\" ", td->td_wmesg); 1145 *ownerp = sx_xholder(sx); 1146 if (sx->sx_lock & SX_LOCK_SHARED) 1147 db_printf("SLOCK (count %ju)\n", 1148 (uintmax_t)SX_SHARERS(sx->sx_lock)); 1149 else 1150 db_printf("XLOCK\n"); 1151 return (1); 1152} 1153#endif 1154