kern_sx.c revision 168330
1113828Simp/*- 2113828Simp * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3113828Simp * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4113828Simp * All rights reserved. 5113828Simp * 6113828Simp * Redistribution and use in source and binary forms, with or without 7113828Simp * modification, are permitted provided that the following conditions 8113828Simp * are met: 9113828Simp * 1. Redistributions of source code must retain the above copyright 10113828Simp * notice(s), this list of conditions and the following disclaimer as 11113828Simp * the first lines of this file unmodified other than the possible 12113828Simp * addition of one or more copyright notices. 13113828Simp * 2. Redistributions in binary form must reproduce the above copyright 14113828Simp * notice(s), this list of conditions and the following disclaimer in the 15113828Simp * documentation and/or other materials provided with the distribution. 16113828Simp * 17113828Simp * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18113828Simp * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19113828Simp * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20113828Simp * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21113828Simp * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22113828Simp * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23113828Simp * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24113828Simp * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25113828Simp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26113828Simp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27113828Simp * DAMAGE. 28113828Simp */ 29113828Simp 30113828Simp/* 31113828Simp * Shared/exclusive locks. This implementation attempts to ensure 32113828Simp * deterministic lock granting behavior, so that slocks and xlocks are 33113828Simp * interleaved. 34113828Simp * 35115418Sru * Priority propagation will not generally raise the priority of lock holders, 36115418Sru * so should not be relied upon in combination with sx locks. 37113828Simp */ 38113828Simp 39113828Simp#include "opt_adaptive_sx.h" 40122173Sdes#include "opt_ddb.h" 41113828Simp 42122173Sdes#include <sys/cdefs.h> 43113828Simp__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 168330 2007-04-03 22:58:21Z kmacy $"); 44113828Simp 45115418Sru#include <sys/param.h> 46113828Simp#include <sys/ktr.h> 47115418Sru#include <sys/lock.h> 48113828Simp#include <sys/lock_profile.h> 49115418Sru#include <sys/mutex.h> 50113828Simp#include <sys/proc.h> 51115418Sru#include <sys/sleepqueue.h> 52113828Simp#include <sys/sx.h> 53113828Simp#include <sys/systm.h> 54113828Simp 55121414Shmp#ifdef ADAPTIVE_SX 56113828Simp#include <machine/cpu.h> 57#endif 58 59#ifdef DDB 60#include <ddb/ddb.h> 61#endif 62 63#if !defined(SMP) && defined(ADAPTIVE_SX) 64#error "You must have SMP to enable the ADAPTIVE_SX option" 65#endif 66 67/* Handy macros for sleep queues. */ 68#define SQ_EXCLUSIVE_QUEUE 0 69#define SQ_SHARED_QUEUE 1 70 71/* 72 * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We 73 * drop Giant anytime we have to sleep or if we adaptively spin. 74 */ 75#define GIANT_DECLARE \ 76 int _giantcnt = 0; \ 77 WITNESS_SAVE_DECL(Giant) \ 78 79#define GIANT_SAVE() do { \ 80 if (mtx_owned(&Giant)) { \ 81 WITNESS_SAVE(&Giant.lock_object, Giant); \ 82 while (mtx_owned(&Giant)) { \ 83 _giantcnt++; \ 84 mtx_unlock(&Giant); \ 85 } \ 86 } \ 87} while (0) 88 89#define GIANT_RESTORE() do { \ 90 if (_giantcnt > 0) { \ 91 mtx_assert(&Giant, MA_NOTOWNED); \ 92 while (_giantcnt--) \ 93 mtx_lock(&Giant); \ 94 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 95 } \ 96} while (0) 97 98/* 99 * Returns true if an exclusive lock is recursed. It curthread 100 * currently has an exclusive lock. 101 */ 102#define sx_recursed(sx) ((sx)->sx_recurse != 0) 103 104/* 105 * Return a pointer to the owning thread if the lock is exclusively 106 * locked. 107 */ 108#define sx_xholder(sx) \ 109 ((sx)->sx_lock & SX_LOCK_SHARED ? NULL : \ 110 (struct thread *)SX_OWNER((sx)->sx_lock)) 111 112#ifdef DDB 113static void db_show_sx(struct lock_object *lock); 114#endif 115static void lock_sx(struct lock_object *lock, int how); 116static int unlock_sx(struct lock_object *lock); 117 118struct lock_class lock_class_sx = { 119 .lc_name = "sx", 120 .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 121#ifdef DDB 122 .lc_ddb_show = db_show_sx, 123#endif 124 .lc_lock = lock_sx, 125 .lc_unlock = unlock_sx, 126}; 127 128#ifndef INVARIANTS 129#define _sx_assert(sx, what, file, line) 130#endif 131 132void 133lock_sx(struct lock_object *lock, int how) 134{ 135 struct sx *sx; 136 137 sx = (struct sx *)lock; 138 if (how) 139 sx_xlock(sx); 140 else 141 sx_slock(sx); 142} 143 144int 145unlock_sx(struct lock_object *lock) 146{ 147 struct sx *sx; 148 149 sx = (struct sx *)lock; 150 sx_assert(sx, SX_LOCKED | SX_NOTRECURSED); 151 if (sx_xlocked(sx)) { 152 sx_xunlock(sx); 153 return (1); 154 } else { 155 sx_sunlock(sx); 156 return (0); 157 } 158} 159 160void 161sx_sysinit(void *arg) 162{ 163 struct sx_args *sargs = arg; 164 165 sx_init(sargs->sa_sx, sargs->sa_desc); 166} 167 168void 169sx_init_flags(struct sx *sx, const char *description, int opts) 170{ 171 int flags; 172 173 flags = LO_SLEEPABLE | LO_UPGRADABLE | LO_RECURSABLE;; 174 175 if (opts & SX_DUPOK) 176 flags |= LO_DUPOK; 177 if (opts & SX_NOPROFILE) 178 flags |= LO_NOPROFILE; 179 if (!(opts & SX_NOWITNESS)) 180 flags |= LO_WITNESS; 181 if (opts & SX_QUIET) 182 flags |= LO_QUIET; 183 184 flags |= opts & SX_ADAPTIVESPIN; 185 sx->sx_lock = SX_LOCK_UNLOCKED; 186 sx->sx_recurse = 0; 187 lock_profile_object_init(&sx->lock_object, &lock_class_sx, description); 188 lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags); 189} 190 191void 192sx_destroy(struct sx *sx) 193{ 194 195 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held")); 196 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed")); 197 lock_profile_object_destroy(&sx->lock_object); 198 lock_destroy(&sx->lock_object); 199} 200 201void 202_sx_slock(struct sx *sx, const char *file, int line) 203{ 204 205 MPASS(curthread != NULL); 206 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line); 207 __sx_slock(sx, file, line); 208 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 209 WITNESS_LOCK(&sx->lock_object, 0, file, line); 210 curthread->td_locks++; 211} 212 213int 214_sx_try_slock(struct sx *sx, const char *file, int line) 215{ 216 uintptr_t x; 217 218 x = sx->sx_lock; 219 if ((x & SX_LOCK_SHARED) && atomic_cmpset_acq_ptr(&sx->sx_lock, x, 220 x + SX_ONE_SHARER)) { 221 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 222 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); 223 curthread->td_locks++; 224 return (1); 225 } 226 227 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 228 return (0); 229} 230 231void 232_sx_xlock(struct sx *sx, const char *file, int line) 233{ 234 235 MPASS(curthread != NULL); 236 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 237 line); 238 __sx_xlock(sx, curthread, file, line); 239 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, file, line); 240 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 241 curthread->td_locks++; 242} 243 244int 245_sx_try_xlock(struct sx *sx, const char *file, int line) 246{ 247 int rval; 248 249 MPASS(curthread != NULL); 250 251 if (sx_xlocked(sx)) { 252 sx->sx_recurse++; 253 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 254 rval = 1; 255 } else 256 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 257 (uintptr_t)curthread); 258 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 259 if (rval) { 260 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 261 file, line); 262 curthread->td_locks++; 263 } 264 265 return (rval); 266} 267 268void 269_sx_sunlock(struct sx *sx, const char *file, int line) 270{ 271 272 MPASS(curthread != NULL); 273 _sx_assert(sx, SX_SLOCKED, file, line); 274 curthread->td_locks--; 275 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 276 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 277 __sx_sunlock(sx, file, line); 278} 279 280void 281_sx_xunlock(struct sx *sx, const char *file, int line) 282{ 283 284 MPASS(curthread != NULL); 285 _sx_assert(sx, SX_XLOCKED, file, line); 286 curthread->td_locks--; 287 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 288 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 289 line); 290 if (!sx_recursed(sx)) 291 lock_profile_release_lock(&sx->lock_object); 292 __sx_xunlock(sx, curthread, file, line); 293} 294 295/* 296 * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. 297 * This will only succeed if this thread holds a single shared lock. 298 * Return 1 if if the upgrade succeed, 0 otherwise. 299 */ 300int 301_sx_try_upgrade(struct sx *sx, const char *file, int line) 302{ 303 uintptr_t x; 304 int success; 305 306 _sx_assert(sx, SX_SLOCKED, file, line); 307 308 /* 309 * Try to switch from one shared lock to an exclusive lock. We need 310 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 311 * we will wake up the exclusive waiters when we drop the lock. 312 */ 313 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; 314 success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, 315 (uintptr_t)curthread | x); 316 LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); 317 if (success) 318 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 319 file, line); 320 return (success); 321} 322 323/* 324 * Downgrade an unrecursed exclusive lock into a single shared lock. 325 */ 326void 327_sx_downgrade(struct sx *sx, const char *file, int line) 328{ 329 uintptr_t x; 330 331 _sx_assert(sx, SX_XLOCKED | SX_NOTRECURSED, file, line); 332#ifndef INVARIANTS 333 if (sx_recursed(sx)) 334 panic("downgrade of a recursed lock"); 335#endif 336 337 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); 338 339 /* 340 * Try to switch from an exclusive lock with no shared waiters 341 * to one sharer with no shared waiters. If there are 342 * exclusive waiters, we don't need to lock the sleep queue so 343 * long as we preserve the flag. We do one quick try and if 344 * that fails we grab the sleepq lock to keep the flags from 345 * changing and do it the slow way. 346 * 347 * We have to lock the sleep queue if there are shared waiters 348 * so we can wake them up. 349 */ 350 x = sx->sx_lock; 351 if (!(x & SX_LOCK_SHARED_WAITERS) && 352 atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | 353 (x & SX_LOCK_EXCLUSIVE_WAITERS))) { 354 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 355 return; 356 } 357 358 /* 359 * Lock the sleep queue so we can read the waiters bits 360 * without any races and wakeup any shared waiters. 361 */ 362 sleepq_lock(&sx->lock_object); 363 364 /* 365 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single 366 * shared lock. If there are any shared waiters, wake them up. 367 */ 368 x = sx->sx_lock; 369 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | 370 (x & SX_LOCK_EXCLUSIVE_WAITERS)); 371 if (x & SX_LOCK_SHARED_WAITERS) 372 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, 373 SQ_SHARED_QUEUE); 374 else 375 sleepq_release(&sx->lock_object); 376 377 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 378} 379 380/* 381 * This function represents the so-called 'hard case' for sx_xlock 382 * operation. All 'easy case' failures are redirected to this. Note 383 * that ideally this would be a static function, but it needs to be 384 * accessible from at least sx.h. 385 */ 386void 387_sx_xlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 388{ 389 GIANT_DECLARE; 390#ifdef ADAPTIVE_SX 391 volatile struct thread *owner; 392#endif 393 uintptr_t x; 394 int contested = 0; 395 uint64_t waitstart = 0; 396 397 /* If we already hold an exclusive lock, then recurse. */ 398 if (sx_xlocked(sx)) { 399 sx->sx_recurse++; 400 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 401 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 402 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 403 return; 404 } 405 lock_profile_obtain_lock_failed(&(sx)->lock_object, 406 &contested, &waitstart); 407 408 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 409 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 410 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 411 412 while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { 413#ifdef ADAPTIVE_SX 414 /* 415 * If the lock is write locked and the owner is 416 * running on another CPU, spin until the owner stops 417 * running or the state of the lock changes. 418 */ 419 x = sx->sx_lock; 420 if (!(x & SX_LOCK_SHARED) && 421 (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) { 422 x = SX_OWNER(x); 423 owner = (struct thread *)x; 424 if (TD_IS_RUNNING(owner)) { 425 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 426 CTR3(KTR_LOCK, 427 "%s: spinning on %p held by %p", 428 __func__, sx, owner); 429 GIANT_SAVE(); 430 while (SX_OWNER(sx->sx_lock) == x && 431 TD_IS_RUNNING(owner)) 432 cpu_spinwait(); 433 continue; 434 } 435 } 436#endif 437 438 sleepq_lock(&sx->lock_object); 439 x = sx->sx_lock; 440 441 /* 442 * If the lock was released while spinning on the 443 * sleep queue chain lock, try again. 444 */ 445 if (x == SX_LOCK_UNLOCKED) { 446 sleepq_release(&sx->lock_object); 447 continue; 448 } 449 450#ifdef ADAPTIVE_SX 451 /* 452 * The current lock owner might have started executing 453 * on another CPU (or the lock could have changed 454 * owners) while we were waiting on the sleep queue 455 * chain lock. If so, drop the sleep queue lock and try 456 * again. 457 */ 458 if (!(x & SX_LOCK_SHARED) && 459 (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) { 460 owner = (struct thread *)SX_OWNER(x); 461 if (TD_IS_RUNNING(owner)) { 462 sleepq_release(&sx->lock_object); 463 continue; 464 } 465 } 466#endif 467 468 /* 469 * If an exclusive lock was released with both shared 470 * and exclusive waiters and a shared waiter hasn't 471 * woken up and acquired the lock yet, sx_lock will be 472 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS. 473 * If we see that value, try to acquire it once. Note 474 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS 475 * as there are other exclusive waiters still. If we 476 * fail, restart the loop. 477 */ 478 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) { 479 if (atomic_cmpset_acq_ptr(&sx->sx_lock, 480 SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS, 481 tid | SX_LOCK_EXCLUSIVE_WAITERS)) { 482 sleepq_release(&sx->lock_object); 483 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 484 __func__, sx); 485 break; 486 } 487 sleepq_release(&sx->lock_object); 488 continue; 489 } 490 491 /* 492 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail, 493 * than loop back and retry. 494 */ 495 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 496 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 497 x | SX_LOCK_EXCLUSIVE_WAITERS)) { 498 sleepq_release(&sx->lock_object); 499 continue; 500 } 501 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 502 CTR2(KTR_LOCK, "%s: %p set excl waiters flag", 503 __func__, sx); 504 } 505 506 /* 507 * Since we have been unable to acquire the exclusive 508 * lock and the exclusive waiters flag is set, we have 509 * to sleep. 510 */ 511 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 512 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 513 __func__, sx); 514 515 GIANT_SAVE(); 516 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 517 SLEEPQ_SX, SQ_EXCLUSIVE_QUEUE); 518 sleepq_wait(&sx->lock_object); 519 520 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 521 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 522 __func__, sx); 523 } 524 525 GIANT_RESTORE(); 526 lock_profile_obtain_lock_success(&(sx)->lock_object, contested, 527 waitstart, (file), (line)); 528} 529 530/* 531 * This function represents the so-called 'hard case' for sx_xunlock 532 * operation. All 'easy case' failures are redirected to this. Note 533 * that ideally this would be a static function, but it needs to be 534 * accessible from at least sx.h. 535 */ 536void 537_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 538{ 539 uintptr_t x; 540 int queue; 541 542 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 543 544 /* If the lock is recursed, then unrecurse one level. */ 545 if (sx_xlocked(sx) && sx_recursed(sx)) { 546 if ((--sx->sx_recurse) == 0) 547 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 548 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 549 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); 550 return; 551 } 552 MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | 553 SX_LOCK_EXCLUSIVE_WAITERS)); 554 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 555 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); 556 557 sleepq_lock(&sx->lock_object); 558 x = SX_LOCK_UNLOCKED; 559 560 /* 561 * The wake up algorithm here is quite simple and probably not 562 * ideal. It gives precedence to shared waiters if they are 563 * present. For this condition, we have to preserve the 564 * state of the exclusive waiters flag. 565 */ 566 if (sx->sx_lock & SX_LOCK_SHARED_WAITERS) { 567 queue = SQ_SHARED_QUEUE; 568 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); 569 } else 570 queue = SQ_EXCLUSIVE_QUEUE; 571 572 /* Wake up all the waiters for the specific queue. */ 573 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 574 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", 575 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : 576 "exclusive"); 577 atomic_store_rel_ptr(&sx->sx_lock, x); 578 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, queue); 579} 580 581/* 582 * This function represents the so-called 'hard case' for sx_slock 583 * operation. All 'easy case' failures are redirected to this. Note 584 * that ideally this would be a static function, but it needs to be 585 * accessible from at least sx.h. 586 */ 587void 588_sx_slock_hard(struct sx *sx, const char *file, int line) 589{ 590 GIANT_DECLARE; 591#ifdef ADAPTIVE_SX 592 volatile struct thread *owner; 593#endif 594 uintptr_t x; 595 uint64_t waitstart = 0; 596 int contested = 0; 597 /* 598 * As with rwlocks, we don't make any attempt to try to block 599 * shared locks once there is an exclusive waiter. 600 */ 601 602 for (;;) { 603 x = sx->sx_lock; 604 605 /* 606 * If no other thread has an exclusive lock then try to bump up 607 * the count of sharers. Since we have to preserve the state 608 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the 609 * shared lock loop back and retry. 610 */ 611 if (x & SX_LOCK_SHARED) { 612 MPASS(!(x & SX_LOCK_SHARED_WAITERS)); 613 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, 614 x + SX_ONE_SHARER)) { 615 if (SX_SHARERS(x) == 0) 616 lock_profile_obtain_lock_success( 617 &sx->lock_object, contested, 618 waitstart, file, line); 619 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 620 CTR4(KTR_LOCK, 621 "%s: %p succeed %p -> %p", __func__, 622 sx, (void *)x, 623 (void *)(x + SX_ONE_SHARER)); 624 break; 625 } 626 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 627 &waitstart); 628 629 continue; 630 } 631 632#ifdef ADAPTIVE_SX 633 /* 634 * If the owner is running on another CPU, spin until 635 * the owner stops running or the state of the lock 636 * changes. 637 */ 638 else if (sx->lock_object.lo_flags & SX_ADAPTIVESPIN) { 639 x = SX_OWNER(x); 640 owner = (struct thread *)x; 641 if (TD_IS_RUNNING(owner)) { 642 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 643 &waitstart); 644 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 645 CTR3(KTR_LOCK, 646 "%s: spinning on %p held by %p", 647 __func__, sx, owner); 648 GIANT_SAVE(); 649 while (SX_OWNER(sx->sx_lock) == x && 650 TD_IS_RUNNING(owner)) 651 cpu_spinwait(); 652 continue; 653 } 654 } 655#endif 656 else 657 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 658 &waitstart); 659 660 /* 661 * Some other thread already has an exclusive lock, so 662 * start the process of blocking. 663 */ 664 sleepq_lock(&sx->lock_object); 665 x = sx->sx_lock; 666 667 /* 668 * The lock could have been released while we spun. 669 * In this case loop back and retry. 670 */ 671 if (x & SX_LOCK_SHARED) { 672 sleepq_release(&sx->lock_object); 673 continue; 674 } 675 676#ifdef ADAPTIVE_SX 677 /* 678 * If the owner is running on another CPU, spin until 679 * the owner stops running or the state of the lock 680 * changes. 681 */ 682 if (!(x & SX_LOCK_SHARED) && 683 (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) { 684 owner = (struct thread *)SX_OWNER(x); 685 if (TD_IS_RUNNING(owner)) { 686 sleepq_release(&sx->lock_object); 687 continue; 688 } 689 } 690#endif 691 692 /* 693 * Try to set the SX_LOCK_SHARED_WAITERS flag. If we 694 * fail to set it drop the sleep queue lock and loop 695 * back. 696 */ 697 if (!(x & SX_LOCK_SHARED_WAITERS)) { 698 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 699 x | SX_LOCK_SHARED_WAITERS)) { 700 sleepq_release(&sx->lock_object); 701 continue; 702 } 703 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 704 CTR2(KTR_LOCK, "%s: %p set shared waiters flag", 705 __func__, sx); 706 } 707 708 /* 709 * Since we have been unable to acquire the shared lock, 710 * we have to sleep. 711 */ 712 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 713 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 714 __func__, sx); 715 716 GIANT_SAVE(); 717 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 718 SLEEPQ_SX, SQ_SHARED_QUEUE); 719 sleepq_wait(&sx->lock_object); 720 721 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 722 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 723 __func__, sx); 724 } 725 726 GIANT_RESTORE(); 727} 728 729/* 730 * This function represents the so-called 'hard case' for sx_sunlock 731 * operation. All 'easy case' failures are redirected to this. Note 732 * that ideally this would be a static function, but it needs to be 733 * accessible from at least sx.h. 734 */ 735void 736_sx_sunlock_hard(struct sx *sx, const char *file, int line) 737{ 738 uintptr_t x; 739 740 for (;;) { 741 x = sx->sx_lock; 742 743 /* 744 * We should never have sharers while at least one thread 745 * holds a shared lock. 746 */ 747 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), 748 ("%s: waiting sharers", __func__)); 749 750 /* 751 * See if there is more than one shared lock held. If 752 * so, just drop one and return. 753 */ 754 if (SX_SHARERS(x) > 1) { 755 if (atomic_cmpset_ptr(&sx->sx_lock, x, 756 x - SX_ONE_SHARER)) { 757 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 758 CTR4(KTR_LOCK, 759 "%s: %p succeeded %p -> %p", 760 __func__, sx, (void *)x, 761 (void *)(x - SX_ONE_SHARER)); 762 break; 763 } 764 continue; 765 } 766 767 /* 768 * If there aren't any waiters for an exclusive lock, 769 * then try to drop it quickly. 770 */ 771 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 772 MPASS(x == SX_SHARERS_LOCK(1)); 773 if (atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1), 774 SX_LOCK_UNLOCKED)) { 775 lock_profile_release_lock(&sx->lock_object); 776 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 777 CTR2(KTR_LOCK, "%s: %p last succeeded", 778 __func__, sx); 779 break; 780 } 781 continue; 782 } 783 784 /* 785 * At this point, there should just be one sharer with 786 * exclusive waiters. 787 */ 788 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS)); 789 790 lock_profile_release_lock(&sx->lock_object); 791 sleepq_lock(&sx->lock_object); 792 793 /* 794 * Wake up semantic here is quite simple: 795 * Just wake up all the exclusive waiters. 796 * Note that the state of the lock could have changed, 797 * so if it fails loop back and retry. 798 */ 799 if (!atomic_cmpset_ptr(&sx->sx_lock, 800 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS, 801 SX_LOCK_UNLOCKED)) { 802 sleepq_release(&sx->lock_object); 803 continue; 804 } 805 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 806 CTR2(KTR_LOCK, "%s: %p waking up all thread on" 807 "exclusive queue", __func__, sx); 808 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, -1, 809 SQ_EXCLUSIVE_QUEUE); 810 break; 811 } 812} 813 814#ifdef INVARIANT_SUPPORT 815#ifndef INVARIANTS 816#undef _sx_assert 817#endif 818 819/* 820 * In the non-WITNESS case, sx_assert() can only detect that at least 821 * *some* thread owns an slock, but it cannot guarantee that *this* 822 * thread owns an slock. 823 */ 824void 825_sx_assert(struct sx *sx, int what, const char *file, int line) 826{ 827#ifndef WITNESS 828 int slocked = 0; 829#endif 830 831 if (panicstr != NULL) 832 return; 833 switch (what) { 834 case SX_SLOCKED: 835 case SX_SLOCKED | SX_NOTRECURSED: 836 case SX_SLOCKED | SX_RECURSED: 837#ifndef WITNESS 838 slocked = 1; 839 /* FALLTHROUGH */ 840#endif 841 case SX_LOCKED: 842 case SX_LOCKED | SX_NOTRECURSED: 843 case SX_LOCKED | SX_RECURSED: 844#ifdef WITNESS 845 witness_assert(&sx->lock_object, what, file, line); 846#else 847 /* 848 * If some other thread has an exclusive lock or we 849 * have one and are asserting a shared lock, fail. 850 * Also, if no one has a lock at all, fail. 851 */ 852 if (sx->sx_lock == SX_LOCK_UNLOCKED || 853 (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked || 854 sx_xholder(sx) != curthread))) 855 panic("Lock %s not %slocked @ %s:%d\n", 856 sx->lock_object.lo_name, slocked ? "share " : "", 857 file, line); 858 859 if (!(sx->sx_lock & SX_LOCK_SHARED)) { 860 if (sx_recursed(sx)) { 861 if (what & SX_NOTRECURSED) 862 panic("Lock %s recursed @ %s:%d\n", 863 sx->lock_object.lo_name, file, 864 line); 865 } else if (what & SX_RECURSED) 866 panic("Lock %s not recursed @ %s:%d\n", 867 sx->lock_object.lo_name, file, line); 868 } 869#endif 870 break; 871 case SX_XLOCKED: 872 case SX_XLOCKED | SX_NOTRECURSED: 873 case SX_XLOCKED | SX_RECURSED: 874 if (sx_xholder(sx) != curthread) 875 panic("Lock %s not exclusively locked @ %s:%d\n", 876 sx->lock_object.lo_name, file, line); 877 if (sx_recursed(sx)) { 878 if (what & SX_NOTRECURSED) 879 panic("Lock %s recursed @ %s:%d\n", 880 sx->lock_object.lo_name, file, line); 881 } else if (what & SX_RECURSED) 882 panic("Lock %s not recursed @ %s:%d\n", 883 sx->lock_object.lo_name, file, line); 884 break; 885 case SX_UNLOCKED: 886#ifdef WITNESS 887 witness_assert(&sx->lock_object, what, file, line); 888#else 889 /* 890 * If we hold an exclusve lock fail. We can't 891 * reliably check to see if we hold a shared lock or 892 * not. 893 */ 894 if (sx_xholder(sx) == curthread) 895 panic("Lock %s exclusively locked @ %s:%d\n", 896 sx->lock_object.lo_name, file, line); 897#endif 898 break; 899 default: 900 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 901 line); 902 } 903} 904#endif /* INVARIANT_SUPPORT */ 905 906#ifdef DDB 907static void 908db_show_sx(struct lock_object *lock) 909{ 910 struct thread *td; 911 struct sx *sx; 912 913 sx = (struct sx *)lock; 914 915 db_printf(" state: "); 916 if (sx->sx_lock == SX_LOCK_UNLOCKED) 917 db_printf("UNLOCKED\n"); 918 else if (sx->sx_lock & SX_LOCK_SHARED) 919 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock)); 920 else { 921 td = sx_xholder(sx); 922 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 923 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 924 if (sx_recursed(sx)) 925 db_printf(" recursed: %d\n", sx->sx_recurse); 926 } 927 928 db_printf(" waiters: "); 929 switch(sx->sx_lock & 930 (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) { 931 case SX_LOCK_SHARED_WAITERS: 932 db_printf("shared\n"); 933 break; 934 case SX_LOCK_EXCLUSIVE_WAITERS: 935 db_printf("exclusive\n"); 936 break; 937 case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS: 938 db_printf("exclusive and shared\n"); 939 break; 940 default: 941 db_printf("none\n"); 942 } 943} 944 945/* 946 * Check to see if a thread that is blocked on a sleep queue is actually 947 * blocked on an sx lock. If so, output some details and return true. 948 * If the lock has an exclusive owner, return that in *ownerp. 949 */ 950int 951sx_chain(struct thread *td, struct thread **ownerp) 952{ 953 struct sx *sx; 954 955 /* 956 * Check to see if this thread is blocked on an sx lock. 957 * First, we check the lock class. If that is ok, then we 958 * compare the lock name against the wait message. 959 */ 960 sx = td->td_wchan; 961 if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx || 962 sx->lock_object.lo_name != td->td_wmesg) 963 return (0); 964 965 /* We think we have an sx lock, so output some details. */ 966 db_printf("blocked on sx \"%s\" ", td->td_wmesg); 967 *ownerp = sx_xholder(sx); 968 if (sx->sx_lock & SX_LOCK_SHARED) 969 db_printf("SLOCK (count %ju)\n", 970 (uintmax_t)SX_SHARERS(sx->sx_lock)); 971 else 972 db_printf("XLOCK\n"); 973 return (1); 974} 975#endif 976