kern_lock.c revision 177982
1/*- 2 * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29#include "opt_ddb.h" 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sys/kern/kern_lock.c 177982 2008-04-07 14:46:38Z attilio $"); 33 34#include <sys/param.h> 35#include <sys/ktr.h> 36#include <sys/lock.h> 37#include <sys/lock_profile.h> 38#include <sys/lockmgr.h> 39#include <sys/mutex.h> 40#include <sys/proc.h> 41#include <sys/sleepqueue.h> 42#ifdef DEBUG_LOCKS 43#include <sys/stack.h> 44#endif 45#include <sys/systm.h> 46 47#include <machine/cpu.h> 48 49#ifdef DDB 50#include <ddb/ddb.h> 51#endif 52 53CTASSERT(((LK_CANRECURSE | LK_NOSHARE) & LO_CLASSFLAGS) == 54 (LK_CANRECURSE | LK_NOSHARE)); 55 56#define SQ_EXCLUSIVE_QUEUE 0 57#define SQ_SHARED_QUEUE 1 58 59#ifndef INVARIANTS 60#define _lockmgr_assert(lk, what, file, line) 61#define TD_LOCKS_INC(td) 62#define TD_LOCKS_DEC(td) 63#else 64#define TD_LOCKS_INC(td) ((td)->td_locks++) 65#define TD_LOCKS_DEC(td) ((td)->td_locks--) 66#endif 67#define TD_SLOCKS_INC(td) ((td)->td_lk_slocks++) 68#define TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--) 69 70#ifndef DEBUG_LOCKS 71#define STACK_PRINT(lk) 72#define STACK_SAVE(lk) 73#define STACK_ZERO(lk) 74#else 75#define STACK_PRINT(lk) stack_print_ddb(&(lk)->lk_stack) 76#define STACK_SAVE(lk) stack_save(&(lk)->lk_stack) 77#define STACK_ZERO(lk) stack_zero(&(lk)->lk_stack) 78#endif 79 80#define LOCK_LOG2(lk, string, arg1, arg2) \ 81 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \ 82 CTR2(KTR_LOCK, (string), (arg1), (arg2)) 83#define LOCK_LOG3(lk, string, arg1, arg2, arg3) \ 84 if (LOCK_LOG_TEST(&(lk)->lock_object, 0)) \ 85 CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3)) 86 87#define LK_TRYOP(x) \ 88 ((x) & LK_NOWAIT) 89#define LK_CAN_SHARE(x) \ 90 (((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 || \ 91 curthread->td_lk_slocks || (curthread->td_pflags & TDP_DEADLKTREAT))) 92 93#define lockmgr_disowned(lk) \ 94 (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) 95 96#define lockmgr_xlocked(lk) \ 97 (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread) 98 99static void assert_lockmgr(struct lock_object *lock, int how); 100#ifdef DDB 101static void db_show_lockmgr(struct lock_object *lock); 102#endif 103static void lock_lockmgr(struct lock_object *lock, int how); 104static int unlock_lockmgr(struct lock_object *lock); 105 106struct lock_class lock_class_lockmgr = { 107 .lc_name = "lockmgr", 108 .lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE, 109 .lc_assert = assert_lockmgr, 110#ifdef DDB 111 .lc_ddb_show = db_show_lockmgr, 112#endif 113 .lc_lock = lock_lockmgr, 114 .lc_unlock = unlock_lockmgr 115}; 116 117static __inline struct thread * 118lockmgr_xholder(struct lock *lk) 119{ 120 uintptr_t x; 121 122 x = lk->lk_lock; 123 return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x)); 124} 125 126/* 127 * It assumes sleepq_lock held and returns with this one unheld. 128 * It also assumes the generic interlock is sane and previously checked. 129 * If LK_INTERLOCK is specified the interlock is not reacquired after the 130 * sleep. 131 */ 132static __inline int 133sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk, 134 const char *wmesg, int pri, int timo, int queue) 135{ 136 struct lock_class *class; 137 int catch, error; 138 139 class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; 140 catch = (pri) ? (pri & PCATCH) : 0; 141 pri &= PRIMASK; 142 error = 0; 143 144 LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk, 145 (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared"); 146 147 if (flags & LK_INTERLOCK) 148 class->lc_unlock(ilk); 149 DROP_GIANT(); 150 sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ? 151 SLEEPQ_INTERRUPTIBLE : 0), queue); 152 if ((flags & LK_TIMELOCK) && timo) 153 sleepq_set_timeout(&lk->lock_object, timo); 154 155 /* 156 * Decisional switch for real sleeping. 157 */ 158 if ((flags & LK_TIMELOCK) && timo && catch) 159 error = sleepq_timedwait_sig(&lk->lock_object, pri); 160 else if ((flags & LK_TIMELOCK) && timo) 161 error = sleepq_timedwait(&lk->lock_object, pri); 162 else if (catch) 163 error = sleepq_wait_sig(&lk->lock_object, pri); 164 else 165 sleepq_wait(&lk->lock_object, pri); 166 PICKUP_GIANT(); 167 if ((flags & LK_SLEEPFAIL) && error == 0) 168 error = ENOLCK; 169 170 return (error); 171} 172 173static __inline void 174wakeupshlk(struct lock *lk, const char *file, int line) 175{ 176 uintptr_t v, x; 177 int queue; 178 179 TD_LOCKS_DEC(curthread); 180 TD_SLOCKS_DEC(curthread); 181 LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line); 182 183 for (;;) { 184 x = lk->lk_lock; 185 186 /* 187 * If there is more than one shared lock held, just drop one 188 * and return. 189 */ 190 if (LK_SHARERS(x) > 1) { 191 if (atomic_cmpset_ptr(&lk->lk_lock, x, 192 x - LK_ONE_SHARER)) 193 break; 194 continue; 195 } 196 197 /* 198 * If there are not waiters on the exclusive queue, drop the 199 * lock quickly. 200 */ 201 if ((x & LK_ALL_WAITERS) == 0) { 202 MPASS(x == LK_SHARERS_LOCK(1)); 203 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1), 204 LK_UNLOCKED)) 205 break; 206 continue; 207 } 208 209 /* 210 * We should have a sharer with waiters, so enter the hard 211 * path in order to handle wakeups correctly. 212 */ 213 sleepq_lock(&lk->lock_object); 214 x = lk->lk_lock & LK_ALL_WAITERS; 215 v = LK_UNLOCKED; 216 217 /* 218 * If the lock has exclusive waiters, give them preference in 219 * order to avoid deadlock with shared runners up. 220 */ 221 if (x & LK_EXCLUSIVE_WAITERS) { 222 queue = SQ_EXCLUSIVE_QUEUE; 223 v |= (x & LK_SHARED_WAITERS); 224 } else { 225 MPASS(x == LK_SHARED_WAITERS); 226 queue = SQ_SHARED_QUEUE; 227 } 228 229 if (!atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x, 230 v)) { 231 sleepq_release(&lk->lock_object); 232 continue; 233 } 234 LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", 235 __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : 236 "exclusive"); 237 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); 238 sleepq_release(&lk->lock_object); 239 break; 240 } 241 242 lock_profile_release_lock(&lk->lock_object); 243} 244 245static void 246assert_lockmgr(struct lock_object *lock, int what) 247{ 248 249 panic("lockmgr locks do not support assertions"); 250} 251 252static void 253lock_lockmgr(struct lock_object *lock, int how) 254{ 255 256 panic("lockmgr locks do not support sleep interlocking"); 257} 258 259static int 260unlock_lockmgr(struct lock_object *lock) 261{ 262 263 panic("lockmgr locks do not support sleep interlocking"); 264} 265 266void 267lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags) 268{ 269 int iflags; 270 271 MPASS((flags & ~LK_INIT_MASK) == 0); 272 273 iflags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE; 274 if ((flags & LK_NODUP) == 0) 275 iflags |= LO_DUPOK; 276 if (flags & LK_NOPROFILE) 277 iflags |= LO_NOPROFILE; 278 if ((flags & LK_NOWITNESS) == 0) 279 iflags |= LO_WITNESS; 280 if (flags & LK_QUIET) 281 iflags |= LO_QUIET; 282 iflags |= flags & (LK_CANRECURSE | LK_NOSHARE); 283 284 lk->lk_lock = LK_UNLOCKED; 285 lk->lk_recurse = 0; 286 lk->lk_timo = timo; 287 lk->lk_pri = pri; 288 lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags); 289 STACK_ZERO(lk); 290} 291 292void 293lockdestroy(struct lock *lk) 294{ 295 296 KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held")); 297 KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed")); 298 lock_destroy(&lk->lock_object); 299} 300 301int 302__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, 303 const char *wmesg, int pri, int timo, const char *file, int line) 304{ 305 uint64_t waittime; 306 struct lock_class *class; 307 const char *iwmesg; 308 uintptr_t tid, v, x; 309 u_int op; 310 int contested, error, ipri, itimo, queue; 311 312 contested = 0; 313 error = 0; 314 waittime = 0; 315 tid = (uintptr_t)curthread; 316 op = (flags & LK_TYPE_MASK); 317 iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg; 318 ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri; 319 itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo; 320 321 MPASS((flags & ~LK_TOTAL_MASK) == 0); 322 KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 || 323 (op != LK_DOWNGRADE && op != LK_RELEASE), 324 ("%s: Invalid flags in regard of the operation desired @ %s:%d", 325 __func__, file, line)); 326 KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL, 327 ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d", 328 __func__, file, line)); 329 330 class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL; 331 if (panicstr != NULL) { 332 if (flags & LK_INTERLOCK) 333 class->lc_unlock(ilk); 334 return (0); 335 } 336 337 if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE)) 338 op = LK_EXCLUSIVE; 339 340 switch (op) { 341 case LK_SHARED: 342 for (;;) { 343 x = lk->lk_lock; 344 345 /* 346 * If no other thread has an exclusive lock, or 347 * no exclusive waiter is present, bump the count of 348 * sharers. Since we have to preserve the state of 349 * waiters, if we fail to acquire the shared lock 350 * loop back and retry. 351 */ 352 if (LK_CAN_SHARE(x)) { 353 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, 354 x + LK_ONE_SHARER)) 355 break; 356 continue; 357 } 358 lock_profile_obtain_lock_failed(&lk->lock_object, 359 &contested, &waittime); 360 361 /* 362 * If the lock is alredy held by curthread in 363 * exclusive way avoid a deadlock. 364 */ 365 if (LK_HOLDER(x) == tid) { 366 LOCK_LOG2(lk, 367 "%s: %p alredy held in exclusive mode", 368 __func__, lk); 369 error = EDEADLK; 370 break; 371 } 372 373 /* 374 * If the lock is expected to not sleep just give up 375 * and return. 376 */ 377 if (LK_TRYOP(flags)) { 378 LOCK_LOG2(lk, "%s: %p fails the try operation", 379 __func__, lk); 380 error = EBUSY; 381 break; 382 } 383 384 /* 385 * Acquire the sleepqueue chain lock because we 386 * probabilly will need to manipulate waiters flags. 387 */ 388 sleepq_lock(&lk->lock_object); 389 x = lk->lk_lock; 390 391 /* 392 * if the lock can be acquired in shared mode, try 393 * again. 394 */ 395 if (LK_CAN_SHARE(x)) { 396 sleepq_release(&lk->lock_object); 397 continue; 398 } 399 400 /* 401 * Try to set the LK_SHARED_WAITERS flag. If we fail, 402 * loop back and retry. 403 */ 404 if ((x & LK_SHARED_WAITERS) == 0) { 405 if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x, 406 x | LK_SHARED_WAITERS)) { 407 sleepq_release(&lk->lock_object); 408 continue; 409 } 410 LOCK_LOG2(lk, "%s: %p set shared waiters flag", 411 __func__, lk); 412 } 413 414 /* 415 * As far as we have been unable to acquire the 416 * shared lock and the shared waiters flag is set, 417 * we will sleep. 418 */ 419 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, 420 SQ_SHARED_QUEUE); 421 flags &= ~LK_INTERLOCK; 422 if (error) { 423 LOCK_LOG3(lk, 424 "%s: interrupted sleep for %p with %d", 425 __func__, lk, error); 426 break; 427 } 428 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 429 __func__, lk); 430 } 431 if (error == 0) { 432 lock_profile_obtain_lock_success(&lk->lock_object, 433 contested, waittime, file, line); 434 LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, 435 line); 436 TD_LOCKS_INC(curthread); 437 TD_SLOCKS_INC(curthread); 438 STACK_SAVE(lk); 439 } 440 break; 441 case LK_UPGRADE: 442 _lockmgr_assert(lk, KA_SLOCKED, file, line); 443 x = lk->lk_lock & LK_ALL_WAITERS; 444 445 /* 446 * Try to switch from one shared lock to an exclusive one. 447 * We need to preserve waiters flags during the operation. 448 */ 449 if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x, 450 tid | x)) { 451 LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, 452 line); 453 TD_SLOCKS_DEC(curthread); 454 break; 455 } 456 457 /* 458 * We have been unable to succeed in upgrading, so just 459 * give up the shared lock. 460 */ 461 wakeupshlk(lk, file, line); 462 463 /* FALLTHROUGH */ 464 case LK_EXCLUSIVE: 465 466 /* 467 * If curthread alredy holds the lock and this one is 468 * allowed to recurse, simply recurse on it. 469 */ 470 if (lockmgr_xlocked(lk)) { 471 if ((flags & LK_CANRECURSE) == 0 && 472 (lk->lock_object.lo_flags & LK_CANRECURSE) == 0) { 473 474 /* 475 * If the lock is expected to not panic just 476 * give up and return. 477 */ 478 if (LK_TRYOP(flags)) { 479 LOCK_LOG2(lk, 480 "%s: %p fails the try operation", 481 __func__, lk); 482 error = EBUSY; 483 break; 484 } 485 if (flags & LK_INTERLOCK) 486 class->lc_unlock(ilk); 487 panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n", 488 __func__, iwmesg, file, line); 489 } 490 lk->lk_recurse++; 491 LOCK_LOG2(lk, "%s: %p recursing", __func__, lk); 492 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, 493 lk->lk_recurse, file, line); 494 TD_LOCKS_INC(curthread); 495 break; 496 } 497 498 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, 499 tid)) { 500 lock_profile_obtain_lock_failed(&lk->lock_object, 501 &contested, &waittime); 502 503 /* 504 * If the lock is expected to not sleep just give up 505 * and return. 506 */ 507 if (LK_TRYOP(flags)) { 508 LOCK_LOG2(lk, "%s: %p fails the try operation", 509 __func__, lk); 510 error = EBUSY; 511 break; 512 } 513 514 /* 515 * Acquire the sleepqueue chain lock because we 516 * probabilly will need to manipulate waiters flags. 517 */ 518 sleepq_lock(&lk->lock_object); 519 x = lk->lk_lock; 520 v = x & LK_ALL_WAITERS; 521 522 /* 523 * if the lock has been released while we spun on 524 * the sleepqueue chain lock just try again. 525 */ 526 if (x == LK_UNLOCKED) { 527 sleepq_release(&lk->lock_object); 528 continue; 529 } 530 531 /* 532 * The lock can be in the state where there is a 533 * pending queue of waiters, but still no owner. 534 * This happens when the lock is contested and an 535 * owner is going to claim the lock. 536 * If curthread is the one successfully acquiring it 537 * claim lock ownership and return, preserving waiters 538 * flags. 539 */ 540 if (x == (LK_UNLOCKED | v)) { 541 if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, 542 tid | v)) { 543 sleepq_release(&lk->lock_object); 544 LOCK_LOG2(lk, 545 "%s: %p claimed by a new writer", 546 __func__, lk); 547 break; 548 } 549 sleepq_release(&lk->lock_object); 550 continue; 551 } 552 553 /* 554 * Try to set the LK_EXCLUSIVE_WAITERS flag. If we 555 * fail, loop back and retry. 556 */ 557 if ((x & LK_EXCLUSIVE_WAITERS) == 0) { 558 if (!atomic_cmpset_ptr(&lk->lk_lock, x, 559 x | LK_EXCLUSIVE_WAITERS)) { 560 sleepq_release(&lk->lock_object); 561 continue; 562 } 563 LOCK_LOG2(lk, "%s: %p set excl waiters flag", 564 __func__, lk); 565 } 566 567 /* 568 * As far as we have been unable to acquire the 569 * exclusive lock and the exclusive waiters flag 570 * is set, we will sleep. 571 */ 572 error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, 573 SQ_EXCLUSIVE_QUEUE); 574 flags &= ~LK_INTERLOCK; 575 if (error) { 576 LOCK_LOG3(lk, 577 "%s: interrupted sleep for %p with %d", 578 __func__, lk, error); 579 break; 580 } 581 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 582 __func__, lk); 583 } 584 if (error == 0) { 585 lock_profile_obtain_lock_success(&lk->lock_object, 586 contested, waittime, file, line); 587 LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, 588 lk->lk_recurse, file, line); 589 TD_LOCKS_INC(curthread); 590 STACK_SAVE(lk); 591 } 592 break; 593 case LK_DOWNGRADE: 594 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line); 595 596 /* 597 * In order to preserve waiters flags, just spin. 598 */ 599 for (;;) { 600 x = lk->lk_lock & LK_ALL_WAITERS; 601 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, 602 LK_SHARERS_LOCK(1) | x)) { 603 LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 604 0, 0, file, line); 605 TD_SLOCKS_INC(curthread); 606 break; 607 } 608 cpu_spinwait(); 609 } 610 break; 611 case LK_RELEASE: 612 _lockmgr_assert(lk, KA_LOCKED, file, line); 613 x = lk->lk_lock; 614 615 if ((x & LK_SHARE) == 0) { 616 617 /* 618 * As first option, treact the lock as if it has not 619 * any waiter. 620 * Fix-up the tid var if the lock has been disowned. 621 */ 622 if (LK_HOLDER(x) == LK_KERNPROC) 623 tid = LK_KERNPROC; 624 else 625 TD_LOCKS_DEC(curthread); 626 LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, 627 lk->lk_recurse, file, line); 628 629 /* 630 * The lock is held in exclusive mode. 631 * If the lock is recursed also, then unrecurse it. 632 */ 633 if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { 634 LOCK_LOG2(lk, "%s: %p unrecursing", __func__, 635 lk); 636 lk->lk_recurse--; 637 break; 638 } 639 lock_profile_release_lock(&lk->lock_object); 640 641 if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid, 642 LK_UNLOCKED)) 643 break; 644 645 sleepq_lock(&lk->lock_object); 646 x = lk->lk_lock & LK_ALL_WAITERS; 647 v = LK_UNLOCKED; 648 649 /* 650 * If the lock has exclusive waiters, give them 651 * preference in order to avoid deadlock with 652 * shared runners up. 653 */ 654 if (x & LK_EXCLUSIVE_WAITERS) { 655 queue = SQ_EXCLUSIVE_QUEUE; 656 v |= (x & LK_SHARED_WAITERS); 657 } else { 658 MPASS(x == LK_SHARED_WAITERS); 659 queue = SQ_SHARED_QUEUE; 660 } 661 662 LOCK_LOG3(lk, 663 "%s: %p waking up threads on the %s queue", 664 __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : 665 "exclusive"); 666 atomic_store_rel_ptr(&lk->lk_lock, v); 667 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); 668 sleepq_release(&lk->lock_object); 669 break; 670 } else 671 wakeupshlk(lk, file, line); 672 break; 673 case LK_DRAIN: 674 675 /* 676 * Trying to drain a lock we alredy own will result in a 677 * deadlock. 678 */ 679 if (lockmgr_xlocked(lk)) { 680 if (flags & LK_INTERLOCK) 681 class->lc_unlock(ilk); 682 panic("%s: draining %s with the lock held @ %s:%d\n", 683 __func__, iwmesg, file, line); 684 } 685 686 while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { 687 lock_profile_obtain_lock_failed(&lk->lock_object, 688 &contested, &waittime); 689 690 /* 691 * If the lock is expected to not sleep just give up 692 * and return. 693 */ 694 if (LK_TRYOP(flags)) { 695 LOCK_LOG2(lk, "%s: %p fails the try operation", 696 __func__, lk); 697 error = EBUSY; 698 break; 699 } 700 701 /* 702 * Acquire the sleepqueue chain lock because we 703 * probabilly will need to manipulate waiters flags. 704 */ 705 sleepq_lock(&lk->lock_object); 706 x = lk->lk_lock; 707 v = x & LK_ALL_WAITERS; 708 709 /* 710 * if the lock has been released while we spun on 711 * the sleepqueue chain lock just try again. 712 */ 713 if (x == LK_UNLOCKED) { 714 sleepq_release(&lk->lock_object); 715 continue; 716 } 717 718 if (x == (LK_UNLOCKED | v)) { 719 v = x; 720 if (v & LK_EXCLUSIVE_WAITERS) { 721 queue = SQ_EXCLUSIVE_QUEUE; 722 v &= ~LK_EXCLUSIVE_WAITERS; 723 } else { 724 MPASS(v & LK_SHARED_WAITERS); 725 queue = SQ_SHARED_QUEUE; 726 v &= ~LK_SHARED_WAITERS; 727 } 728 if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) { 729 sleepq_release(&lk->lock_object); 730 continue; 731 } 732 LOCK_LOG3(lk, 733 "%s: %p waking up all threads on the %s queue", 734 __func__, lk, queue == SQ_SHARED_QUEUE ? 735 "shared" : "exclusive"); 736 sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 737 0, queue); 738 739 /* 740 * If shared waiters have been woken up we need 741 * to wait for one of them to acquire the lock 742 * before to set the exclusive waiters in 743 * order to avoid a deadlock. 744 */ 745 if (queue == SQ_SHARED_QUEUE) { 746 for (v = lk->lk_lock; 747 (v & LK_SHARE) && !LK_SHARERS(v); 748 v = lk->lk_lock) 749 cpu_spinwait(); 750 } 751 } 752 753 /* 754 * Try to set the LK_EXCLUSIVE_WAITERS flag. If we 755 * fail, loop back and retry. 756 */ 757 if ((x & LK_EXCLUSIVE_WAITERS) == 0) { 758 if (!atomic_cmpset_ptr(&lk->lk_lock, x, 759 x | LK_EXCLUSIVE_WAITERS)) { 760 sleepq_release(&lk->lock_object); 761 continue; 762 } 763 LOCK_LOG2(lk, "%s: %p set drain waiters flag", 764 __func__, lk); 765 } 766 767 /* 768 * As far as we have been unable to acquire the 769 * exclusive lock and the exclusive waiters flag 770 * is set, we will sleep. 771 */ 772 if (flags & LK_INTERLOCK) { 773 class->lc_unlock(ilk); 774 flags &= ~LK_INTERLOCK; 775 } 776 DROP_GIANT(); 777 sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK, 778 SQ_EXCLUSIVE_QUEUE); 779 sleepq_wait(&lk->lock_object, ipri & PRIMASK); 780 PICKUP_GIANT(); 781 LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", 782 __func__, lk); 783 } 784 785 if (error == 0) { 786 lock_profile_obtain_lock_success(&lk->lock_object, 787 contested, waittime, file, line); 788 LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0, 789 lk->lk_recurse, file, line); 790 TD_LOCKS_INC(curthread); 791 STACK_SAVE(lk); 792 } 793 break; 794 default: 795 if (flags & LK_INTERLOCK) 796 class->lc_unlock(ilk); 797 panic("%s: unknown lockmgr request 0x%x\n", __func__, op); 798 } 799 800 /* 801 * We could have exited from the switch without reacquiring the 802 * interlock, so we need to check for the interlock ownership. 803 */ 804 if (flags & LK_INTERLOCK) 805 class->lc_unlock(ilk); 806 807 return (error); 808} 809 810void 811_lockmgr_disown(struct lock *lk, const char *file, int line) 812{ 813 uintptr_t tid, x; 814 815 tid = (uintptr_t)curthread; 816 _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line); 817 818 /* 819 * If the owner is alredy LK_KERNPROC just skip the whole operation. 820 */ 821 if (LK_HOLDER(lk->lk_lock) != tid) 822 return; 823 824 /* 825 * In order to preserve waiters flags, just spin. 826 */ 827 for (;;) { 828 x = lk->lk_lock & LK_ALL_WAITERS; 829 if (atomic_cmpset_ptr(&lk->lk_lock, tid | x, 830 LK_KERNPROC | x)) { 831 LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, 832 line); 833 TD_LOCKS_DEC(curthread); 834 return; 835 } 836 cpu_spinwait(); 837 } 838} 839 840void 841lockmgr_printinfo(struct lock *lk) 842{ 843 struct thread *td; 844 uintptr_t x; 845 846 if (lk->lk_lock == LK_UNLOCKED) 847 printf(" lock type %s: UNLOCKED\n", lk->lock_object.lo_name); 848 else if (lk->lk_lock & LK_SHARE) 849 printf(" lock type %s: SHARED (count %ju)\n", 850 lk->lock_object.lo_name, 851 (uintmax_t)LK_SHARERS(lk->lk_lock)); 852 else { 853 td = lockmgr_xholder(lk); 854 printf(" lock type %s: EXCL by thread %p (pid %d)\n", 855 lk->lock_object.lo_name, td, td->td_proc->p_pid); 856 } 857 858 x = lk->lk_lock; 859 if (x & LK_EXCLUSIVE_WAITERS) 860 printf(" with exclusive waiters pending\n"); 861 if (x & LK_SHARED_WAITERS) 862 printf(" with shared waiters pending\n"); 863 864 STACK_PRINT(lk); 865} 866 867int 868lockstatus(struct lock *lk) 869{ 870 uintptr_t v, x; 871 int ret; 872 873 ret = LK_SHARED; 874 x = lk->lk_lock; 875 v = LK_HOLDER(x); 876 877 if ((x & LK_SHARE) == 0) { 878 if (v == (uintptr_t)curthread || v == LK_KERNPROC) 879 ret = LK_EXCLUSIVE; 880 else 881 ret = LK_EXCLOTHER; 882 } else if (x == LK_UNLOCKED) 883 ret = 0; 884 885 return (ret); 886} 887 888#ifdef INVARIANT_SUPPORT 889#ifndef INVARIANTS 890#undef _lockmgr_assert 891#endif 892 893void 894_lockmgr_assert(struct lock *lk, int what, const char *file, int line) 895{ 896 int slocked = 0; 897 898 if (panicstr != NULL) 899 return; 900 switch (what) { 901 case KA_SLOCKED: 902 case KA_SLOCKED | KA_NOTRECURSED: 903 case KA_SLOCKED | KA_RECURSED: 904 slocked = 1; 905 case KA_LOCKED: 906 case KA_LOCKED | KA_NOTRECURSED: 907 case KA_LOCKED | KA_RECURSED: 908 if (lk->lk_lock == LK_UNLOCKED || 909 ((lk->lk_lock & LK_SHARE) == 0 && (slocked || 910 (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))))) 911 panic("Lock %s not %slocked @ %s:%d\n", 912 lk->lock_object.lo_name, slocked ? "share" : "", 913 file, line); 914 915 if ((lk->lk_lock & LK_SHARE) == 0) { 916 if (lockmgr_recursed(lk)) { 917 if (what & KA_NOTRECURSED) 918 panic("Lock %s recursed @ %s:%d\n", 919 lk->lock_object.lo_name, file, 920 line); 921 } else if (what & KA_RECURSED) 922 panic("Lock %s not recursed @ %s:%d\n", 923 lk->lock_object.lo_name, file, line); 924 } 925 break; 926 case KA_XLOCKED: 927 case KA_XLOCKED | KA_NOTRECURSED: 928 case KA_XLOCKED | KA_RECURSED: 929 if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)) 930 panic("Lock %s not exclusively locked @ %s:%d\n", 931 lk->lock_object.lo_name, file, line); 932 if (lockmgr_recursed(lk)) { 933 if (what & KA_NOTRECURSED) 934 panic("Lock %s recursed @ %s:%d\n", 935 lk->lock_object.lo_name, file, line); 936 } else if (what & KA_RECURSED) 937 panic("Lock %s not recursed @ %s:%d\n", 938 lk->lock_object.lo_name, file, line); 939 break; 940 case KA_UNLOCKED: 941 if (lockmgr_xlocked(lk) || lockmgr_disowned(lk)) 942 panic("Lock %s exclusively locked @ %s:%d\n", 943 lk->lock_object.lo_name, file, line); 944 break; 945 default: 946 panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file, 947 line); 948 } 949} 950#endif 951 952#ifdef DDB 953int 954lockmgr_chain(struct thread *td, struct thread **ownerp) 955{ 956 struct lock *lk; 957 958 lk = td->td_wchan; 959 960 if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr) 961 return (0); 962 db_printf("blocked on lockmgr %s", lk->lock_object.lo_name); 963 if (lk->lk_lock & LK_SHARE) 964 db_printf("SHARED (count %ju)\n", 965 (uintmax_t)LK_SHARERS(lk->lk_lock)); 966 else 967 db_printf("EXCL\n"); 968 *ownerp = lockmgr_xholder(lk); 969 970 return (1); 971} 972 973static void 974db_show_lockmgr(struct lock_object *lock) 975{ 976 struct thread *td; 977 struct lock *lk; 978 979 lk = (struct lock *)lock; 980 981 db_printf(" state: "); 982 if (lk->lk_lock == LK_UNLOCKED) 983 db_printf("UNLOCKED\n"); 984 else if (lk->lk_lock & LK_SHARE) 985 db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock)); 986 else { 987 td = lockmgr_xholder(lk); 988 if (td == (struct thread *)LK_KERNPROC) 989 db_printf("XLOCK: LK_KERNPROC\n"); 990 else 991 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 992 td->td_tid, td->td_proc->p_pid, 993 td->td_proc->p_comm); 994 if (lockmgr_recursed(lk)) 995 db_printf(" recursed: %d\n", lk->lk_recurse); 996 } 997 db_printf(" waiters: "); 998 switch (lk->lk_lock & LK_ALL_WAITERS) { 999 case LK_SHARED_WAITERS: 1000 db_printf("shared\n"); 1001 case LK_EXCLUSIVE_WAITERS: 1002 db_printf("exclusive\n"); 1003 break; 1004 case LK_ALL_WAITERS: 1005 db_printf("shared and exclusive\n"); 1006 break; 1007 default: 1008 db_printf("none\n"); 1009 } 1010} 1011#endif 1012