kern_mutex.c revision 164159
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> 37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 164159 2006-11-11 03:18:07Z kmacy $"); 38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_mutex_wake_all.h" 43#include "opt_sched.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/bus.h> 48#include <sys/conf.h> 49#include <sys/kdb.h> 50#include <sys/kernel.h> 51#include <sys/ktr.h> 52#include <sys/lock.h> 53#include <sys/malloc.h> 54#include <sys/mutex.h> 55#include <sys/proc.h> 56#include <sys/resourcevar.h> 57#include <sys/sched.h> 58#include <sys/sbuf.h> 59#include <sys/sysctl.h> 60#include <sys/turnstile.h> 61#include <sys/vmmeter.h> 62#include <sys/lock_profile.h> 63 64#include <machine/atomic.h> 65#include <machine/bus.h> 66#include <machine/cpu.h> 67 68#include <ddb/ddb.h> 69 70#include <fs/devfs/devfs_int.h> 71 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74 75/* 76 * Force MUTEX_WAKE_ALL for now. 77 * single thread wakeup needs fixes to avoid race conditions with 78 * priority inheritance. 79 */ 80#ifndef MUTEX_WAKE_ALL 81#define MUTEX_WAKE_ALL 82#endif 83 84/* 85 * Internal utility macros. 86 */ 87#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 88 89#define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 90 91#ifdef DDB 92static void db_show_mtx(struct lock_object *lock); 93#endif 94 95/* 96 * Lock classes for sleep and spin mutexes. 97 */ 98struct lock_class lock_class_mtx_sleep = { 99 "sleep mutex", 100 LC_SLEEPLOCK | LC_RECURSABLE, 101#ifdef DDB 102 db_show_mtx 103#endif 104}; 105struct lock_class lock_class_mtx_spin = { 106 "spin mutex", 107 LC_SPINLOCK | LC_RECURSABLE, 108#ifdef DDB 109 db_show_mtx 110#endif 111}; 112 113/* 114 * System-wide mutexes 115 */ 116struct mtx sched_lock; 117struct mtx Giant; 118 119/* 120 * Function versions of the inlined __mtx_* macros. These are used by 121 * modules and can also be called from assembly language if needed. 122 */ 123void 124_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 125{ 126 uint64_t waittime; 127 128 MPASS(curthread != NULL); 129 KASSERT(m->mtx_lock != MTX_DESTROYED, 130 ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 131 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 132 ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 133 file, line)); 134 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 135 file, line); 136 137 lock_profile_waitstart(&waittime); 138 _get_sleep_lock(m, curthread, opts, file, line); 139 LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 140 line); 141 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 142 curthread->td_locks++; 143 lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line); 144} 145 146void 147_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 148{ 149 150 MPASS(curthread != NULL); 151 KASSERT(m->mtx_lock != MTX_DESTROYED, 152 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 153 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 154 ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 155 file, line)); 156 curthread->td_locks--; 157 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 158 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 159 line); 160 mtx_assert(m, MA_OWNED); 161 162 lock_profile_release_lock(&m->mtx_object); 163 _rel_sleep_lock(m, curthread, opts, file, line); 164} 165 166void 167_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 168{ 169 170 uint64_t waittime; 171 172 MPASS(curthread != NULL); 173 KASSERT(m->mtx_lock != MTX_DESTROYED, 174 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 175 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, 176 ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 177 m->mtx_object.lo_name, file, line)); 178 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 179 file, line); 180 lock_profile_waitstart(&waittime); 181 _get_spin_lock(m, curthread, opts, file, line); 182 LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 183 line); 184 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 185 lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line); 186} 187 188void 189_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 190{ 191 192 MPASS(curthread != NULL); 193 KASSERT(m->mtx_lock != MTX_DESTROYED, 194 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 195 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, 196 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 197 m->mtx_object.lo_name, file, line)); 198 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 199 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 200 line); 201 mtx_assert(m, MA_OWNED); 202 lock_profile_release_lock(&m->mtx_object); 203 _rel_spin_lock(m); 204} 205 206/* 207 * The important part of mtx_trylock{,_flags}() 208 * Tries to acquire lock `m.' If this function is called on a mutex that 209 * is already owned, it will recursively acquire the lock. 210 */ 211int 212_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 213{ 214 int rval; 215 uint64_t waittime = 0; 216 217 MPASS(curthread != NULL); 218 KASSERT(m->mtx_lock != MTX_DESTROYED, 219 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 220 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 221 ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 222 file, line)); 223 224 if (mtx_owned(m) && (m->mtx_object.lo_flags & LO_RECURSABLE) != 0) { 225 m->mtx_recurse++; 226 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 227 rval = 1; 228 } else 229 rval = _obtain_lock(m, (uintptr_t)curthread); 230 231 LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 232 if (rval) { 233 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 234 file, line); 235 curthread->td_locks++; 236 lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line); 237 238 } 239 240 return (rval); 241} 242 243/* 244 * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 245 * 246 * We call this if the lock is either contested (i.e. we need to go to 247 * sleep waiting for it), or if we need to recurse on it. 248 */ 249void 250_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, 251 int line) 252{ 253#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 254 volatile struct thread *owner; 255#endif 256#ifdef KTR 257 int cont_logged = 0; 258#endif 259 uintptr_t v; 260 int contested; 261 262 if (mtx_owned(m)) { 263 KASSERT((m->mtx_object.lo_flags & LO_RECURSABLE) != 0, 264 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 265 m->mtx_object.lo_name, file, line)); 266 m->mtx_recurse++; 267 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 268 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 269 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 270 return; 271 } 272 273 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 274 CTR4(KTR_LOCK, 275 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 276 m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 277 278 while (!_obtain_lock(m, tid)) { 279 lock_profile_obtain_lock_failed(&m->mtx_object, &contested); 280 turnstile_lock(&m->mtx_object); 281 v = m->mtx_lock; 282 283 /* 284 * Check if the lock has been released while spinning for 285 * the turnstile chain lock. 286 */ 287 if (v == MTX_UNOWNED) { 288 turnstile_release(&m->mtx_object); 289 cpu_spinwait(); 290 continue; 291 } 292 293#ifdef MUTEX_WAKE_ALL 294 MPASS(v != MTX_CONTESTED); 295#else 296 /* 297 * The mutex was marked contested on release. This means that 298 * there are other threads blocked on it. Grab ownership of 299 * it and propagate its priority to the current thread if 300 * necessary. 301 */ 302 if (v == MTX_CONTESTED) { 303 m->mtx_lock = tid | MTX_CONTESTED; 304 turnstile_claim(&m->mtx_object); 305 break; 306 } 307#endif 308 309 /* 310 * If the mutex isn't already contested and a failure occurs 311 * setting the contested bit, the mutex was either released 312 * or the state of the MTX_RECURSED bit changed. 313 */ 314 if ((v & MTX_CONTESTED) == 0 && 315 !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 316 turnstile_release(&m->mtx_object); 317 cpu_spinwait(); 318 continue; 319 } 320 321#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 322 /* 323 * If the current owner of the lock is executing on another 324 * CPU, spin instead of blocking. 325 */ 326 owner = (struct thread *)(v & ~MTX_FLAGMASK); 327#ifdef ADAPTIVE_GIANT 328 if (TD_IS_RUNNING(owner)) { 329#else 330 if (m != &Giant && TD_IS_RUNNING(owner)) { 331#endif 332 turnstile_release(&m->mtx_object); 333 while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) { 334 cpu_spinwait(); 335 } 336 continue; 337 } 338#endif /* SMP && !NO_ADAPTIVE_MUTEXES */ 339 340 /* 341 * We definitely must sleep for this lock. 342 */ 343 mtx_assert(m, MA_NOTOWNED); 344 345#ifdef KTR 346 if (!cont_logged) { 347 CTR6(KTR_CONTENTION, 348 "contention: %p at %s:%d wants %s, taken by %s:%d", 349 (void *)tid, file, line, m->mtx_object.lo_name, 350 WITNESS_FILE(&m->mtx_object), 351 WITNESS_LINE(&m->mtx_object)); 352 cont_logged = 1; 353 } 354#endif 355 356 /* 357 * Block on the turnstile. 358 */ 359 turnstile_wait(&m->mtx_object, mtx_owner(m), 360 TS_EXCLUSIVE_QUEUE); 361 } 362 363#ifdef KTR 364 if (cont_logged) { 365 CTR4(KTR_CONTENTION, 366 "contention end: %s acquired by %p at %s:%d", 367 m->mtx_object.lo_name, (void *)tid, file, line); 368 } 369#endif 370#ifdef LOCK_PROFILING 371 if (contested) 372 m->mtx_object.lo_profile_obj.lpo_contest_locking++; 373 m->mtx_object.lo_profile_obj.lpo_contest_holding = 0; 374#endif 375 return; 376} 377 378#ifdef SMP 379/* 380 * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 381 * 382 * This is only called if we need to actually spin for the lock. Recursion 383 * is handled inline. 384 */ 385void 386_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, 387 int line) 388{ 389 struct thread *td; 390 int contested, i = 0; 391 392 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 393 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 394 395 while (!_obtain_lock(m, tid)) { 396 lock_profile_obtain_lock_failed(&m->mtx_object, &contested); 397 398 /* Give interrupts a chance while we spin. */ 399 spinlock_exit(); 400 while (m->mtx_lock != MTX_UNOWNED) { 401 if (i++ < 10000000) { 402 cpu_spinwait(); 403 continue; 404 } 405 if (i < 60000000 || kdb_active || panicstr != NULL) 406 DELAY(1); 407 else { 408 td = mtx_owner(m); 409 410 /* If the mutex is unlocked, try again. */ 411 if (td == NULL) 412 continue; 413 printf( 414 "spin lock %p (%s) held by %p (tid %d) too long\n", 415 m, m->mtx_object.lo_name, td, td->td_tid); 416#ifdef WITNESS 417 witness_display_spinlock(&m->mtx_object, td); 418#endif 419 panic("spin lock held too long"); 420 } 421 cpu_spinwait(); 422 } 423 spinlock_enter(); 424 } 425 426 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 427 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 428 429 return; 430} 431#endif /* SMP */ 432 433/* 434 * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 435 * 436 * We are only called here if the lock is recursed or contested (i.e. we 437 * need to wake up a blocked thread). 438 */ 439void 440_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 441{ 442 struct turnstile *ts; 443#ifndef PREEMPTION 444 struct thread *td, *td1; 445#endif 446 447 if (mtx_recursed(m)) { 448 if (--(m->mtx_recurse) == 0) 449 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 450 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 451 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 452 return; 453 } 454 455 turnstile_lock(&m->mtx_object); 456 ts = turnstile_lookup(&m->mtx_object); 457 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 458 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 459 460#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 461 if (ts == NULL) { 462 _release_lock_quick(m); 463 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 464 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 465 turnstile_release(&m->mtx_object); 466 return; 467 } 468#else 469 MPASS(ts != NULL); 470#endif 471#ifndef PREEMPTION 472 /* XXX */ 473 td1 = turnstile_head(ts, TS_EXCLUSIVE_QUEUE); 474#endif 475#ifdef MUTEX_WAKE_ALL 476 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 477 _release_lock_quick(m); 478#else 479 if (turnstile_signal(ts, TS_EXCLUSIVE_QUEUE)) { 480 _release_lock_quick(m); 481 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 482 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 483 } else { 484 m->mtx_lock = MTX_CONTESTED; 485 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 486 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p still contested", 487 m); 488 } 489#endif 490 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 491 492#ifndef PREEMPTION 493 /* 494 * XXX: This is just a hack until preemption is done. However, 495 * once preemption is done we need to either wrap the 496 * turnstile_signal() and release of the actual lock in an 497 * extra critical section or change the preemption code to 498 * always just set a flag and never do instant-preempts. 499 */ 500 td = curthread; 501 if (td->td_critnest > 0 || td1->td_priority >= td->td_priority) 502 return; 503 mtx_lock_spin(&sched_lock); 504 if (!TD_IS_RUNNING(td1)) { 505#ifdef notyet 506 if (td->td_ithd != NULL) { 507 struct ithd *it = td->td_ithd; 508 509 if (it->it_interrupted) { 510 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 511 CTR2(KTR_LOCK, 512 "_mtx_unlock_sleep: %p interrupted %p", 513 it, it->it_interrupted); 514 intr_thd_fixup(it); 515 } 516 } 517#endif 518 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 519 CTR2(KTR_LOCK, 520 "_mtx_unlock_sleep: %p switching out lock=%p", m, 521 (void *)m->mtx_lock); 522 523 mi_switch(SW_INVOL, NULL); 524 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 525 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 526 m, (void *)m->mtx_lock); 527 } 528 mtx_unlock_spin(&sched_lock); 529#endif 530 531 return; 532} 533 534/* 535 * All the unlocking of MTX_SPIN locks is done inline. 536 * See the _rel_spin_lock() macro for the details. 537 */ 538 539/* 540 * The backing function for the INVARIANTS-enabled mtx_assert() 541 */ 542#ifdef INVARIANT_SUPPORT 543void 544_mtx_assert(struct mtx *m, int what, const char *file, int line) 545{ 546 547 if (panicstr != NULL || dumping) 548 return; 549 switch (what) { 550 case MA_OWNED: 551 case MA_OWNED | MA_RECURSED: 552 case MA_OWNED | MA_NOTRECURSED: 553 if (!mtx_owned(m)) 554 panic("mutex %s not owned at %s:%d", 555 m->mtx_object.lo_name, file, line); 556 if (mtx_recursed(m)) { 557 if ((what & MA_NOTRECURSED) != 0) 558 panic("mutex %s recursed at %s:%d", 559 m->mtx_object.lo_name, file, line); 560 } else if ((what & MA_RECURSED) != 0) { 561 panic("mutex %s unrecursed at %s:%d", 562 m->mtx_object.lo_name, file, line); 563 } 564 break; 565 case MA_NOTOWNED: 566 if (mtx_owned(m)) 567 panic("mutex %s owned at %s:%d", 568 m->mtx_object.lo_name, file, line); 569 break; 570 default: 571 panic("unknown mtx_assert at %s:%d", file, line); 572 } 573} 574#endif 575 576/* 577 * The MUTEX_DEBUG-enabled mtx_validate() 578 * 579 * Most of these checks have been moved off into the LO_INITIALIZED flag 580 * maintained by the witness code. 581 */ 582#ifdef MUTEX_DEBUG 583 584void mtx_validate(struct mtx *); 585 586void 587mtx_validate(struct mtx *m) 588{ 589 590/* 591 * XXX: When kernacc() does not require Giant we can reenable this check 592 */ 593#ifdef notyet 594 /* 595 * Can't call kernacc() from early init386(), especially when 596 * initializing Giant mutex, because some stuff in kernacc() 597 * requires Giant itself. 598 */ 599 if (!cold) 600 if (!kernacc((caddr_t)m, sizeof(m), 601 VM_PROT_READ | VM_PROT_WRITE)) 602 panic("Can't read and write to mutex %p", m); 603#endif 604} 605#endif 606 607/* 608 * General init routine used by the MTX_SYSINIT() macro. 609 */ 610void 611mtx_sysinit(void *arg) 612{ 613 struct mtx_args *margs = arg; 614 615 mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 616} 617 618/* 619 * Mutex initialization routine; initialize lock `m' of type contained in 620 * `opts' with options contained in `opts' and name `name.' The optional 621 * lock type `type' is used as a general lock category name for use with 622 * witness. 623 */ 624void 625mtx_init(struct mtx *m, const char *name, const char *type, int opts) 626{ 627 struct lock_class *class; 628 int flags; 629 630 MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 631 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); 632 633#ifdef MUTEX_DEBUG 634 /* Diagnostic and error correction */ 635 mtx_validate(m); 636#endif 637 638 /* Determine lock class and lock flags. */ 639 if (opts & MTX_SPIN) 640 class = &lock_class_mtx_spin; 641 else 642 class = &lock_class_mtx_sleep; 643 flags = 0; 644 if (opts & MTX_QUIET) 645 flags |= LO_QUIET; 646 if (opts & MTX_RECURSE) 647 flags |= LO_RECURSABLE; 648 if ((opts & MTX_NOWITNESS) == 0) 649 flags |= LO_WITNESS; 650 if (opts & MTX_DUPOK) 651 flags |= LO_DUPOK; 652 if (opts & MTX_NOPROFILE) 653 flags |= LO_NOPROFILE; 654 655 /* Initialize mutex. */ 656 m->mtx_lock = MTX_UNOWNED; 657 m->mtx_recurse = 0; 658 659 lock_profile_object_init(&m->mtx_object, name); 660 lock_init(&m->mtx_object, class, name, type, flags); 661} 662 663/* 664 * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 665 * passed in as a flag here because if the corresponding mtx_init() was 666 * called with MTX_QUIET set, then it will already be set in the mutex's 667 * flags. 668 */ 669void 670mtx_destroy(struct mtx *m) 671{ 672 673 if (!mtx_owned(m)) 674 MPASS(mtx_unowned(m)); 675 else { 676 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 677 678 /* Perform the non-mtx related part of mtx_unlock_spin(). */ 679 if (LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin) 680 spinlock_exit(); 681 else 682 curthread->td_locks--; 683 684 /* Tell witness this isn't locked to make it happy. */ 685 WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, 686 __LINE__); 687 } 688 689 m->mtx_lock = MTX_DESTROYED; 690 lock_profile_object_destroy(&m->mtx_object); 691 lock_destroy(&m->mtx_object); 692} 693 694/* 695 * Intialize the mutex code and system mutexes. This is called from the MD 696 * startup code prior to mi_startup(). The per-CPU data space needs to be 697 * setup before this is called. 698 */ 699void 700mutex_init(void) 701{ 702 703 /* Setup turnstiles so that sleep mutexes work. */ 704 init_turnstiles(); 705 706 /* 707 * Initialize mutexes. 708 */ 709 mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 710 mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 711 mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 712 mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 713 mtx_lock(&Giant); 714 715 lock_profile_init(); 716} 717 718#ifdef DDB 719void 720db_show_mtx(struct lock_object *lock) 721{ 722 struct thread *td; 723 struct mtx *m; 724 725 m = (struct mtx *)lock; 726 727 db_printf(" flags: {"); 728 if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 729 db_printf("SPIN"); 730 else 731 db_printf("DEF"); 732 if (m->mtx_object.lo_flags & LO_RECURSABLE) 733 db_printf(", RECURSE"); 734 if (m->mtx_object.lo_flags & LO_DUPOK) 735 db_printf(", DUPOK"); 736 db_printf("}\n"); 737 db_printf(" state: {"); 738 if (mtx_unowned(m)) 739 db_printf("UNOWNED"); 740 else { 741 db_printf("OWNED"); 742 if (m->mtx_lock & MTX_CONTESTED) 743 db_printf(", CONTESTED"); 744 if (m->mtx_lock & MTX_RECURSED) 745 db_printf(", RECURSED"); 746 } 747 db_printf("}\n"); 748 if (!mtx_unowned(m)) { 749 td = mtx_owner(m); 750 db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 751 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 752 if (mtx_recursed(m)) 753 db_printf(" recursed: %d\n", m->mtx_recurse); 754 } 755} 756#endif 757