kern_mutex.c revision 167365
1251875Speter/*- 2251875Speter * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3251875Speter * 4251875Speter * Redistribution and use in source and binary forms, with or without 5251875Speter * modification, are permitted provided that the following conditions 6251875Speter * are met: 7251875Speter * 1. Redistributions of source code must retain the above copyright 8251875Speter * notice, this list of conditions and the following disclaimer. 9251875Speter * 2. Redistributions in binary form must reproduce the above copyright 10251875Speter * notice, this list of conditions and the following disclaimer in the 11251875Speter * documentation and/or other materials provided with the distribution. 12251875Speter * 3. Berkeley Software Design Inc's name may not be used to endorse or 13251875Speter * promote products derived from this software without specific prior 14251875Speter * written permission. 15251875Speter * 16251875Speter * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17251875Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18251875Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19251875Speter * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20251875Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21251875Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22251875Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23251875Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24251875Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25251875Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26251875Speter * SUCH DAMAGE. 27251875Speter * 28251875Speter * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29251875Speter * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30251875Speter */ 31251875Speter 32251875Speter/* 33251875Speter * Machine independent bits of mutex implementation. 34251875Speter */ 35251875Speter 36251875Speter#include <sys/cdefs.h> 37251875Speter__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 167365 2007-03-09 16:04:44Z jhb $"); 38251875Speter 39251875Speter#include "opt_adaptive_mutexes.h" 40251875Speter#include "opt_ddb.h" 41251875Speter#include "opt_global.h" 42251875Speter#include "opt_mutex_wake_all.h" 43251875Speter#include "opt_sched.h" 44251875Speter 45251875Speter#include <sys/param.h> 46251875Speter#include <sys/systm.h> 47251875Speter#include <sys/bus.h> 48251875Speter#include <sys/conf.h> 49251875Speter#include <sys/kdb.h> 50251875Speter#include <sys/kernel.h> 51251875Speter#include <sys/ktr.h> 52251875Speter#include <sys/lock.h> 53251875Speter#include <sys/malloc.h> 54251875Speter#include <sys/mutex.h> 55251875Speter#include <sys/proc.h> 56251875Speter#include <sys/resourcevar.h> 57251875Speter#include <sys/sched.h> 58251875Speter#include <sys/sbuf.h> 59251875Speter#include <sys/sysctl.h> 60251875Speter#include <sys/turnstile.h> 61251875Speter#include <sys/vmmeter.h> 62251875Speter#include <sys/lock_profile.h> 63251875Speter 64251875Speter#include <machine/atomic.h> 65251875Speter#include <machine/bus.h> 66251875Speter#include <machine/cpu.h> 67251875Speter 68251875Speter#include <ddb/ddb.h> 69251875Speter 70251875Speter#include <fs/devfs/devfs_int.h> 71251875Speter 72251875Speter#include <vm/vm.h> 73251875Speter#include <vm/vm_extern.h> 74251875Speter 75251875Speter/* 76251875Speter * Force MUTEX_WAKE_ALL for now. 77251875Speter * single thread wakeup needs fixes to avoid race conditions with 78251875Speter * priority inheritance. 79251875Speter */ 80251875Speter#ifndef MUTEX_WAKE_ALL 81251875Speter#define MUTEX_WAKE_ALL 82251875Speter#endif 83251875Speter 84251875Speter/* 85251875Speter * Internal utility macros. 86251875Speter */ 87251875Speter#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 88251875Speter 89251875Speter#define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 90251875Speter 91251875Speter#ifdef DDB 92251875Speterstatic void db_show_mtx(struct lock_object *lock); 93251875Speter#endif 94251875Speter 95251875Speter/* 96251875Speter * Lock classes for sleep and spin mutexes. 97251875Speter */ 98251875Speterstruct lock_class lock_class_mtx_sleep = { 99251875Speter .lc_name = "sleep mutex", 100251875Speter .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 101251875Speter#ifdef DDB 102251875Speter .lc_ddb_show = db_show_mtx, 103251875Speter#endif 104251875Speter}; 105251875Speterstruct lock_class lock_class_mtx_spin = { 106251875Speter .lc_name = "spin mutex", 107251875Speter .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 108251875Speter#ifdef DDB 109251875Speter .lc_ddb_show = db_show_mtx, 110251875Speter#endif 111251875Speter}; 112251875Speter 113251875Speter/* 114251875Speter * System-wide mutexes 115251875Speter */ 116251875Speterstruct mtx sched_lock; 117251875Speterstruct mtx Giant; 118251875Speter 119251875Speter#ifdef LOCK_PROFILING 120251875Speterstatic inline void lock_profile_init(void) 121251875Speter{ 122251875Speter int i; 123251875Speter /* Initialize the mutex profiling locks */ 124251875Speter for (i = 0; i < LPROF_LOCK_SIZE; i++) { 125251875Speter mtx_init(&lprof_locks[i], "mprof lock", 126251875Speter NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE); 127251875Speter } 128251875Speter} 129251875Speter#else 130251875Speterstatic inline void lock_profile_init(void) {;} 131251875Speter#endif 132251875Speter 133251875Speter/* 134251875Speter * Function versions of the inlined __mtx_* macros. These are used by 135251875Speter * modules and can also be called from assembly language if needed. 136251875Speter */ 137251875Spetervoid 138251875Speter_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 139251875Speter{ 140251875Speter 141251875Speter MPASS(curthread != NULL); 142251875Speter KASSERT(m->mtx_lock != MTX_DESTROYED, 143251875Speter ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 144269847Speter KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 145269847Speter ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 146269847Speter file, line)); 147269847Speter WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 148269847Speter file, line); 149269847Speter 150269847Speter _get_sleep_lock(m, curthread, opts, file, line); 151269847Speter LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 152269847Speter line); 153269847Speter WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 154269847Speter curthread->td_locks++; 155269847Speter} 156251875Speter 157251875Spetervoid 158251875Speter_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 159251875Speter{ 160251875Speter MPASS(curthread != NULL); 161251875Speter KASSERT(m->mtx_lock != MTX_DESTROYED, 162251875Speter ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 163251875Speter KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 164251875Speter ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 165251875Speter file, line)); 166251875Speter curthread->td_locks--; 167251875Speter WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 168251875Speter LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 169251875Speter line); 170251875Speter mtx_assert(m, MA_OWNED); 171251875Speter 172251875Speter lock_profile_release_lock(&m->mtx_object); 173251875Speter _rel_sleep_lock(m, curthread, opts, file, line); 174251875Speter} 175251875Speter 176251875Spetervoid 177251875Speter_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 178251875Speter{ 179251875Speter 180251875Speter MPASS(curthread != NULL); 181251875Speter KASSERT(m->mtx_lock != MTX_DESTROYED, 182251875Speter ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 183251875Speter KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, 184251875Speter ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 185251875Speter m->mtx_object.lo_name, file, line)); 186251875Speter WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 187251875Speter file, line); 188251875Speter _get_spin_lock(m, curthread, opts, file, line); 189251875Speter LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 190251875Speter line); 191251875Speter WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 192251875Speter} 193251875Speter 194251875Spetervoid 195251875Speter_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 196251875Speter{ 197251875Speter 198251875Speter MPASS(curthread != NULL); 199251875Speter KASSERT(m->mtx_lock != MTX_DESTROYED, 200251875Speter ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 201251875Speter KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, 202251875Speter ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 203251875Speter m->mtx_object.lo_name, file, line)); 204251875Speter WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 205251875Speter LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 206251875Speter line); 207251875Speter mtx_assert(m, MA_OWNED); 208251875Speter 209251875Speter lock_profile_release_lock(&m->mtx_object); 210251875Speter _rel_spin_lock(m); 211251875Speter} 212251875Speter 213251875Speter/* 214251875Speter * The important part of mtx_trylock{,_flags}() 215251875Speter * Tries to acquire lock `m.' If this function is called on a mutex that 216251875Speter * is already owned, it will recursively acquire the lock. 217251875Speter */ 218251875Speterint 219251875Speter_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 220251875Speter{ 221251875Speter int rval, contested = 0; 222251875Speter uint64_t waittime = 0; 223251875Speter 224251875Speter MPASS(curthread != NULL); 225251875Speter KASSERT(m->mtx_lock != MTX_DESTROYED, 226251875Speter ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 227251875Speter KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 228251875Speter ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 229251875Speter file, line)); 230251875Speter 231251875Speter if (mtx_owned(m) && (m->mtx_object.lo_flags & LO_RECURSABLE) != 0) { 232251875Speter m->mtx_recurse++; 233251875Speter atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 234251875Speter rval = 1; 235251875Speter } else 236251875Speter rval = _obtain_lock(m, (uintptr_t)curthread); 237251875Speter 238251875Speter LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 239251875Speter if (rval) { 240251875Speter WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 241251875Speter file, line); 242251875Speter curthread->td_locks++; 243251875Speter if (m->mtx_recurse == 0) 244251875Speter lock_profile_obtain_lock_success(&m->mtx_object, contested, 245251875Speter waittime, file, line); 246251875Speter 247251875Speter } 248251875Speter 249251875Speter return (rval); 250251875Speter} 251251875Speter 252251875Speter/* 253251875Speter * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 254251875Speter * 255251875Speter * We call this if the lock is either contested (i.e. we need to go to 256251875Speter * sleep waiting for it), or if we need to recurse on it. 257251875Speter */ 258251875Spetervoid 259251875Speter_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, 260251875Speter int line) 261251875Speter{ 262251875Speter#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 263251875Speter volatile struct thread *owner; 264251875Speter#endif 265251875Speter#ifdef KTR 266251875Speter int cont_logged = 0; 267251875Speter#endif 268251875Speter uintptr_t v; 269251875Speter 270251875Speter if (mtx_owned(m)) { 271251875Speter KASSERT((m->mtx_object.lo_flags & LO_RECURSABLE) != 0, 272251875Speter ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 273251875Speter m->mtx_object.lo_name, file, line)); 274251875Speter m->mtx_recurse++; 275251875Speter atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 276251875Speter if (LOCK_LOG_TEST(&m->mtx_object, opts)) 277251875Speter CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 278251875Speter return; 279251875Speter } 280251875Speter 281251875Speter if (LOCK_LOG_TEST(&m->mtx_object, opts)) 282251875Speter CTR4(KTR_LOCK, 283251875Speter "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 284251875Speter m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 285251875Speter 286251875Speter while (!_obtain_lock(m, tid)) { 287251875Speter turnstile_lock(&m->mtx_object); 288251875Speter v = m->mtx_lock; 289251875Speter 290251875Speter /* 291251875Speter * Check if the lock has been released while spinning for 292251875Speter * the turnstile chain lock. 293251875Speter */ 294251875Speter if (v == MTX_UNOWNED) { 295251875Speter turnstile_release(&m->mtx_object); 296251875Speter cpu_spinwait(); 297251875Speter continue; 298251875Speter } 299251875Speter 300251875Speter#ifdef MUTEX_WAKE_ALL 301251875Speter MPASS(v != MTX_CONTESTED); 302251875Speter#else 303251875Speter /* 304251875Speter * The mutex was marked contested on release. This means that 305251875Speter * there are other threads blocked on it. Grab ownership of 306251875Speter * it and propagate its priority to the current thread if 307251875Speter * necessary. 308251875Speter */ 309251875Speter if (v == MTX_CONTESTED) { 310251875Speter m->mtx_lock = tid | MTX_CONTESTED; 311251875Speter turnstile_claim(&m->mtx_object); 312251875Speter break; 313251875Speter } 314251875Speter#endif 315251875Speter 316251875Speter /* 317251875Speter * If the mutex isn't already contested and a failure occurs 318251875Speter * setting the contested bit, the mutex was either released 319251875Speter * or the state of the MTX_RECURSED bit changed. 320251875Speter */ 321251875Speter if ((v & MTX_CONTESTED) == 0 && 322251875Speter !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 323251875Speter turnstile_release(&m->mtx_object); 324251875Speter cpu_spinwait(); 325251875Speter continue; 326251875Speter } 327251875Speter 328251875Speter#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 329251875Speter /* 330251875Speter * If the current owner of the lock is executing on another 331251875Speter * CPU, spin instead of blocking. 332251875Speter */ 333251875Speter owner = (struct thread *)(v & ~MTX_FLAGMASK); 334251875Speter#ifdef ADAPTIVE_GIANT 335251875Speter if (TD_IS_RUNNING(owner)) 336251875Speter#else 337251875Speter if (m != &Giant && TD_IS_RUNNING(owner)) 338251875Speter#endif 339251875Speter { 340251875Speter turnstile_release(&m->mtx_object); 341251875Speter while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) { 342251875Speter cpu_spinwait(); 343251875Speter } 344251875Speter continue; 345251875Speter } 346251875Speter#endif /* SMP && !NO_ADAPTIVE_MUTEXES */ 347251875Speter 348251875Speter /* 349251875Speter * We definitely must sleep for this lock. 350251875Speter */ 351251875Speter mtx_assert(m, MA_NOTOWNED); 352251875Speter 353251875Speter#ifdef KTR 354251875Speter if (!cont_logged) { 355251875Speter CTR6(KTR_CONTENTION, 356251875Speter "contention: %p at %s:%d wants %s, taken by %s:%d", 357251875Speter (void *)tid, file, line, m->mtx_object.lo_name, 358251875Speter WITNESS_FILE(&m->mtx_object), 359251875Speter WITNESS_LINE(&m->mtx_object)); 360251875Speter cont_logged = 1; 361251875Speter } 362251875Speter#endif 363251875Speter 364251875Speter /* 365251875Speter * Block on the turnstile. 366251875Speter */ 367251875Speter turnstile_wait(&m->mtx_object, mtx_owner(m), 368251875Speter TS_EXCLUSIVE_QUEUE); 369251875Speter } 370251875Speter#ifdef KTR 371251875Speter if (cont_logged) { 372251875Speter CTR4(KTR_CONTENTION, 373251875Speter "contention end: %s acquired by %p at %s:%d", 374251875Speter m->mtx_object.lo_name, (void *)tid, file, line); 375251875Speter } 376251875Speter#endif 377251875Speter return; 378251875Speter} 379251875Speter 380251875Speter#ifdef SMP 381251875Speter/* 382251875Speter * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 383251875Speter * 384251875Speter * This is only called if we need to actually spin for the lock. Recursion 385251875Speter * is handled inline. 386251875Speter */ 387251875Spetervoid 388251875Speter_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, 389251875Speter int line) 390251875Speter{ 391251875Speter int i = 0; 392251875Speter struct thread *td; 393251875Speter 394251875Speter if (LOCK_LOG_TEST(&m->mtx_object, opts)) 395251875Speter CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 396253734Speter 397253734Speter while (!_obtain_lock(m, tid)) { 398251875Speter 399253734Speter /* Give interrupts a chance while we spin. */ 400253734Speter spinlock_exit(); 401253734Speter while (m->mtx_lock != MTX_UNOWNED) { 402253734Speter if (i++ < 10000000) { 403251875Speter cpu_spinwait(); 404253734Speter continue; 405251875Speter } 406253734Speter if (i < 60000000 || kdb_active || panicstr != NULL) 407253734Speter DELAY(1); 408253734Speter else { 409253734Speter td = mtx_owner(m); 410253734Speter 411253734Speter /* If the mutex is unlocked, try again. */ 412253734Speter if (td == NULL) 413253734Speter continue; 414253734Speter printf( 415253734Speter "spin lock %p (%s) held by %p (tid %d) too long\n", 416253734Speter m, m->mtx_object.lo_name, td, td->td_tid); 417253734Speter#ifdef WITNESS 418253734Speter witness_display_spinlock(&m->mtx_object, td); 419253734Speter#endif 420253734Speter panic("spin lock held too long"); 421253734Speter } 422253734Speter cpu_spinwait(); 423253734Speter } 424251875Speter spinlock_enter(); 425251875Speter } 426251875Speter 427251875Speter if (LOCK_LOG_TEST(&m->mtx_object, opts)) 428251875Speter CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 429251875Speter 430251875Speter return; 431} 432#endif /* SMP */ 433 434/* 435 * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 436 * 437 * We are only called here if the lock is recursed or contested (i.e. we 438 * need to wake up a blocked thread). 439 */ 440void 441_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 442{ 443 struct turnstile *ts; 444#ifndef PREEMPTION 445 struct thread *td, *td1; 446#endif 447 448 if (mtx_recursed(m)) { 449 if (--(m->mtx_recurse) == 0) 450 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 451 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 452 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 453 return; 454 } 455 456 turnstile_lock(&m->mtx_object); 457 ts = turnstile_lookup(&m->mtx_object); 458 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 459 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 460 461#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 462 if (ts == NULL) { 463 _release_lock_quick(m); 464 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 465 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 466 turnstile_release(&m->mtx_object); 467 return; 468 } 469#else 470 MPASS(ts != NULL); 471#endif 472#ifndef PREEMPTION 473 /* XXX */ 474 td1 = turnstile_head(ts, TS_EXCLUSIVE_QUEUE); 475#endif 476#ifdef MUTEX_WAKE_ALL 477 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 478 _release_lock_quick(m); 479#else 480 if (turnstile_signal(ts, TS_EXCLUSIVE_QUEUE)) { 481 _release_lock_quick(m); 482 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 483 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 484 } else { 485 m->mtx_lock = MTX_CONTESTED; 486 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 487 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p still contested", 488 m); 489 } 490#endif 491 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 492 493#ifndef PREEMPTION 494 /* 495 * XXX: This is just a hack until preemption is done. However, 496 * once preemption is done we need to either wrap the 497 * turnstile_signal() and release of the actual lock in an 498 * extra critical section or change the preemption code to 499 * always just set a flag and never do instant-preempts. 500 */ 501 td = curthread; 502 if (td->td_critnest > 0 || td1->td_priority >= td->td_priority) 503 return; 504 mtx_lock_spin(&sched_lock); 505 if (!TD_IS_RUNNING(td1)) { 506#ifdef notyet 507 if (td->td_ithd != NULL) { 508 struct ithd *it = td->td_ithd; 509 510 if (it->it_interrupted) { 511 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 512 CTR2(KTR_LOCK, 513 "_mtx_unlock_sleep: %p interrupted %p", 514 it, it->it_interrupted); 515 intr_thd_fixup(it); 516 } 517 } 518#endif 519 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 520 CTR2(KTR_LOCK, 521 "_mtx_unlock_sleep: %p switching out lock=%p", m, 522 (void *)m->mtx_lock); 523 524 mi_switch(SW_INVOL, NULL); 525 if (LOCK_LOG_TEST(&m->mtx_object, opts)) 526 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 527 m, (void *)m->mtx_lock); 528 } 529 mtx_unlock_spin(&sched_lock); 530#endif 531 532 return; 533} 534 535/* 536 * All the unlocking of MTX_SPIN locks is done inline. 537 * See the _rel_spin_lock() macro for the details. 538 */ 539 540/* 541 * The backing function for the INVARIANTS-enabled mtx_assert() 542 */ 543#ifdef INVARIANT_SUPPORT 544void 545_mtx_assert(struct mtx *m, int what, const char *file, int line) 546{ 547 548 if (panicstr != NULL || dumping) 549 return; 550 switch (what) { 551 case MA_OWNED: 552 case MA_OWNED | MA_RECURSED: 553 case MA_OWNED | MA_NOTRECURSED: 554 if (!mtx_owned(m)) 555 panic("mutex %s not owned at %s:%d", 556 m->mtx_object.lo_name, file, line); 557 if (mtx_recursed(m)) { 558 if ((what & MA_NOTRECURSED) != 0) 559 panic("mutex %s recursed at %s:%d", 560 m->mtx_object.lo_name, file, line); 561 } else if ((what & MA_RECURSED) != 0) { 562 panic("mutex %s unrecursed at %s:%d", 563 m->mtx_object.lo_name, file, line); 564 } 565 break; 566 case MA_NOTOWNED: 567 if (mtx_owned(m)) 568 panic("mutex %s owned at %s:%d", 569 m->mtx_object.lo_name, file, line); 570 break; 571 default: 572 panic("unknown mtx_assert at %s:%d", file, line); 573 } 574} 575#endif 576 577/* 578 * The MUTEX_DEBUG-enabled mtx_validate() 579 * 580 * Most of these checks have been moved off into the LO_INITIALIZED flag 581 * maintained by the witness code. 582 */ 583#ifdef MUTEX_DEBUG 584 585void mtx_validate(struct mtx *); 586 587void 588mtx_validate(struct mtx *m) 589{ 590 591/* 592 * XXX: When kernacc() does not require Giant we can reenable this check 593 */ 594#ifdef notyet 595 /* 596 * Can't call kernacc() from early init386(), especially when 597 * initializing Giant mutex, because some stuff in kernacc() 598 * requires Giant itself. 599 */ 600 if (!cold) 601 if (!kernacc((caddr_t)m, sizeof(m), 602 VM_PROT_READ | VM_PROT_WRITE)) 603 panic("Can't read and write to mutex %p", m); 604#endif 605} 606#endif 607 608/* 609 * General init routine used by the MTX_SYSINIT() macro. 610 */ 611void 612mtx_sysinit(void *arg) 613{ 614 struct mtx_args *margs = arg; 615 616 mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 617} 618 619/* 620 * Mutex initialization routine; initialize lock `m' of type contained in 621 * `opts' with options contained in `opts' and name `name.' The optional 622 * lock type `type' is used as a general lock category name for use with 623 * witness. 624 */ 625void 626mtx_init(struct mtx *m, const char *name, const char *type, int opts) 627{ 628 struct lock_class *class; 629 int flags; 630 631 MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 632 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); 633 634#ifdef MUTEX_DEBUG 635 /* Diagnostic and error correction */ 636 mtx_validate(m); 637#endif 638 639 /* Determine lock class and lock flags. */ 640 if (opts & MTX_SPIN) 641 class = &lock_class_mtx_spin; 642 else 643 class = &lock_class_mtx_sleep; 644 flags = 0; 645 if (opts & MTX_QUIET) 646 flags |= LO_QUIET; 647 if (opts & MTX_RECURSE) 648 flags |= LO_RECURSABLE; 649 if ((opts & MTX_NOWITNESS) == 0) 650 flags |= LO_WITNESS; 651 if (opts & MTX_DUPOK) 652 flags |= LO_DUPOK; 653 if (opts & MTX_NOPROFILE) 654 flags |= LO_NOPROFILE; 655 656 /* Initialize mutex. */ 657 m->mtx_lock = MTX_UNOWNED; 658 m->mtx_recurse = 0; 659 660 lock_profile_object_init(&m->mtx_object, class, name); 661 lock_init(&m->mtx_object, class, name, type, flags); 662} 663 664/* 665 * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 666 * passed in as a flag here because if the corresponding mtx_init() was 667 * called with MTX_QUIET set, then it will already be set in the mutex's 668 * flags. 669 */ 670void 671mtx_destroy(struct mtx *m) 672{ 673 674 if (!mtx_owned(m)) 675 MPASS(mtx_unowned(m)); 676 else { 677 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 678 679 /* Perform the non-mtx related part of mtx_unlock_spin(). */ 680 if (LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin) 681 spinlock_exit(); 682 else 683 curthread->td_locks--; 684 685 /* Tell witness this isn't locked to make it happy. */ 686 WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, 687 __LINE__); 688 } 689 690 m->mtx_lock = MTX_DESTROYED; 691 lock_profile_object_destroy(&m->mtx_object); 692 lock_destroy(&m->mtx_object); 693} 694 695/* 696 * Intialize the mutex code and system mutexes. This is called from the MD 697 * startup code prior to mi_startup(). The per-CPU data space needs to be 698 * setup before this is called. 699 */ 700void 701mutex_init(void) 702{ 703 704 /* Setup turnstiles so that sleep mutexes work. */ 705 init_turnstiles(); 706 707 /* 708 * Initialize mutexes. 709 */ 710 mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 711 mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 712 mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 713 mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 714 mtx_lock(&Giant); 715 716 lock_profile_init(); 717} 718 719#ifdef DDB 720void 721db_show_mtx(struct lock_object *lock) 722{ 723 struct thread *td; 724 struct mtx *m; 725 726 m = (struct mtx *)lock; 727 728 db_printf(" flags: {"); 729 if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 730 db_printf("SPIN"); 731 else 732 db_printf("DEF"); 733 if (m->mtx_object.lo_flags & LO_RECURSABLE) 734 db_printf(", RECURSE"); 735 if (m->mtx_object.lo_flags & LO_DUPOK) 736 db_printf(", DUPOK"); 737 db_printf("}\n"); 738 db_printf(" state: {"); 739 if (mtx_unowned(m)) 740 db_printf("UNOWNED"); 741 else { 742 db_printf("OWNED"); 743 if (m->mtx_lock & MTX_CONTESTED) 744 db_printf(", CONTESTED"); 745 if (m->mtx_lock & MTX_RECURSED) 746 db_printf(", RECURSED"); 747 } 748 db_printf("}\n"); 749 if (!mtx_unowned(m)) { 750 td = mtx_owner(m); 751 db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 752 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 753 if (mtx_recursed(m)) 754 db_printf(" recursed: %d\n", m->mtx_recurse); 755 } 756} 757#endif 758