kern_mutex.c revision 167368
1274955Ssvnmir/*- 2274955Ssvnmir * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3274955Ssvnmir * 4274955Ssvnmir * Redistribution and use in source and binary forms, with or without 5274955Ssvnmir * modification, are permitted provided that the following conditions 6274955Ssvnmir * are met: 7274955Ssvnmir * 1. Redistributions of source code must retain the above copyright 8274955Ssvnmir * notice, this list of conditions and the following disclaimer. 9274955Ssvnmir * 2. Redistributions in binary form must reproduce the above copyright 10274955Ssvnmir * notice, this list of conditions and the following disclaimer in the 11274955Ssvnmir * documentation and/or other materials provided with the distribution. 12274955Ssvnmir * 3. Berkeley Software Design Inc's name may not be used to endorse or 13274955Ssvnmir * promote products derived from this software without specific prior 14274955Ssvnmir * written permission. 15274955Ssvnmir * 16280031Sdim * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17274955Ssvnmir * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18274955Ssvnmir * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19288943Sdim * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20280031Sdim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21274955Ssvnmir * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22274955Ssvnmir * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23274955Ssvnmir * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24274955Ssvnmir * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25274955Ssvnmir * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26274955Ssvnmir * SUCH DAMAGE. 27274955Ssvnmir * 28274955Ssvnmir * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29280031Sdim * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30280031Sdim */ 31280031Sdim 32274955Ssvnmir/* 33274955Ssvnmir * Machine independent bits of mutex implementation. 34274955Ssvnmir */ 35274955Ssvnmir 36274955Ssvnmir#include <sys/cdefs.h> 37274955Ssvnmir__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 167368 2007-03-09 16:27:11Z jhb $"); 38274955Ssvnmir 39274955Ssvnmir#include "opt_adaptive_mutexes.h" 40274955Ssvnmir#include "opt_ddb.h" 41274955Ssvnmir#include "opt_global.h" 42274955Ssvnmir#include "opt_mutex_wake_all.h" 43274955Ssvnmir#include "opt_sched.h" 44296417Sdim 45296417Sdim#include <sys/param.h> 46296417Sdim#include <sys/systm.h> 47274955Ssvnmir#include <sys/bus.h> 48274955Ssvnmir#include <sys/conf.h> 49274955Ssvnmir#include <sys/kdb.h> 50274955Ssvnmir#include <sys/kernel.h> 51296417Sdim#include <sys/ktr.h> 52296417Sdim#include <sys/lock.h> 53296417Sdim#include <sys/malloc.h> 54274955Ssvnmir#include <sys/mutex.h> 55296417Sdim#include <sys/proc.h> 56296417Sdim#include <sys/resourcevar.h> 57274955Ssvnmir#include <sys/sched.h> 58296417Sdim#include <sys/sbuf.h> 59296417Sdim#include <sys/sysctl.h> 60296417Sdim#include <sys/turnstile.h> 61296417Sdim#include <sys/vmmeter.h> 62296417Sdim#include <sys/lock_profile.h> 63296417Sdim 64296417Sdim#include <machine/atomic.h> 65296417Sdim#include <machine/bus.h> 66296417Sdim#include <machine/cpu.h> 67296417Sdim 68296417Sdim#include <ddb/ddb.h> 69296417Sdim 70296417Sdim#include <fs/devfs/devfs_int.h> 71296417Sdim 72296417Sdim#include <vm/vm.h> 73296417Sdim#include <vm/vm_extern.h> 74296417Sdim 75296417Sdim/* 76296417Sdim * Force MUTEX_WAKE_ALL for now. 77296417Sdim * single thread wakeup needs fixes to avoid race conditions with 78296417Sdim * priority inheritance. 79296417Sdim */ 80296417Sdim#ifndef MUTEX_WAKE_ALL 81274955Ssvnmir#define MUTEX_WAKE_ALL 82274955Ssvnmir#endif 83296417Sdim 84296417Sdim/* 85296417Sdim * Internal utility macros. 86274955Ssvnmir */ 87274955Ssvnmir#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 88274955Ssvnmir 89296417Sdim#define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 90274955Ssvnmir 91274955Ssvnmir#ifdef DDB 92274955Ssvnmirstatic void db_show_mtx(struct lock_object *lock); 93274955Ssvnmir#endif 94274955Ssvnmirstatic void lock_mtx(struct lock_object *lock, int how); 95296417Sdimstatic void lock_spin(struct lock_object *lock, int how); 96274955Ssvnmirstatic int unlock_mtx(struct lock_object *lock); 97296417Sdimstatic int unlock_spin(struct lock_object *lock); 98296417Sdim 99296417Sdim/* 100296417Sdim * Lock classes for sleep and spin mutexes. 101296417Sdim */ 102296417Sdimstruct lock_class lock_class_mtx_sleep = { 103274955Ssvnmir .lc_name = "sleep mutex", 104274955Ssvnmir .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 105274955Ssvnmir#ifdef DDB 106274955Ssvnmir .lc_ddb_show = db_show_mtx, 107274955Ssvnmir#endif 108274955Ssvnmir .lc_lock = lock_mtx, 109296417Sdim .lc_unlock = unlock_mtx, 110296417Sdim}; 111274955Ssvnmirstruct lock_class lock_class_mtx_spin = { 112296417Sdim .lc_name = "spin mutex", 113296417Sdim .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 114296417Sdim#ifdef DDB 115296417Sdim .lc_ddb_show = db_show_mtx, 116296417Sdim#endif 117274955Ssvnmir .lc_lock = lock_spin, 118274955Ssvnmir .lc_unlock = unlock_spin, 119274955Ssvnmir}; 120274955Ssvnmir 121274955Ssvnmir/* 122296417Sdim * System-wide mutexes 123274955Ssvnmir */ 124274955Ssvnmirstruct mtx sched_lock; 125274955Ssvnmirstruct mtx Giant; 126274955Ssvnmir 127274955Ssvnmir#ifdef LOCK_PROFILING 128274955Ssvnmirstatic inline void lock_profile_init(void) 129274955Ssvnmir{ 130296417Sdim int i; 131296417Sdim /* Initialize the mutex profiling locks */ 132296417Sdim for (i = 0; i < LPROF_LOCK_SIZE; i++) { 133296417Sdim mtx_init(&lprof_locks[i], "mprof lock", 134274955Ssvnmir NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE); 135296417Sdim } 136274955Ssvnmir} 137296417Sdim#else 138296417Sdimstatic inline void lock_profile_init(void) {;} 139274955Ssvnmir#endif 140296417Sdim 141296417Sdimvoid 142274955Ssvnmirlock_mtx(struct lock_object *lock, int how) 143296417Sdim{ 144296417Sdim 145296417Sdim mtx_lock((struct mtx *)lock); 146296417Sdim} 147296417Sdim 148296417Sdimvoid 149296417Sdimlock_spin(struct lock_object *lock, int how) 150296417Sdim{ 151296417Sdim 152274955Ssvnmir panic("spin locks can only use msleep_spin"); 153274955Ssvnmir} 154274955Ssvnmir 155296417Sdimint 156274955Ssvnmirunlock_mtx(struct lock_object *lock) 157274955Ssvnmir{ 158274955Ssvnmir struct mtx *m; 159280031Sdim 160274955Ssvnmir m = (struct mtx *)lock; 161296417Sdim mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 162296417Sdim mtx_unlock(m); 163296417Sdim return (0); 164296417Sdim} 165274955Ssvnmir 166274955Ssvnmirint 167274955Ssvnmirunlock_spin(struct lock_object *lock) 168274955Ssvnmir{ 169274955Ssvnmir 170274955Ssvnmir panic("spin locks can only use msleep_spin"); 171296417Sdim} 172296417Sdim 173274955Ssvnmir/* 174274955Ssvnmir * Function versions of the inlined __mtx_* macros. These are used by 175274955Ssvnmir * modules and can also be called from assembly language if needed. 176274955Ssvnmir */ 177274955Ssvnmirvoid 178274955Ssvnmir_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 179274955Ssvnmir{ 180288943Sdim 181296417Sdim MPASS(curthread != NULL); 182296417Sdim KASSERT(m->mtx_lock != MTX_DESTROYED, 183296417Sdim ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 184296417Sdim KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 185288943Sdim ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 186274955Ssvnmir file, line)); 187274955Ssvnmir WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 188274955Ssvnmir file, line); 189296417Sdim 190296417Sdim _get_sleep_lock(m, curthread, opts, file, line); 191296417Sdim LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 192296417Sdim line); 193296417Sdim WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 194296417Sdim curthread->td_locks++; 195274955Ssvnmir} 196296417Sdim 197296417Sdimvoid 198296417Sdim_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 199296417Sdim{ 200296417Sdim MPASS(curthread != NULL); 201296417Sdim KASSERT(m->mtx_lock != MTX_DESTROYED, 202296417Sdim ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 203296417Sdim KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 204296417Sdim ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 205296417Sdim file, line)); 206296417Sdim curthread->td_locks--; 207296417Sdim WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 208296417Sdim LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 209296417Sdim line); 210296417Sdim mtx_assert(m, MA_OWNED); 211296417Sdim 212296417Sdim lock_profile_release_lock(&m->mtx_object); 213296417Sdim _rel_sleep_lock(m, curthread, opts, file, line); 214296417Sdim} 215296417Sdim 216296417Sdimvoid 217296417Sdim_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 218296417Sdim{ 219296417Sdim 220296417Sdim MPASS(curthread != NULL); 221296417Sdim KASSERT(m->mtx_lock != MTX_DESTROYED, 222296417Sdim ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 223296417Sdim KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, 224296417Sdim ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 225296417Sdim m->mtx_object.lo_name, file, line)); 226296417Sdim WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 227296417Sdim file, line); 228296417Sdim _get_spin_lock(m, curthread, opts, file, line); 229296417Sdim LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, 230296417Sdim line); 231296417Sdim WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 232296417Sdim} 233296417Sdim 234296417Sdimvoid 235296417Sdim_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 236296417Sdim{ 237296417Sdim 238296417Sdim MPASS(curthread != NULL); 239296417Sdim KASSERT(m->mtx_lock != MTX_DESTROYED, 240296417Sdim ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 241296417Sdim KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, 242296417Sdim ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 243296417Sdim m->mtx_object.lo_name, file, line)); 244296417Sdim WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); 245296417Sdim LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file, 246296417Sdim line); 247296417Sdim mtx_assert(m, MA_OWNED); 248296417Sdim 249296417Sdim lock_profile_release_lock(&m->mtx_object); 250296417Sdim _rel_spin_lock(m); 251296417Sdim} 252296417Sdim 253296417Sdim/* 254296417Sdim * The important part of mtx_trylock{,_flags}() 255296417Sdim * Tries to acquire lock `m.' If this function is called on a mutex that 256296417Sdim * is already owned, it will recursively acquire the lock. 257296417Sdim */ 258296417Sdimint 259296417Sdim_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 260296417Sdim{ 261296417Sdim int rval, contested = 0; 262296417Sdim uint64_t waittime = 0; 263296417Sdim 264296417Sdim MPASS(curthread != NULL); 265296417Sdim KASSERT(m->mtx_lock != MTX_DESTROYED, 266296417Sdim ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 267296417Sdim KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, 268296417Sdim ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, 269296417Sdim file, line)); 270296417Sdim 271274955Ssvnmir if (mtx_owned(m) && (m->mtx_object.lo_flags & LO_RECURSABLE) != 0) { 272274955Ssvnmir m->mtx_recurse++; 273296417Sdim atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 274296417Sdim rval = 1; 275296417Sdim } else 276296417Sdim rval = _obtain_lock(m, (uintptr_t)curthread); 277296417Sdim 278296417Sdim LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 279296417Sdim if (rval) { 280274955Ssvnmir WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 281296417Sdim file, line); 282296417Sdim curthread->td_locks++; 283296417Sdim if (m->mtx_recurse == 0) 284296417Sdim lock_profile_obtain_lock_success(&m->mtx_object, contested, 285274955Ssvnmir waittime, file, line); 286274955Ssvnmir 287274955Ssvnmir } 288274955Ssvnmir 289296417Sdim return (rval); 290296417Sdim} 291296417Sdim 292296417Sdim/* 293274955Ssvnmir * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 294274955Ssvnmir * 295274955Ssvnmir * We call this if the lock is either contested (i.e. we need to go to 296296417Sdim * sleep waiting for it), or if we need to recurse on it. 297296417Sdim */ 298296417Sdimvoid 299296417Sdim_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, 300274955Ssvnmir int line) 301274955Ssvnmir{ 302274955Ssvnmir#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 303274955Ssvnmir volatile struct thread *owner; 304288943Sdim#endif 305288943Sdim#ifdef KTR 306288943Sdim int cont_logged = 0; 307288943Sdim#endif 308288943Sdim uintptr_t v; 309288943Sdim 310288943Sdim if (mtx_owned(m)) { 311288943Sdim KASSERT((m->mtx_object.lo_flags & LO_RECURSABLE) != 0, 312288943Sdim ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 313288943Sdim m->mtx_object.lo_name, file, line)); 314288943Sdim m->mtx_recurse++; 315288943Sdim atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 316288943Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 317296417Sdim CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 318296417Sdim return; 319296417Sdim } 320296417Sdim 321288943Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 322288943Sdim CTR4(KTR_LOCK, 323288943Sdim "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 324288943Sdim m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 325288943Sdim 326288943Sdim while (!_obtain_lock(m, tid)) { 327288943Sdim turnstile_lock(&m->mtx_object); 328288943Sdim v = m->mtx_lock; 329288943Sdim 330288943Sdim /* 331288943Sdim * Check if the lock has been released while spinning for 332288943Sdim * the turnstile chain lock. 333288943Sdim */ 334288943Sdim if (v == MTX_UNOWNED) { 335288943Sdim turnstile_release(&m->mtx_object); 336288943Sdim cpu_spinwait(); 337296417Sdim continue; 338296417Sdim } 339296417Sdim 340296417Sdim#ifdef MUTEX_WAKE_ALL 341288943Sdim MPASS(v != MTX_CONTESTED); 342288943Sdim#else 343288943Sdim /* 344288943Sdim * The mutex was marked contested on release. This means that 345288943Sdim * there are other threads blocked on it. Grab ownership of 346296417Sdim * it and propagate its priority to the current thread if 347296417Sdim * necessary. 348296417Sdim */ 349296417Sdim if (v == MTX_CONTESTED) { 350296417Sdim m->mtx_lock = tid | MTX_CONTESTED; 351296417Sdim turnstile_claim(&m->mtx_object); 352296417Sdim break; 353296417Sdim } 354288943Sdim#endif 355288943Sdim 356288943Sdim /* 357274955Ssvnmir * If the mutex isn't already contested and a failure occurs 358274955Ssvnmir * setting the contested bit, the mutex was either released 359274955Ssvnmir * or the state of the MTX_RECURSED bit changed. 360274955Ssvnmir */ 361274955Ssvnmir if ((v & MTX_CONTESTED) == 0 && 362274955Ssvnmir !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 363274955Ssvnmir turnstile_release(&m->mtx_object); 364274955Ssvnmir cpu_spinwait(); 365274955Ssvnmir continue; 366274955Ssvnmir } 367274955Ssvnmir 368274955Ssvnmir#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 369274955Ssvnmir /* 370296417Sdim * If the current owner of the lock is executing on another 371296417Sdim * CPU, spin instead of blocking. 372296417Sdim */ 373296417Sdim owner = (struct thread *)(v & ~MTX_FLAGMASK); 374296417Sdim#ifdef ADAPTIVE_GIANT 375296417Sdim if (TD_IS_RUNNING(owner)) 376296417Sdim#else 377296417Sdim if (m != &Giant && TD_IS_RUNNING(owner)) 378274955Ssvnmir#endif 379274955Ssvnmir { 380274955Ssvnmir turnstile_release(&m->mtx_object); 381274955Ssvnmir while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) { 382274955Ssvnmir cpu_spinwait(); 383274955Ssvnmir } 384274955Ssvnmir continue; 385274955Ssvnmir } 386274955Ssvnmir#endif /* SMP && !NO_ADAPTIVE_MUTEXES */ 387274955Ssvnmir 388274955Ssvnmir /* 389274955Ssvnmir * We definitely must sleep for this lock. 390274955Ssvnmir */ 391274955Ssvnmir mtx_assert(m, MA_NOTOWNED); 392274955Ssvnmir 393274955Ssvnmir#ifdef KTR 394274955Ssvnmir if (!cont_logged) { 395274955Ssvnmir CTR6(KTR_CONTENTION, 396274955Ssvnmir "contention: %p at %s:%d wants %s, taken by %s:%d", 397274955Ssvnmir (void *)tid, file, line, m->mtx_object.lo_name, 398274955Ssvnmir WITNESS_FILE(&m->mtx_object), 399288943Sdim WITNESS_LINE(&m->mtx_object)); 400288943Sdim cont_logged = 1; 401288943Sdim } 402296417Sdim#endif 403296417Sdim 404296417Sdim /* 405296417Sdim * Block on the turnstile. 406296417Sdim */ 407296417Sdim turnstile_wait(&m->mtx_object, mtx_owner(m), 408296417Sdim TS_EXCLUSIVE_QUEUE); 409296417Sdim } 410296417Sdim#ifdef KTR 411296417Sdim if (cont_logged) { 412296417Sdim CTR4(KTR_CONTENTION, 413296417Sdim "contention end: %s acquired by %p at %s:%d", 414274955Ssvnmir m->mtx_object.lo_name, (void *)tid, file, line); 415274955Ssvnmir } 416274955Ssvnmir#endif 417296417Sdim return; 418296417Sdim} 419296417Sdim 420296417Sdim#ifdef SMP 421296417Sdim/* 422296417Sdim * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 423296417Sdim * 424296417Sdim * This is only called if we need to actually spin for the lock. Recursion 425296417Sdim * is handled inline. 426296417Sdim */ 427296417Sdimvoid 428296417Sdim_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, 429296417Sdim int line) 430296417Sdim{ 431296417Sdim int i = 0; 432296417Sdim struct thread *td; 433296417Sdim 434296417Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 435296417Sdim CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 436296417Sdim 437296417Sdim while (!_obtain_lock(m, tid)) { 438296417Sdim 439296417Sdim /* Give interrupts a chance while we spin. */ 440296417Sdim spinlock_exit(); 441296417Sdim while (m->mtx_lock != MTX_UNOWNED) { 442296417Sdim if (i++ < 10000000) { 443296417Sdim cpu_spinwait(); 444296417Sdim continue; 445296417Sdim } 446296417Sdim if (i < 60000000 || kdb_active || panicstr != NULL) 447274955Ssvnmir DELAY(1); 448274955Ssvnmir else { 449274955Ssvnmir td = mtx_owner(m); 450274955Ssvnmir 451274955Ssvnmir /* If the mutex is unlocked, try again. */ 452274955Ssvnmir if (td == NULL) 453274955Ssvnmir continue; 454274955Ssvnmir printf( 455274955Ssvnmir "spin lock %p (%s) held by %p (tid %d) too long\n", 456274955Ssvnmir m, m->mtx_object.lo_name, td, td->td_tid); 457296417Sdim#ifdef WITNESS 458296417Sdim witness_display_spinlock(&m->mtx_object, td); 459296417Sdim#endif 460296417Sdim panic("spin lock held too long"); 461274955Ssvnmir } 462274955Ssvnmir cpu_spinwait(); 463274955Ssvnmir } 464274955Ssvnmir spinlock_enter(); 465274955Ssvnmir } 466274955Ssvnmir 467274955Ssvnmir if (LOCK_LOG_TEST(&m->mtx_object, opts)) 468274955Ssvnmir CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 469274955Ssvnmir 470274955Ssvnmir return; 471296417Sdim} 472296417Sdim#endif /* SMP */ 473296417Sdim 474296417Sdim/* 475274955Ssvnmir * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 476274955Ssvnmir * 477274955Ssvnmir * We are only called here if the lock is recursed or contested (i.e. we 478274955Ssvnmir * need to wake up a blocked thread). 479288943Sdim */ 480288943Sdimvoid 481296417Sdim_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 482296417Sdim{ 483296417Sdim struct turnstile *ts; 484296417Sdim#ifndef PREEMPTION 485296417Sdim struct thread *td, *td1; 486296417Sdim#endif 487296417Sdim 488296417Sdim if (mtx_recursed(m)) { 489296417Sdim if (--(m->mtx_recurse) == 0) 490296417Sdim atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 491296417Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 492296417Sdim CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 493296417Sdim return; 494296417Sdim } 495296417Sdim 496296417Sdim turnstile_lock(&m->mtx_object); 497296417Sdim ts = turnstile_lookup(&m->mtx_object); 498296417Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 499296417Sdim CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 500296417Sdim 501296417Sdim#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 502296417Sdim if (ts == NULL) { 503274955Ssvnmir _release_lock_quick(m); 504274955Ssvnmir if (LOCK_LOG_TEST(&m->mtx_object, opts)) 505274955Ssvnmir CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 506274955Ssvnmir turnstile_release(&m->mtx_object); 507274955Ssvnmir return; 508274955Ssvnmir } 509274955Ssvnmir#else 510274955Ssvnmir MPASS(ts != NULL); 511274955Ssvnmir#endif 512274955Ssvnmir#ifndef PREEMPTION 513274955Ssvnmir /* XXX */ 514274955Ssvnmir td1 = turnstile_head(ts, TS_EXCLUSIVE_QUEUE); 515274955Ssvnmir#endif 516296417Sdim#ifdef MUTEX_WAKE_ALL 517296417Sdim turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 518296417Sdim _release_lock_quick(m); 519296417Sdim#else 520274955Ssvnmir if (turnstile_signal(ts, TS_EXCLUSIVE_QUEUE)) { 521274955Ssvnmir _release_lock_quick(m); 522274955Ssvnmir if (LOCK_LOG_TEST(&m->mtx_object, opts)) 523274955Ssvnmir CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 524274955Ssvnmir } else { 525274955Ssvnmir m->mtx_lock = MTX_CONTESTED; 526274955Ssvnmir if (LOCK_LOG_TEST(&m->mtx_object, opts)) 527274955Ssvnmir CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p still contested", 528274955Ssvnmir m); 529274955Ssvnmir } 530296417Sdim#endif 531296417Sdim turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 532296417Sdim 533296417Sdim#ifndef PREEMPTION 534274955Ssvnmir /* 535274955Ssvnmir * XXX: This is just a hack until preemption is done. However, 536274955Ssvnmir * once preemption is done we need to either wrap the 537274955Ssvnmir * turnstile_signal() and release of the actual lock in an 538288943Sdim * extra critical section or change the preemption code to 539288943Sdim * always just set a flag and never do instant-preempts. 540296417Sdim */ 541296417Sdim td = curthread; 542296417Sdim if (td->td_critnest > 0 || td1->td_priority >= td->td_priority) 543296417Sdim return; 544296417Sdim mtx_lock_spin(&sched_lock); 545296417Sdim if (!TD_IS_RUNNING(td1)) { 546296417Sdim#ifdef notyet 547296417Sdim if (td->td_ithd != NULL) { 548296417Sdim struct ithd *it = td->td_ithd; 549296417Sdim 550296417Sdim if (it->it_interrupted) { 551296417Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 552296417Sdim CTR2(KTR_LOCK, 553296417Sdim "_mtx_unlock_sleep: %p interrupted %p", 554296417Sdim it, it->it_interrupted); 555296417Sdim intr_thd_fixup(it); 556296417Sdim } 557296417Sdim } 558296417Sdim#endif 559296417Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 560296417Sdim CTR2(KTR_LOCK, 561296417Sdim "_mtx_unlock_sleep: %p switching out lock=%p", m, 562274955Ssvnmir (void *)m->mtx_lock); 563274955Ssvnmir 564274955Ssvnmir mi_switch(SW_INVOL, NULL); 565296417Sdim if (LOCK_LOG_TEST(&m->mtx_object, opts)) 566296417Sdim CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 567296417Sdim m, (void *)m->mtx_lock); 568296417Sdim } 569296417Sdim mtx_unlock_spin(&sched_lock); 570296417Sdim#endif 571296417Sdim 572296417Sdim return; 573296417Sdim} 574296417Sdim 575296417Sdim/* 576296417Sdim * All the unlocking of MTX_SPIN locks is done inline. 577296417Sdim * See the _rel_spin_lock() macro for the details. 578296417Sdim */ 579296417Sdim 580296417Sdim/* 581296417Sdim * The backing function for the INVARIANTS-enabled mtx_assert() 582296417Sdim */ 583296417Sdim#ifdef INVARIANT_SUPPORT 584296417Sdimvoid 585296417Sdim_mtx_assert(struct mtx *m, int what, const char *file, int line) 586296417Sdim{ 587296417Sdim 588296417Sdim if (panicstr != NULL || dumping) 589296417Sdim return; 590296417Sdim switch (what) { 591296417Sdim case MA_OWNED: 592296417Sdim case MA_OWNED | MA_RECURSED: 593296417Sdim case MA_OWNED | MA_NOTRECURSED: 594296417Sdim if (!mtx_owned(m)) 595296417Sdim panic("mutex %s not owned at %s:%d", 596296417Sdim m->mtx_object.lo_name, file, line); 597296417Sdim if (mtx_recursed(m)) { 598296417Sdim if ((what & MA_NOTRECURSED) != 0) 599296417Sdim panic("mutex %s recursed at %s:%d", 600296417Sdim m->mtx_object.lo_name, file, line); 601296417Sdim } else if ((what & MA_RECURSED) != 0) { 602296417Sdim panic("mutex %s unrecursed at %s:%d", 603296417Sdim m->mtx_object.lo_name, file, line); 604296417Sdim } 605296417Sdim break; 606296417Sdim case MA_NOTOWNED: 607296417Sdim if (mtx_owned(m)) 608296417Sdim panic("mutex %s owned at %s:%d", 609296417Sdim m->mtx_object.lo_name, file, line); 610296417Sdim break; 611296417Sdim default: 612296417Sdim panic("unknown mtx_assert at %s:%d", file, line); 613296417Sdim } 614296417Sdim} 615296417Sdim#endif 616274955Ssvnmir 617274955Ssvnmir/* 618274955Ssvnmir * The MUTEX_DEBUG-enabled mtx_validate() 619296417Sdim * 620274955Ssvnmir * Most of these checks have been moved off into the LO_INITIALIZED flag 621274955Ssvnmir * maintained by the witness code. 622274955Ssvnmir */ 623274955Ssvnmir#ifdef MUTEX_DEBUG 624274955Ssvnmir 625274955Ssvnmirvoid mtx_validate(struct mtx *); 626274955Ssvnmir 627274955Ssvnmirvoid 628274955Ssvnmirmtx_validate(struct mtx *m) 629296417Sdim{ 630288943Sdim 631288943Sdim/* 632296417Sdim * XXX: When kernacc() does not require Giant we can reenable this check 633296417Sdim */ 634274955Ssvnmir#ifdef notyet 635296417Sdim /* 636288943Sdim * Can't call kernacc() from early init386(), especially when 637274955Ssvnmir * initializing Giant mutex, because some stuff in kernacc() 638274955Ssvnmir * requires Giant itself. 639274955Ssvnmir */ 640274955Ssvnmir if (!cold) 641274955Ssvnmir if (!kernacc((caddr_t)m, sizeof(m), 642296417Sdim VM_PROT_READ | VM_PROT_WRITE)) 643296417Sdim panic("Can't read and write to mutex %p", m); 644274955Ssvnmir#endif 645274955Ssvnmir} 646274955Ssvnmir#endif 647296417Sdim 648296417Sdim/* 649274955Ssvnmir * General init routine used by the MTX_SYSINIT() macro. 650274955Ssvnmir */ 651288943Sdimvoid 652288943Sdimmtx_sysinit(void *arg) 653288943Sdim{ 654288943Sdim struct mtx_args *margs = arg; 655288943Sdim 656274955Ssvnmir mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 657274955Ssvnmir} 658274955Ssvnmir 659274955Ssvnmir/* 660274955Ssvnmir * Mutex initialization routine; initialize lock `m' of type contained in 661296417Sdim * `opts' with options contained in `opts' and name `name.' The optional 662296417Sdim * lock type `type' is used as a general lock category name for use with 663296417Sdim * witness. 664296417Sdim */ 665296417Sdimvoid 666296417Sdimmtx_init(struct mtx *m, const char *name, const char *type, int opts) 667296417Sdim{ 668296417Sdim struct lock_class *class; 669296417Sdim int flags; 670296417Sdim 671296417Sdim MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 672296417Sdim MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); 673296417Sdim 674296417Sdim#ifdef MUTEX_DEBUG 675296417Sdim /* Diagnostic and error correction */ 676296417Sdim mtx_validate(m); 677296417Sdim#endif 678296417Sdim 679296417Sdim /* Determine lock class and lock flags. */ 680296417Sdim if (opts & MTX_SPIN) 681296417Sdim class = &lock_class_mtx_spin; 682296417Sdim else 683296417Sdim class = &lock_class_mtx_sleep; 684296417Sdim flags = 0; 685296417Sdim if (opts & MTX_QUIET) 686296417Sdim flags |= LO_QUIET; 687296417Sdim if (opts & MTX_RECURSE) 688296417Sdim flags |= LO_RECURSABLE; 689296417Sdim if ((opts & MTX_NOWITNESS) == 0) 690296417Sdim flags |= LO_WITNESS; 691296417Sdim if (opts & MTX_DUPOK) 692296417Sdim flags |= LO_DUPOK; 693296417Sdim if (opts & MTX_NOPROFILE) 694296417Sdim flags |= LO_NOPROFILE; 695296417Sdim 696296417Sdim /* Initialize mutex. */ 697296417Sdim m->mtx_lock = MTX_UNOWNED; 698296417Sdim m->mtx_recurse = 0; 699296417Sdim 700296417Sdim lock_profile_object_init(&m->mtx_object, class, name); 701296417Sdim lock_init(&m->mtx_object, class, name, type, flags); 702296417Sdim} 703296417Sdim 704296417Sdim/* 705296417Sdim * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 706296417Sdim * passed in as a flag here because if the corresponding mtx_init() was 707296417Sdim * called with MTX_QUIET set, then it will already be set in the mutex's 708296417Sdim * flags. 709296417Sdim */ 710296417Sdimvoid 711296417Sdimmtx_destroy(struct mtx *m) 712296417Sdim{ 713296417Sdim 714296417Sdim if (!mtx_owned(m)) 715296417Sdim MPASS(mtx_unowned(m)); 716296417Sdim else { 717296417Sdim MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 718296417Sdim 719296417Sdim /* Perform the non-mtx related part of mtx_unlock_spin(). */ 720296417Sdim if (LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin) 721296417Sdim spinlock_exit(); 722296417Sdim else 723296417Sdim curthread->td_locks--; 724296417Sdim 725296417Sdim /* Tell witness this isn't locked to make it happy. */ 726296417Sdim WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__, 727296417Sdim __LINE__); 728296417Sdim } 729296417Sdim 730296417Sdim m->mtx_lock = MTX_DESTROYED; 731296417Sdim lock_profile_object_destroy(&m->mtx_object); 732296417Sdim lock_destroy(&m->mtx_object); 733296417Sdim} 734296417Sdim 735296417Sdim/* 736296417Sdim * Intialize the mutex code and system mutexes. This is called from the MD 737296417Sdim * startup code prior to mi_startup(). The per-CPU data space needs to be 738296417Sdim * setup before this is called. 739296417Sdim */ 740296417Sdimvoid 741296417Sdimmutex_init(void) 742296417Sdim{ 743296417Sdim 744296417Sdim /* Setup turnstiles so that sleep mutexes work. */ 745296417Sdim init_turnstiles(); 746296417Sdim 747296417Sdim /* 748296417Sdim * Initialize mutexes. 749296417Sdim */ 750296417Sdim mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 751296417Sdim mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 752296417Sdim mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 753296417Sdim mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 754296417Sdim mtx_lock(&Giant); 755296417Sdim 756296417Sdim lock_profile_init(); 757296417Sdim} 758296417Sdim 759296417Sdim#ifdef DDB 760274955Ssvnmirvoid 761296417Sdimdb_show_mtx(struct lock_object *lock) 762296417Sdim{ 763296417Sdim struct thread *td; 764296417Sdim struct mtx *m; 765296417Sdim 766296417Sdim m = (struct mtx *)lock; 767296417Sdim 768296417Sdim db_printf(" flags: {"); 769296417Sdim if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 770296417Sdim db_printf("SPIN"); 771296417Sdim else 772296417Sdim db_printf("DEF"); 773296417Sdim if (m->mtx_object.lo_flags & LO_RECURSABLE) 774296417Sdim db_printf(", RECURSE"); 775296417Sdim if (m->mtx_object.lo_flags & LO_DUPOK) 776296417Sdim db_printf(", DUPOK"); 777296417Sdim db_printf("}\n"); 778296417Sdim db_printf(" state: {"); 779296417Sdim if (mtx_unowned(m)) 780296417Sdim db_printf("UNOWNED"); 781296417Sdim else { 782296417Sdim db_printf("OWNED"); 783296417Sdim if (m->mtx_lock & MTX_CONTESTED) 784296417Sdim db_printf(", CONTESTED"); 785296417Sdim if (m->mtx_lock & MTX_RECURSED) 786274955Ssvnmir db_printf(", RECURSED"); 787274955Ssvnmir } 788274955Ssvnmir db_printf("}\n"); 789274955Ssvnmir if (!mtx_unowned(m)) { 790274955Ssvnmir td = mtx_owner(m); 791274955Ssvnmir db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 792274955Ssvnmir td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 793274955Ssvnmir if (mtx_recursed(m)) 794274955Ssvnmir db_printf(" recursed: %d\n", m->mtx_recurse); 795274955Ssvnmir } 796274955Ssvnmir} 797288943Sdim#endif 798288943Sdim