subr_lock.c revision 164159
153913Sarchie/*- 253913Sarchie * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 353913Sarchie * All rights reserved. 453913Sarchie * 553913Sarchie * Redistribution and use in source and binary forms, with or without 653913Sarchie * modification, are permitted provided that the following conditions 753913Sarchie * are met: 853913Sarchie * 1. Redistributions of source code must retain the above copyright 953913Sarchie * notice, this list of conditions and the following disclaimer. 1053913Sarchie * 2. Redistributions in binary form must reproduce the above copyright 1153913Sarchie * notice, this list of conditions and the following disclaimer in the 1253913Sarchie * documentation and/or other materials provided with the distribution. 1353913Sarchie * 3. Neither the name of the author nor the names of any co-contributors 1453913Sarchie * may be used to endorse or promote products derived from this software 1553913Sarchie * without specific prior written permission. 1653913Sarchie * 1753913Sarchie * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1853913Sarchie * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1953913Sarchie * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2053913Sarchie * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2153913Sarchie * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2253913Sarchie * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2353913Sarchie * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2453913Sarchie * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2553913Sarchie * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2653913Sarchie * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2753913Sarchie * SUCH DAMAGE. 2853913Sarchie */ 2953913Sarchie 3053913Sarchie/* 3153913Sarchie * This module holds the global variables and functions used to maintain 3253913Sarchie * lock_object structures. 3353913Sarchie */ 3453913Sarchie 3553913Sarchie#include <sys/cdefs.h> 3653913Sarchie__FBSDID("$FreeBSD: head/sys/kern/subr_lock.c 164159 2006-11-11 03:18:07Z kmacy $"); 3767506Sjulian 3853913Sarchie#include "opt_ddb.h" 3953913Sarchie#include "opt_mprof.h" 4053913Sarchie 4153913Sarchie#include <sys/param.h> 4253913Sarchie#include <sys/systm.h> 4353913Sarchie#include <sys/ktr.h> 4453913Sarchie#include <sys/linker_set.h> 4553913Sarchie#include <sys/lock.h> 4653913Sarchie#include <sys/sbuf.h> 4753913Sarchie#include <sys/sysctl.h> 4853913Sarchie#include <sys/lock_profile.h> 4953913Sarchie 5053998Sarchie#ifdef DDB 5153998Sarchie#include <ddb/ddb.h> 5253998Sarchie#endif 5353913Sarchie 5453998SarchieCTASSERT(LOCK_CLASS_MAX == 15); 5553998Sarchie 5653998Sarchiestruct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { 5753998Sarchie &lock_class_mtx_spin, 5853998Sarchie &lock_class_mtx_sleep, 5953998Sarchie &lock_class_sx, 6053998Sarchie &lock_class_rw, 6153998Sarchie}; 6253998Sarchie 6353913Sarchie#ifdef LOCK_PROFILING 6453913Sarchie#include <machine/cpufunc.h> 6553913Sarchie 6653913SarchieSYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging"); 6753913SarchieSYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, "lock profiling"); 6853913Sarchieint lock_prof_enable = 0; 6953913SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, enable, CTLFLAG_RW, 7053913Sarchie &lock_prof_enable, 0, "Enable lock profiling"); 7153913Sarchie 7253913Sarchie/* 7353913Sarchie * lprof_buf is a static pool of profiling records to avoid possible 7453913Sarchie * reentrance of the memory allocation functions. 7553913Sarchie * 7653913Sarchie * Note: NUM_LPROF_BUFFERS must be smaller than LPROF_HASH_SIZE. 7753913Sarchie */ 7853913Sarchiestruct lock_prof lprof_buf[LPROF_HASH_SIZE]; 7953913Sarchiestatic int allocated_lprof_buf; 8053913Sarchiestruct mtx lprof_locks[LPROF_LOCK_SIZE]; 8153913Sarchie 8253913Sarchie 8353913Sarchie/* SWAG: sbuf size = avg stat. line size * number of locks */ 8453913Sarchie#define LPROF_SBUF_SIZE 256 * 400 8553913Sarchie 8653913Sarchiestatic int lock_prof_acquisitions; 8753913SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, acquisitions, CTLFLAG_RD, 8853998Sarchie &lock_prof_acquisitions, 0, "Number of mutex acquistions recorded"); 8953913Sarchiestatic int lock_prof_records; 9053913SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, records, CTLFLAG_RD, 9153913Sarchie &lock_prof_records, 0, "Number of profiling records"); 9253913Sarchiestatic int lock_prof_maxrecords = LPROF_HASH_SIZE; 9353998SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, maxrecords, CTLFLAG_RD, 9453998Sarchie &lock_prof_maxrecords, 0, "Maximum number of profiling records"); 9553998Sarchiestatic int lock_prof_rejected; 9653998SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD, 9753913Sarchie &lock_prof_rejected, 0, "Number of rejected profiling records"); 9853998Sarchiestatic int lock_prof_hashsize = LPROF_HASH_SIZE; 9953998SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, hashsize, CTLFLAG_RD, 10053913Sarchie &lock_prof_hashsize, 0, "Hash size"); 10153998Sarchiestatic int lock_prof_collisions = 0; 10253998SarchieSYSCTL_INT(_debug_lock_prof, OID_AUTO, collisions, CTLFLAG_RD, 10353998Sarchie &lock_prof_collisions, 0, "Number of hash collisions"); 10453998Sarchie 10553998Sarchie#ifndef USE_CPU_NANOSECONDS 10653998Sarchiestatic u_int64_t 10753998Sarchienanoseconds(void) 10853913Sarchie{ 10953998Sarchie struct timespec tv; 11053998Sarchie 11153913Sarchie nanotime(&tv); 11253998Sarchie return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec); 11353998Sarchie} 11453998Sarchie#endif 11553998Sarchie 11653998Sarchiestatic int 11753998Sarchiedump_lock_prof_stats(SYSCTL_HANDLER_ARGS) 11853998Sarchie{ 11953998Sarchie struct sbuf *sb; 12053998Sarchie int error, i; 12153998Sarchie static int multiplier = 1; 12253998Sarchie const char *p; 12353998Sarchie 12453998Sarchie if (allocated_lprof_buf == 0) 12553998Sarchie return (SYSCTL_OUT(req, "No locking recorded", 12653998Sarchie sizeof("No locking recorded"))); 12753998Sarchie 12853998Sarchieretry_sbufops: 12958011Sarchie sb = sbuf_new(NULL, NULL, LPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN); 13053998Sarchie sbuf_printf(sb, "\n%6s %12s %12s %11s %5s %5s %12s %12s %s\n", 13153998Sarchie "max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cn\ 13253998Sarchiet_lock", "name"); 13353998Sarchie for (i = 0; i < LPROF_HASH_SIZE; ++i) { 13453998Sarchie if (lprof_buf[i].name == NULL) 13553998Sarchie continue; 13653998Sarchie for (p = lprof_buf[i].file; 13753998Sarchie p != NULL && strncmp(p, "../", 3) == 0; p += 3) 13853998Sarchie /* nothing */ ; 13953998Sarchie sbuf_printf(sb, "%6ju %12ju %12ju %11ju %5ju %5ju %12ju %12ju %s:%d (\ 14053998Sarchie%s)\n", 14153998Sarchie lprof_buf[i].cnt_max / 1000, 14253998Sarchie lprof_buf[i].cnt_tot / 1000, 14353998Sarchie lprof_buf[i].cnt_wait / 1000, 14453998Sarchie lprof_buf[i].cnt_cur, 14553998Sarchie lprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 : 14653998Sarchie lprof_buf[i].cnt_tot / (lprof_buf[i].cnt_cur * 1000), 14753998Sarchie lprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 : 14853998Sarchie lprof_buf[i].cnt_wait / (lprof_buf[i].cnt_cur * 1000), 14953998Sarchie lprof_buf[i].cnt_contest_holding, 15053998Sarchie lprof_buf[i].cnt_contest_locking, 15153998Sarchie p, lprof_buf[i].line, lprof_buf[i].name); 15253998Sarchie if (sbuf_overflowed(sb)) { 15353998Sarchie sbuf_delete(sb); 15453998Sarchie multiplier++; 15553998Sarchie goto retry_sbufops; 15653998Sarchie } 15753998Sarchie } 15853998Sarchie 15953998Sarchie sbuf_finish(sb); 16053998Sarchie error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 16153998Sarchie sbuf_delete(sb); 16253998Sarchie return (error); 16353998Sarchie} 16453998Sarchiestatic int 16553998Sarchiereset_lock_prof_stats(SYSCTL_HANDLER_ARGS) 16664506Sarchie{ 16764506Sarchie int error, v; 16853998Sarchie 16953998Sarchie if (allocated_lprof_buf == 0) 17064506Sarchie return (0); 17153998Sarchie 17253998Sarchie v = 0; 17364506Sarchie error = sysctl_handle_int(oidp, &v, 0, req); 17453998Sarchie if (error) 17553998Sarchie return (error); 17653998Sarchie if (req->newptr == NULL) 17753998Sarchie return (error); 17853998Sarchie if (v == 0) 17953998Sarchie return (0); 18053998Sarchie 18153998Sarchie bzero(lprof_buf, LPROF_HASH_SIZE*sizeof(*lprof_buf)); 18253913Sarchie allocated_lprof_buf = 0; 18353998Sarchie return (0); 18453998Sarchie} 18553998Sarchie 18653998SarchieSYSCTL_PROC(_debug_lock_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD, 18753913Sarchie NULL, 0, dump_lock_prof_stats, "A", "Mutex profiling statistics"); 18853998Sarchie 18953998SarchieSYSCTL_PROC(_debug_lock_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW, 19053998Sarchie NULL, 0, reset_lock_prof_stats, "I", "Reset mutex profiling statistics"); 19153913Sarchie#endif 19253913Sarchie 19353913Sarchievoid 19453913Sarchielock_init(struct lock_object *lock, struct lock_class *class, const char *name, 19553913Sarchie const char *type, int flags) 19653913Sarchie{ 19753913Sarchie int i; 19853913Sarchie 19953998Sarchie /* Check for double-init and zero object. */ 20053998Sarchie KASSERT(!lock_initalized(lock), ("lock \"%s\" %p already initialized", 20153913Sarchie name, lock)); 20253913Sarchie 20353913Sarchie /* Look up lock class to find its index. */ 20453913Sarchie for (i = 0; i < LOCK_CLASS_MAX; i++) 20553913Sarchie if (lock_classes[i] == class) { 20653913Sarchie lock->lo_flags = i << LO_CLASSSHIFT; 20753913Sarchie break; 20853913Sarchie } 20953913Sarchie KASSERT(i < LOCK_CLASS_MAX, ("unknown lock class %p", class)); 21053913Sarchie 21153913Sarchie /* Initialize the lock object. */ 21253913Sarchie lock->lo_name = name; 21353913Sarchie lock->lo_type = type != NULL ? type : name; 21453913Sarchie lock->lo_flags |= flags | LO_INITIALIZED; 21553913Sarchie LOCK_LOG_INIT(lock, 0); 21653913Sarchie WITNESS_INIT(lock); 21753913Sarchie} 21853913Sarchie 21953913Sarchievoid 22053913Sarchielock_destroy(struct lock_object *lock) 22153913Sarchie{ 22253913Sarchie 22353913Sarchie KASSERT(lock_initalized(lock), ("lock %p is not initialized", lock)); 22453913Sarchie WITNESS_DESTROY(lock); 22553913Sarchie LOCK_LOG_DESTROY(lock, 0); 22653913Sarchie lock->lo_flags &= ~LO_INITIALIZED; 22753913Sarchie} 22853913Sarchie 22953913Sarchie#ifdef DDB 23053913SarchieDB_SHOW_COMMAND(lock, db_show_lock) 23153913Sarchie{ 23253913Sarchie struct lock_object *lock; 23353913Sarchie struct lock_class *class; 23453913Sarchie 23553913Sarchie if (!have_addr) 23653913Sarchie return; 23753913Sarchie lock = (struct lock_object *)addr; 23853913Sarchie if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) { 23953913Sarchie db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock)); 24053913Sarchie return; 24153913Sarchie } 24253913Sarchie class = LOCK_CLASS(lock); 24353913Sarchie db_printf(" class: %s\n", class->lc_name); 24453913Sarchie db_printf(" name: %s\n", lock->lo_name); 24553913Sarchie if (lock->lo_type && lock->lo_type != lock->lo_name) 24653913Sarchie db_printf(" type: %s\n", lock->lo_type); 24753913Sarchie class->lc_ddb_show(lock); 24853913Sarchie} 24953913Sarchie#endif 25053913Sarchie 25153913Sarchie#ifdef LOCK_PROFILING 25253913Sarchievoid _lock_profile_obtain_lock_success(struct lock_object *lo, uint64_t waittime, con\ 25353913Sarchiest char *file, int line) 25453913Sarchie{ 25553913Sarchie struct lock_profile_object *l = &lo->lo_profile_obj; 25653913Sarchie 25753913Sarchie /* don't reset the timer when/if recursing */ 25853913Sarchie if (l->lpo_acqtime == 0) { 25953913Sarchie l->lpo_filename = file; 26053913Sarchie l->lpo_lineno = line; 26153913Sarchie l->lpo_acqtime = nanoseconds(); 26253913Sarchie if (waittime) { 26353913Sarchie if (l->lpo_acqtime > waittime) 26453913Sarchie l->lpo_waittime = l->lpo_acqtime - waittime; 26553913Sarchie } 26653913Sarchie } 26753913Sarchie} 26853913Sarchie 26953913Sarchievoid _lock_profile_update_wait(struct lock_object *lo, uint64_t waitstart) 27053998Sarchie{ 27153998Sarchie struct lock_profile_object *l = &lo->lo_profile_obj; 27253998Sarchie 27353998Sarchie if (lock_prof_enable && waitstart) { 27453998Sarchie uint64_t now, waittime; 27553998Sarchie struct lock_prof *mpp; 27653998Sarchie u_int hash; 27753998Sarchie const char *p = l->lpo_filename; 27853998Sarchie int collision = 0; 27953998Sarchie now = nanoseconds(); 28053913Sarchie if (now < waitstart) 28153913Sarchie return; 28253913Sarchie waittime = now - waitstart; 28353913Sarchie hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & LPROF_HASH_MASK; 28453913Sarchie 28553913Sarchie mpp = &lprof_buf[hash]; 28653913Sarchie while (mpp->name != NULL) { 28753913Sarchie if (mpp->line == l->lpo_lineno && 28853913Sarchie mpp->file == p && 28953913Sarchie mpp->namehash == l->lpo_namehash) 29053913Sarchie break; 29153913Sarchie /* If the lprof_hash entry is allocated to someone else, try the next one */ 29253913Sarchie collision = 1; 29353913Sarchie CTR4(KTR_SPARE1, "Hash collision, %s:%d %s(%x)", mpp->file, mpp->line, mpp->name, mpp->namehash); 29453913Sarchie hash = (hash + 1) & LPROF_HASH_MASK; 29553913Sarchie mpp = &lprof_buf[hash]; 29653998Sarchie } 29753913Sarchie if (mpp->name == NULL) { 29853998Sarchie int buf; 29953998Sarchie 30053998Sarchie buf = atomic_fetchadd_int(&allocated_lprof_buf, 1); 30153998Sarchie /* Just exit if we cannot get a trace buffer */ 30253913Sarchie if (buf >= LPROF_HASH_SIZE) { 30353913Sarchie ++lock_prof_rejected; 30453913Sarchie return; 30553913Sarchie } 30653913Sarchie mpp->file = p; 30753913Sarchie mpp->line = l->lpo_lineno; 30853913Sarchie mpp->name = lo->lo_name; 30953913Sarchie mpp->namehash = l->lpo_namehash; 31053913Sarchie if (collision) 31153913Sarchie ++lock_prof_collisions; 31253913Sarchie /* We might have raced someone else but who cares, they'll try again next time */ 31353913Sarchie ++lock_prof_records; 31453913Sarchie } 31553913Sarchie LPROF_LOCK(hash); 31653913Sarchie mpp->cnt_wait += waittime; 31753913Sarchie LPROF_UNLOCK(hash); 31853913Sarchie } 31953913Sarchie} 32053913Sarchie 32153998Sarchievoid _lock_profile_release_lock(struct lock_object *lo) 32253913Sarchie{ 32353998Sarchie struct lock_profile_object *l = &lo->lo_profile_obj; 32453998Sarchie 32553998Sarchie if (l->lpo_acqtime && !(lo->lo_flags & LO_NOPROFILE)) { 32653913Sarchie const char *unknown = "(unknown)"; 32753913Sarchie u_int64_t acqtime, now, waittime; 32853913Sarchie struct lock_prof *mpp; 32953913Sarchie u_int hash; 33053998Sarchie const char *p = l->lpo_filename; 33153998Sarchie int collision = 0; 33253998Sarchie 33353998Sarchie now = nanoseconds(); 33453998Sarchie acqtime = l->lpo_acqtime; 33553998Sarchie waittime = l->lpo_waittime; 33653998Sarchie if (now <= acqtime) 33753913Sarchie return; 33853913Sarchie if (p == NULL || *p == '\0') 33953913Sarchie p = unknown; 34053913Sarchie hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & LPROF_HASH_MASK; 34153913Sarchie CTR5(KTR_SPARE1, "Hashing %s(%x) %s:%d to %d", l->lpo_name, 34253913Sarchie l->lpo_namehash, p, l->lpo_lineno, hash); 34353913Sarchie mpp = &lprof_buf[hash]; 34453913Sarchie while (mpp->name != NULL) { 34553913Sarchie if (mpp->line == l->lpo_lineno && 34653913Sarchie mpp->file == p && 34753913Sarchie mpp->namehash == l->lpo_namehash) 34853998Sarchie break; 34953913Sarchie /* If the lprof_hash entry is allocated to someone 35053998Sarchie * else, try the next one 35153998Sarchie */ 35253998Sarchie collision = 1; 35353913Sarchie CTR4(KTR_SPARE1, "Hash collision, %s:%d %s(%x)", mpp->file, 35453913Sarchie mpp->line, mpp->name, mpp->namehash); 35553913Sarchie hash = (hash + 1) & LPROF_HASH_MASK; 35653913Sarchie mpp = &lprof_buf[hash]; 35753913Sarchie } 35853998Sarchie if (mpp->name == NULL) { 35953998Sarchie int buf; 36053998Sarchie 36153998Sarchie buf = atomic_fetchadd_int(&allocated_lprof_buf, 1); 36253998Sarchie /* Just exit if we cannot get a trace buffer */ 36353998Sarchie if (buf >= LPROF_HASH_SIZE) { 36453998Sarchie ++lock_prof_rejected; 36553998Sarchie return; 36653998Sarchie } 36753913Sarchie mpp->file = p; 36853913Sarchie mpp->line = l->lpo_lineno; 36953913Sarchie mpp->name = lo->lo_name; 37053913Sarchie mpp->namehash = l->lpo_namehash; 37153913Sarchie if (collision) 37253913Sarchie ++lock_prof_collisions; 37353913Sarchie 37453998Sarchie /* 37553913Sarchie * We might have raced someone else but who cares, 37653998Sarchie * they'll try again next time 37753998Sarchie */ 37853913Sarchie ++lock_prof_records; 37953913Sarchie } 38053913Sarchie LPROF_LOCK(hash); 38153913Sarchie /* 38253913Sarchie * Record if the mutex has been held longer now than ever 38353913Sarchie * before. 38453998Sarchie */ 38553913Sarchie if (now - acqtime > mpp->cnt_max) 38653998Sarchie mpp->cnt_max = now - acqtime; 38753998Sarchie mpp->cnt_tot += now - acqtime; 38853998Sarchie mpp->cnt_wait += waittime; 38953913Sarchie mpp->cnt_cur++; 39058011Sarchie /* 39153913Sarchie * There's a small race, really we should cmpxchg 39253913Sarchie * 0 with the current value, but that would bill 39353913Sarchie * the contention to the wrong lock instance if 39458011Sarchie * it followed this also. 39553913Sarchie */ 39653913Sarchie mpp->cnt_contest_holding += l->lpo_contest_holding; 39753913Sarchie mpp->cnt_contest_locking += l->lpo_contest_locking; 39853913Sarchie LPROF_UNLOCK(hash); 39953998Sarchie 40053913Sarchie } 40153913Sarchie l->lpo_acqtime = 0; 40253913Sarchie l->lpo_waittime = 0; 40353913Sarchie l->lpo_contest_locking = 0; 40453913Sarchie l->lpo_contest_holding = 0; 40553913Sarchie} 40653913Sarchie#endif 40753913Sarchie