subr_lock.c revision 215034
113546Sjulian/*- 235509Sjb * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 313546Sjulian * All rights reserved. 413546Sjulian * 513546Sjulian * Redistribution and use in source and binary forms, with or without 613546Sjulian * modification, are permitted provided that the following conditions 713546Sjulian * are met: 813546Sjulian * 1. Redistributions of source code must retain the above copyright 913546Sjulian * notice, this list of conditions and the following disclaimer. 1013546Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1113546Sjulian * notice, this list of conditions and the following disclaimer in the 1213546Sjulian * documentation and/or other materials provided with the distribution. 1313546Sjulian * 3. Neither the name of the author nor the names of any co-contributors 1413546Sjulian * may be used to endorse or promote products derived from this software 1513546Sjulian * without specific prior written permission. 1613546Sjulian * 1713546Sjulian * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1813546Sjulian * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1913546Sjulian * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2013546Sjulian * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2113546Sjulian * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2213546Sjulian * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2344963Sjb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2413546Sjulian * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2513546Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2613546Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2713546Sjulian * SUCH DAMAGE. 2813546Sjulian */ 2913546Sjulian 3013546Sjulian/* 3113546Sjulian * This module holds the global variables and functions used to maintain 3250476Speter * lock_object structures. 3324520Sjb */ 3413546Sjulian 3513546Sjulian#include <sys/cdefs.h> 3648046Sjb__FBSDID("$FreeBSD: head/sys/kern/subr_lock.c 215034 2010-11-09 10:59:09Z brucec $"); 3713546Sjulian 3813546Sjulian#include "opt_ddb.h" 3913546Sjulian#include "opt_mprof.h" 4013546Sjulian 4156310Sjasone#include <sys/param.h> 4213546Sjulian#include <sys/systm.h> 4356310Sjasone#include <sys/kernel.h> 4413546Sjulian#include <sys/ktr.h> 4513546Sjulian#include <sys/linker_set.h> 4613546Sjulian#include <sys/lock.h> 4713546Sjulian#include <sys/lock_profile.h> 4813546Sjulian#include <sys/malloc.h> 4913546Sjulian#include <sys/mutex.h> 5013546Sjulian#include <sys/pcpu.h> 51103388Smini#include <sys/proc.h> 5213546Sjulian#include <sys/sbuf.h> 5367097Sdeischen#include <sys/sched.h> 5467097Sdeischen#include <sys/smp.h> 5567097Sdeischen#include <sys/sysctl.h> 5667097Sdeischen 5767097Sdeischen#ifdef DDB 5867097Sdeischen#include <ddb/ddb.h> 5967097Sdeischen#endif 6013546Sjulian 6168516Sdeischen#include <machine/cpufunc.h> 62103419Smini 6313546SjulianCTASSERT(LOCK_CLASS_MAX == 15); 6448046Sjb 6548046Sjbstruct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { 6648046Sjb &lock_class_mtx_spin, 6744963Sjb &lock_class_mtx_sleep, 6844963Sjb &lock_class_sx, 6944963Sjb &lock_class_rm, 7067097Sdeischen &lock_class_rw, 7167097Sdeischen &lock_class_lockmgr, 7267097Sdeischen}; 7313546Sjulian 74103419Sminivoid 7513546Sjulianlock_init(struct lock_object *lock, struct lock_class *class, const char *name, 7671581Sdeischen const char *type, int flags) 7771581Sdeischen{ 7867097Sdeischen int i; 7967097Sdeischen 80103419Smini /* Check for double-init and zero object. */ 81103419Smini KASSERT(!lock_initalized(lock), ("lock \"%s\" %p already initialized", 8267097Sdeischen name, lock)); 8367097Sdeischen 8413546Sjulian /* Look up lock class to find its index. */ 85103419Smini for (i = 0; i < LOCK_CLASS_MAX; i++) 86103419Smini if (lock_classes[i] == class) { 87103419Smini lock->lo_flags = i << LO_CLASSSHIFT; 8867097Sdeischen break; 8935509Sjb } 90103419Smini KASSERT(i < LOCK_CLASS_MAX, ("unknown lock class %p", class)); 91103419Smini 92103419Smini /* Initialize the lock object. */ 93103419Smini lock->lo_name = name; 9435509Sjb lock->lo_flags |= flags | LO_INITIALIZED; 95103419Smini LOCK_LOG_INIT(lock, 0); 9613546Sjulian WITNESS_INIT(lock, (type != NULL) ? type : name); 97103419Smini} 98103419Smini 99103419Sminivoid 100103419Sminilock_destroy(struct lock_object *lock) 101103419Smini{ 10267097Sdeischen 103103419Smini KASSERT(lock_initalized(lock), ("lock %p is not initialized", lock)); 104103419Smini WITNESS_DESTROY(lock); 105103419Smini LOCK_LOG_DESTROY(lock, 0); 106103419Smini lock->lo_flags &= ~LO_INITIALIZED; 107103419Smini} 108103419Smini 10953812Salfred#ifdef DDB 110103419SminiDB_SHOW_COMMAND(lock, db_show_lock) 111103419Smini{ 11253812Salfred struct lock_object *lock; 113103419Smini struct lock_class *class; 114103419Smini 115103419Smini if (!have_addr) 11667097Sdeischen return; 11767097Sdeischen lock = (struct lock_object *)addr; 11848046Sjb if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) { 11967097Sdeischen db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock)); 12067097Sdeischen return; 12167097Sdeischen } 12267097Sdeischen class = LOCK_CLASS(lock); 12367097Sdeischen db_printf(" class: %s\n", class->lc_name); 12471581Sdeischen db_printf(" name: %s\n", lock->lo_name); 12567097Sdeischen class->lc_ddb_show(lock); 12667097Sdeischen} 12767097Sdeischen#endif 12867097Sdeischen 12913546Sjulian#ifdef LOCK_PROFILING 13044963Sjb 13113546Sjulian/* 132103419Smini * One object per-thread for each lock the thread owns. Tracks individual 133103419Smini * lock instances. 13413546Sjulian */ 13548046Sjbstruct lock_profile_object { 136103419Smini LIST_ENTRY(lock_profile_object) lpo_link; 137103419Smini struct lock_object *lpo_obj; 138103419Smini const char *lpo_file; 139103419Smini int lpo_line; 140103419Smini uint16_t lpo_ref; 14113546Sjulian uint16_t lpo_cnt; 14267097Sdeischen uint64_t lpo_acqtime; 14313546Sjulian uint64_t lpo_waittime; 14467097Sdeischen u_int lpo_contest_locking; 14513546Sjulian}; 14667097Sdeischen 14771581Sdeischen/* 14844963Sjb * One lock_prof for each (file, line, lock object) triple. 14948046Sjb */ 15044963Sjbstruct lock_prof { 15171581Sdeischen SLIST_ENTRY(lock_prof) link; 15267097Sdeischen struct lock_class *class; 15367097Sdeischen const char *file; 15468516Sdeischen const char *name; 15567097Sdeischen int line; 15671581Sdeischen int ticks; 15771581Sdeischen uintmax_t cnt_wait_max; 15871581Sdeischen uintmax_t cnt_max; 15967097Sdeischen uintmax_t cnt_tot; 16071581Sdeischen uintmax_t cnt_wait; 16167097Sdeischen uintmax_t cnt_cur; 16267097Sdeischen uintmax_t cnt_contest_locking; 16367097Sdeischen}; 16467097Sdeischen 16548046SjbSLIST_HEAD(lphead, lock_prof); 16648046Sjb 16748046Sjb#define LPROF_HASH_SIZE 4096 16848046Sjb#define LPROF_HASH_MASK (LPROF_HASH_SIZE - 1) 16971581Sdeischen#define LPROF_CACHE_SIZE 4096 17048046Sjb 17153812Salfred/* 17258094Sdeischen * Array of objects and profs for each type of object for each cpu. Spinlocks 17344963Sjb * are handled separately because a thread may be preempted and acquire a 17458094Sdeischen * spinlock while in the lock profiling code of a non-spinlock. In this way 17558094Sdeischen * we only need a critical section to protect the per-cpu lists. 17644963Sjb */ 17748046Sjbstruct lock_prof_type { 17844963Sjb struct lphead lpt_lpalloc; 17948046Sjb struct lpohead lpt_lpoalloc; 18013546Sjulian struct lphead lpt_hash[LPROF_HASH_SIZE]; 18148046Sjb struct lock_prof lpt_prof[LPROF_CACHE_SIZE]; 18248046Sjb struct lock_profile_object lpt_objs[LPROF_CACHE_SIZE]; 18348046Sjb}; 18448046Sjb 18513546Sjulianstruct lock_prof_cpu { 18667097Sdeischen struct lock_prof_type lpc_types[2]; /* One for spin one for other. */ 18748046Sjb}; 18813546Sjulian 18948046Sjbstruct lock_prof_cpu *lp_cpu[MAXCPU]; 19048046Sjb 19168516Sdeischenvolatile int lock_prof_enable = 0; 19248046Sjbstatic volatile int lock_prof_resetting; 19348046Sjb 19448046Sjb#define LPROF_SBUF_SIZE 256 19548046Sjb 19648046Sjbstatic int lock_prof_rejected; 19748046Sjbstatic int lock_prof_skipspin; 19871581Sdeischenstatic int lock_prof_skipcount; 19971581Sdeischen 20013546Sjulian#ifndef USE_CPU_NANOSECONDS 20148046Sjbuint64_t 20271581Sdeischennanoseconds(void) 20344963Sjb{ 20448046Sjb struct bintime bt; 20571581Sdeischen uint64_t ns; 20648046Sjb 20744963Sjb binuptime(&bt); 20848046Sjb /* From bintime2timespec */ 20948046Sjb ns = bt.sec * (uint64_t)1000000000; 21048046Sjb ns += ((uint64_t)1000000000 * (uint32_t)(bt.frac >> 32)) >> 32; 21148046Sjb return (ns); 21271581Sdeischen} 21344963Sjb#endif 21448046Sjb 21571581Sdeischenstatic void 21648046Sjblock_prof_init_type(struct lock_prof_type *type) 21748046Sjb{ 21848046Sjb int i; 21948046Sjb 22048046Sjb SLIST_INIT(&type->lpt_lpalloc); 22171581Sdeischen LIST_INIT(&type->lpt_lpoalloc); 22271581Sdeischen for (i = 0; i < LPROF_CACHE_SIZE; i++) { 22348046Sjb SLIST_INSERT_HEAD(&type->lpt_lpalloc, &type->lpt_prof[i], 22448046Sjb link); 22548046Sjb LIST_INSERT_HEAD(&type->lpt_lpoalloc, &type->lpt_objs[i], 22648046Sjb lpo_link); 22767097Sdeischen } 22813546Sjulian} 22913546Sjulian 23013546Sjulianstatic void 23167097Sdeischenlock_prof_init(void *arg) 23267097Sdeischen{ 23348046Sjb int cpu; 23448046Sjb 23548046Sjb for (cpu = 0; cpu <= mp_maxid; cpu++) { 23648046Sjb lp_cpu[cpu] = malloc(sizeof(*lp_cpu[cpu]), M_DEVBUF, 23748046Sjb M_WAITOK | M_ZERO); 23848046Sjb lock_prof_init_type(&lp_cpu[cpu]->lpc_types[0]); 23948046Sjb lock_prof_init_type(&lp_cpu[cpu]->lpc_types[1]); 24048046Sjb } 24148046Sjb} 24248046SjbSYSINIT(lockprof, SI_SUB_SMP, SI_ORDER_ANY, lock_prof_init, NULL); 24348046Sjb 24448046Sjb/* 24548046Sjb * To be certain that lock profiling has idled on all cpus before we 24613546Sjulian * reset, we schedule the resetting thread on all active cpus. Since 247103419Smini * all operations happen within critical sections we can be sure that 248103419Smini * it is safe to zero the profiling structures. 249103419Smini */ 250103419Sministatic void 251103419Sminilock_prof_idle(void) 252103419Smini{ 253103419Smini struct thread *td; 254103419Smini int cpu; 255103419Smini 256103419Smini td = curthread; 25748046Sjb thread_lock(td); 25813546Sjulian CPU_FOREACH(cpu) { 25948046Sjb sched_bind(td, cpu); 26048046Sjb } 26148046Sjb sched_unbind(td); 26213546Sjulian thread_unlock(td); 26348046Sjb} 26467097Sdeischen 26567097Sdeischenstatic void 26648046Sjblock_prof_reset_wait(void) 26767097Sdeischen{ 26867097Sdeischen 26967097Sdeischen /* 27068516Sdeischen * Spin relinquishing our cpu so that lock_prof_idle may 27167097Sdeischen * run on it. 27267097Sdeischen */ 27371581Sdeischen while (lock_prof_resetting) 27471581Sdeischen sched_relinquish(curthread); 27571581Sdeischen} 27667097Sdeischen 27771581Sdeischenstatic void 27867097Sdeischenlock_prof_reset(void) 27967097Sdeischen{ 28071581Sdeischen struct lock_prof_cpu *lpc; 28171581Sdeischen int enabled, i, cpu; 28267097Sdeischen 28367097Sdeischen /* 28467097Sdeischen * We not only race with acquiring and releasing locks but also 28567097Sdeischen * thread exit. To be certain that threads exit without valid head 28671581Sdeischen * pointers they must see resetting set before enabled is cleared. 28771581Sdeischen * Otherwise a lock may not be removed from a per-thread list due 28871581Sdeischen * to disabled being set but not wait for reset() to remove it below. 28967097Sdeischen */ 29067097Sdeischen atomic_store_rel_int(&lock_prof_resetting, 1); 29171581Sdeischen enabled = lock_prof_enable; 29271581Sdeischen lock_prof_enable = 0; 29367097Sdeischen lock_prof_idle(); 29467097Sdeischen /* 29571581Sdeischen * Some objects may have migrated between CPUs. Clear all links 29613546Sjulian * before we zero the structures. Some items may still be linked 29748046Sjb * into per-thread lists as well. 29848046Sjb */ 29948046Sjb for (cpu = 0; cpu <= mp_maxid; cpu++) { 30048046Sjb lpc = lp_cpu[cpu]; 30113546Sjulian for (i = 0; i < LPROF_CACHE_SIZE; i++) { 30271581Sdeischen LIST_REMOVE(&lpc->lpc_types[0].lpt_objs[i], lpo_link); 30348046Sjb LIST_REMOVE(&lpc->lpc_types[1].lpt_objs[i], lpo_link); 30413546Sjulian } 30548046Sjb } 30648046Sjb for (cpu = 0; cpu <= mp_maxid; cpu++) { 30748046Sjb lpc = lp_cpu[cpu]; 30813546Sjulian bzero(lpc, sizeof(*lpc)); 30971581Sdeischen lock_prof_init_type(&lpc->lpc_types[0]); 31013546Sjulian lock_prof_init_type(&lpc->lpc_types[1]); 31113546Sjulian } 31213546Sjulian atomic_store_rel_int(&lock_prof_resetting, 0); 31313546Sjulian lock_prof_enable = enabled; 31444963Sjb} 31513546Sjulian 31648046Sjbstatic void 31713546Sjulianlock_prof_output(struct lock_prof *lp, struct sbuf *sb) 31813546Sjulian{ 31944963Sjb const char *p; 32013546Sjulian 32113546Sjulian for (p = lp->file; p != NULL && strncmp(p, "../", 3) == 0; p += 3); 32213546Sjulian sbuf_printf(sb, 32368516Sdeischen "%8ju %9ju %11ju %11ju %11ju %6ju %6ju %2ju %6ju %s:%d (%s:%s)\n", 32413546Sjulian lp->cnt_max / 1000, lp->cnt_wait_max / 1000, lp->cnt_tot / 1000, 32571581Sdeischen lp->cnt_wait / 1000, lp->cnt_cur, 32671581Sdeischen lp->cnt_cur == 0 ? (uintmax_t)0 : 32771581Sdeischen lp->cnt_tot / (lp->cnt_cur * 1000), 32867097Sdeischen lp->cnt_cur == 0 ? (uintmax_t)0 : 32971581Sdeischen lp->cnt_wait / (lp->cnt_cur * 1000), 33013546Sjulian (uintmax_t)0, lp->cnt_contest_locking, 33113546Sjulian p, lp->line, lp->class->lc_name, lp->name); 33213546Sjulian} 33368516Sdeischen 33413546Sjulianstatic void 335103419Sminilock_prof_sum(struct lock_prof *match, struct lock_prof *dst, int hash, 33667097Sdeischen int spin, int t) 33767097Sdeischen{ 33867097Sdeischen struct lock_prof_type *type; 33967097Sdeischen struct lock_prof *l; 34067097Sdeischen int cpu; 34167097Sdeischen 34267097Sdeischen dst->file = match->file; 34367097Sdeischen dst->line = match->line; 34467097Sdeischen dst->class = match->class; 34567097Sdeischen dst->name = match->name; 34667097Sdeischen 34767097Sdeischen for (cpu = 0; cpu <= mp_maxid; cpu++) { 34867097Sdeischen if (lp_cpu[cpu] == NULL) 34967097Sdeischen continue; 35067097Sdeischen type = &lp_cpu[cpu]->lpc_types[spin]; 35167097Sdeischen SLIST_FOREACH(l, &type->lpt_hash[hash], link) { 35248046Sjb if (l->ticks == t) 35348046Sjb continue; 35448046Sjb if (l->file != match->file || l->line != match->line || 35513546Sjulian l->name != match->name) 35671581Sdeischen continue; 35771581Sdeischen l->ticks = t; 35813546Sjulian if (l->cnt_max > dst->cnt_max) 35913546Sjulian dst->cnt_max = l->cnt_max; 36013546Sjulian if (l->cnt_wait_max > dst->cnt_wait_max) 36168516Sdeischen dst->cnt_wait_max = l->cnt_wait_max; 36213546Sjulian dst->cnt_tot += l->cnt_tot; 36367097Sdeischen dst->cnt_wait += l->cnt_wait; 36471581Sdeischen dst->cnt_cur += l->cnt_cur; 36513546Sjulian dst->cnt_contest_locking += l->cnt_contest_locking; 36613546Sjulian } 36713546Sjulian } 36813546Sjulian 36968516Sdeischen} 37013546Sjulian 37171581Sdeischenstatic void 37213546Sjulianlock_prof_type_stats(struct lock_prof_type *type, struct sbuf *sb, int spin, 37371581Sdeischen int t) 37413546Sjulian{ 37513546Sjulian struct lock_prof *l; 37656277Sjasone int i; 37767097Sdeischen 37867097Sdeischen for (i = 0; i < LPROF_HASH_SIZE; ++i) { 37956277Sjasone SLIST_FOREACH(l, &type->lpt_hash[i], link) { 38067097Sdeischen struct lock_prof lp = {}; 38171581Sdeischen 38267097Sdeischen if (l->ticks == t) 38371581Sdeischen continue; 38456277Sjasone lock_prof_sum(l, &lp, i, spin, t); 38556277Sjasone lock_prof_output(&lp, sb); 38667097Sdeischen } 38756277Sjasone } 388103419Smini} 38913546Sjulian 39013546Sjulianstatic int 39113546Sjuliandump_lock_prof_stats(SYSCTL_HANDLER_ARGS) 39213546Sjulian{ 39313546Sjulian struct sbuf *sb; 39413546Sjulian int error, cpu, t; 39513546Sjulian int enabled; 39613546Sjulian 39713546Sjulian sb = sbuf_new_for_sysctl(NULL, NULL, LPROF_SBUF_SIZE, req); 39813546Sjulian sbuf_printf(sb, "\n%8s %9s %11s %11s %11s %6s %6s %2s %6s %s\n", 39971581Sdeischen "max", "wait_max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name"); 40071581Sdeischen enabled = lock_prof_enable; 40148046Sjb lock_prof_enable = 0; 40248046Sjb lock_prof_idle(); 40348046Sjb t = ticks; 40448046Sjb for (cpu = 0; cpu <= mp_maxid; cpu++) { 40548046Sjb if (lp_cpu[cpu] == NULL) 40648046Sjb continue; 40748046Sjb lock_prof_type_stats(&lp_cpu[cpu]->lpc_types[0], sb, 0, t); 40813546Sjulian lock_prof_type_stats(&lp_cpu[cpu]->lpc_types[1], sb, 1, t); 40971581Sdeischen } 41071581Sdeischen lock_prof_enable = enabled; 41171581Sdeischen 41213546Sjulian error = sbuf_finish(sb); 41313546Sjulian /* Output a trailing NUL. */ 414103419Smini if (error == 0) 41513546Sjulian error = SYSCTL_OUT(req, "", 1); 41613546Sjulian sbuf_delete(sb); 41741164Sjb return (error); 41841164Sjb} 41941164Sjb 42041164Sjbstatic int 42171581Sdeischenenable_lock_prof(SYSCTL_HANDLER_ARGS) 42271581Sdeischen{ 42348046Sjb int error, v; 42448046Sjb 42548046Sjb v = lock_prof_enable; 42648046Sjb error = sysctl_handle_int(oidp, &v, v, req); 42748046Sjb if (error) 42848046Sjb return (error); 42948046Sjb if (req->newptr == NULL) 43041164Sjb return (error); 43171581Sdeischen if (v == lock_prof_enable) 43271581Sdeischen return (0); 43371581Sdeischen if (v == 1) 43441164Sjb lock_prof_reset(); 43541164Sjb lock_prof_enable = !!v; 43641164Sjb 43741164Sjb return (0); 438103419Smini} 43941164Sjb 44041164Sjbstatic int 44113546Sjulianreset_lock_prof_stats(SYSCTL_HANDLER_ARGS) 442103419Smini{ 44313546Sjulian int error, v; 44448046Sjb 44548046Sjb v = 0; 44648046Sjb error = sysctl_handle_int(oidp, &v, 0, req); 44748046Sjb if (error) 44853812Salfred return (error); 44913546Sjulian if (req->newptr == NULL) 45013546Sjulian return (error); 45113546Sjulian if (v == 0) 452103419Smini return (0); 453103419Smini lock_prof_reset(); 454103419Smini 45548046Sjb return (0); 456103419Smini} 45748046Sjb 458103419Sministatic struct lock_prof * 459103419Sminilock_profile_lookup(struct lock_object *lo, int spin, const char *file, 460103419Smini int line) 461103419Smini{ 462103419Smini const char *unknown = "(unknown)"; 463103419Smini struct lock_prof_type *type; 46413546Sjulian struct lock_prof *lp; 465103419Smini struct lphead *head; 466103419Smini const char *p; 467103419Smini u_int hash; 468103419Smini 469103419Smini p = file; 470103419Smini if (p == NULL || *p == '\0') 471103419Smini p = unknown; 472103419Smini hash = (uintptr_t)lo->lo_name * 31 + (uintptr_t)p * 31 + line; 473103419Smini hash &= LPROF_HASH_MASK; 474103419Smini type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin]; 475103419Smini head = &type->lpt_hash[hash]; 476103419Smini SLIST_FOREACH(lp, head, link) { 477103419Smini if (lp->line == line && lp->file == p && 478103419Smini lp->name == lo->lo_name) 479103419Smini return (lp); 480103419Smini 48149661Sdeischen } 48248046Sjb lp = SLIST_FIRST(&type->lpt_lpalloc); 48349661Sdeischen if (lp == NULL) { 48449661Sdeischen lock_prof_rejected++; 48549661Sdeischen return (lp); 48649661Sdeischen } 48724520Sjb SLIST_REMOVE_HEAD(&type->lpt_lpalloc, link); 48849661Sdeischen lp->file = p; 48949661Sdeischen lp->line = line; 49024520Sjb lp->class = LOCK_CLASS(lo); 49149661Sdeischen lp->name = lo->lo_name; 49248046Sjb SLIST_INSERT_HEAD(&type->lpt_hash[hash], lp, link); 49313546Sjulian return (lp); 49448046Sjb} 495103419Smini 49648046Sjbstatic struct lock_profile_object * 497103419Sminilock_profile_object_lookup(struct lock_object *lo, int spin, const char *file, 49848046Sjb int line) 499103419Smini{ 50013546Sjulian struct lock_profile_object *l; 50148046Sjb struct lock_prof_type *type; 50248046Sjb struct lpohead *head; 50348046Sjb 50448046Sjb head = &curthread->td_lprof[spin]; 50548046Sjb LIST_FOREACH(l, head, lpo_link) 50648046Sjb if (l->lpo_obj == lo && l->lpo_file == file && 50713546Sjulian l->lpo_line == line) 50848046Sjb return (l); 50913546Sjulian type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin]; 51048046Sjb l = LIST_FIRST(&type->lpt_lpoalloc); 51148046Sjb if (l == NULL) { 51248046Sjb lock_prof_rejected++; 51348046Sjb return (NULL); 51448046Sjb } 51513546Sjulian LIST_REMOVE(l, lpo_link); 51613546Sjulian l->lpo_obj = lo; 51748046Sjb l->lpo_file = file; 51813546Sjulian l->lpo_line = line; 51948046Sjb l->lpo_cnt = 0; 52048046Sjb LIST_INSERT_HEAD(head, l, lpo_link); 52148046Sjb 52248046Sjb return (l); 52348046Sjb} 52448046Sjb 52513546Sjulianvoid 52613546Sjulianlock_profile_obtain_lock_success(struct lock_object *lo, int contested, 52713546Sjulian uint64_t waittime, const char *file, int line) 52864346Sjlemon{ 52913546Sjulian static int lock_prof_count; 53071581Sdeischen struct lock_profile_object *l; 53113546Sjulian int spin; 53213546Sjulian 53313546Sjulian /* don't reset the timer when/if recursing */ 53413546Sjulian if (!lock_prof_enable || (lo->lo_flags & LO_NOPROFILE)) 53571581Sdeischen return; 53613546Sjulian if (lock_prof_skipcount && 53713546Sjulian (++lock_prof_count % lock_prof_skipcount) != 0) 53813546Sjulian return; 53913546Sjulian spin = (LOCK_CLASS(lo)->lc_flags & LC_SPINLOCK) ? 1 : 0; 54013546Sjulian if (spin && lock_prof_skipspin == 1) 54168516Sdeischen return; 54213546Sjulian critical_enter(); 54371581Sdeischen /* Recheck enabled now that we're in a critical section. */ 54471581Sdeischen if (lock_prof_enable == 0) 54513546Sjulian goto out; 54613546Sjulian l = lock_profile_object_lookup(lo, spin, file, line); 54717706Sjulian if (l == NULL) 54813546Sjulian goto out; 54971581Sdeischen l->lpo_cnt++; 55071581Sdeischen if (++l->lpo_ref > 1) 55113546Sjulian goto out; 55213546Sjulian l->lpo_contest_locking = contested; 55367097Sdeischen l->lpo_acqtime = nanoseconds(); 55413546Sjulian if (waittime && (l->lpo_acqtime > waittime)) 55513546Sjulian l->lpo_waittime = l->lpo_acqtime - waittime; 55613546Sjulian else 55771581Sdeischen l->lpo_waittime = 0; 55871581Sdeischenout: 55913546Sjulian critical_exit(); 56013546Sjulian} 56171581Sdeischen 56213546Sjulianvoid 56371581Sdeischenlock_profile_thread_exit(struct thread *td) 56471581Sdeischen{ 56513546Sjulian#ifdef INVARIANTS 56613546Sjulian struct lock_profile_object *l; 56713546Sjulian 56844963Sjb MPASS(curthread->td_critnest == 0); 56944963Sjb#endif 57048046Sjb /* 57144963Sjb * If lock profiling was disabled we have to wait for reset to 57271581Sdeischen * clear our pointers before we can exit safely. 57371581Sdeischen */ 57448046Sjb lock_prof_reset_wait(); 57571581Sdeischen#ifdef INVARIANTS 57644963Sjb LIST_FOREACH(l, &td->td_lprof[0], lpo_link) 57744963Sjb printf("thread still holds lock acquired at %s:%d\n", 57844963Sjb l->lpo_file, l->lpo_line); 57948046Sjb LIST_FOREACH(l, &td->td_lprof[1], lpo_link) 58044963Sjb printf("thread still holds lock acquired at %s:%d\n", 58171581Sdeischen l->lpo_file, l->lpo_line); 58271581Sdeischen#endif 58344963Sjb MPASS(LIST_FIRST(&td->td_lprof[0]) == NULL); 58444963Sjb MPASS(LIST_FIRST(&td->td_lprof[1]) == NULL); 58548046Sjb} 58644963Sjb 58771581Sdeischenvoid 58848046Sjblock_profile_release_lock(struct lock_object *lo) 58971581Sdeischen{ 59048046Sjb struct lock_profile_object *l; 59171581Sdeischen struct lock_prof_type *type; 59248046Sjb struct lock_prof *lp; 59371581Sdeischen uint64_t curtime, holdtime; 59448046Sjb struct lpohead *head; 59544963Sjb int spin; 59667097Sdeischen 59767097Sdeischen if (lo->lo_flags & LO_NOPROFILE) 59867097Sdeischen return; 59971581Sdeischen spin = (LOCK_CLASS(lo)->lc_flags & LC_SPINLOCK) ? 1 : 0; 60071581Sdeischen head = &curthread->td_lprof[spin]; 60167097Sdeischen if (LIST_FIRST(head) == NULL) 60244963Sjb return; 60348046Sjb critical_enter(); 60444963Sjb /* Recheck enabled now that we're in a critical section. */ 60544963Sjb if (lock_prof_enable == 0 && lock_prof_resetting == 1) 60644963Sjb goto out; 60744963Sjb /* 60844963Sjb * If lock profiling is not enabled we still want to remove the 60944963Sjb * lpo from our queue. 61044963Sjb */ 61144963Sjb LIST_FOREACH(l, head, lpo_link) 61253812Salfred if (l->lpo_obj == lo) 61344963Sjb break; 61444963Sjb if (l == NULL) 61553812Salfred goto out; 61644963Sjb if (--l->lpo_ref > 0) 61744963Sjb goto out; 61844963Sjb lp = lock_profile_lookup(lo, spin, l->lpo_file, l->lpo_line); 61944963Sjb if (lp == NULL) 62044963Sjb goto release; 62144963Sjb curtime = nanoseconds(); 62244963Sjb if (curtime < l->lpo_acqtime) 62371581Sdeischen goto release; 62471581Sdeischen holdtime = curtime - l->lpo_acqtime; 62571581Sdeischen 62671581Sdeischen /* 62771581Sdeischen * Record if the lock has been held longer now than ever 62871581Sdeischen * before. 62971581Sdeischen */ 63071581Sdeischen if (holdtime > lp->cnt_max) 63171581Sdeischen lp->cnt_max = holdtime; 63271581Sdeischen if (l->lpo_waittime > lp->cnt_wait_max) 63371581Sdeischen lp->cnt_wait_max = l->lpo_waittime; 63471581Sdeischen lp->cnt_tot += holdtime; 63571581Sdeischen lp->cnt_wait += l->lpo_waittime; 63671581Sdeischen lp->cnt_contest_locking += l->lpo_contest_locking; 63771581Sdeischen lp->cnt_cur += l->lpo_cnt; 638release: 639 LIST_REMOVE(l, lpo_link); 640 type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin]; 641 LIST_INSERT_HEAD(&type->lpt_lpoalloc, l, lpo_link); 642out: 643 critical_exit(); 644} 645 646SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging"); 647SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, "lock profiling"); 648SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW, 649 &lock_prof_skipspin, 0, "Skip profiling on spinlocks."); 650SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW, 651 &lock_prof_skipcount, 0, "Sample approximately every N lock acquisitions."); 652SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD, 653 &lock_prof_rejected, 0, "Number of rejected profiling records"); 654SYSCTL_PROC(_debug_lock_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD, 655 NULL, 0, dump_lock_prof_stats, "A", "Lock profiling statistics"); 656SYSCTL_PROC(_debug_lock_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW, 657 NULL, 0, reset_lock_prof_stats, "I", "Reset lock profiling statistics"); 658SYSCTL_PROC(_debug_lock_prof, OID_AUTO, enable, CTLTYPE_INT | CTLFLAG_RW, 659 NULL, 0, enable_lock_prof, "I", "Enable lock profiling"); 660 661#endif 662