Deleted Added
sdiff udiff text old ( 161336 ) new ( 164159 )
full compact
kern_mutex.c (161336) kern_mutex.c (164159)
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 20 unchanged lines hidden (view full) ---

29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32/*
33 * Machine independent bits of mutex implementation.
34 */
35
36#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 20 unchanged lines hidden (view full) ---

29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
30 */
31
32/*
33 * Machine independent bits of mutex implementation.
34 */
35
36#include <sys/cdefs.h>
37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 161336 2006-08-15 18:26:12Z jhb $");
37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 164159 2006-11-11 03:18:07Z kmacy $");
38
39#include "opt_adaptive_mutexes.h"
40#include "opt_ddb.h"
38
39#include "opt_adaptive_mutexes.h"
40#include "opt_ddb.h"
41#include "opt_mprof.h"
41#include "opt_global.h"
42#include "opt_mutex_wake_all.h"
43#include "opt_sched.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/bus.h>
48#include <sys/conf.h>
49#include <sys/kdb.h>

--- 4 unchanged lines hidden (view full) ---

54#include <sys/mutex.h>
55#include <sys/proc.h>
56#include <sys/resourcevar.h>
57#include <sys/sched.h>
58#include <sys/sbuf.h>
59#include <sys/sysctl.h>
60#include <sys/turnstile.h>
61#include <sys/vmmeter.h>
42#include "opt_mutex_wake_all.h"
43#include "opt_sched.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/bus.h>
48#include <sys/conf.h>
49#include <sys/kdb.h>

--- 4 unchanged lines hidden (view full) ---

54#include <sys/mutex.h>
55#include <sys/proc.h>
56#include <sys/resourcevar.h>
57#include <sys/sched.h>
58#include <sys/sbuf.h>
59#include <sys/sysctl.h>
60#include <sys/turnstile.h>
61#include <sys/vmmeter.h>
62#include <sys/lock_profile.h>
62
63#include <machine/atomic.h>
64#include <machine/bus.h>
65#include <machine/cpu.h>
66
67#include <ddb/ddb.h>
68
69#include <fs/devfs/devfs_int.h>

--- 40 unchanged lines hidden (view full) ---

110};
111
112/*
113 * System-wide mutexes
114 */
115struct mtx sched_lock;
116struct mtx Giant;
117
63
64#include <machine/atomic.h>
65#include <machine/bus.h>
66#include <machine/cpu.h>
67
68#include <ddb/ddb.h>
69
70#include <fs/devfs/devfs_int.h>

--- 40 unchanged lines hidden (view full) ---

111};
112
113/*
114 * System-wide mutexes
115 */
116struct mtx sched_lock;
117struct mtx Giant;
118
118#ifdef MUTEX_PROFILING
119SYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging");
120SYSCTL_NODE(_debug_mutex, OID_AUTO, prof, CTLFLAG_RD, NULL, "mutex profiling");
121static int mutex_prof_enable = 0;
122SYSCTL_INT(_debug_mutex_prof, OID_AUTO, enable, CTLFLAG_RW,
123 &mutex_prof_enable, 0, "Enable tracing of mutex holdtime");
124
125struct mutex_prof {
126 const char *name;
127 const char *file;
128 int line;
129 uintmax_t cnt_max;
130 uintmax_t cnt_tot;
131 uintmax_t cnt_cur;
132 uintmax_t cnt_contest_holding;
133 uintmax_t cnt_contest_locking;
134 struct mutex_prof *next;
135};
136
137/*
119/*
138 * mprof_buf is a static pool of profiling records to avoid possible
139 * reentrance of the memory allocation functions.
140 *
141 * Note: NUM_MPROF_BUFFERS must be smaller than MPROF_HASH_SIZE.
142 */
143#ifdef MPROF_BUFFERS
144#define NUM_MPROF_BUFFERS MPROF_BUFFERS
145#else
146#define NUM_MPROF_BUFFERS 1000
147#endif
148static struct mutex_prof mprof_buf[NUM_MPROF_BUFFERS];
149static int first_free_mprof_buf;
150#ifndef MPROF_HASH_SIZE
151#define MPROF_HASH_SIZE 1009
152#endif
153#if NUM_MPROF_BUFFERS >= MPROF_HASH_SIZE
154#error MPROF_BUFFERS must be larger than MPROF_HASH_SIZE
155#endif
156static struct mutex_prof *mprof_hash[MPROF_HASH_SIZE];
157/* SWAG: sbuf size = avg stat. line size * number of locks */
158#define MPROF_SBUF_SIZE 256 * 400
159
160static int mutex_prof_acquisitions;
161SYSCTL_INT(_debug_mutex_prof, OID_AUTO, acquisitions, CTLFLAG_RD,
162 &mutex_prof_acquisitions, 0, "Number of mutex acquistions recorded");
163static int mutex_prof_records;
164SYSCTL_INT(_debug_mutex_prof, OID_AUTO, records, CTLFLAG_RD,
165 &mutex_prof_records, 0, "Number of profiling records");
166static int mutex_prof_maxrecords = NUM_MPROF_BUFFERS;
167SYSCTL_INT(_debug_mutex_prof, OID_AUTO, maxrecords, CTLFLAG_RD,
168 &mutex_prof_maxrecords, 0, "Maximum number of profiling records");
169static int mutex_prof_rejected;
170SYSCTL_INT(_debug_mutex_prof, OID_AUTO, rejected, CTLFLAG_RD,
171 &mutex_prof_rejected, 0, "Number of rejected profiling records");
172static int mutex_prof_hashsize = MPROF_HASH_SIZE;
173SYSCTL_INT(_debug_mutex_prof, OID_AUTO, hashsize, CTLFLAG_RD,
174 &mutex_prof_hashsize, 0, "Hash size");
175static int mutex_prof_collisions = 0;
176SYSCTL_INT(_debug_mutex_prof, OID_AUTO, collisions, CTLFLAG_RD,
177 &mutex_prof_collisions, 0, "Number of hash collisions");
178
179/*
180 * mprof_mtx protects the profiling buffers and the hash.
181 */
182static struct mtx mprof_mtx;
183MTX_SYSINIT(mprof, &mprof_mtx, "mutex profiling lock", MTX_SPIN | MTX_QUIET);
184
185static u_int64_t
186nanoseconds(void)
187{
188 struct timespec tv;
189
190 nanotime(&tv);
191 return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec);
192}
193
194static int
195dump_mutex_prof_stats(SYSCTL_HANDLER_ARGS)
196{
197 struct sbuf *sb;
198 int error, i;
199 static int multiplier = 1;
200
201 if (first_free_mprof_buf == 0)
202 return (SYSCTL_OUT(req, "No locking recorded",
203 sizeof("No locking recorded")));
204
205retry_sbufops:
206 sb = sbuf_new(NULL, NULL, MPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN);
207 sbuf_printf(sb, "\n%6s %12s %11s %5s %12s %12s %s\n",
208 "max", "total", "count", "avg", "cnt_hold", "cnt_lock", "name");
209 /*
210 * XXX this spinlock seems to be by far the largest perpetrator
211 * of spinlock latency (1.6 msec on an Athlon1600 was recorded
212 * even before I pessimized it further by moving the average
213 * computation here).
214 */
215 mtx_lock_spin(&mprof_mtx);
216 for (i = 0; i < first_free_mprof_buf; ++i) {
217 sbuf_printf(sb, "%6ju %12ju %11ju %5ju %12ju %12ju %s:%d (%s)\n",
218 mprof_buf[i].cnt_max / 1000,
219 mprof_buf[i].cnt_tot / 1000,
220 mprof_buf[i].cnt_cur,
221 mprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 :
222 mprof_buf[i].cnt_tot / (mprof_buf[i].cnt_cur * 1000),
223 mprof_buf[i].cnt_contest_holding,
224 mprof_buf[i].cnt_contest_locking,
225 mprof_buf[i].file, mprof_buf[i].line, mprof_buf[i].name);
226 if (sbuf_overflowed(sb)) {
227 mtx_unlock_spin(&mprof_mtx);
228 sbuf_delete(sb);
229 multiplier++;
230 goto retry_sbufops;
231 }
232 }
233 mtx_unlock_spin(&mprof_mtx);
234 sbuf_finish(sb);
235 error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
236 sbuf_delete(sb);
237 return (error);
238}
239SYSCTL_PROC(_debug_mutex_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD,
240 NULL, 0, dump_mutex_prof_stats, "A", "Mutex profiling statistics");
241
242static int
243reset_mutex_prof_stats(SYSCTL_HANDLER_ARGS)
244{
245 int error, v;
246
247 if (first_free_mprof_buf == 0)
248 return (0);
249
250 v = 0;
251 error = sysctl_handle_int(oidp, &v, 0, req);
252 if (error)
253 return (error);
254 if (req->newptr == NULL)
255 return (error);
256 if (v == 0)
257 return (0);
258
259 mtx_lock_spin(&mprof_mtx);
260 bzero(mprof_buf, sizeof(*mprof_buf) * first_free_mprof_buf);
261 bzero(mprof_hash, sizeof(struct mtx *) * MPROF_HASH_SIZE);
262 first_free_mprof_buf = 0;
263 mtx_unlock_spin(&mprof_mtx);
264 return (0);
265}
266SYSCTL_PROC(_debug_mutex_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW,
267 NULL, 0, reset_mutex_prof_stats, "I", "Reset mutex profiling statistics");
268#endif
269
270/*
271 * Function versions of the inlined __mtx_* macros. These are used by
272 * modules and can also be called from assembly language if needed.
273 */
274void
275_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
276{
120 * Function versions of the inlined __mtx_* macros. These are used by
121 * modules and can also be called from assembly language if needed.
122 */
123void
124_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
125{
126 uint64_t waittime;
277
278 MPASS(curthread != NULL);
279 KASSERT(m->mtx_lock != MTX_DESTROYED,
280 ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
281 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
282 ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
283 file, line));
284 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
285 file, line);
127
128 MPASS(curthread != NULL);
129 KASSERT(m->mtx_lock != MTX_DESTROYED,
130 ("mtx_lock() of destroyed mutex @ %s:%d", file, line));
131 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
132 ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
133 file, line));
134 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
135 file, line);
136
137 lock_profile_waitstart(&waittime);
286 _get_sleep_lock(m, curthread, opts, file, line);
287 LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file,
288 line);
289 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
290 curthread->td_locks++;
138 _get_sleep_lock(m, curthread, opts, file, line);
139 LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file,
140 line);
141 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
142 curthread->td_locks++;
291#ifdef MUTEX_PROFILING
292 /* don't reset the timer when/if recursing */
293 if (m->mtx_acqtime == 0) {
294 m->mtx_filename = file;
295 m->mtx_lineno = line;
296 m->mtx_acqtime = mutex_prof_enable ? nanoseconds() : 0;
297 ++mutex_prof_acquisitions;
298 }
299#endif
143 lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line);
300}
301
302void
303_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
304{
305
306 MPASS(curthread != NULL);
307 KASSERT(m->mtx_lock != MTX_DESTROYED,
308 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
309 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
310 ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
311 file, line));
312 curthread->td_locks--;
313 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
314 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file,
315 line);
316 mtx_assert(m, MA_OWNED);
144}
145
146void
147_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
148{
149
150 MPASS(curthread != NULL);
151 KASSERT(m->mtx_lock != MTX_DESTROYED,
152 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line));
153 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
154 ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
155 file, line));
156 curthread->td_locks--;
157 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
158 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file,
159 line);
160 mtx_assert(m, MA_OWNED);
317#ifdef MUTEX_PROFILING
318 if (m->mtx_acqtime != 0) {
319 static const char *unknown = "(unknown)";
320 struct mutex_prof *mpp;
321 u_int64_t acqtime, now;
322 const char *p, *q;
323 volatile u_int hash;
324
161
325 now = nanoseconds();
326 acqtime = m->mtx_acqtime;
327 m->mtx_acqtime = 0;
328 if (now <= acqtime)
329 goto out;
330 for (p = m->mtx_filename;
331 p != NULL && strncmp(p, "../", 3) == 0; p += 3)
332 /* nothing */ ;
333 if (p == NULL || *p == '\0')
334 p = unknown;
335 for (hash = m->mtx_lineno, q = p; *q != '\0'; ++q)
336 hash = (hash * 2 + *q) % MPROF_HASH_SIZE;
337 mtx_lock_spin(&mprof_mtx);
338 for (mpp = mprof_hash[hash]; mpp != NULL; mpp = mpp->next)
339 if (mpp->line == m->mtx_lineno &&
340 strcmp(mpp->file, p) == 0)
341 break;
342 if (mpp == NULL) {
343 /* Just exit if we cannot get a trace buffer */
344 if (first_free_mprof_buf >= NUM_MPROF_BUFFERS) {
345 ++mutex_prof_rejected;
346 goto unlock;
347 }
348 mpp = &mprof_buf[first_free_mprof_buf++];
349 mpp->name = mtx_name(m);
350 mpp->file = p;
351 mpp->line = m->mtx_lineno;
352 mpp->next = mprof_hash[hash];
353 if (mprof_hash[hash] != NULL)
354 ++mutex_prof_collisions;
355 mprof_hash[hash] = mpp;
356 ++mutex_prof_records;
357 }
358 /*
359 * Record if the mutex has been held longer now than ever
360 * before.
361 */
362 if (now - acqtime > mpp->cnt_max)
363 mpp->cnt_max = now - acqtime;
364 mpp->cnt_tot += now - acqtime;
365 mpp->cnt_cur++;
366 /*
367 * There's a small race, really we should cmpxchg
368 * 0 with the current value, but that would bill
369 * the contention to the wrong lock instance if
370 * it followed this also.
371 */
372 mpp->cnt_contest_holding += m->mtx_contest_holding;
373 m->mtx_contest_holding = 0;
374 mpp->cnt_contest_locking += m->mtx_contest_locking;
375 m->mtx_contest_locking = 0;
376unlock:
377 mtx_unlock_spin(&mprof_mtx);
378 }
379out:
380#endif
162 lock_profile_release_lock(&m->mtx_object);
381 _rel_sleep_lock(m, curthread, opts, file, line);
382}
383
384void
385_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
386{
163 _rel_sleep_lock(m, curthread, opts, file, line);
164}
165
166void
167_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
168{
169
170 uint64_t waittime;
387
388 MPASS(curthread != NULL);
389 KASSERT(m->mtx_lock != MTX_DESTROYED,
390 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
391 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin,
392 ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
393 m->mtx_object.lo_name, file, line));
394 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
395 file, line);
171
172 MPASS(curthread != NULL);
173 KASSERT(m->mtx_lock != MTX_DESTROYED,
174 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line));
175 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin,
176 ("mtx_lock_spin() of sleep mutex %s @ %s:%d",
177 m->mtx_object.lo_name, file, line));
178 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
179 file, line);
180 lock_profile_waitstart(&waittime);
396 _get_spin_lock(m, curthread, opts, file, line);
397 LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file,
398 line);
399 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
181 _get_spin_lock(m, curthread, opts, file, line);
182 LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file,
183 line);
184 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
185 lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line);
400}
401
402void
403_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
404{
405
406 MPASS(curthread != NULL);
407 KASSERT(m->mtx_lock != MTX_DESTROYED,
408 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
409 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin,
410 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
411 m->mtx_object.lo_name, file, line));
412 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
413 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file,
414 line);
415 mtx_assert(m, MA_OWNED);
186}
187
188void
189_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
190{
191
192 MPASS(curthread != NULL);
193 KASSERT(m->mtx_lock != MTX_DESTROYED,
194 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line));
195 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin,
196 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
197 m->mtx_object.lo_name, file, line));
198 WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
199 LOCK_LOG_LOCK("UNLOCK", &m->mtx_object, opts, m->mtx_recurse, file,
200 line);
201 mtx_assert(m, MA_OWNED);
202 lock_profile_release_lock(&m->mtx_object);
416 _rel_spin_lock(m);
417}
418
419/*
420 * The important part of mtx_trylock{,_flags}()
421 * Tries to acquire lock `m.' If this function is called on a mutex that
422 * is already owned, it will recursively acquire the lock.
423 */
424int
425_mtx_trylock(struct mtx *m, int opts, const char *file, int line)
426{
427 int rval;
203 _rel_spin_lock(m);
204}
205
206/*
207 * The important part of mtx_trylock{,_flags}()
208 * Tries to acquire lock `m.' If this function is called on a mutex that
209 * is already owned, it will recursively acquire the lock.
210 */
211int
212_mtx_trylock(struct mtx *m, int opts, const char *file, int line)
213{
214 int rval;
215 uint64_t waittime = 0;
428
429 MPASS(curthread != NULL);
430 KASSERT(m->mtx_lock != MTX_DESTROYED,
431 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
432 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
433 ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
434 file, line));
435

--- 4 unchanged lines hidden (view full) ---

440 } else
441 rval = _obtain_lock(m, (uintptr_t)curthread);
442
443 LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line);
444 if (rval) {
445 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
446 file, line);
447 curthread->td_locks++;
216
217 MPASS(curthread != NULL);
218 KASSERT(m->mtx_lock != MTX_DESTROYED,
219 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line));
220 KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep,
221 ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
222 file, line));
223

--- 4 unchanged lines hidden (view full) ---

228 } else
229 rval = _obtain_lock(m, (uintptr_t)curthread);
230
231 LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line);
232 if (rval) {
233 WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK,
234 file, line);
235 curthread->td_locks++;
236 lock_profile_obtain_lock_success(&m->mtx_object, waittime, file, line);
237
448 }
449
450 return (rval);
451}
452
453/*
454 * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
455 *
456 * We call this if the lock is either contested (i.e. we need to go to
457 * sleep waiting for it), or if we need to recurse on it.
458 */
459void
460_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
461 int line)
462{
463#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
464 volatile struct thread *owner;
465#endif
238 }
239
240 return (rval);
241}
242
243/*
244 * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock.
245 *
246 * We call this if the lock is either contested (i.e. we need to go to
247 * sleep waiting for it), or if we need to recurse on it.
248 */
249void
250_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
251 int line)
252{
253#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES)
254 volatile struct thread *owner;
255#endif
466 uintptr_t v;
467#ifdef KTR
468 int cont_logged = 0;
469#endif
256#ifdef KTR
257 int cont_logged = 0;
258#endif
470#ifdef MUTEX_PROFILING
259 uintptr_t v;
471 int contested;
260 int contested;
472#endif
473
474 if (mtx_owned(m)) {
475 KASSERT((m->mtx_object.lo_flags & LO_RECURSABLE) != 0,
476 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
477 m->mtx_object.lo_name, file, line));
478 m->mtx_recurse++;
479 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
480 if (LOCK_LOG_TEST(&m->mtx_object, opts))
481 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
482 return;
483 }
484
485 if (LOCK_LOG_TEST(&m->mtx_object, opts))
486 CTR4(KTR_LOCK,
487 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
488 m->mtx_object.lo_name, (void *)m->mtx_lock, file, line);
489
261
262 if (mtx_owned(m)) {
263 KASSERT((m->mtx_object.lo_flags & LO_RECURSABLE) != 0,
264 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n",
265 m->mtx_object.lo_name, file, line));
266 m->mtx_recurse++;
267 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
268 if (LOCK_LOG_TEST(&m->mtx_object, opts))
269 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m);
270 return;
271 }
272
273 if (LOCK_LOG_TEST(&m->mtx_object, opts))
274 CTR4(KTR_LOCK,
275 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d",
276 m->mtx_object.lo_name, (void *)m->mtx_lock, file, line);
277
490#ifdef MUTEX_PROFILING
491 contested = 0;
492#endif
493 while (!_obtain_lock(m, tid)) {
278 while (!_obtain_lock(m, tid)) {
494#ifdef MUTEX_PROFILING
495 contested = 1;
496 atomic_add_int(&m->mtx_contest_holding, 1);
497#endif
279 lock_profile_obtain_lock_failed(&m->mtx_object, &contested);
498 turnstile_lock(&m->mtx_object);
499 v = m->mtx_lock;
500
501 /*
502 * Check if the lock has been released while spinning for
503 * the turnstile chain lock.
504 */
505 if (v == MTX_UNOWNED) {

--- 74 unchanged lines hidden (view full) ---

580
581#ifdef KTR
582 if (cont_logged) {
583 CTR4(KTR_CONTENTION,
584 "contention end: %s acquired by %p at %s:%d",
585 m->mtx_object.lo_name, (void *)tid, file, line);
586 }
587#endif
280 turnstile_lock(&m->mtx_object);
281 v = m->mtx_lock;
282
283 /*
284 * Check if the lock has been released while spinning for
285 * the turnstile chain lock.
286 */
287 if (v == MTX_UNOWNED) {

--- 74 unchanged lines hidden (view full) ---

362
363#ifdef KTR
364 if (cont_logged) {
365 CTR4(KTR_CONTENTION,
366 "contention end: %s acquired by %p at %s:%d",
367 m->mtx_object.lo_name, (void *)tid, file, line);
368 }
369#endif
588#ifdef MUTEX_PROFILING
370#ifdef LOCK_PROFILING
589 if (contested)
371 if (contested)
590 m->mtx_contest_locking++;
591 m->mtx_contest_holding = 0;
372 m->mtx_object.lo_profile_obj.lpo_contest_locking++;
373 m->mtx_object.lo_profile_obj.lpo_contest_holding = 0;
592#endif
593 return;
594}
595
596#ifdef SMP
597/*
598 * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
599 *
600 * This is only called if we need to actually spin for the lock. Recursion
601 * is handled inline.
602 */
603void
604_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
605 int line)
606{
607 struct thread *td;
374#endif
375 return;
376}
377
378#ifdef SMP
379/*
380 * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
381 *
382 * This is only called if we need to actually spin for the lock. Recursion
383 * is handled inline.
384 */
385void
386_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
387 int line)
388{
389 struct thread *td;
608 int i = 0;
390 int contested, i = 0;
609
610 if (LOCK_LOG_TEST(&m->mtx_object, opts))
611 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
612
613 while (!_obtain_lock(m, tid)) {
391
392 if (LOCK_LOG_TEST(&m->mtx_object, opts))
393 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
394
395 while (!_obtain_lock(m, tid)) {
396 lock_profile_obtain_lock_failed(&m->mtx_object, &contested);
614
615 /* Give interrupts a chance while we spin. */
616 spinlock_exit();
617 while (m->mtx_lock != MTX_UNOWNED) {
618 if (i++ < 10000000) {
619 cpu_spinwait();
620 continue;
621 }

--- 218 unchanged lines hidden (view full) ---

840 */
841void
842mtx_init(struct mtx *m, const char *name, const char *type, int opts)
843{
844 struct lock_class *class;
845 int flags;
846
847 MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
397
398 /* Give interrupts a chance while we spin. */
399 spinlock_exit();
400 while (m->mtx_lock != MTX_UNOWNED) {
401 if (i++ < 10000000) {
402 cpu_spinwait();
403 continue;
404 }

--- 218 unchanged lines hidden (view full) ---

623 */
624void
625mtx_init(struct mtx *m, const char *name, const char *type, int opts)
626{
627 struct lock_class *class;
628 int flags;
629
630 MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE |
848 MTX_NOWITNESS | MTX_DUPOK)) == 0);
631 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0);
849
850#ifdef MUTEX_DEBUG
851 /* Diagnostic and error correction */
852 mtx_validate(m);
853#endif
854
855 /* Determine lock class and lock flags. */
856 if (opts & MTX_SPIN)

--- 4 unchanged lines hidden (view full) ---

861 if (opts & MTX_QUIET)
862 flags |= LO_QUIET;
863 if (opts & MTX_RECURSE)
864 flags |= LO_RECURSABLE;
865 if ((opts & MTX_NOWITNESS) == 0)
866 flags |= LO_WITNESS;
867 if (opts & MTX_DUPOK)
868 flags |= LO_DUPOK;
632
633#ifdef MUTEX_DEBUG
634 /* Diagnostic and error correction */
635 mtx_validate(m);
636#endif
637
638 /* Determine lock class and lock flags. */
639 if (opts & MTX_SPIN)

--- 4 unchanged lines hidden (view full) ---

644 if (opts & MTX_QUIET)
645 flags |= LO_QUIET;
646 if (opts & MTX_RECURSE)
647 flags |= LO_RECURSABLE;
648 if ((opts & MTX_NOWITNESS) == 0)
649 flags |= LO_WITNESS;
650 if (opts & MTX_DUPOK)
651 flags |= LO_DUPOK;
652 if (opts & MTX_NOPROFILE)
653 flags |= LO_NOPROFILE;
869
870 /* Initialize mutex. */
871 m->mtx_lock = MTX_UNOWNED;
872 m->mtx_recurse = 0;
654
655 /* Initialize mutex. */
656 m->mtx_lock = MTX_UNOWNED;
657 m->mtx_recurse = 0;
873#ifdef MUTEX_PROFILING
874 m->mtx_acqtime = 0;
875 m->mtx_filename = NULL;
876 m->mtx_lineno = 0;
877 m->mtx_contest_holding = 0;
878 m->mtx_contest_locking = 0;
879#endif
880
658
659 lock_profile_object_init(&m->mtx_object, name);
881 lock_init(&m->mtx_object, class, name, type, flags);
882}
883
884/*
885 * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be
886 * passed in as a flag here because if the corresponding mtx_init() was
887 * called with MTX_QUIET set, then it will already be set in the mutex's
888 * flags.

--- 14 unchanged lines hidden (view full) ---

903 curthread->td_locks--;
904
905 /* Tell witness this isn't locked to make it happy. */
906 WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__,
907 __LINE__);
908 }
909
910 m->mtx_lock = MTX_DESTROYED;
660 lock_init(&m->mtx_object, class, name, type, flags);
661}
662
663/*
664 * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be
665 * passed in as a flag here because if the corresponding mtx_init() was
666 * called with MTX_QUIET set, then it will already be set in the mutex's
667 * flags.

--- 14 unchanged lines hidden (view full) ---

682 curthread->td_locks--;
683
684 /* Tell witness this isn't locked to make it happy. */
685 WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__,
686 __LINE__);
687 }
688
689 m->mtx_lock = MTX_DESTROYED;
690 lock_profile_object_destroy(&m->mtx_object);
911 lock_destroy(&m->mtx_object);
912}
913
914/*
915 * Intialize the mutex code and system mutexes. This is called from the MD
916 * startup code prior to mi_startup(). The per-CPU data space needs to be
917 * setup before this is called.
918 */

--- 7 unchanged lines hidden (view full) ---

926 /*
927 * Initialize mutexes.
928 */
929 mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
930 mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE);
931 mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
932 mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
933 mtx_lock(&Giant);
691 lock_destroy(&m->mtx_object);
692}
693
694/*
695 * Intialize the mutex code and system mutexes. This is called from the MD
696 * startup code prior to mi_startup(). The per-CPU data space needs to be
697 * setup before this is called.
698 */

--- 7 unchanged lines hidden (view full) ---

706 /*
707 * Initialize mutexes.
708 */
709 mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
710 mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE);
711 mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
712 mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
713 mtx_lock(&Giant);
714
715 lock_profile_init();
934}
935
936#ifdef DDB
937void
938db_show_mtx(struct lock_object *lock)
939{
940 struct thread *td;
941 struct mtx *m;

--- 33 unchanged lines hidden --- 		716}
717
718#ifdef DDB
719void
720db_show_mtx(struct lock_object *lock)
721{
722 struct thread *td;
723 struct mtx *m;

--- 33 unchanged lines hidden ---