kern_mutex.c (227758) | kern_mutex.c (228424) |
---|---|
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 20 unchanged lines hidden (view full) --- 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. --- 20 unchanged lines hidden (view full) --- 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> |
37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 227758 2011-11-20 16:33:09Z attilio $"); | 37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 228424 2011-12-11 21:02:01Z avg $"); |
38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_kdtrace.h" 43#include "opt_sched.h" 44 45#include <sys/param.h> --- 141 unchanged lines hidden (view full) --- 187/* 188 * Function versions of the inlined __mtx_* macros. These are used by 189 * modules and can also be called from assembly language if needed. 190 */ 191void 192_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 193{ 194 | 38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_kdtrace.h" 43#include "opt_sched.h" 44 45#include <sys/param.h> --- 141 unchanged lines hidden (view full) --- 187/* 188 * Function versions of the inlined __mtx_* macros. These are used by 189 * modules and can also be called from assembly language if needed. 190 */ 191void 192_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 193{ 194 |
195 if (SCHEDULER_STOPPED()) 196 return; |
|
195 MPASS(curthread != NULL); 196 KASSERT(m->mtx_lock != MTX_DESTROYED, 197 ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 198 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 199 ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 200 file, line)); 201 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 202 file, line, NULL); 203 204 __mtx_lock(m, curthread, opts, file, line); 205 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 206 line); 207 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 208 curthread->td_locks++; 209} 210 211void 212_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 213{ | 197 MPASS(curthread != NULL); 198 KASSERT(m->mtx_lock != MTX_DESTROYED, 199 ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 200 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 201 ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 202 file, line)); 203 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 204 file, line, NULL); 205 206 __mtx_lock(m, curthread, opts, file, line); 207 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 208 line); 209 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 210 curthread->td_locks++; 211} 212 213void 214_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 215{ |
216 217 if (SCHEDULER_STOPPED()) 218 return; |
|
214 MPASS(curthread != NULL); 215 KASSERT(m->mtx_lock != MTX_DESTROYED, 216 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 217 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 218 ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 219 file, line)); 220 curthread->td_locks--; 221 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); --- 5 unchanged lines hidden (view full) --- 227 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m); 228 __mtx_unlock(m, curthread, opts, file, line); 229} 230 231void 232_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 233{ 234 | 219 MPASS(curthread != NULL); 220 KASSERT(m->mtx_lock != MTX_DESTROYED, 221 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 222 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 223 ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 224 file, line)); 225 curthread->td_locks--; 226 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); --- 5 unchanged lines hidden (view full) --- 232 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_UNLOCK_RELEASE, m); 233 __mtx_unlock(m, curthread, opts, file, line); 234} 235 236void 237_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 238{ 239 |
240 if (SCHEDULER_STOPPED()) 241 return; |
|
235 MPASS(curthread != NULL); 236 KASSERT(m->mtx_lock != MTX_DESTROYED, 237 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 238 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 239 ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 240 m->lock_object.lo_name, file, line)); 241 if (mtx_owned(m)) 242 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, --- 6 unchanged lines hidden (view full) --- 249 line); 250 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 251} 252 253void 254_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 255{ 256 | 242 MPASS(curthread != NULL); 243 KASSERT(m->mtx_lock != MTX_DESTROYED, 244 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 245 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 246 ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 247 m->lock_object.lo_name, file, line)); 248 if (mtx_owned(m)) 249 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, --- 6 unchanged lines hidden (view full) --- 256 line); 257 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 258} 259 260void 261_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 262{ 263 |
264 if (SCHEDULER_STOPPED()) 265 return; |
|
257 MPASS(curthread != NULL); 258 KASSERT(m->mtx_lock != MTX_DESTROYED, 259 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 260 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 261 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 262 m->lock_object.lo_name, file, line)); 263 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 264 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, --- 12 unchanged lines hidden (view full) --- 277mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line) 278{ 279#ifdef LOCK_PROFILING 280 uint64_t waittime = 0; 281 int contested = 0; 282#endif 283 int rval; 284 | 266 MPASS(curthread != NULL); 267 KASSERT(m->mtx_lock != MTX_DESTROYED, 268 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 269 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 270 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 271 m->lock_object.lo_name, file, line)); 272 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 273 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, --- 12 unchanged lines hidden (view full) --- 286mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line) 287{ 288#ifdef LOCK_PROFILING 289 uint64_t waittime = 0; 290 int contested = 0; 291#endif 292 int rval; 293 |
294 if (SCHEDULER_STOPPED()) 295 return (1); 296 |
|
285 MPASS(curthread != NULL); 286 KASSERT(m->mtx_lock != MTX_DESTROYED, 287 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 288 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 289 ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 290 file, line)); 291 292 if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) { --- 40 unchanged lines hidden (view full) --- 333 uint64_t waittime = 0; 334#endif 335#ifdef KDTRACE_HOOKS 336 uint64_t spin_cnt = 0; 337 uint64_t sleep_cnt = 0; 338 int64_t sleep_time = 0; 339#endif 340 | 297 MPASS(curthread != NULL); 298 KASSERT(m->mtx_lock != MTX_DESTROYED, 299 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 300 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 301 ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 302 file, line)); 303 304 if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) { --- 40 unchanged lines hidden (view full) --- 345 uint64_t waittime = 0; 346#endif 347#ifdef KDTRACE_HOOKS 348 uint64_t spin_cnt = 0; 349 uint64_t sleep_cnt = 0; 350 int64_t sleep_time = 0; 351#endif 352 |
353 if (SCHEDULER_STOPPED()) 354 return; 355 |
|
341 if (mtx_owned(m)) { 342 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 343 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 344 m->lock_object.lo_name, file, line)); 345 m->mtx_recurse++; 346 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 347 if (LOCK_LOG_TEST(&m->lock_object, opts)) 348 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); --- 154 unchanged lines hidden (view full) --- 503 int line) 504{ 505 int i = 0; 506#ifdef LOCK_PROFILING 507 int contested = 0; 508 uint64_t waittime = 0; 509#endif 510 | 356 if (mtx_owned(m)) { 357 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 358 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 359 m->lock_object.lo_name, file, line)); 360 m->mtx_recurse++; 361 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 362 if (LOCK_LOG_TEST(&m->lock_object, opts)) 363 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); --- 154 unchanged lines hidden (view full) --- 518 int line) 519{ 520 int i = 0; 521#ifdef LOCK_PROFILING 522 int contested = 0; 523 uint64_t waittime = 0; 524#endif 525 |
526 if (SCHEDULER_STOPPED()) 527 return; 528 |
|
511 if (LOCK_LOG_TEST(&m->lock_object, opts)) 512 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 513 514 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 515 while (!_mtx_obtain_lock(m, tid)) { 516 517 /* Give interrupts a chance while we spin. */ 518 spinlock_exit(); --- 31 unchanged lines hidden (view full) --- 550 uint64_t waittime = 0; 551#endif 552#ifdef KDTRACE_HOOKS 553 uint64_t spin_cnt = 0; 554#endif 555 556 i = 0; 557 tid = (uintptr_t)curthread; | 529 if (LOCK_LOG_TEST(&m->lock_object, opts)) 530 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 531 532 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 533 while (!_mtx_obtain_lock(m, tid)) { 534 535 /* Give interrupts a chance while we spin. */ 536 spinlock_exit(); --- 31 unchanged lines hidden (view full) --- 568 uint64_t waittime = 0; 569#endif 570#ifdef KDTRACE_HOOKS 571 uint64_t spin_cnt = 0; 572#endif 573 574 i = 0; 575 tid = (uintptr_t)curthread; |
576 577 if (SCHEDULER_STOPPED()) 578 return; 579 |
|
558 for (;;) { 559retry: 560 spinlock_enter(); 561 m = td->td_lock; 562 KASSERT(m->mtx_lock != MTX_DESTROYED, 563 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 564 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 565 ("thread_lock() of sleep mutex %s @ %s:%d", --- 85 unchanged lines hidden (view full) --- 651 * We are only called here if the lock is recursed or contested (i.e. we 652 * need to wake up a blocked thread). 653 */ 654void 655_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 656{ 657 struct turnstile *ts; 658 | 580 for (;;) { 581retry: 582 spinlock_enter(); 583 m = td->td_lock; 584 KASSERT(m->mtx_lock != MTX_DESTROYED, 585 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 586 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 587 ("thread_lock() of sleep mutex %s @ %s:%d", --- 85 unchanged lines hidden (view full) --- 673 * We are only called here if the lock is recursed or contested (i.e. we 674 * need to wake up a blocked thread). 675 */ 676void 677_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 678{ 679 struct turnstile *ts; 680 |
681 if (SCHEDULER_STOPPED()) 682 return; 683 |
|
659 if (mtx_recursed(m)) { 660 if (--(m->mtx_recurse) == 0) 661 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 662 if (LOCK_LOG_TEST(&m->lock_object, opts)) 663 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 664 return; 665 } 666 --- 247 unchanged lines hidden --- | 684 if (mtx_recursed(m)) { 685 if (--(m->mtx_recurse) == 0) 686 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 687 if (LOCK_LOG_TEST(&m->lock_object, opts)) 688 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 689 return; 690 } 691 --- 247 unchanged lines hidden --- |