kern_rwlock.c (278694) | kern_rwlock.c (284998) |
---|---|
1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 15 unchanged lines hidden (view full) --- 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 15 unchanged lines hidden (view full) --- 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31#include <sys/cdefs.h> |
32__FBSDID("$FreeBSD: stable/10/sys/kern/kern_rwlock.c 278694 2015-02-13 19:06:22Z sbruno $"); | 32__FBSDID("$FreeBSD: stable/10/sys/kern/kern_rwlock.c 284998 2015-07-01 10:15:49Z avg $"); |
33 34#include "opt_ddb.h" 35#include "opt_hwpmc_hooks.h" 36#include "opt_kdtrace.h" 37#include "opt_no_adaptive_rwlocks.h" 38 39#include <sys/param.h> 40#include <sys/kdb.h> --- 253 unchanged lines hidden (view full) --- 294 } else 295 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, 296 (uintptr_t)curthread); 297 298 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 299 if (rval) { 300 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 301 file, line); | 33 34#include "opt_ddb.h" 35#include "opt_hwpmc_hooks.h" 36#include "opt_kdtrace.h" 37#include "opt_no_adaptive_rwlocks.h" 38 39#include <sys/param.h> 40#include <sys/kdb.h> --- 253 unchanged lines hidden (view full) --- 294 } else 295 rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, 296 (uintptr_t)curthread); 297 298 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); 299 if (rval) { 300 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 301 file, line); |
302 if (!rw_recursed(rw)) 303 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, 304 rw, 0, 0, file, line); |
|
302 curthread->td_locks++; 303 } 304 return (rval); 305} 306 307void 308_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) 309{ --- 38 unchanged lines hidden (view full) --- 348 int i; 349#endif 350#ifdef LOCK_PROFILING 351 uint64_t waittime = 0; 352 int contested = 0; 353#endif 354 uintptr_t v; 355#ifdef KDTRACE_HOOKS | 305 curthread->td_locks++; 306 } 307 return (rval); 308} 309 310void 311_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) 312{ --- 38 unchanged lines hidden (view full) --- 351 int i; 352#endif 353#ifdef LOCK_PROFILING 354 uint64_t waittime = 0; 355 int contested = 0; 356#endif 357 uintptr_t v; 358#ifdef KDTRACE_HOOKS |
359 uintptr_t state; |
|
356 uint64_t spin_cnt = 0; 357 uint64_t sleep_cnt = 0; 358 int64_t sleep_time = 0; | 360 uint64_t spin_cnt = 0; 361 uint64_t sleep_cnt = 0; 362 int64_t sleep_time = 0; |
363 int64_t all_time = 0; |
|
359#endif 360 361 if (SCHEDULER_STOPPED()) 362 return; 363 364 rw = rwlock2rw(c); 365 366 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 367 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", 368 curthread, rw->lock_object.lo_name, file, line)); 369 KASSERT(rw->rw_lock != RW_DESTROYED, 370 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 371 KASSERT(rw_wowner(rw) != curthread, 372 ("rw_rlock: wlock already held for %s @ %s:%d", 373 rw->lock_object.lo_name, file, line)); 374 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 375 | 364#endif 365 366 if (SCHEDULER_STOPPED()) 367 return; 368 369 rw = rwlock2rw(c); 370 371 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 372 ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", 373 curthread, rw->lock_object.lo_name, file, line)); 374 KASSERT(rw->rw_lock != RW_DESTROYED, 375 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 376 KASSERT(rw_wowner(rw) != curthread, 377 ("rw_rlock: wlock already held for %s @ %s:%d", 378 rw->lock_object.lo_name, file, line)); 379 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 380 |
381#ifdef KDTRACE_HOOKS 382 all_time -= lockstat_nsecs(); 383 state = rw->rw_lock; 384#endif |
|
376 for (;;) { 377#ifdef KDTRACE_HOOKS 378 spin_cnt++; 379#endif 380 /* 381 * Handle the easy case. If no other thread has a write 382 * lock, then try to bump up the count of read locks. Note 383 * that we have to preserve the current state of the --- 143 unchanged lines hidden (view full) --- 527#ifdef KDTRACE_HOOKS 528 sleep_time += lockstat_nsecs(); 529 sleep_cnt++; 530#endif 531 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 532 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 533 __func__, rw); 534 } | 385 for (;;) { 386#ifdef KDTRACE_HOOKS 387 spin_cnt++; 388#endif 389 /* 390 * Handle the easy case. If no other thread has a write 391 * lock, then try to bump up the count of read locks. Note 392 * that we have to preserve the current state of the --- 143 unchanged lines hidden (view full) --- 536#ifdef KDTRACE_HOOKS 537 sleep_time += lockstat_nsecs(); 538 sleep_cnt++; 539#endif 540 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 541 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 542 __func__, rw); 543 } |
544#ifdef KDTRACE_HOOKS 545 all_time += lockstat_nsecs(); 546 if (sleep_time) 547 LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time, 548 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 549 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); |
|
535 | 550 |
551 /* Record only the loops spinning and not sleeping. */ 552 if (spin_cnt > sleep_cnt) 553 LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, all_time - sleep_time, 554 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 555 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 556#endif |
|
536 /* 537 * TODO: acquire "owner of record" here. Here be turnstile dragons 538 * however. turnstiles don't like owners changing between calls to 539 * turnstile_wait() currently. 540 */ 541 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, 542 waittime, file, line); 543 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 544 WITNESS_LOCK(&rw->lock_object, 0, file, line); 545 curthread->td_locks++; 546 curthread->td_rw_rlocks++; | 557 /* 558 * TODO: acquire "owner of record" here. Here be turnstile dragons 559 * however. turnstiles don't like owners changing between calls to 560 * turnstile_wait() currently. 561 */ 562 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, 563 waittime, file, line); 564 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 565 WITNESS_LOCK(&rw->lock_object, 0, file, line); 566 curthread->td_locks++; 567 curthread->td_rw_rlocks++; |
547#ifdef KDTRACE_HOOKS 548 if (sleep_time) 549 LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time); 550 551 /* 552 * Record only the loops spinning and not sleeping. 553 */ 554 if (spin_cnt > sleep_cnt) 555 LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); 556#endif | |
557} 558 559int 560__rw_try_rlock(volatile uintptr_t *c, const char *file, int line) 561{ 562 struct rwlock *rw; 563 uintptr_t x; 564 --- 11 unchanged lines hidden (view full) --- 576 KASSERT(rw->rw_lock != RW_DESTROYED, 577 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 578 if (!(x & RW_LOCK_READ)) 579 break; 580 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 581 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 582 line); 583 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); | 568} 569 570int 571__rw_try_rlock(volatile uintptr_t *c, const char *file, int line) 572{ 573 struct rwlock *rw; 574 uintptr_t x; 575 --- 11 unchanged lines hidden (view full) --- 587 KASSERT(rw->rw_lock != RW_DESTROYED, 588 ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); 589 if (!(x & RW_LOCK_READ)) 590 break; 591 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { 592 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, 593 line); 594 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); |
595 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, 596 rw, 0, 0, file, line); |
|
584 curthread->td_locks++; 585 curthread->td_rw_rlocks++; 586 return (1); 587 } 588 } 589 590 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 591 return (0); --- 128 unchanged lines hidden (view full) --- 720 int i; 721#endif 722 uintptr_t v, x; 723#ifdef LOCK_PROFILING 724 uint64_t waittime = 0; 725 int contested = 0; 726#endif 727#ifdef KDTRACE_HOOKS | 597 curthread->td_locks++; 598 curthread->td_rw_rlocks++; 599 return (1); 600 } 601 } 602 603 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); 604 return (0); --- 128 unchanged lines hidden (view full) --- 733 int i; 734#endif 735 uintptr_t v, x; 736#ifdef LOCK_PROFILING 737 uint64_t waittime = 0; 738 int contested = 0; 739#endif 740#ifdef KDTRACE_HOOKS |
741 uintptr_t state; |
|
728 uint64_t spin_cnt = 0; 729 uint64_t sleep_cnt = 0; 730 int64_t sleep_time = 0; | 742 uint64_t spin_cnt = 0; 743 uint64_t sleep_cnt = 0; 744 int64_t sleep_time = 0; |
745 int64_t all_time = 0; |
|
731#endif 732 733 if (SCHEDULER_STOPPED()) 734 return; 735 736 rw = rwlock2rw(c); 737 738 if (rw_wlocked(rw)) { --- 5 unchanged lines hidden (view full) --- 744 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 745 return; 746 } 747 748 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 749 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 750 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 751 | 746#endif 747 748 if (SCHEDULER_STOPPED()) 749 return; 750 751 rw = rwlock2rw(c); 752 753 if (rw_wlocked(rw)) { --- 5 unchanged lines hidden (view full) --- 759 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); 760 return; 761 } 762 763 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 764 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 765 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 766 |
767#ifdef KDTRACE_HOOKS 768 all_time -= lockstat_nsecs(); 769 state = rw->rw_lock; 770#endif |
|
752 while (!_rw_write_lock(rw, tid)) { 753#ifdef KDTRACE_HOOKS 754 spin_cnt++; 755#endif 756#ifdef HWPMC_HOOKS 757 PMC_SOFT_CALL( , , lock, failed); 758#endif 759 lock_profile_obtain_lock_failed(&rw->lock_object, --- 121 unchanged lines hidden (view full) --- 881#endif 882 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 883 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 884 __func__, rw); 885#ifdef ADAPTIVE_RWLOCKS 886 spintries = 0; 887#endif 888 } | 771 while (!_rw_write_lock(rw, tid)) { 772#ifdef KDTRACE_HOOKS 773 spin_cnt++; 774#endif 775#ifdef HWPMC_HOOKS 776 PMC_SOFT_CALL( , , lock, failed); 777#endif 778 lock_profile_obtain_lock_failed(&rw->lock_object, --- 121 unchanged lines hidden (view full) --- 900#endif 901 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 902 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 903 __func__, rw); 904#ifdef ADAPTIVE_RWLOCKS 905 spintries = 0; 906#endif 907 } |
889 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, 890 waittime, file, line); | |
891#ifdef KDTRACE_HOOKS | 908#ifdef KDTRACE_HOOKS |
909 all_time += lockstat_nsecs(); |
|
892 if (sleep_time) | 910 if (sleep_time) |
893 LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time); | 911 LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time, 912 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 913 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); |
894 | 914 |
895 /* 896 * Record only the loops spinning and not sleeping. 897 */ | 915 /* Record only the loops spinning and not sleeping. */ |
898 if (spin_cnt > sleep_cnt) | 916 if (spin_cnt > sleep_cnt) |
899 LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); | 917 LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, all_time - sleep_time, 918 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 919 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); |
900#endif | 920#endif |
921 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, 922 waittime, file, line); |
|
901} 902 903/* 904 * This function is called if the first try at releasing a write lock failed. 905 * This means that one of the 2 waiter bits must be set indicating that at 906 * least one thread is waiting on this lock. 907 */ 908void --- 342 unchanged lines hidden --- | 923} 924 925/* 926 * This function is called if the first try at releasing a write lock failed. 927 * This means that one of the 2 waiter bits must be set indicating that at 928 * least one thread is waiting on this lock. 929 */ 930void --- 342 unchanged lines hidden --- |