| kern_rwlock.c (284998) | kern_rwlock.c (285759) |
|---|---|
| 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 15 unchanged lines hidden (view full) --- 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 15 unchanged lines hidden (view full) --- 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * Machine independent bits of reader/writer lock implementation. 29 */ 30 31#include <sys/cdefs.h> |
| 32__FBSDID("$FreeBSD: stable/10/sys/kern/kern_rwlock.c 284998 2015-07-01 10:15:49Z avg $"); | 32__FBSDID("$FreeBSD: stable/10/sys/kern/kern_rwlock.c 285759 2015-07-21 17:16:37Z markj $"); |
| 33 34#include "opt_ddb.h" 35#include "opt_hwpmc_hooks.h" 36#include "opt_kdtrace.h" 37#include "opt_no_adaptive_rwlocks.h" 38 39#include <sys/param.h> 40#include <sys/kdb.h> --- 333 unchanged lines hidden (view full) --- 374 KASSERT(rw->rw_lock != RW_DESTROYED, 375 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 376 KASSERT(rw_wowner(rw) != curthread, 377 ("rw_rlock: wlock already held for %s @ %s:%d", 378 rw->lock_object.lo_name, file, line)); 379 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 380 381#ifdef KDTRACE_HOOKS | 33 34#include "opt_ddb.h" 35#include "opt_hwpmc_hooks.h" 36#include "opt_kdtrace.h" 37#include "opt_no_adaptive_rwlocks.h" 38 39#include <sys/param.h> 40#include <sys/kdb.h> --- 333 unchanged lines hidden (view full) --- 374 KASSERT(rw->rw_lock != RW_DESTROYED, 375 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 376 KASSERT(rw_wowner(rw) != curthread, 377 ("rw_rlock: wlock already held for %s @ %s:%d", 378 rw->lock_object.lo_name, file, line)); 379 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); 380 381#ifdef KDTRACE_HOOKS |
| 382 all_time -= lockstat_nsecs(); | 382 all_time -= lockstat_nsecs(&rw->lock_object); |
| 383 state = rw->rw_lock; 384#endif 385 for (;;) { 386#ifdef KDTRACE_HOOKS 387 spin_cnt++; 388#endif 389 /* 390 * Handle the easy case. If no other thread has a write --- 134 unchanged lines hidden (view full) --- 525 /* 526 * We were unable to acquire the lock and the read waiters 527 * flag is set, so we must block on the turnstile. 528 */ 529 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 530 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 531 rw); 532#ifdef KDTRACE_HOOKS | 383 state = rw->rw_lock; 384#endif 385 for (;;) { 386#ifdef KDTRACE_HOOKS 387 spin_cnt++; 388#endif 389 /* 390 * Handle the easy case. If no other thread has a write --- 134 unchanged lines hidden (view full) --- 525 /* 526 * We were unable to acquire the lock and the read waiters 527 * flag is set, so we must block on the turnstile. 528 */ 529 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 530 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 531 rw); 532#ifdef KDTRACE_HOOKS |
| 533 sleep_time -= lockstat_nsecs(); | 533 sleep_time -= lockstat_nsecs(&rw->lock_object); |
| 534#endif 535 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 536#ifdef KDTRACE_HOOKS | 534#endif 535 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 536#ifdef KDTRACE_HOOKS |
| 537 sleep_time += lockstat_nsecs(); | 537 sleep_time += lockstat_nsecs(&rw->lock_object); |
| 538 sleep_cnt++; 539#endif 540 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 541 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 542 __func__, rw); 543 } 544#ifdef KDTRACE_HOOKS | 538 sleep_cnt++; 539#endif 540 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 541 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 542 __func__, rw); 543 } 544#ifdef KDTRACE_HOOKS |
| 545 all_time += lockstat_nsecs(); | 545 all_time += lockstat_nsecs(&rw->lock_object); |
| 546 if (sleep_time) 547 LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time, 548 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 549 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 550 551 /* Record only the loops spinning and not sleeping. */ 552 if (spin_cnt > sleep_cnt) 553 LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, all_time - sleep_time, --- 206 unchanged lines hidden (view full) --- 760 return; 761 } 762 763 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 764 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 765 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 766 767#ifdef KDTRACE_HOOKS | 546 if (sleep_time) 547 LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time, 548 LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, 549 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 550 551 /* Record only the loops spinning and not sleeping. */ 552 if (spin_cnt > sleep_cnt) 553 LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, all_time - sleep_time, --- 206 unchanged lines hidden (view full) --- 760 return; 761 } 762 763 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 764 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 765 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 766 767#ifdef KDTRACE_HOOKS |
| 768 all_time -= lockstat_nsecs(); | 768 all_time -= lockstat_nsecs(&rw->lock_object); |
| 769 state = rw->rw_lock; 770#endif 771 while (!_rw_write_lock(rw, tid)) { 772#ifdef KDTRACE_HOOKS 773 spin_cnt++; 774#endif 775#ifdef HWPMC_HOOKS 776 PMC_SOFT_CALL( , , lock, failed); --- 109 unchanged lines hidden (view full) --- 886 /* 887 * We were unable to acquire the lock and the write waiters 888 * flag is set, so we must block on the turnstile. 889 */ 890 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 891 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 892 rw); 893#ifdef KDTRACE_HOOKS | 769 state = rw->rw_lock; 770#endif 771 while (!_rw_write_lock(rw, tid)) { 772#ifdef KDTRACE_HOOKS 773 spin_cnt++; 774#endif 775#ifdef HWPMC_HOOKS 776 PMC_SOFT_CALL( , , lock, failed); --- 109 unchanged lines hidden (view full) --- 886 /* 887 * We were unable to acquire the lock and the write waiters 888 * flag is set, so we must block on the turnstile. 889 */ 890 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 891 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 892 rw); 893#ifdef KDTRACE_HOOKS |
| 894 sleep_time -= lockstat_nsecs(); | 894 sleep_time -= lockstat_nsecs(&rw->lock_object); |
| 895#endif 896 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 897#ifdef KDTRACE_HOOKS | 895#endif 896 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 897#ifdef KDTRACE_HOOKS |
| 898 sleep_time += lockstat_nsecs(); | 898 sleep_time += lockstat_nsecs(&rw->lock_object); |
| 899 sleep_cnt++; 900#endif 901 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 902 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 903 __func__, rw); 904#ifdef ADAPTIVE_RWLOCKS 905 spintries = 0; 906#endif 907 } 908#ifdef KDTRACE_HOOKS | 899 sleep_cnt++; 900#endif 901 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 902 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 903 __func__, rw); 904#ifdef ADAPTIVE_RWLOCKS 905 spintries = 0; 906#endif 907 } 908#ifdef KDTRACE_HOOKS |
| 909 all_time += lockstat_nsecs(); | 909 all_time += lockstat_nsecs(&rw->lock_object); |
| 910 if (sleep_time) 911 LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time, 912 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 913 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 914 915 /* Record only the loops spinning and not sleeping. */ 916 if (spin_cnt > sleep_cnt) 917 LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, all_time - sleep_time, --- 355 unchanged lines hidden --- | 910 if (sleep_time) 911 LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time, 912 LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, 913 (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); 914 915 /* Record only the loops spinning and not sleeping. */ 916 if (spin_cnt > sleep_cnt) 917 LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, all_time - sleep_time, --- 355 unchanged lines hidden --- |