kern_rwlock.c (173960) | kern_rwlock.c (174629) |
---|---|
1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 18 unchanged lines hidden (view full) --- 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Machine independent bits of reader/writer lock implementation. 32 */ 33 34#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 18 unchanged lines hidden (view full) --- 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Machine independent bits of reader/writer lock implementation. 32 */ 33 34#include <sys/cdefs.h> |
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 173960 2007-11-26 22:37:35Z attilio $"); | 35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 174629 2007-12-15 23:13:31Z jeff $"); |
36 37#include "opt_ddb.h" 38#include "opt_no_adaptive_rwlocks.h" 39 40#include <sys/param.h> 41#include <sys/ktr.h> 42#include <sys/lock.h> 43#include <sys/mutex.h> --- 178 unchanged lines hidden (view full) --- 222 223void 224_rw_rlock(struct rwlock *rw, const char *file, int line) 225{ 226 struct turnstile *ts; 227#ifdef ADAPTIVE_RWLOCKS 228 volatile struct thread *owner; 229#endif | 36 37#include "opt_ddb.h" 38#include "opt_no_adaptive_rwlocks.h" 39 40#include <sys/param.h> 41#include <sys/ktr.h> 42#include <sys/lock.h> 43#include <sys/mutex.h> --- 178 unchanged lines hidden (view full) --- 222 223void 224_rw_rlock(struct rwlock *rw, const char *file, int line) 225{ 226 struct turnstile *ts; 227#ifdef ADAPTIVE_RWLOCKS 228 volatile struct thread *owner; 229#endif |
230#ifdef LOCK_PROFILING_SHARED | |
231 uint64_t waittime = 0; 232 int contested = 0; | 230 uint64_t waittime = 0; 231 int contested = 0; |
233#endif | |
234 uintptr_t x; 235 236 KASSERT(rw->rw_lock != RW_DESTROYED, 237 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 238 KASSERT(rw_wowner(rw) != curthread, 239 ("%s (%s): wlock already held @ %s:%d", __func__, 240 rw->lock_object.lo_name, file, line)); 241 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line); --- 26 unchanged lines hidden (view full) --- 268 /* 269 * The RW_LOCK_READ_WAITERS flag should only be set 270 * if another thread currently holds a write lock, 271 * and in that case RW_LOCK_READ should be clear. 272 */ 273 MPASS((x & RW_LOCK_READ_WAITERS) == 0); 274 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, 275 x + RW_ONE_READER)) { | 232 uintptr_t x; 233 234 KASSERT(rw->rw_lock != RW_DESTROYED, 235 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); 236 KASSERT(rw_wowner(rw) != curthread, 237 ("%s (%s): wlock already held @ %s:%d", __func__, 238 rw->lock_object.lo_name, file, line)); 239 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line); --- 26 unchanged lines hidden (view full) --- 266 /* 267 * The RW_LOCK_READ_WAITERS flag should only be set 268 * if another thread currently holds a write lock, 269 * and in that case RW_LOCK_READ should be clear. 270 */ 271 MPASS((x & RW_LOCK_READ_WAITERS) == 0); 272 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, 273 x + RW_ONE_READER)) { |
276#ifdef LOCK_PROFILING_SHARED 277 if (RW_READERS(x) == 0) 278 lock_profile_obtain_lock_success( 279 &rw->lock_object, contested, 280 waittime, file, line); 281#endif | |
282 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 283 CTR4(KTR_LOCK, 284 "%s: %p succeed %p -> %p", __func__, 285 rw, (void *)x, 286 (void *)(x + RW_ONE_READER)); 287 break; 288 } 289 cpu_spinwait(); 290 continue; 291 } | 274 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 275 CTR4(KTR_LOCK, 276 "%s: %p succeed %p -> %p", __func__, 277 rw, (void *)x, 278 (void *)(x + RW_ONE_READER)); 279 break; 280 } 281 cpu_spinwait(); 282 continue; 283 } |
284 lock_profile_obtain_lock_failed(&rw->lock_object, 285 &contested, &waittime); |
|
292 293#ifdef ADAPTIVE_RWLOCKS 294 /* 295 * If the owner is running on another CPU, spin until 296 * the owner stops running or the state of the lock 297 * changes. 298 */ 299 owner = (struct thread *)RW_OWNER(x); 300 if (TD_IS_RUNNING(owner)) { 301 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 302 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 303 __func__, rw, owner); | 286 287#ifdef ADAPTIVE_RWLOCKS 288 /* 289 * If the owner is running on another CPU, spin until 290 * the owner stops running or the state of the lock 291 * changes. 292 */ 293 owner = (struct thread *)RW_OWNER(x); 294 if (TD_IS_RUNNING(owner)) { 295 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 296 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 297 __func__, rw, owner); |
304#ifdef LOCK_PROFILING_SHARED 305 lock_profile_obtain_lock_failed(&rw->lock_object, 306 &contested, &waittime); 307#endif | |
308 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 309 TD_IS_RUNNING(owner)) 310 cpu_spinwait(); 311 continue; 312 } 313#endif 314 315 /* --- 48 unchanged lines hidden (view full) --- 364 365 /* 366 * We were unable to acquire the lock and the read waiters 367 * flag is set, so we must block on the turnstile. 368 */ 369 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 370 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 371 rw); | 298 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 299 TD_IS_RUNNING(owner)) 300 cpu_spinwait(); 301 continue; 302 } 303#endif 304 305 /* --- 48 unchanged lines hidden (view full) --- 354 355 /* 356 * We were unable to acquire the lock and the read waiters 357 * flag is set, so we must block on the turnstile. 358 */ 359 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 360 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 361 rw); |
372#ifdef LOCK_PROFILING_SHARED 373 lock_profile_obtain_lock_failed(&rw->lock_object, &contested, 374 &waittime); 375#endif | |
376 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 377 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 378 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 379 __func__, rw); 380 } 381 382 /* 383 * TODO: acquire "owner of record" here. Here be turnstile dragons 384 * however. turnstiles don't like owners changing between calls to 385 * turnstile_wait() currently. 386 */ | 362 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); 363 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 364 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 365 __func__, rw); 366 } 367 368 /* 369 * TODO: acquire "owner of record" here. Here be turnstile dragons 370 * however. turnstiles don't like owners changing between calls to 371 * turnstile_wait() currently. 372 */ |
387 | 373 lock_profile_obtain_lock_success( &rw->lock_object, contested, 374 waittime, file, line); |
388 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 389 WITNESS_LOCK(&rw->lock_object, 0, file, line); 390 curthread->td_locks++; 391} 392 393void 394_rw_runlock(struct rwlock *rw, const char *file, int line) 395{ --- 30 unchanged lines hidden (view full) --- 426 427 428 /* 429 * We should never have read waiters while at least one 430 * thread holds a read lock. (See note above) 431 */ 432 KASSERT(!(x & RW_LOCK_READ_WAITERS), 433 ("%s: waiting readers", __func__)); | 375 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); 376 WITNESS_LOCK(&rw->lock_object, 0, file, line); 377 curthread->td_locks++; 378} 379 380void 381_rw_runlock(struct rwlock *rw, const char *file, int line) 382{ --- 30 unchanged lines hidden (view full) --- 413 414 415 /* 416 * We should never have read waiters while at least one 417 * thread holds a read lock. (See note above) 418 */ 419 KASSERT(!(x & RW_LOCK_READ_WAITERS), 420 ("%s: waiting readers", __func__)); |
434#ifdef LOCK_PROFILING_SHARED 435 lock_profile_release_lock(&rw->lock_object); 436#endif | |
437 438 /* 439 * If there aren't any waiters for a write lock, then try 440 * to drop it quickly. 441 */ 442 if (!(x & RW_LOCK_WRITE_WAITERS)) { 443 444 /* --- 60 unchanged lines hidden (view full) --- 505 */ 506 ts = turnstile_lookup(&rw->lock_object); 507 MPASS(ts != NULL); 508 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 509 turnstile_unpend(ts, TS_SHARED_LOCK); 510 turnstile_chain_unlock(&rw->lock_object); 511 break; 512 } | 421 422 /* 423 * If there aren't any waiters for a write lock, then try 424 * to drop it quickly. 425 */ 426 if (!(x & RW_LOCK_WRITE_WAITERS)) { 427 428 /* --- 60 unchanged lines hidden (view full) --- 489 */ 490 ts = turnstile_lookup(&rw->lock_object); 491 MPASS(ts != NULL); 492 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 493 turnstile_unpend(ts, TS_SHARED_LOCK); 494 turnstile_chain_unlock(&rw->lock_object); 495 break; 496 } |
497 lock_profile_release_lock(&rw->lock_object); |
|
513} 514 515/* 516 * This function is called when we are unable to obtain a write lock on the 517 * first try. This means that at least one other thread holds either a 518 * read or write lock. 519 */ 520void --- 18 unchanged lines hidden (view full) --- 539 return; 540 } 541 542 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 543 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 544 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 545 546 while (!_rw_write_lock(rw, tid)) { | 498} 499 500/* 501 * This function is called when we are unable to obtain a write lock on the 502 * first try. This means that at least one other thread holds either a 503 * read or write lock. 504 */ 505void --- 18 unchanged lines hidden (view full) --- 524 return; 525 } 526 527 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 528 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 529 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); 530 531 while (!_rw_write_lock(rw, tid)) { |
532 lock_profile_obtain_lock_failed(&rw->lock_object, 533 &contested, &waittime); |
|
547#ifdef ADAPTIVE_RWLOCKS 548 /* 549 * If the lock is write locked and the owner is 550 * running on another CPU, spin until the owner stops 551 * running or the state of the lock changes. 552 */ 553 v = rw->rw_lock; 554 owner = (struct thread *)RW_OWNER(v); 555 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 556 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 557 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 558 __func__, rw, owner); | 534#ifdef ADAPTIVE_RWLOCKS 535 /* 536 * If the lock is write locked and the owner is 537 * running on another CPU, spin until the owner stops 538 * running or the state of the lock changes. 539 */ 540 v = rw->rw_lock; 541 owner = (struct thread *)RW_OWNER(v); 542 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { 543 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 544 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 545 __func__, rw, owner); |
559 lock_profile_obtain_lock_failed(&rw->lock_object, 560 &contested, &waittime); | |
561 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 562 TD_IS_RUNNING(owner)) 563 cpu_spinwait(); 564 continue; 565 } 566#endif 567 568 ts = turnstile_trywait(&rw->lock_object); --- 67 unchanged lines hidden (view full) --- 636 637 /* 638 * We were unable to acquire the lock and the write waiters 639 * flag is set, so we must block on the turnstile. 640 */ 641 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 642 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 643 rw); | 546 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && 547 TD_IS_RUNNING(owner)) 548 cpu_spinwait(); 549 continue; 550 } 551#endif 552 553 ts = turnstile_trywait(&rw->lock_object); --- 67 unchanged lines hidden (view full) --- 621 622 /* 623 * We were unable to acquire the lock and the write waiters 624 * flag is set, so we must block on the turnstile. 625 */ 626 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 627 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, 628 rw); |
644 lock_profile_obtain_lock_failed(&rw->lock_object, &contested, 645 &waittime); | |
646 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 647 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 648 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 649 __func__, rw); 650 } 651 lock_profile_obtain_lock_success(&rw->lock_object, contested, waittime, 652 file, line); 653} --- 304 unchanged lines hidden --- | 629 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); 630 if (LOCK_LOG_TEST(&rw->lock_object, 0)) 631 CTR2(KTR_LOCK, "%s: %p resuming from turnstile", 632 __func__, rw); 633 } 634 lock_profile_obtain_lock_success(&rw->lock_object, contested, waittime, 635 file, line); 636} --- 304 unchanged lines hidden --- |