Deleted Added
sdiff udiff text old ( 173960 ) new ( 174629 )
full compact
kern_rwlock.c (173960) kern_rwlock.c (174629)
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 173960 2007-11-26 22:37:35Z attilio $");
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 174629 2007-12-15 23:13:31Z jeff $");
36
37#include "opt_ddb.h"
38#include "opt_no_adaptive_rwlocks.h"
39
40#include <sys/param.h>
41#include <sys/ktr.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>

--- 178 unchanged lines hidden (view full) ---

222
223void
224_rw_rlock(struct rwlock *rw, const char *file, int line)
225{
226 struct turnstile *ts;
227#ifdef ADAPTIVE_RWLOCKS
228 volatile struct thread *owner;
229#endif
36
37#include "opt_ddb.h"
38#include "opt_no_adaptive_rwlocks.h"
39
40#include <sys/param.h>
41#include <sys/ktr.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>

--- 178 unchanged lines hidden (view full) ---

222
223void
224_rw_rlock(struct rwlock *rw, const char *file, int line)
225{
226 struct turnstile *ts;
227#ifdef ADAPTIVE_RWLOCKS
228 volatile struct thread *owner;
229#endif
230#ifdef LOCK_PROFILING_SHARED
231 uint64_t waittime = 0;
232 int contested = 0;
230 uint64_t waittime = 0;
231 int contested = 0;
233#endif
234 uintptr_t x;
235
236 KASSERT(rw->rw_lock != RW_DESTROYED,
237 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
238 KASSERT(rw_wowner(rw) != curthread,
239 ("%s (%s): wlock already held @ %s:%d", __func__,
240 rw->lock_object.lo_name, file, line));
241 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line);

--- 26 unchanged lines hidden (view full) ---

268 /*
269 * The RW_LOCK_READ_WAITERS flag should only be set
270 * if another thread currently holds a write lock,
271 * and in that case RW_LOCK_READ should be clear.
272 */
273 MPASS((x & RW_LOCK_READ_WAITERS) == 0);
274 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x,
275 x + RW_ONE_READER)) {
232 uintptr_t x;
233
234 KASSERT(rw->rw_lock != RW_DESTROYED,
235 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
236 KASSERT(rw_wowner(rw) != curthread,
237 ("%s (%s): wlock already held @ %s:%d", __func__,
238 rw->lock_object.lo_name, file, line));
239 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line);

--- 26 unchanged lines hidden (view full) ---

266 /*
267 * The RW_LOCK_READ_WAITERS flag should only be set
268 * if another thread currently holds a write lock,
269 * and in that case RW_LOCK_READ should be clear.
270 */
271 MPASS((x & RW_LOCK_READ_WAITERS) == 0);
272 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x,
273 x + RW_ONE_READER)) {
276#ifdef LOCK_PROFILING_SHARED
277 if (RW_READERS(x) == 0)
278 lock_profile_obtain_lock_success(
279 &rw->lock_object, contested,
280 waittime, file, line);
281#endif
282 if (LOCK_LOG_TEST(&rw->lock_object, 0))
283 CTR4(KTR_LOCK,
284 "%s: %p succeed %p -> %p", __func__,
285 rw, (void *)x,
286 (void *)(x + RW_ONE_READER));
287 break;
288 }
289 cpu_spinwait();
290 continue;
291 }
274 if (LOCK_LOG_TEST(&rw->lock_object, 0))
275 CTR4(KTR_LOCK,
276 "%s: %p succeed %p -> %p", __func__,
277 rw, (void *)x,
278 (void *)(x + RW_ONE_READER));
279 break;
280 }
281 cpu_spinwait();
282 continue;
283 }
284 lock_profile_obtain_lock_failed(&rw->lock_object,
285 &contested, &waittime);
292
293#ifdef ADAPTIVE_RWLOCKS
294 /*
295 * If the owner is running on another CPU, spin until
296 * the owner stops running or the state of the lock
297 * changes.
298 */
299 owner = (struct thread *)RW_OWNER(x);
300 if (TD_IS_RUNNING(owner)) {
301 if (LOCK_LOG_TEST(&rw->lock_object, 0))
302 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
303 __func__, rw, owner);
286
287#ifdef ADAPTIVE_RWLOCKS
288 /*
289 * If the owner is running on another CPU, spin until
290 * the owner stops running or the state of the lock
291 * changes.
292 */
293 owner = (struct thread *)RW_OWNER(x);
294 if (TD_IS_RUNNING(owner)) {
295 if (LOCK_LOG_TEST(&rw->lock_object, 0))
296 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
297 __func__, rw, owner);
304#ifdef LOCK_PROFILING_SHARED
305 lock_profile_obtain_lock_failed(&rw->lock_object,
306 &contested, &waittime);
307#endif
308 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
309 TD_IS_RUNNING(owner))
310 cpu_spinwait();
311 continue;
312 }
313#endif
314
315 /*

--- 48 unchanged lines hidden (view full) ---

364
365 /*
366 * We were unable to acquire the lock and the read waiters
367 * flag is set, so we must block on the turnstile.
368 */
369 if (LOCK_LOG_TEST(&rw->lock_object, 0))
370 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
371 rw);
298 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
299 TD_IS_RUNNING(owner))
300 cpu_spinwait();
301 continue;
302 }
303#endif
304
305 /*

--- 48 unchanged lines hidden (view full) ---

354
355 /*
356 * We were unable to acquire the lock and the read waiters
357 * flag is set, so we must block on the turnstile.
358 */
359 if (LOCK_LOG_TEST(&rw->lock_object, 0))
360 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
361 rw);
372#ifdef LOCK_PROFILING_SHARED
373 lock_profile_obtain_lock_failed(&rw->lock_object, &contested,
374 &waittime);
375#endif
376 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE);
377 if (LOCK_LOG_TEST(&rw->lock_object, 0))
378 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
379 __func__, rw);
380 }
381
382 /*
383 * TODO: acquire "owner of record" here. Here be turnstile dragons
384 * however. turnstiles don't like owners changing between calls to
385 * turnstile_wait() currently.
386 */
362 turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE);
363 if (LOCK_LOG_TEST(&rw->lock_object, 0))
364 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
365 __func__, rw);
366 }
367
368 /*
369 * TODO: acquire "owner of record" here. Here be turnstile dragons
370 * however. turnstiles don't like owners changing between calls to
371 * turnstile_wait() currently.
372 */
387
373 lock_profile_obtain_lock_success( &rw->lock_object, contested,
374 waittime, file, line);
388 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
389 WITNESS_LOCK(&rw->lock_object, 0, file, line);
390 curthread->td_locks++;
391}
392
393void
394_rw_runlock(struct rwlock *rw, const char *file, int line)
395{

--- 30 unchanged lines hidden (view full) ---

426
427
428 /*
429 * We should never have read waiters while at least one
430 * thread holds a read lock. (See note above)
431 */
432 KASSERT(!(x & RW_LOCK_READ_WAITERS),
433 ("%s: waiting readers", __func__));
375 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
376 WITNESS_LOCK(&rw->lock_object, 0, file, line);
377 curthread->td_locks++;
378}
379
380void
381_rw_runlock(struct rwlock *rw, const char *file, int line)
382{

--- 30 unchanged lines hidden (view full) ---

413
414
415 /*
416 * We should never have read waiters while at least one
417 * thread holds a read lock. (See note above)
418 */
419 KASSERT(!(x & RW_LOCK_READ_WAITERS),
420 ("%s: waiting readers", __func__));
434#ifdef LOCK_PROFILING_SHARED
435 lock_profile_release_lock(&rw->lock_object);
436#endif
437
438 /*
439 * If there aren't any waiters for a write lock, then try
440 * to drop it quickly.
441 */
442 if (!(x & RW_LOCK_WRITE_WAITERS)) {
443
444 /*

--- 60 unchanged lines hidden (view full) ---

505 */
506 ts = turnstile_lookup(&rw->lock_object);
507 MPASS(ts != NULL);
508 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
509 turnstile_unpend(ts, TS_SHARED_LOCK);
510 turnstile_chain_unlock(&rw->lock_object);
511 break;
512 }
421
422 /*
423 * If there aren't any waiters for a write lock, then try
424 * to drop it quickly.
425 */
426 if (!(x & RW_LOCK_WRITE_WAITERS)) {
427
428 /*

--- 60 unchanged lines hidden (view full) ---

489 */
490 ts = turnstile_lookup(&rw->lock_object);
491 MPASS(ts != NULL);
492 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
493 turnstile_unpend(ts, TS_SHARED_LOCK);
494 turnstile_chain_unlock(&rw->lock_object);
495 break;
496 }
497 lock_profile_release_lock(&rw->lock_object);
513}
514
515/*
516 * This function is called when we are unable to obtain a write lock on the
517 * first try. This means that at least one other thread holds either a
518 * read or write lock.
519 */
520void

--- 18 unchanged lines hidden (view full) ---

539 return;
540 }
541
542 if (LOCK_LOG_TEST(&rw->lock_object, 0))
543 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
544 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
545
546 while (!_rw_write_lock(rw, tid)) {
498}
499
500/*
501 * This function is called when we are unable to obtain a write lock on the
502 * first try. This means that at least one other thread holds either a
503 * read or write lock.
504 */
505void

--- 18 unchanged lines hidden (view full) ---

524 return;
525 }
526
527 if (LOCK_LOG_TEST(&rw->lock_object, 0))
528 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
529 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
530
531 while (!_rw_write_lock(rw, tid)) {
532 lock_profile_obtain_lock_failed(&rw->lock_object,
533 &contested, &waittime);
547#ifdef ADAPTIVE_RWLOCKS
548 /*
549 * If the lock is write locked and the owner is
550 * running on another CPU, spin until the owner stops
551 * running or the state of the lock changes.
552 */
553 v = rw->rw_lock;
554 owner = (struct thread *)RW_OWNER(v);
555 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
556 if (LOCK_LOG_TEST(&rw->lock_object, 0))
557 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
558 __func__, rw, owner);
534#ifdef ADAPTIVE_RWLOCKS
535 /*
536 * If the lock is write locked and the owner is
537 * running on another CPU, spin until the owner stops
538 * running or the state of the lock changes.
539 */
540 v = rw->rw_lock;
541 owner = (struct thread *)RW_OWNER(v);
542 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
543 if (LOCK_LOG_TEST(&rw->lock_object, 0))
544 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
545 __func__, rw, owner);
559 lock_profile_obtain_lock_failed(&rw->lock_object,
560 &contested, &waittime);
561 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
562 TD_IS_RUNNING(owner))
563 cpu_spinwait();
564 continue;
565 }
566#endif
567
568 ts = turnstile_trywait(&rw->lock_object);

--- 67 unchanged lines hidden (view full) ---

636
637 /*
638 * We were unable to acquire the lock and the write waiters
639 * flag is set, so we must block on the turnstile.
640 */
641 if (LOCK_LOG_TEST(&rw->lock_object, 0))
642 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
643 rw);
546 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
547 TD_IS_RUNNING(owner))
548 cpu_spinwait();
549 continue;
550 }
551#endif
552
553 ts = turnstile_trywait(&rw->lock_object);

--- 67 unchanged lines hidden (view full) ---

621
622 /*
623 * We were unable to acquire the lock and the write waiters
624 * flag is set, so we must block on the turnstile.
625 */
626 if (LOCK_LOG_TEST(&rw->lock_object, 0))
627 CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
628 rw);
644 lock_profile_obtain_lock_failed(&rw->lock_object, &contested,
645 &waittime);
646 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE);
647 if (LOCK_LOG_TEST(&rw->lock_object, 0))
648 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
649 __func__, rw);
650 }
651 lock_profile_obtain_lock_success(&rw->lock_object, contested, waittime,
652 file, line);
653}

--- 304 unchanged lines hidden --- 		629 turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE);
630 if (LOCK_LOG_TEST(&rw->lock_object, 0))
631 CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
632 __func__, rw);
633 }
634 lock_profile_obtain_lock_success(&rw->lock_object, contested, waittime,
635 file, line);
636}

--- 304 unchanged lines hidden ---