Deleted Added
sdiff udiff text old ( 177843 ) new ( 177912 )
full compact
kern_rwlock.c (177843) kern_rwlock.c (177912)
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 18 unchanged lines hidden (view full) ---

27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 177843 2008-04-01 20:31:55Z attilio $");
35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 177912 2008-04-04 10:00:46Z jeff $");
36
37#include "opt_ddb.h"
38#include "opt_no_adaptive_rwlocks.h"
39
40#include <sys/param.h>
41#include <sys/ktr.h>
36
37#include "opt_ddb.h"
38#include "opt_no_adaptive_rwlocks.h"
39
40#include <sys/param.h>
41#include <sys/ktr.h>
42#include <sys/kernel.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>
44#include <sys/proc.h>
45#include <sys/rwlock.h>
43#include <sys/lock.h>
44#include <sys/mutex.h>
45#include <sys/proc.h>
46#include <sys/rwlock.h>
47#include <sys/sysctl.h>
46#include <sys/systm.h>
47#include <sys/turnstile.h>
48
49#include <machine/cpu.h>
50
51CTASSERT((RW_RECURSE & LO_CLASSFLAGS) == RW_RECURSE);
52
53#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
54#define ADAPTIVE_RWLOCKS
55#endif
56
48#include <sys/systm.h>
49#include <sys/turnstile.h>
50
51#include <machine/cpu.h>
52
53CTASSERT((RW_RECURSE & LO_CLASSFLAGS) == RW_RECURSE);
54
55#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
56#define ADAPTIVE_RWLOCKS
57#endif
58
59#ifdef ADAPTIVE_RWLOCKS
60static int rowner_retries = 10;
61static int rowner_loops = 10000;
62SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, "rwlock debugging");
63SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, "");
64SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, "");
65#endif
66
57#ifdef DDB
58#include <ddb/ddb.h>
59
60static void db_show_rwlock(struct lock_object *lock);
61#endif
62static void assert_rw(struct lock_object *lock, int what);
63static void lock_rw(struct lock_object *lock, int how);
64static int unlock_rw(struct lock_object *lock);

--- 191 unchanged lines hidden (view full) ---

256 RW_LOCK_READ)
257
258void
259_rw_rlock(struct rwlock *rw, const char *file, int line)
260{
261 struct turnstile *ts;
262#ifdef ADAPTIVE_RWLOCKS
263 volatile struct thread *owner;
67#ifdef DDB
68#include <ddb/ddb.h>
69
70static void db_show_rwlock(struct lock_object *lock);
71#endif
72static void assert_rw(struct lock_object *lock, int what);
73static void lock_rw(struct lock_object *lock, int how);
74static int unlock_rw(struct lock_object *lock);

--- 191 unchanged lines hidden (view full) ---

266 RW_LOCK_READ)
267
268void
269_rw_rlock(struct rwlock *rw, const char *file, int line)
270{
271 struct turnstile *ts;
272#ifdef ADAPTIVE_RWLOCKS
273 volatile struct thread *owner;
274 int spintries = 0;
275 int i;
264#endif
265 uint64_t waittime = 0;
266 int contested = 0;
267 uintptr_t v;
268
269 KASSERT(rw->rw_lock != RW_DESTROYED,
270 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
271 KASSERT(rw_wowner(rw) != curthread,

--- 47 unchanged lines hidden (view full) ---

319 CTR3(KTR_LOCK,
320 "%s: spinning on %p held by %p",
321 __func__, rw, owner);
322 while ((struct thread*)RW_OWNER(rw->rw_lock) ==
323 owner && TD_IS_RUNNING(owner))
324 cpu_spinwait();
325 continue;
326 }
276#endif
277 uint64_t waittime = 0;
278 int contested = 0;
279 uintptr_t v;
280
281 KASSERT(rw->rw_lock != RW_DESTROYED,
282 ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
283 KASSERT(rw_wowner(rw) != curthread,

--- 47 unchanged lines hidden (view full) ---

331 CTR3(KTR_LOCK,
332 "%s: spinning on %p held by %p",
333 __func__, rw, owner);
334 while ((struct thread*)RW_OWNER(rw->rw_lock) ==
335 owner && TD_IS_RUNNING(owner))
336 cpu_spinwait();
337 continue;
338 }
339 } else if (spintries < rowner_retries) {
340 spintries++;
341 for (i = 0; i < rowner_loops; i++) {
342 v = rw->rw_lock;
343 if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v))
344 break;
345 cpu_spinwait();
346 }
347 if (i != rowner_loops)
348 continue;
327 }
328#endif
329
330 /*
331 * Okay, now it's the hard case. Some other thread already
332 * has a write lock or there are write waiters present,
333 * acquire the turnstile lock so we can begin the process
334 * of blocking.

--- 252 unchanged lines hidden (view full) ---

587 if (LOCK_LOG_TEST(&rw->lock_object, 0))
588 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
589 __func__, rw, owner);
590 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
591 TD_IS_RUNNING(owner))
592 cpu_spinwait();
593 continue;
594 }
349 }
350#endif
351
352 /*
353 * Okay, now it's the hard case. Some other thread already
354 * has a write lock or there are write waiters present,
355 * acquire the turnstile lock so we can begin the process
356 * of blocking.

--- 252 unchanged lines hidden (view full) ---

609 if (LOCK_LOG_TEST(&rw->lock_object, 0))
610 CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
611 __func__, rw, owner);
612 while ((struct thread*)RW_OWNER(rw->rw_lock) == owner &&
613 TD_IS_RUNNING(owner))
614 cpu_spinwait();
615 continue;
616 }
595 if ((v & RW_LOCK_READ) && RW_READERS(v) && spintries < 100) {
617 if ((v & RW_LOCK_READ) && RW_READERS(v) &&
618 spintries < rowner_retries) {
596 if (!(v & RW_LOCK_WRITE_SPINNER)) {
597 if (!atomic_cmpset_ptr(&rw->rw_lock, v,
598 v | RW_LOCK_WRITE_SPINNER)) {
599 cpu_spinwait();
600 continue;
601 }
602 }
603 spintries++;
619 if (!(v & RW_LOCK_WRITE_SPINNER)) {
620 if (!atomic_cmpset_ptr(&rw->rw_lock, v,
621 v | RW_LOCK_WRITE_SPINNER)) {
622 cpu_spinwait();
623 continue;
624 }
625 }
626 spintries++;
604 for (i = 100000; i > 0; i--) {
627 for (i = 0; i < rowner_loops; i++) {
605 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0)
606 break;
607 cpu_spinwait();
608 }
628 if ((rw->rw_lock & RW_LOCK_WRITE_SPINNER) == 0)
629 break;
630 cpu_spinwait();
631 }
609 if (i)
632 if (i != rowner_loops)
610 continue;
611 }
612#endif
613 ts = turnstile_trywait(&rw->lock_object);
614 v = rw->rw_lock;
615
616#ifdef ADAPTIVE_RWLOCKS
617 /*

--- 388 unchanged lines hidden --- 		633 continue;
634 }
635#endif
636 ts = turnstile_trywait(&rw->lock_object);
637 v = rw->rw_lock;
638
639#ifdef ADAPTIVE_RWLOCKS
640 /*

--- 388 unchanged lines hidden ---