Deleted Added
sdiff udiff text old ( 164246 ) new ( 167012 )
full compact
kern_lock.c (164246) kern_lock.c (167012)
1/*-
2 * Copyright (c) 1995
3 * The Regents of the University of California. All rights reserved.
4 *
5 * Copyright (C) 1997
6 * John S. Dyson. All rights reserved.
7 *
8 * This code contains ideas from software contributed to Berkeley by

--- 27 unchanged lines hidden (view full) ---

36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95
41 */
42
43#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1995
3 * The Regents of the University of California. All rights reserved.
4 *
5 * Copyright (C) 1997
6 * John S. Dyson. All rights reserved.
7 *
8 * This code contains ideas from software contributed to Berkeley by

--- 27 unchanged lines hidden (view full) ---

36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95
41 */
42
43#include <sys/cdefs.h>
44__FBSDID("$FreeBSD: head/sys/kern/kern_lock.c 164246 2006-11-13 05:41:46Z kmacy $");
44__FBSDID("$FreeBSD: head/sys/kern/kern_lock.c 167012 2007-02-26 08:26:44Z kmacy $");
45
46#include "opt_ddb.h"
47#include "opt_global.h"
48
49#include <sys/param.h>
50#include <sys/kdb.h>
51#include <sys/kernel.h>
52#include <sys/ktr.h>

--- 26 unchanged lines hidden (view full) ---

79 * Locking primitives implementation.
80 * Locks provide shared/exclusive sychronization.
81 */
82
83#define COUNT(td, x) if ((td)) (td)->td_locks += (x)
84#define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \
85 LK_SHARE_NONZERO | LK_WAIT_NONZERO)
86
45
46#include "opt_ddb.h"
47#include "opt_global.h"
48
49#include <sys/param.h>
50#include <sys/kdb.h>
51#include <sys/kernel.h>
52#include <sys/ktr.h>

--- 26 unchanged lines hidden (view full) ---

79 * Locking primitives implementation.
80 * Locks provide shared/exclusive sychronization.
81 */
82
83#define COUNT(td, x) if ((td)) (td)->td_locks += (x)
84#define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \
85 LK_SHARE_NONZERO | LK_WAIT_NONZERO)
86
87static int acquire(struct lock **lkpp, int extflags, int wanted);
87static int acquire(struct lock **lkpp, int extflags, int wanted, int *contested, uint64_t *waittime);
88static int acquiredrain(struct lock *lkp, int extflags) ;
89
90static __inline void
91sharelock(struct thread *td, struct lock *lkp, int incr) {
92 lkp->lk_flags |= LK_SHARE_NONZERO;
93 lkp->lk_sharecount += incr;
94 COUNT(td, incr);
95}

--- 11 unchanged lines hidden (view full) ---

107 }
108 lkp->lk_sharecount = 0;
109 } else {
110 lkp->lk_sharecount -= decr;
111 }
112}
113
114static int
88static int acquiredrain(struct lock *lkp, int extflags) ;
89
90static __inline void
91sharelock(struct thread *td, struct lock *lkp, int incr) {
92 lkp->lk_flags |= LK_SHARE_NONZERO;
93 lkp->lk_sharecount += incr;
94 COUNT(td, incr);
95}

--- 11 unchanged lines hidden (view full) ---

107 }
108 lkp->lk_sharecount = 0;
109 } else {
110 lkp->lk_sharecount -= decr;
111 }
112}
113
114static int
115acquire(struct lock **lkpp, int extflags, int wanted)
115acquire(struct lock **lkpp, int extflags, int wanted, int *contested, uint64_t *waittime)
116{
117 struct lock *lkp = *lkpp;
118 int error;
119 CTR3(KTR_LOCK,
120 "acquire(): lkp == %p, extflags == 0x%x, wanted == 0x%x",
121 lkp, extflags, wanted);
122
123 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted))
124 return EBUSY;
125 error = 0;
116{
117 struct lock *lkp = *lkpp;
118 int error;
119 CTR3(KTR_LOCK,
120 "acquire(): lkp == %p, extflags == 0x%x, wanted == 0x%x",
121 lkp, extflags, wanted);
122
123 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted))
124 return EBUSY;
125 error = 0;
126 if ((lkp->lk_flags & wanted) != 0)
127 lock_profile_obtain_lock_failed(&lkp->lk_object, contested, waittime);
128
126 while ((lkp->lk_flags & wanted) != 0) {
127 CTR2(KTR_LOCK,
128 "acquire(): lkp == %p, lk_flags == 0x%x sleeping",
129 lkp, lkp->lk_flags);
130 lkp->lk_flags |= LK_WAIT_NONZERO;
131 lkp->lk_waitcount++;
132 error = msleep(lkp, lkp->lk_interlock, lkp->lk_prio,
133 lkp->lk_wmesg,

--- 29 unchanged lines hidden (view full) ---

163int
164_lockmgr(struct lock *lkp, int flags, struct mtx *interlkp,
165 struct thread *td, char *file, int line)
166
167{
168 int error;
169 struct thread *thr;
170 int extflags, lockflags;
129 while ((lkp->lk_flags & wanted) != 0) {
130 CTR2(KTR_LOCK,
131 "acquire(): lkp == %p, lk_flags == 0x%x sleeping",
132 lkp, lkp->lk_flags);
133 lkp->lk_flags |= LK_WAIT_NONZERO;
134 lkp->lk_waitcount++;
135 error = msleep(lkp, lkp->lk_interlock, lkp->lk_prio,
136 lkp->lk_wmesg,

--- 29 unchanged lines hidden (view full) ---

166int
167_lockmgr(struct lock *lkp, int flags, struct mtx *interlkp,
168 struct thread *td, char *file, int line)
169
170{
171 int error;
172 struct thread *thr;
173 int extflags, lockflags;
171 uint64_t waitstart;
172
174 int contested = 0;
175 uint64_t waitstart = 0;
176
173 error = 0;
174 if (td == NULL)
175 thr = LK_KERNPROC;
176 else
177 thr = td;
178
177 error = 0;
178 if (td == NULL)
179 thr = LK_KERNPROC;
180 else
181 thr = td;
182
179 lock_profile_waitstart(&waitstart);
180 if ((flags & LK_INTERNAL) == 0)
181 mtx_lock(lkp->lk_interlock);
182 CTR6(KTR_LOCK,
183 "lockmgr(): lkp == %p (lk_wmesg == \"%s\"), owner == %p, exclusivecount == %d, flags == 0x%x, "
184 "td == %p", lkp, lkp->lk_wmesg, lkp->lk_lockholder,
185 lkp->lk_exclusivecount, flags, td);
186#ifdef DEBUG_LOCKS
187 {

--- 35 unchanged lines hidden (view full) ---

223 * However, if TDP_DEADLKTREAT is set, we override exclusive
224 * lock requests or upgrade requests ( but not the exclusive
225 * lock itself ).
226 */
227 if (lkp->lk_lockholder != thr) {
228 lockflags = LK_HAVE_EXCL;
229 if (td != NULL && !(td->td_pflags & TDP_DEADLKTREAT))
230 lockflags |= LK_WANT_EXCL | LK_WANT_UPGRADE;
183 if ((flags & LK_INTERNAL) == 0)
184 mtx_lock(lkp->lk_interlock);
185 CTR6(KTR_LOCK,
186 "lockmgr(): lkp == %p (lk_wmesg == \"%s\"), owner == %p, exclusivecount == %d, flags == 0x%x, "
187 "td == %p", lkp, lkp->lk_wmesg, lkp->lk_lockholder,
188 lkp->lk_exclusivecount, flags, td);
189#ifdef DEBUG_LOCKS
190 {

--- 35 unchanged lines hidden (view full) ---

226 * However, if TDP_DEADLKTREAT is set, we override exclusive
227 * lock requests or upgrade requests ( but not the exclusive
228 * lock itself ).
229 */
230 if (lkp->lk_lockholder != thr) {
231 lockflags = LK_HAVE_EXCL;
232 if (td != NULL && !(td->td_pflags & TDP_DEADLKTREAT))
233 lockflags |= LK_WANT_EXCL | LK_WANT_UPGRADE;
231 error = acquire(&lkp, extflags, lockflags);
234 error = acquire(&lkp, extflags, lockflags, &contested, &waitstart);
232 if (error)
233 break;
234 sharelock(td, lkp, 1);
235 if (lkp->lk_sharecount == 1)
235 if (error)
236 break;
237 sharelock(td, lkp, 1);
238 if (lkp->lk_sharecount == 1)
236 lock_profile_obtain_lock_success(&lkp->lk_object, waitstart, file, line);
239 lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
237
238#if defined(DEBUG_LOCKS)
239 stack_save(&lkp->lk_stack);
240#endif
241 break;
242 }
243 /*
244 * We hold an exclusive lock, so downgrade it to shared.
245 * An alternative would be to fail with EDEADLK.
246 */
247 sharelock(td, lkp, 1);
248 if (lkp->lk_sharecount == 1)
240
241#if defined(DEBUG_LOCKS)
242 stack_save(&lkp->lk_stack);
243#endif
244 break;
245 }
246 /*
247 * We hold an exclusive lock, so downgrade it to shared.
248 * An alternative would be to fail with EDEADLK.
249 */
250 sharelock(td, lkp, 1);
251 if (lkp->lk_sharecount == 1)
249 lock_profile_obtain_lock_success(&lkp->lk_object, waitstart, file, line);
252 lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
250 /* FALLTHROUGH downgrade */
251
252 case LK_DOWNGRADE:
253 KASSERT(lkp->lk_lockholder == thr && lkp->lk_exclusivecount != 0,
254 ("lockmgr: not holding exclusive lock "
255 "(owner thread (%p) != thread (%p), exlcnt (%d) != 0",
256 lkp->lk_lockholder, thr, lkp->lk_exclusivecount));
257 sharelock(td, lkp, lkp->lk_exclusivecount);

--- 45 unchanged lines hidden (view full) ---

303 }
304 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
305 /*
306 * We are first shared lock to request an upgrade, so
307 * request upgrade and wait for the shared count to
308 * drop to zero, then take exclusive lock.
309 */
310 lkp->lk_flags |= LK_WANT_UPGRADE;
253 /* FALLTHROUGH downgrade */
254
255 case LK_DOWNGRADE:
256 KASSERT(lkp->lk_lockholder == thr && lkp->lk_exclusivecount != 0,
257 ("lockmgr: not holding exclusive lock "
258 "(owner thread (%p) != thread (%p), exlcnt (%d) != 0",
259 lkp->lk_lockholder, thr, lkp->lk_exclusivecount));
260 sharelock(td, lkp, lkp->lk_exclusivecount);

--- 45 unchanged lines hidden (view full) ---

306 }
307 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
308 /*
309 * We are first shared lock to request an upgrade, so
310 * request upgrade and wait for the shared count to
311 * drop to zero, then take exclusive lock.
312 */
313 lkp->lk_flags |= LK_WANT_UPGRADE;
311 error = acquire(&lkp, extflags, LK_SHARE_NONZERO);
314 error = acquire(&lkp, extflags, LK_SHARE_NONZERO, &contested, &waitstart);
312 lkp->lk_flags &= ~LK_WANT_UPGRADE;
313
314 if (error) {
315 if ((lkp->lk_flags & ( LK_WANT_EXCL | LK_WAIT_NONZERO)) == (LK_WANT_EXCL | LK_WAIT_NONZERO))
316 wakeup((void *)lkp);
317 break;
318 }
319 if (lkp->lk_exclusivecount != 0)
320 panic("lockmgr: non-zero exclusive count");
321 lkp->lk_flags |= LK_HAVE_EXCL;
322 lkp->lk_lockholder = thr;
323 lkp->lk_exclusivecount = 1;
324 COUNT(td, 1);
315 lkp->lk_flags &= ~LK_WANT_UPGRADE;
316
317 if (error) {
318 if ((lkp->lk_flags & ( LK_WANT_EXCL | LK_WAIT_NONZERO)) == (LK_WANT_EXCL | LK_WAIT_NONZERO))
319 wakeup((void *)lkp);
320 break;
321 }
322 if (lkp->lk_exclusivecount != 0)
323 panic("lockmgr: non-zero exclusive count");
324 lkp->lk_flags |= LK_HAVE_EXCL;
325 lkp->lk_lockholder = thr;
326 lkp->lk_exclusivecount = 1;
327 COUNT(td, 1);
325 lock_profile_obtain_lock_success(&lkp->lk_object, waitstart, file, line);
328 lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
326#if defined(DEBUG_LOCKS)
327 stack_save(&lkp->lk_stack);
328#endif
329 break;
330 }
331 /*
332 * Someone else has requested upgrade. Release our shared
333 * lock, awaken upgrade requestor if we are the last shared

--- 23 unchanged lines hidden (view full) ---

357 if ((extflags & LK_NOWAIT) &&
358 (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) {
359 error = EBUSY;
360 break;
361 }
362 /*
363 * Try to acquire the want_exclusive flag.
364 */
329#if defined(DEBUG_LOCKS)
330 stack_save(&lkp->lk_stack);
331#endif
332 break;
333 }
334 /*
335 * Someone else has requested upgrade. Release our shared
336 * lock, awaken upgrade requestor if we are the last shared

--- 23 unchanged lines hidden (view full) ---

360 if ((extflags & LK_NOWAIT) &&
361 (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) {
362 error = EBUSY;
363 break;
364 }
365 /*
366 * Try to acquire the want_exclusive flag.
367 */
365 error = acquire(&lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL));
368 error = acquire(&lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL), &contested, &waitstart);
366 if (error)
367 break;
368 lkp->lk_flags |= LK_WANT_EXCL;
369 /*
370 * Wait for shared locks and upgrades to finish.
371 */
369 if (error)
370 break;
371 lkp->lk_flags |= LK_WANT_EXCL;
372 /*
373 * Wait for shared locks and upgrades to finish.
374 */
372 error = acquire(&lkp, extflags, LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO);
375 error = acquire(&lkp, extflags, LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, &contested, &waitstart);
373 lkp->lk_flags &= ~LK_WANT_EXCL;
374 if (error) {
375 if (lkp->lk_flags & LK_WAIT_NONZERO)
376 wakeup((void *)lkp);
377 break;
378 }
379 lkp->lk_flags |= LK_HAVE_EXCL;
380 lkp->lk_lockholder = thr;
381 if (lkp->lk_exclusivecount != 0)
382 panic("lockmgr: non-zero exclusive count");
383 lkp->lk_exclusivecount = 1;
384 COUNT(td, 1);
376 lkp->lk_flags &= ~LK_WANT_EXCL;
377 if (error) {
378 if (lkp->lk_flags & LK_WAIT_NONZERO)
379 wakeup((void *)lkp);
380 break;
381 }
382 lkp->lk_flags |= LK_HAVE_EXCL;
383 lkp->lk_lockholder = thr;
384 if (lkp->lk_exclusivecount != 0)
385 panic("lockmgr: non-zero exclusive count");
386 lkp->lk_exclusivecount = 1;
387 COUNT(td, 1);
385 lock_profile_obtain_lock_success(&lkp->lk_object, waitstart, file, line);
388 lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
386#if defined(DEBUG_LOCKS)
387 stack_save(&lkp->lk_stack);
388#endif
389 break;
390
391 case LK_RELEASE:
392 if (lkp->lk_exclusivecount != 0) {
393 if (lkp->lk_lockholder != thr &&

--- 289 unchanged lines hidden --- 		389#if defined(DEBUG_LOCKS)
390 stack_save(&lkp->lk_stack);
391#endif
392 break;
393
394 case LK_RELEASE:
395 if (lkp->lk_exclusivecount != 0) {
396 if (lkp->lk_lockholder != thr &&

--- 289 unchanged lines hidden ---