Deleted Added
sdiff udiff text old ( 227788 ) new ( 228424 )
full compact
kern_sx.c (227788) kern_sx.c (228424)
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 27 unchanged lines hidden (view full) ---

36 * so should not be relied upon in combination with sx locks.
37 */
38
39#include "opt_ddb.h"
40#include "opt_kdtrace.h"
41#include "opt_no_adaptive_sx.h"
42
43#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 27 unchanged lines hidden (view full) ---

36 * so should not be relied upon in combination with sx locks.
37 */
38
39#include "opt_ddb.h"
40#include "opt_kdtrace.h"
41#include "opt_no_adaptive_sx.h"
42
43#include <sys/cdefs.h>
44__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 227788 2011-11-21 12:59:52Z attilio $");
44__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 228424 2011-12-11 21:02:01Z avg $");
45
46#include <sys/param.h>
47#include <sys/ktr.h>
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/proc.h>
51#include <sys/sleepqueue.h>
52#include <sys/sx.h>

--- 183 unchanged lines hidden (view full) ---

236 lock_destroy(&sx->lock_object);
237}
238
239int
240_sx_slock(struct sx *sx, int opts, const char *file, int line)
241{
242 int error = 0;
243
45
46#include <sys/param.h>
47#include <sys/ktr.h>
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/proc.h>
51#include <sys/sleepqueue.h>
52#include <sys/sx.h>

--- 183 unchanged lines hidden (view full) ---

236 lock_destroy(&sx->lock_object);
237}
238
239int
240_sx_slock(struct sx *sx, int opts, const char *file, int line)
241{
242 int error = 0;
243
244 if (SCHEDULER_STOPPED())
245 return (0);
244 MPASS(curthread != NULL);
245 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
246 ("sx_slock() of destroyed sx @ %s:%d", file, line));
247 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
248 error = __sx_slock(sx, opts, file, line);
249 if (!error) {
250 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
251 WITNESS_LOCK(&sx->lock_object, 0, file, line);
252 curthread->td_locks++;
253 }
254
255 return (error);
256}
257
258int
259sx_try_slock_(struct sx *sx, const char *file, int line)
260{
261 uintptr_t x;
262
246 MPASS(curthread != NULL);
247 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
248 ("sx_slock() of destroyed sx @ %s:%d", file, line));
249 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL);
250 error = __sx_slock(sx, opts, file, line);
251 if (!error) {
252 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
253 WITNESS_LOCK(&sx->lock_object, 0, file, line);
254 curthread->td_locks++;
255 }
256
257 return (error);
258}
259
260int
261sx_try_slock_(struct sx *sx, const char *file, int line)
262{
263 uintptr_t x;
264
265 if (SCHEDULER_STOPPED())
266 return (1);
267
263 for (;;) {
264 x = sx->sx_lock;
265 KASSERT(x != SX_LOCK_DESTROYED,
266 ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
267 if (!(x & SX_LOCK_SHARED))
268 break;
269 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
270 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);

--- 7 unchanged lines hidden (view full) ---

278 return (0);
279}
280
281int
282_sx_xlock(struct sx *sx, int opts, const char *file, int line)
283{
284 int error = 0;
285
268 for (;;) {
269 x = sx->sx_lock;
270 KASSERT(x != SX_LOCK_DESTROYED,
271 ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
272 if (!(x & SX_LOCK_SHARED))
273 break;
274 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
275 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);

--- 7 unchanged lines hidden (view full) ---

283 return (0);
284}
285
286int
287_sx_xlock(struct sx *sx, int opts, const char *file, int line)
288{
289 int error = 0;
290
291 if (SCHEDULER_STOPPED())
292 return (0);
286 MPASS(curthread != NULL);
287 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
288 ("sx_xlock() of destroyed sx @ %s:%d", file, line));
289 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
290 line, NULL);
291 error = __sx_xlock(sx, curthread, opts, file, line);
292 if (!error) {
293 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,

--- 5 unchanged lines hidden (view full) ---

299 return (error);
300}
301
302int
303sx_try_xlock_(struct sx *sx, const char *file, int line)
304{
305 int rval;
306
293 MPASS(curthread != NULL);
294 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
295 ("sx_xlock() of destroyed sx @ %s:%d", file, line));
296 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
297 line, NULL);
298 error = __sx_xlock(sx, curthread, opts, file, line);
299 if (!error) {
300 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,

--- 5 unchanged lines hidden (view full) ---

306 return (error);
307}
308
309int
310sx_try_xlock_(struct sx *sx, const char *file, int line)
311{
312 int rval;
313
314 if (SCHEDULER_STOPPED())
315 return (1);
316
307 MPASS(curthread != NULL);
308 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
309 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
310
311 if (sx_xlocked(sx) &&
312 (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) {
313 sx->sx_recurse++;
314 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);

--- 10 unchanged lines hidden (view full) ---

325
326 return (rval);
327}
328
329void
330_sx_sunlock(struct sx *sx, const char *file, int line)
331{
332
317 MPASS(curthread != NULL);
318 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
319 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
320
321 if (sx_xlocked(sx) &&
322 (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) {
323 sx->sx_recurse++;
324 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);

--- 10 unchanged lines hidden (view full) ---

335
336 return (rval);
337}
338
339void
340_sx_sunlock(struct sx *sx, const char *file, int line)
341{
342
343 if (SCHEDULER_STOPPED())
344 return;
333 MPASS(curthread != NULL);
334 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
335 ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
336 _sx_assert(sx, SA_SLOCKED, file, line);
337 curthread->td_locks--;
338 WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
339 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
340 __sx_sunlock(sx, file, line);
341 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx);
342}
343
344void
345_sx_xunlock(struct sx *sx, const char *file, int line)
346{
347
345 MPASS(curthread != NULL);
346 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
347 ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
348 _sx_assert(sx, SA_SLOCKED, file, line);
349 curthread->td_locks--;
350 WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
351 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
352 __sx_sunlock(sx, file, line);
353 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx);
354}
355
356void
357_sx_xunlock(struct sx *sx, const char *file, int line)
358{
359
360 if (SCHEDULER_STOPPED())
361 return;
348 MPASS(curthread != NULL);
349 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
350 ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
351 _sx_assert(sx, SA_XLOCKED, file, line);
352 curthread->td_locks--;
353 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
354 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
355 line);

--- 8 unchanged lines hidden (view full) ---

364 * Return 1 if if the upgrade succeed, 0 otherwise.
365 */
366int
367sx_try_upgrade_(struct sx *sx, const char *file, int line)
368{
369 uintptr_t x;
370 int success;
371
362 MPASS(curthread != NULL);
363 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
364 ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
365 _sx_assert(sx, SA_XLOCKED, file, line);
366 curthread->td_locks--;
367 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
368 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
369 line);

--- 8 unchanged lines hidden (view full) ---

378 * Return 1 if if the upgrade succeed, 0 otherwise.
379 */
380int
381sx_try_upgrade_(struct sx *sx, const char *file, int line)
382{
383 uintptr_t x;
384 int success;
385
386 if (SCHEDULER_STOPPED())
387 return (1);
388
372 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
373 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
374 _sx_assert(sx, SA_SLOCKED, file, line);
375
376 /*
377 * Try to switch from one shared lock to an exclusive lock. We need
378 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
379 * we will wake up the exclusive waiters when we drop the lock.

--- 14 unchanged lines hidden (view full) ---

394 * Downgrade an unrecursed exclusive lock into a single shared lock.
395 */
396void
397sx_downgrade_(struct sx *sx, const char *file, int line)
398{
399 uintptr_t x;
400 int wakeup_swapper;
401
389 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
390 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
391 _sx_assert(sx, SA_SLOCKED, file, line);
392
393 /*
394 * Try to switch from one shared lock to an exclusive lock. We need
395 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
396 * we will wake up the exclusive waiters when we drop the lock.

--- 14 unchanged lines hidden (view full) ---

411 * Downgrade an unrecursed exclusive lock into a single shared lock.
412 */
413void
414sx_downgrade_(struct sx *sx, const char *file, int line)
415{
416 uintptr_t x;
417 int wakeup_swapper;
418
419 if (SCHEDULER_STOPPED())
420 return;
421
402 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
403 ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
404 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
405#ifndef INVARIANTS
406 if (sx_recursed(sx))
407 panic("downgrade of a recursed lock");
408#endif
409

--- 66 unchanged lines hidden (view full) ---

476#endif
477 int error = 0;
478#ifdef KDTRACE_HOOKS
479 uint64_t spin_cnt = 0;
480 uint64_t sleep_cnt = 0;
481 int64_t sleep_time = 0;
482#endif
483
422 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
423 ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
424 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
425#ifndef INVARIANTS
426 if (sx_recursed(sx))
427 panic("downgrade of a recursed lock");
428#endif
429

--- 66 unchanged lines hidden (view full) ---

496#endif
497 int error = 0;
498#ifdef KDTRACE_HOOKS
499 uint64_t spin_cnt = 0;
500 uint64_t sleep_cnt = 0;
501 int64_t sleep_time = 0;
502#endif
503
504 if (SCHEDULER_STOPPED())
505 return (0);
506
484 /* If we already hold an exclusive lock, then recurse. */
485 if (sx_xlocked(sx)) {
486 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
487 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
488 sx->lock_object.lo_name, file, line));
489 sx->sx_recurse++;
490 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
491 if (LOCK_LOG_TEST(&sx->lock_object, 0))

--- 184 unchanged lines hidden (view full) ---

676 * accessible from at least sx.h.
677 */
678void
679_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
680{
681 uintptr_t x;
682 int queue, wakeup_swapper;
683
507 /* If we already hold an exclusive lock, then recurse. */
508 if (sx_xlocked(sx)) {
509 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
510 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
511 sx->lock_object.lo_name, file, line));
512 sx->sx_recurse++;
513 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
514 if (LOCK_LOG_TEST(&sx->lock_object, 0))

--- 184 unchanged lines hidden (view full) ---

699 * accessible from at least sx.h.
700 */
701void
702_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
703{
704 uintptr_t x;
705 int queue, wakeup_swapper;
706
707 if (SCHEDULER_STOPPED())
708 return;
709
684 MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
685
686 /* If the lock is recursed, then unrecurse one level. */
687 if (sx_xlocked(sx) && sx_recursed(sx)) {
688 if ((--sx->sx_recurse) == 0)
689 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
690 if (LOCK_LOG_TEST(&sx->lock_object, 0))
691 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);

--- 56 unchanged lines hidden (view full) ---

748 uintptr_t x;
749 int error = 0;
750#ifdef KDTRACE_HOOKS
751 uint64_t spin_cnt = 0;
752 uint64_t sleep_cnt = 0;
753 int64_t sleep_time = 0;
754#endif
755
710 MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
711
712 /* If the lock is recursed, then unrecurse one level. */
713 if (sx_xlocked(sx) && sx_recursed(sx)) {
714 if ((--sx->sx_recurse) == 0)
715 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
716 if (LOCK_LOG_TEST(&sx->lock_object, 0))
717 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);

--- 56 unchanged lines hidden (view full) ---

774 uintptr_t x;
775 int error = 0;
776#ifdef KDTRACE_HOOKS
777 uint64_t spin_cnt = 0;
778 uint64_t sleep_cnt = 0;
779 int64_t sleep_time = 0;
780#endif
781
782 if (SCHEDULER_STOPPED())
783 return (0);
784
756 /*
757 * As with rwlocks, we don't make any attempt to try to block
758 * shared locks once there is an exclusive waiter.
759 */
760 for (;;) {
761#ifdef KDTRACE_HOOKS
762 spin_cnt++;
763#endif

--- 150 unchanged lines hidden (view full) ---

914 * accessible from at least sx.h.
915 */
916void
917_sx_sunlock_hard(struct sx *sx, const char *file, int line)
918{
919 uintptr_t x;
920 int wakeup_swapper;
921
785 /*
786 * As with rwlocks, we don't make any attempt to try to block
787 * shared locks once there is an exclusive waiter.
788 */
789 for (;;) {
790#ifdef KDTRACE_HOOKS
791 spin_cnt++;
792#endif

--- 150 unchanged lines hidden (view full) ---

943 * accessible from at least sx.h.
944 */
945void
946_sx_sunlock_hard(struct sx *sx, const char *file, int line)
947{
948 uintptr_t x;
949 int wakeup_swapper;
950
951 if (SCHEDULER_STOPPED())
952 return;
953
922 for (;;) {
923 x = sx->sx_lock;
924
925 /*
926 * We should never have sharers while at least one thread
927 * holds a shared lock.
928 */
929 KASSERT(!(x & SX_LOCK_SHARED_WAITERS),

--- 232 unchanged lines hidden --- 		954 for (;;) {
955 x = sx->sx_lock;
956
957 /*
958 * We should never have sharers while at least one thread
959 * holds a shared lock.
960 */
961 KASSERT(!(x & SX_LOCK_SHARED_WAITERS),

--- 232 unchanged lines hidden ---