Deleted Added
sdiff udiff text old ( 179025 ) new ( 181334 )
full compact
kern_sx.c (179025) kern_sx.c (181334)
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 26 unchanged lines hidden (view full) ---

35 * Priority propagation will not generally raise the priority of lock holders,
36 * so should not be relied upon in combination with sx locks.
37 */
38
39#include "opt_adaptive_sx.h"
40#include "opt_ddb.h"
41
42#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 26 unchanged lines hidden (view full) ---

35 * Priority propagation will not generally raise the priority of lock holders,
36 * so should not be relied upon in combination with sx locks.
37 */
38
39#include "opt_adaptive_sx.h"
40#include "opt_ddb.h"
41
42#include <sys/cdefs.h>
43__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 179025 2008-05-15 20:10:06Z attilio $");
43__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 181334 2008-08-05 20:02:31Z jhb $");
44
45#include <sys/param.h>
46#include <sys/ktr.h>
47#include <sys/lock.h>
48#include <sys/mutex.h>
49#include <sys/proc.h>
50#include <sys/sleepqueue.h>
51#include <sys/sx.h>

--- 304 unchanged lines hidden (view full) ---

356
357/*
358 * Downgrade an unrecursed exclusive lock into a single shared lock.
359 */
360void
361_sx_downgrade(struct sx *sx, const char *file, int line)
362{
363 uintptr_t x;
44
45#include <sys/param.h>
46#include <sys/ktr.h>
47#include <sys/lock.h>
48#include <sys/mutex.h>
49#include <sys/proc.h>
50#include <sys/sleepqueue.h>
51#include <sys/sx.h>

--- 304 unchanged lines hidden (view full) ---

356
357/*
358 * Downgrade an unrecursed exclusive lock into a single shared lock.
359 */
360void
361_sx_downgrade(struct sx *sx, const char *file, int line)
362{
363 uintptr_t x;
364 int wakeup_swapper;
364
365 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
366 ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
367 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
368#ifndef INVARIANTS
369 if (sx_recursed(sx))
370 panic("downgrade of a recursed lock");
371#endif

--- 24 unchanged lines hidden (view full) ---

396 * without any races and wakeup any shared waiters.
397 */
398 sleepq_lock(&sx->lock_object);
399
400 /*
401 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
402 * shared lock. If there are any shared waiters, wake them up.
403 */
365
366 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
367 ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
368 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
369#ifndef INVARIANTS
370 if (sx_recursed(sx))
371 panic("downgrade of a recursed lock");
372#endif

--- 24 unchanged lines hidden (view full) ---

397 * without any races and wakeup any shared waiters.
398 */
399 sleepq_lock(&sx->lock_object);
400
401 /*
402 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
403 * shared lock. If there are any shared waiters, wake them up.
404 */
405 wakeup_swapper = 0;
404 x = sx->sx_lock;
405 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
406 (x & SX_LOCK_EXCLUSIVE_WAITERS));
407 if (x & SX_LOCK_SHARED_WAITERS)
406 x = sx->sx_lock;
407 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
408 (x & SX_LOCK_EXCLUSIVE_WAITERS));
409 if (x & SX_LOCK_SHARED_WAITERS)
408 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
409 SQ_SHARED_QUEUE);
410 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
411 0, SQ_SHARED_QUEUE);
410 sleepq_release(&sx->lock_object);
411
412 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
412 sleepq_release(&sx->lock_object);
413
414 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
415
416 if (wakeup_swapper)
417 kick_proc0();
413}
414
415/*
416 * This function represents the so-called 'hard case' for sx_xlock
417 * operation. All 'easy case' failures are redirected to this. Note
418 * that ideally this would be a static function, but it needs to be
419 * accessible from at least sx.h.
420 */

--- 163 unchanged lines hidden (view full) ---

584 * operation. All 'easy case' failures are redirected to this. Note
585 * that ideally this would be a static function, but it needs to be
586 * accessible from at least sx.h.
587 */
588void
589_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
590{
591 uintptr_t x;
418}
419
420/*
421 * This function represents the so-called 'hard case' for sx_xlock
422 * operation. All 'easy case' failures are redirected to this. Note
423 * that ideally this would be a static function, but it needs to be
424 * accessible from at least sx.h.
425 */

--- 163 unchanged lines hidden (view full) ---

589 * operation. All 'easy case' failures are redirected to this. Note
590 * that ideally this would be a static function, but it needs to be
591 * accessible from at least sx.h.
592 */
593void
594_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
595{
596 uintptr_t x;
592 int queue;
597 int queue, wakeup_swapper;
593
594 MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
595
596 /* If the lock is recursed, then unrecurse one level. */
597 if (sx_xlocked(sx) && sx_recursed(sx)) {
598 if ((--sx->sx_recurse) == 0)
599 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
600 if (LOCK_LOG_TEST(&sx->lock_object, 0))

--- 21 unchanged lines hidden (view full) ---

622 queue = SQ_EXCLUSIVE_QUEUE;
623
624 /* Wake up all the waiters for the specific queue. */
625 if (LOCK_LOG_TEST(&sx->lock_object, 0))
626 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
627 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
628 "exclusive");
629 atomic_store_rel_ptr(&sx->sx_lock, x);
598
599 MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
600
601 /* If the lock is recursed, then unrecurse one level. */
602 if (sx_xlocked(sx) && sx_recursed(sx)) {
603 if ((--sx->sx_recurse) == 0)
604 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
605 if (LOCK_LOG_TEST(&sx->lock_object, 0))

--- 21 unchanged lines hidden (view full) ---

627 queue = SQ_EXCLUSIVE_QUEUE;
628
629 /* Wake up all the waiters for the specific queue. */
630 if (LOCK_LOG_TEST(&sx->lock_object, 0))
631 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
632 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
633 "exclusive");
634 atomic_store_rel_ptr(&sx->sx_lock, x);
630 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, queue);
635 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
636 queue);
631 sleepq_release(&sx->lock_object);
637 sleepq_release(&sx->lock_object);
638 if (wakeup_swapper)
639 kick_proc0();
632}
633
634/*
635 * This function represents the so-called 'hard case' for sx_slock
636 * operation. All 'easy case' failures are redirected to this. Note
637 * that ideally this would be a static function, but it needs to be
638 * accessible from at least sx.h.
639 */

--- 150 unchanged lines hidden (view full) ---

790 * operation. All 'easy case' failures are redirected to this. Note
791 * that ideally this would be a static function, but it needs to be
792 * accessible from at least sx.h.
793 */
794void
795_sx_sunlock_hard(struct sx *sx, const char *file, int line)
796{
797 uintptr_t x;
640}
641
642/*
643 * This function represents the so-called 'hard case' for sx_slock
644 * operation. All 'easy case' failures are redirected to this. Note
645 * that ideally this would be a static function, but it needs to be
646 * accessible from at least sx.h.
647 */

--- 150 unchanged lines hidden (view full) ---

798 * operation. All 'easy case' failures are redirected to this. Note
799 * that ideally this would be a static function, but it needs to be
800 * accessible from at least sx.h.
801 */
802void
803_sx_sunlock_hard(struct sx *sx, const char *file, int line)
804{
805 uintptr_t x;
806 int wakeup_swapper;
798
799 for (;;) {
800 x = sx->sx_lock;
801
802 /*
803 * We should never have sharers while at least one thread
804 * holds a shared lock.
805 */

--- 51 unchanged lines hidden (view full) ---

857 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS,
858 SX_LOCK_UNLOCKED)) {
859 sleepq_release(&sx->lock_object);
860 continue;
861 }
862 if (LOCK_LOG_TEST(&sx->lock_object, 0))
863 CTR2(KTR_LOCK, "%s: %p waking up all thread on"
864 "exclusive queue", __func__, sx);
807
808 for (;;) {
809 x = sx->sx_lock;
810
811 /*
812 * We should never have sharers while at least one thread
813 * holds a shared lock.
814 */

--- 51 unchanged lines hidden (view full) ---

866 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS,
867 SX_LOCK_UNLOCKED)) {
868 sleepq_release(&sx->lock_object);
869 continue;
870 }
871 if (LOCK_LOG_TEST(&sx->lock_object, 0))
872 CTR2(KTR_LOCK, "%s: %p waking up all thread on"
873 "exclusive queue", __func__, sx);
865 sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
866 SQ_EXCLUSIVE_QUEUE);
874 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
875 0, SQ_EXCLUSIVE_QUEUE);
867 sleepq_release(&sx->lock_object);
876 sleepq_release(&sx->lock_object);
877 if (wakeup_swapper)
878 kick_proc0();
868 break;
869 }
870}
871
872#ifdef INVARIANT_SUPPORT
873#ifndef INVARIANTS
874#undef _sx_assert
875#endif

--- 161 unchanged lines hidden --- 		879 break;
880 }
881}
882
883#ifdef INVARIANT_SUPPORT
884#ifndef INVARIANTS
885#undef _sx_assert
886#endif

--- 161 unchanged lines hidden ---