kern_sx.c (227788) | kern_sx.c (228424) |
---|---|
1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 27 unchanged lines hidden (view full) --- 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39#include "opt_ddb.h" 40#include "opt_kdtrace.h" 41#include "opt_no_adaptive_sx.h" 42 43#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 27 unchanged lines hidden (view full) --- 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39#include "opt_ddb.h" 40#include "opt_kdtrace.h" 41#include "opt_no_adaptive_sx.h" 42 43#include <sys/cdefs.h> |
44__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 227788 2011-11-21 12:59:52Z attilio $"); | 44__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 228424 2011-12-11 21:02:01Z avg $"); |
45 46#include <sys/param.h> 47#include <sys/ktr.h> 48#include <sys/lock.h> 49#include <sys/mutex.h> 50#include <sys/proc.h> 51#include <sys/sleepqueue.h> 52#include <sys/sx.h> --- 183 unchanged lines hidden (view full) --- 236 lock_destroy(&sx->lock_object); 237} 238 239int 240_sx_slock(struct sx *sx, int opts, const char *file, int line) 241{ 242 int error = 0; 243 | 45 46#include <sys/param.h> 47#include <sys/ktr.h> 48#include <sys/lock.h> 49#include <sys/mutex.h> 50#include <sys/proc.h> 51#include <sys/sleepqueue.h> 52#include <sys/sx.h> --- 183 unchanged lines hidden (view full) --- 236 lock_destroy(&sx->lock_object); 237} 238 239int 240_sx_slock(struct sx *sx, int opts, const char *file, int line) 241{ 242 int error = 0; 243 |
244 if (SCHEDULER_STOPPED()) 245 return (0); |
|
244 MPASS(curthread != NULL); 245 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 246 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 247 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); 248 error = __sx_slock(sx, opts, file, line); 249 if (!error) { 250 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 251 WITNESS_LOCK(&sx->lock_object, 0, file, line); 252 curthread->td_locks++; 253 } 254 255 return (error); 256} 257 258int 259sx_try_slock_(struct sx *sx, const char *file, int line) 260{ 261 uintptr_t x; 262 | 246 MPASS(curthread != NULL); 247 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 248 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 249 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); 250 error = __sx_slock(sx, opts, file, line); 251 if (!error) { 252 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 253 WITNESS_LOCK(&sx->lock_object, 0, file, line); 254 curthread->td_locks++; 255 } 256 257 return (error); 258} 259 260int 261sx_try_slock_(struct sx *sx, const char *file, int line) 262{ 263 uintptr_t x; 264 |
265 if (SCHEDULER_STOPPED()) 266 return (1); 267 |
|
263 for (;;) { 264 x = sx->sx_lock; 265 KASSERT(x != SX_LOCK_DESTROYED, 266 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 267 if (!(x & SX_LOCK_SHARED)) 268 break; 269 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 270 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); --- 7 unchanged lines hidden (view full) --- 278 return (0); 279} 280 281int 282_sx_xlock(struct sx *sx, int opts, const char *file, int line) 283{ 284 int error = 0; 285 | 268 for (;;) { 269 x = sx->sx_lock; 270 KASSERT(x != SX_LOCK_DESTROYED, 271 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 272 if (!(x & SX_LOCK_SHARED)) 273 break; 274 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 275 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); --- 7 unchanged lines hidden (view full) --- 283 return (0); 284} 285 286int 287_sx_xlock(struct sx *sx, int opts, const char *file, int line) 288{ 289 int error = 0; 290 |
291 if (SCHEDULER_STOPPED()) 292 return (0); |
|
286 MPASS(curthread != NULL); 287 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 288 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 289 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 290 line, NULL); 291 error = __sx_xlock(sx, curthread, opts, file, line); 292 if (!error) { 293 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, --- 5 unchanged lines hidden (view full) --- 299 return (error); 300} 301 302int 303sx_try_xlock_(struct sx *sx, const char *file, int line) 304{ 305 int rval; 306 | 293 MPASS(curthread != NULL); 294 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 295 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 296 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 297 line, NULL); 298 error = __sx_xlock(sx, curthread, opts, file, line); 299 if (!error) { 300 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, --- 5 unchanged lines hidden (view full) --- 306 return (error); 307} 308 309int 310sx_try_xlock_(struct sx *sx, const char *file, int line) 311{ 312 int rval; 313 |
314 if (SCHEDULER_STOPPED()) 315 return (1); 316 |
|
307 MPASS(curthread != NULL); 308 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 309 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 310 311 if (sx_xlocked(sx) && 312 (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) { 313 sx->sx_recurse++; 314 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); --- 10 unchanged lines hidden (view full) --- 325 326 return (rval); 327} 328 329void 330_sx_sunlock(struct sx *sx, const char *file, int line) 331{ 332 | 317 MPASS(curthread != NULL); 318 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 319 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 320 321 if (sx_xlocked(sx) && 322 (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) { 323 sx->sx_recurse++; 324 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); --- 10 unchanged lines hidden (view full) --- 335 336 return (rval); 337} 338 339void 340_sx_sunlock(struct sx *sx, const char *file, int line) 341{ 342 |
343 if (SCHEDULER_STOPPED()) 344 return; |
|
333 MPASS(curthread != NULL); 334 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 335 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 336 _sx_assert(sx, SA_SLOCKED, file, line); 337 curthread->td_locks--; 338 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 339 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 340 __sx_sunlock(sx, file, line); 341 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx); 342} 343 344void 345_sx_xunlock(struct sx *sx, const char *file, int line) 346{ 347 | 345 MPASS(curthread != NULL); 346 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 347 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 348 _sx_assert(sx, SA_SLOCKED, file, line); 349 curthread->td_locks--; 350 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 351 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 352 __sx_sunlock(sx, file, line); 353 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx); 354} 355 356void 357_sx_xunlock(struct sx *sx, const char *file, int line) 358{ 359 |
360 if (SCHEDULER_STOPPED()) 361 return; |
|
348 MPASS(curthread != NULL); 349 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 350 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 351 _sx_assert(sx, SA_XLOCKED, file, line); 352 curthread->td_locks--; 353 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 354 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 355 line); --- 8 unchanged lines hidden (view full) --- 364 * Return 1 if if the upgrade succeed, 0 otherwise. 365 */ 366int 367sx_try_upgrade_(struct sx *sx, const char *file, int line) 368{ 369 uintptr_t x; 370 int success; 371 | 362 MPASS(curthread != NULL); 363 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 364 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 365 _sx_assert(sx, SA_XLOCKED, file, line); 366 curthread->td_locks--; 367 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 368 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 369 line); --- 8 unchanged lines hidden (view full) --- 378 * Return 1 if if the upgrade succeed, 0 otherwise. 379 */ 380int 381sx_try_upgrade_(struct sx *sx, const char *file, int line) 382{ 383 uintptr_t x; 384 int success; 385 |
386 if (SCHEDULER_STOPPED()) 387 return (1); 388 |
|
372 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 373 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 374 _sx_assert(sx, SA_SLOCKED, file, line); 375 376 /* 377 * Try to switch from one shared lock to an exclusive lock. We need 378 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 379 * we will wake up the exclusive waiters when we drop the lock. --- 14 unchanged lines hidden (view full) --- 394 * Downgrade an unrecursed exclusive lock into a single shared lock. 395 */ 396void 397sx_downgrade_(struct sx *sx, const char *file, int line) 398{ 399 uintptr_t x; 400 int wakeup_swapper; 401 | 389 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 390 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 391 _sx_assert(sx, SA_SLOCKED, file, line); 392 393 /* 394 * Try to switch from one shared lock to an exclusive lock. We need 395 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 396 * we will wake up the exclusive waiters when we drop the lock. --- 14 unchanged lines hidden (view full) --- 411 * Downgrade an unrecursed exclusive lock into a single shared lock. 412 */ 413void 414sx_downgrade_(struct sx *sx, const char *file, int line) 415{ 416 uintptr_t x; 417 int wakeup_swapper; 418 |
419 if (SCHEDULER_STOPPED()) 420 return; 421 |
|
402 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 403 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 404 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); 405#ifndef INVARIANTS 406 if (sx_recursed(sx)) 407 panic("downgrade of a recursed lock"); 408#endif 409 --- 66 unchanged lines hidden (view full) --- 476#endif 477 int error = 0; 478#ifdef KDTRACE_HOOKS 479 uint64_t spin_cnt = 0; 480 uint64_t sleep_cnt = 0; 481 int64_t sleep_time = 0; 482#endif 483 | 422 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 423 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 424 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); 425#ifndef INVARIANTS 426 if (sx_recursed(sx)) 427 panic("downgrade of a recursed lock"); 428#endif 429 --- 66 unchanged lines hidden (view full) --- 496#endif 497 int error = 0; 498#ifdef KDTRACE_HOOKS 499 uint64_t spin_cnt = 0; 500 uint64_t sleep_cnt = 0; 501 int64_t sleep_time = 0; 502#endif 503 |
504 if (SCHEDULER_STOPPED()) 505 return (0); 506 |
|
484 /* If we already hold an exclusive lock, then recurse. */ 485 if (sx_xlocked(sx)) { 486 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, 487 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n", 488 sx->lock_object.lo_name, file, line)); 489 sx->sx_recurse++; 490 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 491 if (LOCK_LOG_TEST(&sx->lock_object, 0)) --- 184 unchanged lines hidden (view full) --- 676 * accessible from at least sx.h. 677 */ 678void 679_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 680{ 681 uintptr_t x; 682 int queue, wakeup_swapper; 683 | 507 /* If we already hold an exclusive lock, then recurse. */ 508 if (sx_xlocked(sx)) { 509 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, 510 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n", 511 sx->lock_object.lo_name, file, line)); 512 sx->sx_recurse++; 513 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 514 if (LOCK_LOG_TEST(&sx->lock_object, 0)) --- 184 unchanged lines hidden (view full) --- 699 * accessible from at least sx.h. 700 */ 701void 702_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 703{ 704 uintptr_t x; 705 int queue, wakeup_swapper; 706 |
707 if (SCHEDULER_STOPPED()) 708 return; 709 |
|
684 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 685 686 /* If the lock is recursed, then unrecurse one level. */ 687 if (sx_xlocked(sx) && sx_recursed(sx)) { 688 if ((--sx->sx_recurse) == 0) 689 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 690 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 691 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); --- 56 unchanged lines hidden (view full) --- 748 uintptr_t x; 749 int error = 0; 750#ifdef KDTRACE_HOOKS 751 uint64_t spin_cnt = 0; 752 uint64_t sleep_cnt = 0; 753 int64_t sleep_time = 0; 754#endif 755 | 710 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 711 712 /* If the lock is recursed, then unrecurse one level. */ 713 if (sx_xlocked(sx) && sx_recursed(sx)) { 714 if ((--sx->sx_recurse) == 0) 715 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 716 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 717 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); --- 56 unchanged lines hidden (view full) --- 774 uintptr_t x; 775 int error = 0; 776#ifdef KDTRACE_HOOKS 777 uint64_t spin_cnt = 0; 778 uint64_t sleep_cnt = 0; 779 int64_t sleep_time = 0; 780#endif 781 |
782 if (SCHEDULER_STOPPED()) 783 return (0); 784 |
|
756 /* 757 * As with rwlocks, we don't make any attempt to try to block 758 * shared locks once there is an exclusive waiter. 759 */ 760 for (;;) { 761#ifdef KDTRACE_HOOKS 762 spin_cnt++; 763#endif --- 150 unchanged lines hidden (view full) --- 914 * accessible from at least sx.h. 915 */ 916void 917_sx_sunlock_hard(struct sx *sx, const char *file, int line) 918{ 919 uintptr_t x; 920 int wakeup_swapper; 921 | 785 /* 786 * As with rwlocks, we don't make any attempt to try to block 787 * shared locks once there is an exclusive waiter. 788 */ 789 for (;;) { 790#ifdef KDTRACE_HOOKS 791 spin_cnt++; 792#endif --- 150 unchanged lines hidden (view full) --- 943 * accessible from at least sx.h. 944 */ 945void 946_sx_sunlock_hard(struct sx *sx, const char *file, int line) 947{ 948 uintptr_t x; 949 int wakeup_swapper; 950 |
951 if (SCHEDULER_STOPPED()) 952 return; 953 |
|
922 for (;;) { 923 x = sx->sx_lock; 924 925 /* 926 * We should never have sharers while at least one thread 927 * holds a shared lock. 928 */ 929 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), --- 232 unchanged lines hidden --- | 954 for (;;) { 955 x = sx->sx_lock; 956 957 /* 958 * We should never have sharers while at least one thread 959 * holds a shared lock. 960 */ 961 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), --- 232 unchanged lines hidden --- |