| kern_rwlock.c (164246) | kern_rwlock.c (167012) |
|---|---|
| 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 18 unchanged lines hidden (view full) --- 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Machine independent bits of reader/writer lock implementation. 32 */ 33 34#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 18 unchanged lines hidden (view full) --- 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Machine independent bits of reader/writer lock implementation. 32 */ 33 34#include <sys/cdefs.h> |
| 35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 164246 2006-11-13 05:41:46Z kmacy $"); | 35__FBSDID("$FreeBSD: head/sys/kern/kern_rwlock.c 167012 2007-02-26 08:26:44Z kmacy $"); |
| 36 37#include "opt_ddb.h" 38 39#include <sys/param.h> 40#include <sys/ktr.h> 41#include <sys/lock.h> 42#include <sys/mutex.h> 43#include <sys/proc.h> --- 62 unchanged lines hidden (view full) --- 106 struct rw_args *args = arg; 107 108 rw_init(args->ra_rw, args->ra_desc); 109} 110 111void 112_rw_wlock(struct rwlock *rw, const char *file, int line) 113{ | 36 37#include "opt_ddb.h" 38 39#include <sys/param.h> 40#include <sys/ktr.h> 41#include <sys/lock.h> 42#include <sys/mutex.h> 43#include <sys/proc.h> --- 62 unchanged lines hidden (view full) --- 106 struct rw_args *args = arg; 107 108 rw_init(args->ra_rw, args->ra_desc); 109} 110 111void 112_rw_wlock(struct rwlock *rw, const char *file, int line) 113{ |
| 114 uint64_t waitstart; | |
| 115 116 MPASS(curthread != NULL); 117 KASSERT(rw_wowner(rw) != curthread, 118 ("%s (%s): wlock already held @ %s:%d", __func__, 119 rw->rw_object.lo_name, file, line)); 120 WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 121 line); | 114 115 MPASS(curthread != NULL); 116 KASSERT(rw_wowner(rw) != curthread, 117 ("%s (%s): wlock already held @ %s:%d", __func__, 118 rw->rw_object.lo_name, file, line)); 119 WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 120 line); |
| 122 lock_profile_waitstart(&waitstart); | |
| 123 __rw_wlock(rw, curthread, file, line); | 121 __rw_wlock(rw, curthread, file, line); |
| 124 lock_profile_obtain_lock_success(&rw->rw_object, waitstart, file, line); | |
| 125 LOCK_LOG_LOCK("WLOCK", &rw->rw_object, 0, 0, file, line); 126 WITNESS_LOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); 127 curthread->td_locks++; 128} 129 130void 131_rw_wunlock(struct rwlock *rw, const char *file, int line) 132{ --- 28 unchanged lines hidden (view full) --- 161 * that we currently allow for read locks to recurse and we 162 * don't keep track of all the holders of read locks. Thus, if 163 * we were to block readers once a writer blocked and a reader 164 * tried to recurse on their reader lock after a writer had 165 * blocked we would end up in a deadlock since the reader would 166 * be blocked on the writer, and the writer would be blocked 167 * waiting for the reader to release its original read lock. 168 */ | 122 LOCK_LOG_LOCK("WLOCK", &rw->rw_object, 0, 0, file, line); 123 WITNESS_LOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); 124 curthread->td_locks++; 125} 126 127void 128_rw_wunlock(struct rwlock *rw, const char *file, int line) 129{ --- 28 unchanged lines hidden (view full) --- 158 * that we currently allow for read locks to recurse and we 159 * don't keep track of all the holders of read locks. Thus, if 160 * we were to block readers once a writer blocked and a reader 161 * tried to recurse on their reader lock after a writer had 162 * blocked we would end up in a deadlock since the reader would 163 * be blocked on the writer, and the writer would be blocked 164 * waiting for the reader to release its original read lock. 165 */ |
| 169 lock_profile_waitstart(&waitstart); | |
| 170 for (;;) { 171 /* 172 * Handle the easy case. If no other thread has a write 173 * lock, then try to bump up the count of read locks. Note 174 * that we have to preserve the current state of the 175 * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 176 * read lock, then rw_lock must have changed, so restart 177 * the loop. Note that this handles the case of a --- 6 unchanged lines hidden (view full) --- 184 /* 185 * The RW_LOCK_READ_WAITERS flag should only be set 186 * if another thread currently holds a write lock, 187 * and in that case RW_LOCK_READ should be clear. 188 */ 189 MPASS((x & RW_LOCK_READ_WAITERS) == 0); 190 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, 191 x + RW_ONE_READER)) { | 166 for (;;) { 167 /* 168 * Handle the easy case. If no other thread has a write 169 * lock, then try to bump up the count of read locks. Note 170 * that we have to preserve the current state of the 171 * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a 172 * read lock, then rw_lock must have changed, so restart 173 * the loop. Note that this handles the case of a --- 6 unchanged lines hidden (view full) --- 180 /* 181 * The RW_LOCK_READ_WAITERS flag should only be set 182 * if another thread currently holds a write lock, 183 * and in that case RW_LOCK_READ should be clear. 184 */ 185 MPASS((x & RW_LOCK_READ_WAITERS) == 0); 186 if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, 187 x + RW_ONE_READER)) { |
| 192 lock_profile_obtain_lock_success(&rw->rw_object, waitstart, file, line); | 188 lock_profile_obtain_lock_success(&rw->rw_object, contested, waitstart, file, line); |
| 193 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 194 CTR4(KTR_LOCK, 195 "%s: %p succeed %p -> %p", __func__, 196 rw, (void *)x, 197 (void *)(x + RW_ONE_READER)); 198 break; 199 } | 189 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 190 CTR4(KTR_LOCK, 191 "%s: %p succeed %p -> %p", __func__, 192 rw, (void *)x, 193 (void *)(x + RW_ONE_READER)); 194 break; 195 } |
| 196 lock_profile_obtain_lock_failed(&rw->rw_object, &contested, &waitstart); |
|
| 200 cpu_spinwait(); | 197 cpu_spinwait(); |
| 201 lock_profile_obtain_lock_failed(&rw->rw_object, &contested); | |
| 202 continue; 203 } 204 205 /* 206 * Okay, now it's the hard case. Some other thread already 207 * has a write lock, so acquire the turnstile lock so we can 208 * begin the process of blocking. 209 */ --- 32 unchanged lines hidden (view full) --- 242#ifdef SMP 243 /* 244 * If the owner is running on another CPU, spin until 245 * the owner stops running or the state of the lock 246 * changes. 247 */ 248 owner = (struct thread *)RW_OWNER(x); 249 if (TD_IS_RUNNING(owner)) { | 198 continue; 199 } 200 201 /* 202 * Okay, now it's the hard case. Some other thread already 203 * has a write lock, so acquire the turnstile lock so we can 204 * begin the process of blocking. 205 */ --- 32 unchanged lines hidden (view full) --- 238#ifdef SMP 239 /* 240 * If the owner is running on another CPU, spin until 241 * the owner stops running or the state of the lock 242 * changes. 243 */ 244 owner = (struct thread *)RW_OWNER(x); 245 if (TD_IS_RUNNING(owner)) { |
| 250 lock_profile_obtain_lock_failed(&rw->rw_object, &contested); | 246 lock_profile_obtain_lock_failed(&rw->rw_object, &contested, &waitstart); |
| 251 turnstile_release(&rw->rw_object); 252 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 253 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 254 __func__, rw, owner); 255 while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && 256 TD_IS_RUNNING(owner)) 257 cpu_spinwait(); 258 continue; --- 147 unchanged lines hidden (view full) --- 406 * read or write lock. 407 */ 408void 409_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 410{ 411#ifdef SMP 412 volatile struct thread *owner; 413#endif | 247 turnstile_release(&rw->rw_object); 248 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 249 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 250 __func__, rw, owner); 251 while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && 252 TD_IS_RUNNING(owner)) 253 cpu_spinwait(); 254 continue; --- 147 unchanged lines hidden (view full) --- 402 * read or write lock. 403 */ 404void 405_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) 406{ 407#ifdef SMP 408 volatile struct thread *owner; 409#endif |
| 414 int contested; | |
| 415 uintptr_t v; 416 417 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 418 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 419 rw->rw_object.lo_name, (void *)rw->rw_lock, file, line); 420 421 while (!_rw_write_lock(rw, tid)) { 422 turnstile_lock(&rw->rw_object); --- 25 unchanged lines hidden (view full) --- 448 tid | RW_LOCK_WRITE_WAITERS)) { 449 turnstile_claim(&rw->rw_object); 450 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 451 __func__, rw); 452 break; 453 } 454 turnstile_release(&rw->rw_object); 455 cpu_spinwait(); | 410 uintptr_t v; 411 412 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 413 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 414 rw->rw_object.lo_name, (void *)rw->rw_lock, file, line); 415 416 while (!_rw_write_lock(rw, tid)) { 417 turnstile_lock(&rw->rw_object); --- 25 unchanged lines hidden (view full) --- 443 tid | RW_LOCK_WRITE_WAITERS)) { 444 turnstile_claim(&rw->rw_object); 445 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 446 __func__, rw); 447 break; 448 } 449 turnstile_release(&rw->rw_object); 450 cpu_spinwait(); |
| 456 lock_profile_obtain_lock_failed(&rw->rw_object, &contested); | |
| 457 continue; 458 } 459 460 /* 461 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 462 * set it. If we fail to set it, then loop back and try 463 * again. 464 */ 465 if (!(v & RW_LOCK_WRITE_WAITERS)) { 466 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 467 v | RW_LOCK_WRITE_WAITERS)) { 468 turnstile_release(&rw->rw_object); 469 cpu_spinwait(); | 451 continue; 452 } 453 454 /* 455 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to 456 * set it. If we fail to set it, then loop back and try 457 * again. 458 */ 459 if (!(v & RW_LOCK_WRITE_WAITERS)) { 460 if (!atomic_cmpset_ptr(&rw->rw_lock, v, 461 v | RW_LOCK_WRITE_WAITERS)) { 462 turnstile_release(&rw->rw_object); 463 cpu_spinwait(); |
| 470 lock_profile_obtain_lock_failed(&rw->rw_object, &contested); | |
| 471 continue; 472 } 473 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 474 CTR2(KTR_LOCK, "%s: %p set write waiters flag", 475 __func__, rw); 476 } 477 478#ifdef SMP 479 /* 480 * If the lock is write locked and the owner is 481 * running on another CPU, spin until the owner stops 482 * running or the state of the lock changes. 483 */ 484 owner = (struct thread *)RW_OWNER(v); 485 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { | 464 continue; 465 } 466 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 467 CTR2(KTR_LOCK, "%s: %p set write waiters flag", 468 __func__, rw); 469 } 470 471#ifdef SMP 472 /* 473 * If the lock is write locked and the owner is 474 * running on another CPU, spin until the owner stops 475 * running or the state of the lock changes. 476 */ 477 owner = (struct thread *)RW_OWNER(v); 478 if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) { |
| 486 lock_profile_obtain_lock_failed(&rw->rw_object, &contested); | |
| 487 turnstile_release(&rw->rw_object); 488 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 489 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 490 __func__, rw, owner); 491 while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && 492 TD_IS_RUNNING(owner)) 493 cpu_spinwait(); 494 continue; --- 344 unchanged lines hidden --- | 479 turnstile_release(&rw->rw_object); 480 if (LOCK_LOG_TEST(&rw->rw_object, 0)) 481 CTR3(KTR_LOCK, "%s: spinning on %p held by %p", 482 __func__, rw, owner); 483 while ((struct thread*)RW_OWNER(rw->rw_lock)== owner && 484 TD_IS_RUNNING(owner)) 485 cpu_spinwait(); 486 continue; --- 344 unchanged lines hidden --- |