kern_lock.c revision 149574
1/*- 2 * Copyright (c) 1995 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Copyright (C) 1997 6 * John S. Dyson. All rights reserved. 7 * 8 * This code contains ideas from software contributed to Berkeley by 9 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 10 * System project at Carnegie-Mellon University. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 41 */ 42 43#include <sys/cdefs.h> 44__FBSDID("$FreeBSD: head/sys/kern/kern_lock.c 149574 2005-08-29 11:34:08Z pjd $"); 45 46#include <sys/param.h> 47#include <sys/kernel.h> 48#include <sys/ktr.h> 49#include <sys/lock.h> 50#include <sys/lockmgr.h> 51#include <sys/mutex.h> 52#include <sys/proc.h> 53#include <sys/systm.h> 54#ifdef DEBUG_LOCKS 55#include <sys/stack.h> 56#include <sys/sysctl.h> 57#endif 58 59/* 60 * Locking primitives implementation. 61 * Locks provide shared/exclusive sychronization. 62 */ 63 64#define COUNT(td, x) if ((td)) (td)->td_locks += (x) 65#define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \ 66 LK_SHARE_NONZERO | LK_WAIT_NONZERO) 67 68static int acquire(struct lock **lkpp, int extflags, int wanted); 69static int acquiredrain(struct lock *lkp, int extflags) ; 70 71static __inline void 72sharelock(struct thread *td, struct lock *lkp, int incr) { 73 lkp->lk_flags |= LK_SHARE_NONZERO; 74 lkp->lk_sharecount += incr; 75 COUNT(td, incr); 76} 77 78static __inline void 79shareunlock(struct thread *td, struct lock *lkp, int decr) { 80 81 KASSERT(lkp->lk_sharecount >= decr, ("shareunlock: count < decr")); 82 83 COUNT(td, -decr); 84 if (lkp->lk_sharecount == decr) { 85 lkp->lk_flags &= ~LK_SHARE_NONZERO; 86 if (lkp->lk_flags & (LK_WANT_UPGRADE | LK_WANT_EXCL)) { 87 wakeup(lkp); 88 } 89 lkp->lk_sharecount = 0; 90 } else { 91 lkp->lk_sharecount -= decr; 92 } 93} 94 95static int 96acquire(struct lock **lkpp, int extflags, int wanted) 97{ 98 struct lock *lkp = *lkpp; 99 int error; 100 CTR3(KTR_LOCK, 101 "acquire(): lkp == %p, extflags == 0x%x, wanted == 0x%x", 102 lkp, extflags, wanted); 103 104 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted)) 105 return EBUSY; 106 error = 0; 107 while ((lkp->lk_flags & wanted) != 0) { 108 CTR2(KTR_LOCK, 109 "acquire(): lkp == %p, lk_flags == 0x%x sleeping", 110 lkp, lkp->lk_flags); 111 lkp->lk_flags |= LK_WAIT_NONZERO; 112 lkp->lk_waitcount++; 113 error = msleep(lkp, lkp->lk_interlock, lkp->lk_prio, 114 lkp->lk_wmesg, 115 ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0)); 116 lkp->lk_waitcount--; 117 if (lkp->lk_waitcount == 0) 118 lkp->lk_flags &= ~LK_WAIT_NONZERO; 119 if (error) 120 break; 121 if (extflags & LK_SLEEPFAIL) { 122 error = ENOLCK; 123 break; 124 } 125 if (lkp->lk_newlock != NULL) { 126 mtx_lock(lkp->lk_newlock->lk_interlock); 127 mtx_unlock(lkp->lk_interlock); 128 if (lkp->lk_waitcount == 0) 129 wakeup((void *)(&lkp->lk_newlock)); 130 *lkpp = lkp = lkp->lk_newlock; 131 } 132 } 133 mtx_assert(lkp->lk_interlock, MA_OWNED); 134 return (error); 135} 136 137/* 138 * Set, change, or release a lock. 139 * 140 * Shared requests increment the shared count. Exclusive requests set the 141 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 142 * accepted shared locks and shared-to-exclusive upgrades to go away. 143 */ 144int 145lockmgr(lkp, flags, interlkp, td) 146 struct lock *lkp; 147 u_int flags; 148 struct mtx *interlkp; 149 struct thread *td; 150{ 151 int error; 152 struct thread *thr; 153 int extflags, lockflags; 154 155 error = 0; 156 if (td == NULL) 157 thr = LK_KERNPROC; 158 else 159 thr = td; 160 161 if ((flags & LK_INTERNAL) == 0) 162 mtx_lock(lkp->lk_interlock); 163 CTR6(KTR_LOCK, 164 "lockmgr(): lkp == %p (lk_wmesg == \"%s\"), owner == %p, exclusivecount == %d, flags == 0x%x, " 165 "td == %p", lkp, lkp->lk_wmesg, lkp->lk_lockholder, 166 lkp->lk_exclusivecount, flags, td); 167#ifdef DEBUG_LOCKS 168 { 169 struct stack stack; /* XXX */ 170 stack_save(&stack); 171 CTRSTACK(KTR_LOCK, &stack, 0, 1); 172 } 173#endif 174 175 if (flags & LK_INTERLOCK) { 176 mtx_assert(interlkp, MA_OWNED | MA_NOTRECURSED); 177 mtx_unlock(interlkp); 178 } 179 180 if ((flags & (LK_NOWAIT|LK_RELEASE)) == 0) 181 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, 182 &lkp->lk_interlock->mtx_object, 183 "Acquiring lockmgr lock \"%s\"", lkp->lk_wmesg); 184 185 if (panicstr != NULL) { 186 mtx_unlock(lkp->lk_interlock); 187 return (0); 188 } 189 if ((lkp->lk_flags & LK_NOSHARE) && 190 (flags & LK_TYPE_MASK) == LK_SHARED) { 191 flags &= ~LK_TYPE_MASK; 192 flags |= LK_EXCLUSIVE; 193 } 194 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 195 196 switch (flags & LK_TYPE_MASK) { 197 198 case LK_SHARED: 199 /* 200 * If we are not the exclusive lock holder, we have to block 201 * while there is an exclusive lock holder or while an 202 * exclusive lock request or upgrade request is in progress. 203 * 204 * However, if TDP_DEADLKTREAT is set, we override exclusive 205 * lock requests or upgrade requests ( but not the exclusive 206 * lock itself ). 207 */ 208 if (lkp->lk_lockholder != thr) { 209 lockflags = LK_HAVE_EXCL; 210 if (td != NULL && !(td->td_pflags & TDP_DEADLKTREAT)) 211 lockflags |= LK_WANT_EXCL | LK_WANT_UPGRADE; 212 error = acquire(&lkp, extflags, lockflags); 213 if (error) 214 break; 215 sharelock(td, lkp, 1); 216#if defined(DEBUG_LOCKS) 217 stack_save(&lkp->lk_stack); 218#endif 219 break; 220 } 221 /* 222 * We hold an exclusive lock, so downgrade it to shared. 223 * An alternative would be to fail with EDEADLK. 224 */ 225 sharelock(td, lkp, 1); 226 /* FALLTHROUGH downgrade */ 227 228 case LK_DOWNGRADE: 229 KASSERT(lkp->lk_lockholder == thr && lkp->lk_exclusivecount != 0, 230 ("lockmgr: not holding exclusive lock " 231 "(owner thread (%p) != thread (%p), exlcnt (%d) != 0", 232 lkp->lk_lockholder, thr, lkp->lk_exclusivecount)); 233 sharelock(td, lkp, lkp->lk_exclusivecount); 234 COUNT(td, -lkp->lk_exclusivecount); 235 lkp->lk_exclusivecount = 0; 236 lkp->lk_flags &= ~LK_HAVE_EXCL; 237 lkp->lk_lockholder = LK_NOPROC; 238 if (lkp->lk_waitcount) 239 wakeup((void *)lkp); 240 break; 241 242 case LK_EXCLUPGRADE: 243 /* 244 * If another process is ahead of us to get an upgrade, 245 * then we want to fail rather than have an intervening 246 * exclusive access. 247 */ 248 if (lkp->lk_flags & LK_WANT_UPGRADE) { 249 shareunlock(td, lkp, 1); 250 error = EBUSY; 251 break; 252 } 253 /* FALLTHROUGH normal upgrade */ 254 255 case LK_UPGRADE: 256 /* 257 * Upgrade a shared lock to an exclusive one. If another 258 * shared lock has already requested an upgrade to an 259 * exclusive lock, our shared lock is released and an 260 * exclusive lock is requested (which will be granted 261 * after the upgrade). If we return an error, the file 262 * will always be unlocked. 263 */ 264 if (lkp->lk_lockholder == thr) 265 panic("lockmgr: upgrade exclusive lock"); 266 if (lkp->lk_sharecount <= 0) 267 panic("lockmgr: upgrade without shared"); 268 shareunlock(td, lkp, 1); 269 /* 270 * If we are just polling, check to see if we will block. 271 */ 272 if ((extflags & LK_NOWAIT) && 273 ((lkp->lk_flags & LK_WANT_UPGRADE) || 274 lkp->lk_sharecount > 1)) { 275 error = EBUSY; 276 break; 277 } 278 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 279 /* 280 * We are first shared lock to request an upgrade, so 281 * request upgrade and wait for the shared count to 282 * drop to zero, then take exclusive lock. 283 */ 284 lkp->lk_flags |= LK_WANT_UPGRADE; 285 error = acquire(&lkp, extflags, LK_SHARE_NONZERO); 286 lkp->lk_flags &= ~LK_WANT_UPGRADE; 287 288 if (error) { 289 if ((lkp->lk_flags & ( LK_WANT_EXCL | LK_WAIT_NONZERO)) == (LK_WANT_EXCL | LK_WAIT_NONZERO)) 290 wakeup((void *)lkp); 291 break; 292 } 293 if (lkp->lk_exclusivecount != 0) 294 panic("lockmgr: non-zero exclusive count"); 295 lkp->lk_flags |= LK_HAVE_EXCL; 296 lkp->lk_lockholder = thr; 297 lkp->lk_exclusivecount = 1; 298 COUNT(td, 1); 299#if defined(DEBUG_LOCKS) 300 stack_save(&lkp->lk_stack); 301#endif 302 break; 303 } 304 /* 305 * Someone else has requested upgrade. Release our shared 306 * lock, awaken upgrade requestor if we are the last shared 307 * lock, then request an exclusive lock. 308 */ 309 if ( (lkp->lk_flags & (LK_SHARE_NONZERO|LK_WAIT_NONZERO)) == 310 LK_WAIT_NONZERO) 311 wakeup((void *)lkp); 312 /* FALLTHROUGH exclusive request */ 313 314 case LK_EXCLUSIVE: 315 if (lkp->lk_lockholder == thr && thr != LK_KERNPROC) { 316 /* 317 * Recursive lock. 318 */ 319 if ((extflags & (LK_NOWAIT | LK_CANRECURSE)) == 0) 320 panic("lockmgr: locking against myself"); 321 if ((extflags & LK_CANRECURSE) != 0) { 322 lkp->lk_exclusivecount++; 323 COUNT(td, 1); 324 break; 325 } 326 } 327 /* 328 * If we are just polling, check to see if we will sleep. 329 */ 330 if ((extflags & LK_NOWAIT) && 331 (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) { 332 error = EBUSY; 333 break; 334 } 335 /* 336 * Try to acquire the want_exclusive flag. 337 */ 338 error = acquire(&lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL)); 339 if (error) 340 break; 341 lkp->lk_flags |= LK_WANT_EXCL; 342 /* 343 * Wait for shared locks and upgrades to finish. 344 */ 345 error = acquire(&lkp, extflags, LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO); 346 lkp->lk_flags &= ~LK_WANT_EXCL; 347 if (error) { 348 if (lkp->lk_flags & LK_WAIT_NONZERO) 349 wakeup((void *)lkp); 350 break; 351 } 352 lkp->lk_flags |= LK_HAVE_EXCL; 353 lkp->lk_lockholder = thr; 354 if (lkp->lk_exclusivecount != 0) 355 panic("lockmgr: non-zero exclusive count"); 356 lkp->lk_exclusivecount = 1; 357 COUNT(td, 1); 358#if defined(DEBUG_LOCKS) 359 stack_save(&lkp->lk_stack); 360#endif 361 break; 362 363 case LK_RELEASE: 364 if (lkp->lk_exclusivecount != 0) { 365 if (lkp->lk_lockholder != thr && 366 lkp->lk_lockholder != LK_KERNPROC) { 367 panic("lockmgr: thread %p, not %s %p unlocking", 368 thr, "exclusive lock holder", 369 lkp->lk_lockholder); 370 } 371 if (lkp->lk_lockholder != LK_KERNPROC) 372 COUNT(td, -1); 373 if (lkp->lk_exclusivecount == 1) { 374 lkp->lk_flags &= ~LK_HAVE_EXCL; 375 lkp->lk_lockholder = LK_NOPROC; 376 lkp->lk_exclusivecount = 0; 377 } else { 378 lkp->lk_exclusivecount--; 379 } 380 } else if (lkp->lk_flags & LK_SHARE_NONZERO) 381 shareunlock(td, lkp, 1); 382 if (lkp->lk_flags & LK_WAIT_NONZERO) 383 wakeup((void *)lkp); 384 break; 385 386 case LK_DRAIN: 387 /* 388 * Check that we do not already hold the lock, as it can 389 * never drain if we do. Unfortunately, we have no way to 390 * check for holding a shared lock, but at least we can 391 * check for an exclusive one. 392 */ 393 if (lkp->lk_lockholder == thr) 394 panic("lockmgr: draining against myself"); 395 396 error = acquiredrain(lkp, extflags); 397 if (error) 398 break; 399 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 400 lkp->lk_lockholder = thr; 401 lkp->lk_exclusivecount = 1; 402 COUNT(td, 1); 403#if defined(DEBUG_LOCKS) 404 stack_save(&lkp->lk_stack); 405#endif 406 break; 407 408 default: 409 mtx_unlock(lkp->lk_interlock); 410 panic("lockmgr: unknown locktype request %d", 411 flags & LK_TYPE_MASK); 412 /* NOTREACHED */ 413 } 414 if ((lkp->lk_flags & LK_WAITDRAIN) && 415 (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 416 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0) { 417 lkp->lk_flags &= ~LK_WAITDRAIN; 418 wakeup((void *)&lkp->lk_flags); 419 } 420 mtx_unlock(lkp->lk_interlock); 421 return (error); 422} 423 424static int 425acquiredrain(struct lock *lkp, int extflags) { 426 int error; 427 428 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & LK_ALL)) { 429 return EBUSY; 430 } 431 while (lkp->lk_flags & LK_ALL) { 432 lkp->lk_flags |= LK_WAITDRAIN; 433 error = msleep(&lkp->lk_flags, lkp->lk_interlock, lkp->lk_prio, 434 lkp->lk_wmesg, 435 ((extflags & LK_TIMELOCK) ? lkp->lk_timo : 0)); 436 if (error) 437 return error; 438 if (extflags & LK_SLEEPFAIL) { 439 return ENOLCK; 440 } 441 } 442 return 0; 443} 444 445/* 446 * Transfer any waiting processes from one lock to another. 447 */ 448void 449transferlockers(from, to) 450 struct lock *from; 451 struct lock *to; 452{ 453 454 KASSERT(from != to, ("lock transfer to self")); 455 KASSERT((from->lk_flags&LK_WAITDRAIN) == 0, ("transfer draining lock")); 456 457 mtx_lock(from->lk_interlock); 458 if (from->lk_waitcount == 0) { 459 mtx_unlock(from->lk_interlock); 460 return; 461 } 462 from->lk_newlock = to; 463 wakeup((void *)from); 464 msleep(&from->lk_newlock, from->lk_interlock, from->lk_prio, 465 "lkxfer", 0); 466 from->lk_newlock = NULL; 467 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 468 KASSERT(from->lk_waitcount == 0, ("active lock")); 469 mtx_unlock(from->lk_interlock); 470} 471 472 473/* 474 * Initialize a lock; required before use. 475 */ 476void 477lockinit(lkp, prio, wmesg, timo, flags) 478 struct lock *lkp; 479 int prio; 480 const char *wmesg; 481 int timo; 482 int flags; 483{ 484 CTR5(KTR_LOCK, "lockinit(): lkp == %p, prio == %d, wmesg == \"%s\", " 485 "timo == %d, flags = 0x%x\n", lkp, prio, wmesg, timo, flags); 486 487 lkp->lk_interlock = mtx_pool_alloc(mtxpool_lockbuilder); 488 lkp->lk_flags = (flags & LK_EXTFLG_MASK); 489 lkp->lk_sharecount = 0; 490 lkp->lk_waitcount = 0; 491 lkp->lk_exclusivecount = 0; 492 lkp->lk_prio = prio; 493 lkp->lk_wmesg = wmesg; 494 lkp->lk_timo = timo; 495 lkp->lk_lockholder = LK_NOPROC; 496 lkp->lk_newlock = NULL; 497#ifdef DEBUG_LOCKS 498 stack_zero(&lkp->lk_stack); 499#endif 500} 501 502/* 503 * Destroy a lock. 504 */ 505void 506lockdestroy(lkp) 507 struct lock *lkp; 508{ 509 CTR2(KTR_LOCK, "lockdestroy(): lkp == %p (lk_wmesg == \"%s\")", 510 lkp, lkp->lk_wmesg); 511} 512 513/* 514 * Determine the status of a lock. 515 */ 516int 517lockstatus(lkp, td) 518 struct lock *lkp; 519 struct thread *td; 520{ 521 int lock_type = 0; 522 523 mtx_lock(lkp->lk_interlock); 524 if (lkp->lk_exclusivecount != 0) { 525 if (td == NULL || lkp->lk_lockholder == td) 526 lock_type = LK_EXCLUSIVE; 527 else 528 lock_type = LK_EXCLOTHER; 529 } else if (lkp->lk_sharecount != 0) 530 lock_type = LK_SHARED; 531 mtx_unlock(lkp->lk_interlock); 532 return (lock_type); 533} 534 535/* 536 * Determine the number of holders of a lock. 537 */ 538int 539lockcount(lkp) 540 struct lock *lkp; 541{ 542 int count; 543 544 mtx_lock(lkp->lk_interlock); 545 count = lkp->lk_exclusivecount + lkp->lk_sharecount; 546 mtx_unlock(lkp->lk_interlock); 547 return (count); 548} 549 550/* 551 * Print out information about state of a lock. Used by VOP_PRINT 552 * routines to display status about contained locks. 553 */ 554void 555lockmgr_printinfo(lkp) 556 struct lock *lkp; 557{ 558 559 if (lkp->lk_sharecount) 560 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 561 lkp->lk_sharecount); 562 else if (lkp->lk_flags & LK_HAVE_EXCL) 563 printf(" lock type %s: EXCL (count %d) by thread %p (pid %d)", 564 lkp->lk_wmesg, lkp->lk_exclusivecount, 565 lkp->lk_lockholder, lkp->lk_lockholder->td_proc->p_pid); 566 if (lkp->lk_waitcount > 0) 567 printf(" with %d pending", lkp->lk_waitcount); 568#ifdef DEBUG_LOCKS 569 stack_print(&lkp->lk_stack); 570#endif 571} 572