1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29/* 30 * @OSF_COPYRIGHT@ 31 */ 32 33/* 34 * Mach Operating System 35 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 36 * All Rights Reserved. 37 * 38 * Permission to use, copy, modify and distribute this software and its 39 * documentation is hereby granted, provided that both the copyright 40 * notice and this permission notice appear in all copies of the 41 * software, derivative works or modified versions, and any portions 42 * thereof, and that both notices appear in supporting documentation. 43 * 44 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 45 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 46 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 47 * 48 * Carnegie Mellon requests users of this software to return to 49 * 50 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 51 * School of Computer Science 52 * Carnegie Mellon University 53 * Pittsburgh PA 15213-3890 54 * 55 * any improvements or extensions that they make and grant Carnegie Mellon 56 * the rights to redistribute these changes. 57 */ 58 59/* 60 * File: kern/lock.c 61 * Author: Avadis Tevanian, Jr., Michael Wayne Young 62 * Date: 1985 63 * 64 * Locking primitives implementation 65 */ 66 67/* 68 * ARM locks. 69 */ 70 71#include <mach_ldebug.h> 72 73#include <kern/lock.h> 74#include <kern/locks.h> 75#include <kern/kalloc.h> 76#include <kern/misc_protos.h> 77#include <kern/thread.h> 78#include <kern/processor.h> 79#include <kern/cpu_data.h> 80#include <kern/cpu_number.h> 81#include <kern/sched_prim.h> 82#include <kern/xpr.h> 83#include <kern/debug.h> 84#include <string.h> 85 86#include <arm/machine_routines.h> 87#include <machine/machine_cpu.h> 88#include <arm/mp.h> 89 90#include <sys/kdebug.h> 91#include <mach/branch_predicates.h> 92 93#include <arm/misc_protos.h> 94 95/* 96 * This file works, don't mess with it. 97 */ 98 99unsigned int lock_wait_time[2] = { (unsigned int) -1, 0 }; 100 101#define lck_mtx_data lck_mtx_sw.lck_mtxd.lck_mtxd_data 102#define lck_mtx_waiters lck_mtx_sw.lck_mtxd.lck_mtxd_waiters 103#define lck_mtx_pri lck_mtx_sw.lck_mtxd.lck_mtxd_pri 104 105uint32_t LcksOpts; 106 107void lck_rw_ilk_lock(lck_rw_t * lck) 108{ 109 lck_spin_lock(lck); 110} 111 112void lck_rw_ilk_unlock(lck_rw_t * lck) 113{ 114 lck_spin_unlock(lck); 115} 116 117static void lck_mtx_ext_init(lck_mtx_ext_t * lck, lck_grp_t * grp, 118 lck_attr_t * attr) 119{ 120 bzero((void *) lck, sizeof(lck_mtx_ext_t)); 121 122 lck->lck_mtx_grp = grp; 123 124 if (grp->lck_grp_attr & LCK_GRP_ATTR_STAT) 125 lck->lck_mtx_attr |= LCK_MTX_ATTR_STAT; 126} 127 128lck_mtx_t *lck_mtx_alloc_init(lck_grp_t * grp, lck_attr_t * attr) 129{ 130 lck_mtx_t *lck; 131 132 if ((lck = (lck_mtx_t *) kalloc(sizeof(lck_mtx_t))) != 0) 133 lck_mtx_init(lck, grp, attr); 134 135 return (lck); 136} 137 138void lck_mtx_free(lck_mtx_t * lck, lck_grp_t * grp) 139{ 140 lck_mtx_destroy(lck, grp); 141 kfree((void *) lck, sizeof(lck_mtx_t)); 142} 143 144void lck_mtx_init(lck_mtx_t * lck, lck_grp_t * grp, lck_attr_t * attr) 145{ 146 lck_mtx_ext_t *lck_ext; 147 lck_attr_t *lck_attr; 148 149 if (attr != LCK_ATTR_NULL) 150 lck_attr = attr; 151 else 152 lck_attr = &LockDefaultLckAttr; 153 154 lck->lck_mtx_data = 0; 155 lck->lck_mtx_waiters = 0; 156 lck->lck_mtx_state = 0; 157 158 lck_grp_reference(grp); 159 lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX); 160} 161 162void lck_mtx_destroy(lck_mtx_t * lck, lck_grp_t * grp) 163{ 164 boolean_t lck_is_indirect; 165 166 if (lck->lck_mtx_tag == LCK_MTX_TAG_DESTROYED) 167 return; 168 169 lck_is_indirect = (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT); 170 171 lck_mtx_lock_mark_destroyed(lck); 172 173 if (lck_is_indirect) 174 kfree(lck->lck_mtx_ptr, sizeof(lck_mtx_ext_t)); 175 176 lck_grp_lckcnt_decr(grp, LCK_TYPE_MTX); 177 lck_grp_deallocate(grp); 178 return; 179} 180 181void lck_mtx_init_ext(lck_mtx_t * lck, lck_mtx_ext_t * lck_ext, lck_grp_t * grp, 182 lck_attr_t * attr) 183{ 184 lck_attr_t *lck_attr; 185 186 if (attr != LCK_ATTR_NULL) 187 lck_attr = attr; 188 else 189 lck_attr = &LockDefaultLckAttr; 190 191 lck->lck_mtx_data = 0; 192 lck->lck_mtx_waiters = 0; 193 lck->lck_mtx_state = 0; 194 195 lck_grp_reference(grp); 196 lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX); 197 198} 199 200lck_rw_t *lck_rw_alloc_init(lck_grp_t * grp, lck_attr_t * attr) 201{ 202 lck_rw_t *lck; 203 204 if ((lck = (lck_rw_t *) kalloc(sizeof(lck_rw_t))) != 0) { 205 bzero(lck, sizeof(lck_rw_t)); 206 lck_rw_init(lck, grp, attr); 207 } 208 209 return (lck); 210} 211 212void lck_rw_destroy(lck_rw_t * lck, lck_grp_t * grp) 213{ 214 if (lck->lck_rw_tag == LCK_RW_TAG_DESTROYED) 215 return; 216 lck->lck_rw_tag = LCK_RW_TAG_DESTROYED; 217 lck_grp_lckcnt_decr(grp, LCK_TYPE_RW); 218 lck_grp_deallocate(grp); 219 return; 220} 221 222void lck_rw_free(lck_rw_t * lck, lck_grp_t * grp) 223{ 224 lck_rw_destroy(lck, grp); 225 kfree(lck, sizeof(lck_rw_t)); 226} 227 228void lck_rw_init(lck_rw_t * lck, lck_grp_t * grp, lck_attr_t * attr) 229{ 230 lck_attr_t *lck_attr = (attr != LCK_ATTR_NULL) ? attr : &LockDefaultLckAttr; 231 232 lck->lck_rw_interlock = 0; 233 lck->lck_rw_want_excl = FALSE; 234 lck->lck_rw_want_upgrade = FALSE; 235 lck->lck_rw_shared_count = 0; 236 lck->lck_rw_waiting = 0; 237 lck->lck_rw_tag = 0; 238 lck->lck_rw_priv_excl = 239 ((lck_attr->lck_attr_val & LCK_ATTR_RW_SHARED_PRIORITY) == 0); 240 241 lck_grp_reference(grp); 242 lck_grp_lckcnt_incr(grp, LCK_TYPE_RW); 243} 244 245void lck_spin_destroy(lck_spin_t * lck, lck_grp_t * grp) 246{ 247 if (lck->interlock == LCK_SPIN_TAG_DESTROYED) 248 return; 249 lck->interlock = LCK_SPIN_TAG_DESTROYED; 250 lck_grp_lckcnt_decr(grp, LCK_TYPE_SPIN); 251 lck_grp_deallocate(grp); 252} 253 254void lck_spin_init(lck_spin_t * lck, lck_grp_t * grp, 255 __unused lck_attr_t * attr) 256{ 257 lck->interlock = 0; 258 lck_grp_reference(grp); 259 lck_grp_lckcnt_incr(grp, LCK_TYPE_SPIN); 260} 261 262void lock_init(lock_t * l, boolean_t can_sleep, __unused unsigned short tag, 263 __unused unsigned short tag1) 264{ 265 l->lck_rw_interlock = 0; 266 l->lck_rw_want_excl = FALSE; 267 l->lck_rw_want_upgrade = FALSE; 268 l->lck_rw_shared_count = 0; 269 l->lck_rw_tag = tag; 270 l->lck_rw_priv_excl = 1; 271 l->lck_rw_waiting = 0; 272} 273 274void lck_rw_assert(lck_rw_t * lck, unsigned int type) 275{ 276 switch (type) { 277 case LCK_RW_ASSERT_SHARED: 278 if (lck->lck_rw_shared_count != 0) { 279 return; 280 } 281 break; 282 case LCK_RW_ASSERT_EXCLUSIVE: 283 if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) 284 && lck->lck_rw_shared_count == 0) { 285 return; 286 } 287 break; 288 case LCK_RW_ASSERT_HELD: 289 if (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade 290 || lck->lck_rw_shared_count != 0) { 291 return; 292 } 293 break; 294 default: 295 break; 296 } 297 298 panic("rw lock (%p) not held (mode=%u), first word %08x\n", lck, type, 299 *(uint32_t *) lck); 300} 301 302void lck_rw_lock(lck_rw_t * lck, lck_rw_type_t lck_rw_type) 303{ 304 if (lck_rw_type == LCK_RW_TYPE_SHARED) 305 lck_rw_lock_shared(lck); 306 else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) 307 lck_rw_lock_exclusive(lck); 308 else 309 panic("lck_rw_lock(): Invalid RW lock type: %d\n", lck_rw_type); 310 return; 311} 312 313void lck_spin_free(lck_spin_t * lck, lck_grp_t * grp) 314{ 315 lck_spin_destroy(lck, grp); 316 kfree((void *) lck, sizeof(lck_spin_t)); 317} 318 319boolean_t lck_rw_try_lock(lck_rw_t * lck, lck_rw_type_t lck_rw_type) 320{ 321 if (lck_rw_type == LCK_RW_TYPE_SHARED) 322 return lck_rw_try_lock_shared(lck); 323 else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) 324 return lck_rw_try_lock_exclusive(lck); 325 else 326 panic("lck_rw_try_lock(): Invalid RW lock type: %d\n", lck_rw_type); 327 return FALSE; 328} 329 330lck_spin_t *lck_spin_alloc_init(lck_grp_t * grp, lck_attr_t * attr) 331{ 332 lck_spin_t *lck; 333 334 if ((lck = (lck_spin_t *) kalloc(sizeof(lck_spin_t))) != 0) 335 lck_spin_init(lck, grp, attr); 336 337 return (lck); 338} 339 340void usimple_lock(usimple_lock_t l) 341{ 342 lck_spin_lock((lck_spin_t *) l); 343} 344 345void usimple_unlock(usimple_lock_t l) 346{ 347 lck_spin_unlock((lck_spin_t *) l); 348} 349 350unsigned int usimple_lock_try(usimple_lock_t l) 351{ 352 return (lck_spin_try_lock((lck_spin_t *) l)); 353} 354 355extern void arm_usimple_lock_init(usimple_lock_t, unsigned short); 356 357void usimple_lock_init(usimple_lock_t l, unsigned short tag) 358{ 359 arm_usimple_lock_init(l, tag); 360} 361 362void lck_rw_unlock_shared(lck_rw_t * lck) 363{ 364 lck_rw_type_t ret; 365 366 ret = lck_rw_done(lck); 367 368 if (ret != LCK_RW_TYPE_SHARED) 369 panic("lck_rw_unlock(): lock held in mode: %d\n", ret); 370} 371 372void lck_rw_unlock_exclusive(lck_rw_t * lck) 373{ 374 lck_rw_type_t ret; 375 376 ret = lck_rw_done(lck); 377 378 if (ret != LCK_RW_TYPE_EXCLUSIVE) 379 panic("lck_rw_unlock_exclusive(): lock held in mode: %d\n", ret); 380} 381 382void lck_rw_lock_shared_gen(lck_rw_t * lck) 383{ 384 int i; 385 wait_result_t res; 386 387 lck_rw_ilk_lock(lck); 388 389 while ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) 390 && ((lck->lck_rw_shared_count == 0) || (lck->lck_rw_priv_excl))) { 391 i = lock_wait_time[1]; 392 393 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | 394 DBG_FUNC_START, (int) lck, lck->lck_rw_want_excl, 395 lck->lck_rw_want_upgrade, i, 0); 396 397 if (i != 0) { 398 lck_rw_ilk_unlock(lck); 399 while (--i != 0 400 && (lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) 401 && ((lck->lck_rw_shared_count == 0) 402 || (lck->lck_rw_priv_excl))) 403 continue; 404 lck_rw_ilk_lock(lck); 405 } 406 407 if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) 408 && ((lck->lck_rw_shared_count == 0) || (lck->lck_rw_priv_excl))) { 409 lck->lck_rw_waiting = TRUE; 410 res = 411 assert_wait((event_t) 412 (((unsigned int *) lck) + 413 ((sizeof(lck_rw_t) - 1) / sizeof(unsigned int))), 414 THREAD_UNINT); 415 if (res == THREAD_WAITING) { 416 lck_rw_ilk_unlock(lck); 417 res = thread_block(THREAD_CONTINUE_NULL); 418 lck_rw_ilk_lock(lck); 419 } 420 } 421 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | 422 DBG_FUNC_END, (int) lck, lck->lck_rw_want_excl, 423 lck->lck_rw_want_upgrade, res, 0); 424 } 425 426 lck->lck_rw_shared_count++; 427 428 lck_rw_ilk_unlock(lck); 429} 430 431void lck_rw_lock_exclusive_gen(lck_rw_t * lck) 432{ 433 int i; 434 wait_result_t res; 435 436 lck_rw_ilk_lock(lck); 437 /* 438 * Try to acquire the lck_rw_want_excl bit. 439 */ 440 441 while (lck->lck_rw_want_excl) { 442 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | 443 DBG_FUNC_START, (int) lck, 0, 0, 0, 0); 444 445 i = lock_wait_time[1]; 446 if (i != 0) { 447 lck_rw_ilk_unlock(lck); 448 while (--i != 0 && lck->lck_rw_want_excl) 449 continue; 450 lck_rw_ilk_lock(lck); 451 } 452 453 if (lck->lck_rw_want_excl) { 454 lck->lck_rw_waiting = TRUE; 455 res = 456 assert_wait((event_t) 457 (((unsigned int *) lck) + 458 ((sizeof(lck_rw_t) - 1) / sizeof(unsigned int))), 459 THREAD_UNINT); 460 if (res == THREAD_WAITING) { 461 lck_rw_ilk_unlock(lck); 462 res = thread_block(THREAD_CONTINUE_NULL); 463 lck_rw_ilk_lock(lck); 464 } 465 } 466 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | 467 DBG_FUNC_END, (int) lck, res, 0, 0, 0); 468 } 469 lck->lck_rw_want_excl = TRUE; 470 471 /* 472 * Wait for readers (and upgrades) to finish 473 */ 474 475 while ((lck->lck_rw_shared_count != 0) || lck->lck_rw_want_upgrade) { 476 477 i = lock_wait_time[1]; 478 479 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | 480 DBG_FUNC_START, (int) lck, lck->lck_rw_shared_count, 481 lck->lck_rw_want_upgrade, i, 0); 482 483 if (i != 0) { 484 lck_rw_ilk_unlock(lck); 485 while (--i != 0 486 && (lck->lck_rw_shared_count != 0 487 || lck->lck_rw_want_upgrade)) 488 continue; 489 lck_rw_ilk_lock(lck); 490 } 491 492 if (lck->lck_rw_shared_count != 0 || lck->lck_rw_want_upgrade) { 493 lck->lck_rw_waiting = TRUE; 494 res = 495 assert_wait((event_t) 496 (((unsigned int *) lck) + 497 ((sizeof(lck_rw_t) - 1) / sizeof(unsigned int))), 498 THREAD_UNINT); 499 500 if (res == THREAD_WAITING) { 501 lck_rw_ilk_unlock(lck); 502 res = thread_block(THREAD_CONTINUE_NULL); 503 lck_rw_ilk_lock(lck); 504 } 505 } 506 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | 507 DBG_FUNC_END, (int) lck, lck->lck_rw_shared_count, 508 lck->lck_rw_want_upgrade, res, 0); 509 } 510 511 lck_rw_ilk_unlock(lck); 512} 513 514void lck_mtx_lock_mark_destroyed(lck_mtx_t * mutex) 515{ 516 return; 517} 518 519boolean_t lck_rw_lock_shared_to_exclusive_gen(lck_rw_t * lck) 520{ 521 int i; 522 boolean_t do_wakeup = FALSE; 523 wait_result_t res; 524 525 lck_rw_ilk_lock(lck); 526 527 lck->lck_rw_shared_count--; 528 529 if (lck->lck_rw_want_upgrade) { 530 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | 531 DBG_FUNC_START, (int) lck, lck->lck_rw_shared_count, 532 lck->lck_rw_want_upgrade, 0, 0); 533 534 /* 535 * Someone else has requested upgrade. 536 * Since we've released a read lock, wake 537 * him up. 538 */ 539 if (lck->lck_rw_waiting && (lck->lck_rw_shared_count == 0)) { 540 lck->lck_rw_waiting = FALSE; 541 do_wakeup = TRUE; 542 } 543 544 lck_rw_ilk_unlock(lck); 545 546 if (do_wakeup) 547 thread_wakeup((event_t) 548 (((unsigned int *) lck) + 549 ((sizeof(lck_rw_t) - 1) / sizeof(unsigned int)))); 550 551 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | 552 DBG_FUNC_END, (int) lck, lck->lck_rw_shared_count, 553 lck->lck_rw_want_upgrade, 0, 0); 554 555 return (FALSE); 556 } 557 558 lck->lck_rw_want_upgrade = TRUE; 559 560 while (lck->lck_rw_shared_count != 0) { 561 i = lock_wait_time[1]; 562 563 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | 564 DBG_FUNC_START, (int) lck, lck->lck_rw_shared_count, i, 0, 565 0); 566 if (i != 0) { 567 lck_rw_ilk_unlock(lck); 568 while (--i != 0 && lck->lck_rw_shared_count != 0) 569 continue; 570 lck_rw_ilk_lock(lck); 571 } 572 573 if (lck->lck_rw_shared_count != 0) { 574 lck->lck_rw_waiting = TRUE; 575 res = 576 assert_wait((event_t) 577 (((unsigned int *) lck) + 578 ((sizeof(lck_rw_t) - 1) / sizeof(unsigned int))), 579 THREAD_UNINT); 580 if (res == THREAD_WAITING) { 581 lck_rw_ilk_unlock(lck); 582 res = thread_block(THREAD_CONTINUE_NULL); 583 lck_rw_ilk_lock(lck); 584 } 585 } 586 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | 587 DBG_FUNC_END, (int) lck, lck->lck_rw_shared_count, 0, 0, 588 0); 589 } 590 591 lck_rw_ilk_unlock(lck); 592 593 return (TRUE); 594} 595