1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56#include <mach_ldebug.h> 57#include <debug.h> 58 59#include <mach/kern_return.h> 60#include <mach/mach_host_server.h> 61#include <mach_debug/lockgroup_info.h> 62 63#include <kern/locks.h> 64#include <kern/misc_protos.h> 65#include <kern/kalloc.h> 66#include <kern/thread.h> 67#include <kern/processor.h> 68#include <kern/sched_prim.h> 69#include <kern/debug.h> 70#include <string.h> 71 72 73#include <sys/kdebug.h> 74 75#if CONFIG_DTRACE 76/* 77 * We need only enough declarations from the BSD-side to be able to 78 * test if our probe is active, and to call __dtrace_probe(). Setting 79 * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in. 80 */ 81#define NEED_DTRACE_DEFS 82#include <../bsd/sys/lockstat.h> 83#endif 84 85#define LCK_MTX_SLEEP_CODE 0 86#define LCK_MTX_SLEEP_DEADLINE_CODE 1 87#define LCK_MTX_LCK_WAIT_CODE 2 88#define LCK_MTX_UNLCK_WAKEUP_CODE 3 89 90 91static queue_head_t lck_grp_queue; 92static unsigned int lck_grp_cnt; 93 94decl_lck_mtx_data(static,lck_grp_lock) 95static lck_mtx_ext_t lck_grp_lock_ext; 96 97lck_grp_attr_t LockDefaultGroupAttr; 98lck_grp_t LockCompatGroup; 99lck_attr_t LockDefaultLckAttr; 100 101/* 102 * Routine: lck_mod_init 103 */ 104 105void 106lck_mod_init( 107 void) 108{ 109 /* 110 * Obtain "lcks" options:this currently controls lock statistics 111 */ 112 if (!PE_parse_boot_argn("lcks", &LcksOpts, sizeof (LcksOpts))) 113 LcksOpts = 0; 114 115 queue_init(&lck_grp_queue); 116 117 /* 118 * Need to bootstrap the LockCompatGroup instead of calling lck_grp_init() here. This avoids 119 * grabbing the lck_grp_lock before it is initialized. 120 */ 121 122 bzero(&LockCompatGroup, sizeof(lck_grp_t)); 123 (void) strncpy(LockCompatGroup.lck_grp_name, "Compatibility APIs", LCK_GRP_MAX_NAME); 124 125 if (LcksOpts & enaLkStat) 126 LockCompatGroup.lck_grp_attr = LCK_GRP_ATTR_STAT; 127 else 128 LockCompatGroup.lck_grp_attr = LCK_ATTR_NONE; 129 130 LockCompatGroup.lck_grp_refcnt = 1; 131 132 enqueue_tail(&lck_grp_queue, (queue_entry_t)&LockCompatGroup); 133 lck_grp_cnt = 1; 134 135 lck_grp_attr_setdefault(&LockDefaultGroupAttr); 136 lck_attr_setdefault(&LockDefaultLckAttr); 137 138 lck_mtx_init_ext(&lck_grp_lock, &lck_grp_lock_ext, &LockCompatGroup, &LockDefaultLckAttr); 139 140} 141 142/* 143 * Routine: lck_grp_attr_alloc_init 144 */ 145 146lck_grp_attr_t * 147lck_grp_attr_alloc_init( 148 void) 149{ 150 lck_grp_attr_t *attr; 151 152 if ((attr = (lck_grp_attr_t *)kalloc(sizeof(lck_grp_attr_t))) != 0) 153 lck_grp_attr_setdefault(attr); 154 155 return(attr); 156} 157 158 159/* 160 * Routine: lck_grp_attr_setdefault 161 */ 162 163void 164lck_grp_attr_setdefault( 165 lck_grp_attr_t *attr) 166{ 167 if (LcksOpts & enaLkStat) 168 attr->grp_attr_val = LCK_GRP_ATTR_STAT; 169 else 170 attr->grp_attr_val = 0; 171} 172 173 174/* 175 * Routine: lck_grp_attr_setstat 176 */ 177 178void 179lck_grp_attr_setstat( 180 lck_grp_attr_t *attr) 181{ 182 (void)hw_atomic_or(&attr->grp_attr_val, LCK_GRP_ATTR_STAT); 183} 184 185 186/* 187 * Routine: lck_grp_attr_free 188 */ 189 190void 191lck_grp_attr_free( 192 lck_grp_attr_t *attr) 193{ 194 kfree(attr, sizeof(lck_grp_attr_t)); 195} 196 197 198/* 199 * Routine: lck_grp_alloc_init 200 */ 201 202lck_grp_t * 203lck_grp_alloc_init( 204 const char* grp_name, 205 lck_grp_attr_t *attr) 206{ 207 lck_grp_t *grp; 208 209 if ((grp = (lck_grp_t *)kalloc(sizeof(lck_grp_t))) != 0) 210 lck_grp_init(grp, grp_name, attr); 211 212 return(grp); 213} 214 215 216/* 217 * Routine: lck_grp_init 218 */ 219 220void 221lck_grp_init( 222 lck_grp_t *grp, 223 const char* grp_name, 224 lck_grp_attr_t *attr) 225{ 226 bzero((void *)grp, sizeof(lck_grp_t)); 227 228 (void) strncpy(grp->lck_grp_name, grp_name, LCK_GRP_MAX_NAME); 229 230 if (attr != LCK_GRP_ATTR_NULL) 231 grp->lck_grp_attr = attr->grp_attr_val; 232 else if (LcksOpts & enaLkStat) 233 grp->lck_grp_attr = LCK_GRP_ATTR_STAT; 234 else 235 grp->lck_grp_attr = LCK_ATTR_NONE; 236 237 grp->lck_grp_refcnt = 1; 238 239 lck_mtx_lock(&lck_grp_lock); 240 enqueue_tail(&lck_grp_queue, (queue_entry_t)grp); 241 lck_grp_cnt++; 242 lck_mtx_unlock(&lck_grp_lock); 243 244} 245 246 247/* 248 * Routine: lck_grp_free 249 */ 250 251void 252lck_grp_free( 253 lck_grp_t *grp) 254{ 255 lck_mtx_lock(&lck_grp_lock); 256 lck_grp_cnt--; 257 (void)remque((queue_entry_t)grp); 258 lck_mtx_unlock(&lck_grp_lock); 259 lck_grp_deallocate(grp); 260} 261 262 263/* 264 * Routine: lck_grp_reference 265 */ 266 267void 268lck_grp_reference( 269 lck_grp_t *grp) 270{ 271 (void)hw_atomic_add(&grp->lck_grp_refcnt, 1); 272} 273 274 275/* 276 * Routine: lck_grp_deallocate 277 */ 278 279void 280lck_grp_deallocate( 281 lck_grp_t *grp) 282{ 283 if (hw_atomic_sub(&grp->lck_grp_refcnt, 1) == 0) 284 kfree(grp, sizeof(lck_grp_t)); 285} 286 287/* 288 * Routine: lck_grp_lckcnt_incr 289 */ 290 291void 292lck_grp_lckcnt_incr( 293 lck_grp_t *grp, 294 lck_type_t lck_type) 295{ 296 unsigned int *lckcnt; 297 298 switch (lck_type) { 299 case LCK_TYPE_SPIN: 300 lckcnt = &grp->lck_grp_spincnt; 301 break; 302 case LCK_TYPE_MTX: 303 lckcnt = &grp->lck_grp_mtxcnt; 304 break; 305 case LCK_TYPE_RW: 306 lckcnt = &grp->lck_grp_rwcnt; 307 break; 308 default: 309 return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type); 310 } 311 312 (void)hw_atomic_add(lckcnt, 1); 313} 314 315/* 316 * Routine: lck_grp_lckcnt_decr 317 */ 318 319void 320lck_grp_lckcnt_decr( 321 lck_grp_t *grp, 322 lck_type_t lck_type) 323{ 324 unsigned int *lckcnt; 325 326 switch (lck_type) { 327 case LCK_TYPE_SPIN: 328 lckcnt = &grp->lck_grp_spincnt; 329 break; 330 case LCK_TYPE_MTX: 331 lckcnt = &grp->lck_grp_mtxcnt; 332 break; 333 case LCK_TYPE_RW: 334 lckcnt = &grp->lck_grp_rwcnt; 335 break; 336 default: 337 return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type); 338 } 339 340 (void)hw_atomic_sub(lckcnt, 1); 341} 342 343/* 344 * Routine: lck_attr_alloc_init 345 */ 346 347lck_attr_t * 348lck_attr_alloc_init( 349 void) 350{ 351 lck_attr_t *attr; 352 353 if ((attr = (lck_attr_t *)kalloc(sizeof(lck_attr_t))) != 0) 354 lck_attr_setdefault(attr); 355 356 return(attr); 357} 358 359 360/* 361 * Routine: lck_attr_setdefault 362 */ 363 364void 365lck_attr_setdefault( 366 lck_attr_t *attr) 367{ 368#if __i386__ || __x86_64__ 369#if !DEBUG 370 if (LcksOpts & enaLkDeb) 371 attr->lck_attr_val = LCK_ATTR_DEBUG; 372 else 373 attr->lck_attr_val = LCK_ATTR_NONE; 374#else 375 attr->lck_attr_val = LCK_ATTR_DEBUG; 376#endif /* !DEBUG */ 377#elif defined(__arm__) 378 attr->lck_attr_val = LCK_ATTR_DEBUG; 379#else 380#error Unknown architecture. 381#endif /* __arm__ */ 382} 383 384 385/* 386 * Routine: lck_attr_setdebug 387 */ 388void 389lck_attr_setdebug( 390 lck_attr_t *attr) 391{ 392 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG); 393} 394 395/* 396 * Routine: lck_attr_setdebug 397 */ 398void 399lck_attr_cleardebug( 400 lck_attr_t *attr) 401{ 402 (void)hw_atomic_and(&attr->lck_attr_val, ~LCK_ATTR_DEBUG); 403} 404 405 406/* 407 * Routine: lck_attr_rw_shared_priority 408 */ 409void 410lck_attr_rw_shared_priority( 411 lck_attr_t *attr) 412{ 413 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY); 414} 415 416 417/* 418 * Routine: lck_attr_free 419 */ 420void 421lck_attr_free( 422 lck_attr_t *attr) 423{ 424 kfree(attr, sizeof(lck_attr_t)); 425} 426 427 428/* 429 * Routine: lck_spin_sleep 430 */ 431wait_result_t 432lck_spin_sleep( 433 lck_spin_t *lck, 434 lck_sleep_action_t lck_sleep_action, 435 event_t event, 436 wait_interrupt_t interruptible) 437{ 438 wait_result_t res; 439 440 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 441 panic("Invalid lock sleep action %x\n", lck_sleep_action); 442 443 res = assert_wait(event, interruptible); 444 if (res == THREAD_WAITING) { 445 lck_spin_unlock(lck); 446 res = thread_block(THREAD_CONTINUE_NULL); 447 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) 448 lck_spin_lock(lck); 449 } 450 else 451 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 452 lck_spin_unlock(lck); 453 454 return res; 455} 456 457 458/* 459 * Routine: lck_spin_sleep_deadline 460 */ 461wait_result_t 462lck_spin_sleep_deadline( 463 lck_spin_t *lck, 464 lck_sleep_action_t lck_sleep_action, 465 event_t event, 466 wait_interrupt_t interruptible, 467 uint64_t deadline) 468{ 469 wait_result_t res; 470 471 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 472 panic("Invalid lock sleep action %x\n", lck_sleep_action); 473 474 res = assert_wait_deadline(event, interruptible, deadline); 475 if (res == THREAD_WAITING) { 476 lck_spin_unlock(lck); 477 res = thread_block(THREAD_CONTINUE_NULL); 478 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) 479 lck_spin_lock(lck); 480 } 481 else 482 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 483 lck_spin_unlock(lck); 484 485 return res; 486} 487 488 489/* 490 * Routine: lck_mtx_sleep 491 */ 492wait_result_t 493lck_mtx_sleep( 494 lck_mtx_t *lck, 495 lck_sleep_action_t lck_sleep_action, 496 event_t event, 497 wait_interrupt_t interruptible) 498{ 499 wait_result_t res; 500 501 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START, 502 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0); 503 504 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 505 panic("Invalid lock sleep action %x\n", lck_sleep_action); 506 507 res = assert_wait(event, interruptible); 508 if (res == THREAD_WAITING) { 509 lck_mtx_unlock(lck); 510 res = thread_block(THREAD_CONTINUE_NULL); 511 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { 512 if ((lck_sleep_action & LCK_SLEEP_SPIN)) 513 lck_mtx_lock_spin(lck); 514 else 515 lck_mtx_lock(lck); 516 } 517 } 518 else 519 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 520 lck_mtx_unlock(lck); 521 522 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); 523 524 return res; 525} 526 527 528/* 529 * Routine: lck_mtx_sleep_deadline 530 */ 531wait_result_t 532lck_mtx_sleep_deadline( 533 lck_mtx_t *lck, 534 lck_sleep_action_t lck_sleep_action, 535 event_t event, 536 wait_interrupt_t interruptible, 537 uint64_t deadline) 538{ 539 wait_result_t res; 540 541 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START, 542 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0); 543 544 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 545 panic("Invalid lock sleep action %x\n", lck_sleep_action); 546 547 res = assert_wait_deadline(event, interruptible, deadline); 548 if (res == THREAD_WAITING) { 549 lck_mtx_unlock(lck); 550 res = thread_block(THREAD_CONTINUE_NULL); 551 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { 552 if ((lck_sleep_action & LCK_SLEEP_SPIN)) 553 lck_mtx_lock_spin(lck); 554 else 555 lck_mtx_lock(lck); 556 } 557 } 558 else 559 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 560 lck_mtx_unlock(lck); 561 562 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); 563 564 return res; 565} 566 567/* 568 * Routine: lck_mtx_lock_wait 569 * 570 * Invoked in order to wait on contention. 571 * 572 * Called with the interlock locked and 573 * returns it unlocked. 574 */ 575void 576lck_mtx_lock_wait ( 577 lck_mtx_t *lck, 578 thread_t holder) 579{ 580 thread_t self = current_thread(); 581 lck_mtx_t *mutex; 582 integer_t priority; 583 spl_t s = splsched(); 584#if CONFIG_DTRACE 585 uint64_t sleep_start = 0; 586 587 if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) { 588 sleep_start = mach_absolute_time(); 589 } 590#endif 591 592 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) 593 mutex = lck; 594 else 595 mutex = &lck->lck_mtx_ptr->lck_mtx; 596 597 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0); 598 599 priority = self->sched_pri; 600 if (priority < self->priority) 601 priority = self->priority; 602 if (priority < BASEPRI_DEFAULT) 603 priority = BASEPRI_DEFAULT; 604 605 thread_lock(holder); 606 if (mutex->lck_mtx_pri == 0) 607 holder->promotions++; 608 holder->sched_flags |= TH_SFLAG_PROMOTED; 609 if ( mutex->lck_mtx_pri < priority && 610 holder->sched_pri < priority ) { 611 KERNEL_DEBUG_CONSTANT( 612 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, 613 holder->sched_pri, priority, holder, lck, 0); 614 615 set_sched_pri(holder, priority); 616 } 617 thread_unlock(holder); 618 splx(s); 619 620 if (mutex->lck_mtx_pri < priority) 621 mutex->lck_mtx_pri = priority; 622 if (self->pending_promoter[self->pending_promoter_index] == NULL) { 623 self->pending_promoter[self->pending_promoter_index] = mutex; 624 mutex->lck_mtx_waiters++; 625 } 626 else 627 if (self->pending_promoter[self->pending_promoter_index] != mutex) { 628 self->pending_promoter[++self->pending_promoter_index] = mutex; 629 mutex->lck_mtx_waiters++; 630 } 631 632 assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))), THREAD_UNINT); 633 lck_mtx_ilk_unlock(mutex); 634 635 thread_block(THREAD_CONTINUE_NULL); 636 637 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); 638#if CONFIG_DTRACE 639 /* 640 * Record the Dtrace lockstat probe for blocking, block time 641 * measured from when we were entered. 642 */ 643 if (sleep_start) { 644 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) { 645 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck, 646 mach_absolute_time() - sleep_start); 647 } else { 648 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck, 649 mach_absolute_time() - sleep_start); 650 } 651 } 652#endif 653} 654 655/* 656 * Routine: lck_mtx_lock_acquire 657 * 658 * Invoked on acquiring the mutex when there is 659 * contention. 660 * 661 * Returns the current number of waiters. 662 * 663 * Called with the interlock locked. 664 */ 665int 666lck_mtx_lock_acquire( 667 lck_mtx_t *lck) 668{ 669 thread_t thread = current_thread(); 670 lck_mtx_t *mutex; 671 672 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) 673 mutex = lck; 674 else 675 mutex = &lck->lck_mtx_ptr->lck_mtx; 676 677 if (thread->pending_promoter[thread->pending_promoter_index] == mutex) { 678 thread->pending_promoter[thread->pending_promoter_index] = NULL; 679 if (thread->pending_promoter_index > 0) 680 thread->pending_promoter_index--; 681 mutex->lck_mtx_waiters--; 682 } 683 684 if (mutex->lck_mtx_waiters > 0) { 685 integer_t priority = mutex->lck_mtx_pri; 686 spl_t s = splsched(); 687 688 thread_lock(thread); 689 thread->promotions++; 690 thread->sched_flags |= TH_SFLAG_PROMOTED; 691 if (thread->sched_pri < priority) { 692 KERNEL_DEBUG_CONSTANT( 693 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, 694 thread->sched_pri, priority, 0, lck, 0); 695 696 set_sched_pri(thread, priority); 697 } 698 thread_unlock(thread); 699 splx(s); 700 } 701 else 702 mutex->lck_mtx_pri = 0; 703 704 return (mutex->lck_mtx_waiters); 705} 706 707/* 708 * Routine: lck_mtx_unlock_wakeup 709 * 710 * Invoked on unlock when there is contention. 711 * 712 * Called with the interlock locked. 713 */ 714void 715lck_mtx_unlock_wakeup ( 716 lck_mtx_t *lck, 717 thread_t holder) 718{ 719 thread_t thread = current_thread(); 720 lck_mtx_t *mutex; 721 722 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) 723 mutex = lck; 724 else 725 mutex = &lck->lck_mtx_ptr->lck_mtx; 726 727 if (thread != holder) 728 panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex, holder); 729 730 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0); 731 732 assert(mutex->lck_mtx_waiters > 0); 733 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int))); 734 735 if (thread->promotions > 0) { 736 spl_t s = splsched(); 737 738 thread_lock(thread); 739 if ( --thread->promotions == 0 && 740 (thread->sched_flags & TH_SFLAG_PROMOTED) ) { 741 thread->sched_flags &= ~TH_SFLAG_PROMOTED; 742 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) { 743 KERNEL_DEBUG_CONSTANT( 744 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | DBG_FUNC_NONE, 745 thread->sched_pri, DEPRESSPRI, 0, lck, 0); 746 747 set_sched_pri(thread, DEPRESSPRI); 748 } 749 else { 750 if (thread->priority < thread->sched_pri) { 751 KERNEL_DEBUG_CONSTANT( 752 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | 753 DBG_FUNC_NONE, 754 thread->sched_pri, thread->priority, 755 0, lck, 0); 756 } 757 758 SCHED(compute_priority)(thread, FALSE); 759 } 760 } 761 thread_unlock(thread); 762 splx(s); 763 } 764 765 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); 766} 767 768void 769lck_mtx_unlockspin_wakeup ( 770 lck_mtx_t *lck) 771{ 772 assert(lck->lck_mtx_waiters > 0); 773 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int))); 774 775 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_NONE, (int)lck, 0, 0, 1, 0); 776#if CONFIG_DTRACE 777 /* 778 * When there are waiters, we skip the hot-patch spot in the 779 * fastpath, so we record it here. 780 */ 781 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lck, 0); 782#endif 783} 784 785 786/* 787 * Routine: mutex_pause 788 * 789 * Called by former callers of simple_lock_pause(). 790 */ 791#define MAX_COLLISION_COUNTS 32 792#define MAX_COLLISION 8 793 794unsigned int max_collision_count[MAX_COLLISION_COUNTS]; 795 796uint32_t collision_backoffs[MAX_COLLISION] = { 797 10, 50, 100, 200, 400, 600, 800, 1000 798}; 799 800 801void 802mutex_pause(uint32_t collisions) 803{ 804 wait_result_t wait_result; 805 uint32_t back_off; 806 807 if (collisions >= MAX_COLLISION_COUNTS) 808 collisions = MAX_COLLISION_COUNTS - 1; 809 max_collision_count[collisions]++; 810 811 if (collisions >= MAX_COLLISION) 812 collisions = MAX_COLLISION - 1; 813 back_off = collision_backoffs[collisions]; 814 815 wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, back_off, NSEC_PER_USEC); 816 assert(wait_result == THREAD_WAITING); 817 818 wait_result = thread_block(THREAD_CONTINUE_NULL); 819 assert(wait_result == THREAD_TIMED_OUT); 820} 821 822 823unsigned int mutex_yield_wait = 0; 824unsigned int mutex_yield_no_wait = 0; 825 826void 827lck_mtx_yield( 828 lck_mtx_t *lck) 829{ 830 int waiters; 831 832#if DEBUG 833 lck_mtx_assert(lck, LCK_MTX_ASSERT_OWNED); 834#endif /* DEBUG */ 835 836 if (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT) 837 waiters = lck->lck_mtx_ptr->lck_mtx.lck_mtx_waiters; 838 else 839 waiters = lck->lck_mtx_waiters; 840 841 if ( !waiters) { 842 mutex_yield_no_wait++; 843 } else { 844 mutex_yield_wait++; 845 lck_mtx_unlock(lck); 846 mutex_pause(0); 847 lck_mtx_lock(lck); 848 } 849} 850 851 852/* 853 * Routine: lck_rw_sleep 854 */ 855wait_result_t 856lck_rw_sleep( 857 lck_rw_t *lck, 858 lck_sleep_action_t lck_sleep_action, 859 event_t event, 860 wait_interrupt_t interruptible) 861{ 862 wait_result_t res; 863 lck_rw_type_t lck_rw_type; 864 865 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 866 panic("Invalid lock sleep action %x\n", lck_sleep_action); 867 868 res = assert_wait(event, interruptible); 869 if (res == THREAD_WAITING) { 870 lck_rw_type = lck_rw_done(lck); 871 res = thread_block(THREAD_CONTINUE_NULL); 872 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { 873 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE))) 874 lck_rw_lock(lck, lck_rw_type); 875 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) 876 lck_rw_lock_exclusive(lck); 877 else 878 lck_rw_lock_shared(lck); 879 } 880 } 881 else 882 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 883 (void)lck_rw_done(lck); 884 885 return res; 886} 887 888 889/* 890 * Routine: lck_rw_sleep_deadline 891 */ 892wait_result_t 893lck_rw_sleep_deadline( 894 lck_rw_t *lck, 895 lck_sleep_action_t lck_sleep_action, 896 event_t event, 897 wait_interrupt_t interruptible, 898 uint64_t deadline) 899{ 900 wait_result_t res; 901 lck_rw_type_t lck_rw_type; 902 903 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 904 panic("Invalid lock sleep action %x\n", lck_sleep_action); 905 906 res = assert_wait_deadline(event, interruptible, deadline); 907 if (res == THREAD_WAITING) { 908 lck_rw_type = lck_rw_done(lck); 909 res = thread_block(THREAD_CONTINUE_NULL); 910 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { 911 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE))) 912 lck_rw_lock(lck, lck_rw_type); 913 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) 914 lck_rw_lock_exclusive(lck); 915 else 916 lck_rw_lock_shared(lck); 917 } 918 } 919 else 920 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 921 (void)lck_rw_done(lck); 922 923 return res; 924} 925 926kern_return_t 927host_lockgroup_info( 928 host_t host, 929 lockgroup_info_array_t *lockgroup_infop, 930 mach_msg_type_number_t *lockgroup_infoCntp) 931{ 932 lockgroup_info_t *lockgroup_info_base; 933 lockgroup_info_t *lockgroup_info; 934 vm_offset_t lockgroup_info_addr; 935 vm_size_t lockgroup_info_size; 936 lck_grp_t *lck_grp; 937 unsigned int i; 938 vm_size_t used; 939 vm_map_copy_t copy; 940 kern_return_t kr; 941 942 if (host == HOST_NULL) 943 return KERN_INVALID_HOST; 944 945 lck_mtx_lock(&lck_grp_lock); 946 947 lockgroup_info_size = round_page(lck_grp_cnt * sizeof *lockgroup_info); 948 kr = kmem_alloc_pageable(ipc_kernel_map, 949 &lockgroup_info_addr, lockgroup_info_size); 950 if (kr != KERN_SUCCESS) { 951 lck_mtx_unlock(&lck_grp_lock); 952 return(kr); 953 } 954 955 lockgroup_info_base = (lockgroup_info_t *) lockgroup_info_addr; 956 lck_grp = (lck_grp_t *)queue_first(&lck_grp_queue); 957 lockgroup_info = lockgroup_info_base; 958 959 for (i = 0; i < lck_grp_cnt; i++) { 960 961 lockgroup_info->lock_spin_cnt = lck_grp->lck_grp_spincnt; 962 lockgroup_info->lock_spin_util_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_util_cnt; 963 lockgroup_info->lock_spin_held_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cnt; 964 lockgroup_info->lock_spin_miss_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_miss_cnt; 965 lockgroup_info->lock_spin_held_max = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_max; 966 lockgroup_info->lock_spin_held_cum = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cum; 967 968 lockgroup_info->lock_mtx_cnt = lck_grp->lck_grp_mtxcnt; 969 lockgroup_info->lock_mtx_util_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt; 970 lockgroup_info->lock_mtx_held_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt; 971 lockgroup_info->lock_mtx_miss_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt; 972 lockgroup_info->lock_mtx_wait_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt; 973 lockgroup_info->lock_mtx_held_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_max; 974 lockgroup_info->lock_mtx_held_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cum; 975 lockgroup_info->lock_mtx_wait_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_max; 976 lockgroup_info->lock_mtx_wait_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cum; 977 978 lockgroup_info->lock_rw_cnt = lck_grp->lck_grp_rwcnt; 979 lockgroup_info->lock_rw_util_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt; 980 lockgroup_info->lock_rw_held_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cnt; 981 lockgroup_info->lock_rw_miss_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt; 982 lockgroup_info->lock_rw_wait_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cnt; 983 lockgroup_info->lock_rw_held_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_max; 984 lockgroup_info->lock_rw_held_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cum; 985 lockgroup_info->lock_rw_wait_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_max; 986 lockgroup_info->lock_rw_wait_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cum; 987 988 (void) strncpy(lockgroup_info->lockgroup_name,lck_grp->lck_grp_name, LOCKGROUP_MAX_NAME); 989 990 lck_grp = (lck_grp_t *)(queue_next((queue_entry_t)(lck_grp))); 991 lockgroup_info++; 992 } 993 994 *lockgroup_infoCntp = lck_grp_cnt; 995 lck_mtx_unlock(&lck_grp_lock); 996 997 used = (*lockgroup_infoCntp) * sizeof *lockgroup_info; 998 999 if (used != lockgroup_info_size) 1000 bzero((char *) lockgroup_info, lockgroup_info_size - used); 1001 1002 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)lockgroup_info_addr, 1003 (vm_map_size_t)lockgroup_info_size, TRUE, ©); 1004 assert(kr == KERN_SUCCESS); 1005 1006 *lockgroup_infop = (lockgroup_info_t *) copy; 1007 1008 return(KERN_SUCCESS); 1009} 1010 1011/* 1012 * Compatibility module 1013 */ 1014 1015extern lck_rw_t *lock_alloc_EXT( boolean_t can_sleep, unsigned short tag0, unsigned short tag1); 1016extern void lock_done_EXT(lck_rw_t *lock); 1017extern void lock_free_EXT(lck_rw_t *lock); 1018extern void lock_init_EXT(lck_rw_t *lock, boolean_t can_sleep, unsigned short tag0, unsigned short tag1); 1019extern void lock_read_EXT(lck_rw_t *lock); 1020extern boolean_t lock_read_to_write_EXT(lck_rw_t *lock); 1021extern void lock_write_EXT(lck_rw_t *lock); 1022extern void lock_write_to_read_EXT(lck_rw_t *lock); 1023extern wait_result_t thread_sleep_lock_write_EXT( 1024 event_t event, lck_rw_t *lock, wait_interrupt_t interruptible); 1025 1026extern void usimple_lock_EXT(lck_spin_t *lock); 1027extern void usimple_lock_init_EXT(lck_spin_t *lock, unsigned short tag); 1028extern unsigned int usimple_lock_try_EXT(lck_spin_t *lock); 1029extern void usimple_unlock_EXT(lck_spin_t *lock); 1030extern wait_result_t thread_sleep_usimple_lock_EXT(event_t event, lck_spin_t *lock, wait_interrupt_t interruptible); 1031 1032 1033lck_mtx_t* mutex_alloc_EXT(__unused unsigned short tag); 1034void mutex_free_EXT(lck_mtx_t *mutex); 1035void mutex_init_EXT(lck_mtx_t *mutex, __unused unsigned short tag); 1036wait_result_t thread_sleep_mutex_EXT(event_t event, lck_mtx_t *mutex, wait_interrupt_t interruptible); 1037wait_result_t thread_sleep_mutex_deadline_EXT(event_t event, lck_mtx_t *mutex, uint64_t deadline, wait_interrupt_t interruptible); 1038 1039lck_rw_t * 1040lock_alloc_EXT( 1041 __unused boolean_t can_sleep, 1042 __unused unsigned short tag0, 1043 __unused unsigned short tag1) 1044{ 1045 return( lck_rw_alloc_init( &LockCompatGroup, LCK_ATTR_NULL)); 1046} 1047 1048void 1049lock_done_EXT( 1050 lck_rw_t *lock) 1051{ 1052 (void) lck_rw_done(lock); 1053} 1054 1055void 1056lock_free_EXT( 1057 lck_rw_t *lock) 1058{ 1059 lck_rw_free(lock, &LockCompatGroup); 1060} 1061 1062void 1063lock_init_EXT( 1064 lck_rw_t *lock, 1065 __unused boolean_t can_sleep, 1066 __unused unsigned short tag0, 1067 __unused unsigned short tag1) 1068{ 1069 lck_rw_init(lock, &LockCompatGroup, LCK_ATTR_NULL); 1070} 1071 1072void 1073lock_read_EXT( 1074 lck_rw_t *lock) 1075{ 1076 lck_rw_lock_shared( lock); 1077} 1078 1079boolean_t 1080lock_read_to_write_EXT( 1081 lck_rw_t *lock) 1082{ 1083 return( lck_rw_lock_shared_to_exclusive(lock)); 1084} 1085 1086void 1087lock_write_EXT( 1088 lck_rw_t *lock) 1089{ 1090 lck_rw_lock_exclusive(lock); 1091} 1092 1093void 1094lock_write_to_read_EXT( 1095 lck_rw_t *lock) 1096{ 1097 lck_rw_lock_exclusive_to_shared(lock); 1098} 1099 1100wait_result_t 1101thread_sleep_lock_write_EXT( 1102 event_t event, 1103 lck_rw_t *lock, 1104 wait_interrupt_t interruptible) 1105{ 1106 return( lck_rw_sleep(lock, LCK_SLEEP_EXCLUSIVE, event, interruptible)); 1107} 1108 1109void 1110usimple_lock_EXT( 1111 lck_spin_t *lock) 1112{ 1113 lck_spin_lock(lock); 1114} 1115 1116void 1117usimple_lock_init_EXT( 1118 lck_spin_t *lock, 1119 __unused unsigned short tag) 1120{ 1121 lck_spin_init(lock, &LockCompatGroup, LCK_ATTR_NULL); 1122} 1123 1124unsigned int 1125usimple_lock_try_EXT( 1126 lck_spin_t *lock) 1127{ 1128 return(lck_spin_try_lock(lock)); 1129} 1130 1131void 1132usimple_unlock_EXT( 1133 lck_spin_t *lock) 1134{ 1135 lck_spin_unlock(lock); 1136} 1137 1138wait_result_t 1139thread_sleep_usimple_lock_EXT( 1140 event_t event, 1141 lck_spin_t *lock, 1142 wait_interrupt_t interruptible) 1143{ 1144 return( lck_spin_sleep(lock, LCK_SLEEP_DEFAULT, event, interruptible)); 1145} 1146lck_mtx_t * 1147mutex_alloc_EXT( 1148 __unused unsigned short tag) 1149{ 1150 return(lck_mtx_alloc_init(&LockCompatGroup, LCK_ATTR_NULL)); 1151} 1152 1153void 1154mutex_free_EXT( 1155 lck_mtx_t *mutex) 1156{ 1157 lck_mtx_free(mutex, &LockCompatGroup); 1158} 1159 1160void 1161mutex_init_EXT( 1162 lck_mtx_t *mutex, 1163 __unused unsigned short tag) 1164{ 1165 lck_mtx_init(mutex, &LockCompatGroup, LCK_ATTR_NULL); 1166} 1167 1168wait_result_t 1169thread_sleep_mutex_EXT( 1170 event_t event, 1171 lck_mtx_t *mutex, 1172 wait_interrupt_t interruptible) 1173{ 1174 return( lck_mtx_sleep(mutex, LCK_SLEEP_DEFAULT, event, interruptible)); 1175} 1176 1177wait_result_t 1178thread_sleep_mutex_deadline_EXT( 1179 event_t event, 1180 lck_mtx_t *mutex, 1181 uint64_t deadline, 1182 wait_interrupt_t interruptible) 1183{ 1184 return( lck_mtx_sleep_deadline(mutex, LCK_SLEEP_DEFAULT, event, interruptible, deadline)); 1185} 1186