1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56#include <mach_kdb.h> 57#include <mach_ldebug.h> 58#include <debug.h> 59 60#include <mach/kern_return.h> 61#include <mach/mach_host_server.h> 62#include <mach_debug/lockgroup_info.h> 63 64#include <kern/locks.h> 65#include <kern/misc_protos.h> 66#include <kern/kalloc.h> 67#include <kern/thread.h> 68#include <kern/processor.h> 69#include <kern/sched_prim.h> 70#include <kern/debug.h> 71#include <string.h> 72 73 74#include <sys/kdebug.h> 75 76#if CONFIG_DTRACE 77/* 78 * We need only enough declarations from the BSD-side to be able to 79 * test if our probe is active, and to call __dtrace_probe(). Setting 80 * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in. 81 */ 82#define NEED_DTRACE_DEFS 83#include <../bsd/sys/lockstat.h> 84#endif 85 86#define LCK_MTX_SLEEP_CODE 0 87#define LCK_MTX_SLEEP_DEADLINE_CODE 1 88#define LCK_MTX_LCK_WAIT_CODE 2 89#define LCK_MTX_UNLCK_WAKEUP_CODE 3 90 91 92static queue_head_t lck_grp_queue; 93static unsigned int lck_grp_cnt; 94 95decl_mutex_data(static,lck_grp_lock) 96 97lck_grp_attr_t LockDefaultGroupAttr; 98lck_grp_t LockCompatGroup; 99lck_attr_t LockDefaultLckAttr; 100 101/* 102 * Routine: lck_mod_init 103 */ 104 105void 106lck_mod_init( 107 void) 108{ 109 queue_init(&lck_grp_queue); 110 mutex_init(&lck_grp_lock, 0); 111 lck_grp_cnt = 0; 112 lck_grp_attr_setdefault( &LockDefaultGroupAttr); 113 lck_grp_init( &LockCompatGroup, "Compatibility APIs", LCK_GRP_ATTR_NULL); 114 lck_attr_setdefault(&LockDefaultLckAttr); 115} 116 117/* 118 * Routine: lck_grp_attr_alloc_init 119 */ 120 121lck_grp_attr_t * 122lck_grp_attr_alloc_init( 123 void) 124{ 125 lck_grp_attr_t *attr; 126 127 if ((attr = (lck_grp_attr_t *)kalloc(sizeof(lck_grp_attr_t))) != 0) 128 lck_grp_attr_setdefault(attr); 129 130 return(attr); 131} 132 133 134/* 135 * Routine: lck_grp_attr_setdefault 136 */ 137 138void 139lck_grp_attr_setdefault( 140 lck_grp_attr_t *attr) 141{ 142 if (LcksOpts & enaLkStat) 143 attr->grp_attr_val = LCK_GRP_ATTR_STAT; 144 else 145 attr->grp_attr_val = 0; 146} 147 148 149/* 150 * Routine: lck_grp_attr_setstat 151 */ 152 153void 154lck_grp_attr_setstat( 155 lck_grp_attr_t *attr) 156{ 157 (void)hw_atomic_or(&attr->grp_attr_val, LCK_GRP_ATTR_STAT); 158} 159 160 161/* 162 * Routine: lck_grp_attr_free 163 */ 164 165void 166lck_grp_attr_free( 167 lck_grp_attr_t *attr) 168{ 169 kfree(attr, sizeof(lck_grp_attr_t)); 170} 171 172 173/* 174 * Routine: lck_grp_alloc_init 175 */ 176 177lck_grp_t * 178lck_grp_alloc_init( 179 const char* grp_name, 180 lck_grp_attr_t *attr) 181{ 182 lck_grp_t *grp; 183 184 if ((grp = (lck_grp_t *)kalloc(sizeof(lck_grp_t))) != 0) 185 lck_grp_init(grp, grp_name, attr); 186 187 return(grp); 188} 189 190 191/* 192 * Routine: lck_grp_init 193 */ 194 195void 196lck_grp_init( 197 lck_grp_t *grp, 198 const char* grp_name, 199 lck_grp_attr_t *attr) 200{ 201 bzero((void *)grp, sizeof(lck_grp_t)); 202 203 (void) strncpy(grp->lck_grp_name, grp_name, LCK_GRP_MAX_NAME); 204 205 if (attr != LCK_GRP_ATTR_NULL) 206 grp->lck_grp_attr = attr->grp_attr_val; 207 else if (LcksOpts & enaLkStat) 208 grp->lck_grp_attr = LCK_GRP_ATTR_STAT; 209 else 210 grp->lck_grp_attr = LCK_ATTR_NONE; 211 212 grp->lck_grp_refcnt = 1; 213 214 mutex_lock(&lck_grp_lock); 215 enqueue_tail(&lck_grp_queue, (queue_entry_t)grp); 216 lck_grp_cnt++; 217 mutex_unlock(&lck_grp_lock); 218 219} 220 221 222/* 223 * Routine: lck_grp_free 224 */ 225 226void 227lck_grp_free( 228 lck_grp_t *grp) 229{ 230 mutex_lock(&lck_grp_lock); 231 lck_grp_cnt--; 232 (void)remque((queue_entry_t)grp); 233 mutex_unlock(&lck_grp_lock); 234 lck_grp_deallocate(grp); 235} 236 237 238/* 239 * Routine: lck_grp_reference 240 */ 241 242void 243lck_grp_reference( 244 lck_grp_t *grp) 245{ 246 (void)hw_atomic_add(&grp->lck_grp_refcnt, 1); 247} 248 249 250/* 251 * Routine: lck_grp_deallocate 252 */ 253 254void 255lck_grp_deallocate( 256 lck_grp_t *grp) 257{ 258 if (hw_atomic_sub(&grp->lck_grp_refcnt, 1) == 0) 259 kfree(grp, sizeof(lck_grp_t)); 260} 261 262/* 263 * Routine: lck_grp_lckcnt_incr 264 */ 265 266void 267lck_grp_lckcnt_incr( 268 lck_grp_t *grp, 269 lck_type_t lck_type) 270{ 271 unsigned int *lckcnt; 272 273 switch (lck_type) { 274 case LCK_TYPE_SPIN: 275 lckcnt = &grp->lck_grp_spincnt; 276 break; 277 case LCK_TYPE_MTX: 278 lckcnt = &grp->lck_grp_mtxcnt; 279 break; 280 case LCK_TYPE_RW: 281 lckcnt = &grp->lck_grp_rwcnt; 282 break; 283 default: 284 return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type); 285 } 286 287 (void)hw_atomic_add(lckcnt, 1); 288} 289 290/* 291 * Routine: lck_grp_lckcnt_decr 292 */ 293 294void 295lck_grp_lckcnt_decr( 296 lck_grp_t *grp, 297 lck_type_t lck_type) 298{ 299 unsigned int *lckcnt; 300 301 switch (lck_type) { 302 case LCK_TYPE_SPIN: 303 lckcnt = &grp->lck_grp_spincnt; 304 break; 305 case LCK_TYPE_MTX: 306 lckcnt = &grp->lck_grp_mtxcnt; 307 break; 308 case LCK_TYPE_RW: 309 lckcnt = &grp->lck_grp_rwcnt; 310 break; 311 default: 312 return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type); 313 } 314 315 (void)hw_atomic_sub(lckcnt, 1); 316} 317 318/* 319 * Routine: lck_attr_alloc_init 320 */ 321 322lck_attr_t * 323lck_attr_alloc_init( 324 void) 325{ 326 lck_attr_t *attr; 327 328 if ((attr = (lck_attr_t *)kalloc(sizeof(lck_attr_t))) != 0) 329 lck_attr_setdefault(attr); 330 331 return(attr); 332} 333 334 335/* 336 * Routine: lck_attr_setdefault 337 */ 338 339void 340lck_attr_setdefault( 341 lck_attr_t *attr) 342{ 343#if !DEBUG 344 if (LcksOpts & enaLkDeb) 345 attr->lck_attr_val = LCK_ATTR_DEBUG; 346 else 347 attr->lck_attr_val = LCK_ATTR_NONE; 348#else 349 attr->lck_attr_val = LCK_ATTR_DEBUG; 350#endif /* !DEBUG */ 351} 352 353 354/* 355 * Routine: lck_attr_setdebug 356 */ 357void 358lck_attr_setdebug( 359 lck_attr_t *attr) 360{ 361 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG); 362} 363 364/* 365 * Routine: lck_attr_setdebug 366 */ 367void 368lck_attr_cleardebug( 369 lck_attr_t *attr) 370{ 371 (void)hw_atomic_and(&attr->lck_attr_val, ~LCK_ATTR_DEBUG); 372} 373 374 375/* 376 * Routine: lck_attr_rw_shared_priority 377 */ 378void 379lck_attr_rw_shared_priority( 380 lck_attr_t *attr) 381{ 382 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY); 383} 384 385 386/* 387 * Routine: lck_attr_free 388 */ 389void 390lck_attr_free( 391 lck_attr_t *attr) 392{ 393 kfree(attr, sizeof(lck_attr_t)); 394} 395 396 397/* 398 * Routine: lck_spin_sleep 399 */ 400wait_result_t 401lck_spin_sleep( 402 lck_spin_t *lck, 403 lck_sleep_action_t lck_sleep_action, 404 event_t event, 405 wait_interrupt_t interruptible) 406{ 407 wait_result_t res; 408 409 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 410 panic("Invalid lock sleep action %x\n", lck_sleep_action); 411 412 res = assert_wait(event, interruptible); 413 if (res == THREAD_WAITING) { 414 lck_spin_unlock(lck); 415 res = thread_block(THREAD_CONTINUE_NULL); 416 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) 417 lck_spin_lock(lck); 418 } 419 else 420 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 421 lck_spin_unlock(lck); 422 423 return res; 424} 425 426 427/* 428 * Routine: lck_spin_sleep_deadline 429 */ 430wait_result_t 431lck_spin_sleep_deadline( 432 lck_spin_t *lck, 433 lck_sleep_action_t lck_sleep_action, 434 event_t event, 435 wait_interrupt_t interruptible, 436 uint64_t deadline) 437{ 438 wait_result_t res; 439 440 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 441 panic("Invalid lock sleep action %x\n", lck_sleep_action); 442 443 res = assert_wait_deadline(event, interruptible, deadline); 444 if (res == THREAD_WAITING) { 445 lck_spin_unlock(lck); 446 res = thread_block(THREAD_CONTINUE_NULL); 447 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) 448 lck_spin_lock(lck); 449 } 450 else 451 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 452 lck_spin_unlock(lck); 453 454 return res; 455} 456 457 458/* 459 * Routine: lck_mtx_sleep 460 */ 461wait_result_t 462lck_mtx_sleep( 463 lck_mtx_t *lck, 464 lck_sleep_action_t lck_sleep_action, 465 event_t event, 466 wait_interrupt_t interruptible) 467{ 468 wait_result_t res; 469 470 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START, 471 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0); 472 473 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 474 panic("Invalid lock sleep action %x\n", lck_sleep_action); 475 476 res = assert_wait(event, interruptible); 477 if (res == THREAD_WAITING) { 478 lck_mtx_unlock(lck); 479 res = thread_block(THREAD_CONTINUE_NULL); 480 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) 481 lck_mtx_lock(lck); 482 } 483 else 484 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 485 lck_mtx_unlock(lck); 486 487 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); 488 489 return res; 490} 491 492 493/* 494 * Routine: lck_mtx_sleep_deadline 495 */ 496wait_result_t 497lck_mtx_sleep_deadline( 498 lck_mtx_t *lck, 499 lck_sleep_action_t lck_sleep_action, 500 event_t event, 501 wait_interrupt_t interruptible, 502 uint64_t deadline) 503{ 504 wait_result_t res; 505 506 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START, 507 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0); 508 509 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 510 panic("Invalid lock sleep action %x\n", lck_sleep_action); 511 512 res = assert_wait_deadline(event, interruptible, deadline); 513 if (res == THREAD_WAITING) { 514 lck_mtx_unlock(lck); 515 res = thread_block(THREAD_CONTINUE_NULL); 516 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) 517 lck_mtx_lock(lck); 518 } 519 else 520 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 521 lck_mtx_unlock(lck); 522 523 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); 524 525 return res; 526} 527 528/* 529 * Routine: lck_mtx_lock_wait 530 * 531 * Invoked in order to wait on contention. 532 * 533 * Called with the interlock locked and 534 * returns it unlocked. 535 */ 536void 537lck_mtx_lock_wait ( 538 lck_mtx_t *lck, 539 thread_t holder) 540{ 541 thread_t self = current_thread(); 542 lck_mtx_t *mutex; 543 integer_t priority; 544 spl_t s = splsched(); 545#if CONFIG_DTRACE 546 uint64_t sleep_start = 0; 547 548 if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) { 549 sleep_start = mach_absolute_time(); 550 } 551#endif 552 553 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) 554 mutex = lck; 555 else 556 mutex = &lck->lck_mtx_ptr->lck_mtx; 557 558 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0); 559 560 priority = self->sched_pri; 561 if (priority < self->priority) 562 priority = self->priority; 563 if (priority < BASEPRI_DEFAULT) 564 priority = BASEPRI_DEFAULT; 565 566 thread_lock(holder); 567 if (mutex->lck_mtx_pri == 0) 568 holder->promotions++; 569 holder->sched_mode |= TH_MODE_PROMOTED; 570 if ( mutex->lck_mtx_pri < priority && 571 holder->sched_pri < priority ) { 572 KERNEL_DEBUG_CONSTANT( 573 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, 574 holder->sched_pri, priority, (int)holder, (int)lck, 0); 575 576 set_sched_pri(holder, priority); 577 } 578 thread_unlock(holder); 579 splx(s); 580 581 if (mutex->lck_mtx_pri < priority) 582 mutex->lck_mtx_pri = priority; 583 if (self->pending_promoter[self->pending_promoter_index] == NULL) { 584 self->pending_promoter[self->pending_promoter_index] = mutex; 585 mutex->lck_mtx_waiters++; 586 } 587 else 588 if (self->pending_promoter[self->pending_promoter_index] != mutex) { 589 self->pending_promoter[++self->pending_promoter_index] = mutex; 590 mutex->lck_mtx_waiters++; 591 } 592 593 assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))), THREAD_UNINT); 594 lck_mtx_ilk_unlock(mutex); 595 596 thread_block(THREAD_CONTINUE_NULL); 597 598 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); 599#if CONFIG_DTRACE 600 /* 601 * Record the Dtrace lockstat probe for blocking, block time 602 * measured from when we were entered. 603 */ 604 if (sleep_start) { 605 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) { 606 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck, 607 mach_absolute_time() - sleep_start); 608 } else { 609 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck, 610 mach_absolute_time() - sleep_start); 611 } 612 } 613#endif 614} 615 616/* 617 * Routine: lck_mtx_lock_acquire 618 * 619 * Invoked on acquiring the mutex when there is 620 * contention. 621 * 622 * Returns the current number of waiters. 623 * 624 * Called with the interlock locked. 625 */ 626int 627lck_mtx_lock_acquire( 628 lck_mtx_t *lck) 629{ 630 thread_t thread = current_thread(); 631 lck_mtx_t *mutex; 632 633 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) 634 mutex = lck; 635 else 636 mutex = &lck->lck_mtx_ptr->lck_mtx; 637 638 if (thread->pending_promoter[thread->pending_promoter_index] == mutex) { 639 thread->pending_promoter[thread->pending_promoter_index] = NULL; 640 if (thread->pending_promoter_index > 0) 641 thread->pending_promoter_index--; 642 mutex->lck_mtx_waiters--; 643 } 644 645 if (mutex->lck_mtx_waiters > 0) { 646 integer_t priority = mutex->lck_mtx_pri; 647 spl_t s = splsched(); 648 649 thread_lock(thread); 650 thread->promotions++; 651 thread->sched_mode |= TH_MODE_PROMOTED; 652 if (thread->sched_pri < priority) { 653 KERNEL_DEBUG_CONSTANT( 654 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE, 655 thread->sched_pri, priority, 0, (int)lck, 0); 656 657 set_sched_pri(thread, priority); 658 } 659 thread_unlock(thread); 660 splx(s); 661 } 662 else 663 mutex->lck_mtx_pri = 0; 664 665 return (mutex->lck_mtx_waiters); 666} 667 668/* 669 * Routine: lck_mtx_unlock_wakeup 670 * 671 * Invoked on unlock when there is contention. 672 * 673 * Called with the interlock locked. 674 */ 675void 676lck_mtx_unlock_wakeup ( 677 lck_mtx_t *lck, 678 thread_t holder) 679{ 680 thread_t thread = current_thread(); 681 lck_mtx_t *mutex; 682 683 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) 684 mutex = lck; 685 else 686 mutex = &lck->lck_mtx_ptr->lck_mtx; 687 688 689 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0); 690 691 if (thread != holder) 692 panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex, holder); 693 694 if (thread->promotions > 0) { 695 spl_t s = splsched(); 696 697 thread_lock(thread); 698 if ( --thread->promotions == 0 && 699 (thread->sched_mode & TH_MODE_PROMOTED) ) { 700 thread->sched_mode &= ~TH_MODE_PROMOTED; 701 if (thread->sched_mode & TH_MODE_ISDEPRESSED) { 702 KERNEL_DEBUG_CONSTANT( 703 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | DBG_FUNC_NONE, 704 thread->sched_pri, DEPRESSPRI, 0, (int)lck, 0); 705 706 set_sched_pri(thread, DEPRESSPRI); 707 } 708 else { 709 if (thread->priority < thread->sched_pri) { 710 KERNEL_DEBUG_CONSTANT( 711 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | 712 DBG_FUNC_NONE, 713 thread->sched_pri, thread->priority, 714 0, (int)lck, 0); 715 } 716 717 compute_priority(thread, FALSE); 718 } 719 } 720 thread_unlock(thread); 721 splx(s); 722 } 723 assert(mutex->lck_mtx_waiters > 0); 724 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int))); 725 726 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0); 727} 728 729void 730lck_mtx_unlockspin_wakeup ( 731 lck_mtx_t *lck) 732{ 733 assert(lck->lck_mtx_waiters > 0); 734 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int))); 735 736 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_NONE, (int)lck, 0, 0, 1, 0); 737#if CONFIG_DTRACE 738 /* 739 * When there are waiters, we skip the hot-patch spot in the 740 * fastpath, so we record it here. 741 */ 742 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lck, 0); 743#endif 744} 745 746 747/* 748 * Routine: mutex_pause 749 * 750 * Called by former callers of simple_lock_pause(). 751 */ 752#define MAX_COLLISION_COUNTS 32 753#define MAX_COLLISION 8 754 755unsigned int max_collision_count[MAX_COLLISION_COUNTS]; 756 757uint32_t collision_backoffs[MAX_COLLISION] = { 758 10, 50, 100, 200, 400, 600, 800, 1000 759}; 760 761 762void 763mutex_pause(uint32_t collisions) 764{ 765 wait_result_t wait_result; 766 uint32_t back_off; 767 768 if (collisions >= MAX_COLLISION_COUNTS) 769 collisions = MAX_COLLISION_COUNTS - 1; 770 max_collision_count[collisions]++; 771 772 if (collisions >= MAX_COLLISION) 773 collisions = MAX_COLLISION - 1; 774 back_off = collision_backoffs[collisions]; 775 776 wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, back_off, NSEC_PER_USEC); 777 assert(wait_result == THREAD_WAITING); 778 779 wait_result = thread_block(THREAD_CONTINUE_NULL); 780 assert(wait_result == THREAD_TIMED_OUT); 781} 782 783 784unsigned int mutex_yield_wait = 0; 785unsigned int mutex_yield_no_wait = 0; 786 787void 788mutex_yield( 789 mutex_t *mutex) 790{ 791 lck_mtx_t *lck; 792 793#if DEBUG 794 _mutex_assert(mutex, MA_OWNED); 795#endif /* DEBUG */ 796 797 lck = (lck_mtx_t *) mutex; 798 if (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT) 799 lck = &lck->lck_mtx_ptr->lck_mtx; 800 801 if (! lck->lck_mtx_waiters) { 802 mutex_yield_no_wait++; 803 } else { 804 mutex_yield_wait++; 805 mutex_unlock(mutex); 806 mutex_pause(0); 807 mutex_lock(mutex); 808 } 809} 810 811 812/* 813 * Routine: lck_rw_sleep 814 */ 815wait_result_t 816lck_rw_sleep( 817 lck_rw_t *lck, 818 lck_sleep_action_t lck_sleep_action, 819 event_t event, 820 wait_interrupt_t interruptible) 821{ 822 wait_result_t res; 823 lck_rw_type_t lck_rw_type; 824 825 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 826 panic("Invalid lock sleep action %x\n", lck_sleep_action); 827 828 res = assert_wait(event, interruptible); 829 if (res == THREAD_WAITING) { 830 lck_rw_type = lck_rw_done(lck); 831 res = thread_block(THREAD_CONTINUE_NULL); 832 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { 833 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE))) 834 lck_rw_lock(lck, lck_rw_type); 835 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) 836 lck_rw_lock_exclusive(lck); 837 else 838 lck_rw_lock_shared(lck); 839 } 840 } 841 else 842 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 843 (void)lck_rw_done(lck); 844 845 return res; 846} 847 848 849/* 850 * Routine: lck_rw_sleep_deadline 851 */ 852wait_result_t 853lck_rw_sleep_deadline( 854 lck_rw_t *lck, 855 lck_sleep_action_t lck_sleep_action, 856 event_t event, 857 wait_interrupt_t interruptible, 858 uint64_t deadline) 859{ 860 wait_result_t res; 861 lck_rw_type_t lck_rw_type; 862 863 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) 864 panic("Invalid lock sleep action %x\n", lck_sleep_action); 865 866 res = assert_wait_deadline(event, interruptible, deadline); 867 if (res == THREAD_WAITING) { 868 lck_rw_type = lck_rw_done(lck); 869 res = thread_block(THREAD_CONTINUE_NULL); 870 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { 871 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE))) 872 lck_rw_lock(lck, lck_rw_type); 873 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) 874 lck_rw_lock_exclusive(lck); 875 else 876 lck_rw_lock_shared(lck); 877 } 878 } 879 else 880 if (lck_sleep_action & LCK_SLEEP_UNLOCK) 881 (void)lck_rw_done(lck); 882 883 return res; 884} 885 886kern_return_t 887host_lockgroup_info( 888 host_t host, 889 lockgroup_info_array_t *lockgroup_infop, 890 mach_msg_type_number_t *lockgroup_infoCntp) 891{ 892 lockgroup_info_t *lockgroup_info_base; 893 lockgroup_info_t *lockgroup_info; 894 vm_offset_t lockgroup_info_addr; 895 vm_size_t lockgroup_info_size; 896 lck_grp_t *lck_grp; 897 unsigned int i; 898 vm_size_t used; 899 vm_map_copy_t copy; 900 kern_return_t kr; 901 902 if (host == HOST_NULL) 903 return KERN_INVALID_HOST; 904 905 mutex_lock(&lck_grp_lock); 906 907 lockgroup_info_size = round_page(lck_grp_cnt * sizeof *lockgroup_info); 908 kr = kmem_alloc_pageable(ipc_kernel_map, 909 &lockgroup_info_addr, lockgroup_info_size); 910 if (kr != KERN_SUCCESS) { 911 mutex_unlock(&lck_grp_lock); 912 return(kr); 913 } 914 915 lockgroup_info_base = (lockgroup_info_t *) lockgroup_info_addr; 916 lck_grp = (lck_grp_t *)queue_first(&lck_grp_queue); 917 lockgroup_info = lockgroup_info_base; 918 919 for (i = 0; i < lck_grp_cnt; i++) { 920 921 lockgroup_info->lock_spin_cnt = lck_grp->lck_grp_spincnt; 922 lockgroup_info->lock_spin_util_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_util_cnt; 923 lockgroup_info->lock_spin_held_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cnt; 924 lockgroup_info->lock_spin_miss_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_miss_cnt; 925 lockgroup_info->lock_spin_held_max = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_max; 926 lockgroup_info->lock_spin_held_cum = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cum; 927 928 lockgroup_info->lock_mtx_cnt = lck_grp->lck_grp_mtxcnt; 929 lockgroup_info->lock_mtx_util_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt; 930 lockgroup_info->lock_mtx_held_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt; 931 lockgroup_info->lock_mtx_miss_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt; 932 lockgroup_info->lock_mtx_wait_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt; 933 lockgroup_info->lock_mtx_held_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_max; 934 lockgroup_info->lock_mtx_held_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cum; 935 lockgroup_info->lock_mtx_wait_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_max; 936 lockgroup_info->lock_mtx_wait_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cum; 937 938 lockgroup_info->lock_rw_cnt = lck_grp->lck_grp_rwcnt; 939 lockgroup_info->lock_rw_util_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt; 940 lockgroup_info->lock_rw_held_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cnt; 941 lockgroup_info->lock_rw_miss_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt; 942 lockgroup_info->lock_rw_wait_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cnt; 943 lockgroup_info->lock_rw_held_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_max; 944 lockgroup_info->lock_rw_held_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cum; 945 lockgroup_info->lock_rw_wait_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_max; 946 lockgroup_info->lock_rw_wait_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cum; 947 948 (void) strncpy(lockgroup_info->lockgroup_name,lck_grp->lck_grp_name, LOCKGROUP_MAX_NAME); 949 950 lck_grp = (lck_grp_t *)(queue_next((queue_entry_t)(lck_grp))); 951 lockgroup_info++; 952 } 953 954 *lockgroup_infoCntp = lck_grp_cnt; 955 mutex_unlock(&lck_grp_lock); 956 957 used = (*lockgroup_infoCntp) * sizeof *lockgroup_info; 958 959 if (used != lockgroup_info_size) 960 bzero((char *) lockgroup_info, lockgroup_info_size - used); 961 962 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)lockgroup_info_addr, 963 (vm_map_size_t)lockgroup_info_size, TRUE, ©); 964 assert(kr == KERN_SUCCESS); 965 966 *lockgroup_infop = (lockgroup_info_t *) copy; 967 968 return(KERN_SUCCESS); 969} 970 971/* 972 * Compatibility module 973 */ 974 975extern lck_rw_t *lock_alloc_EXT( boolean_t can_sleep, unsigned short tag0, unsigned short tag1); 976extern void lock_done_EXT(lck_rw_t *lock); 977extern void lock_free_EXT(lck_rw_t *lock); 978extern void lock_init_EXT(lck_rw_t *lock, boolean_t can_sleep, unsigned short tag0, unsigned short tag1); 979extern void lock_read_EXT(lck_rw_t *lock); 980extern boolean_t lock_read_to_write_EXT(lck_rw_t *lock); 981extern void lock_write_EXT(lck_rw_t *lock); 982extern void lock_write_to_read_EXT(lck_rw_t *lock); 983extern wait_result_t thread_sleep_lock_write_EXT( 984 event_t event, lck_rw_t *lock, wait_interrupt_t interruptible); 985 986extern lck_mtx_t *mutex_alloc_EXT(unsigned short tag); 987extern void mutex_free_EXT(lck_mtx_t *mutex); 988extern void mutex_init_EXT(lck_mtx_t *mutex, unsigned short tag); 989extern void mutex_lock_EXT(lck_mtx_t *mutex); 990extern boolean_t mutex_try_EXT(lck_mtx_t *mutex); 991extern void mutex_unlock_EXT(lck_mtx_t *mutex); 992extern wait_result_t thread_sleep_mutex_EXT( 993 event_t event, lck_mtx_t *mutex, wait_interrupt_t interruptible); 994extern wait_result_t thread_sleep_mutex_deadline_EXT( 995 event_t event, lck_mtx_t *mutex, uint64_t deadline, wait_interrupt_t interruptible); 996 997extern void usimple_lock_EXT(lck_spin_t *lock); 998extern void usimple_lock_init_EXT(lck_spin_t *lock, unsigned short tag); 999extern unsigned int usimple_lock_try_EXT(lck_spin_t *lock); 1000extern void usimple_unlock_EXT(lck_spin_t *lock); 1001extern wait_result_t thread_sleep_usimple_lock_EXT(event_t event, lck_spin_t *lock, wait_interrupt_t interruptible); 1002 1003lck_rw_t * 1004lock_alloc_EXT( 1005 __unused boolean_t can_sleep, 1006 __unused unsigned short tag0, 1007 __unused unsigned short tag1) 1008{ 1009 return( lck_rw_alloc_init( &LockCompatGroup, LCK_ATTR_NULL)); 1010} 1011 1012void 1013lock_done_EXT( 1014 lck_rw_t *lock) 1015{ 1016 (void) lck_rw_done(lock); 1017} 1018 1019void 1020lock_free_EXT( 1021 lck_rw_t *lock) 1022{ 1023 lck_rw_free(lock, &LockCompatGroup); 1024} 1025 1026void 1027lock_init_EXT( 1028 lck_rw_t *lock, 1029 __unused boolean_t can_sleep, 1030 __unused unsigned short tag0, 1031 __unused unsigned short tag1) 1032{ 1033 lck_rw_init(lock, &LockCompatGroup, LCK_ATTR_NULL); 1034} 1035 1036void 1037lock_read_EXT( 1038 lck_rw_t *lock) 1039{ 1040 lck_rw_lock_shared( lock); 1041} 1042 1043boolean_t 1044lock_read_to_write_EXT( 1045 lck_rw_t *lock) 1046{ 1047 return( lck_rw_lock_shared_to_exclusive(lock)); 1048} 1049 1050void 1051lock_write_EXT( 1052 lck_rw_t *lock) 1053{ 1054 lck_rw_lock_exclusive(lock); 1055} 1056 1057void 1058lock_write_to_read_EXT( 1059 lck_rw_t *lock) 1060{ 1061 lck_rw_lock_exclusive_to_shared(lock); 1062} 1063 1064wait_result_t 1065thread_sleep_lock_write_EXT( 1066 event_t event, 1067 lck_rw_t *lock, 1068 wait_interrupt_t interruptible) 1069{ 1070 return( lck_rw_sleep(lock, LCK_SLEEP_EXCLUSIVE, event, interruptible)); 1071} 1072 1073lck_mtx_t * 1074mutex_alloc_EXT( 1075 __unused unsigned short tag) 1076{ 1077 return(lck_mtx_alloc_init(&LockCompatGroup, LCK_ATTR_NULL)); 1078} 1079 1080void 1081mutex_free_EXT( 1082 lck_mtx_t *mutex) 1083{ 1084 lck_mtx_free(mutex, &LockCompatGroup); 1085} 1086 1087void 1088mutex_init_EXT( 1089 lck_mtx_t *mutex, 1090 __unused unsigned short tag) 1091{ 1092 lck_mtx_init(mutex, &LockCompatGroup, LCK_ATTR_NULL); 1093} 1094 1095void 1096mutex_lock_EXT( 1097 lck_mtx_t *mutex) 1098{ 1099 lck_mtx_lock(mutex); 1100} 1101 1102boolean_t 1103mutex_try_EXT( 1104 lck_mtx_t *mutex) 1105{ 1106 return(lck_mtx_try_lock(mutex)); 1107} 1108 1109void 1110mutex_unlock_EXT( 1111 lck_mtx_t *mutex) 1112{ 1113 lck_mtx_unlock(mutex); 1114} 1115 1116wait_result_t 1117thread_sleep_mutex_EXT( 1118 event_t event, 1119 lck_mtx_t *mutex, 1120 wait_interrupt_t interruptible) 1121{ 1122 return( lck_mtx_sleep(mutex, LCK_SLEEP_DEFAULT, event, interruptible)); 1123} 1124 1125wait_result_t 1126thread_sleep_mutex_deadline_EXT( 1127 event_t event, 1128 lck_mtx_t *mutex, 1129 uint64_t deadline, 1130 wait_interrupt_t interruptible) 1131{ 1132 return( lck_mtx_sleep_deadline(mutex, LCK_SLEEP_DEFAULT, event, interruptible, deadline)); 1133} 1134 1135void 1136usimple_lock_EXT( 1137 lck_spin_t *lock) 1138{ 1139 lck_spin_lock(lock); 1140} 1141 1142void 1143usimple_lock_init_EXT( 1144 lck_spin_t *lock, 1145 __unused unsigned short tag) 1146{ 1147 lck_spin_init(lock, &LockCompatGroup, LCK_ATTR_NULL); 1148} 1149 1150unsigned int 1151usimple_lock_try_EXT( 1152 lck_spin_t *lock) 1153{ 1154 return(lck_spin_try_lock(lock)); 1155} 1156 1157void 1158usimple_unlock_EXT( 1159 lck_spin_t *lock) 1160{ 1161 lck_spin_unlock(lock); 1162} 1163 1164wait_result_t 1165thread_sleep_usimple_lock_EXT( 1166 event_t event, 1167 lck_spin_t *lock, 1168 wait_interrupt_t interruptible) 1169{ 1170 return( lck_spin_sleep(lock, LCK_SLEEP_DEFAULT, event, interruptible)); 1171} 1172