1/* 2 * Copyright (c) 2013 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <mach/mach_types.h> 30#include <mach/notify.h> 31#include <ipc/ipc_types.h> 32#include <ipc/ipc_importance.h> 33#include <ipc/ipc_port.h> 34#include <ipc/ipc_voucher.h> 35#include <kern/ipc_kobject.h> 36#include <kern/ipc_tt.h> 37#include <kern/mach_param.h> 38#include <kern/misc_protos.h> 39#include <kern/kalloc.h> 40#include <kern/zalloc.h> 41#include <kern/queue.h> 42#include <kern/task.h> 43 44#include <sys/kdebug.h> 45 46#include <mach/mach_voucher_attr_control.h> 47#include <mach/machine/sdt.h> 48 49extern int proc_pid(void *); 50extern int proc_selfpid(void); 51extern uint64_t proc_uniqueid(void *p); 52extern char *proc_name_address(void *p); 53 54/* 55 * Globals for delayed boost drop processing. 56 */ 57static queue_head_t ipc_importance_delayed_drop_queue; 58static thread_call_t ipc_importance_delayed_drop_call; 59static uint64_t ipc_importance_delayed_drop_timestamp; 60static boolean_t ipc_importance_delayed_drop_call_requested = FALSE; 61 62#define DENAP_DROP_TARGET (1000 * NSEC_PER_MSEC) /* optimum denap delay */ 63#define DENAP_DROP_SKEW (100 * NSEC_PER_MSEC) /* request skew for wakeup */ 64#define DENAP_DROP_LEEWAY (2 * DENAP_DROP_SKEW) /* specified wakeup leeway */ 65 66#define DENAP_DROP_DELAY (DENAP_DROP_TARGET + DENAP_DROP_SKEW) 67#define DENAP_DROP_FLAGS (THREAD_CALL_DELAY_SYS_NORMAL | THREAD_CALL_DELAY_LEEWAY) 68 69/* 70 * Importance Voucher Attribute Manager 71 */ 72 73static lck_spin_t ipc_importance_lock_data; /* single lock for now */ 74 75 76#define ipc_importance_lock_init() \ 77 lck_spin_init(&ipc_importance_lock_data, &ipc_lck_grp, &ipc_lck_attr) 78#define ipc_importance_lock_destroy() \ 79 lck_spin_destroy(&ipc_importance_lock_data, &ipc_lck_grp) 80#define ipc_importance_lock() \ 81 lck_spin_lock(&ipc_importance_lock_data) 82#define ipc_importance_lock_try() \ 83 lck_spin_try_lock(&ipc_importance_lock_data) 84#define ipc_importance_unlock() \ 85 lck_spin_unlock(&ipc_importance_lock_data) 86#define ipc_importance_sleep(elem) lck_spin_sleep(&ipc_importance_lock_data, \ 87 LCK_SLEEP_DEFAULT, \ 88 (event_t)(elem), \ 89 THREAD_UNINT) 90#define ipc_importance_wakeup(elem) thread_wakeup((event_t)(elem)) 91 92#if IIE_REF_DEBUG 93#define incr_ref_counter(x) (hw_atomic_add(&(x), 1)) 94 95static inline 96uint32_t ipc_importance_reference_internal(ipc_importance_elem_t elem) 97{ 98 incr_ref_counter(elem->iie_refs_added); 99 return (hw_atomic_add(&elem->iie_bits, 1) & IIE_REFS_MASK); 100} 101 102static inline 103uint32_t ipc_importance_release_internal(ipc_importance_elem_t elem) 104{ 105 incr_ref_counter(elem->iie_refs_dropped); 106 return (hw_atomic_sub(&elem->iie_bits, 1) & IIE_REFS_MASK); 107} 108 109static inline 110uint32_t ipc_importance_task_reference_internal(ipc_importance_task_t task_imp) 111{ 112 uint32_t out; 113 out = ipc_importance_reference_internal(&task_imp->iit_elem); 114 incr_ref_counter(task_imp->iit_elem.iie_task_refs_added); 115 return out; 116} 117 118static inline 119uint32_t ipc_importance_task_release_internal(ipc_importance_task_t task_imp) 120{ 121 uint32_t out; 122 123 assert(1 < IIT_REFS(task_imp)); 124 incr_ref_counter(task_imp->iit_elem.iie_task_refs_dropped); 125 out = ipc_importance_release_internal(&task_imp->iit_elem); 126 return out; 127} 128 129static inline 130void ipc_importance_counter_init(ipc_importance_elem_t elem) 131{ 132 133 elem->iie_refs_added = 0; 134 elem->iie_refs_dropped = 0; 135 elem->iie_kmsg_refs_added = 0; 136 elem->iie_kmsg_refs_inherited = 0; 137 elem->iie_kmsg_refs_coalesced = 0; 138 elem->iie_kmsg_refs_dropped = 0; 139 elem->iie_task_refs_added = 0; 140 elem->iie_task_refs_added_inherit_from = 0; 141 elem->iie_task_refs_added_transition = 0; 142 elem->iie_task_refs_self_added = 0; 143 elem->iie_task_refs_inherited = 0; 144 elem->iie_task_refs_coalesced = 0; 145 elem->iie_task_refs_dropped = 0; 146} 147#else 148#define incr_ref_counter(x) 149#endif 150 151#if DEVELOPMENT || DEBUG 152static queue_head_t global_iit_alloc_queue; 153#endif 154 155/* TODO: remove this varibale when interactive daemon audit is complete */ 156boolean_t ipc_importance_interactive_receiver = FALSE; 157 158static zone_t ipc_importance_task_zone; 159static zone_t ipc_importance_inherit_zone; 160 161static ipc_voucher_attr_control_t ipc_importance_control; 162 163/* 164 * Routine: ipc_importance_kmsg_link 165 * Purpose: 166 * Link the kmsg onto the appropriate propagation chain. 167 * If the element is a task importance, we link directly 168 * on its propagation chain. Otherwise, we link onto the 169 * destination task of the inherit. 170 * Conditions: 171 * Importance lock held. 172 * Caller is donating an importance elem reference to the kmsg. 173 */ 174static void 175ipc_importance_kmsg_link( 176 ipc_kmsg_t kmsg, 177 ipc_importance_elem_t elem) 178{ 179 ipc_importance_elem_t link_elem; 180 181 assert(IIE_NULL == kmsg->ikm_importance); 182 183 link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? 184 (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : 185 elem; 186 187 queue_enter(&link_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance); 188 kmsg->ikm_importance = elem; 189} 190 191/* 192 * Routine: ipc_importance_kmsg_unlink 193 * Purpose: 194 * Unlink the kmsg from its current propagation chain. 195 * If the element is a task importance, we unlink directly 196 * from its propagation chain. Otherwise, we unlink from the 197 * destination task of the inherit. 198 * Returns: 199 * The reference to the importance element it was linked on. 200 * Conditions: 201 * Importance lock held. 202 * Caller is responsible for dropping reference on returned elem. 203 */ 204static ipc_importance_elem_t 205ipc_importance_kmsg_unlink( 206 ipc_kmsg_t kmsg) 207{ 208 ipc_importance_elem_t elem = kmsg->ikm_importance; 209 210 if (IIE_NULL != elem) { 211 ipc_importance_elem_t unlink_elem; 212 213 unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? 214 (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : 215 elem; 216 217 queue_remove(&unlink_elem->iie_kmsgs, kmsg, ipc_kmsg_t, ikm_inheritance); 218 kmsg->ikm_importance = IIE_NULL; 219 } 220 return elem; 221} 222 223/* 224 * Routine: ipc_importance_inherit_link 225 * Purpose: 226 * Link the inherit onto the appropriate propagation chain. 227 * If the element is a task importance, we link directly 228 * on its propagation chain. Otherwise, we link onto the 229 * destination task of the inherit. 230 * Conditions: 231 * Importance lock held. 232 * Caller is donating an elem importance reference to the inherit. 233 */ 234static void 235ipc_importance_inherit_link( 236 ipc_importance_inherit_t inherit, 237 ipc_importance_elem_t elem) 238{ 239 ipc_importance_elem_t link_elem; 240 241 assert(IIE_NULL == inherit->iii_from_elem); 242 link_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? 243 (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : 244 elem; 245 246 queue_enter(&link_elem->iie_inherits, inherit, 247 ipc_importance_inherit_t, iii_inheritance); 248 inherit->iii_from_elem = elem; 249} 250 251/* 252 * Routine: ipc_importance_inherit_unlink 253 * Purpose: 254 * Unlink the inherit from its current propagation chain. 255 * If the element is a task importance, we unlink directly 256 * from its propagation chain. Otherwise, we unlink from the 257 * destination task of the inherit. 258 * Returns: 259 * The reference to the importance element it was linked on. 260 * Conditions: 261 * Importance lock held. 262 * Caller is responsible for dropping reference on returned elem. 263 */ 264static ipc_importance_elem_t 265ipc_importance_inherit_unlink( 266 ipc_importance_inherit_t inherit) 267{ 268 ipc_importance_elem_t elem = inherit->iii_from_elem; 269 270 if (IIE_NULL != elem) { 271 ipc_importance_elem_t unlink_elem; 272 273 unlink_elem = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? 274 (ipc_importance_elem_t)((ipc_importance_inherit_t)elem)->iii_to_task : 275 elem; 276 277 queue_remove(&unlink_elem->iie_inherits, inherit, 278 ipc_importance_inherit_t, iii_inheritance); 279 inherit->iii_from_elem = IIE_NULL; 280 } 281 return elem; 282} 283 284/* 285 * Routine: ipc_importance_reference 286 * Purpose: 287 * Add a reference to the importance element. 288 * Conditions: 289 * Caller must hold a reference on the element. 290 */ 291void 292ipc_importance_reference(ipc_importance_elem_t elem) 293{ 294 assert(0 < IIE_REFS(elem)); 295 ipc_importance_reference_internal(elem); 296} 297 298/* 299 * Routine: ipc_importance_release_locked 300 * Purpose: 301 * Release a reference on an importance attribute value, 302 * unlinking and deallocating the attribute if the last reference. 303 * Conditions: 304 * Entered with importance lock held, leaves with it unlocked. 305 */ 306static void 307ipc_importance_release_locked(ipc_importance_elem_t elem) 308{ 309 assert(0 < IIE_REFS(elem)); 310 311 if (0 < ipc_importance_release_internal(elem)) { 312 313#if DEVELOPMENT || DEBUG 314 ipc_importance_inherit_t temp_inherit; 315 ipc_importance_task_t link_task; 316 ipc_kmsg_t temp_kmsg; 317 uint32_t expected = 0; 318 319 if (0 < elem->iie_made) 320 expected++; 321 322 link_task = (IIE_TYPE_INHERIT == IIE_TYPE(elem)) ? 323 ((ipc_importance_inherit_t)elem)->iii_to_task : 324 (ipc_importance_task_t)elem; 325 326 queue_iterate(&link_task->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) 327 if (temp_kmsg->ikm_importance == elem) 328 expected++; 329 queue_iterate(&link_task->iit_inherits, temp_inherit, 330 ipc_importance_inherit_t, iii_inheritance) 331 if (temp_inherit->iii_from_elem == elem) 332 expected++; 333 334 if (IIE_REFS(elem) < expected) 335 panic("ipc_importance_release_locked (%p)", elem); 336#endif 337 ipc_importance_unlock(); 338 return; 339 } 340 341 /* last ref */ 342 /* can't get to no refs if we contribute to something else's importance */ 343 assert(queue_empty(&elem->iie_kmsgs)); 344 assert(queue_empty(&elem->iie_inherits)); 345 346 switch (IIE_TYPE(elem)) { 347 348 /* just a "from" task reference to drop */ 349 case IIE_TYPE_TASK: 350 { 351 ipc_importance_task_t task_elem; 352 353 task_elem = (ipc_importance_task_t)elem; 354 assert(TASK_NULL == task_elem->iit_task); 355 356#if DEVELOPMENT || DEBUG 357 queue_remove(&global_iit_alloc_queue, task_elem, ipc_importance_task_t, iit_allocation); 358#endif 359 360 ipc_importance_unlock(); 361 362 zfree(ipc_importance_task_zone, task_elem); 363 break; 364 } 365 366 /* dropping an inherit element */ 367 case IIE_TYPE_INHERIT: 368 { 369 ipc_importance_inherit_t inherit; 370 ipc_importance_elem_t from_elem; 371 ipc_importance_task_t to_task; 372 373 374 inherit = (ipc_importance_inherit_t)elem; 375 to_task = inherit->iii_to_task; 376 assert(IIT_NULL != to_task); 377 assert(!inherit->iii_donating); 378 379 /* unlink and release the inherit */ 380 assert(ipc_importance_task_is_any_receiver_type(to_task)); 381 from_elem = ipc_importance_inherit_unlink(inherit); 382 assert(IIE_NULL != from_elem); 383 ipc_importance_release_locked(from_elem); 384 /* unlocked on return */ 385 386 ipc_importance_task_release(to_task); 387 388 zfree(ipc_importance_inherit_zone, inherit); 389 break; 390 } 391 } 392} 393 394/* 395 * Routine: ipc_importance_release 396 * Purpose: 397 * Release a reference on an importance attribute value, 398 * unlinking and deallocating the attribute if the last reference. 399 * Conditions: 400 * nothing locked on entrance, nothing locked on exit. 401 * May block. 402 */ 403void 404ipc_importance_release(ipc_importance_elem_t elem) 405{ 406 if (IIE_NULL == elem) 407 return; 408 409 ipc_importance_lock(); 410 ipc_importance_release_locked(elem); 411 /* unlocked */ 412} 413 414/* 415 * Routine: ipc_importance_task_reference 416 417 418 * Purpose: 419 * Retain a reference on a task importance attribute value. 420 * Conditions: 421 * nothing locked on entrance, nothing locked on exit. 422 * caller holds a reference already. 423 */ 424void 425ipc_importance_task_reference(ipc_importance_task_t task_elem) 426{ 427 if (IIT_NULL == task_elem) 428 return; 429#if IIE_REF_DEBUG 430 incr_ref_counter(task_elem->iit_elem.iie_task_refs_added); 431#endif 432 ipc_importance_reference(&task_elem->iit_elem); 433} 434 435/* 436 * Routine: ipc_importance_task_release 437 * Purpose: 438 * Release a reference on a task importance attribute value, 439 * unlinking and deallocating the attribute if the last reference. 440 * Conditions: 441 * nothing locked on entrance, nothing locked on exit. 442 * May block. 443 */ 444void 445ipc_importance_task_release(ipc_importance_task_t task_elem) 446{ 447 if (IIT_NULL == task_elem) 448 return; 449 450 ipc_importance_lock(); 451#if IIE_REF_DEBUG 452 incr_ref_counter(task_elem->iit_elem.iie_task_refs_dropped); 453#endif 454 ipc_importance_release_locked(&task_elem->iit_elem); 455 /* unlocked */ 456} 457 458/* 459 * Routine: ipc_importance_task_release_locked 460 * Purpose: 461 * Release a reference on a task importance attribute value, 462 * unlinking and deallocating the attribute if the last reference. 463 * Conditions: 464 * importance lock held on entry, nothing locked on exit. 465 * May block. 466 */ 467static void 468ipc_importance_task_release_locked(ipc_importance_task_t task_elem) 469{ 470 if (IIT_NULL == task_elem) { 471 ipc_importance_unlock(); 472 return; 473 } 474#if IIE_REF_DEBUG 475 incr_ref_counter(task_elem->iit_elem.iie_task_refs_dropped); 476#endif 477 ipc_importance_release_locked(&task_elem->iit_elem); 478 /* unlocked */ 479} 480 481/* 482 * Routines for importance donation/inheritance/boosting 483 */ 484 485 486/* 487 * External importance assertions are managed by the process in userspace 488 * Internal importance assertions are the responsibility of the kernel 489 * Assertions are changed from internal to external via task_importance_externalize_assertion 490 */ 491 492/* 493 * Routine: ipc_importance_task_check_transition 494 * Purpose: 495 * Increase or decrement the internal task importance counter of the 496 * specified task and determine if propagation and a task policy 497 * update is required. 498 * 499 * If it is already enqueued for a policy update, steal it from that queue 500 * (as we are reversing that update before it happens). 501 * 502 * Conditions: 503 * Called with the importance lock held. 504 * It is the caller's responsibility to perform the propagation of the 505 * transition and/or policy changes by checking the return value. 506 */ 507static boolean_t 508ipc_importance_task_check_transition( 509 ipc_importance_task_t task_imp, 510 iit_update_type_t type, 511 uint32_t delta) 512{ 513 514 task_t target_task = task_imp->iit_task; 515 boolean_t boost = (IIT_UPDATE_HOLD == type); 516 boolean_t before_boosted, after_boosted; 517 518 if (!ipc_importance_task_is_any_receiver_type(task_imp)) 519 return FALSE; 520 521#if IMPORTANCE_DEBUG 522 int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1; 523 524 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_START, 525 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); 526#endif 527 528 /* snapshot the effective boosting status before making any changes */ 529 before_boosted = (task_imp->iit_assertcnt > 0); 530 531 /* Adjust the assertcnt appropriately */ 532 if (boost) { 533 task_imp->iit_assertcnt += delta; 534#if IMPORTANCE_DEBUG 535 DTRACE_BOOST6(send_boost, task_t, target_task, int, target_pid, 536 task_t, current_task(), int, proc_selfpid(), int, delta, int, task_imp->iit_assertcnt); 537#endif 538 } else { 539 // assert(delta <= task_imp->iit_assertcnt); 540 if (delta > task_imp->iit_assertcnt - IIT_EXTERN(task_imp)) { 541 /* TODO: Turn this back into a panic <rdar://problem/12592649> */ 542 if (target_task != TASK_NULL) { 543 printf("Over-release of kernel-internal importance assertions for pid %d (%s), " 544 "dropping %d assertion(s) but task only has %d remaining (%d external).\n", 545 audit_token_pid_from_task(target_task), 546 (target_task->bsd_info == NULL) ? "" : proc_name_address(target_task->bsd_info), 547 delta, 548 task_imp->iit_assertcnt, 549 IIT_EXTERN(task_imp)); 550 } 551 task_imp->iit_assertcnt = IIT_EXTERN(task_imp); 552 } else { 553 task_imp->iit_assertcnt -= delta; 554 } 555#if IMPORTANCE_DEBUG 556 // This convers both legacy and voucher-based importance. 557 DTRACE_BOOST4(drop_boost, task_t, target_task, int, target_pid, int, delta, int, task_imp->iit_assertcnt); 558#endif 559 } 560 561#if IMPORTANCE_DEBUG 562 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (((boost) ? IMP_HOLD : IMP_DROP) | TASK_POLICY_INTERNAL))) | DBG_FUNC_END, 563 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); 564#endif 565 566 /* did the change result in an effective donor status change? */ 567 after_boosted = (task_imp->iit_assertcnt > 0); 568 569 if (after_boosted != before_boosted) { 570 571 /* 572 * If the task importance is already on an update queue, we just reversed the need for a 573 * pending policy update. If the queue is any other than the delayed-drop-queue, pull it 574 * off that queue and release the reference it got going onto the update queue. If it is 575 * the delayed-drop-queue we leave it in place in case it comes back into the drop state 576 * before its time delay is up. 577 * 578 * We still need to propagate the change downstream to reverse the assertcnt effects, 579 * but we no longer need to update this task's boost policy state. 580 * 581 * Otherwise, mark it as needing a policy update. 582 */ 583 assert(0 == task_imp->iit_updatepolicy); 584 if (NULL != task_imp->iit_updateq) { 585 if (&ipc_importance_delayed_drop_queue != task_imp->iit_updateq) { 586 queue_remove(task_imp->iit_updateq, task_imp, ipc_importance_task_t, iit_updates); 587 task_imp->iit_updateq = NULL; 588 ipc_importance_task_release_internal(task_imp); /* can't be last ref */ 589 } 590 } else { 591 task_imp->iit_updatepolicy = 1; 592 } 593 return TRUE; 594 } 595 596 return FALSE; 597} 598 599 600/* 601 * Routine: ipc_importance_task_propagate_helper 602 * Purpose: 603 * Increase or decrement the internal task importance counter of all 604 * importance tasks inheriting from the specified one. If this causes 605 * that importance task to change state, add it to the list of tasks 606 * to do a policy update against. 607 * Conditions: 608 * Called with the importance lock held. 609 * It is the caller's responsibility to iterate down the generated list 610 * and propagate any subsequent assertion changes from there. 611 */ 612static void 613ipc_importance_task_propagate_helper( 614 ipc_importance_task_t task_imp, 615 iit_update_type_t type, 616 queue_t propagation) 617{ 618 ipc_importance_task_t temp_task_imp; 619 620 /* 621 * iterate the downstream kmsgs, adjust their boosts, 622 * and capture the next task to adjust for each message 623 */ 624 625 ipc_kmsg_t temp_kmsg; 626 627 queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) { 628 mach_msg_header_t *hdr = temp_kmsg->ikm_header; 629 mach_port_delta_t delta; 630 ipc_port_t port; 631 632 /* toggle the kmsg importance bit as a barrier to parallel adjusts */ 633 if (IIT_UPDATE_HOLD == type) { 634 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr->msgh_bits)) { 635 continue; 636 } 637 638 /* mark the message as now carrying importance */ 639 hdr->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; 640 delta = 1; 641 } else { 642 if (!MACH_MSGH_BITS_RAISED_IMPORTANCE(hdr->msgh_bits)) { 643 continue; 644 } 645 646 /* clear the message as now carrying importance */ 647 hdr->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; 648 delta = -1; 649 } 650 651 /* determine the task importance to adjust as result (if any) */ 652 port = (ipc_port_t) hdr->msgh_remote_port; 653 assert(IP_VALID(port)); 654 ip_lock(port); 655 temp_task_imp = IIT_NULL; 656 if (!ipc_port_importance_delta_internal(port, &delta, &temp_task_imp)) { 657 ip_unlock(port); 658 } 659 660 /* no task importance to adjust associated with the port? */ 661 if (IIT_NULL == temp_task_imp) { 662 continue; 663 } 664 665 /* hold a reference on temp_task_imp */ 666 667 /* Adjust the task assertions and determine if an edge was crossed */ 668 if (ipc_importance_task_check_transition(temp_task_imp, type, 1)) { 669 incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); 670 queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); 671 /* reference donated */ 672 } else { 673 ipc_importance_task_release_internal(temp_task_imp); 674 } 675 } 676 677 /* 678 * iterate the downstream importance inherits 679 * and capture the next task importance to boost for each 680 */ 681 ipc_importance_inherit_t temp_inherit; 682 683 queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) { 684 uint32_t assertcnt = III_EXTERN(temp_inherit); 685 686 temp_task_imp = temp_inherit->iii_to_task; 687 assert(IIT_NULL != temp_task_imp); 688 689 if (IIT_UPDATE_HOLD == type) { 690 /* if no undropped externcnts in the inherit, nothing to do */ 691 if (0 == assertcnt) { 692 assert(temp_inherit->iii_donating == FALSE); 693 continue; 694 } 695 696 /* nothing to do if the inherit is already donating (forced donation) */ 697 if (temp_inherit->iii_donating) { 698 continue; 699 } 700 701 /* mark it donating and contribute to the task externcnts */ 702 temp_inherit->iii_donating = TRUE; 703 temp_task_imp->iit_externcnt += temp_inherit->iii_externcnt; 704 temp_task_imp->iit_externdrop += temp_inherit->iii_externdrop; 705 706 } else { 707 /* if no contributing assertions, move on */ 708 if (0 == assertcnt) { 709 assert(temp_inherit->iii_donating == FALSE); 710 continue; 711 } 712 713 /* nothing to do if the inherit is not donating */ 714 if (!temp_inherit->iii_donating) { 715 continue; 716 } 717 718 /* mark it no longer donating */ 719 temp_inherit->iii_donating = FALSE; 720 721 /* remove the contribution the inherit made to the to-task */ 722 assert(IIT_EXTERN(temp_task_imp) >= III_EXTERN(temp_inherit)); 723 assert(temp_task_imp->iit_externcnt >= temp_inherit->iii_externcnt); 724 assert(temp_task_imp->iit_externdrop >= temp_inherit->iii_externdrop); 725 temp_task_imp->iit_externcnt -= temp_inherit->iii_externcnt; 726 temp_task_imp->iit_externdrop -= temp_inherit->iii_externdrop; 727 728 } 729 730 /* Adjust the task assertions and determine if an edge was crossed */ 731 assert(ipc_importance_task_is_any_receiver_type(temp_task_imp)); 732 if (ipc_importance_task_check_transition(temp_task_imp, type, assertcnt)) { 733 ipc_importance_task_reference(temp_task_imp); 734 incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_transition); 735 queue_enter(propagation, temp_task_imp, ipc_importance_task_t, iit_props); 736 } 737 } 738} 739 740/* 741 * Routine: ipc_importance_task_process_updates 742 * Purpose: 743 * Process the queue of task importances and apply the policy 744 * update called for. Only process tasks in the queue with an 745 * update timestamp less than the supplied max. 746 * Conditions: 747 * Called and returns with importance locked. 748 * May drop importance lock and block temporarily. 749 */ 750static void 751ipc_importance_task_process_updates( 752 queue_t supplied_queue, 753 boolean_t boost, 754 uint64_t max_timestamp) 755{ 756 ipc_importance_task_t task_imp; 757 queue_head_t second_chance; 758 queue_t queue = supplied_queue; 759 760 /* 761 * This queue will hold the task's we couldn't trylock on first pass. 762 * By using a second (private) queue, we guarantee all tasks that get 763 * entered on this queue have a timestamp under the maximum. 764 */ 765 queue_init(&second_chance); 766 767 /* process any resulting policy updates */ 768 retry: 769 while(!queue_empty(queue)) { 770 task_t target_task; 771 struct task_pend_token pend_token = {}; 772 773 task_imp = (ipc_importance_task_t)queue_first(queue); 774 assert(0 == task_imp->iit_updatepolicy); 775 assert(queue == task_imp->iit_updateq); 776 777 /* if timestamp is too big, we're done */ 778 if (task_imp->iit_updatetime > max_timestamp) { 779 break; 780 } 781 782 /* we were given a reference on each task in the queue */ 783 784 /* remove it from the supplied queue */ 785 queue_remove(queue, task_imp, ipc_importance_task_t, iit_updates); 786 task_imp->iit_updateq = NULL; 787 788 target_task = task_imp->iit_task; 789 790 /* Is it well on the way to exiting? */ 791 if (TASK_NULL == target_task) { 792 ipc_importance_task_release_locked(task_imp); 793 /* importance unlocked */ 794 ipc_importance_lock(); 795 continue; 796 } 797 798 /* Has the update been reversed on the hysteresis queue? */ 799 if (0 < task_imp->iit_assertcnt && 800 queue == &ipc_importance_delayed_drop_queue) { 801 ipc_importance_task_release_locked(task_imp); 802 /* importance unlocked */ 803 ipc_importance_lock(); 804 continue; 805 } 806 807 /* 808 * Can we get the task lock out-of-order? 809 * If not, stick this back on the second-chance queue. 810 */ 811 if (!task_lock_try(target_task)) { 812 boolean_t should_wait_lock = (queue == &second_chance); 813 task_imp->iit_updateq = &second_chance; 814 815 /* 816 * If we're already processing second-chances on 817 * tasks, keep this task on the front of the queue. 818 * We will wait for the task lock before coming 819 * back and trying again, and we have a better 820 * chance of re-acquiring the lock if we come back 821 * to it right away. 822 */ 823 if (should_wait_lock){ 824 task_reference(target_task); 825 queue_enter_first(&second_chance, task_imp, 826 ipc_importance_task_t, iit_updates); 827 } else { 828 queue_enter(&second_chance, task_imp, 829 ipc_importance_task_t, iit_updates); 830 } 831 ipc_importance_unlock(); 832 833 if (should_wait_lock) { 834 task_lock(target_task); 835 task_unlock(target_task); 836 task_deallocate(target_task); 837 } 838 839 ipc_importance_lock(); 840 continue; 841 } 842 843 /* is it going away? */ 844 if (!target_task->active) { 845 task_unlock(target_task); 846 ipc_importance_task_release_locked(task_imp); 847 /* importance unlocked */ 848 ipc_importance_lock(); 849 continue; 850 } 851 852 /* take a task reference for while we don't have the importance lock */ 853 task_reference(target_task); 854 855 /* count the transition */ 856 if (boost) 857 task_imp->iit_transitions++; 858 859 ipc_importance_unlock(); 860 861 /* apply the policy adjust to the target task (while it is still locked) */ 862 task_update_boost_locked(target_task, boost, &pend_token); 863 864 /* complete the policy update with the task unlocked */ 865 ipc_importance_task_release(task_imp); 866 task_unlock(target_task); 867 task_policy_update_complete_unlocked(target_task, THREAD_NULL, &pend_token); 868 task_deallocate(target_task); 869 870 ipc_importance_lock(); 871 } 872 873 /* If there are tasks we couldn't update the first time, try again */ 874 if (!queue_empty(&second_chance)) { 875 queue = &second_chance; 876 goto retry; 877 } 878} 879 880 881/* 882 * Routine: ipc_importance_task_delayed_drop_scan 883 * Purpose: 884 * The thread call routine to scan the delayed drop queue, 885 * requesting all updates with a deadline up to the last target 886 * for the thread-call (which is DENAP_DROP_SKEW beyond the first 887 * thread's optimum delay). 888 * update to drop its boost. 889 * Conditions: 890 * Nothing locked 891 */ 892static void 893ipc_importance_task_delayed_drop_scan( 894 __unused void *arg1, 895 __unused void *arg2) 896{ 897 ipc_importance_lock(); 898 899 /* process all queued task drops with timestamps up to TARGET(first)+SKEW */ 900 ipc_importance_task_process_updates(&ipc_importance_delayed_drop_queue, 901 FALSE, 902 ipc_importance_delayed_drop_timestamp); 903 904 /* importance lock may have been temporarily dropped */ 905 906 /* If there are any entries left in the queue, re-arm the call here */ 907 if (!queue_empty(&ipc_importance_delayed_drop_queue)) { 908 ipc_importance_task_t task_imp; 909 uint64_t deadline; 910 uint64_t leeway; 911 912 task_imp = (ipc_importance_task_t)queue_first(&ipc_importance_delayed_drop_queue); 913 914 nanoseconds_to_absolutetime(DENAP_DROP_DELAY, &deadline); 915 deadline += task_imp->iit_updatetime; 916 ipc_importance_delayed_drop_timestamp = deadline; 917 918 nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway); 919 920 thread_call_enter_delayed_with_leeway( 921 ipc_importance_delayed_drop_call, 922 NULL, 923 deadline, 924 leeway, 925 DENAP_DROP_FLAGS); 926 } else { 927 ipc_importance_delayed_drop_call_requested = FALSE; 928 } 929 ipc_importance_unlock(); 930} 931 932/* 933 * Routine: ipc_importance_task_delayed_drop 934 * Purpose: 935 * Queue the specified task importance for delayed policy 936 * update to drop its boost. 937 * Conditions: 938 * Called with the importance lock held. 939 */ 940static void 941ipc_importance_task_delayed_drop(ipc_importance_task_t task_imp) 942{ 943 uint64_t timestamp = mach_absolute_time(); /* no mach_approximate_time() in kernel */ 944 945 assert(ipc_importance_delayed_drop_call != NULL); 946 947 /* 948 * If still on an update queue from a previous change, 949 * remove it first (and use that reference). Otherwise, take 950 * a new reference for the delay drop update queue. 951 */ 952 if (NULL != task_imp->iit_updateq) { 953 queue_remove(task_imp->iit_updateq, task_imp, 954 ipc_importance_task_t, iit_updates); 955 } else { 956 ipc_importance_task_reference_internal(task_imp); 957 } 958 959 task_imp->iit_updateq = &ipc_importance_delayed_drop_queue; 960 task_imp->iit_updatetime = timestamp; 961 962 queue_enter(&ipc_importance_delayed_drop_queue, task_imp, 963 ipc_importance_task_t, iit_updates); 964 965 /* request the delayed thread-call if not already requested */ 966 if (!ipc_importance_delayed_drop_call_requested) { 967 uint64_t deadline; 968 uint64_t leeway; 969 970 nanoseconds_to_absolutetime(DENAP_DROP_DELAY, &deadline); 971 deadline += task_imp->iit_updatetime; 972 ipc_importance_delayed_drop_timestamp = deadline; 973 974 nanoseconds_to_absolutetime(DENAP_DROP_LEEWAY, &leeway); 975 976 ipc_importance_delayed_drop_call_requested = TRUE; 977 thread_call_enter_delayed_with_leeway( 978 ipc_importance_delayed_drop_call, 979 NULL, 980 deadline, 981 leeway, 982 DENAP_DROP_FLAGS); 983 } 984} 985 986 987/* 988 * Routine: ipc_importance_task_propagate_assertion_locked 989 * Purpose: 990 * Propagate the importance transition type to every item 991 * If this causes a boost to be applied, determine if that 992 * boost should propagate downstream. 993 * Conditions: 994 * Called with the importance lock held. 995 */ 996static void 997ipc_importance_task_propagate_assertion_locked( 998 ipc_importance_task_t task_imp, 999 iit_update_type_t type, 1000 boolean_t update_task_imp) 1001{ 1002 boolean_t boost = (IIT_UPDATE_HOLD == type); 1003 ipc_importance_task_t temp_task_imp; 1004 queue_head_t propagate; 1005 queue_head_t updates; 1006 1007 queue_init(&updates); 1008 queue_init(&propagate); 1009 1010 /* 1011 * If we're going to update the policy for the provided task, 1012 * enqueue it on the propagate queue itself. Otherwise, only 1013 * enqueue downstream things. 1014 */ 1015 if (update_task_imp) { 1016 queue_enter(&propagate, task_imp, ipc_importance_task_t, iit_props); 1017 } else { 1018 ipc_importance_task_propagate_helper(task_imp, type, &propagate); 1019 } 1020 1021 /* 1022 * for each item on the propagation list, propagate any change downstream, 1023 * adding new tasks to propagate further if they transistioned as well. 1024 */ 1025 while (!queue_empty(&propagate)) { 1026 boolean_t need_update; 1027 1028 queue_remove_first(&propagate, temp_task_imp, ipc_importance_task_t, iit_props); 1029 assert(IIT_NULL != temp_task_imp); 1030 1031 /* only propagate for receivers not already marked as a donor */ 1032 if (!ipc_importance_task_is_marked_donor(temp_task_imp) && 1033 ipc_importance_task_is_marked_receiver(temp_task_imp)) { 1034 ipc_importance_task_propagate_helper(temp_task_imp, type, &propagate); 1035 } 1036 1037 /* if we have a policy update to apply, enqueue a reference for later processing */ 1038 need_update = (0 != temp_task_imp->iit_updatepolicy); 1039 temp_task_imp->iit_updatepolicy = 0; 1040 if (need_update && TASK_NULL != temp_task_imp->iit_task) { 1041 if (NULL == temp_task_imp->iit_updateq) { 1042 temp_task_imp->iit_updatetime = 0; 1043 temp_task_imp->iit_updateq = &updates; 1044 ipc_importance_task_reference_internal(temp_task_imp); 1045 if (boost) { 1046 queue_enter(&updates, temp_task_imp, 1047 ipc_importance_task_t, iit_updates); 1048 } else { 1049 queue_enter_first(&updates, temp_task_imp, 1050 ipc_importance_task_t, iit_updates); 1051 } 1052 } else { 1053 /* Must already be on the AppNap hysteresis queue */ 1054 assert(&ipc_importance_delayed_drop_queue); 1055 assert(ipc_importance_task_is_marked_denap_receiver(temp_task_imp)); 1056 } 1057 } 1058 } 1059 1060 /* apply updates to task (may drop importance lock) */ 1061 if (!queue_empty(&updates)) { 1062 ipc_importance_task_process_updates(&updates, boost, 0); 1063 } 1064} 1065 1066/* 1067 * Routine: ipc_importance_task_hold_internal_assertion_locked 1068 * Purpose: 1069 * Increment the assertion count on the task importance. 1070 * If this results in a boost state change in that task, 1071 * prepare to update task policy for this task AND, if 1072 * if not just waking out of App Nap, all down-stream 1073 * tasks that have a similar transition through inheriting 1074 * this update. 1075 * Conditions: 1076 * importance locked on entry and exit. 1077 * May temporarily drop importance lock and block. 1078 */ 1079static kern_return_t 1080ipc_importance_task_hold_internal_assertion_locked(ipc_importance_task_t task_imp, uint32_t count) 1081{ 1082 if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_HOLD, count)) { 1083 ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_HOLD, TRUE); 1084 } 1085 return KERN_SUCCESS; 1086} 1087 1088/* 1089 * Routine: ipc_importance_task_drop_internal_assertion_locked 1090 * Purpose: 1091 * Decrement the assertion count on the task importance. 1092 * If this results in a boost state change in that task, 1093 * prepare to update task policy for this task AND, if 1094 * if not just waking out of App Nap, all down-stream 1095 * tasks that have a similar transition through inheriting 1096 * this update. 1097 * Conditions: 1098 * importance locked on entry and exit. 1099 * May temporarily drop importance lock and block. 1100 */ 1101static kern_return_t 1102ipc_importance_task_drop_internal_assertion_locked(ipc_importance_task_t task_imp, uint32_t count) 1103{ 1104 if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) { 1105 ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE); 1106 } 1107 return KERN_SUCCESS; 1108} 1109 1110/* 1111 * Routine: ipc_importance_task_hold_internal_assertion 1112 * Purpose: 1113 * Increment the assertion count on the task importance. 1114 * If this results in a 0->1 change in that count, 1115 * prepare to update task policy for this task AND 1116 * (potentially) all down-stream tasks that have a 1117 * similar transition through inheriting this update. 1118 * Conditions: 1119 * Nothing locked 1120 * May block after dropping importance lock. 1121 */ 1122int 1123ipc_importance_task_hold_internal_assertion(ipc_importance_task_t task_imp, uint32_t count) 1124{ 1125 int ret = KERN_SUCCESS; 1126 1127 if (ipc_importance_task_is_any_receiver_type(task_imp)) { 1128 ipc_importance_lock(); 1129 ret = ipc_importance_task_hold_internal_assertion_locked(task_imp, count); 1130 ipc_importance_unlock(); 1131 } 1132 return ret; 1133} 1134 1135/* 1136 * Routine: ipc_importance_task_drop_internal_assertion 1137 * Purpose: 1138 * Decrement the assertion count on the task importance. 1139 * If this results in a X->0 change in that count, 1140 * prepare to update task policy for this task AND 1141 * all down-stream tasks that have a similar transition 1142 * through inheriting this drop update. 1143 * Conditions: 1144 * Nothing locked on entry. 1145 * May block after dropping importance lock. 1146 */ 1147kern_return_t 1148ipc_importance_task_drop_internal_assertion(ipc_importance_task_t task_imp, uint32_t count) 1149{ 1150 kern_return_t ret = KERN_SUCCESS; 1151 1152 if (ipc_importance_task_is_any_receiver_type(task_imp)) { 1153 ipc_importance_lock(); 1154 ret = ipc_importance_task_drop_internal_assertion_locked(task_imp, count); 1155 ipc_importance_unlock(); 1156 } 1157 return ret; 1158} 1159 1160/* 1161 * Routine: ipc_importance_task_hold_file_lock_assertion 1162 * Purpose: 1163 * Increment the file lock assertion count on the task importance. 1164 * If this results in a 0->1 change in that count, 1165 * prepare to update task policy for this task AND 1166 * (potentially) all down-stream tasks that have a 1167 * similar transition through inheriting this update. 1168 * Conditions: 1169 * Nothing locked 1170 * May block after dropping importance lock. 1171 */ 1172kern_return_t 1173ipc_importance_task_hold_file_lock_assertion(ipc_importance_task_t task_imp, uint32_t count) 1174{ 1175 kern_return_t ret = KERN_SUCCESS; 1176 1177 if (ipc_importance_task_is_any_receiver_type(task_imp)) { 1178 ipc_importance_lock(); 1179 ret = ipc_importance_task_hold_internal_assertion_locked(task_imp, count); 1180 if (KERN_SUCCESS == ret) { 1181 task_imp->iit_filelocks += count; 1182 } 1183 ipc_importance_unlock(); 1184 } 1185 return ret; 1186} 1187 1188/* 1189 * Routine: ipc_importance_task_drop_file_lock_assertion 1190 * Purpose: 1191 * Decrement the assertion count on the task importance. 1192 * If this results in a X->0 change in that count, 1193 * prepare to update task policy for this task AND 1194 * all down-stream tasks that have a similar transition 1195 * through inheriting this drop update. 1196 * Conditions: 1197 * Nothing locked on entry. 1198 * May block after dropping importance lock. 1199 */ 1200kern_return_t 1201ipc_importance_task_drop_file_lock_assertion(ipc_importance_task_t task_imp, uint32_t count) 1202{ 1203 kern_return_t ret = KERN_SUCCESS; 1204 1205 if (ipc_importance_task_is_any_receiver_type(task_imp)) { 1206 ipc_importance_lock(); 1207 if (count <= task_imp->iit_filelocks) { 1208 task_imp->iit_filelocks -= count; 1209 ret = ipc_importance_task_drop_internal_assertion_locked(task_imp, count); 1210 } else { 1211 ret = KERN_INVALID_ARGUMENT; 1212 } 1213 ipc_importance_unlock(); 1214 } 1215 return ret; 1216} 1217 1218/* 1219 * Routine: ipc_importance_task_hold_legacy_external_assertion 1220 * Purpose: 1221 * Increment the external assertion count on the task importance. 1222 * This cannot result in an 0->1 transition, as the caller must 1223 * already hold an external boost. 1224 * Conditions: 1225 * Nothing locked on entry. 1226 * May block after dropping importance lock. 1227 * A queue of task importance structures is returned 1228 * by ipc_importance_task_hold_assertion_locked(). Each 1229 * needs to be updated (outside the importance lock hold). 1230 */ 1231kern_return_t 1232ipc_importance_task_hold_legacy_external_assertion(ipc_importance_task_t task_imp, uint32_t count) 1233{ 1234 task_t target_task; 1235 uint32_t target_assertcnt; 1236 uint32_t target_externcnt; 1237 uint32_t target_legacycnt; 1238 1239 kern_return_t ret; 1240 1241 ipc_importance_lock(); 1242 target_task = task_imp->iit_task; 1243 1244#if IMPORTANCE_DEBUG 1245 int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1; 1246 1247 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, 1248 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); 1249#endif 1250 1251 if (IIT_LEGACY_EXTERN(task_imp) == 0) { 1252 /* Only allowed to take a new boost assertion when holding an external boost */ 1253 /* save data for diagnostic printf below */ 1254 target_assertcnt = task_imp->iit_assertcnt; 1255 target_externcnt = IIT_EXTERN(task_imp); 1256 target_legacycnt = IIT_LEGACY_EXTERN(task_imp); 1257 ret = KERN_FAILURE; 1258 count = 0; 1259 } else { 1260 assert(ipc_importance_task_is_any_receiver_type(task_imp)); 1261 assert(0 < task_imp->iit_assertcnt); 1262 assert(0 < IIT_EXTERN(task_imp)); 1263 task_imp->iit_assertcnt += count; 1264 task_imp->iit_externcnt += count; 1265 task_imp->iit_legacy_externcnt += count; 1266 ret = KERN_SUCCESS; 1267 } 1268 ipc_importance_unlock(); 1269 1270#if IMPORTANCE_DEBUG 1271 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_HOLD | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, 1272 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); 1273 // This covers the legacy case where a task takes an extra boost. 1274 DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, proc_selfpid(), int, count, int, task_imp->iit_assertcnt); 1275#endif 1276 1277 if (KERN_FAILURE == ret && target_task != TASK_NULL) { 1278 printf("BUG in process %s[%d]: " 1279 "attempt to acquire an additional legacy external boost assertion without holding an existing legacy external assertion. " 1280 "(%d total, %d external, %d legacy-external)\n", 1281 proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task), 1282 target_assertcnt, target_externcnt, target_legacycnt); 1283 } 1284 1285 return(ret); 1286} 1287 1288/* 1289 * Routine: ipc_importance_task_drop_legacy_external_assertion 1290 * Purpose: 1291 * Drop the legacy external assertion count on the task and 1292 * reflect that change to total external assertion count and 1293 * then onto the internal importance count. 1294 * 1295 * If this results in a X->0 change in the internal, 1296 * count, prepare to update task policy for this task AND 1297 * all down-stream tasks that have a similar transition 1298 * through inheriting this update. 1299 * Conditions: 1300 * Nothing locked on entry. 1301 */ 1302kern_return_t 1303ipc_importance_task_drop_legacy_external_assertion(ipc_importance_task_t task_imp, uint32_t count) 1304{ 1305 int ret = KERN_SUCCESS; 1306 task_t target_task; 1307 uint32_t target_assertcnt; 1308 uint32_t target_externcnt; 1309 uint32_t target_legacycnt; 1310 1311 if (count > 1) { 1312 return KERN_INVALID_ARGUMENT; 1313 } 1314 1315 ipc_importance_lock(); 1316 target_task = task_imp->iit_task; 1317 1318#if IMPORTANCE_DEBUG 1319 int target_pid = (TASK_NULL != target_task) ? audit_token_pid_from_task(target_task) : -1; 1320 1321 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_START, 1322 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); 1323#endif 1324 1325 if (count > IIT_LEGACY_EXTERN(task_imp)) { 1326 /* Process over-released its boost count - save data for diagnostic printf */ 1327 /* TODO: If count > 1, we should clear out as many external assertions as there are left. */ 1328 target_assertcnt = task_imp->iit_assertcnt; 1329 target_externcnt = IIT_EXTERN(task_imp); 1330 target_legacycnt = IIT_LEGACY_EXTERN(task_imp); 1331 ret = KERN_FAILURE; 1332 } else { 1333 /* 1334 * decrement legacy external count from the top level and reflect 1335 * into internal for this and all subsequent updates. 1336 */ 1337 assert(ipc_importance_task_is_any_receiver_type(task_imp)); 1338 assert(IIT_EXTERN(task_imp) >= count); 1339 1340 task_imp->iit_legacy_externdrop += count; 1341 task_imp->iit_externdrop += count; 1342 1343 /* reset extern counters (if appropriate) */ 1344 if (IIT_LEGACY_EXTERN(task_imp) == 0) { 1345 if (IIT_EXTERN(task_imp) != 0) { 1346 task_imp->iit_externcnt -= task_imp->iit_legacy_externcnt; 1347 task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop; 1348 } else { 1349 task_imp->iit_externcnt = 0; 1350 task_imp->iit_externdrop = 0; 1351 } 1352 task_imp->iit_legacy_externcnt = 0; 1353 task_imp->iit_legacy_externdrop = 0; 1354 } 1355 1356 /* reflect the drop to the internal assertion count (and effect any importance change) */ 1357 if (ipc_importance_task_check_transition(task_imp, IIT_UPDATE_DROP, count)) { 1358 ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, TRUE); 1359 } 1360 ret = KERN_SUCCESS; 1361 } 1362 1363#if IMPORTANCE_DEBUG 1364 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, (IMP_DROP | TASK_POLICY_EXTERNAL))) | DBG_FUNC_END, 1365 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); 1366#endif 1367 1368 ipc_importance_unlock(); 1369 1370 /* delayed printf for user-supplied data failures */ 1371 if (KERN_FAILURE == ret && TASK_NULL != target_task) { 1372 printf("BUG in process %s[%d]: over-released legacy external boost assertions (%d total, %d external, %d legacy-external)\n", 1373 proc_name_address(target_task->bsd_info), audit_token_pid_from_task(target_task), 1374 target_assertcnt, target_externcnt, target_legacycnt); 1375 } 1376 1377 return(ret); 1378} 1379 1380 1381 1382/* Transfer an assertion to legacy userspace responsibility */ 1383static kern_return_t 1384ipc_importance_task_externalize_legacy_assertion(ipc_importance_task_t task_imp, uint32_t count, __unused int sender_pid) 1385{ 1386 task_t target_task; 1387 1388 assert(IIT_NULL != task_imp); 1389 target_task = task_imp->iit_task; 1390 1391 if (TASK_NULL == target_task || 1392 !ipc_importance_task_is_any_receiver_type(task_imp)) { 1393 return KERN_FAILURE; 1394 } 1395 1396#if IMPORTANCE_DEBUG 1397 int target_pid = audit_token_pid_from_task(target_task); 1398 1399 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_START, 1400 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_EXTERN(task_imp), 0); 1401#endif 1402 1403 ipc_importance_lock(); 1404 /* assert(task_imp->iit_assertcnt >= IIT_EXTERN(task_imp) + count); */ 1405 assert(IIT_EXTERN(task_imp) >= IIT_LEGACY_EXTERN(task_imp)); 1406 task_imp->iit_legacy_externcnt += count; 1407 task_imp->iit_externcnt += count; 1408 ipc_importance_unlock(); 1409 1410#if IMPORTANCE_DEBUG 1411 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_ASSERTION, IMP_EXTERN)) | DBG_FUNC_END, 1412 proc_selfpid(), target_pid, task_imp->iit_assertcnt, IIT_LEGACY_EXTERN(task_imp), 0); 1413 // This is the legacy boosting path 1414 DTRACE_BOOST5(receive_boost, task_t, target_task, int, target_pid, int, sender_pid, int, count, int, IIT_LEGACY_EXTERN(task_imp)); 1415#endif /* IMPORTANCE_DEBUG */ 1416 1417 return(KERN_SUCCESS); 1418} 1419 1420/* 1421 * Routine: ipc_importance_task_update_live_donor 1422 * Purpose: 1423 * Read the live donor status and update the live_donor bit/propagate the change in importance. 1424 * Conditions: 1425 * Nothing locked on entrance, nothing locked on exit. 1426 * 1427 * TODO: Need tracepoints around this function... 1428 */ 1429void 1430ipc_importance_task_update_live_donor(ipc_importance_task_t task_imp) 1431{ 1432 uint32_t task_live_donor; 1433 boolean_t before_donor; 1434 boolean_t after_donor; 1435 task_t target_task; 1436 1437 assert(task_imp != NULL); 1438 1439 /* 1440 * Nothing to do if the task is not marked as expecting 1441 * live donor updates. 1442 */ 1443 if (!ipc_importance_task_is_marked_live_donor(task_imp)) { 1444 return; 1445 } 1446 1447 ipc_importance_lock(); 1448 1449 /* If the task got disconnected on the way here, no use (or ability) adjusting live donor status */ 1450 target_task = task_imp->iit_task; 1451 if (TASK_NULL == target_task) { 1452 ipc_importance_unlock(); 1453 return; 1454 } 1455 before_donor = ipc_importance_task_is_marked_donor(task_imp); 1456 1457 /* snapshot task live donor status - may change, but another call will accompany the change */ 1458 task_live_donor = target_task->effective_policy.t_live_donor; 1459 1460#if IMPORTANCE_DEBUG 1461 int target_pid = audit_token_pid_from_task(target_task); 1462 1463 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 1464 (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_START, 1465 target_pid, task_imp->iit_donor, task_live_donor, before_donor, 0); 1466#endif 1467 1468 /* update the task importance live donor status based on the task's value */ 1469 task_imp->iit_donor = task_live_donor; 1470 1471 after_donor = ipc_importance_task_is_marked_donor(task_imp); 1472 1473 /* Has the effectiveness of being a donor changed as a result of this update? */ 1474 if (before_donor != after_donor) { 1475 iit_update_type_t type; 1476 1477 /* propagate assertions without updating the current task policy (already handled) */ 1478 if (0 == before_donor) { 1479 task_imp->iit_transitions++; 1480 type = IIT_UPDATE_HOLD; 1481 } else { 1482 type = IIT_UPDATE_DROP; 1483 } 1484 ipc_importance_task_propagate_assertion_locked(task_imp, type, FALSE); 1485 } 1486 1487#if IMPORTANCE_DEBUG 1488 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 1489 (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_UPDATE_LIVE_DONOR_STATE)) | DBG_FUNC_END, 1490 target_pid, task_imp->iit_donor, task_live_donor, after_donor, 0); 1491#endif 1492 1493 ipc_importance_unlock(); 1494} 1495 1496 1497/* 1498 * Routine: ipc_importance_task_mark_donor 1499 * Purpose: 1500 * Set the task importance donor flag. 1501 * Conditions: 1502 * Nothing locked on entrance, nothing locked on exit. 1503 * 1504 * This is only called while the task is being constructed, 1505 * so no need to update task policy or propagate downstream. 1506 */ 1507void 1508ipc_importance_task_mark_donor(ipc_importance_task_t task_imp, boolean_t donating) 1509{ 1510 assert(task_imp != NULL); 1511 1512 ipc_importance_lock(); 1513 1514 int old_donor = task_imp->iit_donor; 1515 1516 task_imp->iit_donor = (donating ? 1 : 0); 1517 1518 if (task_imp->iit_donor > 0 && old_donor == 0) 1519 task_imp->iit_transitions++; 1520 1521 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 1522 (IMPORTANCE_CODE(IMP_DONOR_CHANGE, IMP_DONOR_INIT_DONOR_STATE)) | DBG_FUNC_NONE, 1523 audit_token_pid_from_task(task_imp->iit_task), donating, 1524 old_donor, task_imp->iit_donor, 0); 1525 1526 ipc_importance_unlock(); 1527} 1528 1529/* 1530 * Routine: ipc_importance_task_marked_donor 1531 * Purpose: 1532 * Query the donor flag for the given task importance. 1533 * Conditions: 1534 * May be called without taking the importance lock. 1535 * In that case, donor status can change so you must 1536 * check only once for each donation event. 1537 */ 1538boolean_t 1539ipc_importance_task_is_marked_donor(ipc_importance_task_t task_imp) 1540{ 1541 if (IIT_NULL == task_imp) { 1542 return FALSE; 1543 } 1544 return (0 != task_imp->iit_donor); 1545} 1546 1547/* 1548 * Routine: ipc_importance_task_mark_live_donor 1549 * Purpose: 1550 * Indicate that the task is eligible for live donor updates. 1551 * Conditions: 1552 * Nothing locked on entrance, nothing locked on exit. 1553 * 1554 * This is only called while the task is being constructed. 1555 */ 1556void 1557ipc_importance_task_mark_live_donor(ipc_importance_task_t task_imp, boolean_t live_donating) 1558{ 1559 assert(task_imp != NULL); 1560 1561 ipc_importance_lock(); 1562 task_imp->iit_live_donor = (live_donating ? 1 : 0); 1563 ipc_importance_unlock(); 1564} 1565 1566/* 1567 * Routine: ipc_importance_task_marked_live_donor 1568 * Purpose: 1569 * Query the live donor and donor flags for the given task importance. 1570 * Conditions: 1571 * May be called without taking the importance lock. 1572 * In that case, donor status can change so you must 1573 * check only once for each donation event. 1574 */ 1575boolean_t 1576ipc_importance_task_is_marked_live_donor(ipc_importance_task_t task_imp) 1577{ 1578 if (IIT_NULL == task_imp) { 1579 return FALSE; 1580 } 1581 return (0 != task_imp->iit_live_donor); 1582} 1583 1584/* 1585 * Routine: ipc_importance_task_is_donor 1586 * Purpose: 1587 * Query the full donor status for the given task importance. 1588 * Conditions: 1589 * May be called without taking the importance lock. 1590 * In that case, donor status can change so you must 1591 * check only once for each donation event. 1592 */ 1593boolean_t 1594ipc_importance_task_is_donor(ipc_importance_task_t task_imp) 1595{ 1596 if (IIT_NULL == task_imp) { 1597 return FALSE; 1598 } 1599 return (ipc_importance_task_is_marked_donor(task_imp) || 1600 (ipc_importance_task_is_marked_receiver(task_imp) && 1601 task_imp->iit_assertcnt > 0)); 1602} 1603 1604/* 1605 * Routine: ipc_importance_task_is_never_donor 1606 * Purpose: 1607 * Query if a given task can ever donate importance. 1608 * Conditions: 1609 * May be called without taking the importance lock. 1610 * Condition is permanent for a give task. 1611 */ 1612boolean_t 1613ipc_importance_task_is_never_donor(ipc_importance_task_t task_imp) 1614{ 1615 if (IIT_NULL == task_imp) { 1616 return FALSE; 1617 } 1618 return (!ipc_importance_task_is_marked_donor(task_imp) && 1619 !ipc_importance_task_is_marked_live_donor(task_imp) && 1620 !ipc_importance_task_is_marked_receiver(task_imp)); 1621} 1622 1623/* 1624 * Routine: ipc_importance_task_mark_receiver 1625 * Purpose: 1626 * Update the task importance receiver flag. 1627 * Conditions: 1628 * Nothing locked on entrance, nothing locked on exit. 1629 * This can only be invoked before the task is discoverable, 1630 * so no worries about atomicity(?) 1631 */ 1632void 1633ipc_importance_task_mark_receiver(ipc_importance_task_t task_imp, boolean_t receiving) 1634{ 1635 assert(task_imp != NULL); 1636 1637 ipc_importance_lock(); 1638 if (receiving) { 1639 assert(task_imp->iit_assertcnt == 0); 1640 assert(task_imp->iit_externcnt == 0); 1641 assert(task_imp->iit_externdrop == 0); 1642 assert(task_imp->iit_denap == 0); 1643 task_imp->iit_receiver = 1; /* task can receive importance boost */ 1644 } else if (task_imp->iit_receiver) { 1645 assert(task_imp->iit_denap == 0); 1646 if (task_imp->iit_assertcnt != 0 || IIT_EXTERN(task_imp) != 0) { 1647 panic("disabling imp_receiver on task with pending importance boosts!"); 1648 } 1649 task_imp->iit_receiver = 0; 1650 } 1651 ipc_importance_unlock(); 1652} 1653 1654 1655/* 1656 * Routine: ipc_importance_task_marked_receiver 1657 * Purpose: 1658 * Query the receiver flag for the given task importance. 1659 * Conditions: 1660 * May be called without taking the importance lock as 1661 * the importance flag can never change after task init. 1662 */ 1663boolean_t 1664ipc_importance_task_is_marked_receiver(ipc_importance_task_t task_imp) 1665{ 1666 return (IIT_NULL != task_imp && 0 != task_imp->iit_receiver); 1667} 1668 1669 1670/* 1671 * Routine: ipc_importance_task_mark_denap_receiver 1672 * Purpose: 1673 * Update the task importance de-nap receiver flag. 1674 * Conditions: 1675 * Nothing locked on entrance, nothing locked on exit. 1676 * This can only be invoked before the task is discoverable, 1677 * so no worries about atomicity(?) 1678 */ 1679void 1680ipc_importance_task_mark_denap_receiver(ipc_importance_task_t task_imp, boolean_t denap) 1681{ 1682 assert(task_imp != NULL); 1683 1684 ipc_importance_lock(); 1685 if (denap) { 1686 assert(task_imp->iit_assertcnt == 0); 1687 assert(task_imp->iit_externcnt == 0); 1688 assert(task_imp->iit_receiver == 0); 1689 task_imp->iit_denap = 1; /* task can receive de-nap boost */ 1690 } else if (task_imp->iit_denap) { 1691 assert(task_imp->iit_receiver == 0); 1692 if (0 < task_imp->iit_assertcnt || 0 < IIT_EXTERN(task_imp)) { 1693 panic("disabling de-nap on task with pending de-nap boosts!"); 1694 } 1695 task_imp->iit_denap = 0; 1696 } 1697 ipc_importance_unlock(); 1698} 1699 1700 1701/* 1702 * Routine: ipc_importance_task_marked_denap_receiver 1703 * Purpose: 1704 * Query the de-nap receiver flag for the given task importance. 1705 * Conditions: 1706 * May be called without taking the importance lock as 1707 * the de-nap flag can never change after task init. 1708 */ 1709boolean_t 1710ipc_importance_task_is_marked_denap_receiver(ipc_importance_task_t task_imp) 1711{ 1712 return (IIT_NULL != task_imp && 0 != task_imp->iit_denap); 1713} 1714 1715/* 1716 * Routine: ipc_importance_task_is_denap_receiver 1717 * Purpose: 1718 * Query the full de-nap receiver status for the given task importance. 1719 * For now, that is simply whether the receiver flag is set. 1720 * Conditions: 1721 * May be called without taking the importance lock as 1722 * the de-nap receiver flag can never change after task init. 1723 */ 1724boolean_t 1725ipc_importance_task_is_denap_receiver(ipc_importance_task_t task_imp) 1726{ 1727 return (ipc_importance_task_is_marked_denap_receiver(task_imp)); 1728} 1729 1730/* 1731 * Routine: ipc_importance_task_is_any_receiver_type 1732 * Purpose: 1733 * Query if the task is marked to receive boosts - either 1734 * importance or denap. 1735 * Conditions: 1736 * May be called without taking the importance lock as both 1737 * the importance and de-nap receiver flags can never change 1738 * after task init. 1739 */ 1740boolean_t 1741ipc_importance_task_is_any_receiver_type(ipc_importance_task_t task_imp) 1742{ 1743 return (ipc_importance_task_is_marked_receiver(task_imp) || 1744 ipc_importance_task_is_marked_denap_receiver(task_imp)); 1745} 1746 1747#if 0 /* currently unused */ 1748 1749/* 1750 * Routine: ipc_importance_inherit_reference 1751 * Purpose: 1752 * Add a reference to the inherit importance element. 1753 * Conditions: 1754 * Caller most hold a reference on the inherit element. 1755 */ 1756static inline void 1757ipc_importance_inherit_reference(ipc_importance_inherit_t inherit) 1758{ 1759 ipc_importance_reference(&inherit->iii_elem); 1760} 1761#endif /* currently unused */ 1762 1763/* 1764 * Routine: ipc_importance_inherit_release_locked 1765 * Purpose: 1766 * Release a reference on an inherit importance attribute value, 1767 * unlinking and deallocating the attribute if the last reference. 1768 * Conditions: 1769 * Entered with importance lock held, leaves with it unlocked. 1770 */ 1771static inline void 1772ipc_importance_inherit_release_locked(ipc_importance_inherit_t inherit) 1773{ 1774 ipc_importance_release_locked(&inherit->iii_elem); 1775} 1776 1777#if 0 /* currently unused */ 1778/* 1779 * Routine: ipc_importance_inherit_release 1780 * Purpose: 1781 * Release a reference on an inherit importance attribute value, 1782 * unlinking and deallocating the attribute if the last reference. 1783 * Conditions: 1784 * nothing locked on entrance, nothing locked on exit. 1785 * May block. 1786 */ 1787void 1788ipc_importance_inherit_release(ipc_importance_inherit_t inherit) 1789{ 1790 if (III_NULL != inherit) 1791 ipc_importance_release(&inherit->iii_elem); 1792} 1793#endif /* 0 currently unused */ 1794 1795/* 1796 * Routine: ipc_importance_for_task 1797 * Purpose: 1798 * Create a reference for the specified task's base importance 1799 * element. If the base importance element doesn't exist, make it and 1800 * bind it to the active task. If the task is inactive, there isn't 1801 * any need to return a new reference. 1802 * Conditions: 1803 * If made is true, a "made" reference is returned (for donating to 1804 * the voucher system). Otherwise an internal reference is returned. 1805 * 1806 * Nothing locked on entry. May block. 1807 */ 1808ipc_importance_task_t 1809ipc_importance_for_task(task_t task, boolean_t made) 1810{ 1811 ipc_importance_task_t task_elem; 1812 boolean_t first_pass = TRUE; 1813 1814 assert(TASK_NULL != task); 1815 1816 retry: 1817 /* No use returning anything for inactive task */ 1818 if (!task->active) 1819 return IIT_NULL; 1820 1821 ipc_importance_lock(); 1822 task_elem = task->task_imp_base; 1823 if (IIT_NULL != task_elem) { 1824 /* Add a made reference (borrowing active task ref to do it) */ 1825 if (made) { 1826 if (0 == task_elem->iit_made++) { 1827 assert(IIT_REFS_MAX > IIT_REFS(task_elem)); 1828 ipc_importance_task_reference_internal(task_elem); 1829 } 1830 } else { 1831 assert(IIT_REFS_MAX > IIT_REFS(task_elem)); 1832 ipc_importance_task_reference_internal(task_elem); 1833 } 1834 ipc_importance_unlock(); 1835 return task_elem; 1836 } 1837 ipc_importance_unlock(); 1838 1839 if (!first_pass) 1840 return IIT_NULL; 1841 first_pass = FALSE; 1842 1843 /* Need to make one - may race with others (be prepared to drop) */ 1844 task_elem = (ipc_importance_task_t)zalloc(ipc_importance_task_zone); 1845 if (IIT_NULL == task_elem) 1846 goto retry; 1847 1848 task_elem->iit_bits = IIE_TYPE_TASK | 2; /* one for task, one for return/made */ 1849 task_elem->iit_made = (made) ? 1 : 0; 1850 task_elem->iit_task = task; /* take actual ref when we're sure */ 1851 task_elem->iit_updateq = NULL; 1852 task_elem->iit_receiver = 0; 1853 task_elem->iit_denap = 0; 1854 task_elem->iit_donor = 0; 1855 task_elem->iit_live_donor = 0; 1856 task_elem->iit_updatepolicy = 0; 1857 task_elem->iit_reserved = 0; 1858 task_elem->iit_filelocks = 0; 1859 task_elem->iit_updatetime = 0; 1860 task_elem->iit_transitions = 0; 1861 task_elem->iit_assertcnt = 0; 1862 task_elem->iit_externcnt = 0; 1863 task_elem->iit_externdrop = 0; 1864 task_elem->iit_legacy_externcnt = 0; 1865 task_elem->iit_legacy_externdrop = 0; 1866#if IIE_REF_DEBUG 1867 ipc_importance_counter_init(&task_elem->iit_elem); 1868#endif 1869 queue_init(&task_elem->iit_kmsgs); 1870 queue_init(&task_elem->iit_inherits); 1871 1872 ipc_importance_lock(); 1873 if (!task->active) { 1874 ipc_importance_unlock(); 1875 zfree(ipc_importance_task_zone, task_elem); 1876 return IIT_NULL; 1877 } 1878 1879 /* did we lose the race? */ 1880 if (IIT_NULL != task->task_imp_base) { 1881 ipc_importance_unlock(); 1882 zfree(ipc_importance_task_zone, task_elem); 1883 goto retry; 1884 } 1885 1886 /* we won the race */ 1887 task->task_imp_base = task_elem; 1888 task_reference(task); 1889#if DEVELOPMENT || DEBUG 1890 queue_enter(&global_iit_alloc_queue, task_elem, ipc_importance_task_t, iit_allocation); 1891 task_importance_update_owner_info(task); 1892#endif 1893 ipc_importance_unlock(); 1894 1895 return task_elem; 1896} 1897 1898#if DEVELOPMENT || DEBUG 1899void task_importance_update_owner_info(task_t task) { 1900 1901 if (task != TASK_NULL && task->task_imp_base != IIT_NULL) { 1902 ipc_importance_task_t task_elem = task->task_imp_base; 1903 1904 task_elem->iit_bsd_pid = audit_token_pid_from_task(task); 1905 if (task->bsd_info) { 1906 strncpy(&task_elem->iit_procname[0], proc_name_address(task->bsd_info), 16); 1907 task_elem->iit_procname[16] = '\0'; 1908 } else { 1909 strncpy(&task_elem->iit_procname[0], "unknown", 16); 1910 } 1911 } 1912} 1913#endif 1914 1915/* 1916 * Routine: ipc_importance_reset_locked 1917 * Purpose: 1918 * Reset a task's IPC importance (the task is going away or exec'ing) 1919 * 1920 * Remove the donor bit and legacy externalized assertions from the 1921 * current task importance and see if that wipes out downstream donations. 1922 * Conditions: 1923 * importance lock held. 1924 */ 1925 1926static void 1927ipc_importance_reset_locked(ipc_importance_task_t task_imp, boolean_t donor) 1928{ 1929 boolean_t before_donor, after_donor; 1930 1931 /* remove the donor bit, live-donor bit and externalized boosts */ 1932 before_donor = ipc_importance_task_is_donor(task_imp); 1933 if (donor) { 1934 task_imp->iit_donor = 0; 1935 } 1936 assert(IIT_LEGACY_EXTERN(task_imp) <= IIT_EXTERN(task_imp)); 1937 assert(task_imp->iit_legacy_externcnt <= task_imp->iit_externcnt); 1938 assert(task_imp->iit_legacy_externdrop <= task_imp->iit_externdrop); 1939 task_imp->iit_externcnt -= task_imp->iit_legacy_externcnt; 1940 task_imp->iit_externdrop -= task_imp->iit_legacy_externdrop; 1941 1942 /* assert(IIT_LEGACY_EXTERN(task_imp) <= task_imp->iit_assertcnt); */ 1943 if (IIT_LEGACY_EXTERN(task_imp) < task_imp->iit_assertcnt) { 1944 task_imp->iit_assertcnt -= IIT_LEGACY_EXTERN(task_imp); 1945 } else { 1946 assert(IIT_LEGACY_EXTERN(task_imp) == task_imp->iit_assertcnt); 1947 task_imp->iit_assertcnt = 0; 1948 } 1949 task_imp->iit_legacy_externcnt = 0; 1950 task_imp->iit_legacy_externdrop = 0; 1951 after_donor = ipc_importance_task_is_donor(task_imp); 1952 1953#if DEVELOPMENT || DEBUG 1954 if (task_imp->iit_assertcnt > 0 && task_imp->iit_live_donor) { 1955 printf("Live donor task %s[%d] still has %d importance assertions after reset\n", 1956 task_imp->iit_procname, task_imp->iit_bsd_pid, task_imp->iit_assertcnt); 1957 } 1958#endif 1959 1960 /* propagate a downstream drop if there was a change in donor status */ 1961 if (after_donor != before_donor) { 1962 ipc_importance_task_propagate_assertion_locked(task_imp, IIT_UPDATE_DROP, FALSE); 1963 } 1964} 1965 1966/* 1967 * Routine: ipc_importance_reset 1968 * Purpose: 1969 * Reset a task's IPC importance 1970 * 1971 * The task is being reset, although staying around. Arrange to have the 1972 * external state of the task reset from the importance. 1973 * Conditions: 1974 * importance lock not held. 1975 */ 1976 1977void 1978ipc_importance_reset(ipc_importance_task_t task_imp, boolean_t donor) 1979{ 1980 if (IIT_NULL == task_imp) { 1981 return; 1982 } 1983 ipc_importance_lock(); 1984 ipc_importance_reset_locked(task_imp, donor); 1985 ipc_importance_unlock(); 1986} 1987 1988/* 1989 * Routine: ipc_importance_disconnect_task 1990 * Purpose: 1991 * Disconnect a task from its importance. 1992 * 1993 * Clear the task pointer from the importance and drop the 1994 * reference the task held on the importance object. Before 1995 * doing that, reset the effects the current task holds on 1996 * the importance and see if that wipes out downstream donations. 1997 * 1998 * We allow the upstream boosts to continue to affect downstream 1999 * even though the local task is being effectively pulled from 2000 * the chain. 2001 * Conditions: 2002 * Nothing locked. 2003 */ 2004void 2005ipc_importance_disconnect_task(task_t task) 2006{ 2007 ipc_importance_task_t task_imp; 2008 2009 task_lock(task); 2010 ipc_importance_lock(); 2011 task_imp = task->task_imp_base; 2012 2013 /* did somebody beat us to it? */ 2014 if (IIT_NULL == task_imp) { 2015 ipc_importance_unlock(); 2016 task_unlock(task); 2017 return; 2018 } 2019 2020 /* disconnect the task from this importance */ 2021 assert(task_imp->iit_task == task); 2022 task_imp->iit_task = TASK_NULL; 2023 task->task_imp_base = IIT_NULL; 2024 task_unlock(task); 2025 2026 /* reset the effects the current task hold on the importance */ 2027 ipc_importance_reset_locked(task_imp, TRUE); 2028 2029 ipc_importance_task_release_locked(task_imp); 2030 /* importance unlocked */ 2031 2032 /* deallocate the task now that the importance is unlocked */ 2033 task_deallocate(task); 2034} 2035 2036/* 2037 * Routine: ipc_importance_send 2038 * Purpose: 2039 * Post the importance voucher attribute [if sent] or a static 2040 * importance boost depending upon options and conditions. 2041 * Conditions: 2042 * Destination port locked on entry and exit, may be dropped during the call. 2043 * Returns: 2044 * A boolean identifying if the port lock was tempoarily dropped. 2045 */ 2046boolean_t 2047ipc_importance_send( 2048 ipc_kmsg_t kmsg, 2049 mach_msg_option_t option) 2050{ 2051 ipc_port_t port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; 2052 boolean_t port_lock_dropped = FALSE; 2053 ipc_importance_elem_t elem; 2054 task_t task; 2055 ipc_importance_task_t task_imp; 2056 kern_return_t kr; 2057 2058 2059 assert(IP_VALID(port)); 2060 2061 /* If no donation to be made, return quickly */ 2062 if ((port->ip_impdonation == 0) || 2063 (option & MACH_SEND_NOIMPORTANCE) != 0) { 2064 return port_lock_dropped; 2065 } 2066 2067 task = current_task(); 2068 2069 /* If forced sending a static boost, go update the port */ 2070 if ((option & MACH_SEND_IMPORTANCE) != 0) { 2071 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; 2072 goto portupdate; 2073 } 2074 2075 task_imp = task->task_imp_base; 2076 assert(IIT_NULL != task_imp); 2077 2078 /* If the sender can never donate importance, nothing to do */ 2079 if (ipc_importance_task_is_never_donor(task_imp)) { 2080 return port_lock_dropped; 2081 } 2082 2083 elem = IIE_NULL; 2084 2085 /* If importance receiver and passing a voucher, look for importance in there */ 2086 if (IP_VALID(kmsg->ikm_voucher) && 2087 ipc_importance_task_is_marked_receiver(task_imp)) { 2088 mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED]; 2089 mach_voucher_attr_value_handle_array_size_t val_count; 2090 ipc_voucher_t voucher; 2091 2092 assert(ip_kotype(kmsg->ikm_voucher) == IKOT_VOUCHER); 2093 voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject; 2094 2095 /* check to see if the voucher has an importance attribute */ 2096 val_count = MACH_VOUCHER_ATTR_VALUE_MAX_NESTED; 2097 kr = mach_voucher_attr_control_get_values(ipc_importance_control, voucher, 2098 vals, &val_count); 2099 assert(KERN_SUCCESS == kr); 2100 2101 /* 2102 * Only use importance associated with our task (either directly 2103 * or through an inherit that donates to our task). 2104 */ 2105 if (0 < val_count) { 2106 ipc_importance_elem_t check_elem; 2107 2108 check_elem = (ipc_importance_elem_t)vals[0]; 2109 assert(IIE_NULL != check_elem); 2110 if (IIE_TYPE_INHERIT == IIE_TYPE(check_elem)) { 2111 ipc_importance_inherit_t inherit; 2112 inherit = (ipc_importance_inherit_t) check_elem; 2113 if (inherit->iii_to_task == task_imp) { 2114 elem = check_elem; 2115 } 2116 } else if (check_elem == (ipc_importance_elem_t)task_imp) { 2117 elem = check_elem; 2118 } 2119 } 2120 } 2121 2122 /* If we haven't found an importance attribute to send yet, use the task's */ 2123 if (IIE_NULL == elem) { 2124 elem = (ipc_importance_elem_t)task_imp; 2125 } 2126 2127 /* take a reference for the message to hold */ 2128 ipc_importance_reference_internal(elem); 2129 2130 /* acquire the importance lock while trying to hang on to port lock */ 2131 if (!ipc_importance_lock_try()) { 2132 port_lock_dropped = TRUE; 2133 ip_unlock(port); 2134 ipc_importance_lock(); 2135 } 2136 2137 /* link kmsg onto the donor element propagation chain */ 2138 ipc_importance_kmsg_link(kmsg, elem); 2139 /* elem reference transfered to kmsg */ 2140 2141 incr_ref_counter(elem->iie_kmsg_refs_added); 2142 2143 /* If the sender isn't currently a donor, no need to apply boost */ 2144 if (!ipc_importance_task_is_donor(task_imp)) { 2145 ipc_importance_unlock(); 2146 2147 /* re-acquire port lock, if needed */ 2148 if (TRUE == port_lock_dropped) 2149 ip_lock(port); 2150 2151 return port_lock_dropped; 2152 } 2153 2154 /* Mark the fact that we are (currently) donating through this message */ 2155 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; 2156 2157 /* 2158 * If we need to relock the port, do it with the importance still locked. 2159 * This assures we get to add the importance boost through the port to 2160 * the task BEFORE anyone else can attempt to undo that operation because 2161 * the sender lost donor status. 2162 */ 2163 if (TRUE == port_lock_dropped) { 2164 ip_lock(port); 2165 } 2166 ipc_importance_unlock(); 2167 2168 portupdate: 2169 2170#if IMPORTANCE_DEBUG 2171 if (kdebug_enable) { 2172 mach_msg_max_trailer_t *dbgtrailer = (mach_msg_max_trailer_t *) 2173 ((vm_offset_t)kmsg->ikm_header + round_msg(kmsg->ikm_header->msgh_size)); 2174 unsigned int sender_pid = dbgtrailer->msgh_audit.val[5]; 2175 mach_msg_id_t imp_msgh_id = kmsg->ikm_header->msgh_id; 2176 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_START, 2177 audit_token_pid_from_task(task), sender_pid, imp_msgh_id, 0, 0); 2178 } 2179#endif /* IMPORTANCE_DEBUG */ 2180 2181 /* adjust port boost count (with port locked) */ 2182 if (TRUE == ipc_port_importance_delta(port, 1)) { 2183 port_lock_dropped = TRUE; 2184 ip_lock(port); 2185 } 2186 return port_lock_dropped; 2187} 2188 2189/* 2190 * Routine: ipc_importance_inherit_from 2191 * Purpose: 2192 * Create a "made" reference for an importance attribute representing 2193 * an inheritance between the sender of a message (if linked) and the 2194 * current task importance. If the message is not linked, a static 2195 * boost may be created, based on the boost state of the message. 2196 * 2197 * Any transfer from kmsg linkage to inherit linkage must be atomic. 2198 * 2199 * If the task is inactive, there isn't any need to return a new reference. 2200 * Conditions: 2201 * Nothing locked on entry. May block. 2202 */ 2203static ipc_importance_inherit_t 2204ipc_importance_inherit_from(ipc_kmsg_t kmsg) 2205{ 2206 ipc_importance_task_t task_imp = IIT_NULL; 2207 ipc_importance_elem_t from_elem = kmsg->ikm_importance; 2208 ipc_importance_elem_t elem; 2209 task_t task_self = current_task(); 2210 2211 ipc_port_t port = kmsg->ikm_header->msgh_remote_port; 2212 ipc_importance_inherit_t inherit = III_NULL; 2213 ipc_importance_inherit_t alloc = III_NULL; 2214 ipc_importance_inherit_t temp_inherit; 2215 boolean_t cleared_self_donation = FALSE; 2216 boolean_t donating; 2217 uint32_t depth = 1; 2218 2219 /* The kmsg must have an importance donor or static boost to proceed */ 2220 if (IIE_NULL == kmsg->ikm_importance && 2221 !MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) { 2222 return III_NULL; 2223 } 2224 2225 /* 2226 * No need to set up an inherit linkage if the dest isn't a receiver 2227 * of one type or the other. 2228 */ 2229 if (!ipc_importance_task_is_any_receiver_type(task_self->task_imp_base)) { 2230 ipc_importance_lock(); 2231 goto out_locked; 2232 } 2233 2234 /* Grab a reference on the importance of the destination */ 2235 task_imp = ipc_importance_for_task(task_self, FALSE); 2236 2237 ipc_importance_lock(); 2238 2239 if (IIT_NULL == task_imp) { 2240 goto out_locked; 2241 } 2242 2243 incr_ref_counter(task_imp->iit_elem.iie_task_refs_added_inherit_from); 2244 2245 /* If message is already associated with an inherit... */ 2246 if (IIE_TYPE_INHERIT == IIE_TYPE(from_elem)) { 2247 ipc_importance_inherit_t from_inherit = (ipc_importance_inherit_t)from_elem; 2248 2249 /* already targeting our task? - just use it */ 2250 if (from_inherit->iii_to_task == task_imp) { 2251 /* clear self-donation if not also present in inherit */ 2252 if (!from_inherit->iii_donating && 2253 MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) { 2254 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; 2255 cleared_self_donation = TRUE; 2256 } 2257 inherit = from_inherit; 2258 2259 } else if (III_DEPTH_MAX == III_DEPTH(from_inherit)) { 2260 ipc_importance_task_t to_task; 2261 ipc_importance_elem_t unlinked_from; 2262 2263 /* 2264 * Chain too long. Switch to looking 2265 * directly at the from_inherit's to-task 2266 * as our source of importance. 2267 */ 2268 to_task = from_inherit->iii_to_task; 2269 ipc_importance_task_reference(to_task); 2270 from_elem = (ipc_importance_elem_t)to_task; 2271 depth = III_DEPTH_RESET | 1; 2272 2273 /* Fixup the kmsg linkage to reflect change */ 2274 unlinked_from = ipc_importance_kmsg_unlink(kmsg); 2275 assert(unlinked_from == (ipc_importance_elem_t)from_inherit); 2276 ipc_importance_kmsg_link(kmsg, from_elem); 2277 ipc_importance_inherit_release_locked(from_inherit); 2278 /* importance unlocked */ 2279 ipc_importance_lock(); 2280 2281 } else { 2282 /* inheriting from an inherit */ 2283 depth = from_inherit->iii_depth + 1; 2284 } 2285 } 2286 2287 /* 2288 * Don't allow a task to inherit from itself (would keep it permanently 2289 * boosted even if all other donors to the task went away). 2290 */ 2291 2292 if (from_elem == (ipc_importance_elem_t)task_imp) { 2293 goto out_locked; 2294 } 2295 2296 /* 2297 * But if the message isn't associated with any linked source, it is 2298 * intended to be permanently boosting (static boost from kernel). 2299 * In that case DO let the process permanently boost itself. 2300 */ 2301 if (IIE_NULL == from_elem) { 2302 assert(MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)); 2303 ipc_importance_task_reference_internal(task_imp); 2304 from_elem = (ipc_importance_elem_t)task_imp; 2305 } 2306 2307 /* 2308 * Now that we have the from_elem figured out, 2309 * check to see if we already have an inherit for this pairing 2310 */ 2311 while (III_NULL == inherit) { 2312 queue_iterate(&from_elem->iie_inherits, temp_inherit, 2313 ipc_importance_inherit_t, iii_inheritance) { 2314 if (temp_inherit->iii_to_task == task_imp && 2315 temp_inherit->iii_depth == depth) { 2316 inherit = temp_inherit; 2317 break; 2318 } 2319 } 2320 2321 /* Do we have to allocate a new inherit */ 2322 if (III_NULL == inherit) { 2323 if (III_NULL != alloc) { 2324 break; 2325 } 2326 2327 /* allocate space */ 2328 ipc_importance_unlock(); 2329 alloc = (ipc_importance_inherit_t) 2330 zalloc(ipc_importance_inherit_zone); 2331 ipc_importance_lock(); 2332 } 2333 } 2334 2335 /* snapshot the donating status while we have importance locked */ 2336 donating = MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits); 2337 2338 if (III_NULL != inherit) { 2339 /* We found one, piggyback on that */ 2340 assert(0 < III_REFS(inherit)); 2341 assert(0 < IIE_REFS(inherit->iii_from_elem)); 2342 assert(inherit->iii_externcnt >= inherit->iii_made); 2343 2344 /* add in a made reference */ 2345 if (0 == inherit->iii_made++) { 2346 assert(III_REFS_MAX > III_REFS(inherit)); 2347 ipc_importance_inherit_reference_internal(inherit); 2348 } 2349 2350 /* Reflect the inherit's change of status into the task boosts */ 2351 if (0 == III_EXTERN(inherit)) { 2352 assert(!inherit->iii_donating); 2353 inherit->iii_donating = donating; 2354 if (donating) { 2355 task_imp->iit_externcnt += inherit->iii_externcnt; 2356 task_imp->iit_externdrop += inherit->iii_externdrop; 2357 } 2358 } else { 2359 assert(donating == inherit->iii_donating); 2360 } 2361 2362 /* add in a external reference for this use of the inherit */ 2363 inherit->iii_externcnt++; 2364 if (donating) { 2365 task_imp->iit_externcnt++; 2366 } 2367 } else { 2368 /* initialize the previously allocated space */ 2369 inherit = alloc; 2370 inherit->iii_bits = IIE_TYPE_INHERIT | 1; 2371 inherit->iii_made = 1; 2372 inherit->iii_externcnt = 1; 2373 inherit->iii_externdrop = 0; 2374 inherit->iii_depth = depth; 2375 inherit->iii_to_task = task_imp; 2376 inherit->iii_from_elem = IIE_NULL; 2377 queue_init(&inherit->iii_kmsgs); 2378 queue_init(&inherit->iii_inherits); 2379 2380 /* If donating, reflect that in the task externcnt */ 2381 if (donating) { 2382 inherit->iii_donating = TRUE; 2383 task_imp->iit_externcnt++; 2384 } else { 2385 inherit->iii_donating = FALSE; 2386 } 2387 2388 /* 2389 * Chain our new inherit on the element it inherits from. 2390 * The new inherit takes our reference on from_elem. 2391 */ 2392 ipc_importance_inherit_link(inherit, from_elem); 2393 2394#if IIE_REF_DEBUG 2395 ipc_importance_counter_init(&inherit->iii_elem); 2396 from_elem->iie_kmsg_refs_inherited++; 2397 task_imp->iit_elem.iie_task_refs_inherited++; 2398#endif 2399 } 2400 2401 out_locked: 2402 /* 2403 * for those paths that came straight here: snapshot the donating status 2404 * (this should match previous snapshot for other paths). 2405 */ 2406 donating = MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits); 2407 2408 /* unlink the kmsg inheritance (if any) */ 2409 elem = ipc_importance_kmsg_unlink(kmsg); 2410 assert(elem == from_elem); 2411 2412 /* If we didn't create a new inherit, we have some resources to release */ 2413 if (III_NULL == inherit || inherit != alloc) { 2414 if (IIE_NULL != from_elem) { 2415 if (III_NULL != inherit) { 2416 incr_ref_counter(from_elem->iie_kmsg_refs_coalesced); 2417 } else { 2418 incr_ref_counter(from_elem->iie_kmsg_refs_dropped); 2419 } 2420 ipc_importance_release_locked(from_elem); 2421 /* importance unlocked */ 2422 } else { 2423 ipc_importance_unlock(); 2424 } 2425 2426 if (IIT_NULL != task_imp) { 2427 if (III_NULL != inherit) { 2428 incr_ref_counter(task_imp->iit_elem.iie_task_refs_coalesced); 2429 } 2430 ipc_importance_task_release(task_imp); 2431 } 2432 2433 if (III_NULL != alloc) 2434 zfree(ipc_importance_inherit_zone, alloc); 2435 } else { 2436 /* from_elem and task_imp references transferred to new inherit */ 2437 ipc_importance_unlock(); 2438 } 2439 2440 /* decrement port boost count */ 2441 if (donating) { 2442 ip_lock(port); 2443 if (III_NULL != inherit) { 2444 /* task assertions transferred to inherit, just adjust port count */ 2445 ipc_port_impcount_delta(port, -1, IP_NULL); 2446 ip_unlock(port); 2447 } else { 2448 /* drop importance from port and destination task */ 2449 if (ipc_port_importance_delta(port, -1) == FALSE) { 2450 ip_unlock(port); 2451 } 2452 } 2453 } else if (cleared_self_donation) { 2454 ip_lock(port); 2455 /* drop cleared donation from port and destination task */ 2456 if (ipc_port_importance_delta(port, -1) == FALSE) { 2457 ip_unlock(port); 2458 } 2459 } 2460 2461 if (III_NULL != inherit) { 2462 /* have an associated importance attr, even if currently not donating */ 2463 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_RAISEIMP; 2464 } else { 2465 /* we won't have an importance attribute associated with our message */ 2466 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; 2467 } 2468 2469 return inherit; 2470} 2471 2472/* 2473 * Routine: ipc_importance_receive 2474 * Purpose: 2475 * Process importance attributes in a received message. 2476 * 2477 * If an importance voucher attribute was sent, transform 2478 * that into an attribute value reflecting the inheritance 2479 * from the sender to the receiver. 2480 * 2481 * If a static boost is received (or the voucher isn't on 2482 * a voucher-based boost), export a static boost. 2483 * Conditions: 2484 * Nothing locked. 2485 */ 2486void 2487ipc_importance_receive( 2488 ipc_kmsg_t kmsg, 2489 mach_msg_option_t option) 2490{ 2491 unsigned int sender_pid = ((mach_msg_max_trailer_t *) 2492 ((vm_offset_t)kmsg->ikm_header + 2493 round_msg(kmsg->ikm_header->msgh_size)))->msgh_audit.val[5]; 2494 task_t task_self = current_task(); 2495 int impresult = -1; 2496 2497 /* convert to a voucher with an inherit importance attribute? */ 2498 if ((option & MACH_RCV_VOUCHER) != 0) { 2499 uint8_t recipes[2 * sizeof(ipc_voucher_attr_recipe_data_t) + 2500 sizeof(mach_voucher_attr_value_handle_t)]; 2501 ipc_voucher_attr_raw_recipe_array_size_t recipe_size = 0; 2502 ipc_voucher_attr_recipe_t recipe = (ipc_voucher_attr_recipe_t)recipes; 2503 ipc_voucher_t recv_voucher; 2504 mach_voucher_attr_value_handle_t handle; 2505 ipc_importance_inherit_t inherit; 2506 kern_return_t kr; 2507 2508 /* set up recipe to copy the old voucher */ 2509 if (IP_VALID(kmsg->ikm_voucher)) { 2510 ipc_voucher_t sent_voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject; 2511 2512 recipe->key = MACH_VOUCHER_ATTR_KEY_ALL; 2513 recipe->command = MACH_VOUCHER_ATTR_COPY; 2514 recipe->previous_voucher = sent_voucher; 2515 recipe->content_size = 0; 2516 recipe_size += sizeof(*recipe); 2517 } 2518 2519 /* 2520 * create an inheritance attribute from the kmsg (may be NULL) 2521 * transferring any boosts from the kmsg linkage through the 2522 * port directly to the new inheritance object. 2523 */ 2524 inherit = ipc_importance_inherit_from(kmsg); 2525 handle = (mach_voucher_attr_value_handle_t)inherit; 2526 2527 assert(IIE_NULL == kmsg->ikm_importance); 2528 2529 /* replace the importance attribute with the handle we created */ 2530 /* our made reference on the inhert is donated to the voucher */ 2531 recipe = (ipc_voucher_attr_recipe_t)&recipes[recipe_size]; 2532 recipe->key = MACH_VOUCHER_ATTR_KEY_IMPORTANCE; 2533 recipe->command = MACH_VOUCHER_ATTR_SET_VALUE_HANDLE; 2534 recipe->previous_voucher = IPC_VOUCHER_NULL; 2535 recipe->content_size = sizeof(mach_voucher_attr_value_handle_t); 2536 *(mach_voucher_attr_value_handle_t *)(void *)recipe->content = handle; 2537 recipe_size += sizeof(*recipe) + sizeof(mach_voucher_attr_value_handle_t); 2538 2539 kr = ipc_voucher_attr_control_create_mach_voucher(ipc_importance_control, 2540 recipes, 2541 recipe_size, 2542 &recv_voucher); 2543 assert(KERN_SUCCESS == kr); 2544 2545 /* swap the voucher port (and set voucher bits in case it didn't already exist) */ 2546 kmsg->ikm_header->msgh_bits |= (MACH_MSG_TYPE_MOVE_SEND << 16); 2547 ipc_port_release_send(kmsg->ikm_voucher); 2548 kmsg->ikm_voucher = convert_voucher_to_port(recv_voucher); 2549 if (III_NULL != inherit) 2550 impresult = 2; 2551 2552 } else { /* Don't want a voucher */ 2553 2554 /* got linked importance? have to drop */ 2555 if (IIE_NULL != kmsg->ikm_importance) { 2556 ipc_importance_elem_t elem; 2557 2558 ipc_importance_lock(); 2559 elem = ipc_importance_kmsg_unlink(kmsg); 2560#if IIE_REF_DEBUG 2561 elem->iie_kmsg_refs_dropped++; 2562#endif 2563 ipc_importance_release_locked(elem); 2564 /* importance unlocked */ 2565 } 2566 2567 /* With kmsg unlinked, can safely examine message importance attribute. */ 2568 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) { 2569 ipc_importance_task_t task_imp = task_self->task_imp_base; 2570 ipc_port_t port = kmsg->ikm_header->msgh_remote_port; 2571 2572 /* defensive deduction for release builds lacking the assert */ 2573 ip_lock(port); 2574 ipc_port_impcount_delta(port, -1, IP_NULL); 2575 ip_unlock(port); 2576 2577 /* will user accept legacy responsibility for the importance boost */ 2578 if (KERN_SUCCESS == ipc_importance_task_externalize_legacy_assertion(task_imp, 1, sender_pid)) { 2579 impresult = 1; 2580 } else { 2581 /* The importance boost never applied to task (clear the bit) */ 2582 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; 2583 impresult = 0; 2584 } 2585 } 2586 } 2587 2588#if IMPORTANCE_DEBUG 2589 if (-1 < impresult) 2590 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_DELV)) | DBG_FUNC_NONE, 2591 sender_pid, audit_token_pid_from_task(task_self), 2592 kmsg->ikm_header->msgh_id, impresult, 0); 2593 if (impresult == 2){ 2594 /* 2595 * This probe only covers new voucher-based path. Legacy importance 2596 * will trigger the probe in ipc_importance_task_externalize_assertion() 2597 * above and have impresult==1 here. 2598 */ 2599 DTRACE_BOOST5(receive_boost, task_t, task_self, int, audit_token_pid_from_task(task_self), int, sender_pid, int, 1, int, task_self->task_imp_base->iit_assertcnt); 2600 } 2601#endif /* IMPORTANCE_DEBUG */ 2602} 2603 2604/* 2605 * Routine: ipc_importance_unreceive 2606 * Purpose: 2607 * Undo receive of importance attributes in a message. 2608 * 2609 * Conditions: 2610 * Nothing locked. 2611 */ 2612void 2613ipc_importance_unreceive( 2614 ipc_kmsg_t kmsg, 2615 mach_msg_option_t __unused option) 2616{ 2617 /* importance should already be in the voucher and out of the kmsg */ 2618 assert(IIE_NULL == kmsg->ikm_importance); 2619 2620 /* See if there is a legacy boost to be dropped from receiver */ 2621 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) { 2622 ipc_importance_task_t task_imp; 2623 2624 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; 2625 task_imp = current_task()->task_imp_base; 2626 if (!IP_VALID(kmsg->ikm_voucher) && IIT_NULL != task_imp) { 2627 ipc_importance_task_drop_legacy_external_assertion(task_imp, 1); 2628 } 2629 /* 2630 * ipc_kmsg_copyout_dest() will consume the voucher 2631 * and any contained importance. 2632 */ 2633 } 2634} 2635 2636/* 2637 * Routine: ipc_importance_clean 2638 * Purpose: 2639 * Clean up importance state in a kmsg that is being cleaned. 2640 * Unlink the importance chain if one was set up, and drop 2641 * the reference this kmsg held on the donor. Then check to 2642 * if importance was carried to the port, and remove that if 2643 * needed. 2644 * Conditions: 2645 * Nothing locked. 2646 */ 2647void 2648ipc_importance_clean( 2649 ipc_kmsg_t kmsg) 2650{ 2651 ipc_port_t port; 2652 2653 /* Is the kmsg still linked? If so, remove that first */ 2654 if (IIE_NULL != kmsg->ikm_importance) { 2655 ipc_importance_elem_t elem; 2656 2657 ipc_importance_lock(); 2658 elem = ipc_importance_kmsg_unlink(kmsg); 2659 assert(IIE_NULL != elem); 2660 ipc_importance_release_locked(elem); 2661 /* importance unlocked */ 2662 } 2663 2664 /* See if there is a legacy importance boost to be dropped from port */ 2665 if (MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)) { 2666 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_RAISEIMP; 2667 port = kmsg->ikm_header->msgh_remote_port; 2668 if (IP_VALID(port)) { 2669 ip_lock(port); 2670 /* inactive ports already had their importance boosts dropped */ 2671 if (!ip_active(port) || 2672 ipc_port_importance_delta(port, -1) == FALSE) { 2673 ip_unlock(port); 2674 } 2675 } 2676 } 2677} 2678 2679void 2680ipc_importance_assert_clean(__assert_only ipc_kmsg_t kmsg) 2681{ 2682 assert(IIE_NULL == kmsg->ikm_importance); 2683 assert(!MACH_MSGH_BITS_RAISED_IMPORTANCE(kmsg->ikm_header->msgh_bits)); 2684} 2685 2686/* 2687 * IPC Importance Attribute Manager definition 2688 */ 2689 2690static kern_return_t 2691ipc_importance_release_value( 2692 ipc_voucher_attr_manager_t manager, 2693 mach_voucher_attr_key_t key, 2694 mach_voucher_attr_value_handle_t value, 2695 mach_voucher_attr_value_reference_t sync); 2696 2697static kern_return_t 2698ipc_importance_get_value( 2699 ipc_voucher_attr_manager_t manager, 2700 mach_voucher_attr_key_t key, 2701 mach_voucher_attr_recipe_command_t command, 2702 mach_voucher_attr_value_handle_array_t prev_values, 2703 mach_voucher_attr_value_handle_array_size_t prev_value_count, 2704 mach_voucher_attr_content_t content, 2705 mach_voucher_attr_content_size_t content_size, 2706 mach_voucher_attr_value_handle_t *out_value, 2707 ipc_voucher_t *out_value_voucher); 2708 2709static kern_return_t 2710ipc_importance_extract_content( 2711 ipc_voucher_attr_manager_t manager, 2712 mach_voucher_attr_key_t key, 2713 mach_voucher_attr_value_handle_array_t values, 2714 mach_voucher_attr_value_handle_array_size_t value_count, 2715 mach_voucher_attr_recipe_command_t *out_command, 2716 mach_voucher_attr_content_t out_content, 2717 mach_voucher_attr_content_size_t *in_out_content_size); 2718 2719static kern_return_t 2720ipc_importance_command( 2721 ipc_voucher_attr_manager_t manager, 2722 mach_voucher_attr_key_t key, 2723 mach_voucher_attr_value_handle_array_t values, 2724 mach_msg_type_number_t value_count, 2725 mach_voucher_attr_command_t command, 2726 mach_voucher_attr_content_t in_content, 2727 mach_voucher_attr_content_size_t in_content_size, 2728 mach_voucher_attr_content_t out_content, 2729 mach_voucher_attr_content_size_t *out_content_size); 2730 2731static void 2732ipc_importance_manager_release( 2733 ipc_voucher_attr_manager_t manager); 2734 2735struct ipc_voucher_attr_manager ipc_importance_manager = { 2736 .ivam_release_value = ipc_importance_release_value, 2737 .ivam_get_value = ipc_importance_get_value, 2738 .ivam_extract_content = ipc_importance_extract_content, 2739 .ivam_command = ipc_importance_command, 2740 .ivam_release = ipc_importance_manager_release, 2741}; 2742 2743#define IMPORTANCE_ASSERT_KEY(key) assert(MACH_VOUCHER_ATTR_KEY_IMPORTANCE == (key)) 2744#define IMPORTANCE_ASSERT_MANAGER(manager) assert(&ipc_importance_manager == (manager)) 2745 2746/* 2747 * Routine: ipc_importance_release_value [Voucher Attribute Manager Interface] 2748 * Purpose: 2749 * Release what the voucher system believes is the last "made" reference 2750 * on an importance attribute value handle. The sync parameter is used to 2751 * avoid races with new made references concurrently being returned to the 2752 * voucher system in other threads. 2753 * Conditions: 2754 * Nothing locked on entry. May block. 2755 */ 2756static kern_return_t 2757ipc_importance_release_value( 2758 ipc_voucher_attr_manager_t __assert_only manager, 2759 mach_voucher_attr_key_t __assert_only key, 2760 mach_voucher_attr_value_handle_t value, 2761 mach_voucher_attr_value_reference_t sync) 2762{ 2763 ipc_importance_elem_t elem; 2764 2765 IMPORTANCE_ASSERT_MANAGER(manager); 2766 IMPORTANCE_ASSERT_KEY(key); 2767 assert(0 < sync); 2768 2769 elem = (ipc_importance_elem_t)value; 2770 2771 ipc_importance_lock(); 2772 2773 /* Any oustanding made refs? */ 2774 if (sync != elem->iie_made) { 2775 assert(sync < elem->iie_made); 2776 ipc_importance_unlock(); 2777 return KERN_FAILURE; 2778 } 2779 2780 /* clear made */ 2781 elem->iie_made = 0; 2782 2783 /* 2784 * If there are pending external boosts represented by this attribute, 2785 * drop them from the apropriate task 2786 */ 2787 if (IIE_TYPE_INHERIT == IIE_TYPE(elem)) { 2788 ipc_importance_inherit_t inherit = (ipc_importance_inherit_t)elem; 2789 2790 assert(inherit->iii_externcnt >= inherit->iii_externdrop); 2791 2792 if (inherit->iii_donating) { 2793 ipc_importance_task_t imp_task = inherit->iii_to_task; 2794 uint32_t assertcnt = III_EXTERN(inherit); 2795 2796 assert(ipc_importance_task_is_any_receiver_type(imp_task)); 2797 assert(imp_task->iit_externcnt >= inherit->iii_externcnt); 2798 assert(imp_task->iit_externdrop >= inherit->iii_externdrop); 2799 imp_task->iit_externcnt -= inherit->iii_externcnt; 2800 imp_task->iit_externdrop -= inherit->iii_externdrop; 2801 inherit->iii_externcnt = 0; 2802 inherit->iii_externdrop = 0; 2803 inherit->iii_donating = FALSE; 2804 2805 /* adjust the internal assertions - and propagate if needed */ 2806 if (ipc_importance_task_check_transition(imp_task, IIT_UPDATE_DROP, assertcnt)) { 2807 ipc_importance_task_propagate_assertion_locked(imp_task, IIT_UPDATE_DROP, TRUE); 2808 } 2809 } else { 2810 inherit->iii_externcnt = 0; 2811 inherit->iii_externdrop = 0; 2812 } 2813 } 2814 2815 /* drop the made reference on elem */ 2816 ipc_importance_release_locked(elem); 2817 /* returns unlocked */ 2818 2819 return KERN_SUCCESS; 2820} 2821 2822 2823/* 2824 * Routine: ipc_importance_get_value [Voucher Attribute Manager Interface] 2825 * Purpose: 2826 * Convert command and content data into a reference on a [potentially new] 2827 * attribute value. The importance attribute manager will only allow the 2828 * caller to get a value for the current task's importance, or to redeem 2829 * an importance attribute from an existing voucher. 2830 * Conditions: 2831 * Nothing locked on entry. May block. 2832 */ 2833static kern_return_t 2834ipc_importance_get_value( 2835 ipc_voucher_attr_manager_t __assert_only manager, 2836 mach_voucher_attr_key_t __assert_only key, 2837 mach_voucher_attr_recipe_command_t command, 2838 mach_voucher_attr_value_handle_array_t prev_values, 2839 mach_voucher_attr_value_handle_array_size_t prev_value_count, 2840 mach_voucher_attr_content_t __unused content, 2841 mach_voucher_attr_content_size_t content_size, 2842 mach_voucher_attr_value_handle_t *out_value, 2843 ipc_voucher_t *out_value_voucher) 2844{ 2845 ipc_importance_elem_t elem; 2846 task_t self; 2847 2848 IMPORTANCE_ASSERT_MANAGER(manager); 2849 IMPORTANCE_ASSERT_KEY(key); 2850 2851 if (0 != content_size) 2852 return KERN_INVALID_ARGUMENT; 2853 2854 /* never an out voucher */ 2855 2856 switch (command) { 2857 2858 case MACH_VOUCHER_ATTR_REDEEM: 2859 2860 /* redeem of previous values is the value */ 2861 if (0 < prev_value_count) { 2862 elem = (ipc_importance_elem_t)prev_values[0]; 2863 assert(IIE_NULL != elem); 2864 2865 ipc_importance_lock(); 2866 assert(0 < elem->iie_made); 2867 elem->iie_made++; 2868 ipc_importance_unlock(); 2869 2870 *out_value = prev_values[0]; 2871 return KERN_SUCCESS; 2872 } 2873 2874 /* redeem of default is default */ 2875 *out_value = 0; 2876 *out_value_voucher = IPC_VOUCHER_NULL; 2877 return KERN_SUCCESS; 2878 2879 case MACH_VOUCHER_ATTR_IMPORTANCE_SELF: 2880 self = current_task(); 2881 2882 elem = (ipc_importance_elem_t)ipc_importance_for_task(self, TRUE); 2883 /* made reference added (or IIE_NULL which isn't referenced) */ 2884 2885 *out_value = (mach_voucher_attr_value_handle_t)elem; 2886 *out_value_voucher = IPC_VOUCHER_NULL; 2887 return KERN_SUCCESS; 2888 2889 default: 2890 /* 2891 * every other command is unknown 2892 * 2893 * Specifically, there is no mechanism provided to construct an 2894 * importance attribute for a task/process from just a pid or 2895 * task port. It has to be copied (or redeemed) from a previous 2896 * voucher that has it. 2897 */ 2898 return KERN_INVALID_ARGUMENT; 2899 } 2900} 2901 2902/* 2903 * Routine: ipc_importance_extract_content [Voucher Attribute Manager Interface] 2904 * Purpose: 2905 * Extract meaning from the attribute value present in a voucher. While 2906 * the real goal is to provide commands and data that can reproduce the 2907 * voucher's value "out of thin air", this isn't possible with importance 2908 * attribute values. Instead, return debug info to help track down dependencies. 2909 * Conditions: 2910 * Nothing locked on entry. May block. 2911 */ 2912static kern_return_t 2913ipc_importance_extract_content( 2914 ipc_voucher_attr_manager_t __assert_only manager, 2915 mach_voucher_attr_key_t __assert_only key, 2916 mach_voucher_attr_value_handle_array_t values, 2917 mach_voucher_attr_value_handle_array_size_t value_count, 2918 mach_voucher_attr_recipe_command_t *out_command, 2919 mach_voucher_attr_content_t out_content, 2920 mach_voucher_attr_content_size_t *in_out_content_size) 2921{ 2922 mach_voucher_attr_content_size_t size = 0; 2923 ipc_importance_elem_t elem; 2924 unsigned int i; 2925 2926 IMPORTANCE_ASSERT_MANAGER(manager); 2927 IMPORTANCE_ASSERT_KEY(key); 2928 2929 /* the first non-default value provides the data */ 2930 for (i = 0; i < value_count ; i++) { 2931 elem = (ipc_importance_elem_t)values[i]; 2932 if (IIE_NULL == elem) 2933 continue; 2934 2935 snprintf((char *)out_content, *in_out_content_size, "Importance for pid "); 2936 size = (mach_voucher_attr_content_size_t)strlen((char *)out_content); 2937 2938 for(;;) { 2939 ipc_importance_inherit_t inherit = III_NULL; 2940 ipc_importance_task_t task_imp; 2941 task_t task; 2942 int task_pid; 2943 2944 if (IIE_TYPE_TASK == IIE_TYPE(elem)) { 2945 task_imp = (ipc_importance_task_t)elem; 2946 task = task_imp->iit_task; 2947 task_pid = (TASK_NULL != task) ? 2948 audit_token_pid_from_task(task) : -1; 2949 snprintf((char *)out_content + size, *in_out_content_size - size, "%d", task_pid); 2950 } else { 2951 inherit = (ipc_importance_inherit_t)elem; 2952 task_imp = inherit->iii_to_task; 2953 task = task_imp->iit_task; 2954 task_pid = (TASK_NULL != task) ? 2955 audit_token_pid_from_task(task) : -1; 2956 snprintf((char *)out_content + size, *in_out_content_size - size, 2957 "%d (%d of %d boosts) %s from pid ", task_pid, 2958 III_EXTERN(inherit), inherit->iii_externcnt, 2959 (inherit->iii_donating) ? "donated" : "linked"); 2960 } 2961 2962 size = (mach_voucher_attr_content_size_t)strlen((char *)out_content); 2963 2964 if (III_NULL == inherit) 2965 break; 2966 2967 elem = inherit->iii_from_elem; 2968 } 2969 size++; /* account for NULL */ 2970 } 2971 *out_command = MACH_VOUCHER_ATTR_NOOP; /* cannot be used to regenerate value */ 2972 *in_out_content_size = size; 2973 return KERN_SUCCESS; 2974} 2975 2976/* 2977 * Routine: ipc_importance_command [Voucher Attribute Manager Interface] 2978 * Purpose: 2979 * Run commands against the importance attribute value found in a voucher. 2980 * No such commands are currently supported. 2981 * Conditions: 2982 * Nothing locked on entry. May block. 2983 */ 2984static kern_return_t 2985ipc_importance_command( 2986 ipc_voucher_attr_manager_t __assert_only manager, 2987 mach_voucher_attr_key_t __assert_only key, 2988 mach_voucher_attr_value_handle_array_t values, 2989 mach_msg_type_number_t value_count, 2990 mach_voucher_attr_command_t command, 2991 mach_voucher_attr_content_t in_content, 2992 mach_voucher_attr_content_size_t in_content_size, 2993 mach_voucher_attr_content_t out_content, 2994 mach_voucher_attr_content_size_t *out_content_size) 2995{ 2996 ipc_importance_inherit_t inherit; 2997 ipc_importance_task_t to_task; 2998 uint32_t refs, *outrefsp; 2999 mach_msg_type_number_t i; 3000 uint32_t externcnt; 3001 3002 IMPORTANCE_ASSERT_MANAGER(manager); 3003 IMPORTANCE_ASSERT_KEY(key); 3004 3005 if (in_content_size != sizeof(refs) || 3006 (*out_content_size != 0 && *out_content_size != sizeof(refs))) { 3007 return KERN_INVALID_ARGUMENT; 3008 } 3009 refs = *(uint32_t *)(void *)in_content; 3010 outrefsp = (*out_content_size != 0) ? (uint32_t *)(void *)out_content : NULL; 3011 3012 if (MACH_VOUCHER_IMPORTANCE_ATTR_DROP_EXTERNAL != command) { 3013 return KERN_NOT_SUPPORTED; 3014 } 3015 3016 /* the first non-default value of the apropos type provides the data */ 3017 inherit = III_NULL; 3018 for (i = 0; i < value_count; i++) { 3019 ipc_importance_elem_t elem = (ipc_importance_elem_t)values[i]; 3020 3021 if (IIE_NULL != elem && IIE_TYPE_INHERIT == IIE_TYPE(elem)) { 3022 inherit = (ipc_importance_inherit_t)elem; 3023 break; 3024 } 3025 } 3026 if (III_NULL == inherit) { 3027 return KERN_INVALID_ARGUMENT; 3028 } 3029 3030 ipc_importance_lock(); 3031 3032 if (0 == refs) { 3033 if (NULL != outrefsp) { 3034 *outrefsp = III_EXTERN(inherit); 3035 } 3036 ipc_importance_unlock(); 3037 return KERN_SUCCESS; 3038 } 3039 3040 /* Enough external references left to drop? */ 3041 if (III_EXTERN(inherit) < refs) { 3042 ipc_importance_unlock(); 3043 return KERN_FAILURE; 3044 } 3045 3046 to_task = inherit->iii_to_task; 3047 assert(ipc_importance_task_is_any_receiver_type(to_task)); 3048 3049 /* re-base external and internal counters at the inherit and the to-task (if apropos) */ 3050 if (inherit->iii_donating) { 3051 assert(IIT_EXTERN(to_task) >= III_EXTERN(inherit)); 3052 assert(to_task->iit_externcnt >= inherit->iii_externcnt); 3053 assert(to_task->iit_externdrop >= inherit->iii_externdrop); 3054 inherit->iii_externdrop += refs; 3055 to_task->iit_externdrop += refs; 3056 externcnt = III_EXTERN(inherit); 3057 if (0 == externcnt) { 3058 inherit->iii_donating = FALSE; 3059 to_task->iit_externcnt -= inherit->iii_externcnt; 3060 to_task->iit_externdrop -= inherit->iii_externdrop; 3061 3062 3063 /* Start AppNap delay hysteresis - even if not the last boost for the task. */ 3064 if (ipc_importance_delayed_drop_call != NULL && 3065 ipc_importance_task_is_marked_denap_receiver(to_task)) { 3066 ipc_importance_task_delayed_drop(to_task); 3067 } 3068 3069 /* drop task assertions associated with the dropped boosts */ 3070 if (ipc_importance_task_check_transition(to_task, IIT_UPDATE_DROP, refs)) { 3071 ipc_importance_task_propagate_assertion_locked(to_task, IIT_UPDATE_DROP, TRUE); 3072 /* may have dropped and retaken importance lock */ 3073 } 3074 } else { 3075 /* assert(to_task->iit_assertcnt >= refs + externcnt); */ 3076 /* defensive deduction in case of assertcnt underflow */ 3077 if (to_task->iit_assertcnt > refs + externcnt) { 3078 to_task->iit_assertcnt -= refs; 3079 } else { 3080 to_task->iit_assertcnt = externcnt; 3081 } 3082 } 3083 } else { 3084 inherit->iii_externdrop += refs; 3085 externcnt = III_EXTERN(inherit); 3086 } 3087 3088 /* capture result (if requested) */ 3089 if (NULL != outrefsp) { 3090 *outrefsp = externcnt; 3091 } 3092 3093 ipc_importance_unlock(); 3094 return KERN_SUCCESS; 3095} 3096 3097/* 3098 * Routine: ipc_importance_manager_release [Voucher Attribute Manager Interface] 3099 * Purpose: 3100 * Release the Voucher system's reference on the IPC importance attribute 3101 * manager. 3102 * Conditions: 3103 * As this can only occur after the manager drops the Attribute control 3104 * reference granted back at registration time, and that reference is never 3105 * dropped, this should never be called. 3106 */ 3107static void 3108ipc_importance_manager_release( 3109 ipc_voucher_attr_manager_t __assert_only manager) 3110{ 3111 IMPORTANCE_ASSERT_MANAGER(manager); 3112 panic("Voucher importance manager released"); 3113} 3114 3115/* 3116 * Routine: ipc_importance_init 3117 * Purpose: 3118 * Initialize the IPC importance manager. 3119 * Conditions: 3120 * Zones and Vouchers are already initialized. 3121 */ 3122void 3123ipc_importance_init(void) 3124{ 3125 natural_t ipc_importance_max = (task_max + thread_max) * 2; 3126 char temp_buf[26]; 3127 kern_return_t kr; 3128 3129 if (PE_parse_boot_argn("imp_interactive_receiver", temp_buf, sizeof(temp_buf))) { 3130 ipc_importance_interactive_receiver = TRUE; 3131 } 3132 3133 ipc_importance_task_zone = zinit(sizeof(struct ipc_importance_task), 3134 ipc_importance_max * sizeof(struct ipc_importance_task), 3135 sizeof(struct ipc_importance_task), 3136 "ipc task importance"); 3137 zone_change(ipc_importance_task_zone, Z_NOENCRYPT, TRUE); 3138 3139 ipc_importance_inherit_zone = zinit(sizeof(struct ipc_importance_inherit), 3140 ipc_importance_max * sizeof(struct ipc_importance_inherit), 3141 sizeof(struct ipc_importance_inherit), 3142 "ipc importance inherit"); 3143 zone_change(ipc_importance_inherit_zone, Z_NOENCRYPT, TRUE); 3144 3145 3146#if DEVELOPMENT || DEBUG 3147 queue_init(&global_iit_alloc_queue); 3148#endif 3149 3150 /* initialize global locking */ 3151 ipc_importance_lock_init(); 3152 3153 kr = ipc_register_well_known_mach_voucher_attr_manager(&ipc_importance_manager, 3154 (mach_voucher_attr_value_handle_t)0, 3155 MACH_VOUCHER_ATTR_KEY_IMPORTANCE, 3156 &ipc_importance_control); 3157 if (KERN_SUCCESS != kr) 3158 printf("Voucher importance manager register returned %d", kr); 3159} 3160 3161/* 3162 * Routine: ipc_importance_thread_call_init 3163 * Purpose: 3164 * Initialize the IPC importance code dependent upon 3165 * thread-call support being available. 3166 * Conditions: 3167 * Thread-call mechanism is already initialized. 3168 */ 3169void 3170ipc_importance_thread_call_init(void) 3171{ 3172 /* initialize delayed drop queue and thread-call */ 3173 queue_init(&ipc_importance_delayed_drop_queue); 3174 ipc_importance_delayed_drop_call = 3175 thread_call_allocate(ipc_importance_task_delayed_drop_scan, NULL); 3176 if (NULL == ipc_importance_delayed_drop_call) { 3177 panic("ipc_importance_init"); 3178 } 3179} 3180 3181/* 3182 * Routing: task_importance_list_pids 3183 * Purpose: list pids where task in donating importance. 3184 * Conditions: To be called only from kdp stackshot code. 3185 * Will panic the system otherwise. 3186 */ 3187extern int 3188task_importance_list_pids(task_t task, int flags, int *pid_list, unsigned int max_count) 3189{ 3190 if (lck_spin_is_acquired(&ipc_importance_lock_data) || 3191 max_count < 1 || 3192 task->task_imp_base == IIT_NULL || 3193 pid_list == NULL || 3194 flags != TASK_IMP_LIST_DONATING_PIDS) { 3195 return 0; 3196 } 3197 unsigned int pidcount = 0; 3198 task_t temp_task; 3199 ipc_importance_task_t task_imp = task->task_imp_base; 3200 ipc_kmsg_t temp_kmsg; 3201 ipc_importance_inherit_t temp_inherit; 3202 ipc_importance_elem_t elem; 3203 int target_pid; 3204 3205 queue_iterate(&task_imp->iit_inherits, temp_inherit, ipc_importance_inherit_t, iii_inheritance) { 3206 /* check space in buffer */ 3207 if (pidcount >= max_count) 3208 break; 3209 target_pid = -1; 3210 3211 if (temp_inherit->iii_donating) { 3212 3213#if DEVELOPMENT || DEBUG 3214 target_pid = temp_inherit->iii_to_task->iit_bsd_pid; 3215#else 3216 temp_task = temp_inherit->iii_to_task->iit_task; 3217 if (temp_task != TASK_NULL) { 3218 target_pid = audit_token_pid_from_task(temp_task); 3219 } 3220#endif 3221 } 3222 3223 if (target_pid != -1) { 3224 pid_list[pidcount++] = target_pid; 3225 } 3226 3227 } 3228 3229 queue_iterate(&task_imp->iit_kmsgs, temp_kmsg, ipc_kmsg_t, ikm_inheritance) { 3230 if (pidcount >= max_count) 3231 break; 3232 target_pid = -1; 3233 elem = temp_kmsg->ikm_importance; 3234 temp_task = TASK_NULL; 3235 3236 if (elem == IIE_NULL) { 3237 continue; 3238 } 3239 3240 if (!(temp_kmsg->ikm_header && MACH_MSGH_BITS_RAISED_IMPORTANCE(temp_kmsg->ikm_header->msgh_bits))) { 3241 continue; 3242 } 3243 3244 if (IIE_TYPE_TASK == IIE_TYPE(elem) && 3245 (((ipc_importance_task_t)elem)->iit_task != TASK_NULL)) { 3246 target_pid = audit_token_pid_from_task(((ipc_importance_task_t)elem)->iit_task); 3247 } else { 3248 temp_inherit = (ipc_importance_inherit_t)elem; 3249#if DEVELOPMENT || DEBUG 3250 target_pid = temp_inherit->iii_to_task->iit_bsd_pid; 3251#else 3252 temp_task = temp_inherit->iii_to_task->iit_task; 3253 if (temp_task != TASK_NULL) { 3254 target_pid = audit_token_pid_from_task(temp_task); 3255 } 3256#endif 3257 } 3258 3259 if (target_pid != -1) { 3260 pid_list[pidcount++] = target_pid; 3261 } 3262 } 3263 3264 return pidcount; 3265} 3266 3267