1/* 2 * Copyright (c) 2000-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_FREE_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58/* 59 * File: kern/thread.c 60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub 61 * Date: 1986 62 * 63 * Thread management primitives implementation. 64 */ 65/* 66 * Copyright (c) 1993 The University of Utah and 67 * the Computer Systems Laboratory (CSL). All rights reserved. 68 * 69 * Permission to use, copy, modify and distribute this software and its 70 * documentation is hereby granted, provided that both the copyright 71 * notice and this permission notice appear in all copies of the 72 * software, derivative works or modified versions, and any portions 73 * thereof, and that both notices appear in supporting documentation. 74 * 75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS 76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF 77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 78 * 79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any 80 * improvements that they make and grant CSL redistribution rights. 81 * 82 */ 83 84#include <mach/mach_types.h> 85#include <mach/boolean.h> 86#include <mach/policy.h> 87#include <mach/thread_info.h> 88#include <mach/thread_special_ports.h> 89#include <mach/thread_status.h> 90#include <mach/time_value.h> 91#include <mach/vm_param.h> 92 93#include <machine/thread.h> 94#include <machine/pal_routines.h> 95#include <machine/limits.h> 96 97#include <kern/kern_types.h> 98#include <kern/kalloc.h> 99#include <kern/cpu_data.h> 100#include <kern/counters.h> 101#include <kern/extmod_statistics.h> 102#include <kern/ipc_mig.h> 103#include <kern/ipc_tt.h> 104#include <kern/mach_param.h> 105#include <kern/machine.h> 106#include <kern/misc_protos.h> 107#include <kern/processor.h> 108#include <kern/queue.h> 109#include <kern/sched.h> 110#include <kern/sched_prim.h> 111#include <kern/sync_lock.h> 112#include <kern/syscall_subr.h> 113#include <kern/task.h> 114#include <kern/thread.h> 115#include <kern/host.h> 116#include <kern/zalloc.h> 117#include <kern/assert.h> 118#include <kern/exc_resource.h> 119#include <kern/telemetry.h> 120#if KPC 121#include <kern/kpc.h> 122#endif 123 124#include <ipc/ipc_kmsg.h> 125#include <ipc/ipc_port.h> 126#include <bank/bank_types.h> 127 128#include <vm/vm_kern.h> 129#include <vm/vm_pageout.h> 130 131#include <sys/kdebug.h> 132 133#include <mach/sdt.h> 134 135/* 136 * Exported interfaces 137 */ 138#include <mach/task_server.h> 139#include <mach/thread_act_server.h> 140#include <mach/mach_host_server.h> 141#include <mach/host_priv_server.h> 142#include <mach/mach_voucher_server.h> 143 144static struct zone *thread_zone; 145static lck_grp_attr_t thread_lck_grp_attr; 146lck_attr_t thread_lck_attr; 147lck_grp_t thread_lck_grp; 148 149decl_simple_lock_data(static,thread_stack_lock) 150static queue_head_t thread_stack_queue; 151 152decl_simple_lock_data(static,thread_terminate_lock) 153static queue_head_t thread_terminate_queue; 154 155static struct thread thread_template, init_thread; 156 157static void sched_call_null( 158 int type, 159 thread_t thread); 160 161#ifdef MACH_BSD 162extern void proc_exit(void *); 163extern uint64_t get_dispatchqueue_offset_from_proc(void *); 164extern int proc_selfpid(void); 165extern char * proc_name_address(void *p); 166#endif /* MACH_BSD */ 167 168extern int disable_exc_resource; 169extern int audio_active; 170extern int debug_task; 171int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */ 172int task_threadmax = CONFIG_THREAD_MAX; 173 174static uint64_t thread_unique_id = 100; 175 176struct _thread_ledger_indices thread_ledgers = { -1 }; 177static ledger_template_t thread_ledger_template = NULL; 178void init_thread_ledgers(void); 179int task_disable_cpumon(task_t task); 180 181#if CONFIG_JETSAM 182void jetsam_on_ledger_cpulimit_exceeded(void); 183#endif 184 185/* 186 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry. 187 * 188 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user 189 * stacktraces, aka micro-stackshots) 190 */ 191#define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70 192 193int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */ 194void __attribute__((noinline)) THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(void); 195 196/* 197 * The smallest interval over which we support limiting CPU consumption is 1ms 198 */ 199#define MINIMUM_CPULIMIT_INTERVAL_MS 1 200 201void 202thread_bootstrap(void) 203{ 204 /* 205 * Fill in a template thread for fast initialization. 206 */ 207 208 thread_template.runq = PROCESSOR_NULL; 209 210 thread_template.ref_count = 2; 211 212 thread_template.reason = AST_NONE; 213 thread_template.at_safe_point = FALSE; 214 thread_template.wait_event = NO_EVENT64; 215 thread_template.wait_queue = WAIT_QUEUE_NULL; 216 thread_template.wait_result = THREAD_WAITING; 217 thread_template.options = THREAD_ABORTSAFE; 218 thread_template.state = TH_WAIT | TH_UNINT; 219 thread_template.wake_active = FALSE; 220 thread_template.continuation = THREAD_CONTINUE_NULL; 221 thread_template.parameter = NULL; 222 223 thread_template.importance = 0; 224 thread_template.sched_mode = TH_MODE_NONE; 225 thread_template.sched_flags = 0; 226 thread_template.saved_mode = TH_MODE_NONE; 227 thread_template.safe_release = 0; 228 229 thread_template.sfi_class = SFI_CLASS_UNSPECIFIED; 230 thread_template.sfi_wait_class = SFI_CLASS_UNSPECIFIED; 231 232 thread_template.active = 0; 233 thread_template.started = 0; 234 thread_template.static_param = 0; 235 thread_template.policy_reset = 0; 236 237 thread_template.priority = 0; 238 thread_template.sched_pri = 0; 239 thread_template.max_priority = 0; 240 thread_template.task_priority = 0; 241 thread_template.promotions = 0; 242 thread_template.pending_promoter_index = 0; 243 thread_template.pending_promoter[0] = 244 thread_template.pending_promoter[1] = NULL; 245 thread_template.rwlock_count = 0; 246 247#if MACH_ASSERT 248 thread_template.SHARE_COUNT = 0; 249 thread_template.BG_COUNT = 0; 250#endif /* MACH_ASSERT */ 251 252 thread_template.realtime.deadline = UINT64_MAX; 253 254 thread_template.quantum_remaining = 0; 255 thread_template.last_run_time = 0; 256 257 thread_template.computation_metered = 0; 258 thread_template.computation_epoch = 0; 259 260#if defined(CONFIG_SCHED_TIMESHARE_CORE) 261 thread_template.sched_stamp = 0; 262 thread_template.pri_shift = INT8_MAX; 263 thread_template.sched_usage = 0; 264 thread_template.cpu_usage = thread_template.cpu_delta = 0; 265#endif 266 thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0; 267 268 thread_template.bound_processor = PROCESSOR_NULL; 269 thread_template.last_processor = PROCESSOR_NULL; 270 271 thread_template.sched_call = sched_call_null; 272 273 timer_init(&thread_template.user_timer); 274 timer_init(&thread_template.system_timer); 275 thread_template.user_timer_save = 0; 276 thread_template.system_timer_save = 0; 277 thread_template.vtimer_user_save = 0; 278 thread_template.vtimer_prof_save = 0; 279 thread_template.vtimer_rlim_save = 0; 280 281 thread_template.wait_sfi_begin_time = 0; 282 283 thread_template.wait_timer_is_set = FALSE; 284 thread_template.wait_timer_active = 0; 285 286 thread_template.depress_timer_active = 0; 287 288 thread_template.special_handler.handler = special_handler; 289 thread_template.special_handler.next = NULL; 290 291 thread_template.recover = (vm_offset_t)NULL; 292 293 thread_template.map = VM_MAP_NULL; 294 295#if CONFIG_DTRACE 296 thread_template.t_dtrace_predcache = 0; 297 thread_template.t_dtrace_vtime = 0; 298 thread_template.t_dtrace_tracing = 0; 299#endif /* CONFIG_DTRACE */ 300 301#if KPC 302 thread_template.kpc_buf = NULL; 303#endif 304 305#if HYPERVISOR 306 thread_template.hv_thread_target = NULL; 307#endif /* HYPERVISOR */ 308 309 thread_template.t_chud = 0; 310 thread_template.t_page_creation_count = 0; 311 thread_template.t_page_creation_time = 0; 312 313 thread_template.affinity_set = NULL; 314 315 thread_template.syscalls_unix = 0; 316 thread_template.syscalls_mach = 0; 317 318 thread_template.t_ledger = LEDGER_NULL; 319 thread_template.t_threadledger = LEDGER_NULL; 320#ifdef CONFIG_BANK 321 thread_template.t_bankledger = LEDGER_NULL; 322 thread_template.t_deduct_bank_ledger_time = 0; 323#endif 324 325 thread_template.requested_policy = default_task_requested_policy; 326 thread_template.effective_policy = default_task_effective_policy; 327 thread_template.pended_policy = default_task_pended_policy; 328 329 thread_template.usynch_override_contended_resource_count = 0; 330 331 thread_template.iotier_override = THROTTLE_LEVEL_NONE; 332 thread_template.thread_io_stats = NULL; 333 thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0; 334 335 thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0; 336 thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0; 337 338 thread_template.thread_tag = 0; 339 340 thread_template.ith_voucher_name = MACH_PORT_NULL; 341 thread_template.ith_voucher = IPC_VOUCHER_NULL; 342 343 init_thread = thread_template; 344 machine_set_current_thread(&init_thread); 345} 346 347extern boolean_t allow_qos_policy_set; 348 349void 350thread_init(void) 351{ 352 thread_zone = zinit( 353 sizeof(struct thread), 354 thread_max * sizeof(struct thread), 355 THREAD_CHUNK * sizeof(struct thread), 356 "threads"); 357 358 lck_grp_attr_setdefault(&thread_lck_grp_attr); 359 lck_grp_init(&thread_lck_grp, "thread", &thread_lck_grp_attr); 360 lck_attr_setdefault(&thread_lck_attr); 361 362 stack_init(); 363 364 /* 365 * Initialize any machine-dependent 366 * per-thread structures necessary. 367 */ 368 machine_thread_init(); 369 370 if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct, 371 sizeof (cpumon_ustackshots_trigger_pct))) { 372 cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT; 373 } 374 375 PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set)); 376 377 init_thread_ledgers(); 378} 379 380static void 381thread_terminate_continue(void) 382{ 383 panic("thread_terminate_continue"); 384 /*NOTREACHED*/ 385} 386 387/* 388 * thread_terminate_self: 389 */ 390void 391thread_terminate_self(void) 392{ 393 thread_t thread = current_thread(); 394 395 task_t task; 396 spl_t s; 397 int threadcnt; 398 399 pal_thread_terminate_self(thread); 400 401 DTRACE_PROC(lwp__exit); 402 403 thread_mtx_lock(thread); 404 405 ipc_thread_disable(thread); 406 407 thread_mtx_unlock(thread); 408 409 s = splsched(); 410 thread_lock(thread); 411 412 assert_thread_sched_count(thread); 413 414 /* 415 * Cancel priority depression, wait for concurrent expirations 416 * on other processors. 417 */ 418 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) { 419 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK; 420 421 /* If our priority was low because of a depressed yield, restore it in case we block below */ 422 set_sched_pri(thread, thread->priority); 423 424 if (timer_call_cancel(&thread->depress_timer)) 425 thread->depress_timer_active--; 426 } 427 428 while (thread->depress_timer_active > 0) { 429 thread_unlock(thread); 430 splx(s); 431 432 delay(1); 433 434 s = splsched(); 435 thread_lock(thread); 436 } 437 438 thread_sched_call(thread, NULL); 439 440 thread_unlock(thread); 441 splx(s); 442 443 444 thread_mtx_lock(thread); 445 446 thread_policy_reset(thread); 447 448 thread_mtx_unlock(thread); 449 450 task = thread->task; 451 uthread_cleanup(task, thread->uthread, task->bsd_info); 452 threadcnt = hw_atomic_sub(&task->active_thread_count, 1); 453 454 /* 455 * If we are the last thread to terminate and the task is 456 * associated with a BSD process, perform BSD process exit. 457 */ 458 if (threadcnt == 0 && task->bsd_info != NULL) 459 proc_exit(task->bsd_info); 460 461 uthread_cred_free(thread->uthread); 462 463 s = splsched(); 464 thread_lock(thread); 465 466 /* 467 * Cancel wait timer, and wait for 468 * concurrent expirations. 469 */ 470 if (thread->wait_timer_is_set) { 471 thread->wait_timer_is_set = FALSE; 472 473 if (timer_call_cancel(&thread->wait_timer)) 474 thread->wait_timer_active--; 475 } 476 477 while (thread->wait_timer_active > 0) { 478 thread_unlock(thread); 479 splx(s); 480 481 delay(1); 482 483 s = splsched(); 484 thread_lock(thread); 485 } 486 487 /* 488 * If there is a reserved stack, release it. 489 */ 490 if (thread->reserved_stack != 0) { 491 stack_free_reserved(thread); 492 thread->reserved_stack = 0; 493 } 494 495 /* 496 * Mark thread as terminating, and block. 497 */ 498 thread->state |= TH_TERMINATE; 499 thread_mark_wait_locked(thread, THREAD_UNINT); 500 assert(thread->promotions == 0); 501 assert(thread->rwlock_count == 0); 502 thread_unlock(thread); 503 /* splsched */ 504 505 thread_block((thread_continue_t)thread_terminate_continue); 506 /*NOTREACHED*/ 507} 508 509void 510thread_deallocate( 511 thread_t thread) 512{ 513 task_t task; 514 515 if (thread == THREAD_NULL) 516 return; 517 518 if (thread_deallocate_internal(thread) > 0) 519 return; 520 521 if(!(thread->state & TH_TERMINATE2)) 522 panic("thread_deallocate: thread not properly terminated\n"); 523 524#if KPC 525 kpc_thread_destroy(thread); 526#endif 527 528 ipc_thread_terminate(thread); 529 530 task = thread->task; 531 532#ifdef MACH_BSD 533 { 534 void *ut = thread->uthread; 535 536 thread->uthread = NULL; 537 uthread_zone_free(ut); 538 } 539#endif /* MACH_BSD */ 540 541 if (thread->t_ledger) 542 ledger_dereference(thread->t_ledger); 543 if (thread->t_threadledger) 544 ledger_dereference(thread->t_threadledger); 545 546 if (IPC_VOUCHER_NULL != thread->ith_voucher) 547 ipc_voucher_release(thread->ith_voucher); 548 549 if (thread->thread_io_stats) 550 kfree(thread->thread_io_stats, sizeof(struct io_stat_info)); 551 552 if (thread->kernel_stack != 0) 553 stack_free(thread); 554 555 lck_mtx_destroy(&thread->mutex, &thread_lck_grp); 556 machine_thread_destroy(thread); 557 558 task_deallocate(task); 559 560 zfree(thread_zone, thread); 561} 562 563/* 564 * thread_terminate_daemon: 565 * 566 * Perform final clean up for terminating threads. 567 */ 568static void 569thread_terminate_daemon(void) 570{ 571 thread_t self, thread; 572 task_t task; 573 574 self = current_thread(); 575 self->options |= TH_OPT_SYSTEM_CRITICAL; 576 577 (void)splsched(); 578 simple_lock(&thread_terminate_lock); 579 580 while ((thread = (thread_t)dequeue_head(&thread_terminate_queue)) != THREAD_NULL) { 581 simple_unlock(&thread_terminate_lock); 582 (void)spllo(); 583 584 assert(thread->SHARE_COUNT == 0); 585 assert(thread->BG_COUNT == 0); 586 587 task = thread->task; 588 589 task_lock(task); 590 task->total_user_time += timer_grab(&thread->user_timer); 591 if (thread->precise_user_kernel_time) { 592 task->total_system_time += timer_grab(&thread->system_timer); 593 } else { 594 task->total_user_time += timer_grab(&thread->system_timer); 595 } 596 597 task->c_switch += thread->c_switch; 598 task->p_switch += thread->p_switch; 599 task->ps_switch += thread->ps_switch; 600 601 task->syscalls_unix += thread->syscalls_unix; 602 task->syscalls_mach += thread->syscalls_mach; 603 604 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1; 605 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2; 606 task->task_gpu_ns += ml_gpu_stat(thread); 607 608 thread_update_qos_cpu_time(thread, FALSE); 609 queue_remove(&task->threads, thread, thread_t, task_threads); 610 task->thread_count--; 611 612 /* 613 * If the task is being halted, and there is only one thread 614 * left in the task after this one, then wakeup that thread. 615 */ 616 if (task->thread_count == 1 && task->halting) 617 thread_wakeup((event_t)&task->halting); 618 619 task_unlock(task); 620 621 lck_mtx_lock(&tasks_threads_lock); 622 queue_remove(&threads, thread, thread_t, threads); 623 threads_count--; 624 lck_mtx_unlock(&tasks_threads_lock); 625 626 thread_deallocate(thread); 627 628 (void)splsched(); 629 simple_lock(&thread_terminate_lock); 630 } 631 632 assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT); 633 simple_unlock(&thread_terminate_lock); 634 /* splsched */ 635 636 self->options &= ~TH_OPT_SYSTEM_CRITICAL; 637 thread_block((thread_continue_t)thread_terminate_daemon); 638 /*NOTREACHED*/ 639} 640 641/* 642 * thread_terminate_enqueue: 643 * 644 * Enqueue a terminating thread for final disposition. 645 * 646 * Called at splsched. 647 */ 648void 649thread_terminate_enqueue( 650 thread_t thread) 651{ 652 KERNEL_DEBUG_CONSTANT(TRACEDBG_CODE(DBG_TRACE_DATA, TRACE_DATA_THREAD_TERMINATE) | DBG_FUNC_NONE, thread->thread_id, 0, 0, 0, 0); 653 654 simple_lock(&thread_terminate_lock); 655 enqueue_tail(&thread_terminate_queue, (queue_entry_t)thread); 656 simple_unlock(&thread_terminate_lock); 657 658 thread_wakeup((event_t)&thread_terminate_queue); 659} 660 661/* 662 * thread_stack_daemon: 663 * 664 * Perform stack allocation as required due to 665 * invoke failures. 666 */ 667static void 668thread_stack_daemon(void) 669{ 670 thread_t thread; 671 spl_t s; 672 673 s = splsched(); 674 simple_lock(&thread_stack_lock); 675 676 while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) { 677 simple_unlock(&thread_stack_lock); 678 splx(s); 679 680 /* allocate stack with interrupts enabled so that we can call into VM */ 681 stack_alloc(thread); 682 683 s = splsched(); 684 thread_lock(thread); 685 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ); 686 thread_unlock(thread); 687 688 simple_lock(&thread_stack_lock); 689 } 690 691 assert_wait((event_t)&thread_stack_queue, THREAD_UNINT); 692 simple_unlock(&thread_stack_lock); 693 splx(s); 694 695 thread_block((thread_continue_t)thread_stack_daemon); 696 /*NOTREACHED*/ 697} 698 699/* 700 * thread_stack_enqueue: 701 * 702 * Enqueue a thread for stack allocation. 703 * 704 * Called at splsched. 705 */ 706void 707thread_stack_enqueue( 708 thread_t thread) 709{ 710 simple_lock(&thread_stack_lock); 711 enqueue_tail(&thread_stack_queue, (queue_entry_t)thread); 712 simple_unlock(&thread_stack_lock); 713 714 thread_wakeup((event_t)&thread_stack_queue); 715} 716 717void 718thread_daemon_init(void) 719{ 720 kern_return_t result; 721 thread_t thread = NULL; 722 723 simple_lock_init(&thread_terminate_lock, 0); 724 queue_init(&thread_terminate_queue); 725 726 result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread); 727 if (result != KERN_SUCCESS) 728 panic("thread_daemon_init: thread_terminate_daemon"); 729 730 thread_deallocate(thread); 731 732 simple_lock_init(&thread_stack_lock, 0); 733 queue_init(&thread_stack_queue); 734 735 result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread); 736 if (result != KERN_SUCCESS) 737 panic("thread_daemon_init: thread_stack_daemon"); 738 739 thread_deallocate(thread); 740} 741 742/* 743 * Create a new thread. 744 * Doesn't start the thread running. 745 * 746 * Task and tasks_threads_lock are returned locked on success. 747 */ 748static kern_return_t 749thread_create_internal( 750 task_t parent_task, 751 integer_t priority, 752 thread_continue_t continuation, 753 int options, 754#define TH_OPTION_NONE 0x00 755#define TH_OPTION_NOCRED 0x01 756#define TH_OPTION_NOSUSP 0x02 757 thread_t *out_thread) 758{ 759 thread_t new_thread; 760 static thread_t first_thread; 761 762 /* 763 * Allocate a thread and initialize static fields 764 */ 765 if (first_thread == THREAD_NULL) 766 new_thread = first_thread = current_thread(); 767 else 768 new_thread = (thread_t)zalloc(thread_zone); 769 if (new_thread == THREAD_NULL) 770 return (KERN_RESOURCE_SHORTAGE); 771 772 if (new_thread != first_thread) 773 *new_thread = thread_template; 774 775#ifdef MACH_BSD 776 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0); 777 if (new_thread->uthread == NULL) { 778 zfree(thread_zone, new_thread); 779 return (KERN_RESOURCE_SHORTAGE); 780 } 781#endif /* MACH_BSD */ 782 783 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) { 784#ifdef MACH_BSD 785 void *ut = new_thread->uthread; 786 787 new_thread->uthread = NULL; 788 /* cred free may not be necessary */ 789 uthread_cleanup(parent_task, ut, parent_task->bsd_info); 790 uthread_cred_free(ut); 791 uthread_zone_free(ut); 792#endif /* MACH_BSD */ 793 794 zfree(thread_zone, new_thread); 795 return (KERN_FAILURE); 796 } 797 798 new_thread->task = parent_task; 799 800 thread_lock_init(new_thread); 801 wake_lock_init(new_thread); 802 803 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, &thread_lck_attr); 804 805 ipc_thread_init(new_thread); 806 807 new_thread->continuation = continuation; 808 809 /* Allocate I/O Statistics structure */ 810 new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info)); 811 assert(new_thread->thread_io_stats != NULL); 812 bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info)); 813 814#if CONFIG_IOSCHED 815 /* Clear out the I/O Scheduling info for AppleFSCompression */ 816 new_thread->decmp_upl = NULL; 817#endif /* CONFIG_IOSCHED */ 818 819 lck_mtx_lock(&tasks_threads_lock); 820 task_lock(parent_task); 821 822 if ( !parent_task->active || parent_task->halting || 823 ((options & TH_OPTION_NOSUSP) != 0 && 824 parent_task->suspend_count > 0) || 825 (parent_task->thread_count >= task_threadmax && 826 parent_task != kernel_task) ) { 827 task_unlock(parent_task); 828 lck_mtx_unlock(&tasks_threads_lock); 829 830#ifdef MACH_BSD 831 { 832 void *ut = new_thread->uthread; 833 834 new_thread->uthread = NULL; 835 uthread_cleanup(parent_task, ut, parent_task->bsd_info); 836 /* cred free may not be necessary */ 837 uthread_cred_free(ut); 838 uthread_zone_free(ut); 839 } 840#endif /* MACH_BSD */ 841 ipc_thread_disable(new_thread); 842 ipc_thread_terminate(new_thread); 843 kfree(new_thread->thread_io_stats, sizeof(struct io_stat_info)); 844 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp); 845 machine_thread_destroy(new_thread); 846 zfree(thread_zone, new_thread); 847 return (KERN_FAILURE); 848 } 849 850 /* New threads inherit any default state on the task */ 851 machine_thread_inherit_taskwide(new_thread, parent_task); 852 853 task_reference_internal(parent_task); 854 855 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) { 856 /* 857 * This task has a per-thread CPU limit; make sure this new thread 858 * gets its limit set too, before it gets out of the kernel. 859 */ 860 set_astledger(new_thread); 861 } 862 863 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */ 864 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template, 865 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) { 866 867 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time); 868 } 869 870 new_thread->cpu_time_last_qos = 0; 871#ifdef CONFIG_BANK 872 new_thread->t_bankledger = LEDGER_NULL; 873 new_thread->t_deduct_bank_ledger_time = 0; 874#endif 875 876 new_thread->t_ledger = new_thread->task->ledger; 877 if (new_thread->t_ledger) 878 ledger_reference(new_thread->t_ledger); 879 880#if defined(CONFIG_SCHED_MULTIQ) 881 /* Cache the task's sched_group */ 882 new_thread->sched_group = parent_task->sched_group; 883#endif /* defined(CONFIG_SCHED_MULTIQ) */ 884 885 /* Cache the task's map */ 886 new_thread->map = parent_task->map; 887 888 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread); 889 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread); 890 891#if CONFIG_COUNTERS 892 /* 893 * If parent task has any reservations, they need to be propagated to this 894 * thread. 895 */ 896 new_thread->t_chud = (TASK_PMC_FLAG == (parent_task->t_chud & TASK_PMC_FLAG)) ? 897 THREAD_PMC_FLAG : 0U; 898#endif 899#if KPC 900 kpc_thread_create(new_thread); 901#endif 902 903 /* Only need to update policies pushed from task to thread */ 904 new_thread->requested_policy.bg_iotier = parent_task->effective_policy.bg_iotier; 905 new_thread->requested_policy.terminated = parent_task->effective_policy.terminated; 906 907 /* Set the thread's scheduling parameters */ 908 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task); 909 new_thread->sched_flags = 0; 910 new_thread->max_priority = parent_task->max_priority; 911 new_thread->task_priority = parent_task->priority; 912 new_thread->priority = (priority < 0)? parent_task->priority: priority; 913 if (new_thread->priority > new_thread->max_priority) 914 new_thread->priority = new_thread->max_priority; 915 new_thread->importance = new_thread->priority - new_thread->task_priority; 916 new_thread->saved_importance = new_thread->importance; 917 918#if defined(CONFIG_SCHED_TIMESHARE_CORE) 919 new_thread->sched_stamp = sched_tick; 920 new_thread->pri_shift = sched_pri_shift; 921#endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */ 922 923 if (parent_task->max_priority <= MAXPRI_THROTTLE) { 924 sched_set_thread_throttled(new_thread, TRUE); 925 } 926 927 SCHED(compute_priority)(new_thread, FALSE); 928 929 thread_policy_create(new_thread); 930 931 /* Chain the thread onto the task's list */ 932 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads); 933 parent_task->thread_count++; 934 935 /* So terminating threads don't need to take the task lock to decrement */ 936 hw_atomic_add(&parent_task->active_thread_count, 1); 937 938 /* Protected by the tasks_threads_lock */ 939 new_thread->thread_id = ++thread_unique_id; 940 941 queue_enter(&threads, new_thread, thread_t, threads); 942 threads_count++; 943 944 new_thread->active = TRUE; 945 946 *out_thread = new_thread; 947 948 { 949 long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4; 950 951 kdbg_trace_data(parent_task->bsd_info, &dbg_arg2); 952 953 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 954 TRACEDBG_CODE(DBG_TRACE_DATA, 1) | DBG_FUNC_NONE, 955 (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, 0, 0, 0); 956 957 kdbg_trace_string(parent_task->bsd_info, 958 &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4); 959 960 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 961 TRACEDBG_CODE(DBG_TRACE_STRING, 1) | DBG_FUNC_NONE, 962 dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0); 963 } 964 965 DTRACE_PROC1(lwp__create, thread_t, *out_thread); 966 967 return (KERN_SUCCESS); 968} 969 970static kern_return_t 971thread_create_internal2( 972 task_t task, 973 thread_t *new_thread, 974 boolean_t from_user) 975{ 976 kern_return_t result; 977 thread_t thread; 978 979 if (task == TASK_NULL || task == kernel_task) 980 return (KERN_INVALID_ARGUMENT); 981 982 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread); 983 if (result != KERN_SUCCESS) 984 return (result); 985 986 thread->user_stop_count = 1; 987 thread_hold(thread); 988 if (task->suspend_count > 0) 989 thread_hold(thread); 990 991 if (from_user) 992 extmod_statistics_incr_thread_create(task); 993 994 task_unlock(task); 995 lck_mtx_unlock(&tasks_threads_lock); 996 997 *new_thread = thread; 998 999 return (KERN_SUCCESS); 1000} 1001 1002/* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */ 1003kern_return_t 1004thread_create( 1005 task_t task, 1006 thread_t *new_thread); 1007 1008kern_return_t 1009thread_create( 1010 task_t task, 1011 thread_t *new_thread) 1012{ 1013 return thread_create_internal2(task, new_thread, FALSE); 1014} 1015 1016kern_return_t 1017thread_create_from_user( 1018 task_t task, 1019 thread_t *new_thread) 1020{ 1021 return thread_create_internal2(task, new_thread, TRUE); 1022} 1023 1024static kern_return_t 1025thread_create_running_internal2( 1026 register task_t task, 1027 int flavor, 1028 thread_state_t new_state, 1029 mach_msg_type_number_t new_state_count, 1030 thread_t *new_thread, 1031 boolean_t from_user) 1032{ 1033 register kern_return_t result; 1034 thread_t thread; 1035 1036 if (task == TASK_NULL || task == kernel_task) 1037 return (KERN_INVALID_ARGUMENT); 1038 1039 result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread); 1040 if (result != KERN_SUCCESS) 1041 return (result); 1042 1043 result = machine_thread_set_state( 1044 thread, flavor, new_state, new_state_count); 1045 if (result != KERN_SUCCESS) { 1046 task_unlock(task); 1047 lck_mtx_unlock(&tasks_threads_lock); 1048 1049 thread_terminate(thread); 1050 thread_deallocate(thread); 1051 return (result); 1052 } 1053 1054 thread_mtx_lock(thread); 1055 thread_start_internal(thread); 1056 thread_mtx_unlock(thread); 1057 1058 if (from_user) 1059 extmod_statistics_incr_thread_create(task); 1060 1061 task_unlock(task); 1062 lck_mtx_unlock(&tasks_threads_lock); 1063 1064 *new_thread = thread; 1065 1066 return (result); 1067} 1068 1069/* Prototype, see justification above */ 1070kern_return_t 1071thread_create_running( 1072 register task_t task, 1073 int flavor, 1074 thread_state_t new_state, 1075 mach_msg_type_number_t new_state_count, 1076 thread_t *new_thread); 1077 1078kern_return_t 1079thread_create_running( 1080 register task_t task, 1081 int flavor, 1082 thread_state_t new_state, 1083 mach_msg_type_number_t new_state_count, 1084 thread_t *new_thread) 1085{ 1086 return thread_create_running_internal2( 1087 task, flavor, new_state, new_state_count, 1088 new_thread, FALSE); 1089} 1090 1091kern_return_t 1092thread_create_running_from_user( 1093 register task_t task, 1094 int flavor, 1095 thread_state_t new_state, 1096 mach_msg_type_number_t new_state_count, 1097 thread_t *new_thread) 1098{ 1099 return thread_create_running_internal2( 1100 task, flavor, new_state, new_state_count, 1101 new_thread, TRUE); 1102} 1103 1104kern_return_t 1105thread_create_workq( 1106 task_t task, 1107 thread_continue_t thread_return, 1108 thread_t *new_thread) 1109{ 1110 kern_return_t result; 1111 thread_t thread; 1112 1113 if (task == TASK_NULL || task == kernel_task) 1114 return (KERN_INVALID_ARGUMENT); 1115 1116 result = thread_create_internal(task, -1, thread_return, TH_OPTION_NOCRED | TH_OPTION_NOSUSP, &thread); 1117 if (result != KERN_SUCCESS) 1118 return (result); 1119 1120 thread->user_stop_count = 1; 1121 thread_hold(thread); 1122 if (task->suspend_count > 0) 1123 thread_hold(thread); 1124 1125 task_unlock(task); 1126 lck_mtx_unlock(&tasks_threads_lock); 1127 1128 *new_thread = thread; 1129 1130 return (KERN_SUCCESS); 1131} 1132 1133/* 1134 * kernel_thread_create: 1135 * 1136 * Create a thread in the kernel task 1137 * to execute in kernel context. 1138 */ 1139kern_return_t 1140kernel_thread_create( 1141 thread_continue_t continuation, 1142 void *parameter, 1143 integer_t priority, 1144 thread_t *new_thread) 1145{ 1146 kern_return_t result; 1147 thread_t thread; 1148 task_t task = kernel_task; 1149 1150 result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread); 1151 if (result != KERN_SUCCESS) 1152 return (result); 1153 1154 task_unlock(task); 1155 lck_mtx_unlock(&tasks_threads_lock); 1156 1157 stack_alloc(thread); 1158 assert(thread->kernel_stack != 0); 1159 thread->reserved_stack = thread->kernel_stack; 1160 1161 thread->parameter = parameter; 1162 1163if(debug_task & 1) 1164 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation); 1165 *new_thread = thread; 1166 1167 return (result); 1168} 1169 1170kern_return_t 1171kernel_thread_start_priority( 1172 thread_continue_t continuation, 1173 void *parameter, 1174 integer_t priority, 1175 thread_t *new_thread) 1176{ 1177 kern_return_t result; 1178 thread_t thread; 1179 1180 result = kernel_thread_create(continuation, parameter, priority, &thread); 1181 if (result != KERN_SUCCESS) 1182 return (result); 1183 1184 *new_thread = thread; 1185 1186 thread_mtx_lock(thread); 1187 thread_start_internal(thread); 1188 thread_mtx_unlock(thread); 1189 1190 return (result); 1191} 1192 1193kern_return_t 1194kernel_thread_start( 1195 thread_continue_t continuation, 1196 void *parameter, 1197 thread_t *new_thread) 1198{ 1199 return kernel_thread_start_priority(continuation, parameter, -1, new_thread); 1200} 1201 1202 1203kern_return_t 1204thread_info_internal( 1205 register thread_t thread, 1206 thread_flavor_t flavor, 1207 thread_info_t thread_info_out, /* ptr to OUT array */ 1208 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/ 1209{ 1210 int state, flags; 1211 spl_t s; 1212 1213 if (thread == THREAD_NULL) 1214 return (KERN_INVALID_ARGUMENT); 1215 1216 if (flavor == THREAD_BASIC_INFO) { 1217 register thread_basic_info_t basic_info; 1218 1219 if (*thread_info_count < THREAD_BASIC_INFO_COUNT) 1220 return (KERN_INVALID_ARGUMENT); 1221 1222 basic_info = (thread_basic_info_t) thread_info_out; 1223 1224 s = splsched(); 1225 thread_lock(thread); 1226 1227 /* fill in info */ 1228 1229 thread_read_times(thread, &basic_info->user_time, 1230 &basic_info->system_time); 1231 1232 /* 1233 * Update lazy-evaluated scheduler info because someone wants it. 1234 */ 1235 if (SCHED(can_update_priority)(thread)) 1236 SCHED(update_priority)(thread); 1237 1238 basic_info->sleep_time = 0; 1239 1240 /* 1241 * To calculate cpu_usage, first correct for timer rate, 1242 * then for 5/8 ageing. The correction factor [3/5] is 1243 * (1/(5/8) - 1). 1244 */ 1245 basic_info->cpu_usage = 0; 1246#if defined(CONFIG_SCHED_TIMESHARE_CORE) 1247 if (sched_tick_interval) { 1248 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage 1249 * TH_USAGE_SCALE) / sched_tick_interval); 1250 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5; 1251 } 1252#endif 1253 1254 if (basic_info->cpu_usage > TH_USAGE_SCALE) 1255 basic_info->cpu_usage = TH_USAGE_SCALE; 1256 1257 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)? 1258 POLICY_TIMESHARE: POLICY_RR); 1259 1260 flags = 0; 1261 if (thread->options & TH_OPT_IDLE_THREAD) 1262 flags |= TH_FLAGS_IDLE; 1263 1264 if (!thread->kernel_stack) 1265 flags |= TH_FLAGS_SWAPPED; 1266 1267 state = 0; 1268 if (thread->state & TH_TERMINATE) 1269 state = TH_STATE_HALTED; 1270 else 1271 if (thread->state & TH_RUN) 1272 state = TH_STATE_RUNNING; 1273 else 1274 if (thread->state & TH_UNINT) 1275 state = TH_STATE_UNINTERRUPTIBLE; 1276 else 1277 if (thread->state & TH_SUSP) 1278 state = TH_STATE_STOPPED; 1279 else 1280 if (thread->state & TH_WAIT) 1281 state = TH_STATE_WAITING; 1282 1283 basic_info->run_state = state; 1284 basic_info->flags = flags; 1285 1286 basic_info->suspend_count = thread->user_stop_count; 1287 1288 thread_unlock(thread); 1289 splx(s); 1290 1291 *thread_info_count = THREAD_BASIC_INFO_COUNT; 1292 1293 return (KERN_SUCCESS); 1294 } 1295 else 1296 if (flavor == THREAD_IDENTIFIER_INFO) { 1297 register thread_identifier_info_t identifier_info; 1298 1299 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) 1300 return (KERN_INVALID_ARGUMENT); 1301 1302 identifier_info = (thread_identifier_info_t) thread_info_out; 1303 1304 s = splsched(); 1305 thread_lock(thread); 1306 1307 identifier_info->thread_id = thread->thread_id; 1308 identifier_info->thread_handle = thread->machine.cthread_self; 1309 if(thread->task->bsd_info) { 1310 identifier_info->dispatch_qaddr = identifier_info->thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info); 1311 } else { 1312 thread_unlock(thread); 1313 splx(s); 1314 return KERN_INVALID_ARGUMENT; 1315 } 1316 1317 thread_unlock(thread); 1318 splx(s); 1319 return KERN_SUCCESS; 1320 } 1321 else 1322 if (flavor == THREAD_SCHED_TIMESHARE_INFO) { 1323 policy_timeshare_info_t ts_info; 1324 1325 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) 1326 return (KERN_INVALID_ARGUMENT); 1327 1328 ts_info = (policy_timeshare_info_t)thread_info_out; 1329 1330 s = splsched(); 1331 thread_lock(thread); 1332 1333 if (thread->sched_mode != TH_MODE_TIMESHARE) { 1334 thread_unlock(thread); 1335 splx(s); 1336 1337 return (KERN_INVALID_POLICY); 1338 } 1339 1340 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0; 1341 if (ts_info->depressed) { 1342 ts_info->base_priority = DEPRESSPRI; 1343 ts_info->depress_priority = thread->priority; 1344 } 1345 else { 1346 ts_info->base_priority = thread->priority; 1347 ts_info->depress_priority = -1; 1348 } 1349 1350 ts_info->cur_priority = thread->sched_pri; 1351 ts_info->max_priority = thread->max_priority; 1352 1353 thread_unlock(thread); 1354 splx(s); 1355 1356 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT; 1357 1358 return (KERN_SUCCESS); 1359 } 1360 else 1361 if (flavor == THREAD_SCHED_FIFO_INFO) { 1362 if (*thread_info_count < POLICY_FIFO_INFO_COUNT) 1363 return (KERN_INVALID_ARGUMENT); 1364 1365 return (KERN_INVALID_POLICY); 1366 } 1367 else 1368 if (flavor == THREAD_SCHED_RR_INFO) { 1369 policy_rr_info_t rr_info; 1370 uint32_t quantum_time; 1371 uint64_t quantum_ns; 1372 1373 if (*thread_info_count < POLICY_RR_INFO_COUNT) 1374 return (KERN_INVALID_ARGUMENT); 1375 1376 rr_info = (policy_rr_info_t) thread_info_out; 1377 1378 s = splsched(); 1379 thread_lock(thread); 1380 1381 if (thread->sched_mode == TH_MODE_TIMESHARE) { 1382 thread_unlock(thread); 1383 splx(s); 1384 1385 return (KERN_INVALID_POLICY); 1386 } 1387 1388 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0; 1389 if (rr_info->depressed) { 1390 rr_info->base_priority = DEPRESSPRI; 1391 rr_info->depress_priority = thread->priority; 1392 } 1393 else { 1394 rr_info->base_priority = thread->priority; 1395 rr_info->depress_priority = -1; 1396 } 1397 1398 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL); 1399 absolutetime_to_nanoseconds(quantum_time, &quantum_ns); 1400 1401 rr_info->max_priority = thread->max_priority; 1402 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000); 1403 1404 thread_unlock(thread); 1405 splx(s); 1406 1407 *thread_info_count = POLICY_RR_INFO_COUNT; 1408 1409 return (KERN_SUCCESS); 1410 } 1411 1412 return (KERN_INVALID_ARGUMENT); 1413} 1414 1415void 1416thread_read_times( 1417 thread_t thread, 1418 time_value_t *user_time, 1419 time_value_t *system_time) 1420{ 1421 clock_sec_t secs; 1422 clock_usec_t usecs; 1423 uint64_t tval_user, tval_system; 1424 1425 tval_user = timer_grab(&thread->user_timer); 1426 tval_system = timer_grab(&thread->system_timer); 1427 1428 if (thread->precise_user_kernel_time) { 1429 absolutetime_to_microtime(tval_user, &secs, &usecs); 1430 user_time->seconds = (typeof(user_time->seconds))secs; 1431 user_time->microseconds = usecs; 1432 1433 absolutetime_to_microtime(tval_system, &secs, &usecs); 1434 system_time->seconds = (typeof(system_time->seconds))secs; 1435 system_time->microseconds = usecs; 1436 } else { 1437 /* system_timer may represent either sys or user */ 1438 tval_user += tval_system; 1439 absolutetime_to_microtime(tval_user, &secs, &usecs); 1440 user_time->seconds = (typeof(user_time->seconds))secs; 1441 user_time->microseconds = usecs; 1442 1443 system_time->seconds = 0; 1444 system_time->microseconds = 0; 1445 } 1446} 1447 1448uint64_t thread_get_runtime_self(void) 1449{ 1450 boolean_t interrupt_state; 1451 uint64_t runtime; 1452 thread_t thread = NULL; 1453 processor_t processor = NULL; 1454 1455 thread = current_thread(); 1456 1457 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */ 1458 interrupt_state = ml_set_interrupts_enabled(FALSE); 1459 processor = current_processor(); 1460 timer_switch(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time(), PROCESSOR_DATA(processor, thread_timer)); 1461 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer)); 1462 ml_set_interrupts_enabled(interrupt_state); 1463 1464 return runtime; 1465} 1466 1467kern_return_t 1468thread_assign( 1469 __unused thread_t thread, 1470 __unused processor_set_t new_pset) 1471{ 1472 return (KERN_FAILURE); 1473} 1474 1475/* 1476 * thread_assign_default: 1477 * 1478 * Special version of thread_assign for assigning threads to default 1479 * processor set. 1480 */ 1481kern_return_t 1482thread_assign_default( 1483 thread_t thread) 1484{ 1485 return (thread_assign(thread, &pset0)); 1486} 1487 1488/* 1489 * thread_get_assignment 1490 * 1491 * Return current assignment for this thread. 1492 */ 1493kern_return_t 1494thread_get_assignment( 1495 thread_t thread, 1496 processor_set_t *pset) 1497{ 1498 if (thread == NULL) 1499 return (KERN_INVALID_ARGUMENT); 1500 1501 *pset = &pset0; 1502 1503 return (KERN_SUCCESS); 1504} 1505 1506/* 1507 * thread_wire_internal: 1508 * 1509 * Specify that the target thread must always be able 1510 * to run and to allocate memory. 1511 */ 1512kern_return_t 1513thread_wire_internal( 1514 host_priv_t host_priv, 1515 thread_t thread, 1516 boolean_t wired, 1517 boolean_t *prev_state) 1518{ 1519 if (host_priv == NULL || thread != current_thread()) 1520 return (KERN_INVALID_ARGUMENT); 1521 1522 assert(host_priv == &realhost); 1523 1524 if (prev_state) 1525 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0; 1526 1527 if (wired) { 1528 if (!(thread->options & TH_OPT_VMPRIV)) 1529 vm_page_free_reserve(1); /* XXX */ 1530 thread->options |= TH_OPT_VMPRIV; 1531 } 1532 else { 1533 if (thread->options & TH_OPT_VMPRIV) 1534 vm_page_free_reserve(-1); /* XXX */ 1535 thread->options &= ~TH_OPT_VMPRIV; 1536 } 1537 1538 return (KERN_SUCCESS); 1539} 1540 1541 1542/* 1543 * thread_wire: 1544 * 1545 * User-api wrapper for thread_wire_internal() 1546 */ 1547kern_return_t 1548thread_wire( 1549 host_priv_t host_priv, 1550 thread_t thread, 1551 boolean_t wired) 1552{ 1553 return (thread_wire_internal(host_priv, thread, wired, NULL)); 1554} 1555 1556 1557boolean_t 1558set_vm_privilege(boolean_t privileged) 1559{ 1560 boolean_t was_vmpriv; 1561 1562 if (current_thread()->options & TH_OPT_VMPRIV) 1563 was_vmpriv = TRUE; 1564 else 1565 was_vmpriv = FALSE; 1566 1567 if (privileged != FALSE) 1568 current_thread()->options |= TH_OPT_VMPRIV; 1569 else 1570 current_thread()->options &= ~TH_OPT_VMPRIV; 1571 1572 return (was_vmpriv); 1573} 1574 1575 1576/* 1577 * XXX assuming current thread only, for now... 1578 */ 1579void 1580thread_guard_violation(thread_t thread, unsigned type) 1581{ 1582 assert(thread == current_thread()); 1583 1584 spl_t s = splsched(); 1585 /* 1586 * Use the saved state area of the thread structure 1587 * to store all info required to handle the AST when 1588 * returning to userspace 1589 */ 1590 thread->guard_exc_info.type = type; 1591 thread_ast_set(thread, AST_GUARD); 1592 ast_propagate(thread->ast); 1593 1594 splx(s); 1595} 1596 1597/* 1598 * guard_ast: 1599 * 1600 * Handle AST_GUARD for a thread. This routine looks at the 1601 * state saved in the thread structure to determine the cause 1602 * of this exception. Based on this value, it invokes the 1603 * appropriate routine which determines other exception related 1604 * info and raises the exception. 1605 */ 1606void 1607guard_ast(thread_t thread) 1608{ 1609 if (thread->guard_exc_info.type == GUARD_TYPE_MACH_PORT) 1610 mach_port_guard_ast(thread); 1611 else 1612 fd_guard_ast(thread); 1613} 1614 1615static void 1616thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1) 1617{ 1618 if (warning == LEDGER_WARNING_ROSE_ABOVE) { 1619#if CONFIG_TELEMETRY 1620 /* 1621 * This thread is in danger of violating the CPU usage monitor. Enable telemetry 1622 * on the entire task so there are micro-stackshots available if and when 1623 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots 1624 * for this thread only; but now that this task is suspect, knowing what all of 1625 * its threads are up to will be useful. 1626 */ 1627 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1); 1628#endif 1629 return; 1630 } 1631 1632#if CONFIG_TELEMETRY 1633 /* 1634 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or 1635 * exceeded the limit, turn telemetry off for the task. 1636 */ 1637 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0); 1638#endif 1639 1640 if (warning == 0) { 1641 THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(); 1642 } 1643} 1644 1645void __attribute__((noinline)) 1646THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU__SENDING_EXC_RESOURCE(void) 1647{ 1648 int pid = 0; 1649 task_t task = current_task(); 1650 thread_t thread = current_thread(); 1651 uint64_t tid = thread->thread_id; 1652 char *procname = (char *) "unknown"; 1653 time_value_t thread_total_time = {0, 0}; 1654 time_value_t thread_system_time; 1655 time_value_t thread_user_time; 1656 int action; 1657 uint8_t percentage; 1658 uint32_t limit_percent; 1659 uint32_t usage_percent; 1660 uint32_t interval_sec; 1661 uint64_t interval_ns; 1662 uint64_t balance_ns; 1663 boolean_t fatal = FALSE; 1664 1665 mach_exception_data_type_t code[EXCEPTION_CODE_MAX]; 1666 struct ledger_entry_info lei; 1667 1668 assert(thread->t_threadledger != LEDGER_NULL); 1669 1670 /* 1671 * Now that a thread has tripped the monitor, disable it for the entire task. 1672 */ 1673 task_lock(task); 1674 if ((task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) { 1675 /* 1676 * The CPU usage monitor has been disabled on our task, so some other 1677 * thread must have gotten here first. We only send one exception per 1678 * task lifetime, so there's nothing left for us to do here. 1679 */ 1680 task_unlock(task); 1681 return; 1682 } 1683 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) { 1684 fatal = TRUE; 1685 } 1686 task_disable_cpumon(task); 1687 task_unlock(task); 1688 1689#ifdef MACH_BSD 1690 pid = proc_selfpid(); 1691 if (task->bsd_info != NULL) 1692 procname = proc_name_address(task->bsd_info); 1693#endif 1694 1695 thread_get_cpulimit(&action, &percentage, &interval_ns); 1696 1697 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC); 1698 1699 thread_read_times(thread, &thread_user_time, &thread_system_time); 1700 time_value_add(&thread_total_time, &thread_user_time); 1701 time_value_add(&thread_total_time, &thread_system_time); 1702 1703 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei); 1704 1705 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns); 1706 usage_percent = (uint32_t) ((balance_ns * 100ULL) / lei.lei_last_refill); 1707 1708 /* Show refill period in the same units as balance, limit, etc */ 1709 nanoseconds_to_absolutetime(lei.lei_refill_period, &lei.lei_refill_period); 1710 1711 limit_percent = (uint32_t) ((lei.lei_limit * 100ULL) / lei.lei_refill_period); 1712 1713 /* TODO: show task total runtime as well? see TASK_ABSOLUTETIME_INFO */ 1714 1715 if (disable_exc_resource) { 1716 printf("process %s[%d] thread %llu caught burning CPU!; EXC_RESOURCE " 1717 "supressed by a boot-arg\n", procname, pid, tid); 1718 return; 1719 } 1720 1721 if (audio_active) { 1722 printf("process %s[%d] thread %llu caught burning CPU!; EXC_RESOURCE " 1723 "supressed due to audio playback\n", procname, pid, tid); 1724 return; 1725 } 1726 printf("process %s[%d] thread %llu caught burning CPU! " 1727 "It used more than %d%% CPU (Actual recent usage: %d%%) over %d seconds. " 1728 "thread lifetime cpu usage %d.%06d seconds, (%d.%06d user, %d.%06d system) " 1729 "ledger info: balance: %lld credit: %lld debit: %lld limit: %llu (%d%%) " 1730 "period: %llu time since last refill (ns): %llu %s\n", 1731 procname, pid, tid, 1732 percentage, usage_percent, interval_sec, 1733 thread_total_time.seconds, thread_total_time.microseconds, 1734 thread_user_time.seconds, thread_user_time.microseconds, 1735 thread_system_time.seconds, thread_system_time.microseconds, 1736 lei.lei_balance, 1737 lei.lei_credit, lei.lei_debit, 1738 lei.lei_limit, limit_percent, 1739 lei.lei_refill_period, lei.lei_last_refill, 1740 (fatal ? "[fatal violation]" : "")); 1741 1742 1743 code[0] = code[1] = 0; 1744 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU); 1745 if (fatal) { 1746 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL); 1747 }else { 1748 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR); 1749 } 1750 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec); 1751 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], limit_percent); 1752 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent); 1753 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX); 1754 1755 if (fatal) { 1756#if CONFIG_JETSAM 1757 jetsam_on_ledger_cpulimit_exceeded(); 1758#else 1759 task_terminate_internal(task); 1760#endif 1761 } 1762} 1763 1764#define UPDATE_IO_STATS(info, size) \ 1765{ \ 1766 info.count++; \ 1767 info.size += size; \ 1768} 1769 1770#define UPDATE_IO_STATS_ATOMIC(info, size) \ 1771{ \ 1772 OSIncrementAtomic64((SInt64 *)&(info.count)); \ 1773 OSAddAtomic64(size, (SInt64 *)&(info.size)); \ 1774} 1775 1776void thread_update_io_stats(thread_t thread, int size, int io_flags) 1777{ 1778 int io_tier; 1779 1780 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL) 1781 return; 1782 1783 if (io_flags & DKIO_READ) { 1784 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size); 1785 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size); 1786 } 1787 1788 if (io_flags & DKIO_META) { 1789 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size); 1790 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size); 1791 } 1792 1793 if (io_flags & DKIO_PAGING) { 1794 UPDATE_IO_STATS(thread->thread_io_stats->paging, size); 1795 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size); 1796 } 1797 1798 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT); 1799 assert (io_tier < IO_NUM_PRIORITIES); 1800 1801 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size); 1802 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size); 1803 1804 /* Update Total I/O Counts */ 1805 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size); 1806 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size); 1807 1808} 1809 1810void 1811init_thread_ledgers(void) { 1812 ledger_template_t t; 1813 int idx; 1814 1815 assert(thread_ledger_template == NULL); 1816 1817 if ((t = ledger_template_create("Per-thread ledger")) == NULL) 1818 panic("couldn't create thread ledger template"); 1819 1820 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) { 1821 panic("couldn't create cpu_time entry for thread ledger template"); 1822 } 1823 1824 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) { 1825 panic("couldn't set thread ledger callback for cpu_time entry"); 1826 } 1827 1828 thread_ledgers.cpu_time = idx; 1829 1830 thread_ledger_template = t; 1831} 1832 1833/* 1834 * Returns currently applied CPU usage limit, or 0/0 if none is applied. 1835 */ 1836int 1837thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns) 1838{ 1839 int64_t abstime = 0; 1840 uint64_t limittime = 0; 1841 thread_t thread = current_thread(); 1842 1843 *percentage = 0; 1844 *interval_ns = 0; 1845 *action = 0; 1846 1847 if (thread->t_threadledger == LEDGER_NULL) { 1848 /* 1849 * This thread has no per-thread ledger, so it can't possibly 1850 * have a CPU limit applied. 1851 */ 1852 return (KERN_SUCCESS); 1853 } 1854 1855 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns); 1856 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime); 1857 1858 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) { 1859 /* 1860 * This thread's CPU time ledger has no period or limit; so it 1861 * doesn't have a CPU limit applied. 1862 */ 1863 return (KERN_SUCCESS); 1864 } 1865 1866 /* 1867 * This calculation is the converse to the one in thread_set_cpulimit(). 1868 */ 1869 absolutetime_to_nanoseconds(abstime, &limittime); 1870 *percentage = (limittime * 100ULL) / *interval_ns; 1871 assert(*percentage <= 100); 1872 1873 if (thread->options & TH_OPT_PROC_CPULIMIT) { 1874 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0); 1875 1876 *action = THREAD_CPULIMIT_BLOCK; 1877 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) { 1878 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0); 1879 1880 *action = THREAD_CPULIMIT_EXCEPTION; 1881 } else { 1882 *action = THREAD_CPULIMIT_DISABLE; 1883 } 1884 1885 return (KERN_SUCCESS); 1886} 1887 1888/* 1889 * Set CPU usage limit on a thread. 1890 * 1891 * Calling with percentage of 0 will unset the limit for this thread. 1892 */ 1893int 1894thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns) 1895{ 1896 thread_t thread = current_thread(); 1897 ledger_t l; 1898 uint64_t limittime = 0; 1899 uint64_t abstime = 0; 1900 1901 assert(percentage <= 100); 1902 1903 if (action == THREAD_CPULIMIT_DISABLE) { 1904 /* 1905 * Remove CPU limit, if any exists. 1906 */ 1907 if (thread->t_threadledger != LEDGER_NULL) { 1908 l = thread->t_threadledger; 1909 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0); 1910 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE); 1911 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT); 1912 } 1913 1914 return (0); 1915 } 1916 1917 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) { 1918 return (KERN_INVALID_ARGUMENT); 1919 } 1920 1921 l = thread->t_threadledger; 1922 if (l == LEDGER_NULL) { 1923 /* 1924 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active. 1925 */ 1926 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL) 1927 return (KERN_RESOURCE_SHORTAGE); 1928 1929 /* 1930 * We are the first to create this thread's ledger, so only activate our entry. 1931 */ 1932 ledger_entry_setactive(l, thread_ledgers.cpu_time); 1933 thread->t_threadledger = l; 1934 } 1935 1936 /* 1937 * The limit is specified as a percentage of CPU over an interval in nanoseconds. 1938 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit. 1939 */ 1940 limittime = (interval_ns * percentage) / 100; 1941 nanoseconds_to_absolutetime(limittime, &abstime); 1942 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct); 1943 /* 1944 * Refill the thread's allotted CPU time every interval_ns nanoseconds. 1945 */ 1946 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns); 1947 1948 if (action == THREAD_CPULIMIT_EXCEPTION) { 1949 /* 1950 * We don't support programming the CPU usage monitor on a task if any of its 1951 * threads have a per-thread blocking CPU limit configured. 1952 */ 1953 if (thread->options & TH_OPT_PRVT_CPULIMIT) { 1954 panic("CPU usage monitor activated, but blocking thread limit exists"); 1955 } 1956 1957 /* 1958 * Make a note that this thread's CPU limit is being used for the task-wide CPU 1959 * usage monitor. We don't have to arm the callback which will trigger the 1960 * exception, because that was done for us in ledger_instantiate (because the 1961 * ledger template used has a default callback). 1962 */ 1963 thread->options |= TH_OPT_PROC_CPULIMIT; 1964 } else { 1965 /* 1966 * We deliberately override any CPU limit imposed by a task-wide limit (eg 1967 * CPU usage monitor). 1968 */ 1969 thread->options &= ~TH_OPT_PROC_CPULIMIT; 1970 1971 thread->options |= TH_OPT_PRVT_CPULIMIT; 1972 /* The per-thread ledger template by default has a callback for CPU time */ 1973 ledger_disable_callback(l, thread_ledgers.cpu_time); 1974 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK); 1975 } 1976 1977 return (0); 1978} 1979 1980static void 1981sched_call_null( 1982__unused int type, 1983__unused thread_t thread) 1984{ 1985 return; 1986} 1987 1988void 1989thread_sched_call( 1990 thread_t thread, 1991 sched_call_t call) 1992{ 1993 thread->sched_call = (call != NULL)? call: sched_call_null; 1994} 1995 1996void 1997thread_static_param( 1998 thread_t thread, 1999 boolean_t state) 2000{ 2001 thread_mtx_lock(thread); 2002 thread->static_param = state; 2003 thread_mtx_unlock(thread); 2004} 2005 2006uint64_t 2007thread_tid( 2008 thread_t thread) 2009{ 2010 return (thread != THREAD_NULL? thread->thread_id: 0); 2011} 2012 2013uint16_t thread_set_tag(thread_t th, uint16_t tag) { 2014 return thread_set_tag_internal(th, tag); 2015} 2016uint16_t thread_get_tag(thread_t th) { 2017 return thread_get_tag_internal(th); 2018} 2019 2020uint64_t 2021thread_dispatchqaddr( 2022 thread_t thread) 2023{ 2024 uint64_t dispatchqueue_addr = 0; 2025 uint64_t thread_handle = 0; 2026 2027 if (thread != THREAD_NULL) { 2028 thread_handle = thread->machine.cthread_self; 2029 2030 if (thread->task->bsd_info) 2031 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info); 2032 } 2033 2034 return (dispatchqueue_addr); 2035} 2036 2037/* 2038 * Export routines to other components for things that are done as macros 2039 * within the osfmk component. 2040 */ 2041 2042#undef thread_reference 2043void thread_reference(thread_t thread); 2044void 2045thread_reference( 2046 thread_t thread) 2047{ 2048 if (thread != THREAD_NULL) 2049 thread_reference_internal(thread); 2050} 2051 2052#undef thread_should_halt 2053 2054boolean_t 2055thread_should_halt( 2056 thread_t th) 2057{ 2058 return (thread_should_halt_fast(th)); 2059} 2060 2061/* 2062 * thread_set_voucher_name - reset the voucher port name bound to this thread 2063 * 2064 * Conditions: nothing locked 2065 * 2066 * If we already converted the previous name to a cached voucher 2067 * reference, then we discard that reference here. The next lookup 2068 * will cache it again. 2069 */ 2070 2071kern_return_t 2072thread_set_voucher_name(mach_port_name_t voucher_name) 2073{ 2074 thread_t thread = current_thread(); 2075 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL; 2076 ipc_voucher_t voucher; 2077#ifdef CONFIG_BANK 2078 ledger_t bankledger = NULL; 2079#endif 2080 2081 if (MACH_PORT_DEAD == voucher_name) 2082 return KERN_INVALID_RIGHT; 2083 2084 /* 2085 * agressively convert to voucher reference 2086 */ 2087 if (MACH_PORT_VALID(voucher_name)) { 2088 new_voucher = convert_port_name_to_voucher(voucher_name); 2089 if (IPC_VOUCHER_NULL == new_voucher) 2090 return KERN_INVALID_ARGUMENT; 2091 } 2092#ifdef CONFIG_BANK 2093 bankledger = bank_get_voucher_ledger(new_voucher); 2094#endif 2095 2096 thread_mtx_lock(thread); 2097 voucher = thread->ith_voucher; 2098 thread->ith_voucher_name = voucher_name; 2099 thread->ith_voucher = new_voucher; 2100#ifdef CONFIG_BANK 2101 bank_swap_thread_bank_ledger(thread, bankledger); 2102#endif 2103 thread_mtx_unlock(thread); 2104 2105 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2106 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 2107 (uintptr_t)thread_tid(thread), 2108 (uintptr_t)voucher_name, 2109 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher), 2110 1, 0); 2111 2112 if (IPC_VOUCHER_NULL != voucher) 2113 ipc_voucher_release(voucher); 2114 2115 return KERN_SUCCESS; 2116} 2117 2118/* 2119 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher 2120 * 2121 * Conditions: nothing locked 2122 * 2123 * A reference to the voucher may be lazily pending, if someone set the voucher name 2124 * but nobody has done a lookup yet. In that case, we'll have to do the equivalent 2125 * lookup here. 2126 * 2127 * NOTE: At the moment, there is no distinction between the current and effective 2128 * vouchers because we only set them at the thread level currently. 2129 */ 2130kern_return_t 2131thread_get_mach_voucher( 2132 thread_act_t thread, 2133 mach_voucher_selector_t __unused which, 2134 ipc_voucher_t *voucherp) 2135{ 2136 ipc_voucher_t voucher; 2137 mach_port_name_t voucher_name; 2138 2139 if (THREAD_NULL == thread) 2140 return KERN_INVALID_ARGUMENT; 2141 2142 thread_mtx_lock(thread); 2143 voucher = thread->ith_voucher; 2144 2145 /* if already cached, just return a ref */ 2146 if (IPC_VOUCHER_NULL != voucher) { 2147 ipc_voucher_reference(voucher); 2148 thread_mtx_unlock(thread); 2149 *voucherp = voucher; 2150 return KERN_SUCCESS; 2151 } 2152 2153 voucher_name = thread->ith_voucher_name; 2154 2155 /* convert the name to a port, then voucher reference */ 2156 if (MACH_PORT_VALID(voucher_name)) { 2157 ipc_port_t port; 2158 2159 if (KERN_SUCCESS != 2160 ipc_object_copyin(thread->task->itk_space, voucher_name, 2161 MACH_MSG_TYPE_COPY_SEND, (ipc_object_t *)&port)) { 2162 thread->ith_voucher_name = MACH_PORT_NULL; 2163 thread_mtx_unlock(thread); 2164 *voucherp = IPC_VOUCHER_NULL; 2165 return KERN_SUCCESS; 2166 } 2167 2168 /* convert to a voucher ref to return, and cache a ref on thread */ 2169 voucher = convert_port_to_voucher(port); 2170 ipc_voucher_reference(voucher); 2171 thread->ith_voucher = voucher; 2172 thread_mtx_unlock(thread); 2173 2174 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2175 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 2176 (uintptr_t)thread_tid(thread), 2177 (uintptr_t)port, 2178 VM_KERNEL_ADDRPERM((uintptr_t)voucher), 2179 2, 0); 2180 2181 2182 ipc_port_release_send(port); 2183 } else 2184 thread_mtx_unlock(thread); 2185 2186 *voucherp = voucher; 2187 return KERN_SUCCESS; 2188} 2189 2190/* 2191 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher 2192 * 2193 * Conditions: callers holds a reference on the voucher. 2194 * nothing locked. 2195 * 2196 * We grab another reference to the voucher and bind it to the thread. Any lazy 2197 * binding is erased. The old voucher reference associated with the thread is 2198 * discarded. 2199 */ 2200kern_return_t 2201thread_set_mach_voucher( 2202 thread_t thread, 2203 ipc_voucher_t voucher) 2204{ 2205 ipc_voucher_t old_voucher; 2206#ifdef CONFIG_BANK 2207 ledger_t bankledger = NULL; 2208#endif 2209 2210 if (THREAD_NULL == thread) 2211 return KERN_INVALID_ARGUMENT; 2212 2213 if (thread != current_thread() || thread->started) 2214 return KERN_INVALID_ARGUMENT; 2215 2216 2217 ipc_voucher_reference(voucher); 2218#ifdef CONFIG_BANK 2219 bankledger = bank_get_voucher_ledger(voucher); 2220#endif 2221 thread_mtx_lock(thread); 2222 old_voucher = thread->ith_voucher; 2223 thread->ith_voucher = voucher; 2224 thread->ith_voucher_name = MACH_PORT_NULL; 2225#ifdef CONFIG_BANK 2226 bank_swap_thread_bank_ledger(thread, bankledger); 2227#endif 2228 thread_mtx_unlock(thread); 2229 2230 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2231 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 2232 (uintptr_t)thread_tid(thread), 2233 (uintptr_t)MACH_PORT_NULL, 2234 VM_KERNEL_ADDRPERM((uintptr_t)voucher), 2235 3, 0); 2236 2237 ipc_voucher_release(old_voucher); 2238 2239 return KERN_SUCCESS; 2240} 2241 2242/* 2243 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher 2244 * 2245 * Conditions: callers holds a reference on the new and presumed old voucher(s). 2246 * nothing locked. 2247 * 2248 * If the old voucher is still the same as passed in, replace it with new voucher 2249 * and discard the old (and the reference passed in). Otherwise, discard the new 2250 * and return an updated old voucher. 2251 */ 2252kern_return_t 2253thread_swap_mach_voucher( 2254 thread_t thread, 2255 ipc_voucher_t new_voucher, 2256 ipc_voucher_t *in_out_old_voucher) 2257{ 2258 mach_port_name_t old_voucher_name; 2259 ipc_voucher_t old_voucher; 2260#ifdef CONFIG_BANK 2261 ledger_t bankledger = NULL; 2262#endif 2263 2264 if (THREAD_NULL == thread) 2265 return KERN_INVALID_TASK; 2266 2267 if (thread != current_thread() || thread->started) 2268 return KERN_INVALID_ARGUMENT; 2269 2270#ifdef CONFIG_BANK 2271 bankledger = bank_get_voucher_ledger(new_voucher); 2272#endif 2273 2274 thread_mtx_lock(thread); 2275 2276 old_voucher = thread->ith_voucher; 2277 2278 if (IPC_VOUCHER_NULL == old_voucher) { 2279 old_voucher_name = thread->ith_voucher_name; 2280 2281 /* perform lazy binding if needed */ 2282 if (MACH_PORT_VALID(old_voucher_name)) { 2283 old_voucher = convert_port_name_to_voucher(old_voucher_name); 2284 thread->ith_voucher_name = MACH_PORT_NULL; 2285 thread->ith_voucher = old_voucher; 2286 2287 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2288 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 2289 (uintptr_t)thread_tid(thread), 2290 (uintptr_t)old_voucher_name, 2291 VM_KERNEL_ADDRPERM((uintptr_t)old_voucher), 2292 4, 0); 2293 2294 } 2295 } 2296 2297 /* swap in new voucher, if old voucher matches the one supplied */ 2298 if (old_voucher == *in_out_old_voucher) { 2299 ipc_voucher_reference(new_voucher); 2300 thread->ith_voucher = new_voucher; 2301 thread->ith_voucher_name = MACH_PORT_NULL; 2302#ifdef CONFIG_BANK 2303 bank_swap_thread_bank_ledger(thread, bankledger); 2304#endif 2305 thread_mtx_unlock(thread); 2306 2307 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2308 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 2309 (uintptr_t)thread_tid(thread), 2310 (uintptr_t)MACH_PORT_NULL, 2311 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher), 2312 5, 0); 2313 2314 ipc_voucher_release(old_voucher); 2315 2316 *in_out_old_voucher = IPC_VOUCHER_NULL; 2317 return KERN_SUCCESS; 2318 } 2319 2320 /* Otherwise, just return old voucher reference */ 2321 ipc_voucher_reference(old_voucher); 2322 thread_mtx_unlock(thread); 2323 *in_out_old_voucher = old_voucher; 2324 return KERN_SUCCESS; 2325} 2326 2327/* 2328 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher. 2329 */ 2330kern_return_t 2331thread_get_current_voucher_origin_pid( 2332 int32_t *pid) 2333{ 2334 uint32_t buf_size; 2335 kern_return_t kr; 2336 thread_t thread = current_thread(); 2337 2338 buf_size = sizeof(*pid); 2339 kr = mach_voucher_attr_command(thread->ith_voucher, 2340 MACH_VOUCHER_ATTR_KEY_BANK, 2341 BANK_ORIGINATOR_PID, 2342 NULL, 2343 0, 2344 (mach_voucher_attr_content_t)pid, 2345 &buf_size); 2346 2347 return kr; 2348} 2349 2350#if CONFIG_DTRACE 2351uint32_t dtrace_get_thread_predcache(thread_t thread) 2352{ 2353 if (thread != THREAD_NULL) 2354 return thread->t_dtrace_predcache; 2355 else 2356 return 0; 2357} 2358 2359int64_t dtrace_get_thread_vtime(thread_t thread) 2360{ 2361 if (thread != THREAD_NULL) 2362 return thread->t_dtrace_vtime; 2363 else 2364 return 0; 2365} 2366 2367int64_t dtrace_get_thread_tracing(thread_t thread) 2368{ 2369 if (thread != THREAD_NULL) 2370 return thread->t_dtrace_tracing; 2371 else 2372 return 0; 2373} 2374 2375boolean_t dtrace_get_thread_reentering(thread_t thread) 2376{ 2377 if (thread != THREAD_NULL) 2378 return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE; 2379 else 2380 return 0; 2381} 2382 2383vm_offset_t dtrace_get_kernel_stack(thread_t thread) 2384{ 2385 if (thread != THREAD_NULL) 2386 return thread->kernel_stack; 2387 else 2388 return 0; 2389} 2390 2391int64_t dtrace_calc_thread_recent_vtime(thread_t thread) 2392{ 2393 if (thread != THREAD_NULL) { 2394 processor_t processor = current_processor(); 2395 uint64_t abstime = mach_absolute_time(); 2396 timer_t timer; 2397 2398 timer = PROCESSOR_DATA(processor, thread_timer); 2399 2400 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) + 2401 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */ 2402 } else 2403 return 0; 2404} 2405 2406void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache) 2407{ 2408 if (thread != THREAD_NULL) 2409 thread->t_dtrace_predcache = predcache; 2410} 2411 2412void dtrace_set_thread_vtime(thread_t thread, int64_t vtime) 2413{ 2414 if (thread != THREAD_NULL) 2415 thread->t_dtrace_vtime = vtime; 2416} 2417 2418void dtrace_set_thread_tracing(thread_t thread, int64_t accum) 2419{ 2420 if (thread != THREAD_NULL) 2421 thread->t_dtrace_tracing = accum; 2422} 2423 2424void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool) 2425{ 2426 if (thread != THREAD_NULL) { 2427 if (vbool) 2428 thread->options |= TH_OPT_DTRACE; 2429 else 2430 thread->options &= (~TH_OPT_DTRACE); 2431 } 2432} 2433 2434vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover) 2435{ 2436 vm_offset_t prev = 0; 2437 2438 if (thread != THREAD_NULL) { 2439 prev = thread->recover; 2440 thread->recover = recover; 2441 } 2442 return prev; 2443} 2444 2445void dtrace_thread_bootstrap(void) 2446{ 2447 task_t task = current_task(); 2448 2449 if (task->thread_count == 1) { 2450 thread_t thread = current_thread(); 2451 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) { 2452 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS; 2453 DTRACE_PROC(exec__success); 2454 } 2455 DTRACE_PROC(start); 2456 } 2457 DTRACE_PROC(lwp__start); 2458 2459} 2460 2461void 2462dtrace_thread_didexec(thread_t thread) 2463{ 2464 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS; 2465} 2466#endif /* CONFIG_DTRACE */ 2467