1/* 2 * Copyright (c) 2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#define PTHREAD_INTERNAL 1 30 31#include <kern/debug.h> 32#include <kern/mach_param.h> 33#include <kern/sched_prim.h> 34#include <kern/task.h> 35#include <kern/thread.h> 36#include <kern/affinity.h> 37#include <kern/zalloc.h> 38#include <machine/machine_routines.h> 39#include <mach/task.h> 40#include <mach/thread_act.h> 41#include <sys/param.h> 42#include <sys/pthread_shims.h> 43#include <sys/proc_internal.h> 44#include <sys/sysproto.h> 45#include <sys/systm.h> 46#include <vm/vm_map.h> 47#include <vm/vm_protos.h> 48 49/* version number of the in-kernel shims given to pthread.kext */ 50#define PTHREAD_SHIMS_VERSION 1 51 52/* on arm, the callbacks function has two #ifdef arm ponters */ 53#define PTHREAD_CALLBACK_MEMBER ml_get_max_cpus 54 55/* compile time asserts to check the length of structures in pthread_shims.h */ 56char pthread_functions_size_compile_assert[(sizeof(struct pthread_functions_s) - offsetof(struct pthread_functions_s, psynch_rw_yieldwrlock) - sizeof(void*)) == (sizeof(void*) * 100) ? 1 : -1]; 57char pthread_callbacks_size_compile_assert[(sizeof(struct pthread_callbacks_s) - offsetof(struct pthread_callbacks_s, PTHREAD_CALLBACK_MEMBER) - sizeof(void*)) == (sizeof(void*) * 100) ? 1 : -1]; 58 59/* old pthread code had definitions for these as they don't exist in headers */ 60extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t); 61extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t); 62 63#define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \ 64 static rettype \ 65 get(structtype x) { \ 66 return (x)->member; \ 67 } \ 68 static void \ 69 set(structtype x, rettype y) { \ 70 (x)->member = y; \ 71 } 72 73PTHREAD_STRUCT_ACCESSOR(proc_get_threadstart, proc_set_threadstart, user_addr_t, struct proc*, p_threadstart); 74PTHREAD_STRUCT_ACCESSOR(proc_get_pthsize, proc_set_pthsize, int, struct proc*, p_pthsize); 75PTHREAD_STRUCT_ACCESSOR(proc_get_wqthread, proc_set_wqthread, user_addr_t, struct proc*, p_wqthread); 76PTHREAD_STRUCT_ACCESSOR(proc_get_targconc, proc_set_targconc, user_addr_t, struct proc*, p_targconc); 77PTHREAD_STRUCT_ACCESSOR(proc_get_stack_addr_hint, proc_set_stack_addr_hint, user_addr_t, struct proc *, p_stack_addr_hint); 78PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset); 79PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset); 80PTHREAD_STRUCT_ACCESSOR(proc_get_pthread_tsd_offset, proc_set_pthread_tsd_offset, uint32_t, struct proc *, p_pth_tsd_offset); 81PTHREAD_STRUCT_ACCESSOR(proc_get_wqptr, proc_set_wqptr, void*, struct proc*, p_wqptr); 82PTHREAD_STRUCT_ACCESSOR(proc_get_wqsize, proc_set_wqsize, int, struct proc*, p_wqsize); 83PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash); 84 85PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist); 86PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask); 87PTHREAD_STRUCT_ACCESSOR(uthread_get_returnval, uthread_set_returnval, int, struct uthread*, uu_rval[0]); 88 89static void 90pthread_returning_to_userspace(void) 91{ 92 thread_exception_return(); 93} 94 95static uint32_t 96get_task_threadmax(void) { 97 return task_threadmax; 98} 99 100static task_t 101proc_get_task(struct proc *p) { 102 return p->task; 103} 104 105static lck_spin_t* 106proc_get_wqlockptr(struct proc *p) { 107 return &(p->p_wqlock); 108} 109 110static boolean_t* 111proc_get_wqinitingptr(struct proc *p) { 112 return &(p->p_wqiniting); 113} 114 115static uint64_t 116proc_get_register(struct proc *p) { 117 return (p->p_lflag & P_LREGISTER); 118} 119 120static void 121proc_set_register(struct proc *p) { 122 proc_setregister(p); 123} 124 125static void* 126uthread_get_uukwe(struct uthread *t) 127{ 128 return &t->uu_kevent.uu_kwe; 129} 130 131static int 132uthread_is_cancelled(struct uthread *t) 133{ 134 return (t->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL; 135} 136 137static vm_map_t 138_current_map(void) 139{ 140 return current_map(); 141} 142 143static boolean_t 144qos_main_thread_active(void) 145{ 146 return TRUE; 147} 148 149 150static int proc_usynch_get_requested_thread_qos(struct uthread *uth) 151{ 152 task_t task = current_task(); 153 thread_t thread = uth ? uth->uu_thread : current_thread(); 154 int requested_qos; 155 156 requested_qos = proc_get_task_policy(task, thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS); 157 158 /* 159 * For the purposes of userspace synchronization, it doesn't make sense to place an override of UNSPECIFIED 160 * on another thread, if the current thread doesn't have any QoS set. In these cases, upgrade to 161 * THREAD_QOS_USER_INTERACTIVE. 162 */ 163 if (requested_qos == THREAD_QOS_UNSPECIFIED) { 164 requested_qos = THREAD_QOS_USER_INTERACTIVE; 165 } 166 167 return requested_qos; 168} 169 170static boolean_t proc_usynch_thread_qos_add_override(struct uthread *uth, uint64_t tid, int override_qos, boolean_t first_override_for_resource) 171{ 172 task_t task = current_task(); 173 thread_t thread = uth ? uth->uu_thread : THREAD_NULL; 174 175 return proc_thread_qos_add_override(task, thread, tid, override_qos, first_override_for_resource); 176} 177 178static boolean_t proc_usynch_thread_qos_remove_override(struct uthread *uth, uint64_t tid) 179{ 180 task_t task = current_task(); 181 thread_t thread = uth ? uth->uu_thread : THREAD_NULL; 182 183 return proc_thread_qos_remove_override(task, thread, tid); 184} 185 186/* kernel (core) to kext shims */ 187 188void 189pthread_init(void) 190{ 191 if (!pthread_functions) { 192 panic("pthread kernel extension not loaded (function table is NULL)."); 193 } 194 pthread_functions->pthread_init(); 195} 196 197int 198fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo) 199{ 200 return pthread_functions->fill_procworkqueue(p, pwqinfo); 201} 202 203void 204workqueue_init_lock(proc_t p) 205{ 206 pthread_functions->workqueue_init_lock(p); 207} 208 209void 210workqueue_destroy_lock(proc_t p) 211{ 212 pthread_functions->workqueue_destroy_lock(p); 213} 214 215void 216workqueue_exit(struct proc *p) 217{ 218 pthread_functions->workqueue_exit(p); 219} 220 221void 222workqueue_mark_exiting(struct proc *p) 223{ 224 pthread_functions->workqueue_mark_exiting(p); 225} 226 227void 228workqueue_thread_yielded(void) 229{ 230 pthread_functions->workqueue_thread_yielded(); 231} 232 233sched_call_t 234workqueue_get_sched_callback(void) 235{ 236 if (pthread_functions->workqueue_get_sched_callback) { 237 return pthread_functions->workqueue_get_sched_callback(); 238 } 239 return NULL; 240} 241 242void 243pth_proc_hashinit(proc_t p) 244{ 245 pthread_functions->pth_proc_hashinit(p); 246} 247 248void 249pth_proc_hashdelete(proc_t p) 250{ 251 pthread_functions->pth_proc_hashdelete(p); 252} 253 254/* syscall shims */ 255int 256bsdthread_create(struct proc *p, struct bsdthread_create_args *uap, user_addr_t *retval) 257{ 258 return pthread_functions->bsdthread_create(p, uap->func, uap->func_arg, uap->stack, uap->pthread, uap->flags, retval); 259} 260 261int 262bsdthread_register(struct proc *p, struct bsdthread_register_args *uap, __unused int32_t *retval) 263{ 264 if (pthread_functions->version >= 1) { 265 return pthread_functions->bsdthread_register2(p, uap->threadstart, uap->wqthread, 266 uap->flags, uap->stack_addr_hint, 267 uap->targetconc_ptr, uap->dispatchqueue_offset, 268 uap->tsd_offset, retval); 269 } else { 270 return pthread_functions->bsdthread_register(p, uap->threadstart, uap->wqthread, 271 uap->flags, uap->stack_addr_hint, 272 uap->targetconc_ptr, uap->dispatchqueue_offset, 273 retval); 274 } 275} 276 277int 278bsdthread_terminate(struct proc *p, struct bsdthread_terminate_args *uap, int32_t *retval) 279{ 280 return pthread_functions->bsdthread_terminate(p, uap->stackaddr, uap->freesize, uap->port, uap->sem, retval); 281} 282 283int 284bsdthread_ctl(struct proc *p, struct bsdthread_ctl_args *uap, int *retval) 285{ 286 return pthread_functions->bsdthread_ctl(p, uap->cmd, uap->arg1, uap->arg2, uap->arg3, retval); 287} 288 289 290int 291thread_selfid(struct proc *p, __unused struct thread_selfid_args *uap, uint64_t *retval) 292{ 293 return pthread_functions->thread_selfid(p, retval); 294} 295 296int 297workq_kernreturn(struct proc *p, struct workq_kernreturn_args *uap, int32_t *retval) 298{ 299 return pthread_functions->workq_kernreturn(p, uap->options, uap->item, uap->affinity, uap->prio, retval); 300} 301 302int 303workq_open(struct proc *p, __unused struct workq_open_args *uap, int32_t *retval) 304{ 305 return pthread_functions->workq_open(p, retval); 306} 307 308/* pthread synchroniser syscalls */ 309 310int 311psynch_mutexwait(proc_t p, struct psynch_mutexwait_args *uap, uint32_t *retval) 312{ 313 return pthread_functions->psynch_mutexwait(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval); 314} 315 316int 317psynch_mutexdrop(proc_t p, struct psynch_mutexdrop_args *uap, uint32_t *retval) 318{ 319 return pthread_functions->psynch_mutexdrop(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval); 320} 321 322int 323psynch_cvbroad(proc_t p, struct psynch_cvbroad_args *uap, uint32_t *retval) 324{ 325 return pthread_functions->psynch_cvbroad(p, uap->cv, uap->cvlsgen, uap->cvudgen, uap->flags, uap->mutex, uap->mugen, uap->tid, retval); 326} 327 328int 329psynch_cvsignal(proc_t p, struct psynch_cvsignal_args *uap, uint32_t *retval) 330{ 331 return pthread_functions->psynch_cvsignal(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->thread_port, uap->mutex, uap->mugen, uap->tid, uap->flags, retval); 332} 333 334int 335psynch_cvwait(proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval) 336{ 337 return pthread_functions->psynch_cvwait(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->mutex, uap->mugen, uap->flags, uap->sec, uap->nsec, retval); 338} 339 340int 341psynch_cvclrprepost(proc_t p, struct psynch_cvclrprepost_args * uap, int *retval) 342{ 343 return pthread_functions->psynch_cvclrprepost(p, uap->cv, uap->cvgen, uap->cvugen, uap->cvsgen, uap->prepocnt, uap->preposeq, uap->flags, retval); 344} 345 346int 347psynch_rw_longrdlock(proc_t p, struct psynch_rw_longrdlock_args * uap, uint32_t *retval) 348{ 349 return pthread_functions->psynch_rw_longrdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 350} 351 352int 353psynch_rw_rdlock(proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval) 354{ 355 return pthread_functions->psynch_rw_rdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 356} 357 358int 359psynch_rw_unlock(proc_t p, struct psynch_rw_unlock_args *uap, uint32_t *retval) 360{ 361 return pthread_functions->psynch_rw_unlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 362} 363 364int 365psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args *uap, __unused uint32_t *retval) 366{ 367 return ENOTSUP; 368} 369 370int 371psynch_rw_wrlock(proc_t p, struct psynch_rw_wrlock_args *uap, uint32_t *retval) 372{ 373 return pthread_functions->psynch_rw_wrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 374} 375 376int 377psynch_rw_yieldwrlock(proc_t p, struct psynch_rw_yieldwrlock_args *uap, uint32_t *retval) 378{ 379 return pthread_functions->psynch_rw_yieldwrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 380} 381 382int 383psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t *retval) 384{ 385 return 0; 386} 387 388int 389psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int *retval) 390{ 391 return 0; 392} 393 394/* 395 * The callbacks structure (defined in pthread_shims.h) contains a collection 396 * of kernel functions that were not deemed sensible to expose as a KPI to all 397 * kernel extensions. So the kext is given them in the form of a structure of 398 * function pointers. 399 */ 400static struct pthread_callbacks_s pthread_callbacks = { 401 .version = PTHREAD_SHIMS_VERSION, 402 .config_thread_max = CONFIG_THREAD_MAX, 403 .get_task_threadmax = get_task_threadmax, 404 405 .proc_get_threadstart = proc_get_threadstart, 406 .proc_set_threadstart = proc_set_threadstart, 407 .proc_get_pthsize = proc_get_pthsize, 408 .proc_set_pthsize = proc_set_pthsize, 409 .proc_get_wqthread = proc_get_wqthread, 410 .proc_set_wqthread = proc_set_wqthread, 411 .proc_get_targconc = proc_get_targconc, 412 .proc_set_targconc = proc_set_targconc, 413 .proc_get_dispatchqueue_offset = proc_get_dispatchqueue_offset, 414 .proc_set_dispatchqueue_offset = proc_set_dispatchqueue_offset, 415 .proc_get_wqptr = proc_get_wqptr, 416 .proc_set_wqptr = proc_set_wqptr, 417 .proc_get_wqsize = proc_get_wqsize, 418 .proc_set_wqsize = proc_set_wqsize, 419 .proc_get_wqlockptr = proc_get_wqlockptr, 420 .proc_get_wqinitingptr = proc_get_wqinitingptr, 421 .proc_get_pthhash = proc_get_pthhash, 422 .proc_set_pthhash = proc_set_pthhash, 423 .proc_get_task = proc_get_task, 424 .proc_lock = proc_lock, 425 .proc_unlock = proc_unlock, 426 .proc_get_register = proc_get_register, 427 .proc_set_register = proc_set_register, 428 429 /* kernel IPI interfaces */ 430 .ipc_port_copyout_send = ipc_port_copyout_send, 431 .task_get_ipcspace = get_task_ipcspace, 432 .vm_map_page_info = vm_map_page_info, 433 .vm_map_switch = vm_map_switch, 434 .thread_set_wq_state32 = thread_set_wq_state32, 435 .thread_set_wq_state64 = thread_set_wq_state64, 436 437 .uthread_get_threadlist = uthread_get_threadlist, 438 .uthread_set_threadlist = uthread_set_threadlist, 439 .uthread_get_sigmask = uthread_get_sigmask, 440 .uthread_set_sigmask = uthread_set_sigmask, 441 .uthread_get_uukwe = uthread_get_uukwe, 442 .uthread_get_returnval = uthread_get_returnval, 443 .uthread_set_returnval = uthread_set_returnval, 444 .uthread_is_cancelled = uthread_is_cancelled, 445 446 .thread_exception_return = pthread_returning_to_userspace, 447 .thread_bootstrap_return = thread_bootstrap_return, 448 .unix_syscall_return = unix_syscall_return, 449 450 .absolutetime_to_microtime = absolutetime_to_microtime, 451 452 .proc_restore_workq_bgthreadpolicy = proc_restore_workq_bgthreadpolicy, 453 .proc_apply_workq_bgthreadpolicy = proc_apply_workq_bgthreadpolicy, 454 455 .get_bsdthread_info = (void*)get_bsdthread_info, 456 .thread_sched_call = thread_sched_call, 457 .thread_static_param = thread_static_param, 458 .thread_create_workq = thread_create_workq, 459 .thread_policy_set_internal = thread_policy_set_internal, 460 .thread_policy_get = thread_policy_get, 461 .thread_set_voucher_name = thread_set_voucher_name, 462 463 .thread_affinity_set = thread_affinity_set, 464 465 .zalloc = zalloc, 466 .zfree = zfree, 467 .zinit = zinit, 468 469 .__pthread_testcancel = __pthread_testcancel, 470 471 .mach_port_deallocate = mach_port_deallocate, 472 .semaphore_signal_internal_trap = semaphore_signal_internal_trap, 473 .current_map = _current_map, 474 .thread_create = thread_create, 475 .thread_resume = thread_resume, 476 477 .convert_thread_to_port = convert_thread_to_port, 478 .ml_get_max_cpus = (void*)ml_get_max_cpus, 479 480 481 .proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset, 482 .proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset, 483 484 .proc_get_stack_addr_hint = proc_get_stack_addr_hint, 485 .proc_set_stack_addr_hint = proc_set_stack_addr_hint, 486 .proc_get_pthread_tsd_offset = proc_get_pthread_tsd_offset, 487 .proc_set_pthread_tsd_offset = proc_set_pthread_tsd_offset, 488 489 .thread_set_tsd_base = thread_set_tsd_base, 490 491 .proc_usynch_get_requested_thread_qos = proc_usynch_get_requested_thread_qos, 492 .proc_usynch_thread_qos_add_override = proc_usynch_thread_qos_add_override, 493 .proc_usynch_thread_qos_remove_override = proc_usynch_thread_qos_remove_override, 494 495 .qos_main_thread_active = qos_main_thread_active, 496}; 497 498pthread_callbacks_t pthread_kern = &pthread_callbacks; 499pthread_functions_t pthread_functions = NULL; 500 501/* 502 * pthread_kext_register is called by pthread.kext upon load, it has to provide 503 * us with a function pointer table of pthread internal calls. In return, this 504 * file provides it with a table of function pointers it needs. 505 */ 506 507void 508pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks) 509{ 510 if (pthread_functions != NULL) { 511 panic("Re-initialisation of pthread kext callbacks."); 512 } 513 514 if (callbacks != NULL) { 515 *callbacks = &pthread_callbacks; 516 } else { 517 panic("pthread_kext_register called without callbacks pointer."); 518 } 519 520 if (fns) { 521 pthread_functions = fns; 522 } 523} 524