1/* 2 * Copyright (c) 2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#define PTHREAD_INTERNAL 1 30 31#include <kern/debug.h> 32#include <kern/mach_param.h> 33#include <kern/sched_prim.h> 34#include <kern/task.h> 35#include <kern/thread.h> 36#include <kern/affinity.h> 37#include <kern/zalloc.h> 38#include <machine/machine_routines.h> 39#include <mach/task.h> 40#include <mach/thread_act.h> 41#include <sys/param.h> 42#include <sys/pthread_shims.h> 43#include <sys/proc_internal.h> 44#include <sys/sysproto.h> 45#include <sys/systm.h> 46#include <vm/vm_map.h> 47#include <vm/vm_protos.h> 48 49/* version number of the in-kernel shims given to pthread.kext */ 50#define PTHREAD_SHIMS_VERSION 1 51 52/* old pthread code had definitions for these as they don't exist in headers */ 53extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t); 54extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t); 55 56#define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \ 57 static rettype \ 58 get(structtype x) { \ 59 return (x)->member; \ 60 } \ 61 static void \ 62 set(structtype x, rettype y) { \ 63 (x)->member = y; \ 64 } 65 66PTHREAD_STRUCT_ACCESSOR(proc_get_threadstart, proc_set_threadstart, user_addr_t, struct proc*, p_threadstart); 67PTHREAD_STRUCT_ACCESSOR(proc_get_pthsize, proc_set_pthsize, int, struct proc*, p_pthsize); 68PTHREAD_STRUCT_ACCESSOR(proc_get_wqthread, proc_set_wqthread, user_addr_t, struct proc*, p_wqthread); 69PTHREAD_STRUCT_ACCESSOR(proc_get_targconc, proc_set_targconc, user_addr_t, struct proc*, p_targconc); 70PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset); 71PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset); 72PTHREAD_STRUCT_ACCESSOR(proc_get_wqptr, proc_set_wqptr, void*, struct proc*, p_wqptr); 73PTHREAD_STRUCT_ACCESSOR(proc_get_wqsize, proc_set_wqsize, int, struct proc*, p_wqsize); 74PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash); 75 76PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist); 77PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask); 78PTHREAD_STRUCT_ACCESSOR(uthread_get_returnval, uthread_set_returnval, int, struct uthread*, uu_rval[0]); 79 80static void 81pthread_returning_to_userspace(void) 82{ 83 thread_exception_return(); 84} 85 86static uint32_t 87get_task_threadmax(void) { 88 return task_threadmax; 89} 90 91static task_t 92proc_get_task(struct proc *p) { 93 return p->task; 94} 95 96static lck_spin_t* 97proc_get_wqlockptr(struct proc *p) { 98 return &(p->p_wqlock); 99} 100 101static boolean_t* 102proc_get_wqinitingptr(struct proc *p) { 103 return &(p->p_wqiniting); 104} 105 106static uint64_t 107proc_get_register(struct proc *p) { 108 return (p->p_lflag & P_LREGISTER); 109} 110 111static void 112proc_set_register(struct proc *p) { 113 proc_setregister(p); 114} 115 116static void* 117uthread_get_uukwe(struct uthread *t) 118{ 119 return &t->uu_kevent.uu_kwe; 120} 121 122static int 123uthread_is_cancelled(struct uthread *t) 124{ 125 return (t->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL; 126} 127 128static vm_map_t 129_current_map(void) 130{ 131 return current_map(); 132} 133 134 135/* kernel (core) to kext shims */ 136 137void 138pthread_init(void) 139{ 140 if (!pthread_functions) { 141 panic("pthread kernel extension not loaded (function table is NULL)."); 142 } 143 pthread_functions->pthread_init(); 144} 145 146int 147fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo) 148{ 149 return pthread_functions->fill_procworkqueue(p, pwqinfo); 150} 151 152void 153workqueue_init_lock(proc_t p) 154{ 155 pthread_functions->workqueue_init_lock(p); 156} 157 158void 159workqueue_destroy_lock(proc_t p) 160{ 161 pthread_functions->workqueue_destroy_lock(p); 162} 163 164void 165workqueue_exit(struct proc *p) 166{ 167 pthread_functions->workqueue_exit(p); 168} 169 170void 171workqueue_mark_exiting(struct proc *p) 172{ 173 pthread_functions->workqueue_mark_exiting(p); 174} 175 176void 177workqueue_thread_yielded(void) 178{ 179 pthread_functions->workqueue_thread_yielded(); 180} 181 182sched_call_t 183workqueue_get_sched_callback(void) 184{ 185 if (pthread_functions->workqueue_get_sched_callback) { 186 return pthread_functions->workqueue_get_sched_callback(); 187 } 188 return NULL; 189} 190 191void 192pth_proc_hashinit(proc_t p) 193{ 194 pthread_functions->pth_proc_hashinit(p); 195} 196 197void 198pth_proc_hashdelete(proc_t p) 199{ 200 pthread_functions->pth_proc_hashdelete(p); 201} 202 203/* syscall shims */ 204int 205bsdthread_create(struct proc *p, struct bsdthread_create_args *uap, user_addr_t *retval) 206{ 207 return pthread_functions->bsdthread_create(p, uap->func, uap->func_arg, uap->stack, uap->pthread, uap->flags, retval); 208} 209 210int 211bsdthread_register(struct proc *p, struct bsdthread_register_args *uap, __unused int32_t *retval) 212{ 213 return pthread_functions->bsdthread_register(p, uap->threadstart, uap->wqthread, uap->pthsize, uap->dummy_value, 214 uap->targetconc_ptr, uap->dispatchqueue_offset, retval); 215} 216 217int 218bsdthread_terminate(struct proc *p, struct bsdthread_terminate_args *uap, int32_t *retval) 219{ 220 return pthread_functions->bsdthread_terminate(p, uap->stackaddr, uap->freesize, uap->port, uap->sem, retval); 221} 222 223int 224thread_selfid(struct proc *p, __unused struct thread_selfid_args *uap, uint64_t *retval) 225{ 226 return pthread_functions->thread_selfid(p, retval); 227} 228 229int 230workq_kernreturn(struct proc *p, struct workq_kernreturn_args *uap, int32_t *retval) 231{ 232 return pthread_functions->workq_kernreturn(p, uap->options, uap->item, uap->affinity, uap->prio, retval); 233} 234 235int 236workq_open(struct proc *p, __unused struct workq_open_args *uap, int32_t *retval) 237{ 238 return pthread_functions->workq_open(p, retval); 239} 240 241/* pthread synchroniser syscalls */ 242 243int 244psynch_mutexwait(proc_t p, struct psynch_mutexwait_args *uap, uint32_t *retval) 245{ 246 return pthread_functions->psynch_mutexwait(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval); 247} 248 249int 250psynch_mutexdrop(proc_t p, struct psynch_mutexdrop_args *uap, uint32_t *retval) 251{ 252 return pthread_functions->psynch_mutexdrop(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval); 253} 254 255int 256psynch_cvbroad(proc_t p, struct psynch_cvbroad_args *uap, uint32_t *retval) 257{ 258 return pthread_functions->psynch_cvbroad(p, uap->cv, uap->cvlsgen, uap->cvudgen, uap->flags, uap->mutex, uap->mugen, uap->tid, retval); 259} 260 261int 262psynch_cvsignal(proc_t p, struct psynch_cvsignal_args *uap, uint32_t *retval) 263{ 264 return pthread_functions->psynch_cvsignal(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->thread_port, uap->mutex, uap->mugen, uap->tid, uap->flags, retval); 265} 266 267int 268psynch_cvwait(proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval) 269{ 270 return pthread_functions->psynch_cvwait(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->mutex, uap->mugen, uap->flags, uap->sec, uap->nsec, retval); 271} 272 273int 274psynch_cvclrprepost(proc_t p, struct psynch_cvclrprepost_args * uap, int *retval) 275{ 276 return pthread_functions->psynch_cvclrprepost(p, uap->cv, uap->cvgen, uap->cvugen, uap->cvsgen, uap->prepocnt, uap->preposeq, uap->flags, retval); 277} 278 279int 280psynch_rw_longrdlock(proc_t p, struct psynch_rw_longrdlock_args * uap, uint32_t *retval) 281{ 282 return pthread_functions->psynch_rw_longrdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 283} 284 285int 286psynch_rw_rdlock(proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval) 287{ 288 return pthread_functions->psynch_rw_rdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 289} 290 291int 292psynch_rw_unlock(proc_t p, struct psynch_rw_unlock_args *uap, uint32_t *retval) 293{ 294 return pthread_functions->psynch_rw_unlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 295} 296 297int 298psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args *uap, __unused uint32_t *retval) 299{ 300 return ENOTSUP; 301} 302 303int 304psynch_rw_wrlock(proc_t p, struct psynch_rw_wrlock_args *uap, uint32_t *retval) 305{ 306 return pthread_functions->psynch_rw_wrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 307} 308 309int 310psynch_rw_yieldwrlock(proc_t p, struct psynch_rw_yieldwrlock_args *uap, uint32_t *retval) 311{ 312 return pthread_functions->psynch_rw_yieldwrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); 313} 314 315int 316psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t *retval) 317{ 318 return 0; 319} 320 321int 322psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int *retval) 323{ 324 return 0; 325} 326 327/* unimplemented guard */ 328 329// static void 330// unhooked_panic(void) 331// { 332// panic("pthread system call not hooked up"); 333// } 334 335/* 336 * The callbacks structure (defined in pthread_shims.h) contains a collection 337 * of kernel functions that were not deemed sensible to expose as a KPI to all 338 * kernel extensions. So the kext is given them in the form of a structure of 339 * function pointers. 340 */ 341static struct pthread_callbacks_s pthread_callbacks = { 342 .version = PTHREAD_SHIMS_VERSION, 343 .config_thread_max = CONFIG_THREAD_MAX, 344 .get_task_threadmax = get_task_threadmax, 345 346 .proc_get_threadstart = proc_get_threadstart, 347 .proc_set_threadstart = proc_set_threadstart, 348 .proc_get_pthsize = proc_get_pthsize, 349 .proc_set_pthsize = proc_set_pthsize, 350 .proc_get_wqthread = proc_get_wqthread, 351 .proc_set_wqthread = proc_set_wqthread, 352 .proc_get_targconc = proc_get_targconc, 353 .proc_set_targconc = proc_set_targconc, 354 .proc_get_dispatchqueue_offset = proc_get_dispatchqueue_offset, 355 .proc_set_dispatchqueue_offset = proc_set_dispatchqueue_offset, 356 .proc_get_wqptr = proc_get_wqptr, 357 .proc_set_wqptr = proc_set_wqptr, 358 .proc_get_wqsize = proc_get_wqsize, 359 .proc_set_wqsize = proc_set_wqsize, 360 .proc_get_wqlockptr = proc_get_wqlockptr, 361 .proc_get_wqinitingptr = proc_get_wqinitingptr, 362 .proc_get_pthhash = proc_get_pthhash, 363 .proc_set_pthhash = proc_set_pthhash, 364 .proc_get_task = proc_get_task, 365 .proc_lock = proc_lock, 366 .proc_unlock = proc_unlock, 367 .proc_get_register = proc_get_register, 368 .proc_set_register = proc_set_register, 369 370 /* kernel IPI interfaces */ 371 .ipc_port_copyout_send = ipc_port_copyout_send, 372 .task_get_ipcspace = get_task_ipcspace, 373 .vm_map_page_info = vm_map_page_info, 374 .vm_map_switch = vm_map_switch, 375 .thread_set_wq_state32 = thread_set_wq_state32, 376 .thread_set_wq_state64 = thread_set_wq_state64, 377 378 .uthread_get_threadlist = uthread_get_threadlist, 379 .uthread_set_threadlist = uthread_set_threadlist, 380 .uthread_get_sigmask = uthread_get_sigmask, 381 .uthread_set_sigmask = uthread_set_sigmask, 382 .uthread_get_uukwe = uthread_get_uukwe, 383 .uthread_get_returnval = uthread_get_returnval, 384 .uthread_set_returnval = uthread_set_returnval, 385 .uthread_is_cancelled = uthread_is_cancelled, 386 387 .thread_exception_return = pthread_returning_to_userspace, 388 .thread_bootstrap_return = thread_bootstrap_return, 389 .unix_syscall_return = unix_syscall_return, 390 391 .absolutetime_to_microtime = absolutetime_to_microtime, 392 393 .proc_restore_workq_bgthreadpolicy = proc_restore_workq_bgthreadpolicy, 394 .proc_apply_workq_bgthreadpolicy = proc_apply_workq_bgthreadpolicy, 395 396 .get_bsdthread_info = (void*)get_bsdthread_info, 397 .thread_sched_call = thread_sched_call, 398 .thread_static_param = thread_static_param, 399 .thread_create_workq = thread_create_workq, 400 .thread_policy_set_internal = thread_policy_set_internal, 401 402 .thread_affinity_set = thread_affinity_set, 403 404 .zalloc = zalloc, 405 .zfree = zfree, 406 .zinit = zinit, 407 408 .__pthread_testcancel = __pthread_testcancel, 409 410 .mach_port_deallocate = mach_port_deallocate, 411 .semaphore_signal_internal_trap = semaphore_signal_internal_trap, 412 .current_map = _current_map, 413 .thread_create = thread_create, 414 .thread_resume = thread_resume, 415 416 .convert_thread_to_port = convert_thread_to_port, 417 .ml_get_max_cpus = (void*)ml_get_max_cpus, 418 419 420 .proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset, 421 .proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset, 422}; 423 424pthread_callbacks_t pthread_kern = &pthread_callbacks; 425pthread_functions_t pthread_functions = NULL; 426 427/* 428 * pthread_kext_register is called by pthread.kext upon load, it has to provide 429 * us with a function pointer table of pthread internal calls. In return, this 430 * file provides it with a table of function pointers it needs. 431 */ 432 433void 434pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks) 435{ 436 if (pthread_functions != NULL) { 437 panic("Re-initialisation of pthread kext callbacks."); 438 } 439 440 if (callbacks != NULL) { 441 *callbacks = &pthread_callbacks; 442 } else { 443 panic("pthread_kext_register called without callbacks pointer."); 444 } 445 446 if (fns) { 447 pthread_functions = fns; 448 } 449} 450