1/*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#define PTHREAD_INTERNAL 1
30
31#include <kern/debug.h>
32#include <kern/mach_param.h>
33#include <kern/sched_prim.h>
34#include <kern/task.h>
35#include <kern/thread.h>
36#include <kern/affinity.h>
37#include <kern/zalloc.h>
38#include <machine/machine_routines.h>
39#include <mach/task.h>
40#include <mach/thread_act.h>
41#include <sys/param.h>
42#include <sys/pthread_shims.h>
43#include <sys/proc_internal.h>
44#include <sys/sysproto.h>
45#include <sys/systm.h>
46#include <vm/vm_map.h>
47#include <vm/vm_protos.h>
48
49/* version number of the in-kernel shims given to pthread.kext */
50#define PTHREAD_SHIMS_VERSION 1
51
52/* old pthread code had definitions for these as they don't exist in headers */
53extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t);
54extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t);
55
56#define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \
57	static rettype \
58	get(structtype x) { \
59		return (x)->member; \
60	} \
61	static void \
62	set(structtype x, rettype y) { \
63		(x)->member = y; \
64	}
65
66PTHREAD_STRUCT_ACCESSOR(proc_get_threadstart, proc_set_threadstart, user_addr_t, struct proc*, p_threadstart);
67PTHREAD_STRUCT_ACCESSOR(proc_get_pthsize, proc_set_pthsize, int, struct proc*, p_pthsize);
68PTHREAD_STRUCT_ACCESSOR(proc_get_wqthread, proc_set_wqthread, user_addr_t, struct proc*, p_wqthread);
69PTHREAD_STRUCT_ACCESSOR(proc_get_targconc, proc_set_targconc, user_addr_t, struct proc*, p_targconc);
70PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset);
71PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset);
72PTHREAD_STRUCT_ACCESSOR(proc_get_wqptr, proc_set_wqptr, void*, struct proc*, p_wqptr);
73PTHREAD_STRUCT_ACCESSOR(proc_get_wqsize, proc_set_wqsize, int, struct proc*, p_wqsize);
74PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash);
75
76PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist);
77PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask);
78PTHREAD_STRUCT_ACCESSOR(uthread_get_returnval, uthread_set_returnval, int, struct uthread*, uu_rval[0]);
79
80static void
81pthread_returning_to_userspace(void)
82{
83	thread_exception_return();
84}
85
86static uint32_t
87get_task_threadmax(void) {
88	return task_threadmax;
89}
90
91static task_t
92proc_get_task(struct proc *p) {
93	return p->task;
94}
95
96static lck_spin_t*
97proc_get_wqlockptr(struct proc *p) {
98	return &(p->p_wqlock);
99}
100
101static boolean_t*
102proc_get_wqinitingptr(struct proc *p) {
103	return &(p->p_wqiniting);
104}
105
106static uint64_t
107proc_get_register(struct proc *p) {
108	return (p->p_lflag & P_LREGISTER);
109}
110
111static void
112proc_set_register(struct proc *p) {
113	proc_setregister(p);
114}
115
116static void*
117uthread_get_uukwe(struct uthread *t)
118{
119	return &t->uu_kevent.uu_kwe;
120}
121
122static int
123uthread_is_cancelled(struct uthread *t)
124{
125	return (t->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL;
126}
127
128static vm_map_t
129_current_map(void)
130{
131	return current_map();
132}
133
134
135/* kernel (core) to kext shims */
136
137void
138pthread_init(void)
139{
140	if (!pthread_functions) {
141		panic("pthread kernel extension not loaded (function table is NULL).");
142	}
143	pthread_functions->pthread_init();
144}
145
146int
147fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo)
148{
149	return pthread_functions->fill_procworkqueue(p, pwqinfo);
150}
151
152void
153workqueue_init_lock(proc_t p)
154{
155	pthread_functions->workqueue_init_lock(p);
156}
157
158void
159workqueue_destroy_lock(proc_t p)
160{
161	pthread_functions->workqueue_destroy_lock(p);
162}
163
164void
165workqueue_exit(struct proc *p)
166{
167	pthread_functions->workqueue_exit(p);
168}
169
170void
171workqueue_mark_exiting(struct proc *p)
172{
173	pthread_functions->workqueue_mark_exiting(p);
174}
175
176void
177workqueue_thread_yielded(void)
178{
179	pthread_functions->workqueue_thread_yielded();
180}
181
182sched_call_t
183workqueue_get_sched_callback(void)
184{
185	if (pthread_functions->workqueue_get_sched_callback) {
186		return pthread_functions->workqueue_get_sched_callback();
187	}
188	return NULL;
189}
190
191void
192pth_proc_hashinit(proc_t p)
193{
194	pthread_functions->pth_proc_hashinit(p);
195}
196
197void
198pth_proc_hashdelete(proc_t p)
199{
200	pthread_functions->pth_proc_hashdelete(p);
201}
202
203/* syscall shims */
204int
205bsdthread_create(struct proc *p, struct bsdthread_create_args *uap, user_addr_t *retval)
206{
207	return pthread_functions->bsdthread_create(p, uap->func, uap->func_arg, uap->stack, uap->pthread, uap->flags, retval);
208}
209
210int
211bsdthread_register(struct proc *p, struct bsdthread_register_args *uap, __unused int32_t *retval)
212{
213	return pthread_functions->bsdthread_register(p, uap->threadstart, uap->wqthread, uap->pthsize, uap->dummy_value,
214			uap->targetconc_ptr, uap->dispatchqueue_offset, retval);
215}
216
217int
218bsdthread_terminate(struct proc *p, struct bsdthread_terminate_args *uap, int32_t *retval)
219{
220	return pthread_functions->bsdthread_terminate(p, uap->stackaddr, uap->freesize, uap->port, uap->sem, retval);
221}
222
223int
224thread_selfid(struct proc *p, __unused struct thread_selfid_args *uap, uint64_t *retval)
225{
226	return pthread_functions->thread_selfid(p, retval);
227}
228
229int
230workq_kernreturn(struct proc *p, struct workq_kernreturn_args *uap, int32_t *retval)
231{
232	return pthread_functions->workq_kernreturn(p, uap->options, uap->item, uap->affinity, uap->prio, retval);
233}
234
235int
236workq_open(struct proc *p, __unused struct workq_open_args  *uap, int32_t *retval)
237{
238	return pthread_functions->workq_open(p, retval);
239}
240
241/* pthread synchroniser syscalls */
242
243int
244psynch_mutexwait(proc_t p, struct psynch_mutexwait_args *uap, uint32_t *retval)
245{
246	return pthread_functions->psynch_mutexwait(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval);
247}
248
249int
250psynch_mutexdrop(proc_t p, struct psynch_mutexdrop_args *uap, uint32_t *retval)
251{
252	return pthread_functions->psynch_mutexdrop(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval);
253}
254
255int
256psynch_cvbroad(proc_t p, struct psynch_cvbroad_args *uap, uint32_t *retval)
257{
258	return pthread_functions->psynch_cvbroad(p, uap->cv, uap->cvlsgen, uap->cvudgen, uap->flags, uap->mutex, uap->mugen, uap->tid, retval);
259}
260
261int
262psynch_cvsignal(proc_t p, struct psynch_cvsignal_args *uap, uint32_t *retval)
263{
264	return pthread_functions->psynch_cvsignal(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->thread_port, uap->mutex, uap->mugen, uap->tid, uap->flags, retval);
265}
266
267int
268psynch_cvwait(proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval)
269{
270	return pthread_functions->psynch_cvwait(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->mutex, uap->mugen, uap->flags, uap->sec, uap->nsec, retval);
271}
272
273int
274psynch_cvclrprepost(proc_t p, struct psynch_cvclrprepost_args * uap, int *retval)
275{
276	return pthread_functions->psynch_cvclrprepost(p, uap->cv, uap->cvgen, uap->cvugen, uap->cvsgen, uap->prepocnt, uap->preposeq, uap->flags, retval);
277}
278
279int
280psynch_rw_longrdlock(proc_t p, struct psynch_rw_longrdlock_args * uap,  uint32_t *retval)
281{
282	return pthread_functions->psynch_rw_longrdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
283}
284
285int
286psynch_rw_rdlock(proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval)
287{
288	return pthread_functions->psynch_rw_rdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
289}
290
291int
292psynch_rw_unlock(proc_t p, struct psynch_rw_unlock_args *uap, uint32_t *retval)
293{
294	return pthread_functions->psynch_rw_unlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
295}
296
297int
298psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args *uap, __unused uint32_t *retval)
299{
300	return ENOTSUP;
301}
302
303int
304psynch_rw_wrlock(proc_t p, struct psynch_rw_wrlock_args *uap, uint32_t *retval)
305{
306	return pthread_functions->psynch_rw_wrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
307}
308
309int
310psynch_rw_yieldwrlock(proc_t p, struct psynch_rw_yieldwrlock_args *uap, uint32_t *retval)
311{
312	return pthread_functions->psynch_rw_yieldwrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
313}
314
315int
316psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t *retval)
317{
318	return 0;
319}
320
321int
322psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int *retval)
323{
324	return 0;
325}
326
327/* unimplemented guard */
328
329// static void
330// unhooked_panic(void)
331// {
332// 	panic("pthread system call not hooked up");
333// }
334
335/*
336 * The callbacks structure (defined in pthread_shims.h) contains a collection
337 * of kernel functions that were not deemed sensible to expose as a KPI to all
338 * kernel extensions. So the kext is given them in the form of a structure of
339 * function pointers.
340 */
341static struct pthread_callbacks_s pthread_callbacks = {
342	.version = PTHREAD_SHIMS_VERSION,
343	.config_thread_max = CONFIG_THREAD_MAX,
344	.get_task_threadmax = get_task_threadmax,
345
346	.proc_get_threadstart = proc_get_threadstart,
347	.proc_set_threadstart = proc_set_threadstart,
348	.proc_get_pthsize = proc_get_pthsize,
349	.proc_set_pthsize = proc_set_pthsize,
350	.proc_get_wqthread = proc_get_wqthread,
351	.proc_set_wqthread = proc_set_wqthread,
352	.proc_get_targconc = proc_get_targconc,
353	.proc_set_targconc = proc_set_targconc,
354	.proc_get_dispatchqueue_offset = proc_get_dispatchqueue_offset,
355	.proc_set_dispatchqueue_offset = proc_set_dispatchqueue_offset,
356	.proc_get_wqptr = proc_get_wqptr,
357	.proc_set_wqptr = proc_set_wqptr,
358	.proc_get_wqsize = proc_get_wqsize,
359	.proc_set_wqsize = proc_set_wqsize,
360	.proc_get_wqlockptr = proc_get_wqlockptr,
361	.proc_get_wqinitingptr = proc_get_wqinitingptr,
362	.proc_get_pthhash = proc_get_pthhash,
363	.proc_set_pthhash = proc_set_pthhash,
364	.proc_get_task = proc_get_task,
365	.proc_lock = proc_lock,
366	.proc_unlock = proc_unlock,
367	.proc_get_register = proc_get_register,
368	.proc_set_register = proc_set_register,
369
370	/* kernel IPI interfaces */
371	.ipc_port_copyout_send = ipc_port_copyout_send,
372	.task_get_ipcspace = get_task_ipcspace,
373	.vm_map_page_info = vm_map_page_info,
374	.vm_map_switch = vm_map_switch,
375	.thread_set_wq_state32 = thread_set_wq_state32,
376	.thread_set_wq_state64 = thread_set_wq_state64,
377
378	.uthread_get_threadlist = uthread_get_threadlist,
379	.uthread_set_threadlist = uthread_set_threadlist,
380	.uthread_get_sigmask = uthread_get_sigmask,
381	.uthread_set_sigmask = uthread_set_sigmask,
382	.uthread_get_uukwe = uthread_get_uukwe,
383	.uthread_get_returnval = uthread_get_returnval,
384	.uthread_set_returnval = uthread_set_returnval,
385	.uthread_is_cancelled = uthread_is_cancelled,
386
387	.thread_exception_return = pthread_returning_to_userspace,
388	.thread_bootstrap_return = thread_bootstrap_return,
389	.unix_syscall_return = unix_syscall_return,
390
391	.absolutetime_to_microtime = absolutetime_to_microtime,
392
393	.proc_restore_workq_bgthreadpolicy = proc_restore_workq_bgthreadpolicy,
394	.proc_apply_workq_bgthreadpolicy = proc_apply_workq_bgthreadpolicy,
395
396	.get_bsdthread_info = (void*)get_bsdthread_info,
397	.thread_sched_call = thread_sched_call,
398	.thread_static_param = thread_static_param,
399	.thread_create_workq = thread_create_workq,
400	.thread_policy_set_internal = thread_policy_set_internal,
401
402	.thread_affinity_set = thread_affinity_set,
403
404	.zalloc = zalloc,
405	.zfree = zfree,
406	.zinit = zinit,
407
408	.__pthread_testcancel = __pthread_testcancel,
409
410	.mach_port_deallocate = mach_port_deallocate,
411	.semaphore_signal_internal_trap = semaphore_signal_internal_trap,
412	.current_map = _current_map,
413	.thread_create = thread_create,
414	.thread_resume = thread_resume,
415
416	.convert_thread_to_port = convert_thread_to_port,
417	.ml_get_max_cpus = (void*)ml_get_max_cpus,
418
419
420	.proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset,
421	.proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset,
422};
423
424pthread_callbacks_t pthread_kern = &pthread_callbacks;
425pthread_functions_t pthread_functions = NULL;
426
427/*
428 * pthread_kext_register is called by pthread.kext upon load, it has to provide
429 * us with a function pointer table of pthread internal calls. In return, this
430 * file provides it with a table of function pointers it needs.
431 */
432
433void
434pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks)
435{
436	if (pthread_functions != NULL) {
437		panic("Re-initialisation of pthread kext callbacks.");
438	}
439
440	if (callbacks != NULL) {
441		*callbacks = &pthread_callbacks;
442	} else {
443		panic("pthread_kext_register called without callbacks pointer.");
444	}
445
446	if (fns) {
447		pthread_functions = fns;
448	}
449}
450