1/*
2 * Copyright (c) 2003 Daniel M. Eischen <deischen@freebsd.org>
3 * Copyright (c) 1995-1998 John Birrell <jb@cimlogic.com.au>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the author nor the names of any co-contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * $FreeBSD$
31 */
32
33/* Allocate space for global thread variables here: */
34#define GLOBAL_PTHREAD_PRIVATE
35
36#include "namespace.h"
37#include <sys/param.h>
38#include <sys/types.h>
39#include <sys/signalvar.h>
40#include <machine/reg.h>
41
42#include <sys/ioctl.h>
43#include <sys/mount.h>
44#include <sys/uio.h>
45#include <sys/socket.h>
46#include <sys/event.h>
47#include <sys/stat.h>
48#include <sys/sysctl.h>
49#include <sys/time.h>
50#include <sys/ttycom.h>
51#include <sys/wait.h>
52#include <sys/mman.h>
53#include <dirent.h>
54#include <errno.h>
55#include <fcntl.h>
56#include <paths.h>
57#include <pthread.h>
58#include <pthread_np.h>
59#include <signal.h>
60#include <stdio.h>
61#include <stdlib.h>
62#include <string.h>
63#include <unistd.h>
64#include "un-namespace.h"
65
66#include "libc_private.h"
67#include "thr_private.h"
68
69int	__pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *);
70int	__pthread_mutex_lock(pthread_mutex_t *);
71int	__pthread_mutex_trylock(pthread_mutex_t *);
72void	_thread_init_hack(void);
73extern int _thread_state_running;
74
75static void init_private(void);
76static void init_main_thread(struct pthread *thread);
77
78/*
79 * All weak references used within libc should be in this table.
80 * This is so that static libraries will work.
81 */
82static void *references[] = {
83	&_accept,
84	&_bind,
85	&_close,
86	&_connect,
87	&_dup,
88	&_dup2,
89	&_execve,
90	&_fcntl,
91	&_flock,
92	&_flockfile,
93	&_fstat,
94	&_fstatfs,
95	&_fsync,
96	&_funlockfile,
97	&_getdirentries,
98	&_getlogin,
99	&_getpeername,
100	&_getsockname,
101	&_getsockopt,
102	&_ioctl,
103	&_kevent,
104	&_listen,
105	&_nanosleep,
106	&_open,
107	&_pthread_getspecific,
108	&_pthread_key_create,
109	&_pthread_key_delete,
110	&_pthread_mutex_destroy,
111	&_pthread_mutex_init,
112	&_pthread_mutex_lock,
113	&_pthread_mutex_trylock,
114	&_pthread_mutex_unlock,
115	&_pthread_mutexattr_init,
116	&_pthread_mutexattr_destroy,
117	&_pthread_mutexattr_settype,
118	&_pthread_once,
119	&_pthread_setspecific,
120	&_read,
121	&_readv,
122	&_recvfrom,
123	&_recvmsg,
124	&_select,
125	&_sendmsg,
126	&_sendto,
127	&_setsockopt,
128	&_sigaction,
129	&_sigprocmask,
130	&_sigsuspend,
131	&_socket,
132	&_socketpair,
133	&_thread_init_hack,
134	&_wait4,
135	&_write,
136	&_writev
137};
138
139/*
140 * These are needed when linking statically.  All references within
141 * libgcc (and in the future libc) to these routines are weak, but
142 * if they are not (strongly) referenced by the application or other
143 * libraries, then the actual functions will not be loaded.
144 */
145static void *libgcc_references[] = {
146	&_pthread_once,
147	&_pthread_key_create,
148	&_pthread_key_delete,
149	&_pthread_getspecific,
150	&_pthread_setspecific,
151	&_pthread_mutex_init,
152	&_pthread_mutex_destroy,
153	&_pthread_mutex_lock,
154	&_pthread_mutex_trylock,
155	&_pthread_mutex_unlock
156};
157
158#define	DUAL_ENTRY(entry)	\
159	(pthread_func_t)entry, (pthread_func_t)entry
160
161static pthread_func_t jmp_table[][2] = {
162	{DUAL_ENTRY(_pthread_atfork)},	/* PJT_ATFORK */
163	{DUAL_ENTRY(_pthread_attr_destroy)},	/* PJT_ATTR_DESTROY */
164	{DUAL_ENTRY(_pthread_attr_getdetachstate)},	/* PJT_ATTR_GETDETACHSTATE */
165	{DUAL_ENTRY(_pthread_attr_getguardsize)},	/* PJT_ATTR_GETGUARDSIZE */
166	{DUAL_ENTRY(_pthread_attr_getinheritsched)},	/* PJT_ATTR_GETINHERITSCHED */
167	{DUAL_ENTRY(_pthread_attr_getschedparam)},	/* PJT_ATTR_GETSCHEDPARAM */
168	{DUAL_ENTRY(_pthread_attr_getschedpolicy)},	/* PJT_ATTR_GETSCHEDPOLICY */
169	{DUAL_ENTRY(_pthread_attr_getscope)},	/* PJT_ATTR_GETSCOPE */
170	{DUAL_ENTRY(_pthread_attr_getstackaddr)},	/* PJT_ATTR_GETSTACKADDR */
171	{DUAL_ENTRY(_pthread_attr_getstacksize)},	/* PJT_ATTR_GETSTACKSIZE */
172	{DUAL_ENTRY(_pthread_attr_init)},	/* PJT_ATTR_INIT */
173	{DUAL_ENTRY(_pthread_attr_setdetachstate)},	/* PJT_ATTR_SETDETACHSTATE */
174	{DUAL_ENTRY(_pthread_attr_setguardsize)},	/* PJT_ATTR_SETGUARDSIZE */
175	{DUAL_ENTRY(_pthread_attr_setinheritsched)},	/* PJT_ATTR_SETINHERITSCHED */
176	{DUAL_ENTRY(_pthread_attr_setschedparam)},	/* PJT_ATTR_SETSCHEDPARAM */
177	{DUAL_ENTRY(_pthread_attr_setschedpolicy)},	/* PJT_ATTR_SETSCHEDPOLICY */
178	{DUAL_ENTRY(_pthread_attr_setscope)},	/* PJT_ATTR_SETSCOPE */
179	{DUAL_ENTRY(_pthread_attr_setstackaddr)},	/* PJT_ATTR_SETSTACKADDR */
180	{DUAL_ENTRY(_pthread_attr_setstacksize)},	/* PJT_ATTR_SETSTACKSIZE */
181	{DUAL_ENTRY(_pthread_cancel)},	/* PJT_CANCEL */
182	{DUAL_ENTRY(_pthread_cleanup_pop)},	/* PJT_CLEANUP_POP */
183	{DUAL_ENTRY(_pthread_cleanup_push)},	/* PJT_CLEANUP_PUSH */
184	{DUAL_ENTRY(_pthread_cond_broadcast)},	/* PJT_COND_BROADCAST */
185	{DUAL_ENTRY(_pthread_cond_destroy)},	/* PJT_COND_DESTROY */
186	{DUAL_ENTRY(_pthread_cond_init)},	/* PJT_COND_INIT */
187	{DUAL_ENTRY(_pthread_cond_signal)},	/* PJT_COND_SIGNAL */
188	{DUAL_ENTRY(_pthread_cond_timedwait)},	/* PJT_COND_TIMEDWAIT */
189	{(pthread_func_t)__pthread_cond_wait,
190	 (pthread_func_t)_pthread_cond_wait},	/* PJT_COND_WAIT */
191	{DUAL_ENTRY(_pthread_detach)},	/* PJT_DETACH */
192	{DUAL_ENTRY(_pthread_equal)},	/* PJT_EQUAL */
193	{DUAL_ENTRY(_pthread_exit)},	/* PJT_EXIT */
194	{DUAL_ENTRY(_pthread_getspecific)},	/* PJT_GETSPECIFIC */
195	{DUAL_ENTRY(_pthread_join)},	/* PJT_JOIN */
196	{DUAL_ENTRY(_pthread_key_create)},	/* PJT_KEY_CREATE */
197	{DUAL_ENTRY(_pthread_key_delete)},	/* PJT_KEY_DELETE*/
198	{DUAL_ENTRY(_pthread_kill)},	/* PJT_KILL */
199	{DUAL_ENTRY(_pthread_main_np)},		/* PJT_MAIN_NP */
200	{DUAL_ENTRY(_pthread_mutexattr_destroy)}, /* PJT_MUTEXATTR_DESTROY */
201	{DUAL_ENTRY(_pthread_mutexattr_init)},	/* PJT_MUTEXATTR_INIT */
202	{DUAL_ENTRY(_pthread_mutexattr_settype)}, /* PJT_MUTEXATTR_SETTYPE */
203	{DUAL_ENTRY(_pthread_mutex_destroy)},	/* PJT_MUTEX_DESTROY */
204	{DUAL_ENTRY(_pthread_mutex_init)},	/* PJT_MUTEX_INIT */
205	{(pthread_func_t)__pthread_mutex_lock,
206	 (pthread_func_t)_pthread_mutex_lock},	/* PJT_MUTEX_LOCK */
207	{(pthread_func_t)__pthread_mutex_trylock,
208	 (pthread_func_t)_pthread_mutex_trylock},/* PJT_MUTEX_TRYLOCK */
209	{DUAL_ENTRY(_pthread_mutex_unlock)},	/* PJT_MUTEX_UNLOCK */
210	{DUAL_ENTRY(_pthread_once)},		/* PJT_ONCE */
211	{DUAL_ENTRY(_pthread_rwlock_destroy)},	/* PJT_RWLOCK_DESTROY */
212	{DUAL_ENTRY(_pthread_rwlock_init)},	/* PJT_RWLOCK_INIT */
213	{DUAL_ENTRY(_pthread_rwlock_rdlock)},	/* PJT_RWLOCK_RDLOCK */
214	{DUAL_ENTRY(_pthread_rwlock_tryrdlock)},/* PJT_RWLOCK_TRYRDLOCK */
215	{DUAL_ENTRY(_pthread_rwlock_trywrlock)},/* PJT_RWLOCK_TRYWRLOCK */
216	{DUAL_ENTRY(_pthread_rwlock_unlock)},	/* PJT_RWLOCK_UNLOCK */
217	{DUAL_ENTRY(_pthread_rwlock_wrlock)},	/* PJT_RWLOCK_WRLOCK */
218	{DUAL_ENTRY(_pthread_self)},		/* PJT_SELF */
219	{DUAL_ENTRY(_pthread_setcancelstate)},	/* PJT_SETCANCELSTATE */
220	{DUAL_ENTRY(_pthread_setcanceltype)},	/* PJT_SETCANCELTYPE */
221	{DUAL_ENTRY(_pthread_setspecific)},	/* PJT_SETSPECIFIC */
222	{DUAL_ENTRY(_pthread_sigmask)},		/* PJT_SIGMASK */
223	{DUAL_ENTRY(_pthread_testcancel)}	/* PJT_TESTCANCEL */
224};
225
226static int	init_once = 0;
227
228/*
229 * Threaded process initialization.
230 *
231 * This is only called under two conditions:
232 *
233 *   1) Some thread routines have detected that the library hasn't yet
234 *      been initialized (_thr_initial == NULL && curthread == NULL), or
235 *
236 *   2) An explicit call to reinitialize after a fork (indicated
237 *      by curthread != NULL)
238 */
239void
240_libpthread_init(struct pthread *curthread)
241{
242	int fd;
243
244	/* Check if this function has already been called: */
245	if ((_thr_initial != NULL) && (curthread == NULL))
246		/* Only initialize the threaded application once. */
247		return;
248
249	/*
250	 * Make gcc quiescent about {,libgcc_}references not being
251	 * referenced:
252	 */
253	if ((references[0] == NULL) || (libgcc_references[0] == NULL))
254		PANIC("Failed loading mandatory references in _thread_init");
255
256	/* Pull debug symbols in for static binary */
257	_thread_state_running = PS_RUNNING;
258
259	/*
260	 * Check the size of the jump table to make sure it is preset
261	 * with the correct number of entries.
262	 */
263	if (sizeof(jmp_table) != (sizeof(pthread_func_t) * PJT_MAX * 2))
264		PANIC("Thread jump table not properly initialized");
265	memcpy(__thr_jtable, jmp_table, sizeof(jmp_table));
266
267	/*
268	 * Check for the special case of this process running as
269	 * or in place of init as pid = 1:
270	 */
271	if ((_thr_pid = getpid()) == 1) {
272		/*
273		 * Setup a new session for this process which is
274		 * assumed to be running as root.
275		 */
276		if (setsid() == -1)
277			PANIC("Can't set session ID");
278		if (revoke(_PATH_CONSOLE) != 0)
279			PANIC("Can't revoke console");
280		if ((fd = __sys_open(_PATH_CONSOLE, O_RDWR)) < 0)
281			PANIC("Can't open console");
282		if (setlogin("root") == -1)
283			PANIC("Can't set login to root");
284		if (__sys_ioctl(fd, TIOCSCTTY, (char *) NULL) == -1)
285			PANIC("Can't set controlling terminal");
286	}
287
288	/* Initialize pthread private data. */
289	init_private();
290	_kse_init();
291
292	/* Initialize the initial kse and kseg. */
293	_kse_initial = _kse_alloc(NULL, _thread_scope_system > 0);
294	if (_kse_initial == NULL)
295		PANIC("Can't allocate initial kse.");
296	_kse_initial->k_kseg = _kseg_alloc(NULL);
297	if (_kse_initial->k_kseg == NULL)
298		PANIC("Can't allocate initial kseg.");
299	_kse_initial->k_kseg->kg_flags |= KGF_SINGLE_THREAD;
300	_kse_initial->k_schedq = &_kse_initial->k_kseg->kg_schedq;
301
302	TAILQ_INSERT_TAIL(&_kse_initial->k_kseg->kg_kseq, _kse_initial, k_kgqe);
303	_kse_initial->k_kseg->kg_ksecount = 1;
304
305	/* Set the initial thread. */
306	if (curthread == NULL) {
307		/* Create and initialize the initial thread. */
308		curthread = _thr_alloc(NULL);
309		if (curthread == NULL)
310			PANIC("Can't allocate initial thread");
311		_thr_initial = curthread;
312		init_main_thread(curthread);
313	} else {
314		/*
315		 * The initial thread is the current thread.  It is
316		 * assumed that the current thread is already initialized
317		 * because it is left over from a fork().
318		 */
319		_thr_initial = curthread;
320	}
321	_kse_initial->k_kseg->kg_threadcount = 0;
322	_thr_initial->kse = _kse_initial;
323	_thr_initial->kseg = _kse_initial->k_kseg;
324	_thr_initial->active = 1;
325
326	/*
327	 * Add the thread to the thread list and to the KSEG's thread
328         * queue.
329	 */
330	THR_LIST_ADD(_thr_initial);
331	KSEG_THRQ_ADD(_kse_initial->k_kseg, _thr_initial);
332
333	/* Setup the KSE/thread specific data for the current KSE/thread. */
334	_thr_initial->kse->k_curthread = _thr_initial;
335	_kcb_set(_thr_initial->kse->k_kcb);
336	_tcb_set(_thr_initial->kse->k_kcb, _thr_initial->tcb);
337	_thr_initial->kse->k_flags |= KF_INITIALIZED;
338
339	_thr_signal_init();
340	_kse_critical_leave(&_thr_initial->tcb->tcb_tmbx);
341	/*
342	 * activate threaded mode as soon as possible if we are
343	 * being debugged
344	 */
345	if (_libkse_debug)
346		_kse_setthreaded(1);
347}
348
349/*
350 * This function and pthread_create() do a lot of the same things.
351 * It'd be nice to consolidate the common stuff in one place.
352 */
353static void
354init_main_thread(struct pthread *thread)
355{
356	/* Setup the thread attributes. */
357	thread->attr = _pthread_attr_default;
358	thread->attr.flags |= PTHREAD_SCOPE_SYSTEM;
359	/*
360	 * Set up the thread stack.
361	 *
362	 * Create a red zone below the main stack.  All other stacks
363	 * are constrained to a maximum size by the parameters
364	 * passed to mmap(), but this stack is only limited by
365	 * resource limits, so this stack needs an explicitly mapped
366	 * red zone to protect the thread stack that is just beyond.
367	 */
368	if (mmap((void *)((uintptr_t)_usrstack - _thr_stack_initial -
369	    _thr_guard_default), _thr_guard_default, 0, MAP_ANON,
370	    -1, 0) == MAP_FAILED)
371		PANIC("Cannot allocate red zone for initial thread");
372
373	/*
374	 * Mark the stack as an application supplied stack so that it
375	 * isn't deallocated.
376	 *
377	 * XXX - I'm not sure it would hurt anything to deallocate
378	 *       the main thread stack because deallocation doesn't
379	 *       actually free() it; it just puts it in the free
380	 *       stack queue for later reuse.
381	 */
382	thread->attr.stackaddr_attr = (void *)((uintptr_t)_usrstack -
383	     _thr_stack_initial);
384	thread->attr.stacksize_attr = _thr_stack_initial;
385	thread->attr.guardsize_attr = _thr_guard_default;
386	thread->attr.flags |= THR_STACK_USER;
387
388	/*
389	 * Write a magic value to the thread structure
390	 * to help identify valid ones:
391	 */
392	thread->magic = THR_MAGIC;
393
394	thread->slice_usec = -1;
395	thread->cancelflags = PTHREAD_CANCEL_ENABLE | PTHREAD_CANCEL_DEFERRED;
396	thread->name = strdup("initial thread");
397
398	/* Initialize the thread for signals: */
399	SIGEMPTYSET(thread->sigmask);
400
401	/*
402	 * Set up the thread mailbox.  The threads saved context
403	 * is also in the mailbox.
404	 */
405	thread->tcb->tcb_tmbx.tm_udata = thread;
406	thread->tcb->tcb_tmbx.tm_context.uc_stack.ss_size =
407	    thread->attr.stacksize_attr;
408	thread->tcb->tcb_tmbx.tm_context.uc_stack.ss_sp =
409	    thread->attr.stackaddr_attr;
410
411	/* Default the priority of the initial thread: */
412	thread->base_priority = THR_DEFAULT_PRIORITY;
413	thread->active_priority = THR_DEFAULT_PRIORITY;
414	thread->inherited_priority = 0;
415
416	/* Initialize the mutex queue: */
417	TAILQ_INIT(&thread->mutexq);
418
419	/* Initialize hooks in the thread structure: */
420	thread->specific = NULL;
421	thread->cleanup = NULL;
422	thread->flags = 0;
423	thread->sigbackout = NULL;
424	thread->continuation = NULL;
425
426	thread->state = PS_RUNNING;
427	thread->uniqueid = 0;
428}
429
430static void
431init_private(void)
432{
433	struct clockinfo clockinfo;
434	size_t len;
435	int mib[2];
436
437	/*
438	 * Avoid reinitializing some things if they don't need to be,
439	 * e.g. after a fork().
440	 */
441	if (init_once == 0) {
442		/* Find the stack top */
443		mib[0] = CTL_KERN;
444		mib[1] = KERN_USRSTACK;
445		len = sizeof (_usrstack);
446		if (sysctl(mib, 2, &_usrstack, &len, NULL, 0) == -1)
447			PANIC("Cannot get kern.usrstack from sysctl");
448		/* Get the kernel clockrate: */
449		mib[0] = CTL_KERN;
450		mib[1] = KERN_CLOCKRATE;
451		len = sizeof (struct clockinfo);
452		if (sysctl(mib, 2, &clockinfo, &len, NULL, 0) == 0)
453			_clock_res_usec = 1000000 / clockinfo.stathz;
454		else
455			_clock_res_usec = CLOCK_RES_USEC;
456
457		_thr_page_size = getpagesize();
458		_thr_guard_default = _thr_page_size;
459		if (sizeof(void *) == 8) {
460			_thr_stack_default = THR_STACK64_DEFAULT;
461			_thr_stack_initial = THR_STACK64_INITIAL;
462		}
463		else {
464			_thr_stack_default = THR_STACK32_DEFAULT;
465			_thr_stack_initial = THR_STACK32_INITIAL;
466		}
467		_pthread_attr_default.guardsize_attr = _thr_guard_default;
468		_pthread_attr_default.stacksize_attr = _thr_stack_default;
469		TAILQ_INIT(&_thr_atfork_list);
470		init_once = 1;	/* Don't do this again. */
471	} else {
472		/*
473		 * Destroy the locks before creating them.  We don't
474		 * know what state they are in so it is better to just
475		 * recreate them.
476		 */
477		_lock_destroy(&_thread_signal_lock);
478		_lock_destroy(&_mutex_static_lock);
479		_lock_destroy(&_rwlock_static_lock);
480		_lock_destroy(&_keytable_lock);
481	}
482
483	/* Initialize everything else. */
484	TAILQ_INIT(&_thread_list);
485	TAILQ_INIT(&_thread_gc_list);
486	_pthread_mutex_init(&_thr_atfork_mutex, NULL);
487
488	/*
489	 * Initialize the lock for temporary installation of signal
490	 * handlers (to support sigwait() semantics) and for the
491	 * process signal mask and pending signal sets.
492	 */
493	if (_lock_init(&_thread_signal_lock, LCK_ADAPTIVE,
494	    _kse_lock_wait, _kse_lock_wakeup, calloc) != 0)
495		PANIC("Cannot initialize _thread_signal_lock");
496	if (_lock_init(&_mutex_static_lock, LCK_ADAPTIVE,
497	    _thr_lock_wait, _thr_lock_wakeup, calloc) != 0)
498		PANIC("Cannot initialize mutex static init lock");
499	if (_lock_init(&_rwlock_static_lock, LCK_ADAPTIVE,
500	    _thr_lock_wait, _thr_lock_wakeup, calloc) != 0)
501		PANIC("Cannot initialize rwlock static init lock");
502	if (_lock_init(&_keytable_lock, LCK_ADAPTIVE,
503	    _thr_lock_wait, _thr_lock_wakeup, calloc) != 0)
504		PANIC("Cannot initialize thread specific keytable lock");
505	_thr_spinlock_init();
506
507	/* Clear pending signals and get the process signal mask. */
508	SIGEMPTYSET(_thr_proc_sigpending);
509
510	/* Are we in M:N mode (default) or 1:1 mode? */
511#ifdef SYSTEM_SCOPE_ONLY
512	_thread_scope_system = 1;
513#else
514	if (getenv("LIBPTHREAD_SYSTEM_SCOPE") != NULL)
515		_thread_scope_system = 1;
516	else if (getenv("LIBPTHREAD_PROCESS_SCOPE") != NULL)
517		_thread_scope_system = -1;
518#endif
519	if (getenv("LIBPTHREAD_DEBUG") != NULL)
520		_thr_debug_flags |= DBG_INFO_DUMP;
521
522	/*
523	 * _thread_list_lock and _kse_count are initialized
524	 * by _kse_init()
525	 */
526}
527