1#pragma once
2
3#include "atomic.h"
4#include "libc.h"
5#include "pthread_arch.h"
6#include <assert.h>
7#include <errno.h>
8#include <limits.h>
9#include <locale.h>
10#include <pthread.h>
11#include <signal.h>
12#include <sys/uio.h>
13#include <threads.h>
14
15#include <zircon/stack.h>
16#include <zircon/tls.h>
17#include <runtime/thread.h>
18#include <runtime/tls.h>
19
20#define pthread __pthread
21
22// This is what the thread pointer points to directly.  On TLS_ABOVE_TP
23// machines, the size of this is part of the ABI known to the compiler
24// and linker.
25typedef struct {
26    // The position of this pointer is part of the ABI on x86.
27    // It has the same value as the thread pointer itself.
28    uintptr_t tp;
29    void** dtv;
30} tcbhead_t;
31
32// The locations of these fields is part of the ABI known to the compiler.
33typedef struct {
34    uintptr_t stack_guard;
35    uintptr_t unsafe_sp;
36} tp_abi_t;
37
38struct tls_dtor;
39
40struct pthread {
41#ifndef TLS_ABOVE_TP
42    // These must be the very first members.
43    tcbhead_t head;
44    tp_abi_t abi;
45#endif
46
47    zxr_thread_t zxr_thread;
48
49    // The *_region fields describe whole memory regions reserved,
50    // including guard pages (for deallocation).  safe_stack and
51    // unsafe_stack describe just the actual stack block between the
52    // guards.
53    struct iovec tcb_region;
54    struct iovec safe_stack, safe_stack_region;
55    struct iovec unsafe_stack, unsafe_stack_region;
56
57    struct tls_dtor* tls_dtors;
58    void* tsd[PTHREAD_KEYS_MAX];
59    int tsd_used;
60    int errno_value;
61
62    void* sanitizer_hook;
63    void* start_arg;
64    void* (*start)(void*);
65    void* result;
66    locale_t locale;
67    char* dlerror_buf;
68    int dlerror_flag;
69
70#ifdef TLS_ABOVE_TP
71    // These must be the very last members.
72    tp_abi_t abi;
73    tcbhead_t head;
74#endif
75};
76
77#ifdef TLS_ABOVE_TP
78#define PTHREAD_TP_OFFSET offsetof(struct pthread, head)
79#else
80#define PTHREAD_TP_OFFSET 0
81#endif
82
83#define TP_OFFSETOF(field) \
84    ((ptrdiff_t)offsetof(struct pthread, field) - PTHREAD_TP_OFFSET)
85
86static_assert(TP_OFFSETOF(head) == 0,
87              "ABI tcbhead_t misplaced in struct pthread");
88
89#ifdef ABI_TCBHEAD_SIZE
90static_assert((sizeof(struct pthread) -
91               offsetof(struct pthread, head)) == ABI_TCBHEAD_SIZE,
92              "ABI tcbhead_t misplaced in struct pthread");
93#endif
94
95#if defined(__x86_64__) || defined(__aarch64__)
96// The tlsdesc.s assembly code assumes this, though it's not part of the ABI.
97static_assert(TP_OFFSETOF(head.dtv) == 8, "dtv misplaced in struct pthread");
98#endif
99
100static_assert(TP_OFFSETOF(abi.stack_guard) == ZX_TLS_STACK_GUARD_OFFSET,
101              "stack_guard not at ABI-mandated offset from thread pointer");
102static_assert(TP_OFFSETOF(abi.unsafe_sp) == ZX_TLS_UNSAFE_SP_OFFSET,
103              "unsafe_sp not at ABI-mandated offset from thread pointer");
104
105static inline void* pthread_to_tp(struct pthread* thread) {
106    return (void*)((char*)thread + PTHREAD_TP_OFFSET);
107}
108
109static inline struct pthread* tp_to_pthread(void* tp) {
110    return (struct pthread*)((char*)tp - PTHREAD_TP_OFFSET);
111}
112
113#define SIGALL_SET ((sigset_t*)(const unsigned long long[2]){-1, -1})
114
115#define PTHREAD_MUTEX_MASK (PTHREAD_MUTEX_RECURSIVE | PTHREAD_MUTEX_ERRORCHECK)
116// The bit used in the recursive and errorchecking cases, which track thread owners.
117#define PTHREAD_MUTEX_OWNED_LOCK_BIT 0x80000000
118#define PTHREAD_MUTEX_OWNED_LOCK_MASK 0x7fffffff
119
120extern void* __pthread_tsd_main[];
121extern volatile size_t __pthread_tsd_size;
122
123void* __tls_get_new(size_t*) ATTR_LIBC_VISIBILITY;
124
125static inline pthread_t __pthread_self(void) {
126    return tp_to_pthread(zxr_tp_get());
127}
128
129static inline thrd_t __thrd_current(void) {
130    return (thrd_t)__pthread_self();
131}
132
133static inline pid_t __thread_get_tid(void) {
134    // We rely on the fact that the high bit is not set. For now,
135    // let's incur the cost of this check, until we consider the
136    // userspace handle value representation completely baked.
137    pid_t id = zxr_thread_get_handle(&__pthread_self()->zxr_thread);
138    if (id & PTHREAD_MUTEX_OWNED_LOCK_BIT) {
139        __builtin_trap();
140    }
141    return id;
142}
143
144int __pthread_create(pthread_t* __restrict, const pthread_attr_t* __restrict,
145                     void* (*)(void*), void* __restrict) ATTR_LIBC_VISIBILITY;
146int __pthread_detach(pthread_t t) ATTR_LIBC_VISIBILITY;
147_Noreturn void __pthread_exit(void* result) ATTR_LIBC_VISIBILITY;
148int __pthread_join(pthread_t t, void** result) ATTR_LIBC_VISIBILITY;
149
150// Signal n (or all, for -1) threads on a pthread_cond_t or cnd_t.
151void __private_cond_signal(void* condvar, int n) ATTR_LIBC_VISIBILITY;
152
153// This is guaranteed to only return 0, EINVAL, or ETIMEDOUT.
154int __timedwait(atomic_int*, int, clockid_t, const struct timespec*)
155    ATTR_LIBC_VISIBILITY;
156
157// Loading a library can introduce more thread_local variables. Thread
158// allocation bases bookkeeping decisions based on the current state
159// of thread_locals in the program, so thread creation needs to be
160// inhibited by a concurrent dlopen. This lock implements that
161// exclusion.
162void __thread_allocation_inhibit(void) ATTR_LIBC_VISIBILITY;
163void __thread_allocation_release(void) ATTR_LIBC_VISIBILITY;
164
165void __thread_tsd_run_dtors(void) ATTR_LIBC_VISIBILITY;
166
167#define DEFAULT_PTHREAD_ATTR                                                  \
168    ((pthread_attr_t){                                                        \
169        ._a_stacksize = ZIRCON_DEFAULT_STACK_SIZE,                           \
170        ._a_guardsize = PAGE_SIZE,                                            \
171    })
172
173thrd_t __allocate_thread(size_t guard_size,
174                         size_t stack_size,
175                         const char* thread_name,
176                         char default_name[ZX_MAX_NAME_LEN])
177    __attribute__((nonnull(3))) ATTR_LIBC_VISIBILITY;
178
179pthread_t __init_main_thread(zx_handle_t thread_self) ATTR_LIBC_VISIBILITY;
180
181int __clock_gettime(clockid_t, struct timespec*) ATTR_LIBC_VISIBILITY;
182