1#include <elf.h>
2#include <limits.h>
3#include <sys/mman.h>
4#include <string.h>
5#include <stddef.h>
6#include "pthread_impl.h"
7#include "libc.h"
8#include "atomic.h"
9#include "syscall.h"
10
11int __init_tp(void *p)
12{
13	pthread_t td = p;
14	td->self = td;
15	int r = __set_thread_area(TP_ADJ(p));
16	if (r < 0) return -1;
17	if (!r) libc.can_do_threads = 1;
18	td->tid = __syscall(SYS_set_tid_address, &td->tid);
19	td->locale = &libc.global_locale;
20	td->robust_list.head = &td->robust_list.head;
21	return 0;
22}
23
24static struct builtin_tls {
25	char c;
26	struct pthread pt;
27	void *space[16];
28} builtin_tls[1];
29#define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
30
31static struct tls_module main_tls;
32
33size_t libc_get_tls_size()
34{
35    return libc.tls_size;
36}
37
38void libc_set_threaded(void) {
39	/// Once set, does not get unset.
40	a_store(&libc.threaded, 1);
41}
42
43int libc_get_threads_minus_1(void) {
44	return libc.threads_minus_1;
45}
46
47int libc_inc_threads_minus_1(void) {
48	return a_fetch_add(&libc.threads_minus_1, 1);
49}
50
51int libc_dec_threads_minus_1(void) {
52	return a_fetch_add(&libc.threads_minus_1, -1);
53}
54
55uintptr_t libc_tp_adj(uintptr_t ptr) {
56    return (uintptr_t)TP_ADJ(ptr);
57}
58
59void *__copy_tls(unsigned char *mem)
60{
61	pthread_t td;
62	struct tls_module *p;
63	size_t i;
64	void **dtv;
65
66#ifdef TLS_ABOVE_TP
67	dtv = (void **)(mem + libc.tls_size) - (libc.tls_cnt + 1);
68
69	mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1);
70	td = (pthread_t)mem;
71	mem += sizeof(struct pthread);
72
73	for (i=1, p=libc.tls_head; p; i++, p=p->next) {
74		dtv[i] = mem + p->offset;
75		memcpy(dtv[i], p->image, p->len);
76	}
77#else
78	dtv = (void **)mem;
79
80	mem += libc.tls_size - sizeof(struct pthread);
81	mem -= (uintptr_t)mem & (libc.tls_align-1);
82	td = (pthread_t)mem;
83
84	for (i=1, p=libc.tls_head; p; i++, p=p->next) {
85		dtv[i] = mem - p->offset;
86		memcpy(dtv[i], p->image, p->len);
87	}
88#endif
89	dtv[0] = (void *)libc.tls_cnt;
90	td->dtv = td->dtv_copy = dtv;
91	return td;
92}
93
94#if ULONG_MAX == 0xffffffff
95typedef Elf32_Phdr Phdr;
96#else
97typedef Elf64_Phdr Phdr;
98#endif
99
100__attribute__((__weak__, __visibility__("hidden")))
101extern const size_t _DYNAMIC[];
102
103static void static_init_tls(size_t *aux)
104{
105	unsigned char *p;
106	size_t n;
107	Phdr *phdr, *tls_phdr=0;
108	size_t base = 0;
109	void *mem;
110
111	for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) {
112		phdr = (void *)p;
113		if (phdr->p_type == PT_PHDR)
114			base = aux[AT_PHDR] - phdr->p_vaddr;
115		if (phdr->p_type == PT_DYNAMIC && _DYNAMIC)
116			base = (size_t)_DYNAMIC - phdr->p_vaddr;
117		if (phdr->p_type == PT_TLS)
118			tls_phdr = phdr;
119	}
120
121	if (tls_phdr) {
122		main_tls.image = (void *)(base + tls_phdr->p_vaddr);
123		main_tls.len = tls_phdr->p_filesz;
124		main_tls.size = tls_phdr->p_memsz;
125		main_tls.align = tls_phdr->p_align;
126		libc.tls_cnt = 1;
127		libc.tls_head = &main_tls;
128	}
129
130	main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image)
131		& (main_tls.align-1);
132	if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN;
133#ifndef TLS_ABOVE_TP
134	main_tls.offset = main_tls.size;
135#endif
136
137	libc.tls_align = main_tls.align;
138	libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread)
139		+ main_tls.size + main_tls.align
140		+ MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN;
141
142	if (libc.tls_size > sizeof builtin_tls) {
143#ifndef SYS_mmap2
144#define SYS_mmap2 SYS_mmap
145#endif
146		mem = (void *)__syscall(
147			SYS_mmap2,
148			0, libc.tls_size, PROT_READ|PROT_WRITE,
149			MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
150		/* -4095...-1 cast to void * will crash on dereference anyway,
151		 * so don't bloat the init code checking for error codes and
152		 * explicitly calling a_crash(). */
153	} else {
154		mem = builtin_tls;
155	}
156
157	/* Failure to initialize thread pointer is always fatal. */
158	if (__init_tp(__copy_tls(mem)) < 0)
159		a_crash();
160}
161
162weak_alias(static_init_tls, __init_tls);
163