1#include <elf.h> 2#include <limits.h> 3#include <sys/mman.h> 4#include <string.h> 5#include <stddef.h> 6#include "pthread_impl.h" 7#include "libc.h" 8#include "atomic.h" 9#include "syscall.h" 10 11int __init_tp(void *p) 12{ 13 pthread_t td = p; 14 td->self = td; 15 int r = __set_thread_area(TP_ADJ(p)); 16 if (r < 0) return -1; 17 if (!r) libc.can_do_threads = 1; 18 td->tid = __syscall(SYS_set_tid_address, &td->tid); 19 td->locale = &libc.global_locale; 20 td->robust_list.head = &td->robust_list.head; 21 return 0; 22} 23 24static struct builtin_tls { 25 char c; 26 struct pthread pt; 27 void *space[16]; 28} builtin_tls[1]; 29#define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt) 30 31static struct tls_module main_tls; 32 33size_t libc_get_tls_size() 34{ 35 return libc.tls_size; 36} 37 38void libc_set_threaded(void) { 39 /// Once set, does not get unset. 40 a_store(&libc.threaded, 1); 41} 42 43int libc_get_threads_minus_1(void) { 44 return libc.threads_minus_1; 45} 46 47int libc_inc_threads_minus_1(void) { 48 return a_fetch_add(&libc.threads_minus_1, 1); 49} 50 51int libc_dec_threads_minus_1(void) { 52 return a_fetch_add(&libc.threads_minus_1, -1); 53} 54 55uintptr_t libc_tp_adj(uintptr_t ptr) { 56 return (uintptr_t)TP_ADJ(ptr); 57} 58 59void *__copy_tls(unsigned char *mem) 60{ 61 pthread_t td; 62 struct tls_module *p; 63 size_t i; 64 void **dtv; 65 66#ifdef TLS_ABOVE_TP 67 dtv = (void **)(mem + libc.tls_size) - (libc.tls_cnt + 1); 68 69 mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1); 70 td = (pthread_t)mem; 71 mem += sizeof(struct pthread); 72 73 for (i=1, p=libc.tls_head; p; i++, p=p->next) { 74 dtv[i] = mem + p->offset; 75 memcpy(dtv[i], p->image, p->len); 76 } 77#else 78 dtv = (void **)mem; 79 80 mem += libc.tls_size - sizeof(struct pthread); 81 mem -= (uintptr_t)mem & (libc.tls_align-1); 82 td = (pthread_t)mem; 83 84 for (i=1, p=libc.tls_head; p; i++, p=p->next) { 85 dtv[i] = mem - p->offset; 86 memcpy(dtv[i], p->image, p->len); 87 } 88#endif 89 dtv[0] = (void *)libc.tls_cnt; 90 td->dtv = td->dtv_copy = dtv; 91 return td; 92} 93 94#if ULONG_MAX == 0xffffffff 95typedef Elf32_Phdr Phdr; 96#else 97typedef Elf64_Phdr Phdr; 98#endif 99 100__attribute__((__weak__, __visibility__("hidden"))) 101extern const size_t _DYNAMIC[]; 102 103static void static_init_tls(size_t *aux) 104{ 105 unsigned char *p; 106 size_t n; 107 Phdr *phdr, *tls_phdr=0; 108 size_t base = 0; 109 void *mem; 110 111 for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) { 112 phdr = (void *)p; 113 if (phdr->p_type == PT_PHDR) 114 base = aux[AT_PHDR] - phdr->p_vaddr; 115 if (phdr->p_type == PT_DYNAMIC && _DYNAMIC) 116 base = (size_t)_DYNAMIC - phdr->p_vaddr; 117 if (phdr->p_type == PT_TLS) 118 tls_phdr = phdr; 119 } 120 121 if (tls_phdr) { 122 main_tls.image = (void *)(base + tls_phdr->p_vaddr); 123 main_tls.len = tls_phdr->p_filesz; 124 main_tls.size = tls_phdr->p_memsz; 125 main_tls.align = tls_phdr->p_align; 126 libc.tls_cnt = 1; 127 libc.tls_head = &main_tls; 128 } 129 130 main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image) 131 & (main_tls.align-1); 132 if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN; 133#ifndef TLS_ABOVE_TP 134 main_tls.offset = main_tls.size; 135#endif 136 137 libc.tls_align = main_tls.align; 138 libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread) 139 + main_tls.size + main_tls.align 140 + MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN; 141 142 if (libc.tls_size > sizeof builtin_tls) { 143#ifndef SYS_mmap2 144#define SYS_mmap2 SYS_mmap 145#endif 146 mem = (void *)__syscall( 147 SYS_mmap2, 148 0, libc.tls_size, PROT_READ|PROT_WRITE, 149 MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); 150 /* -4095...-1 cast to void * will crash on dereference anyway, 151 * so don't bloat the init code checking for error codes and 152 * explicitly calling a_crash(). */ 153 } else { 154 mem = builtin_tls; 155 } 156 157 /* Failure to initialize thread pointer is always fatal. */ 158 if (__init_tp(__copy_tls(mem)) < 0) 159 a_crash(); 160} 161 162weak_alias(static_init_tls, __init_tls); 163