1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2004 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31/* 32 * Define stubs for TLS internals so that programs and libraries can 33 * link. These functions will be replaced by functional versions at 34 * runtime from ld-elf.so.1. 35 */ 36 37#include <sys/cdefs.h> 38#include <sys/param.h> 39#include <stdlib.h> 40#include <string.h> 41#include <elf.h> 42#include <unistd.h> 43 44#include "rtld.h" 45#include "libc_private.h" 46 47#define tls_assert(cond) ((cond) ? (void) 0 : \ 48 (tls_msg(#cond ": assert failed: " __FILE__ ":" \ 49 __XSTRING(__LINE__) "\n"), abort())) 50#define tls_msg(s) write(STDOUT_FILENO, s, strlen(s)) 51 52/* Provided by jemalloc to avoid bootstrapping issues. */ 53void *__je_bootstrap_malloc(size_t size); 54void *__je_bootstrap_calloc(size_t num, size_t size); 55void __je_bootstrap_free(void *ptr); 56 57__weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 58__weak_reference(__libc_free_tls, _rtld_free_tls); 59 60#ifdef __i386__ 61 62__weak_reference(___libc_tls_get_addr, ___tls_get_addr); 63__attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 64 65#endif 66 67void * __libc_tls_get_addr(void *); 68__weak_reference(__libc_tls_get_addr, __tls_get_addr); 69 70void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 71void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 72void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 73void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 74 75#if defined(__amd64__) 76#define TLS_TCB_ALIGN 16 77#elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \ 78 defined(__mips__) || defined(__powerpc__) || defined(__riscv) 79#define TLS_TCB_ALIGN sizeof(void *) 80#else 81#error TLS_TCB_ALIGN undefined for target architecture 82#endif 83 84#if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \ 85 defined(__powerpc__) || defined(__riscv) 86#define TLS_VARIANT_I 87#endif 88#if defined(__i386__) || defined(__amd64__) 89#define TLS_VARIANT_II 90#endif 91 92#if defined(__mips__) || defined(__powerpc__) || defined(__riscv) 93#define DTV_OFFSET 0x8000 94#else 95#define DTV_OFFSET 0 96#endif 97 98#ifndef PIC 99 100static size_t libc_tls_static_space; 101static size_t libc_tls_init_size; 102static size_t libc_tls_init_align; 103static void *libc_tls_init; 104#endif 105 106void * 107__libc_tls_get_addr(void *vti) 108{ 109 Elf_Addr **dtvp, *dtv; 110 tls_index *ti; 111 112 dtvp = _get_tp(); 113 dtv = *dtvp; 114 ti = vti; 115 return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) + 116 TLS_DTV_OFFSET); 117} 118 119#ifdef __i386__ 120 121/* GNU ABI */ 122 123__attribute__((__regparm__(1))) 124void * 125___libc_tls_get_addr(void *vti) 126{ 127 return (__libc_tls_get_addr(vti)); 128} 129 130#endif 131 132#ifndef PIC 133 134static void * 135libc_malloc_aligned(size_t size, size_t align) 136{ 137 void *mem, *res; 138 139 if (align < sizeof(void *)) 140 align = sizeof(void *); 141 142 mem = __je_bootstrap_malloc(size + sizeof(void *) + align - 1); 143 res = (void *)roundup2((uintptr_t)mem + sizeof(void *), align); 144 *(void **)((uintptr_t)res - sizeof(void *)) = mem; 145 return (res); 146} 147 148static void 149libc_free_aligned(void *ptr) 150{ 151 void *mem; 152 uintptr_t x; 153 154 if (ptr == NULL) 155 return; 156 157 x = (uintptr_t)ptr; 158 x -= sizeof(void *); 159 mem = *(void **)x; 160 __je_bootstrap_free(mem); 161} 162 163#ifdef TLS_VARIANT_I 164 165/* 166 * There are two versions of variant I of TLS 167 * 168 * - ARM and aarch64 uses original variant I as is described in [1] and [2], 169 * where TP points to start of TCB followed by aligned TLS segment. 170 * Both TCB and TLS must be aligned to alignment of TLS section. The TCB[0] 171 * points to DTV vector and DTV values are real addresses (without bias). 172 * Note: for Local Exec TLS Model, the offsets from TP (TCB in this case) to 173 * TLS variables are computed by linker, so we cannot overalign TLS section. 174 * 175 * - MIPS, PowerPC and RISC-V use modified version of variant I, 176 * described in [3] where TP points (with bias) to TLS and TCB immediately 177 * precedes TLS without any alignment gap[4]. Only TLS should be aligned. 178 * The TCB[0] points to DTV vector and DTV values are biased by constant 179 * value (0x8000) from real addresses[5]. 180 * 181 * [1] Ulrich Drepper: ELF Handling for Thread-Local Storage 182 * www.akkadia.org/drepper/tls.pdf 183 * 184 * [2] ARM IHI 0045E: Addenda to, and Errata in, the ABI for the ARM(r) 185 * Architecture 186 * infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf 187 * 188 * [3] OpenPOWER: Power Architecture 64-Bit ELF V2 ABI Specification 189 * https://members.openpowerfoundation.org/document/dl/576 190 * 191 * [4] Its unclear if "without any alignment gap" is hard ABI requirement, 192 * but we must follow this rule due to suboptimal _set_tp() 193 * (aka <ARCH>_SET_TP) implementation. This function doesn't expect TP but 194 * TCB as argument. 195 * 196 * [5] I'm not able to validate "values are biased" assertions. 197 */ 198 199/* 200 * Return pointer to allocated TLS block 201 */ 202static void * 203get_tls_block_ptr(void *tcb, size_t tcbsize) 204{ 205 size_t extra_size, post_size, pre_size, tls_block_size; 206 207 /* Compute fragments sizes. */ 208 extra_size = tcbsize - TLS_TCB_SIZE; 209#if defined(__aarch64__) || defined(__arm__) 210 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 211#else 212 post_size = 0; 213#endif 214 tls_block_size = tcbsize + post_size; 215 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 216 tls_block_size; 217 218 return ((char *)tcb - pre_size - extra_size); 219} 220 221/* 222 * Free Static TLS using the Variant I method. The tcbsize 223 * and tcbalign parameters must be the same as those used to allocate 224 * the block. 225 */ 226void 227__libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 228{ 229 Elf_Addr *dtv; 230 Elf_Addr **tls; 231 232 tls = (Elf_Addr **)tcb; 233 dtv = tls[0]; 234 __je_bootstrap_free(dtv); 235 libc_free_aligned(get_tls_block_ptr(tcb, tcbsize)); 236} 237 238/* 239 * Allocate Static TLS using the Variant I method. 240 * 241 * To handle all above requirements, we setup the following layout for 242 * TLS block: 243 * (whole memory block is aligned with MAX(TLS_TCB_ALIGN, tls_init_align)) 244 * 245 * +----------+--------------+--------------+-----------+------------------+ 246 * | pre gap | extended TCB | TCB | post gap | TLS segment | 247 * | pre_size | extra_size | TLS_TCB_SIZE | post_size | tls_static_space | 248 * +----------+--------------+--------------+-----------+------------------+ 249 * 250 * where: 251 * extra_size is tcbsize - TLS_TCB_SIZE 252 * post_size is used to adjust TCB to TLS aligment for first version of TLS 253 * layout and is always 0 for second version. 254 * pre_size is used to adjust TCB aligment for first version and to adjust 255 * TLS alignment for second version. 256 * 257 */ 258void * 259__libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign) 260{ 261 Elf_Addr *dtv, **tcb; 262 char *tls_block, *tls; 263 size_t extra_size, maxalign, post_size, pre_size, tls_block_size; 264 265 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 266 return (oldtcb); 267 268 tls_assert(tcbalign >= TLS_TCB_ALIGN); 269 maxalign = MAX(tcbalign, libc_tls_init_align); 270 271 /* Compute fragmets sizes. */ 272 extra_size = tcbsize - TLS_TCB_SIZE; 273#if defined(__aarch64__) || defined(__arm__) 274 post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; 275#else 276 post_size = 0; 277#endif 278 tls_block_size = tcbsize + post_size; 279 pre_size = roundup2(tls_block_size, libc_tls_init_align) - 280 tls_block_size; 281 tls_block_size += pre_size + libc_tls_static_space; 282 283 /* Allocate whole TLS block */ 284 tls_block = libc_malloc_aligned(tls_block_size, maxalign); 285 if (tls_block == NULL) { 286 tls_msg("__libc_allocate_tls: Out of memory.\n"); 287 abort(); 288 } 289 memset(tls_block, 0, tls_block_size); 290 tcb = (Elf_Addr **)(tls_block + pre_size + extra_size); 291 tls = (char *)tcb + TLS_TCB_SIZE + post_size; 292 293 if (oldtcb != NULL) { 294 memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), 295 tls_block_size); 296 libc_free_aligned(oldtcb); 297 298 /* Adjust the DTV. */ 299 dtv = tcb[0]; 300 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET); 301 } else { 302 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 303 if (dtv == NULL) { 304 tls_msg("__libc_allocate_tls: Out of memory.\n"); 305 abort(); 306 } 307 /* Build the DTV. */ 308 tcb[0] = dtv; 309 dtv[0] = 1; /* Generation. */ 310 dtv[1] = 1; /* Segments count. */ 311 dtv[2] = (Elf_Addr)(tls + DTV_OFFSET); 312 313 if (libc_tls_init_size > 0) 314 memcpy(tls, libc_tls_init, libc_tls_init_size); 315 } 316 317 return (tcb); 318} 319 320#endif 321 322#ifdef TLS_VARIANT_II 323 324#define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 325 326/* 327 * Free Static TLS using the Variant II method. 328 */ 329void 330__libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 331{ 332 size_t size; 333 Elf_Addr* dtv; 334 Elf_Addr tlsstart, tlsend; 335 336 /* 337 * Figure out the size of the initial TLS block so that we can 338 * find stuff which ___tls_get_addr() allocated dynamically. 339 */ 340 tcbalign = MAX(tcbalign, libc_tls_init_align); 341 size = roundup2(libc_tls_static_space, tcbalign); 342 343 dtv = ((Elf_Addr**)tcb)[1]; 344 tlsend = (Elf_Addr) tcb; 345 tlsstart = tlsend - size; 346 libc_free_aligned((void*)tlsstart); 347 __je_bootstrap_free(dtv); 348} 349 350/* 351 * Allocate Static TLS using the Variant II method. 352 */ 353void * 354__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 355{ 356 size_t size; 357 char *tls; 358 Elf_Addr *dtv; 359 Elf_Addr segbase, oldsegbase; 360 361 tcbalign = MAX(tcbalign, libc_tls_init_align); 362 size = roundup2(libc_tls_static_space, tcbalign); 363 364 if (tcbsize < 2 * sizeof(Elf_Addr)) 365 tcbsize = 2 * sizeof(Elf_Addr); 366 tls = libc_malloc_aligned(size + tcbsize, tcbalign); 367 if (tls == NULL) { 368 tls_msg("__libc_allocate_tls: Out of memory.\n"); 369 abort(); 370 } 371 memset(tls, 0, size + tcbsize); 372 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 373 if (dtv == NULL) { 374 tls_msg("__libc_allocate_tls: Out of memory.\n"); 375 abort(); 376 } 377 378 segbase = (Elf_Addr)(tls + size); 379 ((Elf_Addr*)segbase)[0] = segbase; 380 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 381 382 dtv[0] = 1; 383 dtv[1] = 1; 384 dtv[2] = segbase - libc_tls_static_space; 385 386 if (oldtls) { 387 /* 388 * Copy the static TLS block over whole. 389 */ 390 oldsegbase = (Elf_Addr) oldtls; 391 memcpy((void *)(segbase - libc_tls_static_space), 392 (const void *)(oldsegbase - libc_tls_static_space), 393 libc_tls_static_space); 394 395 /* 396 * We assume that this block was the one we created with 397 * allocate_initial_tls(). 398 */ 399 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 400 } else { 401 memcpy((void *)(segbase - libc_tls_static_space), 402 libc_tls_init, libc_tls_init_size); 403 memset((void *)(segbase - libc_tls_static_space + 404 libc_tls_init_size), 0, 405 libc_tls_static_space - libc_tls_init_size); 406 } 407 408 return (void*) segbase; 409} 410 411#endif /* TLS_VARIANT_II */ 412 413#else 414 415void * 416__libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 417 size_t tcbalign __unused) 418{ 419 return (0); 420} 421 422void 423__libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 424 size_t tcbalign __unused) 425{ 426} 427 428#endif /* PIC */ 429 430extern char **environ; 431 432void 433_init_tls(void) 434{ 435#ifndef PIC 436 Elf_Addr *sp; 437 Elf_Auxinfo *aux, *auxp; 438 Elf_Phdr *phdr; 439 size_t phent, phnum; 440 int i; 441 void *tls; 442 443 sp = (Elf_Addr *) environ; 444 while (*sp++ != 0) 445 ; 446 aux = (Elf_Auxinfo *) sp; 447 phdr = NULL; 448 phent = phnum = 0; 449 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 450 switch (auxp->a_type) { 451 case AT_PHDR: 452 phdr = auxp->a_un.a_ptr; 453 break; 454 455 case AT_PHENT: 456 phent = auxp->a_un.a_val; 457 break; 458 459 case AT_PHNUM: 460 phnum = auxp->a_un.a_val; 461 break; 462 } 463 } 464 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 465 return; 466 467 for (i = 0; (unsigned) i < phnum; i++) { 468 if (phdr[i].p_type == PT_TLS) { 469 libc_tls_static_space = roundup2(phdr[i].p_memsz, 470 phdr[i].p_align); 471 libc_tls_init_size = phdr[i].p_filesz; 472 libc_tls_init_align = phdr[i].p_align; 473 libc_tls_init = (void *)phdr[i].p_vaddr; 474 break; 475 } 476 } 477 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 478 479 _set_tp(tls); 480#endif 481} 482