1133064Sdfr/*- 2133064Sdfr * Copyright (c) 2004 Doug Rabson 3133064Sdfr * All rights reserved. 4133064Sdfr * 5133064Sdfr * Redistribution and use in source and binary forms, with or without 6133064Sdfr * modification, are permitted provided that the following conditions 7133064Sdfr * are met: 8133064Sdfr * 1. Redistributions of source code must retain the above copyright 9133064Sdfr * notice, this list of conditions and the following disclaimer. 10133064Sdfr * 2. Redistributions in binary form must reproduce the above copyright 11133064Sdfr * notice, this list of conditions and the following disclaimer in the 12133064Sdfr * documentation and/or other materials provided with the distribution. 13133064Sdfr * 14133064Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15133064Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16133064Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17133064Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18133064Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19133064Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20133064Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21133064Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22133064Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23133064Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24133064Sdfr * SUCH DAMAGE. 25133064Sdfr * 26133064Sdfr * $FreeBSD: releng/11.0/lib/libc/gen/tls.c 298215 2016-04-18 16:25:37Z pfg $ 27133064Sdfr */ 28133064Sdfr 29133064Sdfr/* 30133064Sdfr * Define stubs for TLS internals so that programs and libraries can 31133064Sdfr * link. These functions will be replaced by functional versions at 32133064Sdfr * runtime from ld-elf.so.1. 33133064Sdfr */ 34133064Sdfr 35143921Sdavidxu#include <sys/cdefs.h> 36298215Spfg#include <sys/param.h> 37133754Sdfr#include <stdlib.h> 38133754Sdfr#include <string.h> 39133754Sdfr#include <elf.h> 40143921Sdavidxu 41133754Sdfr#include "libc_private.h" 42133064Sdfr 43234370Sjasone/* Provided by jemalloc to avoid bootstrapping issues. */ 44286866Sjasonevoid *__je_bootstrap_malloc(size_t size); 45286866Sjasonevoid *__je_bootstrap_calloc(size_t num, size_t size); 46286866Sjasonevoid __je_bootstrap_free(void *ptr); 47234370Sjasone 48143921Sdavidxu__weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 49143921Sdavidxu__weak_reference(__libc_free_tls, _rtld_free_tls); 50143921Sdavidxu 51143921Sdavidxu#ifdef __i386__ 52143921Sdavidxu 53143921Sdavidxu__weak_reference(___libc_tls_get_addr, ___tls_get_addr); 54143921Sdavidxu__attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 55143921Sdavidxu 56143921Sdavidxu#endif 57143921Sdavidxu 58143921Sdavidxuvoid * __libc_tls_get_addr(void *); 59143921Sdavidxu__weak_reference(__libc_tls_get_addr, __tls_get_addr); 60143921Sdavidxu 61143921Sdavidxuvoid *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 62143921Sdavidxuvoid _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 63143921Sdavidxuvoid *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 64143921Sdavidxuvoid __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 65143921Sdavidxu 66268351Smarcel#if defined(__amd64__) 67163118Skmacy#define TLS_TCB_ALIGN 16 68294227Sbr#elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \ 69294227Sbr defined(__mips__) || defined(__powerpc__) || defined(__riscv__) || \ 70294227Sbr defined(__sparc64__) 71163118Skmacy#define TLS_TCB_ALIGN sizeof(void *) 72163118Skmacy#else 73163118Skmacy#error TLS_TCB_ALIGN undefined for target architecture 74163118Skmacy#endif 75163118Skmacy 76294227Sbr#if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \ 77294227Sbr defined(__powerpc__) || defined(__riscv__) 78133754Sdfr#define TLS_VARIANT_I 79133754Sdfr#endif 80232582Sgonzo#if defined(__i386__) || defined(__amd64__) || defined(__sparc64__) 81133754Sdfr#define TLS_VARIANT_II 82133754Sdfr#endif 83133754Sdfr 84275004Semaste#ifndef PIC 85133754Sdfr 86133754Sdfrstatic size_t tls_static_space; 87133754Sdfrstatic size_t tls_init_size; 88133754Sdfrstatic void *tls_init; 89133754Sdfr#endif 90133754Sdfr 91133064Sdfr#ifdef __i386__ 92133064Sdfr 93143921Sdavidxu/* GNU ABI */ 94133064Sdfr 95133064Sdfr__attribute__((__regparm__(1))) 96133064Sdfrvoid * 97143921Sdavidxu___libc_tls_get_addr(void *ti __unused) 98133064Sdfr{ 99133064Sdfr return (0); 100133064Sdfr} 101133064Sdfr 102133064Sdfr#endif 103133064Sdfr 104133064Sdfrvoid * 105143921Sdavidxu__libc_tls_get_addr(void *ti __unused) 106133064Sdfr{ 107133064Sdfr return (0); 108133064Sdfr} 109133064Sdfr 110275004Semaste#ifndef PIC 111143921Sdavidxu 112133754Sdfr#ifdef TLS_VARIANT_I 113133754Sdfr 114161800Smarcel#define TLS_TCB_SIZE (2 * sizeof(void *)) 115161800Smarcel 116142560Sdavidxu/* 117142959Sdavidxu * Free Static TLS using the Variant I method. 118142560Sdavidxu */ 119133754Sdfrvoid 120161800Smarcel__libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 121133754Sdfr{ 122161800Smarcel Elf_Addr *dtv; 123161800Smarcel Elf_Addr **tls; 124133754Sdfr 125161800Smarcel tls = (Elf_Addr **)((Elf_Addr)tcb + tcbsize - TLS_TCB_SIZE); 126161800Smarcel dtv = tls[0]; 127286866Sjasone __je_bootstrap_free(dtv); 128286866Sjasone __je_bootstrap_free(tcb); 129133754Sdfr} 130133754Sdfr 131133754Sdfr/* 132133754Sdfr * Allocate Static TLS using the Variant I method. 133133754Sdfr */ 134133064Sdfrvoid * 135161800Smarcel__libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign __unused) 136133064Sdfr{ 137133754Sdfr Elf_Addr *dtv; 138161800Smarcel Elf_Addr **tls; 139161800Smarcel char *tcb; 140133754Sdfr 141161800Smarcel if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 142161800Smarcel return (oldtcb); 143133754Sdfr 144286866Sjasone tcb = __je_bootstrap_calloc(1, tls_static_space + tcbsize - TLS_TCB_SIZE); 145161800Smarcel tls = (Elf_Addr **)(tcb + tcbsize - TLS_TCB_SIZE); 146133754Sdfr 147161800Smarcel if (oldtcb != NULL) { 148203946Smarcel memcpy(tls, oldtcb, tls_static_space); 149286866Sjasone __je_bootstrap_free(oldtcb); 150133754Sdfr 151161800Smarcel /* Adjust the DTV. */ 152161800Smarcel dtv = tls[0]; 153161800Smarcel dtv[2] = (Elf_Addr)tls + TLS_TCB_SIZE; 154161800Smarcel } else { 155286866Sjasone dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 156161800Smarcel tls[0] = dtv; 157161800Smarcel dtv[0] = 1; 158161800Smarcel dtv[1] = 1; 159161800Smarcel dtv[2] = (Elf_Addr)tls + TLS_TCB_SIZE; 160133754Sdfr 161161800Smarcel if (tls_init_size > 0) 162161827Smarcel memcpy((void*)dtv[2], tls_init, tls_init_size); 163161800Smarcel if (tls_static_space > tls_init_size) 164161827Smarcel memset((void*)(dtv[2] + tls_init_size), 0, 165161827Smarcel tls_static_space - tls_init_size); 166133754Sdfr } 167133754Sdfr 168161800Smarcel return(tcb); 169133064Sdfr} 170133064Sdfr 171133754Sdfr#endif 172133754Sdfr 173133754Sdfr#ifdef TLS_VARIANT_II 174133754Sdfr 175161800Smarcel#define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 176161800Smarcel 177133754Sdfr/* 178133754Sdfr * Free Static TLS using the Variant II method. 179133754Sdfr */ 180133064Sdfrvoid 181143921Sdavidxu__libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 182133064Sdfr{ 183133754Sdfr size_t size; 184133754Sdfr Elf_Addr* dtv; 185133754Sdfr Elf_Addr tlsstart, tlsend; 186133754Sdfr 187133754Sdfr /* 188133754Sdfr * Figure out the size of the initial TLS block so that we can 189133754Sdfr * find stuff which ___tls_get_addr() allocated dynamically. 190133754Sdfr */ 191298215Spfg size = roundup2(tls_static_space, tcbalign); 192133754Sdfr 193133754Sdfr dtv = ((Elf_Addr**)tcb)[1]; 194133754Sdfr tlsend = (Elf_Addr) tcb; 195133754Sdfr tlsstart = tlsend - size; 196286866Sjasone __je_bootstrap_free((void*) tlsstart); 197286866Sjasone __je_bootstrap_free(dtv); 198133064Sdfr} 199133754Sdfr 200133754Sdfr/* 201133754Sdfr * Allocate Static TLS using the Variant II method. 202133754Sdfr */ 203133754Sdfrvoid * 204143921Sdavidxu__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 205133754Sdfr{ 206133754Sdfr size_t size; 207133754Sdfr char *tls; 208133754Sdfr Elf_Addr *dtv; 209133754Sdfr Elf_Addr segbase, oldsegbase; 210133754Sdfr 211298215Spfg size = roundup2(tls_static_space, tcbalign); 212133754Sdfr 213166995Skientzle if (tcbsize < 2 * sizeof(Elf_Addr)) 214166995Skientzle tcbsize = 2 * sizeof(Elf_Addr); 215286866Sjasone tls = __je_bootstrap_calloc(1, size + tcbsize); 216286866Sjasone dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr)); 217133754Sdfr 218133754Sdfr segbase = (Elf_Addr)(tls + size); 219133754Sdfr ((Elf_Addr*)segbase)[0] = segbase; 220133754Sdfr ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 221133754Sdfr 222133754Sdfr dtv[0] = 1; 223133754Sdfr dtv[1] = 1; 224133754Sdfr dtv[2] = segbase - tls_static_space; 225133754Sdfr 226133754Sdfr if (oldtls) { 227133754Sdfr /* 228133754Sdfr * Copy the static TLS block over whole. 229133754Sdfr */ 230133754Sdfr oldsegbase = (Elf_Addr) oldtls; 231133754Sdfr memcpy((void *)(segbase - tls_static_space), 232133754Sdfr (const void *)(oldsegbase - tls_static_space), 233133754Sdfr tls_static_space); 234133754Sdfr 235133754Sdfr /* 236133754Sdfr * We assume that this block was the one we created with 237133754Sdfr * allocate_initial_tls(). 238133754Sdfr */ 239133754Sdfr _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 240133754Sdfr } else { 241133754Sdfr memcpy((void *)(segbase - tls_static_space), 242133754Sdfr tls_init, tls_init_size); 243133754Sdfr memset((void *)(segbase - tls_static_space + tls_init_size), 244133754Sdfr 0, tls_static_space - tls_init_size); 245133754Sdfr } 246133754Sdfr 247133754Sdfr return (void*) segbase; 248143921Sdavidxu} 249143921Sdavidxu 250143921Sdavidxu#endif /* TLS_VARIANT_II */ 251143921Sdavidxu 252133754Sdfr#else 253143921Sdavidxu 254143921Sdavidxuvoid * 255143921Sdavidxu__libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 256143921Sdavidxu size_t tcbalign __unused) 257143921Sdavidxu{ 258133754Sdfr return (0); 259133754Sdfr} 260133754Sdfr 261143921Sdavidxuvoid 262143921Sdavidxu__libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 263143921Sdavidxu size_t tcbalign __unused) 264143921Sdavidxu{ 265143921Sdavidxu} 266133754Sdfr 267275004Semaste#endif /* PIC */ 268143921Sdavidxu 269143921Sdavidxuextern char **environ; 270143921Sdavidxu 271133754Sdfrvoid 272288029Srodrigc_init_tls(void) 273133754Sdfr{ 274275004Semaste#ifndef PIC 275133754Sdfr Elf_Addr *sp; 276133754Sdfr Elf_Auxinfo *aux, *auxp; 277133754Sdfr Elf_Phdr *phdr; 278133754Sdfr size_t phent, phnum; 279133754Sdfr int i; 280133949Sdfr void *tls; 281133754Sdfr 282133754Sdfr sp = (Elf_Addr *) environ; 283133754Sdfr while (*sp++ != 0) 284133754Sdfr ; 285133754Sdfr aux = (Elf_Auxinfo *) sp; 286297790Spfg phdr = NULL; 287133754Sdfr phent = phnum = 0; 288133754Sdfr for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 289133754Sdfr switch (auxp->a_type) { 290133754Sdfr case AT_PHDR: 291133754Sdfr phdr = auxp->a_un.a_ptr; 292133754Sdfr break; 293133754Sdfr 294133754Sdfr case AT_PHENT: 295133754Sdfr phent = auxp->a_un.a_val; 296133754Sdfr break; 297133754Sdfr 298133754Sdfr case AT_PHNUM: 299133754Sdfr phnum = auxp->a_un.a_val; 300133754Sdfr break; 301133754Sdfr } 302133754Sdfr } 303297790Spfg if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0) 304133754Sdfr return; 305133754Sdfr 306143921Sdavidxu for (i = 0; (unsigned) i < phnum; i++) { 307133754Sdfr if (phdr[i].p_type == PT_TLS) { 308298215Spfg tls_static_space = roundup2(phdr[i].p_memsz, 309133754Sdfr phdr[i].p_align); 310133754Sdfr tls_init_size = phdr[i].p_filesz; 311133754Sdfr tls_init = (void*) phdr[i].p_vaddr; 312133754Sdfr } 313133754Sdfr } 314133754Sdfr 315232582Sgonzo#ifdef TLS_VARIANT_I 316232582Sgonzo /* 317232582Sgonzo * tls_static_space should include space for TLS structure 318232582Sgonzo */ 319232582Sgonzo tls_static_space += TLS_TCB_SIZE; 320232582Sgonzo#endif 321232582Sgonzo 322163118Skmacy tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 323133754Sdfr 324133949Sdfr _set_tp(tls); 325133754Sdfr#endif 326133754Sdfr} 327