1/*	$NetBSD: tls.c,v 1.19 2023/06/07 13:50:04 joerg Exp $	*/
2/*-
3 * Copyright (c) 2011 The NetBSD Foundation, Inc.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The NetBSD Foundation
7 * by Joerg Sonnenberger.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32__RCSID("$NetBSD: tls.c,v 1.19 2023/06/07 13:50:04 joerg Exp $");
33
34#include <sys/param.h>
35#include <sys/ucontext.h>
36#include <lwp.h>
37#include <stdalign.h>
38#include <stddef.h>
39#include <string.h>
40#include "debug.h"
41#include "rtld.h"
42
43#if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
44
45static struct tls_tcb *_rtld_tls_allocate_locked(void);
46static void *_rtld_tls_module_allocate(struct tls_tcb *, size_t);
47
48#ifndef TLS_DTV_OFFSET
49#define	TLS_DTV_OFFSET	0
50#endif
51
52static size_t _rtld_tls_static_space;	/* Static TLS space allocated */
53static size_t _rtld_tls_static_offset;	/* Next offset for static TLS to use */
54size_t _rtld_tls_dtv_generation = 1;
55size_t _rtld_tls_max_index = 1;
56
57#define	DTV_GENERATION(dtv)		((size_t)((dtv)[0]))
58#define	DTV_MAX_INDEX(dtv)		((size_t)((dtv)[-1]))
59#define	SET_DTV_GENERATION(dtv, val)	(dtv)[0] = (void *)(size_t)(val)
60#define	SET_DTV_MAX_INDEX(dtv, val)	(dtv)[-1] = (void *)(size_t)(val)
61
62void *
63_rtld_tls_get_addr(void *tls, size_t idx, size_t offset)
64{
65	struct tls_tcb *tcb = tls;
66	void **dtv, **new_dtv;
67	sigset_t mask;
68
69	_rtld_exclusive_enter(&mask);
70
71	dtv = tcb->tcb_dtv;
72
73	if (__predict_false(DTV_GENERATION(dtv) != _rtld_tls_dtv_generation)) {
74		size_t to_copy = DTV_MAX_INDEX(dtv);
75
76		new_dtv = xcalloc((2 + _rtld_tls_max_index) * sizeof(*dtv));
77		++new_dtv;
78		if (to_copy > _rtld_tls_max_index)
79			to_copy = _rtld_tls_max_index;
80		memcpy(new_dtv + 1, dtv + 1, to_copy * sizeof(*dtv));
81		xfree(dtv - 1);
82		dtv = tcb->tcb_dtv = new_dtv;
83		SET_DTV_MAX_INDEX(dtv, _rtld_tls_max_index);
84		SET_DTV_GENERATION(dtv, _rtld_tls_dtv_generation);
85	}
86
87	if (__predict_false(dtv[idx] == NULL))
88		dtv[idx] = _rtld_tls_module_allocate(tcb, idx);
89
90	_rtld_exclusive_exit(&mask);
91
92	return (uint8_t *)dtv[idx] + offset;
93}
94
95void
96_rtld_tls_initial_allocation(void)
97{
98	struct tls_tcb *tcb;
99
100	_rtld_tls_static_space = _rtld_tls_static_offset +
101	    RTLD_STATIC_TLS_RESERVATION;
102
103#ifndef __HAVE_TLS_VARIANT_I
104	_rtld_tls_static_space = roundup2(_rtld_tls_static_space,
105	    alignof(max_align_t));
106#endif
107	dbg(("_rtld_tls_static_space %zu", _rtld_tls_static_space));
108
109	tcb = _rtld_tls_allocate_locked();
110#ifdef __HAVE___LWP_SETTCB
111	__lwp_settcb(tcb);
112#else
113	_lwp_setprivate(tcb);
114#endif
115}
116
117static struct tls_tcb *
118_rtld_tls_allocate_locked(void)
119{
120	Obj_Entry *obj;
121	struct tls_tcb *tcb;
122	uint8_t *p, *q;
123
124	p = xcalloc(_rtld_tls_static_space + sizeof(struct tls_tcb));
125#ifdef __HAVE_TLS_VARIANT_I
126	tcb = (struct tls_tcb *)p;
127	p += sizeof(struct tls_tcb);
128#else
129	p += _rtld_tls_static_space;
130	tcb = (struct tls_tcb *)p;
131	tcb->tcb_self = tcb;
132#endif
133	dbg(("lwp %d tls tcb %p", _lwp_self(), tcb));
134	tcb->tcb_dtv = xcalloc(sizeof(*tcb->tcb_dtv) * (2 + _rtld_tls_max_index));
135	++tcb->tcb_dtv;
136	SET_DTV_MAX_INDEX(tcb->tcb_dtv, _rtld_tls_max_index);
137	SET_DTV_GENERATION(tcb->tcb_dtv, _rtld_tls_dtv_generation);
138
139	for (obj = _rtld_objlist; obj != NULL; obj = obj->next) {
140		if (obj->tls_static) {
141#ifdef __HAVE_TLS_VARIANT_I
142			q = p + obj->tlsoffset;
143#else
144			q = p - obj->tlsoffset;
145#endif
146			dbg(("%s: [lwp %d] tls dtv %p index %zu offset %zu",
147			    obj->path, _lwp_self(),
148			    q, obj->tlsindex, obj->tlsoffset));
149			if (obj->tlsinitsize)
150				memcpy(q, obj->tlsinit, obj->tlsinitsize);
151			tcb->tcb_dtv[obj->tlsindex] = q;
152		}
153	}
154
155	return tcb;
156}
157
158struct tls_tcb *
159_rtld_tls_allocate(void)
160{
161	struct tls_tcb *tcb;
162	sigset_t mask;
163
164	_rtld_exclusive_enter(&mask);
165	tcb = _rtld_tls_allocate_locked();
166	_rtld_exclusive_exit(&mask);
167
168	return tcb;
169}
170
171void
172_rtld_tls_free(struct tls_tcb *tcb)
173{
174	size_t i, max_index;
175	uint8_t *p, *p_end;
176	sigset_t mask;
177
178	_rtld_exclusive_enter(&mask);
179
180#ifdef __HAVE_TLS_VARIANT_I
181	p = (uint8_t *)tcb;
182#else
183	p = (uint8_t *)tcb - _rtld_tls_static_space;
184#endif
185	p_end = p + _rtld_tls_static_space;
186
187	max_index = DTV_MAX_INDEX(tcb->tcb_dtv);
188	for (i = 1; i <= max_index; ++i) {
189		if ((uint8_t *)tcb->tcb_dtv[i] < p ||
190		    (uint8_t *)tcb->tcb_dtv[i] >= p_end)
191			xfree(tcb->tcb_dtv[i]);
192	}
193	xfree(tcb->tcb_dtv - 1);
194	xfree(p);
195
196	_rtld_exclusive_exit(&mask);
197}
198
199static void *
200_rtld_tls_module_allocate(struct tls_tcb *tcb, size_t idx)
201{
202	Obj_Entry *obj;
203	uint8_t *p;
204
205	for (obj = _rtld_objlist; obj != NULL; obj = obj->next) {
206		if (obj->tlsindex == idx)
207			break;
208	}
209	if (obj == NULL) {
210		_rtld_error("Module for TLS index %zu missing", idx);
211		_rtld_die();
212	}
213	if (obj->tls_static) {
214#ifdef __HAVE_TLS_VARIANT_I
215		p = (uint8_t *)tcb + obj->tlsoffset + sizeof(struct tls_tcb);
216#else
217		p = (uint8_t *)tcb - obj->tlsoffset;
218#endif
219		return p;
220	}
221
222	p = xmalloc(obj->tlssize);
223	memcpy(p, obj->tlsinit, obj->tlsinitsize);
224	memset(p + obj->tlsinitsize, 0, obj->tlssize - obj->tlsinitsize);
225
226	obj->tls_dynamic = 1;
227
228	return p;
229}
230
231int
232_rtld_tls_offset_allocate(Obj_Entry *obj)
233{
234	size_t offset, next_offset;
235
236	if (obj->tls_dynamic)
237		return -1;
238
239	if (obj->tls_static)
240		return 0;
241	if (obj->tlssize == 0) {
242		obj->tlsoffset = 0;
243		obj->tls_static = 1;
244		return 0;
245	}
246
247#ifdef __HAVE_TLS_VARIANT_I
248	offset = roundup2(_rtld_tls_static_offset, obj->tlsalign);
249	next_offset = offset + obj->tlssize;
250#else
251	offset = roundup2(_rtld_tls_static_offset + obj->tlssize,
252	    obj->tlsalign);
253	next_offset = offset;
254#endif
255
256	/*
257	 * Check if the static allocation was already done.
258	 * This happens if dynamically loaded modules want to use
259	 * static TLS space.
260	 *
261	 * XXX Keep an actual free list and callbacks for initialisation.
262	 */
263	if (_rtld_tls_static_space) {
264		if (obj->tlsinitsize) {
265			_rtld_error("%s: Use of initialized "
266			    "Thread Local Storage with model initial-exec "
267			    "and dlopen is not supported",
268			    obj->path);
269			return -1;
270		}
271		if (next_offset > _rtld_tls_static_space) {
272			_rtld_error("%s: No space available "
273			    "for static Thread Local Storage",
274			    obj->path);
275			return -1;
276		}
277	}
278	obj->tlsoffset = offset;
279	dbg(("%s: static tls offset 0x%zx size %zu\n",
280	    obj->path, obj->tlsoffset, obj->tlssize));
281	_rtld_tls_static_offset = next_offset;
282	obj->tls_static = 1;
283
284	return 0;
285}
286
287void
288_rtld_tls_offset_free(Obj_Entry *obj)
289{
290
291	/*
292	 * XXX See above.
293	 */
294	obj->tls_static = 0;
295	return;
296}
297
298#if defined(__HAVE_COMMON___TLS_GET_ADDR) && defined(RTLD_LOADER)
299/*
300 * The fast path is access to an already allocated DTV entry.
301 * This checks the current limit and the entry without needing any
302 * locking. Entries are only freed on dlclose() and it is an application
303 * bug if code of the module is still running at that point.
304 */
305void *
306__tls_get_addr(void *arg_)
307{
308	size_t *arg = (size_t *)arg_;
309	void **dtv;
310#ifdef __HAVE___LWP_GETTCB_FAST
311	struct tls_tcb * const tcb = __lwp_gettcb_fast();
312#else
313	struct tls_tcb * const tcb = __lwp_getprivate_fast();
314#endif
315	size_t idx = arg[0], offset = arg[1] + TLS_DTV_OFFSET;
316
317	dtv = tcb->tcb_dtv;
318
319	if (__predict_true(idx < DTV_MAX_INDEX(dtv) && dtv[idx] != NULL))
320		return (uint8_t *)dtv[idx] + offset;
321
322	return _rtld_tls_get_addr(tcb, idx, offset);
323}
324#endif
325
326#endif /* __HAVE_TLS_VARIANT_I || __HAVE_TLS_VARIANT_II */
327