tsd.h revision 296221 
1313535Sngie/******************************************************************************/
2272343Sngie#ifdef JEMALLOC_H_TYPES
3272343Sngie
4272343Sngie/* Maximum number of malloc_tsd users with cleanup functions. */
5272343Sngie#define	MALLOC_TSD_CLEANUPS_MAX	2
6272343Sngie
7272343Sngietypedef bool (*malloc_tsd_cleanup_t)(void);
8272343Sngie
9272343Sngie#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
10272343Sngie    !defined(_WIN32))
11272343Sngietypedef struct tsd_init_block_s tsd_init_block_t;
12272343Sngietypedef struct tsd_init_head_s tsd_init_head_t;
13272343Sngie#endif
14272343Sngie
15272343Sngietypedef struct tsd_s tsd_t;
16272343Sngie
17272343Sngietypedef enum {
18272343Sngie	tsd_state_uninitialized,
19272343Sngie	tsd_state_nominal,
20272343Sngie	tsd_state_purgatory,
21272343Sngie	tsd_state_reincarnated
22272343Sngie} tsd_state_t;
23272343Sngie
24272343Sngie/*
25272343Sngie * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
26272343Sngie * are five macros that support (at least) three use cases: file-private,
27272343Sngie * library-private, and library-private inlined.  Following is an example
28272343Sngie * library-private tsd variable:
29313535Sngie *
30313535Sngie * In example.h:
31313535Sngie *   typedef struct {
32313535Sngie *           int x;
33313535Sngie *           int y;
34272343Sngie *   } example_t;
35272343Sngie *   #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
36313535Sngie *   malloc_tsd_types(example_, example_t)
37272343Sngie *   malloc_tsd_protos(, example_, example_t)
38272343Sngie *   malloc_tsd_externs(example_, example_t)
39272343Sngie * In example.c:
40272343Sngie *   malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
41272343Sngie *   malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
42272343Sngie *       example_tsd_cleanup)
43272343Sngie *
44272343Sngie * The result is a set of generated functions, e.g.:
45272343Sngie *
46272343Sngie *   bool example_tsd_boot(void) {...}
47272343Sngie *   example_t *example_tsd_get() {...}
48272343Sngie *   void example_tsd_set(example_t *val) {...}
49272343Sngie *
50272343Sngie * Note that all of the functions deal in terms of (a_type *) rather than
51272343Sngie * (a_type) so that it is possible to support non-pointer types (unlike
52272343Sngie * pthreads TSD).  example_tsd_cleanup() is passed an (a_type *) pointer that is
53272343Sngie * cast to (void *).  This means that the cleanup function needs to cast the
54272343Sngie * function argument to (a_type *), then dereference the resulting pointer to
55272343Sngie * access fields, e.g.
56272343Sngie *
57272343Sngie *   void
58272343Sngie *   example_tsd_cleanup(void *arg)
59272343Sngie *   {
60272343Sngie *           example_t *example = (example_t *)arg;
61272343Sngie *
62272343Sngie *           example->x = 42;
63272343Sngie *           [...]
64272343Sngie *           if ([want the cleanup function to be called again])
65272343Sngie *                   example_tsd_set(example);
66272343Sngie *   }
67272343Sngie *
68272343Sngie * If example_tsd_set() is called within example_tsd_cleanup(), it will be
69272343Sngie * called again.  This is similar to how pthreads TSD destruction works, except
70272343Sngie * that pthreads only calls the cleanup function again if the value was set to
71272343Sngie * non-NULL.
72272343Sngie */
73272343Sngie
74272343Sngie/* malloc_tsd_types(). */
75272343Sngie#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
76272343Sngie#define	malloc_tsd_types(a_name, a_type)
77272343Sngie#elif (defined(JEMALLOC_TLS))
78272343Sngie#define	malloc_tsd_types(a_name, a_type)
79272343Sngie#elif (defined(_WIN32))
80272343Sngie#define	malloc_tsd_types(a_name, a_type)				\
81272343Sngietypedef struct {							\
82272343Sngie	bool	initialized;						\
83276478Sngie	a_type	val;							\
84272343Sngie} a_name##tsd_wrapper_t;
85276478Sngie#else
86272343Sngie#define	malloc_tsd_types(a_name, a_type)				\
87272343Sngietypedef struct {							\
88272343Sngie	bool	initialized;						\
89272343Sngie	a_type	val;							\
90272343Sngie} a_name##tsd_wrapper_t;
91272343Sngie#endif
92272343Sngie
93272343Sngie/* malloc_tsd_protos(). */
94272343Sngie#define	malloc_tsd_protos(a_attr, a_name, a_type)			\
95272343Sngiea_attr bool								\
96272343Sngiea_name##tsd_boot0(void);						\
97272343Sngiea_attr void								\
98272343Sngiea_name##tsd_boot1(void);						\
99272343Sngiea_attr bool								\
100272343Sngiea_name##tsd_boot(void);							\
101272343Sngiea_attr a_type *								\
102272343Sngiea_name##tsd_get(void);							\
103272343Sngiea_attr void								\
104272343Sngiea_name##tsd_set(a_type *val);
105272343Sngie
106272343Sngie/* malloc_tsd_externs(). */
107272343Sngie#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
108272343Sngie#define	malloc_tsd_externs(a_name, a_type)				\
109272343Sngieextern __thread a_type	a_name##tsd_tls;				\
110272343Sngieextern __thread bool	a_name##tsd_initialized;			\
111272343Sngieextern bool		a_name##tsd_booted;
112272343Sngie#elif (defined(JEMALLOC_TLS))
113272343Sngie#define	malloc_tsd_externs(a_name, a_type)				\
114272343Sngieextern __thread a_type	a_name##tsd_tls;				\
115272343Sngieextern pthread_key_t	a_name##tsd_tsd;				\
116272343Sngieextern bool		a_name##tsd_booted;
117272343Sngie#elif (defined(_WIN32))
118272343Sngie#define	malloc_tsd_externs(a_name, a_type)				\
119272343Sngieextern DWORD		a_name##tsd_tsd;				\
120272343Sngieextern a_name##tsd_wrapper_t	a_name##tsd_boot_wrapper;		\
121272343Sngieextern bool		a_name##tsd_booted;
122272343Sngie#else
123276478Sngie#define	malloc_tsd_externs(a_name, a_type)				\
124272343Sngieextern pthread_key_t	a_name##tsd_tsd;				\
125272343Sngieextern tsd_init_head_t	a_name##tsd_init_head;				\
126272343Sngieextern a_name##tsd_wrapper_t	a_name##tsd_boot_wrapper;		\
127272343Sngieextern bool		a_name##tsd_booted;
128272343Sngie#endif
129272343Sngie
130272343Sngie/* malloc_tsd_data(). */
131272343Sngie#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
132272343Sngie#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
133272343Sngiea_attr __thread a_type JEMALLOC_TLS_MODEL				\
134272343Sngie    a_name##tsd_tls = a_initializer;					\
135272343Sngiea_attr __thread bool JEMALLOC_TLS_MODEL					\
136272343Sngie    a_name##tsd_initialized = false;					\
137276478Sngiea_attr bool		a_name##tsd_booted = false;
138272343Sngie#elif (defined(JEMALLOC_TLS))
139272343Sngie#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
140272343Sngiea_attr __thread a_type JEMALLOC_TLS_MODEL				\
141272343Sngie    a_name##tsd_tls = a_initializer;					\
142272343Sngiea_attr pthread_key_t	a_name##tsd_tsd;				\
143272343Sngiea_attr bool		a_name##tsd_booted = false;
144272343Sngie#elif (defined(_WIN32))
145272343Sngie#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
146272343Sngiea_attr DWORD		a_name##tsd_tsd;				\
147272343Sngiea_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {		\
148272343Sngie	false,								\
149272343Sngie	a_initializer							\
150272343Sngie};									\
151272343Sngiea_attr bool		a_name##tsd_booted = false;
152272343Sngie#else
153272343Sngie#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
154272343Sngiea_attr pthread_key_t	a_name##tsd_tsd;				\
155272343Sngiea_attr tsd_init_head_t	a_name##tsd_init_head = {			\
156272343Sngie	ql_head_initializer(blocks),					\
157272343Sngie	MALLOC_MUTEX_INITIALIZER					\
158272343Sngie};									\
159272343Sngiea_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {		\
160272343Sngie	false,								\
161272343Sngie	a_initializer							\
162272343Sngie};									\
163272343Sngiea_attr bool		a_name##tsd_booted = false;
164272343Sngie#endif
165272343Sngie
166272343Sngie/* malloc_tsd_funcs(). */
167272343Sngie#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
168272343Sngie#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
169272343Sngie    a_cleanup)								\
170272343Sngie/* Initialization/cleanup. */						\
171272343Sngiea_attr bool								\
172272343Sngiea_name##tsd_cleanup_wrapper(void)					\
173272343Sngie{									\
174272343Sngie									\
175272343Sngie	if (a_name##tsd_initialized) {					\
176272343Sngie		a_name##tsd_initialized = false;			\
177272343Sngie		a_cleanup(&a_name##tsd_tls);				\
178272343Sngie	}								\
179272343Sngie	return (a_name##tsd_initialized);				\
180313498Sngie}									\
181313498Sngiea_attr bool								\
182272343Sngiea_name##tsd_boot0(void)							\
183272343Sngie{									\
184272343Sngie									\
185272343Sngie	if (a_cleanup != malloc_tsd_no_cleanup) {			\
186272343Sngie		malloc_tsd_cleanup_register(				\
187272343Sngie		    &a_name##tsd_cleanup_wrapper);			\
188272343Sngie	}								\
189272343Sngie	a_name##tsd_booted = true;					\
190272343Sngie	return (false);							\
191272343Sngie}									\
192272343Sngiea_attr void								\
193272343Sngiea_name##tsd_boot1(void)							\
194272343Sngie{									\
195272343Sngie									\
196272343Sngie	/* Do nothing. */						\
197272343Sngie}									\
198272343Sngiea_attr bool								\
199272343Sngiea_name##tsd_boot(void)							\
200272343Sngie{									\
201272343Sngie									\
202272343Sngie	return (a_name##tsd_boot0());					\
203272343Sngie}									\
204272343Sngie/* Get/set. */								\
205272343Sngiea_attr a_type *								\
206272343Sngiea_name##tsd_get(void)							\
207272343Sngie{									\
208272343Sngie									\
209272343Sngie	assert(a_name##tsd_booted);					\
210272343Sngie	return (&a_name##tsd_tls);					\
211272343Sngie}									\
212272343Sngiea_attr void								\
213272343Sngiea_name##tsd_set(a_type *val)						\
214272343Sngie{									\
215272343Sngie									\
216272343Sngie	assert(a_name##tsd_booted);					\
217272343Sngie	a_name##tsd_tls = (*val);					\
218272343Sngie	if (a_cleanup != malloc_tsd_no_cleanup)				\
219272343Sngie		a_name##tsd_initialized = true;				\
220276478Sngie}
221272343Sngie#elif (defined(JEMALLOC_TLS))
222272343Sngie#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
223272343Sngie    a_cleanup)								\
224272343Sngie/* Initialization/cleanup. */						\
225272343Sngiea_attr bool								\
226272343Sngiea_name##tsd_boot0(void)							\
227272343Sngie{									\
228272343Sngie									\
229272343Sngie	if (a_cleanup != malloc_tsd_no_cleanup) {			\
230272343Sngie		if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) !=	\
231272343Sngie		    0)							\
232272343Sngie			return (true);					\
233272343Sngie	}								\
234276478Sngie	a_name##tsd_booted = true;					\
235272343Sngie	return (false);							\
236272343Sngie}									\
237272343Sngiea_attr void								\
238272343Sngiea_name##tsd_boot1(void)							\
239272343Sngie{									\
240272343Sngie									\
241272343Sngie	/* Do nothing. */						\
242272343Sngie}									\
243272343Sngiea_attr bool								\
244272343Sngiea_name##tsd_boot(void)							\
245272343Sngie{									\
246272343Sngie									\
247272343Sngie	return (a_name##tsd_boot0());					\
248272343Sngie}									\
249272343Sngie/* Get/set. */								\
250272343Sngiea_attr a_type *								\
251272343Sngiea_name##tsd_get(void)							\
252272343Sngie{									\
253272343Sngie									\
254272343Sngie	assert(a_name##tsd_booted);					\
255272343Sngie	return (&a_name##tsd_tls);					\
256272343Sngie}									\
257272343Sngiea_attr void								\
258272343Sngiea_name##tsd_set(a_type *val)						\
259272343Sngie{									\
260272343Sngie									\
261272343Sngie	assert(a_name##tsd_booted);					\
262272343Sngie	a_name##tsd_tls = (*val);					\
263272343Sngie	if (a_cleanup != malloc_tsd_no_cleanup) {			\
264272343Sngie		if (pthread_setspecific(a_name##tsd_tsd,		\
265272343Sngie		    (void *)(&a_name##tsd_tls))) {			\
266272343Sngie			malloc_write("<jemalloc>: Error"		\
267272343Sngie			    " setting TSD for "#a_name"\n");		\
268272343Sngie			if (opt_abort)					\
269272343Sngie				abort();				\
270272343Sngie		}							\
271272343Sngie	}								\
272272343Sngie}
273272343Sngie#elif (defined(_WIN32))
274272343Sngie#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
275272343Sngie    a_cleanup)								\
276272343Sngie/* Initialization/cleanup. */						\
277272343Sngiea_attr bool								\
278272343Sngiea_name##tsd_cleanup_wrapper(void)					\
279272343Sngie{									\
280272343Sngie	DWORD error = GetLastError();					\
281272343Sngie	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
282272343Sngie	    TlsGetValue(a_name##tsd_tsd);				\
283272343Sngie	SetLastError(error);						\
284272343Sngie									\
285272343Sngie	if (wrapper == NULL)						\
286272343Sngie		return (false);						\
287272343Sngie	if (a_cleanup != malloc_tsd_no_cleanup &&			\
288272343Sngie	    wrapper->initialized) {					\
289272343Sngie		wrapper->initialized = false;				\
290272343Sngie		a_cleanup(&wrapper->val);				\
291272343Sngie		if (wrapper->initialized) {				\
292272343Sngie			/* Trigger another cleanup round. */		\
293272343Sngie			return (true);					\
294272343Sngie		}							\
295272343Sngie	}								\
296272343Sngie	malloc_tsd_dalloc(wrapper);					\
297272343Sngie	return (false);							\
298272343Sngie}									\
299272343Sngiea_attr void								\
300272343Sngiea_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)			\
301272343Sngie{									\
302272343Sngie									\
303272343Sngie	if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) {		\
304272343Sngie		malloc_write("<jemalloc>: Error setting"		\
305272343Sngie		    " TSD for "#a_name"\n");				\
306272343Sngie		abort();						\
307272343Sngie	}								\
308272343Sngie}									\
309272343Sngiea_attr a_name##tsd_wrapper_t *						\
310272343Sngiea_name##tsd_wrapper_get(void)						\
311272343Sngie{									\
312272343Sngie	DWORD error = GetLastError();					\
313272343Sngie	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
314272343Sngie	    TlsGetValue(a_name##tsd_tsd);				\
315272343Sngie	SetLastError(error);						\
316272343Sngie									\
317272343Sngie	if (unlikely(wrapper == NULL)) {				\
318272343Sngie		wrapper = (a_name##tsd_wrapper_t *)			\
319272343Sngie		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
320272343Sngie		if (wrapper == NULL) {					\
321272343Sngie			malloc_write("<jemalloc>: Error allocating"	\
322272343Sngie			    " TSD for "#a_name"\n");			\
323272343Sngie			abort();					\
324272343Sngie		} else {						\
325272343Sngie			wrapper->initialized = false;			\
326272343Sngie			wrapper->val = a_initializer;			\
327272343Sngie		}							\
328272343Sngie		a_name##tsd_wrapper_set(wrapper);			\
329272343Sngie	}								\
330272343Sngie	return (wrapper);						\
331272343Sngie}									\
332272343Sngiea_attr bool								\
333272343Sngiea_name##tsd_boot0(void)							\
334272343Sngie{									\
335272343Sngie									\
336272343Sngie	a_name##tsd_tsd = TlsAlloc();					\
337272343Sngie	if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES)			\
338272343Sngie		return (true);						\
339272343Sngie	if (a_cleanup != malloc_tsd_no_cleanup) {			\
340272343Sngie		malloc_tsd_cleanup_register(				\
341276478Sngie		    &a_name##tsd_cleanup_wrapper);			\
342272343Sngie	}								\
343276478Sngie	a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);		\
344272343Sngie	a_name##tsd_booted = true;					\
345272343Sngie	return (false);							\
346272343Sngie}									\
347272343Sngiea_attr void								\
348272343Sngiea_name##tsd_boot1(void)							\
349{									\
350	a_name##tsd_wrapper_t *wrapper;					\
351	wrapper = (a_name##tsd_wrapper_t *)				\
352	    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));		\
353	if (wrapper == NULL) {						\
354		malloc_write("<jemalloc>: Error allocating"		\
355		    " TSD for "#a_name"\n");				\
356		abort();						\
357	}								\
358	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
359	    sizeof(a_name##tsd_wrapper_t));				\
360	a_name##tsd_wrapper_set(wrapper);				\
361}									\
362a_attr bool								\
363a_name##tsd_boot(void)							\
364{									\
365									\
366	if (a_name##tsd_boot0())					\
367		return (true);						\
368	a_name##tsd_boot1();						\
369	return (false);							\
370}									\
371/* Get/set. */								\
372a_attr a_type *								\
373a_name##tsd_get(void)							\
374{									\
375	a_name##tsd_wrapper_t *wrapper;					\
376									\
377	assert(a_name##tsd_booted);					\
378	wrapper = a_name##tsd_wrapper_get();				\
379	return (&wrapper->val);						\
380}									\
381a_attr void								\
382a_name##tsd_set(a_type *val)						\
383{									\
384	a_name##tsd_wrapper_t *wrapper;					\
385									\
386	assert(a_name##tsd_booted);					\
387	wrapper = a_name##tsd_wrapper_get();				\
388	wrapper->val = *(val);						\
389	if (a_cleanup != malloc_tsd_no_cleanup)				\
390		wrapper->initialized = true;				\
391}
392#else
393#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
394    a_cleanup)								\
395/* Initialization/cleanup. */						\
396a_attr void								\
397a_name##tsd_cleanup_wrapper(void *arg)					\
398{									\
399	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg;	\
400									\
401	if (a_cleanup != malloc_tsd_no_cleanup &&			\
402	    wrapper->initialized) {					\
403		wrapper->initialized = false;				\
404		a_cleanup(&wrapper->val);				\
405		if (wrapper->initialized) {				\
406			/* Trigger another cleanup round. */		\
407			if (pthread_setspecific(a_name##tsd_tsd,	\
408			    (void *)wrapper)) {				\
409				malloc_write("<jemalloc>: Error"	\
410				    " setting TSD for "#a_name"\n");	\
411				if (opt_abort)				\
412					abort();			\
413			}						\
414			return;						\
415		}							\
416	}								\
417	malloc_tsd_dalloc(wrapper);					\
418}									\
419a_attr void								\
420a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)			\
421{									\
422									\
423	if (pthread_setspecific(a_name##tsd_tsd,			\
424	    (void *)wrapper)) {						\
425		malloc_write("<jemalloc>: Error setting"		\
426		    " TSD for "#a_name"\n");				\
427		abort();						\
428	}								\
429}									\
430a_attr a_name##tsd_wrapper_t *						\
431a_name##tsd_wrapper_get(void)						\
432{									\
433	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
434	    pthread_getspecific(a_name##tsd_tsd);			\
435									\
436	if (unlikely(wrapper == NULL)) {				\
437		tsd_init_block_t block;					\
438		wrapper = tsd_init_check_recursion(			\
439		    &a_name##tsd_init_head, &block);			\
440		if (wrapper)						\
441		    return (wrapper);					\
442		wrapper = (a_name##tsd_wrapper_t *)			\
443		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
444		block.data = wrapper;					\
445		if (wrapper == NULL) {					\
446			malloc_write("<jemalloc>: Error allocating"	\
447			    " TSD for "#a_name"\n");			\
448			abort();					\
449		} else {						\
450			wrapper->initialized = false;			\
451			wrapper->val = a_initializer;			\
452		}							\
453		a_name##tsd_wrapper_set(wrapper);			\
454		tsd_init_finish(&a_name##tsd_init_head, &block);	\
455	}								\
456	return (wrapper);						\
457}									\
458a_attr bool								\
459a_name##tsd_boot0(void)							\
460{									\
461									\
462	if (pthread_key_create(&a_name##tsd_tsd,			\
463	    a_name##tsd_cleanup_wrapper) != 0)				\
464		return (true);						\
465	a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);		\
466	a_name##tsd_booted = true;					\
467	return (false);							\
468}									\
469a_attr void								\
470a_name##tsd_boot1(void)							\
471{									\
472	a_name##tsd_wrapper_t *wrapper;					\
473	wrapper = (a_name##tsd_wrapper_t *)				\
474	    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));		\
475	if (wrapper == NULL) {						\
476		malloc_write("<jemalloc>: Error allocating"		\
477		    " TSD for "#a_name"\n");				\
478		abort();						\
479	}								\
480	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
481	    sizeof(a_name##tsd_wrapper_t));				\
482	a_name##tsd_wrapper_set(wrapper);				\
483}									\
484a_attr bool								\
485a_name##tsd_boot(void)							\
486{									\
487									\
488	if (a_name##tsd_boot0())					\
489		return (true);						\
490	a_name##tsd_boot1();						\
491	return (false);							\
492}									\
493/* Get/set. */								\
494a_attr a_type *								\
495a_name##tsd_get(void)							\
496{									\
497	a_name##tsd_wrapper_t *wrapper;					\
498									\
499	assert(a_name##tsd_booted);					\
500	wrapper = a_name##tsd_wrapper_get();				\
501	return (&wrapper->val);						\
502}									\
503a_attr void								\
504a_name##tsd_set(a_type *val)						\
505{									\
506	a_name##tsd_wrapper_t *wrapper;					\
507									\
508	assert(a_name##tsd_booted);					\
509	wrapper = a_name##tsd_wrapper_get();				\
510	wrapper->val = *(val);						\
511	if (a_cleanup != malloc_tsd_no_cleanup)				\
512		wrapper->initialized = true;				\
513}
514#endif
515
516#endif /* JEMALLOC_H_TYPES */
517/******************************************************************************/
518#ifdef JEMALLOC_H_STRUCTS
519
520#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
521    !defined(_WIN32))
522struct tsd_init_block_s {
523	ql_elm(tsd_init_block_t)	link;
524	pthread_t			thread;
525	void				*data;
526};
527struct tsd_init_head_s {
528	ql_head(tsd_init_block_t)	blocks;
529	malloc_mutex_t			lock;
530};
531#endif
532
533#define	MALLOC_TSD							\
534/*  O(name,			type) */				\
535    O(tcache,			tcache_t *)				\
536    O(thread_allocated,		uint64_t)				\
537    O(thread_deallocated,	uint64_t)				\
538    O(prof_tdata,		prof_tdata_t *)				\
539    O(arena,			arena_t *)				\
540    O(arenas_tdata,		arena_tdata_t *)			\
541    O(narenas_tdata,		unsigned)				\
542    O(arenas_tdata_bypass,	bool)					\
543    O(tcache_enabled,		tcache_enabled_t)			\
544    O(quarantine,		quarantine_t *)				\
545
546#define	TSD_INITIALIZER {						\
547    tsd_state_uninitialized,						\
548    NULL,								\
549    0,									\
550    0,									\
551    NULL,								\
552    NULL,								\
553    NULL,								\
554    0,									\
555    false,								\
556    tcache_enabled_default,						\
557    NULL								\
558}
559
560struct tsd_s {
561	tsd_state_t	state;
562#define	O(n, t)								\
563	t		n;
564MALLOC_TSD
565#undef O
566};
567
568static const tsd_t tsd_initializer = TSD_INITIALIZER;
569
570malloc_tsd_types(, tsd_t)
571
572#endif /* JEMALLOC_H_STRUCTS */
573/******************************************************************************/
574#ifdef JEMALLOC_H_EXTERNS
575
576void	*malloc_tsd_malloc(size_t size);
577void	malloc_tsd_dalloc(void *wrapper);
578void	malloc_tsd_no_cleanup(void *arg);
579void	malloc_tsd_cleanup_register(bool (*f)(void));
580bool	malloc_tsd_boot0(void);
581void	malloc_tsd_boot1(void);
582#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
583    !defined(_WIN32))
584void	*tsd_init_check_recursion(tsd_init_head_t *head,
585    tsd_init_block_t *block);
586void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
587#endif
588void	tsd_cleanup(void *arg);
589
590#endif /* JEMALLOC_H_EXTERNS */
591/******************************************************************************/
592#ifdef JEMALLOC_H_INLINES
593
594#ifndef JEMALLOC_ENABLE_INLINE
595malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
596
597tsd_t	*tsd_fetch(void);
598bool	tsd_nominal(tsd_t *tsd);
599#define	O(n, t)								\
600t	*tsd_##n##p_get(tsd_t *tsd);					\
601t	tsd_##n##_get(tsd_t *tsd);					\
602void	tsd_##n##_set(tsd_t *tsd, t n);
603MALLOC_TSD
604#undef O
605#endif
606
607#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
608malloc_tsd_externs(, tsd_t)
609malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
610
611JEMALLOC_ALWAYS_INLINE tsd_t *
612tsd_fetch(void)
613{
614	tsd_t *tsd = tsd_get();
615
616	if (unlikely(tsd->state != tsd_state_nominal)) {
617		if (tsd->state == tsd_state_uninitialized) {
618			tsd->state = tsd_state_nominal;
619			/* Trigger cleanup handler registration. */
620			tsd_set(tsd);
621		} else if (tsd->state == tsd_state_purgatory) {
622			tsd->state = tsd_state_reincarnated;
623			tsd_set(tsd);
624		} else
625			assert(tsd->state == tsd_state_reincarnated);
626	}
627
628	return (tsd);
629}
630
631JEMALLOC_INLINE bool
632tsd_nominal(tsd_t *tsd)
633{
634
635	return (tsd->state == tsd_state_nominal);
636}
637
638#define	O(n, t)								\
639JEMALLOC_ALWAYS_INLINE t *						\
640tsd_##n##p_get(tsd_t *tsd)						\
641{									\
642									\
643	return (&tsd->n);						\
644}									\
645									\
646JEMALLOC_ALWAYS_INLINE t						\
647tsd_##n##_get(tsd_t *tsd)						\
648{									\
649									\
650	return (*tsd_##n##p_get(tsd));					\
651}									\
652									\
653JEMALLOC_ALWAYS_INLINE void						\
654tsd_##n##_set(tsd_t *tsd, t n)						\
655{									\
656									\
657	assert(tsd->state == tsd_state_nominal);			\
658	tsd->n = n;							\
659}
660MALLOC_TSD
661#undef O
662#endif
663
664#endif /* JEMALLOC_H_INLINES */
665/******************************************************************************/
666