1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4/* Maximum number of malloc_tsd users with cleanup functions. */
5#define	MALLOC_TSD_CLEANUPS_MAX	2
6
7typedef bool (*malloc_tsd_cleanup_t)(void);
8
9#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
10    !defined(_WIN32))
11typedef struct tsd_init_block_s tsd_init_block_t;
12typedef struct tsd_init_head_s tsd_init_head_t;
13#endif
14
15typedef struct tsd_s tsd_t;
16typedef struct tsdn_s tsdn_t;
17
18#define	TSDN_NULL	((tsdn_t *)0)
19
20typedef enum {
21	tsd_state_uninitialized,
22	tsd_state_nominal,
23	tsd_state_purgatory,
24	tsd_state_reincarnated
25} tsd_state_t;
26
27/*
28 * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
29 * are five macros that support (at least) three use cases: file-private,
30 * library-private, and library-private inlined.  Following is an example
31 * library-private tsd variable:
32 *
33 * In example.h:
34 *   typedef struct {
35 *           int x;
36 *           int y;
37 *   } example_t;
38 *   #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
39 *   malloc_tsd_types(example_, example_t)
40 *   malloc_tsd_protos(, example_, example_t)
41 *   malloc_tsd_externs(example_, example_t)
42 * In example.c:
43 *   malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
44 *   malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
45 *       example_tsd_cleanup)
46 *
47 * The result is a set of generated functions, e.g.:
48 *
49 *   bool example_tsd_boot(void) {...}
50 *   bool example_tsd_booted_get(void) {...}
51 *   example_t *example_tsd_get() {...}
52 *   void example_tsd_set(example_t *val) {...}
53 *
54 * Note that all of the functions deal in terms of (a_type *) rather than
55 * (a_type) so that it is possible to support non-pointer types (unlike
56 * pthreads TSD).  example_tsd_cleanup() is passed an (a_type *) pointer that is
57 * cast to (void *).  This means that the cleanup function needs to cast the
58 * function argument to (a_type *), then dereference the resulting pointer to
59 * access fields, e.g.
60 *
61 *   void
62 *   example_tsd_cleanup(void *arg)
63 *   {
64 *           example_t *example = (example_t *)arg;
65 *
66 *           example->x = 42;
67 *           [...]
68 *           if ([want the cleanup function to be called again])
69 *                   example_tsd_set(example);
70 *   }
71 *
72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be
73 * called again.  This is similar to how pthreads TSD destruction works, except
74 * that pthreads only calls the cleanup function again if the value was set to
75 * non-NULL.
76 */
77
78/* malloc_tsd_types(). */
79#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
80#define	malloc_tsd_types(a_name, a_type)
81#elif (defined(JEMALLOC_TLS))
82#define	malloc_tsd_types(a_name, a_type)
83#elif (defined(_WIN32))
84#define	malloc_tsd_types(a_name, a_type)				\
85typedef struct {							\
86	bool	initialized;						\
87	a_type	val;							\
88} a_name##tsd_wrapper_t;
89#else
90#define	malloc_tsd_types(a_name, a_type)				\
91typedef struct {							\
92	bool	initialized;						\
93	a_type	val;							\
94} a_name##tsd_wrapper_t;
95#endif
96
97/* malloc_tsd_protos(). */
98#define	malloc_tsd_protos(a_attr, a_name, a_type)			\
99a_attr bool								\
100a_name##tsd_boot0(void);						\
101a_attr void								\
102a_name##tsd_boot1(void);						\
103a_attr bool								\
104a_name##tsd_boot(void);							\
105a_attr bool								\
106a_name##tsd_booted_get(void);						\
107a_attr a_type *								\
108a_name##tsd_get(void);							\
109a_attr void								\
110a_name##tsd_set(a_type *val);
111
112/* malloc_tsd_externs(). */
113#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
114#define	malloc_tsd_externs(a_name, a_type)				\
115extern __thread a_type	a_name##tsd_tls;				\
116extern __thread bool	a_name##tsd_initialized;			\
117extern bool		a_name##tsd_booted;
118#elif (defined(JEMALLOC_TLS))
119#define	malloc_tsd_externs(a_name, a_type)				\
120extern __thread a_type	a_name##tsd_tls;				\
121extern pthread_key_t	a_name##tsd_tsd;				\
122extern bool		a_name##tsd_booted;
123#elif (defined(_WIN32))
124#define	malloc_tsd_externs(a_name, a_type)				\
125extern DWORD		a_name##tsd_tsd;				\
126extern a_name##tsd_wrapper_t	a_name##tsd_boot_wrapper;		\
127extern bool		a_name##tsd_booted;
128#else
129#define	malloc_tsd_externs(a_name, a_type)				\
130extern pthread_key_t	a_name##tsd_tsd;				\
131extern tsd_init_head_t	a_name##tsd_init_head;				\
132extern a_name##tsd_wrapper_t	a_name##tsd_boot_wrapper;		\
133extern bool		a_name##tsd_booted;
134#endif
135
136/* malloc_tsd_data(). */
137#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
138#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
139a_attr __thread a_type JEMALLOC_TLS_MODEL				\
140    a_name##tsd_tls = a_initializer;					\
141a_attr __thread bool JEMALLOC_TLS_MODEL					\
142    a_name##tsd_initialized = false;					\
143a_attr bool		a_name##tsd_booted = false;
144#elif (defined(JEMALLOC_TLS))
145#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
146a_attr __thread a_type JEMALLOC_TLS_MODEL				\
147    a_name##tsd_tls = a_initializer;					\
148a_attr pthread_key_t	a_name##tsd_tsd;				\
149a_attr bool		a_name##tsd_booted = false;
150#elif (defined(_WIN32))
151#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
152a_attr DWORD		a_name##tsd_tsd;				\
153a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {		\
154	false,								\
155	a_initializer							\
156};									\
157a_attr bool		a_name##tsd_booted = false;
158#else
159#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
160a_attr pthread_key_t	a_name##tsd_tsd;				\
161a_attr tsd_init_head_t	a_name##tsd_init_head = {			\
162	ql_head_initializer(blocks),					\
163	MALLOC_MUTEX_INITIALIZER					\
164};									\
165a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {		\
166	false,								\
167	a_initializer							\
168};									\
169a_attr bool		a_name##tsd_booted = false;
170#endif
171
172/* malloc_tsd_funcs(). */
173#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
174#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
175    a_cleanup)								\
176/* Initialization/cleanup. */						\
177a_attr bool								\
178a_name##tsd_cleanup_wrapper(void)					\
179{									\
180									\
181	if (a_name##tsd_initialized) {					\
182		a_name##tsd_initialized = false;			\
183		a_cleanup(&a_name##tsd_tls);				\
184	}								\
185	return (a_name##tsd_initialized);				\
186}									\
187a_attr bool								\
188a_name##tsd_boot0(void)							\
189{									\
190									\
191	if (a_cleanup != malloc_tsd_no_cleanup) {			\
192		malloc_tsd_cleanup_register(				\
193		    &a_name##tsd_cleanup_wrapper);			\
194	}								\
195	a_name##tsd_booted = true;					\
196	return (false);							\
197}									\
198a_attr void								\
199a_name##tsd_boot1(void)							\
200{									\
201									\
202	/* Do nothing. */						\
203}									\
204a_attr bool								\
205a_name##tsd_boot(void)							\
206{									\
207									\
208	return (a_name##tsd_boot0());					\
209}									\
210a_attr bool								\
211a_name##tsd_booted_get(void)						\
212{									\
213									\
214	return (a_name##tsd_booted);					\
215}									\
216/* Get/set. */								\
217a_attr a_type *								\
218a_name##tsd_get(void)							\
219{									\
220									\
221	assert(a_name##tsd_booted);					\
222	return (&a_name##tsd_tls);					\
223}									\
224a_attr void								\
225a_name##tsd_set(a_type *val)						\
226{									\
227									\
228	assert(a_name##tsd_booted);					\
229	a_name##tsd_tls = (*val);					\
230	if (a_cleanup != malloc_tsd_no_cleanup)				\
231		a_name##tsd_initialized = true;				\
232}
233#elif (defined(JEMALLOC_TLS))
234#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
235    a_cleanup)								\
236/* Initialization/cleanup. */						\
237a_attr bool								\
238a_name##tsd_boot0(void)							\
239{									\
240									\
241	if (a_cleanup != malloc_tsd_no_cleanup) {			\
242		if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) !=	\
243		    0)							\
244			return (true);					\
245	}								\
246	a_name##tsd_booted = true;					\
247	return (false);							\
248}									\
249a_attr void								\
250a_name##tsd_boot1(void)							\
251{									\
252									\
253	/* Do nothing. */						\
254}									\
255a_attr bool								\
256a_name##tsd_boot(void)							\
257{									\
258									\
259	return (a_name##tsd_boot0());					\
260}									\
261a_attr bool								\
262a_name##tsd_booted_get(void)						\
263{									\
264									\
265	return (a_name##tsd_booted);					\
266}									\
267/* Get/set. */								\
268a_attr a_type *								\
269a_name##tsd_get(void)							\
270{									\
271									\
272	assert(a_name##tsd_booted);					\
273	return (&a_name##tsd_tls);					\
274}									\
275a_attr void								\
276a_name##tsd_set(a_type *val)						\
277{									\
278									\
279	assert(a_name##tsd_booted);					\
280	a_name##tsd_tls = (*val);					\
281	if (a_cleanup != malloc_tsd_no_cleanup) {			\
282		if (pthread_setspecific(a_name##tsd_tsd,		\
283		    (void *)(&a_name##tsd_tls))) {			\
284			malloc_write("<jemalloc>: Error"		\
285			    " setting TSD for "#a_name"\n");		\
286			if (opt_abort)					\
287				abort();				\
288		}							\
289	}								\
290}
291#elif (defined(_WIN32))
292#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
293    a_cleanup)								\
294/* Initialization/cleanup. */						\
295a_attr bool								\
296a_name##tsd_cleanup_wrapper(void)					\
297{									\
298	DWORD error = GetLastError();					\
299	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
300	    TlsGetValue(a_name##tsd_tsd);				\
301	SetLastError(error);						\
302									\
303	if (wrapper == NULL)						\
304		return (false);						\
305	if (a_cleanup != malloc_tsd_no_cleanup &&			\
306	    wrapper->initialized) {					\
307		wrapper->initialized = false;				\
308		a_cleanup(&wrapper->val);				\
309		if (wrapper->initialized) {				\
310			/* Trigger another cleanup round. */		\
311			return (true);					\
312		}							\
313	}								\
314	malloc_tsd_dalloc(wrapper);					\
315	return (false);							\
316}									\
317a_attr void								\
318a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)			\
319{									\
320									\
321	if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) {		\
322		malloc_write("<jemalloc>: Error setting"		\
323		    " TSD for "#a_name"\n");				\
324		abort();						\
325	}								\
326}									\
327a_attr a_name##tsd_wrapper_t *						\
328a_name##tsd_wrapper_get(void)						\
329{									\
330	DWORD error = GetLastError();					\
331	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
332	    TlsGetValue(a_name##tsd_tsd);				\
333	SetLastError(error);						\
334									\
335	if (unlikely(wrapper == NULL)) {				\
336		wrapper = (a_name##tsd_wrapper_t *)			\
337		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
338		if (wrapper == NULL) {					\
339			malloc_write("<jemalloc>: Error allocating"	\
340			    " TSD for "#a_name"\n");			\
341			abort();					\
342		} else {						\
343			wrapper->initialized = false;			\
344			wrapper->val = a_initializer;			\
345		}							\
346		a_name##tsd_wrapper_set(wrapper);			\
347	}								\
348	return (wrapper);						\
349}									\
350a_attr bool								\
351a_name##tsd_boot0(void)							\
352{									\
353									\
354	a_name##tsd_tsd = TlsAlloc();					\
355	if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES)			\
356		return (true);						\
357	if (a_cleanup != malloc_tsd_no_cleanup) {			\
358		malloc_tsd_cleanup_register(				\
359		    &a_name##tsd_cleanup_wrapper);			\
360	}								\
361	a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);		\
362	a_name##tsd_booted = true;					\
363	return (false);							\
364}									\
365a_attr void								\
366a_name##tsd_boot1(void)							\
367{									\
368	a_name##tsd_wrapper_t *wrapper;					\
369	wrapper = (a_name##tsd_wrapper_t *)				\
370	    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));		\
371	if (wrapper == NULL) {						\
372		malloc_write("<jemalloc>: Error allocating"		\
373		    " TSD for "#a_name"\n");				\
374		abort();						\
375	}								\
376	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
377	    sizeof(a_name##tsd_wrapper_t));				\
378	a_name##tsd_wrapper_set(wrapper);				\
379}									\
380a_attr bool								\
381a_name##tsd_boot(void)							\
382{									\
383									\
384	if (a_name##tsd_boot0())					\
385		return (true);						\
386	a_name##tsd_boot1();						\
387	return (false);							\
388}									\
389a_attr bool								\
390a_name##tsd_booted_get(void)						\
391{									\
392									\
393	return (a_name##tsd_booted);					\
394}									\
395/* Get/set. */								\
396a_attr a_type *								\
397a_name##tsd_get(void)							\
398{									\
399	a_name##tsd_wrapper_t *wrapper;					\
400									\
401	assert(a_name##tsd_booted);					\
402	wrapper = a_name##tsd_wrapper_get();				\
403	return (&wrapper->val);						\
404}									\
405a_attr void								\
406a_name##tsd_set(a_type *val)						\
407{									\
408	a_name##tsd_wrapper_t *wrapper;					\
409									\
410	assert(a_name##tsd_booted);					\
411	wrapper = a_name##tsd_wrapper_get();				\
412	wrapper->val = *(val);						\
413	if (a_cleanup != malloc_tsd_no_cleanup)				\
414		wrapper->initialized = true;				\
415}
416#else
417#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
418    a_cleanup)								\
419/* Initialization/cleanup. */						\
420a_attr void								\
421a_name##tsd_cleanup_wrapper(void *arg)					\
422{									\
423	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg;	\
424									\
425	if (a_cleanup != malloc_tsd_no_cleanup &&			\
426	    wrapper->initialized) {					\
427		wrapper->initialized = false;				\
428		a_cleanup(&wrapper->val);				\
429		if (wrapper->initialized) {				\
430			/* Trigger another cleanup round. */		\
431			if (pthread_setspecific(a_name##tsd_tsd,	\
432			    (void *)wrapper)) {				\
433				malloc_write("<jemalloc>: Error"	\
434				    " setting TSD for "#a_name"\n");	\
435				if (opt_abort)				\
436					abort();			\
437			}						\
438			return;						\
439		}							\
440	}								\
441	malloc_tsd_dalloc(wrapper);					\
442}									\
443a_attr void								\
444a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)			\
445{									\
446									\
447	if (pthread_setspecific(a_name##tsd_tsd,			\
448	    (void *)wrapper)) {						\
449		malloc_write("<jemalloc>: Error setting"		\
450		    " TSD for "#a_name"\n");				\
451		abort();						\
452	}								\
453}									\
454a_attr a_name##tsd_wrapper_t *						\
455a_name##tsd_wrapper_get(void)						\
456{									\
457	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
458	    pthread_getspecific(a_name##tsd_tsd);			\
459									\
460	if (unlikely(wrapper == NULL)) {				\
461		tsd_init_block_t block;					\
462		wrapper = tsd_init_check_recursion(			\
463		    &a_name##tsd_init_head, &block);			\
464		if (wrapper)						\
465		    return (wrapper);					\
466		wrapper = (a_name##tsd_wrapper_t *)			\
467		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
468		block.data = wrapper;					\
469		if (wrapper == NULL) {					\
470			malloc_write("<jemalloc>: Error allocating"	\
471			    " TSD for "#a_name"\n");			\
472			abort();					\
473		} else {						\
474			wrapper->initialized = false;			\
475			wrapper->val = a_initializer;			\
476		}							\
477		a_name##tsd_wrapper_set(wrapper);			\
478		tsd_init_finish(&a_name##tsd_init_head, &block);	\
479	}								\
480	return (wrapper);						\
481}									\
482a_attr bool								\
483a_name##tsd_boot0(void)							\
484{									\
485									\
486	if (pthread_key_create(&a_name##tsd_tsd,			\
487	    a_name##tsd_cleanup_wrapper) != 0)				\
488		return (true);						\
489	a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);		\
490	a_name##tsd_booted = true;					\
491	return (false);							\
492}									\
493a_attr void								\
494a_name##tsd_boot1(void)							\
495{									\
496	a_name##tsd_wrapper_t *wrapper;					\
497	wrapper = (a_name##tsd_wrapper_t *)				\
498	    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));		\
499	if (wrapper == NULL) {						\
500		malloc_write("<jemalloc>: Error allocating"		\
501		    " TSD for "#a_name"\n");				\
502		abort();						\
503	}								\
504	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
505	    sizeof(a_name##tsd_wrapper_t));				\
506	a_name##tsd_wrapper_set(wrapper);				\
507}									\
508a_attr bool								\
509a_name##tsd_boot(void)							\
510{									\
511									\
512	if (a_name##tsd_boot0())					\
513		return (true);						\
514	a_name##tsd_boot1();						\
515	return (false);							\
516}									\
517a_attr bool								\
518a_name##tsd_booted_get(void)						\
519{									\
520									\
521	return (a_name##tsd_booted);					\
522}									\
523/* Get/set. */								\
524a_attr a_type *								\
525a_name##tsd_get(void)							\
526{									\
527	a_name##tsd_wrapper_t *wrapper;					\
528									\
529	assert(a_name##tsd_booted);					\
530	wrapper = a_name##tsd_wrapper_get();				\
531	return (&wrapper->val);						\
532}									\
533a_attr void								\
534a_name##tsd_set(a_type *val)						\
535{									\
536	a_name##tsd_wrapper_t *wrapper;					\
537									\
538	assert(a_name##tsd_booted);					\
539	wrapper = a_name##tsd_wrapper_get();				\
540	wrapper->val = *(val);						\
541	if (a_cleanup != malloc_tsd_no_cleanup)				\
542		wrapper->initialized = true;				\
543}
544#endif
545
546#endif /* JEMALLOC_H_TYPES */
547/******************************************************************************/
548#ifdef JEMALLOC_H_STRUCTS
549
550#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
551    !defined(_WIN32))
552struct tsd_init_block_s {
553	ql_elm(tsd_init_block_t)	link;
554	pthread_t			thread;
555	void				*data;
556};
557struct tsd_init_head_s {
558	ql_head(tsd_init_block_t)	blocks;
559	malloc_mutex_t			lock;
560};
561#endif
562
563#define	MALLOC_TSD							\
564/*  O(name,			type) */				\
565    O(tcache,			tcache_t *)				\
566    O(thread_allocated,		uint64_t)				\
567    O(thread_deallocated,	uint64_t)				\
568    O(prof_tdata,		prof_tdata_t *)				\
569    O(iarena,			arena_t *)				\
570    O(arena,			arena_t *)				\
571    O(arenas_tdata,		arena_tdata_t *)			\
572    O(narenas_tdata,		unsigned)				\
573    O(arenas_tdata_bypass,	bool)					\
574    O(tcache_enabled,		tcache_enabled_t)			\
575    O(quarantine,		quarantine_t *)				\
576    O(witnesses,		witness_list_t)				\
577    O(witness_fork,		bool)					\
578
579#define	TSD_INITIALIZER {						\
580    tsd_state_uninitialized,						\
581    NULL,								\
582    0,									\
583    0,									\
584    NULL,								\
585    NULL,								\
586    NULL,								\
587    NULL,								\
588    0,									\
589    false,								\
590    tcache_enabled_default,						\
591    NULL,								\
592    ql_head_initializer(witnesses),					\
593    false								\
594}
595
596struct tsd_s {
597	tsd_state_t	state;
598#define	O(n, t)								\
599	t		n;
600MALLOC_TSD
601#undef O
602};
603
604/*
605 * Wrapper around tsd_t that makes it possible to avoid implicit conversion
606 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be
607 * explicitly converted to tsd_t, which is non-nullable.
608 */
609struct tsdn_s {
610	tsd_t	tsd;
611};
612
613static const tsd_t tsd_initializer = TSD_INITIALIZER;
614
615malloc_tsd_types(, tsd_t)
616
617#endif /* JEMALLOC_H_STRUCTS */
618/******************************************************************************/
619#ifdef JEMALLOC_H_EXTERNS
620
621void	*malloc_tsd_malloc(size_t size);
622void	malloc_tsd_dalloc(void *wrapper);
623void	malloc_tsd_no_cleanup(void *arg);
624void	malloc_tsd_cleanup_register(bool (*f)(void));
625tsd_t	*malloc_tsd_boot0(void);
626void	malloc_tsd_boot1(void);
627#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
628    !defined(_WIN32))
629void	*tsd_init_check_recursion(tsd_init_head_t *head,
630    tsd_init_block_t *block);
631void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
632#endif
633void	tsd_cleanup(void *arg);
634
635#endif /* JEMALLOC_H_EXTERNS */
636/******************************************************************************/
637#ifdef JEMALLOC_H_INLINES
638
639#ifndef JEMALLOC_ENABLE_INLINE
640malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
641
642tsd_t	*tsd_fetch(void);
643tsdn_t	*tsd_tsdn(tsd_t *tsd);
644bool	tsd_nominal(tsd_t *tsd);
645#define	O(n, t)								\
646t	*tsd_##n##p_get(tsd_t *tsd);					\
647t	tsd_##n##_get(tsd_t *tsd);					\
648void	tsd_##n##_set(tsd_t *tsd, t n);
649MALLOC_TSD
650#undef O
651tsdn_t	*tsdn_fetch(void);
652bool	tsdn_null(const tsdn_t *tsdn);
653tsd_t	*tsdn_tsd(tsdn_t *tsdn);
654#endif
655
656#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
657malloc_tsd_externs(, tsd_t)
658malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
659
660JEMALLOC_ALWAYS_INLINE tsd_t *
661tsd_fetch(void)
662{
663	tsd_t *tsd = tsd_get();
664
665	if (unlikely(tsd->state != tsd_state_nominal)) {
666		if (tsd->state == tsd_state_uninitialized) {
667			tsd->state = tsd_state_nominal;
668			/* Trigger cleanup handler registration. */
669			tsd_set(tsd);
670		} else if (tsd->state == tsd_state_purgatory) {
671			tsd->state = tsd_state_reincarnated;
672			tsd_set(tsd);
673		} else
674			assert(tsd->state == tsd_state_reincarnated);
675	}
676
677	return (tsd);
678}
679
680JEMALLOC_ALWAYS_INLINE tsdn_t *
681tsd_tsdn(tsd_t *tsd)
682{
683
684	return ((tsdn_t *)tsd);
685}
686
687JEMALLOC_INLINE bool
688tsd_nominal(tsd_t *tsd)
689{
690
691	return (tsd->state == tsd_state_nominal);
692}
693
694#define	O(n, t)								\
695JEMALLOC_ALWAYS_INLINE t *						\
696tsd_##n##p_get(tsd_t *tsd)						\
697{									\
698									\
699	return (&tsd->n);						\
700}									\
701									\
702JEMALLOC_ALWAYS_INLINE t						\
703tsd_##n##_get(tsd_t *tsd)						\
704{									\
705									\
706	return (*tsd_##n##p_get(tsd));					\
707}									\
708									\
709JEMALLOC_ALWAYS_INLINE void						\
710tsd_##n##_set(tsd_t *tsd, t n)						\
711{									\
712									\
713	assert(tsd->state == tsd_state_nominal);			\
714	tsd->n = n;							\
715}
716MALLOC_TSD
717#undef O
718
719JEMALLOC_ALWAYS_INLINE tsdn_t *
720tsdn_fetch(void)
721{
722
723	if (!tsd_booted_get())
724		return (NULL);
725
726	return (tsd_tsdn(tsd_fetch()));
727}
728
729JEMALLOC_ALWAYS_INLINE bool
730tsdn_null(const tsdn_t *tsdn)
731{
732
733	return (tsdn == NULL);
734}
735
736JEMALLOC_ALWAYS_INLINE tsd_t *
737tsdn_tsd(tsdn_t *tsdn)
738{
739
740	assert(!tsdn_null(tsdn));
741
742	return (&tsdn->tsd);
743}
744#endif
745
746#endif /* JEMALLOC_H_INLINES */
747/******************************************************************************/
748