tsd.h revision 288090 
1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4/* Maximum number of malloc_tsd users with cleanup functions. */
5#define	MALLOC_TSD_CLEANUPS_MAX	2
6
7typedef bool (*malloc_tsd_cleanup_t)(void);
8
9#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
10    !defined(_WIN32))
11typedef struct tsd_init_block_s tsd_init_block_t;
12typedef struct tsd_init_head_s tsd_init_head_t;
13#endif
14
15typedef struct tsd_s tsd_t;
16
17typedef enum {
18	tsd_state_uninitialized,
19	tsd_state_nominal,
20	tsd_state_purgatory,
21	tsd_state_reincarnated
22} tsd_state_t;
23
24/*
25 * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
26 * are five macros that support (at least) three use cases: file-private,
27 * library-private, and library-private inlined.  Following is an example
28 * library-private tsd variable:
29 *
30 * In example.h:
31 *   typedef struct {
32 *           int x;
33 *           int y;
34 *   } example_t;
35 *   #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
36 *   malloc_tsd_types(example_, example_t)
37 *   malloc_tsd_protos(, example_, example_t)
38 *   malloc_tsd_externs(example_, example_t)
39 * In example.c:
40 *   malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
41 *   malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
42 *       example_tsd_cleanup)
43 *
44 * The result is a set of generated functions, e.g.:
45 *
46 *   bool example_tsd_boot(void) {...}
47 *   example_t *example_tsd_get() {...}
48 *   void example_tsd_set(example_t *val) {...}
49 *
50 * Note that all of the functions deal in terms of (a_type *) rather than
51 * (a_type) so that it is possible to support non-pointer types (unlike
52 * pthreads TSD).  example_tsd_cleanup() is passed an (a_type *) pointer that is
53 * cast to (void *).  This means that the cleanup function needs to cast the
54 * function argument to (a_type *), then dereference the resulting pointer to
55 * access fields, e.g.
56 *
57 *   void
58 *   example_tsd_cleanup(void *arg)
59 *   {
60 *           example_t *example = (example_t *)arg;
61 *
62 *           example->x = 42;
63 *           [...]
64 *           if ([want the cleanup function to be called again])
65 *                   example_tsd_set(example);
66 *   }
67 *
68 * If example_tsd_set() is called within example_tsd_cleanup(), it will be
69 * called again.  This is similar to how pthreads TSD destruction works, except
70 * that pthreads only calls the cleanup function again if the value was set to
71 * non-NULL.
72 */
73
74/* malloc_tsd_types(). */
75#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
76#define	malloc_tsd_types(a_name, a_type)
77#elif (defined(JEMALLOC_TLS))
78#define	malloc_tsd_types(a_name, a_type)
79#elif (defined(_WIN32))
80#define	malloc_tsd_types(a_name, a_type)				\
81typedef struct {							\
82	bool	initialized;						\
83	a_type	val;							\
84} a_name##tsd_wrapper_t;
85#else
86#define	malloc_tsd_types(a_name, a_type)				\
87typedef struct {							\
88	bool	initialized;						\
89	a_type	val;							\
90} a_name##tsd_wrapper_t;
91#endif
92
93/* malloc_tsd_protos(). */
94#define	malloc_tsd_protos(a_attr, a_name, a_type)			\
95a_attr bool								\
96a_name##tsd_boot0(void);						\
97a_attr void								\
98a_name##tsd_boot1(void);						\
99a_attr bool								\
100a_name##tsd_boot(void);							\
101a_attr a_type *								\
102a_name##tsd_get(void);							\
103a_attr void								\
104a_name##tsd_set(a_type *val);
105
106/* malloc_tsd_externs(). */
107#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
108#define	malloc_tsd_externs(a_name, a_type)				\
109extern __thread a_type	a_name##tsd_tls;				\
110extern __thread bool	a_name##tsd_initialized;			\
111extern bool		a_name##tsd_booted;
112#elif (defined(JEMALLOC_TLS))
113#define	malloc_tsd_externs(a_name, a_type)				\
114extern __thread a_type	a_name##tsd_tls;				\
115extern pthread_key_t	a_name##tsd_tsd;				\
116extern bool		a_name##tsd_booted;
117#elif (defined(_WIN32))
118#define	malloc_tsd_externs(a_name, a_type)				\
119extern DWORD		a_name##tsd_tsd;				\
120extern a_name##tsd_wrapper_t	a_name##tsd_boot_wrapper;		\
121extern bool		a_name##tsd_booted;
122#else
123#define	malloc_tsd_externs(a_name, a_type)				\
124extern pthread_key_t	a_name##tsd_tsd;				\
125extern tsd_init_head_t	a_name##tsd_init_head;				\
126extern a_name##tsd_wrapper_t	a_name##tsd_boot_wrapper;		\
127extern bool		a_name##tsd_booted;
128#endif
129
130/* malloc_tsd_data(). */
131#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
132#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
133a_attr __thread a_type JEMALLOC_TLS_MODEL				\
134    a_name##tsd_tls = a_initializer;					\
135a_attr __thread bool JEMALLOC_TLS_MODEL					\
136    a_name##tsd_initialized = false;					\
137a_attr bool		a_name##tsd_booted = false;
138#elif (defined(JEMALLOC_TLS))
139#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
140a_attr __thread a_type JEMALLOC_TLS_MODEL				\
141    a_name##tsd_tls = a_initializer;					\
142a_attr pthread_key_t	a_name##tsd_tsd;				\
143a_attr bool		a_name##tsd_booted = false;
144#elif (defined(_WIN32))
145#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
146a_attr DWORD		a_name##tsd_tsd;				\
147a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {		\
148	false,								\
149	a_initializer							\
150};									\
151a_attr bool		a_name##tsd_booted = false;
152#else
153#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
154a_attr pthread_key_t	a_name##tsd_tsd;				\
155a_attr tsd_init_head_t	a_name##tsd_init_head = {			\
156	ql_head_initializer(blocks),					\
157	MALLOC_MUTEX_INITIALIZER					\
158};									\
159a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {		\
160	false,								\
161	a_initializer							\
162};									\
163a_attr bool		a_name##tsd_booted = false;
164#endif
165
166/* malloc_tsd_funcs(). */
167#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
168#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
169    a_cleanup)								\
170/* Initialization/cleanup. */						\
171a_attr bool								\
172a_name##tsd_cleanup_wrapper(void)					\
173{									\
174									\
175	if (a_name##tsd_initialized) {					\
176		a_name##tsd_initialized = false;			\
177		a_cleanup(&a_name##tsd_tls);				\
178	}								\
179	return (a_name##tsd_initialized);				\
180}									\
181a_attr bool								\
182a_name##tsd_boot0(void)							\
183{									\
184									\
185	if (a_cleanup != malloc_tsd_no_cleanup) {			\
186		malloc_tsd_cleanup_register(				\
187		    &a_name##tsd_cleanup_wrapper);			\
188	}								\
189	a_name##tsd_booted = true;					\
190	return (false);							\
191}									\
192a_attr void								\
193a_name##tsd_boot1(void)							\
194{									\
195									\
196	/* Do nothing. */						\
197}									\
198a_attr bool								\
199a_name##tsd_boot(void)							\
200{									\
201									\
202	return (a_name##tsd_boot0());					\
203}									\
204/* Get/set. */								\
205a_attr a_type *								\
206a_name##tsd_get(void)							\
207{									\
208									\
209	assert(a_name##tsd_booted);					\
210	return (&a_name##tsd_tls);					\
211}									\
212a_attr void								\
213a_name##tsd_set(a_type *val)						\
214{									\
215									\
216	assert(a_name##tsd_booted);					\
217	a_name##tsd_tls = (*val);					\
218	if (a_cleanup != malloc_tsd_no_cleanup)				\
219		a_name##tsd_initialized = true;				\
220}
221#elif (defined(JEMALLOC_TLS))
222#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
223    a_cleanup)								\
224/* Initialization/cleanup. */						\
225a_attr bool								\
226a_name##tsd_boot0(void)							\
227{									\
228									\
229	if (a_cleanup != malloc_tsd_no_cleanup) {			\
230		if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) !=	\
231		    0)							\
232			return (true);					\
233	}								\
234	a_name##tsd_booted = true;					\
235	return (false);							\
236}									\
237a_attr void								\
238a_name##tsd_boot1(void)							\
239{									\
240									\
241	/* Do nothing. */						\
242}									\
243a_attr bool								\
244a_name##tsd_boot(void)							\
245{									\
246									\
247	return (a_name##tsd_boot0());					\
248}									\
249/* Get/set. */								\
250a_attr a_type *								\
251a_name##tsd_get(void)							\
252{									\
253									\
254	assert(a_name##tsd_booted);					\
255	return (&a_name##tsd_tls);					\
256}									\
257a_attr void								\
258a_name##tsd_set(a_type *val)						\
259{									\
260									\
261	assert(a_name##tsd_booted);					\
262	a_name##tsd_tls = (*val);					\
263	if (a_cleanup != malloc_tsd_no_cleanup) {			\
264		if (pthread_setspecific(a_name##tsd_tsd,		\
265		    (void *)(&a_name##tsd_tls))) {			\
266			malloc_write("<jemalloc>: Error"		\
267			    " setting TSD for "#a_name"\n");		\
268			if (opt_abort)					\
269				abort();				\
270		}							\
271	}								\
272}
273#elif (defined(_WIN32))
274#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
275    a_cleanup)								\
276/* Initialization/cleanup. */						\
277a_attr bool								\
278a_name##tsd_cleanup_wrapper(void)					\
279{									\
280	DWORD error = GetLastError();					\
281	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
282	    TlsGetValue(a_name##tsd_tsd);				\
283	SetLastError(error);						\
284									\
285	if (wrapper == NULL)						\
286		return (false);						\
287	if (a_cleanup != malloc_tsd_no_cleanup &&			\
288	    wrapper->initialized) {					\
289		wrapper->initialized = false;				\
290		a_cleanup(&wrapper->val);				\
291		if (wrapper->initialized) {				\
292			/* Trigger another cleanup round. */		\
293			return (true);					\
294		}							\
295	}								\
296	malloc_tsd_dalloc(wrapper);					\
297	return (false);							\
298}									\
299a_attr void								\
300a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)			\
301{									\
302									\
303	if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) {		\
304		malloc_write("<jemalloc>: Error setting"		\
305		    " TSD for "#a_name"\n");				\
306		abort();						\
307	}								\
308}									\
309a_attr a_name##tsd_wrapper_t *						\
310a_name##tsd_wrapper_get(void)						\
311{									\
312	DWORD error = GetLastError();					\
313	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
314	    TlsGetValue(a_name##tsd_tsd);				\
315	SetLastError(error);						\
316									\
317	if (unlikely(wrapper == NULL)) {				\
318		wrapper = (a_name##tsd_wrapper_t *)			\
319		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
320		if (wrapper == NULL) {					\
321			malloc_write("<jemalloc>: Error allocating"	\
322			    " TSD for "#a_name"\n");			\
323			abort();					\
324		} else {						\
325			wrapper->initialized = false;			\
326			wrapper->val = a_initializer;			\
327		}							\
328		a_name##tsd_wrapper_set(wrapper);			\
329	}								\
330	return (wrapper);						\
331}									\
332a_attr bool								\
333a_name##tsd_boot0(void)							\
334{									\
335									\
336	a_name##tsd_tsd = TlsAlloc();					\
337	if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES)			\
338		return (true);						\
339	if (a_cleanup != malloc_tsd_no_cleanup) {			\
340		malloc_tsd_cleanup_register(				\
341		    &a_name##tsd_cleanup_wrapper);			\
342	}								\
343	a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);		\
344	a_name##tsd_booted = true;					\
345	return (false);							\
346}									\
347a_attr void								\
348a_name##tsd_boot1(void)							\
349{									\
350	a_name##tsd_wrapper_t *wrapper;					\
351	wrapper = (a_name##tsd_wrapper_t *)				\
352	    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));		\
353	if (wrapper == NULL) {						\
354		malloc_write("<jemalloc>: Error allocating"		\
355		    " TSD for "#a_name"\n");				\
356		abort();						\
357	}								\
358	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
359	    sizeof(a_name##tsd_wrapper_t));				\
360	a_name##tsd_wrapper_set(wrapper);				\
361}									\
362a_attr bool								\
363a_name##tsd_boot(void)							\
364{									\
365									\
366	if (a_name##tsd_boot0())					\
367		return (true);						\
368	a_name##tsd_boot1();						\
369	return (false);							\
370}									\
371/* Get/set. */								\
372a_attr a_type *								\
373a_name##tsd_get(void)							\
374{									\
375	a_name##tsd_wrapper_t *wrapper;					\
376									\
377	assert(a_name##tsd_booted);					\
378	wrapper = a_name##tsd_wrapper_get();				\
379	return (&wrapper->val);						\
380}									\
381a_attr void								\
382a_name##tsd_set(a_type *val)						\
383{									\
384	a_name##tsd_wrapper_t *wrapper;					\
385									\
386	assert(a_name##tsd_booted);					\
387	wrapper = a_name##tsd_wrapper_get();				\
388	wrapper->val = *(val);						\
389	if (a_cleanup != malloc_tsd_no_cleanup)				\
390		wrapper->initialized = true;				\
391}
392#else
393#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
394    a_cleanup)								\
395/* Initialization/cleanup. */						\
396a_attr void								\
397a_name##tsd_cleanup_wrapper(void *arg)					\
398{									\
399	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg;	\
400									\
401	if (a_cleanup != malloc_tsd_no_cleanup &&			\
402	    wrapper->initialized) {					\
403		wrapper->initialized = false;				\
404		a_cleanup(&wrapper->val);				\
405		if (wrapper->initialized) {				\
406			/* Trigger another cleanup round. */		\
407			if (pthread_setspecific(a_name##tsd_tsd,	\
408			    (void *)wrapper)) {				\
409				malloc_write("<jemalloc>: Error"	\
410				    " setting TSD for "#a_name"\n");	\
411				if (opt_abort)				\
412					abort();			\
413			}						\
414			return;						\
415		}							\
416	}								\
417	malloc_tsd_dalloc(wrapper);					\
418}									\
419a_attr void								\
420a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)			\
421{									\
422									\
423	if (pthread_setspecific(a_name##tsd_tsd,			\
424	    (void *)wrapper)) {						\
425		malloc_write("<jemalloc>: Error setting"		\
426		    " TSD for "#a_name"\n");				\
427		abort();						\
428	}								\
429}									\
430a_attr a_name##tsd_wrapper_t *						\
431a_name##tsd_wrapper_get(void)						\
432{									\
433	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
434	    pthread_getspecific(a_name##tsd_tsd);			\
435									\
436	if (unlikely(wrapper == NULL)) {				\
437		tsd_init_block_t block;					\
438		wrapper = tsd_init_check_recursion(			\
439		    &a_name##tsd_init_head, &block);			\
440		if (wrapper)						\
441		    return (wrapper);					\
442		wrapper = (a_name##tsd_wrapper_t *)			\
443		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
444		block.data = wrapper;					\
445		if (wrapper == NULL) {					\
446			malloc_write("<jemalloc>: Error allocating"	\
447			    " TSD for "#a_name"\n");			\
448			abort();					\
449		} else {						\
450			wrapper->initialized = false;			\
451			wrapper->val = a_initializer;			\
452		}							\
453		a_name##tsd_wrapper_set(wrapper);			\
454		tsd_init_finish(&a_name##tsd_init_head, &block);	\
455	}								\
456	return (wrapper);						\
457}									\
458a_attr bool								\
459a_name##tsd_boot0(void)							\
460{									\
461									\
462	if (pthread_key_create(&a_name##tsd_tsd,			\
463	    a_name##tsd_cleanup_wrapper) != 0)				\
464		return (true);						\
465	a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);		\
466	a_name##tsd_booted = true;					\
467	return (false);							\
468}									\
469a_attr void								\
470a_name##tsd_boot1(void)							\
471{									\
472	a_name##tsd_wrapper_t *wrapper;					\
473	wrapper = (a_name##tsd_wrapper_t *)				\
474	    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));		\
475	if (wrapper == NULL) {						\
476		malloc_write("<jemalloc>: Error allocating"		\
477		    " TSD for "#a_name"\n");				\
478		abort();						\
479	}								\
480	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
481	    sizeof(a_name##tsd_wrapper_t));				\
482	a_name##tsd_wrapper_set(wrapper);				\
483}									\
484a_attr bool								\
485a_name##tsd_boot(void)							\
486{									\
487									\
488	if (a_name##tsd_boot0())					\
489		return (true);						\
490	a_name##tsd_boot1();						\
491	return (false);							\
492}									\
493/* Get/set. */								\
494a_attr a_type *								\
495a_name##tsd_get(void)							\
496{									\
497	a_name##tsd_wrapper_t *wrapper;					\
498									\
499	assert(a_name##tsd_booted);					\
500	wrapper = a_name##tsd_wrapper_get();				\
501	return (&wrapper->val);						\
502}									\
503a_attr void								\
504a_name##tsd_set(a_type *val)						\
505{									\
506	a_name##tsd_wrapper_t *wrapper;					\
507									\
508	assert(a_name##tsd_booted);					\
509	wrapper = a_name##tsd_wrapper_get();				\
510	wrapper->val = *(val);						\
511	if (a_cleanup != malloc_tsd_no_cleanup)				\
512		wrapper->initialized = true;				\
513}
514#endif
515
516#endif /* JEMALLOC_H_TYPES */
517/******************************************************************************/
518#ifdef JEMALLOC_H_STRUCTS
519
520#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
521    !defined(_WIN32))
522struct tsd_init_block_s {
523	ql_elm(tsd_init_block_t)	link;
524	pthread_t			thread;
525	void				*data;
526};
527struct tsd_init_head_s {
528	ql_head(tsd_init_block_t)	blocks;
529	malloc_mutex_t			lock;
530};
531#endif
532
533#define	MALLOC_TSD							\
534/*  O(name,			type) */				\
535    O(tcache,			tcache_t *)				\
536    O(thread_allocated,		uint64_t)				\
537    O(thread_deallocated,	uint64_t)				\
538    O(prof_tdata,		prof_tdata_t *)				\
539    O(arena,			arena_t *)				\
540    O(arenas_cache,		arena_t **)				\
541    O(narenas_cache,		unsigned)				\
542    O(arenas_cache_bypass,	bool)					\
543    O(tcache_enabled,		tcache_enabled_t)			\
544    O(quarantine,		quarantine_t *)				\
545
546#define	TSD_INITIALIZER {						\
547    tsd_state_uninitialized,						\
548    NULL,								\
549    0,									\
550    0,									\
551    NULL,								\
552    NULL,								\
553    NULL,								\
554    0,									\
555    false,								\
556    tcache_enabled_default,						\
557    NULL								\
558}
559
560struct tsd_s {
561	tsd_state_t	state;
562#define	O(n, t)								\
563	t		n;
564MALLOC_TSD
565#undef O
566};
567
568static const tsd_t tsd_initializer = TSD_INITIALIZER;
569
570malloc_tsd_types(, tsd_t)
571
572#endif /* JEMALLOC_H_STRUCTS */
573/******************************************************************************/
574#ifdef JEMALLOC_H_EXTERNS
575
576void	*malloc_tsd_malloc(size_t size);
577void	malloc_tsd_dalloc(void *wrapper);
578void	malloc_tsd_no_cleanup(void *arg);
579void	malloc_tsd_cleanup_register(bool (*f)(void));
580bool	malloc_tsd_boot0(void);
581void	malloc_tsd_boot1(void);
582#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
583    !defined(_WIN32))
584void	*tsd_init_check_recursion(tsd_init_head_t *head,
585    tsd_init_block_t *block);
586void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
587#endif
588void	tsd_cleanup(void *arg);
589
590#endif /* JEMALLOC_H_EXTERNS */
591/******************************************************************************/
592#ifdef JEMALLOC_H_INLINES
593
594#ifndef JEMALLOC_ENABLE_INLINE
595malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
596
597tsd_t	*tsd_fetch(void);
598bool	tsd_nominal(tsd_t *tsd);
599#define	O(n, t)								\
600t	*tsd_##n##p_get(tsd_t *tsd);					\
601t	tsd_##n##_get(tsd_t *tsd);					\
602void	tsd_##n##_set(tsd_t *tsd, t n);
603MALLOC_TSD
604#undef O
605#endif
606
607#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
608malloc_tsd_externs(, tsd_t)
609malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
610
611JEMALLOC_ALWAYS_INLINE tsd_t *
612tsd_fetch(void)
613{
614	tsd_t *tsd = tsd_get();
615
616	if (unlikely(tsd->state != tsd_state_nominal)) {
617		if (tsd->state == tsd_state_uninitialized) {
618			tsd->state = tsd_state_nominal;
619			/* Trigger cleanup handler registration. */
620			tsd_set(tsd);
621		} else if (tsd->state == tsd_state_purgatory) {
622			tsd->state = tsd_state_reincarnated;
623			tsd_set(tsd);
624		} else
625			assert(tsd->state == tsd_state_reincarnated);
626	}
627
628	return (tsd);
629}
630
631JEMALLOC_INLINE bool
632tsd_nominal(tsd_t *tsd)
633{
634
635	return (tsd->state == tsd_state_nominal);
636}
637
638#define	O(n, t)								\
639JEMALLOC_ALWAYS_INLINE t *						\
640tsd_##n##p_get(tsd_t *tsd)						\
641{									\
642									\
643	return (&tsd->n);						\
644}									\
645									\
646JEMALLOC_ALWAYS_INLINE t						\
647tsd_##n##_get(tsd_t *tsd)						\
648{									\
649									\
650	return (*tsd_##n##p_get(tsd));					\
651}									\
652									\
653JEMALLOC_ALWAYS_INLINE void						\
654tsd_##n##_set(tsd_t *tsd, t n)						\
655{									\
656									\
657	assert(tsd->state == tsd_state_nominal);			\
658	tsd->n = n;							\
659}
660MALLOC_TSD
661#undef O
662#endif
663
664#endif /* JEMALLOC_H_INLINES */
665/******************************************************************************/
666