1/******************************************************************************/ 2#ifdef JEMALLOC_H_TYPES 3 4/* Maximum number of malloc_tsd users with cleanup functions. */ 5#define MALLOC_TSD_CLEANUPS_MAX 2 6 7typedef bool (*malloc_tsd_cleanup_t)(void); 8 9#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10 !defined(_WIN32)) 11typedef struct tsd_init_block_s tsd_init_block_t; 12typedef struct tsd_init_head_s tsd_init_head_t; 13#endif 14 15typedef struct tsd_s tsd_t; 16typedef struct tsdn_s tsdn_t; 17 18#define TSDN_NULL ((tsdn_t *)0) 19 20typedef enum { 21 tsd_state_uninitialized, 22 tsd_state_nominal, 23 tsd_state_purgatory, 24 tsd_state_reincarnated 25} tsd_state_t; 26 27/* 28 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 29 * are five macros that support (at least) three use cases: file-private, 30 * library-private, and library-private inlined. Following is an example 31 * library-private tsd variable: 32 * 33 * In example.h: 34 * typedef struct { 35 * int x; 36 * int y; 37 * } example_t; 38 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 39 * malloc_tsd_types(example_, example_t) 40 * malloc_tsd_protos(, example_, example_t) 41 * malloc_tsd_externs(example_, example_t) 42 * In example.c: 43 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) 44 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, 45 * example_tsd_cleanup) 46 * 47 * The result is a set of generated functions, e.g.: 48 * 49 * bool example_tsd_boot(void) {...} 50 * bool example_tsd_booted_get(void) {...} 51 * example_t *example_tsd_get() {...} 52 * void example_tsd_set(example_t *val) {...} 53 * 54 * Note that all of the functions deal in terms of (a_type *) rather than 55 * (a_type) so that it is possible to support non-pointer types (unlike 56 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 57 * cast to (void *). This means that the cleanup function needs to cast the 58 * function argument to (a_type *), then dereference the resulting pointer to 59 * access fields, e.g. 60 * 61 * void 62 * example_tsd_cleanup(void *arg) 63 * { 64 * example_t *example = (example_t *)arg; 65 * 66 * example->x = 42; 67 * [...] 68 * if ([want the cleanup function to be called again]) 69 * example_tsd_set(example); 70 * } 71 * 72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be 73 * called again. This is similar to how pthreads TSD destruction works, except 74 * that pthreads only calls the cleanup function again if the value was set to 75 * non-NULL. 76 */ 77 78/* malloc_tsd_types(). */ 79#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 80#define malloc_tsd_types(a_name, a_type) 81#elif (defined(JEMALLOC_TLS)) 82#define malloc_tsd_types(a_name, a_type) 83#elif (defined(_WIN32)) 84#define malloc_tsd_types(a_name, a_type) \ 85typedef struct { \ 86 bool initialized; \ 87 a_type val; \ 88} a_name##tsd_wrapper_t; 89#else 90#define malloc_tsd_types(a_name, a_type) \ 91typedef struct { \ 92 bool initialized; \ 93 a_type val; \ 94} a_name##tsd_wrapper_t; 95#endif 96 97/* malloc_tsd_protos(). */ 98#define malloc_tsd_protos(a_attr, a_name, a_type) \ 99a_attr bool \ 100a_name##tsd_boot0(void); \ 101a_attr void \ 102a_name##tsd_boot1(void); \ 103a_attr bool \ 104a_name##tsd_boot(void); \ 105a_attr bool \ 106a_name##tsd_booted_get(void); \ 107a_attr a_type * \ 108a_name##tsd_get(void); \ 109a_attr void \ 110a_name##tsd_set(a_type *val); 111 112/* malloc_tsd_externs(). */ 113#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 114#define malloc_tsd_externs(a_name, a_type) \ 115extern __thread a_type a_name##tsd_tls; \ 116extern __thread bool a_name##tsd_initialized; \ 117extern bool a_name##tsd_booted; 118#elif (defined(JEMALLOC_TLS)) 119#define malloc_tsd_externs(a_name, a_type) \ 120extern __thread a_type a_name##tsd_tls; \ 121extern pthread_key_t a_name##tsd_tsd; \ 122extern bool a_name##tsd_booted; 123#elif (defined(_WIN32)) 124#define malloc_tsd_externs(a_name, a_type) \ 125extern DWORD a_name##tsd_tsd; \ 126extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 127extern bool a_name##tsd_booted; 128#else 129#define malloc_tsd_externs(a_name, a_type) \ 130extern pthread_key_t a_name##tsd_tsd; \ 131extern tsd_init_head_t a_name##tsd_init_head; \ 132extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 133extern bool a_name##tsd_booted; 134#endif 135 136/* malloc_tsd_data(). */ 137#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 138#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 139a_attr __thread a_type JEMALLOC_TLS_MODEL \ 140 a_name##tsd_tls = a_initializer; \ 141a_attr __thread bool JEMALLOC_TLS_MODEL \ 142 a_name##tsd_initialized = false; \ 143a_attr bool a_name##tsd_booted = false; 144#elif (defined(JEMALLOC_TLS)) 145#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 146a_attr __thread a_type JEMALLOC_TLS_MODEL \ 147 a_name##tsd_tls = a_initializer; \ 148a_attr pthread_key_t a_name##tsd_tsd; \ 149a_attr bool a_name##tsd_booted = false; 150#elif (defined(_WIN32)) 151#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 152a_attr DWORD a_name##tsd_tsd; \ 153a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 154 false, \ 155 a_initializer \ 156}; \ 157a_attr bool a_name##tsd_booted = false; 158#else 159#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 160a_attr pthread_key_t a_name##tsd_tsd; \ 161a_attr tsd_init_head_t a_name##tsd_init_head = { \ 162 ql_head_initializer(blocks), \ 163 MALLOC_MUTEX_INITIALIZER \ 164}; \ 165a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 166 false, \ 167 a_initializer \ 168}; \ 169a_attr bool a_name##tsd_booted = false; 170#endif 171 172/* malloc_tsd_funcs(). */ 173#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 174#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 175 a_cleanup) \ 176/* Initialization/cleanup. */ \ 177a_attr bool \ 178a_name##tsd_cleanup_wrapper(void) \ 179{ \ 180 \ 181 if (a_name##tsd_initialized) { \ 182 a_name##tsd_initialized = false; \ 183 a_cleanup(&a_name##tsd_tls); \ 184 } \ 185 return (a_name##tsd_initialized); \ 186} \ 187a_attr bool \ 188a_name##tsd_boot0(void) \ 189{ \ 190 \ 191 if (a_cleanup != malloc_tsd_no_cleanup) { \ 192 malloc_tsd_cleanup_register( \ 193 &a_name##tsd_cleanup_wrapper); \ 194 } \ 195 a_name##tsd_booted = true; \ 196 return (false); \ 197} \ 198a_attr void \ 199a_name##tsd_boot1(void) \ 200{ \ 201 \ 202 /* Do nothing. */ \ 203} \ 204a_attr bool \ 205a_name##tsd_boot(void) \ 206{ \ 207 \ 208 return (a_name##tsd_boot0()); \ 209} \ 210a_attr bool \ 211a_name##tsd_booted_get(void) \ 212{ \ 213 \ 214 return (a_name##tsd_booted); \ 215} \ 216/* Get/set. */ \ 217a_attr a_type * \ 218a_name##tsd_get(void) \ 219{ \ 220 \ 221 assert(a_name##tsd_booted); \ 222 return (&a_name##tsd_tls); \ 223} \ 224a_attr void \ 225a_name##tsd_set(a_type *val) \ 226{ \ 227 \ 228 assert(a_name##tsd_booted); \ 229 a_name##tsd_tls = (*val); \ 230 if (a_cleanup != malloc_tsd_no_cleanup) \ 231 a_name##tsd_initialized = true; \ 232} 233#elif (defined(JEMALLOC_TLS)) 234#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 235 a_cleanup) \ 236/* Initialization/cleanup. */ \ 237a_attr bool \ 238a_name##tsd_boot0(void) \ 239{ \ 240 \ 241 if (a_cleanup != malloc_tsd_no_cleanup) { \ 242 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ 243 0) \ 244 return (true); \ 245 } \ 246 a_name##tsd_booted = true; \ 247 return (false); \ 248} \ 249a_attr void \ 250a_name##tsd_boot1(void) \ 251{ \ 252 \ 253 /* Do nothing. */ \ 254} \ 255a_attr bool \ 256a_name##tsd_boot(void) \ 257{ \ 258 \ 259 return (a_name##tsd_boot0()); \ 260} \ 261a_attr bool \ 262a_name##tsd_booted_get(void) \ 263{ \ 264 \ 265 return (a_name##tsd_booted); \ 266} \ 267/* Get/set. */ \ 268a_attr a_type * \ 269a_name##tsd_get(void) \ 270{ \ 271 \ 272 assert(a_name##tsd_booted); \ 273 return (&a_name##tsd_tls); \ 274} \ 275a_attr void \ 276a_name##tsd_set(a_type *val) \ 277{ \ 278 \ 279 assert(a_name##tsd_booted); \ 280 a_name##tsd_tls = (*val); \ 281 if (a_cleanup != malloc_tsd_no_cleanup) { \ 282 if (pthread_setspecific(a_name##tsd_tsd, \ 283 (void *)(&a_name##tsd_tls))) { \ 284 malloc_write("<jemalloc>: Error" \ 285 " setting TSD for "#a_name"\n"); \ 286 if (opt_abort) \ 287 abort(); \ 288 } \ 289 } \ 290} 291#elif (defined(_WIN32)) 292#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 293 a_cleanup) \ 294/* Initialization/cleanup. */ \ 295a_attr bool \ 296a_name##tsd_cleanup_wrapper(void) \ 297{ \ 298 DWORD error = GetLastError(); \ 299 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 300 TlsGetValue(a_name##tsd_tsd); \ 301 SetLastError(error); \ 302 \ 303 if (wrapper == NULL) \ 304 return (false); \ 305 if (a_cleanup != malloc_tsd_no_cleanup && \ 306 wrapper->initialized) { \ 307 wrapper->initialized = false; \ 308 a_cleanup(&wrapper->val); \ 309 if (wrapper->initialized) { \ 310 /* Trigger another cleanup round. */ \ 311 return (true); \ 312 } \ 313 } \ 314 malloc_tsd_dalloc(wrapper); \ 315 return (false); \ 316} \ 317a_attr void \ 318a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 319{ \ 320 \ 321 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ 322 malloc_write("<jemalloc>: Error setting" \ 323 " TSD for "#a_name"\n"); \ 324 abort(); \ 325 } \ 326} \ 327a_attr a_name##tsd_wrapper_t * \ 328a_name##tsd_wrapper_get(void) \ 329{ \ 330 DWORD error = GetLastError(); \ 331 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 332 TlsGetValue(a_name##tsd_tsd); \ 333 SetLastError(error); \ 334 \ 335 if (unlikely(wrapper == NULL)) { \ 336 wrapper = (a_name##tsd_wrapper_t *) \ 337 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 338 if (wrapper == NULL) { \ 339 malloc_write("<jemalloc>: Error allocating" \ 340 " TSD for "#a_name"\n"); \ 341 abort(); \ 342 } else { \ 343 wrapper->initialized = false; \ 344 wrapper->val = a_initializer; \ 345 } \ 346 a_name##tsd_wrapper_set(wrapper); \ 347 } \ 348 return (wrapper); \ 349} \ 350a_attr bool \ 351a_name##tsd_boot0(void) \ 352{ \ 353 \ 354 a_name##tsd_tsd = TlsAlloc(); \ 355 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ 356 return (true); \ 357 if (a_cleanup != malloc_tsd_no_cleanup) { \ 358 malloc_tsd_cleanup_register( \ 359 &a_name##tsd_cleanup_wrapper); \ 360 } \ 361 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 362 a_name##tsd_booted = true; \ 363 return (false); \ 364} \ 365a_attr void \ 366a_name##tsd_boot1(void) \ 367{ \ 368 a_name##tsd_wrapper_t *wrapper; \ 369 wrapper = (a_name##tsd_wrapper_t *) \ 370 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 371 if (wrapper == NULL) { \ 372 malloc_write("<jemalloc>: Error allocating" \ 373 " TSD for "#a_name"\n"); \ 374 abort(); \ 375 } \ 376 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 377 sizeof(a_name##tsd_wrapper_t)); \ 378 a_name##tsd_wrapper_set(wrapper); \ 379} \ 380a_attr bool \ 381a_name##tsd_boot(void) \ 382{ \ 383 \ 384 if (a_name##tsd_boot0()) \ 385 return (true); \ 386 a_name##tsd_boot1(); \ 387 return (false); \ 388} \ 389a_attr bool \ 390a_name##tsd_booted_get(void) \ 391{ \ 392 \ 393 return (a_name##tsd_booted); \ 394} \ 395/* Get/set. */ \ 396a_attr a_type * \ 397a_name##tsd_get(void) \ 398{ \ 399 a_name##tsd_wrapper_t *wrapper; \ 400 \ 401 assert(a_name##tsd_booted); \ 402 wrapper = a_name##tsd_wrapper_get(); \ 403 return (&wrapper->val); \ 404} \ 405a_attr void \ 406a_name##tsd_set(a_type *val) \ 407{ \ 408 a_name##tsd_wrapper_t *wrapper; \ 409 \ 410 assert(a_name##tsd_booted); \ 411 wrapper = a_name##tsd_wrapper_get(); \ 412 wrapper->val = *(val); \ 413 if (a_cleanup != malloc_tsd_no_cleanup) \ 414 wrapper->initialized = true; \ 415} 416#else 417#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 418 a_cleanup) \ 419/* Initialization/cleanup. */ \ 420a_attr void \ 421a_name##tsd_cleanup_wrapper(void *arg) \ 422{ \ 423 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ 424 \ 425 if (a_cleanup != malloc_tsd_no_cleanup && \ 426 wrapper->initialized) { \ 427 wrapper->initialized = false; \ 428 a_cleanup(&wrapper->val); \ 429 if (wrapper->initialized) { \ 430 /* Trigger another cleanup round. */ \ 431 if (pthread_setspecific(a_name##tsd_tsd, \ 432 (void *)wrapper)) { \ 433 malloc_write("<jemalloc>: Error" \ 434 " setting TSD for "#a_name"\n"); \ 435 if (opt_abort) \ 436 abort(); \ 437 } \ 438 return; \ 439 } \ 440 } \ 441 malloc_tsd_dalloc(wrapper); \ 442} \ 443a_attr void \ 444a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 445{ \ 446 \ 447 if (pthread_setspecific(a_name##tsd_tsd, \ 448 (void *)wrapper)) { \ 449 malloc_write("<jemalloc>: Error setting" \ 450 " TSD for "#a_name"\n"); \ 451 abort(); \ 452 } \ 453} \ 454a_attr a_name##tsd_wrapper_t * \ 455a_name##tsd_wrapper_get(void) \ 456{ \ 457 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 458 pthread_getspecific(a_name##tsd_tsd); \ 459 \ 460 if (unlikely(wrapper == NULL)) { \ 461 tsd_init_block_t block; \ 462 wrapper = tsd_init_check_recursion( \ 463 &a_name##tsd_init_head, &block); \ 464 if (wrapper) \ 465 return (wrapper); \ 466 wrapper = (a_name##tsd_wrapper_t *) \ 467 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 468 block.data = wrapper; \ 469 if (wrapper == NULL) { \ 470 malloc_write("<jemalloc>: Error allocating" \ 471 " TSD for "#a_name"\n"); \ 472 abort(); \ 473 } else { \ 474 wrapper->initialized = false; \ 475 wrapper->val = a_initializer; \ 476 } \ 477 a_name##tsd_wrapper_set(wrapper); \ 478 tsd_init_finish(&a_name##tsd_init_head, &block); \ 479 } \ 480 return (wrapper); \ 481} \ 482a_attr bool \ 483a_name##tsd_boot0(void) \ 484{ \ 485 \ 486 if (pthread_key_create(&a_name##tsd_tsd, \ 487 a_name##tsd_cleanup_wrapper) != 0) \ 488 return (true); \ 489 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 490 a_name##tsd_booted = true; \ 491 return (false); \ 492} \ 493a_attr void \ 494a_name##tsd_boot1(void) \ 495{ \ 496 a_name##tsd_wrapper_t *wrapper; \ 497 wrapper = (a_name##tsd_wrapper_t *) \ 498 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 499 if (wrapper == NULL) { \ 500 malloc_write("<jemalloc>: Error allocating" \ 501 " TSD for "#a_name"\n"); \ 502 abort(); \ 503 } \ 504 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 505 sizeof(a_name##tsd_wrapper_t)); \ 506 a_name##tsd_wrapper_set(wrapper); \ 507} \ 508a_attr bool \ 509a_name##tsd_boot(void) \ 510{ \ 511 \ 512 if (a_name##tsd_boot0()) \ 513 return (true); \ 514 a_name##tsd_boot1(); \ 515 return (false); \ 516} \ 517a_attr bool \ 518a_name##tsd_booted_get(void) \ 519{ \ 520 \ 521 return (a_name##tsd_booted); \ 522} \ 523/* Get/set. */ \ 524a_attr a_type * \ 525a_name##tsd_get(void) \ 526{ \ 527 a_name##tsd_wrapper_t *wrapper; \ 528 \ 529 assert(a_name##tsd_booted); \ 530 wrapper = a_name##tsd_wrapper_get(); \ 531 return (&wrapper->val); \ 532} \ 533a_attr void \ 534a_name##tsd_set(a_type *val) \ 535{ \ 536 a_name##tsd_wrapper_t *wrapper; \ 537 \ 538 assert(a_name##tsd_booted); \ 539 wrapper = a_name##tsd_wrapper_get(); \ 540 wrapper->val = *(val); \ 541 if (a_cleanup != malloc_tsd_no_cleanup) \ 542 wrapper->initialized = true; \ 543} 544#endif 545 546#endif /* JEMALLOC_H_TYPES */ 547/******************************************************************************/ 548#ifdef JEMALLOC_H_STRUCTS 549 550#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 551 !defined(_WIN32)) 552struct tsd_init_block_s { 553 ql_elm(tsd_init_block_t) link; 554 pthread_t thread; 555 void *data; 556}; 557struct tsd_init_head_s { 558 ql_head(tsd_init_block_t) blocks; 559 malloc_mutex_t lock; 560}; 561#endif 562 563#define MALLOC_TSD \ 564/* O(name, type) */ \ 565 O(tcache, tcache_t *) \ 566 O(thread_allocated, uint64_t) \ 567 O(thread_deallocated, uint64_t) \ 568 O(prof_tdata, prof_tdata_t *) \ 569 O(iarena, arena_t *) \ 570 O(arena, arena_t *) \ 571 O(arenas_tdata, arena_tdata_t *) \ 572 O(narenas_tdata, unsigned) \ 573 O(arenas_tdata_bypass, bool) \ 574 O(tcache_enabled, tcache_enabled_t) \ 575 O(quarantine, quarantine_t *) \ 576 O(witnesses, witness_list_t) \ 577 O(witness_fork, bool) \ 578 579#define TSD_INITIALIZER { \ 580 tsd_state_uninitialized, \ 581 NULL, \ 582 0, \ 583 0, \ 584 NULL, \ 585 NULL, \ 586 NULL, \ 587 NULL, \ 588 0, \ 589 false, \ 590 tcache_enabled_default, \ 591 NULL, \ 592 ql_head_initializer(witnesses), \ 593 false \ 594} 595 596struct tsd_s { 597 tsd_state_t state; 598#define O(n, t) \ 599 t n; 600MALLOC_TSD 601#undef O 602}; 603 604/* 605 * Wrapper around tsd_t that makes it possible to avoid implicit conversion 606 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be 607 * explicitly converted to tsd_t, which is non-nullable. 608 */ 609struct tsdn_s { 610 tsd_t tsd; 611}; 612 613static const tsd_t tsd_initializer = TSD_INITIALIZER; 614 615malloc_tsd_types(, tsd_t) 616 617#endif /* JEMALLOC_H_STRUCTS */ 618/******************************************************************************/ 619#ifdef JEMALLOC_H_EXTERNS 620 621void *malloc_tsd_malloc(size_t size); 622void malloc_tsd_dalloc(void *wrapper); 623void malloc_tsd_no_cleanup(void *arg); 624void malloc_tsd_cleanup_register(bool (*f)(void)); 625tsd_t *malloc_tsd_boot0(void); 626void malloc_tsd_boot1(void); 627#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 628 !defined(_WIN32)) 629void *tsd_init_check_recursion(tsd_init_head_t *head, 630 tsd_init_block_t *block); 631void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); 632#endif 633void tsd_cleanup(void *arg); 634 635#endif /* JEMALLOC_H_EXTERNS */ 636/******************************************************************************/ 637#ifdef JEMALLOC_H_INLINES 638 639#ifndef JEMALLOC_ENABLE_INLINE 640malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) 641 642tsd_t *tsd_fetch(void); 643tsdn_t *tsd_tsdn(tsd_t *tsd); 644bool tsd_nominal(tsd_t *tsd); 645#define O(n, t) \ 646t *tsd_##n##p_get(tsd_t *tsd); \ 647t tsd_##n##_get(tsd_t *tsd); \ 648void tsd_##n##_set(tsd_t *tsd, t n); 649MALLOC_TSD 650#undef O 651tsdn_t *tsdn_fetch(void); 652bool tsdn_null(const tsdn_t *tsdn); 653tsd_t *tsdn_tsd(tsdn_t *tsdn); 654#endif 655 656#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) 657malloc_tsd_externs(, tsd_t) 658malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) 659 660JEMALLOC_ALWAYS_INLINE tsd_t * 661tsd_fetch(void) 662{ 663 tsd_t *tsd = tsd_get(); 664 665 if (unlikely(tsd->state != tsd_state_nominal)) { 666 if (tsd->state == tsd_state_uninitialized) { 667 tsd->state = tsd_state_nominal; 668 /* Trigger cleanup handler registration. */ 669 tsd_set(tsd); 670 } else if (tsd->state == tsd_state_purgatory) { 671 tsd->state = tsd_state_reincarnated; 672 tsd_set(tsd); 673 } else 674 assert(tsd->state == tsd_state_reincarnated); 675 } 676 677 return (tsd); 678} 679 680JEMALLOC_ALWAYS_INLINE tsdn_t * 681tsd_tsdn(tsd_t *tsd) 682{ 683 684 return ((tsdn_t *)tsd); 685} 686 687JEMALLOC_INLINE bool 688tsd_nominal(tsd_t *tsd) 689{ 690 691 return (tsd->state == tsd_state_nominal); 692} 693 694#define O(n, t) \ 695JEMALLOC_ALWAYS_INLINE t * \ 696tsd_##n##p_get(tsd_t *tsd) \ 697{ \ 698 \ 699 return (&tsd->n); \ 700} \ 701 \ 702JEMALLOC_ALWAYS_INLINE t \ 703tsd_##n##_get(tsd_t *tsd) \ 704{ \ 705 \ 706 return (*tsd_##n##p_get(tsd)); \ 707} \ 708 \ 709JEMALLOC_ALWAYS_INLINE void \ 710tsd_##n##_set(tsd_t *tsd, t n) \ 711{ \ 712 \ 713 assert(tsd->state == tsd_state_nominal); \ 714 tsd->n = n; \ 715} 716MALLOC_TSD 717#undef O 718 719JEMALLOC_ALWAYS_INLINE tsdn_t * 720tsdn_fetch(void) 721{ 722 723 if (!tsd_booted_get()) 724 return (NULL); 725 726 return (tsd_tsdn(tsd_fetch())); 727} 728 729JEMALLOC_ALWAYS_INLINE bool 730tsdn_null(const tsdn_t *tsdn) 731{ 732 733 return (tsdn == NULL); 734} 735 736JEMALLOC_ALWAYS_INLINE tsd_t * 737tsdn_tsd(tsdn_t *tsdn) 738{ 739 740 assert(!tsdn_null(tsdn)); 741 742 return (&tsdn->tsd); 743} 744#endif 745 746#endif /* JEMALLOC_H_INLINES */ 747/******************************************************************************/ 748