1234370Sjasone/******************************************************************************/ 2234370Sjasone#ifdef JEMALLOC_H_TYPES 3234370Sjasone 4234370Sjasone/* Maximum number of malloc_tsd users with cleanup functions. */ 5286866Sjasone#define MALLOC_TSD_CLEANUPS_MAX 2 6234370Sjasone 7234543Sjasonetypedef bool (*malloc_tsd_cleanup_t)(void); 8234370Sjasone 9261071Sjasone#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10261071Sjasone !defined(_WIN32)) 11261071Sjasonetypedef struct tsd_init_block_s tsd_init_block_t; 12261071Sjasonetypedef struct tsd_init_head_s tsd_init_head_t; 13261071Sjasone#endif 14261071Sjasone 15286866Sjasonetypedef struct tsd_s tsd_t; 16299587Sjasonetypedef struct tsdn_s tsdn_t; 17286866Sjasone 18299587Sjasone#define TSDN_NULL ((tsdn_t *)0) 19299587Sjasone 20286866Sjasonetypedef enum { 21286866Sjasone tsd_state_uninitialized, 22286866Sjasone tsd_state_nominal, 23286866Sjasone tsd_state_purgatory, 24286866Sjasone tsd_state_reincarnated 25286866Sjasone} tsd_state_t; 26286866Sjasone 27234370Sjasone/* 28234370Sjasone * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 29286866Sjasone * are five macros that support (at least) three use cases: file-private, 30234370Sjasone * library-private, and library-private inlined. Following is an example 31234370Sjasone * library-private tsd variable: 32234370Sjasone * 33234370Sjasone * In example.h: 34234370Sjasone * typedef struct { 35234370Sjasone * int x; 36234370Sjasone * int y; 37234370Sjasone * } example_t; 38234370Sjasone * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 39286866Sjasone * malloc_tsd_types(example_, example_t) 40286866Sjasone * malloc_tsd_protos(, example_, example_t) 41286866Sjasone * malloc_tsd_externs(example_, example_t) 42234370Sjasone * In example.c: 43286866Sjasone * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) 44286866Sjasone * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, 45234370Sjasone * example_tsd_cleanup) 46234370Sjasone * 47234370Sjasone * The result is a set of generated functions, e.g.: 48234370Sjasone * 49234370Sjasone * bool example_tsd_boot(void) {...} 50299587Sjasone * bool example_tsd_booted_get(void) {...} 51286866Sjasone * example_t *example_tsd_get() {...} 52286866Sjasone * void example_tsd_set(example_t *val) {...} 53234370Sjasone * 54234370Sjasone * Note that all of the functions deal in terms of (a_type *) rather than 55286866Sjasone * (a_type) so that it is possible to support non-pointer types (unlike 56234370Sjasone * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 57286866Sjasone * cast to (void *). This means that the cleanup function needs to cast the 58286866Sjasone * function argument to (a_type *), then dereference the resulting pointer to 59286866Sjasone * access fields, e.g. 60234370Sjasone * 61234370Sjasone * void 62234370Sjasone * example_tsd_cleanup(void *arg) 63234370Sjasone * { 64286866Sjasone * example_t *example = (example_t *)arg; 65234370Sjasone * 66286866Sjasone * example->x = 42; 67234370Sjasone * [...] 68286866Sjasone * if ([want the cleanup function to be called again]) 69286866Sjasone * example_tsd_set(example); 70234370Sjasone * } 71234370Sjasone * 72234370Sjasone * If example_tsd_set() is called within example_tsd_cleanup(), it will be 73234370Sjasone * called again. This is similar to how pthreads TSD destruction works, except 74234370Sjasone * that pthreads only calls the cleanup function again if the value was set to 75234370Sjasone * non-NULL. 76234370Sjasone */ 77234370Sjasone 78286866Sjasone/* malloc_tsd_types(). */ 79286866Sjasone#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 80286866Sjasone#define malloc_tsd_types(a_name, a_type) 81286866Sjasone#elif (defined(JEMALLOC_TLS)) 82286866Sjasone#define malloc_tsd_types(a_name, a_type) 83286866Sjasone#elif (defined(_WIN32)) 84286866Sjasone#define malloc_tsd_types(a_name, a_type) \ 85286866Sjasonetypedef struct { \ 86286866Sjasone bool initialized; \ 87286866Sjasone a_type val; \ 88286866Sjasone} a_name##tsd_wrapper_t; 89286866Sjasone#else 90286866Sjasone#define malloc_tsd_types(a_name, a_type) \ 91286866Sjasonetypedef struct { \ 92286866Sjasone bool initialized; \ 93286866Sjasone a_type val; \ 94286866Sjasone} a_name##tsd_wrapper_t; 95286866Sjasone#endif 96286866Sjasone 97234370Sjasone/* malloc_tsd_protos(). */ 98234370Sjasone#define malloc_tsd_protos(a_attr, a_name, a_type) \ 99234370Sjasonea_attr bool \ 100286866Sjasonea_name##tsd_boot0(void); \ 101286866Sjasonea_attr void \ 102286866Sjasonea_name##tsd_boot1(void); \ 103286866Sjasonea_attr bool \ 104286866Sjasonea_name##tsd_boot(void); \ 105299587Sjasonea_attr bool \ 106299587Sjasonea_name##tsd_booted_get(void); \ 107234370Sjasonea_attr a_type * \ 108286866Sjasonea_name##tsd_get(void); \ 109234370Sjasonea_attr void \ 110286866Sjasonea_name##tsd_set(a_type *val); 111234370Sjasone 112234370Sjasone/* malloc_tsd_externs(). */ 113234370Sjasone#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 114234370Sjasone#define malloc_tsd_externs(a_name, a_type) \ 115286866Sjasoneextern __thread a_type a_name##tsd_tls; \ 116286866Sjasoneextern __thread bool a_name##tsd_initialized; \ 117286866Sjasoneextern bool a_name##tsd_booted; 118234370Sjasone#elif (defined(JEMALLOC_TLS)) 119234370Sjasone#define malloc_tsd_externs(a_name, a_type) \ 120286866Sjasoneextern __thread a_type a_name##tsd_tls; \ 121286866Sjasoneextern pthread_key_t a_name##tsd_tsd; \ 122286866Sjasoneextern bool a_name##tsd_booted; 123235238Sjasone#elif (defined(_WIN32)) 124261071Sjasone#define malloc_tsd_externs(a_name, a_type) \ 125286866Sjasoneextern DWORD a_name##tsd_tsd; \ 126286866Sjasoneextern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 127286866Sjasoneextern bool a_name##tsd_booted; 128234370Sjasone#else 129234370Sjasone#define malloc_tsd_externs(a_name, a_type) \ 130286866Sjasoneextern pthread_key_t a_name##tsd_tsd; \ 131286866Sjasoneextern tsd_init_head_t a_name##tsd_init_head; \ 132286866Sjasoneextern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 133286866Sjasoneextern bool a_name##tsd_booted; 134234370Sjasone#endif 135234370Sjasone 136234370Sjasone/* malloc_tsd_data(). */ 137234370Sjasone#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 138234370Sjasone#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 139234370Sjasonea_attr __thread a_type JEMALLOC_TLS_MODEL \ 140286866Sjasone a_name##tsd_tls = a_initializer; \ 141234370Sjasonea_attr __thread bool JEMALLOC_TLS_MODEL \ 142286866Sjasone a_name##tsd_initialized = false; \ 143286866Sjasonea_attr bool a_name##tsd_booted = false; 144234370Sjasone#elif (defined(JEMALLOC_TLS)) 145234370Sjasone#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 146234370Sjasonea_attr __thread a_type JEMALLOC_TLS_MODEL \ 147286866Sjasone a_name##tsd_tls = a_initializer; \ 148286866Sjasonea_attr pthread_key_t a_name##tsd_tsd; \ 149286866Sjasonea_attr bool a_name##tsd_booted = false; 150235238Sjasone#elif (defined(_WIN32)) 151235238Sjasone#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 152286866Sjasonea_attr DWORD a_name##tsd_tsd; \ 153286866Sjasonea_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 154286866Sjasone false, \ 155286866Sjasone a_initializer \ 156286866Sjasone}; \ 157286866Sjasonea_attr bool a_name##tsd_booted = false; 158234370Sjasone#else 159234370Sjasone#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 160286866Sjasonea_attr pthread_key_t a_name##tsd_tsd; \ 161286866Sjasonea_attr tsd_init_head_t a_name##tsd_init_head = { \ 162261071Sjasone ql_head_initializer(blocks), \ 163261071Sjasone MALLOC_MUTEX_INITIALIZER \ 164261071Sjasone}; \ 165286866Sjasonea_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 166286866Sjasone false, \ 167286866Sjasone a_initializer \ 168286866Sjasone}; \ 169286866Sjasonea_attr bool a_name##tsd_booted = false; 170234370Sjasone#endif 171234370Sjasone 172234370Sjasone/* malloc_tsd_funcs(). */ 173234370Sjasone#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 174234370Sjasone#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 175234370Sjasone a_cleanup) \ 176234370Sjasone/* Initialization/cleanup. */ \ 177234370Sjasonea_attr bool \ 178286866Sjasonea_name##tsd_cleanup_wrapper(void) \ 179234370Sjasone{ \ 180234370Sjasone \ 181286866Sjasone if (a_name##tsd_initialized) { \ 182286866Sjasone a_name##tsd_initialized = false; \ 183286866Sjasone a_cleanup(&a_name##tsd_tls); \ 184234370Sjasone } \ 185286866Sjasone return (a_name##tsd_initialized); \ 186234370Sjasone} \ 187234370Sjasonea_attr bool \ 188286866Sjasonea_name##tsd_boot0(void) \ 189234370Sjasone{ \ 190234370Sjasone \ 191234370Sjasone if (a_cleanup != malloc_tsd_no_cleanup) { \ 192234370Sjasone malloc_tsd_cleanup_register( \ 193286866Sjasone &a_name##tsd_cleanup_wrapper); \ 194234370Sjasone } \ 195286866Sjasone a_name##tsd_booted = true; \ 196234370Sjasone return (false); \ 197234370Sjasone} \ 198286866Sjasonea_attr void \ 199288090Sjasonea_name##tsd_boot1(void) \ 200286866Sjasone{ \ 201286866Sjasone \ 202286866Sjasone /* Do nothing. */ \ 203286866Sjasone} \ 204286866Sjasonea_attr bool \ 205286866Sjasonea_name##tsd_boot(void) \ 206286866Sjasone{ \ 207286866Sjasone \ 208286866Sjasone return (a_name##tsd_boot0()); \ 209286866Sjasone} \ 210299587Sjasonea_attr bool \ 211299587Sjasonea_name##tsd_booted_get(void) \ 212299587Sjasone{ \ 213299587Sjasone \ 214299587Sjasone return (a_name##tsd_booted); \ 215299587Sjasone} \ 216234370Sjasone/* Get/set. */ \ 217234370Sjasonea_attr a_type * \ 218286866Sjasonea_name##tsd_get(void) \ 219234370Sjasone{ \ 220234370Sjasone \ 221286866Sjasone assert(a_name##tsd_booted); \ 222286866Sjasone return (&a_name##tsd_tls); \ 223234370Sjasone} \ 224234370Sjasonea_attr void \ 225286866Sjasonea_name##tsd_set(a_type *val) \ 226234370Sjasone{ \ 227234370Sjasone \ 228286866Sjasone assert(a_name##tsd_booted); \ 229286866Sjasone a_name##tsd_tls = (*val); \ 230234370Sjasone if (a_cleanup != malloc_tsd_no_cleanup) \ 231286866Sjasone a_name##tsd_initialized = true; \ 232234370Sjasone} 233234370Sjasone#elif (defined(JEMALLOC_TLS)) 234234370Sjasone#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 235234370Sjasone a_cleanup) \ 236234370Sjasone/* Initialization/cleanup. */ \ 237234370Sjasonea_attr bool \ 238286866Sjasonea_name##tsd_boot0(void) \ 239234370Sjasone{ \ 240234370Sjasone \ 241234370Sjasone if (a_cleanup != malloc_tsd_no_cleanup) { \ 242286866Sjasone if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ 243286866Sjasone 0) \ 244234370Sjasone return (true); \ 245234370Sjasone } \ 246286866Sjasone a_name##tsd_booted = true; \ 247234370Sjasone return (false); \ 248234370Sjasone} \ 249286866Sjasonea_attr void \ 250288090Sjasonea_name##tsd_boot1(void) \ 251286866Sjasone{ \ 252286866Sjasone \ 253286866Sjasone /* Do nothing. */ \ 254286866Sjasone} \ 255286866Sjasonea_attr bool \ 256286866Sjasonea_name##tsd_boot(void) \ 257286866Sjasone{ \ 258286866Sjasone \ 259286866Sjasone return (a_name##tsd_boot0()); \ 260286866Sjasone} \ 261299587Sjasonea_attr bool \ 262299587Sjasonea_name##tsd_booted_get(void) \ 263299587Sjasone{ \ 264299587Sjasone \ 265299587Sjasone return (a_name##tsd_booted); \ 266299587Sjasone} \ 267234370Sjasone/* Get/set. */ \ 268234370Sjasonea_attr a_type * \ 269286866Sjasonea_name##tsd_get(void) \ 270234370Sjasone{ \ 271234370Sjasone \ 272286866Sjasone assert(a_name##tsd_booted); \ 273286866Sjasone return (&a_name##tsd_tls); \ 274234370Sjasone} \ 275234370Sjasonea_attr void \ 276286866Sjasonea_name##tsd_set(a_type *val) \ 277234370Sjasone{ \ 278234370Sjasone \ 279286866Sjasone assert(a_name##tsd_booted); \ 280286866Sjasone a_name##tsd_tls = (*val); \ 281234370Sjasone if (a_cleanup != malloc_tsd_no_cleanup) { \ 282286866Sjasone if (pthread_setspecific(a_name##tsd_tsd, \ 283286866Sjasone (void *)(&a_name##tsd_tls))) { \ 284234370Sjasone malloc_write("<jemalloc>: Error" \ 285234370Sjasone " setting TSD for "#a_name"\n"); \ 286234370Sjasone if (opt_abort) \ 287234370Sjasone abort(); \ 288234370Sjasone } \ 289234370Sjasone } \ 290234370Sjasone} 291235238Sjasone#elif (defined(_WIN32)) 292235238Sjasone#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 293235238Sjasone a_cleanup) \ 294235238Sjasone/* Initialization/cleanup. */ \ 295235238Sjasonea_attr bool \ 296286866Sjasonea_name##tsd_cleanup_wrapper(void) \ 297235238Sjasone{ \ 298286866Sjasone DWORD error = GetLastError(); \ 299286866Sjasone a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 300286866Sjasone TlsGetValue(a_name##tsd_tsd); \ 301286866Sjasone SetLastError(error); \ 302235238Sjasone \ 303235238Sjasone if (wrapper == NULL) \ 304235238Sjasone return (false); \ 305235238Sjasone if (a_cleanup != malloc_tsd_no_cleanup && \ 306235238Sjasone wrapper->initialized) { \ 307235238Sjasone wrapper->initialized = false; \ 308286866Sjasone a_cleanup(&wrapper->val); \ 309235238Sjasone if (wrapper->initialized) { \ 310235238Sjasone /* Trigger another cleanup round. */ \ 311235238Sjasone return (true); \ 312235238Sjasone } \ 313235238Sjasone } \ 314235238Sjasone malloc_tsd_dalloc(wrapper); \ 315235238Sjasone return (false); \ 316235238Sjasone} \ 317286866Sjasonea_attr void \ 318286866Sjasonea_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 319235238Sjasone{ \ 320235238Sjasone \ 321286866Sjasone if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ 322286866Sjasone malloc_write("<jemalloc>: Error setting" \ 323286866Sjasone " TSD for "#a_name"\n"); \ 324286866Sjasone abort(); \ 325235238Sjasone } \ 326235238Sjasone} \ 327286866Sjasonea_attr a_name##tsd_wrapper_t * \ 328286866Sjasonea_name##tsd_wrapper_get(void) \ 329235238Sjasone{ \ 330286866Sjasone DWORD error = GetLastError(); \ 331286866Sjasone a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 332286866Sjasone TlsGetValue(a_name##tsd_tsd); \ 333286866Sjasone SetLastError(error); \ 334235238Sjasone \ 335286866Sjasone if (unlikely(wrapper == NULL)) { \ 336286866Sjasone wrapper = (a_name##tsd_wrapper_t *) \ 337286866Sjasone malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 338235238Sjasone if (wrapper == NULL) { \ 339235238Sjasone malloc_write("<jemalloc>: Error allocating" \ 340235238Sjasone " TSD for "#a_name"\n"); \ 341235238Sjasone abort(); \ 342235238Sjasone } else { \ 343235238Sjasone wrapper->initialized = false; \ 344286866Sjasone wrapper->val = a_initializer; \ 345235238Sjasone } \ 346286866Sjasone a_name##tsd_wrapper_set(wrapper); \ 347235238Sjasone } \ 348235238Sjasone return (wrapper); \ 349235238Sjasone} \ 350286866Sjasonea_attr bool \ 351286866Sjasonea_name##tsd_boot0(void) \ 352286866Sjasone{ \ 353286866Sjasone \ 354286866Sjasone a_name##tsd_tsd = TlsAlloc(); \ 355286866Sjasone if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ 356286866Sjasone return (true); \ 357286866Sjasone if (a_cleanup != malloc_tsd_no_cleanup) { \ 358286866Sjasone malloc_tsd_cleanup_register( \ 359286866Sjasone &a_name##tsd_cleanup_wrapper); \ 360286866Sjasone } \ 361286866Sjasone a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 362286866Sjasone a_name##tsd_booted = true; \ 363286866Sjasone return (false); \ 364286866Sjasone} \ 365286866Sjasonea_attr void \ 366288090Sjasonea_name##tsd_boot1(void) \ 367286866Sjasone{ \ 368286866Sjasone a_name##tsd_wrapper_t *wrapper; \ 369286866Sjasone wrapper = (a_name##tsd_wrapper_t *) \ 370286866Sjasone malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 371286866Sjasone if (wrapper == NULL) { \ 372286866Sjasone malloc_write("<jemalloc>: Error allocating" \ 373286866Sjasone " TSD for "#a_name"\n"); \ 374286866Sjasone abort(); \ 375286866Sjasone } \ 376286866Sjasone memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 377286866Sjasone sizeof(a_name##tsd_wrapper_t)); \ 378286866Sjasone a_name##tsd_wrapper_set(wrapper); \ 379286866Sjasone} \ 380286866Sjasonea_attr bool \ 381286866Sjasonea_name##tsd_boot(void) \ 382286866Sjasone{ \ 383286866Sjasone \ 384286866Sjasone if (a_name##tsd_boot0()) \ 385286866Sjasone return (true); \ 386286866Sjasone a_name##tsd_boot1(); \ 387286866Sjasone return (false); \ 388286866Sjasone} \ 389299587Sjasonea_attr bool \ 390299587Sjasonea_name##tsd_booted_get(void) \ 391299587Sjasone{ \ 392299587Sjasone \ 393299587Sjasone return (a_name##tsd_booted); \ 394299587Sjasone} \ 395286866Sjasone/* Get/set. */ \ 396235238Sjasonea_attr a_type * \ 397286866Sjasonea_name##tsd_get(void) \ 398235238Sjasone{ \ 399286866Sjasone a_name##tsd_wrapper_t *wrapper; \ 400235238Sjasone \ 401286866Sjasone assert(a_name##tsd_booted); \ 402286866Sjasone wrapper = a_name##tsd_wrapper_get(); \ 403235238Sjasone return (&wrapper->val); \ 404235238Sjasone} \ 405235238Sjasonea_attr void \ 406286866Sjasonea_name##tsd_set(a_type *val) \ 407235238Sjasone{ \ 408286866Sjasone a_name##tsd_wrapper_t *wrapper; \ 409235238Sjasone \ 410286866Sjasone assert(a_name##tsd_booted); \ 411286866Sjasone wrapper = a_name##tsd_wrapper_get(); \ 412235238Sjasone wrapper->val = *(val); \ 413235238Sjasone if (a_cleanup != malloc_tsd_no_cleanup) \ 414235238Sjasone wrapper->initialized = true; \ 415235238Sjasone} 416234370Sjasone#else 417234370Sjasone#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 418234370Sjasone a_cleanup) \ 419234370Sjasone/* Initialization/cleanup. */ \ 420234370Sjasonea_attr void \ 421286866Sjasonea_name##tsd_cleanup_wrapper(void *arg) \ 422234370Sjasone{ \ 423286866Sjasone a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ 424234370Sjasone \ 425234370Sjasone if (a_cleanup != malloc_tsd_no_cleanup && \ 426234370Sjasone wrapper->initialized) { \ 427234370Sjasone wrapper->initialized = false; \ 428234370Sjasone a_cleanup(&wrapper->val); \ 429234370Sjasone if (wrapper->initialized) { \ 430234370Sjasone /* Trigger another cleanup round. */ \ 431286866Sjasone if (pthread_setspecific(a_name##tsd_tsd, \ 432234370Sjasone (void *)wrapper)) { \ 433234370Sjasone malloc_write("<jemalloc>: Error" \ 434234370Sjasone " setting TSD for "#a_name"\n"); \ 435234370Sjasone if (opt_abort) \ 436234370Sjasone abort(); \ 437234370Sjasone } \ 438234370Sjasone return; \ 439234370Sjasone } \ 440234370Sjasone } \ 441234543Sjasone malloc_tsd_dalloc(wrapper); \ 442234370Sjasone} \ 443286866Sjasonea_attr void \ 444286866Sjasonea_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 445234370Sjasone{ \ 446234370Sjasone \ 447286866Sjasone if (pthread_setspecific(a_name##tsd_tsd, \ 448286866Sjasone (void *)wrapper)) { \ 449286866Sjasone malloc_write("<jemalloc>: Error setting" \ 450286866Sjasone " TSD for "#a_name"\n"); \ 451286866Sjasone abort(); \ 452286866Sjasone } \ 453234370Sjasone} \ 454286866Sjasonea_attr a_name##tsd_wrapper_t * \ 455286866Sjasonea_name##tsd_wrapper_get(void) \ 456234370Sjasone{ \ 457286866Sjasone a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 458286866Sjasone pthread_getspecific(a_name##tsd_tsd); \ 459234370Sjasone \ 460286866Sjasone if (unlikely(wrapper == NULL)) { \ 461261071Sjasone tsd_init_block_t block; \ 462261071Sjasone wrapper = tsd_init_check_recursion( \ 463286866Sjasone &a_name##tsd_init_head, &block); \ 464261071Sjasone if (wrapper) \ 465261071Sjasone return (wrapper); \ 466286866Sjasone wrapper = (a_name##tsd_wrapper_t *) \ 467286866Sjasone malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 468261071Sjasone block.data = wrapper; \ 469234370Sjasone if (wrapper == NULL) { \ 470234370Sjasone malloc_write("<jemalloc>: Error allocating" \ 471234370Sjasone " TSD for "#a_name"\n"); \ 472234543Sjasone abort(); \ 473234370Sjasone } else { \ 474234543Sjasone wrapper->initialized = false; \ 475286866Sjasone wrapper->val = a_initializer; \ 476234370Sjasone } \ 477286866Sjasone a_name##tsd_wrapper_set(wrapper); \ 478286866Sjasone tsd_init_finish(&a_name##tsd_init_head, &block); \ 479234370Sjasone } \ 480234370Sjasone return (wrapper); \ 481234370Sjasone} \ 482286866Sjasonea_attr bool \ 483286866Sjasonea_name##tsd_boot0(void) \ 484286866Sjasone{ \ 485286866Sjasone \ 486286866Sjasone if (pthread_key_create(&a_name##tsd_tsd, \ 487286866Sjasone a_name##tsd_cleanup_wrapper) != 0) \ 488286866Sjasone return (true); \ 489286866Sjasone a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 490286866Sjasone a_name##tsd_booted = true; \ 491286866Sjasone return (false); \ 492286866Sjasone} \ 493286866Sjasonea_attr void \ 494288090Sjasonea_name##tsd_boot1(void) \ 495286866Sjasone{ \ 496286866Sjasone a_name##tsd_wrapper_t *wrapper; \ 497286866Sjasone wrapper = (a_name##tsd_wrapper_t *) \ 498286866Sjasone malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 499286866Sjasone if (wrapper == NULL) { \ 500286866Sjasone malloc_write("<jemalloc>: Error allocating" \ 501286866Sjasone " TSD for "#a_name"\n"); \ 502286866Sjasone abort(); \ 503286866Sjasone } \ 504286866Sjasone memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 505286866Sjasone sizeof(a_name##tsd_wrapper_t)); \ 506286866Sjasone a_name##tsd_wrapper_set(wrapper); \ 507286866Sjasone} \ 508286866Sjasonea_attr bool \ 509286866Sjasonea_name##tsd_boot(void) \ 510286866Sjasone{ \ 511286866Sjasone \ 512286866Sjasone if (a_name##tsd_boot0()) \ 513286866Sjasone return (true); \ 514286866Sjasone a_name##tsd_boot1(); \ 515286866Sjasone return (false); \ 516286866Sjasone} \ 517299587Sjasonea_attr bool \ 518299587Sjasonea_name##tsd_booted_get(void) \ 519299587Sjasone{ \ 520299587Sjasone \ 521299587Sjasone return (a_name##tsd_booted); \ 522299587Sjasone} \ 523286866Sjasone/* Get/set. */ \ 524234370Sjasonea_attr a_type * \ 525286866Sjasonea_name##tsd_get(void) \ 526234370Sjasone{ \ 527286866Sjasone a_name##tsd_wrapper_t *wrapper; \ 528234370Sjasone \ 529286866Sjasone assert(a_name##tsd_booted); \ 530286866Sjasone wrapper = a_name##tsd_wrapper_get(); \ 531234370Sjasone return (&wrapper->val); \ 532234370Sjasone} \ 533234370Sjasonea_attr void \ 534286866Sjasonea_name##tsd_set(a_type *val) \ 535234370Sjasone{ \ 536286866Sjasone a_name##tsd_wrapper_t *wrapper; \ 537234370Sjasone \ 538286866Sjasone assert(a_name##tsd_booted); \ 539286866Sjasone wrapper = a_name##tsd_wrapper_get(); \ 540234370Sjasone wrapper->val = *(val); \ 541234370Sjasone if (a_cleanup != malloc_tsd_no_cleanup) \ 542234370Sjasone wrapper->initialized = true; \ 543234370Sjasone} 544234370Sjasone#endif 545234370Sjasone 546234370Sjasone#endif /* JEMALLOC_H_TYPES */ 547234370Sjasone/******************************************************************************/ 548234370Sjasone#ifdef JEMALLOC_H_STRUCTS 549234370Sjasone 550261071Sjasone#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 551261071Sjasone !defined(_WIN32)) 552261071Sjasonestruct tsd_init_block_s { 553261071Sjasone ql_elm(tsd_init_block_t) link; 554261071Sjasone pthread_t thread; 555261071Sjasone void *data; 556261071Sjasone}; 557261071Sjasonestruct tsd_init_head_s { 558261071Sjasone ql_head(tsd_init_block_t) blocks; 559261071Sjasone malloc_mutex_t lock; 560261071Sjasone}; 561261071Sjasone#endif 562261071Sjasone 563286866Sjasone#define MALLOC_TSD \ 564286866Sjasone/* O(name, type) */ \ 565286866Sjasone O(tcache, tcache_t *) \ 566286866Sjasone O(thread_allocated, uint64_t) \ 567286866Sjasone O(thread_deallocated, uint64_t) \ 568286866Sjasone O(prof_tdata, prof_tdata_t *) \ 569299587Sjasone O(iarena, arena_t *) \ 570286866Sjasone O(arena, arena_t *) \ 571296221Sjasone O(arenas_tdata, arena_tdata_t *) \ 572296221Sjasone O(narenas_tdata, unsigned) \ 573296221Sjasone O(arenas_tdata_bypass, bool) \ 574286866Sjasone O(tcache_enabled, tcache_enabled_t) \ 575286866Sjasone O(quarantine, quarantine_t *) \ 576299587Sjasone O(witnesses, witness_list_t) \ 577299587Sjasone O(witness_fork, bool) \ 578286866Sjasone 579286866Sjasone#define TSD_INITIALIZER { \ 580286866Sjasone tsd_state_uninitialized, \ 581286866Sjasone NULL, \ 582286866Sjasone 0, \ 583286866Sjasone 0, \ 584286866Sjasone NULL, \ 585286866Sjasone NULL, \ 586286866Sjasone NULL, \ 587299587Sjasone NULL, \ 588286866Sjasone 0, \ 589286866Sjasone false, \ 590286866Sjasone tcache_enabled_default, \ 591299587Sjasone NULL, \ 592299587Sjasone ql_head_initializer(witnesses), \ 593299587Sjasone false \ 594286866Sjasone} 595286866Sjasone 596286866Sjasonestruct tsd_s { 597286866Sjasone tsd_state_t state; 598286866Sjasone#define O(n, t) \ 599286866Sjasone t n; 600286866SjasoneMALLOC_TSD 601286866Sjasone#undef O 602286866Sjasone}; 603286866Sjasone 604299587Sjasone/* 605299587Sjasone * Wrapper around tsd_t that makes it possible to avoid implicit conversion 606299587Sjasone * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be 607299587Sjasone * explicitly converted to tsd_t, which is non-nullable. 608299587Sjasone */ 609299587Sjasonestruct tsdn_s { 610299587Sjasone tsd_t tsd; 611299587Sjasone}; 612299587Sjasone 613286866Sjasonestatic const tsd_t tsd_initializer = TSD_INITIALIZER; 614286866Sjasone 615286866Sjasonemalloc_tsd_types(, tsd_t) 616286866Sjasone 617234370Sjasone#endif /* JEMALLOC_H_STRUCTS */ 618234370Sjasone/******************************************************************************/ 619234370Sjasone#ifdef JEMALLOC_H_EXTERNS 620234370Sjasone 621234370Sjasonevoid *malloc_tsd_malloc(size_t size); 622234370Sjasonevoid malloc_tsd_dalloc(void *wrapper); 623286866Sjasonevoid malloc_tsd_no_cleanup(void *arg); 624234543Sjasonevoid malloc_tsd_cleanup_register(bool (*f)(void)); 625299587Sjasonetsd_t *malloc_tsd_boot0(void); 626286866Sjasonevoid malloc_tsd_boot1(void); 627261071Sjasone#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 628261071Sjasone !defined(_WIN32)) 629261071Sjasonevoid *tsd_init_check_recursion(tsd_init_head_t *head, 630261071Sjasone tsd_init_block_t *block); 631261071Sjasonevoid tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); 632261071Sjasone#endif 633286866Sjasonevoid tsd_cleanup(void *arg); 634234370Sjasone 635234370Sjasone#endif /* JEMALLOC_H_EXTERNS */ 636234370Sjasone/******************************************************************************/ 637234370Sjasone#ifdef JEMALLOC_H_INLINES 638234370Sjasone 639286866Sjasone#ifndef JEMALLOC_ENABLE_INLINE 640286866Sjasonemalloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) 641286866Sjasone 642286866Sjasonetsd_t *tsd_fetch(void); 643299587Sjasonetsdn_t *tsd_tsdn(tsd_t *tsd); 644286866Sjasonebool tsd_nominal(tsd_t *tsd); 645286866Sjasone#define O(n, t) \ 646286866Sjasonet *tsd_##n##p_get(tsd_t *tsd); \ 647286866Sjasonet tsd_##n##_get(tsd_t *tsd); \ 648286866Sjasonevoid tsd_##n##_set(tsd_t *tsd, t n); 649286866SjasoneMALLOC_TSD 650286866Sjasone#undef O 651299587Sjasonetsdn_t *tsdn_fetch(void); 652299587Sjasonebool tsdn_null(const tsdn_t *tsdn); 653299587Sjasonetsd_t *tsdn_tsd(tsdn_t *tsdn); 654286866Sjasone#endif 655286866Sjasone 656286866Sjasone#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) 657286866Sjasonemalloc_tsd_externs(, tsd_t) 658286866Sjasonemalloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) 659286866Sjasone 660286866SjasoneJEMALLOC_ALWAYS_INLINE tsd_t * 661286866Sjasonetsd_fetch(void) 662286866Sjasone{ 663286866Sjasone tsd_t *tsd = tsd_get(); 664286866Sjasone 665286866Sjasone if (unlikely(tsd->state != tsd_state_nominal)) { 666286866Sjasone if (tsd->state == tsd_state_uninitialized) { 667286866Sjasone tsd->state = tsd_state_nominal; 668286866Sjasone /* Trigger cleanup handler registration. */ 669286866Sjasone tsd_set(tsd); 670286866Sjasone } else if (tsd->state == tsd_state_purgatory) { 671286866Sjasone tsd->state = tsd_state_reincarnated; 672286866Sjasone tsd_set(tsd); 673286866Sjasone } else 674286866Sjasone assert(tsd->state == tsd_state_reincarnated); 675286866Sjasone } 676286866Sjasone 677286866Sjasone return (tsd); 678286866Sjasone} 679286866Sjasone 680299587SjasoneJEMALLOC_ALWAYS_INLINE tsdn_t * 681299587Sjasonetsd_tsdn(tsd_t *tsd) 682299587Sjasone{ 683299587Sjasone 684299587Sjasone return ((tsdn_t *)tsd); 685299587Sjasone} 686299587Sjasone 687286866SjasoneJEMALLOC_INLINE bool 688286866Sjasonetsd_nominal(tsd_t *tsd) 689286866Sjasone{ 690286866Sjasone 691286866Sjasone return (tsd->state == tsd_state_nominal); 692286866Sjasone} 693286866Sjasone 694286866Sjasone#define O(n, t) \ 695286866SjasoneJEMALLOC_ALWAYS_INLINE t * \ 696286866Sjasonetsd_##n##p_get(tsd_t *tsd) \ 697286866Sjasone{ \ 698286866Sjasone \ 699286866Sjasone return (&tsd->n); \ 700286866Sjasone} \ 701286866Sjasone \ 702286866SjasoneJEMALLOC_ALWAYS_INLINE t \ 703286866Sjasonetsd_##n##_get(tsd_t *tsd) \ 704286866Sjasone{ \ 705286866Sjasone \ 706286866Sjasone return (*tsd_##n##p_get(tsd)); \ 707286866Sjasone} \ 708286866Sjasone \ 709286866SjasoneJEMALLOC_ALWAYS_INLINE void \ 710286866Sjasonetsd_##n##_set(tsd_t *tsd, t n) \ 711286866Sjasone{ \ 712286866Sjasone \ 713286866Sjasone assert(tsd->state == tsd_state_nominal); \ 714286866Sjasone tsd->n = n; \ 715286866Sjasone} 716286866SjasoneMALLOC_TSD 717286866Sjasone#undef O 718299587Sjasone 719299587SjasoneJEMALLOC_ALWAYS_INLINE tsdn_t * 720299587Sjasonetsdn_fetch(void) 721299587Sjasone{ 722299587Sjasone 723299587Sjasone if (!tsd_booted_get()) 724299587Sjasone return (NULL); 725299587Sjasone 726299587Sjasone return (tsd_tsdn(tsd_fetch())); 727299587Sjasone} 728299587Sjasone 729299587SjasoneJEMALLOC_ALWAYS_INLINE bool 730299587Sjasonetsdn_null(const tsdn_t *tsdn) 731299587Sjasone{ 732299587Sjasone 733299587Sjasone return (tsdn == NULL); 734299587Sjasone} 735299587Sjasone 736299587SjasoneJEMALLOC_ALWAYS_INLINE tsd_t * 737299587Sjasonetsdn_tsd(tsdn_t *tsdn) 738299587Sjasone{ 739299587Sjasone 740299587Sjasone assert(!tsdn_null(tsdn)); 741299587Sjasone 742299587Sjasone return (&tsdn->tsd); 743299587Sjasone} 744286866Sjasone#endif 745286866Sjasone 746234370Sjasone#endif /* JEMALLOC_H_INLINES */ 747234370Sjasone/******************************************************************************/ 748