jemalloc.c revision 235322
1234370Sjasone#define JEMALLOC_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone 4234370Sjasone/******************************************************************************/ 5234370Sjasone/* Data. */ 6234370Sjasone 7234370Sjasonemalloc_tsd_data(, arenas, arena_t *, NULL) 8234370Sjasonemalloc_tsd_data(, thread_allocated, thread_allocated_t, 9234370Sjasone THREAD_ALLOCATED_INITIALIZER) 10234370Sjasone 11234658Sdim/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 12234658Sdimconst char *__malloc_options_1_0 = NULL; 13234370Sjasone__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 14234370Sjasone 15234370Sjasone/* Runtime configuration options. */ 16235238Sjasoneconst char *je_malloc_conf; 17234370Sjasone#ifdef JEMALLOC_DEBUG 18234370Sjasonebool opt_abort = true; 19234370Sjasone# ifdef JEMALLOC_FILL 20234370Sjasonebool opt_junk = true; 21234370Sjasone# else 22234370Sjasonebool opt_junk = false; 23234370Sjasone# endif 24234370Sjasone#else 25234370Sjasonebool opt_abort = false; 26234370Sjasonebool opt_junk = false; 27234370Sjasone#endif 28234370Sjasonesize_t opt_quarantine = ZU(0); 29234370Sjasonebool opt_redzone = false; 30234370Sjasonebool opt_utrace = false; 31234370Sjasonebool opt_valgrind = false; 32234370Sjasonebool opt_xmalloc = false; 33234370Sjasonebool opt_zero = false; 34234370Sjasonesize_t opt_narenas = 0; 35234370Sjasone 36234370Sjasoneunsigned ncpus; 37234370Sjasone 38234370Sjasonemalloc_mutex_t arenas_lock; 39234370Sjasonearena_t **arenas; 40234370Sjasoneunsigned narenas; 41234370Sjasone 42234370Sjasone/* Set to true once the allocator has been initialized. */ 43234370Sjasonestatic bool malloc_initialized = false; 44234370Sjasone 45234370Sjasone#ifdef JEMALLOC_THREADED_INIT 46234370Sjasone/* Used to let the initializing thread recursively allocate. */ 47234370Sjasone# define NO_INITIALIZER ((unsigned long)0) 48234370Sjasone# define INITIALIZER pthread_self() 49234370Sjasone# define IS_INITIALIZER (malloc_initializer == pthread_self()) 50234370Sjasonestatic pthread_t malloc_initializer = NO_INITIALIZER; 51234370Sjasone#else 52234370Sjasone# define NO_INITIALIZER false 53234370Sjasone# define INITIALIZER true 54234370Sjasone# define IS_INITIALIZER malloc_initializer 55234370Sjasonestatic bool malloc_initializer = NO_INITIALIZER; 56234370Sjasone#endif 57234370Sjasone 58234370Sjasone/* Used to avoid initialization races. */ 59235238Sjasone#ifdef _WIN32 60235238Sjasonestatic malloc_mutex_t init_lock; 61235238Sjasone 62235238SjasoneJEMALLOC_ATTR(constructor) 63235238Sjasonestatic void WINAPI 64235238Sjasone_init_init_lock(void) 65235238Sjasone{ 66235238Sjasone 67235238Sjasone malloc_mutex_init(&init_lock); 68235238Sjasone} 69235238Sjasone 70235238Sjasone#ifdef _MSC_VER 71235238Sjasone# pragma section(".CRT$XCU", read) 72235238SjasoneJEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 73235238Sjasonestatic const void (WINAPI *init_init_lock)(void) = _init_init_lock; 74235238Sjasone#endif 75235238Sjasone 76235238Sjasone#else 77234370Sjasonestatic malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 78235238Sjasone#endif 79234370Sjasone 80234370Sjasonetypedef struct { 81234370Sjasone void *p; /* Input pointer (as in realloc(p, s)). */ 82234370Sjasone size_t s; /* Request size. */ 83234370Sjasone void *r; /* Result pointer. */ 84234370Sjasone} malloc_utrace_t; 85234370Sjasone 86234370Sjasone#ifdef JEMALLOC_UTRACE 87234370Sjasone# define UTRACE(a, b, c) do { \ 88234370Sjasone if (opt_utrace) { \ 89234370Sjasone malloc_utrace_t ut; \ 90234370Sjasone ut.p = (a); \ 91234370Sjasone ut.s = (b); \ 92234370Sjasone ut.r = (c); \ 93234370Sjasone utrace(&ut, sizeof(ut)); \ 94234370Sjasone } \ 95234370Sjasone} while (0) 96234370Sjasone#else 97234370Sjasone# define UTRACE(a, b, c) 98234370Sjasone#endif 99234370Sjasone 100234370Sjasone/******************************************************************************/ 101234370Sjasone/* Function prototypes for non-inline static functions. */ 102234370Sjasone 103234370Sjasonestatic void stats_print_atexit(void); 104234370Sjasonestatic unsigned malloc_ncpus(void); 105234370Sjasonestatic bool malloc_conf_next(char const **opts_p, char const **k_p, 106234370Sjasone size_t *klen_p, char const **v_p, size_t *vlen_p); 107234370Sjasonestatic void malloc_conf_error(const char *msg, const char *k, size_t klen, 108234370Sjasone const char *v, size_t vlen); 109234370Sjasonestatic void malloc_conf_init(void); 110234370Sjasonestatic bool malloc_init_hard(void); 111234370Sjasonestatic int imemalign(void **memptr, size_t alignment, size_t size, 112234370Sjasone size_t min_alignment); 113234370Sjasone 114234370Sjasone/******************************************************************************/ 115234370Sjasone/* 116234370Sjasone * Begin miscellaneous support functions. 117234370Sjasone */ 118234370Sjasone 119234370Sjasone/* Create a new arena and insert it into the arenas array at index ind. */ 120234370Sjasonearena_t * 121234370Sjasonearenas_extend(unsigned ind) 122234370Sjasone{ 123234370Sjasone arena_t *ret; 124234370Sjasone 125234370Sjasone ret = (arena_t *)base_alloc(sizeof(arena_t)); 126234370Sjasone if (ret != NULL && arena_new(ret, ind) == false) { 127234370Sjasone arenas[ind] = ret; 128234370Sjasone return (ret); 129234370Sjasone } 130234370Sjasone /* Only reached if there is an OOM error. */ 131234370Sjasone 132234370Sjasone /* 133234370Sjasone * OOM here is quite inconvenient to propagate, since dealing with it 134234370Sjasone * would require a check for failure in the fast path. Instead, punt 135234370Sjasone * by using arenas[0]. In practice, this is an extremely unlikely 136234370Sjasone * failure. 137234370Sjasone */ 138234370Sjasone malloc_write("<jemalloc>: Error initializing arena\n"); 139234370Sjasone if (opt_abort) 140234370Sjasone abort(); 141234370Sjasone 142234370Sjasone return (arenas[0]); 143234370Sjasone} 144234370Sjasone 145234370Sjasone/* Slow path, called only by choose_arena(). */ 146234370Sjasonearena_t * 147234370Sjasonechoose_arena_hard(void) 148234370Sjasone{ 149234370Sjasone arena_t *ret; 150234370Sjasone 151234370Sjasone if (narenas > 1) { 152234370Sjasone unsigned i, choose, first_null; 153234370Sjasone 154234370Sjasone choose = 0; 155234370Sjasone first_null = narenas; 156234370Sjasone malloc_mutex_lock(&arenas_lock); 157234370Sjasone assert(arenas[0] != NULL); 158234370Sjasone for (i = 1; i < narenas; i++) { 159234370Sjasone if (arenas[i] != NULL) { 160234370Sjasone /* 161234370Sjasone * Choose the first arena that has the lowest 162234370Sjasone * number of threads assigned to it. 163234370Sjasone */ 164234370Sjasone if (arenas[i]->nthreads < 165234370Sjasone arenas[choose]->nthreads) 166234370Sjasone choose = i; 167234370Sjasone } else if (first_null == narenas) { 168234370Sjasone /* 169234370Sjasone * Record the index of the first uninitialized 170234370Sjasone * arena, in case all extant arenas are in use. 171234370Sjasone * 172234370Sjasone * NB: It is possible for there to be 173234370Sjasone * discontinuities in terms of initialized 174234370Sjasone * versus uninitialized arenas, due to the 175234370Sjasone * "thread.arena" mallctl. 176234370Sjasone */ 177234370Sjasone first_null = i; 178234370Sjasone } 179234370Sjasone } 180234370Sjasone 181234370Sjasone if (arenas[choose]->nthreads == 0 || first_null == narenas) { 182234370Sjasone /* 183234370Sjasone * Use an unloaded arena, or the least loaded arena if 184234370Sjasone * all arenas are already initialized. 185234370Sjasone */ 186234370Sjasone ret = arenas[choose]; 187234370Sjasone } else { 188234370Sjasone /* Initialize a new arena. */ 189234370Sjasone ret = arenas_extend(first_null); 190234370Sjasone } 191234370Sjasone ret->nthreads++; 192234370Sjasone malloc_mutex_unlock(&arenas_lock); 193234370Sjasone } else { 194234370Sjasone ret = arenas[0]; 195234370Sjasone malloc_mutex_lock(&arenas_lock); 196234370Sjasone ret->nthreads++; 197234370Sjasone malloc_mutex_unlock(&arenas_lock); 198234370Sjasone } 199234370Sjasone 200234370Sjasone arenas_tsd_set(&ret); 201234370Sjasone 202234370Sjasone return (ret); 203234370Sjasone} 204234370Sjasone 205234370Sjasonestatic void 206234370Sjasonestats_print_atexit(void) 207234370Sjasone{ 208234370Sjasone 209234370Sjasone if (config_tcache && config_stats) { 210234370Sjasone unsigned i; 211234370Sjasone 212234370Sjasone /* 213234370Sjasone * Merge stats from extant threads. This is racy, since 214234370Sjasone * individual threads do not lock when recording tcache stats 215234370Sjasone * events. As a consequence, the final stats may be slightly 216234370Sjasone * out of date by the time they are reported, if other threads 217234370Sjasone * continue to allocate. 218234370Sjasone */ 219234370Sjasone for (i = 0; i < narenas; i++) { 220234370Sjasone arena_t *arena = arenas[i]; 221234370Sjasone if (arena != NULL) { 222234370Sjasone tcache_t *tcache; 223234370Sjasone 224234370Sjasone /* 225234370Sjasone * tcache_stats_merge() locks bins, so if any 226234370Sjasone * code is introduced that acquires both arena 227234370Sjasone * and bin locks in the opposite order, 228234370Sjasone * deadlocks may result. 229234370Sjasone */ 230234370Sjasone malloc_mutex_lock(&arena->lock); 231234370Sjasone ql_foreach(tcache, &arena->tcache_ql, link) { 232234370Sjasone tcache_stats_merge(tcache, arena); 233234370Sjasone } 234234370Sjasone malloc_mutex_unlock(&arena->lock); 235234370Sjasone } 236234370Sjasone } 237234370Sjasone } 238234370Sjasone je_malloc_stats_print(NULL, NULL, NULL); 239234370Sjasone} 240234370Sjasone 241234370Sjasone/* 242234370Sjasone * End miscellaneous support functions. 243234370Sjasone */ 244234370Sjasone/******************************************************************************/ 245234370Sjasone/* 246234370Sjasone * Begin initialization functions. 247234370Sjasone */ 248234370Sjasone 249234370Sjasonestatic unsigned 250234370Sjasonemalloc_ncpus(void) 251234370Sjasone{ 252234370Sjasone unsigned ret; 253234370Sjasone long result; 254234370Sjasone 255235238Sjasone#ifdef _WIN32 256235238Sjasone SYSTEM_INFO si; 257235238Sjasone GetSystemInfo(&si); 258235238Sjasone result = si.dwNumberOfProcessors; 259235238Sjasone#else 260234370Sjasone result = sysconf(_SC_NPROCESSORS_ONLN); 261234370Sjasone if (result == -1) { 262234370Sjasone /* Error. */ 263234370Sjasone ret = 1; 264234370Sjasone } 265235238Sjasone#endif 266234370Sjasone ret = (unsigned)result; 267234370Sjasone 268234370Sjasone return (ret); 269234370Sjasone} 270234370Sjasone 271234370Sjasonevoid 272234370Sjasonearenas_cleanup(void *arg) 273234370Sjasone{ 274234370Sjasone arena_t *arena = *(arena_t **)arg; 275234370Sjasone 276234370Sjasone malloc_mutex_lock(&arenas_lock); 277234370Sjasone arena->nthreads--; 278234370Sjasone malloc_mutex_unlock(&arenas_lock); 279234370Sjasone} 280234370Sjasone 281234370Sjasonestatic inline bool 282234370Sjasonemalloc_init(void) 283234370Sjasone{ 284234370Sjasone 285234370Sjasone if (malloc_initialized == false) 286234370Sjasone return (malloc_init_hard()); 287234370Sjasone 288234370Sjasone return (false); 289234370Sjasone} 290234370Sjasone 291234370Sjasonestatic bool 292234370Sjasonemalloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 293234370Sjasone char const **v_p, size_t *vlen_p) 294234370Sjasone{ 295234370Sjasone bool accept; 296234370Sjasone const char *opts = *opts_p; 297234370Sjasone 298234370Sjasone *k_p = opts; 299234370Sjasone 300234370Sjasone for (accept = false; accept == false;) { 301234370Sjasone switch (*opts) { 302234370Sjasone case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 303234370Sjasone case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 304234370Sjasone case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 305234370Sjasone case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 306234370Sjasone case 'Y': case 'Z': 307234370Sjasone case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 308234370Sjasone case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 309234370Sjasone case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 310234370Sjasone case 's': case 't': case 'u': case 'v': case 'w': case 'x': 311234370Sjasone case 'y': case 'z': 312234370Sjasone case '0': case '1': case '2': case '3': case '4': case '5': 313234370Sjasone case '6': case '7': case '8': case '9': 314234370Sjasone case '_': 315234370Sjasone opts++; 316234370Sjasone break; 317234370Sjasone case ':': 318234370Sjasone opts++; 319234370Sjasone *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 320234370Sjasone *v_p = opts; 321234370Sjasone accept = true; 322234370Sjasone break; 323234370Sjasone case '\0': 324234370Sjasone if (opts != *opts_p) { 325234370Sjasone malloc_write("<jemalloc>: Conf string ends " 326234370Sjasone "with key\n"); 327234370Sjasone } 328234370Sjasone return (true); 329234370Sjasone default: 330234370Sjasone malloc_write("<jemalloc>: Malformed conf string\n"); 331234370Sjasone return (true); 332234370Sjasone } 333234370Sjasone } 334234370Sjasone 335234370Sjasone for (accept = false; accept == false;) { 336234370Sjasone switch (*opts) { 337234370Sjasone case ',': 338234370Sjasone opts++; 339234370Sjasone /* 340234370Sjasone * Look ahead one character here, because the next time 341234370Sjasone * this function is called, it will assume that end of 342234370Sjasone * input has been cleanly reached if no input remains, 343234370Sjasone * but we have optimistically already consumed the 344234370Sjasone * comma if one exists. 345234370Sjasone */ 346234370Sjasone if (*opts == '\0') { 347234370Sjasone malloc_write("<jemalloc>: Conf string ends " 348234370Sjasone "with comma\n"); 349234370Sjasone } 350234370Sjasone *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 351234370Sjasone accept = true; 352234370Sjasone break; 353234370Sjasone case '\0': 354234370Sjasone *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 355234370Sjasone accept = true; 356234370Sjasone break; 357234370Sjasone default: 358234370Sjasone opts++; 359234370Sjasone break; 360234370Sjasone } 361234370Sjasone } 362234370Sjasone 363234370Sjasone *opts_p = opts; 364234370Sjasone return (false); 365234370Sjasone} 366234370Sjasone 367234370Sjasonestatic void 368234370Sjasonemalloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 369234370Sjasone size_t vlen) 370234370Sjasone{ 371234370Sjasone 372234370Sjasone malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 373234370Sjasone (int)vlen, v); 374234370Sjasone} 375234370Sjasone 376234370Sjasonestatic void 377234370Sjasonemalloc_conf_init(void) 378234370Sjasone{ 379234370Sjasone unsigned i; 380234370Sjasone char buf[PATH_MAX + 1]; 381234370Sjasone const char *opts, *k, *v; 382234370Sjasone size_t klen, vlen; 383234370Sjasone 384234370Sjasone for (i = 0; i < 3; i++) { 385234370Sjasone /* Get runtime configuration. */ 386234370Sjasone switch (i) { 387234370Sjasone case 0: 388234370Sjasone if (je_malloc_conf != NULL) { 389234370Sjasone /* 390234370Sjasone * Use options that were compiled into the 391234370Sjasone * program. 392234370Sjasone */ 393234370Sjasone opts = je_malloc_conf; 394234370Sjasone } else { 395234370Sjasone /* No configuration specified. */ 396234370Sjasone buf[0] = '\0'; 397234370Sjasone opts = buf; 398234370Sjasone } 399234370Sjasone break; 400234370Sjasone case 1: { 401235238Sjasone#ifndef _WIN32 402234370Sjasone int linklen; 403234370Sjasone const char *linkname = 404235238Sjasone# ifdef JEMALLOC_PREFIX 405234370Sjasone "/etc/"JEMALLOC_PREFIX"malloc.conf" 406235238Sjasone# else 407234370Sjasone "/etc/malloc.conf" 408235238Sjasone# endif 409234370Sjasone ; 410234370Sjasone 411234370Sjasone if ((linklen = readlink(linkname, buf, 412234370Sjasone sizeof(buf) - 1)) != -1) { 413234370Sjasone /* 414234370Sjasone * Use the contents of the "/etc/malloc.conf" 415234370Sjasone * symbolic link's name. 416234370Sjasone */ 417234370Sjasone buf[linklen] = '\0'; 418234370Sjasone opts = buf; 419235238Sjasone } else 420235238Sjasone#endif 421235238Sjasone { 422234370Sjasone /* No configuration specified. */ 423234370Sjasone buf[0] = '\0'; 424234370Sjasone opts = buf; 425234370Sjasone } 426234370Sjasone break; 427234370Sjasone } case 2: { 428234370Sjasone const char *envname = 429234370Sjasone#ifdef JEMALLOC_PREFIX 430234370Sjasone JEMALLOC_CPREFIX"MALLOC_CONF" 431234370Sjasone#else 432234370Sjasone "MALLOC_CONF" 433234370Sjasone#endif 434234370Sjasone ; 435234370Sjasone 436234370Sjasone if (issetugid() == 0 && (opts = getenv(envname)) != 437234370Sjasone NULL) { 438234370Sjasone /* 439234370Sjasone * Do nothing; opts is already initialized to 440234370Sjasone * the value of the MALLOC_CONF environment 441234370Sjasone * variable. 442234370Sjasone */ 443234370Sjasone } else { 444234370Sjasone /* No configuration specified. */ 445234370Sjasone buf[0] = '\0'; 446234370Sjasone opts = buf; 447234370Sjasone } 448234370Sjasone break; 449234370Sjasone } default: 450234370Sjasone /* NOTREACHED */ 451234370Sjasone assert(false); 452234370Sjasone buf[0] = '\0'; 453234370Sjasone opts = buf; 454234370Sjasone } 455234370Sjasone 456234370Sjasone while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 457234370Sjasone &vlen) == false) { 458234370Sjasone#define CONF_HANDLE_BOOL_HIT(o, n, hit) \ 459234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 460234370Sjasone klen) == 0) { \ 461234370Sjasone if (strncmp("true", v, vlen) == 0 && \ 462234370Sjasone vlen == sizeof("true")-1) \ 463234370Sjasone o = true; \ 464234370Sjasone else if (strncmp("false", v, vlen) == \ 465234370Sjasone 0 && vlen == sizeof("false")-1) \ 466234370Sjasone o = false; \ 467234370Sjasone else { \ 468234370Sjasone malloc_conf_error( \ 469234370Sjasone "Invalid conf value", \ 470234370Sjasone k, klen, v, vlen); \ 471234370Sjasone } \ 472234370Sjasone hit = true; \ 473234370Sjasone } else \ 474234370Sjasone hit = false; 475234370Sjasone#define CONF_HANDLE_BOOL(o, n) { \ 476234370Sjasone bool hit; \ 477234370Sjasone CONF_HANDLE_BOOL_HIT(o, n, hit); \ 478234370Sjasone if (hit) \ 479234370Sjasone continue; \ 480234370Sjasone} 481234370Sjasone#define CONF_HANDLE_SIZE_T(o, n, min, max) \ 482234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 483234370Sjasone klen) == 0) { \ 484234370Sjasone uintmax_t um; \ 485234370Sjasone char *end; \ 486234370Sjasone \ 487235238Sjasone set_errno(0); \ 488234370Sjasone um = malloc_strtoumax(v, &end, 0); \ 489235238Sjasone if (get_errno() != 0 || (uintptr_t)end -\ 490234370Sjasone (uintptr_t)v != vlen) { \ 491234370Sjasone malloc_conf_error( \ 492234370Sjasone "Invalid conf value", \ 493234370Sjasone k, klen, v, vlen); \ 494234370Sjasone } else if (um < min || um > max) { \ 495234370Sjasone malloc_conf_error( \ 496234370Sjasone "Out-of-range conf value", \ 497234370Sjasone k, klen, v, vlen); \ 498234370Sjasone } else \ 499234370Sjasone o = um; \ 500234370Sjasone continue; \ 501234370Sjasone } 502234370Sjasone#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 503234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 504234370Sjasone klen) == 0) { \ 505234370Sjasone long l; \ 506234370Sjasone char *end; \ 507234370Sjasone \ 508235238Sjasone set_errno(0); \ 509234370Sjasone l = strtol(v, &end, 0); \ 510235238Sjasone if (get_errno() != 0 || (uintptr_t)end -\ 511234370Sjasone (uintptr_t)v != vlen) { \ 512234370Sjasone malloc_conf_error( \ 513234370Sjasone "Invalid conf value", \ 514234370Sjasone k, klen, v, vlen); \ 515234370Sjasone } else if (l < (ssize_t)min || l > \ 516234370Sjasone (ssize_t)max) { \ 517234370Sjasone malloc_conf_error( \ 518234370Sjasone "Out-of-range conf value", \ 519234370Sjasone k, klen, v, vlen); \ 520234370Sjasone } else \ 521234370Sjasone o = l; \ 522234370Sjasone continue; \ 523234370Sjasone } 524234370Sjasone#define CONF_HANDLE_CHAR_P(o, n, d) \ 525234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 526234370Sjasone klen) == 0) { \ 527234370Sjasone size_t cpylen = (vlen <= \ 528234370Sjasone sizeof(o)-1) ? vlen : \ 529234370Sjasone sizeof(o)-1; \ 530234370Sjasone strncpy(o, v, cpylen); \ 531234370Sjasone o[cpylen] = '\0'; \ 532234370Sjasone continue; \ 533234370Sjasone } 534234370Sjasone 535234543Sjasone CONF_HANDLE_BOOL(opt_abort, "abort") 536234370Sjasone /* 537234370Sjasone * Chunks always require at least one header page, plus 538234370Sjasone * one data page in the absence of redzones, or three 539234370Sjasone * pages in the presence of redzones. In order to 540234370Sjasone * simplify options processing, fix the limit based on 541234370Sjasone * config_fill. 542234370Sjasone */ 543234543Sjasone CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 544234370Sjasone (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1) 545234543Sjasone CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 546234543Sjasone SIZE_T_MAX) 547234543Sjasone CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 548234370Sjasone -1, (sizeof(size_t) << 3) - 1) 549234543Sjasone CONF_HANDLE_BOOL(opt_stats_print, "stats_print") 550234370Sjasone if (config_fill) { 551234543Sjasone CONF_HANDLE_BOOL(opt_junk, "junk") 552234543Sjasone CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 553234370Sjasone 0, SIZE_T_MAX) 554234543Sjasone CONF_HANDLE_BOOL(opt_redzone, "redzone") 555234543Sjasone CONF_HANDLE_BOOL(opt_zero, "zero") 556234370Sjasone } 557234370Sjasone if (config_utrace) { 558234543Sjasone CONF_HANDLE_BOOL(opt_utrace, "utrace") 559234370Sjasone } 560234370Sjasone if (config_valgrind) { 561234370Sjasone bool hit; 562234370Sjasone CONF_HANDLE_BOOL_HIT(opt_valgrind, 563234543Sjasone "valgrind", hit) 564234370Sjasone if (config_fill && opt_valgrind && hit) { 565234370Sjasone opt_junk = false; 566234370Sjasone opt_zero = false; 567234370Sjasone if (opt_quarantine == 0) { 568234370Sjasone opt_quarantine = 569234370Sjasone JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 570234370Sjasone } 571234370Sjasone opt_redzone = true; 572234370Sjasone } 573234370Sjasone if (hit) 574234370Sjasone continue; 575234370Sjasone } 576234370Sjasone if (config_xmalloc) { 577234543Sjasone CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") 578234370Sjasone } 579234370Sjasone if (config_tcache) { 580234543Sjasone CONF_HANDLE_BOOL(opt_tcache, "tcache") 581234370Sjasone CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 582234543Sjasone "lg_tcache_max", -1, 583234370Sjasone (sizeof(size_t) << 3) - 1) 584234370Sjasone } 585234370Sjasone if (config_prof) { 586234543Sjasone CONF_HANDLE_BOOL(opt_prof, "prof") 587234543Sjasone CONF_HANDLE_CHAR_P(opt_prof_prefix, 588234543Sjasone "prof_prefix", "jeprof") 589234543Sjasone CONF_HANDLE_BOOL(opt_prof_active, "prof_active") 590234370Sjasone CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, 591234543Sjasone "lg_prof_sample", 0, 592234370Sjasone (sizeof(uint64_t) << 3) - 1) 593234543Sjasone CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") 594234370Sjasone CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 595234543Sjasone "lg_prof_interval", -1, 596234370Sjasone (sizeof(uint64_t) << 3) - 1) 597234543Sjasone CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") 598234543Sjasone CONF_HANDLE_BOOL(opt_prof_final, "prof_final") 599234543Sjasone CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") 600234370Sjasone } 601234370Sjasone malloc_conf_error("Invalid conf pair", k, klen, v, 602234370Sjasone vlen); 603234370Sjasone#undef CONF_HANDLE_BOOL 604234370Sjasone#undef CONF_HANDLE_SIZE_T 605234370Sjasone#undef CONF_HANDLE_SSIZE_T 606234370Sjasone#undef CONF_HANDLE_CHAR_P 607234370Sjasone } 608234370Sjasone } 609234370Sjasone} 610234370Sjasone 611234370Sjasonestatic bool 612234370Sjasonemalloc_init_hard(void) 613234370Sjasone{ 614234370Sjasone arena_t *init_arenas[1]; 615234370Sjasone 616234370Sjasone malloc_mutex_lock(&init_lock); 617234370Sjasone if (malloc_initialized || IS_INITIALIZER) { 618234370Sjasone /* 619234370Sjasone * Another thread initialized the allocator before this one 620234370Sjasone * acquired init_lock, or this thread is the initializing 621234370Sjasone * thread, and it is recursively allocating. 622234370Sjasone */ 623234370Sjasone malloc_mutex_unlock(&init_lock); 624234370Sjasone return (false); 625234370Sjasone } 626234370Sjasone#ifdef JEMALLOC_THREADED_INIT 627234370Sjasone if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { 628234370Sjasone /* Busy-wait until the initializing thread completes. */ 629234370Sjasone do { 630234370Sjasone malloc_mutex_unlock(&init_lock); 631234370Sjasone CPU_SPINWAIT; 632234370Sjasone malloc_mutex_lock(&init_lock); 633234370Sjasone } while (malloc_initialized == false); 634234370Sjasone malloc_mutex_unlock(&init_lock); 635234370Sjasone return (false); 636234370Sjasone } 637234370Sjasone#endif 638234370Sjasone malloc_initializer = INITIALIZER; 639234370Sjasone 640234370Sjasone malloc_tsd_boot(); 641234370Sjasone if (config_prof) 642234370Sjasone prof_boot0(); 643234370Sjasone 644234370Sjasone malloc_conf_init(); 645234370Sjasone 646235238Sjasone#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 647235238Sjasone && !defined(_WIN32)) 648234370Sjasone /* Register fork handlers. */ 649234370Sjasone if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 650234370Sjasone jemalloc_postfork_child) != 0) { 651234370Sjasone malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 652234370Sjasone if (opt_abort) 653234370Sjasone abort(); 654234370Sjasone } 655234370Sjasone#endif 656234370Sjasone 657234370Sjasone if (opt_stats_print) { 658234370Sjasone /* Print statistics at exit. */ 659234370Sjasone if (atexit(stats_print_atexit) != 0) { 660234370Sjasone malloc_write("<jemalloc>: Error in atexit()\n"); 661234370Sjasone if (opt_abort) 662234370Sjasone abort(); 663234370Sjasone } 664234370Sjasone } 665234370Sjasone 666234370Sjasone if (base_boot()) { 667234370Sjasone malloc_mutex_unlock(&init_lock); 668234370Sjasone return (true); 669234370Sjasone } 670234370Sjasone 671234569Sjasone if (chunk_boot()) { 672234370Sjasone malloc_mutex_unlock(&init_lock); 673234370Sjasone return (true); 674234370Sjasone } 675234370Sjasone 676234370Sjasone if (ctl_boot()) { 677234370Sjasone malloc_mutex_unlock(&init_lock); 678234370Sjasone return (true); 679234370Sjasone } 680234370Sjasone 681234370Sjasone if (config_prof) 682234370Sjasone prof_boot1(); 683234370Sjasone 684234370Sjasone arena_boot(); 685234370Sjasone 686234370Sjasone if (config_tcache && tcache_boot0()) { 687234370Sjasone malloc_mutex_unlock(&init_lock); 688234370Sjasone return (true); 689234370Sjasone } 690234370Sjasone 691234370Sjasone if (huge_boot()) { 692234370Sjasone malloc_mutex_unlock(&init_lock); 693234370Sjasone return (true); 694234370Sjasone } 695234370Sjasone 696234370Sjasone if (malloc_mutex_init(&arenas_lock)) 697234370Sjasone return (true); 698234370Sjasone 699234370Sjasone /* 700234370Sjasone * Create enough scaffolding to allow recursive allocation in 701234370Sjasone * malloc_ncpus(). 702234370Sjasone */ 703234370Sjasone narenas = 1; 704234370Sjasone arenas = init_arenas; 705234370Sjasone memset(arenas, 0, sizeof(arena_t *) * narenas); 706234370Sjasone 707234370Sjasone /* 708234370Sjasone * Initialize one arena here. The rest are lazily created in 709234370Sjasone * choose_arena_hard(). 710234370Sjasone */ 711234370Sjasone arenas_extend(0); 712234370Sjasone if (arenas[0] == NULL) { 713234370Sjasone malloc_mutex_unlock(&init_lock); 714234370Sjasone return (true); 715234370Sjasone } 716234370Sjasone 717234370Sjasone /* Initialize allocation counters before any allocations can occur. */ 718234370Sjasone if (config_stats && thread_allocated_tsd_boot()) { 719234370Sjasone malloc_mutex_unlock(&init_lock); 720234370Sjasone return (true); 721234370Sjasone } 722234370Sjasone 723234370Sjasone if (arenas_tsd_boot()) { 724234370Sjasone malloc_mutex_unlock(&init_lock); 725234370Sjasone return (true); 726234370Sjasone } 727234370Sjasone 728234370Sjasone if (config_tcache && tcache_boot1()) { 729234370Sjasone malloc_mutex_unlock(&init_lock); 730234370Sjasone return (true); 731234370Sjasone } 732234370Sjasone 733234370Sjasone if (config_fill && quarantine_boot()) { 734234370Sjasone malloc_mutex_unlock(&init_lock); 735234370Sjasone return (true); 736234370Sjasone } 737234370Sjasone 738234370Sjasone if (config_prof && prof_boot2()) { 739234370Sjasone malloc_mutex_unlock(&init_lock); 740234370Sjasone return (true); 741234370Sjasone } 742234370Sjasone 743234370Sjasone /* Get number of CPUs. */ 744234370Sjasone malloc_mutex_unlock(&init_lock); 745234370Sjasone ncpus = malloc_ncpus(); 746234370Sjasone malloc_mutex_lock(&init_lock); 747234370Sjasone 748234370Sjasone if (mutex_boot()) { 749234370Sjasone malloc_mutex_unlock(&init_lock); 750234370Sjasone return (true); 751234370Sjasone } 752234370Sjasone 753234370Sjasone if (opt_narenas == 0) { 754234370Sjasone /* 755234370Sjasone * For SMP systems, create more than one arena per CPU by 756234370Sjasone * default. 757234370Sjasone */ 758234370Sjasone if (ncpus > 1) 759234370Sjasone opt_narenas = ncpus << 2; 760234370Sjasone else 761234370Sjasone opt_narenas = 1; 762234370Sjasone } 763234370Sjasone narenas = opt_narenas; 764234370Sjasone /* 765234370Sjasone * Make sure that the arenas array can be allocated. In practice, this 766234370Sjasone * limit is enough to allow the allocator to function, but the ctl 767234370Sjasone * machinery will fail to allocate memory at far lower limits. 768234370Sjasone */ 769234370Sjasone if (narenas > chunksize / sizeof(arena_t *)) { 770234370Sjasone narenas = chunksize / sizeof(arena_t *); 771234370Sjasone malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 772234370Sjasone narenas); 773234370Sjasone } 774234370Sjasone 775234370Sjasone /* Allocate and initialize arenas. */ 776234370Sjasone arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); 777234370Sjasone if (arenas == NULL) { 778234370Sjasone malloc_mutex_unlock(&init_lock); 779234370Sjasone return (true); 780234370Sjasone } 781234370Sjasone /* 782234370Sjasone * Zero the array. In practice, this should always be pre-zeroed, 783234370Sjasone * since it was just mmap()ed, but let's be sure. 784234370Sjasone */ 785234370Sjasone memset(arenas, 0, sizeof(arena_t *) * narenas); 786234370Sjasone /* Copy the pointer to the one arena that was already initialized. */ 787234370Sjasone arenas[0] = init_arenas[0]; 788234370Sjasone 789234370Sjasone malloc_initialized = true; 790234370Sjasone malloc_mutex_unlock(&init_lock); 791234370Sjasone return (false); 792234370Sjasone} 793234370Sjasone 794234370Sjasone/* 795234370Sjasone * End initialization functions. 796234370Sjasone */ 797234370Sjasone/******************************************************************************/ 798234370Sjasone/* 799234370Sjasone * Begin malloc(3)-compatible functions. 800234370Sjasone */ 801234370Sjasone 802234370Sjasonevoid * 803234370Sjasoneje_malloc(size_t size) 804234370Sjasone{ 805234370Sjasone void *ret; 806235238Sjasone size_t usize JEMALLOC_CC_SILENCE_INIT(0); 807234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 808234370Sjasone 809234370Sjasone if (malloc_init()) { 810234370Sjasone ret = NULL; 811234370Sjasone goto label_oom; 812234370Sjasone } 813234370Sjasone 814234370Sjasone if (size == 0) 815234370Sjasone size = 1; 816234370Sjasone 817234370Sjasone if (config_prof && opt_prof) { 818234370Sjasone usize = s2u(size); 819234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 820234370Sjasone if (cnt == NULL) { 821234370Sjasone ret = NULL; 822234370Sjasone goto label_oom; 823234370Sjasone } 824234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 825234370Sjasone SMALL_MAXCLASS) { 826234370Sjasone ret = imalloc(SMALL_MAXCLASS+1); 827234370Sjasone if (ret != NULL) 828234370Sjasone arena_prof_promoted(ret, usize); 829234370Sjasone } else 830234370Sjasone ret = imalloc(size); 831234370Sjasone } else { 832234370Sjasone if (config_stats || (config_valgrind && opt_valgrind)) 833234370Sjasone usize = s2u(size); 834234370Sjasone ret = imalloc(size); 835234370Sjasone } 836234370Sjasone 837234370Sjasonelabel_oom: 838234370Sjasone if (ret == NULL) { 839234370Sjasone if (config_xmalloc && opt_xmalloc) { 840234370Sjasone malloc_write("<jemalloc>: Error in malloc(): " 841234370Sjasone "out of memory\n"); 842234370Sjasone abort(); 843234370Sjasone } 844235238Sjasone set_errno(ENOMEM); 845234370Sjasone } 846234370Sjasone if (config_prof && opt_prof && ret != NULL) 847234370Sjasone prof_malloc(ret, usize, cnt); 848234370Sjasone if (config_stats && ret != NULL) { 849234370Sjasone assert(usize == isalloc(ret, config_prof)); 850234370Sjasone thread_allocated_tsd_get()->allocated += usize; 851234370Sjasone } 852234370Sjasone UTRACE(0, size, ret); 853234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 854234370Sjasone return (ret); 855234370Sjasone} 856234370Sjasone 857234370SjasoneJEMALLOC_ATTR(nonnull(1)) 858234370Sjasone#ifdef JEMALLOC_PROF 859234370Sjasone/* 860234370Sjasone * Avoid any uncertainty as to how many backtrace frames to ignore in 861234370Sjasone * PROF_ALLOC_PREP(). 862234370Sjasone */ 863234370SjasoneJEMALLOC_ATTR(noinline) 864234370Sjasone#endif 865234370Sjasonestatic int 866234370Sjasoneimemalign(void **memptr, size_t alignment, size_t size, 867234370Sjasone size_t min_alignment) 868234370Sjasone{ 869234370Sjasone int ret; 870234370Sjasone size_t usize; 871234370Sjasone void *result; 872234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 873234370Sjasone 874234370Sjasone assert(min_alignment != 0); 875234370Sjasone 876234370Sjasone if (malloc_init()) 877234370Sjasone result = NULL; 878234370Sjasone else { 879234370Sjasone if (size == 0) 880234370Sjasone size = 1; 881234370Sjasone 882234370Sjasone /* Make sure that alignment is a large enough power of 2. */ 883234370Sjasone if (((alignment - 1) & alignment) != 0 884234370Sjasone || (alignment < min_alignment)) { 885234370Sjasone if (config_xmalloc && opt_xmalloc) { 886234370Sjasone malloc_write("<jemalloc>: Error allocating " 887234370Sjasone "aligned memory: invalid alignment\n"); 888234370Sjasone abort(); 889234370Sjasone } 890234370Sjasone result = NULL; 891234370Sjasone ret = EINVAL; 892234370Sjasone goto label_return; 893234370Sjasone } 894234370Sjasone 895234370Sjasone usize = sa2u(size, alignment); 896234370Sjasone if (usize == 0) { 897234370Sjasone result = NULL; 898234370Sjasone ret = ENOMEM; 899234370Sjasone goto label_return; 900234370Sjasone } 901234370Sjasone 902234370Sjasone if (config_prof && opt_prof) { 903234370Sjasone PROF_ALLOC_PREP(2, usize, cnt); 904234370Sjasone if (cnt == NULL) { 905234370Sjasone result = NULL; 906234370Sjasone ret = EINVAL; 907234370Sjasone } else { 908234370Sjasone if (prof_promote && (uintptr_t)cnt != 909234370Sjasone (uintptr_t)1U && usize <= SMALL_MAXCLASS) { 910234370Sjasone assert(sa2u(SMALL_MAXCLASS+1, 911234370Sjasone alignment) != 0); 912234370Sjasone result = ipalloc(sa2u(SMALL_MAXCLASS+1, 913234370Sjasone alignment), alignment, false); 914234370Sjasone if (result != NULL) { 915234370Sjasone arena_prof_promoted(result, 916234370Sjasone usize); 917234370Sjasone } 918234370Sjasone } else { 919234370Sjasone result = ipalloc(usize, alignment, 920234370Sjasone false); 921234370Sjasone } 922234370Sjasone } 923234370Sjasone } else 924234370Sjasone result = ipalloc(usize, alignment, false); 925234370Sjasone } 926234370Sjasone 927234370Sjasone if (result == NULL) { 928234370Sjasone if (config_xmalloc && opt_xmalloc) { 929234370Sjasone malloc_write("<jemalloc>: Error allocating aligned " 930234370Sjasone "memory: out of memory\n"); 931234370Sjasone abort(); 932234370Sjasone } 933234370Sjasone ret = ENOMEM; 934234370Sjasone goto label_return; 935234370Sjasone } 936234370Sjasone 937234370Sjasone *memptr = result; 938234370Sjasone ret = 0; 939234370Sjasone 940234370Sjasonelabel_return: 941234370Sjasone if (config_stats && result != NULL) { 942234370Sjasone assert(usize == isalloc(result, config_prof)); 943234370Sjasone thread_allocated_tsd_get()->allocated += usize; 944234370Sjasone } 945234370Sjasone if (config_prof && opt_prof && result != NULL) 946234370Sjasone prof_malloc(result, usize, cnt); 947234370Sjasone UTRACE(0, size, result); 948234370Sjasone return (ret); 949234370Sjasone} 950234370Sjasone 951234370Sjasoneint 952234370Sjasoneje_posix_memalign(void **memptr, size_t alignment, size_t size) 953234370Sjasone{ 954234370Sjasone int ret = imemalign(memptr, alignment, size, sizeof(void *)); 955234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 956234370Sjasone config_prof), false); 957234370Sjasone return (ret); 958234370Sjasone} 959234370Sjasone 960234370Sjasonevoid * 961234370Sjasoneje_aligned_alloc(size_t alignment, size_t size) 962234370Sjasone{ 963234370Sjasone void *ret; 964234370Sjasone int err; 965234370Sjasone 966234370Sjasone if ((err = imemalign(&ret, alignment, size, 1)) != 0) { 967234370Sjasone ret = NULL; 968235238Sjasone set_errno(err); 969234370Sjasone } 970234370Sjasone JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 971234370Sjasone false); 972234370Sjasone return (ret); 973234370Sjasone} 974234370Sjasone 975234370Sjasonevoid * 976234370Sjasoneje_calloc(size_t num, size_t size) 977234370Sjasone{ 978234370Sjasone void *ret; 979234370Sjasone size_t num_size; 980235238Sjasone size_t usize JEMALLOC_CC_SILENCE_INIT(0); 981234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 982234370Sjasone 983234370Sjasone if (malloc_init()) { 984234370Sjasone num_size = 0; 985234370Sjasone ret = NULL; 986234370Sjasone goto label_return; 987234370Sjasone } 988234370Sjasone 989234370Sjasone num_size = num * size; 990234370Sjasone if (num_size == 0) { 991234370Sjasone if (num == 0 || size == 0) 992234370Sjasone num_size = 1; 993234370Sjasone else { 994234370Sjasone ret = NULL; 995234370Sjasone goto label_return; 996234370Sjasone } 997234370Sjasone /* 998234370Sjasone * Try to avoid division here. We know that it isn't possible to 999234370Sjasone * overflow during multiplication if neither operand uses any of the 1000234370Sjasone * most significant half of the bits in a size_t. 1001234370Sjasone */ 1002234370Sjasone } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1003234370Sjasone && (num_size / size != num)) { 1004234370Sjasone /* size_t overflow. */ 1005234370Sjasone ret = NULL; 1006234370Sjasone goto label_return; 1007234370Sjasone } 1008234370Sjasone 1009234370Sjasone if (config_prof && opt_prof) { 1010234370Sjasone usize = s2u(num_size); 1011234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1012234370Sjasone if (cnt == NULL) { 1013234370Sjasone ret = NULL; 1014234370Sjasone goto label_return; 1015234370Sjasone } 1016234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize 1017234370Sjasone <= SMALL_MAXCLASS) { 1018234370Sjasone ret = icalloc(SMALL_MAXCLASS+1); 1019234370Sjasone if (ret != NULL) 1020234370Sjasone arena_prof_promoted(ret, usize); 1021234370Sjasone } else 1022234370Sjasone ret = icalloc(num_size); 1023234370Sjasone } else { 1024234370Sjasone if (config_stats || (config_valgrind && opt_valgrind)) 1025234370Sjasone usize = s2u(num_size); 1026234370Sjasone ret = icalloc(num_size); 1027234370Sjasone } 1028234370Sjasone 1029234370Sjasonelabel_return: 1030234370Sjasone if (ret == NULL) { 1031234370Sjasone if (config_xmalloc && opt_xmalloc) { 1032234370Sjasone malloc_write("<jemalloc>: Error in calloc(): out of " 1033234370Sjasone "memory\n"); 1034234370Sjasone abort(); 1035234370Sjasone } 1036235238Sjasone set_errno(ENOMEM); 1037234370Sjasone } 1038234370Sjasone 1039234370Sjasone if (config_prof && opt_prof && ret != NULL) 1040234370Sjasone prof_malloc(ret, usize, cnt); 1041234370Sjasone if (config_stats && ret != NULL) { 1042234370Sjasone assert(usize == isalloc(ret, config_prof)); 1043234370Sjasone thread_allocated_tsd_get()->allocated += usize; 1044234370Sjasone } 1045234370Sjasone UTRACE(0, num_size, ret); 1046234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1047234370Sjasone return (ret); 1048234370Sjasone} 1049234370Sjasone 1050234370Sjasonevoid * 1051234370Sjasoneje_realloc(void *ptr, size_t size) 1052234370Sjasone{ 1053234370Sjasone void *ret; 1054235238Sjasone size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1055234370Sjasone size_t old_size = 0; 1056234370Sjasone size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1057234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 1058234370Sjasone prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); 1059234370Sjasone 1060234370Sjasone if (size == 0) { 1061234370Sjasone if (ptr != NULL) { 1062234370Sjasone /* realloc(ptr, 0) is equivalent to free(p). */ 1063234370Sjasone if (config_prof) { 1064234370Sjasone old_size = isalloc(ptr, true); 1065234370Sjasone if (config_valgrind && opt_valgrind) 1066234370Sjasone old_rzsize = p2rz(ptr); 1067234370Sjasone } else if (config_stats) { 1068234370Sjasone old_size = isalloc(ptr, false); 1069234370Sjasone if (config_valgrind && opt_valgrind) 1070234370Sjasone old_rzsize = u2rz(old_size); 1071234370Sjasone } else if (config_valgrind && opt_valgrind) { 1072234370Sjasone old_size = isalloc(ptr, false); 1073234370Sjasone old_rzsize = u2rz(old_size); 1074234370Sjasone } 1075234370Sjasone if (config_prof && opt_prof) { 1076234370Sjasone old_ctx = prof_ctx_get(ptr); 1077234370Sjasone cnt = NULL; 1078234370Sjasone } 1079234370Sjasone iqalloc(ptr); 1080234370Sjasone ret = NULL; 1081234370Sjasone goto label_return; 1082234370Sjasone } else 1083234370Sjasone size = 1; 1084234370Sjasone } 1085234370Sjasone 1086234370Sjasone if (ptr != NULL) { 1087234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1088234370Sjasone 1089234370Sjasone if (config_prof) { 1090234370Sjasone old_size = isalloc(ptr, true); 1091234370Sjasone if (config_valgrind && opt_valgrind) 1092234370Sjasone old_rzsize = p2rz(ptr); 1093234370Sjasone } else if (config_stats) { 1094234370Sjasone old_size = isalloc(ptr, false); 1095234370Sjasone if (config_valgrind && opt_valgrind) 1096234370Sjasone old_rzsize = u2rz(old_size); 1097234370Sjasone } else if (config_valgrind && opt_valgrind) { 1098234370Sjasone old_size = isalloc(ptr, false); 1099234370Sjasone old_rzsize = u2rz(old_size); 1100234370Sjasone } 1101234370Sjasone if (config_prof && opt_prof) { 1102234370Sjasone usize = s2u(size); 1103234370Sjasone old_ctx = prof_ctx_get(ptr); 1104234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1105234370Sjasone if (cnt == NULL) { 1106234370Sjasone old_ctx = NULL; 1107234370Sjasone ret = NULL; 1108234370Sjasone goto label_oom; 1109234370Sjasone } 1110234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && 1111234370Sjasone usize <= SMALL_MAXCLASS) { 1112234370Sjasone ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0, 1113234370Sjasone false, false); 1114234370Sjasone if (ret != NULL) 1115234370Sjasone arena_prof_promoted(ret, usize); 1116234370Sjasone else 1117234370Sjasone old_ctx = NULL; 1118234370Sjasone } else { 1119234370Sjasone ret = iralloc(ptr, size, 0, 0, false, false); 1120234370Sjasone if (ret == NULL) 1121234370Sjasone old_ctx = NULL; 1122234370Sjasone } 1123234370Sjasone } else { 1124234370Sjasone if (config_stats || (config_valgrind && opt_valgrind)) 1125234370Sjasone usize = s2u(size); 1126234370Sjasone ret = iralloc(ptr, size, 0, 0, false, false); 1127234370Sjasone } 1128234370Sjasone 1129234370Sjasonelabel_oom: 1130234370Sjasone if (ret == NULL) { 1131234370Sjasone if (config_xmalloc && opt_xmalloc) { 1132234370Sjasone malloc_write("<jemalloc>: Error in realloc(): " 1133234370Sjasone "out of memory\n"); 1134234370Sjasone abort(); 1135234370Sjasone } 1136235238Sjasone set_errno(ENOMEM); 1137234370Sjasone } 1138234370Sjasone } else { 1139234370Sjasone /* realloc(NULL, size) is equivalent to malloc(size). */ 1140234370Sjasone if (config_prof && opt_prof) 1141234370Sjasone old_ctx = NULL; 1142234370Sjasone if (malloc_init()) { 1143234370Sjasone if (config_prof && opt_prof) 1144234370Sjasone cnt = NULL; 1145234370Sjasone ret = NULL; 1146234370Sjasone } else { 1147234370Sjasone if (config_prof && opt_prof) { 1148234370Sjasone usize = s2u(size); 1149234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1150234370Sjasone if (cnt == NULL) 1151234370Sjasone ret = NULL; 1152234370Sjasone else { 1153234370Sjasone if (prof_promote && (uintptr_t)cnt != 1154234370Sjasone (uintptr_t)1U && usize <= 1155234370Sjasone SMALL_MAXCLASS) { 1156234370Sjasone ret = imalloc(SMALL_MAXCLASS+1); 1157234370Sjasone if (ret != NULL) { 1158234370Sjasone arena_prof_promoted(ret, 1159234370Sjasone usize); 1160234370Sjasone } 1161234370Sjasone } else 1162234370Sjasone ret = imalloc(size); 1163234370Sjasone } 1164234370Sjasone } else { 1165234370Sjasone if (config_stats || (config_valgrind && 1166234370Sjasone opt_valgrind)) 1167234370Sjasone usize = s2u(size); 1168234370Sjasone ret = imalloc(size); 1169234370Sjasone } 1170234370Sjasone } 1171234370Sjasone 1172234370Sjasone if (ret == NULL) { 1173234370Sjasone if (config_xmalloc && opt_xmalloc) { 1174234370Sjasone malloc_write("<jemalloc>: Error in realloc(): " 1175234370Sjasone "out of memory\n"); 1176234370Sjasone abort(); 1177234370Sjasone } 1178235238Sjasone set_errno(ENOMEM); 1179234370Sjasone } 1180234370Sjasone } 1181234370Sjasone 1182234370Sjasonelabel_return: 1183234370Sjasone if (config_prof && opt_prof) 1184234370Sjasone prof_realloc(ret, usize, cnt, old_size, old_ctx); 1185234370Sjasone if (config_stats && ret != NULL) { 1186234370Sjasone thread_allocated_t *ta; 1187234370Sjasone assert(usize == isalloc(ret, config_prof)); 1188234370Sjasone ta = thread_allocated_tsd_get(); 1189234370Sjasone ta->allocated += usize; 1190234370Sjasone ta->deallocated += old_size; 1191234370Sjasone } 1192234370Sjasone UTRACE(ptr, size, ret); 1193234370Sjasone JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); 1194234370Sjasone return (ret); 1195234370Sjasone} 1196234370Sjasone 1197234370Sjasonevoid 1198234370Sjasoneje_free(void *ptr) 1199234370Sjasone{ 1200234370Sjasone 1201234370Sjasone UTRACE(ptr, 0, 0); 1202234370Sjasone if (ptr != NULL) { 1203234370Sjasone size_t usize; 1204234370Sjasone size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1205234370Sjasone 1206234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1207234370Sjasone 1208234370Sjasone if (config_prof && opt_prof) { 1209234370Sjasone usize = isalloc(ptr, config_prof); 1210234370Sjasone prof_free(ptr, usize); 1211234370Sjasone } else if (config_stats || config_valgrind) 1212234370Sjasone usize = isalloc(ptr, config_prof); 1213234370Sjasone if (config_stats) 1214234370Sjasone thread_allocated_tsd_get()->deallocated += usize; 1215234370Sjasone if (config_valgrind && opt_valgrind) 1216234370Sjasone rzsize = p2rz(ptr); 1217234370Sjasone iqalloc(ptr); 1218234370Sjasone JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1219234370Sjasone } 1220234370Sjasone} 1221234370Sjasone 1222234370Sjasone/* 1223234370Sjasone * End malloc(3)-compatible functions. 1224234370Sjasone */ 1225234370Sjasone/******************************************************************************/ 1226234370Sjasone/* 1227234370Sjasone * Begin non-standard override functions. 1228234370Sjasone */ 1229234370Sjasone 1230234370Sjasone#ifdef JEMALLOC_OVERRIDE_MEMALIGN 1231234370Sjasonevoid * 1232234370Sjasoneje_memalign(size_t alignment, size_t size) 1233234370Sjasone{ 1234234370Sjasone void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1235234370Sjasone imemalign(&ret, alignment, size, 1); 1236234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1237234370Sjasone return (ret); 1238234370Sjasone} 1239234370Sjasone#endif 1240234370Sjasone 1241234370Sjasone#ifdef JEMALLOC_OVERRIDE_VALLOC 1242234370Sjasonevoid * 1243234370Sjasoneje_valloc(size_t size) 1244234370Sjasone{ 1245234370Sjasone void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1246234370Sjasone imemalign(&ret, PAGE, size, 1); 1247234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1248234370Sjasone return (ret); 1249234370Sjasone} 1250234370Sjasone#endif 1251234370Sjasone 1252234370Sjasone/* 1253234370Sjasone * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1254234370Sjasone * #define je_malloc malloc 1255234370Sjasone */ 1256234370Sjasone#define malloc_is_malloc 1 1257234370Sjasone#define is_malloc_(a) malloc_is_ ## a 1258234370Sjasone#define is_malloc(a) is_malloc_(a) 1259234370Sjasone 1260234370Sjasone#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) 1261234370Sjasone/* 1262234370Sjasone * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1263234370Sjasone * to inconsistently reference libc's malloc(3)-compatible functions 1264234370Sjasone * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1265234370Sjasone * 1266234370Sjasone * These definitions interpose hooks in glibc. The functions are actually 1267234370Sjasone * passed an extra argument for the caller return address, which will be 1268234370Sjasone * ignored. 1269234370Sjasone */ 1270235238SjasoneJEMALLOC_EXPORT void (* const __free_hook)(void *ptr) = je_free; 1271235238SjasoneJEMALLOC_EXPORT void *(* const __malloc_hook)(size_t size) = je_malloc; 1272235238SjasoneJEMALLOC_EXPORT void *(* const __realloc_hook)(void *ptr, size_t size) = 1273235238Sjasone je_realloc; 1274235238SjasoneJEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) = 1275235238Sjasone je_memalign; 1276234370Sjasone#endif 1277234370Sjasone 1278234370Sjasone/* 1279234370Sjasone * End non-standard override functions. 1280234370Sjasone */ 1281234370Sjasone/******************************************************************************/ 1282234370Sjasone/* 1283234370Sjasone * Begin non-standard functions. 1284234370Sjasone */ 1285234370Sjasone 1286234370Sjasonesize_t 1287234370Sjasoneje_malloc_usable_size(const void *ptr) 1288234370Sjasone{ 1289234370Sjasone size_t ret; 1290234370Sjasone 1291234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1292234370Sjasone 1293234370Sjasone if (config_ivsalloc) 1294234370Sjasone ret = ivsalloc(ptr, config_prof); 1295234370Sjasone else 1296234370Sjasone ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 1297234370Sjasone 1298234370Sjasone return (ret); 1299234370Sjasone} 1300234370Sjasone 1301234370Sjasonevoid 1302234370Sjasoneje_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 1303234370Sjasone const char *opts) 1304234370Sjasone{ 1305234370Sjasone 1306234370Sjasone stats_print(write_cb, cbopaque, opts); 1307234370Sjasone} 1308234370Sjasone 1309234370Sjasoneint 1310234370Sjasoneje_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 1311234370Sjasone size_t newlen) 1312234370Sjasone{ 1313234370Sjasone 1314234370Sjasone if (malloc_init()) 1315234370Sjasone return (EAGAIN); 1316234370Sjasone 1317234370Sjasone return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1318234370Sjasone} 1319234370Sjasone 1320234370Sjasoneint 1321234370Sjasoneje_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 1322234370Sjasone{ 1323234370Sjasone 1324234370Sjasone if (malloc_init()) 1325234370Sjasone return (EAGAIN); 1326234370Sjasone 1327234370Sjasone return (ctl_nametomib(name, mibp, miblenp)); 1328234370Sjasone} 1329234370Sjasone 1330234370Sjasoneint 1331234370Sjasoneje_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1332234370Sjasone void *newp, size_t newlen) 1333234370Sjasone{ 1334234370Sjasone 1335234370Sjasone if (malloc_init()) 1336234370Sjasone return (EAGAIN); 1337234370Sjasone 1338234370Sjasone return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1339234370Sjasone} 1340234370Sjasone 1341234370Sjasone/* 1342234370Sjasone * End non-standard functions. 1343234370Sjasone */ 1344234370Sjasone/******************************************************************************/ 1345234370Sjasone/* 1346234370Sjasone * Begin experimental functions. 1347234370Sjasone */ 1348234370Sjasone#ifdef JEMALLOC_EXPERIMENTAL 1349234370Sjasone 1350234370SjasoneJEMALLOC_INLINE void * 1351234370Sjasoneiallocm(size_t usize, size_t alignment, bool zero) 1352234370Sjasone{ 1353234370Sjasone 1354234370Sjasone assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, 1355234370Sjasone alignment))); 1356234370Sjasone 1357234370Sjasone if (alignment != 0) 1358234370Sjasone return (ipalloc(usize, alignment, zero)); 1359234370Sjasone else if (zero) 1360234370Sjasone return (icalloc(usize)); 1361234370Sjasone else 1362234370Sjasone return (imalloc(usize)); 1363234370Sjasone} 1364234370Sjasone 1365234370Sjasoneint 1366234370Sjasoneje_allocm(void **ptr, size_t *rsize, size_t size, int flags) 1367234370Sjasone{ 1368234370Sjasone void *p; 1369234370Sjasone size_t usize; 1370234370Sjasone size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1371234370Sjasone & (SIZE_T_MAX-1)); 1372234370Sjasone bool zero = flags & ALLOCM_ZERO; 1373234370Sjasone 1374234370Sjasone assert(ptr != NULL); 1375234370Sjasone assert(size != 0); 1376234370Sjasone 1377234370Sjasone if (malloc_init()) 1378234370Sjasone goto label_oom; 1379234370Sjasone 1380234370Sjasone usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1381234370Sjasone if (usize == 0) 1382234370Sjasone goto label_oom; 1383234370Sjasone 1384234370Sjasone if (config_prof && opt_prof) { 1385235238Sjasone prof_thr_cnt_t *cnt; 1386235238Sjasone 1387234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1388234370Sjasone if (cnt == NULL) 1389234370Sjasone goto label_oom; 1390234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 1391234370Sjasone SMALL_MAXCLASS) { 1392234370Sjasone size_t usize_promoted = (alignment == 0) ? 1393234370Sjasone s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, 1394234370Sjasone alignment); 1395234370Sjasone assert(usize_promoted != 0); 1396234370Sjasone p = iallocm(usize_promoted, alignment, zero); 1397234370Sjasone if (p == NULL) 1398234370Sjasone goto label_oom; 1399234370Sjasone arena_prof_promoted(p, usize); 1400234370Sjasone } else { 1401234370Sjasone p = iallocm(usize, alignment, zero); 1402234370Sjasone if (p == NULL) 1403234370Sjasone goto label_oom; 1404234370Sjasone } 1405234370Sjasone prof_malloc(p, usize, cnt); 1406234370Sjasone } else { 1407234370Sjasone p = iallocm(usize, alignment, zero); 1408234370Sjasone if (p == NULL) 1409234370Sjasone goto label_oom; 1410234370Sjasone } 1411234370Sjasone if (rsize != NULL) 1412234370Sjasone *rsize = usize; 1413234370Sjasone 1414234370Sjasone *ptr = p; 1415234370Sjasone if (config_stats) { 1416234370Sjasone assert(usize == isalloc(p, config_prof)); 1417234370Sjasone thread_allocated_tsd_get()->allocated += usize; 1418234370Sjasone } 1419234370Sjasone UTRACE(0, size, p); 1420234370Sjasone JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); 1421234370Sjasone return (ALLOCM_SUCCESS); 1422234370Sjasonelabel_oom: 1423234370Sjasone if (config_xmalloc && opt_xmalloc) { 1424234370Sjasone malloc_write("<jemalloc>: Error in allocm(): " 1425234370Sjasone "out of memory\n"); 1426234370Sjasone abort(); 1427234370Sjasone } 1428234370Sjasone *ptr = NULL; 1429234370Sjasone UTRACE(0, size, 0); 1430234370Sjasone return (ALLOCM_ERR_OOM); 1431234370Sjasone} 1432234370Sjasone 1433234370Sjasoneint 1434234370Sjasoneje_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 1435234370Sjasone{ 1436234370Sjasone void *p, *q; 1437234370Sjasone size_t usize; 1438234370Sjasone size_t old_size; 1439234370Sjasone size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1440234370Sjasone size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1441234370Sjasone & (SIZE_T_MAX-1)); 1442234370Sjasone bool zero = flags & ALLOCM_ZERO; 1443234370Sjasone bool no_move = flags & ALLOCM_NO_MOVE; 1444234370Sjasone 1445234370Sjasone assert(ptr != NULL); 1446234370Sjasone assert(*ptr != NULL); 1447234370Sjasone assert(size != 0); 1448234370Sjasone assert(SIZE_T_MAX - size >= extra); 1449234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1450234370Sjasone 1451234370Sjasone p = *ptr; 1452234370Sjasone if (config_prof && opt_prof) { 1453235238Sjasone prof_thr_cnt_t *cnt; 1454235238Sjasone 1455234370Sjasone /* 1456234370Sjasone * usize isn't knowable before iralloc() returns when extra is 1457234370Sjasone * non-zero. Therefore, compute its maximum possible value and 1458234370Sjasone * use that in PROF_ALLOC_PREP() to decide whether to capture a 1459234370Sjasone * backtrace. prof_realloc() will use the actual usize to 1460234370Sjasone * decide whether to sample. 1461234370Sjasone */ 1462234370Sjasone size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1463234370Sjasone sa2u(size+extra, alignment); 1464234370Sjasone prof_ctx_t *old_ctx = prof_ctx_get(p); 1465234370Sjasone old_size = isalloc(p, true); 1466234370Sjasone if (config_valgrind && opt_valgrind) 1467234370Sjasone old_rzsize = p2rz(p); 1468234370Sjasone PROF_ALLOC_PREP(1, max_usize, cnt); 1469234370Sjasone if (cnt == NULL) 1470234370Sjasone goto label_oom; 1471234370Sjasone /* 1472234370Sjasone * Use minimum usize to determine whether promotion may happen. 1473234370Sjasone */ 1474234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U 1475234370Sjasone && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) 1476234370Sjasone <= SMALL_MAXCLASS) { 1477234370Sjasone q = iralloc(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1478234370Sjasone size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 1479234370Sjasone alignment, zero, no_move); 1480234370Sjasone if (q == NULL) 1481234370Sjasone goto label_err; 1482234370Sjasone if (max_usize < PAGE) { 1483234370Sjasone usize = max_usize; 1484234370Sjasone arena_prof_promoted(q, usize); 1485234370Sjasone } else 1486234370Sjasone usize = isalloc(q, config_prof); 1487234370Sjasone } else { 1488234370Sjasone q = iralloc(p, size, extra, alignment, zero, no_move); 1489234370Sjasone if (q == NULL) 1490234370Sjasone goto label_err; 1491234370Sjasone usize = isalloc(q, config_prof); 1492234370Sjasone } 1493234370Sjasone prof_realloc(q, usize, cnt, old_size, old_ctx); 1494234370Sjasone if (rsize != NULL) 1495234370Sjasone *rsize = usize; 1496234370Sjasone } else { 1497234370Sjasone if (config_stats) { 1498234370Sjasone old_size = isalloc(p, false); 1499234370Sjasone if (config_valgrind && opt_valgrind) 1500234370Sjasone old_rzsize = u2rz(old_size); 1501234370Sjasone } else if (config_valgrind && opt_valgrind) { 1502234370Sjasone old_size = isalloc(p, false); 1503234370Sjasone old_rzsize = u2rz(old_size); 1504234370Sjasone } 1505234370Sjasone q = iralloc(p, size, extra, alignment, zero, no_move); 1506234370Sjasone if (q == NULL) 1507234370Sjasone goto label_err; 1508234370Sjasone if (config_stats) 1509234370Sjasone usize = isalloc(q, config_prof); 1510234370Sjasone if (rsize != NULL) { 1511234370Sjasone if (config_stats == false) 1512234370Sjasone usize = isalloc(q, config_prof); 1513234370Sjasone *rsize = usize; 1514234370Sjasone } 1515234370Sjasone } 1516234370Sjasone 1517234370Sjasone *ptr = q; 1518234370Sjasone if (config_stats) { 1519234370Sjasone thread_allocated_t *ta; 1520234370Sjasone ta = thread_allocated_tsd_get(); 1521234370Sjasone ta->allocated += usize; 1522234370Sjasone ta->deallocated += old_size; 1523234370Sjasone } 1524234370Sjasone UTRACE(p, size, q); 1525234370Sjasone JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); 1526234370Sjasone return (ALLOCM_SUCCESS); 1527234370Sjasonelabel_err: 1528234370Sjasone if (no_move) { 1529234370Sjasone UTRACE(p, size, q); 1530234370Sjasone return (ALLOCM_ERR_NOT_MOVED); 1531234370Sjasone } 1532234370Sjasonelabel_oom: 1533234370Sjasone if (config_xmalloc && opt_xmalloc) { 1534234370Sjasone malloc_write("<jemalloc>: Error in rallocm(): " 1535234370Sjasone "out of memory\n"); 1536234370Sjasone abort(); 1537234370Sjasone } 1538234370Sjasone UTRACE(p, size, 0); 1539234370Sjasone return (ALLOCM_ERR_OOM); 1540234370Sjasone} 1541234370Sjasone 1542234370Sjasoneint 1543234370Sjasoneje_sallocm(const void *ptr, size_t *rsize, int flags) 1544234370Sjasone{ 1545234370Sjasone size_t sz; 1546234370Sjasone 1547234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1548234370Sjasone 1549234370Sjasone if (config_ivsalloc) 1550234370Sjasone sz = ivsalloc(ptr, config_prof); 1551234370Sjasone else { 1552234370Sjasone assert(ptr != NULL); 1553234370Sjasone sz = isalloc(ptr, config_prof); 1554234370Sjasone } 1555234370Sjasone assert(rsize != NULL); 1556234370Sjasone *rsize = sz; 1557234370Sjasone 1558234370Sjasone return (ALLOCM_SUCCESS); 1559234370Sjasone} 1560234370Sjasone 1561234370Sjasoneint 1562234370Sjasoneje_dallocm(void *ptr, int flags) 1563234370Sjasone{ 1564234370Sjasone size_t usize; 1565234370Sjasone size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1566234370Sjasone 1567234370Sjasone assert(ptr != NULL); 1568234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1569234370Sjasone 1570234370Sjasone UTRACE(ptr, 0, 0); 1571234370Sjasone if (config_stats || config_valgrind) 1572234370Sjasone usize = isalloc(ptr, config_prof); 1573234370Sjasone if (config_prof && opt_prof) { 1574234370Sjasone if (config_stats == false && config_valgrind == false) 1575234370Sjasone usize = isalloc(ptr, config_prof); 1576234370Sjasone prof_free(ptr, usize); 1577234370Sjasone } 1578234370Sjasone if (config_stats) 1579234370Sjasone thread_allocated_tsd_get()->deallocated += usize; 1580234370Sjasone if (config_valgrind && opt_valgrind) 1581234370Sjasone rzsize = p2rz(ptr); 1582234370Sjasone iqalloc(ptr); 1583234370Sjasone JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1584234370Sjasone 1585234370Sjasone return (ALLOCM_SUCCESS); 1586234370Sjasone} 1587234370Sjasone 1588234370Sjasoneint 1589234370Sjasoneje_nallocm(size_t *rsize, size_t size, int flags) 1590234370Sjasone{ 1591234370Sjasone size_t usize; 1592234370Sjasone size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1593234370Sjasone & (SIZE_T_MAX-1)); 1594234370Sjasone 1595234370Sjasone assert(size != 0); 1596234370Sjasone 1597234370Sjasone if (malloc_init()) 1598234370Sjasone return (ALLOCM_ERR_OOM); 1599234370Sjasone 1600234370Sjasone usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1601234370Sjasone if (usize == 0) 1602234370Sjasone return (ALLOCM_ERR_OOM); 1603234370Sjasone 1604234370Sjasone if (rsize != NULL) 1605234370Sjasone *rsize = usize; 1606234370Sjasone return (ALLOCM_SUCCESS); 1607234370Sjasone} 1608234370Sjasone 1609234370Sjasone#endif 1610234370Sjasone/* 1611234370Sjasone * End experimental functions. 1612234370Sjasone */ 1613234370Sjasone/******************************************************************************/ 1614234370Sjasone/* 1615234370Sjasone * The following functions are used by threading libraries for protection of 1616234370Sjasone * malloc during fork(). 1617234370Sjasone */ 1618234370Sjasone 1619234370Sjasone#ifndef JEMALLOC_MUTEX_INIT_CB 1620234370Sjasonevoid 1621234370Sjasonejemalloc_prefork(void) 1622234370Sjasone#else 1623235238SjasoneJEMALLOC_EXPORT void 1624234370Sjasone_malloc_prefork(void) 1625234370Sjasone#endif 1626234370Sjasone{ 1627234370Sjasone unsigned i; 1628234370Sjasone 1629235322Sjasone#ifdef JEMALLOC_MUTEX_INIT_CB 1630235322Sjasone if (malloc_initialized == false) 1631235322Sjasone return; 1632235322Sjasone#endif 1633235322Sjasone assert(malloc_initialized); 1634235322Sjasone 1635234370Sjasone /* Acquire all mutexes in a safe order. */ 1636234370Sjasone malloc_mutex_prefork(&arenas_lock); 1637234370Sjasone for (i = 0; i < narenas; i++) { 1638234370Sjasone if (arenas[i] != NULL) 1639234370Sjasone arena_prefork(arenas[i]); 1640234370Sjasone } 1641234370Sjasone base_prefork(); 1642234370Sjasone huge_prefork(); 1643234370Sjasone chunk_dss_prefork(); 1644234370Sjasone} 1645234370Sjasone 1646234370Sjasone#ifndef JEMALLOC_MUTEX_INIT_CB 1647234370Sjasonevoid 1648234370Sjasonejemalloc_postfork_parent(void) 1649234370Sjasone#else 1650235238SjasoneJEMALLOC_EXPORT void 1651234370Sjasone_malloc_postfork(void) 1652234370Sjasone#endif 1653234370Sjasone{ 1654234370Sjasone unsigned i; 1655234370Sjasone 1656235322Sjasone#ifdef JEMALLOC_MUTEX_INIT_CB 1657235322Sjasone if (malloc_initialized == false) 1658235322Sjasone return; 1659235322Sjasone#endif 1660235322Sjasone assert(malloc_initialized); 1661235322Sjasone 1662234370Sjasone /* Release all mutexes, now that fork() has completed. */ 1663234370Sjasone chunk_dss_postfork_parent(); 1664234370Sjasone huge_postfork_parent(); 1665234370Sjasone base_postfork_parent(); 1666234370Sjasone for (i = 0; i < narenas; i++) { 1667234370Sjasone if (arenas[i] != NULL) 1668234370Sjasone arena_postfork_parent(arenas[i]); 1669234370Sjasone } 1670234370Sjasone malloc_mutex_postfork_parent(&arenas_lock); 1671234370Sjasone} 1672234370Sjasone 1673234370Sjasonevoid 1674234370Sjasonejemalloc_postfork_child(void) 1675234370Sjasone{ 1676234370Sjasone unsigned i; 1677234370Sjasone 1678235322Sjasone assert(malloc_initialized); 1679235322Sjasone 1680234370Sjasone /* Release all mutexes, now that fork() has completed. */ 1681234370Sjasone chunk_dss_postfork_child(); 1682234370Sjasone huge_postfork_child(); 1683234370Sjasone base_postfork_child(); 1684234370Sjasone for (i = 0; i < narenas; i++) { 1685234370Sjasone if (arenas[i] != NULL) 1686234370Sjasone arena_postfork_child(arenas[i]); 1687234370Sjasone } 1688234370Sjasone malloc_mutex_postfork_child(&arenas_lock); 1689234370Sjasone} 1690234370Sjasone 1691234370Sjasone/******************************************************************************/ 1692234370Sjasone/* 1693234370Sjasone * The following functions are used for TLS allocation/deallocation in static 1694234370Sjasone * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() 1695234370Sjasone * is that these avoid accessing TLS variables. 1696234370Sjasone */ 1697234370Sjasone 1698234370Sjasonestatic void * 1699234370Sjasonea0alloc(size_t size, bool zero) 1700234370Sjasone{ 1701234370Sjasone 1702234370Sjasone if (malloc_init()) 1703234370Sjasone return (NULL); 1704234370Sjasone 1705234370Sjasone if (size == 0) 1706234370Sjasone size = 1; 1707234370Sjasone 1708234370Sjasone if (size <= arena_maxclass) 1709234370Sjasone return (arena_malloc(arenas[0], size, zero, false)); 1710234370Sjasone else 1711234370Sjasone return (huge_malloc(size, zero)); 1712234370Sjasone} 1713234370Sjasone 1714234370Sjasonevoid * 1715234370Sjasonea0malloc(size_t size) 1716234370Sjasone{ 1717234370Sjasone 1718234370Sjasone return (a0alloc(size, false)); 1719234370Sjasone} 1720234370Sjasone 1721234370Sjasonevoid * 1722234370Sjasonea0calloc(size_t num, size_t size) 1723234370Sjasone{ 1724234370Sjasone 1725234370Sjasone return (a0alloc(num * size, true)); 1726234370Sjasone} 1727234370Sjasone 1728234370Sjasonevoid 1729234370Sjasonea0free(void *ptr) 1730234370Sjasone{ 1731234370Sjasone arena_chunk_t *chunk; 1732234370Sjasone 1733234370Sjasone if (ptr == NULL) 1734234370Sjasone return; 1735234370Sjasone 1736234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1737234370Sjasone if (chunk != ptr) 1738234370Sjasone arena_dalloc(chunk->arena, chunk, ptr, false); 1739234370Sjasone else 1740234370Sjasone huge_dalloc(ptr, true); 1741234370Sjasone} 1742234370Sjasone 1743234370Sjasone/******************************************************************************/ 1744