jemalloc.c revision 234543
1234370Sjasone#define JEMALLOC_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone 4234370Sjasone/******************************************************************************/ 5234370Sjasone/* Data. */ 6234370Sjasone 7234370Sjasonemalloc_tsd_data(, arenas, arena_t *, NULL) 8234370Sjasonemalloc_tsd_data(, thread_allocated, thread_allocated_t, 9234370Sjasone THREAD_ALLOCATED_INITIALIZER) 10234370Sjasone 11234370Sjasoneconst char *__malloc_options_1_0; 12234370Sjasone__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 13234370Sjasone 14234370Sjasone/* Runtime configuration options. */ 15234370Sjasoneconst char *je_malloc_conf JEMALLOC_ATTR(visibility("default")); 16234370Sjasone#ifdef JEMALLOC_DEBUG 17234370Sjasonebool opt_abort = true; 18234370Sjasone# ifdef JEMALLOC_FILL 19234370Sjasonebool opt_junk = true; 20234370Sjasone# else 21234370Sjasonebool opt_junk = false; 22234370Sjasone# endif 23234370Sjasone#else 24234370Sjasonebool opt_abort = false; 25234370Sjasonebool opt_junk = false; 26234370Sjasone#endif 27234370Sjasonesize_t opt_quarantine = ZU(0); 28234370Sjasonebool opt_redzone = false; 29234370Sjasonebool opt_utrace = false; 30234370Sjasonebool opt_valgrind = false; 31234370Sjasonebool opt_xmalloc = false; 32234370Sjasonebool opt_zero = false; 33234370Sjasonesize_t opt_narenas = 0; 34234370Sjasone 35234370Sjasoneunsigned ncpus; 36234370Sjasone 37234370Sjasonemalloc_mutex_t arenas_lock; 38234370Sjasonearena_t **arenas; 39234370Sjasoneunsigned narenas; 40234370Sjasone 41234370Sjasone/* Set to true once the allocator has been initialized. */ 42234370Sjasonestatic bool malloc_initialized = false; 43234370Sjasone 44234370Sjasone#ifdef JEMALLOC_THREADED_INIT 45234370Sjasone/* Used to let the initializing thread recursively allocate. */ 46234370Sjasone# define NO_INITIALIZER ((unsigned long)0) 47234370Sjasone# define INITIALIZER pthread_self() 48234370Sjasone# define IS_INITIALIZER (malloc_initializer == pthread_self()) 49234370Sjasonestatic pthread_t malloc_initializer = NO_INITIALIZER; 50234370Sjasone#else 51234370Sjasone# define NO_INITIALIZER false 52234370Sjasone# define INITIALIZER true 53234370Sjasone# define IS_INITIALIZER malloc_initializer 54234370Sjasonestatic bool malloc_initializer = NO_INITIALIZER; 55234370Sjasone#endif 56234370Sjasone 57234370Sjasone/* Used to avoid initialization races. */ 58234370Sjasonestatic malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 59234370Sjasone 60234370Sjasonetypedef struct { 61234370Sjasone void *p; /* Input pointer (as in realloc(p, s)). */ 62234370Sjasone size_t s; /* Request size. */ 63234370Sjasone void *r; /* Result pointer. */ 64234370Sjasone} malloc_utrace_t; 65234370Sjasone 66234370Sjasone#ifdef JEMALLOC_UTRACE 67234370Sjasone# define UTRACE(a, b, c) do { \ 68234370Sjasone if (opt_utrace) { \ 69234370Sjasone malloc_utrace_t ut; \ 70234370Sjasone ut.p = (a); \ 71234370Sjasone ut.s = (b); \ 72234370Sjasone ut.r = (c); \ 73234370Sjasone utrace(&ut, sizeof(ut)); \ 74234370Sjasone } \ 75234370Sjasone} while (0) 76234370Sjasone#else 77234370Sjasone# define UTRACE(a, b, c) 78234370Sjasone#endif 79234370Sjasone 80234370Sjasone/******************************************************************************/ 81234370Sjasone/* Function prototypes for non-inline static functions. */ 82234370Sjasone 83234370Sjasonestatic void stats_print_atexit(void); 84234370Sjasonestatic unsigned malloc_ncpus(void); 85234370Sjasonestatic bool malloc_conf_next(char const **opts_p, char const **k_p, 86234370Sjasone size_t *klen_p, char const **v_p, size_t *vlen_p); 87234370Sjasonestatic void malloc_conf_error(const char *msg, const char *k, size_t klen, 88234370Sjasone const char *v, size_t vlen); 89234370Sjasonestatic void malloc_conf_init(void); 90234370Sjasonestatic bool malloc_init_hard(void); 91234370Sjasonestatic int imemalign(void **memptr, size_t alignment, size_t size, 92234370Sjasone size_t min_alignment); 93234370Sjasone 94234370Sjasone/******************************************************************************/ 95234370Sjasone/* 96234370Sjasone * Begin miscellaneous support functions. 97234370Sjasone */ 98234370Sjasone 99234370Sjasone/* Create a new arena and insert it into the arenas array at index ind. */ 100234370Sjasonearena_t * 101234370Sjasonearenas_extend(unsigned ind) 102234370Sjasone{ 103234370Sjasone arena_t *ret; 104234370Sjasone 105234370Sjasone ret = (arena_t *)base_alloc(sizeof(arena_t)); 106234370Sjasone if (ret != NULL && arena_new(ret, ind) == false) { 107234370Sjasone arenas[ind] = ret; 108234370Sjasone return (ret); 109234370Sjasone } 110234370Sjasone /* Only reached if there is an OOM error. */ 111234370Sjasone 112234370Sjasone /* 113234370Sjasone * OOM here is quite inconvenient to propagate, since dealing with it 114234370Sjasone * would require a check for failure in the fast path. Instead, punt 115234370Sjasone * by using arenas[0]. In practice, this is an extremely unlikely 116234370Sjasone * failure. 117234370Sjasone */ 118234370Sjasone malloc_write("<jemalloc>: Error initializing arena\n"); 119234370Sjasone if (opt_abort) 120234370Sjasone abort(); 121234370Sjasone 122234370Sjasone return (arenas[0]); 123234370Sjasone} 124234370Sjasone 125234370Sjasone/* Slow path, called only by choose_arena(). */ 126234370Sjasonearena_t * 127234370Sjasonechoose_arena_hard(void) 128234370Sjasone{ 129234370Sjasone arena_t *ret; 130234370Sjasone 131234370Sjasone if (narenas > 1) { 132234370Sjasone unsigned i, choose, first_null; 133234370Sjasone 134234370Sjasone choose = 0; 135234370Sjasone first_null = narenas; 136234370Sjasone malloc_mutex_lock(&arenas_lock); 137234370Sjasone assert(arenas[0] != NULL); 138234370Sjasone for (i = 1; i < narenas; i++) { 139234370Sjasone if (arenas[i] != NULL) { 140234370Sjasone /* 141234370Sjasone * Choose the first arena that has the lowest 142234370Sjasone * number of threads assigned to it. 143234370Sjasone */ 144234370Sjasone if (arenas[i]->nthreads < 145234370Sjasone arenas[choose]->nthreads) 146234370Sjasone choose = i; 147234370Sjasone } else if (first_null == narenas) { 148234370Sjasone /* 149234370Sjasone * Record the index of the first uninitialized 150234370Sjasone * arena, in case all extant arenas are in use. 151234370Sjasone * 152234370Sjasone * NB: It is possible for there to be 153234370Sjasone * discontinuities in terms of initialized 154234370Sjasone * versus uninitialized arenas, due to the 155234370Sjasone * "thread.arena" mallctl. 156234370Sjasone */ 157234370Sjasone first_null = i; 158234370Sjasone } 159234370Sjasone } 160234370Sjasone 161234370Sjasone if (arenas[choose]->nthreads == 0 || first_null == narenas) { 162234370Sjasone /* 163234370Sjasone * Use an unloaded arena, or the least loaded arena if 164234370Sjasone * all arenas are already initialized. 165234370Sjasone */ 166234370Sjasone ret = arenas[choose]; 167234370Sjasone } else { 168234370Sjasone /* Initialize a new arena. */ 169234370Sjasone ret = arenas_extend(first_null); 170234370Sjasone } 171234370Sjasone ret->nthreads++; 172234370Sjasone malloc_mutex_unlock(&arenas_lock); 173234370Sjasone } else { 174234370Sjasone ret = arenas[0]; 175234370Sjasone malloc_mutex_lock(&arenas_lock); 176234370Sjasone ret->nthreads++; 177234370Sjasone malloc_mutex_unlock(&arenas_lock); 178234370Sjasone } 179234370Sjasone 180234370Sjasone arenas_tsd_set(&ret); 181234370Sjasone 182234370Sjasone return (ret); 183234370Sjasone} 184234370Sjasone 185234370Sjasonestatic void 186234370Sjasonestats_print_atexit(void) 187234370Sjasone{ 188234370Sjasone 189234370Sjasone if (config_tcache && config_stats) { 190234370Sjasone unsigned i; 191234370Sjasone 192234370Sjasone /* 193234370Sjasone * Merge stats from extant threads. This is racy, since 194234370Sjasone * individual threads do not lock when recording tcache stats 195234370Sjasone * events. As a consequence, the final stats may be slightly 196234370Sjasone * out of date by the time they are reported, if other threads 197234370Sjasone * continue to allocate. 198234370Sjasone */ 199234370Sjasone for (i = 0; i < narenas; i++) { 200234370Sjasone arena_t *arena = arenas[i]; 201234370Sjasone if (arena != NULL) { 202234370Sjasone tcache_t *tcache; 203234370Sjasone 204234370Sjasone /* 205234370Sjasone * tcache_stats_merge() locks bins, so if any 206234370Sjasone * code is introduced that acquires both arena 207234370Sjasone * and bin locks in the opposite order, 208234370Sjasone * deadlocks may result. 209234370Sjasone */ 210234370Sjasone malloc_mutex_lock(&arena->lock); 211234370Sjasone ql_foreach(tcache, &arena->tcache_ql, link) { 212234370Sjasone tcache_stats_merge(tcache, arena); 213234370Sjasone } 214234370Sjasone malloc_mutex_unlock(&arena->lock); 215234370Sjasone } 216234370Sjasone } 217234370Sjasone } 218234370Sjasone je_malloc_stats_print(NULL, NULL, NULL); 219234370Sjasone} 220234370Sjasone 221234370Sjasone/* 222234370Sjasone * End miscellaneous support functions. 223234370Sjasone */ 224234370Sjasone/******************************************************************************/ 225234370Sjasone/* 226234370Sjasone * Begin initialization functions. 227234370Sjasone */ 228234370Sjasone 229234370Sjasonestatic unsigned 230234370Sjasonemalloc_ncpus(void) 231234370Sjasone{ 232234370Sjasone unsigned ret; 233234370Sjasone long result; 234234370Sjasone 235234370Sjasone result = sysconf(_SC_NPROCESSORS_ONLN); 236234370Sjasone if (result == -1) { 237234370Sjasone /* Error. */ 238234370Sjasone ret = 1; 239234370Sjasone } 240234370Sjasone ret = (unsigned)result; 241234370Sjasone 242234370Sjasone return (ret); 243234370Sjasone} 244234370Sjasone 245234370Sjasonevoid 246234370Sjasonearenas_cleanup(void *arg) 247234370Sjasone{ 248234370Sjasone arena_t *arena = *(arena_t **)arg; 249234370Sjasone 250234370Sjasone malloc_mutex_lock(&arenas_lock); 251234370Sjasone arena->nthreads--; 252234370Sjasone malloc_mutex_unlock(&arenas_lock); 253234370Sjasone} 254234370Sjasone 255234370Sjasonestatic inline bool 256234370Sjasonemalloc_init(void) 257234370Sjasone{ 258234370Sjasone 259234370Sjasone if (malloc_initialized == false) 260234370Sjasone return (malloc_init_hard()); 261234370Sjasone 262234370Sjasone return (false); 263234370Sjasone} 264234370Sjasone 265234370Sjasonestatic bool 266234370Sjasonemalloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 267234370Sjasone char const **v_p, size_t *vlen_p) 268234370Sjasone{ 269234370Sjasone bool accept; 270234370Sjasone const char *opts = *opts_p; 271234370Sjasone 272234370Sjasone *k_p = opts; 273234370Sjasone 274234370Sjasone for (accept = false; accept == false;) { 275234370Sjasone switch (*opts) { 276234370Sjasone case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 277234370Sjasone case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 278234370Sjasone case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 279234370Sjasone case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 280234370Sjasone case 'Y': case 'Z': 281234370Sjasone case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 282234370Sjasone case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 283234370Sjasone case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 284234370Sjasone case 's': case 't': case 'u': case 'v': case 'w': case 'x': 285234370Sjasone case 'y': case 'z': 286234370Sjasone case '0': case '1': case '2': case '3': case '4': case '5': 287234370Sjasone case '6': case '7': case '8': case '9': 288234370Sjasone case '_': 289234370Sjasone opts++; 290234370Sjasone break; 291234370Sjasone case ':': 292234370Sjasone opts++; 293234370Sjasone *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 294234370Sjasone *v_p = opts; 295234370Sjasone accept = true; 296234370Sjasone break; 297234370Sjasone case '\0': 298234370Sjasone if (opts != *opts_p) { 299234370Sjasone malloc_write("<jemalloc>: Conf string ends " 300234370Sjasone "with key\n"); 301234370Sjasone } 302234370Sjasone return (true); 303234370Sjasone default: 304234370Sjasone malloc_write("<jemalloc>: Malformed conf string\n"); 305234370Sjasone return (true); 306234370Sjasone } 307234370Sjasone } 308234370Sjasone 309234370Sjasone for (accept = false; accept == false;) { 310234370Sjasone switch (*opts) { 311234370Sjasone case ',': 312234370Sjasone opts++; 313234370Sjasone /* 314234370Sjasone * Look ahead one character here, because the next time 315234370Sjasone * this function is called, it will assume that end of 316234370Sjasone * input has been cleanly reached if no input remains, 317234370Sjasone * but we have optimistically already consumed the 318234370Sjasone * comma if one exists. 319234370Sjasone */ 320234370Sjasone if (*opts == '\0') { 321234370Sjasone malloc_write("<jemalloc>: Conf string ends " 322234370Sjasone "with comma\n"); 323234370Sjasone } 324234370Sjasone *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 325234370Sjasone accept = true; 326234370Sjasone break; 327234370Sjasone case '\0': 328234370Sjasone *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 329234370Sjasone accept = true; 330234370Sjasone break; 331234370Sjasone default: 332234370Sjasone opts++; 333234370Sjasone break; 334234370Sjasone } 335234370Sjasone } 336234370Sjasone 337234370Sjasone *opts_p = opts; 338234370Sjasone return (false); 339234370Sjasone} 340234370Sjasone 341234370Sjasonestatic void 342234370Sjasonemalloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 343234370Sjasone size_t vlen) 344234370Sjasone{ 345234370Sjasone 346234370Sjasone malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 347234370Sjasone (int)vlen, v); 348234370Sjasone} 349234370Sjasone 350234370Sjasonestatic void 351234370Sjasonemalloc_conf_init(void) 352234370Sjasone{ 353234370Sjasone unsigned i; 354234370Sjasone char buf[PATH_MAX + 1]; 355234370Sjasone const char *opts, *k, *v; 356234370Sjasone size_t klen, vlen; 357234370Sjasone 358234370Sjasone for (i = 0; i < 3; i++) { 359234370Sjasone /* Get runtime configuration. */ 360234370Sjasone switch (i) { 361234370Sjasone case 0: 362234370Sjasone if (je_malloc_conf != NULL) { 363234370Sjasone /* 364234370Sjasone * Use options that were compiled into the 365234370Sjasone * program. 366234370Sjasone */ 367234370Sjasone opts = je_malloc_conf; 368234370Sjasone } else { 369234370Sjasone /* No configuration specified. */ 370234370Sjasone buf[0] = '\0'; 371234370Sjasone opts = buf; 372234370Sjasone } 373234370Sjasone break; 374234370Sjasone case 1: { 375234370Sjasone int linklen; 376234370Sjasone const char *linkname = 377234370Sjasone#ifdef JEMALLOC_PREFIX 378234370Sjasone "/etc/"JEMALLOC_PREFIX"malloc.conf" 379234370Sjasone#else 380234370Sjasone "/etc/malloc.conf" 381234370Sjasone#endif 382234370Sjasone ; 383234370Sjasone 384234370Sjasone if ((linklen = readlink(linkname, buf, 385234370Sjasone sizeof(buf) - 1)) != -1) { 386234370Sjasone /* 387234370Sjasone * Use the contents of the "/etc/malloc.conf" 388234370Sjasone * symbolic link's name. 389234370Sjasone */ 390234370Sjasone buf[linklen] = '\0'; 391234370Sjasone opts = buf; 392234370Sjasone } else { 393234370Sjasone /* No configuration specified. */ 394234370Sjasone buf[0] = '\0'; 395234370Sjasone opts = buf; 396234370Sjasone } 397234370Sjasone break; 398234370Sjasone } case 2: { 399234370Sjasone const char *envname = 400234370Sjasone#ifdef JEMALLOC_PREFIX 401234370Sjasone JEMALLOC_CPREFIX"MALLOC_CONF" 402234370Sjasone#else 403234370Sjasone "MALLOC_CONF" 404234370Sjasone#endif 405234370Sjasone ; 406234370Sjasone 407234370Sjasone if (issetugid() == 0 && (opts = getenv(envname)) != 408234370Sjasone NULL) { 409234370Sjasone /* 410234370Sjasone * Do nothing; opts is already initialized to 411234370Sjasone * the value of the MALLOC_CONF environment 412234370Sjasone * variable. 413234370Sjasone */ 414234370Sjasone } else { 415234370Sjasone /* No configuration specified. */ 416234370Sjasone buf[0] = '\0'; 417234370Sjasone opts = buf; 418234370Sjasone } 419234370Sjasone break; 420234370Sjasone } default: 421234370Sjasone /* NOTREACHED */ 422234370Sjasone assert(false); 423234370Sjasone buf[0] = '\0'; 424234370Sjasone opts = buf; 425234370Sjasone } 426234370Sjasone 427234370Sjasone while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 428234370Sjasone &vlen) == false) { 429234370Sjasone#define CONF_HANDLE_BOOL_HIT(o, n, hit) \ 430234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 431234370Sjasone klen) == 0) { \ 432234370Sjasone if (strncmp("true", v, vlen) == 0 && \ 433234370Sjasone vlen == sizeof("true")-1) \ 434234370Sjasone o = true; \ 435234370Sjasone else if (strncmp("false", v, vlen) == \ 436234370Sjasone 0 && vlen == sizeof("false")-1) \ 437234370Sjasone o = false; \ 438234370Sjasone else { \ 439234370Sjasone malloc_conf_error( \ 440234370Sjasone "Invalid conf value", \ 441234370Sjasone k, klen, v, vlen); \ 442234370Sjasone } \ 443234370Sjasone hit = true; \ 444234370Sjasone } else \ 445234370Sjasone hit = false; 446234370Sjasone#define CONF_HANDLE_BOOL(o, n) { \ 447234370Sjasone bool hit; \ 448234370Sjasone CONF_HANDLE_BOOL_HIT(o, n, hit); \ 449234370Sjasone if (hit) \ 450234370Sjasone continue; \ 451234370Sjasone} 452234370Sjasone#define CONF_HANDLE_SIZE_T(o, n, min, max) \ 453234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 454234370Sjasone klen) == 0) { \ 455234370Sjasone uintmax_t um; \ 456234370Sjasone char *end; \ 457234370Sjasone \ 458234370Sjasone errno = 0; \ 459234370Sjasone um = malloc_strtoumax(v, &end, 0); \ 460234370Sjasone if (errno != 0 || (uintptr_t)end - \ 461234370Sjasone (uintptr_t)v != vlen) { \ 462234370Sjasone malloc_conf_error( \ 463234370Sjasone "Invalid conf value", \ 464234370Sjasone k, klen, v, vlen); \ 465234370Sjasone } else if (um < min || um > max) { \ 466234370Sjasone malloc_conf_error( \ 467234370Sjasone "Out-of-range conf value", \ 468234370Sjasone k, klen, v, vlen); \ 469234370Sjasone } else \ 470234370Sjasone o = um; \ 471234370Sjasone continue; \ 472234370Sjasone } 473234370Sjasone#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 474234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 475234370Sjasone klen) == 0) { \ 476234370Sjasone long l; \ 477234370Sjasone char *end; \ 478234370Sjasone \ 479234370Sjasone errno = 0; \ 480234370Sjasone l = strtol(v, &end, 0); \ 481234370Sjasone if (errno != 0 || (uintptr_t)end - \ 482234370Sjasone (uintptr_t)v != vlen) { \ 483234370Sjasone malloc_conf_error( \ 484234370Sjasone "Invalid conf value", \ 485234370Sjasone k, klen, v, vlen); \ 486234370Sjasone } else if (l < (ssize_t)min || l > \ 487234370Sjasone (ssize_t)max) { \ 488234370Sjasone malloc_conf_error( \ 489234370Sjasone "Out-of-range conf value", \ 490234370Sjasone k, klen, v, vlen); \ 491234370Sjasone } else \ 492234370Sjasone o = l; \ 493234370Sjasone continue; \ 494234370Sjasone } 495234370Sjasone#define CONF_HANDLE_CHAR_P(o, n, d) \ 496234543Sjasone if (sizeof(n)-1 == klen && strncmp(n, k, \ 497234370Sjasone klen) == 0) { \ 498234370Sjasone size_t cpylen = (vlen <= \ 499234370Sjasone sizeof(o)-1) ? vlen : \ 500234370Sjasone sizeof(o)-1; \ 501234370Sjasone strncpy(o, v, cpylen); \ 502234370Sjasone o[cpylen] = '\0'; \ 503234370Sjasone continue; \ 504234370Sjasone } 505234370Sjasone 506234543Sjasone CONF_HANDLE_BOOL(opt_abort, "abort") 507234370Sjasone /* 508234370Sjasone * Chunks always require at least one header page, plus 509234370Sjasone * one data page in the absence of redzones, or three 510234370Sjasone * pages in the presence of redzones. In order to 511234370Sjasone * simplify options processing, fix the limit based on 512234370Sjasone * config_fill. 513234370Sjasone */ 514234543Sjasone CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 515234370Sjasone (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1) 516234543Sjasone CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 517234543Sjasone SIZE_T_MAX) 518234543Sjasone CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 519234370Sjasone -1, (sizeof(size_t) << 3) - 1) 520234543Sjasone CONF_HANDLE_BOOL(opt_stats_print, "stats_print") 521234370Sjasone if (config_fill) { 522234543Sjasone CONF_HANDLE_BOOL(opt_junk, "junk") 523234543Sjasone CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 524234370Sjasone 0, SIZE_T_MAX) 525234543Sjasone CONF_HANDLE_BOOL(opt_redzone, "redzone") 526234543Sjasone CONF_HANDLE_BOOL(opt_zero, "zero") 527234370Sjasone } 528234370Sjasone if (config_utrace) { 529234543Sjasone CONF_HANDLE_BOOL(opt_utrace, "utrace") 530234370Sjasone } 531234370Sjasone if (config_valgrind) { 532234370Sjasone bool hit; 533234370Sjasone CONF_HANDLE_BOOL_HIT(opt_valgrind, 534234543Sjasone "valgrind", hit) 535234370Sjasone if (config_fill && opt_valgrind && hit) { 536234370Sjasone opt_junk = false; 537234370Sjasone opt_zero = false; 538234370Sjasone if (opt_quarantine == 0) { 539234370Sjasone opt_quarantine = 540234370Sjasone JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 541234370Sjasone } 542234370Sjasone opt_redzone = true; 543234370Sjasone } 544234370Sjasone if (hit) 545234370Sjasone continue; 546234370Sjasone } 547234370Sjasone if (config_xmalloc) { 548234543Sjasone CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") 549234370Sjasone } 550234370Sjasone if (config_tcache) { 551234543Sjasone CONF_HANDLE_BOOL(opt_tcache, "tcache") 552234370Sjasone CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 553234543Sjasone "lg_tcache_max", -1, 554234370Sjasone (sizeof(size_t) << 3) - 1) 555234370Sjasone } 556234370Sjasone if (config_prof) { 557234543Sjasone CONF_HANDLE_BOOL(opt_prof, "prof") 558234543Sjasone CONF_HANDLE_CHAR_P(opt_prof_prefix, 559234543Sjasone "prof_prefix", "jeprof") 560234543Sjasone CONF_HANDLE_BOOL(opt_prof_active, "prof_active") 561234370Sjasone CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, 562234543Sjasone "lg_prof_sample", 0, 563234370Sjasone (sizeof(uint64_t) << 3) - 1) 564234543Sjasone CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") 565234370Sjasone CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 566234543Sjasone "lg_prof_interval", -1, 567234370Sjasone (sizeof(uint64_t) << 3) - 1) 568234543Sjasone CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") 569234543Sjasone CONF_HANDLE_BOOL(opt_prof_final, "prof_final") 570234543Sjasone CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") 571234370Sjasone } 572234370Sjasone malloc_conf_error("Invalid conf pair", k, klen, v, 573234370Sjasone vlen); 574234370Sjasone#undef CONF_HANDLE_BOOL 575234370Sjasone#undef CONF_HANDLE_SIZE_T 576234370Sjasone#undef CONF_HANDLE_SSIZE_T 577234370Sjasone#undef CONF_HANDLE_CHAR_P 578234370Sjasone } 579234370Sjasone } 580234370Sjasone} 581234370Sjasone 582234370Sjasonestatic bool 583234370Sjasonemalloc_init_hard(void) 584234370Sjasone{ 585234370Sjasone arena_t *init_arenas[1]; 586234370Sjasone 587234370Sjasone malloc_mutex_lock(&init_lock); 588234370Sjasone if (malloc_initialized || IS_INITIALIZER) { 589234370Sjasone /* 590234370Sjasone * Another thread initialized the allocator before this one 591234370Sjasone * acquired init_lock, or this thread is the initializing 592234370Sjasone * thread, and it is recursively allocating. 593234370Sjasone */ 594234370Sjasone malloc_mutex_unlock(&init_lock); 595234370Sjasone return (false); 596234370Sjasone } 597234370Sjasone#ifdef JEMALLOC_THREADED_INIT 598234370Sjasone if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { 599234370Sjasone /* Busy-wait until the initializing thread completes. */ 600234370Sjasone do { 601234370Sjasone malloc_mutex_unlock(&init_lock); 602234370Sjasone CPU_SPINWAIT; 603234370Sjasone malloc_mutex_lock(&init_lock); 604234370Sjasone } while (malloc_initialized == false); 605234370Sjasone malloc_mutex_unlock(&init_lock); 606234370Sjasone return (false); 607234370Sjasone } 608234370Sjasone#endif 609234370Sjasone malloc_initializer = INITIALIZER; 610234370Sjasone 611234370Sjasone malloc_tsd_boot(); 612234370Sjasone if (config_prof) 613234370Sjasone prof_boot0(); 614234370Sjasone 615234370Sjasone malloc_conf_init(); 616234370Sjasone 617234370Sjasone#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE)) 618234370Sjasone /* Register fork handlers. */ 619234370Sjasone if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 620234370Sjasone jemalloc_postfork_child) != 0) { 621234370Sjasone malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 622234370Sjasone if (opt_abort) 623234370Sjasone abort(); 624234370Sjasone } 625234370Sjasone#endif 626234370Sjasone 627234370Sjasone if (opt_stats_print) { 628234370Sjasone /* Print statistics at exit. */ 629234370Sjasone if (atexit(stats_print_atexit) != 0) { 630234370Sjasone malloc_write("<jemalloc>: Error in atexit()\n"); 631234370Sjasone if (opt_abort) 632234370Sjasone abort(); 633234370Sjasone } 634234370Sjasone } 635234370Sjasone 636234370Sjasone if (base_boot()) { 637234370Sjasone malloc_mutex_unlock(&init_lock); 638234370Sjasone return (true); 639234370Sjasone } 640234370Sjasone 641234370Sjasone if (chunk_boot0()) { 642234370Sjasone malloc_mutex_unlock(&init_lock); 643234370Sjasone return (true); 644234370Sjasone } 645234370Sjasone 646234370Sjasone if (ctl_boot()) { 647234370Sjasone malloc_mutex_unlock(&init_lock); 648234370Sjasone return (true); 649234370Sjasone } 650234370Sjasone 651234370Sjasone if (config_prof) 652234370Sjasone prof_boot1(); 653234370Sjasone 654234370Sjasone arena_boot(); 655234370Sjasone 656234370Sjasone if (config_tcache && tcache_boot0()) { 657234370Sjasone malloc_mutex_unlock(&init_lock); 658234370Sjasone return (true); 659234370Sjasone } 660234370Sjasone 661234370Sjasone if (huge_boot()) { 662234370Sjasone malloc_mutex_unlock(&init_lock); 663234370Sjasone return (true); 664234370Sjasone } 665234370Sjasone 666234370Sjasone if (malloc_mutex_init(&arenas_lock)) 667234370Sjasone return (true); 668234370Sjasone 669234370Sjasone /* 670234370Sjasone * Create enough scaffolding to allow recursive allocation in 671234370Sjasone * malloc_ncpus(). 672234370Sjasone */ 673234370Sjasone narenas = 1; 674234370Sjasone arenas = init_arenas; 675234370Sjasone memset(arenas, 0, sizeof(arena_t *) * narenas); 676234370Sjasone 677234370Sjasone /* 678234370Sjasone * Initialize one arena here. The rest are lazily created in 679234370Sjasone * choose_arena_hard(). 680234370Sjasone */ 681234370Sjasone arenas_extend(0); 682234370Sjasone if (arenas[0] == NULL) { 683234370Sjasone malloc_mutex_unlock(&init_lock); 684234370Sjasone return (true); 685234370Sjasone } 686234370Sjasone 687234370Sjasone /* Initialize allocation counters before any allocations can occur. */ 688234370Sjasone if (config_stats && thread_allocated_tsd_boot()) { 689234370Sjasone malloc_mutex_unlock(&init_lock); 690234370Sjasone return (true); 691234370Sjasone } 692234370Sjasone 693234370Sjasone if (arenas_tsd_boot()) { 694234370Sjasone malloc_mutex_unlock(&init_lock); 695234370Sjasone return (true); 696234370Sjasone } 697234370Sjasone 698234370Sjasone if (config_tcache && tcache_boot1()) { 699234370Sjasone malloc_mutex_unlock(&init_lock); 700234370Sjasone return (true); 701234370Sjasone } 702234370Sjasone 703234370Sjasone if (config_fill && quarantine_boot()) { 704234370Sjasone malloc_mutex_unlock(&init_lock); 705234370Sjasone return (true); 706234370Sjasone } 707234370Sjasone 708234370Sjasone if (config_prof && prof_boot2()) { 709234370Sjasone malloc_mutex_unlock(&init_lock); 710234370Sjasone return (true); 711234370Sjasone } 712234370Sjasone 713234370Sjasone /* Get number of CPUs. */ 714234370Sjasone malloc_mutex_unlock(&init_lock); 715234370Sjasone ncpus = malloc_ncpus(); 716234370Sjasone malloc_mutex_lock(&init_lock); 717234370Sjasone 718234370Sjasone if (chunk_boot1()) { 719234370Sjasone malloc_mutex_unlock(&init_lock); 720234370Sjasone return (true); 721234370Sjasone } 722234370Sjasone 723234370Sjasone if (mutex_boot()) { 724234370Sjasone malloc_mutex_unlock(&init_lock); 725234370Sjasone return (true); 726234370Sjasone } 727234370Sjasone 728234370Sjasone if (opt_narenas == 0) { 729234370Sjasone /* 730234370Sjasone * For SMP systems, create more than one arena per CPU by 731234370Sjasone * default. 732234370Sjasone */ 733234370Sjasone if (ncpus > 1) 734234370Sjasone opt_narenas = ncpus << 2; 735234370Sjasone else 736234370Sjasone opt_narenas = 1; 737234370Sjasone } 738234370Sjasone narenas = opt_narenas; 739234370Sjasone /* 740234370Sjasone * Make sure that the arenas array can be allocated. In practice, this 741234370Sjasone * limit is enough to allow the allocator to function, but the ctl 742234370Sjasone * machinery will fail to allocate memory at far lower limits. 743234370Sjasone */ 744234370Sjasone if (narenas > chunksize / sizeof(arena_t *)) { 745234370Sjasone narenas = chunksize / sizeof(arena_t *); 746234370Sjasone malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 747234370Sjasone narenas); 748234370Sjasone } 749234370Sjasone 750234370Sjasone /* Allocate and initialize arenas. */ 751234370Sjasone arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); 752234370Sjasone if (arenas == NULL) { 753234370Sjasone malloc_mutex_unlock(&init_lock); 754234370Sjasone return (true); 755234370Sjasone } 756234370Sjasone /* 757234370Sjasone * Zero the array. In practice, this should always be pre-zeroed, 758234370Sjasone * since it was just mmap()ed, but let's be sure. 759234370Sjasone */ 760234370Sjasone memset(arenas, 0, sizeof(arena_t *) * narenas); 761234370Sjasone /* Copy the pointer to the one arena that was already initialized. */ 762234370Sjasone arenas[0] = init_arenas[0]; 763234370Sjasone 764234370Sjasone malloc_initialized = true; 765234370Sjasone malloc_mutex_unlock(&init_lock); 766234370Sjasone return (false); 767234370Sjasone} 768234370Sjasone 769234370Sjasone/* 770234370Sjasone * End initialization functions. 771234370Sjasone */ 772234370Sjasone/******************************************************************************/ 773234370Sjasone/* 774234370Sjasone * Begin malloc(3)-compatible functions. 775234370Sjasone */ 776234370Sjasone 777234370SjasoneJEMALLOC_ATTR(malloc) 778234370SjasoneJEMALLOC_ATTR(visibility("default")) 779234370Sjasonevoid * 780234370Sjasoneje_malloc(size_t size) 781234370Sjasone{ 782234370Sjasone void *ret; 783234370Sjasone size_t usize; 784234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 785234370Sjasone 786234370Sjasone if (malloc_init()) { 787234370Sjasone ret = NULL; 788234370Sjasone goto label_oom; 789234370Sjasone } 790234370Sjasone 791234370Sjasone if (size == 0) 792234370Sjasone size = 1; 793234370Sjasone 794234370Sjasone if (config_prof && opt_prof) { 795234370Sjasone usize = s2u(size); 796234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 797234370Sjasone if (cnt == NULL) { 798234370Sjasone ret = NULL; 799234370Sjasone goto label_oom; 800234370Sjasone } 801234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 802234370Sjasone SMALL_MAXCLASS) { 803234370Sjasone ret = imalloc(SMALL_MAXCLASS+1); 804234370Sjasone if (ret != NULL) 805234370Sjasone arena_prof_promoted(ret, usize); 806234370Sjasone } else 807234370Sjasone ret = imalloc(size); 808234370Sjasone } else { 809234370Sjasone if (config_stats || (config_valgrind && opt_valgrind)) 810234370Sjasone usize = s2u(size); 811234370Sjasone ret = imalloc(size); 812234370Sjasone } 813234370Sjasone 814234370Sjasonelabel_oom: 815234370Sjasone if (ret == NULL) { 816234370Sjasone if (config_xmalloc && opt_xmalloc) { 817234370Sjasone malloc_write("<jemalloc>: Error in malloc(): " 818234370Sjasone "out of memory\n"); 819234370Sjasone abort(); 820234370Sjasone } 821234370Sjasone errno = ENOMEM; 822234370Sjasone } 823234370Sjasone if (config_prof && opt_prof && ret != NULL) 824234370Sjasone prof_malloc(ret, usize, cnt); 825234370Sjasone if (config_stats && ret != NULL) { 826234370Sjasone assert(usize == isalloc(ret, config_prof)); 827234370Sjasone thread_allocated_tsd_get()->allocated += usize; 828234370Sjasone } 829234370Sjasone UTRACE(0, size, ret); 830234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 831234370Sjasone return (ret); 832234370Sjasone} 833234370Sjasone 834234370SjasoneJEMALLOC_ATTR(nonnull(1)) 835234370Sjasone#ifdef JEMALLOC_PROF 836234370Sjasone/* 837234370Sjasone * Avoid any uncertainty as to how many backtrace frames to ignore in 838234370Sjasone * PROF_ALLOC_PREP(). 839234370Sjasone */ 840234370SjasoneJEMALLOC_ATTR(noinline) 841234370Sjasone#endif 842234370Sjasonestatic int 843234370Sjasoneimemalign(void **memptr, size_t alignment, size_t size, 844234370Sjasone size_t min_alignment) 845234370Sjasone{ 846234370Sjasone int ret; 847234370Sjasone size_t usize; 848234370Sjasone void *result; 849234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 850234370Sjasone 851234370Sjasone assert(min_alignment != 0); 852234370Sjasone 853234370Sjasone if (malloc_init()) 854234370Sjasone result = NULL; 855234370Sjasone else { 856234370Sjasone if (size == 0) 857234370Sjasone size = 1; 858234370Sjasone 859234370Sjasone /* Make sure that alignment is a large enough power of 2. */ 860234370Sjasone if (((alignment - 1) & alignment) != 0 861234370Sjasone || (alignment < min_alignment)) { 862234370Sjasone if (config_xmalloc && opt_xmalloc) { 863234370Sjasone malloc_write("<jemalloc>: Error allocating " 864234370Sjasone "aligned memory: invalid alignment\n"); 865234370Sjasone abort(); 866234370Sjasone } 867234370Sjasone result = NULL; 868234370Sjasone ret = EINVAL; 869234370Sjasone goto label_return; 870234370Sjasone } 871234370Sjasone 872234370Sjasone usize = sa2u(size, alignment); 873234370Sjasone if (usize == 0) { 874234370Sjasone result = NULL; 875234370Sjasone ret = ENOMEM; 876234370Sjasone goto label_return; 877234370Sjasone } 878234370Sjasone 879234370Sjasone if (config_prof && opt_prof) { 880234370Sjasone PROF_ALLOC_PREP(2, usize, cnt); 881234370Sjasone if (cnt == NULL) { 882234370Sjasone result = NULL; 883234370Sjasone ret = EINVAL; 884234370Sjasone } else { 885234370Sjasone if (prof_promote && (uintptr_t)cnt != 886234370Sjasone (uintptr_t)1U && usize <= SMALL_MAXCLASS) { 887234370Sjasone assert(sa2u(SMALL_MAXCLASS+1, 888234370Sjasone alignment) != 0); 889234370Sjasone result = ipalloc(sa2u(SMALL_MAXCLASS+1, 890234370Sjasone alignment), alignment, false); 891234370Sjasone if (result != NULL) { 892234370Sjasone arena_prof_promoted(result, 893234370Sjasone usize); 894234370Sjasone } 895234370Sjasone } else { 896234370Sjasone result = ipalloc(usize, alignment, 897234370Sjasone false); 898234370Sjasone } 899234370Sjasone } 900234370Sjasone } else 901234370Sjasone result = ipalloc(usize, alignment, false); 902234370Sjasone } 903234370Sjasone 904234370Sjasone if (result == NULL) { 905234370Sjasone if (config_xmalloc && opt_xmalloc) { 906234370Sjasone malloc_write("<jemalloc>: Error allocating aligned " 907234370Sjasone "memory: out of memory\n"); 908234370Sjasone abort(); 909234370Sjasone } 910234370Sjasone ret = ENOMEM; 911234370Sjasone goto label_return; 912234370Sjasone } 913234370Sjasone 914234370Sjasone *memptr = result; 915234370Sjasone ret = 0; 916234370Sjasone 917234370Sjasonelabel_return: 918234370Sjasone if (config_stats && result != NULL) { 919234370Sjasone assert(usize == isalloc(result, config_prof)); 920234370Sjasone thread_allocated_tsd_get()->allocated += usize; 921234370Sjasone } 922234370Sjasone if (config_prof && opt_prof && result != NULL) 923234370Sjasone prof_malloc(result, usize, cnt); 924234370Sjasone UTRACE(0, size, result); 925234370Sjasone return (ret); 926234370Sjasone} 927234370Sjasone 928234370SjasoneJEMALLOC_ATTR(nonnull(1)) 929234370SjasoneJEMALLOC_ATTR(visibility("default")) 930234370Sjasoneint 931234370Sjasoneje_posix_memalign(void **memptr, size_t alignment, size_t size) 932234370Sjasone{ 933234370Sjasone int ret = imemalign(memptr, alignment, size, sizeof(void *)); 934234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 935234370Sjasone config_prof), false); 936234370Sjasone return (ret); 937234370Sjasone} 938234370Sjasone 939234370SjasoneJEMALLOC_ATTR(malloc) 940234370SjasoneJEMALLOC_ATTR(visibility("default")) 941234370Sjasonevoid * 942234370Sjasoneje_aligned_alloc(size_t alignment, size_t size) 943234370Sjasone{ 944234370Sjasone void *ret; 945234370Sjasone int err; 946234370Sjasone 947234370Sjasone if ((err = imemalign(&ret, alignment, size, 1)) != 0) { 948234370Sjasone ret = NULL; 949234370Sjasone errno = err; 950234370Sjasone } 951234370Sjasone JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 952234370Sjasone false); 953234370Sjasone return (ret); 954234370Sjasone} 955234370Sjasone 956234370SjasoneJEMALLOC_ATTR(malloc) 957234370SjasoneJEMALLOC_ATTR(visibility("default")) 958234370Sjasonevoid * 959234370Sjasoneje_calloc(size_t num, size_t size) 960234370Sjasone{ 961234370Sjasone void *ret; 962234370Sjasone size_t num_size; 963234370Sjasone size_t usize; 964234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 965234370Sjasone 966234370Sjasone if (malloc_init()) { 967234370Sjasone num_size = 0; 968234370Sjasone ret = NULL; 969234370Sjasone goto label_return; 970234370Sjasone } 971234370Sjasone 972234370Sjasone num_size = num * size; 973234370Sjasone if (num_size == 0) { 974234370Sjasone if (num == 0 || size == 0) 975234370Sjasone num_size = 1; 976234370Sjasone else { 977234370Sjasone ret = NULL; 978234370Sjasone goto label_return; 979234370Sjasone } 980234370Sjasone /* 981234370Sjasone * Try to avoid division here. We know that it isn't possible to 982234370Sjasone * overflow during multiplication if neither operand uses any of the 983234370Sjasone * most significant half of the bits in a size_t. 984234370Sjasone */ 985234370Sjasone } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 986234370Sjasone && (num_size / size != num)) { 987234370Sjasone /* size_t overflow. */ 988234370Sjasone ret = NULL; 989234370Sjasone goto label_return; 990234370Sjasone } 991234370Sjasone 992234370Sjasone if (config_prof && opt_prof) { 993234370Sjasone usize = s2u(num_size); 994234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 995234370Sjasone if (cnt == NULL) { 996234370Sjasone ret = NULL; 997234370Sjasone goto label_return; 998234370Sjasone } 999234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize 1000234370Sjasone <= SMALL_MAXCLASS) { 1001234370Sjasone ret = icalloc(SMALL_MAXCLASS+1); 1002234370Sjasone if (ret != NULL) 1003234370Sjasone arena_prof_promoted(ret, usize); 1004234370Sjasone } else 1005234370Sjasone ret = icalloc(num_size); 1006234370Sjasone } else { 1007234370Sjasone if (config_stats || (config_valgrind && opt_valgrind)) 1008234370Sjasone usize = s2u(num_size); 1009234370Sjasone ret = icalloc(num_size); 1010234370Sjasone } 1011234370Sjasone 1012234370Sjasonelabel_return: 1013234370Sjasone if (ret == NULL) { 1014234370Sjasone if (config_xmalloc && opt_xmalloc) { 1015234370Sjasone malloc_write("<jemalloc>: Error in calloc(): out of " 1016234370Sjasone "memory\n"); 1017234370Sjasone abort(); 1018234370Sjasone } 1019234370Sjasone errno = ENOMEM; 1020234370Sjasone } 1021234370Sjasone 1022234370Sjasone if (config_prof && opt_prof && ret != NULL) 1023234370Sjasone prof_malloc(ret, usize, cnt); 1024234370Sjasone if (config_stats && ret != NULL) { 1025234370Sjasone assert(usize == isalloc(ret, config_prof)); 1026234370Sjasone thread_allocated_tsd_get()->allocated += usize; 1027234370Sjasone } 1028234370Sjasone UTRACE(0, num_size, ret); 1029234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1030234370Sjasone return (ret); 1031234370Sjasone} 1032234370Sjasone 1033234370SjasoneJEMALLOC_ATTR(visibility("default")) 1034234370Sjasonevoid * 1035234370Sjasoneje_realloc(void *ptr, size_t size) 1036234370Sjasone{ 1037234370Sjasone void *ret; 1038234370Sjasone size_t usize; 1039234370Sjasone size_t old_size = 0; 1040234370Sjasone size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1041234370Sjasone prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 1042234370Sjasone prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); 1043234370Sjasone 1044234370Sjasone if (size == 0) { 1045234370Sjasone if (ptr != NULL) { 1046234370Sjasone /* realloc(ptr, 0) is equivalent to free(p). */ 1047234370Sjasone if (config_prof) { 1048234370Sjasone old_size = isalloc(ptr, true); 1049234370Sjasone if (config_valgrind && opt_valgrind) 1050234370Sjasone old_rzsize = p2rz(ptr); 1051234370Sjasone } else if (config_stats) { 1052234370Sjasone old_size = isalloc(ptr, false); 1053234370Sjasone if (config_valgrind && opt_valgrind) 1054234370Sjasone old_rzsize = u2rz(old_size); 1055234370Sjasone } else if (config_valgrind && opt_valgrind) { 1056234370Sjasone old_size = isalloc(ptr, false); 1057234370Sjasone old_rzsize = u2rz(old_size); 1058234370Sjasone } 1059234370Sjasone if (config_prof && opt_prof) { 1060234370Sjasone old_ctx = prof_ctx_get(ptr); 1061234370Sjasone cnt = NULL; 1062234370Sjasone } 1063234370Sjasone iqalloc(ptr); 1064234370Sjasone ret = NULL; 1065234370Sjasone goto label_return; 1066234370Sjasone } else 1067234370Sjasone size = 1; 1068234370Sjasone } 1069234370Sjasone 1070234370Sjasone if (ptr != NULL) { 1071234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1072234370Sjasone 1073234370Sjasone if (config_prof) { 1074234370Sjasone old_size = isalloc(ptr, true); 1075234370Sjasone if (config_valgrind && opt_valgrind) 1076234370Sjasone old_rzsize = p2rz(ptr); 1077234370Sjasone } else if (config_stats) { 1078234370Sjasone old_size = isalloc(ptr, false); 1079234370Sjasone if (config_valgrind && opt_valgrind) 1080234370Sjasone old_rzsize = u2rz(old_size); 1081234370Sjasone } else if (config_valgrind && opt_valgrind) { 1082234370Sjasone old_size = isalloc(ptr, false); 1083234370Sjasone old_rzsize = u2rz(old_size); 1084234370Sjasone } 1085234370Sjasone if (config_prof && opt_prof) { 1086234370Sjasone usize = s2u(size); 1087234370Sjasone old_ctx = prof_ctx_get(ptr); 1088234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1089234370Sjasone if (cnt == NULL) { 1090234370Sjasone old_ctx = NULL; 1091234370Sjasone ret = NULL; 1092234370Sjasone goto label_oom; 1093234370Sjasone } 1094234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && 1095234370Sjasone usize <= SMALL_MAXCLASS) { 1096234370Sjasone ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0, 1097234370Sjasone false, false); 1098234370Sjasone if (ret != NULL) 1099234370Sjasone arena_prof_promoted(ret, usize); 1100234370Sjasone else 1101234370Sjasone old_ctx = NULL; 1102234370Sjasone } else { 1103234370Sjasone ret = iralloc(ptr, size, 0, 0, false, false); 1104234370Sjasone if (ret == NULL) 1105234370Sjasone old_ctx = NULL; 1106234370Sjasone } 1107234370Sjasone } else { 1108234370Sjasone if (config_stats || (config_valgrind && opt_valgrind)) 1109234370Sjasone usize = s2u(size); 1110234370Sjasone ret = iralloc(ptr, size, 0, 0, false, false); 1111234370Sjasone } 1112234370Sjasone 1113234370Sjasonelabel_oom: 1114234370Sjasone if (ret == NULL) { 1115234370Sjasone if (config_xmalloc && opt_xmalloc) { 1116234370Sjasone malloc_write("<jemalloc>: Error in realloc(): " 1117234370Sjasone "out of memory\n"); 1118234370Sjasone abort(); 1119234370Sjasone } 1120234370Sjasone errno = ENOMEM; 1121234370Sjasone } 1122234370Sjasone } else { 1123234370Sjasone /* realloc(NULL, size) is equivalent to malloc(size). */ 1124234370Sjasone if (config_prof && opt_prof) 1125234370Sjasone old_ctx = NULL; 1126234370Sjasone if (malloc_init()) { 1127234370Sjasone if (config_prof && opt_prof) 1128234370Sjasone cnt = NULL; 1129234370Sjasone ret = NULL; 1130234370Sjasone } else { 1131234370Sjasone if (config_prof && opt_prof) { 1132234370Sjasone usize = s2u(size); 1133234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1134234370Sjasone if (cnt == NULL) 1135234370Sjasone ret = NULL; 1136234370Sjasone else { 1137234370Sjasone if (prof_promote && (uintptr_t)cnt != 1138234370Sjasone (uintptr_t)1U && usize <= 1139234370Sjasone SMALL_MAXCLASS) { 1140234370Sjasone ret = imalloc(SMALL_MAXCLASS+1); 1141234370Sjasone if (ret != NULL) { 1142234370Sjasone arena_prof_promoted(ret, 1143234370Sjasone usize); 1144234370Sjasone } 1145234370Sjasone } else 1146234370Sjasone ret = imalloc(size); 1147234370Sjasone } 1148234370Sjasone } else { 1149234370Sjasone if (config_stats || (config_valgrind && 1150234370Sjasone opt_valgrind)) 1151234370Sjasone usize = s2u(size); 1152234370Sjasone ret = imalloc(size); 1153234370Sjasone } 1154234370Sjasone } 1155234370Sjasone 1156234370Sjasone if (ret == NULL) { 1157234370Sjasone if (config_xmalloc && opt_xmalloc) { 1158234370Sjasone malloc_write("<jemalloc>: Error in realloc(): " 1159234370Sjasone "out of memory\n"); 1160234370Sjasone abort(); 1161234370Sjasone } 1162234370Sjasone errno = ENOMEM; 1163234370Sjasone } 1164234370Sjasone } 1165234370Sjasone 1166234370Sjasonelabel_return: 1167234370Sjasone if (config_prof && opt_prof) 1168234370Sjasone prof_realloc(ret, usize, cnt, old_size, old_ctx); 1169234370Sjasone if (config_stats && ret != NULL) { 1170234370Sjasone thread_allocated_t *ta; 1171234370Sjasone assert(usize == isalloc(ret, config_prof)); 1172234370Sjasone ta = thread_allocated_tsd_get(); 1173234370Sjasone ta->allocated += usize; 1174234370Sjasone ta->deallocated += old_size; 1175234370Sjasone } 1176234370Sjasone UTRACE(ptr, size, ret); 1177234370Sjasone JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); 1178234370Sjasone return (ret); 1179234370Sjasone} 1180234370Sjasone 1181234370SjasoneJEMALLOC_ATTR(visibility("default")) 1182234370Sjasonevoid 1183234370Sjasoneje_free(void *ptr) 1184234370Sjasone{ 1185234370Sjasone 1186234370Sjasone UTRACE(ptr, 0, 0); 1187234370Sjasone if (ptr != NULL) { 1188234370Sjasone size_t usize; 1189234370Sjasone size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1190234370Sjasone 1191234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1192234370Sjasone 1193234370Sjasone if (config_prof && opt_prof) { 1194234370Sjasone usize = isalloc(ptr, config_prof); 1195234370Sjasone prof_free(ptr, usize); 1196234370Sjasone } else if (config_stats || config_valgrind) 1197234370Sjasone usize = isalloc(ptr, config_prof); 1198234370Sjasone if (config_stats) 1199234370Sjasone thread_allocated_tsd_get()->deallocated += usize; 1200234370Sjasone if (config_valgrind && opt_valgrind) 1201234370Sjasone rzsize = p2rz(ptr); 1202234370Sjasone iqalloc(ptr); 1203234370Sjasone JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1204234370Sjasone } 1205234370Sjasone} 1206234370Sjasone 1207234370Sjasone/* 1208234370Sjasone * End malloc(3)-compatible functions. 1209234370Sjasone */ 1210234370Sjasone/******************************************************************************/ 1211234370Sjasone/* 1212234370Sjasone * Begin non-standard override functions. 1213234370Sjasone */ 1214234370Sjasone 1215234370Sjasone#ifdef JEMALLOC_OVERRIDE_MEMALIGN 1216234370SjasoneJEMALLOC_ATTR(malloc) 1217234370SjasoneJEMALLOC_ATTR(visibility("default")) 1218234370Sjasonevoid * 1219234370Sjasoneje_memalign(size_t alignment, size_t size) 1220234370Sjasone{ 1221234370Sjasone void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1222234370Sjasone imemalign(&ret, alignment, size, 1); 1223234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1224234370Sjasone return (ret); 1225234370Sjasone} 1226234370Sjasone#endif 1227234370Sjasone 1228234370Sjasone#ifdef JEMALLOC_OVERRIDE_VALLOC 1229234370SjasoneJEMALLOC_ATTR(malloc) 1230234370SjasoneJEMALLOC_ATTR(visibility("default")) 1231234370Sjasonevoid * 1232234370Sjasoneje_valloc(size_t size) 1233234370Sjasone{ 1234234370Sjasone void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1235234370Sjasone imemalign(&ret, PAGE, size, 1); 1236234370Sjasone JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1237234370Sjasone return (ret); 1238234370Sjasone} 1239234370Sjasone#endif 1240234370Sjasone 1241234370Sjasone/* 1242234370Sjasone * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1243234370Sjasone * #define je_malloc malloc 1244234370Sjasone */ 1245234370Sjasone#define malloc_is_malloc 1 1246234370Sjasone#define is_malloc_(a) malloc_is_ ## a 1247234370Sjasone#define is_malloc(a) is_malloc_(a) 1248234370Sjasone 1249234370Sjasone#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) 1250234370Sjasone/* 1251234370Sjasone * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1252234370Sjasone * to inconsistently reference libc's malloc(3)-compatible functions 1253234370Sjasone * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1254234370Sjasone * 1255234370Sjasone * These definitions interpose hooks in glibc. The functions are actually 1256234370Sjasone * passed an extra argument for the caller return address, which will be 1257234370Sjasone * ignored. 1258234370Sjasone */ 1259234370SjasoneJEMALLOC_ATTR(visibility("default")) 1260234370Sjasonevoid (* const __free_hook)(void *ptr) = je_free; 1261234370Sjasone 1262234370SjasoneJEMALLOC_ATTR(visibility("default")) 1263234370Sjasonevoid *(* const __malloc_hook)(size_t size) = je_malloc; 1264234370Sjasone 1265234370SjasoneJEMALLOC_ATTR(visibility("default")) 1266234370Sjasonevoid *(* const __realloc_hook)(void *ptr, size_t size) = je_realloc; 1267234370Sjasone 1268234370SjasoneJEMALLOC_ATTR(visibility("default")) 1269234370Sjasonevoid *(* const __memalign_hook)(size_t alignment, size_t size) = je_memalign; 1270234370Sjasone#endif 1271234370Sjasone 1272234370Sjasone/* 1273234370Sjasone * End non-standard override functions. 1274234370Sjasone */ 1275234370Sjasone/******************************************************************************/ 1276234370Sjasone/* 1277234370Sjasone * Begin non-standard functions. 1278234370Sjasone */ 1279234370Sjasone 1280234370SjasoneJEMALLOC_ATTR(visibility("default")) 1281234370Sjasonesize_t 1282234370Sjasoneje_malloc_usable_size(const void *ptr) 1283234370Sjasone{ 1284234370Sjasone size_t ret; 1285234370Sjasone 1286234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1287234370Sjasone 1288234370Sjasone if (config_ivsalloc) 1289234370Sjasone ret = ivsalloc(ptr, config_prof); 1290234370Sjasone else 1291234370Sjasone ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 1292234370Sjasone 1293234370Sjasone return (ret); 1294234370Sjasone} 1295234370Sjasone 1296234370SjasoneJEMALLOC_ATTR(visibility("default")) 1297234370Sjasonevoid 1298234370Sjasoneje_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 1299234370Sjasone const char *opts) 1300234370Sjasone{ 1301234370Sjasone 1302234370Sjasone stats_print(write_cb, cbopaque, opts); 1303234370Sjasone} 1304234370Sjasone 1305234370SjasoneJEMALLOC_ATTR(visibility("default")) 1306234370Sjasoneint 1307234370Sjasoneje_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 1308234370Sjasone size_t newlen) 1309234370Sjasone{ 1310234370Sjasone 1311234370Sjasone if (malloc_init()) 1312234370Sjasone return (EAGAIN); 1313234370Sjasone 1314234370Sjasone return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1315234370Sjasone} 1316234370Sjasone 1317234370SjasoneJEMALLOC_ATTR(visibility("default")) 1318234370Sjasoneint 1319234370Sjasoneje_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 1320234370Sjasone{ 1321234370Sjasone 1322234370Sjasone if (malloc_init()) 1323234370Sjasone return (EAGAIN); 1324234370Sjasone 1325234370Sjasone return (ctl_nametomib(name, mibp, miblenp)); 1326234370Sjasone} 1327234370Sjasone 1328234370SjasoneJEMALLOC_ATTR(visibility("default")) 1329234370Sjasoneint 1330234370Sjasoneje_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1331234370Sjasone void *newp, size_t newlen) 1332234370Sjasone{ 1333234370Sjasone 1334234370Sjasone if (malloc_init()) 1335234370Sjasone return (EAGAIN); 1336234370Sjasone 1337234370Sjasone return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1338234370Sjasone} 1339234370Sjasone 1340234370Sjasone/* 1341234370Sjasone * End non-standard functions. 1342234370Sjasone */ 1343234370Sjasone/******************************************************************************/ 1344234370Sjasone/* 1345234370Sjasone * Begin experimental functions. 1346234370Sjasone */ 1347234370Sjasone#ifdef JEMALLOC_EXPERIMENTAL 1348234370Sjasone 1349234370SjasoneJEMALLOC_INLINE void * 1350234370Sjasoneiallocm(size_t usize, size_t alignment, bool zero) 1351234370Sjasone{ 1352234370Sjasone 1353234370Sjasone assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, 1354234370Sjasone alignment))); 1355234370Sjasone 1356234370Sjasone if (alignment != 0) 1357234370Sjasone return (ipalloc(usize, alignment, zero)); 1358234370Sjasone else if (zero) 1359234370Sjasone return (icalloc(usize)); 1360234370Sjasone else 1361234370Sjasone return (imalloc(usize)); 1362234370Sjasone} 1363234370Sjasone 1364234370SjasoneJEMALLOC_ATTR(nonnull(1)) 1365234370SjasoneJEMALLOC_ATTR(visibility("default")) 1366234370Sjasoneint 1367234370Sjasoneje_allocm(void **ptr, size_t *rsize, size_t size, int flags) 1368234370Sjasone{ 1369234370Sjasone void *p; 1370234370Sjasone size_t usize; 1371234370Sjasone size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1372234370Sjasone & (SIZE_T_MAX-1)); 1373234370Sjasone bool zero = flags & ALLOCM_ZERO; 1374234370Sjasone prof_thr_cnt_t *cnt; 1375234370Sjasone 1376234370Sjasone assert(ptr != NULL); 1377234370Sjasone assert(size != 0); 1378234370Sjasone 1379234370Sjasone if (malloc_init()) 1380234370Sjasone goto label_oom; 1381234370Sjasone 1382234370Sjasone usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1383234370Sjasone if (usize == 0) 1384234370Sjasone goto label_oom; 1385234370Sjasone 1386234370Sjasone if (config_prof && opt_prof) { 1387234370Sjasone PROF_ALLOC_PREP(1, usize, cnt); 1388234370Sjasone if (cnt == NULL) 1389234370Sjasone goto label_oom; 1390234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 1391234370Sjasone SMALL_MAXCLASS) { 1392234370Sjasone size_t usize_promoted = (alignment == 0) ? 1393234370Sjasone s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, 1394234370Sjasone alignment); 1395234370Sjasone assert(usize_promoted != 0); 1396234370Sjasone p = iallocm(usize_promoted, alignment, zero); 1397234370Sjasone if (p == NULL) 1398234370Sjasone goto label_oom; 1399234370Sjasone arena_prof_promoted(p, usize); 1400234370Sjasone } else { 1401234370Sjasone p = iallocm(usize, alignment, zero); 1402234370Sjasone if (p == NULL) 1403234370Sjasone goto label_oom; 1404234370Sjasone } 1405234370Sjasone prof_malloc(p, usize, cnt); 1406234370Sjasone } else { 1407234370Sjasone p = iallocm(usize, alignment, zero); 1408234370Sjasone if (p == NULL) 1409234370Sjasone goto label_oom; 1410234370Sjasone } 1411234370Sjasone if (rsize != NULL) 1412234370Sjasone *rsize = usize; 1413234370Sjasone 1414234370Sjasone *ptr = p; 1415234370Sjasone if (config_stats) { 1416234370Sjasone assert(usize == isalloc(p, config_prof)); 1417234370Sjasone thread_allocated_tsd_get()->allocated += usize; 1418234370Sjasone } 1419234370Sjasone UTRACE(0, size, p); 1420234370Sjasone JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); 1421234370Sjasone return (ALLOCM_SUCCESS); 1422234370Sjasonelabel_oom: 1423234370Sjasone if (config_xmalloc && opt_xmalloc) { 1424234370Sjasone malloc_write("<jemalloc>: Error in allocm(): " 1425234370Sjasone "out of memory\n"); 1426234370Sjasone abort(); 1427234370Sjasone } 1428234370Sjasone *ptr = NULL; 1429234370Sjasone UTRACE(0, size, 0); 1430234370Sjasone return (ALLOCM_ERR_OOM); 1431234370Sjasone} 1432234370Sjasone 1433234370SjasoneJEMALLOC_ATTR(nonnull(1)) 1434234370SjasoneJEMALLOC_ATTR(visibility("default")) 1435234370Sjasoneint 1436234370Sjasoneje_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 1437234370Sjasone{ 1438234370Sjasone void *p, *q; 1439234370Sjasone size_t usize; 1440234370Sjasone size_t old_size; 1441234370Sjasone size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1442234370Sjasone size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1443234370Sjasone & (SIZE_T_MAX-1)); 1444234370Sjasone bool zero = flags & ALLOCM_ZERO; 1445234370Sjasone bool no_move = flags & ALLOCM_NO_MOVE; 1446234370Sjasone prof_thr_cnt_t *cnt; 1447234370Sjasone 1448234370Sjasone assert(ptr != NULL); 1449234370Sjasone assert(*ptr != NULL); 1450234370Sjasone assert(size != 0); 1451234370Sjasone assert(SIZE_T_MAX - size >= extra); 1452234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1453234370Sjasone 1454234370Sjasone p = *ptr; 1455234370Sjasone if (config_prof && opt_prof) { 1456234370Sjasone /* 1457234370Sjasone * usize isn't knowable before iralloc() returns when extra is 1458234370Sjasone * non-zero. Therefore, compute its maximum possible value and 1459234370Sjasone * use that in PROF_ALLOC_PREP() to decide whether to capture a 1460234370Sjasone * backtrace. prof_realloc() will use the actual usize to 1461234370Sjasone * decide whether to sample. 1462234370Sjasone */ 1463234370Sjasone size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1464234370Sjasone sa2u(size+extra, alignment); 1465234370Sjasone prof_ctx_t *old_ctx = prof_ctx_get(p); 1466234370Sjasone old_size = isalloc(p, true); 1467234370Sjasone if (config_valgrind && opt_valgrind) 1468234370Sjasone old_rzsize = p2rz(p); 1469234370Sjasone PROF_ALLOC_PREP(1, max_usize, cnt); 1470234370Sjasone if (cnt == NULL) 1471234370Sjasone goto label_oom; 1472234370Sjasone /* 1473234370Sjasone * Use minimum usize to determine whether promotion may happen. 1474234370Sjasone */ 1475234370Sjasone if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U 1476234370Sjasone && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) 1477234370Sjasone <= SMALL_MAXCLASS) { 1478234370Sjasone q = iralloc(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1479234370Sjasone size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 1480234370Sjasone alignment, zero, no_move); 1481234370Sjasone if (q == NULL) 1482234370Sjasone goto label_err; 1483234370Sjasone if (max_usize < PAGE) { 1484234370Sjasone usize = max_usize; 1485234370Sjasone arena_prof_promoted(q, usize); 1486234370Sjasone } else 1487234370Sjasone usize = isalloc(q, config_prof); 1488234370Sjasone } else { 1489234370Sjasone q = iralloc(p, size, extra, alignment, zero, no_move); 1490234370Sjasone if (q == NULL) 1491234370Sjasone goto label_err; 1492234370Sjasone usize = isalloc(q, config_prof); 1493234370Sjasone } 1494234370Sjasone prof_realloc(q, usize, cnt, old_size, old_ctx); 1495234370Sjasone if (rsize != NULL) 1496234370Sjasone *rsize = usize; 1497234370Sjasone } else { 1498234370Sjasone if (config_stats) { 1499234370Sjasone old_size = isalloc(p, false); 1500234370Sjasone if (config_valgrind && opt_valgrind) 1501234370Sjasone old_rzsize = u2rz(old_size); 1502234370Sjasone } else if (config_valgrind && opt_valgrind) { 1503234370Sjasone old_size = isalloc(p, false); 1504234370Sjasone old_rzsize = u2rz(old_size); 1505234370Sjasone } 1506234370Sjasone q = iralloc(p, size, extra, alignment, zero, no_move); 1507234370Sjasone if (q == NULL) 1508234370Sjasone goto label_err; 1509234370Sjasone if (config_stats) 1510234370Sjasone usize = isalloc(q, config_prof); 1511234370Sjasone if (rsize != NULL) { 1512234370Sjasone if (config_stats == false) 1513234370Sjasone usize = isalloc(q, config_prof); 1514234370Sjasone *rsize = usize; 1515234370Sjasone } 1516234370Sjasone } 1517234370Sjasone 1518234370Sjasone *ptr = q; 1519234370Sjasone if (config_stats) { 1520234370Sjasone thread_allocated_t *ta; 1521234370Sjasone ta = thread_allocated_tsd_get(); 1522234370Sjasone ta->allocated += usize; 1523234370Sjasone ta->deallocated += old_size; 1524234370Sjasone } 1525234370Sjasone UTRACE(p, size, q); 1526234370Sjasone JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); 1527234370Sjasone return (ALLOCM_SUCCESS); 1528234370Sjasonelabel_err: 1529234370Sjasone if (no_move) { 1530234370Sjasone UTRACE(p, size, q); 1531234370Sjasone return (ALLOCM_ERR_NOT_MOVED); 1532234370Sjasone } 1533234370Sjasonelabel_oom: 1534234370Sjasone if (config_xmalloc && opt_xmalloc) { 1535234370Sjasone malloc_write("<jemalloc>: Error in rallocm(): " 1536234370Sjasone "out of memory\n"); 1537234370Sjasone abort(); 1538234370Sjasone } 1539234370Sjasone UTRACE(p, size, 0); 1540234370Sjasone return (ALLOCM_ERR_OOM); 1541234370Sjasone} 1542234370Sjasone 1543234370SjasoneJEMALLOC_ATTR(nonnull(1)) 1544234370SjasoneJEMALLOC_ATTR(visibility("default")) 1545234370Sjasoneint 1546234370Sjasoneje_sallocm(const void *ptr, size_t *rsize, int flags) 1547234370Sjasone{ 1548234370Sjasone size_t sz; 1549234370Sjasone 1550234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1551234370Sjasone 1552234370Sjasone if (config_ivsalloc) 1553234370Sjasone sz = ivsalloc(ptr, config_prof); 1554234370Sjasone else { 1555234370Sjasone assert(ptr != NULL); 1556234370Sjasone sz = isalloc(ptr, config_prof); 1557234370Sjasone } 1558234370Sjasone assert(rsize != NULL); 1559234370Sjasone *rsize = sz; 1560234370Sjasone 1561234370Sjasone return (ALLOCM_SUCCESS); 1562234370Sjasone} 1563234370Sjasone 1564234370SjasoneJEMALLOC_ATTR(nonnull(1)) 1565234370SjasoneJEMALLOC_ATTR(visibility("default")) 1566234370Sjasoneint 1567234370Sjasoneje_dallocm(void *ptr, int flags) 1568234370Sjasone{ 1569234370Sjasone size_t usize; 1570234370Sjasone size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1571234370Sjasone 1572234370Sjasone assert(ptr != NULL); 1573234370Sjasone assert(malloc_initialized || IS_INITIALIZER); 1574234370Sjasone 1575234370Sjasone UTRACE(ptr, 0, 0); 1576234370Sjasone if (config_stats || config_valgrind) 1577234370Sjasone usize = isalloc(ptr, config_prof); 1578234370Sjasone if (config_prof && opt_prof) { 1579234370Sjasone if (config_stats == false && config_valgrind == false) 1580234370Sjasone usize = isalloc(ptr, config_prof); 1581234370Sjasone prof_free(ptr, usize); 1582234370Sjasone } 1583234370Sjasone if (config_stats) 1584234370Sjasone thread_allocated_tsd_get()->deallocated += usize; 1585234370Sjasone if (config_valgrind && opt_valgrind) 1586234370Sjasone rzsize = p2rz(ptr); 1587234370Sjasone iqalloc(ptr); 1588234370Sjasone JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1589234370Sjasone 1590234370Sjasone return (ALLOCM_SUCCESS); 1591234370Sjasone} 1592234370Sjasone 1593234370SjasoneJEMALLOC_ATTR(visibility("default")) 1594234370Sjasoneint 1595234370Sjasoneje_nallocm(size_t *rsize, size_t size, int flags) 1596234370Sjasone{ 1597234370Sjasone size_t usize; 1598234370Sjasone size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1599234370Sjasone & (SIZE_T_MAX-1)); 1600234370Sjasone 1601234370Sjasone assert(size != 0); 1602234370Sjasone 1603234370Sjasone if (malloc_init()) 1604234370Sjasone return (ALLOCM_ERR_OOM); 1605234370Sjasone 1606234370Sjasone usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1607234370Sjasone if (usize == 0) 1608234370Sjasone return (ALLOCM_ERR_OOM); 1609234370Sjasone 1610234370Sjasone if (rsize != NULL) 1611234370Sjasone *rsize = usize; 1612234370Sjasone return (ALLOCM_SUCCESS); 1613234370Sjasone} 1614234370Sjasone 1615234370Sjasone#endif 1616234370Sjasone/* 1617234370Sjasone * End experimental functions. 1618234370Sjasone */ 1619234370Sjasone/******************************************************************************/ 1620234370Sjasone/* 1621234370Sjasone * The following functions are used by threading libraries for protection of 1622234370Sjasone * malloc during fork(). 1623234370Sjasone */ 1624234370Sjasone 1625234370Sjasone#ifndef JEMALLOC_MUTEX_INIT_CB 1626234370Sjasonevoid 1627234370Sjasonejemalloc_prefork(void) 1628234370Sjasone#else 1629234543SjasoneJEMALLOC_ATTR(visibility("default")) 1630234370Sjasonevoid 1631234370Sjasone_malloc_prefork(void) 1632234370Sjasone#endif 1633234370Sjasone{ 1634234370Sjasone unsigned i; 1635234370Sjasone 1636234370Sjasone /* Acquire all mutexes in a safe order. */ 1637234370Sjasone malloc_mutex_prefork(&arenas_lock); 1638234370Sjasone for (i = 0; i < narenas; i++) { 1639234370Sjasone if (arenas[i] != NULL) 1640234370Sjasone arena_prefork(arenas[i]); 1641234370Sjasone } 1642234370Sjasone base_prefork(); 1643234370Sjasone huge_prefork(); 1644234370Sjasone chunk_dss_prefork(); 1645234370Sjasone} 1646234370Sjasone 1647234370Sjasone#ifndef JEMALLOC_MUTEX_INIT_CB 1648234370Sjasonevoid 1649234370Sjasonejemalloc_postfork_parent(void) 1650234370Sjasone#else 1651234543SjasoneJEMALLOC_ATTR(visibility("default")) 1652234370Sjasonevoid 1653234370Sjasone_malloc_postfork(void) 1654234370Sjasone#endif 1655234370Sjasone{ 1656234370Sjasone unsigned i; 1657234370Sjasone 1658234370Sjasone /* Release all mutexes, now that fork() has completed. */ 1659234370Sjasone chunk_dss_postfork_parent(); 1660234370Sjasone huge_postfork_parent(); 1661234370Sjasone base_postfork_parent(); 1662234370Sjasone for (i = 0; i < narenas; i++) { 1663234370Sjasone if (arenas[i] != NULL) 1664234370Sjasone arena_postfork_parent(arenas[i]); 1665234370Sjasone } 1666234370Sjasone malloc_mutex_postfork_parent(&arenas_lock); 1667234370Sjasone} 1668234370Sjasone 1669234370Sjasonevoid 1670234370Sjasonejemalloc_postfork_child(void) 1671234370Sjasone{ 1672234370Sjasone unsigned i; 1673234370Sjasone 1674234370Sjasone /* Release all mutexes, now that fork() has completed. */ 1675234370Sjasone chunk_dss_postfork_child(); 1676234370Sjasone huge_postfork_child(); 1677234370Sjasone base_postfork_child(); 1678234370Sjasone for (i = 0; i < narenas; i++) { 1679234370Sjasone if (arenas[i] != NULL) 1680234370Sjasone arena_postfork_child(arenas[i]); 1681234370Sjasone } 1682234370Sjasone malloc_mutex_postfork_child(&arenas_lock); 1683234370Sjasone} 1684234370Sjasone 1685234370Sjasone/******************************************************************************/ 1686234370Sjasone/* 1687234370Sjasone * The following functions are used for TLS allocation/deallocation in static 1688234370Sjasone * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() 1689234370Sjasone * is that these avoid accessing TLS variables. 1690234370Sjasone */ 1691234370Sjasone 1692234370Sjasonestatic void * 1693234370Sjasonea0alloc(size_t size, bool zero) 1694234370Sjasone{ 1695234370Sjasone 1696234370Sjasone if (malloc_init()) 1697234370Sjasone return (NULL); 1698234370Sjasone 1699234370Sjasone if (size == 0) 1700234370Sjasone size = 1; 1701234370Sjasone 1702234370Sjasone if (size <= arena_maxclass) 1703234370Sjasone return (arena_malloc(arenas[0], size, zero, false)); 1704234370Sjasone else 1705234370Sjasone return (huge_malloc(size, zero)); 1706234370Sjasone} 1707234370Sjasone 1708234370Sjasonevoid * 1709234370Sjasonea0malloc(size_t size) 1710234370Sjasone{ 1711234370Sjasone 1712234370Sjasone return (a0alloc(size, false)); 1713234370Sjasone} 1714234370Sjasone 1715234370Sjasonevoid * 1716234370Sjasonea0calloc(size_t num, size_t size) 1717234370Sjasone{ 1718234370Sjasone 1719234370Sjasone return (a0alloc(num * size, true)); 1720234370Sjasone} 1721234370Sjasone 1722234370Sjasonevoid 1723234370Sjasonea0free(void *ptr) 1724234370Sjasone{ 1725234370Sjasone arena_chunk_t *chunk; 1726234370Sjasone 1727234370Sjasone if (ptr == NULL) 1728234370Sjasone return; 1729234370Sjasone 1730234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1731234370Sjasone if (chunk != ptr) 1732234370Sjasone arena_dalloc(chunk->arena, chunk, ptr, false); 1733234370Sjasone else 1734234370Sjasone huge_dalloc(ptr, true); 1735234370Sjasone} 1736234370Sjasone 1737234370Sjasone/******************************************************************************/ 1738