prof.c revision 234543
1234370Sjasone#define JEMALLOC_PROF_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone/******************************************************************************/ 4234370Sjasone 5234370Sjasone#ifdef JEMALLOC_PROF_LIBUNWIND 6234370Sjasone#define UNW_LOCAL_ONLY 7234370Sjasone#include <libunwind.h> 8234370Sjasone#endif 9234370Sjasone 10234370Sjasone#ifdef JEMALLOC_PROF_LIBGCC 11234370Sjasone#include <unwind.h> 12234370Sjasone#endif 13234370Sjasone 14234370Sjasone/******************************************************************************/ 15234370Sjasone/* Data. */ 16234370Sjasone 17234370Sjasonemalloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL) 18234370Sjasone 19234370Sjasonebool opt_prof = false; 20234370Sjasonebool opt_prof_active = true; 21234370Sjasonesize_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; 22234370Sjasonessize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; 23234370Sjasonebool opt_prof_gdump = false; 24234543Sjasonebool opt_prof_final = true; 25234370Sjasonebool opt_prof_leak = false; 26234543Sjasonebool opt_prof_accum = false; 27234370Sjasonechar opt_prof_prefix[PATH_MAX + 1]; 28234370Sjasone 29234370Sjasoneuint64_t prof_interval; 30234370Sjasonebool prof_promote; 31234370Sjasone 32234370Sjasone/* 33234370Sjasone * Table of mutexes that are shared among ctx's. These are leaf locks, so 34234370Sjasone * there is no problem with using them for more than one ctx at the same time. 35234370Sjasone * The primary motivation for this sharing though is that ctx's are ephemeral, 36234370Sjasone * and destroying mutexes causes complications for systems that allocate when 37234370Sjasone * creating/destroying mutexes. 38234370Sjasone */ 39234370Sjasonestatic malloc_mutex_t *ctx_locks; 40234370Sjasonestatic unsigned cum_ctxs; /* Atomic counter. */ 41234370Sjasone 42234370Sjasone/* 43234370Sjasone * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data 44234370Sjasone * structure that knows about all backtraces currently captured. 45234370Sjasone */ 46234370Sjasonestatic ckh_t bt2ctx; 47234370Sjasonestatic malloc_mutex_t bt2ctx_mtx; 48234370Sjasone 49234370Sjasonestatic malloc_mutex_t prof_dump_seq_mtx; 50234370Sjasonestatic uint64_t prof_dump_seq; 51234370Sjasonestatic uint64_t prof_dump_iseq; 52234370Sjasonestatic uint64_t prof_dump_mseq; 53234370Sjasonestatic uint64_t prof_dump_useq; 54234370Sjasone 55234370Sjasone/* 56234370Sjasone * This buffer is rather large for stack allocation, so use a single buffer for 57234370Sjasone * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since 58234370Sjasone * it must be locked anyway during dumping. 59234370Sjasone */ 60234370Sjasonestatic char prof_dump_buf[PROF_DUMP_BUFSIZE]; 61234370Sjasonestatic unsigned prof_dump_buf_end; 62234370Sjasonestatic int prof_dump_fd; 63234370Sjasone 64234370Sjasone/* Do not dump any profiles until bootstrapping is complete. */ 65234370Sjasonestatic bool prof_booted = false; 66234370Sjasone 67234370Sjasonestatic malloc_mutex_t enq_mtx; 68234370Sjasonestatic bool enq; 69234370Sjasonestatic bool enq_idump; 70234370Sjasonestatic bool enq_gdump; 71234370Sjasone 72234370Sjasone/******************************************************************************/ 73234370Sjasone/* Function prototypes for non-inline static functions. */ 74234370Sjasone 75234370Sjasonestatic prof_bt_t *bt_dup(prof_bt_t *bt); 76234370Sjasonestatic void bt_destroy(prof_bt_t *bt); 77234370Sjasone#ifdef JEMALLOC_PROF_LIBGCC 78234370Sjasonestatic _Unwind_Reason_Code prof_unwind_init_callback( 79234370Sjasone struct _Unwind_Context *context, void *arg); 80234370Sjasonestatic _Unwind_Reason_Code prof_unwind_callback( 81234370Sjasone struct _Unwind_Context *context, void *arg); 82234370Sjasone#endif 83234370Sjasonestatic bool prof_flush(bool propagate_err); 84234370Sjasonestatic bool prof_write(bool propagate_err, const char *s); 85234370Sjasonestatic bool prof_printf(bool propagate_err, const char *format, ...) 86234370Sjasone JEMALLOC_ATTR(format(printf, 2, 3)); 87234370Sjasonestatic void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, 88234370Sjasone size_t *leak_nctx); 89234370Sjasonestatic void prof_ctx_destroy(prof_ctx_t *ctx); 90234370Sjasonestatic void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); 91234370Sjasonestatic bool prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, 92234370Sjasone prof_bt_t *bt); 93234370Sjasonestatic bool prof_dump_maps(bool propagate_err); 94234370Sjasonestatic bool prof_dump(bool propagate_err, const char *filename, 95234370Sjasone bool leakcheck); 96234370Sjasonestatic void prof_dump_filename(char *filename, char v, int64_t vseq); 97234370Sjasonestatic void prof_fdump(void); 98234370Sjasonestatic void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, 99234370Sjasone size_t *hash2); 100234370Sjasonestatic bool prof_bt_keycomp(const void *k1, const void *k2); 101234370Sjasonestatic malloc_mutex_t *prof_ctx_mutex_choose(void); 102234370Sjasone 103234370Sjasone/******************************************************************************/ 104234370Sjasone 105234370Sjasonevoid 106234370Sjasonebt_init(prof_bt_t *bt, void **vec) 107234370Sjasone{ 108234370Sjasone 109234370Sjasone cassert(config_prof); 110234370Sjasone 111234370Sjasone bt->vec = vec; 112234370Sjasone bt->len = 0; 113234370Sjasone} 114234370Sjasone 115234370Sjasonestatic void 116234370Sjasonebt_destroy(prof_bt_t *bt) 117234370Sjasone{ 118234370Sjasone 119234370Sjasone cassert(config_prof); 120234370Sjasone 121234370Sjasone idalloc(bt); 122234370Sjasone} 123234370Sjasone 124234370Sjasonestatic prof_bt_t * 125234370Sjasonebt_dup(prof_bt_t *bt) 126234370Sjasone{ 127234370Sjasone prof_bt_t *ret; 128234370Sjasone 129234370Sjasone cassert(config_prof); 130234370Sjasone 131234370Sjasone /* 132234370Sjasone * Create a single allocation that has space for vec immediately 133234370Sjasone * following the prof_bt_t structure. The backtraces that get 134234370Sjasone * stored in the backtrace caches are copied from stack-allocated 135234370Sjasone * temporary variables, so size is known at creation time. Making this 136234370Sjasone * a contiguous object improves cache locality. 137234370Sjasone */ 138234370Sjasone ret = (prof_bt_t *)imalloc(QUANTUM_CEILING(sizeof(prof_bt_t)) + 139234370Sjasone (bt->len * sizeof(void *))); 140234370Sjasone if (ret == NULL) 141234370Sjasone return (NULL); 142234370Sjasone ret->vec = (void **)((uintptr_t)ret + 143234370Sjasone QUANTUM_CEILING(sizeof(prof_bt_t))); 144234370Sjasone memcpy(ret->vec, bt->vec, bt->len * sizeof(void *)); 145234370Sjasone ret->len = bt->len; 146234370Sjasone 147234370Sjasone return (ret); 148234370Sjasone} 149234370Sjasone 150234370Sjasonestatic inline void 151234370Sjasoneprof_enter(void) 152234370Sjasone{ 153234370Sjasone 154234370Sjasone cassert(config_prof); 155234370Sjasone 156234370Sjasone malloc_mutex_lock(&enq_mtx); 157234370Sjasone enq = true; 158234370Sjasone malloc_mutex_unlock(&enq_mtx); 159234370Sjasone 160234370Sjasone malloc_mutex_lock(&bt2ctx_mtx); 161234370Sjasone} 162234370Sjasone 163234370Sjasonestatic inline void 164234370Sjasoneprof_leave(void) 165234370Sjasone{ 166234370Sjasone bool idump, gdump; 167234370Sjasone 168234370Sjasone cassert(config_prof); 169234370Sjasone 170234370Sjasone malloc_mutex_unlock(&bt2ctx_mtx); 171234370Sjasone 172234370Sjasone malloc_mutex_lock(&enq_mtx); 173234370Sjasone enq = false; 174234370Sjasone idump = enq_idump; 175234370Sjasone enq_idump = false; 176234370Sjasone gdump = enq_gdump; 177234370Sjasone enq_gdump = false; 178234370Sjasone malloc_mutex_unlock(&enq_mtx); 179234370Sjasone 180234370Sjasone if (idump) 181234370Sjasone prof_idump(); 182234370Sjasone if (gdump) 183234370Sjasone prof_gdump(); 184234370Sjasone} 185234370Sjasone 186234370Sjasone#ifdef JEMALLOC_PROF_LIBUNWIND 187234370Sjasonevoid 188234370Sjasoneprof_backtrace(prof_bt_t *bt, unsigned nignore) 189234370Sjasone{ 190234370Sjasone unw_context_t uc; 191234370Sjasone unw_cursor_t cursor; 192234370Sjasone unsigned i; 193234370Sjasone int err; 194234370Sjasone 195234370Sjasone cassert(config_prof); 196234370Sjasone assert(bt->len == 0); 197234370Sjasone assert(bt->vec != NULL); 198234370Sjasone 199234370Sjasone unw_getcontext(&uc); 200234370Sjasone unw_init_local(&cursor, &uc); 201234370Sjasone 202234370Sjasone /* Throw away (nignore+1) stack frames, if that many exist. */ 203234370Sjasone for (i = 0; i < nignore + 1; i++) { 204234370Sjasone err = unw_step(&cursor); 205234370Sjasone if (err <= 0) 206234370Sjasone return; 207234370Sjasone } 208234370Sjasone 209234370Sjasone /* 210234370Sjasone * Iterate over stack frames until there are no more, or until no space 211234370Sjasone * remains in bt. 212234370Sjasone */ 213234370Sjasone for (i = 0; i < PROF_BT_MAX; i++) { 214234370Sjasone unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); 215234370Sjasone bt->len++; 216234370Sjasone err = unw_step(&cursor); 217234370Sjasone if (err <= 0) 218234370Sjasone break; 219234370Sjasone } 220234370Sjasone} 221234370Sjasone#elif (defined(JEMALLOC_PROF_LIBGCC)) 222234370Sjasonestatic _Unwind_Reason_Code 223234370Sjasoneprof_unwind_init_callback(struct _Unwind_Context *context, void *arg) 224234370Sjasone{ 225234370Sjasone 226234370Sjasone cassert(config_prof); 227234370Sjasone 228234370Sjasone return (_URC_NO_REASON); 229234370Sjasone} 230234370Sjasone 231234370Sjasonestatic _Unwind_Reason_Code 232234370Sjasoneprof_unwind_callback(struct _Unwind_Context *context, void *arg) 233234370Sjasone{ 234234370Sjasone prof_unwind_data_t *data = (prof_unwind_data_t *)arg; 235234370Sjasone 236234370Sjasone cassert(config_prof); 237234370Sjasone 238234370Sjasone if (data->nignore > 0) 239234370Sjasone data->nignore--; 240234370Sjasone else { 241234370Sjasone data->bt->vec[data->bt->len] = (void *)_Unwind_GetIP(context); 242234370Sjasone data->bt->len++; 243234370Sjasone if (data->bt->len == data->max) 244234370Sjasone return (_URC_END_OF_STACK); 245234370Sjasone } 246234370Sjasone 247234370Sjasone return (_URC_NO_REASON); 248234370Sjasone} 249234370Sjasone 250234370Sjasonevoid 251234370Sjasoneprof_backtrace(prof_bt_t *bt, unsigned nignore) 252234370Sjasone{ 253234370Sjasone prof_unwind_data_t data = {bt, nignore, PROF_BT_MAX}; 254234370Sjasone 255234370Sjasone cassert(config_prof); 256234370Sjasone 257234370Sjasone _Unwind_Backtrace(prof_unwind_callback, &data); 258234370Sjasone} 259234370Sjasone#elif (defined(JEMALLOC_PROF_GCC)) 260234370Sjasonevoid 261234370Sjasoneprof_backtrace(prof_bt_t *bt, unsigned nignore) 262234370Sjasone{ 263234370Sjasone#define BT_FRAME(i) \ 264234370Sjasone if ((i) < nignore + PROF_BT_MAX) { \ 265234370Sjasone void *p; \ 266234370Sjasone if (__builtin_frame_address(i) == 0) \ 267234370Sjasone return; \ 268234370Sjasone p = __builtin_return_address(i); \ 269234370Sjasone if (p == NULL) \ 270234370Sjasone return; \ 271234370Sjasone if (i >= nignore) { \ 272234370Sjasone bt->vec[(i) - nignore] = p; \ 273234370Sjasone bt->len = (i) - nignore + 1; \ 274234370Sjasone } \ 275234370Sjasone } else \ 276234370Sjasone return; 277234370Sjasone 278234370Sjasone cassert(config_prof); 279234370Sjasone assert(nignore <= 3); 280234370Sjasone 281234370Sjasone BT_FRAME(0) 282234370Sjasone BT_FRAME(1) 283234370Sjasone BT_FRAME(2) 284234370Sjasone BT_FRAME(3) 285234370Sjasone BT_FRAME(4) 286234370Sjasone BT_FRAME(5) 287234370Sjasone BT_FRAME(6) 288234370Sjasone BT_FRAME(7) 289234370Sjasone BT_FRAME(8) 290234370Sjasone BT_FRAME(9) 291234370Sjasone 292234370Sjasone BT_FRAME(10) 293234370Sjasone BT_FRAME(11) 294234370Sjasone BT_FRAME(12) 295234370Sjasone BT_FRAME(13) 296234370Sjasone BT_FRAME(14) 297234370Sjasone BT_FRAME(15) 298234370Sjasone BT_FRAME(16) 299234370Sjasone BT_FRAME(17) 300234370Sjasone BT_FRAME(18) 301234370Sjasone BT_FRAME(19) 302234370Sjasone 303234370Sjasone BT_FRAME(20) 304234370Sjasone BT_FRAME(21) 305234370Sjasone BT_FRAME(22) 306234370Sjasone BT_FRAME(23) 307234370Sjasone BT_FRAME(24) 308234370Sjasone BT_FRAME(25) 309234370Sjasone BT_FRAME(26) 310234370Sjasone BT_FRAME(27) 311234370Sjasone BT_FRAME(28) 312234370Sjasone BT_FRAME(29) 313234370Sjasone 314234370Sjasone BT_FRAME(30) 315234370Sjasone BT_FRAME(31) 316234370Sjasone BT_FRAME(32) 317234370Sjasone BT_FRAME(33) 318234370Sjasone BT_FRAME(34) 319234370Sjasone BT_FRAME(35) 320234370Sjasone BT_FRAME(36) 321234370Sjasone BT_FRAME(37) 322234370Sjasone BT_FRAME(38) 323234370Sjasone BT_FRAME(39) 324234370Sjasone 325234370Sjasone BT_FRAME(40) 326234370Sjasone BT_FRAME(41) 327234370Sjasone BT_FRAME(42) 328234370Sjasone BT_FRAME(43) 329234370Sjasone BT_FRAME(44) 330234370Sjasone BT_FRAME(45) 331234370Sjasone BT_FRAME(46) 332234370Sjasone BT_FRAME(47) 333234370Sjasone BT_FRAME(48) 334234370Sjasone BT_FRAME(49) 335234370Sjasone 336234370Sjasone BT_FRAME(50) 337234370Sjasone BT_FRAME(51) 338234370Sjasone BT_FRAME(52) 339234370Sjasone BT_FRAME(53) 340234370Sjasone BT_FRAME(54) 341234370Sjasone BT_FRAME(55) 342234370Sjasone BT_FRAME(56) 343234370Sjasone BT_FRAME(57) 344234370Sjasone BT_FRAME(58) 345234370Sjasone BT_FRAME(59) 346234370Sjasone 347234370Sjasone BT_FRAME(60) 348234370Sjasone BT_FRAME(61) 349234370Sjasone BT_FRAME(62) 350234370Sjasone BT_FRAME(63) 351234370Sjasone BT_FRAME(64) 352234370Sjasone BT_FRAME(65) 353234370Sjasone BT_FRAME(66) 354234370Sjasone BT_FRAME(67) 355234370Sjasone BT_FRAME(68) 356234370Sjasone BT_FRAME(69) 357234370Sjasone 358234370Sjasone BT_FRAME(70) 359234370Sjasone BT_FRAME(71) 360234370Sjasone BT_FRAME(72) 361234370Sjasone BT_FRAME(73) 362234370Sjasone BT_FRAME(74) 363234370Sjasone BT_FRAME(75) 364234370Sjasone BT_FRAME(76) 365234370Sjasone BT_FRAME(77) 366234370Sjasone BT_FRAME(78) 367234370Sjasone BT_FRAME(79) 368234370Sjasone 369234370Sjasone BT_FRAME(80) 370234370Sjasone BT_FRAME(81) 371234370Sjasone BT_FRAME(82) 372234370Sjasone BT_FRAME(83) 373234370Sjasone BT_FRAME(84) 374234370Sjasone BT_FRAME(85) 375234370Sjasone BT_FRAME(86) 376234370Sjasone BT_FRAME(87) 377234370Sjasone BT_FRAME(88) 378234370Sjasone BT_FRAME(89) 379234370Sjasone 380234370Sjasone BT_FRAME(90) 381234370Sjasone BT_FRAME(91) 382234370Sjasone BT_FRAME(92) 383234370Sjasone BT_FRAME(93) 384234370Sjasone BT_FRAME(94) 385234370Sjasone BT_FRAME(95) 386234370Sjasone BT_FRAME(96) 387234370Sjasone BT_FRAME(97) 388234370Sjasone BT_FRAME(98) 389234370Sjasone BT_FRAME(99) 390234370Sjasone 391234370Sjasone BT_FRAME(100) 392234370Sjasone BT_FRAME(101) 393234370Sjasone BT_FRAME(102) 394234370Sjasone BT_FRAME(103) 395234370Sjasone BT_FRAME(104) 396234370Sjasone BT_FRAME(105) 397234370Sjasone BT_FRAME(106) 398234370Sjasone BT_FRAME(107) 399234370Sjasone BT_FRAME(108) 400234370Sjasone BT_FRAME(109) 401234370Sjasone 402234370Sjasone BT_FRAME(110) 403234370Sjasone BT_FRAME(111) 404234370Sjasone BT_FRAME(112) 405234370Sjasone BT_FRAME(113) 406234370Sjasone BT_FRAME(114) 407234370Sjasone BT_FRAME(115) 408234370Sjasone BT_FRAME(116) 409234370Sjasone BT_FRAME(117) 410234370Sjasone BT_FRAME(118) 411234370Sjasone BT_FRAME(119) 412234370Sjasone 413234370Sjasone BT_FRAME(120) 414234370Sjasone BT_FRAME(121) 415234370Sjasone BT_FRAME(122) 416234370Sjasone BT_FRAME(123) 417234370Sjasone BT_FRAME(124) 418234370Sjasone BT_FRAME(125) 419234370Sjasone BT_FRAME(126) 420234370Sjasone BT_FRAME(127) 421234370Sjasone 422234370Sjasone /* Extras to compensate for nignore. */ 423234370Sjasone BT_FRAME(128) 424234370Sjasone BT_FRAME(129) 425234370Sjasone BT_FRAME(130) 426234370Sjasone#undef BT_FRAME 427234370Sjasone} 428234370Sjasone#else 429234370Sjasonevoid 430234370Sjasoneprof_backtrace(prof_bt_t *bt, unsigned nignore) 431234370Sjasone{ 432234370Sjasone 433234370Sjasone cassert(config_prof); 434234370Sjasone assert(false); 435234370Sjasone} 436234370Sjasone#endif 437234370Sjasone 438234370Sjasoneprof_thr_cnt_t * 439234370Sjasoneprof_lookup(prof_bt_t *bt) 440234370Sjasone{ 441234370Sjasone union { 442234370Sjasone prof_thr_cnt_t *p; 443234370Sjasone void *v; 444234370Sjasone } ret; 445234370Sjasone prof_tdata_t *prof_tdata; 446234370Sjasone 447234370Sjasone cassert(config_prof); 448234370Sjasone 449234370Sjasone prof_tdata = *prof_tdata_tsd_get(); 450234370Sjasone if (prof_tdata == NULL) { 451234370Sjasone prof_tdata = prof_tdata_init(); 452234370Sjasone if (prof_tdata == NULL) 453234370Sjasone return (NULL); 454234370Sjasone } 455234370Sjasone 456234370Sjasone if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { 457234370Sjasone union { 458234370Sjasone prof_bt_t *p; 459234370Sjasone void *v; 460234370Sjasone } btkey; 461234370Sjasone union { 462234370Sjasone prof_ctx_t *p; 463234370Sjasone void *v; 464234370Sjasone } ctx; 465234370Sjasone bool new_ctx; 466234370Sjasone 467234370Sjasone /* 468234370Sjasone * This thread's cache lacks bt. Look for it in the global 469234370Sjasone * cache. 470234370Sjasone */ 471234370Sjasone prof_enter(); 472234370Sjasone if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { 473234370Sjasone /* bt has never been seen before. Insert it. */ 474234370Sjasone ctx.v = imalloc(sizeof(prof_ctx_t)); 475234370Sjasone if (ctx.v == NULL) { 476234370Sjasone prof_leave(); 477234370Sjasone return (NULL); 478234370Sjasone } 479234370Sjasone btkey.p = bt_dup(bt); 480234370Sjasone if (btkey.v == NULL) { 481234370Sjasone prof_leave(); 482234370Sjasone idalloc(ctx.v); 483234370Sjasone return (NULL); 484234370Sjasone } 485234370Sjasone ctx.p->bt = btkey.p; 486234370Sjasone ctx.p->lock = prof_ctx_mutex_choose(); 487234370Sjasone memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); 488234370Sjasone ql_new(&ctx.p->cnts_ql); 489234370Sjasone if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { 490234370Sjasone /* OOM. */ 491234370Sjasone prof_leave(); 492234370Sjasone idalloc(btkey.v); 493234370Sjasone idalloc(ctx.v); 494234370Sjasone return (NULL); 495234370Sjasone } 496234370Sjasone /* 497234370Sjasone * Artificially raise curobjs, in order to avoid a race 498234370Sjasone * condition with prof_ctx_merge()/prof_ctx_destroy(). 499234370Sjasone * 500234370Sjasone * No locking is necessary for ctx here because no other 501234370Sjasone * threads have had the opportunity to fetch it from 502234370Sjasone * bt2ctx yet. 503234370Sjasone */ 504234370Sjasone ctx.p->cnt_merged.curobjs++; 505234370Sjasone new_ctx = true; 506234370Sjasone } else { 507234370Sjasone /* 508234370Sjasone * Artificially raise curobjs, in order to avoid a race 509234370Sjasone * condition with prof_ctx_merge()/prof_ctx_destroy(). 510234370Sjasone */ 511234370Sjasone malloc_mutex_lock(ctx.p->lock); 512234370Sjasone ctx.p->cnt_merged.curobjs++; 513234370Sjasone malloc_mutex_unlock(ctx.p->lock); 514234370Sjasone new_ctx = false; 515234370Sjasone } 516234370Sjasone prof_leave(); 517234370Sjasone 518234370Sjasone /* Link a prof_thd_cnt_t into ctx for this thread. */ 519234370Sjasone if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { 520234370Sjasone assert(ckh_count(&prof_tdata->bt2cnt) > 0); 521234370Sjasone /* 522234370Sjasone * Flush the least recently used cnt in order to keep 523234370Sjasone * bt2cnt from becoming too large. 524234370Sjasone */ 525234370Sjasone ret.p = ql_last(&prof_tdata->lru_ql, lru_link); 526234370Sjasone assert(ret.v != NULL); 527234370Sjasone if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, 528234370Sjasone NULL, NULL)) 529234370Sjasone assert(false); 530234370Sjasone ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); 531234370Sjasone prof_ctx_merge(ret.p->ctx, ret.p); 532234370Sjasone /* ret can now be re-used. */ 533234370Sjasone } else { 534234370Sjasone assert(ckh_count(&prof_tdata->bt2cnt) < PROF_TCMAX); 535234370Sjasone /* Allocate and partially initialize a new cnt. */ 536234370Sjasone ret.v = imalloc(sizeof(prof_thr_cnt_t)); 537234370Sjasone if (ret.p == NULL) { 538234370Sjasone if (new_ctx) 539234370Sjasone prof_ctx_destroy(ctx.p); 540234370Sjasone return (NULL); 541234370Sjasone } 542234370Sjasone ql_elm_new(ret.p, cnts_link); 543234370Sjasone ql_elm_new(ret.p, lru_link); 544234370Sjasone } 545234370Sjasone /* Finish initializing ret. */ 546234370Sjasone ret.p->ctx = ctx.p; 547234370Sjasone ret.p->epoch = 0; 548234370Sjasone memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); 549234370Sjasone if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) { 550234370Sjasone if (new_ctx) 551234370Sjasone prof_ctx_destroy(ctx.p); 552234370Sjasone idalloc(ret.v); 553234370Sjasone return (NULL); 554234370Sjasone } 555234370Sjasone ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); 556234370Sjasone malloc_mutex_lock(ctx.p->lock); 557234370Sjasone ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); 558234370Sjasone ctx.p->cnt_merged.curobjs--; 559234370Sjasone malloc_mutex_unlock(ctx.p->lock); 560234370Sjasone } else { 561234370Sjasone /* Move ret to the front of the LRU. */ 562234370Sjasone ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); 563234370Sjasone ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); 564234370Sjasone } 565234370Sjasone 566234370Sjasone return (ret.p); 567234370Sjasone} 568234370Sjasone 569234370Sjasonestatic bool 570234370Sjasoneprof_flush(bool propagate_err) 571234370Sjasone{ 572234370Sjasone bool ret = false; 573234370Sjasone ssize_t err; 574234370Sjasone 575234370Sjasone cassert(config_prof); 576234370Sjasone 577234370Sjasone err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); 578234370Sjasone if (err == -1) { 579234370Sjasone if (propagate_err == false) { 580234370Sjasone malloc_write("<jemalloc>: write() failed during heap " 581234370Sjasone "profile flush\n"); 582234370Sjasone if (opt_abort) 583234370Sjasone abort(); 584234370Sjasone } 585234370Sjasone ret = true; 586234370Sjasone } 587234370Sjasone prof_dump_buf_end = 0; 588234370Sjasone 589234370Sjasone return (ret); 590234370Sjasone} 591234370Sjasone 592234370Sjasonestatic bool 593234370Sjasoneprof_write(bool propagate_err, const char *s) 594234370Sjasone{ 595234370Sjasone unsigned i, slen, n; 596234370Sjasone 597234370Sjasone cassert(config_prof); 598234370Sjasone 599234370Sjasone i = 0; 600234370Sjasone slen = strlen(s); 601234370Sjasone while (i < slen) { 602234370Sjasone /* Flush the buffer if it is full. */ 603234370Sjasone if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) 604234370Sjasone if (prof_flush(propagate_err) && propagate_err) 605234370Sjasone return (true); 606234370Sjasone 607234370Sjasone if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { 608234370Sjasone /* Finish writing. */ 609234370Sjasone n = slen - i; 610234370Sjasone } else { 611234370Sjasone /* Write as much of s as will fit. */ 612234370Sjasone n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; 613234370Sjasone } 614234370Sjasone memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); 615234370Sjasone prof_dump_buf_end += n; 616234370Sjasone i += n; 617234370Sjasone } 618234370Sjasone 619234370Sjasone return (false); 620234370Sjasone} 621234370Sjasone 622234370SjasoneJEMALLOC_ATTR(format(printf, 2, 3)) 623234370Sjasonestatic bool 624234370Sjasoneprof_printf(bool propagate_err, const char *format, ...) 625234370Sjasone{ 626234370Sjasone bool ret; 627234370Sjasone va_list ap; 628234370Sjasone char buf[PROF_PRINTF_BUFSIZE]; 629234370Sjasone 630234370Sjasone va_start(ap, format); 631234370Sjasone malloc_vsnprintf(buf, sizeof(buf), format, ap); 632234370Sjasone va_end(ap); 633234370Sjasone ret = prof_write(propagate_err, buf); 634234370Sjasone 635234370Sjasone return (ret); 636234370Sjasone} 637234370Sjasone 638234370Sjasonestatic void 639234370Sjasoneprof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) 640234370Sjasone{ 641234370Sjasone prof_thr_cnt_t *thr_cnt; 642234370Sjasone prof_cnt_t tcnt; 643234370Sjasone 644234370Sjasone cassert(config_prof); 645234370Sjasone 646234370Sjasone malloc_mutex_lock(ctx->lock); 647234370Sjasone 648234370Sjasone memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); 649234370Sjasone ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { 650234370Sjasone volatile unsigned *epoch = &thr_cnt->epoch; 651234370Sjasone 652234370Sjasone while (true) { 653234370Sjasone unsigned epoch0 = *epoch; 654234370Sjasone 655234370Sjasone /* Make sure epoch is even. */ 656234370Sjasone if (epoch0 & 1U) 657234370Sjasone continue; 658234370Sjasone 659234370Sjasone memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); 660234370Sjasone 661234370Sjasone /* Terminate if epoch didn't change while reading. */ 662234370Sjasone if (*epoch == epoch0) 663234370Sjasone break; 664234370Sjasone } 665234370Sjasone 666234370Sjasone ctx->cnt_summed.curobjs += tcnt.curobjs; 667234370Sjasone ctx->cnt_summed.curbytes += tcnt.curbytes; 668234370Sjasone if (opt_prof_accum) { 669234370Sjasone ctx->cnt_summed.accumobjs += tcnt.accumobjs; 670234370Sjasone ctx->cnt_summed.accumbytes += tcnt.accumbytes; 671234370Sjasone } 672234370Sjasone } 673234370Sjasone 674234370Sjasone if (ctx->cnt_summed.curobjs != 0) 675234370Sjasone (*leak_nctx)++; 676234370Sjasone 677234370Sjasone /* Add to cnt_all. */ 678234370Sjasone cnt_all->curobjs += ctx->cnt_summed.curobjs; 679234370Sjasone cnt_all->curbytes += ctx->cnt_summed.curbytes; 680234370Sjasone if (opt_prof_accum) { 681234370Sjasone cnt_all->accumobjs += ctx->cnt_summed.accumobjs; 682234370Sjasone cnt_all->accumbytes += ctx->cnt_summed.accumbytes; 683234370Sjasone } 684234370Sjasone 685234370Sjasone malloc_mutex_unlock(ctx->lock); 686234370Sjasone} 687234370Sjasone 688234370Sjasonestatic void 689234370Sjasoneprof_ctx_destroy(prof_ctx_t *ctx) 690234370Sjasone{ 691234370Sjasone 692234370Sjasone cassert(config_prof); 693234370Sjasone 694234370Sjasone /* 695234370Sjasone * Check that ctx is still unused by any thread cache before destroying 696234370Sjasone * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in 697234370Sjasone * order to avoid a race condition with this function, as does 698234370Sjasone * prof_ctx_merge() in order to avoid a race between the main body of 699234370Sjasone * prof_ctx_merge() and entry into this function. 700234370Sjasone */ 701234370Sjasone prof_enter(); 702234370Sjasone malloc_mutex_lock(ctx->lock); 703234370Sjasone if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { 704234370Sjasone assert(ctx->cnt_merged.curbytes == 0); 705234370Sjasone assert(ctx->cnt_merged.accumobjs == 0); 706234370Sjasone assert(ctx->cnt_merged.accumbytes == 0); 707234370Sjasone /* Remove ctx from bt2ctx. */ 708234370Sjasone if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) 709234370Sjasone assert(false); 710234370Sjasone prof_leave(); 711234370Sjasone /* Destroy ctx. */ 712234370Sjasone malloc_mutex_unlock(ctx->lock); 713234370Sjasone bt_destroy(ctx->bt); 714234370Sjasone idalloc(ctx); 715234370Sjasone } else { 716234370Sjasone /* 717234370Sjasone * Compensate for increment in prof_ctx_merge() or 718234370Sjasone * prof_lookup(). 719234370Sjasone */ 720234370Sjasone ctx->cnt_merged.curobjs--; 721234370Sjasone malloc_mutex_unlock(ctx->lock); 722234370Sjasone prof_leave(); 723234370Sjasone } 724234370Sjasone} 725234370Sjasone 726234370Sjasonestatic void 727234370Sjasoneprof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) 728234370Sjasone{ 729234370Sjasone bool destroy; 730234370Sjasone 731234370Sjasone cassert(config_prof); 732234370Sjasone 733234370Sjasone /* Merge cnt stats and detach from ctx. */ 734234370Sjasone malloc_mutex_lock(ctx->lock); 735234370Sjasone ctx->cnt_merged.curobjs += cnt->cnts.curobjs; 736234370Sjasone ctx->cnt_merged.curbytes += cnt->cnts.curbytes; 737234370Sjasone ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; 738234370Sjasone ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; 739234370Sjasone ql_remove(&ctx->cnts_ql, cnt, cnts_link); 740234370Sjasone if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && 741234370Sjasone ctx->cnt_merged.curobjs == 0) { 742234370Sjasone /* 743234370Sjasone * Artificially raise ctx->cnt_merged.curobjs in order to keep 744234370Sjasone * another thread from winning the race to destroy ctx while 745234370Sjasone * this one has ctx->lock dropped. Without this, it would be 746234370Sjasone * possible for another thread to: 747234370Sjasone * 748234370Sjasone * 1) Sample an allocation associated with ctx. 749234370Sjasone * 2) Deallocate the sampled object. 750234370Sjasone * 3) Successfully prof_ctx_destroy(ctx). 751234370Sjasone * 752234370Sjasone * The result would be that ctx no longer exists by the time 753234370Sjasone * this thread accesses it in prof_ctx_destroy(). 754234370Sjasone */ 755234370Sjasone ctx->cnt_merged.curobjs++; 756234370Sjasone destroy = true; 757234370Sjasone } else 758234370Sjasone destroy = false; 759234370Sjasone malloc_mutex_unlock(ctx->lock); 760234370Sjasone if (destroy) 761234370Sjasone prof_ctx_destroy(ctx); 762234370Sjasone} 763234370Sjasone 764234370Sjasonestatic bool 765234370Sjasoneprof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, prof_bt_t *bt) 766234370Sjasone{ 767234370Sjasone unsigned i; 768234370Sjasone 769234370Sjasone cassert(config_prof); 770234370Sjasone 771234370Sjasone if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { 772234370Sjasone assert(ctx->cnt_summed.curbytes == 0); 773234370Sjasone assert(ctx->cnt_summed.accumobjs == 0); 774234370Sjasone assert(ctx->cnt_summed.accumbytes == 0); 775234370Sjasone return (false); 776234370Sjasone } 777234370Sjasone 778234370Sjasone if (prof_printf(propagate_err, "%"PRId64": %"PRId64 779234370Sjasone " [%"PRIu64": %"PRIu64"] @", 780234370Sjasone ctx->cnt_summed.curobjs, ctx->cnt_summed.curbytes, 781234370Sjasone ctx->cnt_summed.accumobjs, ctx->cnt_summed.accumbytes)) 782234370Sjasone return (true); 783234370Sjasone 784234370Sjasone for (i = 0; i < bt->len; i++) { 785234370Sjasone if (prof_printf(propagate_err, " %#"PRIxPTR, 786234370Sjasone (uintptr_t)bt->vec[i])) 787234370Sjasone return (true); 788234370Sjasone } 789234370Sjasone 790234370Sjasone if (prof_write(propagate_err, "\n")) 791234370Sjasone return (true); 792234370Sjasone 793234370Sjasone return (false); 794234370Sjasone} 795234370Sjasone 796234370Sjasonestatic bool 797234370Sjasoneprof_dump_maps(bool propagate_err) 798234370Sjasone{ 799234370Sjasone int mfd; 800234370Sjasone char filename[PATH_MAX + 1]; 801234370Sjasone 802234370Sjasone cassert(config_prof); 803234370Sjasone 804234370Sjasone malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps", 805234370Sjasone (int)getpid()); 806234370Sjasone mfd = open(filename, O_RDONLY); 807234370Sjasone if (mfd != -1) { 808234370Sjasone ssize_t nread; 809234370Sjasone 810234370Sjasone if (prof_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && 811234370Sjasone propagate_err) 812234370Sjasone return (true); 813234370Sjasone nread = 0; 814234370Sjasone do { 815234370Sjasone prof_dump_buf_end += nread; 816234370Sjasone if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { 817234370Sjasone /* Make space in prof_dump_buf before read(). */ 818234370Sjasone if (prof_flush(propagate_err) && propagate_err) 819234370Sjasone return (true); 820234370Sjasone } 821234370Sjasone nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], 822234370Sjasone PROF_DUMP_BUFSIZE - prof_dump_buf_end); 823234370Sjasone } while (nread > 0); 824234370Sjasone close(mfd); 825234370Sjasone } else 826234370Sjasone return (true); 827234370Sjasone 828234370Sjasone return (false); 829234370Sjasone} 830234370Sjasone 831234370Sjasonestatic bool 832234370Sjasoneprof_dump(bool propagate_err, const char *filename, bool leakcheck) 833234370Sjasone{ 834234370Sjasone prof_cnt_t cnt_all; 835234370Sjasone size_t tabind; 836234370Sjasone union { 837234370Sjasone prof_bt_t *p; 838234370Sjasone void *v; 839234370Sjasone } bt; 840234370Sjasone union { 841234370Sjasone prof_ctx_t *p; 842234370Sjasone void *v; 843234370Sjasone } ctx; 844234370Sjasone size_t leak_nctx; 845234370Sjasone 846234370Sjasone cassert(config_prof); 847234370Sjasone 848234370Sjasone prof_enter(); 849234370Sjasone prof_dump_fd = creat(filename, 0644); 850234370Sjasone if (prof_dump_fd == -1) { 851234370Sjasone if (propagate_err == false) { 852234370Sjasone malloc_printf( 853234370Sjasone "<jemalloc>: creat(\"%s\"), 0644) failed\n", 854234370Sjasone filename); 855234370Sjasone if (opt_abort) 856234370Sjasone abort(); 857234370Sjasone } 858234370Sjasone goto label_error; 859234370Sjasone } 860234370Sjasone 861234370Sjasone /* Merge per thread profile stats, and sum them in cnt_all. */ 862234370Sjasone memset(&cnt_all, 0, sizeof(prof_cnt_t)); 863234370Sjasone leak_nctx = 0; 864234370Sjasone for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) 865234370Sjasone prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); 866234370Sjasone 867234370Sjasone /* Dump profile header. */ 868234370Sjasone if (opt_lg_prof_sample == 0) { 869234370Sjasone if (prof_printf(propagate_err, 870234370Sjasone "heap profile: %"PRId64": %"PRId64 871234370Sjasone " [%"PRIu64": %"PRIu64"] @ heapprofile\n", 872234370Sjasone cnt_all.curobjs, cnt_all.curbytes, 873234370Sjasone cnt_all.accumobjs, cnt_all.accumbytes)) 874234370Sjasone goto label_error; 875234370Sjasone } else { 876234370Sjasone if (prof_printf(propagate_err, 877234370Sjasone "heap profile: %"PRId64": %"PRId64 878234370Sjasone " [%"PRIu64": %"PRIu64"] @ heap_v2/%"PRIu64"\n", 879234370Sjasone cnt_all.curobjs, cnt_all.curbytes, 880234370Sjasone cnt_all.accumobjs, cnt_all.accumbytes, 881234370Sjasone ((uint64_t)1U << opt_lg_prof_sample))) 882234370Sjasone goto label_error; 883234370Sjasone } 884234370Sjasone 885234370Sjasone /* Dump per ctx profile stats. */ 886234370Sjasone for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) 887234370Sjasone == false;) { 888234370Sjasone if (prof_dump_ctx(propagate_err, ctx.p, bt.p)) 889234370Sjasone goto label_error; 890234370Sjasone } 891234370Sjasone 892234370Sjasone /* Dump /proc/<pid>/maps if possible. */ 893234370Sjasone if (prof_dump_maps(propagate_err)) 894234370Sjasone goto label_error; 895234370Sjasone 896234370Sjasone if (prof_flush(propagate_err)) 897234370Sjasone goto label_error; 898234370Sjasone close(prof_dump_fd); 899234370Sjasone prof_leave(); 900234370Sjasone 901234370Sjasone if (leakcheck && cnt_all.curbytes != 0) { 902234370Sjasone malloc_printf("<jemalloc>: Leak summary: %"PRId64" byte%s, %" 903234370Sjasone PRId64" object%s, %zu context%s\n", 904234370Sjasone cnt_all.curbytes, (cnt_all.curbytes != 1) ? "s" : "", 905234370Sjasone cnt_all.curobjs, (cnt_all.curobjs != 1) ? "s" : "", 906234370Sjasone leak_nctx, (leak_nctx != 1) ? "s" : ""); 907234370Sjasone malloc_printf( 908234370Sjasone "<jemalloc>: Run pprof on \"%s\" for leak detail\n", 909234370Sjasone filename); 910234370Sjasone } 911234370Sjasone 912234370Sjasone return (false); 913234370Sjasonelabel_error: 914234370Sjasone prof_leave(); 915234370Sjasone return (true); 916234370Sjasone} 917234370Sjasone 918234370Sjasone#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) 919234370Sjasonestatic void 920234370Sjasoneprof_dump_filename(char *filename, char v, int64_t vseq) 921234370Sjasone{ 922234370Sjasone 923234370Sjasone cassert(config_prof); 924234370Sjasone 925234370Sjasone if (vseq != UINT64_C(0xffffffffffffffff)) { 926234370Sjasone /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ 927234370Sjasone malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 928234370Sjasone "%s.%d.%"PRIu64".%c%"PRId64".heap", 929234370Sjasone opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq); 930234370Sjasone } else { 931234370Sjasone /* "<prefix>.<pid>.<seq>.<v>.heap" */ 932234370Sjasone malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 933234370Sjasone "%s.%d.%"PRIu64".%c.heap", 934234370Sjasone opt_prof_prefix, (int)getpid(), prof_dump_seq, v); 935234370Sjasone } 936234370Sjasone} 937234370Sjasone 938234370Sjasonestatic void 939234370Sjasoneprof_fdump(void) 940234370Sjasone{ 941234370Sjasone char filename[DUMP_FILENAME_BUFSIZE]; 942234370Sjasone 943234370Sjasone cassert(config_prof); 944234370Sjasone 945234370Sjasone if (prof_booted == false) 946234370Sjasone return; 947234370Sjasone 948234543Sjasone if (opt_prof_final && opt_prof_prefix[0] != '\0') { 949234370Sjasone malloc_mutex_lock(&prof_dump_seq_mtx); 950234370Sjasone prof_dump_filename(filename, 'f', UINT64_C(0xffffffffffffffff)); 951234370Sjasone malloc_mutex_unlock(&prof_dump_seq_mtx); 952234370Sjasone prof_dump(false, filename, opt_prof_leak); 953234370Sjasone } 954234370Sjasone} 955234370Sjasone 956234370Sjasonevoid 957234370Sjasoneprof_idump(void) 958234370Sjasone{ 959234370Sjasone char filename[PATH_MAX + 1]; 960234370Sjasone 961234370Sjasone cassert(config_prof); 962234370Sjasone 963234370Sjasone if (prof_booted == false) 964234370Sjasone return; 965234370Sjasone malloc_mutex_lock(&enq_mtx); 966234370Sjasone if (enq) { 967234370Sjasone enq_idump = true; 968234370Sjasone malloc_mutex_unlock(&enq_mtx); 969234370Sjasone return; 970234370Sjasone } 971234370Sjasone malloc_mutex_unlock(&enq_mtx); 972234370Sjasone 973234370Sjasone if (opt_prof_prefix[0] != '\0') { 974234370Sjasone malloc_mutex_lock(&prof_dump_seq_mtx); 975234370Sjasone prof_dump_filename(filename, 'i', prof_dump_iseq); 976234370Sjasone prof_dump_iseq++; 977234370Sjasone malloc_mutex_unlock(&prof_dump_seq_mtx); 978234370Sjasone prof_dump(false, filename, false); 979234370Sjasone } 980234370Sjasone} 981234370Sjasone 982234370Sjasonebool 983234370Sjasoneprof_mdump(const char *filename) 984234370Sjasone{ 985234370Sjasone char filename_buf[DUMP_FILENAME_BUFSIZE]; 986234370Sjasone 987234370Sjasone cassert(config_prof); 988234370Sjasone 989234370Sjasone if (opt_prof == false || prof_booted == false) 990234370Sjasone return (true); 991234370Sjasone 992234370Sjasone if (filename == NULL) { 993234370Sjasone /* No filename specified, so automatically generate one. */ 994234370Sjasone if (opt_prof_prefix[0] == '\0') 995234370Sjasone return (true); 996234370Sjasone malloc_mutex_lock(&prof_dump_seq_mtx); 997234370Sjasone prof_dump_filename(filename_buf, 'm', prof_dump_mseq); 998234370Sjasone prof_dump_mseq++; 999234370Sjasone malloc_mutex_unlock(&prof_dump_seq_mtx); 1000234370Sjasone filename = filename_buf; 1001234370Sjasone } 1002234370Sjasone return (prof_dump(true, filename, false)); 1003234370Sjasone} 1004234370Sjasone 1005234370Sjasonevoid 1006234370Sjasoneprof_gdump(void) 1007234370Sjasone{ 1008234370Sjasone char filename[DUMP_FILENAME_BUFSIZE]; 1009234370Sjasone 1010234370Sjasone cassert(config_prof); 1011234370Sjasone 1012234370Sjasone if (prof_booted == false) 1013234370Sjasone return; 1014234370Sjasone malloc_mutex_lock(&enq_mtx); 1015234370Sjasone if (enq) { 1016234370Sjasone enq_gdump = true; 1017234370Sjasone malloc_mutex_unlock(&enq_mtx); 1018234370Sjasone return; 1019234370Sjasone } 1020234370Sjasone malloc_mutex_unlock(&enq_mtx); 1021234370Sjasone 1022234370Sjasone if (opt_prof_prefix[0] != '\0') { 1023234370Sjasone malloc_mutex_lock(&prof_dump_seq_mtx); 1024234370Sjasone prof_dump_filename(filename, 'u', prof_dump_useq); 1025234370Sjasone prof_dump_useq++; 1026234370Sjasone malloc_mutex_unlock(&prof_dump_seq_mtx); 1027234370Sjasone prof_dump(false, filename, false); 1028234370Sjasone } 1029234370Sjasone} 1030234370Sjasone 1031234370Sjasonestatic void 1032234370Sjasoneprof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) 1033234370Sjasone{ 1034234370Sjasone size_t ret1, ret2; 1035234370Sjasone uint64_t h; 1036234370Sjasone prof_bt_t *bt = (prof_bt_t *)key; 1037234370Sjasone 1038234370Sjasone cassert(config_prof); 1039234370Sjasone assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); 1040234370Sjasone assert(hash1 != NULL); 1041234370Sjasone assert(hash2 != NULL); 1042234370Sjasone 1043234370Sjasone h = hash(bt->vec, bt->len * sizeof(void *), 1044234370Sjasone UINT64_C(0x94122f335b332aea)); 1045234370Sjasone if (minbits <= 32) { 1046234370Sjasone /* 1047234370Sjasone * Avoid doing multiple hashes, since a single hash provides 1048234370Sjasone * enough bits. 1049234370Sjasone */ 1050234370Sjasone ret1 = h & ZU(0xffffffffU); 1051234370Sjasone ret2 = h >> 32; 1052234370Sjasone } else { 1053234370Sjasone ret1 = h; 1054234370Sjasone ret2 = hash(bt->vec, bt->len * sizeof(void *), 1055234370Sjasone UINT64_C(0x8432a476666bbc13)); 1056234370Sjasone } 1057234370Sjasone 1058234370Sjasone *hash1 = ret1; 1059234370Sjasone *hash2 = ret2; 1060234370Sjasone} 1061234370Sjasone 1062234370Sjasonestatic bool 1063234370Sjasoneprof_bt_keycomp(const void *k1, const void *k2) 1064234370Sjasone{ 1065234370Sjasone const prof_bt_t *bt1 = (prof_bt_t *)k1; 1066234370Sjasone const prof_bt_t *bt2 = (prof_bt_t *)k2; 1067234370Sjasone 1068234370Sjasone cassert(config_prof); 1069234370Sjasone 1070234370Sjasone if (bt1->len != bt2->len) 1071234370Sjasone return (false); 1072234370Sjasone return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); 1073234370Sjasone} 1074234370Sjasone 1075234370Sjasonestatic malloc_mutex_t * 1076234370Sjasoneprof_ctx_mutex_choose(void) 1077234370Sjasone{ 1078234370Sjasone unsigned nctxs = atomic_add_u(&cum_ctxs, 1); 1079234370Sjasone 1080234370Sjasone return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); 1081234370Sjasone} 1082234370Sjasone 1083234370Sjasoneprof_tdata_t * 1084234370Sjasoneprof_tdata_init(void) 1085234370Sjasone{ 1086234370Sjasone prof_tdata_t *prof_tdata; 1087234370Sjasone 1088234370Sjasone cassert(config_prof); 1089234370Sjasone 1090234370Sjasone /* Initialize an empty cache for this thread. */ 1091234370Sjasone prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); 1092234370Sjasone if (prof_tdata == NULL) 1093234370Sjasone return (NULL); 1094234370Sjasone 1095234370Sjasone if (ckh_new(&prof_tdata->bt2cnt, PROF_CKH_MINITEMS, 1096234370Sjasone prof_bt_hash, prof_bt_keycomp)) { 1097234370Sjasone idalloc(prof_tdata); 1098234370Sjasone return (NULL); 1099234370Sjasone } 1100234370Sjasone ql_new(&prof_tdata->lru_ql); 1101234370Sjasone 1102234370Sjasone prof_tdata->vec = imalloc(sizeof(void *) * PROF_BT_MAX); 1103234370Sjasone if (prof_tdata->vec == NULL) { 1104234370Sjasone ckh_delete(&prof_tdata->bt2cnt); 1105234370Sjasone idalloc(prof_tdata); 1106234370Sjasone return (NULL); 1107234370Sjasone } 1108234370Sjasone 1109234370Sjasone prof_tdata->prng_state = 0; 1110234370Sjasone prof_tdata->threshold = 0; 1111234370Sjasone prof_tdata->accum = 0; 1112234370Sjasone 1113234370Sjasone prof_tdata_tsd_set(&prof_tdata); 1114234370Sjasone 1115234370Sjasone return (prof_tdata); 1116234370Sjasone} 1117234370Sjasone 1118234370Sjasonevoid 1119234370Sjasoneprof_tdata_cleanup(void *arg) 1120234370Sjasone{ 1121234370Sjasone prof_thr_cnt_t *cnt; 1122234370Sjasone prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg; 1123234370Sjasone 1124234370Sjasone cassert(config_prof); 1125234370Sjasone 1126234370Sjasone /* 1127234370Sjasone * Delete the hash table. All of its contents can still be iterated 1128234370Sjasone * over via the LRU. 1129234370Sjasone */ 1130234370Sjasone ckh_delete(&prof_tdata->bt2cnt); 1131234370Sjasone 1132234370Sjasone /* Iteratively merge cnt's into the global stats and delete them. */ 1133234370Sjasone while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { 1134234370Sjasone ql_remove(&prof_tdata->lru_ql, cnt, lru_link); 1135234370Sjasone prof_ctx_merge(cnt->ctx, cnt); 1136234370Sjasone idalloc(cnt); 1137234370Sjasone } 1138234370Sjasone 1139234370Sjasone idalloc(prof_tdata->vec); 1140234370Sjasone 1141234370Sjasone idalloc(prof_tdata); 1142234370Sjasone prof_tdata = NULL; 1143234370Sjasone prof_tdata_tsd_set(&prof_tdata); 1144234370Sjasone} 1145234370Sjasone 1146234370Sjasonevoid 1147234370Sjasoneprof_boot0(void) 1148234370Sjasone{ 1149234370Sjasone 1150234370Sjasone cassert(config_prof); 1151234370Sjasone 1152234370Sjasone memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, 1153234370Sjasone sizeof(PROF_PREFIX_DEFAULT)); 1154234370Sjasone} 1155234370Sjasone 1156234370Sjasonevoid 1157234370Sjasoneprof_boot1(void) 1158234370Sjasone{ 1159234370Sjasone 1160234370Sjasone cassert(config_prof); 1161234370Sjasone 1162234370Sjasone /* 1163234370Sjasone * opt_prof and prof_promote must be in their final state before any 1164234370Sjasone * arenas are initialized, so this function must be executed early. 1165234370Sjasone */ 1166234370Sjasone 1167234370Sjasone if (opt_prof_leak && opt_prof == false) { 1168234370Sjasone /* 1169234370Sjasone * Enable opt_prof, but in such a way that profiles are never 1170234370Sjasone * automatically dumped. 1171234370Sjasone */ 1172234370Sjasone opt_prof = true; 1173234370Sjasone opt_prof_gdump = false; 1174234370Sjasone prof_interval = 0; 1175234370Sjasone } else if (opt_prof) { 1176234370Sjasone if (opt_lg_prof_interval >= 0) { 1177234370Sjasone prof_interval = (((uint64_t)1U) << 1178234370Sjasone opt_lg_prof_interval); 1179234370Sjasone } else 1180234370Sjasone prof_interval = 0; 1181234370Sjasone } 1182234370Sjasone 1183234370Sjasone prof_promote = (opt_prof && opt_lg_prof_sample > LG_PAGE); 1184234370Sjasone} 1185234370Sjasone 1186234370Sjasonebool 1187234370Sjasoneprof_boot2(void) 1188234370Sjasone{ 1189234370Sjasone 1190234370Sjasone cassert(config_prof); 1191234370Sjasone 1192234370Sjasone if (opt_prof) { 1193234370Sjasone unsigned i; 1194234370Sjasone 1195234370Sjasone if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, 1196234370Sjasone prof_bt_keycomp)) 1197234370Sjasone return (true); 1198234370Sjasone if (malloc_mutex_init(&bt2ctx_mtx)) 1199234370Sjasone return (true); 1200234370Sjasone if (prof_tdata_tsd_boot()) { 1201234370Sjasone malloc_write( 1202234370Sjasone "<jemalloc>: Error in pthread_key_create()\n"); 1203234370Sjasone abort(); 1204234370Sjasone } 1205234370Sjasone 1206234370Sjasone if (malloc_mutex_init(&prof_dump_seq_mtx)) 1207234370Sjasone return (true); 1208234370Sjasone 1209234370Sjasone if (malloc_mutex_init(&enq_mtx)) 1210234370Sjasone return (true); 1211234370Sjasone enq = false; 1212234370Sjasone enq_idump = false; 1213234370Sjasone enq_gdump = false; 1214234370Sjasone 1215234370Sjasone if (atexit(prof_fdump) != 0) { 1216234370Sjasone malloc_write("<jemalloc>: Error in atexit()\n"); 1217234370Sjasone if (opt_abort) 1218234370Sjasone abort(); 1219234370Sjasone } 1220234370Sjasone 1221234370Sjasone ctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * 1222234370Sjasone sizeof(malloc_mutex_t)); 1223234370Sjasone if (ctx_locks == NULL) 1224234370Sjasone return (true); 1225234370Sjasone for (i = 0; i < PROF_NCTX_LOCKS; i++) { 1226234370Sjasone if (malloc_mutex_init(&ctx_locks[i])) 1227234370Sjasone return (true); 1228234370Sjasone } 1229234370Sjasone } 1230234370Sjasone 1231234370Sjasone#ifdef JEMALLOC_PROF_LIBGCC 1232234370Sjasone /* 1233234370Sjasone * Cause the backtracing machinery to allocate its internal state 1234234370Sjasone * before enabling profiling. 1235234370Sjasone */ 1236234370Sjasone _Unwind_Backtrace(prof_unwind_init_callback, NULL); 1237234370Sjasone#endif 1238234370Sjasone 1239234370Sjasone prof_booted = true; 1240234370Sjasone 1241234370Sjasone return (false); 1242234370Sjasone} 1243234370Sjasone 1244234370Sjasone/******************************************************************************/ 1245