1#define JEMALLOC_PROF_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3/******************************************************************************/ 4 5#ifdef JEMALLOC_PROF_LIBUNWIND 6#define UNW_LOCAL_ONLY 7#include <libunwind.h> 8#endif 9 10#ifdef JEMALLOC_PROF_LIBGCC 11#include <unwind.h> 12#endif 13 14/******************************************************************************/ 15/* Data. */ 16 17bool opt_prof = false; 18bool opt_prof_active = true; 19bool opt_prof_thread_active_init = true; 20size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; 21ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; 22bool opt_prof_gdump = false; 23bool opt_prof_final = false; 24bool opt_prof_leak = false; 25bool opt_prof_accum = false; 26char opt_prof_prefix[ 27 /* Minimize memory bloat for non-prof builds. */ 28#ifdef JEMALLOC_PROF 29 PATH_MAX + 30#endif 31 1]; 32 33/* 34 * Initialized as opt_prof_active, and accessed via 35 * prof_active_[gs]et{_unlocked,}(). 36 */ 37bool prof_active; 38static malloc_mutex_t prof_active_mtx; 39 40/* 41 * Initialized as opt_prof_thread_active_init, and accessed via 42 * prof_thread_active_init_[gs]et(). 43 */ 44static bool prof_thread_active_init; 45static malloc_mutex_t prof_thread_active_init_mtx; 46 47/* 48 * Initialized as opt_prof_gdump, and accessed via 49 * prof_gdump_[gs]et{_unlocked,}(). 50 */ 51bool prof_gdump_val; 52static malloc_mutex_t prof_gdump_mtx; 53 54uint64_t prof_interval = 0; 55 56size_t lg_prof_sample; 57 58/* 59 * Table of mutexes that are shared among gctx's. These are leaf locks, so 60 * there is no problem with using them for more than one gctx at the same time. 61 * The primary motivation for this sharing though is that gctx's are ephemeral, 62 * and destroying mutexes causes complications for systems that allocate when 63 * creating/destroying mutexes. 64 */ 65static malloc_mutex_t *gctx_locks; 66static unsigned cum_gctxs; /* Atomic counter. */ 67 68/* 69 * Table of mutexes that are shared among tdata's. No operations require 70 * holding multiple tdata locks, so there is no problem with using them for more 71 * than one tdata at the same time, even though a gctx lock may be acquired 72 * while holding a tdata lock. 73 */ 74static malloc_mutex_t *tdata_locks; 75 76/* 77 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data 78 * structure that knows about all backtraces currently captured. 79 */ 80static ckh_t bt2gctx; 81static malloc_mutex_t bt2gctx_mtx; 82 83/* 84 * Tree of all extant prof_tdata_t structures, regardless of state, 85 * {attached,detached,expired}. 86 */ 87static prof_tdata_tree_t tdatas; 88static malloc_mutex_t tdatas_mtx; 89 90static uint64_t next_thr_uid; 91static malloc_mutex_t next_thr_uid_mtx; 92 93static malloc_mutex_t prof_dump_seq_mtx; 94static uint64_t prof_dump_seq; 95static uint64_t prof_dump_iseq; 96static uint64_t prof_dump_mseq; 97static uint64_t prof_dump_useq; 98 99/* 100 * This buffer is rather large for stack allocation, so use a single buffer for 101 * all profile dumps. 102 */ 103static malloc_mutex_t prof_dump_mtx; 104static char prof_dump_buf[ 105 /* Minimize memory bloat for non-prof builds. */ 106#ifdef JEMALLOC_PROF 107 PROF_DUMP_BUFSIZE 108#else 109 1 110#endif 111]; 112static size_t prof_dump_buf_end; 113static int prof_dump_fd; 114 115/* Do not dump any profiles until bootstrapping is complete. */ 116static bool prof_booted = false; 117 118/******************************************************************************/ 119/* 120 * Function prototypes for static functions that are referenced prior to 121 * definition. 122 */ 123 124static bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx); 125static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); 126static bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, 127 bool even_if_attached); 128static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, 129 bool even_if_attached); 130static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); 131 132/******************************************************************************/ 133/* Red-black trees. */ 134 135JEMALLOC_INLINE_C int 136prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) 137{ 138 uint64_t a_thr_uid = a->thr_uid; 139 uint64_t b_thr_uid = b->thr_uid; 140 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); 141 if (ret == 0) { 142 uint64_t a_thr_discrim = a->thr_discrim; 143 uint64_t b_thr_discrim = b->thr_discrim; 144 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < 145 b_thr_discrim); 146 if (ret == 0) { 147 uint64_t a_tctx_uid = a->tctx_uid; 148 uint64_t b_tctx_uid = b->tctx_uid; 149 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < 150 b_tctx_uid); 151 } 152 } 153 return (ret); 154} 155 156rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, 157 tctx_link, prof_tctx_comp) 158 159JEMALLOC_INLINE_C int 160prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) 161{ 162 unsigned a_len = a->bt.len; 163 unsigned b_len = b->bt.len; 164 unsigned comp_len = (a_len < b_len) ? a_len : b_len; 165 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); 166 if (ret == 0) 167 ret = (a_len > b_len) - (a_len < b_len); 168 return (ret); 169} 170 171rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, 172 prof_gctx_comp) 173 174JEMALLOC_INLINE_C int 175prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) 176{ 177 int ret; 178 uint64_t a_uid = a->thr_uid; 179 uint64_t b_uid = b->thr_uid; 180 181 ret = ((a_uid > b_uid) - (a_uid < b_uid)); 182 if (ret == 0) { 183 uint64_t a_discrim = a->thr_discrim; 184 uint64_t b_discrim = b->thr_discrim; 185 186 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); 187 } 188 return (ret); 189} 190 191rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, 192 prof_tdata_comp) 193 194/******************************************************************************/ 195 196void 197prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) 198{ 199 prof_tdata_t *tdata; 200 201 cassert(config_prof); 202 203 if (updated) { 204 /* 205 * Compute a new sample threshold. This isn't very important in 206 * practice, because this function is rarely executed, so the 207 * potential for sample bias is minimal except in contrived 208 * programs. 209 */ 210 tdata = prof_tdata_get(tsd, true); 211 if (tdata != NULL) 212 prof_sample_threshold_update(tdata); 213 } 214 215 if ((uintptr_t)tctx > (uintptr_t)1U) { 216 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); 217 tctx->prepared = false; 218 if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) 219 prof_tctx_destroy(tsd, tctx); 220 else 221 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); 222 } 223} 224 225void 226prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent, const void *ptr, 227 size_t usize, prof_tctx_t *tctx) 228{ 229 prof_tctx_set(tsdn, extent, ptr, usize, tctx); 230 231 malloc_mutex_lock(tsdn, tctx->tdata->lock); 232 tctx->cnts.curobjs++; 233 tctx->cnts.curbytes += usize; 234 if (opt_prof_accum) { 235 tctx->cnts.accumobjs++; 236 tctx->cnts.accumbytes += usize; 237 } 238 tctx->prepared = false; 239 malloc_mutex_unlock(tsdn, tctx->tdata->lock); 240} 241 242void 243prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 244{ 245 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); 246 assert(tctx->cnts.curobjs > 0); 247 assert(tctx->cnts.curbytes >= usize); 248 tctx->cnts.curobjs--; 249 tctx->cnts.curbytes -= usize; 250 251 if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) 252 prof_tctx_destroy(tsd, tctx); 253 else 254 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); 255} 256 257void 258bt_init(prof_bt_t *bt, void **vec) 259{ 260 cassert(config_prof); 261 262 bt->vec = vec; 263 bt->len = 0; 264} 265 266JEMALLOC_INLINE_C void 267prof_enter(tsd_t *tsd, prof_tdata_t *tdata) 268{ 269 cassert(config_prof); 270 assert(tdata == prof_tdata_get(tsd, false)); 271 272 if (tdata != NULL) { 273 assert(!tdata->enq); 274 tdata->enq = true; 275 } 276 277 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); 278} 279 280JEMALLOC_INLINE_C void 281prof_leave(tsd_t *tsd, prof_tdata_t *tdata) 282{ 283 cassert(config_prof); 284 assert(tdata == prof_tdata_get(tsd, false)); 285 286 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); 287 288 if (tdata != NULL) { 289 bool idump, gdump; 290 291 assert(tdata->enq); 292 tdata->enq = false; 293 idump = tdata->enq_idump; 294 tdata->enq_idump = false; 295 gdump = tdata->enq_gdump; 296 tdata->enq_gdump = false; 297 298 if (idump) 299 prof_idump(tsd_tsdn(tsd)); 300 if (gdump) 301 prof_gdump(tsd_tsdn(tsd)); 302 } 303} 304 305#ifdef JEMALLOC_PROF_LIBUNWIND 306void 307prof_backtrace(prof_bt_t *bt) 308{ 309 int nframes; 310 311 cassert(config_prof); 312 assert(bt->len == 0); 313 assert(bt->vec != NULL); 314 315 nframes = unw_backtrace(bt->vec, PROF_BT_MAX); 316 if (nframes <= 0) 317 return; 318 bt->len = nframes; 319} 320#elif (defined(JEMALLOC_PROF_LIBGCC)) 321static _Unwind_Reason_Code 322prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) 323{ 324 cassert(config_prof); 325 326 return (_URC_NO_REASON); 327} 328 329static _Unwind_Reason_Code 330prof_unwind_callback(struct _Unwind_Context *context, void *arg) 331{ 332 prof_unwind_data_t *data = (prof_unwind_data_t *)arg; 333 void *ip; 334 335 cassert(config_prof); 336 337 ip = (void *)_Unwind_GetIP(context); 338 if (ip == NULL) 339 return (_URC_END_OF_STACK); 340 data->bt->vec[data->bt->len] = ip; 341 data->bt->len++; 342 if (data->bt->len == data->max) 343 return (_URC_END_OF_STACK); 344 345 return (_URC_NO_REASON); 346} 347 348void 349prof_backtrace(prof_bt_t *bt) 350{ 351 prof_unwind_data_t data = {bt, PROF_BT_MAX}; 352 353 cassert(config_prof); 354 355 _Unwind_Backtrace(prof_unwind_callback, &data); 356} 357#elif (defined(JEMALLOC_PROF_GCC)) 358void 359prof_backtrace(prof_bt_t *bt) 360{ 361#define BT_FRAME(i) \ 362 if ((i) < PROF_BT_MAX) { \ 363 void *p; \ 364 if (__builtin_frame_address(i) == 0) \ 365 return; \ 366 p = __builtin_return_address(i); \ 367 if (p == NULL) \ 368 return; \ 369 bt->vec[(i)] = p; \ 370 bt->len = (i) + 1; \ 371 } else \ 372 return; 373 374 cassert(config_prof); 375 376 BT_FRAME(0) 377 BT_FRAME(1) 378 BT_FRAME(2) 379 BT_FRAME(3) 380 BT_FRAME(4) 381 BT_FRAME(5) 382 BT_FRAME(6) 383 BT_FRAME(7) 384 BT_FRAME(8) 385 BT_FRAME(9) 386 387 BT_FRAME(10) 388 BT_FRAME(11) 389 BT_FRAME(12) 390 BT_FRAME(13) 391 BT_FRAME(14) 392 BT_FRAME(15) 393 BT_FRAME(16) 394 BT_FRAME(17) 395 BT_FRAME(18) 396 BT_FRAME(19) 397 398 BT_FRAME(20) 399 BT_FRAME(21) 400 BT_FRAME(22) 401 BT_FRAME(23) 402 BT_FRAME(24) 403 BT_FRAME(25) 404 BT_FRAME(26) 405 BT_FRAME(27) 406 BT_FRAME(28) 407 BT_FRAME(29) 408 409 BT_FRAME(30) 410 BT_FRAME(31) 411 BT_FRAME(32) 412 BT_FRAME(33) 413 BT_FRAME(34) 414 BT_FRAME(35) 415 BT_FRAME(36) 416 BT_FRAME(37) 417 BT_FRAME(38) 418 BT_FRAME(39) 419 420 BT_FRAME(40) 421 BT_FRAME(41) 422 BT_FRAME(42) 423 BT_FRAME(43) 424 BT_FRAME(44) 425 BT_FRAME(45) 426 BT_FRAME(46) 427 BT_FRAME(47) 428 BT_FRAME(48) 429 BT_FRAME(49) 430 431 BT_FRAME(50) 432 BT_FRAME(51) 433 BT_FRAME(52) 434 BT_FRAME(53) 435 BT_FRAME(54) 436 BT_FRAME(55) 437 BT_FRAME(56) 438 BT_FRAME(57) 439 BT_FRAME(58) 440 BT_FRAME(59) 441 442 BT_FRAME(60) 443 BT_FRAME(61) 444 BT_FRAME(62) 445 BT_FRAME(63) 446 BT_FRAME(64) 447 BT_FRAME(65) 448 BT_FRAME(66) 449 BT_FRAME(67) 450 BT_FRAME(68) 451 BT_FRAME(69) 452 453 BT_FRAME(70) 454 BT_FRAME(71) 455 BT_FRAME(72) 456 BT_FRAME(73) 457 BT_FRAME(74) 458 BT_FRAME(75) 459 BT_FRAME(76) 460 BT_FRAME(77) 461 BT_FRAME(78) 462 BT_FRAME(79) 463 464 BT_FRAME(80) 465 BT_FRAME(81) 466 BT_FRAME(82) 467 BT_FRAME(83) 468 BT_FRAME(84) 469 BT_FRAME(85) 470 BT_FRAME(86) 471 BT_FRAME(87) 472 BT_FRAME(88) 473 BT_FRAME(89) 474 475 BT_FRAME(90) 476 BT_FRAME(91) 477 BT_FRAME(92) 478 BT_FRAME(93) 479 BT_FRAME(94) 480 BT_FRAME(95) 481 BT_FRAME(96) 482 BT_FRAME(97) 483 BT_FRAME(98) 484 BT_FRAME(99) 485 486 BT_FRAME(100) 487 BT_FRAME(101) 488 BT_FRAME(102) 489 BT_FRAME(103) 490 BT_FRAME(104) 491 BT_FRAME(105) 492 BT_FRAME(106) 493 BT_FRAME(107) 494 BT_FRAME(108) 495 BT_FRAME(109) 496 497 BT_FRAME(110) 498 BT_FRAME(111) 499 BT_FRAME(112) 500 BT_FRAME(113) 501 BT_FRAME(114) 502 BT_FRAME(115) 503 BT_FRAME(116) 504 BT_FRAME(117) 505 BT_FRAME(118) 506 BT_FRAME(119) 507 508 BT_FRAME(120) 509 BT_FRAME(121) 510 BT_FRAME(122) 511 BT_FRAME(123) 512 BT_FRAME(124) 513 BT_FRAME(125) 514 BT_FRAME(126) 515 BT_FRAME(127) 516#undef BT_FRAME 517} 518#else 519void 520prof_backtrace(prof_bt_t *bt) 521{ 522 cassert(config_prof); 523 not_reached(); 524} 525#endif 526 527static malloc_mutex_t * 528prof_gctx_mutex_choose(void) 529{ 530 unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); 531 532 return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); 533} 534 535static malloc_mutex_t * 536prof_tdata_mutex_choose(uint64_t thr_uid) 537{ 538 return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); 539} 540 541static prof_gctx_t * 542prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) 543{ 544 /* 545 * Create a single allocation that has space for vec of length bt->len. 546 */ 547 size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *)); 548 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size, 549 size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true), 550 true); 551 if (gctx == NULL) 552 return (NULL); 553 gctx->lock = prof_gctx_mutex_choose(); 554 /* 555 * Set nlimbo to 1, in order to avoid a race condition with 556 * prof_tctx_destroy()/prof_gctx_try_destroy(). 557 */ 558 gctx->nlimbo = 1; 559 tctx_tree_new(&gctx->tctxs); 560 /* Duplicate bt. */ 561 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); 562 gctx->bt.vec = gctx->vec; 563 gctx->bt.len = bt->len; 564 return (gctx); 565} 566 567static void 568prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, 569 prof_tdata_t *tdata) 570{ 571 cassert(config_prof); 572 573 /* 574 * Check that gctx is still unused by any thread cache before destroying 575 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race 576 * condition with this function, as does prof_tctx_destroy() in order to 577 * avoid a race between the main body of prof_tctx_destroy() and entry 578 * into this function. 579 */ 580 prof_enter(tsd, tdata_self); 581 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 582 assert(gctx->nlimbo != 0); 583 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { 584 /* Remove gctx from bt2gctx. */ 585 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) 586 not_reached(); 587 prof_leave(tsd, tdata_self); 588 /* Destroy gctx. */ 589 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 590 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), gctx), gctx, 591 NULL, true, true); 592 } else { 593 /* 594 * Compensate for increment in prof_tctx_destroy() or 595 * prof_lookup(). 596 */ 597 gctx->nlimbo--; 598 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 599 prof_leave(tsd, tdata_self); 600 } 601} 602 603static bool 604prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) 605{ 606 malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); 607 608 if (opt_prof_accum) 609 return (false); 610 if (tctx->cnts.curobjs != 0) 611 return (false); 612 if (tctx->prepared) 613 return (false); 614 return (true); 615} 616 617static bool 618prof_gctx_should_destroy(prof_gctx_t *gctx) 619{ 620 if (opt_prof_accum) 621 return (false); 622 if (!tctx_tree_empty(&gctx->tctxs)) 623 return (false); 624 if (gctx->nlimbo != 0) 625 return (false); 626 return (true); 627} 628 629static void 630prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) 631{ 632 prof_tdata_t *tdata = tctx->tdata; 633 prof_gctx_t *gctx = tctx->gctx; 634 bool destroy_tdata, destroy_tctx, destroy_gctx; 635 636 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); 637 638 assert(tctx->cnts.curobjs == 0); 639 assert(tctx->cnts.curbytes == 0); 640 assert(!opt_prof_accum); 641 assert(tctx->cnts.accumobjs == 0); 642 assert(tctx->cnts.accumbytes == 0); 643 644 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); 645 destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false); 646 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 647 648 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 649 switch (tctx->state) { 650 case prof_tctx_state_nominal: 651 tctx_tree_remove(&gctx->tctxs, tctx); 652 destroy_tctx = true; 653 if (prof_gctx_should_destroy(gctx)) { 654 /* 655 * Increment gctx->nlimbo in order to keep another 656 * thread from winning the race to destroy gctx while 657 * this one has gctx->lock dropped. Without this, it 658 * would be possible for another thread to: 659 * 660 * 1) Sample an allocation associated with gctx. 661 * 2) Deallocate the sampled object. 662 * 3) Successfully prof_gctx_try_destroy(gctx). 663 * 664 * The result would be that gctx no longer exists by the 665 * time this thread accesses it in 666 * prof_gctx_try_destroy(). 667 */ 668 gctx->nlimbo++; 669 destroy_gctx = true; 670 } else 671 destroy_gctx = false; 672 break; 673 case prof_tctx_state_dumping: 674 /* 675 * A dumping thread needs tctx to remain valid until dumping 676 * has finished. Change state such that the dumping thread will 677 * complete destruction during a late dump iteration phase. 678 */ 679 tctx->state = prof_tctx_state_purgatory; 680 destroy_tctx = false; 681 destroy_gctx = false; 682 break; 683 default: 684 not_reached(); 685 destroy_tctx = false; 686 destroy_gctx = false; 687 } 688 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 689 if (destroy_gctx) { 690 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, 691 tdata); 692 } 693 694 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock); 695 696 if (destroy_tdata) 697 prof_tdata_destroy(tsd, tdata, false); 698 699 if (destroy_tctx) 700 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tctx), tctx, 701 NULL, true, true); 702} 703 704static bool 705prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, 706 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) 707{ 708 union { 709 prof_gctx_t *p; 710 void *v; 711 } gctx; 712 union { 713 prof_bt_t *p; 714 void *v; 715 } btkey; 716 bool new_gctx; 717 718 prof_enter(tsd, tdata); 719 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { 720 /* bt has never been seen before. Insert it. */ 721 gctx.p = prof_gctx_create(tsd_tsdn(tsd), bt); 722 if (gctx.v == NULL) { 723 prof_leave(tsd, tdata); 724 return (true); 725 } 726 btkey.p = &gctx.p->bt; 727 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { 728 /* OOM. */ 729 prof_leave(tsd, tdata); 730 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), gctx.v), 731 gctx.v, NULL, true, true); 732 return (true); 733 } 734 new_gctx = true; 735 } else { 736 /* 737 * Increment nlimbo, in order to avoid a race condition with 738 * prof_tctx_destroy()/prof_gctx_try_destroy(). 739 */ 740 malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock); 741 gctx.p->nlimbo++; 742 malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock); 743 new_gctx = false; 744 } 745 prof_leave(tsd, tdata); 746 747 *p_btkey = btkey.v; 748 *p_gctx = gctx.p; 749 *p_new_gctx = new_gctx; 750 return (false); 751} 752 753prof_tctx_t * 754prof_lookup(tsd_t *tsd, prof_bt_t *bt) 755{ 756 union { 757 prof_tctx_t *p; 758 void *v; 759 } ret; 760 prof_tdata_t *tdata; 761 bool not_found; 762 763 cassert(config_prof); 764 765 tdata = prof_tdata_get(tsd, false); 766 if (tdata == NULL) 767 return (NULL); 768 769 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); 770 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); 771 if (!not_found) /* Note double negative! */ 772 ret.p->prepared = true; 773 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 774 if (not_found) { 775 void *btkey; 776 prof_gctx_t *gctx; 777 bool new_gctx, error; 778 779 /* 780 * This thread's cache lacks bt. Look for it in the global 781 * cache. 782 */ 783 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, 784 &new_gctx)) 785 return (NULL); 786 787 /* Link a prof_tctx_t into gctx for this thread. */ 788 ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t), 789 size2index(sizeof(prof_tctx_t)), false, NULL, true, 790 arena_ichoose(tsd, NULL), true); 791 if (ret.p == NULL) { 792 if (new_gctx) 793 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 794 return (NULL); 795 } 796 ret.p->tdata = tdata; 797 ret.p->thr_uid = tdata->thr_uid; 798 ret.p->thr_discrim = tdata->thr_discrim; 799 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); 800 ret.p->gctx = gctx; 801 ret.p->tctx_uid = tdata->tctx_uid_next++; 802 ret.p->prepared = true; 803 ret.p->state = prof_tctx_state_initializing; 804 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); 805 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v); 806 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 807 if (error) { 808 if (new_gctx) 809 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 810 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), ret.v), 811 ret.v, NULL, true, true); 812 return (NULL); 813 } 814 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 815 ret.p->state = prof_tctx_state_nominal; 816 tctx_tree_insert(&gctx->tctxs, ret.p); 817 gctx->nlimbo--; 818 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 819 } 820 821 return (ret.p); 822} 823 824/* 825 * The bodies of this function and prof_leakcheck() are compiled out unless heap 826 * profiling is enabled, so that it is possible to compile jemalloc with 827 * floating point support completely disabled. Avoiding floating point code is 828 * important on memory-constrained systems, but it also enables a workaround for 829 * versions of glibc that don't properly save/restore floating point registers 830 * during dynamic lazy symbol loading (which internally calls into whatever 831 * malloc implementation happens to be integrated into the application). Note 832 * that some compilers (e.g. gcc 4.8) may use floating point registers for fast 833 * memory moves, so jemalloc must be compiled with such optimizations disabled 834 * (e.g. 835 * -mno-sse) in order for the workaround to be complete. 836 */ 837void 838prof_sample_threshold_update(prof_tdata_t *tdata) 839{ 840#ifdef JEMALLOC_PROF 841 uint64_t r; 842 double u; 843 844 if (!config_prof) 845 return; 846 847 if (lg_prof_sample == 0) { 848 tdata->bytes_until_sample = 0; 849 return; 850 } 851 852 /* 853 * Compute sample interval as a geometrically distributed random 854 * variable with mean (2^lg_prof_sample). 855 * 856 * __ __ 857 * | log(u) | 1 858 * tdata->bytes_until_sample = | -------- |, where p = --------------- 859 * | log(1-p) | lg_prof_sample 860 * 2 861 * 862 * For more information on the math, see: 863 * 864 * Non-Uniform Random Variate Generation 865 * Luc Devroye 866 * Springer-Verlag, New York, 1986 867 * pp 500 868 * (http://luc.devroye.org/rnbookindex.html) 869 */ 870 r = prng_lg_range_u64(&tdata->prng_state, 53); 871 u = (double)r * (1.0/9007199254740992.0L); 872 tdata->bytes_until_sample = (uint64_t)(log(u) / 873 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) 874 + (uint64_t)1U; 875#endif 876} 877 878#ifdef JEMALLOC_JET 879static prof_tdata_t * 880prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 881{ 882 size_t *tdata_count = (size_t *)arg; 883 884 (*tdata_count)++; 885 886 return (NULL); 887} 888 889size_t 890prof_tdata_count(void) 891{ 892 size_t tdata_count = 0; 893 tsdn_t *tsdn; 894 895 tsdn = tsdn_fetch(); 896 malloc_mutex_lock(tsdn, &tdatas_mtx); 897 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, 898 (void *)&tdata_count); 899 malloc_mutex_unlock(tsdn, &tdatas_mtx); 900 901 return (tdata_count); 902} 903#endif 904 905#ifdef JEMALLOC_JET 906size_t 907prof_bt_count(void) 908{ 909 size_t bt_count; 910 tsd_t *tsd; 911 prof_tdata_t *tdata; 912 913 tsd = tsd_fetch(); 914 tdata = prof_tdata_get(tsd, false); 915 if (tdata == NULL) 916 return (0); 917 918 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); 919 bt_count = ckh_count(&bt2gctx); 920 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); 921 922 return (bt_count); 923} 924#endif 925 926#ifdef JEMALLOC_JET 927#undef prof_dump_open 928#define prof_dump_open JEMALLOC_N(prof_dump_open_impl) 929#endif 930static int 931prof_dump_open(bool propagate_err, const char *filename) 932{ 933 int fd; 934 935 fd = creat(filename, 0644); 936 if (fd == -1 && !propagate_err) { 937 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n", 938 filename); 939 if (opt_abort) 940 abort(); 941 } 942 943 return (fd); 944} 945#ifdef JEMALLOC_JET 946#undef prof_dump_open 947#define prof_dump_open JEMALLOC_N(prof_dump_open) 948prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); 949#endif 950 951static bool 952prof_dump_flush(bool propagate_err) 953{ 954 bool ret = false; 955 ssize_t err; 956 957 cassert(config_prof); 958 959 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); 960 if (err == -1) { 961 if (!propagate_err) { 962 malloc_write("<jemalloc>: write() failed during heap " 963 "profile flush\n"); 964 if (opt_abort) 965 abort(); 966 } 967 ret = true; 968 } 969 prof_dump_buf_end = 0; 970 971 return (ret); 972} 973 974static bool 975prof_dump_close(bool propagate_err) 976{ 977 bool ret; 978 979 assert(prof_dump_fd != -1); 980 ret = prof_dump_flush(propagate_err); 981 close(prof_dump_fd); 982 prof_dump_fd = -1; 983 984 return (ret); 985} 986 987static bool 988prof_dump_write(bool propagate_err, const char *s) 989{ 990 size_t i, slen, n; 991 992 cassert(config_prof); 993 994 i = 0; 995 slen = strlen(s); 996 while (i < slen) { 997 /* Flush the buffer if it is full. */ 998 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) 999 if (prof_dump_flush(propagate_err) && propagate_err) 1000 return (true); 1001 1002 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { 1003 /* Finish writing. */ 1004 n = slen - i; 1005 } else { 1006 /* Write as much of s as will fit. */ 1007 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; 1008 } 1009 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); 1010 prof_dump_buf_end += n; 1011 i += n; 1012 } 1013 1014 return (false); 1015} 1016 1017JEMALLOC_FORMAT_PRINTF(2, 3) 1018static bool 1019prof_dump_printf(bool propagate_err, const char *format, ...) 1020{ 1021 bool ret; 1022 va_list ap; 1023 char buf[PROF_PRINTF_BUFSIZE]; 1024 1025 va_start(ap, format); 1026 malloc_vsnprintf(buf, sizeof(buf), format, ap); 1027 va_end(ap); 1028 ret = prof_dump_write(propagate_err, buf); 1029 1030 return (ret); 1031} 1032 1033static void 1034prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) 1035{ 1036 malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); 1037 1038 malloc_mutex_lock(tsdn, tctx->gctx->lock); 1039 1040 switch (tctx->state) { 1041 case prof_tctx_state_initializing: 1042 malloc_mutex_unlock(tsdn, tctx->gctx->lock); 1043 return; 1044 case prof_tctx_state_nominal: 1045 tctx->state = prof_tctx_state_dumping; 1046 malloc_mutex_unlock(tsdn, tctx->gctx->lock); 1047 1048 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); 1049 1050 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1051 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1052 if (opt_prof_accum) { 1053 tdata->cnt_summed.accumobjs += 1054 tctx->dump_cnts.accumobjs; 1055 tdata->cnt_summed.accumbytes += 1056 tctx->dump_cnts.accumbytes; 1057 } 1058 break; 1059 case prof_tctx_state_dumping: 1060 case prof_tctx_state_purgatory: 1061 not_reached(); 1062 } 1063} 1064 1065static void 1066prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) 1067{ 1068 malloc_mutex_assert_owner(tsdn, gctx->lock); 1069 1070 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1071 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1072 if (opt_prof_accum) { 1073 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; 1074 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; 1075 } 1076} 1077 1078static prof_tctx_t * 1079prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1080{ 1081 tsdn_t *tsdn = (tsdn_t *)arg; 1082 1083 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); 1084 1085 switch (tctx->state) { 1086 case prof_tctx_state_nominal: 1087 /* New since dumping started; ignore. */ 1088 break; 1089 case prof_tctx_state_dumping: 1090 case prof_tctx_state_purgatory: 1091 prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx); 1092 break; 1093 default: 1094 not_reached(); 1095 } 1096 1097 return (NULL); 1098} 1099 1100struct prof_tctx_dump_iter_arg_s { 1101 tsdn_t *tsdn; 1102 bool propagate_err; 1103}; 1104 1105static prof_tctx_t * 1106prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) 1107{ 1108 struct prof_tctx_dump_iter_arg_s *arg = 1109 (struct prof_tctx_dump_iter_arg_s *)opaque; 1110 1111 malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock); 1112 1113 switch (tctx->state) { 1114 case prof_tctx_state_initializing: 1115 case prof_tctx_state_nominal: 1116 /* Not captured by this dump. */ 1117 break; 1118 case prof_tctx_state_dumping: 1119 case prof_tctx_state_purgatory: 1120 if (prof_dump_printf(arg->propagate_err, 1121 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": " 1122 "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs, 1123 tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, 1124 tctx->dump_cnts.accumbytes)) 1125 return (tctx); 1126 break; 1127 default: 1128 not_reached(); 1129 } 1130 return (NULL); 1131} 1132 1133static prof_tctx_t * 1134prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1135{ 1136 tsdn_t *tsdn = (tsdn_t *)arg; 1137 prof_tctx_t *ret; 1138 1139 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); 1140 1141 switch (tctx->state) { 1142 case prof_tctx_state_nominal: 1143 /* New since dumping started; ignore. */ 1144 break; 1145 case prof_tctx_state_dumping: 1146 tctx->state = prof_tctx_state_nominal; 1147 break; 1148 case prof_tctx_state_purgatory: 1149 ret = tctx; 1150 goto label_return; 1151 default: 1152 not_reached(); 1153 } 1154 1155 ret = NULL; 1156label_return: 1157 return (ret); 1158} 1159 1160static void 1161prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) 1162{ 1163 cassert(config_prof); 1164 1165 malloc_mutex_lock(tsdn, gctx->lock); 1166 1167 /* 1168 * Increment nlimbo so that gctx won't go away before dump. 1169 * Additionally, link gctx into the dump list so that it is included in 1170 * prof_dump()'s second pass. 1171 */ 1172 gctx->nlimbo++; 1173 gctx_tree_insert(gctxs, gctx); 1174 1175 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); 1176 1177 malloc_mutex_unlock(tsdn, gctx->lock); 1178} 1179 1180struct prof_gctx_merge_iter_arg_s { 1181 tsdn_t *tsdn; 1182 size_t leak_ngctx; 1183}; 1184 1185static prof_gctx_t * 1186prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) 1187{ 1188 struct prof_gctx_merge_iter_arg_s *arg = 1189 (struct prof_gctx_merge_iter_arg_s *)opaque; 1190 1191 malloc_mutex_lock(arg->tsdn, gctx->lock); 1192 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, 1193 (void *)arg->tsdn); 1194 if (gctx->cnt_summed.curobjs != 0) 1195 arg->leak_ngctx++; 1196 malloc_mutex_unlock(arg->tsdn, gctx->lock); 1197 1198 return (NULL); 1199} 1200 1201static void 1202prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) 1203{ 1204 prof_tdata_t *tdata = prof_tdata_get(tsd, false); 1205 prof_gctx_t *gctx; 1206 1207 /* 1208 * Standard tree iteration won't work here, because as soon as we 1209 * decrement gctx->nlimbo and unlock gctx, another thread can 1210 * concurrently destroy it, which will corrupt the tree. Therefore, 1211 * tear down the tree one node at a time during iteration. 1212 */ 1213 while ((gctx = gctx_tree_first(gctxs)) != NULL) { 1214 gctx_tree_remove(gctxs, gctx); 1215 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 1216 { 1217 prof_tctx_t *next; 1218 1219 next = NULL; 1220 do { 1221 prof_tctx_t *to_destroy = 1222 tctx_tree_iter(&gctx->tctxs, next, 1223 prof_tctx_finish_iter, 1224 (void *)tsd_tsdn(tsd)); 1225 if (to_destroy != NULL) { 1226 next = tctx_tree_next(&gctx->tctxs, 1227 to_destroy); 1228 tctx_tree_remove(&gctx->tctxs, 1229 to_destroy); 1230 idalloctm(tsd_tsdn(tsd), 1231 iealloc(tsd_tsdn(tsd), to_destroy), 1232 to_destroy, NULL, true, true); 1233 } else 1234 next = NULL; 1235 } while (next != NULL); 1236 } 1237 gctx->nlimbo--; 1238 if (prof_gctx_should_destroy(gctx)) { 1239 gctx->nlimbo++; 1240 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 1241 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 1242 } else 1243 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 1244 } 1245} 1246 1247struct prof_tdata_merge_iter_arg_s { 1248 tsdn_t *tsdn; 1249 prof_cnt_t cnt_all; 1250}; 1251 1252static prof_tdata_t * 1253prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, 1254 void *opaque) 1255{ 1256 struct prof_tdata_merge_iter_arg_s *arg = 1257 (struct prof_tdata_merge_iter_arg_s *)opaque; 1258 1259 malloc_mutex_lock(arg->tsdn, tdata->lock); 1260 if (!tdata->expired) { 1261 size_t tabind; 1262 union { 1263 prof_tctx_t *p; 1264 void *v; 1265 } tctx; 1266 1267 tdata->dumping = true; 1268 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); 1269 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, 1270 &tctx.v);) 1271 prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata); 1272 1273 arg->cnt_all.curobjs += tdata->cnt_summed.curobjs; 1274 arg->cnt_all.curbytes += tdata->cnt_summed.curbytes; 1275 if (opt_prof_accum) { 1276 arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs; 1277 arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes; 1278 } 1279 } else 1280 tdata->dumping = false; 1281 malloc_mutex_unlock(arg->tsdn, tdata->lock); 1282 1283 return (NULL); 1284} 1285 1286static prof_tdata_t * 1287prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1288{ 1289 bool propagate_err = *(bool *)arg; 1290 1291 if (!tdata->dumping) 1292 return (NULL); 1293 1294 if (prof_dump_printf(propagate_err, 1295 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n", 1296 tdata->thr_uid, tdata->cnt_summed.curobjs, 1297 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, 1298 tdata->cnt_summed.accumbytes, 1299 (tdata->thread_name != NULL) ? " " : "", 1300 (tdata->thread_name != NULL) ? tdata->thread_name : "")) 1301 return (tdata); 1302 return (NULL); 1303} 1304 1305#ifdef JEMALLOC_JET 1306#undef prof_dump_header 1307#define prof_dump_header JEMALLOC_N(prof_dump_header_impl) 1308#endif 1309static bool 1310prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) 1311{ 1312 bool ret; 1313 1314 if (prof_dump_printf(propagate_err, 1315 "heap_v2/%"FMTu64"\n" 1316 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1317 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, 1318 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) 1319 return (true); 1320 1321 malloc_mutex_lock(tsdn, &tdatas_mtx); 1322 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, 1323 (void *)&propagate_err) != NULL); 1324 malloc_mutex_unlock(tsdn, &tdatas_mtx); 1325 return (ret); 1326} 1327#ifdef JEMALLOC_JET 1328#undef prof_dump_header 1329#define prof_dump_header JEMALLOC_N(prof_dump_header) 1330prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); 1331#endif 1332 1333static bool 1334prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx, 1335 const prof_bt_t *bt, prof_gctx_tree_t *gctxs) 1336{ 1337 bool ret; 1338 unsigned i; 1339 struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg; 1340 1341 cassert(config_prof); 1342 malloc_mutex_assert_owner(tsdn, gctx->lock); 1343 1344 /* Avoid dumping such gctx's that have no useful data. */ 1345 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || 1346 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { 1347 assert(gctx->cnt_summed.curobjs == 0); 1348 assert(gctx->cnt_summed.curbytes == 0); 1349 assert(gctx->cnt_summed.accumobjs == 0); 1350 assert(gctx->cnt_summed.accumbytes == 0); 1351 ret = false; 1352 goto label_return; 1353 } 1354 1355 if (prof_dump_printf(propagate_err, "@")) { 1356 ret = true; 1357 goto label_return; 1358 } 1359 for (i = 0; i < bt->len; i++) { 1360 if (prof_dump_printf(propagate_err, " %#"FMTxPTR, 1361 (uintptr_t)bt->vec[i])) { 1362 ret = true; 1363 goto label_return; 1364 } 1365 } 1366 1367 if (prof_dump_printf(propagate_err, 1368 "\n" 1369 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1370 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, 1371 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { 1372 ret = true; 1373 goto label_return; 1374 } 1375 1376 prof_tctx_dump_iter_arg.tsdn = tsdn; 1377 prof_tctx_dump_iter_arg.propagate_err = propagate_err; 1378 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, 1379 (void *)&prof_tctx_dump_iter_arg) != NULL) { 1380 ret = true; 1381 goto label_return; 1382 } 1383 1384 ret = false; 1385label_return: 1386 return (ret); 1387} 1388 1389#ifndef _WIN32 1390JEMALLOC_FORMAT_PRINTF(1, 2) 1391static int 1392prof_open_maps(const char *format, ...) 1393{ 1394 int mfd; 1395 va_list ap; 1396 char filename[PATH_MAX + 1]; 1397 1398 va_start(ap, format); 1399 malloc_vsnprintf(filename, sizeof(filename), format, ap); 1400 va_end(ap); 1401 mfd = open(filename, O_RDONLY); 1402 1403 return (mfd); 1404} 1405#endif 1406 1407static int 1408prof_getpid(void) 1409{ 1410#ifdef _WIN32 1411 return (GetCurrentProcessId()); 1412#else 1413 return (getpid()); 1414#endif 1415} 1416 1417static bool 1418prof_dump_maps(bool propagate_err) 1419{ 1420 bool ret; 1421 int mfd; 1422 1423 cassert(config_prof); 1424#ifdef __FreeBSD__ 1425 mfd = prof_open_maps("/proc/curproc/map"); 1426#elif defined(_WIN32) 1427 mfd = -1; // Not implemented 1428#else 1429 { 1430 int pid = prof_getpid(); 1431 1432 mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); 1433 if (mfd == -1) 1434 mfd = prof_open_maps("/proc/%d/maps", pid); 1435 } 1436#endif 1437 if (mfd != -1) { 1438 ssize_t nread; 1439 1440 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && 1441 propagate_err) { 1442 ret = true; 1443 goto label_return; 1444 } 1445 nread = 0; 1446 do { 1447 prof_dump_buf_end += nread; 1448 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { 1449 /* Make space in prof_dump_buf before read(). */ 1450 if (prof_dump_flush(propagate_err) && 1451 propagate_err) { 1452 ret = true; 1453 goto label_return; 1454 } 1455 } 1456 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], 1457 PROF_DUMP_BUFSIZE - prof_dump_buf_end); 1458 } while (nread > 0); 1459 } else { 1460 ret = true; 1461 goto label_return; 1462 } 1463 1464 ret = false; 1465label_return: 1466 if (mfd != -1) 1467 close(mfd); 1468 return (ret); 1469} 1470 1471/* 1472 * See prof_sample_threshold_update() comment for why the body of this function 1473 * is conditionally compiled. 1474 */ 1475static void 1476prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, 1477 const char *filename) 1478{ 1479#ifdef JEMALLOC_PROF 1480 /* 1481 * Scaling is equivalent AdjustSamples() in jeprof, but the result may 1482 * differ slightly from what jeprof reports, because here we scale the 1483 * summary values, whereas jeprof scales each context individually and 1484 * reports the sums of the scaled values. 1485 */ 1486 if (cnt_all->curbytes != 0) { 1487 double sample_period = (double)((uint64_t)1 << lg_prof_sample); 1488 double ratio = (((double)cnt_all->curbytes) / 1489 (double)cnt_all->curobjs) / sample_period; 1490 double scale_factor = 1.0 / (1.0 - exp(-ratio)); 1491 uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes) 1492 * scale_factor); 1493 uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) * 1494 scale_factor); 1495 1496 malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64 1497 " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n", 1498 curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs != 1499 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : ""); 1500 malloc_printf( 1501 "<jemalloc>: Run jeprof on \"%s\" for leak detail\n", 1502 filename); 1503 } 1504#endif 1505} 1506 1507struct prof_gctx_dump_iter_arg_s { 1508 tsdn_t *tsdn; 1509 bool propagate_err; 1510}; 1511 1512static prof_gctx_t * 1513prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) 1514{ 1515 prof_gctx_t *ret; 1516 struct prof_gctx_dump_iter_arg_s *arg = 1517 (struct prof_gctx_dump_iter_arg_s *)opaque; 1518 1519 malloc_mutex_lock(arg->tsdn, gctx->lock); 1520 1521 if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt, 1522 gctxs)) { 1523 ret = gctx; 1524 goto label_return; 1525 } 1526 1527 ret = NULL; 1528label_return: 1529 malloc_mutex_unlock(arg->tsdn, gctx->lock); 1530 return (ret); 1531} 1532 1533static void 1534prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata, 1535 struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg, 1536 struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg, 1537 prof_gctx_tree_t *gctxs) 1538{ 1539 size_t tabind; 1540 union { 1541 prof_gctx_t *p; 1542 void *v; 1543 } gctx; 1544 1545 prof_enter(tsd, tdata); 1546 1547 /* 1548 * Put gctx's in limbo and clear their counters in preparation for 1549 * summing. 1550 */ 1551 gctx_tree_new(gctxs); 1552 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) { 1553 prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs); 1554 } 1555 1556 /* 1557 * Iterate over tdatas, and for the non-expired ones snapshot their tctx 1558 * stats and merge them into the associated gctx's. 1559 */ 1560 prof_tdata_merge_iter_arg->tsdn = tsd_tsdn(tsd); 1561 memset(&prof_tdata_merge_iter_arg->cnt_all, 0, sizeof(prof_cnt_t)); 1562 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); 1563 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, 1564 (void *)prof_tdata_merge_iter_arg); 1565 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); 1566 1567 /* Merge tctx stats into gctx's. */ 1568 prof_gctx_merge_iter_arg->tsdn = tsd_tsdn(tsd); 1569 prof_gctx_merge_iter_arg->leak_ngctx = 0; 1570 gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter, 1571 (void *)prof_gctx_merge_iter_arg); 1572 1573 prof_leave(tsd, tdata); 1574} 1575 1576static bool 1577prof_dump_file(tsd_t *tsd, bool propagate_err, const char *filename, 1578 bool leakcheck, prof_tdata_t *tdata, 1579 struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg, 1580 struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg, 1581 struct prof_gctx_dump_iter_arg_s *prof_gctx_dump_iter_arg, 1582 prof_gctx_tree_t *gctxs) 1583{ 1584 /* Create dump file. */ 1585 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) { 1586 return true; 1587 } 1588 1589 /* Dump profile header. */ 1590 if (prof_dump_header(tsd_tsdn(tsd), propagate_err, 1591 &prof_tdata_merge_iter_arg->cnt_all)) { 1592 goto label_write_error; 1593 } 1594 1595 /* Dump per gctx profile stats. */ 1596 prof_gctx_dump_iter_arg->tsdn = tsd_tsdn(tsd); 1597 prof_gctx_dump_iter_arg->propagate_err = propagate_err; 1598 if (gctx_tree_iter(gctxs, NULL, prof_gctx_dump_iter, 1599 (void *)prof_gctx_dump_iter_arg) != NULL) { 1600 goto label_write_error; 1601 } 1602 1603 /* Dump /proc/<pid>/maps if possible. */ 1604 if (prof_dump_maps(propagate_err)) { 1605 goto label_write_error; 1606 } 1607 1608 if (prof_dump_close(propagate_err)) { 1609 return true; 1610 } 1611 1612 return false; 1613label_write_error: 1614 prof_dump_close(propagate_err); 1615 return true; 1616} 1617 1618static bool 1619prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) 1620{ 1621 prof_tdata_t *tdata; 1622 struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg; 1623 struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg; 1624 struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg; 1625 prof_gctx_tree_t gctxs; 1626 bool err; 1627 1628 cassert(config_prof); 1629 1630 tdata = prof_tdata_get(tsd, true); 1631 if (tdata == NULL) { 1632 return true; 1633 } 1634 1635 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); 1636 1637 prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg, 1638 &prof_gctx_merge_iter_arg, &gctxs); 1639 err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata, 1640 &prof_tdata_merge_iter_arg, &prof_gctx_merge_iter_arg, 1641 &prof_gctx_dump_iter_arg, &gctxs); 1642 prof_gctx_finish(tsd, &gctxs); 1643 1644 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); 1645 1646 if (err) { 1647 return true; 1648 } 1649 1650 if (leakcheck) { 1651 prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all, 1652 prof_gctx_merge_iter_arg.leak_ngctx, filename); 1653 } 1654 return false; 1655} 1656 1657#ifdef JEMALLOC_JET 1658void 1659prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs, 1660 uint64_t *accumbytes) 1661{ 1662 tsd_t *tsd; 1663 prof_tdata_t *tdata; 1664 struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg; 1665 struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg; 1666 prof_gctx_tree_t gctxs; 1667 1668 tsd = tsd_fetch(); 1669 tdata = prof_tdata_get(tsd, false); 1670 if (tdata == NULL) { 1671 if (curobjs != NULL) { 1672 *curobjs = 0; 1673 } 1674 if (curbytes != NULL) { 1675 *curbytes = 0; 1676 } 1677 if (accumobjs != NULL) { 1678 *accumobjs = 0; 1679 } 1680 if (accumbytes != NULL) { 1681 *accumbytes = 0; 1682 } 1683 return; 1684 } 1685 1686 prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg, 1687 &prof_gctx_merge_iter_arg, &gctxs); 1688 prof_gctx_finish(tsd, &gctxs); 1689 1690 if (curobjs != NULL) { 1691 *curobjs = prof_tdata_merge_iter_arg.cnt_all.curobjs; 1692 } 1693 if (curbytes != NULL) { 1694 *curbytes = prof_tdata_merge_iter_arg.cnt_all.curbytes; 1695 } 1696 if (accumobjs != NULL) { 1697 *accumobjs = prof_tdata_merge_iter_arg.cnt_all.accumobjs; 1698 } 1699 if (accumbytes != NULL) { 1700 *accumbytes = prof_tdata_merge_iter_arg.cnt_all.accumbytes; 1701 } 1702} 1703#endif 1704 1705#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) 1706#define VSEQ_INVALID UINT64_C(0xffffffffffffffff) 1707static void 1708prof_dump_filename(char *filename, char v, uint64_t vseq) 1709{ 1710 cassert(config_prof); 1711 1712 if (vseq != VSEQ_INVALID) { 1713 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ 1714 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1715 "%s.%d.%"FMTu64".%c%"FMTu64".heap", 1716 opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq); 1717 } else { 1718 /* "<prefix>.<pid>.<seq>.<v>.heap" */ 1719 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1720 "%s.%d.%"FMTu64".%c.heap", 1721 opt_prof_prefix, prof_getpid(), prof_dump_seq, v); 1722 } 1723 prof_dump_seq++; 1724} 1725 1726static void 1727prof_fdump(void) 1728{ 1729 tsd_t *tsd; 1730 char filename[DUMP_FILENAME_BUFSIZE]; 1731 1732 cassert(config_prof); 1733 assert(opt_prof_final); 1734 assert(opt_prof_prefix[0] != '\0'); 1735 1736 if (!prof_booted) 1737 return; 1738 tsd = tsd_fetch(); 1739 1740 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1741 prof_dump_filename(filename, 'f', VSEQ_INVALID); 1742 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1743 prof_dump(tsd, false, filename, opt_prof_leak); 1744} 1745 1746void 1747prof_idump(tsdn_t *tsdn) 1748{ 1749 tsd_t *tsd; 1750 prof_tdata_t *tdata; 1751 1752 cassert(config_prof); 1753 1754 if (!prof_booted || tsdn_null(tsdn)) 1755 return; 1756 tsd = tsdn_tsd(tsdn); 1757 tdata = prof_tdata_get(tsd, false); 1758 if (tdata == NULL) 1759 return; 1760 if (tdata->enq) { 1761 tdata->enq_idump = true; 1762 return; 1763 } 1764 1765 if (opt_prof_prefix[0] != '\0') { 1766 char filename[PATH_MAX + 1]; 1767 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1768 prof_dump_filename(filename, 'i', prof_dump_iseq); 1769 prof_dump_iseq++; 1770 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1771 prof_dump(tsd, false, filename, false); 1772 } 1773} 1774 1775bool 1776prof_mdump(tsd_t *tsd, const char *filename) 1777{ 1778 char filename_buf[DUMP_FILENAME_BUFSIZE]; 1779 1780 cassert(config_prof); 1781 1782 if (!opt_prof || !prof_booted) 1783 return (true); 1784 1785 if (filename == NULL) { 1786 /* No filename specified, so automatically generate one. */ 1787 if (opt_prof_prefix[0] == '\0') 1788 return (true); 1789 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1790 prof_dump_filename(filename_buf, 'm', prof_dump_mseq); 1791 prof_dump_mseq++; 1792 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1793 filename = filename_buf; 1794 } 1795 return (prof_dump(tsd, true, filename, false)); 1796} 1797 1798void 1799prof_gdump(tsdn_t *tsdn) 1800{ 1801 tsd_t *tsd; 1802 prof_tdata_t *tdata; 1803 1804 cassert(config_prof); 1805 1806 if (!prof_booted || tsdn_null(tsdn)) 1807 return; 1808 tsd = tsdn_tsd(tsdn); 1809 tdata = prof_tdata_get(tsd, false); 1810 if (tdata == NULL) 1811 return; 1812 if (tdata->enq) { 1813 tdata->enq_gdump = true; 1814 return; 1815 } 1816 1817 if (opt_prof_prefix[0] != '\0') { 1818 char filename[DUMP_FILENAME_BUFSIZE]; 1819 malloc_mutex_lock(tsdn, &prof_dump_seq_mtx); 1820 prof_dump_filename(filename, 'u', prof_dump_useq); 1821 prof_dump_useq++; 1822 malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx); 1823 prof_dump(tsd, false, filename, false); 1824 } 1825} 1826 1827static void 1828prof_bt_hash(const void *key, size_t r_hash[2]) 1829{ 1830 prof_bt_t *bt = (prof_bt_t *)key; 1831 1832 cassert(config_prof); 1833 1834 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); 1835} 1836 1837static bool 1838prof_bt_keycomp(const void *k1, const void *k2) 1839{ 1840 const prof_bt_t *bt1 = (prof_bt_t *)k1; 1841 const prof_bt_t *bt2 = (prof_bt_t *)k2; 1842 1843 cassert(config_prof); 1844 1845 if (bt1->len != bt2->len) 1846 return (false); 1847 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); 1848} 1849 1850JEMALLOC_INLINE_C uint64_t 1851prof_thr_uid_alloc(tsdn_t *tsdn) 1852{ 1853 uint64_t thr_uid; 1854 1855 malloc_mutex_lock(tsdn, &next_thr_uid_mtx); 1856 thr_uid = next_thr_uid; 1857 next_thr_uid++; 1858 malloc_mutex_unlock(tsdn, &next_thr_uid_mtx); 1859 1860 return (thr_uid); 1861} 1862 1863static prof_tdata_t * 1864prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, 1865 char *thread_name, bool active) 1866{ 1867 prof_tdata_t *tdata; 1868 1869 cassert(config_prof); 1870 1871 /* Initialize an empty cache for this thread. */ 1872 tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t), 1873 size2index(sizeof(prof_tdata_t)), false, NULL, true, 1874 arena_get(TSDN_NULL, 0, true), true); 1875 if (tdata == NULL) 1876 return (NULL); 1877 1878 tdata->lock = prof_tdata_mutex_choose(thr_uid); 1879 tdata->thr_uid = thr_uid; 1880 tdata->thr_discrim = thr_discrim; 1881 tdata->thread_name = thread_name; 1882 tdata->attached = true; 1883 tdata->expired = false; 1884 tdata->tctx_uid_next = 0; 1885 1886 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, 1887 prof_bt_keycomp)) { 1888 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tdata), tdata, 1889 NULL, true, true); 1890 return (NULL); 1891 } 1892 1893 tdata->prng_state = (uint64_t)(uintptr_t)tdata; 1894 prof_sample_threshold_update(tdata); 1895 1896 tdata->enq = false; 1897 tdata->enq_idump = false; 1898 tdata->enq_gdump = false; 1899 1900 tdata->dumping = false; 1901 tdata->active = active; 1902 1903 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); 1904 tdata_tree_insert(&tdatas, tdata); 1905 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); 1906 1907 return (tdata); 1908} 1909 1910prof_tdata_t * 1911prof_tdata_init(tsd_t *tsd) 1912{ 1913 return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0, 1914 NULL, prof_thread_active_init_get(tsd_tsdn(tsd)))); 1915} 1916 1917static bool 1918prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) 1919{ 1920 if (tdata->attached && !even_if_attached) 1921 return (false); 1922 if (ckh_count(&tdata->bt2tctx) != 0) 1923 return (false); 1924 return (true); 1925} 1926 1927static bool 1928prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, 1929 bool even_if_attached) 1930{ 1931 malloc_mutex_assert_owner(tsdn, tdata->lock); 1932 1933 return (prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); 1934} 1935 1936static void 1937prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, 1938 bool even_if_attached) 1939{ 1940 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx); 1941 1942 tdata_tree_remove(&tdatas, tdata); 1943 1944 assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); 1945 1946 if (tdata->thread_name != NULL) { 1947 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), 1948 tdata->thread_name), tdata->thread_name, NULL, true, true); 1949 } 1950 ckh_delete(tsd, &tdata->bt2tctx); 1951 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tdata), tdata, NULL, 1952 true, true); 1953} 1954 1955static void 1956prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) 1957{ 1958 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); 1959 prof_tdata_destroy_locked(tsd, tdata, even_if_attached); 1960 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); 1961} 1962 1963static void 1964prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) 1965{ 1966 bool destroy_tdata; 1967 1968 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); 1969 if (tdata->attached) { 1970 destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, 1971 true); 1972 /* 1973 * Only detach if !destroy_tdata, because detaching would allow 1974 * another thread to win the race to destroy tdata. 1975 */ 1976 if (!destroy_tdata) 1977 tdata->attached = false; 1978 tsd_prof_tdata_set(tsd, NULL); 1979 } else 1980 destroy_tdata = false; 1981 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 1982 if (destroy_tdata) 1983 prof_tdata_destroy(tsd, tdata, true); 1984} 1985 1986prof_tdata_t * 1987prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) 1988{ 1989 uint64_t thr_uid = tdata->thr_uid; 1990 uint64_t thr_discrim = tdata->thr_discrim + 1; 1991 char *thread_name = (tdata->thread_name != NULL) ? 1992 prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL; 1993 bool active = tdata->active; 1994 1995 prof_tdata_detach(tsd, tdata); 1996 return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, 1997 active)); 1998} 1999 2000static bool 2001prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) 2002{ 2003 bool destroy_tdata; 2004 2005 malloc_mutex_lock(tsdn, tdata->lock); 2006 if (!tdata->expired) { 2007 tdata->expired = true; 2008 destroy_tdata = tdata->attached ? false : 2009 prof_tdata_should_destroy(tsdn, tdata, false); 2010 } else 2011 destroy_tdata = false; 2012 malloc_mutex_unlock(tsdn, tdata->lock); 2013 2014 return (destroy_tdata); 2015} 2016 2017static prof_tdata_t * 2018prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 2019{ 2020 tsdn_t *tsdn = (tsdn_t *)arg; 2021 2022 return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL); 2023} 2024 2025void 2026prof_reset(tsd_t *tsd, size_t lg_sample) 2027{ 2028 prof_tdata_t *next; 2029 2030 assert(lg_sample < (sizeof(uint64_t) << 3)); 2031 2032 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); 2033 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); 2034 2035 lg_prof_sample = lg_sample; 2036 2037 next = NULL; 2038 do { 2039 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, 2040 prof_tdata_reset_iter, (void *)tsd); 2041 if (to_destroy != NULL) { 2042 next = tdata_tree_next(&tdatas, to_destroy); 2043 prof_tdata_destroy_locked(tsd, to_destroy, false); 2044 } else 2045 next = NULL; 2046 } while (next != NULL); 2047 2048 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); 2049 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); 2050} 2051 2052void 2053prof_tdata_cleanup(tsd_t *tsd) 2054{ 2055 prof_tdata_t *tdata; 2056 2057 if (!config_prof) 2058 return; 2059 2060 tdata = tsd_prof_tdata_get(tsd); 2061 if (tdata != NULL) 2062 prof_tdata_detach(tsd, tdata); 2063} 2064 2065bool 2066prof_active_get(tsdn_t *tsdn) 2067{ 2068 bool prof_active_current; 2069 2070 malloc_mutex_lock(tsdn, &prof_active_mtx); 2071 prof_active_current = prof_active; 2072 malloc_mutex_unlock(tsdn, &prof_active_mtx); 2073 return (prof_active_current); 2074} 2075 2076bool 2077prof_active_set(tsdn_t *tsdn, bool active) 2078{ 2079 bool prof_active_old; 2080 2081 malloc_mutex_lock(tsdn, &prof_active_mtx); 2082 prof_active_old = prof_active; 2083 prof_active = active; 2084 malloc_mutex_unlock(tsdn, &prof_active_mtx); 2085 return (prof_active_old); 2086} 2087 2088const char * 2089prof_thread_name_get(tsd_t *tsd) 2090{ 2091 prof_tdata_t *tdata; 2092 2093 tdata = prof_tdata_get(tsd, true); 2094 if (tdata == NULL) 2095 return (""); 2096 return (tdata->thread_name != NULL ? tdata->thread_name : ""); 2097} 2098 2099static char * 2100prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) 2101{ 2102 char *ret; 2103 size_t size; 2104 2105 if (thread_name == NULL) 2106 return (NULL); 2107 2108 size = strlen(thread_name) + 1; 2109 if (size == 1) 2110 return (char*)(""); 2111 2112 ret = iallocztm(tsdn, size, size2index(size), false, NULL, true, 2113 arena_get(TSDN_NULL, 0, true), true); 2114 if (ret == NULL) 2115 return (NULL); 2116 memcpy(ret, thread_name, size); 2117 return (ret); 2118} 2119 2120int 2121prof_thread_name_set(tsd_t *tsd, const char *thread_name) 2122{ 2123 prof_tdata_t *tdata; 2124 unsigned i; 2125 char *s; 2126 2127 tdata = prof_tdata_get(tsd, true); 2128 if (tdata == NULL) 2129 return (EAGAIN); 2130 2131 /* Validate input. */ 2132 if (thread_name == NULL) 2133 return (EFAULT); 2134 for (i = 0; thread_name[i] != '\0'; i++) { 2135 char c = thread_name[i]; 2136 if (!isgraph(c) && !isblank(c)) 2137 return (EFAULT); 2138 } 2139 2140 s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name); 2141 if (s == NULL) 2142 return (EAGAIN); 2143 2144 if (tdata->thread_name != NULL) { 2145 idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), 2146 tdata->thread_name), tdata->thread_name, NULL, true, true); 2147 tdata->thread_name = NULL; 2148 } 2149 if (strlen(s) > 0) 2150 tdata->thread_name = s; 2151 return (0); 2152} 2153 2154bool 2155prof_thread_active_get(tsd_t *tsd) 2156{ 2157 prof_tdata_t *tdata; 2158 2159 tdata = prof_tdata_get(tsd, true); 2160 if (tdata == NULL) 2161 return (false); 2162 return (tdata->active); 2163} 2164 2165bool 2166prof_thread_active_set(tsd_t *tsd, bool active) 2167{ 2168 prof_tdata_t *tdata; 2169 2170 tdata = prof_tdata_get(tsd, true); 2171 if (tdata == NULL) 2172 return (true); 2173 tdata->active = active; 2174 return (false); 2175} 2176 2177bool 2178prof_thread_active_init_get(tsdn_t *tsdn) 2179{ 2180 bool active_init; 2181 2182 malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); 2183 active_init = prof_thread_active_init; 2184 malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); 2185 return (active_init); 2186} 2187 2188bool 2189prof_thread_active_init_set(tsdn_t *tsdn, bool active_init) 2190{ 2191 bool active_init_old; 2192 2193 malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); 2194 active_init_old = prof_thread_active_init; 2195 prof_thread_active_init = active_init; 2196 malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); 2197 return (active_init_old); 2198} 2199 2200bool 2201prof_gdump_get(tsdn_t *tsdn) 2202{ 2203 bool prof_gdump_current; 2204 2205 malloc_mutex_lock(tsdn, &prof_gdump_mtx); 2206 prof_gdump_current = prof_gdump_val; 2207 malloc_mutex_unlock(tsdn, &prof_gdump_mtx); 2208 return (prof_gdump_current); 2209} 2210 2211bool 2212prof_gdump_set(tsdn_t *tsdn, bool gdump) 2213{ 2214 bool prof_gdump_old; 2215 2216 malloc_mutex_lock(tsdn, &prof_gdump_mtx); 2217 prof_gdump_old = prof_gdump_val; 2218 prof_gdump_val = gdump; 2219 malloc_mutex_unlock(tsdn, &prof_gdump_mtx); 2220 return (prof_gdump_old); 2221} 2222 2223void 2224prof_boot0(void) 2225{ 2226 cassert(config_prof); 2227 2228 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, 2229 sizeof(PROF_PREFIX_DEFAULT)); 2230} 2231 2232void 2233prof_boot1(void) 2234{ 2235 cassert(config_prof); 2236 2237 /* 2238 * opt_prof must be in its final state before any arenas are 2239 * initialized, so this function must be executed early. 2240 */ 2241 2242 if (opt_prof_leak && !opt_prof) { 2243 /* 2244 * Enable opt_prof, but in such a way that profiles are never 2245 * automatically dumped. 2246 */ 2247 opt_prof = true; 2248 opt_prof_gdump = false; 2249 } else if (opt_prof) { 2250 if (opt_lg_prof_interval >= 0) { 2251 prof_interval = (((uint64_t)1U) << 2252 opt_lg_prof_interval); 2253 } 2254 } 2255} 2256 2257bool 2258prof_boot2(tsd_t *tsd) 2259{ 2260 cassert(config_prof); 2261 2262 if (opt_prof) { 2263 unsigned i; 2264 2265 lg_prof_sample = opt_lg_prof_sample; 2266 2267 prof_active = opt_prof_active; 2268 if (malloc_mutex_init(&prof_active_mtx, "prof_active", 2269 WITNESS_RANK_PROF_ACTIVE)) 2270 return (true); 2271 2272 prof_gdump_val = opt_prof_gdump; 2273 if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump", 2274 WITNESS_RANK_PROF_GDUMP)) 2275 return (true); 2276 2277 prof_thread_active_init = opt_prof_thread_active_init; 2278 if (malloc_mutex_init(&prof_thread_active_init_mtx, 2279 "prof_thread_active_init", 2280 WITNESS_RANK_PROF_THREAD_ACTIVE_INIT)) 2281 return (true); 2282 2283 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, 2284 prof_bt_keycomp)) 2285 return (true); 2286 if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx", 2287 WITNESS_RANK_PROF_BT2GCTX)) 2288 return (true); 2289 2290 tdata_tree_new(&tdatas); 2291 if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas", 2292 WITNESS_RANK_PROF_TDATAS)) 2293 return (true); 2294 2295 next_thr_uid = 0; 2296 if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid", 2297 WITNESS_RANK_PROF_NEXT_THR_UID)) 2298 return (true); 2299 2300 if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq", 2301 WITNESS_RANK_PROF_DUMP_SEQ)) 2302 return (true); 2303 if (malloc_mutex_init(&prof_dump_mtx, "prof_dump", 2304 WITNESS_RANK_PROF_DUMP)) 2305 return (true); 2306 2307 if (opt_prof_final && opt_prof_prefix[0] != '\0' && 2308 atexit(prof_fdump) != 0) { 2309 malloc_write("<jemalloc>: Error in atexit()\n"); 2310 if (opt_abort) 2311 abort(); 2312 } 2313 2314 gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), 2315 b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), 2316 CACHELINE); 2317 if (gctx_locks == NULL) 2318 return (true); 2319 for (i = 0; i < PROF_NCTX_LOCKS; i++) { 2320 if (malloc_mutex_init(&gctx_locks[i], "prof_gctx", 2321 WITNESS_RANK_PROF_GCTX)) 2322 return (true); 2323 } 2324 2325 tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), 2326 b0get(), PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t), 2327 CACHELINE); 2328 if (tdata_locks == NULL) 2329 return (true); 2330 for (i = 0; i < PROF_NTDATA_LOCKS; i++) { 2331 if (malloc_mutex_init(&tdata_locks[i], "prof_tdata", 2332 WITNESS_RANK_PROF_TDATA)) 2333 return (true); 2334 } 2335 } 2336 2337#ifdef JEMALLOC_PROF_LIBGCC 2338 /* 2339 * Cause the backtracing machinery to allocate its internal state 2340 * before enabling profiling. 2341 */ 2342 _Unwind_Backtrace(prof_unwind_init_callback, NULL); 2343#endif 2344 2345 prof_booted = true; 2346 2347 return (false); 2348} 2349 2350void 2351prof_prefork0(tsdn_t *tsdn) 2352{ 2353 if (opt_prof) { 2354 unsigned i; 2355 2356 malloc_mutex_prefork(tsdn, &prof_dump_mtx); 2357 malloc_mutex_prefork(tsdn, &bt2gctx_mtx); 2358 malloc_mutex_prefork(tsdn, &tdatas_mtx); 2359 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2360 malloc_mutex_prefork(tsdn, &tdata_locks[i]); 2361 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2362 malloc_mutex_prefork(tsdn, &gctx_locks[i]); 2363 } 2364} 2365 2366void 2367prof_prefork1(tsdn_t *tsdn) 2368{ 2369 if (opt_prof) { 2370 malloc_mutex_prefork(tsdn, &prof_active_mtx); 2371 malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx); 2372 malloc_mutex_prefork(tsdn, &prof_gdump_mtx); 2373 malloc_mutex_prefork(tsdn, &next_thr_uid_mtx); 2374 malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx); 2375 } 2376} 2377 2378void 2379prof_postfork_parent(tsdn_t *tsdn) 2380{ 2381 if (opt_prof) { 2382 unsigned i; 2383 2384 malloc_mutex_postfork_parent(tsdn, 2385 &prof_thread_active_init_mtx); 2386 malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx); 2387 malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx); 2388 malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx); 2389 malloc_mutex_postfork_parent(tsdn, &prof_active_mtx); 2390 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2391 malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]); 2392 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2393 malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]); 2394 malloc_mutex_postfork_parent(tsdn, &tdatas_mtx); 2395 malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx); 2396 malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx); 2397 } 2398} 2399 2400void 2401prof_postfork_child(tsdn_t *tsdn) 2402{ 2403 if (opt_prof) { 2404 unsigned i; 2405 2406 malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx); 2407 malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx); 2408 malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx); 2409 malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx); 2410 malloc_mutex_postfork_child(tsdn, &prof_active_mtx); 2411 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2412 malloc_mutex_postfork_child(tsdn, &gctx_locks[i]); 2413 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2414 malloc_mutex_postfork_child(tsdn, &tdata_locks[i]); 2415 malloc_mutex_postfork_child(tsdn, &tdatas_mtx); 2416 malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx); 2417 malloc_mutex_postfork_child(tsdn, &prof_dump_mtx); 2418 } 2419} 2420 2421/******************************************************************************/ 2422