1234370Sjasone#define JEMALLOC_PROF_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone/******************************************************************************/ 4234370Sjasone 5234370Sjasone#ifdef JEMALLOC_PROF_LIBUNWIND 6234370Sjasone#define UNW_LOCAL_ONLY 7234370Sjasone#include <libunwind.h> 8234370Sjasone#endif 9234370Sjasone 10234370Sjasone#ifdef JEMALLOC_PROF_LIBGCC 11234370Sjasone#include <unwind.h> 12234370Sjasone#endif 13234370Sjasone 14234370Sjasone/******************************************************************************/ 15234370Sjasone/* Data. */ 16234370Sjasone 17234370Sjasonebool opt_prof = false; 18234370Sjasonebool opt_prof_active = true; 19286866Sjasonebool opt_prof_thread_active_init = true; 20234370Sjasonesize_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; 21234370Sjasonessize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; 22234370Sjasonebool opt_prof_gdump = false; 23286866Sjasonebool opt_prof_final = false; 24234370Sjasonebool opt_prof_leak = false; 25234543Sjasonebool opt_prof_accum = false; 26261071Sjasonechar opt_prof_prefix[ 27261071Sjasone /* Minimize memory bloat for non-prof builds. */ 28261071Sjasone#ifdef JEMALLOC_PROF 29261071Sjasone PATH_MAX + 30261071Sjasone#endif 31261071Sjasone 1]; 32234370Sjasone 33286866Sjasone/* 34286866Sjasone * Initialized as opt_prof_active, and accessed via 35286866Sjasone * prof_active_[gs]et{_unlocked,}(). 36286866Sjasone */ 37286866Sjasonebool prof_active; 38286866Sjasonestatic malloc_mutex_t prof_active_mtx; 39286866Sjasone 40286866Sjasone/* 41286866Sjasone * Initialized as opt_prof_thread_active_init, and accessed via 42286866Sjasone * prof_thread_active_init_[gs]et(). 43286866Sjasone */ 44286866Sjasonestatic bool prof_thread_active_init; 45286866Sjasonestatic malloc_mutex_t prof_thread_active_init_mtx; 46286866Sjasone 47286866Sjasone/* 48286866Sjasone * Initialized as opt_prof_gdump, and accessed via 49286866Sjasone * prof_gdump_[gs]et{_unlocked,}(). 50286866Sjasone */ 51286866Sjasonebool prof_gdump_val; 52286866Sjasonestatic malloc_mutex_t prof_gdump_mtx; 53286866Sjasone 54245868Sjasoneuint64_t prof_interval = 0; 55234370Sjasone 56286866Sjasonesize_t lg_prof_sample; 57286866Sjasone 58234370Sjasone/* 59286866Sjasone * Table of mutexes that are shared among gctx's. These are leaf locks, so 60286866Sjasone * there is no problem with using them for more than one gctx at the same time. 61286866Sjasone * The primary motivation for this sharing though is that gctx's are ephemeral, 62234370Sjasone * and destroying mutexes causes complications for systems that allocate when 63234370Sjasone * creating/destroying mutexes. 64234370Sjasone */ 65286866Sjasonestatic malloc_mutex_t *gctx_locks; 66286866Sjasonestatic unsigned cum_gctxs; /* Atomic counter. */ 67234370Sjasone 68234370Sjasone/* 69286866Sjasone * Table of mutexes that are shared among tdata's. No operations require 70286866Sjasone * holding multiple tdata locks, so there is no problem with using them for more 71286866Sjasone * than one tdata at the same time, even though a gctx lock may be acquired 72286866Sjasone * while holding a tdata lock. 73286866Sjasone */ 74286866Sjasonestatic malloc_mutex_t *tdata_locks; 75286866Sjasone 76286866Sjasone/* 77286866Sjasone * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data 78234370Sjasone * structure that knows about all backtraces currently captured. 79234370Sjasone */ 80286866Sjasonestatic ckh_t bt2gctx; 81286866Sjasonestatic malloc_mutex_t bt2gctx_mtx; 82234370Sjasone 83286866Sjasone/* 84286866Sjasone * Tree of all extant prof_tdata_t structures, regardless of state, 85286866Sjasone * {attached,detached,expired}. 86286866Sjasone */ 87286866Sjasonestatic prof_tdata_tree_t tdatas; 88286866Sjasonestatic malloc_mutex_t tdatas_mtx; 89286866Sjasone 90286866Sjasonestatic uint64_t next_thr_uid; 91286866Sjasonestatic malloc_mutex_t next_thr_uid_mtx; 92286866Sjasone 93234370Sjasonestatic malloc_mutex_t prof_dump_seq_mtx; 94234370Sjasonestatic uint64_t prof_dump_seq; 95234370Sjasonestatic uint64_t prof_dump_iseq; 96234370Sjasonestatic uint64_t prof_dump_mseq; 97234370Sjasonestatic uint64_t prof_dump_useq; 98234370Sjasone 99234370Sjasone/* 100234370Sjasone * This buffer is rather large for stack allocation, so use a single buffer for 101261071Sjasone * all profile dumps. 102234370Sjasone */ 103261071Sjasonestatic malloc_mutex_t prof_dump_mtx; 104261071Sjasonestatic char prof_dump_buf[ 105261071Sjasone /* Minimize memory bloat for non-prof builds. */ 106261071Sjasone#ifdef JEMALLOC_PROF 107261071Sjasone PROF_DUMP_BUFSIZE 108261071Sjasone#else 109261071Sjasone 1 110261071Sjasone#endif 111261071Sjasone]; 112296221Sjasonestatic size_t prof_dump_buf_end; 113234370Sjasonestatic int prof_dump_fd; 114234370Sjasone 115234370Sjasone/* Do not dump any profiles until bootstrapping is complete. */ 116234370Sjasonestatic bool prof_booted = false; 117234370Sjasone 118234370Sjasone/******************************************************************************/ 119286866Sjasone/* 120286866Sjasone * Function prototypes for static functions that are referenced prior to 121286866Sjasone * definition. 122286866Sjasone */ 123234370Sjasone 124299587Sjasonestatic bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx); 125286866Sjasonestatic void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); 126299587Sjasonestatic bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, 127286866Sjasone bool even_if_attached); 128299587Sjasonestatic void prof_tdata_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, 129286866Sjasone bool even_if_attached); 130299587Sjasonestatic char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); 131286866Sjasone 132286866Sjasone/******************************************************************************/ 133286866Sjasone/* Red-black trees. */ 134286866Sjasone 135286866SjasoneJEMALLOC_INLINE_C int 136286866Sjasoneprof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) 137286866Sjasone{ 138286866Sjasone uint64_t a_thr_uid = a->thr_uid; 139286866Sjasone uint64_t b_thr_uid = b->thr_uid; 140286866Sjasone int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); 141286866Sjasone if (ret == 0) { 142288090Sjasone uint64_t a_thr_discrim = a->thr_discrim; 143288090Sjasone uint64_t b_thr_discrim = b->thr_discrim; 144288090Sjasone ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < 145288090Sjasone b_thr_discrim); 146288090Sjasone if (ret == 0) { 147288090Sjasone uint64_t a_tctx_uid = a->tctx_uid; 148288090Sjasone uint64_t b_tctx_uid = b->tctx_uid; 149288090Sjasone ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < 150288090Sjasone b_tctx_uid); 151288090Sjasone } 152286866Sjasone } 153286866Sjasone return (ret); 154286866Sjasone} 155286866Sjasone 156286866Sjasonerb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, 157286866Sjasone tctx_link, prof_tctx_comp) 158286866Sjasone 159286866SjasoneJEMALLOC_INLINE_C int 160286866Sjasoneprof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) 161286866Sjasone{ 162286866Sjasone unsigned a_len = a->bt.len; 163286866Sjasone unsigned b_len = b->bt.len; 164286866Sjasone unsigned comp_len = (a_len < b_len) ? a_len : b_len; 165286866Sjasone int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); 166286866Sjasone if (ret == 0) 167286866Sjasone ret = (a_len > b_len) - (a_len < b_len); 168286866Sjasone return (ret); 169286866Sjasone} 170286866Sjasone 171286866Sjasonerb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, 172286866Sjasone prof_gctx_comp) 173286866Sjasone 174286866SjasoneJEMALLOC_INLINE_C int 175286866Sjasoneprof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) 176286866Sjasone{ 177286866Sjasone int ret; 178286866Sjasone uint64_t a_uid = a->thr_uid; 179286866Sjasone uint64_t b_uid = b->thr_uid; 180286866Sjasone 181286866Sjasone ret = ((a_uid > b_uid) - (a_uid < b_uid)); 182286866Sjasone if (ret == 0) { 183286866Sjasone uint64_t a_discrim = a->thr_discrim; 184286866Sjasone uint64_t b_discrim = b->thr_discrim; 185286866Sjasone 186286866Sjasone ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); 187286866Sjasone } 188286866Sjasone return (ret); 189286866Sjasone} 190286866Sjasone 191286866Sjasonerb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, 192286866Sjasone prof_tdata_comp) 193286866Sjasone 194286866Sjasone/******************************************************************************/ 195286866Sjasone 196234370Sjasonevoid 197286866Sjasoneprof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) 198234370Sjasone{ 199286866Sjasone prof_tdata_t *tdata; 200234370Sjasone 201234370Sjasone cassert(config_prof); 202234370Sjasone 203286866Sjasone if (updated) { 204286866Sjasone /* 205286866Sjasone * Compute a new sample threshold. This isn't very important in 206286866Sjasone * practice, because this function is rarely executed, so the 207286866Sjasone * potential for sample bias is minimal except in contrived 208286866Sjasone * programs. 209286866Sjasone */ 210286866Sjasone tdata = prof_tdata_get(tsd, true); 211286866Sjasone if (tdata != NULL) 212288090Sjasone prof_sample_threshold_update(tdata); 213286866Sjasone } 214286866Sjasone 215286866Sjasone if ((uintptr_t)tctx > (uintptr_t)1U) { 216299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); 217286866Sjasone tctx->prepared = false; 218299587Sjasone if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) 219286866Sjasone prof_tctx_destroy(tsd, tctx); 220286866Sjasone else 221299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); 222286866Sjasone } 223234370Sjasone} 224234370Sjasone 225286866Sjasonevoid 226299587Sjasoneprof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, 227299587Sjasone prof_tctx_t *tctx) 228234370Sjasone{ 229234370Sjasone 230299587Sjasone prof_tctx_set(tsdn, ptr, usize, tctx); 231234370Sjasone 232299587Sjasone malloc_mutex_lock(tsdn, tctx->tdata->lock); 233286866Sjasone tctx->cnts.curobjs++; 234286866Sjasone tctx->cnts.curbytes += usize; 235286866Sjasone if (opt_prof_accum) { 236286866Sjasone tctx->cnts.accumobjs++; 237286866Sjasone tctx->cnts.accumbytes += usize; 238286866Sjasone } 239286866Sjasone tctx->prepared = false; 240299587Sjasone malloc_mutex_unlock(tsdn, tctx->tdata->lock); 241234370Sjasone} 242234370Sjasone 243286866Sjasonevoid 244286866Sjasoneprof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 245234370Sjasone{ 246234370Sjasone 247299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); 248286866Sjasone assert(tctx->cnts.curobjs > 0); 249286866Sjasone assert(tctx->cnts.curbytes >= usize); 250286866Sjasone tctx->cnts.curobjs--; 251286866Sjasone tctx->cnts.curbytes -= usize; 252286866Sjasone 253299587Sjasone if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) 254286866Sjasone prof_tctx_destroy(tsd, tctx); 255286866Sjasone else 256299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); 257286866Sjasone} 258286866Sjasone 259286866Sjasonevoid 260286866Sjasonebt_init(prof_bt_t *bt, void **vec) 261286866Sjasone{ 262286866Sjasone 263234370Sjasone cassert(config_prof); 264234370Sjasone 265286866Sjasone bt->vec = vec; 266286866Sjasone bt->len = 0; 267234370Sjasone} 268234370Sjasone 269286866SjasoneJEMALLOC_INLINE_C void 270286866Sjasoneprof_enter(tsd_t *tsd, prof_tdata_t *tdata) 271234370Sjasone{ 272234370Sjasone 273234370Sjasone cassert(config_prof); 274286866Sjasone assert(tdata == prof_tdata_get(tsd, false)); 275234370Sjasone 276286866Sjasone if (tdata != NULL) { 277286866Sjasone assert(!tdata->enq); 278286866Sjasone tdata->enq = true; 279286866Sjasone } 280234370Sjasone 281299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); 282234370Sjasone} 283234370Sjasone 284286866SjasoneJEMALLOC_INLINE_C void 285286866Sjasoneprof_leave(tsd_t *tsd, prof_tdata_t *tdata) 286234370Sjasone{ 287234370Sjasone 288234370Sjasone cassert(config_prof); 289286866Sjasone assert(tdata == prof_tdata_get(tsd, false)); 290234370Sjasone 291299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); 292234370Sjasone 293286866Sjasone if (tdata != NULL) { 294286866Sjasone bool idump, gdump; 295234370Sjasone 296286866Sjasone assert(tdata->enq); 297286866Sjasone tdata->enq = false; 298286866Sjasone idump = tdata->enq_idump; 299286866Sjasone tdata->enq_idump = false; 300286866Sjasone gdump = tdata->enq_gdump; 301286866Sjasone tdata->enq_gdump = false; 302286866Sjasone 303286866Sjasone if (idump) 304299587Sjasone prof_idump(tsd_tsdn(tsd)); 305286866Sjasone if (gdump) 306299587Sjasone prof_gdump(tsd_tsdn(tsd)); 307286866Sjasone } 308234370Sjasone} 309234370Sjasone 310234370Sjasone#ifdef JEMALLOC_PROF_LIBUNWIND 311234370Sjasonevoid 312286866Sjasoneprof_backtrace(prof_bt_t *bt) 313234370Sjasone{ 314286866Sjasone int nframes; 315234370Sjasone 316234370Sjasone cassert(config_prof); 317234370Sjasone assert(bt->len == 0); 318234370Sjasone assert(bt->vec != NULL); 319234370Sjasone 320286866Sjasone nframes = unw_backtrace(bt->vec, PROF_BT_MAX); 321286866Sjasone if (nframes <= 0) 322286866Sjasone return; 323286866Sjasone bt->len = nframes; 324234370Sjasone} 325234370Sjasone#elif (defined(JEMALLOC_PROF_LIBGCC)) 326234370Sjasonestatic _Unwind_Reason_Code 327234370Sjasoneprof_unwind_init_callback(struct _Unwind_Context *context, void *arg) 328234370Sjasone{ 329234370Sjasone 330234370Sjasone cassert(config_prof); 331234370Sjasone 332234370Sjasone return (_URC_NO_REASON); 333234370Sjasone} 334234370Sjasone 335234370Sjasonestatic _Unwind_Reason_Code 336234370Sjasoneprof_unwind_callback(struct _Unwind_Context *context, void *arg) 337234370Sjasone{ 338234370Sjasone prof_unwind_data_t *data = (prof_unwind_data_t *)arg; 339286866Sjasone void *ip; 340234370Sjasone 341234370Sjasone cassert(config_prof); 342234370Sjasone 343286866Sjasone ip = (void *)_Unwind_GetIP(context); 344286866Sjasone if (ip == NULL) 345286866Sjasone return (_URC_END_OF_STACK); 346286866Sjasone data->bt->vec[data->bt->len] = ip; 347286866Sjasone data->bt->len++; 348286866Sjasone if (data->bt->len == data->max) 349286866Sjasone return (_URC_END_OF_STACK); 350234370Sjasone 351234370Sjasone return (_URC_NO_REASON); 352234370Sjasone} 353234370Sjasone 354234370Sjasonevoid 355286866Sjasoneprof_backtrace(prof_bt_t *bt) 356234370Sjasone{ 357286866Sjasone prof_unwind_data_t data = {bt, PROF_BT_MAX}; 358234370Sjasone 359234370Sjasone cassert(config_prof); 360234370Sjasone 361234370Sjasone _Unwind_Backtrace(prof_unwind_callback, &data); 362234370Sjasone} 363234370Sjasone#elif (defined(JEMALLOC_PROF_GCC)) 364234370Sjasonevoid 365286866Sjasoneprof_backtrace(prof_bt_t *bt) 366234370Sjasone{ 367234370Sjasone#define BT_FRAME(i) \ 368286866Sjasone if ((i) < PROF_BT_MAX) { \ 369234370Sjasone void *p; \ 370234370Sjasone if (__builtin_frame_address(i) == 0) \ 371234370Sjasone return; \ 372234370Sjasone p = __builtin_return_address(i); \ 373234370Sjasone if (p == NULL) \ 374234370Sjasone return; \ 375286866Sjasone bt->vec[(i)] = p; \ 376286866Sjasone bt->len = (i) + 1; \ 377234370Sjasone } else \ 378234370Sjasone return; 379234370Sjasone 380234370Sjasone cassert(config_prof); 381234370Sjasone 382234370Sjasone BT_FRAME(0) 383234370Sjasone BT_FRAME(1) 384234370Sjasone BT_FRAME(2) 385234370Sjasone BT_FRAME(3) 386234370Sjasone BT_FRAME(4) 387234370Sjasone BT_FRAME(5) 388234370Sjasone BT_FRAME(6) 389234370Sjasone BT_FRAME(7) 390234370Sjasone BT_FRAME(8) 391234370Sjasone BT_FRAME(9) 392234370Sjasone 393234370Sjasone BT_FRAME(10) 394234370Sjasone BT_FRAME(11) 395234370Sjasone BT_FRAME(12) 396234370Sjasone BT_FRAME(13) 397234370Sjasone BT_FRAME(14) 398234370Sjasone BT_FRAME(15) 399234370Sjasone BT_FRAME(16) 400234370Sjasone BT_FRAME(17) 401234370Sjasone BT_FRAME(18) 402234370Sjasone BT_FRAME(19) 403234370Sjasone 404234370Sjasone BT_FRAME(20) 405234370Sjasone BT_FRAME(21) 406234370Sjasone BT_FRAME(22) 407234370Sjasone BT_FRAME(23) 408234370Sjasone BT_FRAME(24) 409234370Sjasone BT_FRAME(25) 410234370Sjasone BT_FRAME(26) 411234370Sjasone BT_FRAME(27) 412234370Sjasone BT_FRAME(28) 413234370Sjasone BT_FRAME(29) 414234370Sjasone 415234370Sjasone BT_FRAME(30) 416234370Sjasone BT_FRAME(31) 417234370Sjasone BT_FRAME(32) 418234370Sjasone BT_FRAME(33) 419234370Sjasone BT_FRAME(34) 420234370Sjasone BT_FRAME(35) 421234370Sjasone BT_FRAME(36) 422234370Sjasone BT_FRAME(37) 423234370Sjasone BT_FRAME(38) 424234370Sjasone BT_FRAME(39) 425234370Sjasone 426234370Sjasone BT_FRAME(40) 427234370Sjasone BT_FRAME(41) 428234370Sjasone BT_FRAME(42) 429234370Sjasone BT_FRAME(43) 430234370Sjasone BT_FRAME(44) 431234370Sjasone BT_FRAME(45) 432234370Sjasone BT_FRAME(46) 433234370Sjasone BT_FRAME(47) 434234370Sjasone BT_FRAME(48) 435234370Sjasone BT_FRAME(49) 436234370Sjasone 437234370Sjasone BT_FRAME(50) 438234370Sjasone BT_FRAME(51) 439234370Sjasone BT_FRAME(52) 440234370Sjasone BT_FRAME(53) 441234370Sjasone BT_FRAME(54) 442234370Sjasone BT_FRAME(55) 443234370Sjasone BT_FRAME(56) 444234370Sjasone BT_FRAME(57) 445234370Sjasone BT_FRAME(58) 446234370Sjasone BT_FRAME(59) 447234370Sjasone 448234370Sjasone BT_FRAME(60) 449234370Sjasone BT_FRAME(61) 450234370Sjasone BT_FRAME(62) 451234370Sjasone BT_FRAME(63) 452234370Sjasone BT_FRAME(64) 453234370Sjasone BT_FRAME(65) 454234370Sjasone BT_FRAME(66) 455234370Sjasone BT_FRAME(67) 456234370Sjasone BT_FRAME(68) 457234370Sjasone BT_FRAME(69) 458234370Sjasone 459234370Sjasone BT_FRAME(70) 460234370Sjasone BT_FRAME(71) 461234370Sjasone BT_FRAME(72) 462234370Sjasone BT_FRAME(73) 463234370Sjasone BT_FRAME(74) 464234370Sjasone BT_FRAME(75) 465234370Sjasone BT_FRAME(76) 466234370Sjasone BT_FRAME(77) 467234370Sjasone BT_FRAME(78) 468234370Sjasone BT_FRAME(79) 469234370Sjasone 470234370Sjasone BT_FRAME(80) 471234370Sjasone BT_FRAME(81) 472234370Sjasone BT_FRAME(82) 473234370Sjasone BT_FRAME(83) 474234370Sjasone BT_FRAME(84) 475234370Sjasone BT_FRAME(85) 476234370Sjasone BT_FRAME(86) 477234370Sjasone BT_FRAME(87) 478234370Sjasone BT_FRAME(88) 479234370Sjasone BT_FRAME(89) 480234370Sjasone 481234370Sjasone BT_FRAME(90) 482234370Sjasone BT_FRAME(91) 483234370Sjasone BT_FRAME(92) 484234370Sjasone BT_FRAME(93) 485234370Sjasone BT_FRAME(94) 486234370Sjasone BT_FRAME(95) 487234370Sjasone BT_FRAME(96) 488234370Sjasone BT_FRAME(97) 489234370Sjasone BT_FRAME(98) 490234370Sjasone BT_FRAME(99) 491234370Sjasone 492234370Sjasone BT_FRAME(100) 493234370Sjasone BT_FRAME(101) 494234370Sjasone BT_FRAME(102) 495234370Sjasone BT_FRAME(103) 496234370Sjasone BT_FRAME(104) 497234370Sjasone BT_FRAME(105) 498234370Sjasone BT_FRAME(106) 499234370Sjasone BT_FRAME(107) 500234370Sjasone BT_FRAME(108) 501234370Sjasone BT_FRAME(109) 502234370Sjasone 503234370Sjasone BT_FRAME(110) 504234370Sjasone BT_FRAME(111) 505234370Sjasone BT_FRAME(112) 506234370Sjasone BT_FRAME(113) 507234370Sjasone BT_FRAME(114) 508234370Sjasone BT_FRAME(115) 509234370Sjasone BT_FRAME(116) 510234370Sjasone BT_FRAME(117) 511234370Sjasone BT_FRAME(118) 512234370Sjasone BT_FRAME(119) 513234370Sjasone 514234370Sjasone BT_FRAME(120) 515234370Sjasone BT_FRAME(121) 516234370Sjasone BT_FRAME(122) 517234370Sjasone BT_FRAME(123) 518234370Sjasone BT_FRAME(124) 519234370Sjasone BT_FRAME(125) 520234370Sjasone BT_FRAME(126) 521234370Sjasone BT_FRAME(127) 522234370Sjasone#undef BT_FRAME 523234370Sjasone} 524234370Sjasone#else 525234370Sjasonevoid 526286866Sjasoneprof_backtrace(prof_bt_t *bt) 527234370Sjasone{ 528234370Sjasone 529234370Sjasone cassert(config_prof); 530261071Sjasone not_reached(); 531234370Sjasone} 532234370Sjasone#endif 533234370Sjasone 534261071Sjasonestatic malloc_mutex_t * 535286866Sjasoneprof_gctx_mutex_choose(void) 536261071Sjasone{ 537286866Sjasone unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); 538261071Sjasone 539286866Sjasone return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); 540261071Sjasone} 541261071Sjasone 542286866Sjasonestatic malloc_mutex_t * 543286866Sjasoneprof_tdata_mutex_choose(uint64_t thr_uid) 544261071Sjasone{ 545261071Sjasone 546286866Sjasone return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); 547286866Sjasone} 548286866Sjasone 549286866Sjasonestatic prof_gctx_t * 550299587Sjasoneprof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) 551286866Sjasone{ 552261071Sjasone /* 553286866Sjasone * Create a single allocation that has space for vec of length bt->len. 554286866Sjasone */ 555296221Sjasone size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *)); 556299587Sjasone prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size, 557299587Sjasone size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true), 558299587Sjasone true); 559286866Sjasone if (gctx == NULL) 560286866Sjasone return (NULL); 561286866Sjasone gctx->lock = prof_gctx_mutex_choose(); 562286866Sjasone /* 563261071Sjasone * Set nlimbo to 1, in order to avoid a race condition with 564286866Sjasone * prof_tctx_destroy()/prof_gctx_try_destroy(). 565261071Sjasone */ 566286866Sjasone gctx->nlimbo = 1; 567286866Sjasone tctx_tree_new(&gctx->tctxs); 568286866Sjasone /* Duplicate bt. */ 569286866Sjasone memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); 570286866Sjasone gctx->bt.vec = gctx->vec; 571286866Sjasone gctx->bt.len = bt->len; 572286866Sjasone return (gctx); 573261071Sjasone} 574261071Sjasone 575261071Sjasonestatic void 576286866Sjasoneprof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, 577286866Sjasone prof_tdata_t *tdata) 578261071Sjasone{ 579261071Sjasone 580261071Sjasone cassert(config_prof); 581261071Sjasone 582261071Sjasone /* 583286866Sjasone * Check that gctx is still unused by any thread cache before destroying 584286866Sjasone * it. prof_lookup() increments gctx->nlimbo in order to avoid a race 585286866Sjasone * condition with this function, as does prof_tctx_destroy() in order to 586286866Sjasone * avoid a race between the main body of prof_tctx_destroy() and entry 587261071Sjasone * into this function. 588261071Sjasone */ 589286866Sjasone prof_enter(tsd, tdata_self); 590299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 591286866Sjasone assert(gctx->nlimbo != 0); 592286866Sjasone if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { 593286866Sjasone /* Remove gctx from bt2gctx. */ 594299587Sjasone if (ckh_remove(tsd_tsdn(tsd), &bt2gctx, &gctx->bt, NULL, NULL)) 595261071Sjasone not_reached(); 596286866Sjasone prof_leave(tsd, tdata_self); 597286866Sjasone /* Destroy gctx. */ 598299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 599299587Sjasone idalloctm(tsd_tsdn(tsd), gctx, NULL, true, true); 600261071Sjasone } else { 601261071Sjasone /* 602286866Sjasone * Compensate for increment in prof_tctx_destroy() or 603261071Sjasone * prof_lookup(). 604261071Sjasone */ 605286866Sjasone gctx->nlimbo--; 606299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 607286866Sjasone prof_leave(tsd, tdata_self); 608261071Sjasone } 609261071Sjasone} 610261071Sjasone 611286866Sjasonestatic bool 612299587Sjasoneprof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) 613286866Sjasone{ 614286866Sjasone 615299587Sjasone malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); 616299587Sjasone 617286866Sjasone if (opt_prof_accum) 618286866Sjasone return (false); 619286866Sjasone if (tctx->cnts.curobjs != 0) 620286866Sjasone return (false); 621286866Sjasone if (tctx->prepared) 622286866Sjasone return (false); 623286866Sjasone return (true); 624286866Sjasone} 625286866Sjasone 626286866Sjasonestatic bool 627286866Sjasoneprof_gctx_should_destroy(prof_gctx_t *gctx) 628286866Sjasone{ 629286866Sjasone 630286866Sjasone if (opt_prof_accum) 631286866Sjasone return (false); 632286866Sjasone if (!tctx_tree_empty(&gctx->tctxs)) 633286866Sjasone return (false); 634286866Sjasone if (gctx->nlimbo != 0) 635286866Sjasone return (false); 636286866Sjasone return (true); 637286866Sjasone} 638286866Sjasone 639261071Sjasonestatic void 640286866Sjasoneprof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) 641261071Sjasone{ 642286866Sjasone prof_tdata_t *tdata = tctx->tdata; 643286866Sjasone prof_gctx_t *gctx = tctx->gctx; 644286866Sjasone bool destroy_tdata, destroy_tctx, destroy_gctx; 645261071Sjasone 646299587Sjasone malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); 647299587Sjasone 648286866Sjasone assert(tctx->cnts.curobjs == 0); 649286866Sjasone assert(tctx->cnts.curbytes == 0); 650286866Sjasone assert(!opt_prof_accum); 651286866Sjasone assert(tctx->cnts.accumobjs == 0); 652286866Sjasone assert(tctx->cnts.accumbytes == 0); 653261071Sjasone 654299587Sjasone ckh_remove(tsd_tsdn(tsd), &tdata->bt2tctx, &gctx->bt, NULL, NULL); 655299587Sjasone destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false); 656299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 657286866Sjasone 658299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 659286866Sjasone switch (tctx->state) { 660286866Sjasone case prof_tctx_state_nominal: 661286866Sjasone tctx_tree_remove(&gctx->tctxs, tctx); 662286866Sjasone destroy_tctx = true; 663286866Sjasone if (prof_gctx_should_destroy(gctx)) { 664286866Sjasone /* 665286866Sjasone * Increment gctx->nlimbo in order to keep another 666286866Sjasone * thread from winning the race to destroy gctx while 667286866Sjasone * this one has gctx->lock dropped. Without this, it 668286866Sjasone * would be possible for another thread to: 669286866Sjasone * 670286866Sjasone * 1) Sample an allocation associated with gctx. 671286866Sjasone * 2) Deallocate the sampled object. 672286866Sjasone * 3) Successfully prof_gctx_try_destroy(gctx). 673286866Sjasone * 674286866Sjasone * The result would be that gctx no longer exists by the 675286866Sjasone * time this thread accesses it in 676286866Sjasone * prof_gctx_try_destroy(). 677286866Sjasone */ 678286866Sjasone gctx->nlimbo++; 679286866Sjasone destroy_gctx = true; 680286866Sjasone } else 681286866Sjasone destroy_gctx = false; 682286866Sjasone break; 683286866Sjasone case prof_tctx_state_dumping: 684261071Sjasone /* 685286866Sjasone * A dumping thread needs tctx to remain valid until dumping 686286866Sjasone * has finished. Change state such that the dumping thread will 687286866Sjasone * complete destruction during a late dump iteration phase. 688261071Sjasone */ 689286866Sjasone tctx->state = prof_tctx_state_purgatory; 690286866Sjasone destroy_tctx = false; 691286866Sjasone destroy_gctx = false; 692286866Sjasone break; 693286866Sjasone default: 694286866Sjasone not_reached(); 695286866Sjasone destroy_tctx = false; 696286866Sjasone destroy_gctx = false; 697286866Sjasone } 698299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 699286866Sjasone if (destroy_gctx) { 700286866Sjasone prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, 701286866Sjasone tdata); 702286866Sjasone } 703286866Sjasone 704299587Sjasone malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock); 705299587Sjasone 706286866Sjasone if (destroy_tdata) 707299587Sjasone prof_tdata_destroy(tsd_tsdn(tsd), tdata, false); 708286866Sjasone 709286866Sjasone if (destroy_tctx) 710299587Sjasone idalloctm(tsd_tsdn(tsd), tctx, NULL, true, true); 711261071Sjasone} 712261071Sjasone 713261071Sjasonestatic bool 714286866Sjasoneprof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, 715286866Sjasone void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) 716261071Sjasone{ 717261071Sjasone union { 718286866Sjasone prof_gctx_t *p; 719261071Sjasone void *v; 720286866Sjasone } gctx; 721261071Sjasone union { 722261071Sjasone prof_bt_t *p; 723261071Sjasone void *v; 724261071Sjasone } btkey; 725286866Sjasone bool new_gctx; 726261071Sjasone 727286866Sjasone prof_enter(tsd, tdata); 728286866Sjasone if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { 729261071Sjasone /* bt has never been seen before. Insert it. */ 730299587Sjasone gctx.p = prof_gctx_create(tsd_tsdn(tsd), bt); 731286866Sjasone if (gctx.v == NULL) { 732286866Sjasone prof_leave(tsd, tdata); 733261071Sjasone return (true); 734261071Sjasone } 735286866Sjasone btkey.p = &gctx.p->bt; 736299587Sjasone if (ckh_insert(tsd_tsdn(tsd), &bt2gctx, btkey.v, gctx.v)) { 737261071Sjasone /* OOM. */ 738286866Sjasone prof_leave(tsd, tdata); 739299587Sjasone idalloctm(tsd_tsdn(tsd), gctx.v, NULL, true, true); 740261071Sjasone return (true); 741261071Sjasone } 742286866Sjasone new_gctx = true; 743261071Sjasone } else { 744261071Sjasone /* 745261071Sjasone * Increment nlimbo, in order to avoid a race condition with 746286866Sjasone * prof_tctx_destroy()/prof_gctx_try_destroy(). 747261071Sjasone */ 748299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock); 749286866Sjasone gctx.p->nlimbo++; 750299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock); 751286866Sjasone new_gctx = false; 752261071Sjasone } 753286866Sjasone prof_leave(tsd, tdata); 754261071Sjasone 755261071Sjasone *p_btkey = btkey.v; 756286866Sjasone *p_gctx = gctx.p; 757286866Sjasone *p_new_gctx = new_gctx; 758261071Sjasone return (false); 759261071Sjasone} 760261071Sjasone 761286866Sjasoneprof_tctx_t * 762286866Sjasoneprof_lookup(tsd_t *tsd, prof_bt_t *bt) 763234370Sjasone{ 764234370Sjasone union { 765286866Sjasone prof_tctx_t *p; 766234370Sjasone void *v; 767234370Sjasone } ret; 768286866Sjasone prof_tdata_t *tdata; 769286866Sjasone bool not_found; 770234370Sjasone 771234370Sjasone cassert(config_prof); 772234370Sjasone 773286866Sjasone tdata = prof_tdata_get(tsd, false); 774286866Sjasone if (tdata == NULL) 775235238Sjasone return (NULL); 776234370Sjasone 777299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); 778286866Sjasone not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); 779286866Sjasone if (!not_found) /* Note double negative! */ 780286866Sjasone ret.p->prepared = true; 781299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 782286866Sjasone if (not_found) { 783261071Sjasone void *btkey; 784286866Sjasone prof_gctx_t *gctx; 785286866Sjasone bool new_gctx, error; 786234370Sjasone 787234370Sjasone /* 788234370Sjasone * This thread's cache lacks bt. Look for it in the global 789234370Sjasone * cache. 790234370Sjasone */ 791286866Sjasone if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, 792286866Sjasone &new_gctx)) 793261071Sjasone return (NULL); 794234370Sjasone 795286866Sjasone /* Link a prof_tctx_t into gctx for this thread. */ 796299587Sjasone ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t), 797299587Sjasone size2index(sizeof(prof_tctx_t)), false, NULL, true, 798299587Sjasone arena_ichoose(tsd_tsdn(tsd), NULL), true); 799286866Sjasone if (ret.p == NULL) { 800286866Sjasone if (new_gctx) 801286866Sjasone prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 802286866Sjasone return (NULL); 803234370Sjasone } 804286866Sjasone ret.p->tdata = tdata; 805286866Sjasone ret.p->thr_uid = tdata->thr_uid; 806288090Sjasone ret.p->thr_discrim = tdata->thr_discrim; 807234370Sjasone memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); 808286866Sjasone ret.p->gctx = gctx; 809286866Sjasone ret.p->tctx_uid = tdata->tctx_uid_next++; 810286866Sjasone ret.p->prepared = true; 811286866Sjasone ret.p->state = prof_tctx_state_initializing; 812299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); 813299587Sjasone error = ckh_insert(tsd_tsdn(tsd), &tdata->bt2tctx, btkey, 814299587Sjasone ret.v); 815299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 816286866Sjasone if (error) { 817286866Sjasone if (new_gctx) 818286866Sjasone prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 819299587Sjasone idalloctm(tsd_tsdn(tsd), ret.v, NULL, true, true); 820234370Sjasone return (NULL); 821234370Sjasone } 822299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 823286866Sjasone ret.p->state = prof_tctx_state_nominal; 824286866Sjasone tctx_tree_insert(&gctx->tctxs, ret.p); 825286866Sjasone gctx->nlimbo--; 826299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 827234370Sjasone } 828234370Sjasone 829234370Sjasone return (ret.p); 830234370Sjasone} 831234370Sjasone 832299587Sjasone/* 833299587Sjasone * The bodies of this function and prof_leakcheck() are compiled out unless heap 834299587Sjasone * profiling is enabled, so that it is possible to compile jemalloc with 835299587Sjasone * floating point support completely disabled. Avoiding floating point code is 836299587Sjasone * important on memory-constrained systems, but it also enables a workaround for 837299587Sjasone * versions of glibc that don't properly save/restore floating point registers 838299587Sjasone * during dynamic lazy symbol loading (which internally calls into whatever 839299587Sjasone * malloc implementation happens to be integrated into the application). Note 840299587Sjasone * that some compilers (e.g. gcc 4.8) may use floating point registers for fast 841299587Sjasone * memory moves, so jemalloc must be compiled with such optimizations disabled 842299587Sjasone * (e.g. 843299587Sjasone * -mno-sse) in order for the workaround to be complete. 844299587Sjasone */ 845286866Sjasonevoid 846286866Sjasoneprof_sample_threshold_update(prof_tdata_t *tdata) 847286866Sjasone{ 848286866Sjasone#ifdef JEMALLOC_PROF 849286866Sjasone uint64_t r; 850286866Sjasone double u; 851286866Sjasone 852286866Sjasone if (!config_prof) 853286866Sjasone return; 854286866Sjasone 855286866Sjasone if (lg_prof_sample == 0) { 856286866Sjasone tdata->bytes_until_sample = 0; 857286866Sjasone return; 858286866Sjasone } 859286866Sjasone 860286866Sjasone /* 861286866Sjasone * Compute sample interval as a geometrically distributed random 862286866Sjasone * variable with mean (2^lg_prof_sample). 863286866Sjasone * 864286866Sjasone * __ __ 865286866Sjasone * | log(u) | 1 866286866Sjasone * tdata->bytes_until_sample = | -------- |, where p = --------------- 867286866Sjasone * | log(1-p) | lg_prof_sample 868286866Sjasone * 2 869286866Sjasone * 870286866Sjasone * For more information on the math, see: 871286866Sjasone * 872286866Sjasone * Non-Uniform Random Variate Generation 873286866Sjasone * Luc Devroye 874286866Sjasone * Springer-Verlag, New York, 1986 875286866Sjasone * pp 500 876286866Sjasone * (http://luc.devroye.org/rnbookindex.html) 877286866Sjasone */ 878296221Sjasone r = prng_lg_range(&tdata->prng_state, 53); 879286866Sjasone u = (double)r * (1.0/9007199254740992.0L); 880286866Sjasone tdata->bytes_until_sample = (uint64_t)(log(u) / 881286866Sjasone log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) 882286866Sjasone + (uint64_t)1U; 883286866Sjasone#endif 884286866Sjasone} 885286866Sjasone 886261071Sjasone#ifdef JEMALLOC_JET 887286866Sjasonestatic prof_tdata_t * 888286866Sjasoneprof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 889286866Sjasone{ 890286866Sjasone size_t *tdata_count = (size_t *)arg; 891286866Sjasone 892286866Sjasone (*tdata_count)++; 893286866Sjasone 894286866Sjasone return (NULL); 895286866Sjasone} 896286866Sjasone 897261071Sjasonesize_t 898286866Sjasoneprof_tdata_count(void) 899286866Sjasone{ 900286866Sjasone size_t tdata_count = 0; 901299587Sjasone tsdn_t *tsdn; 902286866Sjasone 903299587Sjasone tsdn = tsdn_fetch(); 904299587Sjasone malloc_mutex_lock(tsdn, &tdatas_mtx); 905286866Sjasone tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, 906286866Sjasone (void *)&tdata_count); 907299587Sjasone malloc_mutex_unlock(tsdn, &tdatas_mtx); 908286866Sjasone 909286866Sjasone return (tdata_count); 910286866Sjasone} 911286866Sjasone#endif 912286866Sjasone 913286866Sjasone#ifdef JEMALLOC_JET 914286866Sjasonesize_t 915261071Sjasoneprof_bt_count(void) 916261071Sjasone{ 917261071Sjasone size_t bt_count; 918286866Sjasone tsd_t *tsd; 919286866Sjasone prof_tdata_t *tdata; 920261071Sjasone 921286866Sjasone tsd = tsd_fetch(); 922286866Sjasone tdata = prof_tdata_get(tsd, false); 923286866Sjasone if (tdata == NULL) 924261071Sjasone return (0); 925261071Sjasone 926299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); 927286866Sjasone bt_count = ckh_count(&bt2gctx); 928299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); 929261071Sjasone 930261071Sjasone return (bt_count); 931261071Sjasone} 932261071Sjasone#endif 933261071Sjasone 934261071Sjasone#ifdef JEMALLOC_JET 935261071Sjasone#undef prof_dump_open 936261071Sjasone#define prof_dump_open JEMALLOC_N(prof_dump_open_impl) 937261071Sjasone#endif 938261071Sjasonestatic int 939261071Sjasoneprof_dump_open(bool propagate_err, const char *filename) 940261071Sjasone{ 941261071Sjasone int fd; 942261071Sjasone 943261071Sjasone fd = creat(filename, 0644); 944286866Sjasone if (fd == -1 && !propagate_err) { 945261071Sjasone malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n", 946261071Sjasone filename); 947261071Sjasone if (opt_abort) 948261071Sjasone abort(); 949261071Sjasone } 950261071Sjasone 951261071Sjasone return (fd); 952261071Sjasone} 953261071Sjasone#ifdef JEMALLOC_JET 954261071Sjasone#undef prof_dump_open 955261071Sjasone#define prof_dump_open JEMALLOC_N(prof_dump_open) 956261071Sjasoneprof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); 957261071Sjasone#endif 958261071Sjasone 959234370Sjasonestatic bool 960261071Sjasoneprof_dump_flush(bool propagate_err) 961234370Sjasone{ 962234370Sjasone bool ret = false; 963234370Sjasone ssize_t err; 964234370Sjasone 965234370Sjasone cassert(config_prof); 966234370Sjasone 967234370Sjasone err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); 968234370Sjasone if (err == -1) { 969286866Sjasone if (!propagate_err) { 970234370Sjasone malloc_write("<jemalloc>: write() failed during heap " 971234370Sjasone "profile flush\n"); 972234370Sjasone if (opt_abort) 973234370Sjasone abort(); 974234370Sjasone } 975234370Sjasone ret = true; 976234370Sjasone } 977234370Sjasone prof_dump_buf_end = 0; 978234370Sjasone 979234370Sjasone return (ret); 980234370Sjasone} 981234370Sjasone 982234370Sjasonestatic bool 983261071Sjasoneprof_dump_close(bool propagate_err) 984234370Sjasone{ 985261071Sjasone bool ret; 986261071Sjasone 987261071Sjasone assert(prof_dump_fd != -1); 988261071Sjasone ret = prof_dump_flush(propagate_err); 989261071Sjasone close(prof_dump_fd); 990261071Sjasone prof_dump_fd = -1; 991261071Sjasone 992261071Sjasone return (ret); 993261071Sjasone} 994261071Sjasone 995261071Sjasonestatic bool 996261071Sjasoneprof_dump_write(bool propagate_err, const char *s) 997261071Sjasone{ 998296221Sjasone size_t i, slen, n; 999234370Sjasone 1000234370Sjasone cassert(config_prof); 1001234370Sjasone 1002234370Sjasone i = 0; 1003234370Sjasone slen = strlen(s); 1004234370Sjasone while (i < slen) { 1005234370Sjasone /* Flush the buffer if it is full. */ 1006234370Sjasone if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) 1007261071Sjasone if (prof_dump_flush(propagate_err) && propagate_err) 1008234370Sjasone return (true); 1009234370Sjasone 1010234370Sjasone if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { 1011234370Sjasone /* Finish writing. */ 1012234370Sjasone n = slen - i; 1013234370Sjasone } else { 1014234370Sjasone /* Write as much of s as will fit. */ 1015234370Sjasone n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; 1016234370Sjasone } 1017234370Sjasone memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); 1018234370Sjasone prof_dump_buf_end += n; 1019234370Sjasone i += n; 1020234370Sjasone } 1021234370Sjasone 1022234370Sjasone return (false); 1023234370Sjasone} 1024234370Sjasone 1025286866SjasoneJEMALLOC_FORMAT_PRINTF(2, 3) 1026234370Sjasonestatic bool 1027261071Sjasoneprof_dump_printf(bool propagate_err, const char *format, ...) 1028234370Sjasone{ 1029234370Sjasone bool ret; 1030234370Sjasone va_list ap; 1031234370Sjasone char buf[PROF_PRINTF_BUFSIZE]; 1032234370Sjasone 1033234370Sjasone va_start(ap, format); 1034234370Sjasone malloc_vsnprintf(buf, sizeof(buf), format, ap); 1035234370Sjasone va_end(ap); 1036261071Sjasone ret = prof_dump_write(propagate_err, buf); 1037234370Sjasone 1038234370Sjasone return (ret); 1039234370Sjasone} 1040234370Sjasone 1041234370Sjasonestatic void 1042299587Sjasoneprof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) 1043234370Sjasone{ 1044234370Sjasone 1045299587Sjasone malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); 1046286866Sjasone 1047299587Sjasone malloc_mutex_lock(tsdn, tctx->gctx->lock); 1048299587Sjasone 1049286866Sjasone switch (tctx->state) { 1050286866Sjasone case prof_tctx_state_initializing: 1051299587Sjasone malloc_mutex_unlock(tsdn, tctx->gctx->lock); 1052286866Sjasone return; 1053286866Sjasone case prof_tctx_state_nominal: 1054286866Sjasone tctx->state = prof_tctx_state_dumping; 1055299587Sjasone malloc_mutex_unlock(tsdn, tctx->gctx->lock); 1056286866Sjasone 1057286866Sjasone memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); 1058286866Sjasone 1059286866Sjasone tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1060286866Sjasone tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1061286866Sjasone if (opt_prof_accum) { 1062286866Sjasone tdata->cnt_summed.accumobjs += 1063286866Sjasone tctx->dump_cnts.accumobjs; 1064286866Sjasone tdata->cnt_summed.accumbytes += 1065286866Sjasone tctx->dump_cnts.accumbytes; 1066286866Sjasone } 1067286866Sjasone break; 1068286866Sjasone case prof_tctx_state_dumping: 1069286866Sjasone case prof_tctx_state_purgatory: 1070286866Sjasone not_reached(); 1071286866Sjasone } 1072286866Sjasone} 1073286866Sjasone 1074286866Sjasonestatic void 1075299587Sjasoneprof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) 1076286866Sjasone{ 1077286866Sjasone 1078299587Sjasone malloc_mutex_assert_owner(tsdn, gctx->lock); 1079299587Sjasone 1080286866Sjasone gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1081286866Sjasone gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1082286866Sjasone if (opt_prof_accum) { 1083286866Sjasone gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; 1084286866Sjasone gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; 1085286866Sjasone } 1086286866Sjasone} 1087286866Sjasone 1088286866Sjasonestatic prof_tctx_t * 1089286866Sjasoneprof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1090286866Sjasone{ 1091299587Sjasone tsdn_t *tsdn = (tsdn_t *)arg; 1092286866Sjasone 1093299587Sjasone malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); 1094299587Sjasone 1095286866Sjasone switch (tctx->state) { 1096286866Sjasone case prof_tctx_state_nominal: 1097286866Sjasone /* New since dumping started; ignore. */ 1098286866Sjasone break; 1099286866Sjasone case prof_tctx_state_dumping: 1100286866Sjasone case prof_tctx_state_purgatory: 1101299587Sjasone prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx); 1102286866Sjasone break; 1103286866Sjasone default: 1104286866Sjasone not_reached(); 1105286866Sjasone } 1106286866Sjasone 1107286866Sjasone return (NULL); 1108286866Sjasone} 1109286866Sjasone 1110299587Sjasonestruct prof_tctx_dump_iter_arg_s { 1111299587Sjasone tsdn_t *tsdn; 1112299587Sjasone bool propagate_err; 1113299587Sjasone}; 1114299587Sjasone 1115286866Sjasonestatic prof_tctx_t * 1116299587Sjasoneprof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) 1117286866Sjasone{ 1118299587Sjasone struct prof_tctx_dump_iter_arg_s *arg = 1119299587Sjasone (struct prof_tctx_dump_iter_arg_s *)opaque; 1120286866Sjasone 1121299587Sjasone malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock); 1122299587Sjasone 1123289900Sjasone switch (tctx->state) { 1124289900Sjasone case prof_tctx_state_initializing: 1125289900Sjasone case prof_tctx_state_nominal: 1126289900Sjasone /* Not captured by this dump. */ 1127289900Sjasone break; 1128289900Sjasone case prof_tctx_state_dumping: 1129289900Sjasone case prof_tctx_state_purgatory: 1130299587Sjasone if (prof_dump_printf(arg->propagate_err, 1131289900Sjasone " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": " 1132289900Sjasone "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs, 1133289900Sjasone tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, 1134289900Sjasone tctx->dump_cnts.accumbytes)) 1135289900Sjasone return (tctx); 1136289900Sjasone break; 1137289900Sjasone default: 1138289900Sjasone not_reached(); 1139289900Sjasone } 1140286866Sjasone return (NULL); 1141286866Sjasone} 1142286866Sjasone 1143286866Sjasonestatic prof_tctx_t * 1144286866Sjasoneprof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1145286866Sjasone{ 1146299587Sjasone tsdn_t *tsdn = (tsdn_t *)arg; 1147286866Sjasone prof_tctx_t *ret; 1148286866Sjasone 1149299587Sjasone malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); 1150299587Sjasone 1151286866Sjasone switch (tctx->state) { 1152286866Sjasone case prof_tctx_state_nominal: 1153286866Sjasone /* New since dumping started; ignore. */ 1154286866Sjasone break; 1155286866Sjasone case prof_tctx_state_dumping: 1156286866Sjasone tctx->state = prof_tctx_state_nominal; 1157286866Sjasone break; 1158286866Sjasone case prof_tctx_state_purgatory: 1159286866Sjasone ret = tctx; 1160286866Sjasone goto label_return; 1161286866Sjasone default: 1162286866Sjasone not_reached(); 1163286866Sjasone } 1164286866Sjasone 1165286866Sjasone ret = NULL; 1166286866Sjasonelabel_return: 1167286866Sjasone return (ret); 1168286866Sjasone} 1169286866Sjasone 1170286866Sjasonestatic void 1171299587Sjasoneprof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) 1172286866Sjasone{ 1173286866Sjasone 1174234370Sjasone cassert(config_prof); 1175234370Sjasone 1176299587Sjasone malloc_mutex_lock(tsdn, gctx->lock); 1177234370Sjasone 1178261071Sjasone /* 1179286866Sjasone * Increment nlimbo so that gctx won't go away before dump. 1180286866Sjasone * Additionally, link gctx into the dump list so that it is included in 1181261071Sjasone * prof_dump()'s second pass. 1182261071Sjasone */ 1183286866Sjasone gctx->nlimbo++; 1184286866Sjasone gctx_tree_insert(gctxs, gctx); 1185261071Sjasone 1186286866Sjasone memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); 1187234370Sjasone 1188299587Sjasone malloc_mutex_unlock(tsdn, gctx->lock); 1189286866Sjasone} 1190234370Sjasone 1191299587Sjasonestruct prof_gctx_merge_iter_arg_s { 1192299587Sjasone tsdn_t *tsdn; 1193299587Sjasone size_t leak_ngctx; 1194299587Sjasone}; 1195299587Sjasone 1196286866Sjasonestatic prof_gctx_t * 1197299587Sjasoneprof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) 1198286866Sjasone{ 1199299587Sjasone struct prof_gctx_merge_iter_arg_s *arg = 1200299587Sjasone (struct prof_gctx_merge_iter_arg_s *)opaque; 1201234370Sjasone 1202299587Sjasone malloc_mutex_lock(arg->tsdn, gctx->lock); 1203299587Sjasone tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, 1204299587Sjasone (void *)arg->tsdn); 1205286866Sjasone if (gctx->cnt_summed.curobjs != 0) 1206299587Sjasone arg->leak_ngctx++; 1207299587Sjasone malloc_mutex_unlock(arg->tsdn, gctx->lock); 1208234370Sjasone 1209286866Sjasone return (NULL); 1210286866Sjasone} 1211234370Sjasone 1212286866Sjasonestatic void 1213286866Sjasoneprof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) 1214286866Sjasone{ 1215286866Sjasone prof_tdata_t *tdata = prof_tdata_get(tsd, false); 1216286866Sjasone prof_gctx_t *gctx; 1217234370Sjasone 1218286866Sjasone /* 1219286866Sjasone * Standard tree iteration won't work here, because as soon as we 1220286866Sjasone * decrement gctx->nlimbo and unlock gctx, another thread can 1221286866Sjasone * concurrently destroy it, which will corrupt the tree. Therefore, 1222286866Sjasone * tear down the tree one node at a time during iteration. 1223286866Sjasone */ 1224286866Sjasone while ((gctx = gctx_tree_first(gctxs)) != NULL) { 1225286866Sjasone gctx_tree_remove(gctxs, gctx); 1226299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); 1227286866Sjasone { 1228286866Sjasone prof_tctx_t *next; 1229234370Sjasone 1230286866Sjasone next = NULL; 1231286866Sjasone do { 1232286866Sjasone prof_tctx_t *to_destroy = 1233286866Sjasone tctx_tree_iter(&gctx->tctxs, next, 1234299587Sjasone prof_tctx_finish_iter, 1235299587Sjasone (void *)tsd_tsdn(tsd)); 1236286866Sjasone if (to_destroy != NULL) { 1237286866Sjasone next = tctx_tree_next(&gctx->tctxs, 1238286866Sjasone to_destroy); 1239286866Sjasone tctx_tree_remove(&gctx->tctxs, 1240286866Sjasone to_destroy); 1241299587Sjasone idalloctm(tsd_tsdn(tsd), to_destroy, 1242299587Sjasone NULL, true, true); 1243286866Sjasone } else 1244286866Sjasone next = NULL; 1245286866Sjasone } while (next != NULL); 1246286866Sjasone } 1247286866Sjasone gctx->nlimbo--; 1248286866Sjasone if (prof_gctx_should_destroy(gctx)) { 1249286866Sjasone gctx->nlimbo++; 1250299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 1251286866Sjasone prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 1252286866Sjasone } else 1253299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); 1254234370Sjasone } 1255234370Sjasone} 1256234370Sjasone 1257299587Sjasonestruct prof_tdata_merge_iter_arg_s { 1258299587Sjasone tsdn_t *tsdn; 1259299587Sjasone prof_cnt_t cnt_all; 1260299587Sjasone}; 1261299587Sjasone 1262286866Sjasonestatic prof_tdata_t * 1263299587Sjasoneprof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, 1264299587Sjasone void *opaque) 1265234370Sjasone{ 1266299587Sjasone struct prof_tdata_merge_iter_arg_s *arg = 1267299587Sjasone (struct prof_tdata_merge_iter_arg_s *)opaque; 1268234370Sjasone 1269299587Sjasone malloc_mutex_lock(arg->tsdn, tdata->lock); 1270286866Sjasone if (!tdata->expired) { 1271286866Sjasone size_t tabind; 1272286866Sjasone union { 1273286866Sjasone prof_tctx_t *p; 1274286866Sjasone void *v; 1275286866Sjasone } tctx; 1276261071Sjasone 1277286866Sjasone tdata->dumping = true; 1278286866Sjasone memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); 1279286866Sjasone for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, 1280286866Sjasone &tctx.v);) 1281299587Sjasone prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata); 1282286866Sjasone 1283299587Sjasone arg->cnt_all.curobjs += tdata->cnt_summed.curobjs; 1284299587Sjasone arg->cnt_all.curbytes += tdata->cnt_summed.curbytes; 1285286866Sjasone if (opt_prof_accum) { 1286299587Sjasone arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs; 1287299587Sjasone arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes; 1288286866Sjasone } 1289286866Sjasone } else 1290286866Sjasone tdata->dumping = false; 1291299587Sjasone malloc_mutex_unlock(arg->tsdn, tdata->lock); 1292286866Sjasone 1293286866Sjasone return (NULL); 1294234370Sjasone} 1295234370Sjasone 1296286866Sjasonestatic prof_tdata_t * 1297286866Sjasoneprof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1298234370Sjasone{ 1299286866Sjasone bool propagate_err = *(bool *)arg; 1300234370Sjasone 1301286866Sjasone if (!tdata->dumping) 1302286866Sjasone return (NULL); 1303286866Sjasone 1304286866Sjasone if (prof_dump_printf(propagate_err, 1305286866Sjasone " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n", 1306286866Sjasone tdata->thr_uid, tdata->cnt_summed.curobjs, 1307286866Sjasone tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, 1308286866Sjasone tdata->cnt_summed.accumbytes, 1309286866Sjasone (tdata->thread_name != NULL) ? " " : "", 1310286866Sjasone (tdata->thread_name != NULL) ? tdata->thread_name : "")) 1311286866Sjasone return (tdata); 1312286866Sjasone return (NULL); 1313261071Sjasone} 1314234370Sjasone 1315286866Sjasone#ifdef JEMALLOC_JET 1316286866Sjasone#undef prof_dump_header 1317286866Sjasone#define prof_dump_header JEMALLOC_N(prof_dump_header_impl) 1318286866Sjasone#endif 1319286866Sjasonestatic bool 1320299587Sjasoneprof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) 1321261071Sjasone{ 1322286866Sjasone bool ret; 1323261071Sjasone 1324286866Sjasone if (prof_dump_printf(propagate_err, 1325286866Sjasone "heap_v2/%"FMTu64"\n" 1326286866Sjasone " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1327286866Sjasone ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, 1328286866Sjasone cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) 1329286866Sjasone return (true); 1330286866Sjasone 1331299587Sjasone malloc_mutex_lock(tsdn, &tdatas_mtx); 1332286866Sjasone ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, 1333286866Sjasone (void *)&propagate_err) != NULL); 1334299587Sjasone malloc_mutex_unlock(tsdn, &tdatas_mtx); 1335286866Sjasone return (ret); 1336234370Sjasone} 1337286866Sjasone#ifdef JEMALLOC_JET 1338286866Sjasone#undef prof_dump_header 1339286866Sjasone#define prof_dump_header JEMALLOC_N(prof_dump_header) 1340286866Sjasoneprof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); 1341286866Sjasone#endif 1342234370Sjasone 1343234370Sjasonestatic bool 1344299587Sjasoneprof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx, 1345299587Sjasone const prof_bt_t *bt, prof_gctx_tree_t *gctxs) 1346234370Sjasone{ 1347261071Sjasone bool ret; 1348234370Sjasone unsigned i; 1349299587Sjasone struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg; 1350234370Sjasone 1351234370Sjasone cassert(config_prof); 1352299587Sjasone malloc_mutex_assert_owner(tsdn, gctx->lock); 1353234370Sjasone 1354286866Sjasone /* Avoid dumping such gctx's that have no useful data. */ 1355286866Sjasone if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || 1356286866Sjasone (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { 1357286866Sjasone assert(gctx->cnt_summed.curobjs == 0); 1358286866Sjasone assert(gctx->cnt_summed.curbytes == 0); 1359286866Sjasone assert(gctx->cnt_summed.accumobjs == 0); 1360286866Sjasone assert(gctx->cnt_summed.accumbytes == 0); 1361261071Sjasone ret = false; 1362261071Sjasone goto label_return; 1363234370Sjasone } 1364234370Sjasone 1365286866Sjasone if (prof_dump_printf(propagate_err, "@")) { 1366261071Sjasone ret = true; 1367261071Sjasone goto label_return; 1368261071Sjasone } 1369234370Sjasone for (i = 0; i < bt->len; i++) { 1370286866Sjasone if (prof_dump_printf(propagate_err, " %#"FMTxPTR, 1371261071Sjasone (uintptr_t)bt->vec[i])) { 1372261071Sjasone ret = true; 1373261071Sjasone goto label_return; 1374261071Sjasone } 1375234370Sjasone } 1376234370Sjasone 1377286866Sjasone if (prof_dump_printf(propagate_err, 1378286866Sjasone "\n" 1379286866Sjasone " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1380286866Sjasone gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, 1381286866Sjasone gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { 1382261071Sjasone ret = true; 1383261071Sjasone goto label_return; 1384261071Sjasone } 1385234370Sjasone 1386299587Sjasone prof_tctx_dump_iter_arg.tsdn = tsdn; 1387299587Sjasone prof_tctx_dump_iter_arg.propagate_err = propagate_err; 1388286866Sjasone if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, 1389299587Sjasone (void *)&prof_tctx_dump_iter_arg) != NULL) { 1390286866Sjasone ret = true; 1391286866Sjasone goto label_return; 1392286866Sjasone } 1393286866Sjasone 1394261071Sjasone ret = false; 1395261071Sjasonelabel_return: 1396261071Sjasone return (ret); 1397234370Sjasone} 1398234370Sjasone 1399296221Sjasone#ifndef _WIN32 1400286866SjasoneJEMALLOC_FORMAT_PRINTF(1, 2) 1401286866Sjasonestatic int 1402286866Sjasoneprof_open_maps(const char *format, ...) 1403286866Sjasone{ 1404286866Sjasone int mfd; 1405286866Sjasone va_list ap; 1406286866Sjasone char filename[PATH_MAX + 1]; 1407286866Sjasone 1408286866Sjasone va_start(ap, format); 1409286866Sjasone malloc_vsnprintf(filename, sizeof(filename), format, ap); 1410286866Sjasone va_end(ap); 1411286866Sjasone mfd = open(filename, O_RDONLY); 1412286866Sjasone 1413286866Sjasone return (mfd); 1414286866Sjasone} 1415296221Sjasone#endif 1416286866Sjasone 1417296221Sjasonestatic int 1418296221Sjasoneprof_getpid(void) 1419296221Sjasone{ 1420296221Sjasone 1421296221Sjasone#ifdef _WIN32 1422296221Sjasone return (GetCurrentProcessId()); 1423296221Sjasone#else 1424296221Sjasone return (getpid()); 1425296221Sjasone#endif 1426296221Sjasone} 1427296221Sjasone 1428234370Sjasonestatic bool 1429234370Sjasoneprof_dump_maps(bool propagate_err) 1430234370Sjasone{ 1431261071Sjasone bool ret; 1432234370Sjasone int mfd; 1433234370Sjasone 1434234370Sjasone cassert(config_prof); 1435263974Sjasone#ifdef __FreeBSD__ 1436286866Sjasone mfd = prof_open_maps("/proc/curproc/map"); 1437296221Sjasone#elif defined(_WIN32) 1438296221Sjasone mfd = -1; // Not implemented 1439263974Sjasone#else 1440286866Sjasone { 1441296221Sjasone int pid = prof_getpid(); 1442286866Sjasone 1443286866Sjasone mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); 1444286866Sjasone if (mfd == -1) 1445286866Sjasone mfd = prof_open_maps("/proc/%d/maps", pid); 1446286866Sjasone } 1447263974Sjasone#endif 1448234370Sjasone if (mfd != -1) { 1449234370Sjasone ssize_t nread; 1450234370Sjasone 1451261071Sjasone if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && 1452261071Sjasone propagate_err) { 1453261071Sjasone ret = true; 1454261071Sjasone goto label_return; 1455261071Sjasone } 1456234370Sjasone nread = 0; 1457234370Sjasone do { 1458234370Sjasone prof_dump_buf_end += nread; 1459234370Sjasone if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { 1460234370Sjasone /* Make space in prof_dump_buf before read(). */ 1461261071Sjasone if (prof_dump_flush(propagate_err) && 1462261071Sjasone propagate_err) { 1463261071Sjasone ret = true; 1464261071Sjasone goto label_return; 1465261071Sjasone } 1466234370Sjasone } 1467234370Sjasone nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], 1468234370Sjasone PROF_DUMP_BUFSIZE - prof_dump_buf_end); 1469234370Sjasone } while (nread > 0); 1470261071Sjasone } else { 1471261071Sjasone ret = true; 1472261071Sjasone goto label_return; 1473261071Sjasone } 1474261071Sjasone 1475261071Sjasone ret = false; 1476261071Sjasonelabel_return: 1477261071Sjasone if (mfd != -1) 1478234370Sjasone close(mfd); 1479261071Sjasone return (ret); 1480261071Sjasone} 1481234370Sjasone 1482299587Sjasone/* 1483299587Sjasone * See prof_sample_threshold_update() comment for why the body of this function 1484299587Sjasone * is conditionally compiled. 1485299587Sjasone */ 1486261071Sjasonestatic void 1487286866Sjasoneprof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, 1488261071Sjasone const char *filename) 1489261071Sjasone{ 1490261071Sjasone 1491299587Sjasone#ifdef JEMALLOC_PROF 1492299587Sjasone /* 1493299587Sjasone * Scaling is equivalent AdjustSamples() in jeprof, but the result may 1494299587Sjasone * differ slightly from what jeprof reports, because here we scale the 1495299587Sjasone * summary values, whereas jeprof scales each context individually and 1496299587Sjasone * reports the sums of the scaled values. 1497299587Sjasone */ 1498261071Sjasone if (cnt_all->curbytes != 0) { 1499299587Sjasone double sample_period = (double)((uint64_t)1 << lg_prof_sample); 1500299587Sjasone double ratio = (((double)cnt_all->curbytes) / 1501299587Sjasone (double)cnt_all->curobjs) / sample_period; 1502299587Sjasone double scale_factor = 1.0 / (1.0 - exp(-ratio)); 1503299587Sjasone uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes) 1504299587Sjasone * scale_factor); 1505299587Sjasone uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) * 1506299587Sjasone scale_factor); 1507299587Sjasone 1508299587Sjasone malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64 1509299587Sjasone " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n", 1510299587Sjasone curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs != 1511299587Sjasone 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : ""); 1512261071Sjasone malloc_printf( 1513286866Sjasone "<jemalloc>: Run jeprof on \"%s\" for leak detail\n", 1514261071Sjasone filename); 1515261071Sjasone } 1516299587Sjasone#endif 1517234370Sjasone} 1518234370Sjasone 1519299587Sjasonestruct prof_gctx_dump_iter_arg_s { 1520299587Sjasone tsdn_t *tsdn; 1521299587Sjasone bool propagate_err; 1522299587Sjasone}; 1523299587Sjasone 1524286866Sjasonestatic prof_gctx_t * 1525299587Sjasoneprof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) 1526286866Sjasone{ 1527286866Sjasone prof_gctx_t *ret; 1528299587Sjasone struct prof_gctx_dump_iter_arg_s *arg = 1529299587Sjasone (struct prof_gctx_dump_iter_arg_s *)opaque; 1530286866Sjasone 1531299587Sjasone malloc_mutex_lock(arg->tsdn, gctx->lock); 1532286866Sjasone 1533299587Sjasone if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt, 1534299587Sjasone gctxs)) { 1535286866Sjasone ret = gctx; 1536286866Sjasone goto label_return; 1537286866Sjasone } 1538286866Sjasone 1539286866Sjasone ret = NULL; 1540286866Sjasonelabel_return: 1541299587Sjasone malloc_mutex_unlock(arg->tsdn, gctx->lock); 1542286866Sjasone return (ret); 1543286866Sjasone} 1544286866Sjasone 1545234370Sjasonestatic bool 1546286866Sjasoneprof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) 1547234370Sjasone{ 1548286866Sjasone prof_tdata_t *tdata; 1549299587Sjasone struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg; 1550234370Sjasone size_t tabind; 1551234370Sjasone union { 1552286866Sjasone prof_gctx_t *p; 1553234370Sjasone void *v; 1554286866Sjasone } gctx; 1555299587Sjasone struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg; 1556299587Sjasone struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg; 1557286866Sjasone prof_gctx_tree_t gctxs; 1558234370Sjasone 1559234370Sjasone cassert(config_prof); 1560234370Sjasone 1561286866Sjasone tdata = prof_tdata_get(tsd, true); 1562286866Sjasone if (tdata == NULL) 1563235238Sjasone return (true); 1564234370Sjasone 1565299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); 1566286866Sjasone prof_enter(tsd, tdata); 1567261071Sjasone 1568286866Sjasone /* 1569286866Sjasone * Put gctx's in limbo and clear their counters in preparation for 1570286866Sjasone * summing. 1571286866Sjasone */ 1572286866Sjasone gctx_tree_new(&gctxs); 1573286866Sjasone for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) 1574299587Sjasone prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, &gctxs); 1575286866Sjasone 1576286866Sjasone /* 1577286866Sjasone * Iterate over tdatas, and for the non-expired ones snapshot their tctx 1578286866Sjasone * stats and merge them into the associated gctx's. 1579286866Sjasone */ 1580299587Sjasone prof_tdata_merge_iter_arg.tsdn = tsd_tsdn(tsd); 1581299587Sjasone memset(&prof_tdata_merge_iter_arg.cnt_all, 0, sizeof(prof_cnt_t)); 1582299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); 1583299587Sjasone tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, 1584299587Sjasone (void *)&prof_tdata_merge_iter_arg); 1585299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); 1586234370Sjasone 1587286866Sjasone /* Merge tctx stats into gctx's. */ 1588299587Sjasone prof_gctx_merge_iter_arg.tsdn = tsd_tsdn(tsd); 1589299587Sjasone prof_gctx_merge_iter_arg.leak_ngctx = 0; 1590299587Sjasone gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, 1591299587Sjasone (void *)&prof_gctx_merge_iter_arg); 1592286866Sjasone 1593286866Sjasone prof_leave(tsd, tdata); 1594286866Sjasone 1595261071Sjasone /* Create dump file. */ 1596261071Sjasone if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) 1597261071Sjasone goto label_open_close_error; 1598261071Sjasone 1599234370Sjasone /* Dump profile header. */ 1600299587Sjasone if (prof_dump_header(tsd_tsdn(tsd), propagate_err, 1601299587Sjasone &prof_tdata_merge_iter_arg.cnt_all)) 1602261071Sjasone goto label_write_error; 1603234370Sjasone 1604286866Sjasone /* Dump per gctx profile stats. */ 1605299587Sjasone prof_gctx_dump_iter_arg.tsdn = tsd_tsdn(tsd); 1606299587Sjasone prof_gctx_dump_iter_arg.propagate_err = propagate_err; 1607286866Sjasone if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, 1608299587Sjasone (void *)&prof_gctx_dump_iter_arg) != NULL) 1609286866Sjasone goto label_write_error; 1610234370Sjasone 1611234370Sjasone /* Dump /proc/<pid>/maps if possible. */ 1612234370Sjasone if (prof_dump_maps(propagate_err)) 1613261071Sjasone goto label_write_error; 1614234370Sjasone 1615261071Sjasone if (prof_dump_close(propagate_err)) 1616261071Sjasone goto label_open_close_error; 1617234370Sjasone 1618286866Sjasone prof_gctx_finish(tsd, &gctxs); 1619299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); 1620234370Sjasone 1621299587Sjasone if (leakcheck) { 1622299587Sjasone prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all, 1623299587Sjasone prof_gctx_merge_iter_arg.leak_ngctx, filename); 1624299587Sjasone } 1625234370Sjasone return (false); 1626261071Sjasonelabel_write_error: 1627261071Sjasone prof_dump_close(propagate_err); 1628261071Sjasonelabel_open_close_error: 1629286866Sjasone prof_gctx_finish(tsd, &gctxs); 1630299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); 1631234370Sjasone return (true); 1632234370Sjasone} 1633234370Sjasone 1634234370Sjasone#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) 1635261071Sjasone#define VSEQ_INVALID UINT64_C(0xffffffffffffffff) 1636234370Sjasonestatic void 1637286866Sjasoneprof_dump_filename(char *filename, char v, uint64_t vseq) 1638234370Sjasone{ 1639234370Sjasone 1640234370Sjasone cassert(config_prof); 1641234370Sjasone 1642261071Sjasone if (vseq != VSEQ_INVALID) { 1643234370Sjasone /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ 1644234370Sjasone malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1645286866Sjasone "%s.%d.%"FMTu64".%c%"FMTu64".heap", 1646296221Sjasone opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq); 1647234370Sjasone } else { 1648234370Sjasone /* "<prefix>.<pid>.<seq>.<v>.heap" */ 1649234370Sjasone malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1650286866Sjasone "%s.%d.%"FMTu64".%c.heap", 1651296221Sjasone opt_prof_prefix, prof_getpid(), prof_dump_seq, v); 1652234370Sjasone } 1653235238Sjasone prof_dump_seq++; 1654234370Sjasone} 1655234370Sjasone 1656234370Sjasonestatic void 1657234370Sjasoneprof_fdump(void) 1658234370Sjasone{ 1659286866Sjasone tsd_t *tsd; 1660234370Sjasone char filename[DUMP_FILENAME_BUFSIZE]; 1661234370Sjasone 1662234370Sjasone cassert(config_prof); 1663286866Sjasone assert(opt_prof_final); 1664286866Sjasone assert(opt_prof_prefix[0] != '\0'); 1665234370Sjasone 1666286866Sjasone if (!prof_booted) 1667234370Sjasone return; 1668286866Sjasone tsd = tsd_fetch(); 1669234370Sjasone 1670299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1671286866Sjasone prof_dump_filename(filename, 'f', VSEQ_INVALID); 1672299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1673286866Sjasone prof_dump(tsd, false, filename, opt_prof_leak); 1674234370Sjasone} 1675234370Sjasone 1676234370Sjasonevoid 1677299587Sjasoneprof_idump(tsdn_t *tsdn) 1678234370Sjasone{ 1679286866Sjasone tsd_t *tsd; 1680286866Sjasone prof_tdata_t *tdata; 1681234370Sjasone 1682234370Sjasone cassert(config_prof); 1683234370Sjasone 1684299587Sjasone if (!prof_booted || tsdn_null(tsdn)) 1685234370Sjasone return; 1686299587Sjasone tsd = tsdn_tsd(tsdn); 1687286866Sjasone tdata = prof_tdata_get(tsd, false); 1688286866Sjasone if (tdata == NULL) 1689234370Sjasone return; 1690286866Sjasone if (tdata->enq) { 1691286866Sjasone tdata->enq_idump = true; 1692235238Sjasone return; 1693234370Sjasone } 1694234370Sjasone 1695234370Sjasone if (opt_prof_prefix[0] != '\0') { 1696288090Sjasone char filename[PATH_MAX + 1]; 1697299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1698234370Sjasone prof_dump_filename(filename, 'i', prof_dump_iseq); 1699234370Sjasone prof_dump_iseq++; 1700299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1701286866Sjasone prof_dump(tsd, false, filename, false); 1702234370Sjasone } 1703234370Sjasone} 1704234370Sjasone 1705234370Sjasonebool 1706299587Sjasoneprof_mdump(tsd_t *tsd, const char *filename) 1707234370Sjasone{ 1708234370Sjasone char filename_buf[DUMP_FILENAME_BUFSIZE]; 1709234370Sjasone 1710234370Sjasone cassert(config_prof); 1711234370Sjasone 1712286866Sjasone if (!opt_prof || !prof_booted) 1713234370Sjasone return (true); 1714234370Sjasone 1715234370Sjasone if (filename == NULL) { 1716234370Sjasone /* No filename specified, so automatically generate one. */ 1717234370Sjasone if (opt_prof_prefix[0] == '\0') 1718234370Sjasone return (true); 1719299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1720234370Sjasone prof_dump_filename(filename_buf, 'm', prof_dump_mseq); 1721234370Sjasone prof_dump_mseq++; 1722299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); 1723234370Sjasone filename = filename_buf; 1724234370Sjasone } 1725286866Sjasone return (prof_dump(tsd, true, filename, false)); 1726234370Sjasone} 1727234370Sjasone 1728234370Sjasonevoid 1729299587Sjasoneprof_gdump(tsdn_t *tsdn) 1730234370Sjasone{ 1731286866Sjasone tsd_t *tsd; 1732286866Sjasone prof_tdata_t *tdata; 1733234370Sjasone 1734234370Sjasone cassert(config_prof); 1735234370Sjasone 1736299587Sjasone if (!prof_booted || tsdn_null(tsdn)) 1737234370Sjasone return; 1738299587Sjasone tsd = tsdn_tsd(tsdn); 1739286866Sjasone tdata = prof_tdata_get(tsd, false); 1740286866Sjasone if (tdata == NULL) 1741234370Sjasone return; 1742286866Sjasone if (tdata->enq) { 1743286866Sjasone tdata->enq_gdump = true; 1744235238Sjasone return; 1745234370Sjasone } 1746234370Sjasone 1747234370Sjasone if (opt_prof_prefix[0] != '\0') { 1748288090Sjasone char filename[DUMP_FILENAME_BUFSIZE]; 1749299587Sjasone malloc_mutex_lock(tsdn, &prof_dump_seq_mtx); 1750234370Sjasone prof_dump_filename(filename, 'u', prof_dump_useq); 1751234370Sjasone prof_dump_useq++; 1752299587Sjasone malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx); 1753286866Sjasone prof_dump(tsd, false, filename, false); 1754234370Sjasone } 1755234370Sjasone} 1756234370Sjasone 1757234370Sjasonestatic void 1758245868Sjasoneprof_bt_hash(const void *key, size_t r_hash[2]) 1759234370Sjasone{ 1760234370Sjasone prof_bt_t *bt = (prof_bt_t *)key; 1761234370Sjasone 1762234370Sjasone cassert(config_prof); 1763234370Sjasone 1764245868Sjasone hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); 1765234370Sjasone} 1766234370Sjasone 1767234370Sjasonestatic bool 1768234370Sjasoneprof_bt_keycomp(const void *k1, const void *k2) 1769234370Sjasone{ 1770234370Sjasone const prof_bt_t *bt1 = (prof_bt_t *)k1; 1771234370Sjasone const prof_bt_t *bt2 = (prof_bt_t *)k2; 1772234370Sjasone 1773234370Sjasone cassert(config_prof); 1774234370Sjasone 1775234370Sjasone if (bt1->len != bt2->len) 1776234370Sjasone return (false); 1777234370Sjasone return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); 1778234370Sjasone} 1779234370Sjasone 1780286866SjasoneJEMALLOC_INLINE_C uint64_t 1781299587Sjasoneprof_thr_uid_alloc(tsdn_t *tsdn) 1782234370Sjasone{ 1783286866Sjasone uint64_t thr_uid; 1784234370Sjasone 1785299587Sjasone malloc_mutex_lock(tsdn, &next_thr_uid_mtx); 1786286866Sjasone thr_uid = next_thr_uid; 1787286866Sjasone next_thr_uid++; 1788299587Sjasone malloc_mutex_unlock(tsdn, &next_thr_uid_mtx); 1789286866Sjasone 1790286866Sjasone return (thr_uid); 1791286866Sjasone} 1792286866Sjasone 1793286866Sjasonestatic prof_tdata_t * 1794299587Sjasoneprof_tdata_init_impl(tsdn_t *tsdn, uint64_t thr_uid, uint64_t thr_discrim, 1795286866Sjasone char *thread_name, bool active) 1796286866Sjasone{ 1797286866Sjasone prof_tdata_t *tdata; 1798286866Sjasone 1799234370Sjasone cassert(config_prof); 1800234370Sjasone 1801234370Sjasone /* Initialize an empty cache for this thread. */ 1802299587Sjasone tdata = (prof_tdata_t *)iallocztm(tsdn, sizeof(prof_tdata_t), 1803299587Sjasone size2index(sizeof(prof_tdata_t)), false, NULL, true, 1804299587Sjasone arena_get(TSDN_NULL, 0, true), true); 1805286866Sjasone if (tdata == NULL) 1806234370Sjasone return (NULL); 1807234370Sjasone 1808286866Sjasone tdata->lock = prof_tdata_mutex_choose(thr_uid); 1809286866Sjasone tdata->thr_uid = thr_uid; 1810286866Sjasone tdata->thr_discrim = thr_discrim; 1811286866Sjasone tdata->thread_name = thread_name; 1812286866Sjasone tdata->attached = true; 1813286866Sjasone tdata->expired = false; 1814286866Sjasone tdata->tctx_uid_next = 0; 1815286866Sjasone 1816299587Sjasone if (ckh_new(tsdn, &tdata->bt2tctx, PROF_CKH_MINITEMS, 1817234370Sjasone prof_bt_hash, prof_bt_keycomp)) { 1818299587Sjasone idalloctm(tsdn, tdata, NULL, true, true); 1819234370Sjasone return (NULL); 1820234370Sjasone } 1821234370Sjasone 1822286866Sjasone tdata->prng_state = (uint64_t)(uintptr_t)tdata; 1823286866Sjasone prof_sample_threshold_update(tdata); 1824234370Sjasone 1825286866Sjasone tdata->enq = false; 1826286866Sjasone tdata->enq_idump = false; 1827286866Sjasone tdata->enq_gdump = false; 1828234370Sjasone 1829286866Sjasone tdata->dumping = false; 1830286866Sjasone tdata->active = active; 1831235238Sjasone 1832299587Sjasone malloc_mutex_lock(tsdn, &tdatas_mtx); 1833286866Sjasone tdata_tree_insert(&tdatas, tdata); 1834299587Sjasone malloc_mutex_unlock(tsdn, &tdatas_mtx); 1835234370Sjasone 1836286866Sjasone return (tdata); 1837234370Sjasone} 1838234370Sjasone 1839286866Sjasoneprof_tdata_t * 1840299587Sjasoneprof_tdata_init(tsdn_t *tsdn) 1841234370Sjasone{ 1842234370Sjasone 1843299587Sjasone return (prof_tdata_init_impl(tsdn, prof_thr_uid_alloc(tsdn), 0, NULL, 1844299587Sjasone prof_thread_active_init_get(tsdn))); 1845286866Sjasone} 1846234370Sjasone 1847286866Sjasonestatic bool 1848299587Sjasoneprof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) 1849286866Sjasone{ 1850286866Sjasone 1851286866Sjasone if (tdata->attached && !even_if_attached) 1852286866Sjasone return (false); 1853286866Sjasone if (ckh_count(&tdata->bt2tctx) != 0) 1854286866Sjasone return (false); 1855286866Sjasone return (true); 1856286866Sjasone} 1857286866Sjasone 1858299587Sjasonestatic bool 1859299587Sjasoneprof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, 1860299587Sjasone bool even_if_attached) 1861299587Sjasone{ 1862299587Sjasone 1863299587Sjasone malloc_mutex_assert_owner(tsdn, tdata->lock); 1864299587Sjasone 1865299587Sjasone return (prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); 1866299587Sjasone} 1867299587Sjasone 1868286866Sjasonestatic void 1869299587Sjasoneprof_tdata_destroy_locked(tsdn_t *tsdn, prof_tdata_t *tdata, 1870286866Sjasone bool even_if_attached) 1871286866Sjasone{ 1872286866Sjasone 1873299587Sjasone malloc_mutex_assert_owner(tsdn, &tdatas_mtx); 1874286866Sjasone 1875299587Sjasone assert(tsdn_null(tsdn) || tsd_prof_tdata_get(tsdn_tsd(tsdn)) != tdata); 1876299587Sjasone 1877286866Sjasone tdata_tree_remove(&tdatas, tdata); 1878286866Sjasone 1879299587Sjasone assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); 1880299587Sjasone 1881286866Sjasone if (tdata->thread_name != NULL) 1882299587Sjasone idalloctm(tsdn, tdata->thread_name, NULL, true, true); 1883299587Sjasone ckh_delete(tsdn, &tdata->bt2tctx); 1884299587Sjasone idalloctm(tsdn, tdata, NULL, true, true); 1885286866Sjasone} 1886286866Sjasone 1887286866Sjasonestatic void 1888299587Sjasoneprof_tdata_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, bool even_if_attached) 1889286866Sjasone{ 1890286866Sjasone 1891299587Sjasone malloc_mutex_lock(tsdn, &tdatas_mtx); 1892299587Sjasone prof_tdata_destroy_locked(tsdn, tdata, even_if_attached); 1893299587Sjasone malloc_mutex_unlock(tsdn, &tdatas_mtx); 1894286866Sjasone} 1895286866Sjasone 1896286866Sjasonestatic void 1897286866Sjasoneprof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) 1898286866Sjasone{ 1899286866Sjasone bool destroy_tdata; 1900286866Sjasone 1901299587Sjasone malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); 1902286866Sjasone if (tdata->attached) { 1903299587Sjasone destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, 1904299587Sjasone true); 1905235238Sjasone /* 1906286866Sjasone * Only detach if !destroy_tdata, because detaching would allow 1907286866Sjasone * another thread to win the race to destroy tdata. 1908235238Sjasone */ 1909286866Sjasone if (!destroy_tdata) 1910286866Sjasone tdata->attached = false; 1911286866Sjasone tsd_prof_tdata_set(tsd, NULL); 1912286866Sjasone } else 1913286866Sjasone destroy_tdata = false; 1914299587Sjasone malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); 1915286866Sjasone if (destroy_tdata) 1916299587Sjasone prof_tdata_destroy(tsd_tsdn(tsd), tdata, true); 1917286866Sjasone} 1918286866Sjasone 1919286866Sjasoneprof_tdata_t * 1920286866Sjasoneprof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) 1921286866Sjasone{ 1922286866Sjasone uint64_t thr_uid = tdata->thr_uid; 1923286866Sjasone uint64_t thr_discrim = tdata->thr_discrim + 1; 1924286866Sjasone char *thread_name = (tdata->thread_name != NULL) ? 1925299587Sjasone prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL; 1926286866Sjasone bool active = tdata->active; 1927286866Sjasone 1928286866Sjasone prof_tdata_detach(tsd, tdata); 1929299587Sjasone return (prof_tdata_init_impl(tsd_tsdn(tsd), thr_uid, thr_discrim, 1930299587Sjasone thread_name, active)); 1931286866Sjasone} 1932286866Sjasone 1933286866Sjasonestatic bool 1934299587Sjasoneprof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) 1935286866Sjasone{ 1936286866Sjasone bool destroy_tdata; 1937286866Sjasone 1938299587Sjasone malloc_mutex_lock(tsdn, tdata->lock); 1939286866Sjasone if (!tdata->expired) { 1940286866Sjasone tdata->expired = true; 1941286866Sjasone destroy_tdata = tdata->attached ? false : 1942299587Sjasone prof_tdata_should_destroy(tsdn, tdata, false); 1943286866Sjasone } else 1944286866Sjasone destroy_tdata = false; 1945299587Sjasone malloc_mutex_unlock(tsdn, tdata->lock); 1946286866Sjasone 1947286866Sjasone return (destroy_tdata); 1948286866Sjasone} 1949286866Sjasone 1950286866Sjasonestatic prof_tdata_t * 1951286866Sjasoneprof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1952286866Sjasone{ 1953299587Sjasone tsdn_t *tsdn = (tsdn_t *)arg; 1954286866Sjasone 1955299587Sjasone return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL); 1956286866Sjasone} 1957286866Sjasone 1958286866Sjasonevoid 1959299587Sjasoneprof_reset(tsdn_t *tsdn, size_t lg_sample) 1960286866Sjasone{ 1961286866Sjasone prof_tdata_t *next; 1962286866Sjasone 1963286866Sjasone assert(lg_sample < (sizeof(uint64_t) << 3)); 1964286866Sjasone 1965299587Sjasone malloc_mutex_lock(tsdn, &prof_dump_mtx); 1966299587Sjasone malloc_mutex_lock(tsdn, &tdatas_mtx); 1967286866Sjasone 1968286866Sjasone lg_prof_sample = lg_sample; 1969286866Sjasone 1970286866Sjasone next = NULL; 1971286866Sjasone do { 1972286866Sjasone prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, 1973299587Sjasone prof_tdata_reset_iter, (void *)tsdn); 1974286866Sjasone if (to_destroy != NULL) { 1975286866Sjasone next = tdata_tree_next(&tdatas, to_destroy); 1976299587Sjasone prof_tdata_destroy_locked(tsdn, to_destroy, false); 1977286866Sjasone } else 1978286866Sjasone next = NULL; 1979286866Sjasone } while (next != NULL); 1980286866Sjasone 1981299587Sjasone malloc_mutex_unlock(tsdn, &tdatas_mtx); 1982299587Sjasone malloc_mutex_unlock(tsdn, &prof_dump_mtx); 1983286866Sjasone} 1984286866Sjasone 1985286866Sjasonevoid 1986286866Sjasoneprof_tdata_cleanup(tsd_t *tsd) 1987286866Sjasone{ 1988286866Sjasone prof_tdata_t *tdata; 1989286866Sjasone 1990286866Sjasone if (!config_prof) 1991286866Sjasone return; 1992286866Sjasone 1993286866Sjasone tdata = tsd_prof_tdata_get(tsd); 1994286866Sjasone if (tdata != NULL) 1995286866Sjasone prof_tdata_detach(tsd, tdata); 1996286866Sjasone} 1997286866Sjasone 1998286866Sjasonebool 1999299587Sjasoneprof_active_get(tsdn_t *tsdn) 2000286866Sjasone{ 2001286866Sjasone bool prof_active_current; 2002286866Sjasone 2003299587Sjasone malloc_mutex_lock(tsdn, &prof_active_mtx); 2004286866Sjasone prof_active_current = prof_active; 2005299587Sjasone malloc_mutex_unlock(tsdn, &prof_active_mtx); 2006286866Sjasone return (prof_active_current); 2007286866Sjasone} 2008286866Sjasone 2009286866Sjasonebool 2010299587Sjasoneprof_active_set(tsdn_t *tsdn, bool active) 2011286866Sjasone{ 2012286866Sjasone bool prof_active_old; 2013286866Sjasone 2014299587Sjasone malloc_mutex_lock(tsdn, &prof_active_mtx); 2015286866Sjasone prof_active_old = prof_active; 2016286866Sjasone prof_active = active; 2017299587Sjasone malloc_mutex_unlock(tsdn, &prof_active_mtx); 2018286866Sjasone return (prof_active_old); 2019286866Sjasone} 2020286866Sjasone 2021286866Sjasoneconst char * 2022299587Sjasoneprof_thread_name_get(tsd_t *tsd) 2023286866Sjasone{ 2024286866Sjasone prof_tdata_t *tdata; 2025286866Sjasone 2026286866Sjasone tdata = prof_tdata_get(tsd, true); 2027286866Sjasone if (tdata == NULL) 2028286866Sjasone return (""); 2029286866Sjasone return (tdata->thread_name != NULL ? tdata->thread_name : ""); 2030286866Sjasone} 2031286866Sjasone 2032286866Sjasonestatic char * 2033299587Sjasoneprof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) 2034286866Sjasone{ 2035286866Sjasone char *ret; 2036286866Sjasone size_t size; 2037286866Sjasone 2038286866Sjasone if (thread_name == NULL) 2039286866Sjasone return (NULL); 2040286866Sjasone 2041286866Sjasone size = strlen(thread_name) + 1; 2042286866Sjasone if (size == 1) 2043286866Sjasone return (""); 2044286866Sjasone 2045299587Sjasone ret = iallocztm(tsdn, size, size2index(size), false, NULL, true, 2046299587Sjasone arena_get(TSDN_NULL, 0, true), true); 2047286866Sjasone if (ret == NULL) 2048286866Sjasone return (NULL); 2049286866Sjasone memcpy(ret, thread_name, size); 2050286866Sjasone return (ret); 2051286866Sjasone} 2052286866Sjasone 2053286866Sjasoneint 2054286866Sjasoneprof_thread_name_set(tsd_t *tsd, const char *thread_name) 2055286866Sjasone{ 2056286866Sjasone prof_tdata_t *tdata; 2057286866Sjasone unsigned i; 2058286866Sjasone char *s; 2059286866Sjasone 2060286866Sjasone tdata = prof_tdata_get(tsd, true); 2061286866Sjasone if (tdata == NULL) 2062286866Sjasone return (EAGAIN); 2063286866Sjasone 2064286866Sjasone /* Validate input. */ 2065286866Sjasone if (thread_name == NULL) 2066286866Sjasone return (EFAULT); 2067286866Sjasone for (i = 0; thread_name[i] != '\0'; i++) { 2068286866Sjasone char c = thread_name[i]; 2069286866Sjasone if (!isgraph(c) && !isblank(c)) 2070286866Sjasone return (EFAULT); 2071234370Sjasone } 2072286866Sjasone 2073299587Sjasone s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name); 2074286866Sjasone if (s == NULL) 2075286866Sjasone return (EAGAIN); 2076286866Sjasone 2077286866Sjasone if (tdata->thread_name != NULL) { 2078299587Sjasone idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true); 2079286866Sjasone tdata->thread_name = NULL; 2080286866Sjasone } 2081286866Sjasone if (strlen(s) > 0) 2082286866Sjasone tdata->thread_name = s; 2083286866Sjasone return (0); 2084234370Sjasone} 2085234370Sjasone 2086286866Sjasonebool 2087299587Sjasoneprof_thread_active_get(tsd_t *tsd) 2088286866Sjasone{ 2089286866Sjasone prof_tdata_t *tdata; 2090286866Sjasone 2091286866Sjasone tdata = prof_tdata_get(tsd, true); 2092286866Sjasone if (tdata == NULL) 2093286866Sjasone return (false); 2094286866Sjasone return (tdata->active); 2095286866Sjasone} 2096286866Sjasone 2097286866Sjasonebool 2098299587Sjasoneprof_thread_active_set(tsd_t *tsd, bool active) 2099286866Sjasone{ 2100286866Sjasone prof_tdata_t *tdata; 2101286866Sjasone 2102286866Sjasone tdata = prof_tdata_get(tsd, true); 2103286866Sjasone if (tdata == NULL) 2104286866Sjasone return (true); 2105286866Sjasone tdata->active = active; 2106286866Sjasone return (false); 2107286866Sjasone} 2108286866Sjasone 2109286866Sjasonebool 2110299587Sjasoneprof_thread_active_init_get(tsdn_t *tsdn) 2111286866Sjasone{ 2112286866Sjasone bool active_init; 2113286866Sjasone 2114299587Sjasone malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); 2115286866Sjasone active_init = prof_thread_active_init; 2116299587Sjasone malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); 2117286866Sjasone return (active_init); 2118286866Sjasone} 2119286866Sjasone 2120286866Sjasonebool 2121299587Sjasoneprof_thread_active_init_set(tsdn_t *tsdn, bool active_init) 2122286866Sjasone{ 2123286866Sjasone bool active_init_old; 2124286866Sjasone 2125299587Sjasone malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); 2126286866Sjasone active_init_old = prof_thread_active_init; 2127286866Sjasone prof_thread_active_init = active_init; 2128299587Sjasone malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); 2129286866Sjasone return (active_init_old); 2130286866Sjasone} 2131286866Sjasone 2132286866Sjasonebool 2133299587Sjasoneprof_gdump_get(tsdn_t *tsdn) 2134286866Sjasone{ 2135286866Sjasone bool prof_gdump_current; 2136286866Sjasone 2137299587Sjasone malloc_mutex_lock(tsdn, &prof_gdump_mtx); 2138286866Sjasone prof_gdump_current = prof_gdump_val; 2139299587Sjasone malloc_mutex_unlock(tsdn, &prof_gdump_mtx); 2140286866Sjasone return (prof_gdump_current); 2141286866Sjasone} 2142286866Sjasone 2143286866Sjasonebool 2144299587Sjasoneprof_gdump_set(tsdn_t *tsdn, bool gdump) 2145286866Sjasone{ 2146286866Sjasone bool prof_gdump_old; 2147286866Sjasone 2148299587Sjasone malloc_mutex_lock(tsdn, &prof_gdump_mtx); 2149286866Sjasone prof_gdump_old = prof_gdump_val; 2150286866Sjasone prof_gdump_val = gdump; 2151299587Sjasone malloc_mutex_unlock(tsdn, &prof_gdump_mtx); 2152286866Sjasone return (prof_gdump_old); 2153286866Sjasone} 2154286866Sjasone 2155234370Sjasonevoid 2156234370Sjasoneprof_boot0(void) 2157234370Sjasone{ 2158234370Sjasone 2159234370Sjasone cassert(config_prof); 2160234370Sjasone 2161234370Sjasone memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, 2162234370Sjasone sizeof(PROF_PREFIX_DEFAULT)); 2163234370Sjasone} 2164234370Sjasone 2165234370Sjasonevoid 2166234370Sjasoneprof_boot1(void) 2167234370Sjasone{ 2168234370Sjasone 2169234370Sjasone cassert(config_prof); 2170234370Sjasone 2171234370Sjasone /* 2172286866Sjasone * opt_prof must be in its final state before any arenas are 2173286866Sjasone * initialized, so this function must be executed early. 2174234370Sjasone */ 2175234370Sjasone 2176286866Sjasone if (opt_prof_leak && !opt_prof) { 2177234370Sjasone /* 2178234370Sjasone * Enable opt_prof, but in such a way that profiles are never 2179234370Sjasone * automatically dumped. 2180234370Sjasone */ 2181234370Sjasone opt_prof = true; 2182234370Sjasone opt_prof_gdump = false; 2183234370Sjasone } else if (opt_prof) { 2184234370Sjasone if (opt_lg_prof_interval >= 0) { 2185234370Sjasone prof_interval = (((uint64_t)1U) << 2186234370Sjasone opt_lg_prof_interval); 2187245868Sjasone } 2188234370Sjasone } 2189234370Sjasone} 2190234370Sjasone 2191234370Sjasonebool 2192299587Sjasoneprof_boot2(tsdn_t *tsdn) 2193234370Sjasone{ 2194234370Sjasone 2195234370Sjasone cassert(config_prof); 2196234370Sjasone 2197234370Sjasone if (opt_prof) { 2198234370Sjasone unsigned i; 2199234370Sjasone 2200286866Sjasone lg_prof_sample = opt_lg_prof_sample; 2201286866Sjasone 2202286866Sjasone prof_active = opt_prof_active; 2203299587Sjasone if (malloc_mutex_init(&prof_active_mtx, "prof_active", 2204299587Sjasone WITNESS_RANK_PROF_ACTIVE)) 2205286866Sjasone return (true); 2206286866Sjasone 2207286866Sjasone prof_gdump_val = opt_prof_gdump; 2208299587Sjasone if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump", 2209299587Sjasone WITNESS_RANK_PROF_GDUMP)) 2210286866Sjasone return (true); 2211286866Sjasone 2212286866Sjasone prof_thread_active_init = opt_prof_thread_active_init; 2213299587Sjasone if (malloc_mutex_init(&prof_thread_active_init_mtx, 2214299587Sjasone "prof_thread_active_init", 2215299587Sjasone WITNESS_RANK_PROF_THREAD_ACTIVE_INIT)) 2216286866Sjasone return (true); 2217286866Sjasone 2218299587Sjasone if (ckh_new(tsdn, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, 2219234370Sjasone prof_bt_keycomp)) 2220234370Sjasone return (true); 2221299587Sjasone if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx", 2222299587Sjasone WITNESS_RANK_PROF_BT2GCTX)) 2223234370Sjasone return (true); 2224234370Sjasone 2225286866Sjasone tdata_tree_new(&tdatas); 2226299587Sjasone if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas", 2227299587Sjasone WITNESS_RANK_PROF_TDATAS)) 2228286866Sjasone return (true); 2229286866Sjasone 2230286866Sjasone next_thr_uid = 0; 2231299587Sjasone if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid", 2232299587Sjasone WITNESS_RANK_PROF_NEXT_THR_UID)) 2233286866Sjasone return (true); 2234286866Sjasone 2235299587Sjasone if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq", 2236299587Sjasone WITNESS_RANK_PROF_DUMP_SEQ)) 2237234370Sjasone return (true); 2238299587Sjasone if (malloc_mutex_init(&prof_dump_mtx, "prof_dump", 2239299587Sjasone WITNESS_RANK_PROF_DUMP)) 2240261071Sjasone return (true); 2241234370Sjasone 2242286866Sjasone if (opt_prof_final && opt_prof_prefix[0] != '\0' && 2243286866Sjasone atexit(prof_fdump) != 0) { 2244234370Sjasone malloc_write("<jemalloc>: Error in atexit()\n"); 2245234370Sjasone if (opt_abort) 2246234370Sjasone abort(); 2247234370Sjasone } 2248234370Sjasone 2249299587Sjasone gctx_locks = (malloc_mutex_t *)base_alloc(tsdn, PROF_NCTX_LOCKS 2250299587Sjasone * sizeof(malloc_mutex_t)); 2251286866Sjasone if (gctx_locks == NULL) 2252234370Sjasone return (true); 2253234370Sjasone for (i = 0; i < PROF_NCTX_LOCKS; i++) { 2254299587Sjasone if (malloc_mutex_init(&gctx_locks[i], "prof_gctx", 2255299587Sjasone WITNESS_RANK_PROF_GCTX)) 2256234370Sjasone return (true); 2257234370Sjasone } 2258286866Sjasone 2259299587Sjasone tdata_locks = (malloc_mutex_t *)base_alloc(tsdn, 2260299587Sjasone PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t)); 2261286866Sjasone if (tdata_locks == NULL) 2262286866Sjasone return (true); 2263286866Sjasone for (i = 0; i < PROF_NTDATA_LOCKS; i++) { 2264299587Sjasone if (malloc_mutex_init(&tdata_locks[i], "prof_tdata", 2265299587Sjasone WITNESS_RANK_PROF_TDATA)) 2266286866Sjasone return (true); 2267286866Sjasone } 2268234370Sjasone } 2269234370Sjasone 2270234370Sjasone#ifdef JEMALLOC_PROF_LIBGCC 2271234370Sjasone /* 2272234370Sjasone * Cause the backtracing machinery to allocate its internal state 2273234370Sjasone * before enabling profiling. 2274234370Sjasone */ 2275234370Sjasone _Unwind_Backtrace(prof_unwind_init_callback, NULL); 2276234370Sjasone#endif 2277234370Sjasone 2278234370Sjasone prof_booted = true; 2279234370Sjasone 2280234370Sjasone return (false); 2281234370Sjasone} 2282234370Sjasone 2283242844Sjasonevoid 2284299587Sjasoneprof_prefork0(tsdn_t *tsdn) 2285242844Sjasone{ 2286242844Sjasone 2287242844Sjasone if (opt_prof) { 2288242844Sjasone unsigned i; 2289242844Sjasone 2290299587Sjasone malloc_mutex_prefork(tsdn, &prof_dump_mtx); 2291299587Sjasone malloc_mutex_prefork(tsdn, &bt2gctx_mtx); 2292299587Sjasone malloc_mutex_prefork(tsdn, &tdatas_mtx); 2293299587Sjasone for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2294299587Sjasone malloc_mutex_prefork(tsdn, &tdata_locks[i]); 2295242844Sjasone for (i = 0; i < PROF_NCTX_LOCKS; i++) 2296299587Sjasone malloc_mutex_prefork(tsdn, &gctx_locks[i]); 2297242844Sjasone } 2298242844Sjasone} 2299242844Sjasone 2300242844Sjasonevoid 2301299587Sjasoneprof_prefork1(tsdn_t *tsdn) 2302242844Sjasone{ 2303242844Sjasone 2304242844Sjasone if (opt_prof) { 2305299587Sjasone malloc_mutex_prefork(tsdn, &prof_active_mtx); 2306299587Sjasone malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx); 2307299587Sjasone malloc_mutex_prefork(tsdn, &prof_gdump_mtx); 2308299587Sjasone malloc_mutex_prefork(tsdn, &next_thr_uid_mtx); 2309299587Sjasone malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx); 2310299587Sjasone } 2311299587Sjasone} 2312299587Sjasone 2313299587Sjasonevoid 2314299587Sjasoneprof_postfork_parent(tsdn_t *tsdn) 2315299587Sjasone{ 2316299587Sjasone 2317299587Sjasone if (opt_prof) { 2318242844Sjasone unsigned i; 2319242844Sjasone 2320299587Sjasone malloc_mutex_postfork_parent(tsdn, 2321299587Sjasone &prof_thread_active_init_mtx); 2322299587Sjasone malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx); 2323299587Sjasone malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx); 2324299587Sjasone malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx); 2325299587Sjasone malloc_mutex_postfork_parent(tsdn, &prof_active_mtx); 2326299587Sjasone for (i = 0; i < PROF_NCTX_LOCKS; i++) 2327299587Sjasone malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]); 2328286866Sjasone for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2329299587Sjasone malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]); 2330299587Sjasone malloc_mutex_postfork_parent(tsdn, &tdatas_mtx); 2331299587Sjasone malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx); 2332299587Sjasone malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx); 2333242844Sjasone } 2334242844Sjasone} 2335242844Sjasone 2336242844Sjasonevoid 2337299587Sjasoneprof_postfork_child(tsdn_t *tsdn) 2338242844Sjasone{ 2339242844Sjasone 2340242844Sjasone if (opt_prof) { 2341242844Sjasone unsigned i; 2342242844Sjasone 2343299587Sjasone malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx); 2344299587Sjasone malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx); 2345299587Sjasone malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx); 2346299587Sjasone malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx); 2347299587Sjasone malloc_mutex_postfork_child(tsdn, &prof_active_mtx); 2348299587Sjasone for (i = 0; i < PROF_NCTX_LOCKS; i++) 2349299587Sjasone malloc_mutex_postfork_child(tsdn, &gctx_locks[i]); 2350286866Sjasone for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2351299587Sjasone malloc_mutex_postfork_child(tsdn, &tdata_locks[i]); 2352299587Sjasone malloc_mutex_postfork_child(tsdn, &tdatas_mtx); 2353299587Sjasone malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx); 2354299587Sjasone malloc_mutex_postfork_child(tsdn, &prof_dump_mtx); 2355242844Sjasone } 2356242844Sjasone} 2357242844Sjasone 2358234370Sjasone/******************************************************************************/ 2359