1234370Sjasone/******************************************************************************/ 2234370Sjasone#ifdef JEMALLOC_H_TYPES 3234370Sjasone 4234370Sjasonetypedef struct prof_bt_s prof_bt_t; 5234370Sjasonetypedef struct prof_cnt_s prof_cnt_t; 6286866Sjasonetypedef struct prof_tctx_s prof_tctx_t; 7286866Sjasonetypedef struct prof_gctx_s prof_gctx_t; 8234370Sjasonetypedef struct prof_tdata_s prof_tdata_t; 9234370Sjasone 10234370Sjasone/* Option defaults. */ 11262521Sjasone#ifdef JEMALLOC_PROF 12262521Sjasone# define PROF_PREFIX_DEFAULT "jeprof" 13262521Sjasone#else 14262521Sjasone# define PROF_PREFIX_DEFAULT "" 15262521Sjasone#endif 16234543Sjasone#define LG_PROF_SAMPLE_DEFAULT 19 17234370Sjasone#define LG_PROF_INTERVAL_DEFAULT -1 18234370Sjasone 19234370Sjasone/* 20234370Sjasone * Hard limit on stack backtrace depth. The version of prof_backtrace() that 21234370Sjasone * is based on __builtin_return_address() necessarily has a hard-coded number 22234370Sjasone * of backtrace frame handlers, and should be kept in sync with this setting. 23234370Sjasone */ 24234370Sjasone#define PROF_BT_MAX 128 25234370Sjasone 26234370Sjasone/* Initial hash table size. */ 27234370Sjasone#define PROF_CKH_MINITEMS 64 28234370Sjasone 29234370Sjasone/* Size of memory buffer to use when writing dump files. */ 30234370Sjasone#define PROF_DUMP_BUFSIZE 65536 31234370Sjasone 32234370Sjasone/* Size of stack-allocated buffer used by prof_printf(). */ 33234370Sjasone#define PROF_PRINTF_BUFSIZE 128 34234370Sjasone 35234370Sjasone/* 36286866Sjasone * Number of mutexes shared among all gctx's. No space is allocated for these 37234370Sjasone * unless profiling is enabled, so it's okay to over-provision. 38234370Sjasone */ 39234370Sjasone#define PROF_NCTX_LOCKS 1024 40234370Sjasone 41235238Sjasone/* 42286866Sjasone * Number of mutexes shared among all tdata's. No space is allocated for these 43286866Sjasone * unless profiling is enabled, so it's okay to over-provision. 44286866Sjasone */ 45286866Sjasone#define PROF_NTDATA_LOCKS 256 46286866Sjasone 47286866Sjasone/* 48235238Sjasone * prof_tdata pointers close to NULL are used to encode state information that 49235238Sjasone * is used for cleaning up during thread shutdown. 50235238Sjasone */ 51235238Sjasone#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) 52235238Sjasone#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) 53235238Sjasone#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY 54235238Sjasone 55234370Sjasone#endif /* JEMALLOC_H_TYPES */ 56234370Sjasone/******************************************************************************/ 57234370Sjasone#ifdef JEMALLOC_H_STRUCTS 58234370Sjasone 59234370Sjasonestruct prof_bt_s { 60234370Sjasone /* Backtrace, stored as len program counters. */ 61234370Sjasone void **vec; 62234370Sjasone unsigned len; 63234370Sjasone}; 64234370Sjasone 65234370Sjasone#ifdef JEMALLOC_PROF_LIBGCC 66234370Sjasone/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ 67234370Sjasonetypedef struct { 68234370Sjasone prof_bt_t *bt; 69234370Sjasone unsigned max; 70234370Sjasone} prof_unwind_data_t; 71234370Sjasone#endif 72234370Sjasone 73234370Sjasonestruct prof_cnt_s { 74286866Sjasone /* Profiling counters. */ 75286866Sjasone uint64_t curobjs; 76286866Sjasone uint64_t curbytes; 77234370Sjasone uint64_t accumobjs; 78234370Sjasone uint64_t accumbytes; 79234370Sjasone}; 80234370Sjasone 81286866Sjasonetypedef enum { 82286866Sjasone prof_tctx_state_initializing, 83286866Sjasone prof_tctx_state_nominal, 84286866Sjasone prof_tctx_state_dumping, 85286866Sjasone prof_tctx_state_purgatory /* Dumper must finish destroying. */ 86286866Sjasone} prof_tctx_state_t; 87234370Sjasone 88286866Sjasonestruct prof_tctx_s { 89286866Sjasone /* Thread data for thread that performed the allocation. */ 90286866Sjasone prof_tdata_t *tdata; 91234370Sjasone 92234370Sjasone /* 93288090Sjasone * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be 94288090Sjasone * defunct during teardown. 95234370Sjasone */ 96286866Sjasone uint64_t thr_uid; 97288090Sjasone uint64_t thr_discrim; 98234370Sjasone 99286866Sjasone /* Profiling counters, protected by tdata->lock. */ 100286866Sjasone prof_cnt_t cnts; 101286866Sjasone 102286866Sjasone /* Associated global context. */ 103286866Sjasone prof_gctx_t *gctx; 104286866Sjasone 105234370Sjasone /* 106286866Sjasone * UID that distinguishes multiple tctx's created by the same thread, 107286866Sjasone * but coexisting in gctx->tctxs. There are two ways that such 108286866Sjasone * coexistence can occur: 109286866Sjasone * - A dumper thread can cause a tctx to be retained in the purgatory 110286866Sjasone * state. 111286866Sjasone * - Although a single "producer" thread must create all tctx's which 112286866Sjasone * share the same thr_uid, multiple "consumers" can each concurrently 113286866Sjasone * execute portions of prof_tctx_destroy(). prof_tctx_destroy() only 114286866Sjasone * gets called once each time cnts.cur{objs,bytes} drop to 0, but this 115286866Sjasone * threshold can be hit again before the first consumer finishes 116286866Sjasone * executing prof_tctx_destroy(). 117234370Sjasone */ 118286866Sjasone uint64_t tctx_uid; 119234370Sjasone 120286866Sjasone /* Linkage into gctx's tctxs. */ 121286866Sjasone rb_node(prof_tctx_t) tctx_link; 122286866Sjasone 123286866Sjasone /* 124286866Sjasone * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents 125286866Sjasone * sample vs destroy race. 126286866Sjasone */ 127286866Sjasone bool prepared; 128286866Sjasone 129286866Sjasone /* Current dump-related state, protected by gctx->lock. */ 130286866Sjasone prof_tctx_state_t state; 131286866Sjasone 132286866Sjasone /* 133286866Sjasone * Copy of cnts snapshotted during early dump phase, protected by 134286866Sjasone * dump_mtx. 135286866Sjasone */ 136286866Sjasone prof_cnt_t dump_cnts; 137234370Sjasone}; 138286866Sjasonetypedef rb_tree(prof_tctx_t) prof_tctx_tree_t; 139234370Sjasone 140286866Sjasonestruct prof_gctx_s { 141286866Sjasone /* Protects nlimbo, cnt_summed, and tctxs. */ 142234370Sjasone malloc_mutex_t *lock; 143234370Sjasone 144235238Sjasone /* 145286866Sjasone * Number of threads that currently cause this gctx to be in a state of 146235238Sjasone * limbo due to one of: 147286866Sjasone * - Initializing this gctx. 148286866Sjasone * - Initializing per thread counters associated with this gctx. 149286866Sjasone * - Preparing to destroy this gctx. 150286866Sjasone * - Dumping a heap profile that includes this gctx. 151235238Sjasone * nlimbo must be 1 (single destroyer) in order to safely destroy the 152286866Sjasone * gctx. 153235238Sjasone */ 154235238Sjasone unsigned nlimbo; 155235238Sjasone 156286866Sjasone /* 157286866Sjasone * Tree of profile counters, one for each thread that has allocated in 158286866Sjasone * this context. 159286866Sjasone */ 160286866Sjasone prof_tctx_tree_t tctxs; 161286866Sjasone 162286866Sjasone /* Linkage for tree of contexts to be dumped. */ 163286866Sjasone rb_node(prof_gctx_t) dump_link; 164286866Sjasone 165234370Sjasone /* Temporary storage for summation during dump. */ 166234370Sjasone prof_cnt_t cnt_summed; 167234370Sjasone 168286866Sjasone /* Associated backtrace. */ 169286866Sjasone prof_bt_t bt; 170234370Sjasone 171286866Sjasone /* Backtrace vector, variable size, referred to by bt. */ 172286866Sjasone void *vec[1]; 173286866Sjasone}; 174286866Sjasonetypedef rb_tree(prof_gctx_t) prof_gctx_tree_t; 175286866Sjasone 176286866Sjasonestruct prof_tdata_s { 177286866Sjasone malloc_mutex_t *lock; 178286866Sjasone 179286866Sjasone /* Monotonically increasing unique thread identifier. */ 180286866Sjasone uint64_t thr_uid; 181286866Sjasone 182234370Sjasone /* 183286866Sjasone * Monotonically increasing discriminator among tdata structures 184286866Sjasone * associated with the same thr_uid. 185234370Sjasone */ 186286866Sjasone uint64_t thr_discrim; 187261071Sjasone 188286866Sjasone /* Included in heap profile dumps if non-NULL. */ 189286866Sjasone char *thread_name; 190234370Sjasone 191286866Sjasone bool attached; 192286866Sjasone bool expired; 193286866Sjasone 194286866Sjasone rb_node(prof_tdata_t) tdata_link; 195286866Sjasone 196234370Sjasone /* 197286866Sjasone * Counter used to initialize prof_tctx_t's tctx_uid. No locking is 198286866Sjasone * necessary when incrementing this field, because only one thread ever 199286866Sjasone * does so. 200234370Sjasone */ 201286866Sjasone uint64_t tctx_uid_next; 202234370Sjasone 203286866Sjasone /* 204286866Sjasone * Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks 205286866Sjasone * backtraces for which it has non-zero allocation/deallocation counters 206286866Sjasone * associated with thread-specific prof_tctx_t objects. Other threads 207286866Sjasone * may write to prof_tctx_t contents when freeing associated objects. 208286866Sjasone */ 209286866Sjasone ckh_t bt2tctx; 210234370Sjasone 211234370Sjasone /* Sampling state. */ 212234370Sjasone uint64_t prng_state; 213286866Sjasone uint64_t bytes_until_sample; 214235238Sjasone 215235238Sjasone /* State used to avoid dumping while operating on prof internals. */ 216235238Sjasone bool enq; 217235238Sjasone bool enq_idump; 218235238Sjasone bool enq_gdump; 219286866Sjasone 220286866Sjasone /* 221286866Sjasone * Set to true during an early dump phase for tdata's which are 222286866Sjasone * currently being dumped. New threads' tdata's have this initialized 223286866Sjasone * to false so that they aren't accidentally included in later dump 224286866Sjasone * phases. 225286866Sjasone */ 226286866Sjasone bool dumping; 227286866Sjasone 228286866Sjasone /* 229286866Sjasone * True if profiling is active for this tdata's thread 230286866Sjasone * (thread.prof.active mallctl). 231286866Sjasone */ 232286866Sjasone bool active; 233286866Sjasone 234286866Sjasone /* Temporary storage for summation during dump. */ 235286866Sjasone prof_cnt_t cnt_summed; 236286866Sjasone 237286866Sjasone /* Backtrace vector, used for calls to prof_backtrace(). */ 238286866Sjasone void *vec[PROF_BT_MAX]; 239234370Sjasone}; 240286866Sjasonetypedef rb_tree(prof_tdata_t) prof_tdata_tree_t; 241234370Sjasone 242234370Sjasone#endif /* JEMALLOC_H_STRUCTS */ 243234370Sjasone/******************************************************************************/ 244234370Sjasone#ifdef JEMALLOC_H_EXTERNS 245234370Sjasone 246234370Sjasoneextern bool opt_prof; 247234370Sjasoneextern bool opt_prof_active; 248286866Sjasoneextern bool opt_prof_thread_active_init; 249234370Sjasoneextern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ 250234370Sjasoneextern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ 251234370Sjasoneextern bool opt_prof_gdump; /* High-water memory dumping. */ 252234543Sjasoneextern bool opt_prof_final; /* Final profile dumping. */ 253234370Sjasoneextern bool opt_prof_leak; /* Dump leak summary at exit. */ 254234370Sjasoneextern bool opt_prof_accum; /* Report cumulative bytes. */ 255261071Sjasoneextern char opt_prof_prefix[ 256261071Sjasone /* Minimize memory bloat for non-prof builds. */ 257261071Sjasone#ifdef JEMALLOC_PROF 258261071Sjasone PATH_MAX + 259261071Sjasone#endif 260261071Sjasone 1]; 261234370Sjasone 262286866Sjasone/* Accessed via prof_active_[gs]et{_unlocked,}(). */ 263286866Sjasoneextern bool prof_active; 264286866Sjasone 265286866Sjasone/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */ 266286866Sjasoneextern bool prof_gdump_val; 267286866Sjasone 268234370Sjasone/* 269234370Sjasone * Profile dump interval, measured in bytes allocated. Each arena triggers a 270234370Sjasone * profile dump when it reaches this threshold. The effect is that the 271234370Sjasone * interval between profile dumps averages prof_interval, though the actual 272234370Sjasone * interval between dumps will tend to be sporadic, and the interval will be a 273234370Sjasone * maximum of approximately (prof_interval * narenas). 274234370Sjasone */ 275234370Sjasoneextern uint64_t prof_interval; 276234370Sjasone 277234370Sjasone/* 278286866Sjasone * Initialized as opt_lg_prof_sample, and potentially modified during profiling 279286866Sjasone * resets. 280234370Sjasone */ 281286866Sjasoneextern size_t lg_prof_sample; 282234370Sjasone 283286866Sjasonevoid prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); 284299587Sjasonevoid prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, 285286866Sjasone prof_tctx_t *tctx); 286286866Sjasonevoid prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); 287234370Sjasonevoid bt_init(prof_bt_t *bt, void **vec); 288286866Sjasonevoid prof_backtrace(prof_bt_t *bt); 289286866Sjasoneprof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); 290261071Sjasone#ifdef JEMALLOC_JET 291286866Sjasonesize_t prof_tdata_count(void); 292261071Sjasonesize_t prof_bt_count(void); 293286866Sjasoneconst prof_cnt_t *prof_cnt_all(void); 294261071Sjasonetypedef int (prof_dump_open_t)(bool, const char *); 295261071Sjasoneextern prof_dump_open_t *prof_dump_open; 296299587Sjasonetypedef bool (prof_dump_header_t)(tsdn_t *, bool, const prof_cnt_t *); 297286866Sjasoneextern prof_dump_header_t *prof_dump_header; 298261071Sjasone#endif 299299587Sjasonevoid prof_idump(tsdn_t *tsdn); 300299587Sjasonebool prof_mdump(tsd_t *tsd, const char *filename); 301299587Sjasonevoid prof_gdump(tsdn_t *tsdn); 302299587Sjasoneprof_tdata_t *prof_tdata_init(tsdn_t *tsdn); 303286866Sjasoneprof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); 304299587Sjasonevoid prof_reset(tsdn_t *tsdn, size_t lg_sample); 305286866Sjasonevoid prof_tdata_cleanup(tsd_t *tsd); 306299587Sjasonebool prof_active_get(tsdn_t *tsdn); 307299587Sjasonebool prof_active_set(tsdn_t *tsdn, bool active); 308299587Sjasoneconst char *prof_thread_name_get(tsd_t *tsd); 309286866Sjasoneint prof_thread_name_set(tsd_t *tsd, const char *thread_name); 310299587Sjasonebool prof_thread_active_get(tsd_t *tsd); 311299587Sjasonebool prof_thread_active_set(tsd_t *tsd, bool active); 312299587Sjasonebool prof_thread_active_init_get(tsdn_t *tsdn); 313299587Sjasonebool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init); 314299587Sjasonebool prof_gdump_get(tsdn_t *tsdn); 315299587Sjasonebool prof_gdump_set(tsdn_t *tsdn, bool active); 316234370Sjasonevoid prof_boot0(void); 317234370Sjasonevoid prof_boot1(void); 318299587Sjasonebool prof_boot2(tsdn_t *tsdn); 319299587Sjasonevoid prof_prefork0(tsdn_t *tsdn); 320299587Sjasonevoid prof_prefork1(tsdn_t *tsdn); 321299587Sjasonevoid prof_postfork_parent(tsdn_t *tsdn); 322299587Sjasonevoid prof_postfork_child(tsdn_t *tsdn); 323286866Sjasonevoid prof_sample_threshold_update(prof_tdata_t *tdata); 324234370Sjasone 325234370Sjasone#endif /* JEMALLOC_H_EXTERNS */ 326234370Sjasone/******************************************************************************/ 327234370Sjasone#ifdef JEMALLOC_H_INLINES 328234370Sjasone 329234370Sjasone#ifndef JEMALLOC_ENABLE_INLINE 330286866Sjasonebool prof_active_get_unlocked(void); 331286866Sjasonebool prof_gdump_get_unlocked(void); 332286866Sjasoneprof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); 333299587Sjasoneprof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr); 334299587Sjasonevoid prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, 335299587Sjasone prof_tctx_t *tctx); 336299587Sjasonevoid prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, 337299587Sjasone const void *old_ptr, prof_tctx_t *tctx); 338286866Sjasonebool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, 339286866Sjasone prof_tdata_t **tdata_out); 340288090Sjasoneprof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, 341288090Sjasone bool update); 342299587Sjasonevoid prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, 343288090Sjasone prof_tctx_t *tctx); 344286866Sjasonevoid prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, 345288090Sjasone prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr, 346288090Sjasone size_t old_usize, prof_tctx_t *old_tctx); 347286866Sjasonevoid prof_free(tsd_t *tsd, const void *ptr, size_t usize); 348234370Sjasone#endif 349234370Sjasone 350234370Sjasone#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) 351286866SjasoneJEMALLOC_ALWAYS_INLINE bool 352286866Sjasoneprof_active_get_unlocked(void) 353235238Sjasone{ 354235238Sjasone 355286866Sjasone /* 356286866Sjasone * Even if opt_prof is true, sampling can be temporarily disabled by 357286866Sjasone * setting prof_active to false. No locking is used when reading 358286866Sjasone * prof_active in the fast path, so there are no guarantees regarding 359286866Sjasone * how long it will take for all threads to notice state changes. 360286866Sjasone */ 361286866Sjasone return (prof_active); 362235238Sjasone} 363235238Sjasone 364286866SjasoneJEMALLOC_ALWAYS_INLINE bool 365286866Sjasoneprof_gdump_get_unlocked(void) 366234370Sjasone{ 367286866Sjasone 368261071Sjasone /* 369286866Sjasone * No locking is used when reading prof_gdump_val in the fast path, so 370286866Sjasone * there are no guarantees regarding how long it will take for all 371286866Sjasone * threads to notice state changes. 372261071Sjasone */ 373286866Sjasone return (prof_gdump_val); 374286866Sjasone} 375234370Sjasone 376286866SjasoneJEMALLOC_ALWAYS_INLINE prof_tdata_t * 377286866Sjasoneprof_tdata_get(tsd_t *tsd, bool create) 378286866Sjasone{ 379286866Sjasone prof_tdata_t *tdata; 380286866Sjasone 381234370Sjasone cassert(config_prof); 382234370Sjasone 383286866Sjasone tdata = tsd_prof_tdata_get(tsd); 384286866Sjasone if (create) { 385286866Sjasone if (unlikely(tdata == NULL)) { 386286866Sjasone if (tsd_nominal(tsd)) { 387299587Sjasone tdata = prof_tdata_init(tsd_tsdn(tsd)); 388286866Sjasone tsd_prof_tdata_set(tsd, tdata); 389286866Sjasone } 390286866Sjasone } else if (unlikely(tdata->expired)) { 391286866Sjasone tdata = prof_tdata_reinit(tsd, tdata); 392286866Sjasone tsd_prof_tdata_set(tsd, tdata); 393286866Sjasone } 394286866Sjasone assert(tdata == NULL || tdata->attached); 395286866Sjasone } 396286866Sjasone 397286866Sjasone return (tdata); 398234370Sjasone} 399234370Sjasone 400286866SjasoneJEMALLOC_ALWAYS_INLINE prof_tctx_t * 401299587Sjasoneprof_tctx_get(tsdn_t *tsdn, const void *ptr) 402234370Sjasone{ 403234370Sjasone 404234370Sjasone cassert(config_prof); 405234370Sjasone assert(ptr != NULL); 406234370Sjasone 407299587Sjasone return (arena_prof_tctx_get(tsdn, ptr)); 408234370Sjasone} 409234370Sjasone 410286866SjasoneJEMALLOC_ALWAYS_INLINE void 411299587Sjasoneprof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) 412234370Sjasone{ 413234370Sjasone 414234370Sjasone cassert(config_prof); 415234370Sjasone assert(ptr != NULL); 416234370Sjasone 417299587Sjasone arena_prof_tctx_set(tsdn, ptr, usize, tctx); 418234370Sjasone} 419234370Sjasone 420288090SjasoneJEMALLOC_ALWAYS_INLINE void 421299587Sjasoneprof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, const void *old_ptr, 422288090Sjasone prof_tctx_t *old_tctx) 423288090Sjasone{ 424288090Sjasone 425288090Sjasone cassert(config_prof); 426288090Sjasone assert(ptr != NULL); 427288090Sjasone 428299587Sjasone arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx); 429288090Sjasone} 430288090Sjasone 431286866SjasoneJEMALLOC_ALWAYS_INLINE bool 432286866Sjasoneprof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, 433286866Sjasone prof_tdata_t **tdata_out) 434234370Sjasone{ 435286866Sjasone prof_tdata_t *tdata; 436234370Sjasone 437234370Sjasone cassert(config_prof); 438234370Sjasone 439286866Sjasone tdata = prof_tdata_get(tsd, true); 440296221Sjasone if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) 441286866Sjasone tdata = NULL; 442286866Sjasone 443286866Sjasone if (tdata_out != NULL) 444286866Sjasone *tdata_out = tdata; 445286866Sjasone 446296221Sjasone if (unlikely(tdata == NULL)) 447235238Sjasone return (true); 448234370Sjasone 449296221Sjasone if (likely(tdata->bytes_until_sample >= usize)) { 450286866Sjasone if (update) 451286866Sjasone tdata->bytes_until_sample -= usize; 452286866Sjasone return (true); 453286866Sjasone } else { 454234370Sjasone /* Compute new sample threshold. */ 455286866Sjasone if (update) 456286866Sjasone prof_sample_threshold_update(tdata); 457286866Sjasone return (!tdata->active); 458234370Sjasone } 459234370Sjasone} 460234370Sjasone 461286866SjasoneJEMALLOC_ALWAYS_INLINE prof_tctx_t * 462288090Sjasoneprof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update) 463234370Sjasone{ 464286866Sjasone prof_tctx_t *ret; 465286866Sjasone prof_tdata_t *tdata; 466286866Sjasone prof_bt_t bt; 467234370Sjasone 468286866Sjasone assert(usize == s2u(usize)); 469286866Sjasone 470288090Sjasone if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update, 471288090Sjasone &tdata))) 472286866Sjasone ret = (prof_tctx_t *)(uintptr_t)1U; 473286866Sjasone else { 474286866Sjasone bt_init(&bt, tdata->vec); 475286866Sjasone prof_backtrace(&bt); 476286866Sjasone ret = prof_lookup(tsd, &bt); 477286866Sjasone } 478286866Sjasone 479286866Sjasone return (ret); 480286866Sjasone} 481286866Sjasone 482286866SjasoneJEMALLOC_ALWAYS_INLINE void 483299587Sjasoneprof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) 484286866Sjasone{ 485286866Sjasone 486234370Sjasone cassert(config_prof); 487234370Sjasone assert(ptr != NULL); 488299587Sjasone assert(usize == isalloc(tsdn, ptr, true)); 489234370Sjasone 490286866Sjasone if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 491299587Sjasone prof_malloc_sample_object(tsdn, ptr, usize, tctx); 492286866Sjasone else 493299587Sjasone prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U); 494234370Sjasone} 495234370Sjasone 496286866SjasoneJEMALLOC_ALWAYS_INLINE void 497286866Sjasoneprof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, 498288090Sjasone bool prof_active, bool updated, const void *old_ptr, size_t old_usize, 499288090Sjasone prof_tctx_t *old_tctx) 500234370Sjasone{ 501288090Sjasone bool sampled, old_sampled; 502234370Sjasone 503234370Sjasone cassert(config_prof); 504286866Sjasone assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); 505234370Sjasone 506288090Sjasone if (prof_active && !updated && ptr != NULL) { 507299587Sjasone assert(usize == isalloc(tsd_tsdn(tsd), ptr, true)); 508286866Sjasone if (prof_sample_accum_update(tsd, usize, true, NULL)) { 509234370Sjasone /* 510288090Sjasone * Don't sample. The usize passed to prof_alloc_prep() 511286866Sjasone * was larger than what actually got allocated, so a 512286866Sjasone * backtrace was captured for this allocation, even 513286866Sjasone * though its actual usize was insufficient to cross the 514286866Sjasone * sample threshold. 515234370Sjasone */ 516301718Sjasone prof_alloc_rollback(tsd, tctx, true); 517286866Sjasone tctx = (prof_tctx_t *)(uintptr_t)1U; 518234370Sjasone } 519286866Sjasone } 520234370Sjasone 521288090Sjasone sampled = ((uintptr_t)tctx > (uintptr_t)1U); 522288090Sjasone old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U); 523288090Sjasone 524288090Sjasone if (unlikely(sampled)) 525299587Sjasone prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx); 526286866Sjasone else 527299587Sjasone prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx); 528288090Sjasone 529288090Sjasone if (unlikely(old_sampled)) 530288090Sjasone prof_free_sampled_object(tsd, old_usize, old_tctx); 531234370Sjasone} 532234370Sjasone 533286866SjasoneJEMALLOC_ALWAYS_INLINE void 534286866Sjasoneprof_free(tsd_t *tsd, const void *ptr, size_t usize) 535234370Sjasone{ 536299587Sjasone prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); 537234370Sjasone 538234370Sjasone cassert(config_prof); 539299587Sjasone assert(usize == isalloc(tsd_tsdn(tsd), ptr, true)); 540234370Sjasone 541286866Sjasone if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) 542286866Sjasone prof_free_sampled_object(tsd, usize, tctx); 543234370Sjasone} 544234370Sjasone#endif 545234370Sjasone 546234370Sjasone#endif /* JEMALLOC_H_INLINES */ 547234370Sjasone/******************************************************************************/ 548