ctl.c revision 1.2
1#define JEMALLOC_CTL_C_ 2#include "jemalloc/internal/jemalloc_preamble.h" 3#include "jemalloc/internal/jemalloc_internal_includes.h" 4 5#include "jemalloc/internal/assert.h" 6#include "jemalloc/internal/ctl.h" 7#include "jemalloc/internal/extent_dss.h" 8#include "jemalloc/internal/extent_mmap.h" 9#include "jemalloc/internal/mutex.h" 10#include "jemalloc/internal/nstime.h" 11#include "jemalloc/internal/size_classes.h" 12#include "jemalloc/internal/util.h" 13 14/******************************************************************************/ 15/* Data. */ 16 17/* 18 * ctl_mtx protects the following: 19 * - ctl_stats->* 20 */ 21static malloc_mutex_t ctl_mtx; 22static bool ctl_initialized; 23static ctl_stats_t *ctl_stats; 24static ctl_arenas_t *ctl_arenas; 25 26/******************************************************************************/ 27/* Helpers for named and indexed nodes. */ 28 29static const ctl_named_node_t * 30ctl_named_node(const ctl_node_t *node) { 31 return ((node->named) ? (const ctl_named_node_t *)node : NULL); 32} 33 34static const ctl_named_node_t * 35ctl_named_children(const ctl_named_node_t *node, size_t index) { 36 const ctl_named_node_t *children = ctl_named_node(node->children); 37 38 return (children ? &children[index] : NULL); 39} 40 41static const ctl_indexed_node_t * 42ctl_indexed_node(const ctl_node_t *node) { 43 return (!node->named ? (const ctl_indexed_node_t *)node : NULL); 44} 45 46/******************************************************************************/ 47/* Function prototypes for non-inline static functions. */ 48 49#define CTL_PROTO(n) \ 50static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 51 void *oldp, size_t *oldlenp, void *newp, size_t newlen); 52 53#define INDEX_PROTO(n) \ 54static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \ 55 const size_t *mib, size_t miblen, size_t i); 56 57CTL_PROTO(version) 58CTL_PROTO(epoch) 59CTL_PROTO(background_thread) 60CTL_PROTO(max_background_threads) 61CTL_PROTO(thread_tcache_enabled) 62CTL_PROTO(thread_tcache_flush) 63CTL_PROTO(thread_prof_name) 64CTL_PROTO(thread_prof_active) 65CTL_PROTO(thread_arena) 66CTL_PROTO(thread_allocated) 67CTL_PROTO(thread_allocatedp) 68CTL_PROTO(thread_deallocated) 69CTL_PROTO(thread_deallocatedp) 70CTL_PROTO(config_cache_oblivious) 71CTL_PROTO(config_debug) 72CTL_PROTO(config_fill) 73CTL_PROTO(config_lazy_lock) 74CTL_PROTO(config_malloc_conf) 75CTL_PROTO(config_prof) 76CTL_PROTO(config_prof_libgcc) 77CTL_PROTO(config_prof_libunwind) 78CTL_PROTO(config_stats) 79CTL_PROTO(config_utrace) 80CTL_PROTO(config_xmalloc) 81CTL_PROTO(opt_abort) 82CTL_PROTO(opt_abort_conf) 83CTL_PROTO(opt_metadata_thp) 84CTL_PROTO(opt_retain) 85CTL_PROTO(opt_dss) 86CTL_PROTO(opt_narenas) 87CTL_PROTO(opt_percpu_arena) 88CTL_PROTO(opt_background_thread) 89CTL_PROTO(opt_max_background_threads) 90CTL_PROTO(opt_dirty_decay_ms) 91CTL_PROTO(opt_muzzy_decay_ms) 92CTL_PROTO(opt_stats_print) 93CTL_PROTO(opt_stats_print_opts) 94CTL_PROTO(opt_junk) 95CTL_PROTO(opt_zero) 96CTL_PROTO(opt_utrace) 97CTL_PROTO(opt_xmalloc) 98CTL_PROTO(opt_tcache) 99CTL_PROTO(opt_thp) 100CTL_PROTO(opt_lg_extent_max_active_fit) 101CTL_PROTO(opt_lg_tcache_max) 102CTL_PROTO(opt_prof) 103CTL_PROTO(opt_prof_prefix) 104CTL_PROTO(opt_prof_active) 105CTL_PROTO(opt_prof_thread_active_init) 106CTL_PROTO(opt_lg_prof_sample) 107CTL_PROTO(opt_lg_prof_interval) 108CTL_PROTO(opt_prof_gdump) 109CTL_PROTO(opt_prof_final) 110CTL_PROTO(opt_prof_leak) 111CTL_PROTO(opt_prof_accum) 112CTL_PROTO(tcache_create) 113CTL_PROTO(tcache_flush) 114CTL_PROTO(tcache_destroy) 115CTL_PROTO(arena_i_initialized) 116CTL_PROTO(arena_i_decay) 117CTL_PROTO(arena_i_purge) 118CTL_PROTO(arena_i_reset) 119CTL_PROTO(arena_i_destroy) 120CTL_PROTO(arena_i_dss) 121CTL_PROTO(arena_i_dirty_decay_ms) 122CTL_PROTO(arena_i_muzzy_decay_ms) 123CTL_PROTO(arena_i_extent_hooks) 124CTL_PROTO(arena_i_retain_grow_limit) 125INDEX_PROTO(arena_i) 126CTL_PROTO(arenas_bin_i_size) 127CTL_PROTO(arenas_bin_i_nregs) 128CTL_PROTO(arenas_bin_i_slab_size) 129INDEX_PROTO(arenas_bin_i) 130CTL_PROTO(arenas_lextent_i_size) 131INDEX_PROTO(arenas_lextent_i) 132CTL_PROTO(arenas_narenas) 133CTL_PROTO(arenas_dirty_decay_ms) 134CTL_PROTO(arenas_muzzy_decay_ms) 135CTL_PROTO(arenas_quantum) 136CTL_PROTO(arenas_page) 137CTL_PROTO(arenas_tcache_max) 138CTL_PROTO(arenas_nbins) 139CTL_PROTO(arenas_nhbins) 140CTL_PROTO(arenas_nlextents) 141CTL_PROTO(arenas_create) 142CTL_PROTO(arenas_lookup) 143CTL_PROTO(prof_thread_active_init) 144CTL_PROTO(prof_active) 145CTL_PROTO(prof_dump) 146CTL_PROTO(prof_gdump) 147CTL_PROTO(prof_reset) 148CTL_PROTO(prof_interval) 149CTL_PROTO(lg_prof_sample) 150CTL_PROTO(stats_arenas_i_small_allocated) 151CTL_PROTO(stats_arenas_i_small_nmalloc) 152CTL_PROTO(stats_arenas_i_small_ndalloc) 153CTL_PROTO(stats_arenas_i_small_nrequests) 154CTL_PROTO(stats_arenas_i_large_allocated) 155CTL_PROTO(stats_arenas_i_large_nmalloc) 156CTL_PROTO(stats_arenas_i_large_ndalloc) 157CTL_PROTO(stats_arenas_i_large_nrequests) 158CTL_PROTO(stats_arenas_i_bins_j_nmalloc) 159CTL_PROTO(stats_arenas_i_bins_j_ndalloc) 160CTL_PROTO(stats_arenas_i_bins_j_nrequests) 161CTL_PROTO(stats_arenas_i_bins_j_curregs) 162CTL_PROTO(stats_arenas_i_bins_j_nfills) 163CTL_PROTO(stats_arenas_i_bins_j_nflushes) 164CTL_PROTO(stats_arenas_i_bins_j_nslabs) 165CTL_PROTO(stats_arenas_i_bins_j_nreslabs) 166CTL_PROTO(stats_arenas_i_bins_j_curslabs) 167INDEX_PROTO(stats_arenas_i_bins_j) 168CTL_PROTO(stats_arenas_i_lextents_j_nmalloc) 169CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) 170CTL_PROTO(stats_arenas_i_lextents_j_nrequests) 171CTL_PROTO(stats_arenas_i_lextents_j_curlextents) 172INDEX_PROTO(stats_arenas_i_lextents_j) 173CTL_PROTO(stats_arenas_i_nthreads) 174CTL_PROTO(stats_arenas_i_uptime) 175CTL_PROTO(stats_arenas_i_dss) 176CTL_PROTO(stats_arenas_i_dirty_decay_ms) 177CTL_PROTO(stats_arenas_i_muzzy_decay_ms) 178CTL_PROTO(stats_arenas_i_pactive) 179CTL_PROTO(stats_arenas_i_pdirty) 180CTL_PROTO(stats_arenas_i_pmuzzy) 181CTL_PROTO(stats_arenas_i_mapped) 182CTL_PROTO(stats_arenas_i_retained) 183CTL_PROTO(stats_arenas_i_dirty_npurge) 184CTL_PROTO(stats_arenas_i_dirty_nmadvise) 185CTL_PROTO(stats_arenas_i_dirty_purged) 186CTL_PROTO(stats_arenas_i_muzzy_npurge) 187CTL_PROTO(stats_arenas_i_muzzy_nmadvise) 188CTL_PROTO(stats_arenas_i_muzzy_purged) 189CTL_PROTO(stats_arenas_i_base) 190CTL_PROTO(stats_arenas_i_internal) 191CTL_PROTO(stats_arenas_i_metadata_thp) 192CTL_PROTO(stats_arenas_i_tcache_bytes) 193CTL_PROTO(stats_arenas_i_resident) 194INDEX_PROTO(stats_arenas_i) 195CTL_PROTO(stats_allocated) 196CTL_PROTO(stats_active) 197CTL_PROTO(stats_background_thread_num_threads) 198CTL_PROTO(stats_background_thread_num_runs) 199CTL_PROTO(stats_background_thread_run_interval) 200CTL_PROTO(stats_metadata) 201CTL_PROTO(stats_metadata_thp) 202CTL_PROTO(stats_resident) 203CTL_PROTO(stats_mapped) 204CTL_PROTO(stats_retained) 205 206#define MUTEX_STATS_CTL_PROTO_GEN(n) \ 207CTL_PROTO(stats_##n##_num_ops) \ 208CTL_PROTO(stats_##n##_num_wait) \ 209CTL_PROTO(stats_##n##_num_spin_acq) \ 210CTL_PROTO(stats_##n##_num_owner_switch) \ 211CTL_PROTO(stats_##n##_total_wait_time) \ 212CTL_PROTO(stats_##n##_max_wait_time) \ 213CTL_PROTO(stats_##n##_max_num_thds) 214 215/* Global mutexes. */ 216#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx) 217MUTEX_PROF_GLOBAL_MUTEXES 218#undef OP 219 220/* Per arena mutexes. */ 221#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx) 222MUTEX_PROF_ARENA_MUTEXES 223#undef OP 224 225/* Arena bin mutexes. */ 226MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) 227#undef MUTEX_STATS_CTL_PROTO_GEN 228 229CTL_PROTO(stats_mutexes_reset) 230 231/******************************************************************************/ 232/* mallctl tree. */ 233 234#define NAME(n) {true}, n 235#define CHILD(t, c) \ 236 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ 237 (const ctl_node_t *)c##_node, \ 238 NULL 239#define CTL(c) 0, NULL, c##_ctl 240 241/* 242 * Only handles internal indexed nodes, since there are currently no external 243 * ones. 244 */ 245#define INDEX(i) {false}, i##_index 246 247static const ctl_named_node_t thread_tcache_node[] = { 248 {NAME("enabled"), CTL(thread_tcache_enabled)}, 249 {NAME("flush"), CTL(thread_tcache_flush)} 250}; 251 252static const ctl_named_node_t thread_prof_node[] = { 253 {NAME("name"), CTL(thread_prof_name)}, 254 {NAME("active"), CTL(thread_prof_active)} 255}; 256 257static const ctl_named_node_t thread_node[] = { 258 {NAME("arena"), CTL(thread_arena)}, 259 {NAME("allocated"), CTL(thread_allocated)}, 260 {NAME("allocatedp"), CTL(thread_allocatedp)}, 261 {NAME("deallocated"), CTL(thread_deallocated)}, 262 {NAME("deallocatedp"), CTL(thread_deallocatedp)}, 263 {NAME("tcache"), CHILD(named, thread_tcache)}, 264 {NAME("prof"), CHILD(named, thread_prof)} 265}; 266 267static const ctl_named_node_t config_node[] = { 268 {NAME("cache_oblivious"), CTL(config_cache_oblivious)}, 269 {NAME("debug"), CTL(config_debug)}, 270 {NAME("fill"), CTL(config_fill)}, 271 {NAME("lazy_lock"), CTL(config_lazy_lock)}, 272 {NAME("malloc_conf"), CTL(config_malloc_conf)}, 273 {NAME("prof"), CTL(config_prof)}, 274 {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, 275 {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, 276 {NAME("stats"), CTL(config_stats)}, 277 {NAME("utrace"), CTL(config_utrace)}, 278 {NAME("xmalloc"), CTL(config_xmalloc)} 279}; 280 281static const ctl_named_node_t opt_node[] = { 282 {NAME("abort"), CTL(opt_abort)}, 283 {NAME("abort_conf"), CTL(opt_abort_conf)}, 284 {NAME("metadata_thp"), CTL(opt_metadata_thp)}, 285 {NAME("retain"), CTL(opt_retain)}, 286 {NAME("dss"), CTL(opt_dss)}, 287 {NAME("narenas"), CTL(opt_narenas)}, 288 {NAME("percpu_arena"), CTL(opt_percpu_arena)}, 289 {NAME("background_thread"), CTL(opt_background_thread)}, 290 {NAME("max_background_threads"), CTL(opt_max_background_threads)}, 291 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, 292 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, 293 {NAME("stats_print"), CTL(opt_stats_print)}, 294 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)}, 295 {NAME("junk"), CTL(opt_junk)}, 296 {NAME("zero"), CTL(opt_zero)}, 297 {NAME("utrace"), CTL(opt_utrace)}, 298 {NAME("xmalloc"), CTL(opt_xmalloc)}, 299 {NAME("tcache"), CTL(opt_tcache)}, 300 {NAME("thp"), CTL(opt_thp)}, 301 {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, 302 {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, 303 {NAME("prof"), CTL(opt_prof)}, 304 {NAME("prof_prefix"), CTL(opt_prof_prefix)}, 305 {NAME("prof_active"), CTL(opt_prof_active)}, 306 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, 307 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, 308 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, 309 {NAME("prof_gdump"), CTL(opt_prof_gdump)}, 310 {NAME("prof_final"), CTL(opt_prof_final)}, 311 {NAME("prof_leak"), CTL(opt_prof_leak)}, 312 {NAME("prof_accum"), CTL(opt_prof_accum)} 313}; 314 315static const ctl_named_node_t tcache_node[] = { 316 {NAME("create"), CTL(tcache_create)}, 317 {NAME("flush"), CTL(tcache_flush)}, 318 {NAME("destroy"), CTL(tcache_destroy)} 319}; 320 321static const ctl_named_node_t arena_i_node[] = { 322 {NAME("initialized"), CTL(arena_i_initialized)}, 323 {NAME("decay"), CTL(arena_i_decay)}, 324 {NAME("purge"), CTL(arena_i_purge)}, 325 {NAME("reset"), CTL(arena_i_reset)}, 326 {NAME("destroy"), CTL(arena_i_destroy)}, 327 {NAME("dss"), CTL(arena_i_dss)}, 328 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, 329 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, 330 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, 331 {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)} 332}; 333static const ctl_named_node_t super_arena_i_node[] = { 334 {NAME(""), CHILD(named, arena_i)} 335}; 336 337static const ctl_indexed_node_t arena_node[] = { 338 {INDEX(arena_i)} 339}; 340 341static const ctl_named_node_t arenas_bin_i_node[] = { 342 {NAME("size"), CTL(arenas_bin_i_size)}, 343 {NAME("nregs"), CTL(arenas_bin_i_nregs)}, 344 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} 345}; 346static const ctl_named_node_t super_arenas_bin_i_node[] = { 347 {NAME(""), CHILD(named, arenas_bin_i)} 348}; 349 350static const ctl_indexed_node_t arenas_bin_node[] = { 351 {INDEX(arenas_bin_i)} 352}; 353 354static const ctl_named_node_t arenas_lextent_i_node[] = { 355 {NAME("size"), CTL(arenas_lextent_i_size)} 356}; 357static const ctl_named_node_t super_arenas_lextent_i_node[] = { 358 {NAME(""), CHILD(named, arenas_lextent_i)} 359}; 360 361static const ctl_indexed_node_t arenas_lextent_node[] = { 362 {INDEX(arenas_lextent_i)} 363}; 364 365static const ctl_named_node_t arenas_node[] = { 366 {NAME("narenas"), CTL(arenas_narenas)}, 367 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)}, 368 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)}, 369 {NAME("quantum"), CTL(arenas_quantum)}, 370 {NAME("page"), CTL(arenas_page)}, 371 {NAME("tcache_max"), CTL(arenas_tcache_max)}, 372 {NAME("nbins"), CTL(arenas_nbins)}, 373 {NAME("nhbins"), CTL(arenas_nhbins)}, 374 {NAME("bin"), CHILD(indexed, arenas_bin)}, 375 {NAME("nlextents"), CTL(arenas_nlextents)}, 376 {NAME("lextent"), CHILD(indexed, arenas_lextent)}, 377 {NAME("create"), CTL(arenas_create)}, 378 {NAME("lookup"), CTL(arenas_lookup)} 379}; 380 381static const ctl_named_node_t prof_node[] = { 382 {NAME("thread_active_init"), CTL(prof_thread_active_init)}, 383 {NAME("active"), CTL(prof_active)}, 384 {NAME("dump"), CTL(prof_dump)}, 385 {NAME("gdump"), CTL(prof_gdump)}, 386 {NAME("reset"), CTL(prof_reset)}, 387 {NAME("interval"), CTL(prof_interval)}, 388 {NAME("lg_sample"), CTL(lg_prof_sample)} 389}; 390 391static const ctl_named_node_t stats_arenas_i_small_node[] = { 392 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, 393 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, 394 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, 395 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} 396}; 397 398static const ctl_named_node_t stats_arenas_i_large_node[] = { 399 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, 400 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, 401 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, 402 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} 403}; 404 405#define MUTEX_PROF_DATA_NODE(prefix) \ 406static const ctl_named_node_t stats_##prefix##_node[] = { \ 407 {NAME("num_ops"), \ 408 CTL(stats_##prefix##_num_ops)}, \ 409 {NAME("num_wait"), \ 410 CTL(stats_##prefix##_num_wait)}, \ 411 {NAME("num_spin_acq"), \ 412 CTL(stats_##prefix##_num_spin_acq)}, \ 413 {NAME("num_owner_switch"), \ 414 CTL(stats_##prefix##_num_owner_switch)}, \ 415 {NAME("total_wait_time"), \ 416 CTL(stats_##prefix##_total_wait_time)}, \ 417 {NAME("max_wait_time"), \ 418 CTL(stats_##prefix##_max_wait_time)}, \ 419 {NAME("max_num_thds"), \ 420 CTL(stats_##prefix##_max_num_thds)} \ 421 /* Note that # of current waiting thread not provided. */ \ 422}; 423 424MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex) 425 426static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { 427 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, 428 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, 429 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, 430 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, 431 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, 432 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, 433 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, 434 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, 435 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, 436 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} 437}; 438 439static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { 440 {NAME(""), CHILD(named, stats_arenas_i_bins_j)} 441}; 442 443static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { 444 {INDEX(stats_arenas_i_bins_j)} 445}; 446 447static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = { 448 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)}, 449 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)}, 450 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)}, 451 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)} 452}; 453static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = { 454 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)} 455}; 456 457static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { 458 {INDEX(stats_arenas_i_lextents_j)} 459}; 460 461#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) 462MUTEX_PROF_ARENA_MUTEXES 463#undef OP 464 465static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { 466#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)}, 467MUTEX_PROF_ARENA_MUTEXES 468#undef OP 469}; 470 471static const ctl_named_node_t stats_arenas_i_node[] = { 472 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, 473 {NAME("uptime"), CTL(stats_arenas_i_uptime)}, 474 {NAME("dss"), CTL(stats_arenas_i_dss)}, 475 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)}, 476 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)}, 477 {NAME("pactive"), CTL(stats_arenas_i_pactive)}, 478 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, 479 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, 480 {NAME("mapped"), CTL(stats_arenas_i_mapped)}, 481 {NAME("retained"), CTL(stats_arenas_i_retained)}, 482 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, 483 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, 484 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, 485 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, 486 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, 487 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, 488 {NAME("base"), CTL(stats_arenas_i_base)}, 489 {NAME("internal"), CTL(stats_arenas_i_internal)}, 490 {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, 491 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, 492 {NAME("resident"), CTL(stats_arenas_i_resident)}, 493 {NAME("small"), CHILD(named, stats_arenas_i_small)}, 494 {NAME("large"), CHILD(named, stats_arenas_i_large)}, 495 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, 496 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, 497 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} 498}; 499static const ctl_named_node_t super_stats_arenas_i_node[] = { 500 {NAME(""), CHILD(named, stats_arenas_i)} 501}; 502 503static const ctl_indexed_node_t stats_arenas_node[] = { 504 {INDEX(stats_arenas_i)} 505}; 506 507static const ctl_named_node_t stats_background_thread_node[] = { 508 {NAME("num_threads"), CTL(stats_background_thread_num_threads)}, 509 {NAME("num_runs"), CTL(stats_background_thread_num_runs)}, 510 {NAME("run_interval"), CTL(stats_background_thread_run_interval)} 511}; 512 513#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx) 514MUTEX_PROF_GLOBAL_MUTEXES 515#undef OP 516 517static const ctl_named_node_t stats_mutexes_node[] = { 518#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)}, 519MUTEX_PROF_GLOBAL_MUTEXES 520#undef OP 521 {NAME("reset"), CTL(stats_mutexes_reset)} 522}; 523#undef MUTEX_PROF_DATA_NODE 524 525static const ctl_named_node_t stats_node[] = { 526 {NAME("allocated"), CTL(stats_allocated)}, 527 {NAME("active"), CTL(stats_active)}, 528 {NAME("metadata"), CTL(stats_metadata)}, 529 {NAME("metadata_thp"), CTL(stats_metadata_thp)}, 530 {NAME("resident"), CTL(stats_resident)}, 531 {NAME("mapped"), CTL(stats_mapped)}, 532 {NAME("retained"), CTL(stats_retained)}, 533 {NAME("background_thread"), 534 CHILD(named, stats_background_thread)}, 535 {NAME("mutexes"), CHILD(named, stats_mutexes)}, 536 {NAME("arenas"), CHILD(indexed, stats_arenas)} 537}; 538 539static const ctl_named_node_t root_node[] = { 540 {NAME("version"), CTL(version)}, 541 {NAME("epoch"), CTL(epoch)}, 542 {NAME("background_thread"), CTL(background_thread)}, 543 {NAME("max_background_threads"), CTL(max_background_threads)}, 544 {NAME("thread"), CHILD(named, thread)}, 545 {NAME("config"), CHILD(named, config)}, 546 {NAME("opt"), CHILD(named, opt)}, 547 {NAME("tcache"), CHILD(named, tcache)}, 548 {NAME("arena"), CHILD(indexed, arena)}, 549 {NAME("arenas"), CHILD(named, arenas)}, 550 {NAME("prof"), CHILD(named, prof)}, 551 {NAME("stats"), CHILD(named, stats)} 552}; 553static const ctl_named_node_t super_root_node[] = { 554 {NAME(""), CHILD(named, root)} 555}; 556 557#undef NAME 558#undef CHILD 559#undef CTL 560#undef INDEX 561 562/******************************************************************************/ 563 564/* 565 * Sets *dst + *src non-atomically. This is safe, since everything is 566 * synchronized by the ctl mutex. 567 */ 568static void 569ctl_accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) { 570#ifdef JEMALLOC_ATOMIC_U64 571 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); 572 uint64_t cur_src = atomic_load_u64(src, ATOMIC_RELAXED); 573 atomic_store_u64(dst, cur_dst + cur_src, ATOMIC_RELAXED); 574#else 575 *dst += *src; 576#endif 577} 578 579/* Likewise: with ctl mutex synchronization, reading is simple. */ 580static uint64_t 581ctl_arena_stats_read_u64(arena_stats_u64_t *p) { 582#ifdef JEMALLOC_ATOMIC_U64 583 return atomic_load_u64(p, ATOMIC_RELAXED); 584#else 585 return *p; 586#endif 587} 588 589static void 590accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) { 591 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); 592 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED); 593 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED); 594} 595 596/******************************************************************************/ 597 598static unsigned 599arenas_i2a_impl(size_t i, bool compat, bool validate) { 600 unsigned a; 601 602 switch (i) { 603 case MALLCTL_ARENAS_ALL: 604 a = 0; 605 break; 606 case MALLCTL_ARENAS_DESTROYED: 607 a = 1; 608 break; 609 default: 610 if (compat && i == ctl_arenas->narenas) { 611 /* 612 * Provide deprecated backward compatibility for 613 * accessing the merged stats at index narenas rather 614 * than via MALLCTL_ARENAS_ALL. This is scheduled for 615 * removal in 6.0.0. 616 */ 617 a = 0; 618 } else if (validate && i >= ctl_arenas->narenas) { 619 a = UINT_MAX; 620 } else { 621 /* 622 * This function should never be called for an index 623 * more than one past the range of indices that have 624 * initialized ctl data. 625 */ 626 assert(i < ctl_arenas->narenas || (!validate && i == 627 ctl_arenas->narenas)); 628 a = (unsigned)i + 2; 629 } 630 break; 631 } 632 633 return a; 634} 635 636static unsigned 637arenas_i2a(size_t i) { 638 return arenas_i2a_impl(i, true, false); 639} 640 641static ctl_arena_t * 642arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) { 643 ctl_arena_t *ret; 644 645 assert(!compat || !init); 646 647 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)]; 648 if (init && ret == NULL) { 649 if (config_stats) { 650 struct container_s { 651 ctl_arena_t ctl_arena; 652 ctl_arena_stats_t astats; 653 }; 654 struct container_s *cont = 655 (struct container_s *)base_alloc(tsd_tsdn(tsd), 656 b0get(), sizeof(struct container_s), QUANTUM); 657 if (cont == NULL) { 658 return NULL; 659 } 660 ret = &cont->ctl_arena; 661 ret->astats = &cont->astats; 662 } else { 663 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(), 664 sizeof(ctl_arena_t), QUANTUM); 665 if (ret == NULL) { 666 return NULL; 667 } 668 } 669 ret->arena_ind = (unsigned)i; 670 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret; 671 } 672 673 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i)); 674 return ret; 675} 676 677static ctl_arena_t * 678arenas_i(size_t i) { 679 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false); 680 assert(ret != NULL); 681 return ret; 682} 683 684static void 685ctl_arena_clear(ctl_arena_t *ctl_arena) { 686 ctl_arena->nthreads = 0; 687 ctl_arena->dss = dss_prec_names[dss_prec_limit]; 688 ctl_arena->dirty_decay_ms = -1; 689 ctl_arena->muzzy_decay_ms = -1; 690 ctl_arena->pactive = 0; 691 ctl_arena->pdirty = 0; 692 ctl_arena->pmuzzy = 0; 693 if (config_stats) { 694 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); 695 ctl_arena->astats->allocated_small = 0; 696 ctl_arena->astats->nmalloc_small = 0; 697 ctl_arena->astats->ndalloc_small = 0; 698 ctl_arena->astats->nrequests_small = 0; 699 memset(ctl_arena->astats->bstats, 0, NBINS * 700 sizeof(bin_stats_t)); 701 memset(ctl_arena->astats->lstats, 0, (NSIZES - NBINS) * 702 sizeof(arena_stats_large_t)); 703 } 704} 705 706static void 707ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { 708 unsigned i; 709 710 if (config_stats) { 711 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads, 712 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 713 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 714 &ctl_arena->pdirty, &ctl_arena->pmuzzy, 715 &ctl_arena->astats->astats, ctl_arena->astats->bstats, 716 ctl_arena->astats->lstats); 717 718 for (i = 0; i < NBINS; i++) { 719 ctl_arena->astats->allocated_small += 720 ctl_arena->astats->bstats[i].curregs * 721 sz_index2size(i); 722 ctl_arena->astats->nmalloc_small += 723 ctl_arena->astats->bstats[i].nmalloc; 724 ctl_arena->astats->ndalloc_small += 725 ctl_arena->astats->bstats[i].ndalloc; 726 ctl_arena->astats->nrequests_small += 727 ctl_arena->astats->bstats[i].nrequests; 728 } 729 } else { 730 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, 731 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 732 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 733 &ctl_arena->pdirty, &ctl_arena->pmuzzy); 734 } 735} 736 737static void 738ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, 739 bool destroyed) { 740 unsigned i; 741 742 if (!destroyed) { 743 ctl_sdarena->nthreads += ctl_arena->nthreads; 744 ctl_sdarena->pactive += ctl_arena->pactive; 745 ctl_sdarena->pdirty += ctl_arena->pdirty; 746 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy; 747 } else { 748 assert(ctl_arena->nthreads == 0); 749 assert(ctl_arena->pactive == 0); 750 assert(ctl_arena->pdirty == 0); 751 assert(ctl_arena->pmuzzy == 0); 752 } 753 754 if (config_stats) { 755 ctl_arena_stats_t *sdstats = ctl_sdarena->astats; 756 ctl_arena_stats_t *astats = ctl_arena->astats; 757 758 if (!destroyed) { 759 accum_atomic_zu(&sdstats->astats.mapped, 760 &astats->astats.mapped); 761 accum_atomic_zu(&sdstats->astats.retained, 762 &astats->astats.retained); 763 } 764 765 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, 766 &astats->astats.decay_dirty.npurge); 767 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise, 768 &astats->astats.decay_dirty.nmadvise); 769 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged, 770 &astats->astats.decay_dirty.purged); 771 772 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge, 773 &astats->astats.decay_muzzy.npurge); 774 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise, 775 &astats->astats.decay_muzzy.nmadvise); 776 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, 777 &astats->astats.decay_muzzy.purged); 778 779#define OP(mtx) malloc_mutex_prof_merge( \ 780 &(sdstats->astats.mutex_prof_data[ \ 781 arena_prof_mutex_##mtx]), \ 782 &(astats->astats.mutex_prof_data[ \ 783 arena_prof_mutex_##mtx])); 784MUTEX_PROF_ARENA_MUTEXES 785#undef OP 786 if (!destroyed) { 787 accum_atomic_zu(&sdstats->astats.base, 788 &astats->astats.base); 789 accum_atomic_zu(&sdstats->astats.internal, 790 &astats->astats.internal); 791 accum_atomic_zu(&sdstats->astats.resident, 792 &astats->astats.resident); 793 accum_atomic_zu(&sdstats->astats.metadata_thp, 794 &astats->astats.metadata_thp); 795 } else { 796 assert(atomic_load_zu( 797 &astats->astats.internal, ATOMIC_RELAXED) == 0); 798 } 799 800 if (!destroyed) { 801 sdstats->allocated_small += astats->allocated_small; 802 } else { 803 assert(astats->allocated_small == 0); 804 } 805 sdstats->nmalloc_small += astats->nmalloc_small; 806 sdstats->ndalloc_small += astats->ndalloc_small; 807 sdstats->nrequests_small += astats->nrequests_small; 808 809 if (!destroyed) { 810 accum_atomic_zu(&sdstats->astats.allocated_large, 811 &astats->astats.allocated_large); 812 } else { 813 assert(atomic_load_zu(&astats->astats.allocated_large, 814 ATOMIC_RELAXED) == 0); 815 } 816 ctl_accum_arena_stats_u64(&sdstats->astats.nmalloc_large, 817 &astats->astats.nmalloc_large); 818 ctl_accum_arena_stats_u64(&sdstats->astats.ndalloc_large, 819 &astats->astats.ndalloc_large); 820 ctl_accum_arena_stats_u64(&sdstats->astats.nrequests_large, 821 &astats->astats.nrequests_large); 822 823 accum_atomic_zu(&sdstats->astats.tcache_bytes, 824 &astats->astats.tcache_bytes); 825 826 if (ctl_arena->arena_ind == 0) { 827 sdstats->astats.uptime = astats->astats.uptime; 828 } 829 830 for (i = 0; i < NBINS; i++) { 831 sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; 832 sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; 833 sdstats->bstats[i].nrequests += 834 astats->bstats[i].nrequests; 835 if (!destroyed) { 836 sdstats->bstats[i].curregs += 837 astats->bstats[i].curregs; 838 } else { 839 assert(astats->bstats[i].curregs == 0); 840 } 841 sdstats->bstats[i].nfills += astats->bstats[i].nfills; 842 sdstats->bstats[i].nflushes += 843 astats->bstats[i].nflushes; 844 sdstats->bstats[i].nslabs += astats->bstats[i].nslabs; 845 sdstats->bstats[i].reslabs += astats->bstats[i].reslabs; 846 if (!destroyed) { 847 sdstats->bstats[i].curslabs += 848 astats->bstats[i].curslabs; 849 } else { 850 assert(astats->bstats[i].curslabs == 0); 851 } 852 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, 853 &astats->bstats[i].mutex_data); 854 } 855 856 for (i = 0; i < NSIZES - NBINS; i++) { 857 ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, 858 &astats->lstats[i].nmalloc); 859 ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, 860 &astats->lstats[i].ndalloc); 861 ctl_accum_arena_stats_u64(&sdstats->lstats[i].nrequests, 862 &astats->lstats[i].nrequests); 863 if (!destroyed) { 864 sdstats->lstats[i].curlextents += 865 astats->lstats[i].curlextents; 866 } else { 867 assert(astats->lstats[i].curlextents == 0); 868 } 869 } 870 } 871} 872 873static void 874ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena, 875 unsigned i, bool destroyed) { 876 ctl_arena_t *ctl_arena = arenas_i(i); 877 878 ctl_arena_clear(ctl_arena); 879 ctl_arena_stats_amerge(tsdn, ctl_arena, arena); 880 /* Merge into sum stats as well. */ 881 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed); 882} 883 884static unsigned 885ctl_arena_init(tsd_t *tsd, extent_hooks_t *extent_hooks) { 886 unsigned arena_ind; 887 ctl_arena_t *ctl_arena; 888 889 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) != 890 NULL) { 891 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 892 arena_ind = ctl_arena->arena_ind; 893 } else { 894 arena_ind = ctl_arenas->narenas; 895 } 896 897 /* Trigger stats allocation. */ 898 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) { 899 return UINT_MAX; 900 } 901 902 /* Initialize new arena. */ 903 if (arena_init(tsd_tsdn(tsd), arena_ind, extent_hooks) == NULL) { 904 return UINT_MAX; 905 } 906 907 if (arena_ind == ctl_arenas->narenas) { 908 ctl_arenas->narenas++; 909 } 910 911 return arena_ind; 912} 913 914static void 915ctl_background_thread_stats_read(tsdn_t *tsdn) { 916 background_thread_stats_t *stats = &ctl_stats->background_thread; 917 if (!have_background_thread || 918 background_thread_stats_read(tsdn, stats)) { 919 memset(stats, 0, sizeof(background_thread_stats_t)); 920 nstime_init(&stats->run_interval, 0); 921 } 922} 923 924static void 925ctl_refresh(tsdn_t *tsdn) { 926 unsigned i; 927 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL); 928 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas); 929 930 /* 931 * Clear sum stats, since they will be merged into by 932 * ctl_arena_refresh(). 933 */ 934 ctl_arena_clear(ctl_sarena); 935 936 for (i = 0; i < ctl_arenas->narenas; i++) { 937 tarenas[i] = arena_get(tsdn, i, false); 938 } 939 940 for (i = 0; i < ctl_arenas->narenas; i++) { 941 ctl_arena_t *ctl_arena = arenas_i(i); 942 bool initialized = (tarenas[i] != NULL); 943 944 ctl_arena->initialized = initialized; 945 if (initialized) { 946 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i, 947 false); 948 } 949 } 950 951 if (config_stats) { 952 ctl_stats->allocated = ctl_sarena->astats->allocated_small + 953 atomic_load_zu(&ctl_sarena->astats->astats.allocated_large, 954 ATOMIC_RELAXED); 955 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE); 956 ctl_stats->metadata = atomic_load_zu( 957 &ctl_sarena->astats->astats.base, ATOMIC_RELAXED) + 958 atomic_load_zu(&ctl_sarena->astats->astats.internal, 959 ATOMIC_RELAXED); 960 ctl_stats->metadata_thp = atomic_load_zu( 961 &ctl_sarena->astats->astats.metadata_thp, ATOMIC_RELAXED); 962 ctl_stats->resident = atomic_load_zu( 963 &ctl_sarena->astats->astats.resident, ATOMIC_RELAXED); 964 ctl_stats->mapped = atomic_load_zu( 965 &ctl_sarena->astats->astats.mapped, ATOMIC_RELAXED); 966 ctl_stats->retained = atomic_load_zu( 967 &ctl_sarena->astats->astats.retained, ATOMIC_RELAXED); 968 969 ctl_background_thread_stats_read(tsdn); 970 971#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ 972 malloc_mutex_lock(tsdn, &mtx); \ 973 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ 974 malloc_mutex_unlock(tsdn, &mtx); 975 976 if (config_prof && opt_prof) { 977 READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_prof, 978 bt2gctx_mtx); 979 } 980 if (have_background_thread) { 981 READ_GLOBAL_MUTEX_PROF_DATA( 982 global_prof_mutex_background_thread, 983 background_thread_lock); 984 } else { 985 memset(&ctl_stats->mutex_prof_data[ 986 global_prof_mutex_background_thread], 0, 987 sizeof(mutex_prof_data_t)); 988 } 989 /* We own ctl mutex already. */ 990 malloc_mutex_prof_read(tsdn, 991 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl], 992 &ctl_mtx); 993#undef READ_GLOBAL_MUTEX_PROF_DATA 994 } 995 ctl_arenas->epoch++; 996} 997 998static bool 999ctl_init(tsd_t *tsd) { 1000 bool ret; 1001 tsdn_t *tsdn = tsd_tsdn(tsd); 1002 1003 malloc_mutex_lock(tsdn, &ctl_mtx); 1004 if (!ctl_initialized) { 1005 ctl_arena_t *ctl_sarena, *ctl_darena; 1006 unsigned i; 1007 1008 /* 1009 * Allocate demand-zeroed space for pointers to the full 1010 * range of supported arena indices. 1011 */ 1012 if (ctl_arenas == NULL) { 1013 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn, 1014 b0get(), sizeof(ctl_arenas_t), QUANTUM); 1015 if (ctl_arenas == NULL) { 1016 ret = true; 1017 goto label_return; 1018 } 1019 } 1020 1021 if (config_stats && ctl_stats == NULL) { 1022 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(), 1023 sizeof(ctl_stats_t), QUANTUM); 1024 if (ctl_stats == NULL) { 1025 ret = true; 1026 goto label_return; 1027 } 1028 } 1029 1030 /* 1031 * Allocate space for the current full range of arenas 1032 * here rather than doing it lazily elsewhere, in order 1033 * to limit when OOM-caused errors can occur. 1034 */ 1035 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false, 1036 true)) == NULL) { 1037 ret = true; 1038 goto label_return; 1039 } 1040 ctl_sarena->initialized = true; 1041 1042 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED, 1043 false, true)) == NULL) { 1044 ret = true; 1045 goto label_return; 1046 } 1047 ctl_arena_clear(ctl_darena); 1048 /* 1049 * Don't toggle ctl_darena to initialized until an arena is 1050 * actually destroyed, so that arena.<i>.initialized can be used 1051 * to query whether the stats are relevant. 1052 */ 1053 1054 ctl_arenas->narenas = narenas_total_get(); 1055 for (i = 0; i < ctl_arenas->narenas; i++) { 1056 if (arenas_i_impl(tsd, i, false, true) == NULL) { 1057 ret = true; 1058 goto label_return; 1059 } 1060 } 1061 1062 ql_new(&ctl_arenas->destroyed); 1063 ctl_refresh(tsdn); 1064 1065 ctl_initialized = true; 1066 } 1067 1068 ret = false; 1069label_return: 1070 malloc_mutex_unlock(tsdn, &ctl_mtx); 1071 return ret; 1072} 1073 1074static int 1075ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp, 1076 size_t *mibp, size_t *depthp) { 1077 int ret; 1078 const char *elm, *tdot, *dot; 1079 size_t elen, i, j; 1080 const ctl_named_node_t *node; 1081 1082 elm = name; 1083 /* Equivalent to strchrnul(). */ 1084 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); 1085 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1086 if (elen == 0) { 1087 ret = ENOENT; 1088 goto label_return; 1089 } 1090 node = super_root_node; 1091 for (i = 0; i < *depthp; i++) { 1092 assert(node); 1093 assert(node->nchildren > 0); 1094 if (ctl_named_node(node->children) != NULL) { 1095 const ctl_named_node_t *pnode = node; 1096 1097 /* Children are named. */ 1098 for (j = 0; j < node->nchildren; j++) { 1099 const ctl_named_node_t *child = 1100 ctl_named_children(node, j); 1101 if (strlen(child->name) == elen && 1102 strncmp(elm, child->name, elen) == 0) { 1103 node = child; 1104 if (nodesp != NULL) { 1105 nodesp[i] = 1106 (const ctl_node_t *)node; 1107 } 1108 mibp[i] = j; 1109 break; 1110 } 1111 } 1112 if (node == pnode) { 1113 ret = ENOENT; 1114 goto label_return; 1115 } 1116 } else { 1117 uintmax_t index; 1118 const ctl_indexed_node_t *inode; 1119 1120 /* Children are indexed. */ 1121 index = malloc_strtoumax(elm, NULL, 10); 1122 if (index == UINTMAX_MAX || index > SIZE_T_MAX) { 1123 ret = ENOENT; 1124 goto label_return; 1125 } 1126 1127 inode = ctl_indexed_node(node->children); 1128 node = inode->index(tsdn, mibp, *depthp, (size_t)index); 1129 if (node == NULL) { 1130 ret = ENOENT; 1131 goto label_return; 1132 } 1133 1134 if (nodesp != NULL) { 1135 nodesp[i] = (const ctl_node_t *)node; 1136 } 1137 mibp[i] = (size_t)index; 1138 } 1139 1140 if (node->ctl != NULL) { 1141 /* Terminal node. */ 1142 if (*dot != '\0') { 1143 /* 1144 * The name contains more elements than are 1145 * in this path through the tree. 1146 */ 1147 ret = ENOENT; 1148 goto label_return; 1149 } 1150 /* Complete lookup successful. */ 1151 *depthp = i + 1; 1152 break; 1153 } 1154 1155 /* Update elm. */ 1156 if (*dot == '\0') { 1157 /* No more elements. */ 1158 ret = ENOENT; 1159 goto label_return; 1160 } 1161 elm = &dot[1]; 1162 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : 1163 strchr(elm, '\0'); 1164 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1165 } 1166 1167 ret = 0; 1168label_return: 1169 return ret; 1170} 1171 1172int 1173ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, 1174 void *newp, size_t newlen) { 1175 int ret; 1176 size_t depth; 1177 ctl_node_t const *nodes[CTL_MAX_DEPTH]; 1178 size_t mib[CTL_MAX_DEPTH]; 1179 const ctl_named_node_t *node; 1180 1181 if (!ctl_initialized && ctl_init(tsd)) { 1182 ret = EAGAIN; 1183 goto label_return; 1184 } 1185 1186 depth = CTL_MAX_DEPTH; 1187 ret = ctl_lookup(tsd_tsdn(tsd), name, nodes, mib, &depth); 1188 if (ret != 0) { 1189 goto label_return; 1190 } 1191 1192 node = ctl_named_node(nodes[depth-1]); 1193 if (node != NULL && node->ctl) { 1194 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen); 1195 } else { 1196 /* The name refers to a partial path through the ctl tree. */ 1197 ret = ENOENT; 1198 } 1199 1200label_return: 1201 return(ret); 1202} 1203 1204int 1205ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) { 1206 int ret; 1207 1208 if (!ctl_initialized && ctl_init(tsd)) { 1209 ret = EAGAIN; 1210 goto label_return; 1211 } 1212 1213 ret = ctl_lookup(tsd_tsdn(tsd), name, NULL, mibp, miblenp); 1214label_return: 1215 return(ret); 1216} 1217 1218int 1219ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1220 size_t *oldlenp, void *newp, size_t newlen) { 1221 int ret; 1222 const ctl_named_node_t *node; 1223 size_t i; 1224 1225 if (!ctl_initialized && ctl_init(tsd)) { 1226 ret = EAGAIN; 1227 goto label_return; 1228 } 1229 1230 /* Iterate down the tree. */ 1231 node = super_root_node; 1232 for (i = 0; i < miblen; i++) { 1233 assert(node); 1234 assert(node->nchildren > 0); 1235 if (ctl_named_node(node->children) != NULL) { 1236 /* Children are named. */ 1237 if (node->nchildren <= mib[i]) { 1238 ret = ENOENT; 1239 goto label_return; 1240 } 1241 node = ctl_named_children(node, mib[i]); 1242 } else { 1243 const ctl_indexed_node_t *inode; 1244 1245 /* Indexed element. */ 1246 inode = ctl_indexed_node(node->children); 1247 node = inode->index(tsd_tsdn(tsd), mib, miblen, mib[i]); 1248 if (node == NULL) { 1249 ret = ENOENT; 1250 goto label_return; 1251 } 1252 } 1253 } 1254 1255 /* Call the ctl function. */ 1256 if (node && node->ctl) { 1257 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 1258 } else { 1259 /* Partial MIB. */ 1260 ret = ENOENT; 1261 } 1262 1263label_return: 1264 return(ret); 1265} 1266 1267bool 1268ctl_boot(void) { 1269 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL, 1270 malloc_mutex_rank_exclusive)) { 1271 return true; 1272 } 1273 1274 ctl_initialized = false; 1275 1276 return false; 1277} 1278 1279void 1280ctl_prefork(tsdn_t *tsdn) { 1281 malloc_mutex_prefork(tsdn, &ctl_mtx); 1282} 1283 1284void 1285ctl_postfork_parent(tsdn_t *tsdn) { 1286 malloc_mutex_postfork_parent(tsdn, &ctl_mtx); 1287} 1288 1289void 1290ctl_postfork_child(tsdn_t *tsdn) { 1291 malloc_mutex_postfork_child(tsdn, &ctl_mtx); 1292} 1293 1294/******************************************************************************/ 1295/* *_ctl() functions. */ 1296 1297#define READONLY() do { \ 1298 if (newp != NULL || newlen != 0) { \ 1299 ret = EPERM; \ 1300 goto label_return; \ 1301 } \ 1302} while (/*CONSTCOND*/0) 1303 1304#define WRITEONLY() do { \ 1305 if (oldp != NULL || oldlenp != NULL) { \ 1306 ret = EPERM; \ 1307 goto label_return; \ 1308 } \ 1309} while (/*CONSTCOND*/0) 1310 1311#define READ_XOR_WRITE() do { \ 1312 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ 1313 newlen != 0)) { \ 1314 ret = EPERM; \ 1315 goto label_return; \ 1316 } \ 1317} while (/*CONSTCOND*/0) 1318 1319#define READ(v, t) do { \ 1320 if (oldp != NULL && oldlenp != NULL) { \ 1321 if (*oldlenp != sizeof(t)) { \ 1322 size_t copylen = (sizeof(t) <= *oldlenp) \ 1323 ? sizeof(t) : *oldlenp; \ 1324 memcpy(oldp, (void *)&(v), copylen); \ 1325 ret = EINVAL; \ 1326 goto label_return; \ 1327 } \ 1328 *(t *)oldp = (v); \ 1329 } \ 1330} while (/*CONSTCOND*/0) 1331 1332#define WRITE(v, t) do { \ 1333 if (newp != NULL) { \ 1334 if (newlen != sizeof(t)) { \ 1335 ret = EINVAL; \ 1336 goto label_return; \ 1337 } \ 1338 (v) = *(t *)newp; \ 1339 } \ 1340} while (/*CONSTCOND*/0) 1341 1342#define MIB_UNSIGNED(v, i) do { \ 1343 if (mib[i] > UINT_MAX) { \ 1344 ret = EFAULT; \ 1345 goto label_return; \ 1346 } \ 1347 v = (unsigned)mib[i]; \ 1348} while (/*CONSTCOND*/0) 1349 1350/* 1351 * There's a lot of code duplication in the following macros due to limitations 1352 * in how nested cpp macros are expanded. 1353 */ 1354#define CTL_RO_CLGEN(c, l, n, v, t) \ 1355static int \ 1356n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1357 size_t *oldlenp, void *newp, size_t newlen) { \ 1358 int ret; \ 1359 t oldval; \ 1360 \ 1361 if (!(c)) { \ 1362 return ENOENT; \ 1363 } \ 1364 if (l) { \ 1365 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1366 } \ 1367 READONLY(); \ 1368 oldval = (v); \ 1369 READ(oldval, t); \ 1370 \ 1371 ret = 0; \ 1372label_return: \ 1373 if (l) { \ 1374 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1375 } \ 1376 return ret; \ 1377} 1378 1379#define CTL_RO_CGEN(c, n, v, t) \ 1380static int \ 1381n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1382 size_t *oldlenp, void *newp, size_t newlen) { \ 1383 int ret; \ 1384 t oldval; \ 1385 \ 1386 if (!(c)) { \ 1387 return ENOENT; \ 1388 } \ 1389 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1390 READONLY(); \ 1391 oldval = (v); \ 1392 READ(oldval, t); \ 1393 \ 1394 ret = 0; \ 1395label_return: \ 1396 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1397 return ret; \ 1398} 1399 1400#define CTL_RO_GEN(n, v, t) \ 1401static int \ 1402n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1403 size_t *oldlenp, void *newp, size_t newlen) { \ 1404 int ret; \ 1405 t oldval; \ 1406 \ 1407 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1408 READONLY(); \ 1409 oldval = (v); \ 1410 READ(oldval, t); \ 1411 \ 1412 ret = 0; \ 1413label_return: \ 1414 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1415 return ret; \ 1416} 1417 1418/* 1419 * ctl_mtx is not acquired, under the assumption that no pertinent data will 1420 * mutate during the call. 1421 */ 1422#define CTL_RO_NL_CGEN(c, n, v, t) \ 1423static int \ 1424n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1425 size_t *oldlenp, void *newp, size_t newlen) { \ 1426 int ret; \ 1427 t oldval; \ 1428 \ 1429 if (!(c)) { \ 1430 return ENOENT; \ 1431 } \ 1432 READONLY(); \ 1433 oldval = (v); \ 1434 READ(oldval, t); \ 1435 \ 1436 ret = 0; \ 1437label_return: \ 1438 return ret; \ 1439} 1440 1441#define CTL_RO_NL_GEN(n, v, t) \ 1442static int \ 1443n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1444 size_t *oldlenp, void *newp, size_t newlen) { \ 1445 int ret; \ 1446 t oldval; \ 1447 \ 1448 READONLY(); \ 1449 oldval = (v); \ 1450 READ(oldval, t); \ 1451 \ 1452 ret = 0; \ 1453label_return: \ 1454 return ret; \ 1455} 1456 1457#define CTL_TSD_RO_NL_CGEN(c, n, m, t) \ 1458static int \ 1459n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1460 size_t *oldlenp, void *newp, size_t newlen) { \ 1461 int ret; \ 1462 t oldval; \ 1463 \ 1464 if (!(c)) { \ 1465 return ENOENT; \ 1466 } \ 1467 READONLY(); \ 1468 oldval = (m(tsd)); \ 1469 READ(oldval, t); \ 1470 \ 1471 ret = 0; \ 1472label_return: \ 1473 return ret; \ 1474} 1475 1476#define CTL_RO_CONFIG_GEN(n, t) \ 1477static int \ 1478n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1479 size_t *oldlenp, void *newp, size_t newlen) { \ 1480 int ret; \ 1481 t oldval; \ 1482 \ 1483 READONLY(); \ 1484 oldval = n; \ 1485 READ(oldval, t); \ 1486 \ 1487 ret = 0; \ 1488label_return: \ 1489 return ret; \ 1490} 1491 1492/******************************************************************************/ 1493 1494CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) 1495 1496static int 1497epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1498 size_t *oldlenp, void *newp, size_t newlen) { 1499 int ret; 1500 UNUSED uint64_t newval; 1501 1502 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1503 WRITE(newval, uint64_t); 1504 if (newp != NULL) { 1505 ctl_refresh(tsd_tsdn(tsd)); 1506 } 1507 READ(ctl_arenas->epoch, uint64_t); 1508 1509 ret = 0; 1510label_return: 1511 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1512 return ret; 1513} 1514 1515static int 1516background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1517 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1518 int ret; 1519 bool oldval; 1520 1521 if (!have_background_thread) { 1522 return ENOENT; 1523 } 1524 background_thread_ctl_init(tsd_tsdn(tsd)); 1525 1526 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1527 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1528 if (newp == NULL) { 1529 oldval = background_thread_enabled(); 1530 READ(oldval, bool); 1531 } else { 1532 if (newlen != sizeof(bool)) { 1533 ret = EINVAL; 1534 goto label_return; 1535 } 1536 oldval = background_thread_enabled(); 1537 READ(oldval, bool); 1538 1539 bool newval = *(bool *)newp; 1540 if (newval == oldval) { 1541 ret = 0; 1542 goto label_return; 1543 } 1544 1545 background_thread_enabled_set(tsd_tsdn(tsd), newval); 1546 if (newval) { 1547 if (!can_enable_background_thread) { 1548 malloc_printf("<jemalloc>: Error in dlsym(" 1549 "RTLD_NEXT, \"pthread_create\"). Cannot " 1550 "enable background_thread\n"); 1551 ret = EFAULT; 1552 goto label_return; 1553 } 1554 if (background_threads_enable(tsd)) { 1555 ret = EFAULT; 1556 goto label_return; 1557 } 1558 } else { 1559 if (background_threads_disable(tsd)) { 1560 ret = EFAULT; 1561 goto label_return; 1562 } 1563 } 1564 } 1565 ret = 0; 1566label_return: 1567 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1568 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1569 1570 return ret; 1571} 1572 1573static int 1574max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1575 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1576 int ret; 1577 size_t oldval; 1578 1579 if (!have_background_thread) { 1580 return ENOENT; 1581 } 1582 background_thread_ctl_init(tsd_tsdn(tsd)); 1583 1584 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1585 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1586 if (newp == NULL) { 1587 oldval = max_background_threads; 1588 READ(oldval, size_t); 1589 } else { 1590 if (newlen != sizeof(size_t)) { 1591 ret = EINVAL; 1592 goto label_return; 1593 } 1594 oldval = max_background_threads; 1595 READ(oldval, size_t); 1596 1597 size_t newval = *(size_t *)newp; 1598 if (newval == oldval) { 1599 ret = 0; 1600 goto label_return; 1601 } 1602 if (newval > opt_max_background_threads) { 1603 ret = EINVAL; 1604 goto label_return; 1605 } 1606 1607 if (background_thread_enabled()) { 1608 if (!can_enable_background_thread) { 1609 malloc_printf("<jemalloc>: Error in dlsym(" 1610 "RTLD_NEXT, \"pthread_create\"). Cannot " 1611 "enable background_thread\n"); 1612 ret = EFAULT; 1613 goto label_return; 1614 } 1615 background_thread_enabled_set(tsd_tsdn(tsd), false); 1616 if (background_threads_disable(tsd)) { 1617 ret = EFAULT; 1618 goto label_return; 1619 } 1620 max_background_threads = newval; 1621 background_thread_enabled_set(tsd_tsdn(tsd), true); 1622 if (background_threads_enable(tsd)) { 1623 ret = EFAULT; 1624 goto label_return; 1625 } 1626 } else { 1627 max_background_threads = newval; 1628 } 1629 } 1630 ret = 0; 1631label_return: 1632 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1633 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1634 1635 return ret; 1636} 1637 1638/******************************************************************************/ 1639 1640CTL_RO_CONFIG_GEN(config_cache_oblivious, bool) 1641CTL_RO_CONFIG_GEN(config_debug, bool) 1642CTL_RO_CONFIG_GEN(config_fill, bool) 1643CTL_RO_CONFIG_GEN(config_lazy_lock, bool) 1644CTL_RO_CONFIG_GEN(config_malloc_conf, const char *) 1645CTL_RO_CONFIG_GEN(config_prof, bool) 1646CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) 1647CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) 1648CTL_RO_CONFIG_GEN(config_stats, bool) 1649CTL_RO_CONFIG_GEN(config_utrace, bool) 1650CTL_RO_CONFIG_GEN(config_xmalloc, bool) 1651 1652/******************************************************************************/ 1653 1654CTL_RO_NL_GEN(opt_abort, opt_abort, bool) 1655CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) 1656CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], 1657 const char *) 1658CTL_RO_NL_GEN(opt_retain, opt_retain, bool) 1659CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) 1660CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) 1661CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], 1662 const char *) 1663CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) 1664CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) 1665CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) 1666CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t) 1667CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) 1668CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *) 1669CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) 1670CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) 1671CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) 1672CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) 1673CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) 1674CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *) 1675CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, 1676 size_t) 1677CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) 1678CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) 1679CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) 1680CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) 1681CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, 1682 opt_prof_thread_active_init, bool) 1683CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) 1684CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) 1685CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) 1686CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) 1687CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) 1688CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) 1689 1690/******************************************************************************/ 1691 1692static int 1693thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1694 size_t *oldlenp, void *newp, size_t newlen) { 1695 int ret; 1696 arena_t *oldarena; 1697 unsigned newind, oldind; 1698 1699 oldarena = arena_choose(tsd, NULL); 1700 if (oldarena == NULL) { 1701 return EAGAIN; 1702 } 1703 newind = oldind = arena_ind_get(oldarena); 1704 WRITE(newind, unsigned); 1705 READ(oldind, unsigned); 1706 1707 if (newind != oldind) { 1708 arena_t *newarena; 1709 1710 if (newind >= narenas_total_get()) { 1711 /* New arena index is out of range. */ 1712 ret = EFAULT; 1713 goto label_return; 1714 } 1715 1716 if (have_percpu_arena && 1717 PERCPU_ARENA_ENABLED(opt_percpu_arena)) { 1718 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) { 1719 /* 1720 * If perCPU arena is enabled, thread_arena 1721 * control is not allowed for the auto arena 1722 * range. 1723 */ 1724 ret = EPERM; 1725 goto label_return; 1726 } 1727 } 1728 1729 /* Initialize arena if necessary. */ 1730 newarena = arena_get(tsd_tsdn(tsd), newind, true); 1731 if (newarena == NULL) { 1732 ret = EAGAIN; 1733 goto label_return; 1734 } 1735 /* Set new arena/tcache associations. */ 1736 arena_migrate(tsd, oldind, newind); 1737 if (tcache_available(tsd)) { 1738 tcache_arena_reassociate(tsd_tsdn(tsd), 1739 tsd_tcachep_get(tsd), newarena); 1740 } 1741 } 1742 1743 ret = 0; 1744label_return: 1745 return ret; 1746} 1747 1748CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get, 1749 uint64_t) 1750CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get, 1751 uint64_t *) 1752CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get, 1753 uint64_t) 1754CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, 1755 tsd_thread_deallocatedp_get, uint64_t *) 1756 1757static int 1758thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1759 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1760 int ret; 1761 bool oldval; 1762 1763 oldval = tcache_enabled_get(tsd); 1764 if (newp != NULL) { 1765 if (newlen != sizeof(bool)) { 1766 ret = EINVAL; 1767 goto label_return; 1768 } 1769 tcache_enabled_set(tsd, *(bool *)newp); 1770 } 1771 READ(oldval, bool); 1772 1773 ret = 0; 1774label_return: 1775 return ret; 1776} 1777 1778static int 1779thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1780 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1781 int ret; 1782 1783 if (!tcache_available(tsd)) { 1784 ret = EFAULT; 1785 goto label_return; 1786 } 1787 1788 READONLY(); 1789 WRITEONLY(); 1790 1791 tcache_flush(tsd); 1792 1793 ret = 0; 1794label_return: 1795 return ret; 1796} 1797 1798static int 1799thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1800 size_t *oldlenp, void *newp, size_t newlen) { 1801 int ret; 1802 1803 if (!config_prof) { 1804 return ENOENT; 1805 } 1806 1807 READ_XOR_WRITE(); 1808 1809 if (newp != NULL) { 1810 if (newlen != sizeof(const char *)) { 1811 ret = EINVAL; 1812 goto label_return; 1813 } 1814 1815 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != 1816 0) { 1817 goto label_return; 1818 } 1819 } else { 1820 const char *oldname = prof_thread_name_get(tsd); 1821 READ(oldname, const char *); 1822 } 1823 1824 ret = 0; 1825label_return: 1826 return ret; 1827} 1828 1829static int 1830thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1831 size_t *oldlenp, void *newp, size_t newlen) { 1832 int ret; 1833 bool oldval; 1834 1835 if (!config_prof) { 1836 return ENOENT; 1837 } 1838 1839 oldval = prof_thread_active_get(tsd); 1840 if (newp != NULL) { 1841 if (newlen != sizeof(bool)) { 1842 ret = EINVAL; 1843 goto label_return; 1844 } 1845 if (prof_thread_active_set(tsd, *(bool *)newp)) { 1846 ret = EAGAIN; 1847 goto label_return; 1848 } 1849 } 1850 READ(oldval, bool); 1851 1852 ret = 0; 1853label_return: 1854 return ret; 1855} 1856 1857/******************************************************************************/ 1858 1859static int 1860tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1861 size_t *oldlenp, void *newp, size_t newlen) { 1862 int ret; 1863 unsigned tcache_ind; 1864 1865 READONLY(); 1866 if (tcaches_create(tsd, &tcache_ind)) { 1867 ret = EFAULT; 1868 goto label_return; 1869 } 1870 READ(tcache_ind, unsigned); 1871 1872 ret = 0; 1873label_return: 1874 return ret; 1875} 1876 1877static int 1878tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1879 size_t *oldlenp, void *newp, size_t newlen) { 1880 int ret; 1881 unsigned tcache_ind; 1882 1883 WRITEONLY(); 1884 tcache_ind = UINT_MAX; 1885 WRITE(tcache_ind, unsigned); 1886 if (tcache_ind == UINT_MAX) { 1887 ret = EFAULT; 1888 goto label_return; 1889 } 1890 tcaches_flush(tsd, tcache_ind); 1891 1892 ret = 0; 1893label_return: 1894 return ret; 1895} 1896 1897static int 1898tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1899 size_t *oldlenp, void *newp, size_t newlen) { 1900 int ret; 1901 unsigned tcache_ind; 1902 1903 WRITEONLY(); 1904 tcache_ind = UINT_MAX; 1905 WRITE(tcache_ind, unsigned); 1906 if (tcache_ind == UINT_MAX) { 1907 ret = EFAULT; 1908 goto label_return; 1909 } 1910 tcaches_destroy(tsd, tcache_ind); 1911 1912 ret = 0; 1913label_return: 1914 return ret; 1915} 1916 1917/******************************************************************************/ 1918 1919static int 1920arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1921 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1922 int ret; 1923 tsdn_t *tsdn = tsd_tsdn(tsd); 1924 unsigned arena_ind; 1925 bool initialized; 1926 1927 READONLY(); 1928 MIB_UNSIGNED(arena_ind, 1); 1929 1930 malloc_mutex_lock(tsdn, &ctl_mtx); 1931 initialized = arenas_i(arena_ind)->initialized; 1932 malloc_mutex_unlock(tsdn, &ctl_mtx); 1933 1934 READ(initialized, bool); 1935 1936 ret = 0; 1937label_return: 1938 return ret; 1939} 1940 1941static void 1942arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { 1943 malloc_mutex_lock(tsdn, &ctl_mtx); 1944 { 1945 unsigned narenas = ctl_arenas->narenas; 1946 1947 /* 1948 * Access via index narenas is deprecated, and scheduled for 1949 * removal in 6.0.0. 1950 */ 1951 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) { 1952 unsigned i; 1953 VARIABLE_ARRAY(arena_t *, tarenas, narenas); 1954 1955 for (i = 0; i < narenas; i++) { 1956 tarenas[i] = arena_get(tsdn, i, false); 1957 } 1958 1959 /* 1960 * No further need to hold ctl_mtx, since narenas and 1961 * tarenas contain everything needed below. 1962 */ 1963 malloc_mutex_unlock(tsdn, &ctl_mtx); 1964 1965 for (i = 0; i < narenas; i++) { 1966 if (tarenas[i] != NULL) { 1967 arena_decay(tsdn, tarenas[i], false, 1968 all); 1969 } 1970 } 1971 } else { 1972 arena_t *tarena; 1973 1974 assert(arena_ind < narenas); 1975 1976 tarena = arena_get(tsdn, arena_ind, false); 1977 1978 /* No further need to hold ctl_mtx. */ 1979 malloc_mutex_unlock(tsdn, &ctl_mtx); 1980 1981 if (tarena != NULL) { 1982 arena_decay(tsdn, tarena, false, all); 1983 } 1984 } 1985 } 1986} 1987 1988static int 1989arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1990 size_t *oldlenp, void *newp, size_t newlen) { 1991 int ret; 1992 unsigned arena_ind; 1993 1994 READONLY(); 1995 WRITEONLY(); 1996 MIB_UNSIGNED(arena_ind, 1); 1997 arena_i_decay(tsd_tsdn(tsd), arena_ind, false); 1998 1999 ret = 0; 2000label_return: 2001 return ret; 2002} 2003 2004static int 2005arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2006 size_t *oldlenp, void *newp, size_t newlen) { 2007 int ret; 2008 unsigned arena_ind; 2009 2010 READONLY(); 2011 WRITEONLY(); 2012 MIB_UNSIGNED(arena_ind, 1); 2013 arena_i_decay(tsd_tsdn(tsd), arena_ind, true); 2014 2015 ret = 0; 2016label_return: 2017 return ret; 2018} 2019 2020static int 2021arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen, 2022 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind, 2023 arena_t **arena) { 2024 int ret; 2025 2026 READONLY(); 2027 WRITEONLY(); 2028 MIB_UNSIGNED(*arena_ind, 1); 2029 2030 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false); 2031 if (*arena == NULL || arena_is_auto(*arena)) { 2032 ret = EFAULT; 2033 goto label_return; 2034 } 2035 2036 ret = 0; 2037label_return: 2038 return ret; 2039} 2040 2041static void 2042arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { 2043 /* Temporarily disable the background thread during arena reset. */ 2044 if (have_background_thread) { 2045 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 2046 if (background_thread_enabled()) { 2047 unsigned ind = arena_ind % ncpus; 2048 background_thread_info_t *info = 2049 &background_thread_info[ind]; 2050 assert(info->state == background_thread_started); 2051 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 2052 info->state = background_thread_paused; 2053 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 2054 } 2055 } 2056} 2057 2058static void 2059arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { 2060 if (have_background_thread) { 2061 if (background_thread_enabled()) { 2062 unsigned ind = arena_ind % ncpus; 2063 background_thread_info_t *info = 2064 &background_thread_info[ind]; 2065 assert(info->state == background_thread_paused); 2066 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 2067 info->state = background_thread_started; 2068 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 2069 } 2070 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 2071 } 2072} 2073 2074static int 2075arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2076 size_t *oldlenp, void *newp, size_t newlen) { 2077 int ret; 2078 unsigned arena_ind; 2079 arena_t *arena; 2080 2081 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2082 newp, newlen, &arena_ind, &arena); 2083 if (ret != 0) { 2084 return ret; 2085 } 2086 2087 arena_reset_prepare_background_thread(tsd, arena_ind); 2088 arena_reset(tsd, arena); 2089 arena_reset_finish_background_thread(tsd, arena_ind); 2090 2091 return ret; 2092} 2093 2094static int 2095arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2096 size_t *oldlenp, void *newp, size_t newlen) { 2097 int ret; 2098 unsigned arena_ind; 2099 arena_t *arena; 2100 ctl_arena_t *ctl_darena, *ctl_arena; 2101 2102 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2103 newp, newlen, &arena_ind, &arena); 2104 if (ret != 0) { 2105 goto label_return; 2106 } 2107 2108 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena, 2109 true) != 0) { 2110 ret = EFAULT; 2111 goto label_return; 2112 } 2113 2114 arena_reset_prepare_background_thread(tsd, arena_ind); 2115 /* Merge stats after resetting and purging arena. */ 2116 arena_reset(tsd, arena); 2117 arena_decay(tsd_tsdn(tsd), arena, false, true); 2118 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); 2119 ctl_darena->initialized = true; 2120 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); 2121 /* Destroy arena. */ 2122 arena_destroy(tsd, arena); 2123 ctl_arena = arenas_i(arena_ind); 2124 ctl_arena->initialized = false; 2125 /* Record arena index for later recycling via arenas.create. */ 2126 ql_elm_new(ctl_arena, destroyed_link); 2127 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 2128 arena_reset_finish_background_thread(tsd, arena_ind); 2129 2130 assert(ret == 0); 2131label_return: 2132 return ret; 2133} 2134 2135static int 2136arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2137 size_t *oldlenp, void *newp, size_t newlen) { 2138 int ret; 2139 const char *dss = NULL; 2140 unsigned arena_ind; 2141 dss_prec_t dss_prec_old = dss_prec_limit; 2142 dss_prec_t dss_prec = dss_prec_limit; 2143 2144 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2145 WRITE(dss, const char *); 2146 MIB_UNSIGNED(arena_ind, 1); 2147 if (dss != NULL) { 2148 int i; 2149 bool match = false; 2150 2151 for (i = 0; i < dss_prec_limit; i++) { 2152 if (strcmp(dss_prec_names[i], dss) == 0) { 2153 dss_prec = i; 2154 match = true; 2155 break; 2156 } 2157 } 2158 2159 if (!match) { 2160 ret = EINVAL; 2161 goto label_return; 2162 } 2163 } 2164 2165 /* 2166 * Access via index narenas is deprecated, and scheduled for removal in 2167 * 6.0.0. 2168 */ 2169 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == 2170 ctl_arenas->narenas) { 2171 if (dss_prec != dss_prec_limit && 2172 extent_dss_prec_set(dss_prec)) { 2173 ret = EFAULT; 2174 goto label_return; 2175 } 2176 dss_prec_old = extent_dss_prec_get(); 2177 } else { 2178 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2179 if (arena == NULL || (dss_prec != dss_prec_limit && 2180 arena_dss_prec_set(arena, dss_prec))) { 2181 ret = EFAULT; 2182 goto label_return; 2183 } 2184 dss_prec_old = arena_dss_prec_get(arena); 2185 } 2186 2187 dss = dss_prec_names[dss_prec_old]; 2188 READ(dss, const char *); 2189 2190 ret = 0; 2191label_return: 2192 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2193 return ret; 2194} 2195 2196static int 2197arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, 2198 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { 2199 int ret; 2200 unsigned arena_ind; 2201 arena_t *arena; 2202 2203 MIB_UNSIGNED(arena_ind, 1); 2204 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2205 if (arena == NULL) { 2206 ret = EFAULT; 2207 goto label_return; 2208 } 2209 2210 if (oldp != NULL && oldlenp != NULL) { 2211 size_t oldval = dirty ? arena_dirty_decay_ms_get(arena) : 2212 arena_muzzy_decay_ms_get(arena); 2213 READ(oldval, ssize_t); 2214 } 2215 if (newp != NULL) { 2216 if (newlen != sizeof(ssize_t)) { 2217 ret = EINVAL; 2218 goto label_return; 2219 } 2220 if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, 2221 *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), 2222 arena, *(ssize_t *)newp)) { 2223 ret = EFAULT; 2224 goto label_return; 2225 } 2226 } 2227 2228 ret = 0; 2229label_return: 2230 return ret; 2231} 2232 2233static int 2234arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2235 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2236 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2237 newlen, true); 2238} 2239 2240static int 2241arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2242 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2243 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2244 newlen, false); 2245} 2246 2247static int 2248arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2249 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2250 int ret; 2251 unsigned arena_ind; 2252 arena_t *arena; 2253 2254 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2255 MIB_UNSIGNED(arena_ind, 1); 2256 if (arena_ind < narenas_total_get()) { 2257 extent_hooks_t *old_extent_hooks; 2258 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2259 if (arena == NULL) { 2260 if (arena_ind >= narenas_auto) { 2261 ret = EFAULT; 2262 goto label_return; 2263 } 2264 old_extent_hooks = 2265 (extent_hooks_t *)__UNCONST(&extent_hooks_default); 2266 READ(old_extent_hooks, extent_hooks_t *); 2267 if (newp != NULL) { 2268 /* Initialize a new arena as a side effect. */ 2269 extent_hooks_t *new_extent_hooks 2270 JEMALLOC_CC_SILENCE_INIT(NULL); 2271 WRITE(new_extent_hooks, extent_hooks_t *); 2272 arena = arena_init(tsd_tsdn(tsd), arena_ind, 2273 new_extent_hooks); 2274 if (arena == NULL) { 2275 ret = EFAULT; 2276 goto label_return; 2277 } 2278 } 2279 } else { 2280 if (newp != NULL) { 2281 extent_hooks_t *new_extent_hooks 2282 JEMALLOC_CC_SILENCE_INIT(NULL); 2283 WRITE(new_extent_hooks, extent_hooks_t *); 2284 old_extent_hooks = extent_hooks_set(tsd, arena, 2285 new_extent_hooks); 2286 READ(old_extent_hooks, extent_hooks_t *); 2287 } else { 2288 old_extent_hooks = extent_hooks_get(arena); 2289 READ(old_extent_hooks, extent_hooks_t *); 2290 } 2291 } 2292 } else { 2293 ret = EFAULT; 2294 goto label_return; 2295 } 2296 ret = 0; 2297label_return: 2298 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2299 return ret; 2300} 2301 2302static int 2303arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2304 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2305 int ret; 2306 unsigned arena_ind; 2307 arena_t *arena; 2308 2309 if (!opt_retain) { 2310 /* Only relevant when retain is enabled. */ 2311 return ENOENT; 2312 } 2313 2314 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2315 MIB_UNSIGNED(arena_ind, 1); 2316 if (arena_ind < narenas_total_get() && (arena = 2317 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { 2318 size_t old_limit, new_limit; 2319 if (newp != NULL) { 2320 WRITE(new_limit, size_t); 2321 } 2322 bool err = arena_retain_grow_limit_get_set(tsd, arena, 2323 &old_limit, newp != NULL ? &new_limit : NULL); 2324 if (!err) { 2325 READ(old_limit, size_t); 2326 ret = 0; 2327 } else { 2328 ret = EFAULT; 2329 } 2330 } else { 2331 ret = EFAULT; 2332 } 2333label_return: 2334 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2335 return ret; 2336} 2337 2338static const ctl_named_node_t * 2339arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { 2340 const ctl_named_node_t *ret; 2341 2342 malloc_mutex_lock(tsdn, &ctl_mtx); 2343 switch (i) { 2344 case MALLCTL_ARENAS_ALL: 2345 case MALLCTL_ARENAS_DESTROYED: 2346 break; 2347 default: 2348 if (i > ctl_arenas->narenas) { 2349 ret = NULL; 2350 goto label_return; 2351 } 2352 break; 2353 } 2354 2355 ret = super_arena_i_node; 2356label_return: 2357 malloc_mutex_unlock(tsdn, &ctl_mtx); 2358 return ret; 2359} 2360 2361/******************************************************************************/ 2362 2363static int 2364arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2365 size_t *oldlenp, void *newp, size_t newlen) { 2366 int ret; 2367 unsigned narenas; 2368 2369 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2370 READONLY(); 2371 if (*oldlenp != sizeof(unsigned)) { 2372 ret = EINVAL; 2373 goto label_return; 2374 } 2375 narenas = ctl_arenas->narenas; 2376 READ(narenas, unsigned); 2377 2378 ret = 0; 2379label_return: 2380 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2381 return ret; 2382} 2383 2384static int 2385arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, 2386 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { 2387 int ret; 2388 2389 if (oldp != NULL && oldlenp != NULL) { 2390 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() : 2391 arena_muzzy_decay_ms_default_get()); 2392 READ(oldval, ssize_t); 2393 } 2394 if (newp != NULL) { 2395 if (newlen != sizeof(ssize_t)) { 2396 ret = EINVAL; 2397 goto label_return; 2398 } 2399 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) 2400 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { 2401 ret = EFAULT; 2402 goto label_return; 2403 } 2404 } 2405 2406 ret = 0; 2407label_return: 2408 return ret; 2409} 2410 2411static int 2412arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2413 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2414 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2415 newlen, true); 2416} 2417 2418static int 2419arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2420 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2421 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2422 newlen, false); 2423} 2424 2425CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) 2426CTL_RO_NL_GEN(arenas_page, PAGE, size_t) 2427CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) 2428CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) 2429CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) 2430CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) 2431CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) 2432CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) 2433static const ctl_named_node_t * 2434arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { 2435 if (i > NBINS) { 2436 return NULL; 2437 } 2438 return super_arenas_bin_i_node; 2439} 2440 2441CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned) 2442CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(NBINS+(szind_t)mib[2]), 2443 size_t) 2444static const ctl_named_node_t * 2445arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2446 size_t i) { 2447 if (i > NSIZES - NBINS) { 2448 return NULL; 2449 } 2450 return super_arenas_lextent_i_node; 2451} 2452 2453static int 2454arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2455 size_t *oldlenp, void *newp, size_t newlen) { 2456 int ret; 2457 extent_hooks_t *extent_hooks; 2458 unsigned arena_ind; 2459 2460 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2461 2462 extent_hooks = (extent_hooks_t *)__UNCONST(&extent_hooks_default); 2463 WRITE(extent_hooks, extent_hooks_t *); 2464 if ((arena_ind = ctl_arena_init(tsd, extent_hooks)) == UINT_MAX) { 2465 ret = EAGAIN; 2466 goto label_return; 2467 } 2468 READ(arena_ind, unsigned); 2469 2470 ret = 0; 2471label_return: 2472 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2473 return ret; 2474} 2475 2476static int 2477arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2478 size_t *oldlenp, void *newp, size_t newlen) { 2479 int ret; 2480 unsigned arena_ind; 2481 void *ptr; 2482 extent_t *extent; 2483 arena_t *arena; 2484 2485 ptr = NULL; 2486 ret = EINVAL; 2487 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2488 WRITE(ptr, void *); 2489 extent = iealloc(tsd_tsdn(tsd), ptr); 2490 if (extent == NULL) 2491 goto label_return; 2492 2493 arena = extent_arena_get(extent); 2494 if (arena == NULL) 2495 goto label_return; 2496 2497 arena_ind = arena_ind_get(arena); 2498 READ(arena_ind, unsigned); 2499 2500 ret = 0; 2501label_return: 2502 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2503 return ret; 2504} 2505 2506/******************************************************************************/ 2507 2508static int 2509prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2510 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2511 int ret; 2512 bool oldval; 2513 2514 if (!config_prof) { 2515 return ENOENT; 2516 } 2517 2518 if (newp != NULL) { 2519 if (newlen != sizeof(bool)) { 2520 ret = EINVAL; 2521 goto label_return; 2522 } 2523 oldval = prof_thread_active_init_set(tsd_tsdn(tsd), 2524 *(bool *)newp); 2525 } else { 2526 oldval = prof_thread_active_init_get(tsd_tsdn(tsd)); 2527 } 2528 READ(oldval, bool); 2529 2530 ret = 0; 2531label_return: 2532 return ret; 2533} 2534 2535static int 2536prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2537 size_t *oldlenp, void *newp, size_t newlen) { 2538 int ret; 2539 bool oldval; 2540 2541 if (!config_prof) { 2542 return ENOENT; 2543 } 2544 2545 if (newp != NULL) { 2546 if (newlen != sizeof(bool)) { 2547 ret = EINVAL; 2548 goto label_return; 2549 } 2550 oldval = prof_active_set(tsd_tsdn(tsd), *(bool *)newp); 2551 } else { 2552 oldval = prof_active_get(tsd_tsdn(tsd)); 2553 } 2554 READ(oldval, bool); 2555 2556 ret = 0; 2557label_return: 2558 return ret; 2559} 2560 2561static int 2562prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2563 size_t *oldlenp, void *newp, size_t newlen) { 2564 int ret; 2565 const char *filename = NULL; 2566 2567 if (!config_prof) { 2568 return ENOENT; 2569 } 2570 2571 WRITEONLY(); 2572 WRITE(filename, const char *); 2573 2574 if (prof_mdump(tsd, filename)) { 2575 ret = EFAULT; 2576 goto label_return; 2577 } 2578 2579 ret = 0; 2580label_return: 2581 return ret; 2582} 2583 2584static int 2585prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2586 size_t *oldlenp, void *newp, size_t newlen) { 2587 int ret; 2588 bool oldval; 2589 2590 if (!config_prof) { 2591 return ENOENT; 2592 } 2593 2594 if (newp != NULL) { 2595 if (newlen != sizeof(bool)) { 2596 ret = EINVAL; 2597 goto label_return; 2598 } 2599 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp); 2600 } else { 2601 oldval = prof_gdump_get(tsd_tsdn(tsd)); 2602 } 2603 READ(oldval, bool); 2604 2605 ret = 0; 2606label_return: 2607 return ret; 2608} 2609 2610static int 2611prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2612 size_t *oldlenp, void *newp, size_t newlen) { 2613 int ret; 2614 size_t lg_sample = lg_prof_sample; 2615 2616 if (!config_prof) { 2617 return ENOENT; 2618 } 2619 2620 WRITEONLY(); 2621 WRITE(lg_sample, size_t); 2622 if (lg_sample >= (sizeof(uint64_t) << 3)) { 2623 lg_sample = (sizeof(uint64_t) << 3) - 1; 2624 } 2625 2626 prof_reset(tsd, lg_sample); 2627 2628 ret = 0; 2629label_return: 2630 return ret; 2631} 2632 2633CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) 2634CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) 2635 2636/******************************************************************************/ 2637 2638CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) 2639CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t) 2640CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t) 2641CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t) 2642CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t) 2643CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) 2644CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) 2645 2646CTL_RO_CGEN(config_stats, stats_background_thread_num_threads, 2647 ctl_stats->background_thread.num_threads, size_t) 2648CTL_RO_CGEN(config_stats, stats_background_thread_num_runs, 2649 ctl_stats->background_thread.num_runs, uint64_t) 2650CTL_RO_CGEN(config_stats, stats_background_thread_run_interval, 2651 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t) 2652 2653CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) 2654CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms, 2655 ssize_t) 2656CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms, 2657 ssize_t) 2658CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) 2659CTL_RO_GEN(stats_arenas_i_uptime, 2660 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t) 2661CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t) 2662CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t) 2663CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t) 2664CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, 2665 atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED), 2666 size_t) 2667CTL_RO_CGEN(config_stats, stats_arenas_i_retained, 2668 atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), 2669 size_t) 2670 2671CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, 2672 ctl_arena_stats_read_u64( 2673 &arenas_i(mib[2])->astats->astats.decay_dirty.npurge), uint64_t) 2674CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, 2675 ctl_arena_stats_read_u64( 2676 &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t) 2677CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, 2678 ctl_arena_stats_read_u64( 2679 &arenas_i(mib[2])->astats->astats.decay_dirty.purged), uint64_t) 2680 2681CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, 2682 ctl_arena_stats_read_u64( 2683 &arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), uint64_t) 2684CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, 2685 ctl_arena_stats_read_u64( 2686 &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t) 2687CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, 2688 ctl_arena_stats_read_u64( 2689 &arenas_i(mib[2])->astats->astats.decay_muzzy.purged), uint64_t) 2690 2691CTL_RO_CGEN(config_stats, stats_arenas_i_base, 2692 atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED), 2693 size_t) 2694CTL_RO_CGEN(config_stats, stats_arenas_i_internal, 2695 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), 2696 size_t) 2697CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp, 2698 atomic_load_zu(&arenas_i(mib[2])->astats->astats.metadata_thp, 2699 ATOMIC_RELAXED), size_t) 2700CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, 2701 atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes, 2702 ATOMIC_RELAXED), size_t) 2703CTL_RO_CGEN(config_stats, stats_arenas_i_resident, 2704 atomic_load_zu(&arenas_i(mib[2])->astats->astats.resident, ATOMIC_RELAXED), 2705 size_t) 2706 2707CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, 2708 arenas_i(mib[2])->astats->allocated_small, size_t) 2709CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, 2710 arenas_i(mib[2])->astats->nmalloc_small, uint64_t) 2711CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, 2712 arenas_i(mib[2])->astats->ndalloc_small, uint64_t) 2713CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, 2714 arenas_i(mib[2])->astats->nrequests_small, uint64_t) 2715CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, 2716 atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large, 2717 ATOMIC_RELAXED), size_t) 2718CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, 2719 ctl_arena_stats_read_u64( 2720 &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) 2721CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, 2722 ctl_arena_stats_read_u64( 2723 &arenas_i(mib[2])->astats->astats.ndalloc_large), uint64_t) 2724/* 2725 * Note: "nmalloc" here instead of "nrequests" in the read. This is intentional. 2726 */ 2727CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, 2728 ctl_arena_stats_read_u64( 2729 &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) /* Intentional. */ 2730 2731/* Lock profiling related APIs below. */ 2732#define RO_MUTEX_CTL_GEN(n, l) \ 2733CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \ 2734 l.n_lock_ops, uint64_t) \ 2735CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \ 2736 l.n_wait_times, uint64_t) \ 2737CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \ 2738 l.n_spin_acquired, uint64_t) \ 2739CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ 2740 l.n_owner_switches, uint64_t) \ 2741CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ 2742 nstime_ns(&l.tot_wait_time), uint64_t) \ 2743CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ 2744 nstime_ns(&l.max_wait_time), uint64_t) \ 2745CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ 2746 l.max_n_thds, uint32_t) 2747 2748/* Global mutexes. */ 2749#define OP(mtx) \ 2750 RO_MUTEX_CTL_GEN(mutexes_##mtx, \ 2751 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) 2752MUTEX_PROF_GLOBAL_MUTEXES 2753#undef OP 2754 2755/* Per arena mutexes */ 2756#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ 2757 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) 2758MUTEX_PROF_ARENA_MUTEXES 2759#undef OP 2760 2761/* tcache bin mutex */ 2762RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, 2763 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) 2764#undef RO_MUTEX_CTL_GEN 2765 2766/* Resets all mutex stats, including global, arena and bin mutexes. */ 2767static int 2768stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2769 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2770 if (!config_stats) { 2771 return ENOENT; 2772 } 2773 2774 tsdn_t *tsdn = tsd_tsdn(tsd); 2775 2776#define MUTEX_PROF_RESET(mtx) \ 2777 malloc_mutex_lock(tsdn, &mtx); \ 2778 malloc_mutex_prof_data_reset(tsdn, &mtx); \ 2779 malloc_mutex_unlock(tsdn, &mtx); 2780 2781 /* Global mutexes: ctl and prof. */ 2782 MUTEX_PROF_RESET(ctl_mtx); 2783 if (have_background_thread) { 2784 MUTEX_PROF_RESET(background_thread_lock); 2785 } 2786 if (config_prof && opt_prof) { 2787 MUTEX_PROF_RESET(bt2gctx_mtx); 2788 } 2789 2790 2791 /* Per arena mutexes. */ 2792 unsigned n = narenas_total_get(); 2793 2794 for (unsigned i = 0; i < n; i++) { 2795 arena_t *arena = arena_get(tsdn, i, false); 2796 if (!arena) { 2797 continue; 2798 } 2799 MUTEX_PROF_RESET(arena->large_mtx); 2800 MUTEX_PROF_RESET(arena->extent_avail_mtx); 2801 MUTEX_PROF_RESET(arena->extents_dirty.mtx); 2802 MUTEX_PROF_RESET(arena->extents_muzzy.mtx); 2803 MUTEX_PROF_RESET(arena->extents_retained.mtx); 2804 MUTEX_PROF_RESET(arena->decay_dirty.mtx); 2805 MUTEX_PROF_RESET(arena->decay_muzzy.mtx); 2806 MUTEX_PROF_RESET(arena->tcache_ql_mtx); 2807 MUTEX_PROF_RESET(arena->base->mtx); 2808 2809 for (szind_t j = 0; j < NBINS; j++) { 2810 bin_t *bin = &arena->bins[j]; 2811 MUTEX_PROF_RESET(bin->lock); 2812 } 2813 } 2814#undef MUTEX_PROF_RESET 2815 return 0; 2816} 2817 2818CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, 2819 arenas_i(mib[2])->astats->bstats[mib[4]].nmalloc, uint64_t) 2820CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, 2821 arenas_i(mib[2])->astats->bstats[mib[4]].ndalloc, uint64_t) 2822CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, 2823 arenas_i(mib[2])->astats->bstats[mib[4]].nrequests, uint64_t) 2824CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, 2825 arenas_i(mib[2])->astats->bstats[mib[4]].curregs, size_t) 2826CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills, 2827 arenas_i(mib[2])->astats->bstats[mib[4]].nfills, uint64_t) 2828CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes, 2829 arenas_i(mib[2])->astats->bstats[mib[4]].nflushes, uint64_t) 2830CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs, 2831 arenas_i(mib[2])->astats->bstats[mib[4]].nslabs, uint64_t) 2832CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, 2833 arenas_i(mib[2])->astats->bstats[mib[4]].reslabs, uint64_t) 2834CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, 2835 arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) 2836 2837static const ctl_named_node_t * 2838stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2839 size_t j) { 2840 if (j > NBINS) { 2841 return NULL; 2842 } 2843 return super_stats_arenas_i_bins_j_node; 2844} 2845 2846CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc, 2847 ctl_arena_stats_read_u64( 2848 &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t) 2849CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, 2850 ctl_arena_stats_read_u64( 2851 &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t) 2852CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, 2853 ctl_arena_stats_read_u64( 2854 &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) 2855CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, 2856 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) 2857 2858static const ctl_named_node_t * 2859stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2860 size_t j) { 2861 if (j > NSIZES - NBINS) { 2862 return NULL; 2863 } 2864 return super_stats_arenas_i_lextents_j_node; 2865} 2866 2867static const ctl_named_node_t * 2868stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { 2869 const ctl_named_node_t *ret; 2870 size_t a; 2871 2872 malloc_mutex_lock(tsdn, &ctl_mtx); 2873 a = arenas_i2a_impl(i, true, true); 2874 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { 2875 ret = NULL; 2876 goto label_return; 2877 } 2878 2879 ret = super_stats_arenas_i_node; 2880label_return: 2881 malloc_mutex_unlock(tsdn, &ctl_mtx); 2882 return ret; 2883} 2884