1#ifndef JEMALLOC_INTERNAL_ARENA_STATS_H 2#define JEMALLOC_INTERNAL_ARENA_STATS_H 3 4#include "jemalloc/internal/atomic.h" 5#include "jemalloc/internal/mutex.h" 6#include "jemalloc/internal/mutex_prof.h" 7#include "jemalloc/internal/size_classes.h" 8 9/* 10 * In those architectures that support 64-bit atomics, we use atomic updates for 11 * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize 12 * externally. 13 */ 14#ifdef JEMALLOC_ATOMIC_U64 15typedef atomic_u64_t arena_stats_u64_t; 16#else 17/* Must hold the arena stats mutex while reading atomically. */ 18typedef uint64_t arena_stats_u64_t; 19#endif 20 21typedef struct arena_stats_large_s arena_stats_large_t; 22struct arena_stats_large_s { 23 /* 24 * Total number of allocation/deallocation requests served directly by 25 * the arena. 26 */ 27 arena_stats_u64_t nmalloc; 28 arena_stats_u64_t ndalloc; 29 30 /* 31 * Number of allocation requests that correspond to this size class. 32 * This includes requests served by tcache, though tcache only 33 * periodically merges into this counter. 34 */ 35 arena_stats_u64_t nrequests; /* Partially derived. */ 36 37 /* Current number of allocations of this size class. */ 38 size_t curlextents; /* Derived. */ 39}; 40 41typedef struct arena_stats_decay_s arena_stats_decay_t; 42struct arena_stats_decay_s { 43 /* Total number of purge sweeps. */ 44 arena_stats_u64_t npurge; 45 /* Total number of madvise calls made. */ 46 arena_stats_u64_t nmadvise; 47 /* Total number of pages purged. */ 48 arena_stats_u64_t purged; 49}; 50 51/* 52 * Arena stats. Note that fields marked "derived" are not directly maintained 53 * within the arena code; rather their values are derived during stats merge 54 * requests. 55 */ 56typedef struct arena_stats_s arena_stats_t; 57struct arena_stats_s { 58#ifndef JEMALLOC_ATOMIC_U64 59 malloc_mutex_t mtx; 60#endif 61 62 /* Number of bytes currently mapped, excluding retained memory. */ 63 atomic_zu_t mapped; /* Partially derived. */ 64 65 /* 66 * Number of unused virtual memory bytes currently retained. Retained 67 * bytes are technically mapped (though always decommitted or purged), 68 * but they are excluded from the mapped statistic (above). 69 */ 70 atomic_zu_t retained; /* Derived. */ 71 72 arena_stats_decay_t decay_dirty; 73 arena_stats_decay_t decay_muzzy; 74 75 atomic_zu_t base; /* Derived. */ 76 atomic_zu_t internal; 77 atomic_zu_t resident; /* Derived. */ 78 atomic_zu_t metadata_thp; 79 80 atomic_zu_t allocated_large; /* Derived. */ 81 arena_stats_u64_t nmalloc_large; /* Derived. */ 82 arena_stats_u64_t ndalloc_large; /* Derived. */ 83 arena_stats_u64_t nrequests_large; /* Derived. */ 84 85 /* Number of bytes cached in tcache associated with this arena. */ 86 atomic_zu_t tcache_bytes; /* Derived. */ 87 88 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; 89 90 /* One element for each large size class. */ 91 arena_stats_large_t lstats[NSIZES - NBINS]; 92 93 /* Arena uptime. */ 94 nstime_t uptime; 95}; 96 97static inline bool 98arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) { 99 if (config_debug) { 100 for (size_t i = 0; i < sizeof(arena_stats_t); i++) { 101 assert(((char *)arena_stats)[i] == 0); 102 } 103 } 104#ifndef JEMALLOC_ATOMIC_U64 105 if (malloc_mutex_init(&arena_stats->mtx, "arena_stats", 106 WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) { 107 return true; 108 } 109#endif 110 /* Memory is zeroed, so there is no need to clear stats. */ 111 return false; 112} 113 114static inline void 115arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) { 116#ifndef JEMALLOC_ATOMIC_U64 117 malloc_mutex_lock(tsdn, &arena_stats->mtx); 118#endif 119} 120 121static inline void 122arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) { 123#ifndef JEMALLOC_ATOMIC_U64 124 malloc_mutex_unlock(tsdn, &arena_stats->mtx); 125#endif 126} 127 128static inline uint64_t 129arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, 130 arena_stats_u64_t *p) { 131#ifdef JEMALLOC_ATOMIC_U64 132 return atomic_load_u64(p, ATOMIC_RELAXED); 133#else 134 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); 135 return *p; 136#endif 137} 138 139static inline void 140arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, 141 arena_stats_u64_t *p, uint64_t x) { 142#ifdef JEMALLOC_ATOMIC_U64 143 atomic_fetch_add_u64(p, x, ATOMIC_RELAXED); 144#else 145 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); 146 *p += x; 147#endif 148} 149 150UNUSED static inline void 151arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, 152 arena_stats_u64_t *p, uint64_t x) { 153#ifdef JEMALLOC_ATOMIC_U64 154 UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); 155 assert(r - x <= r); 156#else 157 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); 158 *p -= x; 159 assert(*p + x >= *p); 160#endif 161} 162 163/* 164 * Non-atomically sets *dst += src. *dst needs external synchronization. 165 * This lets us avoid the cost of a fetch_add when its unnecessary (note that 166 * the types here are atomic). 167 */ 168static inline void 169arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) { 170#ifdef JEMALLOC_ATOMIC_U64 171 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); 172 atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED); 173#else 174 *dst += src; 175#endif 176} 177 178static inline size_t 179arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { 180#ifdef JEMALLOC_ATOMIC_U64 181 return atomic_load_zu(p, ATOMIC_RELAXED); 182#else 183 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); 184 return atomic_load_zu(p, ATOMIC_RELAXED); 185#endif 186} 187 188static inline void 189arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, 190 size_t x) { 191#ifdef JEMALLOC_ATOMIC_U64 192 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED); 193#else 194 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); 195 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED); 196 atomic_store_zu(p, cur + x, ATOMIC_RELAXED); 197#endif 198} 199 200static inline void 201arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, 202 size_t x) { 203#ifdef JEMALLOC_ATOMIC_U64 204 UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); 205 assert(r - x <= r); 206#else 207 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); 208 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED); 209 atomic_store_zu(p, cur - x, ATOMIC_RELAXED); 210#endif 211} 212 213/* Like the _u64 variant, needs an externally synchronized *dst. */ 214static inline void 215arena_stats_accum_zu(atomic_zu_t *dst, size_t src) { 216 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); 217 atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED); 218} 219 220static inline void 221arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, 222 szind_t szind, uint64_t nrequests) { 223 arena_stats_lock(tsdn, arena_stats); 224 arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind - 225 NBINS].nrequests, nrequests); 226 arena_stats_unlock(tsdn, arena_stats); 227} 228 229static inline void 230arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { 231 arena_stats_lock(tsdn, arena_stats); 232 arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size); 233 arena_stats_unlock(tsdn, arena_stats); 234} 235 236 237#endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */ 238