arena.h revision 235238
1234370Sjasone/******************************************************************************/ 2234370Sjasone#ifdef JEMALLOC_H_TYPES 3234370Sjasone 4234370Sjasone/* 5234370Sjasone * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized 6234370Sjasone * as small as possible such that this setting is still honored, without 7234370Sjasone * violating other constraints. The goal is to make runs as small as possible 8234370Sjasone * without exceeding a per run external fragmentation threshold. 9234370Sjasone * 10234370Sjasone * We use binary fixed point math for overhead computations, where the binary 11234370Sjasone * point is implicitly RUN_BFP bits to the left. 12234370Sjasone * 13234370Sjasone * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be 14234370Sjasone * honored for some/all object sizes, since when heap profiling is enabled 15234370Sjasone * there is one pointer of header overhead per object (plus a constant). This 16234370Sjasone * constraint is relaxed (ignored) for runs that are so small that the 17234370Sjasone * per-region overhead is greater than: 18234370Sjasone * 19234370Sjasone * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) 20234370Sjasone */ 21234370Sjasone#define RUN_BFP 12 22234370Sjasone/* \/ Implicit binary fixed point. */ 23234370Sjasone#define RUN_MAX_OVRHD 0x0000003dU 24234370Sjasone#define RUN_MAX_OVRHD_RELAX 0x00001800U 25234370Sjasone 26234370Sjasone/* Maximum number of regions in one run. */ 27234370Sjasone#define LG_RUN_MAXREGS 11 28234370Sjasone#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 29234370Sjasone 30234370Sjasone/* 31234370Sjasone * Minimum redzone size. Redzones may be larger than this if necessary to 32234370Sjasone * preserve region alignment. 33234370Sjasone */ 34234370Sjasone#define REDZONE_MINSIZE 16 35234370Sjasone 36234370Sjasone/* 37234370Sjasone * The minimum ratio of active:dirty pages per arena is computed as: 38234370Sjasone * 39234370Sjasone * (nactive >> opt_lg_dirty_mult) >= ndirty 40234370Sjasone * 41234370Sjasone * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 42234370Sjasone * times as many active pages as dirty pages. 43234370Sjasone */ 44234370Sjasone#define LG_DIRTY_MULT_DEFAULT 5 45234370Sjasone 46234370Sjasonetypedef struct arena_chunk_map_s arena_chunk_map_t; 47234370Sjasonetypedef struct arena_chunk_s arena_chunk_t; 48234370Sjasonetypedef struct arena_run_s arena_run_t; 49234370Sjasonetypedef struct arena_bin_info_s arena_bin_info_t; 50234370Sjasonetypedef struct arena_bin_s arena_bin_t; 51234370Sjasonetypedef struct arena_s arena_t; 52234370Sjasone 53234370Sjasone#endif /* JEMALLOC_H_TYPES */ 54234370Sjasone/******************************************************************************/ 55234370Sjasone#ifdef JEMALLOC_H_STRUCTS 56234370Sjasone 57234370Sjasone/* Each element of the chunk map corresponds to one page within the chunk. */ 58234370Sjasonestruct arena_chunk_map_s { 59234370Sjasone#ifndef JEMALLOC_PROF 60234370Sjasone /* 61234370Sjasone * Overlay prof_ctx in order to allow it to be referenced by dead code. 62234370Sjasone * Such antics aren't warranted for per arena data structures, but 63234370Sjasone * chunk map overhead accounts for a percentage of memory, rather than 64234370Sjasone * being just a fixed cost. 65234370Sjasone */ 66234370Sjasone union { 67234370Sjasone#endif 68234370Sjasone union { 69234370Sjasone /* 70234370Sjasone * Linkage for run trees. There are two disjoint uses: 71234370Sjasone * 72234370Sjasone * 1) arena_t's runs_avail_{clean,dirty} trees. 73234370Sjasone * 2) arena_run_t conceptually uses this linkage for in-use 74234370Sjasone * non-full runs, rather than directly embedding linkage. 75234370Sjasone */ 76234370Sjasone rb_node(arena_chunk_map_t) rb_link; 77234370Sjasone /* 78234370Sjasone * List of runs currently in purgatory. arena_chunk_purge() 79234370Sjasone * temporarily allocates runs that contain dirty pages while 80234370Sjasone * purging, so that other threads cannot use the runs while the 81234370Sjasone * purging thread is operating without the arena lock held. 82234370Sjasone */ 83234370Sjasone ql_elm(arena_chunk_map_t) ql_link; 84234370Sjasone } u; 85234370Sjasone 86234370Sjasone /* Profile counters, used for large object runs. */ 87234370Sjasone prof_ctx_t *prof_ctx; 88234370Sjasone#ifndef JEMALLOC_PROF 89234370Sjasone }; /* union { ... }; */ 90234370Sjasone#endif 91234370Sjasone 92234370Sjasone /* 93234370Sjasone * Run address (or size) and various flags are stored together. The bit 94234370Sjasone * layout looks like (assuming 32-bit system): 95234370Sjasone * 96234370Sjasone * ???????? ???????? ????---- ----dula 97234370Sjasone * 98234370Sjasone * ? : Unallocated: Run address for first/last pages, unset for internal 99234370Sjasone * pages. 100234370Sjasone * Small: Run page offset. 101234370Sjasone * Large: Run size for first page, unset for trailing pages. 102234370Sjasone * - : Unused. 103234370Sjasone * d : dirty? 104234370Sjasone * u : unzeroed? 105234370Sjasone * l : large? 106234370Sjasone * a : allocated? 107234370Sjasone * 108234370Sjasone * Following are example bit patterns for the three types of runs. 109234370Sjasone * 110234370Sjasone * p : run page offset 111234370Sjasone * s : run size 112235238Sjasone * n : binind for size class; large objects set these to BININD_INVALID 113235238Sjasone * except for promoted allocations (see prof_promote) 114234370Sjasone * x : don't care 115234370Sjasone * - : 0 116234370Sjasone * + : 1 117234370Sjasone * [DULA] : bit set 118234370Sjasone * [dula] : bit unset 119234370Sjasone * 120234370Sjasone * Unallocated (clean): 121235238Sjasone * ssssssss ssssssss ssss1111 1111du-a 122235238Sjasone * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx 123235238Sjasone * ssssssss ssssssss ssss1111 1111dU-a 124234370Sjasone * 125234370Sjasone * Unallocated (dirty): 126235238Sjasone * ssssssss ssssssss ssss1111 1111D--a 127235238Sjasone * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 128235238Sjasone * ssssssss ssssssss ssss1111 1111D--a 129234370Sjasone * 130234370Sjasone * Small: 131235238Sjasone * pppppppp pppppppp ppppnnnn nnnnd--A 132235238Sjasone * pppppppp pppppppp ppppnnnn nnnn---A 133235238Sjasone * pppppppp pppppppp ppppnnnn nnnnd--A 134234370Sjasone * 135234370Sjasone * Large: 136235238Sjasone * ssssssss ssssssss ssss1111 1111D-LA 137235238Sjasone * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 138235238Sjasone * -------- -------- ----1111 1111D-LA 139234370Sjasone * 140234370Sjasone * Large (sampled, size <= PAGE): 141235238Sjasone * ssssssss ssssssss ssssnnnn nnnnD-LA 142234370Sjasone * 143234370Sjasone * Large (not sampled, size == PAGE): 144235238Sjasone * ssssssss ssssssss ssss1111 1111D-LA 145234370Sjasone */ 146234370Sjasone size_t bits; 147235238Sjasone#define CHUNK_MAP_BININD_SHIFT 4 148235238Sjasone#define BININD_INVALID ((size_t)0xffU) 149235238Sjasone/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ 150235238Sjasone#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) 151235238Sjasone#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 152235238Sjasone#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) 153234370Sjasone#define CHUNK_MAP_DIRTY ((size_t)0x8U) 154234370Sjasone#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 155234370Sjasone#define CHUNK_MAP_LARGE ((size_t)0x2U) 156234370Sjasone#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 157234370Sjasone#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 158234370Sjasone}; 159234370Sjasonetypedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; 160234370Sjasonetypedef rb_tree(arena_chunk_map_t) arena_run_tree_t; 161234370Sjasone 162234370Sjasone/* Arena chunk header. */ 163234370Sjasonestruct arena_chunk_s { 164234370Sjasone /* Arena that owns the chunk. */ 165234370Sjasone arena_t *arena; 166234370Sjasone 167234370Sjasone /* Linkage for the arena's chunks_dirty list. */ 168234370Sjasone ql_elm(arena_chunk_t) link_dirty; 169234370Sjasone 170234370Sjasone /* 171234370Sjasone * True if the chunk is currently in the chunks_dirty list, due to 172234370Sjasone * having at some point contained one or more dirty pages. Removal 173234370Sjasone * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. 174234370Sjasone */ 175234370Sjasone bool dirtied; 176234370Sjasone 177234370Sjasone /* Number of dirty pages. */ 178234370Sjasone size_t ndirty; 179234370Sjasone 180234370Sjasone /* 181234370Sjasone * Map of pages within chunk that keeps track of free/large/small. The 182234370Sjasone * first map_bias entries are omitted, since the chunk header does not 183234370Sjasone * need to be tracked in the map. This omission saves a header page 184234370Sjasone * for common chunk sizes (e.g. 4 MiB). 185234370Sjasone */ 186234370Sjasone arena_chunk_map_t map[1]; /* Dynamically sized. */ 187234370Sjasone}; 188234370Sjasonetypedef rb_tree(arena_chunk_t) arena_chunk_tree_t; 189234370Sjasone 190234370Sjasonestruct arena_run_s { 191234370Sjasone /* Bin this run is associated with. */ 192234370Sjasone arena_bin_t *bin; 193234370Sjasone 194234370Sjasone /* Index of next region that has never been allocated, or nregs. */ 195234370Sjasone uint32_t nextind; 196234370Sjasone 197234370Sjasone /* Number of free regions in run. */ 198234370Sjasone unsigned nfree; 199234370Sjasone}; 200234370Sjasone 201234370Sjasone/* 202234370Sjasone * Read-only information associated with each element of arena_t's bins array 203234370Sjasone * is stored separately, partly to reduce memory usage (only one copy, rather 204234370Sjasone * than one per arena), but mainly to avoid false cacheline sharing. 205234370Sjasone * 206234370Sjasone * Each run has the following layout: 207234370Sjasone * 208234370Sjasone * /--------------------\ 209234370Sjasone * | arena_run_t header | 210234370Sjasone * | ... | 211234370Sjasone * bitmap_offset | bitmap | 212234370Sjasone * | ... | 213234370Sjasone * ctx0_offset | ctx map | 214234370Sjasone * | ... | 215234370Sjasone * |--------------------| 216234370Sjasone * | redzone | 217234370Sjasone * reg0_offset | region 0 | 218234370Sjasone * | redzone | 219234370Sjasone * |--------------------| \ 220234370Sjasone * | redzone | | 221234370Sjasone * | region 1 | > reg_interval 222234370Sjasone * | redzone | / 223234370Sjasone * |--------------------| 224234370Sjasone * | ... | 225234370Sjasone * | ... | 226234370Sjasone * | ... | 227234370Sjasone * |--------------------| 228234370Sjasone * | redzone | 229234370Sjasone * | region nregs-1 | 230234370Sjasone * | redzone | 231234370Sjasone * |--------------------| 232234370Sjasone * | alignment pad? | 233234370Sjasone * \--------------------/ 234234370Sjasone * 235234370Sjasone * reg_interval has at least the same minimum alignment as reg_size; this 236234370Sjasone * preserves the alignment constraint that sa2u() depends on. Alignment pad is 237234370Sjasone * either 0 or redzone_size; it is present only if needed to align reg0_offset. 238234370Sjasone */ 239234370Sjasonestruct arena_bin_info_s { 240234370Sjasone /* Size of regions in a run for this bin's size class. */ 241234370Sjasone size_t reg_size; 242234370Sjasone 243234370Sjasone /* Redzone size. */ 244234370Sjasone size_t redzone_size; 245234370Sjasone 246234370Sjasone /* Interval between regions (reg_size + (redzone_size << 1)). */ 247234370Sjasone size_t reg_interval; 248234370Sjasone 249234370Sjasone /* Total size of a run for this bin's size class. */ 250234370Sjasone size_t run_size; 251234370Sjasone 252234370Sjasone /* Total number of regions in a run for this bin's size class. */ 253234370Sjasone uint32_t nregs; 254234370Sjasone 255234370Sjasone /* 256234370Sjasone * Offset of first bitmap_t element in a run header for this bin's size 257234370Sjasone * class. 258234370Sjasone */ 259234370Sjasone uint32_t bitmap_offset; 260234370Sjasone 261234370Sjasone /* 262234370Sjasone * Metadata used to manipulate bitmaps for runs associated with this 263234370Sjasone * bin. 264234370Sjasone */ 265234370Sjasone bitmap_info_t bitmap_info; 266234370Sjasone 267234370Sjasone /* 268234370Sjasone * Offset of first (prof_ctx_t *) in a run header for this bin's size 269234370Sjasone * class, or 0 if (config_prof == false || opt_prof == false). 270234370Sjasone */ 271234370Sjasone uint32_t ctx0_offset; 272234370Sjasone 273234370Sjasone /* Offset of first region in a run for this bin's size class. */ 274234370Sjasone uint32_t reg0_offset; 275234370Sjasone}; 276234370Sjasone 277234370Sjasonestruct arena_bin_s { 278234370Sjasone /* 279234370Sjasone * All operations on runcur, runs, and stats require that lock be 280234370Sjasone * locked. Run allocation/deallocation are protected by the arena lock, 281234370Sjasone * which may be acquired while holding one or more bin locks, but not 282234370Sjasone * vise versa. 283234370Sjasone */ 284234370Sjasone malloc_mutex_t lock; 285234370Sjasone 286234370Sjasone /* 287234370Sjasone * Current run being used to service allocations of this bin's size 288234370Sjasone * class. 289234370Sjasone */ 290234370Sjasone arena_run_t *runcur; 291234370Sjasone 292234370Sjasone /* 293234370Sjasone * Tree of non-full runs. This tree is used when looking for an 294234370Sjasone * existing run when runcur is no longer usable. We choose the 295234370Sjasone * non-full run that is lowest in memory; this policy tends to keep 296234370Sjasone * objects packed well, and it can also help reduce the number of 297234370Sjasone * almost-empty chunks. 298234370Sjasone */ 299234370Sjasone arena_run_tree_t runs; 300234370Sjasone 301234370Sjasone /* Bin statistics. */ 302234370Sjasone malloc_bin_stats_t stats; 303234370Sjasone}; 304234370Sjasone 305234370Sjasonestruct arena_s { 306234370Sjasone /* This arena's index within the arenas array. */ 307234370Sjasone unsigned ind; 308234370Sjasone 309234370Sjasone /* 310234370Sjasone * Number of threads currently assigned to this arena. This field is 311234370Sjasone * protected by arenas_lock. 312234370Sjasone */ 313234370Sjasone unsigned nthreads; 314234370Sjasone 315234370Sjasone /* 316234370Sjasone * There are three classes of arena operations from a locking 317234370Sjasone * perspective: 318234370Sjasone * 1) Thread asssignment (modifies nthreads) is protected by 319234370Sjasone * arenas_lock. 320234370Sjasone * 2) Bin-related operations are protected by bin locks. 321234370Sjasone * 3) Chunk- and run-related operations are protected by this mutex. 322234370Sjasone */ 323234370Sjasone malloc_mutex_t lock; 324234370Sjasone 325234370Sjasone arena_stats_t stats; 326234370Sjasone /* 327234370Sjasone * List of tcaches for extant threads associated with this arena. 328234370Sjasone * Stats from these are merged incrementally, and at exit. 329234370Sjasone */ 330234370Sjasone ql_head(tcache_t) tcache_ql; 331234370Sjasone 332234370Sjasone uint64_t prof_accumbytes; 333234370Sjasone 334234370Sjasone /* List of dirty-page-containing chunks this arena manages. */ 335234370Sjasone ql_head(arena_chunk_t) chunks_dirty; 336234370Sjasone 337234370Sjasone /* 338234370Sjasone * In order to avoid rapid chunk allocation/deallocation when an arena 339234370Sjasone * oscillates right on the cusp of needing a new chunk, cache the most 340234370Sjasone * recently freed chunk. The spare is left in the arena's chunk trees 341234370Sjasone * until it is deleted. 342234370Sjasone * 343234370Sjasone * There is one spare chunk per arena, rather than one spare total, in 344234370Sjasone * order to avoid interactions between multiple threads that could make 345234370Sjasone * a single spare inadequate. 346234370Sjasone */ 347234370Sjasone arena_chunk_t *spare; 348234370Sjasone 349234370Sjasone /* Number of pages in active runs. */ 350234370Sjasone size_t nactive; 351234370Sjasone 352234370Sjasone /* 353234370Sjasone * Current count of pages within unused runs that are potentially 354234370Sjasone * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 355234370Sjasone * By tracking this, we can institute a limit on how much dirty unused 356234370Sjasone * memory is mapped for each arena. 357234370Sjasone */ 358234370Sjasone size_t ndirty; 359234370Sjasone 360234370Sjasone /* 361234370Sjasone * Approximate number of pages being purged. It is possible for 362234370Sjasone * multiple threads to purge dirty pages concurrently, and they use 363234370Sjasone * npurgatory to indicate the total number of pages all threads are 364234370Sjasone * attempting to purge. 365234370Sjasone */ 366234370Sjasone size_t npurgatory; 367234370Sjasone 368234370Sjasone /* 369234370Sjasone * Size/address-ordered trees of this arena's available runs. The trees 370234370Sjasone * are used for first-best-fit run allocation. The dirty tree contains 371234370Sjasone * runs with dirty pages (i.e. very likely to have been touched and 372234370Sjasone * therefore have associated physical pages), whereas the clean tree 373234370Sjasone * contains runs with pages that either have no associated physical 374234370Sjasone * pages, or have pages that the kernel may recycle at any time due to 375234370Sjasone * previous madvise(2) calls. The dirty tree is used in preference to 376234370Sjasone * the clean tree for allocations, because using dirty pages reduces 377234370Sjasone * the amount of dirty purging necessary to keep the active:dirty page 378234370Sjasone * ratio below the purge threshold. 379234370Sjasone */ 380234370Sjasone arena_avail_tree_t runs_avail_clean; 381234370Sjasone arena_avail_tree_t runs_avail_dirty; 382234370Sjasone 383234370Sjasone /* bins is used to store trees of free regions. */ 384234370Sjasone arena_bin_t bins[NBINS]; 385234370Sjasone}; 386234370Sjasone 387234370Sjasone#endif /* JEMALLOC_H_STRUCTS */ 388234370Sjasone/******************************************************************************/ 389234370Sjasone#ifdef JEMALLOC_H_EXTERNS 390234370Sjasone 391234370Sjasoneextern ssize_t opt_lg_dirty_mult; 392234370Sjasone/* 393234370Sjasone * small_size2bin is a compact lookup table that rounds request sizes up to 394234370Sjasone * size classes. In order to reduce cache footprint, the table is compressed, 395234370Sjasone * and all accesses are via the SMALL_SIZE2BIN macro. 396234370Sjasone */ 397234370Sjasoneextern uint8_t const small_size2bin[]; 398234370Sjasone#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) 399234370Sjasone 400234370Sjasoneextern arena_bin_info_t arena_bin_info[NBINS]; 401234370Sjasone 402234370Sjasone/* Number of large size classes. */ 403234370Sjasone#define nlclasses (chunk_npages - map_bias) 404234370Sjasone 405234370Sjasonevoid arena_purge_all(arena_t *arena); 406234370Sjasonevoid arena_prof_accum(arena_t *arena, uint64_t accumbytes); 407234370Sjasonevoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 408234370Sjasone size_t binind, uint64_t prof_accumbytes); 409234370Sjasonevoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 410234370Sjasone bool zero); 411234370Sjasonevoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 412234370Sjasonevoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 413234370Sjasonevoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 414234370Sjasonevoid *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); 415234370Sjasonevoid arena_prof_promoted(const void *ptr, size_t size); 416235238Sjasonevoid arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 417235238Sjasone arena_chunk_map_t *mapelm); 418234370Sjasonevoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 419235238Sjasone size_t pageind, arena_chunk_map_t *mapelm); 420235238Sjasonevoid arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 421235238Sjasone size_t pageind); 422235238Sjasonevoid arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, 423235238Sjasone void *ptr); 424234370Sjasonevoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 425234370Sjasonevoid arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 426234370Sjasone arena_stats_t *astats, malloc_bin_stats_t *bstats, 427234370Sjasone malloc_large_stats_t *lstats); 428234370Sjasonevoid *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 429234370Sjasone size_t extra, bool zero); 430234370Sjasonevoid *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, 431234370Sjasone size_t alignment, bool zero, bool try_tcache); 432234370Sjasonebool arena_new(arena_t *arena, unsigned ind); 433234370Sjasonevoid arena_boot(void); 434234370Sjasonevoid arena_prefork(arena_t *arena); 435234370Sjasonevoid arena_postfork_parent(arena_t *arena); 436234370Sjasonevoid arena_postfork_child(arena_t *arena); 437234370Sjasone 438234370Sjasone#endif /* JEMALLOC_H_EXTERNS */ 439234370Sjasone/******************************************************************************/ 440234370Sjasone#ifdef JEMALLOC_H_INLINES 441234370Sjasone 442234370Sjasone#ifndef JEMALLOC_ENABLE_INLINE 443235238Sjasonearena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); 444235238Sjasonesize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 445235238Sjasonesize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 446235238Sjasonesize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 447235238Sjasone size_t pageind); 448235238Sjasonesize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 449235238Sjasonesize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 450235238Sjasonesize_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 451235238Sjasonesize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 452235238Sjasonesize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 453235238Sjasonesize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 454235238Sjasonesize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 455235238Sjasonevoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 456235238Sjasone size_t size, size_t flags); 457235238Sjasonevoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 458235238Sjasone size_t size); 459235238Sjasonevoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 460235238Sjasone size_t size, size_t flags); 461235238Sjasonevoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 462235238Sjasone size_t binind); 463235238Sjasonevoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 464235238Sjasone size_t runind, size_t binind, size_t flags); 465235238Sjasonevoid arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 466235238Sjasone size_t unzeroed); 467235238Sjasonesize_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 468234370Sjasonesize_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 469234370Sjasoneunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 470234370Sjasone const void *ptr); 471234370Sjasoneprof_ctx_t *arena_prof_ctx_get(const void *ptr); 472234370Sjasonevoid arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 473234370Sjasonevoid *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); 474234543Sjasonesize_t arena_salloc(const void *ptr, bool demote); 475234370Sjasonevoid arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, 476234370Sjasone bool try_tcache); 477234370Sjasone#endif 478234370Sjasone 479234370Sjasone#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 480235238Sjasone# ifdef JEMALLOC_ARENA_INLINE_A 481235238SjasoneJEMALLOC_INLINE arena_chunk_map_t * 482235238Sjasonearena_mapp_get(arena_chunk_t *chunk, size_t pageind) 483235238Sjasone{ 484235238Sjasone 485235238Sjasone assert(pageind >= map_bias); 486235238Sjasone assert(pageind < chunk_npages); 487235238Sjasone 488235238Sjasone return (&chunk->map[pageind-map_bias]); 489235238Sjasone} 490235238Sjasone 491235238SjasoneJEMALLOC_INLINE size_t * 492235238Sjasonearena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 493235238Sjasone{ 494235238Sjasone 495235238Sjasone return (&arena_mapp_get(chunk, pageind)->bits); 496235238Sjasone} 497235238Sjasone 498234370SjasoneJEMALLOC_INLINE size_t 499235238Sjasonearena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 500235238Sjasone{ 501235238Sjasone 502235238Sjasone return (*arena_mapbitsp_get(chunk, pageind)); 503235238Sjasone} 504235238Sjasone 505235238SjasoneJEMALLOC_INLINE size_t 506235238Sjasonearena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 507235238Sjasone{ 508235238Sjasone size_t mapbits; 509235238Sjasone 510235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 511235238Sjasone assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 512235238Sjasone return (mapbits & ~PAGE_MASK); 513235238Sjasone} 514235238Sjasone 515235238SjasoneJEMALLOC_INLINE size_t 516235238Sjasonearena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 517235238Sjasone{ 518235238Sjasone size_t mapbits; 519235238Sjasone 520235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 521235238Sjasone assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 522235238Sjasone (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 523235238Sjasone return (mapbits & ~PAGE_MASK); 524235238Sjasone} 525235238Sjasone 526235238SjasoneJEMALLOC_INLINE size_t 527235238Sjasonearena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 528235238Sjasone{ 529235238Sjasone size_t mapbits; 530235238Sjasone 531235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 532235238Sjasone assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 533235238Sjasone CHUNK_MAP_ALLOCATED); 534235238Sjasone return (mapbits >> LG_PAGE); 535235238Sjasone} 536235238Sjasone 537235238SjasoneJEMALLOC_INLINE size_t 538235238Sjasonearena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 539235238Sjasone{ 540235238Sjasone size_t mapbits; 541235238Sjasone size_t binind; 542235238Sjasone 543235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 544235238Sjasone binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 545235238Sjasone assert(binind < NBINS || binind == BININD_INVALID); 546235238Sjasone return (binind); 547235238Sjasone} 548235238Sjasone 549235238SjasoneJEMALLOC_INLINE size_t 550235238Sjasonearena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 551235238Sjasone{ 552235238Sjasone size_t mapbits; 553235238Sjasone 554235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 555235238Sjasone return (mapbits & CHUNK_MAP_DIRTY); 556235238Sjasone} 557235238Sjasone 558235238SjasoneJEMALLOC_INLINE size_t 559235238Sjasonearena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 560235238Sjasone{ 561235238Sjasone size_t mapbits; 562235238Sjasone 563235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 564235238Sjasone return (mapbits & CHUNK_MAP_UNZEROED); 565235238Sjasone} 566235238Sjasone 567235238SjasoneJEMALLOC_INLINE size_t 568235238Sjasonearena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 569235238Sjasone{ 570235238Sjasone size_t mapbits; 571235238Sjasone 572235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 573235238Sjasone return (mapbits & CHUNK_MAP_LARGE); 574235238Sjasone} 575235238Sjasone 576235238SjasoneJEMALLOC_INLINE size_t 577235238Sjasonearena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 578235238Sjasone{ 579235238Sjasone size_t mapbits; 580235238Sjasone 581235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 582235238Sjasone return (mapbits & CHUNK_MAP_ALLOCATED); 583235238Sjasone} 584235238Sjasone 585235238SjasoneJEMALLOC_INLINE void 586235238Sjasonearena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 587235238Sjasone size_t flags) 588235238Sjasone{ 589235238Sjasone size_t *mapbitsp; 590235238Sjasone 591235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 592235238Sjasone assert((size & PAGE_MASK) == 0); 593235238Sjasone assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 594235238Sjasone *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags; 595235238Sjasone} 596235238Sjasone 597235238SjasoneJEMALLOC_INLINE void 598235238Sjasonearena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 599235238Sjasone size_t size) 600235238Sjasone{ 601235238Sjasone size_t *mapbitsp; 602235238Sjasone 603235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 604235238Sjasone assert((size & PAGE_MASK) == 0); 605235238Sjasone assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 606235238Sjasone *mapbitsp = size | (*mapbitsp & PAGE_MASK); 607235238Sjasone} 608235238Sjasone 609235238SjasoneJEMALLOC_INLINE void 610235238Sjasonearena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 611235238Sjasone size_t flags) 612235238Sjasone{ 613235238Sjasone size_t *mapbitsp; 614235238Sjasone 615235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 616235238Sjasone assert((size & PAGE_MASK) == 0); 617235238Sjasone assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 618235238Sjasone *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | 619235238Sjasone CHUNK_MAP_ALLOCATED; 620235238Sjasone} 621235238Sjasone 622235238SjasoneJEMALLOC_INLINE void 623235238Sjasonearena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 624235238Sjasone size_t binind) 625235238Sjasone{ 626235238Sjasone size_t *mapbitsp; 627235238Sjasone 628235238Sjasone assert(binind <= BININD_INVALID); 629235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 630235238Sjasone assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); 631235238Sjasone *mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind << 632235238Sjasone CHUNK_MAP_BININD_SHIFT); 633235238Sjasone} 634235238Sjasone 635235238SjasoneJEMALLOC_INLINE void 636235238Sjasonearena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 637235238Sjasone size_t binind, size_t flags) 638235238Sjasone{ 639235238Sjasone size_t *mapbitsp; 640235238Sjasone 641235238Sjasone assert(binind < BININD_INVALID); 642235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 643235238Sjasone assert(pageind - runind >= map_bias); 644235238Sjasone assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 645235238Sjasone *mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) | 646235238Sjasone flags | CHUNK_MAP_ALLOCATED; 647235238Sjasone} 648235238Sjasone 649235238SjasoneJEMALLOC_INLINE void 650235238Sjasonearena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 651235238Sjasone size_t unzeroed) 652235238Sjasone{ 653235238Sjasone size_t *mapbitsp; 654235238Sjasone 655235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 656235238Sjasone *mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed; 657235238Sjasone} 658235238Sjasone 659235238SjasoneJEMALLOC_INLINE size_t 660235238Sjasonearena_ptr_small_binind_get(const void *ptr, size_t mapbits) 661235238Sjasone{ 662235238Sjasone size_t binind; 663235238Sjasone 664235238Sjasone binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 665235238Sjasone 666235238Sjasone if (config_debug) { 667235238Sjasone arena_chunk_t *chunk; 668235238Sjasone arena_t *arena; 669235238Sjasone size_t pageind; 670235238Sjasone size_t actual_mapbits; 671235238Sjasone arena_run_t *run; 672235238Sjasone arena_bin_t *bin; 673235238Sjasone size_t actual_binind; 674235238Sjasone arena_bin_info_t *bin_info; 675235238Sjasone 676235238Sjasone assert(binind != BININD_INVALID); 677235238Sjasone assert(binind < NBINS); 678235238Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 679235238Sjasone arena = chunk->arena; 680235238Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 681235238Sjasone actual_mapbits = arena_mapbits_get(chunk, pageind); 682235238Sjasone assert(mapbits == actual_mapbits); 683235238Sjasone assert(arena_mapbits_large_get(chunk, pageind) == 0); 684235238Sjasone assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 685235238Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 686235238Sjasone (actual_mapbits >> LG_PAGE)) << LG_PAGE)); 687235238Sjasone bin = run->bin; 688235238Sjasone actual_binind = bin - arena->bins; 689235238Sjasone assert(binind == actual_binind); 690235238Sjasone bin_info = &arena_bin_info[actual_binind]; 691235238Sjasone assert(((uintptr_t)ptr - ((uintptr_t)run + 692235238Sjasone (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 693235238Sjasone == 0); 694235238Sjasone } 695235238Sjasone 696235238Sjasone return (binind); 697235238Sjasone} 698235238Sjasone# endif /* JEMALLOC_ARENA_INLINE_A */ 699235238Sjasone 700235238Sjasone# ifdef JEMALLOC_ARENA_INLINE_B 701235238SjasoneJEMALLOC_INLINE size_t 702234370Sjasonearena_bin_index(arena_t *arena, arena_bin_t *bin) 703234370Sjasone{ 704234370Sjasone size_t binind = bin - arena->bins; 705234370Sjasone assert(binind < NBINS); 706234370Sjasone return (binind); 707234370Sjasone} 708234370Sjasone 709234370SjasoneJEMALLOC_INLINE unsigned 710234370Sjasonearena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 711234370Sjasone{ 712234370Sjasone unsigned shift, diff, regind; 713234370Sjasone size_t interval; 714234370Sjasone 715234370Sjasone /* 716234370Sjasone * Freeing a pointer lower than region zero can cause assertion 717234370Sjasone * failure. 718234370Sjasone */ 719234370Sjasone assert((uintptr_t)ptr >= (uintptr_t)run + 720234370Sjasone (uintptr_t)bin_info->reg0_offset); 721234370Sjasone 722234370Sjasone /* 723234370Sjasone * Avoid doing division with a variable divisor if possible. Using 724234370Sjasone * actual division here can reduce allocator throughput by over 20%! 725234370Sjasone */ 726234370Sjasone diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - 727234370Sjasone bin_info->reg0_offset); 728234370Sjasone 729234370Sjasone /* Rescale (factor powers of 2 out of the numerator and denominator). */ 730234370Sjasone interval = bin_info->reg_interval; 731234370Sjasone shift = ffs(interval) - 1; 732234370Sjasone diff >>= shift; 733234370Sjasone interval >>= shift; 734234370Sjasone 735234370Sjasone if (interval == 1) { 736234370Sjasone /* The divisor was a power of 2. */ 737234370Sjasone regind = diff; 738234370Sjasone } else { 739234370Sjasone /* 740234370Sjasone * To divide by a number D that is not a power of two we 741234370Sjasone * multiply by (2^21 / D) and then right shift by 21 positions. 742234370Sjasone * 743234370Sjasone * X / D 744234370Sjasone * 745234370Sjasone * becomes 746234370Sjasone * 747234370Sjasone * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 748234370Sjasone * 749234370Sjasone * We can omit the first three elements, because we never 750234370Sjasone * divide by 0, and 1 and 2 are both powers of two, which are 751234370Sjasone * handled above. 752234370Sjasone */ 753234370Sjasone#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 754234370Sjasone#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 755234370Sjasone static const unsigned interval_invs[] = { 756234370Sjasone SIZE_INV(3), 757234370Sjasone SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 758234370Sjasone SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 759234370Sjasone SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 760234370Sjasone SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 761234370Sjasone SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 762234370Sjasone SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 763234370Sjasone SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 764234370Sjasone }; 765234370Sjasone 766234370Sjasone if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + 767234370Sjasone 2)) { 768234370Sjasone regind = (diff * interval_invs[interval - 3]) >> 769234370Sjasone SIZE_INV_SHIFT; 770234370Sjasone } else 771234370Sjasone regind = diff / interval; 772234370Sjasone#undef SIZE_INV 773234370Sjasone#undef SIZE_INV_SHIFT 774234370Sjasone } 775234370Sjasone assert(diff == regind * interval); 776234370Sjasone assert(regind < bin_info->nregs); 777234370Sjasone 778234370Sjasone return (regind); 779234370Sjasone} 780234370Sjasone 781234370SjasoneJEMALLOC_INLINE prof_ctx_t * 782234370Sjasonearena_prof_ctx_get(const void *ptr) 783234370Sjasone{ 784234370Sjasone prof_ctx_t *ret; 785234370Sjasone arena_chunk_t *chunk; 786234370Sjasone size_t pageind, mapbits; 787234370Sjasone 788234370Sjasone cassert(config_prof); 789234370Sjasone assert(ptr != NULL); 790234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 791234370Sjasone 792234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 793234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 794235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 795234370Sjasone assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 796234370Sjasone if ((mapbits & CHUNK_MAP_LARGE) == 0) { 797234370Sjasone if (prof_promote) 798234370Sjasone ret = (prof_ctx_t *)(uintptr_t)1U; 799234370Sjasone else { 800234370Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 801234370Sjasone (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << 802234370Sjasone LG_PAGE)); 803235238Sjasone size_t binind = arena_ptr_small_binind_get(ptr, 804235238Sjasone mapbits); 805234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 806234370Sjasone unsigned regind; 807234370Sjasone 808234370Sjasone regind = arena_run_regind(run, bin_info, ptr); 809234370Sjasone ret = *(prof_ctx_t **)((uintptr_t)run + 810234370Sjasone bin_info->ctx0_offset + (regind * 811234370Sjasone sizeof(prof_ctx_t *))); 812234370Sjasone } 813234370Sjasone } else 814235238Sjasone ret = arena_mapp_get(chunk, pageind)->prof_ctx; 815234370Sjasone 816234370Sjasone return (ret); 817234370Sjasone} 818234370Sjasone 819234370SjasoneJEMALLOC_INLINE void 820234370Sjasonearena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) 821234370Sjasone{ 822234370Sjasone arena_chunk_t *chunk; 823234370Sjasone size_t pageind, mapbits; 824234370Sjasone 825234370Sjasone cassert(config_prof); 826234370Sjasone assert(ptr != NULL); 827234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 828234370Sjasone 829234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 830234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 831235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 832234370Sjasone assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 833234370Sjasone if ((mapbits & CHUNK_MAP_LARGE) == 0) { 834234370Sjasone if (prof_promote == false) { 835234370Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 836234370Sjasone (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << 837234370Sjasone LG_PAGE)); 838234370Sjasone size_t binind; 839234370Sjasone arena_bin_info_t *bin_info; 840234370Sjasone unsigned regind; 841234370Sjasone 842235238Sjasone binind = arena_ptr_small_binind_get(ptr, mapbits); 843234370Sjasone bin_info = &arena_bin_info[binind]; 844234370Sjasone regind = arena_run_regind(run, bin_info, ptr); 845234370Sjasone 846234370Sjasone *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset 847234370Sjasone + (regind * sizeof(prof_ctx_t *)))) = ctx; 848234370Sjasone } else 849234370Sjasone assert((uintptr_t)ctx == (uintptr_t)1U); 850234370Sjasone } else 851235238Sjasone arena_mapp_get(chunk, pageind)->prof_ctx = ctx; 852234370Sjasone} 853234370Sjasone 854234370SjasoneJEMALLOC_INLINE void * 855234370Sjasonearena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) 856234370Sjasone{ 857234370Sjasone tcache_t *tcache; 858234370Sjasone 859234370Sjasone assert(size != 0); 860234370Sjasone assert(size <= arena_maxclass); 861234370Sjasone 862234370Sjasone if (size <= SMALL_MAXCLASS) { 863234370Sjasone if (try_tcache && (tcache = tcache_get(true)) != NULL) 864234370Sjasone return (tcache_alloc_small(tcache, size, zero)); 865234370Sjasone else { 866234370Sjasone return (arena_malloc_small(choose_arena(arena), size, 867234370Sjasone zero)); 868234370Sjasone } 869234370Sjasone } else { 870234370Sjasone /* 871234370Sjasone * Initialize tcache after checking size in order to avoid 872234370Sjasone * infinite recursion during tcache initialization. 873234370Sjasone */ 874234370Sjasone if (try_tcache && size <= tcache_maxclass && (tcache = 875234370Sjasone tcache_get(true)) != NULL) 876234370Sjasone return (tcache_alloc_large(tcache, size, zero)); 877234370Sjasone else { 878234370Sjasone return (arena_malloc_large(choose_arena(arena), size, 879234370Sjasone zero)); 880234370Sjasone } 881234370Sjasone } 882234370Sjasone} 883234370Sjasone 884234543Sjasone/* Return the size of the allocation pointed to by ptr. */ 885234543SjasoneJEMALLOC_INLINE size_t 886234543Sjasonearena_salloc(const void *ptr, bool demote) 887234543Sjasone{ 888234543Sjasone size_t ret; 889234543Sjasone arena_chunk_t *chunk; 890235238Sjasone size_t pageind, binind; 891234543Sjasone 892234543Sjasone assert(ptr != NULL); 893234543Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 894234543Sjasone 895234543Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 896234543Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 897235238Sjasone assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 898235238Sjasone binind = arena_mapbits_binind_get(chunk, pageind); 899235238Sjasone if (binind == BININD_INVALID || (config_prof && demote == false && 900235238Sjasone prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) { 901235238Sjasone /* 902235238Sjasone * Large allocation. In the common case (demote == true), and 903235238Sjasone * as this is an inline function, most callers will only end up 904235238Sjasone * looking at binind to determine that ptr is a small 905235238Sjasone * allocation. 906235238Sjasone */ 907234543Sjasone assert(((uintptr_t)ptr & PAGE_MASK) == 0); 908235238Sjasone ret = arena_mapbits_large_size_get(chunk, pageind); 909234543Sjasone assert(ret != 0); 910235238Sjasone assert(pageind + (ret>>LG_PAGE) <= chunk_npages); 911235238Sjasone assert(ret == PAGE || arena_mapbits_large_size_get(chunk, 912235238Sjasone pageind+(ret>>LG_PAGE)-1) == 0); 913235238Sjasone assert(binind == arena_mapbits_binind_get(chunk, 914235238Sjasone pageind+(ret>>LG_PAGE)-1)); 915235238Sjasone assert(arena_mapbits_dirty_get(chunk, pageind) == 916235238Sjasone arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); 917235238Sjasone } else { 918235238Sjasone /* 919235238Sjasone * Small allocation (possibly promoted to a large object due to 920235238Sjasone * prof_promote). 921235238Sjasone */ 922235238Sjasone assert(arena_mapbits_large_get(chunk, pageind) != 0 || 923235238Sjasone arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 924235238Sjasone pageind)) == binind); 925235238Sjasone ret = arena_bin_info[binind].reg_size; 926234543Sjasone } 927234543Sjasone 928234543Sjasone return (ret); 929234543Sjasone} 930234543Sjasone 931234370SjasoneJEMALLOC_INLINE void 932234370Sjasonearena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) 933234370Sjasone{ 934235238Sjasone size_t pageind, mapbits; 935234370Sjasone tcache_t *tcache; 936234370Sjasone 937234370Sjasone assert(arena != NULL); 938234370Sjasone assert(chunk->arena == arena); 939234370Sjasone assert(ptr != NULL); 940234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 941234370Sjasone 942234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 943235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 944235238Sjasone assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 945235238Sjasone if ((mapbits & CHUNK_MAP_LARGE) == 0) { 946234370Sjasone /* Small allocation. */ 947235238Sjasone if (try_tcache && (tcache = tcache_get(false)) != NULL) { 948235238Sjasone size_t binind; 949234370Sjasone 950235238Sjasone binind = arena_ptr_small_binind_get(ptr, mapbits); 951235238Sjasone tcache_dalloc_small(tcache, ptr, binind); 952235238Sjasone } else 953235238Sjasone arena_dalloc_small(arena, chunk, ptr, pageind); 954234370Sjasone } else { 955235238Sjasone size_t size = arena_mapbits_large_size_get(chunk, pageind); 956234370Sjasone 957234370Sjasone assert(((uintptr_t)ptr & PAGE_MASK) == 0); 958234370Sjasone 959234370Sjasone if (try_tcache && size <= tcache_maxclass && (tcache = 960234370Sjasone tcache_get(false)) != NULL) { 961234370Sjasone tcache_dalloc_large(tcache, ptr, size); 962235238Sjasone } else 963234370Sjasone arena_dalloc_large(arena, chunk, ptr); 964234370Sjasone } 965234370Sjasone} 966235238Sjasone# endif /* JEMALLOC_ARENA_INLINE_B */ 967234370Sjasone#endif 968234370Sjasone 969234370Sjasone#endif /* JEMALLOC_H_INLINES */ 970234370Sjasone/******************************************************************************/ 971