arena.c revision 245868
1234370Sjasone#define JEMALLOC_ARENA_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone 4234370Sjasone/******************************************************************************/ 5234370Sjasone/* Data. */ 6234370Sjasone 7234370Sjasonessize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8234370Sjasonearena_bin_info_t arena_bin_info[NBINS]; 9234370Sjasone 10235238SjasoneJEMALLOC_ALIGNED(CACHELINE) 11234370Sjasoneconst uint8_t small_size2bin[] = { 12234370Sjasone#define S2B_8(i) i, 13234370Sjasone#define S2B_16(i) S2B_8(i) S2B_8(i) 14234370Sjasone#define S2B_32(i) S2B_16(i) S2B_16(i) 15234370Sjasone#define S2B_64(i) S2B_32(i) S2B_32(i) 16234370Sjasone#define S2B_128(i) S2B_64(i) S2B_64(i) 17234370Sjasone#define S2B_256(i) S2B_128(i) S2B_128(i) 18234370Sjasone#define S2B_512(i) S2B_256(i) S2B_256(i) 19234370Sjasone#define S2B_1024(i) S2B_512(i) S2B_512(i) 20234370Sjasone#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21234370Sjasone#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22234370Sjasone#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23234370Sjasone#define SIZE_CLASS(bin, delta, size) \ 24234370Sjasone S2B_##delta(bin) 25234370Sjasone SIZE_CLASSES 26234370Sjasone#undef S2B_8 27234370Sjasone#undef S2B_16 28234370Sjasone#undef S2B_32 29234370Sjasone#undef S2B_64 30234370Sjasone#undef S2B_128 31234370Sjasone#undef S2B_256 32234370Sjasone#undef S2B_512 33234370Sjasone#undef S2B_1024 34234370Sjasone#undef S2B_2048 35234370Sjasone#undef S2B_4096 36234370Sjasone#undef S2B_8192 37234370Sjasone#undef SIZE_CLASS 38234370Sjasone}; 39234370Sjasone 40234370Sjasone/******************************************************************************/ 41234370Sjasone/* Function prototypes for non-inline static functions. */ 42234370Sjasone 43242844Sjasonestatic void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, 44242844Sjasone size_t pageind, size_t npages, bool maybe_adjac_pred, 45242844Sjasone bool maybe_adjac_succ); 46242844Sjasonestatic void arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, 47242844Sjasone size_t pageind, size_t npages, bool maybe_adjac_pred, 48242844Sjasone bool maybe_adjac_succ); 49234370Sjasonestatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 50235238Sjasone bool large, size_t binind, bool zero); 51234370Sjasonestatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 52234370Sjasonestatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 53235322Sjasonestatic arena_run_t *arena_run_alloc_helper(arena_t *arena, size_t size, 54235322Sjasone bool large, size_t binind, bool zero); 55234370Sjasonestatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 56235238Sjasone size_t binind, bool zero); 57242844Sjasonestatic arena_chunk_t *chunks_dirty_iter_cb(arena_chunk_tree_t *tree, 58242844Sjasone arena_chunk_t *chunk, void *arg); 59234370Sjasonestatic void arena_purge(arena_t *arena, bool all); 60242844Sjasonestatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 61242844Sjasone bool cleaned); 62234370Sjasonestatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 63234370Sjasone arena_run_t *run, size_t oldsize, size_t newsize); 64234370Sjasonestatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 65234370Sjasone arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 66234370Sjasonestatic arena_run_t *arena_bin_runs_first(arena_bin_t *bin); 67234370Sjasonestatic void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run); 68234370Sjasonestatic void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run); 69234370Sjasonestatic arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin); 70234370Sjasonestatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 71234370Sjasonestatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 72234370Sjasonestatic void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 73234370Sjasone arena_bin_t *bin); 74234370Sjasonestatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 75234370Sjasone arena_run_t *run, arena_bin_t *bin); 76234370Sjasonestatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 77234370Sjasone arena_run_t *run, arena_bin_t *bin); 78234370Sjasonestatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 79234370Sjasone void *ptr, size_t oldsize, size_t size); 80234370Sjasonestatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 81234370Sjasone void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); 82234370Sjasonestatic bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, 83234370Sjasone size_t extra, bool zero); 84234370Sjasonestatic size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, 85234370Sjasone size_t min_run_size); 86234370Sjasonestatic void bin_info_init(void); 87234370Sjasone 88234370Sjasone/******************************************************************************/ 89234370Sjasone 90234370Sjasonestatic inline int 91234370Sjasonearena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 92234370Sjasone{ 93234370Sjasone uintptr_t a_mapelm = (uintptr_t)a; 94234370Sjasone uintptr_t b_mapelm = (uintptr_t)b; 95234370Sjasone 96234370Sjasone assert(a != NULL); 97234370Sjasone assert(b != NULL); 98234370Sjasone 99234370Sjasone return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 100234370Sjasone} 101234370Sjasone 102234370Sjasone/* Generate red-black tree functions. */ 103234370Sjasonerb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 104234370Sjasone u.rb_link, arena_run_comp) 105234370Sjasone 106234370Sjasonestatic inline int 107234370Sjasonearena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 108234370Sjasone{ 109234370Sjasone int ret; 110234370Sjasone size_t a_size = a->bits & ~PAGE_MASK; 111234370Sjasone size_t b_size = b->bits & ~PAGE_MASK; 112234370Sjasone 113234370Sjasone ret = (a_size > b_size) - (a_size < b_size); 114234370Sjasone if (ret == 0) { 115234370Sjasone uintptr_t a_mapelm, b_mapelm; 116234370Sjasone 117234370Sjasone if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 118234370Sjasone a_mapelm = (uintptr_t)a; 119234370Sjasone else { 120234370Sjasone /* 121234370Sjasone * Treat keys as though they are lower than anything 122234370Sjasone * else. 123234370Sjasone */ 124234370Sjasone a_mapelm = 0; 125234370Sjasone } 126234370Sjasone b_mapelm = (uintptr_t)b; 127234370Sjasone 128234370Sjasone ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 129234370Sjasone } 130234370Sjasone 131234370Sjasone return (ret); 132234370Sjasone} 133234370Sjasone 134234370Sjasone/* Generate red-black tree functions. */ 135234370Sjasonerb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 136234370Sjasone u.rb_link, arena_avail_comp) 137234370Sjasone 138242844Sjasonestatic inline int 139242844Sjasonearena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 140242844Sjasone{ 141242844Sjasone 142242844Sjasone assert(a != NULL); 143242844Sjasone assert(b != NULL); 144242844Sjasone 145242844Sjasone /* 146242844Sjasone * Short-circuit for self comparison. The following comparison code 147242844Sjasone * would come to the same result, but at the cost of executing the slow 148242844Sjasone * path. 149242844Sjasone */ 150242844Sjasone if (a == b) 151242844Sjasone return (0); 152242844Sjasone 153242844Sjasone /* 154242844Sjasone * Order such that chunks with higher fragmentation are "less than" 155242844Sjasone * those with lower fragmentation -- purging order is from "least" to 156242844Sjasone * "greatest". Fragmentation is measured as: 157242844Sjasone * 158242844Sjasone * mean current avail run size 159242844Sjasone * -------------------------------- 160242844Sjasone * mean defragmented avail run size 161242844Sjasone * 162242844Sjasone * navail 163242844Sjasone * ----------- 164242844Sjasone * nruns_avail nruns_avail-nruns_adjac 165242844Sjasone * = ========================= = ----------------------- 166242844Sjasone * navail nruns_avail 167242844Sjasone * ----------------------- 168242844Sjasone * nruns_avail-nruns_adjac 169242844Sjasone * 170242844Sjasone * The following code multiplies away the denominator prior to 171242844Sjasone * comparison, in order to avoid division. 172242844Sjasone * 173242844Sjasone */ 174242844Sjasone { 175242844Sjasone size_t a_val = (a->nruns_avail - a->nruns_adjac) * 176242844Sjasone b->nruns_avail; 177242844Sjasone size_t b_val = (b->nruns_avail - b->nruns_adjac) * 178242844Sjasone a->nruns_avail; 179242844Sjasone 180242844Sjasone if (a_val < b_val) 181242844Sjasone return (1); 182242844Sjasone if (a_val > b_val) 183242844Sjasone return (-1); 184242844Sjasone } 185242844Sjasone /* 186242844Sjasone * Break ties by chunk address. For fragmented chunks, report lower 187242844Sjasone * addresses as "lower", so that fragmentation reduction happens first 188242844Sjasone * at lower addresses. However, use the opposite ordering for 189242844Sjasone * unfragmented chunks, in order to increase the chances of 190242844Sjasone * re-allocating dirty runs. 191242844Sjasone */ 192242844Sjasone { 193242844Sjasone uintptr_t a_chunk = (uintptr_t)a; 194242844Sjasone uintptr_t b_chunk = (uintptr_t)b; 195242844Sjasone int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 196242844Sjasone if (a->nruns_adjac == 0) { 197242844Sjasone assert(b->nruns_adjac == 0); 198242844Sjasone ret = -ret; 199242844Sjasone } 200242844Sjasone return (ret); 201242844Sjasone } 202242844Sjasone} 203242844Sjasone 204242844Sjasone/* Generate red-black tree functions. */ 205242844Sjasonerb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 206242844Sjasone dirty_link, arena_chunk_dirty_comp) 207242844Sjasone 208242844Sjasonestatic inline bool 209242844Sjasonearena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 210242844Sjasone{ 211242844Sjasone bool ret; 212242844Sjasone 213242844Sjasone if (pageind-1 < map_bias) 214242844Sjasone ret = false; 215242844Sjasone else { 216242844Sjasone ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 217242844Sjasone assert(ret == false || arena_mapbits_dirty_get(chunk, 218242844Sjasone pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 219242844Sjasone } 220242844Sjasone return (ret); 221242844Sjasone} 222242844Sjasone 223242844Sjasonestatic inline bool 224242844Sjasonearena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 225242844Sjasone{ 226242844Sjasone bool ret; 227242844Sjasone 228242844Sjasone if (pageind+npages == chunk_npages) 229242844Sjasone ret = false; 230242844Sjasone else { 231242844Sjasone assert(pageind+npages < chunk_npages); 232242844Sjasone ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 233242844Sjasone assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 234242844Sjasone != arena_mapbits_dirty_get(chunk, pageind+npages)); 235242844Sjasone } 236242844Sjasone return (ret); 237242844Sjasone} 238242844Sjasone 239242844Sjasonestatic inline bool 240242844Sjasonearena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 241242844Sjasone{ 242242844Sjasone 243242844Sjasone return (arena_avail_adjac_pred(chunk, pageind) || 244242844Sjasone arena_avail_adjac_succ(chunk, pageind, npages)); 245242844Sjasone} 246242844Sjasone 247242844Sjasonestatic void 248242844Sjasonearena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 249242844Sjasone size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 250242844Sjasone{ 251242844Sjasone 252242844Sjasone assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 253242844Sjasone LG_PAGE)); 254242844Sjasone 255242844Sjasone /* 256242844Sjasone * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 257242844Sjasone * removed and reinserted even if the run to be inserted is clean. 258242844Sjasone */ 259242844Sjasone if (chunk->ndirty != 0) 260242844Sjasone arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 261242844Sjasone 262242844Sjasone if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 263242844Sjasone chunk->nruns_adjac++; 264242844Sjasone if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 265242844Sjasone chunk->nruns_adjac++; 266242844Sjasone chunk->nruns_avail++; 267242844Sjasone assert(chunk->nruns_avail > chunk->nruns_adjac); 268242844Sjasone 269242844Sjasone if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 270242844Sjasone arena->ndirty += npages; 271242844Sjasone chunk->ndirty += npages; 272242844Sjasone } 273242844Sjasone if (chunk->ndirty != 0) 274242844Sjasone arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 275242844Sjasone 276242844Sjasone arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 277242844Sjasone pageind)); 278242844Sjasone} 279242844Sjasone 280242844Sjasonestatic void 281242844Sjasonearena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 282242844Sjasone size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 283242844Sjasone{ 284242844Sjasone 285242844Sjasone assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 286242844Sjasone LG_PAGE)); 287242844Sjasone 288242844Sjasone /* 289242844Sjasone * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 290242844Sjasone * removed and reinserted even if the run to be removed is clean. 291242844Sjasone */ 292242844Sjasone if (chunk->ndirty != 0) 293242844Sjasone arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 294242844Sjasone 295242844Sjasone if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 296242844Sjasone chunk->nruns_adjac--; 297242844Sjasone if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 298242844Sjasone chunk->nruns_adjac--; 299242844Sjasone chunk->nruns_avail--; 300242844Sjasone assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 301242844Sjasone == 0 && chunk->nruns_adjac == 0)); 302242844Sjasone 303242844Sjasone if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 304242844Sjasone arena->ndirty -= npages; 305242844Sjasone chunk->ndirty -= npages; 306242844Sjasone } 307242844Sjasone if (chunk->ndirty != 0) 308242844Sjasone arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 309242844Sjasone 310242844Sjasone arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 311242844Sjasone pageind)); 312242844Sjasone} 313242844Sjasone 314234370Sjasonestatic inline void * 315234370Sjasonearena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 316234370Sjasone{ 317234370Sjasone void *ret; 318234370Sjasone unsigned regind; 319234370Sjasone bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 320234370Sjasone (uintptr_t)bin_info->bitmap_offset); 321234370Sjasone 322234370Sjasone assert(run->nfree > 0); 323234370Sjasone assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 324234370Sjasone 325234370Sjasone regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 326234370Sjasone ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 327234370Sjasone (uintptr_t)(bin_info->reg_interval * regind)); 328234370Sjasone run->nfree--; 329234370Sjasone if (regind == run->nextind) 330234370Sjasone run->nextind++; 331234370Sjasone assert(regind < run->nextind); 332234370Sjasone return (ret); 333234370Sjasone} 334234370Sjasone 335234370Sjasonestatic inline void 336234370Sjasonearena_run_reg_dalloc(arena_run_t *run, void *ptr) 337234370Sjasone{ 338234370Sjasone arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 339235238Sjasone size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 340235238Sjasone size_t mapbits = arena_mapbits_get(chunk, pageind); 341235238Sjasone size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 342234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 343234370Sjasone unsigned regind = arena_run_regind(run, bin_info, ptr); 344234370Sjasone bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 345234370Sjasone (uintptr_t)bin_info->bitmap_offset); 346234370Sjasone 347234370Sjasone assert(run->nfree < bin_info->nregs); 348234370Sjasone /* Freeing an interior pointer can cause assertion failure. */ 349234370Sjasone assert(((uintptr_t)ptr - ((uintptr_t)run + 350234370Sjasone (uintptr_t)bin_info->reg0_offset)) % 351234370Sjasone (uintptr_t)bin_info->reg_interval == 0); 352234370Sjasone assert((uintptr_t)ptr >= (uintptr_t)run + 353234370Sjasone (uintptr_t)bin_info->reg0_offset); 354234370Sjasone /* Freeing an unallocated pointer can cause assertion failure. */ 355234370Sjasone assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 356234370Sjasone 357234370Sjasone bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 358234370Sjasone run->nfree++; 359234370Sjasone} 360234370Sjasone 361234370Sjasonestatic inline void 362245868Sjasonearena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 363234370Sjasone{ 364245868Sjasone 365245868Sjasone VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 366245868Sjasone LG_PAGE)), (npages << LG_PAGE)); 367245868Sjasone memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 368245868Sjasone (npages << LG_PAGE)); 369245868Sjasone VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 370245868Sjasone LG_PAGE)), (npages << LG_PAGE)); 371245868Sjasone} 372245868Sjasone 373245868Sjasonestatic inline void 374245868Sjasonearena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 375245868Sjasone{ 376234370Sjasone size_t i; 377234370Sjasone UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 378234370Sjasone 379245868Sjasone VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << 380245868Sjasone LG_PAGE)), PAGE); 381234370Sjasone for (i = 0; i < PAGE / sizeof(size_t); i++) 382234370Sjasone assert(p[i] == 0); 383245868Sjasone VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 384245868Sjasone LG_PAGE)), PAGE); 385234370Sjasone} 386234370Sjasone 387234370Sjasonestatic void 388234370Sjasonearena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 389235238Sjasone size_t binind, bool zero) 390234370Sjasone{ 391234370Sjasone arena_chunk_t *chunk; 392234370Sjasone size_t run_ind, total_pages, need_pages, rem_pages, i; 393234370Sjasone size_t flag_dirty; 394234370Sjasone 395235238Sjasone assert((large && binind == BININD_INVALID) || (large == false && binind 396235238Sjasone != BININD_INVALID)); 397235238Sjasone 398234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 399234370Sjasone run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 400235238Sjasone flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 401235238Sjasone total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 402234370Sjasone LG_PAGE; 403235238Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 404235238Sjasone flag_dirty); 405234370Sjasone need_pages = (size >> LG_PAGE); 406234370Sjasone assert(need_pages > 0); 407234370Sjasone assert(need_pages <= total_pages); 408234370Sjasone rem_pages = total_pages - need_pages; 409234370Sjasone 410242844Sjasone arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 411234370Sjasone if (config_stats) { 412234370Sjasone /* 413234370Sjasone * Update stats_cactive if nactive is crossing a chunk 414234370Sjasone * multiple. 415234370Sjasone */ 416234370Sjasone size_t cactive_diff = CHUNK_CEILING((arena->nactive + 417234370Sjasone need_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 418234370Sjasone LG_PAGE); 419234370Sjasone if (cactive_diff != 0) 420234370Sjasone stats_cactive_add(cactive_diff); 421234370Sjasone } 422234370Sjasone arena->nactive += need_pages; 423234370Sjasone 424234370Sjasone /* Keep track of trailing unused pages for later use. */ 425234370Sjasone if (rem_pages > 0) { 426234370Sjasone if (flag_dirty != 0) { 427235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 428235238Sjasone (rem_pages << LG_PAGE), CHUNK_MAP_DIRTY); 429235238Sjasone arena_mapbits_unallocated_set(chunk, 430235238Sjasone run_ind+total_pages-1, (rem_pages << LG_PAGE), 431235238Sjasone CHUNK_MAP_DIRTY); 432234370Sjasone } else { 433235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 434235238Sjasone (rem_pages << LG_PAGE), 435235238Sjasone arena_mapbits_unzeroed_get(chunk, 436235238Sjasone run_ind+need_pages)); 437235238Sjasone arena_mapbits_unallocated_set(chunk, 438235238Sjasone run_ind+total_pages-1, (rem_pages << LG_PAGE), 439235238Sjasone arena_mapbits_unzeroed_get(chunk, 440235238Sjasone run_ind+total_pages-1)); 441234370Sjasone } 442242844Sjasone arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 443242844Sjasone false, true); 444234370Sjasone } 445234370Sjasone 446234370Sjasone /* 447234370Sjasone * Update the page map separately for large vs. small runs, since it is 448234370Sjasone * possible to avoid iteration for large mallocs. 449234370Sjasone */ 450234370Sjasone if (large) { 451234370Sjasone if (zero) { 452234370Sjasone if (flag_dirty == 0) { 453234370Sjasone /* 454234370Sjasone * The run is clean, so some pages may be 455234370Sjasone * zeroed (i.e. never before touched). 456234370Sjasone */ 457234370Sjasone for (i = 0; i < need_pages; i++) { 458235238Sjasone if (arena_mapbits_unzeroed_get(chunk, 459235238Sjasone run_ind+i) != 0) { 460245868Sjasone arena_run_zero(chunk, run_ind+i, 461245868Sjasone 1); 462234370Sjasone } else if (config_debug) { 463245868Sjasone arena_run_page_validate_zeroed( 464234370Sjasone chunk, run_ind+i); 465234370Sjasone } 466234370Sjasone } 467234370Sjasone } else { 468234370Sjasone /* 469234370Sjasone * The run is dirty, so all pages must be 470234370Sjasone * zeroed. 471234370Sjasone */ 472245868Sjasone arena_run_zero(chunk, run_ind, need_pages); 473234370Sjasone } 474234370Sjasone } 475234370Sjasone 476234370Sjasone /* 477234370Sjasone * Set the last element first, in case the run only contains one 478234370Sjasone * page (i.e. both statements set the same element). 479234370Sjasone */ 480235238Sjasone arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, 481235238Sjasone flag_dirty); 482235238Sjasone arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 483234370Sjasone } else { 484234370Sjasone assert(zero == false); 485234370Sjasone /* 486234370Sjasone * Propagate the dirty and unzeroed flags to the allocated 487234370Sjasone * small run, so that arena_dalloc_bin_run() has the ability to 488234370Sjasone * conditionally trim clean pages. 489234370Sjasone */ 490235322Sjasone arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 491234370Sjasone /* 492234370Sjasone * The first page will always be dirtied during small run 493234370Sjasone * initialization, so a validation failure here would not 494234370Sjasone * actually cause an observable failure. 495234370Sjasone */ 496234370Sjasone if (config_debug && flag_dirty == 0 && 497235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind) == 0) 498245868Sjasone arena_run_page_validate_zeroed(chunk, run_ind); 499234370Sjasone for (i = 1; i < need_pages - 1; i++) { 500235322Sjasone arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 501234370Sjasone if (config_debug && flag_dirty == 0 && 502245868Sjasone arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) { 503245868Sjasone arena_run_page_validate_zeroed(chunk, 504245868Sjasone run_ind+i); 505245868Sjasone } 506234370Sjasone } 507235238Sjasone arena_mapbits_small_set(chunk, run_ind+need_pages-1, 508235322Sjasone need_pages-1, binind, flag_dirty); 509234370Sjasone if (config_debug && flag_dirty == 0 && 510235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == 511235238Sjasone 0) { 512245868Sjasone arena_run_page_validate_zeroed(chunk, 513234370Sjasone run_ind+need_pages-1); 514234370Sjasone } 515234370Sjasone } 516234370Sjasone} 517234370Sjasone 518234370Sjasonestatic arena_chunk_t * 519234370Sjasonearena_chunk_alloc(arena_t *arena) 520234370Sjasone{ 521234370Sjasone arena_chunk_t *chunk; 522234370Sjasone size_t i; 523234370Sjasone 524234370Sjasone if (arena->spare != NULL) { 525234370Sjasone chunk = arena->spare; 526234370Sjasone arena->spare = NULL; 527234370Sjasone 528235322Sjasone assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 529235322Sjasone assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 530235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 531235238Sjasone arena_maxclass); 532235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, 533235238Sjasone chunk_npages-1) == arena_maxclass); 534235238Sjasone assert(arena_mapbits_dirty_get(chunk, map_bias) == 535235238Sjasone arena_mapbits_dirty_get(chunk, chunk_npages-1)); 536234370Sjasone } else { 537234370Sjasone bool zero; 538234370Sjasone size_t unzeroed; 539234370Sjasone 540234370Sjasone zero = false; 541234370Sjasone malloc_mutex_unlock(&arena->lock); 542234370Sjasone chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, 543242844Sjasone false, &zero, arena->dss_prec); 544234370Sjasone malloc_mutex_lock(&arena->lock); 545234370Sjasone if (chunk == NULL) 546234370Sjasone return (NULL); 547234370Sjasone if (config_stats) 548234370Sjasone arena->stats.mapped += chunksize; 549234370Sjasone 550234370Sjasone chunk->arena = arena; 551234370Sjasone 552234370Sjasone /* 553234370Sjasone * Claim that no pages are in use, since the header is merely 554234370Sjasone * overhead. 555234370Sjasone */ 556234370Sjasone chunk->ndirty = 0; 557234370Sjasone 558242844Sjasone chunk->nruns_avail = 0; 559242844Sjasone chunk->nruns_adjac = 0; 560242844Sjasone 561234370Sjasone /* 562234370Sjasone * Initialize the map to contain one maximal free untouched run. 563234370Sjasone * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 564234370Sjasone * chunk. 565234370Sjasone */ 566234370Sjasone unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 567235238Sjasone arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 568235238Sjasone unzeroed); 569234370Sjasone /* 570234370Sjasone * There is no need to initialize the internal page map entries 571234370Sjasone * unless the chunk is not zeroed. 572234370Sjasone */ 573234370Sjasone if (zero == false) { 574234370Sjasone for (i = map_bias+1; i < chunk_npages-1; i++) 575235238Sjasone arena_mapbits_unzeroed_set(chunk, i, unzeroed); 576234370Sjasone } else if (config_debug) { 577235238Sjasone for (i = map_bias+1; i < chunk_npages-1; i++) { 578235238Sjasone assert(arena_mapbits_unzeroed_get(chunk, i) == 579235238Sjasone unzeroed); 580235238Sjasone } 581234370Sjasone } 582235238Sjasone arena_mapbits_unallocated_set(chunk, chunk_npages-1, 583235238Sjasone arena_maxclass, unzeroed); 584234370Sjasone } 585234370Sjasone 586242844Sjasone /* Insert the run into the runs_avail tree. */ 587242844Sjasone arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 588242844Sjasone false, false); 589242844Sjasone 590234370Sjasone return (chunk); 591234370Sjasone} 592234370Sjasone 593234370Sjasonestatic void 594234370Sjasonearena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 595234370Sjasone{ 596235322Sjasone assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 597235322Sjasone assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 598235322Sjasone assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 599235322Sjasone arena_maxclass); 600235322Sjasone assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 601235322Sjasone arena_maxclass); 602235322Sjasone assert(arena_mapbits_dirty_get(chunk, map_bias) == 603235322Sjasone arena_mapbits_dirty_get(chunk, chunk_npages-1)); 604235322Sjasone 605234370Sjasone /* 606242844Sjasone * Remove run from the runs_avail tree, so that the arena does not use 607242844Sjasone * it. 608234370Sjasone */ 609242844Sjasone arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 610242844Sjasone false, false); 611234370Sjasone 612234370Sjasone if (arena->spare != NULL) { 613234370Sjasone arena_chunk_t *spare = arena->spare; 614234370Sjasone 615234370Sjasone arena->spare = chunk; 616234370Sjasone malloc_mutex_unlock(&arena->lock); 617234370Sjasone chunk_dealloc((void *)spare, chunksize, true); 618234370Sjasone malloc_mutex_lock(&arena->lock); 619234370Sjasone if (config_stats) 620234370Sjasone arena->stats.mapped -= chunksize; 621234370Sjasone } else 622234370Sjasone arena->spare = chunk; 623234370Sjasone} 624234370Sjasone 625234370Sjasonestatic arena_run_t * 626235322Sjasonearena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, 627235238Sjasone bool zero) 628234370Sjasone{ 629234370Sjasone arena_run_t *run; 630234370Sjasone arena_chunk_map_t *mapelm, key; 631234370Sjasone 632234370Sjasone key.bits = size | CHUNK_MAP_KEY; 633242844Sjasone mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); 634234370Sjasone if (mapelm != NULL) { 635234370Sjasone arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 636234370Sjasone size_t pageind = (((uintptr_t)mapelm - 637234370Sjasone (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 638234370Sjasone + map_bias; 639234370Sjasone 640234370Sjasone run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 641234370Sjasone LG_PAGE)); 642235238Sjasone arena_run_split(arena, run, size, large, binind, zero); 643234370Sjasone return (run); 644234370Sjasone } 645234370Sjasone 646235322Sjasone return (NULL); 647235322Sjasone} 648235322Sjasone 649235322Sjasonestatic arena_run_t * 650235322Sjasonearena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, 651235322Sjasone bool zero) 652235322Sjasone{ 653235322Sjasone arena_chunk_t *chunk; 654235322Sjasone arena_run_t *run; 655235322Sjasone 656235322Sjasone assert(size <= arena_maxclass); 657235322Sjasone assert((size & PAGE_MASK) == 0); 658235322Sjasone assert((large && binind == BININD_INVALID) || (large == false && binind 659235322Sjasone != BININD_INVALID)); 660235322Sjasone 661235322Sjasone /* Search the arena's chunks for the lowest best fit. */ 662235322Sjasone run = arena_run_alloc_helper(arena, size, large, binind, zero); 663235322Sjasone if (run != NULL) 664235322Sjasone return (run); 665235322Sjasone 666234370Sjasone /* 667234370Sjasone * No usable runs. Create a new chunk from which to allocate the run. 668234370Sjasone */ 669234370Sjasone chunk = arena_chunk_alloc(arena); 670234370Sjasone if (chunk != NULL) { 671234370Sjasone run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 672235238Sjasone arena_run_split(arena, run, size, large, binind, zero); 673234370Sjasone return (run); 674234370Sjasone } 675234370Sjasone 676234370Sjasone /* 677234370Sjasone * arena_chunk_alloc() failed, but another thread may have made 678234370Sjasone * sufficient memory available while this one dropped arena->lock in 679234370Sjasone * arena_chunk_alloc(), so search one more time. 680234370Sjasone */ 681235322Sjasone return (arena_run_alloc_helper(arena, size, large, binind, zero)); 682234370Sjasone} 683234370Sjasone 684234370Sjasonestatic inline void 685234370Sjasonearena_maybe_purge(arena_t *arena) 686234370Sjasone{ 687242844Sjasone size_t npurgeable, threshold; 688234370Sjasone 689242844Sjasone /* Don't purge if the option is disabled. */ 690242844Sjasone if (opt_lg_dirty_mult < 0) 691242844Sjasone return; 692242844Sjasone /* Don't purge if all dirty pages are already being purged. */ 693242844Sjasone if (arena->ndirty <= arena->npurgatory) 694242844Sjasone return; 695242844Sjasone npurgeable = arena->ndirty - arena->npurgatory; 696242844Sjasone threshold = (arena->nactive >> opt_lg_dirty_mult); 697242844Sjasone /* 698242844Sjasone * Don't purge unless the number of purgeable pages exceeds the 699242844Sjasone * threshold. 700242844Sjasone */ 701242844Sjasone if (npurgeable <= threshold) 702242844Sjasone return; 703242844Sjasone 704242844Sjasone arena_purge(arena, false); 705234370Sjasone} 706234370Sjasone 707242844Sjasonestatic inline size_t 708242844Sjasonearena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 709234370Sjasone{ 710242844Sjasone size_t npurged; 711234370Sjasone ql_head(arena_chunk_map_t) mapelms; 712234370Sjasone arena_chunk_map_t *mapelm; 713242844Sjasone size_t pageind, npages; 714234370Sjasone size_t nmadvise; 715234370Sjasone 716234370Sjasone ql_new(&mapelms); 717234370Sjasone 718234370Sjasone /* 719234370Sjasone * If chunk is the spare, temporarily re-allocate it, 1) so that its 720242844Sjasone * run is reinserted into runs_avail, and 2) so that it cannot be 721234370Sjasone * completely discarded by another thread while arena->lock is dropped 722234370Sjasone * by this thread. Note that the arena_run_dalloc() call will 723234370Sjasone * implicitly deallocate the chunk, so no explicit action is required 724234370Sjasone * in this function to deallocate the chunk. 725234370Sjasone * 726234370Sjasone * Note that once a chunk contains dirty pages, it cannot again contain 727234370Sjasone * a single run unless 1) it is a dirty run, or 2) this function purges 728234370Sjasone * dirty pages and causes the transition to a single clean run. Thus 729234370Sjasone * (chunk == arena->spare) is possible, but it is not possible for 730234370Sjasone * this function to be called on the spare unless it contains a dirty 731234370Sjasone * run. 732234370Sjasone */ 733234370Sjasone if (chunk == arena->spare) { 734235238Sjasone assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 735235322Sjasone assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 736235322Sjasone 737234370Sjasone arena_chunk_alloc(arena); 738234370Sjasone } 739234370Sjasone 740242844Sjasone if (config_stats) 741242844Sjasone arena->stats.purged += chunk->ndirty; 742242844Sjasone 743242844Sjasone /* 744242844Sjasone * Operate on all dirty runs if there is no clean/dirty run 745242844Sjasone * fragmentation. 746242844Sjasone */ 747242844Sjasone if (chunk->nruns_adjac == 0) 748242844Sjasone all = true; 749242844Sjasone 750242844Sjasone /* 751242844Sjasone * Temporarily allocate free dirty runs within chunk. If all is false, 752242844Sjasone * only operate on dirty runs that are fragments; otherwise operate on 753242844Sjasone * all dirty runs. 754242844Sjasone */ 755242844Sjasone for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 756235238Sjasone mapelm = arena_mapp_get(chunk, pageind); 757235238Sjasone if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 758242844Sjasone size_t run_size = 759242844Sjasone arena_mapbits_unallocated_size_get(chunk, pageind); 760234370Sjasone 761242844Sjasone npages = run_size >> LG_PAGE; 762234370Sjasone assert(pageind + npages <= chunk_npages); 763235322Sjasone assert(arena_mapbits_dirty_get(chunk, pageind) == 764235322Sjasone arena_mapbits_dirty_get(chunk, pageind+npages-1)); 765234370Sjasone 766242844Sjasone if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 767242844Sjasone (all || arena_avail_adjac(chunk, pageind, 768242844Sjasone npages))) { 769242844Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t) 770242844Sjasone chunk + (uintptr_t)(pageind << LG_PAGE)); 771234370Sjasone 772242844Sjasone arena_run_split(arena, run, run_size, true, 773242844Sjasone BININD_INVALID, false); 774234370Sjasone /* Append to list for later processing. */ 775234370Sjasone ql_elm_new(mapelm, u.ql_link); 776234370Sjasone ql_tail_insert(&mapelms, mapelm, u.ql_link); 777234370Sjasone } 778234370Sjasone } else { 779242844Sjasone /* Skip run. */ 780242844Sjasone if (arena_mapbits_large_get(chunk, pageind) != 0) { 781242844Sjasone npages = arena_mapbits_large_size_get(chunk, 782235238Sjasone pageind) >> LG_PAGE; 783242844Sjasone } else { 784235238Sjasone size_t binind; 785235238Sjasone arena_bin_info_t *bin_info; 786234370Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t) 787234370Sjasone chunk + (uintptr_t)(pageind << LG_PAGE)); 788234370Sjasone 789235238Sjasone assert(arena_mapbits_small_runind_get(chunk, 790235238Sjasone pageind) == 0); 791235238Sjasone binind = arena_bin_index(arena, run->bin); 792235238Sjasone bin_info = &arena_bin_info[binind]; 793242844Sjasone npages = bin_info->run_size >> LG_PAGE; 794234370Sjasone } 795234370Sjasone } 796234370Sjasone } 797234370Sjasone assert(pageind == chunk_npages); 798242844Sjasone assert(chunk->ndirty == 0 || all == false); 799242844Sjasone assert(chunk->nruns_adjac == 0); 800234370Sjasone 801234370Sjasone malloc_mutex_unlock(&arena->lock); 802234370Sjasone if (config_stats) 803234370Sjasone nmadvise = 0; 804242844Sjasone npurged = 0; 805234370Sjasone ql_foreach(mapelm, &mapelms, u.ql_link) { 806242844Sjasone bool unzeroed; 807242844Sjasone size_t flag_unzeroed, i; 808242844Sjasone 809242844Sjasone pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 810234370Sjasone sizeof(arena_chunk_map_t)) + map_bias; 811242844Sjasone npages = arena_mapbits_large_size_get(chunk, pageind) >> 812235238Sjasone LG_PAGE; 813234370Sjasone assert(pageind + npages <= chunk_npages); 814242844Sjasone unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 815242844Sjasone LG_PAGE)), (npages << LG_PAGE)); 816242844Sjasone flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 817242844Sjasone /* 818242844Sjasone * Set the unzeroed flag for all pages, now that pages_purge() 819242844Sjasone * has returned whether the pages were zeroed as a side effect 820242844Sjasone * of purging. This chunk map modification is safe even though 821242844Sjasone * the arena mutex isn't currently owned by this thread, 822242844Sjasone * because the run is marked as allocated, thus protecting it 823242844Sjasone * from being modified by any other thread. As long as these 824242844Sjasone * writes don't perturb the first and last elements' 825242844Sjasone * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 826242844Sjasone */ 827242844Sjasone for (i = 0; i < npages; i++) { 828242844Sjasone arena_mapbits_unzeroed_set(chunk, pageind+i, 829242844Sjasone flag_unzeroed); 830242844Sjasone } 831242844Sjasone npurged += npages; 832234370Sjasone if (config_stats) 833234370Sjasone nmadvise++; 834234370Sjasone } 835234370Sjasone malloc_mutex_lock(&arena->lock); 836234370Sjasone if (config_stats) 837234370Sjasone arena->stats.nmadvise += nmadvise; 838234370Sjasone 839234370Sjasone /* Deallocate runs. */ 840234370Sjasone for (mapelm = ql_first(&mapelms); mapelm != NULL; 841234370Sjasone mapelm = ql_first(&mapelms)) { 842242844Sjasone arena_run_t *run; 843242844Sjasone 844242844Sjasone pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 845234370Sjasone sizeof(arena_chunk_map_t)) + map_bias; 846242844Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 847242844Sjasone LG_PAGE)); 848234370Sjasone ql_remove(&mapelms, mapelm, u.ql_link); 849242844Sjasone arena_run_dalloc(arena, run, false, true); 850234370Sjasone } 851242844Sjasone 852242844Sjasone return (npurged); 853234370Sjasone} 854234370Sjasone 855242844Sjasonestatic arena_chunk_t * 856242844Sjasonechunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 857242844Sjasone{ 858242844Sjasone size_t *ndirty = (size_t *)arg; 859242844Sjasone 860242844Sjasone assert(chunk->ndirty != 0); 861242844Sjasone *ndirty += chunk->ndirty; 862242844Sjasone return (NULL); 863242844Sjasone} 864242844Sjasone 865234370Sjasonestatic void 866234370Sjasonearena_purge(arena_t *arena, bool all) 867234370Sjasone{ 868234370Sjasone arena_chunk_t *chunk; 869234370Sjasone size_t npurgatory; 870234370Sjasone if (config_debug) { 871234370Sjasone size_t ndirty = 0; 872234370Sjasone 873242844Sjasone arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 874242844Sjasone chunks_dirty_iter_cb, (void *)&ndirty); 875234370Sjasone assert(ndirty == arena->ndirty); 876234370Sjasone } 877234370Sjasone assert(arena->ndirty > arena->npurgatory || all); 878234370Sjasone assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 879234370Sjasone arena->npurgatory) || all); 880234370Sjasone 881234370Sjasone if (config_stats) 882234370Sjasone arena->stats.npurge++; 883234370Sjasone 884234370Sjasone /* 885234370Sjasone * Compute the minimum number of pages that this thread should try to 886234370Sjasone * purge, and add the result to arena->npurgatory. This will keep 887234370Sjasone * multiple threads from racing to reduce ndirty below the threshold. 888234370Sjasone */ 889242844Sjasone { 890242844Sjasone size_t npurgeable = arena->ndirty - arena->npurgatory; 891242844Sjasone 892242844Sjasone if (all == false) { 893242844Sjasone size_t threshold = (arena->nactive >> 894242844Sjasone opt_lg_dirty_mult); 895242844Sjasone 896242844Sjasone npurgatory = npurgeable - threshold; 897242844Sjasone } else 898242844Sjasone npurgatory = npurgeable; 899234370Sjasone } 900234370Sjasone arena->npurgatory += npurgatory; 901234370Sjasone 902234370Sjasone while (npurgatory > 0) { 903242844Sjasone size_t npurgeable, npurged, nunpurged; 904242844Sjasone 905234370Sjasone /* Get next chunk with dirty pages. */ 906242844Sjasone chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 907234370Sjasone if (chunk == NULL) { 908234370Sjasone /* 909234370Sjasone * This thread was unable to purge as many pages as 910234370Sjasone * originally intended, due to races with other threads 911234370Sjasone * that either did some of the purging work, or re-used 912234370Sjasone * dirty pages. 913234370Sjasone */ 914234370Sjasone arena->npurgatory -= npurgatory; 915234370Sjasone return; 916234370Sjasone } 917242844Sjasone npurgeable = chunk->ndirty; 918242844Sjasone assert(npurgeable != 0); 919234370Sjasone 920242844Sjasone if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 921234370Sjasone /* 922242844Sjasone * This thread will purge all the dirty pages in chunk, 923242844Sjasone * so set npurgatory to reflect this thread's intent to 924242844Sjasone * purge the pages. This tends to reduce the chances 925242844Sjasone * of the following scenario: 926234370Sjasone * 927234370Sjasone * 1) This thread sets arena->npurgatory such that 928234370Sjasone * (arena->ndirty - arena->npurgatory) is at the 929234370Sjasone * threshold. 930234370Sjasone * 2) This thread drops arena->lock. 931234370Sjasone * 3) Another thread causes one or more pages to be 932234370Sjasone * dirtied, and immediately determines that it must 933234370Sjasone * purge dirty pages. 934234370Sjasone * 935234370Sjasone * If this scenario *does* play out, that's okay, 936234370Sjasone * because all of the purging work being done really 937234370Sjasone * needs to happen. 938234370Sjasone */ 939242844Sjasone arena->npurgatory += npurgeable - npurgatory; 940242844Sjasone npurgatory = npurgeable; 941234370Sjasone } 942234370Sjasone 943242844Sjasone /* 944242844Sjasone * Keep track of how many pages are purgeable, versus how many 945242844Sjasone * actually get purged, and adjust counters accordingly. 946242844Sjasone */ 947242844Sjasone arena->npurgatory -= npurgeable; 948242844Sjasone npurgatory -= npurgeable; 949242844Sjasone npurged = arena_chunk_purge(arena, chunk, all); 950242844Sjasone nunpurged = npurgeable - npurged; 951242844Sjasone arena->npurgatory += nunpurged; 952242844Sjasone npurgatory += nunpurged; 953234370Sjasone } 954234370Sjasone} 955234370Sjasone 956234370Sjasonevoid 957234370Sjasonearena_purge_all(arena_t *arena) 958234370Sjasone{ 959234370Sjasone 960234370Sjasone malloc_mutex_lock(&arena->lock); 961234370Sjasone arena_purge(arena, true); 962234370Sjasone malloc_mutex_unlock(&arena->lock); 963234370Sjasone} 964234370Sjasone 965234370Sjasonestatic void 966242844Sjasonearena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 967234370Sjasone{ 968234370Sjasone arena_chunk_t *chunk; 969234370Sjasone size_t size, run_ind, run_pages, flag_dirty; 970234370Sjasone 971234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 972234370Sjasone run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 973234370Sjasone assert(run_ind >= map_bias); 974234370Sjasone assert(run_ind < chunk_npages); 975235238Sjasone if (arena_mapbits_large_get(chunk, run_ind) != 0) { 976235238Sjasone size = arena_mapbits_large_size_get(chunk, run_ind); 977234370Sjasone assert(size == PAGE || 978235238Sjasone arena_mapbits_large_size_get(chunk, 979235238Sjasone run_ind+(size>>LG_PAGE)-1) == 0); 980234370Sjasone } else { 981234370Sjasone size_t binind = arena_bin_index(arena, run->bin); 982234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 983234370Sjasone size = bin_info->run_size; 984234370Sjasone } 985234370Sjasone run_pages = (size >> LG_PAGE); 986234370Sjasone if (config_stats) { 987234370Sjasone /* 988234370Sjasone * Update stats_cactive if nactive is crossing a chunk 989234370Sjasone * multiple. 990234370Sjasone */ 991234370Sjasone size_t cactive_diff = CHUNK_CEILING(arena->nactive << LG_PAGE) - 992234370Sjasone CHUNK_CEILING((arena->nactive - run_pages) << LG_PAGE); 993234370Sjasone if (cactive_diff != 0) 994234370Sjasone stats_cactive_sub(cactive_diff); 995234370Sjasone } 996234370Sjasone arena->nactive -= run_pages; 997234370Sjasone 998234370Sjasone /* 999234370Sjasone * The run is dirty if the caller claims to have dirtied it, as well as 1000242844Sjasone * if it was already dirty before being allocated and the caller 1001242844Sjasone * doesn't claim to have cleaned it. 1002234370Sjasone */ 1003235322Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == 1004235322Sjasone arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1005242844Sjasone if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1006234370Sjasone dirty = true; 1007234370Sjasone flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1008234370Sjasone 1009234370Sjasone /* Mark pages as unallocated in the chunk map. */ 1010234370Sjasone if (dirty) { 1011235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind, size, 1012235238Sjasone CHUNK_MAP_DIRTY); 1013235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1014235238Sjasone CHUNK_MAP_DIRTY); 1015234370Sjasone } else { 1016235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind, size, 1017235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind)); 1018235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1019235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1020234370Sjasone } 1021234370Sjasone 1022234370Sjasone /* Try to coalesce forward. */ 1023234370Sjasone if (run_ind + run_pages < chunk_npages && 1024235238Sjasone arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1025235238Sjasone arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1026235238Sjasone size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1027235238Sjasone run_ind+run_pages); 1028234370Sjasone size_t nrun_pages = nrun_size >> LG_PAGE; 1029234370Sjasone 1030234370Sjasone /* 1031234370Sjasone * Remove successor from runs_avail; the coalesced run is 1032234370Sjasone * inserted later. 1033234370Sjasone */ 1034235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, 1035235238Sjasone run_ind+run_pages+nrun_pages-1) == nrun_size); 1036235238Sjasone assert(arena_mapbits_dirty_get(chunk, 1037235238Sjasone run_ind+run_pages+nrun_pages-1) == flag_dirty); 1038242844Sjasone arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1039242844Sjasone false, true); 1040234370Sjasone 1041234370Sjasone size += nrun_size; 1042234370Sjasone run_pages += nrun_pages; 1043234370Sjasone 1044235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1045235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1046235238Sjasone size); 1047234370Sjasone } 1048234370Sjasone 1049234370Sjasone /* Try to coalesce backward. */ 1050235238Sjasone if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1) 1051235238Sjasone == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) { 1052235238Sjasone size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1053235238Sjasone run_ind-1); 1054234370Sjasone size_t prun_pages = prun_size >> LG_PAGE; 1055234370Sjasone 1056234370Sjasone run_ind -= prun_pages; 1057234370Sjasone 1058234370Sjasone /* 1059234370Sjasone * Remove predecessor from runs_avail; the coalesced run is 1060234370Sjasone * inserted later. 1061234370Sjasone */ 1062235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1063235238Sjasone prun_size); 1064235238Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1065242844Sjasone arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1066242844Sjasone false); 1067234370Sjasone 1068234370Sjasone size += prun_size; 1069234370Sjasone run_pages += prun_pages; 1070234370Sjasone 1071235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1072235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1073235238Sjasone size); 1074234370Sjasone } 1075234370Sjasone 1076234370Sjasone /* Insert into runs_avail, now that coalescing is complete. */ 1077235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1078235238Sjasone arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1079235238Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == 1080235238Sjasone arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1081242844Sjasone arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 1082234370Sjasone 1083235238Sjasone /* Deallocate chunk if it is now completely unused. */ 1084235238Sjasone if (size == arena_maxclass) { 1085235238Sjasone assert(run_ind == map_bias); 1086235238Sjasone assert(run_pages == (arena_maxclass >> LG_PAGE)); 1087234370Sjasone arena_chunk_dealloc(arena, chunk); 1088235238Sjasone } 1089234370Sjasone 1090234370Sjasone /* 1091234370Sjasone * It is okay to do dirty page processing here even if the chunk was 1092234370Sjasone * deallocated above, since in that case it is the spare. Waiting 1093234370Sjasone * until after possible chunk deallocation to do dirty processing 1094234370Sjasone * allows for an old spare to be fully deallocated, thus decreasing the 1095234370Sjasone * chances of spuriously crossing the dirty page purging threshold. 1096234370Sjasone */ 1097234370Sjasone if (dirty) 1098234370Sjasone arena_maybe_purge(arena); 1099234370Sjasone} 1100234370Sjasone 1101234370Sjasonestatic void 1102234370Sjasonearena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1103234370Sjasone size_t oldsize, size_t newsize) 1104234370Sjasone{ 1105234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1106234370Sjasone size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1107235238Sjasone size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1108234370Sjasone 1109234370Sjasone assert(oldsize > newsize); 1110234370Sjasone 1111234370Sjasone /* 1112234370Sjasone * Update the chunk map so that arena_run_dalloc() can treat the 1113234370Sjasone * leading run as separately allocated. Set the last element of each 1114234370Sjasone * run first, in case of single-page runs. 1115234370Sjasone */ 1116235238Sjasone assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1117235322Sjasone arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1118235322Sjasone arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1119234370Sjasone 1120234370Sjasone if (config_debug) { 1121234370Sjasone UNUSED size_t tail_npages = newsize >> LG_PAGE; 1122235238Sjasone assert(arena_mapbits_large_size_get(chunk, 1123235238Sjasone pageind+head_npages+tail_npages-1) == 0); 1124235238Sjasone assert(arena_mapbits_dirty_get(chunk, 1125235238Sjasone pageind+head_npages+tail_npages-1) == flag_dirty); 1126234370Sjasone } 1127235322Sjasone arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1128235322Sjasone flag_dirty); 1129234370Sjasone 1130242844Sjasone arena_run_dalloc(arena, run, false, false); 1131234370Sjasone} 1132234370Sjasone 1133234370Sjasonestatic void 1134234370Sjasonearena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1135234370Sjasone size_t oldsize, size_t newsize, bool dirty) 1136234370Sjasone{ 1137234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1138234370Sjasone size_t head_npages = newsize >> LG_PAGE; 1139235238Sjasone size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1140234370Sjasone 1141234370Sjasone assert(oldsize > newsize); 1142234370Sjasone 1143234370Sjasone /* 1144234370Sjasone * Update the chunk map so that arena_run_dalloc() can treat the 1145234370Sjasone * trailing run as separately allocated. Set the last element of each 1146234370Sjasone * run first, in case of single-page runs. 1147234370Sjasone */ 1148235238Sjasone assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1149235322Sjasone arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1150235322Sjasone arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1151234370Sjasone 1152235238Sjasone if (config_debug) { 1153235238Sjasone UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1154235238Sjasone assert(arena_mapbits_large_size_get(chunk, 1155235238Sjasone pageind+head_npages+tail_npages-1) == 0); 1156235238Sjasone assert(arena_mapbits_dirty_get(chunk, 1157235238Sjasone pageind+head_npages+tail_npages-1) == flag_dirty); 1158235238Sjasone } 1159235238Sjasone arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1160235322Sjasone flag_dirty); 1161234370Sjasone 1162234370Sjasone arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1163242844Sjasone dirty, false); 1164234370Sjasone} 1165234370Sjasone 1166234370Sjasonestatic arena_run_t * 1167234370Sjasonearena_bin_runs_first(arena_bin_t *bin) 1168234370Sjasone{ 1169234370Sjasone arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1170234370Sjasone if (mapelm != NULL) { 1171234370Sjasone arena_chunk_t *chunk; 1172234370Sjasone size_t pageind; 1173235238Sjasone arena_run_t *run; 1174234370Sjasone 1175234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 1176234370Sjasone pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 1177234370Sjasone sizeof(arena_chunk_map_t))) + map_bias; 1178235238Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1179235238Sjasone arena_mapbits_small_runind_get(chunk, pageind)) << 1180234370Sjasone LG_PAGE)); 1181234370Sjasone return (run); 1182234370Sjasone } 1183234370Sjasone 1184234370Sjasone return (NULL); 1185234370Sjasone} 1186234370Sjasone 1187234370Sjasonestatic void 1188234370Sjasonearena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1189234370Sjasone{ 1190234370Sjasone arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1191234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1192235238Sjasone arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1193234370Sjasone 1194234370Sjasone assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1195234370Sjasone 1196234370Sjasone arena_run_tree_insert(&bin->runs, mapelm); 1197234370Sjasone} 1198234370Sjasone 1199234370Sjasonestatic void 1200234370Sjasonearena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1201234370Sjasone{ 1202234370Sjasone arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1203234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1204235238Sjasone arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1205234370Sjasone 1206234370Sjasone assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1207234370Sjasone 1208234370Sjasone arena_run_tree_remove(&bin->runs, mapelm); 1209234370Sjasone} 1210234370Sjasone 1211234370Sjasonestatic arena_run_t * 1212234370Sjasonearena_bin_nonfull_run_tryget(arena_bin_t *bin) 1213234370Sjasone{ 1214234370Sjasone arena_run_t *run = arena_bin_runs_first(bin); 1215234370Sjasone if (run != NULL) { 1216234370Sjasone arena_bin_runs_remove(bin, run); 1217234370Sjasone if (config_stats) 1218234370Sjasone bin->stats.reruns++; 1219234370Sjasone } 1220234370Sjasone return (run); 1221234370Sjasone} 1222234370Sjasone 1223234370Sjasonestatic arena_run_t * 1224234370Sjasonearena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1225234370Sjasone{ 1226234370Sjasone arena_run_t *run; 1227234370Sjasone size_t binind; 1228234370Sjasone arena_bin_info_t *bin_info; 1229234370Sjasone 1230234370Sjasone /* Look for a usable run. */ 1231234370Sjasone run = arena_bin_nonfull_run_tryget(bin); 1232234370Sjasone if (run != NULL) 1233234370Sjasone return (run); 1234234370Sjasone /* No existing runs have any space available. */ 1235234370Sjasone 1236234370Sjasone binind = arena_bin_index(arena, bin); 1237234370Sjasone bin_info = &arena_bin_info[binind]; 1238234370Sjasone 1239234370Sjasone /* Allocate a new run. */ 1240234370Sjasone malloc_mutex_unlock(&bin->lock); 1241234370Sjasone /******************************/ 1242234370Sjasone malloc_mutex_lock(&arena->lock); 1243235238Sjasone run = arena_run_alloc(arena, bin_info->run_size, false, binind, false); 1244234370Sjasone if (run != NULL) { 1245234370Sjasone bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 1246234370Sjasone (uintptr_t)bin_info->bitmap_offset); 1247234370Sjasone 1248234370Sjasone /* Initialize run internals. */ 1249235238Sjasone VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset - 1250235238Sjasone bin_info->redzone_size); 1251234370Sjasone run->bin = bin; 1252234370Sjasone run->nextind = 0; 1253234370Sjasone run->nfree = bin_info->nregs; 1254234370Sjasone bitmap_init(bitmap, &bin_info->bitmap_info); 1255234370Sjasone } 1256234370Sjasone malloc_mutex_unlock(&arena->lock); 1257234370Sjasone /********************************/ 1258234370Sjasone malloc_mutex_lock(&bin->lock); 1259234370Sjasone if (run != NULL) { 1260234370Sjasone if (config_stats) { 1261234370Sjasone bin->stats.nruns++; 1262234370Sjasone bin->stats.curruns++; 1263234370Sjasone } 1264234370Sjasone return (run); 1265234370Sjasone } 1266234370Sjasone 1267234370Sjasone /* 1268234370Sjasone * arena_run_alloc() failed, but another thread may have made 1269234370Sjasone * sufficient memory available while this one dropped bin->lock above, 1270234370Sjasone * so search one more time. 1271234370Sjasone */ 1272234370Sjasone run = arena_bin_nonfull_run_tryget(bin); 1273234370Sjasone if (run != NULL) 1274234370Sjasone return (run); 1275234370Sjasone 1276234370Sjasone return (NULL); 1277234370Sjasone} 1278234370Sjasone 1279234370Sjasone/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1280234370Sjasonestatic void * 1281234370Sjasonearena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1282234370Sjasone{ 1283234370Sjasone void *ret; 1284234370Sjasone size_t binind; 1285234370Sjasone arena_bin_info_t *bin_info; 1286234370Sjasone arena_run_t *run; 1287234370Sjasone 1288234370Sjasone binind = arena_bin_index(arena, bin); 1289234370Sjasone bin_info = &arena_bin_info[binind]; 1290234370Sjasone bin->runcur = NULL; 1291234370Sjasone run = arena_bin_nonfull_run_get(arena, bin); 1292234370Sjasone if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1293234370Sjasone /* 1294234370Sjasone * Another thread updated runcur while this one ran without the 1295234370Sjasone * bin lock in arena_bin_nonfull_run_get(). 1296234370Sjasone */ 1297234370Sjasone assert(bin->runcur->nfree > 0); 1298234370Sjasone ret = arena_run_reg_alloc(bin->runcur, bin_info); 1299234370Sjasone if (run != NULL) { 1300234370Sjasone arena_chunk_t *chunk; 1301234370Sjasone 1302234370Sjasone /* 1303234370Sjasone * arena_run_alloc() may have allocated run, or it may 1304234370Sjasone * have pulled run from the bin's run tree. Therefore 1305234370Sjasone * it is unsafe to make any assumptions about how run 1306234370Sjasone * has previously been used, and arena_bin_lower_run() 1307234370Sjasone * must be called, as if a region were just deallocated 1308234370Sjasone * from the run. 1309234370Sjasone */ 1310234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1311234370Sjasone if (run->nfree == bin_info->nregs) 1312234370Sjasone arena_dalloc_bin_run(arena, chunk, run, bin); 1313234370Sjasone else 1314234370Sjasone arena_bin_lower_run(arena, chunk, run, bin); 1315234370Sjasone } 1316234370Sjasone return (ret); 1317234370Sjasone } 1318234370Sjasone 1319234370Sjasone if (run == NULL) 1320234370Sjasone return (NULL); 1321234370Sjasone 1322234370Sjasone bin->runcur = run; 1323234370Sjasone 1324234370Sjasone assert(bin->runcur->nfree > 0); 1325234370Sjasone 1326234370Sjasone return (arena_run_reg_alloc(bin->runcur, bin_info)); 1327234370Sjasone} 1328234370Sjasone 1329234370Sjasonevoid 1330234370Sjasonearena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 1331234370Sjasone uint64_t prof_accumbytes) 1332234370Sjasone{ 1333234370Sjasone unsigned i, nfill; 1334234370Sjasone arena_bin_t *bin; 1335234370Sjasone arena_run_t *run; 1336234370Sjasone void *ptr; 1337234370Sjasone 1338234370Sjasone assert(tbin->ncached == 0); 1339234370Sjasone 1340245868Sjasone if (config_prof) 1341234370Sjasone arena_prof_accum(arena, prof_accumbytes); 1342234370Sjasone bin = &arena->bins[binind]; 1343234370Sjasone malloc_mutex_lock(&bin->lock); 1344234370Sjasone for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 1345234370Sjasone tbin->lg_fill_div); i < nfill; i++) { 1346234370Sjasone if ((run = bin->runcur) != NULL && run->nfree > 0) 1347234370Sjasone ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1348234370Sjasone else 1349234370Sjasone ptr = arena_bin_malloc_hard(arena, bin); 1350234370Sjasone if (ptr == NULL) 1351234370Sjasone break; 1352234370Sjasone if (config_fill && opt_junk) { 1353234370Sjasone arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1354234370Sjasone true); 1355234370Sjasone } 1356234370Sjasone /* Insert such that low regions get used first. */ 1357234370Sjasone tbin->avail[nfill - 1 - i] = ptr; 1358234370Sjasone } 1359234370Sjasone if (config_stats) { 1360234370Sjasone bin->stats.allocated += i * arena_bin_info[binind].reg_size; 1361234370Sjasone bin->stats.nmalloc += i; 1362234370Sjasone bin->stats.nrequests += tbin->tstats.nrequests; 1363234370Sjasone bin->stats.nfills++; 1364234370Sjasone tbin->tstats.nrequests = 0; 1365234370Sjasone } 1366234370Sjasone malloc_mutex_unlock(&bin->lock); 1367234370Sjasone tbin->ncached = i; 1368234370Sjasone} 1369234370Sjasone 1370234370Sjasonevoid 1371234370Sjasonearena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1372234370Sjasone{ 1373234370Sjasone 1374234370Sjasone if (zero) { 1375234370Sjasone size_t redzone_size = bin_info->redzone_size; 1376234370Sjasone memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1377234370Sjasone redzone_size); 1378234370Sjasone memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1379234370Sjasone redzone_size); 1380234370Sjasone } else { 1381234370Sjasone memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1382234370Sjasone bin_info->reg_interval); 1383234370Sjasone } 1384234370Sjasone} 1385234370Sjasone 1386234370Sjasonevoid 1387234370Sjasonearena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 1388234370Sjasone{ 1389234370Sjasone size_t size = bin_info->reg_size; 1390234370Sjasone size_t redzone_size = bin_info->redzone_size; 1391234370Sjasone size_t i; 1392234370Sjasone bool error = false; 1393234370Sjasone 1394234370Sjasone for (i = 1; i <= redzone_size; i++) { 1395234370Sjasone unsigned byte; 1396234370Sjasone if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) { 1397234370Sjasone error = true; 1398234370Sjasone malloc_printf("<jemalloc>: Corrupt redzone " 1399234370Sjasone "%zu byte%s before %p (size %zu), byte=%#x\n", i, 1400234370Sjasone (i == 1) ? "" : "s", ptr, size, byte); 1401234370Sjasone } 1402234370Sjasone } 1403234370Sjasone for (i = 0; i < redzone_size; i++) { 1404234370Sjasone unsigned byte; 1405234370Sjasone if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) { 1406234370Sjasone error = true; 1407234370Sjasone malloc_printf("<jemalloc>: Corrupt redzone " 1408234370Sjasone "%zu byte%s after end of %p (size %zu), byte=%#x\n", 1409234370Sjasone i, (i == 1) ? "" : "s", ptr, size, byte); 1410234370Sjasone } 1411234370Sjasone } 1412234370Sjasone if (opt_abort && error) 1413234370Sjasone abort(); 1414234370Sjasone 1415234370Sjasone memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1416234370Sjasone bin_info->reg_interval); 1417234370Sjasone} 1418234370Sjasone 1419234370Sjasonevoid * 1420234370Sjasonearena_malloc_small(arena_t *arena, size_t size, bool zero) 1421234370Sjasone{ 1422234370Sjasone void *ret; 1423234370Sjasone arena_bin_t *bin; 1424234370Sjasone arena_run_t *run; 1425234370Sjasone size_t binind; 1426234370Sjasone 1427234370Sjasone binind = SMALL_SIZE2BIN(size); 1428234370Sjasone assert(binind < NBINS); 1429234370Sjasone bin = &arena->bins[binind]; 1430234370Sjasone size = arena_bin_info[binind].reg_size; 1431234370Sjasone 1432234370Sjasone malloc_mutex_lock(&bin->lock); 1433234370Sjasone if ((run = bin->runcur) != NULL && run->nfree > 0) 1434234370Sjasone ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1435234370Sjasone else 1436234370Sjasone ret = arena_bin_malloc_hard(arena, bin); 1437234370Sjasone 1438234370Sjasone if (ret == NULL) { 1439234370Sjasone malloc_mutex_unlock(&bin->lock); 1440234370Sjasone return (NULL); 1441234370Sjasone } 1442234370Sjasone 1443234370Sjasone if (config_stats) { 1444234370Sjasone bin->stats.allocated += size; 1445234370Sjasone bin->stats.nmalloc++; 1446234370Sjasone bin->stats.nrequests++; 1447234370Sjasone } 1448234370Sjasone malloc_mutex_unlock(&bin->lock); 1449245868Sjasone if (config_prof && isthreaded == false) 1450234370Sjasone arena_prof_accum(arena, size); 1451234370Sjasone 1452234370Sjasone if (zero == false) { 1453234370Sjasone if (config_fill) { 1454234370Sjasone if (opt_junk) { 1455234370Sjasone arena_alloc_junk_small(ret, 1456234370Sjasone &arena_bin_info[binind], false); 1457234370Sjasone } else if (opt_zero) 1458234370Sjasone memset(ret, 0, size); 1459234370Sjasone } 1460234370Sjasone } else { 1461234370Sjasone if (config_fill && opt_junk) { 1462234370Sjasone arena_alloc_junk_small(ret, &arena_bin_info[binind], 1463234370Sjasone true); 1464234370Sjasone } 1465234370Sjasone VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1466234370Sjasone memset(ret, 0, size); 1467245868Sjasone VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1468234370Sjasone } 1469234370Sjasone 1470234370Sjasone return (ret); 1471234370Sjasone} 1472234370Sjasone 1473234370Sjasonevoid * 1474234370Sjasonearena_malloc_large(arena_t *arena, size_t size, bool zero) 1475234370Sjasone{ 1476234370Sjasone void *ret; 1477234370Sjasone 1478234370Sjasone /* Large allocation. */ 1479234370Sjasone size = PAGE_CEILING(size); 1480234370Sjasone malloc_mutex_lock(&arena->lock); 1481235238Sjasone ret = (void *)arena_run_alloc(arena, size, true, BININD_INVALID, zero); 1482234370Sjasone if (ret == NULL) { 1483234370Sjasone malloc_mutex_unlock(&arena->lock); 1484234370Sjasone return (NULL); 1485234370Sjasone } 1486234370Sjasone if (config_stats) { 1487234370Sjasone arena->stats.nmalloc_large++; 1488234370Sjasone arena->stats.nrequests_large++; 1489234370Sjasone arena->stats.allocated_large += size; 1490234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1491234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1492234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1493234370Sjasone } 1494234370Sjasone if (config_prof) 1495245868Sjasone arena_prof_accum_locked(arena, size); 1496234370Sjasone malloc_mutex_unlock(&arena->lock); 1497234370Sjasone 1498234370Sjasone if (zero == false) { 1499234370Sjasone if (config_fill) { 1500234370Sjasone if (opt_junk) 1501234370Sjasone memset(ret, 0xa5, size); 1502234370Sjasone else if (opt_zero) 1503234370Sjasone memset(ret, 0, size); 1504234370Sjasone } 1505234370Sjasone } 1506234370Sjasone 1507234370Sjasone return (ret); 1508234370Sjasone} 1509234370Sjasone 1510234370Sjasone/* Only handles large allocations that require more than page alignment. */ 1511234370Sjasonevoid * 1512234370Sjasonearena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1513234370Sjasone{ 1514234370Sjasone void *ret; 1515234370Sjasone size_t alloc_size, leadsize, trailsize; 1516234370Sjasone arena_run_t *run; 1517234370Sjasone arena_chunk_t *chunk; 1518234370Sjasone 1519234370Sjasone assert((size & PAGE_MASK) == 0); 1520234370Sjasone 1521234370Sjasone alignment = PAGE_CEILING(alignment); 1522234370Sjasone alloc_size = size + alignment - PAGE; 1523234370Sjasone 1524234370Sjasone malloc_mutex_lock(&arena->lock); 1525235238Sjasone run = arena_run_alloc(arena, alloc_size, true, BININD_INVALID, zero); 1526234370Sjasone if (run == NULL) { 1527234370Sjasone malloc_mutex_unlock(&arena->lock); 1528234370Sjasone return (NULL); 1529234370Sjasone } 1530234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1531234370Sjasone 1532234370Sjasone leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 1533234370Sjasone (uintptr_t)run; 1534234370Sjasone assert(alloc_size >= leadsize + size); 1535234370Sjasone trailsize = alloc_size - leadsize - size; 1536234370Sjasone ret = (void *)((uintptr_t)run + leadsize); 1537234370Sjasone if (leadsize != 0) { 1538234370Sjasone arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 1539234370Sjasone leadsize); 1540234370Sjasone } 1541234370Sjasone if (trailsize != 0) { 1542234370Sjasone arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 1543234370Sjasone false); 1544234370Sjasone } 1545234370Sjasone 1546234370Sjasone if (config_stats) { 1547234370Sjasone arena->stats.nmalloc_large++; 1548234370Sjasone arena->stats.nrequests_large++; 1549234370Sjasone arena->stats.allocated_large += size; 1550234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1551234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1552234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1553234370Sjasone } 1554234370Sjasone malloc_mutex_unlock(&arena->lock); 1555234370Sjasone 1556234370Sjasone if (config_fill && zero == false) { 1557234370Sjasone if (opt_junk) 1558234370Sjasone memset(ret, 0xa5, size); 1559234370Sjasone else if (opt_zero) 1560234370Sjasone memset(ret, 0, size); 1561234370Sjasone } 1562234370Sjasone return (ret); 1563234370Sjasone} 1564234370Sjasone 1565234370Sjasonevoid 1566234370Sjasonearena_prof_promoted(const void *ptr, size_t size) 1567234370Sjasone{ 1568234370Sjasone arena_chunk_t *chunk; 1569234370Sjasone size_t pageind, binind; 1570234370Sjasone 1571234543Sjasone cassert(config_prof); 1572234370Sjasone assert(ptr != NULL); 1573234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 1574234370Sjasone assert(isalloc(ptr, false) == PAGE); 1575234370Sjasone assert(isalloc(ptr, true) == PAGE); 1576234370Sjasone assert(size <= SMALL_MAXCLASS); 1577234370Sjasone 1578234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1579234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1580234370Sjasone binind = SMALL_SIZE2BIN(size); 1581234370Sjasone assert(binind < NBINS); 1582235238Sjasone arena_mapbits_large_binind_set(chunk, pageind, binind); 1583234370Sjasone 1584234370Sjasone assert(isalloc(ptr, false) == PAGE); 1585234370Sjasone assert(isalloc(ptr, true) == size); 1586234370Sjasone} 1587234370Sjasone 1588234370Sjasonestatic void 1589234370Sjasonearena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1590234370Sjasone arena_bin_t *bin) 1591234370Sjasone{ 1592234370Sjasone 1593234370Sjasone /* Dissociate run from bin. */ 1594234370Sjasone if (run == bin->runcur) 1595234370Sjasone bin->runcur = NULL; 1596234370Sjasone else { 1597234370Sjasone size_t binind = arena_bin_index(chunk->arena, bin); 1598234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1599234370Sjasone 1600234370Sjasone if (bin_info->nregs != 1) { 1601234370Sjasone /* 1602234370Sjasone * This block's conditional is necessary because if the 1603234370Sjasone * run only contains one region, then it never gets 1604234370Sjasone * inserted into the non-full runs tree. 1605234370Sjasone */ 1606234370Sjasone arena_bin_runs_remove(bin, run); 1607234370Sjasone } 1608234370Sjasone } 1609234370Sjasone} 1610234370Sjasone 1611234370Sjasonestatic void 1612234370Sjasonearena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1613234370Sjasone arena_bin_t *bin) 1614234370Sjasone{ 1615234370Sjasone size_t binind; 1616234370Sjasone arena_bin_info_t *bin_info; 1617234370Sjasone size_t npages, run_ind, past; 1618234370Sjasone 1619234370Sjasone assert(run != bin->runcur); 1620235238Sjasone assert(arena_run_tree_search(&bin->runs, 1621235238Sjasone arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1622235238Sjasone == NULL); 1623234370Sjasone 1624234370Sjasone binind = arena_bin_index(chunk->arena, run->bin); 1625234370Sjasone bin_info = &arena_bin_info[binind]; 1626234370Sjasone 1627234370Sjasone malloc_mutex_unlock(&bin->lock); 1628234370Sjasone /******************************/ 1629234370Sjasone npages = bin_info->run_size >> LG_PAGE; 1630234370Sjasone run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1631234370Sjasone past = (size_t)(PAGE_CEILING((uintptr_t)run + 1632234370Sjasone (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1633234370Sjasone bin_info->reg_interval - bin_info->redzone_size) - 1634234370Sjasone (uintptr_t)chunk) >> LG_PAGE); 1635234370Sjasone malloc_mutex_lock(&arena->lock); 1636234370Sjasone 1637234370Sjasone /* 1638234370Sjasone * If the run was originally clean, and some pages were never touched, 1639234370Sjasone * trim the clean pages before deallocating the dirty portion of the 1640234370Sjasone * run. 1641234370Sjasone */ 1642235322Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == 1643235322Sjasone arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1644235238Sjasone if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1645235238Sjasone npages) { 1646235322Sjasone /* Trim clean pages. Convert to large run beforehand. */ 1647235322Sjasone assert(npages > 0); 1648235322Sjasone arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1649235322Sjasone arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1650234370Sjasone arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1651234370Sjasone ((past - run_ind) << LG_PAGE), false); 1652234370Sjasone /* npages = past - run_ind; */ 1653234370Sjasone } 1654242844Sjasone arena_run_dalloc(arena, run, true, false); 1655234370Sjasone malloc_mutex_unlock(&arena->lock); 1656234370Sjasone /****************************/ 1657234370Sjasone malloc_mutex_lock(&bin->lock); 1658234370Sjasone if (config_stats) 1659234370Sjasone bin->stats.curruns--; 1660234370Sjasone} 1661234370Sjasone 1662234370Sjasonestatic void 1663234370Sjasonearena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1664234370Sjasone arena_bin_t *bin) 1665234370Sjasone{ 1666234370Sjasone 1667234370Sjasone /* 1668234370Sjasone * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1669234370Sjasone * non-full run. It is okay to NULL runcur out rather than proactively 1670234370Sjasone * keeping it pointing at the lowest non-full run. 1671234370Sjasone */ 1672234370Sjasone if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1673234370Sjasone /* Switch runcur. */ 1674234370Sjasone if (bin->runcur->nfree > 0) 1675234370Sjasone arena_bin_runs_insert(bin, bin->runcur); 1676234370Sjasone bin->runcur = run; 1677234370Sjasone if (config_stats) 1678234370Sjasone bin->stats.reruns++; 1679234370Sjasone } else 1680234370Sjasone arena_bin_runs_insert(bin, run); 1681234370Sjasone} 1682234370Sjasone 1683234370Sjasonevoid 1684235238Sjasonearena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1685234370Sjasone arena_chunk_map_t *mapelm) 1686234370Sjasone{ 1687234370Sjasone size_t pageind; 1688234370Sjasone arena_run_t *run; 1689234370Sjasone arena_bin_t *bin; 1690235238Sjasone arena_bin_info_t *bin_info; 1691235238Sjasone size_t size, binind; 1692234370Sjasone 1693234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1694234370Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1695235238Sjasone arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1696234370Sjasone bin = run->bin; 1697235238Sjasone binind = arena_ptr_small_binind_get(ptr, mapelm->bits); 1698235238Sjasone bin_info = &arena_bin_info[binind]; 1699234370Sjasone if (config_fill || config_stats) 1700234370Sjasone size = bin_info->reg_size; 1701234370Sjasone 1702234370Sjasone if (config_fill && opt_junk) 1703234370Sjasone arena_dalloc_junk_small(ptr, bin_info); 1704234370Sjasone 1705234370Sjasone arena_run_reg_dalloc(run, ptr); 1706234370Sjasone if (run->nfree == bin_info->nregs) { 1707234370Sjasone arena_dissociate_bin_run(chunk, run, bin); 1708234370Sjasone arena_dalloc_bin_run(arena, chunk, run, bin); 1709234370Sjasone } else if (run->nfree == 1 && run != bin->runcur) 1710234370Sjasone arena_bin_lower_run(arena, chunk, run, bin); 1711234370Sjasone 1712234370Sjasone if (config_stats) { 1713234370Sjasone bin->stats.allocated -= size; 1714234370Sjasone bin->stats.ndalloc++; 1715234370Sjasone } 1716234370Sjasone} 1717234370Sjasone 1718234370Sjasonevoid 1719235238Sjasonearena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1720235238Sjasone size_t pageind, arena_chunk_map_t *mapelm) 1721235238Sjasone{ 1722235238Sjasone arena_run_t *run; 1723235238Sjasone arena_bin_t *bin; 1724235238Sjasone 1725235238Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1726235238Sjasone arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1727235238Sjasone bin = run->bin; 1728235238Sjasone malloc_mutex_lock(&bin->lock); 1729235238Sjasone arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 1730235238Sjasone malloc_mutex_unlock(&bin->lock); 1731235238Sjasone} 1732235238Sjasone 1733235238Sjasonevoid 1734235238Sjasonearena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1735235238Sjasone size_t pageind) 1736235238Sjasone{ 1737235238Sjasone arena_chunk_map_t *mapelm; 1738235238Sjasone 1739235238Sjasone if (config_debug) { 1740235238Sjasone /* arena_ptr_small_binind_get() does extra sanity checking. */ 1741235238Sjasone assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 1742235238Sjasone pageind)) != BININD_INVALID); 1743235238Sjasone } 1744235238Sjasone mapelm = arena_mapp_get(chunk, pageind); 1745235238Sjasone arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 1746235238Sjasone} 1747234370Sjasone 1748234370Sjasonevoid 1749235238Sjasonearena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1750234370Sjasone{ 1751234370Sjasone 1752234370Sjasone if (config_fill || config_stats) { 1753234370Sjasone size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1754235238Sjasone size_t size = arena_mapbits_large_size_get(chunk, pageind); 1755234370Sjasone 1756234370Sjasone if (config_fill && config_stats && opt_junk) 1757234370Sjasone memset(ptr, 0x5a, size); 1758234370Sjasone if (config_stats) { 1759234370Sjasone arena->stats.ndalloc_large++; 1760234370Sjasone arena->stats.allocated_large -= size; 1761234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].ndalloc++; 1762234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns--; 1763234370Sjasone } 1764234370Sjasone } 1765234370Sjasone 1766242844Sjasone arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 1767234370Sjasone} 1768234370Sjasone 1769235238Sjasonevoid 1770235238Sjasonearena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1771235238Sjasone{ 1772235238Sjasone 1773235238Sjasone malloc_mutex_lock(&arena->lock); 1774235238Sjasone arena_dalloc_large_locked(arena, chunk, ptr); 1775235238Sjasone malloc_mutex_unlock(&arena->lock); 1776235238Sjasone} 1777235238Sjasone 1778234370Sjasonestatic void 1779234370Sjasonearena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1780234370Sjasone size_t oldsize, size_t size) 1781234370Sjasone{ 1782234370Sjasone 1783234370Sjasone assert(size < oldsize); 1784234370Sjasone 1785234370Sjasone /* 1786234370Sjasone * Shrink the run, and make trailing pages available for other 1787234370Sjasone * allocations. 1788234370Sjasone */ 1789234370Sjasone malloc_mutex_lock(&arena->lock); 1790234370Sjasone arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1791234370Sjasone true); 1792234370Sjasone if (config_stats) { 1793234370Sjasone arena->stats.ndalloc_large++; 1794234370Sjasone arena->stats.allocated_large -= oldsize; 1795234370Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1796234370Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 1797234370Sjasone 1798234370Sjasone arena->stats.nmalloc_large++; 1799234370Sjasone arena->stats.nrequests_large++; 1800234370Sjasone arena->stats.allocated_large += size; 1801234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1802234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1803234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1804234370Sjasone } 1805234370Sjasone malloc_mutex_unlock(&arena->lock); 1806234370Sjasone} 1807234370Sjasone 1808234370Sjasonestatic bool 1809234370Sjasonearena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1810234370Sjasone size_t oldsize, size_t size, size_t extra, bool zero) 1811234370Sjasone{ 1812234370Sjasone size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1813234370Sjasone size_t npages = oldsize >> LG_PAGE; 1814234370Sjasone size_t followsize; 1815234370Sjasone 1816235238Sjasone assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 1817234370Sjasone 1818234370Sjasone /* Try to extend the run. */ 1819234370Sjasone assert(size + extra > oldsize); 1820234370Sjasone malloc_mutex_lock(&arena->lock); 1821234370Sjasone if (pageind + npages < chunk_npages && 1822235238Sjasone arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 1823235238Sjasone (followsize = arena_mapbits_unallocated_size_get(chunk, 1824235238Sjasone pageind+npages)) >= size - oldsize) { 1825234370Sjasone /* 1826234370Sjasone * The next run is available and sufficiently large. Split the 1827234370Sjasone * following run, then merge the first part with the existing 1828234370Sjasone * allocation. 1829234370Sjasone */ 1830234370Sjasone size_t flag_dirty; 1831234370Sjasone size_t splitsize = (oldsize + followsize <= size + extra) 1832234370Sjasone ? followsize : size + extra - oldsize; 1833234370Sjasone arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1834235238Sjasone ((pageind+npages) << LG_PAGE)), splitsize, true, 1835235238Sjasone BININD_INVALID, zero); 1836234370Sjasone 1837234370Sjasone size = oldsize + splitsize; 1838234370Sjasone npages = size >> LG_PAGE; 1839234370Sjasone 1840234370Sjasone /* 1841234370Sjasone * Mark the extended run as dirty if either portion of the run 1842234370Sjasone * was dirty before allocation. This is rather pedantic, 1843234370Sjasone * because there's not actually any sequence of events that 1844234370Sjasone * could cause the resulting run to be passed to 1845234370Sjasone * arena_run_dalloc() with the dirty argument set to false 1846234370Sjasone * (which is when dirty flag consistency would really matter). 1847234370Sjasone */ 1848235238Sjasone flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 1849235238Sjasone arena_mapbits_dirty_get(chunk, pageind+npages-1); 1850235238Sjasone arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 1851235238Sjasone arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 1852234370Sjasone 1853234370Sjasone if (config_stats) { 1854234370Sjasone arena->stats.ndalloc_large++; 1855234370Sjasone arena->stats.allocated_large -= oldsize; 1856235238Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1857235238Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 1858234370Sjasone 1859234370Sjasone arena->stats.nmalloc_large++; 1860234370Sjasone arena->stats.nrequests_large++; 1861234370Sjasone arena->stats.allocated_large += size; 1862234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1863235238Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1864234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1865234370Sjasone } 1866234370Sjasone malloc_mutex_unlock(&arena->lock); 1867234370Sjasone return (false); 1868234370Sjasone } 1869234370Sjasone malloc_mutex_unlock(&arena->lock); 1870234370Sjasone 1871234370Sjasone return (true); 1872234370Sjasone} 1873234370Sjasone 1874234370Sjasone/* 1875234370Sjasone * Try to resize a large allocation, in order to avoid copying. This will 1876234370Sjasone * always fail if growing an object, and the following run is already in use. 1877234370Sjasone */ 1878234370Sjasonestatic bool 1879234370Sjasonearena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 1880234370Sjasone bool zero) 1881234370Sjasone{ 1882234370Sjasone size_t psize; 1883234370Sjasone 1884234370Sjasone psize = PAGE_CEILING(size + extra); 1885234370Sjasone if (psize == oldsize) { 1886234370Sjasone /* Same size class. */ 1887234370Sjasone if (config_fill && opt_junk && size < oldsize) { 1888234370Sjasone memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1889234370Sjasone size); 1890234370Sjasone } 1891234370Sjasone return (false); 1892234370Sjasone } else { 1893234370Sjasone arena_chunk_t *chunk; 1894234370Sjasone arena_t *arena; 1895234370Sjasone 1896234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1897234370Sjasone arena = chunk->arena; 1898234370Sjasone 1899234370Sjasone if (psize < oldsize) { 1900234370Sjasone /* Fill before shrinking in order avoid a race. */ 1901234370Sjasone if (config_fill && opt_junk) { 1902234370Sjasone memset((void *)((uintptr_t)ptr + size), 0x5a, 1903234370Sjasone oldsize - size); 1904234370Sjasone } 1905234370Sjasone arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 1906234370Sjasone psize); 1907234370Sjasone return (false); 1908234370Sjasone } else { 1909234370Sjasone bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 1910234370Sjasone oldsize, PAGE_CEILING(size), 1911234370Sjasone psize - PAGE_CEILING(size), zero); 1912234370Sjasone if (config_fill && ret == false && zero == false && 1913234370Sjasone opt_zero) { 1914234370Sjasone memset((void *)((uintptr_t)ptr + oldsize), 0, 1915234370Sjasone size - oldsize); 1916234370Sjasone } 1917234370Sjasone return (ret); 1918234370Sjasone } 1919234370Sjasone } 1920234370Sjasone} 1921234370Sjasone 1922234370Sjasonevoid * 1923234370Sjasonearena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 1924234370Sjasone bool zero) 1925234370Sjasone{ 1926234370Sjasone 1927234370Sjasone /* 1928234370Sjasone * Avoid moving the allocation if the size class can be left the same. 1929234370Sjasone */ 1930234370Sjasone if (oldsize <= arena_maxclass) { 1931234370Sjasone if (oldsize <= SMALL_MAXCLASS) { 1932234370Sjasone assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 1933234370Sjasone == oldsize); 1934234370Sjasone if ((size + extra <= SMALL_MAXCLASS && 1935234370Sjasone SMALL_SIZE2BIN(size + extra) == 1936234370Sjasone SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 1937234370Sjasone size + extra >= oldsize)) { 1938234370Sjasone if (config_fill && opt_junk && size < oldsize) { 1939234370Sjasone memset((void *)((uintptr_t)ptr + size), 1940234370Sjasone 0x5a, oldsize - size); 1941234370Sjasone } 1942234370Sjasone return (ptr); 1943234370Sjasone } 1944234370Sjasone } else { 1945234370Sjasone assert(size <= arena_maxclass); 1946234370Sjasone if (size + extra > SMALL_MAXCLASS) { 1947234370Sjasone if (arena_ralloc_large(ptr, oldsize, size, 1948234370Sjasone extra, zero) == false) 1949234370Sjasone return (ptr); 1950234370Sjasone } 1951234370Sjasone } 1952234370Sjasone } 1953234370Sjasone 1954234370Sjasone /* Reallocation would require a move. */ 1955234370Sjasone return (NULL); 1956234370Sjasone} 1957234370Sjasone 1958234370Sjasonevoid * 1959242844Sjasonearena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 1960242844Sjasone size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 1961242844Sjasone bool try_tcache_dalloc) 1962234370Sjasone{ 1963234370Sjasone void *ret; 1964234370Sjasone size_t copysize; 1965234370Sjasone 1966234370Sjasone /* Try to avoid moving the allocation. */ 1967234370Sjasone ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); 1968234370Sjasone if (ret != NULL) 1969234370Sjasone return (ret); 1970234370Sjasone 1971234370Sjasone /* 1972234370Sjasone * size and oldsize are different enough that we need to move the 1973234370Sjasone * object. In that case, fall back to allocating new space and 1974234370Sjasone * copying. 1975234370Sjasone */ 1976234370Sjasone if (alignment != 0) { 1977234370Sjasone size_t usize = sa2u(size + extra, alignment); 1978234370Sjasone if (usize == 0) 1979234370Sjasone return (NULL); 1980242844Sjasone ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); 1981234370Sjasone } else 1982242844Sjasone ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 1983234370Sjasone 1984234370Sjasone if (ret == NULL) { 1985234370Sjasone if (extra == 0) 1986234370Sjasone return (NULL); 1987234370Sjasone /* Try again, this time without extra. */ 1988234370Sjasone if (alignment != 0) { 1989234370Sjasone size_t usize = sa2u(size, alignment); 1990234370Sjasone if (usize == 0) 1991234370Sjasone return (NULL); 1992242844Sjasone ret = ipallocx(usize, alignment, zero, try_tcache_alloc, 1993242844Sjasone arena); 1994234370Sjasone } else 1995242844Sjasone ret = arena_malloc(arena, size, zero, try_tcache_alloc); 1996234370Sjasone 1997234370Sjasone if (ret == NULL) 1998234370Sjasone return (NULL); 1999234370Sjasone } 2000234370Sjasone 2001234370Sjasone /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 2002234370Sjasone 2003234370Sjasone /* 2004234370Sjasone * Copy at most size bytes (not size+extra), since the caller has no 2005234370Sjasone * expectation that the extra bytes will be reliably preserved. 2006234370Sjasone */ 2007234370Sjasone copysize = (size < oldsize) ? size : oldsize; 2008235238Sjasone VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2009234370Sjasone memcpy(ret, ptr, copysize); 2010242844Sjasone iqallocx(ptr, try_tcache_dalloc); 2011234370Sjasone return (ret); 2012234370Sjasone} 2013234370Sjasone 2014242844Sjasonedss_prec_t 2015242844Sjasonearena_dss_prec_get(arena_t *arena) 2016242844Sjasone{ 2017242844Sjasone dss_prec_t ret; 2018242844Sjasone 2019242844Sjasone malloc_mutex_lock(&arena->lock); 2020242844Sjasone ret = arena->dss_prec; 2021242844Sjasone malloc_mutex_unlock(&arena->lock); 2022242844Sjasone return (ret); 2023242844Sjasone} 2024242844Sjasone 2025242844Sjasonevoid 2026242844Sjasonearena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2027242844Sjasone{ 2028242844Sjasone 2029242844Sjasone malloc_mutex_lock(&arena->lock); 2030242844Sjasone arena->dss_prec = dss_prec; 2031242844Sjasone malloc_mutex_unlock(&arena->lock); 2032242844Sjasone} 2033242844Sjasone 2034242844Sjasonevoid 2035242844Sjasonearena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2036242844Sjasone size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2037242844Sjasone malloc_large_stats_t *lstats) 2038242844Sjasone{ 2039242844Sjasone unsigned i; 2040242844Sjasone 2041242844Sjasone malloc_mutex_lock(&arena->lock); 2042242844Sjasone *dss = dss_prec_names[arena->dss_prec]; 2043242844Sjasone *nactive += arena->nactive; 2044242844Sjasone *ndirty += arena->ndirty; 2045242844Sjasone 2046242844Sjasone astats->mapped += arena->stats.mapped; 2047242844Sjasone astats->npurge += arena->stats.npurge; 2048242844Sjasone astats->nmadvise += arena->stats.nmadvise; 2049242844Sjasone astats->purged += arena->stats.purged; 2050242844Sjasone astats->allocated_large += arena->stats.allocated_large; 2051242844Sjasone astats->nmalloc_large += arena->stats.nmalloc_large; 2052242844Sjasone astats->ndalloc_large += arena->stats.ndalloc_large; 2053242844Sjasone astats->nrequests_large += arena->stats.nrequests_large; 2054242844Sjasone 2055242844Sjasone for (i = 0; i < nlclasses; i++) { 2056242844Sjasone lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2057242844Sjasone lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2058242844Sjasone lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2059242844Sjasone lstats[i].curruns += arena->stats.lstats[i].curruns; 2060242844Sjasone } 2061242844Sjasone malloc_mutex_unlock(&arena->lock); 2062242844Sjasone 2063242844Sjasone for (i = 0; i < NBINS; i++) { 2064242844Sjasone arena_bin_t *bin = &arena->bins[i]; 2065242844Sjasone 2066242844Sjasone malloc_mutex_lock(&bin->lock); 2067242844Sjasone bstats[i].allocated += bin->stats.allocated; 2068242844Sjasone bstats[i].nmalloc += bin->stats.nmalloc; 2069242844Sjasone bstats[i].ndalloc += bin->stats.ndalloc; 2070242844Sjasone bstats[i].nrequests += bin->stats.nrequests; 2071242844Sjasone if (config_tcache) { 2072242844Sjasone bstats[i].nfills += bin->stats.nfills; 2073242844Sjasone bstats[i].nflushes += bin->stats.nflushes; 2074242844Sjasone } 2075242844Sjasone bstats[i].nruns += bin->stats.nruns; 2076242844Sjasone bstats[i].reruns += bin->stats.reruns; 2077242844Sjasone bstats[i].curruns += bin->stats.curruns; 2078242844Sjasone malloc_mutex_unlock(&bin->lock); 2079242844Sjasone } 2080242844Sjasone} 2081242844Sjasone 2082234370Sjasonebool 2083234370Sjasonearena_new(arena_t *arena, unsigned ind) 2084234370Sjasone{ 2085234370Sjasone unsigned i; 2086234370Sjasone arena_bin_t *bin; 2087234370Sjasone 2088234370Sjasone arena->ind = ind; 2089234370Sjasone arena->nthreads = 0; 2090234370Sjasone 2091234370Sjasone if (malloc_mutex_init(&arena->lock)) 2092234370Sjasone return (true); 2093234370Sjasone 2094234370Sjasone if (config_stats) { 2095234370Sjasone memset(&arena->stats, 0, sizeof(arena_stats_t)); 2096234370Sjasone arena->stats.lstats = 2097234370Sjasone (malloc_large_stats_t *)base_alloc(nlclasses * 2098234370Sjasone sizeof(malloc_large_stats_t)); 2099234370Sjasone if (arena->stats.lstats == NULL) 2100234370Sjasone return (true); 2101234370Sjasone memset(arena->stats.lstats, 0, nlclasses * 2102234370Sjasone sizeof(malloc_large_stats_t)); 2103234370Sjasone if (config_tcache) 2104234370Sjasone ql_new(&arena->tcache_ql); 2105234370Sjasone } 2106234370Sjasone 2107234370Sjasone if (config_prof) 2108234370Sjasone arena->prof_accumbytes = 0; 2109234370Sjasone 2110242844Sjasone arena->dss_prec = chunk_dss_prec_get(); 2111242844Sjasone 2112234370Sjasone /* Initialize chunks. */ 2113242844Sjasone arena_chunk_dirty_new(&arena->chunks_dirty); 2114234370Sjasone arena->spare = NULL; 2115234370Sjasone 2116234370Sjasone arena->nactive = 0; 2117234370Sjasone arena->ndirty = 0; 2118234370Sjasone arena->npurgatory = 0; 2119234370Sjasone 2120242844Sjasone arena_avail_tree_new(&arena->runs_avail); 2121234370Sjasone 2122234370Sjasone /* Initialize bins. */ 2123234370Sjasone for (i = 0; i < NBINS; i++) { 2124234370Sjasone bin = &arena->bins[i]; 2125234370Sjasone if (malloc_mutex_init(&bin->lock)) 2126234370Sjasone return (true); 2127234370Sjasone bin->runcur = NULL; 2128234370Sjasone arena_run_tree_new(&bin->runs); 2129234370Sjasone if (config_stats) 2130234370Sjasone memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2131234370Sjasone } 2132234370Sjasone 2133234370Sjasone return (false); 2134234370Sjasone} 2135234370Sjasone 2136234370Sjasone/* 2137234370Sjasone * Calculate bin_info->run_size such that it meets the following constraints: 2138234370Sjasone * 2139234370Sjasone * *) bin_info->run_size >= min_run_size 2140234370Sjasone * *) bin_info->run_size <= arena_maxclass 2141234370Sjasone * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 2142234370Sjasone * *) bin_info->nregs <= RUN_MAXREGS 2143234370Sjasone * 2144234370Sjasone * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 2145234370Sjasone * calculated here, since these settings are all interdependent. 2146234370Sjasone */ 2147234370Sjasonestatic size_t 2148234370Sjasonebin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 2149234370Sjasone{ 2150234370Sjasone size_t pad_size; 2151234370Sjasone size_t try_run_size, good_run_size; 2152234370Sjasone uint32_t try_nregs, good_nregs; 2153234370Sjasone uint32_t try_hdr_size, good_hdr_size; 2154234370Sjasone uint32_t try_bitmap_offset, good_bitmap_offset; 2155234370Sjasone uint32_t try_ctx0_offset, good_ctx0_offset; 2156234370Sjasone uint32_t try_redzone0_offset, good_redzone0_offset; 2157234370Sjasone 2158234370Sjasone assert(min_run_size >= PAGE); 2159234370Sjasone assert(min_run_size <= arena_maxclass); 2160234370Sjasone 2161234370Sjasone /* 2162234370Sjasone * Determine redzone size based on minimum alignment and minimum 2163234370Sjasone * redzone size. Add padding to the end of the run if it is needed to 2164234370Sjasone * align the regions. The padding allows each redzone to be half the 2165234370Sjasone * minimum alignment; without the padding, each redzone would have to 2166234370Sjasone * be twice as large in order to maintain alignment. 2167234370Sjasone */ 2168234370Sjasone if (config_fill && opt_redzone) { 2169234370Sjasone size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); 2170234370Sjasone if (align_min <= REDZONE_MINSIZE) { 2171234370Sjasone bin_info->redzone_size = REDZONE_MINSIZE; 2172234370Sjasone pad_size = 0; 2173234370Sjasone } else { 2174234370Sjasone bin_info->redzone_size = align_min >> 1; 2175234370Sjasone pad_size = bin_info->redzone_size; 2176234370Sjasone } 2177234370Sjasone } else { 2178234370Sjasone bin_info->redzone_size = 0; 2179234370Sjasone pad_size = 0; 2180234370Sjasone } 2181234370Sjasone bin_info->reg_interval = bin_info->reg_size + 2182234370Sjasone (bin_info->redzone_size << 1); 2183234370Sjasone 2184234370Sjasone /* 2185234370Sjasone * Calculate known-valid settings before entering the run_size 2186234370Sjasone * expansion loop, so that the first part of the loop always copies 2187234370Sjasone * valid settings. 2188234370Sjasone * 2189234370Sjasone * The do..while loop iteratively reduces the number of regions until 2190234370Sjasone * the run header and the regions no longer overlap. A closed formula 2191234370Sjasone * would be quite messy, since there is an interdependency between the 2192234370Sjasone * header's mask length and the number of regions. 2193234370Sjasone */ 2194234370Sjasone try_run_size = min_run_size; 2195234370Sjasone try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2196234370Sjasone bin_info->reg_interval) 2197234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2198234370Sjasone if (try_nregs > RUN_MAXREGS) { 2199234370Sjasone try_nregs = RUN_MAXREGS 2200234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2201234370Sjasone } 2202234370Sjasone do { 2203234370Sjasone try_nregs--; 2204234370Sjasone try_hdr_size = sizeof(arena_run_t); 2205234370Sjasone /* Pad to a long boundary. */ 2206234370Sjasone try_hdr_size = LONG_CEILING(try_hdr_size); 2207234370Sjasone try_bitmap_offset = try_hdr_size; 2208234370Sjasone /* Add space for bitmap. */ 2209234370Sjasone try_hdr_size += bitmap_size(try_nregs); 2210234370Sjasone if (config_prof && opt_prof && prof_promote == false) { 2211234370Sjasone /* Pad to a quantum boundary. */ 2212234370Sjasone try_hdr_size = QUANTUM_CEILING(try_hdr_size); 2213234370Sjasone try_ctx0_offset = try_hdr_size; 2214234370Sjasone /* Add space for one (prof_ctx_t *) per region. */ 2215234370Sjasone try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 2216234370Sjasone } else 2217234370Sjasone try_ctx0_offset = 0; 2218234370Sjasone try_redzone0_offset = try_run_size - (try_nregs * 2219234370Sjasone bin_info->reg_interval) - pad_size; 2220234370Sjasone } while (try_hdr_size > try_redzone0_offset); 2221234370Sjasone 2222234370Sjasone /* run_size expansion loop. */ 2223234370Sjasone do { 2224234370Sjasone /* 2225234370Sjasone * Copy valid settings before trying more aggressive settings. 2226234370Sjasone */ 2227234370Sjasone good_run_size = try_run_size; 2228234370Sjasone good_nregs = try_nregs; 2229234370Sjasone good_hdr_size = try_hdr_size; 2230234370Sjasone good_bitmap_offset = try_bitmap_offset; 2231234370Sjasone good_ctx0_offset = try_ctx0_offset; 2232234370Sjasone good_redzone0_offset = try_redzone0_offset; 2233234370Sjasone 2234234370Sjasone /* Try more aggressive settings. */ 2235234370Sjasone try_run_size += PAGE; 2236234370Sjasone try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2237234370Sjasone bin_info->reg_interval) 2238234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2239234370Sjasone if (try_nregs > RUN_MAXREGS) { 2240234370Sjasone try_nregs = RUN_MAXREGS 2241234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2242234370Sjasone } 2243234370Sjasone do { 2244234370Sjasone try_nregs--; 2245234370Sjasone try_hdr_size = sizeof(arena_run_t); 2246234370Sjasone /* Pad to a long boundary. */ 2247234370Sjasone try_hdr_size = LONG_CEILING(try_hdr_size); 2248234370Sjasone try_bitmap_offset = try_hdr_size; 2249234370Sjasone /* Add space for bitmap. */ 2250234370Sjasone try_hdr_size += bitmap_size(try_nregs); 2251234370Sjasone if (config_prof && opt_prof && prof_promote == false) { 2252234370Sjasone /* Pad to a quantum boundary. */ 2253234370Sjasone try_hdr_size = QUANTUM_CEILING(try_hdr_size); 2254234370Sjasone try_ctx0_offset = try_hdr_size; 2255234370Sjasone /* 2256234370Sjasone * Add space for one (prof_ctx_t *) per region. 2257234370Sjasone */ 2258234370Sjasone try_hdr_size += try_nregs * 2259234370Sjasone sizeof(prof_ctx_t *); 2260234370Sjasone } 2261234370Sjasone try_redzone0_offset = try_run_size - (try_nregs * 2262234370Sjasone bin_info->reg_interval) - pad_size; 2263234370Sjasone } while (try_hdr_size > try_redzone0_offset); 2264234370Sjasone } while (try_run_size <= arena_maxclass 2265234370Sjasone && try_run_size <= arena_maxclass 2266234370Sjasone && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2267234370Sjasone RUN_MAX_OVRHD_RELAX 2268234370Sjasone && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 2269234370Sjasone && try_nregs < RUN_MAXREGS); 2270234370Sjasone 2271234370Sjasone assert(good_hdr_size <= good_redzone0_offset); 2272234370Sjasone 2273234370Sjasone /* Copy final settings. */ 2274234370Sjasone bin_info->run_size = good_run_size; 2275234370Sjasone bin_info->nregs = good_nregs; 2276234370Sjasone bin_info->bitmap_offset = good_bitmap_offset; 2277234370Sjasone bin_info->ctx0_offset = good_ctx0_offset; 2278234370Sjasone bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2279234370Sjasone 2280234370Sjasone assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2281234370Sjasone * bin_info->reg_interval) + pad_size == bin_info->run_size); 2282234370Sjasone 2283234370Sjasone return (good_run_size); 2284234370Sjasone} 2285234370Sjasone 2286234370Sjasonestatic void 2287234370Sjasonebin_info_init(void) 2288234370Sjasone{ 2289234370Sjasone arena_bin_info_t *bin_info; 2290234370Sjasone size_t prev_run_size = PAGE; 2291234370Sjasone 2292234370Sjasone#define SIZE_CLASS(bin, delta, size) \ 2293234370Sjasone bin_info = &arena_bin_info[bin]; \ 2294234370Sjasone bin_info->reg_size = size; \ 2295234370Sjasone prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2296234370Sjasone bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2297234370Sjasone SIZE_CLASSES 2298234370Sjasone#undef SIZE_CLASS 2299234370Sjasone} 2300234370Sjasone 2301234370Sjasonevoid 2302234370Sjasonearena_boot(void) 2303234370Sjasone{ 2304234370Sjasone size_t header_size; 2305234370Sjasone unsigned i; 2306234370Sjasone 2307234370Sjasone /* 2308234370Sjasone * Compute the header size such that it is large enough to contain the 2309234370Sjasone * page map. The page map is biased to omit entries for the header 2310234370Sjasone * itself, so some iteration is necessary to compute the map bias. 2311234370Sjasone * 2312234370Sjasone * 1) Compute safe header_size and map_bias values that include enough 2313234370Sjasone * space for an unbiased page map. 2314234370Sjasone * 2) Refine map_bias based on (1) to omit the header pages in the page 2315234370Sjasone * map. The resulting map_bias may be one too small. 2316234370Sjasone * 3) Refine map_bias based on (2). The result will be >= the result 2317234370Sjasone * from (2), and will always be correct. 2318234370Sjasone */ 2319234370Sjasone map_bias = 0; 2320234370Sjasone for (i = 0; i < 3; i++) { 2321234370Sjasone header_size = offsetof(arena_chunk_t, map) + 2322234370Sjasone (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2323234370Sjasone map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2324234370Sjasone != 0); 2325234370Sjasone } 2326234370Sjasone assert(map_bias > 0); 2327234370Sjasone 2328234370Sjasone arena_maxclass = chunksize - (map_bias << LG_PAGE); 2329234370Sjasone 2330234370Sjasone bin_info_init(); 2331234370Sjasone} 2332234370Sjasone 2333234370Sjasonevoid 2334234370Sjasonearena_prefork(arena_t *arena) 2335234370Sjasone{ 2336234370Sjasone unsigned i; 2337234370Sjasone 2338234370Sjasone malloc_mutex_prefork(&arena->lock); 2339234370Sjasone for (i = 0; i < NBINS; i++) 2340234370Sjasone malloc_mutex_prefork(&arena->bins[i].lock); 2341234370Sjasone} 2342234370Sjasone 2343234370Sjasonevoid 2344234370Sjasonearena_postfork_parent(arena_t *arena) 2345234370Sjasone{ 2346234370Sjasone unsigned i; 2347234370Sjasone 2348234370Sjasone for (i = 0; i < NBINS; i++) 2349234370Sjasone malloc_mutex_postfork_parent(&arena->bins[i].lock); 2350234370Sjasone malloc_mutex_postfork_parent(&arena->lock); 2351234370Sjasone} 2352234370Sjasone 2353234370Sjasonevoid 2354234370Sjasonearena_postfork_child(arena_t *arena) 2355234370Sjasone{ 2356234370Sjasone unsigned i; 2357234370Sjasone 2358234370Sjasone for (i = 0; i < NBINS; i++) 2359234370Sjasone malloc_mutex_postfork_child(&arena->bins[i].lock); 2360234370Sjasone malloc_mutex_postfork_child(&arena->lock); 2361234370Sjasone} 2362