1234370Sjasone#define JEMALLOC_ARENA_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone 4234370Sjasone/******************************************************************************/ 5234370Sjasone/* Data. */ 6234370Sjasone 7234370Sjasonessize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8234370Sjasonearena_bin_info_t arena_bin_info[NBINS]; 9234370Sjasone 10235238SjasoneJEMALLOC_ALIGNED(CACHELINE) 11234370Sjasoneconst uint8_t small_size2bin[] = { 12234370Sjasone#define S2B_8(i) i, 13234370Sjasone#define S2B_16(i) S2B_8(i) S2B_8(i) 14234370Sjasone#define S2B_32(i) S2B_16(i) S2B_16(i) 15234370Sjasone#define S2B_64(i) S2B_32(i) S2B_32(i) 16234370Sjasone#define S2B_128(i) S2B_64(i) S2B_64(i) 17234370Sjasone#define S2B_256(i) S2B_128(i) S2B_128(i) 18234370Sjasone#define S2B_512(i) S2B_256(i) S2B_256(i) 19234370Sjasone#define S2B_1024(i) S2B_512(i) S2B_512(i) 20234370Sjasone#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21234370Sjasone#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22234370Sjasone#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23234370Sjasone#define SIZE_CLASS(bin, delta, size) \ 24234370Sjasone S2B_##delta(bin) 25234370Sjasone SIZE_CLASSES 26234370Sjasone#undef S2B_8 27234370Sjasone#undef S2B_16 28234370Sjasone#undef S2B_32 29234370Sjasone#undef S2B_64 30234370Sjasone#undef S2B_128 31234370Sjasone#undef S2B_256 32234370Sjasone#undef S2B_512 33234370Sjasone#undef S2B_1024 34234370Sjasone#undef S2B_2048 35234370Sjasone#undef S2B_4096 36234370Sjasone#undef S2B_8192 37234370Sjasone#undef SIZE_CLASS 38234370Sjasone}; 39234370Sjasone 40234370Sjasone/******************************************************************************/ 41234370Sjasone/* Function prototypes for non-inline static functions. */ 42234370Sjasone 43242844Sjasonestatic void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, 44242844Sjasone size_t pageind, size_t npages, bool maybe_adjac_pred, 45242844Sjasone bool maybe_adjac_succ); 46242844Sjasonestatic void arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, 47242844Sjasone size_t pageind, size_t npages, bool maybe_adjac_pred, 48242844Sjasone bool maybe_adjac_succ); 49234370Sjasonestatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 50235238Sjasone bool large, size_t binind, bool zero); 51234370Sjasonestatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 52234370Sjasonestatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 53235322Sjasonestatic arena_run_t *arena_run_alloc_helper(arena_t *arena, size_t size, 54235322Sjasone bool large, size_t binind, bool zero); 55234370Sjasonestatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 56235238Sjasone size_t binind, bool zero); 57242844Sjasonestatic arena_chunk_t *chunks_dirty_iter_cb(arena_chunk_tree_t *tree, 58242844Sjasone arena_chunk_t *chunk, void *arg); 59234370Sjasonestatic void arena_purge(arena_t *arena, bool all); 60242844Sjasonestatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 61242844Sjasone bool cleaned); 62234370Sjasonestatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 63234370Sjasone arena_run_t *run, size_t oldsize, size_t newsize); 64234370Sjasonestatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 65234370Sjasone arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 66234370Sjasonestatic arena_run_t *arena_bin_runs_first(arena_bin_t *bin); 67234370Sjasonestatic void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run); 68234370Sjasonestatic void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run); 69234370Sjasonestatic arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin); 70234370Sjasonestatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 71234370Sjasonestatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 72234370Sjasonestatic void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 73234370Sjasone arena_bin_t *bin); 74234370Sjasonestatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 75234370Sjasone arena_run_t *run, arena_bin_t *bin); 76234370Sjasonestatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 77234370Sjasone arena_run_t *run, arena_bin_t *bin); 78234370Sjasonestatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 79234370Sjasone void *ptr, size_t oldsize, size_t size); 80234370Sjasonestatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 81234370Sjasone void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); 82234370Sjasonestatic bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, 83234370Sjasone size_t extra, bool zero); 84234370Sjasonestatic size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, 85234370Sjasone size_t min_run_size); 86234370Sjasonestatic void bin_info_init(void); 87234370Sjasone 88234370Sjasone/******************************************************************************/ 89234370Sjasone 90234370Sjasonestatic inline int 91234370Sjasonearena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 92234370Sjasone{ 93234370Sjasone uintptr_t a_mapelm = (uintptr_t)a; 94234370Sjasone uintptr_t b_mapelm = (uintptr_t)b; 95234370Sjasone 96234370Sjasone assert(a != NULL); 97234370Sjasone assert(b != NULL); 98234370Sjasone 99234370Sjasone return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 100234370Sjasone} 101234370Sjasone 102234370Sjasone/* Generate red-black tree functions. */ 103234370Sjasonerb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 104234370Sjasone u.rb_link, arena_run_comp) 105234370Sjasone 106234370Sjasonestatic inline int 107234370Sjasonearena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 108234370Sjasone{ 109234370Sjasone int ret; 110234370Sjasone size_t a_size = a->bits & ~PAGE_MASK; 111234370Sjasone size_t b_size = b->bits & ~PAGE_MASK; 112234370Sjasone 113234370Sjasone ret = (a_size > b_size) - (a_size < b_size); 114234370Sjasone if (ret == 0) { 115234370Sjasone uintptr_t a_mapelm, b_mapelm; 116234370Sjasone 117234370Sjasone if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 118234370Sjasone a_mapelm = (uintptr_t)a; 119234370Sjasone else { 120234370Sjasone /* 121234370Sjasone * Treat keys as though they are lower than anything 122234370Sjasone * else. 123234370Sjasone */ 124234370Sjasone a_mapelm = 0; 125234370Sjasone } 126234370Sjasone b_mapelm = (uintptr_t)b; 127234370Sjasone 128234370Sjasone ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 129234370Sjasone } 130234370Sjasone 131234370Sjasone return (ret); 132234370Sjasone} 133234370Sjasone 134234370Sjasone/* Generate red-black tree functions. */ 135234370Sjasonerb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 136234370Sjasone u.rb_link, arena_avail_comp) 137234370Sjasone 138242844Sjasonestatic inline int 139242844Sjasonearena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 140242844Sjasone{ 141242844Sjasone 142242844Sjasone assert(a != NULL); 143242844Sjasone assert(b != NULL); 144242844Sjasone 145242844Sjasone /* 146242844Sjasone * Short-circuit for self comparison. The following comparison code 147242844Sjasone * would come to the same result, but at the cost of executing the slow 148242844Sjasone * path. 149242844Sjasone */ 150242844Sjasone if (a == b) 151242844Sjasone return (0); 152242844Sjasone 153242844Sjasone /* 154242844Sjasone * Order such that chunks with higher fragmentation are "less than" 155242844Sjasone * those with lower fragmentation -- purging order is from "least" to 156242844Sjasone * "greatest". Fragmentation is measured as: 157242844Sjasone * 158242844Sjasone * mean current avail run size 159242844Sjasone * -------------------------------- 160242844Sjasone * mean defragmented avail run size 161242844Sjasone * 162242844Sjasone * navail 163242844Sjasone * ----------- 164242844Sjasone * nruns_avail nruns_avail-nruns_adjac 165242844Sjasone * = ========================= = ----------------------- 166242844Sjasone * navail nruns_avail 167242844Sjasone * ----------------------- 168242844Sjasone * nruns_avail-nruns_adjac 169242844Sjasone * 170242844Sjasone * The following code multiplies away the denominator prior to 171242844Sjasone * comparison, in order to avoid division. 172242844Sjasone * 173242844Sjasone */ 174242844Sjasone { 175242844Sjasone size_t a_val = (a->nruns_avail - a->nruns_adjac) * 176242844Sjasone b->nruns_avail; 177242844Sjasone size_t b_val = (b->nruns_avail - b->nruns_adjac) * 178242844Sjasone a->nruns_avail; 179242844Sjasone 180242844Sjasone if (a_val < b_val) 181242844Sjasone return (1); 182242844Sjasone if (a_val > b_val) 183242844Sjasone return (-1); 184242844Sjasone } 185242844Sjasone /* 186242844Sjasone * Break ties by chunk address. For fragmented chunks, report lower 187242844Sjasone * addresses as "lower", so that fragmentation reduction happens first 188242844Sjasone * at lower addresses. However, use the opposite ordering for 189242844Sjasone * unfragmented chunks, in order to increase the chances of 190242844Sjasone * re-allocating dirty runs. 191242844Sjasone */ 192242844Sjasone { 193242844Sjasone uintptr_t a_chunk = (uintptr_t)a; 194242844Sjasone uintptr_t b_chunk = (uintptr_t)b; 195242844Sjasone int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 196242844Sjasone if (a->nruns_adjac == 0) { 197242844Sjasone assert(b->nruns_adjac == 0); 198242844Sjasone ret = -ret; 199242844Sjasone } 200242844Sjasone return (ret); 201242844Sjasone } 202242844Sjasone} 203242844Sjasone 204242844Sjasone/* Generate red-black tree functions. */ 205242844Sjasonerb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 206242844Sjasone dirty_link, arena_chunk_dirty_comp) 207242844Sjasone 208242844Sjasonestatic inline bool 209242844Sjasonearena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 210242844Sjasone{ 211242844Sjasone bool ret; 212242844Sjasone 213242844Sjasone if (pageind-1 < map_bias) 214242844Sjasone ret = false; 215242844Sjasone else { 216242844Sjasone ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 217242844Sjasone assert(ret == false || arena_mapbits_dirty_get(chunk, 218242844Sjasone pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 219242844Sjasone } 220242844Sjasone return (ret); 221242844Sjasone} 222242844Sjasone 223242844Sjasonestatic inline bool 224242844Sjasonearena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 225242844Sjasone{ 226242844Sjasone bool ret; 227242844Sjasone 228242844Sjasone if (pageind+npages == chunk_npages) 229242844Sjasone ret = false; 230242844Sjasone else { 231242844Sjasone assert(pageind+npages < chunk_npages); 232242844Sjasone ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 233242844Sjasone assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 234242844Sjasone != arena_mapbits_dirty_get(chunk, pageind+npages)); 235242844Sjasone } 236242844Sjasone return (ret); 237242844Sjasone} 238242844Sjasone 239242844Sjasonestatic inline bool 240242844Sjasonearena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 241242844Sjasone{ 242242844Sjasone 243242844Sjasone return (arena_avail_adjac_pred(chunk, pageind) || 244242844Sjasone arena_avail_adjac_succ(chunk, pageind, npages)); 245242844Sjasone} 246242844Sjasone 247242844Sjasonestatic void 248242844Sjasonearena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 249242844Sjasone size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 250242844Sjasone{ 251242844Sjasone 252242844Sjasone assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 253242844Sjasone LG_PAGE)); 254242844Sjasone 255242844Sjasone /* 256242844Sjasone * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 257242844Sjasone * removed and reinserted even if the run to be inserted is clean. 258242844Sjasone */ 259242844Sjasone if (chunk->ndirty != 0) 260242844Sjasone arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 261242844Sjasone 262242844Sjasone if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 263242844Sjasone chunk->nruns_adjac++; 264242844Sjasone if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 265242844Sjasone chunk->nruns_adjac++; 266242844Sjasone chunk->nruns_avail++; 267242844Sjasone assert(chunk->nruns_avail > chunk->nruns_adjac); 268242844Sjasone 269242844Sjasone if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 270242844Sjasone arena->ndirty += npages; 271242844Sjasone chunk->ndirty += npages; 272242844Sjasone } 273242844Sjasone if (chunk->ndirty != 0) 274242844Sjasone arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 275242844Sjasone 276242844Sjasone arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 277242844Sjasone pageind)); 278242844Sjasone} 279242844Sjasone 280242844Sjasonestatic void 281242844Sjasonearena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 282242844Sjasone size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 283242844Sjasone{ 284242844Sjasone 285242844Sjasone assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 286242844Sjasone LG_PAGE)); 287242844Sjasone 288242844Sjasone /* 289242844Sjasone * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 290242844Sjasone * removed and reinserted even if the run to be removed is clean. 291242844Sjasone */ 292242844Sjasone if (chunk->ndirty != 0) 293242844Sjasone arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 294242844Sjasone 295242844Sjasone if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 296242844Sjasone chunk->nruns_adjac--; 297242844Sjasone if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 298242844Sjasone chunk->nruns_adjac--; 299242844Sjasone chunk->nruns_avail--; 300242844Sjasone assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 301242844Sjasone == 0 && chunk->nruns_adjac == 0)); 302242844Sjasone 303242844Sjasone if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 304242844Sjasone arena->ndirty -= npages; 305242844Sjasone chunk->ndirty -= npages; 306242844Sjasone } 307242844Sjasone if (chunk->ndirty != 0) 308242844Sjasone arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 309242844Sjasone 310242844Sjasone arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 311242844Sjasone pageind)); 312242844Sjasone} 313242844Sjasone 314234370Sjasonestatic inline void * 315234370Sjasonearena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 316234370Sjasone{ 317234370Sjasone void *ret; 318234370Sjasone unsigned regind; 319234370Sjasone bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 320234370Sjasone (uintptr_t)bin_info->bitmap_offset); 321234370Sjasone 322234370Sjasone assert(run->nfree > 0); 323234370Sjasone assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 324234370Sjasone 325234370Sjasone regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 326234370Sjasone ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 327234370Sjasone (uintptr_t)(bin_info->reg_interval * regind)); 328234370Sjasone run->nfree--; 329234370Sjasone if (regind == run->nextind) 330234370Sjasone run->nextind++; 331234370Sjasone assert(regind < run->nextind); 332234370Sjasone return (ret); 333234370Sjasone} 334234370Sjasone 335234370Sjasonestatic inline void 336234370Sjasonearena_run_reg_dalloc(arena_run_t *run, void *ptr) 337234370Sjasone{ 338234370Sjasone arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 339235238Sjasone size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 340235238Sjasone size_t mapbits = arena_mapbits_get(chunk, pageind); 341235238Sjasone size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 342234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 343234370Sjasone unsigned regind = arena_run_regind(run, bin_info, ptr); 344234370Sjasone bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 345234370Sjasone (uintptr_t)bin_info->bitmap_offset); 346234370Sjasone 347234370Sjasone assert(run->nfree < bin_info->nregs); 348234370Sjasone /* Freeing an interior pointer can cause assertion failure. */ 349234370Sjasone assert(((uintptr_t)ptr - ((uintptr_t)run + 350234370Sjasone (uintptr_t)bin_info->reg0_offset)) % 351234370Sjasone (uintptr_t)bin_info->reg_interval == 0); 352234370Sjasone assert((uintptr_t)ptr >= (uintptr_t)run + 353234370Sjasone (uintptr_t)bin_info->reg0_offset); 354234370Sjasone /* Freeing an unallocated pointer can cause assertion failure. */ 355234370Sjasone assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 356234370Sjasone 357234370Sjasone bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 358234370Sjasone run->nfree++; 359234370Sjasone} 360234370Sjasone 361234370Sjasonestatic inline void 362245868Sjasonearena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 363234370Sjasone{ 364245868Sjasone 365245868Sjasone VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 366245868Sjasone LG_PAGE)), (npages << LG_PAGE)); 367245868Sjasone memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 368245868Sjasone (npages << LG_PAGE)); 369245868Sjasone} 370245868Sjasone 371245868Sjasonestatic inline void 372245868Sjasonearena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 373245868Sjasone{ 374234370Sjasone size_t i; 375234370Sjasone UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 376234370Sjasone 377245868Sjasone VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << 378245868Sjasone LG_PAGE)), PAGE); 379234370Sjasone for (i = 0; i < PAGE / sizeof(size_t); i++) 380234370Sjasone assert(p[i] == 0); 381234370Sjasone} 382234370Sjasone 383234370Sjasonestatic void 384234370Sjasonearena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 385235238Sjasone size_t binind, bool zero) 386234370Sjasone{ 387234370Sjasone arena_chunk_t *chunk; 388234370Sjasone size_t run_ind, total_pages, need_pages, rem_pages, i; 389234370Sjasone size_t flag_dirty; 390234370Sjasone 391235238Sjasone assert((large && binind == BININD_INVALID) || (large == false && binind 392235238Sjasone != BININD_INVALID)); 393235238Sjasone 394234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 395234370Sjasone run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 396235238Sjasone flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 397235238Sjasone total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 398234370Sjasone LG_PAGE; 399235238Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 400235238Sjasone flag_dirty); 401234370Sjasone need_pages = (size >> LG_PAGE); 402234370Sjasone assert(need_pages > 0); 403234370Sjasone assert(need_pages <= total_pages); 404234370Sjasone rem_pages = total_pages - need_pages; 405234370Sjasone 406242844Sjasone arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 407234370Sjasone if (config_stats) { 408234370Sjasone /* 409234370Sjasone * Update stats_cactive if nactive is crossing a chunk 410234370Sjasone * multiple. 411234370Sjasone */ 412234370Sjasone size_t cactive_diff = CHUNK_CEILING((arena->nactive + 413234370Sjasone need_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 414234370Sjasone LG_PAGE); 415234370Sjasone if (cactive_diff != 0) 416234370Sjasone stats_cactive_add(cactive_diff); 417234370Sjasone } 418234370Sjasone arena->nactive += need_pages; 419234370Sjasone 420234370Sjasone /* Keep track of trailing unused pages for later use. */ 421234370Sjasone if (rem_pages > 0) { 422234370Sjasone if (flag_dirty != 0) { 423235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 424235238Sjasone (rem_pages << LG_PAGE), CHUNK_MAP_DIRTY); 425235238Sjasone arena_mapbits_unallocated_set(chunk, 426235238Sjasone run_ind+total_pages-1, (rem_pages << LG_PAGE), 427235238Sjasone CHUNK_MAP_DIRTY); 428234370Sjasone } else { 429235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 430235238Sjasone (rem_pages << LG_PAGE), 431235238Sjasone arena_mapbits_unzeroed_get(chunk, 432235238Sjasone run_ind+need_pages)); 433235238Sjasone arena_mapbits_unallocated_set(chunk, 434235238Sjasone run_ind+total_pages-1, (rem_pages << LG_PAGE), 435235238Sjasone arena_mapbits_unzeroed_get(chunk, 436235238Sjasone run_ind+total_pages-1)); 437234370Sjasone } 438242844Sjasone arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 439242844Sjasone false, true); 440234370Sjasone } 441234370Sjasone 442234370Sjasone /* 443234370Sjasone * Update the page map separately for large vs. small runs, since it is 444234370Sjasone * possible to avoid iteration for large mallocs. 445234370Sjasone */ 446234370Sjasone if (large) { 447234370Sjasone if (zero) { 448234370Sjasone if (flag_dirty == 0) { 449234370Sjasone /* 450234370Sjasone * The run is clean, so some pages may be 451234370Sjasone * zeroed (i.e. never before touched). 452234370Sjasone */ 453234370Sjasone for (i = 0; i < need_pages; i++) { 454235238Sjasone if (arena_mapbits_unzeroed_get(chunk, 455235238Sjasone run_ind+i) != 0) { 456245868Sjasone arena_run_zero(chunk, run_ind+i, 457245868Sjasone 1); 458234370Sjasone } else if (config_debug) { 459245868Sjasone arena_run_page_validate_zeroed( 460234370Sjasone chunk, run_ind+i); 461234370Sjasone } 462234370Sjasone } 463234370Sjasone } else { 464234370Sjasone /* 465234370Sjasone * The run is dirty, so all pages must be 466234370Sjasone * zeroed. 467234370Sjasone */ 468245868Sjasone arena_run_zero(chunk, run_ind, need_pages); 469234370Sjasone } 470234370Sjasone } 471234370Sjasone 472234370Sjasone /* 473234370Sjasone * Set the last element first, in case the run only contains one 474234370Sjasone * page (i.e. both statements set the same element). 475234370Sjasone */ 476235238Sjasone arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, 477235238Sjasone flag_dirty); 478235238Sjasone arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 479234370Sjasone } else { 480234370Sjasone assert(zero == false); 481234370Sjasone /* 482234370Sjasone * Propagate the dirty and unzeroed flags to the allocated 483234370Sjasone * small run, so that arena_dalloc_bin_run() has the ability to 484234370Sjasone * conditionally trim clean pages. 485234370Sjasone */ 486235322Sjasone arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 487234370Sjasone /* 488234370Sjasone * The first page will always be dirtied during small run 489234370Sjasone * initialization, so a validation failure here would not 490234370Sjasone * actually cause an observable failure. 491234370Sjasone */ 492234370Sjasone if (config_debug && flag_dirty == 0 && 493235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind) == 0) 494245868Sjasone arena_run_page_validate_zeroed(chunk, run_ind); 495234370Sjasone for (i = 1; i < need_pages - 1; i++) { 496235322Sjasone arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 497234370Sjasone if (config_debug && flag_dirty == 0 && 498245868Sjasone arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) { 499245868Sjasone arena_run_page_validate_zeroed(chunk, 500245868Sjasone run_ind+i); 501245868Sjasone } 502234370Sjasone } 503235238Sjasone arena_mapbits_small_set(chunk, run_ind+need_pages-1, 504235322Sjasone need_pages-1, binind, flag_dirty); 505234370Sjasone if (config_debug && flag_dirty == 0 && 506235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == 507235238Sjasone 0) { 508245868Sjasone arena_run_page_validate_zeroed(chunk, 509234370Sjasone run_ind+need_pages-1); 510234370Sjasone } 511234370Sjasone } 512251300Sjasone VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 513251300Sjasone LG_PAGE)), (need_pages << LG_PAGE)); 514234370Sjasone} 515234370Sjasone 516234370Sjasonestatic arena_chunk_t * 517234370Sjasonearena_chunk_alloc(arena_t *arena) 518234370Sjasone{ 519234370Sjasone arena_chunk_t *chunk; 520234370Sjasone size_t i; 521234370Sjasone 522234370Sjasone if (arena->spare != NULL) { 523234370Sjasone chunk = arena->spare; 524234370Sjasone arena->spare = NULL; 525234370Sjasone 526235322Sjasone assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 527235322Sjasone assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 528235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 529235238Sjasone arena_maxclass); 530235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, 531235238Sjasone chunk_npages-1) == arena_maxclass); 532235238Sjasone assert(arena_mapbits_dirty_get(chunk, map_bias) == 533235238Sjasone arena_mapbits_dirty_get(chunk, chunk_npages-1)); 534234370Sjasone } else { 535234370Sjasone bool zero; 536234370Sjasone size_t unzeroed; 537234370Sjasone 538234370Sjasone zero = false; 539234370Sjasone malloc_mutex_unlock(&arena->lock); 540234370Sjasone chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, 541242844Sjasone false, &zero, arena->dss_prec); 542234370Sjasone malloc_mutex_lock(&arena->lock); 543234370Sjasone if (chunk == NULL) 544234370Sjasone return (NULL); 545234370Sjasone if (config_stats) 546234370Sjasone arena->stats.mapped += chunksize; 547234370Sjasone 548234370Sjasone chunk->arena = arena; 549234370Sjasone 550234370Sjasone /* 551234370Sjasone * Claim that no pages are in use, since the header is merely 552234370Sjasone * overhead. 553234370Sjasone */ 554234370Sjasone chunk->ndirty = 0; 555234370Sjasone 556242844Sjasone chunk->nruns_avail = 0; 557242844Sjasone chunk->nruns_adjac = 0; 558242844Sjasone 559234370Sjasone /* 560234370Sjasone * Initialize the map to contain one maximal free untouched run. 561234370Sjasone * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 562234370Sjasone * chunk. 563234370Sjasone */ 564234370Sjasone unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 565235238Sjasone arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 566235238Sjasone unzeroed); 567234370Sjasone /* 568234370Sjasone * There is no need to initialize the internal page map entries 569234370Sjasone * unless the chunk is not zeroed. 570234370Sjasone */ 571234370Sjasone if (zero == false) { 572234370Sjasone for (i = map_bias+1; i < chunk_npages-1; i++) 573235238Sjasone arena_mapbits_unzeroed_set(chunk, i, unzeroed); 574234370Sjasone } else if (config_debug) { 575251300Sjasone VALGRIND_MAKE_MEM_DEFINED( 576251300Sjasone (void *)arena_mapp_get(chunk, map_bias+1), 577251300Sjasone (void *)((uintptr_t) 578251300Sjasone arena_mapp_get(chunk, chunk_npages-1) 579251300Sjasone - (uintptr_t)arena_mapp_get(chunk, map_bias+1))); 580235238Sjasone for (i = map_bias+1; i < chunk_npages-1; i++) { 581235238Sjasone assert(arena_mapbits_unzeroed_get(chunk, i) == 582235238Sjasone unzeroed); 583235238Sjasone } 584234370Sjasone } 585235238Sjasone arena_mapbits_unallocated_set(chunk, chunk_npages-1, 586235238Sjasone arena_maxclass, unzeroed); 587234370Sjasone } 588234370Sjasone 589242844Sjasone /* Insert the run into the runs_avail tree. */ 590242844Sjasone arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 591242844Sjasone false, false); 592242844Sjasone 593234370Sjasone return (chunk); 594234370Sjasone} 595234370Sjasone 596234370Sjasonestatic void 597234370Sjasonearena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 598234370Sjasone{ 599235322Sjasone assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 600235322Sjasone assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 601235322Sjasone assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 602235322Sjasone arena_maxclass); 603235322Sjasone assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 604235322Sjasone arena_maxclass); 605235322Sjasone assert(arena_mapbits_dirty_get(chunk, map_bias) == 606235322Sjasone arena_mapbits_dirty_get(chunk, chunk_npages-1)); 607235322Sjasone 608234370Sjasone /* 609242844Sjasone * Remove run from the runs_avail tree, so that the arena does not use 610242844Sjasone * it. 611234370Sjasone */ 612242844Sjasone arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 613242844Sjasone false, false); 614234370Sjasone 615234370Sjasone if (arena->spare != NULL) { 616234370Sjasone arena_chunk_t *spare = arena->spare; 617234370Sjasone 618234370Sjasone arena->spare = chunk; 619234370Sjasone malloc_mutex_unlock(&arena->lock); 620234370Sjasone chunk_dealloc((void *)spare, chunksize, true); 621234370Sjasone malloc_mutex_lock(&arena->lock); 622234370Sjasone if (config_stats) 623234370Sjasone arena->stats.mapped -= chunksize; 624234370Sjasone } else 625234370Sjasone arena->spare = chunk; 626234370Sjasone} 627234370Sjasone 628234370Sjasonestatic arena_run_t * 629235322Sjasonearena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, 630235238Sjasone bool zero) 631234370Sjasone{ 632234370Sjasone arena_run_t *run; 633234370Sjasone arena_chunk_map_t *mapelm, key; 634234370Sjasone 635234370Sjasone key.bits = size | CHUNK_MAP_KEY; 636242844Sjasone mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); 637234370Sjasone if (mapelm != NULL) { 638234370Sjasone arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 639234370Sjasone size_t pageind = (((uintptr_t)mapelm - 640234370Sjasone (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 641234370Sjasone + map_bias; 642234370Sjasone 643234370Sjasone run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 644234370Sjasone LG_PAGE)); 645235238Sjasone arena_run_split(arena, run, size, large, binind, zero); 646234370Sjasone return (run); 647234370Sjasone } 648234370Sjasone 649235322Sjasone return (NULL); 650235322Sjasone} 651235322Sjasone 652235322Sjasonestatic arena_run_t * 653235322Sjasonearena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, 654235322Sjasone bool zero) 655235322Sjasone{ 656235322Sjasone arena_chunk_t *chunk; 657235322Sjasone arena_run_t *run; 658235322Sjasone 659235322Sjasone assert(size <= arena_maxclass); 660235322Sjasone assert((size & PAGE_MASK) == 0); 661235322Sjasone assert((large && binind == BININD_INVALID) || (large == false && binind 662235322Sjasone != BININD_INVALID)); 663235322Sjasone 664235322Sjasone /* Search the arena's chunks for the lowest best fit. */ 665235322Sjasone run = arena_run_alloc_helper(arena, size, large, binind, zero); 666235322Sjasone if (run != NULL) 667235322Sjasone return (run); 668235322Sjasone 669234370Sjasone /* 670234370Sjasone * No usable runs. Create a new chunk from which to allocate the run. 671234370Sjasone */ 672234370Sjasone chunk = arena_chunk_alloc(arena); 673234370Sjasone if (chunk != NULL) { 674234370Sjasone run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 675235238Sjasone arena_run_split(arena, run, size, large, binind, zero); 676234370Sjasone return (run); 677234370Sjasone } 678234370Sjasone 679234370Sjasone /* 680234370Sjasone * arena_chunk_alloc() failed, but another thread may have made 681234370Sjasone * sufficient memory available while this one dropped arena->lock in 682234370Sjasone * arena_chunk_alloc(), so search one more time. 683234370Sjasone */ 684235322Sjasone return (arena_run_alloc_helper(arena, size, large, binind, zero)); 685234370Sjasone} 686234370Sjasone 687234370Sjasonestatic inline void 688234370Sjasonearena_maybe_purge(arena_t *arena) 689234370Sjasone{ 690242844Sjasone size_t npurgeable, threshold; 691234370Sjasone 692242844Sjasone /* Don't purge if the option is disabled. */ 693242844Sjasone if (opt_lg_dirty_mult < 0) 694242844Sjasone return; 695242844Sjasone /* Don't purge if all dirty pages are already being purged. */ 696242844Sjasone if (arena->ndirty <= arena->npurgatory) 697242844Sjasone return; 698242844Sjasone npurgeable = arena->ndirty - arena->npurgatory; 699242844Sjasone threshold = (arena->nactive >> opt_lg_dirty_mult); 700242844Sjasone /* 701242844Sjasone * Don't purge unless the number of purgeable pages exceeds the 702242844Sjasone * threshold. 703242844Sjasone */ 704242844Sjasone if (npurgeable <= threshold) 705242844Sjasone return; 706242844Sjasone 707242844Sjasone arena_purge(arena, false); 708234370Sjasone} 709234370Sjasone 710242844Sjasonestatic inline size_t 711242844Sjasonearena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 712234370Sjasone{ 713242844Sjasone size_t npurged; 714234370Sjasone ql_head(arena_chunk_map_t) mapelms; 715234370Sjasone arena_chunk_map_t *mapelm; 716242844Sjasone size_t pageind, npages; 717234370Sjasone size_t nmadvise; 718234370Sjasone 719234370Sjasone ql_new(&mapelms); 720234370Sjasone 721234370Sjasone /* 722234370Sjasone * If chunk is the spare, temporarily re-allocate it, 1) so that its 723242844Sjasone * run is reinserted into runs_avail, and 2) so that it cannot be 724234370Sjasone * completely discarded by another thread while arena->lock is dropped 725234370Sjasone * by this thread. Note that the arena_run_dalloc() call will 726234370Sjasone * implicitly deallocate the chunk, so no explicit action is required 727234370Sjasone * in this function to deallocate the chunk. 728234370Sjasone * 729234370Sjasone * Note that once a chunk contains dirty pages, it cannot again contain 730234370Sjasone * a single run unless 1) it is a dirty run, or 2) this function purges 731234370Sjasone * dirty pages and causes the transition to a single clean run. Thus 732234370Sjasone * (chunk == arena->spare) is possible, but it is not possible for 733234370Sjasone * this function to be called on the spare unless it contains a dirty 734234370Sjasone * run. 735234370Sjasone */ 736234370Sjasone if (chunk == arena->spare) { 737235238Sjasone assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 738235322Sjasone assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 739235322Sjasone 740234370Sjasone arena_chunk_alloc(arena); 741234370Sjasone } 742234370Sjasone 743242844Sjasone if (config_stats) 744242844Sjasone arena->stats.purged += chunk->ndirty; 745242844Sjasone 746242844Sjasone /* 747242844Sjasone * Operate on all dirty runs if there is no clean/dirty run 748242844Sjasone * fragmentation. 749242844Sjasone */ 750242844Sjasone if (chunk->nruns_adjac == 0) 751242844Sjasone all = true; 752242844Sjasone 753242844Sjasone /* 754242844Sjasone * Temporarily allocate free dirty runs within chunk. If all is false, 755242844Sjasone * only operate on dirty runs that are fragments; otherwise operate on 756242844Sjasone * all dirty runs. 757242844Sjasone */ 758242844Sjasone for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 759235238Sjasone mapelm = arena_mapp_get(chunk, pageind); 760235238Sjasone if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 761242844Sjasone size_t run_size = 762242844Sjasone arena_mapbits_unallocated_size_get(chunk, pageind); 763234370Sjasone 764242844Sjasone npages = run_size >> LG_PAGE; 765234370Sjasone assert(pageind + npages <= chunk_npages); 766235322Sjasone assert(arena_mapbits_dirty_get(chunk, pageind) == 767235322Sjasone arena_mapbits_dirty_get(chunk, pageind+npages-1)); 768234370Sjasone 769242844Sjasone if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 770242844Sjasone (all || arena_avail_adjac(chunk, pageind, 771242844Sjasone npages))) { 772242844Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t) 773242844Sjasone chunk + (uintptr_t)(pageind << LG_PAGE)); 774234370Sjasone 775242844Sjasone arena_run_split(arena, run, run_size, true, 776242844Sjasone BININD_INVALID, false); 777234370Sjasone /* Append to list for later processing. */ 778234370Sjasone ql_elm_new(mapelm, u.ql_link); 779234370Sjasone ql_tail_insert(&mapelms, mapelm, u.ql_link); 780234370Sjasone } 781234370Sjasone } else { 782242844Sjasone /* Skip run. */ 783242844Sjasone if (arena_mapbits_large_get(chunk, pageind) != 0) { 784242844Sjasone npages = arena_mapbits_large_size_get(chunk, 785235238Sjasone pageind) >> LG_PAGE; 786242844Sjasone } else { 787235238Sjasone size_t binind; 788235238Sjasone arena_bin_info_t *bin_info; 789234370Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t) 790234370Sjasone chunk + (uintptr_t)(pageind << LG_PAGE)); 791234370Sjasone 792235238Sjasone assert(arena_mapbits_small_runind_get(chunk, 793235238Sjasone pageind) == 0); 794235238Sjasone binind = arena_bin_index(arena, run->bin); 795235238Sjasone bin_info = &arena_bin_info[binind]; 796242844Sjasone npages = bin_info->run_size >> LG_PAGE; 797234370Sjasone } 798234370Sjasone } 799234370Sjasone } 800234370Sjasone assert(pageind == chunk_npages); 801242844Sjasone assert(chunk->ndirty == 0 || all == false); 802242844Sjasone assert(chunk->nruns_adjac == 0); 803234370Sjasone 804234370Sjasone malloc_mutex_unlock(&arena->lock); 805234370Sjasone if (config_stats) 806234370Sjasone nmadvise = 0; 807242844Sjasone npurged = 0; 808234370Sjasone ql_foreach(mapelm, &mapelms, u.ql_link) { 809242844Sjasone bool unzeroed; 810242844Sjasone size_t flag_unzeroed, i; 811242844Sjasone 812242844Sjasone pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 813234370Sjasone sizeof(arena_chunk_map_t)) + map_bias; 814242844Sjasone npages = arena_mapbits_large_size_get(chunk, pageind) >> 815235238Sjasone LG_PAGE; 816234370Sjasone assert(pageind + npages <= chunk_npages); 817242844Sjasone unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 818242844Sjasone LG_PAGE)), (npages << LG_PAGE)); 819242844Sjasone flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 820242844Sjasone /* 821242844Sjasone * Set the unzeroed flag for all pages, now that pages_purge() 822242844Sjasone * has returned whether the pages were zeroed as a side effect 823242844Sjasone * of purging. This chunk map modification is safe even though 824242844Sjasone * the arena mutex isn't currently owned by this thread, 825242844Sjasone * because the run is marked as allocated, thus protecting it 826242844Sjasone * from being modified by any other thread. As long as these 827242844Sjasone * writes don't perturb the first and last elements' 828242844Sjasone * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 829242844Sjasone */ 830242844Sjasone for (i = 0; i < npages; i++) { 831242844Sjasone arena_mapbits_unzeroed_set(chunk, pageind+i, 832242844Sjasone flag_unzeroed); 833242844Sjasone } 834242844Sjasone npurged += npages; 835234370Sjasone if (config_stats) 836234370Sjasone nmadvise++; 837234370Sjasone } 838234370Sjasone malloc_mutex_lock(&arena->lock); 839234370Sjasone if (config_stats) 840234370Sjasone arena->stats.nmadvise += nmadvise; 841234370Sjasone 842234370Sjasone /* Deallocate runs. */ 843234370Sjasone for (mapelm = ql_first(&mapelms); mapelm != NULL; 844234370Sjasone mapelm = ql_first(&mapelms)) { 845242844Sjasone arena_run_t *run; 846242844Sjasone 847242844Sjasone pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 848234370Sjasone sizeof(arena_chunk_map_t)) + map_bias; 849242844Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 850242844Sjasone LG_PAGE)); 851234370Sjasone ql_remove(&mapelms, mapelm, u.ql_link); 852242844Sjasone arena_run_dalloc(arena, run, false, true); 853234370Sjasone } 854242844Sjasone 855242844Sjasone return (npurged); 856234370Sjasone} 857234370Sjasone 858242844Sjasonestatic arena_chunk_t * 859242844Sjasonechunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 860242844Sjasone{ 861242844Sjasone size_t *ndirty = (size_t *)arg; 862242844Sjasone 863242844Sjasone assert(chunk->ndirty != 0); 864242844Sjasone *ndirty += chunk->ndirty; 865242844Sjasone return (NULL); 866242844Sjasone} 867242844Sjasone 868234370Sjasonestatic void 869234370Sjasonearena_purge(arena_t *arena, bool all) 870234370Sjasone{ 871234370Sjasone arena_chunk_t *chunk; 872234370Sjasone size_t npurgatory; 873234370Sjasone if (config_debug) { 874234370Sjasone size_t ndirty = 0; 875234370Sjasone 876242844Sjasone arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 877242844Sjasone chunks_dirty_iter_cb, (void *)&ndirty); 878234370Sjasone assert(ndirty == arena->ndirty); 879234370Sjasone } 880234370Sjasone assert(arena->ndirty > arena->npurgatory || all); 881234370Sjasone assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 882234370Sjasone arena->npurgatory) || all); 883234370Sjasone 884234370Sjasone if (config_stats) 885234370Sjasone arena->stats.npurge++; 886234370Sjasone 887234370Sjasone /* 888234370Sjasone * Compute the minimum number of pages that this thread should try to 889234370Sjasone * purge, and add the result to arena->npurgatory. This will keep 890234370Sjasone * multiple threads from racing to reduce ndirty below the threshold. 891234370Sjasone */ 892242844Sjasone { 893242844Sjasone size_t npurgeable = arena->ndirty - arena->npurgatory; 894242844Sjasone 895242844Sjasone if (all == false) { 896242844Sjasone size_t threshold = (arena->nactive >> 897242844Sjasone opt_lg_dirty_mult); 898242844Sjasone 899242844Sjasone npurgatory = npurgeable - threshold; 900242844Sjasone } else 901242844Sjasone npurgatory = npurgeable; 902234370Sjasone } 903234370Sjasone arena->npurgatory += npurgatory; 904234370Sjasone 905234370Sjasone while (npurgatory > 0) { 906242844Sjasone size_t npurgeable, npurged, nunpurged; 907242844Sjasone 908234370Sjasone /* Get next chunk with dirty pages. */ 909242844Sjasone chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 910234370Sjasone if (chunk == NULL) { 911234370Sjasone /* 912234370Sjasone * This thread was unable to purge as many pages as 913234370Sjasone * originally intended, due to races with other threads 914234370Sjasone * that either did some of the purging work, or re-used 915234370Sjasone * dirty pages. 916234370Sjasone */ 917234370Sjasone arena->npurgatory -= npurgatory; 918234370Sjasone return; 919234370Sjasone } 920242844Sjasone npurgeable = chunk->ndirty; 921242844Sjasone assert(npurgeable != 0); 922234370Sjasone 923242844Sjasone if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 924234370Sjasone /* 925242844Sjasone * This thread will purge all the dirty pages in chunk, 926242844Sjasone * so set npurgatory to reflect this thread's intent to 927242844Sjasone * purge the pages. This tends to reduce the chances 928242844Sjasone * of the following scenario: 929234370Sjasone * 930234370Sjasone * 1) This thread sets arena->npurgatory such that 931234370Sjasone * (arena->ndirty - arena->npurgatory) is at the 932234370Sjasone * threshold. 933234370Sjasone * 2) This thread drops arena->lock. 934234370Sjasone * 3) Another thread causes one or more pages to be 935234370Sjasone * dirtied, and immediately determines that it must 936234370Sjasone * purge dirty pages. 937234370Sjasone * 938234370Sjasone * If this scenario *does* play out, that's okay, 939234370Sjasone * because all of the purging work being done really 940234370Sjasone * needs to happen. 941234370Sjasone */ 942242844Sjasone arena->npurgatory += npurgeable - npurgatory; 943242844Sjasone npurgatory = npurgeable; 944234370Sjasone } 945234370Sjasone 946242844Sjasone /* 947242844Sjasone * Keep track of how many pages are purgeable, versus how many 948242844Sjasone * actually get purged, and adjust counters accordingly. 949242844Sjasone */ 950242844Sjasone arena->npurgatory -= npurgeable; 951242844Sjasone npurgatory -= npurgeable; 952242844Sjasone npurged = arena_chunk_purge(arena, chunk, all); 953242844Sjasone nunpurged = npurgeable - npurged; 954242844Sjasone arena->npurgatory += nunpurged; 955242844Sjasone npurgatory += nunpurged; 956234370Sjasone } 957234370Sjasone} 958234370Sjasone 959234370Sjasonevoid 960234370Sjasonearena_purge_all(arena_t *arena) 961234370Sjasone{ 962234370Sjasone 963234370Sjasone malloc_mutex_lock(&arena->lock); 964234370Sjasone arena_purge(arena, true); 965234370Sjasone malloc_mutex_unlock(&arena->lock); 966234370Sjasone} 967234370Sjasone 968234370Sjasonestatic void 969242844Sjasonearena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 970234370Sjasone{ 971234370Sjasone arena_chunk_t *chunk; 972234370Sjasone size_t size, run_ind, run_pages, flag_dirty; 973234370Sjasone 974234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 975234370Sjasone run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 976234370Sjasone assert(run_ind >= map_bias); 977234370Sjasone assert(run_ind < chunk_npages); 978235238Sjasone if (arena_mapbits_large_get(chunk, run_ind) != 0) { 979235238Sjasone size = arena_mapbits_large_size_get(chunk, run_ind); 980234370Sjasone assert(size == PAGE || 981235238Sjasone arena_mapbits_large_size_get(chunk, 982235238Sjasone run_ind+(size>>LG_PAGE)-1) == 0); 983234370Sjasone } else { 984234370Sjasone size_t binind = arena_bin_index(arena, run->bin); 985234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 986234370Sjasone size = bin_info->run_size; 987234370Sjasone } 988234370Sjasone run_pages = (size >> LG_PAGE); 989234370Sjasone if (config_stats) { 990234370Sjasone /* 991234370Sjasone * Update stats_cactive if nactive is crossing a chunk 992234370Sjasone * multiple. 993234370Sjasone */ 994234370Sjasone size_t cactive_diff = CHUNK_CEILING(arena->nactive << LG_PAGE) - 995234370Sjasone CHUNK_CEILING((arena->nactive - run_pages) << LG_PAGE); 996234370Sjasone if (cactive_diff != 0) 997234370Sjasone stats_cactive_sub(cactive_diff); 998234370Sjasone } 999234370Sjasone arena->nactive -= run_pages; 1000234370Sjasone 1001234370Sjasone /* 1002234370Sjasone * The run is dirty if the caller claims to have dirtied it, as well as 1003242844Sjasone * if it was already dirty before being allocated and the caller 1004242844Sjasone * doesn't claim to have cleaned it. 1005234370Sjasone */ 1006235322Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == 1007235322Sjasone arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1008242844Sjasone if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1009234370Sjasone dirty = true; 1010234370Sjasone flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1011234370Sjasone 1012234370Sjasone /* Mark pages as unallocated in the chunk map. */ 1013234370Sjasone if (dirty) { 1014235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind, size, 1015235238Sjasone CHUNK_MAP_DIRTY); 1016235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1017235238Sjasone CHUNK_MAP_DIRTY); 1018234370Sjasone } else { 1019235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind, size, 1020235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind)); 1021235238Sjasone arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1022235238Sjasone arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1023234370Sjasone } 1024234370Sjasone 1025234370Sjasone /* Try to coalesce forward. */ 1026234370Sjasone if (run_ind + run_pages < chunk_npages && 1027235238Sjasone arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1028235238Sjasone arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1029235238Sjasone size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1030235238Sjasone run_ind+run_pages); 1031234370Sjasone size_t nrun_pages = nrun_size >> LG_PAGE; 1032234370Sjasone 1033234370Sjasone /* 1034234370Sjasone * Remove successor from runs_avail; the coalesced run is 1035234370Sjasone * inserted later. 1036234370Sjasone */ 1037235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, 1038235238Sjasone run_ind+run_pages+nrun_pages-1) == nrun_size); 1039235238Sjasone assert(arena_mapbits_dirty_get(chunk, 1040235238Sjasone run_ind+run_pages+nrun_pages-1) == flag_dirty); 1041242844Sjasone arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1042242844Sjasone false, true); 1043234370Sjasone 1044234370Sjasone size += nrun_size; 1045234370Sjasone run_pages += nrun_pages; 1046234370Sjasone 1047235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1048235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1049235238Sjasone size); 1050234370Sjasone } 1051234370Sjasone 1052234370Sjasone /* Try to coalesce backward. */ 1053235238Sjasone if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1) 1054235238Sjasone == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) { 1055235238Sjasone size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1056235238Sjasone run_ind-1); 1057234370Sjasone size_t prun_pages = prun_size >> LG_PAGE; 1058234370Sjasone 1059234370Sjasone run_ind -= prun_pages; 1060234370Sjasone 1061234370Sjasone /* 1062234370Sjasone * Remove predecessor from runs_avail; the coalesced run is 1063234370Sjasone * inserted later. 1064234370Sjasone */ 1065235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1066235238Sjasone prun_size); 1067235238Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1068242844Sjasone arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1069242844Sjasone false); 1070234370Sjasone 1071234370Sjasone size += prun_size; 1072234370Sjasone run_pages += prun_pages; 1073234370Sjasone 1074235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1075235238Sjasone arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1076235238Sjasone size); 1077234370Sjasone } 1078234370Sjasone 1079234370Sjasone /* Insert into runs_avail, now that coalescing is complete. */ 1080235238Sjasone assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1081235238Sjasone arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1082235238Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == 1083235238Sjasone arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1084242844Sjasone arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 1085234370Sjasone 1086235238Sjasone /* Deallocate chunk if it is now completely unused. */ 1087235238Sjasone if (size == arena_maxclass) { 1088235238Sjasone assert(run_ind == map_bias); 1089235238Sjasone assert(run_pages == (arena_maxclass >> LG_PAGE)); 1090234370Sjasone arena_chunk_dealloc(arena, chunk); 1091235238Sjasone } 1092234370Sjasone 1093234370Sjasone /* 1094234370Sjasone * It is okay to do dirty page processing here even if the chunk was 1095234370Sjasone * deallocated above, since in that case it is the spare. Waiting 1096234370Sjasone * until after possible chunk deallocation to do dirty processing 1097234370Sjasone * allows for an old spare to be fully deallocated, thus decreasing the 1098234370Sjasone * chances of spuriously crossing the dirty page purging threshold. 1099234370Sjasone */ 1100234370Sjasone if (dirty) 1101234370Sjasone arena_maybe_purge(arena); 1102234370Sjasone} 1103234370Sjasone 1104234370Sjasonestatic void 1105234370Sjasonearena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1106234370Sjasone size_t oldsize, size_t newsize) 1107234370Sjasone{ 1108234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1109234370Sjasone size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1110235238Sjasone size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1111234370Sjasone 1112234370Sjasone assert(oldsize > newsize); 1113234370Sjasone 1114234370Sjasone /* 1115234370Sjasone * Update the chunk map so that arena_run_dalloc() can treat the 1116234370Sjasone * leading run as separately allocated. Set the last element of each 1117234370Sjasone * run first, in case of single-page runs. 1118234370Sjasone */ 1119235238Sjasone assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1120235322Sjasone arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1121235322Sjasone arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1122234370Sjasone 1123234370Sjasone if (config_debug) { 1124234370Sjasone UNUSED size_t tail_npages = newsize >> LG_PAGE; 1125235238Sjasone assert(arena_mapbits_large_size_get(chunk, 1126235238Sjasone pageind+head_npages+tail_npages-1) == 0); 1127235238Sjasone assert(arena_mapbits_dirty_get(chunk, 1128235238Sjasone pageind+head_npages+tail_npages-1) == flag_dirty); 1129234370Sjasone } 1130235322Sjasone arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1131235322Sjasone flag_dirty); 1132234370Sjasone 1133242844Sjasone arena_run_dalloc(arena, run, false, false); 1134234370Sjasone} 1135234370Sjasone 1136234370Sjasonestatic void 1137234370Sjasonearena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1138234370Sjasone size_t oldsize, size_t newsize, bool dirty) 1139234370Sjasone{ 1140234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1141234370Sjasone size_t head_npages = newsize >> LG_PAGE; 1142235238Sjasone size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1143234370Sjasone 1144234370Sjasone assert(oldsize > newsize); 1145234370Sjasone 1146234370Sjasone /* 1147234370Sjasone * Update the chunk map so that arena_run_dalloc() can treat the 1148234370Sjasone * trailing run as separately allocated. Set the last element of each 1149234370Sjasone * run first, in case of single-page runs. 1150234370Sjasone */ 1151235238Sjasone assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1152235322Sjasone arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1153235322Sjasone arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1154234370Sjasone 1155235238Sjasone if (config_debug) { 1156235238Sjasone UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1157235238Sjasone assert(arena_mapbits_large_size_get(chunk, 1158235238Sjasone pageind+head_npages+tail_npages-1) == 0); 1159235238Sjasone assert(arena_mapbits_dirty_get(chunk, 1160235238Sjasone pageind+head_npages+tail_npages-1) == flag_dirty); 1161235238Sjasone } 1162235238Sjasone arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1163235322Sjasone flag_dirty); 1164234370Sjasone 1165234370Sjasone arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1166242844Sjasone dirty, false); 1167234370Sjasone} 1168234370Sjasone 1169234370Sjasonestatic arena_run_t * 1170234370Sjasonearena_bin_runs_first(arena_bin_t *bin) 1171234370Sjasone{ 1172234370Sjasone arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1173234370Sjasone if (mapelm != NULL) { 1174234370Sjasone arena_chunk_t *chunk; 1175234370Sjasone size_t pageind; 1176235238Sjasone arena_run_t *run; 1177234370Sjasone 1178234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 1179234370Sjasone pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 1180234370Sjasone sizeof(arena_chunk_map_t))) + map_bias; 1181235238Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1182235238Sjasone arena_mapbits_small_runind_get(chunk, pageind)) << 1183234370Sjasone LG_PAGE)); 1184234370Sjasone return (run); 1185234370Sjasone } 1186234370Sjasone 1187234370Sjasone return (NULL); 1188234370Sjasone} 1189234370Sjasone 1190234370Sjasonestatic void 1191234370Sjasonearena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1192234370Sjasone{ 1193234370Sjasone arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1194234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1195235238Sjasone arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1196234370Sjasone 1197234370Sjasone assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1198234370Sjasone 1199234370Sjasone arena_run_tree_insert(&bin->runs, mapelm); 1200234370Sjasone} 1201234370Sjasone 1202234370Sjasonestatic void 1203234370Sjasonearena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1204234370Sjasone{ 1205234370Sjasone arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1206234370Sjasone size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1207235238Sjasone arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1208234370Sjasone 1209234370Sjasone assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1210234370Sjasone 1211234370Sjasone arena_run_tree_remove(&bin->runs, mapelm); 1212234370Sjasone} 1213234370Sjasone 1214234370Sjasonestatic arena_run_t * 1215234370Sjasonearena_bin_nonfull_run_tryget(arena_bin_t *bin) 1216234370Sjasone{ 1217234370Sjasone arena_run_t *run = arena_bin_runs_first(bin); 1218234370Sjasone if (run != NULL) { 1219234370Sjasone arena_bin_runs_remove(bin, run); 1220234370Sjasone if (config_stats) 1221234370Sjasone bin->stats.reruns++; 1222234370Sjasone } 1223234370Sjasone return (run); 1224234370Sjasone} 1225234370Sjasone 1226234370Sjasonestatic arena_run_t * 1227234370Sjasonearena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1228234370Sjasone{ 1229234370Sjasone arena_run_t *run; 1230234370Sjasone size_t binind; 1231234370Sjasone arena_bin_info_t *bin_info; 1232234370Sjasone 1233234370Sjasone /* Look for a usable run. */ 1234234370Sjasone run = arena_bin_nonfull_run_tryget(bin); 1235234370Sjasone if (run != NULL) 1236234370Sjasone return (run); 1237234370Sjasone /* No existing runs have any space available. */ 1238234370Sjasone 1239234370Sjasone binind = arena_bin_index(arena, bin); 1240234370Sjasone bin_info = &arena_bin_info[binind]; 1241234370Sjasone 1242234370Sjasone /* Allocate a new run. */ 1243234370Sjasone malloc_mutex_unlock(&bin->lock); 1244234370Sjasone /******************************/ 1245234370Sjasone malloc_mutex_lock(&arena->lock); 1246235238Sjasone run = arena_run_alloc(arena, bin_info->run_size, false, binind, false); 1247234370Sjasone if (run != NULL) { 1248234370Sjasone bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 1249234370Sjasone (uintptr_t)bin_info->bitmap_offset); 1250234370Sjasone 1251234370Sjasone /* Initialize run internals. */ 1252234370Sjasone run->bin = bin; 1253234370Sjasone run->nextind = 0; 1254234370Sjasone run->nfree = bin_info->nregs; 1255234370Sjasone bitmap_init(bitmap, &bin_info->bitmap_info); 1256234370Sjasone } 1257234370Sjasone malloc_mutex_unlock(&arena->lock); 1258234370Sjasone /********************************/ 1259234370Sjasone malloc_mutex_lock(&bin->lock); 1260234370Sjasone if (run != NULL) { 1261234370Sjasone if (config_stats) { 1262234370Sjasone bin->stats.nruns++; 1263234370Sjasone bin->stats.curruns++; 1264234370Sjasone } 1265234370Sjasone return (run); 1266234370Sjasone } 1267234370Sjasone 1268234370Sjasone /* 1269234370Sjasone * arena_run_alloc() failed, but another thread may have made 1270234370Sjasone * sufficient memory available while this one dropped bin->lock above, 1271234370Sjasone * so search one more time. 1272234370Sjasone */ 1273234370Sjasone run = arena_bin_nonfull_run_tryget(bin); 1274234370Sjasone if (run != NULL) 1275234370Sjasone return (run); 1276234370Sjasone 1277234370Sjasone return (NULL); 1278234370Sjasone} 1279234370Sjasone 1280234370Sjasone/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1281234370Sjasonestatic void * 1282234370Sjasonearena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1283234370Sjasone{ 1284234370Sjasone void *ret; 1285234370Sjasone size_t binind; 1286234370Sjasone arena_bin_info_t *bin_info; 1287234370Sjasone arena_run_t *run; 1288234370Sjasone 1289234370Sjasone binind = arena_bin_index(arena, bin); 1290234370Sjasone bin_info = &arena_bin_info[binind]; 1291234370Sjasone bin->runcur = NULL; 1292234370Sjasone run = arena_bin_nonfull_run_get(arena, bin); 1293234370Sjasone if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1294234370Sjasone /* 1295234370Sjasone * Another thread updated runcur while this one ran without the 1296234370Sjasone * bin lock in arena_bin_nonfull_run_get(). 1297234370Sjasone */ 1298234370Sjasone assert(bin->runcur->nfree > 0); 1299234370Sjasone ret = arena_run_reg_alloc(bin->runcur, bin_info); 1300234370Sjasone if (run != NULL) { 1301234370Sjasone arena_chunk_t *chunk; 1302234370Sjasone 1303234370Sjasone /* 1304234370Sjasone * arena_run_alloc() may have allocated run, or it may 1305234370Sjasone * have pulled run from the bin's run tree. Therefore 1306234370Sjasone * it is unsafe to make any assumptions about how run 1307234370Sjasone * has previously been used, and arena_bin_lower_run() 1308234370Sjasone * must be called, as if a region were just deallocated 1309234370Sjasone * from the run. 1310234370Sjasone */ 1311234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1312234370Sjasone if (run->nfree == bin_info->nregs) 1313234370Sjasone arena_dalloc_bin_run(arena, chunk, run, bin); 1314234370Sjasone else 1315234370Sjasone arena_bin_lower_run(arena, chunk, run, bin); 1316234370Sjasone } 1317234370Sjasone return (ret); 1318234370Sjasone } 1319234370Sjasone 1320234370Sjasone if (run == NULL) 1321234370Sjasone return (NULL); 1322234370Sjasone 1323234370Sjasone bin->runcur = run; 1324234370Sjasone 1325234370Sjasone assert(bin->runcur->nfree > 0); 1326234370Sjasone 1327234370Sjasone return (arena_run_reg_alloc(bin->runcur, bin_info)); 1328234370Sjasone} 1329234370Sjasone 1330234370Sjasonevoid 1331234370Sjasonearena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 1332234370Sjasone uint64_t prof_accumbytes) 1333234370Sjasone{ 1334234370Sjasone unsigned i, nfill; 1335234370Sjasone arena_bin_t *bin; 1336234370Sjasone arena_run_t *run; 1337234370Sjasone void *ptr; 1338234370Sjasone 1339234370Sjasone assert(tbin->ncached == 0); 1340234370Sjasone 1341251300Sjasone if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 1342251300Sjasone prof_idump(); 1343234370Sjasone bin = &arena->bins[binind]; 1344234370Sjasone malloc_mutex_lock(&bin->lock); 1345234370Sjasone for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 1346234370Sjasone tbin->lg_fill_div); i < nfill; i++) { 1347234370Sjasone if ((run = bin->runcur) != NULL && run->nfree > 0) 1348234370Sjasone ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1349234370Sjasone else 1350234370Sjasone ptr = arena_bin_malloc_hard(arena, bin); 1351234370Sjasone if (ptr == NULL) 1352234370Sjasone break; 1353234370Sjasone if (config_fill && opt_junk) { 1354234370Sjasone arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1355234370Sjasone true); 1356234370Sjasone } 1357234370Sjasone /* Insert such that low regions get used first. */ 1358234370Sjasone tbin->avail[nfill - 1 - i] = ptr; 1359234370Sjasone } 1360234370Sjasone if (config_stats) { 1361234370Sjasone bin->stats.allocated += i * arena_bin_info[binind].reg_size; 1362234370Sjasone bin->stats.nmalloc += i; 1363234370Sjasone bin->stats.nrequests += tbin->tstats.nrequests; 1364234370Sjasone bin->stats.nfills++; 1365234370Sjasone tbin->tstats.nrequests = 0; 1366234370Sjasone } 1367234370Sjasone malloc_mutex_unlock(&bin->lock); 1368234370Sjasone tbin->ncached = i; 1369234370Sjasone} 1370234370Sjasone 1371234370Sjasonevoid 1372234370Sjasonearena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1373234370Sjasone{ 1374234370Sjasone 1375234370Sjasone if (zero) { 1376234370Sjasone size_t redzone_size = bin_info->redzone_size; 1377234370Sjasone memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1378234370Sjasone redzone_size); 1379234370Sjasone memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1380234370Sjasone redzone_size); 1381234370Sjasone } else { 1382234370Sjasone memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1383234370Sjasone bin_info->reg_interval); 1384234370Sjasone } 1385234370Sjasone} 1386234370Sjasone 1387234370Sjasonevoid 1388234370Sjasonearena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 1389234370Sjasone{ 1390234370Sjasone size_t size = bin_info->reg_size; 1391234370Sjasone size_t redzone_size = bin_info->redzone_size; 1392234370Sjasone size_t i; 1393234370Sjasone bool error = false; 1394234370Sjasone 1395234370Sjasone for (i = 1; i <= redzone_size; i++) { 1396234370Sjasone unsigned byte; 1397234370Sjasone if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) { 1398234370Sjasone error = true; 1399234370Sjasone malloc_printf("<jemalloc>: Corrupt redzone " 1400234370Sjasone "%zu byte%s before %p (size %zu), byte=%#x\n", i, 1401234370Sjasone (i == 1) ? "" : "s", ptr, size, byte); 1402234370Sjasone } 1403234370Sjasone } 1404234370Sjasone for (i = 0; i < redzone_size; i++) { 1405234370Sjasone unsigned byte; 1406234370Sjasone if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) { 1407234370Sjasone error = true; 1408234370Sjasone malloc_printf("<jemalloc>: Corrupt redzone " 1409234370Sjasone "%zu byte%s after end of %p (size %zu), byte=%#x\n", 1410234370Sjasone i, (i == 1) ? "" : "s", ptr, size, byte); 1411234370Sjasone } 1412234370Sjasone } 1413234370Sjasone if (opt_abort && error) 1414234370Sjasone abort(); 1415234370Sjasone 1416234370Sjasone memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1417234370Sjasone bin_info->reg_interval); 1418234370Sjasone} 1419234370Sjasone 1420234370Sjasonevoid * 1421234370Sjasonearena_malloc_small(arena_t *arena, size_t size, bool zero) 1422234370Sjasone{ 1423234370Sjasone void *ret; 1424234370Sjasone arena_bin_t *bin; 1425234370Sjasone arena_run_t *run; 1426234370Sjasone size_t binind; 1427234370Sjasone 1428234370Sjasone binind = SMALL_SIZE2BIN(size); 1429234370Sjasone assert(binind < NBINS); 1430234370Sjasone bin = &arena->bins[binind]; 1431234370Sjasone size = arena_bin_info[binind].reg_size; 1432234370Sjasone 1433234370Sjasone malloc_mutex_lock(&bin->lock); 1434234370Sjasone if ((run = bin->runcur) != NULL && run->nfree > 0) 1435234370Sjasone ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1436234370Sjasone else 1437234370Sjasone ret = arena_bin_malloc_hard(arena, bin); 1438234370Sjasone 1439234370Sjasone if (ret == NULL) { 1440234370Sjasone malloc_mutex_unlock(&bin->lock); 1441234370Sjasone return (NULL); 1442234370Sjasone } 1443234370Sjasone 1444234370Sjasone if (config_stats) { 1445234370Sjasone bin->stats.allocated += size; 1446234370Sjasone bin->stats.nmalloc++; 1447234370Sjasone bin->stats.nrequests++; 1448234370Sjasone } 1449234370Sjasone malloc_mutex_unlock(&bin->lock); 1450251300Sjasone if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) 1451251300Sjasone prof_idump(); 1452234370Sjasone 1453234370Sjasone if (zero == false) { 1454234370Sjasone if (config_fill) { 1455234370Sjasone if (opt_junk) { 1456234370Sjasone arena_alloc_junk_small(ret, 1457234370Sjasone &arena_bin_info[binind], false); 1458234370Sjasone } else if (opt_zero) 1459234370Sjasone memset(ret, 0, size); 1460234370Sjasone } 1461234370Sjasone } else { 1462234370Sjasone if (config_fill && opt_junk) { 1463234370Sjasone arena_alloc_junk_small(ret, &arena_bin_info[binind], 1464234370Sjasone true); 1465234370Sjasone } 1466234370Sjasone VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1467234370Sjasone memset(ret, 0, size); 1468234370Sjasone } 1469251300Sjasone VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1470234370Sjasone 1471234370Sjasone return (ret); 1472234370Sjasone} 1473234370Sjasone 1474234370Sjasonevoid * 1475234370Sjasonearena_malloc_large(arena_t *arena, size_t size, bool zero) 1476234370Sjasone{ 1477234370Sjasone void *ret; 1478251300Sjasone UNUSED bool idump; 1479234370Sjasone 1480234370Sjasone /* Large allocation. */ 1481234370Sjasone size = PAGE_CEILING(size); 1482234370Sjasone malloc_mutex_lock(&arena->lock); 1483235238Sjasone ret = (void *)arena_run_alloc(arena, size, true, BININD_INVALID, zero); 1484234370Sjasone if (ret == NULL) { 1485234370Sjasone malloc_mutex_unlock(&arena->lock); 1486234370Sjasone return (NULL); 1487234370Sjasone } 1488234370Sjasone if (config_stats) { 1489234370Sjasone arena->stats.nmalloc_large++; 1490234370Sjasone arena->stats.nrequests_large++; 1491234370Sjasone arena->stats.allocated_large += size; 1492234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1493234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1494234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1495234370Sjasone } 1496234370Sjasone if (config_prof) 1497251300Sjasone idump = arena_prof_accum_locked(arena, size); 1498234370Sjasone malloc_mutex_unlock(&arena->lock); 1499251300Sjasone if (config_prof && idump) 1500251300Sjasone prof_idump(); 1501234370Sjasone 1502234370Sjasone if (zero == false) { 1503234370Sjasone if (config_fill) { 1504234370Sjasone if (opt_junk) 1505234370Sjasone memset(ret, 0xa5, size); 1506234370Sjasone else if (opt_zero) 1507234370Sjasone memset(ret, 0, size); 1508234370Sjasone } 1509234370Sjasone } 1510234370Sjasone 1511234370Sjasone return (ret); 1512234370Sjasone} 1513234370Sjasone 1514234370Sjasone/* Only handles large allocations that require more than page alignment. */ 1515234370Sjasonevoid * 1516234370Sjasonearena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1517234370Sjasone{ 1518234370Sjasone void *ret; 1519234370Sjasone size_t alloc_size, leadsize, trailsize; 1520234370Sjasone arena_run_t *run; 1521234370Sjasone arena_chunk_t *chunk; 1522234370Sjasone 1523234370Sjasone assert((size & PAGE_MASK) == 0); 1524234370Sjasone 1525234370Sjasone alignment = PAGE_CEILING(alignment); 1526234370Sjasone alloc_size = size + alignment - PAGE; 1527234370Sjasone 1528234370Sjasone malloc_mutex_lock(&arena->lock); 1529235238Sjasone run = arena_run_alloc(arena, alloc_size, true, BININD_INVALID, zero); 1530234370Sjasone if (run == NULL) { 1531234370Sjasone malloc_mutex_unlock(&arena->lock); 1532234370Sjasone return (NULL); 1533234370Sjasone } 1534234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1535234370Sjasone 1536234370Sjasone leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 1537234370Sjasone (uintptr_t)run; 1538234370Sjasone assert(alloc_size >= leadsize + size); 1539234370Sjasone trailsize = alloc_size - leadsize - size; 1540234370Sjasone ret = (void *)((uintptr_t)run + leadsize); 1541234370Sjasone if (leadsize != 0) { 1542234370Sjasone arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 1543234370Sjasone leadsize); 1544234370Sjasone } 1545234370Sjasone if (trailsize != 0) { 1546234370Sjasone arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 1547234370Sjasone false); 1548234370Sjasone } 1549234370Sjasone 1550234370Sjasone if (config_stats) { 1551234370Sjasone arena->stats.nmalloc_large++; 1552234370Sjasone arena->stats.nrequests_large++; 1553234370Sjasone arena->stats.allocated_large += size; 1554234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1555234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1556234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1557234370Sjasone } 1558234370Sjasone malloc_mutex_unlock(&arena->lock); 1559234370Sjasone 1560234370Sjasone if (config_fill && zero == false) { 1561234370Sjasone if (opt_junk) 1562234370Sjasone memset(ret, 0xa5, size); 1563234370Sjasone else if (opt_zero) 1564234370Sjasone memset(ret, 0, size); 1565234370Sjasone } 1566234370Sjasone return (ret); 1567234370Sjasone} 1568234370Sjasone 1569234370Sjasonevoid 1570234370Sjasonearena_prof_promoted(const void *ptr, size_t size) 1571234370Sjasone{ 1572234370Sjasone arena_chunk_t *chunk; 1573234370Sjasone size_t pageind, binind; 1574234370Sjasone 1575234543Sjasone cassert(config_prof); 1576234370Sjasone assert(ptr != NULL); 1577234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 1578234370Sjasone assert(isalloc(ptr, false) == PAGE); 1579234370Sjasone assert(isalloc(ptr, true) == PAGE); 1580234370Sjasone assert(size <= SMALL_MAXCLASS); 1581234370Sjasone 1582234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1583234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1584234370Sjasone binind = SMALL_SIZE2BIN(size); 1585234370Sjasone assert(binind < NBINS); 1586235238Sjasone arena_mapbits_large_binind_set(chunk, pageind, binind); 1587234370Sjasone 1588234370Sjasone assert(isalloc(ptr, false) == PAGE); 1589234370Sjasone assert(isalloc(ptr, true) == size); 1590234370Sjasone} 1591234370Sjasone 1592234370Sjasonestatic void 1593234370Sjasonearena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1594234370Sjasone arena_bin_t *bin) 1595234370Sjasone{ 1596234370Sjasone 1597234370Sjasone /* Dissociate run from bin. */ 1598234370Sjasone if (run == bin->runcur) 1599234370Sjasone bin->runcur = NULL; 1600234370Sjasone else { 1601234370Sjasone size_t binind = arena_bin_index(chunk->arena, bin); 1602234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1603234370Sjasone 1604234370Sjasone if (bin_info->nregs != 1) { 1605234370Sjasone /* 1606234370Sjasone * This block's conditional is necessary because if the 1607234370Sjasone * run only contains one region, then it never gets 1608234370Sjasone * inserted into the non-full runs tree. 1609234370Sjasone */ 1610234370Sjasone arena_bin_runs_remove(bin, run); 1611234370Sjasone } 1612234370Sjasone } 1613234370Sjasone} 1614234370Sjasone 1615234370Sjasonestatic void 1616234370Sjasonearena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1617234370Sjasone arena_bin_t *bin) 1618234370Sjasone{ 1619234370Sjasone size_t binind; 1620234370Sjasone arena_bin_info_t *bin_info; 1621234370Sjasone size_t npages, run_ind, past; 1622234370Sjasone 1623234370Sjasone assert(run != bin->runcur); 1624235238Sjasone assert(arena_run_tree_search(&bin->runs, 1625235238Sjasone arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1626235238Sjasone == NULL); 1627234370Sjasone 1628234370Sjasone binind = arena_bin_index(chunk->arena, run->bin); 1629234370Sjasone bin_info = &arena_bin_info[binind]; 1630234370Sjasone 1631234370Sjasone malloc_mutex_unlock(&bin->lock); 1632234370Sjasone /******************************/ 1633234370Sjasone npages = bin_info->run_size >> LG_PAGE; 1634234370Sjasone run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1635234370Sjasone past = (size_t)(PAGE_CEILING((uintptr_t)run + 1636234370Sjasone (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1637234370Sjasone bin_info->reg_interval - bin_info->redzone_size) - 1638234370Sjasone (uintptr_t)chunk) >> LG_PAGE); 1639234370Sjasone malloc_mutex_lock(&arena->lock); 1640234370Sjasone 1641234370Sjasone /* 1642234370Sjasone * If the run was originally clean, and some pages were never touched, 1643234370Sjasone * trim the clean pages before deallocating the dirty portion of the 1644234370Sjasone * run. 1645234370Sjasone */ 1646235322Sjasone assert(arena_mapbits_dirty_get(chunk, run_ind) == 1647235322Sjasone arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1648235238Sjasone if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1649235238Sjasone npages) { 1650235322Sjasone /* Trim clean pages. Convert to large run beforehand. */ 1651235322Sjasone assert(npages > 0); 1652235322Sjasone arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1653235322Sjasone arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1654234370Sjasone arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1655234370Sjasone ((past - run_ind) << LG_PAGE), false); 1656234370Sjasone /* npages = past - run_ind; */ 1657234370Sjasone } 1658242844Sjasone arena_run_dalloc(arena, run, true, false); 1659234370Sjasone malloc_mutex_unlock(&arena->lock); 1660234370Sjasone /****************************/ 1661234370Sjasone malloc_mutex_lock(&bin->lock); 1662234370Sjasone if (config_stats) 1663234370Sjasone bin->stats.curruns--; 1664234370Sjasone} 1665234370Sjasone 1666234370Sjasonestatic void 1667234370Sjasonearena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1668234370Sjasone arena_bin_t *bin) 1669234370Sjasone{ 1670234370Sjasone 1671234370Sjasone /* 1672234370Sjasone * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1673234370Sjasone * non-full run. It is okay to NULL runcur out rather than proactively 1674234370Sjasone * keeping it pointing at the lowest non-full run. 1675234370Sjasone */ 1676234370Sjasone if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1677234370Sjasone /* Switch runcur. */ 1678234370Sjasone if (bin->runcur->nfree > 0) 1679234370Sjasone arena_bin_runs_insert(bin, bin->runcur); 1680234370Sjasone bin->runcur = run; 1681234370Sjasone if (config_stats) 1682234370Sjasone bin->stats.reruns++; 1683234370Sjasone } else 1684234370Sjasone arena_bin_runs_insert(bin, run); 1685234370Sjasone} 1686234370Sjasone 1687234370Sjasonevoid 1688235238Sjasonearena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1689234370Sjasone arena_chunk_map_t *mapelm) 1690234370Sjasone{ 1691234370Sjasone size_t pageind; 1692234370Sjasone arena_run_t *run; 1693234370Sjasone arena_bin_t *bin; 1694235238Sjasone arena_bin_info_t *bin_info; 1695235238Sjasone size_t size, binind; 1696234370Sjasone 1697234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1698234370Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1699235238Sjasone arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1700234370Sjasone bin = run->bin; 1701235238Sjasone binind = arena_ptr_small_binind_get(ptr, mapelm->bits); 1702235238Sjasone bin_info = &arena_bin_info[binind]; 1703234370Sjasone if (config_fill || config_stats) 1704234370Sjasone size = bin_info->reg_size; 1705234370Sjasone 1706234370Sjasone if (config_fill && opt_junk) 1707234370Sjasone arena_dalloc_junk_small(ptr, bin_info); 1708234370Sjasone 1709234370Sjasone arena_run_reg_dalloc(run, ptr); 1710234370Sjasone if (run->nfree == bin_info->nregs) { 1711234370Sjasone arena_dissociate_bin_run(chunk, run, bin); 1712234370Sjasone arena_dalloc_bin_run(arena, chunk, run, bin); 1713234370Sjasone } else if (run->nfree == 1 && run != bin->runcur) 1714234370Sjasone arena_bin_lower_run(arena, chunk, run, bin); 1715234370Sjasone 1716234370Sjasone if (config_stats) { 1717234370Sjasone bin->stats.allocated -= size; 1718234370Sjasone bin->stats.ndalloc++; 1719234370Sjasone } 1720234370Sjasone} 1721234370Sjasone 1722234370Sjasonevoid 1723235238Sjasonearena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1724235238Sjasone size_t pageind, arena_chunk_map_t *mapelm) 1725235238Sjasone{ 1726235238Sjasone arena_run_t *run; 1727235238Sjasone arena_bin_t *bin; 1728235238Sjasone 1729235238Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1730235238Sjasone arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1731235238Sjasone bin = run->bin; 1732235238Sjasone malloc_mutex_lock(&bin->lock); 1733235238Sjasone arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 1734235238Sjasone malloc_mutex_unlock(&bin->lock); 1735235238Sjasone} 1736235238Sjasone 1737235238Sjasonevoid 1738235238Sjasonearena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1739235238Sjasone size_t pageind) 1740235238Sjasone{ 1741235238Sjasone arena_chunk_map_t *mapelm; 1742235238Sjasone 1743235238Sjasone if (config_debug) { 1744235238Sjasone /* arena_ptr_small_binind_get() does extra sanity checking. */ 1745235238Sjasone assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 1746235238Sjasone pageind)) != BININD_INVALID); 1747235238Sjasone } 1748235238Sjasone mapelm = arena_mapp_get(chunk, pageind); 1749235238Sjasone arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 1750235238Sjasone} 1751234370Sjasone 1752234370Sjasonevoid 1753235238Sjasonearena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1754234370Sjasone{ 1755234370Sjasone 1756234370Sjasone if (config_fill || config_stats) { 1757234370Sjasone size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1758235238Sjasone size_t size = arena_mapbits_large_size_get(chunk, pageind); 1759234370Sjasone 1760234370Sjasone if (config_fill && config_stats && opt_junk) 1761234370Sjasone memset(ptr, 0x5a, size); 1762234370Sjasone if (config_stats) { 1763234370Sjasone arena->stats.ndalloc_large++; 1764234370Sjasone arena->stats.allocated_large -= size; 1765234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].ndalloc++; 1766234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns--; 1767234370Sjasone } 1768234370Sjasone } 1769234370Sjasone 1770242844Sjasone arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 1771234370Sjasone} 1772234370Sjasone 1773235238Sjasonevoid 1774235238Sjasonearena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1775235238Sjasone{ 1776235238Sjasone 1777235238Sjasone malloc_mutex_lock(&arena->lock); 1778235238Sjasone arena_dalloc_large_locked(arena, chunk, ptr); 1779235238Sjasone malloc_mutex_unlock(&arena->lock); 1780235238Sjasone} 1781235238Sjasone 1782234370Sjasonestatic void 1783234370Sjasonearena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1784234370Sjasone size_t oldsize, size_t size) 1785234370Sjasone{ 1786234370Sjasone 1787234370Sjasone assert(size < oldsize); 1788234370Sjasone 1789234370Sjasone /* 1790234370Sjasone * Shrink the run, and make trailing pages available for other 1791234370Sjasone * allocations. 1792234370Sjasone */ 1793234370Sjasone malloc_mutex_lock(&arena->lock); 1794234370Sjasone arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1795234370Sjasone true); 1796234370Sjasone if (config_stats) { 1797234370Sjasone arena->stats.ndalloc_large++; 1798234370Sjasone arena->stats.allocated_large -= oldsize; 1799234370Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1800234370Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 1801234370Sjasone 1802234370Sjasone arena->stats.nmalloc_large++; 1803234370Sjasone arena->stats.nrequests_large++; 1804234370Sjasone arena->stats.allocated_large += size; 1805234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1806234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1807234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1808234370Sjasone } 1809234370Sjasone malloc_mutex_unlock(&arena->lock); 1810234370Sjasone} 1811234370Sjasone 1812234370Sjasonestatic bool 1813234370Sjasonearena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1814234370Sjasone size_t oldsize, size_t size, size_t extra, bool zero) 1815234370Sjasone{ 1816234370Sjasone size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1817234370Sjasone size_t npages = oldsize >> LG_PAGE; 1818234370Sjasone size_t followsize; 1819234370Sjasone 1820235238Sjasone assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 1821234370Sjasone 1822234370Sjasone /* Try to extend the run. */ 1823234370Sjasone assert(size + extra > oldsize); 1824234370Sjasone malloc_mutex_lock(&arena->lock); 1825234370Sjasone if (pageind + npages < chunk_npages && 1826235238Sjasone arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 1827235238Sjasone (followsize = arena_mapbits_unallocated_size_get(chunk, 1828235238Sjasone pageind+npages)) >= size - oldsize) { 1829234370Sjasone /* 1830234370Sjasone * The next run is available and sufficiently large. Split the 1831234370Sjasone * following run, then merge the first part with the existing 1832234370Sjasone * allocation. 1833234370Sjasone */ 1834234370Sjasone size_t flag_dirty; 1835234370Sjasone size_t splitsize = (oldsize + followsize <= size + extra) 1836234370Sjasone ? followsize : size + extra - oldsize; 1837234370Sjasone arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1838235238Sjasone ((pageind+npages) << LG_PAGE)), splitsize, true, 1839235238Sjasone BININD_INVALID, zero); 1840234370Sjasone 1841234370Sjasone size = oldsize + splitsize; 1842234370Sjasone npages = size >> LG_PAGE; 1843234370Sjasone 1844234370Sjasone /* 1845234370Sjasone * Mark the extended run as dirty if either portion of the run 1846234370Sjasone * was dirty before allocation. This is rather pedantic, 1847234370Sjasone * because there's not actually any sequence of events that 1848234370Sjasone * could cause the resulting run to be passed to 1849234370Sjasone * arena_run_dalloc() with the dirty argument set to false 1850234370Sjasone * (which is when dirty flag consistency would really matter). 1851234370Sjasone */ 1852235238Sjasone flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 1853235238Sjasone arena_mapbits_dirty_get(chunk, pageind+npages-1); 1854235238Sjasone arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 1855235238Sjasone arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 1856234370Sjasone 1857234370Sjasone if (config_stats) { 1858234370Sjasone arena->stats.ndalloc_large++; 1859234370Sjasone arena->stats.allocated_large -= oldsize; 1860235238Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1861235238Sjasone arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 1862234370Sjasone 1863234370Sjasone arena->stats.nmalloc_large++; 1864234370Sjasone arena->stats.nrequests_large++; 1865234370Sjasone arena->stats.allocated_large += size; 1866234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1867235238Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1868234370Sjasone arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1869234370Sjasone } 1870234370Sjasone malloc_mutex_unlock(&arena->lock); 1871234370Sjasone return (false); 1872234370Sjasone } 1873234370Sjasone malloc_mutex_unlock(&arena->lock); 1874234370Sjasone 1875234370Sjasone return (true); 1876234370Sjasone} 1877234370Sjasone 1878234370Sjasone/* 1879234370Sjasone * Try to resize a large allocation, in order to avoid copying. This will 1880234370Sjasone * always fail if growing an object, and the following run is already in use. 1881234370Sjasone */ 1882234370Sjasonestatic bool 1883234370Sjasonearena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 1884234370Sjasone bool zero) 1885234370Sjasone{ 1886234370Sjasone size_t psize; 1887234370Sjasone 1888234370Sjasone psize = PAGE_CEILING(size + extra); 1889234370Sjasone if (psize == oldsize) { 1890234370Sjasone /* Same size class. */ 1891234370Sjasone if (config_fill && opt_junk && size < oldsize) { 1892234370Sjasone memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1893234370Sjasone size); 1894234370Sjasone } 1895234370Sjasone return (false); 1896234370Sjasone } else { 1897234370Sjasone arena_chunk_t *chunk; 1898234370Sjasone arena_t *arena; 1899234370Sjasone 1900234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1901234370Sjasone arena = chunk->arena; 1902234370Sjasone 1903234370Sjasone if (psize < oldsize) { 1904234370Sjasone /* Fill before shrinking in order avoid a race. */ 1905234370Sjasone if (config_fill && opt_junk) { 1906234370Sjasone memset((void *)((uintptr_t)ptr + size), 0x5a, 1907234370Sjasone oldsize - size); 1908234370Sjasone } 1909234370Sjasone arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 1910234370Sjasone psize); 1911234370Sjasone return (false); 1912234370Sjasone } else { 1913234370Sjasone bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 1914234370Sjasone oldsize, PAGE_CEILING(size), 1915234370Sjasone psize - PAGE_CEILING(size), zero); 1916234370Sjasone if (config_fill && ret == false && zero == false && 1917234370Sjasone opt_zero) { 1918234370Sjasone memset((void *)((uintptr_t)ptr + oldsize), 0, 1919234370Sjasone size - oldsize); 1920234370Sjasone } 1921234370Sjasone return (ret); 1922234370Sjasone } 1923234370Sjasone } 1924234370Sjasone} 1925234370Sjasone 1926234370Sjasonevoid * 1927234370Sjasonearena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 1928234370Sjasone bool zero) 1929234370Sjasone{ 1930234370Sjasone 1931234370Sjasone /* 1932234370Sjasone * Avoid moving the allocation if the size class can be left the same. 1933234370Sjasone */ 1934234370Sjasone if (oldsize <= arena_maxclass) { 1935234370Sjasone if (oldsize <= SMALL_MAXCLASS) { 1936234370Sjasone assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 1937234370Sjasone == oldsize); 1938234370Sjasone if ((size + extra <= SMALL_MAXCLASS && 1939234370Sjasone SMALL_SIZE2BIN(size + extra) == 1940234370Sjasone SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 1941234370Sjasone size + extra >= oldsize)) { 1942234370Sjasone if (config_fill && opt_junk && size < oldsize) { 1943234370Sjasone memset((void *)((uintptr_t)ptr + size), 1944234370Sjasone 0x5a, oldsize - size); 1945234370Sjasone } 1946234370Sjasone return (ptr); 1947234370Sjasone } 1948234370Sjasone } else { 1949234370Sjasone assert(size <= arena_maxclass); 1950234370Sjasone if (size + extra > SMALL_MAXCLASS) { 1951234370Sjasone if (arena_ralloc_large(ptr, oldsize, size, 1952234370Sjasone extra, zero) == false) 1953234370Sjasone return (ptr); 1954234370Sjasone } 1955234370Sjasone } 1956234370Sjasone } 1957234370Sjasone 1958234370Sjasone /* Reallocation would require a move. */ 1959234370Sjasone return (NULL); 1960234370Sjasone} 1961234370Sjasone 1962234370Sjasonevoid * 1963242844Sjasonearena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 1964242844Sjasone size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 1965242844Sjasone bool try_tcache_dalloc) 1966234370Sjasone{ 1967234370Sjasone void *ret; 1968234370Sjasone size_t copysize; 1969234370Sjasone 1970234370Sjasone /* Try to avoid moving the allocation. */ 1971234370Sjasone ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); 1972234370Sjasone if (ret != NULL) 1973234370Sjasone return (ret); 1974234370Sjasone 1975234370Sjasone /* 1976234370Sjasone * size and oldsize are different enough that we need to move the 1977234370Sjasone * object. In that case, fall back to allocating new space and 1978234370Sjasone * copying. 1979234370Sjasone */ 1980234370Sjasone if (alignment != 0) { 1981234370Sjasone size_t usize = sa2u(size + extra, alignment); 1982234370Sjasone if (usize == 0) 1983234370Sjasone return (NULL); 1984242844Sjasone ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); 1985234370Sjasone } else 1986242844Sjasone ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 1987234370Sjasone 1988234370Sjasone if (ret == NULL) { 1989234370Sjasone if (extra == 0) 1990234370Sjasone return (NULL); 1991234370Sjasone /* Try again, this time without extra. */ 1992234370Sjasone if (alignment != 0) { 1993234370Sjasone size_t usize = sa2u(size, alignment); 1994234370Sjasone if (usize == 0) 1995234370Sjasone return (NULL); 1996242844Sjasone ret = ipallocx(usize, alignment, zero, try_tcache_alloc, 1997242844Sjasone arena); 1998234370Sjasone } else 1999242844Sjasone ret = arena_malloc(arena, size, zero, try_tcache_alloc); 2000234370Sjasone 2001234370Sjasone if (ret == NULL) 2002234370Sjasone return (NULL); 2003234370Sjasone } 2004234370Sjasone 2005234370Sjasone /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 2006234370Sjasone 2007234370Sjasone /* 2008234370Sjasone * Copy at most size bytes (not size+extra), since the caller has no 2009234370Sjasone * expectation that the extra bytes will be reliably preserved. 2010234370Sjasone */ 2011234370Sjasone copysize = (size < oldsize) ? size : oldsize; 2012235238Sjasone VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2013234370Sjasone memcpy(ret, ptr, copysize); 2014242844Sjasone iqallocx(ptr, try_tcache_dalloc); 2015234370Sjasone return (ret); 2016234370Sjasone} 2017234370Sjasone 2018242844Sjasonedss_prec_t 2019242844Sjasonearena_dss_prec_get(arena_t *arena) 2020242844Sjasone{ 2021242844Sjasone dss_prec_t ret; 2022242844Sjasone 2023242844Sjasone malloc_mutex_lock(&arena->lock); 2024242844Sjasone ret = arena->dss_prec; 2025242844Sjasone malloc_mutex_unlock(&arena->lock); 2026242844Sjasone return (ret); 2027242844Sjasone} 2028242844Sjasone 2029242844Sjasonevoid 2030242844Sjasonearena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2031242844Sjasone{ 2032242844Sjasone 2033242844Sjasone malloc_mutex_lock(&arena->lock); 2034242844Sjasone arena->dss_prec = dss_prec; 2035242844Sjasone malloc_mutex_unlock(&arena->lock); 2036242844Sjasone} 2037242844Sjasone 2038242844Sjasonevoid 2039242844Sjasonearena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2040242844Sjasone size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2041242844Sjasone malloc_large_stats_t *lstats) 2042242844Sjasone{ 2043242844Sjasone unsigned i; 2044242844Sjasone 2045242844Sjasone malloc_mutex_lock(&arena->lock); 2046242844Sjasone *dss = dss_prec_names[arena->dss_prec]; 2047242844Sjasone *nactive += arena->nactive; 2048242844Sjasone *ndirty += arena->ndirty; 2049242844Sjasone 2050242844Sjasone astats->mapped += arena->stats.mapped; 2051242844Sjasone astats->npurge += arena->stats.npurge; 2052242844Sjasone astats->nmadvise += arena->stats.nmadvise; 2053242844Sjasone astats->purged += arena->stats.purged; 2054242844Sjasone astats->allocated_large += arena->stats.allocated_large; 2055242844Sjasone astats->nmalloc_large += arena->stats.nmalloc_large; 2056242844Sjasone astats->ndalloc_large += arena->stats.ndalloc_large; 2057242844Sjasone astats->nrequests_large += arena->stats.nrequests_large; 2058242844Sjasone 2059242844Sjasone for (i = 0; i < nlclasses; i++) { 2060242844Sjasone lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2061242844Sjasone lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2062242844Sjasone lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2063242844Sjasone lstats[i].curruns += arena->stats.lstats[i].curruns; 2064242844Sjasone } 2065242844Sjasone malloc_mutex_unlock(&arena->lock); 2066242844Sjasone 2067242844Sjasone for (i = 0; i < NBINS; i++) { 2068242844Sjasone arena_bin_t *bin = &arena->bins[i]; 2069242844Sjasone 2070242844Sjasone malloc_mutex_lock(&bin->lock); 2071242844Sjasone bstats[i].allocated += bin->stats.allocated; 2072242844Sjasone bstats[i].nmalloc += bin->stats.nmalloc; 2073242844Sjasone bstats[i].ndalloc += bin->stats.ndalloc; 2074242844Sjasone bstats[i].nrequests += bin->stats.nrequests; 2075242844Sjasone if (config_tcache) { 2076242844Sjasone bstats[i].nfills += bin->stats.nfills; 2077242844Sjasone bstats[i].nflushes += bin->stats.nflushes; 2078242844Sjasone } 2079242844Sjasone bstats[i].nruns += bin->stats.nruns; 2080242844Sjasone bstats[i].reruns += bin->stats.reruns; 2081242844Sjasone bstats[i].curruns += bin->stats.curruns; 2082242844Sjasone malloc_mutex_unlock(&bin->lock); 2083242844Sjasone } 2084242844Sjasone} 2085242844Sjasone 2086234370Sjasonebool 2087234370Sjasonearena_new(arena_t *arena, unsigned ind) 2088234370Sjasone{ 2089234370Sjasone unsigned i; 2090234370Sjasone arena_bin_t *bin; 2091234370Sjasone 2092234370Sjasone arena->ind = ind; 2093234370Sjasone arena->nthreads = 0; 2094234370Sjasone 2095234370Sjasone if (malloc_mutex_init(&arena->lock)) 2096234370Sjasone return (true); 2097234370Sjasone 2098234370Sjasone if (config_stats) { 2099234370Sjasone memset(&arena->stats, 0, sizeof(arena_stats_t)); 2100234370Sjasone arena->stats.lstats = 2101234370Sjasone (malloc_large_stats_t *)base_alloc(nlclasses * 2102234370Sjasone sizeof(malloc_large_stats_t)); 2103234370Sjasone if (arena->stats.lstats == NULL) 2104234370Sjasone return (true); 2105234370Sjasone memset(arena->stats.lstats, 0, nlclasses * 2106234370Sjasone sizeof(malloc_large_stats_t)); 2107234370Sjasone if (config_tcache) 2108234370Sjasone ql_new(&arena->tcache_ql); 2109234370Sjasone } 2110234370Sjasone 2111234370Sjasone if (config_prof) 2112234370Sjasone arena->prof_accumbytes = 0; 2113234370Sjasone 2114242844Sjasone arena->dss_prec = chunk_dss_prec_get(); 2115242844Sjasone 2116234370Sjasone /* Initialize chunks. */ 2117242844Sjasone arena_chunk_dirty_new(&arena->chunks_dirty); 2118234370Sjasone arena->spare = NULL; 2119234370Sjasone 2120234370Sjasone arena->nactive = 0; 2121234370Sjasone arena->ndirty = 0; 2122234370Sjasone arena->npurgatory = 0; 2123234370Sjasone 2124242844Sjasone arena_avail_tree_new(&arena->runs_avail); 2125234370Sjasone 2126234370Sjasone /* Initialize bins. */ 2127234370Sjasone for (i = 0; i < NBINS; i++) { 2128234370Sjasone bin = &arena->bins[i]; 2129234370Sjasone if (malloc_mutex_init(&bin->lock)) 2130234370Sjasone return (true); 2131234370Sjasone bin->runcur = NULL; 2132234370Sjasone arena_run_tree_new(&bin->runs); 2133234370Sjasone if (config_stats) 2134234370Sjasone memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2135234370Sjasone } 2136234370Sjasone 2137234370Sjasone return (false); 2138234370Sjasone} 2139234370Sjasone 2140234370Sjasone/* 2141234370Sjasone * Calculate bin_info->run_size such that it meets the following constraints: 2142234370Sjasone * 2143234370Sjasone * *) bin_info->run_size >= min_run_size 2144234370Sjasone * *) bin_info->run_size <= arena_maxclass 2145234370Sjasone * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 2146234370Sjasone * *) bin_info->nregs <= RUN_MAXREGS 2147234370Sjasone * 2148234370Sjasone * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 2149234370Sjasone * calculated here, since these settings are all interdependent. 2150234370Sjasone */ 2151234370Sjasonestatic size_t 2152234370Sjasonebin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 2153234370Sjasone{ 2154234370Sjasone size_t pad_size; 2155234370Sjasone size_t try_run_size, good_run_size; 2156234370Sjasone uint32_t try_nregs, good_nregs; 2157234370Sjasone uint32_t try_hdr_size, good_hdr_size; 2158234370Sjasone uint32_t try_bitmap_offset, good_bitmap_offset; 2159234370Sjasone uint32_t try_ctx0_offset, good_ctx0_offset; 2160234370Sjasone uint32_t try_redzone0_offset, good_redzone0_offset; 2161234370Sjasone 2162234370Sjasone assert(min_run_size >= PAGE); 2163234370Sjasone assert(min_run_size <= arena_maxclass); 2164234370Sjasone 2165234370Sjasone /* 2166234370Sjasone * Determine redzone size based on minimum alignment and minimum 2167234370Sjasone * redzone size. Add padding to the end of the run if it is needed to 2168234370Sjasone * align the regions. The padding allows each redzone to be half the 2169234370Sjasone * minimum alignment; without the padding, each redzone would have to 2170234370Sjasone * be twice as large in order to maintain alignment. 2171234370Sjasone */ 2172234370Sjasone if (config_fill && opt_redzone) { 2173234370Sjasone size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); 2174234370Sjasone if (align_min <= REDZONE_MINSIZE) { 2175234370Sjasone bin_info->redzone_size = REDZONE_MINSIZE; 2176234370Sjasone pad_size = 0; 2177234370Sjasone } else { 2178234370Sjasone bin_info->redzone_size = align_min >> 1; 2179234370Sjasone pad_size = bin_info->redzone_size; 2180234370Sjasone } 2181234370Sjasone } else { 2182234370Sjasone bin_info->redzone_size = 0; 2183234370Sjasone pad_size = 0; 2184234370Sjasone } 2185234370Sjasone bin_info->reg_interval = bin_info->reg_size + 2186234370Sjasone (bin_info->redzone_size << 1); 2187234370Sjasone 2188234370Sjasone /* 2189234370Sjasone * Calculate known-valid settings before entering the run_size 2190234370Sjasone * expansion loop, so that the first part of the loop always copies 2191234370Sjasone * valid settings. 2192234370Sjasone * 2193234370Sjasone * The do..while loop iteratively reduces the number of regions until 2194234370Sjasone * the run header and the regions no longer overlap. A closed formula 2195234370Sjasone * would be quite messy, since there is an interdependency between the 2196234370Sjasone * header's mask length and the number of regions. 2197234370Sjasone */ 2198234370Sjasone try_run_size = min_run_size; 2199234370Sjasone try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2200234370Sjasone bin_info->reg_interval) 2201234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2202234370Sjasone if (try_nregs > RUN_MAXREGS) { 2203234370Sjasone try_nregs = RUN_MAXREGS 2204234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2205234370Sjasone } 2206234370Sjasone do { 2207234370Sjasone try_nregs--; 2208234370Sjasone try_hdr_size = sizeof(arena_run_t); 2209234370Sjasone /* Pad to a long boundary. */ 2210234370Sjasone try_hdr_size = LONG_CEILING(try_hdr_size); 2211234370Sjasone try_bitmap_offset = try_hdr_size; 2212234370Sjasone /* Add space for bitmap. */ 2213234370Sjasone try_hdr_size += bitmap_size(try_nregs); 2214234370Sjasone if (config_prof && opt_prof && prof_promote == false) { 2215234370Sjasone /* Pad to a quantum boundary. */ 2216234370Sjasone try_hdr_size = QUANTUM_CEILING(try_hdr_size); 2217234370Sjasone try_ctx0_offset = try_hdr_size; 2218234370Sjasone /* Add space for one (prof_ctx_t *) per region. */ 2219234370Sjasone try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 2220234370Sjasone } else 2221234370Sjasone try_ctx0_offset = 0; 2222234370Sjasone try_redzone0_offset = try_run_size - (try_nregs * 2223234370Sjasone bin_info->reg_interval) - pad_size; 2224234370Sjasone } while (try_hdr_size > try_redzone0_offset); 2225234370Sjasone 2226234370Sjasone /* run_size expansion loop. */ 2227234370Sjasone do { 2228234370Sjasone /* 2229234370Sjasone * Copy valid settings before trying more aggressive settings. 2230234370Sjasone */ 2231234370Sjasone good_run_size = try_run_size; 2232234370Sjasone good_nregs = try_nregs; 2233234370Sjasone good_hdr_size = try_hdr_size; 2234234370Sjasone good_bitmap_offset = try_bitmap_offset; 2235234370Sjasone good_ctx0_offset = try_ctx0_offset; 2236234370Sjasone good_redzone0_offset = try_redzone0_offset; 2237234370Sjasone 2238234370Sjasone /* Try more aggressive settings. */ 2239234370Sjasone try_run_size += PAGE; 2240234370Sjasone try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2241234370Sjasone bin_info->reg_interval) 2242234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2243234370Sjasone if (try_nregs > RUN_MAXREGS) { 2244234370Sjasone try_nregs = RUN_MAXREGS 2245234370Sjasone + 1; /* Counter-act try_nregs-- in loop. */ 2246234370Sjasone } 2247234370Sjasone do { 2248234370Sjasone try_nregs--; 2249234370Sjasone try_hdr_size = sizeof(arena_run_t); 2250234370Sjasone /* Pad to a long boundary. */ 2251234370Sjasone try_hdr_size = LONG_CEILING(try_hdr_size); 2252234370Sjasone try_bitmap_offset = try_hdr_size; 2253234370Sjasone /* Add space for bitmap. */ 2254234370Sjasone try_hdr_size += bitmap_size(try_nregs); 2255234370Sjasone if (config_prof && opt_prof && prof_promote == false) { 2256234370Sjasone /* Pad to a quantum boundary. */ 2257234370Sjasone try_hdr_size = QUANTUM_CEILING(try_hdr_size); 2258234370Sjasone try_ctx0_offset = try_hdr_size; 2259234370Sjasone /* 2260234370Sjasone * Add space for one (prof_ctx_t *) per region. 2261234370Sjasone */ 2262234370Sjasone try_hdr_size += try_nregs * 2263234370Sjasone sizeof(prof_ctx_t *); 2264234370Sjasone } 2265234370Sjasone try_redzone0_offset = try_run_size - (try_nregs * 2266234370Sjasone bin_info->reg_interval) - pad_size; 2267234370Sjasone } while (try_hdr_size > try_redzone0_offset); 2268234370Sjasone } while (try_run_size <= arena_maxclass 2269234370Sjasone && try_run_size <= arena_maxclass 2270234370Sjasone && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2271234370Sjasone RUN_MAX_OVRHD_RELAX 2272234370Sjasone && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 2273234370Sjasone && try_nregs < RUN_MAXREGS); 2274234370Sjasone 2275234370Sjasone assert(good_hdr_size <= good_redzone0_offset); 2276234370Sjasone 2277234370Sjasone /* Copy final settings. */ 2278234370Sjasone bin_info->run_size = good_run_size; 2279234370Sjasone bin_info->nregs = good_nregs; 2280234370Sjasone bin_info->bitmap_offset = good_bitmap_offset; 2281234370Sjasone bin_info->ctx0_offset = good_ctx0_offset; 2282234370Sjasone bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2283234370Sjasone 2284234370Sjasone assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2285234370Sjasone * bin_info->reg_interval) + pad_size == bin_info->run_size); 2286234370Sjasone 2287234370Sjasone return (good_run_size); 2288234370Sjasone} 2289234370Sjasone 2290234370Sjasonestatic void 2291234370Sjasonebin_info_init(void) 2292234370Sjasone{ 2293234370Sjasone arena_bin_info_t *bin_info; 2294234370Sjasone size_t prev_run_size = PAGE; 2295234370Sjasone 2296234370Sjasone#define SIZE_CLASS(bin, delta, size) \ 2297234370Sjasone bin_info = &arena_bin_info[bin]; \ 2298234370Sjasone bin_info->reg_size = size; \ 2299234370Sjasone prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2300234370Sjasone bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2301234370Sjasone SIZE_CLASSES 2302234370Sjasone#undef SIZE_CLASS 2303234370Sjasone} 2304234370Sjasone 2305234370Sjasonevoid 2306234370Sjasonearena_boot(void) 2307234370Sjasone{ 2308234370Sjasone size_t header_size; 2309234370Sjasone unsigned i; 2310234370Sjasone 2311234370Sjasone /* 2312234370Sjasone * Compute the header size such that it is large enough to contain the 2313234370Sjasone * page map. The page map is biased to omit entries for the header 2314234370Sjasone * itself, so some iteration is necessary to compute the map bias. 2315234370Sjasone * 2316234370Sjasone * 1) Compute safe header_size and map_bias values that include enough 2317234370Sjasone * space for an unbiased page map. 2318234370Sjasone * 2) Refine map_bias based on (1) to omit the header pages in the page 2319234370Sjasone * map. The resulting map_bias may be one too small. 2320234370Sjasone * 3) Refine map_bias based on (2). The result will be >= the result 2321234370Sjasone * from (2), and will always be correct. 2322234370Sjasone */ 2323234370Sjasone map_bias = 0; 2324234370Sjasone for (i = 0; i < 3; i++) { 2325234370Sjasone header_size = offsetof(arena_chunk_t, map) + 2326234370Sjasone (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2327234370Sjasone map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2328234370Sjasone != 0); 2329234370Sjasone } 2330234370Sjasone assert(map_bias > 0); 2331234370Sjasone 2332234370Sjasone arena_maxclass = chunksize - (map_bias << LG_PAGE); 2333234370Sjasone 2334234370Sjasone bin_info_init(); 2335234370Sjasone} 2336234370Sjasone 2337234370Sjasonevoid 2338234370Sjasonearena_prefork(arena_t *arena) 2339234370Sjasone{ 2340234370Sjasone unsigned i; 2341234370Sjasone 2342234370Sjasone malloc_mutex_prefork(&arena->lock); 2343234370Sjasone for (i = 0; i < NBINS; i++) 2344234370Sjasone malloc_mutex_prefork(&arena->bins[i].lock); 2345234370Sjasone} 2346234370Sjasone 2347234370Sjasonevoid 2348234370Sjasonearena_postfork_parent(arena_t *arena) 2349234370Sjasone{ 2350234370Sjasone unsigned i; 2351234370Sjasone 2352234370Sjasone for (i = 0; i < NBINS; i++) 2353234370Sjasone malloc_mutex_postfork_parent(&arena->bins[i].lock); 2354234370Sjasone malloc_mutex_postfork_parent(&arena->lock); 2355234370Sjasone} 2356234370Sjasone 2357234370Sjasonevoid 2358234370Sjasonearena_postfork_child(arena_t *arena) 2359234370Sjasone{ 2360234370Sjasone unsigned i; 2361234370Sjasone 2362234370Sjasone for (i = 0; i < NBINS; i++) 2363234370Sjasone malloc_mutex_postfork_child(&arena->bins[i].lock); 2364234370Sjasone malloc_mutex_postfork_child(&arena->lock); 2365234370Sjasone} 2366