14c14
< size_t arena_maxclass; /* Max size class for arenas. */
---
> size_t large_maxclass; /* Max large size class. */
42c42
< return ((arena_chunk_map_misc_t *)((size << CHUNK_MAP_SIZE_SHIFT) |
---
> return ((arena_chunk_map_misc_t *)(arena_mapbits_size_encode(size) |
61,62c61
< return (((uintptr_t)miscelm & CHUNK_MAP_SIZE_MASK) >>
< CHUNK_MAP_SIZE_SHIFT);
---
> return (arena_mapbits_size_decode((uintptr_t)miscelm));
76c75
< return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT);
---
> return (arena_mapbits_size_decode(mapbits));
318c317
< index_t binind = arena_ptr_small_binind_get(ptr, mapbits);
---
> szind_t binind = arena_ptr_small_binind_get(ptr, mapbits);
429c428
< size_t flag_dirty, flag_decommitted, run_ind, need_pages, i;
---
> size_t flag_dirty, flag_decommitted, run_ind, need_pages;
462a462
> size_t i;
511c511
< index_t binind)
---
> szind_t binind)
783c783
< index_t index = size2index(usize) - nlclasses - NBINS;
---
> szind_t index = size2index(usize) - nlclasses - NBINS;
796c796
< index_t index = size2index(usize) - nlclasses - NBINS;
---
> szind_t index = size2index(usize) - nlclasses - NBINS;
809c809
< index_t index = size2index(usize) - nlclasses - NBINS;
---
> szind_t index = size2index(usize) - nlclasses - NBINS;
822c822
< index_t index = size2index(usize) - nlclasses - NBINS;
---
> szind_t index = size2index(usize) - nlclasses - NBINS;
1128c1128
< arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind)
---
> arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind)
1139c1139
< arena_run_alloc_small(arena_t *arena, size_t size, index_t binind)
---
> arena_run_alloc_small(arena_t *arena, size_t size, szind_t binind)
1892c1892
< index_t binind;
---
> szind_t binind;
1942,1943c1942
< void *ret;
< index_t binind;
---
> szind_t binind;
1955a1955
> void *ret;
1989c1989
< arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind,
---
> arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind,
1994,1995d1993
< arena_run_t *run;
< void *ptr;
2004a2003,2004
> arena_run_t *run;
> void *ptr;
2079,2081d2078
< size_t size = bin_info->reg_size;
< size_t redzone_size = bin_info->redzone_size;
< size_t i;
2084a2082,2085
> size_t size = bin_info->reg_size;
> size_t redzone_size = bin_info->redzone_size;
> size_t i;
>
2134c2135
< index_t binind;
---
> szind_t binind;
2152c2153
< index_t binind;
---
> szind_t binind;
2236c2237
< index_t index = size2index(usize) - NBINS;
---
> szind_t index = size2index(usize) - NBINS;
2329c2330
< index_t index = size2index(usize) - NBINS;
---
> szind_t index = size2index(usize) - NBINS;
2359c2360
< } else if (usize <= arena_maxclass && alignment <= PAGE) {
---
> } else if (usize <= large_maxclass && alignment <= PAGE) {
2370c2371
< if (likely(usize <= arena_maxclass)) {
---
> if (likely(usize <= large_maxclass)) {
2388c2389
< index_t binind;
---
> szind_t binind;
2416c2417
< index_t binind = arena_bin_index(extent_node_arena_get(
---
> szind_t binind = arena_bin_index(extent_node_arena_get(
2480c2481
< index_t binind;
---
> szind_t binind;
2562c2563
< void
---
> static void
2577c2578
< index_t index = size2index(usize) - NBINS;
---
> szind_t index = size2index(usize) - NBINS;
2624,2625c2625,2626
< index_t oldindex = size2index(oldsize) - NBINS;
< index_t index = size2index(size) - NBINS;
---
> szind_t oldindex = size2index(oldsize) - NBINS;
> szind_t index = size2index(size) - NBINS;
2644c2645
< size_t oldsize, size_t size, size_t extra, bool zero)
---
> size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
2649d2649
< size_t usize_min = s2u(size);
2655d2654
< assert(usize_min > oldsize);
2657,2660c2656,2660
< if (pageind+npages < chunk_npages &&
< arena_mapbits_allocated_get(chunk, pageind+npages) == 0 &&
< (followsize = arena_mapbits_unallocated_size_get(chunk,
< pageind+npages)) >= usize_min - oldsize) {
---
> if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk,
> pageind+npages) != 0)
> goto label_fail;
> followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages);
> if (oldsize + followsize >= usize_min) {
2667c2667
< size_t flag_dirty, flag_unzeroed_mask, splitsize, usize;
---
> size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask;
2669c2669
< usize = s2u(size + extra);
---
> usize = usize_max;
2672a2673
> assert(usize >= oldsize);
2673a2675,2676
> if (splitsize == 0)
> goto label_fail;
2676,2679c2679,2680
< if (arena_run_split_large(arena, run, splitsize, zero)) {
< malloc_mutex_unlock(&arena->lock);
< return (true);
< }
---
> if (arena_run_split_large(arena, run, splitsize, zero))
> goto label_fail;
2703,2704c2704,2705
< index_t oldindex = size2index(oldsize) - NBINS;
< index_t index = size2index(size) - NBINS;
---
> szind_t oldindex = size2index(oldsize) - NBINS;
> szind_t index = size2index(size) - NBINS;
2720a2722
> label_fail:
2722d2723
<
2751,2752c2752,2753
< arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
< bool zero)
---
> arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min,
> size_t usize_max, bool zero)
2754c2755,2756
< size_t usize;
---
> arena_chunk_t *chunk;
> arena_t *arena;
2756,2762c2758,2759
< /* Make sure extra can't cause size_t overflow. */
< if (unlikely(extra >= arena_maxclass))
< return (true);
<
< usize = s2u(size + extra);
< if (usize == oldsize) {
< /* Same size class. */
---
> if (oldsize == usize_max) {
> /* Current size class is compatible and maximal. */
2764,2766c2761
< } else {
< arena_chunk_t *chunk;
< arena_t *arena;
---
> }
2768,2769c2763,2764
< chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
< arena = extent_node_arena_get(&chunk->node);
---
> chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
> arena = extent_node_arena_get(&chunk->node);
2771,2789c2766,2775
< if (usize < oldsize) {
< /* Fill before shrinking in order avoid a race. */
< arena_ralloc_junk_large(ptr, oldsize, usize);
< arena_ralloc_large_shrink(arena, chunk, ptr, oldsize,
< usize);
< return (false);
< } else {
< bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
< oldsize, size, extra, zero);
< if (config_fill && !ret && !zero) {
< if (unlikely(opt_junk_alloc)) {
< memset((void *)((uintptr_t)ptr +
< oldsize), 0xa5, isalloc(ptr,
< config_prof) - oldsize);
< } else if (unlikely(opt_zero)) {
< memset((void *)((uintptr_t)ptr +
< oldsize), 0, isalloc(ptr,
< config_prof) - oldsize);
< }
---
> if (oldsize < usize_max) {
> bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize,
> usize_min, usize_max, zero);
> if (config_fill && !ret && !zero) {
> if (unlikely(opt_junk_alloc)) {
> memset((void *)((uintptr_t)ptr + oldsize), 0xa5,
> isalloc(ptr, config_prof) - oldsize);
> } else if (unlikely(opt_zero)) {
> memset((void *)((uintptr_t)ptr + oldsize), 0,
> isalloc(ptr, config_prof) - oldsize);
2791d2776
< return (ret);
2792a2778
> return (ret);
2793a2780,2785
>
> assert(oldsize > usize_max);
> /* Fill before shrinking in order avoid a race. */
> arena_ralloc_junk_large(ptr, oldsize, usize_max);
> arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, usize_max);
> return (false);
2799a2792
> size_t usize_min, usize_max;
2801c2794,2796
< if (likely(size <= arena_maxclass)) {
---
> usize_min = s2u(size);
> usize_max = s2u(size + extra);
> if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) {
2806,2814c2801,2811
< if (likely(oldsize <= arena_maxclass)) {
< if (oldsize <= SMALL_MAXCLASS) {
< assert(
< arena_bin_info[size2index(oldsize)].reg_size
< == oldsize);
< if ((size + extra <= SMALL_MAXCLASS &&
< size2index(size + extra) ==
< size2index(oldsize)) || (size <= oldsize &&
< size + extra >= oldsize))
---
> if (oldsize <= SMALL_MAXCLASS) {
> assert(arena_bin_info[size2index(oldsize)].reg_size ==
> oldsize);
> if ((usize_max <= SMALL_MAXCLASS &&
> size2index(usize_max) == size2index(oldsize)) ||
> (size <= oldsize && usize_max >= oldsize))
> return (false);
> } else {
> if (usize_max > SMALL_MAXCLASS) {
> if (!arena_ralloc_large(ptr, oldsize, usize_min,
> usize_max, zero))
2816,2822d2812
< } else {
< assert(size <= arena_maxclass);
< if (size + extra > SMALL_MAXCLASS) {
< if (!arena_ralloc_large(ptr, oldsize,
< size, extra, zero))
< return (false);
< }
2828,2829c2818,2821
< } else
< return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));
---
> } else {
> return (huge_ralloc_no_move(ptr, oldsize, usize_min, usize_max,
> zero));
> }
2831a2824,2836
> static void *
> arena_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize,
> size_t alignment, bool zero, tcache_t *tcache)
> {
>
> if (alignment == 0)
> return (arena_malloc(tsd, arena, usize, zero, tcache));
> usize = sa2u(usize, alignment);
> if (usize == 0)
> return (NULL);
> return (ipalloct(tsd, usize, alignment, zero, tcache, arena));
> }
>
2834c2839
< size_t extra, size_t alignment, bool zero, tcache_t *tcache)
---
> size_t alignment, bool zero, tcache_t *tcache)
2836a2842
> size_t usize;
2838c2844,2848
< if (likely(size <= arena_maxclass)) {
---
> usize = s2u(size);
> if (usize == 0)
> return (NULL);
>
> if (likely(usize <= large_maxclass)) {
2842c2852
< if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero))
---
> if (!arena_ralloc_no_move(ptr, oldsize, usize, 0, zero))
2850,2859c2860,2863
< if (alignment != 0) {
< size_t usize = sa2u(size + extra, alignment);
< if (usize == 0)
< return (NULL);
< ret = ipalloct(tsd, usize, alignment, zero, tcache,
< arena);
< } else {
< ret = arena_malloc(tsd, arena, size + extra, zero,
< tcache);
< }
---
> ret = arena_ralloc_move_helper(tsd, arena, usize, alignment,
> zero, tcache);
> if (ret == NULL)
> return (NULL);
2861,2879d2864
< if (ret == NULL) {
< if (extra == 0)
< return (NULL);
< /* Try again, this time without extra. */
< if (alignment != 0) {
< size_t usize = sa2u(size, alignment);
< if (usize == 0)
< return (NULL);
< ret = ipalloct(tsd, usize, alignment, zero,
< tcache, arena);
< } else {
< ret = arena_malloc(tsd, arena, size, zero,
< tcache);
< }
<
< if (ret == NULL)
< return (NULL);
< }
<
2885,2890c2870
< /*
< * Copy at most size bytes (not size+extra), since the caller
< * has no expectation that the extra bytes will be reliably
< * preserved.
< */
< copysize = (size < oldsize) ? size : oldsize;
---
> copysize = (usize < oldsize) ? usize : oldsize;
2895,2896c2875,2876
< ret = huge_ralloc(tsd, arena, ptr, oldsize, size, extra,
< alignment, zero, tcache);
---
> ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment,
> zero, tcache);
3244d3223
< size_t header_size;
3263c3242
< header_size = offsetof(arena_chunk_t, map_bits) +
---
> size_t header_size = offsetof(arena_chunk_t, map_bits) +
3275,3276c3254,3255
< arena_maxclass = index2size(size2index(chunksize)-1);
< if (arena_maxclass > arena_maxrun) {
---
> large_maxclass = index2size(size2index(chunksize)-1);
> if (large_maxclass > arena_maxrun) {
3282c3261
< arena_maxclass = arena_maxrun;
---
> large_maxclass = arena_maxrun;
3284,3285c3263,3264
< assert(arena_maxclass > 0);
< nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS);
---
> assert(large_maxclass > 0);
> nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS);