1#define JEMALLOC_ARENA_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
8static ssize_t lg_dirty_mult_default;
9arena_bin_info_t arena_bin_info[NBINS];
10
11size_t map_bias;
12size_t map_misc_offset;
13size_t arena_maxrun; /* Max run size for arenas. */
| 1#define JEMALLOC_ARENA_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
8static ssize_t lg_dirty_mult_default;
9arena_bin_info_t arena_bin_info[NBINS];
10
11size_t map_bias;
12size_t map_misc_offset;
13size_t arena_maxrun; /* Max run size for arenas. */
|
14size_t arena_maxclass; /* Max size class for arenas. */
| 14size_t large_maxclass; /* Max large size class. */
|
15static size_t small_maxrun; /* Max run size used for small size classes. */
16static bool *small_run_tab; /* Valid small run page multiples. */
17unsigned nlclasses; /* Number of large size classes. */
18unsigned nhclasses; /* Number of huge size classes. */
19
20/******************************************************************************/
21/*
22 * Function prototypes for static functions that are referenced prior to
23 * definition.
24 */
25
26static void arena_purge(arena_t *arena, bool all);
27static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty,
28 bool cleaned, bool decommitted);
29static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk,
30 arena_run_t *run, arena_bin_t *bin);
31static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
32 arena_run_t *run, arena_bin_t *bin);
33
34/******************************************************************************/
35
36#define CHUNK_MAP_KEY ((uintptr_t)0x1U)
37
38JEMALLOC_INLINE_C arena_chunk_map_misc_t *
39arena_miscelm_key_create(size_t size)
40{
41
| 15static size_t small_maxrun; /* Max run size used for small size classes. */
16static bool *small_run_tab; /* Valid small run page multiples. */
17unsigned nlclasses; /* Number of large size classes. */
18unsigned nhclasses; /* Number of huge size classes. */
19
20/******************************************************************************/
21/*
22 * Function prototypes for static functions that are referenced prior to
23 * definition.
24 */
25
26static void arena_purge(arena_t *arena, bool all);
27static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty,
28 bool cleaned, bool decommitted);
29static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk,
30 arena_run_t *run, arena_bin_t *bin);
31static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
32 arena_run_t *run, arena_bin_t *bin);
33
34/******************************************************************************/
35
36#define CHUNK_MAP_KEY ((uintptr_t)0x1U)
37
38JEMALLOC_INLINE_C arena_chunk_map_misc_t *
39arena_miscelm_key_create(size_t size)
40{
41
|
42 return ((arena_chunk_map_misc_t *)((size << CHUNK_MAP_SIZE_SHIFT) |
| 42 return ((arena_chunk_map_misc_t *)(arena_mapbits_size_encode(size) |
|
43 CHUNK_MAP_KEY));
44}
45
46JEMALLOC_INLINE_C bool
47arena_miscelm_is_key(const arena_chunk_map_misc_t *miscelm)
48{
49
50 return (((uintptr_t)miscelm & CHUNK_MAP_KEY) != 0);
51}
52
53#undef CHUNK_MAP_KEY
54
55JEMALLOC_INLINE_C size_t
56arena_miscelm_key_size_get(const arena_chunk_map_misc_t *miscelm)
57{
58
59 assert(arena_miscelm_is_key(miscelm));
60
| 43 CHUNK_MAP_KEY));
44}
45
46JEMALLOC_INLINE_C bool
47arena_miscelm_is_key(const arena_chunk_map_misc_t *miscelm)
48{
49
50 return (((uintptr_t)miscelm & CHUNK_MAP_KEY) != 0);
51}
52
53#undef CHUNK_MAP_KEY
54
55JEMALLOC_INLINE_C size_t
56arena_miscelm_key_size_get(const arena_chunk_map_misc_t *miscelm)
57{
58
59 assert(arena_miscelm_is_key(miscelm));
60
|
61 return (((uintptr_t)miscelm & CHUNK_MAP_SIZE_MASK) >>
62 CHUNK_MAP_SIZE_SHIFT);
| 61 return (arena_mapbits_size_decode((uintptr_t)miscelm));
|
63}
64
65JEMALLOC_INLINE_C size_t
66arena_miscelm_size_get(arena_chunk_map_misc_t *miscelm)
67{
68 arena_chunk_t *chunk;
69 size_t pageind, mapbits;
70
71 assert(!arena_miscelm_is_key(miscelm));
72
73 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm);
74 pageind = arena_miscelm_to_pageind(miscelm);
75 mapbits = arena_mapbits_get(chunk, pageind);
| 62}
63
64JEMALLOC_INLINE_C size_t
65arena_miscelm_size_get(arena_chunk_map_misc_t *miscelm)
66{
67 arena_chunk_t *chunk;
68 size_t pageind, mapbits;
69
70 assert(!arena_miscelm_is_key(miscelm));
71
72 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm);
73 pageind = arena_miscelm_to_pageind(miscelm);
74 mapbits = arena_mapbits_get(chunk, pageind);
|
76 return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT);
| 75 return (arena_mapbits_size_decode(mapbits));
|
77}
78
79JEMALLOC_INLINE_C int
80arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b)
81{
82 uintptr_t a_miscelm = (uintptr_t)a;
83 uintptr_t b_miscelm = (uintptr_t)b;
84
85 assert(a != NULL);
86 assert(b != NULL);
87
88 return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm));
89}
90
91/* Generate red-black tree functions. */
92rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_misc_t,
93 rb_link, arena_run_comp)
94
95static size_t
96run_quantize(size_t size)
97{
98 size_t qsize;
99
100 assert(size != 0);
101 assert(size == PAGE_CEILING(size));
102
103 /* Don't change sizes that are valid small run sizes. */
104 if (size <= small_maxrun && small_run_tab[size >> LG_PAGE])
105 return (size);
106
107 /*
108 * Round down to the nearest run size that can actually be requested
109 * during normal large allocation. Add large_pad so that cache index
110 * randomization can offset the allocation from the page boundary.
111 */
112 qsize = index2size(size2index(size - large_pad + 1) - 1) + large_pad;
113 if (qsize <= SMALL_MAXCLASS + large_pad)
114 return (run_quantize(size - large_pad));
115 assert(qsize <= size);
116 return (qsize);
117}
118
119static size_t
120run_quantize_next(size_t size)
121{
122 size_t large_run_size_next;
123
124 assert(size != 0);
125 assert(size == PAGE_CEILING(size));
126
127 /*
128 * Return the next quantized size greater than the input size.
129 * Quantized sizes comprise the union of run sizes that back small
130 * region runs, and run sizes that back large regions with no explicit
131 * alignment constraints.
132 */
133
134 if (size > SMALL_MAXCLASS) {
135 large_run_size_next = PAGE_CEILING(index2size(size2index(size -
136 large_pad) + 1) + large_pad);
137 } else
138 large_run_size_next = SIZE_T_MAX;
139 if (size >= small_maxrun)
140 return (large_run_size_next);
141
142 while (true) {
143 size += PAGE;
144 assert(size <= small_maxrun);
145 if (small_run_tab[size >> LG_PAGE]) {
146 if (large_run_size_next < size)
147 return (large_run_size_next);
148 return (size);
149 }
150 }
151}
152
153static size_t
154run_quantize_first(size_t size)
155{
156 size_t qsize = run_quantize(size);
157
158 if (qsize < size) {
159 /*
160 * Skip a quantization that may have an adequately large run,
161 * because under-sized runs may be mixed in. This only happens
162 * when an unusual size is requested, i.e. for aligned
163 * allocation, and is just one of several places where linear
164 * search would potentially find sufficiently aligned available
165 * memory somewhere lower.
166 */
167 qsize = run_quantize_next(size);
168 }
169 return (qsize);
170}
171
172JEMALLOC_INLINE_C int
173arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b)
174{
175 int ret;
176 uintptr_t a_miscelm = (uintptr_t)a;
177 size_t a_qsize = run_quantize(arena_miscelm_is_key(a) ?
178 arena_miscelm_key_size_get(a) : arena_miscelm_size_get(a));
179 size_t b_qsize = run_quantize(arena_miscelm_size_get(b));
180
181 /*
182 * Compare based on quantized size rather than size, in order to sort
183 * equally useful runs only by address.
184 */
185 ret = (a_qsize > b_qsize) - (a_qsize < b_qsize);
186 if (ret == 0) {
187 if (!arena_miscelm_is_key(a)) {
188 uintptr_t b_miscelm = (uintptr_t)b;
189
190 ret = (a_miscelm > b_miscelm) - (a_miscelm < b_miscelm);
191 } else {
192 /*
193 * Treat keys as if they are lower than anything else.
194 */
195 ret = -1;
196 }
197 }
198
199 return (ret);
200}
201
202/* Generate red-black tree functions. */
203rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t,
204 arena_chunk_map_misc_t, rb_link, arena_avail_comp)
205
206static void
207arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
208 size_t npages)
209{
210
211 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
212 LG_PAGE));
213 arena_avail_tree_insert(&arena->runs_avail, arena_miscelm_get(chunk,
214 pageind));
215}
216
217static void
218arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
219 size_t npages)
220{
221
222 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
223 LG_PAGE));
224 arena_avail_tree_remove(&arena->runs_avail, arena_miscelm_get(chunk,
225 pageind));
226}
227
228static void
229arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
230 size_t npages)
231{
232 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
233
234 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
235 LG_PAGE));
236 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY);
237 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
238 CHUNK_MAP_DIRTY);
239
240 qr_new(&miscelm->rd, rd_link);
241 qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link);
242 arena->ndirty += npages;
243}
244
245static void
246arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
247 size_t npages)
248{
249 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
250
251 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
252 LG_PAGE));
253 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY);
254 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
255 CHUNK_MAP_DIRTY);
256
257 qr_remove(&miscelm->rd, rd_link);
258 assert(arena->ndirty >= npages);
259 arena->ndirty -= npages;
260}
261
262static size_t
263arena_chunk_dirty_npages(const extent_node_t *node)
264{
265
266 return (extent_node_size_get(node) >> LG_PAGE);
267}
268
269void
270arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache)
271{
272
273 if (cache) {
274 extent_node_dirty_linkage_init(node);
275 extent_node_dirty_insert(node, &arena->runs_dirty,
276 &arena->chunks_cache);
277 arena->ndirty += arena_chunk_dirty_npages(node);
278 }
279}
280
281void
282arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty)
283{
284
285 if (dirty) {
286 extent_node_dirty_remove(node);
287 assert(arena->ndirty >= arena_chunk_dirty_npages(node));
288 arena->ndirty -= arena_chunk_dirty_npages(node);
289 }
290}
291
292JEMALLOC_INLINE_C void *
293arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info)
294{
295 void *ret;
296 unsigned regind;
297 arena_chunk_map_misc_t *miscelm;
298 void *rpages;
299
300 assert(run->nfree > 0);
301 assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info));
302
303 regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info);
304 miscelm = arena_run_to_miscelm(run);
305 rpages = arena_miscelm_to_rpages(miscelm);
306 ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset +
307 (uintptr_t)(bin_info->reg_interval * regind));
308 run->nfree--;
309 return (ret);
310}
311
312JEMALLOC_INLINE_C void
313arena_run_reg_dalloc(arena_run_t *run, void *ptr)
314{
315 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
316 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
317 size_t mapbits = arena_mapbits_get(chunk, pageind);
| 76}
77
78JEMALLOC_INLINE_C int
79arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b)
80{
81 uintptr_t a_miscelm = (uintptr_t)a;
82 uintptr_t b_miscelm = (uintptr_t)b;
83
84 assert(a != NULL);
85 assert(b != NULL);
86
87 return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm));
88}
89
90/* Generate red-black tree functions. */
91rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_misc_t,
92 rb_link, arena_run_comp)
93
94static size_t
95run_quantize(size_t size)
96{
97 size_t qsize;
98
99 assert(size != 0);
100 assert(size == PAGE_CEILING(size));
101
102 /* Don't change sizes that are valid small run sizes. */
103 if (size <= small_maxrun && small_run_tab[size >> LG_PAGE])
104 return (size);
105
106 /*
107 * Round down to the nearest run size that can actually be requested
108 * during normal large allocation. Add large_pad so that cache index
109 * randomization can offset the allocation from the page boundary.
110 */
111 qsize = index2size(size2index(size - large_pad + 1) - 1) + large_pad;
112 if (qsize <= SMALL_MAXCLASS + large_pad)
113 return (run_quantize(size - large_pad));
114 assert(qsize <= size);
115 return (qsize);
116}
117
118static size_t
119run_quantize_next(size_t size)
120{
121 size_t large_run_size_next;
122
123 assert(size != 0);
124 assert(size == PAGE_CEILING(size));
125
126 /*
127 * Return the next quantized size greater than the input size.
128 * Quantized sizes comprise the union of run sizes that back small
129 * region runs, and run sizes that back large regions with no explicit
130 * alignment constraints.
131 */
132
133 if (size > SMALL_MAXCLASS) {
134 large_run_size_next = PAGE_CEILING(index2size(size2index(size -
135 large_pad) + 1) + large_pad);
136 } else
137 large_run_size_next = SIZE_T_MAX;
138 if (size >= small_maxrun)
139 return (large_run_size_next);
140
141 while (true) {
142 size += PAGE;
143 assert(size <= small_maxrun);
144 if (small_run_tab[size >> LG_PAGE]) {
145 if (large_run_size_next < size)
146 return (large_run_size_next);
147 return (size);
148 }
149 }
150}
151
152static size_t
153run_quantize_first(size_t size)
154{
155 size_t qsize = run_quantize(size);
156
157 if (qsize < size) {
158 /*
159 * Skip a quantization that may have an adequately large run,
160 * because under-sized runs may be mixed in. This only happens
161 * when an unusual size is requested, i.e. for aligned
162 * allocation, and is just one of several places where linear
163 * search would potentially find sufficiently aligned available
164 * memory somewhere lower.
165 */
166 qsize = run_quantize_next(size);
167 }
168 return (qsize);
169}
170
171JEMALLOC_INLINE_C int
172arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b)
173{
174 int ret;
175 uintptr_t a_miscelm = (uintptr_t)a;
176 size_t a_qsize = run_quantize(arena_miscelm_is_key(a) ?
177 arena_miscelm_key_size_get(a) : arena_miscelm_size_get(a));
178 size_t b_qsize = run_quantize(arena_miscelm_size_get(b));
179
180 /*
181 * Compare based on quantized size rather than size, in order to sort
182 * equally useful runs only by address.
183 */
184 ret = (a_qsize > b_qsize) - (a_qsize < b_qsize);
185 if (ret == 0) {
186 if (!arena_miscelm_is_key(a)) {
187 uintptr_t b_miscelm = (uintptr_t)b;
188
189 ret = (a_miscelm > b_miscelm) - (a_miscelm < b_miscelm);
190 } else {
191 /*
192 * Treat keys as if they are lower than anything else.
193 */
194 ret = -1;
195 }
196 }
197
198 return (ret);
199}
200
201/* Generate red-black tree functions. */
202rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t,
203 arena_chunk_map_misc_t, rb_link, arena_avail_comp)
204
205static void
206arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
207 size_t npages)
208{
209
210 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
211 LG_PAGE));
212 arena_avail_tree_insert(&arena->runs_avail, arena_miscelm_get(chunk,
213 pageind));
214}
215
216static void
217arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
218 size_t npages)
219{
220
221 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
222 LG_PAGE));
223 arena_avail_tree_remove(&arena->runs_avail, arena_miscelm_get(chunk,
224 pageind));
225}
226
227static void
228arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
229 size_t npages)
230{
231 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
232
233 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
234 LG_PAGE));
235 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY);
236 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
237 CHUNK_MAP_DIRTY);
238
239 qr_new(&miscelm->rd, rd_link);
240 qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link);
241 arena->ndirty += npages;
242}
243
244static void
245arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
246 size_t npages)
247{
248 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
249
250 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
251 LG_PAGE));
252 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY);
253 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
254 CHUNK_MAP_DIRTY);
255
256 qr_remove(&miscelm->rd, rd_link);
257 assert(arena->ndirty >= npages);
258 arena->ndirty -= npages;
259}
260
261static size_t
262arena_chunk_dirty_npages(const extent_node_t *node)
263{
264
265 return (extent_node_size_get(node) >> LG_PAGE);
266}
267
268void
269arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache)
270{
271
272 if (cache) {
273 extent_node_dirty_linkage_init(node);
274 extent_node_dirty_insert(node, &arena->runs_dirty,
275 &arena->chunks_cache);
276 arena->ndirty += arena_chunk_dirty_npages(node);
277 }
278}
279
280void
281arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty)
282{
283
284 if (dirty) {
285 extent_node_dirty_remove(node);
286 assert(arena->ndirty >= arena_chunk_dirty_npages(node));
287 arena->ndirty -= arena_chunk_dirty_npages(node);
288 }
289}
290
291JEMALLOC_INLINE_C void *
292arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info)
293{
294 void *ret;
295 unsigned regind;
296 arena_chunk_map_misc_t *miscelm;
297 void *rpages;
298
299 assert(run->nfree > 0);
300 assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info));
301
302 regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info);
303 miscelm = arena_run_to_miscelm(run);
304 rpages = arena_miscelm_to_rpages(miscelm);
305 ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset +
306 (uintptr_t)(bin_info->reg_interval * regind));
307 run->nfree--;
308 return (ret);
309}
310
311JEMALLOC_INLINE_C void
312arena_run_reg_dalloc(arena_run_t *run, void *ptr)
313{
314 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
315 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
316 size_t mapbits = arena_mapbits_get(chunk, pageind);
|
318 index_t binind = arena_ptr_small_binind_get(ptr, mapbits);
| 317 szind_t binind = arena_ptr_small_binind_get(ptr, mapbits);
|
319 arena_bin_info_t *bin_info = &arena_bin_info[binind];
320 unsigned regind = arena_run_regind(run, bin_info, ptr);
321
322 assert(run->nfree < bin_info->nregs);
323 /* Freeing an interior pointer can cause assertion failure. */
324 assert(((uintptr_t)ptr -
325 ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) +
326 (uintptr_t)bin_info->reg0_offset)) %
327 (uintptr_t)bin_info->reg_interval == 0);
328 assert((uintptr_t)ptr >=
329 (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) +
330 (uintptr_t)bin_info->reg0_offset);
331 /* Freeing an unallocated pointer can cause assertion failure. */
332 assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind));
333
334 bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind);
335 run->nfree++;
336}
337
338JEMALLOC_INLINE_C void
339arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
340{
341
342 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
343 (run_ind << LG_PAGE)), (npages << LG_PAGE));
344 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,
345 (npages << LG_PAGE));
346}
347
348JEMALLOC_INLINE_C void
349arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind)
350{
351
352 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind
353 << LG_PAGE)), PAGE);
354}
355
356JEMALLOC_INLINE_C void
357arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
358{
359 size_t i;
360 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));
361
362 arena_run_page_mark_zeroed(chunk, run_ind);
363 for (i = 0; i < PAGE / sizeof(size_t); i++)
364 assert(p[i] == 0);
365}
366
367static void
368arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages)
369{
370
371 if (config_stats) {
372 ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + add_pages
373 - sub_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive <<
374 LG_PAGE);
375 if (cactive_diff != 0)
376 stats_cactive_add(cactive_diff);
377 }
378}
379
380static void
381arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
382 size_t flag_dirty, size_t flag_decommitted, size_t need_pages)
383{
384 size_t total_pages, rem_pages;
385
386 assert(flag_dirty == 0 || flag_decommitted == 0);
387
388 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>
389 LG_PAGE;
390 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==
391 flag_dirty);
392 assert(need_pages <= total_pages);
393 rem_pages = total_pages - need_pages;
394
395 arena_avail_remove(arena, chunk, run_ind, total_pages);
396 if (flag_dirty != 0)
397 arena_run_dirty_remove(arena, chunk, run_ind, total_pages);
398 arena_cactive_update(arena, need_pages, 0);
399 arena->nactive += need_pages;
400
401 /* Keep track of trailing unused pages for later use. */
402 if (rem_pages > 0) {
403 size_t flags = flag_dirty | flag_decommitted;
404 size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED :
405 0;
406
407 arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
408 (rem_pages << LG_PAGE), flags |
409 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) &
410 flag_unzeroed_mask));
411 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1,
412 (rem_pages << LG_PAGE), flags |
413 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) &
414 flag_unzeroed_mask));
415 if (flag_dirty != 0) {
416 arena_run_dirty_insert(arena, chunk, run_ind+need_pages,
417 rem_pages);
418 }
419 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages);
420 }
421}
422
423static bool
424arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
425 bool remove, bool zero)
426{
427 arena_chunk_t *chunk;
428 arena_chunk_map_misc_t *miscelm;
| 318 arena_bin_info_t *bin_info = &arena_bin_info[binind];
319 unsigned regind = arena_run_regind(run, bin_info, ptr);
320
321 assert(run->nfree < bin_info->nregs);
322 /* Freeing an interior pointer can cause assertion failure. */
323 assert(((uintptr_t)ptr -
324 ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) +
325 (uintptr_t)bin_info->reg0_offset)) %
326 (uintptr_t)bin_info->reg_interval == 0);
327 assert((uintptr_t)ptr >=
328 (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) +
329 (uintptr_t)bin_info->reg0_offset);
330 /* Freeing an unallocated pointer can cause assertion failure. */
331 assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind));
332
333 bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind);
334 run->nfree++;
335}
336
337JEMALLOC_INLINE_C void
338arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
339{
340
341 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
342 (run_ind << LG_PAGE)), (npages << LG_PAGE));
343 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,
344 (npages << LG_PAGE));
345}
346
347JEMALLOC_INLINE_C void
348arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind)
349{
350
351 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind
352 << LG_PAGE)), PAGE);
353}
354
355JEMALLOC_INLINE_C void
356arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
357{
358 size_t i;
359 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));
360
361 arena_run_page_mark_zeroed(chunk, run_ind);
362 for (i = 0; i < PAGE / sizeof(size_t); i++)
363 assert(p[i] == 0);
364}
365
366static void
367arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages)
368{
369
370 if (config_stats) {
371 ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + add_pages
372 - sub_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive <<
373 LG_PAGE);
374 if (cactive_diff != 0)
375 stats_cactive_add(cactive_diff);
376 }
377}
378
379static void
380arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
381 size_t flag_dirty, size_t flag_decommitted, size_t need_pages)
382{
383 size_t total_pages, rem_pages;
384
385 assert(flag_dirty == 0 || flag_decommitted == 0);
386
387 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>
388 LG_PAGE;
389 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==
390 flag_dirty);
391 assert(need_pages <= total_pages);
392 rem_pages = total_pages - need_pages;
393
394 arena_avail_remove(arena, chunk, run_ind, total_pages);
395 if (flag_dirty != 0)
396 arena_run_dirty_remove(arena, chunk, run_ind, total_pages);
397 arena_cactive_update(arena, need_pages, 0);
398 arena->nactive += need_pages;
399
400 /* Keep track of trailing unused pages for later use. */
401 if (rem_pages > 0) {
402 size_t flags = flag_dirty | flag_decommitted;
403 size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED :
404 0;
405
406 arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
407 (rem_pages << LG_PAGE), flags |
408 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) &
409 flag_unzeroed_mask));
410 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1,
411 (rem_pages << LG_PAGE), flags |
412 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) &
413 flag_unzeroed_mask));
414 if (flag_dirty != 0) {
415 arena_run_dirty_insert(arena, chunk, run_ind+need_pages,
416 rem_pages);
417 }
418 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages);
419 }
420}
421
422static bool
423arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
424 bool remove, bool zero)
425{
426 arena_chunk_t *chunk;
427 arena_chunk_map_misc_t *miscelm;
|
429 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i;
| 428 size_t flag_dirty, flag_decommitted, run_ind, need_pages;
|
430 size_t flag_unzeroed_mask;
431
432 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
433 miscelm = arena_run_to_miscelm(run);
434 run_ind = arena_miscelm_to_pageind(miscelm);
435 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
436 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind);
437 need_pages = (size >> LG_PAGE);
438 assert(need_pages > 0);
439
440 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize,
441 run_ind << LG_PAGE, size, arena->ind))
442 return (true);
443
444 if (remove) {
445 arena_run_split_remove(arena, chunk, run_ind, flag_dirty,
446 flag_decommitted, need_pages);
447 }
448
449 if (zero) {
450 if (flag_decommitted != 0) {
451 /* The run is untouched, and therefore zeroed. */
452 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void
453 *)((uintptr_t)chunk + (run_ind << LG_PAGE)),
454 (need_pages << LG_PAGE));
455 } else if (flag_dirty != 0) {
456 /* The run is dirty, so all pages must be zeroed. */
457 arena_run_zero(chunk, run_ind, need_pages);
458 } else {
459 /*
460 * The run is clean, so some pages may be zeroed (i.e.
461 * never before touched).
462 */
| 429 size_t flag_unzeroed_mask;
430
431 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
432 miscelm = arena_run_to_miscelm(run);
433 run_ind = arena_miscelm_to_pageind(miscelm);
434 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
435 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind);
436 need_pages = (size >> LG_PAGE);
437 assert(need_pages > 0);
438
439 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize,
440 run_ind << LG_PAGE, size, arena->ind))
441 return (true);
442
443 if (remove) {
444 arena_run_split_remove(arena, chunk, run_ind, flag_dirty,
445 flag_decommitted, need_pages);
446 }
447
448 if (zero) {
449 if (flag_decommitted != 0) {
450 /* The run is untouched, and therefore zeroed. */
451 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void
452 *)((uintptr_t)chunk + (run_ind << LG_PAGE)),
453 (need_pages << LG_PAGE));
454 } else if (flag_dirty != 0) {
455 /* The run is dirty, so all pages must be zeroed. */
456 arena_run_zero(chunk, run_ind, need_pages);
457 } else {
458 /*
459 * The run is clean, so some pages may be zeroed (i.e.
460 * never before touched).
461 */
|
| 462 size_t i;
|
463 for (i = 0; i < need_pages; i++) {
464 if (arena_mapbits_unzeroed_get(chunk, run_ind+i)
465 != 0)
466 arena_run_zero(chunk, run_ind+i, 1);
467 else if (config_debug) {
468 arena_run_page_validate_zeroed(chunk,
469 run_ind+i);
470 } else {
471 arena_run_page_mark_zeroed(chunk,
472 run_ind+i);
473 }
474 }
475 }
476 } else {
477 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
478 (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
479 }
480
481 /*
482 * Set the last element first, in case the run only contains one page
483 * (i.e. both statements set the same element).
484 */
485 flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ?
486 CHUNK_MAP_UNZEROED : 0;
487 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty |
488 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
489 run_ind+need_pages-1)));
490 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty |
491 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind)));
492 return (false);
493}
494
495static bool
496arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
497{
498
499 return (arena_run_split_large_helper(arena, run, size, true, zero));
500}
501
502static bool
503arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
504{
505
506 return (arena_run_split_large_helper(arena, run, size, false, zero));
507}
508
509static bool
510arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
| 463 for (i = 0; i < need_pages; i++) {
464 if (arena_mapbits_unzeroed_get(chunk, run_ind+i)
465 != 0)
466 arena_run_zero(chunk, run_ind+i, 1);
467 else if (config_debug) {
468 arena_run_page_validate_zeroed(chunk,
469 run_ind+i);
470 } else {
471 arena_run_page_mark_zeroed(chunk,
472 run_ind+i);
473 }
474 }
475 }
476 } else {
477 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
478 (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
479 }
480
481 /*
482 * Set the last element first, in case the run only contains one page
483 * (i.e. both statements set the same element).
484 */
485 flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ?
486 CHUNK_MAP_UNZEROED : 0;
487 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty |
488 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
489 run_ind+need_pages-1)));
490 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty |
491 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind)));
492 return (false);
493}
494
495static bool
496arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
497{
498
499 return (arena_run_split_large_helper(arena, run, size, true, zero));
500}
501
502static bool
503arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
504{
505
506 return (arena_run_split_large_helper(arena, run, size, false, zero));
507}
508
509static bool
510arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
|
511 index_t binind)
| 511 szind_t binind)
|
512{
513 arena_chunk_t *chunk;
514 arena_chunk_map_misc_t *miscelm;
515 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i;
516
517 assert(binind != BININD_INVALID);
518
519 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
520 miscelm = arena_run_to_miscelm(run);
521 run_ind = arena_miscelm_to_pageind(miscelm);
522 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
523 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind);
524 need_pages = (size >> LG_PAGE);
525 assert(need_pages > 0);
526
527 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize,
528 run_ind << LG_PAGE, size, arena->ind))
529 return (true);
530
531 arena_run_split_remove(arena, chunk, run_ind, flag_dirty,
532 flag_decommitted, need_pages);
533
534 for (i = 0; i < need_pages; i++) {
535 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk,
536 run_ind+i);
537 arena_mapbits_small_set(chunk, run_ind+i, i, binind,
538 flag_unzeroed);
539 if (config_debug && flag_dirty == 0 && flag_unzeroed == 0)
540 arena_run_page_validate_zeroed(chunk, run_ind+i);
541 }
542 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
543 (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
544 return (false);
545}
546
547static arena_chunk_t *
548arena_chunk_init_spare(arena_t *arena)
549{
550 arena_chunk_t *chunk;
551
552 assert(arena->spare != NULL);
553
554 chunk = arena->spare;
555 arena->spare = NULL;
556
557 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
558 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
559 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
560 arena_maxrun);
561 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
562 arena_maxrun);
563 assert(arena_mapbits_dirty_get(chunk, map_bias) ==
564 arena_mapbits_dirty_get(chunk, chunk_npages-1));
565
566 return (chunk);
567}
568
569static bool
570arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero)
571{
572
573 /*
574 * The extent node notion of "committed" doesn't directly apply to
575 * arena chunks. Arbitrarily mark them as committed. The commit state
576 * of runs is tracked individually, and upon chunk deallocation the
577 * entire chunk is in a consistent commit state.
578 */
579 extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true);
580 extent_node_achunk_set(&chunk->node, true);
581 return (chunk_register(chunk, &chunk->node));
582}
583
584static arena_chunk_t *
585arena_chunk_alloc_internal_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
586 bool *zero, bool *commit)
587{
588 arena_chunk_t *chunk;
589
590 malloc_mutex_unlock(&arena->lock);
591
592 chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_hooks, NULL,
593 chunksize, chunksize, zero, commit);
594 if (chunk != NULL && !*commit) {
595 /* Commit header. */
596 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias <<
597 LG_PAGE, arena->ind)) {
598 chunk_dalloc_wrapper(arena, chunk_hooks,
599 (void *)chunk, chunksize, *commit);
600 chunk = NULL;
601 }
602 }
603 if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) {
604 if (!*commit) {
605 /* Undo commit of header. */
606 chunk_hooks->decommit(chunk, chunksize, 0, map_bias <<
607 LG_PAGE, arena->ind);
608 }
609 chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk,
610 chunksize, *commit);
611 chunk = NULL;
612 }
613
614 malloc_mutex_lock(&arena->lock);
615 return (chunk);
616}
617
618static arena_chunk_t *
619arena_chunk_alloc_internal(arena_t *arena, bool *zero, bool *commit)
620{
621 arena_chunk_t *chunk;
622 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
623
624 chunk = chunk_alloc_cache(arena, &chunk_hooks, NULL, chunksize,
625 chunksize, zero, true);
626 if (chunk != NULL) {
627 if (arena_chunk_register(arena, chunk, *zero)) {
628 chunk_dalloc_cache(arena, &chunk_hooks, chunk,
629 chunksize, true);
630 return (NULL);
631 }
632 *commit = true;
633 }
634 if (chunk == NULL) {
635 chunk = arena_chunk_alloc_internal_hard(arena, &chunk_hooks,
636 zero, commit);
637 }
638
639 if (config_stats && chunk != NULL) {
640 arena->stats.mapped += chunksize;
641 arena->stats.metadata_mapped += (map_bias << LG_PAGE);
642 }
643
644 return (chunk);
645}
646
647static arena_chunk_t *
648arena_chunk_init_hard(arena_t *arena)
649{
650 arena_chunk_t *chunk;
651 bool zero, commit;
652 size_t flag_unzeroed, flag_decommitted, i;
653
654 assert(arena->spare == NULL);
655
656 zero = false;
657 commit = false;
658 chunk = arena_chunk_alloc_internal(arena, &zero, &commit);
659 if (chunk == NULL)
660 return (NULL);
661
662 /*
663 * Initialize the map to contain one maximal free untouched run. Mark
664 * the pages as zeroed if chunk_alloc() returned a zeroed or decommitted
665 * chunk.
666 */
667 flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED;
668 flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED;
669 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun,
670 flag_unzeroed | flag_decommitted);
671 /*
672 * There is no need to initialize the internal page map entries unless
673 * the chunk is not zeroed.
674 */
675 if (!zero) {
676 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(
677 (void *)arena_bitselm_get(chunk, map_bias+1),
678 (size_t)((uintptr_t) arena_bitselm_get(chunk,
679 chunk_npages-1) - (uintptr_t)arena_bitselm_get(chunk,
680 map_bias+1)));
681 for (i = map_bias+1; i < chunk_npages-1; i++)
682 arena_mapbits_internal_set(chunk, i, flag_unzeroed);
683 } else {
684 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void
685 *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t)
686 arena_bitselm_get(chunk, chunk_npages-1) -
687 (uintptr_t)arena_bitselm_get(chunk, map_bias+1)));
688 if (config_debug) {
689 for (i = map_bias+1; i < chunk_npages-1; i++) {
690 assert(arena_mapbits_unzeroed_get(chunk, i) ==
691 flag_unzeroed);
692 }
693 }
694 }
695 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun,
696 flag_unzeroed);
697
698 return (chunk);
699}
700
701static arena_chunk_t *
702arena_chunk_alloc(arena_t *arena)
703{
704 arena_chunk_t *chunk;
705
706 if (arena->spare != NULL)
707 chunk = arena_chunk_init_spare(arena);
708 else {
709 chunk = arena_chunk_init_hard(arena);
710 if (chunk == NULL)
711 return (NULL);
712 }
713
714 /* Insert the run into the runs_avail tree. */
715 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias);
716
717 return (chunk);
718}
719
720static void
721arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
722{
723
724 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
725 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
726 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
727 arena_maxrun);
728 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
729 arena_maxrun);
730 assert(arena_mapbits_dirty_get(chunk, map_bias) ==
731 arena_mapbits_dirty_get(chunk, chunk_npages-1));
732 assert(arena_mapbits_decommitted_get(chunk, map_bias) ==
733 arena_mapbits_decommitted_get(chunk, chunk_npages-1));
734
735 /*
736 * Remove run from the runs_avail tree, so that the arena does not use
737 * it.
738 */
739 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias);
740
741 if (arena->spare != NULL) {
742 arena_chunk_t *spare = arena->spare;
743 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
744 bool committed;
745
746 arena->spare = chunk;
747 if (arena_mapbits_dirty_get(spare, map_bias) != 0) {
748 arena_run_dirty_remove(arena, spare, map_bias,
749 chunk_npages-map_bias);
750 }
751
752 chunk_deregister(spare, &spare->node);
753
754 committed = (arena_mapbits_decommitted_get(spare, map_bias) ==
755 0);
756 if (!committed) {
757 /*
758 * Decommit the header. Mark the chunk as decommitted
759 * even if header decommit fails, since treating a
760 * partially committed chunk as committed has a high
761 * potential for causing later access of decommitted
762 * memory.
763 */
764 chunk_hooks = chunk_hooks_get(arena);
765 chunk_hooks.decommit(spare, chunksize, 0, map_bias <<
766 LG_PAGE, arena->ind);
767 }
768
769 chunk_dalloc_cache(arena, &chunk_hooks, (void *)spare,
770 chunksize, committed);
771
772 if (config_stats) {
773 arena->stats.mapped -= chunksize;
774 arena->stats.metadata_mapped -= (map_bias << LG_PAGE);
775 }
776 } else
777 arena->spare = chunk;
778}
779
780static void
781arena_huge_malloc_stats_update(arena_t *arena, size_t usize)
782{
| 512{
513 arena_chunk_t *chunk;
514 arena_chunk_map_misc_t *miscelm;
515 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i;
516
517 assert(binind != BININD_INVALID);
518
519 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
520 miscelm = arena_run_to_miscelm(run);
521 run_ind = arena_miscelm_to_pageind(miscelm);
522 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
523 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind);
524 need_pages = (size >> LG_PAGE);
525 assert(need_pages > 0);
526
527 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize,
528 run_ind << LG_PAGE, size, arena->ind))
529 return (true);
530
531 arena_run_split_remove(arena, chunk, run_ind, flag_dirty,
532 flag_decommitted, need_pages);
533
534 for (i = 0; i < need_pages; i++) {
535 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk,
536 run_ind+i);
537 arena_mapbits_small_set(chunk, run_ind+i, i, binind,
538 flag_unzeroed);
539 if (config_debug && flag_dirty == 0 && flag_unzeroed == 0)
540 arena_run_page_validate_zeroed(chunk, run_ind+i);
541 }
542 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
543 (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
544 return (false);
545}
546
547static arena_chunk_t *
548arena_chunk_init_spare(arena_t *arena)
549{
550 arena_chunk_t *chunk;
551
552 assert(arena->spare != NULL);
553
554 chunk = arena->spare;
555 arena->spare = NULL;
556
557 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
558 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
559 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
560 arena_maxrun);
561 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
562 arena_maxrun);
563 assert(arena_mapbits_dirty_get(chunk, map_bias) ==
564 arena_mapbits_dirty_get(chunk, chunk_npages-1));
565
566 return (chunk);
567}
568
569static bool
570arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero)
571{
572
573 /*
574 * The extent node notion of "committed" doesn't directly apply to
575 * arena chunks. Arbitrarily mark them as committed. The commit state
576 * of runs is tracked individually, and upon chunk deallocation the
577 * entire chunk is in a consistent commit state.
578 */
579 extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true);
580 extent_node_achunk_set(&chunk->node, true);
581 return (chunk_register(chunk, &chunk->node));
582}
583
584static arena_chunk_t *
585arena_chunk_alloc_internal_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
586 bool *zero, bool *commit)
587{
588 arena_chunk_t *chunk;
589
590 malloc_mutex_unlock(&arena->lock);
591
592 chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_hooks, NULL,
593 chunksize, chunksize, zero, commit);
594 if (chunk != NULL && !*commit) {
595 /* Commit header. */
596 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias <<
597 LG_PAGE, arena->ind)) {
598 chunk_dalloc_wrapper(arena, chunk_hooks,
599 (void *)chunk, chunksize, *commit);
600 chunk = NULL;
601 }
602 }
603 if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) {
604 if (!*commit) {
605 /* Undo commit of header. */
606 chunk_hooks->decommit(chunk, chunksize, 0, map_bias <<
607 LG_PAGE, arena->ind);
608 }
609 chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk,
610 chunksize, *commit);
611 chunk = NULL;
612 }
613
614 malloc_mutex_lock(&arena->lock);
615 return (chunk);
616}
617
618static arena_chunk_t *
619arena_chunk_alloc_internal(arena_t *arena, bool *zero, bool *commit)
620{
621 arena_chunk_t *chunk;
622 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
623
624 chunk = chunk_alloc_cache(arena, &chunk_hooks, NULL, chunksize,
625 chunksize, zero, true);
626 if (chunk != NULL) {
627 if (arena_chunk_register(arena, chunk, *zero)) {
628 chunk_dalloc_cache(arena, &chunk_hooks, chunk,
629 chunksize, true);
630 return (NULL);
631 }
632 *commit = true;
633 }
634 if (chunk == NULL) {
635 chunk = arena_chunk_alloc_internal_hard(arena, &chunk_hooks,
636 zero, commit);
637 }
638
639 if (config_stats && chunk != NULL) {
640 arena->stats.mapped += chunksize;
641 arena->stats.metadata_mapped += (map_bias << LG_PAGE);
642 }
643
644 return (chunk);
645}
646
647static arena_chunk_t *
648arena_chunk_init_hard(arena_t *arena)
649{
650 arena_chunk_t *chunk;
651 bool zero, commit;
652 size_t flag_unzeroed, flag_decommitted, i;
653
654 assert(arena->spare == NULL);
655
656 zero = false;
657 commit = false;
658 chunk = arena_chunk_alloc_internal(arena, &zero, &commit);
659 if (chunk == NULL)
660 return (NULL);
661
662 /*
663 * Initialize the map to contain one maximal free untouched run. Mark
664 * the pages as zeroed if chunk_alloc() returned a zeroed or decommitted
665 * chunk.
666 */
667 flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED;
668 flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED;
669 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun,
670 flag_unzeroed | flag_decommitted);
671 /*
672 * There is no need to initialize the internal page map entries unless
673 * the chunk is not zeroed.
674 */
675 if (!zero) {
676 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(
677 (void *)arena_bitselm_get(chunk, map_bias+1),
678 (size_t)((uintptr_t) arena_bitselm_get(chunk,
679 chunk_npages-1) - (uintptr_t)arena_bitselm_get(chunk,
680 map_bias+1)));
681 for (i = map_bias+1; i < chunk_npages-1; i++)
682 arena_mapbits_internal_set(chunk, i, flag_unzeroed);
683 } else {
684 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void
685 *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t)
686 arena_bitselm_get(chunk, chunk_npages-1) -
687 (uintptr_t)arena_bitselm_get(chunk, map_bias+1)));
688 if (config_debug) {
689 for (i = map_bias+1; i < chunk_npages-1; i++) {
690 assert(arena_mapbits_unzeroed_get(chunk, i) ==
691 flag_unzeroed);
692 }
693 }
694 }
695 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun,
696 flag_unzeroed);
697
698 return (chunk);
699}
700
701static arena_chunk_t *
702arena_chunk_alloc(arena_t *arena)
703{
704 arena_chunk_t *chunk;
705
706 if (arena->spare != NULL)
707 chunk = arena_chunk_init_spare(arena);
708 else {
709 chunk = arena_chunk_init_hard(arena);
710 if (chunk == NULL)
711 return (NULL);
712 }
713
714 /* Insert the run into the runs_avail tree. */
715 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias);
716
717 return (chunk);
718}
719
720static void
721arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
722{
723
724 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
725 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
726 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
727 arena_maxrun);
728 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
729 arena_maxrun);
730 assert(arena_mapbits_dirty_get(chunk, map_bias) ==
731 arena_mapbits_dirty_get(chunk, chunk_npages-1));
732 assert(arena_mapbits_decommitted_get(chunk, map_bias) ==
733 arena_mapbits_decommitted_get(chunk, chunk_npages-1));
734
735 /*
736 * Remove run from the runs_avail tree, so that the arena does not use
737 * it.
738 */
739 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias);
740
741 if (arena->spare != NULL) {
742 arena_chunk_t *spare = arena->spare;
743 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
744 bool committed;
745
746 arena->spare = chunk;
747 if (arena_mapbits_dirty_get(spare, map_bias) != 0) {
748 arena_run_dirty_remove(arena, spare, map_bias,
749 chunk_npages-map_bias);
750 }
751
752 chunk_deregister(spare, &spare->node);
753
754 committed = (arena_mapbits_decommitted_get(spare, map_bias) ==
755 0);
756 if (!committed) {
757 /*
758 * Decommit the header. Mark the chunk as decommitted
759 * even if header decommit fails, since treating a
760 * partially committed chunk as committed has a high
761 * potential for causing later access of decommitted
762 * memory.
763 */
764 chunk_hooks = chunk_hooks_get(arena);
765 chunk_hooks.decommit(spare, chunksize, 0, map_bias <<
766 LG_PAGE, arena->ind);
767 }
768
769 chunk_dalloc_cache(arena, &chunk_hooks, (void *)spare,
770 chunksize, committed);
771
772 if (config_stats) {
773 arena->stats.mapped -= chunksize;
774 arena->stats.metadata_mapped -= (map_bias << LG_PAGE);
775 }
776 } else
777 arena->spare = chunk;
778}
779
780static void
781arena_huge_malloc_stats_update(arena_t *arena, size_t usize)
782{
|
783 index_t index = size2index(usize) - nlclasses - NBINS;
| 783 szind_t index = size2index(usize) - nlclasses - NBINS;
|
784
785 cassert(config_stats);
786
787 arena->stats.nmalloc_huge++;
788 arena->stats.allocated_huge += usize;
789 arena->stats.hstats[index].nmalloc++;
790 arena->stats.hstats[index].curhchunks++;
791}
792
793static void
794arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize)
795{
| 784
785 cassert(config_stats);
786
787 arena->stats.nmalloc_huge++;
788 arena->stats.allocated_huge += usize;
789 arena->stats.hstats[index].nmalloc++;
790 arena->stats.hstats[index].curhchunks++;
791}
792
793static void
794arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize)
795{
|
796 index_t index = size2index(usize) - nlclasses - NBINS;
| 796 szind_t index = size2index(usize) - nlclasses - NBINS;
|
797
798 cassert(config_stats);
799
800 arena->stats.nmalloc_huge--;
801 arena->stats.allocated_huge -= usize;
802 arena->stats.hstats[index].nmalloc--;
803 arena->stats.hstats[index].curhchunks--;
804}
805
806static void
807arena_huge_dalloc_stats_update(arena_t *arena, size_t usize)
808{
| 797
798 cassert(config_stats);
799
800 arena->stats.nmalloc_huge--;
801 arena->stats.allocated_huge -= usize;
802 arena->stats.hstats[index].nmalloc--;
803 arena->stats.hstats[index].curhchunks--;
804}
805
806static void
807arena_huge_dalloc_stats_update(arena_t *arena, size_t usize)
808{
|
809 index_t index = size2index(usize) - nlclasses - NBINS;
| 809 szind_t index = size2index(usize) - nlclasses - NBINS;
|
810
811 cassert(config_stats);
812
813 arena->stats.ndalloc_huge++;
814 arena->stats.allocated_huge -= usize;
815 arena->stats.hstats[index].ndalloc++;
816 arena->stats.hstats[index].curhchunks--;
817}
818
819static void
820arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize)
821{
| 810
811 cassert(config_stats);
812
813 arena->stats.ndalloc_huge++;
814 arena->stats.allocated_huge -= usize;
815 arena->stats.hstats[index].ndalloc++;
816 arena->stats.hstats[index].curhchunks--;
817}
818
819static void
820arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize)
821{
|
822 index_t index = size2index(usize) - nlclasses - NBINS;
| 822 szind_t index = size2index(usize) - nlclasses - NBINS;
|
823
824 cassert(config_stats);
825
826 arena->stats.ndalloc_huge--;
827 arena->stats.allocated_huge += usize;
828 arena->stats.hstats[index].ndalloc--;
829 arena->stats.hstats[index].curhchunks++;
830}
831
832static void
833arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize)
834{
835
836 arena_huge_dalloc_stats_update(arena, oldsize);
837 arena_huge_malloc_stats_update(arena, usize);
838}
839
840static void
841arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize,
842 size_t usize)
843{
844
845 arena_huge_dalloc_stats_update_undo(arena, oldsize);
846 arena_huge_malloc_stats_update_undo(arena, usize);
847}
848
849extent_node_t *
850arena_node_alloc(arena_t *arena)
851{
852 extent_node_t *node;
853
854 malloc_mutex_lock(&arena->node_cache_mtx);
855 node = ql_last(&arena->node_cache, ql_link);
856 if (node == NULL) {
857 malloc_mutex_unlock(&arena->node_cache_mtx);
858 return (base_alloc(sizeof(extent_node_t)));
859 }
860 ql_tail_remove(&arena->node_cache, extent_node_t, ql_link);
861 malloc_mutex_unlock(&arena->node_cache_mtx);
862 return (node);
863}
864
865void
866arena_node_dalloc(arena_t *arena, extent_node_t *node)
867{
868
869 malloc_mutex_lock(&arena->node_cache_mtx);
870 ql_elm_new(node, ql_link);
871 ql_tail_insert(&arena->node_cache, node, ql_link);
872 malloc_mutex_unlock(&arena->node_cache_mtx);
873}
874
875static void *
876arena_chunk_alloc_huge_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
877 size_t usize, size_t alignment, bool *zero, size_t csize)
878{
879 void *ret;
880 bool commit = true;
881
882 ret = chunk_alloc_wrapper(arena, chunk_hooks, NULL, csize, alignment,
883 zero, &commit);
884 if (ret == NULL) {
885 /* Revert optimistic stats updates. */
886 malloc_mutex_lock(&arena->lock);
887 if (config_stats) {
888 arena_huge_malloc_stats_update_undo(arena, usize);
889 arena->stats.mapped -= usize;
890 }
891 arena->nactive -= (usize >> LG_PAGE);
892 malloc_mutex_unlock(&arena->lock);
893 }
894
895 return (ret);
896}
897
898void *
899arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
900 bool *zero)
901{
902 void *ret;
903 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
904 size_t csize = CHUNK_CEILING(usize);
905
906 malloc_mutex_lock(&arena->lock);
907
908 /* Optimistically update stats. */
909 if (config_stats) {
910 arena_huge_malloc_stats_update(arena, usize);
911 arena->stats.mapped += usize;
912 }
913 arena->nactive += (usize >> LG_PAGE);
914
915 ret = chunk_alloc_cache(arena, &chunk_hooks, NULL, csize, alignment,
916 zero, true);
917 malloc_mutex_unlock(&arena->lock);
918 if (ret == NULL) {
919 ret = arena_chunk_alloc_huge_hard(arena, &chunk_hooks, usize,
920 alignment, zero, csize);
921 }
922
923 if (config_stats && ret != NULL)
924 stats_cactive_add(usize);
925 return (ret);
926}
927
928void
929arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize)
930{
931 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
932 size_t csize;
933
934 csize = CHUNK_CEILING(usize);
935 malloc_mutex_lock(&arena->lock);
936 if (config_stats) {
937 arena_huge_dalloc_stats_update(arena, usize);
938 arena->stats.mapped -= usize;
939 stats_cactive_sub(usize);
940 }
941 arena->nactive -= (usize >> LG_PAGE);
942
943 chunk_dalloc_cache(arena, &chunk_hooks, chunk, csize, true);
944 malloc_mutex_unlock(&arena->lock);
945}
946
947void
948arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize,
949 size_t usize)
950{
951
952 assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize));
953 assert(oldsize != usize);
954
955 malloc_mutex_lock(&arena->lock);
956 if (config_stats)
957 arena_huge_ralloc_stats_update(arena, oldsize, usize);
958 if (oldsize < usize) {
959 size_t udiff = usize - oldsize;
960 arena->nactive += udiff >> LG_PAGE;
961 if (config_stats)
962 stats_cactive_add(udiff);
963 } else {
964 size_t udiff = oldsize - usize;
965 arena->nactive -= udiff >> LG_PAGE;
966 if (config_stats)
967 stats_cactive_sub(udiff);
968 }
969 malloc_mutex_unlock(&arena->lock);
970}
971
972void
973arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize,
974 size_t usize)
975{
976 size_t udiff = oldsize - usize;
977 size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
978
979 malloc_mutex_lock(&arena->lock);
980 if (config_stats) {
981 arena_huge_ralloc_stats_update(arena, oldsize, usize);
982 if (cdiff != 0) {
983 arena->stats.mapped -= cdiff;
984 stats_cactive_sub(udiff);
985 }
986 }
987 arena->nactive -= udiff >> LG_PAGE;
988
989 if (cdiff != 0) {
990 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
991 void *nchunk = (void *)((uintptr_t)chunk +
992 CHUNK_CEILING(usize));
993
994 chunk_dalloc_cache(arena, &chunk_hooks, nchunk, cdiff, true);
995 }
996 malloc_mutex_unlock(&arena->lock);
997}
998
999static bool
1000arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
1001 void *chunk, size_t oldsize, size_t usize, bool *zero, void *nchunk,
1002 size_t udiff, size_t cdiff)
1003{
1004 bool err;
1005 bool commit = true;
1006
1007 err = (chunk_alloc_wrapper(arena, chunk_hooks, nchunk, cdiff, chunksize,
1008 zero, &commit) == NULL);
1009 if (err) {
1010 /* Revert optimistic stats updates. */
1011 malloc_mutex_lock(&arena->lock);
1012 if (config_stats) {
1013 arena_huge_ralloc_stats_update_undo(arena, oldsize,
1014 usize);
1015 arena->stats.mapped -= cdiff;
1016 }
1017 arena->nactive -= (udiff >> LG_PAGE);
1018 malloc_mutex_unlock(&arena->lock);
1019 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk,
1020 cdiff, true, arena->ind)) {
1021 chunk_dalloc_arena(arena, chunk_hooks, nchunk, cdiff, *zero,
1022 true);
1023 err = true;
1024 }
1025 return (err);
1026}
1027
1028bool
1029arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize,
1030 size_t usize, bool *zero)
1031{
1032 bool err;
1033 chunk_hooks_t chunk_hooks = chunk_hooks_get(arena);
1034 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize));
1035 size_t udiff = usize - oldsize;
1036 size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
1037
1038 malloc_mutex_lock(&arena->lock);
1039
1040 /* Optimistically update stats. */
1041 if (config_stats) {
1042 arena_huge_ralloc_stats_update(arena, oldsize, usize);
1043 arena->stats.mapped += cdiff;
1044 }
1045 arena->nactive += (udiff >> LG_PAGE);
1046
1047 err = (chunk_alloc_cache(arena, &arena->chunk_hooks, nchunk, cdiff,
1048 chunksize, zero, true) == NULL);
1049 malloc_mutex_unlock(&arena->lock);
1050 if (err) {
1051 err = arena_chunk_ralloc_huge_expand_hard(arena, &chunk_hooks,
1052 chunk, oldsize, usize, zero, nchunk, udiff,
1053 cdiff);
1054 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk,
1055 cdiff, true, arena->ind)) {
1056 chunk_dalloc_arena(arena, &chunk_hooks, nchunk, cdiff, *zero,
1057 true);
1058 err = true;
1059 }
1060
1061 if (config_stats && !err)
1062 stats_cactive_add(udiff);
1063 return (err);
1064}
1065
1066/*
1067 * Do first-best-fit run selection, i.e. select the lowest run that best fits.
1068 * Run sizes are quantized, so not all candidate runs are necessarily exactly
1069 * the same size.
1070 */
1071static arena_run_t *
1072arena_run_first_best_fit(arena_t *arena, size_t size)
1073{
1074 size_t search_size = run_quantize_first(size);
1075 arena_chunk_map_misc_t *key = arena_miscelm_key_create(search_size);
1076 arena_chunk_map_misc_t *miscelm =
1077 arena_avail_tree_nsearch(&arena->runs_avail, key);
1078 if (miscelm == NULL)
1079 return (NULL);
1080 return (&miscelm->run);
1081}
1082
1083static arena_run_t *
1084arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
1085{
1086 arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));
1087 if (run != NULL) {
1088 if (arena_run_split_large(arena, run, size, zero))
1089 run = NULL;
1090 }
1091 return (run);
1092}
1093
1094static arena_run_t *
1095arena_run_alloc_large(arena_t *arena, size_t size, bool zero)
1096{
1097 arena_chunk_t *chunk;
1098 arena_run_t *run;
1099
1100 assert(size <= arena_maxrun);
1101 assert(size == PAGE_CEILING(size));
1102
1103 /* Search the arena's chunks for the lowest best fit. */
1104 run = arena_run_alloc_large_helper(arena, size, zero);
1105 if (run != NULL)
1106 return (run);
1107
1108 /*
1109 * No usable runs. Create a new chunk from which to allocate the run.
1110 */
1111 chunk = arena_chunk_alloc(arena);
1112 if (chunk != NULL) {
1113 run = &arena_miscelm_get(chunk, map_bias)->run;
1114 if (arena_run_split_large(arena, run, size, zero))
1115 run = NULL;
1116 return (run);
1117 }
1118
1119 /*
1120 * arena_chunk_alloc() failed, but another thread may have made
1121 * sufficient memory available while this one dropped arena->lock in
1122 * arena_chunk_alloc(), so search one more time.
1123 */
1124 return (arena_run_alloc_large_helper(arena, size, zero));
1125}
1126
1127static arena_run_t *
| 823
824 cassert(config_stats);
825
826 arena->stats.ndalloc_huge--;
827 arena->stats.allocated_huge += usize;
828 arena->stats.hstats[index].ndalloc--;
829 arena->stats.hstats[index].curhchunks++;
830}
831
832static void
833arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize)
834{
835
836 arena_huge_dalloc_stats_update(arena, oldsize);
837 arena_huge_malloc_stats_update(arena, usize);
838}
839
840static void
841arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize,
842 size_t usize)
843{
844
845 arena_huge_dalloc_stats_update_undo(arena, oldsize);
846 arena_huge_malloc_stats_update_undo(arena, usize);
847}
848
849extent_node_t *
850arena_node_alloc(arena_t *arena)
851{
852 extent_node_t *node;
853
854 malloc_mutex_lock(&arena->node_cache_mtx);
855 node = ql_last(&arena->node_cache, ql_link);
856 if (node == NULL) {
857 malloc_mutex_unlock(&arena->node_cache_mtx);
858 return (base_alloc(sizeof(extent_node_t)));
859 }
860 ql_tail_remove(&arena->node_cache, extent_node_t, ql_link);
861 malloc_mutex_unlock(&arena->node_cache_mtx);
862 return (node);
863}
864
865void
866arena_node_dalloc(arena_t *arena, extent_node_t *node)
867{
868
869 malloc_mutex_lock(&arena->node_cache_mtx);
870 ql_elm_new(node, ql_link);
871 ql_tail_insert(&arena->node_cache, node, ql_link);
872 malloc_mutex_unlock(&arena->node_cache_mtx);
873}
874
875static void *
876arena_chunk_alloc_huge_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
877 size_t usize, size_t alignment, bool *zero, size_t csize)
878{
879 void *ret;
880 bool commit = true;
881
882 ret = chunk_alloc_wrapper(arena, chunk_hooks, NULL, csize, alignment,
883 zero, &commit);
884 if (ret == NULL) {
885 /* Revert optimistic stats updates. */
886 malloc_mutex_lock(&arena->lock);
887 if (config_stats) {
888 arena_huge_malloc_stats_update_undo(arena, usize);
889 arena->stats.mapped -= usize;
890 }
891 arena->nactive -= (usize >> LG_PAGE);
892 malloc_mutex_unlock(&arena->lock);
893 }
894
895 return (ret);
896}
897
898void *
899arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
900 bool *zero)
901{
902 void *ret;
903 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
904 size_t csize = CHUNK_CEILING(usize);
905
906 malloc_mutex_lock(&arena->lock);
907
908 /* Optimistically update stats. */
909 if (config_stats) {
910 arena_huge_malloc_stats_update(arena, usize);
911 arena->stats.mapped += usize;
912 }
913 arena->nactive += (usize >> LG_PAGE);
914
915 ret = chunk_alloc_cache(arena, &chunk_hooks, NULL, csize, alignment,
916 zero, true);
917 malloc_mutex_unlock(&arena->lock);
918 if (ret == NULL) {
919 ret = arena_chunk_alloc_huge_hard(arena, &chunk_hooks, usize,
920 alignment, zero, csize);
921 }
922
923 if (config_stats && ret != NULL)
924 stats_cactive_add(usize);
925 return (ret);
926}
927
928void
929arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize)
930{
931 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
932 size_t csize;
933
934 csize = CHUNK_CEILING(usize);
935 malloc_mutex_lock(&arena->lock);
936 if (config_stats) {
937 arena_huge_dalloc_stats_update(arena, usize);
938 arena->stats.mapped -= usize;
939 stats_cactive_sub(usize);
940 }
941 arena->nactive -= (usize >> LG_PAGE);
942
943 chunk_dalloc_cache(arena, &chunk_hooks, chunk, csize, true);
944 malloc_mutex_unlock(&arena->lock);
945}
946
947void
948arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize,
949 size_t usize)
950{
951
952 assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize));
953 assert(oldsize != usize);
954
955 malloc_mutex_lock(&arena->lock);
956 if (config_stats)
957 arena_huge_ralloc_stats_update(arena, oldsize, usize);
958 if (oldsize < usize) {
959 size_t udiff = usize - oldsize;
960 arena->nactive += udiff >> LG_PAGE;
961 if (config_stats)
962 stats_cactive_add(udiff);
963 } else {
964 size_t udiff = oldsize - usize;
965 arena->nactive -= udiff >> LG_PAGE;
966 if (config_stats)
967 stats_cactive_sub(udiff);
968 }
969 malloc_mutex_unlock(&arena->lock);
970}
971
972void
973arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize,
974 size_t usize)
975{
976 size_t udiff = oldsize - usize;
977 size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
978
979 malloc_mutex_lock(&arena->lock);
980 if (config_stats) {
981 arena_huge_ralloc_stats_update(arena, oldsize, usize);
982 if (cdiff != 0) {
983 arena->stats.mapped -= cdiff;
984 stats_cactive_sub(udiff);
985 }
986 }
987 arena->nactive -= udiff >> LG_PAGE;
988
989 if (cdiff != 0) {
990 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
991 void *nchunk = (void *)((uintptr_t)chunk +
992 CHUNK_CEILING(usize));
993
994 chunk_dalloc_cache(arena, &chunk_hooks, nchunk, cdiff, true);
995 }
996 malloc_mutex_unlock(&arena->lock);
997}
998
999static bool
1000arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
1001 void *chunk, size_t oldsize, size_t usize, bool *zero, void *nchunk,
1002 size_t udiff, size_t cdiff)
1003{
1004 bool err;
1005 bool commit = true;
1006
1007 err = (chunk_alloc_wrapper(arena, chunk_hooks, nchunk, cdiff, chunksize,
1008 zero, &commit) == NULL);
1009 if (err) {
1010 /* Revert optimistic stats updates. */
1011 malloc_mutex_lock(&arena->lock);
1012 if (config_stats) {
1013 arena_huge_ralloc_stats_update_undo(arena, oldsize,
1014 usize);
1015 arena->stats.mapped -= cdiff;
1016 }
1017 arena->nactive -= (udiff >> LG_PAGE);
1018 malloc_mutex_unlock(&arena->lock);
1019 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk,
1020 cdiff, true, arena->ind)) {
1021 chunk_dalloc_arena(arena, chunk_hooks, nchunk, cdiff, *zero,
1022 true);
1023 err = true;
1024 }
1025 return (err);
1026}
1027
1028bool
1029arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize,
1030 size_t usize, bool *zero)
1031{
1032 bool err;
1033 chunk_hooks_t chunk_hooks = chunk_hooks_get(arena);
1034 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize));
1035 size_t udiff = usize - oldsize;
1036 size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
1037
1038 malloc_mutex_lock(&arena->lock);
1039
1040 /* Optimistically update stats. */
1041 if (config_stats) {
1042 arena_huge_ralloc_stats_update(arena, oldsize, usize);
1043 arena->stats.mapped += cdiff;
1044 }
1045 arena->nactive += (udiff >> LG_PAGE);
1046
1047 err = (chunk_alloc_cache(arena, &arena->chunk_hooks, nchunk, cdiff,
1048 chunksize, zero, true) == NULL);
1049 malloc_mutex_unlock(&arena->lock);
1050 if (err) {
1051 err = arena_chunk_ralloc_huge_expand_hard(arena, &chunk_hooks,
1052 chunk, oldsize, usize, zero, nchunk, udiff,
1053 cdiff);
1054 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk,
1055 cdiff, true, arena->ind)) {
1056 chunk_dalloc_arena(arena, &chunk_hooks, nchunk, cdiff, *zero,
1057 true);
1058 err = true;
1059 }
1060
1061 if (config_stats && !err)
1062 stats_cactive_add(udiff);
1063 return (err);
1064}
1065
1066/*
1067 * Do first-best-fit run selection, i.e. select the lowest run that best fits.
1068 * Run sizes are quantized, so not all candidate runs are necessarily exactly
1069 * the same size.
1070 */
1071static arena_run_t *
1072arena_run_first_best_fit(arena_t *arena, size_t size)
1073{
1074 size_t search_size = run_quantize_first(size);
1075 arena_chunk_map_misc_t *key = arena_miscelm_key_create(search_size);
1076 arena_chunk_map_misc_t *miscelm =
1077 arena_avail_tree_nsearch(&arena->runs_avail, key);
1078 if (miscelm == NULL)
1079 return (NULL);
1080 return (&miscelm->run);
1081}
1082
1083static arena_run_t *
1084arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
1085{
1086 arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));
1087 if (run != NULL) {
1088 if (arena_run_split_large(arena, run, size, zero))
1089 run = NULL;
1090 }
1091 return (run);
1092}
1093
1094static arena_run_t *
1095arena_run_alloc_large(arena_t *arena, size_t size, bool zero)
1096{
1097 arena_chunk_t *chunk;
1098 arena_run_t *run;
1099
1100 assert(size <= arena_maxrun);
1101 assert(size == PAGE_CEILING(size));
1102
1103 /* Search the arena's chunks for the lowest best fit. */
1104 run = arena_run_alloc_large_helper(arena, size, zero);
1105 if (run != NULL)
1106 return (run);
1107
1108 /*
1109 * No usable runs. Create a new chunk from which to allocate the run.
1110 */
1111 chunk = arena_chunk_alloc(arena);
1112 if (chunk != NULL) {
1113 run = &arena_miscelm_get(chunk, map_bias)->run;
1114 if (arena_run_split_large(arena, run, size, zero))
1115 run = NULL;
1116 return (run);
1117 }
1118
1119 /*
1120 * arena_chunk_alloc() failed, but another thread may have made
1121 * sufficient memory available while this one dropped arena->lock in
1122 * arena_chunk_alloc(), so search one more time.
1123 */
1124 return (arena_run_alloc_large_helper(arena, size, zero));
1125}
1126
1127static arena_run_t *
|
1128arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind)
| 1128arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind)
|
1129{
1130 arena_run_t *run = arena_run_first_best_fit(arena, size);
1131 if (run != NULL) {
1132 if (arena_run_split_small(arena, run, size, binind))
1133 run = NULL;
1134 }
1135 return (run);
1136}
1137
1138static arena_run_t *
| 1129{
1130 arena_run_t *run = arena_run_first_best_fit(arena, size);
1131 if (run != NULL) {
1132 if (arena_run_split_small(arena, run, size, binind))
1133 run = NULL;
1134 }
1135 return (run);
1136}
1137
1138static arena_run_t *
|
1139arena_run_alloc_small(arena_t *arena, size_t size, index_t binind)
| 1139arena_run_alloc_small(arena_t *arena, size_t size, szind_t binind)
|
1140{
1141 arena_chunk_t *chunk;
1142 arena_run_t *run;
1143
1144 assert(size <= arena_maxrun);
1145 assert(size == PAGE_CEILING(size));
1146 assert(binind != BININD_INVALID);
1147
1148 /* Search the arena's chunks for the lowest best fit. */
1149 run = arena_run_alloc_small_helper(arena, size, binind);
1150 if (run != NULL)
1151 return (run);
1152
1153 /*
1154 * No usable runs. Create a new chunk from which to allocate the run.
1155 */
1156 chunk = arena_chunk_alloc(arena);
1157 if (chunk != NULL) {
1158 run = &arena_miscelm_get(chunk, map_bias)->run;
1159 if (arena_run_split_small(arena, run, size, binind))
1160 run = NULL;
1161 return (run);
1162 }
1163
1164 /*
1165 * arena_chunk_alloc() failed, but another thread may have made
1166 * sufficient memory available while this one dropped arena->lock in
1167 * arena_chunk_alloc(), so search one more time.
1168 */
1169 return (arena_run_alloc_small_helper(arena, size, binind));
1170}
1171
1172static bool
1173arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult)
1174{
1175
1176 return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t)
1177 << 3));
1178}
1179
1180ssize_t
1181arena_lg_dirty_mult_get(arena_t *arena)
1182{
1183 ssize_t lg_dirty_mult;
1184
1185 malloc_mutex_lock(&arena->lock);
1186 lg_dirty_mult = arena->lg_dirty_mult;
1187 malloc_mutex_unlock(&arena->lock);
1188
1189 return (lg_dirty_mult);
1190}
1191
1192bool
1193arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult)
1194{
1195
1196 if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
1197 return (true);
1198
1199 malloc_mutex_lock(&arena->lock);
1200 arena->lg_dirty_mult = lg_dirty_mult;
1201 arena_maybe_purge(arena);
1202 malloc_mutex_unlock(&arena->lock);
1203
1204 return (false);
1205}
1206
1207void
1208arena_maybe_purge(arena_t *arena)
1209{
1210
1211 /* Don't purge if the option is disabled. */
1212 if (arena->lg_dirty_mult < 0)
1213 return;
1214 /* Don't recursively purge. */
1215 if (arena->purging)
1216 return;
1217 /*
1218 * Iterate, since preventing recursive purging could otherwise leave too
1219 * many dirty pages.
1220 */
1221 while (true) {
1222 size_t threshold = (arena->nactive >> arena->lg_dirty_mult);
1223 if (threshold < chunk_npages)
1224 threshold = chunk_npages;
1225 /*
1226 * Don't purge unless the number of purgeable pages exceeds the
1227 * threshold.
1228 */
1229 if (arena->ndirty <= threshold)
1230 return;
1231 arena_purge(arena, false);
1232 }
1233}
1234
1235static size_t
1236arena_dirty_count(arena_t *arena)
1237{
1238 size_t ndirty = 0;
1239 arena_runs_dirty_link_t *rdelm;
1240 extent_node_t *chunkselm;
1241
1242 for (rdelm = qr_next(&arena->runs_dirty, rd_link),
1243 chunkselm = qr_next(&arena->chunks_cache, cc_link);
1244 rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) {
1245 size_t npages;
1246
1247 if (rdelm == &chunkselm->rd) {
1248 npages = extent_node_size_get(chunkselm) >> LG_PAGE;
1249 chunkselm = qr_next(chunkselm, cc_link);
1250 } else {
1251 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
1252 rdelm);
1253 arena_chunk_map_misc_t *miscelm =
1254 arena_rd_to_miscelm(rdelm);
1255 size_t pageind = arena_miscelm_to_pageind(miscelm);
1256 assert(arena_mapbits_allocated_get(chunk, pageind) ==
1257 0);
1258 assert(arena_mapbits_large_get(chunk, pageind) == 0);
1259 assert(arena_mapbits_dirty_get(chunk, pageind) != 0);
1260 npages = arena_mapbits_unallocated_size_get(chunk,
1261 pageind) >> LG_PAGE;
1262 }
1263 ndirty += npages;
1264 }
1265
1266 return (ndirty);
1267}
1268
1269static size_t
1270arena_compute_npurge(arena_t *arena, bool all)
1271{
1272 size_t npurge;
1273
1274 /*
1275 * Compute the minimum number of pages that this thread should try to
1276 * purge.
1277 */
1278 if (!all) {
1279 size_t threshold = (arena->nactive >> arena->lg_dirty_mult);
1280 threshold = threshold < chunk_npages ? chunk_npages : threshold;
1281
1282 npurge = arena->ndirty - threshold;
1283 } else
1284 npurge = arena->ndirty;
1285
1286 return (npurge);
1287}
1288
1289static size_t
1290arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks, bool all,
1291 size_t npurge, arena_runs_dirty_link_t *purge_runs_sentinel,
1292 extent_node_t *purge_chunks_sentinel)
1293{
1294 arena_runs_dirty_link_t *rdelm, *rdelm_next;
1295 extent_node_t *chunkselm;
1296 size_t nstashed = 0;
1297
1298 /* Stash at least npurge pages. */
1299 for (rdelm = qr_next(&arena->runs_dirty, rd_link),
1300 chunkselm = qr_next(&arena->chunks_cache, cc_link);
1301 rdelm != &arena->runs_dirty; rdelm = rdelm_next) {
1302 size_t npages;
1303 rdelm_next = qr_next(rdelm, rd_link);
1304
1305 if (rdelm == &chunkselm->rd) {
1306 extent_node_t *chunkselm_next;
1307 bool zero;
1308 UNUSED void *chunk;
1309
1310 chunkselm_next = qr_next(chunkselm, cc_link);
1311 /*
1312 * Allocate. chunkselm remains valid due to the
1313 * dalloc_node=false argument to chunk_alloc_cache().
1314 */
1315 zero = false;
1316 chunk = chunk_alloc_cache(arena, chunk_hooks,
1317 extent_node_addr_get(chunkselm),
1318 extent_node_size_get(chunkselm), chunksize, &zero,
1319 false);
1320 assert(chunk == extent_node_addr_get(chunkselm));
1321 assert(zero == extent_node_zeroed_get(chunkselm));
1322 extent_node_dirty_insert(chunkselm, purge_runs_sentinel,
1323 purge_chunks_sentinel);
1324 npages = extent_node_size_get(chunkselm) >> LG_PAGE;
1325 chunkselm = chunkselm_next;
1326 } else {
1327 arena_chunk_t *chunk =
1328 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
1329 arena_chunk_map_misc_t *miscelm =
1330 arena_rd_to_miscelm(rdelm);
1331 size_t pageind = arena_miscelm_to_pageind(miscelm);
1332 arena_run_t *run = &miscelm->run;
1333 size_t run_size =
1334 arena_mapbits_unallocated_size_get(chunk, pageind);
1335
1336 npages = run_size >> LG_PAGE;
1337
1338 assert(pageind + npages <= chunk_npages);
1339 assert(arena_mapbits_dirty_get(chunk, pageind) ==
1340 arena_mapbits_dirty_get(chunk, pageind+npages-1));
1341
1342 /*
1343 * If purging the spare chunk's run, make it available
1344 * prior to allocation.
1345 */
1346 if (chunk == arena->spare)
1347 arena_chunk_alloc(arena);
1348
1349 /* Temporarily allocate the free dirty run. */
1350 arena_run_split_large(arena, run, run_size, false);
1351 /* Stash. */
1352 if (false)
1353 qr_new(rdelm, rd_link); /* Redundant. */
1354 else {
1355 assert(qr_next(rdelm, rd_link) == rdelm);
1356 assert(qr_prev(rdelm, rd_link) == rdelm);
1357 }
1358 qr_meld(purge_runs_sentinel, rdelm, rd_link);
1359 }
1360
1361 nstashed += npages;
1362 if (!all && nstashed >= npurge)
1363 break;
1364 }
1365
1366 return (nstashed);
1367}
1368
1369static size_t
1370arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
1371 arena_runs_dirty_link_t *purge_runs_sentinel,
1372 extent_node_t *purge_chunks_sentinel)
1373{
1374 size_t npurged, nmadvise;
1375 arena_runs_dirty_link_t *rdelm;
1376 extent_node_t *chunkselm;
1377
1378 if (config_stats)
1379 nmadvise = 0;
1380 npurged = 0;
1381
1382 malloc_mutex_unlock(&arena->lock);
1383 for (rdelm = qr_next(purge_runs_sentinel, rd_link),
1384 chunkselm = qr_next(purge_chunks_sentinel, cc_link);
1385 rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) {
1386 size_t npages;
1387
1388 if (rdelm == &chunkselm->rd) {
1389 /*
1390 * Don't actually purge the chunk here because 1)
1391 * chunkselm is embedded in the chunk and must remain
1392 * valid, and 2) we deallocate the chunk in
1393 * arena_unstash_purged(), where it is destroyed,
1394 * decommitted, or purged, depending on chunk
1395 * deallocation policy.
1396 */
1397 size_t size = extent_node_size_get(chunkselm);
1398 npages = size >> LG_PAGE;
1399 chunkselm = qr_next(chunkselm, cc_link);
1400 } else {
1401 size_t pageind, run_size, flag_unzeroed, flags, i;
1402 bool decommitted;
1403 arena_chunk_t *chunk =
1404 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
1405 arena_chunk_map_misc_t *miscelm =
1406 arena_rd_to_miscelm(rdelm);
1407 pageind = arena_miscelm_to_pageind(miscelm);
1408 run_size = arena_mapbits_large_size_get(chunk, pageind);
1409 npages = run_size >> LG_PAGE;
1410
1411 assert(pageind + npages <= chunk_npages);
1412 assert(!arena_mapbits_decommitted_get(chunk, pageind));
1413 assert(!arena_mapbits_decommitted_get(chunk,
1414 pageind+npages-1));
1415 decommitted = !chunk_hooks->decommit(chunk, chunksize,
1416 pageind << LG_PAGE, npages << LG_PAGE, arena->ind);
1417 if (decommitted) {
1418 flag_unzeroed = 0;
1419 flags = CHUNK_MAP_DECOMMITTED;
1420 } else {
1421 flag_unzeroed = chunk_purge_wrapper(arena,
1422 chunk_hooks, chunk, chunksize, pageind <<
1423 LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0;
1424 flags = flag_unzeroed;
1425 }
1426 arena_mapbits_large_set(chunk, pageind+npages-1, 0,
1427 flags);
1428 arena_mapbits_large_set(chunk, pageind, run_size,
1429 flags);
1430
1431 /*
1432 * Set the unzeroed flag for internal pages, now that
1433 * chunk_purge_wrapper() has returned whether the pages
1434 * were zeroed as a side effect of purging. This chunk
1435 * map modification is safe even though the arena mutex
1436 * isn't currently owned by this thread, because the run
1437 * is marked as allocated, thus protecting it from being
1438 * modified by any other thread. As long as these
1439 * writes don't perturb the first and last elements'
1440 * CHUNK_MAP_ALLOCATED bits, behavior is well defined.
1441 */
1442 for (i = 1; i < npages-1; i++) {
1443 arena_mapbits_internal_set(chunk, pageind+i,
1444 flag_unzeroed);
1445 }
1446 }
1447
1448 npurged += npages;
1449 if (config_stats)
1450 nmadvise++;
1451 }
1452 malloc_mutex_lock(&arena->lock);
1453
1454 if (config_stats) {
1455 arena->stats.nmadvise += nmadvise;
1456 arena->stats.purged += npurged;
1457 }
1458
1459 return (npurged);
1460}
1461
1462static void
1463arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
1464 arena_runs_dirty_link_t *purge_runs_sentinel,
1465 extent_node_t *purge_chunks_sentinel)
1466{
1467 arena_runs_dirty_link_t *rdelm, *rdelm_next;
1468 extent_node_t *chunkselm;
1469
1470 /* Deallocate chunks/runs. */
1471 for (rdelm = qr_next(purge_runs_sentinel, rd_link),
1472 chunkselm = qr_next(purge_chunks_sentinel, cc_link);
1473 rdelm != purge_runs_sentinel; rdelm = rdelm_next) {
1474 rdelm_next = qr_next(rdelm, rd_link);
1475 if (rdelm == &chunkselm->rd) {
1476 extent_node_t *chunkselm_next = qr_next(chunkselm,
1477 cc_link);
1478 void *addr = extent_node_addr_get(chunkselm);
1479 size_t size = extent_node_size_get(chunkselm);
1480 bool zeroed = extent_node_zeroed_get(chunkselm);
1481 bool committed = extent_node_committed_get(chunkselm);
1482 extent_node_dirty_remove(chunkselm);
1483 arena_node_dalloc(arena, chunkselm);
1484 chunkselm = chunkselm_next;
1485 chunk_dalloc_arena(arena, chunk_hooks, addr, size,
1486 zeroed, committed);
1487 } else {
1488 arena_chunk_t *chunk =
1489 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
1490 arena_chunk_map_misc_t *miscelm =
1491 arena_rd_to_miscelm(rdelm);
1492 size_t pageind = arena_miscelm_to_pageind(miscelm);
1493 bool decommitted = (arena_mapbits_decommitted_get(chunk,
1494 pageind) != 0);
1495 arena_run_t *run = &miscelm->run;
1496 qr_remove(rdelm, rd_link);
1497 arena_run_dalloc(arena, run, false, true, decommitted);
1498 }
1499 }
1500}
1501
1502static void
1503arena_purge(arena_t *arena, bool all)
1504{
1505 chunk_hooks_t chunk_hooks = chunk_hooks_get(arena);
1506 size_t npurge, npurgeable, npurged;
1507 arena_runs_dirty_link_t purge_runs_sentinel;
1508 extent_node_t purge_chunks_sentinel;
1509
1510 arena->purging = true;
1511
1512 /*
1513 * Calls to arena_dirty_count() are disabled even for debug builds
1514 * because overhead grows nonlinearly as memory usage increases.
1515 */
1516 if (false && config_debug) {
1517 size_t ndirty = arena_dirty_count(arena);
1518 assert(ndirty == arena->ndirty);
1519 }
1520 assert((arena->nactive >> arena->lg_dirty_mult) < arena->ndirty || all);
1521
1522 if (config_stats)
1523 arena->stats.npurge++;
1524
1525 npurge = arena_compute_npurge(arena, all);
1526 qr_new(&purge_runs_sentinel, rd_link);
1527 extent_node_dirty_linkage_init(&purge_chunks_sentinel);
1528
1529 npurgeable = arena_stash_dirty(arena, &chunk_hooks, all, npurge,
1530 &purge_runs_sentinel, &purge_chunks_sentinel);
1531 assert(npurgeable >= npurge);
1532 npurged = arena_purge_stashed(arena, &chunk_hooks, &purge_runs_sentinel,
1533 &purge_chunks_sentinel);
1534 assert(npurged == npurgeable);
1535 arena_unstash_purged(arena, &chunk_hooks, &purge_runs_sentinel,
1536 &purge_chunks_sentinel);
1537
1538 arena->purging = false;
1539}
1540
1541void
1542arena_purge_all(arena_t *arena)
1543{
1544
1545 malloc_mutex_lock(&arena->lock);
1546 arena_purge(arena, true);
1547 malloc_mutex_unlock(&arena->lock);
1548}
1549
1550static void
1551arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size,
1552 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty,
1553 size_t flag_decommitted)
1554{
1555 size_t size = *p_size;
1556 size_t run_ind = *p_run_ind;
1557 size_t run_pages = *p_run_pages;
1558
1559 /* Try to coalesce forward. */
1560 if (run_ind + run_pages < chunk_npages &&
1561 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 &&
1562 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty &&
1563 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) ==
1564 flag_decommitted) {
1565 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk,
1566 run_ind+run_pages);
1567 size_t nrun_pages = nrun_size >> LG_PAGE;
1568
1569 /*
1570 * Remove successor from runs_avail; the coalesced run is
1571 * inserted later.
1572 */
1573 assert(arena_mapbits_unallocated_size_get(chunk,
1574 run_ind+run_pages+nrun_pages-1) == nrun_size);
1575 assert(arena_mapbits_dirty_get(chunk,
1576 run_ind+run_pages+nrun_pages-1) == flag_dirty);
1577 assert(arena_mapbits_decommitted_get(chunk,
1578 run_ind+run_pages+nrun_pages-1) == flag_decommitted);
1579 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages);
1580
1581 /*
1582 * If the successor is dirty, remove it from the set of dirty
1583 * pages.
1584 */
1585 if (flag_dirty != 0) {
1586 arena_run_dirty_remove(arena, chunk, run_ind+run_pages,
1587 nrun_pages);
1588 }
1589
1590 size += nrun_size;
1591 run_pages += nrun_pages;
1592
1593 arena_mapbits_unallocated_size_set(chunk, run_ind, size);
1594 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
1595 size);
1596 }
1597
1598 /* Try to coalesce backward. */
1599 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk,
1600 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) ==
1601 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) ==
1602 flag_decommitted) {
1603 size_t prun_size = arena_mapbits_unallocated_size_get(chunk,
1604 run_ind-1);
1605 size_t prun_pages = prun_size >> LG_PAGE;
1606
1607 run_ind -= prun_pages;
1608
1609 /*
1610 * Remove predecessor from runs_avail; the coalesced run is
1611 * inserted later.
1612 */
1613 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
1614 prun_size);
1615 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);
1616 assert(arena_mapbits_decommitted_get(chunk, run_ind) ==
1617 flag_decommitted);
1618 arena_avail_remove(arena, chunk, run_ind, prun_pages);
1619
1620 /*
1621 * If the predecessor is dirty, remove it from the set of dirty
1622 * pages.
1623 */
1624 if (flag_dirty != 0) {
1625 arena_run_dirty_remove(arena, chunk, run_ind,
1626 prun_pages);
1627 }
1628
1629 size += prun_size;
1630 run_pages += prun_pages;
1631
1632 arena_mapbits_unallocated_size_set(chunk, run_ind, size);
1633 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
1634 size);
1635 }
1636
1637 *p_size = size;
1638 *p_run_ind = run_ind;
1639 *p_run_pages = run_pages;
1640}
1641
1642static size_t
1643arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1644 size_t run_ind)
1645{
1646 size_t size;
1647
1648 assert(run_ind >= map_bias);
1649 assert(run_ind < chunk_npages);
1650
1651 if (arena_mapbits_large_get(chunk, run_ind) != 0) {
1652 size = arena_mapbits_large_size_get(chunk, run_ind);
1653 assert(size == PAGE || arena_mapbits_large_size_get(chunk,
1654 run_ind+(size>>LG_PAGE)-1) == 0);
1655 } else {
1656 arena_bin_info_t *bin_info = &arena_bin_info[run->binind];
1657 size = bin_info->run_size;
1658 }
1659
1660 return (size);
1661}
1662
1663static bool
1664arena_run_decommit(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run)
1665{
1666 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1667 size_t run_ind = arena_miscelm_to_pageind(miscelm);
1668 size_t offset = run_ind << LG_PAGE;
1669 size_t length = arena_run_size_get(arena, chunk, run, run_ind);
1670
1671 return (arena->chunk_hooks.decommit(chunk, chunksize, offset, length,
1672 arena->ind));
1673}
1674
1675static void
1676arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned,
1677 bool decommitted)
1678{
1679 arena_chunk_t *chunk;
1680 arena_chunk_map_misc_t *miscelm;
1681 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted;
1682
1683 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
1684 miscelm = arena_run_to_miscelm(run);
1685 run_ind = arena_miscelm_to_pageind(miscelm);
1686 assert(run_ind >= map_bias);
1687 assert(run_ind < chunk_npages);
1688 size = arena_run_size_get(arena, chunk, run, run_ind);
1689 run_pages = (size >> LG_PAGE);
1690 arena_cactive_update(arena, 0, run_pages);
1691 arena->nactive -= run_pages;
1692
1693 /*
1694 * The run is dirty if the caller claims to have dirtied it, as well as
1695 * if it was already dirty before being allocated and the caller
1696 * doesn't claim to have cleaned it.
1697 */
1698 assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1699 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
1700 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind)
1701 != 0)
1702 dirty = true;
1703 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0;
1704 flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0;
1705
1706 /* Mark pages as unallocated in the chunk map. */
1707 if (dirty || decommitted) {
1708 size_t flags = flag_dirty | flag_decommitted;
1709 arena_mapbits_unallocated_set(chunk, run_ind, size, flags);
1710 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
1711 flags);
1712 } else {
1713 arena_mapbits_unallocated_set(chunk, run_ind, size,
1714 arena_mapbits_unzeroed_get(chunk, run_ind));
1715 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
1716 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1));
1717 }
1718
1719 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages,
1720 flag_dirty, flag_decommitted);
1721
1722 /* Insert into runs_avail, now that coalescing is complete. */
1723 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
1724 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));
1725 assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1726 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
1727 assert(arena_mapbits_decommitted_get(chunk, run_ind) ==
1728 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1));
1729 arena_avail_insert(arena, chunk, run_ind, run_pages);
1730
1731 if (dirty)
1732 arena_run_dirty_insert(arena, chunk, run_ind, run_pages);
1733
1734 /* Deallocate chunk if it is now completely unused. */
1735 if (size == arena_maxrun) {
1736 assert(run_ind == map_bias);
1737 assert(run_pages == (arena_maxrun >> LG_PAGE));
1738 arena_chunk_dalloc(arena, chunk);
1739 }
1740
1741 /*
1742 * It is okay to do dirty page processing here even if the chunk was
1743 * deallocated above, since in that case it is the spare. Waiting
1744 * until after possible chunk deallocation to do dirty processing
1745 * allows for an old spare to be fully deallocated, thus decreasing the
1746 * chances of spuriously crossing the dirty page purging threshold.
1747 */
1748 if (dirty)
1749 arena_maybe_purge(arena);
1750}
1751
1752static void
1753arena_run_dalloc_decommit(arena_t *arena, arena_chunk_t *chunk,
1754 arena_run_t *run)
1755{
1756 bool committed = arena_run_decommit(arena, chunk, run);
1757
1758 arena_run_dalloc(arena, run, committed, false, !committed);
1759}
1760
1761static void
1762arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1763 size_t oldsize, size_t newsize)
1764{
1765 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1766 size_t pageind = arena_miscelm_to_pageind(miscelm);
1767 size_t head_npages = (oldsize - newsize) >> LG_PAGE;
1768 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
1769 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind);
1770 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ?
1771 CHUNK_MAP_UNZEROED : 0;
1772
1773 assert(oldsize > newsize);
1774
1775 /*
1776 * Update the chunk map so that arena_run_dalloc() can treat the
1777 * leading run as separately allocated. Set the last element of each
1778 * run first, in case of single-page runs.
1779 */
1780 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
1781 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty |
1782 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1783 pageind+head_npages-1)));
1784 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty |
1785 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind)));
1786
1787 if (config_debug) {
1788 UNUSED size_t tail_npages = newsize >> LG_PAGE;
1789 assert(arena_mapbits_large_size_get(chunk,
1790 pageind+head_npages+tail_npages-1) == 0);
1791 assert(arena_mapbits_dirty_get(chunk,
1792 pageind+head_npages+tail_npages-1) == flag_dirty);
1793 }
1794 arena_mapbits_large_set(chunk, pageind+head_npages, newsize,
1795 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1796 pageind+head_npages)));
1797
1798 arena_run_dalloc(arena, run, false, false, (flag_decommitted != 0));
1799}
1800
1801static void
1802arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1803 size_t oldsize, size_t newsize, bool dirty)
1804{
1805 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1806 size_t pageind = arena_miscelm_to_pageind(miscelm);
1807 size_t head_npages = newsize >> LG_PAGE;
1808 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
1809 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind);
1810 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ?
1811 CHUNK_MAP_UNZEROED : 0;
1812 arena_chunk_map_misc_t *tail_miscelm;
1813 arena_run_t *tail_run;
1814
1815 assert(oldsize > newsize);
1816
1817 /*
1818 * Update the chunk map so that arena_run_dalloc() can treat the
1819 * trailing run as separately allocated. Set the last element of each
1820 * run first, in case of single-page runs.
1821 */
1822 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
1823 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty |
1824 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1825 pageind+head_npages-1)));
1826 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty |
1827 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind)));
1828
1829 if (config_debug) {
1830 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE;
1831 assert(arena_mapbits_large_size_get(chunk,
1832 pageind+head_npages+tail_npages-1) == 0);
1833 assert(arena_mapbits_dirty_get(chunk,
1834 pageind+head_npages+tail_npages-1) == flag_dirty);
1835 }
1836 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize,
1837 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1838 pageind+head_npages)));
1839
1840 tail_miscelm = arena_miscelm_get(chunk, pageind + head_npages);
1841 tail_run = &tail_miscelm->run;
1842 arena_run_dalloc(arena, tail_run, dirty, false, (flag_decommitted !=
1843 0));
1844}
1845
1846static arena_run_t *
1847arena_bin_runs_first(arena_bin_t *bin)
1848{
1849 arena_chunk_map_misc_t *miscelm = arena_run_tree_first(&bin->runs);
1850 if (miscelm != NULL)
1851 return (&miscelm->run);
1852
1853 return (NULL);
1854}
1855
1856static void
1857arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run)
1858{
1859 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1860
1861 assert(arena_run_tree_search(&bin->runs, miscelm) == NULL);
1862
1863 arena_run_tree_insert(&bin->runs, miscelm);
1864}
1865
1866static void
1867arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run)
1868{
1869 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1870
1871 assert(arena_run_tree_search(&bin->runs, miscelm) != NULL);
1872
1873 arena_run_tree_remove(&bin->runs, miscelm);
1874}
1875
1876static arena_run_t *
1877arena_bin_nonfull_run_tryget(arena_bin_t *bin)
1878{
1879 arena_run_t *run = arena_bin_runs_first(bin);
1880 if (run != NULL) {
1881 arena_bin_runs_remove(bin, run);
1882 if (config_stats)
1883 bin->stats.reruns++;
1884 }
1885 return (run);
1886}
1887
1888static arena_run_t *
1889arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
1890{
1891 arena_run_t *run;
| 1140{
1141 arena_chunk_t *chunk;
1142 arena_run_t *run;
1143
1144 assert(size <= arena_maxrun);
1145 assert(size == PAGE_CEILING(size));
1146 assert(binind != BININD_INVALID);
1147
1148 /* Search the arena's chunks for the lowest best fit. */
1149 run = arena_run_alloc_small_helper(arena, size, binind);
1150 if (run != NULL)
1151 return (run);
1152
1153 /*
1154 * No usable runs. Create a new chunk from which to allocate the run.
1155 */
1156 chunk = arena_chunk_alloc(arena);
1157 if (chunk != NULL) {
1158 run = &arena_miscelm_get(chunk, map_bias)->run;
1159 if (arena_run_split_small(arena, run, size, binind))
1160 run = NULL;
1161 return (run);
1162 }
1163
1164 /*
1165 * arena_chunk_alloc() failed, but another thread may have made
1166 * sufficient memory available while this one dropped arena->lock in
1167 * arena_chunk_alloc(), so search one more time.
1168 */
1169 return (arena_run_alloc_small_helper(arena, size, binind));
1170}
1171
1172static bool
1173arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult)
1174{
1175
1176 return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t)
1177 << 3));
1178}
1179
1180ssize_t
1181arena_lg_dirty_mult_get(arena_t *arena)
1182{
1183 ssize_t lg_dirty_mult;
1184
1185 malloc_mutex_lock(&arena->lock);
1186 lg_dirty_mult = arena->lg_dirty_mult;
1187 malloc_mutex_unlock(&arena->lock);
1188
1189 return (lg_dirty_mult);
1190}
1191
1192bool
1193arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult)
1194{
1195
1196 if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
1197 return (true);
1198
1199 malloc_mutex_lock(&arena->lock);
1200 arena->lg_dirty_mult = lg_dirty_mult;
1201 arena_maybe_purge(arena);
1202 malloc_mutex_unlock(&arena->lock);
1203
1204 return (false);
1205}
1206
1207void
1208arena_maybe_purge(arena_t *arena)
1209{
1210
1211 /* Don't purge if the option is disabled. */
1212 if (arena->lg_dirty_mult < 0)
1213 return;
1214 /* Don't recursively purge. */
1215 if (arena->purging)
1216 return;
1217 /*
1218 * Iterate, since preventing recursive purging could otherwise leave too
1219 * many dirty pages.
1220 */
1221 while (true) {
1222 size_t threshold = (arena->nactive >> arena->lg_dirty_mult);
1223 if (threshold < chunk_npages)
1224 threshold = chunk_npages;
1225 /*
1226 * Don't purge unless the number of purgeable pages exceeds the
1227 * threshold.
1228 */
1229 if (arena->ndirty <= threshold)
1230 return;
1231 arena_purge(arena, false);
1232 }
1233}
1234
1235static size_t
1236arena_dirty_count(arena_t *arena)
1237{
1238 size_t ndirty = 0;
1239 arena_runs_dirty_link_t *rdelm;
1240 extent_node_t *chunkselm;
1241
1242 for (rdelm = qr_next(&arena->runs_dirty, rd_link),
1243 chunkselm = qr_next(&arena->chunks_cache, cc_link);
1244 rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) {
1245 size_t npages;
1246
1247 if (rdelm == &chunkselm->rd) {
1248 npages = extent_node_size_get(chunkselm) >> LG_PAGE;
1249 chunkselm = qr_next(chunkselm, cc_link);
1250 } else {
1251 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
1252 rdelm);
1253 arena_chunk_map_misc_t *miscelm =
1254 arena_rd_to_miscelm(rdelm);
1255 size_t pageind = arena_miscelm_to_pageind(miscelm);
1256 assert(arena_mapbits_allocated_get(chunk, pageind) ==
1257 0);
1258 assert(arena_mapbits_large_get(chunk, pageind) == 0);
1259 assert(arena_mapbits_dirty_get(chunk, pageind) != 0);
1260 npages = arena_mapbits_unallocated_size_get(chunk,
1261 pageind) >> LG_PAGE;
1262 }
1263 ndirty += npages;
1264 }
1265
1266 return (ndirty);
1267}
1268
1269static size_t
1270arena_compute_npurge(arena_t *arena, bool all)
1271{
1272 size_t npurge;
1273
1274 /*
1275 * Compute the minimum number of pages that this thread should try to
1276 * purge.
1277 */
1278 if (!all) {
1279 size_t threshold = (arena->nactive >> arena->lg_dirty_mult);
1280 threshold = threshold < chunk_npages ? chunk_npages : threshold;
1281
1282 npurge = arena->ndirty - threshold;
1283 } else
1284 npurge = arena->ndirty;
1285
1286 return (npurge);
1287}
1288
1289static size_t
1290arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks, bool all,
1291 size_t npurge, arena_runs_dirty_link_t *purge_runs_sentinel,
1292 extent_node_t *purge_chunks_sentinel)
1293{
1294 arena_runs_dirty_link_t *rdelm, *rdelm_next;
1295 extent_node_t *chunkselm;
1296 size_t nstashed = 0;
1297
1298 /* Stash at least npurge pages. */
1299 for (rdelm = qr_next(&arena->runs_dirty, rd_link),
1300 chunkselm = qr_next(&arena->chunks_cache, cc_link);
1301 rdelm != &arena->runs_dirty; rdelm = rdelm_next) {
1302 size_t npages;
1303 rdelm_next = qr_next(rdelm, rd_link);
1304
1305 if (rdelm == &chunkselm->rd) {
1306 extent_node_t *chunkselm_next;
1307 bool zero;
1308 UNUSED void *chunk;
1309
1310 chunkselm_next = qr_next(chunkselm, cc_link);
1311 /*
1312 * Allocate. chunkselm remains valid due to the
1313 * dalloc_node=false argument to chunk_alloc_cache().
1314 */
1315 zero = false;
1316 chunk = chunk_alloc_cache(arena, chunk_hooks,
1317 extent_node_addr_get(chunkselm),
1318 extent_node_size_get(chunkselm), chunksize, &zero,
1319 false);
1320 assert(chunk == extent_node_addr_get(chunkselm));
1321 assert(zero == extent_node_zeroed_get(chunkselm));
1322 extent_node_dirty_insert(chunkselm, purge_runs_sentinel,
1323 purge_chunks_sentinel);
1324 npages = extent_node_size_get(chunkselm) >> LG_PAGE;
1325 chunkselm = chunkselm_next;
1326 } else {
1327 arena_chunk_t *chunk =
1328 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
1329 arena_chunk_map_misc_t *miscelm =
1330 arena_rd_to_miscelm(rdelm);
1331 size_t pageind = arena_miscelm_to_pageind(miscelm);
1332 arena_run_t *run = &miscelm->run;
1333 size_t run_size =
1334 arena_mapbits_unallocated_size_get(chunk, pageind);
1335
1336 npages = run_size >> LG_PAGE;
1337
1338 assert(pageind + npages <= chunk_npages);
1339 assert(arena_mapbits_dirty_get(chunk, pageind) ==
1340 arena_mapbits_dirty_get(chunk, pageind+npages-1));
1341
1342 /*
1343 * If purging the spare chunk's run, make it available
1344 * prior to allocation.
1345 */
1346 if (chunk == arena->spare)
1347 arena_chunk_alloc(arena);
1348
1349 /* Temporarily allocate the free dirty run. */
1350 arena_run_split_large(arena, run, run_size, false);
1351 /* Stash. */
1352 if (false)
1353 qr_new(rdelm, rd_link); /* Redundant. */
1354 else {
1355 assert(qr_next(rdelm, rd_link) == rdelm);
1356 assert(qr_prev(rdelm, rd_link) == rdelm);
1357 }
1358 qr_meld(purge_runs_sentinel, rdelm, rd_link);
1359 }
1360
1361 nstashed += npages;
1362 if (!all && nstashed >= npurge)
1363 break;
1364 }
1365
1366 return (nstashed);
1367}
1368
1369static size_t
1370arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
1371 arena_runs_dirty_link_t *purge_runs_sentinel,
1372 extent_node_t *purge_chunks_sentinel)
1373{
1374 size_t npurged, nmadvise;
1375 arena_runs_dirty_link_t *rdelm;
1376 extent_node_t *chunkselm;
1377
1378 if (config_stats)
1379 nmadvise = 0;
1380 npurged = 0;
1381
1382 malloc_mutex_unlock(&arena->lock);
1383 for (rdelm = qr_next(purge_runs_sentinel, rd_link),
1384 chunkselm = qr_next(purge_chunks_sentinel, cc_link);
1385 rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) {
1386 size_t npages;
1387
1388 if (rdelm == &chunkselm->rd) {
1389 /*
1390 * Don't actually purge the chunk here because 1)
1391 * chunkselm is embedded in the chunk and must remain
1392 * valid, and 2) we deallocate the chunk in
1393 * arena_unstash_purged(), where it is destroyed,
1394 * decommitted, or purged, depending on chunk
1395 * deallocation policy.
1396 */
1397 size_t size = extent_node_size_get(chunkselm);
1398 npages = size >> LG_PAGE;
1399 chunkselm = qr_next(chunkselm, cc_link);
1400 } else {
1401 size_t pageind, run_size, flag_unzeroed, flags, i;
1402 bool decommitted;
1403 arena_chunk_t *chunk =
1404 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
1405 arena_chunk_map_misc_t *miscelm =
1406 arena_rd_to_miscelm(rdelm);
1407 pageind = arena_miscelm_to_pageind(miscelm);
1408 run_size = arena_mapbits_large_size_get(chunk, pageind);
1409 npages = run_size >> LG_PAGE;
1410
1411 assert(pageind + npages <= chunk_npages);
1412 assert(!arena_mapbits_decommitted_get(chunk, pageind));
1413 assert(!arena_mapbits_decommitted_get(chunk,
1414 pageind+npages-1));
1415 decommitted = !chunk_hooks->decommit(chunk, chunksize,
1416 pageind << LG_PAGE, npages << LG_PAGE, arena->ind);
1417 if (decommitted) {
1418 flag_unzeroed = 0;
1419 flags = CHUNK_MAP_DECOMMITTED;
1420 } else {
1421 flag_unzeroed = chunk_purge_wrapper(arena,
1422 chunk_hooks, chunk, chunksize, pageind <<
1423 LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0;
1424 flags = flag_unzeroed;
1425 }
1426 arena_mapbits_large_set(chunk, pageind+npages-1, 0,
1427 flags);
1428 arena_mapbits_large_set(chunk, pageind, run_size,
1429 flags);
1430
1431 /*
1432 * Set the unzeroed flag for internal pages, now that
1433 * chunk_purge_wrapper() has returned whether the pages
1434 * were zeroed as a side effect of purging. This chunk
1435 * map modification is safe even though the arena mutex
1436 * isn't currently owned by this thread, because the run
1437 * is marked as allocated, thus protecting it from being
1438 * modified by any other thread. As long as these
1439 * writes don't perturb the first and last elements'
1440 * CHUNK_MAP_ALLOCATED bits, behavior is well defined.
1441 */
1442 for (i = 1; i < npages-1; i++) {
1443 arena_mapbits_internal_set(chunk, pageind+i,
1444 flag_unzeroed);
1445 }
1446 }
1447
1448 npurged += npages;
1449 if (config_stats)
1450 nmadvise++;
1451 }
1452 malloc_mutex_lock(&arena->lock);
1453
1454 if (config_stats) {
1455 arena->stats.nmadvise += nmadvise;
1456 arena->stats.purged += npurged;
1457 }
1458
1459 return (npurged);
1460}
1461
1462static void
1463arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
1464 arena_runs_dirty_link_t *purge_runs_sentinel,
1465 extent_node_t *purge_chunks_sentinel)
1466{
1467 arena_runs_dirty_link_t *rdelm, *rdelm_next;
1468 extent_node_t *chunkselm;
1469
1470 /* Deallocate chunks/runs. */
1471 for (rdelm = qr_next(purge_runs_sentinel, rd_link),
1472 chunkselm = qr_next(purge_chunks_sentinel, cc_link);
1473 rdelm != purge_runs_sentinel; rdelm = rdelm_next) {
1474 rdelm_next = qr_next(rdelm, rd_link);
1475 if (rdelm == &chunkselm->rd) {
1476 extent_node_t *chunkselm_next = qr_next(chunkselm,
1477 cc_link);
1478 void *addr = extent_node_addr_get(chunkselm);
1479 size_t size = extent_node_size_get(chunkselm);
1480 bool zeroed = extent_node_zeroed_get(chunkselm);
1481 bool committed = extent_node_committed_get(chunkselm);
1482 extent_node_dirty_remove(chunkselm);
1483 arena_node_dalloc(arena, chunkselm);
1484 chunkselm = chunkselm_next;
1485 chunk_dalloc_arena(arena, chunk_hooks, addr, size,
1486 zeroed, committed);
1487 } else {
1488 arena_chunk_t *chunk =
1489 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
1490 arena_chunk_map_misc_t *miscelm =
1491 arena_rd_to_miscelm(rdelm);
1492 size_t pageind = arena_miscelm_to_pageind(miscelm);
1493 bool decommitted = (arena_mapbits_decommitted_get(chunk,
1494 pageind) != 0);
1495 arena_run_t *run = &miscelm->run;
1496 qr_remove(rdelm, rd_link);
1497 arena_run_dalloc(arena, run, false, true, decommitted);
1498 }
1499 }
1500}
1501
1502static void
1503arena_purge(arena_t *arena, bool all)
1504{
1505 chunk_hooks_t chunk_hooks = chunk_hooks_get(arena);
1506 size_t npurge, npurgeable, npurged;
1507 arena_runs_dirty_link_t purge_runs_sentinel;
1508 extent_node_t purge_chunks_sentinel;
1509
1510 arena->purging = true;
1511
1512 /*
1513 * Calls to arena_dirty_count() are disabled even for debug builds
1514 * because overhead grows nonlinearly as memory usage increases.
1515 */
1516 if (false && config_debug) {
1517 size_t ndirty = arena_dirty_count(arena);
1518 assert(ndirty == arena->ndirty);
1519 }
1520 assert((arena->nactive >> arena->lg_dirty_mult) < arena->ndirty || all);
1521
1522 if (config_stats)
1523 arena->stats.npurge++;
1524
1525 npurge = arena_compute_npurge(arena, all);
1526 qr_new(&purge_runs_sentinel, rd_link);
1527 extent_node_dirty_linkage_init(&purge_chunks_sentinel);
1528
1529 npurgeable = arena_stash_dirty(arena, &chunk_hooks, all, npurge,
1530 &purge_runs_sentinel, &purge_chunks_sentinel);
1531 assert(npurgeable >= npurge);
1532 npurged = arena_purge_stashed(arena, &chunk_hooks, &purge_runs_sentinel,
1533 &purge_chunks_sentinel);
1534 assert(npurged == npurgeable);
1535 arena_unstash_purged(arena, &chunk_hooks, &purge_runs_sentinel,
1536 &purge_chunks_sentinel);
1537
1538 arena->purging = false;
1539}
1540
1541void
1542arena_purge_all(arena_t *arena)
1543{
1544
1545 malloc_mutex_lock(&arena->lock);
1546 arena_purge(arena, true);
1547 malloc_mutex_unlock(&arena->lock);
1548}
1549
1550static void
1551arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size,
1552 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty,
1553 size_t flag_decommitted)
1554{
1555 size_t size = *p_size;
1556 size_t run_ind = *p_run_ind;
1557 size_t run_pages = *p_run_pages;
1558
1559 /* Try to coalesce forward. */
1560 if (run_ind + run_pages < chunk_npages &&
1561 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 &&
1562 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty &&
1563 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) ==
1564 flag_decommitted) {
1565 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk,
1566 run_ind+run_pages);
1567 size_t nrun_pages = nrun_size >> LG_PAGE;
1568
1569 /*
1570 * Remove successor from runs_avail; the coalesced run is
1571 * inserted later.
1572 */
1573 assert(arena_mapbits_unallocated_size_get(chunk,
1574 run_ind+run_pages+nrun_pages-1) == nrun_size);
1575 assert(arena_mapbits_dirty_get(chunk,
1576 run_ind+run_pages+nrun_pages-1) == flag_dirty);
1577 assert(arena_mapbits_decommitted_get(chunk,
1578 run_ind+run_pages+nrun_pages-1) == flag_decommitted);
1579 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages);
1580
1581 /*
1582 * If the successor is dirty, remove it from the set of dirty
1583 * pages.
1584 */
1585 if (flag_dirty != 0) {
1586 arena_run_dirty_remove(arena, chunk, run_ind+run_pages,
1587 nrun_pages);
1588 }
1589
1590 size += nrun_size;
1591 run_pages += nrun_pages;
1592
1593 arena_mapbits_unallocated_size_set(chunk, run_ind, size);
1594 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
1595 size);
1596 }
1597
1598 /* Try to coalesce backward. */
1599 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk,
1600 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) ==
1601 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) ==
1602 flag_decommitted) {
1603 size_t prun_size = arena_mapbits_unallocated_size_get(chunk,
1604 run_ind-1);
1605 size_t prun_pages = prun_size >> LG_PAGE;
1606
1607 run_ind -= prun_pages;
1608
1609 /*
1610 * Remove predecessor from runs_avail; the coalesced run is
1611 * inserted later.
1612 */
1613 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
1614 prun_size);
1615 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);
1616 assert(arena_mapbits_decommitted_get(chunk, run_ind) ==
1617 flag_decommitted);
1618 arena_avail_remove(arena, chunk, run_ind, prun_pages);
1619
1620 /*
1621 * If the predecessor is dirty, remove it from the set of dirty
1622 * pages.
1623 */
1624 if (flag_dirty != 0) {
1625 arena_run_dirty_remove(arena, chunk, run_ind,
1626 prun_pages);
1627 }
1628
1629 size += prun_size;
1630 run_pages += prun_pages;
1631
1632 arena_mapbits_unallocated_size_set(chunk, run_ind, size);
1633 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
1634 size);
1635 }
1636
1637 *p_size = size;
1638 *p_run_ind = run_ind;
1639 *p_run_pages = run_pages;
1640}
1641
1642static size_t
1643arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1644 size_t run_ind)
1645{
1646 size_t size;
1647
1648 assert(run_ind >= map_bias);
1649 assert(run_ind < chunk_npages);
1650
1651 if (arena_mapbits_large_get(chunk, run_ind) != 0) {
1652 size = arena_mapbits_large_size_get(chunk, run_ind);
1653 assert(size == PAGE || arena_mapbits_large_size_get(chunk,
1654 run_ind+(size>>LG_PAGE)-1) == 0);
1655 } else {
1656 arena_bin_info_t *bin_info = &arena_bin_info[run->binind];
1657 size = bin_info->run_size;
1658 }
1659
1660 return (size);
1661}
1662
1663static bool
1664arena_run_decommit(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run)
1665{
1666 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1667 size_t run_ind = arena_miscelm_to_pageind(miscelm);
1668 size_t offset = run_ind << LG_PAGE;
1669 size_t length = arena_run_size_get(arena, chunk, run, run_ind);
1670
1671 return (arena->chunk_hooks.decommit(chunk, chunksize, offset, length,
1672 arena->ind));
1673}
1674
1675static void
1676arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned,
1677 bool decommitted)
1678{
1679 arena_chunk_t *chunk;
1680 arena_chunk_map_misc_t *miscelm;
1681 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted;
1682
1683 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
1684 miscelm = arena_run_to_miscelm(run);
1685 run_ind = arena_miscelm_to_pageind(miscelm);
1686 assert(run_ind >= map_bias);
1687 assert(run_ind < chunk_npages);
1688 size = arena_run_size_get(arena, chunk, run, run_ind);
1689 run_pages = (size >> LG_PAGE);
1690 arena_cactive_update(arena, 0, run_pages);
1691 arena->nactive -= run_pages;
1692
1693 /*
1694 * The run is dirty if the caller claims to have dirtied it, as well as
1695 * if it was already dirty before being allocated and the caller
1696 * doesn't claim to have cleaned it.
1697 */
1698 assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1699 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
1700 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind)
1701 != 0)
1702 dirty = true;
1703 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0;
1704 flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0;
1705
1706 /* Mark pages as unallocated in the chunk map. */
1707 if (dirty || decommitted) {
1708 size_t flags = flag_dirty | flag_decommitted;
1709 arena_mapbits_unallocated_set(chunk, run_ind, size, flags);
1710 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
1711 flags);
1712 } else {
1713 arena_mapbits_unallocated_set(chunk, run_ind, size,
1714 arena_mapbits_unzeroed_get(chunk, run_ind));
1715 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
1716 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1));
1717 }
1718
1719 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages,
1720 flag_dirty, flag_decommitted);
1721
1722 /* Insert into runs_avail, now that coalescing is complete. */
1723 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
1724 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));
1725 assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1726 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
1727 assert(arena_mapbits_decommitted_get(chunk, run_ind) ==
1728 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1));
1729 arena_avail_insert(arena, chunk, run_ind, run_pages);
1730
1731 if (dirty)
1732 arena_run_dirty_insert(arena, chunk, run_ind, run_pages);
1733
1734 /* Deallocate chunk if it is now completely unused. */
1735 if (size == arena_maxrun) {
1736 assert(run_ind == map_bias);
1737 assert(run_pages == (arena_maxrun >> LG_PAGE));
1738 arena_chunk_dalloc(arena, chunk);
1739 }
1740
1741 /*
1742 * It is okay to do dirty page processing here even if the chunk was
1743 * deallocated above, since in that case it is the spare. Waiting
1744 * until after possible chunk deallocation to do dirty processing
1745 * allows for an old spare to be fully deallocated, thus decreasing the
1746 * chances of spuriously crossing the dirty page purging threshold.
1747 */
1748 if (dirty)
1749 arena_maybe_purge(arena);
1750}
1751
1752static void
1753arena_run_dalloc_decommit(arena_t *arena, arena_chunk_t *chunk,
1754 arena_run_t *run)
1755{
1756 bool committed = arena_run_decommit(arena, chunk, run);
1757
1758 arena_run_dalloc(arena, run, committed, false, !committed);
1759}
1760
1761static void
1762arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1763 size_t oldsize, size_t newsize)
1764{
1765 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1766 size_t pageind = arena_miscelm_to_pageind(miscelm);
1767 size_t head_npages = (oldsize - newsize) >> LG_PAGE;
1768 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
1769 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind);
1770 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ?
1771 CHUNK_MAP_UNZEROED : 0;
1772
1773 assert(oldsize > newsize);
1774
1775 /*
1776 * Update the chunk map so that arena_run_dalloc() can treat the
1777 * leading run as separately allocated. Set the last element of each
1778 * run first, in case of single-page runs.
1779 */
1780 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
1781 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty |
1782 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1783 pageind+head_npages-1)));
1784 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty |
1785 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind)));
1786
1787 if (config_debug) {
1788 UNUSED size_t tail_npages = newsize >> LG_PAGE;
1789 assert(arena_mapbits_large_size_get(chunk,
1790 pageind+head_npages+tail_npages-1) == 0);
1791 assert(arena_mapbits_dirty_get(chunk,
1792 pageind+head_npages+tail_npages-1) == flag_dirty);
1793 }
1794 arena_mapbits_large_set(chunk, pageind+head_npages, newsize,
1795 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1796 pageind+head_npages)));
1797
1798 arena_run_dalloc(arena, run, false, false, (flag_decommitted != 0));
1799}
1800
1801static void
1802arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1803 size_t oldsize, size_t newsize, bool dirty)
1804{
1805 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1806 size_t pageind = arena_miscelm_to_pageind(miscelm);
1807 size_t head_npages = newsize >> LG_PAGE;
1808 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
1809 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind);
1810 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ?
1811 CHUNK_MAP_UNZEROED : 0;
1812 arena_chunk_map_misc_t *tail_miscelm;
1813 arena_run_t *tail_run;
1814
1815 assert(oldsize > newsize);
1816
1817 /*
1818 * Update the chunk map so that arena_run_dalloc() can treat the
1819 * trailing run as separately allocated. Set the last element of each
1820 * run first, in case of single-page runs.
1821 */
1822 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
1823 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty |
1824 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1825 pageind+head_npages-1)));
1826 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty |
1827 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind)));
1828
1829 if (config_debug) {
1830 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE;
1831 assert(arena_mapbits_large_size_get(chunk,
1832 pageind+head_npages+tail_npages-1) == 0);
1833 assert(arena_mapbits_dirty_get(chunk,
1834 pageind+head_npages+tail_npages-1) == flag_dirty);
1835 }
1836 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize,
1837 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
1838 pageind+head_npages)));
1839
1840 tail_miscelm = arena_miscelm_get(chunk, pageind + head_npages);
1841 tail_run = &tail_miscelm->run;
1842 arena_run_dalloc(arena, tail_run, dirty, false, (flag_decommitted !=
1843 0));
1844}
1845
1846static arena_run_t *
1847arena_bin_runs_first(arena_bin_t *bin)
1848{
1849 arena_chunk_map_misc_t *miscelm = arena_run_tree_first(&bin->runs);
1850 if (miscelm != NULL)
1851 return (&miscelm->run);
1852
1853 return (NULL);
1854}
1855
1856static void
1857arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run)
1858{
1859 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1860
1861 assert(arena_run_tree_search(&bin->runs, miscelm) == NULL);
1862
1863 arena_run_tree_insert(&bin->runs, miscelm);
1864}
1865
1866static void
1867arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run)
1868{
1869 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
1870
1871 assert(arena_run_tree_search(&bin->runs, miscelm) != NULL);
1872
1873 arena_run_tree_remove(&bin->runs, miscelm);
1874}
1875
1876static arena_run_t *
1877arena_bin_nonfull_run_tryget(arena_bin_t *bin)
1878{
1879 arena_run_t *run = arena_bin_runs_first(bin);
1880 if (run != NULL) {
1881 arena_bin_runs_remove(bin, run);
1882 if (config_stats)
1883 bin->stats.reruns++;
1884 }
1885 return (run);
1886}
1887
1888static arena_run_t *
1889arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
1890{
1891 arena_run_t *run;
|
1892 index_t binind;
| 1892 szind_t binind;
|
1893 arena_bin_info_t *bin_info;
1894
1895 /* Look for a usable run. */
1896 run = arena_bin_nonfull_run_tryget(bin);
1897 if (run != NULL)
1898 return (run);
1899 /* No existing runs have any space available. */
1900
1901 binind = arena_bin_index(arena, bin);
1902 bin_info = &arena_bin_info[binind];
1903
1904 /* Allocate a new run. */
1905 malloc_mutex_unlock(&bin->lock);
1906 /******************************/
1907 malloc_mutex_lock(&arena->lock);
1908 run = arena_run_alloc_small(arena, bin_info->run_size, binind);
1909 if (run != NULL) {
1910 /* Initialize run internals. */
1911 run->binind = binind;
1912 run->nfree = bin_info->nregs;
1913 bitmap_init(run->bitmap, &bin_info->bitmap_info);
1914 }
1915 malloc_mutex_unlock(&arena->lock);
1916 /********************************/
1917 malloc_mutex_lock(&bin->lock);
1918 if (run != NULL) {
1919 if (config_stats) {
1920 bin->stats.nruns++;
1921 bin->stats.curruns++;
1922 }
1923 return (run);
1924 }
1925
1926 /*
1927 * arena_run_alloc_small() failed, but another thread may have made
1928 * sufficient memory available while this one dropped bin->lock above,
1929 * so search one more time.
1930 */
1931 run = arena_bin_nonfull_run_tryget(bin);
1932 if (run != NULL)
1933 return (run);
1934
1935 return (NULL);
1936}
1937
1938/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */
1939static void *
1940arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
1941{
| 1893 arena_bin_info_t *bin_info;
1894
1895 /* Look for a usable run. */
1896 run = arena_bin_nonfull_run_tryget(bin);
1897 if (run != NULL)
1898 return (run);
1899 /* No existing runs have any space available. */
1900
1901 binind = arena_bin_index(arena, bin);
1902 bin_info = &arena_bin_info[binind];
1903
1904 /* Allocate a new run. */
1905 malloc_mutex_unlock(&bin->lock);
1906 /******************************/
1907 malloc_mutex_lock(&arena->lock);
1908 run = arena_run_alloc_small(arena, bin_info->run_size, binind);
1909 if (run != NULL) {
1910 /* Initialize run internals. */
1911 run->binind = binind;
1912 run->nfree = bin_info->nregs;
1913 bitmap_init(run->bitmap, &bin_info->bitmap_info);
1914 }
1915 malloc_mutex_unlock(&arena->lock);
1916 /********************************/
1917 malloc_mutex_lock(&bin->lock);
1918 if (run != NULL) {
1919 if (config_stats) {
1920 bin->stats.nruns++;
1921 bin->stats.curruns++;
1922 }
1923 return (run);
1924 }
1925
1926 /*
1927 * arena_run_alloc_small() failed, but another thread may have made
1928 * sufficient memory available while this one dropped bin->lock above,
1929 * so search one more time.
1930 */
1931 run = arena_bin_nonfull_run_tryget(bin);
1932 if (run != NULL)
1933 return (run);
1934
1935 return (NULL);
1936}
1937
1938/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */
1939static void *
1940arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
1941{
|
1942 void *ret;
1943 index_t binind;
| 1942 szind_t binind;
|
1944 arena_bin_info_t *bin_info;
1945 arena_run_t *run;
1946
1947 binind = arena_bin_index(arena, bin);
1948 bin_info = &arena_bin_info[binind];
1949 bin->runcur = NULL;
1950 run = arena_bin_nonfull_run_get(arena, bin);
1951 if (bin->runcur != NULL && bin->runcur->nfree > 0) {
1952 /*
1953 * Another thread updated runcur while this one ran without the
1954 * bin lock in arena_bin_nonfull_run_get().
1955 */
| 1943 arena_bin_info_t *bin_info;
1944 arena_run_t *run;
1945
1946 binind = arena_bin_index(arena, bin);
1947 bin_info = &arena_bin_info[binind];
1948 bin->runcur = NULL;
1949 run = arena_bin_nonfull_run_get(arena, bin);
1950 if (bin->runcur != NULL && bin->runcur->nfree > 0) {
1951 /*
1952 * Another thread updated runcur while this one ran without the
1953 * bin lock in arena_bin_nonfull_run_get().
1954 */
|
| 1955 void *ret;
|
1956 assert(bin->runcur->nfree > 0);
1957 ret = arena_run_reg_alloc(bin->runcur, bin_info);
1958 if (run != NULL) {
1959 arena_chunk_t *chunk;
1960
1961 /*
1962 * arena_run_alloc_small() may have allocated run, or
1963 * it may have pulled run from the bin's run tree.
1964 * Therefore it is unsafe to make any assumptions about
1965 * how run has previously been used, and
1966 * arena_bin_lower_run() must be called, as if a region
1967 * were just deallocated from the run.
1968 */
1969 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
1970 if (run->nfree == bin_info->nregs)
1971 arena_dalloc_bin_run(arena, chunk, run, bin);
1972 else
1973 arena_bin_lower_run(arena, chunk, run, bin);
1974 }
1975 return (ret);
1976 }
1977
1978 if (run == NULL)
1979 return (NULL);
1980
1981 bin->runcur = run;
1982
1983 assert(bin->runcur->nfree > 0);
1984
1985 return (arena_run_reg_alloc(bin->runcur, bin_info));
1986}
1987
1988void
| 1956 assert(bin->runcur->nfree > 0);
1957 ret = arena_run_reg_alloc(bin->runcur, bin_info);
1958 if (run != NULL) {
1959 arena_chunk_t *chunk;
1960
1961 /*
1962 * arena_run_alloc_small() may have allocated run, or
1963 * it may have pulled run from the bin's run tree.
1964 * Therefore it is unsafe to make any assumptions about
1965 * how run has previously been used, and
1966 * arena_bin_lower_run() must be called, as if a region
1967 * were just deallocated from the run.
1968 */
1969 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
1970 if (run->nfree == bin_info->nregs)
1971 arena_dalloc_bin_run(arena, chunk, run, bin);
1972 else
1973 arena_bin_lower_run(arena, chunk, run, bin);
1974 }
1975 return (ret);
1976 }
1977
1978 if (run == NULL)
1979 return (NULL);
1980
1981 bin->runcur = run;
1982
1983 assert(bin->runcur->nfree > 0);
1984
1985 return (arena_run_reg_alloc(bin->runcur, bin_info));
1986}
1987
1988void
|
1989arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind,
| 1989arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind,
|
1990 uint64_t prof_accumbytes)
1991{
1992 unsigned i, nfill;
1993 arena_bin_t *bin;
| 1990 uint64_t prof_accumbytes)
1991{
1992 unsigned i, nfill;
1993 arena_bin_t *bin;
|
1994 arena_run_t *run;
1995 void *ptr;
| |
1996
1997 assert(tbin->ncached == 0);
1998
1999 if (config_prof && arena_prof_accum(arena, prof_accumbytes))
2000 prof_idump();
2001 bin = &arena->bins[binind];
2002 malloc_mutex_lock(&bin->lock);
2003 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
2004 tbin->lg_fill_div); i < nfill; i++) {
| 1994
1995 assert(tbin->ncached == 0);
1996
1997 if (config_prof && arena_prof_accum(arena, prof_accumbytes))
1998 prof_idump();
1999 bin = &arena->bins[binind];
2000 malloc_mutex_lock(&bin->lock);
2001 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
2002 tbin->lg_fill_div); i < nfill; i++) {
|
| 2003 arena_run_t *run;
2004 void *ptr;
|
2005 if ((run = bin->runcur) != NULL && run->nfree > 0)
2006 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]);
2007 else
2008 ptr = arena_bin_malloc_hard(arena, bin);
2009 if (ptr == NULL) {
2010 /*
2011 * OOM. tbin->avail isn't yet filled down to its first
2012 * element, so the successful allocations (if any) must
2013 * be moved to the base of tbin->avail before bailing
2014 * out.
2015 */
2016 if (i > 0) {
2017 memmove(tbin->avail, &tbin->avail[nfill - i],
2018 i * sizeof(void *));
2019 }
2020 break;
2021 }
2022 if (config_fill && unlikely(opt_junk_alloc)) {
2023 arena_alloc_junk_small(ptr, &arena_bin_info[binind],
2024 true);
2025 }
2026 /* Insert such that low regions get used first. */
2027 tbin->avail[nfill - 1 - i] = ptr;
2028 }
2029 if (config_stats) {
2030 bin->stats.nmalloc += i;
2031 bin->stats.nrequests += tbin->tstats.nrequests;
2032 bin->stats.curregs += i;
2033 bin->stats.nfills++;
2034 tbin->tstats.nrequests = 0;
2035 }
2036 malloc_mutex_unlock(&bin->lock);
2037 tbin->ncached = i;
2038}
2039
2040void
2041arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero)
2042{
2043
2044 if (zero) {
2045 size_t redzone_size = bin_info->redzone_size;
2046 memset((void *)((uintptr_t)ptr - redzone_size), 0xa5,
2047 redzone_size);
2048 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5,
2049 redzone_size);
2050 } else {
2051 memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5,
2052 bin_info->reg_interval);
2053 }
2054}
2055
2056#ifdef JEMALLOC_JET
2057#undef arena_redzone_corruption
2058#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl)
2059#endif
2060static void
2061arena_redzone_corruption(void *ptr, size_t usize, bool after,
2062 size_t offset, uint8_t byte)
2063{
2064
2065 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p "
2066 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s",
2067 after ? "after" : "before", ptr, usize, byte);
2068}
2069#ifdef JEMALLOC_JET
2070#undef arena_redzone_corruption
2071#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption)
2072arena_redzone_corruption_t *arena_redzone_corruption =
2073 JEMALLOC_N(arena_redzone_corruption_impl);
2074#endif
2075
2076static void
2077arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset)
2078{
| 2005 if ((run = bin->runcur) != NULL && run->nfree > 0)
2006 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]);
2007 else
2008 ptr = arena_bin_malloc_hard(arena, bin);
2009 if (ptr == NULL) {
2010 /*
2011 * OOM. tbin->avail isn't yet filled down to its first
2012 * element, so the successful allocations (if any) must
2013 * be moved to the base of tbin->avail before bailing
2014 * out.
2015 */
2016 if (i > 0) {
2017 memmove(tbin->avail, &tbin->avail[nfill - i],
2018 i * sizeof(void *));
2019 }
2020 break;
2021 }
2022 if (config_fill && unlikely(opt_junk_alloc)) {
2023 arena_alloc_junk_small(ptr, &arena_bin_info[binind],
2024 true);
2025 }
2026 /* Insert such that low regions get used first. */
2027 tbin->avail[nfill - 1 - i] = ptr;
2028 }
2029 if (config_stats) {
2030 bin->stats.nmalloc += i;
2031 bin->stats.nrequests += tbin->tstats.nrequests;
2032 bin->stats.curregs += i;
2033 bin->stats.nfills++;
2034 tbin->tstats.nrequests = 0;
2035 }
2036 malloc_mutex_unlock(&bin->lock);
2037 tbin->ncached = i;
2038}
2039
2040void
2041arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero)
2042{
2043
2044 if (zero) {
2045 size_t redzone_size = bin_info->redzone_size;
2046 memset((void *)((uintptr_t)ptr - redzone_size), 0xa5,
2047 redzone_size);
2048 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5,
2049 redzone_size);
2050 } else {
2051 memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5,
2052 bin_info->reg_interval);
2053 }
2054}
2055
2056#ifdef JEMALLOC_JET
2057#undef arena_redzone_corruption
2058#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl)
2059#endif
2060static void
2061arena_redzone_corruption(void *ptr, size_t usize, bool after,
2062 size_t offset, uint8_t byte)
2063{
2064
2065 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p "
2066 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s",
2067 after ? "after" : "before", ptr, usize, byte);
2068}
2069#ifdef JEMALLOC_JET
2070#undef arena_redzone_corruption
2071#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption)
2072arena_redzone_corruption_t *arena_redzone_corruption =
2073 JEMALLOC_N(arena_redzone_corruption_impl);
2074#endif
2075
2076static void
2077arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset)
2078{
|
2079 size_t size = bin_info->reg_size;
2080 size_t redzone_size = bin_info->redzone_size;
2081 size_t i;
| |
2082 bool error = false;
2083
2084 if (opt_junk_alloc) {
| 2079 bool error = false;
2080
2081 if (opt_junk_alloc) {
|
| 2082 size_t size = bin_info->reg_size;
2083 size_t redzone_size = bin_info->redzone_size;
2084 size_t i;
2085
|
2085 for (i = 1; i <= redzone_size; i++) {
2086 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i);
2087 if (*byte != 0xa5) {
2088 error = true;
2089 arena_redzone_corruption(ptr, size, false, i,
2090 *byte);
2091 if (reset)
2092 *byte = 0xa5;
2093 }
2094 }
2095 for (i = 0; i < redzone_size; i++) {
2096 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i);
2097 if (*byte != 0xa5) {
2098 error = true;
2099 arena_redzone_corruption(ptr, size, true, i,
2100 *byte);
2101 if (reset)
2102 *byte = 0xa5;
2103 }
2104 }
2105 }
2106
2107 if (opt_abort && error)
2108 abort();
2109}
2110
2111#ifdef JEMALLOC_JET
2112#undef arena_dalloc_junk_small
2113#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl)
2114#endif
2115void
2116arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info)
2117{
2118 size_t redzone_size = bin_info->redzone_size;
2119
2120 arena_redzones_validate(ptr, bin_info, false);
2121 memset((void *)((uintptr_t)ptr - redzone_size), 0x5a,
2122 bin_info->reg_interval);
2123}
2124#ifdef JEMALLOC_JET
2125#undef arena_dalloc_junk_small
2126#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
2127arena_dalloc_junk_small_t *arena_dalloc_junk_small =
2128 JEMALLOC_N(arena_dalloc_junk_small_impl);
2129#endif
2130
2131void
2132arena_quarantine_junk_small(void *ptr, size_t usize)
2133{
| 2086 for (i = 1; i <= redzone_size; i++) {
2087 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i);
2088 if (*byte != 0xa5) {
2089 error = true;
2090 arena_redzone_corruption(ptr, size, false, i,
2091 *byte);
2092 if (reset)
2093 *byte = 0xa5;
2094 }
2095 }
2096 for (i = 0; i < redzone_size; i++) {
2097 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i);
2098 if (*byte != 0xa5) {
2099 error = true;
2100 arena_redzone_corruption(ptr, size, true, i,
2101 *byte);
2102 if (reset)
2103 *byte = 0xa5;
2104 }
2105 }
2106 }
2107
2108 if (opt_abort && error)
2109 abort();
2110}
2111
2112#ifdef JEMALLOC_JET
2113#undef arena_dalloc_junk_small
2114#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl)
2115#endif
2116void
2117arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info)
2118{
2119 size_t redzone_size = bin_info->redzone_size;
2120
2121 arena_redzones_validate(ptr, bin_info, false);
2122 memset((void *)((uintptr_t)ptr - redzone_size), 0x5a,
2123 bin_info->reg_interval);
2124}
2125#ifdef JEMALLOC_JET
2126#undef arena_dalloc_junk_small
2127#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
2128arena_dalloc_junk_small_t *arena_dalloc_junk_small =
2129 JEMALLOC_N(arena_dalloc_junk_small_impl);
2130#endif
2131
2132void
2133arena_quarantine_junk_small(void *ptr, size_t usize)
2134{
|
2134 index_t binind;
| 2135 szind_t binind;
|
2135 arena_bin_info_t *bin_info;
2136 cassert(config_fill);
2137 assert(opt_junk_free);
2138 assert(opt_quarantine);
2139 assert(usize <= SMALL_MAXCLASS);
2140
2141 binind = size2index(usize);
2142 bin_info = &arena_bin_info[binind];
2143 arena_redzones_validate(ptr, bin_info, true);
2144}
2145
2146void *
2147arena_malloc_small(arena_t *arena, size_t size, bool zero)
2148{
2149 void *ret;
2150 arena_bin_t *bin;
2151 arena_run_t *run;
| 2136 arena_bin_info_t *bin_info;
2137 cassert(config_fill);
2138 assert(opt_junk_free);
2139 assert(opt_quarantine);
2140 assert(usize <= SMALL_MAXCLASS);
2141
2142 binind = size2index(usize);
2143 bin_info = &arena_bin_info[binind];
2144 arena_redzones_validate(ptr, bin_info, true);
2145}
2146
2147void *
2148arena_malloc_small(arena_t *arena, size_t size, bool zero)
2149{
2150 void *ret;
2151 arena_bin_t *bin;
2152 arena_run_t *run;
|
2152 index_t binind;
| 2153 szind_t binind;
|
2153
2154 binind = size2index(size);
2155 assert(binind < NBINS);
2156 bin = &arena->bins[binind];
2157 size = index2size(binind);
2158
2159 malloc_mutex_lock(&bin->lock);
2160 if ((run = bin->runcur) != NULL && run->nfree > 0)
2161 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]);
2162 else
2163 ret = arena_bin_malloc_hard(arena, bin);
2164
2165 if (ret == NULL) {
2166 malloc_mutex_unlock(&bin->lock);
2167 return (NULL);
2168 }
2169
2170 if (config_stats) {
2171 bin->stats.nmalloc++;
2172 bin->stats.nrequests++;
2173 bin->stats.curregs++;
2174 }
2175 malloc_mutex_unlock(&bin->lock);
2176 if (config_prof && !isthreaded && arena_prof_accum(arena, size))
2177 prof_idump();
2178
2179 if (!zero) {
2180 if (config_fill) {
2181 if (unlikely(opt_junk_alloc)) {
2182 arena_alloc_junk_small(ret,
2183 &arena_bin_info[binind], false);
2184 } else if (unlikely(opt_zero))
2185 memset(ret, 0, size);
2186 }
2187 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
2188 } else {
2189 if (config_fill && unlikely(opt_junk_alloc)) {
2190 arena_alloc_junk_small(ret, &arena_bin_info[binind],
2191 true);
2192 }
2193 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
2194 memset(ret, 0, size);
2195 }
2196
2197 return (ret);
2198}
2199
2200void *
2201arena_malloc_large(arena_t *arena, size_t size, bool zero)
2202{
2203 void *ret;
2204 size_t usize;
2205 uintptr_t random_offset;
2206 arena_run_t *run;
2207 arena_chunk_map_misc_t *miscelm;
2208 UNUSED bool idump;
2209
2210 /* Large allocation. */
2211 usize = s2u(size);
2212 malloc_mutex_lock(&arena->lock);
2213 if (config_cache_oblivious) {
2214 uint64_t r;
2215
2216 /*
2217 * Compute a uniformly distributed offset within the first page
2218 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64
2219 * for 4 KiB pages and 64-byte cachelines.
2220 */
2221 prng64(r, LG_PAGE - LG_CACHELINE, arena->offset_state,
2222 UINT64_C(6364136223846793009),
2223 UINT64_C(1442695040888963409));
2224 random_offset = ((uintptr_t)r) << LG_CACHELINE;
2225 } else
2226 random_offset = 0;
2227 run = arena_run_alloc_large(arena, usize + large_pad, zero);
2228 if (run == NULL) {
2229 malloc_mutex_unlock(&arena->lock);
2230 return (NULL);
2231 }
2232 miscelm = arena_run_to_miscelm(run);
2233 ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) +
2234 random_offset);
2235 if (config_stats) {
| 2154
2155 binind = size2index(size);
2156 assert(binind < NBINS);
2157 bin = &arena->bins[binind];
2158 size = index2size(binind);
2159
2160 malloc_mutex_lock(&bin->lock);
2161 if ((run = bin->runcur) != NULL && run->nfree > 0)
2162 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]);
2163 else
2164 ret = arena_bin_malloc_hard(arena, bin);
2165
2166 if (ret == NULL) {
2167 malloc_mutex_unlock(&bin->lock);
2168 return (NULL);
2169 }
2170
2171 if (config_stats) {
2172 bin->stats.nmalloc++;
2173 bin->stats.nrequests++;
2174 bin->stats.curregs++;
2175 }
2176 malloc_mutex_unlock(&bin->lock);
2177 if (config_prof && !isthreaded && arena_prof_accum(arena, size))
2178 prof_idump();
2179
2180 if (!zero) {
2181 if (config_fill) {
2182 if (unlikely(opt_junk_alloc)) {
2183 arena_alloc_junk_small(ret,
2184 &arena_bin_info[binind], false);
2185 } else if (unlikely(opt_zero))
2186 memset(ret, 0, size);
2187 }
2188 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
2189 } else {
2190 if (config_fill && unlikely(opt_junk_alloc)) {
2191 arena_alloc_junk_small(ret, &arena_bin_info[binind],
2192 true);
2193 }
2194 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
2195 memset(ret, 0, size);
2196 }
2197
2198 return (ret);
2199}
2200
2201void *
2202arena_malloc_large(arena_t *arena, size_t size, bool zero)
2203{
2204 void *ret;
2205 size_t usize;
2206 uintptr_t random_offset;
2207 arena_run_t *run;
2208 arena_chunk_map_misc_t *miscelm;
2209 UNUSED bool idump;
2210
2211 /* Large allocation. */
2212 usize = s2u(size);
2213 malloc_mutex_lock(&arena->lock);
2214 if (config_cache_oblivious) {
2215 uint64_t r;
2216
2217 /*
2218 * Compute a uniformly distributed offset within the first page
2219 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64
2220 * for 4 KiB pages and 64-byte cachelines.
2221 */
2222 prng64(r, LG_PAGE - LG_CACHELINE, arena->offset_state,
2223 UINT64_C(6364136223846793009),
2224 UINT64_C(1442695040888963409));
2225 random_offset = ((uintptr_t)r) << LG_CACHELINE;
2226 } else
2227 random_offset = 0;
2228 run = arena_run_alloc_large(arena, usize + large_pad, zero);
2229 if (run == NULL) {
2230 malloc_mutex_unlock(&arena->lock);
2231 return (NULL);
2232 }
2233 miscelm = arena_run_to_miscelm(run);
2234 ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) +
2235 random_offset);
2236 if (config_stats) {
|
2236 index_t index = size2index(usize) - NBINS;
| 2237 szind_t index = size2index(usize) - NBINS;
|
2237
2238 arena->stats.nmalloc_large++;
2239 arena->stats.nrequests_large++;
2240 arena->stats.allocated_large += usize;
2241 arena->stats.lstats[index].nmalloc++;
2242 arena->stats.lstats[index].nrequests++;
2243 arena->stats.lstats[index].curruns++;
2244 }
2245 if (config_prof)
2246 idump = arena_prof_accum_locked(arena, usize);
2247 malloc_mutex_unlock(&arena->lock);
2248 if (config_prof && idump)
2249 prof_idump();
2250
2251 if (!zero) {
2252 if (config_fill) {
2253 if (unlikely(opt_junk_alloc))
2254 memset(ret, 0xa5, usize);
2255 else if (unlikely(opt_zero))
2256 memset(ret, 0, usize);
2257 }
2258 }
2259
2260 return (ret);
2261}
2262
2263/* Only handles large allocations that require more than page alignment. */
2264static void *
2265arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
2266 bool zero)
2267{
2268 void *ret;
2269 size_t alloc_size, leadsize, trailsize;
2270 arena_run_t *run;
2271 arena_chunk_t *chunk;
2272 arena_chunk_map_misc_t *miscelm;
2273 void *rpages;
2274
2275 assert(usize == PAGE_CEILING(usize));
2276
2277 arena = arena_choose(tsd, arena);
2278 if (unlikely(arena == NULL))
2279 return (NULL);
2280
2281 alignment = PAGE_CEILING(alignment);
2282 alloc_size = usize + large_pad + alignment - PAGE;
2283
2284 malloc_mutex_lock(&arena->lock);
2285 run = arena_run_alloc_large(arena, alloc_size, false);
2286 if (run == NULL) {
2287 malloc_mutex_unlock(&arena->lock);
2288 return (NULL);
2289 }
2290 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
2291 miscelm = arena_run_to_miscelm(run);
2292 rpages = arena_miscelm_to_rpages(miscelm);
2293
2294 leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) -
2295 (uintptr_t)rpages;
2296 assert(alloc_size >= leadsize + usize);
2297 trailsize = alloc_size - leadsize - usize - large_pad;
2298 if (leadsize != 0) {
2299 arena_chunk_map_misc_t *head_miscelm = miscelm;
2300 arena_run_t *head_run = run;
2301
2302 miscelm = arena_miscelm_get(chunk,
2303 arena_miscelm_to_pageind(head_miscelm) + (leadsize >>
2304 LG_PAGE));
2305 run = &miscelm->run;
2306
2307 arena_run_trim_head(arena, chunk, head_run, alloc_size,
2308 alloc_size - leadsize);
2309 }
2310 if (trailsize != 0) {
2311 arena_run_trim_tail(arena, chunk, run, usize + large_pad +
2312 trailsize, usize + large_pad, false);
2313 }
2314 if (arena_run_init_large(arena, run, usize + large_pad, zero)) {
2315 size_t run_ind =
2316 arena_miscelm_to_pageind(arena_run_to_miscelm(run));
2317 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0);
2318 bool decommitted = (arena_mapbits_decommitted_get(chunk,
2319 run_ind) != 0);
2320
2321 assert(decommitted); /* Cause of OOM. */
2322 arena_run_dalloc(arena, run, dirty, false, decommitted);
2323 malloc_mutex_unlock(&arena->lock);
2324 return (NULL);
2325 }
2326 ret = arena_miscelm_to_rpages(miscelm);
2327
2328 if (config_stats) {
| 2238
2239 arena->stats.nmalloc_large++;
2240 arena->stats.nrequests_large++;
2241 arena->stats.allocated_large += usize;
2242 arena->stats.lstats[index].nmalloc++;
2243 arena->stats.lstats[index].nrequests++;
2244 arena->stats.lstats[index].curruns++;
2245 }
2246 if (config_prof)
2247 idump = arena_prof_accum_locked(arena, usize);
2248 malloc_mutex_unlock(&arena->lock);
2249 if (config_prof && idump)
2250 prof_idump();
2251
2252 if (!zero) {
2253 if (config_fill) {
2254 if (unlikely(opt_junk_alloc))
2255 memset(ret, 0xa5, usize);
2256 else if (unlikely(opt_zero))
2257 memset(ret, 0, usize);
2258 }
2259 }
2260
2261 return (ret);
2262}
2263
2264/* Only handles large allocations that require more than page alignment. */
2265static void *
2266arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
2267 bool zero)
2268{
2269 void *ret;
2270 size_t alloc_size, leadsize, trailsize;
2271 arena_run_t *run;
2272 arena_chunk_t *chunk;
2273 arena_chunk_map_misc_t *miscelm;
2274 void *rpages;
2275
2276 assert(usize == PAGE_CEILING(usize));
2277
2278 arena = arena_choose(tsd, arena);
2279 if (unlikely(arena == NULL))
2280 return (NULL);
2281
2282 alignment = PAGE_CEILING(alignment);
2283 alloc_size = usize + large_pad + alignment - PAGE;
2284
2285 malloc_mutex_lock(&arena->lock);
2286 run = arena_run_alloc_large(arena, alloc_size, false);
2287 if (run == NULL) {
2288 malloc_mutex_unlock(&arena->lock);
2289 return (NULL);
2290 }
2291 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
2292 miscelm = arena_run_to_miscelm(run);
2293 rpages = arena_miscelm_to_rpages(miscelm);
2294
2295 leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) -
2296 (uintptr_t)rpages;
2297 assert(alloc_size >= leadsize + usize);
2298 trailsize = alloc_size - leadsize - usize - large_pad;
2299 if (leadsize != 0) {
2300 arena_chunk_map_misc_t *head_miscelm = miscelm;
2301 arena_run_t *head_run = run;
2302
2303 miscelm = arena_miscelm_get(chunk,
2304 arena_miscelm_to_pageind(head_miscelm) + (leadsize >>
2305 LG_PAGE));
2306 run = &miscelm->run;
2307
2308 arena_run_trim_head(arena, chunk, head_run, alloc_size,
2309 alloc_size - leadsize);
2310 }
2311 if (trailsize != 0) {
2312 arena_run_trim_tail(arena, chunk, run, usize + large_pad +
2313 trailsize, usize + large_pad, false);
2314 }
2315 if (arena_run_init_large(arena, run, usize + large_pad, zero)) {
2316 size_t run_ind =
2317 arena_miscelm_to_pageind(arena_run_to_miscelm(run));
2318 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0);
2319 bool decommitted = (arena_mapbits_decommitted_get(chunk,
2320 run_ind) != 0);
2321
2322 assert(decommitted); /* Cause of OOM. */
2323 arena_run_dalloc(arena, run, dirty, false, decommitted);
2324 malloc_mutex_unlock(&arena->lock);
2325 return (NULL);
2326 }
2327 ret = arena_miscelm_to_rpages(miscelm);
2328
2329 if (config_stats) {
|
2329 index_t index = size2index(usize) - NBINS;
| 2330 szind_t index = size2index(usize) - NBINS;
|
2330
2331 arena->stats.nmalloc_large++;
2332 arena->stats.nrequests_large++;
2333 arena->stats.allocated_large += usize;
2334 arena->stats.lstats[index].nmalloc++;
2335 arena->stats.lstats[index].nrequests++;
2336 arena->stats.lstats[index].curruns++;
2337 }
2338 malloc_mutex_unlock(&arena->lock);
2339
2340 if (config_fill && !zero) {
2341 if (unlikely(opt_junk_alloc))
2342 memset(ret, 0xa5, usize);
2343 else if (unlikely(opt_zero))
2344 memset(ret, 0, usize);
2345 }
2346 return (ret);
2347}
2348
2349void *
2350arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
2351 bool zero, tcache_t *tcache)
2352{
2353 void *ret;
2354
2355 if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE
2356 && (usize & PAGE_MASK) == 0))) {
2357 /* Small; alignment doesn't require special run placement. */
2358 ret = arena_malloc(tsd, arena, usize, zero, tcache);
| 2331
2332 arena->stats.nmalloc_large++;
2333 arena->stats.nrequests_large++;
2334 arena->stats.allocated_large += usize;
2335 arena->stats.lstats[index].nmalloc++;
2336 arena->stats.lstats[index].nrequests++;
2337 arena->stats.lstats[index].curruns++;
2338 }
2339 malloc_mutex_unlock(&arena->lock);
2340
2341 if (config_fill && !zero) {
2342 if (unlikely(opt_junk_alloc))
2343 memset(ret, 0xa5, usize);
2344 else if (unlikely(opt_zero))
2345 memset(ret, 0, usize);
2346 }
2347 return (ret);
2348}
2349
2350void *
2351arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
2352 bool zero, tcache_t *tcache)
2353{
2354 void *ret;
2355
2356 if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE
2357 && (usize & PAGE_MASK) == 0))) {
2358 /* Small; alignment doesn't require special run placement. */
2359 ret = arena_malloc(tsd, arena, usize, zero, tcache);
|
2359 } else if (usize <= arena_maxclass && alignment <= PAGE) {
| 2360 } else if (usize <= large_maxclass && alignment <= PAGE) {
|
2360 /*
2361 * Large; alignment doesn't require special run placement.
2362 * However, the cached pointer may be at a random offset from
2363 * the base of the run, so do some bit manipulation to retrieve
2364 * the base.
2365 */
2366 ret = arena_malloc(tsd, arena, usize, zero, tcache);
2367 if (config_cache_oblivious)
2368 ret = (void *)((uintptr_t)ret & ~PAGE_MASK);
2369 } else {
| 2361 /*
2362 * Large; alignment doesn't require special run placement.
2363 * However, the cached pointer may be at a random offset from
2364 * the base of the run, so do some bit manipulation to retrieve
2365 * the base.
2366 */
2367 ret = arena_malloc(tsd, arena, usize, zero, tcache);
2368 if (config_cache_oblivious)
2369 ret = (void *)((uintptr_t)ret & ~PAGE_MASK);
2370 } else {
|
2370 if (likely(usize <= arena_maxclass)) {
| 2371 if (likely(usize <= large_maxclass)) {
|
2371 ret = arena_palloc_large(tsd, arena, usize, alignment,
2372 zero);
2373 } else if (likely(alignment <= chunksize))
2374 ret = huge_malloc(tsd, arena, usize, zero, tcache);
2375 else {
2376 ret = huge_palloc(tsd, arena, usize, alignment, zero,
2377 tcache);
2378 }
2379 }
2380 return (ret);
2381}
2382
2383void
2384arena_prof_promoted(const void *ptr, size_t size)
2385{
2386 arena_chunk_t *chunk;
2387 size_t pageind;
| 2372 ret = arena_palloc_large(tsd, arena, usize, alignment,
2373 zero);
2374 } else if (likely(alignment <= chunksize))
2375 ret = huge_malloc(tsd, arena, usize, zero, tcache);
2376 else {
2377 ret = huge_palloc(tsd, arena, usize, alignment, zero,
2378 tcache);
2379 }
2380 }
2381 return (ret);
2382}
2383
2384void
2385arena_prof_promoted(const void *ptr, size_t size)
2386{
2387 arena_chunk_t *chunk;
2388 size_t pageind;
|
2388 index_t binind;
| 2389 szind_t binind;
|
2389
2390 cassert(config_prof);
2391 assert(ptr != NULL);
2392 assert(CHUNK_ADDR2BASE(ptr) != ptr);
2393 assert(isalloc(ptr, false) == LARGE_MINCLASS);
2394 assert(isalloc(ptr, true) == LARGE_MINCLASS);
2395 assert(size <= SMALL_MAXCLASS);
2396
2397 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2398 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2399 binind = size2index(size);
2400 assert(binind < NBINS);
2401 arena_mapbits_large_binind_set(chunk, pageind, binind);
2402
2403 assert(isalloc(ptr, false) == LARGE_MINCLASS);
2404 assert(isalloc(ptr, true) == size);
2405}
2406
2407static void
2408arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
2409 arena_bin_t *bin)
2410{
2411
2412 /* Dissociate run from bin. */
2413 if (run == bin->runcur)
2414 bin->runcur = NULL;
2415 else {
| 2390
2391 cassert(config_prof);
2392 assert(ptr != NULL);
2393 assert(CHUNK_ADDR2BASE(ptr) != ptr);
2394 assert(isalloc(ptr, false) == LARGE_MINCLASS);
2395 assert(isalloc(ptr, true) == LARGE_MINCLASS);
2396 assert(size <= SMALL_MAXCLASS);
2397
2398 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2399 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2400 binind = size2index(size);
2401 assert(binind < NBINS);
2402 arena_mapbits_large_binind_set(chunk, pageind, binind);
2403
2404 assert(isalloc(ptr, false) == LARGE_MINCLASS);
2405 assert(isalloc(ptr, true) == size);
2406}
2407
2408static void
2409arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
2410 arena_bin_t *bin)
2411{
2412
2413 /* Dissociate run from bin. */
2414 if (run == bin->runcur)
2415 bin->runcur = NULL;
2416 else {
|
2416 index_t binind = arena_bin_index(extent_node_arena_get(
| 2417 szind_t binind = arena_bin_index(extent_node_arena_get(
|
2417 &chunk->node), bin);
2418 arena_bin_info_t *bin_info = &arena_bin_info[binind];
2419
2420 if (bin_info->nregs != 1) {
2421 /*
2422 * This block's conditional is necessary because if the
2423 * run only contains one region, then it never gets
2424 * inserted into the non-full runs tree.
2425 */
2426 arena_bin_runs_remove(bin, run);
2427 }
2428 }
2429}
2430
2431static void
2432arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
2433 arena_bin_t *bin)
2434{
2435
2436 assert(run != bin->runcur);
2437 assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) ==
2438 NULL);
2439
2440 malloc_mutex_unlock(&bin->lock);
2441 /******************************/
2442 malloc_mutex_lock(&arena->lock);
2443 arena_run_dalloc_decommit(arena, chunk, run);
2444 malloc_mutex_unlock(&arena->lock);
2445 /****************************/
2446 malloc_mutex_lock(&bin->lock);
2447 if (config_stats)
2448 bin->stats.curruns--;
2449}
2450
2451static void
2452arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
2453 arena_bin_t *bin)
2454{
2455
2456 /*
2457 * Make sure that if bin->runcur is non-NULL, it refers to the lowest
2458 * non-full run. It is okay to NULL runcur out rather than proactively
2459 * keeping it pointing at the lowest non-full run.
2460 */
2461 if ((uintptr_t)run < (uintptr_t)bin->runcur) {
2462 /* Switch runcur. */
2463 if (bin->runcur->nfree > 0)
2464 arena_bin_runs_insert(bin, bin->runcur);
2465 bin->runcur = run;
2466 if (config_stats)
2467 bin->stats.reruns++;
2468 } else
2469 arena_bin_runs_insert(bin, run);
2470}
2471
2472static void
2473arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2474 arena_chunk_map_bits_t *bitselm, bool junked)
2475{
2476 size_t pageind, rpages_ind;
2477 arena_run_t *run;
2478 arena_bin_t *bin;
2479 arena_bin_info_t *bin_info;
| 2418 &chunk->node), bin);
2419 arena_bin_info_t *bin_info = &arena_bin_info[binind];
2420
2421 if (bin_info->nregs != 1) {
2422 /*
2423 * This block's conditional is necessary because if the
2424 * run only contains one region, then it never gets
2425 * inserted into the non-full runs tree.
2426 */
2427 arena_bin_runs_remove(bin, run);
2428 }
2429 }
2430}
2431
2432static void
2433arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
2434 arena_bin_t *bin)
2435{
2436
2437 assert(run != bin->runcur);
2438 assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) ==
2439 NULL);
2440
2441 malloc_mutex_unlock(&bin->lock);
2442 /******************************/
2443 malloc_mutex_lock(&arena->lock);
2444 arena_run_dalloc_decommit(arena, chunk, run);
2445 malloc_mutex_unlock(&arena->lock);
2446 /****************************/
2447 malloc_mutex_lock(&bin->lock);
2448 if (config_stats)
2449 bin->stats.curruns--;
2450}
2451
2452static void
2453arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
2454 arena_bin_t *bin)
2455{
2456
2457 /*
2458 * Make sure that if bin->runcur is non-NULL, it refers to the lowest
2459 * non-full run. It is okay to NULL runcur out rather than proactively
2460 * keeping it pointing at the lowest non-full run.
2461 */
2462 if ((uintptr_t)run < (uintptr_t)bin->runcur) {
2463 /* Switch runcur. */
2464 if (bin->runcur->nfree > 0)
2465 arena_bin_runs_insert(bin, bin->runcur);
2466 bin->runcur = run;
2467 if (config_stats)
2468 bin->stats.reruns++;
2469 } else
2470 arena_bin_runs_insert(bin, run);
2471}
2472
2473static void
2474arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2475 arena_chunk_map_bits_t *bitselm, bool junked)
2476{
2477 size_t pageind, rpages_ind;
2478 arena_run_t *run;
2479 arena_bin_t *bin;
2480 arena_bin_info_t *bin_info;
|
2480 index_t binind;
| 2481 szind_t binind;
|
2481
2482 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2483 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
2484 run = &arena_miscelm_get(chunk, rpages_ind)->run;
2485 binind = run->binind;
2486 bin = &arena->bins[binind];
2487 bin_info = &arena_bin_info[binind];
2488
2489 if (!junked && config_fill && unlikely(opt_junk_free))
2490 arena_dalloc_junk_small(ptr, bin_info);
2491
2492 arena_run_reg_dalloc(run, ptr);
2493 if (run->nfree == bin_info->nregs) {
2494 arena_dissociate_bin_run(chunk, run, bin);
2495 arena_dalloc_bin_run(arena, chunk, run, bin);
2496 } else if (run->nfree == 1 && run != bin->runcur)
2497 arena_bin_lower_run(arena, chunk, run, bin);
2498
2499 if (config_stats) {
2500 bin->stats.ndalloc++;
2501 bin->stats.curregs--;
2502 }
2503}
2504
2505void
2506arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2507 arena_chunk_map_bits_t *bitselm)
2508{
2509
2510 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true);
2511}
2512
2513void
2514arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2515 size_t pageind, arena_chunk_map_bits_t *bitselm)
2516{
2517 arena_run_t *run;
2518 arena_bin_t *bin;
2519 size_t rpages_ind;
2520
2521 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
2522 run = &arena_miscelm_get(chunk, rpages_ind)->run;
2523 bin = &arena->bins[run->binind];
2524 malloc_mutex_lock(&bin->lock);
2525 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false);
2526 malloc_mutex_unlock(&bin->lock);
2527}
2528
2529void
2530arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2531 size_t pageind)
2532{
2533 arena_chunk_map_bits_t *bitselm;
2534
2535 if (config_debug) {
2536 /* arena_ptr_small_binind_get() does extra sanity checking. */
2537 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
2538 pageind)) != BININD_INVALID);
2539 }
2540 bitselm = arena_bitselm_get(chunk, pageind);
2541 arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm);
2542}
2543
2544#ifdef JEMALLOC_JET
2545#undef arena_dalloc_junk_large
2546#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl)
2547#endif
2548void
2549arena_dalloc_junk_large(void *ptr, size_t usize)
2550{
2551
2552 if (config_fill && unlikely(opt_junk_free))
2553 memset(ptr, 0x5a, usize);
2554}
2555#ifdef JEMALLOC_JET
2556#undef arena_dalloc_junk_large
2557#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large)
2558arena_dalloc_junk_large_t *arena_dalloc_junk_large =
2559 JEMALLOC_N(arena_dalloc_junk_large_impl);
2560#endif
2561
| 2482
2483 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2484 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
2485 run = &arena_miscelm_get(chunk, rpages_ind)->run;
2486 binind = run->binind;
2487 bin = &arena->bins[binind];
2488 bin_info = &arena_bin_info[binind];
2489
2490 if (!junked && config_fill && unlikely(opt_junk_free))
2491 arena_dalloc_junk_small(ptr, bin_info);
2492
2493 arena_run_reg_dalloc(run, ptr);
2494 if (run->nfree == bin_info->nregs) {
2495 arena_dissociate_bin_run(chunk, run, bin);
2496 arena_dalloc_bin_run(arena, chunk, run, bin);
2497 } else if (run->nfree == 1 && run != bin->runcur)
2498 arena_bin_lower_run(arena, chunk, run, bin);
2499
2500 if (config_stats) {
2501 bin->stats.ndalloc++;
2502 bin->stats.curregs--;
2503 }
2504}
2505
2506void
2507arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2508 arena_chunk_map_bits_t *bitselm)
2509{
2510
2511 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true);
2512}
2513
2514void
2515arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2516 size_t pageind, arena_chunk_map_bits_t *bitselm)
2517{
2518 arena_run_t *run;
2519 arena_bin_t *bin;
2520 size_t rpages_ind;
2521
2522 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
2523 run = &arena_miscelm_get(chunk, rpages_ind)->run;
2524 bin = &arena->bins[run->binind];
2525 malloc_mutex_lock(&bin->lock);
2526 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false);
2527 malloc_mutex_unlock(&bin->lock);
2528}
2529
2530void
2531arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2532 size_t pageind)
2533{
2534 arena_chunk_map_bits_t *bitselm;
2535
2536 if (config_debug) {
2537 /* arena_ptr_small_binind_get() does extra sanity checking. */
2538 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
2539 pageind)) != BININD_INVALID);
2540 }
2541 bitselm = arena_bitselm_get(chunk, pageind);
2542 arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm);
2543}
2544
2545#ifdef JEMALLOC_JET
2546#undef arena_dalloc_junk_large
2547#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl)
2548#endif
2549void
2550arena_dalloc_junk_large(void *ptr, size_t usize)
2551{
2552
2553 if (config_fill && unlikely(opt_junk_free))
2554 memset(ptr, 0x5a, usize);
2555}
2556#ifdef JEMALLOC_JET
2557#undef arena_dalloc_junk_large
2558#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large)
2559arena_dalloc_junk_large_t *arena_dalloc_junk_large =
2560 JEMALLOC_N(arena_dalloc_junk_large_impl);
2561#endif
2562
|
2562void
| 2563static void
|
2563arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk,
2564 void *ptr, bool junked)
2565{
2566 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2567 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
2568 arena_run_t *run = &miscelm->run;
2569
2570 if (config_fill || config_stats) {
2571 size_t usize = arena_mapbits_large_size_get(chunk, pageind) -
2572 large_pad;
2573
2574 if (!junked)
2575 arena_dalloc_junk_large(ptr, usize);
2576 if (config_stats) {
| 2564arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk,
2565 void *ptr, bool junked)
2566{
2567 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2568 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
2569 arena_run_t *run = &miscelm->run;
2570
2571 if (config_fill || config_stats) {
2572 size_t usize = arena_mapbits_large_size_get(chunk, pageind) -
2573 large_pad;
2574
2575 if (!junked)
2576 arena_dalloc_junk_large(ptr, usize);
2577 if (config_stats) {
|
2577 index_t index = size2index(usize) - NBINS;
| 2578 szind_t index = size2index(usize) - NBINS;
|
2578
2579 arena->stats.ndalloc_large++;
2580 arena->stats.allocated_large -= usize;
2581 arena->stats.lstats[index].ndalloc++;
2582 arena->stats.lstats[index].curruns--;
2583 }
2584 }
2585
2586 arena_run_dalloc_decommit(arena, chunk, run);
2587}
2588
2589void
2590arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk,
2591 void *ptr)
2592{
2593
2594 arena_dalloc_large_locked_impl(arena, chunk, ptr, true);
2595}
2596
2597void
2598arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
2599{
2600
2601 malloc_mutex_lock(&arena->lock);
2602 arena_dalloc_large_locked_impl(arena, chunk, ptr, false);
2603 malloc_mutex_unlock(&arena->lock);
2604}
2605
2606static void
2607arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2608 size_t oldsize, size_t size)
2609{
2610 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2611 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
2612 arena_run_t *run = &miscelm->run;
2613
2614 assert(size < oldsize);
2615
2616 /*
2617 * Shrink the run, and make trailing pages available for other
2618 * allocations.
2619 */
2620 malloc_mutex_lock(&arena->lock);
2621 arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size +
2622 large_pad, true);
2623 if (config_stats) {
| 2579
2580 arena->stats.ndalloc_large++;
2581 arena->stats.allocated_large -= usize;
2582 arena->stats.lstats[index].ndalloc++;
2583 arena->stats.lstats[index].curruns--;
2584 }
2585 }
2586
2587 arena_run_dalloc_decommit(arena, chunk, run);
2588}
2589
2590void
2591arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk,
2592 void *ptr)
2593{
2594
2595 arena_dalloc_large_locked_impl(arena, chunk, ptr, true);
2596}
2597
2598void
2599arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
2600{
2601
2602 malloc_mutex_lock(&arena->lock);
2603 arena_dalloc_large_locked_impl(arena, chunk, ptr, false);
2604 malloc_mutex_unlock(&arena->lock);
2605}
2606
2607static void
2608arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
2609 size_t oldsize, size_t size)
2610{
2611 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2612 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
2613 arena_run_t *run = &miscelm->run;
2614
2615 assert(size < oldsize);
2616
2617 /*
2618 * Shrink the run, and make trailing pages available for other
2619 * allocations.
2620 */
2621 malloc_mutex_lock(&arena->lock);
2622 arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size +
2623 large_pad, true);
2624 if (config_stats) {
|
2624 index_t oldindex = size2index(oldsize) - NBINS;
2625 index_t index = size2index(size) - NBINS;
| 2625 szind_t oldindex = size2index(oldsize) - NBINS;
2626 szind_t index = size2index(size) - NBINS;
|
2626
2627 arena->stats.ndalloc_large++;
2628 arena->stats.allocated_large -= oldsize;
2629 arena->stats.lstats[oldindex].ndalloc++;
2630 arena->stats.lstats[oldindex].curruns--;
2631
2632 arena->stats.nmalloc_large++;
2633 arena->stats.nrequests_large++;
2634 arena->stats.allocated_large += size;
2635 arena->stats.lstats[index].nmalloc++;
2636 arena->stats.lstats[index].nrequests++;
2637 arena->stats.lstats[index].curruns++;
2638 }
2639 malloc_mutex_unlock(&arena->lock);
2640}
2641
2642static bool
2643arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
| 2627
2628 arena->stats.ndalloc_large++;
2629 arena->stats.allocated_large -= oldsize;
2630 arena->stats.lstats[oldindex].ndalloc++;
2631 arena->stats.lstats[oldindex].curruns--;
2632
2633 arena->stats.nmalloc_large++;
2634 arena->stats.nrequests_large++;
2635 arena->stats.allocated_large += size;
2636 arena->stats.lstats[index].nmalloc++;
2637 arena->stats.lstats[index].nrequests++;
2638 arena->stats.lstats[index].curruns++;
2639 }
2640 malloc_mutex_unlock(&arena->lock);
2641}
2642
2643static bool
2644arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
2644 size_t oldsize, size_t size, size_t extra, bool zero)
| 2645 size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
|
2645{
2646 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2647 size_t npages = (oldsize + large_pad) >> LG_PAGE;
2648 size_t followsize;
| 2646{
2647 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
2648 size_t npages = (oldsize + large_pad) >> LG_PAGE;
2649 size_t followsize;
|
2649 size_t usize_min = s2u(size);
| |
2650
2651 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) -
2652 large_pad);
2653
2654 /* Try to extend the run. */
| 2650
2651 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) -
2652 large_pad);
2653
2654 /* Try to extend the run. */
|
2655 assert(usize_min > oldsize);
| |
2656 malloc_mutex_lock(&arena->lock);
| 2655 malloc_mutex_lock(&arena->lock);
|
2657 if (pageind+npages < chunk_npages &&
2658 arena_mapbits_allocated_get(chunk, pageind+npages) == 0 &&
2659 (followsize = arena_mapbits_unallocated_size_get(chunk,
2660 pageind+npages)) >= usize_min - oldsize) {
| 2656 if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk,
2657 pageind+npages) != 0)
2658 goto label_fail;
2659 followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages);
2660 if (oldsize + followsize >= usize_min) {
|
2661 /*
2662 * The next run is available and sufficiently large. Split the
2663 * following run, then merge the first part with the existing
2664 * allocation.
2665 */
2666 arena_run_t *run;
| 2661 /*
2662 * The next run is available and sufficiently large. Split the
2663 * following run, then merge the first part with the existing
2664 * allocation.
2665 */
2666 arena_run_t *run;
|
2667 size_t flag_dirty, flag_unzeroed_mask, splitsize, usize;
| 2667 size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask;
|
2668
| 2668
|
2669 usize = s2u(size + extra);
| 2669 usize = usize_max;
|
2670 while (oldsize + followsize < usize)
2671 usize = index2size(size2index(usize)-1);
2672 assert(usize >= usize_min);
| 2670 while (oldsize + followsize < usize)
2671 usize = index2size(size2index(usize)-1);
2672 assert(usize >= usize_min);
|
| 2673 assert(usize >= oldsize);
|
2673 splitsize = usize - oldsize;
| 2674 splitsize = usize - oldsize;
|
| 2675 if (splitsize == 0)
2676 goto label_fail;
|
2674
2675 run = &arena_miscelm_get(chunk, pageind+npages)->run;
| 2677
2678 run = &arena_miscelm_get(chunk, pageind+npages)->run;
|
2676 if (arena_run_split_large(arena, run, splitsize, zero)) {
2677 malloc_mutex_unlock(&arena->lock);
2678 return (true);
2679 }
| 2679 if (arena_run_split_large(arena, run, splitsize, zero))
2680 goto label_fail;
|
2680
2681 size = oldsize + splitsize;
2682 npages = (size + large_pad) >> LG_PAGE;
2683
2684 /*
2685 * Mark the extended run as dirty if either portion of the run
2686 * was dirty before allocation. This is rather pedantic,
2687 * because there's not actually any sequence of events that
2688 * could cause the resulting run to be passed to
2689 * arena_run_dalloc() with the dirty argument set to false
2690 * (which is when dirty flag consistency would really matter).
2691 */
2692 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) |
2693 arena_mapbits_dirty_get(chunk, pageind+npages-1);
2694 flag_unzeroed_mask = flag_dirty == 0 ? CHUNK_MAP_UNZEROED : 0;
2695 arena_mapbits_large_set(chunk, pageind, size + large_pad,
2696 flag_dirty | (flag_unzeroed_mask &
2697 arena_mapbits_unzeroed_get(chunk, pageind)));
2698 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty |
2699 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
2700 pageind+npages-1)));
2701
2702 if (config_stats) {
| 2681
2682 size = oldsize + splitsize;
2683 npages = (size + large_pad) >> LG_PAGE;
2684
2685 /*
2686 * Mark the extended run as dirty if either portion of the run
2687 * was dirty before allocation. This is rather pedantic,
2688 * because there's not actually any sequence of events that
2689 * could cause the resulting run to be passed to
2690 * arena_run_dalloc() with the dirty argument set to false
2691 * (which is when dirty flag consistency would really matter).
2692 */
2693 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) |
2694 arena_mapbits_dirty_get(chunk, pageind+npages-1);
2695 flag_unzeroed_mask = flag_dirty == 0 ? CHUNK_MAP_UNZEROED : 0;
2696 arena_mapbits_large_set(chunk, pageind, size + large_pad,
2697 flag_dirty | (flag_unzeroed_mask &
2698 arena_mapbits_unzeroed_get(chunk, pageind)));
2699 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty |
2700 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
2701 pageind+npages-1)));
2702
2703 if (config_stats) {
|
2703 index_t oldindex = size2index(oldsize) - NBINS;
2704 index_t index = size2index(size) - NBINS;
| 2704 szind_t oldindex = size2index(oldsize) - NBINS;
2705 szind_t index = size2index(size) - NBINS;
|
2705
2706 arena->stats.ndalloc_large++;
2707 arena->stats.allocated_large -= oldsize;
2708 arena->stats.lstats[oldindex].ndalloc++;
2709 arena->stats.lstats[oldindex].curruns--;
2710
2711 arena->stats.nmalloc_large++;
2712 arena->stats.nrequests_large++;
2713 arena->stats.allocated_large += size;
2714 arena->stats.lstats[index].nmalloc++;
2715 arena->stats.lstats[index].nrequests++;
2716 arena->stats.lstats[index].curruns++;
2717 }
2718 malloc_mutex_unlock(&arena->lock);
2719 return (false);
2720 }
| 2706
2707 arena->stats.ndalloc_large++;
2708 arena->stats.allocated_large -= oldsize;
2709 arena->stats.lstats[oldindex].ndalloc++;
2710 arena->stats.lstats[oldindex].curruns--;
2711
2712 arena->stats.nmalloc_large++;
2713 arena->stats.nrequests_large++;
2714 arena->stats.allocated_large += size;
2715 arena->stats.lstats[index].nmalloc++;
2716 arena->stats.lstats[index].nrequests++;
2717 arena->stats.lstats[index].curruns++;
2718 }
2719 malloc_mutex_unlock(&arena->lock);
2720 return (false);
2721 }
|
| 2722label_fail:
|
2721 malloc_mutex_unlock(&arena->lock);
| 2723 malloc_mutex_unlock(&arena->lock);
|
2722
| |
2723 return (true);
2724}
2725
2726#ifdef JEMALLOC_JET
2727#undef arena_ralloc_junk_large
2728#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl)
2729#endif
2730static void
2731arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize)
2732{
2733
2734 if (config_fill && unlikely(opt_junk_free)) {
2735 memset((void *)((uintptr_t)ptr + usize), 0x5a,
2736 old_usize - usize);
2737 }
2738}
2739#ifdef JEMALLOC_JET
2740#undef arena_ralloc_junk_large
2741#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large)
2742arena_ralloc_junk_large_t *arena_ralloc_junk_large =
2743 JEMALLOC_N(arena_ralloc_junk_large_impl);
2744#endif
2745
2746/*
2747 * Try to resize a large allocation, in order to avoid copying. This will
2748 * always fail if growing an object, and the following run is already in use.
2749 */
2750static bool
| 2724 return (true);
2725}
2726
2727#ifdef JEMALLOC_JET
2728#undef arena_ralloc_junk_large
2729#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl)
2730#endif
2731static void
2732arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize)
2733{
2734
2735 if (config_fill && unlikely(opt_junk_free)) {
2736 memset((void *)((uintptr_t)ptr + usize), 0x5a,
2737 old_usize - usize);
2738 }
2739}
2740#ifdef JEMALLOC_JET
2741#undef arena_ralloc_junk_large
2742#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large)
2743arena_ralloc_junk_large_t *arena_ralloc_junk_large =
2744 JEMALLOC_N(arena_ralloc_junk_large_impl);
2745#endif
2746
2747/*
2748 * Try to resize a large allocation, in order to avoid copying. This will
2749 * always fail if growing an object, and the following run is already in use.
2750 */
2751static bool
|
2751arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
2752 bool zero)
| 2752arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min,
2753 size_t usize_max, bool zero)
|
2753{
| 2754{
|
2754 size_t usize;
| 2755 arena_chunk_t *chunk;
2756 arena_t *arena;
|
2755
| 2757
|
2756 /* Make sure extra can't cause size_t overflow. */
2757 if (unlikely(extra >= arena_maxclass))
2758 return (true);
2759
2760 usize = s2u(size + extra);
2761 if (usize == oldsize) {
2762 /* Same size class. */
| 2758 if (oldsize == usize_max) {
2759 /* Current size class is compatible and maximal. */
|
2763 return (false);
| 2760 return (false);
|
2764 } else {
2765 arena_chunk_t *chunk;
2766 arena_t *arena;
| 2761 }
|
2767
| 2762
|
2768 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2769 arena = extent_node_arena_get(&chunk->node);
| 2763 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
2764 arena = extent_node_arena_get(&chunk->node);
|
2770
| 2765
|
2771 if (usize < oldsize) {
2772 /* Fill before shrinking in order avoid a race. */
2773 arena_ralloc_junk_large(ptr, oldsize, usize);
2774 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize,
2775 usize);
2776 return (false);
2777 } else {
2778 bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
2779 oldsize, size, extra, zero);
2780 if (config_fill && !ret && !zero) {
2781 if (unlikely(opt_junk_alloc)) {
2782 memset((void *)((uintptr_t)ptr +
2783 oldsize), 0xa5, isalloc(ptr,
2784 config_prof) - oldsize);
2785 } else if (unlikely(opt_zero)) {
2786 memset((void *)((uintptr_t)ptr +
2787 oldsize), 0, isalloc(ptr,
2788 config_prof) - oldsize);
2789 }
| 2766 if (oldsize < usize_max) {
2767 bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize,
2768 usize_min, usize_max, zero);
2769 if (config_fill && !ret && !zero) {
2770 if (unlikely(opt_junk_alloc)) {
2771 memset((void *)((uintptr_t)ptr + oldsize), 0xa5,
2772 isalloc(ptr, config_prof) - oldsize);
2773 } else if (unlikely(opt_zero)) {
2774 memset((void *)((uintptr_t)ptr + oldsize), 0,
2775 isalloc(ptr, config_prof) - oldsize);
|
2790 }
| 2776 }
|
2791 return (ret);
| |
2792 }
| 2777 }
|
| 2778 return (ret);
|
2793 }
| 2779 }
|
| 2780
2781 assert(oldsize > usize_max);
2782 /* Fill before shrinking in order avoid a race. */
2783 arena_ralloc_junk_large(ptr, oldsize, usize_max);
2784 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, usize_max);
2785 return (false);
|
2794}
2795
2796bool
2797arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
2798 bool zero)
2799{
| 2786}
2787
2788bool
2789arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
2790 bool zero)
2791{
|
| 2792 size_t usize_min, usize_max;
|
2800
| 2793
|
2801 if (likely(size <= arena_maxclass)) {
| 2794 usize_min = s2u(size);
2795 usize_max = s2u(size + extra);
2796 if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) {
|
2802 /*
2803 * Avoid moving the allocation if the size class can be left the
2804 * same.
2805 */
| 2797 /*
2798 * Avoid moving the allocation if the size class can be left the
2799 * same.
2800 */
|
2806 if (likely(oldsize <= arena_maxclass)) {
2807 if (oldsize <= SMALL_MAXCLASS) {
2808 assert(
2809 arena_bin_info[size2index(oldsize)].reg_size
2810 == oldsize);
2811 if ((size + extra <= SMALL_MAXCLASS &&
2812 size2index(size + extra) ==
2813 size2index(oldsize)) || (size <= oldsize &&
2814 size + extra >= oldsize))
| 2801 if (oldsize <= SMALL_MAXCLASS) {
2802 assert(arena_bin_info[size2index(oldsize)].reg_size ==
2803 oldsize);
2804 if ((usize_max <= SMALL_MAXCLASS &&
2805 size2index(usize_max) == size2index(oldsize)) ||
2806 (size <= oldsize && usize_max >= oldsize))
2807 return (false);
2808 } else {
2809 if (usize_max > SMALL_MAXCLASS) {
2810 if (!arena_ralloc_large(ptr, oldsize, usize_min,
2811 usize_max, zero))
|
2815 return (false);
| 2812 return (false);
|
2816 } else {
2817 assert(size <= arena_maxclass);
2818 if (size + extra > SMALL_MAXCLASS) {
2819 if (!arena_ralloc_large(ptr, oldsize,
2820 size, extra, zero))
2821 return (false);
2822 }
| |
2823 }
2824 }
2825
2826 /* Reallocation would require a move. */
2827 return (true);
| 2813 }
2814 }
2815
2816 /* Reallocation would require a move. */
2817 return (true);
|
2828 } else
2829 return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));
| 2818 } else {
2819 return (huge_ralloc_no_move(ptr, oldsize, usize_min, usize_max,
2820 zero));
2821 }
|
2830}
2831
| 2822}
2823
|
| 2824static void *
2825arena_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize,
2826 size_t alignment, bool zero, tcache_t *tcache)
2827{
2828
2829 if (alignment == 0)
2830 return (arena_malloc(tsd, arena, usize, zero, tcache));
2831 usize = sa2u(usize, alignment);
2832 if (usize == 0)
2833 return (NULL);
2834 return (ipalloct(tsd, usize, alignment, zero, tcache, arena));
2835}
2836
|
2832void *
2833arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
| 2837void *
2838arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
|
2834 size_t extra, size_t alignment, bool zero, tcache_t *tcache)
| 2839 size_t alignment, bool zero, tcache_t *tcache)
|
2835{
2836 void *ret;
| 2840{
2841 void *ret;
|
| 2842 size_t usize;
|
2837
| 2843
|
2838 if (likely(size <= arena_maxclass)) {
| 2844 usize = s2u(size);
2845 if (usize == 0)
2846 return (NULL);
2847
2848 if (likely(usize <= large_maxclass)) {
|
2839 size_t copysize;
2840
2841 /* Try to avoid moving the allocation. */
| 2849 size_t copysize;
2850
2851 /* Try to avoid moving the allocation. */
|
2842 if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero))
| 2852 if (!arena_ralloc_no_move(ptr, oldsize, usize, 0, zero))
|
2843 return (ptr);
2844
2845 /*
2846 * size and oldsize are different enough that we need to move
2847 * the object. In that case, fall back to allocating new space
2848 * and copying.
2849 */
| 2853 return (ptr);
2854
2855 /*
2856 * size and oldsize are different enough that we need to move
2857 * the object. In that case, fall back to allocating new space
2858 * and copying.
2859 */
|
2850 if (alignment != 0) {
2851 size_t usize = sa2u(size + extra, alignment);
2852 if (usize == 0)
2853 return (NULL);
2854 ret = ipalloct(tsd, usize, alignment, zero, tcache,
2855 arena);
2856 } else {
2857 ret = arena_malloc(tsd, arena, size + extra, zero,
2858 tcache);
2859 }
| 2860 ret = arena_ralloc_move_helper(tsd, arena, usize, alignment,
2861 zero, tcache);
2862 if (ret == NULL)
2863 return (NULL);
|
2860
| 2864
|
2861 if (ret == NULL) {
2862 if (extra == 0)
2863 return (NULL);
2864 /* Try again, this time without extra. */
2865 if (alignment != 0) {
2866 size_t usize = sa2u(size, alignment);
2867 if (usize == 0)
2868 return (NULL);
2869 ret = ipalloct(tsd, usize, alignment, zero,
2870 tcache, arena);
2871 } else {
2872 ret = arena_malloc(tsd, arena, size, zero,
2873 tcache);
2874 }
2875
2876 if (ret == NULL)
2877 return (NULL);
2878 }
2879
| |
2880 /*
2881 * Junk/zero-filling were already done by
2882 * ipalloc()/arena_malloc().
2883 */
2884
| 2865 /*
2866 * Junk/zero-filling were already done by
2867 * ipalloc()/arena_malloc().
2868 */
2869
|
2885 /*
2886 * Copy at most size bytes (not size+extra), since the caller
2887 * has no expectation that the extra bytes will be reliably
2888 * preserved.
2889 */
2890 copysize = (size < oldsize) ? size : oldsize;
| 2870 copysize = (usize < oldsize) ? usize : oldsize;
|
2891 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
2892 memcpy(ret, ptr, copysize);
2893 isqalloc(tsd, ptr, oldsize, tcache);
2894 } else {
| 2871 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
2872 memcpy(ret, ptr, copysize);
2873 isqalloc(tsd, ptr, oldsize, tcache);
2874 } else {
|
2895 ret = huge_ralloc(tsd, arena, ptr, oldsize, size, extra,
2896 alignment, zero, tcache);
| 2875 ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment,
2876 zero, tcache);
|
2897 }
2898 return (ret);
2899}
2900
2901dss_prec_t
2902arena_dss_prec_get(arena_t *arena)
2903{
2904 dss_prec_t ret;
2905
2906 malloc_mutex_lock(&arena->lock);
2907 ret = arena->dss_prec;
2908 malloc_mutex_unlock(&arena->lock);
2909 return (ret);
2910}
2911
2912bool
2913arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec)
2914{
2915
2916 if (!have_dss)
2917 return (dss_prec != dss_prec_disabled);
2918 malloc_mutex_lock(&arena->lock);
2919 arena->dss_prec = dss_prec;
2920 malloc_mutex_unlock(&arena->lock);
2921 return (false);
2922}
2923
2924ssize_t
2925arena_lg_dirty_mult_default_get(void)
2926{
2927
2928 return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default));
2929}
2930
2931bool
2932arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult)
2933{
2934
2935 if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
2936 return (true);
2937 atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult);
2938 return (false);
2939}
2940
2941void
2942arena_stats_merge(arena_t *arena, const char **dss, ssize_t *lg_dirty_mult,
2943 size_t *nactive, size_t *ndirty, arena_stats_t *astats,
2944 malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats,
2945 malloc_huge_stats_t *hstats)
2946{
2947 unsigned i;
2948
2949 malloc_mutex_lock(&arena->lock);
2950 *dss = dss_prec_names[arena->dss_prec];
2951 *lg_dirty_mult = arena->lg_dirty_mult;
2952 *nactive += arena->nactive;
2953 *ndirty += arena->ndirty;
2954
2955 astats->mapped += arena->stats.mapped;
2956 astats->npurge += arena->stats.npurge;
2957 astats->nmadvise += arena->stats.nmadvise;
2958 astats->purged += arena->stats.purged;
2959 astats->metadata_mapped += arena->stats.metadata_mapped;
2960 astats->metadata_allocated += arena_metadata_allocated_get(arena);
2961 astats->allocated_large += arena->stats.allocated_large;
2962 astats->nmalloc_large += arena->stats.nmalloc_large;
2963 astats->ndalloc_large += arena->stats.ndalloc_large;
2964 astats->nrequests_large += arena->stats.nrequests_large;
2965 astats->allocated_huge += arena->stats.allocated_huge;
2966 astats->nmalloc_huge += arena->stats.nmalloc_huge;
2967 astats->ndalloc_huge += arena->stats.ndalloc_huge;
2968
2969 for (i = 0; i < nlclasses; i++) {
2970 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
2971 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
2972 lstats[i].nrequests += arena->stats.lstats[i].nrequests;
2973 lstats[i].curruns += arena->stats.lstats[i].curruns;
2974 }
2975
2976 for (i = 0; i < nhclasses; i++) {
2977 hstats[i].nmalloc += arena->stats.hstats[i].nmalloc;
2978 hstats[i].ndalloc += arena->stats.hstats[i].ndalloc;
2979 hstats[i].curhchunks += arena->stats.hstats[i].curhchunks;
2980 }
2981 malloc_mutex_unlock(&arena->lock);
2982
2983 for (i = 0; i < NBINS; i++) {
2984 arena_bin_t *bin = &arena->bins[i];
2985
2986 malloc_mutex_lock(&bin->lock);
2987 bstats[i].nmalloc += bin->stats.nmalloc;
2988 bstats[i].ndalloc += bin->stats.ndalloc;
2989 bstats[i].nrequests += bin->stats.nrequests;
2990 bstats[i].curregs += bin->stats.curregs;
2991 if (config_tcache) {
2992 bstats[i].nfills += bin->stats.nfills;
2993 bstats[i].nflushes += bin->stats.nflushes;
2994 }
2995 bstats[i].nruns += bin->stats.nruns;
2996 bstats[i].reruns += bin->stats.reruns;
2997 bstats[i].curruns += bin->stats.curruns;
2998 malloc_mutex_unlock(&bin->lock);
2999 }
3000}
3001
3002arena_t *
3003arena_new(unsigned ind)
3004{
3005 arena_t *arena;
3006 unsigned i;
3007 arena_bin_t *bin;
3008
3009 /*
3010 * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly
3011 * because there is no way to clean up if base_alloc() OOMs.
3012 */
3013 if (config_stats) {
3014 arena = (arena_t *)base_alloc(CACHELINE_CEILING(sizeof(arena_t))
3015 + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) +
3016 nhclasses) * sizeof(malloc_huge_stats_t));
3017 } else
3018 arena = (arena_t *)base_alloc(sizeof(arena_t));
3019 if (arena == NULL)
3020 return (NULL);
3021
3022 arena->ind = ind;
3023 arena->nthreads = 0;
3024 if (malloc_mutex_init(&arena->lock))
3025 return (NULL);
3026
3027 if (config_stats) {
3028 memset(&arena->stats, 0, sizeof(arena_stats_t));
3029 arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena
3030 + CACHELINE_CEILING(sizeof(arena_t)));
3031 memset(arena->stats.lstats, 0, nlclasses *
3032 sizeof(malloc_large_stats_t));
3033 arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena
3034 + CACHELINE_CEILING(sizeof(arena_t)) +
3035 QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t)));
3036 memset(arena->stats.hstats, 0, nhclasses *
3037 sizeof(malloc_huge_stats_t));
3038 if (config_tcache)
3039 ql_new(&arena->tcache_ql);
3040 }
3041
3042 if (config_prof)
3043 arena->prof_accumbytes = 0;
3044
3045 if (config_cache_oblivious) {
3046 /*
3047 * A nondeterministic seed based on the address of arena reduces
3048 * the likelihood of lockstep non-uniform cache index
3049 * utilization among identical concurrent processes, but at the
3050 * cost of test repeatability. For debug builds, instead use a
3051 * deterministic seed.
3052 */
3053 arena->offset_state = config_debug ? ind :
3054 (uint64_t)(uintptr_t)arena;
3055 }
3056
3057 arena->dss_prec = chunk_dss_prec_get();
3058
3059 arena->spare = NULL;
3060
3061 arena->lg_dirty_mult = arena_lg_dirty_mult_default_get();
3062 arena->purging = false;
3063 arena->nactive = 0;
3064 arena->ndirty = 0;
3065
3066 arena_avail_tree_new(&arena->runs_avail);
3067 qr_new(&arena->runs_dirty, rd_link);
3068 qr_new(&arena->chunks_cache, cc_link);
3069
3070 ql_new(&arena->huge);
3071 if (malloc_mutex_init(&arena->huge_mtx))
3072 return (NULL);
3073
3074 extent_tree_szad_new(&arena->chunks_szad_cached);
3075 extent_tree_ad_new(&arena->chunks_ad_cached);
3076 extent_tree_szad_new(&arena->chunks_szad_retained);
3077 extent_tree_ad_new(&arena->chunks_ad_retained);
3078 if (malloc_mutex_init(&arena->chunks_mtx))
3079 return (NULL);
3080 ql_new(&arena->node_cache);
3081 if (malloc_mutex_init(&arena->node_cache_mtx))
3082 return (NULL);
3083
3084 arena->chunk_hooks = chunk_hooks_default;
3085
3086 /* Initialize bins. */
3087 for (i = 0; i < NBINS; i++) {
3088 bin = &arena->bins[i];
3089 if (malloc_mutex_init(&bin->lock))
3090 return (NULL);
3091 bin->runcur = NULL;
3092 arena_run_tree_new(&bin->runs);
3093 if (config_stats)
3094 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
3095 }
3096
3097 return (arena);
3098}
3099
3100/*
3101 * Calculate bin_info->run_size such that it meets the following constraints:
3102 *
3103 * *) bin_info->run_size <= arena_maxrun
3104 * *) bin_info->nregs <= RUN_MAXREGS
3105 *
3106 * bin_info->nregs and bin_info->reg0_offset are also calculated here, since
3107 * these settings are all interdependent.
3108 */
3109static void
3110bin_info_run_size_calc(arena_bin_info_t *bin_info)
3111{
3112 size_t pad_size;
3113 size_t try_run_size, perfect_run_size, actual_run_size;
3114 uint32_t try_nregs, perfect_nregs, actual_nregs;
3115
3116 /*
3117 * Determine redzone size based on minimum alignment and minimum
3118 * redzone size. Add padding to the end of the run if it is needed to
3119 * align the regions. The padding allows each redzone to be half the
3120 * minimum alignment; without the padding, each redzone would have to
3121 * be twice as large in order to maintain alignment.
3122 */
3123 if (config_fill && unlikely(opt_redzone)) {
3124 size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) -
3125 1);
3126 if (align_min <= REDZONE_MINSIZE) {
3127 bin_info->redzone_size = REDZONE_MINSIZE;
3128 pad_size = 0;
3129 } else {
3130 bin_info->redzone_size = align_min >> 1;
3131 pad_size = bin_info->redzone_size;
3132 }
3133 } else {
3134 bin_info->redzone_size = 0;
3135 pad_size = 0;
3136 }
3137 bin_info->reg_interval = bin_info->reg_size +
3138 (bin_info->redzone_size << 1);
3139
3140 /*
3141 * Compute run size under ideal conditions (no redzones, no limit on run
3142 * size).
3143 */
3144 try_run_size = PAGE;
3145 try_nregs = try_run_size / bin_info->reg_size;
3146 do {
3147 perfect_run_size = try_run_size;
3148 perfect_nregs = try_nregs;
3149
3150 try_run_size += PAGE;
3151 try_nregs = try_run_size / bin_info->reg_size;
3152 } while (perfect_run_size != perfect_nregs * bin_info->reg_size);
3153 assert(perfect_nregs <= RUN_MAXREGS);
3154
3155 actual_run_size = perfect_run_size;
3156 actual_nregs = (actual_run_size - pad_size) / bin_info->reg_interval;
3157
3158 /*
3159 * Redzones can require enough padding that not even a single region can
3160 * fit within the number of pages that would normally be dedicated to a
3161 * run for this size class. Increase the run size until at least one
3162 * region fits.
3163 */
3164 while (actual_nregs == 0) {
3165 assert(config_fill && unlikely(opt_redzone));
3166
3167 actual_run_size += PAGE;
3168 actual_nregs = (actual_run_size - pad_size) /
3169 bin_info->reg_interval;
3170 }
3171
3172 /*
3173 * Make sure that the run will fit within an arena chunk.
3174 */
3175 while (actual_run_size > arena_maxrun) {
3176 actual_run_size -= PAGE;
3177 actual_nregs = (actual_run_size - pad_size) /
3178 bin_info->reg_interval;
3179 }
3180 assert(actual_nregs > 0);
3181 assert(actual_run_size == s2u(actual_run_size));
3182
3183 /* Copy final settings. */
3184 bin_info->run_size = actual_run_size;
3185 bin_info->nregs = actual_nregs;
3186 bin_info->reg0_offset = actual_run_size - (actual_nregs *
3187 bin_info->reg_interval) - pad_size + bin_info->redzone_size;
3188
3189 if (actual_run_size > small_maxrun)
3190 small_maxrun = actual_run_size;
3191
3192 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs
3193 * bin_info->reg_interval) + pad_size == bin_info->run_size);
3194}
3195
3196static void
3197bin_info_init(void)
3198{
3199 arena_bin_info_t *bin_info;
3200
3201#define BIN_INFO_INIT_bin_yes(index, size) \
3202 bin_info = &arena_bin_info[index]; \
3203 bin_info->reg_size = size; \
3204 bin_info_run_size_calc(bin_info); \
3205 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs);
3206#define BIN_INFO_INIT_bin_no(index, size)
3207#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
3208 BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
3209 SIZE_CLASSES
3210#undef BIN_INFO_INIT_bin_yes
3211#undef BIN_INFO_INIT_bin_no
3212#undef SC
3213}
3214
3215static bool
3216small_run_size_init(void)
3217{
3218
3219 assert(small_maxrun != 0);
3220
3221 small_run_tab = (bool *)base_alloc(sizeof(bool) * (small_maxrun >>
3222 LG_PAGE));
3223 if (small_run_tab == NULL)
3224 return (true);
3225
3226#define TAB_INIT_bin_yes(index, size) { \
3227 arena_bin_info_t *bin_info = &arena_bin_info[index]; \
3228 small_run_tab[bin_info->run_size >> LG_PAGE] = true; \
3229 }
3230#define TAB_INIT_bin_no(index, size)
3231#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
3232 TAB_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
3233 SIZE_CLASSES
3234#undef TAB_INIT_bin_yes
3235#undef TAB_INIT_bin_no
3236#undef SC
3237
3238 return (false);
3239}
3240
3241bool
3242arena_boot(void)
3243{
| 2877 }
2878 return (ret);
2879}
2880
2881dss_prec_t
2882arena_dss_prec_get(arena_t *arena)
2883{
2884 dss_prec_t ret;
2885
2886 malloc_mutex_lock(&arena->lock);
2887 ret = arena->dss_prec;
2888 malloc_mutex_unlock(&arena->lock);
2889 return (ret);
2890}
2891
2892bool
2893arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec)
2894{
2895
2896 if (!have_dss)
2897 return (dss_prec != dss_prec_disabled);
2898 malloc_mutex_lock(&arena->lock);
2899 arena->dss_prec = dss_prec;
2900 malloc_mutex_unlock(&arena->lock);
2901 return (false);
2902}
2903
2904ssize_t
2905arena_lg_dirty_mult_default_get(void)
2906{
2907
2908 return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default));
2909}
2910
2911bool
2912arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult)
2913{
2914
2915 if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
2916 return (true);
2917 atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult);
2918 return (false);
2919}
2920
2921void
2922arena_stats_merge(arena_t *arena, const char **dss, ssize_t *lg_dirty_mult,
2923 size_t *nactive, size_t *ndirty, arena_stats_t *astats,
2924 malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats,
2925 malloc_huge_stats_t *hstats)
2926{
2927 unsigned i;
2928
2929 malloc_mutex_lock(&arena->lock);
2930 *dss = dss_prec_names[arena->dss_prec];
2931 *lg_dirty_mult = arena->lg_dirty_mult;
2932 *nactive += arena->nactive;
2933 *ndirty += arena->ndirty;
2934
2935 astats->mapped += arena->stats.mapped;
2936 astats->npurge += arena->stats.npurge;
2937 astats->nmadvise += arena->stats.nmadvise;
2938 astats->purged += arena->stats.purged;
2939 astats->metadata_mapped += arena->stats.metadata_mapped;
2940 astats->metadata_allocated += arena_metadata_allocated_get(arena);
2941 astats->allocated_large += arena->stats.allocated_large;
2942 astats->nmalloc_large += arena->stats.nmalloc_large;
2943 astats->ndalloc_large += arena->stats.ndalloc_large;
2944 astats->nrequests_large += arena->stats.nrequests_large;
2945 astats->allocated_huge += arena->stats.allocated_huge;
2946 astats->nmalloc_huge += arena->stats.nmalloc_huge;
2947 astats->ndalloc_huge += arena->stats.ndalloc_huge;
2948
2949 for (i = 0; i < nlclasses; i++) {
2950 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
2951 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
2952 lstats[i].nrequests += arena->stats.lstats[i].nrequests;
2953 lstats[i].curruns += arena->stats.lstats[i].curruns;
2954 }
2955
2956 for (i = 0; i < nhclasses; i++) {
2957 hstats[i].nmalloc += arena->stats.hstats[i].nmalloc;
2958 hstats[i].ndalloc += arena->stats.hstats[i].ndalloc;
2959 hstats[i].curhchunks += arena->stats.hstats[i].curhchunks;
2960 }
2961 malloc_mutex_unlock(&arena->lock);
2962
2963 for (i = 0; i < NBINS; i++) {
2964 arena_bin_t *bin = &arena->bins[i];
2965
2966 malloc_mutex_lock(&bin->lock);
2967 bstats[i].nmalloc += bin->stats.nmalloc;
2968 bstats[i].ndalloc += bin->stats.ndalloc;
2969 bstats[i].nrequests += bin->stats.nrequests;
2970 bstats[i].curregs += bin->stats.curregs;
2971 if (config_tcache) {
2972 bstats[i].nfills += bin->stats.nfills;
2973 bstats[i].nflushes += bin->stats.nflushes;
2974 }
2975 bstats[i].nruns += bin->stats.nruns;
2976 bstats[i].reruns += bin->stats.reruns;
2977 bstats[i].curruns += bin->stats.curruns;
2978 malloc_mutex_unlock(&bin->lock);
2979 }
2980}
2981
2982arena_t *
2983arena_new(unsigned ind)
2984{
2985 arena_t *arena;
2986 unsigned i;
2987 arena_bin_t *bin;
2988
2989 /*
2990 * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly
2991 * because there is no way to clean up if base_alloc() OOMs.
2992 */
2993 if (config_stats) {
2994 arena = (arena_t *)base_alloc(CACHELINE_CEILING(sizeof(arena_t))
2995 + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) +
2996 nhclasses) * sizeof(malloc_huge_stats_t));
2997 } else
2998 arena = (arena_t *)base_alloc(sizeof(arena_t));
2999 if (arena == NULL)
3000 return (NULL);
3001
3002 arena->ind = ind;
3003 arena->nthreads = 0;
3004 if (malloc_mutex_init(&arena->lock))
3005 return (NULL);
3006
3007 if (config_stats) {
3008 memset(&arena->stats, 0, sizeof(arena_stats_t));
3009 arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena
3010 + CACHELINE_CEILING(sizeof(arena_t)));
3011 memset(arena->stats.lstats, 0, nlclasses *
3012 sizeof(malloc_large_stats_t));
3013 arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena
3014 + CACHELINE_CEILING(sizeof(arena_t)) +
3015 QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t)));
3016 memset(arena->stats.hstats, 0, nhclasses *
3017 sizeof(malloc_huge_stats_t));
3018 if (config_tcache)
3019 ql_new(&arena->tcache_ql);
3020 }
3021
3022 if (config_prof)
3023 arena->prof_accumbytes = 0;
3024
3025 if (config_cache_oblivious) {
3026 /*
3027 * A nondeterministic seed based on the address of arena reduces
3028 * the likelihood of lockstep non-uniform cache index
3029 * utilization among identical concurrent processes, but at the
3030 * cost of test repeatability. For debug builds, instead use a
3031 * deterministic seed.
3032 */
3033 arena->offset_state = config_debug ? ind :
3034 (uint64_t)(uintptr_t)arena;
3035 }
3036
3037 arena->dss_prec = chunk_dss_prec_get();
3038
3039 arena->spare = NULL;
3040
3041 arena->lg_dirty_mult = arena_lg_dirty_mult_default_get();
3042 arena->purging = false;
3043 arena->nactive = 0;
3044 arena->ndirty = 0;
3045
3046 arena_avail_tree_new(&arena->runs_avail);
3047 qr_new(&arena->runs_dirty, rd_link);
3048 qr_new(&arena->chunks_cache, cc_link);
3049
3050 ql_new(&arena->huge);
3051 if (malloc_mutex_init(&arena->huge_mtx))
3052 return (NULL);
3053
3054 extent_tree_szad_new(&arena->chunks_szad_cached);
3055 extent_tree_ad_new(&arena->chunks_ad_cached);
3056 extent_tree_szad_new(&arena->chunks_szad_retained);
3057 extent_tree_ad_new(&arena->chunks_ad_retained);
3058 if (malloc_mutex_init(&arena->chunks_mtx))
3059 return (NULL);
3060 ql_new(&arena->node_cache);
3061 if (malloc_mutex_init(&arena->node_cache_mtx))
3062 return (NULL);
3063
3064 arena->chunk_hooks = chunk_hooks_default;
3065
3066 /* Initialize bins. */
3067 for (i = 0; i < NBINS; i++) {
3068 bin = &arena->bins[i];
3069 if (malloc_mutex_init(&bin->lock))
3070 return (NULL);
3071 bin->runcur = NULL;
3072 arena_run_tree_new(&bin->runs);
3073 if (config_stats)
3074 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
3075 }
3076
3077 return (arena);
3078}
3079
3080/*
3081 * Calculate bin_info->run_size such that it meets the following constraints:
3082 *
3083 * *) bin_info->run_size <= arena_maxrun
3084 * *) bin_info->nregs <= RUN_MAXREGS
3085 *
3086 * bin_info->nregs and bin_info->reg0_offset are also calculated here, since
3087 * these settings are all interdependent.
3088 */
3089static void
3090bin_info_run_size_calc(arena_bin_info_t *bin_info)
3091{
3092 size_t pad_size;
3093 size_t try_run_size, perfect_run_size, actual_run_size;
3094 uint32_t try_nregs, perfect_nregs, actual_nregs;
3095
3096 /*
3097 * Determine redzone size based on minimum alignment and minimum
3098 * redzone size. Add padding to the end of the run if it is needed to
3099 * align the regions. The padding allows each redzone to be half the
3100 * minimum alignment; without the padding, each redzone would have to
3101 * be twice as large in order to maintain alignment.
3102 */
3103 if (config_fill && unlikely(opt_redzone)) {
3104 size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) -
3105 1);
3106 if (align_min <= REDZONE_MINSIZE) {
3107 bin_info->redzone_size = REDZONE_MINSIZE;
3108 pad_size = 0;
3109 } else {
3110 bin_info->redzone_size = align_min >> 1;
3111 pad_size = bin_info->redzone_size;
3112 }
3113 } else {
3114 bin_info->redzone_size = 0;
3115 pad_size = 0;
3116 }
3117 bin_info->reg_interval = bin_info->reg_size +
3118 (bin_info->redzone_size << 1);
3119
3120 /*
3121 * Compute run size under ideal conditions (no redzones, no limit on run
3122 * size).
3123 */
3124 try_run_size = PAGE;
3125 try_nregs = try_run_size / bin_info->reg_size;
3126 do {
3127 perfect_run_size = try_run_size;
3128 perfect_nregs = try_nregs;
3129
3130 try_run_size += PAGE;
3131 try_nregs = try_run_size / bin_info->reg_size;
3132 } while (perfect_run_size != perfect_nregs * bin_info->reg_size);
3133 assert(perfect_nregs <= RUN_MAXREGS);
3134
3135 actual_run_size = perfect_run_size;
3136 actual_nregs = (actual_run_size - pad_size) / bin_info->reg_interval;
3137
3138 /*
3139 * Redzones can require enough padding that not even a single region can
3140 * fit within the number of pages that would normally be dedicated to a
3141 * run for this size class. Increase the run size until at least one
3142 * region fits.
3143 */
3144 while (actual_nregs == 0) {
3145 assert(config_fill && unlikely(opt_redzone));
3146
3147 actual_run_size += PAGE;
3148 actual_nregs = (actual_run_size - pad_size) /
3149 bin_info->reg_interval;
3150 }
3151
3152 /*
3153 * Make sure that the run will fit within an arena chunk.
3154 */
3155 while (actual_run_size > arena_maxrun) {
3156 actual_run_size -= PAGE;
3157 actual_nregs = (actual_run_size - pad_size) /
3158 bin_info->reg_interval;
3159 }
3160 assert(actual_nregs > 0);
3161 assert(actual_run_size == s2u(actual_run_size));
3162
3163 /* Copy final settings. */
3164 bin_info->run_size = actual_run_size;
3165 bin_info->nregs = actual_nregs;
3166 bin_info->reg0_offset = actual_run_size - (actual_nregs *
3167 bin_info->reg_interval) - pad_size + bin_info->redzone_size;
3168
3169 if (actual_run_size > small_maxrun)
3170 small_maxrun = actual_run_size;
3171
3172 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs
3173 * bin_info->reg_interval) + pad_size == bin_info->run_size);
3174}
3175
3176static void
3177bin_info_init(void)
3178{
3179 arena_bin_info_t *bin_info;
3180
3181#define BIN_INFO_INIT_bin_yes(index, size) \
3182 bin_info = &arena_bin_info[index]; \
3183 bin_info->reg_size = size; \
3184 bin_info_run_size_calc(bin_info); \
3185 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs);
3186#define BIN_INFO_INIT_bin_no(index, size)
3187#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
3188 BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
3189 SIZE_CLASSES
3190#undef BIN_INFO_INIT_bin_yes
3191#undef BIN_INFO_INIT_bin_no
3192#undef SC
3193}
3194
3195static bool
3196small_run_size_init(void)
3197{
3198
3199 assert(small_maxrun != 0);
3200
3201 small_run_tab = (bool *)base_alloc(sizeof(bool) * (small_maxrun >>
3202 LG_PAGE));
3203 if (small_run_tab == NULL)
3204 return (true);
3205
3206#define TAB_INIT_bin_yes(index, size) { \
3207 arena_bin_info_t *bin_info = &arena_bin_info[index]; \
3208 small_run_tab[bin_info->run_size >> LG_PAGE] = true; \
3209 }
3210#define TAB_INIT_bin_no(index, size)
3211#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
3212 TAB_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
3213 SIZE_CLASSES
3214#undef TAB_INIT_bin_yes
3215#undef TAB_INIT_bin_no
3216#undef SC
3217
3218 return (false);
3219}
3220
3221bool
3222arena_boot(void)
3223{
|
3244 size_t header_size;
| |
3245 unsigned i;
3246
3247 arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
3248
3249 /*
3250 * Compute the header size such that it is large enough to contain the
3251 * page map. The page map is biased to omit entries for the header
3252 * itself, so some iteration is necessary to compute the map bias.
3253 *
3254 * 1) Compute safe header_size and map_bias values that include enough
3255 * space for an unbiased page map.
3256 * 2) Refine map_bias based on (1) to omit the header pages in the page
3257 * map. The resulting map_bias may be one too small.
3258 * 3) Refine map_bias based on (2). The result will be >= the result
3259 * from (2), and will always be correct.
3260 */
3261 map_bias = 0;
3262 for (i = 0; i < 3; i++) {
| 3224 unsigned i;
3225
3226 arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
3227
3228 /*
3229 * Compute the header size such that it is large enough to contain the
3230 * page map. The page map is biased to omit entries for the header
3231 * itself, so some iteration is necessary to compute the map bias.
3232 *
3233 * 1) Compute safe header_size and map_bias values that include enough
3234 * space for an unbiased page map.
3235 * 2) Refine map_bias based on (1) to omit the header pages in the page
3236 * map. The resulting map_bias may be one too small.
3237 * 3) Refine map_bias based on (2). The result will be >= the result
3238 * from (2), and will always be correct.
3239 */
3240 map_bias = 0;
3241 for (i = 0; i < 3; i++) {
|
3263 header_size = offsetof(arena_chunk_t, map_bits) +
| 3242 size_t header_size = offsetof(arena_chunk_t, map_bits) +
|
3264 ((sizeof(arena_chunk_map_bits_t) +
3265 sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias));
3266 map_bias = (header_size + PAGE_MASK) >> LG_PAGE;
3267 }
3268 assert(map_bias > 0);
3269
3270 map_misc_offset = offsetof(arena_chunk_t, map_bits) +
3271 sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias);
3272
3273 arena_maxrun = chunksize - (map_bias << LG_PAGE);
3274 assert(arena_maxrun > 0);
| 3243 ((sizeof(arena_chunk_map_bits_t) +
3244 sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias));
3245 map_bias = (header_size + PAGE_MASK) >> LG_PAGE;
3246 }
3247 assert(map_bias > 0);
3248
3249 map_misc_offset = offsetof(arena_chunk_t, map_bits) +
3250 sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias);
3251
3252 arena_maxrun = chunksize - (map_bias << LG_PAGE);
3253 assert(arena_maxrun > 0);
|
3275 arena_maxclass = index2size(size2index(chunksize)-1);
3276 if (arena_maxclass > arena_maxrun) {
| 3254 large_maxclass = index2size(size2index(chunksize)-1);
3255 if (large_maxclass > arena_maxrun) {
|
3277 /*
3278 * For small chunk sizes it's possible for there to be fewer
3279 * non-header pages available than are necessary to serve the
3280 * size classes just below chunksize.
3281 */
| 3256 /*
3257 * For small chunk sizes it's possible for there to be fewer
3258 * non-header pages available than are necessary to serve the
3259 * size classes just below chunksize.
3260 */
|
3282 arena_maxclass = arena_maxrun;
| 3261 large_maxclass = arena_maxrun;
|
3283 }
| 3262 }
|
3284 assert(arena_maxclass > 0);
3285 nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS);
| 3263 assert(large_maxclass > 0);
3264 nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS);
|
3286 nhclasses = NSIZES - nlclasses - NBINS;
3287
3288 bin_info_init();
3289 return (small_run_size_init());
3290}
3291
3292void
3293arena_prefork(arena_t *arena)
3294{
3295 unsigned i;
3296
3297 malloc_mutex_prefork(&arena->lock);
3298 malloc_mutex_prefork(&arena->huge_mtx);
3299 malloc_mutex_prefork(&arena->chunks_mtx);
3300 malloc_mutex_prefork(&arena->node_cache_mtx);
3301 for (i = 0; i < NBINS; i++)
3302 malloc_mutex_prefork(&arena->bins[i].lock);
3303}
3304
3305void
3306arena_postfork_parent(arena_t *arena)
3307{
3308 unsigned i;
3309
3310 for (i = 0; i < NBINS; i++)
3311 malloc_mutex_postfork_parent(&arena->bins[i].lock);
3312 malloc_mutex_postfork_parent(&arena->node_cache_mtx);
3313 malloc_mutex_postfork_parent(&arena->chunks_mtx);
3314 malloc_mutex_postfork_parent(&arena->huge_mtx);
3315 malloc_mutex_postfork_parent(&arena->lock);
3316}
3317
3318void
3319arena_postfork_child(arena_t *arena)
3320{
3321 unsigned i;
3322
3323 for (i = 0; i < NBINS; i++)
3324 malloc_mutex_postfork_child(&arena->bins[i].lock);
3325 malloc_mutex_postfork_child(&arena->node_cache_mtx);
3326 malloc_mutex_postfork_child(&arena->chunks_mtx);
3327 malloc_mutex_postfork_child(&arena->huge_mtx);
3328 malloc_mutex_postfork_child(&arena->lock);
3329}
| 3265 nhclasses = NSIZES - nlclasses - NBINS;
3266
3267 bin_info_init();
3268 return (small_run_size_init());
3269}
3270
3271void
3272arena_prefork(arena_t *arena)
3273{
3274 unsigned i;
3275
3276 malloc_mutex_prefork(&arena->lock);
3277 malloc_mutex_prefork(&arena->huge_mtx);
3278 malloc_mutex_prefork(&arena->chunks_mtx);
3279 malloc_mutex_prefork(&arena->node_cache_mtx);
3280 for (i = 0; i < NBINS; i++)
3281 malloc_mutex_prefork(&arena->bins[i].lock);
3282}
3283
3284void
3285arena_postfork_parent(arena_t *arena)
3286{
3287 unsigned i;
3288
3289 for (i = 0; i < NBINS; i++)
3290 malloc_mutex_postfork_parent(&arena->bins[i].lock);
3291 malloc_mutex_postfork_parent(&arena->node_cache_mtx);
3292 malloc_mutex_postfork_parent(&arena->chunks_mtx);
3293 malloc_mutex_postfork_parent(&arena->huge_mtx);
3294 malloc_mutex_postfork_parent(&arena->lock);
3295}
3296
3297void
3298arena_postfork_child(arena_t *arena)
3299{
3300 unsigned i;
3301
3302 for (i = 0; i < NBINS; i++)
3303 malloc_mutex_postfork_child(&arena->bins[i].lock);
3304 malloc_mutex_postfork_child(&arena->node_cache_mtx);
3305 malloc_mutex_postfork_child(&arena->chunks_mtx);
3306 malloc_mutex_postfork_child(&arena->huge_mtx);
3307 malloc_mutex_postfork_child(&arena->lock);
3308}
|