1#define JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9 extent_node_t *node;
10
11 node = chunk_lookup(ptr, true);
12 assert(!extent_node_achunk_get(node));
13
14 return (node);
15}
16
17static bool
18huge_node_set(const void *ptr, extent_node_t *node)
19{
20
21 assert(extent_node_addr_get(node) == ptr);
22 assert(!extent_node_achunk_get(node));
23 return (chunk_register(ptr, node));
24}
25
26static void
27huge_node_unset(const void *ptr, const extent_node_t *node)
28{
29
30 chunk_deregister(ptr, node);
31}
32
33void *
34huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
35 tcache_t *tcache)
36{
37 size_t usize;
38
39 usize = s2u(size);
40 if (usize == 0) {
41 /* size_t overflow. */
42 return (NULL);
43 }
44
45 return (huge_palloc(tsd, arena, usize, chunksize, zero, tcache));
46}
47
48void *
49huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
50 bool zero, tcache_t *tcache)
51{
52 void *ret;
53 size_t usize;
54 extent_node_t *node;
55 bool is_zeroed;
56
57 /* Allocate one or more contiguous chunks for this request. */
58
59 usize = sa2u(size, alignment);
60 if (unlikely(usize == 0))
61 return (NULL);
62 assert(usize >= chunksize);
63
64 /* Allocate an extent node with which to track the chunk. */
65 node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
66 CACHELINE, false, tcache, true, arena);
67 if (node == NULL)
68 return (NULL);
69
70 /*
71 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
72 * it is possible to make correct junk/zero fill decisions below.
73 */
74 is_zeroed = zero;
75 arena = arena_choose(tsd, arena);
76 if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena,
77 size, alignment, &is_zeroed)) == NULL) {
78 idalloctm(tsd, node, tcache, true);
79 return (NULL);
80 }
81
82 extent_node_init(node, arena, ret, size, is_zeroed, true);
83
84 if (huge_node_set(ret, node)) {
85 arena_chunk_dalloc_huge(arena, ret, size);
86 idalloctm(tsd, node, tcache, true);
87 return (NULL);
88 }
89
90 /* Insert node into huge. */
91 malloc_mutex_lock(&arena->huge_mtx);
92 ql_elm_new(node, ql_link);
93 ql_tail_insert(&arena->huge, node, ql_link);
94 malloc_mutex_unlock(&arena->huge_mtx);
95
96 if (zero || (config_fill && unlikely(opt_zero))) {
97 if (!is_zeroed)
98 memset(ret, 0, size);
99 } else if (config_fill && unlikely(opt_junk_alloc))
100 memset(ret, 0xa5, size);
101
102 return (ret);
103}
104
105#ifdef JEMALLOC_JET
106#undef huge_dalloc_junk
107#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
108#endif
109static void
110huge_dalloc_junk(void *ptr, size_t usize)
111{
112
113 if (config_fill && have_dss && unlikely(opt_junk_free)) {
114 /*
115 * Only bother junk filling if the chunk isn't about to be
116 * unmapped.
117 */
118 if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
119 memset(ptr, 0x5a, usize);
120 }
121}
122#ifdef JEMALLOC_JET
123#undef huge_dalloc_junk
124#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
125huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
126#endif
127
128static void
129huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
130 size_t usize_max, bool zero)
131{
132 size_t usize, usize_next;
133 extent_node_t *node;
134 arena_t *arena;
135 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
136 bool zeroed;
137
138 /* Increase usize to incorporate extra. */
139 for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
140 <= oldsize; usize = usize_next)
141 ; /* Do nothing. */
142
143 if (oldsize == usize)
144 return;
145
146 node = huge_node_get(ptr);
147 arena = extent_node_arena_get(node);
148
149 /* Fill if necessary (shrinking). */
150 if (oldsize > usize) {
151 size_t sdiff = oldsize - usize;
152 if (config_fill && unlikely(opt_junk_free)) {
153 memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff);
154 zeroed = false;
155 } else {
156 zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr,
157 CHUNK_CEILING(oldsize), usize, sdiff);
158 }
159 } else
160 zeroed = true;
161
162 malloc_mutex_lock(&arena->huge_mtx);
163 /* Update the size of the huge allocation. */
164 assert(extent_node_size_get(node) != usize);
165 extent_node_size_set(node, usize);
166 /* Clear node's zeroed field if zeroing failed above. */
167 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed);
168 malloc_mutex_unlock(&arena->huge_mtx);
169
170 arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
171
172 /* Fill if necessary (growing). */
173 if (oldsize < usize) {
174 if (zero || (config_fill && unlikely(opt_zero))) {
175 if (!zeroed) {
176 memset((void *)((uintptr_t)ptr + oldsize), 0,
177 usize - oldsize);
178 }
179 } else if (config_fill && unlikely(opt_junk_alloc)) {
180 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
181 oldsize);
182 }
183 }
184}
185
186static bool
187huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
188{
189 extent_node_t *node;
190 arena_t *arena;
191 chunk_hooks_t chunk_hooks;
192 size_t cdiff;
193 bool zeroed;
194
195 node = huge_node_get(ptr);
196 arena = extent_node_arena_get(node);
197 chunk_hooks = chunk_hooks_get(arena);
198
199 assert(oldsize > usize);
200
201 /* Split excess chunks. */
202 cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
203 if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
204 CHUNK_CEILING(usize), cdiff, true, arena->ind))
205 return (true);
206
207 if (oldsize > usize) {
208 size_t sdiff = oldsize - usize;
209 if (config_fill && unlikely(opt_junk_free)) {
210 huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
211 sdiff);
212 zeroed = false;
213 } else {
214 zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
215 CHUNK_ADDR2BASE((uintptr_t)ptr + usize),
216 CHUNK_CEILING(oldsize),
217 CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
218 }
219 } else
220 zeroed = true;
221
222 malloc_mutex_lock(&arena->huge_mtx);
223 /* Update the size of the huge allocation. */
224 extent_node_size_set(node, usize);
225 /* Clear node's zeroed field if zeroing failed above. */
226 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed);
227 malloc_mutex_unlock(&arena->huge_mtx);
228
229 /* Zap the excess chunks. */
230 arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize);
231
232 return (false);
233}
234
235static bool
236huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) {
237 extent_node_t *node;
238 arena_t *arena;
239 bool is_zeroed_subchunk, is_zeroed_chunk;
240
241 node = huge_node_get(ptr);
242 arena = extent_node_arena_get(node);
243 malloc_mutex_lock(&arena->huge_mtx);
244 is_zeroed_subchunk = extent_node_zeroed_get(node);
245 malloc_mutex_unlock(&arena->huge_mtx);
246
247 /*
248 * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so
249 * that it is possible to make correct junk/zero fill decisions below.
250 */
251 is_zeroed_chunk = zero;
252
253 if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize,
254 &is_zeroed_chunk))
255 return (true);
256
257 malloc_mutex_lock(&arena->huge_mtx);
258 /* Update the size of the huge allocation. */
259 extent_node_size_set(node, usize);
260 malloc_mutex_unlock(&arena->huge_mtx);
261
262 if (zero || (config_fill && unlikely(opt_zero))) {
263 if (!is_zeroed_subchunk) {
264 memset((void *)((uintptr_t)ptr + oldsize), 0,
265 CHUNK_CEILING(oldsize) - oldsize);
266 }
267 if (!is_zeroed_chunk) {
268 memset((void *)((uintptr_t)ptr +
269 CHUNK_CEILING(oldsize)), 0, usize -
270 CHUNK_CEILING(oldsize));
271 }
272 } else if (config_fill && unlikely(opt_junk_alloc)) {
273 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
274 oldsize);
275 }
276
277 return (false);
278}
279
280bool
281huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min,
282 size_t usize_max, bool zero)
283{
284
285 assert(s2u(oldsize) == oldsize);
286
287 /* Both allocations must be huge to avoid a move. */
288 if (oldsize < chunksize || usize_max < chunksize)
289 return (true);
290
291 if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
292 /* Attempt to expand the allocation in-place. */
293 if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max, zero))
294 return (false);
295 /* Try again, this time with usize_min. */
296 if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
297 CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr,
298 oldsize, usize_min, zero))
299 return (false);
300 }
301
302 /*
303 * Avoid moving the allocation if the existing chunk size accommodates
304 * the new size.
305 */
306 if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
307 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
308 huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max,
309 zero);
310 return (false);
311 }
312
313 /* Attempt to shrink the allocation in-place. */
314 if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max))
315 return (huge_ralloc_no_move_shrink(ptr, oldsize, usize_max));
316 return (true);
317}
318
319static void *
320huge_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize,
321 size_t alignment, bool zero, tcache_t *tcache)
322{
323
324 if (alignment <= chunksize)
325 return (huge_malloc(tsd, arena, usize, zero, tcache));
326 return (huge_palloc(tsd, arena, usize, alignment, zero, tcache));
327}
328
329void *
330huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize,
331 size_t alignment, bool zero, tcache_t *tcache)
332{
333 void *ret;
334 size_t copysize;
335
336 /* Try to avoid moving the allocation. */
337 if (!huge_ralloc_no_move(ptr, oldsize, usize, usize, zero))
338 return (ptr);
339
340 /*
341 * usize and oldsize are different enough that we need to use a
342 * different size class. In that case, fall back to allocating new
343 * space and copying.
344 */
345 ret = huge_ralloc_move_helper(tsd, arena, usize, alignment, zero,
346 tcache);
347 if (ret == NULL)
348 return (NULL);
349
350 copysize = (usize < oldsize) ? usize : oldsize;
351 memcpy(ret, ptr, copysize);
352 isqalloc(tsd, ptr, oldsize, tcache);
353 return (ret);
354}
355
356void
357huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
358{
359 extent_node_t *node;
360 arena_t *arena;
361
362 node = huge_node_get(ptr);
363 arena = extent_node_arena_get(node);
364 huge_node_unset(ptr, node);
365 malloc_mutex_lock(&arena->huge_mtx);
366 ql_remove(&arena->huge, node, ql_link);
367 malloc_mutex_unlock(&arena->huge_mtx);
368
369 huge_dalloc_junk(extent_node_addr_get(node),
370 extent_node_size_get(node));
371 arena_chunk_dalloc_huge(extent_node_arena_get(node),
372 extent_node_addr_get(node), extent_node_size_get(node));
373 idalloctm(tsd, node, tcache, true);
374}
375
376arena_t *
377huge_aalloc(const void *ptr)
378{
379
380 return (extent_node_arena_get(huge_node_get(ptr)));
381}
382
383size_t
384huge_salloc(const void *ptr)
385{
386 size_t size;
387 extent_node_t *node;
388 arena_t *arena;
389
390 node = huge_node_get(ptr);
391 arena = extent_node_arena_get(node);
392 malloc_mutex_lock(&arena->huge_mtx);
393 size = extent_node_size_get(node);
394 malloc_mutex_unlock(&arena->huge_mtx);
395
396 return (size);
397}
398
399prof_tctx_t *
400huge_prof_tctx_get(const void *ptr)
401{
402 prof_tctx_t *tctx;
403 extent_node_t *node;
404 arena_t *arena;
405
406 node = huge_node_get(ptr);
407 arena = extent_node_arena_get(node);
408 malloc_mutex_lock(&arena->huge_mtx);
409 tctx = extent_node_prof_tctx_get(node);
410 malloc_mutex_unlock(&arena->huge_mtx);
411
412 return (tctx);
413}
414
415void
416huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
417{
418 extent_node_t *node;
419 arena_t *arena;
420
421 node = huge_node_get(ptr);
422 arena = extent_node_arena_get(node);
423 malloc_mutex_lock(&arena->huge_mtx);
424 extent_node_prof_tctx_set(node, tctx);
425 malloc_mutex_unlock(&arena->huge_mtx);
426}
427
428void
429huge_prof_tctx_reset(const void *ptr)
430{
431
432 huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U);
433}
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9 extent_node_t *node;
10
11 node = chunk_lookup(ptr, true);
12 assert(!extent_node_achunk_get(node));
13
14 return (node);
15}
16
17static bool
18huge_node_set(const void *ptr, extent_node_t *node)
19{
20
21 assert(extent_node_addr_get(node) == ptr);
22 assert(!extent_node_achunk_get(node));
23 return (chunk_register(ptr, node));
24}
25
26static void
27huge_node_unset(const void *ptr, const extent_node_t *node)
28{
29
30 chunk_deregister(ptr, node);
31}
32
33void *
34huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
35 tcache_t *tcache)
36{
37 size_t usize;
38
39 usize = s2u(size);
40 if (usize == 0) {
41 /* size_t overflow. */
42 return (NULL);
43 }
44
45 return (huge_palloc(tsd, arena, usize, chunksize, zero, tcache));
46}
47
48void *
49huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
50 bool zero, tcache_t *tcache)
51{
52 void *ret;
53 size_t usize;
54 extent_node_t *node;
55 bool is_zeroed;
56
57 /* Allocate one or more contiguous chunks for this request. */
58
59 usize = sa2u(size, alignment);
60 if (unlikely(usize == 0))
61 return (NULL);
62 assert(usize >= chunksize);
63
64 /* Allocate an extent node with which to track the chunk. */
65 node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
66 CACHELINE, false, tcache, true, arena);
67 if (node == NULL)
68 return (NULL);
69
70 /*
71 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
72 * it is possible to make correct junk/zero fill decisions below.
73 */
74 is_zeroed = zero;
75 arena = arena_choose(tsd, arena);
76 if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena,
77 size, alignment, &is_zeroed)) == NULL) {
78 idalloctm(tsd, node, tcache, true);
79 return (NULL);
80 }
81
82 extent_node_init(node, arena, ret, size, is_zeroed, true);
83
84 if (huge_node_set(ret, node)) {
85 arena_chunk_dalloc_huge(arena, ret, size);
86 idalloctm(tsd, node, tcache, true);
87 return (NULL);
88 }
89
90 /* Insert node into huge. */
91 malloc_mutex_lock(&arena->huge_mtx);
92 ql_elm_new(node, ql_link);
93 ql_tail_insert(&arena->huge, node, ql_link);
94 malloc_mutex_unlock(&arena->huge_mtx);
95
96 if (zero || (config_fill && unlikely(opt_zero))) {
97 if (!is_zeroed)
98 memset(ret, 0, size);
99 } else if (config_fill && unlikely(opt_junk_alloc))
100 memset(ret, 0xa5, size);
101
102 return (ret);
103}
104
105#ifdef JEMALLOC_JET
106#undef huge_dalloc_junk
107#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
108#endif
109static void
110huge_dalloc_junk(void *ptr, size_t usize)
111{
112
113 if (config_fill && have_dss && unlikely(opt_junk_free)) {
114 /*
115 * Only bother junk filling if the chunk isn't about to be
116 * unmapped.
117 */
118 if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
119 memset(ptr, 0x5a, usize);
120 }
121}
122#ifdef JEMALLOC_JET
123#undef huge_dalloc_junk
124#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
125huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
126#endif
127
128static void
129huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
130 size_t usize_max, bool zero)
131{
132 size_t usize, usize_next;
133 extent_node_t *node;
134 arena_t *arena;
135 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
136 bool zeroed;
137
138 /* Increase usize to incorporate extra. */
139 for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
140 <= oldsize; usize = usize_next)
141 ; /* Do nothing. */
142
143 if (oldsize == usize)
144 return;
145
146 node = huge_node_get(ptr);
147 arena = extent_node_arena_get(node);
148
149 /* Fill if necessary (shrinking). */
150 if (oldsize > usize) {
151 size_t sdiff = oldsize - usize;
152 if (config_fill && unlikely(opt_junk_free)) {
153 memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff);
154 zeroed = false;
155 } else {
156 zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr,
157 CHUNK_CEILING(oldsize), usize, sdiff);
158 }
159 } else
160 zeroed = true;
161
162 malloc_mutex_lock(&arena->huge_mtx);
163 /* Update the size of the huge allocation. */
164 assert(extent_node_size_get(node) != usize);
165 extent_node_size_set(node, usize);
166 /* Clear node's zeroed field if zeroing failed above. */
167 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed);
168 malloc_mutex_unlock(&arena->huge_mtx);
169
170 arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
171
172 /* Fill if necessary (growing). */
173 if (oldsize < usize) {
174 if (zero || (config_fill && unlikely(opt_zero))) {
175 if (!zeroed) {
176 memset((void *)((uintptr_t)ptr + oldsize), 0,
177 usize - oldsize);
178 }
179 } else if (config_fill && unlikely(opt_junk_alloc)) {
180 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
181 oldsize);
182 }
183 }
184}
185
186static bool
187huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
188{
189 extent_node_t *node;
190 arena_t *arena;
191 chunk_hooks_t chunk_hooks;
192 size_t cdiff;
193 bool zeroed;
194
195 node = huge_node_get(ptr);
196 arena = extent_node_arena_get(node);
197 chunk_hooks = chunk_hooks_get(arena);
198
199 assert(oldsize > usize);
200
201 /* Split excess chunks. */
202 cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
203 if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
204 CHUNK_CEILING(usize), cdiff, true, arena->ind))
205 return (true);
206
207 if (oldsize > usize) {
208 size_t sdiff = oldsize - usize;
209 if (config_fill && unlikely(opt_junk_free)) {
210 huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
211 sdiff);
212 zeroed = false;
213 } else {
214 zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
215 CHUNK_ADDR2BASE((uintptr_t)ptr + usize),
216 CHUNK_CEILING(oldsize),
217 CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
218 }
219 } else
220 zeroed = true;
221
222 malloc_mutex_lock(&arena->huge_mtx);
223 /* Update the size of the huge allocation. */
224 extent_node_size_set(node, usize);
225 /* Clear node's zeroed field if zeroing failed above. */
226 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed);
227 malloc_mutex_unlock(&arena->huge_mtx);
228
229 /* Zap the excess chunks. */
230 arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize);
231
232 return (false);
233}
234
235static bool
236huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) {
237 extent_node_t *node;
238 arena_t *arena;
239 bool is_zeroed_subchunk, is_zeroed_chunk;
240
241 node = huge_node_get(ptr);
242 arena = extent_node_arena_get(node);
243 malloc_mutex_lock(&arena->huge_mtx);
244 is_zeroed_subchunk = extent_node_zeroed_get(node);
245 malloc_mutex_unlock(&arena->huge_mtx);
246
247 /*
248 * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so
249 * that it is possible to make correct junk/zero fill decisions below.
250 */
251 is_zeroed_chunk = zero;
252
253 if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize,
254 &is_zeroed_chunk))
255 return (true);
256
257 malloc_mutex_lock(&arena->huge_mtx);
258 /* Update the size of the huge allocation. */
259 extent_node_size_set(node, usize);
260 malloc_mutex_unlock(&arena->huge_mtx);
261
262 if (zero || (config_fill && unlikely(opt_zero))) {
263 if (!is_zeroed_subchunk) {
264 memset((void *)((uintptr_t)ptr + oldsize), 0,
265 CHUNK_CEILING(oldsize) - oldsize);
266 }
267 if (!is_zeroed_chunk) {
268 memset((void *)((uintptr_t)ptr +
269 CHUNK_CEILING(oldsize)), 0, usize -
270 CHUNK_CEILING(oldsize));
271 }
272 } else if (config_fill && unlikely(opt_junk_alloc)) {
273 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize -
274 oldsize);
275 }
276
277 return (false);
278}
279
280bool
281huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min,
282 size_t usize_max, bool zero)
283{
284
285 assert(s2u(oldsize) == oldsize);
286
287 /* Both allocations must be huge to avoid a move. */
288 if (oldsize < chunksize || usize_max < chunksize)
289 return (true);
290
291 if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
292 /* Attempt to expand the allocation in-place. */
293 if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max, zero))
294 return (false);
295 /* Try again, this time with usize_min. */
296 if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
297 CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr,
298 oldsize, usize_min, zero))
299 return (false);
300 }
301
302 /*
303 * Avoid moving the allocation if the existing chunk size accommodates
304 * the new size.
305 */
306 if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
307 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
308 huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max,
309 zero);
310 return (false);
311 }
312
313 /* Attempt to shrink the allocation in-place. */
314 if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max))
315 return (huge_ralloc_no_move_shrink(ptr, oldsize, usize_max));
316 return (true);
317}
318
319static void *
320huge_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize,
321 size_t alignment, bool zero, tcache_t *tcache)
322{
323
324 if (alignment <= chunksize)
325 return (huge_malloc(tsd, arena, usize, zero, tcache));
326 return (huge_palloc(tsd, arena, usize, alignment, zero, tcache));
327}
328
329void *
330huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize,
331 size_t alignment, bool zero, tcache_t *tcache)
332{
333 void *ret;
334 size_t copysize;
335
336 /* Try to avoid moving the allocation. */
337 if (!huge_ralloc_no_move(ptr, oldsize, usize, usize, zero))
338 return (ptr);
339
340 /*
341 * usize and oldsize are different enough that we need to use a
342 * different size class. In that case, fall back to allocating new
343 * space and copying.
344 */
345 ret = huge_ralloc_move_helper(tsd, arena, usize, alignment, zero,
346 tcache);
347 if (ret == NULL)
348 return (NULL);
349
350 copysize = (usize < oldsize) ? usize : oldsize;
351 memcpy(ret, ptr, copysize);
352 isqalloc(tsd, ptr, oldsize, tcache);
353 return (ret);
354}
355
356void
357huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
358{
359 extent_node_t *node;
360 arena_t *arena;
361
362 node = huge_node_get(ptr);
363 arena = extent_node_arena_get(node);
364 huge_node_unset(ptr, node);
365 malloc_mutex_lock(&arena->huge_mtx);
366 ql_remove(&arena->huge, node, ql_link);
367 malloc_mutex_unlock(&arena->huge_mtx);
368
369 huge_dalloc_junk(extent_node_addr_get(node),
370 extent_node_size_get(node));
371 arena_chunk_dalloc_huge(extent_node_arena_get(node),
372 extent_node_addr_get(node), extent_node_size_get(node));
373 idalloctm(tsd, node, tcache, true);
374}
375
376arena_t *
377huge_aalloc(const void *ptr)
378{
379
380 return (extent_node_arena_get(huge_node_get(ptr)));
381}
382
383size_t
384huge_salloc(const void *ptr)
385{
386 size_t size;
387 extent_node_t *node;
388 arena_t *arena;
389
390 node = huge_node_get(ptr);
391 arena = extent_node_arena_get(node);
392 malloc_mutex_lock(&arena->huge_mtx);
393 size = extent_node_size_get(node);
394 malloc_mutex_unlock(&arena->huge_mtx);
395
396 return (size);
397}
398
399prof_tctx_t *
400huge_prof_tctx_get(const void *ptr)
401{
402 prof_tctx_t *tctx;
403 extent_node_t *node;
404 arena_t *arena;
405
406 node = huge_node_get(ptr);
407 arena = extent_node_arena_get(node);
408 malloc_mutex_lock(&arena->huge_mtx);
409 tctx = extent_node_prof_tctx_get(node);
410 malloc_mutex_unlock(&arena->huge_mtx);
411
412 return (tctx);
413}
414
415void
416huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
417{
418 extent_node_t *node;
419 arena_t *arena;
420
421 node = huge_node_get(ptr);
422 arena = extent_node_arena_get(node);
423 malloc_mutex_lock(&arena->huge_mtx);
424 extent_node_prof_tctx_set(node, tctx);
425 malloc_mutex_unlock(&arena->huge_mtx);
426}
427
428void
429huge_prof_tctx_reset(const void *ptr)
430{
431
432 huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U);
433}