huge.c revision 301718 
1#define	JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9	extent_node_t *node;
10
11	node = chunk_lookup(ptr, true);
12	assert(!extent_node_achunk_get(node));
13
14	return (node);
15}
16
17static bool
18huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
19{
20
21	assert(extent_node_addr_get(node) == ptr);
22	assert(!extent_node_achunk_get(node));
23	return (chunk_register(tsdn, ptr, node));
24}
25
26static void
27huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
28{
29	bool err;
30
31	err = huge_node_set(tsdn, ptr, node);
32	assert(!err);
33}
34
35static void
36huge_node_unset(const void *ptr, const extent_node_t *node)
37{
38
39	chunk_deregister(ptr, node);
40}
41
42void *
43huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
44{
45
46	assert(usize == s2u(usize));
47
48	return (huge_palloc(tsdn, arena, usize, chunksize, zero));
49}
50
51void *
52huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
53    bool zero)
54{
55	void *ret;
56	size_t ausize;
57	extent_node_t *node;
58	bool is_zeroed;
59
60	/* Allocate one or more contiguous chunks for this request. */
61
62	assert(!tsdn_null(tsdn) || arena != NULL);
63
64	ausize = sa2u(usize, alignment);
65	if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
66		return (NULL);
67	assert(ausize >= chunksize);
68
69	/* Allocate an extent node with which to track the chunk. */
70	node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
71	    CACHELINE, false, NULL, true, arena_ichoose(tsdn, arena));
72	if (node == NULL)
73		return (NULL);
74
75	/*
76	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
77	 * it is possible to make correct junk/zero fill decisions below.
78	 */
79	is_zeroed = zero;
80	if (likely(!tsdn_null(tsdn)))
81		arena = arena_choose(tsdn_tsd(tsdn), arena);
82	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
83	    arena, usize, alignment, &is_zeroed)) == NULL) {
84		idalloctm(tsdn, node, NULL, true, true);
85		return (NULL);
86	}
87
88	extent_node_init(node, arena, ret, usize, is_zeroed, true);
89
90	if (huge_node_set(tsdn, ret, node)) {
91		arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
92		idalloctm(tsdn, node, NULL, true, true);
93		return (NULL);
94	}
95
96	/* Insert node into huge. */
97	malloc_mutex_lock(tsdn, &arena->huge_mtx);
98	ql_elm_new(node, ql_link);
99	ql_tail_insert(&arena->huge, node, ql_link);
100	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
101
102	if (zero || (config_fill && unlikely(opt_zero))) {
103		if (!is_zeroed)
104			memset(ret, 0, usize);
105	} else if (config_fill && unlikely(opt_junk_alloc))
106		memset(ret, JEMALLOC_ALLOC_JUNK, usize);
107
108	arena_decay_tick(tsdn, arena);
109	return (ret);
110}
111
112#ifdef JEMALLOC_JET
113#undef huge_dalloc_junk
114#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
115#endif
116static void
117huge_dalloc_junk(tsdn_t *tsdn, void *ptr, size_t usize)
118{
119
120	if (config_fill && have_dss && unlikely(opt_junk_free)) {
121		/*
122		 * Only bother junk filling if the chunk isn't about to be
123		 * unmapped.
124		 */
125		if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
126			memset(ptr, JEMALLOC_FREE_JUNK, usize);
127	}
128}
129#ifdef JEMALLOC_JET
130#undef huge_dalloc_junk
131#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
132huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
133#endif
134
135static void
136huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
137    size_t usize_min, size_t usize_max, bool zero)
138{
139	size_t usize, usize_next;
140	extent_node_t *node;
141	arena_t *arena;
142	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
143	bool pre_zeroed, post_zeroed;
144
145	/* Increase usize to incorporate extra. */
146	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
147	    <= oldsize; usize = usize_next)
148		; /* Do nothing. */
149
150	if (oldsize == usize)
151		return;
152
153	node = huge_node_get(ptr);
154	arena = extent_node_arena_get(node);
155	pre_zeroed = extent_node_zeroed_get(node);
156
157	/* Fill if necessary (shrinking). */
158	if (oldsize > usize) {
159		size_t sdiff = oldsize - usize;
160		if (config_fill && unlikely(opt_junk_free)) {
161			memset((void *)((uintptr_t)ptr + usize),
162			    JEMALLOC_FREE_JUNK, sdiff);
163			post_zeroed = false;
164		} else {
165			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
166			    &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
167			    sdiff);
168		}
169	} else
170		post_zeroed = pre_zeroed;
171
172	malloc_mutex_lock(tsdn, &arena->huge_mtx);
173	/* Update the size of the huge allocation. */
174	huge_node_unset(ptr, node);
175	assert(extent_node_size_get(node) != usize);
176	extent_node_size_set(node, usize);
177	huge_node_reset(tsdn, ptr, node);
178	/* Update zeroed. */
179	extent_node_zeroed_set(node, post_zeroed);
180	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
181
182	arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
183
184	/* Fill if necessary (growing). */
185	if (oldsize < usize) {
186		if (zero || (config_fill && unlikely(opt_zero))) {
187			if (!pre_zeroed) {
188				memset((void *)((uintptr_t)ptr + oldsize), 0,
189				    usize - oldsize);
190			}
191		} else if (config_fill && unlikely(opt_junk_alloc)) {
192			memset((void *)((uintptr_t)ptr + oldsize),
193			    JEMALLOC_ALLOC_JUNK, usize - oldsize);
194		}
195	}
196}
197
198static bool
199huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
200    size_t usize)
201{
202	extent_node_t *node;
203	arena_t *arena;
204	chunk_hooks_t chunk_hooks;
205	size_t cdiff;
206	bool pre_zeroed, post_zeroed;
207
208	node = huge_node_get(ptr);
209	arena = extent_node_arena_get(node);
210	pre_zeroed = extent_node_zeroed_get(node);
211	chunk_hooks = chunk_hooks_get(tsdn, arena);
212
213	assert(oldsize > usize);
214
215	/* Split excess chunks. */
216	cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
217	if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
218	    CHUNK_CEILING(usize), cdiff, true, arena->ind))
219		return (true);
220
221	if (oldsize > usize) {
222		size_t sdiff = oldsize - usize;
223		if (config_fill && unlikely(opt_junk_free)) {
224			huge_dalloc_junk(tsdn, (void *)((uintptr_t)ptr + usize),
225			    sdiff);
226			post_zeroed = false;
227		} else {
228			post_zeroed = !chunk_purge_wrapper(tsdn, arena,
229			    &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
230			    usize), CHUNK_CEILING(oldsize),
231			    CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
232		}
233	} else
234		post_zeroed = pre_zeroed;
235
236	malloc_mutex_lock(tsdn, &arena->huge_mtx);
237	/* Update the size of the huge allocation. */
238	huge_node_unset(ptr, node);
239	extent_node_size_set(node, usize);
240	huge_node_reset(tsdn, ptr, node);
241	/* Update zeroed. */
242	extent_node_zeroed_set(node, post_zeroed);
243	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
244
245	/* Zap the excess chunks. */
246	arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
247
248	return (false);
249}
250
251static bool
252huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
253    size_t usize, bool zero) {
254	extent_node_t *node;
255	arena_t *arena;
256	bool is_zeroed_subchunk, is_zeroed_chunk;
257
258	node = huge_node_get(ptr);
259	arena = extent_node_arena_get(node);
260	malloc_mutex_lock(tsdn, &arena->huge_mtx);
261	is_zeroed_subchunk = extent_node_zeroed_get(node);
262	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
263
264	/*
265	 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
266	 * update extent's zeroed field, and zero as necessary.
267	 */
268	is_zeroed_chunk = false;
269	if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize,
270	     &is_zeroed_chunk))
271		return (true);
272
273	malloc_mutex_lock(tsdn, &arena->huge_mtx);
274	huge_node_unset(ptr, node);
275	extent_node_size_set(node, usize);
276	extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
277	    is_zeroed_chunk);
278	huge_node_reset(tsdn, ptr, node);
279	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
280
281	if (zero || (config_fill && unlikely(opt_zero))) {
282		if (!is_zeroed_subchunk) {
283			memset((void *)((uintptr_t)ptr + oldsize), 0,
284			    CHUNK_CEILING(oldsize) - oldsize);
285		}
286		if (!is_zeroed_chunk) {
287			memset((void *)((uintptr_t)ptr +
288			    CHUNK_CEILING(oldsize)), 0, usize -
289			    CHUNK_CEILING(oldsize));
290		}
291	} else if (config_fill && unlikely(opt_junk_alloc)) {
292		memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
293		    usize - oldsize);
294	}
295
296	return (false);
297}
298
299bool
300huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
301    size_t usize_max, bool zero)
302{
303
304	assert(s2u(oldsize) == oldsize);
305	/* The following should have been caught by callers. */
306	assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
307
308	/* Both allocations must be huge to avoid a move. */
309	if (oldsize < chunksize || usize_max < chunksize)
310		return (true);
311
312	if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
313		/* Attempt to expand the allocation in-place. */
314		if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
315		    zero)) {
316			arena_decay_tick(tsdn, huge_aalloc(ptr));
317			return (false);
318		}
319		/* Try again, this time with usize_min. */
320		if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
321		    CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
322		    ptr, oldsize, usize_min, zero)) {
323			arena_decay_tick(tsdn, huge_aalloc(ptr));
324			return (false);
325		}
326	}
327
328	/*
329	 * Avoid moving the allocation if the existing chunk size accommodates
330	 * the new size.
331	 */
332	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
333	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
334		huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
335		    usize_max, zero);
336		arena_decay_tick(tsdn, huge_aalloc(ptr));
337		return (false);
338	}
339
340	/* Attempt to shrink the allocation in-place. */
341	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
342		if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
343		    usize_max)) {
344			arena_decay_tick(tsdn, huge_aalloc(ptr));
345			return (false);
346		}
347	}
348	return (true);
349}
350
351static void *
352huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
353    size_t alignment, bool zero)
354{
355
356	if (alignment <= chunksize)
357		return (huge_malloc(tsdn, arena, usize, zero));
358	return (huge_palloc(tsdn, arena, usize, alignment, zero));
359}
360
361void *
362huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
363    size_t usize, size_t alignment, bool zero, tcache_t *tcache)
364{
365	void *ret;
366	size_t copysize;
367
368	/* The following should have been caught by callers. */
369	assert(usize > 0 && usize <= HUGE_MAXCLASS);
370
371	/* Try to avoid moving the allocation. */
372	if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize,
373	    zero))
374		return (ptr);
375
376	/*
377	 * usize and oldsize are different enough that we need to use a
378	 * different size class.  In that case, fall back to allocating new
379	 * space and copying.
380	 */
381	ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment,
382	    zero);
383	if (ret == NULL)
384		return (NULL);
385
386	copysize = (usize < oldsize) ? usize : oldsize;
387	memcpy(ret, ptr, copysize);
388	isqalloc(tsd, ptr, oldsize, tcache, true);
389	return (ret);
390}
391
392void
393huge_dalloc(tsdn_t *tsdn, void *ptr)
394{
395	extent_node_t *node;
396	arena_t *arena;
397
398	node = huge_node_get(ptr);
399	arena = extent_node_arena_get(node);
400	huge_node_unset(ptr, node);
401	malloc_mutex_lock(tsdn, &arena->huge_mtx);
402	ql_remove(&arena->huge, node, ql_link);
403	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
404
405	huge_dalloc_junk(tsdn, extent_node_addr_get(node),
406	    extent_node_size_get(node));
407	arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
408	    extent_node_addr_get(node), extent_node_size_get(node));
409	idalloctm(tsdn, node, NULL, true, true);
410
411	arena_decay_tick(tsdn, arena);
412}
413
414arena_t *
415huge_aalloc(const void *ptr)
416{
417
418	return (extent_node_arena_get(huge_node_get(ptr)));
419}
420
421size_t
422huge_salloc(tsdn_t *tsdn, const void *ptr)
423{
424	size_t size;
425	extent_node_t *node;
426	arena_t *arena;
427
428	node = huge_node_get(ptr);
429	arena = extent_node_arena_get(node);
430	malloc_mutex_lock(tsdn, &arena->huge_mtx);
431	size = extent_node_size_get(node);
432	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
433
434	return (size);
435}
436
437prof_tctx_t *
438huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
439{
440	prof_tctx_t *tctx;
441	extent_node_t *node;
442	arena_t *arena;
443
444	node = huge_node_get(ptr);
445	arena = extent_node_arena_get(node);
446	malloc_mutex_lock(tsdn, &arena->huge_mtx);
447	tctx = extent_node_prof_tctx_get(node);
448	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
449
450	return (tctx);
451}
452
453void
454huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
455{
456	extent_node_t *node;
457	arena_t *arena;
458
459	node = huge_node_get(ptr);
460	arena = extent_node_arena_get(node);
461	malloc_mutex_lock(tsdn, &arena->huge_mtx);
462	extent_node_prof_tctx_set(node, tctx);
463	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
464}
465
466void
467huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
468{
469
470	huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
471}
472