1#define	JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7uint64_t	huge_nmalloc;
8uint64_t	huge_ndalloc;
9size_t		huge_allocated;
10
11malloc_mutex_t	huge_mtx;
12
13/******************************************************************************/
14
15/* Tree of chunks that are stand-alone huge allocations. */
16static extent_tree_t	huge;
17
18void *
19huge_malloc(size_t size, bool zero)
20{
21
22	return (huge_palloc(size, chunksize, zero));
23}
24
25void *
26huge_palloc(size_t size, size_t alignment, bool zero)
27{
28	void *ret;
29	size_t csize;
30	extent_node_t *node;
31	bool is_zeroed;
32
33	/* Allocate one or more contiguous chunks for this request. */
34
35	csize = CHUNK_CEILING(size);
36	if (csize == 0) {
37		/* size is large enough to cause size_t wrap-around. */
38		return (NULL);
39	}
40
41	/* Allocate an extent node with which to track the chunk. */
42	node = base_node_alloc();
43	if (node == NULL)
44		return (NULL);
45
46	/*
47	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
48	 * it is possible to make correct junk/zero fill decisions below.
49	 */
50	is_zeroed = zero;
51	ret = chunk_alloc(csize, alignment, false, &is_zeroed,
52	    chunk_dss_prec_get());
53	if (ret == NULL) {
54		base_node_dealloc(node);
55		return (NULL);
56	}
57
58	/* Insert node into huge. */
59	node->addr = ret;
60	node->size = csize;
61
62	malloc_mutex_lock(&huge_mtx);
63	extent_tree_ad_insert(&huge, node);
64	if (config_stats) {
65		stats_cactive_add(csize);
66		huge_nmalloc++;
67		huge_allocated += csize;
68	}
69	malloc_mutex_unlock(&huge_mtx);
70
71	if (config_fill && zero == false) {
72		if (opt_junk)
73			memset(ret, 0xa5, csize);
74		else if (opt_zero && is_zeroed == false)
75			memset(ret, 0, csize);
76	}
77
78	return (ret);
79}
80
81void *
82huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
83{
84
85	/*
86	 * Avoid moving the allocation if the size class can be left the same.
87	 */
88	if (oldsize > arena_maxclass
89	    && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
90	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
91		assert(CHUNK_CEILING(oldsize) == oldsize);
92		if (config_fill && opt_junk && size < oldsize) {
93			memset((void *)((uintptr_t)ptr + size), 0x5a,
94			    oldsize - size);
95		}
96		return (ptr);
97	}
98
99	/* Reallocation would require a move. */
100	return (NULL);
101}
102
103void *
104huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
105    size_t alignment, bool zero, bool try_tcache_dalloc)
106{
107	void *ret;
108	size_t copysize;
109
110	/* Try to avoid moving the allocation. */
111	ret = huge_ralloc_no_move(ptr, oldsize, size, extra);
112	if (ret != NULL)
113		return (ret);
114
115	/*
116	 * size and oldsize are different enough that we need to use a
117	 * different size class.  In that case, fall back to allocating new
118	 * space and copying.
119	 */
120	if (alignment > chunksize)
121		ret = huge_palloc(size + extra, alignment, zero);
122	else
123		ret = huge_malloc(size + extra, zero);
124
125	if (ret == NULL) {
126		if (extra == 0)
127			return (NULL);
128		/* Try again, this time without extra. */
129		if (alignment > chunksize)
130			ret = huge_palloc(size, alignment, zero);
131		else
132			ret = huge_malloc(size, zero);
133
134		if (ret == NULL)
135			return (NULL);
136	}
137
138	/*
139	 * Copy at most size bytes (not size+extra), since the caller has no
140	 * expectation that the extra bytes will be reliably preserved.
141	 */
142	copysize = (size < oldsize) ? size : oldsize;
143
144#ifdef JEMALLOC_MREMAP
145	/*
146	 * Use mremap(2) if this is a huge-->huge reallocation, and neither the
147	 * source nor the destination are in dss.
148	 */
149	if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr)
150	    == false && chunk_in_dss(ret) == false))) {
151		size_t newsize = huge_salloc(ret);
152
153		/*
154		 * Remove ptr from the tree of huge allocations before
155		 * performing the remap operation, in order to avoid the
156		 * possibility of another thread acquiring that mapping before
157		 * this one removes it from the tree.
158		 */
159		huge_dalloc(ptr, false);
160		if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED,
161		    ret) == MAP_FAILED) {
162			/*
163			 * Assuming no chunk management bugs in the allocator,
164			 * the only documented way an error can occur here is
165			 * if the application changed the map type for a
166			 * portion of the old allocation.  This is firmly in
167			 * undefined behavior territory, so write a diagnostic
168			 * message, and optionally abort.
169			 */
170			char buf[BUFERROR_BUF];
171
172			buferror(buf, sizeof(buf));
173			malloc_printf("<jemalloc>: Error in mremap(): %s\n",
174			    buf);
175			if (opt_abort)
176				abort();
177			memcpy(ret, ptr, copysize);
178			chunk_dealloc_mmap(ptr, oldsize);
179		}
180	} else
181#endif
182	{
183		memcpy(ret, ptr, copysize);
184		iqallocx(ptr, try_tcache_dalloc);
185	}
186	return (ret);
187}
188
189void
190huge_dalloc(void *ptr, bool unmap)
191{
192	extent_node_t *node, key;
193
194	malloc_mutex_lock(&huge_mtx);
195
196	/* Extract from tree of huge allocations. */
197	key.addr = ptr;
198	node = extent_tree_ad_search(&huge, &key);
199	assert(node != NULL);
200	assert(node->addr == ptr);
201	extent_tree_ad_remove(&huge, node);
202
203	if (config_stats) {
204		stats_cactive_sub(node->size);
205		huge_ndalloc++;
206		huge_allocated -= node->size;
207	}
208
209	malloc_mutex_unlock(&huge_mtx);
210
211	if (unmap && config_fill && config_dss && opt_junk)
212		memset(node->addr, 0x5a, node->size);
213
214	chunk_dealloc(node->addr, node->size, unmap);
215
216	base_node_dealloc(node);
217}
218
219size_t
220huge_salloc(const void *ptr)
221{
222	size_t ret;
223	extent_node_t *node, key;
224
225	malloc_mutex_lock(&huge_mtx);
226
227	/* Extract from tree of huge allocations. */
228	key.addr = __DECONST(void *, ptr);
229	node = extent_tree_ad_search(&huge, &key);
230	assert(node != NULL);
231
232	ret = node->size;
233
234	malloc_mutex_unlock(&huge_mtx);
235
236	return (ret);
237}
238
239prof_ctx_t *
240huge_prof_ctx_get(const void *ptr)
241{
242	prof_ctx_t *ret;
243	extent_node_t *node, key;
244
245	malloc_mutex_lock(&huge_mtx);
246
247	/* Extract from tree of huge allocations. */
248	key.addr = __DECONST(void *, ptr);
249	node = extent_tree_ad_search(&huge, &key);
250	assert(node != NULL);
251
252	ret = node->prof_ctx;
253
254	malloc_mutex_unlock(&huge_mtx);
255
256	return (ret);
257}
258
259void
260huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
261{
262	extent_node_t *node, key;
263
264	malloc_mutex_lock(&huge_mtx);
265
266	/* Extract from tree of huge allocations. */
267	key.addr = __DECONST(void *, ptr);
268	node = extent_tree_ad_search(&huge, &key);
269	assert(node != NULL);
270
271	node->prof_ctx = ctx;
272
273	malloc_mutex_unlock(&huge_mtx);
274}
275
276bool
277huge_boot(void)
278{
279
280	/* Initialize chunks data. */
281	if (malloc_mutex_init(&huge_mtx))
282		return (true);
283	extent_tree_ad_new(&huge);
284
285	if (config_stats) {
286		huge_nmalloc = 0;
287		huge_ndalloc = 0;
288		huge_allocated = 0;
289	}
290
291	return (false);
292}
293
294void
295huge_prefork(void)
296{
297
298	malloc_mutex_prefork(&huge_mtx);
299}
300
301void
302huge_postfork_parent(void)
303{
304
305	malloc_mutex_postfork_parent(&huge_mtx);
306}
307
308void
309huge_postfork_child(void)
310{
311
312	malloc_mutex_postfork_child(&huge_mtx);
313}
314