1#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
2#define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
3
4#include "jemalloc/internal/jemalloc_internal_types.h"
5#include "jemalloc/internal/mutex.h"
6#include "jemalloc/internal/rtree.h"
7#include "jemalloc/internal/sc.h"
8#include "jemalloc/internal/sz.h"
9#include "jemalloc/internal/ticker.h"
10
11JEMALLOC_ALWAYS_INLINE bool
12arena_has_default_hooks(arena_t *arena) {
13	return (extent_hooks_get(arena) == &extent_hooks_default);
14}
15
16JEMALLOC_ALWAYS_INLINE arena_t *
17arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
18	if (arena != NULL) {
19		return arena;
20	}
21
22	/*
23	 * For huge allocations, use the dedicated huge arena if both are true:
24	 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
25	 * thread is not assigned to a manual arena.
26	 */
27	if (unlikely(size >= oversize_threshold)) {
28		arena_t *tsd_arena = tsd_arena_get(tsd);
29		if (tsd_arena == NULL || arena_is_auto(tsd_arena)) {
30			return arena_choose_huge(tsd);
31		}
32	}
33
34	return arena_choose(tsd, NULL);
35}
36
37JEMALLOC_ALWAYS_INLINE prof_tctx_t *
38arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
39	cassert(config_prof);
40	assert(ptr != NULL);
41
42	/* Static check. */
43	if (alloc_ctx == NULL) {
44		const extent_t *extent = iealloc(tsdn, ptr);
45		if (unlikely(!extent_slab_get(extent))) {
46			return large_prof_tctx_get(tsdn, extent);
47		}
48	} else {
49		if (unlikely(!alloc_ctx->slab)) {
50			return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr));
51		}
52	}
53	return (prof_tctx_t *)(uintptr_t)1U;
54}
55
56JEMALLOC_ALWAYS_INLINE void
57arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
58    alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
59	cassert(config_prof);
60	assert(ptr != NULL);
61
62	/* Static check. */
63	if (alloc_ctx == NULL) {
64		extent_t *extent = iealloc(tsdn, ptr);
65		if (unlikely(!extent_slab_get(extent))) {
66			large_prof_tctx_set(tsdn, extent, tctx);
67		}
68	} else {
69		if (unlikely(!alloc_ctx->slab)) {
70			large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx);
71		}
72	}
73}
74
75static inline void
76arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
77	cassert(config_prof);
78	assert(ptr != NULL);
79
80	extent_t *extent = iealloc(tsdn, ptr);
81	assert(!extent_slab_get(extent));
82
83	large_prof_tctx_reset(tsdn, extent);
84}
85
86JEMALLOC_ALWAYS_INLINE nstime_t
87arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr,
88    alloc_ctx_t *alloc_ctx) {
89	cassert(config_prof);
90	assert(ptr != NULL);
91
92	extent_t *extent = iealloc(tsdn, ptr);
93	/*
94	 * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're
95	 * sure we have a sampled allocation.
96	 */
97	assert(!extent_slab_get(extent));
98	return large_prof_alloc_time_get(extent);
99}
100
101JEMALLOC_ALWAYS_INLINE void
102arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx,
103    nstime_t t) {
104	cassert(config_prof);
105	assert(ptr != NULL);
106
107	extent_t *extent = iealloc(tsdn, ptr);
108	assert(!extent_slab_get(extent));
109	large_prof_alloc_time_set(extent, t);
110}
111
112JEMALLOC_ALWAYS_INLINE void
113arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
114	tsd_t *tsd;
115	ticker_t *decay_ticker;
116
117	if (unlikely(tsdn_null(tsdn))) {
118		return;
119	}
120	tsd = tsdn_tsd(tsdn);
121	decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena));
122	if (unlikely(decay_ticker == NULL)) {
123		return;
124	}
125	if (unlikely(ticker_ticks(decay_ticker, nticks))) {
126		arena_decay(tsdn, arena, false, false);
127	}
128}
129
130JEMALLOC_ALWAYS_INLINE void
131arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
132	malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx);
133	malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx);
134
135	arena_decay_ticks(tsdn, arena, 1);
136}
137
138/* Purge a single extent to retained / unmapped directly. */
139JEMALLOC_ALWAYS_INLINE void
140arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks,
141    extent_t *extent) {
142	size_t extent_size = extent_size_get(extent);
143	extent_dalloc_wrapper(tsdn, arena,
144	    r_extent_hooks, extent);
145	if (config_stats) {
146		/* Update stats accordingly. */
147		arena_stats_lock(tsdn, &arena->stats);
148		arena_stats_add_u64(tsdn, &arena->stats,
149		    &arena->decay_dirty.stats->nmadvise, 1);
150		arena_stats_add_u64(tsdn, &arena->stats,
151		    &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE);
152		arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped,
153		    extent_size);
154		arena_stats_unlock(tsdn, &arena->stats);
155	}
156}
157
158JEMALLOC_ALWAYS_INLINE void *
159arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
160    tcache_t *tcache, bool slow_path) {
161	assert(!tsdn_null(tsdn) || tcache == NULL);
162
163	if (likely(tcache != NULL)) {
164		if (likely(size <= SC_SMALL_MAXCLASS)) {
165			return tcache_alloc_small(tsdn_tsd(tsdn), arena,
166			    tcache, size, ind, zero, slow_path);
167		}
168		if (likely(size <= tcache_maxclass)) {
169			return tcache_alloc_large(tsdn_tsd(tsdn), arena,
170			    tcache, size, ind, zero, slow_path);
171		}
172		/* (size > tcache_maxclass) case falls through. */
173		assert(size > tcache_maxclass);
174	}
175
176	return arena_malloc_hard(tsdn, arena, size, ind, zero);
177}
178
179JEMALLOC_ALWAYS_INLINE arena_t *
180arena_aalloc(tsdn_t *tsdn, const void *ptr) {
181	return extent_arena_get(iealloc(tsdn, ptr));
182}
183
184JEMALLOC_ALWAYS_INLINE size_t
185arena_salloc(tsdn_t *tsdn, const void *ptr) {
186	assert(ptr != NULL);
187
188	rtree_ctx_t rtree_ctx_fallback;
189	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
190
191	szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx,
192	    (uintptr_t)ptr, true);
193	assert(szind != SC_NSIZES);
194
195	return sz_index2size(szind);
196}
197
198JEMALLOC_ALWAYS_INLINE size_t
199arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
200	/*
201	 * Return 0 if ptr is not within an extent managed by jemalloc.  This
202	 * function has two extra costs relative to isalloc():
203	 * - The rtree calls cannot claim to be dependent lookups, which induces
204	 *   rtree lookup load dependencies.
205	 * - The lookup may fail, so there is an extra branch to check for
206	 *   failure.
207	 */
208
209	rtree_ctx_t rtree_ctx_fallback;
210	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
211
212	extent_t *extent;
213	szind_t szind;
214	if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx,
215	    (uintptr_t)ptr, false, &extent, &szind)) {
216		return 0;
217	}
218
219	if (extent == NULL) {
220		return 0;
221	}
222	assert(extent_state_get(extent) == extent_state_active);
223	/* Only slab members should be looked up via interior pointers. */
224	assert(extent_addr_get(extent) == ptr || extent_slab_get(extent));
225
226	assert(szind != SC_NSIZES);
227
228	return sz_index2size(szind);
229}
230
231static inline void
232arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) {
233	if (config_prof && unlikely(szind < SC_NBINS)) {
234		arena_dalloc_promoted(tsdn, ptr, NULL, true);
235	} else {
236		extent_t *extent = iealloc(tsdn, ptr);
237		large_dalloc(tsdn, extent);
238	}
239}
240
241static inline void
242arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
243	assert(ptr != NULL);
244
245	rtree_ctx_t rtree_ctx_fallback;
246	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
247
248	szind_t szind;
249	bool slab;
250	rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
251	    true, &szind, &slab);
252
253	if (config_debug) {
254		extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
255		    rtree_ctx, (uintptr_t)ptr, true);
256		assert(szind == extent_szind_get(extent));
257		assert(szind < SC_NSIZES);
258		assert(slab == extent_slab_get(extent));
259	}
260
261	if (likely(slab)) {
262		/* Small allocation. */
263		arena_dalloc_small(tsdn, ptr);
264	} else {
265		arena_dalloc_large_no_tcache(tsdn, ptr, szind);
266	}
267}
268
269JEMALLOC_ALWAYS_INLINE void
270arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
271    bool slow_path) {
272	if (szind < nhbins) {
273		if (config_prof && unlikely(szind < SC_NBINS)) {
274			arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
275		} else {
276			tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind,
277			    slow_path);
278		}
279	} else {
280		extent_t *extent = iealloc(tsdn, ptr);
281		large_dalloc(tsdn, extent);
282	}
283}
284
285JEMALLOC_ALWAYS_INLINE void
286arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
287    alloc_ctx_t *alloc_ctx, bool slow_path) {
288	assert(!tsdn_null(tsdn) || tcache == NULL);
289	assert(ptr != NULL);
290
291	if (unlikely(tcache == NULL)) {
292		arena_dalloc_no_tcache(tsdn, ptr);
293		return;
294	}
295
296	szind_t szind;
297	bool slab;
298	rtree_ctx_t *rtree_ctx;
299	if (alloc_ctx != NULL) {
300		szind = alloc_ctx->szind;
301		slab = alloc_ctx->slab;
302		assert(szind != SC_NSIZES);
303	} else {
304		rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
305		rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
306		    (uintptr_t)ptr, true, &szind, &slab);
307	}
308
309	if (config_debug) {
310		rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
311		extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
312		    rtree_ctx, (uintptr_t)ptr, true);
313		assert(szind == extent_szind_get(extent));
314		assert(szind < SC_NSIZES);
315		assert(slab == extent_slab_get(extent));
316	}
317
318	if (likely(slab)) {
319		/* Small allocation. */
320		tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
321		    slow_path);
322	} else {
323		arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
324	}
325}
326
327static inline void
328arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
329	assert(ptr != NULL);
330	assert(size <= SC_LARGE_MAXCLASS);
331
332	szind_t szind;
333	bool slab;
334	if (!config_prof || !opt_prof) {
335		/*
336		 * There is no risk of being confused by a promoted sampled
337		 * object, so base szind and slab on the given size.
338		 */
339		szind = sz_size2index(size);
340		slab = (szind < SC_NBINS);
341	}
342
343	if ((config_prof && opt_prof) || config_debug) {
344		rtree_ctx_t rtree_ctx_fallback;
345		rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
346		    &rtree_ctx_fallback);
347
348		rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
349		    (uintptr_t)ptr, true, &szind, &slab);
350
351		assert(szind == sz_size2index(size));
352		assert((config_prof && opt_prof) || slab == (szind < SC_NBINS));
353
354		if (config_debug) {
355			extent_t *extent = rtree_extent_read(tsdn,
356			    &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
357			assert(szind == extent_szind_get(extent));
358			assert(slab == extent_slab_get(extent));
359		}
360	}
361
362	if (likely(slab)) {
363		/* Small allocation. */
364		arena_dalloc_small(tsdn, ptr);
365	} else {
366		arena_dalloc_large_no_tcache(tsdn, ptr, szind);
367	}
368}
369
370JEMALLOC_ALWAYS_INLINE void
371arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
372    alloc_ctx_t *alloc_ctx, bool slow_path) {
373	assert(!tsdn_null(tsdn) || tcache == NULL);
374	assert(ptr != NULL);
375	assert(size <= SC_LARGE_MAXCLASS);
376
377	if (unlikely(tcache == NULL)) {
378		arena_sdalloc_no_tcache(tsdn, ptr, size);
379		return;
380	}
381
382	szind_t szind;
383	bool slab;
384	alloc_ctx_t local_ctx;
385	if (config_prof && opt_prof) {
386		if (alloc_ctx == NULL) {
387			/* Uncommon case and should be a static check. */
388			rtree_ctx_t rtree_ctx_fallback;
389			rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
390			    &rtree_ctx_fallback);
391			rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
392			    (uintptr_t)ptr, true, &local_ctx.szind,
393			    &local_ctx.slab);
394			assert(local_ctx.szind == sz_size2index(size));
395			alloc_ctx = &local_ctx;
396		}
397		slab = alloc_ctx->slab;
398		szind = alloc_ctx->szind;
399	} else {
400		/*
401		 * There is no risk of being confused by a promoted sampled
402		 * object, so base szind and slab on the given size.
403		 */
404		szind = sz_size2index(size);
405		slab = (szind < SC_NBINS);
406	}
407
408	if (config_debug) {
409		rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
410		rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
411		    (uintptr_t)ptr, true, &szind, &slab);
412		extent_t *extent = rtree_extent_read(tsdn,
413		    &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
414		assert(szind == extent_szind_get(extent));
415		assert(slab == extent_slab_get(extent));
416	}
417
418	if (likely(slab)) {
419		/* Small allocation. */
420		tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
421		    slow_path);
422	} else {
423		arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
424	}
425}
426
427#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */
428