1#define	JEMALLOC_QUARANTINE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/*
5 * Quarantine pointers close to NULL are used to encode state information that
6 * is used for cleaning up during thread shutdown.
7 */
8#define	QUARANTINE_STATE_REINCARNATED	((quarantine_t *)(uintptr_t)1)
9#define	QUARANTINE_STATE_PURGATORY	((quarantine_t *)(uintptr_t)2)
10#define	QUARANTINE_STATE_MAX		QUARANTINE_STATE_PURGATORY
11
12/******************************************************************************/
13/* Function prototypes for non-inline static functions. */
14
15static quarantine_t	*quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
16static void	quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine);
17static void	quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine,
18    size_t upper_bound);
19
20/******************************************************************************/
21
22static quarantine_t *
23quarantine_init(tsdn_t *tsdn, size_t lg_maxobjs)
24{
25	quarantine_t *quarantine;
26	size_t size;
27
28	size = offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) *
29	    sizeof(quarantine_obj_t));
30	quarantine = (quarantine_t *)iallocztm(tsdn, size, size2index(size),
31	    false, NULL, true, arena_get(TSDN_NULL, 0, true), true);
32	if (quarantine == NULL)
33		return (NULL);
34	quarantine->curbytes = 0;
35	quarantine->curobjs = 0;
36	quarantine->first = 0;
37	quarantine->lg_maxobjs = lg_maxobjs;
38
39	return (quarantine);
40}
41
42void
43quarantine_alloc_hook_work(tsd_t *tsd)
44{
45	quarantine_t *quarantine;
46
47	if (!tsd_nominal(tsd))
48		return;
49
50	quarantine = quarantine_init(tsd_tsdn(tsd), LG_MAXOBJS_INIT);
51	/*
52	 * Check again whether quarantine has been initialized, because
53	 * quarantine_init() may have triggered recursive initialization.
54	 */
55	if (tsd_quarantine_get(tsd) == NULL)
56		tsd_quarantine_set(tsd, quarantine);
57	else
58		idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
59}
60
61static quarantine_t *
62quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
63{
64	quarantine_t *ret;
65
66	ret = quarantine_init(tsd_tsdn(tsd), quarantine->lg_maxobjs + 1);
67	if (ret == NULL) {
68		quarantine_drain_one(tsd_tsdn(tsd), quarantine);
69		return (quarantine);
70	}
71
72	ret->curbytes = quarantine->curbytes;
73	ret->curobjs = quarantine->curobjs;
74	if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
75	    quarantine->lg_maxobjs)) {
76		/* objs ring buffer data are contiguous. */
77		memcpy(ret->objs, &quarantine->objs[quarantine->first],
78		    quarantine->curobjs * sizeof(quarantine_obj_t));
79	} else {
80		/* objs ring buffer data wrap around. */
81		size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
82		    quarantine->first;
83		size_t ncopy_b = quarantine->curobjs - ncopy_a;
84
85		memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
86		    * sizeof(quarantine_obj_t));
87		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
88		    sizeof(quarantine_obj_t));
89	}
90	idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
91
92	tsd_quarantine_set(tsd, ret);
93	return (ret);
94}
95
96static void
97quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine)
98{
99	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
100	assert(obj->usize == isalloc(tsdn, obj->ptr, config_prof));
101	idalloctm(tsdn, obj->ptr, NULL, false, true);
102	quarantine->curbytes -= obj->usize;
103	quarantine->curobjs--;
104	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
105	    quarantine->lg_maxobjs) - 1);
106}
107
108static void
109quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine, size_t upper_bound)
110{
111
112	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
113		quarantine_drain_one(tsdn, quarantine);
114}
115
116void
117quarantine(tsd_t *tsd, void *ptr)
118{
119	quarantine_t *quarantine;
120	size_t usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
121
122	cassert(config_fill);
123	assert(opt_quarantine);
124
125	if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
126		idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true);
127		return;
128	}
129	/*
130	 * Drain one or more objects if the quarantine size limit would be
131	 * exceeded by appending ptr.
132	 */
133	if (quarantine->curbytes + usize > opt_quarantine) {
134		size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
135		    - usize : 0;
136		quarantine_drain(tsd_tsdn(tsd), quarantine, upper_bound);
137	}
138	/* Grow the quarantine ring buffer if it's full. */
139	if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
140		quarantine = quarantine_grow(tsd, quarantine);
141	/* quarantine_grow() must free a slot if it fails to grow. */
142	assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
143	/* Append ptr if its size doesn't exceed the quarantine size. */
144	if (quarantine->curbytes + usize <= opt_quarantine) {
145		size_t offset = (quarantine->first + quarantine->curobjs) &
146		    ((ZU(1) << quarantine->lg_maxobjs) - 1);
147		quarantine_obj_t *obj = &quarantine->objs[offset];
148		obj->ptr = ptr;
149		obj->usize = usize;
150		quarantine->curbytes += usize;
151		quarantine->curobjs++;
152		if (config_fill && unlikely(opt_junk_free)) {
153			/*
154			 * Only do redzone validation if Valgrind isn't in
155			 * operation.
156			 */
157			if ((!config_valgrind || likely(!in_valgrind))
158			    && usize <= SMALL_MAXCLASS)
159				arena_quarantine_junk_small(ptr, usize);
160			else
161				memset(ptr, JEMALLOC_FREE_JUNK, usize);
162		}
163	} else {
164		assert(quarantine->curbytes == 0);
165		idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true);
166	}
167}
168
169void
170quarantine_cleanup(tsd_t *tsd)
171{
172	quarantine_t *quarantine;
173
174	if (!config_fill)
175		return;
176
177	quarantine = tsd_quarantine_get(tsd);
178	if (quarantine != NULL) {
179		quarantine_drain(tsd_tsdn(tsd), quarantine, 0);
180		idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
181		tsd_quarantine_set(tsd, NULL);
182	}
183}
184