1#define JEMALLOC_QUARANTINE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/*
5 * Quarantine pointers close to NULL are used to encode state information that
6 * is used for cleaning up during thread shutdown.
7 */
8#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1)
9#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
10#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
11
12/******************************************************************************/
13/* Function prototypes for non-inline static functions. */
14
15static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
16static void quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine);
17static void quarantine_drain(tsd_t *tsd, quarantine_t *quarantine,
18 size_t upper_bound);
19
20/******************************************************************************/
21
22static quarantine_t *
23quarantine_init(tsd_t *tsd, size_t lg_maxobjs)
24{
25 quarantine_t *quarantine;
| 1#define JEMALLOC_QUARANTINE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/*
5 * Quarantine pointers close to NULL are used to encode state information that
6 * is used for cleaning up during thread shutdown.
7 */
8#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1)
9#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
10#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
11
12/******************************************************************************/
13/* Function prototypes for non-inline static functions. */
14
15static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
16static void quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine);
17static void quarantine_drain(tsd_t *tsd, quarantine_t *quarantine,
18 size_t upper_bound);
19
20/******************************************************************************/
21
22static quarantine_t *
23quarantine_init(tsd_t *tsd, size_t lg_maxobjs)
24{
25 quarantine_t *quarantine;
|
| 26 size_t size;
|
26
27 assert(tsd_nominal(tsd));
28
| 27
28 assert(tsd_nominal(tsd));
29
|
29 quarantine = (quarantine_t *)iallocztm(tsd, offsetof(quarantine_t, objs)
30 + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)), false,
31 tcache_get(tsd, true), true, NULL);
| 30 size = offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) *
31 sizeof(quarantine_obj_t));
32 quarantine = (quarantine_t *)iallocztm(tsd, size, size2index(size),
33 false, tcache_get(tsd, true), true, NULL, true);
|
32 if (quarantine == NULL)
33 return (NULL);
34 quarantine->curbytes = 0;
35 quarantine->curobjs = 0;
36 quarantine->first = 0;
37 quarantine->lg_maxobjs = lg_maxobjs;
38
39 return (quarantine);
40}
41
42void
43quarantine_alloc_hook_work(tsd_t *tsd)
44{
45 quarantine_t *quarantine;
46
47 if (!tsd_nominal(tsd))
48 return;
49
50 quarantine = quarantine_init(tsd, LG_MAXOBJS_INIT);
51 /*
52 * Check again whether quarantine has been initialized, because
53 * quarantine_init() may have triggered recursive initialization.
54 */
55 if (tsd_quarantine_get(tsd) == NULL)
56 tsd_quarantine_set(tsd, quarantine);
57 else
| 34 if (quarantine == NULL)
35 return (NULL);
36 quarantine->curbytes = 0;
37 quarantine->curobjs = 0;
38 quarantine->first = 0;
39 quarantine->lg_maxobjs = lg_maxobjs;
40
41 return (quarantine);
42}
43
44void
45quarantine_alloc_hook_work(tsd_t *tsd)
46{
47 quarantine_t *quarantine;
48
49 if (!tsd_nominal(tsd))
50 return;
51
52 quarantine = quarantine_init(tsd, LG_MAXOBJS_INIT);
53 /*
54 * Check again whether quarantine has been initialized, because
55 * quarantine_init() may have triggered recursive initialization.
56 */
57 if (tsd_quarantine_get(tsd) == NULL)
58 tsd_quarantine_set(tsd, quarantine);
59 else
|
58 idalloctm(tsd, quarantine, tcache_get(tsd, false), true);
| 60 idalloctm(tsd, quarantine, tcache_get(tsd, false), true, true);
|
59}
60
61static quarantine_t *
62quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
63{
64 quarantine_t *ret;
65
66 ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1);
67 if (ret == NULL) {
68 quarantine_drain_one(tsd, quarantine);
69 return (quarantine);
70 }
71
72 ret->curbytes = quarantine->curbytes;
73 ret->curobjs = quarantine->curobjs;
74 if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
75 quarantine->lg_maxobjs)) {
76 /* objs ring buffer data are contiguous. */
77 memcpy(ret->objs, &quarantine->objs[quarantine->first],
78 quarantine->curobjs * sizeof(quarantine_obj_t));
79 } else {
80 /* objs ring buffer data wrap around. */
81 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
82 quarantine->first;
83 size_t ncopy_b = quarantine->curobjs - ncopy_a;
84
85 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
86 * sizeof(quarantine_obj_t));
87 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
88 sizeof(quarantine_obj_t));
89 }
| 61}
62
63static quarantine_t *
64quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
65{
66 quarantine_t *ret;
67
68 ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1);
69 if (ret == NULL) {
70 quarantine_drain_one(tsd, quarantine);
71 return (quarantine);
72 }
73
74 ret->curbytes = quarantine->curbytes;
75 ret->curobjs = quarantine->curobjs;
76 if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
77 quarantine->lg_maxobjs)) {
78 /* objs ring buffer data are contiguous. */
79 memcpy(ret->objs, &quarantine->objs[quarantine->first],
80 quarantine->curobjs * sizeof(quarantine_obj_t));
81 } else {
82 /* objs ring buffer data wrap around. */
83 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
84 quarantine->first;
85 size_t ncopy_b = quarantine->curobjs - ncopy_a;
86
87 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
88 * sizeof(quarantine_obj_t));
89 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
90 sizeof(quarantine_obj_t));
91 }
|
90 idalloctm(tsd, quarantine, tcache_get(tsd, false), true);
| 92 idalloctm(tsd, quarantine, tcache_get(tsd, false), true, true);
|
91
92 tsd_quarantine_set(tsd, ret);
93 return (ret);
94}
95
96static void
97quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine)
98{
99 quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
100 assert(obj->usize == isalloc(obj->ptr, config_prof));
| 93
94 tsd_quarantine_set(tsd, ret);
95 return (ret);
96}
97
98static void
99quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine)
100{
101 quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
102 assert(obj->usize == isalloc(obj->ptr, config_prof));
|
101 idalloctm(tsd, obj->ptr, NULL, false);
| 103 idalloctm(tsd, obj->ptr, NULL, false, true);
|
102 quarantine->curbytes -= obj->usize;
103 quarantine->curobjs--;
104 quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
105 quarantine->lg_maxobjs) - 1);
106}
107
108static void
109quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound)
110{
111
112 while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
113 quarantine_drain_one(tsd, quarantine);
114}
115
116void
117quarantine(tsd_t *tsd, void *ptr)
118{
119 quarantine_t *quarantine;
120 size_t usize = isalloc(ptr, config_prof);
121
122 cassert(config_fill);
123 assert(opt_quarantine);
124
125 if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
| 104 quarantine->curbytes -= obj->usize;
105 quarantine->curobjs--;
106 quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
107 quarantine->lg_maxobjs) - 1);
108}
109
110static void
111quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound)
112{
113
114 while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
115 quarantine_drain_one(tsd, quarantine);
116}
117
118void
119quarantine(tsd_t *tsd, void *ptr)
120{
121 quarantine_t *quarantine;
122 size_t usize = isalloc(ptr, config_prof);
123
124 cassert(config_fill);
125 assert(opt_quarantine);
126
127 if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
|
126 idalloctm(tsd, ptr, NULL, false);
| 128 idalloctm(tsd, ptr, NULL, false, true);
|
127 return;
128 }
129 /*
130 * Drain one or more objects if the quarantine size limit would be
131 * exceeded by appending ptr.
132 */
133 if (quarantine->curbytes + usize > opt_quarantine) {
134 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
135 - usize : 0;
136 quarantine_drain(tsd, quarantine, upper_bound);
137 }
138 /* Grow the quarantine ring buffer if it's full. */
139 if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
140 quarantine = quarantine_grow(tsd, quarantine);
141 /* quarantine_grow() must free a slot if it fails to grow. */
142 assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
143 /* Append ptr if its size doesn't exceed the quarantine size. */
144 if (quarantine->curbytes + usize <= opt_quarantine) {
145 size_t offset = (quarantine->first + quarantine->curobjs) &
146 ((ZU(1) << quarantine->lg_maxobjs) - 1);
147 quarantine_obj_t *obj = &quarantine->objs[offset];
148 obj->ptr = ptr;
149 obj->usize = usize;
150 quarantine->curbytes += usize;
151 quarantine->curobjs++;
152 if (config_fill && unlikely(opt_junk_free)) {
153 /*
154 * Only do redzone validation if Valgrind isn't in
155 * operation.
156 */
157 if ((!config_valgrind || likely(!in_valgrind))
158 && usize <= SMALL_MAXCLASS)
159 arena_quarantine_junk_small(ptr, usize);
160 else
161 memset(ptr, 0x5a, usize);
162 }
163 } else {
164 assert(quarantine->curbytes == 0);
| 129 return;
130 }
131 /*
132 * Drain one or more objects if the quarantine size limit would be
133 * exceeded by appending ptr.
134 */
135 if (quarantine->curbytes + usize > opt_quarantine) {
136 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
137 - usize : 0;
138 quarantine_drain(tsd, quarantine, upper_bound);
139 }
140 /* Grow the quarantine ring buffer if it's full. */
141 if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
142 quarantine = quarantine_grow(tsd, quarantine);
143 /* quarantine_grow() must free a slot if it fails to grow. */
144 assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
145 /* Append ptr if its size doesn't exceed the quarantine size. */
146 if (quarantine->curbytes + usize <= opt_quarantine) {
147 size_t offset = (quarantine->first + quarantine->curobjs) &
148 ((ZU(1) << quarantine->lg_maxobjs) - 1);
149 quarantine_obj_t *obj = &quarantine->objs[offset];
150 obj->ptr = ptr;
151 obj->usize = usize;
152 quarantine->curbytes += usize;
153 quarantine->curobjs++;
154 if (config_fill && unlikely(opt_junk_free)) {
155 /*
156 * Only do redzone validation if Valgrind isn't in
157 * operation.
158 */
159 if ((!config_valgrind || likely(!in_valgrind))
160 && usize <= SMALL_MAXCLASS)
161 arena_quarantine_junk_small(ptr, usize);
162 else
163 memset(ptr, 0x5a, usize);
164 }
165 } else {
166 assert(quarantine->curbytes == 0);
|
165 idalloctm(tsd, ptr, NULL, false);
| 167 idalloctm(tsd, ptr, NULL, false, true);
|
166 }
167}
168
169void
170quarantine_cleanup(tsd_t *tsd)
171{
172 quarantine_t *quarantine;
173
174 if (!config_fill)
175 return;
176
177 quarantine = tsd_quarantine_get(tsd);
178 if (quarantine != NULL) {
179 quarantine_drain(tsd, quarantine, 0);
| 168 }
169}
170
171void
172quarantine_cleanup(tsd_t *tsd)
173{
174 quarantine_t *quarantine;
175
176 if (!config_fill)
177 return;
178
179 quarantine = tsd_quarantine_get(tsd);
180 if (quarantine != NULL) {
181 quarantine_drain(tsd, quarantine, 0);
|
180 idalloctm(tsd, quarantine, tcache_get(tsd, false), true);
| 182 idalloctm(tsd, quarantine, tcache_get(tsd, false), true, true);
|
181 tsd_quarantine_set(tsd, NULL);
182 }
183}
| 183 tsd_quarantine_set(tsd, NULL);
184 }
185}
|