1#define JEMALLOC_PROF_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3/******************************************************************************/
4
5#ifdef JEMALLOC_PROF_LIBUNWIND
6#define UNW_LOCAL_ONLY
7#include <libunwind.h>
8#endif
9
10#ifdef JEMALLOC_PROF_LIBGCC
11#include <unwind.h>
12#endif
13
14/******************************************************************************/
15/* Data. */
16
17bool opt_prof = false;
18bool opt_prof_active = true;
19bool opt_prof_thread_active_init = true;
20size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
21ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
22bool opt_prof_gdump = false;
23bool opt_prof_final = false;
24bool opt_prof_leak = false;
25bool opt_prof_accum = false;
26char opt_prof_prefix[
27 /* Minimize memory bloat for non-prof builds. */
28#ifdef JEMALLOC_PROF
29 PATH_MAX +
30#endif
31 1];
32
33/*
34 * Initialized as opt_prof_active, and accessed via
35 * prof_active_[gs]et{_unlocked,}().
36 */
37bool prof_active;
38static malloc_mutex_t prof_active_mtx;
39
40/*
41 * Initialized as opt_prof_thread_active_init, and accessed via
42 * prof_thread_active_init_[gs]et().
43 */
44static bool prof_thread_active_init;
45static malloc_mutex_t prof_thread_active_init_mtx;
46
47/*
48 * Initialized as opt_prof_gdump, and accessed via
49 * prof_gdump_[gs]et{_unlocked,}().
50 */
51bool prof_gdump_val;
52static malloc_mutex_t prof_gdump_mtx;
53
54uint64_t prof_interval = 0;
55
56size_t lg_prof_sample;
57
58/*
59 * Table of mutexes that are shared among gctx's. These are leaf locks, so
60 * there is no problem with using them for more than one gctx at the same time.
61 * The primary motivation for this sharing though is that gctx's are ephemeral,
62 * and destroying mutexes causes complications for systems that allocate when
63 * creating/destroying mutexes.
64 */
65static malloc_mutex_t *gctx_locks;
66static unsigned cum_gctxs; /* Atomic counter. */
67
68/*
69 * Table of mutexes that are shared among tdata's. No operations require
70 * holding multiple tdata locks, so there is no problem with using them for more
71 * than one tdata at the same time, even though a gctx lock may be acquired
72 * while holding a tdata lock.
73 */
74static malloc_mutex_t *tdata_locks;
75
76/*
77 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
78 * structure that knows about all backtraces currently captured.
79 */
80static ckh_t bt2gctx;
81static malloc_mutex_t bt2gctx_mtx;
82
83/*
84 * Tree of all extant prof_tdata_t structures, regardless of state,
85 * {attached,detached,expired}.
86 */
87static prof_tdata_tree_t tdatas;
88static malloc_mutex_t tdatas_mtx;
89
90static uint64_t next_thr_uid;
91static malloc_mutex_t next_thr_uid_mtx;
92
93static malloc_mutex_t prof_dump_seq_mtx;
94static uint64_t prof_dump_seq;
95static uint64_t prof_dump_iseq;
96static uint64_t prof_dump_mseq;
97static uint64_t prof_dump_useq;
98
99/*
100 * This buffer is rather large for stack allocation, so use a single buffer for
101 * all profile dumps.
102 */
103static malloc_mutex_t prof_dump_mtx;
104static char prof_dump_buf[
105 /* Minimize memory bloat for non-prof builds. */
106#ifdef JEMALLOC_PROF
107 PROF_DUMP_BUFSIZE
108#else
109 1
110#endif
111];
| 1#define JEMALLOC_PROF_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3/******************************************************************************/
4
5#ifdef JEMALLOC_PROF_LIBUNWIND
6#define UNW_LOCAL_ONLY
7#include <libunwind.h>
8#endif
9
10#ifdef JEMALLOC_PROF_LIBGCC
11#include <unwind.h>
12#endif
13
14/******************************************************************************/
15/* Data. */
16
17bool opt_prof = false;
18bool opt_prof_active = true;
19bool opt_prof_thread_active_init = true;
20size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
21ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
22bool opt_prof_gdump = false;
23bool opt_prof_final = false;
24bool opt_prof_leak = false;
25bool opt_prof_accum = false;
26char opt_prof_prefix[
27 /* Minimize memory bloat for non-prof builds. */
28#ifdef JEMALLOC_PROF
29 PATH_MAX +
30#endif
31 1];
32
33/*
34 * Initialized as opt_prof_active, and accessed via
35 * prof_active_[gs]et{_unlocked,}().
36 */
37bool prof_active;
38static malloc_mutex_t prof_active_mtx;
39
40/*
41 * Initialized as opt_prof_thread_active_init, and accessed via
42 * prof_thread_active_init_[gs]et().
43 */
44static bool prof_thread_active_init;
45static malloc_mutex_t prof_thread_active_init_mtx;
46
47/*
48 * Initialized as opt_prof_gdump, and accessed via
49 * prof_gdump_[gs]et{_unlocked,}().
50 */
51bool prof_gdump_val;
52static malloc_mutex_t prof_gdump_mtx;
53
54uint64_t prof_interval = 0;
55
56size_t lg_prof_sample;
57
58/*
59 * Table of mutexes that are shared among gctx's. These are leaf locks, so
60 * there is no problem with using them for more than one gctx at the same time.
61 * The primary motivation for this sharing though is that gctx's are ephemeral,
62 * and destroying mutexes causes complications for systems that allocate when
63 * creating/destroying mutexes.
64 */
65static malloc_mutex_t *gctx_locks;
66static unsigned cum_gctxs; /* Atomic counter. */
67
68/*
69 * Table of mutexes that are shared among tdata's. No operations require
70 * holding multiple tdata locks, so there is no problem with using them for more
71 * than one tdata at the same time, even though a gctx lock may be acquired
72 * while holding a tdata lock.
73 */
74static malloc_mutex_t *tdata_locks;
75
76/*
77 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
78 * structure that knows about all backtraces currently captured.
79 */
80static ckh_t bt2gctx;
81static malloc_mutex_t bt2gctx_mtx;
82
83/*
84 * Tree of all extant prof_tdata_t structures, regardless of state,
85 * {attached,detached,expired}.
86 */
87static prof_tdata_tree_t tdatas;
88static malloc_mutex_t tdatas_mtx;
89
90static uint64_t next_thr_uid;
91static malloc_mutex_t next_thr_uid_mtx;
92
93static malloc_mutex_t prof_dump_seq_mtx;
94static uint64_t prof_dump_seq;
95static uint64_t prof_dump_iseq;
96static uint64_t prof_dump_mseq;
97static uint64_t prof_dump_useq;
98
99/*
100 * This buffer is rather large for stack allocation, so use a single buffer for
101 * all profile dumps.
102 */
103static malloc_mutex_t prof_dump_mtx;
104static char prof_dump_buf[
105 /* Minimize memory bloat for non-prof builds. */
106#ifdef JEMALLOC_PROF
107 PROF_DUMP_BUFSIZE
108#else
109 1
110#endif
111];
|
112static unsigned prof_dump_buf_end;
| 112static size_t prof_dump_buf_end;
|
113static int prof_dump_fd;
114
115/* Do not dump any profiles until bootstrapping is complete. */
116static bool prof_booted = false;
117
118/******************************************************************************/
119/*
120 * Function prototypes for static functions that are referenced prior to
121 * definition.
122 */
123
124static bool prof_tctx_should_destroy(prof_tctx_t *tctx);
125static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
126static bool prof_tdata_should_destroy(prof_tdata_t *tdata,
127 bool even_if_attached);
128static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
129 bool even_if_attached);
130static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name);
131
132/******************************************************************************/
133/* Red-black trees. */
134
135JEMALLOC_INLINE_C int
136prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b)
137{
138 uint64_t a_thr_uid = a->thr_uid;
139 uint64_t b_thr_uid = b->thr_uid;
140 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
141 if (ret == 0) {
142 uint64_t a_thr_discrim = a->thr_discrim;
143 uint64_t b_thr_discrim = b->thr_discrim;
144 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
145 b_thr_discrim);
146 if (ret == 0) {
147 uint64_t a_tctx_uid = a->tctx_uid;
148 uint64_t b_tctx_uid = b->tctx_uid;
149 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
150 b_tctx_uid);
151 }
152 }
153 return (ret);
154}
155
156rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
157 tctx_link, prof_tctx_comp)
158
159JEMALLOC_INLINE_C int
160prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b)
161{
162 unsigned a_len = a->bt.len;
163 unsigned b_len = b->bt.len;
164 unsigned comp_len = (a_len < b_len) ? a_len : b_len;
165 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
166 if (ret == 0)
167 ret = (a_len > b_len) - (a_len < b_len);
168 return (ret);
169}
170
171rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
172 prof_gctx_comp)
173
174JEMALLOC_INLINE_C int
175prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b)
176{
177 int ret;
178 uint64_t a_uid = a->thr_uid;
179 uint64_t b_uid = b->thr_uid;
180
181 ret = ((a_uid > b_uid) - (a_uid < b_uid));
182 if (ret == 0) {
183 uint64_t a_discrim = a->thr_discrim;
184 uint64_t b_discrim = b->thr_discrim;
185
186 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
187 }
188 return (ret);
189}
190
191rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
192 prof_tdata_comp)
193
194/******************************************************************************/
195
196void
197prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
198{
199 prof_tdata_t *tdata;
200
201 cassert(config_prof);
202
203 if (updated) {
204 /*
205 * Compute a new sample threshold. This isn't very important in
206 * practice, because this function is rarely executed, so the
207 * potential for sample bias is minimal except in contrived
208 * programs.
209 */
210 tdata = prof_tdata_get(tsd, true);
211 if (tdata != NULL)
212 prof_sample_threshold_update(tdata);
213 }
214
215 if ((uintptr_t)tctx > (uintptr_t)1U) {
216 malloc_mutex_lock(tctx->tdata->lock);
217 tctx->prepared = false;
218 if (prof_tctx_should_destroy(tctx))
219 prof_tctx_destroy(tsd, tctx);
220 else
221 malloc_mutex_unlock(tctx->tdata->lock);
222 }
223}
224
225void
226prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx)
227{
228
229 prof_tctx_set(ptr, usize, tctx);
230
231 malloc_mutex_lock(tctx->tdata->lock);
232 tctx->cnts.curobjs++;
233 tctx->cnts.curbytes += usize;
234 if (opt_prof_accum) {
235 tctx->cnts.accumobjs++;
236 tctx->cnts.accumbytes += usize;
237 }
238 tctx->prepared = false;
239 malloc_mutex_unlock(tctx->tdata->lock);
240}
241
242void
243prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
244{
245
246 malloc_mutex_lock(tctx->tdata->lock);
247 assert(tctx->cnts.curobjs > 0);
248 assert(tctx->cnts.curbytes >= usize);
249 tctx->cnts.curobjs--;
250 tctx->cnts.curbytes -= usize;
251
252 if (prof_tctx_should_destroy(tctx))
253 prof_tctx_destroy(tsd, tctx);
254 else
255 malloc_mutex_unlock(tctx->tdata->lock);
256}
257
258void
259bt_init(prof_bt_t *bt, void **vec)
260{
261
262 cassert(config_prof);
263
264 bt->vec = vec;
265 bt->len = 0;
266}
267
268JEMALLOC_INLINE_C void
269prof_enter(tsd_t *tsd, prof_tdata_t *tdata)
270{
271
272 cassert(config_prof);
273 assert(tdata == prof_tdata_get(tsd, false));
274
275 if (tdata != NULL) {
276 assert(!tdata->enq);
277 tdata->enq = true;
278 }
279
280 malloc_mutex_lock(&bt2gctx_mtx);
281}
282
283JEMALLOC_INLINE_C void
284prof_leave(tsd_t *tsd, prof_tdata_t *tdata)
285{
286
287 cassert(config_prof);
288 assert(tdata == prof_tdata_get(tsd, false));
289
290 malloc_mutex_unlock(&bt2gctx_mtx);
291
292 if (tdata != NULL) {
293 bool idump, gdump;
294
295 assert(tdata->enq);
296 tdata->enq = false;
297 idump = tdata->enq_idump;
298 tdata->enq_idump = false;
299 gdump = tdata->enq_gdump;
300 tdata->enq_gdump = false;
301
302 if (idump)
303 prof_idump();
304 if (gdump)
305 prof_gdump();
306 }
307}
308
309#ifdef JEMALLOC_PROF_LIBUNWIND
310void
311prof_backtrace(prof_bt_t *bt)
312{
313 int nframes;
314
315 cassert(config_prof);
316 assert(bt->len == 0);
317 assert(bt->vec != NULL);
318
319 nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
320 if (nframes <= 0)
321 return;
322 bt->len = nframes;
323}
324#elif (defined(JEMALLOC_PROF_LIBGCC))
325static _Unwind_Reason_Code
326prof_unwind_init_callback(struct _Unwind_Context *context, void *arg)
327{
328
329 cassert(config_prof);
330
331 return (_URC_NO_REASON);
332}
333
334static _Unwind_Reason_Code
335prof_unwind_callback(struct _Unwind_Context *context, void *arg)
336{
337 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
338 void *ip;
339
340 cassert(config_prof);
341
342 ip = (void *)_Unwind_GetIP(context);
343 if (ip == NULL)
344 return (_URC_END_OF_STACK);
345 data->bt->vec[data->bt->len] = ip;
346 data->bt->len++;
347 if (data->bt->len == data->max)
348 return (_URC_END_OF_STACK);
349
350 return (_URC_NO_REASON);
351}
352
353void
354prof_backtrace(prof_bt_t *bt)
355{
356 prof_unwind_data_t data = {bt, PROF_BT_MAX};
357
358 cassert(config_prof);
359
360 _Unwind_Backtrace(prof_unwind_callback, &data);
361}
362#elif (defined(JEMALLOC_PROF_GCC))
363void
364prof_backtrace(prof_bt_t *bt)
365{
366#define BT_FRAME(i) \
367 if ((i) < PROF_BT_MAX) { \
368 void *p; \
369 if (__builtin_frame_address(i) == 0) \
370 return; \
371 p = __builtin_return_address(i); \
372 if (p == NULL) \
373 return; \
374 bt->vec[(i)] = p; \
375 bt->len = (i) + 1; \
376 } else \
377 return;
378
379 cassert(config_prof);
380
381 BT_FRAME(0)
382 BT_FRAME(1)
383 BT_FRAME(2)
384 BT_FRAME(3)
385 BT_FRAME(4)
386 BT_FRAME(5)
387 BT_FRAME(6)
388 BT_FRAME(7)
389 BT_FRAME(8)
390 BT_FRAME(9)
391
392 BT_FRAME(10)
393 BT_FRAME(11)
394 BT_FRAME(12)
395 BT_FRAME(13)
396 BT_FRAME(14)
397 BT_FRAME(15)
398 BT_FRAME(16)
399 BT_FRAME(17)
400 BT_FRAME(18)
401 BT_FRAME(19)
402
403 BT_FRAME(20)
404 BT_FRAME(21)
405 BT_FRAME(22)
406 BT_FRAME(23)
407 BT_FRAME(24)
408 BT_FRAME(25)
409 BT_FRAME(26)
410 BT_FRAME(27)
411 BT_FRAME(28)
412 BT_FRAME(29)
413
414 BT_FRAME(30)
415 BT_FRAME(31)
416 BT_FRAME(32)
417 BT_FRAME(33)
418 BT_FRAME(34)
419 BT_FRAME(35)
420 BT_FRAME(36)
421 BT_FRAME(37)
422 BT_FRAME(38)
423 BT_FRAME(39)
424
425 BT_FRAME(40)
426 BT_FRAME(41)
427 BT_FRAME(42)
428 BT_FRAME(43)
429 BT_FRAME(44)
430 BT_FRAME(45)
431 BT_FRAME(46)
432 BT_FRAME(47)
433 BT_FRAME(48)
434 BT_FRAME(49)
435
436 BT_FRAME(50)
437 BT_FRAME(51)
438 BT_FRAME(52)
439 BT_FRAME(53)
440 BT_FRAME(54)
441 BT_FRAME(55)
442 BT_FRAME(56)
443 BT_FRAME(57)
444 BT_FRAME(58)
445 BT_FRAME(59)
446
447 BT_FRAME(60)
448 BT_FRAME(61)
449 BT_FRAME(62)
450 BT_FRAME(63)
451 BT_FRAME(64)
452 BT_FRAME(65)
453 BT_FRAME(66)
454 BT_FRAME(67)
455 BT_FRAME(68)
456 BT_FRAME(69)
457
458 BT_FRAME(70)
459 BT_FRAME(71)
460 BT_FRAME(72)
461 BT_FRAME(73)
462 BT_FRAME(74)
463 BT_FRAME(75)
464 BT_FRAME(76)
465 BT_FRAME(77)
466 BT_FRAME(78)
467 BT_FRAME(79)
468
469 BT_FRAME(80)
470 BT_FRAME(81)
471 BT_FRAME(82)
472 BT_FRAME(83)
473 BT_FRAME(84)
474 BT_FRAME(85)
475 BT_FRAME(86)
476 BT_FRAME(87)
477 BT_FRAME(88)
478 BT_FRAME(89)
479
480 BT_FRAME(90)
481 BT_FRAME(91)
482 BT_FRAME(92)
483 BT_FRAME(93)
484 BT_FRAME(94)
485 BT_FRAME(95)
486 BT_FRAME(96)
487 BT_FRAME(97)
488 BT_FRAME(98)
489 BT_FRAME(99)
490
491 BT_FRAME(100)
492 BT_FRAME(101)
493 BT_FRAME(102)
494 BT_FRAME(103)
495 BT_FRAME(104)
496 BT_FRAME(105)
497 BT_FRAME(106)
498 BT_FRAME(107)
499 BT_FRAME(108)
500 BT_FRAME(109)
501
502 BT_FRAME(110)
503 BT_FRAME(111)
504 BT_FRAME(112)
505 BT_FRAME(113)
506 BT_FRAME(114)
507 BT_FRAME(115)
508 BT_FRAME(116)
509 BT_FRAME(117)
510 BT_FRAME(118)
511 BT_FRAME(119)
512
513 BT_FRAME(120)
514 BT_FRAME(121)
515 BT_FRAME(122)
516 BT_FRAME(123)
517 BT_FRAME(124)
518 BT_FRAME(125)
519 BT_FRAME(126)
520 BT_FRAME(127)
521#undef BT_FRAME
522}
523#else
524void
525prof_backtrace(prof_bt_t *bt)
526{
527
528 cassert(config_prof);
529 not_reached();
530}
531#endif
532
533static malloc_mutex_t *
534prof_gctx_mutex_choose(void)
535{
536 unsigned ngctxs = atomic_add_u(&cum_gctxs, 1);
537
538 return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]);
539}
540
541static malloc_mutex_t *
542prof_tdata_mutex_choose(uint64_t thr_uid)
543{
544
545 return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]);
546}
547
548static prof_gctx_t *
549prof_gctx_create(tsd_t *tsd, prof_bt_t *bt)
550{
551 /*
552 * Create a single allocation that has space for vec of length bt->len.
553 */
| 113static int prof_dump_fd;
114
115/* Do not dump any profiles until bootstrapping is complete. */
116static bool prof_booted = false;
117
118/******************************************************************************/
119/*
120 * Function prototypes for static functions that are referenced prior to
121 * definition.
122 */
123
124static bool prof_tctx_should_destroy(prof_tctx_t *tctx);
125static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
126static bool prof_tdata_should_destroy(prof_tdata_t *tdata,
127 bool even_if_attached);
128static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
129 bool even_if_attached);
130static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name);
131
132/******************************************************************************/
133/* Red-black trees. */
134
135JEMALLOC_INLINE_C int
136prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b)
137{
138 uint64_t a_thr_uid = a->thr_uid;
139 uint64_t b_thr_uid = b->thr_uid;
140 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
141 if (ret == 0) {
142 uint64_t a_thr_discrim = a->thr_discrim;
143 uint64_t b_thr_discrim = b->thr_discrim;
144 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
145 b_thr_discrim);
146 if (ret == 0) {
147 uint64_t a_tctx_uid = a->tctx_uid;
148 uint64_t b_tctx_uid = b->tctx_uid;
149 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
150 b_tctx_uid);
151 }
152 }
153 return (ret);
154}
155
156rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
157 tctx_link, prof_tctx_comp)
158
159JEMALLOC_INLINE_C int
160prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b)
161{
162 unsigned a_len = a->bt.len;
163 unsigned b_len = b->bt.len;
164 unsigned comp_len = (a_len < b_len) ? a_len : b_len;
165 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
166 if (ret == 0)
167 ret = (a_len > b_len) - (a_len < b_len);
168 return (ret);
169}
170
171rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
172 prof_gctx_comp)
173
174JEMALLOC_INLINE_C int
175prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b)
176{
177 int ret;
178 uint64_t a_uid = a->thr_uid;
179 uint64_t b_uid = b->thr_uid;
180
181 ret = ((a_uid > b_uid) - (a_uid < b_uid));
182 if (ret == 0) {
183 uint64_t a_discrim = a->thr_discrim;
184 uint64_t b_discrim = b->thr_discrim;
185
186 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
187 }
188 return (ret);
189}
190
191rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
192 prof_tdata_comp)
193
194/******************************************************************************/
195
196void
197prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
198{
199 prof_tdata_t *tdata;
200
201 cassert(config_prof);
202
203 if (updated) {
204 /*
205 * Compute a new sample threshold. This isn't very important in
206 * practice, because this function is rarely executed, so the
207 * potential for sample bias is minimal except in contrived
208 * programs.
209 */
210 tdata = prof_tdata_get(tsd, true);
211 if (tdata != NULL)
212 prof_sample_threshold_update(tdata);
213 }
214
215 if ((uintptr_t)tctx > (uintptr_t)1U) {
216 malloc_mutex_lock(tctx->tdata->lock);
217 tctx->prepared = false;
218 if (prof_tctx_should_destroy(tctx))
219 prof_tctx_destroy(tsd, tctx);
220 else
221 malloc_mutex_unlock(tctx->tdata->lock);
222 }
223}
224
225void
226prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx)
227{
228
229 prof_tctx_set(ptr, usize, tctx);
230
231 malloc_mutex_lock(tctx->tdata->lock);
232 tctx->cnts.curobjs++;
233 tctx->cnts.curbytes += usize;
234 if (opt_prof_accum) {
235 tctx->cnts.accumobjs++;
236 tctx->cnts.accumbytes += usize;
237 }
238 tctx->prepared = false;
239 malloc_mutex_unlock(tctx->tdata->lock);
240}
241
242void
243prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
244{
245
246 malloc_mutex_lock(tctx->tdata->lock);
247 assert(tctx->cnts.curobjs > 0);
248 assert(tctx->cnts.curbytes >= usize);
249 tctx->cnts.curobjs--;
250 tctx->cnts.curbytes -= usize;
251
252 if (prof_tctx_should_destroy(tctx))
253 prof_tctx_destroy(tsd, tctx);
254 else
255 malloc_mutex_unlock(tctx->tdata->lock);
256}
257
258void
259bt_init(prof_bt_t *bt, void **vec)
260{
261
262 cassert(config_prof);
263
264 bt->vec = vec;
265 bt->len = 0;
266}
267
268JEMALLOC_INLINE_C void
269prof_enter(tsd_t *tsd, prof_tdata_t *tdata)
270{
271
272 cassert(config_prof);
273 assert(tdata == prof_tdata_get(tsd, false));
274
275 if (tdata != NULL) {
276 assert(!tdata->enq);
277 tdata->enq = true;
278 }
279
280 malloc_mutex_lock(&bt2gctx_mtx);
281}
282
283JEMALLOC_INLINE_C void
284prof_leave(tsd_t *tsd, prof_tdata_t *tdata)
285{
286
287 cassert(config_prof);
288 assert(tdata == prof_tdata_get(tsd, false));
289
290 malloc_mutex_unlock(&bt2gctx_mtx);
291
292 if (tdata != NULL) {
293 bool idump, gdump;
294
295 assert(tdata->enq);
296 tdata->enq = false;
297 idump = tdata->enq_idump;
298 tdata->enq_idump = false;
299 gdump = tdata->enq_gdump;
300 tdata->enq_gdump = false;
301
302 if (idump)
303 prof_idump();
304 if (gdump)
305 prof_gdump();
306 }
307}
308
309#ifdef JEMALLOC_PROF_LIBUNWIND
310void
311prof_backtrace(prof_bt_t *bt)
312{
313 int nframes;
314
315 cassert(config_prof);
316 assert(bt->len == 0);
317 assert(bt->vec != NULL);
318
319 nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
320 if (nframes <= 0)
321 return;
322 bt->len = nframes;
323}
324#elif (defined(JEMALLOC_PROF_LIBGCC))
325static _Unwind_Reason_Code
326prof_unwind_init_callback(struct _Unwind_Context *context, void *arg)
327{
328
329 cassert(config_prof);
330
331 return (_URC_NO_REASON);
332}
333
334static _Unwind_Reason_Code
335prof_unwind_callback(struct _Unwind_Context *context, void *arg)
336{
337 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
338 void *ip;
339
340 cassert(config_prof);
341
342 ip = (void *)_Unwind_GetIP(context);
343 if (ip == NULL)
344 return (_URC_END_OF_STACK);
345 data->bt->vec[data->bt->len] = ip;
346 data->bt->len++;
347 if (data->bt->len == data->max)
348 return (_URC_END_OF_STACK);
349
350 return (_URC_NO_REASON);
351}
352
353void
354prof_backtrace(prof_bt_t *bt)
355{
356 prof_unwind_data_t data = {bt, PROF_BT_MAX};
357
358 cassert(config_prof);
359
360 _Unwind_Backtrace(prof_unwind_callback, &data);
361}
362#elif (defined(JEMALLOC_PROF_GCC))
363void
364prof_backtrace(prof_bt_t *bt)
365{
366#define BT_FRAME(i) \
367 if ((i) < PROF_BT_MAX) { \
368 void *p; \
369 if (__builtin_frame_address(i) == 0) \
370 return; \
371 p = __builtin_return_address(i); \
372 if (p == NULL) \
373 return; \
374 bt->vec[(i)] = p; \
375 bt->len = (i) + 1; \
376 } else \
377 return;
378
379 cassert(config_prof);
380
381 BT_FRAME(0)
382 BT_FRAME(1)
383 BT_FRAME(2)
384 BT_FRAME(3)
385 BT_FRAME(4)
386 BT_FRAME(5)
387 BT_FRAME(6)
388 BT_FRAME(7)
389 BT_FRAME(8)
390 BT_FRAME(9)
391
392 BT_FRAME(10)
393 BT_FRAME(11)
394 BT_FRAME(12)
395 BT_FRAME(13)
396 BT_FRAME(14)
397 BT_FRAME(15)
398 BT_FRAME(16)
399 BT_FRAME(17)
400 BT_FRAME(18)
401 BT_FRAME(19)
402
403 BT_FRAME(20)
404 BT_FRAME(21)
405 BT_FRAME(22)
406 BT_FRAME(23)
407 BT_FRAME(24)
408 BT_FRAME(25)
409 BT_FRAME(26)
410 BT_FRAME(27)
411 BT_FRAME(28)
412 BT_FRAME(29)
413
414 BT_FRAME(30)
415 BT_FRAME(31)
416 BT_FRAME(32)
417 BT_FRAME(33)
418 BT_FRAME(34)
419 BT_FRAME(35)
420 BT_FRAME(36)
421 BT_FRAME(37)
422 BT_FRAME(38)
423 BT_FRAME(39)
424
425 BT_FRAME(40)
426 BT_FRAME(41)
427 BT_FRAME(42)
428 BT_FRAME(43)
429 BT_FRAME(44)
430 BT_FRAME(45)
431 BT_FRAME(46)
432 BT_FRAME(47)
433 BT_FRAME(48)
434 BT_FRAME(49)
435
436 BT_FRAME(50)
437 BT_FRAME(51)
438 BT_FRAME(52)
439 BT_FRAME(53)
440 BT_FRAME(54)
441 BT_FRAME(55)
442 BT_FRAME(56)
443 BT_FRAME(57)
444 BT_FRAME(58)
445 BT_FRAME(59)
446
447 BT_FRAME(60)
448 BT_FRAME(61)
449 BT_FRAME(62)
450 BT_FRAME(63)
451 BT_FRAME(64)
452 BT_FRAME(65)
453 BT_FRAME(66)
454 BT_FRAME(67)
455 BT_FRAME(68)
456 BT_FRAME(69)
457
458 BT_FRAME(70)
459 BT_FRAME(71)
460 BT_FRAME(72)
461 BT_FRAME(73)
462 BT_FRAME(74)
463 BT_FRAME(75)
464 BT_FRAME(76)
465 BT_FRAME(77)
466 BT_FRAME(78)
467 BT_FRAME(79)
468
469 BT_FRAME(80)
470 BT_FRAME(81)
471 BT_FRAME(82)
472 BT_FRAME(83)
473 BT_FRAME(84)
474 BT_FRAME(85)
475 BT_FRAME(86)
476 BT_FRAME(87)
477 BT_FRAME(88)
478 BT_FRAME(89)
479
480 BT_FRAME(90)
481 BT_FRAME(91)
482 BT_FRAME(92)
483 BT_FRAME(93)
484 BT_FRAME(94)
485 BT_FRAME(95)
486 BT_FRAME(96)
487 BT_FRAME(97)
488 BT_FRAME(98)
489 BT_FRAME(99)
490
491 BT_FRAME(100)
492 BT_FRAME(101)
493 BT_FRAME(102)
494 BT_FRAME(103)
495 BT_FRAME(104)
496 BT_FRAME(105)
497 BT_FRAME(106)
498 BT_FRAME(107)
499 BT_FRAME(108)
500 BT_FRAME(109)
501
502 BT_FRAME(110)
503 BT_FRAME(111)
504 BT_FRAME(112)
505 BT_FRAME(113)
506 BT_FRAME(114)
507 BT_FRAME(115)
508 BT_FRAME(116)
509 BT_FRAME(117)
510 BT_FRAME(118)
511 BT_FRAME(119)
512
513 BT_FRAME(120)
514 BT_FRAME(121)
515 BT_FRAME(122)
516 BT_FRAME(123)
517 BT_FRAME(124)
518 BT_FRAME(125)
519 BT_FRAME(126)
520 BT_FRAME(127)
521#undef BT_FRAME
522}
523#else
524void
525prof_backtrace(prof_bt_t *bt)
526{
527
528 cassert(config_prof);
529 not_reached();
530}
531#endif
532
533static malloc_mutex_t *
534prof_gctx_mutex_choose(void)
535{
536 unsigned ngctxs = atomic_add_u(&cum_gctxs, 1);
537
538 return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]);
539}
540
541static malloc_mutex_t *
542prof_tdata_mutex_choose(uint64_t thr_uid)
543{
544
545 return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]);
546}
547
548static prof_gctx_t *
549prof_gctx_create(tsd_t *tsd, prof_bt_t *bt)
550{
551 /*
552 * Create a single allocation that has space for vec of length bt->len.
553 */
|
554 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t,
555 vec) + (bt->len * sizeof(void *)), false, tcache_get(tsd, true),
556 true, NULL);
| 554 size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
555 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, size,
556 size2index(size), false, tcache_get(tsd, true), true, NULL, true);
|
557 if (gctx == NULL)
558 return (NULL);
559 gctx->lock = prof_gctx_mutex_choose();
560 /*
561 * Set nlimbo to 1, in order to avoid a race condition with
562 * prof_tctx_destroy()/prof_gctx_try_destroy().
563 */
564 gctx->nlimbo = 1;
565 tctx_tree_new(&gctx->tctxs);
566 /* Duplicate bt. */
567 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
568 gctx->bt.vec = gctx->vec;
569 gctx->bt.len = bt->len;
570 return (gctx);
571}
572
573static void
574prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
575 prof_tdata_t *tdata)
576{
577
578 cassert(config_prof);
579
580 /*
581 * Check that gctx is still unused by any thread cache before destroying
582 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
583 * condition with this function, as does prof_tctx_destroy() in order to
584 * avoid a race between the main body of prof_tctx_destroy() and entry
585 * into this function.
586 */
587 prof_enter(tsd, tdata_self);
588 malloc_mutex_lock(gctx->lock);
589 assert(gctx->nlimbo != 0);
590 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
591 /* Remove gctx from bt2gctx. */
592 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
593 not_reached();
594 prof_leave(tsd, tdata_self);
595 /* Destroy gctx. */
596 malloc_mutex_unlock(gctx->lock);
| 557 if (gctx == NULL)
558 return (NULL);
559 gctx->lock = prof_gctx_mutex_choose();
560 /*
561 * Set nlimbo to 1, in order to avoid a race condition with
562 * prof_tctx_destroy()/prof_gctx_try_destroy().
563 */
564 gctx->nlimbo = 1;
565 tctx_tree_new(&gctx->tctxs);
566 /* Duplicate bt. */
567 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
568 gctx->bt.vec = gctx->vec;
569 gctx->bt.len = bt->len;
570 return (gctx);
571}
572
573static void
574prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
575 prof_tdata_t *tdata)
576{
577
578 cassert(config_prof);
579
580 /*
581 * Check that gctx is still unused by any thread cache before destroying
582 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
583 * condition with this function, as does prof_tctx_destroy() in order to
584 * avoid a race between the main body of prof_tctx_destroy() and entry
585 * into this function.
586 */
587 prof_enter(tsd, tdata_self);
588 malloc_mutex_lock(gctx->lock);
589 assert(gctx->nlimbo != 0);
590 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
591 /* Remove gctx from bt2gctx. */
592 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
593 not_reached();
594 prof_leave(tsd, tdata_self);
595 /* Destroy gctx. */
596 malloc_mutex_unlock(gctx->lock);
|
597 idalloctm(tsd, gctx, tcache_get(tsd, false), true);
| 597 idalloctm(tsd, gctx, tcache_get(tsd, false), true, true);
|
598 } else {
599 /*
600 * Compensate for increment in prof_tctx_destroy() or
601 * prof_lookup().
602 */
603 gctx->nlimbo--;
604 malloc_mutex_unlock(gctx->lock);
605 prof_leave(tsd, tdata_self);
606 }
607}
608
609/* tctx->tdata->lock must be held. */
610static bool
611prof_tctx_should_destroy(prof_tctx_t *tctx)
612{
613
614 if (opt_prof_accum)
615 return (false);
616 if (tctx->cnts.curobjs != 0)
617 return (false);
618 if (tctx->prepared)
619 return (false);
620 return (true);
621}
622
623static bool
624prof_gctx_should_destroy(prof_gctx_t *gctx)
625{
626
627 if (opt_prof_accum)
628 return (false);
629 if (!tctx_tree_empty(&gctx->tctxs))
630 return (false);
631 if (gctx->nlimbo != 0)
632 return (false);
633 return (true);
634}
635
636/* tctx->tdata->lock is held upon entry, and released before return. */
637static void
638prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
639{
640 prof_tdata_t *tdata = tctx->tdata;
641 prof_gctx_t *gctx = tctx->gctx;
642 bool destroy_tdata, destroy_tctx, destroy_gctx;
643
644 assert(tctx->cnts.curobjs == 0);
645 assert(tctx->cnts.curbytes == 0);
646 assert(!opt_prof_accum);
647 assert(tctx->cnts.accumobjs == 0);
648 assert(tctx->cnts.accumbytes == 0);
649
650 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
651 destroy_tdata = prof_tdata_should_destroy(tdata, false);
652 malloc_mutex_unlock(tdata->lock);
653
654 malloc_mutex_lock(gctx->lock);
655 switch (tctx->state) {
656 case prof_tctx_state_nominal:
657 tctx_tree_remove(&gctx->tctxs, tctx);
658 destroy_tctx = true;
659 if (prof_gctx_should_destroy(gctx)) {
660 /*
661 * Increment gctx->nlimbo in order to keep another
662 * thread from winning the race to destroy gctx while
663 * this one has gctx->lock dropped. Without this, it
664 * would be possible for another thread to:
665 *
666 * 1) Sample an allocation associated with gctx.
667 * 2) Deallocate the sampled object.
668 * 3) Successfully prof_gctx_try_destroy(gctx).
669 *
670 * The result would be that gctx no longer exists by the
671 * time this thread accesses it in
672 * prof_gctx_try_destroy().
673 */
674 gctx->nlimbo++;
675 destroy_gctx = true;
676 } else
677 destroy_gctx = false;
678 break;
679 case prof_tctx_state_dumping:
680 /*
681 * A dumping thread needs tctx to remain valid until dumping
682 * has finished. Change state such that the dumping thread will
683 * complete destruction during a late dump iteration phase.
684 */
685 tctx->state = prof_tctx_state_purgatory;
686 destroy_tctx = false;
687 destroy_gctx = false;
688 break;
689 default:
690 not_reached();
691 destroy_tctx = false;
692 destroy_gctx = false;
693 }
694 malloc_mutex_unlock(gctx->lock);
695 if (destroy_gctx) {
696 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
697 tdata);
698 }
699
700 if (destroy_tdata)
701 prof_tdata_destroy(tsd, tdata, false);
702
703 if (destroy_tctx)
| 598 } else {
599 /*
600 * Compensate for increment in prof_tctx_destroy() or
601 * prof_lookup().
602 */
603 gctx->nlimbo--;
604 malloc_mutex_unlock(gctx->lock);
605 prof_leave(tsd, tdata_self);
606 }
607}
608
609/* tctx->tdata->lock must be held. */
610static bool
611prof_tctx_should_destroy(prof_tctx_t *tctx)
612{
613
614 if (opt_prof_accum)
615 return (false);
616 if (tctx->cnts.curobjs != 0)
617 return (false);
618 if (tctx->prepared)
619 return (false);
620 return (true);
621}
622
623static bool
624prof_gctx_should_destroy(prof_gctx_t *gctx)
625{
626
627 if (opt_prof_accum)
628 return (false);
629 if (!tctx_tree_empty(&gctx->tctxs))
630 return (false);
631 if (gctx->nlimbo != 0)
632 return (false);
633 return (true);
634}
635
636/* tctx->tdata->lock is held upon entry, and released before return. */
637static void
638prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
639{
640 prof_tdata_t *tdata = tctx->tdata;
641 prof_gctx_t *gctx = tctx->gctx;
642 bool destroy_tdata, destroy_tctx, destroy_gctx;
643
644 assert(tctx->cnts.curobjs == 0);
645 assert(tctx->cnts.curbytes == 0);
646 assert(!opt_prof_accum);
647 assert(tctx->cnts.accumobjs == 0);
648 assert(tctx->cnts.accumbytes == 0);
649
650 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
651 destroy_tdata = prof_tdata_should_destroy(tdata, false);
652 malloc_mutex_unlock(tdata->lock);
653
654 malloc_mutex_lock(gctx->lock);
655 switch (tctx->state) {
656 case prof_tctx_state_nominal:
657 tctx_tree_remove(&gctx->tctxs, tctx);
658 destroy_tctx = true;
659 if (prof_gctx_should_destroy(gctx)) {
660 /*
661 * Increment gctx->nlimbo in order to keep another
662 * thread from winning the race to destroy gctx while
663 * this one has gctx->lock dropped. Without this, it
664 * would be possible for another thread to:
665 *
666 * 1) Sample an allocation associated with gctx.
667 * 2) Deallocate the sampled object.
668 * 3) Successfully prof_gctx_try_destroy(gctx).
669 *
670 * The result would be that gctx no longer exists by the
671 * time this thread accesses it in
672 * prof_gctx_try_destroy().
673 */
674 gctx->nlimbo++;
675 destroy_gctx = true;
676 } else
677 destroy_gctx = false;
678 break;
679 case prof_tctx_state_dumping:
680 /*
681 * A dumping thread needs tctx to remain valid until dumping
682 * has finished. Change state such that the dumping thread will
683 * complete destruction during a late dump iteration phase.
684 */
685 tctx->state = prof_tctx_state_purgatory;
686 destroy_tctx = false;
687 destroy_gctx = false;
688 break;
689 default:
690 not_reached();
691 destroy_tctx = false;
692 destroy_gctx = false;
693 }
694 malloc_mutex_unlock(gctx->lock);
695 if (destroy_gctx) {
696 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
697 tdata);
698 }
699
700 if (destroy_tdata)
701 prof_tdata_destroy(tsd, tdata, false);
702
703 if (destroy_tctx)
|
704 idalloctm(tsd, tctx, tcache_get(tsd, false), true);
| 704 idalloctm(tsd, tctx, tcache_get(tsd, false), true, true);
|
705}
706
707static bool
708prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
709 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
710{
711 union {
712 prof_gctx_t *p;
713 void *v;
714 } gctx;
715 union {
716 prof_bt_t *p;
717 void *v;
718 } btkey;
719 bool new_gctx;
720
721 prof_enter(tsd, tdata);
722 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
723 /* bt has never been seen before. Insert it. */
724 gctx.p = prof_gctx_create(tsd, bt);
725 if (gctx.v == NULL) {
726 prof_leave(tsd, tdata);
727 return (true);
728 }
729 btkey.p = &gctx.p->bt;
730 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
731 /* OOM. */
732 prof_leave(tsd, tdata);
| 705}
706
707static bool
708prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
709 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
710{
711 union {
712 prof_gctx_t *p;
713 void *v;
714 } gctx;
715 union {
716 prof_bt_t *p;
717 void *v;
718 } btkey;
719 bool new_gctx;
720
721 prof_enter(tsd, tdata);
722 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
723 /* bt has never been seen before. Insert it. */
724 gctx.p = prof_gctx_create(tsd, bt);
725 if (gctx.v == NULL) {
726 prof_leave(tsd, tdata);
727 return (true);
728 }
729 btkey.p = &gctx.p->bt;
730 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
731 /* OOM. */
732 prof_leave(tsd, tdata);
|
733 idalloctm(tsd, gctx.v, tcache_get(tsd, false), true);
| 733 idalloctm(tsd, gctx.v, tcache_get(tsd, false), true,
734 true);
|
734 return (true);
735 }
736 new_gctx = true;
737 } else {
738 /*
739 * Increment nlimbo, in order to avoid a race condition with
740 * prof_tctx_destroy()/prof_gctx_try_destroy().
741 */
742 malloc_mutex_lock(gctx.p->lock);
743 gctx.p->nlimbo++;
744 malloc_mutex_unlock(gctx.p->lock);
745 new_gctx = false;
746 }
747 prof_leave(tsd, tdata);
748
749 *p_btkey = btkey.v;
750 *p_gctx = gctx.p;
751 *p_new_gctx = new_gctx;
752 return (false);
753}
754
755prof_tctx_t *
756prof_lookup(tsd_t *tsd, prof_bt_t *bt)
757{
758 union {
759 prof_tctx_t *p;
760 void *v;
761 } ret;
762 prof_tdata_t *tdata;
763 bool not_found;
764
765 cassert(config_prof);
766
767 tdata = prof_tdata_get(tsd, false);
768 if (tdata == NULL)
769 return (NULL);
770
771 malloc_mutex_lock(tdata->lock);
772 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
773 if (!not_found) /* Note double negative! */
774 ret.p->prepared = true;
775 malloc_mutex_unlock(tdata->lock);
776 if (not_found) {
777 tcache_t *tcache;
778 void *btkey;
779 prof_gctx_t *gctx;
780 bool new_gctx, error;
781
782 /*
783 * This thread's cache lacks bt. Look for it in the global
784 * cache.
785 */
786 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
787 &new_gctx))
788 return (NULL);
789
790 /* Link a prof_tctx_t into gctx for this thread. */
791 tcache = tcache_get(tsd, true);
| 735 return (true);
736 }
737 new_gctx = true;
738 } else {
739 /*
740 * Increment nlimbo, in order to avoid a race condition with
741 * prof_tctx_destroy()/prof_gctx_try_destroy().
742 */
743 malloc_mutex_lock(gctx.p->lock);
744 gctx.p->nlimbo++;
745 malloc_mutex_unlock(gctx.p->lock);
746 new_gctx = false;
747 }
748 prof_leave(tsd, tdata);
749
750 *p_btkey = btkey.v;
751 *p_gctx = gctx.p;
752 *p_new_gctx = new_gctx;
753 return (false);
754}
755
756prof_tctx_t *
757prof_lookup(tsd_t *tsd, prof_bt_t *bt)
758{
759 union {
760 prof_tctx_t *p;
761 void *v;
762 } ret;
763 prof_tdata_t *tdata;
764 bool not_found;
765
766 cassert(config_prof);
767
768 tdata = prof_tdata_get(tsd, false);
769 if (tdata == NULL)
770 return (NULL);
771
772 malloc_mutex_lock(tdata->lock);
773 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
774 if (!not_found) /* Note double negative! */
775 ret.p->prepared = true;
776 malloc_mutex_unlock(tdata->lock);
777 if (not_found) {
778 tcache_t *tcache;
779 void *btkey;
780 prof_gctx_t *gctx;
781 bool new_gctx, error;
782
783 /*
784 * This thread's cache lacks bt. Look for it in the global
785 * cache.
786 */
787 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
788 &new_gctx))
789 return (NULL);
790
791 /* Link a prof_tctx_t into gctx for this thread. */
792 tcache = tcache_get(tsd, true);
|
792 ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, tcache, true,
793 NULL);
| 793 ret.v = iallocztm(tsd, sizeof(prof_tctx_t),
794 size2index(sizeof(prof_tctx_t)), false, tcache, true, NULL,
795 true);
|
794 if (ret.p == NULL) {
795 if (new_gctx)
796 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
797 return (NULL);
798 }
799 ret.p->tdata = tdata;
800 ret.p->thr_uid = tdata->thr_uid;
801 ret.p->thr_discrim = tdata->thr_discrim;
802 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
803 ret.p->gctx = gctx;
804 ret.p->tctx_uid = tdata->tctx_uid_next++;
805 ret.p->prepared = true;
806 ret.p->state = prof_tctx_state_initializing;
807 malloc_mutex_lock(tdata->lock);
808 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
809 malloc_mutex_unlock(tdata->lock);
810 if (error) {
811 if (new_gctx)
812 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
| 796 if (ret.p == NULL) {
797 if (new_gctx)
798 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
799 return (NULL);
800 }
801 ret.p->tdata = tdata;
802 ret.p->thr_uid = tdata->thr_uid;
803 ret.p->thr_discrim = tdata->thr_discrim;
804 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
805 ret.p->gctx = gctx;
806 ret.p->tctx_uid = tdata->tctx_uid_next++;
807 ret.p->prepared = true;
808 ret.p->state = prof_tctx_state_initializing;
809 malloc_mutex_lock(tdata->lock);
810 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
811 malloc_mutex_unlock(tdata->lock);
812 if (error) {
813 if (new_gctx)
814 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
|
813 idalloctm(tsd, ret.v, tcache, true);
| 815 idalloctm(tsd, ret.v, tcache, true, true);
|
814 return (NULL);
815 }
816 malloc_mutex_lock(gctx->lock);
817 ret.p->state = prof_tctx_state_nominal;
818 tctx_tree_insert(&gctx->tctxs, ret.p);
819 gctx->nlimbo--;
820 malloc_mutex_unlock(gctx->lock);
821 }
822
823 return (ret.p);
824}
825
826void
827prof_sample_threshold_update(prof_tdata_t *tdata)
828{
829 /*
830 * The body of this function is compiled out unless heap profiling is
831 * enabled, so that it is possible to compile jemalloc with floating
832 * point support completely disabled. Avoiding floating point code is
833 * important on memory-constrained systems, but it also enables a
834 * workaround for versions of glibc that don't properly save/restore
835 * floating point registers during dynamic lazy symbol loading (which
836 * internally calls into whatever malloc implementation happens to be
837 * integrated into the application). Note that some compilers (e.g.
838 * gcc 4.8) may use floating point registers for fast memory moves, so
839 * jemalloc must be compiled with such optimizations disabled (e.g.
840 * -mno-sse) in order for the workaround to be complete.
841 */
842#ifdef JEMALLOC_PROF
843 uint64_t r;
844 double u;
845
846 if (!config_prof)
847 return;
848
849 if (lg_prof_sample == 0) {
850 tdata->bytes_until_sample = 0;
851 return;
852 }
853
854 /*
855 * Compute sample interval as a geometrically distributed random
856 * variable with mean (2^lg_prof_sample).
857 *
858 * __ __
859 * | log(u) | 1
860 * tdata->bytes_until_sample = | -------- |, where p = ---------------
861 * | log(1-p) | lg_prof_sample
862 * 2
863 *
864 * For more information on the math, see:
865 *
866 * Non-Uniform Random Variate Generation
867 * Luc Devroye
868 * Springer-Verlag, New York, 1986
869 * pp 500
870 * (http://luc.devroye.org/rnbookindex.html)
871 */
| 816 return (NULL);
817 }
818 malloc_mutex_lock(gctx->lock);
819 ret.p->state = prof_tctx_state_nominal;
820 tctx_tree_insert(&gctx->tctxs, ret.p);
821 gctx->nlimbo--;
822 malloc_mutex_unlock(gctx->lock);
823 }
824
825 return (ret.p);
826}
827
828void
829prof_sample_threshold_update(prof_tdata_t *tdata)
830{
831 /*
832 * The body of this function is compiled out unless heap profiling is
833 * enabled, so that it is possible to compile jemalloc with floating
834 * point support completely disabled. Avoiding floating point code is
835 * important on memory-constrained systems, but it also enables a
836 * workaround for versions of glibc that don't properly save/restore
837 * floating point registers during dynamic lazy symbol loading (which
838 * internally calls into whatever malloc implementation happens to be
839 * integrated into the application). Note that some compilers (e.g.
840 * gcc 4.8) may use floating point registers for fast memory moves, so
841 * jemalloc must be compiled with such optimizations disabled (e.g.
842 * -mno-sse) in order for the workaround to be complete.
843 */
844#ifdef JEMALLOC_PROF
845 uint64_t r;
846 double u;
847
848 if (!config_prof)
849 return;
850
851 if (lg_prof_sample == 0) {
852 tdata->bytes_until_sample = 0;
853 return;
854 }
855
856 /*
857 * Compute sample interval as a geometrically distributed random
858 * variable with mean (2^lg_prof_sample).
859 *
860 * __ __
861 * | log(u) | 1
862 * tdata->bytes_until_sample = | -------- |, where p = ---------------
863 * | log(1-p) | lg_prof_sample
864 * 2
865 *
866 * For more information on the math, see:
867 *
868 * Non-Uniform Random Variate Generation
869 * Luc Devroye
870 * Springer-Verlag, New York, 1986
871 * pp 500
872 * (http://luc.devroye.org/rnbookindex.html)
873 */
|
872 prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005),
873 UINT64_C(1442695040888963407));
| 874 r = prng_lg_range(&tdata->prng_state, 53);
|
874 u = (double)r * (1.0/9007199254740992.0L);
875 tdata->bytes_until_sample = (uint64_t)(log(u) /
876 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
877 + (uint64_t)1U;
878#endif
879}
880
881#ifdef JEMALLOC_JET
882static prof_tdata_t *
883prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
884{
885 size_t *tdata_count = (size_t *)arg;
886
887 (*tdata_count)++;
888
889 return (NULL);
890}
891
892size_t
893prof_tdata_count(void)
894{
895 size_t tdata_count = 0;
896
897 malloc_mutex_lock(&tdatas_mtx);
898 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
899 (void *)&tdata_count);
900 malloc_mutex_unlock(&tdatas_mtx);
901
902 return (tdata_count);
903}
904#endif
905
906#ifdef JEMALLOC_JET
907size_t
908prof_bt_count(void)
909{
910 size_t bt_count;
911 tsd_t *tsd;
912 prof_tdata_t *tdata;
913
914 tsd = tsd_fetch();
915 tdata = prof_tdata_get(tsd, false);
916 if (tdata == NULL)
917 return (0);
918
919 malloc_mutex_lock(&bt2gctx_mtx);
920 bt_count = ckh_count(&bt2gctx);
921 malloc_mutex_unlock(&bt2gctx_mtx);
922
923 return (bt_count);
924}
925#endif
926
927#ifdef JEMALLOC_JET
928#undef prof_dump_open
929#define prof_dump_open JEMALLOC_N(prof_dump_open_impl)
930#endif
931static int
932prof_dump_open(bool propagate_err, const char *filename)
933{
934 int fd;
935
936 fd = creat(filename, 0644);
937 if (fd == -1 && !propagate_err) {
938 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
939 filename);
940 if (opt_abort)
941 abort();
942 }
943
944 return (fd);
945}
946#ifdef JEMALLOC_JET
947#undef prof_dump_open
948#define prof_dump_open JEMALLOC_N(prof_dump_open)
949prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
950#endif
951
952static bool
953prof_dump_flush(bool propagate_err)
954{
955 bool ret = false;
956 ssize_t err;
957
958 cassert(config_prof);
959
960 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
961 if (err == -1) {
962 if (!propagate_err) {
963 malloc_write("<jemalloc>: write() failed during heap "
964 "profile flush\n");
965 if (opt_abort)
966 abort();
967 }
968 ret = true;
969 }
970 prof_dump_buf_end = 0;
971
972 return (ret);
973}
974
975static bool
976prof_dump_close(bool propagate_err)
977{
978 bool ret;
979
980 assert(prof_dump_fd != -1);
981 ret = prof_dump_flush(propagate_err);
982 close(prof_dump_fd);
983 prof_dump_fd = -1;
984
985 return (ret);
986}
987
988static bool
989prof_dump_write(bool propagate_err, const char *s)
990{
| 875 u = (double)r * (1.0/9007199254740992.0L);
876 tdata->bytes_until_sample = (uint64_t)(log(u) /
877 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
878 + (uint64_t)1U;
879#endif
880}
881
882#ifdef JEMALLOC_JET
883static prof_tdata_t *
884prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
885{
886 size_t *tdata_count = (size_t *)arg;
887
888 (*tdata_count)++;
889
890 return (NULL);
891}
892
893size_t
894prof_tdata_count(void)
895{
896 size_t tdata_count = 0;
897
898 malloc_mutex_lock(&tdatas_mtx);
899 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
900 (void *)&tdata_count);
901 malloc_mutex_unlock(&tdatas_mtx);
902
903 return (tdata_count);
904}
905#endif
906
907#ifdef JEMALLOC_JET
908size_t
909prof_bt_count(void)
910{
911 size_t bt_count;
912 tsd_t *tsd;
913 prof_tdata_t *tdata;
914
915 tsd = tsd_fetch();
916 tdata = prof_tdata_get(tsd, false);
917 if (tdata == NULL)
918 return (0);
919
920 malloc_mutex_lock(&bt2gctx_mtx);
921 bt_count = ckh_count(&bt2gctx);
922 malloc_mutex_unlock(&bt2gctx_mtx);
923
924 return (bt_count);
925}
926#endif
927
928#ifdef JEMALLOC_JET
929#undef prof_dump_open
930#define prof_dump_open JEMALLOC_N(prof_dump_open_impl)
931#endif
932static int
933prof_dump_open(bool propagate_err, const char *filename)
934{
935 int fd;
936
937 fd = creat(filename, 0644);
938 if (fd == -1 && !propagate_err) {
939 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
940 filename);
941 if (opt_abort)
942 abort();
943 }
944
945 return (fd);
946}
947#ifdef JEMALLOC_JET
948#undef prof_dump_open
949#define prof_dump_open JEMALLOC_N(prof_dump_open)
950prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
951#endif
952
953static bool
954prof_dump_flush(bool propagate_err)
955{
956 bool ret = false;
957 ssize_t err;
958
959 cassert(config_prof);
960
961 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
962 if (err == -1) {
963 if (!propagate_err) {
964 malloc_write("<jemalloc>: write() failed during heap "
965 "profile flush\n");
966 if (opt_abort)
967 abort();
968 }
969 ret = true;
970 }
971 prof_dump_buf_end = 0;
972
973 return (ret);
974}
975
976static bool
977prof_dump_close(bool propagate_err)
978{
979 bool ret;
980
981 assert(prof_dump_fd != -1);
982 ret = prof_dump_flush(propagate_err);
983 close(prof_dump_fd);
984 prof_dump_fd = -1;
985
986 return (ret);
987}
988
989static bool
990prof_dump_write(bool propagate_err, const char *s)
991{
|
991 unsigned i, slen, n;
| 992 size_t i, slen, n;
|
992
993 cassert(config_prof);
994
995 i = 0;
996 slen = strlen(s);
997 while (i < slen) {
998 /* Flush the buffer if it is full. */
999 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
1000 if (prof_dump_flush(propagate_err) && propagate_err)
1001 return (true);
1002
1003 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
1004 /* Finish writing. */
1005 n = slen - i;
1006 } else {
1007 /* Write as much of s as will fit. */
1008 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
1009 }
1010 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
1011 prof_dump_buf_end += n;
1012 i += n;
1013 }
1014
1015 return (false);
1016}
1017
1018JEMALLOC_FORMAT_PRINTF(2, 3)
1019static bool
1020prof_dump_printf(bool propagate_err, const char *format, ...)
1021{
1022 bool ret;
1023 va_list ap;
1024 char buf[PROF_PRINTF_BUFSIZE];
1025
1026 va_start(ap, format);
1027 malloc_vsnprintf(buf, sizeof(buf), format, ap);
1028 va_end(ap);
1029 ret = prof_dump_write(propagate_err, buf);
1030
1031 return (ret);
1032}
1033
1034/* tctx->tdata->lock is held. */
1035static void
1036prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata)
1037{
1038
1039 malloc_mutex_lock(tctx->gctx->lock);
1040
1041 switch (tctx->state) {
1042 case prof_tctx_state_initializing:
1043 malloc_mutex_unlock(tctx->gctx->lock);
1044 return;
1045 case prof_tctx_state_nominal:
1046 tctx->state = prof_tctx_state_dumping;
1047 malloc_mutex_unlock(tctx->gctx->lock);
1048
1049 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
1050
1051 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1052 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1053 if (opt_prof_accum) {
1054 tdata->cnt_summed.accumobjs +=
1055 tctx->dump_cnts.accumobjs;
1056 tdata->cnt_summed.accumbytes +=
1057 tctx->dump_cnts.accumbytes;
1058 }
1059 break;
1060 case prof_tctx_state_dumping:
1061 case prof_tctx_state_purgatory:
1062 not_reached();
1063 }
1064}
1065
1066/* gctx->lock is held. */
1067static void
1068prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx)
1069{
1070
1071 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1072 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1073 if (opt_prof_accum) {
1074 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
1075 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
1076 }
1077}
1078
1079/* tctx->gctx is held. */
1080static prof_tctx_t *
1081prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1082{
1083
1084 switch (tctx->state) {
1085 case prof_tctx_state_nominal:
1086 /* New since dumping started; ignore. */
1087 break;
1088 case prof_tctx_state_dumping:
1089 case prof_tctx_state_purgatory:
1090 prof_tctx_merge_gctx(tctx, tctx->gctx);
1091 break;
1092 default:
1093 not_reached();
1094 }
1095
1096 return (NULL);
1097}
1098
1099/* gctx->lock is held. */
1100static prof_tctx_t *
1101prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1102{
1103 bool propagate_err = *(bool *)arg;
1104
1105 switch (tctx->state) {
1106 case prof_tctx_state_initializing:
1107 case prof_tctx_state_nominal:
1108 /* Not captured by this dump. */
1109 break;
1110 case prof_tctx_state_dumping:
1111 case prof_tctx_state_purgatory:
1112 if (prof_dump_printf(propagate_err,
1113 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
1114 "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
1115 tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
1116 tctx->dump_cnts.accumbytes))
1117 return (tctx);
1118 break;
1119 default:
1120 not_reached();
1121 }
1122 return (NULL);
1123}
1124
1125/* tctx->gctx is held. */
1126static prof_tctx_t *
1127prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1128{
1129 prof_tctx_t *ret;
1130
1131 switch (tctx->state) {
1132 case prof_tctx_state_nominal:
1133 /* New since dumping started; ignore. */
1134 break;
1135 case prof_tctx_state_dumping:
1136 tctx->state = prof_tctx_state_nominal;
1137 break;
1138 case prof_tctx_state_purgatory:
1139 ret = tctx;
1140 goto label_return;
1141 default:
1142 not_reached();
1143 }
1144
1145 ret = NULL;
1146label_return:
1147 return (ret);
1148}
1149
1150static void
1151prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
1152{
1153
1154 cassert(config_prof);
1155
1156 malloc_mutex_lock(gctx->lock);
1157
1158 /*
1159 * Increment nlimbo so that gctx won't go away before dump.
1160 * Additionally, link gctx into the dump list so that it is included in
1161 * prof_dump()'s second pass.
1162 */
1163 gctx->nlimbo++;
1164 gctx_tree_insert(gctxs, gctx);
1165
1166 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
1167
1168 malloc_mutex_unlock(gctx->lock);
1169}
1170
1171static prof_gctx_t *
1172prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1173{
1174 size_t *leak_ngctx = (size_t *)arg;
1175
1176 malloc_mutex_lock(gctx->lock);
1177 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL);
1178 if (gctx->cnt_summed.curobjs != 0)
1179 (*leak_ngctx)++;
1180 malloc_mutex_unlock(gctx->lock);
1181
1182 return (NULL);
1183}
1184
1185static void
1186prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
1187{
1188 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1189 prof_gctx_t *gctx;
1190
1191 /*
1192 * Standard tree iteration won't work here, because as soon as we
1193 * decrement gctx->nlimbo and unlock gctx, another thread can
1194 * concurrently destroy it, which will corrupt the tree. Therefore,
1195 * tear down the tree one node at a time during iteration.
1196 */
1197 while ((gctx = gctx_tree_first(gctxs)) != NULL) {
1198 gctx_tree_remove(gctxs, gctx);
1199 malloc_mutex_lock(gctx->lock);
1200 {
1201 prof_tctx_t *next;
1202
1203 next = NULL;
1204 do {
1205 prof_tctx_t *to_destroy =
1206 tctx_tree_iter(&gctx->tctxs, next,
1207 prof_tctx_finish_iter, NULL);
1208 if (to_destroy != NULL) {
1209 next = tctx_tree_next(&gctx->tctxs,
1210 to_destroy);
1211 tctx_tree_remove(&gctx->tctxs,
1212 to_destroy);
1213 idalloctm(tsd, to_destroy,
| 993
994 cassert(config_prof);
995
996 i = 0;
997 slen = strlen(s);
998 while (i < slen) {
999 /* Flush the buffer if it is full. */
1000 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
1001 if (prof_dump_flush(propagate_err) && propagate_err)
1002 return (true);
1003
1004 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
1005 /* Finish writing. */
1006 n = slen - i;
1007 } else {
1008 /* Write as much of s as will fit. */
1009 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
1010 }
1011 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
1012 prof_dump_buf_end += n;
1013 i += n;
1014 }
1015
1016 return (false);
1017}
1018
1019JEMALLOC_FORMAT_PRINTF(2, 3)
1020static bool
1021prof_dump_printf(bool propagate_err, const char *format, ...)
1022{
1023 bool ret;
1024 va_list ap;
1025 char buf[PROF_PRINTF_BUFSIZE];
1026
1027 va_start(ap, format);
1028 malloc_vsnprintf(buf, sizeof(buf), format, ap);
1029 va_end(ap);
1030 ret = prof_dump_write(propagate_err, buf);
1031
1032 return (ret);
1033}
1034
1035/* tctx->tdata->lock is held. */
1036static void
1037prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata)
1038{
1039
1040 malloc_mutex_lock(tctx->gctx->lock);
1041
1042 switch (tctx->state) {
1043 case prof_tctx_state_initializing:
1044 malloc_mutex_unlock(tctx->gctx->lock);
1045 return;
1046 case prof_tctx_state_nominal:
1047 tctx->state = prof_tctx_state_dumping;
1048 malloc_mutex_unlock(tctx->gctx->lock);
1049
1050 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
1051
1052 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1053 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1054 if (opt_prof_accum) {
1055 tdata->cnt_summed.accumobjs +=
1056 tctx->dump_cnts.accumobjs;
1057 tdata->cnt_summed.accumbytes +=
1058 tctx->dump_cnts.accumbytes;
1059 }
1060 break;
1061 case prof_tctx_state_dumping:
1062 case prof_tctx_state_purgatory:
1063 not_reached();
1064 }
1065}
1066
1067/* gctx->lock is held. */
1068static void
1069prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx)
1070{
1071
1072 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1073 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1074 if (opt_prof_accum) {
1075 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
1076 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
1077 }
1078}
1079
1080/* tctx->gctx is held. */
1081static prof_tctx_t *
1082prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1083{
1084
1085 switch (tctx->state) {
1086 case prof_tctx_state_nominal:
1087 /* New since dumping started; ignore. */
1088 break;
1089 case prof_tctx_state_dumping:
1090 case prof_tctx_state_purgatory:
1091 prof_tctx_merge_gctx(tctx, tctx->gctx);
1092 break;
1093 default:
1094 not_reached();
1095 }
1096
1097 return (NULL);
1098}
1099
1100/* gctx->lock is held. */
1101static prof_tctx_t *
1102prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1103{
1104 bool propagate_err = *(bool *)arg;
1105
1106 switch (tctx->state) {
1107 case prof_tctx_state_initializing:
1108 case prof_tctx_state_nominal:
1109 /* Not captured by this dump. */
1110 break;
1111 case prof_tctx_state_dumping:
1112 case prof_tctx_state_purgatory:
1113 if (prof_dump_printf(propagate_err,
1114 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
1115 "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
1116 tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
1117 tctx->dump_cnts.accumbytes))
1118 return (tctx);
1119 break;
1120 default:
1121 not_reached();
1122 }
1123 return (NULL);
1124}
1125
1126/* tctx->gctx is held. */
1127static prof_tctx_t *
1128prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1129{
1130 prof_tctx_t *ret;
1131
1132 switch (tctx->state) {
1133 case prof_tctx_state_nominal:
1134 /* New since dumping started; ignore. */
1135 break;
1136 case prof_tctx_state_dumping:
1137 tctx->state = prof_tctx_state_nominal;
1138 break;
1139 case prof_tctx_state_purgatory:
1140 ret = tctx;
1141 goto label_return;
1142 default:
1143 not_reached();
1144 }
1145
1146 ret = NULL;
1147label_return:
1148 return (ret);
1149}
1150
1151static void
1152prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
1153{
1154
1155 cassert(config_prof);
1156
1157 malloc_mutex_lock(gctx->lock);
1158
1159 /*
1160 * Increment nlimbo so that gctx won't go away before dump.
1161 * Additionally, link gctx into the dump list so that it is included in
1162 * prof_dump()'s second pass.
1163 */
1164 gctx->nlimbo++;
1165 gctx_tree_insert(gctxs, gctx);
1166
1167 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
1168
1169 malloc_mutex_unlock(gctx->lock);
1170}
1171
1172static prof_gctx_t *
1173prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1174{
1175 size_t *leak_ngctx = (size_t *)arg;
1176
1177 malloc_mutex_lock(gctx->lock);
1178 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL);
1179 if (gctx->cnt_summed.curobjs != 0)
1180 (*leak_ngctx)++;
1181 malloc_mutex_unlock(gctx->lock);
1182
1183 return (NULL);
1184}
1185
1186static void
1187prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
1188{
1189 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1190 prof_gctx_t *gctx;
1191
1192 /*
1193 * Standard tree iteration won't work here, because as soon as we
1194 * decrement gctx->nlimbo and unlock gctx, another thread can
1195 * concurrently destroy it, which will corrupt the tree. Therefore,
1196 * tear down the tree one node at a time during iteration.
1197 */
1198 while ((gctx = gctx_tree_first(gctxs)) != NULL) {
1199 gctx_tree_remove(gctxs, gctx);
1200 malloc_mutex_lock(gctx->lock);
1201 {
1202 prof_tctx_t *next;
1203
1204 next = NULL;
1205 do {
1206 prof_tctx_t *to_destroy =
1207 tctx_tree_iter(&gctx->tctxs, next,
1208 prof_tctx_finish_iter, NULL);
1209 if (to_destroy != NULL) {
1210 next = tctx_tree_next(&gctx->tctxs,
1211 to_destroy);
1212 tctx_tree_remove(&gctx->tctxs,
1213 to_destroy);
1214 idalloctm(tsd, to_destroy,
|
1214 tcache_get(tsd, false), true);
| 1215 tcache_get(tsd, false), true, true);
|
1215 } else
1216 next = NULL;
1217 } while (next != NULL);
1218 }
1219 gctx->nlimbo--;
1220 if (prof_gctx_should_destroy(gctx)) {
1221 gctx->nlimbo++;
1222 malloc_mutex_unlock(gctx->lock);
1223 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
1224 } else
1225 malloc_mutex_unlock(gctx->lock);
1226 }
1227}
1228
1229static prof_tdata_t *
1230prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1231{
1232 prof_cnt_t *cnt_all = (prof_cnt_t *)arg;
1233
1234 malloc_mutex_lock(tdata->lock);
1235 if (!tdata->expired) {
1236 size_t tabind;
1237 union {
1238 prof_tctx_t *p;
1239 void *v;
1240 } tctx;
1241
1242 tdata->dumping = true;
1243 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
1244 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
1245 &tctx.v);)
1246 prof_tctx_merge_tdata(tctx.p, tdata);
1247
1248 cnt_all->curobjs += tdata->cnt_summed.curobjs;
1249 cnt_all->curbytes += tdata->cnt_summed.curbytes;
1250 if (opt_prof_accum) {
1251 cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
1252 cnt_all->accumbytes += tdata->cnt_summed.accumbytes;
1253 }
1254 } else
1255 tdata->dumping = false;
1256 malloc_mutex_unlock(tdata->lock);
1257
1258 return (NULL);
1259}
1260
1261static prof_tdata_t *
1262prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1263{
1264 bool propagate_err = *(bool *)arg;
1265
1266 if (!tdata->dumping)
1267 return (NULL);
1268
1269 if (prof_dump_printf(propagate_err,
1270 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
1271 tdata->thr_uid, tdata->cnt_summed.curobjs,
1272 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
1273 tdata->cnt_summed.accumbytes,
1274 (tdata->thread_name != NULL) ? " " : "",
1275 (tdata->thread_name != NULL) ? tdata->thread_name : ""))
1276 return (tdata);
1277 return (NULL);
1278}
1279
1280#ifdef JEMALLOC_JET
1281#undef prof_dump_header
1282#define prof_dump_header JEMALLOC_N(prof_dump_header_impl)
1283#endif
1284static bool
1285prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
1286{
1287 bool ret;
1288
1289 if (prof_dump_printf(propagate_err,
1290 "heap_v2/%"FMTu64"\n"
1291 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1292 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
1293 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes))
1294 return (true);
1295
1296 malloc_mutex_lock(&tdatas_mtx);
1297 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
1298 (void *)&propagate_err) != NULL);
1299 malloc_mutex_unlock(&tdatas_mtx);
1300 return (ret);
1301}
1302#ifdef JEMALLOC_JET
1303#undef prof_dump_header
1304#define prof_dump_header JEMALLOC_N(prof_dump_header)
1305prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
1306#endif
1307
1308/* gctx->lock is held. */
1309static bool
1310prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt,
1311 prof_gctx_tree_t *gctxs)
1312{
1313 bool ret;
1314 unsigned i;
1315
1316 cassert(config_prof);
1317
1318 /* Avoid dumping such gctx's that have no useful data. */
1319 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
1320 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
1321 assert(gctx->cnt_summed.curobjs == 0);
1322 assert(gctx->cnt_summed.curbytes == 0);
1323 assert(gctx->cnt_summed.accumobjs == 0);
1324 assert(gctx->cnt_summed.accumbytes == 0);
1325 ret = false;
1326 goto label_return;
1327 }
1328
1329 if (prof_dump_printf(propagate_err, "@")) {
1330 ret = true;
1331 goto label_return;
1332 }
1333 for (i = 0; i < bt->len; i++) {
1334 if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
1335 (uintptr_t)bt->vec[i])) {
1336 ret = true;
1337 goto label_return;
1338 }
1339 }
1340
1341 if (prof_dump_printf(propagate_err,
1342 "\n"
1343 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1344 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
1345 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
1346 ret = true;
1347 goto label_return;
1348 }
1349
1350 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
1351 (void *)&propagate_err) != NULL) {
1352 ret = true;
1353 goto label_return;
1354 }
1355
1356 ret = false;
1357label_return:
1358 return (ret);
1359}
1360
| 1216 } else
1217 next = NULL;
1218 } while (next != NULL);
1219 }
1220 gctx->nlimbo--;
1221 if (prof_gctx_should_destroy(gctx)) {
1222 gctx->nlimbo++;
1223 malloc_mutex_unlock(gctx->lock);
1224 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
1225 } else
1226 malloc_mutex_unlock(gctx->lock);
1227 }
1228}
1229
1230static prof_tdata_t *
1231prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1232{
1233 prof_cnt_t *cnt_all = (prof_cnt_t *)arg;
1234
1235 malloc_mutex_lock(tdata->lock);
1236 if (!tdata->expired) {
1237 size_t tabind;
1238 union {
1239 prof_tctx_t *p;
1240 void *v;
1241 } tctx;
1242
1243 tdata->dumping = true;
1244 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
1245 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
1246 &tctx.v);)
1247 prof_tctx_merge_tdata(tctx.p, tdata);
1248
1249 cnt_all->curobjs += tdata->cnt_summed.curobjs;
1250 cnt_all->curbytes += tdata->cnt_summed.curbytes;
1251 if (opt_prof_accum) {
1252 cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
1253 cnt_all->accumbytes += tdata->cnt_summed.accumbytes;
1254 }
1255 } else
1256 tdata->dumping = false;
1257 malloc_mutex_unlock(tdata->lock);
1258
1259 return (NULL);
1260}
1261
1262static prof_tdata_t *
1263prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1264{
1265 bool propagate_err = *(bool *)arg;
1266
1267 if (!tdata->dumping)
1268 return (NULL);
1269
1270 if (prof_dump_printf(propagate_err,
1271 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
1272 tdata->thr_uid, tdata->cnt_summed.curobjs,
1273 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
1274 tdata->cnt_summed.accumbytes,
1275 (tdata->thread_name != NULL) ? " " : "",
1276 (tdata->thread_name != NULL) ? tdata->thread_name : ""))
1277 return (tdata);
1278 return (NULL);
1279}
1280
1281#ifdef JEMALLOC_JET
1282#undef prof_dump_header
1283#define prof_dump_header JEMALLOC_N(prof_dump_header_impl)
1284#endif
1285static bool
1286prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
1287{
1288 bool ret;
1289
1290 if (prof_dump_printf(propagate_err,
1291 "heap_v2/%"FMTu64"\n"
1292 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1293 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
1294 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes))
1295 return (true);
1296
1297 malloc_mutex_lock(&tdatas_mtx);
1298 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
1299 (void *)&propagate_err) != NULL);
1300 malloc_mutex_unlock(&tdatas_mtx);
1301 return (ret);
1302}
1303#ifdef JEMALLOC_JET
1304#undef prof_dump_header
1305#define prof_dump_header JEMALLOC_N(prof_dump_header)
1306prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
1307#endif
1308
1309/* gctx->lock is held. */
1310static bool
1311prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt,
1312 prof_gctx_tree_t *gctxs)
1313{
1314 bool ret;
1315 unsigned i;
1316
1317 cassert(config_prof);
1318
1319 /* Avoid dumping such gctx's that have no useful data. */
1320 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
1321 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
1322 assert(gctx->cnt_summed.curobjs == 0);
1323 assert(gctx->cnt_summed.curbytes == 0);
1324 assert(gctx->cnt_summed.accumobjs == 0);
1325 assert(gctx->cnt_summed.accumbytes == 0);
1326 ret = false;
1327 goto label_return;
1328 }
1329
1330 if (prof_dump_printf(propagate_err, "@")) {
1331 ret = true;
1332 goto label_return;
1333 }
1334 for (i = 0; i < bt->len; i++) {
1335 if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
1336 (uintptr_t)bt->vec[i])) {
1337 ret = true;
1338 goto label_return;
1339 }
1340 }
1341
1342 if (prof_dump_printf(propagate_err,
1343 "\n"
1344 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1345 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
1346 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
1347 ret = true;
1348 goto label_return;
1349 }
1350
1351 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
1352 (void *)&propagate_err) != NULL) {
1353 ret = true;
1354 goto label_return;
1355 }
1356
1357 ret = false;
1358label_return:
1359 return (ret);
1360}
1361
|
| 1362#ifndef _WIN32
|
1361JEMALLOC_FORMAT_PRINTF(1, 2)
1362static int
1363prof_open_maps(const char *format, ...)
1364{
1365 int mfd;
1366 va_list ap;
1367 char filename[PATH_MAX + 1];
1368
1369 va_start(ap, format);
1370 malloc_vsnprintf(filename, sizeof(filename), format, ap);
1371 va_end(ap);
1372 mfd = open(filename, O_RDONLY);
1373
1374 return (mfd);
1375}
| 1363JEMALLOC_FORMAT_PRINTF(1, 2)
1364static int
1365prof_open_maps(const char *format, ...)
1366{
1367 int mfd;
1368 va_list ap;
1369 char filename[PATH_MAX + 1];
1370
1371 va_start(ap, format);
1372 malloc_vsnprintf(filename, sizeof(filename), format, ap);
1373 va_end(ap);
1374 mfd = open(filename, O_RDONLY);
1375
1376 return (mfd);
1377}
|
| 1378#endif
|
1376
| 1379
|
| 1380static int
1381prof_getpid(void)
1382{
1383
1384#ifdef _WIN32
1385 return (GetCurrentProcessId());
1386#else
1387 return (getpid());
1388#endif
1389}
1390
|
1377static bool
1378prof_dump_maps(bool propagate_err)
1379{
1380 bool ret;
1381 int mfd;
1382
1383 cassert(config_prof);
1384#ifdef __FreeBSD__
1385 mfd = prof_open_maps("/proc/curproc/map");
| 1391static bool
1392prof_dump_maps(bool propagate_err)
1393{
1394 bool ret;
1395 int mfd;
1396
1397 cassert(config_prof);
1398#ifdef __FreeBSD__
1399 mfd = prof_open_maps("/proc/curproc/map");
|
| 1400#elif defined(_WIN32)
1401 mfd = -1; // Not implemented
|
1386#else
1387 {
| 1402#else
1403 {
|
1388 int pid = getpid();
| 1404 int pid = prof_getpid();
|
1389
1390 mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
1391 if (mfd == -1)
1392 mfd = prof_open_maps("/proc/%d/maps", pid);
1393 }
1394#endif
1395 if (mfd != -1) {
1396 ssize_t nread;
1397
1398 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
1399 propagate_err) {
1400 ret = true;
1401 goto label_return;
1402 }
1403 nread = 0;
1404 do {
1405 prof_dump_buf_end += nread;
1406 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1407 /* Make space in prof_dump_buf before read(). */
1408 if (prof_dump_flush(propagate_err) &&
1409 propagate_err) {
1410 ret = true;
1411 goto label_return;
1412 }
1413 }
1414 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
1415 PROF_DUMP_BUFSIZE - prof_dump_buf_end);
1416 } while (nread > 0);
1417 } else {
1418 ret = true;
1419 goto label_return;
1420 }
1421
1422 ret = false;
1423label_return:
1424 if (mfd != -1)
1425 close(mfd);
1426 return (ret);
1427}
1428
1429static void
1430prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
1431 const char *filename)
1432{
1433
1434 if (cnt_all->curbytes != 0) {
1435 malloc_printf("<jemalloc>: Leak summary: %"FMTu64" byte%s, %"
1436 FMTu64" object%s, %zu context%s\n",
1437 cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "",
1438 cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "",
1439 leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1440 malloc_printf(
1441 "<jemalloc>: Run jeprof on \"%s\" for leak detail\n",
1442 filename);
1443 }
1444}
1445
1446static prof_gctx_t *
1447prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1448{
1449 prof_gctx_t *ret;
1450 bool propagate_err = *(bool *)arg;
1451
1452 malloc_mutex_lock(gctx->lock);
1453
1454 if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) {
1455 ret = gctx;
1456 goto label_return;
1457 }
1458
1459 ret = NULL;
1460label_return:
1461 malloc_mutex_unlock(gctx->lock);
1462 return (ret);
1463}
1464
1465static bool
1466prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
1467{
1468 prof_tdata_t *tdata;
1469 prof_cnt_t cnt_all;
1470 size_t tabind;
1471 union {
1472 prof_gctx_t *p;
1473 void *v;
1474 } gctx;
1475 size_t leak_ngctx;
1476 prof_gctx_tree_t gctxs;
1477
1478 cassert(config_prof);
1479
1480 tdata = prof_tdata_get(tsd, true);
1481 if (tdata == NULL)
1482 return (true);
1483
1484 malloc_mutex_lock(&prof_dump_mtx);
1485 prof_enter(tsd, tdata);
1486
1487 /*
1488 * Put gctx's in limbo and clear their counters in preparation for
1489 * summing.
1490 */
1491 gctx_tree_new(&gctxs);
1492 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);)
1493 prof_dump_gctx_prep(gctx.p, &gctxs);
1494
1495 /*
1496 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1497 * stats and merge them into the associated gctx's.
1498 */
1499 memset(&cnt_all, 0, sizeof(prof_cnt_t));
1500 malloc_mutex_lock(&tdatas_mtx);
1501 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all);
1502 malloc_mutex_unlock(&tdatas_mtx);
1503
1504 /* Merge tctx stats into gctx's. */
1505 leak_ngctx = 0;
1506 gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx);
1507
1508 prof_leave(tsd, tdata);
1509
1510 /* Create dump file. */
1511 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1)
1512 goto label_open_close_error;
1513
1514 /* Dump profile header. */
1515 if (prof_dump_header(propagate_err, &cnt_all))
1516 goto label_write_error;
1517
1518 /* Dump per gctx profile stats. */
1519 if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter,
1520 (void *)&propagate_err) != NULL)
1521 goto label_write_error;
1522
1523 /* Dump /proc/<pid>/maps if possible. */
1524 if (prof_dump_maps(propagate_err))
1525 goto label_write_error;
1526
1527 if (prof_dump_close(propagate_err))
1528 goto label_open_close_error;
1529
1530 prof_gctx_finish(tsd, &gctxs);
1531 malloc_mutex_unlock(&prof_dump_mtx);
1532
1533 if (leakcheck)
1534 prof_leakcheck(&cnt_all, leak_ngctx, filename);
1535
1536 return (false);
1537label_write_error:
1538 prof_dump_close(propagate_err);
1539label_open_close_error:
1540 prof_gctx_finish(tsd, &gctxs);
1541 malloc_mutex_unlock(&prof_dump_mtx);
1542 return (true);
1543}
1544
1545#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
1546#define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
1547static void
1548prof_dump_filename(char *filename, char v, uint64_t vseq)
1549{
1550
1551 cassert(config_prof);
1552
1553 if (vseq != VSEQ_INVALID) {
1554 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
1555 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1556 "%s.%d.%"FMTu64".%c%"FMTu64".heap",
| 1405
1406 mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
1407 if (mfd == -1)
1408 mfd = prof_open_maps("/proc/%d/maps", pid);
1409 }
1410#endif
1411 if (mfd != -1) {
1412 ssize_t nread;
1413
1414 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
1415 propagate_err) {
1416 ret = true;
1417 goto label_return;
1418 }
1419 nread = 0;
1420 do {
1421 prof_dump_buf_end += nread;
1422 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1423 /* Make space in prof_dump_buf before read(). */
1424 if (prof_dump_flush(propagate_err) &&
1425 propagate_err) {
1426 ret = true;
1427 goto label_return;
1428 }
1429 }
1430 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
1431 PROF_DUMP_BUFSIZE - prof_dump_buf_end);
1432 } while (nread > 0);
1433 } else {
1434 ret = true;
1435 goto label_return;
1436 }
1437
1438 ret = false;
1439label_return:
1440 if (mfd != -1)
1441 close(mfd);
1442 return (ret);
1443}
1444
1445static void
1446prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
1447 const char *filename)
1448{
1449
1450 if (cnt_all->curbytes != 0) {
1451 malloc_printf("<jemalloc>: Leak summary: %"FMTu64" byte%s, %"
1452 FMTu64" object%s, %zu context%s\n",
1453 cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "",
1454 cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "",
1455 leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1456 malloc_printf(
1457 "<jemalloc>: Run jeprof on \"%s\" for leak detail\n",
1458 filename);
1459 }
1460}
1461
1462static prof_gctx_t *
1463prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1464{
1465 prof_gctx_t *ret;
1466 bool propagate_err = *(bool *)arg;
1467
1468 malloc_mutex_lock(gctx->lock);
1469
1470 if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) {
1471 ret = gctx;
1472 goto label_return;
1473 }
1474
1475 ret = NULL;
1476label_return:
1477 malloc_mutex_unlock(gctx->lock);
1478 return (ret);
1479}
1480
1481static bool
1482prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
1483{
1484 prof_tdata_t *tdata;
1485 prof_cnt_t cnt_all;
1486 size_t tabind;
1487 union {
1488 prof_gctx_t *p;
1489 void *v;
1490 } gctx;
1491 size_t leak_ngctx;
1492 prof_gctx_tree_t gctxs;
1493
1494 cassert(config_prof);
1495
1496 tdata = prof_tdata_get(tsd, true);
1497 if (tdata == NULL)
1498 return (true);
1499
1500 malloc_mutex_lock(&prof_dump_mtx);
1501 prof_enter(tsd, tdata);
1502
1503 /*
1504 * Put gctx's in limbo and clear their counters in preparation for
1505 * summing.
1506 */
1507 gctx_tree_new(&gctxs);
1508 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);)
1509 prof_dump_gctx_prep(gctx.p, &gctxs);
1510
1511 /*
1512 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1513 * stats and merge them into the associated gctx's.
1514 */
1515 memset(&cnt_all, 0, sizeof(prof_cnt_t));
1516 malloc_mutex_lock(&tdatas_mtx);
1517 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all);
1518 malloc_mutex_unlock(&tdatas_mtx);
1519
1520 /* Merge tctx stats into gctx's. */
1521 leak_ngctx = 0;
1522 gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx);
1523
1524 prof_leave(tsd, tdata);
1525
1526 /* Create dump file. */
1527 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1)
1528 goto label_open_close_error;
1529
1530 /* Dump profile header. */
1531 if (prof_dump_header(propagate_err, &cnt_all))
1532 goto label_write_error;
1533
1534 /* Dump per gctx profile stats. */
1535 if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter,
1536 (void *)&propagate_err) != NULL)
1537 goto label_write_error;
1538
1539 /* Dump /proc/<pid>/maps if possible. */
1540 if (prof_dump_maps(propagate_err))
1541 goto label_write_error;
1542
1543 if (prof_dump_close(propagate_err))
1544 goto label_open_close_error;
1545
1546 prof_gctx_finish(tsd, &gctxs);
1547 malloc_mutex_unlock(&prof_dump_mtx);
1548
1549 if (leakcheck)
1550 prof_leakcheck(&cnt_all, leak_ngctx, filename);
1551
1552 return (false);
1553label_write_error:
1554 prof_dump_close(propagate_err);
1555label_open_close_error:
1556 prof_gctx_finish(tsd, &gctxs);
1557 malloc_mutex_unlock(&prof_dump_mtx);
1558 return (true);
1559}
1560
1561#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
1562#define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
1563static void
1564prof_dump_filename(char *filename, char v, uint64_t vseq)
1565{
1566
1567 cassert(config_prof);
1568
1569 if (vseq != VSEQ_INVALID) {
1570 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
1571 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1572 "%s.%d.%"FMTu64".%c%"FMTu64".heap",
|
1557 opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq);
| 1573 opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq);
|
1558 } else {
1559 /* "<prefix>.<pid>.<seq>.<v>.heap" */
1560 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1561 "%s.%d.%"FMTu64".%c.heap",
| 1574 } else {
1575 /* "<prefix>.<pid>.<seq>.<v>.heap" */
1576 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1577 "%s.%d.%"FMTu64".%c.heap",
|
1562 opt_prof_prefix, (int)getpid(), prof_dump_seq, v);
| 1578 opt_prof_prefix, prof_getpid(), prof_dump_seq, v);
|
1563 }
1564 prof_dump_seq++;
1565}
1566
1567static void
1568prof_fdump(void)
1569{
1570 tsd_t *tsd;
1571 char filename[DUMP_FILENAME_BUFSIZE];
1572
1573 cassert(config_prof);
1574 assert(opt_prof_final);
1575 assert(opt_prof_prefix[0] != '\0');
1576
1577 if (!prof_booted)
1578 return;
1579 tsd = tsd_fetch();
1580
1581 malloc_mutex_lock(&prof_dump_seq_mtx);
1582 prof_dump_filename(filename, 'f', VSEQ_INVALID);
1583 malloc_mutex_unlock(&prof_dump_seq_mtx);
1584 prof_dump(tsd, false, filename, opt_prof_leak);
1585}
1586
1587void
1588prof_idump(void)
1589{
1590 tsd_t *tsd;
1591 prof_tdata_t *tdata;
1592
1593 cassert(config_prof);
1594
1595 if (!prof_booted)
1596 return;
1597 tsd = tsd_fetch();
1598 tdata = prof_tdata_get(tsd, false);
1599 if (tdata == NULL)
1600 return;
1601 if (tdata->enq) {
1602 tdata->enq_idump = true;
1603 return;
1604 }
1605
1606 if (opt_prof_prefix[0] != '\0') {
1607 char filename[PATH_MAX + 1];
1608 malloc_mutex_lock(&prof_dump_seq_mtx);
1609 prof_dump_filename(filename, 'i', prof_dump_iseq);
1610 prof_dump_iseq++;
1611 malloc_mutex_unlock(&prof_dump_seq_mtx);
1612 prof_dump(tsd, false, filename, false);
1613 }
1614}
1615
1616bool
1617prof_mdump(const char *filename)
1618{
1619 tsd_t *tsd;
1620 char filename_buf[DUMP_FILENAME_BUFSIZE];
1621
1622 cassert(config_prof);
1623
1624 if (!opt_prof || !prof_booted)
1625 return (true);
1626 tsd = tsd_fetch();
1627
1628 if (filename == NULL) {
1629 /* No filename specified, so automatically generate one. */
1630 if (opt_prof_prefix[0] == '\0')
1631 return (true);
1632 malloc_mutex_lock(&prof_dump_seq_mtx);
1633 prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1634 prof_dump_mseq++;
1635 malloc_mutex_unlock(&prof_dump_seq_mtx);
1636 filename = filename_buf;
1637 }
1638 return (prof_dump(tsd, true, filename, false));
1639}
1640
1641void
1642prof_gdump(void)
1643{
1644 tsd_t *tsd;
1645 prof_tdata_t *tdata;
1646
1647 cassert(config_prof);
1648
1649 if (!prof_booted)
1650 return;
1651 tsd = tsd_fetch();
1652 tdata = prof_tdata_get(tsd, false);
1653 if (tdata == NULL)
1654 return;
1655 if (tdata->enq) {
1656 tdata->enq_gdump = true;
1657 return;
1658 }
1659
1660 if (opt_prof_prefix[0] != '\0') {
1661 char filename[DUMP_FILENAME_BUFSIZE];
1662 malloc_mutex_lock(&prof_dump_seq_mtx);
1663 prof_dump_filename(filename, 'u', prof_dump_useq);
1664 prof_dump_useq++;
1665 malloc_mutex_unlock(&prof_dump_seq_mtx);
1666 prof_dump(tsd, false, filename, false);
1667 }
1668}
1669
1670static void
1671prof_bt_hash(const void *key, size_t r_hash[2])
1672{
1673 prof_bt_t *bt = (prof_bt_t *)key;
1674
1675 cassert(config_prof);
1676
1677 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1678}
1679
1680static bool
1681prof_bt_keycomp(const void *k1, const void *k2)
1682{
1683 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1684 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1685
1686 cassert(config_prof);
1687
1688 if (bt1->len != bt2->len)
1689 return (false);
1690 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1691}
1692
1693JEMALLOC_INLINE_C uint64_t
1694prof_thr_uid_alloc(void)
1695{
1696 uint64_t thr_uid;
1697
1698 malloc_mutex_lock(&next_thr_uid_mtx);
1699 thr_uid = next_thr_uid;
1700 next_thr_uid++;
1701 malloc_mutex_unlock(&next_thr_uid_mtx);
1702
1703 return (thr_uid);
1704}
1705
1706static prof_tdata_t *
1707prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1708 char *thread_name, bool active)
1709{
1710 prof_tdata_t *tdata;
1711 tcache_t *tcache;
1712
1713 cassert(config_prof);
1714
1715 /* Initialize an empty cache for this thread. */
1716 tcache = tcache_get(tsd, true);
| 1579 }
1580 prof_dump_seq++;
1581}
1582
1583static void
1584prof_fdump(void)
1585{
1586 tsd_t *tsd;
1587 char filename[DUMP_FILENAME_BUFSIZE];
1588
1589 cassert(config_prof);
1590 assert(opt_prof_final);
1591 assert(opt_prof_prefix[0] != '\0');
1592
1593 if (!prof_booted)
1594 return;
1595 tsd = tsd_fetch();
1596
1597 malloc_mutex_lock(&prof_dump_seq_mtx);
1598 prof_dump_filename(filename, 'f', VSEQ_INVALID);
1599 malloc_mutex_unlock(&prof_dump_seq_mtx);
1600 prof_dump(tsd, false, filename, opt_prof_leak);
1601}
1602
1603void
1604prof_idump(void)
1605{
1606 tsd_t *tsd;
1607 prof_tdata_t *tdata;
1608
1609 cassert(config_prof);
1610
1611 if (!prof_booted)
1612 return;
1613 tsd = tsd_fetch();
1614 tdata = prof_tdata_get(tsd, false);
1615 if (tdata == NULL)
1616 return;
1617 if (tdata->enq) {
1618 tdata->enq_idump = true;
1619 return;
1620 }
1621
1622 if (opt_prof_prefix[0] != '\0') {
1623 char filename[PATH_MAX + 1];
1624 malloc_mutex_lock(&prof_dump_seq_mtx);
1625 prof_dump_filename(filename, 'i', prof_dump_iseq);
1626 prof_dump_iseq++;
1627 malloc_mutex_unlock(&prof_dump_seq_mtx);
1628 prof_dump(tsd, false, filename, false);
1629 }
1630}
1631
1632bool
1633prof_mdump(const char *filename)
1634{
1635 tsd_t *tsd;
1636 char filename_buf[DUMP_FILENAME_BUFSIZE];
1637
1638 cassert(config_prof);
1639
1640 if (!opt_prof || !prof_booted)
1641 return (true);
1642 tsd = tsd_fetch();
1643
1644 if (filename == NULL) {
1645 /* No filename specified, so automatically generate one. */
1646 if (opt_prof_prefix[0] == '\0')
1647 return (true);
1648 malloc_mutex_lock(&prof_dump_seq_mtx);
1649 prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1650 prof_dump_mseq++;
1651 malloc_mutex_unlock(&prof_dump_seq_mtx);
1652 filename = filename_buf;
1653 }
1654 return (prof_dump(tsd, true, filename, false));
1655}
1656
1657void
1658prof_gdump(void)
1659{
1660 tsd_t *tsd;
1661 prof_tdata_t *tdata;
1662
1663 cassert(config_prof);
1664
1665 if (!prof_booted)
1666 return;
1667 tsd = tsd_fetch();
1668 tdata = prof_tdata_get(tsd, false);
1669 if (tdata == NULL)
1670 return;
1671 if (tdata->enq) {
1672 tdata->enq_gdump = true;
1673 return;
1674 }
1675
1676 if (opt_prof_prefix[0] != '\0') {
1677 char filename[DUMP_FILENAME_BUFSIZE];
1678 malloc_mutex_lock(&prof_dump_seq_mtx);
1679 prof_dump_filename(filename, 'u', prof_dump_useq);
1680 prof_dump_useq++;
1681 malloc_mutex_unlock(&prof_dump_seq_mtx);
1682 prof_dump(tsd, false, filename, false);
1683 }
1684}
1685
1686static void
1687prof_bt_hash(const void *key, size_t r_hash[2])
1688{
1689 prof_bt_t *bt = (prof_bt_t *)key;
1690
1691 cassert(config_prof);
1692
1693 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1694}
1695
1696static bool
1697prof_bt_keycomp(const void *k1, const void *k2)
1698{
1699 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1700 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1701
1702 cassert(config_prof);
1703
1704 if (bt1->len != bt2->len)
1705 return (false);
1706 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1707}
1708
1709JEMALLOC_INLINE_C uint64_t
1710prof_thr_uid_alloc(void)
1711{
1712 uint64_t thr_uid;
1713
1714 malloc_mutex_lock(&next_thr_uid_mtx);
1715 thr_uid = next_thr_uid;
1716 next_thr_uid++;
1717 malloc_mutex_unlock(&next_thr_uid_mtx);
1718
1719 return (thr_uid);
1720}
1721
1722static prof_tdata_t *
1723prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1724 char *thread_name, bool active)
1725{
1726 prof_tdata_t *tdata;
1727 tcache_t *tcache;
1728
1729 cassert(config_prof);
1730
1731 /* Initialize an empty cache for this thread. */
1732 tcache = tcache_get(tsd, true);
|
1717 tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false,
1718 tcache, true, NULL);
| 1733 tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t),
1734 size2index(sizeof(prof_tdata_t)), false, tcache, true, NULL, true);
|
1719 if (tdata == NULL)
1720 return (NULL);
1721
1722 tdata->lock = prof_tdata_mutex_choose(thr_uid);
1723 tdata->thr_uid = thr_uid;
1724 tdata->thr_discrim = thr_discrim;
1725 tdata->thread_name = thread_name;
1726 tdata->attached = true;
1727 tdata->expired = false;
1728 tdata->tctx_uid_next = 0;
1729
1730 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS,
1731 prof_bt_hash, prof_bt_keycomp)) {
| 1735 if (tdata == NULL)
1736 return (NULL);
1737
1738 tdata->lock = prof_tdata_mutex_choose(thr_uid);
1739 tdata->thr_uid = thr_uid;
1740 tdata->thr_discrim = thr_discrim;
1741 tdata->thread_name = thread_name;
1742 tdata->attached = true;
1743 tdata->expired = false;
1744 tdata->tctx_uid_next = 0;
1745
1746 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS,
1747 prof_bt_hash, prof_bt_keycomp)) {
|
1732 idalloctm(tsd, tdata, tcache, true);
| 1748 idalloctm(tsd, tdata, tcache, true, true);
|
1733 return (NULL);
1734 }
1735
1736 tdata->prng_state = (uint64_t)(uintptr_t)tdata;
1737 prof_sample_threshold_update(tdata);
1738
1739 tdata->enq = false;
1740 tdata->enq_idump = false;
1741 tdata->enq_gdump = false;
1742
1743 tdata->dumping = false;
1744 tdata->active = active;
1745
1746 malloc_mutex_lock(&tdatas_mtx);
1747 tdata_tree_insert(&tdatas, tdata);
1748 malloc_mutex_unlock(&tdatas_mtx);
1749
1750 return (tdata);
1751}
1752
1753prof_tdata_t *
1754prof_tdata_init(tsd_t *tsd)
1755{
1756
1757 return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL,
1758 prof_thread_active_init_get()));
1759}
1760
1761/* tdata->lock must be held. */
1762static bool
1763prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached)
1764{
1765
1766 if (tdata->attached && !even_if_attached)
1767 return (false);
1768 if (ckh_count(&tdata->bt2tctx) != 0)
1769 return (false);
1770 return (true);
1771}
1772
1773/* tdatas_mtx must be held. */
1774static void
1775prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1776 bool even_if_attached)
1777{
1778 tcache_t *tcache;
1779
1780 assert(prof_tdata_should_destroy(tdata, even_if_attached));
1781 assert(tsd_prof_tdata_get(tsd) != tdata);
1782
1783 tdata_tree_remove(&tdatas, tdata);
1784
1785 tcache = tcache_get(tsd, false);
1786 if (tdata->thread_name != NULL)
| 1749 return (NULL);
1750 }
1751
1752 tdata->prng_state = (uint64_t)(uintptr_t)tdata;
1753 prof_sample_threshold_update(tdata);
1754
1755 tdata->enq = false;
1756 tdata->enq_idump = false;
1757 tdata->enq_gdump = false;
1758
1759 tdata->dumping = false;
1760 tdata->active = active;
1761
1762 malloc_mutex_lock(&tdatas_mtx);
1763 tdata_tree_insert(&tdatas, tdata);
1764 malloc_mutex_unlock(&tdatas_mtx);
1765
1766 return (tdata);
1767}
1768
1769prof_tdata_t *
1770prof_tdata_init(tsd_t *tsd)
1771{
1772
1773 return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL,
1774 prof_thread_active_init_get()));
1775}
1776
1777/* tdata->lock must be held. */
1778static bool
1779prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached)
1780{
1781
1782 if (tdata->attached && !even_if_attached)
1783 return (false);
1784 if (ckh_count(&tdata->bt2tctx) != 0)
1785 return (false);
1786 return (true);
1787}
1788
1789/* tdatas_mtx must be held. */
1790static void
1791prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1792 bool even_if_attached)
1793{
1794 tcache_t *tcache;
1795
1796 assert(prof_tdata_should_destroy(tdata, even_if_attached));
1797 assert(tsd_prof_tdata_get(tsd) != tdata);
1798
1799 tdata_tree_remove(&tdatas, tdata);
1800
1801 tcache = tcache_get(tsd, false);
1802 if (tdata->thread_name != NULL)
|
1787 idalloctm(tsd, tdata->thread_name, tcache, true);
| 1803 idalloctm(tsd, tdata->thread_name, tcache, true, true);
|
1788 ckh_delete(tsd, &tdata->bt2tctx);
| 1804 ckh_delete(tsd, &tdata->bt2tctx);
|
1789 idalloctm(tsd, tdata, tcache, true);
| 1805 idalloctm(tsd, tdata, tcache, true, true);
|
1790}
1791
1792static void
1793prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached)
1794{
1795
1796 malloc_mutex_lock(&tdatas_mtx);
1797 prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1798 malloc_mutex_unlock(&tdatas_mtx);
1799}
1800
1801static void
1802prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
1803{
1804 bool destroy_tdata;
1805
1806 malloc_mutex_lock(tdata->lock);
1807 if (tdata->attached) {
1808 destroy_tdata = prof_tdata_should_destroy(tdata, true);
1809 /*
1810 * Only detach if !destroy_tdata, because detaching would allow
1811 * another thread to win the race to destroy tdata.
1812 */
1813 if (!destroy_tdata)
1814 tdata->attached = false;
1815 tsd_prof_tdata_set(tsd, NULL);
1816 } else
1817 destroy_tdata = false;
1818 malloc_mutex_unlock(tdata->lock);
1819 if (destroy_tdata)
1820 prof_tdata_destroy(tsd, tdata, true);
1821}
1822
1823prof_tdata_t *
1824prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
1825{
1826 uint64_t thr_uid = tdata->thr_uid;
1827 uint64_t thr_discrim = tdata->thr_discrim + 1;
1828 char *thread_name = (tdata->thread_name != NULL) ?
1829 prof_thread_name_alloc(tsd, tdata->thread_name) : NULL;
1830 bool active = tdata->active;
1831
1832 prof_tdata_detach(tsd, tdata);
1833 return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
1834 active));
1835}
1836
1837static bool
1838prof_tdata_expire(prof_tdata_t *tdata)
1839{
1840 bool destroy_tdata;
1841
1842 malloc_mutex_lock(tdata->lock);
1843 if (!tdata->expired) {
1844 tdata->expired = true;
1845 destroy_tdata = tdata->attached ? false :
1846 prof_tdata_should_destroy(tdata, false);
1847 } else
1848 destroy_tdata = false;
1849 malloc_mutex_unlock(tdata->lock);
1850
1851 return (destroy_tdata);
1852}
1853
1854static prof_tdata_t *
1855prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1856{
1857
1858 return (prof_tdata_expire(tdata) ? tdata : NULL);
1859}
1860
1861void
1862prof_reset(tsd_t *tsd, size_t lg_sample)
1863{
1864 prof_tdata_t *next;
1865
1866 assert(lg_sample < (sizeof(uint64_t) << 3));
1867
1868 malloc_mutex_lock(&prof_dump_mtx);
1869 malloc_mutex_lock(&tdatas_mtx);
1870
1871 lg_prof_sample = lg_sample;
1872
1873 next = NULL;
1874 do {
1875 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1876 prof_tdata_reset_iter, NULL);
1877 if (to_destroy != NULL) {
1878 next = tdata_tree_next(&tdatas, to_destroy);
1879 prof_tdata_destroy_locked(tsd, to_destroy, false);
1880 } else
1881 next = NULL;
1882 } while (next != NULL);
1883
1884 malloc_mutex_unlock(&tdatas_mtx);
1885 malloc_mutex_unlock(&prof_dump_mtx);
1886}
1887
1888void
1889prof_tdata_cleanup(tsd_t *tsd)
1890{
1891 prof_tdata_t *tdata;
1892
1893 if (!config_prof)
1894 return;
1895
1896 tdata = tsd_prof_tdata_get(tsd);
1897 if (tdata != NULL)
1898 prof_tdata_detach(tsd, tdata);
1899}
1900
1901bool
1902prof_active_get(void)
1903{
1904 bool prof_active_current;
1905
1906 malloc_mutex_lock(&prof_active_mtx);
1907 prof_active_current = prof_active;
1908 malloc_mutex_unlock(&prof_active_mtx);
1909 return (prof_active_current);
1910}
1911
1912bool
1913prof_active_set(bool active)
1914{
1915 bool prof_active_old;
1916
1917 malloc_mutex_lock(&prof_active_mtx);
1918 prof_active_old = prof_active;
1919 prof_active = active;
1920 malloc_mutex_unlock(&prof_active_mtx);
1921 return (prof_active_old);
1922}
1923
1924const char *
1925prof_thread_name_get(void)
1926{
1927 tsd_t *tsd;
1928 prof_tdata_t *tdata;
1929
1930 tsd = tsd_fetch();
1931 tdata = prof_tdata_get(tsd, true);
1932 if (tdata == NULL)
1933 return ("");
1934 return (tdata->thread_name != NULL ? tdata->thread_name : "");
1935}
1936
1937static char *
1938prof_thread_name_alloc(tsd_t *tsd, const char *thread_name)
1939{
1940 char *ret;
1941 size_t size;
1942
1943 if (thread_name == NULL)
1944 return (NULL);
1945
1946 size = strlen(thread_name) + 1;
1947 if (size == 1)
1948 return ("");
1949
| 1806}
1807
1808static void
1809prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached)
1810{
1811
1812 malloc_mutex_lock(&tdatas_mtx);
1813 prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1814 malloc_mutex_unlock(&tdatas_mtx);
1815}
1816
1817static void
1818prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
1819{
1820 bool destroy_tdata;
1821
1822 malloc_mutex_lock(tdata->lock);
1823 if (tdata->attached) {
1824 destroy_tdata = prof_tdata_should_destroy(tdata, true);
1825 /*
1826 * Only detach if !destroy_tdata, because detaching would allow
1827 * another thread to win the race to destroy tdata.
1828 */
1829 if (!destroy_tdata)
1830 tdata->attached = false;
1831 tsd_prof_tdata_set(tsd, NULL);
1832 } else
1833 destroy_tdata = false;
1834 malloc_mutex_unlock(tdata->lock);
1835 if (destroy_tdata)
1836 prof_tdata_destroy(tsd, tdata, true);
1837}
1838
1839prof_tdata_t *
1840prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
1841{
1842 uint64_t thr_uid = tdata->thr_uid;
1843 uint64_t thr_discrim = tdata->thr_discrim + 1;
1844 char *thread_name = (tdata->thread_name != NULL) ?
1845 prof_thread_name_alloc(tsd, tdata->thread_name) : NULL;
1846 bool active = tdata->active;
1847
1848 prof_tdata_detach(tsd, tdata);
1849 return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
1850 active));
1851}
1852
1853static bool
1854prof_tdata_expire(prof_tdata_t *tdata)
1855{
1856 bool destroy_tdata;
1857
1858 malloc_mutex_lock(tdata->lock);
1859 if (!tdata->expired) {
1860 tdata->expired = true;
1861 destroy_tdata = tdata->attached ? false :
1862 prof_tdata_should_destroy(tdata, false);
1863 } else
1864 destroy_tdata = false;
1865 malloc_mutex_unlock(tdata->lock);
1866
1867 return (destroy_tdata);
1868}
1869
1870static prof_tdata_t *
1871prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1872{
1873
1874 return (prof_tdata_expire(tdata) ? tdata : NULL);
1875}
1876
1877void
1878prof_reset(tsd_t *tsd, size_t lg_sample)
1879{
1880 prof_tdata_t *next;
1881
1882 assert(lg_sample < (sizeof(uint64_t) << 3));
1883
1884 malloc_mutex_lock(&prof_dump_mtx);
1885 malloc_mutex_lock(&tdatas_mtx);
1886
1887 lg_prof_sample = lg_sample;
1888
1889 next = NULL;
1890 do {
1891 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1892 prof_tdata_reset_iter, NULL);
1893 if (to_destroy != NULL) {
1894 next = tdata_tree_next(&tdatas, to_destroy);
1895 prof_tdata_destroy_locked(tsd, to_destroy, false);
1896 } else
1897 next = NULL;
1898 } while (next != NULL);
1899
1900 malloc_mutex_unlock(&tdatas_mtx);
1901 malloc_mutex_unlock(&prof_dump_mtx);
1902}
1903
1904void
1905prof_tdata_cleanup(tsd_t *tsd)
1906{
1907 prof_tdata_t *tdata;
1908
1909 if (!config_prof)
1910 return;
1911
1912 tdata = tsd_prof_tdata_get(tsd);
1913 if (tdata != NULL)
1914 prof_tdata_detach(tsd, tdata);
1915}
1916
1917bool
1918prof_active_get(void)
1919{
1920 bool prof_active_current;
1921
1922 malloc_mutex_lock(&prof_active_mtx);
1923 prof_active_current = prof_active;
1924 malloc_mutex_unlock(&prof_active_mtx);
1925 return (prof_active_current);
1926}
1927
1928bool
1929prof_active_set(bool active)
1930{
1931 bool prof_active_old;
1932
1933 malloc_mutex_lock(&prof_active_mtx);
1934 prof_active_old = prof_active;
1935 prof_active = active;
1936 malloc_mutex_unlock(&prof_active_mtx);
1937 return (prof_active_old);
1938}
1939
1940const char *
1941prof_thread_name_get(void)
1942{
1943 tsd_t *tsd;
1944 prof_tdata_t *tdata;
1945
1946 tsd = tsd_fetch();
1947 tdata = prof_tdata_get(tsd, true);
1948 if (tdata == NULL)
1949 return ("");
1950 return (tdata->thread_name != NULL ? tdata->thread_name : "");
1951}
1952
1953static char *
1954prof_thread_name_alloc(tsd_t *tsd, const char *thread_name)
1955{
1956 char *ret;
1957 size_t size;
1958
1959 if (thread_name == NULL)
1960 return (NULL);
1961
1962 size = strlen(thread_name) + 1;
1963 if (size == 1)
1964 return ("");
1965
|
1950 ret = iallocztm(tsd, size, false, tcache_get(tsd, true), true, NULL);
| 1966 ret = iallocztm(tsd, size, size2index(size), false, tcache_get(tsd,
1967 true), true, NULL, true);
|
1951 if (ret == NULL)
1952 return (NULL);
1953 memcpy(ret, thread_name, size);
1954 return (ret);
1955}
1956
1957int
1958prof_thread_name_set(tsd_t *tsd, const char *thread_name)
1959{
1960 prof_tdata_t *tdata;
1961 unsigned i;
1962 char *s;
1963
1964 tdata = prof_tdata_get(tsd, true);
1965 if (tdata == NULL)
1966 return (EAGAIN);
1967
1968 /* Validate input. */
1969 if (thread_name == NULL)
1970 return (EFAULT);
1971 for (i = 0; thread_name[i] != '\0'; i++) {
1972 char c = thread_name[i];
1973 if (!isgraph(c) && !isblank(c))
1974 return (EFAULT);
1975 }
1976
1977 s = prof_thread_name_alloc(tsd, thread_name);
1978 if (s == NULL)
1979 return (EAGAIN);
1980
1981 if (tdata->thread_name != NULL) {
1982 idalloctm(tsd, tdata->thread_name, tcache_get(tsd, false),
| 1968 if (ret == NULL)
1969 return (NULL);
1970 memcpy(ret, thread_name, size);
1971 return (ret);
1972}
1973
1974int
1975prof_thread_name_set(tsd_t *tsd, const char *thread_name)
1976{
1977 prof_tdata_t *tdata;
1978 unsigned i;
1979 char *s;
1980
1981 tdata = prof_tdata_get(tsd, true);
1982 if (tdata == NULL)
1983 return (EAGAIN);
1984
1985 /* Validate input. */
1986 if (thread_name == NULL)
1987 return (EFAULT);
1988 for (i = 0; thread_name[i] != '\0'; i++) {
1989 char c = thread_name[i];
1990 if (!isgraph(c) && !isblank(c))
1991 return (EFAULT);
1992 }
1993
1994 s = prof_thread_name_alloc(tsd, thread_name);
1995 if (s == NULL)
1996 return (EAGAIN);
1997
1998 if (tdata->thread_name != NULL) {
1999 idalloctm(tsd, tdata->thread_name, tcache_get(tsd, false),
|
1983 true);
| 2000 true, true);
|
1984 tdata->thread_name = NULL;
1985 }
1986 if (strlen(s) > 0)
1987 tdata->thread_name = s;
1988 return (0);
1989}
1990
1991bool
1992prof_thread_active_get(void)
1993{
1994 tsd_t *tsd;
1995 prof_tdata_t *tdata;
1996
1997 tsd = tsd_fetch();
1998 tdata = prof_tdata_get(tsd, true);
1999 if (tdata == NULL)
2000 return (false);
2001 return (tdata->active);
2002}
2003
2004bool
2005prof_thread_active_set(bool active)
2006{
2007 tsd_t *tsd;
2008 prof_tdata_t *tdata;
2009
2010 tsd = tsd_fetch();
2011 tdata = prof_tdata_get(tsd, true);
2012 if (tdata == NULL)
2013 return (true);
2014 tdata->active = active;
2015 return (false);
2016}
2017
2018bool
2019prof_thread_active_init_get(void)
2020{
2021 bool active_init;
2022
2023 malloc_mutex_lock(&prof_thread_active_init_mtx);
2024 active_init = prof_thread_active_init;
2025 malloc_mutex_unlock(&prof_thread_active_init_mtx);
2026 return (active_init);
2027}
2028
2029bool
2030prof_thread_active_init_set(bool active_init)
2031{
2032 bool active_init_old;
2033
2034 malloc_mutex_lock(&prof_thread_active_init_mtx);
2035 active_init_old = prof_thread_active_init;
2036 prof_thread_active_init = active_init;
2037 malloc_mutex_unlock(&prof_thread_active_init_mtx);
2038 return (active_init_old);
2039}
2040
2041bool
2042prof_gdump_get(void)
2043{
2044 bool prof_gdump_current;
2045
2046 malloc_mutex_lock(&prof_gdump_mtx);
2047 prof_gdump_current = prof_gdump_val;
2048 malloc_mutex_unlock(&prof_gdump_mtx);
2049 return (prof_gdump_current);
2050}
2051
2052bool
2053prof_gdump_set(bool gdump)
2054{
2055 bool prof_gdump_old;
2056
2057 malloc_mutex_lock(&prof_gdump_mtx);
2058 prof_gdump_old = prof_gdump_val;
2059 prof_gdump_val = gdump;
2060 malloc_mutex_unlock(&prof_gdump_mtx);
2061 return (prof_gdump_old);
2062}
2063
2064void
2065prof_boot0(void)
2066{
2067
2068 cassert(config_prof);
2069
2070 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
2071 sizeof(PROF_PREFIX_DEFAULT));
2072}
2073
2074void
2075prof_boot1(void)
2076{
2077
2078 cassert(config_prof);
2079
2080 /*
2081 * opt_prof must be in its final state before any arenas are
2082 * initialized, so this function must be executed early.
2083 */
2084
2085 if (opt_prof_leak && !opt_prof) {
2086 /*
2087 * Enable opt_prof, but in such a way that profiles are never
2088 * automatically dumped.
2089 */
2090 opt_prof = true;
2091 opt_prof_gdump = false;
2092 } else if (opt_prof) {
2093 if (opt_lg_prof_interval >= 0) {
2094 prof_interval = (((uint64_t)1U) <<
2095 opt_lg_prof_interval);
2096 }
2097 }
2098}
2099
2100bool
2101prof_boot2(void)
2102{
2103
2104 cassert(config_prof);
2105
2106 if (opt_prof) {
2107 tsd_t *tsd;
2108 unsigned i;
2109
2110 lg_prof_sample = opt_lg_prof_sample;
2111
2112 prof_active = opt_prof_active;
2113 if (malloc_mutex_init(&prof_active_mtx))
2114 return (true);
2115
2116 prof_gdump_val = opt_prof_gdump;
2117 if (malloc_mutex_init(&prof_gdump_mtx))
2118 return (true);
2119
2120 prof_thread_active_init = opt_prof_thread_active_init;
2121 if (malloc_mutex_init(&prof_thread_active_init_mtx))
2122 return (true);
2123
2124 tsd = tsd_fetch();
2125 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
2126 prof_bt_keycomp))
2127 return (true);
2128 if (malloc_mutex_init(&bt2gctx_mtx))
2129 return (true);
2130
2131 tdata_tree_new(&tdatas);
2132 if (malloc_mutex_init(&tdatas_mtx))
2133 return (true);
2134
2135 next_thr_uid = 0;
2136 if (malloc_mutex_init(&next_thr_uid_mtx))
2137 return (true);
2138
2139 if (malloc_mutex_init(&prof_dump_seq_mtx))
2140 return (true);
2141 if (malloc_mutex_init(&prof_dump_mtx))
2142 return (true);
2143
2144 if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
2145 atexit(prof_fdump) != 0) {
2146 malloc_write("<jemalloc>: Error in atexit()\n");
2147 if (opt_abort)
2148 abort();
2149 }
2150
2151 gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS *
2152 sizeof(malloc_mutex_t));
2153 if (gctx_locks == NULL)
2154 return (true);
2155 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2156 if (malloc_mutex_init(&gctx_locks[i]))
2157 return (true);
2158 }
2159
2160 tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS *
2161 sizeof(malloc_mutex_t));
2162 if (tdata_locks == NULL)
2163 return (true);
2164 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2165 if (malloc_mutex_init(&tdata_locks[i]))
2166 return (true);
2167 }
2168 }
2169
2170#ifdef JEMALLOC_PROF_LIBGCC
2171 /*
2172 * Cause the backtracing machinery to allocate its internal state
2173 * before enabling profiling.
2174 */
2175 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
2176#endif
2177
2178 prof_booted = true;
2179
2180 return (false);
2181}
2182
2183void
2184prof_prefork(void)
2185{
2186
2187 if (opt_prof) {
2188 unsigned i;
2189
2190 malloc_mutex_prefork(&tdatas_mtx);
2191 malloc_mutex_prefork(&bt2gctx_mtx);
2192 malloc_mutex_prefork(&next_thr_uid_mtx);
2193 malloc_mutex_prefork(&prof_dump_seq_mtx);
2194 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2195 malloc_mutex_prefork(&gctx_locks[i]);
2196 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2197 malloc_mutex_prefork(&tdata_locks[i]);
2198 }
2199}
2200
2201void
2202prof_postfork_parent(void)
2203{
2204
2205 if (opt_prof) {
2206 unsigned i;
2207
2208 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2209 malloc_mutex_postfork_parent(&tdata_locks[i]);
2210 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2211 malloc_mutex_postfork_parent(&gctx_locks[i]);
2212 malloc_mutex_postfork_parent(&prof_dump_seq_mtx);
2213 malloc_mutex_postfork_parent(&next_thr_uid_mtx);
2214 malloc_mutex_postfork_parent(&bt2gctx_mtx);
2215 malloc_mutex_postfork_parent(&tdatas_mtx);
2216 }
2217}
2218
2219void
2220prof_postfork_child(void)
2221{
2222
2223 if (opt_prof) {
2224 unsigned i;
2225
2226 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2227 malloc_mutex_postfork_child(&tdata_locks[i]);
2228 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2229 malloc_mutex_postfork_child(&gctx_locks[i]);
2230 malloc_mutex_postfork_child(&prof_dump_seq_mtx);
2231 malloc_mutex_postfork_child(&next_thr_uid_mtx);
2232 malloc_mutex_postfork_child(&bt2gctx_mtx);
2233 malloc_mutex_postfork_child(&tdatas_mtx);
2234 }
2235}
2236
2237/******************************************************************************/
| 2001 tdata->thread_name = NULL;
2002 }
2003 if (strlen(s) > 0)
2004 tdata->thread_name = s;
2005 return (0);
2006}
2007
2008bool
2009prof_thread_active_get(void)
2010{
2011 tsd_t *tsd;
2012 prof_tdata_t *tdata;
2013
2014 tsd = tsd_fetch();
2015 tdata = prof_tdata_get(tsd, true);
2016 if (tdata == NULL)
2017 return (false);
2018 return (tdata->active);
2019}
2020
2021bool
2022prof_thread_active_set(bool active)
2023{
2024 tsd_t *tsd;
2025 prof_tdata_t *tdata;
2026
2027 tsd = tsd_fetch();
2028 tdata = prof_tdata_get(tsd, true);
2029 if (tdata == NULL)
2030 return (true);
2031 tdata->active = active;
2032 return (false);
2033}
2034
2035bool
2036prof_thread_active_init_get(void)
2037{
2038 bool active_init;
2039
2040 malloc_mutex_lock(&prof_thread_active_init_mtx);
2041 active_init = prof_thread_active_init;
2042 malloc_mutex_unlock(&prof_thread_active_init_mtx);
2043 return (active_init);
2044}
2045
2046bool
2047prof_thread_active_init_set(bool active_init)
2048{
2049 bool active_init_old;
2050
2051 malloc_mutex_lock(&prof_thread_active_init_mtx);
2052 active_init_old = prof_thread_active_init;
2053 prof_thread_active_init = active_init;
2054 malloc_mutex_unlock(&prof_thread_active_init_mtx);
2055 return (active_init_old);
2056}
2057
2058bool
2059prof_gdump_get(void)
2060{
2061 bool prof_gdump_current;
2062
2063 malloc_mutex_lock(&prof_gdump_mtx);
2064 prof_gdump_current = prof_gdump_val;
2065 malloc_mutex_unlock(&prof_gdump_mtx);
2066 return (prof_gdump_current);
2067}
2068
2069bool
2070prof_gdump_set(bool gdump)
2071{
2072 bool prof_gdump_old;
2073
2074 malloc_mutex_lock(&prof_gdump_mtx);
2075 prof_gdump_old = prof_gdump_val;
2076 prof_gdump_val = gdump;
2077 malloc_mutex_unlock(&prof_gdump_mtx);
2078 return (prof_gdump_old);
2079}
2080
2081void
2082prof_boot0(void)
2083{
2084
2085 cassert(config_prof);
2086
2087 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
2088 sizeof(PROF_PREFIX_DEFAULT));
2089}
2090
2091void
2092prof_boot1(void)
2093{
2094
2095 cassert(config_prof);
2096
2097 /*
2098 * opt_prof must be in its final state before any arenas are
2099 * initialized, so this function must be executed early.
2100 */
2101
2102 if (opt_prof_leak && !opt_prof) {
2103 /*
2104 * Enable opt_prof, but in such a way that profiles are never
2105 * automatically dumped.
2106 */
2107 opt_prof = true;
2108 opt_prof_gdump = false;
2109 } else if (opt_prof) {
2110 if (opt_lg_prof_interval >= 0) {
2111 prof_interval = (((uint64_t)1U) <<
2112 opt_lg_prof_interval);
2113 }
2114 }
2115}
2116
2117bool
2118prof_boot2(void)
2119{
2120
2121 cassert(config_prof);
2122
2123 if (opt_prof) {
2124 tsd_t *tsd;
2125 unsigned i;
2126
2127 lg_prof_sample = opt_lg_prof_sample;
2128
2129 prof_active = opt_prof_active;
2130 if (malloc_mutex_init(&prof_active_mtx))
2131 return (true);
2132
2133 prof_gdump_val = opt_prof_gdump;
2134 if (malloc_mutex_init(&prof_gdump_mtx))
2135 return (true);
2136
2137 prof_thread_active_init = opt_prof_thread_active_init;
2138 if (malloc_mutex_init(&prof_thread_active_init_mtx))
2139 return (true);
2140
2141 tsd = tsd_fetch();
2142 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
2143 prof_bt_keycomp))
2144 return (true);
2145 if (malloc_mutex_init(&bt2gctx_mtx))
2146 return (true);
2147
2148 tdata_tree_new(&tdatas);
2149 if (malloc_mutex_init(&tdatas_mtx))
2150 return (true);
2151
2152 next_thr_uid = 0;
2153 if (malloc_mutex_init(&next_thr_uid_mtx))
2154 return (true);
2155
2156 if (malloc_mutex_init(&prof_dump_seq_mtx))
2157 return (true);
2158 if (malloc_mutex_init(&prof_dump_mtx))
2159 return (true);
2160
2161 if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
2162 atexit(prof_fdump) != 0) {
2163 malloc_write("<jemalloc>: Error in atexit()\n");
2164 if (opt_abort)
2165 abort();
2166 }
2167
2168 gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS *
2169 sizeof(malloc_mutex_t));
2170 if (gctx_locks == NULL)
2171 return (true);
2172 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2173 if (malloc_mutex_init(&gctx_locks[i]))
2174 return (true);
2175 }
2176
2177 tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS *
2178 sizeof(malloc_mutex_t));
2179 if (tdata_locks == NULL)
2180 return (true);
2181 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2182 if (malloc_mutex_init(&tdata_locks[i]))
2183 return (true);
2184 }
2185 }
2186
2187#ifdef JEMALLOC_PROF_LIBGCC
2188 /*
2189 * Cause the backtracing machinery to allocate its internal state
2190 * before enabling profiling.
2191 */
2192 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
2193#endif
2194
2195 prof_booted = true;
2196
2197 return (false);
2198}
2199
2200void
2201prof_prefork(void)
2202{
2203
2204 if (opt_prof) {
2205 unsigned i;
2206
2207 malloc_mutex_prefork(&tdatas_mtx);
2208 malloc_mutex_prefork(&bt2gctx_mtx);
2209 malloc_mutex_prefork(&next_thr_uid_mtx);
2210 malloc_mutex_prefork(&prof_dump_seq_mtx);
2211 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2212 malloc_mutex_prefork(&gctx_locks[i]);
2213 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2214 malloc_mutex_prefork(&tdata_locks[i]);
2215 }
2216}
2217
2218void
2219prof_postfork_parent(void)
2220{
2221
2222 if (opt_prof) {
2223 unsigned i;
2224
2225 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2226 malloc_mutex_postfork_parent(&tdata_locks[i]);
2227 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2228 malloc_mutex_postfork_parent(&gctx_locks[i]);
2229 malloc_mutex_postfork_parent(&prof_dump_seq_mtx);
2230 malloc_mutex_postfork_parent(&next_thr_uid_mtx);
2231 malloc_mutex_postfork_parent(&bt2gctx_mtx);
2232 malloc_mutex_postfork_parent(&tdatas_mtx);
2233 }
2234}
2235
2236void
2237prof_postfork_child(void)
2238{
2239
2240 if (opt_prof) {
2241 unsigned i;
2242
2243 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2244 malloc_mutex_postfork_child(&tdata_locks[i]);
2245 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2246 malloc_mutex_postfork_child(&gctx_locks[i]);
2247 malloc_mutex_postfork_child(&prof_dump_seq_mtx);
2248 malloc_mutex_postfork_child(&next_thr_uid_mtx);
2249 malloc_mutex_postfork_child(&bt2gctx_mtx);
2250 malloc_mutex_postfork_child(&tdatas_mtx);
2251 }
2252}
2253
2254/******************************************************************************/
|