1#define JEMALLOC_MALLOC_IO_C_
2#include "jemalloc/internal/jemalloc_preamble.h"
3#include "jemalloc/internal/jemalloc_internal_includes.h"
4
5#include "jemalloc/internal/malloc_io.h"
6#include "jemalloc/internal/util.h"
7
8#ifdef assert
9#  undef assert
10#endif
11#ifdef not_reached
12#  undef not_reached
13#endif
14#ifdef not_implemented
15#  undef not_implemented
16#endif
17#ifdef assert_not_implemented
18#  undef assert_not_implemented
19#endif
20
21/*
22 * Define simple versions of assertion macros that won't recurse in case
23 * of assertion failures in malloc_*printf().
24 */
25#define assert(e) do {							\
26	if (config_debug && !(e)) {					\
27		malloc_write("<jemalloc>: Failed assertion\n");		\
28		abort();						\
29	}								\
30} while (0)
31
32#define not_reached() do {						\
33	if (config_debug) {						\
34		malloc_write("<jemalloc>: Unreachable code reached\n");	\
35		abort();						\
36	}								\
37	unreachable();							\
38} while (0)
39
40#define not_implemented() do {						\
41	if (config_debug) {						\
42		malloc_write("<jemalloc>: Not implemented\n");		\
43		abort();						\
44	}								\
45} while (0)
46
47#define assert_not_implemented(e) do {					\
48	if (unlikely(config_debug && !(e))) {				\
49		not_implemented();					\
50	}								\
51} while (0)
52
53/******************************************************************************/
54/* Function prototypes for non-inline static functions. */
55
56static void wrtmessage(void *cbopaque, const char *s);
57#define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
58static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
59    size_t *slen_p);
60#define D2S_BUFSIZE (1 + U2S_BUFSIZE)
61static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
62#define O2S_BUFSIZE (1 + U2S_BUFSIZE)
63static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
64#define X2S_BUFSIZE (2 + U2S_BUFSIZE)
65static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
66    size_t *slen_p);
67
68/******************************************************************************/
69
70/* malloc_message() setup. */
71static void
72wrtmessage(void *cbopaque, const char *s) {
73	malloc_write_fd(STDERR_FILENO, s, strlen(s));
74}
75
76JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);
77
78JEMALLOC_ATTR(visibility("hidden"))
79void
80wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
81
82	wrtmessage(NULL, s1);
83	wrtmessage(NULL, s2);
84	wrtmessage(NULL, s3);
85	wrtmessage(NULL, s4);
86}
87
88void	(*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
89    const char *s4) = wrtmessage_1_0;
90__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
91
92/*
93 * Wrapper around malloc_message() that avoids the need for
94 * je_malloc_message(...) throughout the code.
95 */
96void
97malloc_write(const char *s) {
98	if (je_malloc_message != NULL) {
99		je_malloc_message(NULL, s);
100	} else {
101		wrtmessage(NULL, s);
102	}
103}
104
105/*
106 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
107 * provide a wrapper.
108 */
109int
110buferror(int err, char *buf, size_t buflen) {
111#ifdef _WIN32
112	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
113	    (LPSTR)buf, (DWORD)buflen, NULL);
114	return 0;
115#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE)
116	char *b = strerror_r(err, buf, buflen);
117	if (b != buf) {
118		strncpy(buf, b, buflen);
119		buf[buflen-1] = '\0';
120	}
121	return 0;
122#else
123	return strerror_r(err, buf, buflen);
124#endif
125}
126
127uintmax_t
128malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
129	uintmax_t ret, digit;
130	unsigned b;
131	bool neg;
132	const char *p, *ns;
133
134	p = nptr;
135	if (base < 0 || base == 1 || base > 36) {
136		ns = p;
137		set_errno(EINVAL);
138		ret = UINTMAX_MAX;
139		goto label_return;
140	}
141	b = base;
142
143	/* Swallow leading whitespace and get sign, if any. */
144	neg = false;
145	while (true) {
146		switch (*p) {
147		case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
148			p++;
149			break;
150		case '-':
151			neg = true;
152			/* Fall through. */
153		case '+':
154			p++;
155			/* Fall through. */
156		default:
157			goto label_prefix;
158		}
159	}
160
161	/* Get prefix, if any. */
162	label_prefix:
163	/*
164	 * Note where the first non-whitespace/sign character is so that it is
165	 * possible to tell whether any digits are consumed (e.g., "  0" vs.
166	 * "  -x").
167	 */
168	ns = p;
169	if (*p == '0') {
170		switch (p[1]) {
171		case '0': case '1': case '2': case '3': case '4': case '5':
172		case '6': case '7':
173			if (b == 0) {
174				b = 8;
175			}
176			if (b == 8) {
177				p++;
178			}
179			break;
180		case 'X': case 'x':
181			switch (p[2]) {
182			case '0': case '1': case '2': case '3': case '4':
183			case '5': case '6': case '7': case '8': case '9':
184			case 'A': case 'B': case 'C': case 'D': case 'E':
185			case 'F':
186			case 'a': case 'b': case 'c': case 'd': case 'e':
187			case 'f':
188				if (b == 0) {
189					b = 16;
190				}
191				if (b == 16) {
192					p += 2;
193				}
194				break;
195			default:
196				break;
197			}
198			break;
199		default:
200			p++;
201			ret = 0;
202			goto label_return;
203		}
204	}
205	if (b == 0) {
206		b = 10;
207	}
208
209	/* Convert. */
210	ret = 0;
211	while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
212	    || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
213	    || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
214		uintmax_t pret = ret;
215		ret *= b;
216		ret += digit;
217		if (ret < pret) {
218			/* Overflow. */
219			set_errno(ERANGE);
220			ret = UINTMAX_MAX;
221			goto label_return;
222		}
223		p++;
224	}
225	if (neg) {
226		ret = (uintmax_t)(-((intmax_t)ret));
227	}
228
229	if (p == ns) {
230		/* No conversion performed. */
231		set_errno(EINVAL);
232		ret = UINTMAX_MAX;
233		goto label_return;
234	}
235
236label_return:
237	if (endptr != NULL) {
238		if (p == ns) {
239			/* No characters were converted. */
240			*endptr = (char *)nptr;
241		} else {
242			*endptr = (char *)p;
243		}
244	}
245	return ret;
246}
247
248static char *
249u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
250	unsigned i;
251
252	i = U2S_BUFSIZE - 1;
253	s[i] = '\0';
254	switch (base) {
255	case 10:
256		do {
257			i--;
258			s[i] = "0123456789"[x % (uint64_t)10];
259			x /= (uint64_t)10;
260		} while (x > 0);
261		break;
262	case 16: {
263		const char *digits = (uppercase)
264		    ? "0123456789ABCDEF"
265		    : "0123456789abcdef";
266
267		do {
268			i--;
269			s[i] = digits[x & 0xf];
270			x >>= 4;
271		} while (x > 0);
272		break;
273	} default: {
274		const char *digits = (uppercase)
275		    ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
276		    : "0123456789abcdefghijklmnopqrstuvwxyz";
277
278		assert(base >= 2 && base <= 36);
279		do {
280			i--;
281			s[i] = digits[x % (uint64_t)base];
282			x /= (uint64_t)base;
283		} while (x > 0);
284	}}
285
286	*slen_p = U2S_BUFSIZE - 1 - i;
287	return &s[i];
288}
289
290static char *
291d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
292	bool neg;
293
294	if ((neg = (x < 0))) {
295		x = -x;
296	}
297	s = u2s(x, 10, false, s, slen_p);
298	if (neg) {
299		sign = '-';
300	}
301	switch (sign) {
302	case '-':
303		if (!neg) {
304			break;
305		}
306		/* Fall through. */
307	case ' ':
308	case '+':
309		s--;
310		(*slen_p)++;
311		*s = sign;
312		break;
313	default: not_reached();
314	}
315	return s;
316}
317
318static char *
319o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
320	s = u2s(x, 8, false, s, slen_p);
321	if (alt_form && *s != '0') {
322		s--;
323		(*slen_p)++;
324		*s = '0';
325	}
326	return s;
327}
328
329static char *
330x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
331	s = u2s(x, 16, uppercase, s, slen_p);
332	if (alt_form) {
333		s -= 2;
334		(*slen_p) += 2;
335		memcpy(s, uppercase ? "0X" : "0x", 2);
336	}
337	return s;
338}
339
340size_t
341malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
342	size_t i;
343	const char *f;
344
345#define APPEND_C(c) do {						\
346	if (i < size) {							\
347		str[i] = (c);						\
348	}								\
349	i++;								\
350} while (0)
351#define APPEND_S(s, slen) do {						\
352	if (i < size) {							\
353		size_t cpylen = (slen <= size - i) ? slen : size - i;	\
354		memcpy(&str[i], s, cpylen);				\
355	}								\
356	i += slen;							\
357} while (0)
358#define APPEND_PADDED_S(s, slen, width, left_justify) do {		\
359	/* Left padding. */						\
360	size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ?	\
361	    (size_t)width - slen : 0);					\
362	if (!left_justify && pad_len != 0) {				\
363		size_t j;						\
364		for (j = 0; j < pad_len; j++) {				\
365			APPEND_C(' ');					\
366		}							\
367	}								\
368	/* Value. */							\
369	APPEND_S(s, slen);						\
370	/* Right padding. */						\
371	if (left_justify && pad_len != 0) {				\
372		size_t j;						\
373		for (j = 0; j < pad_len; j++) {				\
374			APPEND_C(' ');					\
375		}							\
376	}								\
377} while (0)
378#define GET_ARG_NUMERIC(val, len) do {					\
379	switch ((unsigned char)len) {					\
380	case '?':							\
381		val = va_arg(ap, int);					\
382		break;							\
383	case '?' | 0x80:						\
384		val = va_arg(ap, unsigned int);				\
385		break;							\
386	case 'l':							\
387		val = va_arg(ap, long);					\
388		break;							\
389	case 'l' | 0x80:						\
390		val = va_arg(ap, unsigned long);			\
391		break;							\
392	case 'q':							\
393		val = va_arg(ap, long long);				\
394		break;							\
395	case 'q' | 0x80:						\
396		val = va_arg(ap, unsigned long long);			\
397		break;							\
398	case 'j':							\
399		val = va_arg(ap, intmax_t);				\
400		break;							\
401	case 'j' | 0x80:						\
402		val = va_arg(ap, uintmax_t);				\
403		break;							\
404	case 't':							\
405		val = va_arg(ap, ptrdiff_t);				\
406		break;							\
407	case 'z':							\
408		val = va_arg(ap, ssize_t);				\
409		break;							\
410	case 'z' | 0x80:						\
411		val = va_arg(ap, size_t);				\
412		break;							\
413	case 'p': /* Synthetic; used for %p. */				\
414		val = va_arg(ap, uintptr_t);				\
415		break;							\
416	default:							\
417		not_reached();						\
418		val = 0;						\
419	}								\
420} while (0)
421
422	i = 0;
423	f = format;
424	while (true) {
425		switch (*f) {
426		case '\0': goto label_out;
427		case '%': {
428			bool alt_form = false;
429			bool left_justify = false;
430			bool plus_space = false;
431			bool plus_plus = false;
432			int prec = -1;
433			int width = -1;
434			unsigned char len = '?';
435			char *s;
436			size_t slen;
437
438			f++;
439			/* Flags. */
440			while (true) {
441				switch (*f) {
442				case '#':
443					assert(!alt_form);
444					alt_form = true;
445					break;
446				case '-':
447					assert(!left_justify);
448					left_justify = true;
449					break;
450				case ' ':
451					assert(!plus_space);
452					plus_space = true;
453					break;
454				case '+':
455					assert(!plus_plus);
456					plus_plus = true;
457					break;
458				default: goto label_width;
459				}
460				f++;
461			}
462			/* Width. */
463			label_width:
464			switch (*f) {
465			case '*':
466				width = va_arg(ap, int);
467				f++;
468				if (width < 0) {
469					left_justify = true;
470					width = -width;
471				}
472				break;
473			case '0': case '1': case '2': case '3': case '4':
474			case '5': case '6': case '7': case '8': case '9': {
475				uintmax_t uwidth;
476				set_errno(0);
477				uwidth = malloc_strtoumax(f, (char **)&f, 10);
478				assert(uwidth != UINTMAX_MAX || get_errno() !=
479				    ERANGE);
480				width = (int)uwidth;
481				break;
482			} default:
483				break;
484			}
485			/* Width/precision separator. */
486			if (*f == '.') {
487				f++;
488			} else {
489				goto label_length;
490			}
491			/* Precision. */
492			switch (*f) {
493			case '*':
494				prec = va_arg(ap, int);
495				f++;
496				break;
497			case '0': case '1': case '2': case '3': case '4':
498			case '5': case '6': case '7': case '8': case '9': {
499				uintmax_t uprec;
500				set_errno(0);
501				uprec = malloc_strtoumax(f, (char **)&f, 10);
502				assert(uprec != UINTMAX_MAX || get_errno() !=
503				    ERANGE);
504				prec = (int)uprec;
505				break;
506			}
507			default: break;
508			}
509			/* Length. */
510			label_length:
511			switch (*f) {
512			case 'l':
513				f++;
514				if (*f == 'l') {
515					len = 'q';
516					f++;
517				} else {
518					len = 'l';
519				}
520				break;
521			case 'q': case 'j': case 't': case 'z':
522				len = *f;
523				f++;
524				break;
525			default: break;
526			}
527			/* Conversion specifier. */
528			switch (*f) {
529			case '%':
530				/* %% */
531				APPEND_C(*f);
532				f++;
533				break;
534			case 'd': case 'i': {
535				intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
536				char buf[D2S_BUFSIZE];
537
538				GET_ARG_NUMERIC(val, len);
539				s = d2s(val, (plus_plus ? '+' : (plus_space ?
540				    ' ' : '-')), buf, &slen);
541				APPEND_PADDED_S(s, slen, width, left_justify);
542				f++;
543				break;
544			} case 'o': {
545				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
546				char buf[O2S_BUFSIZE];
547
548				GET_ARG_NUMERIC(val, len | 0x80);
549				s = o2s(val, alt_form, buf, &slen);
550				APPEND_PADDED_S(s, slen, width, left_justify);
551				f++;
552				break;
553			} case 'u': {
554				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
555				char buf[U2S_BUFSIZE];
556
557				GET_ARG_NUMERIC(val, len | 0x80);
558				s = u2s(val, 10, false, buf, &slen);
559				APPEND_PADDED_S(s, slen, width, left_justify);
560				f++;
561				break;
562			} case 'x': case 'X': {
563				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
564				char buf[X2S_BUFSIZE];
565
566				GET_ARG_NUMERIC(val, len | 0x80);
567				s = x2s(val, alt_form, *f == 'X', buf, &slen);
568				APPEND_PADDED_S(s, slen, width, left_justify);
569				f++;
570				break;
571			} case 'c': {
572				unsigned char val;
573				char buf[2];
574
575				assert(len == '?' || len == 'l');
576				assert_not_implemented(len != 'l');
577				val = va_arg(ap, int);
578				buf[0] = val;
579				buf[1] = '\0';
580				APPEND_PADDED_S(buf, 1, width, left_justify);
581				f++;
582				break;
583			} case 's':
584				assert(len == '?' || len == 'l');
585				assert_not_implemented(len != 'l');
586				s = va_arg(ap, char *);
587				slen = (prec < 0) ? strlen(s) : (size_t)prec;
588				APPEND_PADDED_S(s, slen, width, left_justify);
589				f++;
590				break;
591			case 'p': {
592				uintmax_t val;
593				char buf[X2S_BUFSIZE];
594
595				GET_ARG_NUMERIC(val, 'p');
596				s = x2s(val, true, false, buf, &slen);
597				APPEND_PADDED_S(s, slen, width, left_justify);
598				f++;
599				break;
600			} default: not_reached();
601			}
602			break;
603		} default: {
604			APPEND_C(*f);
605			f++;
606			break;
607		}}
608	}
609	label_out:
610	if (i < size) {
611		str[i] = '\0';
612	} else {
613		str[size - 1] = '\0';
614	}
615
616#undef APPEND_C
617#undef APPEND_S
618#undef APPEND_PADDED_S
619#undef GET_ARG_NUMERIC
620	return i;
621}
622
623JEMALLOC_FORMAT_PRINTF(3, 4)
624size_t
625malloc_snprintf(char *str, size_t size, const char *format, ...) {
626	size_t ret;
627	va_list ap;
628
629	va_start(ap, format);
630	ret = malloc_vsnprintf(str, size, format, ap);
631	va_end(ap);
632
633	return ret;
634}
635
636void
637malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
638    const char *format, va_list ap) {
639	char buf[MALLOC_PRINTF_BUFSIZE];
640
641	if (write_cb == NULL) {
642		/*
643		 * The caller did not provide an alternate write_cb callback
644		 * function, so use the default one.  malloc_write() is an
645		 * inline function, so use malloc_message() directly here.
646		 */
647		write_cb = (je_malloc_message != NULL) ? je_malloc_message :
648		    wrtmessage;
649	}
650
651	malloc_vsnprintf(buf, sizeof(buf), format, ap);
652	write_cb(cbopaque, buf);
653}
654
655/*
656 * Print to a callback function in such a way as to (hopefully) avoid memory
657 * allocation.
658 */
659JEMALLOC_FORMAT_PRINTF(3, 4)
660void
661malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
662    const char *format, ...) {
663	va_list ap;
664
665	va_start(ap, format);
666	malloc_vcprintf(write_cb, cbopaque, format, ap);
667	va_end(ap);
668}
669
670/* Print to stderr in such a way as to avoid memory allocation. */
671JEMALLOC_FORMAT_PRINTF(1, 2)
672void
673malloc_printf(const char *format, ...) {
674	va_list ap;
675
676	va_start(ap, format);
677	malloc_vcprintf(NULL, NULL, format, ap);
678	va_end(ap);
679}
680
681/*
682 * Restore normal assertion macros, in order to make it possible to compile all
683 * C files as a single concatenation.
684 */
685#undef assert
686#undef not_reached
687#undef not_implemented
688#undef assert_not_implemented
689#include "jemalloc/internal/assert.h"
690