1#define	assert(e) do {							\
2	if (config_debug && !(e)) {					\
3		malloc_write("<jemalloc>: Failed assertion\n");		\
4		abort();						\
5	}								\
6} while (0)
7
8#define	not_reached() do {						\
9	if (config_debug) {						\
10		malloc_write("<jemalloc>: Unreachable code reached\n");	\
11		abort();						\
12	}								\
13} while (0)
14
15#define	not_implemented() do {						\
16	if (config_debug) {						\
17		malloc_write("<jemalloc>: Not implemented\n");		\
18		abort();						\
19	}								\
20} while (0)
21
22#define	JEMALLOC_UTIL_C_
23#include "jemalloc/internal/jemalloc_internal.h"
24
25/******************************************************************************/
26/* Function prototypes for non-inline static functions. */
27
28static void	wrtmessage(void *cbopaque, const char *s);
29#define	U2S_BUFSIZE	((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
30static char	*u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
31    size_t *slen_p);
32#define	D2S_BUFSIZE	(1 + U2S_BUFSIZE)
33static char	*d2s(intmax_t x, char sign, char *s, size_t *slen_p);
34#define	O2S_BUFSIZE	(1 + U2S_BUFSIZE)
35static char	*o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
36#define	X2S_BUFSIZE	(2 + U2S_BUFSIZE)
37static char	*x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
38    size_t *slen_p);
39
40/******************************************************************************/
41
42/* malloc_message() setup. */
43static void
44wrtmessage(void *cbopaque, const char *s)
45{
46
47#ifdef SYS_write
48	/*
49	 * Use syscall(2) rather than write(2) when possible in order to avoid
50	 * the possibility of memory allocation within libc.  This is necessary
51	 * on FreeBSD; most operating systems do not have this problem though.
52	 */
53	UNUSED int result = syscall(SYS_write, STDERR_FILENO, s, strlen(s));
54#else
55	UNUSED int result = write(STDERR_FILENO, s, strlen(s));
56#endif
57}
58
59JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);
60
61JEMALLOC_ATTR(visibility("hidden"))
62void
63wrtmessage_1_0(const char *s1, const char *s2, const char *s3,
64    const char *s4)
65{
66
67	wrtmessage(NULL, s1);
68	wrtmessage(NULL, s2);
69	wrtmessage(NULL, s3);
70	wrtmessage(NULL, s4);
71}
72
73void	(*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
74    const char *s4) = wrtmessage_1_0;
75__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
76
77/*
78 * Wrapper around malloc_message() that avoids the need for
79 * je_malloc_message(...) throughout the code.
80 */
81void
82malloc_write(const char *s)
83{
84
85	if (je_malloc_message != NULL)
86		je_malloc_message(NULL, s);
87	else
88		wrtmessage(NULL, s);
89}
90
91/*
92 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
93 * provide a wrapper.
94 */
95int
96buferror(char *buf, size_t buflen)
97{
98
99#ifdef _WIN32
100	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0,
101	    (LPSTR)buf, buflen, NULL);
102	return (0);
103#elif defined(_GNU_SOURCE)
104	char *b = strerror_r(errno, buf, buflen);
105	if (b != buf) {
106		strncpy(buf, b, buflen);
107		buf[buflen-1] = '\0';
108	}
109	return (0);
110#else
111	return (strerror_r(errno, buf, buflen));
112#endif
113}
114
115uintmax_t
116malloc_strtoumax(const char *nptr, char **endptr, int base)
117{
118	uintmax_t ret, digit;
119	int b;
120	bool neg;
121	const char *p, *ns;
122
123	if (base < 0 || base == 1 || base > 36) {
124		set_errno(EINVAL);
125		return (UINTMAX_MAX);
126	}
127	b = base;
128
129	/* Swallow leading whitespace and get sign, if any. */
130	neg = false;
131	p = nptr;
132	while (true) {
133		switch (*p) {
134		case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
135			p++;
136			break;
137		case '-':
138			neg = true;
139			/* Fall through. */
140		case '+':
141			p++;
142			/* Fall through. */
143		default:
144			goto label_prefix;
145		}
146	}
147
148	/* Get prefix, if any. */
149	label_prefix:
150	/*
151	 * Note where the first non-whitespace/sign character is so that it is
152	 * possible to tell whether any digits are consumed (e.g., "  0" vs.
153	 * "  -x").
154	 */
155	ns = p;
156	if (*p == '0') {
157		switch (p[1]) {
158		case '0': case '1': case '2': case '3': case '4': case '5':
159		case '6': case '7':
160			if (b == 0)
161				b = 8;
162			if (b == 8)
163				p++;
164			break;
165		case 'x':
166			switch (p[2]) {
167			case '0': case '1': case '2': case '3': case '4':
168			case '5': case '6': case '7': case '8': case '9':
169			case 'A': case 'B': case 'C': case 'D': case 'E':
170			case 'F':
171			case 'a': case 'b': case 'c': case 'd': case 'e':
172			case 'f':
173				if (b == 0)
174					b = 16;
175				if (b == 16)
176					p += 2;
177				break;
178			default:
179				break;
180			}
181			break;
182		default:
183			break;
184		}
185	}
186	if (b == 0)
187		b = 10;
188
189	/* Convert. */
190	ret = 0;
191	while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
192	    || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
193	    || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
194		uintmax_t pret = ret;
195		ret *= b;
196		ret += digit;
197		if (ret < pret) {
198			/* Overflow. */
199			set_errno(ERANGE);
200			return (UINTMAX_MAX);
201		}
202		p++;
203	}
204	if (neg)
205		ret = -ret;
206
207	if (endptr != NULL) {
208		if (p == ns) {
209			/* No characters were converted. */
210			*endptr = (char *)nptr;
211		} else
212			*endptr = (char *)p;
213	}
214
215	return (ret);
216}
217
218static char *
219u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p)
220{
221	unsigned i;
222
223	i = U2S_BUFSIZE - 1;
224	s[i] = '\0';
225	switch (base) {
226	case 10:
227		do {
228			i--;
229			s[i] = "0123456789"[x % (uint64_t)10];
230			x /= (uint64_t)10;
231		} while (x > 0);
232		break;
233	case 16: {
234		const char *digits = (uppercase)
235		    ? "0123456789ABCDEF"
236		    : "0123456789abcdef";
237
238		do {
239			i--;
240			s[i] = digits[x & 0xf];
241			x >>= 4;
242		} while (x > 0);
243		break;
244	} default: {
245		const char *digits = (uppercase)
246		    ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
247		    : "0123456789abcdefghijklmnopqrstuvwxyz";
248
249		assert(base >= 2 && base <= 36);
250		do {
251			i--;
252			s[i] = digits[x % (uint64_t)base];
253			x /= (uint64_t)base;
254		} while (x > 0);
255	}}
256
257	*slen_p = U2S_BUFSIZE - 1 - i;
258	return (&s[i]);
259}
260
261static char *
262d2s(intmax_t x, char sign, char *s, size_t *slen_p)
263{
264	bool neg;
265
266	if ((neg = (x < 0)))
267		x = -x;
268	s = u2s(x, 10, false, s, slen_p);
269	if (neg)
270		sign = '-';
271	switch (sign) {
272	case '-':
273		if (neg == false)
274			break;
275		/* Fall through. */
276	case ' ':
277	case '+':
278		s--;
279		(*slen_p)++;
280		*s = sign;
281		break;
282	default: not_reached();
283	}
284	return (s);
285}
286
287static char *
288o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p)
289{
290
291	s = u2s(x, 8, false, s, slen_p);
292	if (alt_form && *s != '0') {
293		s--;
294		(*slen_p)++;
295		*s = '0';
296	}
297	return (s);
298}
299
300static char *
301x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p)
302{
303
304	s = u2s(x, 16, uppercase, s, slen_p);
305	if (alt_form) {
306		s -= 2;
307		(*slen_p) += 2;
308		memcpy(s, uppercase ? "0X" : "0x", 2);
309	}
310	return (s);
311}
312
313int
314malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
315{
316	int ret;
317	size_t i;
318	const char *f;
319
320#define	APPEND_C(c) do {						\
321	if (i < size)							\
322		str[i] = (c);						\
323	i++;								\
324} while (0)
325#define	APPEND_S(s, slen) do {						\
326	if (i < size) {							\
327		size_t cpylen = (slen <= size - i) ? slen : size - i;	\
328		memcpy(&str[i], s, cpylen);				\
329	}								\
330	i += slen;							\
331} while (0)
332#define	APPEND_PADDED_S(s, slen, width, left_justify) do {		\
333	/* Left padding. */						\
334	size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ?	\
335	    (size_t)width - slen : 0);					\
336	if (left_justify == false && pad_len != 0) {			\
337		size_t j;						\
338		for (j = 0; j < pad_len; j++)				\
339			APPEND_C(' ');					\
340	}								\
341	/* Value. */							\
342	APPEND_S(s, slen);						\
343	/* Right padding. */						\
344	if (left_justify && pad_len != 0) {				\
345		size_t j;						\
346		for (j = 0; j < pad_len; j++)				\
347			APPEND_C(' ');					\
348	}								\
349} while (0)
350#define GET_ARG_NUMERIC(val, len) do {					\
351	switch (len) {							\
352	case '?':							\
353		val = va_arg(ap, int);					\
354		break;							\
355	case '?' | 0x80:						\
356		val = va_arg(ap, unsigned int);				\
357		break;							\
358	case 'l':							\
359		val = va_arg(ap, long);					\
360		break;							\
361	case 'l' | 0x80:						\
362		val = va_arg(ap, unsigned long);			\
363		break;							\
364	case 'q':							\
365		val = va_arg(ap, long long);				\
366		break;							\
367	case 'q' | 0x80:						\
368		val = va_arg(ap, unsigned long long);			\
369		break;							\
370	case 'j':							\
371		val = va_arg(ap, intmax_t);				\
372		break;							\
373	case 't':							\
374		val = va_arg(ap, ptrdiff_t);				\
375		break;							\
376	case 'z':							\
377		val = va_arg(ap, ssize_t);				\
378		break;							\
379	case 'z' | 0x80:						\
380		val = va_arg(ap, size_t);				\
381		break;							\
382	case 'p': /* Synthetic; used for %p. */				\
383		val = va_arg(ap, uintptr_t);				\
384		break;							\
385	default: not_reached();						\
386	}								\
387} while (0)
388
389	i = 0;
390	f = format;
391	while (true) {
392		switch (*f) {
393		case '\0': goto label_out;
394		case '%': {
395			bool alt_form = false;
396			bool left_justify = false;
397			bool plus_space = false;
398			bool plus_plus = false;
399			int prec = -1;
400			int width = -1;
401			unsigned char len = '?';
402
403			f++;
404			if (*f == '%') {
405				/* %% */
406				APPEND_C(*f);
407				break;
408			}
409			/* Flags. */
410			while (true) {
411				switch (*f) {
412				case '#':
413					assert(alt_form == false);
414					alt_form = true;
415					break;
416				case '-':
417					assert(left_justify == false);
418					left_justify = true;
419					break;
420				case ' ':
421					assert(plus_space == false);
422					plus_space = true;
423					break;
424				case '+':
425					assert(plus_plus == false);
426					plus_plus = true;
427					break;
428				default: goto label_width;
429				}
430				f++;
431			}
432			/* Width. */
433			label_width:
434			switch (*f) {
435			case '*':
436				width = va_arg(ap, int);
437				f++;
438				break;
439			case '0': case '1': case '2': case '3': case '4':
440			case '5': case '6': case '7': case '8': case '9': {
441				uintmax_t uwidth;
442				set_errno(0);
443				uwidth = malloc_strtoumax(f, (char **)&f, 10);
444				assert(uwidth != UINTMAX_MAX || get_errno() !=
445				    ERANGE);
446				width = (int)uwidth;
447				if (*f == '.') {
448					f++;
449					goto label_precision;
450				} else
451					goto label_length;
452				break;
453			} case '.':
454				f++;
455				goto label_precision;
456			default: goto label_length;
457			}
458			/* Precision. */
459			label_precision:
460			switch (*f) {
461			case '*':
462				prec = va_arg(ap, int);
463				f++;
464				break;
465			case '0': case '1': case '2': case '3': case '4':
466			case '5': case '6': case '7': case '8': case '9': {
467				uintmax_t uprec;
468				set_errno(0);
469				uprec = malloc_strtoumax(f, (char **)&f, 10);
470				assert(uprec != UINTMAX_MAX || get_errno() !=
471				    ERANGE);
472				prec = (int)uprec;
473				break;
474			}
475			default: break;
476			}
477			/* Length. */
478			label_length:
479			switch (*f) {
480			case 'l':
481				f++;
482				if (*f == 'l') {
483					len = 'q';
484					f++;
485				} else
486					len = 'l';
487				break;
488			case 'j':
489				len = 'j';
490				f++;
491				break;
492			case 't':
493				len = 't';
494				f++;
495				break;
496			case 'z':
497				len = 'z';
498				f++;
499				break;
500			default: break;
501			}
502			/* Conversion specifier. */
503			switch (*f) {
504				char *s;
505				size_t slen;
506			case 'd': case 'i': {
507				intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
508				char buf[D2S_BUFSIZE];
509
510				GET_ARG_NUMERIC(val, len);
511				s = d2s(val, (plus_plus ? '+' : (plus_space ?
512				    ' ' : '-')), buf, &slen);
513				APPEND_PADDED_S(s, slen, width, left_justify);
514				f++;
515				break;
516			} case 'o': {
517				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
518				char buf[O2S_BUFSIZE];
519
520				GET_ARG_NUMERIC(val, len | 0x80);
521				s = o2s(val, alt_form, buf, &slen);
522				APPEND_PADDED_S(s, slen, width, left_justify);
523				f++;
524				break;
525			} case 'u': {
526				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
527				char buf[U2S_BUFSIZE];
528
529				GET_ARG_NUMERIC(val, len | 0x80);
530				s = u2s(val, 10, false, buf, &slen);
531				APPEND_PADDED_S(s, slen, width, left_justify);
532				f++;
533				break;
534			} case 'x': case 'X': {
535				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
536				char buf[X2S_BUFSIZE];
537
538				GET_ARG_NUMERIC(val, len | 0x80);
539				s = x2s(val, alt_form, *f == 'X', buf, &slen);
540				APPEND_PADDED_S(s, slen, width, left_justify);
541				f++;
542				break;
543			} case 'c': {
544				unsigned char val;
545				char buf[2];
546
547				assert(len == '?' || len == 'l');
548				assert_not_implemented(len != 'l');
549				val = va_arg(ap, int);
550				buf[0] = val;
551				buf[1] = '\0';
552				APPEND_PADDED_S(buf, 1, width, left_justify);
553				f++;
554				break;
555			} case 's':
556				assert(len == '?' || len == 'l');
557				assert_not_implemented(len != 'l');
558				s = va_arg(ap, char *);
559				slen = (prec == -1) ? strlen(s) : prec;
560				APPEND_PADDED_S(s, slen, width, left_justify);
561				f++;
562				break;
563			case 'p': {
564				uintmax_t val;
565				char buf[X2S_BUFSIZE];
566
567				GET_ARG_NUMERIC(val, 'p');
568				s = x2s(val, true, false, buf, &slen);
569				APPEND_PADDED_S(s, slen, width, left_justify);
570				f++;
571				break;
572			}
573			default: not_implemented();
574			}
575			break;
576		} default: {
577			APPEND_C(*f);
578			f++;
579			break;
580		}}
581	}
582	label_out:
583	if (i < size)
584		str[i] = '\0';
585	else
586		str[size - 1] = '\0';
587	ret = i;
588
589#undef APPEND_C
590#undef APPEND_S
591#undef APPEND_PADDED_S
592#undef GET_ARG_NUMERIC
593	return (ret);
594}
595
596JEMALLOC_ATTR(format(printf, 3, 4))
597int
598malloc_snprintf(char *str, size_t size, const char *format, ...)
599{
600	int ret;
601	va_list ap;
602
603	va_start(ap, format);
604	ret = malloc_vsnprintf(str, size, format, ap);
605	va_end(ap);
606
607	return (ret);
608}
609
610void
611malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
612    const char *format, va_list ap)
613{
614	char buf[MALLOC_PRINTF_BUFSIZE];
615
616	if (write_cb == NULL) {
617		/*
618		 * The caller did not provide an alternate write_cb callback
619		 * function, so use the default one.  malloc_write() is an
620		 * inline function, so use malloc_message() directly here.
621		 */
622		write_cb = (je_malloc_message != NULL) ? je_malloc_message :
623		    wrtmessage;
624		cbopaque = NULL;
625	}
626
627	malloc_vsnprintf(buf, sizeof(buf), format, ap);
628	write_cb(cbopaque, buf);
629}
630
631/*
632 * Print to a callback function in such a way as to (hopefully) avoid memory
633 * allocation.
634 */
635JEMALLOC_ATTR(format(printf, 3, 4))
636void
637malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
638    const char *format, ...)
639{
640	va_list ap;
641
642	va_start(ap, format);
643	malloc_vcprintf(write_cb, cbopaque, format, ap);
644	va_end(ap);
645}
646
647/* Print to stderr in such a way as to avoid memory allocation. */
648JEMALLOC_ATTR(format(printf, 1, 2))
649void
650malloc_printf(const char *format, ...)
651{
652	va_list ap;
653
654	va_start(ap, format);
655	malloc_vcprintf(NULL, NULL, format, ap);
656	va_end(ap);
657}
658