1/*	$NetBSD: tinytest.c,v 1.6 2020/05/25 20:47:34 christos Exp $	*/
2
3/* tinytest.c -- Copyright 2009-2012 Nick Mathewson
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 *    derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27#ifdef TINYTEST_LOCAL
28#include "tinytest_local.h"
29#endif
30
31#include <stdio.h>
32#include <stdlib.h>
33#include <string.h>
34#include <assert.h>
35
36#ifndef NO_FORKING
37
38#ifdef _WIN32
39#include <windows.h>
40#else
41#include <sys/types.h>
42#include <sys/wait.h>
43#include <unistd.h>
44#endif
45
46#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
47#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
48    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
49/* Workaround for a stupid bug in OSX 10.6 */
50#define FORK_BREAKS_GCOV
51#include <vproc.h>
52#endif
53#endif
54
55#endif /* !NO_FORKING */
56
57#ifndef __GNUC__
58#define __attribute__(x)
59#endif
60
61#include "tinytest.h"
62#include "tinytest_macros.h"
63
64#define LONGEST_TEST_NAME 16384
65
66static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
67static int n_ok = 0; /**< Number of tests that have passed */
68static int n_bad = 0; /**< Number of tests that have failed. */
69static int n_skipped = 0; /**< Number of tests that have been skipped. */
70
71static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
72static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
73static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
74const char *verbosity_flag = "";
75
76const struct testlist_alias_t *cfg_aliases=NULL;
77
78enum outcome { SKIP=2, OK=1, FAIL=0 };
79static enum outcome cur_test_outcome = 0;
80const char *cur_test_prefix = NULL; /**< prefix of the current test group */
81/** Name of the current test, if we haven't logged is yet. Used for --quiet */
82const char *cur_test_name = NULL;
83
84#ifdef _WIN32
85/* Copy of argv[0] for win32. */
86static char commandname[MAX_PATH+1];
87#endif
88
89static void usage(struct testgroup_t *groups, int list_groups)
90  __attribute__((noreturn));
91static int process_test_option(struct testgroup_t *groups, const char *test);
92
93static enum outcome
94testcase_run_bare_(const struct testcase_t *testcase)
95{
96	void *env = NULL;
97	int outcome;
98	if (testcase->setup) {
99		env = testcase->setup->setup_fn(testcase);
100		if (!env)
101			return FAIL;
102		else if (env == (void*)TT_SKIP)
103			return SKIP;
104	}
105
106	cur_test_outcome = OK;
107	testcase->fn(env);
108	outcome = cur_test_outcome;
109
110	if (testcase->setup) {
111		if (testcase->setup->cleanup_fn(testcase, env) == 0)
112			outcome = FAIL;
113	}
114
115	return outcome;
116}
117
118#define MAGIC_EXITCODE 42
119
120#ifndef NO_FORKING
121
122static enum outcome
123testcase_run_forked_(const struct testgroup_t *group,
124		     const struct testcase_t *testcase)
125{
126#ifdef _WIN32
127	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
128	   we'll invoke our own exe (whose name we recall from the command
129	   line) with a command line that tells it to run just the test we
130	   want, and this time without forking.
131
132	   (No, threads aren't an option.  The whole point of forking is to
133	   share no state between tests.)
134	 */
135	int ok;
136	char buffer[LONGEST_TEST_NAME+256];
137	STARTUPINFOA si;
138	PROCESS_INFORMATION info;
139	DWORD exitcode;
140
141	if (!in_tinytest_main) {
142		printf("\nERROR.  On Windows, testcase_run_forked_ must be"
143		       " called from within tinytest_main.\n");
144		abort();
145	}
146	if (opt_verbosity>0)
147		printf("[forking] ");
148
149	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
150		 commandname, verbosity_flag, group->prefix, testcase->name);
151
152	memset(&si, 0, sizeof(si));
153	memset(&info, 0, sizeof(info));
154	si.cb = sizeof(si);
155
156	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
157			   0, NULL, NULL, &si, &info);
158	if (!ok) {
159		printf("CreateProcess failed!\n");
160		return 0;
161	}
162	WaitForSingleObject(info.hProcess, INFINITE);
163	GetExitCodeProcess(info.hProcess, &exitcode);
164	CloseHandle(info.hProcess);
165	CloseHandle(info.hThread);
166	if (exitcode == 0)
167		return OK;
168	else if (exitcode == MAGIC_EXITCODE)
169		return SKIP;
170	else
171		return FAIL;
172#else
173	int outcome_pipe[2];
174	pid_t pid;
175	(void)group;
176
177	if (pipe(outcome_pipe))
178		perror("opening pipe");
179
180	if (opt_verbosity>0)
181		printf("[forking] ");
182	pid = fork();
183#ifdef FORK_BREAKS_GCOV
184	vproc_transaction_begin(0);
185#endif
186	if (!pid) {
187		/* child. */
188		int test_r, write_r;
189		char b[1];
190		close(outcome_pipe[0]);
191		test_r = testcase_run_bare_(testcase);
192		assert(0<=(int)test_r && (int)test_r<=2);
193		b[0] = "NYS"[test_r];
194		write_r = (int)write(outcome_pipe[1], b, 1);
195		if (write_r != 1) {
196			perror("write outcome to pipe");
197			exit(1);
198		}
199		exit(0);
200		return FAIL; /* unreachable */
201	} else {
202		/* parent */
203		int status, r;
204		char b[1];
205		/* Close this now, so that if the other side closes it,
206		 * our read fails. */
207		close(outcome_pipe[1]);
208		r = (int)read(outcome_pipe[0], b, 1);
209		if (r == 0) {
210			printf("[Lost connection!] ");
211			return 0;
212		} else if (r != 1) {
213			perror("read outcome from pipe");
214		}
215		waitpid(pid, &status, 0);
216		close(outcome_pipe[0]);
217		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
218	}
219#endif
220}
221
222#endif /* !NO_FORKING */
223
224int
225testcase_run_one(const struct testgroup_t *group,
226		 const struct testcase_t *testcase)
227{
228	enum outcome outcome;
229
230	if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
231		if (opt_verbosity>0)
232			printf("%s%s: %s\n",
233			   group->prefix, testcase->name,
234			   (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
235		++n_skipped;
236		return SKIP;
237	}
238
239	if (opt_verbosity>0 && !opt_forked) {
240		printf("%s%s: ", group->prefix, testcase->name);
241	} else {
242		if (opt_verbosity==0) printf(".");
243		cur_test_prefix = group->prefix;
244		cur_test_name = testcase->name;
245	}
246
247#ifndef NO_FORKING
248	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
249		outcome = testcase_run_forked_(group, testcase);
250	} else {
251#else
252	{
253#endif
254		outcome = testcase_run_bare_(testcase);
255	}
256
257	if (outcome == OK) {
258		++n_ok;
259		if (opt_verbosity>0 && !opt_forked)
260			puts(opt_verbosity==1?"OK":"");
261	} else if (outcome == SKIP) {
262		++n_skipped;
263		if (opt_verbosity>0 && !opt_forked)
264			puts("SKIPPED");
265	} else {
266		++n_bad;
267		if (!opt_forked)
268			printf("\n  [%s FAILED]\n", testcase->name);
269	}
270
271	if (opt_forked) {
272		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
273		return 1; /* unreachable */
274	} else {
275		return (int)outcome;
276	}
277}
278
279int
280tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
281{
282	int i, j;
283	size_t length = LONGEST_TEST_NAME;
284	char fullname[LONGEST_TEST_NAME];
285	int found=0;
286	if (strstr(arg, ".."))
287		length = strstr(arg,"..")-arg;
288	for (i=0; groups[i].prefix; ++i) {
289		for (j=0; groups[i].cases[j].name; ++j) {
290			struct testcase_t *testcase = &groups[i].cases[j];
291			snprintf(fullname, sizeof(fullname), "%s%s",
292				 groups[i].prefix, testcase->name);
293			if (!flag) { /* Hack! */
294				printf("    %s", fullname);
295				if (testcase->flags & TT_OFF_BY_DEFAULT)
296					puts("   (Off by default)");
297				else if (testcase->flags & TT_SKIP)
298					puts("  (DISABLED)");
299				else
300					puts("");
301			}
302			if (!strncmp(fullname, arg, length)) {
303				if (set)
304					testcase->flags |= flag;
305				else
306					testcase->flags &= ~flag;
307				++found;
308			}
309		}
310	}
311	return found;
312}
313
314static void
315usage(struct testgroup_t *groups, int list_groups)
316{
317	puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
318	puts("  Specify tests by name, or using a prefix ending with '..'");
319	puts("  To skip a test, prefix its name with a colon.");
320	puts("  To enable a disabled test, prefix its name with a plus.");
321	puts("  Use --list-tests for a list of tests.");
322	if (list_groups) {
323		puts("Known tests are:");
324		tinytest_set_flag_(groups, "..", 1, 0);
325	}
326	exit(0);
327}
328
329static int
330process_test_alias(struct testgroup_t *groups, const char *test)
331{
332	int i, j, n, r;
333	for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
334		if (!strcmp(cfg_aliases[i].name, test)) {
335			n = 0;
336			for (j = 0; cfg_aliases[i].tests[j]; ++j) {
337				r = process_test_option(groups, cfg_aliases[i].tests[j]);
338				if (r<0)
339					return -1;
340				n += r;
341			}
342			return n;
343		}
344	}
345	printf("No such test alias as @%s!",test);
346	return -1;
347}
348
349static int
350process_test_option(struct testgroup_t *groups, const char *test)
351{
352	int flag = TT_ENABLED_;
353	int n = 0;
354	if (test[0] == '@') {
355		return process_test_alias(groups, test + 1);
356	} else if (test[0] == ':') {
357		++test;
358		flag = TT_SKIP;
359	} else if (test[0] == '+') {
360		++test;
361		++n;
362		if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
363			printf("No such test as %s!\n", test);
364			return -1;
365		}
366	} else {
367		++n;
368	}
369	if (!tinytest_set_flag_(groups, test, 1, flag)) {
370		printf("No such test as %s!\n", test);
371		return -1;
372	}
373	return n;
374}
375
376void
377tinytest_set_aliases(const struct testlist_alias_t *aliases)
378{
379	cfg_aliases = aliases;
380}
381
382int
383tinytest_main(int c, const char **v, struct testgroup_t *groups)
384{
385	int i, j, n=0;
386
387#ifdef _WIN32
388	const char *sp = strrchr(v[0], '.');
389	const char *extension = "";
390	if (!sp || stricmp(sp, ".exe"))
391		extension = ".exe"; /* Add an exe so CreateProcess will work */
392	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
393	commandname[MAX_PATH]='\0';
394#endif
395	for (i=1; i<c; ++i) {
396		if (v[i][0] == '-') {
397			if (!strcmp(v[i], "--RUNNING-FORKED")) {
398				opt_forked = 1;
399			} else if (!strcmp(v[i], "--no-fork")) {
400				opt_nofork = 1;
401			} else if (!strcmp(v[i], "--quiet")) {
402				opt_verbosity = -1;
403				verbosity_flag = "--quiet";
404			} else if (!strcmp(v[i], "--verbose")) {
405				opt_verbosity = 2;
406				verbosity_flag = "--verbose";
407			} else if (!strcmp(v[i], "--terse")) {
408				opt_verbosity = 0;
409				verbosity_flag = "--terse";
410			} else if (!strcmp(v[i], "--help")) {
411				usage(groups, 0);
412			} else if (!strcmp(v[i], "--list-tests")) {
413				usage(groups, 1);
414			} else {
415				printf("Unknown option %s.  Try --help\n",v[i]);
416				return -1;
417			}
418		} else {
419			int r = process_test_option(groups, v[i]);
420			if (r<0)
421				return -1;
422			n += r;
423		}
424	}
425	if (!n)
426		tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
427
428#ifdef _IONBF
429	setvbuf(stdout, NULL, _IONBF, 0);
430#endif
431
432	++in_tinytest_main;
433	for (i=0; groups[i].prefix; ++i)
434		for (j=0; groups[i].cases[j].name; ++j)
435			if (groups[i].cases[j].flags & TT_ENABLED_)
436				testcase_run_one(&groups[i],
437						 &groups[i].cases[j]);
438
439	--in_tinytest_main;
440
441	if (opt_verbosity==0)
442		puts("");
443
444	if (n_bad)
445		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
446		       n_bad+n_ok,n_skipped);
447	else if (opt_verbosity >= 1)
448		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
449
450	return (n_bad == 0) ? 0 : 1;
451}
452
453int
454tinytest_get_verbosity_(void)
455{
456	return opt_verbosity;
457}
458
459void
460tinytest_set_test_failed_(void)
461{
462	if (opt_verbosity <= 0 && cur_test_name) {
463		if (opt_verbosity==0) puts("");
464		printf("%s%s: ", cur_test_prefix, cur_test_name);
465		cur_test_name = NULL;
466	}
467	cur_test_outcome = 0;
468}
469
470void
471tinytest_set_test_skipped_(void)
472{
473	if (cur_test_outcome==OK)
474		cur_test_outcome = SKIP;
475}
476
477char *
478tinytest_format_hex_(const void *val_, unsigned long len)
479{
480	const unsigned char *val = val_;
481	char *result, *cp;
482	size_t i;
483
484	if (!val)
485		return strdup("null");
486	if (!(result = malloc(len*2+1)))
487		return strdup("<allocation failure>");
488	cp = result;
489	for (i=0;i<len;++i) {
490		*cp++ = "0123456789ABCDEF"[val[i] >> 4];
491		*cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
492	}
493	*cp = 0;
494	return result;
495}
496