tinytest.c revision 290001 
1/* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions
5 * are met:
6 * 1. Redistributions of source code must retain the above copyright
7 *    notice, this list of conditions and the following disclaimer.
8 * 2. Redistributions in binary form must reproduce the above copyright
9 *    notice, this list of conditions and the following disclaimer in the
10 *    documentation and/or other materials provided with the distribution.
11 * 3. The name of the author may not be used to endorse or promote products
12 *    derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25#ifdef TINYTEST_LOCAL
26#include "tinytest_local.h"
27#endif
28
29#include <stdio.h>
30#include <stdlib.h>
31#include <string.h>
32#include <assert.h>
33
34#ifndef NO_FORKING
35
36#ifdef _WIN32
37#include <windows.h>
38#else
39#include <sys/types.h>
40#include <sys/wait.h>
41#include <unistd.h>
42#endif
43
44#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47/* Workaround for a stupid bug in OSX 10.6 */
48#define FORK_BREAKS_GCOV
49#include <vproc.h>
50#endif
51#endif
52
53#endif /* !NO_FORKING */
54
55#ifndef __GNUC__
56#define __attribute__(x)
57#endif
58
59#include "tinytest.h"
60#include "tinytest_macros.h"
61
62#define LONGEST_TEST_NAME 16384
63
64static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
65static int n_ok = 0; /**< Number of tests that have passed */
66static int n_bad = 0; /**< Number of tests that have failed. */
67static int n_skipped = 0; /**< Number of tests that have been skipped. */
68
69static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
70static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
71static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
72const char *verbosity_flag = "";
73
74const struct testlist_alias_t *cfg_aliases=NULL;
75
76enum outcome { SKIP=2, OK=1, FAIL=0 };
77static enum outcome cur_test_outcome = 0;
78const char *cur_test_prefix = NULL; /**< prefix of the current test group */
79/** Name of the current test, if we haven't logged is yet. Used for --quiet */
80const char *cur_test_name = NULL;
81
82#ifdef _WIN32
83/* Copy of argv[0] for win32. */
84static char commandname[MAX_PATH+1];
85#endif
86
87static void usage(struct testgroup_t *groups, int list_groups)
88  __attribute__((noreturn));
89static int process_test_option(struct testgroup_t *groups, const char *test);
90
91static enum outcome
92testcase_run_bare_(const struct testcase_t *testcase)
93{
94	void *env = NULL;
95	int outcome;
96	if (testcase->setup) {
97		env = testcase->setup->setup_fn(testcase);
98		if (!env)
99			return FAIL;
100		else if (env == (void*)TT_SKIP)
101			return SKIP;
102	}
103
104	cur_test_outcome = OK;
105	testcase->fn(env);
106	outcome = cur_test_outcome;
107
108	if (testcase->setup) {
109		if (testcase->setup->cleanup_fn(testcase, env) == 0)
110			outcome = FAIL;
111	}
112
113	return outcome;
114}
115
116#define MAGIC_EXITCODE 42
117
118#ifndef NO_FORKING
119
120static enum outcome
121testcase_run_forked_(const struct testgroup_t *group,
122		     const struct testcase_t *testcase)
123{
124#ifdef _WIN32
125	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
126	   we'll invoke our own exe (whose name we recall from the command
127	   line) with a command line that tells it to run just the test we
128	   want, and this time without forking.
129
130	   (No, threads aren't an option.  The whole point of forking is to
131	   share no state between tests.)
132	 */
133	int ok;
134	char buffer[LONGEST_TEST_NAME+256];
135	STARTUPINFOA si;
136	PROCESS_INFORMATION info;
137	DWORD exitcode;
138
139	if (!in_tinytest_main) {
140		printf("\nERROR.  On Windows, testcase_run_forked_ must be"
141		       " called from within tinytest_main.\n");
142		abort();
143	}
144	if (opt_verbosity>0)
145		printf("[forking] ");
146
147	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
148		 commandname, verbosity_flag, group->prefix, testcase->name);
149
150	memset(&si, 0, sizeof(si));
151	memset(&info, 0, sizeof(info));
152	si.cb = sizeof(si);
153
154	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
155			   0, NULL, NULL, &si, &info);
156	if (!ok) {
157		printf("CreateProcess failed!\n");
158		return 0;
159	}
160	WaitForSingleObject(info.hProcess, INFINITE);
161	GetExitCodeProcess(info.hProcess, &exitcode);
162	CloseHandle(info.hProcess);
163	CloseHandle(info.hThread);
164	if (exitcode == 0)
165		return OK;
166	else if (exitcode == MAGIC_EXITCODE)
167		return SKIP;
168	else
169		return FAIL;
170#else
171	int outcome_pipe[2];
172	pid_t pid;
173	(void)group;
174
175	if (pipe(outcome_pipe))
176		perror("opening pipe");
177
178	if (opt_verbosity>0)
179		printf("[forking] ");
180	pid = fork();
181#ifdef FORK_BREAKS_GCOV
182	vproc_transaction_begin(0);
183#endif
184	if (!pid) {
185		/* child. */
186		int test_r, write_r;
187		char b[1];
188		close(outcome_pipe[0]);
189		test_r = testcase_run_bare_(testcase);
190		assert(0<=(int)test_r && (int)test_r<=2);
191		b[0] = "NYS"[test_r];
192		write_r = (int)write(outcome_pipe[1], b, 1);
193		if (write_r != 1) {
194			perror("write outcome to pipe");
195			exit(1);
196		}
197		exit(0);
198		return FAIL; /* unreachable */
199	} else {
200		/* parent */
201		int status, r;
202		char b[1];
203		/* Close this now, so that if the other side closes it,
204		 * our read fails. */
205		close(outcome_pipe[1]);
206		r = (int)read(outcome_pipe[0], b, 1);
207		if (r == 0) {
208			printf("[Lost connection!] ");
209			return 0;
210		} else if (r != 1) {
211			perror("read outcome from pipe");
212		}
213		waitpid(pid, &status, 0);
214		close(outcome_pipe[0]);
215		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
216	}
217#endif
218}
219
220#endif /* !NO_FORKING */
221
222int
223testcase_run_one(const struct testgroup_t *group,
224		 const struct testcase_t *testcase)
225{
226	enum outcome outcome;
227
228	if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
229		if (opt_verbosity>0)
230			printf("%s%s: %s\n",
231			   group->prefix, testcase->name,
232			   (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
233		++n_skipped;
234		return SKIP;
235	}
236
237	if (opt_verbosity>0 && !opt_forked) {
238		printf("%s%s: ", group->prefix, testcase->name);
239	} else {
240		if (opt_verbosity==0) printf(".");
241		cur_test_prefix = group->prefix;
242		cur_test_name = testcase->name;
243	}
244
245#ifndef NO_FORKING
246	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
247		outcome = testcase_run_forked_(group, testcase);
248	} else {
249#else
250	{
251#endif
252		outcome = testcase_run_bare_(testcase);
253	}
254
255	if (outcome == OK) {
256		++n_ok;
257		if (opt_verbosity>0 && !opt_forked)
258			puts(opt_verbosity==1?"OK":"");
259	} else if (outcome == SKIP) {
260		++n_skipped;
261		if (opt_verbosity>0 && !opt_forked)
262			puts("SKIPPED");
263	} else {
264		++n_bad;
265		if (!opt_forked)
266			printf("\n  [%s FAILED]\n", testcase->name);
267	}
268
269	if (opt_forked) {
270		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
271		return 1; /* unreachable */
272	} else {
273		return (int)outcome;
274	}
275}
276
277int
278tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
279{
280	int i, j;
281	size_t length = LONGEST_TEST_NAME;
282	char fullname[LONGEST_TEST_NAME];
283	int found=0;
284	if (strstr(arg, ".."))
285		length = strstr(arg,"..")-arg;
286	for (i=0; groups[i].prefix; ++i) {
287		for (j=0; groups[i].cases[j].name; ++j) {
288			struct testcase_t *testcase = &groups[i].cases[j];
289			snprintf(fullname, sizeof(fullname), "%s%s",
290				 groups[i].prefix, testcase->name);
291			if (!flag) { /* Hack! */
292				printf("    %s", fullname);
293				if (testcase->flags & TT_OFF_BY_DEFAULT)
294					puts("   (Off by default)");
295				else if (testcase->flags & TT_SKIP)
296					puts("  (DISABLED)");
297				else
298					puts("");
299			}
300			if (!strncmp(fullname, arg, length)) {
301				if (set)
302					testcase->flags |= flag;
303				else
304					testcase->flags &= ~flag;
305				++found;
306			}
307		}
308	}
309	return found;
310}
311
312static void
313usage(struct testgroup_t *groups, int list_groups)
314{
315	puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
316	puts("  Specify tests by name, or using a prefix ending with '..'");
317	puts("  To skip a test, prefix its name with a colon.");
318	puts("  To enable a disabled test, prefix its name with a plus.");
319	puts("  Use --list-tests for a list of tests.");
320	if (list_groups) {
321		puts("Known tests are:");
322		tinytest_set_flag_(groups, "..", 1, 0);
323	}
324	exit(0);
325}
326
327static int
328process_test_alias(struct testgroup_t *groups, const char *test)
329{
330	int i, j, n, r;
331	for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
332		if (!strcmp(cfg_aliases[i].name, test)) {
333			n = 0;
334			for (j = 0; cfg_aliases[i].tests[j]; ++j) {
335				r = process_test_option(groups, cfg_aliases[i].tests[j]);
336				if (r<0)
337					return -1;
338				n += r;
339			}
340			return n;
341		}
342	}
343	printf("No such test alias as @%s!",test);
344	return -1;
345}
346
347static int
348process_test_option(struct testgroup_t *groups, const char *test)
349{
350	int flag = TT_ENABLED_;
351	int n = 0;
352	if (test[0] == '@') {
353		return process_test_alias(groups, test + 1);
354	} else if (test[0] == ':') {
355		++test;
356		flag = TT_SKIP;
357	} else if (test[0] == '+') {
358		++test;
359		++n;
360		if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
361			printf("No such test as %s!\n", test);
362			return -1;
363		}
364	} else {
365		++n;
366	}
367	if (!tinytest_set_flag_(groups, test, 1, flag)) {
368		printf("No such test as %s!\n", test);
369		return -1;
370	}
371	return n;
372}
373
374void
375tinytest_set_aliases(const struct testlist_alias_t *aliases)
376{
377	cfg_aliases = aliases;
378}
379
380int
381tinytest_main(int c, const char **v, struct testgroup_t *groups)
382{
383	int i, j, n=0;
384
385#ifdef _WIN32
386	const char *sp = strrchr(v[0], '.');
387	const char *extension = "";
388	if (!sp || stricmp(sp, ".exe"))
389		extension = ".exe"; /* Add an exe so CreateProcess will work */
390	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
391	commandname[MAX_PATH]='\0';
392#endif
393	for (i=1; i<c; ++i) {
394		if (v[i][0] == '-') {
395			if (!strcmp(v[i], "--RUNNING-FORKED")) {
396				opt_forked = 1;
397			} else if (!strcmp(v[i], "--no-fork")) {
398				opt_nofork = 1;
399			} else if (!strcmp(v[i], "--quiet")) {
400				opt_verbosity = -1;
401				verbosity_flag = "--quiet";
402			} else if (!strcmp(v[i], "--verbose")) {
403				opt_verbosity = 2;
404				verbosity_flag = "--verbose";
405			} else if (!strcmp(v[i], "--terse")) {
406				opt_verbosity = 0;
407				verbosity_flag = "--terse";
408			} else if (!strcmp(v[i], "--help")) {
409				usage(groups, 0);
410			} else if (!strcmp(v[i], "--list-tests")) {
411				usage(groups, 1);
412			} else {
413				printf("Unknown option %s.  Try --help\n",v[i]);
414				return -1;
415			}
416		} else {
417			int r = process_test_option(groups, v[i]);
418			if (r<0)
419				return -1;
420			n += r;
421		}
422	}
423	if (!n)
424		tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
425
426#ifdef _IONBF
427	setvbuf(stdout, NULL, _IONBF, 0);
428#endif
429
430	++in_tinytest_main;
431	for (i=0; groups[i].prefix; ++i)
432		for (j=0; groups[i].cases[j].name; ++j)
433			if (groups[i].cases[j].flags & TT_ENABLED_)
434				testcase_run_one(&groups[i],
435						 &groups[i].cases[j]);
436
437	--in_tinytest_main;
438
439	if (opt_verbosity==0)
440		puts("");
441
442	if (n_bad)
443		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
444		       n_bad+n_ok,n_skipped);
445	else if (opt_verbosity >= 1)
446		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
447
448	return (n_bad == 0) ? 0 : 1;
449}
450
451int
452tinytest_get_verbosity_(void)
453{
454	return opt_verbosity;
455}
456
457void
458tinytest_set_test_failed_(void)
459{
460	if (opt_verbosity <= 0 && cur_test_name) {
461		if (opt_verbosity==0) puts("");
462		printf("%s%s: ", cur_test_prefix, cur_test_name);
463		cur_test_name = NULL;
464	}
465	cur_test_outcome = 0;
466}
467
468void
469tinytest_set_test_skipped_(void)
470{
471	if (cur_test_outcome==OK)
472		cur_test_outcome = SKIP;
473}
474
475char *
476tinytest_format_hex_(const void *val_, unsigned long len)
477{
478	const unsigned char *val = val_;
479	char *result, *cp;
480	size_t i;
481
482	if (!val)
483		return strdup("null");
484	if (!(result = malloc(len*2+1)))
485		return strdup("<allocation failure>");
486	cp = result;
487	for (i=0;i<len;++i) {
488		*cp++ = "0123456789ABCDEF"[val[i] >> 4];
489		*cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
490	}
491	*cp = 0;
492	return result;
493}
494