1/* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions
5 * are met:
6 * 1. Redistributions of source code must retain the above copyright
7 *    notice, this list of conditions and the following disclaimer.
8 * 2. Redistributions in binary form must reproduce the above copyright
9 *    notice, this list of conditions and the following disclaimer in the
10 *    documentation and/or other materials provided with the distribution.
11 * 3. The name of the author may not be used to endorse or promote products
12 *    derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include <stdio.h>
27#include <stdlib.h>
28#include <string.h>
29#include <assert.h>
30
31#ifdef TINYTEST_LOCAL
32#include "tinytest_local.h"
33#endif
34
35#ifdef WIN32
36#include <windows.h>
37#else
38#include <sys/types.h>
39#include <sys/wait.h>
40#include <unistd.h>
41#endif
42
43#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
44#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
45    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
46/* Workaround for a stupid bug in OSX 10.6 */
47#define FORK_BREAKS_GCOV
48#include <vproc.h>
49#endif
50#endif
51
52#ifndef __GNUC__
53#define __attribute__(x)
54#endif
55
56#include "tinytest.h"
57#include "tinytest_macros.h"
58
59#define LONGEST_TEST_NAME 16384
60
61static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
62static int n_ok = 0; /**< Number of tests that have passed */
63static int n_bad = 0; /**< Number of tests that have failed. */
64static int n_skipped = 0; /**< Number of tests that have been skipped. */
65
66static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
67static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
68static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
69const char *verbosity_flag = "";
70
71enum outcome { SKIP=2, OK=1, FAIL=0 };
72static enum outcome cur_test_outcome = 0;
73const char *cur_test_prefix = NULL; /**< prefix of the current test group */
74/** Name of the current test, if we haven't logged is yet. Used for --quiet */
75const char *cur_test_name = NULL;
76
77#ifdef WIN32
78/* Copy of argv[0] for win32. */
79static char commandname[MAX_PATH+1];
80#endif
81
82static void usage(struct testgroup_t *groups, int list_groups)
83  __attribute__((noreturn));
84
85static enum outcome
86_testcase_run_bare(const struct testcase_t *testcase)
87{
88	void *env = NULL;
89	int outcome;
90	if (testcase->setup) {
91		env = testcase->setup->setup_fn(testcase);
92		if (!env)
93			return FAIL;
94		else if (env == (void*)TT_SKIP)
95			return SKIP;
96	}
97
98	cur_test_outcome = OK;
99	testcase->fn(env);
100	outcome = cur_test_outcome;
101
102	if (testcase->setup) {
103		if (testcase->setup->cleanup_fn(testcase, env) == 0)
104			outcome = FAIL;
105	}
106
107	return outcome;
108}
109
110#define MAGIC_EXITCODE 42
111
112static enum outcome
113_testcase_run_forked(const struct testgroup_t *group,
114		     const struct testcase_t *testcase)
115{
116#ifdef WIN32
117	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
118	   we'll invoke our own exe (whose name we recall from the command
119	   line) with a command line that tells it to run just the test we
120	   want, and this time without forking.
121
122	   (No, threads aren't an option.  The whole point of forking is to
123	   share no state between tests.)
124	 */
125	int ok;
126	char buffer[LONGEST_TEST_NAME+256];
127	STARTUPINFOA si;
128	PROCESS_INFORMATION info;
129	DWORD exitcode;
130
131	if (!in_tinytest_main) {
132		printf("\nERROR.  On Windows, _testcase_run_forked must be"
133		       " called from within tinytest_main.\n");
134		abort();
135	}
136	if (opt_verbosity>0)
137		printf("[forking] ");
138
139	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
140		 commandname, verbosity_flag, group->prefix, testcase->name);
141
142	memset(&si, 0, sizeof(si));
143	memset(&info, 0, sizeof(info));
144	si.cb = sizeof(si);
145
146	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
147			   0, NULL, NULL, &si, &info);
148	if (!ok) {
149		printf("CreateProcess failed!\n");
150		return 0;
151	}
152	WaitForSingleObject(info.hProcess, INFINITE);
153	GetExitCodeProcess(info.hProcess, &exitcode);
154	CloseHandle(info.hProcess);
155	CloseHandle(info.hThread);
156	if (exitcode == 0)
157		return OK;
158	else if (exitcode == MAGIC_EXITCODE)
159		return SKIP;
160	else
161		return FAIL;
162#else
163	int outcome_pipe[2];
164	pid_t pid;
165	(void)group;
166
167	if (pipe(outcome_pipe))
168		perror("opening pipe");
169
170	if (opt_verbosity>0)
171		printf("[forking] ");
172	pid = fork();
173#ifdef FORK_BREAKS_GCOV
174	vproc_transaction_begin(0);
175#endif
176	if (!pid) {
177		/* child. */
178		int test_r, write_r;
179		char b[1];
180		close(outcome_pipe[0]);
181		test_r = _testcase_run_bare(testcase);
182		assert(0<=(int)test_r && (int)test_r<=2);
183		b[0] = "NYS"[test_r];
184		write_r = (int)write(outcome_pipe[1], b, 1);
185		if (write_r != 1) {
186			perror("write outcome to pipe");
187			exit(1);
188		}
189		exit(0);
190		return FAIL; /* unreachable */
191	} else {
192		/* parent */
193		int status, r;
194		char b[1];
195		/* Close this now, so that if the other side closes it,
196		 * our read fails. */
197		close(outcome_pipe[1]);
198		r = (int)read(outcome_pipe[0], b, 1);
199		if (r == 0) {
200			printf("[Lost connection!] ");
201			return 0;
202		} else if (r != 1) {
203			perror("read outcome from pipe");
204		}
205		waitpid(pid, &status, 0);
206		close(outcome_pipe[0]);
207		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
208	}
209#endif
210}
211
212int
213testcase_run_one(const struct testgroup_t *group,
214		 const struct testcase_t *testcase)
215{
216	enum outcome outcome;
217
218	if (testcase->flags & TT_SKIP) {
219		if (opt_verbosity>0)
220			printf("%s%s: SKIPPED\n",
221			    group->prefix, testcase->name);
222		++n_skipped;
223		return SKIP;
224	}
225
226	if (opt_verbosity>0 && !opt_forked) {
227		printf("%s%s: ", group->prefix, testcase->name);
228	} else {
229		if (opt_verbosity==0) printf(".");
230		cur_test_prefix = group->prefix;
231		cur_test_name = testcase->name;
232	}
233
234	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
235		outcome = _testcase_run_forked(group, testcase);
236	} else {
237		outcome = _testcase_run_bare(testcase);
238	}
239
240	if (outcome == OK) {
241		++n_ok;
242		if (opt_verbosity>0 && !opt_forked)
243			puts(opt_verbosity==1?"OK":"");
244	} else if (outcome == SKIP) {
245		++n_skipped;
246		if (opt_verbosity>0 && !opt_forked)
247			puts("SKIPPED");
248	} else {
249		++n_bad;
250		if (!opt_forked)
251			printf("\n  [%s FAILED]\n", testcase->name);
252	}
253
254	if (opt_forked) {
255		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
256		return 1; /* unreachable */
257	} else {
258		return (int)outcome;
259	}
260}
261
262int
263_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag)
264{
265	int i, j;
266	size_t length = LONGEST_TEST_NAME;
267	char fullname[LONGEST_TEST_NAME];
268	int found=0;
269	if (strstr(arg, ".."))
270		length = strstr(arg,"..")-arg;
271	for (i=0; groups[i].prefix; ++i) {
272		for (j=0; groups[i].cases[j].name; ++j) {
273			snprintf(fullname, sizeof(fullname), "%s%s",
274				 groups[i].prefix, groups[i].cases[j].name);
275			if (!flag) /* Hack! */
276				printf("    %s\n", fullname);
277			if (!strncmp(fullname, arg, length)) {
278				groups[i].cases[j].flags |= flag;
279				++found;
280			}
281		}
282	}
283	return found;
284}
285
286static void
287usage(struct testgroup_t *groups, int list_groups)
288{
289	puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
290	puts("  Specify tests by name, or using a prefix ending with '..'");
291	puts("  To skip a test, list give its name prefixed with a colon.");
292	puts("  Use --list-tests for a list of tests.");
293	if (list_groups) {
294		puts("Known tests are:");
295		_tinytest_set_flag(groups, "..", 0);
296	}
297	exit(0);
298}
299
300int
301tinytest_main(int c, const char **v, struct testgroup_t *groups)
302{
303	int i, j, n=0;
304
305#ifdef WIN32
306	const char *sp = strrchr(v[0], '.');
307	const char *extension = "";
308	if (!sp || stricmp(sp, ".exe"))
309		extension = ".exe"; /* Add an exe so CreateProcess will work */
310	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
311	commandname[MAX_PATH]='\0';
312#endif
313	for (i=1; i<c; ++i) {
314		if (v[i][0] == '-') {
315			if (!strcmp(v[i], "--RUNNING-FORKED")) {
316				opt_forked = 1;
317			} else if (!strcmp(v[i], "--no-fork")) {
318				opt_nofork = 1;
319			} else if (!strcmp(v[i], "--quiet")) {
320				opt_verbosity = -1;
321				verbosity_flag = "--quiet";
322			} else if (!strcmp(v[i], "--verbose")) {
323				opt_verbosity = 2;
324				verbosity_flag = "--verbose";
325			} else if (!strcmp(v[i], "--terse")) {
326				opt_verbosity = 0;
327				verbosity_flag = "--terse";
328			} else if (!strcmp(v[i], "--help")) {
329				usage(groups, 0);
330			} else if (!strcmp(v[i], "--list-tests")) {
331				usage(groups, 1);
332			} else {
333				printf("Unknown option %s.  Try --help\n",v[i]);
334				return -1;
335			}
336		} else {
337			const char *test = v[i];
338			int flag = _TT_ENABLED;
339			if (test[0] == ':') {
340				++test;
341				flag = TT_SKIP;
342			} else {
343				++n;
344			}
345			if (!_tinytest_set_flag(groups, test, flag)) {
346				printf("No such test as %s!\n", v[i]);
347				return -1;
348			}
349		}
350	}
351	if (!n)
352		_tinytest_set_flag(groups, "..", _TT_ENABLED);
353
354	setvbuf(stdout, NULL, _IONBF, 0);
355
356	++in_tinytest_main;
357	for (i=0; groups[i].prefix; ++i)
358		for (j=0; groups[i].cases[j].name; ++j)
359			if (groups[i].cases[j].flags & _TT_ENABLED)
360				testcase_run_one(&groups[i],
361						 &groups[i].cases[j]);
362
363	--in_tinytest_main;
364
365	if (opt_verbosity==0)
366		puts("");
367
368	if (n_bad)
369		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
370		       n_bad+n_ok,n_skipped);
371	else if (opt_verbosity >= 1)
372		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
373
374	return (n_bad == 0) ? 0 : 1;
375}
376
377int
378_tinytest_get_verbosity(void)
379{
380	return opt_verbosity;
381}
382
383void
384_tinytest_set_test_failed(void)
385{
386	if (opt_verbosity <= 0 && cur_test_name) {
387		if (opt_verbosity==0) puts("");
388		printf("%s%s: ", cur_test_prefix, cur_test_name);
389		cur_test_name = NULL;
390	}
391	cur_test_outcome = 0;
392}
393
394void
395_tinytest_set_test_skipped(void)
396{
397	if (cur_test_outcome==OK)
398		cur_test_outcome = SKIP;
399}
400
401