1/* tinytest.c -- Copyright 2009-2012 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26#include <stdio.h> 27#include <stdlib.h> 28#include <string.h> 29#include <assert.h> 30 31#ifdef TINYTEST_LOCAL 32#include "tinytest_local.h" 33#endif 34 35#ifdef WIN32 36#include <windows.h> 37#else 38#include <sys/types.h> 39#include <sys/wait.h> 40#include <unistd.h> 41#endif 42 43#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 44#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 45 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 46/* Workaround for a stupid bug in OSX 10.6 */ 47#define FORK_BREAKS_GCOV 48#include <vproc.h> 49#endif 50#endif 51 52#ifndef __GNUC__ 53#define __attribute__(x) 54#endif 55 56#include "tinytest.h" 57#include "tinytest_macros.h" 58 59#define LONGEST_TEST_NAME 16384 60 61static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 62static int n_ok = 0; /**< Number of tests that have passed */ 63static int n_bad = 0; /**< Number of tests that have failed. */ 64static int n_skipped = 0; /**< Number of tests that have been skipped. */ 65 66static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 67static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 68static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 69const char *verbosity_flag = ""; 70 71enum outcome { SKIP=2, OK=1, FAIL=0 }; 72static enum outcome cur_test_outcome = 0; 73const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 74/** Name of the current test, if we haven't logged is yet. Used for --quiet */ 75const char *cur_test_name = NULL; 76 77#ifdef WIN32 78/* Copy of argv[0] for win32. */ 79static char commandname[MAX_PATH+1]; 80#endif 81 82static void usage(struct testgroup_t *groups, int list_groups) 83 __attribute__((noreturn)); 84 85static enum outcome 86_testcase_run_bare(const struct testcase_t *testcase) 87{ 88 void *env = NULL; 89 int outcome; 90 if (testcase->setup) { 91 env = testcase->setup->setup_fn(testcase); 92 if (!env) 93 return FAIL; 94 else if (env == (void*)TT_SKIP) 95 return SKIP; 96 } 97 98 cur_test_outcome = OK; 99 testcase->fn(env); 100 outcome = cur_test_outcome; 101 102 if (testcase->setup) { 103 if (testcase->setup->cleanup_fn(testcase, env) == 0) 104 outcome = FAIL; 105 } 106 107 return outcome; 108} 109 110#define MAGIC_EXITCODE 42 111 112static enum outcome 113_testcase_run_forked(const struct testgroup_t *group, 114 const struct testcase_t *testcase) 115{ 116#ifdef WIN32 117 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 118 we'll invoke our own exe (whose name we recall from the command 119 line) with a command line that tells it to run just the test we 120 want, and this time without forking. 121 122 (No, threads aren't an option. The whole point of forking is to 123 share no state between tests.) 124 */ 125 int ok; 126 char buffer[LONGEST_TEST_NAME+256]; 127 STARTUPINFOA si; 128 PROCESS_INFORMATION info; 129 DWORD exitcode; 130 131 if (!in_tinytest_main) { 132 printf("\nERROR. On Windows, _testcase_run_forked must be" 133 " called from within tinytest_main.\n"); 134 abort(); 135 } 136 if (opt_verbosity>0) 137 printf("[forking] "); 138 139 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 140 commandname, verbosity_flag, group->prefix, testcase->name); 141 142 memset(&si, 0, sizeof(si)); 143 memset(&info, 0, sizeof(info)); 144 si.cb = sizeof(si); 145 146 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 147 0, NULL, NULL, &si, &info); 148 if (!ok) { 149 printf("CreateProcess failed!\n"); 150 return 0; 151 } 152 WaitForSingleObject(info.hProcess, INFINITE); 153 GetExitCodeProcess(info.hProcess, &exitcode); 154 CloseHandle(info.hProcess); 155 CloseHandle(info.hThread); 156 if (exitcode == 0) 157 return OK; 158 else if (exitcode == MAGIC_EXITCODE) 159 return SKIP; 160 else 161 return FAIL; 162#else 163 int outcome_pipe[2]; 164 pid_t pid; 165 (void)group; 166 167 if (pipe(outcome_pipe)) 168 perror("opening pipe"); 169 170 if (opt_verbosity>0) 171 printf("[forking] "); 172 pid = fork(); 173#ifdef FORK_BREAKS_GCOV 174 vproc_transaction_begin(0); 175#endif 176 if (!pid) { 177 /* child. */ 178 int test_r, write_r; 179 char b[1]; 180 close(outcome_pipe[0]); 181 test_r = _testcase_run_bare(testcase); 182 assert(0<=(int)test_r && (int)test_r<=2); 183 b[0] = "NYS"[test_r]; 184 write_r = (int)write(outcome_pipe[1], b, 1); 185 if (write_r != 1) { 186 perror("write outcome to pipe"); 187 exit(1); 188 } 189 exit(0); 190 return FAIL; /* unreachable */ 191 } else { 192 /* parent */ 193 int status, r; 194 char b[1]; 195 /* Close this now, so that if the other side closes it, 196 * our read fails. */ 197 close(outcome_pipe[1]); 198 r = (int)read(outcome_pipe[0], b, 1); 199 if (r == 0) { 200 printf("[Lost connection!] "); 201 return 0; 202 } else if (r != 1) { 203 perror("read outcome from pipe"); 204 } 205 waitpid(pid, &status, 0); 206 close(outcome_pipe[0]); 207 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 208 } 209#endif 210} 211 212int 213testcase_run_one(const struct testgroup_t *group, 214 const struct testcase_t *testcase) 215{ 216 enum outcome outcome; 217 218 if (testcase->flags & TT_SKIP) { 219 if (opt_verbosity>0) 220 printf("%s%s: SKIPPED\n", 221 group->prefix, testcase->name); 222 ++n_skipped; 223 return SKIP; 224 } 225 226 if (opt_verbosity>0 && !opt_forked) { 227 printf("%s%s: ", group->prefix, testcase->name); 228 } else { 229 if (opt_verbosity==0) printf("."); 230 cur_test_prefix = group->prefix; 231 cur_test_name = testcase->name; 232 } 233 234 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 235 outcome = _testcase_run_forked(group, testcase); 236 } else { 237 outcome = _testcase_run_bare(testcase); 238 } 239 240 if (outcome == OK) { 241 ++n_ok; 242 if (opt_verbosity>0 && !opt_forked) 243 puts(opt_verbosity==1?"OK":""); 244 } else if (outcome == SKIP) { 245 ++n_skipped; 246 if (opt_verbosity>0 && !opt_forked) 247 puts("SKIPPED"); 248 } else { 249 ++n_bad; 250 if (!opt_forked) 251 printf("\n [%s FAILED]\n", testcase->name); 252 } 253 254 if (opt_forked) { 255 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 256 return 1; /* unreachable */ 257 } else { 258 return (int)outcome; 259 } 260} 261 262int 263_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) 264{ 265 int i, j; 266 size_t length = LONGEST_TEST_NAME; 267 char fullname[LONGEST_TEST_NAME]; 268 int found=0; 269 if (strstr(arg, "..")) 270 length = strstr(arg,"..")-arg; 271 for (i=0; groups[i].prefix; ++i) { 272 for (j=0; groups[i].cases[j].name; ++j) { 273 snprintf(fullname, sizeof(fullname), "%s%s", 274 groups[i].prefix, groups[i].cases[j].name); 275 if (!flag) /* Hack! */ 276 printf(" %s\n", fullname); 277 if (!strncmp(fullname, arg, length)) { 278 groups[i].cases[j].flags |= flag; 279 ++found; 280 } 281 } 282 } 283 return found; 284} 285 286static void 287usage(struct testgroup_t *groups, int list_groups) 288{ 289 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 290 puts(" Specify tests by name, or using a prefix ending with '..'"); 291 puts(" To skip a test, list give its name prefixed with a colon."); 292 puts(" Use --list-tests for a list of tests."); 293 if (list_groups) { 294 puts("Known tests are:"); 295 _tinytest_set_flag(groups, "..", 0); 296 } 297 exit(0); 298} 299 300int 301tinytest_main(int c, const char **v, struct testgroup_t *groups) 302{ 303 int i, j, n=0; 304 305#ifdef WIN32 306 const char *sp = strrchr(v[0], '.'); 307 const char *extension = ""; 308 if (!sp || stricmp(sp, ".exe")) 309 extension = ".exe"; /* Add an exe so CreateProcess will work */ 310 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 311 commandname[MAX_PATH]='\0'; 312#endif 313 for (i=1; i<c; ++i) { 314 if (v[i][0] == '-') { 315 if (!strcmp(v[i], "--RUNNING-FORKED")) { 316 opt_forked = 1; 317 } else if (!strcmp(v[i], "--no-fork")) { 318 opt_nofork = 1; 319 } else if (!strcmp(v[i], "--quiet")) { 320 opt_verbosity = -1; 321 verbosity_flag = "--quiet"; 322 } else if (!strcmp(v[i], "--verbose")) { 323 opt_verbosity = 2; 324 verbosity_flag = "--verbose"; 325 } else if (!strcmp(v[i], "--terse")) { 326 opt_verbosity = 0; 327 verbosity_flag = "--terse"; 328 } else if (!strcmp(v[i], "--help")) { 329 usage(groups, 0); 330 } else if (!strcmp(v[i], "--list-tests")) { 331 usage(groups, 1); 332 } else { 333 printf("Unknown option %s. Try --help\n",v[i]); 334 return -1; 335 } 336 } else { 337 const char *test = v[i]; 338 int flag = _TT_ENABLED; 339 if (test[0] == ':') { 340 ++test; 341 flag = TT_SKIP; 342 } else { 343 ++n; 344 } 345 if (!_tinytest_set_flag(groups, test, flag)) { 346 printf("No such test as %s!\n", v[i]); 347 return -1; 348 } 349 } 350 } 351 if (!n) 352 _tinytest_set_flag(groups, "..", _TT_ENABLED); 353 354 setvbuf(stdout, NULL, _IONBF, 0); 355 356 ++in_tinytest_main; 357 for (i=0; groups[i].prefix; ++i) 358 for (j=0; groups[i].cases[j].name; ++j) 359 if (groups[i].cases[j].flags & _TT_ENABLED) 360 testcase_run_one(&groups[i], 361 &groups[i].cases[j]); 362 363 --in_tinytest_main; 364 365 if (opt_verbosity==0) 366 puts(""); 367 368 if (n_bad) 369 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 370 n_bad+n_ok,n_skipped); 371 else if (opt_verbosity >= 1) 372 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 373 374 return (n_bad == 0) ? 0 : 1; 375} 376 377int 378_tinytest_get_verbosity(void) 379{ 380 return opt_verbosity; 381} 382 383void 384_tinytest_set_test_failed(void) 385{ 386 if (opt_verbosity <= 0 && cur_test_name) { 387 if (opt_verbosity==0) puts(""); 388 printf("%s%s: ", cur_test_prefix, cur_test_name); 389 cur_test_name = NULL; 390 } 391 cur_test_outcome = 0; 392} 393 394void 395_tinytest_set_test_skipped(void) 396{ 397 if (cur_test_outcome==OK) 398 cur_test_outcome = SKIP; 399} 400 401