1/* tinytest.c -- Copyright 2009-2012 Nick Mathewson 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions 5 * are met: 6 * 1. Redistributions of source code must retain the above copyright 7 * notice, this list of conditions and the following disclaimer. 8 * 2. Redistributions in binary form must reproduce the above copyright 9 * notice, this list of conditions and the following disclaimer in the 10 * documentation and/or other materials provided with the distribution. 11 * 3. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26#include <stdio.h> 27#include <stdlib.h> 28#include <string.h> 29#include <assert.h> 30 31#ifdef TINYTEST_LOCAL 32#include "tinytest_local.h" 33#endif 34 35#ifdef WIN32 36#include <windows.h> 37#else 38#include <sys/types.h> 39#include <sys/wait.h> 40#include <unistd.h> 41#endif 42 43#ifndef __GNUC__ 44#define __attribute__(x) 45#endif 46 47#include "tinytest.h" 48#include "tinytest_macros.h" 49 50#define LONGEST_TEST_NAME 16384 51 52static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 53static int n_ok = 0; /**< Number of tests that have passed */ 54static int n_bad = 0; /**< Number of tests that have failed. */ 55static int n_skipped = 0; /**< Number of tests that have been skipped. */ 56 57static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 58static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 59static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 60const char *verbosity_flag = ""; 61 62enum outcome { SKIP=2, OK=1, FAIL=0 }; 63static enum outcome cur_test_outcome = 0; 64const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 65/** Name of the current test, if we haven't logged is yet. Used for --quiet */ 66const char *cur_test_name = NULL; 67 68#ifdef WIN32 69/* Copy of argv[0] for win32. */ 70static char commandname[MAX_PATH+1]; 71#endif 72 73static void usage(struct testgroup_t *groups, int list_groups) 74 __attribute__((noreturn)); 75 76static enum outcome 77_testcase_run_bare(const struct testcase_t *testcase) 78{ 79 void *env = NULL; 80 int outcome; 81 if (testcase->setup) { 82 env = testcase->setup->setup_fn(testcase); 83 if (!env) 84 return FAIL; 85 else if (env == (void*)TT_SKIP) 86 return SKIP; 87 } 88 89 cur_test_outcome = OK; 90 testcase->fn(env); 91 outcome = cur_test_outcome; 92 93 if (testcase->setup) { 94 if (testcase->setup->cleanup_fn(testcase, env) == 0) 95 outcome = FAIL; 96 } 97 98 return outcome; 99} 100 101#define MAGIC_EXITCODE 42 102 103static enum outcome 104_testcase_run_forked(const struct testgroup_t *group, 105 const struct testcase_t *testcase) 106{ 107#ifdef WIN32 108 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 109 we'll invoke our own exe (whose name we recall from the command 110 line) with a command line that tells it to run just the test we 111 want, and this time without forking. 112 113 (No, threads aren't an option. The whole point of forking is to 114 share no state between tests.) 115 */ 116 int ok; 117 char buffer[LONGEST_TEST_NAME+256]; 118 STARTUPINFOA si; 119 PROCESS_INFORMATION info; 120 DWORD exitcode; 121 122 if (!in_tinytest_main) { 123 printf("\nERROR. On Windows, _testcase_run_forked must be" 124 " called from within tinytest_main.\n"); 125 abort(); 126 } 127 if (opt_verbosity>0) 128 printf("[forking] "); 129 130 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 131 commandname, verbosity_flag, group->prefix, testcase->name); 132 133 memset(&si, 0, sizeof(si)); 134 memset(&info, 0, sizeof(info)); 135 si.cb = sizeof(si); 136 137 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 138 0, NULL, NULL, &si, &info); 139 if (!ok) { 140 printf("CreateProcess failed!\n"); 141 return 0; 142 } 143 WaitForSingleObject(info.hProcess, INFINITE); 144 GetExitCodeProcess(info.hProcess, &exitcode); 145 CloseHandle(info.hProcess); 146 CloseHandle(info.hThread); 147 if (exitcode == 0) 148 return OK; 149 else if (exitcode == MAGIC_EXITCODE) 150 return SKIP; 151 else 152 return FAIL; 153#else 154 int outcome_pipe[2]; 155 pid_t pid; 156 (void)group; 157 158 if (pipe(outcome_pipe)) 159 perror("opening pipe"); 160 161 if (opt_verbosity>0) 162 printf("[forking] "); 163 pid = fork(); 164 if (!pid) { 165 /* child. */ 166 int test_r, write_r; 167 char b[1]; 168 close(outcome_pipe[0]); 169 test_r = _testcase_run_bare(testcase); 170 assert(0<=(int)test_r && (int)test_r<=2); 171 b[0] = "NYS"[test_r]; 172 write_r = (int)write(outcome_pipe[1], b, 1); 173 if (write_r != 1) { 174 perror("write outcome to pipe"); 175 exit(1); 176 } 177 exit(0); 178 return FAIL; /* unreachable */ 179 } else { 180 /* parent */ 181 int status, r; 182 char b[1]; 183 /* Close this now, so that if the other side closes it, 184 * our read fails. */ 185 close(outcome_pipe[1]); 186 r = (int)read(outcome_pipe[0], b, 1); 187 if (r == 0) { 188 printf("[Lost connection!] "); 189 return 0; 190 } else if (r != 1) { 191 perror("read outcome from pipe"); 192 } 193 waitpid(pid, &status, 0); 194 close(outcome_pipe[0]); 195 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 196 } 197#endif 198} 199 200int 201testcase_run_one(const struct testgroup_t *group, 202 const struct testcase_t *testcase) 203{ 204 enum outcome outcome; 205 206 if (testcase->flags & TT_SKIP) { 207 if (opt_verbosity>0) 208 printf("%s%s: SKIPPED\n", 209 group->prefix, testcase->name); 210 ++n_skipped; 211 return SKIP; 212 } 213 214 if (opt_verbosity>0 && !opt_forked) { 215 printf("%s%s: ", group->prefix, testcase->name); 216 } else { 217 if (opt_verbosity==0) printf("."); 218 cur_test_prefix = group->prefix; 219 cur_test_name = testcase->name; 220 } 221 222 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 223 outcome = _testcase_run_forked(group, testcase); 224 } else { 225 outcome = _testcase_run_bare(testcase); 226 } 227 228 if (outcome == OK) { 229 ++n_ok; 230 if (opt_verbosity>0 && !opt_forked) 231 puts(opt_verbosity==1?"OK":""); 232 } else if (outcome == SKIP) { 233 ++n_skipped; 234 if (opt_verbosity>0 && !opt_forked) 235 puts("SKIPPED"); 236 } else { 237 ++n_bad; 238 if (!opt_forked) 239 printf("\n [%s FAILED]\n", testcase->name); 240 } 241 242 if (opt_forked) { 243 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 244 return 1; /* unreachable */ 245 } else { 246 return (int)outcome; 247 } 248} 249 250int 251_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) 252{ 253 int i, j; 254 size_t length = LONGEST_TEST_NAME; 255 char fullname[LONGEST_TEST_NAME]; 256 int found=0; 257 if (strstr(arg, "..")) 258 length = strstr(arg,"..")-arg; 259 for (i=0; groups[i].prefix; ++i) { 260 for (j=0; groups[i].cases[j].name; ++j) { 261 snprintf(fullname, sizeof(fullname), "%s%s", 262 groups[i].prefix, groups[i].cases[j].name); 263 if (!flag) /* Hack! */ 264 printf(" %s\n", fullname); 265 if (!strncmp(fullname, arg, length)) { 266 groups[i].cases[j].flags |= flag; 267 ++found; 268 } 269 } 270 } 271 return found; 272} 273 274static void 275usage(struct testgroup_t *groups, int list_groups) 276{ 277 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 278 puts(" Specify tests by name, or using a prefix ending with '..'"); 279 puts(" To skip a test, list give its name prefixed with a colon."); 280 puts(" Use --list-tests for a list of tests."); 281 if (list_groups) { 282 puts("Known tests are:"); 283 _tinytest_set_flag(groups, "..", 0); 284 } 285 exit(0); 286} 287 288int 289tinytest_main(int c, const char **v, struct testgroup_t *groups) 290{ 291 int i, j, n=0; 292 293#ifdef WIN32 294 const char *sp = strrchr(v[0], '.'); 295 const char *extension = ""; 296 if (!sp || stricmp(sp, ".exe")) 297 extension = ".exe"; /* Add an exe so CreateProcess will work */ 298 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 299 commandname[MAX_PATH]='\0'; 300#endif 301 for (i=1; i<c; ++i) { 302 if (v[i][0] == '-') { 303 if (!strcmp(v[i], "--RUNNING-FORKED")) { 304 opt_forked = 1; 305 } else if (!strcmp(v[i], "--no-fork")) { 306 opt_nofork = 1; 307 } else if (!strcmp(v[i], "--quiet")) { 308 opt_verbosity = -1; 309 verbosity_flag = "--quiet"; 310 } else if (!strcmp(v[i], "--verbose")) { 311 opt_verbosity = 2; 312 verbosity_flag = "--verbose"; 313 } else if (!strcmp(v[i], "--terse")) { 314 opt_verbosity = 0; 315 verbosity_flag = "--terse"; 316 } else if (!strcmp(v[i], "--help")) { 317 usage(groups, 0); 318 } else if (!strcmp(v[i], "--list-tests")) { 319 usage(groups, 1); 320 } else { 321 printf("Unknown option %s. Try --help\n",v[i]); 322 return -1; 323 } 324 } else { 325 const char *test = v[i]; 326 int flag = _TT_ENABLED; 327 if (test[0] == ':') { 328 ++test; 329 flag = TT_SKIP; 330 } else { 331 ++n; 332 } 333 if (!_tinytest_set_flag(groups, test, flag)) { 334 printf("No such test as %s!\n", v[i]); 335 return -1; 336 } 337 } 338 } 339 if (!n) 340 _tinytest_set_flag(groups, "..", _TT_ENABLED); 341 342 setvbuf(stdout, NULL, _IONBF, 0); 343 344 ++in_tinytest_main; 345 for (i=0; groups[i].prefix; ++i) 346 for (j=0; groups[i].cases[j].name; ++j) 347 if (groups[i].cases[j].flags & _TT_ENABLED) 348 testcase_run_one(&groups[i], 349 &groups[i].cases[j]); 350 351 --in_tinytest_main; 352 353 if (opt_verbosity==0) 354 puts(""); 355 356 if (n_bad) 357 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 358 n_bad+n_ok,n_skipped); 359 else if (opt_verbosity >= 1) 360 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 361 362 return (n_bad == 0) ? 0 : 1; 363} 364 365int 366_tinytest_get_verbosity(void) 367{ 368 return opt_verbosity; 369} 370 371void 372_tinytest_set_test_failed(void) 373{ 374 if (opt_verbosity <= 0 && cur_test_name) { 375 if (opt_verbosity==0) puts(""); 376 printf("%s%s: ", cur_test_prefix, cur_test_name); 377 cur_test_name = NULL; 378 } 379 cur_test_outcome = 0; 380} 381 382void 383_tinytest_set_test_skipped(void) 384{ 385 if (cur_test_outcome==OK) 386 cur_test_outcome = SKIP; 387} 388 389