1275970Scy/* tinytest.c -- Copyright 2009-2012 Nick Mathewson 2275970Scy * 3275970Scy * Redistribution and use in source and binary forms, with or without 4275970Scy * modification, are permitted provided that the following conditions 5275970Scy * are met: 6275970Scy * 1. Redistributions of source code must retain the above copyright 7275970Scy * notice, this list of conditions and the following disclaimer. 8275970Scy * 2. Redistributions in binary form must reproduce the above copyright 9275970Scy * notice, this list of conditions and the following disclaimer in the 10275970Scy * documentation and/or other materials provided with the distribution. 11275970Scy * 3. The name of the author may not be used to endorse or promote products 12275970Scy * derived from this software without specific prior written permission. 13275970Scy * 14275970Scy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15275970Scy * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16275970Scy * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17275970Scy * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18275970Scy * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19275970Scy * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20275970Scy * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21275970Scy * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22275970Scy * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23275970Scy * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24275970Scy */ 25275970Scy#ifdef TINYTEST_LOCAL 26275970Scy#include "tinytest_local.h" 27275970Scy#endif 28275970Scy 29275970Scy#include <stdio.h> 30275970Scy#include <stdlib.h> 31275970Scy#include <string.h> 32275970Scy#include <assert.h> 33275970Scy 34282408Scy#ifndef NO_FORKING 35282408Scy 36275970Scy#ifdef _WIN32 37275970Scy#include <windows.h> 38275970Scy#else 39275970Scy#include <sys/types.h> 40275970Scy#include <sys/wait.h> 41275970Scy#include <unistd.h> 42275970Scy#endif 43275970Scy 44275970Scy#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 45275970Scy#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 46275970Scy __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 47275970Scy/* Workaround for a stupid bug in OSX 10.6 */ 48275970Scy#define FORK_BREAKS_GCOV 49275970Scy#include <vproc.h> 50275970Scy#endif 51275970Scy#endif 52275970Scy 53282408Scy#endif /* !NO_FORKING */ 54282408Scy 55275970Scy#ifndef __GNUC__ 56275970Scy#define __attribute__(x) 57275970Scy#endif 58275970Scy 59275970Scy#include "tinytest.h" 60275970Scy#include "tinytest_macros.h" 61275970Scy 62275970Scy#define LONGEST_TEST_NAME 16384 63275970Scy 64275970Scystatic int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 65275970Scystatic int n_ok = 0; /**< Number of tests that have passed */ 66275970Scystatic int n_bad = 0; /**< Number of tests that have failed. */ 67275970Scystatic int n_skipped = 0; /**< Number of tests that have been skipped. */ 68275970Scy 69275970Scystatic int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 70275970Scystatic int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 71275970Scystatic int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 72275970Scyconst char *verbosity_flag = ""; 73275970Scy 74275970Scyconst struct testlist_alias_t *cfg_aliases=NULL; 75275970Scy 76275970Scyenum outcome { SKIP=2, OK=1, FAIL=0 }; 77275970Scystatic enum outcome cur_test_outcome = 0; 78275970Scyconst char *cur_test_prefix = NULL; /**< prefix of the current test group */ 79275970Scy/** Name of the current test, if we haven't logged is yet. Used for --quiet */ 80275970Scyconst char *cur_test_name = NULL; 81275970Scy 82275970Scy#ifdef _WIN32 83275970Scy/* Copy of argv[0] for win32. */ 84275970Scystatic char commandname[MAX_PATH+1]; 85275970Scy#endif 86275970Scy 87275970Scystatic void usage(struct testgroup_t *groups, int list_groups) 88275970Scy __attribute__((noreturn)); 89275970Scystatic int process_test_option(struct testgroup_t *groups, const char *test); 90275970Scy 91275970Scystatic enum outcome 92275970Scytestcase_run_bare_(const struct testcase_t *testcase) 93275970Scy{ 94275970Scy void *env = NULL; 95275970Scy int outcome; 96275970Scy if (testcase->setup) { 97275970Scy env = testcase->setup->setup_fn(testcase); 98275970Scy if (!env) 99275970Scy return FAIL; 100275970Scy else if (env == (void*)TT_SKIP) 101275970Scy return SKIP; 102275970Scy } 103275970Scy 104275970Scy cur_test_outcome = OK; 105275970Scy testcase->fn(env); 106275970Scy outcome = cur_test_outcome; 107275970Scy 108275970Scy if (testcase->setup) { 109275970Scy if (testcase->setup->cleanup_fn(testcase, env) == 0) 110275970Scy outcome = FAIL; 111275970Scy } 112275970Scy 113275970Scy return outcome; 114275970Scy} 115275970Scy 116275970Scy#define MAGIC_EXITCODE 42 117275970Scy 118282408Scy#ifndef NO_FORKING 119282408Scy 120275970Scystatic enum outcome 121275970Scytestcase_run_forked_(const struct testgroup_t *group, 122275970Scy const struct testcase_t *testcase) 123275970Scy{ 124275970Scy#ifdef _WIN32 125275970Scy /* Fork? On Win32? How primitive! We'll do what the smart kids do: 126275970Scy we'll invoke our own exe (whose name we recall from the command 127275970Scy line) with a command line that tells it to run just the test we 128275970Scy want, and this time without forking. 129275970Scy 130275970Scy (No, threads aren't an option. The whole point of forking is to 131275970Scy share no state between tests.) 132275970Scy */ 133275970Scy int ok; 134275970Scy char buffer[LONGEST_TEST_NAME+256]; 135275970Scy STARTUPINFOA si; 136275970Scy PROCESS_INFORMATION info; 137275970Scy DWORD exitcode; 138275970Scy 139275970Scy if (!in_tinytest_main) { 140275970Scy printf("\nERROR. On Windows, testcase_run_forked_ must be" 141275970Scy " called from within tinytest_main.\n"); 142275970Scy abort(); 143275970Scy } 144275970Scy if (opt_verbosity>0) 145275970Scy printf("[forking] "); 146275970Scy 147275970Scy snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 148275970Scy commandname, verbosity_flag, group->prefix, testcase->name); 149275970Scy 150275970Scy memset(&si, 0, sizeof(si)); 151275970Scy memset(&info, 0, sizeof(info)); 152275970Scy si.cb = sizeof(si); 153275970Scy 154275970Scy ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 155275970Scy 0, NULL, NULL, &si, &info); 156275970Scy if (!ok) { 157275970Scy printf("CreateProcess failed!\n"); 158275970Scy return 0; 159275970Scy } 160275970Scy WaitForSingleObject(info.hProcess, INFINITE); 161275970Scy GetExitCodeProcess(info.hProcess, &exitcode); 162275970Scy CloseHandle(info.hProcess); 163275970Scy CloseHandle(info.hThread); 164275970Scy if (exitcode == 0) 165275970Scy return OK; 166275970Scy else if (exitcode == MAGIC_EXITCODE) 167275970Scy return SKIP; 168275970Scy else 169275970Scy return FAIL; 170275970Scy#else 171275970Scy int outcome_pipe[2]; 172275970Scy pid_t pid; 173275970Scy (void)group; 174275970Scy 175275970Scy if (pipe(outcome_pipe)) 176275970Scy perror("opening pipe"); 177275970Scy 178275970Scy if (opt_verbosity>0) 179275970Scy printf("[forking] "); 180275970Scy pid = fork(); 181275970Scy#ifdef FORK_BREAKS_GCOV 182275970Scy vproc_transaction_begin(0); 183275970Scy#endif 184275970Scy if (!pid) { 185275970Scy /* child. */ 186275970Scy int test_r, write_r; 187275970Scy char b[1]; 188275970Scy close(outcome_pipe[0]); 189275970Scy test_r = testcase_run_bare_(testcase); 190275970Scy assert(0<=(int)test_r && (int)test_r<=2); 191275970Scy b[0] = "NYS"[test_r]; 192275970Scy write_r = (int)write(outcome_pipe[1], b, 1); 193275970Scy if (write_r != 1) { 194275970Scy perror("write outcome to pipe"); 195275970Scy exit(1); 196275970Scy } 197275970Scy exit(0); 198275970Scy return FAIL; /* unreachable */ 199275970Scy } else { 200275970Scy /* parent */ 201275970Scy int status, r; 202275970Scy char b[1]; 203275970Scy /* Close this now, so that if the other side closes it, 204275970Scy * our read fails. */ 205275970Scy close(outcome_pipe[1]); 206275970Scy r = (int)read(outcome_pipe[0], b, 1); 207275970Scy if (r == 0) { 208275970Scy printf("[Lost connection!] "); 209275970Scy return 0; 210275970Scy } else if (r != 1) { 211275970Scy perror("read outcome from pipe"); 212275970Scy } 213275970Scy waitpid(pid, &status, 0); 214275970Scy close(outcome_pipe[0]); 215275970Scy return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 216275970Scy } 217275970Scy#endif 218275970Scy} 219275970Scy 220282408Scy#endif /* !NO_FORKING */ 221282408Scy 222275970Scyint 223275970Scytestcase_run_one(const struct testgroup_t *group, 224275970Scy const struct testcase_t *testcase) 225275970Scy{ 226275970Scy enum outcome outcome; 227275970Scy 228275970Scy if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { 229275970Scy if (opt_verbosity>0) 230275970Scy printf("%s%s: %s\n", 231275970Scy group->prefix, testcase->name, 232275970Scy (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); 233275970Scy ++n_skipped; 234275970Scy return SKIP; 235275970Scy } 236275970Scy 237275970Scy if (opt_verbosity>0 && !opt_forked) { 238275970Scy printf("%s%s: ", group->prefix, testcase->name); 239275970Scy } else { 240275970Scy if (opt_verbosity==0) printf("."); 241275970Scy cur_test_prefix = group->prefix; 242275970Scy cur_test_name = testcase->name; 243275970Scy } 244275970Scy 245282408Scy#ifndef NO_FORKING 246275970Scy if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 247275970Scy outcome = testcase_run_forked_(group, testcase); 248275970Scy } else { 249282408Scy#else 250282408Scy { 251282408Scy#endif 252275970Scy outcome = testcase_run_bare_(testcase); 253275970Scy } 254275970Scy 255275970Scy if (outcome == OK) { 256275970Scy ++n_ok; 257275970Scy if (opt_verbosity>0 && !opt_forked) 258275970Scy puts(opt_verbosity==1?"OK":""); 259275970Scy } else if (outcome == SKIP) { 260275970Scy ++n_skipped; 261275970Scy if (opt_verbosity>0 && !opt_forked) 262275970Scy puts("SKIPPED"); 263275970Scy } else { 264275970Scy ++n_bad; 265275970Scy if (!opt_forked) 266275970Scy printf("\n [%s FAILED]\n", testcase->name); 267275970Scy } 268275970Scy 269275970Scy if (opt_forked) { 270275970Scy exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 271275970Scy return 1; /* unreachable */ 272275970Scy } else { 273275970Scy return (int)outcome; 274275970Scy } 275275970Scy} 276275970Scy 277275970Scyint 278275970Scytinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) 279275970Scy{ 280275970Scy int i, j; 281275970Scy size_t length = LONGEST_TEST_NAME; 282275970Scy char fullname[LONGEST_TEST_NAME]; 283275970Scy int found=0; 284275970Scy if (strstr(arg, "..")) 285275970Scy length = strstr(arg,"..")-arg; 286275970Scy for (i=0; groups[i].prefix; ++i) { 287275970Scy for (j=0; groups[i].cases[j].name; ++j) { 288275970Scy struct testcase_t *testcase = &groups[i].cases[j]; 289275970Scy snprintf(fullname, sizeof(fullname), "%s%s", 290275970Scy groups[i].prefix, testcase->name); 291275970Scy if (!flag) { /* Hack! */ 292275970Scy printf(" %s", fullname); 293275970Scy if (testcase->flags & TT_OFF_BY_DEFAULT) 294275970Scy puts(" (Off by default)"); 295275970Scy else if (testcase->flags & TT_SKIP) 296275970Scy puts(" (DISABLED)"); 297275970Scy else 298275970Scy puts(""); 299275970Scy } 300275970Scy if (!strncmp(fullname, arg, length)) { 301275970Scy if (set) 302275970Scy testcase->flags |= flag; 303275970Scy else 304275970Scy testcase->flags &= ~flag; 305275970Scy ++found; 306275970Scy } 307275970Scy } 308275970Scy } 309275970Scy return found; 310275970Scy} 311275970Scy 312275970Scystatic void 313275970Scyusage(struct testgroup_t *groups, int list_groups) 314275970Scy{ 315275970Scy puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 316275970Scy puts(" Specify tests by name, or using a prefix ending with '..'"); 317275970Scy puts(" To skip a test, prefix its name with a colon."); 318275970Scy puts(" To enable a disabled test, prefix its name with a plus."); 319275970Scy puts(" Use --list-tests for a list of tests."); 320275970Scy if (list_groups) { 321275970Scy puts("Known tests are:"); 322275970Scy tinytest_set_flag_(groups, "..", 1, 0); 323275970Scy } 324275970Scy exit(0); 325275970Scy} 326275970Scy 327275970Scystatic int 328275970Scyprocess_test_alias(struct testgroup_t *groups, const char *test) 329275970Scy{ 330275970Scy int i, j, n, r; 331275970Scy for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { 332275970Scy if (!strcmp(cfg_aliases[i].name, test)) { 333275970Scy n = 0; 334275970Scy for (j = 0; cfg_aliases[i].tests[j]; ++j) { 335275970Scy r = process_test_option(groups, cfg_aliases[i].tests[j]); 336275970Scy if (r<0) 337275970Scy return -1; 338275970Scy n += r; 339275970Scy } 340275970Scy return n; 341275970Scy } 342275970Scy } 343275970Scy printf("No such test alias as @%s!",test); 344275970Scy return -1; 345275970Scy} 346275970Scy 347275970Scystatic int 348275970Scyprocess_test_option(struct testgroup_t *groups, const char *test) 349275970Scy{ 350275970Scy int flag = TT_ENABLED_; 351275970Scy int n = 0; 352275970Scy if (test[0] == '@') { 353275970Scy return process_test_alias(groups, test + 1); 354275970Scy } else if (test[0] == ':') { 355275970Scy ++test; 356275970Scy flag = TT_SKIP; 357275970Scy } else if (test[0] == '+') { 358275970Scy ++test; 359275970Scy ++n; 360275970Scy if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { 361275970Scy printf("No such test as %s!\n", test); 362275970Scy return -1; 363275970Scy } 364275970Scy } else { 365275970Scy ++n; 366275970Scy } 367275970Scy if (!tinytest_set_flag_(groups, test, 1, flag)) { 368275970Scy printf("No such test as %s!\n", test); 369275970Scy return -1; 370275970Scy } 371275970Scy return n; 372275970Scy} 373275970Scy 374275970Scyvoid 375275970Scytinytest_set_aliases(const struct testlist_alias_t *aliases) 376275970Scy{ 377275970Scy cfg_aliases = aliases; 378275970Scy} 379275970Scy 380275970Scyint 381275970Scytinytest_main(int c, const char **v, struct testgroup_t *groups) 382275970Scy{ 383275970Scy int i, j, n=0; 384275970Scy 385275970Scy#ifdef _WIN32 386275970Scy const char *sp = strrchr(v[0], '.'); 387275970Scy const char *extension = ""; 388275970Scy if (!sp || stricmp(sp, ".exe")) 389275970Scy extension = ".exe"; /* Add an exe so CreateProcess will work */ 390275970Scy snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 391275970Scy commandname[MAX_PATH]='\0'; 392275970Scy#endif 393275970Scy for (i=1; i<c; ++i) { 394275970Scy if (v[i][0] == '-') { 395275970Scy if (!strcmp(v[i], "--RUNNING-FORKED")) { 396275970Scy opt_forked = 1; 397275970Scy } else if (!strcmp(v[i], "--no-fork")) { 398275970Scy opt_nofork = 1; 399275970Scy } else if (!strcmp(v[i], "--quiet")) { 400275970Scy opt_verbosity = -1; 401275970Scy verbosity_flag = "--quiet"; 402275970Scy } else if (!strcmp(v[i], "--verbose")) { 403275970Scy opt_verbosity = 2; 404275970Scy verbosity_flag = "--verbose"; 405275970Scy } else if (!strcmp(v[i], "--terse")) { 406275970Scy opt_verbosity = 0; 407275970Scy verbosity_flag = "--terse"; 408275970Scy } else if (!strcmp(v[i], "--help")) { 409275970Scy usage(groups, 0); 410275970Scy } else if (!strcmp(v[i], "--list-tests")) { 411275970Scy usage(groups, 1); 412275970Scy } else { 413275970Scy printf("Unknown option %s. Try --help\n",v[i]); 414275970Scy return -1; 415275970Scy } 416275970Scy } else { 417275970Scy int r = process_test_option(groups, v[i]); 418275970Scy if (r<0) 419275970Scy return -1; 420275970Scy n += r; 421275970Scy } 422275970Scy } 423275970Scy if (!n) 424275970Scy tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); 425275970Scy 426282408Scy#ifdef _IONBF 427275970Scy setvbuf(stdout, NULL, _IONBF, 0); 428282408Scy#endif 429275970Scy 430275970Scy ++in_tinytest_main; 431275970Scy for (i=0; groups[i].prefix; ++i) 432275970Scy for (j=0; groups[i].cases[j].name; ++j) 433275970Scy if (groups[i].cases[j].flags & TT_ENABLED_) 434275970Scy testcase_run_one(&groups[i], 435275970Scy &groups[i].cases[j]); 436275970Scy 437275970Scy --in_tinytest_main; 438275970Scy 439275970Scy if (opt_verbosity==0) 440275970Scy puts(""); 441275970Scy 442275970Scy if (n_bad) 443275970Scy printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 444275970Scy n_bad+n_ok,n_skipped); 445275970Scy else if (opt_verbosity >= 1) 446275970Scy printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 447275970Scy 448275970Scy return (n_bad == 0) ? 0 : 1; 449275970Scy} 450275970Scy 451275970Scyint 452275970Scytinytest_get_verbosity_(void) 453275970Scy{ 454275970Scy return opt_verbosity; 455275970Scy} 456275970Scy 457275970Scyvoid 458275970Scytinytest_set_test_failed_(void) 459275970Scy{ 460275970Scy if (opt_verbosity <= 0 && cur_test_name) { 461275970Scy if (opt_verbosity==0) puts(""); 462275970Scy printf("%s%s: ", cur_test_prefix, cur_test_name); 463275970Scy cur_test_name = NULL; 464275970Scy } 465275970Scy cur_test_outcome = 0; 466275970Scy} 467275970Scy 468275970Scyvoid 469275970Scytinytest_set_test_skipped_(void) 470275970Scy{ 471275970Scy if (cur_test_outcome==OK) 472275970Scy cur_test_outcome = SKIP; 473275970Scy} 474275970Scy 475282408Scychar * 476282408Scytinytest_format_hex_(const void *val_, unsigned long len) 477282408Scy{ 478282408Scy const unsigned char *val = val_; 479282408Scy char *result, *cp; 480282408Scy size_t i; 481282408Scy 482282408Scy if (!val) 483282408Scy return strdup("null"); 484282408Scy if (!(result = malloc(len*2+1))) 485282408Scy return strdup("<allocation failure>"); 486282408Scy cp = result; 487282408Scy for (i=0;i<len;++i) { 488282408Scy *cp++ = "0123456789ABCDEF"[val[i] >> 4]; 489282408Scy *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; 490282408Scy } 491282408Scy *cp = 0; 492282408Scy return result; 493282408Scy} 494