1// Copyright 2018 The Fuchsia Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <dirent.h> 6#include <errno.h> 7#include <fcntl.h> 8#include <libgen.h> 9#include <limits.h> 10#include <stdio.h> 11#include <stdlib.h> 12#include <sys/stat.h> 13#include <unistd.h> 14 15#include <fbl/auto_call.h> 16#include <fbl/string_buffer.h> 17#include <fbl/unique_fd.h> 18#include <fbl/unique_ptr.h> 19#include <fbl/vector.h> 20#include <runtests-utils/runtests-utils.h> 21#include <unittest/unittest.h> 22 23#include "runtests-utils-test-globals.h" 24#include "runtests-utils-test-utils.h" 25 26 27namespace runtests { 28namespace { 29 30static constexpr char kEchoSuccessAndArgs[] = "echo Success! $@"; 31static constexpr char kEchoFailureAndArgs[] = "echo Failure! $@ 1>&2\nexit 77"; 32 33bool ParseTestNamesEmptyStr() { 34 BEGIN_TEST; 35 36 fbl::String input(""); 37 fbl::Vector<fbl::String> parsed; 38 ParseTestNames(input, &parsed); 39 EXPECT_EQ(0, parsed.size()); 40 41 END_TEST; 42} 43 44bool ParseTestNamesEmptyStrInMiddle() { 45 BEGIN_TEST; 46 47 fbl::String input("a,,b"); 48 fbl::Vector<fbl::String> parsed; 49 ParseTestNames(input, &parsed); 50 ASSERT_EQ(2, parsed.size()); 51 EXPECT_STR_EQ("a", parsed[0].c_str()); 52 EXPECT_STR_EQ("b", parsed[1].c_str()); 53 54 END_TEST; 55} 56 57bool ParseTestNamesTrailingComma() { 58 BEGIN_TEST; 59 60 fbl::String input("a,"); 61 fbl::Vector<fbl::String> parsed; 62 ParseTestNames(input, &parsed); 63 ASSERT_EQ(1, parsed.size()); 64 EXPECT_STR_EQ("a", parsed[0].c_str()); 65 66 END_TEST; 67} 68 69bool ParseTestNamesNormal() { 70 BEGIN_TEST; 71 72 fbl::String input("a,b"); 73 fbl::Vector<fbl::String> parsed; 74 ParseTestNames(input, &parsed); 75 ASSERT_EQ(2, parsed.size()); 76 EXPECT_STR_EQ("a", parsed[0].c_str()); 77 EXPECT_STR_EQ("b", parsed[1].c_str()); 78 79 END_TEST; 80} 81 82bool EmptyWhitelist() { 83 BEGIN_TEST; 84 85 fbl::Vector<fbl::String> whitelist; 86 EXPECT_FALSE(IsInWhitelist("a", whitelist)); 87 88 END_TEST; 89} 90 91bool NonemptyWhitelist() { 92 BEGIN_TEST; 93 94 fbl::Vector<fbl::String> whitelist = {"b", "a"}; 95 EXPECT_TRUE(IsInWhitelist("a", whitelist)); 96 97 END_TEST; 98} 99 100bool JoinPathNoTrailingSlash() { 101 BEGIN_TEST; 102 103 EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b", "c/d").c_str()); 104 105 END_TEST; 106} 107 108bool JoinPathTrailingSlash() { 109 BEGIN_TEST; 110 111 EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b/", "c/d").c_str()); 112 113 END_TEST; 114} 115 116bool JoinPathAbsoluteChild() { 117 BEGIN_TEST; 118 119 EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b/", "/c/d").c_str()); 120 121 END_TEST; 122} 123 124bool MkDirAllTooLong() { 125 BEGIN_TEST; 126 127 char too_long[PATH_MAX + 2]; 128 memset(too_long, 'a', PATH_MAX + 1); 129 too_long[PATH_MAX + 1] = '\0'; 130 EXPECT_EQ(ENAMETOOLONG, MkDirAll(too_long)); 131 132 END_TEST; 133} 134bool MkDirAllAlreadyExists() { 135 BEGIN_TEST; 136 137 ScopedTestDir test_dir; 138 const fbl::String already = JoinPath(test_dir.path(), "already"); 139 const fbl::String exists = JoinPath(already, "exists"); 140 ASSERT_EQ(0, mkdir(already.c_str(), 0755)); 141 ASSERT_EQ(0, mkdir(exists.c_str(), 0755)); 142 EXPECT_EQ(0, MkDirAll(exists)); 143 144 END_TEST; 145} 146bool MkDirAllParentAlreadyExists() { 147 BEGIN_TEST; 148 149 ScopedTestDir test_dir; 150 const fbl::String parent = JoinPath(test_dir.path(), "existing-parent"); 151 const fbl::String child = JoinPath(parent, "child"); 152 ASSERT_EQ(0, mkdir(parent.c_str(), 0755)); 153 EXPECT_EQ(0, MkDirAll(child)); 154 struct stat s; 155 EXPECT_EQ(0, stat(child.c_str(), &s)); 156 157 END_TEST; 158} 159bool MkDirAllParentDoesNotExist() { 160 BEGIN_TEST; 161 162 ScopedTestDir test_dir; 163 const fbl::String parent = JoinPath(test_dir.path(), "not-existing-parent"); 164 const fbl::String child = JoinPath(parent, "child"); 165 struct stat s; 166 ASSERT_NE(0, stat(parent.c_str(), &s)); 167 EXPECT_EQ(0, MkDirAll(child)); 168 EXPECT_EQ(0, stat(child.c_str(), &s)); 169 170 END_TEST; 171} 172 173bool WriteSummaryJSONSucceeds() { 174 BEGIN_TEST; 175 176 // TODO(IN-499): Use fmemopen instead of tmpfile. 177 FILE* output_file = tmpfile(); 178 ASSERT_NONNULL(output_file); 179 fbl::Vector<fbl::unique_ptr<Result>> results; 180 results.push_back(fbl::make_unique<Result>("/a", SUCCESS, 0)); 181 results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0)); 182 ASSERT_EQ(0, WriteSummaryJSON(results, "output.txt", "/tmp/file_path", 183 output_file)); 184 // We don't have a JSON parser in zircon right now, so just hard-code the 185 // expected output. 186 const char kExpectedJSONOutput[] = R"({ 187 "tests": [ 188 { 189 "name": "/a", 190 "output_file": "a/output.txt", 191 "result": "PASS" 192 }, 193 { 194 "name": "b", 195 "output_file": "b/output.txt", 196 "result": "FAIL" 197 } 198 ], 199 "outputs": { 200 "syslog_file": "/tmp/file_path" 201 } 202} 203)"; 204 EXPECT_TRUE(CompareFileContents(output_file, kExpectedJSONOutput)); 205 fclose(output_file); 206 207 END_TEST; 208} 209 210bool WriteSummaryJSONSucceedsWithoutSyslogPath() { 211 BEGIN_TEST; 212 213 // TODO(IN-499): Use fmemopen instead of tmpfile. 214 FILE* output_file = tmpfile(); 215 ASSERT_NONNULL(output_file); 216 fbl::Vector<fbl::unique_ptr<Result>> results; 217 results.push_back(fbl::make_unique<Result>("/a", SUCCESS, 0)); 218 results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0)); 219 ASSERT_EQ(0, WriteSummaryJSON(results, "output.txt", /*syslog_path=*/"", 220 output_file)); 221 // With an empty syslog_path, we expect no values under "outputs" and 222 // "syslog_file" to be generated in the JSON output. 223 const char kExpectedJSONOutput[] = R"({ 224 "tests": [ 225 { 226 "name": "/a", 227 "output_file": "a/output.txt", 228 "result": "PASS" 229 }, 230 { 231 "name": "b", 232 "output_file": "b/output.txt", 233 "result": "FAIL" 234 } 235 ] 236} 237)"; 238 239 EXPECT_TRUE(CompareFileContents(output_file, kExpectedJSONOutput)); 240 fclose(output_file); 241 242 END_TEST; 243} 244 245bool WriteSummaryJSONBadTestName() { 246 BEGIN_TEST; 247 248 // TODO(IN-499): Use fmemopen instead of tmpfile. 249 FILE* output_file = tmpfile(); 250 ASSERT_NONNULL(output_file); 251 // A test name and output file consisting entirely of slashes should trigger 252 // an error. 253 fbl::Vector<fbl::unique_ptr<Result>> results; 254 results.push_back(fbl::make_unique<Result>("///", SUCCESS, 0)); 255 results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0)); 256 ASSERT_NE(0, WriteSummaryJSON(results, /*output_file_basename=*/"///", 257 /*syslog_path=*/"/", output_file)); 258 fclose(output_file); 259 260 END_TEST; 261} 262 263bool ResolveGlobsNoMatches() { 264 BEGIN_TEST; 265 266 ScopedTestDir test_dir; 267 fbl::Vector<fbl::String> resolved; 268 fbl::String test_fs_glob = JoinPath(test_dir.path(), "bar*"); 269 const fbl::Vector<fbl::String> globs = {"/foo/bar/*", test_fs_glob}; 270 ASSERT_EQ(0, ResolveGlobs(globs, &resolved)); 271 EXPECT_EQ(0, resolved.size()); 272 273 END_TEST; 274} 275 276bool ResolveGlobsMultipleMatches() { 277 BEGIN_TEST; 278 279 ScopedTestDir test_dir; 280 fbl::String existing_dir_path = 281 JoinPath(test_dir.path(), "existing-dir/prefix-suffix"); 282 fbl::String existing_file_path = JoinPath(test_dir.path(), "existing-file"); 283 fbl::String existing_dir_glob = 284 JoinPath(test_dir.path(), "existing-dir/prefix*"); 285 const fbl::Vector<fbl::String> globs = { 286 "/does/not/exist/*", 287 existing_dir_glob, // matches existing_dir_path. 288 existing_file_path}; 289 ASSERT_EQ(0, MkDirAll(existing_dir_path)); 290 const int existing_file_fd = open(existing_file_path.c_str(), O_CREAT); 291 ASSERT_NE(-1, existing_file_fd, strerror(errno)); 292 ASSERT_NE(-1, close(existing_file_fd), strerror(errno)); 293 fbl::Vector<fbl::String> resolved; 294 ASSERT_EQ(0, ResolveGlobs(globs, &resolved)); 295 ASSERT_EQ(2, resolved.size()); 296 EXPECT_STR_EQ(existing_dir_path.c_str(), resolved[0].c_str()); 297 298 END_TEST; 299} 300 301bool RunTestSuccess() { 302 BEGIN_TEST; 303 304 ScopedTestDir test_dir; 305 fbl::String test_name = JoinPath(test_dir.path(), "succeed.sh"); 306 const char* argv[] = {test_name.c_str(), nullptr}; 307 ScopedScriptFile script(argv[0], "exit 0"); 308 fbl::unique_ptr<Result> result = PlatformRunTest(argv, nullptr, nullptr); 309 EXPECT_STR_EQ(argv[0], result->name.c_str()); 310 EXPECT_EQ(SUCCESS, result->launch_status); 311 EXPECT_EQ(0, result->return_code); 312 313 END_TEST; 314} 315 316bool RunTestSuccessWithStdout() { 317 BEGIN_TEST; 318 319 ScopedTestDir test_dir; 320 fbl::String test_name = JoinPath(test_dir.path(), "succeed.sh"); 321 const char* argv[] = {test_name.c_str(), nullptr}; 322 const char expected_output[] = "Expect this!\n"; 323 // Produces expected_output, b/c echo adds newline 324 const char script_contents[] = "echo Expect this!"; 325 ScopedScriptFile script(argv[0], script_contents); 326 327 fbl::String output_filename = JoinPath(test_dir.path(), "test.out"); 328 fbl::unique_ptr<Result> result = 329 PlatformRunTest(argv, nullptr, output_filename.c_str()); 330 331 FILE* output_file = fopen(output_filename.c_str(), "r"); 332 ASSERT_TRUE(output_file); 333 char buf[1024]; 334 memset(buf, 0, sizeof(buf)); 335 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file)); 336 fclose(output_file); 337 EXPECT_STR_EQ(expected_output, buf); 338 EXPECT_STR_EQ(argv[0], result->name.c_str()); 339 EXPECT_EQ(SUCCESS, result->launch_status); 340 EXPECT_EQ(0, result->return_code); 341 342 END_TEST; 343} 344 345bool RunTestFailureWithStderr() { 346 BEGIN_TEST; 347 348 ScopedTestDir test_dir; 349 fbl::String test_name = JoinPath(test_dir.path(), "fail.sh"); 350 const char* argv[] = {test_name.c_str(), nullptr}; 351 const char expected_output[] = "Expect this!\n"; 352 // Produces expected_output, b/c echo adds newline 353 const char script_contents[] = "echo Expect this! 1>&2\nexit 77"; 354 ScopedScriptFile script(argv[0], script_contents); 355 356 fbl::String output_filename = JoinPath(test_dir.path(), "test.out"); 357 fbl::unique_ptr<Result> result = 358 PlatformRunTest(argv, nullptr, output_filename.c_str()); 359 360 FILE* output_file = fopen(output_filename.c_str(), "r"); 361 ASSERT_TRUE(output_file); 362 char buf[1024]; 363 memset(buf, 0, sizeof(buf)); 364 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file)); 365 fclose(output_file); 366 EXPECT_STR_EQ(expected_output, buf); 367 EXPECT_STR_EQ(argv[0], result->name.c_str()); 368 EXPECT_EQ(FAILED_NONZERO_RETURN_CODE, result->launch_status); 369 EXPECT_EQ(77, result->return_code); 370 371 END_TEST; 372} 373 374bool RunTestFailureToLoadFile() { 375 BEGIN_TEST; 376 377 const char* argv[] = {"i/do/not/exist/", nullptr}; 378 379 fbl::unique_ptr<Result> result = PlatformRunTest(argv, nullptr, nullptr); 380 EXPECT_STR_EQ(argv[0], result->name.c_str()); 381 EXPECT_EQ(FAILED_TO_LAUNCH, result->launch_status); 382 383 END_TEST; 384} 385 386bool DiscoverTestsInDirGlobsBasic() { 387 BEGIN_TEST; 388 389 ScopedTestDir test_dir; 390 const fbl::String a_file_name = JoinPath(test_dir.path(), "a.sh"); 391 ScopedScriptFile a_file(a_file_name, ""); 392 const fbl::String b_file_name = JoinPath(test_dir.path(), "b.sh"); 393 ScopedScriptFile b_file(b_file_name, ""); 394 fbl::Vector<fbl::String> discovered_paths; 395 EXPECT_EQ(0, DiscoverTestsInDirGlobs({test_dir.path()}, nullptr, {}, 396 &discovered_paths)); 397 EXPECT_EQ(2, discovered_paths.size()); 398 bool discovered_a = false; 399 bool discovered_b = false; 400 // The order of the results is not defined, so just check that each is 401 // present. 402 for (const auto& path : discovered_paths) { 403 if (fbl::StringPiece(path) == a_file.path()) { 404 discovered_a = true; 405 } else if (fbl::StringPiece(path) == b_file.path()) { 406 discovered_b = true; 407 } 408 } 409 EXPECT_TRUE(discovered_a); 410 EXPECT_TRUE(discovered_b); 411 412 END_TEST; 413} 414 415bool DiscoverTestsInDirGlobsFilter() { 416 BEGIN_TEST; 417 418 ScopedTestDir test_dir; 419 const char kHopefullyUniqueFileBasename[] = 420 "e829cea9919fe045ca199945db7ac99a"; 421 const fbl::String unique_file_name = 422 JoinPath(test_dir.path(), kHopefullyUniqueFileBasename); 423 ScopedScriptFile unique_file(unique_file_name, ""); 424 // This one should be ignored because its basename is not in the white list. 425 const fbl::String other_file_name = JoinPath(test_dir.path(), "foo.sh"); 426 ScopedScriptFile fail_file(other_file_name, ""); 427 fbl::Vector<fbl::String> discovered_paths; 428 EXPECT_EQ(0, DiscoverTestsInDirGlobs({JoinPath(TestFsRoot(), "*")}, nullptr, 429 {kHopefullyUniqueFileBasename}, 430 &discovered_paths)); 431 EXPECT_EQ(1, discovered_paths.size()); 432 EXPECT_STR_EQ(unique_file_name.c_str(), discovered_paths[0].c_str()); 433 434 END_TEST; 435} 436 437bool DiscoverTestsInDirGlobsIgnore() { 438 BEGIN_TEST; 439 ScopedTestDir test_dir_a, test_dir_b; 440 const fbl::String a_name = JoinPath(test_dir_a.path(), "foo.sh"); 441 ScopedScriptFile a_file(a_name, ""); 442 const fbl::String b_name = JoinPath(test_dir_b.path(), "foo.sh"); 443 ScopedScriptFile fail_file(b_name, ""); 444 fbl::Vector<fbl::String> discovered_paths; 445 EXPECT_EQ(0, DiscoverTestsInDirGlobs({test_dir_a.path(), test_dir_b.path()}, 446 test_dir_b.basename(), {}, 447 &discovered_paths)); 448 EXPECT_EQ(1, discovered_paths.size()); 449 EXPECT_STR_EQ(a_name.c_str(), discovered_paths[0].c_str()); 450 END_TEST; 451} 452 453bool DiscoverTestsInListFileWithTrailingWhitespace() { 454 BEGIN_TEST; 455 // TODO(IN-499): Use fmemopen instead of tmpfile. 456 FILE* test_list_file = tmpfile(); 457 ASSERT_NONNULL(test_list_file); 458 fprintf(test_list_file, "trailing/tab\t\n"); 459 fprintf(test_list_file, "trailing/space \n"); 460 fprintf(test_list_file, "trailing/return\r"); 461 rewind(test_list_file); 462 fbl::Vector<fbl::String> test_paths; 463 EXPECT_EQ(0, DiscoverTestsInListFile(test_list_file, &test_paths)); 464 EXPECT_EQ(3, test_paths.size()); 465 EXPECT_STR_EQ("trailing/tab", test_paths[0].c_str()); 466 EXPECT_STR_EQ("trailing/space", test_paths[1].c_str()); 467 EXPECT_STR_EQ("trailing/return", test_paths[2].c_str()); 468 fclose(test_list_file); 469 END_TEST; 470} 471 472bool RunTestsWithVerbosity() { 473 BEGIN_TEST; 474 475 ScopedTestDir test_dir; 476 const fbl::String succeed_file_name = 477 JoinPath(test_dir.path(), "succeed.sh"); 478 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 479 int num_failed = 0; 480 fbl::Vector<fbl::unique_ptr<Result>> results; 481 const signed char verbosity = 77; 482 const fbl::String output_dir = JoinPath(test_dir.path(), "output"); 483 const char output_file_base_name[] = "output.txt"; 484 ASSERT_EQ(0, MkDirAll(output_dir)); 485 EXPECT_TRUE(RunTests(PlatformRunTest, {succeed_file_name}, 486 output_dir.c_str(), output_file_base_name, verbosity, 487 &num_failed, &results)); 488 EXPECT_EQ(0, num_failed); 489 EXPECT_EQ(1, results.size()); 490 491 fbl::String output_path = JoinPath( 492 JoinPath(output_dir, succeed_file.path()), output_file_base_name); 493 FILE* output_file = fopen(output_path.c_str(), "r"); 494 ASSERT_TRUE(output_file); 495 char buf[1024]; 496 memset(buf, 0, sizeof(buf)); 497 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file)); 498 fclose(output_file); 499 EXPECT_STR_EQ("Success! v=77\n", buf); 500 501 END_TEST; 502} 503 504bool DiscoverAndRunTestsBasicPass() { 505 BEGIN_TEST; 506 507 ScopedTestDir test_dir; 508 const fbl::String succeed_file_name1 = 509 JoinPath(test_dir.path(), "succeed1.sh"); 510 ScopedScriptFile succeed_file1(succeed_file_name1, kEchoSuccessAndArgs); 511 const fbl::String succeed_file_name2 = 512 JoinPath(test_dir.path(), "succeed2.sh"); 513 ScopedScriptFile succeed_file2(succeed_file_name2, kEchoSuccessAndArgs); 514 const char* const argv[] = {"./runtests", test_dir.path()}; 515 TestStopwatch stopwatch; 516 EXPECT_EQ(EXIT_SUCCESS, DiscoverAndRunTests(PlatformRunTest, 2, argv, {}, 517 &stopwatch, "")); 518 519 END_TEST; 520} 521 522bool DiscoverAndRunTestsBasicFail() { 523 BEGIN_TEST; 524 525 ScopedTestDir test_dir; 526 const fbl::String succeed_file_name = 527 JoinPath(test_dir.path(), "succeed.sh"); 528 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 529 const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh"); 530 ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs); 531 const char* const argv[] = {"./runtests", test_dir.path()}; 532 TestStopwatch stopwatch; 533 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 2, argv, {}, 534 &stopwatch, "")); 535 536 END_TEST; 537} 538 539bool DiscoverAndRunTestsFallsBackToDefaultDirs() { 540 BEGIN_TEST; 541 542 ScopedTestDir test_dir; 543 const fbl::String succeed_file_name = 544 JoinPath(test_dir.path(), "succeed.sh"); 545 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 546 const char* const argv[] = {"./runtests"}; 547 TestStopwatch stopwatch; 548 EXPECT_EQ(EXIT_SUCCESS, 549 DiscoverAndRunTests(PlatformRunTest, 1, argv, {test_dir.path()}, 550 &stopwatch, "")); 551 552 END_TEST; 553} 554 555bool DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs() { 556 BEGIN_TEST; 557 558 ScopedTestDir test_dir; 559 const fbl::String succeed_file_name = 560 JoinPath(test_dir.path(), "succeed.sh"); 561 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 562 const char* const argv[] = {"./runtests"}; 563 TestStopwatch stopwatch; 564 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 1, argv, {}, 565 &stopwatch, "")); 566 567 END_TEST; 568} 569 570bool DiscoverAndRunTestsFailsWithBadArgs() { 571 BEGIN_TEST; 572 573 ScopedTestDir test_dir; 574 const fbl::String succeed_file_name = 575 JoinPath(test_dir.path(), "succeed.sh"); 576 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 577 const char* const argv[] = {"./runtests", "-?", "unknown-arg", 578 test_dir.path()}; 579 TestStopwatch stopwatch; 580 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {}, 581 &stopwatch, "")); 582 583 END_TEST; 584} 585 586bool DiscoverAndRunTestsWithGlobs() { 587 BEGIN_TEST; 588 589 ScopedTestDir test_dir; 590 // Make the directories that the following globs will match. 591 const fbl::String dir1 = JoinPath(test_dir.path(), "A/B/C"); 592 EXPECT_EQ(0, MkDirAll(dir1)); 593 const fbl::String dir2 = JoinPath(test_dir.path(), "A/D/C"); 594 EXPECT_EQ(0, MkDirAll(dir2)); 595 596 const fbl::String succeed_file_name1 = 597 JoinPath(test_dir.path(), "succeed.sh"); 598 ScopedScriptFile succeed_file1(succeed_file_name1, kEchoSuccessAndArgs); 599 const fbl::String succeed_file_name2 = JoinPath(dir1, "succeed.sh"); 600 ScopedScriptFile succeed_file2(succeed_file_name2, kEchoSuccessAndArgs); 601 const fbl::String succeed_file_name3 = JoinPath(dir2, "succeed.sh"); 602 ScopedScriptFile succeed_file3(succeed_file_name3, kEchoSuccessAndArgs); 603 604 fbl::String glob = JoinPath(test_dir.path(), "A/*/C"); 605 const char* const argv[] = {"./runtests", test_dir.path(), glob.c_str()}; 606 TestStopwatch stopwatch; 607 EXPECT_EQ(EXIT_SUCCESS, DiscoverAndRunTests(PlatformRunTest, 3, argv, {}, 608 &stopwatch, "")); 609 610 END_TEST; 611} 612 613// Passing an -o argument should result in output being written to that 614// location. 615bool DiscoverAndRunTestsWithOutput() { 616 BEGIN_TEST; 617 618 ScopedTestDir test_dir; 619 const fbl::String succeed_file_name = 620 JoinPath(test_dir.path(), "succeed.sh"); 621 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 622 const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh"); 623 ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs); 624 625 const fbl::String output_dir = 626 JoinPath(test_dir.path(), "run-all-tests-output-1"); 627 EXPECT_EQ(0, MkDirAll(output_dir)); 628 629 const char* const argv[] = {"./runtests", "-o", output_dir.c_str(), 630 test_dir.path()}; 631 TestStopwatch stopwatch; 632 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {}, 633 &stopwatch, "")); 634 635 // Prepare the expected output. 636 fbl::String success_output_rel_path; 637 ASSERT_TRUE(GetOutputFileRelPath(output_dir, succeed_file_name, 638 &success_output_rel_path)); 639 fbl::String failure_output_rel_path; 640 ASSERT_TRUE(GetOutputFileRelPath(output_dir, fail_file_name, 641 &failure_output_rel_path)); 642 643 fbl::StringBuffer<1024> expected_pass_output_buf; 644 expected_pass_output_buf.AppendPrintf( 645 " {\n" 646 " \"name\": \"%s\",\n" 647 " \"output_file\": \"%s\",\n" 648 " \"result\": \"PASS\"\n" 649 " }", 650 succeed_file_name.c_str(), 651 success_output_rel_path.c_str() + 652 1); // +1 to discard the leading slash. 653 fbl::StringBuffer<1024> expected_fail_output_buf; 654 expected_fail_output_buf.AppendPrintf( 655 " {\n" 656 " \"name\": \"%s\",\n" 657 " \"output_file\": \"%s\",\n" 658 " \"result\": \"FAIL\"\n" 659 " }", 660 fail_file_name.c_str(), 661 failure_output_rel_path.c_str() + 662 1); // +1 to discared the leading slash. 663 664 // Extract the actual output. 665 const fbl::String output_path = JoinPath(output_dir, "summary.json"); 666 FILE* output_file = fopen(output_path.c_str(), "r"); 667 ASSERT_TRUE(output_file); 668 char buf[1024]; 669 memset(buf, 0, sizeof(buf)); 670 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file)); 671 fclose(output_file); 672 673 // The order of the tests in summary.json is not defined, so first check the 674 // prefix, then be permissive about order of the actual tests. 675 size_t buf_index = 0; 676 EXPECT_EQ(0, strncmp(kExpectedJSONOutputPrefix, &buf[buf_index], 677 kExpectedJSONOutputPrefixSize)); 678 buf_index += kExpectedJSONOutputPrefixSize; 679 680 if (!strncmp(expected_pass_output_buf.c_str(), &buf[buf_index], 681 expected_pass_output_buf.size())) { 682 buf_index += expected_pass_output_buf.size(); 683 EXPECT_EQ(0, strncmp(",\n", &buf[buf_index], sizeof(",\n") - 1)); 684 buf_index += sizeof(",\n") - 1; 685 EXPECT_EQ(0, strncmp(expected_fail_output_buf.c_str(), &buf[buf_index], 686 expected_fail_output_buf.size())); 687 buf_index += expected_fail_output_buf.size(); 688 } else if (!strncmp(expected_fail_output_buf.c_str(), &buf[buf_index], 689 expected_fail_output_buf.size())) { 690 buf_index += expected_fail_output_buf.size(); 691 EXPECT_EQ(0, strncmp(",\n", &buf[buf_index], sizeof(",\n") - 1)); 692 buf_index += sizeof(",\n") - 1; 693 EXPECT_EQ(0, strncmp(expected_pass_output_buf.c_str(), &buf[buf_index], 694 expected_pass_output_buf.size())); 695 buf_index += expected_pass_output_buf.size(); 696 } else { 697 printf("Unexpected buffer contents: %s\n", buf); 698 EXPECT_TRUE(false, 699 "output buf didn't contain expected pass or fail strings"); 700 } 701 EXPECT_STR_EQ("\n ]\n}\n", &buf[buf_index]); 702 703 END_TEST; 704} 705 706// Passing an -o argument *and* a syslog file name should result in output being 707// written that includes a syslog reference. 708bool DiscoverAndRunTestsWithSyslogOutput() { 709 BEGIN_TEST; 710 711 ScopedTestDir test_dir; 712 const fbl::String succeed_file_name = 713 JoinPath(test_dir.path(), "succeed.sh"); 714 ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs); 715 const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh"); 716 ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs); 717 718 const fbl::String output_dir = 719 JoinPath(test_dir.path(), "run-all-tests-output-2"); 720 EXPECT_EQ(0, MkDirAll(output_dir)); 721 722 const char* const argv[] = {"./runtests", "-o", output_dir.c_str(), 723 test_dir.path()}; 724 TestStopwatch stopwatch; 725 EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {}, 726 &stopwatch, "syslog.txt")); 727 728 // Prepare the expected output. 729 fbl::String success_output_rel_path; 730 ASSERT_TRUE(GetOutputFileRelPath(output_dir, succeed_file_name, 731 &success_output_rel_path)); 732 fbl::String failure_output_rel_path; 733 ASSERT_TRUE(GetOutputFileRelPath(output_dir, fail_file_name, 734 &failure_output_rel_path)); 735 736 const char kExpectedOutputsStr[] = 737 " \"outputs\": {\n" 738 " \"syslog_file\": \"syslog.txt\"\n" 739 " }"; 740 741 // Extract the actual output. 742 const fbl::String output_path = JoinPath(output_dir, "summary.json"); 743 FILE* output_file = fopen(output_path.c_str(), "r"); 744 ASSERT_TRUE(output_file); 745 char buf[1024]; 746 memset(buf, 0, sizeof(buf)); 747 EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file)); 748 fclose(output_file); 749 750 // We don't actually care if the string is at the beginning or the end of 751 // the JSON, so just search for it anywhere. 752 bool found_expected_outputs_str = false; 753 for (size_t buf_index = 0; buf[buf_index]; ++buf_index) { 754 if (!strncmp(kExpectedOutputsStr, &buf[buf_index], 755 sizeof(kExpectedOutputsStr) - 1)) { 756 found_expected_outputs_str = true; 757 break; 758 } 759 } 760 if (!found_expected_outputs_str) { 761 printf("Unexpected buffer contents: %s\n", buf); 762 } 763 EXPECT_TRUE(found_expected_outputs_str, 764 "Didn't find expected outputs str in buf"); 765 766 END_TEST; 767} 768 769BEGIN_TEST_CASE(ParseTestNames) 770RUN_TEST(ParseTestNamesEmptyStr) 771RUN_TEST(ParseTestNamesEmptyStrInMiddle) 772RUN_TEST(ParseTestNamesNormal) 773RUN_TEST(ParseTestNamesTrailingComma) 774END_TEST_CASE(ParseTestNames) 775 776BEGIN_TEST_CASE(IsInWhitelist) 777RUN_TEST(EmptyWhitelist) 778RUN_TEST(NonemptyWhitelist) 779END_TEST_CASE(IsInWhitelist) 780 781BEGIN_TEST_CASE(JoinPath) 782RUN_TEST(JoinPathNoTrailingSlash) 783RUN_TEST(JoinPathTrailingSlash) 784RUN_TEST(JoinPathAbsoluteChild) 785END_TEST_CASE(JoinPath) 786 787BEGIN_TEST_CASE(MkDirAll) 788RUN_TEST(MkDirAllTooLong) 789RUN_TEST(MkDirAllAlreadyExists) 790RUN_TEST(MkDirAllParentAlreadyExists) 791RUN_TEST(MkDirAllParentDoesNotExist) 792END_TEST_CASE(MkDirAll) 793 794BEGIN_TEST_CASE(WriteSummaryJSON) 795RUN_TEST_MEDIUM(WriteSummaryJSONSucceeds) 796RUN_TEST_MEDIUM(WriteSummaryJSONSucceedsWithoutSyslogPath) 797RUN_TEST_MEDIUM(WriteSummaryJSONBadTestName) 798END_TEST_CASE(WriteSummaryJSON) 799 800BEGIN_TEST_CASE(ResolveGlobs) 801RUN_TEST(ResolveGlobsNoMatches) 802RUN_TEST(ResolveGlobsMultipleMatches) 803END_TEST_CASE(ResolveGlobs) 804 805BEGIN_TEST_CASE(RunTest) 806RUN_TEST(RunTestSuccess) 807RUN_TEST(RunTestSuccessWithStdout) 808RUN_TEST(RunTestFailureWithStderr) 809RUN_TEST(RunTestFailureToLoadFile) 810END_TEST_CASE(RunTest) 811 812BEGIN_TEST_CASE(DiscoverTestsInDirGlobs) 813RUN_TEST(DiscoverTestsInDirGlobsBasic) 814RUN_TEST(DiscoverTestsInDirGlobsFilter) 815RUN_TEST(DiscoverTestsInDirGlobsIgnore) 816END_TEST_CASE(DiscoverTestsInDirGlobs) 817 818BEGIN_TEST_CASE(DiscoverTestsInListFile) 819RUN_TEST(DiscoverTestsInListFileWithTrailingWhitespace) 820END_TEST_CASE(DiscoverTestsInListFile) 821 822BEGIN_TEST_CASE(RunTests) 823RUN_TEST_MEDIUM(RunTestsWithVerbosity) 824END_TEST_CASE(RunTests) 825 826BEGIN_TEST_CASE(DiscoverAndRunTests) 827RUN_TEST_MEDIUM(DiscoverAndRunTestsBasicPass) 828RUN_TEST_MEDIUM(DiscoverAndRunTestsBasicFail) 829RUN_TEST_MEDIUM(DiscoverAndRunTestsFallsBackToDefaultDirs) 830RUN_TEST_MEDIUM(DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs) 831RUN_TEST_MEDIUM(DiscoverAndRunTestsFailsWithBadArgs) 832RUN_TEST_MEDIUM(DiscoverAndRunTestsWithGlobs) 833RUN_TEST_MEDIUM(DiscoverAndRunTestsWithOutput) 834RUN_TEST_MEDIUM(DiscoverAndRunTestsWithSyslogOutput) 835END_TEST_CASE(DiscoverAndRunTests) 836} // namespace 837} // namespace runtests 838