1// Copyright 2018 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <dirent.h>
6#include <errno.h>
7#include <fcntl.h>
8#include <libgen.h>
9#include <limits.h>
10#include <stdio.h>
11#include <stdlib.h>
12#include <sys/stat.h>
13#include <unistd.h>
14
15#include <fbl/auto_call.h>
16#include <fbl/string_buffer.h>
17#include <fbl/unique_fd.h>
18#include <fbl/unique_ptr.h>
19#include <fbl/vector.h>
20#include <runtests-utils/runtests-utils.h>
21#include <unittest/unittest.h>
22
23#include "runtests-utils-test-globals.h"
24#include "runtests-utils-test-utils.h"
25
26
27namespace runtests {
28namespace {
29
30static constexpr char kEchoSuccessAndArgs[] = "echo Success! $@";
31static constexpr char kEchoFailureAndArgs[] = "echo Failure!  $@ 1>&2\nexit 77";
32
33bool ParseTestNamesEmptyStr() {
34    BEGIN_TEST;
35
36    fbl::String input("");
37    fbl::Vector<fbl::String> parsed;
38    ParseTestNames(input, &parsed);
39    EXPECT_EQ(0, parsed.size());
40
41    END_TEST;
42}
43
44bool ParseTestNamesEmptyStrInMiddle() {
45    BEGIN_TEST;
46
47    fbl::String input("a,,b");
48    fbl::Vector<fbl::String> parsed;
49    ParseTestNames(input, &parsed);
50    ASSERT_EQ(2, parsed.size());
51    EXPECT_STR_EQ("a", parsed[0].c_str());
52    EXPECT_STR_EQ("b", parsed[1].c_str());
53
54    END_TEST;
55}
56
57bool ParseTestNamesTrailingComma() {
58    BEGIN_TEST;
59
60    fbl::String input("a,");
61    fbl::Vector<fbl::String> parsed;
62    ParseTestNames(input, &parsed);
63    ASSERT_EQ(1, parsed.size());
64    EXPECT_STR_EQ("a", parsed[0].c_str());
65
66    END_TEST;
67}
68
69bool ParseTestNamesNormal() {
70    BEGIN_TEST;
71
72    fbl::String input("a,b");
73    fbl::Vector<fbl::String> parsed;
74    ParseTestNames(input, &parsed);
75    ASSERT_EQ(2, parsed.size());
76    EXPECT_STR_EQ("a", parsed[0].c_str());
77    EXPECT_STR_EQ("b", parsed[1].c_str());
78
79    END_TEST;
80}
81
82bool EmptyWhitelist() {
83    BEGIN_TEST;
84
85    fbl::Vector<fbl::String> whitelist;
86    EXPECT_FALSE(IsInWhitelist("a", whitelist));
87
88    END_TEST;
89}
90
91bool NonemptyWhitelist() {
92    BEGIN_TEST;
93
94    fbl::Vector<fbl::String> whitelist = {"b", "a"};
95    EXPECT_TRUE(IsInWhitelist("a", whitelist));
96
97    END_TEST;
98}
99
100bool JoinPathNoTrailingSlash() {
101    BEGIN_TEST;
102
103    EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b", "c/d").c_str());
104
105    END_TEST;
106}
107
108bool JoinPathTrailingSlash() {
109    BEGIN_TEST;
110
111    EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b/", "c/d").c_str());
112
113    END_TEST;
114}
115
116bool JoinPathAbsoluteChild() {
117    BEGIN_TEST;
118
119    EXPECT_STR_EQ("a/b/c/d", JoinPath("a/b/", "/c/d").c_str());
120
121    END_TEST;
122}
123
124bool MkDirAllTooLong() {
125    BEGIN_TEST;
126
127    char too_long[PATH_MAX + 2];
128    memset(too_long, 'a', PATH_MAX + 1);
129    too_long[PATH_MAX + 1] = '\0';
130    EXPECT_EQ(ENAMETOOLONG, MkDirAll(too_long));
131
132    END_TEST;
133}
134bool MkDirAllAlreadyExists() {
135    BEGIN_TEST;
136
137    ScopedTestDir test_dir;
138    const fbl::String already = JoinPath(test_dir.path(), "already");
139    const fbl::String exists = JoinPath(already, "exists");
140    ASSERT_EQ(0, mkdir(already.c_str(), 0755));
141    ASSERT_EQ(0, mkdir(exists.c_str(), 0755));
142    EXPECT_EQ(0, MkDirAll(exists));
143
144    END_TEST;
145}
146bool MkDirAllParentAlreadyExists() {
147    BEGIN_TEST;
148
149    ScopedTestDir test_dir;
150    const fbl::String parent = JoinPath(test_dir.path(), "existing-parent");
151    const fbl::String child = JoinPath(parent, "child");
152    ASSERT_EQ(0, mkdir(parent.c_str(), 0755));
153    EXPECT_EQ(0, MkDirAll(child));
154    struct stat s;
155    EXPECT_EQ(0, stat(child.c_str(), &s));
156
157    END_TEST;
158}
159bool MkDirAllParentDoesNotExist() {
160    BEGIN_TEST;
161
162    ScopedTestDir test_dir;
163    const fbl::String parent = JoinPath(test_dir.path(), "not-existing-parent");
164    const fbl::String child = JoinPath(parent, "child");
165    struct stat s;
166    ASSERT_NE(0, stat(parent.c_str(), &s));
167    EXPECT_EQ(0, MkDirAll(child));
168    EXPECT_EQ(0, stat(child.c_str(), &s));
169
170    END_TEST;
171}
172
173bool WriteSummaryJSONSucceeds() {
174    BEGIN_TEST;
175
176    // TODO(IN-499): Use fmemopen instead of tmpfile.
177    FILE* output_file = tmpfile();
178    ASSERT_NONNULL(output_file);
179    fbl::Vector<fbl::unique_ptr<Result>> results;
180    results.push_back(fbl::make_unique<Result>("/a", SUCCESS, 0));
181    results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0));
182    ASSERT_EQ(0, WriteSummaryJSON(results, "output.txt", "/tmp/file_path",
183                                  output_file));
184    // We don't have a JSON parser in zircon right now, so just hard-code the
185    // expected output.
186    const char kExpectedJSONOutput[] = R"({
187  "tests": [
188    {
189      "name": "/a",
190      "output_file": "a/output.txt",
191      "result": "PASS"
192    },
193    {
194      "name": "b",
195      "output_file": "b/output.txt",
196      "result": "FAIL"
197    }
198  ],
199  "outputs": {
200    "syslog_file": "/tmp/file_path"
201  }
202}
203)";
204    EXPECT_TRUE(CompareFileContents(output_file, kExpectedJSONOutput));
205    fclose(output_file);
206
207    END_TEST;
208}
209
210bool WriteSummaryJSONSucceedsWithoutSyslogPath() {
211    BEGIN_TEST;
212
213    // TODO(IN-499): Use fmemopen instead of tmpfile.
214    FILE* output_file = tmpfile();
215    ASSERT_NONNULL(output_file);
216    fbl::Vector<fbl::unique_ptr<Result>> results;
217    results.push_back(fbl::make_unique<Result>("/a", SUCCESS, 0));
218    results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0));
219    ASSERT_EQ(0, WriteSummaryJSON(results, "output.txt", /*syslog_path=*/"",
220                                  output_file));
221    // With an empty syslog_path, we expect no values under "outputs" and
222    // "syslog_file" to be generated in the JSON output.
223    const char kExpectedJSONOutput[] = R"({
224  "tests": [
225    {
226      "name": "/a",
227      "output_file": "a/output.txt",
228      "result": "PASS"
229    },
230    {
231      "name": "b",
232      "output_file": "b/output.txt",
233      "result": "FAIL"
234    }
235  ]
236}
237)";
238
239    EXPECT_TRUE(CompareFileContents(output_file, kExpectedJSONOutput));
240    fclose(output_file);
241
242    END_TEST;
243}
244
245bool WriteSummaryJSONBadTestName() {
246    BEGIN_TEST;
247
248    // TODO(IN-499): Use fmemopen instead of tmpfile.
249    FILE* output_file = tmpfile();
250    ASSERT_NONNULL(output_file);
251    // A test name and output file consisting entirely of slashes should trigger
252    // an error.
253    fbl::Vector<fbl::unique_ptr<Result>> results;
254    results.push_back(fbl::make_unique<Result>("///", SUCCESS, 0));
255    results.push_back(fbl::make_unique<Result>("b", FAILED_TO_LAUNCH, 0));
256    ASSERT_NE(0, WriteSummaryJSON(results, /*output_file_basename=*/"///",
257                                  /*syslog_path=*/"/", output_file));
258    fclose(output_file);
259
260    END_TEST;
261}
262
263bool ResolveGlobsNoMatches() {
264    BEGIN_TEST;
265
266    ScopedTestDir test_dir;
267    fbl::Vector<fbl::String> resolved;
268    fbl::String test_fs_glob = JoinPath(test_dir.path(), "bar*");
269    const fbl::Vector<fbl::String> globs = {"/foo/bar/*", test_fs_glob};
270    ASSERT_EQ(0, ResolveGlobs(globs, &resolved));
271    EXPECT_EQ(0, resolved.size());
272
273    END_TEST;
274}
275
276bool ResolveGlobsMultipleMatches() {
277    BEGIN_TEST;
278
279    ScopedTestDir test_dir;
280    fbl::String existing_dir_path =
281        JoinPath(test_dir.path(), "existing-dir/prefix-suffix");
282    fbl::String existing_file_path = JoinPath(test_dir.path(), "existing-file");
283    fbl::String existing_dir_glob =
284        JoinPath(test_dir.path(), "existing-dir/prefix*");
285    const fbl::Vector<fbl::String> globs = {
286        "/does/not/exist/*",
287        existing_dir_glob, // matches existing_dir_path.
288        existing_file_path};
289    ASSERT_EQ(0, MkDirAll(existing_dir_path));
290    const int existing_file_fd = open(existing_file_path.c_str(), O_CREAT);
291    ASSERT_NE(-1, existing_file_fd, strerror(errno));
292    ASSERT_NE(-1, close(existing_file_fd), strerror(errno));
293    fbl::Vector<fbl::String> resolved;
294    ASSERT_EQ(0, ResolveGlobs(globs, &resolved));
295    ASSERT_EQ(2, resolved.size());
296    EXPECT_STR_EQ(existing_dir_path.c_str(), resolved[0].c_str());
297
298    END_TEST;
299}
300
301bool RunTestSuccess() {
302    BEGIN_TEST;
303
304    ScopedTestDir test_dir;
305    fbl::String test_name = JoinPath(test_dir.path(), "succeed.sh");
306    const char* argv[] = {test_name.c_str(), nullptr};
307    ScopedScriptFile script(argv[0], "exit 0");
308    fbl::unique_ptr<Result> result = PlatformRunTest(argv, nullptr, nullptr);
309    EXPECT_STR_EQ(argv[0], result->name.c_str());
310    EXPECT_EQ(SUCCESS, result->launch_status);
311    EXPECT_EQ(0, result->return_code);
312
313    END_TEST;
314}
315
316bool RunTestSuccessWithStdout() {
317    BEGIN_TEST;
318
319    ScopedTestDir test_dir;
320    fbl::String test_name = JoinPath(test_dir.path(), "succeed.sh");
321    const char* argv[] = {test_name.c_str(), nullptr};
322    const char expected_output[] = "Expect this!\n";
323    // Produces expected_output, b/c echo adds newline
324    const char script_contents[] = "echo Expect this!";
325    ScopedScriptFile script(argv[0], script_contents);
326
327    fbl::String output_filename = JoinPath(test_dir.path(), "test.out");
328    fbl::unique_ptr<Result> result =
329        PlatformRunTest(argv, nullptr, output_filename.c_str());
330
331    FILE* output_file = fopen(output_filename.c_str(), "r");
332    ASSERT_TRUE(output_file);
333    char buf[1024];
334    memset(buf, 0, sizeof(buf));
335    EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
336    fclose(output_file);
337    EXPECT_STR_EQ(expected_output, buf);
338    EXPECT_STR_EQ(argv[0], result->name.c_str());
339    EXPECT_EQ(SUCCESS, result->launch_status);
340    EXPECT_EQ(0, result->return_code);
341
342    END_TEST;
343}
344
345bool RunTestFailureWithStderr() {
346    BEGIN_TEST;
347
348    ScopedTestDir test_dir;
349    fbl::String test_name = JoinPath(test_dir.path(), "fail.sh");
350    const char* argv[] = {test_name.c_str(), nullptr};
351    const char expected_output[] = "Expect this!\n";
352    // Produces expected_output, b/c echo adds newline
353    const char script_contents[] = "echo Expect this! 1>&2\nexit 77";
354    ScopedScriptFile script(argv[0], script_contents);
355
356    fbl::String output_filename = JoinPath(test_dir.path(), "test.out");
357    fbl::unique_ptr<Result> result =
358        PlatformRunTest(argv, nullptr, output_filename.c_str());
359
360    FILE* output_file = fopen(output_filename.c_str(), "r");
361    ASSERT_TRUE(output_file);
362    char buf[1024];
363    memset(buf, 0, sizeof(buf));
364    EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
365    fclose(output_file);
366    EXPECT_STR_EQ(expected_output, buf);
367    EXPECT_STR_EQ(argv[0], result->name.c_str());
368    EXPECT_EQ(FAILED_NONZERO_RETURN_CODE, result->launch_status);
369    EXPECT_EQ(77, result->return_code);
370
371    END_TEST;
372}
373
374bool RunTestFailureToLoadFile() {
375    BEGIN_TEST;
376
377    const char* argv[] = {"i/do/not/exist/", nullptr};
378
379    fbl::unique_ptr<Result> result = PlatformRunTest(argv, nullptr, nullptr);
380    EXPECT_STR_EQ(argv[0], result->name.c_str());
381    EXPECT_EQ(FAILED_TO_LAUNCH, result->launch_status);
382
383    END_TEST;
384}
385
386bool DiscoverTestsInDirGlobsBasic() {
387    BEGIN_TEST;
388
389    ScopedTestDir test_dir;
390    const fbl::String a_file_name = JoinPath(test_dir.path(), "a.sh");
391    ScopedScriptFile a_file(a_file_name, "");
392    const fbl::String b_file_name = JoinPath(test_dir.path(), "b.sh");
393    ScopedScriptFile b_file(b_file_name, "");
394    fbl::Vector<fbl::String> discovered_paths;
395    EXPECT_EQ(0, DiscoverTestsInDirGlobs({test_dir.path()}, nullptr, {},
396                                         &discovered_paths));
397    EXPECT_EQ(2, discovered_paths.size());
398    bool discovered_a = false;
399    bool discovered_b = false;
400    // The order of the results is not defined, so just check that each is
401    // present.
402    for (const auto& path : discovered_paths) {
403        if (fbl::StringPiece(path) == a_file.path()) {
404            discovered_a = true;
405        } else if (fbl::StringPiece(path) == b_file.path()) {
406            discovered_b = true;
407        }
408    }
409    EXPECT_TRUE(discovered_a);
410    EXPECT_TRUE(discovered_b);
411
412    END_TEST;
413}
414
415bool DiscoverTestsInDirGlobsFilter() {
416    BEGIN_TEST;
417
418    ScopedTestDir test_dir;
419    const char kHopefullyUniqueFileBasename[] =
420        "e829cea9919fe045ca199945db7ac99a";
421    const fbl::String unique_file_name =
422        JoinPath(test_dir.path(), kHopefullyUniqueFileBasename);
423    ScopedScriptFile unique_file(unique_file_name, "");
424    // This one should be ignored because its basename is not in the white list.
425    const fbl::String other_file_name = JoinPath(test_dir.path(), "foo.sh");
426    ScopedScriptFile fail_file(other_file_name, "");
427    fbl::Vector<fbl::String> discovered_paths;
428    EXPECT_EQ(0, DiscoverTestsInDirGlobs({JoinPath(TestFsRoot(), "*")}, nullptr,
429                                         {kHopefullyUniqueFileBasename},
430                                         &discovered_paths));
431    EXPECT_EQ(1, discovered_paths.size());
432    EXPECT_STR_EQ(unique_file_name.c_str(), discovered_paths[0].c_str());
433
434    END_TEST;
435}
436
437bool DiscoverTestsInDirGlobsIgnore() {
438    BEGIN_TEST;
439    ScopedTestDir test_dir_a, test_dir_b;
440    const fbl::String a_name = JoinPath(test_dir_a.path(), "foo.sh");
441    ScopedScriptFile a_file(a_name, "");
442    const fbl::String b_name = JoinPath(test_dir_b.path(), "foo.sh");
443    ScopedScriptFile fail_file(b_name, "");
444    fbl::Vector<fbl::String> discovered_paths;
445    EXPECT_EQ(0, DiscoverTestsInDirGlobs({test_dir_a.path(), test_dir_b.path()},
446                                         test_dir_b.basename(), {},
447                                         &discovered_paths));
448    EXPECT_EQ(1, discovered_paths.size());
449    EXPECT_STR_EQ(a_name.c_str(), discovered_paths[0].c_str());
450    END_TEST;
451}
452
453bool DiscoverTestsInListFileWithTrailingWhitespace() {
454    BEGIN_TEST;
455    // TODO(IN-499): Use fmemopen instead of tmpfile.
456    FILE* test_list_file = tmpfile();
457    ASSERT_NONNULL(test_list_file);
458    fprintf(test_list_file, "trailing/tab\t\n");
459    fprintf(test_list_file, "trailing/space \n");
460    fprintf(test_list_file, "trailing/return\r");
461    rewind(test_list_file);
462    fbl::Vector<fbl::String> test_paths;
463    EXPECT_EQ(0, DiscoverTestsInListFile(test_list_file, &test_paths));
464    EXPECT_EQ(3, test_paths.size());
465    EXPECT_STR_EQ("trailing/tab", test_paths[0].c_str());
466    EXPECT_STR_EQ("trailing/space", test_paths[1].c_str());
467    EXPECT_STR_EQ("trailing/return", test_paths[2].c_str());
468    fclose(test_list_file);
469    END_TEST;
470}
471
472bool RunTestsWithVerbosity() {
473    BEGIN_TEST;
474
475    ScopedTestDir test_dir;
476    const fbl::String succeed_file_name =
477        JoinPath(test_dir.path(), "succeed.sh");
478    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
479    int num_failed = 0;
480    fbl::Vector<fbl::unique_ptr<Result>> results;
481    const signed char verbosity = 77;
482    const fbl::String output_dir = JoinPath(test_dir.path(), "output");
483    const char output_file_base_name[] = "output.txt";
484    ASSERT_EQ(0, MkDirAll(output_dir));
485    EXPECT_TRUE(RunTests(PlatformRunTest, {succeed_file_name},
486                         output_dir.c_str(), output_file_base_name, verbosity,
487                         &num_failed, &results));
488    EXPECT_EQ(0, num_failed);
489    EXPECT_EQ(1, results.size());
490
491    fbl::String output_path = JoinPath(
492        JoinPath(output_dir, succeed_file.path()), output_file_base_name);
493    FILE* output_file = fopen(output_path.c_str(), "r");
494    ASSERT_TRUE(output_file);
495    char buf[1024];
496    memset(buf, 0, sizeof(buf));
497    EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
498    fclose(output_file);
499    EXPECT_STR_EQ("Success! v=77\n", buf);
500
501    END_TEST;
502}
503
504bool DiscoverAndRunTestsBasicPass() {
505    BEGIN_TEST;
506
507    ScopedTestDir test_dir;
508    const fbl::String succeed_file_name1 =
509        JoinPath(test_dir.path(), "succeed1.sh");
510    ScopedScriptFile succeed_file1(succeed_file_name1, kEchoSuccessAndArgs);
511    const fbl::String succeed_file_name2 =
512        JoinPath(test_dir.path(), "succeed2.sh");
513    ScopedScriptFile succeed_file2(succeed_file_name2, kEchoSuccessAndArgs);
514    const char* const argv[] = {"./runtests", test_dir.path()};
515    TestStopwatch stopwatch;
516    EXPECT_EQ(EXIT_SUCCESS, DiscoverAndRunTests(PlatformRunTest, 2, argv, {},
517                                                &stopwatch, ""));
518
519    END_TEST;
520}
521
522bool DiscoverAndRunTestsBasicFail() {
523    BEGIN_TEST;
524
525    ScopedTestDir test_dir;
526    const fbl::String succeed_file_name =
527        JoinPath(test_dir.path(), "succeed.sh");
528    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
529    const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh");
530    ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs);
531    const char* const argv[] = {"./runtests", test_dir.path()};
532    TestStopwatch stopwatch;
533    EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 2, argv, {},
534                                                &stopwatch, ""));
535
536    END_TEST;
537}
538
539bool DiscoverAndRunTestsFallsBackToDefaultDirs() {
540    BEGIN_TEST;
541
542    ScopedTestDir test_dir;
543    const fbl::String succeed_file_name =
544        JoinPath(test_dir.path(), "succeed.sh");
545    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
546    const char* const argv[] = {"./runtests"};
547    TestStopwatch stopwatch;
548    EXPECT_EQ(EXIT_SUCCESS,
549              DiscoverAndRunTests(PlatformRunTest, 1, argv, {test_dir.path()},
550                                  &stopwatch, ""));
551
552    END_TEST;
553}
554
555bool DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs() {
556    BEGIN_TEST;
557
558    ScopedTestDir test_dir;
559    const fbl::String succeed_file_name =
560        JoinPath(test_dir.path(), "succeed.sh");
561    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
562    const char* const argv[] = {"./runtests"};
563    TestStopwatch stopwatch;
564    EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 1, argv, {},
565                                                &stopwatch, ""));
566
567    END_TEST;
568}
569
570bool DiscoverAndRunTestsFailsWithBadArgs() {
571    BEGIN_TEST;
572
573    ScopedTestDir test_dir;
574    const fbl::String succeed_file_name =
575        JoinPath(test_dir.path(), "succeed.sh");
576    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
577    const char* const argv[] = {"./runtests", "-?", "unknown-arg",
578                                test_dir.path()};
579    TestStopwatch stopwatch;
580    EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {},
581                                                &stopwatch, ""));
582
583    END_TEST;
584}
585
586bool DiscoverAndRunTestsWithGlobs() {
587    BEGIN_TEST;
588
589    ScopedTestDir test_dir;
590    // Make the directories that the following globs will match.
591    const fbl::String dir1 = JoinPath(test_dir.path(), "A/B/C");
592    EXPECT_EQ(0, MkDirAll(dir1));
593    const fbl::String dir2 = JoinPath(test_dir.path(), "A/D/C");
594    EXPECT_EQ(0, MkDirAll(dir2));
595
596    const fbl::String succeed_file_name1 =
597        JoinPath(test_dir.path(), "succeed.sh");
598    ScopedScriptFile succeed_file1(succeed_file_name1, kEchoSuccessAndArgs);
599    const fbl::String succeed_file_name2 = JoinPath(dir1, "succeed.sh");
600    ScopedScriptFile succeed_file2(succeed_file_name2, kEchoSuccessAndArgs);
601    const fbl::String succeed_file_name3 = JoinPath(dir2, "succeed.sh");
602    ScopedScriptFile succeed_file3(succeed_file_name3, kEchoSuccessAndArgs);
603
604    fbl::String glob = JoinPath(test_dir.path(), "A/*/C");
605    const char* const argv[] = {"./runtests", test_dir.path(), glob.c_str()};
606    TestStopwatch stopwatch;
607    EXPECT_EQ(EXIT_SUCCESS, DiscoverAndRunTests(PlatformRunTest, 3, argv, {},
608                                                &stopwatch, ""));
609
610    END_TEST;
611}
612
613// Passing an -o argument should result in output being written to that
614// location.
615bool DiscoverAndRunTestsWithOutput() {
616    BEGIN_TEST;
617
618    ScopedTestDir test_dir;
619    const fbl::String succeed_file_name =
620        JoinPath(test_dir.path(), "succeed.sh");
621    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
622    const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh");
623    ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs);
624
625    const fbl::String output_dir =
626        JoinPath(test_dir.path(), "run-all-tests-output-1");
627    EXPECT_EQ(0, MkDirAll(output_dir));
628
629    const char* const argv[] = {"./runtests", "-o", output_dir.c_str(),
630                                test_dir.path()};
631    TestStopwatch stopwatch;
632    EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {},
633                                                &stopwatch, ""));
634
635    // Prepare the expected output.
636    fbl::String success_output_rel_path;
637    ASSERT_TRUE(GetOutputFileRelPath(output_dir, succeed_file_name,
638                                     &success_output_rel_path));
639    fbl::String failure_output_rel_path;
640    ASSERT_TRUE(GetOutputFileRelPath(output_dir, fail_file_name,
641                                     &failure_output_rel_path));
642
643    fbl::StringBuffer<1024> expected_pass_output_buf;
644    expected_pass_output_buf.AppendPrintf(
645        "    {\n"
646        "      \"name\": \"%s\",\n"
647        "      \"output_file\": \"%s\",\n"
648        "      \"result\": \"PASS\"\n"
649        "    }",
650        succeed_file_name.c_str(),
651        success_output_rel_path.c_str() +
652            1); // +1 to discard the leading slash.
653    fbl::StringBuffer<1024> expected_fail_output_buf;
654    expected_fail_output_buf.AppendPrintf(
655        "    {\n"
656        "      \"name\": \"%s\",\n"
657        "      \"output_file\": \"%s\",\n"
658        "      \"result\": \"FAIL\"\n"
659        "    }",
660        fail_file_name.c_str(),
661        failure_output_rel_path.c_str() +
662            1); // +1 to discared the leading slash.
663
664    // Extract the actual output.
665    const fbl::String output_path = JoinPath(output_dir, "summary.json");
666    FILE* output_file = fopen(output_path.c_str(), "r");
667    ASSERT_TRUE(output_file);
668    char buf[1024];
669    memset(buf, 0, sizeof(buf));
670    EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
671    fclose(output_file);
672
673    // The order of the tests in summary.json is not defined, so first check the
674    // prefix, then be permissive about order of the actual tests.
675    size_t buf_index = 0;
676    EXPECT_EQ(0, strncmp(kExpectedJSONOutputPrefix, &buf[buf_index],
677                         kExpectedJSONOutputPrefixSize));
678    buf_index += kExpectedJSONOutputPrefixSize;
679
680    if (!strncmp(expected_pass_output_buf.c_str(), &buf[buf_index],
681                 expected_pass_output_buf.size())) {
682        buf_index += expected_pass_output_buf.size();
683        EXPECT_EQ(0, strncmp(",\n", &buf[buf_index], sizeof(",\n") - 1));
684        buf_index += sizeof(",\n") - 1;
685        EXPECT_EQ(0, strncmp(expected_fail_output_buf.c_str(), &buf[buf_index],
686                             expected_fail_output_buf.size()));
687        buf_index += expected_fail_output_buf.size();
688    } else if (!strncmp(expected_fail_output_buf.c_str(), &buf[buf_index],
689                        expected_fail_output_buf.size())) {
690        buf_index += expected_fail_output_buf.size();
691        EXPECT_EQ(0, strncmp(",\n", &buf[buf_index], sizeof(",\n") - 1));
692        buf_index += sizeof(",\n") - 1;
693        EXPECT_EQ(0, strncmp(expected_pass_output_buf.c_str(), &buf[buf_index],
694                             expected_pass_output_buf.size()));
695        buf_index += expected_pass_output_buf.size();
696    } else {
697        printf("Unexpected buffer contents: %s\n", buf);
698        EXPECT_TRUE(false,
699                    "output buf didn't contain expected pass or fail strings");
700    }
701    EXPECT_STR_EQ("\n  ]\n}\n", &buf[buf_index]);
702
703    END_TEST;
704}
705
706// Passing an -o argument *and* a syslog file name should result in output being
707// written that includes a syslog reference.
708bool DiscoverAndRunTestsWithSyslogOutput() {
709    BEGIN_TEST;
710
711    ScopedTestDir test_dir;
712    const fbl::String succeed_file_name =
713        JoinPath(test_dir.path(), "succeed.sh");
714    ScopedScriptFile succeed_file(succeed_file_name, kEchoSuccessAndArgs);
715    const fbl::String fail_file_name = JoinPath(test_dir.path(), "fail.sh");
716    ScopedScriptFile fail_file(fail_file_name, kEchoFailureAndArgs);
717
718    const fbl::String output_dir =
719        JoinPath(test_dir.path(), "run-all-tests-output-2");
720    EXPECT_EQ(0, MkDirAll(output_dir));
721
722    const char* const argv[] = {"./runtests", "-o", output_dir.c_str(),
723                                test_dir.path()};
724    TestStopwatch stopwatch;
725    EXPECT_EQ(EXIT_FAILURE, DiscoverAndRunTests(PlatformRunTest, 4, argv, {},
726                                                &stopwatch, "syslog.txt"));
727
728    // Prepare the expected output.
729    fbl::String success_output_rel_path;
730    ASSERT_TRUE(GetOutputFileRelPath(output_dir, succeed_file_name,
731                                     &success_output_rel_path));
732    fbl::String failure_output_rel_path;
733    ASSERT_TRUE(GetOutputFileRelPath(output_dir, fail_file_name,
734                                     &failure_output_rel_path));
735
736    const char kExpectedOutputsStr[] =
737        "  \"outputs\": {\n"
738        "    \"syslog_file\": \"syslog.txt\"\n"
739        "  }";
740
741    // Extract the actual output.
742    const fbl::String output_path = JoinPath(output_dir, "summary.json");
743    FILE* output_file = fopen(output_path.c_str(), "r");
744    ASSERT_TRUE(output_file);
745    char buf[1024];
746    memset(buf, 0, sizeof(buf));
747    EXPECT_LT(0, fread(buf, sizeof(buf[0]), sizeof(buf), output_file));
748    fclose(output_file);
749
750    // We don't actually care if the string is at the beginning or the end of
751    // the JSON, so just search for it anywhere.
752    bool found_expected_outputs_str = false;
753    for (size_t buf_index = 0; buf[buf_index]; ++buf_index) {
754        if (!strncmp(kExpectedOutputsStr, &buf[buf_index],
755                     sizeof(kExpectedOutputsStr) - 1)) {
756            found_expected_outputs_str = true;
757            break;
758        }
759    }
760    if (!found_expected_outputs_str) {
761        printf("Unexpected buffer contents: %s\n", buf);
762    }
763    EXPECT_TRUE(found_expected_outputs_str,
764                "Didn't find expected outputs str in buf");
765
766    END_TEST;
767}
768
769BEGIN_TEST_CASE(ParseTestNames)
770RUN_TEST(ParseTestNamesEmptyStr)
771RUN_TEST(ParseTestNamesEmptyStrInMiddle)
772RUN_TEST(ParseTestNamesNormal)
773RUN_TEST(ParseTestNamesTrailingComma)
774END_TEST_CASE(ParseTestNames)
775
776BEGIN_TEST_CASE(IsInWhitelist)
777RUN_TEST(EmptyWhitelist)
778RUN_TEST(NonemptyWhitelist)
779END_TEST_CASE(IsInWhitelist)
780
781BEGIN_TEST_CASE(JoinPath)
782RUN_TEST(JoinPathNoTrailingSlash)
783RUN_TEST(JoinPathTrailingSlash)
784RUN_TEST(JoinPathAbsoluteChild)
785END_TEST_CASE(JoinPath)
786
787BEGIN_TEST_CASE(MkDirAll)
788RUN_TEST(MkDirAllTooLong)
789RUN_TEST(MkDirAllAlreadyExists)
790RUN_TEST(MkDirAllParentAlreadyExists)
791RUN_TEST(MkDirAllParentDoesNotExist)
792END_TEST_CASE(MkDirAll)
793
794BEGIN_TEST_CASE(WriteSummaryJSON)
795RUN_TEST_MEDIUM(WriteSummaryJSONSucceeds)
796RUN_TEST_MEDIUM(WriteSummaryJSONSucceedsWithoutSyslogPath)
797RUN_TEST_MEDIUM(WriteSummaryJSONBadTestName)
798END_TEST_CASE(WriteSummaryJSON)
799
800BEGIN_TEST_CASE(ResolveGlobs)
801RUN_TEST(ResolveGlobsNoMatches)
802RUN_TEST(ResolveGlobsMultipleMatches)
803END_TEST_CASE(ResolveGlobs)
804
805BEGIN_TEST_CASE(RunTest)
806RUN_TEST(RunTestSuccess)
807RUN_TEST(RunTestSuccessWithStdout)
808RUN_TEST(RunTestFailureWithStderr)
809RUN_TEST(RunTestFailureToLoadFile)
810END_TEST_CASE(RunTest)
811
812BEGIN_TEST_CASE(DiscoverTestsInDirGlobs)
813RUN_TEST(DiscoverTestsInDirGlobsBasic)
814RUN_TEST(DiscoverTestsInDirGlobsFilter)
815RUN_TEST(DiscoverTestsInDirGlobsIgnore)
816END_TEST_CASE(DiscoverTestsInDirGlobs)
817
818BEGIN_TEST_CASE(DiscoverTestsInListFile)
819RUN_TEST(DiscoverTestsInListFileWithTrailingWhitespace)
820END_TEST_CASE(DiscoverTestsInListFile)
821
822BEGIN_TEST_CASE(RunTests)
823RUN_TEST_MEDIUM(RunTestsWithVerbosity)
824END_TEST_CASE(RunTests)
825
826BEGIN_TEST_CASE(DiscoverAndRunTests)
827RUN_TEST_MEDIUM(DiscoverAndRunTestsBasicPass)
828RUN_TEST_MEDIUM(DiscoverAndRunTestsBasicFail)
829RUN_TEST_MEDIUM(DiscoverAndRunTestsFallsBackToDefaultDirs)
830RUN_TEST_MEDIUM(DiscoverAndRunTestsFailsWithNoTestGlobsOrDefaultDirs)
831RUN_TEST_MEDIUM(DiscoverAndRunTestsFailsWithBadArgs)
832RUN_TEST_MEDIUM(DiscoverAndRunTestsWithGlobs)
833RUN_TEST_MEDIUM(DiscoverAndRunTestsWithOutput)
834RUN_TEST_MEDIUM(DiscoverAndRunTestsWithSyslogOutput)
835END_TEST_CASE(DiscoverAndRunTests)
836} // namespace
837} // namespace runtests
838