1// Copyright 2018 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <runtests-utils/runtests-utils.h>
6
7#include <ctype.h>
8#include <dirent.h>
9#include <errno.h>
10#include <glob.h>
11#include <inttypes.h>
12#include <fcntl.h>
13#include <libgen.h>
14#include <limits.h>
15#include <stdio.h>
16#include <stdlib.h>
17#include <string.h>
18#include <sys/stat.h>
19#include <unistd.h>
20
21#include <fbl/auto_call.h>
22#include <fbl/string.h>
23#include <fbl/string_buffer.h>
24#include <fbl/string_piece.h>
25#include <fbl/string_printf.h>
26#include <fbl/unique_ptr.h>
27#include <fbl/vector.h>
28
29namespace runtests {
30
31void ParseTestNames(const fbl::StringPiece input, fbl::Vector<fbl::String>* output) {
32    // strsep modifies its input, so we have to make a mutable copy.
33    // +1 because StringPiece::size() excludes null terminator.
34    fbl::unique_ptr<char[]> input_copy(new char[input.size() + 1]);
35    memcpy(input_copy.get(), input.data(), input.size());
36    input_copy[input.size()] = '\0';
37
38    // Tokenize the input string into names.
39    char* next_token;
40    for (char* tmp = strtok_r(input_copy.get(), ",", &next_token); tmp != nullptr;
41         tmp = strtok_r(nullptr, ",", &next_token)) {
42        output->push_back(fbl::String(tmp));
43    }
44}
45
46bool IsInWhitelist(const fbl::StringPiece name, const fbl::Vector<fbl::String>& whitelist) {
47    for (const fbl::String& whitelist_entry : whitelist) {
48        if (name == fbl::StringPiece(whitelist_entry)) {
49            return true;
50        }
51    }
52    return false;
53}
54
55int MkDirAll(const fbl::StringPiece dir_name) {
56    fbl::StringBuffer<PATH_MAX> dir_buf;
57    if (dir_name.length() > dir_buf.capacity()) {
58        return ENAMETOOLONG;
59    }
60    dir_buf.Append(dir_name);
61    char* dir = dir_buf.data();
62
63    // Fast path: check if the directory already exists.
64    struct stat s;
65    if (!stat(dir, &s)) {
66        return 0;
67    }
68
69    // Slow path: create the directory and its parents.
70    for (size_t slash = 0u; dir[slash]; slash++) {
71        if (slash != 0u && dir[slash] == '/') {
72            dir[slash] = '\0';
73            if (mkdir(dir, 0755) && errno != EEXIST) {
74                return false;
75            }
76            dir[slash] = '/';
77        }
78    }
79    if (mkdir(dir, 0755) && errno != EEXIST) {
80        return errno;
81    }
82    return 0;
83}
84
85fbl::String JoinPath(const fbl::StringPiece parent, const fbl::StringPiece child) {
86    if (parent.empty()) {
87        return fbl::String(child);
88    }
89    if (child.empty()) {
90        return fbl::String(parent);
91    }
92    if (parent[parent.size() - 1] != '/' && child[0] != '/') {
93        return fbl::String::Concat({parent, "/", child});
94    }
95    if (parent[parent.size() - 1] == '/' && child[0] == '/') {
96        return fbl::String::Concat({parent, &child[1]});
97    }
98    return fbl::String::Concat({parent, child});
99}
100
101int WriteSummaryJSON(const fbl::Vector<fbl::unique_ptr<Result>>& results,
102                     const fbl::StringPiece output_file_basename,
103                     const fbl::StringPiece syslog_path,
104                     FILE* summary_json) {
105    int test_count = 0;
106    fprintf(summary_json, "{\n  \"tests\": [\n");
107    for (const fbl::unique_ptr<Result>& result : results) {
108        if (test_count != 0) {
109            fprintf(summary_json, ",\n");
110        }
111        fprintf(summary_json, "    {\n");
112
113        // Write the name of the test.
114        fprintf(summary_json, "      \"name\": \"%s\",\n", result->name.c_str());
115
116        // Write the path to the output file, relative to the test output root
117        // (i.e. what's passed in via -o). The test name is already a path to
118        // the test binary on the target, so to make this a relative path, we
119        // only have to skip leading '/' characters in the test name.
120        fbl::String output_file = runtests::JoinPath(result->name, output_file_basename);
121        size_t i = strspn(output_file.c_str(), "/");
122        if (i == output_file.size()) {
123            fprintf(stderr, "Error: output_file was empty or all slashes: %s\n",
124                    output_file.c_str());
125            return EINVAL;
126        }
127        fprintf(summary_json, "      \"output_file\": \"%s\",\n", &(output_file.c_str()[i]));
128
129        // Write the result of the test, which is either PASS or FAIL. We only
130        // have one PASS condition in TestResult, which is SUCCESS.
131        fprintf(summary_json, "      \"result\": \"%s\"",
132                result->launch_status == runtests::SUCCESS ? "PASS" : "FAIL");
133
134        // Write all data sinks.
135        if (result->data_sinks.size()) {
136            fprintf(summary_json, ",\n      \"data_sinks\": {\n");
137            int sink_count = 0;
138            for (const auto& sink : result->data_sinks) {
139                if (sink_count != 0) {
140                    fprintf(summary_json, ",\n");
141                }
142                fprintf(summary_json, "        \"%s\": [\n", sink.name.c_str());
143                int file_count = 0;
144                for (const auto& file : sink.files) {
145                    if (file_count != 0) {
146                        fprintf(summary_json, ",\n");
147                    }
148                    fprintf(summary_json, "          {\n"
149                                          "            \"name\": \"%s\",\n"
150                                          "            \"file\": \"%s\"\n"
151                                          "          }",
152                            file.name.c_str(), file.file.c_str());
153                    file_count++;
154                }
155                fprintf(summary_json, "\n        ]");
156                sink_count++;
157            }
158            fprintf(summary_json, "\n      }\n");
159        } else {
160          fprintf(summary_json, "\n");
161        }
162        fprintf(summary_json, "    }");
163        test_count++;
164    }
165    fprintf(summary_json, "\n  ]");
166    if (!syslog_path.empty()) {
167        fprintf(summary_json, ",\n"
168                              "  \"outputs\": {\n"
169                              "    \"syslog_file\": \"%.*s\"\n"
170                              "  }\n",
171                static_cast<int>(syslog_path.length()), syslog_path.data());
172    } else {
173        fprintf(summary_json, "\n");
174    }
175    fprintf(summary_json, "}\n");
176    return 0;
177}
178
179int ResolveGlobs(const fbl::Vector<fbl::String>& globs,
180                 fbl::Vector<fbl::String>* resolved) {
181    glob_t resolved_glob;
182    auto auto_call_glob_free = fbl::MakeAutoCall([&resolved_glob] { globfree(&resolved_glob); });
183    int flags = 0;
184    for (const auto& test_dir_glob : globs) {
185        int err = glob(test_dir_glob.c_str(), flags, nullptr, &resolved_glob);
186
187        // Ignore a lack of matches.
188        if (err && err != GLOB_NOMATCH) {
189            return err;
190        }
191        flags = GLOB_APPEND;
192    }
193    resolved->reserve(resolved_glob.gl_pathc);
194    for (size_t i = 0; i < resolved_glob.gl_pathc; ++i) {
195        resolved->push_back(fbl::String(resolved_glob.gl_pathv[i]));
196    }
197    return 0;
198}
199
200int DiscoverTestsInDirGlobs(const fbl::Vector<fbl::String>& dir_globs,
201                            const char* ignore_dir_name,
202                            const fbl::Vector<fbl::String>& basename_whitelist,
203                            fbl::Vector<fbl::String>* test_paths) {
204    fbl::Vector<fbl::String> test_dirs;
205    const int err = ResolveGlobs(dir_globs, &test_dirs);
206    if (err) {
207        fprintf(stderr, "Error: Failed to resolve globs, error = %d\n", err);
208        return EIO; // glob()'s return values aren't the same as errno. This is somewhat arbitrary.
209    }
210    for (const fbl::String& test_dir : test_dirs) {
211        // In the event of failures around a directory not existing or being an empty node
212        // we will continue to the next entries rather than aborting. This allows us to handle
213        // different sets of default test directories.
214        struct stat st;
215        if (stat(test_dir.c_str(), &st) < 0) {
216            printf("Could not stat %s, skipping...\n", test_dir.c_str());
217            continue;
218        }
219        if (!S_ISDIR(st.st_mode)) {
220            // Silently skip non-directories, as they may have been picked up in
221            // the glob.
222            continue;
223        }
224
225        // Resolve an absolute path to the test directory to ensure output
226        // directory names will never collide.
227        char abs_test_dir[PATH_MAX];
228        if (realpath(test_dir.c_str(), abs_test_dir) == nullptr) {
229            printf("Error: Could not resolve path %s: %s\n", test_dir.c_str(), strerror(errno));
230            continue;
231        }
232
233        // Silently skip |ignore_dir_name|.
234        // The user may have done something like runtests /foo/bar/h*.
235        const auto test_dir_base = basename(abs_test_dir);
236        if (ignore_dir_name && strcmp(test_dir_base, ignore_dir_name) == 0) {
237            continue;
238        }
239
240        DIR* dir = opendir(abs_test_dir);
241        if (dir == nullptr) {
242            fprintf(stderr, "Error: Could not open test dir %s\n", abs_test_dir);
243            return errno;
244        }
245
246        struct dirent* de;
247        while ((de = readdir(dir)) != nullptr) {
248            const char* test_name = de->d_name;
249            if (!basename_whitelist.is_empty() &&
250                !runtests::IsInWhitelist(test_name, basename_whitelist)) {
251                continue;
252            }
253
254            const fbl::String test_path = runtests::JoinPath(abs_test_dir, test_name);
255            if (stat(test_path.c_str(), &st) != 0 || !S_ISREG(st.st_mode)) {
256                continue;
257            }
258            test_paths->push_back(test_path);
259        }
260        closedir(dir);
261    }
262    return 0;
263}
264
265int DiscoverTestsInListFile(FILE* test_list_file, fbl::Vector<fbl::String>* test_paths) {
266    char* line = nullptr;
267    size_t line_capacity = 0;
268    auto free_line = fbl::MakeAutoCall([&line]() {
269        free(line);
270    });
271    while (true) {
272        ssize_t line_length = getline(&line, &line_capacity, test_list_file);
273        if (line_length < 0) {
274            if (feof(test_list_file)) {
275                break;
276            }
277            return errno;
278        }
279        // Don't include trailing space.
280        while (line_length && isspace(line[line_length - 1])) {
281            line_length -= 1;
282        }
283        if (!line_length) {
284            continue;
285        }
286        line[line_length] = '\0';
287        test_paths->push_back(line);
288    }
289    return 0;
290}
291
292bool RunTests(const RunTestFn& RunTest, const fbl::Vector<fbl::String>& test_paths,
293              const char* output_dir,
294              const fbl::StringPiece output_file_basename, signed char verbosity, int* failed_count,
295              fbl::Vector<fbl::unique_ptr<Result>>* results) {
296    for (const fbl::String& test_path : test_paths) {
297        fbl::String output_dir_for_test_str;
298        fbl::String output_filename_str;
299        // Ensure the output directory for this test binary's output exists.
300        if (output_dir != nullptr) {
301            // If output_dir was specified, ask |RunTest| to redirect stdout/stderr
302            // to a file whose name is based on the test name.
303            output_dir_for_test_str = runtests::JoinPath(output_dir, test_path);
304            const int error = runtests::MkDirAll(output_dir_for_test_str);
305            if (error) {
306                fprintf(stderr, "Error: Could not create output directory %s: %s\n",
307                        output_dir_for_test_str.c_str(), strerror(error));
308                return false;
309            }
310            output_filename_str = JoinPath(output_dir_for_test_str, output_file_basename);
311        }
312
313        // Assemble test binary args.
314        fbl::Vector<const char*> argv;
315        argv.push_back(test_path.c_str());
316        fbl::String verbosity_arg;
317        if (verbosity >= 0) {
318            // verbosity defaults to -1: "unspecified". Only pass it along
319            // if it was specified: i.e., non-negative.
320            verbosity_arg = fbl::StringPrintf("v=%d", verbosity);
321            argv.push_back(verbosity_arg.c_str());
322        }
323        argv.push_back(nullptr); // Important, since there's no argc.
324        const char* output_dir_for_test =
325            output_dir_for_test_str.empty() ? nullptr : output_dir_for_test_str.c_str();
326        const char* output_filename =
327            output_filename_str.empty() ? nullptr : output_filename_str.c_str();
328
329        // Execute the test binary.
330        printf("\n------------------------------------------------\n"
331               "RUNNING TEST: %s\n\n",
332               test_path.c_str());
333        fbl::unique_ptr<Result> result = RunTest(argv.get(), output_dir_for_test,
334                                                 output_filename);
335        if (result->launch_status != SUCCESS) {
336            *failed_count += 1;
337        }
338        results->push_back(fbl::move(result));
339    }
340    return true;
341}
342
343} // namespace runtests
344