1// Copyright 2018 The Fuchsia Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <runtests-utils/runtests-utils.h> 6 7#include <ctype.h> 8#include <dirent.h> 9#include <errno.h> 10#include <glob.h> 11#include <inttypes.h> 12#include <fcntl.h> 13#include <libgen.h> 14#include <limits.h> 15#include <stdio.h> 16#include <stdlib.h> 17#include <string.h> 18#include <sys/stat.h> 19#include <unistd.h> 20 21#include <fbl/auto_call.h> 22#include <fbl/string.h> 23#include <fbl/string_buffer.h> 24#include <fbl/string_piece.h> 25#include <fbl/string_printf.h> 26#include <fbl/unique_ptr.h> 27#include <fbl/vector.h> 28 29namespace runtests { 30 31void ParseTestNames(const fbl::StringPiece input, fbl::Vector<fbl::String>* output) { 32 // strsep modifies its input, so we have to make a mutable copy. 33 // +1 because StringPiece::size() excludes null terminator. 34 fbl::unique_ptr<char[]> input_copy(new char[input.size() + 1]); 35 memcpy(input_copy.get(), input.data(), input.size()); 36 input_copy[input.size()] = '\0'; 37 38 // Tokenize the input string into names. 39 char* next_token; 40 for (char* tmp = strtok_r(input_copy.get(), ",", &next_token); tmp != nullptr; 41 tmp = strtok_r(nullptr, ",", &next_token)) { 42 output->push_back(fbl::String(tmp)); 43 } 44} 45 46bool IsInWhitelist(const fbl::StringPiece name, const fbl::Vector<fbl::String>& whitelist) { 47 for (const fbl::String& whitelist_entry : whitelist) { 48 if (name == fbl::StringPiece(whitelist_entry)) { 49 return true; 50 } 51 } 52 return false; 53} 54 55int MkDirAll(const fbl::StringPiece dir_name) { 56 fbl::StringBuffer<PATH_MAX> dir_buf; 57 if (dir_name.length() > dir_buf.capacity()) { 58 return ENAMETOOLONG; 59 } 60 dir_buf.Append(dir_name); 61 char* dir = dir_buf.data(); 62 63 // Fast path: check if the directory already exists. 64 struct stat s; 65 if (!stat(dir, &s)) { 66 return 0; 67 } 68 69 // Slow path: create the directory and its parents. 70 for (size_t slash = 0u; dir[slash]; slash++) { 71 if (slash != 0u && dir[slash] == '/') { 72 dir[slash] = '\0'; 73 if (mkdir(dir, 0755) && errno != EEXIST) { 74 return false; 75 } 76 dir[slash] = '/'; 77 } 78 } 79 if (mkdir(dir, 0755) && errno != EEXIST) { 80 return errno; 81 } 82 return 0; 83} 84 85fbl::String JoinPath(const fbl::StringPiece parent, const fbl::StringPiece child) { 86 if (parent.empty()) { 87 return fbl::String(child); 88 } 89 if (child.empty()) { 90 return fbl::String(parent); 91 } 92 if (parent[parent.size() - 1] != '/' && child[0] != '/') { 93 return fbl::String::Concat({parent, "/", child}); 94 } 95 if (parent[parent.size() - 1] == '/' && child[0] == '/') { 96 return fbl::String::Concat({parent, &child[1]}); 97 } 98 return fbl::String::Concat({parent, child}); 99} 100 101int WriteSummaryJSON(const fbl::Vector<fbl::unique_ptr<Result>>& results, 102 const fbl::StringPiece output_file_basename, 103 const fbl::StringPiece syslog_path, 104 FILE* summary_json) { 105 int test_count = 0; 106 fprintf(summary_json, "{\n \"tests\": [\n"); 107 for (const fbl::unique_ptr<Result>& result : results) { 108 if (test_count != 0) { 109 fprintf(summary_json, ",\n"); 110 } 111 fprintf(summary_json, " {\n"); 112 113 // Write the name of the test. 114 fprintf(summary_json, " \"name\": \"%s\",\n", result->name.c_str()); 115 116 // Write the path to the output file, relative to the test output root 117 // (i.e. what's passed in via -o). The test name is already a path to 118 // the test binary on the target, so to make this a relative path, we 119 // only have to skip leading '/' characters in the test name. 120 fbl::String output_file = runtests::JoinPath(result->name, output_file_basename); 121 size_t i = strspn(output_file.c_str(), "/"); 122 if (i == output_file.size()) { 123 fprintf(stderr, "Error: output_file was empty or all slashes: %s\n", 124 output_file.c_str()); 125 return EINVAL; 126 } 127 fprintf(summary_json, " \"output_file\": \"%s\",\n", &(output_file.c_str()[i])); 128 129 // Write the result of the test, which is either PASS or FAIL. We only 130 // have one PASS condition in TestResult, which is SUCCESS. 131 fprintf(summary_json, " \"result\": \"%s\"", 132 result->launch_status == runtests::SUCCESS ? "PASS" : "FAIL"); 133 134 // Write all data sinks. 135 if (result->data_sinks.size()) { 136 fprintf(summary_json, ",\n \"data_sinks\": {\n"); 137 int sink_count = 0; 138 for (const auto& sink : result->data_sinks) { 139 if (sink_count != 0) { 140 fprintf(summary_json, ",\n"); 141 } 142 fprintf(summary_json, " \"%s\": [\n", sink.name.c_str()); 143 int file_count = 0; 144 for (const auto& file : sink.files) { 145 if (file_count != 0) { 146 fprintf(summary_json, ",\n"); 147 } 148 fprintf(summary_json, " {\n" 149 " \"name\": \"%s\",\n" 150 " \"file\": \"%s\"\n" 151 " }", 152 file.name.c_str(), file.file.c_str()); 153 file_count++; 154 } 155 fprintf(summary_json, "\n ]"); 156 sink_count++; 157 } 158 fprintf(summary_json, "\n }\n"); 159 } else { 160 fprintf(summary_json, "\n"); 161 } 162 fprintf(summary_json, " }"); 163 test_count++; 164 } 165 fprintf(summary_json, "\n ]"); 166 if (!syslog_path.empty()) { 167 fprintf(summary_json, ",\n" 168 " \"outputs\": {\n" 169 " \"syslog_file\": \"%.*s\"\n" 170 " }\n", 171 static_cast<int>(syslog_path.length()), syslog_path.data()); 172 } else { 173 fprintf(summary_json, "\n"); 174 } 175 fprintf(summary_json, "}\n"); 176 return 0; 177} 178 179int ResolveGlobs(const fbl::Vector<fbl::String>& globs, 180 fbl::Vector<fbl::String>* resolved) { 181 glob_t resolved_glob; 182 auto auto_call_glob_free = fbl::MakeAutoCall([&resolved_glob] { globfree(&resolved_glob); }); 183 int flags = 0; 184 for (const auto& test_dir_glob : globs) { 185 int err = glob(test_dir_glob.c_str(), flags, nullptr, &resolved_glob); 186 187 // Ignore a lack of matches. 188 if (err && err != GLOB_NOMATCH) { 189 return err; 190 } 191 flags = GLOB_APPEND; 192 } 193 resolved->reserve(resolved_glob.gl_pathc); 194 for (size_t i = 0; i < resolved_glob.gl_pathc; ++i) { 195 resolved->push_back(fbl::String(resolved_glob.gl_pathv[i])); 196 } 197 return 0; 198} 199 200int DiscoverTestsInDirGlobs(const fbl::Vector<fbl::String>& dir_globs, 201 const char* ignore_dir_name, 202 const fbl::Vector<fbl::String>& basename_whitelist, 203 fbl::Vector<fbl::String>* test_paths) { 204 fbl::Vector<fbl::String> test_dirs; 205 const int err = ResolveGlobs(dir_globs, &test_dirs); 206 if (err) { 207 fprintf(stderr, "Error: Failed to resolve globs, error = %d\n", err); 208 return EIO; // glob()'s return values aren't the same as errno. This is somewhat arbitrary. 209 } 210 for (const fbl::String& test_dir : test_dirs) { 211 // In the event of failures around a directory not existing or being an empty node 212 // we will continue to the next entries rather than aborting. This allows us to handle 213 // different sets of default test directories. 214 struct stat st; 215 if (stat(test_dir.c_str(), &st) < 0) { 216 printf("Could not stat %s, skipping...\n", test_dir.c_str()); 217 continue; 218 } 219 if (!S_ISDIR(st.st_mode)) { 220 // Silently skip non-directories, as they may have been picked up in 221 // the glob. 222 continue; 223 } 224 225 // Resolve an absolute path to the test directory to ensure output 226 // directory names will never collide. 227 char abs_test_dir[PATH_MAX]; 228 if (realpath(test_dir.c_str(), abs_test_dir) == nullptr) { 229 printf("Error: Could not resolve path %s: %s\n", test_dir.c_str(), strerror(errno)); 230 continue; 231 } 232 233 // Silently skip |ignore_dir_name|. 234 // The user may have done something like runtests /foo/bar/h*. 235 const auto test_dir_base = basename(abs_test_dir); 236 if (ignore_dir_name && strcmp(test_dir_base, ignore_dir_name) == 0) { 237 continue; 238 } 239 240 DIR* dir = opendir(abs_test_dir); 241 if (dir == nullptr) { 242 fprintf(stderr, "Error: Could not open test dir %s\n", abs_test_dir); 243 return errno; 244 } 245 246 struct dirent* de; 247 while ((de = readdir(dir)) != nullptr) { 248 const char* test_name = de->d_name; 249 if (!basename_whitelist.is_empty() && 250 !runtests::IsInWhitelist(test_name, basename_whitelist)) { 251 continue; 252 } 253 254 const fbl::String test_path = runtests::JoinPath(abs_test_dir, test_name); 255 if (stat(test_path.c_str(), &st) != 0 || !S_ISREG(st.st_mode)) { 256 continue; 257 } 258 test_paths->push_back(test_path); 259 } 260 closedir(dir); 261 } 262 return 0; 263} 264 265int DiscoverTestsInListFile(FILE* test_list_file, fbl::Vector<fbl::String>* test_paths) { 266 char* line = nullptr; 267 size_t line_capacity = 0; 268 auto free_line = fbl::MakeAutoCall([&line]() { 269 free(line); 270 }); 271 while (true) { 272 ssize_t line_length = getline(&line, &line_capacity, test_list_file); 273 if (line_length < 0) { 274 if (feof(test_list_file)) { 275 break; 276 } 277 return errno; 278 } 279 // Don't include trailing space. 280 while (line_length && isspace(line[line_length - 1])) { 281 line_length -= 1; 282 } 283 if (!line_length) { 284 continue; 285 } 286 line[line_length] = '\0'; 287 test_paths->push_back(line); 288 } 289 return 0; 290} 291 292bool RunTests(const RunTestFn& RunTest, const fbl::Vector<fbl::String>& test_paths, 293 const char* output_dir, 294 const fbl::StringPiece output_file_basename, signed char verbosity, int* failed_count, 295 fbl::Vector<fbl::unique_ptr<Result>>* results) { 296 for (const fbl::String& test_path : test_paths) { 297 fbl::String output_dir_for_test_str; 298 fbl::String output_filename_str; 299 // Ensure the output directory for this test binary's output exists. 300 if (output_dir != nullptr) { 301 // If output_dir was specified, ask |RunTest| to redirect stdout/stderr 302 // to a file whose name is based on the test name. 303 output_dir_for_test_str = runtests::JoinPath(output_dir, test_path); 304 const int error = runtests::MkDirAll(output_dir_for_test_str); 305 if (error) { 306 fprintf(stderr, "Error: Could not create output directory %s: %s\n", 307 output_dir_for_test_str.c_str(), strerror(error)); 308 return false; 309 } 310 output_filename_str = JoinPath(output_dir_for_test_str, output_file_basename); 311 } 312 313 // Assemble test binary args. 314 fbl::Vector<const char*> argv; 315 argv.push_back(test_path.c_str()); 316 fbl::String verbosity_arg; 317 if (verbosity >= 0) { 318 // verbosity defaults to -1: "unspecified". Only pass it along 319 // if it was specified: i.e., non-negative. 320 verbosity_arg = fbl::StringPrintf("v=%d", verbosity); 321 argv.push_back(verbosity_arg.c_str()); 322 } 323 argv.push_back(nullptr); // Important, since there's no argc. 324 const char* output_dir_for_test = 325 output_dir_for_test_str.empty() ? nullptr : output_dir_for_test_str.c_str(); 326 const char* output_filename = 327 output_filename_str.empty() ? nullptr : output_filename_str.c_str(); 328 329 // Execute the test binary. 330 printf("\n------------------------------------------------\n" 331 "RUNNING TEST: %s\n\n", 332 test_path.c_str()); 333 fbl::unique_ptr<Result> result = RunTest(argv.get(), output_dir_for_test, 334 output_filename); 335 if (result->launch_status != SUCCESS) { 336 *failed_count += 1; 337 } 338 results->push_back(fbl::move(result)); 339 } 340 return true; 341} 342 343} // namespace runtests 344