1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2007 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30#if defined(HAVE_CONFIG_H) 31#include "bconfig.h" 32#endif 33 34extern "C" { 35#include <sys/types.h> 36#include <sys/param.h> 37#include <sys/stat.h> 38#include <sys/wait.h> 39#include <unistd.h> 40} 41 42#include <algorithm> 43#include <cerrno> 44#include <cstdlib> 45#include <cstring> 46#include <fstream> 47#include <iostream> 48#include <map> 49#include <string> 50 51#include "atf-c++/detail/application.hpp" 52#include "atf-c++/config.hpp" 53#include "atf-c++/tests.hpp" 54 55#include "atf-c++/detail/env.hpp" 56#include "atf-c++/detail/exceptions.hpp" 57#include "atf-c++/detail/fs.hpp" 58#include "atf-c++/detail/parser.hpp" 59#include "atf-c++/detail/process.hpp" 60#include "atf-c++/detail/sanity.hpp" 61#include "atf-c++/detail/text.hpp" 62 63#include "atffile.hpp" 64#include "config.hpp" 65#include "fs.hpp" 66#include "requirements.hpp" 67#include "test-program.hpp" 68 69namespace impl = atf::atf_run; 70 71#if defined(MAXCOMLEN) 72static const std::string::size_type max_core_name_length = MAXCOMLEN; 73#else 74static const std::string::size_type max_core_name_length = std::string::npos; 75#endif 76 77class atf_run : public atf::application::app { 78 static const char* m_description; 79 80 atf::tests::vars_map m_cmdline_vars; 81 82 static atf::tests::vars_map::value_type parse_var(const std::string&); 83 84 void process_option(int, const char*); 85 std::string specific_args(void) const; 86 options_set specific_options(void) const; 87 88 void parse_vflag(const std::string&); 89 90 std::vector< std::string > conf_args(void) const; 91 92 size_t count_tps(std::vector< std::string >) const; 93 94 int run_test(const atf::fs::path&, impl::atf_tps_writer&, 95 const atf::tests::vars_map&); 96 int run_test_directory(const atf::fs::path&, impl::atf_tps_writer&); 97 int run_test_program(const atf::fs::path&, impl::atf_tps_writer&, 98 const atf::tests::vars_map&); 99 100 impl::test_case_result get_test_case_result(const std::string&, 101 const atf::process::status&, const atf::fs::path&) const; 102 103public: 104 atf_run(void); 105 106 int main(void); 107}; 108 109static void 110sanitize_gdb_env(void) 111{ 112 try { 113 atf::env::unset("TERM"); 114 } catch (...) { 115 // Just swallow exceptions here; they cannot propagate into C, which 116 // is where this function is called from, and even if these exceptions 117 // appear they are benign. 118 } 119} 120 121static void 122dump_stacktrace(const atf::fs::path& tp, const atf::process::status& s, 123 const atf::fs::path& workdir, impl::atf_tps_writer& w) 124{ 125 PRE(s.signaled() && s.coredump()); 126 127 w.stderr_tc("Test program crashed; attempting to get stack trace"); 128 129 const atf::fs::path corename = workdir / 130 (tp.leaf_name().substr(0, max_core_name_length) + ".core"); 131 if (!atf::fs::exists(corename)) { 132 w.stderr_tc("Expected file " + corename.str() + " not found"); 133 return; 134 } 135 136 const atf::fs::path gdb(GDB); 137 const atf::fs::path gdbout = workdir / "gdb.out"; 138 const atf::process::argv_array args(gdb.leaf_name().c_str(), "-batch", 139 "-q", "-ex", "bt", tp.c_str(), 140 corename.c_str(), NULL); 141 atf::process::status status = atf::process::exec( 142 gdb, args, 143 atf::process::stream_redirect_path(gdbout), 144 atf::process::stream_redirect_path(atf::fs::path("/dev/null")), 145 sanitize_gdb_env); 146 if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) { 147 w.stderr_tc("Execution of " GDB " failed"); 148 return; 149 } 150 151 std::ifstream input(gdbout.c_str()); 152 if (input) { 153 std::string line; 154 while (std::getline(input, line).good()) 155 w.stderr_tc(line); 156 input.close(); 157 } 158 159 w.stderr_tc("Stack trace complete"); 160} 161 162const char* atf_run::m_description = 163 "atf-run is a tool that runs tests programs and collects their " 164 "results."; 165 166atf_run::atf_run(void) : 167 app(m_description, "atf-run(1)", "atf(7)") 168{ 169} 170 171void 172atf_run::process_option(int ch, const char* arg) 173{ 174 switch (ch) { 175 case 'v': 176 parse_vflag(arg); 177 break; 178 179 default: 180 UNREACHABLE; 181 } 182} 183 184std::string 185atf_run::specific_args(void) 186 const 187{ 188 return "[test-program1 .. test-programN]"; 189} 190 191atf_run::options_set 192atf_run::specific_options(void) 193 const 194{ 195 using atf::application::option; 196 options_set opts; 197 opts.insert(option('v', "var=value", "Sets the configuration variable " 198 "`var' to `value'; overrides " 199 "values in configuration files")); 200 return opts; 201} 202 203void 204atf_run::parse_vflag(const std::string& str) 205{ 206 if (str.empty()) 207 throw std::runtime_error("-v requires a non-empty argument"); 208 209 std::vector< std::string > ws = atf::text::split(str, "="); 210 if (ws.size() == 1 && str[str.length() - 1] == '=') { 211 m_cmdline_vars[ws[0]] = ""; 212 } else { 213 if (ws.size() != 2) 214 throw std::runtime_error("-v requires an argument of the form " 215 "var=value"); 216 217 m_cmdline_vars[ws[0]] = ws[1]; 218 } 219} 220 221int 222atf_run::run_test(const atf::fs::path& tp, 223 impl::atf_tps_writer& w, 224 const atf::tests::vars_map& config) 225{ 226 atf::fs::file_info fi(tp); 227 228 int errcode; 229 if (fi.get_type() == atf::fs::file_info::dir_type) 230 errcode = run_test_directory(tp, w); 231 else { 232 const atf::tests::vars_map effective_config = 233 impl::merge_configs(config, m_cmdline_vars); 234 235 errcode = run_test_program(tp, w, effective_config); 236 } 237 return errcode; 238} 239 240int 241atf_run::run_test_directory(const atf::fs::path& tp, 242 impl::atf_tps_writer& w) 243{ 244 impl::atffile af = impl::read_atffile(tp / "Atffile"); 245 246 atf::tests::vars_map test_suite_vars; 247 { 248 atf::tests::vars_map::const_iterator iter = 249 af.props().find("test-suite"); 250 INV(iter != af.props().end()); 251 test_suite_vars = impl::read_config_files((*iter).second); 252 } 253 254 bool ok = true; 255 for (std::vector< std::string >::const_iterator iter = af.tps().begin(); 256 iter != af.tps().end(); iter++) { 257 const bool result = run_test(tp / *iter, w, 258 impl::merge_configs(af.conf(), test_suite_vars)); 259 ok &= (result == EXIT_SUCCESS); 260 } 261 262 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 263} 264 265impl::test_case_result 266atf_run::get_test_case_result(const std::string& broken_reason, 267 const atf::process::status& s, 268 const atf::fs::path& resfile) 269 const 270{ 271 using atf::text::to_string; 272 using impl::read_test_case_result; 273 using impl::test_case_result; 274 275 if (!broken_reason.empty()) { 276 test_case_result tcr; 277 278 try { 279 tcr = read_test_case_result(resfile); 280 281 if (tcr.state() == "expected_timeout") { 282 return tcr; 283 } else { 284 return test_case_result("failed", -1, broken_reason); 285 } 286 } catch (const std::runtime_error&) { 287 return test_case_result("failed", -1, broken_reason); 288 } 289 } 290 291 if (s.exited()) { 292 test_case_result tcr; 293 294 try { 295 tcr = read_test_case_result(resfile); 296 } catch (const std::runtime_error& e) { 297 return test_case_result("failed", -1, "Test case exited " 298 "normally but failed to create the results file: " + 299 std::string(e.what())); 300 } 301 302 if (tcr.state() == "expected_death") { 303 return tcr; 304 } else if (tcr.state() == "expected_exit") { 305 if (tcr.value() == -1 || s.exitstatus() == tcr.value()) 306 return tcr; 307 else 308 return test_case_result("failed", -1, "Test case was " 309 "expected to exit with a " + to_string(tcr.value()) + 310 " error code but returned " + to_string(s.exitstatus())); 311 } else if (tcr.state() == "expected_failure") { 312 if (s.exitstatus() == EXIT_SUCCESS) 313 return tcr; 314 else 315 return test_case_result("failed", -1, "Test case returned an " 316 "error in expected_failure mode but it should not have"); 317 } else if (tcr.state() == "expected_signal") { 318 return test_case_result("failed", -1, "Test case exited cleanly " 319 "but was expected to receive a signal"); 320 } else if (tcr.state() == "failed") { 321 if (s.exitstatus() == EXIT_SUCCESS) 322 return test_case_result("failed", -1, "Test case " 323 "exited successfully but reported failure"); 324 else 325 return tcr; 326 } else if (tcr.state() == "passed") { 327 if (s.exitstatus() == EXIT_SUCCESS) 328 return tcr; 329 else 330 return test_case_result("failed", -1, "Test case exited as " 331 "passed but reported an error"); 332 } else if (tcr.state() == "skipped") { 333 if (s.exitstatus() == EXIT_SUCCESS) 334 return tcr; 335 else 336 return test_case_result("failed", -1, "Test case exited as " 337 "skipped but reported an error"); 338 } 339 } else if (s.signaled()) { 340 test_case_result tcr; 341 342 try { 343 tcr = read_test_case_result(resfile); 344 } catch (const std::runtime_error&) { 345 return test_case_result("failed", -1, "Test program received " 346 "signal " + atf::text::to_string(s.termsig()) + 347 (s.coredump() ? " (core dumped)" : "")); 348 } 349 350 if (tcr.state() == "expected_death") { 351 return tcr; 352 } else if (tcr.state() == "expected_signal") { 353 if (tcr.value() == -1 || s.termsig() == tcr.value()) 354 return tcr; 355 else 356 return test_case_result("failed", -1, "Test case was " 357 "expected to exit due to a " + to_string(tcr.value()) + 358 " signal but got " + to_string(s.termsig())); 359 } else { 360 return test_case_result("failed", -1, "Test program received " 361 "signal " + atf::text::to_string(s.termsig()) + 362 (s.coredump() ? " (core dumped)" : "") + " and created a " 363 "bogus results file"); 364 } 365 } 366 UNREACHABLE; 367 return test_case_result(); 368} 369 370int 371atf_run::run_test_program(const atf::fs::path& tp, 372 impl::atf_tps_writer& w, 373 const atf::tests::vars_map& config) 374{ 375 int errcode = EXIT_SUCCESS; 376 377 impl::metadata md; 378 try { 379 md = impl::get_metadata(tp, config); 380 } catch (const atf::parser::format_error& e) { 381 w.start_tp(tp.str(), 0); 382 w.end_tp("Invalid format for test case list: " + std::string(e.what())); 383 return EXIT_FAILURE; 384 } catch (const atf::parser::parse_errors& e) { 385 const std::string reason = atf::text::join(e, "; "); 386 w.start_tp(tp.str(), 0); 387 w.end_tp("Invalid format for test case list: " + reason); 388 return EXIT_FAILURE; 389 } 390 391 impl::temp_dir resdir(atf::fs::path(atf::config::get("atf_workdir")) / 392 "atf-run.XXXXXX"); 393 394 w.start_tp(tp.str(), md.test_cases.size()); 395 if (md.test_cases.empty()) { 396 w.end_tp("Bogus test program: reported 0 test cases"); 397 errcode = EXIT_FAILURE; 398 } else { 399 for (std::map< std::string, atf::tests::vars_map >::const_iterator iter 400 = md.test_cases.begin(); iter != md.test_cases.end(); iter++) { 401 const std::string& tcname = (*iter).first; 402 const atf::tests::vars_map& tcmd = (*iter).second; 403 404 w.start_tc(tcname); 405 406 try { 407 const std::string& reqfail = impl::check_requirements( 408 tcmd, config); 409 if (!reqfail.empty()) { 410 w.end_tc("skipped", reqfail); 411 continue; 412 } 413 } catch (const std::runtime_error& e) { 414 w.end_tc("failed", e.what()); 415 errcode = EXIT_FAILURE; 416 continue; 417 } 418 419 const std::pair< int, int > user = impl::get_required_user( 420 tcmd, config); 421 422 atf::fs::path resfile = resdir.get_path() / "tcr"; 423 INV(!atf::fs::exists(resfile)); 424 try { 425 const bool has_cleanup = atf::text::to_bool( 426 (*tcmd.find("has.cleanup")).second); 427 428 impl::temp_dir workdir(atf::fs::path(atf::config::get( 429 "atf_workdir")) / "atf-run.XXXXXX"); 430 if (user.first != -1 && user.second != -1) { 431 if (::chown(workdir.get_path().c_str(), user.first, 432 user.second) == -1) { 433 throw atf::system_error("chown(" + 434 workdir.get_path().str() + ")", "chown(2) failed", 435 errno); 436 } 437 resfile = workdir.get_path() / "tcr"; 438 } 439 440 std::pair< std::string, const atf::process::status > s = 441 impl::run_test_case(tp, tcname, "body", tcmd, config, 442 resfile, workdir.get_path(), w); 443 if (s.second.signaled() && s.second.coredump()) 444 dump_stacktrace(tp, s.second, workdir.get_path(), w); 445 if (has_cleanup) 446 (void)impl::run_test_case(tp, tcname, "cleanup", tcmd, 447 config, resfile, workdir.get_path(), w); 448 449 // TODO: Force deletion of workdir. 450 451 impl::test_case_result tcr = get_test_case_result(s.first, 452 s.second, resfile); 453 454 w.end_tc(tcr.state(), tcr.reason()); 455 if (tcr.state() == "failed") 456 errcode = EXIT_FAILURE; 457 } catch (...) { 458 if (atf::fs::exists(resfile)) 459 atf::fs::remove(resfile); 460 throw; 461 } 462 if (atf::fs::exists(resfile)) 463 atf::fs::remove(resfile); 464 465 } 466 w.end_tp(""); 467 } 468 469 return errcode; 470} 471 472size_t 473atf_run::count_tps(std::vector< std::string > tps) 474 const 475{ 476 size_t ntps = 0; 477 478 for (std::vector< std::string >::const_iterator iter = tps.begin(); 479 iter != tps.end(); iter++) { 480 atf::fs::path tp(*iter); 481 atf::fs::file_info fi(tp); 482 483 if (fi.get_type() == atf::fs::file_info::dir_type) { 484 impl::atffile af = impl::read_atffile(tp / "Atffile"); 485 std::vector< std::string > aux = af.tps(); 486 for (std::vector< std::string >::iterator i2 = aux.begin(); 487 i2 != aux.end(); i2++) 488 *i2 = (tp / *i2).str(); 489 ntps += count_tps(aux); 490 } else 491 ntps++; 492 } 493 494 return ntps; 495} 496 497static 498void 499call_hook(const std::string& tool, const std::string& hook) 500{ 501 const atf::fs::path sh(atf::config::get("atf_shell")); 502 const atf::fs::path hooks = 503 atf::fs::path(atf::config::get("atf_pkgdatadir")) / (tool + ".hooks"); 504 505 const atf::process::status s = 506 atf::process::exec(sh, 507 atf::process::argv_array(sh.c_str(), hooks.c_str(), 508 hook.c_str(), NULL), 509 atf::process::stream_inherit(), 510 atf::process::stream_inherit()); 511 512 513 if (!s.exited() || s.exitstatus() != EXIT_SUCCESS) 514 throw std::runtime_error("Failed to run the '" + hook + "' hook " 515 "for '" + tool + "'"); 516} 517 518int 519atf_run::main(void) 520{ 521 impl::atffile af = impl::read_atffile(atf::fs::path("Atffile")); 522 523 std::vector< std::string > tps; 524 tps = af.tps(); 525 if (m_argc >= 1) { 526 // TODO: Ensure that the given test names are listed in the 527 // Atffile. Take into account that the file can be using globs. 528 tps.clear(); 529 for (int i = 0; i < m_argc; i++) 530 tps.push_back(m_argv[i]); 531 } 532 533 // Read configuration data for this test suite. 534 atf::tests::vars_map test_suite_vars; 535 { 536 atf::tests::vars_map::const_iterator iter = 537 af.props().find("test-suite"); 538 INV(iter != af.props().end()); 539 test_suite_vars = impl::read_config_files((*iter).second); 540 } 541 542 impl::atf_tps_writer w(std::cout); 543 call_hook("atf-run", "info_start_hook"); 544 w.ntps(count_tps(tps)); 545 546 bool ok = true; 547 for (std::vector< std::string >::const_iterator iter = tps.begin(); 548 iter != tps.end(); iter++) { 549 const bool result = run_test(atf::fs::path(*iter), w, 550 impl::merge_configs(af.conf(), test_suite_vars)); 551 ok &= (result == EXIT_SUCCESS); 552 } 553 554 call_hook("atf-run", "info_end_hook"); 555 556 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 557} 558 559int 560main(int argc, char* const* argv) 561{ 562 return atf_run().run(argc, argv); 563} 564