1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30#if defined(HAVE_CONFIG_H) 31#include "bconfig.h" 32#endif 33 34extern "C" { 35#include <sys/types.h> 36#include <sys/stat.h> 37#include <sys/wait.h> 38#include <unistd.h> 39} 40 41#include <algorithm> 42#include <cerrno> 43#include <cstdlib> 44#include <cstring> 45#include <fstream> 46#include <iostream> 47#include <map> 48#include <string> 49 50#include "atf-c++/detail/application.hpp" 51#include "atf-c++/config.hpp" 52#include "atf-c++/tests.hpp" 53 54#include "atf-c++/detail/env.hpp" 55#include "atf-c++/detail/exceptions.hpp" 56#include "atf-c++/detail/fs.hpp" 57#include "atf-c++/detail/parser.hpp" 58#include "atf-c++/detail/process.hpp" 59#include "atf-c++/detail/sanity.hpp" 60#include "atf-c++/detail/text.hpp" 61 62#include "atffile.hpp" 63#include "config.hpp" 64#include "fs.hpp" 65#include "requirements.hpp" 66#include "test-program.hpp" 67 68namespace impl = atf::atf_run; 69 70class atf_run : public atf::application::app { 71 static const char* m_description; 72 73 atf::tests::vars_map m_cmdline_vars; 74 75 static atf::tests::vars_map::value_type parse_var(const std::string&); 76 77 void process_option(int, const char*); 78 std::string specific_args(void) const; 79 options_set specific_options(void) const; 80 81 void parse_vflag(const std::string&); 82 83 std::vector< std::string > conf_args(void) const; 84 85 size_t count_tps(std::vector< std::string >) const; 86 87 int run_test(const atf::fs::path&, impl::atf_tps_writer&, 88 const atf::tests::vars_map&); 89 int run_test_directory(const atf::fs::path&, impl::atf_tps_writer&); 90 int run_test_program(const atf::fs::path&, impl::atf_tps_writer&, 91 const atf::tests::vars_map&); 92 93 impl::test_case_result get_test_case_result(const std::string&, 94 const atf::process::status&, const atf::fs::path&) const; 95 96public: 97 atf_run(void); 98 99 int main(void); 100}; 101 102const char* atf_run::m_description = 103 "atf-run is a tool that runs tests programs and collects their " 104 "results."; 105 106atf_run::atf_run(void) : 107 app(m_description, "atf-run(1)", "atf(7)") 108{ 109} 110 111void 112atf_run::process_option(int ch, const char* arg) 113{ 114 switch (ch) { 115 case 'v': 116 parse_vflag(arg); 117 break; 118 119 default: 120 UNREACHABLE; 121 } 122} 123 124std::string 125atf_run::specific_args(void) 126 const 127{ 128 return "[test-program1 .. test-programN]"; 129} 130 131atf_run::options_set 132atf_run::specific_options(void) 133 const 134{ 135 using atf::application::option; 136 options_set opts; 137 opts.insert(option('v', "var=value", "Sets the configuration variable " 138 "`var' to `value'; overrides " 139 "values in configuration files")); 140 return opts; 141} 142 143void 144atf_run::parse_vflag(const std::string& str) 145{ 146 if (str.empty()) 147 throw std::runtime_error("-v requires a non-empty argument"); 148 149 std::vector< std::string > ws = atf::text::split(str, "="); 150 if (ws.size() == 1 && str[str.length() - 1] == '=') { 151 m_cmdline_vars[ws[0]] = ""; 152 } else { 153 if (ws.size() != 2) 154 throw std::runtime_error("-v requires an argument of the form " 155 "var=value"); 156 157 m_cmdline_vars[ws[0]] = ws[1]; 158 } 159} 160 161int 162atf_run::run_test(const atf::fs::path& tp, 163 impl::atf_tps_writer& w, 164 const atf::tests::vars_map& config) 165{ 166 atf::fs::file_info fi(tp); 167 168 int errcode; 169 if (fi.get_type() == atf::fs::file_info::dir_type) 170 errcode = run_test_directory(tp, w); 171 else { 172 const atf::tests::vars_map effective_config = 173 impl::merge_configs(config, m_cmdline_vars); 174 175 errcode = run_test_program(tp, w, effective_config); 176 } 177 return errcode; 178} 179 180int 181atf_run::run_test_directory(const atf::fs::path& tp, 182 impl::atf_tps_writer& w) 183{ 184 impl::atffile af = impl::read_atffile(tp / "Atffile"); 185 186 atf::tests::vars_map test_suite_vars; 187 { 188 atf::tests::vars_map::const_iterator iter = 189 af.props().find("test-suite"); 190 INV(iter != af.props().end()); 191 test_suite_vars = impl::read_config_files((*iter).second); 192 } 193 194 bool ok = true; 195 for (std::vector< std::string >::const_iterator iter = af.tps().begin(); 196 iter != af.tps().end(); iter++) { 197 const bool result = run_test(tp / *iter, w, 198 impl::merge_configs(af.conf(), test_suite_vars)); 199 ok &= (result == EXIT_SUCCESS); 200 } 201 202 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 203} 204 205impl::test_case_result 206atf_run::get_test_case_result(const std::string& broken_reason, 207 const atf::process::status& s, 208 const atf::fs::path& resfile) 209 const 210{ 211 using atf::text::to_string; 212 using impl::read_test_case_result; 213 using impl::test_case_result; 214 215 if (!broken_reason.empty()) { 216 test_case_result tcr; 217 218 try { 219 tcr = read_test_case_result(resfile); 220 221 if (tcr.state() == "expected_timeout") { 222 return tcr; 223 } else { 224 return test_case_result("failed", -1, broken_reason); 225 } 226 } catch (const std::runtime_error&) { 227 return test_case_result("failed", -1, broken_reason); 228 } 229 } 230 231 if (s.exited()) { 232 test_case_result tcr; 233 234 try { 235 tcr = read_test_case_result(resfile); 236 } catch (const std::runtime_error& e) { 237 return test_case_result("failed", -1, "Test case exited " 238 "normally but failed to create the results file: " + 239 std::string(e.what())); 240 } 241 242 if (tcr.state() == "expected_death") { 243 return tcr; 244 } else if (tcr.state() == "expected_exit") { 245 if (tcr.value() == -1 || s.exitstatus() == tcr.value()) 246 return tcr; 247 else 248 return test_case_result("failed", -1, "Test case was " 249 "expected to exit with a " + to_string(tcr.value()) + 250 " error code but returned " + to_string(s.exitstatus())); 251 } else if (tcr.state() == "expected_failure") { 252 if (s.exitstatus() == EXIT_SUCCESS) 253 return tcr; 254 else 255 return test_case_result("failed", -1, "Test case returned an " 256 "error in expected_failure mode but it should not have"); 257 } else if (tcr.state() == "expected_signal") { 258 return test_case_result("failed", -1, "Test case exited cleanly " 259 "but was expected to receive a signal"); 260 } else if (tcr.state() == "failed") { 261 if (s.exitstatus() == EXIT_SUCCESS) 262 return test_case_result("failed", -1, "Test case " 263 "exited successfully but reported failure"); 264 else 265 return tcr; 266 } else if (tcr.state() == "passed") { 267 if (s.exitstatus() == EXIT_SUCCESS) 268 return tcr; 269 else 270 return test_case_result("failed", -1, "Test case exited as " 271 "passed but reported an error"); 272 } else if (tcr.state() == "skipped") { 273 if (s.exitstatus() == EXIT_SUCCESS) 274 return tcr; 275 else 276 return test_case_result("failed", -1, "Test case exited as " 277 "skipped but reported an error"); 278 } 279 } else if (s.signaled()) { 280 test_case_result tcr; 281 282 try { 283 tcr = read_test_case_result(resfile); 284 } catch (const std::runtime_error&) { 285 return test_case_result("failed", -1, "Test program received " 286 "signal " + atf::text::to_string(s.termsig()) + 287 (s.coredump() ? " (core dumped)" : "")); 288 } 289 290 if (tcr.state() == "expected_death") { 291 return tcr; 292 } else if (tcr.state() == "expected_signal") { 293 if (tcr.value() == -1 || s.termsig() == tcr.value()) 294 return tcr; 295 else 296 return test_case_result("failed", -1, "Test case was " 297 "expected to exit due to a " + to_string(tcr.value()) + 298 " signal but got " + to_string(s.termsig())); 299 } else { 300 return test_case_result("failed", -1, "Test program received " 301 "signal " + atf::text::to_string(s.termsig()) + 302 (s.coredump() ? " (core dumped)" : "") + " and created a " 303 "bogus results file"); 304 } 305 } 306 UNREACHABLE; 307 return test_case_result(); 308} 309 310int 311atf_run::run_test_program(const atf::fs::path& tp, 312 impl::atf_tps_writer& w, 313 const atf::tests::vars_map& config) 314{ 315 int errcode = EXIT_SUCCESS; 316 317 impl::metadata md; 318 try { 319 md = impl::get_metadata(tp, config); 320 } catch (const atf::parser::format_error& e) { 321 w.start_tp(tp.str(), 0); 322 w.end_tp("Invalid format for test case list: " + std::string(e.what())); 323 return EXIT_FAILURE; 324 } catch (const atf::parser::parse_errors& e) { 325 const std::string reason = atf::text::join(e, "; "); 326 w.start_tp(tp.str(), 0); 327 w.end_tp("Invalid format for test case list: " + reason); 328 return EXIT_FAILURE; 329 } 330 331 impl::temp_dir resdir(atf::fs::path(atf::config::get("atf_workdir")) / 332 "atf-run.XXXXXX"); 333 334 w.start_tp(tp.str(), md.test_cases.size()); 335 if (md.test_cases.empty()) { 336 w.end_tp("Bogus test program: reported 0 test cases"); 337 errcode = EXIT_FAILURE; 338 } else { 339 for (std::map< std::string, atf::tests::vars_map >::const_iterator iter 340 = md.test_cases.begin(); iter != md.test_cases.end(); iter++) { 341 const std::string& tcname = (*iter).first; 342 const atf::tests::vars_map& tcmd = (*iter).second; 343 344 w.start_tc(tcname); 345 346 try { 347 const std::string& reqfail = impl::check_requirements( 348 tcmd, config); 349 if (!reqfail.empty()) { 350 w.end_tc("skipped", reqfail); 351 continue; 352 } 353 } catch (const std::runtime_error& e) { 354 w.end_tc("failed", e.what()); 355 errcode = EXIT_FAILURE; 356 continue; 357 } 358 359 const std::pair< int, int > user = impl::get_required_user( 360 tcmd, config); 361 362 atf::fs::path resfile = resdir.get_path() / "tcr"; 363 INV(!atf::fs::exists(resfile)); 364 try { 365 const bool has_cleanup = atf::text::to_bool( 366 (*tcmd.find("has.cleanup")).second); 367 368 impl::temp_dir workdir(atf::fs::path(atf::config::get( 369 "atf_workdir")) / "atf-run.XXXXXX"); 370 if (user.first != -1 && user.second != -1) { 371 if (::chown(workdir.get_path().c_str(), user.first, 372 user.second) == -1) { 373 throw atf::system_error("chmod(" + 374 workdir.get_path().str() + ")", "chmod(2) failed", 375 errno); 376 } 377 resfile = workdir.get_path() / "tcr"; 378 } 379 380 std::pair< std::string, const atf::process::status > s = 381 impl::run_test_case(tp, tcname, "body", tcmd, config, 382 resfile, workdir.get_path(), w); 383 if (has_cleanup) 384 (void)impl::run_test_case(tp, tcname, "cleanup", tcmd, 385 config, resfile, workdir.get_path(), w); 386 387 // TODO: Force deletion of workdir. 388 389 impl::test_case_result tcr = get_test_case_result(s.first, 390 s.second, resfile); 391 392 w.end_tc(tcr.state(), tcr.reason()); 393 if (tcr.state() == "failed") 394 errcode = EXIT_FAILURE; 395 } catch (...) { 396 if (atf::fs::exists(resfile)) 397 atf::fs::remove(resfile); 398 throw; 399 } 400 if (atf::fs::exists(resfile)) 401 atf::fs::remove(resfile); 402 403 } 404 w.end_tp(""); 405 } 406 407 return errcode; 408} 409 410size_t 411atf_run::count_tps(std::vector< std::string > tps) 412 const 413{ 414 size_t ntps = 0; 415 416 for (std::vector< std::string >::const_iterator iter = tps.begin(); 417 iter != tps.end(); iter++) { 418 atf::fs::path tp(*iter); 419 atf::fs::file_info fi(tp); 420 421 if (fi.get_type() == atf::fs::file_info::dir_type) { 422 impl::atffile af = impl::read_atffile(tp / "Atffile"); 423 std::vector< std::string > aux = af.tps(); 424 for (std::vector< std::string >::iterator i2 = aux.begin(); 425 i2 != aux.end(); i2++) 426 *i2 = (tp / *i2).str(); 427 ntps += count_tps(aux); 428 } else 429 ntps++; 430 } 431 432 return ntps; 433} 434 435static 436void 437call_hook(const std::string& tool, const std::string& hook) 438{ 439 const atf::fs::path sh(atf::config::get("atf_shell")); 440 const atf::fs::path hooks = 441 atf::fs::path(atf::config::get("atf_pkgdatadir")) / (tool + ".hooks"); 442 443 const atf::process::status s = 444 atf::process::exec(sh, 445 atf::process::argv_array(sh.c_str(), hooks.c_str(), 446 hook.c_str(), NULL), 447 atf::process::stream_inherit(), 448 atf::process::stream_inherit()); 449 450 451 if (!s.exited() || s.exitstatus() != EXIT_SUCCESS) 452 throw std::runtime_error("Failed to run the '" + hook + "' hook " 453 "for '" + tool + "'"); 454} 455 456int 457atf_run::main(void) 458{ 459 impl::atffile af = impl::read_atffile(atf::fs::path("Atffile")); 460 461 std::vector< std::string > tps; 462 tps = af.tps(); 463 if (m_argc >= 1) { 464 // TODO: Ensure that the given test names are listed in the 465 // Atffile. Take into account that the file can be using globs. 466 tps.clear(); 467 for (int i = 0; i < m_argc; i++) 468 tps.push_back(m_argv[i]); 469 } 470 471 // Read configuration data for this test suite. 472 atf::tests::vars_map test_suite_vars; 473 { 474 atf::tests::vars_map::const_iterator iter = 475 af.props().find("test-suite"); 476 INV(iter != af.props().end()); 477 test_suite_vars = impl::read_config_files((*iter).second); 478 } 479 480 impl::atf_tps_writer w(std::cout); 481 call_hook("atf-run", "info_start_hook"); 482 w.ntps(count_tps(tps)); 483 484 bool ok = true; 485 for (std::vector< std::string >::const_iterator iter = tps.begin(); 486 iter != tps.end(); iter++) { 487 const bool result = run_test(atf::fs::path(*iter), w, 488 impl::merge_configs(af.conf(), test_suite_vars)); 489 ok &= (result == EXIT_SUCCESS); 490 } 491 492 call_hook("atf-run", "info_end_hook"); 493 494 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 495} 496 497int 498main(int argc, char* const* argv) 499{ 500 return atf_run().run(argc, argv); 501} 502