1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30extern "C" { 31#include <sys/types.h> 32#include <sys/stat.h> 33 34#include <signal.h> 35#include <unistd.h> 36} 37 38#include <cerrno> 39#include <cstdlib> 40#include <cstring> 41#include <fstream> 42#include <iostream> 43 44#include "atf-c++/detail/env.hpp" 45#include "atf-c++/detail/parser.hpp" 46#include "atf-c++/detail/process.hpp" 47#include "atf-c++/detail/sanity.hpp" 48#include "atf-c++/detail/text.hpp" 49 50#include "config.hpp" 51#include "fs.hpp" 52#include "io.hpp" 53#include "requirements.hpp" 54#include "signals.hpp" 55#include "test-program.hpp" 56#include "timer.hpp" 57#include "user.hpp" 58 59namespace impl = atf::atf_run; 60namespace detail = atf::atf_run::detail; 61 62namespace { 63 64static void 65check_stream(std::ostream& os) 66{ 67 // If we receive a signal while writing to the stream, the bad bit gets set. 68 // Things seem to behave fine afterwards if we clear such error condition. 69 // However, I'm not sure if it's safe to query errno at this point. 70 if (os.bad()) { 71 if (errno == EINTR) 72 os.clear(); 73 else 74 throw std::runtime_error("Failed"); 75 } 76} 77 78namespace atf_tp { 79 80static const atf::parser::token_type eof_type = 0; 81static const atf::parser::token_type nl_type = 1; 82static const atf::parser::token_type text_type = 2; 83static const atf::parser::token_type colon_type = 3; 84static const atf::parser::token_type dblquote_type = 4; 85 86class tokenizer : public atf::parser::tokenizer< std::istream > { 87public: 88 tokenizer(std::istream& is, size_t curline) : 89 atf::parser::tokenizer< std::istream > 90 (is, true, eof_type, nl_type, text_type, curline) 91 { 92 add_delim(':', colon_type); 93 add_quote('"', dblquote_type); 94 } 95}; 96 97} // namespace atf_tp 98 99class metadata_reader : public detail::atf_tp_reader { 100 impl::test_cases_map m_tcs; 101 102 void got_tc(const std::string& ident, const atf::tests::vars_map& props) 103 { 104 if (m_tcs.find(ident) != m_tcs.end()) 105 throw(std::runtime_error("Duplicate test case " + ident + 106 " in test program")); 107 m_tcs[ident] = props; 108 109 if (m_tcs[ident].find("has.cleanup") == m_tcs[ident].end()) 110 m_tcs[ident].insert(std::make_pair("has.cleanup", "false")); 111 112 if (m_tcs[ident].find("timeout") == m_tcs[ident].end()) 113 m_tcs[ident].insert(std::make_pair("timeout", "30")); 114 } 115 116public: 117 metadata_reader(std::istream& is) : 118 detail::atf_tp_reader(is) 119 { 120 } 121 122 const impl::test_cases_map& 123 get_tcs(void) 124 const 125 { 126 return m_tcs; 127 } 128}; 129 130struct get_metadata_params { 131 const atf::fs::path& executable; 132 const atf::tests::vars_map& config; 133 134 get_metadata_params(const atf::fs::path& p_executable, 135 const atf::tests::vars_map& p_config) : 136 executable(p_executable), 137 config(p_config) 138 { 139 } 140}; 141 142struct test_case_params { 143 const atf::fs::path& executable; 144 const std::string& test_case_name; 145 const std::string& test_case_part; 146 const atf::tests::vars_map& metadata; 147 const atf::tests::vars_map& config; 148 const atf::fs::path& resfile; 149 const atf::fs::path& workdir; 150 151 test_case_params(const atf::fs::path& p_executable, 152 const std::string& p_test_case_name, 153 const std::string& p_test_case_part, 154 const atf::tests::vars_map& p_metadata, 155 const atf::tests::vars_map& p_config, 156 const atf::fs::path& p_resfile, 157 const atf::fs::path& p_workdir) : 158 executable(p_executable), 159 test_case_name(p_test_case_name), 160 test_case_part(p_test_case_part), 161 metadata(p_metadata), 162 config(p_config), 163 resfile(p_resfile), 164 workdir(p_workdir) 165 { 166 } 167}; 168 169static 170void 171append_to_vector(std::vector< std::string >& v1, 172 const std::vector< std::string >& v2) 173{ 174 std::copy(v2.begin(), v2.end(), 175 std::back_insert_iterator< std::vector< std::string > >(v1)); 176} 177 178static 179char** 180vector_to_argv(const std::vector< std::string >& v) 181{ 182 char** argv = new char*[v.size() + 1]; 183 for (std::vector< std::string >::size_type i = 0; i < v.size(); i++) { 184 argv[i] = strdup(v[i].c_str()); 185 } 186 argv[v.size()] = NULL; 187 return argv; 188} 189 190static 191void 192exec_or_exit(const atf::fs::path& executable, 193 const std::vector< std::string >& argv) 194{ 195 // This leaks memory in case of a failure, but it is OK. Exiting will 196 // do the necessary cleanup. 197 char* const* native_argv = vector_to_argv(argv); 198 199 ::execv(executable.c_str(), native_argv); 200 201 const std::string message = "Failed to execute '" + executable.str() + 202 "': " + std::strerror(errno) + "\n"; 203 if (::write(STDERR_FILENO, message.c_str(), message.length()) == -1) 204 std::abort(); 205 std::exit(EXIT_FAILURE); 206} 207 208static 209std::vector< std::string > 210config_to_args(const atf::tests::vars_map& config) 211{ 212 std::vector< std::string > args; 213 214 for (atf::tests::vars_map::const_iterator iter = config.begin(); 215 iter != config.end(); iter++) 216 args.push_back("-v" + (*iter).first + "=" + (*iter).second); 217 218 return args; 219} 220 221static 222void 223prepare_child(const atf::fs::path& workdir) 224{ 225 const int ret = ::setpgid(::getpid(), 0); 226 INV(ret != -1); 227 228 ::umask(S_IWGRP | S_IWOTH); 229 230 for (int i = 1; i <= impl::last_signo; i++) 231 impl::reset(i); 232 233 atf::env::set("HOME", workdir.str()); 234 atf::env::unset("LANG"); 235 atf::env::unset("LC_ALL"); 236 atf::env::unset("LC_COLLATE"); 237 atf::env::unset("LC_CTYPE"); 238 atf::env::unset("LC_MESSAGES"); 239 atf::env::unset("LC_MONETARY"); 240 atf::env::unset("LC_NUMERIC"); 241 atf::env::unset("LC_TIME"); 242 atf::env::unset("TZ"); 243 244 impl::change_directory(workdir); 245} 246 247static 248void 249get_metadata_child(void* raw_params) 250{ 251 const get_metadata_params* params = 252 static_cast< const get_metadata_params* >(raw_params); 253 254 std::vector< std::string > argv; 255 argv.push_back(params->executable.leaf_name()); 256 argv.push_back("-l"); 257 argv.push_back("-s" + params->executable.branch_path().str()); 258 append_to_vector(argv, config_to_args(params->config)); 259 260 exec_or_exit(params->executable, argv); 261} 262 263void 264run_test_case_child(void* raw_params) 265{ 266 const test_case_params* params = 267 static_cast< const test_case_params* >(raw_params); 268 269 const std::pair< int, int > user = impl::get_required_user( 270 params->metadata, params->config); 271 if (user.first != -1 && user.second != -1) 272 impl::drop_privileges(user); 273 274 // The input 'tp' parameter may be relative and become invalid once 275 // we change the current working directory. 276 const atf::fs::path absolute_executable = params->executable.to_absolute(); 277 278 // Prepare the test program's arguments. We use dynamic memory and 279 // do not care to release it. We are going to die anyway very soon, 280 // either due to exec(2) or to exit(3). 281 std::vector< std::string > argv; 282 argv.push_back(absolute_executable.leaf_name()); 283 argv.push_back("-r" + params->resfile.str()); 284 argv.push_back("-s" + absolute_executable.branch_path().str()); 285 append_to_vector(argv, config_to_args(params->config)); 286 argv.push_back(params->test_case_name + ":" + params->test_case_part); 287 288 prepare_child(params->workdir); 289 exec_or_exit(absolute_executable, argv); 290} 291 292static void 293tokenize_result(const std::string& line, std::string& out_state, 294 std::string& out_arg, std::string& out_reason) 295{ 296 const std::string::size_type pos = line.find_first_of(":("); 297 if (pos == std::string::npos) { 298 out_state = line; 299 out_arg = ""; 300 out_reason = ""; 301 } else if (line[pos] == ':') { 302 out_state = line.substr(0, pos); 303 out_arg = ""; 304 out_reason = atf::text::trim(line.substr(pos + 1)); 305 } else if (line[pos] == '(') { 306 const std::string::size_type pos2 = line.find("):", pos); 307 if (pos2 == std::string::npos) 308 throw std::runtime_error("Invalid test case result '" + line + 309 "': unclosed optional argument"); 310 out_state = line.substr(0, pos); 311 out_arg = line.substr(pos + 1, pos2 - pos - 1); 312 out_reason = atf::text::trim(line.substr(pos2 + 2)); 313 } else 314 UNREACHABLE; 315} 316 317static impl::test_case_result 318handle_result(const std::string& state, const std::string& arg, 319 const std::string& reason) 320{ 321 PRE(state == "passed"); 322 323 if (!arg.empty() || !reason.empty()) 324 throw std::runtime_error("The test case result '" + state + "' cannot " 325 "be accompanied by a reason nor an expected value"); 326 327 return impl::test_case_result(state, -1, reason); 328} 329 330static impl::test_case_result 331handle_result_with_reason(const std::string& state, const std::string& arg, 332 const std::string& reason) 333{ 334 PRE(state == "expected_death" || state == "expected_failure" || 335 state == "expected_timeout" || state == "failed" || state == "skipped"); 336 337 if (!arg.empty() || reason.empty()) 338 throw std::runtime_error("The test case result '" + state + "' must " 339 "be accompanied by a reason but not by an expected value"); 340 341 return impl::test_case_result(state, -1, reason); 342} 343 344static impl::test_case_result 345handle_result_with_reason_and_arg(const std::string& state, 346 const std::string& arg, 347 const std::string& reason) 348{ 349 PRE(state == "expected_exit" || state == "expected_signal"); 350 351 if (reason.empty()) 352 throw std::runtime_error("The test case result '" + state + "' must " 353 "be accompanied by a reason"); 354 355 int value; 356 if (arg.empty()) { 357 value = -1; 358 } else { 359 try { 360 value = atf::text::to_type< int >(arg); 361 } catch (const std::runtime_error&) { 362 throw std::runtime_error("The value '" + arg + "' passed to the '" + 363 state + "' state must be an integer"); 364 } 365 } 366 367 return impl::test_case_result(state, value, reason); 368} 369 370} // anonymous namespace 371 372detail::atf_tp_reader::atf_tp_reader(std::istream& is) : 373 m_is(is) 374{ 375} 376 377detail::atf_tp_reader::~atf_tp_reader(void) 378{ 379} 380 381void 382detail::atf_tp_reader::got_tc(const std::string& ident, 383 const std::map< std::string, std::string >& md) 384{ 385} 386 387void 388detail::atf_tp_reader::got_eof(void) 389{ 390} 391 392void 393detail::atf_tp_reader::validate_and_insert(const std::string& name, 394 const std::string& value, const size_t lineno, 395 std::map< std::string, std::string >& md) 396{ 397 using atf::parser::parse_error; 398 399 if (value.empty()) 400 throw parse_error(lineno, "The value for '" + name +"' cannot be " 401 "empty"); 402 403 const std::string ident_regex = "^[_A-Za-z0-9]+$"; 404 const std::string integer_regex = "^[0-9]+$"; 405 406 if (name == "descr") { 407 // Any non-empty value is valid. 408 } else if (name == "has.cleanup") { 409 try { 410 (void)atf::text::to_bool(value); 411 } catch (const std::runtime_error&) { 412 throw parse_error(lineno, "The has.cleanup property requires a" 413 " boolean value"); 414 } 415 } else if (name == "ident") { 416 if (!atf::text::match(value, ident_regex)) 417 throw parse_error(lineno, "The identifier must match " + 418 ident_regex + "; was '" + value + "'"); 419 } else if (name == "require.arch") { 420 } else if (name == "require.config") { 421 } else if (name == "require.machine") { 422 } else if (name == "require.progs") { 423 } else if (name == "require.user") { 424 } else if (name == "timeout") { 425 if (!atf::text::match(value, integer_regex)) 426 throw parse_error(lineno, "The timeout property requires an integer" 427 " value"); 428 } else if (name == "use.fs") { 429 // Deprecated; ignore it. 430 } else if (name.size() > 2 && name[0] == 'X' && name[1] == '-') { 431 // Any non-empty value is valid. 432 } else { 433 throw parse_error(lineno, "Unknown property '" + name + "'"); 434 } 435 436 md.insert(std::make_pair(name, value)); 437} 438 439void 440detail::atf_tp_reader::read(void) 441{ 442 using atf::parser::parse_error; 443 using namespace atf_tp; 444 445 std::pair< size_t, atf::parser::headers_map > hml = 446 atf::parser::read_headers(m_is, 1); 447 atf::parser::validate_content_type(hml.second, 448 "application/X-atf-tp", 1); 449 450 tokenizer tkz(m_is, hml.first); 451 atf::parser::parser< tokenizer > p(tkz); 452 453 try { 454 atf::parser::token t = p.expect(text_type, "property name"); 455 if (t.text() != "ident") 456 throw parse_error(t.lineno(), "First property of a test case " 457 "must be 'ident'"); 458 459 std::map< std::string, std::string > props; 460 do { 461 const std::string name = t.text(); 462 t = p.expect(colon_type, "`:'"); 463 const std::string value = atf::text::trim(p.rest_of_line()); 464 t = p.expect(nl_type, "new line"); 465 validate_and_insert(name, value, t.lineno(), props); 466 467 t = p.expect(eof_type, nl_type, text_type, "property name, new " 468 "line or eof"); 469 if (t.type() == nl_type || t.type() == eof_type) { 470 const std::map< std::string, std::string >::const_iterator 471 iter = props.find("ident"); 472 if (iter == props.end()) 473 throw parse_error(t.lineno(), "Test case definition did " 474 "not define an 'ident' property"); 475 ATF_PARSER_CALLBACK(p, got_tc((*iter).second, props)); 476 props.clear(); 477 478 if (t.type() == nl_type) { 479 t = p.expect(text_type, "property name"); 480 if (t.text() != "ident") 481 throw parse_error(t.lineno(), "First property of a " 482 "test case must be 'ident'"); 483 } 484 } 485 } while (t.type() != eof_type); 486 ATF_PARSER_CALLBACK(p, got_eof()); 487 } catch (const parse_error& pe) { 488 p.add_error(pe); 489 p.reset(nl_type); 490 } 491} 492 493impl::test_case_result 494detail::parse_test_case_result(const std::string& line) 495{ 496 std::string state, arg, reason; 497 tokenize_result(line, state, arg, reason); 498 499 if (state == "expected_death") 500 return handle_result_with_reason(state, arg, reason); 501 else if (state.compare(0, 13, "expected_exit") == 0) 502 return handle_result_with_reason_and_arg(state, arg, reason); 503 else if (state.compare(0, 16, "expected_failure") == 0) 504 return handle_result_with_reason(state, arg, reason); 505 else if (state.compare(0, 15, "expected_signal") == 0) 506 return handle_result_with_reason_and_arg(state, arg, reason); 507 else if (state.compare(0, 16, "expected_timeout") == 0) 508 return handle_result_with_reason(state, arg, reason); 509 else if (state == "failed") 510 return handle_result_with_reason(state, arg, reason); 511 else if (state == "passed") 512 return handle_result(state, arg, reason); 513 else if (state == "skipped") 514 return handle_result_with_reason(state, arg, reason); 515 else 516 throw std::runtime_error("Unknown test case result type in: " + line); 517} 518 519impl::atf_tps_writer::atf_tps_writer(std::ostream& os) : 520 m_os(os) 521{ 522 atf::parser::headers_map hm; 523 atf::parser::attrs_map ct_attrs; 524 ct_attrs["version"] = "2"; 525 hm["Content-Type"] = 526 atf::parser::header_entry("Content-Type", "application/X-atf-tps", 527 ct_attrs); 528 atf::parser::write_headers(hm, m_os); 529} 530 531void 532impl::atf_tps_writer::info(const std::string& what, const std::string& val) 533{ 534 m_os << "info: " << what << ", " << val << "\n"; 535 m_os.flush(); 536} 537 538void 539impl::atf_tps_writer::ntps(size_t p_ntps) 540{ 541 m_os << "tps-count: " << p_ntps << "\n"; 542 m_os.flush(); 543} 544 545void 546impl::atf_tps_writer::start_tp(const std::string& tp, size_t ntcs) 547{ 548 m_tpname = tp; 549 m_os << "tp-start: " << tp << ", " << ntcs << "\n"; 550 m_os.flush(); 551} 552 553void 554impl::atf_tps_writer::end_tp(const std::string& reason) 555{ 556 PRE(reason.find('\n') == std::string::npos); 557 if (reason.empty()) 558 m_os << "tp-end: " << m_tpname << "\n"; 559 else 560 m_os << "tp-end: " << m_tpname << ", " << reason << "\n"; 561 m_os.flush(); 562} 563 564void 565impl::atf_tps_writer::start_tc(const std::string& tcname) 566{ 567 m_tcname = tcname; 568 m_os << "tc-start: " << tcname << "\n"; 569 m_os.flush(); 570} 571 572void 573impl::atf_tps_writer::stdout_tc(const std::string& line) 574{ 575 m_os << "tc-so:" << line << "\n"; 576 check_stream(m_os); 577 m_os.flush(); 578 check_stream(m_os); 579} 580 581void 582impl::atf_tps_writer::stderr_tc(const std::string& line) 583{ 584 m_os << "tc-se:" << line << "\n"; 585 check_stream(m_os); 586 m_os.flush(); 587 check_stream(m_os); 588} 589 590void 591impl::atf_tps_writer::end_tc(const std::string& state, 592 const std::string& reason) 593{ 594 std::string str = "tc-end: " + m_tcname + ", " + state; 595 if (!reason.empty()) 596 str += ", " + reason; 597 m_os << str << "\n"; 598 m_os.flush(); 599} 600 601impl::metadata 602impl::get_metadata(const atf::fs::path& executable, 603 const atf::tests::vars_map& config) 604{ 605 get_metadata_params params(executable, config); 606 atf::process::child child = 607 atf::process::fork(get_metadata_child, 608 atf::process::stream_capture(), 609 atf::process::stream_inherit(), 610 static_cast< void * >(¶ms)); 611 612 impl::pistream outin(child.stdout_fd()); 613 614 metadata_reader parser(outin); 615 parser.read(); 616 617 const atf::process::status status = child.wait(); 618 if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) 619 throw atf::parser::format_error("Test program returned failure " 620 "exit status for test case list"); 621 622 return metadata(parser.get_tcs()); 623} 624 625impl::test_case_result 626impl::read_test_case_result(const atf::fs::path& results_path) 627{ 628 std::ifstream results_file(results_path.c_str()); 629 if (!results_file) 630 throw std::runtime_error("Failed to open " + results_path.str()); 631 632 std::string line, extra_line; 633 std::getline(results_file, line); 634 if (!results_file.good()) 635 throw std::runtime_error("Results file is empty"); 636 637 while (std::getline(results_file, extra_line).good()) 638 line += "<<NEWLINE UNEXPECTED>>" + extra_line; 639 640 results_file.close(); 641 642 return detail::parse_test_case_result(line); 643} 644 645namespace { 646 647static volatile bool terminate_poll; 648 649static void 650sigchld_handler(const int signo) 651{ 652 terminate_poll = true; 653} 654 655class child_muxer : public impl::muxer { 656 impl::atf_tps_writer& m_writer; 657 658 void 659 line_callback(const size_t index, const std::string& line) 660 { 661 switch (index) { 662 case 0: m_writer.stdout_tc(line); break; 663 case 1: m_writer.stderr_tc(line); break; 664 default: UNREACHABLE; 665 } 666 } 667 668public: 669 child_muxer(const int* fds, const size_t nfds, 670 impl::atf_tps_writer& writer) : 671 muxer(fds, nfds), 672 m_writer(writer) 673 { 674 } 675}; 676 677} // anonymous namespace 678 679std::pair< std::string, atf::process::status > 680impl::run_test_case(const atf::fs::path& executable, 681 const std::string& test_case_name, 682 const std::string& test_case_part, 683 const atf::tests::vars_map& metadata, 684 const atf::tests::vars_map& config, 685 const atf::fs::path& resfile, 686 const atf::fs::path& workdir, 687 atf_tps_writer& writer) 688{ 689 // TODO: Capture termination signals and deliver them to the subprocess 690 // instead. Or maybe do something else; think about it. 691 692 test_case_params params(executable, test_case_name, test_case_part, 693 metadata, config, resfile, workdir); 694 atf::process::child child = 695 atf::process::fork(run_test_case_child, 696 atf::process::stream_capture(), 697 atf::process::stream_capture(), 698 static_cast< void * >(¶ms)); 699 700 terminate_poll = false; 701 702 const atf::tests::vars_map::const_iterator iter = metadata.find("timeout"); 703 INV(iter != metadata.end()); 704 const unsigned int timeout = 705 atf::text::to_type< unsigned int >((*iter).second); 706 const pid_t child_pid = child.pid(); 707 708 // Get the input stream of stdout and stderr. 709 impl::file_handle outfh = child.stdout_fd(); 710 impl::file_handle errfh = child.stderr_fd(); 711 712 bool timed_out = false; 713 714 // Process the test case's output and multiplex it into our output 715 // stream as we read it. 716 int fds[2] = {outfh.get(), errfh.get()}; 717 child_muxer mux(fds, 2, writer); 718 try { 719 child_timer timeout_timer(timeout, child_pid, terminate_poll); 720 signal_programmer sigchld(SIGCHLD, sigchld_handler); 721 mux.mux(terminate_poll); 722 timed_out = timeout_timer.fired(); 723 } catch (...) { 724 UNREACHABLE; 725 } 726 727 ::killpg(child_pid, SIGTERM); 728 mux.flush(); 729 atf::process::status status = child.wait(); 730 ::killpg(child_pid, SIGKILL); 731 732 std::string reason; 733 734 if (timed_out) { 735 // Don't assume the child process has been signaled due to the timeout 736 // expiration as older versions did. The child process may have exited 737 // but we may have timed out due to a subchild process getting stuck. 738 reason = "Test case timed out after " + atf::text::to_string(timeout) + 739 " " + (timeout == 1 ? "second" : "seconds"); 740 } 741 742 return std::make_pair(reason, status); 743} 744