1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2008, 2009, 2010 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30extern "C" { 31#include <sys/types.h> 32#include <sys/wait.h> 33 34#include <limits.h> 35#include <signal.h> 36} 37 38#include <cerrno> 39#include <cstdlib> 40#include <cstring> 41#include <fstream> 42#include <iostream> 43#include <iterator> 44#include <list> 45#include <memory> 46#include <utility> 47 48#include "atf-c++/check.hpp" 49#include "atf-c++/config.hpp" 50#include "atf-c++/utils.hpp" 51 52#include "atf-c++/detail/application.hpp" 53#include "atf-c++/detail/exceptions.hpp" 54#include "atf-c++/detail/fs.hpp" 55#include "atf-c++/detail/process.hpp" 56#include "atf-c++/detail/sanity.hpp" 57#include "atf-c++/detail/text.hpp" 58 59// ------------------------------------------------------------------------ 60// Auxiliary functions. 61// ------------------------------------------------------------------------ 62 63namespace { 64 65enum status_check_t { 66 sc_exit, 67 sc_ignore, 68 sc_signal, 69}; 70 71struct status_check { 72 status_check_t type; 73 bool negated; 74 int value; 75 76 status_check(const status_check_t& p_type, const bool p_negated, 77 const int p_value) : 78 type(p_type), 79 negated(p_negated), 80 value(p_value) 81 { 82 } 83}; 84 85enum output_check_t { 86 oc_ignore, 87 oc_inline, 88 oc_file, 89 oc_empty, 90 oc_match, 91 oc_save 92}; 93 94struct output_check { 95 output_check_t type; 96 bool negated; 97 std::string value; 98 99 output_check(const output_check_t& p_type, const bool p_negated, 100 const std::string& p_value) : 101 type(p_type), 102 negated(p_negated), 103 value(p_value) 104 { 105 } 106}; 107 108class temp_file : public std::ostream { 109 std::auto_ptr< atf::fs::path > m_path; 110 int m_fd; 111 112public: 113 temp_file(const atf::fs::path& p) : 114 std::ostream(NULL), 115 m_fd(-1) 116 { 117 atf::utils::auto_array< char > buf(new char[p.str().length() + 1]); 118 std::strcpy(buf.get(), p.c_str()); 119 120 m_fd = ::mkstemp(buf.get()); 121 if (m_fd == -1) 122 throw atf::system_error("atf_check::temp_file::temp_file(" + 123 p.str() + ")", "mkstemp(3) failed", 124 errno); 125 126 m_path.reset(new atf::fs::path(buf.get())); 127 } 128 129 ~temp_file(void) 130 { 131 close(); 132 try { 133 remove(*m_path); 134 } catch (const atf::system_error&) { 135 // Ignore deletion errors. 136 } 137 } 138 139 const atf::fs::path& 140 get_path(void) const 141 { 142 return *m_path; 143 } 144 145 void 146 write(const std::string& text) 147 { 148 if (::write(m_fd, text.c_str(), text.size()) == -1) 149 throw atf::system_error("atf_check", "write(2) failed", errno); 150 } 151 152 void 153 close(void) 154 { 155 if (m_fd != -1) { 156 flush(); 157 ::close(m_fd); 158 m_fd = -1; 159 } 160 } 161}; 162 163} // anonymous namespace 164 165static int 166parse_exit_code(const std::string& str) 167{ 168 try { 169 const int value = atf::text::to_type< int >(str); 170 if (value < 0 || value > 255) 171 throw std::runtime_error("Unused reason"); 172 return value; 173 } catch (const std::runtime_error&) { 174 throw atf::application::usage_error("Invalid exit code for -s option; " 175 "must be an integer in range 0-255"); 176 } 177} 178 179static struct name_number { 180 const char *name; 181 int signo; 182} signal_names_to_numbers[] = { 183 { "hup", SIGHUP }, 184 { "int", SIGINT }, 185 { "quit", SIGQUIT }, 186 { "trap", SIGTRAP }, 187 { "kill", SIGKILL }, 188 { "segv", SIGSEGV }, 189 { "pipe", SIGPIPE }, 190 { "alrm", SIGALRM }, 191 { "term", SIGTERM }, 192 { "usr1", SIGUSR1 }, 193 { "usr2", SIGUSR2 }, 194 { NULL, INT_MIN }, 195}; 196 197static int 198signal_name_to_number(const std::string& str) 199{ 200 struct name_number* iter = signal_names_to_numbers; 201 int signo = INT_MIN; 202 while (signo == INT_MIN && iter->name != NULL) { 203 if (str == iter->name || str == std::string("sig") + iter->name) 204 signo = iter->signo; 205 else 206 iter++; 207 } 208 return signo; 209} 210 211static int 212parse_signal(const std::string& str) 213{ 214 const int signo = signal_name_to_number(str); 215 if (signo == INT_MIN) { 216 try { 217 return atf::text::to_type< int >(str); 218 } catch (std::runtime_error) { 219 throw atf::application::usage_error("Invalid signal name or number " 220 "in -s option"); 221 } 222 } 223 INV(signo != INT_MIN); 224 return signo; 225} 226 227static status_check 228parse_status_check_arg(const std::string& arg) 229{ 230 const std::string::size_type delimiter = arg.find(':'); 231 bool negated = (arg.compare(0, 4, "not-") == 0); 232 const std::string action_str = arg.substr(0, delimiter); 233 const std::string action = negated ? action_str.substr(4) : action_str; 234 const std::string value_str = ( 235 delimiter == std::string::npos ? "" : arg.substr(delimiter + 1)); 236 int value; 237 238 status_check_t type; 239 if (action == "eq") { 240 // Deprecated; use exit instead. TODO: Remove after 0.10. 241 type = sc_exit; 242 if (negated) 243 throw atf::application::usage_error("Cannot negate eq checker"); 244 negated = false; 245 value = parse_exit_code(value_str); 246 } else if (action == "exit") { 247 type = sc_exit; 248 if (value_str.empty()) 249 value = INT_MIN; 250 else 251 value = parse_exit_code(value_str); 252 } else if (action == "ignore") { 253 if (negated) 254 throw atf::application::usage_error("Cannot negate ignore checker"); 255 type = sc_ignore; 256 value = INT_MIN; 257 } else if (action == "ne") { 258 // Deprecated; use not-exit instead. TODO: Remove after 0.10. 259 type = sc_exit; 260 if (negated) 261 throw atf::application::usage_error("Cannot negate ne checker"); 262 negated = true; 263 value = parse_exit_code(value_str); 264 } else if (action == "signal") { 265 type = sc_signal; 266 if (value_str.empty()) 267 value = INT_MIN; 268 else 269 value = parse_signal(value_str); 270 } else 271 throw atf::application::usage_error("Invalid output checker"); 272 273 return status_check(type, negated, value); 274} 275 276static 277output_check 278parse_output_check_arg(const std::string& arg) 279{ 280 const std::string::size_type delimiter = arg.find(':'); 281 const bool negated = (arg.compare(0, 4, "not-") == 0); 282 const std::string action_str = arg.substr(0, delimiter); 283 const std::string action = negated ? action_str.substr(4) : action_str; 284 285 output_check_t type; 286 if (action == "empty") 287 type = oc_empty; 288 else if (action == "file") 289 type = oc_file; 290 else if (action == "ignore") { 291 if (negated) 292 throw atf::application::usage_error("Cannot negate ignore checker"); 293 type = oc_ignore; 294 } else if (action == "inline") 295 type = oc_inline; 296 else if (action == "match") 297 type = oc_match; 298 else if (action == "save") { 299 if (negated) 300 throw atf::application::usage_error("Cannot negate save checker"); 301 type = oc_save; 302 } else 303 throw atf::application::usage_error("Invalid output checker"); 304 305 return output_check(type, negated, arg.substr(delimiter + 1)); 306} 307 308static 309std::string 310flatten_argv(char* const* argv) 311{ 312 std::string cmdline; 313 314 char* const* arg = &argv[0]; 315 while (*arg != NULL) { 316 if (arg != &argv[0]) 317 cmdline += ' '; 318 319 cmdline += *arg; 320 321 arg++; 322 } 323 324 return cmdline; 325} 326 327static 328std::auto_ptr< atf::check::check_result > 329execute(const char* const* argv) 330{ 331 std::cout << "Executing command [ "; 332 for (int i = 0; argv[i] != NULL; ++i) 333 std::cout << argv[i] << " "; 334 std::cout << "]\n"; 335 336 atf::process::argv_array argva(argv); 337 return atf::check::exec(argva); 338} 339 340static 341std::auto_ptr< atf::check::check_result > 342execute_with_shell(char* const* argv) 343{ 344 const std::string cmd = flatten_argv(argv); 345 346 const char* sh_argv[4]; 347 sh_argv[0] = atf::config::get("atf_shell").c_str(); 348 sh_argv[1] = "-c"; 349 sh_argv[2] = cmd.c_str(); 350 sh_argv[3] = NULL; 351 return execute(sh_argv); 352} 353 354static 355void 356cat_file(const atf::fs::path& path) 357{ 358 std::ifstream stream(path.c_str()); 359 if (!stream) 360 throw std::runtime_error("Failed to open " + path.str()); 361 362 stream >> std::noskipws; 363 std::istream_iterator< char > begin(stream), end; 364 std::ostream_iterator< char > out(std::cerr); 365 std::copy(begin, end, out); 366 367 stream.close(); 368} 369 370static 371bool 372grep_file(const atf::fs::path& path, const std::string& regexp) 373{ 374 std::ifstream stream(path.c_str()); 375 if (!stream) 376 throw std::runtime_error("Failed to open " + path.str()); 377 378 bool found = false; 379 380 std::string line; 381 while (!found && std::getline(stream, line).good()) { 382 if (atf::text::match(line, regexp)) 383 found = true; 384 } 385 386 stream.close(); 387 388 return found; 389} 390 391static 392bool 393file_empty(const atf::fs::path& p) 394{ 395 atf::fs::file_info f(p); 396 397 return (f.get_size() == 0); 398} 399 400static bool 401compare_files(const atf::fs::path& p1, const atf::fs::path& p2) 402{ 403 bool equal = false; 404 405 std::ifstream f1(p1.c_str()); 406 if (!f1) 407 throw std::runtime_error("Failed to open " + p1.str()); 408 409 std::ifstream f2(p2.c_str()); 410 if (!f2) 411 throw std::runtime_error("Failed to open " + p1.str()); 412 413 for (;;) { 414 char buf1[512], buf2[512]; 415 416 f1.read(buf1, sizeof(buf1)); 417 if (f1.bad()) 418 throw std::runtime_error("Failed to read from " + p1.str()); 419 420 f2.read(buf2, sizeof(buf2)); 421 if (f2.bad()) 422 throw std::runtime_error("Failed to read from " + p1.str()); 423 424 std::cout << "1 read: " << f1.gcount() << "\n"; 425 std::cout << "2 read: " << f2.gcount() << "\n"; 426 if ((f1.gcount() == 0) && (f2.gcount() == 0)) { 427 equal = true; 428 break; 429 } 430 431 if ((f1.gcount() != f2.gcount()) || 432 (std::memcmp(buf1, buf2, f1.gcount()) != 0)) { 433 break; 434 } 435 } 436 437 return equal; 438} 439 440static 441void 442print_diff(const atf::fs::path& p1, const atf::fs::path& p2) 443{ 444 const atf::process::status s = 445 atf::process::exec(atf::fs::path("diff"), 446 atf::process::argv_array("diff", "-u", p1.c_str(), 447 p2.c_str(), NULL), 448 atf::process::stream_connect(STDOUT_FILENO, 449 STDERR_FILENO), 450 atf::process::stream_inherit()); 451 452 if (!s.exited()) 453 std::cerr << "Failed to run diff(3)\n"; 454 455 if (s.exitstatus() != 1) 456 std::cerr << "Error while running diff(3)\n"; 457} 458 459static 460std::string 461decode(const std::string& s) 462{ 463 size_t i; 464 std::string res; 465 466 res.reserve(s.length()); 467 468 i = 0; 469 while (i < s.length()) { 470 char c = s[i++]; 471 472 if (c == '\\') { 473 switch (s[i++]) { 474 case 'a': c = '\a'; break; 475 case 'b': c = '\b'; break; 476 case 'c': break; 477 case 'e': c = 033; break; 478 case 'f': c = '\f'; break; 479 case 'n': c = '\n'; break; 480 case 'r': c = '\r'; break; 481 case 't': c = '\t'; break; 482 case 'v': c = '\v'; break; 483 case '\\': break; 484 case '0': 485 { 486 int count = 3; 487 c = 0; 488 while (--count >= 0 && (unsigned)(s[i] - '0') < 8) 489 c = (c << 3) + (s[i++] - '0'); 490 break; 491 } 492 default: 493 --i; 494 break; 495 } 496 } 497 498 res.push_back(c); 499 } 500 501 return res; 502} 503 504static 505bool 506run_status_check(const status_check& sc, const atf::check::check_result& cr) 507{ 508 bool result; 509 510 if (sc.type == sc_exit) { 511 if (cr.exited() && sc.value != INT_MIN) { 512 const int status = cr.exitcode(); 513 514 if (!sc.negated && sc.value != status) { 515 std::cerr << "Fail: incorrect exit status: " 516 << status << ", expected: " 517 << sc.value << "\n"; 518 result = false; 519 } else if (sc.negated && sc.value == status) { 520 std::cerr << "Fail: incorrect exit status: " 521 << status << ", expected: " 522 << "anything else\n"; 523 result = false; 524 } else 525 result = true; 526 } else if (cr.exited() && sc.value == INT_MIN) { 527 result = true; 528 } else { 529 std::cerr << "Fail: program did not exit cleanly\n"; 530 result = false; 531 } 532 } else if (sc.type == sc_ignore) { 533 result = true; 534 } else if (sc.type == sc_signal) { 535 if (cr.signaled() && sc.value != INT_MIN) { 536 const int status = cr.termsig(); 537 538 if (!sc.negated && sc.value != status) { 539 std::cerr << "Fail: incorrect signal received: " 540 << status << ", expected: " << sc.value << "\n"; 541 result = false; 542 } else if (sc.negated && sc.value == status) { 543 std::cerr << "Fail: incorrect signal received: " 544 << status << ", expected: " 545 << "anything else\n"; 546 result = false; 547 } else 548 result = true; 549 } else if (cr.signaled() && sc.value == INT_MIN) { 550 result = true; 551 } else { 552 std::cerr << "Fail: program did not receive a signal\n"; 553 result = false; 554 } 555 } else { 556 UNREACHABLE; 557 result = false; 558 } 559 560 if (result == false) { 561 std::cerr << "stdout:\n"; 562 cat_file(atf::fs::path(cr.stdout_path())); 563 std::cerr << "\n"; 564 565 std::cerr << "stderr:\n"; 566 cat_file(atf::fs::path(cr.stderr_path())); 567 std::cerr << "\n"; 568 } 569 570 return result; 571} 572 573static 574bool 575run_status_checks(const std::vector< status_check >& checks, 576 const atf::check::check_result& result) 577{ 578 bool ok = false; 579 580 for (std::vector< status_check >::const_iterator iter = checks.begin(); 581 !ok && iter != checks.end(); iter++) { 582 ok |= run_status_check(*iter, result); 583 } 584 585 return ok; 586} 587 588static 589bool 590run_output_check(const output_check oc, const atf::fs::path& path, 591 const std::string& stdxxx) 592{ 593 bool result; 594 595 if (oc.type == oc_empty) { 596 const bool is_empty = file_empty(path); 597 if (!oc.negated && !is_empty) { 598 std::cerr << "Fail: " << stdxxx << " not empty\n"; 599 print_diff(atf::fs::path("/dev/null"), path); 600 result = false; 601 } else if (oc.negated && is_empty) { 602 std::cerr << "Fail: " << stdxxx << " is empty\n"; 603 result = false; 604 } else 605 result = true; 606 } else if (oc.type == oc_file) { 607 const bool equals = compare_files(path, atf::fs::path(oc.value)); 608 if (!oc.negated && !equals) { 609 std::cerr << "Fail: " << stdxxx << " does not match golden " 610 "output\n"; 611 print_diff(atf::fs::path(oc.value), path); 612 result = false; 613 } else if (oc.negated && equals) { 614 std::cerr << "Fail: " << stdxxx << " matches golden output\n"; 615 cat_file(atf::fs::path(oc.value)); 616 result = false; 617 } else 618 result = true; 619 } else if (oc.type == oc_ignore) { 620 result = true; 621 } else if (oc.type == oc_inline) { 622 atf::fs::path path2 = atf::fs::path(atf::config::get("atf_workdir")) 623 / "inline.XXXXXX"; 624 temp_file temp(path2); 625 temp.write(decode(oc.value)); 626 temp.close(); 627 628 const bool equals = compare_files(path, temp.get_path()); 629 if (!oc.negated && !equals) { 630 std::cerr << "Fail: " << stdxxx << " does not match expected " 631 "value\n"; 632 print_diff(temp.get_path(), path); 633 result = false; 634 } else if (oc.negated && equals) { 635 std::cerr << "Fail: " << stdxxx << " matches expected value\n"; 636 cat_file(temp.get_path()); 637 result = false; 638 } else 639 result = true; 640 } else if (oc.type == oc_match) { 641 const bool matches = grep_file(path, oc.value); 642 if (!oc.negated && !matches) { 643 std::cerr << "Fail: regexp " + oc.value + " not in " << stdxxx 644 << "\n"; 645 cat_file(path); 646 result = false; 647 } else if (oc.negated && matches) { 648 std::cerr << "Fail: regexp " + oc.value + " is in " << stdxxx 649 << "\n"; 650 cat_file(path); 651 result = false; 652 } else 653 result = true; 654 } else if (oc.type == oc_save) { 655 INV(!oc.negated); 656 std::ifstream ifs(path.c_str(), std::fstream::binary); 657 ifs >> std::noskipws; 658 std::istream_iterator< char > begin(ifs), end; 659 660 std::ofstream ofs(oc.value.c_str(), std::fstream::binary 661 | std::fstream::trunc); 662 std::ostream_iterator <char> obegin(ofs); 663 664 std::copy(begin, end, obegin); 665 result = true; 666 } else { 667 UNREACHABLE; 668 result = false; 669 } 670 671 return result; 672} 673 674static 675bool 676run_output_checks(const std::vector< output_check >& checks, 677 const atf::fs::path& path, const std::string& stdxxx) 678{ 679 bool ok = true; 680 681 for (std::vector< output_check >::const_iterator iter = checks.begin(); 682 iter != checks.end(); iter++) { 683 ok &= run_output_check(*iter, path, stdxxx); 684 } 685 686 return ok; 687} 688 689// ------------------------------------------------------------------------ 690// The "atf_check" application. 691// ------------------------------------------------------------------------ 692 693namespace { 694 695class atf_check : public atf::application::app { 696 bool m_xflag; 697 698 std::vector< status_check > m_status_checks; 699 std::vector< output_check > m_stdout_checks; 700 std::vector< output_check > m_stderr_checks; 701 702 static const char* m_description; 703 704 bool run_output_checks(const atf::check::check_result&, 705 const std::string&) const; 706 707 std::string specific_args(void) const; 708 options_set specific_options(void) const; 709 void process_option(int, const char*); 710 void process_option_s(const std::string&); 711 712public: 713 atf_check(void); 714 int main(void); 715}; 716 717} // anonymous namespace 718 719const char* atf_check::m_description = 720 "atf-check executes given command and analyzes its results."; 721 722atf_check::atf_check(void) : 723 app(m_description, "atf-check(1)", "atf(7)"), 724 m_xflag(false) 725{ 726} 727 728bool 729atf_check::run_output_checks(const atf::check::check_result& r, 730 const std::string& stdxxx) 731 const 732{ 733 if (stdxxx == "stdout") { 734 return ::run_output_checks(m_stdout_checks, 735 atf::fs::path(r.stdout_path()), "stdout"); 736 } else if (stdxxx == "stderr") { 737 return ::run_output_checks(m_stderr_checks, 738 atf::fs::path(r.stderr_path()), "stderr"); 739 } else { 740 UNREACHABLE; 741 return false; 742 } 743} 744 745std::string 746atf_check::specific_args(void) 747 const 748{ 749 return "<command>"; 750} 751 752atf_check::options_set 753atf_check::specific_options(void) 754 const 755{ 756 using atf::application::option; 757 options_set opts; 758 759 opts.insert(option('s', "qual:value", "Handle status. Qualifier " 760 "must be one of: ignore exit:<num> signal:<name|num>")); 761 opts.insert(option('o', "action:arg", "Handle stdout. Action must be " 762 "one of: empty ignore file:<path> inline:<val> match:regexp " 763 "save:<path>")); 764 opts.insert(option('e', "action:arg", "Handle stderr. Action must be " 765 "one of: empty ignore file:<path> inline:<val> match:regexp " 766 "save:<path>")); 767 opts.insert(option('x', "", "Execute command as a shell command")); 768 769 return opts; 770} 771 772void 773atf_check::process_option(int ch, const char* arg) 774{ 775 switch (ch) { 776 case 's': 777 m_status_checks.push_back(parse_status_check_arg(arg)); 778 break; 779 780 case 'o': 781 m_stdout_checks.push_back(parse_output_check_arg(arg)); 782 break; 783 784 case 'e': 785 m_stderr_checks.push_back(parse_output_check_arg(arg)); 786 break; 787 788 case 'x': 789 m_xflag = true; 790 break; 791 792 default: 793 UNREACHABLE; 794 } 795} 796 797int 798atf_check::main(void) 799{ 800 if (m_argc < 1) 801 throw atf::application::usage_error("No command specified"); 802 803 int status = EXIT_FAILURE; 804 805 std::auto_ptr< atf::check::check_result > r = 806 m_xflag ? execute_with_shell(m_argv) : execute(m_argv); 807 808 if (m_status_checks.empty()) 809 m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS)); 810 else if (m_status_checks.size() > 1) { 811 // TODO: Remove this restriction. 812 throw atf::application::usage_error("Cannot specify -s more than once"); 813 } 814 815 if (m_stdout_checks.empty()) 816 m_stdout_checks.push_back(output_check(oc_empty, false, "")); 817 if (m_stderr_checks.empty()) 818 m_stderr_checks.push_back(output_check(oc_empty, false, "")); 819 820 if ((run_status_checks(m_status_checks, *r) == false) || 821 (run_output_checks(*r, "stderr") == false) || 822 (run_output_checks(*r, "stdout") == false)) 823 status = EXIT_FAILURE; 824 else 825 status = EXIT_SUCCESS; 826 827 return status; 828} 829 830int 831main(int argc, char* const* argv) 832{ 833 return atf_check().run(argc, argv); 834} 835