1// Copyright 2010 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/atf_result.hpp" 30 31#include <cstdlib> 32#include <fstream> 33#include <utility> 34 35#include "engine/exceptions.hpp" 36#include "model/test_result.hpp" 37#include "utils/fs/path.hpp" 38#include "utils/format/macros.hpp" 39#include "utils/optional.ipp" 40#include "utils/process/status.hpp" 41#include "utils/sanity.hpp" 42#include "utils/text/exceptions.hpp" 43#include "utils/text/operations.ipp" 44 45namespace fs = utils::fs; 46namespace process = utils::process; 47namespace text = utils::text; 48 49using utils::none; 50using utils::optional; 51 52 53namespace { 54 55 56/// Reads a file and flattens its lines. 57/// 58/// The main purpose of this function is to simplify the parsing of a file 59/// containing the result of a test. Therefore, the return value carries 60/// several assumptions. 61/// 62/// \param input The stream to read from. 63/// 64/// \return A pair (line count, contents) detailing how many lines where read 65/// and their contents. If the file contains a single line with no newline 66/// character, the line count is 0. If the file includes more than one line, 67/// the lines are merged together and separated by the magic string 68/// '<<NEWLINE>>'. 69static std::pair< size_t, std::string > 70read_lines(std::istream& input) 71{ 72 std::pair< size_t, std::string > ret = std::make_pair(0, ""); 73 74 do { 75 std::string line; 76 std::getline(input, line); 77 if (input.eof() && !line.empty()) { 78 if (ret.first == 0) 79 ret.second = line; 80 else { 81 ret.second += "<<NEWLINE>>" + line; 82 ret.first++; 83 } 84 } else if (input.good()) { 85 if (ret.first == 0) 86 ret.second = line; 87 else 88 ret.second += "<<NEWLINE>>" + line; 89 ret.first++; 90 } 91 } while (input.good()); 92 93 return ret; 94} 95 96 97/// Parses a test result that does not accept a reason. 98/// 99/// \param status The result status name. 100/// \param rest The rest of the line after the status name. 101/// 102/// \return An object representing the test result. 103/// 104/// \throw format_error If the result is invalid (i.e. rest is invalid). 105/// 106/// \pre status must be "passed". 107static engine::atf_result 108parse_without_reason(const std::string& status, const std::string& rest) 109{ 110 if (!rest.empty()) 111 throw engine::format_error(F("%s cannot have a reason") % status); 112 PRE(status == "passed"); 113 return engine::atf_result(engine::atf_result::passed); 114} 115 116 117/// Parses a test result that needs a reason. 118/// 119/// \param status The result status name. 120/// \param rest The rest of the line after the status name. 121/// 122/// \return An object representing the test result. 123/// 124/// \throw format_error If the result is invalid (i.e. rest is invalid). 125/// 126/// \pre status must be one of "broken", "expected_death", "expected_failure", 127/// "expected_timeout", "failed" or "skipped". 128static engine::atf_result 129parse_with_reason(const std::string& status, const std::string& rest) 130{ 131 using engine::atf_result; 132 133 if (rest.length() < 3 || rest.substr(0, 2) != ": ") 134 throw engine::format_error(F("%s must be followed by ': <reason>'") % 135 status); 136 const std::string reason = rest.substr(2); 137 INV(!reason.empty()); 138 139 if (status == "broken") 140 return atf_result(atf_result::broken, reason); 141 else if (status == "expected_death") 142 return atf_result(atf_result::expected_death, reason); 143 else if (status == "expected_failure") 144 return atf_result(atf_result::expected_failure, reason); 145 else if (status == "expected_timeout") 146 return atf_result(atf_result::expected_timeout, reason); 147 else if (status == "failed") 148 return atf_result(atf_result::failed, reason); 149 else if (status == "skipped") 150 return atf_result(atf_result::skipped, reason); 151 else 152 PRE_MSG(false, "Unexpected status"); 153} 154 155 156/// Converts a string to an integer. 157/// 158/// \param str The string containing the integer to convert. 159/// 160/// \return The converted integer; none if the parsing fails. 161static optional< int > 162parse_int(const std::string& str) 163{ 164 try { 165 return utils::make_optional(text::to_type< int >(str)); 166 } catch (const text::value_error& e) { 167 return none; 168 } 169} 170 171 172/// Parses a test result that needs a reason and accepts an optional integer. 173/// 174/// \param status The result status name. 175/// \param rest The rest of the line after the status name. 176/// 177/// \return The parsed test result if the data is valid, or a broken result if 178/// the parsing failed. 179/// 180/// \pre status must be one of "expected_exit" or "expected_signal". 181static engine::atf_result 182parse_with_reason_and_arg(const std::string& status, const std::string& rest) 183{ 184 using engine::atf_result; 185 186 std::string::size_type delim = rest.find_first_of(":("); 187 if (delim == std::string::npos) 188 throw engine::format_error(F("Invalid format for '%s' test case " 189 "result; must be followed by '[(num)]: " 190 "<reason>' but found '%s'") % 191 status % rest); 192 193 optional< int > arg; 194 if (rest[delim] == '(') { 195 const std::string::size_type delim2 = rest.find("):", delim); 196 if (delim == std::string::npos) 197 throw engine::format_error(F("Mismatched '(' in %s") % rest); 198 199 const std::string argstr = rest.substr(delim + 1, delim2 - delim - 1); 200 arg = parse_int(argstr); 201 if (!arg) 202 throw engine::format_error(F("Invalid integer argument '%s' to " 203 "'%s' test case result") % 204 argstr % status); 205 delim = delim2 + 1; 206 } 207 208 const std::string reason = rest.substr(delim + 2); 209 210 if (status == "expected_exit") 211 return atf_result(atf_result::expected_exit, arg, reason); 212 else if (status == "expected_signal") 213 return atf_result(atf_result::expected_signal, arg, reason); 214 else 215 PRE_MSG(false, "Unexpected status"); 216} 217 218 219/// Formats the termination status of a process to be used with validate_result. 220/// 221/// \param status The status to format. 222/// 223/// \return A string describing the status. 224static std::string 225format_status(const process::status& status) 226{ 227 if (status.exited()) 228 return F("exited with code %s") % status.exitstatus(); 229 else if (status.signaled()) 230 return F("received signal %s%s") % status.termsig() % 231 (status.coredump() ? " (core dumped)" : ""); 232 else 233 return F("terminated in an unknown manner"); 234} 235 236 237} // anonymous namespace 238 239 240/// Constructs a raw result with a type. 241/// 242/// The reason and the argument are left uninitialized. 243/// 244/// \param type_ The type of the result. 245engine::atf_result::atf_result(const types type_) : 246 _type(type_) 247{ 248} 249 250 251/// Constructs a raw result with a type and a reason. 252/// 253/// The argument is left uninitialized. 254/// 255/// \param type_ The type of the result. 256/// \param reason_ The reason for the result. 257engine::atf_result::atf_result(const types type_, const std::string& reason_) : 258 _type(type_), _reason(reason_) 259{ 260} 261 262 263/// Constructs a raw result with a type, an optional argument and a reason. 264/// 265/// \param type_ The type of the result. 266/// \param argument_ The optional argument for the result. 267/// \param reason_ The reason for the result. 268engine::atf_result::atf_result(const types type_, 269 const utils::optional< int >& argument_, 270 const std::string& reason_) : 271 _type(type_), _argument(argument_), _reason(reason_) 272{ 273} 274 275 276/// Parses an input stream to extract a test result. 277/// 278/// If the parsing fails for any reason, the test result is 'broken' and it 279/// contains the reason for the parsing failure. Test cases that report results 280/// in an inconsistent state cannot be trusted (e.g. the test program code may 281/// have a bug), and thus why they are reported as broken instead of just failed 282/// (which is a legitimate result for a test case). 283/// 284/// \param input The stream to read from. 285/// 286/// \return A generic representation of the result of the test case. 287/// 288/// \throw format_error If the input is invalid. 289engine::atf_result 290engine::atf_result::parse(std::istream& input) 291{ 292 const std::pair< size_t, std::string > data = read_lines(input); 293 if (data.first == 0) 294 throw format_error("Empty test result or no new line"); 295 else if (data.first > 1) 296 throw format_error("Test result contains multiple lines: " + 297 data.second); 298 else { 299 const std::string::size_type delim = data.second.find_first_not_of( 300 "abcdefghijklmnopqrstuvwxyz_"); 301 const std::string status = data.second.substr(0, delim); 302 const std::string rest = data.second.substr(status.length()); 303 304 if (status == "broken") 305 return parse_with_reason(status, rest); 306 else if (status == "expected_death") 307 return parse_with_reason(status, rest); 308 else if (status == "expected_exit") 309 return parse_with_reason_and_arg(status, rest); 310 else if (status == "expected_failure") 311 return parse_with_reason(status, rest); 312 else if (status == "expected_signal") 313 return parse_with_reason_and_arg(status, rest); 314 else if (status == "expected_timeout") 315 return parse_with_reason(status, rest); 316 else if (status == "failed") 317 return parse_with_reason(status, rest); 318 else if (status == "passed") 319 return parse_without_reason(status, rest); 320 else if (status == "skipped") 321 return parse_with_reason(status, rest); 322 else 323 throw format_error(F("Unknown test result '%s'") % status); 324 } 325} 326 327 328/// Loads a test case result from a file. 329/// 330/// \param file The file to parse. 331/// 332/// \return The parsed test case result if all goes well. 333/// 334/// \throw std::runtime_error If the file does not exist. 335/// \throw engine::format_error If the contents of the file are bogus. 336engine::atf_result 337engine::atf_result::load(const fs::path& file) 338{ 339 std::ifstream input(file.c_str()); 340 if (!input) 341 throw std::runtime_error("Cannot open results file"); 342 else 343 return parse(input); 344} 345 346 347/// Gets the type of the result. 348/// 349/// \return A result type. 350engine::atf_result::types 351engine::atf_result::type(void) const 352{ 353 return _type; 354} 355 356 357/// Gets the optional argument of the result. 358/// 359/// \return The argument of the result if present; none otherwise. 360const optional< int >& 361engine::atf_result::argument(void) const 362{ 363 return _argument; 364} 365 366 367/// Gets the optional reason of the result. 368/// 369/// \return The reason of the result if present; none otherwise. 370const optional< std::string >& 371engine::atf_result::reason(void) const 372{ 373 return _reason; 374} 375 376 377/// Checks whether the result should be reported as good or not. 378/// 379/// \return True if the result can be considered "good", false otherwise. 380bool 381engine::atf_result::good(void) const 382{ 383 switch (_type) { 384 case atf_result::expected_death: 385 case atf_result::expected_exit: 386 case atf_result::expected_failure: 387 case atf_result::expected_signal: 388 case atf_result::expected_timeout: 389 case atf_result::passed: 390 case atf_result::skipped: 391 return true; 392 393 case atf_result::broken: 394 case atf_result::failed: 395 return false; 396 397 default: 398 UNREACHABLE; 399 } 400} 401 402 403/// Reinterprets a raw result based on the termination status of the test case. 404/// 405/// This reinterpretation ensures that the termination conditions of the program 406/// match what is expected of the paticular result reported by the test program. 407/// If such conditions do not match, the test program is considered bogus and is 408/// thus reported as broken. 409/// 410/// This is just a helper function for calculate_result(); the real result of 411/// the test case cannot be inferred from apply() only. 412/// 413/// \param status The exit status of the test program, or none if the test 414/// program timed out. 415/// 416/// \result The adjusted result. The original result is transformed into broken 417/// if the exit status of the program does not match our expectations. 418engine::atf_result 419engine::atf_result::apply(const optional< process::status >& status) 420 const 421{ 422 if (!status) { 423 if (_type != atf_result::expected_timeout) 424 return atf_result(atf_result::broken, "Test case body timed out"); 425 else 426 return *this; 427 } 428 429 INV(status); 430 switch (_type) { 431 case atf_result::broken: 432 return *this; 433 434 case atf_result::expected_death: 435 return *this; 436 437 case atf_result::expected_exit: 438 if (status.get().exited()) { 439 if (_argument) { 440 if (_argument.get() == status.get().exitstatus()) 441 return *this; 442 else 443 return atf_result( 444 atf_result::failed, 445 F("Test case expected to exit with code %s but got " 446 "code %s") % 447 _argument.get() % status.get().exitstatus()); 448 } else 449 return *this; 450 } else 451 return atf_result(atf_result::broken, "Expected clean exit but " + 452 format_status(status.get())); 453 454 case atf_result::expected_failure: 455 if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) 456 return *this; 457 else 458 return atf_result(atf_result::broken, "Expected failure should " 459 "have reported success but " + 460 format_status(status.get())); 461 462 case atf_result::expected_signal: 463 if (status.get().signaled()) { 464 if (_argument) { 465 if (_argument.get() == status.get().termsig()) 466 return *this; 467 else 468 return atf_result( 469 atf_result::failed, 470 F("Test case expected to receive signal %s but " 471 "got %s") % 472 _argument.get() % status.get().termsig()); 473 } else 474 return *this; 475 } else 476 return atf_result(atf_result::broken, "Expected signal but " + 477 format_status(status.get())); 478 479 case atf_result::expected_timeout: 480 return atf_result(atf_result::broken, "Expected timeout but " + 481 format_status(status.get())); 482 483 case atf_result::failed: 484 if (status.get().exited() && status.get().exitstatus() == EXIT_FAILURE) 485 return *this; 486 else 487 return atf_result(atf_result::broken, "Failed test case should " 488 "have reported failure but " + 489 format_status(status.get())); 490 491 case atf_result::passed: 492 if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) 493 return *this; 494 else 495 return atf_result(atf_result::broken, "Passed test case should " 496 "have reported success but " + 497 format_status(status.get())); 498 499 case atf_result::skipped: 500 if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) 501 return *this; 502 else 503 return atf_result(atf_result::broken, "Skipped test case should " 504 "have reported success but " + 505 format_status(status.get())); 506 } 507 508 UNREACHABLE; 509} 510 511 512/// Converts an internal result to the interface-agnostic representation. 513/// 514/// \return A generic result instance representing this result. 515model::test_result 516engine::atf_result::externalize(void) const 517{ 518 switch (_type) { 519 case atf_result::broken: 520 return model::test_result(model::test_result_broken, _reason.get()); 521 522 case atf_result::expected_death: 523 case atf_result::expected_exit: 524 case atf_result::expected_failure: 525 case atf_result::expected_signal: 526 case atf_result::expected_timeout: 527 return model::test_result(model::test_result_expected_failure, 528 _reason.get()); 529 530 case atf_result::failed: 531 return model::test_result(model::test_result_failed, _reason.get()); 532 533 case atf_result::passed: 534 return model::test_result(model::test_result_passed); 535 536 case atf_result::skipped: 537 return model::test_result(model::test_result_skipped, _reason.get()); 538 539 default: 540 UNREACHABLE; 541 } 542} 543 544 545/// Compares two raw results for equality. 546/// 547/// \param other The result to compare to. 548/// 549/// \return True if the two raw results are equal; false otherwise. 550bool 551engine::atf_result::operator==(const atf_result& other) const 552{ 553 return _type == other._type && _argument == other._argument && 554 _reason == other._reason; 555} 556 557 558/// Compares two raw results for inequality. 559/// 560/// \param other The result to compare to. 561/// 562/// \return True if the two raw results are different; false otherwise. 563bool 564engine::atf_result::operator!=(const atf_result& other) const 565{ 566 return !(*this == other); 567} 568 569 570/// Injects the object into a stream. 571/// 572/// \param output The stream into which to inject the object. 573/// \param object The object to format. 574/// 575/// \return The output stream. 576std::ostream& 577engine::operator<<(std::ostream& output, const atf_result& object) 578{ 579 std::string result_name; 580 switch (object.type()) { 581 case atf_result::broken: result_name = "broken"; break; 582 case atf_result::expected_death: result_name = "expected_death"; break; 583 case atf_result::expected_exit: result_name = "expected_exit"; break; 584 case atf_result::expected_failure: result_name = "expected_failure"; break; 585 case atf_result::expected_signal: result_name = "expected_signal"; break; 586 case atf_result::expected_timeout: result_name = "expected_timeout"; break; 587 case atf_result::failed: result_name = "failed"; break; 588 case atf_result::passed: result_name = "passed"; break; 589 case atf_result::skipped: result_name = "skipped"; break; 590 } 591 592 const optional< int >& argument = object.argument(); 593 594 const optional< std::string >& reason = object.reason(); 595 596 output << F("model::test_result{type=%s, argument=%s, reason=%s}") 597 % text::quote(result_name, '\'') 598 % (argument ? (F("%s") % argument.get()).str() : "none") 599 % (reason ? text::quote(reason.get(), '\'') : "none"); 600 601 return output; 602} 603 604 605/// Calculates the user-visible result of a test case. 606/// 607/// This function needs to perform magic to ensure that what the test case 608/// reports as its result is what the user should really see: i.e. it adjusts 609/// the reported status of the test to the exit conditions of its body and 610/// cleanup parts. 611/// 612/// \param body_status The termination status of the process that executed 613/// the body of the test. None if the body timed out. 614/// \param results_file The path to the results file that the test case body is 615/// supposed to have created. 616/// 617/// \return The calculated test case result. 618model::test_result 619engine::calculate_atf_result(const optional< process::status >& body_status, 620 const fs::path& results_file) 621{ 622 using engine::atf_result; 623 624 atf_result result(atf_result::broken, "Unknown result"); 625 try { 626 result = atf_result::load(results_file); 627 } catch (const engine::format_error& error) { 628 result = atf_result(atf_result::broken, error.what()); 629 } catch (const std::runtime_error& error) { 630 if (body_status) 631 result = atf_result( 632 atf_result::broken, F("Premature exit; test case %s") % 633 format_status(body_status.get())); 634 else { 635 // The test case timed out. apply() handles this case later. 636 } 637 } 638 639 result = result.apply(body_status); 640 641 return result.externalize(); 642} 643