test_case.cpp revision 1.1.1.2
1// Copyright 2010 Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/test_case.hpp" 30 31extern "C" { 32#include <signal.h> 33} 34 35#include <fstream> 36 37#include "engine/config.hpp" 38#include "engine/exceptions.hpp" 39#include "engine/test_program.hpp" 40#include "engine/test_result.hpp" 41#include "engine/testers.hpp" 42#include "utils/config/tree.ipp" 43#include "utils/datetime.hpp" 44#include "utils/defs.hpp" 45#include "utils/format/macros.hpp" 46#include "utils/logging/operations.hpp" 47#include "utils/fs/auto_cleaners.hpp" 48#include "utils/fs/operations.hpp" 49#include "utils/fs/path.hpp" 50#include "utils/optional.ipp" 51#include "utils/passwd.hpp" 52#include "utils/text/operations.ipp" 53 54namespace config = utils::config; 55namespace fs = utils::fs; 56namespace logging = utils::logging; 57namespace passwd = utils::passwd; 58namespace text = utils::text; 59 60using utils::none; 61using utils::optional; 62 63 64namespace { 65 66 67/// Generates the set of configuration variables for the tester. 68/// 69/// \param metadata The metadata of the test. 70/// \param user_config The configuration variables provided by the user. 71/// \param test_suite The name of the test suite. 72/// 73/// \return The mapping of configuration variables for the tester. 74static config::properties_map 75generate_tester_config(const engine::metadata& metadata, 76 const config::tree& user_config, 77 const std::string& test_suite) 78{ 79 config::properties_map props; 80 81 try { 82 props = user_config.all_properties(F("test_suites.%s") % test_suite, 83 true); 84 } catch (const config::unknown_key_error& unused_error) { 85 // Ignore: not all test suites have entries in the configuration. 86 } 87 88 if (user_config.is_set("unprivileged_user")) { 89 const passwd::user& user = 90 user_config.lookup< engine::user_node >("unprivileged_user"); 91 props["unprivileged-user"] = user.name; 92 } 93 94 // TODO(jmmv): This is an ugly hack to cope with an atf-specific 95 // property. We should not be doing this at all, so just consider this 96 // a temporary optimization... 97 if (metadata.has_cleanup()) 98 props["has.cleanup"] = "true"; 99 else 100 props["has.cleanup"] = "false"; 101 102 return props; 103} 104 105 106/// Creates a tester. 107/// 108/// \param interface_name The name of the tester interface to use. 109/// \param metadata Metadata of the test case. 110/// \param user_config User-provided configuration variables. 111/// 112/// \return The created tester, on which the test() method can be executed. 113static engine::tester 114create_tester(const std::string& interface_name, 115 const engine::metadata& metadata, const config::tree& user_config) 116{ 117 optional< passwd::user > user; 118 if (user_config.is_set("unprivileged_user") && 119 metadata.required_user() == "unprivileged") 120 user = user_config.lookup< engine::user_node >("unprivileged_user"); 121 122 return engine::tester(interface_name, user, 123 utils::make_optional(metadata.timeout())); 124} 125 126 127} // anonymous namespace 128 129 130/// Destructor. 131engine::test_case_hooks::~test_case_hooks(void) 132{ 133} 134 135 136/// Called once the test case's stdout is ready for processing. 137/// 138/// It is important to note that this file is only available within this 139/// callback. Attempting to read the file once the execute function has 140/// returned will result in an error because the file might have been deleted. 141/// 142/// \param unused_file The path to the file containing the stdout. 143void 144engine::test_case_hooks::got_stdout(const fs::path& UTILS_UNUSED_PARAM(file)) 145{ 146} 147 148 149/// Called once the test case's stderr is ready for processing. 150/// 151/// It is important to note that this file is only available within this 152/// callback. Attempting to read the file once the execute function has 153/// returned will result in an error because the file might have been deleted. 154/// 155/// \param unused_file The path to the file containing the stderr. 156void 157engine::test_case_hooks::got_stderr(const fs::path& UTILS_UNUSED_PARAM(file)) 158{ 159} 160 161 162/// Internal implementation for a test_case. 163struct engine::test_case::impl { 164 /// Name of the interface implemented by the test program. 165 const std::string interface_name; 166 167 /// Test program this test case belongs to. 168 const test_program& _test_program; 169 170 /// Name of the test case; must be unique within the test program. 171 std::string name; 172 173 /// Test case metadata. 174 metadata md; 175 176 /// Fake result to return instead of running the test case. 177 optional< test_result > fake_result; 178 179 /// Constructor. 180 /// 181 /// \param interface_name_ Name of the interface implemented by the test 182 /// program. 183 /// \param test_program_ The test program this test case belongs to. 184 /// \param name_ The name of the test case within the test program. 185 /// \param md_ Metadata of the test case. 186 /// \param fake_result_ Fake result to return instead of running the test 187 /// case. 188 impl(const std::string& interface_name_, 189 const test_program& test_program_, 190 const std::string& name_, 191 const metadata& md_, 192 const optional< test_result >& fake_result_) : 193 interface_name(interface_name_), 194 _test_program(test_program_), 195 name(name_), 196 md(md_), 197 fake_result(fake_result_) 198 { 199 } 200 201 /// Equality comparator. 202 /// 203 /// \param other The other object to compare this one to. 204 /// 205 /// \return True if this object and other are equal; false otherwise. 206 bool 207 operator==(const impl& other) const 208 { 209 return (interface_name == other.interface_name && 210 (_test_program.absolute_path() == 211 other._test_program.absolute_path()) && 212 name == other.name && 213 md == other.md && 214 fake_result == other.fake_result); 215 } 216}; 217 218 219/// Constructs a new test case. 220/// 221/// \param interface_name_ Name of the interface implemented by the test 222/// program. 223/// \param test_program_ The test program this test case belongs to. This is a 224/// static reference (instead of a test_program_ptr) because the test 225/// program must exist in order for the test case to exist. 226/// \param name_ The name of the test case within the test program. Must be 227/// unique. 228/// \param md_ Metadata of the test case. 229engine::test_case::test_case(const std::string& interface_name_, 230 const test_program& test_program_, 231 const std::string& name_, 232 const metadata& md_) : 233 _pimpl(new impl(interface_name_, test_program_, name_, md_, none)) 234{ 235} 236 237 238 239/// Constructs a new fake test case. 240/// 241/// A fake test case is a test case that is not really defined by the test 242/// program. Such test cases have a name surrounded by '__' and, when executed, 243/// they return a fixed, pre-recorded result. 244/// 245/// This is necessary for the cases where listing the test cases of a test 246/// program fails. In this scenario, we generate a single test case within 247/// the test program that unconditionally returns a failure. 248/// 249/// TODO(jmmv): Need to get rid of this. We should be able to report the 250/// status of test programs independently of test cases, as some interfaces 251/// don't know about the latter at all. 252/// 253/// \param interface_name_ Name of the interface implemented by the test 254/// program. 255/// \param test_program_ The test program this test case belongs to. 256/// \param name_ The name to give to this fake test case. This name has to be 257/// prefixed and suffixed by '__' to clearly denote that this is internal. 258/// \param description_ The description of the test case, if any. 259/// \param test_result_ The fake result to return when this test case is run. 260engine::test_case::test_case( 261 const std::string& interface_name_, 262 const test_program& test_program_, 263 const std::string& name_, 264 const std::string& description_, 265 const engine::test_result& test_result_) : 266 _pimpl(new impl(interface_name_, test_program_, name_, 267 metadata_builder().set_description(description_).build(), 268 utils::make_optional(test_result_))) 269{ 270 PRE_MSG(name_.length() > 4 && name_.substr(0, 2) == "__" && 271 name_.substr(name_.length() - 2) == "__", 272 "Invalid fake name provided to fake test case"); 273} 274 275 276/// Destroys a test case. 277engine::test_case::~test_case(void) 278{ 279} 280 281 282/// Gets the name of the interface implemented by the test program. 283/// 284/// \return An interface name. 285const std::string& 286engine::test_case::interface_name(void) const 287{ 288 return _pimpl->interface_name; 289} 290 291 292/// Gets the test program this test case belongs to. 293/// 294/// \return A reference to the container test program. 295const engine::test_program& 296engine::test_case::container_test_program(void) const 297{ 298 return _pimpl->_test_program; 299} 300 301 302/// Gets the test case name. 303/// 304/// \return The test case name, relative to the test program. 305const std::string& 306engine::test_case::name(void) const 307{ 308 return _pimpl->name; 309} 310 311 312/// Gets the test case metadata. 313/// 314/// \return The test case metadata. 315const engine::metadata& 316engine::test_case::get_metadata(void) const 317{ 318 return _pimpl->md; 319} 320 321 322/// Gets the fake result pre-stored for this test case. 323/// 324/// \return A fake result, or none if not defined. 325optional< engine::test_result > 326engine::test_case::fake_result(void) const 327{ 328 return _pimpl->fake_result; 329} 330 331 332/// Equality comparator. 333/// 334/// \warning Because test cases reference their container test programs, and 335/// test programs include test cases, we cannot perform a full comparison here: 336/// otherwise, we'd enter an inifinte loop. Therefore, out of necessity, this 337/// does NOT compare whether the container test programs of the affected test 338/// cases are the same. 339/// 340/// \param other The other object to compare this one to. 341/// 342/// \return True if this object and other are equal; false otherwise. 343bool 344engine::test_case::operator==(const test_case& other) const 345{ 346 return _pimpl == other._pimpl || *_pimpl == *other._pimpl; 347} 348 349 350/// Inequality comparator. 351/// 352/// \param other The other object to compare this one to. 353/// 354/// \return True if this object and other are different; false otherwise. 355bool 356engine::test_case::operator!=(const test_case& other) const 357{ 358 return !(*this == other); 359} 360 361 362/// Injects the object into a stream. 363/// 364/// \param output The stream into which to inject the object. 365/// \param object The object to format. 366/// 367/// \return The output stream. 368std::ostream& 369engine::operator<<(std::ostream& output, const test_case& object) 370{ 371 // We skip injecting container_test_program() on purpose to avoid a loop. 372 output << F("test_case{interface=%s, name=%s, metadata=%s}") 373 % text::quote(object.interface_name(), '\'') 374 % text::quote(object.name(), '\'') 375 % object.get_metadata(); 376 return output; 377} 378 379 380/// Runs the test case in debug mode. 381/// 382/// Debug mode gives the caller more control on the execution of the test. It 383/// should not be used for normal execution of tests; instead, call run(). 384/// 385/// \param test_case The test case to debug. 386/// \param user_config The user configuration that defines the execution of this 387/// test case. 388/// \param hooks Hooks to introspect the execution of the test case. 389/// \param work_directory A directory that can be used to place temporary files. 390/// \param stdout_path The file to which to redirect the stdout of the test. 391/// For interactive debugging, '/dev/stdout' is probably a reasonable value. 392/// \param stderr_path The file to which to redirect the stdout of the test. 393/// For interactive debugging, '/dev/stderr' is probably a reasonable value. 394/// 395/// \return The result of the execution of the test case. 396engine::test_result 397engine::debug_test_case(const test_case* test_case, 398 const config::tree& user_config, 399 test_case_hooks& hooks, 400 const fs::path& work_directory, 401 const fs::path& stdout_path, 402 const fs::path& stderr_path) 403{ 404 if (test_case->fake_result()) 405 return test_case->fake_result().get(); 406 407 const std::string skip_reason = check_reqs( 408 test_case->get_metadata(), user_config, 409 test_case->container_test_program().test_suite_name()); 410 if (!skip_reason.empty()) 411 return test_result(test_result::skipped, skip_reason); 412 413 if (!fs::exists(test_case->container_test_program().absolute_path())) 414 return test_result(test_result::broken, "Test program does not exist"); 415 416 const fs::auto_file result_file(work_directory / "result.txt"); 417 418 const engine::test_program& test_program = 419 test_case->container_test_program(); 420 421 try { 422 const engine::tester tester = create_tester( 423 test_program.interface_name(), test_case->get_metadata(), 424 user_config); 425 tester.test(test_program.absolute_path(), test_case->name(), 426 result_file.file(), stdout_path, stderr_path, 427 generate_tester_config(test_case->get_metadata(), 428 user_config, 429 test_program.test_suite_name())); 430 431 hooks.got_stdout(stdout_path); 432 hooks.got_stderr(stderr_path); 433 434 std::ifstream result_input(result_file.file().c_str()); 435 return engine::test_result::parse(result_input); 436 } catch (const std::runtime_error& e) { 437 // One of the possible explanation for us getting here is if the tester 438 // crashes or doesn't behave as expected. We must record any output 439 // from the process so that we can debug it further. 440 hooks.got_stdout(stdout_path); 441 hooks.got_stderr(stderr_path); 442 443 return engine::test_result( 444 engine::test_result::broken, 445 F("Caught unexpected exception: %s") % e.what()); 446 } 447} 448 449 450/// Runs the test case. 451/// 452/// \param test_case The test case to run. 453/// \param user_config The user configuration that defines the execution of this 454/// test case. 455/// \param hooks Hooks to introspect the execution of the test case. 456/// \param work_directory A directory that can be used to place temporary files. 457/// 458/// \return The result of the execution of the test case. 459engine::test_result 460engine::run_test_case(const test_case* test_case, 461 const config::tree& user_config, 462 test_case_hooks& hooks, 463 const fs::path& work_directory) 464{ 465 if (test_case->fake_result()) 466 return test_case->fake_result().get(); 467 468 const std::string skip_reason = check_reqs( 469 test_case->get_metadata(), user_config, 470 test_case->container_test_program().test_suite_name()); 471 if (!skip_reason.empty()) 472 return test_result(test_result::skipped, skip_reason); 473 474 if (!fs::exists(test_case->container_test_program().absolute_path())) 475 return test_result(test_result::broken, "Test program does not exist"); 476 477 const fs::auto_file stdout_file(work_directory / "stdout.txt"); 478 const fs::auto_file stderr_file(work_directory / "stderr.txt"); 479 const fs::auto_file result_file(work_directory / "result.txt"); 480 481 const engine::test_program& test_program = 482 test_case->container_test_program(); 483 484 try { 485 const engine::tester tester = create_tester( 486 test_program.interface_name(), test_case->get_metadata(), 487 user_config); 488 tester.test(test_program.absolute_path(), test_case->name(), 489 result_file.file(), stdout_file.file(), stderr_file.file(), 490 generate_tester_config(test_case->get_metadata(), 491 user_config, 492 test_program.test_suite_name())); 493 494 hooks.got_stdout(stdout_file.file()); 495 hooks.got_stderr(stderr_file.file()); 496 497 std::ifstream result_input(result_file.file().c_str()); 498 return engine::test_result::parse(result_input); 499 } catch (const std::runtime_error& e) { 500 // One of the possible explanation for us getting here is if the tester 501 // crashes or doesn't behave as expected. We must record any output 502 // from the process so that we can debug it further. 503 hooks.got_stdout(stdout_file.file()); 504 hooks.got_stderr(stderr_file.file()); 505 506 return engine::test_result( 507 engine::test_result::broken, 508 F("Caught unexpected exception: %s") % e.what()); 509 } 510} 511