1// Copyright 2014 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/plain.hpp" 30 31extern "C" { 32#include <signal.h> 33} 34 35#include <atf-c++.hpp> 36 37#include "engine/config.hpp" 38#include "engine/scheduler.hpp" 39#include "model/metadata.hpp" 40#include "model/test_case.hpp" 41#include "model/test_program.hpp" 42#include "model/test_result.hpp" 43#include "utils/config/tree.ipp" 44#include "utils/datetime.hpp" 45#include "utils/env.hpp" 46#include "utils/format/containers.ipp" 47#include "utils/format/macros.hpp" 48#include "utils/fs/operations.hpp" 49#include "utils/fs/path.hpp" 50#include "utils/optional.ipp" 51 52namespace config = utils::config; 53namespace datetime = utils::datetime; 54namespace fs = utils::fs; 55namespace scheduler = engine::scheduler; 56 57using utils::none; 58 59 60namespace { 61 62 63/// Copies the plain helper to the work directory, selecting a specific helper. 64/// 65/// \param tc Pointer to the calling test case, to obtain srcdir. 66/// \param name Name of the new binary to create. Must match the name of a 67/// valid helper, as the binary name is used to select it. 68static void 69copy_plain_helper(const atf::tests::tc* tc, const char* name) 70{ 71 const fs::path srcdir(tc->get_config_var("srcdir")); 72 atf::utils::copy_file((srcdir / "plain_helpers").str(), name); 73} 74 75 76/// Runs one plain test program and checks its result. 77/// 78/// \param tc Pointer to the calling test case, to obtain srcdir. 79/// \param test_case_name Name of the "test case" to select from the helper 80/// program. 81/// \param exp_result The expected result. 82/// \param metadata The test case metadata. 83/// \param user_config User-provided configuration variables. 84static void 85run_one(const atf::tests::tc* tc, const char* test_case_name, 86 const model::test_result& exp_result, 87 const model::metadata& metadata = model::metadata_builder().build(), 88 const config::tree& user_config = engine::empty_config()) 89{ 90 copy_plain_helper(tc, test_case_name); 91 const model::test_program_ptr program = model::test_program_builder( 92 "plain", fs::path(test_case_name), fs::current_path(), "the-suite") 93 .add_test_case("main", metadata).build_ptr(); 94 95 scheduler::scheduler_handle handle = scheduler::setup(); 96 (void)handle.spawn_test(program, "main", user_config); 97 98 scheduler::result_handle_ptr result_handle = handle.wait_any(); 99 const scheduler::test_result_handle* test_result_handle = 100 dynamic_cast< const scheduler::test_result_handle* >( 101 result_handle.get()); 102 atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); 103 atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); 104 ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); 105 result_handle->cleanup(); 106 result_handle.reset(); 107 108 handle.cleanup(); 109} 110 111 112} // anonymous namespace 113 114 115ATF_TEST_CASE_WITHOUT_HEAD(list); 116ATF_TEST_CASE_BODY(list) 117{ 118 const model::test_program program = model::test_program_builder( 119 "plain", fs::path("non-existent"), fs::path("."), "unused-suite") 120 .build(); 121 122 scheduler::scheduler_handle handle = scheduler::setup(); 123 const model::test_cases_map test_cases = handle.list_tests( 124 &program, engine::empty_config()); 125 handle.cleanup(); 126 127 const model::test_cases_map exp_test_cases = model::test_cases_map_builder() 128 .add("main").build(); 129 ATF_REQUIRE_EQ(exp_test_cases, test_cases); 130} 131 132 133ATF_TEST_CASE_WITHOUT_HEAD(test__exit_success_is_pass); 134ATF_TEST_CASE_BODY(test__exit_success_is_pass) 135{ 136 const model::test_result exp_result(model::test_result_passed); 137 run_one(this, "pass", exp_result); 138} 139 140 141ATF_TEST_CASE_WITHOUT_HEAD(test__exit_non_zero_is_fail); 142ATF_TEST_CASE_BODY(test__exit_non_zero_is_fail) 143{ 144 const model::test_result exp_result( 145 model::test_result_failed, 146 "Returned non-success exit status 8"); 147 run_one(this, "fail", exp_result); 148} 149 150 151ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken); 152ATF_TEST_CASE_BODY(test__signal_is_broken) 153{ 154 const model::test_result exp_result(model::test_result_broken, 155 F("Received signal %s") % SIGABRT); 156 run_one(this, "crash", exp_result); 157} 158 159 160ATF_TEST_CASE(test__timeout_is_broken); 161ATF_TEST_CASE_HEAD(test__timeout_is_broken) 162{ 163 set_md_var("timeout", "60"); 164} 165ATF_TEST_CASE_BODY(test__timeout_is_broken) 166{ 167 utils::setenv("CONTROL_DIR", fs::current_path().str()); 168 169 const model::metadata metadata = model::metadata_builder() 170 .set_timeout(datetime::delta(1, 0)).build(); 171 const model::test_result exp_result(model::test_result_broken, 172 "Test case timed out"); 173 run_one(this, "timeout", exp_result, metadata); 174 175 ATF_REQUIRE(!atf::utils::file_exists("cookie")); 176} 177 178 179ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables); 180ATF_TEST_CASE_BODY(test__configuration_variables) 181{ 182 config::tree user_config = engine::empty_config(); 183 user_config.set_string("test_suites.a-suite.first", "unused"); 184 user_config.set_string("test_suites.the-suite.first", "some value"); 185 user_config.set_string("test_suites.the-suite.second", "some other value"); 186 user_config.set_string("test_suites.other-suite.first", "unused"); 187 188 const model::test_result exp_result(model::test_result_passed); 189 run_one(this, "check_configuration_variables", exp_result, 190 model::metadata_builder().build(), user_config); 191} 192 193 194ATF_INIT_TEST_CASES(tcs) 195{ 196 scheduler::register_interface( 197 "plain", std::shared_ptr< scheduler::interface >( 198 new engine::plain_interface())); 199 200 ATF_ADD_TEST_CASE(tcs, list); 201 202 ATF_ADD_TEST_CASE(tcs, test__exit_success_is_pass); 203 ATF_ADD_TEST_CASE(tcs, test__exit_non_zero_is_fail); 204 ATF_ADD_TEST_CASE(tcs, test__signal_is_broken); 205 ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken); 206 ATF_ADD_TEST_CASE(tcs, test__configuration_variables); 207} 208