1// Copyright 2011 Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/drivers/run_tests.hpp" 30 31#include "engine/action.hpp" 32#include "engine/context.hpp" 33#include "engine/filters.hpp" 34#include "engine/kyuafile.hpp" 35#include "engine/test_program.hpp" 36#include "engine/test_result.hpp" 37#include "store/backend.hpp" 38#include "store/transaction.hpp" 39#include "utils/datetime.hpp" 40#include "utils/defs.hpp" 41#include "utils/format/macros.hpp" 42#include "utils/fs/auto_cleaners.hpp" 43#include "utils/logging/macros.hpp" 44#include "utils/optional.ipp" 45#include "utils/signals/interrupts.hpp" 46 47namespace config = utils::config; 48namespace datetime = utils::datetime; 49namespace fs = utils::fs; 50namespace run_tests = engine::drivers::run_tests; 51namespace signals = utils::signals; 52 53using utils::optional; 54 55 56namespace { 57 58 59/// Test case hooks to save the output into the database. 60class file_saver_hooks : public engine::test_case_hooks { 61 /// Open write transaction for the test case's data. 62 store::transaction& _tx; 63 64 /// Identifier of the test case being stored. 65 const int64_t _test_case_id; 66 67public: 68 /// Constructs a new set of hooks. 69 /// 70 /// \param tx_ Open write transaction for the test case's data. 71 /// \param test_case_id_ Identifier of the test case being stored. 72 file_saver_hooks(store::transaction& tx_, 73 const int64_t test_case_id_) : 74 _tx(tx_), _test_case_id(test_case_id_) 75 { 76 } 77 78 /// Stores the stdout of the test case into the database. 79 /// 80 /// \param file Path to the stdout of the test case. 81 void 82 got_stdout(const fs::path& file) 83 { 84 _tx.put_test_case_file("__STDOUT__", file, _test_case_id); 85 } 86 87 /// Stores the stderr of the test case into the database. 88 /// 89 /// \param file Path to the stderr of the test case. 90 void 91 got_stderr(const fs::path& file) 92 { 93 _tx.put_test_case_file("__STDERR__", file, _test_case_id); 94 } 95}; 96 97 98/// Runs a test program in a controlled manner. 99/// 100/// If the test program fails to provide a list of test cases, a fake test case 101/// named '__test_program__' is created and it is reported as broken. 102/// 103/// \param program The test program to execute. 104/// \param user_config The configuration variables provided by the user. 105/// \param filters The matching state of the filters. 106/// \param hooks The user hooks to receive asynchronous notifications. 107/// \param work_directory Temporary directory to use. 108/// \param tx The store transaction into which to put the results. 109/// \param action_id The action this program belongs to. 110void 111run_test_program(const engine::test_program& program, 112 const config::tree& user_config, 113 engine::filters_state& filters, 114 run_tests::base_hooks& hooks, 115 const fs::path& work_directory, 116 store::transaction& tx, 117 const int64_t action_id) 118{ 119 LI(F("Processing test program '%s'") % program.relative_path()); 120 const int64_t test_program_id = tx.put_test_program(program, action_id); 121 122 const engine::test_cases_vector& test_cases = program.test_cases(); 123 for (engine::test_cases_vector::const_iterator iter = test_cases.begin(); 124 iter != test_cases.end(); iter++) { 125 const engine::test_case_ptr test_case = *iter; 126 127 if (!filters.match_test_case(program.relative_path(), 128 test_case->name())) 129 continue; 130 131 const int64_t test_case_id = tx.put_test_case(*test_case, 132 test_program_id); 133 file_saver_hooks test_hooks(tx, test_case_id); 134 hooks.got_test_case(test_case); 135 const datetime::timestamp start_time = datetime::timestamp::now(); 136 const engine::test_result result = run_test_case( 137 test_case.get(), user_config, test_hooks, work_directory); 138 const datetime::timestamp end_time = datetime::timestamp::now(); 139 tx.put_result(result, test_case_id, start_time, end_time); 140 hooks.got_result(test_case, result, end_time - start_time); 141 142 signals::check_interrupt(); 143 } 144} 145 146 147} // anonymous namespace 148 149 150/// Pure abstract destructor. 151run_tests::base_hooks::~base_hooks(void) 152{ 153} 154 155 156/// Executes the operation. 157/// 158/// \param kyuafile_path The path to the Kyuafile to be loaded. 159/// \param build_root If not none, path to the built test programs. 160/// \param store_path The path to the store to be used. 161/// \param raw_filters The test case filters as provided by the user. 162/// \param user_config The end-user configuration properties. 163/// \param hooks The hooks for this execution. 164/// 165/// \returns A structure with all results computed by this driver. 166run_tests::result 167run_tests::drive(const fs::path& kyuafile_path, 168 const optional< fs::path > build_root, 169 const fs::path& store_path, 170 const std::set< engine::test_filter >& raw_filters, 171 const config::tree& user_config, 172 base_hooks& hooks) 173{ 174 const engine::kyuafile kyuafile = engine::kyuafile::load( 175 kyuafile_path, build_root); 176 filters_state filters(raw_filters); 177 store::backend db = store::backend::open_rw(store_path); 178 store::transaction tx = db.start(); 179 180 engine::context context = engine::context::current(); 181 const int64_t context_id = tx.put_context(context); 182 183 engine::action action(context); 184 const int64_t action_id = tx.put_action(action, context_id); 185 186 signals::interrupts_handler interrupts; 187 188 const fs::auto_directory work_directory = fs::auto_directory::mkdtemp( 189 "kyua.XXXXXX"); 190 191 for (test_programs_vector::const_iterator iter = 192 kyuafile.test_programs().begin(); 193 iter != kyuafile.test_programs().end(); iter++) { 194 const test_program_ptr& test_program = *iter; 195 196 if (!filters.match_test_program(test_program->relative_path())) 197 continue; 198 199 run_test_program(*test_program, user_config, filters, hooks, 200 work_directory.directory(), tx, action_id); 201 202 signals::check_interrupt(); 203 } 204 205 tx.commit(); 206 207 return result(action_id, filters.unused()); 208} 209