1// Copyright 2011 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "drivers/run_tests.hpp" 30 31#include <utility> 32 33#include "engine/config.hpp" 34#include "engine/filters.hpp" 35#include "engine/kyuafile.hpp" 36#include "engine/scanner.hpp" 37#include "engine/scheduler.hpp" 38#include "model/context.hpp" 39#include "model/metadata.hpp" 40#include "model/test_case.hpp" 41#include "model/test_program.hpp" 42#include "model/test_result.hpp" 43#include "store/write_backend.hpp" 44#include "store/write_transaction.hpp" 45#include "utils/config/tree.ipp" 46#include "utils/datetime.hpp" 47#include "utils/defs.hpp" 48#include "utils/format/macros.hpp" 49#include "utils/logging/macros.hpp" 50#include "utils/noncopyable.hpp" 51#include "utils/optional.ipp" 52#include "utils/passwd.hpp" 53#include "utils/text/operations.ipp" 54 55namespace config = utils::config; 56namespace datetime = utils::datetime; 57namespace fs = utils::fs; 58namespace passwd = utils::passwd; 59namespace scheduler = engine::scheduler; 60namespace text = utils::text; 61 62using utils::none; 63using utils::optional; 64 65 66namespace { 67 68 69/// Map of test program identifiers (relative paths) to their identifiers in the 70/// database. We need to keep this in memory because test programs can be 71/// returned by the scanner in any order, and we only want to put each test 72/// program once. 73typedef std::map< fs::path, int64_t > path_to_id_map; 74 75 76/// Map of in-flight PIDs to their corresponding test case IDs. 77typedef std::map< int, int64_t > pid_to_id_map; 78 79 80/// Pair of PID to a test case ID. 81typedef pid_to_id_map::value_type pid_and_id_pair; 82 83 84/// Puts a test program in the store and returns its identifier. 85/// 86/// This function is idempotent: we maintain a side cache of already-put test 87/// programs so that we can return their identifiers without having to put them 88/// again. 89/// TODO(jmmv): It's possible that the store module should offer this 90/// functionality and not have to do this ourselves here. 91/// 92/// \param test_program The test program being put. 93/// \param [in,out] tx Writable transaction on the store. 94/// \param [in,out] ids_cache Cache of already-put test programs. 95/// 96/// \return A test program identifier. 97static int64_t 98find_test_program_id(const model::test_program_ptr test_program, 99 store::write_transaction& tx, 100 path_to_id_map& ids_cache) 101{ 102 const fs::path& key = test_program->relative_path(); 103 std::map< fs::path, int64_t >::const_iterator iter = ids_cache.find(key); 104 if (iter == ids_cache.end()) { 105 const int64_t id = tx.put_test_program(*test_program); 106 ids_cache.insert(std::make_pair(key, id)); 107 return id; 108 } else { 109 return (*iter).second; 110 } 111} 112 113 114/// Stores the result of an execution in the database. 115/// 116/// \param test_case_id Identifier of the test case in the database. 117/// \param result The result of the execution. 118/// \param [in,out] tx Writable transaction where to store the result data. 119static void 120put_test_result(const int64_t test_case_id, 121 const scheduler::test_result_handle& result, 122 store::write_transaction& tx) 123{ 124 tx.put_result(result.test_result(), test_case_id, 125 result.start_time(), result.end_time()); 126 tx.put_test_case_file("__STDOUT__", result.stdout_file(), test_case_id); 127 tx.put_test_case_file("__STDERR__", result.stderr_file(), test_case_id); 128 129} 130 131 132/// Cleans up a test case and folds any errors into the test result. 133/// 134/// \param handle The result handle for the test. 135/// 136/// \return The test result if the cleanup succeeds; a broken test result 137/// otherwise. 138model::test_result 139safe_cleanup(scheduler::test_result_handle handle) throw() 140{ 141 try { 142 handle.cleanup(); 143 return handle.test_result(); 144 } catch (const std::exception& e) { 145 return model::test_result( 146 model::test_result_broken, 147 F("Failed to clean up test case's work directory %s: %s") % 148 handle.work_directory() % e.what()); 149 } 150} 151 152 153/// Starts a test asynchronously. 154/// 155/// \param handle Scheduler handle. 156/// \param match Test program and test case to start. 157/// \param [in,out] tx Writable transaction to obtain test IDs. 158/// \param [in,out] ids_cache Cache of already-put test cases. 159/// \param user_config The end-user configuration properties. 160/// \param hooks The hooks for this execution. 161/// 162/// \returns The PID for the started test and the test case's identifier in the 163/// store. 164pid_and_id_pair 165start_test(scheduler::scheduler_handle& handle, 166 const engine::scan_result& match, 167 store::write_transaction& tx, 168 path_to_id_map& ids_cache, 169 const config::tree& user_config, 170 drivers::run_tests::base_hooks& hooks) 171{ 172 const model::test_program_ptr test_program = match.first; 173 const std::string& test_case_name = match.second; 174 175 hooks.got_test_case(*test_program, test_case_name); 176 177 const int64_t test_program_id = find_test_program_id( 178 test_program, tx, ids_cache); 179 const int64_t test_case_id = tx.put_test_case( 180 *test_program, test_case_name, test_program_id); 181 182 const scheduler::exec_handle exec_handle = handle.spawn_test( 183 test_program, test_case_name, user_config); 184 return std::make_pair(exec_handle, test_case_id); 185} 186 187 188/// Processes the completion of a test. 189/// 190/// \param [in,out] result_handle The completion handle of the test subprocess. 191/// \param test_case_id Identifier of the test case as returned by start_test(). 192/// \param [in,out] tx Writable transaction to put the test results. 193/// \param hooks The hooks for this execution. 194/// 195/// \post result_handle is cleaned up. The caller cannot clean it up again. 196void 197finish_test(scheduler::result_handle_ptr result_handle, 198 const int64_t test_case_id, 199 store::write_transaction& tx, 200 drivers::run_tests::base_hooks& hooks) 201{ 202 const scheduler::test_result_handle* test_result_handle = 203 dynamic_cast< const scheduler::test_result_handle* >( 204 result_handle.get()); 205 206 put_test_result(test_case_id, *test_result_handle, tx); 207 208 const model::test_result test_result = safe_cleanup(*test_result_handle); 209 hooks.got_result( 210 *test_result_handle->test_program(), 211 test_result_handle->test_case_name(), 212 test_result_handle->test_result(), 213 result_handle->end_time() - result_handle->start_time()); 214} 215 216 217/// Extracts the keys of a pid_to_id_map and returns them as a string. 218/// 219/// \param map The PID to test ID map from which to get the PIDs. 220/// 221/// \return A user-facing string with the collection of PIDs. 222static std::string 223format_pids(const pid_to_id_map& map) 224{ 225 std::set< pid_to_id_map::key_type > pids; 226 for (pid_to_id_map::const_iterator iter = map.begin(); iter != map.end(); 227 ++iter) { 228 pids.insert(iter->first); 229 } 230 return text::join(pids, ","); 231} 232 233 234} // anonymous namespace 235 236 237/// Pure abstract destructor. 238drivers::run_tests::base_hooks::~base_hooks(void) 239{ 240} 241 242 243/// Executes the operation. 244/// 245/// \param kyuafile_path The path to the Kyuafile to be loaded. 246/// \param build_root If not none, path to the built test programs. 247/// \param store_path The path to the store to be used. 248/// \param filters The test case filters as provided by the user. 249/// \param user_config The end-user configuration properties. 250/// \param hooks The hooks for this execution. 251/// 252/// \returns A structure with all results computed by this driver. 253drivers::run_tests::result 254drivers::run_tests::drive(const fs::path& kyuafile_path, 255 const optional< fs::path > build_root, 256 const fs::path& store_path, 257 const std::set< engine::test_filter >& filters, 258 const config::tree& user_config, 259 base_hooks& hooks) 260{ 261 scheduler::scheduler_handle handle = scheduler::setup(); 262 263 const engine::kyuafile kyuafile = engine::kyuafile::load( 264 kyuafile_path, build_root, user_config, handle); 265 store::write_backend db = store::write_backend::open_rw(store_path); 266 store::write_transaction tx = db.start_write(); 267 268 { 269 const model::context context = scheduler::current_context(); 270 (void)tx.put_context(context); 271 } 272 273 engine::scanner scanner(kyuafile.test_programs(), filters); 274 275 path_to_id_map ids_cache; 276 pid_to_id_map in_flight; 277 std::vector< engine::scan_result > exclusive_tests; 278 279 const std::size_t slots = user_config.lookup< config::positive_int_node >( 280 "parallelism"); 281 INV(slots >= 1); 282 do { 283 INV(in_flight.size() <= slots); 284 285 // Spawn as many jobs as needed to fill our execution slots. We do this 286 // first with the assumption that the spawning is faster than any single 287 // job, so we want to keep as many jobs in the background as possible. 288 while (in_flight.size() < slots) { 289 optional< engine::scan_result > match = scanner.yield(); 290 if (!match) 291 break; 292 const model::test_program_ptr test_program = match.get().first; 293 const std::string& test_case_name = match.get().second; 294 295 const model::test_case& test_case = test_program->find( 296 test_case_name); 297 if (test_case.get_metadata().is_exclusive()) { 298 // Exclusive tests get processed later, separately. 299 exclusive_tests.push_back(match.get()); 300 continue; 301 } 302 303 const pid_and_id_pair pid_id = start_test( 304 handle, match.get(), tx, ids_cache, user_config, hooks); 305 INV_MSG(in_flight.find(pid_id.first) == in_flight.end(), 306 F("Spawned test has PID of still-tracked process %s") % 307 pid_id.first); 308 in_flight.insert(pid_id); 309 } 310 311 // If there are any used slots, consume any at random and return the 312 // result. We consume slots one at a time to give preference to the 313 // spawning of new tests as detailed above. 314 if (!in_flight.empty()) { 315 scheduler::result_handle_ptr result_handle = handle.wait_any(); 316 317 const pid_to_id_map::iterator iter = in_flight.find( 318 result_handle->original_pid()); 319 INV_MSG(iter != in_flight.end(), 320 F("Lost track of in-flight PID %s; tracking %s") % 321 result_handle->original_pid() % format_pids(in_flight)); 322 const int64_t test_case_id = (*iter).second; 323 in_flight.erase(iter); 324 325 finish_test(result_handle, test_case_id, tx, hooks); 326 } 327 } while (!in_flight.empty() || !scanner.done()); 328 329 // Run any exclusive tests that we spotted earlier sequentially. 330 for (std::vector< engine::scan_result >::const_iterator 331 iter = exclusive_tests.begin(); iter != exclusive_tests.end(); 332 ++iter) { 333 const pid_and_id_pair data = start_test( 334 handle, *iter, tx, ids_cache, user_config, hooks); 335 scheduler::result_handle_ptr result_handle = handle.wait_any(); 336 finish_test(result_handle, data.second, tx, hooks); 337 } 338 339 tx.commit(); 340 341 handle.cleanup(); 342 343 return result(scanner.unused_filters()); 344} 345