1// Copyright 2011 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "cli/common.hpp" 30 31#include <algorithm> 32#include <fstream> 33#include <iostream> 34#include <stdexcept> 35 36#include "engine/filters.hpp" 37#include "model/test_program.hpp" 38#include "model/test_result.hpp" 39#include "store/layout.hpp" 40#include "utils/cmdline/exceptions.hpp" 41#include "utils/cmdline/options.hpp" 42#include "utils/cmdline/parser.ipp" 43#include "utils/cmdline/ui.hpp" 44#include "utils/datetime.hpp" 45#include "utils/env.hpp" 46#include "utils/format/macros.hpp" 47#include "utils/logging/macros.hpp" 48#include "utils/fs/exceptions.hpp" 49#include "utils/fs/operations.hpp" 50#include "utils/fs/path.hpp" 51#include "utils/optional.ipp" 52#include "utils/sanity.hpp" 53 54#if defined(HAVE_CONFIG_H) 55# include "config.h" 56#endif 57 58namespace cmdline = utils::cmdline; 59namespace datetime = utils::datetime; 60namespace fs = utils::fs; 61namespace layout = store::layout; 62 63using utils::none; 64using utils::optional; 65 66 67/// Standard definition of the option to specify the build root. 68const cmdline::path_option cli::build_root_option( 69 "build-root", 70 "Path to the built test programs, if different from the location of the " 71 "Kyuafile scripts", 72 "path"); 73 74 75/// Standard definition of the option to specify a Kyuafile. 76const cmdline::path_option cli::kyuafile_option( 77 'k', "kyuafile", 78 "Path to the test suite definition", 79 "file", "Kyuafile"); 80 81 82/// Standard definition of the option to specify filters on test results. 83const cmdline::list_option cli::results_filter_option( 84 "results-filter", "Comma-separated list of result types to include in " 85 "the report", "types", "skipped,xfail,broken,failed"); 86 87 88/// Standard definition of the option to specify the results file. 89/// 90/// TODO(jmmv): Should support a git-like syntax to go back in time, like 91/// --results-file=LATEST^N where N indicates how many runs to go back to. 92const cmdline::string_option cli::results_file_create_option( 93 'r', "results-file", 94 "Path to the results file to create; if left to the default value, the " 95 "name of the file is automatically computed for the current test suite", 96 "file", layout::results_auto_create_name); 97 98 99/// Standard definition of the option to specify the results file. 100/// 101/// TODO(jmmv): Should support a git-like syntax to go back in time, like 102/// --results-file=LATEST^N where N indicates how many runs to go back to. 103const cmdline::string_option cli::results_file_open_option( 104 'r', "results-file", 105 "Path to the results file to open or the identifier of the current test " 106 "suite or a previous results file for automatic lookup; if left to the " 107 "default value, uses the current directory as the test suite name", 108 "file", layout::results_auto_open_name); 109 110 111namespace { 112 113 114/// Gets the path to the historical database if it exists. 115/// 116/// TODO(jmmv): This function should go away. It only exists as a temporary 117/// transitional path to force the use of the stale ~/.kyua/store.db if it 118/// exists. 119/// 120/// \return A path if the file is found; none otherwise. 121static optional< fs::path > 122get_historical_db(void) 123{ 124 optional< fs::path > home = utils::get_home(); 125 if (home) { 126 const fs::path old_db = home.get() / ".kyua/store.db"; 127 if (fs::exists(old_db)) { 128 if (old_db.is_absolute()) 129 return utils::make_optional(old_db); 130 else 131 return utils::make_optional(old_db.to_absolute()); 132 } else { 133 return none; 134 } 135 } else { 136 return none; 137 } 138} 139 140 141/// Converts a set of result type names to identifiers. 142/// 143/// \param names The collection of names to process; may be empty. 144/// 145/// \return The result type identifiers corresponding to the input names. 146/// 147/// \throw std::runtime_error If any name in the input names is invalid. 148static cli::result_types 149parse_types(const std::vector< std::string >& names) 150{ 151 typedef std::map< std::string, model::test_result_type > types_map; 152 types_map valid_types; 153 valid_types["broken"] = model::test_result_broken; 154 valid_types["failed"] = model::test_result_failed; 155 valid_types["passed"] = model::test_result_passed; 156 valid_types["skipped"] = model::test_result_skipped; 157 valid_types["xfail"] = model::test_result_expected_failure; 158 159 cli::result_types types; 160 for (std::vector< std::string >::const_iterator iter = names.begin(); 161 iter != names.end(); ++iter) { 162 const types_map::const_iterator match = valid_types.find(*iter); 163 if (match == valid_types.end()) 164 throw std::runtime_error(F("Unknown result type '%s'") % *iter); 165 else 166 types.push_back((*match).second); 167 } 168 return types; 169} 170 171 172} // anonymous namespace 173 174 175/// Gets the path to the build root, if any. 176/// 177/// This is just syntactic sugar to simplify quierying the 'build_root_option'. 178/// 179/// \param cmdline The parsed command line. 180/// 181/// \return The path to the build root, if specified; none otherwise. 182optional< fs::path > 183cli::build_root_path(const cmdline::parsed_cmdline& cmdline) 184{ 185 optional< fs::path > build_root; 186 if (cmdline.has_option(build_root_option.long_name())) 187 build_root = cmdline.get_option< cmdline::path_option >( 188 build_root_option.long_name()); 189 return build_root; 190} 191 192 193/// Gets the path to the Kyuafile to be loaded. 194/// 195/// This is just syntactic sugar to simplify quierying the 'kyuafile_option'. 196/// 197/// \param cmdline The parsed command line. 198/// 199/// \return The path to the Kyuafile to be loaded. 200fs::path 201cli::kyuafile_path(const cmdline::parsed_cmdline& cmdline) 202{ 203 return cmdline.get_option< cmdline::path_option >( 204 kyuafile_option.long_name()); 205} 206 207 208/// Gets the value of the results-file flag for the creation of a new file. 209/// 210/// \param cmdline The parsed command line from which to extract any possible 211/// override for the location of the database via the --results-file flag. 212/// 213/// \return The path to the database to be used. 214/// 215/// \throw cmdline::error If the value passed to the flag is invalid. 216std::string 217cli::results_file_create(const cmdline::parsed_cmdline& cmdline) 218{ 219 std::string results_file = cmdline.get_option< cmdline::string_option >( 220 results_file_create_option.long_name()); 221 if (results_file == results_file_create_option.default_value()) { 222 const optional< fs::path > historical_db = get_historical_db(); 223 if (historical_db) 224 results_file = historical_db.get().str(); 225 } else { 226 try { 227 (void)fs::path(results_file); 228 } catch (const fs::error& e) { 229 throw cmdline::usage_error(F("Invalid value passed to --%s") % 230 results_file_create_option.long_name()); 231 } 232 } 233 return results_file; 234} 235 236 237/// Gets the value of the results-file flag for the lookup of the file. 238/// 239/// \param cmdline The parsed command line from which to extract any possible 240/// override for the location of the database via the --results-file flag. 241/// 242/// \return The path to the database to be used. 243/// 244/// \throw cmdline::error If the value passed to the flag is invalid. 245std::string 246cli::results_file_open(const cmdline::parsed_cmdline& cmdline) 247{ 248 std::string results_file = cmdline.get_option< cmdline::string_option >( 249 results_file_open_option.long_name()); 250 if (results_file == results_file_open_option.default_value()) { 251 const optional< fs::path > historical_db = get_historical_db(); 252 if (historical_db) 253 results_file = historical_db.get().str(); 254 } else { 255 try { 256 (void)fs::path(results_file); 257 } catch (const fs::error& e) { 258 throw cmdline::usage_error(F("Invalid value passed to --%s") % 259 results_file_open_option.long_name()); 260 } 261 } 262 return results_file; 263} 264 265 266/// Gets the filters for the result types. 267/// 268/// \param cmdline The parsed command line. 269/// 270/// \return A collection of result types to be used for filtering. 271/// 272/// \throw std::runtime_error If any of the user-provided filters is invalid. 273cli::result_types 274cli::get_result_types(const utils::cmdline::parsed_cmdline& cmdline) 275{ 276 result_types types = parse_types( 277 cmdline.get_option< cmdline::list_option >("results-filter")); 278 if (types.empty()) { 279 types.push_back(model::test_result_passed); 280 types.push_back(model::test_result_skipped); 281 types.push_back(model::test_result_expected_failure); 282 types.push_back(model::test_result_broken); 283 types.push_back(model::test_result_failed); 284 } 285 return types; 286} 287 288 289/// Parses a set of command-line arguments to construct test filters. 290/// 291/// \param args The command-line arguments representing test filters. 292/// 293/// \return A set of test filters. 294/// 295/// \throw cmdline:error If any of the arguments is invalid, or if they 296/// represent a non-disjoint collection of filters. 297std::set< engine::test_filter > 298cli::parse_filters(const cmdline::args_vector& args) 299{ 300 std::set< engine::test_filter > filters; 301 302 try { 303 for (cmdline::args_vector::const_iterator iter = args.begin(); 304 iter != args.end(); iter++) { 305 const engine::test_filter filter(engine::test_filter::parse(*iter)); 306 if (filters.find(filter) != filters.end()) 307 throw cmdline::error(F("Duplicate filter '%s'") % filter.str()); 308 filters.insert(filter); 309 } 310 check_disjoint_filters(filters); 311 } catch (const std::runtime_error& e) { 312 throw cmdline::error(e.what()); 313 } 314 315 return filters; 316} 317 318 319/// Reports the filters that have not matched any tests as errors. 320/// 321/// \param unused The collection of unused filters to report. 322/// \param ui The user interface object through which errors are to be reported. 323/// 324/// \return True if there are any unused filters. The caller should report this 325/// as an error to the user by means of a non-successful exit code. 326bool 327cli::report_unused_filters(const std::set< engine::test_filter >& unused, 328 cmdline::ui* ui) 329{ 330 for (std::set< engine::test_filter >::const_iterator iter = unused.begin(); 331 iter != unused.end(); iter++) { 332 cmdline::print_warning(ui, F("No test cases matched by the filter " 333 "'%s'.") % (*iter).str()); 334 } 335 336 return !unused.empty(); 337} 338 339 340/// Formats a time delta for user presentation. 341/// 342/// \param delta The time delta to format. 343/// 344/// \return A user-friendly representation of the time delta. 345std::string 346cli::format_delta(const datetime::delta& delta) 347{ 348 return F("%.3ss") % (delta.seconds + (delta.useconds / 1000000.0)); 349} 350 351 352/// Formats a test case result for user presentation. 353/// 354/// \param result The result to format. 355/// 356/// \return A user-friendly representation of the result. 357std::string 358cli::format_result(const model::test_result& result) 359{ 360 std::string text; 361 362 switch (result.type()) { 363 case model::test_result_broken: text = "broken"; break; 364 case model::test_result_expected_failure: text = "expected_failure"; break; 365 case model::test_result_failed: text = "failed"; break; 366 case model::test_result_passed: text = "passed"; break; 367 case model::test_result_skipped: text = "skipped"; break; 368 } 369 INV(!text.empty()); 370 371 if (!result.reason().empty()) 372 text += ": " + result.reason(); 373 374 return text; 375} 376 377 378/// Formats the identifier of a test case for user presentation. 379/// 380/// \param test_program The test program containing the test case. 381/// \param test_case_name The name of the test case. 382/// 383/// \return A string representing the test case uniquely within a test suite. 384std::string 385cli::format_test_case_id(const model::test_program& test_program, 386 const std::string& test_case_name) 387{ 388 return F("%s:%s") % test_program.relative_path() % test_case_name; 389} 390 391 392/// Formats a filter using the same syntax of a test case. 393/// 394/// \param test_filter The filter to format. 395/// 396/// \return A string representing the test filter. 397std::string 398cli::format_test_case_id(const engine::test_filter& test_filter) 399{ 400 return F("%s:%s") % test_filter.test_program % test_filter.test_case; 401} 402 403 404/// Prints the version header information to the interface output. 405/// 406/// \param ui Interface to which to write the version details. 407void 408cli::write_version_header(utils::cmdline::ui* ui) 409{ 410 ui->out(PACKAGE " (" PACKAGE_NAME ") " PACKAGE_VERSION); 411} 412