1// Copyright 2011 Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "cli/common.hpp" 30 31#include <algorithm> 32 33#include "engine/filters.hpp" 34#include "engine/test_case.hpp" 35#include "engine/test_program.hpp" 36#include "engine/test_result.hpp" 37#include "utils/cmdline/exceptions.hpp" 38#include "utils/cmdline/parser.ipp" 39#include "utils/datetime.hpp" 40#include "utils/env.hpp" 41#include "utils/format/macros.hpp" 42#include "utils/logging/macros.hpp" 43#include "utils/fs/exceptions.hpp" 44#include "utils/fs/operations.hpp" 45#include "utils/fs/path.hpp" 46#include "utils/optional.ipp" 47 48namespace cmdline = utils::cmdline; 49namespace datetime = utils::datetime; 50namespace fs = utils::fs; 51 52using utils::none; 53using utils::optional; 54 55 56/// Standard definition of the option to specify the build root. 57const cmdline::path_option cli::build_root_option( 58 "build-root", 59 "Path to the built test programs, if different from the location of the " 60 "Kyuafile scripts", 61 "path"); 62 63 64/// Standard definition of the option to specify a Kyuafile. 65const cmdline::path_option cli::kyuafile_option( 66 'k', "kyuafile", 67 "Path to the test suite definition", 68 "file", "Kyuafile"); 69 70 71/// Standard definition of the option to specify filters on test results. 72const cmdline::list_option cli::results_filter_option( 73 "results-filter", "Comma-separated list of result types to include in " 74 "the report", "types", "skipped,xfail,broken,failed"); 75 76 77/// Standard definition of the option to specify the store. 78const cmdline::path_option cli::store_option( 79 's', "store", 80 "Path to the store database", 81 "file", "~/.kyua/store.db"); 82 83 84namespace { 85 86 87/// Converts a set of result type names to identifiers. 88/// 89/// \param names The collection of names to process; may be empty. 90/// 91/// \return The result type identifiers corresponding to the input names. 92/// 93/// \throw std::runtime_error If any name in the input names is invalid. 94static cli::result_types 95parse_types(const std::vector< std::string >& names) 96{ 97 using engine::test_result; 98 typedef std::map< std::string, test_result::result_type > types_map; 99 types_map valid_types; 100 valid_types["broken"] = test_result::broken; 101 valid_types["failed"] = test_result::failed; 102 valid_types["passed"] = test_result::passed; 103 valid_types["skipped"] = test_result::skipped; 104 valid_types["xfail"] = test_result::expected_failure; 105 106 cli::result_types types; 107 for (std::vector< std::string >::const_iterator iter = names.begin(); 108 iter != names.end(); ++iter) { 109 const types_map::const_iterator match = valid_types.find(*iter); 110 if (match == valid_types.end()) 111 throw std::runtime_error(F("Unknown result type '%s'") % *iter); 112 else 113 types.push_back((*match).second); 114 } 115 return types; 116} 117 118 119} // anonymous namespace 120 121 122/// Gets the path to the build root, if any. 123/// 124/// This is just syntactic sugar to simplify quierying the 'build_root_option'. 125/// 126/// \param cmdline The parsed command line. 127/// 128/// \return The path to the build root, if specified; none otherwise. 129optional< fs::path > 130cli::build_root_path(const cmdline::parsed_cmdline& cmdline) 131{ 132 optional< fs::path > build_root; 133 if (cmdline.has_option(build_root_option.long_name())) 134 build_root = cmdline.get_option< cmdline::path_option >( 135 build_root_option.long_name()); 136 return build_root; 137} 138 139 140/// Gets the value of the HOME environment variable with path validation. 141/// 142/// \return The value of the HOME environment variable if it is a valid path; 143/// none if it is not defined or if it contains an invalid path. 144optional< fs::path > 145cli::get_home(void) 146{ 147 const optional< std::string > home = utils::getenv("HOME"); 148 if (home) { 149 try { 150 return utils::make_optional(fs::path(home.get())); 151 } catch (const fs::error& e) { 152 LW(F("Invalid value '%s' in HOME environment variable: %s") % 153 home.get() % e.what()); 154 return none; 155 } 156 } else 157 return none; 158} 159 160 161/// Gets the path to the Kyuafile to be loaded. 162/// 163/// This is just syntactic sugar to simplify quierying the 'kyuafile_option'. 164/// 165/// \param cmdline The parsed command line. 166/// 167/// \return The path to the Kyuafile to be loaded. 168fs::path 169cli::kyuafile_path(const cmdline::parsed_cmdline& cmdline) 170{ 171 return cmdline.get_option< cmdline::path_option >( 172 kyuafile_option.long_name()); 173} 174 175 176/// Gets the filters for the result types. 177/// 178/// \param cmdline The parsed command line. 179/// 180/// \return A collection of result types to be used for filtering. 181/// 182/// \throw std::runtime_error If any of the user-provided filters is invalid. 183cli::result_types 184cli::get_result_types(const utils::cmdline::parsed_cmdline& cmdline) 185{ 186 result_types types = parse_types( 187 cmdline.get_option< cmdline::list_option >("results-filter")); 188 if (types.empty()) { 189 types.push_back(engine::test_result::passed); 190 types.push_back(engine::test_result::skipped); 191 types.push_back(engine::test_result::expected_failure); 192 types.push_back(engine::test_result::broken); 193 types.push_back(engine::test_result::failed); 194 } 195 return types; 196} 197 198 199/// Gets the path to the store to be used. 200/// 201/// This has the side-effect of creating the directory in which to store the 202/// database if and only if the path to the database matches the default value. 203/// When the user does not specify an override for the location of the database, 204/// he should not care about the directory existing. Any of this is not a big 205/// deal though, because logs are also stored within ~/.kyua and thus we will 206/// most likely end up creating the directory anyway. 207/// 208/// \param cmdline The parsed command line. 209/// 210/// \return The path to the store to be used. 211/// 212/// \throw fs::error If the creation of the directory fails. 213fs::path 214cli::store_path(const cmdline::parsed_cmdline& cmdline) 215{ 216 fs::path store = cmdline.get_option< cmdline::path_option >( 217 store_option.long_name()); 218 if (store == fs::path(store_option.default_value())) { 219 const optional< fs::path > home = cli::get_home(); 220 if (home) { 221 store = home.get() / ".kyua/store.db"; 222 fs::mkdir_p(store.branch_path(), 0777); 223 } else { 224 store = fs::path("kyua-store.db"); 225 LW("HOME not defined; creating store database in current " 226 "directory"); 227 } 228 } 229 LI(F("Store database set to: %s") % store); 230 return store; 231} 232 233 234/// Parses a set of command-line arguments to construct test filters. 235/// 236/// \param args The command-line arguments representing test filters. 237/// 238/// \throw cmdline:error If any of the arguments is invalid, or if they 239/// represent a non-disjoint collection of filters. 240std::set< engine::test_filter > 241cli::parse_filters(const cmdline::args_vector& args) 242{ 243 std::set< engine::test_filter > filters; 244 245 try { 246 for (cmdline::args_vector::const_iterator iter = args.begin(); 247 iter != args.end(); iter++) { 248 const engine::test_filter filter(engine::test_filter::parse(*iter)); 249 if (filters.find(filter) != filters.end()) 250 throw cmdline::error(F("Duplicate filter '%s'") % filter.str()); 251 filters.insert(filter); 252 } 253 check_disjoint_filters(filters); 254 } catch (const std::runtime_error& e) { 255 throw cmdline::error(e.what()); 256 } 257 258 return filters; 259} 260 261 262/// Reports the filters that have not matched any tests as errors. 263/// 264/// \param unused The collection of unused filters to report. 265/// \param ui The user interface object through which errors are to be reported. 266/// 267/// \return True if there are any unused filters. The caller should report this 268/// as an error to the user by means of a non-successful exit code. 269bool 270cli::report_unused_filters(const std::set< engine::test_filter >& unused, 271 cmdline::ui* ui) 272{ 273 for (std::set< engine::test_filter >::const_iterator iter = unused.begin(); 274 iter != unused.end(); iter++) { 275 cmdline::print_warning(ui, F("No test cases matched by the filter '%s'") 276 % (*iter).str()); 277 } 278 279 return !unused.empty(); 280} 281 282 283/// Formats a time delta for user presentation. 284/// 285/// \param delta The time delta to format. 286/// 287/// \return A user-friendly representation of the time delta. 288std::string 289cli::format_delta(const datetime::delta& delta) 290{ 291 return F("%.3ss") % (delta.seconds + (delta.useconds / 1000000.0)); 292} 293 294 295/// Formats a test case result for user presentation. 296/// 297/// \param result The result to format. 298/// 299/// \return A user-friendly representation of the result. 300std::string 301cli::format_result(const engine::test_result& result) 302{ 303 std::string text; 304 305 using engine::test_result; 306 switch (result.type()) { 307 case test_result::broken: text = "broken"; break; 308 case test_result::expected_failure: text = "expected_failure"; break; 309 case test_result::failed: text = "failed"; break; 310 case test_result::passed: text = "passed"; break; 311 case test_result::skipped: text = "skipped"; break; 312 } 313 INV(!text.empty()); 314 315 if (!result.reason().empty()) 316 text += ": " + result.reason(); 317 318 return text; 319} 320 321 322/// Formats the identifier of a test case for user presentation. 323/// 324/// \param test_case The test case whose identifier to format. 325/// 326/// \return A string representing the test case uniquely within a test suite. 327std::string 328cli::format_test_case_id(const engine::test_case& test_case) 329{ 330 return F("%s:%s") % test_case.container_test_program().relative_path() % 331 test_case.name(); 332} 333 334 335/// Formats a filter using the same syntax of a test case. 336/// 337/// \param test_filter The filter to format. 338/// 339/// \return A string representing the test filter. 340std::string 341cli::format_test_case_id(const engine::test_filter& test_filter) 342{ 343 return F("%s:%s") % test_filter.test_program % test_filter.test_case; 344} 345