1// Copyright 2011 Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "cli/cmd_report.hpp" 30 31#include <cstddef> 32#include <cstdlib> 33#include <fstream> 34#include <map> 35#include <vector> 36 37#include "cli/common.ipp" 38#include "engine/action.hpp" 39#include "engine/context.hpp" 40#include "engine/drivers/scan_action.hpp" 41#include "engine/test_result.hpp" 42#include "utils/cmdline/exceptions.hpp" 43#include "utils/cmdline/parser.ipp" 44#include "utils/defs.hpp" 45#include "utils/format/macros.hpp" 46#include "utils/optional.ipp" 47 48namespace cmdline = utils::cmdline; 49namespace config = utils::config; 50namespace datetime = utils::datetime; 51namespace fs = utils::fs; 52namespace scan_action = engine::drivers::scan_action; 53 54using cli::cmd_report; 55using utils::optional; 56 57 58namespace { 59 60 61/// Generates a plain-text report intended to be printed to the console. 62class console_hooks : public scan_action::base_hooks { 63 /// Indirection to print the output to the correct file stream. 64 cli::file_writer _writer; 65 66 /// Whether to include the runtime context in the output or not. 67 const bool _show_context; 68 69 /// Collection of result types to include in the report. 70 const cli::result_types& _results_filters; 71 72 /// The action ID loaded. 73 int64_t _action_id; 74 75 /// The total run time of the tests. 76 datetime::delta _runtime; 77 78 /// Representation of a single result. 79 struct result_data { 80 /// The relative path to the test program. 81 fs::path binary_path; 82 83 /// The name of the test case. 84 std::string test_case_name; 85 86 /// The result of the test case. 87 engine::test_result result; 88 89 /// The duration of the test case execution. 90 datetime::delta duration; 91 92 /// Constructs a new results data. 93 /// 94 /// \param binary_path_ The relative path to the test program. 95 /// \param test_case_name_ The name of the test case. 96 /// \param result_ The result of the test case. 97 /// \param duration_ The duration of the test case execution. 98 result_data(const fs::path& binary_path_, 99 const std::string& test_case_name_, 100 const engine::test_result& result_, 101 const datetime::delta& duration_) : 102 binary_path(binary_path_), test_case_name(test_case_name_), 103 result(result_), duration(duration_) 104 { 105 } 106 }; 107 108 /// Results received, broken down by their type. 109 /// 110 /// Note that this may not include all results, as keeping the whole list in 111 /// memory may be too much. 112 std::map< engine::test_result::result_type, 113 std::vector< result_data > > _results; 114 115 /// Prints the execution context to the output. 116 /// 117 /// \param context The context to dump. 118 void 119 print_context(const engine::context& context) 120 { 121 _writer("===> Execution context"); 122 123 _writer(F("Current directory: %s") % context.cwd()); 124 const std::map< std::string, std::string >& env = context.env(); 125 if (env.empty()) 126 _writer("No environment variables recorded"); 127 else { 128 _writer("Environment variables:"); 129 for (std::map< std::string, std::string >::const_iterator 130 iter = env.begin(); iter != env.end(); iter++) { 131 _writer(F(" %s=%s") % (*iter).first % (*iter).second); 132 } 133 } 134 } 135 136 /// Counts how many results of a given type have been received. 137 std::size_t 138 count_results(const engine::test_result::result_type type) 139 { 140 const std::map< engine::test_result::result_type, 141 std::vector< result_data > >::const_iterator iter = 142 _results.find(type); 143 if (iter == _results.end()) 144 return 0; 145 else 146 return (*iter).second.size(); 147 } 148 149 /// Prints a set of results. 150 void 151 print_results(const engine::test_result::result_type type, 152 const char* title) 153 { 154 const std::map< engine::test_result::result_type, 155 std::vector< result_data > >::const_iterator iter2 = 156 _results.find(type); 157 if (iter2 == _results.end()) 158 return; 159 const std::vector< result_data >& all = (*iter2).second; 160 161 _writer(F("===> %s") % title); 162 for (std::vector< result_data >::const_iterator iter = all.begin(); 163 iter != all.end(); iter++) { 164 _writer(F("%s:%s -> %s [%s]") % (*iter).binary_path % 165 (*iter).test_case_name % 166 cli::format_result((*iter).result) % 167 cli::format_delta((*iter).duration)); 168 } 169 } 170 171public: 172 /// Constructor for the hooks. 173 /// 174 /// \param ui_ The user interface object of the caller command. 175 /// \param outfile_ The file to which to send the output. 176 /// \param show_context_ Whether to include the runtime context in 177 /// the output or not. 178 /// \param results_filters_ The result types to include in the report. 179 /// Cannot be empty. 180 console_hooks(cmdline::ui* ui_, const fs::path& outfile_, 181 const bool show_context_, 182 const cli::result_types& results_filters_) : 183 _writer(ui_, outfile_), 184 _show_context(show_context_), 185 _results_filters(results_filters_) 186 { 187 PRE(!results_filters_.empty()); 188 } 189 190 /// Callback executed when an action is found. 191 /// 192 /// \param action_id The identifier of the loaded action. 193 /// \param action The action loaded from the database. 194 void 195 got_action(const int64_t action_id, const engine::action& action) 196 { 197 _action_id = action_id; 198 if (_show_context) 199 print_context(action.runtime_context()); 200 } 201 202 /// Callback executed when a test results is found. 203 /// 204 /// \param iter Container for the test result's data. 205 void 206 got_result(store::results_iterator& iter) 207 { 208 _runtime += iter.duration(); 209 const engine::test_result result = iter.result(); 210 _results[result.type()].push_back( 211 result_data(iter.test_program()->relative_path(), 212 iter.test_case_name(), iter.result(), iter.duration())); 213 } 214 215 /// Prints the tests summary. 216 void 217 print_tests(void) 218 { 219 using engine::test_result; 220 typedef std::map< test_result::result_type, const char* > types_map; 221 222 types_map titles; 223 titles[engine::test_result::broken] = "Broken tests"; 224 titles[engine::test_result::expected_failure] = "Expected failures"; 225 titles[engine::test_result::failed] = "Failed tests"; 226 titles[engine::test_result::passed] = "Passed tests"; 227 titles[engine::test_result::skipped] = "Skipped tests"; 228 229 for (cli::result_types::const_iterator iter = _results_filters.begin(); 230 iter != _results_filters.end(); ++iter) { 231 const types_map::const_iterator match = titles.find(*iter); 232 INV_MSG(match != titles.end(), "Conditional does not match user " 233 "input validation in parse_types()"); 234 print_results((*match).first, (*match).second); 235 } 236 237 const std::size_t broken = count_results(test_result::broken); 238 const std::size_t failed = count_results(test_result::failed); 239 const std::size_t passed = count_results(test_result::passed); 240 const std::size_t skipped = count_results(test_result::skipped); 241 const std::size_t xfail = count_results(test_result::expected_failure); 242 const std::size_t total = broken + failed + passed + skipped + xfail; 243 244 _writer("===> Summary"); 245 _writer(F("Action: %s") % _action_id); 246 _writer(F("Test cases: %s total, %s skipped, %s expected failures, " 247 "%s broken, %s failed") % 248 total % skipped % xfail % broken % failed); 249 _writer(F("Total time: %s") % cli::format_delta(_runtime)); 250 } 251}; 252 253 254} // anonymous namespace 255 256 257const fs::path cli::file_writer::_stdout_path("/dev/stdout"); 258const fs::path cli::file_writer::_stderr_path("/dev/stderr"); 259 260 261/// Constructs a new file_writer wrapper. 262/// 263/// \param ui_ The UI object of the caller command. 264/// \param path_ The path to the output file. 265cli::file_writer::file_writer(cmdline::ui* const ui_, const fs::path& path_) : 266 _ui(ui_), _output_path(path_) 267{ 268 if (path_ != _stdout_path && path_ != _stderr_path) { 269 _output_file.reset(new std::ofstream(path_.c_str())); 270 if (!*(_output_file)) { 271 throw std::runtime_error(F("Cannot open output file %s") % path_); 272 } 273 } 274} 275 276/// Destructor. 277cli::file_writer::~file_writer(void) 278{ 279} 280 281/// Writes a message to the selected output. 282/// 283/// \param message The message to write; should not include a termination 284/// new line. 285void 286cli::file_writer::operator()(const std::string& message) 287{ 288 if (_output_path == _stdout_path) 289 _ui->out(message); 290 else if (_output_path == _stderr_path) 291 _ui->err(message); 292 else { 293 INV(_output_file.get() != NULL); 294 (*_output_file) << message << '\n'; 295 } 296} 297 298 299/// Default constructor for cmd_report. 300cmd_report::cmd_report(void) : cli_command( 301 "report", "", 0, 0, 302 "Generates a user-friendly, plain-text report with the result of a " 303 "previous action") 304{ 305 add_option(store_option); 306 add_option(cmdline::bool_option( 307 "show-context", "Include the execution context in the report")); 308 add_option(cmdline::int_option( 309 "action", "The action to report; if not specified, defaults to the " 310 "latest action in the database", "id")); 311 add_option(cmdline::path_option( 312 "output", "The file to which to write the report", 313 "path", "/dev/stdout")); 314 add_option(results_filter_option); 315} 316 317 318/// Entry point for the "report" subcommand. 319/// 320/// \param ui Object to interact with the I/O of the program. 321/// \param cmdline Representation of the command line to the subcommand. 322/// \param unused_user_config The runtime configuration of the program. 323/// 324/// \return 0 if everything is OK, 1 if the statement is invalid or if there is 325/// any other problem. 326int 327cmd_report::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline, 328 const config::tree& UTILS_UNUSED_PARAM(user_config)) 329{ 330 optional< int64_t > action_id; 331 if (cmdline.has_option("action")) 332 action_id = cmdline.get_option< cmdline::int_option >("action"); 333 334 const result_types types = get_result_types(cmdline); 335 console_hooks hooks( 336 ui, cmdline.get_option< cmdline::path_option >("output"), 337 cmdline.has_option("show-context"), types); 338 scan_action::drive(store_path(cmdline), action_id, hooks); 339 hooks.print_tests(); 340 341 return EXIT_SUCCESS; 342} 343