1// Copyright 2012 Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "cli/cmd_report_html.hpp" 30 31#include <algorithm> 32#include <cerrno> 33#include <cstdlib> 34#include <stdexcept> 35 36#include "cli/common.ipp" 37#include "engine/action.hpp" 38#include "engine/context.hpp" 39#include "engine/drivers/scan_action.hpp" 40#include "engine/test_result.hpp" 41#include "utils/cmdline/options.hpp" 42#include "utils/cmdline/parser.ipp" 43#include "utils/datetime.hpp" 44#include "utils/env.hpp" 45#include "utils/format/macros.hpp" 46#include "utils/fs/exceptions.hpp" 47#include "utils/fs/operations.hpp" 48#include "utils/fs/path.hpp" 49#include "utils/optional.ipp" 50#include "utils/text/templates.hpp" 51 52namespace cmdline = utils::cmdline; 53namespace config = utils::config; 54namespace datetime = utils::datetime; 55namespace fs = utils::fs; 56namespace scan_action = engine::drivers::scan_action; 57namespace text = utils::text; 58 59using utils::optional; 60 61 62namespace { 63 64 65/// Creates the report's top directory and fails if it exists. 66/// 67/// \param directory The directory to create. 68/// \param force Whether to wipe an existing directory or not. 69/// 70/// \throw std::runtime_error If the directory already exists; this is a user 71/// error that the user must correct. 72/// \throw fs::error If the directory creation fails for any other reason. 73static void 74create_top_directory(const fs::path& directory, const bool force) 75{ 76 if (force) { 77 if (fs::exists(directory)) 78 fs::rm_r(directory); 79 } 80 81 try { 82 fs::mkdir(directory, 0755); 83 } catch (const fs::system_error& e) { 84 if (e.original_errno() == EEXIST) 85 throw std::runtime_error(F("Output directory '%s' already exists; " 86 "maybe use --force?") % 87 directory); 88 else 89 throw e; 90 } 91} 92 93 94/// Generates a flat unique filename for a given test case. 95/// 96/// \param test_case The test case for which to genereate the name. 97/// 98/// \return A filename unique within a directory with a trailing HTML extension. 99static std::string 100test_case_filename(const engine::test_case& test_case) 101{ 102 static const char* special_characters = "/:"; 103 104 std::string name = cli::format_test_case_id(test_case); 105 std::string::size_type pos = name.find_first_of(special_characters); 106 while (pos != std::string::npos) { 107 name.replace(pos, 1, "_"); 108 pos = name.find_first_of(special_characters, pos + 1); 109 } 110 return name + ".html"; 111} 112 113 114/// Adds a string to string map to the templates. 115/// 116/// \param [in,out] templates The templates to add the map to. 117/// \param props The map to add to the templates. 118/// \param key_vector Name of the template vector that holds the keys. 119/// \param value_vector Name of the template vector that holds the values. 120static void 121add_map(text::templates_def& templates, const config::properties_map& props, 122 const std::string& key_vector, const std::string& value_vector) 123{ 124 templates.add_vector(key_vector); 125 templates.add_vector(value_vector); 126 127 for (config::properties_map::const_iterator iter = props.begin(); 128 iter != props.end(); ++iter) { 129 templates.add_to_vector(key_vector, (*iter).first); 130 templates.add_to_vector(value_vector, (*iter).second); 131 } 132} 133 134 135/// Generates an HTML report. 136class html_hooks : public scan_action::base_hooks { 137 /// User interface object where to report progress. 138 cmdline::ui* _ui; 139 140 /// The top directory in which to create the HTML files. 141 fs::path _directory; 142 143 /// Collection of result types to include in the report. 144 const cli::result_types& _results_filters; 145 146 /// Templates accumulator to generate the index.html file. 147 text::templates_def _summary_templates; 148 149 /// Mapping of result types to the amount of tests with such result. 150 std::map< engine::test_result::result_type, std::size_t > _types_count; 151 152 /// Generates a common set of templates for all of our files. 153 /// 154 /// \return A new templates object with common parameters. 155 static text::templates_def 156 common_templates(void) 157 { 158 text::templates_def templates; 159 templates.add_variable("css", "report.css"); 160 return templates; 161 } 162 163 /// Adds a test case result to the summary. 164 /// 165 /// \param test_case The test case to be added. 166 /// \param result The result of the test case. 167 /// \param has_detail If true, the result of the test case has not been 168 /// filtered and therefore there exists a separate file for the test 169 /// with all of its information. 170 void 171 add_to_summary(const engine::test_case& test_case, 172 const engine::test_result& result, 173 const bool has_detail) 174 { 175 ++_types_count[result.type()]; 176 177 if (!has_detail) 178 return; 179 180 std::string test_cases_vector; 181 std::string test_cases_file_vector; 182 switch (result.type()) { 183 case engine::test_result::broken: 184 test_cases_vector = "broken_test_cases"; 185 test_cases_file_vector = "broken_test_cases_file"; 186 break; 187 188 case engine::test_result::expected_failure: 189 test_cases_vector = "xfail_test_cases"; 190 test_cases_file_vector = "xfail_test_cases_file"; 191 break; 192 193 case engine::test_result::failed: 194 test_cases_vector = "failed_test_cases"; 195 test_cases_file_vector = "failed_test_cases_file"; 196 break; 197 198 case engine::test_result::passed: 199 test_cases_vector = "passed_test_cases"; 200 test_cases_file_vector = "passed_test_cases_file"; 201 break; 202 203 case engine::test_result::skipped: 204 test_cases_vector = "skipped_test_cases"; 205 test_cases_file_vector = "skipped_test_cases_file"; 206 break; 207 } 208 INV(!test_cases_vector.empty()); 209 INV(!test_cases_file_vector.empty()); 210 211 _summary_templates.add_to_vector(test_cases_vector, 212 cli::format_test_case_id(test_case)); 213 _summary_templates.add_to_vector(test_cases_file_vector, 214 test_case_filename(test_case)); 215 } 216 217 /// Instantiate a template to generate an HTML file in the output directory. 218 /// 219 /// \param templates The templates to use. 220 /// \param template_name The name of the template. This is automatically 221 /// searched for in the installed directory, so do not provide a path. 222 /// \param output_name The name of the output file. This is a basename to 223 /// be created within the output directory. 224 /// 225 /// \throw text::error If there is any problem applying the templates. 226 void 227 generate(const text::templates_def& templates, 228 const std::string& template_name, 229 const std::string& output_name) const 230 { 231 const fs::path miscdir(utils::getenv_with_default( 232 "KYUA_MISCDIR", KYUA_MISCDIR)); 233 const fs::path template_file = miscdir / template_name; 234 const fs::path output_path(_directory / output_name); 235 236 _ui->out(F("Generating %s") % output_path); 237 text::instantiate(templates, template_file, output_path); 238 } 239 240 /// Gets the number of tests with a given result type. 241 /// 242 /// \param type The type to be queried. 243 /// 244 /// \return The number of tests of the given type, or 0 if none have yet 245 /// been registered by add_to_summary(). 246 std::size_t 247 get_count(const engine::test_result::result_type type) const 248 { 249 const std::map< engine::test_result::result_type, 250 std::size_t >::const_iterator 251 iter = _types_count.find(type); 252 if (iter == _types_count.end()) 253 return 0; 254 else 255 return (*iter).second; 256 } 257 258public: 259 /// Constructor for the hooks. 260 /// 261 /// \param ui_ User interface object where to report progress. 262 /// \param directory_ The directory in which to create the HTML files. 263 /// \param results_filters_ The result types to include in the report. 264 /// Cannot be empty. 265 html_hooks(cmdline::ui* ui_, const fs::path& directory_, 266 const cli::result_types& results_filters_) : 267 _ui(ui_), 268 _directory(directory_), 269 _results_filters(results_filters_), 270 _summary_templates(common_templates()) 271 { 272 PRE(!results_filters_.empty()); 273 274 // Keep in sync with add_to_summary(). 275 _summary_templates.add_vector("broken_test_cases"); 276 _summary_templates.add_vector("broken_test_cases_file"); 277 _summary_templates.add_vector("xfail_test_cases"); 278 _summary_templates.add_vector("xfail_test_cases_file"); 279 _summary_templates.add_vector("failed_test_cases"); 280 _summary_templates.add_vector("failed_test_cases_file"); 281 _summary_templates.add_vector("passed_test_cases"); 282 _summary_templates.add_vector("passed_test_cases_file"); 283 _summary_templates.add_vector("skipped_test_cases"); 284 _summary_templates.add_vector("skipped_test_cases_file"); 285 } 286 287 /// Callback executed when an action is found. 288 /// 289 /// \param action_id The identifier of the loaded action. 290 /// \param action The action loaded from the database. 291 void 292 got_action(const int64_t action_id, 293 const engine::action& action) 294 { 295 _summary_templates.add_variable("action_id", F("%s") % action_id); 296 297 const engine::context& context = action.runtime_context(); 298 text::templates_def templates = common_templates(); 299 templates.add_variable("action_id", F("%s") % action_id); 300 templates.add_variable("cwd", context.cwd().str()); 301 add_map(templates, context.env(), "env_var", "env_var_value"); 302 generate(templates, "context.html", "context.html"); 303 } 304 305 /// Callback executed when a test results is found. 306 /// 307 /// \param iter Container for the test result's data. 308 void 309 got_result(store::results_iterator& iter) 310 { 311 const engine::test_program_ptr test_program = iter.test_program(); 312 const engine::test_result result = iter.result(); 313 314 const engine::test_case& test_case = *test_program->find( 315 iter.test_case_name()); 316 317 if (std::find(_results_filters.begin(), _results_filters.end(), 318 result.type()) == _results_filters.end()) { 319 add_to_summary(test_case, result, false); 320 return; 321 } 322 323 add_to_summary(test_case, result, true); 324 325 text::templates_def templates = common_templates(); 326 templates.add_variable("test_case", 327 cli::format_test_case_id(test_case)); 328 templates.add_variable("test_program", 329 test_program->absolute_path().str()); 330 templates.add_variable("result", cli::format_result(result)); 331 templates.add_variable("duration", cli::format_delta(iter.duration())); 332 333 add_map(templates, test_case.get_metadata().to_properties(), 334 "metadata_var", "metadata_value"); 335 336 { 337 const std::string stdout_text = iter.stdout_contents(); 338 if (!stdout_text.empty()) 339 templates.add_variable("stdout", stdout_text); 340 } 341 { 342 const std::string stderr_text = iter.stderr_contents(); 343 if (!stderr_text.empty()) 344 templates.add_variable("stderr", stderr_text); 345 } 346 347 generate(templates, "test_result.html", test_case_filename(test_case)); 348 } 349 350 /// Writes the index.html file in the output directory. 351 /// 352 /// This should only be called once all the processing has been done; 353 /// i.e. when the scan_action driver returns. 354 void 355 write_summary(void) 356 { 357 const std::size_t n_passed = get_count(engine::test_result::passed); 358 const std::size_t n_failed = get_count(engine::test_result::failed); 359 const std::size_t n_skipped = get_count(engine::test_result::skipped); 360 const std::size_t n_xfail = get_count( 361 engine::test_result::expected_failure); 362 const std::size_t n_broken = get_count(engine::test_result::broken); 363 364 const std::size_t n_bad = n_broken + n_failed; 365 366 _summary_templates.add_variable("passed_tests_count", 367 F("%s") % n_passed); 368 _summary_templates.add_variable("failed_tests_count", 369 F("%s") % n_failed); 370 _summary_templates.add_variable("skipped_tests_count", 371 F("%s") % n_skipped); 372 _summary_templates.add_variable("xfail_tests_count", 373 F("%s") % n_xfail); 374 _summary_templates.add_variable("broken_tests_count", 375 F("%s") % n_broken); 376 _summary_templates.add_variable("bad_tests_count", F("%s") % n_bad); 377 378 generate(text::templates_def(), "report.css", "report.css"); 379 generate(_summary_templates, "index.html", "index.html"); 380 } 381}; 382 383 384} // anonymous namespace 385 386 387/// Default constructor for cmd_report_html. 388cli::cmd_report_html::cmd_report_html(void) : cli_command( 389 "report-html", "", 0, 0, 390 "Generates an HTML report with the result of a previous action") 391{ 392 add_option(store_option); 393 add_option(cmdline::int_option( 394 "action", "The action to report; if not specified, defaults to the " 395 "latest action in the database", "id")); 396 add_option(cmdline::bool_option( 397 "force", "Wipe the output directory before generating the new report; " 398 "use care")); 399 add_option(cmdline::path_option( 400 "output", "The directory in which to store the HTML files", 401 "path", "html")); 402 add_option(cmdline::list_option( 403 "results-filter", "Comma-separated list of result types to include in " 404 "the report", "types", "skipped,xfail,broken,failed")); 405} 406 407 408/// Entry point for the "report-html" subcommand. 409/// 410/// \param ui Object to interact with the I/O of the program. 411/// \param cmdline Representation of the command line to the subcommand. 412/// \param unused_user_config The runtime configuration of the program. 413/// 414/// \return 0 if everything is OK, 1 if the statement is invalid or if there is 415/// any other problem. 416int 417cli::cmd_report_html::run(cmdline::ui* ui, 418 const cmdline::parsed_cmdline& cmdline, 419 const config::tree& UTILS_UNUSED_PARAM(user_config)) 420{ 421 optional< int64_t > action_id; 422 if (cmdline.has_option("action")) 423 action_id = cmdline.get_option< cmdline::int_option >("action"); 424 425 const result_types types = get_result_types(cmdline); 426 const fs::path directory = 427 cmdline.get_option< cmdline::path_option >("output"); 428 create_top_directory(directory, cmdline.has_option("force")); 429 html_hooks hooks(ui, directory, types); 430 scan_action::drive(store_path(cmdline), action_id, hooks); 431 hooks.write_summary(); 432 433 return EXIT_SUCCESS; 434} 435