1// Copyright 2012 Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9//   notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above copyright
11//   notice, this list of conditions and the following disclaimer in the
12//   documentation and/or other materials provided with the distribution.
13// * Neither the name of Google Inc. nor the names of its contributors
14//   may be used to endorse or promote products derived from this software
15//   without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#include "cli/cmd_report_html.hpp"
30
31#include <algorithm>
32#include <cerrno>
33#include <cstdlib>
34#include <stdexcept>
35
36#include "cli/common.ipp"
37#include "engine/action.hpp"
38#include "engine/context.hpp"
39#include "engine/drivers/scan_action.hpp"
40#include "engine/test_result.hpp"
41#include "utils/cmdline/options.hpp"
42#include "utils/cmdline/parser.ipp"
43#include "utils/datetime.hpp"
44#include "utils/env.hpp"
45#include "utils/format/macros.hpp"
46#include "utils/fs/exceptions.hpp"
47#include "utils/fs/operations.hpp"
48#include "utils/fs/path.hpp"
49#include "utils/optional.ipp"
50#include "utils/text/templates.hpp"
51
52namespace cmdline = utils::cmdline;
53namespace config = utils::config;
54namespace datetime = utils::datetime;
55namespace fs = utils::fs;
56namespace scan_action = engine::drivers::scan_action;
57namespace text = utils::text;
58
59using utils::optional;
60
61
62namespace {
63
64
65/// Creates the report's top directory and fails if it exists.
66///
67/// \param directory The directory to create.
68/// \param force Whether to wipe an existing directory or not.
69///
70/// \throw std::runtime_error If the directory already exists; this is a user
71///     error that the user must correct.
72/// \throw fs::error If the directory creation fails for any other reason.
73static void
74create_top_directory(const fs::path& directory, const bool force)
75{
76    if (force) {
77        if (fs::exists(directory))
78            fs::rm_r(directory);
79    }
80
81    try {
82        fs::mkdir(directory, 0755);
83    } catch (const fs::system_error& e) {
84        if (e.original_errno() == EEXIST)
85            throw std::runtime_error(F("Output directory '%s' already exists; "
86                                       "maybe use --force?") %
87                                     directory);
88        else
89            throw e;
90    }
91}
92
93
94/// Generates a flat unique filename for a given test case.
95///
96/// \param test_case The test case for which to genereate the name.
97///
98/// \return A filename unique within a directory with a trailing HTML extension.
99static std::string
100test_case_filename(const engine::test_case& test_case)
101{
102    static const char* special_characters = "/:";
103
104    std::string name = cli::format_test_case_id(test_case);
105    std::string::size_type pos = name.find_first_of(special_characters);
106    while (pos != std::string::npos) {
107        name.replace(pos, 1, "_");
108        pos = name.find_first_of(special_characters, pos + 1);
109    }
110    return name + ".html";
111}
112
113
114/// Adds a string to string map to the templates.
115///
116/// \param [in,out] templates The templates to add the map to.
117/// \param props The map to add to the templates.
118/// \param key_vector Name of the template vector that holds the keys.
119/// \param value_vector Name of the template vector that holds the values.
120static void
121add_map(text::templates_def& templates, const config::properties_map& props,
122        const std::string& key_vector, const std::string& value_vector)
123{
124    templates.add_vector(key_vector);
125    templates.add_vector(value_vector);
126
127    for (config::properties_map::const_iterator iter = props.begin();
128         iter != props.end(); ++iter) {
129        templates.add_to_vector(key_vector, (*iter).first);
130        templates.add_to_vector(value_vector, (*iter).second);
131    }
132}
133
134
135/// Generates an HTML report.
136class html_hooks : public scan_action::base_hooks {
137    /// User interface object where to report progress.
138    cmdline::ui* _ui;
139
140    /// The top directory in which to create the HTML files.
141    fs::path _directory;
142
143    /// Collection of result types to include in the report.
144    const cli::result_types& _results_filters;
145
146    /// Templates accumulator to generate the index.html file.
147    text::templates_def _summary_templates;
148
149    /// Mapping of result types to the amount of tests with such result.
150    std::map< engine::test_result::result_type, std::size_t > _types_count;
151
152    /// Generates a common set of templates for all of our files.
153    ///
154    /// \return A new templates object with common parameters.
155    static text::templates_def
156    common_templates(void)
157    {
158        text::templates_def templates;
159        templates.add_variable("css", "report.css");
160        return templates;
161    }
162
163    /// Adds a test case result to the summary.
164    ///
165    /// \param test_case The test case to be added.
166    /// \param result The result of the test case.
167    /// \param has_detail If true, the result of the test case has not been
168    ///     filtered and therefore there exists a separate file for the test
169    ///     with all of its information.
170    void
171    add_to_summary(const engine::test_case& test_case,
172                   const engine::test_result& result,
173                   const bool has_detail)
174    {
175        ++_types_count[result.type()];
176
177        if (!has_detail)
178            return;
179
180        std::string test_cases_vector;
181        std::string test_cases_file_vector;
182        switch (result.type()) {
183        case engine::test_result::broken:
184            test_cases_vector = "broken_test_cases";
185            test_cases_file_vector = "broken_test_cases_file";
186            break;
187
188        case engine::test_result::expected_failure:
189            test_cases_vector = "xfail_test_cases";
190            test_cases_file_vector = "xfail_test_cases_file";
191            break;
192
193        case engine::test_result::failed:
194            test_cases_vector = "failed_test_cases";
195            test_cases_file_vector = "failed_test_cases_file";
196            break;
197
198        case engine::test_result::passed:
199            test_cases_vector = "passed_test_cases";
200            test_cases_file_vector = "passed_test_cases_file";
201            break;
202
203        case engine::test_result::skipped:
204            test_cases_vector = "skipped_test_cases";
205            test_cases_file_vector = "skipped_test_cases_file";
206            break;
207        }
208        INV(!test_cases_vector.empty());
209        INV(!test_cases_file_vector.empty());
210
211        _summary_templates.add_to_vector(test_cases_vector,
212                                         cli::format_test_case_id(test_case));
213        _summary_templates.add_to_vector(test_cases_file_vector,
214                                         test_case_filename(test_case));
215    }
216
217    /// Instantiate a template to generate an HTML file in the output directory.
218    ///
219    /// \param templates The templates to use.
220    /// \param template_name The name of the template.  This is automatically
221    ///     searched for in the installed directory, so do not provide a path.
222    /// \param output_name The name of the output file.  This is a basename to
223    ///     be created within the output directory.
224    ///
225    /// \throw text::error If there is any problem applying the templates.
226    void
227    generate(const text::templates_def& templates,
228             const std::string& template_name,
229             const std::string& output_name) const
230    {
231        const fs::path miscdir(utils::getenv_with_default(
232             "KYUA_MISCDIR", KYUA_MISCDIR));
233        const fs::path template_file = miscdir / template_name;
234        const fs::path output_path(_directory / output_name);
235
236        _ui->out(F("Generating %s") % output_path);
237        text::instantiate(templates, template_file, output_path);
238    }
239
240    /// Gets the number of tests with a given result type.
241    ///
242    /// \param type The type to be queried.
243    ///
244    /// \return The number of tests of the given type, or 0 if none have yet
245    /// been registered by add_to_summary().
246    std::size_t
247    get_count(const engine::test_result::result_type type) const
248    {
249        const std::map< engine::test_result::result_type,
250                        std::size_t >::const_iterator
251            iter = _types_count.find(type);
252        if (iter == _types_count.end())
253            return 0;
254        else
255            return (*iter).second;
256    }
257
258public:
259    /// Constructor for the hooks.
260    ///
261    /// \param ui_ User interface object where to report progress.
262    /// \param directory_ The directory in which to create the HTML files.
263    /// \param results_filters_ The result types to include in the report.
264    ///     Cannot be empty.
265    html_hooks(cmdline::ui* ui_, const fs::path& directory_,
266               const cli::result_types& results_filters_) :
267        _ui(ui_),
268        _directory(directory_),
269        _results_filters(results_filters_),
270        _summary_templates(common_templates())
271    {
272        PRE(!results_filters_.empty());
273
274        // Keep in sync with add_to_summary().
275        _summary_templates.add_vector("broken_test_cases");
276        _summary_templates.add_vector("broken_test_cases_file");
277        _summary_templates.add_vector("xfail_test_cases");
278        _summary_templates.add_vector("xfail_test_cases_file");
279        _summary_templates.add_vector("failed_test_cases");
280        _summary_templates.add_vector("failed_test_cases_file");
281        _summary_templates.add_vector("passed_test_cases");
282        _summary_templates.add_vector("passed_test_cases_file");
283        _summary_templates.add_vector("skipped_test_cases");
284        _summary_templates.add_vector("skipped_test_cases_file");
285    }
286
287    /// Callback executed when an action is found.
288    ///
289    /// \param action_id The identifier of the loaded action.
290    /// \param action The action loaded from the database.
291    void
292    got_action(const int64_t action_id,
293               const engine::action& action)
294    {
295        _summary_templates.add_variable("action_id", F("%s") % action_id);
296
297        const engine::context& context = action.runtime_context();
298        text::templates_def templates = common_templates();
299        templates.add_variable("action_id", F("%s") % action_id);
300        templates.add_variable("cwd", context.cwd().str());
301        add_map(templates, context.env(), "env_var", "env_var_value");
302        generate(templates, "context.html", "context.html");
303    }
304
305    /// Callback executed when a test results is found.
306    ///
307    /// \param iter Container for the test result's data.
308    void
309    got_result(store::results_iterator& iter)
310    {
311        const engine::test_program_ptr test_program = iter.test_program();
312        const engine::test_result result = iter.result();
313
314        const engine::test_case& test_case = *test_program->find(
315            iter.test_case_name());
316
317        if (std::find(_results_filters.begin(), _results_filters.end(),
318                      result.type()) == _results_filters.end()) {
319            add_to_summary(test_case, result, false);
320            return;
321        }
322
323        add_to_summary(test_case, result, true);
324
325        text::templates_def templates = common_templates();
326        templates.add_variable("test_case",
327                               cli::format_test_case_id(test_case));
328        templates.add_variable("test_program",
329                               test_program->absolute_path().str());
330        templates.add_variable("result", cli::format_result(result));
331        templates.add_variable("duration", cli::format_delta(iter.duration()));
332
333        add_map(templates, test_case.get_metadata().to_properties(),
334                "metadata_var", "metadata_value");
335
336        {
337            const std::string stdout_text = iter.stdout_contents();
338            if (!stdout_text.empty())
339                templates.add_variable("stdout", stdout_text);
340        }
341        {
342            const std::string stderr_text = iter.stderr_contents();
343            if (!stderr_text.empty())
344                templates.add_variable("stderr", stderr_text);
345        }
346
347        generate(templates, "test_result.html", test_case_filename(test_case));
348    }
349
350    /// Writes the index.html file in the output directory.
351    ///
352    /// This should only be called once all the processing has been done;
353    /// i.e. when the scan_action driver returns.
354    void
355    write_summary(void)
356    {
357        const std::size_t n_passed = get_count(engine::test_result::passed);
358        const std::size_t n_failed = get_count(engine::test_result::failed);
359        const std::size_t n_skipped = get_count(engine::test_result::skipped);
360        const std::size_t n_xfail = get_count(
361            engine::test_result::expected_failure);
362        const std::size_t n_broken = get_count(engine::test_result::broken);
363
364        const std::size_t n_bad = n_broken + n_failed;
365
366        _summary_templates.add_variable("passed_tests_count",
367                                        F("%s") % n_passed);
368        _summary_templates.add_variable("failed_tests_count",
369                                        F("%s") % n_failed);
370        _summary_templates.add_variable("skipped_tests_count",
371                                        F("%s") % n_skipped);
372        _summary_templates.add_variable("xfail_tests_count",
373                                        F("%s") % n_xfail);
374        _summary_templates.add_variable("broken_tests_count",
375                                        F("%s") % n_broken);
376        _summary_templates.add_variable("bad_tests_count", F("%s") % n_bad);
377
378        generate(text::templates_def(), "report.css", "report.css");
379        generate(_summary_templates, "index.html", "index.html");
380    }
381};
382
383
384}  // anonymous namespace
385
386
387/// Default constructor for cmd_report_html.
388cli::cmd_report_html::cmd_report_html(void) : cli_command(
389    "report-html", "", 0, 0,
390    "Generates an HTML report with the result of a previous action")
391{
392    add_option(store_option);
393    add_option(cmdline::int_option(
394        "action", "The action to report; if not specified, defaults to the "
395        "latest action in the database", "id"));
396    add_option(cmdline::bool_option(
397        "force", "Wipe the output directory before generating the new report; "
398        "use care"));
399    add_option(cmdline::path_option(
400        "output", "The directory in which to store the HTML files",
401        "path", "html"));
402    add_option(cmdline::list_option(
403        "results-filter", "Comma-separated list of result types to include in "
404        "the report", "types", "skipped,xfail,broken,failed"));
405}
406
407
408/// Entry point for the "report-html" subcommand.
409///
410/// \param ui Object to interact with the I/O of the program.
411/// \param cmdline Representation of the command line to the subcommand.
412/// \param unused_user_config The runtime configuration of the program.
413///
414/// \return 0 if everything is OK, 1 if the statement is invalid or if there is
415/// any other problem.
416int
417cli::cmd_report_html::run(cmdline::ui* ui,
418                          const cmdline::parsed_cmdline& cmdline,
419                          const config::tree& UTILS_UNUSED_PARAM(user_config))
420{
421    optional< int64_t > action_id;
422    if (cmdline.has_option("action"))
423        action_id = cmdline.get_option< cmdline::int_option >("action");
424
425    const result_types types = get_result_types(cmdline);
426    const fs::path directory =
427        cmdline.get_option< cmdline::path_option >("output");
428    create_top_directory(directory, cmdline.has_option("force"));
429    html_hooks hooks(ui, directory, types);
430    scan_action::drive(store_path(cmdline), action_id, hooks);
431    hooks.write_summary();
432
433    return EXIT_SUCCESS;
434}
435