1// Copyright 2011 Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9//   notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above copyright
11//   notice, this list of conditions and the following disclaimer in the
12//   documentation and/or other materials provided with the distribution.
13// * Neither the name of Google Inc. nor the names of its contributors
14//   may be used to endorse or promote products derived from this software
15//   without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#include "cli/cmd_report.hpp"
30
31#include <cstddef>
32#include <cstdlib>
33#include <fstream>
34#include <map>
35#include <vector>
36
37#include "cli/common.ipp"
38#include "engine/action.hpp"
39#include "engine/context.hpp"
40#include "engine/drivers/scan_action.hpp"
41#include "engine/test_result.hpp"
42#include "utils/cmdline/exceptions.hpp"
43#include "utils/cmdline/parser.ipp"
44#include "utils/defs.hpp"
45#include "utils/format/macros.hpp"
46#include "utils/optional.ipp"
47
48namespace cmdline = utils::cmdline;
49namespace config = utils::config;
50namespace datetime = utils::datetime;
51namespace fs = utils::fs;
52namespace scan_action = engine::drivers::scan_action;
53
54using cli::cmd_report;
55using utils::optional;
56
57
58namespace {
59
60
61/// Generates a plain-text report intended to be printed to the console.
62class console_hooks : public scan_action::base_hooks {
63    /// Indirection to print the output to the correct file stream.
64    cli::file_writer _writer;
65
66    /// Whether to include the runtime context in the output or not.
67    const bool _show_context;
68
69    /// Collection of result types to include in the report.
70    const cli::result_types& _results_filters;
71
72    /// The action ID loaded.
73    int64_t _action_id;
74
75    /// The total run time of the tests.
76    datetime::delta _runtime;
77
78    /// Representation of a single result.
79    struct result_data {
80        /// The relative path to the test program.
81        fs::path binary_path;
82
83        /// The name of the test case.
84        std::string test_case_name;
85
86        /// The result of the test case.
87        engine::test_result result;
88
89        /// The duration of the test case execution.
90        datetime::delta duration;
91
92        /// Constructs a new results data.
93        ///
94        /// \param binary_path_ The relative path to the test program.
95        /// \param test_case_name_ The name of the test case.
96        /// \param result_ The result of the test case.
97        /// \param duration_ The duration of the test case execution.
98        result_data(const fs::path& binary_path_,
99                    const std::string& test_case_name_,
100                    const engine::test_result& result_,
101                    const datetime::delta& duration_) :
102            binary_path(binary_path_), test_case_name(test_case_name_),
103            result(result_), duration(duration_)
104        {
105        }
106    };
107
108    /// Results received, broken down by their type.
109    ///
110    /// Note that this may not include all results, as keeping the whole list in
111    /// memory may be too much.
112    std::map< engine::test_result::result_type,
113              std::vector< result_data > > _results;
114
115    /// Prints the execution context to the output.
116    ///
117    /// \param context The context to dump.
118    void
119    print_context(const engine::context& context)
120    {
121        _writer("===> Execution context");
122
123        _writer(F("Current directory: %s") % context.cwd());
124        const std::map< std::string, std::string >& env = context.env();
125        if (env.empty())
126            _writer("No environment variables recorded");
127        else {
128            _writer("Environment variables:");
129            for (std::map< std::string, std::string >::const_iterator
130                     iter = env.begin(); iter != env.end(); iter++) {
131                _writer(F("    %s=%s") % (*iter).first % (*iter).second);
132            }
133        }
134    }
135
136    /// Counts how many results of a given type have been received.
137    std::size_t
138    count_results(const engine::test_result::result_type type)
139    {
140        const std::map< engine::test_result::result_type,
141                        std::vector< result_data > >::const_iterator iter =
142            _results.find(type);
143        if (iter == _results.end())
144            return 0;
145        else
146            return (*iter).second.size();
147    }
148
149    /// Prints a set of results.
150    void
151    print_results(const engine::test_result::result_type type,
152                  const char* title)
153    {
154        const std::map< engine::test_result::result_type,
155                        std::vector< result_data > >::const_iterator iter2 =
156            _results.find(type);
157        if (iter2 == _results.end())
158            return;
159        const std::vector< result_data >& all = (*iter2).second;
160
161        _writer(F("===> %s") % title);
162        for (std::vector< result_data >::const_iterator iter = all.begin();
163             iter != all.end(); iter++) {
164            _writer(F("%s:%s  ->  %s  [%s]") % (*iter).binary_path %
165                    (*iter).test_case_name %
166                    cli::format_result((*iter).result) %
167                    cli::format_delta((*iter).duration));
168        }
169    }
170
171public:
172    /// Constructor for the hooks.
173    ///
174    /// \param ui_ The user interface object of the caller command.
175    /// \param outfile_ The file to which to send the output.
176    /// \param show_context_ Whether to include the runtime context in
177    ///     the output or not.
178    /// \param results_filters_ The result types to include in the report.
179    ///     Cannot be empty.
180    console_hooks(cmdline::ui* ui_, const fs::path& outfile_,
181                  const bool show_context_,
182                  const cli::result_types& results_filters_) :
183        _writer(ui_, outfile_),
184        _show_context(show_context_),
185        _results_filters(results_filters_)
186    {
187        PRE(!results_filters_.empty());
188    }
189
190    /// Callback executed when an action is found.
191    ///
192    /// \param action_id The identifier of the loaded action.
193    /// \param action The action loaded from the database.
194    void
195    got_action(const int64_t action_id, const engine::action& action)
196    {
197        _action_id = action_id;
198        if (_show_context)
199            print_context(action.runtime_context());
200    }
201
202    /// Callback executed when a test results is found.
203    ///
204    /// \param iter Container for the test result's data.
205    void
206    got_result(store::results_iterator& iter)
207    {
208        _runtime += iter.duration();
209        const engine::test_result result = iter.result();
210        _results[result.type()].push_back(
211            result_data(iter.test_program()->relative_path(),
212                        iter.test_case_name(), iter.result(), iter.duration()));
213    }
214
215    /// Prints the tests summary.
216    void
217    print_tests(void)
218    {
219        using engine::test_result;
220        typedef std::map< test_result::result_type, const char* > types_map;
221
222        types_map titles;
223        titles[engine::test_result::broken] = "Broken tests";
224        titles[engine::test_result::expected_failure] = "Expected failures";
225        titles[engine::test_result::failed] = "Failed tests";
226        titles[engine::test_result::passed] = "Passed tests";
227        titles[engine::test_result::skipped] = "Skipped tests";
228
229        for (cli::result_types::const_iterator iter = _results_filters.begin();
230             iter != _results_filters.end(); ++iter) {
231            const types_map::const_iterator match = titles.find(*iter);
232            INV_MSG(match != titles.end(), "Conditional does not match user "
233                    "input validation in parse_types()");
234            print_results((*match).first, (*match).second);
235        }
236
237        const std::size_t broken = count_results(test_result::broken);
238        const std::size_t failed = count_results(test_result::failed);
239        const std::size_t passed = count_results(test_result::passed);
240        const std::size_t skipped = count_results(test_result::skipped);
241        const std::size_t xfail = count_results(test_result::expected_failure);
242        const std::size_t total = broken + failed + passed + skipped + xfail;
243
244        _writer("===> Summary");
245        _writer(F("Action: %s") % _action_id);
246        _writer(F("Test cases: %s total, %s skipped, %s expected failures, "
247                  "%s broken, %s failed") %
248                total % skipped % xfail % broken % failed);
249        _writer(F("Total time: %s") % cli::format_delta(_runtime));
250    }
251};
252
253
254}  // anonymous namespace
255
256
257const fs::path cli::file_writer::_stdout_path("/dev/stdout");
258const fs::path cli::file_writer::_stderr_path("/dev/stderr");
259
260
261/// Constructs a new file_writer wrapper.
262///
263/// \param ui_ The UI object of the caller command.
264/// \param path_ The path to the output file.
265cli::file_writer::file_writer(cmdline::ui* const ui_, const fs::path& path_) :
266    _ui(ui_), _output_path(path_)
267{
268    if (path_ != _stdout_path && path_ != _stderr_path) {
269        _output_file.reset(new std::ofstream(path_.c_str()));
270        if (!*(_output_file)) {
271            throw std::runtime_error(F("Cannot open output file %s") % path_);
272        }
273    }
274}
275
276/// Destructor.
277cli::file_writer::~file_writer(void)
278{
279}
280
281/// Writes a message to the selected output.
282///
283/// \param message The message to write; should not include a termination
284///     new line.
285void
286cli::file_writer::operator()(const std::string& message)
287{
288    if (_output_path == _stdout_path)
289        _ui->out(message);
290    else if (_output_path == _stderr_path)
291        _ui->err(message);
292    else {
293        INV(_output_file.get() != NULL);
294        (*_output_file) << message << '\n';
295    }
296}
297
298
299/// Default constructor for cmd_report.
300cmd_report::cmd_report(void) : cli_command(
301    "report", "", 0, 0,
302    "Generates a user-friendly, plain-text report with the result of a "
303    "previous action")
304{
305    add_option(store_option);
306    add_option(cmdline::bool_option(
307        "show-context", "Include the execution context in the report"));
308    add_option(cmdline::int_option(
309        "action", "The action to report; if not specified, defaults to the "
310        "latest action in the database", "id"));
311    add_option(cmdline::path_option(
312        "output", "The file to which to write the report",
313        "path", "/dev/stdout"));
314    add_option(results_filter_option);
315}
316
317
318/// Entry point for the "report" subcommand.
319///
320/// \param ui Object to interact with the I/O of the program.
321/// \param cmdline Representation of the command line to the subcommand.
322/// \param unused_user_config The runtime configuration of the program.
323///
324/// \return 0 if everything is OK, 1 if the statement is invalid or if there is
325/// any other problem.
326int
327cmd_report::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline,
328                const config::tree& UTILS_UNUSED_PARAM(user_config))
329{
330    optional< int64_t > action_id;
331    if (cmdline.has_option("action"))
332        action_id = cmdline.get_option< cmdline::int_option >("action");
333
334    const result_types types = get_result_types(cmdline);
335    console_hooks hooks(
336        ui, cmdline.get_option< cmdline::path_option >("output"),
337        cmdline.has_option("show-context"), types);
338    scan_action::drive(store_path(cmdline), action_id, hooks);
339    hooks.print_tests();
340
341    return EXIT_SUCCESS;
342}
343