1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30#include <cstdlib>
31#include <fstream>
32#include <iostream>
33#include <memory>
34#include <utility>
35#include <vector>
36
37#include "atf-c++/detail/application.hpp"
38#include "atf-c++/detail/fs.hpp"
39#include "atf-c++/detail/sanity.hpp"
40#include "atf-c++/detail/text.hpp"
41#include "atf-c++/detail/ui.hpp"
42
43#include "reader.hpp"
44
45typedef std::auto_ptr< std::ostream > ostream_ptr;
46
47ostream_ptr
48open_outfile(const atf::fs::path& path)
49{
50    ostream_ptr osp;
51    if (path.str() == "-")
52        osp = ostream_ptr(new std::ofstream("/dev/stdout"));
53    else
54        osp = ostream_ptr(new std::ofstream(path.c_str()));
55    if (!(*osp))
56        throw std::runtime_error("Could not create file " + path.str());
57    return osp;
58}
59
60// ------------------------------------------------------------------------
61// The "writer" interface.
62// ------------------------------------------------------------------------
63
64//!
65//! \brief A base class that defines an output format.
66//!
67//! The writer base class defines a generic interface to output formats.
68//! This is meant to be subclassed, and each subclass can redefine any
69//! method to format the information as it wishes.
70//!
71//! This class is not tied to a output stream nor a file because, depending
72//! on the output format, we will want to write to a single file or to
73//! multiple ones.
74//!
75class writer {
76public:
77    writer(void) {}
78    virtual ~writer(void) {}
79
80    virtual void write_info(const std::string&, const std::string&) {}
81    virtual void write_ntps(size_t) {}
82    virtual void write_tp_start(const std::string&, size_t) {}
83    virtual void write_tp_end(const std::string&) {}
84    virtual void write_tc_start(const std::string&) {}
85    virtual void write_tc_stdout_line(const std::string&) {}
86    virtual void write_tc_stderr_line(const std::string&) {}
87    virtual void write_tc_end(const std::string&, const std::string&) {}
88    virtual void write_eof(void) {}
89};
90
91// ------------------------------------------------------------------------
92// The "csv_writer" class.
93// ------------------------------------------------------------------------
94
95//!
96//! \brief A very simple plain-text output format.
97//!
98//! The csv_writer class implements a very simple plain-text output
99//! format that summarizes the results of each executed test case.  The
100//! results are meant to be easily parseable by third-party tools, hence
101//! they are formatted as a CSV file.
102//!
103class csv_writer : public writer {
104    ostream_ptr m_os;
105    bool m_failed;
106
107    std::string m_tpname;
108    std::string m_tcname;
109
110public:
111    csv_writer(const atf::fs::path& p) :
112        m_os(open_outfile(p))
113    {
114    }
115
116    virtual
117    void
118    write_tp_start(const std::string& name, size_t ntcs)
119    {
120        m_tpname = name;
121        m_failed = false;
122    }
123
124    virtual
125    void
126    write_tp_end(const std::string& reason)
127    {
128        if (!reason.empty())
129            (*m_os) << "tp, " << m_tpname << ", bogus, " << reason << "\n";
130        else if (m_failed)
131            (*m_os) << "tp, " << m_tpname << ", failed\n";
132        else
133            (*m_os) << "tp, " << m_tpname << ", passed\n";
134    }
135
136    virtual
137    void
138    write_tc_start(const std::string& name)
139    {
140        m_tcname = name;
141    }
142
143    virtual
144    void
145    write_tc_end(const std::string& state, const std::string& reason)
146    {
147        std::string str = "tc, " + m_tpname + ", " + m_tcname + ", " + state;
148        if (!reason.empty())
149            str += ", " + reason;
150        (*m_os) << str << "\n";
151
152        if (state == "failed")
153            m_failed = true;
154    }
155};
156
157// ------------------------------------------------------------------------
158// The "ticker_writer" class.
159// ------------------------------------------------------------------------
160
161//!
162//! \brief A console-friendly output format.
163//!
164//! The ticker_writer class implements a formatter that is user-friendly
165//! in the sense that it shows the execution of test cases in an easy to
166//! read format.  It is not meant to be parseable and its format can
167//! freely change across releases.
168//!
169class ticker_writer : public writer {
170    ostream_ptr m_os;
171
172    size_t m_curtp, m_ntps;
173    size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped, m_tcs_expected_failures;
174    std::string m_tcname, m_tpname;
175    std::vector< std::string > m_failed_tcs;
176    std::map< std::string, std::string > m_expected_failures_tcs;
177    std::vector< std::string > m_failed_tps;
178
179    void
180    write_info(const std::string& what, const std::string& val)
181    {
182        if (what == "tests.root") {
183            (*m_os) << "Tests root: " << val << "\n\n";
184        }
185    }
186
187    void
188    write_ntps(size_t ntps)
189    {
190        m_curtp = 1;
191        m_tcs_passed = 0;
192        m_tcs_failed = 0;
193        m_tcs_skipped = 0;
194        m_tcs_expected_failures = 0;
195        m_ntps = ntps;
196    }
197
198    void
199    write_tp_start(const std::string& tp, size_t ntcs)
200    {
201        using atf::text::to_string;
202        using atf::ui::format_text;
203
204        m_tpname = tp;
205
206        (*m_os) << format_text(tp + " (" + to_string(m_curtp) +
207                               "/" + to_string(m_ntps) + "): " +
208                               to_string(ntcs) + " test cases")
209                << "\n";
210        (*m_os).flush();
211    }
212
213    void
214    write_tp_end(const std::string& reason)
215    {
216        using atf::ui::format_text_with_tag;
217
218        m_curtp++;
219
220        if (!reason.empty()) {
221            (*m_os) << format_text_with_tag("BOGUS TEST PROGRAM: Cannot "
222                                            "trust its results because "
223                                            "of `" + reason + "'",
224                                            m_tpname + ": ", false)
225                    << "\n";
226            m_failed_tps.push_back(m_tpname);
227        }
228        (*m_os) << "\n";
229        (*m_os).flush();
230
231        m_tpname.clear();
232    }
233
234    void
235    write_tc_start(const std::string& tcname)
236    {
237        m_tcname = tcname;
238
239        (*m_os) << "    " + tcname + ": ";
240        (*m_os).flush();
241    }
242
243    void
244    write_tc_end(const std::string& state, const std::string& reason)
245    {
246        std::string str;
247
248        if (state == "expected_death" || state == "expected_exit" ||
249            state == "expected_failure" || state == "expected_signal" ||
250            state == "expected_timeout") {
251            str = "Expected failure: " + reason;
252            m_tcs_expected_failures++;
253            m_expected_failures_tcs[m_tpname + ":" + m_tcname] = reason;
254        } else if (state == "failed") {
255            str = "Failed: " + reason;
256            m_tcs_failed++;
257            m_failed_tcs.push_back(m_tpname + ":" + m_tcname);
258        } else if (state == "passed") {
259            str = "Passed.";
260            m_tcs_passed++;
261        } else if (state == "skipped") {
262            str = "Skipped: " + reason;
263            m_tcs_skipped++;
264        } else
265            UNREACHABLE;
266
267        // XXX Wrap text.  format_text_with_tag does not currently allow
268        // to specify the current column, which is needed because we have
269        // already printed the tc's name.
270        (*m_os) << str << "\n";
271
272        m_tcname = "";
273    }
274
275    static void
276    write_expected_failures(const std::map< std::string, std::string >& xfails,
277                            std::ostream& os)
278    {
279        using atf::ui::format_text;
280        using atf::ui::format_text_with_tag;
281
282        os << format_text("Test cases for known bugs:") << "\n";
283
284        for (std::map< std::string, std::string >::const_iterator iter =
285             xfails.begin(); iter != xfails.end(); iter++) {
286            const std::string& name = (*iter).first;
287            const std::string& reason = (*iter).second;
288
289            os << format_text_with_tag(reason, "    " + name + ": ", false)
290               << "\n";
291        }
292    }
293
294    void
295    write_eof(void)
296    {
297        using atf::text::join;
298        using atf::text::to_string;
299        using atf::ui::format_text;
300        using atf::ui::format_text_with_tag;
301
302        if (!m_failed_tps.empty()) {
303            (*m_os) << format_text("Failed (bogus) test programs:")
304                    << "\n";
305            (*m_os) << format_text_with_tag(join(m_failed_tps, ", "),
306                                            "    ", false) << "\n\n";
307        }
308
309        if (!m_expected_failures_tcs.empty()) {
310            write_expected_failures(m_expected_failures_tcs, *m_os);
311            (*m_os) << "\n";
312        }
313
314        if (!m_failed_tcs.empty()) {
315            (*m_os) << format_text("Failed test cases:") << "\n";
316            (*m_os) << format_text_with_tag(join(m_failed_tcs, ", "),
317                                            "    ", false) << "\n\n";
318        }
319
320        (*m_os) << format_text("Summary for " + to_string(m_ntps) +
321                               " test programs:") << "\n";
322        (*m_os) << format_text_with_tag(to_string(m_tcs_passed) +
323                                        " passed test cases.",
324                                        "    ", false) << "\n";
325        (*m_os) << format_text_with_tag(to_string(m_tcs_failed) +
326                                        " failed test cases.",
327                                        "    ", false) << "\n";
328        (*m_os) << format_text_with_tag(to_string(m_tcs_expected_failures) +
329                                        " expected failed test cases.",
330                                        "    ", false) << "\n";
331        (*m_os) << format_text_with_tag(to_string(m_tcs_skipped) +
332                                        " skipped test cases.",
333                                        "    ", false) << "\n";
334    }
335
336public:
337    ticker_writer(const atf::fs::path& p) :
338        m_os(open_outfile(p))
339    {
340    }
341};
342
343// ------------------------------------------------------------------------
344// The "xml" class.
345// ------------------------------------------------------------------------
346
347//!
348//! \brief A single-file XML output format.
349//!
350//! The xml_writer class implements a formatter that prints the results
351//! of test cases in an XML format easily parseable later on by other
352//! utilities.
353//!
354class xml_writer : public writer {
355    ostream_ptr m_os;
356
357    size_t m_curtp, m_ntps;
358    std::string m_tcname, m_tpname;
359
360    static
361    std::string
362    attrval(const std::string& str)
363    {
364        return str;
365    }
366
367    static
368    std::string
369    elemval(const std::string& str)
370    {
371        std::string ostr;
372        for (std::string::const_iterator iter = str.begin();
373             iter != str.end(); iter++) {
374            switch (*iter) {
375            case '&': ostr += "&amp;"; break;
376            case '<': ostr += "&lt;"; break;
377            case '>': ostr += "&gt;"; break;
378            default:  ostr += *iter;
379            }
380        }
381        return ostr;
382    }
383
384    void
385    write_info(const std::string& what, const std::string& val)
386    {
387        (*m_os) << "<info class=\"" << what << "\">" << val << "</info>\n";
388    }
389
390    void
391    write_tp_start(const std::string& tp, size_t ntcs)
392    {
393        (*m_os) << "<tp id=\"" << attrval(tp) << "\">\n";
394    }
395
396    void
397    write_tp_end(const std::string& reason)
398    {
399        if (!reason.empty())
400            (*m_os) << "<failed>" << elemval(reason) << "</failed>\n";
401        (*m_os) << "</tp>\n";
402    }
403
404    void
405    write_tc_start(const std::string& tcname)
406    {
407        (*m_os) << "<tc id=\"" << attrval(tcname) << "\">\n";
408    }
409
410    void
411    write_tc_stdout_line(const std::string& line)
412    {
413        (*m_os) << "<so>" << elemval(line) << "</so>\n";
414    }
415
416    void
417    write_tc_stderr_line(const std::string& line)
418    {
419        (*m_os) << "<se>" << elemval(line) << "</se>\n";
420    }
421
422    void
423    write_tc_end(const std::string& state, const std::string& reason)
424    {
425        std::string str;
426
427        if (state == "expected_death" || state == "expected_exit" ||
428            state == "expected_failure" || state == "expected_signal" ||
429            state == "expected_timeout") {
430            (*m_os) << "<" << state << ">" << elemval(reason)
431                    << "</" << state << ">\n";
432        } else if (state == "passed") {
433            (*m_os) << "<passed />\n";
434        } else if (state == "failed") {
435            (*m_os) << "<failed>" << elemval(reason) << "</failed>\n";
436        } else if (state == "skipped") {
437            (*m_os) << "<skipped>" << elemval(reason) << "</skipped>\n";
438        } else
439            UNREACHABLE;
440        (*m_os) << "</tc>\n";
441    }
442
443    void
444    write_eof(void)
445    {
446        (*m_os) << "</tests-results>\n";
447    }
448
449public:
450    xml_writer(const atf::fs::path& p) :
451        m_os(open_outfile(p))
452    {
453        (*m_os) << "<?xml version=\"1.0\"?>\n"
454                << "<!DOCTYPE tests-results PUBLIC "
455                   "\"-//NetBSD//DTD ATF Tests Results 0.1//EN\" "
456                   "\"http://www.NetBSD.org/XML/atf/tests-results.dtd\">\n\n"
457                   "<tests-results>\n";
458    }
459};
460
461// ------------------------------------------------------------------------
462// The "converter" class.
463// ------------------------------------------------------------------------
464
465//!
466//! \brief A reader that redirects events to multiple writers.
467//!
468//! The converter class implements an atf_tps_reader that, for each event
469//! raised by the parser, redirects it to multiple writers so that they
470//! can reformat it according to their output rules.
471//!
472class converter : public atf::atf_report::atf_tps_reader {
473    typedef std::vector< writer* > outs_vector;
474    outs_vector m_outs;
475
476    void
477    got_info(const std::string& what, const std::string& val)
478    {
479        for (outs_vector::iterator iter = m_outs.begin();
480             iter != m_outs.end(); iter++)
481            (*iter)->write_info(what, val);
482    }
483
484    void
485    got_ntps(size_t ntps)
486    {
487        for (outs_vector::iterator iter = m_outs.begin();
488             iter != m_outs.end(); iter++)
489            (*iter)->write_ntps(ntps);
490    }
491
492    void
493    got_tp_start(const std::string& tp, size_t ntcs)
494    {
495        for (outs_vector::iterator iter = m_outs.begin();
496             iter != m_outs.end(); iter++)
497            (*iter)->write_tp_start(tp, ntcs);
498    }
499
500    void
501    got_tp_end(const std::string& reason)
502    {
503        for (outs_vector::iterator iter = m_outs.begin();
504             iter != m_outs.end(); iter++)
505            (*iter)->write_tp_end(reason);
506    }
507
508    void
509    got_tc_start(const std::string& tcname)
510    {
511        for (outs_vector::iterator iter = m_outs.begin();
512             iter != m_outs.end(); iter++)
513            (*iter)->write_tc_start(tcname);
514    }
515
516    void
517    got_tc_stdout_line(const std::string& line)
518    {
519        for (outs_vector::iterator iter = m_outs.begin();
520             iter != m_outs.end(); iter++)
521            (*iter)->write_tc_stdout_line(line);
522    }
523
524    void
525    got_tc_stderr_line(const std::string& line)
526    {
527        for (outs_vector::iterator iter = m_outs.begin();
528             iter != m_outs.end(); iter++)
529            (*iter)->write_tc_stderr_line(line);
530    }
531
532    void
533    got_tc_end(const std::string& state, const std::string& reason)
534    {
535        for (outs_vector::iterator iter = m_outs.begin();
536             iter != m_outs.end(); iter++)
537            (*iter)->write_tc_end(state, reason);
538    }
539
540    void
541    got_eof(void)
542    {
543        for (outs_vector::iterator iter = m_outs.begin();
544             iter != m_outs.end(); iter++)
545            (*iter)->write_eof();
546    }
547
548public:
549    converter(std::istream& is) :
550        atf::atf_report::atf_tps_reader(is)
551    {
552    }
553
554    ~converter(void)
555    {
556        for (outs_vector::iterator iter = m_outs.begin();
557             iter != m_outs.end(); iter++)
558            delete *iter;
559    }
560
561    void
562    add_output(const std::string& fmt, const atf::fs::path& p)
563    {
564        if (fmt == "csv") {
565            m_outs.push_back(new csv_writer(p));
566        } else if (fmt == "ticker") {
567            m_outs.push_back(new ticker_writer(p));
568        } else if (fmt == "xml") {
569            m_outs.push_back(new xml_writer(p));
570        } else
571            throw std::runtime_error("Unknown format `" + fmt + "'");
572    }
573};
574
575// ------------------------------------------------------------------------
576// The "atf_report" class.
577// ------------------------------------------------------------------------
578
579class atf_report : public atf::application::app {
580    static const char* m_description;
581
582    typedef std::pair< std::string, atf::fs::path > fmt_path_pair;
583    std::vector< fmt_path_pair > m_oflags;
584
585    void process_option(int, const char*);
586    options_set specific_options(void) const;
587
588public:
589    atf_report(void);
590
591    int main(void);
592};
593
594const char* atf_report::m_description =
595    "atf-report is a tool that parses the output of atf-run and "
596    "generates user-friendly reports in multiple different formats.";
597
598atf_report::atf_report(void) :
599    app(m_description, "atf-report(1)", "atf(7)")
600{
601}
602
603void
604atf_report::process_option(int ch, const char* arg)
605{
606    switch (ch) {
607    case 'o':
608        {
609            std::string str(arg);
610            std::string::size_type pos = str.find(':');
611            if (pos == std::string::npos)
612                throw std::runtime_error("Syntax error in -o option");
613            else {
614                std::string fmt = str.substr(0, pos);
615                atf::fs::path path = atf::fs::path(str.substr(pos + 1));
616                m_oflags.push_back(fmt_path_pair(fmt, path));
617            }
618        }
619        break;
620
621    default:
622        UNREACHABLE;
623    }
624}
625
626atf_report::options_set
627atf_report::specific_options(void)
628    const
629{
630    using atf::application::option;
631    options_set opts;
632    opts.insert(option('o', "fmt:path", "Adds a new output file; multiple "
633                                        "ones can be specified, and a - "
634                                        "path means stdout"));
635    return opts;
636}
637
638int
639atf_report::main(void)
640{
641    if (m_oflags.empty())
642        m_oflags.push_back(fmt_path_pair("ticker", atf::fs::path("-")));
643
644    // Look for path duplicates.
645    std::set< atf::fs::path > paths;
646    for (std::vector< fmt_path_pair >::const_iterator iter = m_oflags.begin();
647         iter != m_oflags.end(); iter++) {
648        atf::fs::path p = (*iter).second;
649        if (p == atf::fs::path("/dev/stdout"))
650            p = atf::fs::path("-");
651        if (paths.find(p) != paths.end())
652            throw std::runtime_error("The file `" + p.str() + "' was "
653                                     "specified more than once");
654        paths.insert((*iter).second);
655    }
656
657    // Generate the output files.
658    converter cnv(std::cin);
659    for (std::vector< fmt_path_pair >::const_iterator iter = m_oflags.begin();
660         iter != m_oflags.end(); iter++)
661        cnv.add_output((*iter).first, (*iter).second);
662    cnv.read();
663
664    return EXIT_SUCCESS;
665}
666
667int
668main(int argc, char* const* argv)
669{
670    return atf_report().run(argc, argv);
671}
672