1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30#include <cstdlib> 31#include <fstream> 32#include <iostream> 33#include <memory> 34#include <utility> 35#include <vector> 36 37#include "atf-c++/detail/application.hpp" 38#include "atf-c++/detail/fs.hpp" 39#include "atf-c++/detail/sanity.hpp" 40#include "atf-c++/detail/text.hpp" 41#include "atf-c++/detail/ui.hpp" 42 43#include "reader.hpp" 44 45typedef std::auto_ptr< std::ostream > ostream_ptr; 46 47ostream_ptr 48open_outfile(const atf::fs::path& path) 49{ 50 ostream_ptr osp; 51 if (path.str() == "-") 52 osp = ostream_ptr(new std::ofstream("/dev/stdout")); 53 else 54 osp = ostream_ptr(new std::ofstream(path.c_str())); 55 if (!(*osp)) 56 throw std::runtime_error("Could not create file " + path.str()); 57 return osp; 58} 59 60// ------------------------------------------------------------------------ 61// The "writer" interface. 62// ------------------------------------------------------------------------ 63 64//! 65//! \brief A base class that defines an output format. 66//! 67//! The writer base class defines a generic interface to output formats. 68//! This is meant to be subclassed, and each subclass can redefine any 69//! method to format the information as it wishes. 70//! 71//! This class is not tied to a output stream nor a file because, depending 72//! on the output format, we will want to write to a single file or to 73//! multiple ones. 74//! 75class writer { 76public: 77 writer(void) {} 78 virtual ~writer(void) {} 79 80 virtual void write_info(const std::string&, const std::string&) {} 81 virtual void write_ntps(size_t) {} 82 virtual void write_tp_start(const std::string&, size_t) {} 83 virtual void write_tp_end(const std::string&) {} 84 virtual void write_tc_start(const std::string&) {} 85 virtual void write_tc_stdout_line(const std::string&) {} 86 virtual void write_tc_stderr_line(const std::string&) {} 87 virtual void write_tc_end(const std::string&, const std::string&) {} 88 virtual void write_eof(void) {} 89}; 90 91// ------------------------------------------------------------------------ 92// The "csv_writer" class. 93// ------------------------------------------------------------------------ 94 95//! 96//! \brief A very simple plain-text output format. 97//! 98//! The csv_writer class implements a very simple plain-text output 99//! format that summarizes the results of each executed test case. The 100//! results are meant to be easily parseable by third-party tools, hence 101//! they are formatted as a CSV file. 102//! 103class csv_writer : public writer { 104 ostream_ptr m_os; 105 bool m_failed; 106 107 std::string m_tpname; 108 std::string m_tcname; 109 110public: 111 csv_writer(const atf::fs::path& p) : 112 m_os(open_outfile(p)) 113 { 114 } 115 116 virtual 117 void 118 write_tp_start(const std::string& name, size_t ntcs) 119 { 120 m_tpname = name; 121 m_failed = false; 122 } 123 124 virtual 125 void 126 write_tp_end(const std::string& reason) 127 { 128 if (!reason.empty()) 129 (*m_os) << "tp, " << m_tpname << ", bogus, " << reason << "\n"; 130 else if (m_failed) 131 (*m_os) << "tp, " << m_tpname << ", failed\n"; 132 else 133 (*m_os) << "tp, " << m_tpname << ", passed\n"; 134 } 135 136 virtual 137 void 138 write_tc_start(const std::string& name) 139 { 140 m_tcname = name; 141 } 142 143 virtual 144 void 145 write_tc_end(const std::string& state, const std::string& reason) 146 { 147 std::string str = "tc, " + m_tpname + ", " + m_tcname + ", " + state; 148 if (!reason.empty()) 149 str += ", " + reason; 150 (*m_os) << str << "\n"; 151 152 if (state == "failed") 153 m_failed = true; 154 } 155}; 156 157// ------------------------------------------------------------------------ 158// The "ticker_writer" class. 159// ------------------------------------------------------------------------ 160 161//! 162//! \brief A console-friendly output format. 163//! 164//! The ticker_writer class implements a formatter that is user-friendly 165//! in the sense that it shows the execution of test cases in an easy to 166//! read format. It is not meant to be parseable and its format can 167//! freely change across releases. 168//! 169class ticker_writer : public writer { 170 ostream_ptr m_os; 171 172 size_t m_curtp, m_ntps; 173 size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped, m_tcs_expected_failures; 174 std::string m_tcname, m_tpname; 175 std::vector< std::string > m_failed_tcs; 176 std::map< std::string, std::string > m_expected_failures_tcs; 177 std::vector< std::string > m_failed_tps; 178 179 void 180 write_info(const std::string& what, const std::string& val) 181 { 182 if (what == "tests.root") { 183 (*m_os) << "Tests root: " << val << "\n\n"; 184 } 185 } 186 187 void 188 write_ntps(size_t ntps) 189 { 190 m_curtp = 1; 191 m_tcs_passed = 0; 192 m_tcs_failed = 0; 193 m_tcs_skipped = 0; 194 m_tcs_expected_failures = 0; 195 m_ntps = ntps; 196 } 197 198 void 199 write_tp_start(const std::string& tp, size_t ntcs) 200 { 201 using atf::text::to_string; 202 using atf::ui::format_text; 203 204 m_tpname = tp; 205 206 (*m_os) << format_text(tp + " (" + to_string(m_curtp) + 207 "/" + to_string(m_ntps) + "): " + 208 to_string(ntcs) + " test cases") 209 << "\n"; 210 (*m_os).flush(); 211 } 212 213 void 214 write_tp_end(const std::string& reason) 215 { 216 using atf::ui::format_text_with_tag; 217 218 m_curtp++; 219 220 if (!reason.empty()) { 221 (*m_os) << format_text_with_tag("BOGUS TEST PROGRAM: Cannot " 222 "trust its results because " 223 "of `" + reason + "'", 224 m_tpname + ": ", false) 225 << "\n"; 226 m_failed_tps.push_back(m_tpname); 227 } 228 (*m_os) << "\n"; 229 (*m_os).flush(); 230 231 m_tpname.clear(); 232 } 233 234 void 235 write_tc_start(const std::string& tcname) 236 { 237 m_tcname = tcname; 238 239 (*m_os) << " " + tcname + ": "; 240 (*m_os).flush(); 241 } 242 243 void 244 write_tc_end(const std::string& state, const std::string& reason) 245 { 246 std::string str; 247 248 if (state == "expected_death" || state == "expected_exit" || 249 state == "expected_failure" || state == "expected_signal" || 250 state == "expected_timeout") { 251 str = "Expected failure: " + reason; 252 m_tcs_expected_failures++; 253 m_expected_failures_tcs[m_tpname + ":" + m_tcname] = reason; 254 } else if (state == "failed") { 255 str = "Failed: " + reason; 256 m_tcs_failed++; 257 m_failed_tcs.push_back(m_tpname + ":" + m_tcname); 258 } else if (state == "passed") { 259 str = "Passed."; 260 m_tcs_passed++; 261 } else if (state == "skipped") { 262 str = "Skipped: " + reason; 263 m_tcs_skipped++; 264 } else 265 UNREACHABLE; 266 267 // XXX Wrap text. format_text_with_tag does not currently allow 268 // to specify the current column, which is needed because we have 269 // already printed the tc's name. 270 (*m_os) << str << "\n"; 271 272 m_tcname = ""; 273 } 274 275 static void 276 write_expected_failures(const std::map< std::string, std::string >& xfails, 277 std::ostream& os) 278 { 279 using atf::ui::format_text; 280 using atf::ui::format_text_with_tag; 281 282 os << format_text("Test cases for known bugs:") << "\n"; 283 284 for (std::map< std::string, std::string >::const_iterator iter = 285 xfails.begin(); iter != xfails.end(); iter++) { 286 const std::string& name = (*iter).first; 287 const std::string& reason = (*iter).second; 288 289 os << format_text_with_tag(reason, " " + name + ": ", false) 290 << "\n"; 291 } 292 } 293 294 void 295 write_eof(void) 296 { 297 using atf::text::join; 298 using atf::text::to_string; 299 using atf::ui::format_text; 300 using atf::ui::format_text_with_tag; 301 302 if (!m_failed_tps.empty()) { 303 (*m_os) << format_text("Failed (bogus) test programs:") 304 << "\n"; 305 (*m_os) << format_text_with_tag(join(m_failed_tps, ", "), 306 " ", false) << "\n\n"; 307 } 308 309 if (!m_expected_failures_tcs.empty()) { 310 write_expected_failures(m_expected_failures_tcs, *m_os); 311 (*m_os) << "\n"; 312 } 313 314 if (!m_failed_tcs.empty()) { 315 (*m_os) << format_text("Failed test cases:") << "\n"; 316 (*m_os) << format_text_with_tag(join(m_failed_tcs, ", "), 317 " ", false) << "\n\n"; 318 } 319 320 (*m_os) << format_text("Summary for " + to_string(m_ntps) + 321 " test programs:") << "\n"; 322 (*m_os) << format_text_with_tag(to_string(m_tcs_passed) + 323 " passed test cases.", 324 " ", false) << "\n"; 325 (*m_os) << format_text_with_tag(to_string(m_tcs_failed) + 326 " failed test cases.", 327 " ", false) << "\n"; 328 (*m_os) << format_text_with_tag(to_string(m_tcs_expected_failures) + 329 " expected failed test cases.", 330 " ", false) << "\n"; 331 (*m_os) << format_text_with_tag(to_string(m_tcs_skipped) + 332 " skipped test cases.", 333 " ", false) << "\n"; 334 } 335 336public: 337 ticker_writer(const atf::fs::path& p) : 338 m_os(open_outfile(p)) 339 { 340 } 341}; 342 343// ------------------------------------------------------------------------ 344// The "xml" class. 345// ------------------------------------------------------------------------ 346 347//! 348//! \brief A single-file XML output format. 349//! 350//! The xml_writer class implements a formatter that prints the results 351//! of test cases in an XML format easily parseable later on by other 352//! utilities. 353//! 354class xml_writer : public writer { 355 ostream_ptr m_os; 356 357 size_t m_curtp, m_ntps; 358 std::string m_tcname, m_tpname; 359 360 static 361 std::string 362 attrval(const std::string& str) 363 { 364 return str; 365 } 366 367 static 368 std::string 369 elemval(const std::string& str) 370 { 371 std::string ostr; 372 for (std::string::const_iterator iter = str.begin(); 373 iter != str.end(); iter++) { 374 switch (*iter) { 375 case '&': ostr += "&"; break; 376 case '<': ostr += "<"; break; 377 case '>': ostr += ">"; break; 378 default: ostr += *iter; 379 } 380 } 381 return ostr; 382 } 383 384 void 385 write_info(const std::string& what, const std::string& val) 386 { 387 (*m_os) << "<info class=\"" << what << "\">" << val << "</info>\n"; 388 } 389 390 void 391 write_tp_start(const std::string& tp, size_t ntcs) 392 { 393 (*m_os) << "<tp id=\"" << attrval(tp) << "\">\n"; 394 } 395 396 void 397 write_tp_end(const std::string& reason) 398 { 399 if (!reason.empty()) 400 (*m_os) << "<failed>" << elemval(reason) << "</failed>\n"; 401 (*m_os) << "</tp>\n"; 402 } 403 404 void 405 write_tc_start(const std::string& tcname) 406 { 407 (*m_os) << "<tc id=\"" << attrval(tcname) << "\">\n"; 408 } 409 410 void 411 write_tc_stdout_line(const std::string& line) 412 { 413 (*m_os) << "<so>" << elemval(line) << "</so>\n"; 414 } 415 416 void 417 write_tc_stderr_line(const std::string& line) 418 { 419 (*m_os) << "<se>" << elemval(line) << "</se>\n"; 420 } 421 422 void 423 write_tc_end(const std::string& state, const std::string& reason) 424 { 425 std::string str; 426 427 if (state == "expected_death" || state == "expected_exit" || 428 state == "expected_failure" || state == "expected_signal" || 429 state == "expected_timeout") { 430 (*m_os) << "<" << state << ">" << elemval(reason) 431 << "</" << state << ">\n"; 432 } else if (state == "passed") { 433 (*m_os) << "<passed />\n"; 434 } else if (state == "failed") { 435 (*m_os) << "<failed>" << elemval(reason) << "</failed>\n"; 436 } else if (state == "skipped") { 437 (*m_os) << "<skipped>" << elemval(reason) << "</skipped>\n"; 438 } else 439 UNREACHABLE; 440 (*m_os) << "</tc>\n"; 441 } 442 443 void 444 write_eof(void) 445 { 446 (*m_os) << "</tests-results>\n"; 447 } 448 449public: 450 xml_writer(const atf::fs::path& p) : 451 m_os(open_outfile(p)) 452 { 453 (*m_os) << "<?xml version=\"1.0\"?>\n" 454 << "<!DOCTYPE tests-results PUBLIC " 455 "\"-//NetBSD//DTD ATF Tests Results 0.1//EN\" " 456 "\"http://www.NetBSD.org/XML/atf/tests-results.dtd\">\n\n" 457 "<tests-results>\n"; 458 } 459}; 460 461// ------------------------------------------------------------------------ 462// The "converter" class. 463// ------------------------------------------------------------------------ 464 465//! 466//! \brief A reader that redirects events to multiple writers. 467//! 468//! The converter class implements an atf_tps_reader that, for each event 469//! raised by the parser, redirects it to multiple writers so that they 470//! can reformat it according to their output rules. 471//! 472class converter : public atf::atf_report::atf_tps_reader { 473 typedef std::vector< writer* > outs_vector; 474 outs_vector m_outs; 475 476 void 477 got_info(const std::string& what, const std::string& val) 478 { 479 for (outs_vector::iterator iter = m_outs.begin(); 480 iter != m_outs.end(); iter++) 481 (*iter)->write_info(what, val); 482 } 483 484 void 485 got_ntps(size_t ntps) 486 { 487 for (outs_vector::iterator iter = m_outs.begin(); 488 iter != m_outs.end(); iter++) 489 (*iter)->write_ntps(ntps); 490 } 491 492 void 493 got_tp_start(const std::string& tp, size_t ntcs) 494 { 495 for (outs_vector::iterator iter = m_outs.begin(); 496 iter != m_outs.end(); iter++) 497 (*iter)->write_tp_start(tp, ntcs); 498 } 499 500 void 501 got_tp_end(const std::string& reason) 502 { 503 for (outs_vector::iterator iter = m_outs.begin(); 504 iter != m_outs.end(); iter++) 505 (*iter)->write_tp_end(reason); 506 } 507 508 void 509 got_tc_start(const std::string& tcname) 510 { 511 for (outs_vector::iterator iter = m_outs.begin(); 512 iter != m_outs.end(); iter++) 513 (*iter)->write_tc_start(tcname); 514 } 515 516 void 517 got_tc_stdout_line(const std::string& line) 518 { 519 for (outs_vector::iterator iter = m_outs.begin(); 520 iter != m_outs.end(); iter++) 521 (*iter)->write_tc_stdout_line(line); 522 } 523 524 void 525 got_tc_stderr_line(const std::string& line) 526 { 527 for (outs_vector::iterator iter = m_outs.begin(); 528 iter != m_outs.end(); iter++) 529 (*iter)->write_tc_stderr_line(line); 530 } 531 532 void 533 got_tc_end(const std::string& state, const std::string& reason) 534 { 535 for (outs_vector::iterator iter = m_outs.begin(); 536 iter != m_outs.end(); iter++) 537 (*iter)->write_tc_end(state, reason); 538 } 539 540 void 541 got_eof(void) 542 { 543 for (outs_vector::iterator iter = m_outs.begin(); 544 iter != m_outs.end(); iter++) 545 (*iter)->write_eof(); 546 } 547 548public: 549 converter(std::istream& is) : 550 atf::atf_report::atf_tps_reader(is) 551 { 552 } 553 554 ~converter(void) 555 { 556 for (outs_vector::iterator iter = m_outs.begin(); 557 iter != m_outs.end(); iter++) 558 delete *iter; 559 } 560 561 void 562 add_output(const std::string& fmt, const atf::fs::path& p) 563 { 564 if (fmt == "csv") { 565 m_outs.push_back(new csv_writer(p)); 566 } else if (fmt == "ticker") { 567 m_outs.push_back(new ticker_writer(p)); 568 } else if (fmt == "xml") { 569 m_outs.push_back(new xml_writer(p)); 570 } else 571 throw std::runtime_error("Unknown format `" + fmt + "'"); 572 } 573}; 574 575// ------------------------------------------------------------------------ 576// The "atf_report" class. 577// ------------------------------------------------------------------------ 578 579class atf_report : public atf::application::app { 580 static const char* m_description; 581 582 typedef std::pair< std::string, atf::fs::path > fmt_path_pair; 583 std::vector< fmt_path_pair > m_oflags; 584 585 void process_option(int, const char*); 586 options_set specific_options(void) const; 587 588public: 589 atf_report(void); 590 591 int main(void); 592}; 593 594const char* atf_report::m_description = 595 "atf-report is a tool that parses the output of atf-run and " 596 "generates user-friendly reports in multiple different formats."; 597 598atf_report::atf_report(void) : 599 app(m_description, "atf-report(1)", "atf(7)") 600{ 601} 602 603void 604atf_report::process_option(int ch, const char* arg) 605{ 606 switch (ch) { 607 case 'o': 608 { 609 std::string str(arg); 610 std::string::size_type pos = str.find(':'); 611 if (pos == std::string::npos) 612 throw std::runtime_error("Syntax error in -o option"); 613 else { 614 std::string fmt = str.substr(0, pos); 615 atf::fs::path path = atf::fs::path(str.substr(pos + 1)); 616 m_oflags.push_back(fmt_path_pair(fmt, path)); 617 } 618 } 619 break; 620 621 default: 622 UNREACHABLE; 623 } 624} 625 626atf_report::options_set 627atf_report::specific_options(void) 628 const 629{ 630 using atf::application::option; 631 options_set opts; 632 opts.insert(option('o', "fmt:path", "Adds a new output file; multiple " 633 "ones can be specified, and a - " 634 "path means stdout")); 635 return opts; 636} 637 638int 639atf_report::main(void) 640{ 641 if (m_oflags.empty()) 642 m_oflags.push_back(fmt_path_pair("ticker", atf::fs::path("-"))); 643 644 // Look for path duplicates. 645 std::set< atf::fs::path > paths; 646 for (std::vector< fmt_path_pair >::const_iterator iter = m_oflags.begin(); 647 iter != m_oflags.end(); iter++) { 648 atf::fs::path p = (*iter).second; 649 if (p == atf::fs::path("/dev/stdout")) 650 p = atf::fs::path("-"); 651 if (paths.find(p) != paths.end()) 652 throw std::runtime_error("The file `" + p.str() + "' was " 653 "specified more than once"); 654 paths.insert((*iter).second); 655 } 656 657 // Generate the output files. 658 converter cnv(std::cin); 659 for (std::vector< fmt_path_pair >::const_iterator iter = m_oflags.begin(); 660 iter != m_oflags.end(); iter++) 661 cnv.add_output((*iter).first, (*iter).second); 662 cnv.read(); 663 664 return EXIT_SUCCESS; 665} 666 667int 668main(int argc, char* const* argv) 669{ 670 return atf_report().run(argc, argv); 671} 672