1217309Snwhitehorn// Copyright 2015 The Kyua Authors. 2217309Snwhitehorn// All rights reserved. 3217309Snwhitehorn// 4217309Snwhitehorn// Redistribution and use in source and binary forms, with or without 5217309Snwhitehorn// modification, are permitted provided that the following conditions are 6217309Snwhitehorn// met: 7217309Snwhitehorn// 8217309Snwhitehorn// * Redistributions of source code must retain the above copyright 9217309Snwhitehorn// notice, this list of conditions and the following disclaimer. 10217309Snwhitehorn// * Redistributions in binary form must reproduce the above copyright 11217309Snwhitehorn// notice, this list of conditions and the following disclaimer in the 12217309Snwhitehorn// documentation and/or other materials provided with the distribution. 13217309Snwhitehorn// * Neither the name of Google Inc. nor the names of its contributors 14217309Snwhitehorn// may be used to endorse or promote products derived from this software 15217309Snwhitehorn// without specific prior written permission. 16217309Snwhitehorn// 17217309Snwhitehorn// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18217309Snwhitehorn// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19217309Snwhitehorn// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20217309Snwhitehorn// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21217309Snwhitehorn// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22217309Snwhitehorn// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23217309Snwhitehorn// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24217309Snwhitehorn// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25217309Snwhitehorn// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26217309Snwhitehorn// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/tap_parser.hpp" 30 31#include <fstream> 32 33#include <atf-c++.hpp> 34 35#include "engine/exceptions.hpp" 36#include "utils/format/containers.ipp" 37#include "utils/format/macros.hpp" 38#include "utils/fs/path.hpp" 39 40namespace fs = utils::fs; 41 42 43namespace { 44 45 46/// Helper to execute parse_tap_output() on inline text contents. 47/// 48/// \param contents The TAP output to parse. 49/// 50/// \return The tap_summary object resultingafter the parse. 51/// 52/// \throw engine::load_error If parse_tap_output() fails. 53static engine::tap_summary 54do_parse(const std::string& contents) 55{ 56 std::ofstream output("tap.txt"); 57 ATF_REQUIRE(output); 58 output << contents; 59 output.close(); 60 return engine::parse_tap_output(fs::path("tap.txt")); 61} 62 63 64} // anonymous namespace 65 66 67ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__bailed_out); 68ATF_TEST_CASE_BODY(tap_summary__bailed_out) 69{ 70 const engine::tap_summary summary = engine::tap_summary::new_bailed_out(); 71 ATF_REQUIRE(summary.bailed_out()); 72} 73 74 75ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__some_results); 76ATF_TEST_CASE_BODY(tap_summary__some_results) 77{ 78 const engine::tap_summary summary = engine::tap_summary::new_results( 79 engine::tap_plan(1, 5), 3, 2); 80 ATF_REQUIRE(!summary.bailed_out()); 81 ATF_REQUIRE_EQ(engine::tap_plan(1, 5), summary.plan()); 82 ATF_REQUIRE_EQ(3, summary.ok_count()); 83 ATF_REQUIRE_EQ(2, summary.not_ok_count()); 84} 85 86 87ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__all_skipped); 88ATF_TEST_CASE_BODY(tap_summary__all_skipped) 89{ 90 const engine::tap_summary summary = engine::tap_summary::new_all_skipped( 91 "Skipped"); 92 ATF_REQUIRE(!summary.bailed_out()); 93 ATF_REQUIRE_EQ(engine::tap_plan(1, 0), summary.plan()); 94 ATF_REQUIRE_EQ("Skipped", summary.all_skipped_reason()); 95} 96 97 98ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__equality_operators); 99ATF_TEST_CASE_BODY(tap_summary__equality_operators) 100{ 101 const engine::tap_summary bailed_out = 102 engine::tap_summary::new_bailed_out(); 103 const engine::tap_summary all_skipped_1 = 104 engine::tap_summary::new_all_skipped("Reason 1"); 105 const engine::tap_summary results_1 = 106 engine::tap_summary::new_results(engine::tap_plan(1, 5), 3, 2); 107 108 // Self-equality checks. 109 ATF_REQUIRE( bailed_out == bailed_out); 110 ATF_REQUIRE(!(bailed_out != bailed_out)); 111 ATF_REQUIRE( all_skipped_1 == all_skipped_1); 112 ATF_REQUIRE(!(all_skipped_1 != all_skipped_1)); 113 ATF_REQUIRE( results_1 == results_1); 114 ATF_REQUIRE(!(results_1 != results_1)); 115 116 // Cross-equality checks. 117 ATF_REQUIRE(!(bailed_out == all_skipped_1)); 118 ATF_REQUIRE( bailed_out != all_skipped_1); 119 ATF_REQUIRE(!(bailed_out == results_1)); 120 ATF_REQUIRE( bailed_out != results_1); 121 ATF_REQUIRE(!(all_skipped_1 == results_1)); 122 ATF_REQUIRE( all_skipped_1 != results_1); 123 124 // Checks for the all_skipped "type". 125 const engine::tap_summary all_skipped_2 = 126 engine::tap_summary::new_all_skipped("Reason 2"); 127 ATF_REQUIRE(!(all_skipped_1 == all_skipped_2)); 128 ATF_REQUIRE( all_skipped_1 != all_skipped_2); 129 130 131 // Checks for the results "type", different plan. 132 const engine::tap_summary results_2 = 133 engine::tap_summary::new_results(engine::tap_plan(2, 6), 134 results_1.ok_count(), 135 results_1.not_ok_count()); 136 ATF_REQUIRE(!(results_1 == results_2)); 137 ATF_REQUIRE( results_1 != results_2); 138 139 140 // Checks for the results "type", different counts. 141 const engine::tap_summary results_3 = 142 engine::tap_summary::new_results(results_1.plan(), 143 results_1.not_ok_count(), 144 results_1.ok_count()); 145 ATF_REQUIRE(!(results_1 == results_3)); 146 ATF_REQUIRE( results_1 != results_3); 147} 148 149 150ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__output); 151ATF_TEST_CASE_BODY(tap_summary__output) 152{ 153 { 154 const engine::tap_summary summary = 155 engine::tap_summary::new_bailed_out(); 156 ATF_REQUIRE_EQ( 157 "tap_summary{bailed_out=true}", 158 (F("%s") % summary).str()); 159 } 160 161 { 162 const engine::tap_summary summary = 163 engine::tap_summary::new_results(engine::tap_plan(5, 10), 2, 4); 164 ATF_REQUIRE_EQ( 165 "tap_summary{bailed_out=false, plan=5..10, ok_count=2, " 166 "not_ok_count=4}", 167 (F("%s") % summary).str()); 168 } 169 170 { 171 const engine::tap_summary summary = 172 engine::tap_summary::new_all_skipped("Who knows"); 173 ATF_REQUIRE_EQ( 174 "tap_summary{bailed_out=false, plan=1..0, " 175 "all_skipped_reason=Who knows}", 176 (F("%s") % summary).str()); 177 } 178} 179 180 181ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__only_one_result); 182ATF_TEST_CASE_BODY(parse_tap_output__only_one_result) 183{ 184 const engine::tap_summary summary = do_parse( 185 "1..1\n" 186 "ok - 1\n"); 187 188 const engine::tap_summary exp_summary = 189 engine::tap_summary::new_results(engine::tap_plan(1, 1), 1, 0); 190 ATF_REQUIRE_EQ(exp_summary, summary); 191} 192 193 194ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__all_pass); 195ATF_TEST_CASE_BODY(parse_tap_output__all_pass) 196{ 197 const engine::tap_summary summary = do_parse( 198 "1..8\n" 199 "ok - 1\n" 200 " Some diagnostic message\n" 201 "ok - 2 This test also passed\n" 202 "garbage line\n" 203 "ok - 3 This test passed\n" 204 "not ok 4 # SKIP Some reason\n" 205 "not ok 5 # TODO Another reason\n" 206 "ok - 6 Doesn't make a difference SKIP\n" 207 "ok - 7 Doesn't make a difference either TODO\n" 208 "ok # Also works without a number\n"); 209 210 const engine::tap_summary exp_summary = 211 engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0); 212 ATF_REQUIRE_EQ(exp_summary, summary); 213} 214 215 216ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__some_fail); 217ATF_TEST_CASE_BODY(parse_tap_output__some_fail) 218{ 219 const engine::tap_summary summary = do_parse( 220 "garbage line\n" 221 "not ok - 1 This test failed\n" 222 "ok - 2 This test passed\n" 223 "not ok - 3 This test failed\n" 224 "1..6\n" 225 "not ok - 4 This test failed\n" 226 "ok - 5 This test passed\n" 227 "not ok # Fails as well without a number\n"); 228 229 const engine::tap_summary exp_summary = 230 engine::tap_summary::new_results(engine::tap_plan(1, 6), 2, 4); 231 ATF_REQUIRE_EQ(exp_summary, summary); 232} 233 234 235ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_and_todo_variants); 236ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants) 237{ 238 const engine::tap_summary summary = do_parse( 239 "1..8\n" 240 "not ok - 1 # SKIP Some reason\n" 241 "not ok - 2 # skip Some reason\n" 242 "not ok - 3 # Skipped Some reason\n" 243 "not ok - 4 # skipped Some reason\n" 244 "not ok - 5 # Skipped: Some reason\n" 245 "not ok - 6 # skipped: Some reason\n" 246 "not ok - 7 # TODO Some reason\n" 247 "not ok - 8 # todo Some reason\n"); 248 249 const engine::tap_summary exp_summary = 250 engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0); 251 ATF_REQUIRE_EQ(exp_summary, summary); 252} 253 254 255ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_with_reason); 256ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason) 257{ 258 const engine::tap_summary summary = do_parse( 259 "1..0 SKIP Some reason for skipping\n" 260 "ok - 1\n" 261 " Some diagnostic message\n" 262 "ok - 6 Doesn't make a difference SKIP\n" 263 "ok - 7 Doesn't make a difference either TODO\n"); 264 265 const engine::tap_summary exp_summary = 266 engine::tap_summary::new_all_skipped("Some reason for skipping"); 267 ATF_REQUIRE_EQ(exp_summary, summary); 268} 269 270 271ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_without_reason); 272ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason) 273{ 274 const engine::tap_summary summary = do_parse( 275 "1..0 unrecognized # garbage skip\n"); 276 277 const engine::tap_summary exp_summary = 278 engine::tap_summary::new_all_skipped("No reason specified"); 279 ATF_REQUIRE_EQ(exp_summary, summary); 280} 281 282 283ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_invalid); 284ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid) 285{ 286 ATF_REQUIRE_THROW_RE(engine::load_error, 287 "Skipped plan must be 1\\.\\.0", 288 do_parse("1..3 # skip\n")); 289} 290 291 292ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__plan_at_end); 293ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end) 294{ 295 const engine::tap_summary summary = do_parse( 296 "ok - 1\n" 297 " Some diagnostic message\n" 298 "ok - 2 This test also passed\n" 299 "garbage line\n" 300 "ok - 3 This test passed\n" 301 "not ok 4 # SKIP Some reason\n" 302 "not ok 5 # TODO Another reason\n" 303 "ok - 6 Doesn't make a difference SKIP\n" 304 "ok - 7 Doesn't make a difference either TODO\n" 305 "1..7\n"); 306 307 const engine::tap_summary exp_summary = 308 engine::tap_summary::new_results(engine::tap_plan(1, 7), 7, 0); 309 ATF_REQUIRE_EQ(exp_summary, summary); 310} 311 312 313ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__stray_oks); 314ATF_TEST_CASE_BODY(parse_tap_output__stray_oks) 315{ 316 const engine::tap_summary summary = do_parse( 317 "1..3\n" 318 "ok - 1\n" 319 "ok\n" 320 "ok - 2 This test also passed\n" 321 "not ok\n" 322 "ok - 3 This test passed\n"); 323 324 const engine::tap_summary exp_summary = 325 engine::tap_summary::new_results(engine::tap_plan(1, 3), 3, 0); 326 ATF_REQUIRE_EQ(exp_summary, summary); 327} 328 329 330ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__no_plan); 331ATF_TEST_CASE_BODY(parse_tap_output__no_plan) 332{ 333 ATF_REQUIRE_THROW_RE( 334 engine::load_error, 335 "Output did not contain any TAP plan", 336 do_parse( 337 "not ok - 1 This test failed\n" 338 "ok - 2 This test passed\n")); 339} 340 341 342ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__double_plan); 343ATF_TEST_CASE_BODY(parse_tap_output__double_plan) 344{ 345 ATF_REQUIRE_THROW_RE( 346 engine::load_error, 347 "Found duplicate plan", 348 do_parse( 349 "garbage line\n" 350 "1..5\n" 351 "not ok - 1 This test failed\n" 352 "ok - 2 This test passed\n" 353 "1..8\n" 354 "ok\n")); 355} 356 357 358ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_plan); 359ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan) 360{ 361 ATF_REQUIRE_THROW_RE( 362 engine::load_error, 363 "Reported plan differs from actual executed tests", 364 do_parse( 365 "1..3\n" 366 "not ok - 1 This test failed\n" 367 "ok - 2 This test passed\n")); 368} 369 370 371ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_trailing_plan); 372ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan) 373{ 374 ATF_REQUIRE_THROW_RE( 375 engine::load_error, 376 "Reported plan differs from actual executed tests", 377 do_parse( 378 "not ok - 1 This test failed\n" 379 "ok - 2 This test passed\n" 380 "1..3\n")); 381} 382 383 384ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__insane_plan); 385ATF_TEST_CASE_BODY(parse_tap_output__insane_plan) 386{ 387 ATF_REQUIRE_THROW_RE( 388 engine::load_error, "Invalid value", 389 do_parse("120830981209831..234891793874080981092803981092312\n")); 390} 391 392 393ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__reversed_plan); 394ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan) 395{ 396 ATF_REQUIRE_THROW_RE(engine::load_error, 397 "Found reversed plan 8\\.\\.5", 398 do_parse("8..5\n")); 399} 400 401 402ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out); 403ATF_TEST_CASE_BODY(parse_tap_output__bail_out) 404{ 405 const engine::tap_summary summary = do_parse( 406 "1..3\n" 407 "not ok - 1 This test failed\n" 408 "Bail out! There is some unknown problem\n" 409 "ok - 2 This test passed\n"); 410 411 const engine::tap_summary exp_summary = 412 engine::tap_summary::new_bailed_out(); 413 ATF_REQUIRE_EQ(exp_summary, summary); 414} 415 416 417ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out_wins_over_no_plan); 418ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan) 419{ 420 const engine::tap_summary summary = do_parse( 421 "not ok - 1 This test failed\n" 422 "Bail out! There is some unknown problem\n" 423 "ok - 2 This test passed\n"); 424 425 const engine::tap_summary exp_summary = 426 engine::tap_summary::new_bailed_out(); 427 ATF_REQUIRE_EQ(exp_summary, summary); 428} 429 430 431ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__open_failure); 432ATF_TEST_CASE_BODY(parse_tap_output__open_failure) 433{ 434 ATF_REQUIRE_THROW_RE(engine::load_error, "hello.txt.*Failed to open", 435 engine::parse_tap_output(fs::path("hello.txt"))); 436} 437 438 439ATF_INIT_TEST_CASES(tcs) 440{ 441 ATF_ADD_TEST_CASE(tcs, tap_summary__bailed_out); 442 ATF_ADD_TEST_CASE(tcs, tap_summary__some_results); 443 ATF_ADD_TEST_CASE(tcs, tap_summary__all_skipped); 444 ATF_ADD_TEST_CASE(tcs, tap_summary__equality_operators); 445 ATF_ADD_TEST_CASE(tcs, tap_summary__output); 446 447 ATF_ADD_TEST_CASE(tcs, parse_tap_output__only_one_result); 448 ATF_ADD_TEST_CASE(tcs, parse_tap_output__all_pass); 449 ATF_ADD_TEST_CASE(tcs, parse_tap_output__some_fail); 450 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_and_todo_variants); 451 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_without_reason); 452 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_with_reason); 453 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_invalid); 454 ATF_ADD_TEST_CASE(tcs, parse_tap_output__plan_at_end); 455 ATF_ADD_TEST_CASE(tcs, parse_tap_output__stray_oks); 456 ATF_ADD_TEST_CASE(tcs, parse_tap_output__no_plan); 457 ATF_ADD_TEST_CASE(tcs, parse_tap_output__double_plan); 458 ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_plan); 459 ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_trailing_plan); 460 ATF_ADD_TEST_CASE(tcs, parse_tap_output__insane_plan); 461 ATF_ADD_TEST_CASE(tcs, parse_tap_output__reversed_plan); 462 ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out); 463 ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out_wins_over_no_plan); 464 ATF_ADD_TEST_CASE(tcs, parse_tap_output__open_failure); 465} 466