1# Copyright 2011 Google Inc. 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above copyright 11# notice, this list of conditions and the following disclaimer in the 12# documentation and/or other materials provided with the distribution. 13# * Neither the name of Google Inc. nor the names of its contributors 14# may be used to endorse or promote products derived from this software 15# without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 30utils_test_case one_test_program__all_pass 31one_test_program__all_pass_body() { 32 utils_install_timestamp_wrapper 33 34 cat >Kyuafile <<EOF 35syntax(2) 36test_suite("integration") 37atf_test_program{name="simple_all_pass"} 38EOF 39 40 cat >expout <<EOF 41simple_all_pass:pass -> passed [S.UUUs] 42simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] 43 442/2 passed (0 failed) 45Committed action 1 46EOF 47 48 utils_cp_helper simple_all_pass . 49 atf_check -s exit:0 -o file:expout -e empty kyua test 50} 51 52 53utils_test_case one_test_program__some_fail 54one_test_program__some_fail_body() { 55 utils_install_timestamp_wrapper 56 57 cat >Kyuafile <<EOF 58syntax(2) 59test_suite("integration") 60atf_test_program{name="simple_some_fail"} 61EOF 62 63 cat >expout <<EOF 64simple_some_fail:fail -> failed: This fails on purpose [S.UUUs] 65simple_some_fail:pass -> passed [S.UUUs] 66 671/2 passed (1 failed) 68Committed action 1 69EOF 70 71 utils_cp_helper simple_some_fail . 72 atf_check -s exit:1 -o file:expout -e empty kyua test 73} 74 75 76utils_test_case many_test_programs__all_pass 77many_test_programs__all_pass_body() { 78 utils_install_timestamp_wrapper 79 80 cat >Kyuafile <<EOF 81syntax(2) 82test_suite("integration") 83atf_test_program{name="first"} 84atf_test_program{name="second"} 85atf_test_program{name="third"} 86plain_test_program{name="fourth", required_files="/non-existent/foo"} 87EOF 88 89 cat >expout <<EOF 90first:pass -> passed [S.UUUs] 91first:skip -> skipped: The reason for skipping is this [S.UUUs] 92second:pass -> passed [S.UUUs] 93second:skip -> skipped: The reason for skipping is this [S.UUUs] 94third:pass -> passed [S.UUUs] 95third:skip -> skipped: The reason for skipping is this [S.UUUs] 96fourth:main -> skipped: Required file '/non-existent/foo' not found [S.UUUs] 97 987/7 passed (0 failed) 99Committed action 1 100EOF 101 102 utils_cp_helper simple_all_pass first 103 utils_cp_helper simple_all_pass second 104 utils_cp_helper simple_all_pass third 105 echo "not executed" >fourth; chmod +x fourth 106 atf_check -s exit:0 -o file:expout -e empty kyua test 107} 108 109 110utils_test_case many_test_programs__some_fail 111many_test_programs__some_fail_body() { 112 utils_install_timestamp_wrapper 113 114 cat >Kyuafile <<EOF 115syntax(2) 116test_suite("integration") 117atf_test_program{name="first"} 118atf_test_program{name="second"} 119atf_test_program{name="third"} 120plain_test_program{name="fourth"} 121EOF 122 123 cat >expout <<EOF 124first:fail -> failed: This fails on purpose [S.UUUs] 125first:pass -> passed [S.UUUs] 126second:fail -> failed: This fails on purpose [S.UUUs] 127second:pass -> passed [S.UUUs] 128third:pass -> passed [S.UUUs] 129third:skip -> skipped: The reason for skipping is this [S.UUUs] 130fourth:main -> failed: Returned non-success exit status 76 [S.UUUs] 131 1324/7 passed (3 failed) 133Committed action 1 134EOF 135 136 utils_cp_helper simple_some_fail first 137 utils_cp_helper simple_some_fail second 138 utils_cp_helper simple_all_pass third 139 echo '#! /bin/sh' >fourth 140 echo 'exit 76' >>fourth 141 chmod +x fourth 142 atf_check -s exit:1 -o file:expout -e empty kyua test 143} 144 145 146utils_test_case expect__all_pass 147expect__all_pass_body() { 148 utils_install_timestamp_wrapper 149 150 cat >Kyuafile <<EOF 151syntax(2) 152test_suite("integration") 153atf_test_program{name="expect_all_pass"} 154EOF 155 156 cat >expout <<EOF 157expect_all_pass:die -> expected_failure: This is the reason for death [S.UUUs] 158expect_all_pass:exit -> expected_failure: Exiting with correct code [S.UUUs] 159expect_all_pass:failure -> expected_failure: Oh no: Forced failure [S.UUUs] 160expect_all_pass:signal -> expected_failure: Exiting with correct signal [S.UUUs] 161expect_all_pass:timeout -> expected_failure: This times out [S.UUUs] 162 1635/5 passed (0 failed) 164Committed action 1 165EOF 166 167 utils_cp_helper expect_all_pass . 168 atf_check -s exit:0 -o file:expout -e empty kyua test 169} 170 171 172utils_test_case expect__some_fail 173expect__some_fail_body() { 174 utils_install_timestamp_wrapper 175 176 cat >Kyuafile <<EOF 177syntax(2) 178test_suite("integration") 179atf_test_program{name="expect_some_fail"} 180EOF 181 182 cat >expout <<EOF 183expect_some_fail:die -> failed: Test case was expected to terminate abruptly but it continued execution [S.UUUs] 184expect_some_fail:exit -> failed: Test case expected to exit with code 12 but got code 34 [S.UUUs] 185expect_some_fail:failure -> failed: Test case was expecting a failure but none were raised [S.UUUs] 186expect_some_fail:pass -> passed [S.UUUs] 187expect_some_fail:signal -> failed: Test case expected to receive signal 15 but got 9 [S.UUUs] 188expect_some_fail:timeout -> failed: Test case was expected to hang but it continued execution [S.UUUs] 189 1901/6 passed (5 failed) 191Committed action 1 192EOF 193 194 utils_cp_helper expect_some_fail . 195 atf_check -s exit:1 -o file:expout -e empty kyua test 196} 197 198 199utils_test_case premature_exit 200premature_exit_body() { 201 utils_install_timestamp_wrapper 202 203 cat >Kyuafile <<EOF 204syntax(2) 205test_suite("integration") 206atf_test_program{name="bogus_test_cases"} 207EOF 208 209 cat >expout <<EOF 210bogus_test_cases:die -> broken: Premature exit; test case received signal 9 [S.UUUs] 211bogus_test_cases:exit -> broken: Premature exit; test case exited with code 0 [S.UUUs] 212bogus_test_cases:pass -> passed [S.UUUs] 213 2141/3 passed (2 failed) 215Committed action 1 216EOF 217 218 utils_cp_helper bogus_test_cases . 219 atf_check -s exit:1 -o file:expout -e empty kyua test 220} 221 222 223utils_test_case no_args 224no_args_body() { 225 utils_install_timestamp_wrapper 226 227 cat >Kyuafile <<EOF 228syntax(2) 229test_suite("integration") 230atf_test_program{name="simple_all_pass"} 231include("subdir/Kyuafile") 232EOF 233 utils_cp_helper metadata . 234 utils_cp_helper simple_all_pass . 235 236 mkdir subdir 237 cat >subdir/Kyuafile <<EOF 238syntax(2) 239test_suite("integration2") 240atf_test_program{name="simple_some_fail"} 241EOF 242 utils_cp_helper simple_some_fail subdir 243 244 cat >expout <<EOF 245simple_all_pass:pass -> passed [S.UUUs] 246simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] 247subdir/simple_some_fail:fail -> failed: This fails on purpose [S.UUUs] 248subdir/simple_some_fail:pass -> passed [S.UUUs] 249 2503/4 passed (1 failed) 251Committed action 1 252EOF 253 atf_check -s exit:1 -o file:expout -e empty kyua test 254} 255 256 257utils_test_case one_arg__subdir 258one_arg__subdir_body() { 259 utils_install_timestamp_wrapper 260 261 cat >Kyuafile <<EOF 262syntax(2) 263test_suite("top-level") 264include("subdir/Kyuafile") 265EOF 266 267 mkdir subdir 268 cat >subdir/Kyuafile <<EOF 269syntax(2) 270test_suite("in-subdir") 271atf_test_program{name="simple_all_pass"} 272EOF 273 utils_cp_helper simple_all_pass subdir 274 275 cat >expout <<EOF 276subdir/simple_all_pass:pass -> passed [S.UUUs] 277subdir/simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] 278 2792/2 passed (0 failed) 280Committed action 1 281EOF 282 atf_check -s exit:0 -o file:expout -e empty kyua test subdir 283} 284 285 286utils_test_case one_arg__test_case 287one_arg__test_case_body() { 288 utils_install_timestamp_wrapper 289 290 cat >Kyuafile <<EOF 291syntax(2) 292test_suite("top-level") 293atf_test_program{name="first"} 294atf_test_program{name="second"} 295EOF 296 utils_cp_helper simple_all_pass first 297 utils_cp_helper simple_all_pass second 298 299 cat >expout <<EOF 300first:skip -> skipped: The reason for skipping is this [S.UUUs] 301 3021/1 passed (0 failed) 303Committed action 1 304EOF 305 atf_check -s exit:0 -o file:expout -e empty kyua test first:skip 306} 307 308 309utils_test_case one_arg__test_program 310one_arg__test_program_body() { 311 utils_install_timestamp_wrapper 312 313 cat >Kyuafile <<EOF 314syntax(2) 315test_suite("top-level") 316atf_test_program{name="first"} 317atf_test_program{name="second"} 318EOF 319 utils_cp_helper simple_all_pass first 320 utils_cp_helper simple_some_fail second 321 322 cat >expout <<EOF 323second:fail -> failed: This fails on purpose [S.UUUs] 324second:pass -> passed [S.UUUs] 325 3261/2 passed (1 failed) 327Committed action 1 328EOF 329 atf_check -s exit:1 -o file:expout -e empty kyua test second 330} 331 332 333utils_test_case one_arg__invalid 334one_arg__invalid_body() { 335cat >experr <<EOF 336kyua: E: Test case component in 'foo:' is empty. 337EOF 338 atf_check -s exit:2 -o empty -e file:experr kyua test foo: 339 340cat >experr <<EOF 341kyua: E: Program name '/a/b' must be relative to the test suite, not absolute. 342EOF 343 atf_check -s exit:2 -o empty -e file:experr kyua test /a/b 344} 345 346 347utils_test_case many_args__ok 348many_args__ok_body() { 349 utils_install_timestamp_wrapper 350 351 cat >Kyuafile <<EOF 352syntax(2) 353test_suite("top-level") 354include("subdir/Kyuafile") 355atf_test_program{name="first"} 356EOF 357 utils_cp_helper simple_all_pass first 358 359 mkdir subdir 360 cat >subdir/Kyuafile <<EOF 361syntax(2) 362test_suite("in-subdir") 363atf_test_program{name="second"} 364EOF 365 utils_cp_helper simple_some_fail subdir/second 366 367 cat >expout <<EOF 368subdir/second:fail -> failed: This fails on purpose [S.UUUs] 369subdir/second:pass -> passed [S.UUUs] 370first:pass -> passed [S.UUUs] 371 3722/3 passed (1 failed) 373Committed action 1 374EOF 375 atf_check -s exit:1 -o file:expout -e empty kyua test subdir first:pass 376} 377 378 379utils_test_case many_args__invalid 380many_args__invalid_body() { 381cat >experr <<EOF 382kyua: E: Program name component in ':badbad' is empty. 383EOF 384 atf_check -s exit:2 -o empty -e file:experr kyua test this-is-ok :badbad 385 386cat >experr <<EOF 387kyua: E: Program name '/foo' must be relative to the test suite, not absolute. 388EOF 389 atf_check -s exit:2 -o empty -e file:experr kyua test this-is-ok /foo 390} 391 392 393utils_test_case many_args__no_match__all 394many_args__no_match__all_body() { 395 cat >Kyuafile <<EOF 396syntax(2) 397test_suite("top-level") 398atf_test_program{name="first"} 399atf_test_program{name="second"} 400EOF 401 utils_cp_helper simple_all_pass first 402 utils_cp_helper simple_all_pass second 403 404 cat >expout <<EOF 405Committed action 1 406EOF 407 cat >experr <<EOF 408kyua: W: No test cases matched by the filter 'first1'. 409EOF 410 atf_check -s exit:1 -o file:expout -e file:experr kyua test first1 411} 412 413 414utils_test_case many_args__no_match__some 415many_args__no_match__some_body() { 416 utils_install_timestamp_wrapper 417 418 cat >Kyuafile <<EOF 419syntax(2) 420test_suite("top-level") 421atf_test_program{name="first"} 422atf_test_program{name="second"} 423atf_test_program{name="third"} 424EOF 425 utils_cp_helper simple_all_pass first 426 utils_cp_helper simple_all_pass second 427 utils_cp_helper simple_some_fail third 428 429 cat >expout <<EOF 430first:pass -> passed [S.UUUs] 431first:skip -> skipped: The reason for skipping is this [S.UUUs] 432third:fail -> failed: This fails on purpose [S.UUUs] 433third:pass -> passed [S.UUUs] 434 4353/4 passed (1 failed) 436Committed action 1 437EOF 438 439 cat >experr <<EOF 440kyua: W: No test cases matched by the filter 'fifth'. 441kyua: W: No test cases matched by the filter 'fourth'. 442EOF 443 atf_check -s exit:1 -o file:expout -e file:experr kyua test first fourth \ 444 third fifth 445} 446 447 448utils_test_case args_are_relative 449args_are_relative_body() { 450 utils_install_timestamp_wrapper 451 452 mkdir root 453 cat >root/Kyuafile <<EOF 454syntax(2) 455test_suite("integration-1") 456atf_test_program{name="first"} 457atf_test_program{name="second"} 458include("subdir/Kyuafile") 459EOF 460 utils_cp_helper simple_all_pass root/first 461 utils_cp_helper simple_some_fail root/second 462 463 mkdir root/subdir 464 cat >root/subdir/Kyuafile <<EOF 465syntax(2) 466test_suite("integration-2") 467atf_test_program{name="third"} 468atf_test_program{name="fourth"} 469EOF 470 utils_cp_helper simple_all_pass root/subdir/third 471 utils_cp_helper simple_some_fail root/subdir/fourth 472 473 cat >expout <<EOF 474first:pass -> passed [S.UUUs] 475first:skip -> skipped: The reason for skipping is this [S.UUUs] 476subdir/fourth:fail -> failed: This fails on purpose [S.UUUs] 477 4782/3 passed (1 failed) 479Committed action 1 480EOF 481 atf_check -s exit:1 -o file:expout -e empty kyua test \ 482 -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail 483} 484 485 486utils_test_case only_load_used_test_programs 487only_load_used_test_programs_body() { 488 utils_install_timestamp_wrapper 489 490 cat >Kyuafile <<EOF 491syntax(2) 492test_suite("integration") 493atf_test_program{name="first"} 494atf_test_program{name="second"} 495EOF 496 utils_cp_helper simple_all_pass first 497 utils_cp_helper bad_test_program second 498 499 cat >expout <<EOF 500first:pass -> passed [S.UUUs] 501first:skip -> skipped: The reason for skipping is this [S.UUUs] 502 5032/2 passed (0 failed) 504Committed action 1 505EOF 506 CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE 507 atf_check -s exit:0 -o file:expout -e empty kyua test first 508 if test -f "${CREATE_COOKIE}"; then 509 atf_fail "An unmatched test case has been executed, which harms" \ 510 "performance" 511 fi 512} 513 514 515utils_test_case config_behavior 516config_behavior_body() { 517 cat >"my-config" <<EOF 518syntax(2) 519test_suites.suite1["X-the-variable"] = "value1" 520test_suites.suite2["X-the-variable"] = "override me" 521EOF 522 523 cat >Kyuafile <<EOF 524syntax(2) 525atf_test_program{name="config1", test_suite="suite1"} 526atf_test_program{name="config2", test_suite="suite2"} 527atf_test_program{name="config3", test_suite="suite3"} 528EOF 529 utils_cp_helper config config1 530 utils_cp_helper config config2 531 utils_cp_helper config config3 532 533 atf_check -s exit:1 -o save:stdout -e empty \ 534 kyua -c my-config -v test_suites.suite2.X-the-variable=value2 test 535 atf_check -s exit:0 -o ignore -e empty \ 536 grep 'config1:get_variable.*failed' stdout 537 atf_check -s exit:0 -o ignore -e empty \ 538 grep 'config2:get_variable.*passed' stdout 539 atf_check -s exit:0 -o ignore -e empty \ 540 grep 'config3:get_variable.*skipped' stdout 541} 542 543 544utils_test_case store_contents 545store_contents_body() { 546 utils_install_timestamp_wrapper 547 548 cat >Kyuafile <<EOF 549syntax(2) 550atf_test_program{name="some-program", test_suite="suite1"} 551EOF 552 utils_cp_helper simple_all_pass some-program 553 cat >expout <<EOF 554some-program:pass -> passed [S.UUUs] 555some-program:skip -> skipped: The reason for skipping is this [S.UUUs] 556 5572/2 passed (0 failed) 558Committed action 1 559EOF 560 561 # TODO(jmmv): The tests below should not care about the specific contents of 562 # the database, but we cannot do better yet. Instead, we should bundle 563 # these tests with the tests of the future "report" tests so that we can 564 # actually validate the integration of testing plus reporting. 565 566 atf_check -s exit:0 -o file:expout -e empty kyua test 567 atf_check -s exit:0 -o inline:'1\n' -e empty \ 568 kyua db-exec --no-headers "SELECT COUNT(action_id) FROM ACTIONS" 569 atf_check -s exit:0 -o inline:'1\n' -e empty \ 570 kyua db-exec --no-headers "SELECT COUNT(context_id) FROM CONTEXTS" 571 572 rm -f some-program 573 utils_cp_helper simple_some_fail some-program 574 cat >expout <<EOF 575some-program:fail -> failed: This fails on purpose [S.UUUs] 576some-program:pass -> passed [S.UUUs] 577 5781/2 passed (1 failed) 579Committed action 2 580EOF 581 582 atf_check -s exit:1 -o file:expout -e empty kyua test 583 atf_check -s exit:0 -o inline:'2\n' -e empty \ 584 kyua db-exec --no-headers "SELECT COUNT(action_id) FROM ACTIONS" 585 atf_check -s exit:0 -o inline:'2\n' -e empty \ 586 kyua db-exec --no-headers "SELECT COUNT(context_id) FROM CONTEXTS" 587 588cat >expout <<EOF 5891,some-program,pass,passed,NULL 5901,some-program,skip,skipped,The reason for skipping is this 5912,some-program,fail,failed,This fails on purpose 5922,some-program,pass,passed,NULL 593EOF 594 atf_check -s exit:0 -o file:expout -e empty \ 595 kyua db-exec --no-headers \ 596 "SELECT actions.action_id, " \ 597 " test_programs.relative_path, test_cases.name, " \ 598 " test_results.result_type, test_results.result_reason " \ 599 "FROM actions " \ 600 " JOIN test_programs " \ 601 " ON actions.action_id = test_programs.action_id " \ 602 " JOIN test_cases " \ 603 " ON test_programs.test_program_id = test_cases.test_program_id " \ 604 " JOIN test_results " \ 605 " ON test_cases.test_case_id = test_results.test_case_id " \ 606 "ORDER BY actions.action_id, test_programs.relative_path, " \ 607 " test_cases.name" 608} 609 610 611utils_test_case store_flag__ok 612store_flag__ok_body() { 613 cat >Kyuafile <<EOF 614syntax(2) 615atf_test_program{name="config1", test_suite="suite1"} 616EOF 617 utils_cp_helper config config1 618 619 atf_check -s exit:0 -o ignore -e empty kyua test -s foo1.db 620 test -f foo1.db || atf_fail "-s did not work" 621 atf_check -s exit:0 -o ignore -e empty kyua test --store=foo2.db 622 test -f foo2.db || atf_fail "--store did not work" 623 test ! -f .kyua/store.db || atf_fail "Default database created" 624} 625 626 627utils_test_case store_flag__fail 628store_flag__fail_body() { 629 cat >Kyuafile <<EOF 630syntax(2) 631atf_test_program{name="config1", test_suite="suite1"} 632EOF 633 utils_cp_helper config config1 634 635 atf_check -s exit:3 -o empty -e match:"Invalid.*--store" \ 636 kyua test --store="" 637} 638 639 640utils_test_case build_root_flag 641build_root_flag_body() { 642 utils_install_timestamp_wrapper 643 644 cat >Kyuafile <<EOF 645syntax(2) 646test_suite("integration") 647atf_test_program{name="first"} 648include("subdir/Kyuafile") 649EOF 650 651 mkdir subdir 652 cat >subdir/Kyuafile <<EOF 653syntax(2) 654test_suite("integration") 655atf_test_program{name="second"} 656atf_test_program{name="third"} 657EOF 658 659 cat >expout <<EOF 660first:pass -> passed [S.UUUs] 661first:skip -> skipped: The reason for skipping is this [S.UUUs] 662subdir/second:pass -> passed [S.UUUs] 663subdir/second:skip -> skipped: The reason for skipping is this [S.UUUs] 664subdir/third:pass -> passed [S.UUUs] 665subdir/third:skip -> skipped: The reason for skipping is this [S.UUUs] 666 6676/6 passed (0 failed) 668Committed action 1 669EOF 670 671 mkdir build 672 mkdir build/subdir 673 utils_cp_helper simple_all_pass build/first 674 utils_cp_helper simple_all_pass build/subdir/second 675 utils_cp_helper simple_all_pass build/subdir/third 676 677 atf_check -s exit:0 -o file:expout -e empty kyua test --build-root=build 678} 679 680 681utils_test_case kyuafile_flag__no_args 682kyuafile_flag__no_args_body() { 683 utils_install_timestamp_wrapper 684 685 cat >Kyuafile <<EOF 686This file is bogus but it is not loaded. 687EOF 688 689 cat >myfile <<EOF 690syntax(2) 691test_suite("integration") 692atf_test_program{name="sometest"} 693EOF 694 utils_cp_helper simple_all_pass sometest 695 696 cat >expout <<EOF 697sometest:pass -> passed [S.UUUs] 698sometest:skip -> skipped: The reason for skipping is this [S.UUUs] 699 7002/2 passed (0 failed) 701Committed action 1 702EOF 703 atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile 704 cat >expout <<EOF 705sometest:pass -> passed [S.UUUs] 706sometest:skip -> skipped: The reason for skipping is this [S.UUUs] 707 7082/2 passed (0 failed) 709Committed action 2 710EOF 711 atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile 712} 713 714 715utils_test_case kyuafile_flag__some_args 716kyuafile_flag__some_args_body() { 717 utils_install_timestamp_wrapper 718 719 cat >Kyuafile <<EOF 720This file is bogus but it is not loaded. 721EOF 722 723 cat >myfile <<EOF 724syntax(2) 725test_suite("hello-world") 726atf_test_program{name="sometest"} 727EOF 728 utils_cp_helper simple_all_pass sometest 729 730 cat >expout <<EOF 731sometest:pass -> passed [S.UUUs] 732sometest:skip -> skipped: The reason for skipping is this [S.UUUs] 733 7342/2 passed (0 failed) 735Committed action 1 736EOF 737 atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile sometest 738 cat >expout <<EOF 739sometest:pass -> passed [S.UUUs] 740sometest:skip -> skipped: The reason for skipping is this [S.UUUs] 741 7422/2 passed (0 failed) 743Committed action 2 744EOF 745 atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile \ 746 sometest 747} 748 749 750utils_test_case interrupt 751interrupt_body() { 752 cat >Kyuafile <<EOF 753syntax(2) 754test_suite("integration") 755atf_test_program{name="interrupts"} 756EOF 757 utils_cp_helper interrupts . 758 759 kyua \ 760 -v test_suites.integration.X-body-cookie="$(pwd)/body" \ 761 -v test_suites.integration.X-cleanup-cookie="$(pwd)/cleanup" \ 762 test >stdout 2>stderr & 763 pid=${!} 764 echo "Kyua subprocess is PID ${pid}" 765 766 while [ ! -f body ]; do 767 echo "Waiting for body to start" 768 sleep 1 769 done 770 echo "Body started" 771 sleep 1 772 773 echo "Sending INT signal to ${pid}" 774 kill -INT ${pid} 775 echo "Waiting for process ${pid} to exit" 776 wait ${pid} 777 ret=${?} 778 sed -e 's,^,kyua stdout:,' stdout 779 sed -e 's,^,kyua stderr:,' stderr 780 echo "Process ${pid} exited" 781 [ ${ret} -ne 0 ] || atf_fail 'No error code reported' 782 783 [ -f cleanup ] || atf_fail 'Cleanup part not executed after signal' 784 785 atf_check -s exit:0 -o ignore -e empty grep 'Signal caught' stderr 786 atf_check -s exit:0 -o ignore -e empty \ 787 grep 'kyua: E: Interrupted by signal' stderr 788} 789 790 791utils_test_case no_test_program_match 792no_test_program_match_body() { 793 cat >Kyuafile <<EOF 794syntax(2) 795test_suite("integration") 796atf_test_program{name="first"} 797EOF 798 utils_cp_helper simple_all_pass first 799 utils_cp_helper simple_all_pass second 800 801 cat >expout <<EOF 802Committed action 1 803EOF 804 cat >experr <<EOF 805kyua: W: No test cases matched by the filter 'second'. 806EOF 807 atf_check -s exit:1 -o file:expout -e file:experr kyua test second 808} 809 810 811utils_test_case no_test_case_match 812no_test_case_match_body() { 813 cat >Kyuafile <<EOF 814syntax(2) 815test_suite("integration") 816atf_test_program{name="first"} 817EOF 818 utils_cp_helper simple_all_pass first 819 820 cat >expout <<EOF 821Committed action 1 822EOF 823 cat >experr <<EOF 824kyua: W: No test cases matched by the filter 'first:foobar'. 825EOF 826 atf_check -s exit:1 -o file:expout -e file:experr kyua test first:foobar 827} 828 829 830utils_test_case missing_kyuafile__no_args 831missing_kyuafile__no_args_body() { 832 cat >experr <<EOF 833kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 834EOF 835 atf_check -s exit:2 -o empty -e file:experr kyua test 836} 837 838 839utils_test_case missing_kyuafile__test_program 840missing_kyuafile__test_program_body() { 841 mkdir subdir 842 cat >subdir/Kyuafile <<EOF 843syntax(2) 844test_suite("integration") 845atf_test_program{name="unused"} 846EOF 847 utils_cp_helper simple_all_pass subdir/unused 848 849 cat >experr <<EOF 850kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 851EOF 852 atf_check -s exit:2 -o empty -e file:experr kyua test subdir/unused 853} 854 855 856utils_test_case missing_kyuafile__subdir 857missing_kyuafile__subdir_body() { 858 mkdir subdir 859 cat >subdir/Kyuafile <<EOF 860syntax(2) 861test_suite("integration") 862atf_test_program{name="unused"} 863EOF 864 utils_cp_helper simple_all_pass subdir/unused 865 866 cat >experr <<EOF 867kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 868EOF 869 atf_check -s exit:2 -o empty -e file:experr kyua test subdir 870} 871 872 873utils_test_case bogus_config 874bogus_config_body() { 875 mkdir .kyua 876 cat >"${HOME}/.kyua/kyua.conf" <<EOF 877Hello, world. 878EOF 879 880 file_re='.*\.kyua/kyua.conf' 881 atf_check -s exit:2 -o empty \ 882 -e match:"^kyua: E: Load of '${file_re}' failed: Failed to load Lua" \ 883 kyua test 884} 885 886 887utils_test_case bogus_kyuafile 888bogus_kyuafile_body() { 889 cat >Kyuafile <<EOF 890Hello, world. 891EOF 892 atf_check -s exit:2 -o empty \ 893 -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list 894} 895 896 897utils_test_case bogus_test_program 898bogus_test_program_body() { 899 utils_install_timestamp_wrapper 900 901 cat >Kyuafile <<EOF 902syntax(2) 903test_suite("integration") 904atf_test_program{name="crash_on_list"} 905atf_test_program{name="non_executable"} 906EOF 907 utils_cp_helper bad_test_program crash_on_list 908 echo 'I am not executable' >non_executable 909 910 cat >expout <<EOF 911crash_on_list:__test_cases_list__ -> broken: Tester did not exit cleanly: kyua-atf-tester: Invalid test cases list header 'This is not a valid test program!' [S.UUUs] 912non_executable:__test_cases_list__ -> broken: Tester did not exit cleanly: kyua-atf-tester: execvp failed: Permission denied [S.UUUs] 913 9140/2 passed (2 failed) 915Committed action 1 916EOF 917 atf_check -s exit:1 -o file:expout -e empty kyua test 918} 919 920 921utils_test_case missing_test_program 922missing_test_program_body() { 923 cat >Kyuafile <<EOF 924syntax(2) 925include("subdir/Kyuafile") 926EOF 927 mkdir subdir 928 cat >subdir/Kyuafile <<EOF 929syntax(2) 930test_suite("integration") 931atf_test_program{name="ok"} 932atf_test_program{name="i-am-missing"} 933EOF 934 echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok 935 936 cat >experr <<EOF 937kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'. 938EOF 939 atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list 940} 941 942 943atf_init_test_cases() { 944 atf_add_test_case one_test_program__all_pass 945 atf_add_test_case one_test_program__some_fail 946 atf_add_test_case many_test_programs__all_pass 947 atf_add_test_case many_test_programs__some_fail 948 atf_add_test_case expect__all_pass 949 atf_add_test_case expect__some_fail 950 atf_add_test_case premature_exit 951 952 atf_add_test_case no_args 953 atf_add_test_case one_arg__subdir 954 atf_add_test_case one_arg__test_case 955 atf_add_test_case one_arg__test_program 956 atf_add_test_case one_arg__invalid 957 atf_add_test_case many_args__ok 958 atf_add_test_case many_args__invalid 959 atf_add_test_case many_args__no_match__all 960 atf_add_test_case many_args__no_match__some 961 962 atf_add_test_case args_are_relative 963 964 atf_add_test_case only_load_used_test_programs 965 966 atf_add_test_case config_behavior 967 968 atf_add_test_case store_contents 969 atf_add_test_case store_flag__ok 970 atf_add_test_case store_flag__fail 971 972 atf_add_test_case build_root_flag 973 974 atf_add_test_case kyuafile_flag__no_args 975 atf_add_test_case kyuafile_flag__some_args 976 977 atf_add_test_case interrupt 978 979 atf_add_test_case no_test_program_match 980 atf_add_test_case no_test_case_match 981 982 atf_add_test_case missing_kyuafile__no_args 983 atf_add_test_case missing_kyuafile__test_program 984 atf_add_test_case missing_kyuafile__subdir 985 986 atf_add_test_case bogus_config 987 atf_add_test_case bogus_kyuafile 988 atf_add_test_case bogus_test_program 989 atf_add_test_case missing_test_program 990} 991