1# Copyright 2011 Google Inc. 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above copyright 11# notice, this list of conditions and the following disclaimer in the 12# documentation and/or other materials provided with the distribution. 13# * Neither the name of Google Inc. nor the names of its contributors 14# may be used to endorse or promote products derived from this software 15# without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 30utils_test_case no_args 31no_args_body() { 32 cat >Kyuafile <<EOF 33syntax(2) 34test_suite("integration") 35atf_test_program{name="metadata"} 36atf_test_program{name="simple_all_pass"} 37include("subdir/Kyuafile") 38EOF 39 utils_cp_helper metadata . 40 utils_cp_helper simple_all_pass . 41 42 mkdir subdir 43 cat >subdir/Kyuafile <<EOF 44syntax(2) 45test_suite("integration2") 46atf_test_program{name="simple_some_fail"} 47EOF 48 utils_cp_helper simple_some_fail subdir 49 50 cat >expout <<EOF 51metadata:no_properties 52metadata:one_property 53metadata:many_properties 54metadata:with_cleanup 55simple_all_pass:pass 56simple_all_pass:skip 57subdir/simple_some_fail:fail 58subdir/simple_some_fail:pass 59EOF 60 atf_check -s exit:0 -o file:expout -e empty kyua list 61} 62 63 64utils_test_case one_arg__subdir 65one_arg__subdir_body() { 66 cat >Kyuafile <<EOF 67syntax(2) 68test_suite("top-level") 69include("subdir/Kyuafile") 70EOF 71 72 mkdir subdir 73 cat >subdir/Kyuafile <<EOF 74syntax(2) 75test_suite("in-subdir") 76atf_test_program{name="simple_all_pass"} 77EOF 78 utils_cp_helper simple_all_pass subdir 79 80 cat >expout <<EOF 81subdir/simple_all_pass:pass 82subdir/simple_all_pass:skip 83EOF 84 atf_check -s exit:0 -o file:expout -e empty kyua list subdir 85} 86 87 88utils_test_case one_arg__test_case 89one_arg__test_case_body() { 90 cat >Kyuafile <<EOF 91syntax(2) 92test_suite("top-level") 93atf_test_program{name="first"} 94atf_test_program{name="second"} 95EOF 96 utils_cp_helper simple_all_pass first 97 utils_cp_helper simple_all_pass second 98 99 cat >expout <<EOF 100first:skip 101EOF 102 atf_check -s exit:0 -o file:expout -e empty kyua list first:skip 103} 104 105 106utils_test_case one_arg__test_program 107one_arg__test_program_body() { 108 cat >Kyuafile <<EOF 109syntax(2) 110test_suite("top-level") 111atf_test_program{name="first"} 112atf_test_program{name="second"} 113EOF 114 utils_cp_helper simple_all_pass first 115 utils_cp_helper simple_some_fail second 116 117 cat >expout <<EOF 118second:fail 119second:pass 120EOF 121 atf_check -s exit:0 -o file:expout -e empty kyua list second 122} 123 124 125utils_test_case one_arg__invalid 126one_arg__invalid_body() { 127cat >experr <<EOF 128kyua: E: Test case component in 'foo:' is empty. 129EOF 130 atf_check -s exit:2 -o empty -e file:experr kyua list foo: 131 132cat >experr <<EOF 133kyua: E: Program name '/a/b' must be relative to the test suite, not absolute. 134EOF 135 atf_check -s exit:2 -o empty -e file:experr kyua list /a/b 136} 137 138 139utils_test_case many_args__ok 140many_args__ok_body() { 141 cat >Kyuafile <<EOF 142syntax(2) 143test_suite("top-level") 144include("subdir/Kyuafile") 145atf_test_program{name="first"} 146EOF 147 utils_cp_helper simple_all_pass first 148 149 mkdir subdir 150 cat >subdir/Kyuafile <<EOF 151syntax(2) 152test_suite("in-subdir") 153atf_test_program{name="second"} 154EOF 155 utils_cp_helper simple_some_fail subdir/second 156 157 cat >expout <<EOF 158subdir/second:fail (in-subdir) 159subdir/second:pass (in-subdir) 160first:pass (top-level) 161EOF 162 atf_check -s exit:0 -o file:expout -e empty kyua list -v subdir first:pass 163} 164 165 166utils_test_case many_args__invalid 167many_args__invalid_body() { 168cat >experr <<EOF 169kyua: E: Program name component in ':badbad' is empty. 170EOF 171 atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok :badbad 172 173cat >experr <<EOF 174kyua: E: Program name '/foo' must be relative to the test suite, not absolute. 175EOF 176 atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok /foo 177} 178 179 180utils_test_case many_args__no_match__all 181many_args__no_match__all_body() { 182 cat >Kyuafile <<EOF 183syntax(2) 184test_suite("top-level") 185atf_test_program{name="first"} 186atf_test_program{name="second"} 187EOF 188 utils_cp_helper simple_all_pass first 189 utils_cp_helper simple_all_pass second 190 191 cat >experr <<EOF 192kyua: W: No test cases matched by the filter 'first1'. 193EOF 194 atf_check -s exit:1 -o empty -e file:experr kyua list first1 195} 196 197 198utils_test_case many_args__no_match__some 199many_args__no_match__some_body() { 200 cat >Kyuafile <<EOF 201syntax(2) 202test_suite("top-level") 203atf_test_program{name="first"} 204atf_test_program{name="second"} 205atf_test_program{name="third"} 206EOF 207 utils_cp_helper simple_all_pass first 208 utils_cp_helper simple_all_pass second 209 utils_cp_helper simple_some_fail third 210 211 cat >expout <<EOF 212first:pass 213first:skip 214third:fail 215third:pass 216EOF 217 218 cat >experr <<EOF 219kyua: W: No test cases matched by the filter 'fifth'. 220kyua: W: No test cases matched by the filter 'fourth'. 221EOF 222 atf_check -s exit:1 -o file:expout -e file:experr kyua list first fourth \ 223 third fifth 224} 225 226 227utils_test_case args_are_relative 228args_are_relative_body() { 229 mkdir root 230 cat >root/Kyuafile <<EOF 231syntax(2) 232test_suite("integration-1") 233atf_test_program{name="first"} 234atf_test_program{name="second"} 235include("subdir/Kyuafile") 236EOF 237 utils_cp_helper simple_all_pass root/first 238 utils_cp_helper simple_some_fail root/second 239 240 mkdir root/subdir 241 cat >root/subdir/Kyuafile <<EOF 242syntax(2) 243test_suite("integration-2") 244atf_test_program{name="third"} 245atf_test_program{name="fourth"} 246EOF 247 utils_cp_helper simple_all_pass root/subdir/third 248 utils_cp_helper simple_some_fail root/subdir/fourth 249 250 cat >expout <<EOF 251first:pass (integration-1) 252first:skip (integration-1) 253subdir/fourth:fail (integration-2) 254EOF 255 atf_check -s exit:0 -o file:expout -e empty kyua list \ 256 -v -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail 257} 258 259 260utils_test_case only_load_used_test_programs 261only_load_used_test_programs_body() { 262 cat >Kyuafile <<EOF 263syntax(2) 264test_suite("integration") 265atf_test_program{name="first"} 266atf_test_program{name="second"} 267EOF 268 utils_cp_helper simple_all_pass first 269 utils_cp_helper bad_test_program second 270 271 cat >expout <<EOF 272first:pass 273first:skip 274EOF 275 CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE 276 atf_check -s exit:0 -o file:expout -e empty kyua list first 277 if test -f "${CREATE_COOKIE}"; then 278 atf_fail "An unmatched test case has been executed, which harms" \ 279 "performance" 280 fi 281} 282 283 284utils_test_case build_root_flag 285build_root_flag_body() { 286 mkdir subdir 287 mkdir build 288 mkdir build/subdir 289 290 cat >Kyuafile <<EOF 291syntax(2) 292test_suite("top-level") 293include("subdir/Kyuafile") 294atf_test_program{name="first"} 295EOF 296 echo 'invalid' >first 297 utils_cp_helper simple_all_pass build/first 298 299 cat >subdir/Kyuafile <<EOF 300syntax(2) 301test_suite("in-subdir") 302atf_test_program{name="second"} 303EOF 304 echo 'invalid' >subdir/second 305 utils_cp_helper simple_some_fail build/subdir/second 306 307 cat >expout <<EOF 308subdir/second:fail 309subdir/second:pass 310first:pass 311EOF 312 atf_check -s exit:0 -o file:expout -e empty kyua list --build-root=build \ 313 subdir first:pass 314} 315 316 317utils_test_case kyuafile_flag__no_args 318kyuafile_flag__no_args_body() { 319 cat >Kyuafile <<EOF 320This file is bogus but it is not loaded. 321EOF 322 323 cat >myfile <<EOF 324syntax(2) 325test_suite("integration") 326atf_test_program{name="sometest"} 327EOF 328 utils_cp_helper simple_all_pass sometest 329 330 cat >expout <<EOF 331sometest:pass 332sometest:skip 333EOF 334 atf_check -s exit:0 -o file:expout -e empty kyua list -k myfile 335 atf_check -s exit:0 -o file:expout -e empty kyua list --kyuafile=myfile 336} 337 338 339utils_test_case kyuafile_flag__some_args 340kyuafile_flag__some_args_body() { 341 cat >Kyuafile <<EOF 342This file is bogus but it is not loaded. 343EOF 344 345 cat >myfile <<EOF 346syntax(2) 347test_suite("hello-world") 348atf_test_program{name="sometest"} 349EOF 350 utils_cp_helper simple_all_pass sometest 351 352 cat >expout <<EOF 353sometest:pass (hello-world) 354sometest:skip (hello-world) 355EOF 356 atf_check -s exit:0 -o file:expout -e empty kyua list -v -k myfile sometest 357 atf_check -s exit:0 -o file:expout -e empty kyua list -v --kyuafile=myfile \ 358 sometest 359} 360 361 362utils_test_case verbose_flag 363verbose_flag_body() { 364 cat >Kyuafile <<EOF 365syntax(2) 366test_suite("integration-suite-1") 367atf_test_program{name="simple_all_pass"} 368plain_test_program{name="i_am_plain", timeout=654} 369include("subdir/Kyuafile") 370EOF 371 utils_cp_helper simple_all_pass . 372 touch i_am_plain 373 374 mkdir subdir 375 cat >subdir/Kyuafile <<EOF 376syntax(2) 377test_suite("integration-suite-2") 378atf_test_program{name="metadata"} 379EOF 380 utils_cp_helper metadata subdir 381 382 cat >expout <<EOF 383simple_all_pass:pass (integration-suite-1) 384simple_all_pass:skip (integration-suite-1) 385i_am_plain:main (integration-suite-1) 386 timeout = 654 387subdir/metadata:no_properties (integration-suite-2) 388subdir/metadata:one_property (integration-suite-2) 389 description = Does nothing but has one metadata property 390subdir/metadata:many_properties (integration-suite-2) 391 allowed_architectures = some-architecture 392 allowed_platforms = some-platform 393 custom.X-no-meaning = I am a custom variable 394 description = A description with some padding 395 required_configs = var1 var2 var3 396 required_files = /my/file1 /some/other/file 397 required_programs = /nonexistent/bin3 bin1 bin2 398 required_user = root 399subdir/metadata:with_cleanup (integration-suite-2) 400 has_cleanup = true 401 timeout = 250 402EOF 403 atf_check -s exit:0 -o file:expout -e empty kyua list -v 404 atf_check -s exit:0 -o file:expout -e empty kyua list --verbose 405} 406 407 408utils_test_case no_test_program_match 409no_test_program_match_body() { 410 cat >Kyuafile <<EOF 411syntax(2) 412test_suite("integration") 413atf_test_program{name="first"} 414EOF 415 utils_cp_helper simple_all_pass first 416 utils_cp_helper simple_all_pass second 417 418 cat >experr <<EOF 419kyua: W: No test cases matched by the filter 'second'. 420EOF 421 atf_check -s exit:1 -o empty -e file:experr kyua list second 422} 423 424 425utils_test_case no_test_case_match 426no_test_case_match_body() { 427 cat >Kyuafile <<EOF 428syntax(2) 429test_suite("integration") 430atf_test_program{name="first"} 431EOF 432 utils_cp_helper simple_all_pass first 433 434 cat >experr <<EOF 435kyua: W: No test cases matched by the filter 'first:foobar'. 436EOF 437 atf_check -s exit:1 -o empty -e file:experr kyua list first:foobar 438} 439 440 441utils_test_case missing_kyuafile__no_args 442missing_kyuafile__no_args_body() { 443 cat >experr <<EOF 444kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 445EOF 446 atf_check -s exit:2 -o empty -e file:experr kyua list 447} 448 449 450utils_test_case missing_kyuafile__test_program 451missing_kyuafile__test_program_body() { 452 mkdir subdir 453 cat >subdir/Kyuafile <<EOF 454syntax(2) 455test_suite("integration") 456atf_test_program{name="unused"} 457EOF 458 utils_cp_helper simple_all_pass subdir/unused 459 460 cat >experr <<EOF 461kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 462EOF 463 atf_check -s exit:2 -o empty -e file:experr kyua list subdir/unused 464} 465 466 467utils_test_case missing_kyuafile__subdir 468missing_kyuafile__subdir_body() { 469 mkdir subdir 470 cat >subdir/Kyuafile <<EOF 471syntax(2) 472test_suite("integration") 473atf_test_program{name="unused"} 474EOF 475 utils_cp_helper simple_all_pass subdir/unused 476 477 cat >experr <<EOF 478kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found. 479EOF 480 atf_check -s exit:2 -o empty -e file:experr kyua list subdir 481} 482 483 484utils_test_case bogus_kyuafile 485bogus_kyuafile_body() { 486 cat >Kyuafile <<EOF 487Hello, world. 488EOF 489 atf_check -s exit:2 -o empty \ 490 -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list 491} 492 493 494utils_test_case bogus_test_program 495bogus_test_program_body() { 496 cat >Kyuafile <<EOF 497syntax(2) 498test_suite("integration") 499atf_test_program{name="crash_on_list"} 500atf_test_program{name="non_executable"} 501EOF 502 utils_cp_helper bad_test_program crash_on_list 503 echo 'I am not executable' >non_executable 504 505 cat >expout <<EOF 506crash_on_list:__test_cases_list__ 507non_executable:__test_cases_list__ 508EOF 509 atf_check -s exit:0 -o file:expout -e empty kyua list 510} 511 512 513utils_test_case missing_test_program 514missing_test_program_body() { 515 cat >Kyuafile <<EOF 516syntax(2) 517include("subdir/Kyuafile") 518EOF 519 mkdir subdir 520 cat >subdir/Kyuafile <<EOF 521syntax(2) 522test_suite("integration") 523atf_test_program{name="ok"} 524atf_test_program{name="i-am-missing"} 525EOF 526 echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok 527 528 cat >experr <<EOF 529kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'. 530EOF 531 atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list 532} 533 534 535atf_init_test_cases() { 536 atf_add_test_case no_args 537 atf_add_test_case one_arg__subdir 538 atf_add_test_case one_arg__test_case 539 atf_add_test_case one_arg__test_program 540 atf_add_test_case one_arg__invalid 541 atf_add_test_case many_args__ok 542 atf_add_test_case many_args__invalid 543 atf_add_test_case many_args__no_match__all 544 atf_add_test_case many_args__no_match__some 545 546 atf_add_test_case args_are_relative 547 548 atf_add_test_case only_load_used_test_programs 549 550 atf_add_test_case build_root_flag 551 552 atf_add_test_case kyuafile_flag__no_args 553 atf_add_test_case kyuafile_flag__some_args 554 555 atf_add_test_case verbose_flag 556 557 atf_add_test_case no_test_program_match 558 atf_add_test_case no_test_case_match 559 560 atf_add_test_case missing_kyuafile__no_args 561 atf_add_test_case missing_kyuafile__test_program 562 atf_add_test_case missing_kyuafile__subdir 563 564 atf_add_test_case bogus_kyuafile 565 atf_add_test_case bogus_test_program 566 atf_add_test_case missing_test_program 567} 568