1# 2# Automated Testing Framework (atf) 3# 4# Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28# 29 30create_helpers() 31{ 32 mkdir dir1 33 cp $(atf_get_srcdir)/pass_helper dir1/tp1 34 cp $(atf_get_srcdir)/fail_helper dir1/tp2 35 cp $(atf_get_srcdir)/pass_helper tp3 36 cp $(atf_get_srcdir)/fail_helper tp4 37 38 cat >tp5 <<EOF 39#! $(atf-config -t atf_shell) 40echo foo 41EOF 42 chmod +x tp5 43 44 cat >Atffile <<EOF 45Content-Type: application/X-atf-atffile; version="1" 46 47prop: test-suite = atf 48 49tp: dir1 50tp: tp3 51tp: tp4 52tp: tp5 53EOF 54 55 cat >dir1/Atffile <<EOF 56Content-Type: application/X-atf-atffile; version="1" 57 58prop: test-suite = atf 59 60tp: tp1 61tp: tp2 62EOF 63} 64 65run_helpers() 66{ 67 mkdir etc 68 cat >etc/atf-run.hooks <<EOF 69#! $(atf-config -t atf_shell) 70 71info_start_hook() 72{ 73 atf_tps_writer_info "startinfo" "A value" 74} 75 76info_end_hook() 77{ 78 atf_tps_writer_info "endinfo" "Another value" 79} 80EOF 81 echo "Using atf-run to run helpers" 82 ATF_CONFDIR=$(pwd)/etc atf-run >tps.out 2>/dev/null 83 rm -rf etc 84} 85 86atf_test_case default 87default_head() 88{ 89 atf_set "descr" "Checks that the default output uses the ticker" \ 90 "format" 91} 92default_body() 93{ 94 create_helpers 95 run_helpers 96 97 # Check that the default output uses the ticker format. 98 atf_check -s eq:0 -o match:'test cases' -o match:'Failed test cases' \ 99 -o match:'Summary for' -e empty -x 'atf-report <tps.out' 100} 101 102# XXX The test for all expect_ values should be intermixed with the other 103# tests. However, to do that, we need to migrate to using C helpers for 104# simplicity in raising signals... 105atf_test_case expect 106expect_body() 107{ 108 ln -s "$(atf_get_srcdir)/../atf-run/expect_helpers" . 109 cat >Atffile <<EOF 110Content-Type: application/X-atf-atffile; version="1" 111 112prop: test-suite = atf 113 114tp: expect_helpers 115EOF 116 run_helpers 117 118# NO_CHECK_STYLE_BEGIN 119 cat >expout <<EOF 120tc, expect_helpers, death_and_exit, expected_death, Exit case 121tc, expect_helpers, death_and_signal, expected_death, Signal case 122tc, expect_helpers, death_but_pass, failed, Test case was expected to terminate abruptly but it continued execution 123tc, expect_helpers, exit_any_and_exit, expected_exit, Call will exit 124tc, expect_helpers, exit_but_pass, failed, Test case was expected to exit cleanly but it continued execution 125tc, expect_helpers, exit_code_and_exit, expected_exit, Call will exit 126tc, expect_helpers, fail_and_fail_check, expected_failure, And fail again: 2 checks failed as expected; see output for more details 127tc, expect_helpers, fail_and_fail_requirement, expected_failure, Fail reason: The failure 128tc, expect_helpers, fail_but_pass, failed, Test case was expecting a failure but none were raised 129tc, expect_helpers, pass_and_pass, passed 130tc, expect_helpers, pass_but_fail_check, failed, 1 checks failed; see output for more details 131tc, expect_helpers, pass_but_fail_requirement, failed, Some reason 132tc, expect_helpers, signal_any_and_signal, expected_signal, Call will signal 133tc, expect_helpers, signal_but_pass, failed, Test case was expected to receive a termination signal but it continued execution 134tc, expect_helpers, signal_no_and_signal, expected_signal, Call will signal 135tc, expect_helpers, timeout_and_hang, expected_timeout, Will overrun 136tc, expect_helpers, timeout_but_pass, failed, Test case was expected to hang but it continued execution 137tp, expect_helpers, failed 138EOF 139# NO_CHECK_STYLE_END 140 atf_check -s eq:0 -o file:expout -e empty -x "atf-report -o csv:- <tps.out" 141 142# NO_CHECK_STYLE_BEGIN 143 cat >expout <<EOF 144expect_helpers (1/1): 17 test cases 145 death_and_exit: Expected failure: Exit case 146 death_and_signal: Expected failure: Signal case 147 death_but_pass: Failed: Test case was expected to terminate abruptly but it continued execution 148 exit_any_and_exit: Expected failure: Call will exit 149 exit_but_pass: Failed: Test case was expected to exit cleanly but it continued execution 150 exit_code_and_exit: Expected failure: Call will exit 151 fail_and_fail_check: Expected failure: And fail again: 2 checks failed as expected; see output for more details 152 fail_and_fail_requirement: Expected failure: Fail reason: The failure 153 fail_but_pass: Failed: Test case was expecting a failure but none were raised 154 pass_and_pass: Passed. 155 pass_but_fail_check: Failed: 1 checks failed; see output for more details 156 pass_but_fail_requirement: Failed: Some reason 157 signal_any_and_signal: Expected failure: Call will signal 158 signal_but_pass: Failed: Test case was expected to receive a termination signal but it continued execution 159 signal_no_and_signal: Expected failure: Call will signal 160 timeout_and_hang: Expected failure: Will overrun 161 timeout_but_pass: Failed: Test case was expected to hang but it continued execution 162 163Test cases for known bugs: 164 expect_helpers:death_and_exit: Exit case 165 expect_helpers:death_and_signal: Signal case 166 expect_helpers:exit_any_and_exit: Call will exit 167 expect_helpers:exit_code_and_exit: Call will exit 168 expect_helpers:fail_and_fail_check: And fail again: 2 checks failed as expected; see output for more details 169 expect_helpers:fail_and_fail_requirement: Fail reason: The failure 170 expect_helpers:signal_any_and_signal: Call will signal 171 expect_helpers:signal_no_and_signal: Call will signal 172 expect_helpers:timeout_and_hang: Will overrun 173 174Failed test cases: 175 expect_helpers:death_but_pass, expect_helpers:exit_but_pass, expect_helpers:fail_but_pass, expect_helpers:pass_but_fail_check, expect_helpers:pass_but_fail_requirement, expect_helpers:signal_but_pass, expect_helpers:timeout_but_pass 176 177Summary for 1 test programs: 178 1 passed test cases. 179 7 failed test cases. 180 9 expected failed test cases. 181 0 skipped test cases. 182EOF 183# NO_CHECK_STYLE_END 184 atf_check -s eq:0 -o file:expout -e empty -x \ 185 "atf-report -o ticker:- <tps.out" 186 187 # Just ensure that this does not crash for now... 188 atf_check -s eq:0 -o ignore -e empty -x "atf-report -o xml:- <tps.out" 189} 190 191atf_test_case oflag 192oflag_head() 193{ 194 atf_set "descr" "Checks that the -o flag works" 195} 196oflag_body() 197{ 198 create_helpers 199 run_helpers 200 201 # Get the default output. 202 atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report <tps.out' 203 mv stdout defout 204 205 # Check that changing the stdout output works. 206 atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report -o csv:- <tps.out' 207 atf_check -s eq:1 -o empty -e empty cmp -s defout stdout 208 cp stdout expcsv 209 210 # Check that sending the output to a file does not write to stdout. 211 atf_check -s eq:0 -o empty -e empty -x 'atf-report -o csv:fmt.out <tps.out' 212 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out 213 rm -f fmt.out 214 215 # Check that defining two outputs using the same format works. 216 atf_check -s eq:0 -o empty -e empty -x \ 217 'atf-report -o csv:fmt.out -o csv:fmt2.out <tps.out' 218 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out 219 atf_check -s eq:0 -o empty -e empty cmp -s fmt.out fmt2.out 220 rm -f fmt.out fmt2.out 221 222 # Check that defining two outputs using different formats works. 223 atf_check -s eq:0 -o empty -e empty -x \ 224 'atf-report -o csv:fmt.out -o ticker:fmt2.out <tps.out' 225 atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out 226 atf_check -s eq:1 -o empty -e empty cmp -s fmt.out fmt2.out 227 atf_check -s eq:0 -o ignore -e empty grep "test cases" fmt2.out 228 atf_check -s eq:0 -o ignore -e empty grep "Failed test cases" fmt2.out 229 atf_check -s eq:0 -o ignore -e empty grep "Summary for" fmt2.out 230 rm -f fmt.out fmt2.out 231 232 # Check that defining two outputs over the same file does not work. 233 atf_check -s eq:1 -o empty -e match:'more than once' -x \ 234 'atf-report -o csv:fmt.out -o ticker:fmt.out <tps.out' 235 rm -f fmt.out 236 237 # Check that defining two outputs over stdout (but using different 238 # paths) does not work. 239 atf_check -s eq:1 -o empty -e match:'more than once' -x \ 240 'atf-report -o csv:- -o ticker:/dev/stdout <tps.out' 241 rm -f fmt.out 242} 243 244atf_test_case output_csv 245output_csv_head() 246{ 247 atf_set "descr" "Checks the CSV output format" 248} 249output_csv_body() 250{ 251 create_helpers 252 run_helpers 253 254# NO_CHECK_STYLE_BEGIN 255 cat >expout <<EOF 256tc, dir1/tp1, main, passed 257tp, dir1/tp1, passed 258tc, dir1/tp2, main, failed, This always fails 259tp, dir1/tp2, failed 260tc, tp3, main, passed 261tp, tp3, passed 262tc, tp4, main, failed, This always fails 263tp, tp4, failed 264tp, tp5, bogus, Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:' 265EOF 266# NO_CHECK_STYLE_END 267 268 atf_check -s eq:0 -o file:expout -e empty -x 'atf-report -o csv:- <tps.out' 269} 270 271atf_test_case output_ticker 272output_ticker_head() 273{ 274 atf_set "descr" "Checks the ticker output format" 275} 276output_ticker_body() 277{ 278 create_helpers 279 run_helpers 280 281# NO_CHECK_STYLE_BEGIN 282 cat >expout <<EOF 283dir1/tp1 (1/5): 1 test cases 284 main: Passed. 285 286dir1/tp2 (2/5): 1 test cases 287 main: Failed: This always fails 288 289tp3 (3/5): 1 test cases 290 main: Passed. 291 292tp4 (4/5): 1 test cases 293 main: Failed: This always fails 294 295tp5 (5/5): 0 test cases 296tp5: BOGUS TEST PROGRAM: Cannot trust its results because of \`Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'' 297 298Failed (bogus) test programs: 299 tp5 300 301Failed test cases: 302 dir1/tp2:main, tp4:main 303 304Summary for 5 test programs: 305 2 passed test cases. 306 2 failed test cases. 307 0 expected failed test cases. 308 0 skipped test cases. 309EOF 310 311 atf_check -s eq:0 -o file:expout -e empty -x 'atf-report -o ticker:- <tps.out' 312} 313# NO_CHECK_STYLE_END 314 315atf_test_case output_xml 316output_xml_head() 317{ 318 atf_set "descr" "Checks the XML output format" 319} 320output_xml_body() 321{ 322 create_helpers 323 run_helpers 324 325# NO_CHECK_STYLE_BEGIN 326 cat >expout <<EOF 327<?xml version="1.0"?> 328<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd"> 329 330<tests-results> 331<info class="startinfo">A value</info> 332<tp id="dir1/tp1"> 333<tc id="main"> 334<passed /> 335</tc> 336</tp> 337<tp id="dir1/tp2"> 338<tc id="main"> 339<failed>This always fails</failed> 340</tc> 341</tp> 342<tp id="tp3"> 343<tc id="main"> 344<passed /> 345</tc> 346</tp> 347<tp id="tp4"> 348<tc id="main"> 349<failed>This always fails</failed> 350</tc> 351</tp> 352<tp id="tp5"> 353<failed>Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'</failed> 354</tp> 355<info class="endinfo">Another value</info> 356</tests-results> 357EOF 358# NO_CHECK_STYLE_END 359 360 atf_check -s eq:0 -o file:expout -e empty -x 'atf-report -o xml:- <tps.out' 361} 362 363atf_test_case output_xml_space 364output_xml_space_head() 365{ 366 atf_set "descr" "Checks that the XML output format properly preserves" \ 367 "leading and trailing whitespace in stdout and stderr" \ 368 "lines" 369} 370output_xml_space_body() 371{ 372 cp $(atf_get_srcdir)/misc_helpers . 373 cat >Atffile <<EOF 374Content-Type: application/X-atf-atffile; version="1" 375 376prop: test-suite = atf 377 378tp: misc_helpers 379EOF 380 381# NO_CHECK_STYLE_BEGIN 382 cat >expout <<EOF 383<?xml version="1.0"?> 384<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd"> 385 386<tests-results> 387<info class="startinfo">A value</info> 388<tp id="misc_helpers"> 389<tc id="diff"> 390<so>--- a 2007-11-04 14:00:41.000000000 +0100</so> 391<so>+++ b 2007-11-04 14:00:48.000000000 +0100</so> 392<so>@@ -1,7 +1,7 @@</so> 393<so> This test is meant to simulate a diff.</so> 394<so> Blank space at beginning of context lines must be preserved.</so> 395<so> </so> 396<so>-First original line.</so> 397<so>-Second original line.</so> 398<so>+First modified line.</so> 399<so>+Second modified line.</so> 400<so> </so> 401<so> EOF</so> 402<passed /> 403</tc> 404</tp> 405<info class="endinfo">Another value</info> 406</tests-results> 407EOF 408# NO_CHECK_STYLE_END 409 410 run_helpers 411 atf_check -s eq:0 -o file:expout -e empty -x 'atf-report -o xml:- <tps.out' 412} 413 414atf_init_test_cases() 415{ 416 atf_add_test_case default 417 atf_add_test_case expect 418 atf_add_test_case oflag 419 atf_add_test_case output_csv 420 atf_add_test_case output_ticker 421 atf_add_test_case output_xml 422 atf_add_test_case output_xml_space 423} 424 425# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4 426