1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2008, 2009, 2010 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30extern "C" { 31#include <fcntl.h> 32#include <unistd.h> 33} 34 35#include <cerrno> 36#include <cstdlib> 37#include <iostream> 38#include <stdexcept> 39 40#include "macros.hpp" 41 42#include "detail/fs.hpp" 43#include "detail/process.hpp" 44#include "detail/sanity.hpp" 45#include "detail/test_helpers.hpp" 46#include "detail/text.hpp" 47 48// ------------------------------------------------------------------------ 49// Auxiliary functions. 50// ------------------------------------------------------------------------ 51 52static 53void 54create_ctl_file(const atf::tests::tc& tc, const char *name) 55{ 56 ATF_REQUIRE(open(name, O_CREAT | O_WRONLY | O_TRUNC, 0644) != -1); 57} 58 59// ------------------------------------------------------------------------ 60// Auxiliary test cases. 61// ------------------------------------------------------------------------ 62 63ATF_TEST_CASE(h_pass); 64ATF_TEST_CASE_HEAD(h_pass) 65{ 66 set_md_var("descr", "Helper test case"); 67} 68ATF_TEST_CASE_BODY(h_pass) 69{ 70 create_ctl_file(*this, "before"); 71 ATF_PASS(); 72 create_ctl_file(*this, "after"); 73} 74 75ATF_TEST_CASE(h_fail); 76ATF_TEST_CASE_HEAD(h_fail) 77{ 78 set_md_var("descr", "Helper test case"); 79} 80ATF_TEST_CASE_BODY(h_fail) 81{ 82 create_ctl_file(*this, "before"); 83 ATF_FAIL("Failed on purpose"); 84 create_ctl_file(*this, "after"); 85} 86 87ATF_TEST_CASE(h_skip); 88ATF_TEST_CASE_HEAD(h_skip) 89{ 90 set_md_var("descr", "Helper test case"); 91} 92ATF_TEST_CASE_BODY(h_skip) 93{ 94 create_ctl_file(*this, "before"); 95 ATF_SKIP("Skipped on purpose"); 96 create_ctl_file(*this, "after"); 97} 98 99ATF_TEST_CASE(h_require); 100ATF_TEST_CASE_HEAD(h_require) 101{ 102 set_md_var("descr", "Helper test case"); 103} 104ATF_TEST_CASE_BODY(h_require) 105{ 106 bool condition = atf::text::to_bool(get_config_var("condition")); 107 108 create_ctl_file(*this, "before"); 109 ATF_REQUIRE(condition); 110 create_ctl_file(*this, "after"); 111} 112 113ATF_TEST_CASE(h_require_eq); 114ATF_TEST_CASE_HEAD(h_require_eq) 115{ 116 set_md_var("descr", "Helper test case"); 117} 118ATF_TEST_CASE_BODY(h_require_eq) 119{ 120 long v1 = atf::text::to_type< long >(get_config_var("v1")); 121 long v2 = atf::text::to_type< long >(get_config_var("v2")); 122 123 create_ctl_file(*this, "before"); 124 ATF_REQUIRE_EQ(v1, v2); 125 create_ctl_file(*this, "after"); 126} 127 128ATF_TEST_CASE(h_require_match); 129ATF_TEST_CASE_HEAD(h_require_match) 130{ 131 set_md_var("descr", "Helper test case"); 132} 133ATF_TEST_CASE_BODY(h_require_match) 134{ 135 const std::string regexp = get_config_var("regexp"); 136 const std::string string = get_config_var("string"); 137 138 create_ctl_file(*this, "before"); 139 ATF_REQUIRE_MATCH(regexp, string); 140 create_ctl_file(*this, "after"); 141} 142 143ATF_TEST_CASE(h_require_throw); 144ATF_TEST_CASE_HEAD(h_require_throw) 145{ 146 set_md_var("descr", "Helper test case"); 147} 148ATF_TEST_CASE_BODY(h_require_throw) 149{ 150 create_ctl_file(*this, "before"); 151 152 if (get_config_var("what") == "throw_int") 153 ATF_REQUIRE_THROW(std::runtime_error, if (1) throw int(5)); 154 else if (get_config_var("what") == "throw_rt") 155 ATF_REQUIRE_THROW(std::runtime_error, 156 if (1) throw std::runtime_error("e")); 157 else if (get_config_var("what") == "no_throw_rt") 158 ATF_REQUIRE_THROW(std::runtime_error, 159 if (0) throw std::runtime_error("e")); 160 161 create_ctl_file(*this, "after"); 162} 163 164ATF_TEST_CASE(h_require_throw_re); 165ATF_TEST_CASE_HEAD(h_require_throw_re) 166{ 167 set_md_var("descr", "Helper test case"); 168} 169ATF_TEST_CASE_BODY(h_require_throw_re) 170{ 171 create_ctl_file(*this, "before"); 172 173 if (get_config_var("what") == "throw_int") 174 ATF_REQUIRE_THROW_RE(std::runtime_error, "5", if (1) throw int(5)); 175 else if (get_config_var("what") == "throw_rt_match") 176 ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz", 177 if (1) throw std::runtime_error("a foo bar baz")); 178 else if (get_config_var("what") == "throw_rt_no_match") 179 ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz", 180 if (1) throw std::runtime_error("baz foo bar a")); 181 else if (get_config_var("what") == "no_throw_rt") 182 ATF_REQUIRE_THROW_RE(std::runtime_error, "e", 183 if (0) throw std::runtime_error("e")); 184 185 create_ctl_file(*this, "after"); 186} 187 188static int 189errno_fail_stub(const int raised_errno) 190{ 191 errno = raised_errno; 192 return -1; 193} 194 195static int 196errno_ok_stub(void) 197{ 198 return 0; 199} 200 201ATF_TEST_CASE(h_check_errno); 202ATF_TEST_CASE_HEAD(h_check_errno) 203{ 204 set_md_var("descr", "Helper test case"); 205} 206ATF_TEST_CASE_BODY(h_check_errno) 207{ 208 create_ctl_file(*this, "before"); 209 210 if (get_config_var("what") == "no_error") 211 ATF_CHECK_ERRNO(-1, errno_ok_stub() == -1); 212 else if (get_config_var("what") == "errno_ok") 213 ATF_CHECK_ERRNO(2, errno_fail_stub(2) == -1); 214 else if (get_config_var("what") == "errno_fail") 215 ATF_CHECK_ERRNO(3, errno_fail_stub(4) == -1); 216 else 217 UNREACHABLE; 218 219 create_ctl_file(*this, "after"); 220} 221 222ATF_TEST_CASE(h_require_errno); 223ATF_TEST_CASE_HEAD(h_require_errno) 224{ 225 set_md_var("descr", "Helper test case"); 226} 227ATF_TEST_CASE_BODY(h_require_errno) 228{ 229 create_ctl_file(*this, "before"); 230 231 if (get_config_var("what") == "no_error") 232 ATF_REQUIRE_ERRNO(-1, errno_ok_stub() == -1); 233 else if (get_config_var("what") == "errno_ok") 234 ATF_REQUIRE_ERRNO(2, errno_fail_stub(2) == -1); 235 else if (get_config_var("what") == "errno_fail") 236 ATF_REQUIRE_ERRNO(3, errno_fail_stub(4) == -1); 237 else 238 UNREACHABLE; 239 240 create_ctl_file(*this, "after"); 241} 242 243// ------------------------------------------------------------------------ 244// Test cases for the macros. 245// ------------------------------------------------------------------------ 246 247ATF_TEST_CASE(pass); 248ATF_TEST_CASE_HEAD(pass) 249{ 250 set_md_var("descr", "Tests the ATF_PASS macro"); 251} 252ATF_TEST_CASE_BODY(pass) 253{ 254 run_h_tc< ATF_TEST_CASE_NAME(h_pass) >(); 255 ATF_REQUIRE(grep_file("result", "^passed")); 256 ATF_REQUIRE(atf::fs::exists(atf::fs::path("before"))); 257 ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after"))); 258} 259 260ATF_TEST_CASE(fail); 261ATF_TEST_CASE_HEAD(fail) 262{ 263 set_md_var("descr", "Tests the ATF_FAIL macro"); 264} 265ATF_TEST_CASE_BODY(fail) 266{ 267 run_h_tc< ATF_TEST_CASE_NAME(h_fail) >(); 268 ATF_REQUIRE(grep_file("result", "^failed: Failed on purpose")); 269 ATF_REQUIRE(atf::fs::exists(atf::fs::path("before"))); 270 ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after"))); 271} 272 273ATF_TEST_CASE(skip); 274ATF_TEST_CASE_HEAD(skip) 275{ 276 set_md_var("descr", "Tests the ATF_SKIP macro"); 277} 278ATF_TEST_CASE_BODY(skip) 279{ 280 run_h_tc< ATF_TEST_CASE_NAME(h_skip) >(); 281 ATF_REQUIRE(grep_file("result", "^skipped: Skipped on purpose")); 282 ATF_REQUIRE(atf::fs::exists(atf::fs::path("before"))); 283 ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after"))); 284} 285 286ATF_TEST_CASE(require); 287ATF_TEST_CASE_HEAD(require) 288{ 289 set_md_var("descr", "Tests the ATF_REQUIRE macro"); 290} 291ATF_TEST_CASE_BODY(require) 292{ 293 struct test { 294 const char *cond; 295 bool ok; 296 } *t, tests[] = { 297 { "false", false }, 298 { "true", true }, 299 { NULL, false } 300 }; 301 302 const atf::fs::path before("before"); 303 const atf::fs::path after("after"); 304 305 for (t = &tests[0]; t->cond != NULL; t++) { 306 atf::tests::vars_map config; 307 config["condition"] = t->cond; 308 309 std::cout << "Checking with a " << t->cond << " value\n"; 310 311 run_h_tc< ATF_TEST_CASE_NAME(h_require) >(config); 312 313 ATF_REQUIRE(atf::fs::exists(before)); 314 if (t->ok) { 315 ATF_REQUIRE(grep_file("result", "^passed")); 316 ATF_REQUIRE(atf::fs::exists(after)); 317 } else { 318 ATF_REQUIRE(grep_file("result", "^failed: .*condition not met")); 319 ATF_REQUIRE(!atf::fs::exists(after)); 320 } 321 322 atf::fs::remove(before); 323 if (t->ok) 324 atf::fs::remove(after); 325 } 326} 327 328ATF_TEST_CASE(require_eq); 329ATF_TEST_CASE_HEAD(require_eq) 330{ 331 set_md_var("descr", "Tests the ATF_REQUIRE_EQ macro"); 332} 333ATF_TEST_CASE_BODY(require_eq) 334{ 335 struct test { 336 const char *v1; 337 const char *v2; 338 bool ok; 339 } *t, tests[] = { 340 { "1", "1", true }, 341 { "1", "2", false }, 342 { "2", "1", false }, 343 { "2", "2", true }, 344 { NULL, NULL, false } 345 }; 346 347 const atf::fs::path before("before"); 348 const atf::fs::path after("after"); 349 350 for (t = &tests[0]; t->v1 != NULL; t++) { 351 atf::tests::vars_map config; 352 config["v1"] = t->v1; 353 config["v2"] = t->v2; 354 355 std::cout << "Checking with " << t->v1 << ", " << t->v2 356 << " and expecting " << (t->ok ? "true" : "false") 357 << "\n"; 358 359 run_h_tc< ATF_TEST_CASE_NAME(h_require_eq) >(config); 360 361 ATF_REQUIRE(atf::fs::exists(before)); 362 if (t->ok) { 363 ATF_REQUIRE(grep_file("result", "^passed")); 364 ATF_REQUIRE(atf::fs::exists(after)); 365 } else { 366 ATF_REQUIRE(grep_file("result", "^failed: .*v1 != v2")); 367 ATF_REQUIRE(!atf::fs::exists(after)); 368 } 369 370 atf::fs::remove(before); 371 if (t->ok) 372 atf::fs::remove(after); 373 } 374} 375 376ATF_TEST_CASE(require_match); 377ATF_TEST_CASE_HEAD(require_match) 378{ 379 set_md_var("descr", "Tests the ATF_REQUIRE_MATCH macro"); 380} 381ATF_TEST_CASE_BODY(require_match) 382{ 383 struct test { 384 const char *regexp; 385 const char *string; 386 bool ok; 387 } *t, tests[] = { 388 { "foo.*bar", "this is a foo, bar, baz", true }, 389 { "bar.*baz", "this is a baz, bar, foo", false }, 390 { NULL, NULL, false } 391 }; 392 393 const atf::fs::path before("before"); 394 const atf::fs::path after("after"); 395 396 for (t = &tests[0]; t->regexp != NULL; t++) { 397 atf::tests::vars_map config; 398 config["regexp"] = t->regexp; 399 config["string"] = t->string; 400 401 std::cout << "Checking with " << t->regexp << ", " << t->string 402 << " and expecting " << (t->ok ? "true" : "false") 403 << "\n"; 404 405 run_h_tc< ATF_TEST_CASE_NAME(h_require_match) >(config); 406 407 ATF_REQUIRE(atf::fs::exists(before)); 408 if (t->ok) { 409 ATF_REQUIRE(grep_file("result", "^passed")); 410 ATF_REQUIRE(atf::fs::exists(after)); 411 } else { 412 ATF_REQUIRE(grep_file("result", "^failed: ")); 413 ATF_REQUIRE(!atf::fs::exists(after)); 414 } 415 416 atf::fs::remove(before); 417 if (t->ok) 418 atf::fs::remove(after); 419 } 420} 421 422ATF_TEST_CASE(require_throw); 423ATF_TEST_CASE_HEAD(require_throw) 424{ 425 set_md_var("descr", "Tests the ATF_REQUIRE_THROW macro"); 426} 427ATF_TEST_CASE_BODY(require_throw) 428{ 429 struct test { 430 const char *what; 431 bool ok; 432 const char *msg; 433 } *t, tests[] = { 434 { "throw_int", false, "unexpected error" }, 435 { "throw_rt", true, NULL }, 436 { "no_throw_rt", false, "did not throw" }, 437 { NULL, false, NULL } 438 }; 439 440 const atf::fs::path before("before"); 441 const atf::fs::path after("after"); 442 443 for (t = &tests[0]; t->what != NULL; t++) { 444 atf::tests::vars_map config; 445 config["what"] = t->what; 446 447 std::cout << "Checking with " << t->what << " and expecting " 448 << (t->ok ? "true" : "false") << "\n"; 449 450 run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config); 451 452 ATF_REQUIRE(atf::fs::exists(before)); 453 if (t->ok) { 454 ATF_REQUIRE(grep_file("result", "^passed")); 455 ATF_REQUIRE(atf::fs::exists(after)); 456 } else { 457 std::cout << "Checking that message contains '" << t->msg 458 << "'\n"; 459 std::string exp_result = std::string("^failed: .*") + t->msg; 460 ATF_REQUIRE(grep_file("result", exp_result.c_str())); 461 ATF_REQUIRE(!atf::fs::exists(after)); 462 } 463 464 atf::fs::remove(before); 465 if (t->ok) 466 atf::fs::remove(after); 467 } 468} 469 470ATF_TEST_CASE(require_throw_re); 471ATF_TEST_CASE_HEAD(require_throw_re) 472{ 473 set_md_var("descr", "Tests the ATF_REQUIRE_THROW_RE macro"); 474} 475ATF_TEST_CASE_BODY(require_throw_re) 476{ 477 struct test { 478 const char *what; 479 bool ok; 480 const char *msg; 481 } *t, tests[] = { 482 { "throw_int", false, "unexpected error" }, 483 { "throw_rt_match", true, NULL }, 484 { "throw_rt_no_match", true, "threw.*runtime_error(baz foo bar a).*" 485 "does not match 'a foo bar baz'" }, 486 { "no_throw_rt", false, "did not throw" }, 487 { NULL, false, NULL } 488 }; 489 490 const atf::fs::path before("before"); 491 const atf::fs::path after("after"); 492 493 for (t = &tests[0]; t->what != NULL; t++) { 494 atf::tests::vars_map config; 495 config["what"] = t->what; 496 497 std::cout << "Checking with " << t->what << " and expecting " 498 << (t->ok ? "true" : "false") << "\n"; 499 500 run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config); 501 502 ATF_REQUIRE(atf::fs::exists(before)); 503 if (t->ok) { 504 ATF_REQUIRE(grep_file("result", "^passed")); 505 ATF_REQUIRE(atf::fs::exists(after)); 506 } else { 507 std::cout << "Checking that message contains '" << t->msg 508 << "'\n"; 509 std::string exp_result = std::string("^failed: .*") + t->msg; 510 ATF_REQUIRE(grep_file("result", exp_result.c_str())); 511 ATF_REQUIRE(!atf::fs::exists(after)); 512 } 513 514 atf::fs::remove(before); 515 if (t->ok) 516 atf::fs::remove(after); 517 } 518} 519 520ATF_TEST_CASE(check_errno); 521ATF_TEST_CASE_HEAD(check_errno) 522{ 523 set_md_var("descr", "Tests the ATF_CHECK_ERRNO macro"); 524} 525ATF_TEST_CASE_BODY(check_errno) 526{ 527 struct test { 528 const char *what; 529 bool ok; 530 const char *msg; 531 } *t, tests[] = { 532 { "no_error", false, 533 "Expected true value in errno_ok_stub\\(\\) == -1" }, 534 { "errno_ok", true, NULL }, 535 { "errno_fail", false, 536 "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" }, 537 { NULL, false, NULL } 538 }; 539 540 const atf::fs::path before("before"); 541 const atf::fs::path after("after"); 542 543 for (t = &tests[0]; t->what != NULL; t++) { 544 atf::tests::vars_map config; 545 config["what"] = t->what; 546 547 run_h_tc< ATF_TEST_CASE_NAME(h_check_errno) >(config); 548 549 ATF_REQUIRE(atf::fs::exists(before)); 550 ATF_REQUIRE(atf::fs::exists(after)); 551 552 if (t->ok) { 553 ATF_REQUIRE(grep_file("result", "^passed")); 554 } else { 555 ATF_REQUIRE(grep_file("result", "^failed")); 556 557 std::string exp_result = "macros_test.cpp:[0-9]+: " + 558 std::string(t->msg) + "$"; 559 ATF_REQUIRE(grep_file("stderr", exp_result.c_str())); 560 } 561 562 atf::fs::remove(before); 563 atf::fs::remove(after); 564 } 565} 566 567ATF_TEST_CASE(require_errno); 568ATF_TEST_CASE_HEAD(require_errno) 569{ 570 set_md_var("descr", "Tests the ATF_REQUIRE_ERRNO macro"); 571} 572ATF_TEST_CASE_BODY(require_errno) 573{ 574 struct test { 575 const char *what; 576 bool ok; 577 const char *msg; 578 } *t, tests[] = { 579 { "no_error", false, 580 "Expected true value in errno_ok_stub\\(\\) == -1" }, 581 { "errno_ok", true, NULL }, 582 { "errno_fail", false, 583 "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" }, 584 { NULL, false, NULL } 585 }; 586 587 const atf::fs::path before("before"); 588 const atf::fs::path after("after"); 589 590 for (t = &tests[0]; t->what != NULL; t++) { 591 atf::tests::vars_map config; 592 config["what"] = t->what; 593 594 run_h_tc< ATF_TEST_CASE_NAME(h_require_errno) >(config); 595 596 ATF_REQUIRE(atf::fs::exists(before)); 597 if (t->ok) { 598 ATF_REQUIRE(grep_file("result", "^passed")); 599 ATF_REQUIRE(atf::fs::exists(after)); 600 } else { 601 std::string exp_result = "^failed: .*macros_test.cpp:[0-9]+: " + 602 std::string(t->msg) + "$"; 603 ATF_REQUIRE(grep_file("result", exp_result.c_str())); 604 605 ATF_REQUIRE(!atf::fs::exists(after)); 606 } 607 608 atf::fs::remove(before); 609 if (t->ok) 610 atf::fs::remove(after); 611 } 612} 613 614// ------------------------------------------------------------------------ 615// Tests cases for the header file. 616// ------------------------------------------------------------------------ 617 618HEADER_TC(include, "atf-c++/macros.hpp"); 619BUILD_TC(use, "macros_hpp_test.cpp", 620 "Tests that the macros provided by the atf-c++/macros.hpp file " 621 "do not cause syntax errors when used", 622 "Build of macros_hpp_test.cpp failed; some macros in " 623 "atf-c++/macros.hpp are broken"); 624 625// ------------------------------------------------------------------------ 626// Main. 627// ------------------------------------------------------------------------ 628 629ATF_INIT_TEST_CASES(tcs) 630{ 631 // Add the test cases for the macros. 632 ATF_ADD_TEST_CASE(tcs, pass); 633 ATF_ADD_TEST_CASE(tcs, fail); 634 ATF_ADD_TEST_CASE(tcs, skip); 635 ATF_ADD_TEST_CASE(tcs, check_errno); 636 ATF_ADD_TEST_CASE(tcs, require); 637 ATF_ADD_TEST_CASE(tcs, require_eq); 638 ATF_ADD_TEST_CASE(tcs, require_match); 639 ATF_ADD_TEST_CASE(tcs, require_throw); 640 ATF_ADD_TEST_CASE(tcs, require_throw_re); 641 ATF_ADD_TEST_CASE(tcs, require_errno); 642 643 // Add the test cases for the header file. 644 ATF_ADD_TEST_CASE(tcs, include); 645 ATF_ADD_TEST_CASE(tcs, use); 646} 647