1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2008, 2009, 2010 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <fcntl.h>
32#include <unistd.h>
33}
34
35#include <cerrno>
36#include <cstdlib>
37#include <iostream>
38#include <stdexcept>
39
40#include "macros.hpp"
41
42#include "detail/fs.hpp"
43#include "detail/process.hpp"
44#include "detail/sanity.hpp"
45#include "detail/test_helpers.hpp"
46#include "detail/text.hpp"
47
48// ------------------------------------------------------------------------
49// Auxiliary functions.
50// ------------------------------------------------------------------------
51
52static
53void
54create_ctl_file(const atf::tests::tc& tc, const char *name)
55{
56    ATF_REQUIRE(open(name, O_CREAT | O_WRONLY | O_TRUNC, 0644) != -1);
57}
58
59// ------------------------------------------------------------------------
60// Auxiliary test cases.
61// ------------------------------------------------------------------------
62
63ATF_TEST_CASE(h_pass);
64ATF_TEST_CASE_HEAD(h_pass)
65{
66    set_md_var("descr", "Helper test case");
67}
68ATF_TEST_CASE_BODY(h_pass)
69{
70    create_ctl_file(*this, "before");
71    ATF_PASS();
72    create_ctl_file(*this, "after");
73}
74
75ATF_TEST_CASE(h_fail);
76ATF_TEST_CASE_HEAD(h_fail)
77{
78    set_md_var("descr", "Helper test case");
79}
80ATF_TEST_CASE_BODY(h_fail)
81{
82    create_ctl_file(*this, "before");
83    ATF_FAIL("Failed on purpose");
84    create_ctl_file(*this, "after");
85}
86
87ATF_TEST_CASE(h_skip);
88ATF_TEST_CASE_HEAD(h_skip)
89{
90    set_md_var("descr", "Helper test case");
91}
92ATF_TEST_CASE_BODY(h_skip)
93{
94    create_ctl_file(*this, "before");
95    ATF_SKIP("Skipped on purpose");
96    create_ctl_file(*this, "after");
97}
98
99ATF_TEST_CASE(h_require);
100ATF_TEST_CASE_HEAD(h_require)
101{
102    set_md_var("descr", "Helper test case");
103}
104ATF_TEST_CASE_BODY(h_require)
105{
106    bool condition = atf::text::to_bool(get_config_var("condition"));
107
108    create_ctl_file(*this, "before");
109    ATF_REQUIRE(condition);
110    create_ctl_file(*this, "after");
111}
112
113ATF_TEST_CASE(h_require_eq);
114ATF_TEST_CASE_HEAD(h_require_eq)
115{
116    set_md_var("descr", "Helper test case");
117}
118ATF_TEST_CASE_BODY(h_require_eq)
119{
120    long v1 = atf::text::to_type< long >(get_config_var("v1"));
121    long v2 = atf::text::to_type< long >(get_config_var("v2"));
122
123    create_ctl_file(*this, "before");
124    ATF_REQUIRE_EQ(v1, v2);
125    create_ctl_file(*this, "after");
126}
127
128ATF_TEST_CASE(h_require_match);
129ATF_TEST_CASE_HEAD(h_require_match)
130{
131    set_md_var("descr", "Helper test case");
132}
133ATF_TEST_CASE_BODY(h_require_match)
134{
135    const std::string regexp = get_config_var("regexp");
136    const std::string string = get_config_var("string");
137
138    create_ctl_file(*this, "before");
139    ATF_REQUIRE_MATCH(regexp, string);
140    create_ctl_file(*this, "after");
141}
142
143ATF_TEST_CASE(h_require_throw);
144ATF_TEST_CASE_HEAD(h_require_throw)
145{
146    set_md_var("descr", "Helper test case");
147}
148ATF_TEST_CASE_BODY(h_require_throw)
149{
150    create_ctl_file(*this, "before");
151
152    if (get_config_var("what") == "throw_int")
153        ATF_REQUIRE_THROW(std::runtime_error, if (1) throw int(5));
154    else if (get_config_var("what") == "throw_rt")
155        ATF_REQUIRE_THROW(std::runtime_error,
156                        if (1) throw std::runtime_error("e"));
157    else if (get_config_var("what") == "no_throw_rt")
158        ATF_REQUIRE_THROW(std::runtime_error,
159                        if (0) throw std::runtime_error("e"));
160
161    create_ctl_file(*this, "after");
162}
163
164ATF_TEST_CASE(h_require_throw_re);
165ATF_TEST_CASE_HEAD(h_require_throw_re)
166{
167    set_md_var("descr", "Helper test case");
168}
169ATF_TEST_CASE_BODY(h_require_throw_re)
170{
171    create_ctl_file(*this, "before");
172
173    if (get_config_var("what") == "throw_int")
174        ATF_REQUIRE_THROW_RE(std::runtime_error, "5", if (1) throw int(5));
175    else if (get_config_var("what") == "throw_rt_match")
176        ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz",
177                             if (1) throw std::runtime_error("a foo bar baz"));
178    else if (get_config_var("what") == "throw_rt_no_match")
179        ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz",
180                             if (1) throw std::runtime_error("baz foo bar a"));
181    else if (get_config_var("what") == "no_throw_rt")
182        ATF_REQUIRE_THROW_RE(std::runtime_error, "e",
183                             if (0) throw std::runtime_error("e"));
184
185    create_ctl_file(*this, "after");
186}
187
188static int
189errno_fail_stub(const int raised_errno)
190{
191    errno = raised_errno;
192    return -1;
193}
194
195static int
196errno_ok_stub(void)
197{
198    return 0;
199}
200
201ATF_TEST_CASE(h_check_errno);
202ATF_TEST_CASE_HEAD(h_check_errno)
203{
204    set_md_var("descr", "Helper test case");
205}
206ATF_TEST_CASE_BODY(h_check_errno)
207{
208    create_ctl_file(*this, "before");
209
210    if (get_config_var("what") == "no_error")
211        ATF_CHECK_ERRNO(-1, errno_ok_stub() == -1);
212    else if (get_config_var("what") == "errno_ok")
213        ATF_CHECK_ERRNO(2, errno_fail_stub(2) == -1);
214    else if (get_config_var("what") == "errno_fail")
215        ATF_CHECK_ERRNO(3, errno_fail_stub(4) == -1);
216    else
217        UNREACHABLE;
218
219    create_ctl_file(*this, "after");
220}
221
222ATF_TEST_CASE(h_require_errno);
223ATF_TEST_CASE_HEAD(h_require_errno)
224{
225    set_md_var("descr", "Helper test case");
226}
227ATF_TEST_CASE_BODY(h_require_errno)
228{
229    create_ctl_file(*this, "before");
230
231    if (get_config_var("what") == "no_error")
232        ATF_REQUIRE_ERRNO(-1, errno_ok_stub() == -1);
233    else if (get_config_var("what") == "errno_ok")
234        ATF_REQUIRE_ERRNO(2, errno_fail_stub(2) == -1);
235    else if (get_config_var("what") == "errno_fail")
236        ATF_REQUIRE_ERRNO(3, errno_fail_stub(4) == -1);
237    else
238        UNREACHABLE;
239
240    create_ctl_file(*this, "after");
241}
242
243// ------------------------------------------------------------------------
244// Test cases for the macros.
245// ------------------------------------------------------------------------
246
247ATF_TEST_CASE(pass);
248ATF_TEST_CASE_HEAD(pass)
249{
250    set_md_var("descr", "Tests the ATF_PASS macro");
251}
252ATF_TEST_CASE_BODY(pass)
253{
254    run_h_tc< ATF_TEST_CASE_NAME(h_pass) >();
255    ATF_REQUIRE(grep_file("result", "^passed"));
256    ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
257    ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
258}
259
260ATF_TEST_CASE(fail);
261ATF_TEST_CASE_HEAD(fail)
262{
263    set_md_var("descr", "Tests the ATF_FAIL macro");
264}
265ATF_TEST_CASE_BODY(fail)
266{
267    run_h_tc< ATF_TEST_CASE_NAME(h_fail) >();
268    ATF_REQUIRE(grep_file("result", "^failed: Failed on purpose"));
269    ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
270    ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
271}
272
273ATF_TEST_CASE(skip);
274ATF_TEST_CASE_HEAD(skip)
275{
276    set_md_var("descr", "Tests the ATF_SKIP macro");
277}
278ATF_TEST_CASE_BODY(skip)
279{
280    run_h_tc< ATF_TEST_CASE_NAME(h_skip) >();
281    ATF_REQUIRE(grep_file("result", "^skipped: Skipped on purpose"));
282    ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
283    ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
284}
285
286ATF_TEST_CASE(require);
287ATF_TEST_CASE_HEAD(require)
288{
289    set_md_var("descr", "Tests the ATF_REQUIRE macro");
290}
291ATF_TEST_CASE_BODY(require)
292{
293    struct test {
294        const char *cond;
295        bool ok;
296    } *t, tests[] = {
297        { "false", false },
298        { "true", true },
299        { NULL, false }
300    };
301
302    const atf::fs::path before("before");
303    const atf::fs::path after("after");
304
305    for (t = &tests[0]; t->cond != NULL; t++) {
306        atf::tests::vars_map config;
307        config["condition"] = t->cond;
308
309        std::cout << "Checking with a " << t->cond << " value\n";
310
311        run_h_tc< ATF_TEST_CASE_NAME(h_require) >(config);
312
313        ATF_REQUIRE(atf::fs::exists(before));
314        if (t->ok) {
315            ATF_REQUIRE(grep_file("result", "^passed"));
316            ATF_REQUIRE(atf::fs::exists(after));
317        } else {
318            ATF_REQUIRE(grep_file("result", "^failed: .*condition not met"));
319            ATF_REQUIRE(!atf::fs::exists(after));
320        }
321
322        atf::fs::remove(before);
323        if (t->ok)
324            atf::fs::remove(after);
325    }
326}
327
328ATF_TEST_CASE(require_eq);
329ATF_TEST_CASE_HEAD(require_eq)
330{
331    set_md_var("descr", "Tests the ATF_REQUIRE_EQ macro");
332}
333ATF_TEST_CASE_BODY(require_eq)
334{
335    struct test {
336        const char *v1;
337        const char *v2;
338        bool ok;
339    } *t, tests[] = {
340        { "1", "1", true },
341        { "1", "2", false },
342        { "2", "1", false },
343        { "2", "2", true },
344        { NULL, NULL, false }
345    };
346
347    const atf::fs::path before("before");
348    const atf::fs::path after("after");
349
350    for (t = &tests[0]; t->v1 != NULL; t++) {
351        atf::tests::vars_map config;
352        config["v1"] = t->v1;
353        config["v2"] = t->v2;
354
355        std::cout << "Checking with " << t->v1 << ", " << t->v2
356                  << " and expecting " << (t->ok ? "true" : "false")
357                  << "\n";
358
359        run_h_tc< ATF_TEST_CASE_NAME(h_require_eq) >(config);
360
361        ATF_REQUIRE(atf::fs::exists(before));
362        if (t->ok) {
363            ATF_REQUIRE(grep_file("result", "^passed"));
364            ATF_REQUIRE(atf::fs::exists(after));
365        } else {
366            ATF_REQUIRE(grep_file("result", "^failed: .*v1 != v2"));
367            ATF_REQUIRE(!atf::fs::exists(after));
368        }
369
370        atf::fs::remove(before);
371        if (t->ok)
372            atf::fs::remove(after);
373    }
374}
375
376ATF_TEST_CASE(require_match);
377ATF_TEST_CASE_HEAD(require_match)
378{
379    set_md_var("descr", "Tests the ATF_REQUIRE_MATCH macro");
380}
381ATF_TEST_CASE_BODY(require_match)
382{
383    struct test {
384        const char *regexp;
385        const char *string;
386        bool ok;
387    } *t, tests[] = {
388        { "foo.*bar", "this is a foo, bar, baz", true },
389        { "bar.*baz", "this is a baz, bar, foo", false },
390        { NULL, NULL, false }
391    };
392
393    const atf::fs::path before("before");
394    const atf::fs::path after("after");
395
396    for (t = &tests[0]; t->regexp != NULL; t++) {
397        atf::tests::vars_map config;
398        config["regexp"] = t->regexp;
399        config["string"] = t->string;
400
401        std::cout << "Checking with " << t->regexp << ", " << t->string
402                  << " and expecting " << (t->ok ? "true" : "false")
403                  << "\n";
404
405        run_h_tc< ATF_TEST_CASE_NAME(h_require_match) >(config);
406
407        ATF_REQUIRE(atf::fs::exists(before));
408        if (t->ok) {
409            ATF_REQUIRE(grep_file("result", "^passed"));
410            ATF_REQUIRE(atf::fs::exists(after));
411        } else {
412            ATF_REQUIRE(grep_file("result", "^failed: "));
413            ATF_REQUIRE(!atf::fs::exists(after));
414        }
415
416        atf::fs::remove(before);
417        if (t->ok)
418            atf::fs::remove(after);
419    }
420}
421
422ATF_TEST_CASE(require_throw);
423ATF_TEST_CASE_HEAD(require_throw)
424{
425    set_md_var("descr", "Tests the ATF_REQUIRE_THROW macro");
426}
427ATF_TEST_CASE_BODY(require_throw)
428{
429    struct test {
430        const char *what;
431        bool ok;
432        const char *msg;
433    } *t, tests[] = {
434        { "throw_int", false, "unexpected error" },
435        { "throw_rt", true, NULL },
436        { "no_throw_rt", false, "did not throw" },
437        { NULL, false, NULL }
438    };
439
440    const atf::fs::path before("before");
441    const atf::fs::path after("after");
442
443    for (t = &tests[0]; t->what != NULL; t++) {
444        atf::tests::vars_map config;
445        config["what"] = t->what;
446
447        std::cout << "Checking with " << t->what << " and expecting "
448                  << (t->ok ? "true" : "false") << "\n";
449
450        run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config);
451
452        ATF_REQUIRE(atf::fs::exists(before));
453        if (t->ok) {
454            ATF_REQUIRE(grep_file("result", "^passed"));
455            ATF_REQUIRE(atf::fs::exists(after));
456        } else {
457            std::cout << "Checking that message contains '" << t->msg
458                      << "'\n";
459            std::string exp_result = std::string("^failed: .*") + t->msg;
460            ATF_REQUIRE(grep_file("result", exp_result.c_str()));
461            ATF_REQUIRE(!atf::fs::exists(after));
462        }
463
464        atf::fs::remove(before);
465        if (t->ok)
466            atf::fs::remove(after);
467    }
468}
469
470ATF_TEST_CASE(require_throw_re);
471ATF_TEST_CASE_HEAD(require_throw_re)
472{
473    set_md_var("descr", "Tests the ATF_REQUIRE_THROW_RE macro");
474}
475ATF_TEST_CASE_BODY(require_throw_re)
476{
477    struct test {
478        const char *what;
479        bool ok;
480        const char *msg;
481    } *t, tests[] = {
482        { "throw_int", false, "unexpected error" },
483        { "throw_rt_match", true, NULL },
484        { "throw_rt_no_match", true, "threw.*runtime_error(baz foo bar a).*"
485          "does not match 'a foo bar baz'" },
486        { "no_throw_rt", false, "did not throw" },
487        { NULL, false, NULL }
488    };
489
490    const atf::fs::path before("before");
491    const atf::fs::path after("after");
492
493    for (t = &tests[0]; t->what != NULL; t++) {
494        atf::tests::vars_map config;
495        config["what"] = t->what;
496
497        std::cout << "Checking with " << t->what << " and expecting "
498                  << (t->ok ? "true" : "false") << "\n";
499
500        run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config);
501
502        ATF_REQUIRE(atf::fs::exists(before));
503        if (t->ok) {
504            ATF_REQUIRE(grep_file("result", "^passed"));
505            ATF_REQUIRE(atf::fs::exists(after));
506        } else {
507            std::cout << "Checking that message contains '" << t->msg
508                      << "'\n";
509            std::string exp_result = std::string("^failed: .*") + t->msg;
510            ATF_REQUIRE(grep_file("result", exp_result.c_str()));
511            ATF_REQUIRE(!atf::fs::exists(after));
512        }
513
514        atf::fs::remove(before);
515        if (t->ok)
516            atf::fs::remove(after);
517    }
518}
519
520ATF_TEST_CASE(check_errno);
521ATF_TEST_CASE_HEAD(check_errno)
522{
523    set_md_var("descr", "Tests the ATF_CHECK_ERRNO macro");
524}
525ATF_TEST_CASE_BODY(check_errno)
526{
527    struct test {
528        const char *what;
529        bool ok;
530        const char *msg;
531    } *t, tests[] = {
532        { "no_error", false,
533          "Expected true value in errno_ok_stub\\(\\) == -1" },
534        { "errno_ok", true, NULL },
535        { "errno_fail", false,
536          "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" },
537        { NULL, false, NULL }
538    };
539
540    const atf::fs::path before("before");
541    const atf::fs::path after("after");
542
543    for (t = &tests[0]; t->what != NULL; t++) {
544        atf::tests::vars_map config;
545        config["what"] = t->what;
546
547        run_h_tc< ATF_TEST_CASE_NAME(h_check_errno) >(config);
548
549        ATF_REQUIRE(atf::fs::exists(before));
550        ATF_REQUIRE(atf::fs::exists(after));
551
552        if (t->ok) {
553            ATF_REQUIRE(grep_file("result", "^passed"));
554        } else {
555            ATF_REQUIRE(grep_file("result", "^failed"));
556
557            std::string exp_result = "macros_test.cpp:[0-9]+: " +
558                std::string(t->msg) + "$";
559            ATF_REQUIRE(grep_file("stderr", exp_result.c_str()));
560        }
561
562        atf::fs::remove(before);
563        atf::fs::remove(after);
564    }
565}
566
567ATF_TEST_CASE(require_errno);
568ATF_TEST_CASE_HEAD(require_errno)
569{
570    set_md_var("descr", "Tests the ATF_REQUIRE_ERRNO macro");
571}
572ATF_TEST_CASE_BODY(require_errno)
573{
574    struct test {
575        const char *what;
576        bool ok;
577        const char *msg;
578    } *t, tests[] = {
579        { "no_error", false,
580          "Expected true value in errno_ok_stub\\(\\) == -1" },
581        { "errno_ok", true, NULL },
582        { "errno_fail", false,
583          "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" },
584        { NULL, false, NULL }
585    };
586
587    const atf::fs::path before("before");
588    const atf::fs::path after("after");
589
590    for (t = &tests[0]; t->what != NULL; t++) {
591        atf::tests::vars_map config;
592        config["what"] = t->what;
593
594        run_h_tc< ATF_TEST_CASE_NAME(h_require_errno) >(config);
595
596        ATF_REQUIRE(atf::fs::exists(before));
597        if (t->ok) {
598            ATF_REQUIRE(grep_file("result", "^passed"));
599            ATF_REQUIRE(atf::fs::exists(after));
600        } else {
601            std::string exp_result = "^failed: .*macros_test.cpp:[0-9]+: " +
602                std::string(t->msg) + "$";
603            ATF_REQUIRE(grep_file("result", exp_result.c_str()));
604
605            ATF_REQUIRE(!atf::fs::exists(after));
606        }
607
608        atf::fs::remove(before);
609        if (t->ok)
610            atf::fs::remove(after);
611    }
612}
613
614// ------------------------------------------------------------------------
615// Tests cases for the header file.
616// ------------------------------------------------------------------------
617
618HEADER_TC(include, "atf-c++/macros.hpp");
619BUILD_TC(use, "macros_hpp_test.cpp",
620         "Tests that the macros provided by the atf-c++/macros.hpp file "
621         "do not cause syntax errors when used",
622         "Build of macros_hpp_test.cpp failed; some macros in "
623         "atf-c++/macros.hpp are broken");
624
625// ------------------------------------------------------------------------
626// Main.
627// ------------------------------------------------------------------------
628
629ATF_INIT_TEST_CASES(tcs)
630{
631    // Add the test cases for the macros.
632    ATF_ADD_TEST_CASE(tcs, pass);
633    ATF_ADD_TEST_CASE(tcs, fail);
634    ATF_ADD_TEST_CASE(tcs, skip);
635    ATF_ADD_TEST_CASE(tcs, check_errno);
636    ATF_ADD_TEST_CASE(tcs, require);
637    ATF_ADD_TEST_CASE(tcs, require_eq);
638    ATF_ADD_TEST_CASE(tcs, require_match);
639    ATF_ADD_TEST_CASE(tcs, require_throw);
640    ATF_ADD_TEST_CASE(tcs, require_throw_re);
641    ATF_ADD_TEST_CASE(tcs, require_errno);
642
643    // Add the test cases for the header file.
644    ATF_ADD_TEST_CASE(tcs, include);
645    ATF_ADD_TEST_CASE(tcs, use);
646}
647