1// Copyright 2012 Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9//   notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above copyright
11//   notice, this list of conditions and the following disclaimer in the
12//   documentation and/or other materials provided with the distribution.
13// * Neither the name of Google Inc. nor the names of its contributors
14//   may be used to endorse or promote products derived from this software
15//   without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#include <stdlib.h>
30
31#include <atf-c.h>
32
33#define INTERFACE "atf"
34#include "common_inttest.h"
35
36
37ATF_TC(list__ok);
38ATF_TC_HEAD(list__ok, tc) { setup(tc, true); }
39ATF_TC_BODY(list__ok, tc)
40{
41    char* helpers = helpers_path(tc);
42    check(EXIT_SUCCESS,
43          "test_case{name='fail'}\n"
44          "test_case{name='pass'}\n"
45          "test_case{name='signal'}\n"
46          "test_case{name='sleep'}\n"
47          "test_case{name='cleanup_check_work_directory', has_cleanup='true'}\n"
48          "test_case{name='cleanup_fail', has_cleanup='true'}\n"
49          "test_case{name='cleanup_signal', has_cleanup='true'}\n"
50          "test_case{name='cleanup_sleep', has_cleanup='true'}\n"
51          "test_case{name='body_and_cleanup_fail', has_cleanup='true'}\n"
52          "test_case{name='print_config', has_cleanup='true'}\n",
53          "",
54          "list", helpers, NULL);
55    free(helpers);
56}
57
58
59ATF_TC(test__pass);
60ATF_TC_HEAD(test__pass, tc) { setup(tc, true); }
61ATF_TC_BODY(test__pass, tc)
62{
63    char* helpers = helpers_path(tc);
64    check(EXIT_SUCCESS,
65          "First line to stdout\nSecond line to stdout\n",
66          "First line to stderr\nSecond line to stderr\n",
67          "test", helpers, "pass", "test-result", NULL);
68    free(helpers);
69
70    ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
71}
72
73
74ATF_TC(test__fail);
75ATF_TC_HEAD(test__fail, tc) { setup(tc, true); }
76ATF_TC_BODY(test__fail, tc)
77{
78    char* helpers = helpers_path(tc);
79    check(EXIT_FAILURE, "First line to stdout\n", "First line to stderr\n",
80          "test", helpers, "fail", "test-result", NULL);
81    free(helpers);
82
83    ATF_REQUIRE(atf_utils_compare_file("test-result",
84        "failed: This is the failure message\n"));
85}
86
87
88ATF_TC(test__crash);
89ATF_TC_HEAD(test__crash, tc) { setup(tc, true); }
90ATF_TC_BODY(test__crash, tc)
91{
92    char* helpers = helpers_path(tc);
93    check(EXIT_FAILURE, "", "save:crash.err",
94          "test", helpers, "signal", "test-result", NULL);
95    free(helpers);
96
97    ATF_REQUIRE(atf_utils_compare_file("test-result",
98        "broken: Premature exit; test case received signal 6 (core dumped)\n"));
99
100    ATF_REQUIRE(atf_utils_grep_file("About to die due to SIGABRT!",
101                                    "crash.err"));
102    ATF_REQUIRE(atf_utils_grep_file("attempting to gather stack trace",
103                                    "crash.err"));
104}
105
106
107ATF_TC(test__timeout);
108ATF_TC_HEAD(test__timeout, tc) { setup(tc, true); }
109ATF_TC_BODY(test__timeout, tc)
110{
111    char* helpers = helpers_path(tc);
112    check(EXIT_FAILURE, "", "Subprocess timed out; sending KILL signal...\n",
113          "-t1", "test", helpers, "sleep", "test-result", NULL);
114    free(helpers);
115
116    ATF_REQUIRE(atf_utils_compare_file("test-result",
117                                       "broken: Test case body timed out\n"));
118}
119
120
121ATF_TC(test__result_priority);
122ATF_TC_HEAD(test__result_priority, tc) { setup(tc, true); }
123ATF_TC_BODY(test__result_priority, tc)
124{
125    char* helpers = helpers_path(tc);
126    check(EXIT_FAILURE, "Killing cleanup\n", "",
127          "test", "-vhas.cleanup=true", helpers, "body_and_cleanup_fail",
128          "test-result", NULL);
129    free(helpers);
130
131    ATF_REQUIRE(atf_utils_compare_file("test-result", "failed: Body fails\n"));
132}
133
134
135ATF_TC(test__cleanup__ok);
136ATF_TC_HEAD(test__cleanup__ok, tc) { setup(tc, true); }
137ATF_TC_BODY(test__cleanup__ok, tc)
138{
139    char* helpers = helpers_path(tc);
140    check(EXIT_SUCCESS,
141          "Body stdout\nCleanup stdout\n"
142              "Cleanup properly ran in the same directory as the body\n",
143          "Body stderr\nCleanup stderr\n",
144          "test", "-vhas.cleanup=true", helpers, "cleanup_check_work_directory",
145          "test-result", NULL);
146    free(helpers);
147
148    ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
149}
150
151
152ATF_TC(test__cleanup__fail);
153ATF_TC_HEAD(test__cleanup__fail, tc) { setup(tc, true); }
154ATF_TC_BODY(test__cleanup__fail, tc)
155{
156    char* helpers = helpers_path(tc);
157    check(EXIT_FAILURE, "", "",
158          "test", "-vhas.cleanup=true", helpers, "cleanup_fail", "test-result",
159          NULL);
160    free(helpers);
161
162    ATF_REQUIRE(atf_utils_compare_file("test-result", "broken: Test case "
163        "cleanup exited with code 1\n"));
164}
165
166
167ATF_TC(test__cleanup__crash);
168ATF_TC_HEAD(test__cleanup__crash, tc) { setup(tc, true); }
169ATF_TC_BODY(test__cleanup__crash, tc)
170{
171    char* helpers = helpers_path(tc);
172    check(EXIT_FAILURE, "", "save:crash.err",
173          "test", "-vhas.cleanup=true", helpers, "cleanup_signal",
174          "test-result", NULL);
175    free(helpers);
176
177    ATF_REQUIRE(atf_utils_compare_file("test-result",
178        "broken: Test case cleanup received signal 6 (core dumped)\n"));
179
180    ATF_REQUIRE(atf_utils_grep_file("About to die due to SIGABRT!",
181                                    "crash.err"));
182    ATF_REQUIRE(atf_utils_grep_file("attempting to gather stack trace",
183                                    "crash.err"));
184}
185
186
187ATF_TC(test__cleanup__timeout);
188ATF_TC_HEAD(test__cleanup__timeout, tc) { setup(tc, true); }
189ATF_TC_BODY(test__cleanup__timeout, tc)
190{
191    char* helpers = helpers_path(tc);
192    check(EXIT_FAILURE, "", "Subprocess timed out; sending KILL signal...\n",
193          "-t1", "test", "-vhas.cleanup=true", helpers, "cleanup_sleep",
194          "test-result", NULL);
195    free(helpers);
196
197    ATF_REQUIRE(atf_utils_compare_file("test-result", "broken: Test case "
198        "cleanup timed out\n"));
199}
200
201
202ATF_TC(test__config__builtin);
203ATF_TC_HEAD(test__config__builtin, tc) { setup(tc, true); }
204ATF_TC_BODY(test__config__builtin, tc)
205{
206    char* helpers = helpers_path(tc);
207    check(EXIT_SUCCESS, "", "",
208          "test", helpers, "print_config", "test-result", NULL);
209    free(helpers);
210
211    ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
212}
213
214
215ATF_TC(test__config__custom);
216ATF_TC_HEAD(test__config__custom, tc) { setup(tc, true); }
217ATF_TC_BODY(test__config__custom, tc)
218{
219    char* helpers = helpers_path(tc);
220    check(EXIT_SUCCESS,
221          "body my-var1 value1\n"
222          "body v2 a b c foo\n"
223          "cleanup my-var1 value1\n"
224          "cleanup v2 a b c foo\n",
225          "",
226          "test", "-vmy-var1=value1", "-vv2=a b c foo", helpers,
227          "print_config", "test-result", NULL);
228    free(helpers);
229
230    ATF_REQUIRE(atf_utils_compare_file("test-result", "passed\n"));
231}
232
233
234ATF_TC(test__invalid_test_case_name);
235ATF_TC_HEAD(test__invalid_test_case_name, tc) { setup(tc, false); }
236ATF_TC_BODY(test__invalid_test_case_name, tc)
237{
238    char* helpers = helpers_path(tc);
239    check(EXIT_FAILURE, "",  // TODO(jmmv): Should be EXIT_INTERNAL_ERROR.
240          "atf_helpers: ERROR: Unknown test case `foo'\n"
241          "atf_helpers: See atf-test-program(1) for usage details.\n",
242          "test", "-vhas.cleanup=false", helpers, "foo", "test-result", NULL);
243    free(helpers);
244
245    ATF_REQUIRE(atf_utils_compare_file("test-result",
246        "broken: Premature exit; test case exited with code 1\n"));
247}
248
249
250ATF_TC(test__missing_test_program);
251ATF_TC_HEAD(test__missing_test_program, tc) { setup(tc, false); }
252ATF_TC_BODY(test__missing_test_program, tc)
253{
254    check(EXIT_INTERNAL_ERROR, "",
255          "kyua-atf-tester: execvp failed: No such file or directory\n",
256          "test", "./non-existent", "pass", "test-result", NULL);
257
258    ATF_REQUIRE(!atf_utils_file_exists("test-result"));
259}
260
261
262ATF_TP_ADD_TCS(tp)
263{
264    ATF_TP_ADD_TC(tp, top__missing_command);
265    ATF_TP_ADD_TC(tp, top__unknown_command);
266
267    ATF_TP_ADD_TC(tp, list__ok);
268
269    ATF_TP_ADD_TC(tp, test__pass);
270    ATF_TP_ADD_TC(tp, test__fail);
271    ATF_TP_ADD_TC(tp, test__crash);
272    ATF_TP_ADD_TC(tp, test__timeout);
273    ATF_TP_ADD_TC(tp, test__result_priority);
274    ATF_TP_ADD_TC(tp, test__cleanup__ok);
275    ATF_TP_ADD_TC(tp, test__cleanup__fail);
276    ATF_TP_ADD_TC(tp, test__cleanup__crash);
277    ATF_TP_ADD_TC(tp, test__cleanup__timeout);
278    ATF_TP_ADD_TC(tp, test__config__builtin);
279    ATF_TP_ADD_TC(tp, test__config__custom);
280    ATF_TP_ADD_TC(tp, test__missing_test_program);
281    ATF_TP_ADD_TC(tp, test__invalid_test_case_name);
282
283    return atf_no_error();
284}
285