1// Copyright 2010 The Kyua Authors.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9//   notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above copyright
11//   notice, this list of conditions and the following disclaimer in the
12//   documentation and/or other materials provided with the distribution.
13// * Neither the name of Google Inc. nor the names of its contributors
14//   may be used to endorse or promote products derived from this software
15//   without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#include "engine/atf_result.hpp"
30
31extern "C" {
32#include <signal.h>
33}
34
35#include <cstdlib>
36#include <fstream>
37#include <sstream>
38#include <stdexcept>
39
40#include <atf-c++.hpp>
41
42#include "engine/exceptions.hpp"
43#include "model/test_result.hpp"
44#include "utils/format/macros.hpp"
45#include "utils/fs/path.hpp"
46#include "utils/process/status.hpp"
47
48namespace fs = utils::fs;
49namespace process = utils::process;
50
51using utils::none;
52using utils::optional;
53
54
55namespace {
56
57
58/// Performs a test for results::parse() that should succeed.
59///
60/// \param exp_type The expected type of the result.
61/// \param exp_argument The expected argument in the result, if any.
62/// \param exp_reason The expected reason describing the result, if any.
63/// \param text The literal input to parse; can include multiple lines.
64static void
65parse_ok_test(const engine::atf_result::types& exp_type,
66              const optional< int >& exp_argument,
67              const char* exp_reason, const char* text)
68{
69    std::istringstream input(text);
70    const engine::atf_result actual = engine::atf_result::parse(input);
71    ATF_REQUIRE(exp_type == actual.type());
72    ATF_REQUIRE_EQ(exp_argument, actual.argument());
73    if (exp_reason != NULL) {
74        ATF_REQUIRE(actual.reason());
75        ATF_REQUIRE_EQ(exp_reason, actual.reason().get());
76    } else {
77        ATF_REQUIRE(!actual.reason());
78    }
79}
80
81
82/// Wrapper around parse_ok_test to define a test case.
83///
84/// \param name The name of the test case; will be prefixed with
85///     "atf_result__parse__".
86/// \param exp_type The expected type of the result.
87/// \param exp_argument The expected argument in the result, if any.
88/// \param exp_reason The expected reason describing the result, if any.
89/// \param input The literal input to parse.
90#define PARSE_OK(name, exp_type, exp_argument, exp_reason, input) \
91    ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \
92    ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \
93    { \
94        parse_ok_test(exp_type, exp_argument, exp_reason, input); \
95    }
96
97
98/// Performs a test for results::parse() that should fail.
99///
100/// \param reason_regexp The reason to match against the broken reason.
101/// \param text The literal input to parse; can include multiple lines.
102static void
103parse_broken_test(const char* reason_regexp, const char* text)
104{
105    std::istringstream input(text);
106    ATF_REQUIRE_THROW_RE(engine::format_error, reason_regexp,
107                         engine::atf_result::parse(input));
108}
109
110
111/// Wrapper around parse_broken_test to define a test case.
112///
113/// \param name The name of the test case; will be prefixed with
114///    "atf_result__parse__".
115/// \param reason_regexp The reason to match against the broken reason.
116/// \param input The literal input to parse.
117#define PARSE_BROKEN(name, reason_regexp, input) \
118    ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \
119    ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \
120    { \
121        parse_broken_test(reason_regexp, input); \
122    }
123
124
125}  // anonymous namespace
126
127
128PARSE_BROKEN(empty,
129             "Empty.*no new line",
130             "");
131PARSE_BROKEN(no_newline__unknown,
132             "Empty.*no new line",
133             "foo");
134PARSE_BROKEN(no_newline__known,
135             "Empty.*no new line",
136             "passed");
137PARSE_BROKEN(multiline__no_newline,
138             "multiple lines.*foo<<NEWLINE>>bar",
139             "failed: foo\nbar");
140PARSE_BROKEN(multiline__with_newline,
141             "multiple lines.*foo<<NEWLINE>>bar",
142             "failed: foo\nbar\n");
143PARSE_BROKEN(unknown_status__no_reason,
144             "Unknown.*result.*'cba'",
145             "cba\n");
146PARSE_BROKEN(unknown_status__with_reason,
147             "Unknown.*result.*'hgf'",
148             "hgf: foo\n");
149PARSE_BROKEN(missing_reason__no_delim,
150             "failed.*followed by.*reason",
151             "failed\n");
152PARSE_BROKEN(missing_reason__bad_delim,
153             "failed.*followed by.*reason",
154             "failed:\n");
155PARSE_BROKEN(missing_reason__empty,
156             "failed.*followed by.*reason",
157             "failed: \n");
158
159
160PARSE_OK(broken__ok,
161         engine::atf_result::broken, none, "a b c",
162         "broken: a b c\n");
163PARSE_OK(broken__blanks,
164         engine::atf_result::broken, none, "   ",
165         "broken:    \n");
166
167
168PARSE_OK(expected_death__ok,
169         engine::atf_result::expected_death, none, "a b c",
170         "expected_death: a b c\n");
171PARSE_OK(expected_death__blanks,
172         engine::atf_result::expected_death, none, "   ",
173         "expected_death:    \n");
174
175
176PARSE_OK(expected_exit__ok__any,
177         engine::atf_result::expected_exit, none, "any exit code",
178         "expected_exit: any exit code\n");
179PARSE_OK(expected_exit__ok__specific,
180         engine::atf_result::expected_exit, optional< int >(712),
181         "some known exit code",
182         "expected_exit(712): some known exit code\n");
183PARSE_BROKEN(expected_exit__bad_int,
184             "Invalid integer.*45a3",
185             "expected_exit(45a3): this is broken\n");
186
187
188PARSE_OK(expected_failure__ok,
189         engine::atf_result::expected_failure, none, "a b c",
190         "expected_failure: a b c\n");
191PARSE_OK(expected_failure__blanks,
192         engine::atf_result::expected_failure, none, "   ",
193         "expected_failure:    \n");
194
195
196PARSE_OK(expected_signal__ok__any,
197         engine::atf_result::expected_signal, none, "any signal code",
198         "expected_signal: any signal code\n");
199PARSE_OK(expected_signal__ok__specific,
200         engine::atf_result::expected_signal, optional< int >(712),
201         "some known signal code",
202         "expected_signal(712): some known signal code\n");
203PARSE_BROKEN(expected_signal__bad_int,
204             "Invalid integer.*45a3",
205             "expected_signal(45a3): this is broken\n");
206
207
208PARSE_OK(expected_timeout__ok,
209         engine::atf_result::expected_timeout, none, "a b c",
210         "expected_timeout: a b c\n");
211PARSE_OK(expected_timeout__blanks,
212         engine::atf_result::expected_timeout, none, "   ",
213         "expected_timeout:    \n");
214
215
216PARSE_OK(failed__ok,
217         engine::atf_result::failed, none, "a b c",
218         "failed: a b c\n");
219PARSE_OK(failed__blanks,
220         engine::atf_result::failed, none, "   ",
221         "failed:    \n");
222
223
224PARSE_OK(passed__ok,
225         engine::atf_result::passed, none, NULL,
226         "passed\n");
227PARSE_BROKEN(passed__reason,
228             "cannot have a reason",
229             "passed a b c\n");
230
231
232PARSE_OK(skipped__ok,
233         engine::atf_result::skipped, none, "a b c",
234         "skipped: a b c\n");
235PARSE_OK(skipped__blanks,
236         engine::atf_result::skipped, none, "   ",
237         "skipped:    \n");
238
239
240ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__ok);
241ATF_TEST_CASE_BODY(atf_result__load__ok)
242{
243    std::ofstream output("result.txt");
244    ATF_REQUIRE(output);
245    output << "skipped: a b c\n";
246    output.close();
247
248    const engine::atf_result result = engine::atf_result::load(
249        utils::fs::path("result.txt"));
250    ATF_REQUIRE(engine::atf_result::skipped == result.type());
251    ATF_REQUIRE(!result.argument());
252    ATF_REQUIRE(result.reason());
253    ATF_REQUIRE_EQ("a b c", result.reason().get());
254}
255
256
257ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__missing_file);
258ATF_TEST_CASE_BODY(atf_result__load__missing_file)
259{
260    ATF_REQUIRE_THROW_RE(
261        std::runtime_error, "Cannot open",
262        engine::atf_result::load(utils::fs::path("result.txt")));
263}
264
265
266ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__format_error);
267ATF_TEST_CASE_BODY(atf_result__load__format_error)
268{
269    std::ofstream output("abc.txt");
270    ATF_REQUIRE(output);
271    output << "passed: foo\n";
272    output.close();
273
274    ATF_REQUIRE_THROW_RE(engine::format_error, "cannot have a reason",
275                         engine::atf_result::load(utils::fs::path("abc.txt")));
276}
277
278
279ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__broken__ok);
280ATF_TEST_CASE_BODY(atf_result__apply__broken__ok)
281{
282    const engine::atf_result in_result(engine::atf_result::broken,
283                                       "Passthrough");
284    const process::status status = process::status::fake_exited(EXIT_SUCCESS);
285    ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status)));
286}
287
288
289ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__timed_out);
290ATF_TEST_CASE_BODY(atf_result__apply__timed_out)
291{
292    const engine::atf_result timed_out(engine::atf_result::broken,
293                                       "Some arbitrary error");
294    ATF_REQUIRE_EQ(engine::atf_result(engine::atf_result::broken,
295                                      "Test case body timed out"),
296                   timed_out.apply(none));
297}
298
299
300ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_death__ok);
301ATF_TEST_CASE_BODY(atf_result__apply__expected_death__ok)
302{
303    const engine::atf_result in_result(engine::atf_result::expected_death,
304                                       "Passthrough");
305    const process::status status = process::status::fake_signaled(SIGINT, true);
306    ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status)));
307}
308
309
310ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__ok);
311ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__ok)
312{
313    const process::status success = process::status::fake_exited(EXIT_SUCCESS);
314    const process::status failure = process::status::fake_exited(EXIT_FAILURE);
315
316    const engine::atf_result any_code(engine::atf_result::expected_exit, none,
317                                      "The reason");
318    ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(success)));
319    ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(failure)));
320
321    const engine::atf_result a_code(engine::atf_result::expected_exit,
322                            utils::make_optional(EXIT_FAILURE), "The reason");
323    ATF_REQUIRE_EQ(a_code, a_code.apply(utils::make_optional(failure)));
324}
325
326
327ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__failed);
328ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__failed)
329{
330    const process::status success = process::status::fake_exited(EXIT_SUCCESS);
331
332    const engine::atf_result a_code(engine::atf_result::expected_exit,
333                            utils::make_optional(EXIT_FAILURE), "The reason");
334    ATF_REQUIRE_EQ(
335        engine::atf_result(engine::atf_result::failed,
336                           "Test case expected to exit with code 1 but got "
337                           "code 0"),
338        a_code.apply(utils::make_optional(success)));
339}
340
341
342ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__broken);
343ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__broken)
344{
345    const process::status sig3 = process::status::fake_signaled(3, false);
346
347    const engine::atf_result any_code(engine::atf_result::expected_exit, none,
348                                      "The reason");
349    ATF_REQUIRE_EQ(
350        engine::atf_result(engine::atf_result::broken,
351                           "Expected clean exit but received signal 3"),
352        any_code.apply(utils::make_optional(sig3)));
353}
354
355
356ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__ok);
357ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__ok)
358{
359    const process::status status = process::status::fake_exited(EXIT_SUCCESS);
360    const engine::atf_result xfailure(engine::atf_result::expected_failure,
361                                      "The reason");
362    ATF_REQUIRE_EQ(xfailure, xfailure.apply(utils::make_optional(status)));
363}
364
365
366ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__broken);
367ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__broken)
368{
369    const process::status failure = process::status::fake_exited(EXIT_FAILURE);
370    const process::status sig3 = process::status::fake_signaled(3, true);
371    const process::status sig4 = process::status::fake_signaled(4, false);
372
373    const engine::atf_result xfailure(engine::atf_result::expected_failure,
374                                      "The reason");
375    ATF_REQUIRE_EQ(
376        engine::atf_result(engine::atf_result::broken,
377                           "Expected failure should have reported success but "
378                           "exited with code 1"),
379        xfailure.apply(utils::make_optional(failure)));
380    ATF_REQUIRE_EQ(
381        engine::atf_result(engine::atf_result::broken,
382                           "Expected failure should have reported success but "
383                           "received signal 3 (core dumped)"),
384        xfailure.apply(utils::make_optional(sig3)));
385    ATF_REQUIRE_EQ(
386        engine::atf_result(engine::atf_result::broken,
387                           "Expected failure should have reported success but "
388                           "received signal 4"),
389        xfailure.apply(utils::make_optional(sig4)));
390}
391
392
393ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__ok);
394ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__ok)
395{
396    const process::status sig1 = process::status::fake_signaled(1, false);
397    const process::status sig3 = process::status::fake_signaled(3, true);
398
399    const engine::atf_result any_sig(engine::atf_result::expected_signal, none,
400                                     "The reason");
401    ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig1)));
402    ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig3)));
403
404    const engine::atf_result a_sig(engine::atf_result::expected_signal,
405                           utils::make_optional(3), "The reason");
406    ATF_REQUIRE_EQ(a_sig, a_sig.apply(utils::make_optional(sig3)));
407}
408
409
410ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__failed);
411ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__failed)
412{
413    const process::status sig5 = process::status::fake_signaled(5, false);
414
415    const engine::atf_result a_sig(engine::atf_result::expected_signal,
416                           utils::make_optional(4), "The reason");
417    ATF_REQUIRE_EQ(
418        engine::atf_result(engine::atf_result::failed,
419                           "Test case expected to receive signal 4 but got 5"),
420        a_sig.apply(utils::make_optional(sig5)));
421}
422
423
424ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__broken);
425ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__broken)
426{
427    const process::status success = process::status::fake_exited(EXIT_SUCCESS);
428
429    const engine::atf_result any_sig(engine::atf_result::expected_signal, none,
430                                     "The reason");
431    ATF_REQUIRE_EQ(
432        engine::atf_result(engine::atf_result::broken,
433                           "Expected signal but exited with code 0"),
434        any_sig.apply(utils::make_optional(success)));
435}
436
437
438ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__ok);
439ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__ok)
440{
441    const engine::atf_result timeout(engine::atf_result::expected_timeout,
442                                     "The reason");
443    ATF_REQUIRE_EQ(timeout, timeout.apply(none));
444}
445
446
447ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__broken);
448ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__broken)
449{
450    const process::status status = process::status::fake_exited(EXIT_SUCCESS);
451    const engine::atf_result timeout(engine::atf_result::expected_timeout,
452                                     "The reason");
453    ATF_REQUIRE_EQ(
454        engine::atf_result(engine::atf_result::broken,
455                           "Expected timeout but exited with code 0"),
456        timeout.apply(utils::make_optional(status)));
457}
458
459
460ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__ok);
461ATF_TEST_CASE_BODY(atf_result__apply__failed__ok)
462{
463    const process::status status = process::status::fake_exited(EXIT_FAILURE);
464    const engine::atf_result failed(engine::atf_result::failed, "The reason");
465    ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status)));
466}
467
468
469ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__broken);
470ATF_TEST_CASE_BODY(atf_result__apply__failed__broken)
471{
472    const process::status success = process::status::fake_exited(EXIT_SUCCESS);
473    const process::status sig3 = process::status::fake_signaled(3, true);
474    const process::status sig4 = process::status::fake_signaled(4, false);
475
476    const engine::atf_result failed(engine::atf_result::failed, "The reason");
477    ATF_REQUIRE_EQ(
478        engine::atf_result(engine::atf_result::broken,
479                           "Failed test case should have reported failure but "
480                           "exited with code 0"),
481        failed.apply(utils::make_optional(success)));
482    ATF_REQUIRE_EQ(
483        engine::atf_result(engine::atf_result::broken,
484                           "Failed test case should have reported failure but "
485                           "received signal 3 (core dumped)"),
486        failed.apply(utils::make_optional(sig3)));
487    ATF_REQUIRE_EQ(
488        engine::atf_result(engine::atf_result::broken,
489                           "Failed test case should have reported failure but "
490                           "received signal 4"),
491        failed.apply(utils::make_optional(sig4)));
492}
493
494
495ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__ok);
496ATF_TEST_CASE_BODY(atf_result__apply__passed__ok)
497{
498    const process::status status = process::status::fake_exited(EXIT_SUCCESS);
499    const engine::atf_result passed(engine::atf_result::passed);
500    ATF_REQUIRE_EQ(passed, passed.apply(utils::make_optional(status)));
501}
502
503
504ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__broken);
505ATF_TEST_CASE_BODY(atf_result__apply__passed__broken)
506{
507    const process::status failure = process::status::fake_exited(EXIT_FAILURE);
508    const process::status sig3 = process::status::fake_signaled(3, true);
509    const process::status sig4 = process::status::fake_signaled(4, false);
510
511    const engine::atf_result passed(engine::atf_result::passed);
512    ATF_REQUIRE_EQ(
513        engine::atf_result(engine::atf_result::broken,
514                           "Passed test case should have reported success but "
515                           "exited with code 1"),
516        passed.apply(utils::make_optional(failure)));
517    ATF_REQUIRE_EQ(
518        engine::atf_result(engine::atf_result::broken,
519                           "Passed test case should have reported success but "
520                           "received signal 3 (core dumped)"),
521        passed.apply(utils::make_optional(sig3)));
522    ATF_REQUIRE_EQ(
523        engine::atf_result(engine::atf_result::broken,
524                           "Passed test case should have reported success but "
525                           "received signal 4"),
526        passed.apply(utils::make_optional(sig4)));
527}
528
529
530ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__ok);
531ATF_TEST_CASE_BODY(atf_result__apply__skipped__ok)
532{
533    const process::status status = process::status::fake_exited(EXIT_SUCCESS);
534    const engine::atf_result skipped(engine::atf_result::skipped, "The reason");
535    ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status)));
536}
537
538
539ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__broken);
540ATF_TEST_CASE_BODY(atf_result__apply__skipped__broken)
541{
542    const process::status failure = process::status::fake_exited(EXIT_FAILURE);
543    const process::status sig3 = process::status::fake_signaled(3, true);
544    const process::status sig4 = process::status::fake_signaled(4, false);
545
546    const engine::atf_result skipped(engine::atf_result::skipped, "The reason");
547    ATF_REQUIRE_EQ(
548        engine::atf_result(engine::atf_result::broken,
549                           "Skipped test case should have reported success but "
550                           "exited with code 1"),
551        skipped.apply(utils::make_optional(failure)));
552    ATF_REQUIRE_EQ(
553        engine::atf_result(engine::atf_result::broken,
554                           "Skipped test case should have reported success but "
555                           "received signal 3 (core dumped)"),
556        skipped.apply(utils::make_optional(sig3)));
557    ATF_REQUIRE_EQ(
558        engine::atf_result(engine::atf_result::broken,
559                           "Skipped test case should have reported success but "
560                           "received signal 4"),
561        skipped.apply(utils::make_optional(sig4)));
562}
563
564
565ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__broken);
566ATF_TEST_CASE_BODY(atf_result__externalize__broken)
567{
568    const engine::atf_result raw(engine::atf_result::broken, "The reason");
569    const model::test_result expected(model::test_result_broken,
570                                      "The reason");
571    ATF_REQUIRE_EQ(expected, raw.externalize());
572}
573
574
575ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_death);
576ATF_TEST_CASE_BODY(atf_result__externalize__expected_death)
577{
578    const engine::atf_result raw(engine::atf_result::expected_death,
579                                 "The reason");
580    const model::test_result expected(model::test_result_expected_failure,
581                                      "The reason");
582    ATF_REQUIRE_EQ(expected, raw.externalize());
583}
584
585
586ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_exit);
587ATF_TEST_CASE_BODY(atf_result__externalize__expected_exit)
588{
589    const engine::atf_result raw(engine::atf_result::expected_exit,
590                                 "The reason");
591    const model::test_result expected(model::test_result_expected_failure,
592                                      "The reason");
593    ATF_REQUIRE_EQ(expected, raw.externalize());
594}
595
596
597ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_failure);
598ATF_TEST_CASE_BODY(atf_result__externalize__expected_failure)
599{
600    const engine::atf_result raw(engine::atf_result::expected_failure,
601                                 "The reason");
602    const model::test_result expected(model::test_result_expected_failure,
603                                      "The reason");
604    ATF_REQUIRE_EQ(expected, raw.externalize());
605}
606
607
608ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_signal);
609ATF_TEST_CASE_BODY(atf_result__externalize__expected_signal)
610{
611    const engine::atf_result raw(engine::atf_result::expected_signal,
612                                 "The reason");
613    const model::test_result expected(model::test_result_expected_failure,
614                                      "The reason");
615    ATF_REQUIRE_EQ(expected, raw.externalize());
616}
617
618
619ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_timeout);
620ATF_TEST_CASE_BODY(atf_result__externalize__expected_timeout)
621{
622    const engine::atf_result raw(engine::atf_result::expected_timeout,
623                                 "The reason");
624    const model::test_result expected(model::test_result_expected_failure,
625                                      "The reason");
626    ATF_REQUIRE_EQ(expected, raw.externalize());
627}
628
629
630ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__failed);
631ATF_TEST_CASE_BODY(atf_result__externalize__failed)
632{
633    const engine::atf_result raw(engine::atf_result::failed, "The reason");
634    const model::test_result expected(model::test_result_failed,
635                                      "The reason");
636    ATF_REQUIRE(expected == raw.externalize());
637}
638
639
640ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__passed);
641ATF_TEST_CASE_BODY(atf_result__externalize__passed)
642{
643    const engine::atf_result raw(engine::atf_result::passed);
644    const model::test_result expected(model::test_result_passed);
645    ATF_REQUIRE_EQ(expected, raw.externalize());
646}
647
648
649ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__skipped);
650ATF_TEST_CASE_BODY(atf_result__externalize__skipped)
651{
652    const engine::atf_result raw(engine::atf_result::skipped, "The reason");
653    const model::test_result expected(model::test_result_skipped,
654                                      "The reason");
655    ATF_REQUIRE_EQ(expected, raw.externalize());
656}
657
658
659ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__missing_file);
660ATF_TEST_CASE_BODY(calculate_atf_result__missing_file)
661{
662    using process::status;
663
664    const status body_status = status::fake_exited(EXIT_SUCCESS);
665    const model::test_result expected(
666        model::test_result_broken,
667        "Premature exit; test case exited with code 0");
668    ATF_REQUIRE_EQ(expected, engine::calculate_atf_result(
669        utils::make_optional(body_status), fs::path("foo")));
670}
671
672
673ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__bad_file);
674ATF_TEST_CASE_BODY(calculate_atf_result__bad_file)
675{
676    using process::status;
677
678    const status body_status = status::fake_exited(EXIT_SUCCESS);
679    atf::utils::create_file("foo", "invalid\n");
680    const model::test_result expected(model::test_result_broken,
681                                      "Unknown test result 'invalid'");
682    ATF_REQUIRE_EQ(expected, engine::calculate_atf_result(
683        utils::make_optional(body_status), fs::path("foo")));
684}
685
686
687ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_ok);
688ATF_TEST_CASE_BODY(calculate_atf_result__body_ok)
689{
690    using process::status;
691
692    atf::utils::create_file("result.txt", "skipped: Something\n");
693    const status body_status = status::fake_exited(EXIT_SUCCESS);
694    ATF_REQUIRE_EQ(
695        model::test_result(model::test_result_skipped, "Something"),
696        engine::calculate_atf_result(utils::make_optional(body_status),
697                                     fs::path("result.txt")));
698}
699
700
701ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_bad);
702ATF_TEST_CASE_BODY(calculate_atf_result__body_bad)
703{
704    using process::status;
705
706    atf::utils::create_file("result.txt", "skipped: Something\n");
707    const status body_status = status::fake_exited(EXIT_FAILURE);
708    ATF_REQUIRE_EQ(
709        model::test_result(model::test_result_broken, "Skipped test case "
710                           "should have reported success but exited with "
711                           "code 1"),
712        engine::calculate_atf_result(utils::make_optional(body_status),
713                                     fs::path("result.txt")));
714}
715
716
717ATF_INIT_TEST_CASES(tcs)
718{
719    ATF_ADD_TEST_CASE(tcs, atf_result__parse__empty);
720    ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__unknown);
721    ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__known);
722    ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__no_newline);
723    ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__with_newline);
724    ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__no_reason);
725    ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__with_reason);
726    ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__no_delim);
727    ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__bad_delim);
728    ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__empty);
729    ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__ok);
730    ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__blanks);
731    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__ok);
732    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__blanks);
733    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__any);
734    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__specific);
735    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__bad_int);
736    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__ok);
737    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__blanks);
738    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__any);
739    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__specific);
740    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__bad_int);
741    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__ok);
742    ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__blanks);
743    ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__ok);
744    ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__blanks);
745    ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__ok);
746    ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__reason);
747    ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__ok);
748    ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__blanks);
749
750    ATF_ADD_TEST_CASE(tcs, atf_result__load__ok);
751    ATF_ADD_TEST_CASE(tcs, atf_result__load__missing_file);
752    ATF_ADD_TEST_CASE(tcs, atf_result__load__format_error);
753
754    ATF_ADD_TEST_CASE(tcs, atf_result__apply__broken__ok);
755    ATF_ADD_TEST_CASE(tcs, atf_result__apply__timed_out);
756    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_death__ok);
757    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__ok);
758    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__failed);
759    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__broken);
760    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__ok);
761    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__broken);
762    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__ok);
763    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__failed);
764    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__broken);
765    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__ok);
766    ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__broken);
767    ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__ok);
768    ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__broken);
769    ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__ok);
770    ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__broken);
771    ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__ok);
772    ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__broken);
773
774    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__broken);
775    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_death);
776    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_exit);
777    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_failure);
778    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_signal);
779    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_timeout);
780    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__failed);
781    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__passed);
782    ATF_ADD_TEST_CASE(tcs, atf_result__externalize__skipped);
783
784    ATF_ADD_TEST_CASE(tcs, calculate_atf_result__missing_file);
785    ATF_ADD_TEST_CASE(tcs, calculate_atf_result__bad_file);
786    ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_ok);
787    ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_bad);
788}
789