1// Copyright 2010 The Kyua Authors.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9//   notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above copyright
11//   notice, this list of conditions and the following disclaimer in the
12//   documentation and/or other materials provided with the distribution.
13// * Neither the name of Google Inc. nor the names of its contributors
14//   may be used to endorse or promote products derived from this software
15//   without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29#include "engine/atf_result.hpp"
30
31#include <cstdlib>
32#include <fstream>
33#include <utility>
34
35#include "engine/exceptions.hpp"
36#include "model/test_result.hpp"
37#include "utils/fs/path.hpp"
38#include "utils/format/macros.hpp"
39#include "utils/optional.ipp"
40#include "utils/process/status.hpp"
41#include "utils/sanity.hpp"
42#include "utils/text/exceptions.hpp"
43#include "utils/text/operations.ipp"
44
45namespace fs = utils::fs;
46namespace process = utils::process;
47namespace text = utils::text;
48
49using utils::none;
50using utils::optional;
51
52
53namespace {
54
55
56/// Reads a file and flattens its lines.
57///
58/// The main purpose of this function is to simplify the parsing of a file
59/// containing the result of a test.  Therefore, the return value carries
60/// several assumptions.
61///
62/// \param input The stream to read from.
63///
64/// \return A pair (line count, contents) detailing how many lines where read
65/// and their contents.  If the file contains a single line with no newline
66/// character, the line count is 0.  If the file includes more than one line,
67/// the lines are merged together and separated by the magic string
68/// '&lt;&lt;NEWLINE&gt;&gt;'.
69static std::pair< size_t, std::string >
70read_lines(std::istream& input)
71{
72    std::pair< size_t, std::string > ret = std::make_pair(0, "");
73
74    do {
75        std::string line;
76        std::getline(input, line);
77        if (input.eof() && !line.empty()) {
78            if (ret.first == 0)
79                ret.second = line;
80            else {
81                ret.second += "<<NEWLINE>>" + line;
82                ret.first++;
83            }
84        } else if (input.good()) {
85            if (ret.first == 0)
86                ret.second = line;
87            else
88                ret.second += "<<NEWLINE>>" + line;
89            ret.first++;
90        }
91    } while (input.good());
92
93    return ret;
94}
95
96
97/// Parses a test result that does not accept a reason.
98///
99/// \param status The result status name.
100/// \param rest The rest of the line after the status name.
101///
102/// \return An object representing the test result.
103///
104/// \throw format_error If the result is invalid (i.e. rest is invalid).
105///
106/// \pre status must be "passed".
107static engine::atf_result
108parse_without_reason(const std::string& status, const std::string& rest)
109{
110    if (!rest.empty())
111        throw engine::format_error(F("%s cannot have a reason") % status);
112    PRE(status == "passed");
113    return engine::atf_result(engine::atf_result::passed);
114}
115
116
117/// Parses a test result that needs a reason.
118///
119/// \param status The result status name.
120/// \param rest The rest of the line after the status name.
121///
122/// \return An object representing the test result.
123///
124/// \throw format_error If the result is invalid (i.e. rest is invalid).
125///
126/// \pre status must be one of "broken", "expected_death", "expected_failure",
127/// "expected_timeout", "failed" or "skipped".
128static engine::atf_result
129parse_with_reason(const std::string& status, const std::string& rest)
130{
131    using engine::atf_result;
132
133    if (rest.length() < 3 || rest.substr(0, 2) != ": ")
134        throw engine::format_error(F("%s must be followed by ': <reason>'") %
135                                   status);
136    const std::string reason = rest.substr(2);
137    INV(!reason.empty());
138
139    if (status == "broken")
140        return atf_result(atf_result::broken, reason);
141    else if (status == "expected_death")
142        return atf_result(atf_result::expected_death, reason);
143    else if (status == "expected_failure")
144        return atf_result(atf_result::expected_failure, reason);
145    else if (status == "expected_timeout")
146        return atf_result(atf_result::expected_timeout, reason);
147    else if (status == "failed")
148        return atf_result(atf_result::failed, reason);
149    else if (status == "skipped")
150        return atf_result(atf_result::skipped, reason);
151    else
152        PRE_MSG(false, "Unexpected status");
153}
154
155
156/// Converts a string to an integer.
157///
158/// \param str The string containing the integer to convert.
159///
160/// \return The converted integer; none if the parsing fails.
161static optional< int >
162parse_int(const std::string& str)
163{
164    try {
165        return utils::make_optional(text::to_type< int >(str));
166    } catch (const text::value_error& e) {
167        return none;
168    }
169}
170
171
172/// Parses a test result that needs a reason and accepts an optional integer.
173///
174/// \param status The result status name.
175/// \param rest The rest of the line after the status name.
176///
177/// \return The parsed test result if the data is valid, or a broken result if
178/// the parsing failed.
179///
180/// \pre status must be one of "expected_exit" or "expected_signal".
181static engine::atf_result
182parse_with_reason_and_arg(const std::string& status, const std::string& rest)
183{
184    using engine::atf_result;
185
186    std::string::size_type delim = rest.find_first_of(":(");
187    if (delim == std::string::npos)
188        throw engine::format_error(F("Invalid format for '%s' test case "
189                                     "result; must be followed by '[(num)]: "
190                                     "<reason>' but found '%s'") %
191                                   status % rest);
192
193    optional< int > arg;
194    if (rest[delim] == '(') {
195        const std::string::size_type delim2 = rest.find("):", delim);
196        if (delim == std::string::npos)
197            throw engine::format_error(F("Mismatched '(' in %s") % rest);
198
199        const std::string argstr = rest.substr(delim + 1, delim2 - delim - 1);
200        arg = parse_int(argstr);
201        if (!arg)
202            throw engine::format_error(F("Invalid integer argument '%s' to "
203                                         "'%s' test case result") %
204                                       argstr % status);
205        delim = delim2 + 1;
206    }
207
208    const std::string reason = rest.substr(delim + 2);
209
210    if (status == "expected_exit")
211        return atf_result(atf_result::expected_exit, arg, reason);
212    else if (status == "expected_signal")
213        return atf_result(atf_result::expected_signal, arg, reason);
214    else
215        PRE_MSG(false, "Unexpected status");
216}
217
218
219/// Formats the termination status of a process to be used with validate_result.
220///
221/// \param status The status to format.
222///
223/// \return A string describing the status.
224static std::string
225format_status(const process::status& status)
226{
227    if (status.exited())
228        return F("exited with code %s") % status.exitstatus();
229    else if (status.signaled())
230        return F("received signal %s%s") % status.termsig() %
231            (status.coredump() ? " (core dumped)" : "");
232    else
233        return F("terminated in an unknown manner");
234}
235
236
237}  // anonymous namespace
238
239
240/// Constructs a raw result with a type.
241///
242/// The reason and the argument are left uninitialized.
243///
244/// \param type_ The type of the result.
245engine::atf_result::atf_result(const types type_) :
246    _type(type_)
247{
248}
249
250
251/// Constructs a raw result with a type and a reason.
252///
253/// The argument is left uninitialized.
254///
255/// \param type_ The type of the result.
256/// \param reason_ The reason for the result.
257engine::atf_result::atf_result(const types type_, const std::string& reason_) :
258    _type(type_), _reason(reason_)
259{
260}
261
262
263/// Constructs a raw result with a type, an optional argument and a reason.
264///
265/// \param type_ The type of the result.
266/// \param argument_ The optional argument for the result.
267/// \param reason_ The reason for the result.
268engine::atf_result::atf_result(const types type_,
269                               const utils::optional< int >& argument_,
270                               const std::string& reason_) :
271    _type(type_), _argument(argument_), _reason(reason_)
272{
273}
274
275
276/// Parses an input stream to extract a test result.
277///
278/// If the parsing fails for any reason, the test result is 'broken' and it
279/// contains the reason for the parsing failure.  Test cases that report results
280/// in an inconsistent state cannot be trusted (e.g. the test program code may
281/// have a bug), and thus why they are reported as broken instead of just failed
282/// (which is a legitimate result for a test case).
283///
284/// \param input The stream to read from.
285///
286/// \return A generic representation of the result of the test case.
287///
288/// \throw format_error If the input is invalid.
289engine::atf_result
290engine::atf_result::parse(std::istream& input)
291{
292    const std::pair< size_t, std::string > data = read_lines(input);
293    if (data.first == 0)
294        throw format_error("Empty test result or no new line");
295    else if (data.first > 1)
296        throw format_error("Test result contains multiple lines: " +
297                           data.second);
298    else {
299        const std::string::size_type delim = data.second.find_first_not_of(
300            "abcdefghijklmnopqrstuvwxyz_");
301        const std::string status = data.second.substr(0, delim);
302        const std::string rest = data.second.substr(status.length());
303
304        if (status == "broken")
305            return parse_with_reason(status, rest);
306        else if (status == "expected_death")
307            return parse_with_reason(status, rest);
308        else if (status == "expected_exit")
309            return parse_with_reason_and_arg(status, rest);
310        else if (status == "expected_failure")
311            return parse_with_reason(status, rest);
312        else if (status == "expected_signal")
313            return parse_with_reason_and_arg(status, rest);
314        else if (status == "expected_timeout")
315            return parse_with_reason(status, rest);
316        else if (status == "failed")
317            return parse_with_reason(status, rest);
318        else if (status == "passed")
319            return parse_without_reason(status, rest);
320        else if (status == "skipped")
321            return parse_with_reason(status, rest);
322        else
323            throw format_error(F("Unknown test result '%s'") % status);
324    }
325}
326
327
328/// Loads a test case result from a file.
329///
330/// \param file The file to parse.
331///
332/// \return The parsed test case result if all goes well.
333///
334/// \throw std::runtime_error If the file does not exist.
335/// \throw engine::format_error If the contents of the file are bogus.
336engine::atf_result
337engine::atf_result::load(const fs::path& file)
338{
339    std::ifstream input(file.c_str());
340    if (!input)
341        throw std::runtime_error("Cannot open results file");
342    else
343        return parse(input);
344}
345
346
347/// Gets the type of the result.
348///
349/// \return A result type.
350engine::atf_result::types
351engine::atf_result::type(void) const
352{
353    return _type;
354}
355
356
357/// Gets the optional argument of the result.
358///
359/// \return The argument of the result if present; none otherwise.
360const optional< int >&
361engine::atf_result::argument(void) const
362{
363    return _argument;
364}
365
366
367/// Gets the optional reason of the result.
368///
369/// \return The reason of the result if present; none otherwise.
370const optional< std::string >&
371engine::atf_result::reason(void) const
372{
373    return _reason;
374}
375
376
377/// Checks whether the result should be reported as good or not.
378///
379/// \return True if the result can be considered "good", false otherwise.
380bool
381engine::atf_result::good(void) const
382{
383    switch (_type) {
384    case atf_result::expected_death:
385    case atf_result::expected_exit:
386    case atf_result::expected_failure:
387    case atf_result::expected_signal:
388    case atf_result::expected_timeout:
389    case atf_result::passed:
390    case atf_result::skipped:
391        return true;
392
393    case atf_result::broken:
394    case atf_result::failed:
395        return false;
396
397    default:
398        UNREACHABLE;
399    }
400}
401
402
403/// Reinterprets a raw result based on the termination status of the test case.
404///
405/// This reinterpretation ensures that the termination conditions of the program
406/// match what is expected of the paticular result reported by the test program.
407/// If such conditions do not match, the test program is considered bogus and is
408/// thus reported as broken.
409///
410/// This is just a helper function for calculate_result(); the real result of
411/// the test case cannot be inferred from apply() only.
412///
413/// \param status The exit status of the test program, or none if the test
414/// program timed out.
415///
416/// \result The adjusted result.  The original result is transformed into broken
417/// if the exit status of the program does not match our expectations.
418engine::atf_result
419engine::atf_result::apply(const optional< process::status >& status)
420    const
421{
422    if (!status) {
423        if (_type != atf_result::expected_timeout)
424            return atf_result(atf_result::broken, "Test case body timed out");
425        else
426            return *this;
427    }
428
429    INV(status);
430    switch (_type) {
431    case atf_result::broken:
432        return *this;
433
434    case atf_result::expected_death:
435        return *this;
436
437    case atf_result::expected_exit:
438        if (status.get().exited()) {
439            if (_argument) {
440                if (_argument.get() == status.get().exitstatus())
441                    return *this;
442                else
443                    return atf_result(
444                        atf_result::failed,
445                        F("Test case expected to exit with code %s but got "
446                          "code %s") %
447                        _argument.get() % status.get().exitstatus());
448            } else
449                return *this;
450        } else
451              return atf_result(atf_result::broken, "Expected clean exit but " +
452                                format_status(status.get()));
453
454    case atf_result::expected_failure:
455        if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS)
456            return *this;
457        else
458            return atf_result(atf_result::broken, "Expected failure should "
459                              "have reported success but " +
460                              format_status(status.get()));
461
462    case atf_result::expected_signal:
463        if (status.get().signaled()) {
464            if (_argument) {
465                if (_argument.get() == status.get().termsig())
466                    return *this;
467                else
468                    return atf_result(
469                        atf_result::failed,
470                        F("Test case expected to receive signal %s but "
471                          "got %s") %
472                        _argument.get() % status.get().termsig());
473            } else
474                return *this;
475        } else
476            return atf_result(atf_result::broken, "Expected signal but " +
477                              format_status(status.get()));
478
479    case atf_result::expected_timeout:
480        return atf_result(atf_result::broken, "Expected timeout but " +
481                          format_status(status.get()));
482
483    case atf_result::failed:
484        if (status.get().exited() && status.get().exitstatus() == EXIT_FAILURE)
485            return *this;
486        else
487            return atf_result(atf_result::broken, "Failed test case should "
488                              "have reported failure but " +
489                              format_status(status.get()));
490
491    case atf_result::passed:
492        if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS)
493            return *this;
494        else
495            return atf_result(atf_result::broken, "Passed test case should "
496                              "have reported success but " +
497                              format_status(status.get()));
498
499    case atf_result::skipped:
500        if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS)
501            return *this;
502        else
503            return atf_result(atf_result::broken, "Skipped test case should "
504                              "have reported success but " +
505                              format_status(status.get()));
506    }
507
508    UNREACHABLE;
509}
510
511
512/// Converts an internal result to the interface-agnostic representation.
513///
514/// \return A generic result instance representing this result.
515model::test_result
516engine::atf_result::externalize(void) const
517{
518    switch (_type) {
519    case atf_result::broken:
520        return model::test_result(model::test_result_broken, _reason.get());
521
522    case atf_result::expected_death:
523    case atf_result::expected_exit:
524    case atf_result::expected_failure:
525    case atf_result::expected_signal:
526    case atf_result::expected_timeout:
527        return model::test_result(model::test_result_expected_failure,
528                                  _reason.get());
529
530    case atf_result::failed:
531        return model::test_result(model::test_result_failed, _reason.get());
532
533    case atf_result::passed:
534        return model::test_result(model::test_result_passed);
535
536    case atf_result::skipped:
537        return model::test_result(model::test_result_skipped, _reason.get());
538
539    default:
540        UNREACHABLE;
541    }
542}
543
544
545/// Compares two raw results for equality.
546///
547/// \param other The result to compare to.
548///
549/// \return True if the two raw results are equal; false otherwise.
550bool
551engine::atf_result::operator==(const atf_result& other) const
552{
553    return _type == other._type && _argument == other._argument &&
554        _reason == other._reason;
555}
556
557
558/// Compares two raw results for inequality.
559///
560/// \param other The result to compare to.
561///
562/// \return True if the two raw results are different; false otherwise.
563bool
564engine::atf_result::operator!=(const atf_result& other) const
565{
566    return !(*this == other);
567}
568
569
570/// Injects the object into a stream.
571///
572/// \param output The stream into which to inject the object.
573/// \param object The object to format.
574///
575/// \return The output stream.
576std::ostream&
577engine::operator<<(std::ostream& output, const atf_result& object)
578{
579    std::string result_name;
580    switch (object.type()) {
581    case atf_result::broken: result_name = "broken"; break;
582    case atf_result::expected_death: result_name = "expected_death"; break;
583    case atf_result::expected_exit: result_name = "expected_exit"; break;
584    case atf_result::expected_failure: result_name = "expected_failure"; break;
585    case atf_result::expected_signal: result_name = "expected_signal"; break;
586    case atf_result::expected_timeout: result_name = "expected_timeout"; break;
587    case atf_result::failed: result_name = "failed"; break;
588    case atf_result::passed: result_name = "passed"; break;
589    case atf_result::skipped: result_name = "skipped"; break;
590    }
591
592    const optional< int >& argument = object.argument();
593
594    const optional< std::string >& reason = object.reason();
595
596    output << F("model::test_result{type=%s, argument=%s, reason=%s}")
597        % text::quote(result_name, '\'')
598        % (argument ? (F("%s") % argument.get()).str() : "none")
599        % (reason ? text::quote(reason.get(), '\'') : "none");
600
601    return output;
602}
603
604
605/// Calculates the user-visible result of a test case.
606///
607/// This function needs to perform magic to ensure that what the test case
608/// reports as its result is what the user should really see: i.e. it adjusts
609/// the reported status of the test to the exit conditions of its body and
610/// cleanup parts.
611///
612/// \param body_status The termination status of the process that executed
613///     the body of the test.  None if the body timed out.
614/// \param results_file The path to the results file that the test case body is
615///     supposed to have created.
616///
617/// \return The calculated test case result.
618model::test_result
619engine::calculate_atf_result(const optional< process::status >& body_status,
620                             const fs::path& results_file)
621{
622    using engine::atf_result;
623
624    atf_result result(atf_result::broken, "Unknown result");
625    try {
626        result = atf_result::load(results_file);
627    } catch (const engine::format_error& error) {
628        result = atf_result(atf_result::broken, error.what());
629    } catch (const std::runtime_error& error) {
630        if (body_status)
631            result = atf_result(
632                atf_result::broken, F("Premature exit; test case %s") %
633                format_status(body_status.get()));
634        else {
635            // The test case timed out.  apply() handles this case later.
636        }
637    }
638
639    result = result.apply(body_status);
640
641    return result.externalize();
642}
643