1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2008 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <sys/types.h>
32#include <sys/wait.h>
33
34#include <limits.h>
35#include <signal.h>
36#include <unistd.h>
37}
38
39#include <cerrno>
40#include <cstdlib>
41#include <cstring>
42#include <fstream>
43#include <ios>
44#include <iostream>
45#include <iterator>
46#include <list>
47#include <memory>
48#include <utility>
49
50#include "atf-c++/check.hpp"
51#include "atf-c++/config.hpp"
52
53#include "atf-c++/detail/application.hpp"
54#include "atf-c++/detail/auto_array.hpp"
55#include "atf-c++/detail/exceptions.hpp"
56#include "atf-c++/detail/fs.hpp"
57#include "atf-c++/detail/process.hpp"
58#include "atf-c++/detail/sanity.hpp"
59#include "atf-c++/detail/text.hpp"
60
61// ------------------------------------------------------------------------
62// Auxiliary functions.
63// ------------------------------------------------------------------------
64
65namespace {
66
67enum status_check_t {
68    sc_exit,
69    sc_ignore,
70    sc_signal,
71};
72
73struct status_check {
74    status_check_t type;
75    bool negated;
76    int value;
77
78    status_check(const status_check_t& p_type, const bool p_negated,
79                 const int p_value) :
80        type(p_type),
81        negated(p_negated),
82        value(p_value)
83    {
84    }
85};
86
87enum output_check_t {
88    oc_ignore,
89    oc_inline,
90    oc_file,
91    oc_empty,
92    oc_match,
93    oc_save
94};
95
96struct output_check {
97    output_check_t type;
98    bool negated;
99    std::string value;
100
101    output_check(const output_check_t& p_type, const bool p_negated,
102                 const std::string& p_value) :
103        type(p_type),
104        negated(p_negated),
105        value(p_value)
106    {
107    }
108};
109
110class temp_file : public std::ostream {
111    std::auto_ptr< atf::fs::path > m_path;
112    int m_fd;
113
114public:
115    temp_file(const atf::fs::path& p) :
116        std::ostream(NULL),
117        m_fd(-1)
118    {
119        atf::auto_array< char > buf(new char[p.str().length() + 1]);
120        std::strcpy(buf.get(), p.c_str());
121
122        m_fd = ::mkstemp(buf.get());
123        if (m_fd == -1)
124            throw atf::system_error("atf_check::temp_file::temp_file(" +
125                                    p.str() + ")", "mkstemp(3) failed",
126                                    errno);
127
128        m_path.reset(new atf::fs::path(buf.get()));
129    }
130
131    ~temp_file(void)
132    {
133        close();
134        try {
135            remove(*m_path);
136        } catch (const atf::system_error&) {
137            // Ignore deletion errors.
138        }
139    }
140
141    const atf::fs::path&
142    get_path(void) const
143    {
144        return *m_path;
145    }
146
147    void
148    write(const std::string& text)
149    {
150        if (::write(m_fd, text.c_str(), text.size()) == -1)
151            throw atf::system_error("atf_check", "write(2) failed", errno);
152    }
153
154    void
155    close(void)
156    {
157        if (m_fd != -1) {
158            flush();
159            ::close(m_fd);
160            m_fd = -1;
161        }
162    }
163};
164
165} // anonymous namespace
166
167static int
168parse_exit_code(const std::string& str)
169{
170    try {
171        const int value = atf::text::to_type< int >(str);
172        if (value < 0 || value > 255)
173            throw std::runtime_error("Unused reason");
174        return value;
175    } catch (const std::runtime_error&) {
176        throw atf::application::usage_error("Invalid exit code for -s option; "
177            "must be an integer in range 0-255");
178    }
179}
180
181static struct name_number {
182    const char *name;
183    int signo;
184} signal_names_to_numbers[] = {
185    { "hup", SIGHUP },
186    { "int", SIGINT },
187    { "quit", SIGQUIT },
188    { "trap", SIGTRAP },
189    { "abrt", SIGABRT },
190    { "kill", SIGKILL },
191    { "segv", SIGSEGV },
192    { "pipe", SIGPIPE },
193    { "alrm", SIGALRM },
194    { "term", SIGTERM },
195    { "usr1", SIGUSR1 },
196    { "usr2", SIGUSR2 },
197    { NULL, INT_MIN },
198};
199
200static int
201signal_name_to_number(const std::string& str)
202{
203    struct name_number* iter = signal_names_to_numbers;
204    int signo = INT_MIN;
205    while (signo == INT_MIN && iter->name != NULL) {
206        if (str == iter->name || str == std::string("sig") + iter->name)
207            signo = iter->signo;
208        else
209            iter++;
210    }
211    return signo;
212}
213
214static int
215parse_signal(const std::string& str)
216{
217    const int signo = signal_name_to_number(str);
218    if (signo == INT_MIN) {
219        try {
220            return atf::text::to_type< int >(str);
221        } catch (std::runtime_error) {
222            throw atf::application::usage_error("Invalid signal name or number "
223                "in -s option");
224        }
225    }
226    INV(signo != INT_MIN);
227    return signo;
228}
229
230static status_check
231parse_status_check_arg(const std::string& arg)
232{
233    const std::string::size_type delimiter = arg.find(':');
234    bool negated = (arg.compare(0, 4, "not-") == 0);
235    const std::string action_str = arg.substr(0, delimiter);
236    const std::string action = negated ? action_str.substr(4) : action_str;
237    const std::string value_str = (
238        delimiter == std::string::npos ? "" : arg.substr(delimiter + 1));
239    int value;
240
241    status_check_t type;
242    if (action == "eq") {
243        // Deprecated; use exit instead.  TODO: Remove after 0.10.
244        type = sc_exit;
245        if (negated)
246            throw atf::application::usage_error("Cannot negate eq checker");
247        negated = false;
248        value = parse_exit_code(value_str);
249    } else if (action == "exit") {
250        type = sc_exit;
251        if (value_str.empty())
252            value = INT_MIN;
253        else
254            value = parse_exit_code(value_str);
255    } else if (action == "ignore") {
256        if (negated)
257            throw atf::application::usage_error("Cannot negate ignore checker");
258        type = sc_ignore;
259        value = INT_MIN;
260    } else if (action == "ne") {
261        // Deprecated; use not-exit instead.  TODO: Remove after 0.10.
262        type = sc_exit;
263        if (negated)
264            throw atf::application::usage_error("Cannot negate ne checker");
265        negated = true;
266        value = parse_exit_code(value_str);
267    } else if (action == "signal") {
268        type = sc_signal;
269        if (value_str.empty())
270            value = INT_MIN;
271        else
272            value = parse_signal(value_str);
273    } else
274        throw atf::application::usage_error("Invalid status checker");
275
276    return status_check(type, negated, value);
277}
278
279static
280output_check
281parse_output_check_arg(const std::string& arg)
282{
283    const std::string::size_type delimiter = arg.find(':');
284    const bool negated = (arg.compare(0, 4, "not-") == 0);
285    const std::string action_str = arg.substr(0, delimiter);
286    const std::string action = negated ? action_str.substr(4) : action_str;
287
288    output_check_t type;
289    if (action == "empty")
290        type = oc_empty;
291    else if (action == "file")
292        type = oc_file;
293    else if (action == "ignore") {
294        if (negated)
295            throw atf::application::usage_error("Cannot negate ignore checker");
296        type = oc_ignore;
297    } else if (action == "inline")
298        type = oc_inline;
299    else if (action == "match")
300        type = oc_match;
301    else if (action == "save") {
302        if (negated)
303            throw atf::application::usage_error("Cannot negate save checker");
304        type = oc_save;
305    } else
306        throw atf::application::usage_error("Invalid output checker");
307
308    return output_check(type, negated, arg.substr(delimiter + 1));
309}
310
311static
312std::string
313flatten_argv(char* const* argv)
314{
315    std::string cmdline;
316
317    char* const* arg = &argv[0];
318    while (*arg != NULL) {
319        if (arg != &argv[0])
320            cmdline += ' ';
321
322        cmdline += *arg;
323
324        arg++;
325    }
326
327    return cmdline;
328}
329
330static
331std::auto_ptr< atf::check::check_result >
332execute(const char* const* argv)
333{
334    // TODO: This should go to stderr... but fixing it now may be hard as test
335    // cases out there might be relying on stderr being silent.
336    std::cout << "Executing command [ ";
337    for (int i = 0; argv[i] != NULL; ++i)
338        std::cout << argv[i] << " ";
339    std::cout << "]\n";
340    std::cout.flush();
341
342    atf::process::argv_array argva(argv);
343    return atf::check::exec(argva);
344}
345
346static
347std::auto_ptr< atf::check::check_result >
348execute_with_shell(char* const* argv)
349{
350    const std::string cmd = flatten_argv(argv);
351
352    const char* sh_argv[4];
353    sh_argv[0] = atf::config::get("atf_shell").c_str();
354    sh_argv[1] = "-c";
355    sh_argv[2] = cmd.c_str();
356    sh_argv[3] = NULL;
357    return execute(sh_argv);
358}
359
360static
361void
362cat_file(const atf::fs::path& path)
363{
364    std::ifstream stream(path.c_str());
365    if (!stream)
366        throw std::runtime_error("Failed to open " + path.str());
367
368    stream >> std::noskipws;
369    std::istream_iterator< char > begin(stream), end;
370    std::ostream_iterator< char > out(std::cerr);
371    std::copy(begin, end, out);
372
373    stream.close();
374}
375
376static
377bool
378grep_file(const atf::fs::path& path, const std::string& regexp)
379{
380    std::ifstream stream(path.c_str());
381    if (!stream)
382        throw std::runtime_error("Failed to open " + path.str());
383
384    bool found = false;
385
386    std::string line;
387    while (!found && !std::getline(stream, line).fail()) {
388        if (atf::text::match(line, regexp))
389            found = true;
390    }
391
392    stream.close();
393
394    return found;
395}
396
397static
398bool
399file_empty(const atf::fs::path& p)
400{
401    atf::fs::file_info f(p);
402
403    return (f.get_size() == 0);
404}
405
406static bool
407compare_files(const atf::fs::path& p1, const atf::fs::path& p2)
408{
409    bool equal = false;
410
411    std::ifstream f1(p1.c_str());
412    if (!f1)
413        throw std::runtime_error("Failed to open " + p1.str());
414
415    std::ifstream f2(p2.c_str());
416    if (!f2)
417        throw std::runtime_error("Failed to open " + p1.str());
418
419    for (;;) {
420        char buf1[512], buf2[512];
421
422        f1.read(buf1, sizeof(buf1));
423        if (f1.bad())
424            throw std::runtime_error("Failed to read from " + p1.str());
425
426        f2.read(buf2, sizeof(buf2));
427        if (f2.bad())
428            throw std::runtime_error("Failed to read from " + p1.str());
429
430        if ((f1.gcount() == 0) && (f2.gcount() == 0)) {
431            equal = true;
432            break;
433        }
434
435        if ((f1.gcount() != f2.gcount()) ||
436            (std::memcmp(buf1, buf2, f1.gcount()) != 0)) {
437            break;
438        }
439    }
440
441    return equal;
442}
443
444static
445void
446print_diff(const atf::fs::path& p1, const atf::fs::path& p2)
447{
448    const atf::process::status s =
449        atf::process::exec(atf::fs::path("diff"),
450                           atf::process::argv_array("diff", "-u", p1.c_str(),
451                                                    p2.c_str(), NULL),
452                           atf::process::stream_connect(STDOUT_FILENO,
453                                                        STDERR_FILENO),
454                           atf::process::stream_inherit());
455
456    if (!s.exited())
457        std::cerr << "Failed to run diff(3)\n";
458
459    if (s.exitstatus() != 1)
460        std::cerr << "Error while running diff(3)\n";
461}
462
463static
464std::string
465decode(const std::string& s)
466{
467    size_t i;
468    std::string res;
469
470    res.reserve(s.length());
471
472    i = 0;
473    while (i < s.length()) {
474        char c = s[i++];
475
476        if (c == '\\') {
477            switch (s[i++]) {
478            case 'a': c = '\a'; break;
479            case 'b': c = '\b'; break;
480            case 'c': break;
481            case 'e': c = 033; break;
482            case 'f': c = '\f'; break;
483            case 'n': c = '\n'; break;
484            case 'r': c = '\r'; break;
485            case 't': c = '\t'; break;
486            case 'v': c = '\v'; break;
487            case '\\': break;
488            case '0':
489                {
490                    int count = 3;
491                    c = 0;
492                    while (--count >= 0 && (unsigned)(s[i] - '0') < 8)
493                        c = (c << 3) + (s[i++] - '0');
494                    break;
495                }
496            default:
497                --i;
498                break;
499            }
500        }
501
502        res.push_back(c);
503    }
504
505    return res;
506}
507
508static
509bool
510run_status_check(const status_check& sc, const atf::check::check_result& cr)
511{
512    bool result;
513
514    if (sc.type == sc_exit) {
515        if (cr.exited() && sc.value != INT_MIN) {
516            const int status = cr.exitcode();
517
518            if (!sc.negated && sc.value != status) {
519                std::cerr << "Fail: incorrect exit status: "
520                          << status << ", expected: "
521                          << sc.value << "\n";
522                result = false;
523            } else if (sc.negated && sc.value == status) {
524                std::cerr << "Fail: incorrect exit status: "
525                          << status << ", expected: "
526                          << "anything else\n";
527                result = false;
528            } else
529                result = true;
530        } else if (cr.exited() && sc.value == INT_MIN) {
531            result = true;
532        } else {
533            std::cerr << "Fail: program did not exit cleanly\n";
534            result = false;
535        }
536    } else if (sc.type == sc_ignore) {
537        result = true;
538    } else if (sc.type == sc_signal) {
539        if (cr.signaled() && sc.value != INT_MIN) {
540            const int status = cr.termsig();
541
542            if (!sc.negated && sc.value != status) {
543                std::cerr << "Fail: incorrect signal received: "
544                          << status << ", expected: " << sc.value << "\n";
545                result = false;
546            } else if (sc.negated && sc.value == status) {
547                std::cerr << "Fail: incorrect signal received: "
548                          << status << ", expected: "
549                          << "anything else\n";
550                result = false;
551            } else
552                result = true;
553        } else if (cr.signaled() && sc.value == INT_MIN) {
554            result = true;
555        } else {
556            std::cerr << "Fail: program did not receive a signal\n";
557            result = false;
558        }
559    } else {
560        UNREACHABLE;
561        result = false;
562    }
563
564    if (result == false) {
565        std::cerr << "stdout:\n";
566        cat_file(atf::fs::path(cr.stdout_path()));
567        std::cerr << "\n";
568
569        std::cerr << "stderr:\n";
570        cat_file(atf::fs::path(cr.stderr_path()));
571        std::cerr << "\n";
572    }
573
574    return result;
575}
576
577static
578bool
579run_status_checks(const std::vector< status_check >& checks,
580                  const atf::check::check_result& result)
581{
582    bool ok = false;
583
584    for (std::vector< status_check >::const_iterator iter = checks.begin();
585         !ok && iter != checks.end(); iter++) {
586         ok |= run_status_check(*iter, result);
587    }
588
589    return ok;
590}
591
592static
593bool
594run_output_check(const output_check oc, const atf::fs::path& path,
595                 const std::string& stdxxx)
596{
597    bool result;
598
599    if (oc.type == oc_empty) {
600        const bool is_empty = file_empty(path);
601        if (!oc.negated && !is_empty) {
602            std::cerr << "Fail: " << stdxxx << " not empty\n";
603            print_diff(atf::fs::path("/dev/null"), path);
604            result = false;
605        } else if (oc.negated && is_empty) {
606            std::cerr << "Fail: " << stdxxx << " is empty\n";
607            result = false;
608        } else
609            result = true;
610    } else if (oc.type == oc_file) {
611        const bool equals = compare_files(path, atf::fs::path(oc.value));
612        if (!oc.negated && !equals) {
613            std::cerr << "Fail: " << stdxxx << " does not match golden "
614                "output\n";
615            print_diff(atf::fs::path(oc.value), path);
616            result = false;
617        } else if (oc.negated && equals) {
618            std::cerr << "Fail: " << stdxxx << " matches golden output\n";
619            cat_file(atf::fs::path(oc.value));
620            result = false;
621        } else
622            result = true;
623    } else if (oc.type == oc_ignore) {
624        result = true;
625    } else if (oc.type == oc_inline) {
626        atf::fs::path path2 = atf::fs::path(atf::config::get("atf_workdir"))
627                              / "inline.XXXXXX";
628        temp_file temp(path2);
629        temp.write(decode(oc.value));
630        temp.close();
631
632        const bool equals = compare_files(path, temp.get_path());
633        if (!oc.negated && !equals) {
634            std::cerr << "Fail: " << stdxxx << " does not match expected "
635                "value\n";
636            print_diff(temp.get_path(), path);
637            result = false;
638        } else if (oc.negated && equals) {
639            std::cerr << "Fail: " << stdxxx << " matches expected value\n";
640            cat_file(temp.get_path());
641            result = false;
642        } else
643            result = true;
644    } else if (oc.type == oc_match) {
645        const bool matches = grep_file(path, oc.value);
646        if (!oc.negated && !matches) {
647            std::cerr << "Fail: regexp " + oc.value + " not in " << stdxxx
648                      << "\n";
649            cat_file(path);
650            result = false;
651        } else if (oc.negated && matches) {
652            std::cerr << "Fail: regexp " + oc.value + " is in " << stdxxx
653                      << "\n";
654            cat_file(path);
655            result = false;
656        } else
657            result = true;
658    } else if (oc.type == oc_save) {
659        INV(!oc.negated);
660        std::ifstream ifs(path.c_str(), std::fstream::binary);
661        ifs >> std::noskipws;
662        std::istream_iterator< char > begin(ifs), end;
663
664        std::ofstream ofs(oc.value.c_str(), std::fstream::binary
665                                     | std::fstream::trunc);
666        std::ostream_iterator <char> obegin(ofs);
667
668        std::copy(begin, end, obegin);
669        result = true;
670    } else {
671        UNREACHABLE;
672        result = false;
673    }
674
675    return result;
676}
677
678static
679bool
680run_output_checks(const std::vector< output_check >& checks,
681                  const atf::fs::path& path, const std::string& stdxxx)
682{
683    bool ok = true;
684
685    for (std::vector< output_check >::const_iterator iter = checks.begin();
686         iter != checks.end(); iter++) {
687         ok &= run_output_check(*iter, path, stdxxx);
688    }
689
690    return ok;
691}
692
693// ------------------------------------------------------------------------
694// The "atf_check" application.
695// ------------------------------------------------------------------------
696
697namespace {
698
699class atf_check : public atf::application::app {
700    bool m_xflag;
701
702    std::vector< status_check > m_status_checks;
703    std::vector< output_check > m_stdout_checks;
704    std::vector< output_check > m_stderr_checks;
705
706    static const char* m_description;
707
708    bool run_output_checks(const atf::check::check_result&,
709                           const std::string&) const;
710
711    std::string specific_args(void) const;
712    options_set specific_options(void) const;
713    void process_option(int, const char*);
714    void process_option_s(const std::string&);
715
716public:
717    atf_check(void);
718    int main(void);
719};
720
721} // anonymous namespace
722
723const char* atf_check::m_description =
724    "atf-check executes given command and analyzes its results.";
725
726atf_check::atf_check(void) :
727    app(m_description, "atf-check(1)"),
728    m_xflag(false)
729{
730}
731
732bool
733atf_check::run_output_checks(const atf::check::check_result& r,
734                             const std::string& stdxxx)
735    const
736{
737    if (stdxxx == "stdout") {
738        return ::run_output_checks(m_stdout_checks,
739            atf::fs::path(r.stdout_path()), "stdout");
740    } else if (stdxxx == "stderr") {
741        return ::run_output_checks(m_stderr_checks,
742            atf::fs::path(r.stderr_path()), "stderr");
743    } else {
744        UNREACHABLE;
745        return false;
746    }
747}
748
749std::string
750atf_check::specific_args(void)
751    const
752{
753    return "<command>";
754}
755
756atf_check::options_set
757atf_check::specific_options(void)
758    const
759{
760    using atf::application::option;
761    options_set opts;
762
763    opts.insert(option('s', "qual:value", "Handle status. Qualifier "
764                "must be one of: ignore exit:<num> signal:<name|num>"));
765    opts.insert(option('o', "action:arg", "Handle stdout. Action must be "
766                "one of: empty ignore file:<path> inline:<val> match:regexp "
767                "save:<path>"));
768    opts.insert(option('e', "action:arg", "Handle stderr. Action must be "
769                "one of: empty ignore file:<path> inline:<val> match:regexp "
770                "save:<path>"));
771    opts.insert(option('x', "", "Execute command as a shell command"));
772
773    return opts;
774}
775
776void
777atf_check::process_option(int ch, const char* arg)
778{
779    switch (ch) {
780    case 's':
781        m_status_checks.push_back(parse_status_check_arg(arg));
782        break;
783
784    case 'o':
785        m_stdout_checks.push_back(parse_output_check_arg(arg));
786        break;
787
788    case 'e':
789        m_stderr_checks.push_back(parse_output_check_arg(arg));
790        break;
791
792    case 'x':
793        m_xflag = true;
794        break;
795
796    default:
797        UNREACHABLE;
798    }
799}
800
801int
802atf_check::main(void)
803{
804    if (m_argc < 1)
805        throw atf::application::usage_error("No command specified");
806
807    int status = EXIT_FAILURE;
808
809    std::auto_ptr< atf::check::check_result > r =
810        m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
811
812    if (m_status_checks.empty())
813        m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS));
814    else if (m_status_checks.size() > 1) {
815        // TODO: Remove this restriction.
816        throw atf::application::usage_error("Cannot specify -s more than once");
817    }
818
819    if (m_stdout_checks.empty())
820        m_stdout_checks.push_back(output_check(oc_empty, false, ""));
821    if (m_stderr_checks.empty())
822        m_stderr_checks.push_back(output_check(oc_empty, false, ""));
823
824    if ((run_status_checks(m_status_checks, *r) == false) ||
825        (run_output_checks(*r, "stderr") == false) ||
826        (run_output_checks(*r, "stdout") == false))
827        status = EXIT_FAILURE;
828    else
829        status = EXIT_SUCCESS;
830
831    return status;
832}
833
834int
835main(int argc, char* const* argv)
836{
837    return atf_check().run(argc, argv);
838}
839