1// Copyright (c) 2008 The NetBSD Foundation, Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions
6// are met:
7// 1. Redistributions of source code must retain the above copyright
8//    notice, this list of conditions and the following disclaimer.
9// 2. Redistributions in binary form must reproduce the above copyright
10//    notice, this list of conditions and the following disclaimer in the
11//    documentation and/or other materials provided with the distribution.
12//
13// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
14// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
15// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
16// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
18// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
20// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
24// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26extern "C" {
27#include <sys/types.h>
28#include <sys/wait.h>
29
30#include <limits.h>
31#include <signal.h>
32#include <unistd.h>
33}
34
35#include <cerrno>
36#include <cstdlib>
37#include <cstring>
38#include <fstream>
39#include <ios>
40#include <iostream>
41#include <iterator>
42#include <list>
43#include <memory>
44#include <utility>
45
46#include "atf-c++/check.hpp"
47#include "atf-c++/detail/application.hpp"
48#include "atf-c++/detail/auto_array.hpp"
49#include "atf-c++/detail/env.hpp"
50#include "atf-c++/detail/exceptions.hpp"
51#include "atf-c++/detail/fs.hpp"
52#include "atf-c++/detail/process.hpp"
53#include "atf-c++/detail/sanity.hpp"
54#include "atf-c++/detail/text.hpp"
55
56// ------------------------------------------------------------------------
57// Auxiliary functions.
58// ------------------------------------------------------------------------
59
60namespace {
61
62enum status_check_t {
63    sc_exit,
64    sc_ignore,
65    sc_signal,
66};
67
68struct status_check {
69    status_check_t type;
70    bool negated;
71    int value;
72
73    status_check(const status_check_t& p_type, const bool p_negated,
74                 const int p_value) :
75        type(p_type),
76        negated(p_negated),
77        value(p_value)
78    {
79    }
80};
81
82enum output_check_t {
83    oc_ignore,
84    oc_inline,
85    oc_file,
86    oc_empty,
87    oc_match,
88    oc_save
89};
90
91struct output_check {
92    output_check_t type;
93    bool negated;
94    std::string value;
95
96    output_check(const output_check_t& p_type, const bool p_negated,
97                 const std::string& p_value) :
98        type(p_type),
99        negated(p_negated),
100        value(p_value)
101    {
102    }
103};
104
105class temp_file : public std::ostream {
106    std::auto_ptr< atf::fs::path > m_path;
107    int m_fd;
108
109public:
110    temp_file(const char* pattern) :
111        std::ostream(NULL),
112        m_fd(-1)
113    {
114        const atf::fs::path file = atf::fs::path(
115            atf::env::get("TMPDIR", "/tmp")) / pattern;
116
117        atf::auto_array< char > buf(new char[file.str().length() + 1]);
118        std::strcpy(buf.get(), file.c_str());
119
120        m_fd = ::mkstemp(buf.get());
121        if (m_fd == -1)
122            throw atf::system_error("atf_check::temp_file::temp_file(" +
123                                    file.str() + ")", "mkstemp(3) failed",
124                                    errno);
125
126        m_path.reset(new atf::fs::path(buf.get()));
127    }
128
129    ~temp_file(void)
130    {
131        close();
132        try {
133            remove(*m_path);
134        } catch (const atf::system_error&) {
135            // Ignore deletion errors.
136        }
137    }
138
139    const atf::fs::path&
140    get_path(void) const
141    {
142        return *m_path;
143    }
144
145    void
146    write(const std::string& text)
147    {
148        if (::write(m_fd, text.c_str(), text.size()) == -1)
149            throw atf::system_error("atf_check", "write(2) failed", errno);
150    }
151
152    void
153    close(void)
154    {
155        if (m_fd != -1) {
156            flush();
157            ::close(m_fd);
158            m_fd = -1;
159        }
160    }
161};
162
163} // anonymous namespace
164
165static int
166parse_exit_code(const std::string& str)
167{
168    try {
169        const int value = atf::text::to_type< int >(str);
170        if (value < 0 || value > 255)
171            throw std::runtime_error("Unused reason");
172        return value;
173    } catch (const std::runtime_error&) {
174        throw atf::application::usage_error("Invalid exit code for -s option; "
175            "must be an integer in range 0-255");
176    }
177}
178
179static struct name_number {
180    const char *name;
181    int signo;
182} signal_names_to_numbers[] = {
183    { "hup", SIGHUP },
184    { "int", SIGINT },
185    { "quit", SIGQUIT },
186    { "trap", SIGTRAP },
187    { "abrt", SIGABRT },
188    { "kill", SIGKILL },
189    { "segv", SIGSEGV },
190    { "pipe", SIGPIPE },
191    { "alrm", SIGALRM },
192    { "term", SIGTERM },
193    { "usr1", SIGUSR1 },
194    { "usr2", SIGUSR2 },
195    { NULL, INT_MIN },
196};
197
198static int
199signal_name_to_number(const std::string& str)
200{
201    struct name_number* iter = signal_names_to_numbers;
202    int signo = INT_MIN;
203    while (signo == INT_MIN && iter->name != NULL) {
204        if (str == iter->name || str == std::string("sig") + iter->name)
205            signo = iter->signo;
206        else
207            iter++;
208    }
209    return signo;
210}
211
212static int
213parse_signal(const std::string& str)
214{
215    const int signo = signal_name_to_number(str);
216    if (signo == INT_MIN) {
217        try {
218            return atf::text::to_type< int >(str);
219        } catch (std::runtime_error) {
220            throw atf::application::usage_error("Invalid signal name or number "
221                "in -s option");
222        }
223    }
224    INV(signo != INT_MIN);
225    return signo;
226}
227
228static status_check
229parse_status_check_arg(const std::string& arg)
230{
231    const std::string::size_type delimiter = arg.find(':');
232    bool negated = (arg.compare(0, 4, "not-") == 0);
233    const std::string action_str = arg.substr(0, delimiter);
234    const std::string action = negated ? action_str.substr(4) : action_str;
235    const std::string value_str = (
236        delimiter == std::string::npos ? "" : arg.substr(delimiter + 1));
237    int value;
238
239    status_check_t type;
240    if (action == "eq") {
241        // Deprecated; use exit instead.  TODO: Remove after 0.10.
242        type = sc_exit;
243        if (negated)
244            throw atf::application::usage_error("Cannot negate eq checker");
245        negated = false;
246        value = parse_exit_code(value_str);
247    } else if (action == "exit") {
248        type = sc_exit;
249        if (value_str.empty())
250            value = INT_MIN;
251        else
252            value = parse_exit_code(value_str);
253    } else if (action == "ignore") {
254        if (negated)
255            throw atf::application::usage_error("Cannot negate ignore checker");
256        type = sc_ignore;
257        value = INT_MIN;
258    } else if (action == "ne") {
259        // Deprecated; use not-exit instead.  TODO: Remove after 0.10.
260        type = sc_exit;
261        if (negated)
262            throw atf::application::usage_error("Cannot negate ne checker");
263        negated = true;
264        value = parse_exit_code(value_str);
265    } else if (action == "signal") {
266        type = sc_signal;
267        if (value_str.empty())
268            value = INT_MIN;
269        else
270            value = parse_signal(value_str);
271    } else
272        throw atf::application::usage_error("Invalid status checker");
273
274    return status_check(type, negated, value);
275}
276
277static
278output_check
279parse_output_check_arg(const std::string& arg)
280{
281    const std::string::size_type delimiter = arg.find(':');
282    const bool negated = (arg.compare(0, 4, "not-") == 0);
283    const std::string action_str = arg.substr(0, delimiter);
284    const std::string action = negated ? action_str.substr(4) : action_str;
285
286    output_check_t type;
287    if (action == "empty")
288        type = oc_empty;
289    else if (action == "file")
290        type = oc_file;
291    else if (action == "ignore") {
292        if (negated)
293            throw atf::application::usage_error("Cannot negate ignore checker");
294        type = oc_ignore;
295    } else if (action == "inline")
296        type = oc_inline;
297    else if (action == "match")
298        type = oc_match;
299    else if (action == "save") {
300        if (negated)
301            throw atf::application::usage_error("Cannot negate save checker");
302        type = oc_save;
303    } else
304        throw atf::application::usage_error("Invalid output checker");
305
306    return output_check(type, negated, arg.substr(delimiter + 1));
307}
308
309static
310std::string
311flatten_argv(char* const* argv)
312{
313    std::string cmdline;
314
315    char* const* arg = &argv[0];
316    while (*arg != NULL) {
317        if (arg != &argv[0])
318            cmdline += ' ';
319
320        cmdline += *arg;
321
322        arg++;
323    }
324
325    return cmdline;
326}
327
328static
329std::auto_ptr< atf::check::check_result >
330execute(const char* const* argv)
331{
332    // TODO: This should go to stderr... but fixing it now may be hard as test
333    // cases out there might be relying on stderr being silent.
334    std::cout << "Executing command [ ";
335    for (int i = 0; argv[i] != NULL; ++i)
336        std::cout << argv[i] << " ";
337    std::cout << "]\n";
338    std::cout.flush();
339
340    atf::process::argv_array argva(argv);
341    return atf::check::exec(argva);
342}
343
344static
345std::auto_ptr< atf::check::check_result >
346execute_with_shell(char* const* argv)
347{
348    const std::string cmd = flatten_argv(argv);
349    const std::string shell = atf::env::get("ATF_SHELL", ATF_SHELL);
350
351    const char* sh_argv[4];
352    sh_argv[0] = shell.c_str();
353    sh_argv[1] = "-c";
354    sh_argv[2] = cmd.c_str();
355    sh_argv[3] = NULL;
356    return execute(sh_argv);
357}
358
359static
360void
361cat_file(const atf::fs::path& path)
362{
363    std::ifstream stream(path.c_str());
364    if (!stream)
365        throw std::runtime_error("Failed to open " + path.str());
366
367    stream >> std::noskipws;
368    std::istream_iterator< char > begin(stream), end;
369    std::ostream_iterator< char > out(std::cerr);
370    std::copy(begin, end, out);
371
372    stream.close();
373}
374
375static
376bool
377grep_file(const atf::fs::path& path, const std::string& regexp)
378{
379    std::ifstream stream(path.c_str());
380    if (!stream)
381        throw std::runtime_error("Failed to open " + path.str());
382
383    bool found = false;
384
385    std::string line;
386    while (!found && !std::getline(stream, line).fail()) {
387        if (atf::text::match(line, regexp))
388            found = true;
389    }
390
391    stream.close();
392
393    return found;
394}
395
396static
397bool
398file_empty(const atf::fs::path& p)
399{
400    atf::fs::file_info f(p);
401
402    return (f.get_size() == 0);
403}
404
405static bool
406compare_files(const atf::fs::path& p1, const atf::fs::path& p2)
407{
408    bool equal = false;
409
410    std::ifstream f1(p1.c_str());
411    if (!f1)
412        throw std::runtime_error("Failed to open " + p1.str());
413
414    std::ifstream f2(p2.c_str());
415    if (!f2)
416        throw std::runtime_error("Failed to open " + p1.str());
417
418    for (;;) {
419        char buf1[512], buf2[512];
420
421        f1.read(buf1, sizeof(buf1));
422        if (f1.bad())
423            throw std::runtime_error("Failed to read from " + p1.str());
424
425        f2.read(buf2, sizeof(buf2));
426        if (f2.bad())
427            throw std::runtime_error("Failed to read from " + p1.str());
428
429        if ((f1.gcount() == 0) && (f2.gcount() == 0)) {
430            equal = true;
431            break;
432        }
433
434        if ((f1.gcount() != f2.gcount()) ||
435            (std::memcmp(buf1, buf2, f1.gcount()) != 0)) {
436            break;
437        }
438    }
439
440    return equal;
441}
442
443static
444void
445print_diff(const atf::fs::path& p1, const atf::fs::path& p2)
446{
447    const atf::process::status s =
448        atf::process::exec(atf::fs::path("diff"),
449                           atf::process::argv_array("diff", "-u", p1.c_str(),
450                                                    p2.c_str(), NULL),
451                           atf::process::stream_connect(STDOUT_FILENO,
452                                                        STDERR_FILENO),
453                           atf::process::stream_inherit());
454
455    if (!s.exited())
456        std::cerr << "Failed to run diff(3)\n";
457
458    if (s.exitstatus() != 1)
459        std::cerr << "Error while running diff(3)\n";
460}
461
462static
463std::string
464decode(const std::string& s)
465{
466    size_t i;
467    std::string res;
468
469    res.reserve(s.length());
470
471    i = 0;
472    while (i < s.length()) {
473        char c = s[i++];
474
475        if (c == '\\') {
476            switch (s[i++]) {
477            case 'a': c = '\a'; break;
478            case 'b': c = '\b'; break;
479            case 'c': break;
480            case 'e': c = 033; break;
481            case 'f': c = '\f'; break;
482            case 'n': c = '\n'; break;
483            case 'r': c = '\r'; break;
484            case 't': c = '\t'; break;
485            case 'v': c = '\v'; break;
486            case '\\': break;
487            case '0':
488                {
489                    int count = 3;
490                    c = 0;
491                    while (--count >= 0 && (unsigned)(s[i] - '0') < 8)
492                        c = (c << 3) + (s[i++] - '0');
493                    break;
494                }
495            default:
496                --i;
497                break;
498            }
499        }
500
501        res.push_back(c);
502    }
503
504    return res;
505}
506
507static
508bool
509run_status_check(const status_check& sc, const atf::check::check_result& cr)
510{
511    bool result;
512
513    if (sc.type == sc_exit) {
514        if (cr.exited() && sc.value != INT_MIN) {
515            const int status = cr.exitcode();
516
517            if (!sc.negated && sc.value != status) {
518                std::cerr << "Fail: incorrect exit status: "
519                          << status << ", expected: "
520                          << sc.value << "\n";
521                result = false;
522            } else if (sc.negated && sc.value == status) {
523                std::cerr << "Fail: incorrect exit status: "
524                          << status << ", expected: "
525                          << "anything else\n";
526                result = false;
527            } else
528                result = true;
529        } else if (cr.exited() && sc.value == INT_MIN) {
530            result = true;
531        } else {
532            std::cerr << "Fail: program did not exit cleanly\n";
533            result = false;
534        }
535    } else if (sc.type == sc_ignore) {
536        result = true;
537    } else if (sc.type == sc_signal) {
538        if (cr.signaled() && sc.value != INT_MIN) {
539            const int status = cr.termsig();
540
541            if (!sc.negated && sc.value != status) {
542                std::cerr << "Fail: incorrect signal received: "
543                          << status << ", expected: " << sc.value << "\n";
544                result = false;
545            } else if (sc.negated && sc.value == status) {
546                std::cerr << "Fail: incorrect signal received: "
547                          << status << ", expected: "
548                          << "anything else\n";
549                result = false;
550            } else
551                result = true;
552        } else if (cr.signaled() && sc.value == INT_MIN) {
553            result = true;
554        } else {
555            std::cerr << "Fail: program did not receive a signal\n";
556            result = false;
557        }
558    } else {
559        UNREACHABLE;
560        result = false;
561    }
562
563    if (result == false) {
564        std::cerr << "stdout:\n";
565        cat_file(atf::fs::path(cr.stdout_path()));
566        std::cerr << "\n";
567
568        std::cerr << "stderr:\n";
569        cat_file(atf::fs::path(cr.stderr_path()));
570        std::cerr << "\n";
571    }
572
573    return result;
574}
575
576static
577bool
578run_status_checks(const std::vector< status_check >& checks,
579                  const atf::check::check_result& result)
580{
581    bool ok = false;
582
583    for (std::vector< status_check >::const_iterator iter = checks.begin();
584         !ok && iter != checks.end(); iter++) {
585         ok |= run_status_check(*iter, result);
586    }
587
588    return ok;
589}
590
591static
592bool
593run_output_check(const output_check oc, const atf::fs::path& path,
594                 const std::string& stdxxx)
595{
596    bool result;
597
598    if (oc.type == oc_empty) {
599        const bool is_empty = file_empty(path);
600        if (!oc.negated && !is_empty) {
601            std::cerr << "Fail: " << stdxxx << " not empty\n";
602            print_diff(atf::fs::path("/dev/null"), path);
603            result = false;
604        } else if (oc.negated && is_empty) {
605            std::cerr << "Fail: " << stdxxx << " is empty\n";
606            result = false;
607        } else
608            result = true;
609    } else if (oc.type == oc_file) {
610        const bool equals = compare_files(path, atf::fs::path(oc.value));
611        if (!oc.negated && !equals) {
612            std::cerr << "Fail: " << stdxxx << " does not match golden "
613                "output\n";
614            print_diff(atf::fs::path(oc.value), path);
615            result = false;
616        } else if (oc.negated && equals) {
617            std::cerr << "Fail: " << stdxxx << " matches golden output\n";
618            cat_file(atf::fs::path(oc.value));
619            result = false;
620        } else
621            result = true;
622    } else if (oc.type == oc_ignore) {
623        result = true;
624    } else if (oc.type == oc_inline) {
625        temp_file temp("atf-check.XXXXXX");
626        temp.write(decode(oc.value));
627        temp.close();
628
629        const bool equals = compare_files(path, temp.get_path());
630        if (!oc.negated && !equals) {
631            std::cerr << "Fail: " << stdxxx << " does not match expected "
632                "value\n";
633            print_diff(temp.get_path(), path);
634            result = false;
635        } else if (oc.negated && equals) {
636            std::cerr << "Fail: " << stdxxx << " matches expected value\n";
637            cat_file(temp.get_path());
638            result = false;
639        } else
640            result = true;
641    } else if (oc.type == oc_match) {
642        const bool matches = grep_file(path, oc.value);
643        if (!oc.negated && !matches) {
644            std::cerr << "Fail: regexp " + oc.value + " not in " << stdxxx
645                      << "\n";
646            cat_file(path);
647            result = false;
648        } else if (oc.negated && matches) {
649            std::cerr << "Fail: regexp " + oc.value + " is in " << stdxxx
650                      << "\n";
651            cat_file(path);
652            result = false;
653        } else
654            result = true;
655    } else if (oc.type == oc_save) {
656        INV(!oc.negated);
657        std::ifstream ifs(path.c_str(), std::fstream::binary);
658        ifs >> std::noskipws;
659        std::istream_iterator< char > begin(ifs), end;
660
661        std::ofstream ofs(oc.value.c_str(), std::fstream::binary
662                                     | std::fstream::trunc);
663        std::ostream_iterator <char> obegin(ofs);
664
665        std::copy(begin, end, obegin);
666        result = true;
667    } else {
668        UNREACHABLE;
669        result = false;
670    }
671
672    return result;
673}
674
675static
676bool
677run_output_checks(const std::vector< output_check >& checks,
678                  const atf::fs::path& path, const std::string& stdxxx)
679{
680    bool ok = true;
681
682    for (std::vector< output_check >::const_iterator iter = checks.begin();
683         iter != checks.end(); iter++) {
684         ok &= run_output_check(*iter, path, stdxxx);
685    }
686
687    return ok;
688}
689
690// ------------------------------------------------------------------------
691// The "atf_check" application.
692// ------------------------------------------------------------------------
693
694namespace {
695
696class atf_check : public atf::application::app {
697    bool m_xflag;
698
699    std::vector< status_check > m_status_checks;
700    std::vector< output_check > m_stdout_checks;
701    std::vector< output_check > m_stderr_checks;
702
703    static const char* m_description;
704
705    bool run_output_checks(const atf::check::check_result&,
706                           const std::string&) const;
707
708    std::string specific_args(void) const;
709    options_set specific_options(void) const;
710    void process_option(int, const char*);
711    void process_option_s(const std::string&);
712
713public:
714    atf_check(void);
715    int main(void);
716};
717
718} // anonymous namespace
719
720const char* atf_check::m_description =
721    "atf-check executes given command and analyzes its results.";
722
723atf_check::atf_check(void) :
724    app(m_description, "atf-check(1)"),
725    m_xflag(false)
726{
727}
728
729bool
730atf_check::run_output_checks(const atf::check::check_result& r,
731                             const std::string& stdxxx)
732    const
733{
734    if (stdxxx == "stdout") {
735        return ::run_output_checks(m_stdout_checks,
736            atf::fs::path(r.stdout_path()), "stdout");
737    } else if (stdxxx == "stderr") {
738        return ::run_output_checks(m_stderr_checks,
739            atf::fs::path(r.stderr_path()), "stderr");
740    } else {
741        UNREACHABLE;
742        return false;
743    }
744}
745
746std::string
747atf_check::specific_args(void)
748    const
749{
750    return "<command>";
751}
752
753atf_check::options_set
754atf_check::specific_options(void)
755    const
756{
757    using atf::application::option;
758    options_set opts;
759
760    opts.insert(option('s', "qual:value", "Handle status. Qualifier "
761                "must be one of: ignore exit:<num> signal:<name|num>"));
762    opts.insert(option('o', "action:arg", "Handle stdout. Action must be "
763                "one of: empty ignore file:<path> inline:<val> match:regexp "
764                "save:<path>"));
765    opts.insert(option('e', "action:arg", "Handle stderr. Action must be "
766                "one of: empty ignore file:<path> inline:<val> match:regexp "
767                "save:<path>"));
768    opts.insert(option('x', "", "Execute command as a shell command"));
769
770    return opts;
771}
772
773void
774atf_check::process_option(int ch, const char* arg)
775{
776    switch (ch) {
777    case 's':
778        m_status_checks.push_back(parse_status_check_arg(arg));
779        break;
780
781    case 'o':
782        m_stdout_checks.push_back(parse_output_check_arg(arg));
783        break;
784
785    case 'e':
786        m_stderr_checks.push_back(parse_output_check_arg(arg));
787        break;
788
789    case 'x':
790        m_xflag = true;
791        break;
792
793    default:
794        UNREACHABLE;
795    }
796}
797
798int
799atf_check::main(void)
800{
801    if (m_argc < 1)
802        throw atf::application::usage_error("No command specified");
803
804    int status = EXIT_FAILURE;
805
806    std::auto_ptr< atf::check::check_result > r =
807        m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
808
809    if (m_status_checks.empty())
810        m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS));
811    else if (m_status_checks.size() > 1) {
812        // TODO: Remove this restriction.
813        throw atf::application::usage_error("Cannot specify -s more than once");
814    }
815
816    if (m_stdout_checks.empty())
817        m_stdout_checks.push_back(output_check(oc_empty, false, ""));
818    if (m_stderr_checks.empty())
819        m_stderr_checks.push_back(output_check(oc_empty, false, ""));
820
821    if ((run_status_checks(m_status_checks, *r) == false) ||
822        (run_output_checks(*r, "stderr") == false) ||
823        (run_output_checks(*r, "stdout") == false))
824        status = EXIT_FAILURE;
825    else
826        status = EXIT_SUCCESS;
827
828    return status;
829}
830
831int
832main(int argc, char* const* argv)
833{
834    return atf_check().run(argc, argv);
835}
836