1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2008, 2009, 2010 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <sys/types.h>
32#include <sys/wait.h>
33
34#include <limits.h>
35#include <signal.h>
36}
37
38#include <cerrno>
39#include <cstdlib>
40#include <cstring>
41#include <fstream>
42#include <iostream>
43#include <iterator>
44#include <list>
45#include <memory>
46#include <utility>
47
48#include "atf-c++/check.hpp"
49#include "atf-c++/config.hpp"
50#include "atf-c++/utils.hpp"
51
52#include "atf-c++/detail/application.hpp"
53#include "atf-c++/detail/exceptions.hpp"
54#include "atf-c++/detail/fs.hpp"
55#include "atf-c++/detail/process.hpp"
56#include "atf-c++/detail/sanity.hpp"
57#include "atf-c++/detail/text.hpp"
58
59// ------------------------------------------------------------------------
60// Auxiliary functions.
61// ------------------------------------------------------------------------
62
63namespace {
64
65enum status_check_t {
66    sc_exit,
67    sc_ignore,
68    sc_signal,
69};
70
71struct status_check {
72    status_check_t type;
73    bool negated;
74    int value;
75
76    status_check(const status_check_t& p_type, const bool p_negated,
77                 const int p_value) :
78        type(p_type),
79        negated(p_negated),
80        value(p_value)
81    {
82    }
83};
84
85enum output_check_t {
86    oc_ignore,
87    oc_inline,
88    oc_file,
89    oc_empty,
90    oc_match,
91    oc_save
92};
93
94struct output_check {
95    output_check_t type;
96    bool negated;
97    std::string value;
98
99    output_check(const output_check_t& p_type, const bool p_negated,
100                 const std::string& p_value) :
101        type(p_type),
102        negated(p_negated),
103        value(p_value)
104    {
105    }
106};
107
108class temp_file : public std::ostream {
109    std::auto_ptr< atf::fs::path > m_path;
110    int m_fd;
111
112public:
113    temp_file(const atf::fs::path& p) :
114        std::ostream(NULL),
115        m_fd(-1)
116    {
117        atf::utils::auto_array< char > buf(new char[p.str().length() + 1]);
118        std::strcpy(buf.get(), p.c_str());
119
120        m_fd = ::mkstemp(buf.get());
121        if (m_fd == -1)
122            throw atf::system_error("atf_check::temp_file::temp_file(" +
123                                    p.str() + ")", "mkstemp(3) failed",
124                                    errno);
125
126        m_path.reset(new atf::fs::path(buf.get()));
127    }
128
129    ~temp_file(void)
130    {
131        close();
132        try {
133            remove(*m_path);
134        } catch (const atf::system_error&) {
135            // Ignore deletion errors.
136        }
137    }
138
139    const atf::fs::path&
140    get_path(void) const
141    {
142        return *m_path;
143    }
144
145    void
146    write(const std::string& text)
147    {
148        if (::write(m_fd, text.c_str(), text.size()) == -1)
149            throw atf::system_error("atf_check", "write(2) failed", errno);
150    }
151
152    void
153    close(void)
154    {
155        if (m_fd != -1) {
156            flush();
157            ::close(m_fd);
158            m_fd = -1;
159        }
160    }
161};
162
163} // anonymous namespace
164
165static int
166parse_exit_code(const std::string& str)
167{
168    try {
169        const int value = atf::text::to_type< int >(str);
170        if (value < 0 || value > 255)
171            throw std::runtime_error("Unused reason");
172        return value;
173    } catch (const std::runtime_error&) {
174        throw atf::application::usage_error("Invalid exit code for -s option; "
175            "must be an integer in range 0-255");
176    }
177}
178
179static struct name_number {
180    const char *name;
181    int signo;
182} signal_names_to_numbers[] = {
183    { "hup", SIGHUP },
184    { "int", SIGINT },
185    { "quit", SIGQUIT },
186    { "trap", SIGTRAP },
187    { "kill", SIGKILL },
188    { "segv", SIGSEGV },
189    { "pipe", SIGPIPE },
190    { "alrm", SIGALRM },
191    { "term", SIGTERM },
192    { "usr1", SIGUSR1 },
193    { "usr2", SIGUSR2 },
194    { NULL, INT_MIN },
195};
196
197static int
198signal_name_to_number(const std::string& str)
199{
200    struct name_number* iter = signal_names_to_numbers;
201    int signo = INT_MIN;
202    while (signo == INT_MIN && iter->name != NULL) {
203        if (str == iter->name || str == std::string("sig") + iter->name)
204            signo = iter->signo;
205        else
206            iter++;
207    }
208    return signo;
209}
210
211static int
212parse_signal(const std::string& str)
213{
214    const int signo = signal_name_to_number(str);
215    if (signo == INT_MIN) {
216        try {
217            return atf::text::to_type< int >(str);
218        } catch (std::runtime_error) {
219            throw atf::application::usage_error("Invalid signal name or number "
220                "in -s option");
221        }
222    }
223    INV(signo != INT_MIN);
224    return signo;
225}
226
227static status_check
228parse_status_check_arg(const std::string& arg)
229{
230    const std::string::size_type delimiter = arg.find(':');
231    bool negated = (arg.compare(0, 4, "not-") == 0);
232    const std::string action_str = arg.substr(0, delimiter);
233    const std::string action = negated ? action_str.substr(4) : action_str;
234    const std::string value_str = (
235        delimiter == std::string::npos ? "" : arg.substr(delimiter + 1));
236    int value;
237
238    status_check_t type;
239    if (action == "eq") {
240        // Deprecated; use exit instead.  TODO: Remove after 0.10.
241        type = sc_exit;
242        if (negated)
243            throw atf::application::usage_error("Cannot negate eq checker");
244        negated = false;
245        value = parse_exit_code(value_str);
246    } else if (action == "exit") {
247        type = sc_exit;
248        if (value_str.empty())
249            value = INT_MIN;
250        else
251            value = parse_exit_code(value_str);
252    } else if (action == "ignore") {
253        if (negated)
254            throw atf::application::usage_error("Cannot negate ignore checker");
255        type = sc_ignore;
256        value = INT_MIN;
257    } else if (action == "ne") {
258        // Deprecated; use not-exit instead.  TODO: Remove after 0.10.
259        type = sc_exit;
260        if (negated)
261            throw atf::application::usage_error("Cannot negate ne checker");
262        negated = true;
263        value = parse_exit_code(value_str);
264    } else if (action == "signal") {
265        type = sc_signal;
266        if (value_str.empty())
267            value = INT_MIN;
268        else
269            value = parse_signal(value_str);
270    } else
271        throw atf::application::usage_error("Invalid output checker");
272
273    return status_check(type, negated, value);
274}
275
276static
277output_check
278parse_output_check_arg(const std::string& arg)
279{
280    const std::string::size_type delimiter = arg.find(':');
281    const bool negated = (arg.compare(0, 4, "not-") == 0);
282    const std::string action_str = arg.substr(0, delimiter);
283    const std::string action = negated ? action_str.substr(4) : action_str;
284
285    output_check_t type;
286    if (action == "empty")
287        type = oc_empty;
288    else if (action == "file")
289        type = oc_file;
290    else if (action == "ignore") {
291        if (negated)
292            throw atf::application::usage_error("Cannot negate ignore checker");
293        type = oc_ignore;
294    } else if (action == "inline")
295        type = oc_inline;
296    else if (action == "match")
297        type = oc_match;
298    else if (action == "save") {
299        if (negated)
300            throw atf::application::usage_error("Cannot negate save checker");
301        type = oc_save;
302    } else
303        throw atf::application::usage_error("Invalid output checker");
304
305    return output_check(type, negated, arg.substr(delimiter + 1));
306}
307
308static
309std::string
310flatten_argv(char* const* argv)
311{
312    std::string cmdline;
313
314    char* const* arg = &argv[0];
315    while (*arg != NULL) {
316        if (arg != &argv[0])
317            cmdline += ' ';
318
319        cmdline += *arg;
320
321        arg++;
322    }
323
324    return cmdline;
325}
326
327static
328std::auto_ptr< atf::check::check_result >
329execute(const char* const* argv)
330{
331    std::cout << "Executing command [ ";
332    for (int i = 0; argv[i] != NULL; ++i)
333        std::cout << argv[i] << " ";
334    std::cout << "]\n";
335
336    atf::process::argv_array argva(argv);
337    return atf::check::exec(argva);
338}
339
340static
341std::auto_ptr< atf::check::check_result >
342execute_with_shell(char* const* argv)
343{
344    const std::string cmd = flatten_argv(argv);
345
346    const char* sh_argv[4];
347    sh_argv[0] = atf::config::get("atf_shell").c_str();
348    sh_argv[1] = "-c";
349    sh_argv[2] = cmd.c_str();
350    sh_argv[3] = NULL;
351    return execute(sh_argv);
352}
353
354static
355void
356cat_file(const atf::fs::path& path)
357{
358    std::ifstream stream(path.c_str());
359    if (!stream)
360        throw std::runtime_error("Failed to open " + path.str());
361
362    stream >> std::noskipws;
363    std::istream_iterator< char > begin(stream), end;
364    std::ostream_iterator< char > out(std::cerr);
365    std::copy(begin, end, out);
366
367    stream.close();
368}
369
370static
371bool
372grep_file(const atf::fs::path& path, const std::string& regexp)
373{
374    std::ifstream stream(path.c_str());
375    if (!stream)
376        throw std::runtime_error("Failed to open " + path.str());
377
378    bool found = false;
379
380    std::string line;
381    while (!found && std::getline(stream, line).good()) {
382        if (atf::text::match(line, regexp))
383            found = true;
384    }
385
386    stream.close();
387
388    return found;
389}
390
391static
392bool
393file_empty(const atf::fs::path& p)
394{
395    atf::fs::file_info f(p);
396
397    return (f.get_size() == 0);
398}
399
400static bool
401compare_files(const atf::fs::path& p1, const atf::fs::path& p2)
402{
403    bool equal = false;
404
405    std::ifstream f1(p1.c_str());
406    if (!f1)
407        throw std::runtime_error("Failed to open " + p1.str());
408
409    std::ifstream f2(p2.c_str());
410    if (!f2)
411        throw std::runtime_error("Failed to open " + p1.str());
412
413    for (;;) {
414        char buf1[512], buf2[512];
415
416        f1.read(buf1, sizeof(buf1));
417        if (f1.bad())
418            throw std::runtime_error("Failed to read from " + p1.str());
419
420        f2.read(buf2, sizeof(buf2));
421        if (f2.bad())
422            throw std::runtime_error("Failed to read from " + p1.str());
423
424        std::cout << "1 read: " << f1.gcount() << "\n";
425        std::cout << "2 read: " << f2.gcount() << "\n";
426        if ((f1.gcount() == 0) && (f2.gcount() == 0)) {
427            equal = true;
428            break;
429        }
430
431        if ((f1.gcount() != f2.gcount()) ||
432            (std::memcmp(buf1, buf2, f1.gcount()) != 0)) {
433            break;
434        }
435    }
436
437    return equal;
438}
439
440static
441void
442print_diff(const atf::fs::path& p1, const atf::fs::path& p2)
443{
444    const atf::process::status s =
445        atf::process::exec(atf::fs::path("diff"),
446                           atf::process::argv_array("diff", "-u", p1.c_str(),
447                                                    p2.c_str(), NULL),
448                           atf::process::stream_connect(STDOUT_FILENO,
449                                                        STDERR_FILENO),
450                           atf::process::stream_inherit());
451
452    if (!s.exited())
453        std::cerr << "Failed to run diff(3)\n";
454
455    if (s.exitstatus() != 1)
456        std::cerr << "Error while running diff(3)\n";
457}
458
459static
460std::string
461decode(const std::string& s)
462{
463    size_t i;
464    std::string res;
465
466    res.reserve(s.length());
467
468    i = 0;
469    while (i < s.length()) {
470        char c = s[i++];
471
472        if (c == '\\') {
473            switch (s[i++]) {
474            case 'a': c = '\a'; break;
475            case 'b': c = '\b'; break;
476            case 'c': break;
477            case 'e': c = 033; break;
478            case 'f': c = '\f'; break;
479            case 'n': c = '\n'; break;
480            case 'r': c = '\r'; break;
481            case 't': c = '\t'; break;
482            case 'v': c = '\v'; break;
483            case '\\': break;
484            case '0':
485                {
486                    int count = 3;
487                    c = 0;
488                    while (--count >= 0 && (unsigned)(s[i] - '0') < 8)
489                        c = (c << 3) + (s[i++] - '0');
490                    break;
491                }
492            default:
493                --i;
494                break;
495            }
496        }
497
498        res.push_back(c);
499    }
500
501    return res;
502}
503
504static
505bool
506run_status_check(const status_check& sc, const atf::check::check_result& cr)
507{
508    bool result;
509
510    if (sc.type == sc_exit) {
511        if (cr.exited() && sc.value != INT_MIN) {
512            const int status = cr.exitcode();
513
514            if (!sc.negated && sc.value != status) {
515                std::cerr << "Fail: incorrect exit status: "
516                          << status << ", expected: "
517                          << sc.value << "\n";
518                result = false;
519            } else if (sc.negated && sc.value == status) {
520                std::cerr << "Fail: incorrect exit status: "
521                          << status << ", expected: "
522                          << "anything else\n";
523                result = false;
524            } else
525                result = true;
526        } else if (cr.exited() && sc.value == INT_MIN) {
527            result = true;
528        } else {
529            std::cerr << "Fail: program did not exit cleanly\n";
530            result = false;
531        }
532    } else if (sc.type == sc_ignore) {
533        result = true;
534    } else if (sc.type == sc_signal) {
535        if (cr.signaled() && sc.value != INT_MIN) {
536            const int status = cr.termsig();
537
538            if (!sc.negated && sc.value != status) {
539                std::cerr << "Fail: incorrect signal received: "
540                          << status << ", expected: " << sc.value << "\n";
541                result = false;
542            } else if (sc.negated && sc.value == status) {
543                std::cerr << "Fail: incorrect signal received: "
544                          << status << ", expected: "
545                          << "anything else\n";
546                result = false;
547            } else
548                result = true;
549        } else if (cr.signaled() && sc.value == INT_MIN) {
550            result = true;
551        } else {
552            std::cerr << "Fail: program did not receive a signal\n";
553            result = false;
554        }
555    } else {
556        UNREACHABLE;
557        result = false;
558    }
559
560    if (result == false) {
561        std::cerr << "stdout:\n";
562        cat_file(atf::fs::path(cr.stdout_path()));
563        std::cerr << "\n";
564
565        std::cerr << "stderr:\n";
566        cat_file(atf::fs::path(cr.stderr_path()));
567        std::cerr << "\n";
568    }
569
570    return result;
571}
572
573static
574bool
575run_status_checks(const std::vector< status_check >& checks,
576                  const atf::check::check_result& result)
577{
578    bool ok = false;
579
580    for (std::vector< status_check >::const_iterator iter = checks.begin();
581         !ok && iter != checks.end(); iter++) {
582         ok |= run_status_check(*iter, result);
583    }
584
585    return ok;
586}
587
588static
589bool
590run_output_check(const output_check oc, const atf::fs::path& path,
591                 const std::string& stdxxx)
592{
593    bool result;
594
595    if (oc.type == oc_empty) {
596        const bool is_empty = file_empty(path);
597        if (!oc.negated && !is_empty) {
598            std::cerr << "Fail: " << stdxxx << " not empty\n";
599            print_diff(atf::fs::path("/dev/null"), path);
600            result = false;
601        } else if (oc.negated && is_empty) {
602            std::cerr << "Fail: " << stdxxx << " is empty\n";
603            result = false;
604        } else
605            result = true;
606    } else if (oc.type == oc_file) {
607        const bool equals = compare_files(path, atf::fs::path(oc.value));
608        if (!oc.negated && !equals) {
609            std::cerr << "Fail: " << stdxxx << " does not match golden "
610                "output\n";
611            print_diff(atf::fs::path(oc.value), path);
612            result = false;
613        } else if (oc.negated && equals) {
614            std::cerr << "Fail: " << stdxxx << " matches golden output\n";
615            cat_file(atf::fs::path(oc.value));
616            result = false;
617        } else
618            result = true;
619    } else if (oc.type == oc_ignore) {
620        result = true;
621    } else if (oc.type == oc_inline) {
622        atf::fs::path path2 = atf::fs::path(atf::config::get("atf_workdir"))
623                              / "inline.XXXXXX";
624        temp_file temp(path2);
625        temp.write(decode(oc.value));
626        temp.close();
627
628        const bool equals = compare_files(path, temp.get_path());
629        if (!oc.negated && !equals) {
630            std::cerr << "Fail: " << stdxxx << " does not match expected "
631                "value\n";
632            print_diff(temp.get_path(), path);
633            result = false;
634        } else if (oc.negated && equals) {
635            std::cerr << "Fail: " << stdxxx << " matches expected value\n";
636            cat_file(temp.get_path());
637            result = false;
638        } else
639            result = true;
640    } else if (oc.type == oc_match) {
641        const bool matches = grep_file(path, oc.value);
642        if (!oc.negated && !matches) {
643            std::cerr << "Fail: regexp " + oc.value + " not in " << stdxxx
644                      << "\n";
645            cat_file(path);
646            result = false;
647        } else if (oc.negated && matches) {
648            std::cerr << "Fail: regexp " + oc.value + " is in " << stdxxx
649                      << "\n";
650            cat_file(path);
651            result = false;
652        } else
653            result = true;
654    } else if (oc.type == oc_save) {
655        INV(!oc.negated);
656        std::ifstream ifs(path.c_str(), std::fstream::binary);
657        ifs >> std::noskipws;
658        std::istream_iterator< char > begin(ifs), end;
659
660        std::ofstream ofs(oc.value.c_str(), std::fstream::binary
661                                     | std::fstream::trunc);
662        std::ostream_iterator <char> obegin(ofs);
663
664        std::copy(begin, end, obegin);
665        result = true;
666    } else {
667        UNREACHABLE;
668        result = false;
669    }
670
671    return result;
672}
673
674static
675bool
676run_output_checks(const std::vector< output_check >& checks,
677                  const atf::fs::path& path, const std::string& stdxxx)
678{
679    bool ok = true;
680
681    for (std::vector< output_check >::const_iterator iter = checks.begin();
682         iter != checks.end(); iter++) {
683         ok &= run_output_check(*iter, path, stdxxx);
684    }
685
686    return ok;
687}
688
689// ------------------------------------------------------------------------
690// The "atf_check" application.
691// ------------------------------------------------------------------------
692
693namespace {
694
695class atf_check : public atf::application::app {
696    bool m_xflag;
697
698    std::vector< status_check > m_status_checks;
699    std::vector< output_check > m_stdout_checks;
700    std::vector< output_check > m_stderr_checks;
701
702    static const char* m_description;
703
704    bool run_output_checks(const atf::check::check_result&,
705                           const std::string&) const;
706
707    std::string specific_args(void) const;
708    options_set specific_options(void) const;
709    void process_option(int, const char*);
710    void process_option_s(const std::string&);
711
712public:
713    atf_check(void);
714    int main(void);
715};
716
717} // anonymous namespace
718
719const char* atf_check::m_description =
720    "atf-check executes given command and analyzes its results.";
721
722atf_check::atf_check(void) :
723    app(m_description, "atf-check(1)", "atf(7)"),
724    m_xflag(false)
725{
726}
727
728bool
729atf_check::run_output_checks(const atf::check::check_result& r,
730                             const std::string& stdxxx)
731    const
732{
733    if (stdxxx == "stdout") {
734        return ::run_output_checks(m_stdout_checks,
735            atf::fs::path(r.stdout_path()), "stdout");
736    } else if (stdxxx == "stderr") {
737        return ::run_output_checks(m_stderr_checks,
738            atf::fs::path(r.stderr_path()), "stderr");
739    } else {
740        UNREACHABLE;
741        return false;
742    }
743}
744
745std::string
746atf_check::specific_args(void)
747    const
748{
749    return "<command>";
750}
751
752atf_check::options_set
753atf_check::specific_options(void)
754    const
755{
756    using atf::application::option;
757    options_set opts;
758
759    opts.insert(option('s', "qual:value", "Handle status. Qualifier "
760                "must be one of: ignore exit:<num> signal:<name|num>"));
761    opts.insert(option('o', "action:arg", "Handle stdout. Action must be "
762                "one of: empty ignore file:<path> inline:<val> match:regexp "
763                "save:<path>"));
764    opts.insert(option('e', "action:arg", "Handle stderr. Action must be "
765                "one of: empty ignore file:<path> inline:<val> match:regexp "
766                "save:<path>"));
767    opts.insert(option('x', "", "Execute command as a shell command"));
768
769    return opts;
770}
771
772void
773atf_check::process_option(int ch, const char* arg)
774{
775    switch (ch) {
776    case 's':
777        m_status_checks.push_back(parse_status_check_arg(arg));
778        break;
779
780    case 'o':
781        m_stdout_checks.push_back(parse_output_check_arg(arg));
782        break;
783
784    case 'e':
785        m_stderr_checks.push_back(parse_output_check_arg(arg));
786        break;
787
788    case 'x':
789        m_xflag = true;
790        break;
791
792    default:
793        UNREACHABLE;
794    }
795}
796
797int
798atf_check::main(void)
799{
800    if (m_argc < 1)
801        throw atf::application::usage_error("No command specified");
802
803    int status = EXIT_FAILURE;
804
805    std::auto_ptr< atf::check::check_result > r =
806        m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
807
808    if (m_status_checks.empty())
809        m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS));
810    else if (m_status_checks.size() > 1) {
811        // TODO: Remove this restriction.
812        throw atf::application::usage_error("Cannot specify -s more than once");
813    }
814
815    if (m_stdout_checks.empty())
816        m_stdout_checks.push_back(output_check(oc_empty, false, ""));
817    if (m_stderr_checks.empty())
818        m_stderr_checks.push_back(output_check(oc_empty, false, ""));
819
820    if ((run_status_checks(m_status_checks, *r) == false) ||
821        (run_output_checks(*r, "stderr") == false) ||
822        (run_output_checks(*r, "stdout") == false))
823        status = EXIT_FAILURE;
824    else
825        status = EXIT_SUCCESS;
826
827    return status;
828}
829
830int
831main(int argc, char* const* argv)
832{
833    return atf_check().run(argc, argv);
834}
835