atf-run.cpp revision 1.2.4.1 
1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2007 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <sys/types.h>
32#include <sys/param.h>
33#include <sys/stat.h>
34#include <sys/wait.h>
35#include <unistd.h>
36}
37
38#include <algorithm>
39#include <cassert>
40#include <cerrno>
41#include <cstdlib>
42#include <cstring>
43#include <fstream>
44#include <iostream>
45#include <map>
46#include <string>
47
48#include "application.hpp"
49#include "atffile.hpp"
50#include "config.hpp"
51#include "config_file.hpp"
52#include "env.hpp"
53#include "exceptions.hpp"
54#include "fs.hpp"
55#include "parser.hpp"
56#include "process.hpp"
57#include "requirements.hpp"
58#include "test-program.hpp"
59#include "text.hpp"
60
61namespace {
62
63typedef std::map< std::string, std::string > vars_map;
64
65} // anonymous namespace
66
67class atf_run : public tools::application::app {
68    static const char* m_description;
69
70    vars_map m_cmdline_vars;
71
72    static vars_map::value_type parse_var(const std::string&);
73
74    void process_option(int, const char*);
75    std::string specific_args(void) const;
76    options_set specific_options(void) const;
77
78    void parse_vflag(const std::string&);
79
80    std::vector< std::string > conf_args(void) const;
81
82    size_t count_tps(std::vector< std::string >) const;
83
84    int run_test(const tools::fs::path&, tools::test_program::atf_tps_writer&,
85                 const vars_map&);
86    int run_test_directory(const tools::fs::path&,
87                           tools::test_program::atf_tps_writer&);
88    int run_test_program(const tools::fs::path&,
89                         tools::test_program::atf_tps_writer&,
90                         const vars_map&);
91
92    tools::test_program::test_case_result get_test_case_result(
93        const std::string&, const tools::process::status&,
94        const tools::fs::path&) const;
95
96public:
97    atf_run(void);
98
99    int main(void);
100};
101
102static void
103sanitize_gdb_env(void)
104{
105    try {
106        tools::env::unset("TERM");
107    } catch (...) {
108        // Just swallow exceptions here; they cannot propagate into C, which
109        // is where this function is called from, and even if these exceptions
110        // appear they are benign.
111    }
112}
113
114static void
115dump_stacktrace(const tools::fs::path& tp, const tools::process::status& s,
116                const tools::fs::path& workdir,
117                tools::test_program::atf_tps_writer& w)
118{
119    assert(s.signaled() && s.coredump());
120
121    w.stderr_tc("Test program crashed; attempting to get stack trace");
122
123    const tools::fs::path corename = workdir /
124        (tp.leaf_name().substr(0, MAXCOMLEN) + ".core");
125    if (!tools::fs::exists(corename)) {
126        w.stderr_tc("Expected file " + corename.str() + " not found");
127        return;
128    }
129
130    const tools::fs::path gdb(GDB);
131    const tools::fs::path gdbout = workdir / "gdb.out";
132    const tools::process::argv_array args(gdb.leaf_name().c_str(), "-batch",
133                                        "-q", "-ex", "bt", tp.c_str(),
134                                        corename.c_str(), NULL);
135    tools::process::status status = tools::process::exec(
136        gdb, args,
137        tools::process::stream_redirect_path(gdbout),
138        tools::process::stream_redirect_path(tools::fs::path("/dev/null")),
139        sanitize_gdb_env);
140    if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) {
141        w.stderr_tc("Execution of " GDB " failed");
142        return;
143    }
144
145    std::ifstream input(gdbout.c_str());
146    if (input) {
147        std::string line;
148        while (std::getline(input, line).good())
149            w.stderr_tc(line);
150        input.close();
151    }
152
153    w.stderr_tc("Stack trace complete");
154}
155
156const char* atf_run::m_description =
157    "atf-run is a tool that runs tests programs and collects their "
158    "results.";
159
160atf_run::atf_run(void) :
161    app(m_description, "atf-run(1)", "atf(7)")
162{
163}
164
165void
166atf_run::process_option(int ch, const char* arg)
167{
168    switch (ch) {
169    case 'v':
170        parse_vflag(arg);
171        break;
172
173    default:
174        std::abort();
175    }
176}
177
178std::string
179atf_run::specific_args(void)
180    const
181{
182    return "[test-program1 .. test-programN]";
183}
184
185atf_run::options_set
186atf_run::specific_options(void)
187    const
188{
189    using tools::application::option;
190    options_set opts;
191    opts.insert(option('v', "var=value", "Sets the configuration variable "
192                                         "`var' to `value'; overrides "
193                                         "values in configuration files"));
194    return opts;
195}
196
197void
198atf_run::parse_vflag(const std::string& str)
199{
200    if (str.empty())
201        throw std::runtime_error("-v requires a non-empty argument");
202
203    std::vector< std::string > ws = tools::text::split(str, "=");
204    if (ws.size() == 1 && str[str.length() - 1] == '=') {
205        m_cmdline_vars[ws[0]] = "";
206    } else {
207        if (ws.size() != 2)
208            throw std::runtime_error("-v requires an argument of the form "
209                                     "var=value");
210
211        m_cmdline_vars[ws[0]] = ws[1];
212    }
213}
214
215int
216atf_run::run_test(const tools::fs::path& tp,
217                  tools::test_program::atf_tps_writer& w,
218                  const vars_map& config)
219{
220    tools::fs::file_info fi(tp);
221
222    int errcode;
223    if (fi.get_type() == tools::fs::file_info::dir_type)
224        errcode = run_test_directory(tp, w);
225    else {
226        const vars_map effective_config =
227            tools::config_file::merge_configs(config, m_cmdline_vars);
228
229        errcode = run_test_program(tp, w, effective_config);
230    }
231    return errcode;
232}
233
234int
235atf_run::run_test_directory(const tools::fs::path& tp,
236                            tools::test_program::atf_tps_writer& w)
237{
238    tools::atffile af = tools::read_atffile(tp / "Atffile");
239
240    vars_map test_suite_vars;
241    {
242        vars_map::const_iterator iter = af.props().find("test-suite");
243        assert(iter != af.props().end());
244        test_suite_vars = tools::config_file::read_config_files((*iter).second);
245    }
246
247    bool ok = true;
248    for (std::vector< std::string >::const_iterator iter = af.tps().begin();
249         iter != af.tps().end(); iter++) {
250        const bool result = run_test(tp / *iter, w,
251            tools::config_file::merge_configs(af.conf(), test_suite_vars));
252        ok &= (result == EXIT_SUCCESS);
253    }
254
255    return ok ? EXIT_SUCCESS : EXIT_FAILURE;
256}
257
258tools::test_program::test_case_result
259atf_run::get_test_case_result(const std::string& broken_reason,
260                              const tools::process::status& s,
261                              const tools::fs::path& resfile)
262    const
263{
264    using tools::text::to_string;
265    using tools::test_program::read_test_case_result;
266    using tools::test_program::test_case_result;
267
268    if (!broken_reason.empty()) {
269        test_case_result tcr;
270
271        try {
272            tcr = read_test_case_result(resfile);
273
274            if (tcr.state() == "expected_timeout") {
275                return tcr;
276            } else {
277                return test_case_result("failed", -1, broken_reason);
278            }
279        } catch (const std::runtime_error&) {
280            return test_case_result("failed", -1, broken_reason);
281        }
282    }
283
284    if (s.exited()) {
285        test_case_result tcr;
286
287        try {
288            tcr = read_test_case_result(resfile);
289        } catch (const std::runtime_error& e) {
290            return test_case_result("failed", -1, "Test case exited "
291                "normally but failed to create the results file: " +
292                std::string(e.what()));
293        }
294
295        if (tcr.state() == "expected_death") {
296            return tcr;
297        } else if (tcr.state() == "expected_exit") {
298            if (tcr.value() == -1 || s.exitstatus() == tcr.value())
299                return tcr;
300            else
301                return test_case_result("failed", -1, "Test case was "
302                    "expected to exit with a " + to_string(tcr.value()) +
303                    " error code but returned " + to_string(s.exitstatus()));
304        } else if (tcr.state() == "expected_failure") {
305            if (s.exitstatus() == EXIT_SUCCESS)
306                return tcr;
307            else
308                return test_case_result("failed", -1, "Test case returned an "
309                    "error in expected_failure mode but it should not have");
310        } else if (tcr.state() == "expected_signal") {
311            return test_case_result("failed", -1, "Test case exited cleanly "
312                "but was expected to receive a signal");
313        } else if (tcr.state() == "failed") {
314            if (s.exitstatus() == EXIT_SUCCESS)
315                return test_case_result("failed", -1, "Test case "
316                    "exited successfully but reported failure");
317            else
318                return tcr;
319        } else if (tcr.state() == "passed") {
320            if (s.exitstatus() == EXIT_SUCCESS)
321                return tcr;
322            else
323                return test_case_result("failed", -1, "Test case exited as "
324                    "passed but reported an error");
325        } else if (tcr.state() == "skipped") {
326            if (s.exitstatus() == EXIT_SUCCESS)
327                return tcr;
328            else
329                return test_case_result("failed", -1, "Test case exited as "
330                    "skipped but reported an error");
331        }
332    } else if (s.signaled()) {
333        test_case_result tcr;
334
335        try {
336            tcr = read_test_case_result(resfile);
337        } catch (const std::runtime_error&) {
338            return test_case_result("failed", -1, "Test program received "
339                "signal " + tools::text::to_string(s.termsig()) +
340                (s.coredump() ? " (core dumped)" : ""));
341        }
342
343        if (tcr.state() == "expected_death") {
344            return tcr;
345        } else if (tcr.state() == "expected_signal") {
346            if (tcr.value() == -1 || s.termsig() == tcr.value())
347                return tcr;
348            else
349                return test_case_result("failed", -1, "Test case was "
350                    "expected to exit due to a " + to_string(tcr.value()) +
351                    " signal but got " + to_string(s.termsig()));
352        } else {
353            return test_case_result("failed", -1, "Test program received "
354                "signal " + tools::text::to_string(s.termsig()) +
355                (s.coredump() ? " (core dumped)" : "") + " and created a "
356                "bogus results file");
357        }
358    }
359    std::abort();
360    return test_case_result();
361}
362
363int
364atf_run::run_test_program(const tools::fs::path& tp,
365                          tools::test_program::atf_tps_writer& w,
366                          const vars_map& config)
367{
368    int errcode = EXIT_SUCCESS;
369
370    tools::test_program::metadata md;
371    try {
372        md = tools::test_program::get_metadata(tp, config);
373    } catch (const tools::parser::format_error& e) {
374        w.start_tp(tp.str(), 0);
375        w.end_tp("Invalid format for test case list: " + std::string(e.what()));
376        return EXIT_FAILURE;
377    } catch (const tools::parser::parse_errors& e) {
378        const std::string reason = tools::text::join(e, "; ");
379        w.start_tp(tp.str(), 0);
380        w.end_tp("Invalid format for test case list: " + reason);
381        return EXIT_FAILURE;
382    }
383
384    tools::fs::temp_dir resdir(
385        tools::fs::path(tools::config::get("atf_workdir")) / "atf-run.XXXXXX");
386
387    w.start_tp(tp.str(), md.test_cases.size());
388    if (md.test_cases.empty()) {
389        w.end_tp("Bogus test program: reported 0 test cases");
390        errcode = EXIT_FAILURE;
391    } else {
392        for (std::map< std::string, vars_map >::const_iterator iter
393             = md.test_cases.begin(); iter != md.test_cases.end(); iter++) {
394            const std::string& tcname = (*iter).first;
395            const vars_map& tcmd = (*iter).second;
396
397            w.start_tc(tcname);
398
399            try {
400                const std::string& reqfail = tools::check_requirements(
401                    tcmd, config);
402                if (!reqfail.empty()) {
403                    w.end_tc("skipped", reqfail);
404                    continue;
405                }
406            } catch (const std::runtime_error& e) {
407                w.end_tc("failed", e.what());
408                errcode = EXIT_FAILURE;
409                continue;
410            }
411
412            const std::pair< int, int > user = tools::get_required_user(
413                tcmd, config);
414
415            tools::fs::path resfile = resdir.get_path() / "tcr";
416            assert(!tools::fs::exists(resfile));
417            try {
418                const bool has_cleanup = tools::text::to_bool(
419                    (*tcmd.find("has.cleanup")).second);
420
421                tools::fs::temp_dir workdir(tools::fs::path(tools::config::get(
422                    "atf_workdir")) / "atf-run.XXXXXX");
423                if (user.first != -1 && user.second != -1) {
424                    if (::chown(workdir.get_path().c_str(), user.first,
425                                user.second) == -1) {
426                        throw tools::system_error("chown(" +
427                            workdir.get_path().str() + ")", "chown(2) failed",
428                            errno);
429                    }
430                    resfile = workdir.get_path() / "tcr";
431                }
432
433                std::pair< std::string, const tools::process::status > s =
434                    tools::test_program::run_test_case(
435                        tp, tcname, "body", tcmd, config,
436                        resfile, workdir.get_path(), w);
437                if (s.second.signaled() && s.second.coredump())
438                    dump_stacktrace(tp, s.second, workdir.get_path(), w);
439                if (has_cleanup)
440                    (void)tools::test_program::run_test_case(
441                        tp, tcname, "cleanup", tcmd,
442                        config, resfile, workdir.get_path(), w);
443
444                // TODO: Force deletion of workdir.
445
446                tools::test_program::test_case_result tcr =
447                    get_test_case_result(s.first, s.second, resfile);
448
449                w.end_tc(tcr.state(), tcr.reason());
450                if (tcr.state() == "failed")
451                    errcode = EXIT_FAILURE;
452            } catch (...) {
453                if (tools::fs::exists(resfile))
454                    tools::fs::remove(resfile);
455                throw;
456            }
457            if (tools::fs::exists(resfile))
458                tools::fs::remove(resfile);
459
460        }
461        w.end_tp("");
462    }
463
464    return errcode;
465}
466
467size_t
468atf_run::count_tps(std::vector< std::string > tps)
469    const
470{
471    size_t ntps = 0;
472
473    for (std::vector< std::string >::const_iterator iter = tps.begin();
474         iter != tps.end(); iter++) {
475        tools::fs::path tp(*iter);
476        tools::fs::file_info fi(tp);
477
478        if (fi.get_type() == tools::fs::file_info::dir_type) {
479            tools::atffile af = tools::read_atffile(tp / "Atffile");
480            std::vector< std::string > aux = af.tps();
481            for (std::vector< std::string >::iterator i2 = aux.begin();
482                 i2 != aux.end(); i2++)
483                *i2 = (tp / *i2).str();
484            ntps += count_tps(aux);
485        } else
486            ntps++;
487    }
488
489    return ntps;
490}
491
492static
493void
494call_hook(const std::string& tool, const std::string& hook)
495{
496    const tools::fs::path sh(tools::config::get("atf_shell"));
497    const tools::fs::path hooks =
498        tools::fs::path(tools::config::get("atf_pkgdatadir")) / (tool + ".hooks");
499
500    const tools::process::status s =
501        tools::process::exec(sh,
502                           tools::process::argv_array(sh.c_str(), hooks.c_str(),
503                                                    hook.c_str(), NULL),
504                           tools::process::stream_inherit(),
505                           tools::process::stream_inherit());
506
507
508    if (!s.exited() || s.exitstatus() != EXIT_SUCCESS)
509        throw std::runtime_error("Failed to run the '" + hook + "' hook "
510                                 "for '" + tool + "'");
511}
512
513int
514atf_run::main(void)
515{
516    tools::atffile af = tools::read_atffile(tools::fs::path("Atffile"));
517
518    std::vector< std::string > tps;
519    tps = af.tps();
520    if (m_argc >= 1) {
521        // TODO: Ensure that the given test names are listed in the
522        // Atffile.  Take into account that the file can be using globs.
523        tps.clear();
524        for (int i = 0; i < m_argc; i++)
525            tps.push_back(m_argv[i]);
526    }
527
528    // Read configuration data for this test suite.
529    vars_map test_suite_vars;
530    {
531        vars_map::const_iterator iter = af.props().find("test-suite");
532        assert(iter != af.props().end());
533        test_suite_vars = tools::config_file::read_config_files((*iter).second);
534    }
535
536    tools::test_program::atf_tps_writer w(std::cout);
537    call_hook("atf-run", "info_start_hook");
538    w.ntps(count_tps(tps));
539
540    bool ok = true;
541    for (std::vector< std::string >::const_iterator iter = tps.begin();
542         iter != tps.end(); iter++) {
543        const bool result = run_test(tools::fs::path(*iter), w,
544            tools::config_file::merge_configs(af.conf(), test_suite_vars));
545        ok &= (result == EXIT_SUCCESS);
546    }
547
548    call_hook("atf-run", "info_end_hook");
549
550    return ok ? EXIT_SUCCESS : EXIT_FAILURE;
551}
552
553int
554main(int argc, char* const* argv)
555{
556    return atf_run().run(argc, argv);
557}
558