1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30#if defined(HAVE_CONFIG_H)
31#include "bconfig.h"
32#endif
33
34extern "C" {
35#include <sys/types.h>
36#include <sys/stat.h>
37#include <sys/wait.h>
38#include <unistd.h>
39}
40
41#include <algorithm>
42#include <cerrno>
43#include <cstdlib>
44#include <cstring>
45#include <fstream>
46#include <iostream>
47#include <map>
48#include <string>
49
50#include "atf-c++/detail/application.hpp"
51#include "atf-c++/config.hpp"
52#include "atf-c++/tests.hpp"
53
54#include "atf-c++/detail/env.hpp"
55#include "atf-c++/detail/exceptions.hpp"
56#include "atf-c++/detail/fs.hpp"
57#include "atf-c++/detail/parser.hpp"
58#include "atf-c++/detail/process.hpp"
59#include "atf-c++/detail/sanity.hpp"
60#include "atf-c++/detail/text.hpp"
61
62#include "atffile.hpp"
63#include "config.hpp"
64#include "fs.hpp"
65#include "requirements.hpp"
66#include "test-program.hpp"
67
68namespace impl = atf::atf_run;
69
70class atf_run : public atf::application::app {
71    static const char* m_description;
72
73    atf::tests::vars_map m_cmdline_vars;
74
75    static atf::tests::vars_map::value_type parse_var(const std::string&);
76
77    void process_option(int, const char*);
78    std::string specific_args(void) const;
79    options_set specific_options(void) const;
80
81    void parse_vflag(const std::string&);
82
83    std::vector< std::string > conf_args(void) const;
84
85    size_t count_tps(std::vector< std::string >) const;
86
87    int run_test(const atf::fs::path&, impl::atf_tps_writer&,
88                 const atf::tests::vars_map&);
89    int run_test_directory(const atf::fs::path&, impl::atf_tps_writer&);
90    int run_test_program(const atf::fs::path&, impl::atf_tps_writer&,
91                         const atf::tests::vars_map&);
92
93    impl::test_case_result get_test_case_result(const std::string&,
94        const atf::process::status&, const atf::fs::path&) const;
95
96public:
97    atf_run(void);
98
99    int main(void);
100};
101
102const char* atf_run::m_description =
103    "atf-run is a tool that runs tests programs and collects their "
104    "results.";
105
106atf_run::atf_run(void) :
107    app(m_description, "atf-run(1)", "atf(7)")
108{
109}
110
111void
112atf_run::process_option(int ch, const char* arg)
113{
114    switch (ch) {
115    case 'v':
116        parse_vflag(arg);
117        break;
118
119    default:
120        UNREACHABLE;
121    }
122}
123
124std::string
125atf_run::specific_args(void)
126    const
127{
128    return "[test-program1 .. test-programN]";
129}
130
131atf_run::options_set
132atf_run::specific_options(void)
133    const
134{
135    using atf::application::option;
136    options_set opts;
137    opts.insert(option('v', "var=value", "Sets the configuration variable "
138                                         "`var' to `value'; overrides "
139                                         "values in configuration files"));
140    return opts;
141}
142
143void
144atf_run::parse_vflag(const std::string& str)
145{
146    if (str.empty())
147        throw std::runtime_error("-v requires a non-empty argument");
148
149    std::vector< std::string > ws = atf::text::split(str, "=");
150    if (ws.size() == 1 && str[str.length() - 1] == '=') {
151        m_cmdline_vars[ws[0]] = "";
152    } else {
153        if (ws.size() != 2)
154            throw std::runtime_error("-v requires an argument of the form "
155                                     "var=value");
156
157        m_cmdline_vars[ws[0]] = ws[1];
158    }
159}
160
161int
162atf_run::run_test(const atf::fs::path& tp,
163                  impl::atf_tps_writer& w,
164                  const atf::tests::vars_map& config)
165{
166    atf::fs::file_info fi(tp);
167
168    int errcode;
169    if (fi.get_type() == atf::fs::file_info::dir_type)
170        errcode = run_test_directory(tp, w);
171    else {
172        const atf::tests::vars_map effective_config =
173            impl::merge_configs(config, m_cmdline_vars);
174
175        errcode = run_test_program(tp, w, effective_config);
176    }
177    return errcode;
178}
179
180int
181atf_run::run_test_directory(const atf::fs::path& tp,
182                            impl::atf_tps_writer& w)
183{
184    impl::atffile af = impl::read_atffile(tp / "Atffile");
185
186    atf::tests::vars_map test_suite_vars;
187    {
188        atf::tests::vars_map::const_iterator iter =
189            af.props().find("test-suite");
190        INV(iter != af.props().end());
191        test_suite_vars = impl::read_config_files((*iter).second);
192    }
193
194    bool ok = true;
195    for (std::vector< std::string >::const_iterator iter = af.tps().begin();
196         iter != af.tps().end(); iter++) {
197        const bool result = run_test(tp / *iter, w,
198            impl::merge_configs(af.conf(), test_suite_vars));
199        ok &= (result == EXIT_SUCCESS);
200    }
201
202    return ok ? EXIT_SUCCESS : EXIT_FAILURE;
203}
204
205impl::test_case_result
206atf_run::get_test_case_result(const std::string& broken_reason,
207                              const atf::process::status& s,
208                              const atf::fs::path& resfile)
209    const
210{
211    using atf::text::to_string;
212    using impl::read_test_case_result;
213    using impl::test_case_result;
214
215    if (!broken_reason.empty()) {
216        test_case_result tcr;
217
218        try {
219            tcr = read_test_case_result(resfile);
220
221            if (tcr.state() == "expected_timeout") {
222                return tcr;
223            } else {
224                return test_case_result("failed", -1, broken_reason);
225            }
226        } catch (const std::runtime_error&) {
227            return test_case_result("failed", -1, broken_reason);
228        }
229    }
230
231    if (s.exited()) {
232        test_case_result tcr;
233
234        try {
235            tcr = read_test_case_result(resfile);
236        } catch (const std::runtime_error& e) {
237            return test_case_result("failed", -1, "Test case exited "
238                "normally but failed to create the results file: " +
239                std::string(e.what()));
240        }
241
242        if (tcr.state() == "expected_death") {
243            return tcr;
244        } else if (tcr.state() == "expected_exit") {
245            if (tcr.value() == -1 || s.exitstatus() == tcr.value())
246                return tcr;
247            else
248                return test_case_result("failed", -1, "Test case was "
249                    "expected to exit with a " + to_string(tcr.value()) +
250                    " error code but returned " + to_string(s.exitstatus()));
251        } else if (tcr.state() == "expected_failure") {
252            if (s.exitstatus() == EXIT_SUCCESS)
253                return tcr;
254            else
255                return test_case_result("failed", -1, "Test case returned an "
256                    "error in expected_failure mode but it should not have");
257        } else if (tcr.state() == "expected_signal") {
258            return test_case_result("failed", -1, "Test case exited cleanly "
259                "but was expected to receive a signal");
260        } else if (tcr.state() == "failed") {
261            if (s.exitstatus() == EXIT_SUCCESS)
262                return test_case_result("failed", -1, "Test case "
263                    "exited successfully but reported failure");
264            else
265                return tcr;
266        } else if (tcr.state() == "passed") {
267            if (s.exitstatus() == EXIT_SUCCESS)
268                return tcr;
269            else
270                return test_case_result("failed", -1, "Test case exited as "
271                    "passed but reported an error");
272        } else if (tcr.state() == "skipped") {
273            if (s.exitstatus() == EXIT_SUCCESS)
274                return tcr;
275            else
276                return test_case_result("failed", -1, "Test case exited as "
277                    "skipped but reported an error");
278        }
279    } else if (s.signaled()) {
280        test_case_result tcr;
281
282        try {
283            tcr = read_test_case_result(resfile);
284        } catch (const std::runtime_error&) {
285            return test_case_result("failed", -1, "Test program received "
286                "signal " + atf::text::to_string(s.termsig()) +
287                (s.coredump() ? " (core dumped)" : ""));
288        }
289
290        if (tcr.state() == "expected_death") {
291            return tcr;
292        } else if (tcr.state() == "expected_signal") {
293            if (tcr.value() == -1 || s.termsig() == tcr.value())
294                return tcr;
295            else
296                return test_case_result("failed", -1, "Test case was "
297                    "expected to exit due to a " + to_string(tcr.value()) +
298                    " signal but got " + to_string(s.termsig()));
299        } else {
300            return test_case_result("failed", -1, "Test program received "
301                "signal " + atf::text::to_string(s.termsig()) +
302                (s.coredump() ? " (core dumped)" : "") + " and created a "
303                "bogus results file");
304        }
305    }
306    UNREACHABLE;
307    return test_case_result();
308}
309
310int
311atf_run::run_test_program(const atf::fs::path& tp,
312                          impl::atf_tps_writer& w,
313                          const atf::tests::vars_map& config)
314{
315    int errcode = EXIT_SUCCESS;
316
317    impl::metadata md;
318    try {
319        md = impl::get_metadata(tp, config);
320    } catch (const atf::parser::format_error& e) {
321        w.start_tp(tp.str(), 0);
322        w.end_tp("Invalid format for test case list: " + std::string(e.what()));
323        return EXIT_FAILURE;
324    } catch (const atf::parser::parse_errors& e) {
325        const std::string reason = atf::text::join(e, "; ");
326        w.start_tp(tp.str(), 0);
327        w.end_tp("Invalid format for test case list: " + reason);
328        return EXIT_FAILURE;
329    }
330
331    impl::temp_dir resdir(atf::fs::path(atf::config::get("atf_workdir")) /
332                          "atf-run.XXXXXX");
333
334    w.start_tp(tp.str(), md.test_cases.size());
335    if (md.test_cases.empty()) {
336        w.end_tp("Bogus test program: reported 0 test cases");
337        errcode = EXIT_FAILURE;
338    } else {
339        for (std::map< std::string, atf::tests::vars_map >::const_iterator iter
340             = md.test_cases.begin(); iter != md.test_cases.end(); iter++) {
341            const std::string& tcname = (*iter).first;
342            const atf::tests::vars_map& tcmd = (*iter).second;
343
344            w.start_tc(tcname);
345
346            try {
347                const std::string& reqfail = impl::check_requirements(
348                    tcmd, config);
349                if (!reqfail.empty()) {
350                    w.end_tc("skipped", reqfail);
351                    continue;
352                }
353            } catch (const std::runtime_error& e) {
354                w.end_tc("failed", e.what());
355                errcode = EXIT_FAILURE;
356                continue;
357            }
358
359            const std::pair< int, int > user = impl::get_required_user(
360                tcmd, config);
361
362            atf::fs::path resfile = resdir.get_path() / "tcr";
363            INV(!atf::fs::exists(resfile));
364            try {
365                const bool has_cleanup = atf::text::to_bool(
366                    (*tcmd.find("has.cleanup")).second);
367
368                impl::temp_dir workdir(atf::fs::path(atf::config::get(
369                    "atf_workdir")) / "atf-run.XXXXXX");
370                if (user.first != -1 && user.second != -1) {
371                    if (::chown(workdir.get_path().c_str(), user.first,
372                                user.second) == -1) {
373                        throw atf::system_error("chmod(" +
374                            workdir.get_path().str() + ")", "chmod(2) failed",
375                            errno);
376                    }
377                    resfile = workdir.get_path() / "tcr";
378                }
379
380                std::pair< std::string, const atf::process::status > s =
381                    impl::run_test_case(tp, tcname, "body", tcmd, config,
382                                            resfile, workdir.get_path(), w);
383                if (has_cleanup)
384                    (void)impl::run_test_case(tp, tcname, "cleanup", tcmd,
385                            config, resfile, workdir.get_path(), w);
386
387                // TODO: Force deletion of workdir.
388
389                impl::test_case_result tcr = get_test_case_result(s.first,
390                    s.second, resfile);
391
392                w.end_tc(tcr.state(), tcr.reason());
393                if (tcr.state() == "failed")
394                    errcode = EXIT_FAILURE;
395            } catch (...) {
396                if (atf::fs::exists(resfile))
397                    atf::fs::remove(resfile);
398                throw;
399            }
400            if (atf::fs::exists(resfile))
401                atf::fs::remove(resfile);
402
403        }
404        w.end_tp("");
405    }
406
407    return errcode;
408}
409
410size_t
411atf_run::count_tps(std::vector< std::string > tps)
412    const
413{
414    size_t ntps = 0;
415
416    for (std::vector< std::string >::const_iterator iter = tps.begin();
417         iter != tps.end(); iter++) {
418        atf::fs::path tp(*iter);
419        atf::fs::file_info fi(tp);
420
421        if (fi.get_type() == atf::fs::file_info::dir_type) {
422            impl::atffile af = impl::read_atffile(tp / "Atffile");
423            std::vector< std::string > aux = af.tps();
424            for (std::vector< std::string >::iterator i2 = aux.begin();
425                 i2 != aux.end(); i2++)
426                *i2 = (tp / *i2).str();
427            ntps += count_tps(aux);
428        } else
429            ntps++;
430    }
431
432    return ntps;
433}
434
435static
436void
437call_hook(const std::string& tool, const std::string& hook)
438{
439    const atf::fs::path sh(atf::config::get("atf_shell"));
440    const atf::fs::path hooks =
441        atf::fs::path(atf::config::get("atf_pkgdatadir")) / (tool + ".hooks");
442
443    const atf::process::status s =
444        atf::process::exec(sh,
445                           atf::process::argv_array(sh.c_str(), hooks.c_str(),
446                                                    hook.c_str(), NULL),
447                           atf::process::stream_inherit(),
448                           atf::process::stream_inherit());
449
450
451    if (!s.exited() || s.exitstatus() != EXIT_SUCCESS)
452        throw std::runtime_error("Failed to run the '" + hook + "' hook "
453                                 "for '" + tool + "'");
454}
455
456int
457atf_run::main(void)
458{
459    impl::atffile af = impl::read_atffile(atf::fs::path("Atffile"));
460
461    std::vector< std::string > tps;
462    tps = af.tps();
463    if (m_argc >= 1) {
464        // TODO: Ensure that the given test names are listed in the
465        // Atffile.  Take into account that the file can be using globs.
466        tps.clear();
467        for (int i = 0; i < m_argc; i++)
468            tps.push_back(m_argv[i]);
469    }
470
471    // Read configuration data for this test suite.
472    atf::tests::vars_map test_suite_vars;
473    {
474        atf::tests::vars_map::const_iterator iter =
475            af.props().find("test-suite");
476        INV(iter != af.props().end());
477        test_suite_vars = impl::read_config_files((*iter).second);
478    }
479
480    impl::atf_tps_writer w(std::cout);
481    call_hook("atf-run", "info_start_hook");
482    w.ntps(count_tps(tps));
483
484    bool ok = true;
485    for (std::vector< std::string >::const_iterator iter = tps.begin();
486         iter != tps.end(); iter++) {
487        const bool result = run_test(atf::fs::path(*iter), w,
488            impl::merge_configs(af.conf(), test_suite_vars));
489        ok &= (result == EXIT_SUCCESS);
490    }
491
492    call_hook("atf-run", "info_end_hook");
493
494    return ok ? EXIT_SUCCESS : EXIT_FAILURE;
495}
496
497int
498main(int argc, char* const* argv)
499{
500    return atf_run().run(argc, argv);
501}
502