1// Copyright 2011 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "cli/common.hpp" 30 31#include <fstream> 32 33#include <atf-c++.hpp> 34 35#include "engine/exceptions.hpp" 36#include "engine/filters.hpp" 37#include "model/metadata.hpp" 38#include "model/test_program.hpp" 39#include "model/test_result.hpp" 40#include "store/layout.hpp" 41#include "utils/cmdline/exceptions.hpp" 42#include "utils/cmdline/globals.hpp" 43#include "utils/cmdline/options.hpp" 44#include "utils/cmdline/parser.ipp" 45#include "utils/cmdline/ui_mock.hpp" 46#include "utils/datetime.hpp" 47#include "utils/env.hpp" 48#include "utils/format/macros.hpp" 49#include "utils/fs/exceptions.hpp" 50#include "utils/fs/operations.hpp" 51#include "utils/fs/path.hpp" 52#include "utils/optional.ipp" 53#include "utils/sanity.hpp" 54 55namespace cmdline = utils::cmdline; 56namespace config = utils::config; 57namespace datetime = utils::datetime; 58namespace fs = utils::fs; 59namespace layout = store::layout; 60 61using utils::optional; 62 63 64namespace { 65 66 67/// Syntactic sugar to instantiate engine::test_filter objects. 68/// 69/// \param test_program Test program. 70/// \param test_case Test case. 71/// 72/// \return A \code test_filter \endcode object, based on \p test_program and 73/// \p test_case. 74inline engine::test_filter 75mkfilter(const char* test_program, const char* test_case) 76{ 77 return engine::test_filter(fs::path(test_program), test_case); 78} 79 80 81} // anonymous namespace 82 83 84ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__default); 85ATF_TEST_CASE_BODY(build_root_path__default) 86{ 87 std::map< std::string, std::vector< std::string > > options; 88 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 89 90 ATF_REQUIRE(!cli::build_root_path(mock_cmdline)); 91} 92 93 94ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__explicit); 95ATF_TEST_CASE_BODY(build_root_path__explicit) 96{ 97 std::map< std::string, std::vector< std::string > > options; 98 options["build-root"].push_back("/my//path"); 99 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 100 101 ATF_REQUIRE(cli::build_root_path(mock_cmdline)); 102 ATF_REQUIRE_EQ("/my/path", cli::build_root_path(mock_cmdline).get().str()); 103} 104 105 106ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__default); 107ATF_TEST_CASE_BODY(kyuafile_path__default) 108{ 109 std::map< std::string, std::vector< std::string > > options; 110 options["kyuafile"].push_back(cli::kyuafile_option.default_value()); 111 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 112 113 ATF_REQUIRE_EQ(cli::kyuafile_option.default_value(), 114 cli::kyuafile_path(mock_cmdline).str()); 115} 116 117 118ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__explicit); 119ATF_TEST_CASE_BODY(kyuafile_path__explicit) 120{ 121 std::map< std::string, std::vector< std::string > > options; 122 options["kyuafile"].push_back("/my//path"); 123 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 124 125 ATF_REQUIRE_EQ("/my/path", cli::kyuafile_path(mock_cmdline).str()); 126} 127 128 129ATF_TEST_CASE_WITHOUT_HEAD(result_types__default); 130ATF_TEST_CASE_BODY(result_types__default) 131{ 132 std::map< std::string, std::vector< std::string > > options; 133 options["results-filter"].push_back( 134 cli::results_filter_option.default_value()); 135 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 136 137 cli::result_types exp_types; 138 exp_types.push_back(model::test_result_skipped); 139 exp_types.push_back(model::test_result_expected_failure); 140 exp_types.push_back(model::test_result_broken); 141 exp_types.push_back(model::test_result_failed); 142 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 143} 144 145 146ATF_TEST_CASE_WITHOUT_HEAD(result_types__empty); 147ATF_TEST_CASE_BODY(result_types__empty) 148{ 149 std::map< std::string, std::vector< std::string > > options; 150 options["results-filter"].push_back(""); 151 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 152 153 cli::result_types exp_types; 154 exp_types.push_back(model::test_result_passed); 155 exp_types.push_back(model::test_result_skipped); 156 exp_types.push_back(model::test_result_expected_failure); 157 exp_types.push_back(model::test_result_broken); 158 exp_types.push_back(model::test_result_failed); 159 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 160} 161 162 163ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__all); 164ATF_TEST_CASE_BODY(result_types__explicit__all) 165{ 166 std::map< std::string, std::vector< std::string > > options; 167 options["results-filter"].push_back("passed,skipped,xfail,broken,failed"); 168 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 169 170 cli::result_types exp_types; 171 exp_types.push_back(model::test_result_passed); 172 exp_types.push_back(model::test_result_skipped); 173 exp_types.push_back(model::test_result_expected_failure); 174 exp_types.push_back(model::test_result_broken); 175 exp_types.push_back(model::test_result_failed); 176 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 177} 178 179 180ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__some); 181ATF_TEST_CASE_BODY(result_types__explicit__some) 182{ 183 std::map< std::string, std::vector< std::string > > options; 184 options["results-filter"].push_back("skipped,broken"); 185 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 186 187 cli::result_types exp_types; 188 exp_types.push_back(model::test_result_skipped); 189 exp_types.push_back(model::test_result_broken); 190 ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline)); 191} 192 193 194ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__invalid); 195ATF_TEST_CASE_BODY(result_types__explicit__invalid) 196{ 197 std::map< std::string, std::vector< std::string > > options; 198 options["results-filter"].push_back("skipped,foo,broken"); 199 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 200 201 ATF_REQUIRE_THROW_RE(std::runtime_error, "Unknown result type 'foo'", 202 cli::get_result_types(mock_cmdline)); 203} 204 205 206ATF_TEST_CASE_WITHOUT_HEAD(results_file_create__default__new); 207ATF_TEST_CASE_BODY(results_file_create__default__new) 208{ 209 std::map< std::string, std::vector< std::string > > options; 210 options["results-file"].push_back( 211 cli::results_file_create_option.default_value()); 212 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 213 214 const fs::path home("homedir"); 215 utils::setenv("HOME", home.str()); 216 217 ATF_REQUIRE_EQ(cli::results_file_create_option.default_value(), 218 cli::results_file_create(mock_cmdline)); 219 ATF_REQUIRE(!fs::exists(home / ".kyua")); 220} 221 222 223ATF_TEST_CASE_WITHOUT_HEAD(results_file_create__default__historical); 224ATF_TEST_CASE_BODY(results_file_create__default__historical) 225{ 226 std::map< std::string, std::vector< std::string > > options; 227 options["results-file"].push_back( 228 cli::results_file_create_option.default_value()); 229 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 230 231 const fs::path home("homedir"); 232 utils::setenv("HOME", home.str()); 233 fs::mkdir_p(fs::path("homedir/.kyua"), 0755); 234 atf::utils::create_file("homedir/.kyua/store.db", "fake store"); 235 236 ATF_REQUIRE_EQ(fs::path("homedir/.kyua/store.db").to_absolute(), 237 fs::path(cli::results_file_create(mock_cmdline))); 238} 239 240 241ATF_TEST_CASE_WITHOUT_HEAD(results_file_create__explicit); 242ATF_TEST_CASE_BODY(results_file_create__explicit) 243{ 244 std::map< std::string, std::vector< std::string > > options; 245 options["results-file"].push_back("/my//path/f.db"); 246 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 247 248 ATF_REQUIRE_EQ("/my//path/f.db", 249 cli::results_file_create(mock_cmdline)); 250} 251 252 253ATF_TEST_CASE_WITHOUT_HEAD(results_file_open__default__latest); 254ATF_TEST_CASE_BODY(results_file_open__default__latest) 255{ 256 std::map< std::string, std::vector< std::string > > options; 257 options["results-file"].push_back( 258 cli::results_file_open_option.default_value()); 259 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 260 261 const fs::path home("homedir"); 262 utils::setenv("HOME", home.str()); 263 264 ATF_REQUIRE_EQ(cli::results_file_open_option.default_value(), 265 cli::results_file_open(mock_cmdline)); 266 ATF_REQUIRE(!fs::exists(home / ".kyua")); 267} 268 269 270ATF_TEST_CASE_WITHOUT_HEAD(results_file_open__default__historical); 271ATF_TEST_CASE_BODY(results_file_open__default__historical) 272{ 273 std::map< std::string, std::vector< std::string > > options; 274 options["results-file"].push_back( 275 cli::results_file_open_option.default_value()); 276 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 277 278 const fs::path home("homedir"); 279 utils::setenv("HOME", home.str()); 280 fs::mkdir_p(fs::path("homedir/.kyua"), 0755); 281 atf::utils::create_file("homedir/.kyua/store.db", "fake store"); 282 283 ATF_REQUIRE_EQ(fs::path("homedir/.kyua/store.db").to_absolute(), 284 fs::path(cli::results_file_open(mock_cmdline))); 285} 286 287 288ATF_TEST_CASE_WITHOUT_HEAD(results_file_open__explicit); 289ATF_TEST_CASE_BODY(results_file_open__explicit) 290{ 291 std::map< std::string, std::vector< std::string > > options; 292 options["results-file"].push_back("/my//path/f.db"); 293 const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector()); 294 295 ATF_REQUIRE_EQ("/my//path/f.db", cli::results_file_open(mock_cmdline)); 296} 297 298 299ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__none); 300ATF_TEST_CASE_BODY(parse_filters__none) 301{ 302 const cmdline::args_vector args; 303 const std::set< engine::test_filter > filters = cli::parse_filters(args); 304 ATF_REQUIRE(filters.empty()); 305} 306 307 308ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__ok); 309ATF_TEST_CASE_BODY(parse_filters__ok) 310{ 311 cmdline::args_vector args; 312 args.push_back("foo"); 313 args.push_back("bar/baz"); 314 args.push_back("other:abc"); 315 args.push_back("other:bcd"); 316 const std::set< engine::test_filter > filters = cli::parse_filters(args); 317 318 std::set< engine::test_filter > exp_filters; 319 exp_filters.insert(mkfilter("foo", "")); 320 exp_filters.insert(mkfilter("bar/baz", "")); 321 exp_filters.insert(mkfilter("other", "abc")); 322 exp_filters.insert(mkfilter("other", "bcd")); 323 324 ATF_REQUIRE(exp_filters == filters); 325} 326 327 328ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__duplicate); 329ATF_TEST_CASE_BODY(parse_filters__duplicate) 330{ 331 cmdline::args_vector args; 332 args.push_back("foo/bar//baz"); 333 args.push_back("hello/world:yes"); 334 args.push_back("foo//bar/baz"); 335 ATF_REQUIRE_THROW_RE(cmdline::error, "Duplicate.*'foo/bar/baz'", 336 cli::parse_filters(args)); 337} 338 339 340ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__nondisjoint); 341ATF_TEST_CASE_BODY(parse_filters__nondisjoint) 342{ 343 cmdline::args_vector args; 344 args.push_back("foo/bar"); 345 args.push_back("hello/world:yes"); 346 args.push_back("foo/bar:baz"); 347 ATF_REQUIRE_THROW_RE(cmdline::error, "'foo/bar'.*'foo/bar:baz'.*disjoint", 348 cli::parse_filters(args)); 349} 350 351 352ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__none); 353ATF_TEST_CASE_BODY(report_unused_filters__none) 354{ 355 std::set< engine::test_filter > unused; 356 357 cmdline::ui_mock ui; 358 ATF_REQUIRE(!cli::report_unused_filters(unused, &ui)); 359 ATF_REQUIRE(ui.out_log().empty()); 360 ATF_REQUIRE(ui.err_log().empty()); 361} 362 363 364ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__some); 365ATF_TEST_CASE_BODY(report_unused_filters__some) 366{ 367 std::set< engine::test_filter > unused; 368 unused.insert(mkfilter("a/b", "")); 369 unused.insert(mkfilter("hey/d", "yes")); 370 371 cmdline::ui_mock ui; 372 cmdline::init("progname"); 373 ATF_REQUIRE(cli::report_unused_filters(unused, &ui)); 374 ATF_REQUIRE(ui.out_log().empty()); 375 ATF_REQUIRE_EQ(2, ui.err_log().size()); 376 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'a/b'", 377 ui.err_log())); 378 ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'hey/d:yes'", 379 ui.err_log())); 380} 381 382 383ATF_TEST_CASE_WITHOUT_HEAD(format_delta); 384ATF_TEST_CASE_BODY(format_delta) 385{ 386 ATF_REQUIRE_EQ("0.000s", cli::format_delta(datetime::delta())); 387 ATF_REQUIRE_EQ("0.012s", cli::format_delta(datetime::delta(0, 12300))); 388 ATF_REQUIRE_EQ("0.999s", cli::format_delta(datetime::delta(0, 999000))); 389 ATF_REQUIRE_EQ("51.321s", cli::format_delta(datetime::delta(51, 321000))); 390} 391 392 393ATF_TEST_CASE_WITHOUT_HEAD(format_result__no_reason); 394ATF_TEST_CASE_BODY(format_result__no_reason) 395{ 396 ATF_REQUIRE_EQ("passed", cli::format_result( 397 model::test_result(model::test_result_passed))); 398 ATF_REQUIRE_EQ("failed", cli::format_result( 399 model::test_result(model::test_result_failed))); 400} 401 402 403ATF_TEST_CASE_WITHOUT_HEAD(format_result__with_reason); 404ATF_TEST_CASE_BODY(format_result__with_reason) 405{ 406 ATF_REQUIRE_EQ("broken: Something", cli::format_result( 407 model::test_result(model::test_result_broken, "Something"))); 408 ATF_REQUIRE_EQ("expected_failure: A B C", cli::format_result( 409 model::test_result(model::test_result_expected_failure, "A B C"))); 410 ATF_REQUIRE_EQ("failed: More text", cli::format_result( 411 model::test_result(model::test_result_failed, "More text"))); 412 ATF_REQUIRE_EQ("skipped: Bye", cli::format_result( 413 model::test_result(model::test_result_skipped, "Bye"))); 414} 415 416 417ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_case); 418ATF_TEST_CASE_BODY(format_test_case_id__test_case) 419{ 420 const model::test_program test_program = model::test_program_builder( 421 "mock", fs::path("foo/bar/baz"), fs::path("unused-root"), 422 "unused-suite-name") 423 .add_test_case("abc") 424 .build(); 425 ATF_REQUIRE_EQ("foo/bar/baz:abc", 426 cli::format_test_case_id(test_program, "abc")); 427} 428 429 430ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_filter); 431ATF_TEST_CASE_BODY(format_test_case_id__test_filter) 432{ 433 const engine::test_filter filter(fs::path("foo/bar"), "baz"); 434 ATF_REQUIRE_EQ("foo/bar:baz", cli::format_test_case_id(filter)); 435} 436 437 438ATF_TEST_CASE_WITHOUT_HEAD(write_version_header); 439ATF_TEST_CASE_BODY(write_version_header) 440{ 441 cmdline::ui_mock ui; 442 cli::write_version_header(&ui); 443 ATF_REQUIRE_EQ(1, ui.out_log().size()); 444 ATF_REQUIRE_MATCH("^kyua .*[0-9]+\\.[0-9]+$", ui.out_log()[0]); 445 ATF_REQUIRE(ui.err_log().empty()); 446} 447 448 449ATF_INIT_TEST_CASES(tcs) 450{ 451 ATF_ADD_TEST_CASE(tcs, build_root_path__default); 452 ATF_ADD_TEST_CASE(tcs, build_root_path__explicit); 453 454 ATF_ADD_TEST_CASE(tcs, kyuafile_path__default); 455 ATF_ADD_TEST_CASE(tcs, kyuafile_path__explicit); 456 457 ATF_ADD_TEST_CASE(tcs, result_types__default); 458 ATF_ADD_TEST_CASE(tcs, result_types__empty); 459 ATF_ADD_TEST_CASE(tcs, result_types__explicit__all); 460 ATF_ADD_TEST_CASE(tcs, result_types__explicit__some); 461 ATF_ADD_TEST_CASE(tcs, result_types__explicit__invalid); 462 463 ATF_ADD_TEST_CASE(tcs, results_file_create__default__new); 464 ATF_ADD_TEST_CASE(tcs, results_file_create__default__historical); 465 ATF_ADD_TEST_CASE(tcs, results_file_create__explicit); 466 467 ATF_ADD_TEST_CASE(tcs, results_file_open__default__latest); 468 ATF_ADD_TEST_CASE(tcs, results_file_open__default__historical); 469 ATF_ADD_TEST_CASE(tcs, results_file_open__explicit); 470 471 ATF_ADD_TEST_CASE(tcs, parse_filters__none); 472 ATF_ADD_TEST_CASE(tcs, parse_filters__ok); 473 ATF_ADD_TEST_CASE(tcs, parse_filters__duplicate); 474 ATF_ADD_TEST_CASE(tcs, parse_filters__nondisjoint); 475 476 ATF_ADD_TEST_CASE(tcs, report_unused_filters__none); 477 ATF_ADD_TEST_CASE(tcs, report_unused_filters__some); 478 479 ATF_ADD_TEST_CASE(tcs, format_delta); 480 481 ATF_ADD_TEST_CASE(tcs, format_result__no_reason); 482 ATF_ADD_TEST_CASE(tcs, format_result__with_reason); 483 484 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_case); 485 ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_filter); 486 487 ATF_ADD_TEST_CASE(tcs, write_version_header); 488} 489