1// Copyright 2010 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/kyuafile.hpp" 30 31#include <algorithm> 32#include <iterator> 33#include <stdexcept> 34 35#include <lutok/exceptions.hpp> 36#include <lutok/operations.hpp> 37#include <lutok/stack_cleaner.hpp> 38#include <lutok/state.ipp> 39 40#include "engine/exceptions.hpp" 41#include "engine/scheduler.hpp" 42#include "model/metadata.hpp" 43#include "model/test_program.hpp" 44#include "utils/config/exceptions.hpp" 45#include "utils/config/tree.ipp" 46#include "utils/datetime.hpp" 47#include "utils/format/macros.hpp" 48#include "utils/fs/lua_module.hpp" 49#include "utils/fs/operations.hpp" 50#include "utils/logging/macros.hpp" 51#include "utils/noncopyable.hpp" 52#include "utils/optional.ipp" 53#include "utils/sanity.hpp" 54 55namespace config = utils::config; 56namespace datetime = utils::datetime; 57namespace fs = utils::fs; 58namespace scheduler = engine::scheduler; 59 60using utils::none; 61using utils::optional; 62 63 64// History of Kyuafile file versions: 65// 66// 3 - DOES NOT YET EXIST. Pending changes for when this is introduced: 67// 68// * Revisit what to do about the test_suite definition. Support for 69// per-test program overrides is deprecated and should be removed. 70// But, maybe, the whole test_suite definition idea is wrong and we 71// should instead be explicitly telling which configuration variables 72// to "inject" into each test program. 73// 74// 2 - Changed the syntax() call to take only a version number, instead of the 75// word 'config' as the first argument and the version as the second one. 76// Files now start with syntax(2) instead of syntax('kyuafile', 1). 77// 78// 1 - Initial version. 79 80 81namespace { 82 83 84static int lua_current_kyuafile(lutok::state&); 85static int lua_generic_test_program(lutok::state&); 86static int lua_include(lutok::state&); 87static int lua_syntax(lutok::state&); 88static int lua_test_suite(lutok::state&); 89 90 91/// Concatenates two paths while avoiding paths to start with './'. 92/// 93/// \param root Path to the directory containing the file. 94/// \param file Path to concatenate to root. Cannot be absolute. 95/// 96/// \return The concatenated path. 97static fs::path 98relativize(const fs::path& root, const fs::path& file) 99{ 100 PRE(!file.is_absolute()); 101 102 if (root == fs::path(".")) 103 return file; 104 else 105 return root / file; 106} 107 108 109/// Implementation of a parser for Kyuafiles. 110/// 111/// The main purpose of having this as a class is to keep track of global state 112/// within the Lua files and allowing the Lua callbacks to easily access such 113/// data. 114class parser : utils::noncopyable { 115 /// Lua state to parse a single Kyuafile file. 116 lutok::state _state; 117 118 /// Root directory of the test suite represented by the Kyuafile. 119 const fs::path _source_root; 120 121 /// Root directory of the test programs. 122 const fs::path _build_root; 123 124 /// Name of the Kyuafile to load relative to _source_root. 125 const fs::path _relative_filename; 126 127 /// Version of the Kyuafile file format requested by the parsed file. 128 /// 129 /// This is set once the Kyuafile invokes the syntax() call. 130 optional< int > _version; 131 132 /// Name of the test suite defined by the Kyuafile. 133 /// 134 /// This is set once the Kyuafile invokes the test_suite() call. 135 optional< std::string > _test_suite; 136 137 /// Collection of test programs defined by the Kyuafile. 138 /// 139 /// This acts as an accumulator for all the *_test_program() calls within 140 /// the Kyuafile. 141 model::test_programs_vector _test_programs; 142 143 /// Safely gets _test_suite and respects any test program overrides. 144 /// 145 /// \param program_override The test program-specific test suite name. May 146 /// be empty to indicate no override. 147 /// 148 /// \return The name of the test suite. 149 /// 150 /// \throw std::runtime_error If program_override is empty and the Kyuafile 151 /// did not yet define the global name of the test suite. 152 std::string 153 get_test_suite(const std::string& program_override) 154 { 155 std::string test_suite; 156 157 if (program_override.empty()) { 158 if (!_test_suite) { 159 throw std::runtime_error("No test suite defined in the " 160 "Kyuafile and no override provided in " 161 "the test_program definition"); 162 } 163 test_suite = _test_suite.get(); 164 } else { 165 test_suite = program_override; 166 } 167 168 return test_suite; 169 } 170 171public: 172 /// Initializes the parser and the Lua state. 173 /// 174 /// \param source_root_ The root directory of the test suite represented by 175 /// the Kyuafile. 176 /// \param build_root_ The root directory of the test programs. 177 /// \param relative_filename_ Name of the Kyuafile to load relative to 178 /// source_root_. 179 /// \param user_config User configuration holding any test suite properties 180 /// to be passed to the list operation. 181 /// \param scheduler_handle The scheduler context to use for loading the 182 /// test case lists. 183 parser(const fs::path& source_root_, const fs::path& build_root_, 184 const fs::path& relative_filename_, 185 const config::tree& user_config, 186 scheduler::scheduler_handle& scheduler_handle) : 187 _source_root(source_root_), _build_root(build_root_), 188 _relative_filename(relative_filename_) 189 { 190 lutok::stack_cleaner cleaner(_state); 191 192 _state.push_cxx_function(lua_syntax); 193 _state.set_global("syntax"); 194 195 *_state.new_userdata< parser* >() = this; 196 _state.set_global("_parser"); 197 198 _state.push_cxx_function(lua_current_kyuafile); 199 _state.set_global("current_kyuafile"); 200 201 *_state.new_userdata< const config::tree* >() = &user_config; 202 *_state.new_userdata< scheduler::scheduler_handle* >() = 203 &scheduler_handle; 204 _state.push_cxx_closure(lua_include, 2); 205 _state.set_global("include"); 206 207 _state.push_cxx_function(lua_test_suite); 208 _state.set_global("test_suite"); 209 210 const std::set< std::string > interfaces = 211 scheduler::registered_interface_names(); 212 for (std::set< std::string >::const_iterator iter = interfaces.begin(); 213 iter != interfaces.end(); ++iter) { 214 const std::string& interface = *iter; 215 216 _state.push_string(interface); 217 *_state.new_userdata< const config::tree* >() = &user_config; 218 *_state.new_userdata< scheduler::scheduler_handle* >() = 219 &scheduler_handle; 220 _state.push_cxx_closure(lua_generic_test_program, 3); 221 _state.set_global(interface + "_test_program"); 222 } 223 224 _state.open_base(); 225 _state.open_string(); 226 _state.open_table(); 227 fs::open_fs(_state, callback_current_kyuafile().branch_path()); 228 } 229 230 /// Destructor. 231 ~parser(void) 232 { 233 } 234 235 /// Gets the parser object associated to a Lua state. 236 /// 237 /// \param state The Lua state from which to obtain the parser object. 238 /// 239 /// \return A pointer to the parser. 240 static parser* 241 get_from_state(lutok::state& state) 242 { 243 lutok::stack_cleaner cleaner(state); 244 state.get_global("_parser"); 245 return *state.to_userdata< parser* >(-1); 246 } 247 248 /// Callback for the Kyuafile current_kyuafile() function. 249 /// 250 /// \return Returns the absolute path to the current Kyuafile. 251 fs::path 252 callback_current_kyuafile(void) const 253 { 254 const fs::path file = relativize(_source_root, _relative_filename); 255 if (file.is_absolute()) 256 return file; 257 else 258 return file.to_absolute(); 259 } 260 261 /// Callback for the Kyuafile include() function. 262 /// 263 /// \post _test_programs is extended with the the test programs defined by 264 /// the included file. 265 /// 266 /// \param raw_file Path to the file to include. 267 /// \param user_config User configuration holding any test suite properties 268 /// to be passed to the list operation. 269 /// \param scheduler_handle Scheduler context to run test programs in. 270 void 271 callback_include(const fs::path& raw_file, 272 const config::tree& user_config, 273 scheduler::scheduler_handle& scheduler_handle) 274 { 275 const fs::path file = relativize(_relative_filename.branch_path(), 276 raw_file); 277 const model::test_programs_vector subtps = 278 parser(_source_root, _build_root, file, user_config, 279 scheduler_handle).parse(); 280 281 std::copy(subtps.begin(), subtps.end(), 282 std::back_inserter(_test_programs)); 283 } 284 285 /// Callback for the Kyuafile syntax() function. 286 /// 287 /// \post _version is set to the requested version. 288 /// 289 /// \param version Version of the Kyuafile syntax requested by the file. 290 /// 291 /// \throw std::runtime_error If the format or the version are invalid, or 292 /// if syntax() has already been called. 293 void 294 callback_syntax(const int version) 295 { 296 if (_version) 297 throw std::runtime_error("Can only call syntax() once"); 298 299 if (version < 1 || version > 2) 300 throw std::runtime_error(F("Unsupported file version %s") % 301 version); 302 303 _version = utils::make_optional(version); 304 } 305 306 /// Callback for the various Kyuafile *_test_program() functions. 307 /// 308 /// \post _test_programs is extended to include the newly defined test 309 /// program. 310 /// 311 /// \param interface Name of the test program interface. 312 /// \param raw_path Path to the test program, relative to the Kyuafile. 313 /// This has to be adjusted according to the relative location of this 314 /// Kyuafile to _source_root. 315 /// \param test_suite_override Name of the test suite this test program 316 /// belongs to, if explicitly defined at the test program level. 317 /// \param metadata Metadata variables passed to the test program. 318 /// \param user_config User configuration holding any test suite properties 319 /// to be passed to the list operation. 320 /// \param scheduler_handle Scheduler context to run test programs in. 321 /// 322 /// \throw std::runtime_error If the test program definition is invalid or 323 /// if the test program does not exist. 324 void 325 callback_test_program(const std::string& interface, 326 const fs::path& raw_path, 327 const std::string& test_suite_override, 328 const model::metadata& metadata, 329 const config::tree& user_config, 330 scheduler::scheduler_handle& scheduler_handle) 331 { 332 if (raw_path.is_absolute()) 333 throw std::runtime_error(F("Got unexpected absolute path for test " 334 "program '%s'") % raw_path); 335 else if (raw_path.str() != raw_path.leaf_name()) 336 throw std::runtime_error(F("Test program '%s' cannot contain path " 337 "components") % raw_path); 338 339 const fs::path path = relativize(_relative_filename.branch_path(), 340 raw_path); 341 342 if (!fs::exists(_build_root / path)) 343 throw std::runtime_error(F("Non-existent test program '%s'") % 344 path); 345 346 const std::string test_suite = get_test_suite(test_suite_override); 347 348 _test_programs.push_back(model::test_program_ptr( 349 new scheduler::lazy_test_program(interface, path, _build_root, 350 test_suite, metadata, user_config, 351 scheduler_handle))); 352 } 353 354 /// Callback for the Kyuafile test_suite() function. 355 /// 356 /// \post _version is set to the requested version. 357 /// 358 /// \param name Name of the test suite. 359 /// 360 /// \throw std::runtime_error If test_suite() has already been called. 361 void 362 callback_test_suite(const std::string& name) 363 { 364 if (_test_suite) 365 throw std::runtime_error("Can only call test_suite() once"); 366 _test_suite = utils::make_optional(name); 367 } 368 369 /// Parses the Kyuafile. 370 /// 371 /// \pre Can only be invoked once. 372 /// 373 /// \return The collection of test programs defined by the Kyuafile. 374 /// 375 /// \throw load_error If there is any problem parsing the file. 376 const model::test_programs_vector& 377 parse(void) 378 { 379 PRE(_test_programs.empty()); 380 381 const fs::path load_path = relativize(_source_root, _relative_filename); 382 try { 383 lutok::do_file(_state, load_path.str(), 0, 0, 0); 384 } catch (const std::runtime_error& e) { 385 // It is tempting to think that all of our various auxiliary 386 // functions above could raise load_error by themselves thus making 387 // this exception rewriting here unnecessary. Howver, that would 388 // not work because the helper functions above are executed within a 389 // Lua context, and we lose their type when they are propagated out 390 // of it. 391 throw engine::load_error(load_path, e.what()); 392 } 393 394 if (!_version) 395 throw engine::load_error(load_path, "syntax() never called"); 396 397 return _test_programs; 398 } 399}; 400 401 402/// Glue to invoke parser::callback_test_program() from Lua. 403/// 404/// This is a helper function for the various *_test_program() calls, as they 405/// only differ in the interface of the defined test program. 406/// 407/// \pre state(-1) A table with the arguments that define the test program. The 408/// special argument 'test_suite' provides an override to the global test suite 409/// name. The rest of the arguments are part of the test program metadata. 410/// \pre state(upvalue 1) String with the name of the interface. 411/// \pre state(upvalue 2) User configuration with the per-test suite settings. 412/// \pre state(upvalue 3) Scheduler context to run test programs in. 413/// 414/// \param state The Lua state that executed the function. 415/// 416/// \return Number of return values left on the Lua stack. 417/// 418/// \throw std::runtime_error If the arguments to the function are invalid. 419static int 420lua_generic_test_program(lutok::state& state) 421{ 422 if (!state.is_string(state.upvalue_index(1))) 423 throw std::runtime_error("Found corrupt state for test_program " 424 "function"); 425 const std::string interface = state.to_string(state.upvalue_index(1)); 426 427 if (!state.is_userdata(state.upvalue_index(2))) 428 throw std::runtime_error("Found corrupt state for test_program " 429 "function"); 430 const config::tree* user_config = *state.to_userdata< const config::tree* >( 431 state.upvalue_index(2)); 432 433 if (!state.is_userdata(state.upvalue_index(3))) 434 throw std::runtime_error("Found corrupt state for test_program " 435 "function"); 436 scheduler::scheduler_handle* scheduler_handle = 437 *state.to_userdata< scheduler::scheduler_handle* >( 438 state.upvalue_index(3)); 439 440 if (!state.is_table(-1)) 441 throw std::runtime_error( 442 F("%s_test_program expects a table of properties as its single " 443 "argument") % interface); 444 445 scheduler::ensure_valid_interface(interface); 446 447 lutok::stack_cleaner cleaner(state); 448 449 state.push_string("name"); 450 state.get_table(-2); 451 if (!state.is_string(-1)) 452 throw std::runtime_error("Test program name not defined or not a " 453 "string"); 454 const fs::path path(state.to_string(-1)); 455 state.pop(1); 456 457 state.push_string("test_suite"); 458 state.get_table(-2); 459 std::string test_suite; 460 if (state.is_nil(-1)) { 461 // Leave empty to use the global test-suite value. 462 } else if (state.is_string(-1)) { 463 test_suite = state.to_string(-1); 464 } else { 465 throw std::runtime_error(F("Found non-string value in the test_suite " 466 "property of test program '%s'") % path); 467 } 468 state.pop(1); 469 470 model::metadata_builder mdbuilder; 471 state.push_nil(); 472 while (state.next(-2)) { 473 if (!state.is_string(-2)) 474 throw std::runtime_error(F("Found non-string metadata property " 475 "name in test program '%s'") % 476 path); 477 const std::string property = state.to_string(-2); 478 479 if (property != "name" && property != "test_suite") { 480 std::string value; 481 if (state.is_boolean(-1)) { 482 value = F("%s") % state.to_boolean(-1); 483 } else if (state.is_number(-1)) { 484 value = F("%s") % state.to_integer(-1); 485 } else if (state.is_string(-1)) { 486 value = state.to_string(-1); 487 } else { 488 throw std::runtime_error( 489 F("Metadata property '%s' in test program '%s' cannot be " 490 "converted to a string") % property % path); 491 } 492 493 mdbuilder.set_string(property, value); 494 } 495 496 state.pop(1); 497 } 498 499 parser::get_from_state(state)->callback_test_program( 500 interface, path, test_suite, mdbuilder.build(), *user_config, 501 *scheduler_handle); 502 return 0; 503} 504 505 506/// Glue to invoke parser::callback_current_kyuafile() from Lua. 507/// 508/// \param state The Lua state that executed the function. 509/// 510/// \return Number of return values left on the Lua stack. 511static int 512lua_current_kyuafile(lutok::state& state) 513{ 514 state.push_string(parser::get_from_state(state)-> 515 callback_current_kyuafile().str()); 516 return 1; 517} 518 519 520/// Glue to invoke parser::callback_include() from Lua. 521/// 522/// \param state The Lua state that executed the function. 523/// 524/// \pre state(upvalue 1) User configuration with the per-test suite settings. 525/// \pre state(upvalue 2) Scheduler context to run test programs in. 526/// 527/// \return Number of return values left on the Lua stack. 528static int 529lua_include(lutok::state& state) 530{ 531 if (!state.is_userdata(state.upvalue_index(1))) 532 throw std::runtime_error("Found corrupt state for test_program " 533 "function"); 534 const config::tree* user_config = *state.to_userdata< const config::tree* >( 535 state.upvalue_index(1)); 536 537 if (!state.is_userdata(state.upvalue_index(2))) 538 throw std::runtime_error("Found corrupt state for test_program " 539 "function"); 540 scheduler::scheduler_handle* scheduler_handle = 541 *state.to_userdata< scheduler::scheduler_handle* >( 542 state.upvalue_index(2)); 543 544 parser::get_from_state(state)->callback_include( 545 fs::path(state.to_string(-1)), *user_config, *scheduler_handle); 546 return 0; 547} 548 549 550/// Glue to invoke parser::callback_syntax() from Lua. 551/// 552/// \pre state(-2) The syntax format name, if a v1 file. 553/// \pre state(-1) The syntax format version. 554/// 555/// \param state The Lua state that executed the function. 556/// 557/// \return Number of return values left on the Lua stack. 558static int 559lua_syntax(lutok::state& state) 560{ 561 if (!state.is_number(-1)) 562 throw std::runtime_error("Last argument to syntax must be a number"); 563 const int syntax_version = state.to_integer(-1); 564 565 if (syntax_version == 1) { 566 if (state.get_top() != 2) 567 throw std::runtime_error("Version 1 files need two arguments to " 568 "syntax()"); 569 if (!state.is_string(-2) || state.to_string(-2) != "kyuafile") 570 throw std::runtime_error("First argument to syntax must be " 571 "'kyuafile' for version 1 files"); 572 } else { 573 if (state.get_top() != 1) 574 throw std::runtime_error("syntax() only takes one argument"); 575 } 576 577 parser::get_from_state(state)->callback_syntax(syntax_version); 578 return 0; 579} 580 581 582/// Glue to invoke parser::callback_test_suite() from Lua. 583/// 584/// \param state The Lua state that executed the function. 585/// 586/// \return Number of return values left on the Lua stack. 587static int 588lua_test_suite(lutok::state& state) 589{ 590 parser::get_from_state(state)->callback_test_suite(state.to_string(-1)); 591 return 0; 592} 593 594 595} // anonymous namespace 596 597 598/// Constructs a kyuafile form initialized data. 599/// 600/// Use load() to parse a test suite configuration file and construct a 601/// kyuafile object. 602/// 603/// \param source_root_ The root directory for the test suite represented by the 604/// Kyuafile. In other words, the directory containing the first Kyuafile 605/// processed. 606/// \param build_root_ The root directory for the test programs themselves. In 607/// general, this will be the same as source_root_. If different, the 608/// specified directory must follow the exact same layout of source_root_. 609/// \param tps_ Collection of test programs that belong to this test suite. 610engine::kyuafile::kyuafile(const fs::path& source_root_, 611 const fs::path& build_root_, 612 const model::test_programs_vector& tps_) : 613 _source_root(source_root_), 614 _build_root(build_root_), 615 _test_programs(tps_) 616{ 617} 618 619 620/// Destructor. 621engine::kyuafile::~kyuafile(void) 622{ 623} 624 625 626/// Parses a test suite configuration file. 627/// 628/// \param file The file to parse. 629/// \param user_build_root If not none, specifies a path to a directory 630/// containing the test programs themselves. The layout of the build root 631/// must match the layout of the source root (which is just the directory 632/// from which the Kyuafile is being read). 633/// \param user_config User configuration holding any test suite properties 634/// to be passed to the list operation. 635/// \param scheduler_handle The scheduler context to use for loading the test 636/// case lists. 637/// 638/// \return High-level representation of the configuration file. 639/// 640/// \throw load_error If there is any problem loading the file. This includes 641/// file access errors and syntax errors. 642engine::kyuafile 643engine::kyuafile::load(const fs::path& file, 644 const optional< fs::path > user_build_root, 645 const config::tree& user_config, 646 scheduler::scheduler_handle& scheduler_handle) 647{ 648 const fs::path source_root_ = file.branch_path(); 649 const fs::path build_root_ = user_build_root ? 650 user_build_root.get() : source_root_; 651 652 // test_program.absolute_path() uses the current work directory and that 653 // fails to resolve the correct path once we have used chdir to enter the 654 // test work directory. To prevent this causing issues down the road, 655 // force the build root to be absolute so that absolute_path() does not 656 // need to rely on the current work directory. 657 const fs::path abs_build_root = build_root_.is_absolute() ? 658 build_root_ : build_root_.to_absolute(); 659 660 return kyuafile(source_root_, build_root_, 661 parser(source_root_, abs_build_root, 662 fs::path(file.leaf_name()), user_config, 663 scheduler_handle).parse()); 664} 665 666 667/// Gets the root directory of the test suite. 668/// 669/// \return A path. 670const fs::path& 671engine::kyuafile::source_root(void) const 672{ 673 return _source_root; 674} 675 676 677/// Gets the root directory of the test programs. 678/// 679/// \return A path. 680const fs::path& 681engine::kyuafile::build_root(void) const 682{ 683 return _build_root; 684} 685 686 687/// Gets the collection of test programs that belong to this test suite. 688/// 689/// \return Collection of test program executable names. 690const model::test_programs_vector& 691engine::kyuafile::test_programs(void) const 692{ 693 return _test_programs; 694} 695