1// Copyright 2010 Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/kyuafile.hpp" 30 31#include <algorithm> 32#include <iterator> 33#include <stdexcept> 34 35#include <lutok/exceptions.hpp> 36#include <lutok/operations.hpp> 37#include <lutok/stack_cleaner.hpp> 38#include <lutok/state.ipp> 39 40#include "engine/exceptions.hpp" 41#include "engine/test_program.hpp" 42#include "engine/testers.hpp" 43#include "utils/datetime.hpp" 44#include "utils/format/macros.hpp" 45#include "utils/fs/lua_module.hpp" 46#include "utils/fs/operations.hpp" 47#include "utils/logging/macros.hpp" 48#include "utils/noncopyable.hpp" 49#include "utils/optional.ipp" 50#include "utils/sanity.hpp" 51 52namespace datetime = utils::datetime; 53namespace fs = utils::fs; 54 55using utils::none; 56using utils::optional; 57 58 59// History of Kyuafile file versions: 60// 61// 2 - Changed the syntax() call to take only a version number, instead of the 62// word 'config' as the first argument and the version as the second one. 63// Files now start with syntax(2) instead of syntax('kyuafile', 1). 64// 65// 1 - Initial version. 66 67 68namespace { 69 70 71static int lua_atf_test_program(lutok::state&); 72static int lua_current_kyuafile(lutok::state&); 73static int lua_include(lutok::state&); 74static int lua_plain_test_program(lutok::state&); 75static int lua_syntax(lutok::state&); 76static int lua_test_suite(lutok::state&); 77 78 79/// Concatenates two paths while avoiding paths to start with './'. 80/// 81/// \param root Path to the directory containing the file. 82/// \param file Path to concatenate to root. Cannot be absolute. 83/// 84/// \return The concatenated path. 85static fs::path 86relativize(const fs::path& root, const fs::path& file) 87{ 88 PRE(!file.is_absolute()); 89 90 if (root == fs::path(".")) 91 return file; 92 else 93 return root / file; 94} 95 96 97/// Implementation of a parser for Kyuafiles. 98/// 99/// The main purpose of having this as a class is to keep track of global state 100/// within the Lua files and allowing the Lua callbacks to easily access such 101/// data. 102class parser : utils::noncopyable { 103 /// Lua state to parse a single Kyuafile file. 104 lutok::state _state; 105 106 /// Root directory of the test suite represented by the Kyuafile. 107 const fs::path _source_root; 108 109 /// Root directory of the test programs. 110 const fs::path _build_root; 111 112 /// Name of the Kyuafile to load relative to _source_root. 113 const fs::path _relative_filename; 114 115 /// Version of the Kyuafile file format requested by the parsed file. 116 /// 117 /// This is set once the Kyuafile invokes the syntax() call. 118 optional< int > _version; 119 120 /// Name of the test suite defined by the Kyuafile. 121 /// 122 /// This is set once the Kyuafile invokes the test_suite() call. 123 optional< std::string > _test_suite; 124 125 /// Collection of test programs defined by the Kyuafile. 126 /// 127 /// This acts as an accumulator for all the *_test_program() calls within 128 /// the Kyuafile. 129 engine::test_programs_vector _test_programs; 130 131public: 132 /// Initializes the parser and the Lua state. 133 /// 134 /// \param source_root_ The root directory of the test suite represented by 135 /// the Kyuafile. 136 /// \param build_root_ The root directory of the test programs. 137 /// \param relative_filename_ Name of the Kyuafile to load relative to 138 /// source_root_. 139 parser(const fs::path& source_root_, const fs::path& build_root_, 140 const fs::path& relative_filename_) : 141 _source_root(source_root_), _build_root(build_root_), 142 _relative_filename(relative_filename_) 143 { 144 lutok::stack_cleaner cleaner(_state); 145 146 _state.push_cxx_function(lua_syntax); 147 _state.set_global("syntax"); 148 *_state.new_userdata< parser* >() = this; 149 _state.set_global("_parser"); 150 151 _state.push_cxx_function(lua_atf_test_program); 152 _state.set_global("atf_test_program"); 153 _state.push_cxx_function(lua_current_kyuafile); 154 _state.set_global("current_kyuafile"); 155 _state.push_cxx_function(lua_include); 156 _state.set_global("include"); 157 _state.push_cxx_function(lua_plain_test_program); 158 _state.set_global("plain_test_program"); 159 _state.push_cxx_function(lua_test_suite); 160 _state.set_global("test_suite"); 161 162 _state.open_base(); 163 _state.open_string(); 164 _state.open_table(); 165 fs::open_fs(_state); 166 } 167 168 /// Destructor. 169 ~parser(void) 170 { 171 } 172 173 /// Gets the parser object associated to a Lua state. 174 /// 175 /// \param state The Lua state from which to obtain the parser object. 176 /// 177 /// \return A pointer to the parser. 178 static parser* 179 get_from_state(lutok::state& state) 180 { 181 lutok::stack_cleaner cleaner(state); 182 state.get_global("_parser"); 183 return *state.to_userdata< parser* >(); 184 } 185 186 /// Callback for the Kyuafile current_kyuafile() function. 187 /// 188 /// \return Returns the absolute path to the current Kyuafile. 189 fs::path 190 callback_current_kyuafile(void) const 191 { 192 const fs::path file = relativize(_source_root, _relative_filename); 193 if (file.is_absolute()) 194 return file; 195 else 196 return file.to_absolute(); 197 } 198 199 /// Callback for the Kyuafile include() function. 200 /// 201 /// \post _test_programs is extended with the the test programs defined by 202 /// the included file. 203 /// 204 /// \param raw_file Path to the file to include. 205 void 206 callback_include(const fs::path& raw_file) 207 { 208 const fs::path file = relativize(_relative_filename.branch_path(), 209 raw_file); 210 const engine::test_programs_vector subtps = 211 parser(_source_root, _build_root, file).parse(); 212 213 std::copy(subtps.begin(), subtps.end(), 214 std::back_inserter(_test_programs)); 215 } 216 217 /// Callback for the Kyuafile syntax() function. 218 /// 219 /// \post _version is set to the requested version. 220 /// 221 /// \param version Version of the Kyuafile syntax requested by the file. 222 /// 223 /// \throw std::runtime_error If the format or the version are invalid, or 224 /// if syntax() has already been called. 225 void 226 callback_syntax(const int version) 227 { 228 if (_version) 229 throw std::runtime_error("Can only call syntax() once"); 230 231 if (version < 1 || version > 2) 232 throw std::runtime_error(F("Unsupported file version %s") % 233 version); 234 235 _version = utils::make_optional(version); 236 } 237 238 /// Callback for the various Kyuafile *_test_program() functions. 239 /// 240 /// \post _test_programs is extended to include the newly defined test 241 /// program. 242 /// 243 /// \param interface Name of the test program interface. 244 /// \param raw_path Path to the test program, relative to the Kyuafile. 245 /// This has to be adjusted according to the relative location of this 246 /// Kyuafile to _source_root. 247 /// \param test_suite_override Name of the test suite this test program 248 /// belongs to, if explicitly defined at the test program level. 249 /// \param metadata Metadata variables passed to the test program. 250 /// 251 /// \throw std::runtime_error If the test program definition is invalid or 252 /// if the test program does not exist. 253 void 254 callback_test_program(const std::string& interface, 255 const fs::path& raw_path, 256 const std::string& test_suite_override, 257 const engine::metadata& metadata) 258 { 259 if (raw_path.is_absolute()) 260 throw std::runtime_error(F("Got unexpected absolute path for test " 261 "program '%s'") % raw_path); 262 else if (raw_path.str() != raw_path.leaf_name()) 263 throw std::runtime_error(F("Test program '%s' cannot contain path " 264 "components") % raw_path); 265 266 const fs::path path = relativize(_relative_filename.branch_path(), 267 raw_path); 268 269 if (!fs::exists(_build_root / path)) 270 throw std::runtime_error(F("Non-existent test program '%s'") % 271 path); 272 273 const std::string test_suite = test_suite_override.empty() 274 ? _test_suite.get() : test_suite_override; 275 _test_programs.push_back(engine::test_program_ptr( 276 new engine::test_program(interface, path, _build_root, test_suite, 277 metadata))); 278 } 279 280 /// Callback for the Kyuafile test_suite() function. 281 /// 282 /// \post _version is set to the requested version. 283 /// 284 /// \param name Name of the test suite. 285 /// 286 /// \throw std::runtime_error If test_suite() has already been called. 287 void 288 callback_test_suite(const std::string& name) 289 { 290 if (_test_suite) 291 throw std::runtime_error("Can only call test_suite() once"); 292 _test_suite = utils::make_optional(name); 293 } 294 295 /// Parses the Kyuafile. 296 /// 297 /// \pre Can only be invoked once. 298 /// 299 /// \return The collection of test programs defined by the Kyuafile. 300 /// 301 /// \throw load_error If there is any problem parsing the file. 302 const engine::test_programs_vector& 303 parse(void) 304 { 305 PRE(_test_programs.empty()); 306 307 const fs::path load_path = relativize(_source_root, _relative_filename); 308 try { 309 lutok::do_file(_state, load_path.str()); 310 } catch (const std::runtime_error& e) { 311 // It is tempting to think that all of our various auxiliary 312 // functions above could raise load_error by themselves thus making 313 // this exception rewriting here unnecessary. Howver, that would 314 // not work because the helper functions above are executed within a 315 // Lua context, and we lose their type when they are propagated out 316 // of it. 317 throw engine::load_error(load_path, e.what()); 318 } 319 320 if (!_version) 321 throw engine::load_error(load_path, "syntax() never called"); 322 323 return _test_programs; 324 } 325}; 326 327 328/// Gets a string field from a Lua table. 329/// 330/// \pre state(-1) contains a table. 331/// 332/// \param state The Lua state. 333/// \param field The name of the field to query. 334/// \param error The error message to raise when an error condition is 335/// encoutered. 336/// 337/// \return The string value from the table. 338/// 339/// \throw std::runtime_error If there is any problem accessing the table. 340static inline std::string 341get_table_string(lutok::state& state, const char* field, 342 const std::string& error) 343{ 344 PRE(state.is_table()); 345 346 lutok::stack_cleaner cleaner(state); 347 348 state.push_string(field); 349 state.get_table(); 350 if (!state.is_string()) 351 throw std::runtime_error(error); 352 return state.to_string(); 353} 354 355 356/// Checks if the given interface name is valid. 357/// 358/// \param interface The name of the interface to validate. 359/// 360/// \throw std::runtime_error If the given interface is not supported. 361static void 362ensure_valid_interface(const std::string& interface) 363{ 364 try { 365 (void)engine::tester_path(interface); 366 } catch (const engine::error& e) { 367 throw std::runtime_error(F("Unsupported test interface '%s'") % 368 interface); 369 } 370} 371 372 373/// Glue to invoke parser::callback_test_program() from Lua. 374/// 375/// This is a helper function for the various *_test_program() calls, as they 376/// only differ in the interface of the defined test program. 377/// 378/// \pre state(-1) A table with the arguments that define the test program. The 379/// special argument 'test_suite' provides an override to the global test suite 380/// name. The rest of the arguments are part of the test program metadata. 381/// 382/// \param state The Lua state that executed the function. 383/// \param interface Name of the test program interface. 384/// 385/// \return Number of return values left on the Lua stack. 386/// 387/// \throw std::runtime_error If the arguments to the function are invalid. 388static int 389lua_generic_test_program(lutok::state& state, const std::string& interface) 390{ 391 if (!state.is_table()) 392 throw std::runtime_error( 393 F("%s_test_program expects a table of properties as its single " 394 "argument") % interface); 395 396 ensure_valid_interface(interface); 397 398 lutok::stack_cleaner cleaner(state); 399 400 state.push_string("name"); 401 state.get_table(); 402 if (!state.is_string()) 403 throw std::runtime_error("Test program name not defined or not a " 404 "string"); 405 const fs::path path(state.to_string()); 406 state.pop(1); 407 408 state.push_string("test_suite"); 409 state.get_table(); 410 std::string test_suite; 411 if (state.is_nil()) { 412 // Leave empty to use the global test-suite value. 413 } else if (state.is_string()) { 414 test_suite = state.to_string(); 415 } else { 416 throw std::runtime_error(F("Found non-string value in the test_suite " 417 "property of test program '%s'") % path); 418 } 419 state.pop(1); 420 421 engine::metadata_builder mdbuilder; 422 state.push_nil(); 423 while (state.next()) { 424 if (!state.is_string(-2)) 425 throw std::runtime_error(F("Found non-string metadata property " 426 "name in test program '%s'") % 427 path); 428 const std::string property = state.to_string(-2); 429 430 if (property != "name" && property != "test_suite") { 431 if (!state.is_number(-1) && !state.is_string(-1)) 432 throw std::runtime_error( 433 F("Metadata property '%s' in test program '%s' cannot be " 434 "converted to a string") % property % path); 435 const std::string value = state.to_string(-1); 436 437 mdbuilder.set_string(property, value); 438 } 439 440 state.pop(1); 441 } 442 443 parser::get_from_state(state)->callback_test_program( 444 interface, path, test_suite, mdbuilder.build()); 445 return 0; 446} 447 448 449/// Specialization of lua_generic_test_program for ATF test programs. 450/// 451/// \param state The Lua state that executed the function. 452/// 453/// \return Number of return values left on the Lua stack. 454static int 455lua_atf_test_program(lutok::state& state) 456{ 457 return lua_generic_test_program(state, "atf"); 458} 459 460 461/// Glue to invoke parser::callback_current_kyuafile() from Lua. 462/// 463/// \param state The Lua state that executed the function. 464/// 465/// \return Number of return values left on the Lua stack. 466static int 467lua_current_kyuafile(lutok::state& state) 468{ 469 state.push_string(parser::get_from_state(state)-> 470 callback_current_kyuafile().str()); 471 return 1; 472} 473 474 475/// Glue to invoke parser::callback_include() from Lua. 476/// 477/// \param state The Lua state that executed the function. 478/// 479/// \return Number of return values left on the Lua stack. 480static int 481lua_include(lutok::state& state) 482{ 483 parser::get_from_state(state)->callback_include( 484 fs::path(state.to_string())); 485 return 0; 486} 487 488 489/// Specialization of lua_generic_test_program for plain test programs. 490/// 491/// \param state The Lua state that executed the function. 492/// 493/// \return Number of return values left on the Lua stack. 494static int 495lua_plain_test_program(lutok::state& state) 496{ 497 return lua_generic_test_program(state, "plain"); 498} 499 500 501/// Glue to invoke parser::callback_syntax() from Lua. 502/// 503/// \pre state(-2) The syntax format name, if a v1 file. 504/// \pre state(-1) The syntax format version. 505/// 506/// \param state The Lua state that executed the function. 507/// 508/// \return Number of return values left on the Lua stack. 509static int 510lua_syntax(lutok::state& state) 511{ 512 if (!state.is_number(-1)) 513 throw std::runtime_error("Last argument to syntax must be a number"); 514 const int syntax_version = state.to_integer(-1); 515 516 if (syntax_version == 1) { 517 if (state.get_top() != 2) 518 throw std::runtime_error("Version 1 files need two arguments to " 519 "syntax()"); 520 if (!state.is_string(-2) || state.to_string(-2) != "kyuafile") 521 throw std::runtime_error("First argument to syntax must be " 522 "'kyuafile' for version 1 files"); 523 } else { 524 if (state.get_top() != 1) 525 throw std::runtime_error("syntax() only takes one argument"); 526 } 527 528 parser::get_from_state(state)->callback_syntax(syntax_version); 529 return 0; 530} 531 532 533/// Glue to invoke parser::callback_test_suite() from Lua. 534/// 535/// \param state The Lua state that executed the function. 536/// 537/// \return Number of return values left on the Lua stack. 538static int 539lua_test_suite(lutok::state& state) 540{ 541 parser::get_from_state(state)->callback_test_suite(state.to_string()); 542 return 0; 543} 544 545 546} // anonymous namespace 547 548 549/// Constructs a kyuafile form initialized data. 550/// 551/// Use load() to parse a test suite configuration file and construct a 552/// kyuafile object. 553/// 554/// \param source_root_ The root directory for the test suite represented by the 555/// Kyuafile. In other words, the directory containing the first Kyuafile 556/// processed. 557/// \param build_root_ The root directory for the test programs themselves. In 558/// general, this will be the same as source_root_. If different, the 559/// specified directory must follow the exact same layout of source_root_. 560/// \param tps_ Collection of test programs that belong to this test suite. 561engine::kyuafile::kyuafile(const fs::path& source_root_, 562 const fs::path& build_root_, 563 const test_programs_vector& tps_) : 564 _source_root(source_root_), 565 _build_root(build_root_), 566 _test_programs(tps_) 567{ 568} 569 570 571/// Destructor. 572engine::kyuafile::~kyuafile(void) 573{ 574} 575 576 577/// Parses a test suite configuration file. 578/// 579/// \param file The file to parse. 580/// \param user_build_root If not none, specifies a path to a directory 581/// containing the test programs themselves. The layout of the build root 582/// must match the layout of the source root (which is just the directory 583/// from which the Kyuafile is being read). 584/// 585/// \return High-level representation of the configuration file. 586/// 587/// \throw load_error If there is any problem loading the file. This includes 588/// file access errors and syntax errors. 589engine::kyuafile 590engine::kyuafile::load(const fs::path& file, 591 const optional< fs::path > user_build_root) 592{ 593 const fs::path source_root_ = file.branch_path(); 594 const fs::path build_root_ = user_build_root ? 595 user_build_root.get() : source_root_; 596 597 return kyuafile(source_root_, build_root_, 598 parser(source_root_, build_root_, 599 fs::path(file.leaf_name())).parse()); 600} 601 602 603/// Gets the root directory of the test suite. 604/// 605/// \return A path. 606const fs::path& 607engine::kyuafile::source_root(void) const 608{ 609 return _source_root; 610} 611 612 613/// Gets the root directory of the test programs. 614/// 615/// \return A path. 616const fs::path& 617engine::kyuafile::build_root(void) const 618{ 619 return _build_root; 620} 621 622 623/// Gets the collection of test programs that belong to this test suite. 624/// 625/// \return Collection of test program executable names. 626const engine::test_programs_vector& 627engine::kyuafile::test_programs(void) const 628{ 629 return _test_programs; 630} 631