1// Copyright 2011 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "store/write_transaction.hpp" 30 31extern "C" { 32#include <stdint.h> 33} 34 35#include <fstream> 36#include <map> 37 38#include "model/context.hpp" 39#include "model/metadata.hpp" 40#include "model/test_case.hpp" 41#include "model/test_program.hpp" 42#include "model/test_result.hpp" 43#include "model/types.hpp" 44#include "store/dbtypes.hpp" 45#include "store/exceptions.hpp" 46#include "store/write_backend.hpp" 47#include "utils/datetime.hpp" 48#include "utils/format/macros.hpp" 49#include "utils/fs/path.hpp" 50#include "utils/logging/macros.hpp" 51#include "utils/noncopyable.hpp" 52#include "utils/optional.ipp" 53#include "utils/sanity.hpp" 54#include "utils/stream.hpp" 55#include "utils/sqlite/database.hpp" 56#include "utils/sqlite/exceptions.hpp" 57#include "utils/sqlite/statement.ipp" 58#include "utils/sqlite/transaction.hpp" 59 60namespace datetime = utils::datetime; 61namespace fs = utils::fs; 62namespace sqlite = utils::sqlite; 63 64using utils::none; 65using utils::optional; 66 67 68namespace { 69 70 71/// Stores the environment variables of a context. 72/// 73/// \param db The SQLite database. 74/// \param env The environment variables to store. 75/// 76/// \throw sqlite::error If there is a problem storing the variables. 77static void 78put_env_vars(sqlite::database& db, 79 const std::map< std::string, std::string >& env) 80{ 81 sqlite::statement stmt = db.create_statement( 82 "INSERT INTO env_vars (var_name, var_value) " 83 "VALUES (:var_name, :var_value)"); 84 for (std::map< std::string, std::string >::const_iterator iter = 85 env.begin(); iter != env.end(); iter++) { 86 stmt.bind(":var_name", (*iter).first); 87 stmt.bind(":var_value", (*iter).second); 88 stmt.step_without_results(); 89 stmt.reset(); 90 } 91} 92 93 94/// Calculates the last rowid of a table. 95/// 96/// \param db The SQLite database. 97/// \param table Name of the table. 98/// 99/// \return The last rowid; 0 if the table is empty. 100static int64_t 101last_rowid(sqlite::database& db, const std::string& table) 102{ 103 sqlite::statement stmt = db.create_statement( 104 F("SELECT MAX(ROWID) AS max_rowid FROM %s") % table); 105 stmt.step(); 106 if (stmt.column_type(0) == sqlite::type_null) { 107 return 0; 108 } else { 109 INV(stmt.column_type(0) == sqlite::type_integer); 110 return stmt.column_int64(0); 111 } 112} 113 114 115/// Stores a metadata object. 116/// 117/// \param db The database into which to store the information. 118/// \param md The metadata to store. 119/// 120/// \return The identifier of the new metadata object. 121static int64_t 122put_metadata(sqlite::database& db, const model::metadata& md) 123{ 124 const model::properties_map props = md.to_properties(); 125 126 const int64_t metadata_id = last_rowid(db, "metadatas"); 127 128 sqlite::statement stmt = db.create_statement( 129 "INSERT INTO metadatas (metadata_id, property_name, property_value) " 130 "VALUES (:metadata_id, :property_name, :property_value)"); 131 stmt.bind(":metadata_id", metadata_id); 132 133 for (model::properties_map::const_iterator iter = props.begin(); 134 iter != props.end(); ++iter) { 135 stmt.bind(":property_name", (*iter).first); 136 stmt.bind(":property_value", (*iter).second); 137 stmt.step_without_results(); 138 stmt.reset(); 139 } 140 141 return metadata_id; 142} 143 144 145/// Stores an arbitrary file into the database as a BLOB. 146/// 147/// \param db The database into which to store the file. 148/// \param path Path to the file to be stored. 149/// 150/// \return The identifier of the stored file, or none if the file was empty. 151/// 152/// \throw sqlite::error If there are problems writing to the database. 153static optional< int64_t > 154put_file(sqlite::database& db, const fs::path& path) 155{ 156 std::ifstream input(path.c_str()); 157 if (!input) 158 throw store::error(F("Cannot open file %s") % path); 159 160 try { 161 if (utils::stream_length(input) == 0) 162 return none; 163 } catch (const std::runtime_error& e) { 164 // Skipping empty files is an optimization. If we fail to calculate the 165 // size of the file, just ignore the problem. If there are real issues 166 // with the file, the read below will fail anyway. 167 LD(F("Cannot determine if file is empty: %s") % e.what()); 168 } 169 170 // TODO(jmmv): This will probably cause an unreasonable amount of memory 171 // consumption if we decide to store arbitrary files in the database (other 172 // than stdout or stderr). Should this happen, we need to investigate a 173 // better way to feel blobs into SQLite. 174 const std::string contents = utils::read_stream(input); 175 176 sqlite::statement stmt = db.create_statement( 177 "INSERT INTO files (contents) VALUES (:contents)"); 178 stmt.bind(":contents", sqlite::blob(contents.c_str(), contents.length())); 179 stmt.step_without_results(); 180 181 return optional< int64_t >(db.last_insert_rowid()); 182} 183 184 185} // anonymous namespace 186 187 188/// Internal implementation for a store write-only transaction. 189struct store::write_transaction::impl : utils::noncopyable { 190 /// The backend instance. 191 store::write_backend& _backend; 192 193 /// The SQLite database this transaction deals with. 194 sqlite::database _db; 195 196 /// The backing SQLite transaction. 197 sqlite::transaction _tx; 198 199 /// Opens a transaction. 200 /// 201 /// \param backend_ The backend this transaction is connected to. 202 impl(write_backend& backend_) : 203 _backend(backend_), 204 _db(backend_.database()), 205 _tx(backend_.database().begin_transaction()) 206 { 207 } 208}; 209 210 211/// Creates a new write-only transaction. 212/// 213/// \param backend_ The backend this transaction belongs to. 214store::write_transaction::write_transaction(write_backend& backend_) : 215 _pimpl(new impl(backend_)) 216{ 217} 218 219 220/// Destructor. 221store::write_transaction::~write_transaction(void) 222{ 223} 224 225 226/// Commits the transaction. 227/// 228/// \throw error If there is any problem when talking to the database. 229void 230store::write_transaction::commit(void) 231{ 232 try { 233 _pimpl->_tx.commit(); 234 } catch (const sqlite::error& e) { 235 throw error(e.what()); 236 } 237} 238 239 240/// Rolls the transaction back. 241/// 242/// \throw error If there is any problem when talking to the database. 243void 244store::write_transaction::rollback(void) 245{ 246 try { 247 _pimpl->_tx.rollback(); 248 } catch (const sqlite::error& e) { 249 throw error(e.what()); 250 } 251} 252 253 254/// Puts a context into the database. 255/// 256/// \pre The context has not been put yet. 257/// \post The context is stored into the database with a new identifier. 258/// 259/// \param context The context to put. 260/// 261/// \throw error If there is any problem when talking to the database. 262void 263store::write_transaction::put_context(const model::context& context) 264{ 265 try { 266 sqlite::statement stmt = _pimpl->_db.create_statement( 267 "INSERT INTO contexts (cwd) VALUES (:cwd)"); 268 stmt.bind(":cwd", context.cwd().str()); 269 stmt.step_without_results(); 270 271 put_env_vars(_pimpl->_db, context.env()); 272 } catch (const sqlite::error& e) { 273 throw error(e.what()); 274 } 275} 276 277 278/// Puts a test program into the database. 279/// 280/// \pre The test program has not been put yet. 281/// \post The test program is stored into the database with a new identifier. 282/// 283/// \param test_program The test program to put. 284/// 285/// \return The identifier of the inserted test program. 286/// 287/// \throw error If there is any problem when talking to the database. 288int64_t 289store::write_transaction::put_test_program( 290 const model::test_program& test_program) 291{ 292 try { 293 const int64_t metadata_id = put_metadata( 294 _pimpl->_db, test_program.get_metadata()); 295 296 sqlite::statement stmt = _pimpl->_db.create_statement( 297 "INSERT INTO test_programs (absolute_path, " 298 " root, relative_path, test_suite_name, " 299 " metadata_id, interface) " 300 "VALUES (:absolute_path, :root, :relative_path, " 301 " :test_suite_name, :metadata_id, :interface)"); 302 stmt.bind(":absolute_path", test_program.absolute_path().str()); 303 // TODO(jmmv): The root is not necessarily absolute. We need to ensure 304 // that we can recover the absolute path of the test program. Maybe we 305 // need to change the test_program to always ensure root is absolute? 306 stmt.bind(":root", test_program.root().str()); 307 stmt.bind(":relative_path", test_program.relative_path().str()); 308 stmt.bind(":test_suite_name", test_program.test_suite_name()); 309 stmt.bind(":metadata_id", metadata_id); 310 stmt.bind(":interface", test_program.interface_name()); 311 stmt.step_without_results(); 312 return _pimpl->_db.last_insert_rowid(); 313 } catch (const sqlite::error& e) { 314 throw error(e.what()); 315 } 316} 317 318 319/// Puts a test case into the database. 320/// 321/// \pre The test case has not been put yet. 322/// \post The test case is stored into the database with a new identifier. 323/// 324/// \param test_program The program containing the test case to be stored. 325/// \param test_case_name The name of the test case to put. 326/// \param test_program_id The test program this test case belongs to. 327/// 328/// \return The identifier of the inserted test case. 329/// 330/// \throw error If there is any problem when talking to the database. 331int64_t 332store::write_transaction::put_test_case(const model::test_program& test_program, 333 const std::string& test_case_name, 334 const int64_t test_program_id) 335{ 336 const model::test_case& test_case = test_program.find(test_case_name); 337 338 try { 339 const int64_t metadata_id = put_metadata( 340 _pimpl->_db, test_case.get_raw_metadata()); 341 342 sqlite::statement stmt = _pimpl->_db.create_statement( 343 "INSERT INTO test_cases (test_program_id, name, metadata_id) " 344 "VALUES (:test_program_id, :name, :metadata_id)"); 345 stmt.bind(":test_program_id", test_program_id); 346 stmt.bind(":name", test_case.name()); 347 stmt.bind(":metadata_id", metadata_id); 348 stmt.step_without_results(); 349 return _pimpl->_db.last_insert_rowid(); 350 } catch (const sqlite::error& e) { 351 throw error(e.what()); 352 } 353} 354 355 356/// Stores a file generated by a test case into the database as a BLOB. 357/// 358/// \param name The name of the file to store in the database. This needs to be 359/// unique per test case. The caller is free to decide what names to use 360/// for which files. For example, it might make sense to always call 361/// __STDOUT__ the stdout of the test case so that it is easy to locate. 362/// \param path The path to the file to be stored. 363/// \param test_case_id The identifier of the test case this file belongs to. 364/// 365/// \return The identifier of the stored file, or none if the file was empty. 366/// 367/// \throw store::error If there are problems writing to the database. 368optional< int64_t > 369store::write_transaction::put_test_case_file(const std::string& name, 370 const fs::path& path, 371 const int64_t test_case_id) 372{ 373 LD(F("Storing %s (%s) of test case %s") % name % path % test_case_id); 374 try { 375 const optional< int64_t > file_id = put_file(_pimpl->_db, path); 376 if (!file_id) { 377 LD("Not storing empty file"); 378 return none; 379 } 380 381 sqlite::statement stmt = _pimpl->_db.create_statement( 382 "INSERT INTO test_case_files (test_case_id, file_name, file_id) " 383 "VALUES (:test_case_id, :file_name, :file_id)"); 384 stmt.bind(":test_case_id", test_case_id); 385 stmt.bind(":file_name", name); 386 stmt.bind(":file_id", file_id.get()); 387 stmt.step_without_results(); 388 389 return optional< int64_t >(_pimpl->_db.last_insert_rowid()); 390 } catch (const sqlite::error& e) { 391 throw error(e.what()); 392 } 393} 394 395 396/// Puts a result into the database. 397/// 398/// \pre The result has not been put yet. 399/// \post The result is stored into the database with a new identifier. 400/// 401/// \param result The result to put. 402/// \param test_case_id The test case this result corresponds to. 403/// \param start_time The time when the test started to run. 404/// \param end_time The time when the test finished running. 405/// 406/// \return The identifier of the inserted result. 407/// 408/// \throw error If there is any problem when talking to the database. 409int64_t 410store::write_transaction::put_result(const model::test_result& result, 411 const int64_t test_case_id, 412 const datetime::timestamp& start_time, 413 const datetime::timestamp& end_time) 414{ 415 try { 416 sqlite::statement stmt = _pimpl->_db.create_statement( 417 "INSERT INTO test_results (test_case_id, result_type, " 418 " result_reason, start_time, " 419 " end_time) " 420 "VALUES (:test_case_id, :result_type, :result_reason, " 421 " :start_time, :end_time)"); 422 stmt.bind(":test_case_id", test_case_id); 423 424 store::bind_test_result_type(stmt, ":result_type", result.type()); 425 if (result.reason().empty()) 426 stmt.bind(":result_reason", sqlite::null()); 427 else 428 stmt.bind(":result_reason", result.reason()); 429 430 store::bind_timestamp(stmt, ":start_time", start_time); 431 store::bind_timestamp(stmt, ":end_time", end_time); 432 433 stmt.step_without_results(); 434 const int64_t result_id = _pimpl->_db.last_insert_rowid(); 435 436 return result_id; 437 } catch (const sqlite::error& e) { 438 throw error(e.what()); 439 } 440} 441