1#include <cstdio> 2#include <cstring> 3#include <fstream> 4#include <iostream> 5#include <map> 6#include <memory> 7#include <random> 8#include <sstream> 9#include <streambuf> 10 11#include "../src/benchmark_api_internal.h" 12#include "../src/check.h" // NOTE: check.h is for internal use only! 13#include "../src/re.h" // NOTE: re.h is for internal use only 14#include "output_test.h" 15 16// ========================================================================= // 17// ------------------------------ Internals -------------------------------- // 18// ========================================================================= // 19namespace internal { 20namespace { 21 22using TestCaseList = std::vector<TestCase>; 23 24// Use a vector because the order elements are added matters during iteration. 25// std::map/unordered_map don't guarantee that. 26// For example: 27// SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); 28// Substitute("%HelloWorld") // Always expands to Hello. 29using SubMap = std::vector<std::pair<std::string, std::string>>; 30 31TestCaseList& GetTestCaseList(TestCaseID ID) { 32 // Uses function-local statics to ensure initialization occurs 33 // before first use. 34 static TestCaseList lists[TC_NumID]; 35 return lists[ID]; 36} 37 38SubMap& GetSubstitutions() { 39 // Don't use 'dec_re' from header because it may not yet be initialized. 40 // clang-format off 41 static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; 42 static std::string time_re = "([0-9]+[.])?[0-9]+"; 43 static SubMap map = { 44 {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, 45 // human-readable float 46 {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, 47 {"%int", "[ ]*[0-9]+"}, 48 {" %s ", "[ ]+"}, 49 {"%time", "[ ]*" + time_re + "[ ]+ns"}, 50 {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, 51 {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, 52 {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, 53 {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, 54 {"%csv_header", 55 "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," 56 "items_per_second,label,error_occurred,error_message"}, 57 {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, 58 {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, 59 {"%csv_bytes_report", 60 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, 61 {"%csv_items_report", 62 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, 63 {"%csv_bytes_items_report", 64 "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + 65 "," + safe_dec_re + ",,,"}, 66 {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, 67 {"%csv_label_report_end", ",,"}}; 68 // clang-format on 69 return map; 70} 71 72std::string PerformSubstitutions(std::string source) { 73 SubMap const& subs = GetSubstitutions(); 74 using SizeT = std::string::size_type; 75 for (auto const& KV : subs) { 76 SizeT pos; 77 SizeT next_start = 0; 78 while ((pos = source.find(KV.first, next_start)) != std::string::npos) { 79 next_start = pos + KV.second.size(); 80 source.replace(pos, KV.first.size(), KV.second); 81 } 82 } 83 return source; 84} 85 86void CheckCase(std::stringstream& remaining_output, TestCase const& TC, 87 TestCaseList const& not_checks) { 88 std::string first_line; 89 bool on_first = true; 90 std::string line; 91 while (remaining_output.eof() == false) { 92 CHECK(remaining_output.good()); 93 std::getline(remaining_output, line); 94 if (on_first) { 95 first_line = line; 96 on_first = false; 97 } 98 for (const auto& NC : not_checks) { 99 CHECK(!NC.regex->Match(line)) 100 << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" 101 << NC.regex_str << "\"" 102 << "\n actual regex string \"" << TC.substituted_regex << "\"" 103 << "\n started matching near: " << first_line; 104 } 105 if (TC.regex->Match(line)) return; 106 CHECK(TC.match_rule != MR_Next) 107 << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str 108 << "\"" 109 << "\n actual regex string \"" << TC.substituted_regex << "\"" 110 << "\n started matching near: " << first_line; 111 } 112 CHECK(remaining_output.eof() == false) 113 << "End of output reached before match for regex \"" << TC.regex_str 114 << "\" was found" 115 << "\n actual regex string \"" << TC.substituted_regex << "\"" 116 << "\n started matching near: " << first_line; 117} 118 119void CheckCases(TestCaseList const& checks, std::stringstream& output) { 120 std::vector<TestCase> not_checks; 121 for (size_t i = 0; i < checks.size(); ++i) { 122 const auto& TC = checks[i]; 123 if (TC.match_rule == MR_Not) { 124 not_checks.push_back(TC); 125 continue; 126 } 127 CheckCase(output, TC, not_checks); 128 not_checks.clear(); 129 } 130} 131 132class TestReporter : public benchmark::BenchmarkReporter { 133 public: 134 TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) 135 : reporters_(reps) {} 136 137 virtual bool ReportContext(const Context& context) { 138 bool last_ret = false; 139 bool first = true; 140 for (auto rep : reporters_) { 141 bool new_ret = rep->ReportContext(context); 142 CHECK(first || new_ret == last_ret) 143 << "Reports return different values for ReportContext"; 144 first = false; 145 last_ret = new_ret; 146 } 147 (void)first; 148 return last_ret; 149 } 150 151 void ReportRuns(const std::vector<Run>& report) { 152 for (auto rep : reporters_) rep->ReportRuns(report); 153 } 154 void Finalize() { 155 for (auto rep : reporters_) rep->Finalize(); 156 } 157 158 private: 159 std::vector<benchmark::BenchmarkReporter*> reporters_; 160}; 161} // namespace 162 163} // end namespace internal 164 165// ========================================================================= // 166// -------------------------- Results checking ----------------------------- // 167// ========================================================================= // 168 169namespace internal { 170 171// Utility class to manage subscribers for checking benchmark results. 172// It works by parsing the CSV output to read the results. 173class ResultsChecker { 174 public: 175 struct PatternAndFn : public TestCase { // reusing TestCase for its regexes 176 PatternAndFn(const std::string& rx, ResultsCheckFn fn_) 177 : TestCase(rx), fn(fn_) {} 178 ResultsCheckFn fn; 179 }; 180 181 std::vector<PatternAndFn> check_patterns; 182 std::vector<Results> results; 183 std::vector<std::string> field_names; 184 185 void Add(const std::string& entry_pattern, ResultsCheckFn fn); 186 187 void CheckResults(std::stringstream& output); 188 189 private: 190 void SetHeader_(const std::string& csv_header); 191 void SetValues_(const std::string& entry_csv_line); 192 193 std::vector<std::string> SplitCsv_(const std::string& line); 194}; 195 196// store the static ResultsChecker in a function to prevent initialization 197// order problems 198ResultsChecker& GetResultsChecker() { 199 static ResultsChecker rc; 200 return rc; 201} 202 203// add a results checker for a benchmark 204void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { 205 check_patterns.emplace_back(entry_pattern, fn); 206} 207 208// check the results of all subscribed benchmarks 209void ResultsChecker::CheckResults(std::stringstream& output) { 210 // first reset the stream to the start 211 { 212 auto start = std::stringstream::pos_type(0); 213 // clear before calling tellg() 214 output.clear(); 215 // seek to zero only when needed 216 if (output.tellg() > start) output.seekg(start); 217 // and just in case 218 output.clear(); 219 } 220 // now go over every line and publish it to the ResultsChecker 221 std::string line; 222 bool on_first = true; 223 while (output.eof() == false) { 224 CHECK(output.good()); 225 std::getline(output, line); 226 if (on_first) { 227 SetHeader_(line); // this is important 228 on_first = false; 229 continue; 230 } 231 SetValues_(line); 232 } 233 // finally we can call the subscribed check functions 234 for (const auto& p : check_patterns) { 235 VLOG(2) << "--------------------------------\n"; 236 VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; 237 for (const auto& r : results) { 238 if (!p.regex->Match(r.name)) { 239 VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; 240 continue; 241 } else { 242 VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; 243 } 244 VLOG(1) << "Checking results of " << r.name << ": ... \n"; 245 p.fn(r); 246 VLOG(1) << "Checking results of " << r.name << ": OK.\n"; 247 } 248 } 249} 250 251// prepare for the names in this header 252void ResultsChecker::SetHeader_(const std::string& csv_header) { 253 field_names = SplitCsv_(csv_header); 254} 255 256// set the values for a benchmark 257void ResultsChecker::SetValues_(const std::string& entry_csv_line) { 258 if (entry_csv_line.empty()) return; // some lines are empty 259 CHECK(!field_names.empty()); 260 auto vals = SplitCsv_(entry_csv_line); 261 CHECK_EQ(vals.size(), field_names.size()); 262 results.emplace_back(vals[0]); // vals[0] is the benchmark name 263 auto& entry = results.back(); 264 for (size_t i = 1, e = vals.size(); i < e; ++i) { 265 entry.values[field_names[i]] = vals[i]; 266 } 267} 268 269// a quick'n'dirty csv splitter (eliminating quotes) 270std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) { 271 std::vector<std::string> out; 272 if (line.empty()) return out; 273 if (!field_names.empty()) out.reserve(field_names.size()); 274 size_t prev = 0, pos = line.find_first_of(','), curr = pos; 275 while (pos != line.npos) { 276 CHECK(curr > 0); 277 if (line[prev] == '"') ++prev; 278 if (line[curr - 1] == '"') --curr; 279 out.push_back(line.substr(prev, curr - prev)); 280 prev = pos + 1; 281 pos = line.find_first_of(',', pos + 1); 282 curr = pos; 283 } 284 curr = line.size(); 285 if (line[prev] == '"') ++prev; 286 if (line[curr - 1] == '"') --curr; 287 out.push_back(line.substr(prev, curr - prev)); 288 return out; 289} 290 291} // end namespace internal 292 293size_t AddChecker(const char* bm_name, ResultsCheckFn fn) { 294 auto& rc = internal::GetResultsChecker(); 295 rc.Add(bm_name, fn); 296 return rc.results.size(); 297} 298 299int Results::NumThreads() const { 300 auto pos = name.find("/threads:"); 301 if (pos == name.npos) return 1; 302 auto end = name.find('/', pos + 9); 303 std::stringstream ss; 304 ss << name.substr(pos + 9, end); 305 int num = 1; 306 ss >> num; 307 CHECK(!ss.fail()); 308 return num; 309} 310 311double Results::NumIterations() const { 312 return GetAs<double>("iterations"); 313} 314 315double Results::GetTime(BenchmarkTime which) const { 316 CHECK(which == kCpuTime || which == kRealTime); 317 const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; 318 double val = GetAs<double>(which_str); 319 auto unit = Get("time_unit"); 320 CHECK(unit); 321 if (*unit == "ns") { 322 return val * 1.e-9; 323 } else if (*unit == "us") { 324 return val * 1.e-6; 325 } else if (*unit == "ms") { 326 return val * 1.e-3; 327 } else if (*unit == "s") { 328 return val; 329 } else { 330 CHECK(1 == 0) << "unknown time unit: " << *unit; 331 return 0; 332 } 333} 334 335// ========================================================================= // 336// -------------------------- Public API Definitions------------------------ // 337// ========================================================================= // 338 339TestCase::TestCase(std::string re, int rule) 340 : regex_str(std::move(re)), 341 match_rule(rule), 342 substituted_regex(internal::PerformSubstitutions(regex_str)), 343 regex(std::make_shared<benchmark::Regex>()) { 344 std::string err_str; 345 regex->Init(substituted_regex, &err_str); 346 CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex 347 << "\"" 348 << "\n originally \"" << regex_str << "\"" 349 << "\n got error: " << err_str; 350} 351 352int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { 353 auto& L = internal::GetTestCaseList(ID); 354 L.insert(L.end(), il); 355 return 0; 356} 357 358int SetSubstitutions( 359 std::initializer_list<std::pair<std::string, std::string>> il) { 360 auto& subs = internal::GetSubstitutions(); 361 for (auto KV : il) { 362 bool exists = false; 363 KV.second = internal::PerformSubstitutions(KV.second); 364 for (auto& EKV : subs) { 365 if (EKV.first == KV.first) { 366 EKV.second = std::move(KV.second); 367 exists = true; 368 break; 369 } 370 } 371 if (!exists) subs.push_back(std::move(KV)); 372 } 373 return 0; 374} 375 376void RunOutputTests(int argc, char* argv[]) { 377 using internal::GetTestCaseList; 378 benchmark::Initialize(&argc, argv); 379 auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true); 380 benchmark::ConsoleReporter CR(options); 381 benchmark::JSONReporter JR; 382 benchmark::CSVReporter CSVR; 383 struct ReporterTest { 384 const char* name; 385 std::vector<TestCase>& output_cases; 386 std::vector<TestCase>& error_cases; 387 benchmark::BenchmarkReporter& reporter; 388 std::stringstream out_stream; 389 std::stringstream err_stream; 390 391 ReporterTest(const char* n, std::vector<TestCase>& out_tc, 392 std::vector<TestCase>& err_tc, 393 benchmark::BenchmarkReporter& br) 394 : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { 395 reporter.SetOutputStream(&out_stream); 396 reporter.SetErrorStream(&err_stream); 397 } 398 } TestCases[] = { 399 {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), 400 GetTestCaseList(TC_ConsoleErr), CR}, 401 {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), 402 JR}, 403 {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), 404 CSVR}, 405 }; 406 407 // Create the test reporter and run the benchmarks. 408 std::cout << "Running benchmarks...\n"; 409 internal::TestReporter test_rep({&CR, &JR, &CSVR}); 410 benchmark::RunSpecifiedBenchmarks(&test_rep); 411 412 for (auto& rep_test : TestCases) { 413 std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; 414 std::string banner(msg.size() - 1, '-'); 415 std::cout << banner << msg << banner << "\n"; 416 417 std::cerr << rep_test.err_stream.str(); 418 std::cout << rep_test.out_stream.str(); 419 420 internal::CheckCases(rep_test.error_cases, rep_test.err_stream); 421 internal::CheckCases(rep_test.output_cases, rep_test.out_stream); 422 423 std::cout << "\n"; 424 } 425 426 // now that we know the output is as expected, we can dispatch 427 // the checks to subscribees. 428 auto& csv = TestCases[2]; 429 // would use == but gcc spits a warning 430 CHECK(std::strcmp(csv.name, "CSVReporter") == 0); 431 internal::GetResultsChecker().CheckResults(csv.out_stream); 432} 433 434int SubstrCnt(const std::string& haystack, const std::string& pat) { 435 if (pat.length() == 0) return 0; 436 int count = 0; 437 for (size_t offset = haystack.find(pat); offset != std::string::npos; 438 offset = haystack.find(pat, offset + pat.length())) 439 ++count; 440 return count; 441} 442 443static char ToHex(int ch) { 444 return ch < 10 ? static_cast<char>('0' + ch) 445 : static_cast<char>('a' + (ch - 10)); 446} 447 448static char RandomHexChar() { 449 static std::mt19937 rd{std::random_device{}()}; 450 static std::uniform_int_distribution<int> mrand{0, 15}; 451 return ToHex(mrand(rd)); 452} 453 454static std::string GetRandomFileName() { 455 std::string model = "test.%%%%%%"; 456 for (auto & ch : model) { 457 if (ch == '%') 458 ch = RandomHexChar(); 459 } 460 return model; 461} 462 463static bool FileExists(std::string const& name) { 464 std::ifstream in(name.c_str()); 465 return in.good(); 466} 467 468static std::string GetTempFileName() { 469 // This function attempts to avoid race conditions where two tests 470 // create the same file at the same time. However, it still introduces races 471 // similar to tmpnam. 472 int retries = 3; 473 while (--retries) { 474 std::string name = GetRandomFileName(); 475 if (!FileExists(name)) 476 return name; 477 } 478 std::cerr << "Failed to create unique temporary file name" << std::endl; 479 std::abort(); 480} 481 482std::string GetFileReporterOutput(int argc, char* argv[]) { 483 std::vector<char*> new_argv(argv, argv + argc); 484 assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size()); 485 486 std::string tmp_file_name = GetTempFileName(); 487 std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n'; 488 489 std::string tmp = "--benchmark_out="; 490 tmp += tmp_file_name; 491 new_argv.emplace_back(const_cast<char*>(tmp.c_str())); 492 493 argc = int(new_argv.size()); 494 495 benchmark::Initialize(&argc, new_argv.data()); 496 benchmark::RunSpecifiedBenchmarks(); 497 498 // Read the output back from the file, and delete the file. 499 std::ifstream tmp_stream(tmp_file_name); 500 std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)), 501 std::istreambuf_iterator<char>()); 502 std::remove(tmp_file_name.c_str()); 503 504 return output; 505} 506