1#include <cstdio>
2#include <cstring>
3#include <fstream>
4#include <iostream>
5#include <map>
6#include <memory>
7#include <random>
8#include <sstream>
9#include <streambuf>
10
11#include "../src/benchmark_api_internal.h"
12#include "../src/check.h"  // NOTE: check.h is for internal use only!
13#include "../src/re.h"     // NOTE: re.h is for internal use only
14#include "output_test.h"
15
16// ========================================================================= //
17// ------------------------------ Internals -------------------------------- //
18// ========================================================================= //
19namespace internal {
20namespace {
21
22using TestCaseList = std::vector<TestCase>;
23
24// Use a vector because the order elements are added matters during iteration.
25// std::map/unordered_map don't guarantee that.
26// For example:
27//  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
28//     Substitute("%HelloWorld") // Always expands to Hello.
29using SubMap = std::vector<std::pair<std::string, std::string>>;
30
31TestCaseList& GetTestCaseList(TestCaseID ID) {
32  // Uses function-local statics to ensure initialization occurs
33  // before first use.
34  static TestCaseList lists[TC_NumID];
35  return lists[ID];
36}
37
38SubMap& GetSubstitutions() {
39  // Don't use 'dec_re' from header because it may not yet be initialized.
40  // clang-format off
41  static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
42  static std::string time_re = "([0-9]+[.])?[0-9]+";
43  static SubMap map = {
44      {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
45      // human-readable float
46      {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
47      {"%int", "[ ]*[0-9]+"},
48      {" %s ", "[ ]+"},
49      {"%time", "[ ]*" + time_re + "[ ]+ns"},
50      {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
51      {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
52      {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
53      {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
54      {"%csv_header",
55       "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
56       "items_per_second,label,error_occurred,error_message"},
57      {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
58      {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
59      {"%csv_bytes_report",
60       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
61      {"%csv_items_report",
62       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
63      {"%csv_bytes_items_report",
64       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
65       "," + safe_dec_re + ",,,"},
66      {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
67      {"%csv_label_report_end", ",,"}};
68  // clang-format on
69  return map;
70}
71
72std::string PerformSubstitutions(std::string source) {
73  SubMap const& subs = GetSubstitutions();
74  using SizeT = std::string::size_type;
75  for (auto const& KV : subs) {
76    SizeT pos;
77    SizeT next_start = 0;
78    while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
79      next_start = pos + KV.second.size();
80      source.replace(pos, KV.first.size(), KV.second);
81    }
82  }
83  return source;
84}
85
86void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
87               TestCaseList const& not_checks) {
88  std::string first_line;
89  bool on_first = true;
90  std::string line;
91  while (remaining_output.eof() == false) {
92    CHECK(remaining_output.good());
93    std::getline(remaining_output, line);
94    if (on_first) {
95      first_line = line;
96      on_first = false;
97    }
98    for (const auto& NC : not_checks) {
99      CHECK(!NC.regex->Match(line))
100          << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
101          << NC.regex_str << "\""
102          << "\n    actual regex string \"" << TC.substituted_regex << "\""
103          << "\n    started matching near: " << first_line;
104    }
105    if (TC.regex->Match(line)) return;
106    CHECK(TC.match_rule != MR_Next)
107        << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
108        << "\""
109        << "\n    actual regex string \"" << TC.substituted_regex << "\""
110        << "\n    started matching near: " << first_line;
111  }
112  CHECK(remaining_output.eof() == false)
113      << "End of output reached before match for regex \"" << TC.regex_str
114      << "\" was found"
115      << "\n    actual regex string \"" << TC.substituted_regex << "\""
116      << "\n    started matching near: " << first_line;
117}
118
119void CheckCases(TestCaseList const& checks, std::stringstream& output) {
120  std::vector<TestCase> not_checks;
121  for (size_t i = 0; i < checks.size(); ++i) {
122    const auto& TC = checks[i];
123    if (TC.match_rule == MR_Not) {
124      not_checks.push_back(TC);
125      continue;
126    }
127    CheckCase(output, TC, not_checks);
128    not_checks.clear();
129  }
130}
131
132class TestReporter : public benchmark::BenchmarkReporter {
133 public:
134  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
135      : reporters_(reps) {}
136
137  virtual bool ReportContext(const Context& context) {
138    bool last_ret = false;
139    bool first = true;
140    for (auto rep : reporters_) {
141      bool new_ret = rep->ReportContext(context);
142      CHECK(first || new_ret == last_ret)
143          << "Reports return different values for ReportContext";
144      first = false;
145      last_ret = new_ret;
146    }
147    (void)first;
148    return last_ret;
149  }
150
151  void ReportRuns(const std::vector<Run>& report) {
152    for (auto rep : reporters_) rep->ReportRuns(report);
153  }
154  void Finalize() {
155    for (auto rep : reporters_) rep->Finalize();
156  }
157
158 private:
159  std::vector<benchmark::BenchmarkReporter*> reporters_;
160};
161}  // namespace
162
163}  // end namespace internal
164
165// ========================================================================= //
166// -------------------------- Results checking ----------------------------- //
167// ========================================================================= //
168
169namespace internal {
170
171// Utility class to manage subscribers for checking benchmark results.
172// It works by parsing the CSV output to read the results.
173class ResultsChecker {
174 public:
175  struct PatternAndFn : public TestCase {  // reusing TestCase for its regexes
176    PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
177        : TestCase(rx), fn(fn_) {}
178    ResultsCheckFn fn;
179  };
180
181  std::vector<PatternAndFn> check_patterns;
182  std::vector<Results> results;
183  std::vector<std::string> field_names;
184
185  void Add(const std::string& entry_pattern, ResultsCheckFn fn);
186
187  void CheckResults(std::stringstream& output);
188
189 private:
190  void SetHeader_(const std::string& csv_header);
191  void SetValues_(const std::string& entry_csv_line);
192
193  std::vector<std::string> SplitCsv_(const std::string& line);
194};
195
196// store the static ResultsChecker in a function to prevent initialization
197// order problems
198ResultsChecker& GetResultsChecker() {
199  static ResultsChecker rc;
200  return rc;
201}
202
203// add a results checker for a benchmark
204void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
205  check_patterns.emplace_back(entry_pattern, fn);
206}
207
208// check the results of all subscribed benchmarks
209void ResultsChecker::CheckResults(std::stringstream& output) {
210  // first reset the stream to the start
211  {
212    auto start = std::stringstream::pos_type(0);
213    // clear before calling tellg()
214    output.clear();
215    // seek to zero only when needed
216    if (output.tellg() > start) output.seekg(start);
217    // and just in case
218    output.clear();
219  }
220  // now go over every line and publish it to the ResultsChecker
221  std::string line;
222  bool on_first = true;
223  while (output.eof() == false) {
224    CHECK(output.good());
225    std::getline(output, line);
226    if (on_first) {
227      SetHeader_(line);  // this is important
228      on_first = false;
229      continue;
230    }
231    SetValues_(line);
232  }
233  // finally we can call the subscribed check functions
234  for (const auto& p : check_patterns) {
235    VLOG(2) << "--------------------------------\n";
236    VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
237    for (const auto& r : results) {
238      if (!p.regex->Match(r.name)) {
239        VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
240        continue;
241      } else {
242        VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
243      }
244      VLOG(1) << "Checking results of " << r.name << ": ... \n";
245      p.fn(r);
246      VLOG(1) << "Checking results of " << r.name << ": OK.\n";
247    }
248  }
249}
250
251// prepare for the names in this header
252void ResultsChecker::SetHeader_(const std::string& csv_header) {
253  field_names = SplitCsv_(csv_header);
254}
255
256// set the values for a benchmark
257void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
258  if (entry_csv_line.empty()) return;  // some lines are empty
259  CHECK(!field_names.empty());
260  auto vals = SplitCsv_(entry_csv_line);
261  CHECK_EQ(vals.size(), field_names.size());
262  results.emplace_back(vals[0]);  // vals[0] is the benchmark name
263  auto& entry = results.back();
264  for (size_t i = 1, e = vals.size(); i < e; ++i) {
265    entry.values[field_names[i]] = vals[i];
266  }
267}
268
269// a quick'n'dirty csv splitter (eliminating quotes)
270std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {
271  std::vector<std::string> out;
272  if (line.empty()) return out;
273  if (!field_names.empty()) out.reserve(field_names.size());
274  size_t prev = 0, pos = line.find_first_of(','), curr = pos;
275  while (pos != line.npos) {
276    CHECK(curr > 0);
277    if (line[prev] == '"') ++prev;
278    if (line[curr - 1] == '"') --curr;
279    out.push_back(line.substr(prev, curr - prev));
280    prev = pos + 1;
281    pos = line.find_first_of(',', pos + 1);
282    curr = pos;
283  }
284  curr = line.size();
285  if (line[prev] == '"') ++prev;
286  if (line[curr - 1] == '"') --curr;
287  out.push_back(line.substr(prev, curr - prev));
288  return out;
289}
290
291}  // end namespace internal
292
293size_t AddChecker(const char* bm_name, ResultsCheckFn fn) {
294  auto& rc = internal::GetResultsChecker();
295  rc.Add(bm_name, fn);
296  return rc.results.size();
297}
298
299int Results::NumThreads() const {
300  auto pos = name.find("/threads:");
301  if (pos == name.npos) return 1;
302  auto end = name.find('/', pos + 9);
303  std::stringstream ss;
304  ss << name.substr(pos + 9, end);
305  int num = 1;
306  ss >> num;
307  CHECK(!ss.fail());
308  return num;
309}
310
311double Results::NumIterations() const {
312  return GetAs<double>("iterations");
313}
314
315double Results::GetTime(BenchmarkTime which) const {
316  CHECK(which == kCpuTime || which == kRealTime);
317  const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";
318  double val = GetAs<double>(which_str);
319  auto unit = Get("time_unit");
320  CHECK(unit);
321  if (*unit == "ns") {
322    return val * 1.e-9;
323  } else if (*unit == "us") {
324    return val * 1.e-6;
325  } else if (*unit == "ms") {
326    return val * 1.e-3;
327  } else if (*unit == "s") {
328    return val;
329  } else {
330    CHECK(1 == 0) << "unknown time unit: " << *unit;
331    return 0;
332  }
333}
334
335// ========================================================================= //
336// -------------------------- Public API Definitions------------------------ //
337// ========================================================================= //
338
339TestCase::TestCase(std::string re, int rule)
340    : regex_str(std::move(re)),
341      match_rule(rule),
342      substituted_regex(internal::PerformSubstitutions(regex_str)),
343      regex(std::make_shared<benchmark::Regex>()) {
344  std::string err_str;
345  regex->Init(substituted_regex, &err_str);
346  CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
347                         << "\""
348                         << "\n    originally \"" << regex_str << "\""
349                         << "\n    got error: " << err_str;
350}
351
352int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
353  auto& L = internal::GetTestCaseList(ID);
354  L.insert(L.end(), il);
355  return 0;
356}
357
358int SetSubstitutions(
359    std::initializer_list<std::pair<std::string, std::string>> il) {
360  auto& subs = internal::GetSubstitutions();
361  for (auto KV : il) {
362    bool exists = false;
363    KV.second = internal::PerformSubstitutions(KV.second);
364    for (auto& EKV : subs) {
365      if (EKV.first == KV.first) {
366        EKV.second = std::move(KV.second);
367        exists = true;
368        break;
369      }
370    }
371    if (!exists) subs.push_back(std::move(KV));
372  }
373  return 0;
374}
375
376void RunOutputTests(int argc, char* argv[]) {
377  using internal::GetTestCaseList;
378  benchmark::Initialize(&argc, argv);
379  auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);
380  benchmark::ConsoleReporter CR(options);
381  benchmark::JSONReporter JR;
382  benchmark::CSVReporter CSVR;
383  struct ReporterTest {
384    const char* name;
385    std::vector<TestCase>& output_cases;
386    std::vector<TestCase>& error_cases;
387    benchmark::BenchmarkReporter& reporter;
388    std::stringstream out_stream;
389    std::stringstream err_stream;
390
391    ReporterTest(const char* n, std::vector<TestCase>& out_tc,
392                 std::vector<TestCase>& err_tc,
393                 benchmark::BenchmarkReporter& br)
394        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
395      reporter.SetOutputStream(&out_stream);
396      reporter.SetErrorStream(&err_stream);
397    }
398  } TestCases[] = {
399      {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
400       GetTestCaseList(TC_ConsoleErr), CR},
401      {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
402       JR},
403      {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
404       CSVR},
405  };
406
407  // Create the test reporter and run the benchmarks.
408  std::cout << "Running benchmarks...\n";
409  internal::TestReporter test_rep({&CR, &JR, &CSVR});
410  benchmark::RunSpecifiedBenchmarks(&test_rep);
411
412  for (auto& rep_test : TestCases) {
413    std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
414    std::string banner(msg.size() - 1, '-');
415    std::cout << banner << msg << banner << "\n";
416
417    std::cerr << rep_test.err_stream.str();
418    std::cout << rep_test.out_stream.str();
419
420    internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
421    internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
422
423    std::cout << "\n";
424  }
425
426  // now that we know the output is as expected, we can dispatch
427  // the checks to subscribees.
428  auto& csv = TestCases[2];
429  // would use == but gcc spits a warning
430  CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
431  internal::GetResultsChecker().CheckResults(csv.out_stream);
432}
433
434int SubstrCnt(const std::string& haystack, const std::string& pat) {
435  if (pat.length() == 0) return 0;
436  int count = 0;
437  for (size_t offset = haystack.find(pat); offset != std::string::npos;
438       offset = haystack.find(pat, offset + pat.length()))
439    ++count;
440  return count;
441}
442
443static char ToHex(int ch) {
444  return ch < 10 ? static_cast<char>('0' + ch)
445                 : static_cast<char>('a' + (ch - 10));
446}
447
448static char RandomHexChar() {
449  static std::mt19937 rd{std::random_device{}()};
450  static std::uniform_int_distribution<int> mrand{0, 15};
451  return ToHex(mrand(rd));
452}
453
454static std::string GetRandomFileName() {
455  std::string model = "test.%%%%%%";
456  for (auto & ch :  model) {
457    if (ch == '%')
458      ch = RandomHexChar();
459  }
460  return model;
461}
462
463static bool FileExists(std::string const& name) {
464  std::ifstream in(name.c_str());
465  return in.good();
466}
467
468static std::string GetTempFileName() {
469  // This function attempts to avoid race conditions where two tests
470  // create the same file at the same time. However, it still introduces races
471  // similar to tmpnam.
472  int retries = 3;
473  while (--retries) {
474    std::string name = GetRandomFileName();
475    if (!FileExists(name))
476      return name;
477  }
478  std::cerr << "Failed to create unique temporary file name" << std::endl;
479  std::abort();
480}
481
482std::string GetFileReporterOutput(int argc, char* argv[]) {
483  std::vector<char*> new_argv(argv, argv + argc);
484  assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
485
486  std::string tmp_file_name = GetTempFileName();
487  std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';
488
489  std::string tmp = "--benchmark_out=";
490  tmp += tmp_file_name;
491  new_argv.emplace_back(const_cast<char*>(tmp.c_str()));
492
493  argc = int(new_argv.size());
494
495  benchmark::Initialize(&argc, new_argv.data());
496  benchmark::RunSpecifiedBenchmarks();
497
498  // Read the output back from the file, and delete the file.
499  std::ifstream tmp_stream(tmp_file_name);
500  std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),
501                                   std::istreambuf_iterator<char>());
502  std::remove(tmp_file_name.c_str());
503
504  return output;
505}
506