1#include <iostream>
2#include <map>
3#include <memory>
4#include <sstream>
5#include <cstring>
6
7#include "../src/check.h"  // NOTE: check.h is for internal use only!
8#include "../src/re.h"     // NOTE: re.h is for internal use only
9#include "output_test.h"
10#include "../src/benchmark_api_internal.h"
11
12// ========================================================================= //
13// ------------------------------ Internals -------------------------------- //
14// ========================================================================= //
15namespace internal {
16namespace {
17
18using TestCaseList = std::vector<TestCase>;
19
20// Use a vector because the order elements are added matters during iteration.
21// std::map/unordered_map don't guarantee that.
22// For example:
23//  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
24//     Substitute("%HelloWorld") // Always expands to Hello.
25using SubMap = std::vector<std::pair<std::string, std::string>>;
26
27TestCaseList& GetTestCaseList(TestCaseID ID) {
28  // Uses function-local statics to ensure initialization occurs
29  // before first use.
30  static TestCaseList lists[TC_NumID];
31  return lists[ID];
32}
33
34SubMap& GetSubstitutions() {
35  // Don't use 'dec_re' from header because it may not yet be initialized.
36  static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
37  static SubMap map = {
38      {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
39      // human-readable float
40      {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
41      {"%int", "[ ]*[0-9]+"},
42      {" %s ", "[ ]+"},
43      {"%time", "[ ]*[0-9]{1,6} ns"},
44      {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"},
45      {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
46      {"%csv_header",
47       "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
48       "items_per_second,label,error_occurred,error_message"},
49      {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
50      {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
51      {"%csv_bytes_report",
52       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
53      {"%csv_items_report",
54       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
55      {"%csv_bytes_items_report",
56       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
57       "," + safe_dec_re + ",,,"},
58      {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
59      {"%csv_label_report_end", ",,"}};
60  return map;
61}
62
63std::string PerformSubstitutions(std::string source) {
64  SubMap const& subs = GetSubstitutions();
65  using SizeT = std::string::size_type;
66  for (auto const& KV : subs) {
67    SizeT pos;
68    SizeT next_start = 0;
69    while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
70      next_start = pos + KV.second.size();
71      source.replace(pos, KV.first.size(), KV.second);
72    }
73  }
74  return source;
75}
76
77void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
78               TestCaseList const& not_checks) {
79  std::string first_line;
80  bool on_first = true;
81  std::string line;
82  while (remaining_output.eof() == false) {
83    CHECK(remaining_output.good());
84    std::getline(remaining_output, line);
85    if (on_first) {
86      first_line = line;
87      on_first = false;
88    }
89    for (const auto& NC : not_checks) {
90      CHECK(!NC.regex->Match(line))
91          << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
92          << NC.regex_str << "\""
93          << "\n    actual regex string \"" << TC.substituted_regex << "\""
94          << "\n    started matching near: " << first_line;
95    }
96    if (TC.regex->Match(line)) return;
97    CHECK(TC.match_rule != MR_Next)
98        << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
99        << "\""
100        << "\n    actual regex string \"" << TC.substituted_regex << "\""
101        << "\n    started matching near: " << first_line;
102  }
103  CHECK(remaining_output.eof() == false)
104      << "End of output reached before match for regex \"" << TC.regex_str
105      << "\" was found"
106      << "\n    actual regex string \"" << TC.substituted_regex << "\""
107      << "\n    started matching near: " << first_line;
108}
109
110void CheckCases(TestCaseList const& checks, std::stringstream& output) {
111  std::vector<TestCase> not_checks;
112  for (size_t i = 0; i < checks.size(); ++i) {
113    const auto& TC = checks[i];
114    if (TC.match_rule == MR_Not) {
115      not_checks.push_back(TC);
116      continue;
117    }
118    CheckCase(output, TC, not_checks);
119    not_checks.clear();
120  }
121}
122
123class TestReporter : public benchmark::BenchmarkReporter {
124 public:
125  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
126      : reporters_(reps) {}
127
128  virtual bool ReportContext(const Context& context) {
129    bool last_ret = false;
130    bool first = true;
131    for (auto rep : reporters_) {
132      bool new_ret = rep->ReportContext(context);
133      CHECK(first || new_ret == last_ret)
134          << "Reports return different values for ReportContext";
135      first = false;
136      last_ret = new_ret;
137    }
138    (void)first;
139    return last_ret;
140  }
141
142  void ReportRuns(const std::vector<Run>& report) {
143    for (auto rep : reporters_) rep->ReportRuns(report);
144  }
145  void Finalize() {
146    for (auto rep : reporters_) rep->Finalize();
147  }
148
149 private:
150  std::vector<benchmark::BenchmarkReporter *> reporters_;
151};
152}
153
154}  // end namespace internal
155
156// ========================================================================= //
157// -------------------------- Results checking ----------------------------- //
158// ========================================================================= //
159
160namespace internal {
161
162// Utility class to manage subscribers for checking benchmark results.
163// It works by parsing the CSV output to read the results.
164class ResultsChecker {
165 public:
166
167  struct PatternAndFn : public TestCase { // reusing TestCase for its regexes
168    PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
169    : TestCase(rx), fn(fn_) {}
170    ResultsCheckFn fn;
171  };
172
173  std::vector< PatternAndFn > check_patterns;
174  std::vector< Results > results;
175  std::vector< std::string > field_names;
176
177  void Add(const std::string& entry_pattern, ResultsCheckFn fn);
178
179  void CheckResults(std::stringstream& output);
180
181 private:
182
183  void SetHeader_(const std::string& csv_header);
184  void SetValues_(const std::string& entry_csv_line);
185
186  std::vector< std::string > SplitCsv_(const std::string& line);
187
188};
189
190// store the static ResultsChecker in a function to prevent initialization
191// order problems
192ResultsChecker& GetResultsChecker() {
193  static ResultsChecker rc;
194  return rc;
195}
196
197// add a results checker for a benchmark
198void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
199  check_patterns.emplace_back(entry_pattern, fn);
200}
201
202// check the results of all subscribed benchmarks
203void ResultsChecker::CheckResults(std::stringstream& output) {
204  // first reset the stream to the start
205  {
206    auto start = std::ios::streampos(0);
207    // clear before calling tellg()
208    output.clear();
209    // seek to zero only when needed
210    if(output.tellg() > start) output.seekg(start);
211    // and just in case
212    output.clear();
213  }
214  // now go over every line and publish it to the ResultsChecker
215  std::string line;
216  bool on_first = true;
217  while (output.eof() == false) {
218    CHECK(output.good());
219    std::getline(output, line);
220    if (on_first) {
221      SetHeader_(line); // this is important
222      on_first = false;
223      continue;
224    }
225    SetValues_(line);
226  }
227  // finally we can call the subscribed check functions
228  for(const auto& p : check_patterns) {
229    VLOG(2) << "--------------------------------\n";
230    VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
231    for(const auto& r : results) {
232      if(!p.regex->Match(r.name)) {
233        VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
234        continue;
235      } else {
236        VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
237      }
238      VLOG(1) << "Checking results of " << r.name << ": ... \n";
239      p.fn(r);
240      VLOG(1) << "Checking results of " << r.name << ": OK.\n";
241    }
242  }
243}
244
245// prepare for the names in this header
246void ResultsChecker::SetHeader_(const std::string& csv_header) {
247  field_names = SplitCsv_(csv_header);
248}
249
250// set the values for a benchmark
251void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
252  if(entry_csv_line.empty()) return; // some lines are empty
253  CHECK(!field_names.empty());
254  auto vals = SplitCsv_(entry_csv_line);
255  CHECK_EQ(vals.size(), field_names.size());
256  results.emplace_back(vals[0]); // vals[0] is the benchmark name
257  auto &entry = results.back();
258  for (size_t i = 1, e = vals.size(); i < e; ++i) {
259    entry.values[field_names[i]] = vals[i];
260  }
261}
262
263// a quick'n'dirty csv splitter (eliminating quotes)
264std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) {
265  std::vector< std::string > out;
266  if(line.empty()) return out;
267  if(!field_names.empty()) out.reserve(field_names.size());
268  size_t prev = 0, pos = line.find_first_of(','), curr = pos;
269  while(pos != line.npos) {
270    CHECK(curr > 0);
271    if(line[prev] == '"') ++prev;
272    if(line[curr-1] == '"') --curr;
273    out.push_back(line.substr(prev, curr-prev));
274    prev = pos + 1;
275    pos = line.find_first_of(',', pos + 1);
276    curr = pos;
277  }
278  curr = line.size();
279  if(line[prev] == '"') ++prev;
280  if(line[curr-1] == '"') --curr;
281  out.push_back(line.substr(prev, curr-prev));
282  return out;
283}
284
285}  // end namespace internal
286
287size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
288{
289  auto &rc = internal::GetResultsChecker();
290  rc.Add(bm_name, fn);
291  return rc.results.size();
292}
293
294int Results::NumThreads() const {
295  auto pos = name.find("/threads:");
296  if(pos == name.npos) return 1;
297  auto end = name.find('/', pos + 9);
298  std::stringstream ss;
299  ss << name.substr(pos + 9, end);
300  int num = 1;
301  ss >> num;
302  CHECK(!ss.fail());
303  return num;
304}
305
306double Results::GetTime(BenchmarkTime which) const {
307  CHECK(which == kCpuTime || which == kRealTime);
308  const char *which_str = which == kCpuTime ? "cpu_time" : "real_time";
309  double val = GetAs< double >(which_str);
310  auto unit = Get("time_unit");
311  CHECK(unit);
312  if(*unit == "ns") {
313    return val * 1.e-9;
314  } else if(*unit == "us") {
315    return val * 1.e-6;
316  } else if(*unit == "ms") {
317    return val * 1.e-3;
318  } else if(*unit == "s") {
319    return val;
320  } else {
321    CHECK(1 == 0) << "unknown time unit: " << *unit;
322    return 0;
323  }
324}
325
326// ========================================================================= //
327// -------------------------- Public API Definitions------------------------ //
328// ========================================================================= //
329
330TestCase::TestCase(std::string re, int rule)
331    : regex_str(std::move(re)),
332      match_rule(rule),
333      substituted_regex(internal::PerformSubstitutions(regex_str)),
334      regex(std::make_shared<benchmark::Regex>()) {
335  std::string err_str;
336  regex->Init(substituted_regex,& err_str);
337  CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
338                         << "\""
339                         << "\n    originally \"" << regex_str << "\""
340                         << "\n    got error: " << err_str;
341}
342
343int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
344  auto& L = internal::GetTestCaseList(ID);
345  L.insert(L.end(), il);
346  return 0;
347}
348
349int SetSubstitutions(
350    std::initializer_list<std::pair<std::string, std::string>> il) {
351  auto& subs = internal::GetSubstitutions();
352  for (auto KV : il) {
353    bool exists = false;
354    KV.second = internal::PerformSubstitutions(KV.second);
355    for (auto& EKV : subs) {
356      if (EKV.first == KV.first) {
357        EKV.second = std::move(KV.second);
358        exists = true;
359        break;
360      }
361    }
362    if (!exists) subs.push_back(std::move(KV));
363  }
364  return 0;
365}
366
367void RunOutputTests(int argc, char* argv[]) {
368  using internal::GetTestCaseList;
369  benchmark::Initialize(&argc, argv);
370  auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true);
371  benchmark::ConsoleReporter CR(options);
372  benchmark::JSONReporter JR;
373  benchmark::CSVReporter CSVR;
374  struct ReporterTest {
375    const char* name;
376    std::vector<TestCase>& output_cases;
377    std::vector<TestCase>& error_cases;
378    benchmark::BenchmarkReporter& reporter;
379    std::stringstream out_stream;
380    std::stringstream err_stream;
381
382    ReporterTest(const char* n, std::vector<TestCase>& out_tc,
383                 std::vector<TestCase>& err_tc,
384                 benchmark::BenchmarkReporter& br)
385        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
386      reporter.SetOutputStream(&out_stream);
387      reporter.SetErrorStream(&err_stream);
388    }
389  } TestCases[] = {
390      {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
391       GetTestCaseList(TC_ConsoleErr), CR},
392      {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
393       JR},
394      {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
395       CSVR},
396  };
397
398  // Create the test reporter and run the benchmarks.
399  std::cout << "Running benchmarks...\n";
400  internal::TestReporter test_rep({&CR, &JR, &CSVR});
401  benchmark::RunSpecifiedBenchmarks(&test_rep);
402
403  for (auto& rep_test : TestCases) {
404    std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
405    std::string banner(msg.size() - 1, '-');
406    std::cout << banner << msg << banner << "\n";
407
408    std::cerr << rep_test.err_stream.str();
409    std::cout << rep_test.out_stream.str();
410
411    internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
412    internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
413
414    std::cout << "\n";
415  }
416
417  // now that we know the output is as expected, we can dispatch
418  // the checks to subscribees.
419  auto &csv = TestCases[2];
420  // would use == but gcc spits a warning
421  CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
422  internal::GetResultsChecker().CheckResults(csv.out_stream);
423}
424