1
2#undef NDEBUG
3#include <cassert>
4#include <vector>
5
6#include "../src/check.h"  // NOTE: check.h is for internal use only!
7#include "benchmark/benchmark.h"
8
9namespace {
10
11class TestReporter : public benchmark::ConsoleReporter {
12 public:
13  virtual bool ReportContext(const Context& context) {
14    return ConsoleReporter::ReportContext(context);
15  };
16
17  virtual void ReportRuns(const std::vector<Run>& report) {
18    all_runs_.insert(all_runs_.end(), begin(report), end(report));
19    ConsoleReporter::ReportRuns(report);
20  }
21
22  TestReporter() {}
23  virtual ~TestReporter() {}
24
25  mutable std::vector<Run> all_runs_;
26};
27
28struct TestCase {
29  std::string name;
30  bool error_occurred;
31  std::string error_message;
32
33  typedef benchmark::BenchmarkReporter::Run Run;
34
35  void CheckRun(Run const& run) const {
36    CHECK(name == run.benchmark_name) << "expected " << name << " got "
37                                      << run.benchmark_name;
38    CHECK(error_occurred == run.error_occurred);
39    CHECK(error_message == run.error_message);
40    if (error_occurred) {
41      // CHECK(run.iterations == 0);
42    } else {
43      CHECK(run.iterations != 0);
44    }
45  }
46};
47
48std::vector<TestCase> ExpectedResults;
49
50int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
51  for (auto TC : v) {
52    TC.name = base_name + TC.name;
53    ExpectedResults.push_back(std::move(TC));
54  }
55  return 0;
56}
57
58#define CONCAT(x, y) CONCAT2(x, y)
59#define CONCAT2(x, y) x##y
60#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
61
62}  // end namespace
63
64void BM_error_before_running(benchmark::State& state) {
65  state.SkipWithError("error message");
66  while (state.KeepRunning()) {
67    assert(false);
68  }
69}
70BENCHMARK(BM_error_before_running);
71ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
72
73
74void BM_error_before_running_batch(benchmark::State& state) {
75  state.SkipWithError("error message");
76  while (state.KeepRunningBatch(17)) {
77    assert(false);
78  }
79}
80BENCHMARK(BM_error_before_running_batch);
81ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}});
82
83void BM_error_before_running_range_for(benchmark::State& state) {
84  state.SkipWithError("error message");
85  for (auto _ : state) {
86    assert(false);
87  }
88}
89BENCHMARK(BM_error_before_running_range_for);
90ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}});
91
92void BM_error_during_running(benchmark::State& state) {
93  int first_iter = true;
94  while (state.KeepRunning()) {
95    if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
96      assert(first_iter);
97      first_iter = false;
98      state.SkipWithError("error message");
99    } else {
100      state.PauseTiming();
101      state.ResumeTiming();
102    }
103  }
104}
105BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
106ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
107                                      {"/1/threads:2", true, "error message"},
108                                      {"/1/threads:4", true, "error message"},
109                                      {"/1/threads:8", true, "error message"},
110                                      {"/2/threads:1", false, ""},
111                                      {"/2/threads:2", false, ""},
112                                      {"/2/threads:4", false, ""},
113                                      {"/2/threads:8", false, ""}});
114
115void BM_error_during_running_ranged_for(benchmark::State& state) {
116  assert(state.max_iterations > 3 && "test requires at least a few iterations");
117  int first_iter = true;
118  // NOTE: Users should not write the for loop explicitly.
119  for (auto It = state.begin(), End = state.end(); It != End; ++It) {
120    if (state.range(0) == 1) {
121      assert(first_iter);
122      first_iter = false;
123      state.SkipWithError("error message");
124      // Test the unfortunate but documented behavior that the ranged-for loop
125      // doesn't automatically terminate when SkipWithError is set.
126      assert(++It != End);
127      break; // Required behavior
128    }
129  }
130}
131BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5);
132ADD_CASES("BM_error_during_running_ranged_for",
133          {{"/1/iterations:5", true, "error message"},
134           {"/2/iterations:5", false, ""}});
135
136
137
138void BM_error_after_running(benchmark::State& state) {
139  for (auto _ : state) {
140    benchmark::DoNotOptimize(state.iterations());
141  }
142  if (state.thread_index <= (state.threads / 2))
143    state.SkipWithError("error message");
144}
145BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
146ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
147                                     {"/threads:2", true, "error message"},
148                                     {"/threads:4", true, "error message"},
149                                     {"/threads:8", true, "error message"}});
150
151void BM_error_while_paused(benchmark::State& state) {
152  bool first_iter = true;
153  while (state.KeepRunning()) {
154    if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
155      assert(first_iter);
156      first_iter = false;
157      state.PauseTiming();
158      state.SkipWithError("error message");
159    } else {
160      state.PauseTiming();
161      state.ResumeTiming();
162    }
163  }
164}
165BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
166ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
167                                    {"/1/threads:2", true, "error message"},
168                                    {"/1/threads:4", true, "error message"},
169                                    {"/1/threads:8", true, "error message"},
170                                    {"/2/threads:1", false, ""},
171                                    {"/2/threads:2", false, ""},
172                                    {"/2/threads:4", false, ""},
173                                    {"/2/threads:8", false, ""}});
174
175int main(int argc, char* argv[]) {
176  benchmark::Initialize(&argc, argv);
177
178  TestReporter test_reporter;
179  benchmark::RunSpecifiedBenchmarks(&test_reporter);
180
181  typedef benchmark::BenchmarkReporter::Run Run;
182  auto EB = ExpectedResults.begin();
183
184  for (Run const& run : test_reporter.all_runs_) {
185    assert(EB != ExpectedResults.end());
186    EB->CheckRun(run);
187    ++EB;
188  }
189  assert(EB == ExpectedResults.end());
190
191  return 0;
192}
193