1
2#undef NDEBUG
3
4#include "benchmark/benchmark.h"
5#include "output_test.h"
6
7// ========================================================================= //
8// ---------------------- Testing Prologue Output -------------------------- //
9// ========================================================================= //
10
11// clang-format off
12
13ADD_CASES(TC_ConsoleOut,
14          {{"^[-]+$", MR_Next},
15           {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next},
16           {"^[-]+$", MR_Next}});
17ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}});
18
19// clang-format on
20
21// ========================================================================= //
22// ------------------------- Simple Counters Output ------------------------ //
23// ========================================================================= //
24
25void BM_Counters_Simple(benchmark::State& state) {
26  for (auto _ : state) {
27  }
28  state.counters["foo"] = 1;
29  state.counters["bar"] = 2 * (double)state.iterations();
30}
31BENCHMARK(BM_Counters_Simple);
32ADD_CASES(TC_ConsoleOut,
33          {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
34ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
35                       {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next},
36                       {"\"run_type\": \"iteration\",$", MR_Next},
37                       {"\"iterations\": %int,$", MR_Next},
38                       {"\"real_time\": %float,$", MR_Next},
39                       {"\"cpu_time\": %float,$", MR_Next},
40                       {"\"time_unit\": \"ns\",$", MR_Next},
41                       {"\"bar\": %float,$", MR_Next},
42                       {"\"foo\": %float$", MR_Next},
43                       {"}", MR_Next}});
44ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}});
45// VS2013 does not allow this function to be passed as a lambda argument
46// to CHECK_BENCHMARK_RESULTS()
47void CheckSimple(Results const& e) {
48  double its = e.NumIterations();
49  CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
50  // check that the value of bar is within 0.1% of the expected value
51  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001);
52}
53CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple);
54
55// ========================================================================= //
56// --------------------- Counters+Items+Bytes/s Output --------------------- //
57// ========================================================================= //
58
59namespace {
60int num_calls1 = 0;
61}
62void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
63  for (auto _ : state) {
64  }
65  state.counters["foo"] = 1;
66  state.counters["bar"] = ++num_calls1;
67  state.SetBytesProcessed(364);
68  state.SetItemsProcessed(150);
69}
70BENCHMARK(BM_Counters_WithBytesAndItemsPSec);
71ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
72                           "bar=%hrfloat bytes_per_second=%hrfloat/s "
73                           "foo=%hrfloat items_per_second=%hrfloat/s$"}});
74ADD_CASES(TC_JSONOut,
75          {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
76           {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next},
77           {"\"run_type\": \"iteration\",$", MR_Next},
78           {"\"iterations\": %int,$", MR_Next},
79           {"\"real_time\": %float,$", MR_Next},
80           {"\"cpu_time\": %float,$", MR_Next},
81           {"\"time_unit\": \"ns\",$", MR_Next},
82           {"\"bar\": %float,$", MR_Next},
83           {"\"bytes_per_second\": %float,$", MR_Next},
84           {"\"foo\": %float,$", MR_Next},
85           {"\"items_per_second\": %float$", MR_Next},
86           {"}", MR_Next}});
87ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
88                       "%csv_bytes_items_report,%float,%float$"}});
89// VS2013 does not allow this function to be passed as a lambda argument
90// to CHECK_BENCHMARK_RESULTS()
91void CheckBytesAndItemsPSec(Results const& e) {
92  double t = e.DurationCPUTime();  // this (and not real time) is the time used
93  CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
94  CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1);
95  // check that the values are within 0.1% of the expected values
96  CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364. / t, 0.001);
97  CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150. / t, 0.001);
98}
99CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec",
100                        &CheckBytesAndItemsPSec);
101
102// ========================================================================= //
103// ------------------------- Rate Counters Output -------------------------- //
104// ========================================================================= //
105
106void BM_Counters_Rate(benchmark::State& state) {
107  for (auto _ : state) {
108  }
109  namespace bm = benchmark;
110  state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate};
111  state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate};
112}
113BENCHMARK(BM_Counters_Rate);
114ADD_CASES(
115    TC_ConsoleOut,
116    {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
117ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
118                       {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next},
119                       {"\"run_type\": \"iteration\",$", MR_Next},
120                       {"\"iterations\": %int,$", MR_Next},
121                       {"\"real_time\": %float,$", MR_Next},
122                       {"\"cpu_time\": %float,$", MR_Next},
123                       {"\"time_unit\": \"ns\",$", MR_Next},
124                       {"\"bar\": %float,$", MR_Next},
125                       {"\"foo\": %float$", MR_Next},
126                       {"}", MR_Next}});
127ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}});
128// VS2013 does not allow this function to be passed as a lambda argument
129// to CHECK_BENCHMARK_RESULTS()
130void CheckRate(Results const& e) {
131  double t = e.DurationCPUTime();  // this (and not real time) is the time used
132  // check that the values are within 0.1% of the expected values
133  CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001);
134  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001);
135}
136CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate);
137
138// ========================================================================= //
139// ------------------------- Thread Counters Output ------------------------ //
140// ========================================================================= //
141
142void BM_Counters_Threads(benchmark::State& state) {
143  for (auto _ : state) {
144  }
145  state.counters["foo"] = 1;
146  state.counters["bar"] = 2;
147}
148BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
149ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report "
150                           "bar=%hrfloat foo=%hrfloat$"}});
151ADD_CASES(TC_JSONOut,
152          {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
153           {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next},
154           {"\"run_type\": \"iteration\",$", MR_Next},
155           {"\"iterations\": %int,$", MR_Next},
156           {"\"real_time\": %float,$", MR_Next},
157           {"\"cpu_time\": %float,$", MR_Next},
158           {"\"time_unit\": \"ns\",$", MR_Next},
159           {"\"bar\": %float,$", MR_Next},
160           {"\"foo\": %float$", MR_Next},
161           {"}", MR_Next}});
162ADD_CASES(
163    TC_CSVOut,
164    {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}});
165// VS2013 does not allow this function to be passed as a lambda argument
166// to CHECK_BENCHMARK_RESULTS()
167void CheckThreads(Results const& e) {
168  CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads());
169  CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads());
170}
171CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads);
172
173// ========================================================================= //
174// ---------------------- ThreadAvg Counters Output ------------------------ //
175// ========================================================================= //
176
177void BM_Counters_AvgThreads(benchmark::State& state) {
178  for (auto _ : state) {
179  }
180  namespace bm = benchmark;
181  state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads};
182  state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads};
183}
184BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
185ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int "
186                           "%console_report bar=%hrfloat foo=%hrfloat$"}});
187ADD_CASES(TC_JSONOut,
188          {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
189           {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next},
190           {"\"run_type\": \"iteration\",$", MR_Next},
191           {"\"iterations\": %int,$", MR_Next},
192           {"\"real_time\": %float,$", MR_Next},
193           {"\"cpu_time\": %float,$", MR_Next},
194           {"\"time_unit\": \"ns\",$", MR_Next},
195           {"\"bar\": %float,$", MR_Next},
196           {"\"foo\": %float$", MR_Next},
197           {"}", MR_Next}});
198ADD_CASES(
199    TC_CSVOut,
200    {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}});
201// VS2013 does not allow this function to be passed as a lambda argument
202// to CHECK_BENCHMARK_RESULTS()
203void CheckAvgThreads(Results const& e) {
204  CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1);
205  CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2);
206}
207CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int",
208                        &CheckAvgThreads);
209
210// ========================================================================= //
211// ---------------------- ThreadAvg Counters Output ------------------------ //
212// ========================================================================= //
213
214void BM_Counters_AvgThreadsRate(benchmark::State& state) {
215  for (auto _ : state) {
216  }
217  namespace bm = benchmark;
218  state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate};
219  state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate};
220}
221BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8);
222ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int "
223                           "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
224ADD_CASES(TC_JSONOut,
225          {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
226           {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$",
227            MR_Next},
228           {"\"run_type\": \"iteration\",$", MR_Next},
229           {"\"iterations\": %int,$", MR_Next},
230           {"\"real_time\": %float,$", MR_Next},
231           {"\"cpu_time\": %float,$", MR_Next},
232           {"\"time_unit\": \"ns\",$", MR_Next},
233           {"\"bar\": %float,$", MR_Next},
234           {"\"foo\": %float$", MR_Next},
235           {"}", MR_Next}});
236ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/"
237                       "threads:%int\",%csv_report,%float,%float$"}});
238// VS2013 does not allow this function to be passed as a lambda argument
239// to CHECK_BENCHMARK_RESULTS()
240void CheckAvgThreadsRate(Results const& e) {
241  CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / e.DurationCPUTime(), 0.001);
242  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / e.DurationCPUTime(), 0.001);
243}
244CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int",
245                        &CheckAvgThreadsRate);
246
247// ========================================================================= //
248// ------------------- IterationInvariant Counters Output ------------------ //
249// ========================================================================= //
250
251void BM_Counters_IterationInvariant(benchmark::State& state) {
252  for (auto _ : state) {
253  }
254  namespace bm = benchmark;
255  state.counters["foo"] = bm::Counter{1, bm::Counter::kIsIterationInvariant};
256  state.counters["bar"] = bm::Counter{2, bm::Counter::kIsIterationInvariant};
257}
258BENCHMARK(BM_Counters_IterationInvariant);
259ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report "
260                           "bar=%hrfloat foo=%hrfloat$"}});
261ADD_CASES(TC_JSONOut,
262          {{"\"name\": \"BM_Counters_IterationInvariant\",$"},
263           {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next},
264           {"\"run_type\": \"iteration\",$", MR_Next},
265           {"\"iterations\": %int,$", MR_Next},
266           {"\"real_time\": %float,$", MR_Next},
267           {"\"cpu_time\": %float,$", MR_Next},
268           {"\"time_unit\": \"ns\",$", MR_Next},
269           {"\"bar\": %float,$", MR_Next},
270           {"\"foo\": %float$", MR_Next},
271           {"}", MR_Next}});
272ADD_CASES(TC_CSVOut,
273          {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}});
274// VS2013 does not allow this function to be passed as a lambda argument
275// to CHECK_BENCHMARK_RESULTS()
276void CheckIterationInvariant(Results const& e) {
277  double its = e.NumIterations();
278  // check that the values are within 0.1% of the expected value
279  CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its, 0.001);
280  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001);
281}
282CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant",
283                        &CheckIterationInvariant);
284
285// ========================================================================= //
286// ----------------- IterationInvariantRate Counters Output ---------------- //
287// ========================================================================= //
288
289void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) {
290  for (auto _ : state) {
291  }
292  namespace bm = benchmark;
293  state.counters["foo"] =
294      bm::Counter{1, bm::Counter::kIsIterationInvariantRate};
295  state.counters["bar"] =
296      bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant};
297}
298BENCHMARK(BM_Counters_kIsIterationInvariantRate);
299ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate "
300                           "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
301ADD_CASES(TC_JSONOut,
302          {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"},
303           {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$",
304            MR_Next},
305           {"\"run_type\": \"iteration\",$", MR_Next},
306           {"\"iterations\": %int,$", MR_Next},
307           {"\"real_time\": %float,$", MR_Next},
308           {"\"cpu_time\": %float,$", MR_Next},
309           {"\"time_unit\": \"ns\",$", MR_Next},
310           {"\"bar\": %float,$", MR_Next},
311           {"\"foo\": %float$", MR_Next},
312           {"}", MR_Next}});
313ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kIsIterationInvariantRate\",%csv_report,"
314                       "%float,%float$"}});
315// VS2013 does not allow this function to be passed as a lambda argument
316// to CHECK_BENCHMARK_RESULTS()
317void CheckIsIterationInvariantRate(Results const& e) {
318  double its = e.NumIterations();
319  double t = e.DurationCPUTime();  // this (and not real time) is the time used
320  // check that the values are within 0.1% of the expected values
321  CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * 1. / t, 0.001);
322  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, its * 2. / t, 0.001);
323}
324CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate",
325                        &CheckIsIterationInvariantRate);
326
327// ========================================================================= //
328// ------------------- AvgIterations Counters Output ------------------ //
329// ========================================================================= //
330
331void BM_Counters_AvgIterations(benchmark::State& state) {
332  for (auto _ : state) {
333  }
334  namespace bm = benchmark;
335  state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterations};
336  state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgIterations};
337}
338BENCHMARK(BM_Counters_AvgIterations);
339ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report "
340                           "bar=%hrfloat foo=%hrfloat$"}});
341ADD_CASES(TC_JSONOut,
342          {{"\"name\": \"BM_Counters_AvgIterations\",$"},
343           {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next},
344           {"\"run_type\": \"iteration\",$", MR_Next},
345           {"\"iterations\": %int,$", MR_Next},
346           {"\"real_time\": %float,$", MR_Next},
347           {"\"cpu_time\": %float,$", MR_Next},
348           {"\"time_unit\": \"ns\",$", MR_Next},
349           {"\"bar\": %float,$", MR_Next},
350           {"\"foo\": %float$", MR_Next},
351           {"}", MR_Next}});
352ADD_CASES(TC_CSVOut,
353          {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}});
354// VS2013 does not allow this function to be passed as a lambda argument
355// to CHECK_BENCHMARK_RESULTS()
356void CheckAvgIterations(Results const& e) {
357  double its = e.NumIterations();
358  // check that the values are within 0.1% of the expected value
359  CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its, 0.001);
360  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its, 0.001);
361}
362CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations);
363
364// ========================================================================= //
365// ----------------- AvgIterationsRate Counters Output ---------------- //
366// ========================================================================= //
367
368void BM_Counters_kAvgIterationsRate(benchmark::State& state) {
369  for (auto _ : state) {
370  }
371  namespace bm = benchmark;
372  state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate};
373  state.counters["bar"] =
374      bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations};
375}
376BENCHMARK(BM_Counters_kAvgIterationsRate);
377ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate "
378                           "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
379ADD_CASES(TC_JSONOut,
380          {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
381           {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next},
382           {"\"run_type\": \"iteration\",$", MR_Next},
383           {"\"iterations\": %int,$", MR_Next},
384           {"\"real_time\": %float,$", MR_Next},
385           {"\"cpu_time\": %float,$", MR_Next},
386           {"\"time_unit\": \"ns\",$", MR_Next},
387           {"\"bar\": %float,$", MR_Next},
388           {"\"foo\": %float$", MR_Next},
389           {"}", MR_Next}});
390ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report,"
391                       "%float,%float$"}});
392// VS2013 does not allow this function to be passed as a lambda argument
393// to CHECK_BENCHMARK_RESULTS()
394void CheckAvgIterationsRate(Results const& e) {
395  double its = e.NumIterations();
396  double t = e.DurationCPUTime();  // this (and not real time) is the time used
397  // check that the values are within 0.1% of the expected values
398  CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its / t, 0.001);
399  CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its / t, 0.001);
400}
401CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate",
402                        &CheckAvgIterationsRate);
403
404// ========================================================================= //
405// --------------------------- TEST CASES END ------------------------------ //
406// ========================================================================= //
407
408int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
409