1 2#undef NDEBUG 3 4#include "benchmark/benchmark.h" 5#include "output_test.h" 6 7// ========================================================================= // 8// ---------------------- Testing Prologue Output -------------------------- // 9// ========================================================================= // 10 11// clang-format off 12 13ADD_CASES(TC_ConsoleOut, 14 {{"^[-]+$", MR_Next}, 15 {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next}, 16 {"^[-]+$", MR_Next}}); 17ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}}); 18 19// clang-format on 20 21// ========================================================================= // 22// ------------------------- Simple Counters Output ------------------------ // 23// ========================================================================= // 24 25void BM_Counters_Simple(benchmark::State& state) { 26 for (auto _ : state) { 27 } 28 state.counters["foo"] = 1; 29 state.counters["bar"] = 2 * (double)state.iterations(); 30} 31BENCHMARK(BM_Counters_Simple); 32ADD_CASES(TC_ConsoleOut, 33 {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); 34ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, 35 {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, 36 {"\"run_type\": \"iteration\",$", MR_Next}, 37 {"\"iterations\": %int,$", MR_Next}, 38 {"\"real_time\": %float,$", MR_Next}, 39 {"\"cpu_time\": %float,$", MR_Next}, 40 {"\"time_unit\": \"ns\",$", MR_Next}, 41 {"\"bar\": %float,$", MR_Next}, 42 {"\"foo\": %float$", MR_Next}, 43 {"}", MR_Next}}); 44ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); 45// VS2013 does not allow this function to be passed as a lambda argument 46// to CHECK_BENCHMARK_RESULTS() 47void CheckSimple(Results const& e) { 48 double its = e.NumIterations(); 49 CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); 50 // check that the value of bar is within 0.1% of the expected value 51 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); 52} 53CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); 54 55// ========================================================================= // 56// --------------------- Counters+Items+Bytes/s Output --------------------- // 57// ========================================================================= // 58 59namespace { 60int num_calls1 = 0; 61} 62void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { 63 for (auto _ : state) { 64 } 65 state.counters["foo"] = 1; 66 state.counters["bar"] = ++num_calls1; 67 state.SetBytesProcessed(364); 68 state.SetItemsProcessed(150); 69} 70BENCHMARK(BM_Counters_WithBytesAndItemsPSec); 71ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " 72 "bar=%hrfloat bytes_per_second=%hrfloat/s " 73 "foo=%hrfloat items_per_second=%hrfloat/s$"}}); 74ADD_CASES(TC_JSONOut, 75 {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, 76 {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, 77 {"\"run_type\": \"iteration\",$", MR_Next}, 78 {"\"iterations\": %int,$", MR_Next}, 79 {"\"real_time\": %float,$", MR_Next}, 80 {"\"cpu_time\": %float,$", MR_Next}, 81 {"\"time_unit\": \"ns\",$", MR_Next}, 82 {"\"bar\": %float,$", MR_Next}, 83 {"\"bytes_per_second\": %float,$", MR_Next}, 84 {"\"foo\": %float,$", MR_Next}, 85 {"\"items_per_second\": %float$", MR_Next}, 86 {"}", MR_Next}}); 87ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," 88 "%csv_bytes_items_report,%float,%float$"}}); 89// VS2013 does not allow this function to be passed as a lambda argument 90// to CHECK_BENCHMARK_RESULTS() 91void CheckBytesAndItemsPSec(Results const& e) { 92 double t = e.DurationCPUTime(); // this (and not real time) is the time used 93 CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); 94 CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1); 95 // check that the values are within 0.1% of the expected values 96 CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364. / t, 0.001); 97 CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150. / t, 0.001); 98} 99CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", 100 &CheckBytesAndItemsPSec); 101 102// ========================================================================= // 103// ------------------------- Rate Counters Output -------------------------- // 104// ========================================================================= // 105 106void BM_Counters_Rate(benchmark::State& state) { 107 for (auto _ : state) { 108 } 109 namespace bm = benchmark; 110 state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; 111 state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; 112} 113BENCHMARK(BM_Counters_Rate); 114ADD_CASES( 115 TC_ConsoleOut, 116 {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 117ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, 118 {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, 119 {"\"run_type\": \"iteration\",$", MR_Next}, 120 {"\"iterations\": %int,$", MR_Next}, 121 {"\"real_time\": %float,$", MR_Next}, 122 {"\"cpu_time\": %float,$", MR_Next}, 123 {"\"time_unit\": \"ns\",$", MR_Next}, 124 {"\"bar\": %float,$", MR_Next}, 125 {"\"foo\": %float$", MR_Next}, 126 {"}", MR_Next}}); 127ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}}); 128// VS2013 does not allow this function to be passed as a lambda argument 129// to CHECK_BENCHMARK_RESULTS() 130void CheckRate(Results const& e) { 131 double t = e.DurationCPUTime(); // this (and not real time) is the time used 132 // check that the values are within 0.1% of the expected values 133 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001); 134 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001); 135} 136CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); 137 138// ========================================================================= // 139// ------------------------- Thread Counters Output ------------------------ // 140// ========================================================================= // 141 142void BM_Counters_Threads(benchmark::State& state) { 143 for (auto _ : state) { 144 } 145 state.counters["foo"] = 1; 146 state.counters["bar"] = 2; 147} 148BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); 149ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " 150 "bar=%hrfloat foo=%hrfloat$"}}); 151ADD_CASES(TC_JSONOut, 152 {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, 153 {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, 154 {"\"run_type\": \"iteration\",$", MR_Next}, 155 {"\"iterations\": %int,$", MR_Next}, 156 {"\"real_time\": %float,$", MR_Next}, 157 {"\"cpu_time\": %float,$", MR_Next}, 158 {"\"time_unit\": \"ns\",$", MR_Next}, 159 {"\"bar\": %float,$", MR_Next}, 160 {"\"foo\": %float$", MR_Next}, 161 {"}", MR_Next}}); 162ADD_CASES( 163 TC_CSVOut, 164 {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); 165// VS2013 does not allow this function to be passed as a lambda argument 166// to CHECK_BENCHMARK_RESULTS() 167void CheckThreads(Results const& e) { 168 CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads()); 169 CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads()); 170} 171CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads); 172 173// ========================================================================= // 174// ---------------------- ThreadAvg Counters Output ------------------------ // 175// ========================================================================= // 176 177void BM_Counters_AvgThreads(benchmark::State& state) { 178 for (auto _ : state) { 179 } 180 namespace bm = benchmark; 181 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads}; 182 state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; 183} 184BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); 185ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " 186 "%console_report bar=%hrfloat foo=%hrfloat$"}}); 187ADD_CASES(TC_JSONOut, 188 {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, 189 {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, 190 {"\"run_type\": \"iteration\",$", MR_Next}, 191 {"\"iterations\": %int,$", MR_Next}, 192 {"\"real_time\": %float,$", MR_Next}, 193 {"\"cpu_time\": %float,$", MR_Next}, 194 {"\"time_unit\": \"ns\",$", MR_Next}, 195 {"\"bar\": %float,$", MR_Next}, 196 {"\"foo\": %float$", MR_Next}, 197 {"}", MR_Next}}); 198ADD_CASES( 199 TC_CSVOut, 200 {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); 201// VS2013 does not allow this function to be passed as a lambda argument 202// to CHECK_BENCHMARK_RESULTS() 203void CheckAvgThreads(Results const& e) { 204 CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); 205 CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2); 206} 207CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", 208 &CheckAvgThreads); 209 210// ========================================================================= // 211// ---------------------- ThreadAvg Counters Output ------------------------ // 212// ========================================================================= // 213 214void BM_Counters_AvgThreadsRate(benchmark::State& state) { 215 for (auto _ : state) { 216 } 217 namespace bm = benchmark; 218 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; 219 state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; 220} 221BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); 222ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " 223 "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 224ADD_CASES(TC_JSONOut, 225 {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, 226 {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", 227 MR_Next}, 228 {"\"run_type\": \"iteration\",$", MR_Next}, 229 {"\"iterations\": %int,$", MR_Next}, 230 {"\"real_time\": %float,$", MR_Next}, 231 {"\"cpu_time\": %float,$", MR_Next}, 232 {"\"time_unit\": \"ns\",$", MR_Next}, 233 {"\"bar\": %float,$", MR_Next}, 234 {"\"foo\": %float$", MR_Next}, 235 {"}", MR_Next}}); 236ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/" 237 "threads:%int\",%csv_report,%float,%float$"}}); 238// VS2013 does not allow this function to be passed as a lambda argument 239// to CHECK_BENCHMARK_RESULTS() 240void CheckAvgThreadsRate(Results const& e) { 241 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / e.DurationCPUTime(), 0.001); 242 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / e.DurationCPUTime(), 0.001); 243} 244CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", 245 &CheckAvgThreadsRate); 246 247// ========================================================================= // 248// ------------------- IterationInvariant Counters Output ------------------ // 249// ========================================================================= // 250 251void BM_Counters_IterationInvariant(benchmark::State& state) { 252 for (auto _ : state) { 253 } 254 namespace bm = benchmark; 255 state.counters["foo"] = bm::Counter{1, bm::Counter::kIsIterationInvariant}; 256 state.counters["bar"] = bm::Counter{2, bm::Counter::kIsIterationInvariant}; 257} 258BENCHMARK(BM_Counters_IterationInvariant); 259ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " 260 "bar=%hrfloat foo=%hrfloat$"}}); 261ADD_CASES(TC_JSONOut, 262 {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, 263 {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, 264 {"\"run_type\": \"iteration\",$", MR_Next}, 265 {"\"iterations\": %int,$", MR_Next}, 266 {"\"real_time\": %float,$", MR_Next}, 267 {"\"cpu_time\": %float,$", MR_Next}, 268 {"\"time_unit\": \"ns\",$", MR_Next}, 269 {"\"bar\": %float,$", MR_Next}, 270 {"\"foo\": %float$", MR_Next}, 271 {"}", MR_Next}}); 272ADD_CASES(TC_CSVOut, 273 {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}}); 274// VS2013 does not allow this function to be passed as a lambda argument 275// to CHECK_BENCHMARK_RESULTS() 276void CheckIterationInvariant(Results const& e) { 277 double its = e.NumIterations(); 278 // check that the values are within 0.1% of the expected value 279 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its, 0.001); 280 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); 281} 282CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", 283 &CheckIterationInvariant); 284 285// ========================================================================= // 286// ----------------- IterationInvariantRate Counters Output ---------------- // 287// ========================================================================= // 288 289void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { 290 for (auto _ : state) { 291 } 292 namespace bm = benchmark; 293 state.counters["foo"] = 294 bm::Counter{1, bm::Counter::kIsIterationInvariantRate}; 295 state.counters["bar"] = 296 bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant}; 297} 298BENCHMARK(BM_Counters_kIsIterationInvariantRate); 299ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " 300 "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 301ADD_CASES(TC_JSONOut, 302 {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, 303 {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", 304 MR_Next}, 305 {"\"run_type\": \"iteration\",$", MR_Next}, 306 {"\"iterations\": %int,$", MR_Next}, 307 {"\"real_time\": %float,$", MR_Next}, 308 {"\"cpu_time\": %float,$", MR_Next}, 309 {"\"time_unit\": \"ns\",$", MR_Next}, 310 {"\"bar\": %float,$", MR_Next}, 311 {"\"foo\": %float$", MR_Next}, 312 {"}", MR_Next}}); 313ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kIsIterationInvariantRate\",%csv_report," 314 "%float,%float$"}}); 315// VS2013 does not allow this function to be passed as a lambda argument 316// to CHECK_BENCHMARK_RESULTS() 317void CheckIsIterationInvariantRate(Results const& e) { 318 double its = e.NumIterations(); 319 double t = e.DurationCPUTime(); // this (and not real time) is the time used 320 // check that the values are within 0.1% of the expected values 321 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * 1. / t, 0.001); 322 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, its * 2. / t, 0.001); 323} 324CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", 325 &CheckIsIterationInvariantRate); 326 327// ========================================================================= // 328// ------------------- AvgIterations Counters Output ------------------ // 329// ========================================================================= // 330 331void BM_Counters_AvgIterations(benchmark::State& state) { 332 for (auto _ : state) { 333 } 334 namespace bm = benchmark; 335 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterations}; 336 state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgIterations}; 337} 338BENCHMARK(BM_Counters_AvgIterations); 339ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " 340 "bar=%hrfloat foo=%hrfloat$"}}); 341ADD_CASES(TC_JSONOut, 342 {{"\"name\": \"BM_Counters_AvgIterations\",$"}, 343 {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, 344 {"\"run_type\": \"iteration\",$", MR_Next}, 345 {"\"iterations\": %int,$", MR_Next}, 346 {"\"real_time\": %float,$", MR_Next}, 347 {"\"cpu_time\": %float,$", MR_Next}, 348 {"\"time_unit\": \"ns\",$", MR_Next}, 349 {"\"bar\": %float,$", MR_Next}, 350 {"\"foo\": %float$", MR_Next}, 351 {"}", MR_Next}}); 352ADD_CASES(TC_CSVOut, 353 {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}}); 354// VS2013 does not allow this function to be passed as a lambda argument 355// to CHECK_BENCHMARK_RESULTS() 356void CheckAvgIterations(Results const& e) { 357 double its = e.NumIterations(); 358 // check that the values are within 0.1% of the expected value 359 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its, 0.001); 360 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its, 0.001); 361} 362CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); 363 364// ========================================================================= // 365// ----------------- AvgIterationsRate Counters Output ---------------- // 366// ========================================================================= // 367 368void BM_Counters_kAvgIterationsRate(benchmark::State& state) { 369 for (auto _ : state) { 370 } 371 namespace bm = benchmark; 372 state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; 373 state.counters["bar"] = 374 bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations}; 375} 376BENCHMARK(BM_Counters_kAvgIterationsRate); 377ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " 378 "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); 379ADD_CASES(TC_JSONOut, 380 {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, 381 {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, 382 {"\"run_type\": \"iteration\",$", MR_Next}, 383 {"\"iterations\": %int,$", MR_Next}, 384 {"\"real_time\": %float,$", MR_Next}, 385 {"\"cpu_time\": %float,$", MR_Next}, 386 {"\"time_unit\": \"ns\",$", MR_Next}, 387 {"\"bar\": %float,$", MR_Next}, 388 {"\"foo\": %float$", MR_Next}, 389 {"}", MR_Next}}); 390ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report," 391 "%float,%float$"}}); 392// VS2013 does not allow this function to be passed as a lambda argument 393// to CHECK_BENCHMARK_RESULTS() 394void CheckAvgIterationsRate(Results const& e) { 395 double its = e.NumIterations(); 396 double t = e.DurationCPUTime(); // this (and not real time) is the time used 397 // check that the values are within 0.1% of the expected values 398 CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its / t, 0.001); 399 CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its / t, 0.001); 400} 401CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate", 402 &CheckAvgIterationsRate); 403 404// ========================================================================= // 405// --------------------------- TEST CASES END ------------------------------ // 406// ========================================================================= // 407 408int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 409