1 2#undef NDEBUG 3#include <utility> 4 5#include "benchmark/benchmark.h" 6#include "output_test.h" 7 8// ========================================================================= // 9// ---------------------- Testing Prologue Output -------------------------- // 10// ========================================================================= // 11 12ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next}, 13 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, 14 {"^[-]+$", MR_Next}}); 15static int AddContextCases() { 16 AddCases(TC_ConsoleErr, 17 { 18 {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default}, 19 {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, 20 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, 21 }); 22 AddCases(TC_JSONOut, 23 {{"^\\{", MR_Default}, 24 {"\"context\":", MR_Next}, 25 {"\"date\": \"", MR_Next}, 26 {"\"host_name\":", MR_Next}, 27 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",", 28 MR_Next}, 29 {"\"num_cpus\": %int,$", MR_Next}, 30 {"\"mhz_per_cpu\": %float,$", MR_Next}, 31 {"\"cpu_scaling_enabled\": ", MR_Next}, 32 {"\"caches\": \\[$", MR_Next}}); 33 auto const& Info = benchmark::CPUInfo::Get(); 34 auto const& Caches = Info.caches; 35 if (!Caches.empty()) { 36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}}); 37 } 38 for (size_t I = 0; I < Caches.size(); ++I) { 39 std::string num_caches_str = 40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$"; 41 AddCases( 42 TC_ConsoleErr, 43 {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}}); 44 AddCases(TC_JSONOut, {{"\\{$", MR_Next}, 45 {"\"type\": \"", MR_Next}, 46 {"\"level\": %int,$", MR_Next}, 47 {"\"size\": %int,$", MR_Next}, 48 {"\"num_sharing\": %int$", MR_Next}, 49 {"}[,]{0,1}$", MR_Next}}); 50 } 51 AddCases(TC_JSONOut, {{"],$"}}); 52 auto const& LoadAvg = Info.load_avg; 53 if (!LoadAvg.empty()) { 54 AddCases(TC_ConsoleErr, 55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); 56 } 57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); 58 return 0; 59} 60int dummy_register = AddContextCases(); 61ADD_CASES(TC_CSVOut, {{"%csv_header"}}); 62 63// ========================================================================= // 64// ------------------------ Testing Basic Output --------------------------- // 65// ========================================================================= // 66 67void BM_basic(benchmark::State& state) { 68 for (auto _ : state) { 69 } 70} 71BENCHMARK(BM_basic); 72 73ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); 74ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, 75 {"\"run_name\": \"BM_basic\",$", MR_Next}, 76 {"\"run_type\": \"iteration\",$", MR_Next}, 77 {"\"iterations\": %int,$", MR_Next}, 78 {"\"real_time\": %float,$", MR_Next}, 79 {"\"cpu_time\": %float,$", MR_Next}, 80 {"\"time_unit\": \"ns\"$", MR_Next}, 81 {"}", MR_Next}}); 82ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); 83 84// ========================================================================= // 85// ------------------------ Testing Bytes per Second Output ---------------- // 86// ========================================================================= // 87 88void BM_bytes_per_second(benchmark::State& state) { 89 for (auto _ : state) { 90 } 91 state.SetBytesProcessed(1); 92} 93BENCHMARK(BM_bytes_per_second); 94 95ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " 96 "bytes_per_second=%float[kM]{0,1}/s$"}}); 97ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, 98 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, 99 {"\"run_type\": \"iteration\",$", MR_Next}, 100 {"\"iterations\": %int,$", MR_Next}, 101 {"\"real_time\": %float,$", MR_Next}, 102 {"\"cpu_time\": %float,$", MR_Next}, 103 {"\"time_unit\": \"ns\",$", MR_Next}, 104 {"\"bytes_per_second\": %float$", MR_Next}, 105 {"}", MR_Next}}); 106ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); 107 108// ========================================================================= // 109// ------------------------ Testing Items per Second Output ---------------- // 110// ========================================================================= // 111 112void BM_items_per_second(benchmark::State& state) { 113 for (auto _ : state) { 114 } 115 state.SetItemsProcessed(1); 116} 117BENCHMARK(BM_items_per_second); 118 119ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " 120 "items_per_second=%float[kM]{0,1}/s$"}}); 121ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, 122 {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, 123 {"\"run_type\": \"iteration\",$", MR_Next}, 124 {"\"iterations\": %int,$", MR_Next}, 125 {"\"real_time\": %float,$", MR_Next}, 126 {"\"cpu_time\": %float,$", MR_Next}, 127 {"\"time_unit\": \"ns\",$", MR_Next}, 128 {"\"items_per_second\": %float$", MR_Next}, 129 {"}", MR_Next}}); 130ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}}); 131 132// ========================================================================= // 133// ------------------------ Testing Label Output --------------------------- // 134// ========================================================================= // 135 136void BM_label(benchmark::State& state) { 137 for (auto _ : state) { 138 } 139 state.SetLabel("some label"); 140} 141BENCHMARK(BM_label); 142 143ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); 144ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, 145 {"\"run_name\": \"BM_label\",$", MR_Next}, 146 {"\"run_type\": \"iteration\",$", MR_Next}, 147 {"\"iterations\": %int,$", MR_Next}, 148 {"\"real_time\": %float,$", MR_Next}, 149 {"\"cpu_time\": %float,$", MR_Next}, 150 {"\"time_unit\": \"ns\",$", MR_Next}, 151 {"\"label\": \"some label\"$", MR_Next}, 152 {"}", MR_Next}}); 153ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " 154 "label\"%csv_label_report_end$"}}); 155 156// ========================================================================= // 157// ------------------------ Testing Error Output --------------------------- // 158// ========================================================================= // 159 160void BM_error(benchmark::State& state) { 161 state.SkipWithError("message"); 162 for (auto _ : state) { 163 } 164} 165BENCHMARK(BM_error); 166ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); 167ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, 168 {"\"run_name\": \"BM_error\",$", MR_Next}, 169 {"\"run_type\": \"iteration\",$", MR_Next}, 170 {"\"error_occurred\": true,$", MR_Next}, 171 {"\"error_message\": \"message\",$", MR_Next}}); 172 173ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}}); 174 175// ========================================================================= // 176// ------------------------ Testing No Arg Name Output ----------------------- 177// // 178// ========================================================================= // 179 180void BM_no_arg_name(benchmark::State& state) { 181 for (auto _ : state) { 182 } 183} 184BENCHMARK(BM_no_arg_name)->Arg(3); 185ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); 186ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, 187 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, 188 {"\"run_type\": \"iteration\",$", MR_Next}}); 189ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); 190 191// ========================================================================= // 192// ------------------------ Testing Arg Name Output ----------------------- // 193// ========================================================================= // 194 195void BM_arg_name(benchmark::State& state) { 196 for (auto _ : state) { 197 } 198} 199BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); 200ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); 201ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, 202 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, 203 {"\"run_type\": \"iteration\",$", MR_Next}}); 204ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); 205 206// ========================================================================= // 207// ------------------------ Testing Arg Names Output ----------------------- // 208// ========================================================================= // 209 210void BM_arg_names(benchmark::State& state) { 211 for (auto _ : state) { 212 } 213} 214BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); 215ADD_CASES(TC_ConsoleOut, 216 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); 217ADD_CASES(TC_JSONOut, 218 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, 219 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, 220 {"\"run_type\": \"iteration\",$", MR_Next}}); 221ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); 222 223// ========================================================================= // 224// ------------------------ Testing Big Args Output ------------------------ // 225// ========================================================================= // 226 227void BM_BigArgs(benchmark::State& state) { 228 for (auto _ : state) { 229 } 230} 231BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U); 232ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, 233 {"^BM_BigArgs/2147483648 %console_report$"}}); 234 235// ========================================================================= // 236// ----------------------- Testing Complexity Output ----------------------- // 237// ========================================================================= // 238 239void BM_Complexity_O1(benchmark::State& state) { 240 for (auto _ : state) { 241 } 242 state.SetComplexityN(state.range(0)); 243} 244BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); 245SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"}, 246 {"%RMS", "[ ]*[0-9]+ %"}}); 247ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"}, 248 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}}); 249 250// ========================================================================= // 251// ----------------------- Testing Aggregate Output ------------------------ // 252// ========================================================================= // 253 254// Test that non-aggregate data is printed by default 255void BM_Repeat(benchmark::State& state) { 256 for (auto _ : state) { 257 } 258} 259// need two repetitions min to be able to output any aggregate output 260BENCHMARK(BM_Repeat)->Repetitions(2); 261ADD_CASES(TC_ConsoleOut, 262 {{"^BM_Repeat/repeats:2 %console_report$"}, 263 {"^BM_Repeat/repeats:2 %console_report$"}, 264 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"}, 265 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"}, 266 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); 267ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, 268 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, 269 {"\"run_type\": \"iteration\",$", MR_Next}, 270 {"\"name\": \"BM_Repeat/repeats:2\",$"}, 271 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 272 {"\"run_type\": \"iteration\",$", MR_Next}, 273 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, 274 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 275 {"\"run_type\": \"aggregate\",$", MR_Next}, 276 {"\"aggregate_name\": \"mean\",$", MR_Next}, 277 {"\"iterations\": 2,$", MR_Next}, 278 {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, 279 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 280 {"\"run_type\": \"aggregate\",$", MR_Next}, 281 {"\"aggregate_name\": \"median\",$", MR_Next}, 282 {"\"iterations\": 2,$", MR_Next}, 283 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, 284 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 285 {"\"run_type\": \"aggregate\",$", MR_Next}, 286 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 287 {"\"iterations\": 2,$", MR_Next}}); 288ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, 289 {"^\"BM_Repeat/repeats:2\",%csv_report$"}, 290 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, 291 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"}, 292 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}}); 293// but for two repetitions, mean and median is the same, so let's repeat.. 294BENCHMARK(BM_Repeat)->Repetitions(3); 295ADD_CASES(TC_ConsoleOut, 296 {{"^BM_Repeat/repeats:3 %console_report$"}, 297 {"^BM_Repeat/repeats:3 %console_report$"}, 298 {"^BM_Repeat/repeats:3 %console_report$"}, 299 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"}, 300 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"}, 301 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); 302ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, 303 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 304 {"\"run_type\": \"iteration\",$", MR_Next}, 305 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 306 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 307 {"\"run_type\": \"iteration\",$", MR_Next}, 308 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 309 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 310 {"\"run_type\": \"iteration\",$", MR_Next}, 311 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, 312 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 313 {"\"run_type\": \"aggregate\",$", MR_Next}, 314 {"\"aggregate_name\": \"mean\",$", MR_Next}, 315 {"\"iterations\": 3,$", MR_Next}, 316 {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, 317 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 318 {"\"run_type\": \"aggregate\",$", MR_Next}, 319 {"\"aggregate_name\": \"median\",$", MR_Next}, 320 {"\"iterations\": 3,$", MR_Next}, 321 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, 322 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 323 {"\"run_type\": \"aggregate\",$", MR_Next}, 324 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 325 {"\"iterations\": 3,$", MR_Next}}); 326ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, 327 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 328 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 329 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, 330 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"}, 331 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); 332// median differs between even/odd number of repetitions, so just to be sure 333BENCHMARK(BM_Repeat)->Repetitions(4); 334ADD_CASES(TC_ConsoleOut, 335 {{"^BM_Repeat/repeats:4 %console_report$"}, 336 {"^BM_Repeat/repeats:4 %console_report$"}, 337 {"^BM_Repeat/repeats:4 %console_report$"}, 338 {"^BM_Repeat/repeats:4 %console_report$"}, 339 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"}, 340 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"}, 341 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); 342ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, 343 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 344 {"\"run_type\": \"iteration\",$", MR_Next}, 345 {"\"name\": \"BM_Repeat/repeats:4\",$"}, 346 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 347 {"\"run_type\": \"iteration\",$", MR_Next}, 348 {"\"name\": \"BM_Repeat/repeats:4\",$"}, 349 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 350 {"\"run_type\": \"iteration\",$", MR_Next}, 351 {"\"name\": \"BM_Repeat/repeats:4\",$"}, 352 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 353 {"\"run_type\": \"iteration\",$", MR_Next}, 354 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, 355 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 356 {"\"run_type\": \"aggregate\",$", MR_Next}, 357 {"\"aggregate_name\": \"mean\",$", MR_Next}, 358 {"\"iterations\": 4,$", MR_Next}, 359 {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, 360 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 361 {"\"run_type\": \"aggregate\",$", MR_Next}, 362 {"\"aggregate_name\": \"median\",$", MR_Next}, 363 {"\"iterations\": 4,$", MR_Next}, 364 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, 365 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 366 {"\"run_type\": \"aggregate\",$", MR_Next}, 367 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 368 {"\"iterations\": 4,$", MR_Next}}); 369ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, 370 {"^\"BM_Repeat/repeats:4\",%csv_report$"}, 371 {"^\"BM_Repeat/repeats:4\",%csv_report$"}, 372 {"^\"BM_Repeat/repeats:4\",%csv_report$"}, 373 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"}, 374 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"}, 375 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}}); 376 377// Test that a non-repeated test still prints non-aggregate results even when 378// only-aggregate reports have been requested 379void BM_RepeatOnce(benchmark::State& state) { 380 for (auto _ : state) { 381 } 382} 383BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); 384ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); 385ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, 386 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, 387 {"\"run_type\": \"iteration\",$", MR_Next}}); 388ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); 389 390// Test that non-aggregate data is not reported 391void BM_SummaryRepeat(benchmark::State& state) { 392 for (auto _ : state) { 393 } 394} 395BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); 396ADD_CASES( 397 TC_ConsoleOut, 398 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 399 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"}, 400 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"}, 401 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); 402ADD_CASES(TC_JSONOut, 403 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 404 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, 405 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, 406 {"\"run_type\": \"aggregate\",$", MR_Next}, 407 {"\"aggregate_name\": \"mean\",$", MR_Next}, 408 {"\"iterations\": 3,$", MR_Next}, 409 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, 410 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, 411 {"\"run_type\": \"aggregate\",$", MR_Next}, 412 {"\"aggregate_name\": \"median\",$", MR_Next}, 413 {"\"iterations\": 3,$", MR_Next}, 414 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, 415 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, 416 {"\"run_type\": \"aggregate\",$", MR_Next}, 417 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 418 {"\"iterations\": 3,$", MR_Next}}); 419ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 420 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, 421 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, 422 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); 423 424// Test that non-aggregate data is not displayed. 425// NOTE: this test is kinda bad. we are only testing the display output. 426// But we don't check that the file output still contains everything... 427void BM_SummaryDisplay(benchmark::State& state) { 428 for (auto _ : state) { 429 } 430} 431BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly(); 432ADD_CASES( 433 TC_ConsoleOut, 434 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, 435 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"}, 436 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"}, 437 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}}); 438ADD_CASES(TC_JSONOut, 439 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, 440 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, 441 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, 442 {"\"run_type\": \"aggregate\",$", MR_Next}, 443 {"\"aggregate_name\": \"mean\",$", MR_Next}, 444 {"\"iterations\": 2,$", MR_Next}, 445 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, 446 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, 447 {"\"run_type\": \"aggregate\",$", MR_Next}, 448 {"\"aggregate_name\": \"median\",$", MR_Next}, 449 {"\"iterations\": 2,$", MR_Next}, 450 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, 451 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, 452 {"\"run_type\": \"aggregate\",$", MR_Next}, 453 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 454 {"\"iterations\": 2,$", MR_Next}}); 455ADD_CASES(TC_CSVOut, 456 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, 457 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, 458 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"}, 459 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}}); 460 461// Test repeats with custom time unit. 462void BM_RepeatTimeUnit(benchmark::State& state) { 463 for (auto _ : state) { 464 } 465} 466BENCHMARK(BM_RepeatTimeUnit) 467 ->Repetitions(3) 468 ->ReportAggregatesOnly() 469 ->Unit(benchmark::kMicrosecond); 470ADD_CASES( 471 TC_ConsoleOut, 472 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 473 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"}, 474 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ " 475 "]*3$"}, 476 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ " 477 "]*3$"}}); 478ADD_CASES(TC_JSONOut, 479 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 480 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, 481 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, 482 {"\"run_type\": \"aggregate\",$", MR_Next}, 483 {"\"aggregate_name\": \"mean\",$", MR_Next}, 484 {"\"iterations\": 3,$", MR_Next}, 485 {"\"time_unit\": \"us\",?$"}, 486 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, 487 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, 488 {"\"run_type\": \"aggregate\",$", MR_Next}, 489 {"\"aggregate_name\": \"median\",$", MR_Next}, 490 {"\"iterations\": 3,$", MR_Next}, 491 {"\"time_unit\": \"us\",?$"}, 492 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, 493 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, 494 {"\"run_type\": \"aggregate\",$", MR_Next}, 495 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 496 {"\"iterations\": 3,$", MR_Next}, 497 {"\"time_unit\": \"us\",?$"}}); 498ADD_CASES(TC_CSVOut, 499 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 500 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, 501 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"}, 502 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); 503 504// ========================================================================= // 505// -------------------- Testing user-provided statistics ------------------- // 506// ========================================================================= // 507 508const auto UserStatistics = [](const std::vector<double>& v) { 509 return v.back(); 510}; 511void BM_UserStats(benchmark::State& state) { 512 for (auto _ : state) { 513 state.SetIterationTime(150 / 10e8); 514 } 515} 516// clang-format off 517BENCHMARK(BM_UserStats) 518 ->Repetitions(3) 519 ->Iterations(5) 520 ->UseManualTime() 521 ->ComputeStatistics("", UserStatistics); 522// clang-format on 523 524// check that user-provided stats is calculated, and is after the default-ones 525// empty string as name is intentional, it would sort before anything else 526ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " 527 "]* 150 ns %time [ ]*5$"}, 528 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " 529 "]* 150 ns %time [ ]*5$"}, 530 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " 531 "]* 150 ns %time [ ]*5$"}, 532 {"^BM_UserStats/iterations:5/repeats:3/" 533 "manual_time_mean [ ]* 150 ns %time [ ]*3$"}, 534 {"^BM_UserStats/iterations:5/repeats:3/" 535 "manual_time_median [ ]* 150 ns %time [ ]*3$"}, 536 {"^BM_UserStats/iterations:5/repeats:3/" 537 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"}, 538 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ " 539 "[ ]* 150 ns %time [ ]*3$"}}); 540ADD_CASES( 541 TC_JSONOut, 542 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, 543 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 544 MR_Next}, 545 {"\"run_type\": \"iteration\",$", MR_Next}, 546 {"\"iterations\": 5,$", MR_Next}, 547 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 548 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, 549 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 550 MR_Next}, 551 {"\"run_type\": \"iteration\",$", MR_Next}, 552 {"\"iterations\": 5,$", MR_Next}, 553 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 554 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, 555 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 556 MR_Next}, 557 {"\"run_type\": \"iteration\",$", MR_Next}, 558 {"\"iterations\": 5,$", MR_Next}, 559 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 560 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, 561 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 562 MR_Next}, 563 {"\"run_type\": \"aggregate\",$", MR_Next}, 564 {"\"aggregate_name\": \"mean\",$", MR_Next}, 565 {"\"iterations\": 3,$", MR_Next}, 566 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 567 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, 568 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 569 MR_Next}, 570 {"\"run_type\": \"aggregate\",$", MR_Next}, 571 {"\"aggregate_name\": \"median\",$", MR_Next}, 572 {"\"iterations\": 3,$", MR_Next}, 573 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 574 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, 575 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 576 MR_Next}, 577 {"\"run_type\": \"aggregate\",$", MR_Next}, 578 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 579 {"\"iterations\": 3,$", MR_Next}, 580 {"\"real_time\": %float,$", MR_Next}, 581 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, 582 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 583 MR_Next}, 584 {"\"run_type\": \"aggregate\",$", MR_Next}, 585 {"\"aggregate_name\": \"\",$", MR_Next}, 586 {"\"iterations\": 3,$", MR_Next}, 587 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); 588ADD_CASES( 589 TC_CSVOut, 590 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, 591 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, 592 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, 593 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"}, 594 {"^\"BM_UserStats/iterations:5/repeats:3/" 595 "manual_time_median\",%csv_report$"}, 596 {"^\"BM_UserStats/iterations:5/repeats:3/" 597 "manual_time_stddev\",%csv_report$"}, 598 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); 599 600// ========================================================================= // 601// --------------------------- TEST CASES END ------------------------------ // 602// ========================================================================= // 603 604int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 605