1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2020 Facebook */ 3#include "bench.h" 4#include "trigger_bench.skel.h" 5#include "trace_helpers.h" 6 7/* BPF triggering benchmarks */ 8static struct trigger_ctx { 9 struct trigger_bench *skel; 10} ctx; 11 12static struct counter base_hits; 13 14static void trigger_validate(void) 15{ 16 if (env.consumer_cnt != 0) { 17 fprintf(stderr, "benchmark doesn't support consumer!\n"); 18 exit(1); 19 } 20} 21 22static void *trigger_base_producer(void *input) 23{ 24 while (true) { 25 (void)syscall(__NR_getpgid); 26 atomic_inc(&base_hits.value); 27 } 28 return NULL; 29} 30 31static void trigger_base_measure(struct bench_res *res) 32{ 33 res->hits = atomic_swap(&base_hits.value, 0); 34} 35 36static void *trigger_producer(void *input) 37{ 38 while (true) 39 (void)syscall(__NR_getpgid); 40 return NULL; 41} 42 43static void trigger_measure(struct bench_res *res) 44{ 45 res->hits = atomic_swap(&ctx.skel->bss->hits, 0); 46} 47 48static void setup_ctx(void) 49{ 50 setup_libbpf(); 51 52 ctx.skel = trigger_bench__open_and_load(); 53 if (!ctx.skel) { 54 fprintf(stderr, "failed to open skeleton\n"); 55 exit(1); 56 } 57} 58 59static void attach_bpf(struct bpf_program *prog) 60{ 61 struct bpf_link *link; 62 63 link = bpf_program__attach(prog); 64 if (!link) { 65 fprintf(stderr, "failed to attach program!\n"); 66 exit(1); 67 } 68} 69 70static void trigger_tp_setup(void) 71{ 72 setup_ctx(); 73 attach_bpf(ctx.skel->progs.bench_trigger_tp); 74} 75 76static void trigger_rawtp_setup(void) 77{ 78 setup_ctx(); 79 attach_bpf(ctx.skel->progs.bench_trigger_raw_tp); 80} 81 82static void trigger_kprobe_setup(void) 83{ 84 setup_ctx(); 85 attach_bpf(ctx.skel->progs.bench_trigger_kprobe); 86} 87 88static void trigger_kretprobe_setup(void) 89{ 90 setup_ctx(); 91 attach_bpf(ctx.skel->progs.bench_trigger_kretprobe); 92} 93 94static void trigger_kprobe_multi_setup(void) 95{ 96 setup_ctx(); 97 attach_bpf(ctx.skel->progs.bench_trigger_kprobe_multi); 98} 99 100static void trigger_kretprobe_multi_setup(void) 101{ 102 setup_ctx(); 103 attach_bpf(ctx.skel->progs.bench_trigger_kretprobe_multi); 104} 105 106static void trigger_fentry_setup(void) 107{ 108 setup_ctx(); 109 attach_bpf(ctx.skel->progs.bench_trigger_fentry); 110} 111 112static void trigger_fexit_setup(void) 113{ 114 setup_ctx(); 115 attach_bpf(ctx.skel->progs.bench_trigger_fexit); 116} 117 118static void trigger_fentry_sleep_setup(void) 119{ 120 setup_ctx(); 121 attach_bpf(ctx.skel->progs.bench_trigger_fentry_sleep); 122} 123 124static void trigger_fmodret_setup(void) 125{ 126 setup_ctx(); 127 attach_bpf(ctx.skel->progs.bench_trigger_fmodret); 128} 129 130/* make sure call is not inlined and not avoided by compiler, so __weak and 131 * inline asm volatile in the body of the function 132 * 133 * There is a performance difference between uprobing at nop location vs other 134 * instructions. So use two different targets, one of which starts with nop 135 * and another doesn't. 136 * 137 * GCC doesn't generate stack setup preample for these functions due to them 138 * having no input arguments and doing nothing in the body. 139 */ 140__weak void uprobe_target_nop(void) 141{ 142 asm volatile ("nop"); 143} 144 145__weak void opaque_noop_func(void) 146{ 147} 148 149__weak int uprobe_target_push(void) 150{ 151 /* overhead of function call is negligible compared to uprobe 152 * triggering, so this shouldn't affect benchmark results much 153 */ 154 opaque_noop_func(); 155 return 1; 156} 157 158__weak void uprobe_target_ret(void) 159{ 160 asm volatile (""); 161} 162 163static void *uprobe_base_producer(void *input) 164{ 165 while (true) { 166 uprobe_target_nop(); 167 atomic_inc(&base_hits.value); 168 } 169 return NULL; 170} 171 172static void *uprobe_producer_nop(void *input) 173{ 174 while (true) 175 uprobe_target_nop(); 176 return NULL; 177} 178 179static void *uprobe_producer_push(void *input) 180{ 181 while (true) 182 uprobe_target_push(); 183 return NULL; 184} 185 186static void *uprobe_producer_ret(void *input) 187{ 188 while (true) 189 uprobe_target_ret(); 190 return NULL; 191} 192 193static void usetup(bool use_retprobe, void *target_addr) 194{ 195 size_t uprobe_offset; 196 struct bpf_link *link; 197 198 setup_libbpf(); 199 200 ctx.skel = trigger_bench__open_and_load(); 201 if (!ctx.skel) { 202 fprintf(stderr, "failed to open skeleton\n"); 203 exit(1); 204 } 205 206 uprobe_offset = get_uprobe_offset(target_addr); 207 link = bpf_program__attach_uprobe(ctx.skel->progs.bench_trigger_uprobe, 208 use_retprobe, 209 -1 /* all PIDs */, 210 "/proc/self/exe", 211 uprobe_offset); 212 if (!link) { 213 fprintf(stderr, "failed to attach uprobe!\n"); 214 exit(1); 215 } 216 ctx.skel->links.bench_trigger_uprobe = link; 217} 218 219static void uprobe_setup_nop(void) 220{ 221 usetup(false, &uprobe_target_nop); 222} 223 224static void uretprobe_setup_nop(void) 225{ 226 usetup(true, &uprobe_target_nop); 227} 228 229static void uprobe_setup_push(void) 230{ 231 usetup(false, &uprobe_target_push); 232} 233 234static void uretprobe_setup_push(void) 235{ 236 usetup(true, &uprobe_target_push); 237} 238 239static void uprobe_setup_ret(void) 240{ 241 usetup(false, &uprobe_target_ret); 242} 243 244static void uretprobe_setup_ret(void) 245{ 246 usetup(true, &uprobe_target_ret); 247} 248 249const struct bench bench_trig_base = { 250 .name = "trig-base", 251 .validate = trigger_validate, 252 .producer_thread = trigger_base_producer, 253 .measure = trigger_base_measure, 254 .report_progress = hits_drops_report_progress, 255 .report_final = hits_drops_report_final, 256}; 257 258const struct bench bench_trig_tp = { 259 .name = "trig-tp", 260 .validate = trigger_validate, 261 .setup = trigger_tp_setup, 262 .producer_thread = trigger_producer, 263 .measure = trigger_measure, 264 .report_progress = hits_drops_report_progress, 265 .report_final = hits_drops_report_final, 266}; 267 268const struct bench bench_trig_rawtp = { 269 .name = "trig-rawtp", 270 .validate = trigger_validate, 271 .setup = trigger_rawtp_setup, 272 .producer_thread = trigger_producer, 273 .measure = trigger_measure, 274 .report_progress = hits_drops_report_progress, 275 .report_final = hits_drops_report_final, 276}; 277 278const struct bench bench_trig_kprobe = { 279 .name = "trig-kprobe", 280 .validate = trigger_validate, 281 .setup = trigger_kprobe_setup, 282 .producer_thread = trigger_producer, 283 .measure = trigger_measure, 284 .report_progress = hits_drops_report_progress, 285 .report_final = hits_drops_report_final, 286}; 287 288const struct bench bench_trig_kretprobe = { 289 .name = "trig-kretprobe", 290 .validate = trigger_validate, 291 .setup = trigger_kretprobe_setup, 292 .producer_thread = trigger_producer, 293 .measure = trigger_measure, 294 .report_progress = hits_drops_report_progress, 295 .report_final = hits_drops_report_final, 296}; 297 298const struct bench bench_trig_kprobe_multi = { 299 .name = "trig-kprobe-multi", 300 .validate = trigger_validate, 301 .setup = trigger_kprobe_multi_setup, 302 .producer_thread = trigger_producer, 303 .measure = trigger_measure, 304 .report_progress = hits_drops_report_progress, 305 .report_final = hits_drops_report_final, 306}; 307 308const struct bench bench_trig_kretprobe_multi = { 309 .name = "trig-kretprobe-multi", 310 .validate = trigger_validate, 311 .setup = trigger_kretprobe_multi_setup, 312 .producer_thread = trigger_producer, 313 .measure = trigger_measure, 314 .report_progress = hits_drops_report_progress, 315 .report_final = hits_drops_report_final, 316}; 317 318const struct bench bench_trig_fentry = { 319 .name = "trig-fentry", 320 .validate = trigger_validate, 321 .setup = trigger_fentry_setup, 322 .producer_thread = trigger_producer, 323 .measure = trigger_measure, 324 .report_progress = hits_drops_report_progress, 325 .report_final = hits_drops_report_final, 326}; 327 328const struct bench bench_trig_fexit = { 329 .name = "trig-fexit", 330 .validate = trigger_validate, 331 .setup = trigger_fexit_setup, 332 .producer_thread = trigger_producer, 333 .measure = trigger_measure, 334 .report_progress = hits_drops_report_progress, 335 .report_final = hits_drops_report_final, 336}; 337 338const struct bench bench_trig_fentry_sleep = { 339 .name = "trig-fentry-sleep", 340 .validate = trigger_validate, 341 .setup = trigger_fentry_sleep_setup, 342 .producer_thread = trigger_producer, 343 .measure = trigger_measure, 344 .report_progress = hits_drops_report_progress, 345 .report_final = hits_drops_report_final, 346}; 347 348const struct bench bench_trig_fmodret = { 349 .name = "trig-fmodret", 350 .validate = trigger_validate, 351 .setup = trigger_fmodret_setup, 352 .producer_thread = trigger_producer, 353 .measure = trigger_measure, 354 .report_progress = hits_drops_report_progress, 355 .report_final = hits_drops_report_final, 356}; 357 358const struct bench bench_trig_uprobe_base = { 359 .name = "trig-uprobe-base", 360 .setup = NULL, /* no uprobe/uretprobe is attached */ 361 .producer_thread = uprobe_base_producer, 362 .measure = trigger_base_measure, 363 .report_progress = hits_drops_report_progress, 364 .report_final = hits_drops_report_final, 365}; 366 367const struct bench bench_trig_uprobe_nop = { 368 .name = "trig-uprobe-nop", 369 .setup = uprobe_setup_nop, 370 .producer_thread = uprobe_producer_nop, 371 .measure = trigger_measure, 372 .report_progress = hits_drops_report_progress, 373 .report_final = hits_drops_report_final, 374}; 375 376const struct bench bench_trig_uretprobe_nop = { 377 .name = "trig-uretprobe-nop", 378 .setup = uretprobe_setup_nop, 379 .producer_thread = uprobe_producer_nop, 380 .measure = trigger_measure, 381 .report_progress = hits_drops_report_progress, 382 .report_final = hits_drops_report_final, 383}; 384 385const struct bench bench_trig_uprobe_push = { 386 .name = "trig-uprobe-push", 387 .setup = uprobe_setup_push, 388 .producer_thread = uprobe_producer_push, 389 .measure = trigger_measure, 390 .report_progress = hits_drops_report_progress, 391 .report_final = hits_drops_report_final, 392}; 393 394const struct bench bench_trig_uretprobe_push = { 395 .name = "trig-uretprobe-push", 396 .setup = uretprobe_setup_push, 397 .producer_thread = uprobe_producer_push, 398 .measure = trigger_measure, 399 .report_progress = hits_drops_report_progress, 400 .report_final = hits_drops_report_final, 401}; 402 403const struct bench bench_trig_uprobe_ret = { 404 .name = "trig-uprobe-ret", 405 .setup = uprobe_setup_ret, 406 .producer_thread = uprobe_producer_ret, 407 .measure = trigger_measure, 408 .report_progress = hits_drops_report_progress, 409 .report_final = hits_drops_report_final, 410}; 411 412const struct bench bench_trig_uretprobe_ret = { 413 .name = "trig-uretprobe-ret", 414 .setup = uretprobe_setup_ret, 415 .producer_thread = uprobe_producer_ret, 416 .measure = trigger_measure, 417 .report_progress = hits_drops_report_progress, 418 .report_final = hits_drops_report_final, 419}; 420