1// SPDX-License-Identifier: GPL-2.0 2 3/* Copyright (c) 2019 Facebook */ 4 5#include <assert.h> 6#include <limits.h> 7#include <unistd.h> 8#include <sys/file.h> 9#include <sys/time.h> 10#include <linux/err.h> 11#include <linux/zalloc.h> 12#include <api/fs/fs.h> 13#include <perf/bpf_perf.h> 14 15#include "bpf_counter.h" 16#include "bpf-utils.h" 17#include "counts.h" 18#include "debug.h" 19#include "evsel.h" 20#include "evlist.h" 21#include "target.h" 22#include "cgroup.h" 23#include "cpumap.h" 24#include "thread_map.h" 25 26#include "bpf_skel/bpf_prog_profiler.skel.h" 27#include "bpf_skel/bperf_u.h" 28#include "bpf_skel/bperf_leader.skel.h" 29#include "bpf_skel/bperf_follower.skel.h" 30 31#define ATTR_MAP_SIZE 16 32 33static inline void *u64_to_ptr(__u64 ptr) 34{ 35 return (void *)(unsigned long)ptr; 36} 37 38static struct bpf_counter *bpf_counter_alloc(void) 39{ 40 struct bpf_counter *counter; 41 42 counter = zalloc(sizeof(*counter)); 43 if (counter) 44 INIT_LIST_HEAD(&counter->list); 45 return counter; 46} 47 48static int bpf_program_profiler__destroy(struct evsel *evsel) 49{ 50 struct bpf_counter *counter, *tmp; 51 52 list_for_each_entry_safe(counter, tmp, 53 &evsel->bpf_counter_list, list) { 54 list_del_init(&counter->list); 55 bpf_prog_profiler_bpf__destroy(counter->skel); 56 free(counter); 57 } 58 assert(list_empty(&evsel->bpf_counter_list)); 59 60 return 0; 61} 62 63static char *bpf_target_prog_name(int tgt_fd) 64{ 65 struct bpf_func_info *func_info; 66 struct perf_bpil *info_linear; 67 const struct btf_type *t; 68 struct btf *btf = NULL; 69 char *name = NULL; 70 71 info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO); 72 if (IS_ERR_OR_NULL(info_linear)) { 73 pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd); 74 return NULL; 75 } 76 77 if (info_linear->info.btf_id == 0) { 78 pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd); 79 goto out; 80 } 81 82 btf = btf__load_from_kernel_by_id(info_linear->info.btf_id); 83 if (libbpf_get_error(btf)) { 84 pr_debug("failed to load btf for prog FD %d\n", tgt_fd); 85 goto out; 86 } 87 88 func_info = u64_to_ptr(info_linear->info.func_info); 89 t = btf__type_by_id(btf, func_info[0].type_id); 90 if (!t) { 91 pr_debug("btf %d doesn't have type %d\n", 92 info_linear->info.btf_id, func_info[0].type_id); 93 goto out; 94 } 95 name = strdup(btf__name_by_offset(btf, t->name_off)); 96out: 97 btf__free(btf); 98 free(info_linear); 99 return name; 100} 101 102static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) 103{ 104 struct bpf_prog_profiler_bpf *skel; 105 struct bpf_counter *counter; 106 struct bpf_program *prog; 107 char *prog_name = NULL; 108 int prog_fd; 109 int err; 110 111 prog_fd = bpf_prog_get_fd_by_id(prog_id); 112 if (prog_fd < 0) { 113 pr_err("Failed to open fd for bpf prog %u\n", prog_id); 114 return -1; 115 } 116 counter = bpf_counter_alloc(); 117 if (!counter) { 118 close(prog_fd); 119 return -1; 120 } 121 122 skel = bpf_prog_profiler_bpf__open(); 123 if (!skel) { 124 pr_err("Failed to open bpf skeleton\n"); 125 goto err_out; 126 } 127 128 skel->rodata->num_cpu = evsel__nr_cpus(evsel); 129 130 bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel)); 131 bpf_map__set_max_entries(skel->maps.fentry_readings, 1); 132 bpf_map__set_max_entries(skel->maps.accum_readings, 1); 133 134 prog_name = bpf_target_prog_name(prog_fd); 135 if (!prog_name) { 136 pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id); 137 goto err_out; 138 } 139 140 bpf_object__for_each_program(prog, skel->obj) { 141 err = bpf_program__set_attach_target(prog, prog_fd, prog_name); 142 if (err) { 143 pr_err("bpf_program__set_attach_target failed.\n" 144 "Does bpf prog %u have BTF?\n", prog_id); 145 goto err_out; 146 } 147 } 148 set_max_rlimit(); 149 err = bpf_prog_profiler_bpf__load(skel); 150 if (err) { 151 pr_err("bpf_prog_profiler_bpf__load failed\n"); 152 goto err_out; 153 } 154 155 assert(skel != NULL); 156 counter->skel = skel; 157 list_add(&counter->list, &evsel->bpf_counter_list); 158 free(prog_name); 159 close(prog_fd); 160 return 0; 161err_out: 162 bpf_prog_profiler_bpf__destroy(skel); 163 free(prog_name); 164 free(counter); 165 close(prog_fd); 166 return -1; 167} 168 169static int bpf_program_profiler__load(struct evsel *evsel, struct target *target) 170{ 171 char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p; 172 u32 prog_id; 173 int ret; 174 175 bpf_str_ = bpf_str = strdup(target->bpf_str); 176 if (!bpf_str) 177 return -1; 178 179 while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) { 180 prog_id = strtoul(tok, &p, 10); 181 if (prog_id == 0 || prog_id == UINT_MAX || 182 (*p != '\0' && *p != ',')) { 183 pr_err("Failed to parse bpf prog ids %s\n", 184 target->bpf_str); 185 free(bpf_str_); 186 return -1; 187 } 188 189 ret = bpf_program_profiler_load_one(evsel, prog_id); 190 if (ret) { 191 bpf_program_profiler__destroy(evsel); 192 free(bpf_str_); 193 return -1; 194 } 195 bpf_str = NULL; 196 } 197 free(bpf_str_); 198 return 0; 199} 200 201static int bpf_program_profiler__enable(struct evsel *evsel) 202{ 203 struct bpf_counter *counter; 204 int ret; 205 206 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 207 assert(counter->skel != NULL); 208 ret = bpf_prog_profiler_bpf__attach(counter->skel); 209 if (ret) { 210 bpf_program_profiler__destroy(evsel); 211 return ret; 212 } 213 } 214 return 0; 215} 216 217static int bpf_program_profiler__disable(struct evsel *evsel) 218{ 219 struct bpf_counter *counter; 220 221 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 222 assert(counter->skel != NULL); 223 bpf_prog_profiler_bpf__detach(counter->skel); 224 } 225 return 0; 226} 227 228static int bpf_program_profiler__read(struct evsel *evsel) 229{ 230 // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible 231 // Sometimes possible > online, like on a Ryzen 3900X that has 24 232 // threads but its possible showed 0-31 -acme 233 int num_cpu_bpf = libbpf_num_possible_cpus(); 234 struct bpf_perf_event_value values[num_cpu_bpf]; 235 struct bpf_counter *counter; 236 struct perf_counts_values *counts; 237 int reading_map_fd; 238 __u32 key = 0; 239 int err, idx, bpf_cpu; 240 241 if (list_empty(&evsel->bpf_counter_list)) 242 return -EAGAIN; 243 244 perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) { 245 counts = perf_counts(evsel->counts, idx, 0); 246 counts->val = 0; 247 counts->ena = 0; 248 counts->run = 0; 249 } 250 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 251 struct bpf_prog_profiler_bpf *skel = counter->skel; 252 253 assert(skel != NULL); 254 reading_map_fd = bpf_map__fd(skel->maps.accum_readings); 255 256 err = bpf_map_lookup_elem(reading_map_fd, &key, values); 257 if (err) { 258 pr_err("failed to read value\n"); 259 return err; 260 } 261 262 for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) { 263 idx = perf_cpu_map__idx(evsel__cpus(evsel), 264 (struct perf_cpu){.cpu = bpf_cpu}); 265 if (idx == -1) 266 continue; 267 counts = perf_counts(evsel->counts, idx, 0); 268 counts->val += values[bpf_cpu].counter; 269 counts->ena += values[bpf_cpu].enabled; 270 counts->run += values[bpf_cpu].running; 271 } 272 } 273 return 0; 274} 275 276static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx, 277 int fd) 278{ 279 struct bpf_prog_profiler_bpf *skel; 280 struct bpf_counter *counter; 281 int ret; 282 283 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 284 skel = counter->skel; 285 assert(skel != NULL); 286 287 ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), 288 &cpu_map_idx, &fd, BPF_ANY); 289 if (ret) 290 return ret; 291 } 292 return 0; 293} 294 295struct bpf_counter_ops bpf_program_profiler_ops = { 296 .load = bpf_program_profiler__load, 297 .enable = bpf_program_profiler__enable, 298 .disable = bpf_program_profiler__disable, 299 .read = bpf_program_profiler__read, 300 .destroy = bpf_program_profiler__destroy, 301 .install_pe = bpf_program_profiler__install_pe, 302}; 303 304static bool bperf_attr_map_compatible(int attr_map_fd) 305{ 306 struct bpf_map_info map_info = {0}; 307 __u32 map_info_len = sizeof(map_info); 308 int err; 309 310 err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len); 311 312 if (err) 313 return false; 314 return (map_info.key_size == sizeof(struct perf_event_attr)) && 315 (map_info.value_size == sizeof(struct perf_event_attr_map_entry)); 316} 317 318static int bperf_lock_attr_map(struct target *target) 319{ 320 char path[PATH_MAX]; 321 int map_fd, err; 322 323 if (target->attr_map) { 324 scnprintf(path, PATH_MAX, "%s", target->attr_map); 325 } else { 326 scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(), 327 BPF_PERF_DEFAULT_ATTR_MAP_PATH); 328 } 329 330 if (access(path, F_OK)) { 331 map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, 332 sizeof(struct perf_event_attr), 333 sizeof(struct perf_event_attr_map_entry), 334 ATTR_MAP_SIZE, NULL); 335 if (map_fd < 0) 336 return -1; 337 338 err = bpf_obj_pin(map_fd, path); 339 if (err) { 340 /* someone pinned the map in parallel? */ 341 close(map_fd); 342 map_fd = bpf_obj_get(path); 343 if (map_fd < 0) 344 return -1; 345 } 346 } else { 347 map_fd = bpf_obj_get(path); 348 if (map_fd < 0) 349 return -1; 350 } 351 352 if (!bperf_attr_map_compatible(map_fd)) { 353 close(map_fd); 354 return -1; 355 356 } 357 err = flock(map_fd, LOCK_EX); 358 if (err) { 359 close(map_fd); 360 return -1; 361 } 362 return map_fd; 363} 364 365static int bperf_check_target(struct evsel *evsel, 366 struct target *target, 367 enum bperf_filter_type *filter_type, 368 __u32 *filter_entry_cnt) 369{ 370 if (evsel->core.leader->nr_members > 1) { 371 pr_err("bpf managed perf events do not yet support groups.\n"); 372 return -1; 373 } 374 375 /* determine filter type based on target */ 376 if (target->system_wide) { 377 *filter_type = BPERF_FILTER_GLOBAL; 378 *filter_entry_cnt = 1; 379 } else if (target->cpu_list) { 380 *filter_type = BPERF_FILTER_CPU; 381 *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel)); 382 } else if (target->tid) { 383 *filter_type = BPERF_FILTER_PID; 384 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); 385 } else if (target->pid || evsel->evlist->workload.pid != -1) { 386 *filter_type = BPERF_FILTER_TGID; 387 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); 388 } else { 389 pr_err("bpf managed perf events do not yet support these targets.\n"); 390 return -1; 391 } 392 393 return 0; 394} 395 396static struct perf_cpu_map *all_cpu_map; 397 398static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, 399 struct perf_event_attr_map_entry *entry) 400{ 401 struct bperf_leader_bpf *skel = bperf_leader_bpf__open(); 402 int link_fd, diff_map_fd, err; 403 struct bpf_link *link = NULL; 404 405 if (!skel) { 406 pr_err("Failed to open leader skeleton\n"); 407 return -1; 408 } 409 410 bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus()); 411 err = bperf_leader_bpf__load(skel); 412 if (err) { 413 pr_err("Failed to load leader skeleton\n"); 414 goto out; 415 } 416 417 link = bpf_program__attach(skel->progs.on_switch); 418 if (IS_ERR(link)) { 419 pr_err("Failed to attach leader program\n"); 420 err = PTR_ERR(link); 421 goto out; 422 } 423 424 link_fd = bpf_link__fd(link); 425 diff_map_fd = bpf_map__fd(skel->maps.diff_readings); 426 entry->link_id = bpf_link_get_id(link_fd); 427 entry->diff_map_id = bpf_map_get_id(diff_map_fd); 428 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY); 429 assert(err == 0); 430 431 evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id); 432 assert(evsel->bperf_leader_link_fd >= 0); 433 434 /* 435 * save leader_skel for install_pe, which is called within 436 * following evsel__open_per_cpu call 437 */ 438 evsel->leader_skel = skel; 439 evsel__open_per_cpu(evsel, all_cpu_map, -1); 440 441out: 442 bperf_leader_bpf__destroy(skel); 443 bpf_link__destroy(link); 444 return err; 445} 446 447static int bperf__load(struct evsel *evsel, struct target *target) 448{ 449 struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff}; 450 int attr_map_fd, diff_map_fd = -1, err; 451 enum bperf_filter_type filter_type; 452 __u32 filter_entry_cnt, i; 453 454 if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt)) 455 return -1; 456 457 if (!all_cpu_map) { 458 all_cpu_map = perf_cpu_map__new_online_cpus(); 459 if (!all_cpu_map) 460 return -1; 461 } 462 463 evsel->bperf_leader_prog_fd = -1; 464 evsel->bperf_leader_link_fd = -1; 465 466 /* 467 * Step 1: hold a fd on the leader program and the bpf_link, if 468 * the program is not already gone, reload the program. 469 * Use flock() to ensure exclusive access to the perf_event_attr 470 * map. 471 */ 472 attr_map_fd = bperf_lock_attr_map(target); 473 if (attr_map_fd < 0) { 474 pr_err("Failed to lock perf_event_attr map\n"); 475 return -1; 476 } 477 478 err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry); 479 if (err) { 480 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY); 481 if (err) 482 goto out; 483 } 484 485 evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id); 486 if (evsel->bperf_leader_link_fd < 0 && 487 bperf_reload_leader_program(evsel, attr_map_fd, &entry)) { 488 err = -1; 489 goto out; 490 } 491 /* 492 * The bpf_link holds reference to the leader program, and the 493 * leader program holds reference to the maps. Therefore, if 494 * link_id is valid, diff_map_id should also be valid. 495 */ 496 evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id( 497 bpf_link_get_prog_id(evsel->bperf_leader_link_fd)); 498 assert(evsel->bperf_leader_prog_fd >= 0); 499 500 diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id); 501 assert(diff_map_fd >= 0); 502 503 /* 504 * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check 505 * whether the kernel support it 506 */ 507 err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0); 508 if (err) { 509 pr_err("The kernel does not support test_run for raw_tp BPF programs.\n" 510 "Therefore, --use-bpf might show inaccurate readings\n"); 511 goto out; 512 } 513 514 /* Step 2: load the follower skeleton */ 515 evsel->follower_skel = bperf_follower_bpf__open(); 516 if (!evsel->follower_skel) { 517 err = -1; 518 pr_err("Failed to open follower skeleton\n"); 519 goto out; 520 } 521 522 /* attach fexit program to the leader program */ 523 bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX, 524 evsel->bperf_leader_prog_fd, "on_switch"); 525 526 /* connect to leader diff_reading map */ 527 bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd); 528 529 /* set up reading map */ 530 bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings, 531 filter_entry_cnt); 532 /* set up follower filter based on target */ 533 bpf_map__set_max_entries(evsel->follower_skel->maps.filter, 534 filter_entry_cnt); 535 err = bperf_follower_bpf__load(evsel->follower_skel); 536 if (err) { 537 pr_err("Failed to load follower skeleton\n"); 538 bperf_follower_bpf__destroy(evsel->follower_skel); 539 evsel->follower_skel = NULL; 540 goto out; 541 } 542 543 for (i = 0; i < filter_entry_cnt; i++) { 544 int filter_map_fd; 545 __u32 key; 546 547 if (filter_type == BPERF_FILTER_PID || 548 filter_type == BPERF_FILTER_TGID) 549 key = perf_thread_map__pid(evsel->core.threads, i); 550 else if (filter_type == BPERF_FILTER_CPU) 551 key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu; 552 else 553 break; 554 555 filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter); 556 bpf_map_update_elem(filter_map_fd, &key, &i, BPF_ANY); 557 } 558 559 evsel->follower_skel->bss->type = filter_type; 560 561 err = bperf_follower_bpf__attach(evsel->follower_skel); 562 563out: 564 if (err && evsel->bperf_leader_link_fd >= 0) 565 close(evsel->bperf_leader_link_fd); 566 if (err && evsel->bperf_leader_prog_fd >= 0) 567 close(evsel->bperf_leader_prog_fd); 568 if (diff_map_fd >= 0) 569 close(diff_map_fd); 570 571 flock(attr_map_fd, LOCK_UN); 572 close(attr_map_fd); 573 574 return err; 575} 576 577static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) 578{ 579 struct bperf_leader_bpf *skel = evsel->leader_skel; 580 581 return bpf_map_update_elem(bpf_map__fd(skel->maps.events), 582 &cpu_map_idx, &fd, BPF_ANY); 583} 584 585/* 586 * trigger the leader prog on each cpu, so the accum_reading map could get 587 * the latest readings. 588 */ 589static int bperf_sync_counters(struct evsel *evsel) 590{ 591 int num_cpu, i, cpu; 592 593 num_cpu = perf_cpu_map__nr(all_cpu_map); 594 for (i = 0; i < num_cpu; i++) { 595 cpu = perf_cpu_map__cpu(all_cpu_map, i).cpu; 596 bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu); 597 } 598 return 0; 599} 600 601static int bperf__enable(struct evsel *evsel) 602{ 603 evsel->follower_skel->bss->enabled = 1; 604 return 0; 605} 606 607static int bperf__disable(struct evsel *evsel) 608{ 609 evsel->follower_skel->bss->enabled = 0; 610 return 0; 611} 612 613static int bperf__read(struct evsel *evsel) 614{ 615 struct bperf_follower_bpf *skel = evsel->follower_skel; 616 __u32 num_cpu_bpf = cpu__max_cpu().cpu; 617 struct bpf_perf_event_value values[num_cpu_bpf]; 618 struct perf_counts_values *counts; 619 int reading_map_fd, err = 0; 620 __u32 i; 621 int j; 622 623 bperf_sync_counters(evsel); 624 reading_map_fd = bpf_map__fd(skel->maps.accum_readings); 625 626 for (i = 0; i < bpf_map__max_entries(skel->maps.accum_readings); i++) { 627 struct perf_cpu entry; 628 __u32 cpu; 629 630 err = bpf_map_lookup_elem(reading_map_fd, &i, values); 631 if (err) 632 goto out; 633 switch (evsel->follower_skel->bss->type) { 634 case BPERF_FILTER_GLOBAL: 635 assert(i == 0); 636 637 perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) { 638 counts = perf_counts(evsel->counts, j, 0); 639 counts->val = values[entry.cpu].counter; 640 counts->ena = values[entry.cpu].enabled; 641 counts->run = values[entry.cpu].running; 642 } 643 break; 644 case BPERF_FILTER_CPU: 645 cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu; 646 assert(cpu >= 0); 647 counts = perf_counts(evsel->counts, i, 0); 648 counts->val = values[cpu].counter; 649 counts->ena = values[cpu].enabled; 650 counts->run = values[cpu].running; 651 break; 652 case BPERF_FILTER_PID: 653 case BPERF_FILTER_TGID: 654 counts = perf_counts(evsel->counts, 0, i); 655 counts->val = 0; 656 counts->ena = 0; 657 counts->run = 0; 658 659 for (cpu = 0; cpu < num_cpu_bpf; cpu++) { 660 counts->val += values[cpu].counter; 661 counts->ena += values[cpu].enabled; 662 counts->run += values[cpu].running; 663 } 664 break; 665 default: 666 break; 667 } 668 } 669out: 670 return err; 671} 672 673static int bperf__destroy(struct evsel *evsel) 674{ 675 bperf_follower_bpf__destroy(evsel->follower_skel); 676 close(evsel->bperf_leader_prog_fd); 677 close(evsel->bperf_leader_link_fd); 678 return 0; 679} 680 681/* 682 * bperf: share hardware PMCs with BPF 683 * 684 * perf uses performance monitoring counters (PMC) to monitor system 685 * performance. The PMCs are limited hardware resources. For example, 686 * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu. 687 * 688 * Modern data center systems use these PMCs in many different ways: 689 * system level monitoring, (maybe nested) container level monitoring, per 690 * process monitoring, profiling (in sample mode), etc. In some cases, 691 * there are more active perf_events than available hardware PMCs. To allow 692 * all perf_events to have a chance to run, it is necessary to do expensive 693 * time multiplexing of events. 694 * 695 * On the other hand, many monitoring tools count the common metrics 696 * (cycles, instructions). It is a waste to have multiple tools create 697 * multiple perf_events of "cycles" and occupy multiple PMCs. 698 * 699 * bperf tries to reduce such wastes by allowing multiple perf_events of 700 * "cycles" or "instructions" (at different scopes) to share PMUs. Instead 701 * of having each perf-stat session to read its own perf_events, bperf uses 702 * BPF programs to read the perf_events and aggregate readings to BPF maps. 703 * Then, the perf-stat session(s) reads the values from these BPF maps. 704 * 705 * || 706 * shared progs and maps <- || -> per session progs and maps 707 * || 708 * --------------- || 709 * | perf_events | || 710 * --------------- fexit || ----------------- 711 * | --------||----> | follower prog | 712 * --------------- / || --- ----------------- 713 * cs -> | leader prog |/ ||/ | | 714 * --> --------------- /|| -------------- ------------------ 715 * / | | / || | filter map | | accum_readings | 716 * / ------------ ------------ || -------------- ------------------ 717 * | | prev map | | diff map | || | 718 * | ------------ ------------ || | 719 * \ || | 720 * = \ ==================================================== | ============ 721 * \ / user space 722 * \ / 723 * \ / 724 * BPF_PROG_TEST_RUN BPF_MAP_LOOKUP_ELEM 725 * \ / 726 * \ / 727 * \------ perf-stat ----------------------/ 728 * 729 * The figure above shows the architecture of bperf. Note that the figure 730 * is divided into 3 regions: shared progs and maps (top left), per session 731 * progs and maps (top right), and user space (bottom). 732 * 733 * The leader prog is triggered on each context switch (cs). The leader 734 * prog reads perf_events and stores the difference (current_reading - 735 * previous_reading) to the diff map. For the same metric, e.g. "cycles", 736 * multiple perf-stat sessions share the same leader prog. 737 * 738 * Each perf-stat session creates a follower prog as fexit program to the 739 * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38) 740 * follower progs to the same leader prog. The follower prog checks current 741 * task and processor ID to decide whether to add the value from the diff 742 * map to its accumulated reading map (accum_readings). 743 * 744 * Finally, perf-stat user space reads the value from accum_reading map. 745 * 746 * Besides context switch, it is also necessary to trigger the leader prog 747 * before perf-stat reads the value. Otherwise, the accum_reading map may 748 * not have the latest reading from the perf_events. This is achieved by 749 * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU. 750 * 751 * Comment before the definition of struct perf_event_attr_map_entry 752 * describes how different sessions of perf-stat share information about 753 * the leader prog. 754 */ 755 756struct bpf_counter_ops bperf_ops = { 757 .load = bperf__load, 758 .enable = bperf__enable, 759 .disable = bperf__disable, 760 .read = bperf__read, 761 .install_pe = bperf__install_pe, 762 .destroy = bperf__destroy, 763}; 764 765extern struct bpf_counter_ops bperf_cgrp_ops; 766 767static inline bool bpf_counter_skip(struct evsel *evsel) 768{ 769 return evsel->bpf_counter_ops == NULL; 770} 771 772int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) 773{ 774 if (bpf_counter_skip(evsel)) 775 return 0; 776 return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd); 777} 778 779int bpf_counter__load(struct evsel *evsel, struct target *target) 780{ 781 if (target->bpf_str) 782 evsel->bpf_counter_ops = &bpf_program_profiler_ops; 783 else if (cgrp_event_expanded && target->use_bpf) 784 evsel->bpf_counter_ops = &bperf_cgrp_ops; 785 else if (target->use_bpf || evsel->bpf_counter || 786 evsel__match_bpf_counter_events(evsel->name)) 787 evsel->bpf_counter_ops = &bperf_ops; 788 789 if (evsel->bpf_counter_ops) 790 return evsel->bpf_counter_ops->load(evsel, target); 791 return 0; 792} 793 794int bpf_counter__enable(struct evsel *evsel) 795{ 796 if (bpf_counter_skip(evsel)) 797 return 0; 798 return evsel->bpf_counter_ops->enable(evsel); 799} 800 801int bpf_counter__disable(struct evsel *evsel) 802{ 803 if (bpf_counter_skip(evsel)) 804 return 0; 805 return evsel->bpf_counter_ops->disable(evsel); 806} 807 808int bpf_counter__read(struct evsel *evsel) 809{ 810 if (bpf_counter_skip(evsel)) 811 return -EAGAIN; 812 return evsel->bpf_counter_ops->read(evsel); 813} 814 815void bpf_counter__destroy(struct evsel *evsel) 816{ 817 if (bpf_counter_skip(evsel)) 818 return; 819 evsel->bpf_counter_ops->destroy(evsel); 820 evsel->bpf_counter_ops = NULL; 821 evsel->bpf_skel = NULL; 822} 823