11590Srgrimes// SPDX-License-Identifier: GPL-2.0 21590Srgrimes/* 31590Srgrimes * bpf_kwork.c 41590Srgrimes * 51590Srgrimes * Copyright (c) 2022 Huawei Inc, Yang Jihong <yangjihong1@huawei.com> 61590Srgrimes */ 71590Srgrimes 81590Srgrimes#include <time.h> 91590Srgrimes#include <fcntl.h> 101590Srgrimes#include <signal.h> 111590Srgrimes#include <stdio.h> 121590Srgrimes#include <unistd.h> 131590Srgrimes 141590Srgrimes#include <linux/time64.h> 151590Srgrimes 161590Srgrimes#include "util/debug.h" 171590Srgrimes#include "util/evsel.h" 181590Srgrimes#include "util/kwork.h" 191590Srgrimes 201590Srgrimes#include <bpf/bpf.h> 211590Srgrimes#include <perf/cpumap.h> 221590Srgrimes 231590Srgrimes#include "util/bpf_skel/kwork_trace.skel.h" 241590Srgrimes 251590Srgrimes/* 261590Srgrimes * This should be in sync with "util/kwork_trace.bpf.c" 271590Srgrimes */ 281590Srgrimes#define MAX_KWORKNAME 128 291590Srgrimes 301590Srgrimesstruct work_key { 311590Srgrimes u32 type; 321590Srgrimes u32 cpu; 331590Srgrimes u64 id; 341590Srgrimes}; 351590Srgrimes 361590Srgrimesstruct report_data { 371590Srgrimes u64 nr; 381590Srgrimes u64 total_time; 391590Srgrimes u64 max_time; 401590Srgrimes u64 max_time_start; 411590Srgrimes u64 max_time_end; 421590Srgrimes}; 431590Srgrimes 441590Srgrimesstruct kwork_class_bpf { 451590Srgrimes struct kwork_class *class; 461590Srgrimes 471590Srgrimes void (*load_prepare)(struct perf_kwork *kwork); 481590Srgrimes int (*get_work_name)(struct work_key *key, char **ret_name); 491590Srgrimes}; 501590Srgrimes 511590Srgrimesstatic struct kwork_trace_bpf *skel; 521590Srgrimes 531590Srgrimesstatic struct timespec ts_start; 541590Srgrimesstatic struct timespec ts_end; 551590Srgrimes 561590Srgrimesvoid perf_kwork__trace_start(void) 571590Srgrimes{ 581590Srgrimes clock_gettime(CLOCK_MONOTONIC, &ts_start); 591590Srgrimes skel->bss->enabled = 1; 601590Srgrimes} 611590Srgrimes 621590Srgrimesvoid perf_kwork__trace_finish(void) 631590Srgrimes{ 641590Srgrimes clock_gettime(CLOCK_MONOTONIC, &ts_end); 651590Srgrimes skel->bss->enabled = 0; 661590Srgrimes} 671590Srgrimes 681590Srgrimesstatic int get_work_name_from_map(struct work_key *key, char **ret_name) 691590Srgrimes{ 701590Srgrimes char name[MAX_KWORKNAME] = { 0 }; 711590Srgrimes int fd = bpf_map__fd(skel->maps.perf_kwork_names); 721590Srgrimes 731590Srgrimes *ret_name = NULL; 741590Srgrimes 751590Srgrimes if (fd < 0) { 761590Srgrimes pr_debug("Invalid names map fd\n"); 771590Srgrimes return 0; 781590Srgrimes } 791590Srgrimes 801590Srgrimes if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { 811590Srgrimes *ret_name = strdup(name); 821590Srgrimes if (*ret_name == NULL) { 831590Srgrimes pr_err("Failed to copy work name\n"); 841590Srgrimes return -1; 851590Srgrimes } 861590Srgrimes } 871590Srgrimes 881590Srgrimes return 0; 891590Srgrimes} 901590Srgrimes 911590Srgrimesstatic void irq_load_prepare(struct perf_kwork *kwork) 921590Srgrimes{ 931590Srgrimes if (kwork->report == KWORK_REPORT_RUNTIME) { 941590Srgrimes bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); 951590Srgrimes bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); 961590Srgrimes } 971590Srgrimes} 981590Srgrimes 991590Srgrimesstatic struct kwork_class_bpf kwork_irq_bpf = { 1001590Srgrimes .load_prepare = irq_load_prepare, 1011590Srgrimes .get_work_name = get_work_name_from_map, 1021590Srgrimes}; 1031590Srgrimes 1041590Srgrimesstatic void softirq_load_prepare(struct perf_kwork *kwork) 1051590Srgrimes{ 1061590Srgrimes if (kwork->report == KWORK_REPORT_RUNTIME) { 1071590Srgrimes bpf_program__set_autoload(skel->progs.report_softirq_entry, true); 1081590Srgrimes bpf_program__set_autoload(skel->progs.report_softirq_exit, true); 1091590Srgrimes } else if (kwork->report == KWORK_REPORT_LATENCY) { 1101590Srgrimes bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); 1111590Srgrimes bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); 1121590Srgrimes } 1131590Srgrimes} 1141590Srgrimes 1151590Srgrimesstatic struct kwork_class_bpf kwork_softirq_bpf = { 1161590Srgrimes .load_prepare = softirq_load_prepare, 1171590Srgrimes .get_work_name = get_work_name_from_map, 1181590Srgrimes}; 1191590Srgrimes 1201590Srgrimesstatic void workqueue_load_prepare(struct perf_kwork *kwork) 1211590Srgrimes{ 1221590Srgrimes if (kwork->report == KWORK_REPORT_RUNTIME) { 1238874Srgrimes bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); 1241590Srgrimes bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); 1251590Srgrimes } else if (kwork->report == KWORK_REPORT_LATENCY) { 1261590Srgrimes bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); 1271590Srgrimes bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); 1281590Srgrimes } 1291590Srgrimes} 1301590Srgrimes 1311590Srgrimesstatic struct kwork_class_bpf kwork_workqueue_bpf = { 1321590Srgrimes .load_prepare = workqueue_load_prepare, 1331590Srgrimes .get_work_name = get_work_name_from_map, 1341590Srgrimes}; 1351590Srgrimes 1361590Srgrimesstatic struct kwork_class_bpf * 1371590Srgrimeskwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { 1381590Srgrimes [KWORK_CLASS_IRQ] = &kwork_irq_bpf, 1391590Srgrimes [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, 1401590Srgrimes [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, 1411590Srgrimes}; 1421590Srgrimes 1431590Srgrimesstatic bool valid_kwork_class_type(enum kwork_class_type type) 1441590Srgrimes{ 1451590Srgrimes return type >= 0 && type < KWORK_CLASS_MAX ? true : false; 1461590Srgrimes} 1471590Srgrimes 1481590Srgrimesstatic int setup_filters(struct perf_kwork *kwork) 1498874Srgrimes{ 1501590Srgrimes u8 val = 1; 1511590Srgrimes int i, nr_cpus, key, fd; 1521590Srgrimes struct perf_cpu_map *map; 1531590Srgrimes 1541590Srgrimes if (kwork->cpu_list != NULL) { 1551590Srgrimes fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); 1561590Srgrimes if (fd < 0) { 1571590Srgrimes pr_debug("Invalid cpu filter fd\n"); 1581590Srgrimes return -1; 1591590Srgrimes } 1601590Srgrimes 1611590Srgrimes map = perf_cpu_map__new(kwork->cpu_list); 1621590Srgrimes if (map == NULL) { 1631590Srgrimes pr_debug("Invalid cpu_list\n"); 1641590Srgrimes return -1; 1651590Srgrimes } 1661590Srgrimes 1671590Srgrimes nr_cpus = libbpf_num_possible_cpus(); 1681590Srgrimes for (i = 0; i < perf_cpu_map__nr(map); i++) { 1691590Srgrimes struct perf_cpu cpu = perf_cpu_map__cpu(map, i); 1701590Srgrimes 1711590Srgrimes if (cpu.cpu >= nr_cpus) { 1721590Srgrimes perf_cpu_map__put(map); 1731590Srgrimes pr_err("Requested cpu %d too large\n", cpu.cpu); 1741590Srgrimes return -1; 1751590Srgrimes } 1761590Srgrimes bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); 1771590Srgrimes } 1781590Srgrimes perf_cpu_map__put(map); 1791590Srgrimes 1801590Srgrimes skel->bss->has_cpu_filter = 1; 1811590Srgrimes } 1821590Srgrimes 1831590Srgrimes if (kwork->profile_name != NULL) { 1841590Srgrimes if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { 1851590Srgrimes pr_err("Requested name filter %s too large, limit to %d\n", 1861590Srgrimes kwork->profile_name, MAX_KWORKNAME - 1); 1871590Srgrimes return -1; 1881590Srgrimes } 1891590Srgrimes 1901590Srgrimes fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); 1911590Srgrimes if (fd < 0) { 1921590Srgrimes pr_debug("Invalid name filter fd\n"); 1931590Srgrimes return -1; 1941590Srgrimes } 1951590Srgrimes 1961590Srgrimes key = 0; 1971590Srgrimes bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); 1981590Srgrimes 1991590Srgrimes skel->bss->has_name_filter = 1; 2001590Srgrimes } 2011590Srgrimes 2021590Srgrimes return 0; 2031590Srgrimes} 2041590Srgrimes 2051590Srgrimesint perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) 2061590Srgrimes{ 2071590Srgrimes struct bpf_program *prog; 2081590Srgrimes struct kwork_class *class; 2091590Srgrimes struct kwork_class_bpf *class_bpf; 2101590Srgrimes enum kwork_class_type type; 2111590Srgrimes 2121590Srgrimes skel = kwork_trace_bpf__open(); 2131590Srgrimes if (!skel) { 2141590Srgrimes pr_debug("Failed to open kwork trace skeleton\n"); 2151590Srgrimes return -1; 2161590Srgrimes } 2171590Srgrimes 2181590Srgrimes /* 2191590Srgrimes * set all progs to non-autoload, 2201590Srgrimes * then set corresponding progs according to config 2211590Srgrimes */ 2221590Srgrimes bpf_object__for_each_program(prog, skel->obj) 2231590Srgrimes bpf_program__set_autoload(prog, false); 2241590Srgrimes 2251590Srgrimes list_for_each_entry(class, &kwork->class_list, list) { 2261590Srgrimes type = class->type; 2271590Srgrimes if (!valid_kwork_class_type(type) || 2281590Srgrimes (kwork_class_bpf_supported_list[type] == NULL)) { 2291590Srgrimes pr_err("Unsupported bpf trace class %s\n", class->name); 2301590Srgrimes goto out; 2311590Srgrimes } 2321590Srgrimes 2331590Srgrimes class_bpf = kwork_class_bpf_supported_list[type]; 2341590Srgrimes class_bpf->class = class; 2351590Srgrimes 2361590Srgrimes if (class_bpf->load_prepare != NULL) 2371590Srgrimes class_bpf->load_prepare(kwork); 2381590Srgrimes } 2391590Srgrimes 2401590Srgrimes if (kwork_trace_bpf__load(skel)) { 2411590Srgrimes pr_debug("Failed to load kwork trace skeleton\n"); 2421590Srgrimes goto out; 2431590Srgrimes } 2441590Srgrimes 2451590Srgrimes if (setup_filters(kwork)) 2461590Srgrimes goto out; 2471590Srgrimes 2481590Srgrimes if (kwork_trace_bpf__attach(skel)) { 2491590Srgrimes pr_debug("Failed to attach kwork trace skeleton\n"); 2501590Srgrimes goto out; 2511590Srgrimes } 2521590Srgrimes 2531590Srgrimes return 0; 2541590Srgrimes 2551590Srgrimesout: 2561590Srgrimes kwork_trace_bpf__destroy(skel); 2571590Srgrimes return -1; 2581590Srgrimes} 2591590Srgrimes 2601590Srgrimesstatic int add_work(struct perf_kwork *kwork, 2611590Srgrimes struct work_key *key, 2621590Srgrimes struct report_data *data) 2631590Srgrimes{ 2641590Srgrimes struct kwork_work *work; 2651590Srgrimes struct kwork_class_bpf *bpf_trace; 2661590Srgrimes struct kwork_work tmp = { 2671590Srgrimes .id = key->id, 2681590Srgrimes .name = NULL, 2691590Srgrimes .cpu = key->cpu, 2701590Srgrimes }; 2711590Srgrimes enum kwork_class_type type = key->type; 2721590Srgrimes 2731590Srgrimes if (!valid_kwork_class_type(type)) { 2741590Srgrimes pr_debug("Invalid class type %d to add work\n", type); 2751590Srgrimes return -1; 2761590Srgrimes } 2771590Srgrimes 2781590Srgrimes bpf_trace = kwork_class_bpf_supported_list[type]; 2791590Srgrimes tmp.class = bpf_trace->class; 2801590Srgrimes 2811590Srgrimes if ((bpf_trace->get_work_name != NULL) && 2821590Srgrimes (bpf_trace->get_work_name(key, &tmp.name))) 2831590Srgrimes return -1; 2841590Srgrimes 2851590Srgrimes work = perf_kwork_add_work(kwork, tmp.class, &tmp); 2861590Srgrimes if (work == NULL) 2871590Srgrimes return -1; 2881590Srgrimes 2891590Srgrimes if (kwork->report == KWORK_REPORT_RUNTIME) { 2901590Srgrimes work->nr_atoms = data->nr; 2911590Srgrimes work->total_runtime = data->total_time; 2921590Srgrimes work->max_runtime = data->max_time; 2931590Srgrimes work->max_runtime_start = data->max_time_start; 2941590Srgrimes work->max_runtime_end = data->max_time_end; 2951590Srgrimes } else if (kwork->report == KWORK_REPORT_LATENCY) { 2961590Srgrimes work->nr_atoms = data->nr; 2971590Srgrimes work->total_latency = data->total_time; 2981590Srgrimes work->max_latency = data->max_time; 2991590Srgrimes work->max_latency_start = data->max_time_start; 3001590Srgrimes work->max_latency_end = data->max_time_end; 3011590Srgrimes } else { 3021590Srgrimes pr_debug("Invalid bpf report type %d\n", kwork->report); 3031590Srgrimes return -1; 3041590Srgrimes } 3051590Srgrimes 3061590Srgrimes kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; 3071590Srgrimes kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; 3081590Srgrimes 3091590Srgrimes return 0; 3101590Srgrimes} 3111590Srgrimes 3121590Srgrimesint perf_kwork__report_read_bpf(struct perf_kwork *kwork) 3131590Srgrimes{ 3141590Srgrimes struct report_data data; 3151590Srgrimes struct work_key key = { 3161590Srgrimes .type = 0, 3171590Srgrimes .cpu = 0, 3181590Srgrimes .id = 0, 3191590Srgrimes }; 3201590Srgrimes struct work_key prev = { 3211590Srgrimes .type = 0, 3221590Srgrimes .cpu = 0, 3231590Srgrimes .id = 0, 3241590Srgrimes }; 3251590Srgrimes int fd = bpf_map__fd(skel->maps.perf_kwork_report); 3261590Srgrimes 3271590Srgrimes if (fd < 0) { 3281590Srgrimes pr_debug("Invalid report fd\n"); 3291590Srgrimes return -1; 3301590Srgrimes } 3311590Srgrimes 3321590Srgrimes while (!bpf_map_get_next_key(fd, &prev, &key)) { 3331590Srgrimes if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { 3341590Srgrimes pr_debug("Failed to lookup report elem\n"); 3351590Srgrimes return -1; 3361590Srgrimes } 3371590Srgrimes 3381590Srgrimes if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0)) 3391590Srgrimes return -1; 3401590Srgrimes 3411590Srgrimes prev = key; 3421590Srgrimes } 3431590Srgrimes return 0; 3441590Srgrimes} 3451590Srgrimes 3461590Srgrimesvoid perf_kwork__report_cleanup_bpf(void) 3471590Srgrimes{ 3481590Srgrimes kwork_trace_bpf__destroy(skel); 3491590Srgrimes} 3501590Srgrimes