1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2// Copyright (c) 2022, Huawei
3
4#include "vmlinux.h"
5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>
7
8#define KWORK_COUNT 100
9#define MAX_KWORKNAME 128
10
11/*
12 * This should be in sync with "util/kwork.h"
13 */
14enum kwork_class_type {
15	KWORK_CLASS_IRQ,
16	KWORK_CLASS_SOFTIRQ,
17	KWORK_CLASS_WORKQUEUE,
18	KWORK_CLASS_MAX,
19};
20
21struct work_key {
22	__u32 type;
23	__u32 cpu;
24	__u64 id;
25};
26
27struct report_data {
28	__u64 nr;
29	__u64 total_time;
30	__u64 max_time;
31	__u64 max_time_start;
32	__u64 max_time_end;
33};
34
35struct {
36	__uint(type, BPF_MAP_TYPE_HASH);
37	__uint(key_size, sizeof(struct work_key));
38	__uint(value_size, MAX_KWORKNAME);
39	__uint(max_entries, KWORK_COUNT);
40} perf_kwork_names SEC(".maps");
41
42struct {
43	__uint(type, BPF_MAP_TYPE_HASH);
44	__uint(key_size, sizeof(struct work_key));
45	__uint(value_size, sizeof(__u64));
46	__uint(max_entries, KWORK_COUNT);
47} perf_kwork_time SEC(".maps");
48
49struct {
50	__uint(type, BPF_MAP_TYPE_HASH);
51	__uint(key_size, sizeof(struct work_key));
52	__uint(value_size, sizeof(struct report_data));
53	__uint(max_entries, KWORK_COUNT);
54} perf_kwork_report SEC(".maps");
55
56struct {
57	__uint(type, BPF_MAP_TYPE_HASH);
58	__uint(key_size, sizeof(__u32));
59	__uint(value_size, sizeof(__u8));
60	__uint(max_entries, 1);
61} perf_kwork_cpu_filter SEC(".maps");
62
63struct {
64	__uint(type, BPF_MAP_TYPE_ARRAY);
65	__uint(key_size, sizeof(__u32));
66	__uint(value_size, MAX_KWORKNAME);
67	__uint(max_entries, 1);
68} perf_kwork_name_filter SEC(".maps");
69
70int enabled = 0;
71int has_cpu_filter = 0;
72int has_name_filter = 0;
73
74static __always_inline int local_strncmp(const char *s1,
75					 unsigned int sz, const char *s2)
76{
77	int ret = 0;
78	unsigned int i;
79
80	for (i = 0; i < sz; i++) {
81		ret = (unsigned char)s1[i] - (unsigned char)s2[i];
82		if (ret || !s1[i] || !s2[i])
83			break;
84	}
85
86	return ret;
87}
88
89static __always_inline int trace_event_match(struct work_key *key, char *name)
90{
91	__u8 *cpu_val;
92	char *name_val;
93	__u32 zero = 0;
94	__u32 cpu = bpf_get_smp_processor_id();
95
96	if (!enabled)
97		return 0;
98
99	if (has_cpu_filter) {
100		cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu);
101		if (!cpu_val)
102			return 0;
103	}
104
105	if (has_name_filter && (name != NULL)) {
106		name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero);
107		if (name_val &&
108		    (local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) {
109			return 0;
110		}
111	}
112
113	return 1;
114}
115
116static __always_inline void do_update_time(void *map, struct work_key *key,
117					   __u64 time_start, __u64 time_end)
118{
119	struct report_data zero, *data;
120	__s64 delta = time_end - time_start;
121
122	if (delta < 0)
123		return;
124
125	data = bpf_map_lookup_elem(map, key);
126	if (!data) {
127		__builtin_memset(&zero, 0, sizeof(zero));
128		bpf_map_update_elem(map, key, &zero, BPF_NOEXIST);
129		data = bpf_map_lookup_elem(map, key);
130		if (!data)
131			return;
132	}
133
134	if ((delta > data->max_time) ||
135	    (data->max_time == 0)) {
136		data->max_time       = delta;
137		data->max_time_start = time_start;
138		data->max_time_end   = time_end;
139	}
140
141	data->total_time += delta;
142	data->nr++;
143}
144
145static __always_inline void do_update_timestart(void *map, struct work_key *key)
146{
147	__u64 ts = bpf_ktime_get_ns();
148
149	bpf_map_update_elem(map, key, &ts, BPF_ANY);
150}
151
152static __always_inline void do_update_timeend(void *report_map, void *time_map,
153					      struct work_key *key)
154{
155	__u64 *time = bpf_map_lookup_elem(time_map, key);
156
157	if (time) {
158		bpf_map_delete_elem(time_map, key);
159		do_update_time(report_map, key, *time, bpf_ktime_get_ns());
160	}
161}
162
163static __always_inline void do_update_name(void *map,
164					   struct work_key *key, char *name)
165{
166	if (!bpf_map_lookup_elem(map, key))
167		bpf_map_update_elem(map, key, name, BPF_ANY);
168}
169
170static __always_inline int update_timestart(void *map, struct work_key *key)
171{
172	if (!trace_event_match(key, NULL))
173		return 0;
174
175	do_update_timestart(map, key);
176	return 0;
177}
178
179static __always_inline int update_timestart_and_name(void *time_map,
180						     void *names_map,
181						     struct work_key *key,
182						     char *name)
183{
184	if (!trace_event_match(key, name))
185		return 0;
186
187	do_update_timestart(time_map, key);
188	do_update_name(names_map, key, name);
189
190	return 0;
191}
192
193static __always_inline int update_timeend(void *report_map,
194					  void *time_map, struct work_key *key)
195{
196	if (!trace_event_match(key, NULL))
197		return 0;
198
199	do_update_timeend(report_map, time_map, key);
200
201	return 0;
202}
203
204static __always_inline int update_timeend_and_name(void *report_map,
205						   void *time_map,
206						   void *names_map,
207						   struct work_key *key,
208						   char *name)
209{
210	if (!trace_event_match(key, name))
211		return 0;
212
213	do_update_timeend(report_map, time_map, key);
214	do_update_name(names_map, key, name);
215
216	return 0;
217}
218
219SEC("tracepoint/irq/irq_handler_entry")
220int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx)
221{
222	char name[MAX_KWORKNAME];
223	struct work_key key = {
224		.type = KWORK_CLASS_IRQ,
225		.cpu  = bpf_get_smp_processor_id(),
226		.id   = (__u64)ctx->irq,
227	};
228	void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff);
229
230	bpf_probe_read_kernel_str(name, sizeof(name), name_addr);
231
232	return update_timestart_and_name(&perf_kwork_time,
233					 &perf_kwork_names, &key, name);
234}
235
236SEC("tracepoint/irq/irq_handler_exit")
237int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx)
238{
239	struct work_key key = {
240		.type = KWORK_CLASS_IRQ,
241		.cpu  = bpf_get_smp_processor_id(),
242		.id   = (__u64)ctx->irq,
243	};
244
245	return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
246}
247
248static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = {
249	{ "HI"       },
250	{ "TIMER"    },
251	{ "NET_TX"   },
252	{ "NET_RX"   },
253	{ "BLOCK"    },
254	{ "IRQ_POLL" },
255	{ "TASKLET"  },
256	{ "SCHED"    },
257	{ "HRTIMER"  },
258	{ "RCU"      },
259};
260
261SEC("tracepoint/irq/softirq_entry")
262int report_softirq_entry(struct trace_event_raw_softirq *ctx)
263{
264	unsigned int vec = ctx->vec;
265	struct work_key key = {
266		.type = KWORK_CLASS_SOFTIRQ,
267		.cpu  = bpf_get_smp_processor_id(),
268		.id   = (__u64)vec,
269	};
270
271	if (vec < NR_SOFTIRQS) {
272		return update_timestart_and_name(&perf_kwork_time,
273						 &perf_kwork_names, &key,
274						 softirq_name_list[vec]);
275	}
276
277	return 0;
278}
279
280SEC("tracepoint/irq/softirq_exit")
281int report_softirq_exit(struct trace_event_raw_softirq *ctx)
282{
283	struct work_key key = {
284		.type = KWORK_CLASS_SOFTIRQ,
285		.cpu  = bpf_get_smp_processor_id(),
286		.id   = (__u64)ctx->vec,
287	};
288
289	return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
290}
291
292SEC("tracepoint/irq/softirq_raise")
293int latency_softirq_raise(struct trace_event_raw_softirq *ctx)
294{
295	unsigned int vec = ctx->vec;
296	struct work_key key = {
297		.type = KWORK_CLASS_SOFTIRQ,
298		.cpu  = bpf_get_smp_processor_id(),
299		.id   = (__u64)vec,
300	};
301
302	if (vec < NR_SOFTIRQS) {
303		return update_timestart_and_name(&perf_kwork_time,
304						 &perf_kwork_names, &key,
305						 softirq_name_list[vec]);
306	}
307
308	return 0;
309}
310
311SEC("tracepoint/irq/softirq_entry")
312int latency_softirq_entry(struct trace_event_raw_softirq *ctx)
313{
314	struct work_key key = {
315		.type = KWORK_CLASS_SOFTIRQ,
316		.cpu  = bpf_get_smp_processor_id(),
317		.id   = (__u64)ctx->vec,
318	};
319
320	return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
321}
322
323SEC("tracepoint/workqueue/workqueue_execute_start")
324int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
325{
326	struct work_key key = {
327		.type = KWORK_CLASS_WORKQUEUE,
328		.cpu  = bpf_get_smp_processor_id(),
329		.id   = (__u64)ctx->work,
330	};
331
332	return update_timestart(&perf_kwork_time, &key);
333}
334
335SEC("tracepoint/workqueue/workqueue_execute_end")
336int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx)
337{
338	char name[MAX_KWORKNAME];
339	struct work_key key = {
340		.type = KWORK_CLASS_WORKQUEUE,
341		.cpu  = bpf_get_smp_processor_id(),
342		.id   = (__u64)ctx->work,
343	};
344	unsigned long long func_addr = (unsigned long long)ctx->function;
345
346	__builtin_memset(name, 0, sizeof(name));
347	bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
348
349	return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
350				       &perf_kwork_names, &key, name);
351}
352
353SEC("tracepoint/workqueue/workqueue_activate_work")
354int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx)
355{
356	struct work_key key = {
357		.type = KWORK_CLASS_WORKQUEUE,
358		.cpu  = bpf_get_smp_processor_id(),
359		.id   = (__u64)ctx->work,
360	};
361
362	return update_timestart(&perf_kwork_time, &key);
363}
364
365SEC("tracepoint/workqueue/workqueue_execute_start")
366int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
367{
368	char name[MAX_KWORKNAME];
369	struct work_key key = {
370		.type = KWORK_CLASS_WORKQUEUE,
371		.cpu  = bpf_get_smp_processor_id(),
372		.id   = (__u64)ctx->work,
373	};
374	unsigned long long func_addr = (unsigned long long)ctx->function;
375
376	__builtin_memset(name, 0, sizeof(name));
377	bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
378
379	return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
380				       &perf_kwork_names, &key, name);
381}
382
383char LICENSE[] SEC("license") = "Dual BSD/GPL";
384