1// SPDX-License-Identifier: GPL-2.0
2/*  GPLv2, Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc. */
3#include "xdp_sample.bpf.h"
4
5#include <bpf/bpf_tracing.h>
6#include <bpf/bpf_core_read.h>
7#include <bpf/bpf_helpers.h>
8
9array_map rx_cnt SEC(".maps");
10array_map redir_err_cnt SEC(".maps");
11array_map cpumap_enqueue_cnt SEC(".maps");
12array_map cpumap_kthread_cnt SEC(".maps");
13array_map exception_cnt SEC(".maps");
14array_map devmap_xmit_cnt SEC(".maps");
15
16struct {
17	__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
18	__uint(max_entries, 32 * 32);
19	__type(key, u64);
20	__type(value, struct datarec);
21} devmap_xmit_cnt_multi SEC(".maps");
22
23const volatile int nr_cpus = 0;
24
25/* These can be set before loading so that redundant comparisons can be DCE'd by
26 * the verifier, and only actual matches are tried after loading tp_btf program.
27 * This allows sample to filter tracepoint stats based on net_device.
28 */
29const volatile int from_match[32] = {};
30const volatile int to_match[32] = {};
31
32int cpumap_map_id = 0;
33
34/* Find if b is part of set a, but if a is empty set then evaluate to true */
35#define IN_SET(a, b)                                                 \
36	({                                                           \
37		bool __res = !(a)[0];                                \
38		for (int i = 0; i < ARRAY_SIZE(a) && (a)[i]; i++) { \
39			__res = (a)[i] == (b);                       \
40			if (__res)                                   \
41				break;                               \
42		}                                                    \
43		__res;                                               \
44	})
45
46static __always_inline __u32 xdp_get_err_key(int err)
47{
48	switch (err) {
49	case 0:
50		return 0;
51	case -EINVAL:
52		return 2;
53	case -ENETDOWN:
54		return 3;
55	case -EMSGSIZE:
56		return 4;
57	case -EOPNOTSUPP:
58		return 5;
59	case -ENOSPC:
60		return 6;
61	default:
62		return 1;
63	}
64}
65
66static __always_inline int xdp_redirect_collect_stat(int from, int err)
67{
68	u32 cpu = bpf_get_smp_processor_id();
69	u32 key = XDP_REDIRECT_ERROR;
70	struct datarec *rec;
71	u32 idx;
72
73	if (!IN_SET(from_match, from))
74		return 0;
75
76	key = xdp_get_err_key(err);
77
78	idx = key * nr_cpus + cpu;
79	rec = bpf_map_lookup_elem(&redir_err_cnt, &idx);
80	if (!rec)
81		return 0;
82	if (key)
83		NO_TEAR_INC(rec->dropped);
84	else
85		NO_TEAR_INC(rec->processed);
86	return 0; /* Indicate event was filtered (no further processing)*/
87	/*
88	 * Returning 1 here would allow e.g. a perf-record tracepoint
89	 * to see and record these events, but it doesn't work well
90	 * in-practice as stopping perf-record also unload this
91	 * bpf_prog.  Plus, there is additional overhead of doing so.
92	 */
93}
94
95SEC("tp_btf/xdp_redirect_err")
96int BPF_PROG(tp_xdp_redirect_err, const struct net_device *dev,
97	     const struct bpf_prog *xdp, const void *tgt, int err,
98	     const struct bpf_map *map, u32 index)
99{
100	return xdp_redirect_collect_stat(dev->ifindex, err);
101}
102
103SEC("tp_btf/xdp_redirect_map_err")
104int BPF_PROG(tp_xdp_redirect_map_err, const struct net_device *dev,
105	     const struct bpf_prog *xdp, const void *tgt, int err,
106	     const struct bpf_map *map, u32 index)
107{
108	return xdp_redirect_collect_stat(dev->ifindex, err);
109}
110
111SEC("tp_btf/xdp_redirect")
112int BPF_PROG(tp_xdp_redirect, const struct net_device *dev,
113	     const struct bpf_prog *xdp, const void *tgt, int err,
114	     const struct bpf_map *map, u32 index)
115{
116	return xdp_redirect_collect_stat(dev->ifindex, err);
117}
118
119SEC("tp_btf/xdp_redirect_map")
120int BPF_PROG(tp_xdp_redirect_map, const struct net_device *dev,
121	     const struct bpf_prog *xdp, const void *tgt, int err,
122	     const struct bpf_map *map, u32 index)
123{
124	return xdp_redirect_collect_stat(dev->ifindex, err);
125}
126
127SEC("tp_btf/xdp_cpumap_enqueue")
128int BPF_PROG(tp_xdp_cpumap_enqueue, int map_id, unsigned int processed,
129	     unsigned int drops, int to_cpu)
130{
131	u32 cpu = bpf_get_smp_processor_id();
132	struct datarec *rec;
133	u32 idx;
134
135	if (cpumap_map_id && cpumap_map_id != map_id)
136		return 0;
137
138	idx = to_cpu * nr_cpus + cpu;
139	rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &idx);
140	if (!rec)
141		return 0;
142	NO_TEAR_ADD(rec->processed, processed);
143	NO_TEAR_ADD(rec->dropped, drops);
144	/* Record bulk events, then userspace can calc average bulk size */
145	if (processed > 0)
146		NO_TEAR_INC(rec->issue);
147	/* Inception: It's possible to detect overload situations, via
148	 * this tracepoint.  This can be used for creating a feedback
149	 * loop to XDP, which can take appropriate actions to mitigate
150	 * this overload situation.
151	 */
152	return 0;
153}
154
155SEC("tp_btf/xdp_cpumap_kthread")
156int BPF_PROG(tp_xdp_cpumap_kthread, int map_id, unsigned int processed,
157	     unsigned int drops, int sched, struct xdp_cpumap_stats *xdp_stats)
158{
159	struct datarec *rec;
160	u32 cpu;
161
162	if (cpumap_map_id && cpumap_map_id != map_id)
163		return 0;
164
165	cpu = bpf_get_smp_processor_id();
166	rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &cpu);
167	if (!rec)
168		return 0;
169	NO_TEAR_ADD(rec->processed, processed);
170	NO_TEAR_ADD(rec->dropped, drops);
171	NO_TEAR_ADD(rec->xdp_pass, xdp_stats->pass);
172	NO_TEAR_ADD(rec->xdp_drop, xdp_stats->drop);
173	NO_TEAR_ADD(rec->xdp_redirect, xdp_stats->redirect);
174	/* Count times kthread yielded CPU via schedule call */
175	if (sched)
176		NO_TEAR_INC(rec->issue);
177	return 0;
178}
179
180SEC("tp_btf/xdp_exception")
181int BPF_PROG(tp_xdp_exception, const struct net_device *dev,
182	     const struct bpf_prog *xdp, u32 act)
183{
184	u32 cpu = bpf_get_smp_processor_id();
185	struct datarec *rec;
186	u32 key = act, idx;
187
188	if (!IN_SET(from_match, dev->ifindex))
189		return 0;
190	if (!IN_SET(to_match, dev->ifindex))
191		return 0;
192
193	if (key > XDP_REDIRECT)
194		key = XDP_REDIRECT + 1;
195
196	idx = key * nr_cpus + cpu;
197	rec = bpf_map_lookup_elem(&exception_cnt, &idx);
198	if (!rec)
199		return 0;
200	NO_TEAR_INC(rec->dropped);
201
202	return 0;
203}
204
205SEC("tp_btf/xdp_devmap_xmit")
206int BPF_PROG(tp_xdp_devmap_xmit, const struct net_device *from_dev,
207	     const struct net_device *to_dev, int sent, int drops, int err)
208{
209	struct datarec *rec;
210	int idx_in, idx_out;
211	u32 cpu;
212
213	idx_in = from_dev->ifindex;
214	idx_out = to_dev->ifindex;
215
216	if (!IN_SET(from_match, idx_in))
217		return 0;
218	if (!IN_SET(to_match, idx_out))
219		return 0;
220
221	cpu = bpf_get_smp_processor_id();
222	rec = bpf_map_lookup_elem(&devmap_xmit_cnt, &cpu);
223	if (!rec)
224		return 0;
225	NO_TEAR_ADD(rec->processed, sent);
226	NO_TEAR_ADD(rec->dropped, drops);
227	/* Record bulk events, then userspace can calc average bulk size */
228	NO_TEAR_INC(rec->info);
229	/* Record error cases, where no frame were sent */
230	/* Catch API error of drv ndo_xdp_xmit sent more than count */
231	if (err || drops < 0)
232		NO_TEAR_INC(rec->issue);
233	return 0;
234}
235
236SEC("tp_btf/xdp_devmap_xmit")
237int BPF_PROG(tp_xdp_devmap_xmit_multi, const struct net_device *from_dev,
238	     const struct net_device *to_dev, int sent, int drops, int err)
239{
240	struct datarec empty = {};
241	struct datarec *rec;
242	int idx_in, idx_out;
243	u64 idx;
244
245	idx_in = from_dev->ifindex;
246	idx_out = to_dev->ifindex;
247	idx = idx_in;
248	idx = idx << 32 | idx_out;
249
250	if (!IN_SET(from_match, idx_in))
251		return 0;
252	if (!IN_SET(to_match, idx_out))
253		return 0;
254
255	bpf_map_update_elem(&devmap_xmit_cnt_multi, &idx, &empty, BPF_NOEXIST);
256	rec = bpf_map_lookup_elem(&devmap_xmit_cnt_multi, &idx);
257	if (!rec)
258		return 0;
259
260	NO_TEAR_ADD(rec->processed, sent);
261	NO_TEAR_ADD(rec->dropped, drops);
262	NO_TEAR_INC(rec->info);
263	if (err || drops < 0)
264		NO_TEAR_INC(rec->issue);
265	return 0;
266}
267