1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2023 Isovalent */
3
4#include <linux/bpf.h>
5#include <linux/bpf_mprog.h>
6#include <linux/netdevice.h>
7
8#include <net/tcx.h>
9
10int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
11{
12	bool created, ingress = attr->attach_type == BPF_TCX_INGRESS;
13	struct net *net = current->nsproxy->net_ns;
14	struct bpf_mprog_entry *entry, *entry_new;
15	struct bpf_prog *replace_prog = NULL;
16	struct net_device *dev;
17	int ret;
18
19	rtnl_lock();
20	dev = __dev_get_by_index(net, attr->target_ifindex);
21	if (!dev) {
22		ret = -ENODEV;
23		goto out;
24	}
25	if (attr->attach_flags & BPF_F_REPLACE) {
26		replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
27						 prog->type);
28		if (IS_ERR(replace_prog)) {
29			ret = PTR_ERR(replace_prog);
30			replace_prog = NULL;
31			goto out;
32		}
33	}
34	entry = tcx_entry_fetch_or_create(dev, ingress, &created);
35	if (!entry) {
36		ret = -ENOMEM;
37		goto out;
38	}
39	ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
40			       attr->attach_flags, attr->relative_fd,
41			       attr->expected_revision);
42	if (!ret) {
43		if (entry != entry_new) {
44			tcx_entry_update(dev, entry_new, ingress);
45			tcx_entry_sync();
46			tcx_skeys_inc(ingress);
47		}
48		bpf_mprog_commit(entry);
49	} else if (created) {
50		tcx_entry_free(entry);
51	}
52out:
53	if (replace_prog)
54		bpf_prog_put(replace_prog);
55	rtnl_unlock();
56	return ret;
57}
58
59int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
60{
61	bool ingress = attr->attach_type == BPF_TCX_INGRESS;
62	struct net *net = current->nsproxy->net_ns;
63	struct bpf_mprog_entry *entry, *entry_new;
64	struct net_device *dev;
65	int ret;
66
67	rtnl_lock();
68	dev = __dev_get_by_index(net, attr->target_ifindex);
69	if (!dev) {
70		ret = -ENODEV;
71		goto out;
72	}
73	entry = tcx_entry_fetch(dev, ingress);
74	if (!entry) {
75		ret = -ENOENT;
76		goto out;
77	}
78	ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
79			       attr->relative_fd, attr->expected_revision);
80	if (!ret) {
81		if (!tcx_entry_is_active(entry_new))
82			entry_new = NULL;
83		tcx_entry_update(dev, entry_new, ingress);
84		tcx_entry_sync();
85		tcx_skeys_dec(ingress);
86		bpf_mprog_commit(entry);
87		if (!entry_new)
88			tcx_entry_free(entry);
89	}
90out:
91	rtnl_unlock();
92	return ret;
93}
94
95void tcx_uninstall(struct net_device *dev, bool ingress)
96{
97	struct bpf_mprog_entry *entry, *entry_new = NULL;
98	struct bpf_tuple tuple = {};
99	struct bpf_mprog_fp *fp;
100	struct bpf_mprog_cp *cp;
101	bool active;
102
103	entry = tcx_entry_fetch(dev, ingress);
104	if (!entry)
105		return;
106	active = tcx_entry(entry)->miniq_active;
107	if (active)
108		bpf_mprog_clear_all(entry, &entry_new);
109	tcx_entry_update(dev, entry_new, ingress);
110	tcx_entry_sync();
111	bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
112		if (tuple.link)
113			tcx_link(tuple.link)->dev = NULL;
114		else
115			bpf_prog_put(tuple.prog);
116		tcx_skeys_dec(ingress);
117	}
118	if (!active)
119		tcx_entry_free(entry);
120}
121
122int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
123{
124	bool ingress = attr->query.attach_type == BPF_TCX_INGRESS;
125	struct net *net = current->nsproxy->net_ns;
126	struct net_device *dev;
127	int ret;
128
129	rtnl_lock();
130	dev = __dev_get_by_index(net, attr->query.target_ifindex);
131	if (!dev) {
132		ret = -ENODEV;
133		goto out;
134	}
135	ret = bpf_mprog_query(attr, uattr, tcx_entry_fetch(dev, ingress));
136out:
137	rtnl_unlock();
138	return ret;
139}
140
141static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd,
142				u64 revision)
143{
144	struct tcx_link *tcx = tcx_link(link);
145	bool created, ingress = tcx->location == BPF_TCX_INGRESS;
146	struct bpf_mprog_entry *entry, *entry_new;
147	struct net_device *dev = tcx->dev;
148	int ret;
149
150	ASSERT_RTNL();
151	entry = tcx_entry_fetch_or_create(dev, ingress, &created);
152	if (!entry)
153		return -ENOMEM;
154	ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
155			       id_or_fd, revision);
156	if (!ret) {
157		if (entry != entry_new) {
158			tcx_entry_update(dev, entry_new, ingress);
159			tcx_entry_sync();
160			tcx_skeys_inc(ingress);
161		}
162		bpf_mprog_commit(entry);
163	} else if (created) {
164		tcx_entry_free(entry);
165	}
166	return ret;
167}
168
169static void tcx_link_release(struct bpf_link *link)
170{
171	struct tcx_link *tcx = tcx_link(link);
172	bool ingress = tcx->location == BPF_TCX_INGRESS;
173	struct bpf_mprog_entry *entry, *entry_new;
174	struct net_device *dev;
175	int ret = 0;
176
177	rtnl_lock();
178	dev = tcx->dev;
179	if (!dev)
180		goto out;
181	entry = tcx_entry_fetch(dev, ingress);
182	if (!entry) {
183		ret = -ENOENT;
184		goto out;
185	}
186	ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
187	if (!ret) {
188		if (!tcx_entry_is_active(entry_new))
189			entry_new = NULL;
190		tcx_entry_update(dev, entry_new, ingress);
191		tcx_entry_sync();
192		tcx_skeys_dec(ingress);
193		bpf_mprog_commit(entry);
194		if (!entry_new)
195			tcx_entry_free(entry);
196		tcx->dev = NULL;
197	}
198out:
199	WARN_ON_ONCE(ret);
200	rtnl_unlock();
201}
202
203static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog,
204			   struct bpf_prog *oprog)
205{
206	struct tcx_link *tcx = tcx_link(link);
207	bool ingress = tcx->location == BPF_TCX_INGRESS;
208	struct bpf_mprog_entry *entry, *entry_new;
209	struct net_device *dev;
210	int ret = 0;
211
212	rtnl_lock();
213	dev = tcx->dev;
214	if (!dev) {
215		ret = -ENOLINK;
216		goto out;
217	}
218	if (oprog && link->prog != oprog) {
219		ret = -EPERM;
220		goto out;
221	}
222	oprog = link->prog;
223	if (oprog == nprog) {
224		bpf_prog_put(nprog);
225		goto out;
226	}
227	entry = tcx_entry_fetch(dev, ingress);
228	if (!entry) {
229		ret = -ENOENT;
230		goto out;
231	}
232	ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
233			       BPF_F_REPLACE | BPF_F_ID,
234			       link->prog->aux->id, 0);
235	if (!ret) {
236		WARN_ON_ONCE(entry != entry_new);
237		oprog = xchg(&link->prog, nprog);
238		bpf_prog_put(oprog);
239		bpf_mprog_commit(entry);
240	}
241out:
242	rtnl_unlock();
243	return ret;
244}
245
246static void tcx_link_dealloc(struct bpf_link *link)
247{
248	kfree(tcx_link(link));
249}
250
251static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
252{
253	const struct tcx_link *tcx = tcx_link(link);
254	u32 ifindex = 0;
255
256	rtnl_lock();
257	if (tcx->dev)
258		ifindex = tcx->dev->ifindex;
259	rtnl_unlock();
260
261	seq_printf(seq, "ifindex:\t%u\n", ifindex);
262	seq_printf(seq, "attach_type:\t%u (%s)\n",
263		   tcx->location,
264		   tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress");
265}
266
267static int tcx_link_fill_info(const struct bpf_link *link,
268			      struct bpf_link_info *info)
269{
270	const struct tcx_link *tcx = tcx_link(link);
271	u32 ifindex = 0;
272
273	rtnl_lock();
274	if (tcx->dev)
275		ifindex = tcx->dev->ifindex;
276	rtnl_unlock();
277
278	info->tcx.ifindex = ifindex;
279	info->tcx.attach_type = tcx->location;
280	return 0;
281}
282
283static int tcx_link_detach(struct bpf_link *link)
284{
285	tcx_link_release(link);
286	return 0;
287}
288
289static const struct bpf_link_ops tcx_link_lops = {
290	.release	= tcx_link_release,
291	.detach		= tcx_link_detach,
292	.dealloc	= tcx_link_dealloc,
293	.update_prog	= tcx_link_update,
294	.show_fdinfo	= tcx_link_fdinfo,
295	.fill_link_info	= tcx_link_fill_info,
296};
297
298static int tcx_link_init(struct tcx_link *tcx,
299			 struct bpf_link_primer *link_primer,
300			 const union bpf_attr *attr,
301			 struct net_device *dev,
302			 struct bpf_prog *prog)
303{
304	bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog);
305	tcx->location = attr->link_create.attach_type;
306	tcx->dev = dev;
307	return bpf_link_prime(&tcx->link, link_primer);
308}
309
310int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
311{
312	struct net *net = current->nsproxy->net_ns;
313	struct bpf_link_primer link_primer;
314	struct net_device *dev;
315	struct tcx_link *tcx;
316	int ret;
317
318	rtnl_lock();
319	dev = __dev_get_by_index(net, attr->link_create.target_ifindex);
320	if (!dev) {
321		ret = -ENODEV;
322		goto out;
323	}
324	tcx = kzalloc(sizeof(*tcx), GFP_USER);
325	if (!tcx) {
326		ret = -ENOMEM;
327		goto out;
328	}
329	ret = tcx_link_init(tcx, &link_primer, attr, dev, prog);
330	if (ret) {
331		kfree(tcx);
332		goto out;
333	}
334	ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags,
335				   attr->link_create.tcx.relative_fd,
336				   attr->link_create.tcx.expected_revision);
337	if (ret) {
338		tcx->dev = NULL;
339		bpf_link_cleanup(&link_primer);
340		goto out;
341	}
342	ret = bpf_link_settle(&link_primer);
343out:
344	rtnl_unlock();
345	return ret;
346}
347