1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/* Copyright (c) 2021, Microsoft Corporation. */
3
4#include <linux/inetdevice.h>
5#include <linux/etherdevice.h>
6#include <linux/mm.h>
7#include <linux/bpf.h>
8#include <linux/bpf_trace.h>
9#include <net/xdp.h>
10
11#include <net/mana/mana.h>
12
13void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev)
14{
15	u16 txq_idx = skb_get_queue_mapping(skb);
16	struct netdev_queue *ndevtxq;
17	int rc;
18
19	__skb_push(skb, ETH_HLEN);
20
21	ndevtxq = netdev_get_tx_queue(ndev, txq_idx);
22	__netif_tx_lock(ndevtxq, smp_processor_id());
23
24	rc = mana_start_xmit(skb, ndev);
25
26	__netif_tx_unlock(ndevtxq);
27
28	if (dev_xmit_complete(rc))
29		return;
30
31	dev_kfree_skb_any(skb);
32	ndev->stats.tx_dropped++;
33}
34
35static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame,
36			    u16 q_idx)
37{
38	struct sk_buff *skb;
39
40	skb = xdp_build_skb_from_frame(frame, ndev);
41	if (unlikely(!skb))
42		return -ENOMEM;
43
44	skb_set_queue_mapping(skb, q_idx);
45
46	mana_xdp_tx(skb, ndev);
47
48	return 0;
49}
50
51int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
52		  u32 flags)
53{
54	struct mana_port_context *apc = netdev_priv(ndev);
55	struct mana_stats_tx *tx_stats;
56	int i, count = 0;
57	u16 q_idx;
58
59	if (unlikely(!apc->port_is_up))
60		return 0;
61
62	q_idx = smp_processor_id() % ndev->real_num_tx_queues;
63
64	for (i = 0; i < n; i++) {
65		if (mana_xdp_xmit_fm(ndev, frames[i], q_idx))
66			break;
67
68		count++;
69	}
70
71	tx_stats = &apc->tx_qp[q_idx].txq.stats;
72
73	u64_stats_update_begin(&tx_stats->syncp);
74	tx_stats->xdp_xmit += count;
75	u64_stats_update_end(&tx_stats->syncp);
76
77	return count;
78}
79
80u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
81		 struct xdp_buff *xdp, void *buf_va, uint pkt_len)
82{
83	struct mana_stats_rx *rx_stats;
84	struct bpf_prog *prog;
85	u32 act = XDP_PASS;
86
87	rcu_read_lock();
88	prog = rcu_dereference(rxq->bpf_prog);
89
90	if (!prog)
91		goto out;
92
93	xdp_init_buff(xdp, PAGE_SIZE, &rxq->xdp_rxq);
94	xdp_prepare_buff(xdp, buf_va, XDP_PACKET_HEADROOM, pkt_len, false);
95
96	act = bpf_prog_run_xdp(prog, xdp);
97
98	rx_stats = &rxq->stats;
99
100	switch (act) {
101	case XDP_PASS:
102	case XDP_TX:
103	case XDP_DROP:
104		break;
105
106	case XDP_REDIRECT:
107		rxq->xdp_rc = xdp_do_redirect(ndev, xdp, prog);
108		if (!rxq->xdp_rc) {
109			rxq->xdp_flush = true;
110
111			u64_stats_update_begin(&rx_stats->syncp);
112			rx_stats->packets++;
113			rx_stats->bytes += pkt_len;
114			rx_stats->xdp_redirect++;
115			u64_stats_update_end(&rx_stats->syncp);
116
117			break;
118		}
119
120		fallthrough;
121
122	case XDP_ABORTED:
123		trace_xdp_exception(ndev, prog, act);
124		break;
125
126	default:
127		bpf_warn_invalid_xdp_action(ndev, prog, act);
128	}
129
130out:
131	rcu_read_unlock();
132
133	return act;
134}
135
136struct bpf_prog *mana_xdp_get(struct mana_port_context *apc)
137{
138	ASSERT_RTNL();
139
140	return apc->bpf_prog;
141}
142
143static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc)
144{
145	return rtnl_dereference(apc->rxqs[0]->bpf_prog);
146}
147
148/* Set xdp program on channels */
149void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog)
150{
151	struct bpf_prog *old_prog = mana_chn_xdp_get(apc);
152	unsigned int num_queues = apc->num_queues;
153	int i;
154
155	ASSERT_RTNL();
156
157	if (old_prog == prog)
158		return;
159
160	if (prog)
161		bpf_prog_add(prog, num_queues);
162
163	for (i = 0; i < num_queues; i++)
164		rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog);
165
166	if (old_prog)
167		for (i = 0; i < num_queues; i++)
168			bpf_prog_put(old_prog);
169}
170
171static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
172			struct netlink_ext_ack *extack)
173{
174	struct mana_port_context *apc = netdev_priv(ndev);
175	struct bpf_prog *old_prog;
176	struct gdma_context *gc;
177
178	gc = apc->ac->gdma_dev->gdma_context;
179
180	old_prog = mana_xdp_get(apc);
181
182	if (!old_prog && !prog)
183		return 0;
184
185	if (prog && ndev->mtu > MANA_XDP_MTU_MAX) {
186		netdev_err(ndev, "XDP: mtu:%u too large, mtu_max:%lu\n",
187			   ndev->mtu, MANA_XDP_MTU_MAX);
188		NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");
189
190		return -EOPNOTSUPP;
191	}
192
193	/* One refcnt of the prog is hold by the caller already, so
194	 * don't increase refcnt for this one.
195	 */
196	apc->bpf_prog = prog;
197
198	if (old_prog)
199		bpf_prog_put(old_prog);
200
201	if (apc->port_is_up)
202		mana_chn_setxdp(apc, prog);
203
204	if (prog)
205		ndev->max_mtu = MANA_XDP_MTU_MAX;
206	else
207		ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
208
209	return 0;
210}
211
212int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
213{
214	struct netlink_ext_ack *extack = bpf->extack;
215	int ret;
216
217	switch (bpf->command) {
218	case XDP_SETUP_PROG:
219		return mana_xdp_set(ndev, bpf->prog, extack);
220
221	default:
222		return -EOPNOTSUPP;
223	}
224
225	return ret;
226}
227