1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2019, Microsoft Corporation.
3 *
4 * Author:
5 *   Haiyang Zhang <haiyangz@microsoft.com>
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/netdevice.h>
11#include <linux/etherdevice.h>
12#include <linux/ethtool.h>
13#include <linux/netpoll.h>
14#include <linux/bpf.h>
15#include <linux/bpf_trace.h>
16#include <linux/kernel.h>
17#include <net/xdp.h>
18
19#include <linux/mutex.h>
20#include <linux/rtnetlink.h>
21
22#include "hyperv_net.h"
23
24u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan,
25		   struct xdp_buff *xdp)
26{
27	struct netvsc_stats_rx *rx_stats = &nvchan->rx_stats;
28	void *data = nvchan->rsc.data[0];
29	u32 len = nvchan->rsc.len[0];
30	struct page *page = NULL;
31	struct bpf_prog *prog;
32	u32 act = XDP_PASS;
33	bool drop = true;
34
35	xdp->data_hard_start = NULL;
36
37	rcu_read_lock();
38	prog = rcu_dereference(nvchan->bpf_prog);
39
40	if (!prog)
41		goto out;
42
43	/* Ensure that the below memcpy() won't overflow the page buffer. */
44	if (len > ndev->mtu + ETH_HLEN) {
45		act = XDP_DROP;
46		goto out;
47	}
48
49	/* allocate page buffer for data */
50	page = alloc_page(GFP_ATOMIC);
51	if (!page) {
52		act = XDP_DROP;
53		goto out;
54	}
55
56	xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq);
57	xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false);
58
59	memcpy(xdp->data, data, len);
60
61	act = bpf_prog_run_xdp(prog, xdp);
62
63	switch (act) {
64	case XDP_PASS:
65	case XDP_TX:
66		drop = false;
67		break;
68
69	case XDP_DROP:
70		break;
71
72	case XDP_REDIRECT:
73		if (!xdp_do_redirect(ndev, xdp, prog)) {
74			nvchan->xdp_flush = true;
75			drop = false;
76
77			u64_stats_update_begin(&rx_stats->syncp);
78
79			rx_stats->xdp_redirect++;
80			rx_stats->packets++;
81			rx_stats->bytes += nvchan->rsc.pktlen;
82
83			u64_stats_update_end(&rx_stats->syncp);
84
85			break;
86		} else {
87			u64_stats_update_begin(&rx_stats->syncp);
88			rx_stats->xdp_drop++;
89			u64_stats_update_end(&rx_stats->syncp);
90		}
91
92		fallthrough;
93
94	case XDP_ABORTED:
95		trace_xdp_exception(ndev, prog, act);
96		break;
97
98	default:
99		bpf_warn_invalid_xdp_action(ndev, prog, act);
100	}
101
102out:
103	rcu_read_unlock();
104
105	if (page && drop) {
106		__free_page(page);
107		xdp->data_hard_start = NULL;
108	}
109
110	return act;
111}
112
113unsigned int netvsc_xdp_fraglen(unsigned int len)
114{
115	return SKB_DATA_ALIGN(len) +
116	       SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
117}
118
119struct bpf_prog *netvsc_xdp_get(struct netvsc_device *nvdev)
120{
121	return rtnl_dereference(nvdev->chan_table[0].bpf_prog);
122}
123
124int netvsc_xdp_set(struct net_device *dev, struct bpf_prog *prog,
125		   struct netlink_ext_ack *extack,
126		   struct netvsc_device *nvdev)
127{
128	struct bpf_prog *old_prog;
129	int buf_max, i;
130
131	old_prog = netvsc_xdp_get(nvdev);
132
133	if (!old_prog && !prog)
134		return 0;
135
136	buf_max = NETVSC_XDP_HDRM + netvsc_xdp_fraglen(dev->mtu + ETH_HLEN);
137	if (prog && buf_max > PAGE_SIZE) {
138		netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n",
139			   dev->mtu, buf_max);
140		NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");
141
142		return -EOPNOTSUPP;
143	}
144
145	if (prog && (dev->features & NETIF_F_LRO)) {
146		netdev_err(dev, "XDP: not support LRO\n");
147		NL_SET_ERR_MSG_MOD(extack, "XDP: not support LRO");
148
149		return -EOPNOTSUPP;
150	}
151
152	if (prog)
153		bpf_prog_add(prog, nvdev->num_chn - 1);
154
155	for (i = 0; i < nvdev->num_chn; i++)
156		rcu_assign_pointer(nvdev->chan_table[i].bpf_prog, prog);
157
158	if (old_prog)
159		for (i = 0; i < nvdev->num_chn; i++)
160			bpf_prog_put(old_prog);
161
162	return 0;
163}
164
165int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog)
166{
167	struct netdev_bpf xdp;
168	int ret;
169
170	ASSERT_RTNL();
171
172	if (!vf_netdev)
173		return 0;
174
175	if (!vf_netdev->netdev_ops->ndo_bpf)
176		return 0;
177
178	memset(&xdp, 0, sizeof(xdp));
179
180	if (prog)
181		bpf_prog_inc(prog);
182
183	xdp.command = XDP_SETUP_PROG;
184	xdp.prog = prog;
185
186	ret = vf_netdev->netdev_ops->ndo_bpf(vf_netdev, &xdp);
187
188	if (ret && prog)
189		bpf_prog_put(prog);
190
191	return ret;
192}
193
194int netvsc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
195{
196	struct net_device_context *ndevctx = netdev_priv(dev);
197	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
198	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
199	struct netlink_ext_ack *extack = bpf->extack;
200	int ret;
201
202	if (!nvdev || nvdev->destroy) {
203		return -ENODEV;
204	}
205
206	switch (bpf->command) {
207	case XDP_SETUP_PROG:
208		ret = netvsc_xdp_set(dev, bpf->prog, extack, nvdev);
209
210		if (ret)
211			return ret;
212
213		ret = netvsc_vf_setxdp(vf_netdev, bpf->prog);
214
215		if (ret) {
216			netdev_err(dev, "vf_setxdp failed:%d\n", ret);
217			NL_SET_ERR_MSG_MOD(extack, "vf_setxdp failed");
218
219			netvsc_xdp_set(dev, NULL, extack, nvdev);
220		}
221
222		return ret;
223
224	default:
225		return -EINVAL;
226	}
227}
228
229static int netvsc_ndoxdp_xmit_fm(struct net_device *ndev,
230				 struct xdp_frame *frame, u16 q_idx)
231{
232	struct sk_buff *skb;
233
234	skb = xdp_build_skb_from_frame(frame, ndev);
235	if (unlikely(!skb))
236		return -ENOMEM;
237
238	netvsc_get_hash(skb, netdev_priv(ndev));
239
240	skb_record_rx_queue(skb, q_idx);
241
242	netvsc_xdp_xmit(skb, ndev);
243
244	return 0;
245}
246
247int netvsc_ndoxdp_xmit(struct net_device *ndev, int n,
248		       struct xdp_frame **frames, u32 flags)
249{
250	struct net_device_context *ndev_ctx = netdev_priv(ndev);
251	const struct net_device_ops *vf_ops;
252	struct netvsc_stats_tx *tx_stats;
253	struct netvsc_device *nvsc_dev;
254	struct net_device *vf_netdev;
255	int i, count = 0;
256	u16 q_idx;
257
258	/* Don't transmit if netvsc_device is gone */
259	nvsc_dev = rcu_dereference_bh(ndev_ctx->nvdev);
260	if (unlikely(!nvsc_dev || nvsc_dev->destroy))
261		return 0;
262
263	/* If VF is present and up then redirect packets to it.
264	 * Skip the VF if it is marked down or has no carrier.
265	 * If netpoll is in uses, then VF can not be used either.
266	 */
267	vf_netdev = rcu_dereference_bh(ndev_ctx->vf_netdev);
268	if (vf_netdev && netif_running(vf_netdev) &&
269	    netif_carrier_ok(vf_netdev) && !netpoll_tx_running(ndev) &&
270	    vf_netdev->netdev_ops->ndo_xdp_xmit &&
271	    ndev_ctx->data_path_is_vf) {
272		vf_ops = vf_netdev->netdev_ops;
273		return vf_ops->ndo_xdp_xmit(vf_netdev, n, frames, flags);
274	}
275
276	q_idx = smp_processor_id() % ndev->real_num_tx_queues;
277
278	for (i = 0; i < n; i++) {
279		if (netvsc_ndoxdp_xmit_fm(ndev, frames[i], q_idx))
280			break;
281
282		count++;
283	}
284
285	tx_stats = &nvsc_dev->chan_table[q_idx].tx_stats;
286
287	u64_stats_update_begin(&tx_stats->syncp);
288	tx_stats->xdp_xmit += count;
289	u64_stats_update_end(&tx_stats->syncp);
290
291	return count;
292}
293