1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2/* Copyright (C) 2018 Netronome Systems, Inc */
3/* Copyright (C) 2021 Corigine, Inc */
4
5#include <linux/bpf_trace.h>
6#include <linux/netdevice.h>
7
8#include "../nfp_app.h"
9#include "../nfp_net.h"
10#include "../nfp_net_dp.h"
11#include "../nfp_net_xsk.h"
12#include "nfd3.h"
13
14static bool
15nfp_nfd3_xsk_tx_xdp(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
16		    struct nfp_net_rx_ring *rx_ring,
17		    struct nfp_net_tx_ring *tx_ring,
18		    struct nfp_net_xsk_rx_buf *xrxbuf, unsigned int pkt_len,
19		    int pkt_off)
20{
21	struct xsk_buff_pool *pool = r_vec->xsk_pool;
22	struct nfp_nfd3_tx_buf *txbuf;
23	struct nfp_nfd3_tx_desc *txd;
24	unsigned int wr_idx;
25
26	if (nfp_net_tx_space(tx_ring) < 1)
27		return false;
28
29	xsk_buff_raw_dma_sync_for_device(pool, xrxbuf->dma_addr + pkt_off,
30					 pkt_len);
31
32	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);
33
34	txbuf = &tx_ring->txbufs[wr_idx];
35	txbuf->xdp = xrxbuf->xdp;
36	txbuf->real_len = pkt_len;
37	txbuf->is_xsk_tx = true;
38
39	/* Build TX descriptor */
40	txd = &tx_ring->txds[wr_idx];
41	txd->offset_eop = NFD3_DESC_TX_EOP;
42	txd->dma_len = cpu_to_le16(pkt_len);
43	nfp_desc_set_dma_addr_40b(txd, xrxbuf->dma_addr + pkt_off);
44	txd->data_len = cpu_to_le16(pkt_len);
45
46	txd->flags = 0;
47	txd->mss = 0;
48	txd->lso_hdrlen = 0;
49
50	tx_ring->wr_ptr_add++;
51	tx_ring->wr_p++;
52
53	return true;
54}
55
56static void nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring *rx_ring,
57				const struct nfp_net_rx_desc *rxd,
58				struct nfp_net_xsk_rx_buf *xrxbuf,
59				const struct nfp_meta_parsed *meta,
60				unsigned int pkt_len,
61				bool meta_xdp,
62				unsigned int *skbs_polled)
63{
64	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
65	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
66	struct net_device *netdev;
67	struct sk_buff *skb;
68
69	if (likely(!meta->portid)) {
70		netdev = dp->netdev;
71	} else {
72		struct nfp_net *nn = netdev_priv(dp->netdev);
73
74		netdev = nfp_app_dev_get(nn->app, meta->portid, NULL);
75		if (unlikely(!netdev)) {
76			nfp_net_xsk_rx_drop(r_vec, xrxbuf);
77			return;
78		}
79		nfp_repr_inc_rx_stats(netdev, pkt_len);
80	}
81
82	skb = napi_alloc_skb(&r_vec->napi, pkt_len);
83	if (!skb) {
84		nfp_net_xsk_rx_drop(r_vec, xrxbuf);
85		return;
86	}
87	skb_put_data(skb, xrxbuf->xdp->data, pkt_len);
88
89	skb->mark = meta->mark;
90	skb_set_hash(skb, meta->hash, meta->hash_type);
91
92	skb_record_rx_queue(skb, rx_ring->idx);
93	skb->protocol = eth_type_trans(skb, netdev);
94
95	nfp_nfd3_rx_csum(dp, r_vec, rxd, meta, skb);
96
97	if (unlikely(!nfp_net_vlan_strip(skb, rxd, meta))) {
98		dev_kfree_skb_any(skb);
99		nfp_net_xsk_rx_drop(r_vec, xrxbuf);
100		return;
101	}
102
103	if (meta_xdp)
104		skb_metadata_set(skb,
105				 xrxbuf->xdp->data - xrxbuf->xdp->data_meta);
106
107	napi_gro_receive(&rx_ring->r_vec->napi, skb);
108
109	nfp_net_xsk_rx_free(xrxbuf);
110
111	(*skbs_polled)++;
112}
113
114static unsigned int
115nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget,
116		unsigned int *skbs_polled)
117{
118	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
119	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
120	struct nfp_net_tx_ring *tx_ring;
121	struct bpf_prog *xdp_prog;
122	bool xdp_redir = false;
123	int pkts_polled = 0;
124
125	xdp_prog = READ_ONCE(dp->xdp_prog);
126	tx_ring = r_vec->xdp_ring;
127
128	while (pkts_polled < budget) {
129		unsigned int meta_len, data_len, pkt_len, pkt_off;
130		struct nfp_net_xsk_rx_buf *xrxbuf;
131		struct nfp_net_rx_desc *rxd;
132		struct nfp_meta_parsed meta;
133		int idx, act;
134
135		idx = D_IDX(rx_ring, rx_ring->rd_p);
136
137		rxd = &rx_ring->rxds[idx];
138		if (!(rxd->rxd.meta_len_dd & PCIE_DESC_RX_DD))
139			break;
140
141		rx_ring->rd_p++;
142		pkts_polled++;
143
144		xrxbuf = &rx_ring->xsk_rxbufs[idx];
145
146		/* If starved of buffers "drop" it and scream. */
147		if (rx_ring->rd_p >= rx_ring->wr_p) {
148			nn_dp_warn(dp, "Starved of RX buffers\n");
149			nfp_net_xsk_rx_drop(r_vec, xrxbuf);
150			break;
151		}
152
153		/* Memory barrier to ensure that we won't do other reads
154		 * before the DD bit.
155		 */
156		dma_rmb();
157
158		memset(&meta, 0, sizeof(meta));
159
160		/* Only supporting AF_XDP with dynamic metadata so buffer layout
161		 * is always:
162		 *
163		 *  ---------------------------------------------------------
164		 * |  off | metadata  |             packet           | XXXX  |
165		 *  ---------------------------------------------------------
166		 */
167		meta_len = rxd->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK;
168		data_len = le16_to_cpu(rxd->rxd.data_len);
169		pkt_len = data_len - meta_len;
170
171		if (unlikely(meta_len > NFP_NET_MAX_PREPEND)) {
172			nn_dp_warn(dp, "Oversized RX packet metadata %u\n",
173				   meta_len);
174			nfp_net_xsk_rx_drop(r_vec, xrxbuf);
175			continue;
176		}
177
178		/* Stats update. */
179		u64_stats_update_begin(&r_vec->rx_sync);
180		r_vec->rx_pkts++;
181		r_vec->rx_bytes += pkt_len;
182		u64_stats_update_end(&r_vec->rx_sync);
183
184		xrxbuf->xdp->data += meta_len;
185		xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len;
186		xdp_set_data_meta_invalid(xrxbuf->xdp);
187		xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool);
188		net_prefetch(xrxbuf->xdp->data);
189
190		if (meta_len) {
191			if (unlikely(nfp_nfd3_parse_meta(dp->netdev, &meta,
192							 xrxbuf->xdp->data -
193							 meta_len,
194							 xrxbuf->xdp->data,
195							 pkt_len, meta_len))) {
196				nn_dp_warn(dp, "Invalid RX packet metadata\n");
197				nfp_net_xsk_rx_drop(r_vec, xrxbuf);
198				continue;
199			}
200
201			if (unlikely(meta.portid)) {
202				struct nfp_net *nn = netdev_priv(dp->netdev);
203
204				if (meta.portid != NFP_META_PORT_ID_CTRL) {
205					nfp_nfd3_xsk_rx_skb(rx_ring, rxd,
206							    xrxbuf, &meta,
207							    pkt_len, false,
208							    skbs_polled);
209					continue;
210				}
211
212				nfp_app_ctrl_rx_raw(nn->app, xrxbuf->xdp->data,
213						    pkt_len);
214				nfp_net_xsk_rx_free(xrxbuf);
215				continue;
216			}
217		}
218
219		act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp);
220
221		pkt_len = xrxbuf->xdp->data_end - xrxbuf->xdp->data;
222		pkt_off = xrxbuf->xdp->data - xrxbuf->xdp->data_hard_start;
223
224		switch (act) {
225		case XDP_PASS:
226			nfp_nfd3_xsk_rx_skb(rx_ring, rxd, xrxbuf, &meta, pkt_len,
227					    true, skbs_polled);
228			break;
229		case XDP_TX:
230			if (!nfp_nfd3_xsk_tx_xdp(dp, r_vec, rx_ring, tx_ring,
231						 xrxbuf, pkt_len, pkt_off))
232				nfp_net_xsk_rx_drop(r_vec, xrxbuf);
233			else
234				nfp_net_xsk_rx_unstash(xrxbuf);
235			break;
236		case XDP_REDIRECT:
237			if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) {
238				nfp_net_xsk_rx_drop(r_vec, xrxbuf);
239			} else {
240				nfp_net_xsk_rx_unstash(xrxbuf);
241				xdp_redir = true;
242			}
243			break;
244		default:
245			bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act);
246			fallthrough;
247		case XDP_ABORTED:
248			trace_xdp_exception(dp->netdev, xdp_prog, act);
249			fallthrough;
250		case XDP_DROP:
251			nfp_net_xsk_rx_drop(r_vec, xrxbuf);
252			break;
253		}
254	}
255
256	nfp_net_xsk_rx_ring_fill_freelist(r_vec->rx_ring);
257
258	if (xdp_redir)
259		xdp_do_flush();
260
261	if (tx_ring->wr_ptr_add)
262		nfp_net_tx_xmit_more_flush(tx_ring);
263
264	return pkts_polled;
265}
266
267void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf)
268{
269	xsk_buff_free(txbuf->xdp);
270
271	txbuf->dma_addr = 0;
272	txbuf->xdp = NULL;
273}
274
275static bool nfp_nfd3_xsk_complete(struct nfp_net_tx_ring *tx_ring)
276{
277	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
278	u32 done_pkts = 0, done_bytes = 0, reused = 0;
279	bool done_all;
280	int idx, todo;
281	u32 qcp_rd_p;
282
283	if (tx_ring->wr_p == tx_ring->rd_p)
284		return true;
285
286	/* Work out how many descriptors have been transmitted. */
287	qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);
288
289	if (qcp_rd_p == tx_ring->qcp_rd_p)
290		return true;
291
292	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);
293
294	done_all = todo <= NFP_NET_XDP_MAX_COMPLETE;
295	todo = min(todo, NFP_NET_XDP_MAX_COMPLETE);
296
297	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo);
298
299	done_pkts = todo;
300	while (todo--) {
301		struct nfp_nfd3_tx_buf *txbuf;
302
303		idx = D_IDX(tx_ring, tx_ring->rd_p);
304		tx_ring->rd_p++;
305
306		txbuf = &tx_ring->txbufs[idx];
307		if (unlikely(!txbuf->real_len))
308			continue;
309
310		done_bytes += txbuf->real_len;
311		txbuf->real_len = 0;
312
313		if (txbuf->is_xsk_tx) {
314			nfp_nfd3_xsk_tx_free(txbuf);
315			reused++;
316		}
317	}
318
319	u64_stats_update_begin(&r_vec->tx_sync);
320	r_vec->tx_bytes += done_bytes;
321	r_vec->tx_pkts += done_pkts;
322	u64_stats_update_end(&r_vec->tx_sync);
323
324	xsk_tx_completed(r_vec->xsk_pool, done_pkts - reused);
325
326	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
327		  "XDP TX ring corruption rd_p=%u wr_p=%u cnt=%u\n",
328		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
329
330	return done_all;
331}
332
333static void nfp_nfd3_xsk_tx(struct nfp_net_tx_ring *tx_ring)
334{
335	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
336	struct xdp_desc desc[NFP_NET_XSK_TX_BATCH];
337	struct xsk_buff_pool *xsk_pool;
338	struct nfp_nfd3_tx_desc *txd;
339	u32 pkts = 0, wr_idx;
340	u32 i, got;
341
342	xsk_pool = r_vec->xsk_pool;
343
344	while (nfp_net_tx_space(tx_ring) >= NFP_NET_XSK_TX_BATCH) {
345		for (i = 0; i < NFP_NET_XSK_TX_BATCH; i++)
346			if (!xsk_tx_peek_desc(xsk_pool, &desc[i]))
347				break;
348		got = i;
349		if (!got)
350			break;
351
352		wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);
353		prefetchw(&tx_ring->txds[wr_idx]);
354
355		for (i = 0; i < got; i++)
356			xsk_buff_raw_dma_sync_for_device(xsk_pool, desc[i].addr,
357							 desc[i].len);
358
359		for (i = 0; i < got; i++) {
360			wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);
361
362			tx_ring->txbufs[wr_idx].real_len = desc[i].len;
363			tx_ring->txbufs[wr_idx].is_xsk_tx = false;
364
365			/* Build TX descriptor. */
366			txd = &tx_ring->txds[wr_idx];
367			nfp_desc_set_dma_addr_40b(txd,
368						  xsk_buff_raw_get_dma(xsk_pool, desc[i].addr));
369			txd->offset_eop = NFD3_DESC_TX_EOP;
370			txd->dma_len = cpu_to_le16(desc[i].len);
371			txd->data_len = cpu_to_le16(desc[i].len);
372		}
373
374		tx_ring->wr_p += got;
375		pkts += got;
376	}
377
378	if (!pkts)
379		return;
380
381	xsk_tx_release(xsk_pool);
382	/* Ensure all records are visible before incrementing write counter. */
383	wmb();
384	nfp_qcp_wr_ptr_add(tx_ring->qcp_q, pkts);
385}
386
387int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget)
388{
389	struct nfp_net_r_vector *r_vec =
390		container_of(napi, struct nfp_net_r_vector, napi);
391	unsigned int pkts_polled, skbs = 0;
392
393	pkts_polled = nfp_nfd3_xsk_rx(r_vec->rx_ring, budget, &skbs);
394
395	if (pkts_polled < budget) {
396		if (r_vec->tx_ring)
397			nfp_nfd3_tx_complete(r_vec->tx_ring, budget);
398
399		if (!nfp_nfd3_xsk_complete(r_vec->xdp_ring))
400			pkts_polled = budget;
401
402		nfp_nfd3_xsk_tx(r_vec->xdp_ring);
403
404		if (pkts_polled < budget && napi_complete_done(napi, skbs))
405			nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);
406	}
407
408	return pkts_polled;
409}
410