1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2024 Intel Corporation */
3
4#include <net/libeth/rx.h>
5
6/* Rx buffer management */
7
8/**
9 * libeth_rx_hw_len - get the actual buffer size to be passed to HW
10 * @pp: &page_pool_params of the netdev to calculate the size for
11 * @max_len: maximum buffer size for a single descriptor
12 *
13 * Return: HW-writeable length per one buffer to pass it to the HW accounting:
14 * MTU the @dev has, HW required alignment, minimum and maximum allowed values,
15 * and system's page size.
16 */
17static u32 libeth_rx_hw_len(const struct page_pool_params *pp, u32 max_len)
18{
19	u32 len;
20
21	len = READ_ONCE(pp->netdev->mtu) + LIBETH_RX_LL_LEN;
22	len = ALIGN(len, LIBETH_RX_BUF_STRIDE);
23	len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE),
24		   pp->max_len);
25
26	return len;
27}
28
29/**
30 * libeth_rx_fq_create - create a PP with the default libeth settings
31 * @fq: buffer queue struct to fill
32 * @napi: &napi_struct covering this PP (no usage outside its poll loops)
33 *
34 * Return: %0 on success, -%errno on failure.
35 */
36int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi)
37{
38	struct page_pool_params pp = {
39		.flags		= PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
40		.order		= LIBETH_RX_PAGE_ORDER,
41		.pool_size	= fq->count,
42		.nid		= fq->nid,
43		.dev		= napi->dev->dev.parent,
44		.netdev		= napi->dev,
45		.napi		= napi,
46		.dma_dir	= DMA_FROM_DEVICE,
47		.offset		= LIBETH_SKB_HEADROOM,
48	};
49	struct libeth_fqe *fqes;
50	struct page_pool *pool;
51
52	/* HW-writeable / syncable length per one page */
53	pp.max_len = LIBETH_RX_PAGE_LEN(pp.offset);
54
55	/* HW-writeable length per buffer */
56	fq->buf_len = libeth_rx_hw_len(&pp, fq->buf_len);
57	/* Buffer size to allocate */
58	fq->truesize = roundup_pow_of_two(SKB_HEAD_ALIGN(pp.offset +
59							 fq->buf_len));
60
61	pool = page_pool_create(&pp);
62	if (IS_ERR(pool))
63		return PTR_ERR(pool);
64
65	fqes = kvcalloc_node(fq->count, sizeof(*fqes), GFP_KERNEL, fq->nid);
66	if (!fqes)
67		goto err_buf;
68
69	fq->fqes = fqes;
70	fq->pp = pool;
71
72	return 0;
73
74err_buf:
75	page_pool_destroy(pool);
76
77	return -ENOMEM;
78}
79EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_create, LIBETH);
80
81/**
82 * libeth_rx_fq_destroy - destroy a &page_pool created by libeth
83 * @fq: buffer queue to process
84 */
85void libeth_rx_fq_destroy(struct libeth_fq *fq)
86{
87	kvfree(fq->fqes);
88	page_pool_destroy(fq->pp);
89}
90EXPORT_SYMBOL_NS_GPL(libeth_rx_fq_destroy, LIBETH);
91
92/**
93 * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context
94 * @page: page to recycle
95 *
96 * To be used on exceptions or rare cases not requiring fast inline recycling.
97 */
98void libeth_rx_recycle_slow(struct page *page)
99{
100	page_pool_recycle_direct(page->pp, page);
101}
102EXPORT_SYMBOL_NS_GPL(libeth_rx_recycle_slow, LIBETH);
103
104/* Converting abstract packet type numbers into a software structure with
105 * the packet parameters to do O(1) lookup on Rx.
106 */
107
108static const u16 libeth_rx_pt_xdp_oip[] = {
109	[LIBETH_RX_PT_OUTER_L2]		= XDP_RSS_TYPE_NONE,
110	[LIBETH_RX_PT_OUTER_IPV4]	= XDP_RSS_L3_IPV4,
111	[LIBETH_RX_PT_OUTER_IPV6]	= XDP_RSS_L3_IPV6,
112};
113
114static const u16 libeth_rx_pt_xdp_iprot[] = {
115	[LIBETH_RX_PT_INNER_NONE]	= XDP_RSS_TYPE_NONE,
116	[LIBETH_RX_PT_INNER_UDP]	= XDP_RSS_L4_UDP,
117	[LIBETH_RX_PT_INNER_TCP]	= XDP_RSS_L4_TCP,
118	[LIBETH_RX_PT_INNER_SCTP]	= XDP_RSS_L4_SCTP,
119	[LIBETH_RX_PT_INNER_ICMP]	= XDP_RSS_L4_ICMP,
120	[LIBETH_RX_PT_INNER_TIMESYNC]	= XDP_RSS_TYPE_NONE,
121};
122
123static const u16 libeth_rx_pt_xdp_pl[] = {
124	[LIBETH_RX_PT_PAYLOAD_NONE]	= XDP_RSS_TYPE_NONE,
125	[LIBETH_RX_PT_PAYLOAD_L2]	= XDP_RSS_TYPE_NONE,
126	[LIBETH_RX_PT_PAYLOAD_L3]	= XDP_RSS_TYPE_NONE,
127	[LIBETH_RX_PT_PAYLOAD_L4]	= XDP_RSS_L4,
128};
129
130/**
131 * libeth_rx_pt_gen_hash_type - generate an XDP RSS hash type for a PT
132 * @pt: PT structure to evaluate
133 *
134 * Generates ```hash_type``` field with XDP RSS type values from the parsed
135 * packet parameters if they're obtained dynamically at runtime.
136 */
137void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt)
138{
139	pt->hash_type = 0;
140	pt->hash_type |= libeth_rx_pt_xdp_oip[pt->outer_ip];
141	pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot];
142	pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer];
143}
144EXPORT_SYMBOL_NS_GPL(libeth_rx_pt_gen_hash_type, LIBETH);
145
146/* Module */
147
148MODULE_AUTHOR("Intel Corporation");
149MODULE_DESCRIPTION("Common Ethernet library");
150MODULE_LICENSE("GPL");
151