1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2019 Mellanox Technologies. */
3
4#include "setup.h"
5#include "en/params.h"
6#include "en/txrx.h"
7#include "en/health.h"
8#include <net/xdp_sock_drv.h>
9
10static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
11					struct mlx5e_params *params,
12					struct mlx5e_xsk_param *xsk)
13{
14	if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
15		mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n");
16		return -EINVAL;
17	}
18
19	return 0;
20}
21
22/* The limitation of 2048 can be altered, but shouldn't go beyond the minimal
23 * stride size of striding RQ.
24 */
25#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
26
27bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
28			      struct mlx5e_xsk_param *xsk,
29			      struct mlx5_core_dev *mdev)
30{
31	/* AF_XDP doesn't support frames larger than PAGE_SIZE. */
32	if (xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) {
33		mlx5_core_err(mdev, "XSK chunk size %u out of bounds [%u, %lu]\n", xsk->chunk_size,
34			      MLX5E_MIN_XSK_CHUNK_SIZE, PAGE_SIZE);
35		return false;
36	}
37
38	/* frag_sz is different for regular and XSK RQs, so ensure that linear
39	 * SKB mode is possible.
40	 */
41	switch (params->rq_wq_type) {
42	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
43		return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
44	default: /* MLX5_WQ_TYPE_CYCLIC */
45		return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk);
46	}
47}
48
49static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
50				   struct mlx5e_params *params,
51				   struct mlx5e_xsk_param *xsk,
52				   struct mlx5e_channel_param *cparam)
53{
54	mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq);
55	mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq);
56}
57
58static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
59			     struct mlx5e_params *params,
60			     struct xsk_buff_pool *pool,
61			     struct mlx5e_xsk_param *xsk,
62			     struct mlx5e_rq *rq)
63{
64	struct mlx5_core_dev *mdev = c->mdev;
65	int rq_xdp_ix;
66	int err;
67
68	rq->wq_type      = params->rq_wq_type;
69	rq->pdev         = c->pdev;
70	rq->netdev       = c->netdev;
71	rq->priv         = c->priv;
72	rq->tstamp       = c->tstamp;
73	rq->clock        = &mdev->clock;
74	rq->icosq        = &c->icosq;
75	rq->ix           = c->ix;
76	rq->channel      = c;
77	rq->mdev         = mdev;
78	rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
79	rq->xdpsq        = &c->rq_xdpsq;
80	rq->xsk_pool     = pool;
81	rq->stats        = &c->priv->channel_stats[c->ix]->xskrq;
82	rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
83	rq_xdp_ix        = c->ix;
84	err = mlx5e_rq_set_handlers(rq, params, xsk);
85	if (err)
86		return err;
87
88	return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id);
89}
90
91static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
92			     struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
93			     struct mlx5e_xsk_param *xsk)
94{
95	u16 q_counter = c->priv->q_counter[c->sd_ix];
96	struct mlx5e_rq *xskrq = &c->xskrq;
97	int err;
98
99	err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
100	if (err)
101		return err;
102
103	err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq);
104	if (err)
105		return err;
106
107	__set_bit(MLX5E_RQ_STATE_XSK, &xskrq->state);
108	return 0;
109}
110
111int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
112		   struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
113		   struct mlx5e_channel *c)
114{
115	struct mlx5e_channel_param *cparam;
116	struct mlx5e_create_cq_param ccp;
117	int err;
118
119	mlx5e_build_create_cq_param(&ccp, c);
120
121	if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
122		return -EINVAL;
123
124	cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL);
125	if (!cparam)
126		return -ENOMEM;
127
128	mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam);
129
130	err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
131			    &c->xskrq.cq);
132	if (unlikely(err))
133		goto err_free_cparam;
134
135	err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk);
136	if (unlikely(err))
137		goto err_close_rx_cq;
138
139	err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
140			    &c->xsksq.cq);
141	if (unlikely(err))
142		goto err_close_rq;
143
144	/* Create a separate SQ, so that when the buff pool is disabled, we could
145	 * close this SQ safely and stop receiving CQEs. In other case, e.g., if
146	 * the XDPSQ was used instead, we might run into trouble when the buff pool
147	 * is disabled and then re-enabled, but the SQ continues receiving CQEs
148	 * from the old buff pool.
149	 */
150	err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, pool, &c->xsksq, true);
151	if (unlikely(err))
152		goto err_close_tx_cq;
153
154	kvfree(cparam);
155
156	set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
157
158	return 0;
159
160err_close_tx_cq:
161	mlx5e_close_cq(&c->xsksq.cq);
162
163err_close_rq:
164	mlx5e_close_rq(&c->xskrq);
165
166err_close_rx_cq:
167	mlx5e_close_cq(&c->xskrq.cq);
168
169err_free_cparam:
170	kvfree(cparam);
171
172	return err;
173}
174
175void mlx5e_close_xsk(struct mlx5e_channel *c)
176{
177	clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
178	synchronize_net(); /* Sync with NAPI. */
179
180	mlx5e_close_rq(&c->xskrq);
181	mlx5e_close_cq(&c->xskrq.cq);
182	mlx5e_close_xdpsq(&c->xsksq);
183	mlx5e_close_cq(&c->xsksq.cq);
184
185	memset(&c->xskrq, 0, sizeof(c->xskrq));
186	memset(&c->xsksq, 0, sizeof(c->xsksq));
187}
188
189void mlx5e_activate_xsk(struct mlx5e_channel *c)
190{
191	/* ICOSQ recovery deactivates RQs. Suspend the recovery to avoid
192	 * activating XSKRQ in the middle of recovery.
193	 */
194	mlx5e_reporter_icosq_suspend_recovery(c);
195	set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
196	mlx5e_reporter_icosq_resume_recovery(c);
197
198	/* TX queue is created active. */
199}
200
201void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
202{
203	/* ICOSQ recovery may reactivate XSKRQ if clear_bit is called in the
204	 * middle of recovery. Suspend the recovery to avoid it.
205	 */
206	mlx5e_reporter_icosq_suspend_recovery(c);
207	clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
208	mlx5e_reporter_icosq_resume_recovery(c);
209	synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
210
211	/* TX queue is disabled on close. */
212}
213