1// SPDX-License-Identifier: GPL-2.0-only
2/* Atlantic Network Driver
3 *
4 * Copyright (C) 2014-2019 aQuantia Corporation
5 * Copyright (C) 2019-2020 Marvell International Ltd.
6 */
7
8/* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings.
9 * Definition of functions for Rx and Tx rings. Friendly module for aq_nic.
10 */
11
12#include "aq_vec.h"
13
14struct aq_vec_s {
15	const struct aq_hw_ops *aq_hw_ops;
16	struct aq_hw_s *aq_hw;
17	struct aq_nic_s *aq_nic;
18	unsigned int tx_rings;
19	unsigned int rx_rings;
20	struct aq_ring_param_s aq_ring_param;
21	struct napi_struct napi;
22	struct aq_ring_s ring[AQ_CFG_TCS_MAX][2];
23};
24
25#define AQ_VEC_TX_ID 0
26#define AQ_VEC_RX_ID 1
27
28static int aq_vec_poll(struct napi_struct *napi, int budget)
29{
30	struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
31	unsigned int sw_tail_old = 0U;
32	struct aq_ring_s *ring = NULL;
33	bool was_tx_cleaned = true;
34	unsigned int i = 0U;
35	int work_done = 0;
36	int err = 0;
37
38	if (!self) {
39		err = -EINVAL;
40	} else {
41		for (i = 0U; self->tx_rings > i; ++i) {
42			ring = self->ring[i];
43			u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
44			ring[AQ_VEC_RX_ID].stats.rx.polls++;
45			u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
46			if (self->aq_hw_ops->hw_ring_tx_head_update) {
47				err = self->aq_hw_ops->hw_ring_tx_head_update(
48							self->aq_hw,
49							&ring[AQ_VEC_TX_ID]);
50				if (err < 0)
51					goto err_exit;
52			}
53
54			if (ring[AQ_VEC_TX_ID].sw_head !=
55			    ring[AQ_VEC_TX_ID].hw_head) {
56				was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
57				aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]);
58			}
59
60			err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw,
61					    &ring[AQ_VEC_RX_ID]);
62			if (err < 0)
63				goto err_exit;
64
65			if (ring[AQ_VEC_RX_ID].sw_head !=
66				ring[AQ_VEC_RX_ID].hw_head) {
67				err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID],
68						       napi,
69						       &work_done,
70						       budget - work_done);
71				if (err < 0)
72					goto err_exit;
73
74				sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail;
75
76				err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
77				if (err < 0)
78					goto err_exit;
79
80				err = self->aq_hw_ops->hw_ring_rx_fill(
81					self->aq_hw,
82					&ring[AQ_VEC_RX_ID], sw_tail_old);
83				if (err < 0)
84					goto err_exit;
85			}
86		}
87
88err_exit:
89		if (!was_tx_cleaned)
90			work_done = budget;
91
92		if (work_done < budget) {
93			napi_complete_done(napi, work_done);
94			self->aq_hw_ops->hw_irq_enable(self->aq_hw,
95					1U << self->aq_ring_param.vec_idx);
96		}
97	}
98
99	return work_done;
100}
101
102struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
103			      struct aq_nic_cfg_s *aq_nic_cfg)
104{
105	struct aq_vec_s *self = NULL;
106
107	self = kzalloc(sizeof(*self), GFP_KERNEL);
108	if (!self)
109		goto err_exit;
110
111	self->aq_nic = aq_nic;
112	self->aq_ring_param.vec_idx = idx;
113	self->aq_ring_param.cpu =
114		idx + aq_nic_cfg->aq_rss.base_cpu_number;
115
116	cpumask_set_cpu(self->aq_ring_param.cpu,
117			&self->aq_ring_param.affinity_mask);
118
119	self->tx_rings = 0;
120	self->rx_rings = 0;
121
122	netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll);
123
124err_exit:
125	return self;
126}
127
128int aq_vec_ring_alloc(struct aq_vec_s *self, struct aq_nic_s *aq_nic,
129		      unsigned int idx, struct aq_nic_cfg_s *aq_nic_cfg)
130{
131	struct aq_ring_s *ring = NULL;
132	unsigned int i = 0U;
133	int err = 0;
134
135	for (i = 0; i < aq_nic_cfg->tcs; ++i) {
136		const unsigned int idx_ring = AQ_NIC_CFG_TCVEC2RING(aq_nic_cfg,
137								    i, idx);
138
139		ring = &self->ring[i][AQ_VEC_TX_ID];
140		err = aq_ring_tx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg);
141		if (err)
142			goto err_exit;
143
144		++self->tx_rings;
145
146		aq_nic_set_tx_ring(aq_nic, idx_ring, ring);
147
148		ring = &self->ring[i][AQ_VEC_RX_ID];
149		if (xdp_rxq_info_reg(&ring->xdp_rxq,
150				     aq_nic->ndev, idx,
151				     self->napi.napi_id) < 0) {
152			err = -ENOMEM;
153			goto err_exit;
154		}
155		if (xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
156					       MEM_TYPE_PAGE_SHARED, NULL) < 0) {
157			xdp_rxq_info_unreg(&ring->xdp_rxq);
158			err = -ENOMEM;
159			goto err_exit;
160		}
161
162		err = aq_ring_rx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg);
163		if (err) {
164			xdp_rxq_info_unreg(&ring->xdp_rxq);
165			goto err_exit;
166		}
167
168		++self->rx_rings;
169	}
170
171err_exit:
172	if (err < 0) {
173		aq_vec_ring_free(self);
174		self = NULL;
175	}
176
177	return err;
178}
179
180int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
181		struct aq_hw_s *aq_hw)
182{
183	struct aq_ring_s *ring = NULL;
184	unsigned int i = 0U;
185	int err = 0;
186
187	self->aq_hw_ops = aq_hw_ops;
188	self->aq_hw = aq_hw;
189
190	for (i = 0U; self->tx_rings > i; ++i) {
191		ring = self->ring[i];
192		err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX);
193		if (err < 0)
194			goto err_exit;
195
196		err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw,
197						       &ring[AQ_VEC_TX_ID],
198						       &self->aq_ring_param);
199		if (err < 0)
200			goto err_exit;
201
202		err = aq_ring_init(&ring[AQ_VEC_RX_ID], ATL_RING_RX);
203		if (err < 0)
204			goto err_exit;
205
206		err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw,
207						       &ring[AQ_VEC_RX_ID],
208						       &self->aq_ring_param);
209		if (err < 0)
210			goto err_exit;
211
212		err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
213		if (err < 0)
214			goto err_exit;
215
216		err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw,
217						       &ring[AQ_VEC_RX_ID], 0U);
218		if (err < 0)
219			goto err_exit;
220	}
221
222err_exit:
223	return err;
224}
225
226int aq_vec_start(struct aq_vec_s *self)
227{
228	struct aq_ring_s *ring = NULL;
229	unsigned int i = 0U;
230	int err = 0;
231
232	for (i = 0U; self->tx_rings > i; ++i) {
233		ring = self->ring[i];
234		err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw,
235							&ring[AQ_VEC_TX_ID]);
236		if (err < 0)
237			goto err_exit;
238
239		err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw,
240							&ring[AQ_VEC_RX_ID]);
241		if (err < 0)
242			goto err_exit;
243	}
244
245	napi_enable(&self->napi);
246
247err_exit:
248	return err;
249}
250
251void aq_vec_stop(struct aq_vec_s *self)
252{
253	struct aq_ring_s *ring = NULL;
254	unsigned int i = 0U;
255
256	for (i = 0U; self->tx_rings > i; ++i) {
257		ring = self->ring[i];
258		self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw,
259						 &ring[AQ_VEC_TX_ID]);
260
261		self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw,
262						 &ring[AQ_VEC_RX_ID]);
263	}
264
265	napi_disable(&self->napi);
266}
267
268void aq_vec_deinit(struct aq_vec_s *self)
269{
270	struct aq_ring_s *ring = NULL;
271	unsigned int i = 0U;
272
273	if (!self)
274		goto err_exit;
275
276	for (i = 0U; self->tx_rings > i; ++i) {
277		ring = self->ring[i];
278		aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
279		aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]);
280	}
281
282err_exit:;
283}
284
285void aq_vec_free(struct aq_vec_s *self)
286{
287	if (!self)
288		goto err_exit;
289
290	netif_napi_del(&self->napi);
291
292	kfree(self);
293
294err_exit:;
295}
296
297void aq_vec_ring_free(struct aq_vec_s *self)
298{
299	struct aq_ring_s *ring = NULL;
300	unsigned int i = 0U;
301
302	if (!self)
303		goto err_exit;
304
305	for (i = 0U; self->tx_rings > i; ++i) {
306		ring = self->ring[i];
307		aq_ring_free(&ring[AQ_VEC_TX_ID]);
308		if (i < self->rx_rings) {
309			xdp_rxq_info_unreg(&ring[AQ_VEC_RX_ID].xdp_rxq);
310			aq_ring_free(&ring[AQ_VEC_RX_ID]);
311		}
312	}
313
314	self->tx_rings = 0;
315	self->rx_rings = 0;
316err_exit:;
317}
318
319irqreturn_t aq_vec_isr(int irq, void *private)
320{
321	struct aq_vec_s *self = private;
322	int err = 0;
323
324	if (!self) {
325		err = -EINVAL;
326		goto err_exit;
327	}
328	napi_schedule(&self->napi);
329
330err_exit:
331	return err >= 0 ? IRQ_HANDLED : IRQ_NONE;
332}
333
334irqreturn_t aq_vec_isr_legacy(int irq, void *private)
335{
336	struct aq_vec_s *self = private;
337	u64 irq_mask = 0U;
338	int err;
339
340	if (!self)
341		return IRQ_NONE;
342	err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask);
343	if (err < 0)
344		return IRQ_NONE;
345
346	if (irq_mask) {
347		self->aq_hw_ops->hw_irq_disable(self->aq_hw,
348			      1U << self->aq_ring_param.vec_idx);
349		napi_schedule(&self->napi);
350	} else {
351		self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U);
352		return IRQ_NONE;
353	}
354
355	return IRQ_HANDLED;
356}
357
358cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self)
359{
360	return &self->aq_ring_param.affinity_mask;
361}
362
363bool aq_vec_is_valid_tc(struct aq_vec_s *self, const unsigned int tc)
364{
365	return tc < self->rx_rings && tc < self->tx_rings;
366}
367
368unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u64 *data)
369{
370	unsigned int count;
371
372	if (!aq_vec_is_valid_tc(self, tc))
373		return 0;
374
375	count = aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_RX_ID], data);
376	count += aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_TX_ID], data + count);
377
378	return count;
379}
380