1/* SPDX-License-Identifier: GPL-2.0-only */
2/* Atlantic Network Driver
3 *
4 * Copyright (C) 2014-2019 aQuantia Corporation
5 * Copyright (C) 2019-2020 Marvell International Ltd.
6 */
7
8/* File aq_ring.h: Declaration of functions for Rx/Tx rings. */
9
10#ifndef AQ_RING_H
11#define AQ_RING_H
12
13#include "aq_common.h"
14#include "aq_vec.h"
15
16#define AQ_XDP_HEADROOM		ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)
17#define AQ_XDP_TAILROOM		SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
18
19struct page;
20struct aq_nic_cfg_s;
21
22struct aq_rxpage {
23	struct page *page;
24	dma_addr_t daddr;
25	unsigned int order;
26	unsigned int pg_off;
27};
28
29/*           TxC       SOP        DX         EOP
30 *         +----------+----------+----------+-----------
31 *   8bytes|len l3,l4 | pa       | pa       | pa
32 *         +----------+----------+----------+-----------
33 * 4/8bytes|len pkt   |len pkt   |          | skb
34 *         +----------+----------+----------+-----------
35 * 4/8bytes|is_gso    |len,flags |len       |len,is_eop
36 *         +----------+----------+----------+-----------
37 *
38 *  This aq_ring_buff_s doesn't have endianness dependency.
39 *  It is __packed for cache line optimizations.
40 */
41struct __packed aq_ring_buff_s {
42	union {
43		/* RX/TX */
44		dma_addr_t pa;
45		/* RX */
46		struct {
47			u32 rss_hash;
48			u16 next;
49			u8 is_hash_l4;
50			u8 rsvd1;
51			struct aq_rxpage rxdata;
52			u16 vlan_rx_tag;
53		};
54		/* EOP */
55		struct {
56			dma_addr_t pa_eop;
57			struct sk_buff *skb;
58			struct xdp_frame *xdpf;
59		};
60		/* TxC */
61		struct {
62			u32 mss;
63			u8 len_l2;
64			u8 len_l3;
65			u8 len_l4;
66			u8 is_ipv6:1;
67			u8 rsvd2:7;
68			u32 len_pkt;
69			u16 vlan_tx_tag;
70		};
71	};
72	union {
73		struct {
74			u32 len:16;
75			u32 is_ip_cso:1;
76			u32 is_udp_cso:1;
77			u32 is_tcp_cso:1;
78			u32 is_cso_err:1;
79			u32 is_sop:1;
80			u32 is_eop:1;
81			u32 is_gso_tcp:1;
82			u32 is_gso_udp:1;
83			u32 is_mapped:1;
84			u32 is_cleaned:1;
85			u32 is_error:1;
86			u32 is_vlan:1;
87			u32 is_lro:1;
88			u32 rsvd3:3;
89			u16 eop_index;
90			u16 rsvd4;
91		};
92		u64 flags;
93	};
94};
95
96struct aq_ring_stats_rx_s {
97	struct u64_stats_sync syncp;	/* must be first */
98	u64 errors;
99	u64 packets;
100	u64 bytes;
101	u64 lro_packets;
102	u64 jumbo_packets;
103	u64 alloc_fails;
104	u64 skb_alloc_fails;
105	u64 polls;
106	u64 pg_losts;
107	u64 pg_flips;
108	u64 pg_reuses;
109	u64 xdp_aborted;
110	u64 xdp_drop;
111	u64 xdp_pass;
112	u64 xdp_tx;
113	u64 xdp_invalid;
114	u64 xdp_redirect;
115};
116
117struct aq_ring_stats_tx_s {
118	struct u64_stats_sync syncp;	/* must be first */
119	u64 errors;
120	u64 packets;
121	u64 bytes;
122	u64 queue_restarts;
123};
124
125union aq_ring_stats_s {
126	struct aq_ring_stats_rx_s rx;
127	struct aq_ring_stats_tx_s tx;
128};
129
130enum atl_ring_type {
131	ATL_RING_TX,
132	ATL_RING_RX,
133};
134
135struct aq_ring_s {
136	struct aq_ring_buff_s *buff_ring;
137	u8 *dx_ring;		/* descriptors ring, dma shared mem */
138	struct aq_nic_s *aq_nic;
139	unsigned int idx;	/* for HW layer registers operations */
140	unsigned int hw_head;
141	unsigned int sw_head;
142	unsigned int sw_tail;
143	unsigned int size;	/* descriptors number */
144	unsigned int dx_size;	/* TX or RX descriptor size,  */
145				/* stored here for fater math */
146	u16 page_order;
147	u16 page_offset;
148	u16 frame_max;
149	u16 tail_size;
150	union aq_ring_stats_s stats;
151	dma_addr_t dx_ring_pa;
152	struct bpf_prog *xdp_prog;
153	enum atl_ring_type ring_type;
154	struct xdp_rxq_info xdp_rxq;
155};
156
157struct aq_ring_param_s {
158	unsigned int vec_idx;
159	unsigned int cpu;
160	cpumask_t affinity_mask;
161};
162
163static inline void *aq_buf_vaddr(struct aq_rxpage *rxpage)
164{
165	return page_to_virt(rxpage->page) + rxpage->pg_off;
166}
167
168static inline dma_addr_t aq_buf_daddr(struct aq_rxpage *rxpage)
169{
170	return rxpage->daddr + rxpage->pg_off;
171}
172
173static inline unsigned int aq_ring_next_dx(struct aq_ring_s *self,
174					   unsigned int dx)
175{
176	return (++dx >= self->size) ? 0U : dx;
177}
178
179static inline unsigned int aq_ring_avail_dx(struct aq_ring_s *self)
180{
181	return (((self->sw_tail >= self->sw_head)) ?
182		(self->size - 1) - self->sw_tail + self->sw_head :
183		self->sw_head - self->sw_tail - 1);
184}
185
186int aq_ring_tx_alloc(struct aq_ring_s *self,
187		     struct aq_nic_s *aq_nic,
188		     unsigned int idx,
189		     struct aq_nic_cfg_s *aq_nic_cfg);
190int aq_ring_rx_alloc(struct aq_ring_s *self,
191		     struct aq_nic_s *aq_nic,
192		     unsigned int idx,
193		     struct aq_nic_cfg_s *aq_nic_cfg);
194
195int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type);
196void aq_ring_rx_deinit(struct aq_ring_s *self);
197void aq_ring_free(struct aq_ring_s *self);
198void aq_ring_update_queue_state(struct aq_ring_s *ring);
199void aq_ring_queue_wake(struct aq_ring_s *ring);
200void aq_ring_queue_stop(struct aq_ring_s *ring);
201bool aq_ring_tx_clean(struct aq_ring_s *self);
202int aq_xdp_xmit(struct net_device *dev, int num_frames,
203		struct xdp_frame **frames, u32 flags);
204int aq_ring_rx_clean(struct aq_ring_s *self,
205		     struct napi_struct *napi,
206		     int *work_done,
207		     int budget);
208int aq_ring_rx_fill(struct aq_ring_s *self);
209
210int aq_ring_hwts_rx_alloc(struct aq_ring_s *self,
211			  struct aq_nic_s *aq_nic, unsigned int idx,
212			  unsigned int size, unsigned int dx_size);
213void aq_ring_hwts_rx_free(struct aq_ring_s *self);
214void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic);
215
216unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data);
217
218#endif /* AQ_RING_H */
219