1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include "efsys.h"
27#include "efx.h"
28#include "efx_types.h"
29#include "efx_regs.h"
30#include "efx_impl.h"
31
32#if EFSYS_OPT_QSTATS
33#define	EFX_TX_QSTAT_INCR(_etp, _stat)					\
34	do {								\
35		(_etp)->et_stat[_stat]++;				\
36	_NOTE(CONSTANTCONDITION)					\
37	} while (B_FALSE)
38#else
39#define	EFX_TX_QSTAT_INCR(_etp, _stat)
40#endif
41
42	__checkReturn	int
43efx_tx_init(
44	__in		efx_nic_t *enp)
45{
46	efx_oword_t oword;
47	int rc;
48
49	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
50	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
51
52	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
53		rc = EINVAL;
54		goto fail1;
55	}
56
57	if (enp->en_mod_flags & EFX_MOD_TX) {
58		rc = EINVAL;
59		goto fail2;
60	}
61
62	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
63
64	/*
65	 * Disable the timer-based TX DMA backoff and allow TX DMA to be
66	 * controlled by the RX FIFO fill level (although always allow a
67	 * minimal trickle).
68	 */
69	EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
70	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
71	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
72	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
73	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
74	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
75	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
76	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
77
78	/*
79	 * Filter all packets less than 14 bytes to avoid parsing
80	 * errors.
81	 */
82	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
83	EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
84
85	/*
86	 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
87	 * descriptors (which is bad).
88	 */
89	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
90	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
91	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
92
93	enp->en_mod_flags |= EFX_MOD_TX;
94	return (0);
95
96fail2:
97	EFSYS_PROBE(fail2);
98fail1:
99	EFSYS_PROBE1(fail1, int, rc);
100
101	return (rc);
102}
103
104#if EFSYS_OPT_FILTER
105extern	__checkReturn	int
106efx_tx_filter_insert(
107	__in		efx_txq_t *etp,
108	__inout		efx_filter_spec_t *spec)
109{
110	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
111	EFSYS_ASSERT3P(spec, !=, NULL);
112
113	spec->efs_dmaq_id = (uint16_t)etp->et_index;
114	return efx_filter_insert_filter(etp->et_enp, spec, B_FALSE);
115}
116#endif
117
118#if EFSYS_OPT_FILTER
119extern	__checkReturn	int
120efx_tx_filter_remove(
121	__in		efx_txq_t *etp,
122	__inout		efx_filter_spec_t *spec)
123{
124	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
125	EFSYS_ASSERT3P(spec, !=, NULL);
126
127	spec->efs_dmaq_id = (uint16_t)etp->et_index;
128	return efx_filter_remove_filter(etp->et_enp, spec);
129}
130#endif
131
132#define	EFX_TX_DESC(_etp, _addr, _size, _eop, _added)			\
133	do {								\
134		unsigned int id;					\
135		size_t offset;						\
136		efx_qword_t qword;					\
137									\
138		id = (_added)++ & (_etp)->et_mask;			\
139		offset = id * sizeof (efx_qword_t);			\
140									\
141		EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index,	\
142		    unsigned int, id, efsys_dma_addr_t, (_addr),	\
143		    size_t, (_size), boolean_t, (_eop));		\
144									\
145		EFX_POPULATE_QWORD_4(qword,				\
146		    FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1,			\
147		    FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size),	\
148		    FSF_AZ_TX_KER_BUF_ADDR_DW0,				\
149		    (uint32_t)((_addr) & 0xffffffff),			\
150		    FSF_AZ_TX_KER_BUF_ADDR_DW1,				\
151		    (uint32_t)((_addr) >> 32));				\
152		EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword);	\
153									\
154		_NOTE(CONSTANTCONDITION)				\
155	} while (B_FALSE)
156
157	__checkReturn	int
158efx_tx_qpost(
159	__in		efx_txq_t *etp,
160	__in_ecount(n)	efx_buffer_t *eb,
161	__in		unsigned int n,
162	__in		unsigned int completed,
163	__inout		unsigned int *addedp)
164{
165	unsigned int added = *addedp;
166	unsigned int i;
167	int rc = ENOSPC;
168
169	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
170
171	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
172		goto fail1;
173
174	for (i = 0; i < n; i++) {
175		efx_buffer_t *ebp = &eb[i];
176		efsys_dma_addr_t start = ebp->eb_addr;
177		size_t size = ebp->eb_size;
178		efsys_dma_addr_t end = start + size;
179
180		/* Fragments must not span 4k boundaries. */
181		EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
182
183		EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
184	}
185
186	EFX_TX_QSTAT_INCR(etp, TX_POST);
187
188	*addedp = added;
189	return (0);
190
191fail1:
192	EFSYS_PROBE1(fail1, int, rc);
193
194	return (rc);
195}
196
197		void
198efx_tx_qpush(
199	__in	efx_txq_t *etp,
200	__in	unsigned int added)
201{
202	efx_nic_t *enp = etp->et_enp;
203	uint32_t wptr;
204	efx_dword_t dword;
205	efx_oword_t oword;
206
207	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
208
209	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
210	EFSYS_PIO_WRITE_BARRIER();
211
212	/* Push the populated descriptors out */
213	wptr = added & etp->et_mask;
214
215	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
216
217	/* Only write the third DWORD */
218	EFX_POPULATE_DWORD_1(dword,
219	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
220	EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
221			    etp->et_index, &dword, B_FALSE);
222}
223
224		void
225efx_tx_qflush(
226	__in	efx_txq_t *etp)
227{
228	efx_nic_t *enp = etp->et_enp;
229	efx_oword_t oword;
230	uint32_t label;
231
232	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
233
234	label = etp->et_index;
235
236	/* Flush the queue */
237	EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
238	    FRF_AZ_TX_FLUSH_DESCQ, label);
239	EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
240}
241
242		void
243efx_tx_qenable(
244	__in	efx_txq_t *etp)
245{
246	efx_nic_t *enp = etp->et_enp;
247	efx_oword_t oword;
248
249	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
250
251	EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
252			    etp->et_index, &oword);
253
254	EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
255	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
256	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
257	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
258	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
259
260	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
261	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
262	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
263
264	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
265			    etp->et_index, &oword);
266}
267
268	__checkReturn	int
269efx_tx_qcreate(
270	__in		efx_nic_t *enp,
271	__in		unsigned int idx,
272	__in		unsigned int label,
273	__in		efsys_mem_t *esmp,
274	__in		size_t n,
275	__in		uint32_t id,
276	__in		uint16_t flags,
277	__in		efx_evq_t *eep,
278	__deref_out	efx_txq_t **etpp)
279{
280	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
281	efx_txq_t *etp;
282	efx_oword_t oword;
283	uint32_t size;
284	int rc;
285
286	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
287	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
288
289	EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
290	EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
291	EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
292
293	if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
294		rc = EINVAL;
295		goto fail1;
296	}
297	if (idx >= encp->enc_txq_limit) {
298		rc = EINVAL;
299		goto fail2;
300	}
301	for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
302	    size++)
303		if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
304			break;
305	if (id + (1 << size) >= encp->enc_buftbl_limit) {
306		rc = EINVAL;
307		goto fail3;
308	}
309
310	/* Allocate an TXQ object */
311	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
312
313	if (etp == NULL) {
314		rc = ENOMEM;
315		goto fail4;
316	}
317
318	etp->et_magic = EFX_TXQ_MAGIC;
319	etp->et_enp = enp;
320	etp->et_index = idx;
321	etp->et_mask = n - 1;
322	etp->et_esmp = esmp;
323
324	/* Set up the new descriptor queue */
325	EFX_POPULATE_OWORD_6(oword,
326	    FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
327	    FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
328	    FRF_AZ_TX_DESCQ_OWNER_ID, 0,
329	    FRF_AZ_TX_DESCQ_LABEL, label,
330	    FRF_AZ_TX_DESCQ_SIZE, size,
331	    FRF_AZ_TX_DESCQ_TYPE, 0);
332
333	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
334	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
335	    (flags & EFX_CKSUM_IPV4) ? 0 : 1);
336	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
337	    (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
338
339	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
340	    etp->et_index, &oword);
341
342	enp->en_tx_qcount++;
343	*etpp = etp;
344	return (0);
345
346fail4:
347	EFSYS_PROBE(fail4);
348fail3:
349	EFSYS_PROBE(fail3);
350fail2:
351	EFSYS_PROBE(fail2);
352fail1:
353	EFSYS_PROBE1(fail1, int, rc);
354
355	return (rc);
356}
357
358#if EFSYS_OPT_NAMES
359/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
360static const char 	__cs * __cs __efx_tx_qstat_name[] = {
361	"post",
362	"unaligned_split",
363};
364/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
365
366		const char __cs *
367efx_tx_qstat_name(
368	__in	efx_nic_t *enp,
369	__in	unsigned int id)
370{
371	_NOTE(ARGUNUSED(enp))
372	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
373	EFSYS_ASSERT3U(id, <, TX_NQSTATS);
374
375	return (__efx_tx_qstat_name[id]);
376}
377#endif	/* EFSYS_OPT_NAMES */
378
379#if EFSYS_OPT_QSTATS
380					void
381efx_tx_qstats_update(
382	__in				efx_txq_t *etp,
383	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
384{
385	unsigned int id;
386
387	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
388
389	for (id = 0; id < TX_NQSTATS; id++) {
390		efsys_stat_t *essp = &stat[id];
391
392		EFSYS_STAT_INCR(essp, etp->et_stat[id]);
393		etp->et_stat[id] = 0;
394	}
395}
396#endif	/* EFSYS_OPT_QSTATS */
397
398		void
399efx_tx_qdestroy(
400	__in	efx_txq_t *etp)
401{
402	efx_nic_t *enp = etp->et_enp;
403	efx_oword_t oword;
404
405	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
406
407	EFSYS_ASSERT(enp->en_tx_qcount != 0);
408	--enp->en_tx_qcount;
409
410	/* Purge descriptor queue */
411	EFX_ZERO_OWORD(oword);
412
413	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
414			    etp->et_index, &oword);
415
416	/* Free the TXQ object */
417	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
418}
419
420		void
421efx_tx_fini(
422	__in	efx_nic_t *enp)
423{
424	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
425	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
426	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
427	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
428
429	enp->en_mod_flags &= ~EFX_MOD_TX;
430}
431