1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include "efsys.h"
27#include "efx.h"
28#include "efx_types.h"
29#include "efx_regs.h"
30#include "efx_impl.h"
31
32	__checkReturn	int
33efx_rx_init(
34	__in		efx_nic_t *enp)
35{
36	efx_oword_t oword;
37	unsigned int idx;
38	int rc;
39
40	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
41	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
42
43	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
44		rc = EINVAL;
45		goto fail1;
46	}
47
48	if (enp->en_mod_flags & EFX_MOD_RX) {
49		rc = EINVAL;
50		goto fail2;
51	}
52
53	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
54
55	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
56	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
57	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
58	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
59	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
60	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
61	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
62
63	/* Zero the RSS table */
64	for (idx = 0; idx < FR_BZ_RX_INDIRECTION_TBL_ROWS;
65	    idx++) {
66		EFX_ZERO_OWORD(oword);
67		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
68				    idx, &oword);
69	}
70
71	enp->en_mod_flags |= EFX_MOD_RX;
72	return (0);
73
74fail2:
75	EFSYS_PROBE(fail2);
76fail1:
77	EFSYS_PROBE1(fail1, int, rc);
78
79	return (rc);
80}
81
82#if EFSYS_OPT_RX_HDR_SPLIT
83	__checkReturn	int
84efx_rx_hdr_split_enable(
85	__in		efx_nic_t *enp,
86	__in		unsigned int hdr_buf_size,
87	__in		unsigned int pld_buf_size)
88{
89	unsigned int nhdr32;
90	unsigned int npld32;
91	efx_oword_t oword;
92	int rc;
93
94	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
95	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
96	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
97
98	nhdr32 = hdr_buf_size / 32;
99	if ((nhdr32 == 0) ||
100	    (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
101	    ((hdr_buf_size % 32) != 0)) {
102		rc = EINVAL;
103		goto fail1;
104	}
105
106	npld32 = pld_buf_size / 32;
107	if ((npld32 == 0) ||
108	    (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
109	    ((pld_buf_size % 32) != 0)) {
110		rc = EINVAL;
111		goto fail2;
112	}
113
114	if (enp->en_rx_qcount > 0) {
115		rc = EBUSY;
116		goto fail3;
117	}
118
119	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
120
121	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
122	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
123	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
124
125	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
126
127	return (0);
128
129fail3:
130	EFSYS_PROBE(fail3);
131fail2:
132	EFSYS_PROBE(fail2);
133fail1:
134	EFSYS_PROBE1(fail1, int, rc);
135
136	return (rc);
137}
138#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
139
140
141#if EFSYS_OPT_RX_SCATTER
142	__checkReturn	int
143efx_rx_scatter_enable(
144	__in		efx_nic_t *enp,
145	__in		unsigned int buf_size)
146{
147	unsigned int nbuf32;
148	efx_oword_t oword;
149	int rc;
150
151	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
152	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
153	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
154
155	nbuf32 = buf_size / 32;
156	if ((nbuf32 == 0) ||
157	    (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
158	    ((buf_size % 32) != 0)) {
159		rc = EINVAL;
160		goto fail1;
161	}
162
163	if (enp->en_rx_qcount > 0) {
164		rc = EBUSY;
165		goto fail2;
166	}
167
168	/* Set scatter buffer size */
169	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
170	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
171	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
172
173	/* Enable scatter for packets not matching a filter */
174	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
175	EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
176	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
177
178	return (0);
179
180fail2:
181	EFSYS_PROBE(fail2);
182fail1:
183	EFSYS_PROBE1(fail1, int, rc);
184
185	return (rc);
186}
187#endif	/* EFSYS_OPT_RX_SCATTER */
188
189
190#define	EFX_RX_LFSR_HASH(_enp, _insert)					\
191	do {                                    			\
192		efx_oword_t oword;					\
193									\
194		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
195		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);	\
196		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);	\
197		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);	\
198		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
199		    (_insert) ? 1 : 0);					\
200		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
201									\
202		if ((_enp)->en_family == EFX_FAMILY_SIENA) {		\
203			EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
204			    &oword);					\
205			EFX_SET_OWORD_FIELD(oword,			\
206			    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);	\
207			EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
208			    &oword);					\
209		}							\
210									\
211		_NOTE(CONSTANTCONDITION)				\
212	} while (B_FALSE)
213
214#define	EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)		\
215	do {                                    			\
216		efx_oword_t oword;					\
217									\
218		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG,	&oword);	\
219		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);	\
220		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,		\
221		    (_ip) ? 1 : 0);					\
222		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,		\
223		    (_tcp) ? 0 : 1);					\
224		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
225		    (_insert) ? 1 : 0);					\
226		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
227									\
228		_NOTE(CONSTANTCONDITION)				\
229	} while (B_FALSE)
230
231#define	EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)			\
232	do {                                    			\
233		efx_oword_t oword;					\
234									\
235		if ((_enp)->en_family == EFX_FAMILY_FALCON) {		\
236			(_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;	\
237			break;						\
238		}							\
239									\
240		EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
241		EFX_SET_OWORD_FIELD(oword,				\
242		    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);		\
243		EFX_SET_OWORD_FIELD(oword,				\
244		    FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0);	\
245		EFX_SET_OWORD_FIELD(oword,				\
246		    FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);	\
247		EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
248									\
249		(_rc) = 0;						\
250									\
251		_NOTE(CONSTANTCONDITION)				\
252	} while (B_FALSE)
253
254
255#if EFSYS_OPT_RX_SCALE
256	__checkReturn	int
257efx_rx_scale_mode_set(
258	__in		efx_nic_t *enp,
259	__in		efx_rx_hash_alg_t alg,
260	__in		efx_rx_hash_type_t type,
261	__in		boolean_t insert)
262{
263	int rc;
264
265	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
266	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
267	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
268
269	switch (alg) {
270	case EFX_RX_HASHALG_LFSR:
271		EFX_RX_LFSR_HASH(enp, insert);
272		break;
273
274	case EFX_RX_HASHALG_TOEPLITZ:
275		EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
276		    type & (1 << EFX_RX_HASH_IPV4),
277		    type & (1 << EFX_RX_HASH_TCPIPV4));
278
279		EFX_RX_TOEPLITZ_IPV6_HASH(enp,
280		    type & (1 << EFX_RX_HASH_IPV6),
281		    type & (1 << EFX_RX_HASH_TCPIPV6),
282		    rc);
283		if (rc != 0)
284			goto fail1;
285
286		break;
287
288	default:
289		rc = EINVAL;
290		goto fail2;
291	}
292
293	return (0);
294
295fail2:
296	EFSYS_PROBE(fail2);
297fail1:
298	EFSYS_PROBE1(fail1, int, rc);
299
300	EFX_RX_LFSR_HASH(enp, B_FALSE);
301
302	return (rc);
303}
304#endif
305
306#if EFSYS_OPT_RX_SCALE
307	__checkReturn	int
308efx_rx_scale_toeplitz_ipv4_key_set(
309	__in		efx_nic_t *enp,
310	__in_ecount(n)	uint8_t *key,
311	__in		size_t n)
312{
313	efx_oword_t oword;
314	unsigned int byte;
315	unsigned int offset;
316	int rc;
317
318	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
319	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
320
321	byte = 0;
322
323	/* Write toeplitz hash key */
324	EFX_ZERO_OWORD(oword);
325	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
326	    offset > 0 && byte < n;
327	    --offset)
328		oword.eo_u8[offset - 1] = key[byte++];
329
330	EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
331
332	byte = 0;
333
334	/* Verify toeplitz hash key */
335	EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
336	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
337	    offset > 0 && byte < n;
338	    --offset) {
339		if (oword.eo_u8[offset - 1] != key[byte++]) {
340			rc = EFAULT;
341			goto fail1;
342		}
343	}
344
345	return (0);
346
347fail1:
348	EFSYS_PROBE1(fail1, int, rc);
349
350	return (rc);
351}
352#endif
353
354#if EFSYS_OPT_RX_SCALE
355	__checkReturn	int
356efx_rx_scale_toeplitz_ipv6_key_set(
357	__in		efx_nic_t *enp,
358	__in_ecount(n)	uint8_t *key,
359	__in		size_t n)
360{
361	efx_oword_t oword;
362	unsigned int byte;
363	int offset;
364	int rc;
365
366	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
367	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
368
369	byte = 0;
370
371	/* Write toeplitz hash key 3 */
372	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
373	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
374	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
375	    offset > 0 && byte < n;
376	    --offset)
377		oword.eo_u8[offset - 1] = key[byte++];
378
379	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
380
381	/* Write toeplitz hash key 2 */
382	EFX_ZERO_OWORD(oword);
383	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
384	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
385	    offset > 0 && byte < n;
386	    --offset)
387		oword.eo_u8[offset - 1] = key[byte++];
388
389	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
390
391	/* Write toeplitz hash key 1 */
392	EFX_ZERO_OWORD(oword);
393	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
394	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
395	    offset > 0 && byte < n;
396	    --offset)
397		oword.eo_u8[offset - 1] = key[byte++];
398
399	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
400
401	byte = 0;
402
403	/* Verify toeplitz hash key 3 */
404	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
405	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
406	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
407	    offset > 0 && byte < n;
408	    --offset) {
409		if (oword.eo_u8[offset - 1] != key[byte++]) {
410			rc = EFAULT;
411			goto fail1;
412		}
413	}
414
415	/* Verify toeplitz hash key 2 */
416	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
417	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
418	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
419	    offset > 0 && byte < n;
420	    --offset) {
421		if (oword.eo_u8[offset - 1] != key[byte++]) {
422			rc = EFAULT;
423			goto fail2;
424		}
425	}
426
427	/* Verify toeplitz hash key 1 */
428	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
429	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
430	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
431	    offset > 0 && byte < n;
432	    --offset) {
433		if (oword.eo_u8[offset - 1] != key[byte++]) {
434			rc = EFAULT;
435			goto fail3;
436		}
437	}
438
439	return (0);
440
441fail3:
442	EFSYS_PROBE(fail3);
443fail2:
444	EFSYS_PROBE(fail2);
445fail1:
446	EFSYS_PROBE1(fail1, int, rc);
447
448	return (rc);
449}
450#endif
451
452#if EFSYS_OPT_RX_SCALE
453	__checkReturn	int
454efx_rx_scale_tbl_set(
455	__in		efx_nic_t *enp,
456	__in_ecount(n)	unsigned int *table,
457	__in		size_t n)
458{
459	efx_oword_t oword;
460	int idx;
461	int rc;
462
463	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
464	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
465
466	EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
467	EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
468
469	if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
470		rc = EINVAL;
471		goto fail1;
472	}
473
474	for (idx = 0; idx < FR_BZ_RX_INDIRECTION_TBL_ROWS; idx++) {
475		uint32_t byte;
476
477		/* Calculate the entry to place in the table */
478		byte = (uint32_t)table[idx % n];
479
480		EFSYS_PROBE2(table, int, idx, uint32_t, byte);
481
482		EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
483
484		/* Write the table */
485		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
486				    idx, &oword);
487	}
488
489	for (idx = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; idx >= 0; --idx) {
490		uint32_t byte;
491
492		/* Determine if we're starting a new batch */
493		byte = (uint32_t)table[idx % n];
494
495		/* Read the table */
496		EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
497				    idx, &oword);
498
499		/* Verify the entry */
500		if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
501			rc = EFAULT;
502			goto fail2;
503		}
504	}
505
506	return (0);
507
508fail2:
509	EFSYS_PROBE(fail2);
510fail1:
511	EFSYS_PROBE1(fail1, int, rc);
512
513	return (rc);
514}
515#endif
516
517#if EFSYS_OPT_FILTER
518extern	__checkReturn	int
519efx_rx_filter_insert(
520	__in		efx_rxq_t *erp,
521	__inout		efx_filter_spec_t *spec)
522{
523	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
524	EFSYS_ASSERT3P(spec, !=, NULL);
525
526	spec->efs_dmaq_id = (uint16_t)erp->er_index;
527	return efx_filter_insert_filter(erp->er_enp, spec, B_FALSE);
528}
529#endif
530
531#if EFSYS_OPT_FILTER
532extern	__checkReturn	int
533efx_rx_filter_remove(
534	__in		efx_rxq_t *erp,
535	__inout		efx_filter_spec_t *spec)
536{
537	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
538	EFSYS_ASSERT3P(spec, !=, NULL);
539
540	spec->efs_dmaq_id = (uint16_t)erp->er_index;
541	return efx_filter_remove_filter(erp->er_enp, spec);
542}
543#endif
544
545extern			void
546efx_rx_qpost(
547	__in		efx_rxq_t *erp,
548	__in_ecount(n)	efsys_dma_addr_t *addrp,
549	__in		size_t size,
550	__in		unsigned int n,
551	__in		unsigned int completed,
552	__in		unsigned int added)
553{
554	efx_qword_t qword;
555	unsigned int i;
556	unsigned int offset;
557	unsigned int id;
558
559	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
560
561	/* The client driver must not overfill the queue */
562	EFSYS_ASSERT3U(added - completed + n, <=,
563	    EFX_RXQ_LIMIT(erp->er_mask + 1));
564
565	id = added & (erp->er_mask);
566	for (i = 0; i < n; i++) {
567		EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
568		    unsigned int, id, efsys_dma_addr_t, addrp[i],
569		    size_t, size);
570
571		EFX_POPULATE_QWORD_3(qword,
572		    FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
573		    FSF_AZ_RX_KER_BUF_ADDR_DW0,
574		    (uint32_t)(addrp[i] & 0xffffffff),
575		    FSF_AZ_RX_KER_BUF_ADDR_DW1,
576		    (uint32_t)(addrp[i] >> 32));
577
578		offset = id * sizeof (efx_qword_t);
579		EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
580
581		id = (id + 1) & (erp->er_mask);
582	}
583}
584
585		void
586efx_rx_qpush(
587	__in	efx_rxq_t *erp,
588	__in	unsigned int added)
589{
590	efx_nic_t *enp = erp->er_enp;
591	uint32_t wptr;
592	efx_oword_t oword;
593	efx_dword_t dword;
594
595	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
596
597	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
598	EFSYS_PIO_WRITE_BARRIER();
599
600	/* Push the populated descriptors out */
601	wptr = added & erp->er_mask;
602
603	EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
604
605	/* Only write the third DWORD */
606	EFX_POPULATE_DWORD_1(dword,
607	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
608	EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
609			    erp->er_index, &dword, B_FALSE);
610}
611
612		void
613efx_rx_qflush(
614	__in	efx_rxq_t *erp)
615{
616	efx_nic_t *enp = erp->er_enp;
617	efx_oword_t oword;
618	uint32_t label;
619
620	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
621
622	label = erp->er_index;
623
624	/* Flush the queue */
625	EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
626	    FRF_AZ_RX_FLUSH_DESCQ, label);
627	EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
628}
629
630		void
631efx_rx_qenable(
632	__in	efx_rxq_t *erp)
633{
634	efx_nic_t *enp = erp->er_enp;
635	efx_oword_t oword;
636
637	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
638
639	EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
640			    erp->er_index, &oword);
641
642	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
643	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
644	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
645
646	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
647			    erp->er_index, &oword);
648}
649
650	__checkReturn	int
651efx_rx_qcreate(
652	__in		efx_nic_t *enp,
653	__in		unsigned int idx,
654	__in		unsigned int label,
655	__in		efx_rxq_type_t type,
656	__in		efsys_mem_t *esmp,
657	__in		size_t n,
658	__in		uint32_t id,
659	__in		efx_evq_t *eep,
660	__deref_out	efx_rxq_t **erpp)
661{
662	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
663	efx_rxq_t *erp;
664	efx_oword_t oword;
665	uint32_t size;
666	boolean_t split;
667	boolean_t jumbo;
668	int rc;
669
670	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
671	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
672
673	EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
674	EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
675	EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
676
677	if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
678		rc = EINVAL;
679		goto fail1;
680	}
681	if (idx >= encp->enc_rxq_limit) {
682		rc = EINVAL;
683		goto fail2;
684	}
685	for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
686	    size++)
687		if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
688			break;
689	if (id + (1 << size) >= encp->enc_buftbl_limit) {
690		rc = EINVAL;
691		goto fail3;
692	}
693
694	switch (type) {
695	case EFX_RXQ_TYPE_DEFAULT:
696		split = B_FALSE;
697		jumbo = B_FALSE;
698		break;
699
700#if EFSYS_OPT_RX_HDR_SPLIT
701	case EFX_RXQ_TYPE_SPLIT_HEADER:
702		if ((enp->en_family < EFX_FAMILY_SIENA) || ((idx & 1) != 0)) {
703			rc = EINVAL;
704			goto fail4;
705		}
706		split = B_TRUE;
707		jumbo = B_TRUE;
708		break;
709
710	case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
711		if ((enp->en_family < EFX_FAMILY_SIENA) || ((idx & 1) == 0)) {
712			rc = EINVAL;
713			goto fail4;
714		}
715		split = B_FALSE;
716		jumbo = B_TRUE;
717		break;
718#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
719
720#if EFSYS_OPT_RX_SCATTER
721	case EFX_RXQ_TYPE_SCATTER:
722		if (enp->en_family < EFX_FAMILY_SIENA) {
723			rc = EINVAL;
724			goto fail4;
725		}
726		split = B_FALSE;
727		jumbo = B_TRUE;
728		break;
729#endif	/* EFSYS_OPT_RX_SCATTER */
730
731	default:
732		rc = EINVAL;
733		goto fail4;
734	}
735
736	/* Allocate an RXQ object */
737	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
738
739	if (erp == NULL) {
740		rc = ENOMEM;
741		goto fail5;
742	}
743
744	erp->er_magic = EFX_RXQ_MAGIC;
745	erp->er_enp = enp;
746	erp->er_index = idx;
747	erp->er_mask = n - 1;
748	erp->er_esmp = esmp;
749
750	/* Set up the new descriptor queue */
751	EFX_POPULATE_OWORD_10(oword,
752	    FRF_CZ_RX_HDR_SPLIT, split,
753	    FRF_AZ_RX_ISCSI_DDIG_EN, 0,
754	    FRF_AZ_RX_ISCSI_HDIG_EN, 0,
755	    FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
756	    FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
757	    FRF_AZ_RX_DESCQ_OWNER_ID, 0,
758	    FRF_AZ_RX_DESCQ_LABEL, label,
759	    FRF_AZ_RX_DESCQ_SIZE, size,
760	    FRF_AZ_RX_DESCQ_TYPE, 0,
761	    FRF_AZ_RX_DESCQ_JUMBO, jumbo);
762
763	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
764			    erp->er_index, &oword);
765
766	enp->en_rx_qcount++;
767	*erpp = erp;
768	return (0);
769
770fail5:
771	EFSYS_PROBE(fail5);
772fail4:
773	EFSYS_PROBE(fail4);
774fail3:
775	EFSYS_PROBE(fail3);
776fail2:
777	EFSYS_PROBE(fail2);
778fail1:
779	EFSYS_PROBE1(fail1, int, rc);
780
781	return (rc);
782}
783
784		void
785efx_rx_qdestroy(
786	__in	efx_rxq_t *erp)
787{
788	efx_nic_t *enp = erp->er_enp;
789	efx_oword_t oword;
790
791	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
792
793	EFSYS_ASSERT(enp->en_rx_qcount != 0);
794	--enp->en_rx_qcount;
795
796	/* Purge descriptor queue */
797	EFX_ZERO_OWORD(oword);
798
799	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
800			    erp->er_index, &oword);
801
802	/* Free the RXQ object */
803	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
804}
805
806		void
807efx_rx_fini(
808	__in	efx_nic_t *enp)
809{
810	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
811	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
812	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
813	EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
814
815	enp->en_mod_flags &= ~EFX_MOD_RX;
816}
817