efx_rx.c revision 228078 
1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_rx.c 228078 2011-11-28 17:19:05Z philip $");
28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35	__checkReturn	int
36efx_rx_init(
37	__in		efx_nic_t *enp)
38{
39	efx_oword_t oword;
40	unsigned int index;
41	int rc;
42
43	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
44	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
45
46	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
47		rc = EINVAL;
48		goto fail1;
49	}
50
51	if (enp->en_mod_flags & EFX_MOD_RX) {
52		rc = EINVAL;
53		goto fail2;
54	}
55
56	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
57
58	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
59	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
60	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
61	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
62	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
63	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
64	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
65
66	/* Zero the RSS table */
67	for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
68	    index++) {
69		EFX_ZERO_OWORD(oword);
70		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
71				    index, &oword);
72	}
73
74	enp->en_mod_flags |= EFX_MOD_RX;
75	return (0);
76
77fail2:
78	EFSYS_PROBE(fail2);
79fail1:
80	EFSYS_PROBE1(fail1, int, rc);
81
82	return (rc);
83}
84
85#if EFSYS_OPT_RX_HDR_SPLIT
86	__checkReturn	int
87efx_rx_hdr_split_enable(
88	__in		efx_nic_t *enp,
89	__in		unsigned int hdr_buf_size,
90	__in		unsigned int pld_buf_size)
91{
92	unsigned int nhdr32;
93	unsigned int npld32;
94	efx_oword_t oword;
95	int rc;
96
97	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
98	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
99	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
100
101	nhdr32 = hdr_buf_size / 32;
102	if ((nhdr32 == 0) ||
103	    (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
104	    ((hdr_buf_size % 32) != 0)) {
105		rc = EINVAL;
106		goto fail1;
107	}
108
109	npld32 = pld_buf_size / 32;
110	if ((npld32 == 0) ||
111	    (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
112	    ((pld_buf_size % 32) != 0)) {
113		rc = EINVAL;
114		goto fail2;
115	}
116
117	if (enp->en_rx_qcount > 0) {
118		rc = EBUSY;
119		goto fail3;
120	}
121
122	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
123
124	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
125	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
126	EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
127
128	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
129
130	return (0);
131
132fail3:
133	EFSYS_PROBE(fail3);
134fail2:
135	EFSYS_PROBE(fail2);
136fail1:
137	EFSYS_PROBE1(fail1, int, rc);
138
139	return (rc);
140}
141#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
142
143
144#if EFSYS_OPT_RX_SCATTER
145	__checkReturn	int
146efx_rx_scatter_enable(
147	__in		efx_nic_t *enp,
148	__in		unsigned int buf_size)
149{
150	unsigned int nbuf32;
151	efx_oword_t oword;
152	int rc;
153
154	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
155	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
156	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
157
158	nbuf32 = buf_size / 32;
159	if ((nbuf32 == 0) ||
160	    (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
161	    ((buf_size % 32) != 0)) {
162		rc = EINVAL;
163		goto fail1;
164	}
165
166	if (enp->en_rx_qcount > 0) {
167		rc = EBUSY;
168		goto fail2;
169	}
170
171	/* Set scatter buffer size */
172	EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
173	EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
174	EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
175
176	/* Enable scatter for packets not matching a filter */
177	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
178	EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
179	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
180
181	return (0);
182
183fail2:
184	EFSYS_PROBE(fail2);
185fail1:
186	EFSYS_PROBE1(fail1, int, rc);
187
188	return (rc);
189}
190#endif	/* EFSYS_OPT_RX_SCATTER */
191
192
193#define	EFX_RX_LFSR_HASH(_enp, _insert)					\
194	do {                                    			\
195		efx_oword_t oword;					\
196									\
197		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
198		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);	\
199		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);	\
200		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);	\
201		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
202		    (_insert) ? 1 : 0);					\
203		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
204									\
205		if ((_enp)->en_family == EFX_FAMILY_SIENA) {		\
206			EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
207			    &oword);					\
208			EFX_SET_OWORD_FIELD(oword,			\
209			    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);	\
210			EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,	\
211			    &oword);					\
212		}							\
213									\
214		_NOTE(CONSTANTCONDITION)				\
215	} while (B_FALSE)
216
217#define	EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)		\
218	do {                                    			\
219		efx_oword_t oword;					\
220									\
221		EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG,	&oword);	\
222		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);	\
223		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,		\
224		    (_ip) ? 1 : 0);					\
225		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,		\
226		    (_tcp) ? 0 : 1);					\
227		EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,	\
228		    (_insert) ? 1 : 0);					\
229		EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);	\
230									\
231		_NOTE(CONSTANTCONDITION)				\
232	} while (B_FALSE)
233
234#define	EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)			\
235	do {                                    			\
236		efx_oword_t oword;					\
237									\
238		if ((_enp)->en_family == EFX_FAMILY_FALCON) {		\
239			(_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;	\
240			break;						\
241		}							\
242									\
243		EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
244		EFX_SET_OWORD_FIELD(oword,				\
245		    FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);		\
246		EFX_SET_OWORD_FIELD(oword,				\
247		    FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0);	\
248		EFX_SET_OWORD_FIELD(oword,				\
249		    FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);	\
250		EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);	\
251									\
252		(_rc) = 0;						\
253									\
254		_NOTE(CONSTANTCONDITION)				\
255	} while (B_FALSE)
256
257
258#if EFSYS_OPT_RX_SCALE
259	__checkReturn	int
260efx_rx_scale_mode_set(
261	__in		efx_nic_t *enp,
262	__in		efx_rx_hash_alg_t alg,
263	__in		efx_rx_hash_type_t type,
264	__in		boolean_t insert)
265{
266	int rc;
267
268	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
269	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
270	EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
271
272	switch (alg) {
273	case EFX_RX_HASHALG_LFSR:
274		EFX_RX_LFSR_HASH(enp, insert);
275		break;
276
277	case EFX_RX_HASHALG_TOEPLITZ:
278		EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
279		    type & (1 << EFX_RX_HASH_IPV4),
280		    type & (1 << EFX_RX_HASH_TCPIPV4));
281
282		EFX_RX_TOEPLITZ_IPV6_HASH(enp,
283		    type & (1 << EFX_RX_HASH_IPV6),
284		    type & (1 << EFX_RX_HASH_TCPIPV6),
285		    rc);
286		if (rc != 0)
287			goto fail1;
288
289		break;
290
291	default:
292		rc = EINVAL;
293		goto fail2;
294	}
295
296	return (0);
297
298fail2:
299	EFSYS_PROBE(fail2);
300fail1:
301	EFSYS_PROBE1(fail1, int, rc);
302
303	EFX_RX_LFSR_HASH(enp, B_FALSE);
304
305	return (rc);
306}
307#endif
308
309#if EFSYS_OPT_RX_SCALE
310	__checkReturn	int
311efx_rx_scale_toeplitz_ipv4_key_set(
312	__in		efx_nic_t *enp,
313	__in_ecount(n)	uint8_t *key,
314	__in		size_t n)
315{
316	efx_oword_t oword;
317	unsigned int byte;
318	unsigned int offset;
319	int rc;
320
321	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
322	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
323
324	byte = 0;
325
326	/* Write toeplitz hash key */
327	EFX_ZERO_OWORD(oword);
328	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
329	    offset > 0 && byte < n;
330	    --offset)
331		oword.eo_u8[offset - 1] = key[byte++];
332
333	EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
334
335	byte = 0;
336
337	/* Verify toeplitz hash key */
338	EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
339	for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
340	    offset > 0 && byte < n;
341	    --offset) {
342		if (oword.eo_u8[offset - 1] != key[byte++]) {
343			rc = EFAULT;
344			goto fail1;
345		}
346	}
347
348	return (0);
349
350fail1:
351	EFSYS_PROBE1(fail1, int, rc);
352
353	return (rc);
354}
355#endif
356
357#if EFSYS_OPT_RX_SCALE
358	__checkReturn	int
359efx_rx_scale_toeplitz_ipv6_key_set(
360	__in		efx_nic_t *enp,
361	__in_ecount(n)	uint8_t *key,
362	__in		size_t n)
363{
364	efx_oword_t oword;
365	unsigned int byte;
366	int offset;
367	int rc;
368
369	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
370	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
371
372	byte = 0;
373
374	/* Write toeplitz hash key 3 */
375	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
376	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
377	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
378	    offset > 0 && byte < n;
379	    --offset)
380		oword.eo_u8[offset - 1] = key[byte++];
381
382	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
383
384	/* Write toeplitz hash key 2 */
385	EFX_ZERO_OWORD(oword);
386	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
387	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
388	    offset > 0 && byte < n;
389	    --offset)
390		oword.eo_u8[offset - 1] = key[byte++];
391
392	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
393
394	/* Write toeplitz hash key 1 */
395	EFX_ZERO_OWORD(oword);
396	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
397	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
398	    offset > 0 && byte < n;
399	    --offset)
400		oword.eo_u8[offset - 1] = key[byte++];
401
402	EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
403
404	byte = 0;
405
406	/* Verify toeplitz hash key 3 */
407	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
408	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
409	    FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
410	    offset > 0 && byte < n;
411	    --offset) {
412		if (oword.eo_u8[offset - 1] != key[byte++]) {
413			rc = EFAULT;
414			goto fail1;
415		}
416	}
417
418	/* Verify toeplitz hash key 2 */
419	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
420	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
421	    FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
422	    offset > 0 && byte < n;
423	    --offset) {
424		if (oword.eo_u8[offset - 1] != key[byte++]) {
425			rc = EFAULT;
426			goto fail2;
427		}
428	}
429
430	/* Verify toeplitz hash key 1 */
431	EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
432	for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
433	    FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
434	    offset > 0 && byte < n;
435	    --offset) {
436		if (oword.eo_u8[offset - 1] != key[byte++]) {
437			rc = EFAULT;
438			goto fail3;
439		}
440	}
441
442	return (0);
443
444fail3:
445	EFSYS_PROBE(fail3);
446fail2:
447	EFSYS_PROBE(fail2);
448fail1:
449	EFSYS_PROBE1(fail1, int, rc);
450
451	return (rc);
452}
453#endif
454
455#if EFSYS_OPT_RX_SCALE
456	__checkReturn	int
457efx_rx_scale_tbl_set(
458	__in		efx_nic_t *enp,
459	__in_ecount(n)	unsigned int *table,
460	__in		size_t n)
461{
462	efx_oword_t oword;
463	int index;
464	int rc;
465
466	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
467	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
468
469	EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
470	EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
471
472	if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
473		rc = EINVAL;
474		goto fail1;
475	}
476
477	for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
478		uint32_t byte;
479
480		/* Calculate the entry to place in the table */
481		byte = (uint32_t)table[index % n];
482
483		EFSYS_PROBE2(table, int, index, uint32_t, byte);
484
485		EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
486
487		/* Write the table */
488		EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
489				    index, &oword);
490	}
491
492	for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
493		uint32_t byte;
494
495		/* Determine if we're starting a new batch */
496		byte = (uint32_t)table[index % n];
497
498		/* Read the table */
499		EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
500				    index, &oword);
501
502		/* Verify the entry */
503		if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
504			rc = EFAULT;
505			goto fail2;
506		}
507	}
508
509	return (0);
510
511fail2:
512	EFSYS_PROBE(fail2);
513fail1:
514	EFSYS_PROBE1(fail1, int, rc);
515
516	return (rc);
517}
518#endif
519
520#if EFSYS_OPT_FILTER
521extern	__checkReturn	int
522efx_rx_filter_insert(
523	__in		efx_rxq_t *erp,
524	__inout		efx_filter_spec_t *spec)
525{
526	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
527	EFSYS_ASSERT3P(spec, !=, NULL);
528
529	spec->efs_dmaq_id = (uint16_t)erp->er_index;
530	return efx_filter_insert_filter(erp->er_enp, spec, B_FALSE);
531}
532#endif
533
534#if EFSYS_OPT_FILTER
535extern	__checkReturn	int
536efx_rx_filter_remove(
537	__in		efx_rxq_t *erp,
538	__inout		efx_filter_spec_t *spec)
539{
540	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
541	EFSYS_ASSERT3P(spec, !=, NULL);
542
543	spec->efs_dmaq_id = (uint16_t)erp->er_index;
544	return efx_filter_remove_filter(erp->er_enp, spec);
545}
546#endif
547
548extern			void
549efx_rx_qpost(
550	__in		efx_rxq_t *erp,
551	__in_ecount(n)	efsys_dma_addr_t *addrp,
552	__in		size_t size,
553	__in		unsigned int n,
554	__in		unsigned int completed,
555	__in		unsigned int added)
556{
557	efx_qword_t qword;
558	unsigned int i;
559	unsigned int offset;
560	unsigned int id;
561
562	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
563
564	/* The client driver must not overfill the queue */
565	EFSYS_ASSERT3U(added - completed + n, <=,
566	    EFX_RXQ_LIMIT(erp->er_mask + 1));
567
568	id = added & (erp->er_mask);
569	for (i = 0; i < n; i++) {
570		EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
571		    unsigned int, id, efsys_dma_addr_t, addrp[i],
572		    size_t, size);
573
574		EFX_POPULATE_QWORD_3(qword,
575		    FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
576		    FSF_AZ_RX_KER_BUF_ADDR_DW0,
577		    (uint32_t)(addrp[i] & 0xffffffff),
578		    FSF_AZ_RX_KER_BUF_ADDR_DW1,
579		    (uint32_t)(addrp[i] >> 32));
580
581		offset = id * sizeof (efx_qword_t);
582		EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
583
584		id = (id + 1) & (erp->er_mask);
585	}
586}
587
588		void
589efx_rx_qpush(
590	__in	efx_rxq_t *erp,
591	__in	unsigned int added)
592{
593	efx_nic_t *enp = erp->er_enp;
594	uint32_t wptr;
595	efx_oword_t oword;
596	efx_dword_t dword;
597
598	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
599
600	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
601	EFSYS_PIO_WRITE_BARRIER();
602
603	/* Push the populated descriptors out */
604	wptr = added & erp->er_mask;
605
606	EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
607
608	/* Only write the third DWORD */
609	EFX_POPULATE_DWORD_1(dword,
610	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
611	EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
612			    erp->er_index, &dword, B_FALSE);
613}
614
615		void
616efx_rx_qflush(
617	__in	efx_rxq_t *erp)
618{
619	efx_nic_t *enp = erp->er_enp;
620	efx_oword_t oword;
621	uint32_t label;
622
623	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
624
625	label = erp->er_index;
626
627	/* Flush the queue */
628	EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
629	    FRF_AZ_RX_FLUSH_DESCQ, label);
630	EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
631}
632
633		void
634efx_rx_qenable(
635	__in	efx_rxq_t *erp)
636{
637	efx_nic_t *enp = erp->er_enp;
638	efx_oword_t oword;
639
640	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
641
642	EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
643			    erp->er_index, &oword);
644
645	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
646	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
647	EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
648
649	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
650			    erp->er_index, &oword);
651}
652
653	__checkReturn	int
654efx_rx_qcreate(
655	__in		efx_nic_t *enp,
656	__in		unsigned int index,
657	__in		unsigned int label,
658	__in		efx_rxq_type_t type,
659	__in		efsys_mem_t *esmp,
660	__in		size_t n,
661	__in		uint32_t id,
662	__in		efx_evq_t *eep,
663	__deref_out	efx_rxq_t **erpp)
664{
665	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
666	efx_rxq_t *erp;
667	efx_oword_t oword;
668	uint32_t size;
669	boolean_t split;
670	boolean_t jumbo;
671	int rc;
672
673	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
674	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
675
676	EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
677	EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
678	EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
679
680	if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
681		rc = EINVAL;
682		goto fail1;
683	}
684	if (index >= encp->enc_rxq_limit) {
685		rc = EINVAL;
686		goto fail2;
687	}
688	for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
689	    size++)
690		if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
691			break;
692	if (id + (1 << size) >= encp->enc_buftbl_limit) {
693		rc = EINVAL;
694		goto fail3;
695	}
696
697	switch (type) {
698	case EFX_RXQ_TYPE_DEFAULT:
699		split = B_FALSE;
700		jumbo = B_FALSE;
701		break;
702
703#if EFSYS_OPT_RX_HDR_SPLIT
704	case EFX_RXQ_TYPE_SPLIT_HEADER:
705		if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
706			rc = EINVAL;
707			goto fail4;
708		}
709		split = B_TRUE;
710		jumbo = B_TRUE;
711		break;
712
713	case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
714		if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
715			rc = EINVAL;
716			goto fail4;
717		}
718		split = B_FALSE;
719		jumbo = B_TRUE;
720		break;
721#endif	/* EFSYS_OPT_RX_HDR_SPLIT */
722
723#if EFSYS_OPT_RX_SCATTER
724	case EFX_RXQ_TYPE_SCATTER:
725		if (enp->en_family < EFX_FAMILY_SIENA) {
726			rc = EINVAL;
727			goto fail4;
728		}
729		split = B_FALSE;
730		jumbo = B_TRUE;
731		break;
732#endif	/* EFSYS_OPT_RX_SCATTER */
733
734	default:
735		rc = EINVAL;
736		goto fail4;
737	}
738
739	/* Allocate an RXQ object */
740	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
741
742	if (erp == NULL) {
743		rc = ENOMEM;
744		goto fail5;
745	}
746
747	erp->er_magic = EFX_RXQ_MAGIC;
748	erp->er_enp = enp;
749	erp->er_index = index;
750	erp->er_mask = n - 1;
751	erp->er_esmp = esmp;
752
753	/* Set up the new descriptor queue */
754	EFX_POPULATE_OWORD_10(oword,
755	    FRF_CZ_RX_HDR_SPLIT, split,
756	    FRF_AZ_RX_ISCSI_DDIG_EN, 0,
757	    FRF_AZ_RX_ISCSI_HDIG_EN, 0,
758	    FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
759	    FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
760	    FRF_AZ_RX_DESCQ_OWNER_ID, 0,
761	    FRF_AZ_RX_DESCQ_LABEL, label,
762	    FRF_AZ_RX_DESCQ_SIZE, size,
763	    FRF_AZ_RX_DESCQ_TYPE, 0,
764	    FRF_AZ_RX_DESCQ_JUMBO, jumbo);
765
766	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
767			    erp->er_index, &oword);
768
769	enp->en_rx_qcount++;
770	*erpp = erp;
771	return (0);
772
773fail5:
774	EFSYS_PROBE(fail5);
775fail4:
776	EFSYS_PROBE(fail4);
777fail3:
778	EFSYS_PROBE(fail3);
779fail2:
780	EFSYS_PROBE(fail2);
781fail1:
782	EFSYS_PROBE1(fail1, int, rc);
783
784	return (rc);
785}
786
787		void
788efx_rx_qdestroy(
789	__in	efx_rxq_t *erp)
790{
791	efx_nic_t *enp = erp->er_enp;
792	efx_oword_t oword;
793
794	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
795
796	EFSYS_ASSERT(enp->en_rx_qcount != 0);
797	--enp->en_rx_qcount;
798
799	/* Purge descriptor queue */
800	EFX_ZERO_OWORD(oword);
801
802	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
803			    erp->er_index, &oword);
804
805	/* Free the RXQ object */
806	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
807}
808
809		void
810efx_rx_fini(
811	__in	efx_nic_t *enp)
812{
813	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
814	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
815	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
816	EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
817
818	enp->en_mod_flags &= ~EFX_MOD_RX;
819}
820