28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35 __checkReturn int
36efx_rx_init(
37 __in efx_nic_t *enp)
38{
39 efx_oword_t oword;
40 unsigned int index;
41 int rc;
42
43 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
44 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
45
46 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
47 rc = EINVAL;
48 goto fail1;
49 }
50
51 if (enp->en_mod_flags & EFX_MOD_RX) {
52 rc = EINVAL;
53 goto fail2;
54 }
55
56 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
57
58 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
59 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
60 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
61 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
62 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
63 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
64 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
65
66 /* Zero the RSS table */
67 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
68 index++) {
69 EFX_ZERO_OWORD(oword);
70 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
71 index, &oword);
72 }
73
74 enp->en_mod_flags |= EFX_MOD_RX;
75 return (0);
76
77fail2:
78 EFSYS_PROBE(fail2);
79fail1:
80 EFSYS_PROBE1(fail1, int, rc);
81
82 return (rc);
83}
84
85#if EFSYS_OPT_RX_HDR_SPLIT
86 __checkReturn int
87efx_rx_hdr_split_enable(
88 __in efx_nic_t *enp,
89 __in unsigned int hdr_buf_size,
90 __in unsigned int pld_buf_size)
91{
92 unsigned int nhdr32;
93 unsigned int npld32;
94 efx_oword_t oword;
95 int rc;
96
97 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
98 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
99 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
100
101 nhdr32 = hdr_buf_size / 32;
102 if ((nhdr32 == 0) ||
103 (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
104 ((hdr_buf_size % 32) != 0)) {
105 rc = EINVAL;
106 goto fail1;
107 }
108
109 npld32 = pld_buf_size / 32;
110 if ((npld32 == 0) ||
111 (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
112 ((pld_buf_size % 32) != 0)) {
113 rc = EINVAL;
114 goto fail2;
115 }
116
117 if (enp->en_rx_qcount > 0) {
118 rc = EBUSY;
119 goto fail3;
120 }
121
122 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
123
124 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
125 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
126 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
127
128 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
129
130 return (0);
131
132fail3:
133 EFSYS_PROBE(fail3);
134fail2:
135 EFSYS_PROBE(fail2);
136fail1:
137 EFSYS_PROBE1(fail1, int, rc);
138
139 return (rc);
140}
141#endif /* EFSYS_OPT_RX_HDR_SPLIT */
142
143
144#if EFSYS_OPT_RX_SCATTER
145 __checkReturn int
146efx_rx_scatter_enable(
147 __in efx_nic_t *enp,
148 __in unsigned int buf_size)
149{
150 unsigned int nbuf32;
151 efx_oword_t oword;
152 int rc;
153
154 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
155 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
156 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
157
158 nbuf32 = buf_size / 32;
159 if ((nbuf32 == 0) ||
160 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
161 ((buf_size % 32) != 0)) {
162 rc = EINVAL;
163 goto fail1;
164 }
165
166 if (enp->en_rx_qcount > 0) {
167 rc = EBUSY;
168 goto fail2;
169 }
170
171 /* Set scatter buffer size */
172 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
173 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
174 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
175
176 /* Enable scatter for packets not matching a filter */
177 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
178 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
179 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
180
181 return (0);
182
183fail2:
184 EFSYS_PROBE(fail2);
185fail1:
186 EFSYS_PROBE1(fail1, int, rc);
187
188 return (rc);
189}
190#endif /* EFSYS_OPT_RX_SCATTER */
191
192
193#define EFX_RX_LFSR_HASH(_enp, _insert) \
194 do { \
195 efx_oword_t oword; \
196 \
197 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
198 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
199 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
200 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
201 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
202 (_insert) ? 1 : 0); \
203 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
204 \
205 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
206 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
207 &oword); \
208 EFX_SET_OWORD_FIELD(oword, \
209 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
210 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
211 &oword); \
212 } \
213 \
214 _NOTE(CONSTANTCONDITION) \
215 } while (B_FALSE)
216
217#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
218 do { \
219 efx_oword_t oword; \
220 \
221 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
222 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
223 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
224 (_ip) ? 1 : 0); \
225 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
226 (_tcp) ? 0 : 1); \
227 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
228 (_insert) ? 1 : 0); \
229 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
230 \
231 _NOTE(CONSTANTCONDITION) \
232 } while (B_FALSE)
233
234#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
235 do { \
236 efx_oword_t oword; \
237 \
238 if ((_enp)->en_family == EFX_FAMILY_FALCON) { \
239 (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \
240 break; \
241 } \
242 \
243 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
244 EFX_SET_OWORD_FIELD(oword, \
245 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
246 EFX_SET_OWORD_FIELD(oword, \
247 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
248 EFX_SET_OWORD_FIELD(oword, \
249 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
250 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
251 \
252 (_rc) = 0; \
253 \
254 _NOTE(CONSTANTCONDITION) \
255 } while (B_FALSE)
256
257
258#if EFSYS_OPT_RX_SCALE
259 __checkReturn int
260efx_rx_scale_mode_set(
261 __in efx_nic_t *enp,
262 __in efx_rx_hash_alg_t alg,
263 __in efx_rx_hash_type_t type,
264 __in boolean_t insert)
265{
266 int rc;
267
268 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
269 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
270 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
271
272 switch (alg) {
273 case EFX_RX_HASHALG_LFSR:
274 EFX_RX_LFSR_HASH(enp, insert);
275 break;
276
277 case EFX_RX_HASHALG_TOEPLITZ:
278 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
279 type & (1 << EFX_RX_HASH_IPV4),
280 type & (1 << EFX_RX_HASH_TCPIPV4));
281
282 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
283 type & (1 << EFX_RX_HASH_IPV6),
284 type & (1 << EFX_RX_HASH_TCPIPV6),
285 rc);
286 if (rc != 0)
287 goto fail1;
288
289 break;
290
291 default:
292 rc = EINVAL;
293 goto fail2;
294 }
295
296 return (0);
297
298fail2:
299 EFSYS_PROBE(fail2);
300fail1:
301 EFSYS_PROBE1(fail1, int, rc);
302
303 EFX_RX_LFSR_HASH(enp, B_FALSE);
304
305 return (rc);
306}
307#endif
308
309#if EFSYS_OPT_RX_SCALE
310 __checkReturn int
311efx_rx_scale_toeplitz_ipv4_key_set(
312 __in efx_nic_t *enp,
313 __in_ecount(n) uint8_t *key,
314 __in size_t n)
315{
316 efx_oword_t oword;
317 unsigned int byte;
318 unsigned int offset;
319 int rc;
320
321 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
322 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
323
324 byte = 0;
325
326 /* Write toeplitz hash key */
327 EFX_ZERO_OWORD(oword);
328 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
329 offset > 0 && byte < n;
330 --offset)
331 oword.eo_u8[offset - 1] = key[byte++];
332
333 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
334
335 byte = 0;
336
337 /* Verify toeplitz hash key */
338 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
339 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
340 offset > 0 && byte < n;
341 --offset) {
342 if (oword.eo_u8[offset - 1] != key[byte++]) {
343 rc = EFAULT;
344 goto fail1;
345 }
346 }
347
348 return (0);
349
350fail1:
351 EFSYS_PROBE1(fail1, int, rc);
352
353 return (rc);
354}
355#endif
356
357#if EFSYS_OPT_RX_SCALE
358 __checkReturn int
359efx_rx_scale_toeplitz_ipv6_key_set(
360 __in efx_nic_t *enp,
361 __in_ecount(n) uint8_t *key,
362 __in size_t n)
363{
364 efx_oword_t oword;
365 unsigned int byte;
366 int offset;
367 int rc;
368
369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
371
372 byte = 0;
373
374 /* Write toeplitz hash key 3 */
375 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
376 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
377 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
378 offset > 0 && byte < n;
379 --offset)
380 oword.eo_u8[offset - 1] = key[byte++];
381
382 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
383
384 /* Write toeplitz hash key 2 */
385 EFX_ZERO_OWORD(oword);
386 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
387 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
388 offset > 0 && byte < n;
389 --offset)
390 oword.eo_u8[offset - 1] = key[byte++];
391
392 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
393
394 /* Write toeplitz hash key 1 */
395 EFX_ZERO_OWORD(oword);
396 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
397 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
398 offset > 0 && byte < n;
399 --offset)
400 oword.eo_u8[offset - 1] = key[byte++];
401
402 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
403
404 byte = 0;
405
406 /* Verify toeplitz hash key 3 */
407 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
408 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
409 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
410 offset > 0 && byte < n;
411 --offset) {
412 if (oword.eo_u8[offset - 1] != key[byte++]) {
413 rc = EFAULT;
414 goto fail1;
415 }
416 }
417
418 /* Verify toeplitz hash key 2 */
419 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
420 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
421 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
422 offset > 0 && byte < n;
423 --offset) {
424 if (oword.eo_u8[offset - 1] != key[byte++]) {
425 rc = EFAULT;
426 goto fail2;
427 }
428 }
429
430 /* Verify toeplitz hash key 1 */
431 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
432 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
433 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
434 offset > 0 && byte < n;
435 --offset) {
436 if (oword.eo_u8[offset - 1] != key[byte++]) {
437 rc = EFAULT;
438 goto fail3;
439 }
440 }
441
442 return (0);
443
444fail3:
445 EFSYS_PROBE(fail3);
446fail2:
447 EFSYS_PROBE(fail2);
448fail1:
449 EFSYS_PROBE1(fail1, int, rc);
450
451 return (rc);
452}
453#endif
454
455#if EFSYS_OPT_RX_SCALE
456 __checkReturn int
457efx_rx_scale_tbl_set(
458 __in efx_nic_t *enp,
459 __in_ecount(n) unsigned int *table,
460 __in size_t n)
461{
462 efx_oword_t oword;
463 int index;
464 int rc;
465
466 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
467 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
468
469 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
470 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
471
472 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
473 rc = EINVAL;
474 goto fail1;
475 }
476
477 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
478 uint32_t byte;
479
480 /* Calculate the entry to place in the table */
481 byte = (uint32_t)table[index % n];
482
483 EFSYS_PROBE2(table, int, index, uint32_t, byte);
484
485 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
486
487 /* Write the table */
488 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
489 index, &oword);
490 }
491
492 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
493 uint32_t byte;
494
495 /* Determine if we're starting a new batch */
496 byte = (uint32_t)table[index % n];
497
498 /* Read the table */
499 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
500 index, &oword);
501
502 /* Verify the entry */
503 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
504 rc = EFAULT;
505 goto fail2;
506 }
507 }
508
509 return (0);
510
511fail2:
512 EFSYS_PROBE(fail2);
513fail1:
514 EFSYS_PROBE1(fail1, int, rc);
515
516 return (rc);
517}
518#endif
519
520#if EFSYS_OPT_FILTER
521extern __checkReturn int
522efx_rx_filter_insert(
523 __in efx_rxq_t *erp,
524 __inout efx_filter_spec_t *spec)
525{
526 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
527 EFSYS_ASSERT3P(spec, !=, NULL);
528
529 spec->efs_dmaq_id = (uint16_t)erp->er_index;
| 28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35 __checkReturn int
36efx_rx_init(
37 __in efx_nic_t *enp)
38{
39 efx_oword_t oword;
40 unsigned int index;
41 int rc;
42
43 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
44 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
45
46 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
47 rc = EINVAL;
48 goto fail1;
49 }
50
51 if (enp->en_mod_flags & EFX_MOD_RX) {
52 rc = EINVAL;
53 goto fail2;
54 }
55
56 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
57
58 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
59 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
60 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
61 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
62 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
63 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
64 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
65
66 /* Zero the RSS table */
67 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
68 index++) {
69 EFX_ZERO_OWORD(oword);
70 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
71 index, &oword);
72 }
73
74 enp->en_mod_flags |= EFX_MOD_RX;
75 return (0);
76
77fail2:
78 EFSYS_PROBE(fail2);
79fail1:
80 EFSYS_PROBE1(fail1, int, rc);
81
82 return (rc);
83}
84
85#if EFSYS_OPT_RX_HDR_SPLIT
86 __checkReturn int
87efx_rx_hdr_split_enable(
88 __in efx_nic_t *enp,
89 __in unsigned int hdr_buf_size,
90 __in unsigned int pld_buf_size)
91{
92 unsigned int nhdr32;
93 unsigned int npld32;
94 efx_oword_t oword;
95 int rc;
96
97 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
98 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
99 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
100
101 nhdr32 = hdr_buf_size / 32;
102 if ((nhdr32 == 0) ||
103 (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
104 ((hdr_buf_size % 32) != 0)) {
105 rc = EINVAL;
106 goto fail1;
107 }
108
109 npld32 = pld_buf_size / 32;
110 if ((npld32 == 0) ||
111 (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
112 ((pld_buf_size % 32) != 0)) {
113 rc = EINVAL;
114 goto fail2;
115 }
116
117 if (enp->en_rx_qcount > 0) {
118 rc = EBUSY;
119 goto fail3;
120 }
121
122 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
123
124 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
125 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
126 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
127
128 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
129
130 return (0);
131
132fail3:
133 EFSYS_PROBE(fail3);
134fail2:
135 EFSYS_PROBE(fail2);
136fail1:
137 EFSYS_PROBE1(fail1, int, rc);
138
139 return (rc);
140}
141#endif /* EFSYS_OPT_RX_HDR_SPLIT */
142
143
144#if EFSYS_OPT_RX_SCATTER
145 __checkReturn int
146efx_rx_scatter_enable(
147 __in efx_nic_t *enp,
148 __in unsigned int buf_size)
149{
150 unsigned int nbuf32;
151 efx_oword_t oword;
152 int rc;
153
154 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
155 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
156 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
157
158 nbuf32 = buf_size / 32;
159 if ((nbuf32 == 0) ||
160 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
161 ((buf_size % 32) != 0)) {
162 rc = EINVAL;
163 goto fail1;
164 }
165
166 if (enp->en_rx_qcount > 0) {
167 rc = EBUSY;
168 goto fail2;
169 }
170
171 /* Set scatter buffer size */
172 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
173 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
174 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
175
176 /* Enable scatter for packets not matching a filter */
177 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
178 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
179 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
180
181 return (0);
182
183fail2:
184 EFSYS_PROBE(fail2);
185fail1:
186 EFSYS_PROBE1(fail1, int, rc);
187
188 return (rc);
189}
190#endif /* EFSYS_OPT_RX_SCATTER */
191
192
193#define EFX_RX_LFSR_HASH(_enp, _insert) \
194 do { \
195 efx_oword_t oword; \
196 \
197 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
198 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
199 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
200 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
201 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
202 (_insert) ? 1 : 0); \
203 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
204 \
205 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
206 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
207 &oword); \
208 EFX_SET_OWORD_FIELD(oword, \
209 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
210 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
211 &oword); \
212 } \
213 \
214 _NOTE(CONSTANTCONDITION) \
215 } while (B_FALSE)
216
217#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
218 do { \
219 efx_oword_t oword; \
220 \
221 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
222 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
223 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
224 (_ip) ? 1 : 0); \
225 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
226 (_tcp) ? 0 : 1); \
227 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
228 (_insert) ? 1 : 0); \
229 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
230 \
231 _NOTE(CONSTANTCONDITION) \
232 } while (B_FALSE)
233
234#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
235 do { \
236 efx_oword_t oword; \
237 \
238 if ((_enp)->en_family == EFX_FAMILY_FALCON) { \
239 (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \
240 break; \
241 } \
242 \
243 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
244 EFX_SET_OWORD_FIELD(oword, \
245 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
246 EFX_SET_OWORD_FIELD(oword, \
247 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
248 EFX_SET_OWORD_FIELD(oword, \
249 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
250 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
251 \
252 (_rc) = 0; \
253 \
254 _NOTE(CONSTANTCONDITION) \
255 } while (B_FALSE)
256
257
258#if EFSYS_OPT_RX_SCALE
259 __checkReturn int
260efx_rx_scale_mode_set(
261 __in efx_nic_t *enp,
262 __in efx_rx_hash_alg_t alg,
263 __in efx_rx_hash_type_t type,
264 __in boolean_t insert)
265{
266 int rc;
267
268 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
269 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
270 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
271
272 switch (alg) {
273 case EFX_RX_HASHALG_LFSR:
274 EFX_RX_LFSR_HASH(enp, insert);
275 break;
276
277 case EFX_RX_HASHALG_TOEPLITZ:
278 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
279 type & (1 << EFX_RX_HASH_IPV4),
280 type & (1 << EFX_RX_HASH_TCPIPV4));
281
282 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
283 type & (1 << EFX_RX_HASH_IPV6),
284 type & (1 << EFX_RX_HASH_TCPIPV6),
285 rc);
286 if (rc != 0)
287 goto fail1;
288
289 break;
290
291 default:
292 rc = EINVAL;
293 goto fail2;
294 }
295
296 return (0);
297
298fail2:
299 EFSYS_PROBE(fail2);
300fail1:
301 EFSYS_PROBE1(fail1, int, rc);
302
303 EFX_RX_LFSR_HASH(enp, B_FALSE);
304
305 return (rc);
306}
307#endif
308
309#if EFSYS_OPT_RX_SCALE
310 __checkReturn int
311efx_rx_scale_toeplitz_ipv4_key_set(
312 __in efx_nic_t *enp,
313 __in_ecount(n) uint8_t *key,
314 __in size_t n)
315{
316 efx_oword_t oword;
317 unsigned int byte;
318 unsigned int offset;
319 int rc;
320
321 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
322 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
323
324 byte = 0;
325
326 /* Write toeplitz hash key */
327 EFX_ZERO_OWORD(oword);
328 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
329 offset > 0 && byte < n;
330 --offset)
331 oword.eo_u8[offset - 1] = key[byte++];
332
333 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
334
335 byte = 0;
336
337 /* Verify toeplitz hash key */
338 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
339 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
340 offset > 0 && byte < n;
341 --offset) {
342 if (oword.eo_u8[offset - 1] != key[byte++]) {
343 rc = EFAULT;
344 goto fail1;
345 }
346 }
347
348 return (0);
349
350fail1:
351 EFSYS_PROBE1(fail1, int, rc);
352
353 return (rc);
354}
355#endif
356
357#if EFSYS_OPT_RX_SCALE
358 __checkReturn int
359efx_rx_scale_toeplitz_ipv6_key_set(
360 __in efx_nic_t *enp,
361 __in_ecount(n) uint8_t *key,
362 __in size_t n)
363{
364 efx_oword_t oword;
365 unsigned int byte;
366 int offset;
367 int rc;
368
369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
371
372 byte = 0;
373
374 /* Write toeplitz hash key 3 */
375 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
376 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
377 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
378 offset > 0 && byte < n;
379 --offset)
380 oword.eo_u8[offset - 1] = key[byte++];
381
382 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
383
384 /* Write toeplitz hash key 2 */
385 EFX_ZERO_OWORD(oword);
386 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
387 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
388 offset > 0 && byte < n;
389 --offset)
390 oword.eo_u8[offset - 1] = key[byte++];
391
392 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
393
394 /* Write toeplitz hash key 1 */
395 EFX_ZERO_OWORD(oword);
396 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
397 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
398 offset > 0 && byte < n;
399 --offset)
400 oword.eo_u8[offset - 1] = key[byte++];
401
402 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
403
404 byte = 0;
405
406 /* Verify toeplitz hash key 3 */
407 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
408 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
409 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
410 offset > 0 && byte < n;
411 --offset) {
412 if (oword.eo_u8[offset - 1] != key[byte++]) {
413 rc = EFAULT;
414 goto fail1;
415 }
416 }
417
418 /* Verify toeplitz hash key 2 */
419 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
420 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
421 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
422 offset > 0 && byte < n;
423 --offset) {
424 if (oword.eo_u8[offset - 1] != key[byte++]) {
425 rc = EFAULT;
426 goto fail2;
427 }
428 }
429
430 /* Verify toeplitz hash key 1 */
431 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
432 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
433 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
434 offset > 0 && byte < n;
435 --offset) {
436 if (oword.eo_u8[offset - 1] != key[byte++]) {
437 rc = EFAULT;
438 goto fail3;
439 }
440 }
441
442 return (0);
443
444fail3:
445 EFSYS_PROBE(fail3);
446fail2:
447 EFSYS_PROBE(fail2);
448fail1:
449 EFSYS_PROBE1(fail1, int, rc);
450
451 return (rc);
452}
453#endif
454
455#if EFSYS_OPT_RX_SCALE
456 __checkReturn int
457efx_rx_scale_tbl_set(
458 __in efx_nic_t *enp,
459 __in_ecount(n) unsigned int *table,
460 __in size_t n)
461{
462 efx_oword_t oword;
463 int index;
464 int rc;
465
466 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
467 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
468
469 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
470 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
471
472 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
473 rc = EINVAL;
474 goto fail1;
475 }
476
477 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
478 uint32_t byte;
479
480 /* Calculate the entry to place in the table */
481 byte = (uint32_t)table[index % n];
482
483 EFSYS_PROBE2(table, int, index, uint32_t, byte);
484
485 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
486
487 /* Write the table */
488 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
489 index, &oword);
490 }
491
492 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
493 uint32_t byte;
494
495 /* Determine if we're starting a new batch */
496 byte = (uint32_t)table[index % n];
497
498 /* Read the table */
499 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
500 index, &oword);
501
502 /* Verify the entry */
503 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
504 rc = EFAULT;
505 goto fail2;
506 }
507 }
508
509 return (0);
510
511fail2:
512 EFSYS_PROBE(fail2);
513fail1:
514 EFSYS_PROBE1(fail1, int, rc);
515
516 return (rc);
517}
518#endif
519
520#if EFSYS_OPT_FILTER
521extern __checkReturn int
522efx_rx_filter_insert(
523 __in efx_rxq_t *erp,
524 __inout efx_filter_spec_t *spec)
525{
526 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
527 EFSYS_ASSERT3P(spec, !=, NULL);
528
529 spec->efs_dmaq_id = (uint16_t)erp->er_index;
|
545}
546#endif
547
548extern void
549efx_rx_qpost(
550 __in efx_rxq_t *erp,
551 __in_ecount(n) efsys_dma_addr_t *addrp,
552 __in size_t size,
553 __in unsigned int n,
554 __in unsigned int completed,
555 __in unsigned int added)
556{
557 efx_qword_t qword;
558 unsigned int i;
559 unsigned int offset;
560 unsigned int id;
561
562 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
563
564 /* The client driver must not overfill the queue */
565 EFSYS_ASSERT3U(added - completed + n, <=,
566 EFX_RXQ_LIMIT(erp->er_mask + 1));
567
568 id = added & (erp->er_mask);
569 for (i = 0; i < n; i++) {
570 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
571 unsigned int, id, efsys_dma_addr_t, addrp[i],
572 size_t, size);
573
574 EFX_POPULATE_QWORD_3(qword,
575 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
576 FSF_AZ_RX_KER_BUF_ADDR_DW0,
577 (uint32_t)(addrp[i] & 0xffffffff),
578 FSF_AZ_RX_KER_BUF_ADDR_DW1,
579 (uint32_t)(addrp[i] >> 32));
580
581 offset = id * sizeof (efx_qword_t);
582 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
583
584 id = (id + 1) & (erp->er_mask);
585 }
586}
587
588 void
589efx_rx_qpush(
590 __in efx_rxq_t *erp,
591 __in unsigned int added)
592{
593 efx_nic_t *enp = erp->er_enp;
594 uint32_t wptr;
595 efx_oword_t oword;
596 efx_dword_t dword;
597
598 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
599
600 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
601 EFSYS_PIO_WRITE_BARRIER();
602
603 /* Push the populated descriptors out */
604 wptr = added & erp->er_mask;
605
606 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
607
608 /* Only write the third DWORD */
609 EFX_POPULATE_DWORD_1(dword,
610 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
611 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
612 erp->er_index, &dword, B_FALSE);
613}
614
615 void
616efx_rx_qflush(
617 __in efx_rxq_t *erp)
618{
619 efx_nic_t *enp = erp->er_enp;
620 efx_oword_t oword;
621 uint32_t label;
622
623 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
624
625 label = erp->er_index;
626
627 /* Flush the queue */
628 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
629 FRF_AZ_RX_FLUSH_DESCQ, label);
630 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
631}
632
633 void
634efx_rx_qenable(
635 __in efx_rxq_t *erp)
636{
637 efx_nic_t *enp = erp->er_enp;
638 efx_oword_t oword;
639
640 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
641
642 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
643 erp->er_index, &oword);
644
645 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
646 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
647 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
648
649 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
650 erp->er_index, &oword);
651}
652
653 __checkReturn int
654efx_rx_qcreate(
655 __in efx_nic_t *enp,
656 __in unsigned int index,
657 __in unsigned int label,
658 __in efx_rxq_type_t type,
659 __in efsys_mem_t *esmp,
660 __in size_t n,
661 __in uint32_t id,
662 __in efx_evq_t *eep,
663 __deref_out efx_rxq_t **erpp)
664{
665 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
666 efx_rxq_t *erp;
667 efx_oword_t oword;
668 uint32_t size;
669 boolean_t split;
670 boolean_t jumbo;
671 int rc;
672
673 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
674 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
675
| 545}
546#endif
547
548extern void
549efx_rx_qpost(
550 __in efx_rxq_t *erp,
551 __in_ecount(n) efsys_dma_addr_t *addrp,
552 __in size_t size,
553 __in unsigned int n,
554 __in unsigned int completed,
555 __in unsigned int added)
556{
557 efx_qword_t qword;
558 unsigned int i;
559 unsigned int offset;
560 unsigned int id;
561
562 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
563
564 /* The client driver must not overfill the queue */
565 EFSYS_ASSERT3U(added - completed + n, <=,
566 EFX_RXQ_LIMIT(erp->er_mask + 1));
567
568 id = added & (erp->er_mask);
569 for (i = 0; i < n; i++) {
570 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
571 unsigned int, id, efsys_dma_addr_t, addrp[i],
572 size_t, size);
573
574 EFX_POPULATE_QWORD_3(qword,
575 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
576 FSF_AZ_RX_KER_BUF_ADDR_DW0,
577 (uint32_t)(addrp[i] & 0xffffffff),
578 FSF_AZ_RX_KER_BUF_ADDR_DW1,
579 (uint32_t)(addrp[i] >> 32));
580
581 offset = id * sizeof (efx_qword_t);
582 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
583
584 id = (id + 1) & (erp->er_mask);
585 }
586}
587
588 void
589efx_rx_qpush(
590 __in efx_rxq_t *erp,
591 __in unsigned int added)
592{
593 efx_nic_t *enp = erp->er_enp;
594 uint32_t wptr;
595 efx_oword_t oword;
596 efx_dword_t dword;
597
598 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
599
600 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
601 EFSYS_PIO_WRITE_BARRIER();
602
603 /* Push the populated descriptors out */
604 wptr = added & erp->er_mask;
605
606 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
607
608 /* Only write the third DWORD */
609 EFX_POPULATE_DWORD_1(dword,
610 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
611 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
612 erp->er_index, &dword, B_FALSE);
613}
614
615 void
616efx_rx_qflush(
617 __in efx_rxq_t *erp)
618{
619 efx_nic_t *enp = erp->er_enp;
620 efx_oword_t oword;
621 uint32_t label;
622
623 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
624
625 label = erp->er_index;
626
627 /* Flush the queue */
628 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
629 FRF_AZ_RX_FLUSH_DESCQ, label);
630 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
631}
632
633 void
634efx_rx_qenable(
635 __in efx_rxq_t *erp)
636{
637 efx_nic_t *enp = erp->er_enp;
638 efx_oword_t oword;
639
640 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
641
642 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
643 erp->er_index, &oword);
644
645 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
646 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
647 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
648
649 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
650 erp->er_index, &oword);
651}
652
653 __checkReturn int
654efx_rx_qcreate(
655 __in efx_nic_t *enp,
656 __in unsigned int index,
657 __in unsigned int label,
658 __in efx_rxq_type_t type,
659 __in efsys_mem_t *esmp,
660 __in size_t n,
661 __in uint32_t id,
662 __in efx_evq_t *eep,
663 __deref_out efx_rxq_t **erpp)
664{
665 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
666 efx_rxq_t *erp;
667 efx_oword_t oword;
668 uint32_t size;
669 boolean_t split;
670 boolean_t jumbo;
671 int rc;
672
673 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
674 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
675
|
677 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
678 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
679
680 if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
681 rc = EINVAL;
682 goto fail1;
683 }
684 if (index >= encp->enc_rxq_limit) {
685 rc = EINVAL;
686 goto fail2;
687 }
688 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
689 size++)
690 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
691 break;
692 if (id + (1 << size) >= encp->enc_buftbl_limit) {
693 rc = EINVAL;
694 goto fail3;
695 }
696
697 switch (type) {
698 case EFX_RXQ_TYPE_DEFAULT:
699 split = B_FALSE;
700 jumbo = B_FALSE;
701 break;
702
703#if EFSYS_OPT_RX_HDR_SPLIT
704 case EFX_RXQ_TYPE_SPLIT_HEADER:
705 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
706 rc = EINVAL;
707 goto fail4;
708 }
709 split = B_TRUE;
710 jumbo = B_TRUE;
711 break;
712
713 case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
714 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
715 rc = EINVAL;
716 goto fail4;
717 }
718 split = B_FALSE;
719 jumbo = B_TRUE;
720 break;
721#endif /* EFSYS_OPT_RX_HDR_SPLIT */
722
723#if EFSYS_OPT_RX_SCATTER
724 case EFX_RXQ_TYPE_SCATTER:
725 if (enp->en_family < EFX_FAMILY_SIENA) {
726 rc = EINVAL;
727 goto fail4;
728 }
729 split = B_FALSE;
730 jumbo = B_TRUE;
731 break;
732#endif /* EFSYS_OPT_RX_SCATTER */
733
734 default:
735 rc = EINVAL;
736 goto fail4;
737 }
738
739 /* Allocate an RXQ object */
740 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
741
742 if (erp == NULL) {
743 rc = ENOMEM;
744 goto fail5;
745 }
746
747 erp->er_magic = EFX_RXQ_MAGIC;
748 erp->er_enp = enp;
749 erp->er_index = index;
750 erp->er_mask = n - 1;
751 erp->er_esmp = esmp;
752
753 /* Set up the new descriptor queue */
754 EFX_POPULATE_OWORD_10(oword,
755 FRF_CZ_RX_HDR_SPLIT, split,
756 FRF_AZ_RX_ISCSI_DDIG_EN, 0,
757 FRF_AZ_RX_ISCSI_HDIG_EN, 0,
758 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
759 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
760 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
761 FRF_AZ_RX_DESCQ_LABEL, label,
762 FRF_AZ_RX_DESCQ_SIZE, size,
763 FRF_AZ_RX_DESCQ_TYPE, 0,
764 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
765
766 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
767 erp->er_index, &oword);
768
769 enp->en_rx_qcount++;
770 *erpp = erp;
771 return (0);
772
773fail5:
774 EFSYS_PROBE(fail5);
775fail4:
776 EFSYS_PROBE(fail4);
777fail3:
778 EFSYS_PROBE(fail3);
779fail2:
780 EFSYS_PROBE(fail2);
781fail1:
782 EFSYS_PROBE1(fail1, int, rc);
783
784 return (rc);
785}
786
787 void
788efx_rx_qdestroy(
789 __in efx_rxq_t *erp)
790{
791 efx_nic_t *enp = erp->er_enp;
792 efx_oword_t oword;
793
794 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
795
796 EFSYS_ASSERT(enp->en_rx_qcount != 0);
797 --enp->en_rx_qcount;
798
799 /* Purge descriptor queue */
800 EFX_ZERO_OWORD(oword);
801
802 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
803 erp->er_index, &oword);
804
805 /* Free the RXQ object */
806 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
807}
808
809 void
810efx_rx_fini(
811 __in efx_nic_t *enp)
812{
813 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
814 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
815 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
816 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
817
818 enp->en_mod_flags &= ~EFX_MOD_RX;
819}
| 678 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
679 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
680
681 if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
682 rc = EINVAL;
683 goto fail1;
684 }
685 if (index >= encp->enc_rxq_limit) {
686 rc = EINVAL;
687 goto fail2;
688 }
689 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
690 size++)
691 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
692 break;
693 if (id + (1 << size) >= encp->enc_buftbl_limit) {
694 rc = EINVAL;
695 goto fail3;
696 }
697
698 switch (type) {
699 case EFX_RXQ_TYPE_DEFAULT:
700 split = B_FALSE;
701 jumbo = B_FALSE;
702 break;
703
704#if EFSYS_OPT_RX_HDR_SPLIT
705 case EFX_RXQ_TYPE_SPLIT_HEADER:
706 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
707 rc = EINVAL;
708 goto fail4;
709 }
710 split = B_TRUE;
711 jumbo = B_TRUE;
712 break;
713
714 case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
715 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
716 rc = EINVAL;
717 goto fail4;
718 }
719 split = B_FALSE;
720 jumbo = B_TRUE;
721 break;
722#endif /* EFSYS_OPT_RX_HDR_SPLIT */
723
724#if EFSYS_OPT_RX_SCATTER
725 case EFX_RXQ_TYPE_SCATTER:
726 if (enp->en_family < EFX_FAMILY_SIENA) {
727 rc = EINVAL;
728 goto fail4;
729 }
730 split = B_FALSE;
731 jumbo = B_TRUE;
732 break;
733#endif /* EFSYS_OPT_RX_SCATTER */
734
735 default:
736 rc = EINVAL;
737 goto fail4;
738 }
739
740 /* Allocate an RXQ object */
741 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
742
743 if (erp == NULL) {
744 rc = ENOMEM;
745 goto fail5;
746 }
747
748 erp->er_magic = EFX_RXQ_MAGIC;
749 erp->er_enp = enp;
750 erp->er_index = index;
751 erp->er_mask = n - 1;
752 erp->er_esmp = esmp;
753
754 /* Set up the new descriptor queue */
755 EFX_POPULATE_OWORD_10(oword,
756 FRF_CZ_RX_HDR_SPLIT, split,
757 FRF_AZ_RX_ISCSI_DDIG_EN, 0,
758 FRF_AZ_RX_ISCSI_HDIG_EN, 0,
759 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
760 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
761 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
762 FRF_AZ_RX_DESCQ_LABEL, label,
763 FRF_AZ_RX_DESCQ_SIZE, size,
764 FRF_AZ_RX_DESCQ_TYPE, 0,
765 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
766
767 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
768 erp->er_index, &oword);
769
770 enp->en_rx_qcount++;
771 *erpp = erp;
772 return (0);
773
774fail5:
775 EFSYS_PROBE(fail5);
776fail4:
777 EFSYS_PROBE(fail4);
778fail3:
779 EFSYS_PROBE(fail3);
780fail2:
781 EFSYS_PROBE(fail2);
782fail1:
783 EFSYS_PROBE1(fail1, int, rc);
784
785 return (rc);
786}
787
788 void
789efx_rx_qdestroy(
790 __in efx_rxq_t *erp)
791{
792 efx_nic_t *enp = erp->er_enp;
793 efx_oword_t oword;
794
795 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
796
797 EFSYS_ASSERT(enp->en_rx_qcount != 0);
798 --enp->en_rx_qcount;
799
800 /* Purge descriptor queue */
801 EFX_ZERO_OWORD(oword);
802
803 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
804 erp->er_index, &oword);
805
806 /* Free the RXQ object */
807 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
808}
809
810 void
811efx_rx_fini(
812 __in efx_nic_t *enp)
813{
814 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
815 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
816 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
817 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
818
819 enp->en_mod_flags &= ~EFX_MOD_RX;
820}
|