1/*-
2 * Copyright (c) 2012-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/hunt_rx.c 293752 2016-01-12 13:32:04Z arybchik $");
33
34#include "efsys.h"
35#include "efx.h"
36#include "efx_impl.h"
37
38
39#if EFSYS_OPT_HUNTINGTON
40
41
42static __checkReturn efx_rc_t
43efx_mcdi_init_rxq(
44 __in efx_nic_t *enp,
45 __in uint32_t size,
46 __in uint32_t target_evq,
47 __in uint32_t label,
48 __in uint32_t instance,
49 __in efsys_mem_t *esmp,
50 __in boolean_t disable_scatter)
51{
52 efx_mcdi_req_t req;
53 uint8_t payload[
54 MAX(MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)),
55 MC_CMD_INIT_RXQ_OUT_LEN)];
56 int npages = EFX_RXQ_NBUFS(size);
57 int i;
58 efx_qword_t *dma_addr;
59 uint64_t addr;
60 efx_rc_t rc;
61
62 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
63
64 (void) memset(payload, 0, sizeof (payload));
65 req.emr_cmd = MC_CMD_INIT_RXQ;
66 req.emr_in_buf = payload;
67 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
68 req.emr_out_buf = payload;
69 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
70
71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
74 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
75 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
76 INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
77 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
78 INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
79 INIT_RXQ_IN_CRC_MODE, 0,
80 INIT_RXQ_IN_FLAG_PREFIX, 1,
81 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
83 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
84
85 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
86 addr = EFSYS_MEM_ADDR(esmp);
87
88 for (i = 0; i < npages; i++) {
89 EFX_POPULATE_QWORD_2(*dma_addr,
90 EFX_DWORD_1, (uint32_t)(addr >> 32),
91 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
92
93 dma_addr++;
94 addr += EFX_BUF_SIZE;
95 }
96
97 efx_mcdi_execute(enp, &req);
98
99 if (req.emr_rc != 0) {
100 rc = req.emr_rc;
101 goto fail1;
102 }
103
104 return (0);
105
106fail1:
107 EFSYS_PROBE1(fail1, efx_rc_t, rc);
108
109 return (rc);
110}
111
112static __checkReturn efx_rc_t
113efx_mcdi_fini_rxq(
114 __in efx_nic_t *enp,
115 __in uint32_t instance)
116{
117 efx_mcdi_req_t req;
118 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
119 MC_CMD_FINI_RXQ_OUT_LEN)];
120 efx_rc_t rc;
121
122 (void) memset(payload, 0, sizeof (payload));
123 req.emr_cmd = MC_CMD_FINI_RXQ;
124 req.emr_in_buf = payload;
125 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
126 req.emr_out_buf = payload;
127 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
128
129 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
130
131 efx_mcdi_execute(enp, &req);
132
133 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) {
134 rc = req.emr_rc;
135 goto fail1;
136 }
137
138 return (0);
139
140fail1:
141 EFSYS_PROBE1(fail1, efx_rc_t, rc);
142
143 return (rc);
144}
145
146#if EFSYS_OPT_RX_SCALE
147static __checkReturn efx_rc_t
148efx_mcdi_rss_context_alloc(
149 __in efx_nic_t *enp,
150 __out uint32_t *rss_contextp)
151{
152 efx_mcdi_req_t req;
153 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
154 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
155 uint32_t rss_context;
156 efx_rc_t rc;
157
158 (void) memset(payload, 0, sizeof (payload));
159 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
160 req.emr_in_buf = payload;
161 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
162 req.emr_out_buf = payload;
163 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
164
165 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
166 EVB_PORT_ID_ASSIGNED);
167 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE,
168 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE);
169 /* NUM_QUEUES is only used to validate indirection table offsets */
170 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, 64);
171
172 efx_mcdi_execute(enp, &req);
173
174 if (req.emr_rc != 0) {
175 rc = req.emr_rc;
176 goto fail1;
177 }
178
179 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
180 rc = EMSGSIZE;
181 goto fail2;
182 }
183
184 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
185 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
186 rc = ENOENT;
187 goto fail3;
188 }
189
190 *rss_contextp = rss_context;
191
192 return (0);
193
194fail3:
195 EFSYS_PROBE(fail3);
196fail2:
197 EFSYS_PROBE(fail2);
198fail1:
199 EFSYS_PROBE1(fail1, efx_rc_t, rc);
200
201 return (rc);
202}
203#endif /* EFSYS_OPT_RX_SCALE */
204
205#if EFSYS_OPT_RX_SCALE
206static efx_rc_t
207efx_mcdi_rss_context_free(
208 __in efx_nic_t *enp,
209 __in uint32_t rss_context)
210{
211 efx_mcdi_req_t req;
212 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
213 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
214 efx_rc_t rc;
215
216 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
217 rc = EINVAL;
218 goto fail1;
219 }
220
221 (void) memset(payload, 0, sizeof (payload));
222 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
223 req.emr_in_buf = payload;
224 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
225 req.emr_out_buf = payload;
226 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
227
228 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
229
230 efx_mcdi_execute(enp, &req);
231
232 if (req.emr_rc != 0) {
233 rc = req.emr_rc;
234 goto fail2;
235 }
236
237 return (0);
238
239fail2:
240 EFSYS_PROBE(fail2);
241fail1:
242 EFSYS_PROBE1(fail1, efx_rc_t, rc);
243
244 return (rc);
245}
246#endif /* EFSYS_OPT_RX_SCALE */
247
248#if EFSYS_OPT_RX_SCALE
249static efx_rc_t
250efx_mcdi_rss_context_set_flags(
251 __in efx_nic_t *enp,
252 __in uint32_t rss_context,
253 __in efx_rx_hash_type_t type)
254{
255 efx_mcdi_req_t req;
256 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
257 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
258 efx_rc_t rc;
259
260 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
261 rc = EINVAL;
262 goto fail1;
263 }
264
265 (void) memset(payload, 0, sizeof (payload));
266 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
267 req.emr_in_buf = payload;
268 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
269 req.emr_out_buf = payload;
270 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
271
272 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
273 rss_context);
274
275 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
276 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
277 (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0,
278 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
279 (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0,
280 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
281 (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0,
282 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
283 (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0);
284
285 efx_mcdi_execute(enp, &req);
286
287 if (req.emr_rc != 0) {
288 rc = req.emr_rc;
289 goto fail2;
290 }
291
292 return (0);
293
294fail2:
295 EFSYS_PROBE(fail2);
296fail1:
297 EFSYS_PROBE1(fail1, efx_rc_t, rc);
298
299 return (rc);
300}
301#endif /* EFSYS_OPT_RX_SCALE */
302
303#if EFSYS_OPT_RX_SCALE
304static efx_rc_t
305efx_mcdi_rss_context_set_key(
306 __in efx_nic_t *enp,
307 __in uint32_t rss_context,
308 __in_ecount(n) uint8_t *key,
309 __in size_t n)
310{
311 efx_mcdi_req_t req;
312 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
313 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
314 efx_rc_t rc;
315
316 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
317 rc = EINVAL;
318 goto fail1;
319 }
320
321 (void) memset(payload, 0, sizeof (payload));
322 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
323 req.emr_in_buf = payload;
324 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
325 req.emr_out_buf = payload;
326 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
327
328 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
329 rss_context);
330
331 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
332 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
333 rc = EINVAL;
334 goto fail2;
335 }
336
337 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
338 key, n);
339
340 efx_mcdi_execute(enp, &req);
341
342 if (req.emr_rc != 0) {
343 rc = req.emr_rc;
344 goto fail3;
345 }
346
347 return (0);
348
349fail3:
350 EFSYS_PROBE(fail3);
351fail2:
352 EFSYS_PROBE(fail2);
353fail1:
354 EFSYS_PROBE1(fail1, efx_rc_t, rc);
355
356 return (rc);
357}
358#endif /* EFSYS_OPT_RX_SCALE */
359
360#if EFSYS_OPT_RX_SCALE
361static efx_rc_t
362efx_mcdi_rss_context_set_table(
363 __in efx_nic_t *enp,
364 __in uint32_t rss_context,
365 __in_ecount(n) unsigned int *table,
366 __in size_t n)
367{
368 efx_mcdi_req_t req;
369 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
370 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
371 uint8_t *req_table;
372 int i, rc;
373
374 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
375 rc = EINVAL;
376 goto fail1;
377 }
378
379 (void) memset(payload, 0, sizeof (payload));
380 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
381 req.emr_in_buf = payload;
382 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
383 req.emr_out_buf = payload;
384 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
385
386 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
387 rss_context);
388
389 req_table =
390 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
391
392 for (i = 0;
393 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
394 i++) {
395 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
396 }
397
398 efx_mcdi_execute(enp, &req);
399
400 if (req.emr_rc != 0) {
401 rc = req.emr_rc;
402 goto fail2;
403 }
404
405 return (0);
406
407fail2:
408 EFSYS_PROBE(fail2);
409fail1:
410 EFSYS_PROBE1(fail1, efx_rc_t, rc);
411
412 return (rc);
413}
414#endif /* EFSYS_OPT_RX_SCALE */
415
416
417 __checkReturn efx_rc_t
418hunt_rx_init(
419 __in efx_nic_t *enp)
420{
421#if EFSYS_OPT_RX_SCALE
422
423 if (efx_mcdi_rss_context_alloc(enp, &enp->en_rss_context) == 0) {
424 /*
425 * Allocated an exclusive RSS context, which allows both the
426 * indirection table and key to be modified.
427 */
428 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
429 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
430 } else {
431 /*
432 * Failed to allocate an exclusive RSS context. Continue
433 * operation without support for RSS. The pseudo-header in
434 * received packets will not contain a Toeplitz hash value.
435 */
436 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
437 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
438 }
439
440#endif /* EFSYS_OPT_RX_SCALE */
441
442 return (0);
443}
444
445#if EFSYS_OPT_RX_HDR_SPLIT
446 __checkReturn efx_rc_t
447hunt_rx_hdr_split_enable(
448 __in efx_nic_t *enp,
449 __in unsigned int hdr_buf_size,
450 __in unsigned int pld_buf_size)
451{
452 efx_rc_t rc;
453
454 /* FIXME */
455 _NOTE(ARGUNUSED(enp, hdr_buf_size, pld_buf_size))
456 if (B_FALSE) {
457 rc = ENOTSUP;
458 goto fail1;
459 }
460 /* FIXME */
461
462 return (0);
463
464fail1:
465 EFSYS_PROBE1(fail1, efx_rc_t, rc);
466
467 return (rc);
468}
469#endif /* EFSYS_OPT_RX_HDR_SPLIT */
470
471#if EFSYS_OPT_RX_SCATTER
472 __checkReturn efx_rc_t
473hunt_rx_scatter_enable(
474 __in efx_nic_t *enp,
475 __in unsigned int buf_size)
476{
477 _NOTE(ARGUNUSED(enp, buf_size))
478 return (0);
479}
480#endif /* EFSYS_OPT_RX_SCATTER */
481
482#if EFSYS_OPT_RX_SCALE
483 __checkReturn efx_rc_t
484hunt_rx_scale_mode_set(
485 __in efx_nic_t *enp,
486 __in efx_rx_hash_alg_t alg,
487 __in efx_rx_hash_type_t type,
488 __in boolean_t insert)
489{
490 efx_rc_t rc;
491
492 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
493 EFSYS_ASSERT3U(insert, ==, B_TRUE);
494
495 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
496 rc = EINVAL;
497 goto fail1;
498 }
499
500 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
501 rc = ENOTSUP;
502 goto fail2;
503 }
504
505 if ((rc = efx_mcdi_rss_context_set_flags(enp,
506 enp->en_rss_context, type)) != 0)
507 goto fail3;
508
509 return (0);
510
511fail3:
512 EFSYS_PROBE(fail3);
513fail2:
514 EFSYS_PROBE(fail2);
515fail1:
516 EFSYS_PROBE1(fail1, efx_rc_t, rc);
517
518 return (rc);
519}
520#endif /* EFSYS_OPT_RX_SCALE */
521
522#if EFSYS_OPT_RX_SCALE
523 __checkReturn efx_rc_t
524hunt_rx_scale_key_set(
525 __in efx_nic_t *enp,
526 __in_ecount(n) uint8_t *key,
527 __in size_t n)
528{
529 efx_rc_t rc;
530
531 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
532 rc = ENOTSUP;
533 goto fail1;
534 }
535
536 if ((rc = efx_mcdi_rss_context_set_key(enp,
537 enp->en_rss_context, key, n)) != 0)
538 goto fail2;
539
540 return (0);
541
542fail2:
543 EFSYS_PROBE(fail2);
544fail1:
545 EFSYS_PROBE1(fail1, efx_rc_t, rc);
546
547 return (rc);
548}
549#endif /* EFSYS_OPT_RX_SCALE */
550
551#if EFSYS_OPT_RX_SCALE
552 __checkReturn efx_rc_t
553hunt_rx_scale_tbl_set(
554 __in efx_nic_t *enp,
555 __in_ecount(n) unsigned int *table,
556 __in size_t n)
557{
558 efx_rc_t rc;
559
560 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
561 rc = ENOTSUP;
562 goto fail1;
563 }
564
565 if ((rc = efx_mcdi_rss_context_set_table(enp,
566 enp->en_rss_context, table, n)) != 0)
567 goto fail2;
568
569 return (0);
570
571fail2:
572 EFSYS_PROBE(fail2);
573fail1:
574 EFSYS_PROBE1(fail1, efx_rc_t, rc);
575
576 return (rc);
577}
578#endif /* EFSYS_OPT_RX_SCALE */
579
580 void
581hunt_rx_qpost(
582 __in efx_rxq_t *erp,
583 __in_ecount(n) efsys_dma_addr_t *addrp,
584 __in size_t size,
585 __in unsigned int n,
586 __in unsigned int completed,
587 __in unsigned int added)
588{
589 efx_qword_t qword;
590 unsigned int i;
591 unsigned int offset;
592 unsigned int id;
593
594 /* The client driver must not overfill the queue */
595 EFSYS_ASSERT3U(added - completed + n, <=,
596 EFX_RXQ_LIMIT(erp->er_mask + 1));
597
598 id = added & (erp->er_mask);
599 for (i = 0; i < n; i++) {
600 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
601 unsigned int, id, efsys_dma_addr_t, addrp[i],
602 size_t, size);
603
604 EFX_POPULATE_QWORD_3(qword,
605 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
606 ESF_DZ_RX_KER_BUF_ADDR_DW0,
607 (uint32_t)(addrp[i] & 0xffffffff),
608 ESF_DZ_RX_KER_BUF_ADDR_DW1,
609 (uint32_t)(addrp[i] >> 32));
610
611 offset = id * sizeof (efx_qword_t);
612 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
613
614 id = (id + 1) & (erp->er_mask);
615 }
616}
617
618 void
619hunt_rx_qpush(
620 __in efx_rxq_t *erp,
621 __in unsigned int added,
622 __inout unsigned int *pushedp)
623{
624 efx_nic_t *enp = erp->er_enp;
625 unsigned int pushed = *pushedp;
626 uint32_t wptr;
627 efx_dword_t dword;
628
629 /* Hardware has alignment restriction for WPTR */
630 wptr = P2ALIGN(added, HUNTINGTON_RX_WPTR_ALIGN);
631 if (pushed == wptr)
632 return;
633
634 *pushedp = wptr;
635
636 /* Push the populated descriptors out */
637 wptr &= erp->er_mask;
638
639 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
640
641 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
642 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
643 wptr, pushed & erp->er_mask);
644 EFSYS_PIO_WRITE_BARRIER();
645 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
646 erp->er_index, &dword, B_FALSE);
647}
648
649 __checkReturn efx_rc_t
650hunt_rx_qflush(
651 __in efx_rxq_t *erp)
652{
653 efx_nic_t *enp = erp->er_enp;
654 efx_rc_t rc;
655
656 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
657 goto fail1;
658
659 return (0);
660
661fail1:
662 EFSYS_PROBE1(fail1, efx_rc_t, rc);
663
664 return (rc);
665}
666
667 void
668hunt_rx_qenable(
669 __in efx_rxq_t *erp)
670{
671 /* FIXME */
672 _NOTE(ARGUNUSED(erp))
673 /* FIXME */
674}
675
676 __checkReturn efx_rc_t
677hunt_rx_qcreate(
678 __in efx_nic_t *enp,
679 __in unsigned int index,
680 __in unsigned int label,
681 __in efx_rxq_type_t type,
682 __in efsys_mem_t *esmp,
683 __in size_t n,
684 __in uint32_t id,
685 __in efx_evq_t *eep,
686 __in efx_rxq_t *erp)
687{
688 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
689 efx_rc_t rc;
690 boolean_t disable_scatter;
691
692 _NOTE(ARGUNUSED(erp))
693
694 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
695 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
696 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
697
698 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
699 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
700
701 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
702 rc = EINVAL;
703 goto fail1;
704 }
705 if (index >= encp->enc_rxq_limit) {
706 rc = EINVAL;
707 goto fail2;
708 }
709
710 /* Scatter can only be disabled if the firmware supports doing so */
711 if ((type != EFX_RXQ_TYPE_SCATTER) &&
712 enp->en_nic_cfg.enc_rx_disable_scatter_supported) {
713 disable_scatter = B_TRUE;
714 } else {
715 disable_scatter = B_FALSE;
716 }
717
718 /*
719 * Note: EFX_RXQ_TYPE_SPLIT_HEADER and EFX_RXQ_TYPE_SPLIT_PAYLOAD are
720 * not supported here.
721 */
722
723 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
724 esmp, disable_scatter)) != 0)
725 goto fail3;
726
727 erp->er_eep = eep;
728 erp->er_label = label;
729
730 ef10_ev_rxlabel_init(eep, erp, label);
731
732 return (0);
733
734fail3:
735 EFSYS_PROBE(fail3);
736fail2:
737 EFSYS_PROBE(fail2);
738fail1:
739 EFSYS_PROBE1(fail1, efx_rc_t, rc);
740
741 return (rc);
742}
743
744 void
745hunt_rx_qdestroy(
746 __in efx_rxq_t *erp)
747{
748 efx_nic_t *enp = erp->er_enp;
749 efx_evq_t *eep = erp->er_eep;
750 unsigned int label = erp->er_label;
751
752 ef10_ev_rxlabel_fini(eep, label);
753
754 EFSYS_ASSERT(enp->en_rx_qcount != 0);
755 --enp->en_rx_qcount;
756
757 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
758}
759
760 void
761hunt_rx_fini(
762 __in efx_nic_t *enp)
763{
764#if EFSYS_OPT_RX_SCALE
765 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
766 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
767 }
768 enp->en_rss_context = 0;
769 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
770#else
771 _NOTE(ARGUNUSED(enp))
772#endif /* EFSYS_OPT_RX_SCALE */
773}
774
775#endif /* EFSYS_OPT_HUNTINGTON */
2 * Copyright (c) 2012-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/hunt_rx.c 293752 2016-01-12 13:32:04Z arybchik $");
33
34#include "efsys.h"
35#include "efx.h"
36#include "efx_impl.h"
37
38
39#if EFSYS_OPT_HUNTINGTON
40
41
42static __checkReturn efx_rc_t
43efx_mcdi_init_rxq(
44 __in efx_nic_t *enp,
45 __in uint32_t size,
46 __in uint32_t target_evq,
47 __in uint32_t label,
48 __in uint32_t instance,
49 __in efsys_mem_t *esmp,
50 __in boolean_t disable_scatter)
51{
52 efx_mcdi_req_t req;
53 uint8_t payload[
54 MAX(MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)),
55 MC_CMD_INIT_RXQ_OUT_LEN)];
56 int npages = EFX_RXQ_NBUFS(size);
57 int i;
58 efx_qword_t *dma_addr;
59 uint64_t addr;
60 efx_rc_t rc;
61
62 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
63
64 (void) memset(payload, 0, sizeof (payload));
65 req.emr_cmd = MC_CMD_INIT_RXQ;
66 req.emr_in_buf = payload;
67 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
68 req.emr_out_buf = payload;
69 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
70
71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
74 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
75 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
76 INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
77 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
78 INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
79 INIT_RXQ_IN_CRC_MODE, 0,
80 INIT_RXQ_IN_FLAG_PREFIX, 1,
81 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
83 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
84
85 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
86 addr = EFSYS_MEM_ADDR(esmp);
87
88 for (i = 0; i < npages; i++) {
89 EFX_POPULATE_QWORD_2(*dma_addr,
90 EFX_DWORD_1, (uint32_t)(addr >> 32),
91 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
92
93 dma_addr++;
94 addr += EFX_BUF_SIZE;
95 }
96
97 efx_mcdi_execute(enp, &req);
98
99 if (req.emr_rc != 0) {
100 rc = req.emr_rc;
101 goto fail1;
102 }
103
104 return (0);
105
106fail1:
107 EFSYS_PROBE1(fail1, efx_rc_t, rc);
108
109 return (rc);
110}
111
112static __checkReturn efx_rc_t
113efx_mcdi_fini_rxq(
114 __in efx_nic_t *enp,
115 __in uint32_t instance)
116{
117 efx_mcdi_req_t req;
118 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
119 MC_CMD_FINI_RXQ_OUT_LEN)];
120 efx_rc_t rc;
121
122 (void) memset(payload, 0, sizeof (payload));
123 req.emr_cmd = MC_CMD_FINI_RXQ;
124 req.emr_in_buf = payload;
125 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
126 req.emr_out_buf = payload;
127 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
128
129 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
130
131 efx_mcdi_execute(enp, &req);
132
133 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) {
134 rc = req.emr_rc;
135 goto fail1;
136 }
137
138 return (0);
139
140fail1:
141 EFSYS_PROBE1(fail1, efx_rc_t, rc);
142
143 return (rc);
144}
145
146#if EFSYS_OPT_RX_SCALE
147static __checkReturn efx_rc_t
148efx_mcdi_rss_context_alloc(
149 __in efx_nic_t *enp,
150 __out uint32_t *rss_contextp)
151{
152 efx_mcdi_req_t req;
153 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
154 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
155 uint32_t rss_context;
156 efx_rc_t rc;
157
158 (void) memset(payload, 0, sizeof (payload));
159 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
160 req.emr_in_buf = payload;
161 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
162 req.emr_out_buf = payload;
163 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
164
165 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
166 EVB_PORT_ID_ASSIGNED);
167 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE,
168 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE);
169 /* NUM_QUEUES is only used to validate indirection table offsets */
170 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, 64);
171
172 efx_mcdi_execute(enp, &req);
173
174 if (req.emr_rc != 0) {
175 rc = req.emr_rc;
176 goto fail1;
177 }
178
179 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
180 rc = EMSGSIZE;
181 goto fail2;
182 }
183
184 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
185 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
186 rc = ENOENT;
187 goto fail3;
188 }
189
190 *rss_contextp = rss_context;
191
192 return (0);
193
194fail3:
195 EFSYS_PROBE(fail3);
196fail2:
197 EFSYS_PROBE(fail2);
198fail1:
199 EFSYS_PROBE1(fail1, efx_rc_t, rc);
200
201 return (rc);
202}
203#endif /* EFSYS_OPT_RX_SCALE */
204
205#if EFSYS_OPT_RX_SCALE
206static efx_rc_t
207efx_mcdi_rss_context_free(
208 __in efx_nic_t *enp,
209 __in uint32_t rss_context)
210{
211 efx_mcdi_req_t req;
212 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
213 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
214 efx_rc_t rc;
215
216 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
217 rc = EINVAL;
218 goto fail1;
219 }
220
221 (void) memset(payload, 0, sizeof (payload));
222 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
223 req.emr_in_buf = payload;
224 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
225 req.emr_out_buf = payload;
226 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
227
228 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
229
230 efx_mcdi_execute(enp, &req);
231
232 if (req.emr_rc != 0) {
233 rc = req.emr_rc;
234 goto fail2;
235 }
236
237 return (0);
238
239fail2:
240 EFSYS_PROBE(fail2);
241fail1:
242 EFSYS_PROBE1(fail1, efx_rc_t, rc);
243
244 return (rc);
245}
246#endif /* EFSYS_OPT_RX_SCALE */
247
248#if EFSYS_OPT_RX_SCALE
249static efx_rc_t
250efx_mcdi_rss_context_set_flags(
251 __in efx_nic_t *enp,
252 __in uint32_t rss_context,
253 __in efx_rx_hash_type_t type)
254{
255 efx_mcdi_req_t req;
256 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
257 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
258 efx_rc_t rc;
259
260 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
261 rc = EINVAL;
262 goto fail1;
263 }
264
265 (void) memset(payload, 0, sizeof (payload));
266 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
267 req.emr_in_buf = payload;
268 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
269 req.emr_out_buf = payload;
270 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
271
272 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
273 rss_context);
274
275 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
276 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
277 (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0,
278 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
279 (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0,
280 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
281 (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0,
282 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
283 (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0);
284
285 efx_mcdi_execute(enp, &req);
286
287 if (req.emr_rc != 0) {
288 rc = req.emr_rc;
289 goto fail2;
290 }
291
292 return (0);
293
294fail2:
295 EFSYS_PROBE(fail2);
296fail1:
297 EFSYS_PROBE1(fail1, efx_rc_t, rc);
298
299 return (rc);
300}
301#endif /* EFSYS_OPT_RX_SCALE */
302
303#if EFSYS_OPT_RX_SCALE
304static efx_rc_t
305efx_mcdi_rss_context_set_key(
306 __in efx_nic_t *enp,
307 __in uint32_t rss_context,
308 __in_ecount(n) uint8_t *key,
309 __in size_t n)
310{
311 efx_mcdi_req_t req;
312 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
313 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
314 efx_rc_t rc;
315
316 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
317 rc = EINVAL;
318 goto fail1;
319 }
320
321 (void) memset(payload, 0, sizeof (payload));
322 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
323 req.emr_in_buf = payload;
324 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
325 req.emr_out_buf = payload;
326 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
327
328 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
329 rss_context);
330
331 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
332 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
333 rc = EINVAL;
334 goto fail2;
335 }
336
337 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
338 key, n);
339
340 efx_mcdi_execute(enp, &req);
341
342 if (req.emr_rc != 0) {
343 rc = req.emr_rc;
344 goto fail3;
345 }
346
347 return (0);
348
349fail3:
350 EFSYS_PROBE(fail3);
351fail2:
352 EFSYS_PROBE(fail2);
353fail1:
354 EFSYS_PROBE1(fail1, efx_rc_t, rc);
355
356 return (rc);
357}
358#endif /* EFSYS_OPT_RX_SCALE */
359
360#if EFSYS_OPT_RX_SCALE
361static efx_rc_t
362efx_mcdi_rss_context_set_table(
363 __in efx_nic_t *enp,
364 __in uint32_t rss_context,
365 __in_ecount(n) unsigned int *table,
366 __in size_t n)
367{
368 efx_mcdi_req_t req;
369 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
370 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
371 uint8_t *req_table;
372 int i, rc;
373
374 if (rss_context == HUNTINGTON_RSS_CONTEXT_INVALID) {
375 rc = EINVAL;
376 goto fail1;
377 }
378
379 (void) memset(payload, 0, sizeof (payload));
380 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
381 req.emr_in_buf = payload;
382 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
383 req.emr_out_buf = payload;
384 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
385
386 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
387 rss_context);
388
389 req_table =
390 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
391
392 for (i = 0;
393 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
394 i++) {
395 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
396 }
397
398 efx_mcdi_execute(enp, &req);
399
400 if (req.emr_rc != 0) {
401 rc = req.emr_rc;
402 goto fail2;
403 }
404
405 return (0);
406
407fail2:
408 EFSYS_PROBE(fail2);
409fail1:
410 EFSYS_PROBE1(fail1, efx_rc_t, rc);
411
412 return (rc);
413}
414#endif /* EFSYS_OPT_RX_SCALE */
415
416
417 __checkReturn efx_rc_t
418hunt_rx_init(
419 __in efx_nic_t *enp)
420{
421#if EFSYS_OPT_RX_SCALE
422
423 if (efx_mcdi_rss_context_alloc(enp, &enp->en_rss_context) == 0) {
424 /*
425 * Allocated an exclusive RSS context, which allows both the
426 * indirection table and key to be modified.
427 */
428 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
429 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
430 } else {
431 /*
432 * Failed to allocate an exclusive RSS context. Continue
433 * operation without support for RSS. The pseudo-header in
434 * received packets will not contain a Toeplitz hash value.
435 */
436 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
437 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
438 }
439
440#endif /* EFSYS_OPT_RX_SCALE */
441
442 return (0);
443}
444
445#if EFSYS_OPT_RX_HDR_SPLIT
446 __checkReturn efx_rc_t
447hunt_rx_hdr_split_enable(
448 __in efx_nic_t *enp,
449 __in unsigned int hdr_buf_size,
450 __in unsigned int pld_buf_size)
451{
452 efx_rc_t rc;
453
454 /* FIXME */
455 _NOTE(ARGUNUSED(enp, hdr_buf_size, pld_buf_size))
456 if (B_FALSE) {
457 rc = ENOTSUP;
458 goto fail1;
459 }
460 /* FIXME */
461
462 return (0);
463
464fail1:
465 EFSYS_PROBE1(fail1, efx_rc_t, rc);
466
467 return (rc);
468}
469#endif /* EFSYS_OPT_RX_HDR_SPLIT */
470
471#if EFSYS_OPT_RX_SCATTER
472 __checkReturn efx_rc_t
473hunt_rx_scatter_enable(
474 __in efx_nic_t *enp,
475 __in unsigned int buf_size)
476{
477 _NOTE(ARGUNUSED(enp, buf_size))
478 return (0);
479}
480#endif /* EFSYS_OPT_RX_SCATTER */
481
482#if EFSYS_OPT_RX_SCALE
483 __checkReturn efx_rc_t
484hunt_rx_scale_mode_set(
485 __in efx_nic_t *enp,
486 __in efx_rx_hash_alg_t alg,
487 __in efx_rx_hash_type_t type,
488 __in boolean_t insert)
489{
490 efx_rc_t rc;
491
492 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
493 EFSYS_ASSERT3U(insert, ==, B_TRUE);
494
495 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
496 rc = EINVAL;
497 goto fail1;
498 }
499
500 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
501 rc = ENOTSUP;
502 goto fail2;
503 }
504
505 if ((rc = efx_mcdi_rss_context_set_flags(enp,
506 enp->en_rss_context, type)) != 0)
507 goto fail3;
508
509 return (0);
510
511fail3:
512 EFSYS_PROBE(fail3);
513fail2:
514 EFSYS_PROBE(fail2);
515fail1:
516 EFSYS_PROBE1(fail1, efx_rc_t, rc);
517
518 return (rc);
519}
520#endif /* EFSYS_OPT_RX_SCALE */
521
522#if EFSYS_OPT_RX_SCALE
523 __checkReturn efx_rc_t
524hunt_rx_scale_key_set(
525 __in efx_nic_t *enp,
526 __in_ecount(n) uint8_t *key,
527 __in size_t n)
528{
529 efx_rc_t rc;
530
531 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
532 rc = ENOTSUP;
533 goto fail1;
534 }
535
536 if ((rc = efx_mcdi_rss_context_set_key(enp,
537 enp->en_rss_context, key, n)) != 0)
538 goto fail2;
539
540 return (0);
541
542fail2:
543 EFSYS_PROBE(fail2);
544fail1:
545 EFSYS_PROBE1(fail1, efx_rc_t, rc);
546
547 return (rc);
548}
549#endif /* EFSYS_OPT_RX_SCALE */
550
551#if EFSYS_OPT_RX_SCALE
552 __checkReturn efx_rc_t
553hunt_rx_scale_tbl_set(
554 __in efx_nic_t *enp,
555 __in_ecount(n) unsigned int *table,
556 __in size_t n)
557{
558 efx_rc_t rc;
559
560 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
561 rc = ENOTSUP;
562 goto fail1;
563 }
564
565 if ((rc = efx_mcdi_rss_context_set_table(enp,
566 enp->en_rss_context, table, n)) != 0)
567 goto fail2;
568
569 return (0);
570
571fail2:
572 EFSYS_PROBE(fail2);
573fail1:
574 EFSYS_PROBE1(fail1, efx_rc_t, rc);
575
576 return (rc);
577}
578#endif /* EFSYS_OPT_RX_SCALE */
579
580 void
581hunt_rx_qpost(
582 __in efx_rxq_t *erp,
583 __in_ecount(n) efsys_dma_addr_t *addrp,
584 __in size_t size,
585 __in unsigned int n,
586 __in unsigned int completed,
587 __in unsigned int added)
588{
589 efx_qword_t qword;
590 unsigned int i;
591 unsigned int offset;
592 unsigned int id;
593
594 /* The client driver must not overfill the queue */
595 EFSYS_ASSERT3U(added - completed + n, <=,
596 EFX_RXQ_LIMIT(erp->er_mask + 1));
597
598 id = added & (erp->er_mask);
599 for (i = 0; i < n; i++) {
600 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
601 unsigned int, id, efsys_dma_addr_t, addrp[i],
602 size_t, size);
603
604 EFX_POPULATE_QWORD_3(qword,
605 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
606 ESF_DZ_RX_KER_BUF_ADDR_DW0,
607 (uint32_t)(addrp[i] & 0xffffffff),
608 ESF_DZ_RX_KER_BUF_ADDR_DW1,
609 (uint32_t)(addrp[i] >> 32));
610
611 offset = id * sizeof (efx_qword_t);
612 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
613
614 id = (id + 1) & (erp->er_mask);
615 }
616}
617
618 void
619hunt_rx_qpush(
620 __in efx_rxq_t *erp,
621 __in unsigned int added,
622 __inout unsigned int *pushedp)
623{
624 efx_nic_t *enp = erp->er_enp;
625 unsigned int pushed = *pushedp;
626 uint32_t wptr;
627 efx_dword_t dword;
628
629 /* Hardware has alignment restriction for WPTR */
630 wptr = P2ALIGN(added, HUNTINGTON_RX_WPTR_ALIGN);
631 if (pushed == wptr)
632 return;
633
634 *pushedp = wptr;
635
636 /* Push the populated descriptors out */
637 wptr &= erp->er_mask;
638
639 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
640
641 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
642 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
643 wptr, pushed & erp->er_mask);
644 EFSYS_PIO_WRITE_BARRIER();
645 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
646 erp->er_index, &dword, B_FALSE);
647}
648
649 __checkReturn efx_rc_t
650hunt_rx_qflush(
651 __in efx_rxq_t *erp)
652{
653 efx_nic_t *enp = erp->er_enp;
654 efx_rc_t rc;
655
656 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
657 goto fail1;
658
659 return (0);
660
661fail1:
662 EFSYS_PROBE1(fail1, efx_rc_t, rc);
663
664 return (rc);
665}
666
667 void
668hunt_rx_qenable(
669 __in efx_rxq_t *erp)
670{
671 /* FIXME */
672 _NOTE(ARGUNUSED(erp))
673 /* FIXME */
674}
675
676 __checkReturn efx_rc_t
677hunt_rx_qcreate(
678 __in efx_nic_t *enp,
679 __in unsigned int index,
680 __in unsigned int label,
681 __in efx_rxq_type_t type,
682 __in efsys_mem_t *esmp,
683 __in size_t n,
684 __in uint32_t id,
685 __in efx_evq_t *eep,
686 __in efx_rxq_t *erp)
687{
688 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
689 efx_rc_t rc;
690 boolean_t disable_scatter;
691
692 _NOTE(ARGUNUSED(erp))
693
694 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
695 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
696 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
697
698 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
699 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
700
701 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
702 rc = EINVAL;
703 goto fail1;
704 }
705 if (index >= encp->enc_rxq_limit) {
706 rc = EINVAL;
707 goto fail2;
708 }
709
710 /* Scatter can only be disabled if the firmware supports doing so */
711 if ((type != EFX_RXQ_TYPE_SCATTER) &&
712 enp->en_nic_cfg.enc_rx_disable_scatter_supported) {
713 disable_scatter = B_TRUE;
714 } else {
715 disable_scatter = B_FALSE;
716 }
717
718 /*
719 * Note: EFX_RXQ_TYPE_SPLIT_HEADER and EFX_RXQ_TYPE_SPLIT_PAYLOAD are
720 * not supported here.
721 */
722
723 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
724 esmp, disable_scatter)) != 0)
725 goto fail3;
726
727 erp->er_eep = eep;
728 erp->er_label = label;
729
730 ef10_ev_rxlabel_init(eep, erp, label);
731
732 return (0);
733
734fail3:
735 EFSYS_PROBE(fail3);
736fail2:
737 EFSYS_PROBE(fail2);
738fail1:
739 EFSYS_PROBE1(fail1, efx_rc_t, rc);
740
741 return (rc);
742}
743
744 void
745hunt_rx_qdestroy(
746 __in efx_rxq_t *erp)
747{
748 efx_nic_t *enp = erp->er_enp;
749 efx_evq_t *eep = erp->er_eep;
750 unsigned int label = erp->er_label;
751
752 ef10_ev_rxlabel_fini(eep, label);
753
754 EFSYS_ASSERT(enp->en_rx_qcount != 0);
755 --enp->en_rx_qcount;
756
757 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
758}
759
760 void
761hunt_rx_fini(
762 __in efx_nic_t *enp)
763{
764#if EFSYS_OPT_RX_SCALE
765 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
766 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
767 }
768 enp->en_rss_context = 0;
769 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
770#else
771 _NOTE(ARGUNUSED(enp))
772#endif /* EFSYS_OPT_RX_SCALE */
773}
774
775#endif /* EFSYS_OPT_HUNTINGTON */