1/*-
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 299320 2016-05-10 07:01:06Z arybchik $");
33
34#include "efx.h"
35#include "efx_impl.h"
36
37#if EFSYS_OPT_QSTATS
38#define EFX_TX_QSTAT_INCR(_etp, _stat) \
39 do { \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
42 } while (B_FALSE)
43#else
44#define EFX_TX_QSTAT_INCR(_etp, _stat)
45#endif
46
47#if EFSYS_OPT_SIENA
48
49static __checkReturn efx_rc_t
50falconsiena_tx_init(
51 __in efx_nic_t *enp);
52
53static void
54falconsiena_tx_fini(
55 __in efx_nic_t *enp);
56
57static __checkReturn efx_rc_t
58falconsiena_tx_qcreate(
59 __in efx_nic_t *enp,
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
63 __in size_t n,
64 __in uint32_t id,
65 __in uint16_t flags,
66 __in efx_evq_t *eep,
67 __in efx_txq_t *etp,
68 __out unsigned int *addedp);
69
70static void
71falconsiena_tx_qdestroy(
72 __in efx_txq_t *etp);
73
74static __checkReturn efx_rc_t
75falconsiena_tx_qpost(
76 __in efx_txq_t *etp,
77 __in_ecount(n) efx_buffer_t *eb,
78 __in unsigned int n,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82static void
83falconsiena_tx_qpush(
84 __in efx_txq_t *etp,
85 __in unsigned int added,
86 __in unsigned int pushed);
87
88static __checkReturn efx_rc_t
89falconsiena_tx_qpace(
90 __in efx_txq_t *etp,
91 __in unsigned int ns);
92
93static __checkReturn efx_rc_t
94falconsiena_tx_qflush(
95 __in efx_txq_t *etp);
96
97static void
98falconsiena_tx_qenable(
99 __in efx_txq_t *etp);
100
101 __checkReturn efx_rc_t
102falconsiena_tx_qdesc_post(
103 __in efx_txq_t *etp,
104 __in_ecount(n) efx_desc_t *ed,
105 __in unsigned int n,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
108
109 void
110falconsiena_tx_qdesc_dma_create(
111 __in efx_txq_t *etp,
112 __in efsys_dma_addr_t addr,
113 __in size_t size,
114 __in boolean_t eop,
115 __out efx_desc_t *edp);
116
117#if EFSYS_OPT_QSTATS
118static void
119falconsiena_tx_qstats_update(
120 __in efx_txq_t *etp,
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
122#endif
123
124#endif /* EFSYS_OPT_SIENA */
125
126
127#if EFSYS_OPT_SIENA
128static efx_tx_ops_t __efx_tx_siena_ops = {
129 falconsiena_tx_init, /* etxo_init */
130 falconsiena_tx_fini, /* etxo_fini */
131 falconsiena_tx_qcreate, /* etxo_qcreate */
132 falconsiena_tx_qdestroy, /* etxo_qdestroy */
133 falconsiena_tx_qpost, /* etxo_qpost */
134 falconsiena_tx_qpush, /* etxo_qpush */
135 falconsiena_tx_qpace, /* etxo_qpace */
136 falconsiena_tx_qflush, /* etxo_qflush */
137 falconsiena_tx_qenable, /* etxo_qenable */
138 NULL, /* etxo_qpio_enable */
139 NULL, /* etxo_qpio_disable */
140 NULL, /* etxo_qpio_write */
141 NULL, /* etxo_qpio_post */
142 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
143 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 NULL, /* etxo_qdesc_tso_create */
145 NULL, /* etxo_qdesc_tso2_create */
146 NULL, /* etxo_qdesc_vlantci_create */
147#if EFSYS_OPT_QSTATS
148 falconsiena_tx_qstats_update, /* etxo_qstats_update */
149#endif
150};
151#endif /* EFSYS_OPT_SIENA */
152
153#if EFSYS_OPT_HUNTINGTON
154static efx_tx_ops_t __efx_tx_hunt_ops = {
155 ef10_tx_init, /* etxo_init */
156 ef10_tx_fini, /* etxo_fini */
157 ef10_tx_qcreate, /* etxo_qcreate */
158 ef10_tx_qdestroy, /* etxo_qdestroy */
159 ef10_tx_qpost, /* etxo_qpost */
160 ef10_tx_qpush, /* etxo_qpush */
161 ef10_tx_qpace, /* etxo_qpace */
162 ef10_tx_qflush, /* etxo_qflush */
163 ef10_tx_qenable, /* etxo_qenable */
164 ef10_tx_qpio_enable, /* etxo_qpio_enable */
165 ef10_tx_qpio_disable, /* etxo_qpio_disable */
166 ef10_tx_qpio_write, /* etxo_qpio_write */
167 ef10_tx_qpio_post, /* etxo_qpio_post */
168 ef10_tx_qdesc_post, /* etxo_qdesc_post */
169 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
170 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
171 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
172 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
173#if EFSYS_OPT_QSTATS
174 ef10_tx_qstats_update, /* etxo_qstats_update */
175#endif
176};
177#endif /* EFSYS_OPT_HUNTINGTON */
178
179#if EFSYS_OPT_MEDFORD
180static efx_tx_ops_t __efx_tx_medford_ops = {
181 ef10_tx_init, /* etxo_init */
182 ef10_tx_fini, /* etxo_fini */
183 ef10_tx_qcreate, /* etxo_qcreate */
184 ef10_tx_qdestroy, /* etxo_qdestroy */
185 ef10_tx_qpost, /* etxo_qpost */
186 ef10_tx_qpush, /* etxo_qpush */
187 ef10_tx_qpace, /* etxo_qpace */
188 ef10_tx_qflush, /* etxo_qflush */
189 ef10_tx_qenable, /* etxo_qenable */
190 ef10_tx_qpio_enable, /* etxo_qpio_enable */
191 ef10_tx_qpio_disable, /* etxo_qpio_disable */
192 ef10_tx_qpio_write, /* etxo_qpio_write */
193 ef10_tx_qpio_post, /* etxo_qpio_post */
194 ef10_tx_qdesc_post, /* etxo_qdesc_post */
195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
196 NULL, /* etxo_qdesc_tso_create */
197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
199#if EFSYS_OPT_QSTATS
200 ef10_tx_qstats_update, /* etxo_qstats_update */
201#endif
202};
203#endif /* EFSYS_OPT_MEDFORD */
204
205 __checkReturn efx_rc_t
206efx_tx_init(
207 __in efx_nic_t *enp)
208{
209 efx_tx_ops_t *etxop;
210 efx_rc_t rc;
211
212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
214
215 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
216 rc = EINVAL;
217 goto fail1;
218 }
219
220 if (enp->en_mod_flags & EFX_MOD_TX) {
221 rc = EINVAL;
222 goto fail2;
223 }
224
225 switch (enp->en_family) {
226#if EFSYS_OPT_SIENA
227 case EFX_FAMILY_SIENA:
228 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
229 break;
230#endif /* EFSYS_OPT_SIENA */
231
232#if EFSYS_OPT_HUNTINGTON
233 case EFX_FAMILY_HUNTINGTON:
234 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
235 break;
236#endif /* EFSYS_OPT_HUNTINGTON */
237
238#if EFSYS_OPT_MEDFORD
239 case EFX_FAMILY_MEDFORD:
240 etxop = (efx_tx_ops_t *)&__efx_tx_medford_ops;
241 break;
242#endif /* EFSYS_OPT_MEDFORD */
243
244 default:
245 EFSYS_ASSERT(0);
246 rc = ENOTSUP;
247 goto fail3;
248 }
249
250 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
251
252 if ((rc = etxop->etxo_init(enp)) != 0)
253 goto fail4;
254
255 enp->en_etxop = etxop;
256 enp->en_mod_flags |= EFX_MOD_TX;
257 return (0);
258
259fail4:
260 EFSYS_PROBE(fail4);
261fail3:
262 EFSYS_PROBE(fail3);
263fail2:
264 EFSYS_PROBE(fail2);
265fail1:
266 EFSYS_PROBE1(fail1, efx_rc_t, rc);
267
268 enp->en_etxop = NULL;
269 enp->en_mod_flags &= ~EFX_MOD_TX;
270 return (rc);
271}
272
273 void
274efx_tx_fini(
275 __in efx_nic_t *enp)
276{
277 efx_tx_ops_t *etxop = enp->en_etxop;
278
279 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
280 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
283
284 etxop->etxo_fini(enp);
285
286 enp->en_etxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_TX;
288}
289
290 __checkReturn efx_rc_t
291efx_tx_qcreate(
292 __in efx_nic_t *enp,
293 __in unsigned int index,
294 __in unsigned int label,
295 __in efsys_mem_t *esmp,
296 __in size_t n,
297 __in uint32_t id,
298 __in uint16_t flags,
299 __in efx_evq_t *eep,
300 __deref_out efx_txq_t **etpp,
301 __out unsigned int *addedp)
302{
303 efx_tx_ops_t *etxop = enp->en_etxop;
304 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
305 efx_txq_t *etp;
306 efx_rc_t rc;
307
308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
310
311 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
312
313 /* Allocate an TXQ object */
314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
315
316 if (etp == NULL) {
317 rc = ENOMEM;
318 goto fail1;
319 }
320
321 etp->et_magic = EFX_TXQ_MAGIC;
322 etp->et_enp = enp;
323 etp->et_index = index;
324 etp->et_mask = n - 1;
325 etp->et_esmp = esmp;
326
327 /* Initial descriptor index may be modified by etxo_qcreate */
328 *addedp = 0;
329
330 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
331 n, id, flags, eep, etp, addedp)) != 0)
332 goto fail2;
333
334 enp->en_tx_qcount++;
335 *etpp = etp;
336
337 return (0);
338
339fail2:
340 EFSYS_PROBE(fail2);
341 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
342fail1:
343 EFSYS_PROBE1(fail1, efx_rc_t, rc);
344 return (rc);
345}
346
347 void
348efx_tx_qdestroy(
349 __in efx_txq_t *etp)
350{
351 efx_nic_t *enp = etp->et_enp;
352 efx_tx_ops_t *etxop = enp->en_etxop;
353
354 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
355
356 EFSYS_ASSERT(enp->en_tx_qcount != 0);
357 --enp->en_tx_qcount;
358
359 etxop->etxo_qdestroy(etp);
360
361 /* Free the TXQ object */
362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
363}
364
365 __checkReturn efx_rc_t
366efx_tx_qpost(
367 __in efx_txq_t *etp,
368 __in_ecount(n) efx_buffer_t *eb,
369 __in unsigned int n,
370 __in unsigned int completed,
371 __inout unsigned int *addedp)
372{
373 efx_nic_t *enp = etp->et_enp;
374 efx_tx_ops_t *etxop = enp->en_etxop;
375 efx_rc_t rc;
376
377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
378
379 if ((rc = etxop->etxo_qpost(etp, eb,
380 n, completed, addedp)) != 0)
381 goto fail1;
382
383 return (0);
384
385fail1:
386 EFSYS_PROBE1(fail1, efx_rc_t, rc);
387 return (rc);
388}
389
390 void
391efx_tx_qpush(
392 __in efx_txq_t *etp,
393 __in unsigned int added,
394 __in unsigned int pushed)
395{
396 efx_nic_t *enp = etp->et_enp;
397 efx_tx_ops_t *etxop = enp->en_etxop;
398
399 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
400
401 etxop->etxo_qpush(etp, added, pushed);
402}
403
404 __checkReturn efx_rc_t
405efx_tx_qpace(
406 __in efx_txq_t *etp,
407 __in unsigned int ns)
408{
409 efx_nic_t *enp = etp->et_enp;
410 efx_tx_ops_t *etxop = enp->en_etxop;
411 efx_rc_t rc;
412
413 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
414
415 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
416 goto fail1;
417
418 return (0);
419
420fail1:
421 EFSYS_PROBE1(fail1, efx_rc_t, rc);
422 return (rc);
423}
424
425 __checkReturn efx_rc_t
426efx_tx_qflush(
427 __in efx_txq_t *etp)
428{
429 efx_nic_t *enp = etp->et_enp;
430 efx_tx_ops_t *etxop = enp->en_etxop;
431 efx_rc_t rc;
432
433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
434
435 if ((rc = etxop->etxo_qflush(etp)) != 0)
436 goto fail1;
437
438 return (0);
439
440fail1:
441 EFSYS_PROBE1(fail1, efx_rc_t, rc);
442 return (rc);
443}
444
445 void
446efx_tx_qenable(
447 __in efx_txq_t *etp)
448{
449 efx_nic_t *enp = etp->et_enp;
450 efx_tx_ops_t *etxop = enp->en_etxop;
451
452 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
453
454 etxop->etxo_qenable(etp);
455}
456
457 __checkReturn efx_rc_t
458efx_tx_qpio_enable(
459 __in efx_txq_t *etp)
460{
461 efx_nic_t *enp = etp->et_enp;
462 efx_tx_ops_t *etxop = enp->en_etxop;
463 efx_rc_t rc;
464
465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
466
467 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
468 rc = ENOTSUP;
469 goto fail1;
470 }
471 if (etxop->etxo_qpio_enable == NULL) {
472 rc = ENOTSUP;
473 goto fail2;
474 }
475 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
476 goto fail3;
477
478 return (0);
479
480fail3:
481 EFSYS_PROBE(fail3);
482fail2:
483 EFSYS_PROBE(fail2);
484fail1:
485 EFSYS_PROBE1(fail1, efx_rc_t, rc);
486 return (rc);
487}
488
489 void
490efx_tx_qpio_disable(
491 __in efx_txq_t *etp)
492{
493 efx_nic_t *enp = etp->et_enp;
494 efx_tx_ops_t *etxop = enp->en_etxop;
495
496 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
497
498 if (etxop->etxo_qpio_disable != NULL)
499 etxop->etxo_qpio_disable(etp);
500}
501
502 __checkReturn efx_rc_t
503efx_tx_qpio_write(
504 __in efx_txq_t *etp,
505 __in_ecount(buf_length) uint8_t *buffer,
506 __in size_t buf_length,
507 __in size_t pio_buf_offset)
508{
509 efx_nic_t *enp = etp->et_enp;
510 efx_tx_ops_t *etxop = enp->en_etxop;
511 efx_rc_t rc;
512
513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
514
515 if (etxop->etxo_qpio_write != NULL) {
516 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
517 pio_buf_offset)) != 0)
518 goto fail1;
519 return (0);
520 }
521
522 return (ENOTSUP);
523
524fail1:
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
526 return (rc);
527}
528
529 __checkReturn efx_rc_t
530efx_tx_qpio_post(
531 __in efx_txq_t *etp,
532 __in size_t pkt_length,
533 __in unsigned int completed,
534 __inout unsigned int *addedp)
535{
536 efx_nic_t *enp = etp->et_enp;
537 efx_tx_ops_t *etxop = enp->en_etxop;
538 efx_rc_t rc;
539
540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
541
542 if (etxop->etxo_qpio_post != NULL) {
543 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
544 addedp)) != 0)
545 goto fail1;
546 return (0);
547 }
548
549 return (ENOTSUP);
550
551fail1:
552 EFSYS_PROBE1(fail1, efx_rc_t, rc);
553 return (rc);
554}
555
556 __checkReturn efx_rc_t
557efx_tx_qdesc_post(
558 __in efx_txq_t *etp,
559 __in_ecount(n) efx_desc_t *ed,
560 __in unsigned int n,
561 __in unsigned int completed,
562 __inout unsigned int *addedp)
563{
564 efx_nic_t *enp = etp->et_enp;
565 efx_tx_ops_t *etxop = enp->en_etxop;
566 efx_rc_t rc;
567
568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
569
570 if ((rc = etxop->etxo_qdesc_post(etp, ed,
571 n, completed, addedp)) != 0)
572 goto fail1;
573
574 return (0);
575
576fail1:
577 EFSYS_PROBE1(fail1, efx_rc_t, rc);
578 return (rc);
579}
580
581 void
582efx_tx_qdesc_dma_create(
583 __in efx_txq_t *etp,
584 __in efsys_dma_addr_t addr,
585 __in size_t size,
586 __in boolean_t eop,
587 __out efx_desc_t *edp)
588{
589 efx_nic_t *enp = etp->et_enp;
590 efx_tx_ops_t *etxop = enp->en_etxop;
591
592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
594
595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
596}
597
598 void
599efx_tx_qdesc_tso_create(
600 __in efx_txq_t *etp,
601 __in uint16_t ipv4_id,
602 __in uint32_t tcp_seq,
603 __in uint8_t tcp_flags,
604 __out efx_desc_t *edp)
605{
606 efx_nic_t *enp = etp->et_enp;
607 efx_tx_ops_t *etxop = enp->en_etxop;
608
609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
611
612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
613}
614
615 void
616efx_tx_qdesc_tso2_create(
617 __in efx_txq_t *etp,
618 __in uint16_t ipv4_id,
619 __in uint32_t tcp_seq,
620 __in uint16_t mss,
621 __out_ecount(count) efx_desc_t *edp,
622 __in int count)
623{
624 efx_nic_t *enp = etp->et_enp;
625 efx_tx_ops_t *etxop = enp->en_etxop;
626
627 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
628 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
629
630 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
631}
632
633 void
634efx_tx_qdesc_vlantci_create(
635 __in efx_txq_t *etp,
636 __in uint16_t tci,
637 __out efx_desc_t *edp)
638{
639 efx_nic_t *enp = etp->et_enp;
640 efx_tx_ops_t *etxop = enp->en_etxop;
641
642 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
643 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
644
645 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
646}
647
648
649#if EFSYS_OPT_QSTATS
650 void
651efx_tx_qstats_update(
652 __in efx_txq_t *etp,
653 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
654{
655 efx_nic_t *enp = etp->et_enp;
656 efx_tx_ops_t *etxop = enp->en_etxop;
657
658 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
659
660 etxop->etxo_qstats_update(etp, stat);
661}
662#endif
663
664
665#if EFSYS_OPT_SIENA
666
667static __checkReturn efx_rc_t
668falconsiena_tx_init(
669 __in efx_nic_t *enp)
670{
671 efx_oword_t oword;
672
673 /*
674 * Disable the timer-based TX DMA backoff and allow TX DMA to be
675 * controlled by the RX FIFO fill level (although always allow a
676 * minimal trickle).
677 */
678 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
686
687 /*
688 * Filter all packets less than 14 bytes to avoid parsing
689 * errors.
690 */
691 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
692 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
693
694 /*
695 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
696 * descriptors (which is bad).
697 */
698 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
700 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
701
702 return (0);
703}
704
705#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
706 do { \
707 unsigned int id; \
708 size_t offset; \
709 efx_qword_t qword; \
710 \
711 id = (_added)++ & (_etp)->et_mask; \
712 offset = id * sizeof (efx_qword_t); \
713 \
714 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
715 unsigned int, id, efsys_dma_addr_t, (_addr), \
716 size_t, (_size), boolean_t, (_eop)); \
717 \
718 EFX_POPULATE_QWORD_4(qword, \
719 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
720 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
721 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
722 (uint32_t)((_addr) & 0xffffffff), \
723 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
724 (uint32_t)((_addr) >> 32)); \
725 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
726 \
727 _NOTE(CONSTANTCONDITION) \
728 } while (B_FALSE)
729
730static __checkReturn efx_rc_t
731falconsiena_tx_qpost(
732 __in efx_txq_t *etp,
733 __in_ecount(n) efx_buffer_t *eb,
734 __in unsigned int n,
735 __in unsigned int completed,
736 __inout unsigned int *addedp)
737{
738 unsigned int added = *addedp;
739 unsigned int i;
740 int rc = ENOSPC;
741
742 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
743 goto fail1;
744
745 for (i = 0; i < n; i++) {
746 efx_buffer_t *ebp = &eb[i];
747 efsys_dma_addr_t start = ebp->eb_addr;
748 size_t size = ebp->eb_size;
749 efsys_dma_addr_t end = start + size;
750
751 /* Fragments must not span 4k boundaries. */
752 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
753
754 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
755 }
756
757 EFX_TX_QSTAT_INCR(etp, TX_POST);
758
759 *addedp = added;
760 return (0);
761
762fail1:
763 EFSYS_PROBE1(fail1, efx_rc_t, rc);
764
765 return (rc);
766}
767
768static void
769falconsiena_tx_qpush(
770 __in efx_txq_t *etp,
771 __in unsigned int added,
772 __in unsigned int pushed)
773{
774 efx_nic_t *enp = etp->et_enp;
775 uint32_t wptr;
776 efx_dword_t dword;
777 efx_oword_t oword;
778
779 /* Push the populated descriptors out */
780 wptr = added & etp->et_mask;
781
782 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
783
784 /* Only write the third DWORD */
785 EFX_POPULATE_DWORD_1(dword,
786 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
787
788 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
789 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
790 wptr, pushed & etp->et_mask);
791 EFSYS_PIO_WRITE_BARRIER();
792 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
793 etp->et_index, &dword, B_FALSE);
794}
795
796#define EFX_MAX_PACE_VALUE 20
797#define EFX_TX_PACE_CLOCK_BASE 104
798
799static __checkReturn efx_rc_t
800falconsiena_tx_qpace(
801 __in efx_txq_t *etp,
802 __in unsigned int ns)
803{
804 efx_nic_t *enp = etp->et_enp;
805 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
806 efx_oword_t oword;
807 unsigned int pace_val;
808 unsigned int timer_period;
809 efx_rc_t rc;
810
811 if (ns == 0) {
812 pace_val = 0;
813 } else {
814 /*
815 * The pace_val to write into the table is s.t
816 * ns <= timer_period * (2 ^ pace_val)
817 */
818 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
819 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
820 if ((timer_period << pace_val) >= ns)
821 break;
822 }
823 }
824 if (pace_val > EFX_MAX_PACE_VALUE) {
825 rc = EINVAL;
826 goto fail1;
827 }
828
829 /* Update the pacing table */
830 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
831 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
832 &oword, B_TRUE);
833
834 return (0);
835
836fail1:
837 EFSYS_PROBE1(fail1, efx_rc_t, rc);
838
839 return (rc);
840}
841
842static __checkReturn efx_rc_t
843falconsiena_tx_qflush(
844 __in efx_txq_t *etp)
845{
846 efx_nic_t *enp = etp->et_enp;
847 efx_oword_t oword;
848 uint32_t label;
849
850 efx_tx_qpace(etp, 0);
851
852 label = etp->et_index;
853
854 /* Flush the queue */
855 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
856 FRF_AZ_TX_FLUSH_DESCQ, label);
857 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
858
859 return (0);
860}
861
862static void
863falconsiena_tx_qenable(
864 __in efx_txq_t *etp)
865{
866 efx_nic_t *enp = etp->et_enp;
867 efx_oword_t oword;
868
869 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
870 etp->et_index, &oword, B_TRUE);
871
872 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
873 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
874 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
875 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
876 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
877
878 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
879 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
880 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
881
882 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
883 etp->et_index, &oword, B_TRUE);
884}
885
886static __checkReturn efx_rc_t
887falconsiena_tx_qcreate(
888 __in efx_nic_t *enp,
889 __in unsigned int index,
890 __in unsigned int label,
891 __in efsys_mem_t *esmp,
892 __in size_t n,
893 __in uint32_t id,
894 __in uint16_t flags,
895 __in efx_evq_t *eep,
896 __in efx_txq_t *etp,
897 __out unsigned int *addedp)
898{
899 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
900 efx_oword_t oword;
901 uint32_t size;
902 efx_rc_t rc;
903
904 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
905 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
906 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
907
908 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
909 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
910
911 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
912 rc = EINVAL;
913 goto fail1;
914 }
915 if (index >= encp->enc_txq_limit) {
916 rc = EINVAL;
917 goto fail2;
918 }
919 for (size = 0;
920 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
921 size++)
922 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
923 break;
924 if (id + (1 << size) >= encp->enc_buftbl_limit) {
925 rc = EINVAL;
926 goto fail3;
927 }
928
929 /* Set up the new descriptor queue */
930 *addedp = 0;
931
932 EFX_POPULATE_OWORD_6(oword,
933 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
934 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
935 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
936 FRF_AZ_TX_DESCQ_LABEL, label,
937 FRF_AZ_TX_DESCQ_SIZE, size,
938 FRF_AZ_TX_DESCQ_TYPE, 0);
939
940 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
941 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
942 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
943 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
944 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
945
946 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
947 etp->et_index, &oword, B_TRUE);
948
949 return (0);
950
951fail3:
952 EFSYS_PROBE(fail3);
953fail2:
954 EFSYS_PROBE(fail2);
955fail1:
956 EFSYS_PROBE1(fail1, efx_rc_t, rc);
957
958 return (rc);
959}
960
961 __checkReturn efx_rc_t
962falconsiena_tx_qdesc_post(
963 __in efx_txq_t *etp,
964 __in_ecount(n) efx_desc_t *ed,
965 __in unsigned int n,
966 __in unsigned int completed,
967 __inout unsigned int *addedp)
968{
969 unsigned int added = *addedp;
970 unsigned int i;
971 efx_rc_t rc;
972
973 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
974 rc = ENOSPC;
975 goto fail1;
976 }
977
978 for (i = 0; i < n; i++) {
979 efx_desc_t *edp = &ed[i];
980 unsigned int id;
981 size_t offset;
982
983 id = added++ & etp->et_mask;
984 offset = id * sizeof (efx_desc_t);
985
986 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
987 }
988
989 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
990 unsigned int, added, unsigned int, n);
991
992 EFX_TX_QSTAT_INCR(etp, TX_POST);
993
994 *addedp = added;
995 return (0);
996
997fail1:
998 EFSYS_PROBE1(fail1, efx_rc_t, rc);
999 return (rc);
1000}
1001
1002 void
1003falconsiena_tx_qdesc_dma_create(
1004 __in efx_txq_t *etp,
1005 __in efsys_dma_addr_t addr,
1006 __in size_t size,
1007 __in boolean_t eop,
1008 __out efx_desc_t *edp)
1009{
1010 /* Fragments must not span 4k boundaries. */
1011 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
1012
1013 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1014 efsys_dma_addr_t, addr,
1015 size_t, size, boolean_t, eop);
1016
1017 EFX_POPULATE_QWORD_4(edp->ed_eq,
1018 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1019 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1020 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1021 (uint32_t)(addr & 0xffffffff),
1022 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1023 (uint32_t)(addr >> 32));
1024}
1025
1026#endif /* EFSYS_OPT_SIENA */
1027
1028#if EFSYS_OPT_QSTATS
1029#if EFSYS_OPT_NAMES
1030/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
1031static const char *__efx_tx_qstat_name[] = {
1032 "post",
1033 "post_pio",
1034};
1035/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1036
1037 const char *
1038efx_tx_qstat_name(
1039 __in efx_nic_t *enp,
1040 __in unsigned int id)
1041{
1042 _NOTE(ARGUNUSED(enp))
1043 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1044 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1045
1046 return (__efx_tx_qstat_name[id]);
1047}
1048#endif /* EFSYS_OPT_NAMES */
1049#endif /* EFSYS_OPT_QSTATS */
1050
1051#if EFSYS_OPT_SIENA
1052
1053#if EFSYS_OPT_QSTATS
1054static void
1055falconsiena_tx_qstats_update(
1056 __in efx_txq_t *etp,
1057 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1058{
1059 unsigned int id;
1060
1061 for (id = 0; id < TX_NQSTATS; id++) {
1062 efsys_stat_t *essp = &stat[id];
1063
1064 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1065 etp->et_stat[id] = 0;
1066 }
1067}
1068#endif /* EFSYS_OPT_QSTATS */
1069
1070static void
1071falconsiena_tx_qdestroy(
1072 __in efx_txq_t *etp)
1073{
1074 efx_nic_t *enp = etp->et_enp;
1075 efx_oword_t oword;
1076
1077 /* Purge descriptor queue */
1078 EFX_ZERO_OWORD(oword);
1079
1080 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1081 etp->et_index, &oword, B_TRUE);
1082}
1083
1084static void
1085falconsiena_tx_fini(
1086 __in efx_nic_t *enp)
1087{
1088 _NOTE(ARGUNUSED(enp))
1089}
1090
1091#endif /* EFSYS_OPT_SIENA */
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 299320 2016-05-10 07:01:06Z arybchik $");
33
34#include "efx.h"
35#include "efx_impl.h"
36
37#if EFSYS_OPT_QSTATS
38#define EFX_TX_QSTAT_INCR(_etp, _stat) \
39 do { \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
42 } while (B_FALSE)
43#else
44#define EFX_TX_QSTAT_INCR(_etp, _stat)
45#endif
46
47#if EFSYS_OPT_SIENA
48
49static __checkReturn efx_rc_t
50falconsiena_tx_init(
51 __in efx_nic_t *enp);
52
53static void
54falconsiena_tx_fini(
55 __in efx_nic_t *enp);
56
57static __checkReturn efx_rc_t
58falconsiena_tx_qcreate(
59 __in efx_nic_t *enp,
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
63 __in size_t n,
64 __in uint32_t id,
65 __in uint16_t flags,
66 __in efx_evq_t *eep,
67 __in efx_txq_t *etp,
68 __out unsigned int *addedp);
69
70static void
71falconsiena_tx_qdestroy(
72 __in efx_txq_t *etp);
73
74static __checkReturn efx_rc_t
75falconsiena_tx_qpost(
76 __in efx_txq_t *etp,
77 __in_ecount(n) efx_buffer_t *eb,
78 __in unsigned int n,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82static void
83falconsiena_tx_qpush(
84 __in efx_txq_t *etp,
85 __in unsigned int added,
86 __in unsigned int pushed);
87
88static __checkReturn efx_rc_t
89falconsiena_tx_qpace(
90 __in efx_txq_t *etp,
91 __in unsigned int ns);
92
93static __checkReturn efx_rc_t
94falconsiena_tx_qflush(
95 __in efx_txq_t *etp);
96
97static void
98falconsiena_tx_qenable(
99 __in efx_txq_t *etp);
100
101 __checkReturn efx_rc_t
102falconsiena_tx_qdesc_post(
103 __in efx_txq_t *etp,
104 __in_ecount(n) efx_desc_t *ed,
105 __in unsigned int n,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
108
109 void
110falconsiena_tx_qdesc_dma_create(
111 __in efx_txq_t *etp,
112 __in efsys_dma_addr_t addr,
113 __in size_t size,
114 __in boolean_t eop,
115 __out efx_desc_t *edp);
116
117#if EFSYS_OPT_QSTATS
118static void
119falconsiena_tx_qstats_update(
120 __in efx_txq_t *etp,
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
122#endif
123
124#endif /* EFSYS_OPT_SIENA */
125
126
127#if EFSYS_OPT_SIENA
128static efx_tx_ops_t __efx_tx_siena_ops = {
129 falconsiena_tx_init, /* etxo_init */
130 falconsiena_tx_fini, /* etxo_fini */
131 falconsiena_tx_qcreate, /* etxo_qcreate */
132 falconsiena_tx_qdestroy, /* etxo_qdestroy */
133 falconsiena_tx_qpost, /* etxo_qpost */
134 falconsiena_tx_qpush, /* etxo_qpush */
135 falconsiena_tx_qpace, /* etxo_qpace */
136 falconsiena_tx_qflush, /* etxo_qflush */
137 falconsiena_tx_qenable, /* etxo_qenable */
138 NULL, /* etxo_qpio_enable */
139 NULL, /* etxo_qpio_disable */
140 NULL, /* etxo_qpio_write */
141 NULL, /* etxo_qpio_post */
142 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
143 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 NULL, /* etxo_qdesc_tso_create */
145 NULL, /* etxo_qdesc_tso2_create */
146 NULL, /* etxo_qdesc_vlantci_create */
147#if EFSYS_OPT_QSTATS
148 falconsiena_tx_qstats_update, /* etxo_qstats_update */
149#endif
150};
151#endif /* EFSYS_OPT_SIENA */
152
153#if EFSYS_OPT_HUNTINGTON
154static efx_tx_ops_t __efx_tx_hunt_ops = {
155 ef10_tx_init, /* etxo_init */
156 ef10_tx_fini, /* etxo_fini */
157 ef10_tx_qcreate, /* etxo_qcreate */
158 ef10_tx_qdestroy, /* etxo_qdestroy */
159 ef10_tx_qpost, /* etxo_qpost */
160 ef10_tx_qpush, /* etxo_qpush */
161 ef10_tx_qpace, /* etxo_qpace */
162 ef10_tx_qflush, /* etxo_qflush */
163 ef10_tx_qenable, /* etxo_qenable */
164 ef10_tx_qpio_enable, /* etxo_qpio_enable */
165 ef10_tx_qpio_disable, /* etxo_qpio_disable */
166 ef10_tx_qpio_write, /* etxo_qpio_write */
167 ef10_tx_qpio_post, /* etxo_qpio_post */
168 ef10_tx_qdesc_post, /* etxo_qdesc_post */
169 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
170 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
171 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
172 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
173#if EFSYS_OPT_QSTATS
174 ef10_tx_qstats_update, /* etxo_qstats_update */
175#endif
176};
177#endif /* EFSYS_OPT_HUNTINGTON */
178
179#if EFSYS_OPT_MEDFORD
180static efx_tx_ops_t __efx_tx_medford_ops = {
181 ef10_tx_init, /* etxo_init */
182 ef10_tx_fini, /* etxo_fini */
183 ef10_tx_qcreate, /* etxo_qcreate */
184 ef10_tx_qdestroy, /* etxo_qdestroy */
185 ef10_tx_qpost, /* etxo_qpost */
186 ef10_tx_qpush, /* etxo_qpush */
187 ef10_tx_qpace, /* etxo_qpace */
188 ef10_tx_qflush, /* etxo_qflush */
189 ef10_tx_qenable, /* etxo_qenable */
190 ef10_tx_qpio_enable, /* etxo_qpio_enable */
191 ef10_tx_qpio_disable, /* etxo_qpio_disable */
192 ef10_tx_qpio_write, /* etxo_qpio_write */
193 ef10_tx_qpio_post, /* etxo_qpio_post */
194 ef10_tx_qdesc_post, /* etxo_qdesc_post */
195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
196 NULL, /* etxo_qdesc_tso_create */
197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
199#if EFSYS_OPT_QSTATS
200 ef10_tx_qstats_update, /* etxo_qstats_update */
201#endif
202};
203#endif /* EFSYS_OPT_MEDFORD */
204
205 __checkReturn efx_rc_t
206efx_tx_init(
207 __in efx_nic_t *enp)
208{
209 efx_tx_ops_t *etxop;
210 efx_rc_t rc;
211
212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
214
215 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
216 rc = EINVAL;
217 goto fail1;
218 }
219
220 if (enp->en_mod_flags & EFX_MOD_TX) {
221 rc = EINVAL;
222 goto fail2;
223 }
224
225 switch (enp->en_family) {
226#if EFSYS_OPT_SIENA
227 case EFX_FAMILY_SIENA:
228 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
229 break;
230#endif /* EFSYS_OPT_SIENA */
231
232#if EFSYS_OPT_HUNTINGTON
233 case EFX_FAMILY_HUNTINGTON:
234 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
235 break;
236#endif /* EFSYS_OPT_HUNTINGTON */
237
238#if EFSYS_OPT_MEDFORD
239 case EFX_FAMILY_MEDFORD:
240 etxop = (efx_tx_ops_t *)&__efx_tx_medford_ops;
241 break;
242#endif /* EFSYS_OPT_MEDFORD */
243
244 default:
245 EFSYS_ASSERT(0);
246 rc = ENOTSUP;
247 goto fail3;
248 }
249
250 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
251
252 if ((rc = etxop->etxo_init(enp)) != 0)
253 goto fail4;
254
255 enp->en_etxop = etxop;
256 enp->en_mod_flags |= EFX_MOD_TX;
257 return (0);
258
259fail4:
260 EFSYS_PROBE(fail4);
261fail3:
262 EFSYS_PROBE(fail3);
263fail2:
264 EFSYS_PROBE(fail2);
265fail1:
266 EFSYS_PROBE1(fail1, efx_rc_t, rc);
267
268 enp->en_etxop = NULL;
269 enp->en_mod_flags &= ~EFX_MOD_TX;
270 return (rc);
271}
272
273 void
274efx_tx_fini(
275 __in efx_nic_t *enp)
276{
277 efx_tx_ops_t *etxop = enp->en_etxop;
278
279 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
280 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
283
284 etxop->etxo_fini(enp);
285
286 enp->en_etxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_TX;
288}
289
290 __checkReturn efx_rc_t
291efx_tx_qcreate(
292 __in efx_nic_t *enp,
293 __in unsigned int index,
294 __in unsigned int label,
295 __in efsys_mem_t *esmp,
296 __in size_t n,
297 __in uint32_t id,
298 __in uint16_t flags,
299 __in efx_evq_t *eep,
300 __deref_out efx_txq_t **etpp,
301 __out unsigned int *addedp)
302{
303 efx_tx_ops_t *etxop = enp->en_etxop;
304 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
305 efx_txq_t *etp;
306 efx_rc_t rc;
307
308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
310
311 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
312
313 /* Allocate an TXQ object */
314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
315
316 if (etp == NULL) {
317 rc = ENOMEM;
318 goto fail1;
319 }
320
321 etp->et_magic = EFX_TXQ_MAGIC;
322 etp->et_enp = enp;
323 etp->et_index = index;
324 etp->et_mask = n - 1;
325 etp->et_esmp = esmp;
326
327 /* Initial descriptor index may be modified by etxo_qcreate */
328 *addedp = 0;
329
330 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
331 n, id, flags, eep, etp, addedp)) != 0)
332 goto fail2;
333
334 enp->en_tx_qcount++;
335 *etpp = etp;
336
337 return (0);
338
339fail2:
340 EFSYS_PROBE(fail2);
341 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
342fail1:
343 EFSYS_PROBE1(fail1, efx_rc_t, rc);
344 return (rc);
345}
346
347 void
348efx_tx_qdestroy(
349 __in efx_txq_t *etp)
350{
351 efx_nic_t *enp = etp->et_enp;
352 efx_tx_ops_t *etxop = enp->en_etxop;
353
354 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
355
356 EFSYS_ASSERT(enp->en_tx_qcount != 0);
357 --enp->en_tx_qcount;
358
359 etxop->etxo_qdestroy(etp);
360
361 /* Free the TXQ object */
362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
363}
364
365 __checkReturn efx_rc_t
366efx_tx_qpost(
367 __in efx_txq_t *etp,
368 __in_ecount(n) efx_buffer_t *eb,
369 __in unsigned int n,
370 __in unsigned int completed,
371 __inout unsigned int *addedp)
372{
373 efx_nic_t *enp = etp->et_enp;
374 efx_tx_ops_t *etxop = enp->en_etxop;
375 efx_rc_t rc;
376
377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
378
379 if ((rc = etxop->etxo_qpost(etp, eb,
380 n, completed, addedp)) != 0)
381 goto fail1;
382
383 return (0);
384
385fail1:
386 EFSYS_PROBE1(fail1, efx_rc_t, rc);
387 return (rc);
388}
389
390 void
391efx_tx_qpush(
392 __in efx_txq_t *etp,
393 __in unsigned int added,
394 __in unsigned int pushed)
395{
396 efx_nic_t *enp = etp->et_enp;
397 efx_tx_ops_t *etxop = enp->en_etxop;
398
399 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
400
401 etxop->etxo_qpush(etp, added, pushed);
402}
403
404 __checkReturn efx_rc_t
405efx_tx_qpace(
406 __in efx_txq_t *etp,
407 __in unsigned int ns)
408{
409 efx_nic_t *enp = etp->et_enp;
410 efx_tx_ops_t *etxop = enp->en_etxop;
411 efx_rc_t rc;
412
413 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
414
415 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
416 goto fail1;
417
418 return (0);
419
420fail1:
421 EFSYS_PROBE1(fail1, efx_rc_t, rc);
422 return (rc);
423}
424
425 __checkReturn efx_rc_t
426efx_tx_qflush(
427 __in efx_txq_t *etp)
428{
429 efx_nic_t *enp = etp->et_enp;
430 efx_tx_ops_t *etxop = enp->en_etxop;
431 efx_rc_t rc;
432
433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
434
435 if ((rc = etxop->etxo_qflush(etp)) != 0)
436 goto fail1;
437
438 return (0);
439
440fail1:
441 EFSYS_PROBE1(fail1, efx_rc_t, rc);
442 return (rc);
443}
444
445 void
446efx_tx_qenable(
447 __in efx_txq_t *etp)
448{
449 efx_nic_t *enp = etp->et_enp;
450 efx_tx_ops_t *etxop = enp->en_etxop;
451
452 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
453
454 etxop->etxo_qenable(etp);
455}
456
457 __checkReturn efx_rc_t
458efx_tx_qpio_enable(
459 __in efx_txq_t *etp)
460{
461 efx_nic_t *enp = etp->et_enp;
462 efx_tx_ops_t *etxop = enp->en_etxop;
463 efx_rc_t rc;
464
465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
466
467 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
468 rc = ENOTSUP;
469 goto fail1;
470 }
471 if (etxop->etxo_qpio_enable == NULL) {
472 rc = ENOTSUP;
473 goto fail2;
474 }
475 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
476 goto fail3;
477
478 return (0);
479
480fail3:
481 EFSYS_PROBE(fail3);
482fail2:
483 EFSYS_PROBE(fail2);
484fail1:
485 EFSYS_PROBE1(fail1, efx_rc_t, rc);
486 return (rc);
487}
488
489 void
490efx_tx_qpio_disable(
491 __in efx_txq_t *etp)
492{
493 efx_nic_t *enp = etp->et_enp;
494 efx_tx_ops_t *etxop = enp->en_etxop;
495
496 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
497
498 if (etxop->etxo_qpio_disable != NULL)
499 etxop->etxo_qpio_disable(etp);
500}
501
502 __checkReturn efx_rc_t
503efx_tx_qpio_write(
504 __in efx_txq_t *etp,
505 __in_ecount(buf_length) uint8_t *buffer,
506 __in size_t buf_length,
507 __in size_t pio_buf_offset)
508{
509 efx_nic_t *enp = etp->et_enp;
510 efx_tx_ops_t *etxop = enp->en_etxop;
511 efx_rc_t rc;
512
513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
514
515 if (etxop->etxo_qpio_write != NULL) {
516 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
517 pio_buf_offset)) != 0)
518 goto fail1;
519 return (0);
520 }
521
522 return (ENOTSUP);
523
524fail1:
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
526 return (rc);
527}
528
529 __checkReturn efx_rc_t
530efx_tx_qpio_post(
531 __in efx_txq_t *etp,
532 __in size_t pkt_length,
533 __in unsigned int completed,
534 __inout unsigned int *addedp)
535{
536 efx_nic_t *enp = etp->et_enp;
537 efx_tx_ops_t *etxop = enp->en_etxop;
538 efx_rc_t rc;
539
540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
541
542 if (etxop->etxo_qpio_post != NULL) {
543 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
544 addedp)) != 0)
545 goto fail1;
546 return (0);
547 }
548
549 return (ENOTSUP);
550
551fail1:
552 EFSYS_PROBE1(fail1, efx_rc_t, rc);
553 return (rc);
554}
555
556 __checkReturn efx_rc_t
557efx_tx_qdesc_post(
558 __in efx_txq_t *etp,
559 __in_ecount(n) efx_desc_t *ed,
560 __in unsigned int n,
561 __in unsigned int completed,
562 __inout unsigned int *addedp)
563{
564 efx_nic_t *enp = etp->et_enp;
565 efx_tx_ops_t *etxop = enp->en_etxop;
566 efx_rc_t rc;
567
568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
569
570 if ((rc = etxop->etxo_qdesc_post(etp, ed,
571 n, completed, addedp)) != 0)
572 goto fail1;
573
574 return (0);
575
576fail1:
577 EFSYS_PROBE1(fail1, efx_rc_t, rc);
578 return (rc);
579}
580
581 void
582efx_tx_qdesc_dma_create(
583 __in efx_txq_t *etp,
584 __in efsys_dma_addr_t addr,
585 __in size_t size,
586 __in boolean_t eop,
587 __out efx_desc_t *edp)
588{
589 efx_nic_t *enp = etp->et_enp;
590 efx_tx_ops_t *etxop = enp->en_etxop;
591
592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
594
595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
596}
597
598 void
599efx_tx_qdesc_tso_create(
600 __in efx_txq_t *etp,
601 __in uint16_t ipv4_id,
602 __in uint32_t tcp_seq,
603 __in uint8_t tcp_flags,
604 __out efx_desc_t *edp)
605{
606 efx_nic_t *enp = etp->et_enp;
607 efx_tx_ops_t *etxop = enp->en_etxop;
608
609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
611
612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
613}
614
615 void
616efx_tx_qdesc_tso2_create(
617 __in efx_txq_t *etp,
618 __in uint16_t ipv4_id,
619 __in uint32_t tcp_seq,
620 __in uint16_t mss,
621 __out_ecount(count) efx_desc_t *edp,
622 __in int count)
623{
624 efx_nic_t *enp = etp->et_enp;
625 efx_tx_ops_t *etxop = enp->en_etxop;
626
627 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
628 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
629
630 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
631}
632
633 void
634efx_tx_qdesc_vlantci_create(
635 __in efx_txq_t *etp,
636 __in uint16_t tci,
637 __out efx_desc_t *edp)
638{
639 efx_nic_t *enp = etp->et_enp;
640 efx_tx_ops_t *etxop = enp->en_etxop;
641
642 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
643 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
644
645 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
646}
647
648
649#if EFSYS_OPT_QSTATS
650 void
651efx_tx_qstats_update(
652 __in efx_txq_t *etp,
653 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
654{
655 efx_nic_t *enp = etp->et_enp;
656 efx_tx_ops_t *etxop = enp->en_etxop;
657
658 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
659
660 etxop->etxo_qstats_update(etp, stat);
661}
662#endif
663
664
665#if EFSYS_OPT_SIENA
666
667static __checkReturn efx_rc_t
668falconsiena_tx_init(
669 __in efx_nic_t *enp)
670{
671 efx_oword_t oword;
672
673 /*
674 * Disable the timer-based TX DMA backoff and allow TX DMA to be
675 * controlled by the RX FIFO fill level (although always allow a
676 * minimal trickle).
677 */
678 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
686
687 /*
688 * Filter all packets less than 14 bytes to avoid parsing
689 * errors.
690 */
691 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
692 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
693
694 /*
695 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
696 * descriptors (which is bad).
697 */
698 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
700 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
701
702 return (0);
703}
704
705#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
706 do { \
707 unsigned int id; \
708 size_t offset; \
709 efx_qword_t qword; \
710 \
711 id = (_added)++ & (_etp)->et_mask; \
712 offset = id * sizeof (efx_qword_t); \
713 \
714 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
715 unsigned int, id, efsys_dma_addr_t, (_addr), \
716 size_t, (_size), boolean_t, (_eop)); \
717 \
718 EFX_POPULATE_QWORD_4(qword, \
719 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
720 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
721 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
722 (uint32_t)((_addr) & 0xffffffff), \
723 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
724 (uint32_t)((_addr) >> 32)); \
725 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
726 \
727 _NOTE(CONSTANTCONDITION) \
728 } while (B_FALSE)
729
730static __checkReturn efx_rc_t
731falconsiena_tx_qpost(
732 __in efx_txq_t *etp,
733 __in_ecount(n) efx_buffer_t *eb,
734 __in unsigned int n,
735 __in unsigned int completed,
736 __inout unsigned int *addedp)
737{
738 unsigned int added = *addedp;
739 unsigned int i;
740 int rc = ENOSPC;
741
742 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
743 goto fail1;
744
745 for (i = 0; i < n; i++) {
746 efx_buffer_t *ebp = &eb[i];
747 efsys_dma_addr_t start = ebp->eb_addr;
748 size_t size = ebp->eb_size;
749 efsys_dma_addr_t end = start + size;
750
751 /* Fragments must not span 4k boundaries. */
752 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
753
754 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
755 }
756
757 EFX_TX_QSTAT_INCR(etp, TX_POST);
758
759 *addedp = added;
760 return (0);
761
762fail1:
763 EFSYS_PROBE1(fail1, efx_rc_t, rc);
764
765 return (rc);
766}
767
768static void
769falconsiena_tx_qpush(
770 __in efx_txq_t *etp,
771 __in unsigned int added,
772 __in unsigned int pushed)
773{
774 efx_nic_t *enp = etp->et_enp;
775 uint32_t wptr;
776 efx_dword_t dword;
777 efx_oword_t oword;
778
779 /* Push the populated descriptors out */
780 wptr = added & etp->et_mask;
781
782 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
783
784 /* Only write the third DWORD */
785 EFX_POPULATE_DWORD_1(dword,
786 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
787
788 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
789 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
790 wptr, pushed & etp->et_mask);
791 EFSYS_PIO_WRITE_BARRIER();
792 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
793 etp->et_index, &dword, B_FALSE);
794}
795
796#define EFX_MAX_PACE_VALUE 20
797#define EFX_TX_PACE_CLOCK_BASE 104
798
799static __checkReturn efx_rc_t
800falconsiena_tx_qpace(
801 __in efx_txq_t *etp,
802 __in unsigned int ns)
803{
804 efx_nic_t *enp = etp->et_enp;
805 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
806 efx_oword_t oword;
807 unsigned int pace_val;
808 unsigned int timer_period;
809 efx_rc_t rc;
810
811 if (ns == 0) {
812 pace_val = 0;
813 } else {
814 /*
815 * The pace_val to write into the table is s.t
816 * ns <= timer_period * (2 ^ pace_val)
817 */
818 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
819 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
820 if ((timer_period << pace_val) >= ns)
821 break;
822 }
823 }
824 if (pace_val > EFX_MAX_PACE_VALUE) {
825 rc = EINVAL;
826 goto fail1;
827 }
828
829 /* Update the pacing table */
830 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
831 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
832 &oword, B_TRUE);
833
834 return (0);
835
836fail1:
837 EFSYS_PROBE1(fail1, efx_rc_t, rc);
838
839 return (rc);
840}
841
842static __checkReturn efx_rc_t
843falconsiena_tx_qflush(
844 __in efx_txq_t *etp)
845{
846 efx_nic_t *enp = etp->et_enp;
847 efx_oword_t oword;
848 uint32_t label;
849
850 efx_tx_qpace(etp, 0);
851
852 label = etp->et_index;
853
854 /* Flush the queue */
855 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
856 FRF_AZ_TX_FLUSH_DESCQ, label);
857 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
858
859 return (0);
860}
861
862static void
863falconsiena_tx_qenable(
864 __in efx_txq_t *etp)
865{
866 efx_nic_t *enp = etp->et_enp;
867 efx_oword_t oword;
868
869 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
870 etp->et_index, &oword, B_TRUE);
871
872 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
873 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
874 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
875 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
876 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
877
878 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
879 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
880 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
881
882 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
883 etp->et_index, &oword, B_TRUE);
884}
885
886static __checkReturn efx_rc_t
887falconsiena_tx_qcreate(
888 __in efx_nic_t *enp,
889 __in unsigned int index,
890 __in unsigned int label,
891 __in efsys_mem_t *esmp,
892 __in size_t n,
893 __in uint32_t id,
894 __in uint16_t flags,
895 __in efx_evq_t *eep,
896 __in efx_txq_t *etp,
897 __out unsigned int *addedp)
898{
899 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
900 efx_oword_t oword;
901 uint32_t size;
902 efx_rc_t rc;
903
904 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
905 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
906 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
907
908 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
909 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
910
911 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
912 rc = EINVAL;
913 goto fail1;
914 }
915 if (index >= encp->enc_txq_limit) {
916 rc = EINVAL;
917 goto fail2;
918 }
919 for (size = 0;
920 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
921 size++)
922 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
923 break;
924 if (id + (1 << size) >= encp->enc_buftbl_limit) {
925 rc = EINVAL;
926 goto fail3;
927 }
928
929 /* Set up the new descriptor queue */
930 *addedp = 0;
931
932 EFX_POPULATE_OWORD_6(oword,
933 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
934 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
935 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
936 FRF_AZ_TX_DESCQ_LABEL, label,
937 FRF_AZ_TX_DESCQ_SIZE, size,
938 FRF_AZ_TX_DESCQ_TYPE, 0);
939
940 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
941 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
942 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
943 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
944 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
945
946 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
947 etp->et_index, &oword, B_TRUE);
948
949 return (0);
950
951fail3:
952 EFSYS_PROBE(fail3);
953fail2:
954 EFSYS_PROBE(fail2);
955fail1:
956 EFSYS_PROBE1(fail1, efx_rc_t, rc);
957
958 return (rc);
959}
960
961 __checkReturn efx_rc_t
962falconsiena_tx_qdesc_post(
963 __in efx_txq_t *etp,
964 __in_ecount(n) efx_desc_t *ed,
965 __in unsigned int n,
966 __in unsigned int completed,
967 __inout unsigned int *addedp)
968{
969 unsigned int added = *addedp;
970 unsigned int i;
971 efx_rc_t rc;
972
973 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
974 rc = ENOSPC;
975 goto fail1;
976 }
977
978 for (i = 0; i < n; i++) {
979 efx_desc_t *edp = &ed[i];
980 unsigned int id;
981 size_t offset;
982
983 id = added++ & etp->et_mask;
984 offset = id * sizeof (efx_desc_t);
985
986 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
987 }
988
989 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
990 unsigned int, added, unsigned int, n);
991
992 EFX_TX_QSTAT_INCR(etp, TX_POST);
993
994 *addedp = added;
995 return (0);
996
997fail1:
998 EFSYS_PROBE1(fail1, efx_rc_t, rc);
999 return (rc);
1000}
1001
1002 void
1003falconsiena_tx_qdesc_dma_create(
1004 __in efx_txq_t *etp,
1005 __in efsys_dma_addr_t addr,
1006 __in size_t size,
1007 __in boolean_t eop,
1008 __out efx_desc_t *edp)
1009{
1010 /* Fragments must not span 4k boundaries. */
1011 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
1012
1013 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1014 efsys_dma_addr_t, addr,
1015 size_t, size, boolean_t, eop);
1016
1017 EFX_POPULATE_QWORD_4(edp->ed_eq,
1018 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1019 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1020 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1021 (uint32_t)(addr & 0xffffffff),
1022 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1023 (uint32_t)(addr >> 32));
1024}
1025
1026#endif /* EFSYS_OPT_SIENA */
1027
1028#if EFSYS_OPT_QSTATS
1029#if EFSYS_OPT_NAMES
1030/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
1031static const char *__efx_tx_qstat_name[] = {
1032 "post",
1033 "post_pio",
1034};
1035/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1036
1037 const char *
1038efx_tx_qstat_name(
1039 __in efx_nic_t *enp,
1040 __in unsigned int id)
1041{
1042 _NOTE(ARGUNUSED(enp))
1043 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1044 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1045
1046 return (__efx_tx_qstat_name[id]);
1047}
1048#endif /* EFSYS_OPT_NAMES */
1049#endif /* EFSYS_OPT_QSTATS */
1050
1051#if EFSYS_OPT_SIENA
1052
1053#if EFSYS_OPT_QSTATS
1054static void
1055falconsiena_tx_qstats_update(
1056 __in efx_txq_t *etp,
1057 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1058{
1059 unsigned int id;
1060
1061 for (id = 0; id < TX_NQSTATS; id++) {
1062 efsys_stat_t *essp = &stat[id];
1063
1064 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1065 etp->et_stat[id] = 0;
1066 }
1067}
1068#endif /* EFSYS_OPT_QSTATS */
1069
1070static void
1071falconsiena_tx_qdestroy(
1072 __in efx_txq_t *etp)
1073{
1074 efx_nic_t *enp = etp->et_enp;
1075 efx_oword_t oword;
1076
1077 /* Purge descriptor queue */
1078 EFX_ZERO_OWORD(oword);
1079
1080 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1081 etp->et_index, &oword, B_TRUE);
1082}
1083
1084static void
1085falconsiena_tx_fini(
1086 __in efx_nic_t *enp)
1087{
1088 _NOTE(ARGUNUSED(enp))
1089}
1090
1091#endif /* EFSYS_OPT_SIENA */