1/*-
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 293891 2016-01-14 09:05:51Z arybchik $");
33
34#include "efx.h"
35#include "efx_impl.h"
36
37#if EFSYS_OPT_QSTATS
38#define EFX_TX_QSTAT_INCR(_etp, _stat) \
39 do { \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
42 } while (B_FALSE)
43#else
44#define EFX_TX_QSTAT_INCR(_etp, _stat)
45#endif
46
47#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
48
49static __checkReturn efx_rc_t
50falconsiena_tx_init(
51 __in efx_nic_t *enp);
52
53static void
54falconsiena_tx_fini(
55 __in efx_nic_t *enp);
56
57static __checkReturn efx_rc_t
58falconsiena_tx_qcreate(
59 __in efx_nic_t *enp,
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
63 __in size_t n,
64 __in uint32_t id,
65 __in uint16_t flags,
66 __in efx_evq_t *eep,
67 __in efx_txq_t *etp,
68 __out unsigned int *addedp);
69
70static void
71falconsiena_tx_qdestroy(
72 __in efx_txq_t *etp);
73
74static __checkReturn efx_rc_t
75falconsiena_tx_qpost(
76 __in efx_txq_t *etp,
77 __in_ecount(n) efx_buffer_t *eb,
78 __in unsigned int n,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82static void
83falconsiena_tx_qpush(
84 __in efx_txq_t *etp,
85 __in unsigned int added,
86 __in unsigned int pushed);
87
88static __checkReturn efx_rc_t
89falconsiena_tx_qpace(
90 __in efx_txq_t *etp,
91 __in unsigned int ns);
92
93static __checkReturn efx_rc_t
94falconsiena_tx_qflush(
95 __in efx_txq_t *etp);
96
97static void
98falconsiena_tx_qenable(
99 __in efx_txq_t *etp);
100
101 __checkReturn efx_rc_t
102falconsiena_tx_qdesc_post(
103 __in efx_txq_t *etp,
104 __in_ecount(n) efx_desc_t *ed,
105 __in unsigned int n,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
108
109 void
110falconsiena_tx_qdesc_dma_create(
111 __in efx_txq_t *etp,
112 __in efsys_dma_addr_t addr,
113 __in size_t size,
114 __in boolean_t eop,
115 __out efx_desc_t *edp);
116
117#if EFSYS_OPT_QSTATS
118static void
119falconsiena_tx_qstats_update(
120 __in efx_txq_t *etp,
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
122#endif
123
124#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
125
126
127#if EFSYS_OPT_FALCON
128static efx_tx_ops_t __efx_tx_falcon_ops = {
129 falconsiena_tx_init, /* etxo_init */
130 falconsiena_tx_fini, /* etxo_fini */
131 falconsiena_tx_qcreate, /* etxo_qcreate */
132 falconsiena_tx_qdestroy, /* etxo_qdestroy */
133 falconsiena_tx_qpost, /* etxo_qpost */
134 falconsiena_tx_qpush, /* etxo_qpush */
135 falconsiena_tx_qpace, /* etxo_qpace */
136 falconsiena_tx_qflush, /* etxo_qflush */
137 falconsiena_tx_qenable, /* etxo_qenable */
138 NULL, /* etxo_qpio_enable */
139 NULL, /* etxo_qpio_disable */
140 NULL, /* etxo_qpio_write */
141 NULL, /* etxo_qpio_post */
142 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
143 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 NULL, /* etxo_qdesc_tso_create */
145 NULL, /* etxo_qdesc_tso2_create */
146 NULL, /* etxo_qdesc_vlantci_create */
147#if EFSYS_OPT_QSTATS
148 falconsiena_tx_qstats_update, /* etxo_qstats_update */
149#endif
150};
151#endif /* EFSYS_OPT_FALCON */
152
153#if EFSYS_OPT_SIENA
154static efx_tx_ops_t __efx_tx_siena_ops = {
155 falconsiena_tx_init, /* etxo_init */
156 falconsiena_tx_fini, /* etxo_fini */
157 falconsiena_tx_qcreate, /* etxo_qcreate */
158 falconsiena_tx_qdestroy, /* etxo_qdestroy */
159 falconsiena_tx_qpost, /* etxo_qpost */
160 falconsiena_tx_qpush, /* etxo_qpush */
161 falconsiena_tx_qpace, /* etxo_qpace */
162 falconsiena_tx_qflush, /* etxo_qflush */
163 falconsiena_tx_qenable, /* etxo_qenable */
164 NULL, /* etxo_qpio_enable */
165 NULL, /* etxo_qpio_disable */
166 NULL, /* etxo_qpio_write */
167 NULL, /* etxo_qpio_post */
168 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
169 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
170 NULL, /* etxo_qdesc_tso_create */
171 NULL, /* etxo_qdesc_tso2_create */
172 NULL, /* etxo_qdesc_vlantci_create */
173#if EFSYS_OPT_QSTATS
174 falconsiena_tx_qstats_update, /* etxo_qstats_update */
175#endif
176};
177#endif /* EFSYS_OPT_SIENA */
178
179#if EFSYS_OPT_HUNTINGTON
180static efx_tx_ops_t __efx_tx_hunt_ops = {
181 ef10_tx_init, /* etxo_init */
182 ef10_tx_fini, /* etxo_fini */
183 ef10_tx_qcreate, /* etxo_qcreate */
184 ef10_tx_qdestroy, /* etxo_qdestroy */
185 ef10_tx_qpost, /* etxo_qpost */
186 ef10_tx_qpush, /* etxo_qpush */
187 ef10_tx_qpace, /* etxo_qpace */
188 ef10_tx_qflush, /* etxo_qflush */
189 ef10_tx_qenable, /* etxo_qenable */
190 ef10_tx_qpio_enable, /* etxo_qpio_enable */
191 ef10_tx_qpio_disable, /* etxo_qpio_disable */
192 ef10_tx_qpio_write, /* etxo_qpio_write */
193 ef10_tx_qpio_post, /* etxo_qpio_post */
194 ef10_tx_qdesc_post, /* etxo_qdesc_post */
195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
196 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
199#if EFSYS_OPT_QSTATS
200 ef10_tx_qstats_update, /* etxo_qstats_update */
201#endif
202};
203#endif /* EFSYS_OPT_HUNTINGTON */
204
205#if EFSYS_OPT_MEDFORD
206static efx_tx_ops_t __efx_tx_medford_ops = {
207 ef10_tx_init, /* etxo_init */
208 ef10_tx_fini, /* etxo_fini */
209 ef10_tx_qcreate, /* etxo_qcreate */
210 ef10_tx_qdestroy, /* etxo_qdestroy */
211 ef10_tx_qpost, /* etxo_qpost */
212 ef10_tx_qpush, /* etxo_qpush */
213 ef10_tx_qpace, /* etxo_qpace */
214 ef10_tx_qflush, /* etxo_qflush */
215 ef10_tx_qenable, /* etxo_qenable */
216 ef10_tx_qpio_enable, /* etxo_qpio_enable */
217 ef10_tx_qpio_disable, /* etxo_qpio_disable */
218 ef10_tx_qpio_write, /* etxo_qpio_write */
219 ef10_tx_qpio_post, /* etxo_qpio_post */
220 ef10_tx_qdesc_post, /* etxo_qdesc_post */
221 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
222 NULL, /* etxo_qdesc_tso_create */
223 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
224 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
225#if EFSYS_OPT_QSTATS
226 ef10_tx_qstats_update, /* etxo_qstats_update */
227#endif
228};
229#endif /* EFSYS_OPT_MEDFORD */
230
231 __checkReturn efx_rc_t
232efx_tx_init(
233 __in efx_nic_t *enp)
234{
235 efx_tx_ops_t *etxop;
236 efx_rc_t rc;
237
238 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
239 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
240
241 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
242 rc = EINVAL;
243 goto fail1;
244 }
245
246 if (enp->en_mod_flags & EFX_MOD_TX) {
247 rc = EINVAL;
248 goto fail2;
249 }
250
251 switch (enp->en_family) {
252#if EFSYS_OPT_FALCON
253 case EFX_FAMILY_FALCON:
254 etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops;
255 break;
256#endif /* EFSYS_OPT_FALCON */
257
258#if EFSYS_OPT_SIENA
259 case EFX_FAMILY_SIENA:
260 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
261 break;
262#endif /* EFSYS_OPT_SIENA */
263
264#if EFSYS_OPT_HUNTINGTON
265 case EFX_FAMILY_HUNTINGTON:
266 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
267 break;
268#endif /* EFSYS_OPT_HUNTINGTON */
269
270#if EFSYS_OPT_MEDFORD
271 case EFX_FAMILY_MEDFORD:
272 etxop = (efx_tx_ops_t *)&__efx_tx_medford_ops;
273 break;
274#endif /* EFSYS_OPT_MEDFORD */
275
276 default:
277 EFSYS_ASSERT(0);
278 rc = ENOTSUP;
279 goto fail3;
280 }
281
282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
283
284 if ((rc = etxop->etxo_init(enp)) != 0)
285 goto fail4;
286
287 enp->en_etxop = etxop;
288 enp->en_mod_flags |= EFX_MOD_TX;
289 return (0);
290
291fail4:
292 EFSYS_PROBE(fail4);
293fail3:
294 EFSYS_PROBE(fail3);
295fail2:
296 EFSYS_PROBE(fail2);
297fail1:
298 EFSYS_PROBE1(fail1, efx_rc_t, rc);
299
300 enp->en_etxop = NULL;
301 enp->en_mod_flags &= ~EFX_MOD_TX;
302 return (rc);
303}
304
305 void
306efx_tx_fini(
307 __in efx_nic_t *enp)
308{
309 efx_tx_ops_t *etxop = enp->en_etxop;
310
311 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
312 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
313 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
314 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
315
316 etxop->etxo_fini(enp);
317
318 enp->en_etxop = NULL;
319 enp->en_mod_flags &= ~EFX_MOD_TX;
320}
321
322 __checkReturn efx_rc_t
323efx_tx_qcreate(
324 __in efx_nic_t *enp,
325 __in unsigned int index,
326 __in unsigned int label,
327 __in efsys_mem_t *esmp,
328 __in size_t n,
329 __in uint32_t id,
330 __in uint16_t flags,
331 __in efx_evq_t *eep,
332 __deref_out efx_txq_t **etpp,
333 __out unsigned int *addedp)
334{
335 efx_tx_ops_t *etxop = enp->en_etxop;
336 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
337 efx_txq_t *etp;
338 efx_rc_t rc;
339
340 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
341 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
342
343 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
344
345 /* Allocate an TXQ object */
346 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
347
348 if (etp == NULL) {
349 rc = ENOMEM;
350 goto fail1;
351 }
352
353 etp->et_magic = EFX_TXQ_MAGIC;
354 etp->et_enp = enp;
355 etp->et_index = index;
356 etp->et_mask = n - 1;
357 etp->et_esmp = esmp;
358
359 /* Initial descriptor index may be modified by etxo_qcreate */
360 *addedp = 0;
361
362 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
363 n, id, flags, eep, etp, addedp)) != 0)
364 goto fail2;
365
366 enp->en_tx_qcount++;
367 *etpp = etp;
368
369 return (0);
370
371fail2:
372 EFSYS_PROBE(fail2);
373 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
374fail1:
375 EFSYS_PROBE1(fail1, efx_rc_t, rc);
376 return (rc);
377}
378
379 void
380efx_tx_qdestroy(
381 __in efx_txq_t *etp)
382{
383 efx_nic_t *enp = etp->et_enp;
384 efx_tx_ops_t *etxop = enp->en_etxop;
385
386 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
387
388 EFSYS_ASSERT(enp->en_tx_qcount != 0);
389 --enp->en_tx_qcount;
390
391 etxop->etxo_qdestroy(etp);
392
393 /* Free the TXQ object */
394 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
395}
396
397 __checkReturn efx_rc_t
398efx_tx_qpost(
399 __in efx_txq_t *etp,
400 __in_ecount(n) efx_buffer_t *eb,
401 __in unsigned int n,
402 __in unsigned int completed,
403 __inout unsigned int *addedp)
404{
405 efx_nic_t *enp = etp->et_enp;
406 efx_tx_ops_t *etxop = enp->en_etxop;
407 efx_rc_t rc;
408
409 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
410
411 if ((rc = etxop->etxo_qpost(etp, eb,
412 n, completed, addedp)) != 0)
413 goto fail1;
414
415 return (0);
416
417fail1:
418 EFSYS_PROBE1(fail1, efx_rc_t, rc);
419 return (rc);
420}
421
422 void
423efx_tx_qpush(
424 __in efx_txq_t *etp,
425 __in unsigned int added,
426 __in unsigned int pushed)
427{
428 efx_nic_t *enp = etp->et_enp;
429 efx_tx_ops_t *etxop = enp->en_etxop;
430
431 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
432
433 etxop->etxo_qpush(etp, added, pushed);
434}
435
436 __checkReturn efx_rc_t
437efx_tx_qpace(
438 __in efx_txq_t *etp,
439 __in unsigned int ns)
440{
441 efx_nic_t *enp = etp->et_enp;
442 efx_tx_ops_t *etxop = enp->en_etxop;
443 efx_rc_t rc;
444
445 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
446
447 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
448 goto fail1;
449
450 return (0);
451
452fail1:
453 EFSYS_PROBE1(fail1, efx_rc_t, rc);
454 return (rc);
455}
456
457 __checkReturn efx_rc_t
458efx_tx_qflush(
459 __in efx_txq_t *etp)
460{
461 efx_nic_t *enp = etp->et_enp;
462 efx_tx_ops_t *etxop = enp->en_etxop;
463 efx_rc_t rc;
464
465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
466
467 if ((rc = etxop->etxo_qflush(etp)) != 0)
468 goto fail1;
469
470 return (0);
471
472fail1:
473 EFSYS_PROBE1(fail1, efx_rc_t, rc);
474 return (rc);
475}
476
477 void
478efx_tx_qenable(
479 __in efx_txq_t *etp)
480{
481 efx_nic_t *enp = etp->et_enp;
482 efx_tx_ops_t *etxop = enp->en_etxop;
483
484 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
485
486 etxop->etxo_qenable(etp);
487}
488
489 __checkReturn efx_rc_t
490efx_tx_qpio_enable(
491 __in efx_txq_t *etp)
492{
493 efx_nic_t *enp = etp->et_enp;
494 efx_tx_ops_t *etxop = enp->en_etxop;
495 efx_rc_t rc;
496
497 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
498
499 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
500 rc = ENOTSUP;
501 goto fail1;
502 }
503 if (etxop->etxo_qpio_enable == NULL) {
504 rc = ENOTSUP;
505 goto fail2;
506 }
507 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
508 goto fail3;
509
510 return (0);
511
512fail3:
513 EFSYS_PROBE(fail3);
514fail2:
515 EFSYS_PROBE(fail2);
516fail1:
517 EFSYS_PROBE1(fail1, efx_rc_t, rc);
518 return (rc);
519}
520
521 void
522efx_tx_qpio_disable(
523 __in efx_txq_t *etp)
524{
525 efx_nic_t *enp = etp->et_enp;
526 efx_tx_ops_t *etxop = enp->en_etxop;
527
528 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
529
530 if (etxop->etxo_qpio_disable != NULL)
531 etxop->etxo_qpio_disable(etp);
532}
533
534 __checkReturn efx_rc_t
535efx_tx_qpio_write(
536 __in efx_txq_t *etp,
537 __in_ecount(buf_length) uint8_t *buffer,
538 __in size_t buf_length,
539 __in size_t pio_buf_offset)
540{
541 efx_nic_t *enp = etp->et_enp;
542 efx_tx_ops_t *etxop = enp->en_etxop;
543 efx_rc_t rc;
544
545 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
546
547 if (etxop->etxo_qpio_write != NULL) {
548 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
549 pio_buf_offset)) != 0)
550 goto fail1;
551 return (0);
552 }
553
554 return (ENOTSUP);
555
556fail1:
557 EFSYS_PROBE1(fail1, efx_rc_t, rc);
558 return (rc);
559}
560
561 __checkReturn efx_rc_t
562efx_tx_qpio_post(
563 __in efx_txq_t *etp,
564 __in size_t pkt_length,
565 __in unsigned int completed,
566 __inout unsigned int *addedp)
567{
568 efx_nic_t *enp = etp->et_enp;
569 efx_tx_ops_t *etxop = enp->en_etxop;
570 efx_rc_t rc;
571
572 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
573
574 if (etxop->etxo_qpio_post != NULL) {
575 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
576 addedp)) != 0)
577 goto fail1;
578 return (0);
579 }
580
581 return (ENOTSUP);
582
583fail1:
584 EFSYS_PROBE1(fail1, efx_rc_t, rc);
585 return (rc);
586}
587
588 __checkReturn efx_rc_t
589efx_tx_qdesc_post(
590 __in efx_txq_t *etp,
591 __in_ecount(n) efx_desc_t *ed,
592 __in unsigned int n,
593 __in unsigned int completed,
594 __inout unsigned int *addedp)
595{
596 efx_nic_t *enp = etp->et_enp;
597 efx_tx_ops_t *etxop = enp->en_etxop;
598 efx_rc_t rc;
599
600 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
601
602 if ((rc = etxop->etxo_qdesc_post(etp, ed,
603 n, completed, addedp)) != 0)
604 goto fail1;
605
606 return (0);
607
608fail1:
609 EFSYS_PROBE1(fail1, efx_rc_t, rc);
610 return (rc);
611}
612
613 void
614efx_tx_qdesc_dma_create(
615 __in efx_txq_t *etp,
616 __in efsys_dma_addr_t addr,
617 __in size_t size,
618 __in boolean_t eop,
619 __out efx_desc_t *edp)
620{
621 efx_nic_t *enp = etp->et_enp;
622 efx_tx_ops_t *etxop = enp->en_etxop;
623
624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
625 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
626
627 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
628}
629
630 void
631efx_tx_qdesc_tso_create(
632 __in efx_txq_t *etp,
633 __in uint16_t ipv4_id,
634 __in uint32_t tcp_seq,
635 __in uint8_t tcp_flags,
636 __out efx_desc_t *edp)
637{
638 efx_nic_t *enp = etp->et_enp;
639 efx_tx_ops_t *etxop = enp->en_etxop;
640
641 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
642 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
643
644 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
645}
646
647 void
648efx_tx_qdesc_tso2_create(
649 __in efx_txq_t *etp,
650 __in uint16_t ipv4_id,
651 __in uint32_t tcp_seq,
652 __in uint16_t mss,
653 __out_ecount(count) efx_desc_t *edp,
654 __in int count)
655{
656 efx_nic_t *enp = etp->et_enp;
657 efx_tx_ops_t *etxop = enp->en_etxop;
658
659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
660 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
661
662 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
663}
664
665 void
666efx_tx_qdesc_vlantci_create(
667 __in efx_txq_t *etp,
668 __in uint16_t tci,
669 __out efx_desc_t *edp)
670{
671 efx_nic_t *enp = etp->et_enp;
672 efx_tx_ops_t *etxop = enp->en_etxop;
673
674 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
675 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
676
677 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
678}
679
680
681#if EFSYS_OPT_QSTATS
682 void
683efx_tx_qstats_update(
684 __in efx_txq_t *etp,
685 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
686{
687 efx_nic_t *enp = etp->et_enp;
688 efx_tx_ops_t *etxop = enp->en_etxop;
689
690 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
691
692 etxop->etxo_qstats_update(etp, stat);
693}
694#endif
695
696
697#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
698
699static __checkReturn efx_rc_t
700falconsiena_tx_init(
701 __in efx_nic_t *enp)
702{
703 efx_oword_t oword;
704
705 /*
706 * Disable the timer-based TX DMA backoff and allow TX DMA to be
707 * controlled by the RX FIFO fill level (although always allow a
708 * minimal trickle).
709 */
710 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
711 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
712 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
713 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
714 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
715 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
716 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
717 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
718
719 /*
720 * Filter all packets less than 14 bytes to avoid parsing
721 * errors.
722 */
723 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
724 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
725
726 /*
727 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
728 * descriptors (which is bad).
729 */
730 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
732 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
733
734 return (0);
735}
736
737#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
738 do { \
739 unsigned int id; \
740 size_t offset; \
741 efx_qword_t qword; \
742 \
743 id = (_added)++ & (_etp)->et_mask; \
744 offset = id * sizeof (efx_qword_t); \
745 \
746 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
747 unsigned int, id, efsys_dma_addr_t, (_addr), \
748 size_t, (_size), boolean_t, (_eop)); \
749 \
750 EFX_POPULATE_QWORD_4(qword, \
751 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
752 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
753 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
754 (uint32_t)((_addr) & 0xffffffff), \
755 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
756 (uint32_t)((_addr) >> 32)); \
757 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
758 \
759 _NOTE(CONSTANTCONDITION) \
760 } while (B_FALSE)
761
762static __checkReturn efx_rc_t
763falconsiena_tx_qpost(
764 __in efx_txq_t *etp,
765 __in_ecount(n) efx_buffer_t *eb,
766 __in unsigned int n,
767 __in unsigned int completed,
768 __inout unsigned int *addedp)
769{
770 unsigned int added = *addedp;
771 unsigned int i;
772 int rc = ENOSPC;
773
774 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
775 goto fail1;
776
777 for (i = 0; i < n; i++) {
778 efx_buffer_t *ebp = &eb[i];
779 efsys_dma_addr_t start = ebp->eb_addr;
780 size_t size = ebp->eb_size;
781 efsys_dma_addr_t end = start + size;
782
783 /* Fragments must not span 4k boundaries. */
784 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
785
786 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
787 }
788
789 EFX_TX_QSTAT_INCR(etp, TX_POST);
790
791 *addedp = added;
792 return (0);
793
794fail1:
795 EFSYS_PROBE1(fail1, efx_rc_t, rc);
796
797 return (rc);
798}
799
800static void
801falconsiena_tx_qpush(
802 __in efx_txq_t *etp,
803 __in unsigned int added,
804 __in unsigned int pushed)
805{
806 efx_nic_t *enp = etp->et_enp;
807 uint32_t wptr;
808 efx_dword_t dword;
809 efx_oword_t oword;
810
811 /* Push the populated descriptors out */
812 wptr = added & etp->et_mask;
813
814 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
815
816 /* Only write the third DWORD */
817 EFX_POPULATE_DWORD_1(dword,
818 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
819
820 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
821 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
822 wptr, pushed & etp->et_mask);
823 EFSYS_PIO_WRITE_BARRIER();
824 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
825 etp->et_index, &dword, B_FALSE);
826}
827
828#define EFX_MAX_PACE_VALUE 20
829#define EFX_TX_PACE_CLOCK_BASE 104
830
831static __checkReturn efx_rc_t
832falconsiena_tx_qpace(
833 __in efx_txq_t *etp,
834 __in unsigned int ns)
835{
836 efx_nic_t *enp = etp->et_enp;
837 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
838 efx_oword_t oword;
839 unsigned int pace_val;
840 unsigned int timer_period;
841 efx_rc_t rc;
842
843 if (ns == 0) {
844 pace_val = 0;
845 } else {
846 /*
847 * The pace_val to write into the table is s.t
848 * ns <= timer_period * (2 ^ pace_val)
849 */
850 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
851 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
852 if ((timer_period << pace_val) >= ns)
853 break;
854 }
855 }
856 if (pace_val > EFX_MAX_PACE_VALUE) {
857 rc = EINVAL;
858 goto fail1;
859 }
860
861 /* Update the pacing table */
862 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
863 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
864 &oword, B_TRUE);
865
866 return (0);
867
868fail1:
869 EFSYS_PROBE1(fail1, efx_rc_t, rc);
870
871 return (rc);
872}
873
874static __checkReturn efx_rc_t
875falconsiena_tx_qflush(
876 __in efx_txq_t *etp)
877{
878 efx_nic_t *enp = etp->et_enp;
879 efx_oword_t oword;
880 uint32_t label;
881
882 efx_tx_qpace(etp, 0);
883
884 label = etp->et_index;
885
886 /* Flush the queue */
887 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
888 FRF_AZ_TX_FLUSH_DESCQ, label);
889 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
890
891 return (0);
892}
893
894static void
895falconsiena_tx_qenable(
896 __in efx_txq_t *etp)
897{
898 efx_nic_t *enp = etp->et_enp;
899 efx_oword_t oword;
900
901 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
902 etp->et_index, &oword, B_TRUE);
903
904 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
905 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
906 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
907 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
908 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
909
910 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
911 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
912 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
913
914 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
915 etp->et_index, &oword, B_TRUE);
916}
917
918static __checkReturn efx_rc_t
919falconsiena_tx_qcreate(
920 __in efx_nic_t *enp,
921 __in unsigned int index,
922 __in unsigned int label,
923 __in efsys_mem_t *esmp,
924 __in size_t n,
925 __in uint32_t id,
926 __in uint16_t flags,
927 __in efx_evq_t *eep,
928 __in efx_txq_t *etp,
929 __out unsigned int *addedp)
930{
931 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
932 efx_oword_t oword;
933 uint32_t size;
934 efx_rc_t rc;
935
936 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
937 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
938 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
939
940 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
941 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
942
943 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
944 rc = EINVAL;
945 goto fail1;
946 }
947 if (index >= encp->enc_txq_limit) {
948 rc = EINVAL;
949 goto fail2;
950 }
951 for (size = 0;
952 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
953 size++)
954 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
955 break;
956 if (id + (1 << size) >= encp->enc_buftbl_limit) {
957 rc = EINVAL;
958 goto fail3;
959 }
960
961 /* Set up the new descriptor queue */
962 *addedp = 0;
963
964 EFX_POPULATE_OWORD_6(oword,
965 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
966 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
967 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
968 FRF_AZ_TX_DESCQ_LABEL, label,
969 FRF_AZ_TX_DESCQ_SIZE, size,
970 FRF_AZ_TX_DESCQ_TYPE, 0);
971
972 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
973 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
974 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
975 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
976 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
977
978 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
979 etp->et_index, &oword, B_TRUE);
980
981 return (0);
982
983fail3:
984 EFSYS_PROBE(fail3);
985fail2:
986 EFSYS_PROBE(fail2);
987fail1:
988 EFSYS_PROBE1(fail1, efx_rc_t, rc);
989
990 return (rc);
991}
992
993 __checkReturn efx_rc_t
994falconsiena_tx_qdesc_post(
995 __in efx_txq_t *etp,
996 __in_ecount(n) efx_desc_t *ed,
997 __in unsigned int n,
998 __in unsigned int completed,
999 __inout unsigned int *addedp)
1000{
1001 unsigned int added = *addedp;
1002 unsigned int i;
1003 efx_rc_t rc;
1004
1005 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1006 rc = ENOSPC;
1007 goto fail1;
1008 }
1009
1010 for (i = 0; i < n; i++) {
1011 efx_desc_t *edp = &ed[i];
1012 unsigned int id;
1013 size_t offset;
1014
1015 id = added++ & etp->et_mask;
1016 offset = id * sizeof (efx_desc_t);
1017
1018 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1019 }
1020
1021 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1022 unsigned int, added, unsigned int, n);
1023
1024 EFX_TX_QSTAT_INCR(etp, TX_POST);
1025
1026 *addedp = added;
1027 return (0);
1028
1029fail1:
1030 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1031 return (rc);
1032}
1033
1034 void
1035falconsiena_tx_qdesc_dma_create(
1036 __in efx_txq_t *etp,
1037 __in efsys_dma_addr_t addr,
1038 __in size_t size,
1039 __in boolean_t eop,
1040 __out efx_desc_t *edp)
1041{
1042 /* Fragments must not span 4k boundaries. */
1043 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
1044
1045 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1046 efsys_dma_addr_t, addr,
1047 size_t, size, boolean_t, eop);
1048
1049 EFX_POPULATE_QWORD_4(edp->ed_eq,
1050 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1051 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1052 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1053 (uint32_t)(addr & 0xffffffff),
1054 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1055 (uint32_t)(addr >> 32));
1056}
1057
1058#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1059
1060#if EFSYS_OPT_QSTATS
1061#if EFSYS_OPT_NAMES
1062/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
1063static const char *__efx_tx_qstat_name[] = {
1064 "post",
1065 "post_pio",
1066};
1067/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1068
1069 const char *
1070efx_tx_qstat_name(
1071 __in efx_nic_t *enp,
1072 __in unsigned int id)
1073{
1074 _NOTE(ARGUNUSED(enp))
1075 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1076 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1077
1078 return (__efx_tx_qstat_name[id]);
1079}
1080#endif /* EFSYS_OPT_NAMES */
1081#endif /* EFSYS_OPT_QSTATS */
1082
1083#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
1084
1085#if EFSYS_OPT_QSTATS
1086static void
1087falconsiena_tx_qstats_update(
1088 __in efx_txq_t *etp,
1089 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1090{
1091 unsigned int id;
1092
1093 for (id = 0; id < TX_NQSTATS; id++) {
1094 efsys_stat_t *essp = &stat[id];
1095
1096 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1097 etp->et_stat[id] = 0;
1098 }
1099}
1100#endif /* EFSYS_OPT_QSTATS */
1101
1102static void
1103falconsiena_tx_qdestroy(
1104 __in efx_txq_t *etp)
1105{
1106 efx_nic_t *enp = etp->et_enp;
1107 efx_oword_t oword;
1108
1109 /* Purge descriptor queue */
1110 EFX_ZERO_OWORD(oword);
1111
1112 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1113 etp->et_index, &oword, B_TRUE);
1114}
1115
1116static void
1117falconsiena_tx_fini(
1118 __in efx_nic_t *enp)
1119{
1120 _NOTE(ARGUNUSED(enp))
1121}
1122
1123#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 293891 2016-01-14 09:05:51Z arybchik $");
33
34#include "efx.h"
35#include "efx_impl.h"
36
37#if EFSYS_OPT_QSTATS
38#define EFX_TX_QSTAT_INCR(_etp, _stat) \
39 do { \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
42 } while (B_FALSE)
43#else
44#define EFX_TX_QSTAT_INCR(_etp, _stat)
45#endif
46
47#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
48
49static __checkReturn efx_rc_t
50falconsiena_tx_init(
51 __in efx_nic_t *enp);
52
53static void
54falconsiena_tx_fini(
55 __in efx_nic_t *enp);
56
57static __checkReturn efx_rc_t
58falconsiena_tx_qcreate(
59 __in efx_nic_t *enp,
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
63 __in size_t n,
64 __in uint32_t id,
65 __in uint16_t flags,
66 __in efx_evq_t *eep,
67 __in efx_txq_t *etp,
68 __out unsigned int *addedp);
69
70static void
71falconsiena_tx_qdestroy(
72 __in efx_txq_t *etp);
73
74static __checkReturn efx_rc_t
75falconsiena_tx_qpost(
76 __in efx_txq_t *etp,
77 __in_ecount(n) efx_buffer_t *eb,
78 __in unsigned int n,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82static void
83falconsiena_tx_qpush(
84 __in efx_txq_t *etp,
85 __in unsigned int added,
86 __in unsigned int pushed);
87
88static __checkReturn efx_rc_t
89falconsiena_tx_qpace(
90 __in efx_txq_t *etp,
91 __in unsigned int ns);
92
93static __checkReturn efx_rc_t
94falconsiena_tx_qflush(
95 __in efx_txq_t *etp);
96
97static void
98falconsiena_tx_qenable(
99 __in efx_txq_t *etp);
100
101 __checkReturn efx_rc_t
102falconsiena_tx_qdesc_post(
103 __in efx_txq_t *etp,
104 __in_ecount(n) efx_desc_t *ed,
105 __in unsigned int n,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
108
109 void
110falconsiena_tx_qdesc_dma_create(
111 __in efx_txq_t *etp,
112 __in efsys_dma_addr_t addr,
113 __in size_t size,
114 __in boolean_t eop,
115 __out efx_desc_t *edp);
116
117#if EFSYS_OPT_QSTATS
118static void
119falconsiena_tx_qstats_update(
120 __in efx_txq_t *etp,
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
122#endif
123
124#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
125
126
127#if EFSYS_OPT_FALCON
128static efx_tx_ops_t __efx_tx_falcon_ops = {
129 falconsiena_tx_init, /* etxo_init */
130 falconsiena_tx_fini, /* etxo_fini */
131 falconsiena_tx_qcreate, /* etxo_qcreate */
132 falconsiena_tx_qdestroy, /* etxo_qdestroy */
133 falconsiena_tx_qpost, /* etxo_qpost */
134 falconsiena_tx_qpush, /* etxo_qpush */
135 falconsiena_tx_qpace, /* etxo_qpace */
136 falconsiena_tx_qflush, /* etxo_qflush */
137 falconsiena_tx_qenable, /* etxo_qenable */
138 NULL, /* etxo_qpio_enable */
139 NULL, /* etxo_qpio_disable */
140 NULL, /* etxo_qpio_write */
141 NULL, /* etxo_qpio_post */
142 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
143 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 NULL, /* etxo_qdesc_tso_create */
145 NULL, /* etxo_qdesc_tso2_create */
146 NULL, /* etxo_qdesc_vlantci_create */
147#if EFSYS_OPT_QSTATS
148 falconsiena_tx_qstats_update, /* etxo_qstats_update */
149#endif
150};
151#endif /* EFSYS_OPT_FALCON */
152
153#if EFSYS_OPT_SIENA
154static efx_tx_ops_t __efx_tx_siena_ops = {
155 falconsiena_tx_init, /* etxo_init */
156 falconsiena_tx_fini, /* etxo_fini */
157 falconsiena_tx_qcreate, /* etxo_qcreate */
158 falconsiena_tx_qdestroy, /* etxo_qdestroy */
159 falconsiena_tx_qpost, /* etxo_qpost */
160 falconsiena_tx_qpush, /* etxo_qpush */
161 falconsiena_tx_qpace, /* etxo_qpace */
162 falconsiena_tx_qflush, /* etxo_qflush */
163 falconsiena_tx_qenable, /* etxo_qenable */
164 NULL, /* etxo_qpio_enable */
165 NULL, /* etxo_qpio_disable */
166 NULL, /* etxo_qpio_write */
167 NULL, /* etxo_qpio_post */
168 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
169 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
170 NULL, /* etxo_qdesc_tso_create */
171 NULL, /* etxo_qdesc_tso2_create */
172 NULL, /* etxo_qdesc_vlantci_create */
173#if EFSYS_OPT_QSTATS
174 falconsiena_tx_qstats_update, /* etxo_qstats_update */
175#endif
176};
177#endif /* EFSYS_OPT_SIENA */
178
179#if EFSYS_OPT_HUNTINGTON
180static efx_tx_ops_t __efx_tx_hunt_ops = {
181 ef10_tx_init, /* etxo_init */
182 ef10_tx_fini, /* etxo_fini */
183 ef10_tx_qcreate, /* etxo_qcreate */
184 ef10_tx_qdestroy, /* etxo_qdestroy */
185 ef10_tx_qpost, /* etxo_qpost */
186 ef10_tx_qpush, /* etxo_qpush */
187 ef10_tx_qpace, /* etxo_qpace */
188 ef10_tx_qflush, /* etxo_qflush */
189 ef10_tx_qenable, /* etxo_qenable */
190 ef10_tx_qpio_enable, /* etxo_qpio_enable */
191 ef10_tx_qpio_disable, /* etxo_qpio_disable */
192 ef10_tx_qpio_write, /* etxo_qpio_write */
193 ef10_tx_qpio_post, /* etxo_qpio_post */
194 ef10_tx_qdesc_post, /* etxo_qdesc_post */
195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
196 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
199#if EFSYS_OPT_QSTATS
200 ef10_tx_qstats_update, /* etxo_qstats_update */
201#endif
202};
203#endif /* EFSYS_OPT_HUNTINGTON */
204
205#if EFSYS_OPT_MEDFORD
206static efx_tx_ops_t __efx_tx_medford_ops = {
207 ef10_tx_init, /* etxo_init */
208 ef10_tx_fini, /* etxo_fini */
209 ef10_tx_qcreate, /* etxo_qcreate */
210 ef10_tx_qdestroy, /* etxo_qdestroy */
211 ef10_tx_qpost, /* etxo_qpost */
212 ef10_tx_qpush, /* etxo_qpush */
213 ef10_tx_qpace, /* etxo_qpace */
214 ef10_tx_qflush, /* etxo_qflush */
215 ef10_tx_qenable, /* etxo_qenable */
216 ef10_tx_qpio_enable, /* etxo_qpio_enable */
217 ef10_tx_qpio_disable, /* etxo_qpio_disable */
218 ef10_tx_qpio_write, /* etxo_qpio_write */
219 ef10_tx_qpio_post, /* etxo_qpio_post */
220 ef10_tx_qdesc_post, /* etxo_qdesc_post */
221 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
222 NULL, /* etxo_qdesc_tso_create */
223 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
224 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
225#if EFSYS_OPT_QSTATS
226 ef10_tx_qstats_update, /* etxo_qstats_update */
227#endif
228};
229#endif /* EFSYS_OPT_MEDFORD */
230
231 __checkReturn efx_rc_t
232efx_tx_init(
233 __in efx_nic_t *enp)
234{
235 efx_tx_ops_t *etxop;
236 efx_rc_t rc;
237
238 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
239 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
240
241 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
242 rc = EINVAL;
243 goto fail1;
244 }
245
246 if (enp->en_mod_flags & EFX_MOD_TX) {
247 rc = EINVAL;
248 goto fail2;
249 }
250
251 switch (enp->en_family) {
252#if EFSYS_OPT_FALCON
253 case EFX_FAMILY_FALCON:
254 etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops;
255 break;
256#endif /* EFSYS_OPT_FALCON */
257
258#if EFSYS_OPT_SIENA
259 case EFX_FAMILY_SIENA:
260 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
261 break;
262#endif /* EFSYS_OPT_SIENA */
263
264#if EFSYS_OPT_HUNTINGTON
265 case EFX_FAMILY_HUNTINGTON:
266 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
267 break;
268#endif /* EFSYS_OPT_HUNTINGTON */
269
270#if EFSYS_OPT_MEDFORD
271 case EFX_FAMILY_MEDFORD:
272 etxop = (efx_tx_ops_t *)&__efx_tx_medford_ops;
273 break;
274#endif /* EFSYS_OPT_MEDFORD */
275
276 default:
277 EFSYS_ASSERT(0);
278 rc = ENOTSUP;
279 goto fail3;
280 }
281
282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
283
284 if ((rc = etxop->etxo_init(enp)) != 0)
285 goto fail4;
286
287 enp->en_etxop = etxop;
288 enp->en_mod_flags |= EFX_MOD_TX;
289 return (0);
290
291fail4:
292 EFSYS_PROBE(fail4);
293fail3:
294 EFSYS_PROBE(fail3);
295fail2:
296 EFSYS_PROBE(fail2);
297fail1:
298 EFSYS_PROBE1(fail1, efx_rc_t, rc);
299
300 enp->en_etxop = NULL;
301 enp->en_mod_flags &= ~EFX_MOD_TX;
302 return (rc);
303}
304
305 void
306efx_tx_fini(
307 __in efx_nic_t *enp)
308{
309 efx_tx_ops_t *etxop = enp->en_etxop;
310
311 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
312 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
313 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
314 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
315
316 etxop->etxo_fini(enp);
317
318 enp->en_etxop = NULL;
319 enp->en_mod_flags &= ~EFX_MOD_TX;
320}
321
322 __checkReturn efx_rc_t
323efx_tx_qcreate(
324 __in efx_nic_t *enp,
325 __in unsigned int index,
326 __in unsigned int label,
327 __in efsys_mem_t *esmp,
328 __in size_t n,
329 __in uint32_t id,
330 __in uint16_t flags,
331 __in efx_evq_t *eep,
332 __deref_out efx_txq_t **etpp,
333 __out unsigned int *addedp)
334{
335 efx_tx_ops_t *etxop = enp->en_etxop;
336 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
337 efx_txq_t *etp;
338 efx_rc_t rc;
339
340 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
341 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
342
343 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
344
345 /* Allocate an TXQ object */
346 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
347
348 if (etp == NULL) {
349 rc = ENOMEM;
350 goto fail1;
351 }
352
353 etp->et_magic = EFX_TXQ_MAGIC;
354 etp->et_enp = enp;
355 etp->et_index = index;
356 etp->et_mask = n - 1;
357 etp->et_esmp = esmp;
358
359 /* Initial descriptor index may be modified by etxo_qcreate */
360 *addedp = 0;
361
362 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
363 n, id, flags, eep, etp, addedp)) != 0)
364 goto fail2;
365
366 enp->en_tx_qcount++;
367 *etpp = etp;
368
369 return (0);
370
371fail2:
372 EFSYS_PROBE(fail2);
373 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
374fail1:
375 EFSYS_PROBE1(fail1, efx_rc_t, rc);
376 return (rc);
377}
378
379 void
380efx_tx_qdestroy(
381 __in efx_txq_t *etp)
382{
383 efx_nic_t *enp = etp->et_enp;
384 efx_tx_ops_t *etxop = enp->en_etxop;
385
386 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
387
388 EFSYS_ASSERT(enp->en_tx_qcount != 0);
389 --enp->en_tx_qcount;
390
391 etxop->etxo_qdestroy(etp);
392
393 /* Free the TXQ object */
394 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
395}
396
397 __checkReturn efx_rc_t
398efx_tx_qpost(
399 __in efx_txq_t *etp,
400 __in_ecount(n) efx_buffer_t *eb,
401 __in unsigned int n,
402 __in unsigned int completed,
403 __inout unsigned int *addedp)
404{
405 efx_nic_t *enp = etp->et_enp;
406 efx_tx_ops_t *etxop = enp->en_etxop;
407 efx_rc_t rc;
408
409 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
410
411 if ((rc = etxop->etxo_qpost(etp, eb,
412 n, completed, addedp)) != 0)
413 goto fail1;
414
415 return (0);
416
417fail1:
418 EFSYS_PROBE1(fail1, efx_rc_t, rc);
419 return (rc);
420}
421
422 void
423efx_tx_qpush(
424 __in efx_txq_t *etp,
425 __in unsigned int added,
426 __in unsigned int pushed)
427{
428 efx_nic_t *enp = etp->et_enp;
429 efx_tx_ops_t *etxop = enp->en_etxop;
430
431 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
432
433 etxop->etxo_qpush(etp, added, pushed);
434}
435
436 __checkReturn efx_rc_t
437efx_tx_qpace(
438 __in efx_txq_t *etp,
439 __in unsigned int ns)
440{
441 efx_nic_t *enp = etp->et_enp;
442 efx_tx_ops_t *etxop = enp->en_etxop;
443 efx_rc_t rc;
444
445 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
446
447 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
448 goto fail1;
449
450 return (0);
451
452fail1:
453 EFSYS_PROBE1(fail1, efx_rc_t, rc);
454 return (rc);
455}
456
457 __checkReturn efx_rc_t
458efx_tx_qflush(
459 __in efx_txq_t *etp)
460{
461 efx_nic_t *enp = etp->et_enp;
462 efx_tx_ops_t *etxop = enp->en_etxop;
463 efx_rc_t rc;
464
465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
466
467 if ((rc = etxop->etxo_qflush(etp)) != 0)
468 goto fail1;
469
470 return (0);
471
472fail1:
473 EFSYS_PROBE1(fail1, efx_rc_t, rc);
474 return (rc);
475}
476
477 void
478efx_tx_qenable(
479 __in efx_txq_t *etp)
480{
481 efx_nic_t *enp = etp->et_enp;
482 efx_tx_ops_t *etxop = enp->en_etxop;
483
484 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
485
486 etxop->etxo_qenable(etp);
487}
488
489 __checkReturn efx_rc_t
490efx_tx_qpio_enable(
491 __in efx_txq_t *etp)
492{
493 efx_nic_t *enp = etp->et_enp;
494 efx_tx_ops_t *etxop = enp->en_etxop;
495 efx_rc_t rc;
496
497 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
498
499 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
500 rc = ENOTSUP;
501 goto fail1;
502 }
503 if (etxop->etxo_qpio_enable == NULL) {
504 rc = ENOTSUP;
505 goto fail2;
506 }
507 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
508 goto fail3;
509
510 return (0);
511
512fail3:
513 EFSYS_PROBE(fail3);
514fail2:
515 EFSYS_PROBE(fail2);
516fail1:
517 EFSYS_PROBE1(fail1, efx_rc_t, rc);
518 return (rc);
519}
520
521 void
522efx_tx_qpio_disable(
523 __in efx_txq_t *etp)
524{
525 efx_nic_t *enp = etp->et_enp;
526 efx_tx_ops_t *etxop = enp->en_etxop;
527
528 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
529
530 if (etxop->etxo_qpio_disable != NULL)
531 etxop->etxo_qpio_disable(etp);
532}
533
534 __checkReturn efx_rc_t
535efx_tx_qpio_write(
536 __in efx_txq_t *etp,
537 __in_ecount(buf_length) uint8_t *buffer,
538 __in size_t buf_length,
539 __in size_t pio_buf_offset)
540{
541 efx_nic_t *enp = etp->et_enp;
542 efx_tx_ops_t *etxop = enp->en_etxop;
543 efx_rc_t rc;
544
545 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
546
547 if (etxop->etxo_qpio_write != NULL) {
548 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
549 pio_buf_offset)) != 0)
550 goto fail1;
551 return (0);
552 }
553
554 return (ENOTSUP);
555
556fail1:
557 EFSYS_PROBE1(fail1, efx_rc_t, rc);
558 return (rc);
559}
560
561 __checkReturn efx_rc_t
562efx_tx_qpio_post(
563 __in efx_txq_t *etp,
564 __in size_t pkt_length,
565 __in unsigned int completed,
566 __inout unsigned int *addedp)
567{
568 efx_nic_t *enp = etp->et_enp;
569 efx_tx_ops_t *etxop = enp->en_etxop;
570 efx_rc_t rc;
571
572 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
573
574 if (etxop->etxo_qpio_post != NULL) {
575 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
576 addedp)) != 0)
577 goto fail1;
578 return (0);
579 }
580
581 return (ENOTSUP);
582
583fail1:
584 EFSYS_PROBE1(fail1, efx_rc_t, rc);
585 return (rc);
586}
587
588 __checkReturn efx_rc_t
589efx_tx_qdesc_post(
590 __in efx_txq_t *etp,
591 __in_ecount(n) efx_desc_t *ed,
592 __in unsigned int n,
593 __in unsigned int completed,
594 __inout unsigned int *addedp)
595{
596 efx_nic_t *enp = etp->et_enp;
597 efx_tx_ops_t *etxop = enp->en_etxop;
598 efx_rc_t rc;
599
600 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
601
602 if ((rc = etxop->etxo_qdesc_post(etp, ed,
603 n, completed, addedp)) != 0)
604 goto fail1;
605
606 return (0);
607
608fail1:
609 EFSYS_PROBE1(fail1, efx_rc_t, rc);
610 return (rc);
611}
612
613 void
614efx_tx_qdesc_dma_create(
615 __in efx_txq_t *etp,
616 __in efsys_dma_addr_t addr,
617 __in size_t size,
618 __in boolean_t eop,
619 __out efx_desc_t *edp)
620{
621 efx_nic_t *enp = etp->et_enp;
622 efx_tx_ops_t *etxop = enp->en_etxop;
623
624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
625 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
626
627 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
628}
629
630 void
631efx_tx_qdesc_tso_create(
632 __in efx_txq_t *etp,
633 __in uint16_t ipv4_id,
634 __in uint32_t tcp_seq,
635 __in uint8_t tcp_flags,
636 __out efx_desc_t *edp)
637{
638 efx_nic_t *enp = etp->et_enp;
639 efx_tx_ops_t *etxop = enp->en_etxop;
640
641 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
642 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
643
644 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
645}
646
647 void
648efx_tx_qdesc_tso2_create(
649 __in efx_txq_t *etp,
650 __in uint16_t ipv4_id,
651 __in uint32_t tcp_seq,
652 __in uint16_t mss,
653 __out_ecount(count) efx_desc_t *edp,
654 __in int count)
655{
656 efx_nic_t *enp = etp->et_enp;
657 efx_tx_ops_t *etxop = enp->en_etxop;
658
659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
660 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
661
662 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
663}
664
665 void
666efx_tx_qdesc_vlantci_create(
667 __in efx_txq_t *etp,
668 __in uint16_t tci,
669 __out efx_desc_t *edp)
670{
671 efx_nic_t *enp = etp->et_enp;
672 efx_tx_ops_t *etxop = enp->en_etxop;
673
674 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
675 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
676
677 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
678}
679
680
681#if EFSYS_OPT_QSTATS
682 void
683efx_tx_qstats_update(
684 __in efx_txq_t *etp,
685 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
686{
687 efx_nic_t *enp = etp->et_enp;
688 efx_tx_ops_t *etxop = enp->en_etxop;
689
690 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
691
692 etxop->etxo_qstats_update(etp, stat);
693}
694#endif
695
696
697#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
698
699static __checkReturn efx_rc_t
700falconsiena_tx_init(
701 __in efx_nic_t *enp)
702{
703 efx_oword_t oword;
704
705 /*
706 * Disable the timer-based TX DMA backoff and allow TX DMA to be
707 * controlled by the RX FIFO fill level (although always allow a
708 * minimal trickle).
709 */
710 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
711 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
712 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
713 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
714 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
715 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
716 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
717 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
718
719 /*
720 * Filter all packets less than 14 bytes to avoid parsing
721 * errors.
722 */
723 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
724 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
725
726 /*
727 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
728 * descriptors (which is bad).
729 */
730 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
732 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
733
734 return (0);
735}
736
737#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
738 do { \
739 unsigned int id; \
740 size_t offset; \
741 efx_qword_t qword; \
742 \
743 id = (_added)++ & (_etp)->et_mask; \
744 offset = id * sizeof (efx_qword_t); \
745 \
746 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
747 unsigned int, id, efsys_dma_addr_t, (_addr), \
748 size_t, (_size), boolean_t, (_eop)); \
749 \
750 EFX_POPULATE_QWORD_4(qword, \
751 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
752 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
753 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
754 (uint32_t)((_addr) & 0xffffffff), \
755 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
756 (uint32_t)((_addr) >> 32)); \
757 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
758 \
759 _NOTE(CONSTANTCONDITION) \
760 } while (B_FALSE)
761
762static __checkReturn efx_rc_t
763falconsiena_tx_qpost(
764 __in efx_txq_t *etp,
765 __in_ecount(n) efx_buffer_t *eb,
766 __in unsigned int n,
767 __in unsigned int completed,
768 __inout unsigned int *addedp)
769{
770 unsigned int added = *addedp;
771 unsigned int i;
772 int rc = ENOSPC;
773
774 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
775 goto fail1;
776
777 for (i = 0; i < n; i++) {
778 efx_buffer_t *ebp = &eb[i];
779 efsys_dma_addr_t start = ebp->eb_addr;
780 size_t size = ebp->eb_size;
781 efsys_dma_addr_t end = start + size;
782
783 /* Fragments must not span 4k boundaries. */
784 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
785
786 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
787 }
788
789 EFX_TX_QSTAT_INCR(etp, TX_POST);
790
791 *addedp = added;
792 return (0);
793
794fail1:
795 EFSYS_PROBE1(fail1, efx_rc_t, rc);
796
797 return (rc);
798}
799
800static void
801falconsiena_tx_qpush(
802 __in efx_txq_t *etp,
803 __in unsigned int added,
804 __in unsigned int pushed)
805{
806 efx_nic_t *enp = etp->et_enp;
807 uint32_t wptr;
808 efx_dword_t dword;
809 efx_oword_t oword;
810
811 /* Push the populated descriptors out */
812 wptr = added & etp->et_mask;
813
814 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
815
816 /* Only write the third DWORD */
817 EFX_POPULATE_DWORD_1(dword,
818 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
819
820 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
821 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
822 wptr, pushed & etp->et_mask);
823 EFSYS_PIO_WRITE_BARRIER();
824 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
825 etp->et_index, &dword, B_FALSE);
826}
827
828#define EFX_MAX_PACE_VALUE 20
829#define EFX_TX_PACE_CLOCK_BASE 104
830
831static __checkReturn efx_rc_t
832falconsiena_tx_qpace(
833 __in efx_txq_t *etp,
834 __in unsigned int ns)
835{
836 efx_nic_t *enp = etp->et_enp;
837 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
838 efx_oword_t oword;
839 unsigned int pace_val;
840 unsigned int timer_period;
841 efx_rc_t rc;
842
843 if (ns == 0) {
844 pace_val = 0;
845 } else {
846 /*
847 * The pace_val to write into the table is s.t
848 * ns <= timer_period * (2 ^ pace_val)
849 */
850 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
851 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
852 if ((timer_period << pace_val) >= ns)
853 break;
854 }
855 }
856 if (pace_val > EFX_MAX_PACE_VALUE) {
857 rc = EINVAL;
858 goto fail1;
859 }
860
861 /* Update the pacing table */
862 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
863 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
864 &oword, B_TRUE);
865
866 return (0);
867
868fail1:
869 EFSYS_PROBE1(fail1, efx_rc_t, rc);
870
871 return (rc);
872}
873
874static __checkReturn efx_rc_t
875falconsiena_tx_qflush(
876 __in efx_txq_t *etp)
877{
878 efx_nic_t *enp = etp->et_enp;
879 efx_oword_t oword;
880 uint32_t label;
881
882 efx_tx_qpace(etp, 0);
883
884 label = etp->et_index;
885
886 /* Flush the queue */
887 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
888 FRF_AZ_TX_FLUSH_DESCQ, label);
889 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
890
891 return (0);
892}
893
894static void
895falconsiena_tx_qenable(
896 __in efx_txq_t *etp)
897{
898 efx_nic_t *enp = etp->et_enp;
899 efx_oword_t oword;
900
901 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
902 etp->et_index, &oword, B_TRUE);
903
904 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
905 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
906 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
907 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
908 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
909
910 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
911 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
912 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
913
914 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
915 etp->et_index, &oword, B_TRUE);
916}
917
918static __checkReturn efx_rc_t
919falconsiena_tx_qcreate(
920 __in efx_nic_t *enp,
921 __in unsigned int index,
922 __in unsigned int label,
923 __in efsys_mem_t *esmp,
924 __in size_t n,
925 __in uint32_t id,
926 __in uint16_t flags,
927 __in efx_evq_t *eep,
928 __in efx_txq_t *etp,
929 __out unsigned int *addedp)
930{
931 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
932 efx_oword_t oword;
933 uint32_t size;
934 efx_rc_t rc;
935
936 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
937 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
938 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
939
940 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
941 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
942
943 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
944 rc = EINVAL;
945 goto fail1;
946 }
947 if (index >= encp->enc_txq_limit) {
948 rc = EINVAL;
949 goto fail2;
950 }
951 for (size = 0;
952 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
953 size++)
954 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
955 break;
956 if (id + (1 << size) >= encp->enc_buftbl_limit) {
957 rc = EINVAL;
958 goto fail3;
959 }
960
961 /* Set up the new descriptor queue */
962 *addedp = 0;
963
964 EFX_POPULATE_OWORD_6(oword,
965 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
966 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
967 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
968 FRF_AZ_TX_DESCQ_LABEL, label,
969 FRF_AZ_TX_DESCQ_SIZE, size,
970 FRF_AZ_TX_DESCQ_TYPE, 0);
971
972 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
973 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
974 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
975 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
976 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
977
978 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
979 etp->et_index, &oword, B_TRUE);
980
981 return (0);
982
983fail3:
984 EFSYS_PROBE(fail3);
985fail2:
986 EFSYS_PROBE(fail2);
987fail1:
988 EFSYS_PROBE1(fail1, efx_rc_t, rc);
989
990 return (rc);
991}
992
993 __checkReturn efx_rc_t
994falconsiena_tx_qdesc_post(
995 __in efx_txq_t *etp,
996 __in_ecount(n) efx_desc_t *ed,
997 __in unsigned int n,
998 __in unsigned int completed,
999 __inout unsigned int *addedp)
1000{
1001 unsigned int added = *addedp;
1002 unsigned int i;
1003 efx_rc_t rc;
1004
1005 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1006 rc = ENOSPC;
1007 goto fail1;
1008 }
1009
1010 for (i = 0; i < n; i++) {
1011 efx_desc_t *edp = &ed[i];
1012 unsigned int id;
1013 size_t offset;
1014
1015 id = added++ & etp->et_mask;
1016 offset = id * sizeof (efx_desc_t);
1017
1018 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1019 }
1020
1021 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1022 unsigned int, added, unsigned int, n);
1023
1024 EFX_TX_QSTAT_INCR(etp, TX_POST);
1025
1026 *addedp = added;
1027 return (0);
1028
1029fail1:
1030 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1031 return (rc);
1032}
1033
1034 void
1035falconsiena_tx_qdesc_dma_create(
1036 __in efx_txq_t *etp,
1037 __in efsys_dma_addr_t addr,
1038 __in size_t size,
1039 __in boolean_t eop,
1040 __out efx_desc_t *edp)
1041{
1042 /* Fragments must not span 4k boundaries. */
1043 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
1044
1045 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1046 efsys_dma_addr_t, addr,
1047 size_t, size, boolean_t, eop);
1048
1049 EFX_POPULATE_QWORD_4(edp->ed_eq,
1050 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1051 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1052 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1053 (uint32_t)(addr & 0xffffffff),
1054 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1055 (uint32_t)(addr >> 32));
1056}
1057
1058#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1059
1060#if EFSYS_OPT_QSTATS
1061#if EFSYS_OPT_NAMES
1062/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
1063static const char *__efx_tx_qstat_name[] = {
1064 "post",
1065 "post_pio",
1066};
1067/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1068
1069 const char *
1070efx_tx_qstat_name(
1071 __in efx_nic_t *enp,
1072 __in unsigned int id)
1073{
1074 _NOTE(ARGUNUSED(enp))
1075 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1076 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1077
1078 return (__efx_tx_qstat_name[id]);
1079}
1080#endif /* EFSYS_OPT_NAMES */
1081#endif /* EFSYS_OPT_QSTATS */
1082
1083#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
1084
1085#if EFSYS_OPT_QSTATS
1086static void
1087falconsiena_tx_qstats_update(
1088 __in efx_txq_t *etp,
1089 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1090{
1091 unsigned int id;
1092
1093 for (id = 0; id < TX_NQSTATS; id++) {
1094 efsys_stat_t *essp = &stat[id];
1095
1096 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1097 etp->et_stat[id] = 0;
1098 }
1099}
1100#endif /* EFSYS_OPT_QSTATS */
1101
1102static void
1103falconsiena_tx_qdestroy(
1104 __in efx_txq_t *etp)
1105{
1106 efx_nic_t *enp = etp->et_enp;
1107 efx_oword_t oword;
1108
1109 /* Purge descriptor queue */
1110 EFX_ZERO_OWORD(oword);
1111
1112 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1113 etp->et_index, &oword, B_TRUE);
1114}
1115
1116static void
1117falconsiena_tx_fini(
1118 __in efx_nic_t *enp)
1119{
1120 _NOTE(ARGUNUSED(enp))
1121}
1122
1123#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */