efx_tx.c revision 291396
1227569Sphilip/*- 2283514Sarybchik * Copyright (c) 2007-2015 Solarflare Communications Inc. 3283514Sarybchik * All rights reserved. 4227569Sphilip * 5227569Sphilip * Redistribution and use in source and binary forms, with or without 6283514Sarybchik * modification, are permitted provided that the following conditions are met: 7227569Sphilip * 8283514Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 9283514Sarybchik * this list of conditions and the following disclaimer. 10283514Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 11283514Sarybchik * this list of conditions and the following disclaimer in the documentation 12283514Sarybchik * and/or other materials provided with the distribution. 13283514Sarybchik * 14283514Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15283514Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16283514Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17283514Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18283514Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19283514Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20283514Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21283514Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22283514Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23283514Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24283514Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25283514Sarybchik * 26283514Sarybchik * The views and conclusions contained in the software and documentation are 27283514Sarybchik * those of the authors and should not be interpreted as representing official 28283514Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 29227569Sphilip */ 30227569Sphilip 31228078Sphilip#include <sys/cdefs.h> 32228078Sphilip__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_tx.c 291396 2015-11-27 16:18:59Z arybchik $"); 33228078Sphilip 34227569Sphilip#include "efsys.h" 35227569Sphilip#include "efx.h" 36227569Sphilip#include "efx_types.h" 37227569Sphilip#include "efx_regs.h" 38227569Sphilip#include "efx_impl.h" 39227569Sphilip 40227569Sphilip#if EFSYS_OPT_QSTATS 41227569Sphilip#define EFX_TX_QSTAT_INCR(_etp, _stat) \ 42227569Sphilip do { \ 43227569Sphilip (_etp)->et_stat[_stat]++; \ 44227569Sphilip _NOTE(CONSTANTCONDITION) \ 45227569Sphilip } while (B_FALSE) 46227569Sphilip#else 47227569Sphilip#define EFX_TX_QSTAT_INCR(_etp, _stat) 48227569Sphilip#endif 49227569Sphilip 50283514Sarybchik#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 51283514Sarybchik 52283514Sarybchikstatic __checkReturn int 53283514Sarybchikfalconsiena_tx_init( 54283514Sarybchik __in efx_nic_t *enp); 55283514Sarybchik 56283514Sarybchikstatic void 57283514Sarybchikfalconsiena_tx_fini( 58283514Sarybchik __in efx_nic_t *enp); 59283514Sarybchik 60283514Sarybchikstatic __checkReturn int 61283514Sarybchikfalconsiena_tx_qcreate( 62283514Sarybchik __in efx_nic_t *enp, 63283514Sarybchik __in unsigned int index, 64283514Sarybchik __in unsigned int label, 65283514Sarybchik __in efsys_mem_t *esmp, 66283514Sarybchik __in size_t n, 67283514Sarybchik __in uint32_t id, 68283514Sarybchik __in uint16_t flags, 69283514Sarybchik __in efx_evq_t *eep, 70283514Sarybchik __in efx_txq_t *etp, 71283514Sarybchik __out unsigned int *addedp); 72283514Sarybchik 73283514Sarybchikstatic void 74283514Sarybchikfalconsiena_tx_qdestroy( 75283514Sarybchik __in efx_txq_t *etp); 76283514Sarybchik 77283514Sarybchikstatic __checkReturn int 78283514Sarybchikfalconsiena_tx_qpost( 79283514Sarybchik __in efx_txq_t *etp, 80283514Sarybchik __in_ecount(n) efx_buffer_t *eb, 81283514Sarybchik __in unsigned int n, 82283514Sarybchik __in unsigned int completed, 83283514Sarybchik __inout unsigned int *addedp); 84283514Sarybchik 85283514Sarybchikstatic void 86283514Sarybchikfalconsiena_tx_qpush( 87283514Sarybchik __in efx_txq_t *etp, 88283514Sarybchik __in unsigned int added, 89283514Sarybchik __in unsigned int pushed); 90283514Sarybchik 91283514Sarybchikstatic __checkReturn int 92283514Sarybchikfalconsiena_tx_qpace( 93283514Sarybchik __in efx_txq_t *etp, 94283514Sarybchik __in unsigned int ns); 95283514Sarybchik 96283514Sarybchikstatic __checkReturn int 97283514Sarybchikfalconsiena_tx_qflush( 98283514Sarybchik __in efx_txq_t *etp); 99283514Sarybchik 100283514Sarybchikstatic void 101283514Sarybchikfalconsiena_tx_qenable( 102283514Sarybchik __in efx_txq_t *etp); 103283514Sarybchik 104227569Sphilip __checkReturn int 105283514Sarybchikfalconsiena_tx_qdesc_post( 106283514Sarybchik __in efx_txq_t *etp, 107283514Sarybchik __in_ecount(n) efx_desc_t *ed, 108283514Sarybchik __in unsigned int n, 109283514Sarybchik __in unsigned int completed, 110283514Sarybchik __inout unsigned int *addedp); 111283514Sarybchik 112283514Sarybchik void 113283514Sarybchikfalconsiena_tx_qdesc_dma_create( 114283514Sarybchik __in efx_txq_t *etp, 115283514Sarybchik __in efsys_dma_addr_t addr, 116283514Sarybchik __in size_t size, 117283514Sarybchik __in boolean_t eop, 118283514Sarybchik __out efx_desc_t *edp); 119283514Sarybchik 120283514Sarybchik#if EFSYS_OPT_QSTATS 121283514Sarybchikstatic void 122283514Sarybchikfalconsiena_tx_qstats_update( 123283514Sarybchik __in efx_txq_t *etp, 124283514Sarybchik __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 125283514Sarybchik#endif 126283514Sarybchik 127283514Sarybchik#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 128283514Sarybchik 129283514Sarybchik 130283514Sarybchik#if EFSYS_OPT_FALCON 131283514Sarybchikstatic efx_tx_ops_t __efx_tx_falcon_ops = { 132283514Sarybchik falconsiena_tx_init, /* etxo_init */ 133283514Sarybchik falconsiena_tx_fini, /* etxo_fini */ 134283514Sarybchik falconsiena_tx_qcreate, /* etxo_qcreate */ 135283514Sarybchik falconsiena_tx_qdestroy, /* etxo_qdestroy */ 136283514Sarybchik falconsiena_tx_qpost, /* etxo_qpost */ 137283514Sarybchik falconsiena_tx_qpush, /* etxo_qpush */ 138283514Sarybchik falconsiena_tx_qpace, /* etxo_qpace */ 139283514Sarybchik falconsiena_tx_qflush, /* etxo_qflush */ 140283514Sarybchik falconsiena_tx_qenable, /* etxo_qenable */ 141283514Sarybchik NULL, /* etxo_qpio_enable */ 142283514Sarybchik NULL, /* etxo_qpio_disable */ 143283514Sarybchik NULL, /* etxo_qpio_write */ 144283514Sarybchik NULL, /* etxo_qpio_post */ 145283514Sarybchik falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 146283514Sarybchik falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 147283514Sarybchik NULL, /* etxo_qdesc_tso_create */ 148283514Sarybchik NULL, /* etxo_qdesc_vlantci_create */ 149283514Sarybchik#if EFSYS_OPT_QSTATS 150283514Sarybchik falconsiena_tx_qstats_update, /* etxo_qstats_update */ 151283514Sarybchik#endif 152283514Sarybchik}; 153283514Sarybchik#endif /* EFSYS_OPT_FALCON */ 154283514Sarybchik 155283514Sarybchik#if EFSYS_OPT_SIENA 156283514Sarybchikstatic efx_tx_ops_t __efx_tx_siena_ops = { 157283514Sarybchik falconsiena_tx_init, /* etxo_init */ 158283514Sarybchik falconsiena_tx_fini, /* etxo_fini */ 159283514Sarybchik falconsiena_tx_qcreate, /* etxo_qcreate */ 160283514Sarybchik falconsiena_tx_qdestroy, /* etxo_qdestroy */ 161283514Sarybchik falconsiena_tx_qpost, /* etxo_qpost */ 162283514Sarybchik falconsiena_tx_qpush, /* etxo_qpush */ 163283514Sarybchik falconsiena_tx_qpace, /* etxo_qpace */ 164283514Sarybchik falconsiena_tx_qflush, /* etxo_qflush */ 165283514Sarybchik falconsiena_tx_qenable, /* etxo_qenable */ 166283514Sarybchik NULL, /* etxo_qpio_enable */ 167283514Sarybchik NULL, /* etxo_qpio_disable */ 168283514Sarybchik NULL, /* etxo_qpio_write */ 169283514Sarybchik NULL, /* etxo_qpio_post */ 170283514Sarybchik falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 171283514Sarybchik falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 172283514Sarybchik NULL, /* etxo_qdesc_tso_create */ 173283514Sarybchik NULL, /* etxo_qdesc_vlantci_create */ 174283514Sarybchik#if EFSYS_OPT_QSTATS 175283514Sarybchik falconsiena_tx_qstats_update, /* etxo_qstats_update */ 176283514Sarybchik#endif 177283514Sarybchik}; 178283514Sarybchik#endif /* EFSYS_OPT_SIENA */ 179283514Sarybchik 180283514Sarybchik#if EFSYS_OPT_HUNTINGTON 181283514Sarybchikstatic efx_tx_ops_t __efx_tx_hunt_ops = { 182283514Sarybchik hunt_tx_init, /* etxo_init */ 183283514Sarybchik hunt_tx_fini, /* etxo_fini */ 184283514Sarybchik hunt_tx_qcreate, /* etxo_qcreate */ 185283514Sarybchik hunt_tx_qdestroy, /* etxo_qdestroy */ 186283514Sarybchik hunt_tx_qpost, /* etxo_qpost */ 187283514Sarybchik hunt_tx_qpush, /* etxo_qpush */ 188283514Sarybchik hunt_tx_qpace, /* etxo_qpace */ 189283514Sarybchik hunt_tx_qflush, /* etxo_qflush */ 190283514Sarybchik hunt_tx_qenable, /* etxo_qenable */ 191283514Sarybchik hunt_tx_qpio_enable, /* etxo_qpio_enable */ 192283514Sarybchik hunt_tx_qpio_disable, /* etxo_qpio_disable */ 193283514Sarybchik hunt_tx_qpio_write, /* etxo_qpio_write */ 194283514Sarybchik hunt_tx_qpio_post, /* etxo_qpio_post */ 195283514Sarybchik hunt_tx_qdesc_post, /* etxo_qdesc_post */ 196283514Sarybchik hunt_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 197283514Sarybchik hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 198283514Sarybchik hunt_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 199283514Sarybchik#if EFSYS_OPT_QSTATS 200283514Sarybchik hunt_tx_qstats_update, /* etxo_qstats_update */ 201283514Sarybchik#endif 202283514Sarybchik}; 203283514Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */ 204283514Sarybchik 205283514Sarybchik __checkReturn int 206227569Sphilipefx_tx_init( 207227569Sphilip __in efx_nic_t *enp) 208227569Sphilip{ 209283514Sarybchik efx_tx_ops_t *etxop; 210227569Sphilip int rc; 211227569Sphilip 212227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 213227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 214227569Sphilip 215227569Sphilip if (!(enp->en_mod_flags & EFX_MOD_EV)) { 216227569Sphilip rc = EINVAL; 217227569Sphilip goto fail1; 218227569Sphilip } 219227569Sphilip 220227569Sphilip if (enp->en_mod_flags & EFX_MOD_TX) { 221227569Sphilip rc = EINVAL; 222227569Sphilip goto fail2; 223227569Sphilip } 224227569Sphilip 225283514Sarybchik switch (enp->en_family) { 226283514Sarybchik#if EFSYS_OPT_FALCON 227283514Sarybchik case EFX_FAMILY_FALCON: 228283514Sarybchik etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops; 229283514Sarybchik break; 230283514Sarybchik#endif /* EFSYS_OPT_FALCON */ 231283514Sarybchik 232283514Sarybchik#if EFSYS_OPT_SIENA 233283514Sarybchik case EFX_FAMILY_SIENA: 234283514Sarybchik etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops; 235283514Sarybchik break; 236283514Sarybchik#endif /* EFSYS_OPT_SIENA */ 237283514Sarybchik 238283514Sarybchik#if EFSYS_OPT_HUNTINGTON 239283514Sarybchik case EFX_FAMILY_HUNTINGTON: 240283514Sarybchik etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops; 241283514Sarybchik break; 242283514Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */ 243283514Sarybchik 244283514Sarybchik default: 245283514Sarybchik EFSYS_ASSERT(0); 246283514Sarybchik rc = ENOTSUP; 247283514Sarybchik goto fail3; 248283514Sarybchik } 249283514Sarybchik 250227569Sphilip EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 251227569Sphilip 252283514Sarybchik if ((rc = etxop->etxo_init(enp)) != 0) 253283514Sarybchik goto fail4; 254283514Sarybchik 255283514Sarybchik enp->en_etxop = etxop; 256283514Sarybchik enp->en_mod_flags |= EFX_MOD_TX; 257283514Sarybchik return (0); 258283514Sarybchik 259283514Sarybchikfail4: 260283514Sarybchik EFSYS_PROBE(fail4); 261283514Sarybchikfail3: 262283514Sarybchik EFSYS_PROBE(fail3); 263283514Sarybchikfail2: 264283514Sarybchik EFSYS_PROBE(fail2); 265283514Sarybchikfail1: 266283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 267283514Sarybchik 268283514Sarybchik enp->en_etxop = NULL; 269283514Sarybchik enp->en_mod_flags &= ~EFX_MOD_TX; 270283514Sarybchik return (rc); 271283514Sarybchik} 272283514Sarybchik 273283514Sarybchik void 274283514Sarybchikefx_tx_fini( 275283514Sarybchik __in efx_nic_t *enp) 276283514Sarybchik{ 277283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 278283514Sarybchik 279283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 280283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 281283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 282283514Sarybchik EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 283283514Sarybchik 284283514Sarybchik etxop->etxo_fini(enp); 285283514Sarybchik 286283514Sarybchik enp->en_etxop = NULL; 287283514Sarybchik enp->en_mod_flags &= ~EFX_MOD_TX; 288283514Sarybchik} 289283514Sarybchik 290283514Sarybchik __checkReturn int 291283514Sarybchikefx_tx_qcreate( 292283514Sarybchik __in efx_nic_t *enp, 293283514Sarybchik __in unsigned int index, 294283514Sarybchik __in unsigned int label, 295283514Sarybchik __in efsys_mem_t *esmp, 296283514Sarybchik __in size_t n, 297283514Sarybchik __in uint32_t id, 298283514Sarybchik __in uint16_t flags, 299283514Sarybchik __in efx_evq_t *eep, 300283514Sarybchik __deref_out efx_txq_t **etpp, 301283514Sarybchik __out unsigned int *addedp) 302283514Sarybchik{ 303283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 304283514Sarybchik efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 305283514Sarybchik efx_txq_t *etp; 306283514Sarybchik int rc; 307283514Sarybchik 308283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 309283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 310283514Sarybchik 311283514Sarybchik EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 312283514Sarybchik 313283514Sarybchik /* Allocate an TXQ object */ 314283514Sarybchik EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 315283514Sarybchik 316283514Sarybchik if (etp == NULL) { 317283514Sarybchik rc = ENOMEM; 318283514Sarybchik goto fail1; 319283514Sarybchik } 320283514Sarybchik 321283514Sarybchik etp->et_magic = EFX_TXQ_MAGIC; 322283514Sarybchik etp->et_enp = enp; 323283514Sarybchik etp->et_index = index; 324283514Sarybchik etp->et_mask = n - 1; 325283514Sarybchik etp->et_esmp = esmp; 326283514Sarybchik 327283514Sarybchik /* Initial descriptor index may be modified by etxo_qcreate */ 328283514Sarybchik *addedp = 0; 329283514Sarybchik 330283514Sarybchik if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 331283514Sarybchik n, id, flags, eep, etp, addedp)) != 0) 332283514Sarybchik goto fail2; 333283514Sarybchik 334283514Sarybchik enp->en_tx_qcount++; 335283514Sarybchik *etpp = etp; 336283514Sarybchik 337283514Sarybchik return (0); 338283514Sarybchik 339283514Sarybchikfail2: 340283514Sarybchik EFSYS_PROBE(fail2); 341283514Sarybchik EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 342283514Sarybchikfail1: 343283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 344283514Sarybchik return (rc); 345283514Sarybchik} 346283514Sarybchik 347283514Sarybchik void 348283514Sarybchikefx_tx_qdestroy( 349283514Sarybchik __in efx_txq_t *etp) 350283514Sarybchik{ 351283514Sarybchik efx_nic_t *enp = etp->et_enp; 352283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 353283514Sarybchik 354283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 355283514Sarybchik 356283514Sarybchik EFSYS_ASSERT(enp->en_tx_qcount != 0); 357283514Sarybchik --enp->en_tx_qcount; 358283514Sarybchik 359283514Sarybchik etxop->etxo_qdestroy(etp); 360283514Sarybchik 361283514Sarybchik /* Free the TXQ object */ 362283514Sarybchik EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 363283514Sarybchik} 364283514Sarybchik 365283514Sarybchik __checkReturn int 366283514Sarybchikefx_tx_qpost( 367283514Sarybchik __in efx_txq_t *etp, 368283514Sarybchik __in_ecount(n) efx_buffer_t *eb, 369283514Sarybchik __in unsigned int n, 370283514Sarybchik __in unsigned int completed, 371283514Sarybchik __inout unsigned int *addedp) 372283514Sarybchik{ 373283514Sarybchik efx_nic_t *enp = etp->et_enp; 374283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 375283514Sarybchik int rc; 376283514Sarybchik 377283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 378283514Sarybchik 379283514Sarybchik if ((rc = etxop->etxo_qpost(etp, eb, 380283514Sarybchik n, completed, addedp)) != 0) 381283514Sarybchik goto fail1; 382283514Sarybchik 383283514Sarybchik return (0); 384283514Sarybchik 385283514Sarybchikfail1: 386283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 387283514Sarybchik return (rc); 388283514Sarybchik} 389283514Sarybchik 390283514Sarybchik void 391283514Sarybchikefx_tx_qpush( 392283514Sarybchik __in efx_txq_t *etp, 393283514Sarybchik __in unsigned int added, 394283514Sarybchik __in unsigned int pushed) 395283514Sarybchik{ 396283514Sarybchik efx_nic_t *enp = etp->et_enp; 397283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 398283514Sarybchik 399283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 400283514Sarybchik 401283514Sarybchik etxop->etxo_qpush(etp, added, pushed); 402283514Sarybchik} 403283514Sarybchik 404283514Sarybchik __checkReturn int 405283514Sarybchikefx_tx_qpace( 406283514Sarybchik __in efx_txq_t *etp, 407283514Sarybchik __in unsigned int ns) 408283514Sarybchik{ 409283514Sarybchik efx_nic_t *enp = etp->et_enp; 410283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 411283514Sarybchik int rc; 412283514Sarybchik 413283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 414283514Sarybchik 415283514Sarybchik if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 416283514Sarybchik goto fail1; 417283514Sarybchik 418283514Sarybchik return (0); 419283514Sarybchik 420283514Sarybchikfail1: 421283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 422283514Sarybchik return (rc); 423283514Sarybchik} 424283514Sarybchik 425283514Sarybchik __checkReturn int 426283514Sarybchikefx_tx_qflush( 427283514Sarybchik __in efx_txq_t *etp) 428283514Sarybchik{ 429283514Sarybchik efx_nic_t *enp = etp->et_enp; 430283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 431283514Sarybchik int rc; 432283514Sarybchik 433283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 434283514Sarybchik 435283514Sarybchik if ((rc = etxop->etxo_qflush(etp)) != 0) 436283514Sarybchik goto fail1; 437283514Sarybchik 438283514Sarybchik return (0); 439283514Sarybchik 440283514Sarybchikfail1: 441283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 442283514Sarybchik return (rc); 443283514Sarybchik} 444283514Sarybchik 445283514Sarybchik void 446283514Sarybchikefx_tx_qenable( 447283514Sarybchik __in efx_txq_t *etp) 448283514Sarybchik{ 449283514Sarybchik efx_nic_t *enp = etp->et_enp; 450283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 451283514Sarybchik 452283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 453283514Sarybchik 454283514Sarybchik etxop->etxo_qenable(etp); 455283514Sarybchik} 456283514Sarybchik 457283514Sarybchik __checkReturn int 458283514Sarybchikefx_tx_qpio_enable( 459283514Sarybchik __in efx_txq_t *etp) 460283514Sarybchik{ 461283514Sarybchik efx_nic_t *enp = etp->et_enp; 462283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 463283514Sarybchik int rc; 464283514Sarybchik 465283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 466283514Sarybchik 467283514Sarybchik if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 468283514Sarybchik rc = ENOTSUP; 469283514Sarybchik goto fail1; 470283514Sarybchik } 471283514Sarybchik if (etxop->etxo_qpio_enable == NULL) { 472283514Sarybchik rc = ENOTSUP; 473283514Sarybchik goto fail2; 474283514Sarybchik } 475283514Sarybchik if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 476283514Sarybchik goto fail3; 477283514Sarybchik 478283514Sarybchik return (0); 479283514Sarybchik 480283514Sarybchikfail3: 481283514Sarybchik EFSYS_PROBE(fail3); 482283514Sarybchikfail2: 483283514Sarybchik EFSYS_PROBE(fail2); 484283514Sarybchikfail1: 485283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 486283514Sarybchik return (rc); 487283514Sarybchik} 488283514Sarybchik 489283514Sarybchik void 490283514Sarybchikefx_tx_qpio_disable( 491283514Sarybchik __in efx_txq_t *etp) 492283514Sarybchik{ 493283514Sarybchik efx_nic_t *enp = etp->et_enp; 494283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 495283514Sarybchik 496283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 497283514Sarybchik 498283514Sarybchik if (etxop->etxo_qpio_disable != NULL) 499283514Sarybchik etxop->etxo_qpio_disable(etp); 500283514Sarybchik} 501283514Sarybchik 502283514Sarybchik __checkReturn int 503283514Sarybchikefx_tx_qpio_write( 504283514Sarybchik __in efx_txq_t *etp, 505283514Sarybchik __in_ecount(buf_length) uint8_t *buffer, 506283514Sarybchik __in size_t buf_length, 507283514Sarybchik __in size_t pio_buf_offset) 508283514Sarybchik{ 509283514Sarybchik efx_nic_t *enp = etp->et_enp; 510283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 511283514Sarybchik int rc; 512283514Sarybchik 513283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 514283514Sarybchik 515283514Sarybchik if (etxop->etxo_qpio_write != NULL) { 516283514Sarybchik if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 517283514Sarybchik pio_buf_offset)) != 0) 518283514Sarybchik goto fail1; 519283514Sarybchik return (0); 520283514Sarybchik } 521283514Sarybchik 522283514Sarybchik return (ENOTSUP); 523283514Sarybchik 524283514Sarybchikfail1: 525283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 526283514Sarybchik return (rc); 527283514Sarybchik} 528283514Sarybchik 529283514Sarybchik __checkReturn int 530283514Sarybchikefx_tx_qpio_post( 531283514Sarybchik __in efx_txq_t *etp, 532283514Sarybchik __in size_t pkt_length, 533283514Sarybchik __in unsigned int completed, 534283514Sarybchik __inout unsigned int *addedp) 535283514Sarybchik{ 536283514Sarybchik efx_nic_t *enp = etp->et_enp; 537283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 538283514Sarybchik int rc; 539283514Sarybchik 540283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 541283514Sarybchik 542283514Sarybchik if (etxop->etxo_qpio_post != NULL) { 543283514Sarybchik if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 544283514Sarybchik addedp)) != 0) 545283514Sarybchik goto fail1; 546283514Sarybchik return (0); 547283514Sarybchik } 548283514Sarybchik 549283514Sarybchik return (ENOTSUP); 550283514Sarybchik 551283514Sarybchikfail1: 552283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 553283514Sarybchik return (rc); 554283514Sarybchik} 555283514Sarybchik 556283514Sarybchik __checkReturn int 557283514Sarybchikefx_tx_qdesc_post( 558283514Sarybchik __in efx_txq_t *etp, 559283514Sarybchik __in_ecount(n) efx_desc_t *ed, 560283514Sarybchik __in unsigned int n, 561283514Sarybchik __in unsigned int completed, 562283514Sarybchik __inout unsigned int *addedp) 563283514Sarybchik{ 564283514Sarybchik efx_nic_t *enp = etp->et_enp; 565283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 566283514Sarybchik int rc; 567283514Sarybchik 568283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 569283514Sarybchik 570283514Sarybchik if ((rc = etxop->etxo_qdesc_post(etp, ed, 571283514Sarybchik n, completed, addedp)) != 0) 572283514Sarybchik goto fail1; 573283514Sarybchik 574283514Sarybchik return (0); 575283514Sarybchik 576283514Sarybchikfail1: 577283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 578283514Sarybchik return (rc); 579283514Sarybchik} 580283514Sarybchik 581283514Sarybchik void 582283514Sarybchikefx_tx_qdesc_dma_create( 583283514Sarybchik __in efx_txq_t *etp, 584283514Sarybchik __in efsys_dma_addr_t addr, 585283514Sarybchik __in size_t size, 586283514Sarybchik __in boolean_t eop, 587283514Sarybchik __out efx_desc_t *edp) 588283514Sarybchik{ 589283514Sarybchik efx_nic_t *enp = etp->et_enp; 590283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 591283514Sarybchik 592283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 593283514Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 594283514Sarybchik 595283514Sarybchik etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 596283514Sarybchik} 597283514Sarybchik 598283514Sarybchik void 599283514Sarybchikefx_tx_qdesc_tso_create( 600283514Sarybchik __in efx_txq_t *etp, 601283514Sarybchik __in uint16_t ipv4_id, 602283514Sarybchik __in uint32_t tcp_seq, 603283514Sarybchik __in uint8_t tcp_flags, 604283514Sarybchik __out efx_desc_t *edp) 605283514Sarybchik{ 606283514Sarybchik efx_nic_t *enp = etp->et_enp; 607283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 608283514Sarybchik 609283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 610283514Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 611283514Sarybchik 612283514Sarybchik etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 613283514Sarybchik} 614283514Sarybchik 615283514Sarybchik void 616283514Sarybchikefx_tx_qdesc_vlantci_create( 617283514Sarybchik __in efx_txq_t *etp, 618283514Sarybchik __in uint16_t tci, 619283514Sarybchik __out efx_desc_t *edp) 620283514Sarybchik{ 621283514Sarybchik efx_nic_t *enp = etp->et_enp; 622283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 623283514Sarybchik 624283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 625283514Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 626283514Sarybchik 627283514Sarybchik etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 628283514Sarybchik} 629283514Sarybchik 630283514Sarybchik 631283514Sarybchik#if EFSYS_OPT_QSTATS 632283514Sarybchik void 633283514Sarybchikefx_tx_qstats_update( 634283514Sarybchik __in efx_txq_t *etp, 635283514Sarybchik __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 636283514Sarybchik{ 637283514Sarybchik efx_nic_t *enp = etp->et_enp; 638283514Sarybchik efx_tx_ops_t *etxop = enp->en_etxop; 639283514Sarybchik 640283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 641283514Sarybchik 642283514Sarybchik etxop->etxo_qstats_update(etp, stat); 643283514Sarybchik} 644283514Sarybchik#endif 645283514Sarybchik 646283514Sarybchik 647283514Sarybchik#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 648283514Sarybchik 649283514Sarybchikstatic __checkReturn int 650283514Sarybchikfalconsiena_tx_init( 651283514Sarybchik __in efx_nic_t *enp) 652283514Sarybchik{ 653283514Sarybchik efx_oword_t oword; 654283514Sarybchik 655227569Sphilip /* 656227569Sphilip * Disable the timer-based TX DMA backoff and allow TX DMA to be 657227569Sphilip * controlled by the RX FIFO fill level (although always allow a 658227569Sphilip * minimal trickle). 659227569Sphilip */ 660227569Sphilip EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 661227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 662227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 663227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 664227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 665227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 666227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 667227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 668227569Sphilip 669227569Sphilip /* 670227569Sphilip * Filter all packets less than 14 bytes to avoid parsing 671227569Sphilip * errors. 672227569Sphilip */ 673227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 674227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 675227569Sphilip 676227569Sphilip /* 677227569Sphilip * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 678227569Sphilip * descriptors (which is bad). 679227569Sphilip */ 680227569Sphilip EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 681227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 682227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 683227569Sphilip 684227569Sphilip return (0); 685227569Sphilip} 686227569Sphilip 687227569Sphilip#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 688227569Sphilip do { \ 689227569Sphilip unsigned int id; \ 690227569Sphilip size_t offset; \ 691227569Sphilip efx_qword_t qword; \ 692227569Sphilip \ 693227569Sphilip id = (_added)++ & (_etp)->et_mask; \ 694227569Sphilip offset = id * sizeof (efx_qword_t); \ 695227569Sphilip \ 696227569Sphilip EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 697227569Sphilip unsigned int, id, efsys_dma_addr_t, (_addr), \ 698227569Sphilip size_t, (_size), boolean_t, (_eop)); \ 699227569Sphilip \ 700227569Sphilip EFX_POPULATE_QWORD_4(qword, \ 701227569Sphilip FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 702227569Sphilip FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 703227569Sphilip FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 704227569Sphilip (uint32_t)((_addr) & 0xffffffff), \ 705227569Sphilip FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 706227569Sphilip (uint32_t)((_addr) >> 32)); \ 707227569Sphilip EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 708227569Sphilip \ 709227569Sphilip _NOTE(CONSTANTCONDITION) \ 710227569Sphilip } while (B_FALSE) 711227569Sphilip 712283514Sarybchikstatic __checkReturn int 713283514Sarybchikfalconsiena_tx_qpost( 714227569Sphilip __in efx_txq_t *etp, 715227569Sphilip __in_ecount(n) efx_buffer_t *eb, 716227569Sphilip __in unsigned int n, 717227569Sphilip __in unsigned int completed, 718227569Sphilip __inout unsigned int *addedp) 719227569Sphilip{ 720227569Sphilip unsigned int added = *addedp; 721227569Sphilip unsigned int i; 722227569Sphilip int rc = ENOSPC; 723227569Sphilip 724227569Sphilip if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 725227569Sphilip goto fail1; 726227569Sphilip 727227569Sphilip for (i = 0; i < n; i++) { 728227569Sphilip efx_buffer_t *ebp = &eb[i]; 729227569Sphilip efsys_dma_addr_t start = ebp->eb_addr; 730227569Sphilip size_t size = ebp->eb_size; 731227569Sphilip efsys_dma_addr_t end = start + size; 732227569Sphilip 733227569Sphilip /* Fragments must not span 4k boundaries. */ 734227569Sphilip EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 735227569Sphilip 736227569Sphilip EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 737227569Sphilip } 738227569Sphilip 739227569Sphilip EFX_TX_QSTAT_INCR(etp, TX_POST); 740227569Sphilip 741227569Sphilip *addedp = added; 742227569Sphilip return (0); 743227569Sphilip 744227569Sphilipfail1: 745227569Sphilip EFSYS_PROBE1(fail1, int, rc); 746227569Sphilip 747227569Sphilip return (rc); 748227569Sphilip} 749227569Sphilip 750283514Sarybchikstatic void 751283514Sarybchikfalconsiena_tx_qpush( 752227569Sphilip __in efx_txq_t *etp, 753283514Sarybchik __in unsigned int added, 754283514Sarybchik __in unsigned int pushed) 755227569Sphilip{ 756227569Sphilip efx_nic_t *enp = etp->et_enp; 757227569Sphilip uint32_t wptr; 758227569Sphilip efx_dword_t dword; 759227569Sphilip efx_oword_t oword; 760227569Sphilip 761227569Sphilip /* Push the populated descriptors out */ 762227569Sphilip wptr = added & etp->et_mask; 763227569Sphilip 764227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 765227569Sphilip 766227569Sphilip /* Only write the third DWORD */ 767227569Sphilip EFX_POPULATE_DWORD_1(dword, 768227569Sphilip EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 769283514Sarybchik 770283514Sarybchik /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 771283514Sarybchik EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 772283514Sarybchik wptr, pushed & etp->et_mask); 773283514Sarybchik EFSYS_PIO_WRITE_BARRIER(); 774227569Sphilip EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 775227569Sphilip etp->et_index, &dword, B_FALSE); 776227569Sphilip} 777227569Sphilip 778279183Sarybchik#define EFX_MAX_PACE_VALUE 20 779279183Sarybchik#define EFX_TX_PACE_CLOCK_BASE 104 780279183Sarybchik 781283514Sarybchikstatic __checkReturn int 782283514Sarybchikfalconsiena_tx_qpace( 783279183Sarybchik __in efx_txq_t *etp, 784279183Sarybchik __in unsigned int ns) 785279183Sarybchik{ 786279183Sarybchik efx_nic_t *enp = etp->et_enp; 787279183Sarybchik efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 788279183Sarybchik efx_oword_t oword; 789279183Sarybchik unsigned int pace_val; 790279183Sarybchik unsigned int timer_period; 791279183Sarybchik int rc; 792279183Sarybchik 793279183Sarybchik if (ns == 0) { 794279183Sarybchik pace_val = 0; 795279183Sarybchik } else { 796279183Sarybchik /* 797279183Sarybchik * The pace_val to write into the table is s.t 798279183Sarybchik * ns <= timer_period * (2 ^ pace_val) 799279183Sarybchik */ 800279183Sarybchik timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 801279183Sarybchik for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 802279183Sarybchik if ((timer_period << pace_val) >= ns) 803279183Sarybchik break; 804279183Sarybchik } 805279183Sarybchik } 806279183Sarybchik if (pace_val > EFX_MAX_PACE_VALUE) { 807279183Sarybchik rc = EINVAL; 808279183Sarybchik goto fail1; 809279183Sarybchik } 810279183Sarybchik 811279183Sarybchik /* Update the pacing table */ 812279183Sarybchik EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 813283514Sarybchik EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 814283514Sarybchik &oword, B_TRUE); 815279183Sarybchik 816279183Sarybchik return (0); 817279183Sarybchik 818279183Sarybchikfail1: 819279183Sarybchik EFSYS_PROBE1(fail1, int, rc); 820279183Sarybchik 821279183Sarybchik return (rc); 822279183Sarybchik} 823279183Sarybchik 824283514Sarybchikstatic __checkReturn int 825283514Sarybchikfalconsiena_tx_qflush( 826283514Sarybchik __in efx_txq_t *etp) 827227569Sphilip{ 828227569Sphilip efx_nic_t *enp = etp->et_enp; 829227569Sphilip efx_oword_t oword; 830227569Sphilip uint32_t label; 831227569Sphilip 832279183Sarybchik efx_tx_qpace(etp, 0); 833279183Sarybchik 834227569Sphilip label = etp->et_index; 835227569Sphilip 836227569Sphilip /* Flush the queue */ 837227569Sphilip EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 838227569Sphilip FRF_AZ_TX_FLUSH_DESCQ, label); 839227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 840283514Sarybchik 841283514Sarybchik return (0); 842227569Sphilip} 843227569Sphilip 844283514Sarybchikstatic void 845283514Sarybchikfalconsiena_tx_qenable( 846227569Sphilip __in efx_txq_t *etp) 847227569Sphilip{ 848227569Sphilip efx_nic_t *enp = etp->et_enp; 849227569Sphilip efx_oword_t oword; 850227569Sphilip 851227569Sphilip EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 852283514Sarybchik etp->et_index, &oword, B_TRUE); 853227569Sphilip 854227569Sphilip EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 855227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 856227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 857227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 858227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 859227569Sphilip 860227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 861227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 862227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 863227569Sphilip 864227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 865283514Sarybchik etp->et_index, &oword, B_TRUE); 866227569Sphilip} 867227569Sphilip 868283514Sarybchikstatic __checkReturn int 869283514Sarybchikfalconsiena_tx_qcreate( 870227569Sphilip __in efx_nic_t *enp, 871227569Sphilip __in unsigned int index, 872227569Sphilip __in unsigned int label, 873227569Sphilip __in efsys_mem_t *esmp, 874227569Sphilip __in size_t n, 875227569Sphilip __in uint32_t id, 876227569Sphilip __in uint16_t flags, 877227569Sphilip __in efx_evq_t *eep, 878283514Sarybchik __in efx_txq_t *etp, 879283514Sarybchik __out unsigned int *addedp) 880227569Sphilip{ 881227569Sphilip efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 882227569Sphilip efx_oword_t oword; 883227569Sphilip uint32_t size; 884227569Sphilip int rc; 885227569Sphilip 886279141Sarybchik EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 887279141Sarybchik (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 888279141Sarybchik EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 889227569Sphilip 890283514Sarybchik EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 891283514Sarybchik EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 892283514Sarybchik 893283514Sarybchik if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 894227569Sphilip rc = EINVAL; 895227569Sphilip goto fail1; 896227569Sphilip } 897227569Sphilip if (index >= encp->enc_txq_limit) { 898227569Sphilip rc = EINVAL; 899227569Sphilip goto fail2; 900227569Sphilip } 901283514Sarybchik for (size = 0; 902283514Sarybchik (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 903227569Sphilip size++) 904227569Sphilip if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 905227569Sphilip break; 906227569Sphilip if (id + (1 << size) >= encp->enc_buftbl_limit) { 907227569Sphilip rc = EINVAL; 908227569Sphilip goto fail3; 909227569Sphilip } 910227569Sphilip 911227569Sphilip /* Set up the new descriptor queue */ 912291396Sarybchik *addedp = 0; 913291396Sarybchik 914227569Sphilip EFX_POPULATE_OWORD_6(oword, 915227569Sphilip FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 916227569Sphilip FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 917227569Sphilip FRF_AZ_TX_DESCQ_OWNER_ID, 0, 918227569Sphilip FRF_AZ_TX_DESCQ_LABEL, label, 919227569Sphilip FRF_AZ_TX_DESCQ_SIZE, size, 920227569Sphilip FRF_AZ_TX_DESCQ_TYPE, 0); 921227569Sphilip 922227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 923227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 924227569Sphilip (flags & EFX_CKSUM_IPV4) ? 0 : 1); 925227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 926227569Sphilip (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 927227569Sphilip 928227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 929283514Sarybchik etp->et_index, &oword, B_TRUE); 930227569Sphilip 931227569Sphilip return (0); 932227569Sphilip 933227569Sphilipfail3: 934227569Sphilip EFSYS_PROBE(fail3); 935227569Sphilipfail2: 936227569Sphilip EFSYS_PROBE(fail2); 937227569Sphilipfail1: 938227569Sphilip EFSYS_PROBE1(fail1, int, rc); 939227569Sphilip 940227569Sphilip return (rc); 941227569Sphilip} 942227569Sphilip 943283514Sarybchik __checkReturn int 944283514Sarybchikfalconsiena_tx_qdesc_post( 945283514Sarybchik __in efx_txq_t *etp, 946283514Sarybchik __in_ecount(n) efx_desc_t *ed, 947283514Sarybchik __in unsigned int n, 948283514Sarybchik __in unsigned int completed, 949283514Sarybchik __inout unsigned int *addedp) 950283514Sarybchik{ 951283514Sarybchik unsigned int added = *addedp; 952283514Sarybchik unsigned int i; 953283514Sarybchik int rc; 954283514Sarybchik 955283514Sarybchik if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 956283514Sarybchik rc = ENOSPC; 957283514Sarybchik goto fail1; 958283514Sarybchik } 959283514Sarybchik 960283514Sarybchik for (i = 0; i < n; i++) { 961283514Sarybchik efx_desc_t *edp = &ed[i]; 962283514Sarybchik unsigned int id; 963283514Sarybchik size_t offset; 964283514Sarybchik 965283514Sarybchik id = added++ & etp->et_mask; 966283514Sarybchik offset = id * sizeof (efx_desc_t); 967283514Sarybchik 968283514Sarybchik EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 969283514Sarybchik } 970283514Sarybchik 971283514Sarybchik EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 972283514Sarybchik unsigned int, added, unsigned int, n); 973283514Sarybchik 974283514Sarybchik EFX_TX_QSTAT_INCR(etp, TX_POST); 975283514Sarybchik 976283514Sarybchik *addedp = added; 977283514Sarybchik return (0); 978283514Sarybchik 979283514Sarybchikfail1: 980283514Sarybchik EFSYS_PROBE1(fail1, int, rc); 981283514Sarybchik return (rc); 982283514Sarybchik} 983283514Sarybchik 984283514Sarybchik void 985283514Sarybchikfalconsiena_tx_qdesc_dma_create( 986283514Sarybchik __in efx_txq_t *etp, 987283514Sarybchik __in efsys_dma_addr_t addr, 988283514Sarybchik __in size_t size, 989283514Sarybchik __in boolean_t eop, 990283514Sarybchik __out efx_desc_t *edp) 991283514Sarybchik{ 992283514Sarybchik /* Fragments must not span 4k boundaries. */ 993283514Sarybchik EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); 994283514Sarybchik 995283514Sarybchik EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 996283514Sarybchik efsys_dma_addr_t, addr, 997283514Sarybchik size_t, size, boolean_t, eop); 998283514Sarybchik 999283514Sarybchik EFX_POPULATE_QWORD_4(edp->ed_eq, 1000283514Sarybchik FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1001283514Sarybchik FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1002283514Sarybchik FSF_AZ_TX_KER_BUF_ADDR_DW0, 1003283514Sarybchik (uint32_t)(addr & 0xffffffff), 1004283514Sarybchik FSF_AZ_TX_KER_BUF_ADDR_DW1, 1005283514Sarybchik (uint32_t)(addr >> 32)); 1006283514Sarybchik} 1007283514Sarybchik 1008283514Sarybchik#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1009283514Sarybchik 1010277886Sarybchik#if EFSYS_OPT_QSTATS 1011227569Sphilip#if EFSYS_OPT_NAMES 1012283514Sarybchik/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ 1013283514Sarybchikstatic const char *__efx_tx_qstat_name[] = { 1014227569Sphilip "post", 1015283514Sarybchik "post_pio", 1016227569Sphilip}; 1017227569Sphilip/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1018227569Sphilip 1019283514Sarybchik const char * 1020227569Sphilipefx_tx_qstat_name( 1021227569Sphilip __in efx_nic_t *enp, 1022227569Sphilip __in unsigned int id) 1023227569Sphilip{ 1024227569Sphilip _NOTE(ARGUNUSED(enp)) 1025227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1026227569Sphilip EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1027227569Sphilip 1028227569Sphilip return (__efx_tx_qstat_name[id]); 1029227569Sphilip} 1030227569Sphilip#endif /* EFSYS_OPT_NAMES */ 1031283514Sarybchik#endif /* EFSYS_OPT_QSTATS */ 1032227569Sphilip 1033283514Sarybchik#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 1034283514Sarybchik 1035227569Sphilip#if EFSYS_OPT_QSTATS 1036283514Sarybchikstatic void 1037283514Sarybchikfalconsiena_tx_qstats_update( 1038227569Sphilip __in efx_txq_t *etp, 1039227569Sphilip __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1040227569Sphilip{ 1041227569Sphilip unsigned int id; 1042227569Sphilip 1043227569Sphilip for (id = 0; id < TX_NQSTATS; id++) { 1044227569Sphilip efsys_stat_t *essp = &stat[id]; 1045227569Sphilip 1046227569Sphilip EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1047227569Sphilip etp->et_stat[id] = 0; 1048227569Sphilip } 1049227569Sphilip} 1050227569Sphilip#endif /* EFSYS_OPT_QSTATS */ 1051227569Sphilip 1052283514Sarybchikstatic void 1053283514Sarybchikfalconsiena_tx_qdestroy( 1054227569Sphilip __in efx_txq_t *etp) 1055227569Sphilip{ 1056227569Sphilip efx_nic_t *enp = etp->et_enp; 1057227569Sphilip efx_oword_t oword; 1058227569Sphilip 1059227569Sphilip /* Purge descriptor queue */ 1060227569Sphilip EFX_ZERO_OWORD(oword); 1061227569Sphilip 1062227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1063283514Sarybchik etp->et_index, &oword, B_TRUE); 1064227569Sphilip} 1065227569Sphilip 1066283514Sarybchikstatic void 1067283514Sarybchikfalconsiena_tx_fini( 1068227569Sphilip __in efx_nic_t *enp) 1069227569Sphilip{ 1070283514Sarybchik _NOTE(ARGUNUSED(enp)) 1071283514Sarybchik} 1072227569Sphilip 1073283514Sarybchik#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1074