1227569Sphilip/*- 2300607Sarybchik * Copyright (c) 2007-2016 Solarflare Communications Inc. 3283514Sarybchik * All rights reserved. 4227569Sphilip * 5227569Sphilip * Redistribution and use in source and binary forms, with or without 6283514Sarybchik * modification, are permitted provided that the following conditions are met: 7227569Sphilip * 8283514Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 9283514Sarybchik * this list of conditions and the following disclaimer. 10283514Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 11283514Sarybchik * this list of conditions and the following disclaimer in the documentation 12283514Sarybchik * and/or other materials provided with the distribution. 13283514Sarybchik * 14283514Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15283514Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16283514Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17283514Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18283514Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19283514Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20283514Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21283514Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22283514Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23283514Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24283514Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25283514Sarybchik * 26283514Sarybchik * The views and conclusions contained in the software and documentation are 27283514Sarybchik * those of the authors and should not be interpreted as representing official 28283514Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 29227569Sphilip */ 30227569Sphilip 31228078Sphilip#include <sys/cdefs.h> 32228078Sphilip__FBSDID("$FreeBSD: releng/11.0/sys/dev/sfxge/common/efx_tx.c 300840 2016-05-27 11:44:40Z arybchik $"); 33228078Sphilip 34227569Sphilip#include "efx.h" 35227569Sphilip#include "efx_impl.h" 36227569Sphilip 37227569Sphilip#if EFSYS_OPT_QSTATS 38227569Sphilip#define EFX_TX_QSTAT_INCR(_etp, _stat) \ 39227569Sphilip do { \ 40227569Sphilip (_etp)->et_stat[_stat]++; \ 41227569Sphilip _NOTE(CONSTANTCONDITION) \ 42227569Sphilip } while (B_FALSE) 43227569Sphilip#else 44227569Sphilip#define EFX_TX_QSTAT_INCR(_etp, _stat) 45227569Sphilip#endif 46227569Sphilip 47299320Sarybchik#if EFSYS_OPT_SIENA 48283514Sarybchik 49291436Sarybchikstatic __checkReturn efx_rc_t 50299611Sarybchiksiena_tx_init( 51283514Sarybchik __in efx_nic_t *enp); 52283514Sarybchik 53283514Sarybchikstatic void 54299611Sarybchiksiena_tx_fini( 55283514Sarybchik __in efx_nic_t *enp); 56283514Sarybchik 57291436Sarybchikstatic __checkReturn efx_rc_t 58299611Sarybchiksiena_tx_qcreate( 59283514Sarybchik __in efx_nic_t *enp, 60283514Sarybchik __in unsigned int index, 61283514Sarybchik __in unsigned int label, 62283514Sarybchik __in efsys_mem_t *esmp, 63283514Sarybchik __in size_t n, 64283514Sarybchik __in uint32_t id, 65283514Sarybchik __in uint16_t flags, 66283514Sarybchik __in efx_evq_t *eep, 67283514Sarybchik __in efx_txq_t *etp, 68283514Sarybchik __out unsigned int *addedp); 69283514Sarybchik 70283514Sarybchikstatic void 71299611Sarybchiksiena_tx_qdestroy( 72283514Sarybchik __in efx_txq_t *etp); 73283514Sarybchik 74291436Sarybchikstatic __checkReturn efx_rc_t 75299611Sarybchiksiena_tx_qpost( 76283514Sarybchik __in efx_txq_t *etp, 77283514Sarybchik __in_ecount(n) efx_buffer_t *eb, 78283514Sarybchik __in unsigned int n, 79283514Sarybchik __in unsigned int completed, 80283514Sarybchik __inout unsigned int *addedp); 81283514Sarybchik 82283514Sarybchikstatic void 83299611Sarybchiksiena_tx_qpush( 84283514Sarybchik __in efx_txq_t *etp, 85283514Sarybchik __in unsigned int added, 86283514Sarybchik __in unsigned int pushed); 87283514Sarybchik 88291436Sarybchikstatic __checkReturn efx_rc_t 89299611Sarybchiksiena_tx_qpace( 90283514Sarybchik __in efx_txq_t *etp, 91283514Sarybchik __in unsigned int ns); 92283514Sarybchik 93291436Sarybchikstatic __checkReturn efx_rc_t 94299611Sarybchiksiena_tx_qflush( 95283514Sarybchik __in efx_txq_t *etp); 96283514Sarybchik 97283514Sarybchikstatic void 98299611Sarybchiksiena_tx_qenable( 99283514Sarybchik __in efx_txq_t *etp); 100283514Sarybchik 101291436Sarybchik __checkReturn efx_rc_t 102299611Sarybchiksiena_tx_qdesc_post( 103283514Sarybchik __in efx_txq_t *etp, 104283514Sarybchik __in_ecount(n) efx_desc_t *ed, 105283514Sarybchik __in unsigned int n, 106283514Sarybchik __in unsigned int completed, 107283514Sarybchik __inout unsigned int *addedp); 108283514Sarybchik 109283514Sarybchik void 110299611Sarybchiksiena_tx_qdesc_dma_create( 111283514Sarybchik __in efx_txq_t *etp, 112283514Sarybchik __in efsys_dma_addr_t addr, 113283514Sarybchik __in size_t size, 114283514Sarybchik __in boolean_t eop, 115283514Sarybchik __out efx_desc_t *edp); 116283514Sarybchik 117283514Sarybchik#if EFSYS_OPT_QSTATS 118283514Sarybchikstatic void 119299611Sarybchiksiena_tx_qstats_update( 120283514Sarybchik __in efx_txq_t *etp, 121283514Sarybchik __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 122283514Sarybchik#endif 123283514Sarybchik 124299320Sarybchik#endif /* EFSYS_OPT_SIENA */ 125283514Sarybchik 126283514Sarybchik 127283514Sarybchik#if EFSYS_OPT_SIENA 128299517Sarybchikstatic const efx_tx_ops_t __efx_tx_siena_ops = { 129299611Sarybchik siena_tx_init, /* etxo_init */ 130299611Sarybchik siena_tx_fini, /* etxo_fini */ 131299611Sarybchik siena_tx_qcreate, /* etxo_qcreate */ 132299611Sarybchik siena_tx_qdestroy, /* etxo_qdestroy */ 133299611Sarybchik siena_tx_qpost, /* etxo_qpost */ 134299611Sarybchik siena_tx_qpush, /* etxo_qpush */ 135299611Sarybchik siena_tx_qpace, /* etxo_qpace */ 136299611Sarybchik siena_tx_qflush, /* etxo_qflush */ 137299611Sarybchik siena_tx_qenable, /* etxo_qenable */ 138283514Sarybchik NULL, /* etxo_qpio_enable */ 139283514Sarybchik NULL, /* etxo_qpio_disable */ 140283514Sarybchik NULL, /* etxo_qpio_write */ 141283514Sarybchik NULL, /* etxo_qpio_post */ 142299611Sarybchik siena_tx_qdesc_post, /* etxo_qdesc_post */ 143299611Sarybchik siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 144283514Sarybchik NULL, /* etxo_qdesc_tso_create */ 145293891Sarybchik NULL, /* etxo_qdesc_tso2_create */ 146283514Sarybchik NULL, /* etxo_qdesc_vlantci_create */ 147283514Sarybchik#if EFSYS_OPT_QSTATS 148299611Sarybchik siena_tx_qstats_update, /* etxo_qstats_update */ 149283514Sarybchik#endif 150283514Sarybchik}; 151283514Sarybchik#endif /* EFSYS_OPT_SIENA */ 152283514Sarybchik 153283514Sarybchik#if EFSYS_OPT_HUNTINGTON 154299517Sarybchikstatic const efx_tx_ops_t __efx_tx_hunt_ops = { 155293753Sarybchik ef10_tx_init, /* etxo_init */ 156293753Sarybchik ef10_tx_fini, /* etxo_fini */ 157293753Sarybchik ef10_tx_qcreate, /* etxo_qcreate */ 158293753Sarybchik ef10_tx_qdestroy, /* etxo_qdestroy */ 159293753Sarybchik ef10_tx_qpost, /* etxo_qpost */ 160293753Sarybchik ef10_tx_qpush, /* etxo_qpush */ 161293753Sarybchik ef10_tx_qpace, /* etxo_qpace */ 162293753Sarybchik ef10_tx_qflush, /* etxo_qflush */ 163293753Sarybchik ef10_tx_qenable, /* etxo_qenable */ 164293753Sarybchik ef10_tx_qpio_enable, /* etxo_qpio_enable */ 165293753Sarybchik ef10_tx_qpio_disable, /* etxo_qpio_disable */ 166293753Sarybchik ef10_tx_qpio_write, /* etxo_qpio_write */ 167293753Sarybchik ef10_tx_qpio_post, /* etxo_qpio_post */ 168293753Sarybchik ef10_tx_qdesc_post, /* etxo_qdesc_post */ 169293753Sarybchik ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 170299605Sarybchik ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 171293891Sarybchik ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 172293753Sarybchik ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 173283514Sarybchik#if EFSYS_OPT_QSTATS 174293753Sarybchik ef10_tx_qstats_update, /* etxo_qstats_update */ 175283514Sarybchik#endif 176283514Sarybchik}; 177283514Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */ 178283514Sarybchik 179293753Sarybchik#if EFSYS_OPT_MEDFORD 180299517Sarybchikstatic const efx_tx_ops_t __efx_tx_medford_ops = { 181293753Sarybchik ef10_tx_init, /* etxo_init */ 182293753Sarybchik ef10_tx_fini, /* etxo_fini */ 183293753Sarybchik ef10_tx_qcreate, /* etxo_qcreate */ 184293753Sarybchik ef10_tx_qdestroy, /* etxo_qdestroy */ 185293753Sarybchik ef10_tx_qpost, /* etxo_qpost */ 186293753Sarybchik ef10_tx_qpush, /* etxo_qpush */ 187293753Sarybchik ef10_tx_qpace, /* etxo_qpace */ 188293753Sarybchik ef10_tx_qflush, /* etxo_qflush */ 189293753Sarybchik ef10_tx_qenable, /* etxo_qenable */ 190293753Sarybchik ef10_tx_qpio_enable, /* etxo_qpio_enable */ 191293753Sarybchik ef10_tx_qpio_disable, /* etxo_qpio_disable */ 192293753Sarybchik ef10_tx_qpio_write, /* etxo_qpio_write */ 193293753Sarybchik ef10_tx_qpio_post, /* etxo_qpio_post */ 194293753Sarybchik ef10_tx_qdesc_post, /* etxo_qdesc_post */ 195293753Sarybchik ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 196293753Sarybchik NULL, /* etxo_qdesc_tso_create */ 197293891Sarybchik ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 198293753Sarybchik ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 199293753Sarybchik#if EFSYS_OPT_QSTATS 200293753Sarybchik ef10_tx_qstats_update, /* etxo_qstats_update */ 201293753Sarybchik#endif 202293753Sarybchik}; 203293753Sarybchik#endif /* EFSYS_OPT_MEDFORD */ 204293753Sarybchik 205291436Sarybchik __checkReturn efx_rc_t 206227569Sphilipefx_tx_init( 207227569Sphilip __in efx_nic_t *enp) 208227569Sphilip{ 209299517Sarybchik const efx_tx_ops_t *etxop; 210291436Sarybchik efx_rc_t rc; 211227569Sphilip 212227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 213227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 214227569Sphilip 215227569Sphilip if (!(enp->en_mod_flags & EFX_MOD_EV)) { 216227569Sphilip rc = EINVAL; 217227569Sphilip goto fail1; 218227569Sphilip } 219227569Sphilip 220227569Sphilip if (enp->en_mod_flags & EFX_MOD_TX) { 221227569Sphilip rc = EINVAL; 222227569Sphilip goto fail2; 223227569Sphilip } 224227569Sphilip 225283514Sarybchik switch (enp->en_family) { 226283514Sarybchik#if EFSYS_OPT_SIENA 227283514Sarybchik case EFX_FAMILY_SIENA: 228299517Sarybchik etxop = &__efx_tx_siena_ops; 229283514Sarybchik break; 230283514Sarybchik#endif /* EFSYS_OPT_SIENA */ 231283514Sarybchik 232283514Sarybchik#if EFSYS_OPT_HUNTINGTON 233283514Sarybchik case EFX_FAMILY_HUNTINGTON: 234299517Sarybchik etxop = &__efx_tx_hunt_ops; 235283514Sarybchik break; 236283514Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */ 237283514Sarybchik 238293753Sarybchik#if EFSYS_OPT_MEDFORD 239293753Sarybchik case EFX_FAMILY_MEDFORD: 240299517Sarybchik etxop = &__efx_tx_medford_ops; 241293753Sarybchik break; 242293753Sarybchik#endif /* EFSYS_OPT_MEDFORD */ 243293753Sarybchik 244283514Sarybchik default: 245283514Sarybchik EFSYS_ASSERT(0); 246283514Sarybchik rc = ENOTSUP; 247283514Sarybchik goto fail3; 248283514Sarybchik } 249283514Sarybchik 250227569Sphilip EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 251227569Sphilip 252283514Sarybchik if ((rc = etxop->etxo_init(enp)) != 0) 253283514Sarybchik goto fail4; 254283514Sarybchik 255283514Sarybchik enp->en_etxop = etxop; 256283514Sarybchik enp->en_mod_flags |= EFX_MOD_TX; 257283514Sarybchik return (0); 258283514Sarybchik 259283514Sarybchikfail4: 260283514Sarybchik EFSYS_PROBE(fail4); 261283514Sarybchikfail3: 262283514Sarybchik EFSYS_PROBE(fail3); 263283514Sarybchikfail2: 264283514Sarybchik EFSYS_PROBE(fail2); 265283514Sarybchikfail1: 266291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 267283514Sarybchik 268283514Sarybchik enp->en_etxop = NULL; 269283514Sarybchik enp->en_mod_flags &= ~EFX_MOD_TX; 270283514Sarybchik return (rc); 271283514Sarybchik} 272283514Sarybchik 273283514Sarybchik void 274283514Sarybchikefx_tx_fini( 275283514Sarybchik __in efx_nic_t *enp) 276283514Sarybchik{ 277299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 278283514Sarybchik 279283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 280283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 281283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 282283514Sarybchik EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 283283514Sarybchik 284283514Sarybchik etxop->etxo_fini(enp); 285283514Sarybchik 286283514Sarybchik enp->en_etxop = NULL; 287283514Sarybchik enp->en_mod_flags &= ~EFX_MOD_TX; 288283514Sarybchik} 289283514Sarybchik 290291436Sarybchik __checkReturn efx_rc_t 291283514Sarybchikefx_tx_qcreate( 292283514Sarybchik __in efx_nic_t *enp, 293283514Sarybchik __in unsigned int index, 294283514Sarybchik __in unsigned int label, 295283514Sarybchik __in efsys_mem_t *esmp, 296283514Sarybchik __in size_t n, 297283514Sarybchik __in uint32_t id, 298283514Sarybchik __in uint16_t flags, 299283514Sarybchik __in efx_evq_t *eep, 300283514Sarybchik __deref_out efx_txq_t **etpp, 301283514Sarybchik __out unsigned int *addedp) 302283514Sarybchik{ 303299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 304283514Sarybchik efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 305283514Sarybchik efx_txq_t *etp; 306291436Sarybchik efx_rc_t rc; 307283514Sarybchik 308283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 309283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 310283514Sarybchik 311283514Sarybchik EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 312283514Sarybchik 313283514Sarybchik /* Allocate an TXQ object */ 314283514Sarybchik EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 315283514Sarybchik 316283514Sarybchik if (etp == NULL) { 317283514Sarybchik rc = ENOMEM; 318283514Sarybchik goto fail1; 319283514Sarybchik } 320283514Sarybchik 321283514Sarybchik etp->et_magic = EFX_TXQ_MAGIC; 322283514Sarybchik etp->et_enp = enp; 323283514Sarybchik etp->et_index = index; 324283514Sarybchik etp->et_mask = n - 1; 325283514Sarybchik etp->et_esmp = esmp; 326283514Sarybchik 327283514Sarybchik /* Initial descriptor index may be modified by etxo_qcreate */ 328283514Sarybchik *addedp = 0; 329283514Sarybchik 330283514Sarybchik if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 331283514Sarybchik n, id, flags, eep, etp, addedp)) != 0) 332283514Sarybchik goto fail2; 333283514Sarybchik 334283514Sarybchik enp->en_tx_qcount++; 335283514Sarybchik *etpp = etp; 336283514Sarybchik 337283514Sarybchik return (0); 338283514Sarybchik 339283514Sarybchikfail2: 340283514Sarybchik EFSYS_PROBE(fail2); 341283514Sarybchik EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 342283514Sarybchikfail1: 343291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 344283514Sarybchik return (rc); 345283514Sarybchik} 346283514Sarybchik 347283514Sarybchik void 348283514Sarybchikefx_tx_qdestroy( 349283514Sarybchik __in efx_txq_t *etp) 350283514Sarybchik{ 351283514Sarybchik efx_nic_t *enp = etp->et_enp; 352299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 353283514Sarybchik 354283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 355283514Sarybchik 356283514Sarybchik EFSYS_ASSERT(enp->en_tx_qcount != 0); 357283514Sarybchik --enp->en_tx_qcount; 358283514Sarybchik 359283514Sarybchik etxop->etxo_qdestroy(etp); 360283514Sarybchik 361283514Sarybchik /* Free the TXQ object */ 362283514Sarybchik EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 363283514Sarybchik} 364283514Sarybchik 365291436Sarybchik __checkReturn efx_rc_t 366283514Sarybchikefx_tx_qpost( 367283514Sarybchik __in efx_txq_t *etp, 368283514Sarybchik __in_ecount(n) efx_buffer_t *eb, 369283514Sarybchik __in unsigned int n, 370283514Sarybchik __in unsigned int completed, 371283514Sarybchik __inout unsigned int *addedp) 372283514Sarybchik{ 373283514Sarybchik efx_nic_t *enp = etp->et_enp; 374299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 375291436Sarybchik efx_rc_t rc; 376283514Sarybchik 377283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 378283514Sarybchik 379283514Sarybchik if ((rc = etxop->etxo_qpost(etp, eb, 380283514Sarybchik n, completed, addedp)) != 0) 381283514Sarybchik goto fail1; 382283514Sarybchik 383283514Sarybchik return (0); 384283514Sarybchik 385283514Sarybchikfail1: 386291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 387283514Sarybchik return (rc); 388283514Sarybchik} 389283514Sarybchik 390283514Sarybchik void 391283514Sarybchikefx_tx_qpush( 392283514Sarybchik __in efx_txq_t *etp, 393283514Sarybchik __in unsigned int added, 394283514Sarybchik __in unsigned int pushed) 395283514Sarybchik{ 396283514Sarybchik efx_nic_t *enp = etp->et_enp; 397299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 398283514Sarybchik 399283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 400283514Sarybchik 401283514Sarybchik etxop->etxo_qpush(etp, added, pushed); 402283514Sarybchik} 403283514Sarybchik 404291436Sarybchik __checkReturn efx_rc_t 405283514Sarybchikefx_tx_qpace( 406283514Sarybchik __in efx_txq_t *etp, 407283514Sarybchik __in unsigned int ns) 408283514Sarybchik{ 409283514Sarybchik efx_nic_t *enp = etp->et_enp; 410299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 411291436Sarybchik efx_rc_t rc; 412283514Sarybchik 413283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 414283514Sarybchik 415283514Sarybchik if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 416283514Sarybchik goto fail1; 417283514Sarybchik 418283514Sarybchik return (0); 419283514Sarybchik 420283514Sarybchikfail1: 421291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 422283514Sarybchik return (rc); 423283514Sarybchik} 424283514Sarybchik 425291436Sarybchik __checkReturn efx_rc_t 426283514Sarybchikefx_tx_qflush( 427283514Sarybchik __in efx_txq_t *etp) 428283514Sarybchik{ 429283514Sarybchik efx_nic_t *enp = etp->et_enp; 430299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 431291436Sarybchik efx_rc_t rc; 432283514Sarybchik 433283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 434283514Sarybchik 435283514Sarybchik if ((rc = etxop->etxo_qflush(etp)) != 0) 436283514Sarybchik goto fail1; 437283514Sarybchik 438283514Sarybchik return (0); 439283514Sarybchik 440283514Sarybchikfail1: 441291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 442283514Sarybchik return (rc); 443283514Sarybchik} 444283514Sarybchik 445283514Sarybchik void 446283514Sarybchikefx_tx_qenable( 447283514Sarybchik __in efx_txq_t *etp) 448283514Sarybchik{ 449283514Sarybchik efx_nic_t *enp = etp->et_enp; 450299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 451283514Sarybchik 452283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 453283514Sarybchik 454283514Sarybchik etxop->etxo_qenable(etp); 455283514Sarybchik} 456283514Sarybchik 457291436Sarybchik __checkReturn efx_rc_t 458283514Sarybchikefx_tx_qpio_enable( 459283514Sarybchik __in efx_txq_t *etp) 460283514Sarybchik{ 461283514Sarybchik efx_nic_t *enp = etp->et_enp; 462299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 463291436Sarybchik efx_rc_t rc; 464283514Sarybchik 465283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 466283514Sarybchik 467283514Sarybchik if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 468283514Sarybchik rc = ENOTSUP; 469283514Sarybchik goto fail1; 470283514Sarybchik } 471283514Sarybchik if (etxop->etxo_qpio_enable == NULL) { 472283514Sarybchik rc = ENOTSUP; 473283514Sarybchik goto fail2; 474283514Sarybchik } 475283514Sarybchik if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 476283514Sarybchik goto fail3; 477283514Sarybchik 478283514Sarybchik return (0); 479283514Sarybchik 480283514Sarybchikfail3: 481283514Sarybchik EFSYS_PROBE(fail3); 482283514Sarybchikfail2: 483283514Sarybchik EFSYS_PROBE(fail2); 484283514Sarybchikfail1: 485291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 486283514Sarybchik return (rc); 487283514Sarybchik} 488283514Sarybchik 489283514Sarybchik void 490283514Sarybchikefx_tx_qpio_disable( 491283514Sarybchik __in efx_txq_t *etp) 492283514Sarybchik{ 493283514Sarybchik efx_nic_t *enp = etp->et_enp; 494299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 495283514Sarybchik 496283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 497283514Sarybchik 498283514Sarybchik if (etxop->etxo_qpio_disable != NULL) 499283514Sarybchik etxop->etxo_qpio_disable(etp); 500283514Sarybchik} 501283514Sarybchik 502291436Sarybchik __checkReturn efx_rc_t 503283514Sarybchikefx_tx_qpio_write( 504283514Sarybchik __in efx_txq_t *etp, 505283514Sarybchik __in_ecount(buf_length) uint8_t *buffer, 506283514Sarybchik __in size_t buf_length, 507283514Sarybchik __in size_t pio_buf_offset) 508283514Sarybchik{ 509283514Sarybchik efx_nic_t *enp = etp->et_enp; 510299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 511291436Sarybchik efx_rc_t rc; 512283514Sarybchik 513283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 514283514Sarybchik 515283514Sarybchik if (etxop->etxo_qpio_write != NULL) { 516283514Sarybchik if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 517283514Sarybchik pio_buf_offset)) != 0) 518283514Sarybchik goto fail1; 519283514Sarybchik return (0); 520283514Sarybchik } 521283514Sarybchik 522283514Sarybchik return (ENOTSUP); 523283514Sarybchik 524283514Sarybchikfail1: 525291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 526283514Sarybchik return (rc); 527283514Sarybchik} 528283514Sarybchik 529291436Sarybchik __checkReturn efx_rc_t 530283514Sarybchikefx_tx_qpio_post( 531283514Sarybchik __in efx_txq_t *etp, 532283514Sarybchik __in size_t pkt_length, 533283514Sarybchik __in unsigned int completed, 534283514Sarybchik __inout unsigned int *addedp) 535283514Sarybchik{ 536283514Sarybchik efx_nic_t *enp = etp->et_enp; 537299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 538291436Sarybchik efx_rc_t rc; 539283514Sarybchik 540283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 541283514Sarybchik 542283514Sarybchik if (etxop->etxo_qpio_post != NULL) { 543283514Sarybchik if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 544283514Sarybchik addedp)) != 0) 545283514Sarybchik goto fail1; 546283514Sarybchik return (0); 547283514Sarybchik } 548283514Sarybchik 549283514Sarybchik return (ENOTSUP); 550283514Sarybchik 551283514Sarybchikfail1: 552291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 553283514Sarybchik return (rc); 554283514Sarybchik} 555283514Sarybchik 556291436Sarybchik __checkReturn efx_rc_t 557283514Sarybchikefx_tx_qdesc_post( 558283514Sarybchik __in efx_txq_t *etp, 559283514Sarybchik __in_ecount(n) efx_desc_t *ed, 560283514Sarybchik __in unsigned int n, 561283514Sarybchik __in unsigned int completed, 562283514Sarybchik __inout unsigned int *addedp) 563283514Sarybchik{ 564283514Sarybchik efx_nic_t *enp = etp->et_enp; 565299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 566291436Sarybchik efx_rc_t rc; 567283514Sarybchik 568283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 569283514Sarybchik 570283514Sarybchik if ((rc = etxop->etxo_qdesc_post(etp, ed, 571283514Sarybchik n, completed, addedp)) != 0) 572283514Sarybchik goto fail1; 573283514Sarybchik 574283514Sarybchik return (0); 575283514Sarybchik 576283514Sarybchikfail1: 577291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 578283514Sarybchik return (rc); 579283514Sarybchik} 580283514Sarybchik 581283514Sarybchik void 582283514Sarybchikefx_tx_qdesc_dma_create( 583283514Sarybchik __in efx_txq_t *etp, 584283514Sarybchik __in efsys_dma_addr_t addr, 585283514Sarybchik __in size_t size, 586283514Sarybchik __in boolean_t eop, 587283514Sarybchik __out efx_desc_t *edp) 588283514Sarybchik{ 589283514Sarybchik efx_nic_t *enp = etp->et_enp; 590299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 591283514Sarybchik 592283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 593283514Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 594283514Sarybchik 595283514Sarybchik etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 596283514Sarybchik} 597283514Sarybchik 598283514Sarybchik void 599283514Sarybchikefx_tx_qdesc_tso_create( 600283514Sarybchik __in efx_txq_t *etp, 601283514Sarybchik __in uint16_t ipv4_id, 602283514Sarybchik __in uint32_t tcp_seq, 603283514Sarybchik __in uint8_t tcp_flags, 604283514Sarybchik __out efx_desc_t *edp) 605283514Sarybchik{ 606283514Sarybchik efx_nic_t *enp = etp->et_enp; 607299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 608283514Sarybchik 609283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 610283514Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 611283514Sarybchik 612283514Sarybchik etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 613283514Sarybchik} 614283514Sarybchik 615283514Sarybchik void 616293891Sarybchikefx_tx_qdesc_tso2_create( 617293891Sarybchik __in efx_txq_t *etp, 618293891Sarybchik __in uint16_t ipv4_id, 619293891Sarybchik __in uint32_t tcp_seq, 620293891Sarybchik __in uint16_t mss, 621293891Sarybchik __out_ecount(count) efx_desc_t *edp, 622293891Sarybchik __in int count) 623293891Sarybchik{ 624293891Sarybchik efx_nic_t *enp = etp->et_enp; 625299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 626293891Sarybchik 627293891Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 628293891Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 629293891Sarybchik 630293891Sarybchik etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); 631293891Sarybchik} 632293891Sarybchik 633293891Sarybchik void 634283514Sarybchikefx_tx_qdesc_vlantci_create( 635283514Sarybchik __in efx_txq_t *etp, 636283514Sarybchik __in uint16_t tci, 637283514Sarybchik __out efx_desc_t *edp) 638283514Sarybchik{ 639283514Sarybchik efx_nic_t *enp = etp->et_enp; 640299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 641283514Sarybchik 642283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 643283514Sarybchik EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 644283514Sarybchik 645283514Sarybchik etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 646283514Sarybchik} 647283514Sarybchik 648283514Sarybchik 649283514Sarybchik#if EFSYS_OPT_QSTATS 650283514Sarybchik void 651283514Sarybchikefx_tx_qstats_update( 652283514Sarybchik __in efx_txq_t *etp, 653283514Sarybchik __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 654283514Sarybchik{ 655283514Sarybchik efx_nic_t *enp = etp->et_enp; 656299517Sarybchik const efx_tx_ops_t *etxop = enp->en_etxop; 657283514Sarybchik 658283514Sarybchik EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 659283514Sarybchik 660283514Sarybchik etxop->etxo_qstats_update(etp, stat); 661283514Sarybchik} 662283514Sarybchik#endif 663283514Sarybchik 664283514Sarybchik 665299320Sarybchik#if EFSYS_OPT_SIENA 666283514Sarybchik 667291436Sarybchikstatic __checkReturn efx_rc_t 668299611Sarybchiksiena_tx_init( 669283514Sarybchik __in efx_nic_t *enp) 670283514Sarybchik{ 671283514Sarybchik efx_oword_t oword; 672283514Sarybchik 673227569Sphilip /* 674227569Sphilip * Disable the timer-based TX DMA backoff and allow TX DMA to be 675227569Sphilip * controlled by the RX FIFO fill level (although always allow a 676227569Sphilip * minimal trickle). 677227569Sphilip */ 678227569Sphilip EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 679227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 680227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 681227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 682227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 683227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 684227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 685227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 686227569Sphilip 687227569Sphilip /* 688227569Sphilip * Filter all packets less than 14 bytes to avoid parsing 689227569Sphilip * errors. 690227569Sphilip */ 691227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 692227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 693227569Sphilip 694227569Sphilip /* 695227569Sphilip * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 696227569Sphilip * descriptors (which is bad). 697227569Sphilip */ 698227569Sphilip EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 699227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 700227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 701227569Sphilip 702227569Sphilip return (0); 703227569Sphilip} 704227569Sphilip 705227569Sphilip#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 706227569Sphilip do { \ 707227569Sphilip unsigned int id; \ 708227569Sphilip size_t offset; \ 709227569Sphilip efx_qword_t qword; \ 710227569Sphilip \ 711227569Sphilip id = (_added)++ & (_etp)->et_mask; \ 712227569Sphilip offset = id * sizeof (efx_qword_t); \ 713227569Sphilip \ 714227569Sphilip EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 715227569Sphilip unsigned int, id, efsys_dma_addr_t, (_addr), \ 716227569Sphilip size_t, (_size), boolean_t, (_eop)); \ 717227569Sphilip \ 718227569Sphilip EFX_POPULATE_QWORD_4(qword, \ 719227569Sphilip FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 720227569Sphilip FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 721227569Sphilip FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 722227569Sphilip (uint32_t)((_addr) & 0xffffffff), \ 723227569Sphilip FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 724227569Sphilip (uint32_t)((_addr) >> 32)); \ 725227569Sphilip EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 726227569Sphilip \ 727227569Sphilip _NOTE(CONSTANTCONDITION) \ 728227569Sphilip } while (B_FALSE) 729227569Sphilip 730291436Sarybchikstatic __checkReturn efx_rc_t 731299611Sarybchiksiena_tx_qpost( 732227569Sphilip __in efx_txq_t *etp, 733227569Sphilip __in_ecount(n) efx_buffer_t *eb, 734227569Sphilip __in unsigned int n, 735227569Sphilip __in unsigned int completed, 736227569Sphilip __inout unsigned int *addedp) 737227569Sphilip{ 738227569Sphilip unsigned int added = *addedp; 739227569Sphilip unsigned int i; 740227569Sphilip int rc = ENOSPC; 741227569Sphilip 742227569Sphilip if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 743227569Sphilip goto fail1; 744227569Sphilip 745227569Sphilip for (i = 0; i < n; i++) { 746227569Sphilip efx_buffer_t *ebp = &eb[i]; 747227569Sphilip efsys_dma_addr_t start = ebp->eb_addr; 748227569Sphilip size_t size = ebp->eb_size; 749227569Sphilip efsys_dma_addr_t end = start + size; 750227569Sphilip 751227569Sphilip /* Fragments must not span 4k boundaries. */ 752227569Sphilip EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 753227569Sphilip 754227569Sphilip EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 755227569Sphilip } 756227569Sphilip 757227569Sphilip EFX_TX_QSTAT_INCR(etp, TX_POST); 758227569Sphilip 759227569Sphilip *addedp = added; 760227569Sphilip return (0); 761227569Sphilip 762227569Sphilipfail1: 763291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 764227569Sphilip 765227569Sphilip return (rc); 766227569Sphilip} 767227569Sphilip 768283514Sarybchikstatic void 769299611Sarybchiksiena_tx_qpush( 770227569Sphilip __in efx_txq_t *etp, 771283514Sarybchik __in unsigned int added, 772283514Sarybchik __in unsigned int pushed) 773227569Sphilip{ 774227569Sphilip efx_nic_t *enp = etp->et_enp; 775227569Sphilip uint32_t wptr; 776227569Sphilip efx_dword_t dword; 777227569Sphilip efx_oword_t oword; 778227569Sphilip 779227569Sphilip /* Push the populated descriptors out */ 780227569Sphilip wptr = added & etp->et_mask; 781227569Sphilip 782227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 783227569Sphilip 784227569Sphilip /* Only write the third DWORD */ 785227569Sphilip EFX_POPULATE_DWORD_1(dword, 786227569Sphilip EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 787283514Sarybchik 788283514Sarybchik /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 789283514Sarybchik EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 790283514Sarybchik wptr, pushed & etp->et_mask); 791283514Sarybchik EFSYS_PIO_WRITE_BARRIER(); 792227569Sphilip EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 793227569Sphilip etp->et_index, &dword, B_FALSE); 794227569Sphilip} 795227569Sphilip 796279183Sarybchik#define EFX_MAX_PACE_VALUE 20 797279183Sarybchik#define EFX_TX_PACE_CLOCK_BASE 104 798279183Sarybchik 799291436Sarybchikstatic __checkReturn efx_rc_t 800299611Sarybchiksiena_tx_qpace( 801279183Sarybchik __in efx_txq_t *etp, 802279183Sarybchik __in unsigned int ns) 803279183Sarybchik{ 804279183Sarybchik efx_nic_t *enp = etp->et_enp; 805279183Sarybchik efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 806279183Sarybchik efx_oword_t oword; 807279183Sarybchik unsigned int pace_val; 808279183Sarybchik unsigned int timer_period; 809291436Sarybchik efx_rc_t rc; 810279183Sarybchik 811279183Sarybchik if (ns == 0) { 812279183Sarybchik pace_val = 0; 813279183Sarybchik } else { 814279183Sarybchik /* 815279183Sarybchik * The pace_val to write into the table is s.t 816279183Sarybchik * ns <= timer_period * (2 ^ pace_val) 817279183Sarybchik */ 818279183Sarybchik timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 819279183Sarybchik for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 820279183Sarybchik if ((timer_period << pace_val) >= ns) 821279183Sarybchik break; 822279183Sarybchik } 823279183Sarybchik } 824279183Sarybchik if (pace_val > EFX_MAX_PACE_VALUE) { 825279183Sarybchik rc = EINVAL; 826279183Sarybchik goto fail1; 827279183Sarybchik } 828279183Sarybchik 829279183Sarybchik /* Update the pacing table */ 830279183Sarybchik EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 831283514Sarybchik EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 832283514Sarybchik &oword, B_TRUE); 833279183Sarybchik 834279183Sarybchik return (0); 835279183Sarybchik 836279183Sarybchikfail1: 837291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 838279183Sarybchik 839279183Sarybchik return (rc); 840279183Sarybchik} 841279183Sarybchik 842291436Sarybchikstatic __checkReturn efx_rc_t 843299611Sarybchiksiena_tx_qflush( 844283514Sarybchik __in efx_txq_t *etp) 845227569Sphilip{ 846227569Sphilip efx_nic_t *enp = etp->et_enp; 847227569Sphilip efx_oword_t oword; 848227569Sphilip uint32_t label; 849227569Sphilip 850279183Sarybchik efx_tx_qpace(etp, 0); 851279183Sarybchik 852227569Sphilip label = etp->et_index; 853227569Sphilip 854227569Sphilip /* Flush the queue */ 855227569Sphilip EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 856227569Sphilip FRF_AZ_TX_FLUSH_DESCQ, label); 857227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 858283514Sarybchik 859283514Sarybchik return (0); 860227569Sphilip} 861227569Sphilip 862283514Sarybchikstatic void 863299611Sarybchiksiena_tx_qenable( 864227569Sphilip __in efx_txq_t *etp) 865227569Sphilip{ 866227569Sphilip efx_nic_t *enp = etp->et_enp; 867227569Sphilip efx_oword_t oword; 868227569Sphilip 869227569Sphilip EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 870283514Sarybchik etp->et_index, &oword, B_TRUE); 871227569Sphilip 872227569Sphilip EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 873227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 874227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 875227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 876227569Sphilip uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 877227569Sphilip 878227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 879227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 880227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 881227569Sphilip 882227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 883283514Sarybchik etp->et_index, &oword, B_TRUE); 884227569Sphilip} 885227569Sphilip 886291436Sarybchikstatic __checkReturn efx_rc_t 887299611Sarybchiksiena_tx_qcreate( 888227569Sphilip __in efx_nic_t *enp, 889227569Sphilip __in unsigned int index, 890227569Sphilip __in unsigned int label, 891227569Sphilip __in efsys_mem_t *esmp, 892227569Sphilip __in size_t n, 893227569Sphilip __in uint32_t id, 894227569Sphilip __in uint16_t flags, 895227569Sphilip __in efx_evq_t *eep, 896283514Sarybchik __in efx_txq_t *etp, 897283514Sarybchik __out unsigned int *addedp) 898227569Sphilip{ 899227569Sphilip efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 900227569Sphilip efx_oword_t oword; 901227569Sphilip uint32_t size; 902291436Sarybchik efx_rc_t rc; 903227569Sphilip 904300840Sarybchik _NOTE(ARGUNUSED(esmp)) 905300840Sarybchik 906279141Sarybchik EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 907279141Sarybchik (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 908279141Sarybchik EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 909227569Sphilip 910283514Sarybchik EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 911283514Sarybchik EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 912283514Sarybchik 913283514Sarybchik if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 914227569Sphilip rc = EINVAL; 915227569Sphilip goto fail1; 916227569Sphilip } 917227569Sphilip if (index >= encp->enc_txq_limit) { 918227569Sphilip rc = EINVAL; 919227569Sphilip goto fail2; 920227569Sphilip } 921283514Sarybchik for (size = 0; 922283514Sarybchik (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 923227569Sphilip size++) 924227569Sphilip if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 925227569Sphilip break; 926227569Sphilip if (id + (1 << size) >= encp->enc_buftbl_limit) { 927227569Sphilip rc = EINVAL; 928227569Sphilip goto fail3; 929227569Sphilip } 930227569Sphilip 931227569Sphilip /* Set up the new descriptor queue */ 932291396Sarybchik *addedp = 0; 933291396Sarybchik 934227569Sphilip EFX_POPULATE_OWORD_6(oword, 935227569Sphilip FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 936227569Sphilip FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 937227569Sphilip FRF_AZ_TX_DESCQ_OWNER_ID, 0, 938227569Sphilip FRF_AZ_TX_DESCQ_LABEL, label, 939227569Sphilip FRF_AZ_TX_DESCQ_SIZE, size, 940227569Sphilip FRF_AZ_TX_DESCQ_TYPE, 0); 941227569Sphilip 942227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 943227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 944291924Sarybchik (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 945227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 946291924Sarybchik (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 947227569Sphilip 948227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 949283514Sarybchik etp->et_index, &oword, B_TRUE); 950227569Sphilip 951227569Sphilip return (0); 952227569Sphilip 953227569Sphilipfail3: 954227569Sphilip EFSYS_PROBE(fail3); 955227569Sphilipfail2: 956227569Sphilip EFSYS_PROBE(fail2); 957227569Sphilipfail1: 958291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 959227569Sphilip 960227569Sphilip return (rc); 961227569Sphilip} 962227569Sphilip 963291436Sarybchik __checkReturn efx_rc_t 964299611Sarybchiksiena_tx_qdesc_post( 965283514Sarybchik __in efx_txq_t *etp, 966283514Sarybchik __in_ecount(n) efx_desc_t *ed, 967283514Sarybchik __in unsigned int n, 968283514Sarybchik __in unsigned int completed, 969283514Sarybchik __inout unsigned int *addedp) 970283514Sarybchik{ 971283514Sarybchik unsigned int added = *addedp; 972283514Sarybchik unsigned int i; 973291436Sarybchik efx_rc_t rc; 974283514Sarybchik 975283514Sarybchik if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 976283514Sarybchik rc = ENOSPC; 977283514Sarybchik goto fail1; 978283514Sarybchik } 979283514Sarybchik 980283514Sarybchik for (i = 0; i < n; i++) { 981283514Sarybchik efx_desc_t *edp = &ed[i]; 982283514Sarybchik unsigned int id; 983283514Sarybchik size_t offset; 984283514Sarybchik 985283514Sarybchik id = added++ & etp->et_mask; 986283514Sarybchik offset = id * sizeof (efx_desc_t); 987283514Sarybchik 988283514Sarybchik EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 989283514Sarybchik } 990283514Sarybchik 991283514Sarybchik EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 992283514Sarybchik unsigned int, added, unsigned int, n); 993283514Sarybchik 994283514Sarybchik EFX_TX_QSTAT_INCR(etp, TX_POST); 995283514Sarybchik 996283514Sarybchik *addedp = added; 997283514Sarybchik return (0); 998283514Sarybchik 999283514Sarybchikfail1: 1000291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 1001283514Sarybchik return (rc); 1002283514Sarybchik} 1003283514Sarybchik 1004283514Sarybchik void 1005299611Sarybchiksiena_tx_qdesc_dma_create( 1006283514Sarybchik __in efx_txq_t *etp, 1007283514Sarybchik __in efsys_dma_addr_t addr, 1008283514Sarybchik __in size_t size, 1009283514Sarybchik __in boolean_t eop, 1010283514Sarybchik __out efx_desc_t *edp) 1011283514Sarybchik{ 1012283514Sarybchik /* Fragments must not span 4k boundaries. */ 1013283514Sarybchik EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); 1014283514Sarybchik 1015283514Sarybchik EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1016283514Sarybchik efsys_dma_addr_t, addr, 1017283514Sarybchik size_t, size, boolean_t, eop); 1018283514Sarybchik 1019283514Sarybchik EFX_POPULATE_QWORD_4(edp->ed_eq, 1020283514Sarybchik FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1021283514Sarybchik FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1022283514Sarybchik FSF_AZ_TX_KER_BUF_ADDR_DW0, 1023283514Sarybchik (uint32_t)(addr & 0xffffffff), 1024283514Sarybchik FSF_AZ_TX_KER_BUF_ADDR_DW1, 1025283514Sarybchik (uint32_t)(addr >> 32)); 1026283514Sarybchik} 1027283514Sarybchik 1028299320Sarybchik#endif /* EFSYS_OPT_SIENA */ 1029283514Sarybchik 1030277886Sarybchik#if EFSYS_OPT_QSTATS 1031227569Sphilip#if EFSYS_OPT_NAMES 1032283514Sarybchik/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ 1033283514Sarybchikstatic const char *__efx_tx_qstat_name[] = { 1034227569Sphilip "post", 1035283514Sarybchik "post_pio", 1036227569Sphilip}; 1037227569Sphilip/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1038227569Sphilip 1039283514Sarybchik const char * 1040227569Sphilipefx_tx_qstat_name( 1041227569Sphilip __in efx_nic_t *enp, 1042227569Sphilip __in unsigned int id) 1043227569Sphilip{ 1044227569Sphilip _NOTE(ARGUNUSED(enp)) 1045227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1046227569Sphilip EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1047227569Sphilip 1048227569Sphilip return (__efx_tx_qstat_name[id]); 1049227569Sphilip} 1050227569Sphilip#endif /* EFSYS_OPT_NAMES */ 1051283514Sarybchik#endif /* EFSYS_OPT_QSTATS */ 1052227569Sphilip 1053299320Sarybchik#if EFSYS_OPT_SIENA 1054283514Sarybchik 1055227569Sphilip#if EFSYS_OPT_QSTATS 1056283514Sarybchikstatic void 1057299611Sarybchiksiena_tx_qstats_update( 1058227569Sphilip __in efx_txq_t *etp, 1059227569Sphilip __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1060227569Sphilip{ 1061227569Sphilip unsigned int id; 1062227569Sphilip 1063227569Sphilip for (id = 0; id < TX_NQSTATS; id++) { 1064227569Sphilip efsys_stat_t *essp = &stat[id]; 1065227569Sphilip 1066227569Sphilip EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1067227569Sphilip etp->et_stat[id] = 0; 1068227569Sphilip } 1069227569Sphilip} 1070227569Sphilip#endif /* EFSYS_OPT_QSTATS */ 1071227569Sphilip 1072283514Sarybchikstatic void 1073299611Sarybchiksiena_tx_qdestroy( 1074227569Sphilip __in efx_txq_t *etp) 1075227569Sphilip{ 1076227569Sphilip efx_nic_t *enp = etp->et_enp; 1077227569Sphilip efx_oword_t oword; 1078227569Sphilip 1079227569Sphilip /* Purge descriptor queue */ 1080227569Sphilip EFX_ZERO_OWORD(oword); 1081227569Sphilip 1082227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1083283514Sarybchik etp->et_index, &oword, B_TRUE); 1084227569Sphilip} 1085227569Sphilip 1086283514Sarybchikstatic void 1087299611Sarybchiksiena_tx_fini( 1088227569Sphilip __in efx_nic_t *enp) 1089227569Sphilip{ 1090283514Sarybchik _NOTE(ARGUNUSED(enp)) 1091283514Sarybchik} 1092227569Sphilip 1093299320Sarybchik#endif /* EFSYS_OPT_SIENA */ 1094