1227569Sphilip/*- 2301388Sarybchik * Copyright (c) 2007-2016 Solarflare Communications Inc. 3284555Sarybchik * All rights reserved. 4227569Sphilip * 5227569Sphilip * Redistribution and use in source and binary forms, with or without 6284555Sarybchik * modification, are permitted provided that the following conditions are met: 7227569Sphilip * 8284555Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 9284555Sarybchik * this list of conditions and the following disclaimer. 10284555Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 11284555Sarybchik * this list of conditions and the following disclaimer in the documentation 12284555Sarybchik * and/or other materials provided with the distribution. 13284555Sarybchik * 14284555Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15284555Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16284555Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17284555Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18284555Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19284555Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20284555Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21284555Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22284555Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23284555Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24284555Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25284555Sarybchik * 26284555Sarybchik * The views and conclusions contained in the software and documentation are 27284555Sarybchik * those of the authors and should not be interpreted as representing official 28284555Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 29227569Sphilip */ 30227569Sphilip 31228078Sphilip#include <sys/cdefs.h> 32228078Sphilip__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/common/efx_rx.c 342505 2018-12-26 10:15:31Z arybchik $"); 33228078Sphilip 34227569Sphilip#include "efx.h" 35227569Sphilip#include "efx_impl.h" 36227569Sphilip 37284555Sarybchik 38301324Sarybchik#if EFSYS_OPT_SIENA 39284555Sarybchik 40293927Sarybchikstatic __checkReturn efx_rc_t 41301345Sarybchiksiena_rx_init( 42284555Sarybchik __in efx_nic_t *enp); 43284555Sarybchik 44284555Sarybchikstatic void 45301345Sarybchiksiena_rx_fini( 46284555Sarybchik __in efx_nic_t *enp); 47284555Sarybchik 48284555Sarybchik#if EFSYS_OPT_RX_SCATTER 49293927Sarybchikstatic __checkReturn efx_rc_t 50301345Sarybchiksiena_rx_scatter_enable( 51284555Sarybchik __in efx_nic_t *enp, 52284555Sarybchik __in unsigned int buf_size); 53284555Sarybchik#endif /* EFSYS_OPT_RX_SCATTER */ 54284555Sarybchik 55284555Sarybchik#if EFSYS_OPT_RX_SCALE 56293927Sarybchikstatic __checkReturn efx_rc_t 57301345Sarybchiksiena_rx_scale_mode_set( 58284555Sarybchik __in efx_nic_t *enp, 59284555Sarybchik __in efx_rx_hash_alg_t alg, 60284555Sarybchik __in efx_rx_hash_type_t type, 61284555Sarybchik __in boolean_t insert); 62284555Sarybchik 63293927Sarybchikstatic __checkReturn efx_rc_t 64301345Sarybchiksiena_rx_scale_key_set( 65284555Sarybchik __in efx_nic_t *enp, 66284555Sarybchik __in_ecount(n) uint8_t *key, 67284555Sarybchik __in size_t n); 68284555Sarybchik 69293927Sarybchikstatic __checkReturn efx_rc_t 70301345Sarybchiksiena_rx_scale_tbl_set( 71284555Sarybchik __in efx_nic_t *enp, 72284555Sarybchik __in_ecount(n) unsigned int *table, 73284555Sarybchik __in size_t n); 74284555Sarybchik 75294008Sarybchikstatic __checkReturn uint32_t 76301345Sarybchiksiena_rx_prefix_hash( 77294008Sarybchik __in efx_nic_t *enp, 78294008Sarybchik __in efx_rx_hash_alg_t func, 79294008Sarybchik __in uint8_t *buffer); 80294008Sarybchik 81294380Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 82294380Sarybchik 83294008Sarybchikstatic __checkReturn efx_rc_t 84301345Sarybchiksiena_rx_prefix_pktlen( 85294008Sarybchik __in efx_nic_t *enp, 86294008Sarybchik __in uint8_t *buffer, 87294008Sarybchik __out uint16_t *lengthp); 88294008Sarybchik 89284555Sarybchikstatic void 90301345Sarybchiksiena_rx_qpost( 91284555Sarybchik __in efx_rxq_t *erp, 92284555Sarybchik __in_ecount(n) efsys_dma_addr_t *addrp, 93284555Sarybchik __in size_t size, 94284555Sarybchik __in unsigned int n, 95284555Sarybchik __in unsigned int completed, 96284555Sarybchik __in unsigned int added); 97284555Sarybchik 98284555Sarybchikstatic void 99301345Sarybchiksiena_rx_qpush( 100284555Sarybchik __in efx_rxq_t *erp, 101284555Sarybchik __in unsigned int added, 102284555Sarybchik __inout unsigned int *pushedp); 103284555Sarybchik 104293927Sarybchikstatic __checkReturn efx_rc_t 105301345Sarybchiksiena_rx_qflush( 106284555Sarybchik __in efx_rxq_t *erp); 107284555Sarybchik 108284555Sarybchikstatic void 109301345Sarybchiksiena_rx_qenable( 110284555Sarybchik __in efx_rxq_t *erp); 111284555Sarybchik 112293927Sarybchikstatic __checkReturn efx_rc_t 113301345Sarybchiksiena_rx_qcreate( 114284555Sarybchik __in efx_nic_t *enp, 115284555Sarybchik __in unsigned int index, 116284555Sarybchik __in unsigned int label, 117284555Sarybchik __in efx_rxq_type_t type, 118284555Sarybchik __in efsys_mem_t *esmp, 119284555Sarybchik __in size_t n, 120284555Sarybchik __in uint32_t id, 121284555Sarybchik __in efx_evq_t *eep, 122284555Sarybchik __in efx_rxq_t *erp); 123284555Sarybchik 124284555Sarybchikstatic void 125301345Sarybchiksiena_rx_qdestroy( 126284555Sarybchik __in efx_rxq_t *erp); 127284555Sarybchik 128301324Sarybchik#endif /* EFSYS_OPT_SIENA */ 129284555Sarybchik 130284555Sarybchik 131284555Sarybchik#if EFSYS_OPT_SIENA 132301340Sarybchikstatic const efx_rx_ops_t __efx_rx_siena_ops = { 133301345Sarybchik siena_rx_init, /* erxo_init */ 134301345Sarybchik siena_rx_fini, /* erxo_fini */ 135284555Sarybchik#if EFSYS_OPT_RX_SCATTER 136301345Sarybchik siena_rx_scatter_enable, /* erxo_scatter_enable */ 137284555Sarybchik#endif 138284555Sarybchik#if EFSYS_OPT_RX_SCALE 139301345Sarybchik siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 140301345Sarybchik siena_rx_scale_key_set, /* erxo_scale_key_set */ 141301345Sarybchik siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 142301345Sarybchik siena_rx_prefix_hash, /* erxo_prefix_hash */ 143284555Sarybchik#endif 144301345Sarybchik siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 145301345Sarybchik siena_rx_qpost, /* erxo_qpost */ 146301345Sarybchik siena_rx_qpush, /* erxo_qpush */ 147301345Sarybchik siena_rx_qflush, /* erxo_qflush */ 148301345Sarybchik siena_rx_qenable, /* erxo_qenable */ 149301345Sarybchik siena_rx_qcreate, /* erxo_qcreate */ 150301345Sarybchik siena_rx_qdestroy, /* erxo_qdestroy */ 151284555Sarybchik}; 152284555Sarybchik#endif /* EFSYS_OPT_SIENA */ 153284555Sarybchik 154293988Sarybchik#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 155301340Sarybchikstatic const efx_rx_ops_t __efx_rx_ef10_ops = { 156293988Sarybchik ef10_rx_init, /* erxo_init */ 157293988Sarybchik ef10_rx_fini, /* erxo_fini */ 158284555Sarybchik#if EFSYS_OPT_RX_SCATTER 159293988Sarybchik ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160284555Sarybchik#endif 161284555Sarybchik#if EFSYS_OPT_RX_SCALE 162293988Sarybchik ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 163293988Sarybchik ef10_rx_scale_key_set, /* erxo_scale_key_set */ 164293988Sarybchik ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 165294008Sarybchik ef10_rx_prefix_hash, /* erxo_prefix_hash */ 166284555Sarybchik#endif 167294008Sarybchik ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 168293988Sarybchik ef10_rx_qpost, /* erxo_qpost */ 169293988Sarybchik ef10_rx_qpush, /* erxo_qpush */ 170293988Sarybchik ef10_rx_qflush, /* erxo_qflush */ 171293988Sarybchik ef10_rx_qenable, /* erxo_qenable */ 172293988Sarybchik ef10_rx_qcreate, /* erxo_qcreate */ 173293988Sarybchik ef10_rx_qdestroy, /* erxo_qdestroy */ 174284555Sarybchik}; 175293988Sarybchik#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 176284555Sarybchik 177284555Sarybchik 178293927Sarybchik __checkReturn efx_rc_t 179227569Sphilipefx_rx_init( 180284555Sarybchik __inout efx_nic_t *enp) 181227569Sphilip{ 182301340Sarybchik const efx_rx_ops_t *erxop; 183293927Sarybchik efx_rc_t rc; 184227569Sphilip 185227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 186227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 187227569Sphilip 188227569Sphilip if (!(enp->en_mod_flags & EFX_MOD_EV)) { 189227569Sphilip rc = EINVAL; 190227569Sphilip goto fail1; 191227569Sphilip } 192227569Sphilip 193227569Sphilip if (enp->en_mod_flags & EFX_MOD_RX) { 194227569Sphilip rc = EINVAL; 195227569Sphilip goto fail2; 196227569Sphilip } 197227569Sphilip 198284555Sarybchik switch (enp->en_family) { 199284555Sarybchik#if EFSYS_OPT_SIENA 200284555Sarybchik case EFX_FAMILY_SIENA: 201301340Sarybchik erxop = &__efx_rx_siena_ops; 202284555Sarybchik break; 203284555Sarybchik#endif /* EFSYS_OPT_SIENA */ 204284555Sarybchik 205284555Sarybchik#if EFSYS_OPT_HUNTINGTON 206284555Sarybchik case EFX_FAMILY_HUNTINGTON: 207301340Sarybchik erxop = &__efx_rx_ef10_ops; 208284555Sarybchik break; 209284555Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */ 210284555Sarybchik 211293988Sarybchik#if EFSYS_OPT_MEDFORD 212293988Sarybchik case EFX_FAMILY_MEDFORD: 213301340Sarybchik erxop = &__efx_rx_ef10_ops; 214293988Sarybchik break; 215293988Sarybchik#endif /* EFSYS_OPT_MEDFORD */ 216293988Sarybchik 217284555Sarybchik default: 218284555Sarybchik EFSYS_ASSERT(0); 219284555Sarybchik rc = ENOTSUP; 220284555Sarybchik goto fail3; 221284555Sarybchik } 222284555Sarybchik 223284555Sarybchik if ((rc = erxop->erxo_init(enp)) != 0) 224284555Sarybchik goto fail4; 225284555Sarybchik 226284555Sarybchik enp->en_erxop = erxop; 227284555Sarybchik enp->en_mod_flags |= EFX_MOD_RX; 228284555Sarybchik return (0); 229284555Sarybchik 230284555Sarybchikfail4: 231284555Sarybchik EFSYS_PROBE(fail4); 232284555Sarybchikfail3: 233284555Sarybchik EFSYS_PROBE(fail3); 234284555Sarybchikfail2: 235284555Sarybchik EFSYS_PROBE(fail2); 236284555Sarybchikfail1: 237293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 238284555Sarybchik 239284555Sarybchik enp->en_erxop = NULL; 240284555Sarybchik enp->en_mod_flags &= ~EFX_MOD_RX; 241284555Sarybchik return (rc); 242284555Sarybchik} 243284555Sarybchik 244284555Sarybchik void 245284555Sarybchikefx_rx_fini( 246284555Sarybchik __in efx_nic_t *enp) 247284555Sarybchik{ 248301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 249284555Sarybchik 250284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 251284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 252284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 253284555Sarybchik EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 254284555Sarybchik 255284555Sarybchik erxop->erxo_fini(enp); 256284555Sarybchik 257284555Sarybchik enp->en_erxop = NULL; 258284555Sarybchik enp->en_mod_flags &= ~EFX_MOD_RX; 259284555Sarybchik} 260284555Sarybchik 261284555Sarybchik#if EFSYS_OPT_RX_SCATTER 262293927Sarybchik __checkReturn efx_rc_t 263284555Sarybchikefx_rx_scatter_enable( 264284555Sarybchik __in efx_nic_t *enp, 265284555Sarybchik __in unsigned int buf_size) 266284555Sarybchik{ 267301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 268293927Sarybchik efx_rc_t rc; 269284555Sarybchik 270284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 271284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 272284555Sarybchik 273284555Sarybchik if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 274284555Sarybchik goto fail1; 275284555Sarybchik 276284555Sarybchik return (0); 277284555Sarybchik 278284555Sarybchikfail1: 279293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 280284555Sarybchik return (rc); 281284555Sarybchik} 282284555Sarybchik#endif /* EFSYS_OPT_RX_SCATTER */ 283284555Sarybchik 284284555Sarybchik#if EFSYS_OPT_RX_SCALE 285293927Sarybchik __checkReturn efx_rc_t 286284555Sarybchikefx_rx_hash_support_get( 287284555Sarybchik __in efx_nic_t *enp, 288284555Sarybchik __out efx_rx_hash_support_t *supportp) 289284555Sarybchik{ 290293927Sarybchik efx_rc_t rc; 291284555Sarybchik 292284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 293284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 294284555Sarybchik 295284555Sarybchik if (supportp == NULL) { 296284555Sarybchik rc = EINVAL; 297284555Sarybchik goto fail1; 298284555Sarybchik } 299284555Sarybchik 300284555Sarybchik /* Report if resources are available to insert RX hash value */ 301284555Sarybchik *supportp = enp->en_hash_support; 302284555Sarybchik 303284555Sarybchik return (0); 304284555Sarybchik 305284555Sarybchikfail1: 306293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 307284555Sarybchik 308284555Sarybchik return (rc); 309284555Sarybchik} 310284555Sarybchik 311293927Sarybchik __checkReturn efx_rc_t 312284555Sarybchikefx_rx_scale_support_get( 313284555Sarybchik __in efx_nic_t *enp, 314284555Sarybchik __out efx_rx_scale_support_t *supportp) 315284555Sarybchik{ 316293927Sarybchik efx_rc_t rc; 317284555Sarybchik 318284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 319284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 320284555Sarybchik 321284555Sarybchik if (supportp == NULL) { 322284555Sarybchik rc = EINVAL; 323284555Sarybchik goto fail1; 324284555Sarybchik } 325284555Sarybchik 326284555Sarybchik /* Report if resources are available to support RSS */ 327284555Sarybchik *supportp = enp->en_rss_support; 328284555Sarybchik 329284555Sarybchik return (0); 330284555Sarybchik 331284555Sarybchikfail1: 332293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 333284555Sarybchik 334284555Sarybchik return (rc); 335284555Sarybchik} 336284555Sarybchik 337293927Sarybchik __checkReturn efx_rc_t 338284555Sarybchikefx_rx_scale_mode_set( 339284555Sarybchik __in efx_nic_t *enp, 340284555Sarybchik __in efx_rx_hash_alg_t alg, 341284555Sarybchik __in efx_rx_hash_type_t type, 342284555Sarybchik __in boolean_t insert) 343284555Sarybchik{ 344301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 345293927Sarybchik efx_rc_t rc; 346284555Sarybchik 347284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 348284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 349284555Sarybchik 350284555Sarybchik if (erxop->erxo_scale_mode_set != NULL) { 351284555Sarybchik if ((rc = erxop->erxo_scale_mode_set(enp, alg, 352284555Sarybchik type, insert)) != 0) 353284555Sarybchik goto fail1; 354284555Sarybchik } 355284555Sarybchik 356284555Sarybchik return (0); 357284555Sarybchik 358284555Sarybchikfail1: 359293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 360284555Sarybchik return (rc); 361284555Sarybchik} 362284555Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 363284555Sarybchik 364284555Sarybchik#if EFSYS_OPT_RX_SCALE 365293927Sarybchik __checkReturn efx_rc_t 366284555Sarybchikefx_rx_scale_key_set( 367284555Sarybchik __in efx_nic_t *enp, 368284555Sarybchik __in_ecount(n) uint8_t *key, 369284555Sarybchik __in size_t n) 370284555Sarybchik{ 371301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 372293927Sarybchik efx_rc_t rc; 373284555Sarybchik 374284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 375284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 376284555Sarybchik 377284555Sarybchik if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0) 378284555Sarybchik goto fail1; 379284555Sarybchik 380284555Sarybchik return (0); 381284555Sarybchik 382284555Sarybchikfail1: 383293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 384284555Sarybchik 385284555Sarybchik return (rc); 386284555Sarybchik} 387284555Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 388284555Sarybchik 389284555Sarybchik#if EFSYS_OPT_RX_SCALE 390293927Sarybchik __checkReturn efx_rc_t 391284555Sarybchikefx_rx_scale_tbl_set( 392284555Sarybchik __in efx_nic_t *enp, 393284555Sarybchik __in_ecount(n) unsigned int *table, 394284555Sarybchik __in size_t n) 395284555Sarybchik{ 396301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 397293927Sarybchik efx_rc_t rc; 398284555Sarybchik 399284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 400284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 401284555Sarybchik 402284555Sarybchik if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0) 403284555Sarybchik goto fail1; 404284555Sarybchik 405284555Sarybchik return (0); 406284555Sarybchik 407284555Sarybchikfail1: 408293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 409284555Sarybchik 410284555Sarybchik return (rc); 411284555Sarybchik} 412284555Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 413284555Sarybchik 414284555Sarybchik void 415284555Sarybchikefx_rx_qpost( 416284555Sarybchik __in efx_rxq_t *erp, 417284555Sarybchik __in_ecount(n) efsys_dma_addr_t *addrp, 418284555Sarybchik __in size_t size, 419284555Sarybchik __in unsigned int n, 420284555Sarybchik __in unsigned int completed, 421284555Sarybchik __in unsigned int added) 422284555Sarybchik{ 423284555Sarybchik efx_nic_t *enp = erp->er_enp; 424301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 425284555Sarybchik 426284555Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 427284555Sarybchik 428284555Sarybchik erxop->erxo_qpost(erp, addrp, size, n, completed, added); 429284555Sarybchik} 430284555Sarybchik 431284555Sarybchik void 432284555Sarybchikefx_rx_qpush( 433284555Sarybchik __in efx_rxq_t *erp, 434284555Sarybchik __in unsigned int added, 435284555Sarybchik __inout unsigned int *pushedp) 436284555Sarybchik{ 437284555Sarybchik efx_nic_t *enp = erp->er_enp; 438301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 439284555Sarybchik 440284555Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 441284555Sarybchik 442284555Sarybchik erxop->erxo_qpush(erp, added, pushedp); 443284555Sarybchik} 444284555Sarybchik 445293927Sarybchik __checkReturn efx_rc_t 446284555Sarybchikefx_rx_qflush( 447284555Sarybchik __in efx_rxq_t *erp) 448284555Sarybchik{ 449284555Sarybchik efx_nic_t *enp = erp->er_enp; 450301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 451293927Sarybchik efx_rc_t rc; 452284555Sarybchik 453284555Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 454284555Sarybchik 455284555Sarybchik if ((rc = erxop->erxo_qflush(erp)) != 0) 456284555Sarybchik goto fail1; 457284555Sarybchik 458284555Sarybchik return (0); 459284555Sarybchik 460284555Sarybchikfail1: 461293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 462284555Sarybchik 463284555Sarybchik return (rc); 464284555Sarybchik} 465284555Sarybchik 466284555Sarybchik void 467284555Sarybchikefx_rx_qenable( 468284555Sarybchik __in efx_rxq_t *erp) 469284555Sarybchik{ 470284555Sarybchik efx_nic_t *enp = erp->er_enp; 471301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 472284555Sarybchik 473284555Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 474284555Sarybchik 475284555Sarybchik erxop->erxo_qenable(erp); 476284555Sarybchik} 477284555Sarybchik 478293927Sarybchik __checkReturn efx_rc_t 479284555Sarybchikefx_rx_qcreate( 480284555Sarybchik __in efx_nic_t *enp, 481284555Sarybchik __in unsigned int index, 482284555Sarybchik __in unsigned int label, 483284555Sarybchik __in efx_rxq_type_t type, 484284555Sarybchik __in efsys_mem_t *esmp, 485284555Sarybchik __in size_t n, 486284555Sarybchik __in uint32_t id, 487284555Sarybchik __in efx_evq_t *eep, 488284555Sarybchik __deref_out efx_rxq_t **erpp) 489284555Sarybchik{ 490301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 491284555Sarybchik efx_rxq_t *erp; 492293927Sarybchik efx_rc_t rc; 493284555Sarybchik 494284555Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 495284555Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 496284555Sarybchik 497284555Sarybchik /* Allocate an RXQ object */ 498284555Sarybchik EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 499284555Sarybchik 500284555Sarybchik if (erp == NULL) { 501284555Sarybchik rc = ENOMEM; 502284555Sarybchik goto fail1; 503284555Sarybchik } 504284555Sarybchik 505284555Sarybchik erp->er_magic = EFX_RXQ_MAGIC; 506284555Sarybchik erp->er_enp = enp; 507284555Sarybchik erp->er_index = index; 508284555Sarybchik erp->er_mask = n - 1; 509284555Sarybchik erp->er_esmp = esmp; 510284555Sarybchik 511284555Sarybchik if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id, 512284555Sarybchik eep, erp)) != 0) 513284555Sarybchik goto fail2; 514284555Sarybchik 515284555Sarybchik enp->en_rx_qcount++; 516284555Sarybchik *erpp = erp; 517284555Sarybchik 518284555Sarybchik return (0); 519284555Sarybchik 520284555Sarybchikfail2: 521284555Sarybchik EFSYS_PROBE(fail2); 522284555Sarybchik 523284555Sarybchik EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 524284555Sarybchikfail1: 525293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 526284555Sarybchik 527284555Sarybchik return (rc); 528284555Sarybchik} 529284555Sarybchik 530284555Sarybchik void 531284555Sarybchikefx_rx_qdestroy( 532284555Sarybchik __in efx_rxq_t *erp) 533284555Sarybchik{ 534284555Sarybchik efx_nic_t *enp = erp->er_enp; 535301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 536284555Sarybchik 537284555Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 538284555Sarybchik 539284555Sarybchik erxop->erxo_qdestroy(erp); 540284555Sarybchik} 541284555Sarybchik 542293927Sarybchik __checkReturn efx_rc_t 543311088Sarybchikefx_pseudo_hdr_pkt_length_get( 544311083Sarybchik __in efx_rxq_t *erp, 545284555Sarybchik __in uint8_t *buffer, 546294008Sarybchik __out uint16_t *lengthp) 547284555Sarybchik{ 548311083Sarybchik efx_nic_t *enp = erp->er_enp; 549301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 550284555Sarybchik 551311083Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 552311083Sarybchik 553294008Sarybchik return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 554284555Sarybchik} 555284555Sarybchik 556284555Sarybchik#if EFSYS_OPT_RX_SCALE 557294008Sarybchik __checkReturn uint32_t 558311088Sarybchikefx_pseudo_hdr_hash_get( 559311083Sarybchik __in efx_rxq_t *erp, 560284555Sarybchik __in efx_rx_hash_alg_t func, 561284555Sarybchik __in uint8_t *buffer) 562284555Sarybchik{ 563311083Sarybchik efx_nic_t *enp = erp->er_enp; 564301340Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 565294008Sarybchik 566311083Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 567311083Sarybchik 568294008Sarybchik EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 569294008Sarybchik return (erxop->erxo_prefix_hash(enp, func, buffer)); 570284555Sarybchik} 571284555Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 572284555Sarybchik 573301324Sarybchik#if EFSYS_OPT_SIENA 574284555Sarybchik 575293927Sarybchikstatic __checkReturn efx_rc_t 576301345Sarybchiksiena_rx_init( 577284555Sarybchik __in efx_nic_t *enp) 578284555Sarybchik{ 579284555Sarybchik efx_oword_t oword; 580284555Sarybchik unsigned int index; 581284555Sarybchik 582227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 583227569Sphilip 584227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 585227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 586227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 587227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 588227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 589227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 590227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 591227569Sphilip 592227569Sphilip /* Zero the RSS table */ 593227569Sphilip for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 594227569Sphilip index++) { 595227569Sphilip EFX_ZERO_OWORD(oword); 596227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 597284555Sarybchik index, &oword, B_TRUE); 598227569Sphilip } 599227569Sphilip 600284555Sarybchik#if EFSYS_OPT_RX_SCALE 601284555Sarybchik /* The RSS key and indirection table are writable. */ 602284555Sarybchik enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 603227569Sphilip 604284555Sarybchik /* Hardware can insert RX hash with/without RSS */ 605284555Sarybchik enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 606284555Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 607227569Sphilip 608284555Sarybchik return (0); 609227569Sphilip} 610227569Sphilip 611227569Sphilip#if EFSYS_OPT_RX_SCATTER 612293927Sarybchikstatic __checkReturn efx_rc_t 613301345Sarybchiksiena_rx_scatter_enable( 614227569Sphilip __in efx_nic_t *enp, 615227569Sphilip __in unsigned int buf_size) 616227569Sphilip{ 617227569Sphilip unsigned int nbuf32; 618227569Sphilip efx_oword_t oword; 619293927Sarybchik efx_rc_t rc; 620227569Sphilip 621227569Sphilip nbuf32 = buf_size / 32; 622227569Sphilip if ((nbuf32 == 0) || 623227569Sphilip (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 624227569Sphilip ((buf_size % 32) != 0)) { 625227569Sphilip rc = EINVAL; 626227569Sphilip goto fail1; 627227569Sphilip } 628227569Sphilip 629227569Sphilip if (enp->en_rx_qcount > 0) { 630227569Sphilip rc = EBUSY; 631227569Sphilip goto fail2; 632227569Sphilip } 633227569Sphilip 634227569Sphilip /* Set scatter buffer size */ 635227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 636227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 637227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 638227569Sphilip 639227569Sphilip /* Enable scatter for packets not matching a filter */ 640227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 641227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 642227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 643227569Sphilip 644227569Sphilip return (0); 645227569Sphilip 646227569Sphilipfail2: 647227569Sphilip EFSYS_PROBE(fail2); 648227569Sphilipfail1: 649293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 650227569Sphilip 651227569Sphilip return (rc); 652227569Sphilip} 653227569Sphilip#endif /* EFSYS_OPT_RX_SCATTER */ 654227569Sphilip 655227569Sphilip 656227569Sphilip#define EFX_RX_LFSR_HASH(_enp, _insert) \ 657284555Sarybchik do { \ 658227569Sphilip efx_oword_t oword; \ 659227569Sphilip \ 660227569Sphilip EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 661227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 662227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 663227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 664227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 665227569Sphilip (_insert) ? 1 : 0); \ 666227569Sphilip EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 667227569Sphilip \ 668227569Sphilip if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 669227569Sphilip EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 670227569Sphilip &oword); \ 671227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 672227569Sphilip FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 673227569Sphilip EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 674227569Sphilip &oword); \ 675227569Sphilip } \ 676227569Sphilip \ 677227569Sphilip _NOTE(CONSTANTCONDITION) \ 678227569Sphilip } while (B_FALSE) 679227569Sphilip 680227569Sphilip#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 681284555Sarybchik do { \ 682227569Sphilip efx_oword_t oword; \ 683227569Sphilip \ 684227569Sphilip EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 685227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 686227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 687227569Sphilip (_ip) ? 1 : 0); \ 688227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 689227569Sphilip (_tcp) ? 0 : 1); \ 690227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 691227569Sphilip (_insert) ? 1 : 0); \ 692227569Sphilip EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 693227569Sphilip \ 694227569Sphilip _NOTE(CONSTANTCONDITION) \ 695227569Sphilip } while (B_FALSE) 696227569Sphilip 697227569Sphilip#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 698284555Sarybchik do { \ 699227569Sphilip efx_oword_t oword; \ 700227569Sphilip \ 701227569Sphilip EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 702227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 703227569Sphilip FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 704227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 705227569Sphilip FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 706227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 707227569Sphilip FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 708227569Sphilip EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 709227569Sphilip \ 710227569Sphilip (_rc) = 0; \ 711227569Sphilip \ 712227569Sphilip _NOTE(CONSTANTCONDITION) \ 713227569Sphilip } while (B_FALSE) 714227569Sphilip 715227569Sphilip 716227569Sphilip#if EFSYS_OPT_RX_SCALE 717284555Sarybchik 718293927Sarybchikstatic __checkReturn efx_rc_t 719301345Sarybchiksiena_rx_scale_mode_set( 720227569Sphilip __in efx_nic_t *enp, 721227569Sphilip __in efx_rx_hash_alg_t alg, 722227569Sphilip __in efx_rx_hash_type_t type, 723227569Sphilip __in boolean_t insert) 724227569Sphilip{ 725293927Sarybchik efx_rc_t rc; 726227569Sphilip 727227569Sphilip switch (alg) { 728227569Sphilip case EFX_RX_HASHALG_LFSR: 729227569Sphilip EFX_RX_LFSR_HASH(enp, insert); 730227569Sphilip break; 731227569Sphilip 732227569Sphilip case EFX_RX_HASHALG_TOEPLITZ: 733227569Sphilip EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 734311086Sarybchik type & EFX_RX_HASH_IPV4, 735311086Sarybchik type & EFX_RX_HASH_TCPIPV4); 736227569Sphilip 737227569Sphilip EFX_RX_TOEPLITZ_IPV6_HASH(enp, 738311086Sarybchik type & EFX_RX_HASH_IPV6, 739311086Sarybchik type & EFX_RX_HASH_TCPIPV6, 740227569Sphilip rc); 741227569Sphilip if (rc != 0) 742227569Sphilip goto fail1; 743227569Sphilip 744227569Sphilip break; 745227569Sphilip 746227569Sphilip default: 747227569Sphilip rc = EINVAL; 748227569Sphilip goto fail2; 749227569Sphilip } 750227569Sphilip 751227569Sphilip return (0); 752227569Sphilip 753227569Sphilipfail2: 754227569Sphilip EFSYS_PROBE(fail2); 755227569Sphilipfail1: 756293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 757227569Sphilip 758227569Sphilip EFX_RX_LFSR_HASH(enp, B_FALSE); 759227569Sphilip 760227569Sphilip return (rc); 761227569Sphilip} 762227569Sphilip#endif 763227569Sphilip 764227569Sphilip#if EFSYS_OPT_RX_SCALE 765293927Sarybchikstatic __checkReturn efx_rc_t 766301345Sarybchiksiena_rx_scale_key_set( 767227569Sphilip __in efx_nic_t *enp, 768227569Sphilip __in_ecount(n) uint8_t *key, 769227569Sphilip __in size_t n) 770227569Sphilip{ 771227569Sphilip efx_oword_t oword; 772227569Sphilip unsigned int byte; 773227569Sphilip unsigned int offset; 774293927Sarybchik efx_rc_t rc; 775227569Sphilip 776227569Sphilip byte = 0; 777227569Sphilip 778284555Sarybchik /* Write Toeplitz IPv4 hash key */ 779227569Sphilip EFX_ZERO_OWORD(oword); 780227569Sphilip for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 781227569Sphilip offset > 0 && byte < n; 782227569Sphilip --offset) 783227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 784227569Sphilip 785227569Sphilip EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 786227569Sphilip 787227569Sphilip byte = 0; 788227569Sphilip 789284555Sarybchik /* Verify Toeplitz IPv4 hash key */ 790227569Sphilip EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 791227569Sphilip for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 792227569Sphilip offset > 0 && byte < n; 793227569Sphilip --offset) { 794227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 795227569Sphilip rc = EFAULT; 796227569Sphilip goto fail1; 797227569Sphilip } 798227569Sphilip } 799227569Sphilip 800284555Sarybchik if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 801284555Sarybchik goto done; 802227569Sphilip 803227569Sphilip byte = 0; 804227569Sphilip 805284555Sarybchik /* Write Toeplitz IPv6 hash key 3 */ 806227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 807227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 808227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 809227569Sphilip offset > 0 && byte < n; 810227569Sphilip --offset) 811227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 812227569Sphilip 813227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 814227569Sphilip 815284555Sarybchik /* Write Toeplitz IPv6 hash key 2 */ 816227569Sphilip EFX_ZERO_OWORD(oword); 817227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 818227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 819227569Sphilip offset > 0 && byte < n; 820227569Sphilip --offset) 821227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 822227569Sphilip 823227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 824227569Sphilip 825284555Sarybchik /* Write Toeplitz IPv6 hash key 1 */ 826227569Sphilip EFX_ZERO_OWORD(oword); 827227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 828227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 829227569Sphilip offset > 0 && byte < n; 830227569Sphilip --offset) 831227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 832227569Sphilip 833227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 834227569Sphilip 835227569Sphilip byte = 0; 836227569Sphilip 837284555Sarybchik /* Verify Toeplitz IPv6 hash key 3 */ 838227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 839227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 840227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 841227569Sphilip offset > 0 && byte < n; 842227569Sphilip --offset) { 843227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 844227569Sphilip rc = EFAULT; 845284555Sarybchik goto fail2; 846227569Sphilip } 847227569Sphilip } 848227569Sphilip 849284555Sarybchik /* Verify Toeplitz IPv6 hash key 2 */ 850227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 851227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 852227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 853227569Sphilip offset > 0 && byte < n; 854227569Sphilip --offset) { 855227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 856227569Sphilip rc = EFAULT; 857284555Sarybchik goto fail3; 858227569Sphilip } 859227569Sphilip } 860227569Sphilip 861284555Sarybchik /* Verify Toeplitz IPv6 hash key 1 */ 862227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 863227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 864227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 865227569Sphilip offset > 0 && byte < n; 866227569Sphilip --offset) { 867227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 868227569Sphilip rc = EFAULT; 869284555Sarybchik goto fail4; 870227569Sphilip } 871227569Sphilip } 872227569Sphilip 873284555Sarybchikdone: 874227569Sphilip return (0); 875227569Sphilip 876284555Sarybchikfail4: 877284555Sarybchik EFSYS_PROBE(fail4); 878227569Sphilipfail3: 879227569Sphilip EFSYS_PROBE(fail3); 880227569Sphilipfail2: 881227569Sphilip EFSYS_PROBE(fail2); 882227569Sphilipfail1: 883293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 884227569Sphilip 885227569Sphilip return (rc); 886227569Sphilip} 887227569Sphilip#endif 888227569Sphilip 889227569Sphilip#if EFSYS_OPT_RX_SCALE 890293927Sarybchikstatic __checkReturn efx_rc_t 891301345Sarybchiksiena_rx_scale_tbl_set( 892227569Sphilip __in efx_nic_t *enp, 893227569Sphilip __in_ecount(n) unsigned int *table, 894227569Sphilip __in size_t n) 895227569Sphilip{ 896227569Sphilip efx_oword_t oword; 897227569Sphilip int index; 898293927Sarybchik efx_rc_t rc; 899227569Sphilip 900227569Sphilip EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 901227569Sphilip EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 902227569Sphilip 903227569Sphilip if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 904227569Sphilip rc = EINVAL; 905227569Sphilip goto fail1; 906227569Sphilip } 907227569Sphilip 908227569Sphilip for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 909227569Sphilip uint32_t byte; 910227569Sphilip 911227569Sphilip /* Calculate the entry to place in the table */ 912284555Sarybchik byte = (n > 0) ? (uint32_t)table[index % n] : 0; 913227569Sphilip 914227569Sphilip EFSYS_PROBE2(table, int, index, uint32_t, byte); 915227569Sphilip 916227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 917227569Sphilip 918227569Sphilip /* Write the table */ 919227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 920284555Sarybchik index, &oword, B_TRUE); 921227569Sphilip } 922227569Sphilip 923227569Sphilip for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 924227569Sphilip uint32_t byte; 925227569Sphilip 926227569Sphilip /* Determine if we're starting a new batch */ 927284555Sarybchik byte = (n > 0) ? (uint32_t)table[index % n] : 0; 928227569Sphilip 929227569Sphilip /* Read the table */ 930227569Sphilip EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 931284555Sarybchik index, &oword, B_TRUE); 932227569Sphilip 933227569Sphilip /* Verify the entry */ 934227569Sphilip if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 935227569Sphilip rc = EFAULT; 936227569Sphilip goto fail2; 937227569Sphilip } 938227569Sphilip } 939227569Sphilip 940227569Sphilip return (0); 941227569Sphilip 942227569Sphilipfail2: 943227569Sphilip EFSYS_PROBE(fail2); 944227569Sphilipfail1: 945293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 946227569Sphilip 947227569Sphilip return (rc); 948227569Sphilip} 949227569Sphilip#endif 950227569Sphilip 951294008Sarybchik/* 952311088Sarybchik * Falcon/Siena pseudo-header 953294008Sarybchik * -------------------------- 954294008Sarybchik * 955294008Sarybchik * Receive packets are prefixed by an optional 16 byte pseudo-header. 956311088Sarybchik * The pseudo-header is a byte array of one of the forms: 957294008Sarybchik * 958294008Sarybchik * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 959294008Sarybchik * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 960294008Sarybchik * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 961294008Sarybchik * 962294008Sarybchik * where: 963294008Sarybchik * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 964294008Sarybchik * LL.LL LFSR hash (16-bit big-endian) 965294008Sarybchik */ 966294008Sarybchik 967294008Sarybchik#if EFSYS_OPT_RX_SCALE 968294008Sarybchikstatic __checkReturn uint32_t 969301345Sarybchiksiena_rx_prefix_hash( 970294008Sarybchik __in efx_nic_t *enp, 971294008Sarybchik __in efx_rx_hash_alg_t func, 972294008Sarybchik __in uint8_t *buffer) 973294008Sarybchik{ 974301392Sarybchik _NOTE(ARGUNUSED(enp)) 975301392Sarybchik 976294008Sarybchik switch (func) { 977294008Sarybchik case EFX_RX_HASHALG_TOEPLITZ: 978294008Sarybchik return ((buffer[12] << 24) | 979294008Sarybchik (buffer[13] << 16) | 980294008Sarybchik (buffer[14] << 8) | 981294008Sarybchik buffer[15]); 982294008Sarybchik 983294008Sarybchik case EFX_RX_HASHALG_LFSR: 984294008Sarybchik return ((buffer[14] << 8) | buffer[15]); 985294008Sarybchik 986294008Sarybchik default: 987294008Sarybchik EFSYS_ASSERT(0); 988294008Sarybchik return (0); 989294008Sarybchik } 990294008Sarybchik} 991294008Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 992294008Sarybchik 993294008Sarybchikstatic __checkReturn efx_rc_t 994301345Sarybchiksiena_rx_prefix_pktlen( 995294008Sarybchik __in efx_nic_t *enp, 996294008Sarybchik __in uint8_t *buffer, 997294008Sarybchik __out uint16_t *lengthp) 998294008Sarybchik{ 999301392Sarybchik _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1000301392Sarybchik 1001294008Sarybchik /* Not supported by Falcon/Siena hardware */ 1002294008Sarybchik EFSYS_ASSERT(0); 1003294008Sarybchik return (ENOTSUP); 1004294008Sarybchik} 1005294008Sarybchik 1006294008Sarybchik 1007284555Sarybchikstatic void 1008301345Sarybchiksiena_rx_qpost( 1009227569Sphilip __in efx_rxq_t *erp, 1010227569Sphilip __in_ecount(n) efsys_dma_addr_t *addrp, 1011227569Sphilip __in size_t size, 1012227569Sphilip __in unsigned int n, 1013227569Sphilip __in unsigned int completed, 1014227569Sphilip __in unsigned int added) 1015227569Sphilip{ 1016227569Sphilip efx_qword_t qword; 1017227569Sphilip unsigned int i; 1018227569Sphilip unsigned int offset; 1019227569Sphilip unsigned int id; 1020227569Sphilip 1021227569Sphilip /* The client driver must not overfill the queue */ 1022227569Sphilip EFSYS_ASSERT3U(added - completed + n, <=, 1023227569Sphilip EFX_RXQ_LIMIT(erp->er_mask + 1)); 1024227569Sphilip 1025227569Sphilip id = added & (erp->er_mask); 1026227569Sphilip for (i = 0; i < n; i++) { 1027227569Sphilip EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1028227569Sphilip unsigned int, id, efsys_dma_addr_t, addrp[i], 1029227569Sphilip size_t, size); 1030227569Sphilip 1031227569Sphilip EFX_POPULATE_QWORD_3(qword, 1032227569Sphilip FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1033227569Sphilip FSF_AZ_RX_KER_BUF_ADDR_DW0, 1034227569Sphilip (uint32_t)(addrp[i] & 0xffffffff), 1035227569Sphilip FSF_AZ_RX_KER_BUF_ADDR_DW1, 1036227569Sphilip (uint32_t)(addrp[i] >> 32)); 1037227569Sphilip 1038227569Sphilip offset = id * sizeof (efx_qword_t); 1039227569Sphilip EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1040227569Sphilip 1041227569Sphilip id = (id + 1) & (erp->er_mask); 1042227569Sphilip } 1043227569Sphilip} 1044227569Sphilip 1045284555Sarybchikstatic void 1046301345Sarybchiksiena_rx_qpush( 1047227569Sphilip __in efx_rxq_t *erp, 1048284555Sarybchik __in unsigned int added, 1049284555Sarybchik __inout unsigned int *pushedp) 1050227569Sphilip{ 1051227569Sphilip efx_nic_t *enp = erp->er_enp; 1052284555Sarybchik unsigned int pushed = *pushedp; 1053227569Sphilip uint32_t wptr; 1054227569Sphilip efx_oword_t oword; 1055227569Sphilip efx_dword_t dword; 1056227569Sphilip 1057284555Sarybchik /* All descriptors are pushed */ 1058284555Sarybchik *pushedp = added; 1059227569Sphilip 1060227569Sphilip /* Push the populated descriptors out */ 1061227569Sphilip wptr = added & erp->er_mask; 1062227569Sphilip 1063227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1064227569Sphilip 1065227569Sphilip /* Only write the third DWORD */ 1066227569Sphilip EFX_POPULATE_DWORD_1(dword, 1067227569Sphilip EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1068284555Sarybchik 1069284555Sarybchik /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1070284555Sarybchik EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1071284555Sarybchik wptr, pushed & erp->er_mask); 1072284555Sarybchik EFSYS_PIO_WRITE_BARRIER(); 1073227569Sphilip EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1074227569Sphilip erp->er_index, &dword, B_FALSE); 1075227569Sphilip} 1076227569Sphilip 1077293927Sarybchikstatic __checkReturn efx_rc_t 1078301345Sarybchiksiena_rx_qflush( 1079227569Sphilip __in efx_rxq_t *erp) 1080227569Sphilip{ 1081227569Sphilip efx_nic_t *enp = erp->er_enp; 1082227569Sphilip efx_oword_t oword; 1083227569Sphilip uint32_t label; 1084227569Sphilip 1085227569Sphilip label = erp->er_index; 1086227569Sphilip 1087227569Sphilip /* Flush the queue */ 1088227569Sphilip EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1089227569Sphilip FRF_AZ_RX_FLUSH_DESCQ, label); 1090227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1091284555Sarybchik 1092284555Sarybchik return (0); 1093227569Sphilip} 1094227569Sphilip 1095284555Sarybchikstatic void 1096301345Sarybchiksiena_rx_qenable( 1097227569Sphilip __in efx_rxq_t *erp) 1098227569Sphilip{ 1099227569Sphilip efx_nic_t *enp = erp->er_enp; 1100227569Sphilip efx_oword_t oword; 1101227569Sphilip 1102227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1103227569Sphilip 1104227569Sphilip EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1105284555Sarybchik erp->er_index, &oword, B_TRUE); 1106227569Sphilip 1107227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1108227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1109227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1110227569Sphilip 1111227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1112284555Sarybchik erp->er_index, &oword, B_TRUE); 1113227569Sphilip} 1114227569Sphilip 1115293927Sarybchikstatic __checkReturn efx_rc_t 1116301345Sarybchiksiena_rx_qcreate( 1117227569Sphilip __in efx_nic_t *enp, 1118227569Sphilip __in unsigned int index, 1119227569Sphilip __in unsigned int label, 1120227569Sphilip __in efx_rxq_type_t type, 1121227569Sphilip __in efsys_mem_t *esmp, 1122227569Sphilip __in size_t n, 1123227569Sphilip __in uint32_t id, 1124227569Sphilip __in efx_evq_t *eep, 1125284555Sarybchik __in efx_rxq_t *erp) 1126227569Sphilip{ 1127227569Sphilip efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1128227569Sphilip efx_oword_t oword; 1129227569Sphilip uint32_t size; 1130227569Sphilip boolean_t jumbo; 1131293927Sarybchik efx_rc_t rc; 1132227569Sphilip 1133301392Sarybchik _NOTE(ARGUNUSED(esmp)) 1134301392Sarybchik 1135280535Sarybchik EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1136280535Sarybchik (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1137227569Sphilip EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1138227569Sphilip EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1139227569Sphilip 1140284555Sarybchik EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1141284555Sarybchik EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1142284555Sarybchik 1143284555Sarybchik if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 1144227569Sphilip rc = EINVAL; 1145227569Sphilip goto fail1; 1146227569Sphilip } 1147227569Sphilip if (index >= encp->enc_rxq_limit) { 1148227569Sphilip rc = EINVAL; 1149227569Sphilip goto fail2; 1150227569Sphilip } 1151227569Sphilip for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1152227569Sphilip size++) 1153227569Sphilip if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 1154227569Sphilip break; 1155227569Sphilip if (id + (1 << size) >= encp->enc_buftbl_limit) { 1156227569Sphilip rc = EINVAL; 1157227569Sphilip goto fail3; 1158227569Sphilip } 1159227569Sphilip 1160227569Sphilip switch (type) { 1161227569Sphilip case EFX_RXQ_TYPE_DEFAULT: 1162227569Sphilip jumbo = B_FALSE; 1163227569Sphilip break; 1164227569Sphilip 1165227569Sphilip#if EFSYS_OPT_RX_SCATTER 1166227569Sphilip case EFX_RXQ_TYPE_SCATTER: 1167227569Sphilip jumbo = B_TRUE; 1168227569Sphilip break; 1169227569Sphilip#endif /* EFSYS_OPT_RX_SCATTER */ 1170227569Sphilip 1171227569Sphilip default: 1172227569Sphilip rc = EINVAL; 1173227569Sphilip goto fail4; 1174227569Sphilip } 1175227569Sphilip 1176227569Sphilip /* Set up the new descriptor queue */ 1177294004Sarybchik EFX_POPULATE_OWORD_7(oword, 1178227569Sphilip FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1179227569Sphilip FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1180227569Sphilip FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1181227569Sphilip FRF_AZ_RX_DESCQ_LABEL, label, 1182227569Sphilip FRF_AZ_RX_DESCQ_SIZE, size, 1183227569Sphilip FRF_AZ_RX_DESCQ_TYPE, 0, 1184227569Sphilip FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1185227569Sphilip 1186227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1187284555Sarybchik erp->er_index, &oword, B_TRUE); 1188227569Sphilip 1189227569Sphilip return (0); 1190227569Sphilip 1191227569Sphilipfail4: 1192227569Sphilip EFSYS_PROBE(fail4); 1193227569Sphilipfail3: 1194227569Sphilip EFSYS_PROBE(fail3); 1195227569Sphilipfail2: 1196227569Sphilip EFSYS_PROBE(fail2); 1197227569Sphilipfail1: 1198293927Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 1199227569Sphilip 1200227569Sphilip return (rc); 1201227569Sphilip} 1202227569Sphilip 1203284555Sarybchikstatic void 1204301345Sarybchiksiena_rx_qdestroy( 1205227569Sphilip __in efx_rxq_t *erp) 1206227569Sphilip{ 1207227569Sphilip efx_nic_t *enp = erp->er_enp; 1208227569Sphilip efx_oword_t oword; 1209227569Sphilip 1210227569Sphilip EFSYS_ASSERT(enp->en_rx_qcount != 0); 1211227569Sphilip --enp->en_rx_qcount; 1212227569Sphilip 1213227569Sphilip /* Purge descriptor queue */ 1214227569Sphilip EFX_ZERO_OWORD(oword); 1215227569Sphilip 1216227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1217284555Sarybchik erp->er_index, &oword, B_TRUE); 1218227569Sphilip 1219227569Sphilip /* Free the RXQ object */ 1220227569Sphilip EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1221227569Sphilip} 1222227569Sphilip 1223284555Sarybchikstatic void 1224301345Sarybchiksiena_rx_fini( 1225227569Sphilip __in efx_nic_t *enp) 1226227569Sphilip{ 1227284555Sarybchik _NOTE(ARGUNUSED(enp)) 1228284555Sarybchik} 1229227569Sphilip 1230301324Sarybchik#endif /* EFSYS_OPT_SIENA */ 1231