efx_rx.c revision 299517
1227569Sphilip/*- 2283514Sarybchik * Copyright (c) 2007-2015 Solarflare Communications Inc. 3283514Sarybchik * All rights reserved. 4227569Sphilip * 5227569Sphilip * Redistribution and use in source and binary forms, with or without 6283514Sarybchik * modification, are permitted provided that the following conditions are met: 7227569Sphilip * 8283514Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 9283514Sarybchik * this list of conditions and the following disclaimer. 10283514Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 11283514Sarybchik * this list of conditions and the following disclaimer in the documentation 12283514Sarybchik * and/or other materials provided with the distribution. 13283514Sarybchik * 14283514Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15283514Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16283514Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17283514Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18283514Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19283514Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20283514Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21283514Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22283514Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23283514Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24283514Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25283514Sarybchik * 26283514Sarybchik * The views and conclusions contained in the software and documentation are 27283514Sarybchik * those of the authors and should not be interpreted as representing official 28283514Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 29227569Sphilip */ 30227569Sphilip 31228078Sphilip#include <sys/cdefs.h> 32228078Sphilip__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_rx.c 299517 2016-05-12 06:19:06Z arybchik $"); 33228078Sphilip 34227569Sphilip#include "efx.h" 35227569Sphilip#include "efx_impl.h" 36227569Sphilip 37283514Sarybchik 38299320Sarybchik#if EFSYS_OPT_SIENA 39283514Sarybchik 40291436Sarybchikstatic __checkReturn efx_rc_t 41283514Sarybchikfalconsiena_rx_init( 42283514Sarybchik __in efx_nic_t *enp); 43283514Sarybchik 44283514Sarybchikstatic void 45283514Sarybchikfalconsiena_rx_fini( 46283514Sarybchik __in efx_nic_t *enp); 47283514Sarybchik 48283514Sarybchik#if EFSYS_OPT_RX_SCATTER 49291436Sarybchikstatic __checkReturn efx_rc_t 50283514Sarybchikfalconsiena_rx_scatter_enable( 51283514Sarybchik __in efx_nic_t *enp, 52283514Sarybchik __in unsigned int buf_size); 53283514Sarybchik#endif /* EFSYS_OPT_RX_SCATTER */ 54283514Sarybchik 55283514Sarybchik#if EFSYS_OPT_RX_SCALE 56291436Sarybchikstatic __checkReturn efx_rc_t 57283514Sarybchikfalconsiena_rx_scale_mode_set( 58283514Sarybchik __in efx_nic_t *enp, 59283514Sarybchik __in efx_rx_hash_alg_t alg, 60283514Sarybchik __in efx_rx_hash_type_t type, 61283514Sarybchik __in boolean_t insert); 62283514Sarybchik 63291436Sarybchikstatic __checkReturn efx_rc_t 64283514Sarybchikfalconsiena_rx_scale_key_set( 65283514Sarybchik __in efx_nic_t *enp, 66283514Sarybchik __in_ecount(n) uint8_t *key, 67283514Sarybchik __in size_t n); 68283514Sarybchik 69291436Sarybchikstatic __checkReturn efx_rc_t 70283514Sarybchikfalconsiena_rx_scale_tbl_set( 71283514Sarybchik __in efx_nic_t *enp, 72283514Sarybchik __in_ecount(n) unsigned int *table, 73283514Sarybchik __in size_t n); 74283514Sarybchik 75293807Sarybchikstatic __checkReturn uint32_t 76293807Sarybchikfalconsiena_rx_prefix_hash( 77293807Sarybchik __in efx_nic_t *enp, 78293807Sarybchik __in efx_rx_hash_alg_t func, 79293807Sarybchik __in uint8_t *buffer); 80293807Sarybchik 81293890Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 82293890Sarybchik 83293807Sarybchikstatic __checkReturn efx_rc_t 84293807Sarybchikfalconsiena_rx_prefix_pktlen( 85293807Sarybchik __in efx_nic_t *enp, 86293807Sarybchik __in uint8_t *buffer, 87293807Sarybchik __out uint16_t *lengthp); 88293807Sarybchik 89283514Sarybchikstatic void 90283514Sarybchikfalconsiena_rx_qpost( 91283514Sarybchik __in efx_rxq_t *erp, 92283514Sarybchik __in_ecount(n) efsys_dma_addr_t *addrp, 93283514Sarybchik __in size_t size, 94283514Sarybchik __in unsigned int n, 95283514Sarybchik __in unsigned int completed, 96283514Sarybchik __in unsigned int added); 97283514Sarybchik 98283514Sarybchikstatic void 99283514Sarybchikfalconsiena_rx_qpush( 100283514Sarybchik __in efx_rxq_t *erp, 101283514Sarybchik __in unsigned int added, 102283514Sarybchik __inout unsigned int *pushedp); 103283514Sarybchik 104291436Sarybchikstatic __checkReturn efx_rc_t 105283514Sarybchikfalconsiena_rx_qflush( 106283514Sarybchik __in efx_rxq_t *erp); 107283514Sarybchik 108283514Sarybchikstatic void 109283514Sarybchikfalconsiena_rx_qenable( 110283514Sarybchik __in efx_rxq_t *erp); 111283514Sarybchik 112291436Sarybchikstatic __checkReturn efx_rc_t 113283514Sarybchikfalconsiena_rx_qcreate( 114283514Sarybchik __in efx_nic_t *enp, 115283514Sarybchik __in unsigned int index, 116283514Sarybchik __in unsigned int label, 117283514Sarybchik __in efx_rxq_type_t type, 118283514Sarybchik __in efsys_mem_t *esmp, 119283514Sarybchik __in size_t n, 120283514Sarybchik __in uint32_t id, 121283514Sarybchik __in efx_evq_t *eep, 122283514Sarybchik __in efx_rxq_t *erp); 123283514Sarybchik 124283514Sarybchikstatic void 125283514Sarybchikfalconsiena_rx_qdestroy( 126283514Sarybchik __in efx_rxq_t *erp); 127283514Sarybchik 128299320Sarybchik#endif /* EFSYS_OPT_SIENA */ 129283514Sarybchik 130283514Sarybchik 131283514Sarybchik#if EFSYS_OPT_SIENA 132299517Sarybchikstatic const efx_rx_ops_t __efx_rx_siena_ops = { 133283514Sarybchik falconsiena_rx_init, /* erxo_init */ 134283514Sarybchik falconsiena_rx_fini, /* erxo_fini */ 135283514Sarybchik#if EFSYS_OPT_RX_SCATTER 136283514Sarybchik falconsiena_rx_scatter_enable, /* erxo_scatter_enable */ 137283514Sarybchik#endif 138283514Sarybchik#if EFSYS_OPT_RX_SCALE 139283514Sarybchik falconsiena_rx_scale_mode_set, /* erxo_scale_mode_set */ 140283514Sarybchik falconsiena_rx_scale_key_set, /* erxo_scale_key_set */ 141283514Sarybchik falconsiena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 142293807Sarybchik falconsiena_rx_prefix_hash, /* erxo_prefix_hash */ 143283514Sarybchik#endif 144293807Sarybchik falconsiena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 145283514Sarybchik falconsiena_rx_qpost, /* erxo_qpost */ 146283514Sarybchik falconsiena_rx_qpush, /* erxo_qpush */ 147283514Sarybchik falconsiena_rx_qflush, /* erxo_qflush */ 148283514Sarybchik falconsiena_rx_qenable, /* erxo_qenable */ 149283514Sarybchik falconsiena_rx_qcreate, /* erxo_qcreate */ 150283514Sarybchik falconsiena_rx_qdestroy, /* erxo_qdestroy */ 151283514Sarybchik}; 152283514Sarybchik#endif /* EFSYS_OPT_SIENA */ 153283514Sarybchik 154293754Sarybchik#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 155299517Sarybchikstatic const efx_rx_ops_t __efx_rx_ef10_ops = { 156293754Sarybchik ef10_rx_init, /* erxo_init */ 157293754Sarybchik ef10_rx_fini, /* erxo_fini */ 158283514Sarybchik#if EFSYS_OPT_RX_SCATTER 159293754Sarybchik ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160283514Sarybchik#endif 161283514Sarybchik#if EFSYS_OPT_RX_SCALE 162293754Sarybchik ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 163293754Sarybchik ef10_rx_scale_key_set, /* erxo_scale_key_set */ 164293754Sarybchik ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 165293807Sarybchik ef10_rx_prefix_hash, /* erxo_prefix_hash */ 166283514Sarybchik#endif 167293807Sarybchik ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 168293754Sarybchik ef10_rx_qpost, /* erxo_qpost */ 169293754Sarybchik ef10_rx_qpush, /* erxo_qpush */ 170293754Sarybchik ef10_rx_qflush, /* erxo_qflush */ 171293754Sarybchik ef10_rx_qenable, /* erxo_qenable */ 172293754Sarybchik ef10_rx_qcreate, /* erxo_qcreate */ 173293754Sarybchik ef10_rx_qdestroy, /* erxo_qdestroy */ 174283514Sarybchik}; 175293754Sarybchik#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 176283514Sarybchik 177283514Sarybchik 178291436Sarybchik __checkReturn efx_rc_t 179227569Sphilipefx_rx_init( 180283514Sarybchik __inout efx_nic_t *enp) 181227569Sphilip{ 182299517Sarybchik const efx_rx_ops_t *erxop; 183291436Sarybchik efx_rc_t rc; 184227569Sphilip 185227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 186227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 187227569Sphilip 188227569Sphilip if (!(enp->en_mod_flags & EFX_MOD_EV)) { 189227569Sphilip rc = EINVAL; 190227569Sphilip goto fail1; 191227569Sphilip } 192227569Sphilip 193227569Sphilip if (enp->en_mod_flags & EFX_MOD_RX) { 194227569Sphilip rc = EINVAL; 195227569Sphilip goto fail2; 196227569Sphilip } 197227569Sphilip 198283514Sarybchik switch (enp->en_family) { 199283514Sarybchik#if EFSYS_OPT_SIENA 200283514Sarybchik case EFX_FAMILY_SIENA: 201299517Sarybchik erxop = &__efx_rx_siena_ops; 202283514Sarybchik break; 203283514Sarybchik#endif /* EFSYS_OPT_SIENA */ 204283514Sarybchik 205283514Sarybchik#if EFSYS_OPT_HUNTINGTON 206283514Sarybchik case EFX_FAMILY_HUNTINGTON: 207299517Sarybchik erxop = &__efx_rx_ef10_ops; 208283514Sarybchik break; 209283514Sarybchik#endif /* EFSYS_OPT_HUNTINGTON */ 210283514Sarybchik 211293754Sarybchik#if EFSYS_OPT_MEDFORD 212293754Sarybchik case EFX_FAMILY_MEDFORD: 213299517Sarybchik erxop = &__efx_rx_ef10_ops; 214293754Sarybchik break; 215293754Sarybchik#endif /* EFSYS_OPT_MEDFORD */ 216293754Sarybchik 217283514Sarybchik default: 218283514Sarybchik EFSYS_ASSERT(0); 219283514Sarybchik rc = ENOTSUP; 220283514Sarybchik goto fail3; 221283514Sarybchik } 222283514Sarybchik 223283514Sarybchik if ((rc = erxop->erxo_init(enp)) != 0) 224283514Sarybchik goto fail4; 225283514Sarybchik 226283514Sarybchik enp->en_erxop = erxop; 227283514Sarybchik enp->en_mod_flags |= EFX_MOD_RX; 228283514Sarybchik return (0); 229283514Sarybchik 230283514Sarybchikfail4: 231283514Sarybchik EFSYS_PROBE(fail4); 232283514Sarybchikfail3: 233283514Sarybchik EFSYS_PROBE(fail3); 234283514Sarybchikfail2: 235283514Sarybchik EFSYS_PROBE(fail2); 236283514Sarybchikfail1: 237291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 238283514Sarybchik 239283514Sarybchik enp->en_erxop = NULL; 240283514Sarybchik enp->en_mod_flags &= ~EFX_MOD_RX; 241283514Sarybchik return (rc); 242283514Sarybchik} 243283514Sarybchik 244283514Sarybchik void 245283514Sarybchikefx_rx_fini( 246283514Sarybchik __in efx_nic_t *enp) 247283514Sarybchik{ 248299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 249283514Sarybchik 250283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 251283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 252283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 253283514Sarybchik EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 254283514Sarybchik 255283514Sarybchik erxop->erxo_fini(enp); 256283514Sarybchik 257283514Sarybchik enp->en_erxop = NULL; 258283514Sarybchik enp->en_mod_flags &= ~EFX_MOD_RX; 259283514Sarybchik} 260283514Sarybchik 261283514Sarybchik#if EFSYS_OPT_RX_SCATTER 262291436Sarybchik __checkReturn efx_rc_t 263283514Sarybchikefx_rx_scatter_enable( 264283514Sarybchik __in efx_nic_t *enp, 265283514Sarybchik __in unsigned int buf_size) 266283514Sarybchik{ 267299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 268291436Sarybchik efx_rc_t rc; 269283514Sarybchik 270283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 271283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 272283514Sarybchik 273283514Sarybchik if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 274283514Sarybchik goto fail1; 275283514Sarybchik 276283514Sarybchik return (0); 277283514Sarybchik 278283514Sarybchikfail1: 279291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 280283514Sarybchik return (rc); 281283514Sarybchik} 282283514Sarybchik#endif /* EFSYS_OPT_RX_SCATTER */ 283283514Sarybchik 284283514Sarybchik#if EFSYS_OPT_RX_SCALE 285291436Sarybchik __checkReturn efx_rc_t 286283514Sarybchikefx_rx_hash_support_get( 287283514Sarybchik __in efx_nic_t *enp, 288283514Sarybchik __out efx_rx_hash_support_t *supportp) 289283514Sarybchik{ 290291436Sarybchik efx_rc_t rc; 291283514Sarybchik 292283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 293283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 294283514Sarybchik 295283514Sarybchik if (supportp == NULL) { 296283514Sarybchik rc = EINVAL; 297283514Sarybchik goto fail1; 298283514Sarybchik } 299283514Sarybchik 300283514Sarybchik /* Report if resources are available to insert RX hash value */ 301283514Sarybchik *supportp = enp->en_hash_support; 302283514Sarybchik 303283514Sarybchik return (0); 304283514Sarybchik 305283514Sarybchikfail1: 306291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 307283514Sarybchik 308283514Sarybchik return (rc); 309283514Sarybchik} 310283514Sarybchik 311291436Sarybchik __checkReturn efx_rc_t 312283514Sarybchikefx_rx_scale_support_get( 313283514Sarybchik __in efx_nic_t *enp, 314283514Sarybchik __out efx_rx_scale_support_t *supportp) 315283514Sarybchik{ 316291436Sarybchik efx_rc_t rc; 317283514Sarybchik 318283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 319283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 320283514Sarybchik 321283514Sarybchik if (supportp == NULL) { 322283514Sarybchik rc = EINVAL; 323283514Sarybchik goto fail1; 324283514Sarybchik } 325283514Sarybchik 326283514Sarybchik /* Report if resources are available to support RSS */ 327283514Sarybchik *supportp = enp->en_rss_support; 328283514Sarybchik 329283514Sarybchik return (0); 330283514Sarybchik 331283514Sarybchikfail1: 332291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 333283514Sarybchik 334283514Sarybchik return (rc); 335283514Sarybchik} 336283514Sarybchik 337291436Sarybchik __checkReturn efx_rc_t 338283514Sarybchikefx_rx_scale_mode_set( 339283514Sarybchik __in efx_nic_t *enp, 340283514Sarybchik __in efx_rx_hash_alg_t alg, 341283514Sarybchik __in efx_rx_hash_type_t type, 342283514Sarybchik __in boolean_t insert) 343283514Sarybchik{ 344299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 345291436Sarybchik efx_rc_t rc; 346283514Sarybchik 347283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 348283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 349283514Sarybchik 350283514Sarybchik if (erxop->erxo_scale_mode_set != NULL) { 351283514Sarybchik if ((rc = erxop->erxo_scale_mode_set(enp, alg, 352283514Sarybchik type, insert)) != 0) 353283514Sarybchik goto fail1; 354283514Sarybchik } 355283514Sarybchik 356283514Sarybchik return (0); 357283514Sarybchik 358283514Sarybchikfail1: 359291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 360283514Sarybchik return (rc); 361283514Sarybchik} 362283514Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 363283514Sarybchik 364283514Sarybchik#if EFSYS_OPT_RX_SCALE 365291436Sarybchik __checkReturn efx_rc_t 366283514Sarybchikefx_rx_scale_key_set( 367283514Sarybchik __in efx_nic_t *enp, 368283514Sarybchik __in_ecount(n) uint8_t *key, 369283514Sarybchik __in size_t n) 370283514Sarybchik{ 371299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 372291436Sarybchik efx_rc_t rc; 373283514Sarybchik 374283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 375283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 376283514Sarybchik 377283514Sarybchik if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0) 378283514Sarybchik goto fail1; 379283514Sarybchik 380283514Sarybchik return (0); 381283514Sarybchik 382283514Sarybchikfail1: 383291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 384283514Sarybchik 385283514Sarybchik return (rc); 386283514Sarybchik} 387283514Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 388283514Sarybchik 389283514Sarybchik#if EFSYS_OPT_RX_SCALE 390291436Sarybchik __checkReturn efx_rc_t 391283514Sarybchikefx_rx_scale_tbl_set( 392283514Sarybchik __in efx_nic_t *enp, 393283514Sarybchik __in_ecount(n) unsigned int *table, 394283514Sarybchik __in size_t n) 395283514Sarybchik{ 396299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 397291436Sarybchik efx_rc_t rc; 398283514Sarybchik 399283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 400283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 401283514Sarybchik 402283514Sarybchik if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0) 403283514Sarybchik goto fail1; 404283514Sarybchik 405283514Sarybchik return (0); 406283514Sarybchik 407283514Sarybchikfail1: 408291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 409283514Sarybchik 410283514Sarybchik return (rc); 411283514Sarybchik} 412283514Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 413283514Sarybchik 414283514Sarybchik void 415283514Sarybchikefx_rx_qpost( 416283514Sarybchik __in efx_rxq_t *erp, 417283514Sarybchik __in_ecount(n) efsys_dma_addr_t *addrp, 418283514Sarybchik __in size_t size, 419283514Sarybchik __in unsigned int n, 420283514Sarybchik __in unsigned int completed, 421283514Sarybchik __in unsigned int added) 422283514Sarybchik{ 423283514Sarybchik efx_nic_t *enp = erp->er_enp; 424299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 425283514Sarybchik 426283514Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 427283514Sarybchik 428283514Sarybchik erxop->erxo_qpost(erp, addrp, size, n, completed, added); 429283514Sarybchik} 430283514Sarybchik 431283514Sarybchik void 432283514Sarybchikefx_rx_qpush( 433283514Sarybchik __in efx_rxq_t *erp, 434283514Sarybchik __in unsigned int added, 435283514Sarybchik __inout unsigned int *pushedp) 436283514Sarybchik{ 437283514Sarybchik efx_nic_t *enp = erp->er_enp; 438299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 439283514Sarybchik 440283514Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 441283514Sarybchik 442283514Sarybchik erxop->erxo_qpush(erp, added, pushedp); 443283514Sarybchik} 444283514Sarybchik 445291436Sarybchik __checkReturn efx_rc_t 446283514Sarybchikefx_rx_qflush( 447283514Sarybchik __in efx_rxq_t *erp) 448283514Sarybchik{ 449283514Sarybchik efx_nic_t *enp = erp->er_enp; 450299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 451291436Sarybchik efx_rc_t rc; 452283514Sarybchik 453283514Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 454283514Sarybchik 455283514Sarybchik if ((rc = erxop->erxo_qflush(erp)) != 0) 456283514Sarybchik goto fail1; 457283514Sarybchik 458283514Sarybchik return (0); 459283514Sarybchik 460283514Sarybchikfail1: 461291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 462283514Sarybchik 463283514Sarybchik return (rc); 464283514Sarybchik} 465283514Sarybchik 466283514Sarybchik void 467283514Sarybchikefx_rx_qenable( 468283514Sarybchik __in efx_rxq_t *erp) 469283514Sarybchik{ 470283514Sarybchik efx_nic_t *enp = erp->er_enp; 471299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 472283514Sarybchik 473283514Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 474283514Sarybchik 475283514Sarybchik erxop->erxo_qenable(erp); 476283514Sarybchik} 477283514Sarybchik 478291436Sarybchik __checkReturn efx_rc_t 479283514Sarybchikefx_rx_qcreate( 480283514Sarybchik __in efx_nic_t *enp, 481283514Sarybchik __in unsigned int index, 482283514Sarybchik __in unsigned int label, 483283514Sarybchik __in efx_rxq_type_t type, 484283514Sarybchik __in efsys_mem_t *esmp, 485283514Sarybchik __in size_t n, 486283514Sarybchik __in uint32_t id, 487283514Sarybchik __in efx_evq_t *eep, 488283514Sarybchik __deref_out efx_rxq_t **erpp) 489283514Sarybchik{ 490299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 491283514Sarybchik efx_rxq_t *erp; 492291436Sarybchik efx_rc_t rc; 493283514Sarybchik 494283514Sarybchik EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 495283514Sarybchik EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 496283514Sarybchik 497283514Sarybchik /* Allocate an RXQ object */ 498283514Sarybchik EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 499283514Sarybchik 500283514Sarybchik if (erp == NULL) { 501283514Sarybchik rc = ENOMEM; 502283514Sarybchik goto fail1; 503283514Sarybchik } 504283514Sarybchik 505283514Sarybchik erp->er_magic = EFX_RXQ_MAGIC; 506283514Sarybchik erp->er_enp = enp; 507283514Sarybchik erp->er_index = index; 508283514Sarybchik erp->er_mask = n - 1; 509283514Sarybchik erp->er_esmp = esmp; 510283514Sarybchik 511283514Sarybchik if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id, 512283514Sarybchik eep, erp)) != 0) 513283514Sarybchik goto fail2; 514283514Sarybchik 515283514Sarybchik enp->en_rx_qcount++; 516283514Sarybchik *erpp = erp; 517283514Sarybchik 518283514Sarybchik return (0); 519283514Sarybchik 520283514Sarybchikfail2: 521283514Sarybchik EFSYS_PROBE(fail2); 522283514Sarybchik 523283514Sarybchik EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 524283514Sarybchikfail1: 525291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 526283514Sarybchik 527283514Sarybchik return (rc); 528283514Sarybchik} 529283514Sarybchik 530283514Sarybchik void 531283514Sarybchikefx_rx_qdestroy( 532283514Sarybchik __in efx_rxq_t *erp) 533283514Sarybchik{ 534283514Sarybchik efx_nic_t *enp = erp->er_enp; 535299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 536283514Sarybchik 537283514Sarybchik EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 538283514Sarybchik 539283514Sarybchik erxop->erxo_qdestroy(erp); 540283514Sarybchik} 541283514Sarybchik 542291436Sarybchik __checkReturn efx_rc_t 543283514Sarybchikefx_psuedo_hdr_pkt_length_get( 544283514Sarybchik __in efx_nic_t *enp, 545283514Sarybchik __in uint8_t *buffer, 546293807Sarybchik __out uint16_t *lengthp) 547283514Sarybchik{ 548299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 549283514Sarybchik 550293807Sarybchik return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 551283514Sarybchik} 552283514Sarybchik 553283514Sarybchik#if EFSYS_OPT_RX_SCALE 554293807Sarybchik __checkReturn uint32_t 555283514Sarybchikefx_psuedo_hdr_hash_get( 556283514Sarybchik __in efx_nic_t *enp, 557283514Sarybchik __in efx_rx_hash_alg_t func, 558283514Sarybchik __in uint8_t *buffer) 559283514Sarybchik{ 560299517Sarybchik const efx_rx_ops_t *erxop = enp->en_erxop; 561293807Sarybchik 562293807Sarybchik EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 563293807Sarybchik return (erxop->erxo_prefix_hash(enp, func, buffer)); 564283514Sarybchik} 565283514Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 566283514Sarybchik 567299320Sarybchik#if EFSYS_OPT_SIENA 568283514Sarybchik 569291436Sarybchikstatic __checkReturn efx_rc_t 570283514Sarybchikfalconsiena_rx_init( 571283514Sarybchik __in efx_nic_t *enp) 572283514Sarybchik{ 573283514Sarybchik efx_oword_t oword; 574283514Sarybchik unsigned int index; 575283514Sarybchik 576227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 577227569Sphilip 578227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 579227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 580227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 581227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 582227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 583227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 584227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 585227569Sphilip 586227569Sphilip /* Zero the RSS table */ 587227569Sphilip for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 588227569Sphilip index++) { 589227569Sphilip EFX_ZERO_OWORD(oword); 590227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 591283514Sarybchik index, &oword, B_TRUE); 592227569Sphilip } 593227569Sphilip 594283514Sarybchik#if EFSYS_OPT_RX_SCALE 595283514Sarybchik /* The RSS key and indirection table are writable. */ 596283514Sarybchik enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 597227569Sphilip 598283514Sarybchik /* Hardware can insert RX hash with/without RSS */ 599283514Sarybchik enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 600283514Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 601227569Sphilip 602283514Sarybchik return (0); 603227569Sphilip} 604227569Sphilip 605227569Sphilip#if EFSYS_OPT_RX_SCATTER 606291436Sarybchikstatic __checkReturn efx_rc_t 607283514Sarybchikfalconsiena_rx_scatter_enable( 608227569Sphilip __in efx_nic_t *enp, 609227569Sphilip __in unsigned int buf_size) 610227569Sphilip{ 611227569Sphilip unsigned int nbuf32; 612227569Sphilip efx_oword_t oword; 613291436Sarybchik efx_rc_t rc; 614227569Sphilip 615227569Sphilip nbuf32 = buf_size / 32; 616227569Sphilip if ((nbuf32 == 0) || 617227569Sphilip (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 618227569Sphilip ((buf_size % 32) != 0)) { 619227569Sphilip rc = EINVAL; 620227569Sphilip goto fail1; 621227569Sphilip } 622227569Sphilip 623227569Sphilip if (enp->en_rx_qcount > 0) { 624227569Sphilip rc = EBUSY; 625227569Sphilip goto fail2; 626227569Sphilip } 627227569Sphilip 628227569Sphilip /* Set scatter buffer size */ 629227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 630227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 631227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 632227569Sphilip 633227569Sphilip /* Enable scatter for packets not matching a filter */ 634227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 635227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 636227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 637227569Sphilip 638227569Sphilip return (0); 639227569Sphilip 640227569Sphilipfail2: 641227569Sphilip EFSYS_PROBE(fail2); 642227569Sphilipfail1: 643291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 644227569Sphilip 645227569Sphilip return (rc); 646227569Sphilip} 647227569Sphilip#endif /* EFSYS_OPT_RX_SCATTER */ 648227569Sphilip 649227569Sphilip 650227569Sphilip#define EFX_RX_LFSR_HASH(_enp, _insert) \ 651283514Sarybchik do { \ 652227569Sphilip efx_oword_t oword; \ 653227569Sphilip \ 654227569Sphilip EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 655227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 656227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 657227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 658227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 659227569Sphilip (_insert) ? 1 : 0); \ 660227569Sphilip EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 661227569Sphilip \ 662227569Sphilip if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 663227569Sphilip EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 664227569Sphilip &oword); \ 665227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 666227569Sphilip FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 667227569Sphilip EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 668227569Sphilip &oword); \ 669227569Sphilip } \ 670227569Sphilip \ 671227569Sphilip _NOTE(CONSTANTCONDITION) \ 672227569Sphilip } while (B_FALSE) 673227569Sphilip 674227569Sphilip#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 675283514Sarybchik do { \ 676227569Sphilip efx_oword_t oword; \ 677227569Sphilip \ 678227569Sphilip EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 679227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 680227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 681227569Sphilip (_ip) ? 1 : 0); \ 682227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 683227569Sphilip (_tcp) ? 0 : 1); \ 684227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 685227569Sphilip (_insert) ? 1 : 0); \ 686227569Sphilip EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 687227569Sphilip \ 688227569Sphilip _NOTE(CONSTANTCONDITION) \ 689227569Sphilip } while (B_FALSE) 690227569Sphilip 691227569Sphilip#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 692283514Sarybchik do { \ 693227569Sphilip efx_oword_t oword; \ 694227569Sphilip \ 695227569Sphilip EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 696227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 697227569Sphilip FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 698227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 699227569Sphilip FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 700227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 701227569Sphilip FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 702227569Sphilip EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 703227569Sphilip \ 704227569Sphilip (_rc) = 0; \ 705227569Sphilip \ 706227569Sphilip _NOTE(CONSTANTCONDITION) \ 707227569Sphilip } while (B_FALSE) 708227569Sphilip 709227569Sphilip 710227569Sphilip#if EFSYS_OPT_RX_SCALE 711283514Sarybchik 712291436Sarybchikstatic __checkReturn efx_rc_t 713283514Sarybchikfalconsiena_rx_scale_mode_set( 714227569Sphilip __in efx_nic_t *enp, 715227569Sphilip __in efx_rx_hash_alg_t alg, 716227569Sphilip __in efx_rx_hash_type_t type, 717227569Sphilip __in boolean_t insert) 718227569Sphilip{ 719291436Sarybchik efx_rc_t rc; 720227569Sphilip 721227569Sphilip switch (alg) { 722227569Sphilip case EFX_RX_HASHALG_LFSR: 723227569Sphilip EFX_RX_LFSR_HASH(enp, insert); 724227569Sphilip break; 725227569Sphilip 726227569Sphilip case EFX_RX_HASHALG_TOEPLITZ: 727227569Sphilip EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 728227569Sphilip type & (1 << EFX_RX_HASH_IPV4), 729227569Sphilip type & (1 << EFX_RX_HASH_TCPIPV4)); 730227569Sphilip 731227569Sphilip EFX_RX_TOEPLITZ_IPV6_HASH(enp, 732227569Sphilip type & (1 << EFX_RX_HASH_IPV6), 733227569Sphilip type & (1 << EFX_RX_HASH_TCPIPV6), 734227569Sphilip rc); 735227569Sphilip if (rc != 0) 736227569Sphilip goto fail1; 737227569Sphilip 738227569Sphilip break; 739227569Sphilip 740227569Sphilip default: 741227569Sphilip rc = EINVAL; 742227569Sphilip goto fail2; 743227569Sphilip } 744227569Sphilip 745227569Sphilip return (0); 746227569Sphilip 747227569Sphilipfail2: 748227569Sphilip EFSYS_PROBE(fail2); 749227569Sphilipfail1: 750291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 751227569Sphilip 752227569Sphilip EFX_RX_LFSR_HASH(enp, B_FALSE); 753227569Sphilip 754227569Sphilip return (rc); 755227569Sphilip} 756227569Sphilip#endif 757227569Sphilip 758227569Sphilip#if EFSYS_OPT_RX_SCALE 759291436Sarybchikstatic __checkReturn efx_rc_t 760283514Sarybchikfalconsiena_rx_scale_key_set( 761227569Sphilip __in efx_nic_t *enp, 762227569Sphilip __in_ecount(n) uint8_t *key, 763227569Sphilip __in size_t n) 764227569Sphilip{ 765227569Sphilip efx_oword_t oword; 766227569Sphilip unsigned int byte; 767227569Sphilip unsigned int offset; 768291436Sarybchik efx_rc_t rc; 769227569Sphilip 770227569Sphilip byte = 0; 771227569Sphilip 772283514Sarybchik /* Write Toeplitz IPv4 hash key */ 773227569Sphilip EFX_ZERO_OWORD(oword); 774227569Sphilip for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 775227569Sphilip offset > 0 && byte < n; 776227569Sphilip --offset) 777227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 778227569Sphilip 779227569Sphilip EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 780227569Sphilip 781227569Sphilip byte = 0; 782227569Sphilip 783283514Sarybchik /* Verify Toeplitz IPv4 hash key */ 784227569Sphilip EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 785227569Sphilip for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 786227569Sphilip offset > 0 && byte < n; 787227569Sphilip --offset) { 788227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 789227569Sphilip rc = EFAULT; 790227569Sphilip goto fail1; 791227569Sphilip } 792227569Sphilip } 793227569Sphilip 794283514Sarybchik if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 795283514Sarybchik goto done; 796227569Sphilip 797227569Sphilip byte = 0; 798227569Sphilip 799283514Sarybchik /* Write Toeplitz IPv6 hash key 3 */ 800227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 801227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 802227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 803227569Sphilip offset > 0 && byte < n; 804227569Sphilip --offset) 805227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 806227569Sphilip 807227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 808227569Sphilip 809283514Sarybchik /* Write Toeplitz IPv6 hash key 2 */ 810227569Sphilip EFX_ZERO_OWORD(oword); 811227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 812227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 813227569Sphilip offset > 0 && byte < n; 814227569Sphilip --offset) 815227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 816227569Sphilip 817227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 818227569Sphilip 819283514Sarybchik /* Write Toeplitz IPv6 hash key 1 */ 820227569Sphilip EFX_ZERO_OWORD(oword); 821227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 822227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 823227569Sphilip offset > 0 && byte < n; 824227569Sphilip --offset) 825227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 826227569Sphilip 827227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 828227569Sphilip 829227569Sphilip byte = 0; 830227569Sphilip 831283514Sarybchik /* Verify Toeplitz IPv6 hash key 3 */ 832227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 833227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 834227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 835227569Sphilip offset > 0 && byte < n; 836227569Sphilip --offset) { 837227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 838227569Sphilip rc = EFAULT; 839283514Sarybchik goto fail2; 840227569Sphilip } 841227569Sphilip } 842227569Sphilip 843283514Sarybchik /* Verify Toeplitz IPv6 hash key 2 */ 844227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 845227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 846227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 847227569Sphilip offset > 0 && byte < n; 848227569Sphilip --offset) { 849227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 850227569Sphilip rc = EFAULT; 851283514Sarybchik goto fail3; 852227569Sphilip } 853227569Sphilip } 854227569Sphilip 855283514Sarybchik /* Verify Toeplitz IPv6 hash key 1 */ 856227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 857227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 858227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 859227569Sphilip offset > 0 && byte < n; 860227569Sphilip --offset) { 861227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 862227569Sphilip rc = EFAULT; 863283514Sarybchik goto fail4; 864227569Sphilip } 865227569Sphilip } 866227569Sphilip 867283514Sarybchikdone: 868227569Sphilip return (0); 869227569Sphilip 870283514Sarybchikfail4: 871283514Sarybchik EFSYS_PROBE(fail4); 872227569Sphilipfail3: 873227569Sphilip EFSYS_PROBE(fail3); 874227569Sphilipfail2: 875227569Sphilip EFSYS_PROBE(fail2); 876227569Sphilipfail1: 877291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 878227569Sphilip 879227569Sphilip return (rc); 880227569Sphilip} 881227569Sphilip#endif 882227569Sphilip 883227569Sphilip#if EFSYS_OPT_RX_SCALE 884291436Sarybchikstatic __checkReturn efx_rc_t 885283514Sarybchikfalconsiena_rx_scale_tbl_set( 886227569Sphilip __in efx_nic_t *enp, 887227569Sphilip __in_ecount(n) unsigned int *table, 888227569Sphilip __in size_t n) 889227569Sphilip{ 890227569Sphilip efx_oword_t oword; 891227569Sphilip int index; 892291436Sarybchik efx_rc_t rc; 893227569Sphilip 894227569Sphilip EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 895227569Sphilip EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 896227569Sphilip 897227569Sphilip if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 898227569Sphilip rc = EINVAL; 899227569Sphilip goto fail1; 900227569Sphilip } 901227569Sphilip 902227569Sphilip for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 903227569Sphilip uint32_t byte; 904227569Sphilip 905227569Sphilip /* Calculate the entry to place in the table */ 906283514Sarybchik byte = (n > 0) ? (uint32_t)table[index % n] : 0; 907227569Sphilip 908227569Sphilip EFSYS_PROBE2(table, int, index, uint32_t, byte); 909227569Sphilip 910227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 911227569Sphilip 912227569Sphilip /* Write the table */ 913227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 914283514Sarybchik index, &oword, B_TRUE); 915227569Sphilip } 916227569Sphilip 917227569Sphilip for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 918227569Sphilip uint32_t byte; 919227569Sphilip 920227569Sphilip /* Determine if we're starting a new batch */ 921283514Sarybchik byte = (n > 0) ? (uint32_t)table[index % n] : 0; 922227569Sphilip 923227569Sphilip /* Read the table */ 924227569Sphilip EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 925283514Sarybchik index, &oword, B_TRUE); 926227569Sphilip 927227569Sphilip /* Verify the entry */ 928227569Sphilip if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 929227569Sphilip rc = EFAULT; 930227569Sphilip goto fail2; 931227569Sphilip } 932227569Sphilip } 933227569Sphilip 934227569Sphilip return (0); 935227569Sphilip 936227569Sphilipfail2: 937227569Sphilip EFSYS_PROBE(fail2); 938227569Sphilipfail1: 939291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 940227569Sphilip 941227569Sphilip return (rc); 942227569Sphilip} 943227569Sphilip#endif 944227569Sphilip 945293807Sarybchik/* 946293807Sarybchik * Falcon/Siena psuedo-header 947293807Sarybchik * -------------------------- 948293807Sarybchik * 949293807Sarybchik * Receive packets are prefixed by an optional 16 byte pseudo-header. 950293807Sarybchik * The psuedo-header is a byte array of one of the forms: 951293807Sarybchik * 952293807Sarybchik * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 953293807Sarybchik * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 954293807Sarybchik * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 955293807Sarybchik * 956293807Sarybchik * where: 957293807Sarybchik * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 958293807Sarybchik * LL.LL LFSR hash (16-bit big-endian) 959293807Sarybchik */ 960293807Sarybchik 961293807Sarybchik#if EFSYS_OPT_RX_SCALE 962293807Sarybchikstatic __checkReturn uint32_t 963293807Sarybchikfalconsiena_rx_prefix_hash( 964293807Sarybchik __in efx_nic_t *enp, 965293807Sarybchik __in efx_rx_hash_alg_t func, 966293807Sarybchik __in uint8_t *buffer) 967293807Sarybchik{ 968293807Sarybchik switch (func) { 969293807Sarybchik case EFX_RX_HASHALG_TOEPLITZ: 970293807Sarybchik return ((buffer[12] << 24) | 971293807Sarybchik (buffer[13] << 16) | 972293807Sarybchik (buffer[14] << 8) | 973293807Sarybchik buffer[15]); 974293807Sarybchik 975293807Sarybchik case EFX_RX_HASHALG_LFSR: 976293807Sarybchik return ((buffer[14] << 8) | buffer[15]); 977293807Sarybchik 978293807Sarybchik default: 979293807Sarybchik EFSYS_ASSERT(0); 980293807Sarybchik return (0); 981293807Sarybchik } 982293807Sarybchik} 983293807Sarybchik#endif /* EFSYS_OPT_RX_SCALE */ 984293807Sarybchik 985293807Sarybchikstatic __checkReturn efx_rc_t 986293807Sarybchikfalconsiena_rx_prefix_pktlen( 987293807Sarybchik __in efx_nic_t *enp, 988293807Sarybchik __in uint8_t *buffer, 989293807Sarybchik __out uint16_t *lengthp) 990293807Sarybchik{ 991293807Sarybchik /* Not supported by Falcon/Siena hardware */ 992293807Sarybchik EFSYS_ASSERT(0); 993293807Sarybchik return (ENOTSUP); 994293807Sarybchik} 995293807Sarybchik 996293807Sarybchik 997283514Sarybchikstatic void 998283514Sarybchikfalconsiena_rx_qpost( 999227569Sphilip __in efx_rxq_t *erp, 1000227569Sphilip __in_ecount(n) efsys_dma_addr_t *addrp, 1001227569Sphilip __in size_t size, 1002227569Sphilip __in unsigned int n, 1003227569Sphilip __in unsigned int completed, 1004227569Sphilip __in unsigned int added) 1005227569Sphilip{ 1006227569Sphilip efx_qword_t qword; 1007227569Sphilip unsigned int i; 1008227569Sphilip unsigned int offset; 1009227569Sphilip unsigned int id; 1010227569Sphilip 1011227569Sphilip /* The client driver must not overfill the queue */ 1012227569Sphilip EFSYS_ASSERT3U(added - completed + n, <=, 1013227569Sphilip EFX_RXQ_LIMIT(erp->er_mask + 1)); 1014227569Sphilip 1015227569Sphilip id = added & (erp->er_mask); 1016227569Sphilip for (i = 0; i < n; i++) { 1017227569Sphilip EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1018227569Sphilip unsigned int, id, efsys_dma_addr_t, addrp[i], 1019227569Sphilip size_t, size); 1020227569Sphilip 1021227569Sphilip EFX_POPULATE_QWORD_3(qword, 1022227569Sphilip FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1023227569Sphilip FSF_AZ_RX_KER_BUF_ADDR_DW0, 1024227569Sphilip (uint32_t)(addrp[i] & 0xffffffff), 1025227569Sphilip FSF_AZ_RX_KER_BUF_ADDR_DW1, 1026227569Sphilip (uint32_t)(addrp[i] >> 32)); 1027227569Sphilip 1028227569Sphilip offset = id * sizeof (efx_qword_t); 1029227569Sphilip EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1030227569Sphilip 1031227569Sphilip id = (id + 1) & (erp->er_mask); 1032227569Sphilip } 1033227569Sphilip} 1034227569Sphilip 1035283514Sarybchikstatic void 1036283514Sarybchikfalconsiena_rx_qpush( 1037227569Sphilip __in efx_rxq_t *erp, 1038283514Sarybchik __in unsigned int added, 1039283514Sarybchik __inout unsigned int *pushedp) 1040227569Sphilip{ 1041227569Sphilip efx_nic_t *enp = erp->er_enp; 1042283514Sarybchik unsigned int pushed = *pushedp; 1043227569Sphilip uint32_t wptr; 1044227569Sphilip efx_oword_t oword; 1045227569Sphilip efx_dword_t dword; 1046227569Sphilip 1047283514Sarybchik /* All descriptors are pushed */ 1048283514Sarybchik *pushedp = added; 1049227569Sphilip 1050227569Sphilip /* Push the populated descriptors out */ 1051227569Sphilip wptr = added & erp->er_mask; 1052227569Sphilip 1053227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1054227569Sphilip 1055227569Sphilip /* Only write the third DWORD */ 1056227569Sphilip EFX_POPULATE_DWORD_1(dword, 1057227569Sphilip EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1058283514Sarybchik 1059283514Sarybchik /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1060283514Sarybchik EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1061283514Sarybchik wptr, pushed & erp->er_mask); 1062283514Sarybchik EFSYS_PIO_WRITE_BARRIER(); 1063227569Sphilip EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1064227569Sphilip erp->er_index, &dword, B_FALSE); 1065227569Sphilip} 1066227569Sphilip 1067291436Sarybchikstatic __checkReturn efx_rc_t 1068283514Sarybchikfalconsiena_rx_qflush( 1069227569Sphilip __in efx_rxq_t *erp) 1070227569Sphilip{ 1071227569Sphilip efx_nic_t *enp = erp->er_enp; 1072227569Sphilip efx_oword_t oword; 1073227569Sphilip uint32_t label; 1074227569Sphilip 1075227569Sphilip label = erp->er_index; 1076227569Sphilip 1077227569Sphilip /* Flush the queue */ 1078227569Sphilip EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1079227569Sphilip FRF_AZ_RX_FLUSH_DESCQ, label); 1080227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1081283514Sarybchik 1082283514Sarybchik return (0); 1083227569Sphilip} 1084227569Sphilip 1085283514Sarybchikstatic void 1086283514Sarybchikfalconsiena_rx_qenable( 1087227569Sphilip __in efx_rxq_t *erp) 1088227569Sphilip{ 1089227569Sphilip efx_nic_t *enp = erp->er_enp; 1090227569Sphilip efx_oword_t oword; 1091227569Sphilip 1092227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1093227569Sphilip 1094227569Sphilip EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1095283514Sarybchik erp->er_index, &oword, B_TRUE); 1096227569Sphilip 1097227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1098227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1099227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1100227569Sphilip 1101227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1102283514Sarybchik erp->er_index, &oword, B_TRUE); 1103227569Sphilip} 1104227569Sphilip 1105291436Sarybchikstatic __checkReturn efx_rc_t 1106283514Sarybchikfalconsiena_rx_qcreate( 1107227569Sphilip __in efx_nic_t *enp, 1108227569Sphilip __in unsigned int index, 1109227569Sphilip __in unsigned int label, 1110227569Sphilip __in efx_rxq_type_t type, 1111227569Sphilip __in efsys_mem_t *esmp, 1112227569Sphilip __in size_t n, 1113227569Sphilip __in uint32_t id, 1114227569Sphilip __in efx_evq_t *eep, 1115283514Sarybchik __in efx_rxq_t *erp) 1116227569Sphilip{ 1117227569Sphilip efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1118227569Sphilip efx_oword_t oword; 1119227569Sphilip uint32_t size; 1120227569Sphilip boolean_t jumbo; 1121291436Sarybchik efx_rc_t rc; 1122227569Sphilip 1123278839Sarybchik EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1124278839Sarybchik (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1125227569Sphilip EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1126227569Sphilip EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1127227569Sphilip 1128283514Sarybchik EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1129283514Sarybchik EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1130283514Sarybchik 1131283514Sarybchik if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 1132227569Sphilip rc = EINVAL; 1133227569Sphilip goto fail1; 1134227569Sphilip } 1135227569Sphilip if (index >= encp->enc_rxq_limit) { 1136227569Sphilip rc = EINVAL; 1137227569Sphilip goto fail2; 1138227569Sphilip } 1139227569Sphilip for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1140227569Sphilip size++) 1141227569Sphilip if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 1142227569Sphilip break; 1143227569Sphilip if (id + (1 << size) >= encp->enc_buftbl_limit) { 1144227569Sphilip rc = EINVAL; 1145227569Sphilip goto fail3; 1146227569Sphilip } 1147227569Sphilip 1148227569Sphilip switch (type) { 1149227569Sphilip case EFX_RXQ_TYPE_DEFAULT: 1150227569Sphilip jumbo = B_FALSE; 1151227569Sphilip break; 1152227569Sphilip 1153227569Sphilip#if EFSYS_OPT_RX_SCATTER 1154227569Sphilip case EFX_RXQ_TYPE_SCATTER: 1155227569Sphilip if (enp->en_family < EFX_FAMILY_SIENA) { 1156227569Sphilip rc = EINVAL; 1157227569Sphilip goto fail4; 1158227569Sphilip } 1159227569Sphilip jumbo = B_TRUE; 1160227569Sphilip break; 1161227569Sphilip#endif /* EFSYS_OPT_RX_SCATTER */ 1162227569Sphilip 1163227569Sphilip default: 1164227569Sphilip rc = EINVAL; 1165227569Sphilip goto fail4; 1166227569Sphilip } 1167227569Sphilip 1168227569Sphilip /* Set up the new descriptor queue */ 1169293773Sarybchik EFX_POPULATE_OWORD_7(oword, 1170227569Sphilip FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1171227569Sphilip FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1172227569Sphilip FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1173227569Sphilip FRF_AZ_RX_DESCQ_LABEL, label, 1174227569Sphilip FRF_AZ_RX_DESCQ_SIZE, size, 1175227569Sphilip FRF_AZ_RX_DESCQ_TYPE, 0, 1176227569Sphilip FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1177227569Sphilip 1178227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1179283514Sarybchik erp->er_index, &oword, B_TRUE); 1180227569Sphilip 1181227569Sphilip return (0); 1182227569Sphilip 1183227569Sphilipfail4: 1184227569Sphilip EFSYS_PROBE(fail4); 1185227569Sphilipfail3: 1186227569Sphilip EFSYS_PROBE(fail3); 1187227569Sphilipfail2: 1188227569Sphilip EFSYS_PROBE(fail2); 1189227569Sphilipfail1: 1190291436Sarybchik EFSYS_PROBE1(fail1, efx_rc_t, rc); 1191227569Sphilip 1192227569Sphilip return (rc); 1193227569Sphilip} 1194227569Sphilip 1195283514Sarybchikstatic void 1196283514Sarybchikfalconsiena_rx_qdestroy( 1197227569Sphilip __in efx_rxq_t *erp) 1198227569Sphilip{ 1199227569Sphilip efx_nic_t *enp = erp->er_enp; 1200227569Sphilip efx_oword_t oword; 1201227569Sphilip 1202227569Sphilip EFSYS_ASSERT(enp->en_rx_qcount != 0); 1203227569Sphilip --enp->en_rx_qcount; 1204227569Sphilip 1205227569Sphilip /* Purge descriptor queue */ 1206227569Sphilip EFX_ZERO_OWORD(oword); 1207227569Sphilip 1208227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1209283514Sarybchik erp->er_index, &oword, B_TRUE); 1210227569Sphilip 1211227569Sphilip /* Free the RXQ object */ 1212227569Sphilip EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1213227569Sphilip} 1214227569Sphilip 1215283514Sarybchikstatic void 1216283514Sarybchikfalconsiena_rx_fini( 1217227569Sphilip __in efx_nic_t *enp) 1218227569Sphilip{ 1219283514Sarybchik _NOTE(ARGUNUSED(enp)) 1220283514Sarybchik} 1221227569Sphilip 1222299320Sarybchik#endif /* EFSYS_OPT_SIENA */ 1223