1227569Sphilip/*- 2227569Sphilip * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved. 3227569Sphilip * 4227569Sphilip * Redistribution and use in source and binary forms, with or without 5227569Sphilip * modification, are permitted provided that the following conditions 6227569Sphilip * are met: 7227569Sphilip * 1. Redistributions of source code must retain the above copyright 8227569Sphilip * notice, this list of conditions and the following disclaimer. 9227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 10227569Sphilip * notice, this list of conditions and the following disclaimer in the 11227569Sphilip * documentation and/or other materials provided with the distribution. 12227569Sphilip * 13227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND 14227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23227569Sphilip * SUCH DAMAGE. 24227569Sphilip */ 25227569Sphilip 26228100Sphilip#include <sys/cdefs.h> 27228100Sphilip__FBSDID("$FreeBSD$"); 28228100Sphilip 29227569Sphilip#include "efsys.h" 30227569Sphilip#include "efx.h" 31227569Sphilip#include "efx_types.h" 32227569Sphilip#include "efx_regs.h" 33227569Sphilip#include "efx_impl.h" 34227569Sphilip 35227569Sphilip __checkReturn int 36227569Sphilipefx_rx_init( 37227569Sphilip __in efx_nic_t *enp) 38227569Sphilip{ 39227569Sphilip efx_oword_t oword; 40227569Sphilip unsigned int index; 41227569Sphilip int rc; 42227569Sphilip 43227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 44227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 45227569Sphilip 46227569Sphilip if (!(enp->en_mod_flags & EFX_MOD_EV)) { 47227569Sphilip rc = EINVAL; 48227569Sphilip goto fail1; 49227569Sphilip } 50227569Sphilip 51227569Sphilip if (enp->en_mod_flags & EFX_MOD_RX) { 52227569Sphilip rc = EINVAL; 53227569Sphilip goto fail2; 54227569Sphilip } 55227569Sphilip 56227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 57227569Sphilip 58227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 59227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 60227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 61227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 62227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 63227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 64227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 65227569Sphilip 66227569Sphilip /* Zero the RSS table */ 67227569Sphilip for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 68227569Sphilip index++) { 69227569Sphilip EFX_ZERO_OWORD(oword); 70227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 71227569Sphilip index, &oword); 72227569Sphilip } 73227569Sphilip 74227569Sphilip enp->en_mod_flags |= EFX_MOD_RX; 75227569Sphilip return (0); 76227569Sphilip 77227569Sphilipfail2: 78227569Sphilip EFSYS_PROBE(fail2); 79227569Sphilipfail1: 80227569Sphilip EFSYS_PROBE1(fail1, int, rc); 81227569Sphilip 82227569Sphilip return (rc); 83227569Sphilip} 84227569Sphilip 85227569Sphilip#if EFSYS_OPT_RX_HDR_SPLIT 86227569Sphilip __checkReturn int 87227569Sphilipefx_rx_hdr_split_enable( 88227569Sphilip __in efx_nic_t *enp, 89227569Sphilip __in unsigned int hdr_buf_size, 90227569Sphilip __in unsigned int pld_buf_size) 91227569Sphilip{ 92227569Sphilip unsigned int nhdr32; 93227569Sphilip unsigned int npld32; 94227569Sphilip efx_oword_t oword; 95227569Sphilip int rc; 96227569Sphilip 97227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 98227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 99227569Sphilip EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA); 100227569Sphilip 101227569Sphilip nhdr32 = hdr_buf_size / 32; 102227569Sphilip if ((nhdr32 == 0) || 103227569Sphilip (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) || 104227569Sphilip ((hdr_buf_size % 32) != 0)) { 105227569Sphilip rc = EINVAL; 106227569Sphilip goto fail1; 107227569Sphilip } 108227569Sphilip 109227569Sphilip npld32 = pld_buf_size / 32; 110227569Sphilip if ((npld32 == 0) || 111227569Sphilip (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) || 112227569Sphilip ((pld_buf_size % 32) != 0)) { 113227569Sphilip rc = EINVAL; 114227569Sphilip goto fail2; 115227569Sphilip } 116227569Sphilip 117227569Sphilip if (enp->en_rx_qcount > 0) { 118227569Sphilip rc = EBUSY; 119227569Sphilip goto fail3; 120227569Sphilip } 121227569Sphilip 122227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 123227569Sphilip 124227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1); 125227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32); 126227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32); 127227569Sphilip 128227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 129227569Sphilip 130227569Sphilip return (0); 131227569Sphilip 132227569Sphilipfail3: 133227569Sphilip EFSYS_PROBE(fail3); 134227569Sphilipfail2: 135227569Sphilip EFSYS_PROBE(fail2); 136227569Sphilipfail1: 137227569Sphilip EFSYS_PROBE1(fail1, int, rc); 138227569Sphilip 139227569Sphilip return (rc); 140227569Sphilip} 141227569Sphilip#endif /* EFSYS_OPT_RX_HDR_SPLIT */ 142227569Sphilip 143227569Sphilip 144227569Sphilip#if EFSYS_OPT_RX_SCATTER 145227569Sphilip __checkReturn int 146227569Sphilipefx_rx_scatter_enable( 147227569Sphilip __in efx_nic_t *enp, 148227569Sphilip __in unsigned int buf_size) 149227569Sphilip{ 150227569Sphilip unsigned int nbuf32; 151227569Sphilip efx_oword_t oword; 152227569Sphilip int rc; 153227569Sphilip 154227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 155227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 156227569Sphilip EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON); 157227569Sphilip 158227569Sphilip nbuf32 = buf_size / 32; 159227569Sphilip if ((nbuf32 == 0) || 160227569Sphilip (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 161227569Sphilip ((buf_size % 32) != 0)) { 162227569Sphilip rc = EINVAL; 163227569Sphilip goto fail1; 164227569Sphilip } 165227569Sphilip 166227569Sphilip if (enp->en_rx_qcount > 0) { 167227569Sphilip rc = EBUSY; 168227569Sphilip goto fail2; 169227569Sphilip } 170227569Sphilip 171227569Sphilip /* Set scatter buffer size */ 172227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 173227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 174227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 175227569Sphilip 176227569Sphilip /* Enable scatter for packets not matching a filter */ 177227569Sphilip EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 178227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 179227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 180227569Sphilip 181227569Sphilip return (0); 182227569Sphilip 183227569Sphilipfail2: 184227569Sphilip EFSYS_PROBE(fail2); 185227569Sphilipfail1: 186227569Sphilip EFSYS_PROBE1(fail1, int, rc); 187227569Sphilip 188227569Sphilip return (rc); 189227569Sphilip} 190227569Sphilip#endif /* EFSYS_OPT_RX_SCATTER */ 191227569Sphilip 192227569Sphilip 193227569Sphilip#define EFX_RX_LFSR_HASH(_enp, _insert) \ 194227569Sphilip do { \ 195227569Sphilip efx_oword_t oword; \ 196227569Sphilip \ 197227569Sphilip EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 198227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 199227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 200227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 201227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 202227569Sphilip (_insert) ? 1 : 0); \ 203227569Sphilip EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 204227569Sphilip \ 205227569Sphilip if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 206227569Sphilip EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 207227569Sphilip &oword); \ 208227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 209227569Sphilip FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 210227569Sphilip EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 211227569Sphilip &oword); \ 212227569Sphilip } \ 213227569Sphilip \ 214227569Sphilip _NOTE(CONSTANTCONDITION) \ 215227569Sphilip } while (B_FALSE) 216227569Sphilip 217227569Sphilip#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 218227569Sphilip do { \ 219227569Sphilip efx_oword_t oword; \ 220227569Sphilip \ 221227569Sphilip EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 222227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 223227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 224227569Sphilip (_ip) ? 1 : 0); \ 225227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 226227569Sphilip (_tcp) ? 0 : 1); \ 227227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 228227569Sphilip (_insert) ? 1 : 0); \ 229227569Sphilip EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 230227569Sphilip \ 231227569Sphilip _NOTE(CONSTANTCONDITION) \ 232227569Sphilip } while (B_FALSE) 233227569Sphilip 234227569Sphilip#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 235227569Sphilip do { \ 236227569Sphilip efx_oword_t oword; \ 237227569Sphilip \ 238227569Sphilip if ((_enp)->en_family == EFX_FAMILY_FALCON) { \ 239227569Sphilip (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \ 240227569Sphilip break; \ 241227569Sphilip } \ 242227569Sphilip \ 243227569Sphilip EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 244227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 245227569Sphilip FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 246227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 247227569Sphilip FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 248227569Sphilip EFX_SET_OWORD_FIELD(oword, \ 249227569Sphilip FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 250227569Sphilip EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 251227569Sphilip \ 252227569Sphilip (_rc) = 0; \ 253227569Sphilip \ 254227569Sphilip _NOTE(CONSTANTCONDITION) \ 255227569Sphilip } while (B_FALSE) 256227569Sphilip 257227569Sphilip 258227569Sphilip#if EFSYS_OPT_RX_SCALE 259227569Sphilip __checkReturn int 260227569Sphilipefx_rx_scale_mode_set( 261227569Sphilip __in efx_nic_t *enp, 262227569Sphilip __in efx_rx_hash_alg_t alg, 263227569Sphilip __in efx_rx_hash_type_t type, 264227569Sphilip __in boolean_t insert) 265227569Sphilip{ 266227569Sphilip int rc; 267227569Sphilip 268227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 269227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 270227569Sphilip EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON); 271227569Sphilip 272227569Sphilip switch (alg) { 273227569Sphilip case EFX_RX_HASHALG_LFSR: 274227569Sphilip EFX_RX_LFSR_HASH(enp, insert); 275227569Sphilip break; 276227569Sphilip 277227569Sphilip case EFX_RX_HASHALG_TOEPLITZ: 278227569Sphilip EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 279227569Sphilip type & (1 << EFX_RX_HASH_IPV4), 280227569Sphilip type & (1 << EFX_RX_HASH_TCPIPV4)); 281227569Sphilip 282227569Sphilip EFX_RX_TOEPLITZ_IPV6_HASH(enp, 283227569Sphilip type & (1 << EFX_RX_HASH_IPV6), 284227569Sphilip type & (1 << EFX_RX_HASH_TCPIPV6), 285227569Sphilip rc); 286227569Sphilip if (rc != 0) 287227569Sphilip goto fail1; 288227569Sphilip 289227569Sphilip break; 290227569Sphilip 291227569Sphilip default: 292227569Sphilip rc = EINVAL; 293227569Sphilip goto fail2; 294227569Sphilip } 295227569Sphilip 296227569Sphilip return (0); 297227569Sphilip 298227569Sphilipfail2: 299227569Sphilip EFSYS_PROBE(fail2); 300227569Sphilipfail1: 301227569Sphilip EFSYS_PROBE1(fail1, int, rc); 302227569Sphilip 303227569Sphilip EFX_RX_LFSR_HASH(enp, B_FALSE); 304227569Sphilip 305227569Sphilip return (rc); 306227569Sphilip} 307227569Sphilip#endif 308227569Sphilip 309227569Sphilip#if EFSYS_OPT_RX_SCALE 310227569Sphilip __checkReturn int 311227569Sphilipefx_rx_scale_toeplitz_ipv4_key_set( 312227569Sphilip __in efx_nic_t *enp, 313227569Sphilip __in_ecount(n) uint8_t *key, 314227569Sphilip __in size_t n) 315227569Sphilip{ 316227569Sphilip efx_oword_t oword; 317227569Sphilip unsigned int byte; 318227569Sphilip unsigned int offset; 319227569Sphilip int rc; 320227569Sphilip 321227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 322227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 323227569Sphilip 324227569Sphilip byte = 0; 325227569Sphilip 326227569Sphilip /* Write toeplitz hash key */ 327227569Sphilip EFX_ZERO_OWORD(oword); 328227569Sphilip for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 329227569Sphilip offset > 0 && byte < n; 330227569Sphilip --offset) 331227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 332227569Sphilip 333227569Sphilip EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 334227569Sphilip 335227569Sphilip byte = 0; 336227569Sphilip 337227569Sphilip /* Verify toeplitz hash key */ 338227569Sphilip EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 339227569Sphilip for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 340227569Sphilip offset > 0 && byte < n; 341227569Sphilip --offset) { 342227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 343227569Sphilip rc = EFAULT; 344227569Sphilip goto fail1; 345227569Sphilip } 346227569Sphilip } 347227569Sphilip 348227569Sphilip return (0); 349227569Sphilip 350227569Sphilipfail1: 351227569Sphilip EFSYS_PROBE1(fail1, int, rc); 352227569Sphilip 353227569Sphilip return (rc); 354227569Sphilip} 355227569Sphilip#endif 356227569Sphilip 357227569Sphilip#if EFSYS_OPT_RX_SCALE 358227569Sphilip __checkReturn int 359227569Sphilipefx_rx_scale_toeplitz_ipv6_key_set( 360227569Sphilip __in efx_nic_t *enp, 361227569Sphilip __in_ecount(n) uint8_t *key, 362227569Sphilip __in size_t n) 363227569Sphilip{ 364227569Sphilip efx_oword_t oword; 365227569Sphilip unsigned int byte; 366227569Sphilip int offset; 367227569Sphilip int rc; 368227569Sphilip 369227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 370227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 371227569Sphilip 372227569Sphilip byte = 0; 373227569Sphilip 374227569Sphilip /* Write toeplitz hash key 3 */ 375227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 376227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 377227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 378227569Sphilip offset > 0 && byte < n; 379227569Sphilip --offset) 380227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 381227569Sphilip 382227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 383227569Sphilip 384227569Sphilip /* Write toeplitz hash key 2 */ 385227569Sphilip EFX_ZERO_OWORD(oword); 386227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 387227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 388227569Sphilip offset > 0 && byte < n; 389227569Sphilip --offset) 390227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 391227569Sphilip 392227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 393227569Sphilip 394227569Sphilip /* Write toeplitz hash key 1 */ 395227569Sphilip EFX_ZERO_OWORD(oword); 396227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 397227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 398227569Sphilip offset > 0 && byte < n; 399227569Sphilip --offset) 400227569Sphilip oword.eo_u8[offset - 1] = key[byte++]; 401227569Sphilip 402227569Sphilip EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 403227569Sphilip 404227569Sphilip byte = 0; 405227569Sphilip 406227569Sphilip /* Verify toeplitz hash key 3 */ 407227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 408227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 409227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 410227569Sphilip offset > 0 && byte < n; 411227569Sphilip --offset) { 412227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 413227569Sphilip rc = EFAULT; 414227569Sphilip goto fail1; 415227569Sphilip } 416227569Sphilip } 417227569Sphilip 418227569Sphilip /* Verify toeplitz hash key 2 */ 419227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 420227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 421227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 422227569Sphilip offset > 0 && byte < n; 423227569Sphilip --offset) { 424227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 425227569Sphilip rc = EFAULT; 426227569Sphilip goto fail2; 427227569Sphilip } 428227569Sphilip } 429227569Sphilip 430227569Sphilip /* Verify toeplitz hash key 1 */ 431227569Sphilip EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 432227569Sphilip for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 433227569Sphilip FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 434227569Sphilip offset > 0 && byte < n; 435227569Sphilip --offset) { 436227569Sphilip if (oword.eo_u8[offset - 1] != key[byte++]) { 437227569Sphilip rc = EFAULT; 438227569Sphilip goto fail3; 439227569Sphilip } 440227569Sphilip } 441227569Sphilip 442227569Sphilip return (0); 443227569Sphilip 444227569Sphilipfail3: 445227569Sphilip EFSYS_PROBE(fail3); 446227569Sphilipfail2: 447227569Sphilip EFSYS_PROBE(fail2); 448227569Sphilipfail1: 449227569Sphilip EFSYS_PROBE1(fail1, int, rc); 450227569Sphilip 451227569Sphilip return (rc); 452227569Sphilip} 453227569Sphilip#endif 454227569Sphilip 455227569Sphilip#if EFSYS_OPT_RX_SCALE 456227569Sphilip __checkReturn int 457227569Sphilipefx_rx_scale_tbl_set( 458227569Sphilip __in efx_nic_t *enp, 459227569Sphilip __in_ecount(n) unsigned int *table, 460227569Sphilip __in size_t n) 461227569Sphilip{ 462227569Sphilip efx_oword_t oword; 463227569Sphilip int index; 464227569Sphilip int rc; 465227569Sphilip 466227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 467227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 468227569Sphilip 469227569Sphilip EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 470227569Sphilip EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 471227569Sphilip 472227569Sphilip if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 473227569Sphilip rc = EINVAL; 474227569Sphilip goto fail1; 475227569Sphilip } 476227569Sphilip 477227569Sphilip for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 478227569Sphilip uint32_t byte; 479227569Sphilip 480227569Sphilip /* Calculate the entry to place in the table */ 481227569Sphilip byte = (uint32_t)table[index % n]; 482227569Sphilip 483227569Sphilip EFSYS_PROBE2(table, int, index, uint32_t, byte); 484227569Sphilip 485227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 486227569Sphilip 487227569Sphilip /* Write the table */ 488227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 489227569Sphilip index, &oword); 490227569Sphilip } 491227569Sphilip 492227569Sphilip for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 493227569Sphilip uint32_t byte; 494227569Sphilip 495227569Sphilip /* Determine if we're starting a new batch */ 496227569Sphilip byte = (uint32_t)table[index % n]; 497227569Sphilip 498227569Sphilip /* Read the table */ 499227569Sphilip EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 500227569Sphilip index, &oword); 501227569Sphilip 502227569Sphilip /* Verify the entry */ 503227569Sphilip if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 504227569Sphilip rc = EFAULT; 505227569Sphilip goto fail2; 506227569Sphilip } 507227569Sphilip } 508227569Sphilip 509227569Sphilip return (0); 510227569Sphilip 511227569Sphilipfail2: 512227569Sphilip EFSYS_PROBE(fail2); 513227569Sphilipfail1: 514227569Sphilip EFSYS_PROBE1(fail1, int, rc); 515227569Sphilip 516227569Sphilip return (rc); 517227569Sphilip} 518227569Sphilip#endif 519227569Sphilip 520227569Sphilip#if EFSYS_OPT_FILTER 521227569Sphilipextern __checkReturn int 522227569Sphilipefx_rx_filter_insert( 523227569Sphilip __in efx_rxq_t *erp, 524227569Sphilip __inout efx_filter_spec_t *spec) 525227569Sphilip{ 526227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 527227569Sphilip EFSYS_ASSERT3P(spec, !=, NULL); 528227569Sphilip 529227569Sphilip spec->efs_dmaq_id = (uint16_t)erp->er_index; 530227569Sphilip return efx_filter_insert_filter(erp->er_enp, spec, B_FALSE); 531227569Sphilip} 532227569Sphilip#endif 533227569Sphilip 534227569Sphilip#if EFSYS_OPT_FILTER 535227569Sphilipextern __checkReturn int 536227569Sphilipefx_rx_filter_remove( 537227569Sphilip __in efx_rxq_t *erp, 538227569Sphilip __inout efx_filter_spec_t *spec) 539227569Sphilip{ 540227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 541227569Sphilip EFSYS_ASSERT3P(spec, !=, NULL); 542227569Sphilip 543227569Sphilip spec->efs_dmaq_id = (uint16_t)erp->er_index; 544227569Sphilip return efx_filter_remove_filter(erp->er_enp, spec); 545227569Sphilip} 546227569Sphilip#endif 547227569Sphilip 548227569Sphilipextern void 549227569Sphilipefx_rx_qpost( 550227569Sphilip __in efx_rxq_t *erp, 551227569Sphilip __in_ecount(n) efsys_dma_addr_t *addrp, 552227569Sphilip __in size_t size, 553227569Sphilip __in unsigned int n, 554227569Sphilip __in unsigned int completed, 555227569Sphilip __in unsigned int added) 556227569Sphilip{ 557227569Sphilip efx_qword_t qword; 558227569Sphilip unsigned int i; 559227569Sphilip unsigned int offset; 560227569Sphilip unsigned int id; 561227569Sphilip 562227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 563227569Sphilip 564227569Sphilip /* The client driver must not overfill the queue */ 565227569Sphilip EFSYS_ASSERT3U(added - completed + n, <=, 566227569Sphilip EFX_RXQ_LIMIT(erp->er_mask + 1)); 567227569Sphilip 568227569Sphilip id = added & (erp->er_mask); 569227569Sphilip for (i = 0; i < n; i++) { 570227569Sphilip EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 571227569Sphilip unsigned int, id, efsys_dma_addr_t, addrp[i], 572227569Sphilip size_t, size); 573227569Sphilip 574227569Sphilip EFX_POPULATE_QWORD_3(qword, 575227569Sphilip FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 576227569Sphilip FSF_AZ_RX_KER_BUF_ADDR_DW0, 577227569Sphilip (uint32_t)(addrp[i] & 0xffffffff), 578227569Sphilip FSF_AZ_RX_KER_BUF_ADDR_DW1, 579227569Sphilip (uint32_t)(addrp[i] >> 32)); 580227569Sphilip 581227569Sphilip offset = id * sizeof (efx_qword_t); 582227569Sphilip EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 583227569Sphilip 584227569Sphilip id = (id + 1) & (erp->er_mask); 585227569Sphilip } 586227569Sphilip} 587227569Sphilip 588227569Sphilip void 589227569Sphilipefx_rx_qpush( 590227569Sphilip __in efx_rxq_t *erp, 591227569Sphilip __in unsigned int added) 592227569Sphilip{ 593227569Sphilip efx_nic_t *enp = erp->er_enp; 594227569Sphilip uint32_t wptr; 595227569Sphilip efx_oword_t oword; 596227569Sphilip efx_dword_t dword; 597227569Sphilip 598227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 599227569Sphilip 600227569Sphilip /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 601227569Sphilip EFSYS_PIO_WRITE_BARRIER(); 602227569Sphilip 603227569Sphilip /* Push the populated descriptors out */ 604227569Sphilip wptr = added & erp->er_mask; 605227569Sphilip 606227569Sphilip EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 607227569Sphilip 608227569Sphilip /* Only write the third DWORD */ 609227569Sphilip EFX_POPULATE_DWORD_1(dword, 610227569Sphilip EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 611227569Sphilip EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 612227569Sphilip erp->er_index, &dword, B_FALSE); 613227569Sphilip} 614227569Sphilip 615227569Sphilip void 616227569Sphilipefx_rx_qflush( 617227569Sphilip __in efx_rxq_t *erp) 618227569Sphilip{ 619227569Sphilip efx_nic_t *enp = erp->er_enp; 620227569Sphilip efx_oword_t oword; 621227569Sphilip uint32_t label; 622227569Sphilip 623227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 624227569Sphilip 625227569Sphilip label = erp->er_index; 626227569Sphilip 627227569Sphilip /* Flush the queue */ 628227569Sphilip EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 629227569Sphilip FRF_AZ_RX_FLUSH_DESCQ, label); 630227569Sphilip EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 631227569Sphilip} 632227569Sphilip 633227569Sphilip void 634227569Sphilipefx_rx_qenable( 635227569Sphilip __in efx_rxq_t *erp) 636227569Sphilip{ 637227569Sphilip efx_nic_t *enp = erp->er_enp; 638227569Sphilip efx_oword_t oword; 639227569Sphilip 640227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 641227569Sphilip 642227569Sphilip EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 643227569Sphilip erp->er_index, &oword); 644227569Sphilip 645227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 646227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 647227569Sphilip EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 648227569Sphilip 649227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 650227569Sphilip erp->er_index, &oword); 651227569Sphilip} 652227569Sphilip 653227569Sphilip __checkReturn int 654227569Sphilipefx_rx_qcreate( 655227569Sphilip __in efx_nic_t *enp, 656227569Sphilip __in unsigned int index, 657227569Sphilip __in unsigned int label, 658227569Sphilip __in efx_rxq_type_t type, 659227569Sphilip __in efsys_mem_t *esmp, 660227569Sphilip __in size_t n, 661227569Sphilip __in uint32_t id, 662227569Sphilip __in efx_evq_t *eep, 663227569Sphilip __deref_out efx_rxq_t **erpp) 664227569Sphilip{ 665227569Sphilip efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 666227569Sphilip efx_rxq_t *erp; 667227569Sphilip efx_oword_t oword; 668227569Sphilip uint32_t size; 669227569Sphilip boolean_t split; 670227569Sphilip boolean_t jumbo; 671227569Sphilip int rc; 672227569Sphilip 673227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 674227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 675227569Sphilip 676227569Sphilip EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 677227569Sphilip EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 678227569Sphilip EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 679227569Sphilip 680227569Sphilip if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) { 681227569Sphilip rc = EINVAL; 682227569Sphilip goto fail1; 683227569Sphilip } 684227569Sphilip if (index >= encp->enc_rxq_limit) { 685227569Sphilip rc = EINVAL; 686227569Sphilip goto fail2; 687227569Sphilip } 688227569Sphilip for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 689227569Sphilip size++) 690227569Sphilip if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 691227569Sphilip break; 692227569Sphilip if (id + (1 << size) >= encp->enc_buftbl_limit) { 693227569Sphilip rc = EINVAL; 694227569Sphilip goto fail3; 695227569Sphilip } 696227569Sphilip 697227569Sphilip switch (type) { 698227569Sphilip case EFX_RXQ_TYPE_DEFAULT: 699227569Sphilip split = B_FALSE; 700227569Sphilip jumbo = B_FALSE; 701227569Sphilip break; 702227569Sphilip 703227569Sphilip#if EFSYS_OPT_RX_HDR_SPLIT 704227569Sphilip case EFX_RXQ_TYPE_SPLIT_HEADER: 705227569Sphilip if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) { 706227569Sphilip rc = EINVAL; 707227569Sphilip goto fail4; 708227569Sphilip } 709227569Sphilip split = B_TRUE; 710227569Sphilip jumbo = B_TRUE; 711227569Sphilip break; 712227569Sphilip 713227569Sphilip case EFX_RXQ_TYPE_SPLIT_PAYLOAD: 714227569Sphilip if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) { 715227569Sphilip rc = EINVAL; 716227569Sphilip goto fail4; 717227569Sphilip } 718227569Sphilip split = B_FALSE; 719227569Sphilip jumbo = B_TRUE; 720227569Sphilip break; 721227569Sphilip#endif /* EFSYS_OPT_RX_HDR_SPLIT */ 722227569Sphilip 723227569Sphilip#if EFSYS_OPT_RX_SCATTER 724227569Sphilip case EFX_RXQ_TYPE_SCATTER: 725227569Sphilip if (enp->en_family < EFX_FAMILY_SIENA) { 726227569Sphilip rc = EINVAL; 727227569Sphilip goto fail4; 728227569Sphilip } 729227569Sphilip split = B_FALSE; 730227569Sphilip jumbo = B_TRUE; 731227569Sphilip break; 732227569Sphilip#endif /* EFSYS_OPT_RX_SCATTER */ 733227569Sphilip 734227569Sphilip default: 735227569Sphilip rc = EINVAL; 736227569Sphilip goto fail4; 737227569Sphilip } 738227569Sphilip 739227569Sphilip /* Allocate an RXQ object */ 740227569Sphilip EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 741227569Sphilip 742227569Sphilip if (erp == NULL) { 743227569Sphilip rc = ENOMEM; 744227569Sphilip goto fail5; 745227569Sphilip } 746227569Sphilip 747227569Sphilip erp->er_magic = EFX_RXQ_MAGIC; 748227569Sphilip erp->er_enp = enp; 749227569Sphilip erp->er_index = index; 750227569Sphilip erp->er_mask = n - 1; 751227569Sphilip erp->er_esmp = esmp; 752227569Sphilip 753227569Sphilip /* Set up the new descriptor queue */ 754227569Sphilip EFX_POPULATE_OWORD_10(oword, 755227569Sphilip FRF_CZ_RX_HDR_SPLIT, split, 756227569Sphilip FRF_AZ_RX_ISCSI_DDIG_EN, 0, 757227569Sphilip FRF_AZ_RX_ISCSI_HDIG_EN, 0, 758227569Sphilip FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 759227569Sphilip FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 760227569Sphilip FRF_AZ_RX_DESCQ_OWNER_ID, 0, 761227569Sphilip FRF_AZ_RX_DESCQ_LABEL, label, 762227569Sphilip FRF_AZ_RX_DESCQ_SIZE, size, 763227569Sphilip FRF_AZ_RX_DESCQ_TYPE, 0, 764227569Sphilip FRF_AZ_RX_DESCQ_JUMBO, jumbo); 765227569Sphilip 766227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 767227569Sphilip erp->er_index, &oword); 768227569Sphilip 769227569Sphilip enp->en_rx_qcount++; 770227569Sphilip *erpp = erp; 771227569Sphilip return (0); 772227569Sphilip 773227569Sphilipfail5: 774227569Sphilip EFSYS_PROBE(fail5); 775227569Sphilipfail4: 776227569Sphilip EFSYS_PROBE(fail4); 777227569Sphilipfail3: 778227569Sphilip EFSYS_PROBE(fail3); 779227569Sphilipfail2: 780227569Sphilip EFSYS_PROBE(fail2); 781227569Sphilipfail1: 782227569Sphilip EFSYS_PROBE1(fail1, int, rc); 783227569Sphilip 784227569Sphilip return (rc); 785227569Sphilip} 786227569Sphilip 787227569Sphilip void 788227569Sphilipefx_rx_qdestroy( 789227569Sphilip __in efx_rxq_t *erp) 790227569Sphilip{ 791227569Sphilip efx_nic_t *enp = erp->er_enp; 792227569Sphilip efx_oword_t oword; 793227569Sphilip 794227569Sphilip EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 795227569Sphilip 796227569Sphilip EFSYS_ASSERT(enp->en_rx_qcount != 0); 797227569Sphilip --enp->en_rx_qcount; 798227569Sphilip 799227569Sphilip /* Purge descriptor queue */ 800227569Sphilip EFX_ZERO_OWORD(oword); 801227569Sphilip 802227569Sphilip EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 803227569Sphilip erp->er_index, &oword); 804227569Sphilip 805227569Sphilip /* Free the RXQ object */ 806227569Sphilip EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 807227569Sphilip} 808227569Sphilip 809227569Sphilip void 810227569Sphilipefx_rx_fini( 811227569Sphilip __in efx_nic_t *enp) 812227569Sphilip{ 813227569Sphilip EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 814227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 815227569Sphilip EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 816227569Sphilip EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 817227569Sphilip 818227569Sphilip enp->en_mod_flags &= ~EFX_MOD_RX; 819227569Sphilip} 820