1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD$"); 35 36#include "efx.h" 37#include "efx_impl.h" 38 39#if EFSYS_OPT_SIENA 40 41static __checkReturn efx_rc_t 42siena_rx_init( 43 __in efx_nic_t *enp); 44 45static void 46siena_rx_fini( 47 __in efx_nic_t *enp); 48 49#if EFSYS_OPT_RX_SCATTER 50static __checkReturn efx_rc_t 51siena_rx_scatter_enable( 52 __in efx_nic_t *enp, 53 __in unsigned int buf_size); 54#endif /* EFSYS_OPT_RX_SCATTER */ 55 56#if EFSYS_OPT_RX_SCALE 57static __checkReturn efx_rc_t 58siena_rx_scale_mode_set( 59 __in efx_nic_t *enp, 60 __in uint32_t rss_context, 61 __in efx_rx_hash_alg_t alg, 62 __in efx_rx_hash_type_t type, 63 __in boolean_t insert); 64 65static __checkReturn efx_rc_t 66siena_rx_scale_key_set( 67 __in efx_nic_t *enp, 68 __in uint32_t rss_context, 69 __in_ecount(n) uint8_t *key, 70 __in size_t n); 71 72static __checkReturn efx_rc_t 73siena_rx_scale_tbl_set( 74 __in efx_nic_t *enp, 75 __in uint32_t rss_context, 76 __in_ecount(n) unsigned int *table, 77 __in size_t n); 78 79static __checkReturn uint32_t 80siena_rx_prefix_hash( 81 __in efx_nic_t *enp, 82 __in efx_rx_hash_alg_t func, 83 __in uint8_t *buffer); 84 85#endif /* EFSYS_OPT_RX_SCALE */ 86 87static __checkReturn efx_rc_t 88siena_rx_prefix_pktlen( 89 __in efx_nic_t *enp, 90 __in uint8_t *buffer, 91 __out uint16_t *lengthp); 92 93static void 94siena_rx_qpost( 95 __in efx_rxq_t *erp, 96 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 97 __in size_t size, 98 __in unsigned int ndescs, 99 __in unsigned int completed, 100 __in unsigned int added); 101 102static void 103siena_rx_qpush( 104 __in efx_rxq_t *erp, 105 __in unsigned int added, 106 __inout unsigned int *pushedp); 107 108#if EFSYS_OPT_RX_PACKED_STREAM 109static void 110siena_rx_qpush_ps_credits( 111 __in efx_rxq_t *erp); 112 113static __checkReturn uint8_t * 114siena_rx_qps_packet_info( 115 __in efx_rxq_t *erp, 116 __in uint8_t *buffer, 117 __in uint32_t buffer_length, 118 __in uint32_t current_offset, 119 __out uint16_t *lengthp, 120 __out uint32_t *next_offsetp, 121 __out uint32_t *timestamp); 122#endif 123 124static __checkReturn efx_rc_t 125siena_rx_qflush( 126 __in efx_rxq_t *erp); 127 128static void 129siena_rx_qenable( 130 __in efx_rxq_t *erp); 131 132static __checkReturn efx_rc_t 133siena_rx_qcreate( 134 __in efx_nic_t *enp, 135 __in unsigned int index, 136 __in unsigned int label, 137 __in efx_rxq_type_t type, 138 __in_opt const efx_rxq_type_data_t *type_data, 139 __in efsys_mem_t *esmp, 140 __in size_t ndescs, 141 __in uint32_t id, 142 __in unsigned int flags, 143 __in efx_evq_t *eep, 144 __in efx_rxq_t *erp); 145 146static void 147siena_rx_qdestroy( 148 __in efx_rxq_t *erp); 149 150#endif /* EFSYS_OPT_SIENA */ 151 152#if EFSYS_OPT_SIENA 153static const efx_rx_ops_t __efx_rx_siena_ops = { 154 siena_rx_init, /* erxo_init */ 155 siena_rx_fini, /* erxo_fini */ 156#if EFSYS_OPT_RX_SCATTER 157 siena_rx_scatter_enable, /* erxo_scatter_enable */ 158#endif 159#if EFSYS_OPT_RX_SCALE 160 NULL, /* erxo_scale_context_alloc */ 161 NULL, /* erxo_scale_context_free */ 162 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 163 siena_rx_scale_key_set, /* erxo_scale_key_set */ 164 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 165 siena_rx_prefix_hash, /* erxo_prefix_hash */ 166#endif 167 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 168 siena_rx_qpost, /* erxo_qpost */ 169 siena_rx_qpush, /* erxo_qpush */ 170#if EFSYS_OPT_RX_PACKED_STREAM 171 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 172 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 173#endif 174 siena_rx_qflush, /* erxo_qflush */ 175 siena_rx_qenable, /* erxo_qenable */ 176 siena_rx_qcreate, /* erxo_qcreate */ 177 siena_rx_qdestroy, /* erxo_qdestroy */ 178}; 179#endif /* EFSYS_OPT_SIENA */ 180 181#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 182static const efx_rx_ops_t __efx_rx_ef10_ops = { 183 ef10_rx_init, /* erxo_init */ 184 ef10_rx_fini, /* erxo_fini */ 185#if EFSYS_OPT_RX_SCATTER 186 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 187#endif 188#if EFSYS_OPT_RX_SCALE 189 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 190 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 191 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 192 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 193 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 194 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 195#endif 196 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 197 ef10_rx_qpost, /* erxo_qpost */ 198 ef10_rx_qpush, /* erxo_qpush */ 199#if EFSYS_OPT_RX_PACKED_STREAM 200 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 201 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 202#endif 203 ef10_rx_qflush, /* erxo_qflush */ 204 ef10_rx_qenable, /* erxo_qenable */ 205 ef10_rx_qcreate, /* erxo_qcreate */ 206 ef10_rx_qdestroy, /* erxo_qdestroy */ 207}; 208#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ 209 210 __checkReturn efx_rc_t 211efx_rx_init( 212 __inout efx_nic_t *enp) 213{ 214 const efx_rx_ops_t *erxop; 215 efx_rc_t rc; 216 217 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 218 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 219 220 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 221 rc = EINVAL; 222 goto fail1; 223 } 224 225 if (enp->en_mod_flags & EFX_MOD_RX) { 226 rc = EINVAL; 227 goto fail2; 228 } 229 230 switch (enp->en_family) { 231#if EFSYS_OPT_SIENA 232 case EFX_FAMILY_SIENA: 233 erxop = &__efx_rx_siena_ops; 234 break; 235#endif /* EFSYS_OPT_SIENA */ 236 237#if EFSYS_OPT_HUNTINGTON 238 case EFX_FAMILY_HUNTINGTON: 239 erxop = &__efx_rx_ef10_ops; 240 break; 241#endif /* EFSYS_OPT_HUNTINGTON */ 242 243#if EFSYS_OPT_MEDFORD 244 case EFX_FAMILY_MEDFORD: 245 erxop = &__efx_rx_ef10_ops; 246 break; 247#endif /* EFSYS_OPT_MEDFORD */ 248 249#if EFSYS_OPT_MEDFORD2 250 case EFX_FAMILY_MEDFORD2: 251 erxop = &__efx_rx_ef10_ops; 252 break; 253#endif /* EFSYS_OPT_MEDFORD2 */ 254 255 default: 256 EFSYS_ASSERT(0); 257 rc = ENOTSUP; 258 goto fail3; 259 } 260 261 if ((rc = erxop->erxo_init(enp)) != 0) 262 goto fail4; 263 264 enp->en_erxop = erxop; 265 enp->en_mod_flags |= EFX_MOD_RX; 266 return (0); 267 268fail4: 269 EFSYS_PROBE(fail4); 270fail3: 271 EFSYS_PROBE(fail3); 272fail2: 273 EFSYS_PROBE(fail2); 274fail1: 275 EFSYS_PROBE1(fail1, efx_rc_t, rc); 276 277 enp->en_erxop = NULL; 278 enp->en_mod_flags &= ~EFX_MOD_RX; 279 return (rc); 280} 281 282 void 283efx_rx_fini( 284 __in efx_nic_t *enp) 285{ 286 const efx_rx_ops_t *erxop = enp->en_erxop; 287 288 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 289 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 291 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 292 293 erxop->erxo_fini(enp); 294 295 enp->en_erxop = NULL; 296 enp->en_mod_flags &= ~EFX_MOD_RX; 297} 298 299#if EFSYS_OPT_RX_SCATTER 300 __checkReturn efx_rc_t 301efx_rx_scatter_enable( 302 __in efx_nic_t *enp, 303 __in unsigned int buf_size) 304{ 305 const efx_rx_ops_t *erxop = enp->en_erxop; 306 efx_rc_t rc; 307 308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 310 311 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 312 goto fail1; 313 314 return (0); 315 316fail1: 317 EFSYS_PROBE1(fail1, efx_rc_t, rc); 318 return (rc); 319} 320#endif /* EFSYS_OPT_RX_SCATTER */ 321 322#if EFSYS_OPT_RX_SCALE 323 __checkReturn efx_rc_t 324efx_rx_scale_hash_flags_get( 325 __in efx_nic_t *enp, 326 __in efx_rx_hash_alg_t hash_alg, 327 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 328 __in unsigned int max_nflags, 329 __out unsigned int *nflagsp) 330{ 331 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 332 unsigned int nflags = 0; 333 efx_rc_t rc; 334 335 if (flagsp == NULL || nflagsp == NULL) { 336 rc = EINVAL; 337 goto fail1; 338 } 339 340 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 341 nflags = 0; 342 goto done; 343 } 344 345 /* Helper to add flags word to flags array without buffer overflow */ 346#define INSERT_FLAGS(_flags) \ 347 do { \ 348 if (nflags >= max_nflags) { \ 349 rc = E2BIG; \ 350 goto fail2; \ 351 } \ 352 *(flagsp + nflags) = (_flags); \ 353 nflags++; \ 354 \ 355 _NOTE(CONSTANTCONDITION) \ 356 } while (B_FALSE) 357 358 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 359 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 360 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 361 } 362 363 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 364 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 365 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 366 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 367 368 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 370 371 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 372 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 373 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 374 375 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 376 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 377 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 378 } 379 380 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 381 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 382 383 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 384 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 385 386 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 387 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 388 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 389 390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 391 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 392 393 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 394 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 395 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 396 397 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 398 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 399 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 400 401 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 402 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 403 404 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 405 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 406 } 407 408 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 409 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 410 411 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 412 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 413 414 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 415 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 416 417#undef INSERT_FLAGS 418 419done: 420 *nflagsp = nflags; 421 return (0); 422 423fail2: 424 EFSYS_PROBE(fail2); 425fail1: 426 EFSYS_PROBE1(fail1, efx_rc_t, rc); 427 428 return (rc); 429} 430 431 __checkReturn efx_rc_t 432efx_rx_hash_default_support_get( 433 __in efx_nic_t *enp, 434 __out efx_rx_hash_support_t *supportp) 435{ 436 efx_rc_t rc; 437 438 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 439 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 440 441 if (supportp == NULL) { 442 rc = EINVAL; 443 goto fail1; 444 } 445 446 /* 447 * Report the hashing support the client gets by default if it 448 * does not allocate an RSS context itself. 449 */ 450 *supportp = enp->en_hash_support; 451 452 return (0); 453 454fail1: 455 EFSYS_PROBE1(fail1, efx_rc_t, rc); 456 457 return (rc); 458} 459 460 __checkReturn efx_rc_t 461efx_rx_scale_default_support_get( 462 __in efx_nic_t *enp, 463 __out efx_rx_scale_context_type_t *typep) 464{ 465 efx_rc_t rc; 466 467 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 468 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 469 470 if (typep == NULL) { 471 rc = EINVAL; 472 goto fail1; 473 } 474 475 /* 476 * Report the RSS support the client gets by default if it 477 * does not allocate an RSS context itself. 478 */ 479 *typep = enp->en_rss_context_type; 480 481 return (0); 482 483fail1: 484 EFSYS_PROBE1(fail1, efx_rc_t, rc); 485 486 return (rc); 487} 488#endif /* EFSYS_OPT_RX_SCALE */ 489 490#if EFSYS_OPT_RX_SCALE 491 __checkReturn efx_rc_t 492efx_rx_scale_context_alloc( 493 __in efx_nic_t *enp, 494 __in efx_rx_scale_context_type_t type, 495 __in uint32_t num_queues, 496 __out uint32_t *rss_contextp) 497{ 498 const efx_rx_ops_t *erxop = enp->en_erxop; 499 efx_rc_t rc; 500 501 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 502 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 503 504 if (erxop->erxo_scale_context_alloc == NULL) { 505 rc = ENOTSUP; 506 goto fail1; 507 } 508 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 509 num_queues, rss_contextp)) != 0) { 510 goto fail2; 511 } 512 513 return (0); 514 515fail2: 516 EFSYS_PROBE(fail2); 517fail1: 518 EFSYS_PROBE1(fail1, efx_rc_t, rc); 519 return (rc); 520} 521#endif /* EFSYS_OPT_RX_SCALE */ 522 523#if EFSYS_OPT_RX_SCALE 524 __checkReturn efx_rc_t 525efx_rx_scale_context_free( 526 __in efx_nic_t *enp, 527 __in uint32_t rss_context) 528{ 529 const efx_rx_ops_t *erxop = enp->en_erxop; 530 efx_rc_t rc; 531 532 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 533 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 534 535 if (erxop->erxo_scale_context_free == NULL) { 536 rc = ENOTSUP; 537 goto fail1; 538 } 539 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 540 goto fail2; 541 542 return (0); 543 544fail2: 545 EFSYS_PROBE(fail2); 546fail1: 547 EFSYS_PROBE1(fail1, efx_rc_t, rc); 548 return (rc); 549} 550#endif /* EFSYS_OPT_RX_SCALE */ 551 552#if EFSYS_OPT_RX_SCALE 553 __checkReturn efx_rc_t 554efx_rx_scale_mode_set( 555 __in efx_nic_t *enp, 556 __in uint32_t rss_context, 557 __in efx_rx_hash_alg_t alg, 558 __in efx_rx_hash_type_t type, 559 __in boolean_t insert) 560{ 561 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 562 const efx_rx_ops_t *erxop = enp->en_erxop; 563 efx_rx_hash_type_t type_check; 564 unsigned int i; 565 efx_rc_t rc; 566 567 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 568 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 569 570 /* 571 * Legacy flags and modern bits cannot be 572 * used at the same time in the hash type. 573 */ 574 if ((type & EFX_RX_HASH_LEGACY_MASK) && 575 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 576 rc = EINVAL; 577 goto fail1; 578 } 579 580 /* 581 * If RSS hash type is represented by additional bits 582 * in the value, the latter need to be verified since 583 * not all bit combinations are valid RSS modes. Also, 584 * depending on the firmware, some valid combinations 585 * may be unsupported. Discern additional bits in the 586 * type value and try to recognise valid combinations. 587 * If some bits remain unrecognised, report the error. 588 */ 589 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 590 if (type_check != 0) { 591 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 592 unsigned int type_nflags; 593 594 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 595 EFX_ARRAY_SIZE(type_flags), &type_nflags); 596 if (rc != 0) 597 goto fail2; 598 599 for (i = 0; i < type_nflags; ++i) { 600 if ((type_check & type_flags[i]) == type_flags[i]) 601 type_check &= ~(type_flags[i]); 602 } 603 604 if (type_check != 0) { 605 rc = EINVAL; 606 goto fail3; 607 } 608 } 609 610 /* 611 * Translate EFX_RX_HASH() flags to their legacy counterparts 612 * provided that the FW claims no support for additional modes. 613 */ 614 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 615 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 616 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 617 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 618 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 619 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 620 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 621 622 if ((type & t_ipv4) == t_ipv4) 623 type |= EFX_RX_HASH_IPV4; 624 if ((type & t_ipv6) == t_ipv6) 625 type |= EFX_RX_HASH_IPV6; 626 627 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 628 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 629 type |= EFX_RX_HASH_TCPIPV4; 630 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 631 type |= EFX_RX_HASH_TCPIPV6; 632 } 633 634 type &= EFX_RX_HASH_LEGACY_MASK; 635 } 636 637 if (erxop->erxo_scale_mode_set != NULL) { 638 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 639 type, insert)) != 0) 640 goto fail4; 641 } 642 643 return (0); 644 645fail4: 646 EFSYS_PROBE(fail4); 647fail3: 648 EFSYS_PROBE(fail3); 649fail2: 650 EFSYS_PROBE(fail2); 651fail1: 652 EFSYS_PROBE1(fail1, efx_rc_t, rc); 653 return (rc); 654} 655#endif /* EFSYS_OPT_RX_SCALE */ 656 657#if EFSYS_OPT_RX_SCALE 658 __checkReturn efx_rc_t 659efx_rx_scale_key_set( 660 __in efx_nic_t *enp, 661 __in uint32_t rss_context, 662 __in_ecount(n) uint8_t *key, 663 __in size_t n) 664{ 665 const efx_rx_ops_t *erxop = enp->en_erxop; 666 efx_rc_t rc; 667 668 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 669 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 670 671 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 672 goto fail1; 673 674 return (0); 675 676fail1: 677 EFSYS_PROBE1(fail1, efx_rc_t, rc); 678 679 return (rc); 680} 681#endif /* EFSYS_OPT_RX_SCALE */ 682 683#if EFSYS_OPT_RX_SCALE 684 __checkReturn efx_rc_t 685efx_rx_scale_tbl_set( 686 __in efx_nic_t *enp, 687 __in uint32_t rss_context, 688 __in_ecount(n) unsigned int *table, 689 __in size_t n) 690{ 691 const efx_rx_ops_t *erxop = enp->en_erxop; 692 efx_rc_t rc; 693 694 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 695 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 696 697 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 698 goto fail1; 699 700 return (0); 701 702fail1: 703 EFSYS_PROBE1(fail1, efx_rc_t, rc); 704 705 return (rc); 706} 707#endif /* EFSYS_OPT_RX_SCALE */ 708 709 void 710efx_rx_qpost( 711 __in efx_rxq_t *erp, 712 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 713 __in size_t size, 714 __in unsigned int ndescs, 715 __in unsigned int completed, 716 __in unsigned int added) 717{ 718 efx_nic_t *enp = erp->er_enp; 719 const efx_rx_ops_t *erxop = enp->en_erxop; 720 721 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 722 723 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 724} 725 726#if EFSYS_OPT_RX_PACKED_STREAM 727 728 void 729efx_rx_qpush_ps_credits( 730 __in efx_rxq_t *erp) 731{ 732 efx_nic_t *enp = erp->er_enp; 733 const efx_rx_ops_t *erxop = enp->en_erxop; 734 735 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 736 737 erxop->erxo_qpush_ps_credits(erp); 738} 739 740 __checkReturn uint8_t * 741efx_rx_qps_packet_info( 742 __in efx_rxq_t *erp, 743 __in uint8_t *buffer, 744 __in uint32_t buffer_length, 745 __in uint32_t current_offset, 746 __out uint16_t *lengthp, 747 __out uint32_t *next_offsetp, 748 __out uint32_t *timestamp) 749{ 750 efx_nic_t *enp = erp->er_enp; 751 const efx_rx_ops_t *erxop = enp->en_erxop; 752 753 return (erxop->erxo_qps_packet_info(erp, buffer, 754 buffer_length, current_offset, lengthp, 755 next_offsetp, timestamp)); 756} 757 758#endif /* EFSYS_OPT_RX_PACKED_STREAM */ 759 760 void 761efx_rx_qpush( 762 __in efx_rxq_t *erp, 763 __in unsigned int added, 764 __inout unsigned int *pushedp) 765{ 766 efx_nic_t *enp = erp->er_enp; 767 const efx_rx_ops_t *erxop = enp->en_erxop; 768 769 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 770 771 erxop->erxo_qpush(erp, added, pushedp); 772} 773 774 __checkReturn efx_rc_t 775efx_rx_qflush( 776 __in efx_rxq_t *erp) 777{ 778 efx_nic_t *enp = erp->er_enp; 779 const efx_rx_ops_t *erxop = enp->en_erxop; 780 efx_rc_t rc; 781 782 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 783 784 if ((rc = erxop->erxo_qflush(erp)) != 0) 785 goto fail1; 786 787 return (0); 788 789fail1: 790 EFSYS_PROBE1(fail1, efx_rc_t, rc); 791 792 return (rc); 793} 794 795 void 796efx_rx_qenable( 797 __in efx_rxq_t *erp) 798{ 799 efx_nic_t *enp = erp->er_enp; 800 const efx_rx_ops_t *erxop = enp->en_erxop; 801 802 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 803 804 erxop->erxo_qenable(erp); 805} 806 807static __checkReturn efx_rc_t 808efx_rx_qcreate_internal( 809 __in efx_nic_t *enp, 810 __in unsigned int index, 811 __in unsigned int label, 812 __in efx_rxq_type_t type, 813 __in_opt const efx_rxq_type_data_t *type_data, 814 __in efsys_mem_t *esmp, 815 __in size_t ndescs, 816 __in uint32_t id, 817 __in unsigned int flags, 818 __in efx_evq_t *eep, 819 __deref_out efx_rxq_t **erpp) 820{ 821 const efx_rx_ops_t *erxop = enp->en_erxop; 822 efx_rxq_t *erp; 823 efx_rc_t rc; 824 825 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 826 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 827 828 /* Allocate an RXQ object */ 829 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 830 831 if (erp == NULL) { 832 rc = ENOMEM; 833 goto fail1; 834 } 835 836 erp->er_magic = EFX_RXQ_MAGIC; 837 erp->er_enp = enp; 838 erp->er_index = index; 839 erp->er_mask = ndescs - 1; 840 erp->er_esmp = esmp; 841 842 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 843 ndescs, id, flags, eep, erp)) != 0) 844 goto fail2; 845 846 enp->en_rx_qcount++; 847 *erpp = erp; 848 849 return (0); 850 851fail2: 852 EFSYS_PROBE(fail2); 853 854 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 855fail1: 856 EFSYS_PROBE1(fail1, efx_rc_t, rc); 857 858 return (rc); 859} 860 861 __checkReturn efx_rc_t 862efx_rx_qcreate( 863 __in efx_nic_t *enp, 864 __in unsigned int index, 865 __in unsigned int label, 866 __in efx_rxq_type_t type, 867 __in efsys_mem_t *esmp, 868 __in size_t ndescs, 869 __in uint32_t id, 870 __in unsigned int flags, 871 __in efx_evq_t *eep, 872 __deref_out efx_rxq_t **erpp) 873{ 874 return efx_rx_qcreate_internal(enp, index, label, type, NULL, 875 esmp, ndescs, id, flags, eep, erpp); 876} 877 878#if EFSYS_OPT_RX_PACKED_STREAM 879 880 __checkReturn efx_rc_t 881efx_rx_qcreate_packed_stream( 882 __in efx_nic_t *enp, 883 __in unsigned int index, 884 __in unsigned int label, 885 __in uint32_t ps_buf_size, 886 __in efsys_mem_t *esmp, 887 __in size_t ndescs, 888 __in efx_evq_t *eep, 889 __deref_out efx_rxq_t **erpp) 890{ 891 efx_rxq_type_data_t type_data; 892 893 memset(&type_data, 0, sizeof (type_data)); 894 895 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 896 897 return efx_rx_qcreate_internal(enp, index, label, 898 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 899 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 900} 901 902#endif 903 904#if EFSYS_OPT_RX_ES_SUPER_BUFFER 905 906 __checkReturn efx_rc_t 907efx_rx_qcreate_es_super_buffer( 908 __in efx_nic_t *enp, 909 __in unsigned int index, 910 __in unsigned int label, 911 __in uint32_t n_bufs_per_desc, 912 __in uint32_t max_dma_len, 913 __in uint32_t buf_stride, 914 __in uint32_t hol_block_timeout, 915 __in efsys_mem_t *esmp, 916 __in size_t ndescs, 917 __in unsigned int flags, 918 __in efx_evq_t *eep, 919 __deref_out efx_rxq_t **erpp) 920{ 921 efx_rc_t rc; 922 efx_rxq_type_data_t type_data; 923 924 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 925 rc = EINVAL; 926 goto fail1; 927 } 928 929 memset(&type_data, 0, sizeof (type_data)); 930 931 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 932 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 933 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 934 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 935 hol_block_timeout; 936 937 rc = efx_rx_qcreate_internal(enp, index, label, 938 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 939 0 /* id unused on EF10 */, flags, eep, erpp); 940 if (rc != 0) 941 goto fail2; 942 943 return (0); 944 945fail2: 946 EFSYS_PROBE(fail2); 947fail1: 948 EFSYS_PROBE1(fail1, efx_rc_t, rc); 949 950 return (rc); 951} 952 953#endif 954 955 void 956efx_rx_qdestroy( 957 __in efx_rxq_t *erp) 958{ 959 efx_nic_t *enp = erp->er_enp; 960 const efx_rx_ops_t *erxop = enp->en_erxop; 961 962 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 963 964 erxop->erxo_qdestroy(erp); 965} 966 967 __checkReturn efx_rc_t 968efx_pseudo_hdr_pkt_length_get( 969 __in efx_rxq_t *erp, 970 __in uint8_t *buffer, 971 __out uint16_t *lengthp) 972{ 973 efx_nic_t *enp = erp->er_enp; 974 const efx_rx_ops_t *erxop = enp->en_erxop; 975 976 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 977 978 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 979} 980 981#if EFSYS_OPT_RX_SCALE 982 __checkReturn uint32_t 983efx_pseudo_hdr_hash_get( 984 __in efx_rxq_t *erp, 985 __in efx_rx_hash_alg_t func, 986 __in uint8_t *buffer) 987{ 988 efx_nic_t *enp = erp->er_enp; 989 const efx_rx_ops_t *erxop = enp->en_erxop; 990 991 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 992 993 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 994 return (erxop->erxo_prefix_hash(enp, func, buffer)); 995} 996#endif /* EFSYS_OPT_RX_SCALE */ 997 998#if EFSYS_OPT_SIENA 999 1000static __checkReturn efx_rc_t 1001siena_rx_init( 1002 __in efx_nic_t *enp) 1003{ 1004 efx_oword_t oword; 1005 unsigned int index; 1006 1007 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1008 1009 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1010 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1011 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1012 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1013 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1014 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1015 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1016 1017 /* Zero the RSS table */ 1018 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1019 index++) { 1020 EFX_ZERO_OWORD(oword); 1021 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1022 index, &oword, B_TRUE); 1023 } 1024 1025#if EFSYS_OPT_RX_SCALE 1026 /* The RSS key and indirection table are writable. */ 1027 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1028 1029 /* Hardware can insert RX hash with/without RSS */ 1030 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1031#endif /* EFSYS_OPT_RX_SCALE */ 1032 1033 return (0); 1034} 1035 1036#if EFSYS_OPT_RX_SCATTER 1037static __checkReturn efx_rc_t 1038siena_rx_scatter_enable( 1039 __in efx_nic_t *enp, 1040 __in unsigned int buf_size) 1041{ 1042 unsigned int nbuf32; 1043 efx_oword_t oword; 1044 efx_rc_t rc; 1045 1046 nbuf32 = buf_size / 32; 1047 IF ((NBUF32 == 0) || 1048 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1049 ((buf_size % 32) != 0)) { 1050 rc = EINVAL; 1051 goto fail1; 1052 } 1053 1054 if (enp->en_rx_qcount > 0) { 1055 rc = EBUSY; 1056 goto fail2; 1057 } 1058 1059 /* Set scatter buffer size */ 1060 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1061 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1062 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1063 1064 /* Enable scatter for packets not matching a filter */ 1065 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1066 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1067 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1068 1069 return (0); 1070 1071fail2: 1072 EFSYS_PROBE(fail2); 1073fail1: 1074 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1075 1076 return (rc); 1077} 1078#endif /* EFSYS_OPT_RX_SCATTER */ 1079 1080#define EFX_RX_LFSR_HASH(_enp, _insert) \ 1081 do { \ 1082 efx_oword_t oword; \ 1083 \ 1084 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1085 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1086 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1087 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1088 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1089 (_insert) ? 1 : 0); \ 1090 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1091 \ 1092 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1093 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1094 &oword); \ 1095 EFX_SET_OWORD_FIELD(oword, \ 1096 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1097 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1098 &oword); \ 1099 } \ 1100 \ 1101 _NOTE(CONSTANTCONDITION) \ 1102 } while (B_FALSE) 1103 1104#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1105 do { \ 1106 efx_oword_t oword; \ 1107 \ 1108 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1109 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1110 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1111 (_ip) ? 1 : 0); \ 1112 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1113 (_tcp) ? 0 : 1); \ 1114 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1115 (_insert) ? 1 : 0); \ 1116 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1117 \ 1118 _NOTE(CONSTANTCONDITION) \ 1119 } while (B_FALSE) 1120 1121#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1122 do { \ 1123 efx_oword_t oword; \ 1124 \ 1125 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1126 EFX_SET_OWORD_FIELD(oword, \ 1127 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1128 EFX_SET_OWORD_FIELD(oword, \ 1129 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1130 EFX_SET_OWORD_FIELD(oword, \ 1131 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1132 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1133 \ 1134 (_rc) = 0; \ 1135 \ 1136 _NOTE(CONSTANTCONDITION) \ 1137 } while (B_FALSE) 1138 1139#if EFSYS_OPT_RX_SCALE 1140 1141static __checkReturn efx_rc_t 1142siena_rx_scale_mode_set( 1143 __in efx_nic_t *enp, 1144 __in uint32_t rss_context, 1145 __in efx_rx_hash_alg_t alg, 1146 __in efx_rx_hash_type_t type, 1147 __in boolean_t insert) 1148{ 1149 efx_rc_t rc; 1150 1151 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1152 rc = EINVAL; 1153 goto fail1; 1154 } 1155 1156 switch (alg) { 1157 case EFX_RX_HASHALG_LFSR: 1158 EFX_RX_LFSR_HASH(enp, insert); 1159 break; 1160 1161 case EFX_RX_HASHALG_TOEPLITZ: 1162 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1163 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1164 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1165 1166 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1167 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1168 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1169 rc); 1170 if (rc != 0) 1171 goto fail2; 1172 1173 break; 1174 1175 default: 1176 rc = EINVAL; 1177 goto fail3; 1178 } 1179 1180 return (0); 1181 1182fail3: 1183 EFSYS_PROBE(fail3); 1184fail2: 1185 EFSYS_PROBE(fail2); 1186fail1: 1187 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1188 1189 EFX_RX_LFSR_HASH(enp, B_FALSE); 1190 1191 return (rc); 1192} 1193#endif 1194 1195#if EFSYS_OPT_RX_SCALE 1196static __checkReturn efx_rc_t 1197siena_rx_scale_key_set( 1198 __in efx_nic_t *enp, 1199 __in uint32_t rss_context, 1200 __in_ecount(n) uint8_t *key, 1201 __in size_t n) 1202{ 1203 efx_oword_t oword; 1204 unsigned int byte; 1205 unsigned int offset; 1206 efx_rc_t rc; 1207 1208 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1209 rc = EINVAL; 1210 goto fail1; 1211 } 1212 1213 byte = 0; 1214 1215 /* Write Toeplitz IPv4 hash key */ 1216 EFX_ZERO_OWORD(oword); 1217 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1218 offset > 0 && byte < n; 1219 --offset) 1220 oword.eo_u8[offset - 1] = key[byte++]; 1221 1222 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1223 1224 byte = 0; 1225 1226 /* Verify Toeplitz IPv4 hash key */ 1227 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1228 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1229 offset > 0 && byte < n; 1230 --offset) { 1231 if (oword.eo_u8[offset - 1] != key[byte++]) { 1232 rc = EFAULT; 1233 goto fail2; 1234 } 1235 } 1236 1237 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1238 goto done; 1239 1240 byte = 0; 1241 1242 /* Write Toeplitz IPv6 hash key 3 */ 1243 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1244 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1245 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1246 offset > 0 && byte < n; 1247 --offset) 1248 oword.eo_u8[offset - 1] = key[byte++]; 1249 1250 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1251 1252 /* Write Toeplitz IPv6 hash key 2 */ 1253 EFX_ZERO_OWORD(oword); 1254 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1255 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1256 offset > 0 && byte < n; 1257 --offset) 1258 oword.eo_u8[offset - 1] = key[byte++]; 1259 1260 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1261 1262 /* Write Toeplitz IPv6 hash key 1 */ 1263 EFX_ZERO_OWORD(oword); 1264 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1265 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1266 offset > 0 && byte < n; 1267 --offset) 1268 oword.eo_u8[offset - 1] = key[byte++]; 1269 1270 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1271 1272 byte = 0; 1273 1274 /* Verify Toeplitz IPv6 hash key 3 */ 1275 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1276 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1277 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1278 offset > 0 && byte < n; 1279 --offset) { 1280 if (oword.eo_u8[offset - 1] != key[byte++]) { 1281 rc = EFAULT; 1282 goto fail3; 1283 } 1284 } 1285 1286 /* Verify Toeplitz IPv6 hash key 2 */ 1287 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1288 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1289 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1290 offset > 0 && byte < n; 1291 --offset) { 1292 if (oword.eo_u8[offset - 1] != key[byte++]) { 1293 rc = EFAULT; 1294 goto fail4; 1295 } 1296 } 1297 1298 /* Verify Toeplitz IPv6 hash key 1 */ 1299 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1300 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1301 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1302 offset > 0 && byte < n; 1303 --offset) { 1304 if (oword.eo_u8[offset - 1] != key[byte++]) { 1305 rc = EFAULT; 1306 goto fail5; 1307 } 1308 } 1309 1310done: 1311 return (0); 1312 1313fail5: 1314 EFSYS_PROBE(fail5); 1315fail4: 1316 EFSYS_PROBE(fail4); 1317fail3: 1318 EFSYS_PROBE(fail3); 1319fail2: 1320 EFSYS_PROBE(fail2); 1321fail1: 1322 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1323 1324 return (rc); 1325} 1326#endif 1327 1328#if EFSYS_OPT_RX_SCALE 1329static __checkReturn efx_rc_t 1330siena_rx_scale_tbl_set( 1331 __in efx_nic_t *enp, 1332 __in uint32_t rss_context, 1333 __in_ecount(n) unsigned int *table, 1334 __in size_t n) 1335{ 1336 efx_oword_t oword; 1337 int index; 1338 efx_rc_t rc; 1339 1340 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1341 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1342 1343 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1344 rc = EINVAL; 1345 goto fail1; 1346 } 1347 1348 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1349 rc = EINVAL; 1350 goto fail2; 1351 } 1352 1353 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1354 uint32_t byte; 1355 1356 /* Calculate the entry to place in the table */ 1357 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1358 1359 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1360 1361 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1362 1363 /* Write the table */ 1364 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1365 index, &oword, B_TRUE); 1366 } 1367 1368 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1369 uint32_t byte; 1370 1371 /* Determine if we're starting a new batch */ 1372 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1373 1374 /* Read the table */ 1375 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1376 index, &oword, B_TRUE); 1377 1378 /* Verify the entry */ 1379 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1380 rc = EFAULT; 1381 goto fail3; 1382 } 1383 } 1384 1385 return (0); 1386 1387fail3: 1388 EFSYS_PROBE(fail3); 1389fail2: 1390 EFSYS_PROBE(fail2); 1391fail1: 1392 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1393 1394 return (rc); 1395} 1396#endif 1397 1398/* 1399 * Falcon/Siena pseudo-header 1400 * -------------------------- 1401 * 1402 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1403 * The pseudo-header is a byte array of one of the forms: 1404 * 1405 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1406 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1407 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1408 * 1409 * where: 1410 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1411 * LL.LL LFSR hash (16-bit big-endian) 1412 */ 1413 1414#if EFSYS_OPT_RX_SCALE 1415static __checkReturn uint32_t 1416siena_rx_prefix_hash( 1417 __in efx_nic_t *enp, 1418 __in efx_rx_hash_alg_t func, 1419 __in uint8_t *buffer) 1420{ 1421 _NOTE(ARGUNUSED(enp)) 1422 1423 switch (func) { 1424 case EFX_RX_HASHALG_TOEPLITZ: 1425 return ((buffer[12] << 24) | 1426 (buffer[13] << 16) | 1427 (buffer[14] << 8) | 1428 buffer[15]); 1429 1430 case EFX_RX_HASHALG_LFSR: 1431 return ((buffer[14] << 8) | buffer[15]); 1432 1433 default: 1434 EFSYS_ASSERT(0); 1435 return (0); 1436 } 1437} 1438#endif /* EFSYS_OPT_RX_SCALE */ 1439 1440static __checkReturn efx_rc_t 1441siena_rx_prefix_pktlen( 1442 __in efx_nic_t *enp, 1443 __in uint8_t *buffer, 1444 __out uint16_t *lengthp) 1445{ 1446 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1447 1448 /* Not supported by Falcon/Siena hardware */ 1449 EFSYS_ASSERT(0); 1450 return (ENOTSUP); 1451} 1452 1453static void 1454siena_rx_qpost( 1455 __in efx_rxq_t *erp, 1456 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1457 __in size_t size, 1458 __in unsigned int ndescs, 1459 __in unsigned int completed, 1460 __in unsigned int added) 1461{ 1462 efx_qword_t qword; 1463 unsigned int i; 1464 unsigned int offset; 1465 unsigned int id; 1466 1467 /* The client driver must not overfill the queue */ 1468 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1469 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1470 1471 id = added & (erp->er_mask); 1472 for (i = 0; i < ndescs; i++) { 1473 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1474 unsigned int, id, efsys_dma_addr_t, addrp[i], 1475 size_t, size); 1476 1477 EFX_POPULATE_QWORD_3(qword, 1478 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1479 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1480 (uint32_t)(addrp[i] & 0xffffffff), 1481 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1482 (uint32_t)(addrp[i] >> 32)); 1483 1484 offset = id * sizeof (efx_qword_t); 1485 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1486 1487 id = (id + 1) & (erp->er_mask); 1488 } 1489} 1490 1491static void 1492siena_rx_qpush( 1493 __in efx_rxq_t *erp, 1494 __in unsigned int added, 1495 __inout unsigned int *pushedp) 1496{ 1497 efx_nic_t *enp = erp->er_enp; 1498 unsigned int pushed = *pushedp; 1499 uint32_t wptr; 1500 efx_oword_t oword; 1501 efx_dword_t dword; 1502 1503 /* All descriptors are pushed */ 1504 *pushedp = added; 1505 1506 /* Push the populated descriptors out */ 1507 wptr = added & erp->er_mask; 1508 1509 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1510 1511 /* Only write the third DWORD */ 1512 EFX_POPULATE_DWORD_1(dword, 1513 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1514 1515 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1516 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1517 wptr, pushed & erp->er_mask); 1518 EFSYS_PIO_WRITE_BARRIER(); 1519 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1520 erp->er_index, &dword, B_FALSE); 1521} 1522 1523#if EFSYS_OPT_RX_PACKED_STREAM 1524static void 1525siena_rx_qpush_ps_credits( 1526 __in efx_rxq_t *erp) 1527{ 1528 /* Not supported by Siena hardware */ 1529 EFSYS_ASSERT(0); 1530} 1531 1532static uint8_t * 1533siena_rx_qps_packet_info( 1534 __in efx_rxq_t *erp, 1535 __in uint8_t *buffer, 1536 __in uint32_t buffer_length, 1537 __in uint32_t current_offset, 1538 __out uint16_t *lengthp, 1539 __out uint32_t *next_offsetp, 1540 __out uint32_t *timestamp) 1541{ 1542 /* Not supported by Siena hardware */ 1543 EFSYS_ASSERT(0); 1544 1545 return (NULL); 1546} 1547#endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1548 1549static __checkReturn efx_rc_t 1550siena_rx_qflush( 1551 __in efx_rxq_t *erp) 1552{ 1553 efx_nic_t *enp = erp->er_enp; 1554 efx_oword_t oword; 1555 uint32_t label; 1556 1557 label = erp->er_index; 1558 1559 /* Flush the queue */ 1560 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1561 FRF_AZ_RX_FLUSH_DESCQ, label); 1562 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1563 1564 return (0); 1565} 1566 1567static void 1568siena_rx_qenable( 1569 __in efx_rxq_t *erp) 1570{ 1571 efx_nic_t *enp = erp->er_enp; 1572 efx_oword_t oword; 1573 1574 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1575 1576 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1577 erp->er_index, &oword, B_TRUE); 1578 1579 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1580 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1581 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1582 1583 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1584 erp->er_index, &oword, B_TRUE); 1585} 1586 1587static __checkReturn efx_rc_t 1588siena_rx_qcreate( 1589 __in efx_nic_t *enp, 1590 __in unsigned int index, 1591 __in unsigned int label, 1592 __in efx_rxq_type_t type, 1593 __in_opt const efx_rxq_type_data_t *type_data, 1594 __in efsys_mem_t *esmp, 1595 __in size_t ndescs, 1596 __in uint32_t id, 1597 __in unsigned int flags, 1598 __in efx_evq_t *eep, 1599 __in efx_rxq_t *erp) 1600{ 1601 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1602 efx_oword_t oword; 1603 uint32_t size; 1604 boolean_t jumbo = B_FALSE; 1605 efx_rc_t rc; 1606 1607 _NOTE(ARGUNUSED(esmp)) 1608 _NOTE(ARGUNUSED(type_data)) 1609 1610 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1611 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1612 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1613 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1614 1615 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1616 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1617 1618 if (!ISP2(ndescs) || 1619 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 1620 rc = EINVAL; 1621 goto fail1; 1622 } 1623 if (index >= encp->enc_rxq_limit) { 1624 rc = EINVAL; 1625 goto fail2; 1626 } 1627 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1628 size++) 1629 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS)) 1630 break; 1631 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1632 rc = EINVAL; 1633 goto fail3; 1634 } 1635 1636 switch (type) { 1637 case EFX_RXQ_TYPE_DEFAULT: 1638 break; 1639 1640 default: 1641 rc = EINVAL; 1642 goto fail4; 1643 } 1644 1645 if (flags & EFX_RXQ_FLAG_SCATTER) { 1646#if EFSYS_OPT_RX_SCATTER 1647 jumbo = B_TRUE; 1648#else 1649 rc = EINVAL; 1650 goto fail5; 1651#endif /* EFSYS_OPT_RX_SCATTER */ 1652 } 1653 1654 /* Set up the new descriptor queue */ 1655 EFX_POPULATE_OWORD_7(oword, 1656 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1657 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1658 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1659 FRF_AZ_RX_DESCQ_LABEL, label, 1660 FRF_AZ_RX_DESCQ_SIZE, size, 1661 FRF_AZ_RX_DESCQ_TYPE, 0, 1662 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1663 1664 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1665 erp->er_index, &oword, B_TRUE); 1666 1667 return (0); 1668 1669#if !EFSYS_OPT_RX_SCATTER 1670fail5: 1671 EFSYS_PROBE(fail5); 1672#endif 1673fail4: 1674 EFSYS_PROBE(fail4); 1675fail3: 1676 EFSYS_PROBE(fail3); 1677fail2: 1678 EFSYS_PROBE(fail2); 1679fail1: 1680 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1681 1682 return (rc); 1683} 1684 1685static void 1686siena_rx_qdestroy( 1687 __in efx_rxq_t *erp) 1688{ 1689 efx_nic_t *enp = erp->er_enp; 1690 efx_oword_t oword; 1691 1692 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1693 --enp->en_rx_qcount; 1694 1695 /* Purge descriptor queue */ 1696 EFX_ZERO_OWORD(oword); 1697 1698 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1699 erp->er_index, &oword, B_TRUE); 1700 1701 /* Free the RXQ object */ 1702 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1703} 1704 1705static void 1706siena_rx_fini( 1707 __in efx_nic_t *enp) 1708{ 1709 _NOTE(ARGUNUSED(enp)) 1710} 1711 1712#endif /* EFSYS_OPT_SIENA */ 1713