efx_rx.c revision 291436
1/*- 2 * Copyright (c) 2007-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/efx_rx.c 291436 2015-11-29 05:42:49Z arybchik $"); 33 34#include "efsys.h" 35#include "efx.h" 36#include "efx_types.h" 37#include "efx_regs.h" 38#include "efx_impl.h" 39 40 41#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 42 43static __checkReturn efx_rc_t 44falconsiena_rx_init( 45 __in efx_nic_t *enp); 46 47static void 48falconsiena_rx_fini( 49 __in efx_nic_t *enp); 50 51#if EFSYS_OPT_RX_HDR_SPLIT 52static __checkReturn efx_rc_t 53falconsiena_rx_hdr_split_enable( 54 __in efx_nic_t *enp, 55 __in unsigned int hdr_buf_size, 56 __in unsigned int pld_buf_size); 57#endif /* EFSYS_OPT_RX_HDR_SPLIT */ 58 59#if EFSYS_OPT_RX_SCATTER 60static __checkReturn efx_rc_t 61falconsiena_rx_scatter_enable( 62 __in efx_nic_t *enp, 63 __in unsigned int buf_size); 64#endif /* EFSYS_OPT_RX_SCATTER */ 65 66#if EFSYS_OPT_RX_SCALE 67static __checkReturn efx_rc_t 68falconsiena_rx_scale_mode_set( 69 __in efx_nic_t *enp, 70 __in efx_rx_hash_alg_t alg, 71 __in efx_rx_hash_type_t type, 72 __in boolean_t insert); 73 74static __checkReturn efx_rc_t 75falconsiena_rx_scale_key_set( 76 __in efx_nic_t *enp, 77 __in_ecount(n) uint8_t *key, 78 __in size_t n); 79 80static __checkReturn efx_rc_t 81falconsiena_rx_scale_tbl_set( 82 __in efx_nic_t *enp, 83 __in_ecount(n) unsigned int *table, 84 __in size_t n); 85 86#endif /* EFSYS_OPT_RX_SCALE */ 87 88static void 89falconsiena_rx_qpost( 90 __in efx_rxq_t *erp, 91 __in_ecount(n) efsys_dma_addr_t *addrp, 92 __in size_t size, 93 __in unsigned int n, 94 __in unsigned int completed, 95 __in unsigned int added); 96 97static void 98falconsiena_rx_qpush( 99 __in efx_rxq_t *erp, 100 __in unsigned int added, 101 __inout unsigned int *pushedp); 102 103static __checkReturn efx_rc_t 104falconsiena_rx_qflush( 105 __in efx_rxq_t *erp); 106 107static void 108falconsiena_rx_qenable( 109 __in efx_rxq_t *erp); 110 111static __checkReturn efx_rc_t 112falconsiena_rx_qcreate( 113 __in efx_nic_t *enp, 114 __in unsigned int index, 115 __in unsigned int label, 116 __in efx_rxq_type_t type, 117 __in efsys_mem_t *esmp, 118 __in size_t n, 119 __in uint32_t id, 120 __in efx_evq_t *eep, 121 __in efx_rxq_t *erp); 122 123static void 124falconsiena_rx_qdestroy( 125 __in efx_rxq_t *erp); 126 127#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 128 129 130#if EFSYS_OPT_FALCON 131static efx_rx_ops_t __efx_rx_falcon_ops = { 132 falconsiena_rx_init, /* erxo_init */ 133 falconsiena_rx_fini, /* erxo_fini */ 134#if EFSYS_OPT_RX_HDR_SPLIT 135 falconsiena_rx_hdr_split_enable, /* erxo_hdr_split_enable */ 136#endif 137#if EFSYS_OPT_RX_SCATTER 138 falconsiena_rx_scatter_enable, /* erxo_scatter_enable */ 139#endif 140#if EFSYS_OPT_RX_SCALE 141 falconsiena_rx_scale_mode_set, /* erxo_scale_mode_set */ 142 falconsiena_rx_scale_key_set, /* erxo_scale_key_set */ 143 falconsiena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 144#endif 145 falconsiena_rx_qpost, /* erxo_qpost */ 146 falconsiena_rx_qpush, /* erxo_qpush */ 147 falconsiena_rx_qflush, /* erxo_qflush */ 148 falconsiena_rx_qenable, /* erxo_qenable */ 149 falconsiena_rx_qcreate, /* erxo_qcreate */ 150 falconsiena_rx_qdestroy, /* erxo_qdestroy */ 151}; 152#endif /* EFSYS_OPT_FALCON */ 153 154#if EFSYS_OPT_SIENA 155static efx_rx_ops_t __efx_rx_siena_ops = { 156 falconsiena_rx_init, /* erxo_init */ 157 falconsiena_rx_fini, /* erxo_fini */ 158#if EFSYS_OPT_RX_HDR_SPLIT 159 falconsiena_rx_hdr_split_enable, /* erxo_hdr_split_enable */ 160#endif 161#if EFSYS_OPT_RX_SCATTER 162 falconsiena_rx_scatter_enable, /* erxo_scatter_enable */ 163#endif 164#if EFSYS_OPT_RX_SCALE 165 falconsiena_rx_scale_mode_set, /* erxo_scale_mode_set */ 166 falconsiena_rx_scale_key_set, /* erxo_scale_key_set */ 167 falconsiena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 168#endif 169 falconsiena_rx_qpost, /* erxo_qpost */ 170 falconsiena_rx_qpush, /* erxo_qpush */ 171 falconsiena_rx_qflush, /* erxo_qflush */ 172 falconsiena_rx_qenable, /* erxo_qenable */ 173 falconsiena_rx_qcreate, /* erxo_qcreate */ 174 falconsiena_rx_qdestroy, /* erxo_qdestroy */ 175}; 176#endif /* EFSYS_OPT_SIENA */ 177 178#if EFSYS_OPT_HUNTINGTON 179static efx_rx_ops_t __efx_rx_hunt_ops = { 180 hunt_rx_init, /* erxo_init */ 181 hunt_rx_fini, /* erxo_fini */ 182#if EFSYS_OPT_RX_HDR_SPLIT 183 hunt_rx_hdr_split_enable, /* erxo_hdr_split_enable */ 184#endif 185#if EFSYS_OPT_RX_SCATTER 186 hunt_rx_scatter_enable, /* erxo_scatter_enable */ 187#endif 188#if EFSYS_OPT_RX_SCALE 189 hunt_rx_scale_mode_set, /* erxo_scale_mode_set */ 190 hunt_rx_scale_key_set, /* erxo_scale_key_set */ 191 hunt_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 192#endif 193 hunt_rx_qpost, /* erxo_qpost */ 194 hunt_rx_qpush, /* erxo_qpush */ 195 hunt_rx_qflush, /* erxo_qflush */ 196 hunt_rx_qenable, /* erxo_qenable */ 197 hunt_rx_qcreate, /* erxo_qcreate */ 198 hunt_rx_qdestroy, /* erxo_qdestroy */ 199}; 200#endif /* EFSYS_OPT_HUNTINGTON */ 201 202 203 __checkReturn efx_rc_t 204efx_rx_init( 205 __inout efx_nic_t *enp) 206{ 207 efx_rx_ops_t *erxop; 208 efx_rc_t rc; 209 210 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 211 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 212 213 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 214 rc = EINVAL; 215 goto fail1; 216 } 217 218 if (enp->en_mod_flags & EFX_MOD_RX) { 219 rc = EINVAL; 220 goto fail2; 221 } 222 223 switch (enp->en_family) { 224#if EFSYS_OPT_FALCON 225 case EFX_FAMILY_FALCON: 226 erxop = (efx_rx_ops_t *)&__efx_rx_falcon_ops; 227 break; 228#endif /* EFSYS_OPT_FALCON */ 229 230#if EFSYS_OPT_SIENA 231 case EFX_FAMILY_SIENA: 232 erxop = (efx_rx_ops_t *)&__efx_rx_siena_ops; 233 break; 234#endif /* EFSYS_OPT_SIENA */ 235 236#if EFSYS_OPT_HUNTINGTON 237 case EFX_FAMILY_HUNTINGTON: 238 erxop = (efx_rx_ops_t *)&__efx_rx_hunt_ops; 239 break; 240#endif /* EFSYS_OPT_HUNTINGTON */ 241 242 default: 243 EFSYS_ASSERT(0); 244 rc = ENOTSUP; 245 goto fail3; 246 } 247 248 if ((rc = erxop->erxo_init(enp)) != 0) 249 goto fail4; 250 251 enp->en_erxop = erxop; 252 enp->en_mod_flags |= EFX_MOD_RX; 253 return (0); 254 255fail4: 256 EFSYS_PROBE(fail4); 257fail3: 258 EFSYS_PROBE(fail3); 259fail2: 260 EFSYS_PROBE(fail2); 261fail1: 262 EFSYS_PROBE1(fail1, efx_rc_t, rc); 263 264 enp->en_erxop = NULL; 265 enp->en_mod_flags &= ~EFX_MOD_RX; 266 return (rc); 267} 268 269 void 270efx_rx_fini( 271 __in efx_nic_t *enp) 272{ 273 efx_rx_ops_t *erxop = enp->en_erxop; 274 275 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 276 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 277 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 278 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 279 280 erxop->erxo_fini(enp); 281 282 enp->en_erxop = NULL; 283 enp->en_mod_flags &= ~EFX_MOD_RX; 284} 285 286#if EFSYS_OPT_RX_HDR_SPLIT 287 __checkReturn efx_rc_t 288efx_rx_hdr_split_enable( 289 __in efx_nic_t *enp, 290 __in unsigned int hdr_buf_size, 291 __in unsigned int pld_buf_size) 292{ 293 efx_rx_ops_t *erxop = enp->en_erxop; 294 efx_rc_t rc; 295 296 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 297 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 298 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA); 299 300 if ((rc = erxop->erxo_hdr_split_enable(enp, hdr_buf_size, 301 pld_buf_size)) != 0) 302 goto fail1; 303 304 return (0); 305 306fail1: 307 EFSYS_PROBE1(fail1, efx_rc_t, rc); 308 return (rc); 309} 310#endif /* EFSYS_OPT_RX_HDR_SPLIT */ 311 312#if EFSYS_OPT_RX_SCATTER 313 __checkReturn efx_rc_t 314efx_rx_scatter_enable( 315 __in efx_nic_t *enp, 316 __in unsigned int buf_size) 317{ 318 efx_rx_ops_t *erxop = enp->en_erxop; 319 efx_rc_t rc; 320 321 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 322 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 323 324 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 325 goto fail1; 326 327 return (0); 328 329fail1: 330 EFSYS_PROBE1(fail1, efx_rc_t, rc); 331 return (rc); 332} 333#endif /* EFSYS_OPT_RX_SCATTER */ 334 335#if EFSYS_OPT_RX_SCALE 336 __checkReturn efx_rc_t 337efx_rx_hash_support_get( 338 __in efx_nic_t *enp, 339 __out efx_rx_hash_support_t *supportp) 340{ 341 efx_rc_t rc; 342 343 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 344 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 345 346 if (supportp == NULL) { 347 rc = EINVAL; 348 goto fail1; 349 } 350 351 /* Report if resources are available to insert RX hash value */ 352 *supportp = enp->en_hash_support; 353 354 return (0); 355 356fail1: 357 EFSYS_PROBE1(fail1, efx_rc_t, rc); 358 359 return (rc); 360} 361 362 __checkReturn efx_rc_t 363efx_rx_scale_support_get( 364 __in efx_nic_t *enp, 365 __out efx_rx_scale_support_t *supportp) 366{ 367 efx_rc_t rc; 368 369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 371 372 if (supportp == NULL) { 373 rc = EINVAL; 374 goto fail1; 375 } 376 377 /* Report if resources are available to support RSS */ 378 *supportp = enp->en_rss_support; 379 380 return (0); 381 382fail1: 383 EFSYS_PROBE1(fail1, efx_rc_t, rc); 384 385 return (rc); 386} 387 388 __checkReturn efx_rc_t 389efx_rx_scale_mode_set( 390 __in efx_nic_t *enp, 391 __in efx_rx_hash_alg_t alg, 392 __in efx_rx_hash_type_t type, 393 __in boolean_t insert) 394{ 395 efx_rx_ops_t *erxop = enp->en_erxop; 396 efx_rc_t rc; 397 398 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 399 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 400 401 if (erxop->erxo_scale_mode_set != NULL) { 402 if ((rc = erxop->erxo_scale_mode_set(enp, alg, 403 type, insert)) != 0) 404 goto fail1; 405 } 406 407 return (0); 408 409fail1: 410 EFSYS_PROBE1(fail1, efx_rc_t, rc); 411 return (rc); 412} 413#endif /* EFSYS_OPT_RX_SCALE */ 414 415#if EFSYS_OPT_RX_SCALE 416 __checkReturn efx_rc_t 417efx_rx_scale_key_set( 418 __in efx_nic_t *enp, 419 __in_ecount(n) uint8_t *key, 420 __in size_t n) 421{ 422 efx_rx_ops_t *erxop = enp->en_erxop; 423 efx_rc_t rc; 424 425 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 426 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 427 428 if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0) 429 goto fail1; 430 431 return (0); 432 433fail1: 434 EFSYS_PROBE1(fail1, efx_rc_t, rc); 435 436 return (rc); 437} 438#endif /* EFSYS_OPT_RX_SCALE */ 439 440#if EFSYS_OPT_RX_SCALE 441 __checkReturn efx_rc_t 442efx_rx_scale_tbl_set( 443 __in efx_nic_t *enp, 444 __in_ecount(n) unsigned int *table, 445 __in size_t n) 446{ 447 efx_rx_ops_t *erxop = enp->en_erxop; 448 efx_rc_t rc; 449 450 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 451 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 452 453 if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0) 454 goto fail1; 455 456 return (0); 457 458fail1: 459 EFSYS_PROBE1(fail1, efx_rc_t, rc); 460 461 return (rc); 462} 463#endif /* EFSYS_OPT_RX_SCALE */ 464 465 void 466efx_rx_qpost( 467 __in efx_rxq_t *erp, 468 __in_ecount(n) efsys_dma_addr_t *addrp, 469 __in size_t size, 470 __in unsigned int n, 471 __in unsigned int completed, 472 __in unsigned int added) 473{ 474 efx_nic_t *enp = erp->er_enp; 475 efx_rx_ops_t *erxop = enp->en_erxop; 476 477 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 478 479 erxop->erxo_qpost(erp, addrp, size, n, completed, added); 480} 481 482 void 483efx_rx_qpush( 484 __in efx_rxq_t *erp, 485 __in unsigned int added, 486 __inout unsigned int *pushedp) 487{ 488 efx_nic_t *enp = erp->er_enp; 489 efx_rx_ops_t *erxop = enp->en_erxop; 490 491 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 492 493 erxop->erxo_qpush(erp, added, pushedp); 494} 495 496 __checkReturn efx_rc_t 497efx_rx_qflush( 498 __in efx_rxq_t *erp) 499{ 500 efx_nic_t *enp = erp->er_enp; 501 efx_rx_ops_t *erxop = enp->en_erxop; 502 efx_rc_t rc; 503 504 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 505 506 if ((rc = erxop->erxo_qflush(erp)) != 0) 507 goto fail1; 508 509 return (0); 510 511fail1: 512 EFSYS_PROBE1(fail1, efx_rc_t, rc); 513 514 return (rc); 515} 516 517 void 518efx_rx_qenable( 519 __in efx_rxq_t *erp) 520{ 521 efx_nic_t *enp = erp->er_enp; 522 efx_rx_ops_t *erxop = enp->en_erxop; 523 524 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 525 526 erxop->erxo_qenable(erp); 527} 528 529 __checkReturn efx_rc_t 530efx_rx_qcreate( 531 __in efx_nic_t *enp, 532 __in unsigned int index, 533 __in unsigned int label, 534 __in efx_rxq_type_t type, 535 __in efsys_mem_t *esmp, 536 __in size_t n, 537 __in uint32_t id, 538 __in efx_evq_t *eep, 539 __deref_out efx_rxq_t **erpp) 540{ 541 efx_rx_ops_t *erxop = enp->en_erxop; 542 efx_rxq_t *erp; 543 efx_rc_t rc; 544 545 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 546 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 547 548 /* Allocate an RXQ object */ 549 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 550 551 if (erp == NULL) { 552 rc = ENOMEM; 553 goto fail1; 554 } 555 556 erp->er_magic = EFX_RXQ_MAGIC; 557 erp->er_enp = enp; 558 erp->er_index = index; 559 erp->er_mask = n - 1; 560 erp->er_esmp = esmp; 561 562 if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id, 563 eep, erp)) != 0) 564 goto fail2; 565 566 enp->en_rx_qcount++; 567 *erpp = erp; 568 569 return (0); 570 571fail2: 572 EFSYS_PROBE(fail2); 573 574 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 575fail1: 576 EFSYS_PROBE1(fail1, efx_rc_t, rc); 577 578 return (rc); 579} 580 581 void 582efx_rx_qdestroy( 583 __in efx_rxq_t *erp) 584{ 585 efx_nic_t *enp = erp->er_enp; 586 efx_rx_ops_t *erxop = enp->en_erxop; 587 588 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 589 590 erxop->erxo_qdestroy(erp); 591} 592 593/* 594 * Psuedo-header info for Siena/Falcon. 595 * 596 * The psuedo-header is a byte array of one of the forms: 597 * 598 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 599 * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.TT.TT.TT.TT 600 * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.LL.LL 601 * 602 * where: 603 * 604 * TT.TT.TT.TT is a 32-bit Toeplitz hash 605 * LL.LL is a 16-bit LFSR hash 606 * 607 * Hash values are in network (big-endian) byte order. 608 * 609 * 610 * On Huntington the pseudo-header is laid out as: 611 * (See also SF-109306-TC section 9) 612 * 613 * Toeplitz hash (32 bits, little-endian) 614 * Out-of-band outer VLAN tag 615 * (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag) 616 * Out-of-band inner VLAN tag 617 * (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag) 618 * Packet length (16 bits, little-endian, may be 0) 619 * MAC timestamp (32 bits, little-endian, may be 0) 620 * VLAN tag 621 * (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag) 622 * VLAN tag 623 * (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag) 624 */ 625 626 __checkReturn efx_rc_t 627efx_psuedo_hdr_pkt_length_get( 628 __in efx_nic_t *enp, 629 __in uint8_t *buffer, 630 __out uint16_t *pkt_lengthp) 631{ 632 if (enp->en_family != EFX_FAMILY_HUNTINGTON) { 633 EFSYS_ASSERT(0); 634 return (ENOTSUP); 635 } 636 637 *pkt_lengthp = buffer[8] | (buffer[9] << 8); 638 639 return (0); 640} 641 642#if EFSYS_OPT_RX_SCALE 643 644uint32_t 645efx_psuedo_hdr_hash_get( 646 __in efx_nic_t *enp, 647 __in efx_rx_hash_alg_t func, 648 __in uint8_t *buffer) 649{ 650 if (func == EFX_RX_HASHALG_TOEPLITZ) { 651 switch (enp->en_family) { 652 case EFX_FAMILY_FALCON: 653 case EFX_FAMILY_SIENA: 654 return ((buffer[12] << 24) | 655 (buffer[13] << 16) | 656 (buffer[14] << 8) | 657 buffer[15]); 658 case EFX_FAMILY_HUNTINGTON: 659 return (buffer[0] | 660 (buffer[1] << 8) | 661 (buffer[2] << 16) | 662 (buffer[3] << 24)); 663 default: 664 EFSYS_ASSERT(0); 665 return (0); 666 } 667 } else if (func == EFX_RX_HASHALG_LFSR) { 668 EFSYS_ASSERT(enp->en_family == EFX_FAMILY_FALCON || 669 enp->en_family == EFX_FAMILY_SIENA); 670 return ((buffer[14] << 8) | buffer[15]); 671 } else { 672 EFSYS_ASSERT(0); 673 return (0); 674 } 675} 676 677#endif /* EFSYS_OPT_RX_SCALE */ 678 679#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 680 681static __checkReturn efx_rc_t 682falconsiena_rx_init( 683 __in efx_nic_t *enp) 684{ 685 efx_oword_t oword; 686 unsigned int index; 687 688 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 689 690 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 691 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 692 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 693 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 694 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 695 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 696 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 697 698 /* Zero the RSS table */ 699 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 700 index++) { 701 EFX_ZERO_OWORD(oword); 702 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 703 index, &oword, B_TRUE); 704 } 705 706#if EFSYS_OPT_RX_SCALE 707 /* The RSS key and indirection table are writable. */ 708 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 709 710 /* Hardware can insert RX hash with/without RSS */ 711 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 712#endif /* EFSYS_OPT_RX_SCALE */ 713 714 return (0); 715} 716 717#if EFSYS_OPT_RX_HDR_SPLIT 718static __checkReturn efx_rc_t 719falconsiena_rx_hdr_split_enable( 720 __in efx_nic_t *enp, 721 __in unsigned int hdr_buf_size, 722 __in unsigned int pld_buf_size) 723{ 724 unsigned int nhdr32; 725 unsigned int npld32; 726 efx_oword_t oword; 727 efx_rc_t rc; 728 729 nhdr32 = hdr_buf_size / 32; 730 if ((nhdr32 == 0) || 731 (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) || 732 ((hdr_buf_size % 32) != 0)) { 733 rc = EINVAL; 734 goto fail1; 735 } 736 737 npld32 = pld_buf_size / 32; 738 if ((npld32 == 0) || 739 (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) || 740 ((pld_buf_size % 32) != 0)) { 741 rc = EINVAL; 742 goto fail2; 743 } 744 745 if (enp->en_rx_qcount > 0) { 746 rc = EBUSY; 747 goto fail3; 748 } 749 750 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 751 752 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1); 753 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32); 754 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32); 755 756 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 757 758 return (0); 759 760fail3: 761 EFSYS_PROBE(fail3); 762fail2: 763 EFSYS_PROBE(fail2); 764fail1: 765 EFSYS_PROBE1(fail1, efx_rc_t, rc); 766 767 return (rc); 768} 769#endif /* EFSYS_OPT_RX_HDR_SPLIT */ 770 771#if EFSYS_OPT_RX_SCATTER 772static __checkReturn efx_rc_t 773falconsiena_rx_scatter_enable( 774 __in efx_nic_t *enp, 775 __in unsigned int buf_size) 776{ 777 unsigned int nbuf32; 778 efx_oword_t oword; 779 efx_rc_t rc; 780 781 nbuf32 = buf_size / 32; 782 if ((nbuf32 == 0) || 783 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 784 ((buf_size % 32) != 0)) { 785 rc = EINVAL; 786 goto fail1; 787 } 788 789 if (enp->en_rx_qcount > 0) { 790 rc = EBUSY; 791 goto fail2; 792 } 793 794 /* Set scatter buffer size */ 795 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 796 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 797 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 798 799 /* Enable scatter for packets not matching a filter */ 800 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 801 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 802 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 803 804 return (0); 805 806fail2: 807 EFSYS_PROBE(fail2); 808fail1: 809 EFSYS_PROBE1(fail1, efx_rc_t, rc); 810 811 return (rc); 812} 813#endif /* EFSYS_OPT_RX_SCATTER */ 814 815 816#define EFX_RX_LFSR_HASH(_enp, _insert) \ 817 do { \ 818 efx_oword_t oword; \ 819 \ 820 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 821 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 822 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 823 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 824 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 825 (_insert) ? 1 : 0); \ 826 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 827 \ 828 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 829 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 830 &oword); \ 831 EFX_SET_OWORD_FIELD(oword, \ 832 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 833 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 834 &oword); \ 835 } \ 836 \ 837 _NOTE(CONSTANTCONDITION) \ 838 } while (B_FALSE) 839 840#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 841 do { \ 842 efx_oword_t oword; \ 843 \ 844 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 845 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 846 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 847 (_ip) ? 1 : 0); \ 848 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 849 (_tcp) ? 0 : 1); \ 850 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 851 (_insert) ? 1 : 0); \ 852 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 853 \ 854 _NOTE(CONSTANTCONDITION) \ 855 } while (B_FALSE) 856 857#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 858 do { \ 859 efx_oword_t oword; \ 860 \ 861 if ((_enp)->en_family == EFX_FAMILY_FALCON) { \ 862 (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \ 863 break; \ 864 } \ 865 \ 866 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 867 EFX_SET_OWORD_FIELD(oword, \ 868 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 869 EFX_SET_OWORD_FIELD(oword, \ 870 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 871 EFX_SET_OWORD_FIELD(oword, \ 872 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 873 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 874 \ 875 (_rc) = 0; \ 876 \ 877 _NOTE(CONSTANTCONDITION) \ 878 } while (B_FALSE) 879 880 881#if EFSYS_OPT_RX_SCALE 882 883static __checkReturn efx_rc_t 884falconsiena_rx_scale_mode_set( 885 __in efx_nic_t *enp, 886 __in efx_rx_hash_alg_t alg, 887 __in efx_rx_hash_type_t type, 888 __in boolean_t insert) 889{ 890 efx_rc_t rc; 891 892 switch (alg) { 893 case EFX_RX_HASHALG_LFSR: 894 EFX_RX_LFSR_HASH(enp, insert); 895 break; 896 897 case EFX_RX_HASHALG_TOEPLITZ: 898 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 899 type & (1 << EFX_RX_HASH_IPV4), 900 type & (1 << EFX_RX_HASH_TCPIPV4)); 901 902 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 903 type & (1 << EFX_RX_HASH_IPV6), 904 type & (1 << EFX_RX_HASH_TCPIPV6), 905 rc); 906 if (rc != 0) 907 goto fail1; 908 909 break; 910 911 default: 912 rc = EINVAL; 913 goto fail2; 914 } 915 916 return (0); 917 918fail2: 919 EFSYS_PROBE(fail2); 920fail1: 921 EFSYS_PROBE1(fail1, efx_rc_t, rc); 922 923 EFX_RX_LFSR_HASH(enp, B_FALSE); 924 925 return (rc); 926} 927#endif 928 929#if EFSYS_OPT_RX_SCALE 930static __checkReturn efx_rc_t 931falconsiena_rx_scale_key_set( 932 __in efx_nic_t *enp, 933 __in_ecount(n) uint8_t *key, 934 __in size_t n) 935{ 936 efx_oword_t oword; 937 unsigned int byte; 938 unsigned int offset; 939 efx_rc_t rc; 940 941 byte = 0; 942 943 /* Write Toeplitz IPv4 hash key */ 944 EFX_ZERO_OWORD(oword); 945 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 946 offset > 0 && byte < n; 947 --offset) 948 oword.eo_u8[offset - 1] = key[byte++]; 949 950 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 951 952 byte = 0; 953 954 /* Verify Toeplitz IPv4 hash key */ 955 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 956 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 957 offset > 0 && byte < n; 958 --offset) { 959 if (oword.eo_u8[offset - 1] != key[byte++]) { 960 rc = EFAULT; 961 goto fail1; 962 } 963 } 964 965 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 966 goto done; 967 968 EFSYS_ASSERT3U(enp->en_family, !=, EFX_FAMILY_FALCON); 969 970 byte = 0; 971 972 /* Write Toeplitz IPv6 hash key 3 */ 973 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 974 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 975 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 976 offset > 0 && byte < n; 977 --offset) 978 oword.eo_u8[offset - 1] = key[byte++]; 979 980 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 981 982 /* Write Toeplitz IPv6 hash key 2 */ 983 EFX_ZERO_OWORD(oword); 984 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 985 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 986 offset > 0 && byte < n; 987 --offset) 988 oword.eo_u8[offset - 1] = key[byte++]; 989 990 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 991 992 /* Write Toeplitz IPv6 hash key 1 */ 993 EFX_ZERO_OWORD(oword); 994 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 995 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 996 offset > 0 && byte < n; 997 --offset) 998 oword.eo_u8[offset - 1] = key[byte++]; 999 1000 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1001 1002 byte = 0; 1003 1004 /* Verify Toeplitz IPv6 hash key 3 */ 1005 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1006 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1007 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1008 offset > 0 && byte < n; 1009 --offset) { 1010 if (oword.eo_u8[offset - 1] != key[byte++]) { 1011 rc = EFAULT; 1012 goto fail2; 1013 } 1014 } 1015 1016 /* Verify Toeplitz IPv6 hash key 2 */ 1017 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1018 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1019 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1020 offset > 0 && byte < n; 1021 --offset) { 1022 if (oword.eo_u8[offset - 1] != key[byte++]) { 1023 rc = EFAULT; 1024 goto fail3; 1025 } 1026 } 1027 1028 /* Verify Toeplitz IPv6 hash key 1 */ 1029 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1030 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1031 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1032 offset > 0 && byte < n; 1033 --offset) { 1034 if (oword.eo_u8[offset - 1] != key[byte++]) { 1035 rc = EFAULT; 1036 goto fail4; 1037 } 1038 } 1039 1040done: 1041 return (0); 1042 1043fail4: 1044 EFSYS_PROBE(fail4); 1045fail3: 1046 EFSYS_PROBE(fail3); 1047fail2: 1048 EFSYS_PROBE(fail2); 1049fail1: 1050 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1051 1052 return (rc); 1053} 1054#endif 1055 1056#if EFSYS_OPT_RX_SCALE 1057static __checkReturn efx_rc_t 1058falconsiena_rx_scale_tbl_set( 1059 __in efx_nic_t *enp, 1060 __in_ecount(n) unsigned int *table, 1061 __in size_t n) 1062{ 1063 efx_oword_t oword; 1064 int index; 1065 efx_rc_t rc; 1066 1067 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1068 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1069 1070 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1071 rc = EINVAL; 1072 goto fail1; 1073 } 1074 1075 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1076 uint32_t byte; 1077 1078 /* Calculate the entry to place in the table */ 1079 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1080 1081 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1082 1083 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1084 1085 /* Write the table */ 1086 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1087 index, &oword, B_TRUE); 1088 } 1089 1090 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1091 uint32_t byte; 1092 1093 /* Determine if we're starting a new batch */ 1094 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1095 1096 /* Read the table */ 1097 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1098 index, &oword, B_TRUE); 1099 1100 /* Verify the entry */ 1101 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1102 rc = EFAULT; 1103 goto fail2; 1104 } 1105 } 1106 1107 return (0); 1108 1109fail2: 1110 EFSYS_PROBE(fail2); 1111fail1: 1112 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1113 1114 return (rc); 1115} 1116#endif 1117 1118static void 1119falconsiena_rx_qpost( 1120 __in efx_rxq_t *erp, 1121 __in_ecount(n) efsys_dma_addr_t *addrp, 1122 __in size_t size, 1123 __in unsigned int n, 1124 __in unsigned int completed, 1125 __in unsigned int added) 1126{ 1127 efx_qword_t qword; 1128 unsigned int i; 1129 unsigned int offset; 1130 unsigned int id; 1131 1132 /* The client driver must not overfill the queue */ 1133 EFSYS_ASSERT3U(added - completed + n, <=, 1134 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1135 1136 id = added & (erp->er_mask); 1137 for (i = 0; i < n; i++) { 1138 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1139 unsigned int, id, efsys_dma_addr_t, addrp[i], 1140 size_t, size); 1141 1142 EFX_POPULATE_QWORD_3(qword, 1143 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1144 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1145 (uint32_t)(addrp[i] & 0xffffffff), 1146 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1147 (uint32_t)(addrp[i] >> 32)); 1148 1149 offset = id * sizeof (efx_qword_t); 1150 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1151 1152 id = (id + 1) & (erp->er_mask); 1153 } 1154} 1155 1156static void 1157falconsiena_rx_qpush( 1158 __in efx_rxq_t *erp, 1159 __in unsigned int added, 1160 __inout unsigned int *pushedp) 1161{ 1162 efx_nic_t *enp = erp->er_enp; 1163 unsigned int pushed = *pushedp; 1164 uint32_t wptr; 1165 efx_oword_t oword; 1166 efx_dword_t dword; 1167 1168 /* All descriptors are pushed */ 1169 *pushedp = added; 1170 1171 /* Push the populated descriptors out */ 1172 wptr = added & erp->er_mask; 1173 1174 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1175 1176 /* Only write the third DWORD */ 1177 EFX_POPULATE_DWORD_1(dword, 1178 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1179 1180 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1181 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1182 wptr, pushed & erp->er_mask); 1183 EFSYS_PIO_WRITE_BARRIER(); 1184 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1185 erp->er_index, &dword, B_FALSE); 1186} 1187 1188static __checkReturn efx_rc_t 1189falconsiena_rx_qflush( 1190 __in efx_rxq_t *erp) 1191{ 1192 efx_nic_t *enp = erp->er_enp; 1193 efx_oword_t oword; 1194 uint32_t label; 1195 1196 label = erp->er_index; 1197 1198 /* Flush the queue */ 1199 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1200 FRF_AZ_RX_FLUSH_DESCQ, label); 1201 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1202 1203 return (0); 1204} 1205 1206static void 1207falconsiena_rx_qenable( 1208 __in efx_rxq_t *erp) 1209{ 1210 efx_nic_t *enp = erp->er_enp; 1211 efx_oword_t oword; 1212 1213 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1214 1215 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1216 erp->er_index, &oword, B_TRUE); 1217 1218 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1219 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1220 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1221 1222 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1223 erp->er_index, &oword, B_TRUE); 1224} 1225 1226static __checkReturn efx_rc_t 1227falconsiena_rx_qcreate( 1228 __in efx_nic_t *enp, 1229 __in unsigned int index, 1230 __in unsigned int label, 1231 __in efx_rxq_type_t type, 1232 __in efsys_mem_t *esmp, 1233 __in size_t n, 1234 __in uint32_t id, 1235 __in efx_evq_t *eep, 1236 __in efx_rxq_t *erp) 1237{ 1238 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1239 efx_oword_t oword; 1240 uint32_t size; 1241 boolean_t split; 1242 boolean_t jumbo; 1243 efx_rc_t rc; 1244 1245 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1246 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1247 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1248 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1249 1250 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1251 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1252 1253 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 1254 rc = EINVAL; 1255 goto fail1; 1256 } 1257 if (index >= encp->enc_rxq_limit) { 1258 rc = EINVAL; 1259 goto fail2; 1260 } 1261 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1262 size++) 1263 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 1264 break; 1265 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1266 rc = EINVAL; 1267 goto fail3; 1268 } 1269 1270 switch (type) { 1271 case EFX_RXQ_TYPE_DEFAULT: 1272 split = B_FALSE; 1273 jumbo = B_FALSE; 1274 break; 1275 1276#if EFSYS_OPT_RX_HDR_SPLIT 1277 case EFX_RXQ_TYPE_SPLIT_HEADER: 1278 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) { 1279 rc = EINVAL; 1280 goto fail4; 1281 } 1282 split = B_TRUE; 1283 jumbo = B_TRUE; 1284 break; 1285 1286 case EFX_RXQ_TYPE_SPLIT_PAYLOAD: 1287 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) { 1288 rc = EINVAL; 1289 goto fail4; 1290 } 1291 split = B_FALSE; 1292 jumbo = B_TRUE; 1293 break; 1294#endif /* EFSYS_OPT_RX_HDR_SPLIT */ 1295 1296#if EFSYS_OPT_RX_SCATTER 1297 case EFX_RXQ_TYPE_SCATTER: 1298 if (enp->en_family < EFX_FAMILY_SIENA) { 1299 rc = EINVAL; 1300 goto fail4; 1301 } 1302 split = B_FALSE; 1303 jumbo = B_TRUE; 1304 break; 1305#endif /* EFSYS_OPT_RX_SCATTER */ 1306 1307 default: 1308 rc = EINVAL; 1309 goto fail4; 1310 } 1311 1312 /* Set up the new descriptor queue */ 1313 EFX_POPULATE_OWORD_10(oword, 1314 FRF_CZ_RX_HDR_SPLIT, split, 1315 FRF_AZ_RX_ISCSI_DDIG_EN, 0, 1316 FRF_AZ_RX_ISCSI_HDIG_EN, 0, 1317 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1318 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1319 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1320 FRF_AZ_RX_DESCQ_LABEL, label, 1321 FRF_AZ_RX_DESCQ_SIZE, size, 1322 FRF_AZ_RX_DESCQ_TYPE, 0, 1323 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1324 1325 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1326 erp->er_index, &oword, B_TRUE); 1327 1328 return (0); 1329 1330fail4: 1331 EFSYS_PROBE(fail4); 1332fail3: 1333 EFSYS_PROBE(fail3); 1334fail2: 1335 EFSYS_PROBE(fail2); 1336fail1: 1337 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1338 1339 return (rc); 1340} 1341 1342static void 1343falconsiena_rx_qdestroy( 1344 __in efx_rxq_t *erp) 1345{ 1346 efx_nic_t *enp = erp->er_enp; 1347 efx_oword_t oword; 1348 1349 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1350 --enp->en_rx_qcount; 1351 1352 /* Purge descriptor queue */ 1353 EFX_ZERO_OWORD(oword); 1354 1355 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1356 erp->er_index, &oword, B_TRUE); 1357 1358 /* Free the RXQ object */ 1359 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1360} 1361 1362static void 1363falconsiena_rx_fini( 1364 __in efx_nic_t *enp) 1365{ 1366 _NOTE(ARGUNUSED(enp)) 1367} 1368 1369#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1370