1/*- 2 * Copyright (c) 2012-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: stable/11/sys/dev/sfxge/common/ef10_rx.c 350411 2019-07-29 10:42:21Z arybchik $"); 33 34#include "efx.h" 35#include "efx_impl.h" 36 37 38#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 39 40 41static __checkReturn efx_rc_t 42efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t size, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter) 50{ 51 efx_mcdi_req_t req; 52 EFX_MCDI_DECLARE_BUF(payload, 53 MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)), 54 MC_CMD_INIT_RXQ_OUT_LEN); 55 int npages = EFX_RXQ_NBUFS(size); 56 int i; 57 efx_qword_t *dma_addr; 58 uint64_t addr; 59 efx_rc_t rc; 60 61 /* If this changes, then the payload size might need to change. */ 62 EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0); 63 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); 64 65 req.emr_cmd = MC_CMD_INIT_RXQ; 66 req.emr_in_buf = payload; 67 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages); 68 req.emr_out_buf = payload; 69 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN; 70 71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size); 72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq); 73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label); 74 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance); 75 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS, 76 INIT_RXQ_IN_FLAG_BUFF_MODE, 0, 77 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0, 78 INIT_RXQ_IN_FLAG_TIMESTAMP, 0, 79 INIT_RXQ_IN_CRC_MODE, 0, 80 INIT_RXQ_IN_FLAG_PREFIX, 1, 81 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter); 82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0); 83 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 84 85 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 86 addr = EFSYS_MEM_ADDR(esmp); 87 88 for (i = 0; i < npages; i++) { 89 EFX_POPULATE_QWORD_2(*dma_addr, 90 EFX_DWORD_1, (uint32_t)(addr >> 32), 91 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 92 93 dma_addr++; 94 addr += EFX_BUF_SIZE; 95 } 96 97 efx_mcdi_execute(enp, &req); 98 99 if (req.emr_rc != 0) { 100 rc = req.emr_rc; 101 goto fail1; 102 } 103 104 return (0); 105 106fail1: 107 EFSYS_PROBE1(fail1, efx_rc_t, rc); 108 109 return (rc); 110} 111 112static __checkReturn efx_rc_t 113efx_mcdi_fini_rxq( 114 __in efx_nic_t *enp, 115 __in uint32_t instance) 116{ 117 efx_mcdi_req_t req; 118 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN, 119 MC_CMD_FINI_RXQ_OUT_LEN); 120 efx_rc_t rc; 121 122 req.emr_cmd = MC_CMD_FINI_RXQ; 123 req.emr_in_buf = payload; 124 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 125 req.emr_out_buf = payload; 126 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 127 128 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 129 130 efx_mcdi_execute_quiet(enp, &req); 131 132 if (req.emr_rc != 0) { 133 rc = req.emr_rc; 134 goto fail1; 135 } 136 137 return (0); 138 139fail1: 140 /* 141 * EALREADY is not an error, but indicates that the MC has rebooted and 142 * that the RXQ has already been destroyed. 143 */ 144 if (rc != EALREADY) 145 EFSYS_PROBE1(fail1, efx_rc_t, rc); 146 147 return (rc); 148} 149 150#if EFSYS_OPT_RX_SCALE 151static __checkReturn efx_rc_t 152efx_mcdi_rss_context_alloc( 153 __in efx_nic_t *enp, 154 __in efx_rx_scale_support_t scale_support, 155 __in uint32_t num_queues, 156 __out uint32_t *rss_contextp) 157{ 158 efx_mcdi_req_t req; 159 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 160 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 161 uint32_t rss_context; 162 uint32_t context_type; 163 efx_rc_t rc; 164 165 if (num_queues > EFX_MAXRSS) { 166 rc = EINVAL; 167 goto fail1; 168 } 169 170 switch (scale_support) { 171 case EFX_RX_SCALE_EXCLUSIVE: 172 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 173 break; 174 case EFX_RX_SCALE_SHARED: 175 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 176 break; 177 default: 178 rc = EINVAL; 179 goto fail2; 180 } 181 182 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 183 req.emr_in_buf = payload; 184 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 185 req.emr_out_buf = payload; 186 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 187 188 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 189 EVB_PORT_ID_ASSIGNED); 190 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 191 /* NUM_QUEUES is only used to validate indirection table offsets */ 192 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 193 194 efx_mcdi_execute(enp, &req); 195 196 if (req.emr_rc != 0) { 197 rc = req.emr_rc; 198 goto fail3; 199 } 200 201 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 202 rc = EMSGSIZE; 203 goto fail4; 204 } 205 206 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 207 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 208 rc = ENOENT; 209 goto fail5; 210 } 211 212 *rss_contextp = rss_context; 213 214 return (0); 215 216fail5: 217 EFSYS_PROBE(fail5); 218fail4: 219 EFSYS_PROBE(fail4); 220fail3: 221 EFSYS_PROBE(fail3); 222fail2: 223 EFSYS_PROBE(fail2); 224fail1: 225 EFSYS_PROBE1(fail1, efx_rc_t, rc); 226 227 return (rc); 228} 229#endif /* EFSYS_OPT_RX_SCALE */ 230 231#if EFSYS_OPT_RX_SCALE 232static efx_rc_t 233efx_mcdi_rss_context_free( 234 __in efx_nic_t *enp, 235 __in uint32_t rss_context) 236{ 237 efx_mcdi_req_t req; 238 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 239 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN); 240 efx_rc_t rc; 241 242 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 243 rc = EINVAL; 244 goto fail1; 245 } 246 247 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 248 req.emr_in_buf = payload; 249 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 250 req.emr_out_buf = payload; 251 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 252 253 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 254 255 efx_mcdi_execute_quiet(enp, &req); 256 257 if (req.emr_rc != 0) { 258 rc = req.emr_rc; 259 goto fail2; 260 } 261 262 return (0); 263 264fail2: 265 EFSYS_PROBE(fail2); 266fail1: 267 EFSYS_PROBE1(fail1, efx_rc_t, rc); 268 269 return (rc); 270} 271#endif /* EFSYS_OPT_RX_SCALE */ 272 273#if EFSYS_OPT_RX_SCALE 274static efx_rc_t 275efx_mcdi_rss_context_set_flags( 276 __in efx_nic_t *enp, 277 __in uint32_t rss_context, 278 __in efx_rx_hash_type_t type) 279{ 280 efx_mcdi_req_t req; 281 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 282 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN); 283 efx_rc_t rc; 284 285 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 286 rc = EINVAL; 287 goto fail1; 288 } 289 290 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 291 req.emr_in_buf = payload; 292 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 293 req.emr_out_buf = payload; 294 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 295 296 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 297 rss_context); 298 299 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 300 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 301 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 302 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 303 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 304 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 305 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 306 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 307 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); 308 309 efx_mcdi_execute(enp, &req); 310 311 if (req.emr_rc != 0) { 312 rc = req.emr_rc; 313 goto fail2; 314 } 315 316 return (0); 317 318fail2: 319 EFSYS_PROBE(fail2); 320fail1: 321 EFSYS_PROBE1(fail1, efx_rc_t, rc); 322 323 return (rc); 324} 325#endif /* EFSYS_OPT_RX_SCALE */ 326 327#if EFSYS_OPT_RX_SCALE 328static efx_rc_t 329efx_mcdi_rss_context_set_key( 330 __in efx_nic_t *enp, 331 __in uint32_t rss_context, 332 __in_ecount(n) uint8_t *key, 333 __in size_t n) 334{ 335 efx_mcdi_req_t req; 336 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 337 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN); 338 efx_rc_t rc; 339 340 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 341 rc = EINVAL; 342 goto fail1; 343 } 344 345 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 346 req.emr_in_buf = payload; 347 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 348 req.emr_out_buf = payload; 349 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 350 351 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 352 rss_context); 353 354 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 355 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 356 rc = EINVAL; 357 goto fail2; 358 } 359 360 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 361 key, n); 362 363 efx_mcdi_execute(enp, &req); 364 365 if (req.emr_rc != 0) { 366 rc = req.emr_rc; 367 goto fail3; 368 } 369 370 return (0); 371 372fail3: 373 EFSYS_PROBE(fail3); 374fail2: 375 EFSYS_PROBE(fail2); 376fail1: 377 EFSYS_PROBE1(fail1, efx_rc_t, rc); 378 379 return (rc); 380} 381#endif /* EFSYS_OPT_RX_SCALE */ 382 383#if EFSYS_OPT_RX_SCALE 384static efx_rc_t 385efx_mcdi_rss_context_set_table( 386 __in efx_nic_t *enp, 387 __in uint32_t rss_context, 388 __in_ecount(n) unsigned int *table, 389 __in size_t n) 390{ 391 efx_mcdi_req_t req; 392 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 393 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN); 394 uint8_t *req_table; 395 int i, rc; 396 397 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 398 rc = EINVAL; 399 goto fail1; 400 } 401 402 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 403 req.emr_in_buf = payload; 404 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 405 req.emr_out_buf = payload; 406 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 407 408 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 409 rss_context); 410 411 req_table = 412 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 413 414 for (i = 0; 415 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 416 i++) { 417 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 418 } 419 420 efx_mcdi_execute(enp, &req); 421 422 if (req.emr_rc != 0) { 423 rc = req.emr_rc; 424 goto fail2; 425 } 426 427 return (0); 428 429fail2: 430 EFSYS_PROBE(fail2); 431fail1: 432 EFSYS_PROBE1(fail1, efx_rc_t, rc); 433 434 return (rc); 435} 436#endif /* EFSYS_OPT_RX_SCALE */ 437 438 439 __checkReturn efx_rc_t 440ef10_rx_init( 441 __in efx_nic_t *enp) 442{ 443#if EFSYS_OPT_RX_SCALE 444 445 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 446 &enp->en_rss_context) == 0) { 447 /* 448 * Allocated an exclusive RSS context, which allows both the 449 * indirection table and key to be modified. 450 */ 451 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 452 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 453 } else { 454 /* 455 * Failed to allocate an exclusive RSS context. Continue 456 * operation without support for RSS. The pseudo-header in 457 * received packets will not contain a Toeplitz hash value. 458 */ 459 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 460 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 461 } 462 463#endif /* EFSYS_OPT_RX_SCALE */ 464 465 return (0); 466} 467 468#if EFSYS_OPT_RX_SCATTER 469 __checkReturn efx_rc_t 470ef10_rx_scatter_enable( 471 __in efx_nic_t *enp, 472 __in unsigned int buf_size) 473{ 474 _NOTE(ARGUNUSED(enp, buf_size)) 475 return (0); 476} 477#endif /* EFSYS_OPT_RX_SCATTER */ 478 479#if EFSYS_OPT_RX_SCALE 480 __checkReturn efx_rc_t 481ef10_rx_scale_mode_set( 482 __in efx_nic_t *enp, 483 __in efx_rx_hash_alg_t alg, 484 __in efx_rx_hash_type_t type, 485 __in boolean_t insert) 486{ 487 efx_rc_t rc; 488 489 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 490 EFSYS_ASSERT3U(insert, ==, B_TRUE); 491 492 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 493 rc = EINVAL; 494 goto fail1; 495 } 496 497 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 498 rc = ENOTSUP; 499 goto fail2; 500 } 501 502 if ((rc = efx_mcdi_rss_context_set_flags(enp, 503 enp->en_rss_context, type)) != 0) 504 goto fail3; 505 506 return (0); 507 508fail3: 509 EFSYS_PROBE(fail3); 510fail2: 511 EFSYS_PROBE(fail2); 512fail1: 513 EFSYS_PROBE1(fail1, efx_rc_t, rc); 514 515 return (rc); 516} 517#endif /* EFSYS_OPT_RX_SCALE */ 518 519#if EFSYS_OPT_RX_SCALE 520 __checkReturn efx_rc_t 521ef10_rx_scale_key_set( 522 __in efx_nic_t *enp, 523 __in_ecount(n) uint8_t *key, 524 __in size_t n) 525{ 526 efx_rc_t rc; 527 528 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 529 rc = ENOTSUP; 530 goto fail1; 531 } 532 533 if ((rc = efx_mcdi_rss_context_set_key(enp, 534 enp->en_rss_context, key, n)) != 0) 535 goto fail2; 536 537 return (0); 538 539fail2: 540 EFSYS_PROBE(fail2); 541fail1: 542 EFSYS_PROBE1(fail1, efx_rc_t, rc); 543 544 return (rc); 545} 546#endif /* EFSYS_OPT_RX_SCALE */ 547 548#if EFSYS_OPT_RX_SCALE 549 __checkReturn efx_rc_t 550ef10_rx_scale_tbl_set( 551 __in efx_nic_t *enp, 552 __in_ecount(n) unsigned int *table, 553 __in size_t n) 554{ 555 efx_rc_t rc; 556 557 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 558 rc = ENOTSUP; 559 goto fail1; 560 } 561 562 if ((rc = efx_mcdi_rss_context_set_table(enp, 563 enp->en_rss_context, table, n)) != 0) 564 goto fail2; 565 566 return (0); 567 568fail2: 569 EFSYS_PROBE(fail2); 570fail1: 571 EFSYS_PROBE1(fail1, efx_rc_t, rc); 572 573 return (rc); 574} 575#endif /* EFSYS_OPT_RX_SCALE */ 576 577 578/* 579 * EF10 RX pseudo-header 580 * --------------------- 581 * 582 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 583 * 584 * +00: Toeplitz hash value. 585 * (32bit little-endian) 586 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 587 * (16bit big-endian) 588 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 589 * (16bit big-endian) 590 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 591 * (16bit little-endian) 592 * +10: MAC timestamp. Zero if timestamping is not enabled. 593 * (32bit little-endian) 594 * 595 * See "The RX Pseudo-header" in SF-109306-TC. 596 */ 597 598 __checkReturn efx_rc_t 599ef10_rx_prefix_pktlen( 600 __in efx_nic_t *enp, 601 __in uint8_t *buffer, 602 __out uint16_t *lengthp) 603{ 604 _NOTE(ARGUNUSED(enp)) 605 606 /* 607 * The RX pseudo-header contains the packet length, excluding the 608 * pseudo-header. If the hardware receive datapath was operating in 609 * cut-through mode then the length in the RX pseudo-header will be 610 * zero, and the packet length must be obtained from the DMA length 611 * reported in the RX event. 612 */ 613 *lengthp = buffer[8] | (buffer[9] << 8); 614 return (0); 615} 616 617#if EFSYS_OPT_RX_SCALE 618 __checkReturn uint32_t 619ef10_rx_prefix_hash( 620 __in efx_nic_t *enp, 621 __in efx_rx_hash_alg_t func, 622 __in uint8_t *buffer) 623{ 624 _NOTE(ARGUNUSED(enp)) 625 626 switch (func) { 627 case EFX_RX_HASHALG_TOEPLITZ: 628 return (buffer[0] | 629 (buffer[1] << 8) | 630 (buffer[2] << 16) | 631 (buffer[3] << 24)); 632 633 default: 634 EFSYS_ASSERT(0); 635 return (0); 636 } 637} 638#endif /* EFSYS_OPT_RX_SCALE */ 639 640 void 641ef10_rx_qpost( 642 __in efx_rxq_t *erp, 643 __in_ecount(n) efsys_dma_addr_t *addrp, 644 __in size_t size, 645 __in unsigned int n, 646 __in unsigned int completed, 647 __in unsigned int added) 648{ 649 efx_qword_t qword; 650 unsigned int i; 651 unsigned int offset; 652 unsigned int id; 653 654 _NOTE(ARGUNUSED(completed)) 655 656 /* The client driver must not overfill the queue */ 657 EFSYS_ASSERT3U(added - completed + n, <=, 658 EFX_RXQ_LIMIT(erp->er_mask + 1)); 659 660 id = added & (erp->er_mask); 661 for (i = 0; i < n; i++) { 662 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 663 unsigned int, id, efsys_dma_addr_t, addrp[i], 664 size_t, size); 665 666 EFX_POPULATE_QWORD_3(qword, 667 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 668 ESF_DZ_RX_KER_BUF_ADDR_DW0, 669 (uint32_t)(addrp[i] & 0xffffffff), 670 ESF_DZ_RX_KER_BUF_ADDR_DW1, 671 (uint32_t)(addrp[i] >> 32)); 672 673 offset = id * sizeof (efx_qword_t); 674 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 675 676 id = (id + 1) & (erp->er_mask); 677 } 678} 679 680 void 681ef10_rx_qpush( 682 __in efx_rxq_t *erp, 683 __in unsigned int added, 684 __inout unsigned int *pushedp) 685{ 686 efx_nic_t *enp = erp->er_enp; 687 unsigned int pushed = *pushedp; 688 uint32_t wptr; 689 efx_dword_t dword; 690 691 /* Hardware has alignment restriction for WPTR */ 692 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN); 693 if (pushed == wptr) 694 return; 695 696 *pushedp = wptr; 697 698 /* Push the populated descriptors out */ 699 wptr &= erp->er_mask; 700 701 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 702 703 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 704 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 705 wptr, pushed & erp->er_mask); 706 EFSYS_PIO_WRITE_BARRIER(); 707 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 708 erp->er_index, &dword, B_FALSE); 709} 710 711 __checkReturn efx_rc_t 712ef10_rx_qflush( 713 __in efx_rxq_t *erp) 714{ 715 efx_nic_t *enp = erp->er_enp; 716 efx_rc_t rc; 717 718 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 719 goto fail1; 720 721 return (0); 722 723fail1: 724 /* 725 * EALREADY is not an error, but indicates that the MC has rebooted and 726 * that the RXQ has already been destroyed. Callers need to know that 727 * the RXQ flush has completed to avoid waiting until timeout for a 728 * flush done event that will not be delivered. 729 */ 730 if (rc != EALREADY) 731 EFSYS_PROBE1(fail1, efx_rc_t, rc); 732 733 return (rc); 734} 735 736 void 737ef10_rx_qenable( 738 __in efx_rxq_t *erp) 739{ 740 /* FIXME */ 741 _NOTE(ARGUNUSED(erp)) 742 /* FIXME */ 743} 744 745 __checkReturn efx_rc_t 746ef10_rx_qcreate( 747 __in efx_nic_t *enp, 748 __in unsigned int index, 749 __in unsigned int label, 750 __in efx_rxq_type_t type, 751 __in efsys_mem_t *esmp, 752 __in size_t n, 753 __in uint32_t id, 754 __in efx_evq_t *eep, 755 __in efx_rxq_t *erp) 756{ 757 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 758 efx_rc_t rc; 759 boolean_t disable_scatter; 760 761 _NOTE(ARGUNUSED(id, erp)) 762 763 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 764 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 765 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 766 767 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 768 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 769 770 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 771 rc = EINVAL; 772 goto fail1; 773 } 774 if (index >= encp->enc_rxq_limit) { 775 rc = EINVAL; 776 goto fail2; 777 } 778 779 /* Scatter can only be disabled if the firmware supports doing so */ 780 if (type == EFX_RXQ_TYPE_SCATTER) 781 disable_scatter = B_FALSE; 782 else 783 disable_scatter = encp->enc_rx_disable_scatter_supported; 784 785 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, 786 esmp, disable_scatter)) != 0) 787 goto fail3; 788 789 erp->er_eep = eep; 790 erp->er_label = label; 791 792 ef10_ev_rxlabel_init(eep, erp, label); 793 794 return (0); 795 796fail3: 797 EFSYS_PROBE(fail3); 798fail2: 799 EFSYS_PROBE(fail2); 800fail1: 801 EFSYS_PROBE1(fail1, efx_rc_t, rc); 802 803 return (rc); 804} 805 806 void 807ef10_rx_qdestroy( 808 __in efx_rxq_t *erp) 809{ 810 efx_nic_t *enp = erp->er_enp; 811 efx_evq_t *eep = erp->er_eep; 812 unsigned int label = erp->er_label; 813 814 ef10_ev_rxlabel_fini(eep, label); 815 816 EFSYS_ASSERT(enp->en_rx_qcount != 0); 817 --enp->en_rx_qcount; 818 819 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 820} 821 822 void 823ef10_rx_fini( 824 __in efx_nic_t *enp) 825{ 826#if EFSYS_OPT_RX_SCALE 827 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) { 828 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 829 } 830 enp->en_rss_context = 0; 831 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 832#else 833 _NOTE(ARGUNUSED(enp)) 834#endif /* EFSYS_OPT_RX_SCALE */ 835} 836 837#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 838