ef10_rx.c revision 300840
1/*- 2 * Copyright (c) 2012-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sys/dev/sfxge/common/ef10_rx.c 300840 2016-05-27 11:44:40Z arybchik $"); 33 34#include "efx.h" 35#include "efx_impl.h" 36 37 38#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 39 40 41static __checkReturn efx_rc_t 42efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t size, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter) 50{ 51 efx_mcdi_req_t req; 52 uint8_t payload[ 53 MAX(MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)), 54 MC_CMD_INIT_RXQ_OUT_LEN)]; 55 int npages = EFX_RXQ_NBUFS(size); 56 int i; 57 efx_qword_t *dma_addr; 58 uint64_t addr; 59 efx_rc_t rc; 60 61 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); 62 63 (void) memset(payload, 0, sizeof (payload)); 64 req.emr_cmd = MC_CMD_INIT_RXQ; 65 req.emr_in_buf = payload; 66 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages); 67 req.emr_out_buf = payload; 68 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN; 69 70 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size); 71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq); 72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label); 73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance); 74 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS, 75 INIT_RXQ_IN_FLAG_BUFF_MODE, 0, 76 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0, 77 INIT_RXQ_IN_FLAG_TIMESTAMP, 0, 78 INIT_RXQ_IN_CRC_MODE, 0, 79 INIT_RXQ_IN_FLAG_PREFIX, 1, 80 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter); 81 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0); 82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 83 84 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 85 addr = EFSYS_MEM_ADDR(esmp); 86 87 for (i = 0; i < npages; i++) { 88 EFX_POPULATE_QWORD_2(*dma_addr, 89 EFX_DWORD_1, (uint32_t)(addr >> 32), 90 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 91 92 dma_addr++; 93 addr += EFX_BUF_SIZE; 94 } 95 96 efx_mcdi_execute(enp, &req); 97 98 if (req.emr_rc != 0) { 99 rc = req.emr_rc; 100 goto fail1; 101 } 102 103 return (0); 104 105fail1: 106 EFSYS_PROBE1(fail1, efx_rc_t, rc); 107 108 return (rc); 109} 110 111static __checkReturn efx_rc_t 112efx_mcdi_fini_rxq( 113 __in efx_nic_t *enp, 114 __in uint32_t instance) 115{ 116 efx_mcdi_req_t req; 117 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 118 MC_CMD_FINI_RXQ_OUT_LEN)]; 119 efx_rc_t rc; 120 121 (void) memset(payload, 0, sizeof (payload)); 122 req.emr_cmd = MC_CMD_FINI_RXQ; 123 req.emr_in_buf = payload; 124 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 125 req.emr_out_buf = payload; 126 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 127 128 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 129 130 efx_mcdi_execute_quiet(enp, &req); 131 132 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) { 133 rc = req.emr_rc; 134 goto fail1; 135 } 136 137 return (0); 138 139fail1: 140 EFSYS_PROBE1(fail1, efx_rc_t, rc); 141 142 return (rc); 143} 144 145#if EFSYS_OPT_RX_SCALE 146static __checkReturn efx_rc_t 147efx_mcdi_rss_context_alloc( 148 __in efx_nic_t *enp, 149 __in efx_rx_scale_support_t scale_support, 150 __in uint32_t num_queues, 151 __out uint32_t *rss_contextp) 152{ 153 efx_mcdi_req_t req; 154 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 155 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 156 uint32_t rss_context; 157 uint32_t context_type; 158 efx_rc_t rc; 159 160 if (num_queues > EFX_MAXRSS) { 161 rc = EINVAL; 162 goto fail1; 163 } 164 165 switch (scale_support) { 166 case EFX_RX_SCALE_EXCLUSIVE: 167 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 168 break; 169 case EFX_RX_SCALE_SHARED: 170 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 171 break; 172 default: 173 rc = EINVAL; 174 goto fail2; 175 } 176 177 (void) memset(payload, 0, sizeof (payload)); 178 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 179 req.emr_in_buf = payload; 180 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 181 req.emr_out_buf = payload; 182 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 183 184 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 185 EVB_PORT_ID_ASSIGNED); 186 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 187 /* NUM_QUEUES is only used to validate indirection table offsets */ 188 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 189 190 efx_mcdi_execute(enp, &req); 191 192 if (req.emr_rc != 0) { 193 rc = req.emr_rc; 194 goto fail3; 195 } 196 197 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 198 rc = EMSGSIZE; 199 goto fail4; 200 } 201 202 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 203 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 204 rc = ENOENT; 205 goto fail5; 206 } 207 208 *rss_contextp = rss_context; 209 210 return (0); 211 212fail5: 213 EFSYS_PROBE(fail5); 214fail4: 215 EFSYS_PROBE(fail4); 216fail3: 217 EFSYS_PROBE(fail3); 218fail2: 219 EFSYS_PROBE(fail2); 220fail1: 221 EFSYS_PROBE1(fail1, efx_rc_t, rc); 222 223 return (rc); 224} 225#endif /* EFSYS_OPT_RX_SCALE */ 226 227#if EFSYS_OPT_RX_SCALE 228static efx_rc_t 229efx_mcdi_rss_context_free( 230 __in efx_nic_t *enp, 231 __in uint32_t rss_context) 232{ 233 efx_mcdi_req_t req; 234 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 235 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 236 efx_rc_t rc; 237 238 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 239 rc = EINVAL; 240 goto fail1; 241 } 242 243 (void) memset(payload, 0, sizeof (payload)); 244 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 245 req.emr_in_buf = payload; 246 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 247 req.emr_out_buf = payload; 248 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 249 250 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 251 252 efx_mcdi_execute_quiet(enp, &req); 253 254 if (req.emr_rc != 0) { 255 rc = req.emr_rc; 256 goto fail2; 257 } 258 259 return (0); 260 261fail2: 262 EFSYS_PROBE(fail2); 263fail1: 264 EFSYS_PROBE1(fail1, efx_rc_t, rc); 265 266 return (rc); 267} 268#endif /* EFSYS_OPT_RX_SCALE */ 269 270#if EFSYS_OPT_RX_SCALE 271static efx_rc_t 272efx_mcdi_rss_context_set_flags( 273 __in efx_nic_t *enp, 274 __in uint32_t rss_context, 275 __in efx_rx_hash_type_t type) 276{ 277 efx_mcdi_req_t req; 278 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 279 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 280 efx_rc_t rc; 281 282 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 283 rc = EINVAL; 284 goto fail1; 285 } 286 287 (void) memset(payload, 0, sizeof (payload)); 288 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 289 req.emr_in_buf = payload; 290 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 291 req.emr_out_buf = payload; 292 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 293 294 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 295 rss_context); 296 297 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 298 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 299 (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0, 300 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 301 (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0, 302 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 303 (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0, 304 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 305 (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0); 306 307 efx_mcdi_execute(enp, &req); 308 309 if (req.emr_rc != 0) { 310 rc = req.emr_rc; 311 goto fail2; 312 } 313 314 return (0); 315 316fail2: 317 EFSYS_PROBE(fail2); 318fail1: 319 EFSYS_PROBE1(fail1, efx_rc_t, rc); 320 321 return (rc); 322} 323#endif /* EFSYS_OPT_RX_SCALE */ 324 325#if EFSYS_OPT_RX_SCALE 326static efx_rc_t 327efx_mcdi_rss_context_set_key( 328 __in efx_nic_t *enp, 329 __in uint32_t rss_context, 330 __in_ecount(n) uint8_t *key, 331 __in size_t n) 332{ 333 efx_mcdi_req_t req; 334 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 335 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 336 efx_rc_t rc; 337 338 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 339 rc = EINVAL; 340 goto fail1; 341 } 342 343 (void) memset(payload, 0, sizeof (payload)); 344 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 345 req.emr_in_buf = payload; 346 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 347 req.emr_out_buf = payload; 348 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 349 350 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 351 rss_context); 352 353 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 354 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 355 rc = EINVAL; 356 goto fail2; 357 } 358 359 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 360 key, n); 361 362 efx_mcdi_execute(enp, &req); 363 364 if (req.emr_rc != 0) { 365 rc = req.emr_rc; 366 goto fail3; 367 } 368 369 return (0); 370 371fail3: 372 EFSYS_PROBE(fail3); 373fail2: 374 EFSYS_PROBE(fail2); 375fail1: 376 EFSYS_PROBE1(fail1, efx_rc_t, rc); 377 378 return (rc); 379} 380#endif /* EFSYS_OPT_RX_SCALE */ 381 382#if EFSYS_OPT_RX_SCALE 383static efx_rc_t 384efx_mcdi_rss_context_set_table( 385 __in efx_nic_t *enp, 386 __in uint32_t rss_context, 387 __in_ecount(n) unsigned int *table, 388 __in size_t n) 389{ 390 efx_mcdi_req_t req; 391 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 392 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 393 uint8_t *req_table; 394 int i, rc; 395 396 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 397 rc = EINVAL; 398 goto fail1; 399 } 400 401 (void) memset(payload, 0, sizeof (payload)); 402 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 403 req.emr_in_buf = payload; 404 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 405 req.emr_out_buf = payload; 406 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 407 408 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 409 rss_context); 410 411 req_table = 412 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 413 414 for (i = 0; 415 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 416 i++) { 417 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 418 } 419 420 efx_mcdi_execute(enp, &req); 421 422 if (req.emr_rc != 0) { 423 rc = req.emr_rc; 424 goto fail2; 425 } 426 427 return (0); 428 429fail2: 430 EFSYS_PROBE(fail2); 431fail1: 432 EFSYS_PROBE1(fail1, efx_rc_t, rc); 433 434 return (rc); 435} 436#endif /* EFSYS_OPT_RX_SCALE */ 437 438 439 __checkReturn efx_rc_t 440ef10_rx_init( 441 __in efx_nic_t *enp) 442{ 443#if EFSYS_OPT_RX_SCALE 444 445 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 446 &enp->en_rss_context) == 0) { 447 /* 448 * Allocated an exclusive RSS context, which allows both the 449 * indirection table and key to be modified. 450 */ 451 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 452 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 453 } else { 454 /* 455 * Failed to allocate an exclusive RSS context. Continue 456 * operation without support for RSS. The pseudo-header in 457 * received packets will not contain a Toeplitz hash value. 458 */ 459 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 460 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 461 } 462 463#endif /* EFSYS_OPT_RX_SCALE */ 464 465 return (0); 466} 467 468#if EFSYS_OPT_RX_SCATTER 469 __checkReturn efx_rc_t 470ef10_rx_scatter_enable( 471 __in efx_nic_t *enp, 472 __in unsigned int buf_size) 473{ 474 _NOTE(ARGUNUSED(enp, buf_size)) 475 return (0); 476} 477#endif /* EFSYS_OPT_RX_SCATTER */ 478 479#if EFSYS_OPT_RX_SCALE 480 __checkReturn efx_rc_t 481ef10_rx_scale_mode_set( 482 __in efx_nic_t *enp, 483 __in efx_rx_hash_alg_t alg, 484 __in efx_rx_hash_type_t type, 485 __in boolean_t insert) 486{ 487 efx_rc_t rc; 488 489 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 490 EFSYS_ASSERT3U(insert, ==, B_TRUE); 491 492 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 493 rc = EINVAL; 494 goto fail1; 495 } 496 497 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 498 rc = ENOTSUP; 499 goto fail2; 500 } 501 502 if ((rc = efx_mcdi_rss_context_set_flags(enp, 503 enp->en_rss_context, type)) != 0) 504 goto fail3; 505 506 return (0); 507 508fail3: 509 EFSYS_PROBE(fail3); 510fail2: 511 EFSYS_PROBE(fail2); 512fail1: 513 EFSYS_PROBE1(fail1, efx_rc_t, rc); 514 515 return (rc); 516} 517#endif /* EFSYS_OPT_RX_SCALE */ 518 519#if EFSYS_OPT_RX_SCALE 520 __checkReturn efx_rc_t 521ef10_rx_scale_key_set( 522 __in efx_nic_t *enp, 523 __in_ecount(n) uint8_t *key, 524 __in size_t n) 525{ 526 efx_rc_t rc; 527 528 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 529 rc = ENOTSUP; 530 goto fail1; 531 } 532 533 if ((rc = efx_mcdi_rss_context_set_key(enp, 534 enp->en_rss_context, key, n)) != 0) 535 goto fail2; 536 537 return (0); 538 539fail2: 540 EFSYS_PROBE(fail2); 541fail1: 542 EFSYS_PROBE1(fail1, efx_rc_t, rc); 543 544 return (rc); 545} 546#endif /* EFSYS_OPT_RX_SCALE */ 547 548#if EFSYS_OPT_RX_SCALE 549 __checkReturn efx_rc_t 550ef10_rx_scale_tbl_set( 551 __in efx_nic_t *enp, 552 __in_ecount(n) unsigned int *table, 553 __in size_t n) 554{ 555 efx_rc_t rc; 556 557 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 558 rc = ENOTSUP; 559 goto fail1; 560 } 561 562 if ((rc = efx_mcdi_rss_context_set_table(enp, 563 enp->en_rss_context, table, n)) != 0) 564 goto fail2; 565 566 return (0); 567 568fail2: 569 EFSYS_PROBE(fail2); 570fail1: 571 EFSYS_PROBE1(fail1, efx_rc_t, rc); 572 573 return (rc); 574} 575#endif /* EFSYS_OPT_RX_SCALE */ 576 577 578/* 579 * EF10 RX pseudo-header 580 * --------------------- 581 * 582 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 583 * 584 * +00: Toeplitz hash value. 585 * (32bit little-endian) 586 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 587 * (16bit big-endian) 588 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 589 * (16bit big-endian) 590 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 591 * (16bit little-endian) 592 * +10: MAC timestamp. Zero if timestamping is not enabled. 593 * (32bit little-endian) 594 * 595 * See "The RX Pseudo-header" in SF-109306-TC. 596 */ 597 598 __checkReturn efx_rc_t 599ef10_rx_prefix_pktlen( 600 __in efx_nic_t *enp, 601 __in uint8_t *buffer, 602 __out uint16_t *lengthp) 603{ 604 _NOTE(ARGUNUSED(enp)) 605 606 /* 607 * The RX pseudo-header contains the packet length, excluding the 608 * pseudo-header. If the hardware receive datapath was operating in 609 * cut-through mode then the length in the RX pseudo-header will be 610 * zero, and the packet length must be obtained from the DMA length 611 * reported in the RX event. 612 */ 613 *lengthp = buffer[8] | (buffer[9] << 8); 614 return (0); 615} 616 617#if EFSYS_OPT_RX_SCALE 618 __checkReturn uint32_t 619ef10_rx_prefix_hash( 620 __in efx_nic_t *enp, 621 __in efx_rx_hash_alg_t func, 622 __in uint8_t *buffer) 623{ 624 _NOTE(ARGUNUSED(enp)) 625 626 switch (func) { 627 case EFX_RX_HASHALG_TOEPLITZ: 628 return (buffer[0] | 629 (buffer[1] << 8) | 630 (buffer[2] << 16) | 631 (buffer[3] << 24)); 632 633 default: 634 EFSYS_ASSERT(0); 635 return (0); 636 } 637} 638#endif /* EFSYS_OPT_RX_SCALE */ 639 640 void 641ef10_rx_qpost( 642 __in efx_rxq_t *erp, 643 __in_ecount(n) efsys_dma_addr_t *addrp, 644 __in size_t size, 645 __in unsigned int n, 646 __in unsigned int completed, 647 __in unsigned int added) 648{ 649 efx_qword_t qword; 650 unsigned int i; 651 unsigned int offset; 652 unsigned int id; 653 654 /* The client driver must not overfill the queue */ 655 EFSYS_ASSERT3U(added - completed + n, <=, 656 EFX_RXQ_LIMIT(erp->er_mask + 1)); 657 658 id = added & (erp->er_mask); 659 for (i = 0; i < n; i++) { 660 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 661 unsigned int, id, efsys_dma_addr_t, addrp[i], 662 size_t, size); 663 664 EFX_POPULATE_QWORD_3(qword, 665 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 666 ESF_DZ_RX_KER_BUF_ADDR_DW0, 667 (uint32_t)(addrp[i] & 0xffffffff), 668 ESF_DZ_RX_KER_BUF_ADDR_DW1, 669 (uint32_t)(addrp[i] >> 32)); 670 671 offset = id * sizeof (efx_qword_t); 672 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 673 674 id = (id + 1) & (erp->er_mask); 675 } 676} 677 678 void 679ef10_rx_qpush( 680 __in efx_rxq_t *erp, 681 __in unsigned int added, 682 __inout unsigned int *pushedp) 683{ 684 efx_nic_t *enp = erp->er_enp; 685 unsigned int pushed = *pushedp; 686 uint32_t wptr; 687 efx_dword_t dword; 688 689 /* Hardware has alignment restriction for WPTR */ 690 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 691 if (pushed == wptr) 692 return; 693 694 *pushedp = wptr; 695 696 /* Push the populated descriptors out */ 697 wptr &= erp->er_mask; 698 699 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 700 701 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 702 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 703 wptr, pushed & erp->er_mask); 704 EFSYS_PIO_WRITE_BARRIER(); 705 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 706 erp->er_index, &dword, B_FALSE); 707} 708 709 __checkReturn efx_rc_t 710ef10_rx_qflush( 711 __in efx_rxq_t *erp) 712{ 713 efx_nic_t *enp = erp->er_enp; 714 efx_rc_t rc; 715 716 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 717 goto fail1; 718 719 return (0); 720 721fail1: 722 EFSYS_PROBE1(fail1, efx_rc_t, rc); 723 724 return (rc); 725} 726 727 void 728ef10_rx_qenable( 729 __in efx_rxq_t *erp) 730{ 731 /* FIXME */ 732 _NOTE(ARGUNUSED(erp)) 733 /* FIXME */ 734} 735 736 __checkReturn efx_rc_t 737ef10_rx_qcreate( 738 __in efx_nic_t *enp, 739 __in unsigned int index, 740 __in unsigned int label, 741 __in efx_rxq_type_t type, 742 __in efsys_mem_t *esmp, 743 __in size_t n, 744 __in uint32_t id, 745 __in efx_evq_t *eep, 746 __in efx_rxq_t *erp) 747{ 748 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 749 efx_rc_t rc; 750 boolean_t disable_scatter; 751 752 _NOTE(ARGUNUSED(id, erp)) 753 754 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 755 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 756 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 757 758 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 759 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 760 761 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 762 rc = EINVAL; 763 goto fail1; 764 } 765 if (index >= encp->enc_rxq_limit) { 766 rc = EINVAL; 767 goto fail2; 768 } 769 770 /* Scatter can only be disabled if the firmware supports doing so */ 771 if ((type != EFX_RXQ_TYPE_SCATTER) && 772 enp->en_nic_cfg.enc_rx_disable_scatter_supported) { 773 disable_scatter = B_TRUE; 774 } else { 775 disable_scatter = B_FALSE; 776 } 777 778 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, 779 esmp, disable_scatter)) != 0) 780 goto fail3; 781 782 erp->er_eep = eep; 783 erp->er_label = label; 784 785 ef10_ev_rxlabel_init(eep, erp, label); 786 787 return (0); 788 789fail3: 790 EFSYS_PROBE(fail3); 791fail2: 792 EFSYS_PROBE(fail2); 793fail1: 794 EFSYS_PROBE1(fail1, efx_rc_t, rc); 795 796 return (rc); 797} 798 799 void 800ef10_rx_qdestroy( 801 __in efx_rxq_t *erp) 802{ 803 efx_nic_t *enp = erp->er_enp; 804 efx_evq_t *eep = erp->er_eep; 805 unsigned int label = erp->er_label; 806 807 ef10_ev_rxlabel_fini(eep, label); 808 809 EFSYS_ASSERT(enp->en_rx_qcount != 0); 810 --enp->en_rx_qcount; 811 812 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 813} 814 815 void 816ef10_rx_fini( 817 __in efx_nic_t *enp) 818{ 819#if EFSYS_OPT_RX_SCALE 820 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) { 821 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 822 } 823 enp->en_rss_context = 0; 824 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 825#else 826 _NOTE(ARGUNUSED(enp)) 827#endif /* EFSYS_OPT_RX_SCALE */ 828} 829 830#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 831