1/*- 2 * Copyright (c) 2007-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: stable/11/sys/dev/sfxge/common/efx_ev.c 342450 2018-12-25 07:33:02Z arybchik $"); 33 34#include "efx.h" 35#include "efx_impl.h" 36#if EFSYS_OPT_MON_MCDI 37#include "mcdi_mon.h" 38#endif 39 40#if EFSYS_OPT_QSTATS 41#define EFX_EV_QSTAT_INCR(_eep, _stat) \ 42 do { \ 43 (_eep)->ee_stat[_stat]++; \ 44 _NOTE(CONSTANTCONDITION) \ 45 } while (B_FALSE) 46#else 47#define EFX_EV_QSTAT_INCR(_eep, _stat) 48#endif 49 50#define EFX_EV_PRESENT(_qword) \ 51 (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \ 52 EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff) 53 54 55 56#if EFSYS_OPT_SIENA 57 58static __checkReturn efx_rc_t 59siena_ev_init( 60 __in efx_nic_t *enp); 61 62static void 63siena_ev_fini( 64 __in efx_nic_t *enp); 65 66static __checkReturn efx_rc_t 67siena_ev_qcreate( 68 __in efx_nic_t *enp, 69 __in unsigned int index, 70 __in efsys_mem_t *esmp, 71 __in size_t n, 72 __in uint32_t id, 73 __in uint32_t us, 74 __in uint32_t flags, 75 __in efx_evq_t *eep); 76 77static void 78siena_ev_qdestroy( 79 __in efx_evq_t *eep); 80 81static __checkReturn efx_rc_t 82siena_ev_qprime( 83 __in efx_evq_t *eep, 84 __in unsigned int count); 85 86static void 87siena_ev_qpost( 88 __in efx_evq_t *eep, 89 __in uint16_t data); 90 91static __checkReturn efx_rc_t 92siena_ev_qmoderate( 93 __in efx_evq_t *eep, 94 __in unsigned int us); 95 96#if EFSYS_OPT_QSTATS 97static void 98siena_ev_qstats_update( 99 __in efx_evq_t *eep, 100 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); 101 102#endif 103 104#endif /* EFSYS_OPT_SIENA */ 105 106#if EFSYS_OPT_SIENA 107static const efx_ev_ops_t __efx_ev_siena_ops = { 108 siena_ev_init, /* eevo_init */ 109 siena_ev_fini, /* eevo_fini */ 110 siena_ev_qcreate, /* eevo_qcreate */ 111 siena_ev_qdestroy, /* eevo_qdestroy */ 112 siena_ev_qprime, /* eevo_qprime */ 113 siena_ev_qpost, /* eevo_qpost */ 114 siena_ev_qmoderate, /* eevo_qmoderate */ 115#if EFSYS_OPT_QSTATS 116 siena_ev_qstats_update, /* eevo_qstats_update */ 117#endif 118}; 119#endif /* EFSYS_OPT_SIENA */ 120 121#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 122static const efx_ev_ops_t __efx_ev_ef10_ops = { 123 ef10_ev_init, /* eevo_init */ 124 ef10_ev_fini, /* eevo_fini */ 125 ef10_ev_qcreate, /* eevo_qcreate */ 126 ef10_ev_qdestroy, /* eevo_qdestroy */ 127 ef10_ev_qprime, /* eevo_qprime */ 128 ef10_ev_qpost, /* eevo_qpost */ 129 ef10_ev_qmoderate, /* eevo_qmoderate */ 130#if EFSYS_OPT_QSTATS 131 ef10_ev_qstats_update, /* eevo_qstats_update */ 132#endif 133}; 134#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 135 136 137 __checkReturn efx_rc_t 138efx_ev_init( 139 __in efx_nic_t *enp) 140{ 141 const efx_ev_ops_t *eevop; 142 efx_rc_t rc; 143 144 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 145 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); 146 147 if (enp->en_mod_flags & EFX_MOD_EV) { 148 rc = EINVAL; 149 goto fail1; 150 } 151 152 switch (enp->en_family) { 153#if EFSYS_OPT_SIENA 154 case EFX_FAMILY_SIENA: 155 eevop = &__efx_ev_siena_ops; 156 break; 157#endif /* EFSYS_OPT_SIENA */ 158 159#if EFSYS_OPT_HUNTINGTON 160 case EFX_FAMILY_HUNTINGTON: 161 eevop = &__efx_ev_ef10_ops; 162 break; 163#endif /* EFSYS_OPT_HUNTINGTON */ 164 165#if EFSYS_OPT_MEDFORD 166 case EFX_FAMILY_MEDFORD: 167 eevop = &__efx_ev_ef10_ops; 168 break; 169#endif /* EFSYS_OPT_MEDFORD */ 170 171 default: 172 EFSYS_ASSERT(0); 173 rc = ENOTSUP; 174 goto fail1; 175 } 176 177 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); 178 179 if ((rc = eevop->eevo_init(enp)) != 0) 180 goto fail2; 181 182 enp->en_eevop = eevop; 183 enp->en_mod_flags |= EFX_MOD_EV; 184 return (0); 185 186fail2: 187 EFSYS_PROBE(fail2); 188 189fail1: 190 EFSYS_PROBE1(fail1, efx_rc_t, rc); 191 192 enp->en_eevop = NULL; 193 enp->en_mod_flags &= ~EFX_MOD_EV; 194 return (rc); 195} 196 197 void 198efx_ev_fini( 199 __in efx_nic_t *enp) 200{ 201 const efx_ev_ops_t *eevop = enp->en_eevop; 202 203 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 204 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); 205 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); 206 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); 207 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); 208 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); 209 210 eevop->eevo_fini(enp); 211 212 enp->en_eevop = NULL; 213 enp->en_mod_flags &= ~EFX_MOD_EV; 214} 215 216 217 __checkReturn efx_rc_t 218efx_ev_qcreate( 219 __in efx_nic_t *enp, 220 __in unsigned int index, 221 __in efsys_mem_t *esmp, 222 __in size_t n, 223 __in uint32_t id, 224 __in uint32_t us, 225 __in uint32_t flags, 226 __deref_out efx_evq_t **eepp) 227{ 228 const efx_ev_ops_t *eevop = enp->en_eevop; 229 efx_evq_t *eep; 230 efx_rc_t rc; 231 232 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 233 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); 234 235 EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, 236 enp->en_nic_cfg.enc_evq_limit); 237 238 switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) { 239 case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT: 240 break; 241 case EFX_EVQ_FLAGS_NOTIFY_DISABLED: 242 if (us != 0) { 243 rc = EINVAL; 244 goto fail1; 245 } 246 break; 247 default: 248 rc = EINVAL; 249 goto fail2; 250 } 251 252 /* Allocate an EVQ object */ 253 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep); 254 if (eep == NULL) { 255 rc = ENOMEM; 256 goto fail3; 257 } 258 259 eep->ee_magic = EFX_EVQ_MAGIC; 260 eep->ee_enp = enp; 261 eep->ee_index = index; 262 eep->ee_mask = n - 1; 263 eep->ee_flags = flags; 264 eep->ee_esmp = esmp; 265 266 /* 267 * Set outputs before the queue is created because interrupts may be 268 * raised for events immediately after the queue is created, before the 269 * function call below returns. See bug58606. 270 * 271 * The eepp pointer passed in by the client must therefore point to data 272 * shared with the client's event processing context. 273 */ 274 enp->en_ev_qcount++; 275 *eepp = eep; 276 277 if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, us, flags, 278 eep)) != 0) 279 goto fail4; 280 281 return (0); 282 283fail4: 284 EFSYS_PROBE(fail4); 285 286 *eepp = NULL; 287 enp->en_ev_qcount--; 288 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); 289fail3: 290 EFSYS_PROBE(fail3); 291fail2: 292 EFSYS_PROBE(fail2); 293fail1: 294 EFSYS_PROBE1(fail1, efx_rc_t, rc); 295 return (rc); 296} 297 298 void 299efx_ev_qdestroy( 300 __in efx_evq_t *eep) 301{ 302 efx_nic_t *enp = eep->ee_enp; 303 const efx_ev_ops_t *eevop = enp->en_eevop; 304 305 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 306 307 EFSYS_ASSERT(enp->en_ev_qcount != 0); 308 --enp->en_ev_qcount; 309 310 eevop->eevo_qdestroy(eep); 311 312 /* Free the EVQ object */ 313 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); 314} 315 316 __checkReturn efx_rc_t 317efx_ev_qprime( 318 __in efx_evq_t *eep, 319 __in unsigned int count) 320{ 321 efx_nic_t *enp = eep->ee_enp; 322 const efx_ev_ops_t *eevop = enp->en_eevop; 323 efx_rc_t rc; 324 325 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 326 327 if (!(enp->en_mod_flags & EFX_MOD_INTR)) { 328 rc = EINVAL; 329 goto fail1; 330 } 331 332 if ((rc = eevop->eevo_qprime(eep, count)) != 0) 333 goto fail2; 334 335 return (0); 336 337fail2: 338 EFSYS_PROBE(fail2); 339fail1: 340 EFSYS_PROBE1(fail1, efx_rc_t, rc); 341 return (rc); 342} 343 344 __checkReturn boolean_t 345efx_ev_qpending( 346 __in efx_evq_t *eep, 347 __in unsigned int count) 348{ 349 size_t offset; 350 efx_qword_t qword; 351 352 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 353 354 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 355 EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword); 356 357 return (EFX_EV_PRESENT(qword)); 358} 359 360#if EFSYS_OPT_EV_PREFETCH 361 362 void 363efx_ev_qprefetch( 364 __in efx_evq_t *eep, 365 __in unsigned int count) 366{ 367 unsigned int offset; 368 369 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 370 371 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 372 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 373} 374 375#endif /* EFSYS_OPT_EV_PREFETCH */ 376 377#define EFX_EV_BATCH 8 378 379 void 380efx_ev_qpoll( 381 __in efx_evq_t *eep, 382 __inout unsigned int *countp, 383 __in const efx_ev_callbacks_t *eecp, 384 __in_opt void *arg) 385{ 386 efx_qword_t ev[EFX_EV_BATCH]; 387 unsigned int batch; 388 unsigned int total; 389 unsigned int count; 390 unsigned int index; 391 size_t offset; 392 393 /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */ 394 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN); 395 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH); 396 397 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV); 398 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV); 399 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV); 400 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV == 401 FSE_AZ_EV_CODE_DRV_GEN_EV); 402#if EFSYS_OPT_MCDI 403 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV == 404 FSE_AZ_EV_CODE_MCDI_EVRESPONSE); 405#endif 406 407 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 408 EFSYS_ASSERT(countp != NULL); 409 EFSYS_ASSERT(eecp != NULL); 410 411 count = *countp; 412 do { 413 /* Read up until the end of the batch period */ 414 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1)); 415 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 416 for (total = 0; total < batch; ++total) { 417 EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total])); 418 419 if (!EFX_EV_PRESENT(ev[total])) 420 break; 421 422 EFSYS_PROBE3(event, unsigned int, eep->ee_index, 423 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1), 424 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0)); 425 426 offset += sizeof (efx_qword_t); 427 } 428 429#if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1) 430 /* 431 * Prefetch the next batch when we get within PREFETCH_PERIOD 432 * of a completed batch. If the batch is smaller, then prefetch 433 * immediately. 434 */ 435 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD) 436 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 437#endif /* EFSYS_OPT_EV_PREFETCH */ 438 439 /* Process the batch of events */ 440 for (index = 0; index < total; ++index) { 441 boolean_t should_abort; 442 uint32_t code; 443 444#if EFSYS_OPT_EV_PREFETCH 445 /* Prefetch if we've now reached the batch period */ 446 if (total == batch && 447 index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) { 448 offset = (count + batch) & eep->ee_mask; 449 offset *= sizeof (efx_qword_t); 450 451 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 452 } 453#endif /* EFSYS_OPT_EV_PREFETCH */ 454 455 EFX_EV_QSTAT_INCR(eep, EV_ALL); 456 457 code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE); 458 switch (code) { 459 case FSE_AZ_EV_CODE_RX_EV: 460 should_abort = eep->ee_rx(eep, 461 &(ev[index]), eecp, arg); 462 break; 463 case FSE_AZ_EV_CODE_TX_EV: 464 should_abort = eep->ee_tx(eep, 465 &(ev[index]), eecp, arg); 466 break; 467 case FSE_AZ_EV_CODE_DRIVER_EV: 468 should_abort = eep->ee_driver(eep, 469 &(ev[index]), eecp, arg); 470 break; 471 case FSE_AZ_EV_CODE_DRV_GEN_EV: 472 should_abort = eep->ee_drv_gen(eep, 473 &(ev[index]), eecp, arg); 474 break; 475#if EFSYS_OPT_MCDI 476 case FSE_AZ_EV_CODE_MCDI_EVRESPONSE: 477 should_abort = eep->ee_mcdi(eep, 478 &(ev[index]), eecp, arg); 479 break; 480#endif 481 case FSE_AZ_EV_CODE_GLOBAL_EV: 482 if (eep->ee_global) { 483 should_abort = eep->ee_global(eep, 484 &(ev[index]), eecp, arg); 485 break; 486 } 487 /* else fallthrough */ 488 default: 489 EFSYS_PROBE3(bad_event, 490 unsigned int, eep->ee_index, 491 uint32_t, 492 EFX_QWORD_FIELD(ev[index], EFX_DWORD_1), 493 uint32_t, 494 EFX_QWORD_FIELD(ev[index], EFX_DWORD_0)); 495 496 EFSYS_ASSERT(eecp->eec_exception != NULL); 497 (void) eecp->eec_exception(arg, 498 EFX_EXCEPTION_EV_ERROR, code); 499 should_abort = B_TRUE; 500 } 501 if (should_abort) { 502 /* Ignore subsequent events */ 503 total = index + 1; 504 505 /* 506 * Poison batch to ensure the outer 507 * loop is broken out of. 508 */ 509 EFSYS_ASSERT(batch <= EFX_EV_BATCH); 510 batch += (EFX_EV_BATCH << 1); 511 EFSYS_ASSERT(total != batch); 512 break; 513 } 514 } 515 516 /* 517 * Now that the hardware has most likely moved onto dma'ing 518 * into the next cache line, clear the processed events. Take 519 * care to only clear out events that we've processed 520 */ 521 EFX_SET_QWORD(ev[0]); 522 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 523 for (index = 0; index < total; ++index) { 524 EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0])); 525 offset += sizeof (efx_qword_t); 526 } 527 528 count += total; 529 530 } while (total == batch); 531 532 *countp = count; 533} 534 535 void 536efx_ev_qpost( 537 __in efx_evq_t *eep, 538 __in uint16_t data) 539{ 540 efx_nic_t *enp = eep->ee_enp; 541 const efx_ev_ops_t *eevop = enp->en_eevop; 542 543 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 544 545 EFSYS_ASSERT(eevop != NULL && 546 eevop->eevo_qpost != NULL); 547 548 eevop->eevo_qpost(eep, data); 549} 550 551 __checkReturn efx_rc_t 552efx_ev_usecs_to_ticks( 553 __in efx_nic_t *enp, 554 __in unsigned int us, 555 __out unsigned int *ticksp) 556{ 557 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 558 unsigned int ticks; 559 560 /* Convert microseconds to a timer tick count */ 561 if (us == 0) 562 ticks = 0; 563 else if (us * 1000 < encp->enc_evq_timer_quantum_ns) 564 ticks = 1; /* Never round down to zero */ 565 else 566 ticks = us * 1000 / encp->enc_evq_timer_quantum_ns; 567 568 *ticksp = ticks; 569 return (0); 570} 571 572 __checkReturn efx_rc_t 573efx_ev_qmoderate( 574 __in efx_evq_t *eep, 575 __in unsigned int us) 576{ 577 efx_nic_t *enp = eep->ee_enp; 578 const efx_ev_ops_t *eevop = enp->en_eevop; 579 efx_rc_t rc; 580 581 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 582 583 if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) == 584 EFX_EVQ_FLAGS_NOTIFY_DISABLED) { 585 rc = EINVAL; 586 goto fail1; 587 } 588 589 if ((rc = eevop->eevo_qmoderate(eep, us)) != 0) 590 goto fail2; 591 592 return (0); 593 594fail2: 595 EFSYS_PROBE(fail2); 596fail1: 597 EFSYS_PROBE1(fail1, efx_rc_t, rc); 598 return (rc); 599} 600 601#if EFSYS_OPT_QSTATS 602 void 603efx_ev_qstats_update( 604 __in efx_evq_t *eep, 605 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 606 607{ efx_nic_t *enp = eep->ee_enp; 608 const efx_ev_ops_t *eevop = enp->en_eevop; 609 610 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 611 612 eevop->eevo_qstats_update(eep, stat); 613} 614 615#endif /* EFSYS_OPT_QSTATS */ 616 617#if EFSYS_OPT_SIENA 618 619static __checkReturn efx_rc_t 620siena_ev_init( 621 __in efx_nic_t *enp) 622{ 623 efx_oword_t oword; 624 625 /* 626 * Program the event queue for receive and transmit queue 627 * flush events. 628 */ 629 EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword); 630 EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0); 631 EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword); 632 633 return (0); 634 635} 636 637static __checkReturn boolean_t 638siena_ev_rx_not_ok( 639 __in efx_evq_t *eep, 640 __in efx_qword_t *eqp, 641 __in uint32_t label, 642 __in uint32_t id, 643 __inout uint16_t *flagsp) 644{ 645 boolean_t ignore = B_FALSE; 646 647 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) { 648 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC); 649 EFSYS_PROBE(tobe_disc); 650 /* 651 * Assume this is a unicast address mismatch, unless below 652 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or 653 * EV_RX_PAUSE_FRM_ERR is set. 654 */ 655 (*flagsp) |= EFX_ADDR_MISMATCH; 656 } 657 658 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) { 659 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id); 660 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); 661 (*flagsp) |= EFX_DISCARD; 662 663#if EFSYS_OPT_RX_SCATTER 664 /* 665 * Lookout for payload queue ran dry errors and ignore them. 666 * 667 * Sadly for the header/data split cases, the descriptor 668 * pointer in this event refers to the header queue and 669 * therefore cannot be easily detected as duplicate. 670 * So we drop these and rely on the receive processing seeing 671 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard 672 * the partially received packet. 673 */ 674 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) && 675 (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) && 676 (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0)) 677 ignore = B_TRUE; 678#endif /* EFSYS_OPT_RX_SCATTER */ 679 } 680 681 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) { 682 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); 683 EFSYS_PROBE(crc_err); 684 (*flagsp) &= ~EFX_ADDR_MISMATCH; 685 (*flagsp) |= EFX_DISCARD; 686 } 687 688 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) { 689 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR); 690 EFSYS_PROBE(pause_frm_err); 691 (*flagsp) &= ~EFX_ADDR_MISMATCH; 692 (*flagsp) |= EFX_DISCARD; 693 } 694 695 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) { 696 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR); 697 EFSYS_PROBE(owner_id_err); 698 (*flagsp) |= EFX_DISCARD; 699 } 700 701 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) { 702 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); 703 EFSYS_PROBE(ipv4_err); 704 (*flagsp) &= ~EFX_CKSUM_IPV4; 705 } 706 707 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) { 708 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); 709 EFSYS_PROBE(udp_chk_err); 710 (*flagsp) &= ~EFX_CKSUM_TCPUDP; 711 } 712 713 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) { 714 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR); 715 716 /* 717 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This 718 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error 719 * condition. 720 */ 721 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP); 722 } 723 724 return (ignore); 725} 726 727static __checkReturn boolean_t 728siena_ev_rx( 729 __in efx_evq_t *eep, 730 __in efx_qword_t *eqp, 731 __in const efx_ev_callbacks_t *eecp, 732 __in_opt void *arg) 733{ 734 uint32_t id; 735 uint32_t size; 736 uint32_t label; 737 boolean_t ok; 738#if EFSYS_OPT_RX_SCATTER 739 boolean_t sop; 740 boolean_t jumbo_cont; 741#endif /* EFSYS_OPT_RX_SCATTER */ 742 uint32_t hdr_type; 743 boolean_t is_v6; 744 uint16_t flags; 745 boolean_t ignore; 746 boolean_t should_abort; 747 748 EFX_EV_QSTAT_INCR(eep, EV_RX); 749 750 /* Basic packet information */ 751 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR); 752 size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT); 753 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL); 754 ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0); 755 756#if EFSYS_OPT_RX_SCATTER 757 sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0); 758 jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0); 759#endif /* EFSYS_OPT_RX_SCATTER */ 760 761 hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE); 762 763 is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0); 764 765 /* 766 * If packet is marked as OK and packet type is TCP/IP or 767 * UDP/IP or other IP, then we can rely on the hardware checksums. 768 */ 769 switch (hdr_type) { 770 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP: 771 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP; 772 if (is_v6) { 773 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); 774 flags |= EFX_PKT_IPV6; 775 } else { 776 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); 777 flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 778 } 779 break; 780 781 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP: 782 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP; 783 if (is_v6) { 784 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); 785 flags |= EFX_PKT_IPV6; 786 } else { 787 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); 788 flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 789 } 790 break; 791 792 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER: 793 if (is_v6) { 794 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); 795 flags = EFX_PKT_IPV6; 796 } else { 797 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); 798 flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 799 } 800 break; 801 802 case FSE_AZ_RX_EV_HDR_TYPE_OTHER: 803 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); 804 flags = 0; 805 break; 806 807 default: 808 EFSYS_ASSERT(B_FALSE); 809 flags = 0; 810 break; 811 } 812 813#if EFSYS_OPT_RX_SCATTER 814 /* Report scatter and header/lookahead split buffer flags */ 815 if (sop) 816 flags |= EFX_PKT_START; 817 if (jumbo_cont) 818 flags |= EFX_PKT_CONT; 819#endif /* EFSYS_OPT_RX_SCATTER */ 820 821 /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */ 822 if (!ok) { 823 ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags); 824 if (ignore) { 825 EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, 826 uint32_t, size, uint16_t, flags); 827 828 return (B_FALSE); 829 } 830 } 831 832 /* If we're not discarding the packet then it is ok */ 833 if (~flags & EFX_DISCARD) 834 EFX_EV_QSTAT_INCR(eep, EV_RX_OK); 835 836 /* Detect multicast packets that didn't match the filter */ 837 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) { 838 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT); 839 840 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) { 841 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH); 842 } else { 843 EFSYS_PROBE(mcast_mismatch); 844 flags |= EFX_ADDR_MISMATCH; 845 } 846 } else { 847 flags |= EFX_PKT_UNICAST; 848 } 849 850 /* 851 * The packet parser in Siena can abort parsing packets under 852 * certain error conditions, setting the PKT_NOT_PARSED bit 853 * (which clears PKT_OK). If this is set, then don't trust 854 * the PKT_TYPE field. 855 */ 856 if (!ok) { 857 uint32_t parse_err; 858 859 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED); 860 if (parse_err != 0) 861 flags |= EFX_CHECK_VLAN; 862 } 863 864 if (~flags & EFX_CHECK_VLAN) { 865 uint32_t pkt_type; 866 867 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE); 868 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN) 869 flags |= EFX_PKT_VLAN_TAGGED; 870 } 871 872 EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, 873 uint32_t, size, uint16_t, flags); 874 875 EFSYS_ASSERT(eecp->eec_rx != NULL); 876 should_abort = eecp->eec_rx(arg, label, id, size, flags); 877 878 return (should_abort); 879} 880 881static __checkReturn boolean_t 882siena_ev_tx( 883 __in efx_evq_t *eep, 884 __in efx_qword_t *eqp, 885 __in const efx_ev_callbacks_t *eecp, 886 __in_opt void *arg) 887{ 888 uint32_t id; 889 uint32_t label; 890 boolean_t should_abort; 891 892 EFX_EV_QSTAT_INCR(eep, EV_TX); 893 894 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 && 895 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 && 896 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 && 897 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) { 898 899 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR); 900 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL); 901 902 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); 903 904 EFSYS_ASSERT(eecp->eec_tx != NULL); 905 should_abort = eecp->eec_tx(arg, label, id); 906 907 return (should_abort); 908 } 909 910 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0) 911 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 912 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 913 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 914 915 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0) 916 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR); 917 918 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0) 919 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG); 920 921 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0) 922 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL); 923 924 EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED); 925 return (B_FALSE); 926} 927 928static __checkReturn boolean_t 929siena_ev_global( 930 __in efx_evq_t *eep, 931 __in efx_qword_t *eqp, 932 __in const efx_ev_callbacks_t *eecp, 933 __in_opt void *arg) 934{ 935 _NOTE(ARGUNUSED(eqp, eecp, arg)) 936 937 EFX_EV_QSTAT_INCR(eep, EV_GLOBAL); 938 939 return (B_FALSE); 940} 941 942static __checkReturn boolean_t 943siena_ev_driver( 944 __in efx_evq_t *eep, 945 __in efx_qword_t *eqp, 946 __in const efx_ev_callbacks_t *eecp, 947 __in_opt void *arg) 948{ 949 boolean_t should_abort; 950 951 EFX_EV_QSTAT_INCR(eep, EV_DRIVER); 952 should_abort = B_FALSE; 953 954 switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) { 955 case FSE_AZ_TX_DESCQ_FLS_DONE_EV: { 956 uint32_t txq_index; 957 958 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); 959 960 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 961 962 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); 963 964 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); 965 should_abort = eecp->eec_txq_flush_done(arg, txq_index); 966 967 break; 968 } 969 case FSE_AZ_RX_DESCQ_FLS_DONE_EV: { 970 uint32_t rxq_index; 971 uint32_t failed; 972 973 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); 974 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); 975 976 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); 977 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL); 978 979 if (failed) { 980 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED); 981 982 EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index); 983 984 should_abort = eecp->eec_rxq_flush_failed(arg, 985 rxq_index); 986 } else { 987 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); 988 989 EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); 990 991 should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); 992 } 993 994 break; 995 } 996 case FSE_AZ_EVQ_INIT_DONE_EV: 997 EFSYS_ASSERT(eecp->eec_initialized != NULL); 998 should_abort = eecp->eec_initialized(arg); 999 1000 break; 1001 1002 case FSE_AZ_EVQ_NOT_EN_EV: 1003 EFSYS_PROBE(evq_not_en); 1004 break; 1005 1006 case FSE_AZ_SRM_UPD_DONE_EV: { 1007 uint32_t code; 1008 1009 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE); 1010 1011 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 1012 1013 EFSYS_ASSERT(eecp->eec_sram != NULL); 1014 should_abort = eecp->eec_sram(arg, code); 1015 1016 break; 1017 } 1018 case FSE_AZ_WAKE_UP_EV: { 1019 uint32_t id; 1020 1021 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 1022 1023 EFSYS_ASSERT(eecp->eec_wake_up != NULL); 1024 should_abort = eecp->eec_wake_up(arg, id); 1025 1026 break; 1027 } 1028 case FSE_AZ_TX_PKT_NON_TCP_UDP: 1029 EFSYS_PROBE(tx_pkt_non_tcp_udp); 1030 break; 1031 1032 case FSE_AZ_TIMER_EV: { 1033 uint32_t id; 1034 1035 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 1036 1037 EFSYS_ASSERT(eecp->eec_timer != NULL); 1038 should_abort = eecp->eec_timer(arg, id); 1039 1040 break; 1041 } 1042 case FSE_AZ_RX_DSC_ERROR_EV: 1043 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR); 1044 1045 EFSYS_PROBE(rx_dsc_error); 1046 1047 EFSYS_ASSERT(eecp->eec_exception != NULL); 1048 should_abort = eecp->eec_exception(arg, 1049 EFX_EXCEPTION_RX_DSC_ERROR, 0); 1050 1051 break; 1052 1053 case FSE_AZ_TX_DSC_ERROR_EV: 1054 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR); 1055 1056 EFSYS_PROBE(tx_dsc_error); 1057 1058 EFSYS_ASSERT(eecp->eec_exception != NULL); 1059 should_abort = eecp->eec_exception(arg, 1060 EFX_EXCEPTION_TX_DSC_ERROR, 0); 1061 1062 break; 1063 1064 default: 1065 break; 1066 } 1067 1068 return (should_abort); 1069} 1070 1071static __checkReturn boolean_t 1072siena_ev_drv_gen( 1073 __in efx_evq_t *eep, 1074 __in efx_qword_t *eqp, 1075 __in const efx_ev_callbacks_t *eecp, 1076 __in_opt void *arg) 1077{ 1078 uint32_t data; 1079 boolean_t should_abort; 1080 1081 EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); 1082 1083 data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0); 1084 if (data >= ((uint32_t)1 << 16)) { 1085 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 1086 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 1087 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 1088 return (B_TRUE); 1089 } 1090 1091 EFSYS_ASSERT(eecp->eec_software != NULL); 1092 should_abort = eecp->eec_software(arg, (uint16_t)data); 1093 1094 return (should_abort); 1095} 1096 1097#if EFSYS_OPT_MCDI 1098 1099static __checkReturn boolean_t 1100siena_ev_mcdi( 1101 __in efx_evq_t *eep, 1102 __in efx_qword_t *eqp, 1103 __in const efx_ev_callbacks_t *eecp, 1104 __in_opt void *arg) 1105{ 1106 efx_nic_t *enp = eep->ee_enp; 1107 unsigned int code; 1108 boolean_t should_abort = B_FALSE; 1109 1110 EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); 1111 1112 if (enp->en_family != EFX_FAMILY_SIENA) 1113 goto out; 1114 1115 EFSYS_ASSERT(eecp->eec_link_change != NULL); 1116 EFSYS_ASSERT(eecp->eec_exception != NULL); 1117#if EFSYS_OPT_MON_STATS 1118 EFSYS_ASSERT(eecp->eec_monitor != NULL); 1119#endif 1120 1121 EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); 1122 1123 code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); 1124 switch (code) { 1125 case MCDI_EVENT_CODE_BADSSERT: 1126 efx_mcdi_ev_death(enp, EINTR); 1127 break; 1128 1129 case MCDI_EVENT_CODE_CMDDONE: 1130 efx_mcdi_ev_cpl(enp, 1131 MCDI_EV_FIELD(eqp, CMDDONE_SEQ), 1132 MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), 1133 MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); 1134 break; 1135 1136 case MCDI_EVENT_CODE_LINKCHANGE: { 1137 efx_link_mode_t link_mode; 1138 1139 siena_phy_link_ev(enp, eqp, &link_mode); 1140 should_abort = eecp->eec_link_change(arg, link_mode); 1141 break; 1142 } 1143 case MCDI_EVENT_CODE_SENSOREVT: { 1144#if EFSYS_OPT_MON_STATS 1145 efx_mon_stat_t id; 1146 efx_mon_stat_value_t value; 1147 efx_rc_t rc; 1148 1149 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) 1150 should_abort = eecp->eec_monitor(arg, id, value); 1151 else if (rc == ENOTSUP) { 1152 should_abort = eecp->eec_exception(arg, 1153 EFX_EXCEPTION_UNKNOWN_SENSOREVT, 1154 MCDI_EV_FIELD(eqp, DATA)); 1155 } else 1156 EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ 1157#else 1158 should_abort = B_FALSE; 1159#endif 1160 break; 1161 } 1162 case MCDI_EVENT_CODE_SCHEDERR: 1163 /* Informational only */ 1164 break; 1165 1166 case MCDI_EVENT_CODE_REBOOT: 1167 efx_mcdi_ev_death(enp, EIO); 1168 break; 1169 1170 case MCDI_EVENT_CODE_MAC_STATS_DMA: 1171#if EFSYS_OPT_MAC_STATS 1172 if (eecp->eec_mac_stats != NULL) { 1173 eecp->eec_mac_stats(arg, 1174 MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); 1175 } 1176#endif 1177 break; 1178 1179 case MCDI_EVENT_CODE_FWALERT: { 1180 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); 1181 1182 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) 1183 should_abort = eecp->eec_exception(arg, 1184 EFX_EXCEPTION_FWALERT_SRAM, 1185 MCDI_EV_FIELD(eqp, FWALERT_DATA)); 1186 else 1187 should_abort = eecp->eec_exception(arg, 1188 EFX_EXCEPTION_UNKNOWN_FWALERT, 1189 MCDI_EV_FIELD(eqp, DATA)); 1190 break; 1191 } 1192 1193 default: 1194 EFSYS_PROBE1(mc_pcol_error, int, code); 1195 break; 1196 } 1197 1198out: 1199 return (should_abort); 1200} 1201 1202#endif /* EFSYS_OPT_MCDI */ 1203 1204static __checkReturn efx_rc_t 1205siena_ev_qprime( 1206 __in efx_evq_t *eep, 1207 __in unsigned int count) 1208{ 1209 efx_nic_t *enp = eep->ee_enp; 1210 uint32_t rptr; 1211 efx_dword_t dword; 1212 1213 rptr = count & eep->ee_mask; 1214 1215 EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr); 1216 1217 EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index, 1218 &dword, B_FALSE); 1219 1220 return (0); 1221} 1222 1223static void 1224siena_ev_qpost( 1225 __in efx_evq_t *eep, 1226 __in uint16_t data) 1227{ 1228 efx_nic_t *enp = eep->ee_enp; 1229 efx_qword_t ev; 1230 efx_oword_t oword; 1231 1232 EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV, 1233 FSF_AZ_EV_DATA_DW0, (uint32_t)data); 1234 1235 EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index, 1236 EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0), 1237 EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1)); 1238 1239 EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword); 1240} 1241 1242static __checkReturn efx_rc_t 1243siena_ev_qmoderate( 1244 __in efx_evq_t *eep, 1245 __in unsigned int us) 1246{ 1247 efx_nic_t *enp = eep->ee_enp; 1248 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1249 unsigned int locked; 1250 efx_dword_t dword; 1251 efx_rc_t rc; 1252 1253 if (us > encp->enc_evq_timer_max_us) { 1254 rc = EINVAL; 1255 goto fail1; 1256 } 1257 1258 /* If the value is zero then disable the timer */ 1259 if (us == 0) { 1260 EFX_POPULATE_DWORD_2(dword, 1261 FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS, 1262 FRF_CZ_TC_TIMER_VAL, 0); 1263 } else { 1264 unsigned int ticks; 1265 1266 if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) 1267 goto fail2; 1268 1269 EFSYS_ASSERT(ticks > 0); 1270 EFX_POPULATE_DWORD_2(dword, 1271 FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF, 1272 FRF_CZ_TC_TIMER_VAL, ticks - 1); 1273 } 1274 1275 locked = (eep->ee_index == 0) ? 1 : 0; 1276 1277 EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0, 1278 eep->ee_index, &dword, locked); 1279 1280 return (0); 1281 1282fail2: 1283 EFSYS_PROBE(fail2); 1284fail1: 1285 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1286 1287 return (rc); 1288} 1289 1290static __checkReturn efx_rc_t 1291siena_ev_qcreate( 1292 __in efx_nic_t *enp, 1293 __in unsigned int index, 1294 __in efsys_mem_t *esmp, 1295 __in size_t n, 1296 __in uint32_t id, 1297 __in uint32_t us, 1298 __in uint32_t flags, 1299 __in efx_evq_t *eep) 1300{ 1301 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1302 uint32_t size; 1303 efx_oword_t oword; 1304 efx_rc_t rc; 1305 boolean_t notify_mode; 1306 1307 _NOTE(ARGUNUSED(esmp)) 1308 1309 EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS)); 1310 EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS)); 1311 1312 if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) { 1313 rc = EINVAL; 1314 goto fail1; 1315 } 1316 if (index >= encp->enc_evq_limit) { 1317 rc = EINVAL; 1318 goto fail2; 1319 } 1320#if EFSYS_OPT_RX_SCALE 1321 if (enp->en_intr.ei_type == EFX_INTR_LINE && 1322 index >= EFX_MAXRSS_LEGACY) { 1323 rc = EINVAL; 1324 goto fail3; 1325 } 1326#endif 1327 for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS); 1328 size++) 1329 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS)) 1330 break; 1331 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1332 rc = EINVAL; 1333 goto fail4; 1334 } 1335 1336 /* Set up the handler table */ 1337 eep->ee_rx = siena_ev_rx; 1338 eep->ee_tx = siena_ev_tx; 1339 eep->ee_driver = siena_ev_driver; 1340 eep->ee_global = siena_ev_global; 1341 eep->ee_drv_gen = siena_ev_drv_gen; 1342#if EFSYS_OPT_MCDI 1343 eep->ee_mcdi = siena_ev_mcdi; 1344#endif /* EFSYS_OPT_MCDI */ 1345 1346 notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) != 1347 EFX_EVQ_FLAGS_NOTIFY_INTERRUPT); 1348 1349 /* Set up the new event queue */ 1350 EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1, 1351 FRF_CZ_HOST_NOTIFY_MODE, notify_mode, 1352 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); 1353 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE); 1354 1355 EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size, 1356 FRF_AZ_EVQ_BUF_BASE_ID, id); 1357 1358 EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE); 1359 1360 /* Set initial interrupt moderation */ 1361 siena_ev_qmoderate(eep, us); 1362 1363 return (0); 1364 1365fail4: 1366 EFSYS_PROBE(fail4); 1367#if EFSYS_OPT_RX_SCALE 1368fail3: 1369 EFSYS_PROBE(fail3); 1370#endif 1371fail2: 1372 EFSYS_PROBE(fail2); 1373fail1: 1374 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1375 1376 return (rc); 1377} 1378 1379#endif /* EFSYS_OPT_SIENA */ 1380 1381#if EFSYS_OPT_QSTATS 1382#if EFSYS_OPT_NAMES 1383/* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock c0f3bc5083b40532 */ 1384static const char * const __efx_ev_qstat_name[] = { 1385 "all", 1386 "rx", 1387 "rx_ok", 1388 "rx_frm_trunc", 1389 "rx_tobe_disc", 1390 "rx_pause_frm_err", 1391 "rx_buf_owner_id_err", 1392 "rx_ipv4_hdr_chksum_err", 1393 "rx_tcp_udp_chksum_err", 1394 "rx_eth_crc_err", 1395 "rx_ip_frag_err", 1396 "rx_mcast_pkt", 1397 "rx_mcast_hash_match", 1398 "rx_tcp_ipv4", 1399 "rx_tcp_ipv6", 1400 "rx_udp_ipv4", 1401 "rx_udp_ipv6", 1402 "rx_other_ipv4", 1403 "rx_other_ipv6", 1404 "rx_non_ip", 1405 "rx_batch", 1406 "tx", 1407 "tx_wq_ff_full", 1408 "tx_pkt_err", 1409 "tx_pkt_too_big", 1410 "tx_unexpected", 1411 "global", 1412 "global_mnt", 1413 "driver", 1414 "driver_srm_upd_done", 1415 "driver_tx_descq_fls_done", 1416 "driver_rx_descq_fls_done", 1417 "driver_rx_descq_fls_failed", 1418 "driver_rx_dsc_error", 1419 "driver_tx_dsc_error", 1420 "drv_gen", 1421 "mcdi_response", 1422}; 1423/* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */ 1424 1425 const char * 1426efx_ev_qstat_name( 1427 __in efx_nic_t *enp, 1428 __in unsigned int id) 1429{ 1430 _NOTE(ARGUNUSED(enp)) 1431 1432 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1433 EFSYS_ASSERT3U(id, <, EV_NQSTATS); 1434 1435 return (__efx_ev_qstat_name[id]); 1436} 1437#endif /* EFSYS_OPT_NAMES */ 1438#endif /* EFSYS_OPT_QSTATS */ 1439 1440#if EFSYS_OPT_SIENA 1441 1442#if EFSYS_OPT_QSTATS 1443static void 1444siena_ev_qstats_update( 1445 __in efx_evq_t *eep, 1446 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 1447{ 1448 unsigned int id; 1449 1450 for (id = 0; id < EV_NQSTATS; id++) { 1451 efsys_stat_t *essp = &stat[id]; 1452 1453 EFSYS_STAT_INCR(essp, eep->ee_stat[id]); 1454 eep->ee_stat[id] = 0; 1455 } 1456} 1457#endif /* EFSYS_OPT_QSTATS */ 1458 1459static void 1460siena_ev_qdestroy( 1461 __in efx_evq_t *eep) 1462{ 1463 efx_nic_t *enp = eep->ee_enp; 1464 efx_oword_t oword; 1465 1466 /* Purge event queue */ 1467 EFX_ZERO_OWORD(oword); 1468 1469 EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, 1470 eep->ee_index, &oword, B_TRUE); 1471 1472 EFX_ZERO_OWORD(oword); 1473 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE); 1474} 1475 1476static void 1477siena_ev_fini( 1478 __in efx_nic_t *enp) 1479{ 1480 _NOTE(ARGUNUSED(enp)) 1481} 1482 1483#endif /* EFSYS_OPT_SIENA */ 1484