sfxge_ev.c revision 280517
1/*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_ev.c 280517 2015-03-25 10:21:42Z arybchik $"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35 36#include "common/efx.h" 37 38#include "sfxge.h" 39 40static void 41sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 42{ 43 struct sfxge_softc *sc; 44 unsigned int index; 45 struct sfxge_rxq *rxq; 46 struct sfxge_txq *txq; 47 48 sc = evq->sc; 49 index = evq->index; 50 rxq = sc->rxq[index]; 51 52 if ((txq = evq->txq) != NULL) { 53 evq->txq = NULL; 54 evq->txqs = &(evq->txq); 55 56 do { 57 struct sfxge_txq *next; 58 59 next = txq->next; 60 txq->next = NULL; 61 62 KASSERT(txq->evq_index == index, 63 ("txq->evq_index != index")); 64 65 if (txq->pending != txq->completed) 66 sfxge_tx_qcomplete(txq, evq); 67 68 txq = next; 69 } while (txq != NULL); 70 } 71 72 if (rxq->pending != rxq->completed) 73 sfxge_rx_qcomplete(rxq, eop); 74} 75 76static boolean_t 77sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 78 uint16_t flags) 79{ 80 struct sfxge_evq *evq; 81 struct sfxge_softc *sc; 82 struct sfxge_rxq *rxq; 83 unsigned int expected; 84 struct sfxge_rx_sw_desc *rx_desc; 85 86 evq = arg; 87 sc = evq->sc; 88 89 if (evq->exception) 90 goto done; 91 92 rxq = sc->rxq[label]; 93 KASSERT(rxq != NULL, ("rxq == NULL")); 94 KASSERT(evq->index == rxq->index, 95 ("evq->index != rxq->index")); 96 97 if (rxq->init_state != SFXGE_RXQ_STARTED) 98 goto done; 99 100 expected = rxq->pending++ & rxq->ptr_mask; 101 if (id != expected) { 102 evq->exception = B_TRUE; 103 104 device_printf(sc->dev, "RX completion out of order" 105 " (id=%#x expected=%#x flags=%#x); resetting\n", 106 id, expected, flags); 107 sfxge_schedule_reset(sc); 108 109 goto done; 110 } 111 112 rx_desc = &rxq->queue[id]; 113 114 KASSERT(rx_desc->flags == EFX_DISCARD, 115 ("rx_desc->flags != EFX_DISCARD")); 116 rx_desc->flags = flags; 117 118 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 119 rx_desc->size = (uint16_t)size; 120 prefetch_read_many(rx_desc->mbuf); 121 122 evq->rx_done++; 123 124 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 125 sfxge_ev_qcomplete(evq, B_FALSE); 126 127done: 128 return (evq->rx_done >= SFXGE_EV_BATCH); 129} 130 131static boolean_t 132sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 133{ 134 struct sfxge_evq *evq; 135 struct sfxge_softc *sc; 136 137 evq = (struct sfxge_evq *)arg; 138 sc = evq->sc; 139 140 evq->exception = B_TRUE; 141 142 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 143 device_printf(sc->dev, 144 "hardware exception (code=%u); resetting\n", 145 code); 146 sfxge_schedule_reset(sc); 147 } 148 149 return (B_FALSE); 150} 151 152static boolean_t 153sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 154{ 155 struct sfxge_evq *evq; 156 struct sfxge_softc *sc; 157 struct sfxge_rxq *rxq; 158 unsigned int index; 159 unsigned int label; 160 uint16_t magic; 161 162 evq = (struct sfxge_evq *)arg; 163 sc = evq->sc; 164 rxq = sc->rxq[rxq_index]; 165 166 KASSERT(rxq != NULL, ("rxq == NULL")); 167 168 /* Resend a software event on the correct queue */ 169 index = rxq->index; 170 evq = sc->evq[index]; 171 172 label = rxq_index; 173 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 174 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 175 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 176 177 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 178 ("evq not started")); 179 efx_ev_qpost(evq->common, magic); 180 181 return (B_FALSE); 182} 183 184static boolean_t 185sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 186{ 187 struct sfxge_evq *evq; 188 struct sfxge_softc *sc; 189 struct sfxge_rxq *rxq; 190 unsigned int index; 191 unsigned int label; 192 uint16_t magic; 193 194 evq = (struct sfxge_evq *)arg; 195 sc = evq->sc; 196 rxq = sc->rxq[rxq_index]; 197 198 KASSERT(rxq != NULL, ("rxq == NULL")); 199 200 /* Resend a software event on the correct queue */ 201 index = rxq->index; 202 evq = sc->evq[index]; 203 204 label = rxq_index; 205 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 206 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 207 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 208 209 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 210 ("evq not started")); 211 efx_ev_qpost(evq->common, magic); 212 213 return (B_FALSE); 214} 215 216static struct sfxge_txq * 217sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 218{ 219 unsigned int index; 220 221 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 222 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 223 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 224 return (evq->sc->txq[index]); 225} 226 227static boolean_t 228sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 229{ 230 struct sfxge_evq *evq; 231 struct sfxge_txq *txq; 232 unsigned int stop; 233 unsigned int delta; 234 235 evq = (struct sfxge_evq *)arg; 236 txq = sfxge_get_txq_by_label(evq, label); 237 238 KASSERT(txq != NULL, ("txq == NULL")); 239 KASSERT(evq->index == txq->evq_index, 240 ("evq->index != txq->evq_index")); 241 242 if (txq->init_state != SFXGE_TXQ_STARTED) 243 goto done; 244 245 stop = (id + 1) & txq->ptr_mask; 246 id = txq->pending & txq->ptr_mask; 247 248 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 249 txq->pending += delta; 250 251 evq->tx_done++; 252 253 if (txq->next == NULL && 254 evq->txqs != &(txq->next)) { 255 *(evq->txqs) = txq; 256 evq->txqs = &(txq->next); 257 } 258 259 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 260 sfxge_tx_qcomplete(txq, evq); 261 262done: 263 return (evq->tx_done >= SFXGE_EV_BATCH); 264} 265 266static boolean_t 267sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 268{ 269 struct sfxge_evq *evq; 270 struct sfxge_softc *sc; 271 struct sfxge_txq *txq; 272 unsigned int label; 273 uint16_t magic; 274 275 evq = (struct sfxge_evq *)arg; 276 sc = evq->sc; 277 txq = sc->txq[txq_index]; 278 279 KASSERT(txq != NULL, ("txq == NULL")); 280 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 281 ("txq not initialized")); 282 283 /* Resend a software event on the correct queue */ 284 evq = sc->evq[txq->evq_index]; 285 286 label = txq->type; 287 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 288 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 289 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 290 291 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 292 ("evq not started")); 293 efx_ev_qpost(evq->common, magic); 294 295 return (B_FALSE); 296} 297 298static boolean_t 299sfxge_ev_software(void *arg, uint16_t magic) 300{ 301 struct sfxge_evq *evq; 302 struct sfxge_softc *sc; 303 unsigned int label; 304 305 evq = (struct sfxge_evq *)arg; 306 sc = evq->sc; 307 308 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 309 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 310 311 switch (magic) { 312 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 313 struct sfxge_rxq *rxq = sc->rxq[label]; 314 315 KASSERT(rxq != NULL, ("rxq == NULL")); 316 KASSERT(evq->index == rxq->index, 317 ("evq->index != rxq->index")); 318 319 sfxge_rx_qflush_done(rxq); 320 break; 321 } 322 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 323 struct sfxge_rxq *rxq = sc->rxq[label]; 324 325 KASSERT(rxq != NULL, ("rxq == NULL")); 326 KASSERT(evq->index == rxq->index, 327 ("evq->index != rxq->index")); 328 329 sfxge_rx_qflush_failed(rxq); 330 break; 331 } 332 case SFXGE_MAGIC_RX_QREFILL: { 333 struct sfxge_rxq *rxq = sc->rxq[label]; 334 335 KASSERT(rxq != NULL, ("rxq == NULL")); 336 KASSERT(evq->index == rxq->index, 337 ("evq->index != rxq->index")); 338 339 sfxge_rx_qrefill(rxq); 340 break; 341 } 342 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 343 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 344 345 KASSERT(txq != NULL, ("txq == NULL")); 346 KASSERT(evq->index == txq->evq_index, 347 ("evq->index != txq->evq_index")); 348 349 sfxge_tx_qflush_done(txq); 350 break; 351 } 352 default: 353 break; 354 } 355 356 return (B_FALSE); 357} 358 359static boolean_t 360sfxge_ev_sram(void *arg, uint32_t code) 361{ 362 (void)arg; 363 (void)code; 364 365 switch (code) { 366 case EFX_SRAM_UPDATE: 367 EFSYS_PROBE(sram_update); 368 break; 369 370 case EFX_SRAM_CLEAR: 371 EFSYS_PROBE(sram_clear); 372 break; 373 374 case EFX_SRAM_ILLEGAL_CLEAR: 375 EFSYS_PROBE(sram_illegal_clear); 376 break; 377 378 default: 379 KASSERT(B_FALSE, ("Impossible SRAM event")); 380 break; 381 } 382 383 return (B_FALSE); 384} 385 386static boolean_t 387sfxge_ev_timer(void *arg, uint32_t index) 388{ 389 (void)arg; 390 (void)index; 391 392 return (B_FALSE); 393} 394 395static boolean_t 396sfxge_ev_wake_up(void *arg, uint32_t index) 397{ 398 (void)arg; 399 (void)index; 400 401 return (B_FALSE); 402} 403 404#if EFSYS_OPT_QSTATS 405 406static void 407sfxge_ev_stat_update(struct sfxge_softc *sc) 408{ 409 struct sfxge_evq *evq; 410 unsigned int index; 411 clock_t now; 412 413 sx_xlock(&sc->softc_lock); 414 415 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 416 goto out; 417 418 now = ticks; 419 if (now - sc->ev_stats_update_time < hz) 420 goto out; 421 422 sc->ev_stats_update_time = now; 423 424 /* Add event counts from each event queue in turn */ 425 for (index = 0; index < sc->intr.n_alloc; index++) { 426 evq = sc->evq[index]; 427 mtx_lock(&evq->lock); 428 efx_ev_qstats_update(evq->common, sc->ev_stats); 429 mtx_unlock(&evq->lock); 430 } 431out: 432 sx_xunlock(&sc->softc_lock); 433} 434 435static int 436sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 437{ 438 struct sfxge_softc *sc = arg1; 439 unsigned int id = arg2; 440 441 sfxge_ev_stat_update(sc); 442 443 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 444} 445 446static void 447sfxge_ev_stat_init(struct sfxge_softc *sc) 448{ 449 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 450 struct sysctl_oid_list *stat_list; 451 unsigned int id; 452 char name[40]; 453 454 stat_list = SYSCTL_CHILDREN(sc->stats_node); 455 456 for (id = 0; id < EV_NQSTATS; id++) { 457 snprintf(name, sizeof(name), "ev_%s", 458 efx_ev_qstat_name(sc->enp, id)); 459 SYSCTL_ADD_PROC( 460 ctx, stat_list, 461 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 462 sc, id, sfxge_ev_stat_handler, "Q", 463 ""); 464 } 465} 466 467#endif /* EFSYS_OPT_QSTATS */ 468 469static void 470sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 471{ 472 struct sfxge_evq *evq; 473 efx_evq_t *eep; 474 475 evq = sc->evq[idx]; 476 eep = evq->common; 477 478 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 479 ("evq->init_state != SFXGE_EVQ_STARTED")); 480 481 (void)efx_ev_qmoderate(eep, us); 482} 483 484static int 485sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 486{ 487 struct sfxge_softc *sc = arg1; 488 struct sfxge_intr *intr = &sc->intr; 489 unsigned int moderation; 490 int error; 491 int index; 492 493 sx_xlock(&sc->softc_lock); 494 495 if (req->newptr != NULL) { 496 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 497 != 0) 498 goto out; 499 500 /* We may not be calling efx_ev_qmoderate() now, 501 * so we have to range-check the value ourselves. 502 */ 503 if (moderation > 504 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 505 error = EINVAL; 506 goto out; 507 } 508 509 sc->ev_moderation = moderation; 510 if (intr->state == SFXGE_INTR_STARTED) { 511 for (index = 0; index < intr->n_alloc; index++) 512 sfxge_ev_qmoderate(sc, index, moderation); 513 } 514 } else { 515 error = SYSCTL_OUT(req, &sc->ev_moderation, 516 sizeof(sc->ev_moderation)); 517 } 518 519out: 520 sx_xunlock(&sc->softc_lock); 521 522 return (error); 523} 524 525static boolean_t 526sfxge_ev_initialized(void *arg) 527{ 528 struct sfxge_evq *evq; 529 530 evq = (struct sfxge_evq *)arg; 531 532 KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 533 ("evq not starting")); 534 535 evq->init_state = SFXGE_EVQ_STARTED; 536 537 return (0); 538} 539 540static boolean_t 541sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 542{ 543 struct sfxge_evq *evq; 544 struct sfxge_softc *sc; 545 546 evq = (struct sfxge_evq *)arg; 547 sc = evq->sc; 548 549 sfxge_mac_link_update(sc, link_mode); 550 551 return (0); 552} 553 554static const efx_ev_callbacks_t sfxge_ev_callbacks = { 555 .eec_initialized = sfxge_ev_initialized, 556 .eec_rx = sfxge_ev_rx, 557 .eec_tx = sfxge_ev_tx, 558 .eec_exception = sfxge_ev_exception, 559 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 560 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 561 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 562 .eec_software = sfxge_ev_software, 563 .eec_sram = sfxge_ev_sram, 564 .eec_wake_up = sfxge_ev_wake_up, 565 .eec_timer = sfxge_ev_timer, 566 .eec_link_change = sfxge_ev_link_change, 567}; 568 569 570int 571sfxge_ev_qpoll(struct sfxge_evq *evq) 572{ 573 int rc; 574 575 mtx_lock(&evq->lock); 576 577 if (evq->init_state != SFXGE_EVQ_STARTING && 578 evq->init_state != SFXGE_EVQ_STARTED) { 579 rc = EINVAL; 580 goto fail; 581 } 582 583 /* Synchronize the DMA memory for reading */ 584 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 585 BUS_DMASYNC_POSTREAD); 586 587 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 588 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 589 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 590 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 591 592 /* Poll the queue */ 593 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 594 595 evq->rx_done = 0; 596 evq->tx_done = 0; 597 598 /* Perform any pending completion processing */ 599 sfxge_ev_qcomplete(evq, B_TRUE); 600 601 /* Re-prime the event queue for interrupts */ 602 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 603 goto fail; 604 605 mtx_unlock(&evq->lock); 606 607 return (0); 608 609fail: 610 mtx_unlock(&(evq->lock)); 611 return (rc); 612} 613 614static void 615sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 616{ 617 struct sfxge_evq *evq; 618 619 evq = sc->evq[index]; 620 621 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 622 ("evq->init_state != SFXGE_EVQ_STARTED")); 623 624 mtx_lock(&evq->lock); 625 evq->init_state = SFXGE_EVQ_INITIALIZED; 626 evq->read_ptr = 0; 627 evq->exception = B_FALSE; 628 629#if EFSYS_OPT_QSTATS 630 /* Add event counts before discarding the common evq state */ 631 efx_ev_qstats_update(evq->common, sc->ev_stats); 632#endif 633 634 efx_ev_qdestroy(evq->common); 635 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 636 EFX_EVQ_NBUFS(evq->entries)); 637 mtx_unlock(&evq->lock); 638} 639 640static int 641sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 642{ 643 struct sfxge_evq *evq; 644 efsys_mem_t *esmp; 645 int count; 646 int rc; 647 648 evq = sc->evq[index]; 649 esmp = &evq->mem; 650 651 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 652 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 653 654 /* Clear all events. */ 655 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 656 657 /* Program the buffer table. */ 658 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 659 EFX_EVQ_NBUFS(evq->entries))) != 0) 660 return (rc); 661 662 /* Create the common code event queue. */ 663 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 664 evq->buf_base_id, &evq->common)) != 0) 665 goto fail; 666 667 mtx_lock(&evq->lock); 668 669 /* Set the default moderation */ 670 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 671 672 /* Prime the event queue for interrupts */ 673 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 674 goto fail2; 675 676 evq->init_state = SFXGE_EVQ_STARTING; 677 678 mtx_unlock(&evq->lock); 679 680 /* Wait for the initialization event */ 681 count = 0; 682 do { 683 /* Pause for 100 ms */ 684 pause("sfxge evq init", hz / 10); 685 686 /* Check to see if the test event has been processed */ 687 if (evq->init_state == SFXGE_EVQ_STARTED) 688 goto done; 689 690 } while (++count < 20); 691 692 rc = ETIMEDOUT; 693 goto fail3; 694 695done: 696 return (0); 697 698fail3: 699 mtx_lock(&evq->lock); 700 evq->init_state = SFXGE_EVQ_INITIALIZED; 701fail2: 702 mtx_unlock(&evq->lock); 703 efx_ev_qdestroy(evq->common); 704fail: 705 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 706 EFX_EVQ_NBUFS(evq->entries)); 707 708 return (rc); 709} 710 711void 712sfxge_ev_stop(struct sfxge_softc *sc) 713{ 714 struct sfxge_intr *intr; 715 efx_nic_t *enp; 716 int index; 717 718 intr = &sc->intr; 719 enp = sc->enp; 720 721 KASSERT(intr->state == SFXGE_INTR_STARTED, 722 ("Interrupts not started")); 723 724 /* Stop the event queue(s) */ 725 index = intr->n_alloc; 726 while (--index >= 0) 727 sfxge_ev_qstop(sc, index); 728 729 /* Tear down the event module */ 730 efx_ev_fini(enp); 731} 732 733int 734sfxge_ev_start(struct sfxge_softc *sc) 735{ 736 struct sfxge_intr *intr; 737 int index; 738 int rc; 739 740 intr = &sc->intr; 741 742 KASSERT(intr->state == SFXGE_INTR_STARTED, 743 ("intr->state != SFXGE_INTR_STARTED")); 744 745 /* Initialize the event module */ 746 if ((rc = efx_ev_init(sc->enp)) != 0) 747 return (rc); 748 749 /* Start the event queues */ 750 for (index = 0; index < intr->n_alloc; index++) { 751 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 752 goto fail; 753 } 754 755 return (0); 756 757fail: 758 /* Stop the event queue(s) */ 759 while (--index >= 0) 760 sfxge_ev_qstop(sc, index); 761 762 /* Tear down the event module */ 763 efx_ev_fini(sc->enp); 764 765 return (rc); 766} 767 768static void 769sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 770{ 771 struct sfxge_evq *evq; 772 773 evq = sc->evq[index]; 774 775 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 776 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 777 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 778 779 sfxge_dma_free(&evq->mem); 780 781 sc->evq[index] = NULL; 782 783 mtx_destroy(&evq->lock); 784 785 free(evq, M_SFXGE); 786} 787 788static int 789sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 790{ 791 struct sfxge_evq *evq; 792 efsys_mem_t *esmp; 793 int rc; 794 795 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 796 797 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 798 evq->sc = sc; 799 evq->index = index; 800 sc->evq[index] = evq; 801 esmp = &evq->mem; 802 803 /* Build an event queue with room for one event per tx and rx buffer, 804 * plus some extra for link state events and MCDI completions. 805 * There are three tx queues in the first event queue and one in 806 * other. 807 */ 808 if (index == 0) 809 evq->entries = 810 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 811 3 * sc->txq_entries + 812 128); 813 else 814 evq->entries = 815 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 816 sc->txq_entries + 817 128); 818 819 /* Initialise TX completion list */ 820 evq->txqs = &evq->txq; 821 822 /* Allocate DMA space. */ 823 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 824 return (rc); 825 826 /* Allocate buffer table entries. */ 827 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 828 &evq->buf_base_id); 829 830 mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 831 832 evq->init_state = SFXGE_EVQ_INITIALIZED; 833 834 return (0); 835} 836 837void 838sfxge_ev_fini(struct sfxge_softc *sc) 839{ 840 struct sfxge_intr *intr; 841 int index; 842 843 intr = &sc->intr; 844 845 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 846 ("intr->state != SFXGE_INTR_INITIALIZED")); 847 848 sc->ev_moderation = 0; 849 850 /* Tear down the event queue(s). */ 851 index = intr->n_alloc; 852 while (--index >= 0) 853 sfxge_ev_qfini(sc, index); 854} 855 856int 857sfxge_ev_init(struct sfxge_softc *sc) 858{ 859 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 860 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 861 struct sfxge_intr *intr; 862 int index; 863 int rc; 864 865 intr = &sc->intr; 866 867 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 868 ("intr->state != SFXGE_INTR_INITIALIZED")); 869 870 /* Set default interrupt moderation; add a sysctl to 871 * read and change it. 872 */ 873 sc->ev_moderation = SFXGE_MODERATION; 874 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 875 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 876 sc, 0, sfxge_int_mod_handler, "IU", 877 "sfxge interrupt moderation (us)"); 878 879 /* 880 * Initialize the event queue(s) - one per interrupt. 881 */ 882 for (index = 0; index < intr->n_alloc; index++) { 883 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 884 goto fail; 885 } 886 887#if EFSYS_OPT_QSTATS 888 sfxge_ev_stat_init(sc); 889#endif 890 891 return (0); 892 893fail: 894 while (--index >= 0) 895 sfxge_ev_qfini(sc, index); 896 897 return (rc); 898} 899