sfxge_ev.c revision 280582
1/*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_ev.c 280582 2015-03-25 13:04:28Z arybchik $"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35 36#include "common/efx.h" 37 38#include "sfxge.h" 39 40static void 41sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 42{ 43 struct sfxge_softc *sc; 44 unsigned int index; 45 struct sfxge_rxq *rxq; 46 struct sfxge_txq *txq; 47 48 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 49 50 sc = evq->sc; 51 index = evq->index; 52 rxq = sc->rxq[index]; 53 54 if ((txq = evq->txq) != NULL) { 55 evq->txq = NULL; 56 evq->txqs = &(evq->txq); 57 58 do { 59 struct sfxge_txq *next; 60 61 next = txq->next; 62 txq->next = NULL; 63 64 KASSERT(txq->evq_index == index, 65 ("txq->evq_index != index")); 66 67 if (txq->pending != txq->completed) 68 sfxge_tx_qcomplete(txq, evq); 69 70 txq = next; 71 } while (txq != NULL); 72 } 73 74 if (rxq->pending != rxq->completed) 75 sfxge_rx_qcomplete(rxq, eop); 76} 77 78static boolean_t 79sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 80 uint16_t flags) 81{ 82 struct sfxge_evq *evq; 83 struct sfxge_softc *sc; 84 struct sfxge_rxq *rxq; 85 unsigned int expected; 86 struct sfxge_rx_sw_desc *rx_desc; 87 88 evq = arg; 89 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 90 91 sc = evq->sc; 92 93 if (evq->exception) 94 goto done; 95 96 rxq = sc->rxq[label]; 97 KASSERT(rxq != NULL, ("rxq == NULL")); 98 KASSERT(evq->index == rxq->index, 99 ("evq->index != rxq->index")); 100 101 if (rxq->init_state != SFXGE_RXQ_STARTED) 102 goto done; 103 104 expected = rxq->pending++ & rxq->ptr_mask; 105 if (id != expected) { 106 evq->exception = B_TRUE; 107 108 device_printf(sc->dev, "RX completion out of order" 109 " (id=%#x expected=%#x flags=%#x); resetting\n", 110 id, expected, flags); 111 sfxge_schedule_reset(sc); 112 113 goto done; 114 } 115 116 rx_desc = &rxq->queue[id]; 117 118 KASSERT(rx_desc->flags == EFX_DISCARD, 119 ("rx_desc->flags != EFX_DISCARD")); 120 rx_desc->flags = flags; 121 122 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 123 rx_desc->size = (uint16_t)size; 124 prefetch_read_many(rx_desc->mbuf); 125 126 evq->rx_done++; 127 128 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 129 sfxge_ev_qcomplete(evq, B_FALSE); 130 131done: 132 return (evq->rx_done >= SFXGE_EV_BATCH); 133} 134 135static boolean_t 136sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 137{ 138 struct sfxge_evq *evq; 139 struct sfxge_softc *sc; 140 141 evq = (struct sfxge_evq *)arg; 142 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 143 144 sc = evq->sc; 145 146 evq->exception = B_TRUE; 147 148 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 149 device_printf(sc->dev, 150 "hardware exception (code=%u); resetting\n", 151 code); 152 sfxge_schedule_reset(sc); 153 } 154 155 return (B_FALSE); 156} 157 158static boolean_t 159sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 160{ 161 struct sfxge_evq *evq; 162 struct sfxge_softc *sc; 163 struct sfxge_rxq *rxq; 164 unsigned int index; 165 unsigned int label; 166 uint16_t magic; 167 168 evq = (struct sfxge_evq *)arg; 169 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 170 171 sc = evq->sc; 172 rxq = sc->rxq[rxq_index]; 173 174 KASSERT(rxq != NULL, ("rxq == NULL")); 175 176 /* Resend a software event on the correct queue */ 177 index = rxq->index; 178 evq = sc->evq[index]; 179 180 label = rxq_index; 181 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 182 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 183 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 184 185 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 186 ("evq not started")); 187 efx_ev_qpost(evq->common, magic); 188 189 return (B_FALSE); 190} 191 192static boolean_t 193sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 194{ 195 struct sfxge_evq *evq; 196 struct sfxge_softc *sc; 197 struct sfxge_rxq *rxq; 198 unsigned int index; 199 unsigned int label; 200 uint16_t magic; 201 202 evq = (struct sfxge_evq *)arg; 203 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 204 205 sc = evq->sc; 206 rxq = sc->rxq[rxq_index]; 207 208 KASSERT(rxq != NULL, ("rxq == NULL")); 209 210 /* Resend a software event on the correct queue */ 211 index = rxq->index; 212 evq = sc->evq[index]; 213 214 label = rxq_index; 215 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 216 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 217 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 218 219 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 220 ("evq not started")); 221 efx_ev_qpost(evq->common, magic); 222 223 return (B_FALSE); 224} 225 226static struct sfxge_txq * 227sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 228{ 229 unsigned int index; 230 231 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 232 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 233 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 234 return (evq->sc->txq[index]); 235} 236 237static boolean_t 238sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 239{ 240 struct sfxge_evq *evq; 241 struct sfxge_txq *txq; 242 unsigned int stop; 243 unsigned int delta; 244 245 evq = (struct sfxge_evq *)arg; 246 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 247 248 txq = sfxge_get_txq_by_label(evq, label); 249 250 KASSERT(txq != NULL, ("txq == NULL")); 251 KASSERT(evq->index == txq->evq_index, 252 ("evq->index != txq->evq_index")); 253 254 if (txq->init_state != SFXGE_TXQ_STARTED) 255 goto done; 256 257 stop = (id + 1) & txq->ptr_mask; 258 id = txq->pending & txq->ptr_mask; 259 260 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 261 txq->pending += delta; 262 263 evq->tx_done++; 264 265 if (txq->next == NULL && 266 evq->txqs != &(txq->next)) { 267 *(evq->txqs) = txq; 268 evq->txqs = &(txq->next); 269 } 270 271 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 272 sfxge_tx_qcomplete(txq, evq); 273 274done: 275 return (evq->tx_done >= SFXGE_EV_BATCH); 276} 277 278static boolean_t 279sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 280{ 281 struct sfxge_evq *evq; 282 struct sfxge_softc *sc; 283 struct sfxge_txq *txq; 284 unsigned int label; 285 uint16_t magic; 286 287 evq = (struct sfxge_evq *)arg; 288 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 289 290 sc = evq->sc; 291 txq = sc->txq[txq_index]; 292 293 KASSERT(txq != NULL, ("txq == NULL")); 294 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 295 ("txq not initialized")); 296 297 /* Resend a software event on the correct queue */ 298 evq = sc->evq[txq->evq_index]; 299 300 label = txq->type; 301 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 302 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 303 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 304 305 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 306 ("evq not started")); 307 efx_ev_qpost(evq->common, magic); 308 309 return (B_FALSE); 310} 311 312static boolean_t 313sfxge_ev_software(void *arg, uint16_t magic) 314{ 315 struct sfxge_evq *evq; 316 struct sfxge_softc *sc; 317 unsigned int label; 318 319 evq = (struct sfxge_evq *)arg; 320 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 321 322 sc = evq->sc; 323 324 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 325 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 326 327 switch (magic) { 328 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 329 struct sfxge_rxq *rxq = sc->rxq[label]; 330 331 KASSERT(rxq != NULL, ("rxq == NULL")); 332 KASSERT(evq->index == rxq->index, 333 ("evq->index != rxq->index")); 334 335 sfxge_rx_qflush_done(rxq); 336 break; 337 } 338 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 339 struct sfxge_rxq *rxq = sc->rxq[label]; 340 341 KASSERT(rxq != NULL, ("rxq == NULL")); 342 KASSERT(evq->index == rxq->index, 343 ("evq->index != rxq->index")); 344 345 sfxge_rx_qflush_failed(rxq); 346 break; 347 } 348 case SFXGE_MAGIC_RX_QREFILL: { 349 struct sfxge_rxq *rxq = sc->rxq[label]; 350 351 KASSERT(rxq != NULL, ("rxq == NULL")); 352 KASSERT(evq->index == rxq->index, 353 ("evq->index != rxq->index")); 354 355 sfxge_rx_qrefill(rxq); 356 break; 357 } 358 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 359 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 360 361 KASSERT(txq != NULL, ("txq == NULL")); 362 KASSERT(evq->index == txq->evq_index, 363 ("evq->index != txq->evq_index")); 364 365 sfxge_tx_qflush_done(txq); 366 break; 367 } 368 default: 369 break; 370 } 371 372 return (B_FALSE); 373} 374 375static boolean_t 376sfxge_ev_sram(void *arg, uint32_t code) 377{ 378 (void)arg; 379 (void)code; 380 381 switch (code) { 382 case EFX_SRAM_UPDATE: 383 EFSYS_PROBE(sram_update); 384 break; 385 386 case EFX_SRAM_CLEAR: 387 EFSYS_PROBE(sram_clear); 388 break; 389 390 case EFX_SRAM_ILLEGAL_CLEAR: 391 EFSYS_PROBE(sram_illegal_clear); 392 break; 393 394 default: 395 KASSERT(B_FALSE, ("Impossible SRAM event")); 396 break; 397 } 398 399 return (B_FALSE); 400} 401 402static boolean_t 403sfxge_ev_timer(void *arg, uint32_t index) 404{ 405 (void)arg; 406 (void)index; 407 408 return (B_FALSE); 409} 410 411static boolean_t 412sfxge_ev_wake_up(void *arg, uint32_t index) 413{ 414 (void)arg; 415 (void)index; 416 417 return (B_FALSE); 418} 419 420#if EFSYS_OPT_QSTATS 421 422static void 423sfxge_ev_stat_update(struct sfxge_softc *sc) 424{ 425 struct sfxge_evq *evq; 426 unsigned int index; 427 clock_t now; 428 429 SFXGE_ADAPTER_LOCK(sc); 430 431 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 432 goto out; 433 434 now = ticks; 435 if (now - sc->ev_stats_update_time < hz) 436 goto out; 437 438 sc->ev_stats_update_time = now; 439 440 /* Add event counts from each event queue in turn */ 441 for (index = 0; index < sc->evq_count; index++) { 442 evq = sc->evq[index]; 443 SFXGE_EVQ_LOCK(evq); 444 efx_ev_qstats_update(evq->common, sc->ev_stats); 445 SFXGE_EVQ_UNLOCK(evq); 446 } 447out: 448 SFXGE_ADAPTER_UNLOCK(sc); 449} 450 451static int 452sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 453{ 454 struct sfxge_softc *sc = arg1; 455 unsigned int id = arg2; 456 457 sfxge_ev_stat_update(sc); 458 459 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 460} 461 462static void 463sfxge_ev_stat_init(struct sfxge_softc *sc) 464{ 465 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 466 struct sysctl_oid_list *stat_list; 467 unsigned int id; 468 char name[40]; 469 470 stat_list = SYSCTL_CHILDREN(sc->stats_node); 471 472 for (id = 0; id < EV_NQSTATS; id++) { 473 snprintf(name, sizeof(name), "ev_%s", 474 efx_ev_qstat_name(sc->enp, id)); 475 SYSCTL_ADD_PROC( 476 ctx, stat_list, 477 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 478 sc, id, sfxge_ev_stat_handler, "Q", 479 ""); 480 } 481} 482 483#endif /* EFSYS_OPT_QSTATS */ 484 485static void 486sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 487{ 488 struct sfxge_evq *evq; 489 efx_evq_t *eep; 490 491 evq = sc->evq[idx]; 492 eep = evq->common; 493 494 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 495 ("evq->init_state != SFXGE_EVQ_STARTED")); 496 497 (void)efx_ev_qmoderate(eep, us); 498} 499 500static int 501sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 502{ 503 struct sfxge_softc *sc = arg1; 504 struct sfxge_intr *intr = &sc->intr; 505 unsigned int moderation; 506 int error; 507 unsigned int index; 508 509 SFXGE_ADAPTER_LOCK(sc); 510 511 if (req->newptr != NULL) { 512 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 513 != 0) 514 goto out; 515 516 /* We may not be calling efx_ev_qmoderate() now, 517 * so we have to range-check the value ourselves. 518 */ 519 if (moderation > 520 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 521 error = EINVAL; 522 goto out; 523 } 524 525 sc->ev_moderation = moderation; 526 if (intr->state == SFXGE_INTR_STARTED) { 527 for (index = 0; index < sc->evq_count; index++) 528 sfxge_ev_qmoderate(sc, index, moderation); 529 } 530 } else { 531 error = SYSCTL_OUT(req, &sc->ev_moderation, 532 sizeof(sc->ev_moderation)); 533 } 534 535out: 536 SFXGE_ADAPTER_UNLOCK(sc); 537 538 return (error); 539} 540 541static boolean_t 542sfxge_ev_initialized(void *arg) 543{ 544 struct sfxge_evq *evq; 545 546 evq = (struct sfxge_evq *)arg; 547 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 548 549 KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 550 ("evq not starting")); 551 552 evq->init_state = SFXGE_EVQ_STARTED; 553 554 return (0); 555} 556 557static boolean_t 558sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 559{ 560 struct sfxge_evq *evq; 561 struct sfxge_softc *sc; 562 563 evq = (struct sfxge_evq *)arg; 564 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 565 566 sc = evq->sc; 567 568 sfxge_mac_link_update(sc, link_mode); 569 570 return (0); 571} 572 573static const efx_ev_callbacks_t sfxge_ev_callbacks = { 574 .eec_initialized = sfxge_ev_initialized, 575 .eec_rx = sfxge_ev_rx, 576 .eec_tx = sfxge_ev_tx, 577 .eec_exception = sfxge_ev_exception, 578 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 579 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 580 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 581 .eec_software = sfxge_ev_software, 582 .eec_sram = sfxge_ev_sram, 583 .eec_wake_up = sfxge_ev_wake_up, 584 .eec_timer = sfxge_ev_timer, 585 .eec_link_change = sfxge_ev_link_change, 586}; 587 588 589int 590sfxge_ev_qpoll(struct sfxge_evq *evq) 591{ 592 int rc; 593 594 SFXGE_EVQ_LOCK(evq); 595 596 if (evq->init_state != SFXGE_EVQ_STARTING && 597 evq->init_state != SFXGE_EVQ_STARTED) { 598 rc = EINVAL; 599 goto fail; 600 } 601 602 /* Synchronize the DMA memory for reading */ 603 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 604 BUS_DMASYNC_POSTREAD); 605 606 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 607 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 608 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 609 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 610 611 /* Poll the queue */ 612 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 613 614 evq->rx_done = 0; 615 evq->tx_done = 0; 616 617 /* Perform any pending completion processing */ 618 sfxge_ev_qcomplete(evq, B_TRUE); 619 620 /* Re-prime the event queue for interrupts */ 621 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 622 goto fail; 623 624 SFXGE_EVQ_UNLOCK(evq); 625 626 return (0); 627 628fail: 629 SFXGE_EVQ_UNLOCK(evq); 630 return (rc); 631} 632 633static void 634sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 635{ 636 struct sfxge_evq *evq; 637 638 evq = sc->evq[index]; 639 640 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 641 ("evq->init_state != SFXGE_EVQ_STARTED")); 642 643 SFXGE_EVQ_LOCK(evq); 644 evq->init_state = SFXGE_EVQ_INITIALIZED; 645 evq->read_ptr = 0; 646 evq->exception = B_FALSE; 647 648#if EFSYS_OPT_QSTATS 649 /* Add event counts before discarding the common evq state */ 650 efx_ev_qstats_update(evq->common, sc->ev_stats); 651#endif 652 653 efx_ev_qdestroy(evq->common); 654 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 655 EFX_EVQ_NBUFS(evq->entries)); 656 SFXGE_EVQ_UNLOCK(evq); 657} 658 659static int 660sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 661{ 662 struct sfxge_evq *evq; 663 efsys_mem_t *esmp; 664 int count; 665 int rc; 666 667 evq = sc->evq[index]; 668 esmp = &evq->mem; 669 670 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 671 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 672 673 /* Clear all events. */ 674 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 675 676 /* Program the buffer table. */ 677 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 678 EFX_EVQ_NBUFS(evq->entries))) != 0) 679 return (rc); 680 681 /* Create the common code event queue. */ 682 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 683 evq->buf_base_id, &evq->common)) != 0) 684 goto fail; 685 686 SFXGE_EVQ_LOCK(evq); 687 688 /* Set the default moderation */ 689 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 690 691 /* Prime the event queue for interrupts */ 692 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 693 goto fail2; 694 695 evq->init_state = SFXGE_EVQ_STARTING; 696 697 SFXGE_EVQ_UNLOCK(evq); 698 699 /* Wait for the initialization event */ 700 count = 0; 701 do { 702 /* Pause for 100 ms */ 703 pause("sfxge evq init", hz / 10); 704 705 /* Check to see if the test event has been processed */ 706 if (evq->init_state == SFXGE_EVQ_STARTED) 707 goto done; 708 709 } while (++count < 20); 710 711 rc = ETIMEDOUT; 712 goto fail3; 713 714done: 715 return (0); 716 717fail3: 718 SFXGE_EVQ_LOCK(evq); 719 evq->init_state = SFXGE_EVQ_INITIALIZED; 720fail2: 721 SFXGE_EVQ_UNLOCK(evq); 722 efx_ev_qdestroy(evq->common); 723fail: 724 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 725 EFX_EVQ_NBUFS(evq->entries)); 726 727 return (rc); 728} 729 730void 731sfxge_ev_stop(struct sfxge_softc *sc) 732{ 733 struct sfxge_intr *intr; 734 efx_nic_t *enp; 735 int index; 736 737 intr = &sc->intr; 738 enp = sc->enp; 739 740 KASSERT(intr->state == SFXGE_INTR_STARTED, 741 ("Interrupts not started")); 742 743 /* Stop the event queue(s) */ 744 index = sc->evq_count; 745 while (--index >= 0) 746 sfxge_ev_qstop(sc, index); 747 748 /* Tear down the event module */ 749 efx_ev_fini(enp); 750} 751 752int 753sfxge_ev_start(struct sfxge_softc *sc) 754{ 755 struct sfxge_intr *intr; 756 int index; 757 int rc; 758 759 intr = &sc->intr; 760 761 KASSERT(intr->state == SFXGE_INTR_STARTED, 762 ("intr->state != SFXGE_INTR_STARTED")); 763 764 /* Initialize the event module */ 765 if ((rc = efx_ev_init(sc->enp)) != 0) 766 return (rc); 767 768 /* Start the event queues */ 769 for (index = 0; index < sc->evq_count; index++) { 770 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 771 goto fail; 772 } 773 774 return (0); 775 776fail: 777 /* Stop the event queue(s) */ 778 while (--index >= 0) 779 sfxge_ev_qstop(sc, index); 780 781 /* Tear down the event module */ 782 efx_ev_fini(sc->enp); 783 784 return (rc); 785} 786 787static void 788sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 789{ 790 struct sfxge_evq *evq; 791 792 evq = sc->evq[index]; 793 794 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 795 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 796 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 797 798 sfxge_dma_free(&evq->mem); 799 800 sc->evq[index] = NULL; 801 802 SFXGE_EVQ_LOCK_DESTROY(evq); 803 804 free(evq, M_SFXGE); 805} 806 807static int 808sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 809{ 810 struct sfxge_evq *evq; 811 efsys_mem_t *esmp; 812 int rc; 813 814 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 815 816 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 817 evq->sc = sc; 818 evq->index = index; 819 sc->evq[index] = evq; 820 esmp = &evq->mem; 821 822 /* Build an event queue with room for one event per tx and rx buffer, 823 * plus some extra for link state events and MCDI completions. 824 * There are three tx queues in the first event queue and one in 825 * other. 826 */ 827 if (index == 0) 828 evq->entries = 829 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 830 3 * sc->txq_entries + 831 128); 832 else 833 evq->entries = 834 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 835 sc->txq_entries + 836 128); 837 838 /* Initialise TX completion list */ 839 evq->txqs = &evq->txq; 840 841 /* Allocate DMA space. */ 842 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 843 return (rc); 844 845 /* Allocate buffer table entries. */ 846 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 847 &evq->buf_base_id); 848 849 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 850 851 evq->init_state = SFXGE_EVQ_INITIALIZED; 852 853 return (0); 854} 855 856void 857sfxge_ev_fini(struct sfxge_softc *sc) 858{ 859 struct sfxge_intr *intr; 860 int index; 861 862 intr = &sc->intr; 863 864 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 865 ("intr->state != SFXGE_INTR_INITIALIZED")); 866 867 sc->ev_moderation = 0; 868 869 /* Tear down the event queue(s). */ 870 index = sc->evq_count; 871 while (--index >= 0) 872 sfxge_ev_qfini(sc, index); 873 874 sc->evq_count = 0; 875} 876 877int 878sfxge_ev_init(struct sfxge_softc *sc) 879{ 880 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 881 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 882 struct sfxge_intr *intr; 883 int index; 884 int rc; 885 886 intr = &sc->intr; 887 888 sc->evq_count = intr->n_alloc; 889 890 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 891 ("intr->state != SFXGE_INTR_INITIALIZED")); 892 893 /* Set default interrupt moderation; add a sysctl to 894 * read and change it. 895 */ 896 sc->ev_moderation = SFXGE_MODERATION; 897 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 898 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 899 sc, 0, sfxge_int_mod_handler, "IU", 900 "sfxge interrupt moderation (us)"); 901 902 /* 903 * Initialize the event queue(s) - one per interrupt. 904 */ 905 for (index = 0; index < sc->evq_count; index++) { 906 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 907 goto fail; 908 } 909 910#if EFSYS_OPT_QSTATS 911 sfxge_ev_stat_init(sc); 912#endif 913 914 return (0); 915 916fail: 917 while (--index >= 0) 918 sfxge_ev_qfini(sc, index); 919 920 sc->evq_count = 0; 921 return (rc); 922} 923