1/*- 2 * Copyright (c) 2010-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: stable/11/sys/dev/sfxge/sfxge_ev.c 342455 2018-12-25 07:39:34Z arybchik $"); 36 37#include <sys/param.h> 38#include <sys/kernel.h> 39#include <sys/malloc.h> 40#include <sys/param.h> 41#include <sys/queue.h> 42#include <sys/systm.h> 43#include <sys/taskqueue.h> 44 45#include "common/efx.h" 46 47#include "sfxge.h" 48 49static void 50sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 51{ 52 struct sfxge_softc *sc; 53 unsigned int index; 54 struct sfxge_rxq *rxq; 55 struct sfxge_txq *txq; 56 57 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 58 59 sc = evq->sc; 60 index = evq->index; 61 rxq = sc->rxq[index]; 62 63 if ((txq = evq->txq) != NULL) { 64 evq->txq = NULL; 65 evq->txqs = &(evq->txq); 66 67 do { 68 struct sfxge_txq *next; 69 70 next = txq->next; 71 txq->next = NULL; 72 73 KASSERT(txq->evq_index == index, 74 ("txq->evq_index != index")); 75 76 if (txq->pending != txq->completed) 77 sfxge_tx_qcomplete(txq, evq); 78 79 txq = next; 80 } while (txq != NULL); 81 } 82 83 if (rxq->pending != rxq->completed) 84 sfxge_rx_qcomplete(rxq, eop); 85} 86 87static struct sfxge_rxq * 88sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 89{ 90 struct sfxge_rxq *rxq; 91 92 KASSERT(label == 0, ("unexpected rxq label != 0")); 93 94 rxq = evq->sc->rxq[evq->index]; 95 96 KASSERT(rxq != NULL, ("rxq == NULL")); 97 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 98 99 return (rxq); 100} 101 102static boolean_t 103sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 104 uint16_t flags) 105{ 106 struct sfxge_evq *evq; 107 struct sfxge_softc *sc; 108 struct sfxge_rxq *rxq; 109 unsigned int stop; 110 unsigned int delta; 111 struct sfxge_rx_sw_desc *rx_desc; 112 113 evq = arg; 114 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 115 116 sc = evq->sc; 117 118 if (evq->exception) 119 goto done; 120 121 rxq = sfxge_get_rxq_by_label(evq, label); 122 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 123 goto done; 124 125 stop = (id + 1) & rxq->ptr_mask; 126 id = rxq->pending & rxq->ptr_mask; 127 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 128 rxq->pending += delta; 129 130 if (delta != 1) { 131 if ((delta <= 0) || 132 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 133 evq->exception = B_TRUE; 134 135 device_printf(sc->dev, "RX completion out of order" 136 " (id=%#x delta=%u flags=%#x); resetting\n", 137 id, delta, flags); 138 sfxge_schedule_reset(sc); 139 140 goto done; 141 } 142 } 143 144 rx_desc = &rxq->queue[id]; 145 146 prefetch_read_many(rx_desc->mbuf); 147 148 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 149 rx_desc = &rxq->queue[id]; 150 KASSERT(rx_desc->flags == EFX_DISCARD, 151 ("rx_desc->flags != EFX_DISCARD")); 152 rx_desc->flags = flags; 153 154 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 155 rx_desc->size = (uint16_t)size; 156 } 157 158 evq->rx_done++; 159 160 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 161 sfxge_ev_qcomplete(evq, B_FALSE); 162 163done: 164 return (evq->rx_done >= SFXGE_EV_BATCH); 165} 166 167static boolean_t 168sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 169{ 170 struct sfxge_evq *evq; 171 struct sfxge_softc *sc; 172 173 evq = (struct sfxge_evq *)arg; 174 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 175 176 sc = evq->sc; 177 178 DBGPRINT(sc->dev, "[%d] %s", evq->index, 179 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 180 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 181 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 182 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 183 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 184 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 185 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 186 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 187 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 188 "UNKNOWN"); 189 190 evq->exception = B_TRUE; 191 192 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 193 device_printf(sc->dev, 194 "hardware exception (code=%u); resetting\n", 195 code); 196 sfxge_schedule_reset(sc); 197 } 198 199 return (B_FALSE); 200} 201 202static boolean_t 203sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 204{ 205 struct sfxge_evq *evq; 206 struct sfxge_softc *sc; 207 struct sfxge_rxq *rxq; 208 unsigned int index; 209 uint16_t magic; 210 211 evq = (struct sfxge_evq *)arg; 212 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 213 214 sc = evq->sc; 215 rxq = sc->rxq[rxq_index]; 216 217 KASSERT(rxq != NULL, ("rxq == NULL")); 218 219 /* Resend a software event on the correct queue */ 220 index = rxq->index; 221 if (index == evq->index) { 222 sfxge_rx_qflush_done(rxq); 223 return (B_FALSE); 224 } 225 226 evq = sc->evq[index]; 227 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 228 229 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 230 ("evq not started")); 231 efx_ev_qpost(evq->common, magic); 232 233 return (B_FALSE); 234} 235 236static boolean_t 237sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 238{ 239 struct sfxge_evq *evq; 240 struct sfxge_softc *sc; 241 struct sfxge_rxq *rxq; 242 unsigned int index; 243 uint16_t magic; 244 245 evq = (struct sfxge_evq *)arg; 246 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 247 248 sc = evq->sc; 249 rxq = sc->rxq[rxq_index]; 250 251 KASSERT(rxq != NULL, ("rxq == NULL")); 252 253 /* Resend a software event on the correct queue */ 254 index = rxq->index; 255 evq = sc->evq[index]; 256 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 257 258 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 259 ("evq not started")); 260 efx_ev_qpost(evq->common, magic); 261 262 return (B_FALSE); 263} 264 265static struct sfxge_txq * 266sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 267{ 268 unsigned int index; 269 270 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 271 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 272 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 273 ("unexpected txq label")); 274 275 index = (evq->index == 0) ? 276 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 277 return (evq->sc->txq[index]); 278} 279 280static boolean_t 281sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 282{ 283 struct sfxge_evq *evq; 284 struct sfxge_txq *txq; 285 unsigned int stop; 286 unsigned int delta; 287 288 evq = (struct sfxge_evq *)arg; 289 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 290 291 txq = sfxge_get_txq_by_label(evq, label); 292 293 KASSERT(txq != NULL, ("txq == NULL")); 294 KASSERT(evq->index == txq->evq_index, 295 ("evq->index != txq->evq_index")); 296 297 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 298 goto done; 299 300 stop = (id + 1) & txq->ptr_mask; 301 id = txq->pending & txq->ptr_mask; 302 303 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 304 txq->pending += delta; 305 306 evq->tx_done++; 307 308 if (txq->next == NULL && 309 evq->txqs != &(txq->next)) { 310 *(evq->txqs) = txq; 311 evq->txqs = &(txq->next); 312 } 313 314 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 315 sfxge_tx_qcomplete(txq, evq); 316 317done: 318 return (evq->tx_done >= SFXGE_EV_BATCH); 319} 320 321static boolean_t 322sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 323{ 324 struct sfxge_evq *evq; 325 struct sfxge_softc *sc; 326 struct sfxge_txq *txq; 327 uint16_t magic; 328 329 evq = (struct sfxge_evq *)arg; 330 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 331 332 sc = evq->sc; 333 txq = sc->txq[txq_index]; 334 335 KASSERT(txq != NULL, ("txq == NULL")); 336 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 337 ("txq not initialized")); 338 339 if (txq->evq_index == evq->index) { 340 sfxge_tx_qflush_done(txq); 341 return (B_FALSE); 342 } 343 344 /* Resend a software event on the correct queue */ 345 evq = sc->evq[txq->evq_index]; 346 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 347 348 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 349 ("evq not started")); 350 efx_ev_qpost(evq->common, magic); 351 352 return (B_FALSE); 353} 354 355static boolean_t 356sfxge_ev_software(void *arg, uint16_t magic) 357{ 358 struct sfxge_evq *evq; 359 struct sfxge_softc *sc; 360 unsigned int label; 361 362 evq = (struct sfxge_evq *)arg; 363 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 364 365 sc = evq->sc; 366 367 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 368 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 369 370 switch (magic) { 371 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 372 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 373 break; 374 375 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 376 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 377 break; 378 379 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 380 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 381 break; 382 383 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 384 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 385 386 KASSERT(txq != NULL, ("txq == NULL")); 387 KASSERT(evq->index == txq->evq_index, 388 ("evq->index != txq->evq_index")); 389 390 sfxge_tx_qflush_done(txq); 391 break; 392 } 393 default: 394 break; 395 } 396 397 return (B_FALSE); 398} 399 400static boolean_t 401sfxge_ev_sram(void *arg, uint32_t code) 402{ 403 (void)arg; 404 (void)code; 405 406 switch (code) { 407 case EFX_SRAM_UPDATE: 408 EFSYS_PROBE(sram_update); 409 break; 410 411 case EFX_SRAM_CLEAR: 412 EFSYS_PROBE(sram_clear); 413 break; 414 415 case EFX_SRAM_ILLEGAL_CLEAR: 416 EFSYS_PROBE(sram_illegal_clear); 417 break; 418 419 default: 420 KASSERT(B_FALSE, ("Impossible SRAM event")); 421 break; 422 } 423 424 return (B_FALSE); 425} 426 427static boolean_t 428sfxge_ev_timer(void *arg, uint32_t index) 429{ 430 (void)arg; 431 (void)index; 432 433 return (B_FALSE); 434} 435 436static boolean_t 437sfxge_ev_wake_up(void *arg, uint32_t index) 438{ 439 (void)arg; 440 (void)index; 441 442 return (B_FALSE); 443} 444 445#if EFSYS_OPT_QSTATS 446 447static void 448sfxge_evq_stat_update(struct sfxge_evq *evq) 449{ 450 clock_t now; 451 452 SFXGE_EVQ_LOCK(evq); 453 454 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 455 goto out; 456 457 now = ticks; 458 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 459 goto out; 460 461 evq->stats_update_time = now; 462 efx_ev_qstats_update(evq->common, evq->stats); 463 464out: 465 SFXGE_EVQ_UNLOCK(evq); 466} 467 468static int 469sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 470{ 471 struct sfxge_evq *evq = arg1; 472 struct sfxge_softc *sc = evq->sc; 473 unsigned int id = arg2; 474 475 SFXGE_ADAPTER_LOCK(sc); 476 477 sfxge_evq_stat_update(evq); 478 479 SFXGE_ADAPTER_UNLOCK(sc); 480 481 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 482} 483 484static int 485sfxge_evq_stat_init(struct sfxge_evq *evq) 486{ 487 struct sfxge_softc *sc = evq->sc; 488 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 489 char name[16]; 490 struct sysctl_oid *evq_stats_node; 491 unsigned int id; 492 493 snprintf(name, sizeof(name), "%u", evq->index); 494 evq_stats_node = SYSCTL_ADD_NODE(ctx, 495 SYSCTL_CHILDREN(sc->evqs_stats_node), 496 OID_AUTO, name, CTLFLAG_RD, NULL, ""); 497 if (evq_stats_node == NULL) 498 return (ENOMEM); 499 500 for (id = 0; id < EV_NQSTATS; id++) { 501 SYSCTL_ADD_PROC( 502 ctx, SYSCTL_CHILDREN(evq_stats_node), 503 OID_AUTO, efx_ev_qstat_name(sc->enp, id), 504 CTLTYPE_U64|CTLFLAG_RD, 505 evq, id, sfxge_evq_stat_handler, "Q", 506 ""); 507 } 508 509 return (0); 510} 511 512static void 513sfxge_ev_stat_update(struct sfxge_softc *sc) 514{ 515 struct sfxge_evq *evq; 516 unsigned int index; 517 clock_t now; 518 unsigned int id; 519 520 SFXGE_ADAPTER_LOCK(sc); 521 522 now = ticks; 523 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 524 goto out; 525 526 sc->ev_stats_update_time = now; 527 528 memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 529 530 /* Update and add event counts from each event queue in turn */ 531 for (index = 0; index < sc->evq_count; index++) { 532 evq = sc->evq[index]; 533 sfxge_evq_stat_update(evq); 534 for (id = 0; id < EV_NQSTATS; id++) 535 sc->ev_stats[id] += evq->stats[id]; 536 } 537out: 538 SFXGE_ADAPTER_UNLOCK(sc); 539} 540 541static int 542sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 543{ 544 struct sfxge_softc *sc = arg1; 545 unsigned int id = arg2; 546 547 sfxge_ev_stat_update(sc); 548 549 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 550} 551 552static void 553sfxge_ev_stat_init(struct sfxge_softc *sc) 554{ 555 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 556 struct sysctl_oid_list *stat_list; 557 unsigned int id; 558 char name[40]; 559 560 stat_list = SYSCTL_CHILDREN(sc->stats_node); 561 562 for (id = 0; id < EV_NQSTATS; id++) { 563 snprintf(name, sizeof(name), "ev_%s", 564 efx_ev_qstat_name(sc->enp, id)); 565 SYSCTL_ADD_PROC( 566 ctx, stat_list, 567 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 568 sc, id, sfxge_ev_stat_handler, "Q", 569 ""); 570 } 571} 572 573#endif /* EFSYS_OPT_QSTATS */ 574 575static void 576sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 577{ 578 struct sfxge_evq *evq; 579 efx_evq_t *eep; 580 581 evq = sc->evq[idx]; 582 eep = evq->common; 583 584 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 585 ("evq->init_state != SFXGE_EVQ_STARTED")); 586 587 (void)efx_ev_qmoderate(eep, us); 588} 589 590static int 591sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 592{ 593 struct sfxge_softc *sc = arg1; 594 struct sfxge_intr *intr = &sc->intr; 595 unsigned int moderation; 596 int error; 597 unsigned int index; 598 599 SFXGE_ADAPTER_LOCK(sc); 600 601 if (req->newptr != NULL) { 602 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 603 != 0) 604 goto out; 605 606 /* We may not be calling efx_ev_qmoderate() now, 607 * so we have to range-check the value ourselves. 608 */ 609 if (moderation > 610 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 611 error = EINVAL; 612 goto out; 613 } 614 615 sc->ev_moderation = moderation; 616 if (intr->state == SFXGE_INTR_STARTED) { 617 for (index = 0; index < sc->evq_count; index++) 618 sfxge_ev_qmoderate(sc, index, moderation); 619 } 620 } else { 621 error = SYSCTL_OUT(req, &sc->ev_moderation, 622 sizeof(sc->ev_moderation)); 623 } 624 625out: 626 SFXGE_ADAPTER_UNLOCK(sc); 627 628 return (error); 629} 630 631static boolean_t 632sfxge_ev_initialized(void *arg) 633{ 634 struct sfxge_evq *evq; 635 636 evq = (struct sfxge_evq *)arg; 637 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 638 639 /* Init done events may be duplicated on 7xxx */ 640 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 641 evq->init_state == SFXGE_EVQ_STARTED, 642 ("evq not starting")); 643 644 evq->init_state = SFXGE_EVQ_STARTED; 645 646 return (0); 647} 648 649static boolean_t 650sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 651{ 652 struct sfxge_evq *evq; 653 struct sfxge_softc *sc; 654 655 evq = (struct sfxge_evq *)arg; 656 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 657 658 sc = evq->sc; 659 660 sfxge_mac_link_update(sc, link_mode); 661 662 return (0); 663} 664 665static const efx_ev_callbacks_t sfxge_ev_callbacks = { 666 .eec_initialized = sfxge_ev_initialized, 667 .eec_rx = sfxge_ev_rx, 668 .eec_tx = sfxge_ev_tx, 669 .eec_exception = sfxge_ev_exception, 670 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 671 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 672 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 673 .eec_software = sfxge_ev_software, 674 .eec_sram = sfxge_ev_sram, 675 .eec_wake_up = sfxge_ev_wake_up, 676 .eec_timer = sfxge_ev_timer, 677 .eec_link_change = sfxge_ev_link_change, 678}; 679 680 681int 682sfxge_ev_qpoll(struct sfxge_evq *evq) 683{ 684 int rc; 685 686 SFXGE_EVQ_LOCK(evq); 687 688 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 689 evq->init_state != SFXGE_EVQ_STARTED)) { 690 rc = EINVAL; 691 goto fail; 692 } 693 694 /* Synchronize the DMA memory for reading */ 695 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 696 BUS_DMASYNC_POSTREAD); 697 698 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 699 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 700 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 701 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 702 703 /* Poll the queue */ 704 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 705 706 evq->rx_done = 0; 707 evq->tx_done = 0; 708 709 /* Perform any pending completion processing */ 710 sfxge_ev_qcomplete(evq, B_TRUE); 711 712 /* Re-prime the event queue for interrupts */ 713 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 714 goto fail; 715 716 SFXGE_EVQ_UNLOCK(evq); 717 718 return (0); 719 720fail: 721 SFXGE_EVQ_UNLOCK(evq); 722 return (rc); 723} 724 725static void 726sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 727{ 728 struct sfxge_evq *evq; 729 730 evq = sc->evq[index]; 731 732 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 733 ("evq->init_state != SFXGE_EVQ_STARTED")); 734 735 SFXGE_EVQ_LOCK(evq); 736 evq->init_state = SFXGE_EVQ_INITIALIZED; 737 evq->read_ptr = 0; 738 evq->exception = B_FALSE; 739 740#if EFSYS_OPT_QSTATS 741 /* Add event counts before discarding the common evq state */ 742 efx_ev_qstats_update(evq->common, evq->stats); 743#endif 744 745 efx_ev_qdestroy(evq->common); 746 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 747 EFX_EVQ_NBUFS(evq->entries)); 748 SFXGE_EVQ_UNLOCK(evq); 749} 750 751static int 752sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 753{ 754 struct sfxge_evq *evq; 755 efsys_mem_t *esmp; 756 int count; 757 int rc; 758 759 evq = sc->evq[index]; 760 esmp = &evq->mem; 761 762 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 763 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 764 765 /* Clear all events. */ 766 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 767 768 /* Program the buffer table. */ 769 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 770 EFX_EVQ_NBUFS(evq->entries))) != 0) 771 return (rc); 772 773 /* Create the common code event queue. */ 774 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 775 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 776 &evq->common)) != 0) 777 goto fail; 778 779 SFXGE_EVQ_LOCK(evq); 780 781 /* Prime the event queue for interrupts */ 782 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 783 goto fail2; 784 785 evq->init_state = SFXGE_EVQ_STARTING; 786 787 SFXGE_EVQ_UNLOCK(evq); 788 789 /* Wait for the initialization event */ 790 count = 0; 791 do { 792 /* Pause for 100 ms */ 793 pause("sfxge evq init", hz / 10); 794 795 /* Check to see if the test event has been processed */ 796 if (evq->init_state == SFXGE_EVQ_STARTED) 797 goto done; 798 799 } while (++count < 20); 800 801 rc = ETIMEDOUT; 802 goto fail3; 803 804done: 805 return (0); 806 807fail3: 808 SFXGE_EVQ_LOCK(evq); 809 evq->init_state = SFXGE_EVQ_INITIALIZED; 810fail2: 811 SFXGE_EVQ_UNLOCK(evq); 812 efx_ev_qdestroy(evq->common); 813fail: 814 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 815 EFX_EVQ_NBUFS(evq->entries)); 816 817 return (rc); 818} 819 820void 821sfxge_ev_stop(struct sfxge_softc *sc) 822{ 823 struct sfxge_intr *intr; 824 efx_nic_t *enp; 825 int index; 826 827 intr = &sc->intr; 828 enp = sc->enp; 829 830 KASSERT(intr->state == SFXGE_INTR_STARTED, 831 ("Interrupts not started")); 832 833 /* Stop the event queue(s) */ 834 index = sc->evq_count; 835 while (--index >= 0) 836 sfxge_ev_qstop(sc, index); 837 838 /* Tear down the event module */ 839 efx_ev_fini(enp); 840} 841 842int 843sfxge_ev_start(struct sfxge_softc *sc) 844{ 845 struct sfxge_intr *intr; 846 int index; 847 int rc; 848 849 intr = &sc->intr; 850 851 KASSERT(intr->state == SFXGE_INTR_STARTED, 852 ("intr->state != SFXGE_INTR_STARTED")); 853 854 /* Initialize the event module */ 855 if ((rc = efx_ev_init(sc->enp)) != 0) 856 return (rc); 857 858 /* Start the event queues */ 859 for (index = 0; index < sc->evq_count; index++) { 860 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 861 goto fail; 862 } 863 864 return (0); 865 866fail: 867 /* Stop the event queue(s) */ 868 while (--index >= 0) 869 sfxge_ev_qstop(sc, index); 870 871 /* Tear down the event module */ 872 efx_ev_fini(sc->enp); 873 874 return (rc); 875} 876 877static void 878sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 879{ 880 struct sfxge_evq *evq; 881 882 evq = sc->evq[index]; 883 884 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 885 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 886 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 887 888 sfxge_dma_free(&evq->mem); 889 890 sc->evq[index] = NULL; 891 892 SFXGE_EVQ_LOCK_DESTROY(evq); 893 894 free(evq, M_SFXGE); 895} 896 897static int 898sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 899{ 900 struct sfxge_evq *evq; 901 efsys_mem_t *esmp; 902 int rc; 903 904 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 905 906 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 907 evq->sc = sc; 908 evq->index = index; 909 sc->evq[index] = evq; 910 esmp = &evq->mem; 911 912 /* Build an event queue with room for one event per tx and rx buffer, 913 * plus some extra for link state events and MCDI completions. 914 * There are three tx queues in the first event queue and one in 915 * other. 916 */ 917 if (index == 0) 918 evq->entries = 919 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 920 3 * sc->txq_entries + 921 128); 922 else 923 evq->entries = 924 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 925 sc->txq_entries + 926 128); 927 928 /* Initialise TX completion list */ 929 evq->txqs = &evq->txq; 930 931 /* Allocate DMA space. */ 932 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 933 return (rc); 934 935 /* Allocate buffer table entries. */ 936 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 937 &evq->buf_base_id); 938 939 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 940 941 evq->init_state = SFXGE_EVQ_INITIALIZED; 942 943#if EFSYS_OPT_QSTATS 944 rc = sfxge_evq_stat_init(evq); 945 if (rc != 0) 946 goto fail_evq_stat_init; 947#endif 948 949 return (0); 950 951#if EFSYS_OPT_QSTATS 952fail_evq_stat_init: 953 evq->init_state = SFXGE_EVQ_UNINITIALIZED; 954 SFXGE_EVQ_LOCK_DESTROY(evq); 955 sfxge_dma_free(esmp); 956 sc->evq[index] = NULL; 957 free(evq, M_SFXGE); 958 959 return (rc); 960#endif 961} 962 963void 964sfxge_ev_fini(struct sfxge_softc *sc) 965{ 966 struct sfxge_intr *intr; 967 int index; 968 969 intr = &sc->intr; 970 971 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 972 ("intr->state != SFXGE_INTR_INITIALIZED")); 973 974 sc->ev_moderation = 0; 975 976 /* Tear down the event queue(s). */ 977 index = sc->evq_count; 978 while (--index >= 0) 979 sfxge_ev_qfini(sc, index); 980 981 sc->evq_count = 0; 982} 983 984int 985sfxge_ev_init(struct sfxge_softc *sc) 986{ 987 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 988 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 989 struct sfxge_intr *intr; 990 int index; 991 int rc; 992 993 intr = &sc->intr; 994 995 sc->evq_count = intr->n_alloc; 996 997 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 998 ("intr->state != SFXGE_INTR_INITIALIZED")); 999 1000 /* Set default interrupt moderation; add a sysctl to 1001 * read and change it. 1002 */ 1003 sc->ev_moderation = SFXGE_MODERATION; 1004 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1005 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 1006 sc, 0, sfxge_int_mod_handler, "IU", 1007 "sfxge interrupt moderation (us)"); 1008 1009#if EFSYS_OPT_QSTATS 1010 sc->evqs_stats_node = SYSCTL_ADD_NODE( 1011 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1012 OID_AUTO, "evq", CTLFLAG_RD, NULL, "Event queues stats"); 1013 if (sc->evqs_stats_node == NULL) { 1014 rc = ENOMEM; 1015 goto fail_evqs_stats_node; 1016 } 1017#endif 1018 1019 /* 1020 * Initialize the event queue(s) - one per interrupt. 1021 */ 1022 for (index = 0; index < sc->evq_count; index++) { 1023 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1024 goto fail; 1025 } 1026 1027#if EFSYS_OPT_QSTATS 1028 sfxge_ev_stat_init(sc); 1029#endif 1030 1031 return (0); 1032 1033fail: 1034 while (--index >= 0) 1035 sfxge_ev_qfini(sc, index); 1036 1037#if EFSYS_OPT_QSTATS 1038fail_evqs_stats_node: 1039#endif 1040 sc->evq_count = 0; 1041 return (rc); 1042} 1043