1227569Sphilip/*- 2300607Sarybchik * Copyright (c) 2010-2016 Solarflare Communications Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9283514Sarybchik * modification, are permitted provided that the following conditions are met: 10227569Sphilip * 11283514Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 12283514Sarybchik * this list of conditions and the following disclaimer. 13283514Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 14283514Sarybchik * this list of conditions and the following disclaimer in the documentation 15283514Sarybchik * and/or other materials provided with the distribution. 16283514Sarybchik * 17283514Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18283514Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19283514Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20283514Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21283514Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22283514Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23283514Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24283514Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25283514Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26283514Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27283514Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28283514Sarybchik * 29283514Sarybchik * The views and conclusions contained in the software and documentation are 30283514Sarybchik * those of the authors and should not be interpreted as representing official 31283514Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 32227569Sphilip */ 33227569Sphilip 34227569Sphilip#include <sys/cdefs.h> 35227569Sphilip__FBSDID("$FreeBSD: stable/11/sys/dev/sfxge/sfxge_ev.c 342455 2018-12-25 07:39:34Z arybchik $"); 36227569Sphilip 37227569Sphilip#include <sys/param.h> 38257179Sglebius#include <sys/kernel.h> 39257179Sglebius#include <sys/malloc.h> 40257179Sglebius#include <sys/param.h> 41257179Sglebius#include <sys/queue.h> 42227569Sphilip#include <sys/systm.h> 43257179Sglebius#include <sys/taskqueue.h> 44227569Sphilip 45227569Sphilip#include "common/efx.h" 46227569Sphilip 47227569Sphilip#include "sfxge.h" 48227569Sphilip 49227569Sphilipstatic void 50227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 51227569Sphilip{ 52227569Sphilip struct sfxge_softc *sc; 53227569Sphilip unsigned int index; 54227569Sphilip struct sfxge_rxq *rxq; 55227569Sphilip struct sfxge_txq *txq; 56227569Sphilip 57279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 58279177Sarybchik 59227569Sphilip sc = evq->sc; 60227569Sphilip index = evq->index; 61227569Sphilip rxq = sc->rxq[index]; 62227569Sphilip 63227569Sphilip if ((txq = evq->txq) != NULL) { 64227569Sphilip evq->txq = NULL; 65227569Sphilip evq->txqs = &(evq->txq); 66227569Sphilip 67227569Sphilip do { 68227569Sphilip struct sfxge_txq *next; 69227569Sphilip 70227569Sphilip next = txq->next; 71227569Sphilip txq->next = NULL; 72227569Sphilip 73227569Sphilip KASSERT(txq->evq_index == index, 74227569Sphilip ("txq->evq_index != index")); 75227569Sphilip 76227569Sphilip if (txq->pending != txq->completed) 77277889Sarybchik sfxge_tx_qcomplete(txq, evq); 78227569Sphilip 79227569Sphilip txq = next; 80227569Sphilip } while (txq != NULL); 81227569Sphilip } 82227569Sphilip 83227569Sphilip if (rxq->pending != rxq->completed) 84227569Sphilip sfxge_rx_qcomplete(rxq, eop); 85227569Sphilip} 86227569Sphilip 87298735Sarybchikstatic struct sfxge_rxq * 88298735Sarybchiksfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 89298735Sarybchik{ 90298735Sarybchik struct sfxge_rxq *rxq; 91298735Sarybchik 92298735Sarybchik KASSERT(label == 0, ("unexpected rxq label != 0")); 93298735Sarybchik 94298735Sarybchik rxq = evq->sc->rxq[evq->index]; 95298735Sarybchik 96298735Sarybchik KASSERT(rxq != NULL, ("rxq == NULL")); 97298735Sarybchik KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 98298735Sarybchik 99298735Sarybchik return (rxq); 100298735Sarybchik} 101298735Sarybchik 102227569Sphilipstatic boolean_t 103227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 104283514Sarybchik uint16_t flags) 105227569Sphilip{ 106227569Sphilip struct sfxge_evq *evq; 107227569Sphilip struct sfxge_softc *sc; 108227569Sphilip struct sfxge_rxq *rxq; 109283514Sarybchik unsigned int stop; 110283514Sarybchik unsigned int delta; 111227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 112227569Sphilip 113227569Sphilip evq = arg; 114279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 115279177Sarybchik 116227569Sphilip sc = evq->sc; 117227569Sphilip 118227569Sphilip if (evq->exception) 119227569Sphilip goto done; 120227569Sphilip 121298735Sarybchik rxq = sfxge_get_rxq_by_label(evq, label); 122279351Sarybchik if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 123227569Sphilip goto done; 124227569Sphilip 125283514Sarybchik stop = (id + 1) & rxq->ptr_mask; 126283514Sarybchik id = rxq->pending & rxq->ptr_mask; 127283514Sarybchik delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 128283514Sarybchik rxq->pending += delta; 129227569Sphilip 130283514Sarybchik if (delta != 1) { 131301309Sarybchik if ((delta <= 0) || 132283514Sarybchik (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 133283514Sarybchik evq->exception = B_TRUE; 134227569Sphilip 135283514Sarybchik device_printf(sc->dev, "RX completion out of order" 136283514Sarybchik " (id=%#x delta=%u flags=%#x); resetting\n", 137283514Sarybchik id, delta, flags); 138283514Sarybchik sfxge_schedule_reset(sc); 139283514Sarybchik 140283514Sarybchik goto done; 141283514Sarybchik } 142227569Sphilip } 143227569Sphilip 144227569Sphilip rx_desc = &rxq->queue[id]; 145227569Sphilip 146227569Sphilip prefetch_read_many(rx_desc->mbuf); 147227569Sphilip 148283514Sarybchik for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 149283514Sarybchik rx_desc = &rxq->queue[id]; 150283514Sarybchik KASSERT(rx_desc->flags == EFX_DISCARD, 151283514Sarybchik ("rx_desc->flags != EFX_DISCARD")); 152283514Sarybchik rx_desc->flags = flags; 153283514Sarybchik 154283514Sarybchik KASSERT(size < (1 << 16), ("size > (1 << 16)")); 155283514Sarybchik rx_desc->size = (uint16_t)size; 156283514Sarybchik } 157283514Sarybchik 158227569Sphilip evq->rx_done++; 159227569Sphilip 160227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 161227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 162227569Sphilip 163227569Sphilipdone: 164227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 165227569Sphilip} 166227569Sphilip 167227569Sphilipstatic boolean_t 168227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 169227569Sphilip{ 170227569Sphilip struct sfxge_evq *evq; 171227569Sphilip struct sfxge_softc *sc; 172227569Sphilip 173227569Sphilip evq = (struct sfxge_evq *)arg; 174279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 175279177Sarybchik 176227569Sphilip sc = evq->sc; 177227569Sphilip 178283514Sarybchik DBGPRINT(sc->dev, "[%d] %s", evq->index, 179283514Sarybchik (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 180283514Sarybchik (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 181283514Sarybchik (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 182283514Sarybchik (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 183283514Sarybchik (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 184283514Sarybchik (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 185283514Sarybchik (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 186283514Sarybchik (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 187283514Sarybchik (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 188283514Sarybchik "UNKNOWN"); 189283514Sarybchik 190227569Sphilip evq->exception = B_TRUE; 191227569Sphilip 192227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 193227569Sphilip device_printf(sc->dev, 194227569Sphilip "hardware exception (code=%u); resetting\n", 195227569Sphilip code); 196227569Sphilip sfxge_schedule_reset(sc); 197227569Sphilip } 198227569Sphilip 199227569Sphilip return (B_FALSE); 200227569Sphilip} 201227569Sphilip 202227569Sphilipstatic boolean_t 203264461Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 204227569Sphilip{ 205227569Sphilip struct sfxge_evq *evq; 206227569Sphilip struct sfxge_softc *sc; 207227569Sphilip struct sfxge_rxq *rxq; 208227569Sphilip unsigned int index; 209227569Sphilip uint16_t magic; 210227569Sphilip 211227569Sphilip evq = (struct sfxge_evq *)arg; 212279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 213279177Sarybchik 214227569Sphilip sc = evq->sc; 215264461Sgnn rxq = sc->rxq[rxq_index]; 216227569Sphilip 217227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 218227569Sphilip 219227569Sphilip /* Resend a software event on the correct queue */ 220227569Sphilip index = rxq->index; 221283514Sarybchik if (index == evq->index) { 222283514Sarybchik sfxge_rx_qflush_done(rxq); 223283514Sarybchik return (B_FALSE); 224283514Sarybchik } 225283514Sarybchik 226227569Sphilip evq = sc->evq[index]; 227301105Sarybchik magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 228227569Sphilip 229227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 230227569Sphilip ("evq not started")); 231227569Sphilip efx_ev_qpost(evq->common, magic); 232227569Sphilip 233227569Sphilip return (B_FALSE); 234227569Sphilip} 235227569Sphilip 236227569Sphilipstatic boolean_t 237264461Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 238227569Sphilip{ 239227569Sphilip struct sfxge_evq *evq; 240227569Sphilip struct sfxge_softc *sc; 241227569Sphilip struct sfxge_rxq *rxq; 242227569Sphilip unsigned int index; 243227569Sphilip uint16_t magic; 244227569Sphilip 245227569Sphilip evq = (struct sfxge_evq *)arg; 246279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 247279177Sarybchik 248227569Sphilip sc = evq->sc; 249264461Sgnn rxq = sc->rxq[rxq_index]; 250227569Sphilip 251227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 252227569Sphilip 253227569Sphilip /* Resend a software event on the correct queue */ 254227569Sphilip index = rxq->index; 255227569Sphilip evq = sc->evq[index]; 256301105Sarybchik magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 257227569Sphilip 258227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 259227569Sphilip ("evq not started")); 260227569Sphilip efx_ev_qpost(evq->common, magic); 261227569Sphilip 262227569Sphilip return (B_FALSE); 263227569Sphilip} 264227569Sphilip 265264461Sgnnstatic struct sfxge_txq * 266264461Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 267264461Sgnn{ 268264461Sgnn unsigned int index; 269264461Sgnn 270342455Sarybchik KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 271342455Sarybchik ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 272342455Sarybchik (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 273342455Sarybchik ("unexpected txq label")); 274342455Sarybchik 275342454Sarybchik index = (evq->index == 0) ? 276342454Sarybchik label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 277272325Sgnn return (evq->sc->txq[index]); 278264461Sgnn} 279264461Sgnn 280227569Sphilipstatic boolean_t 281227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 282227569Sphilip{ 283227569Sphilip struct sfxge_evq *evq; 284227569Sphilip struct sfxge_txq *txq; 285227569Sphilip unsigned int stop; 286227569Sphilip unsigned int delta; 287227569Sphilip 288227569Sphilip evq = (struct sfxge_evq *)arg; 289279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 290279177Sarybchik 291264461Sgnn txq = sfxge_get_txq_by_label(evq, label); 292227569Sphilip 293227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 294227569Sphilip KASSERT(evq->index == txq->evq_index, 295227569Sphilip ("evq->index != txq->evq_index")); 296227569Sphilip 297279351Sarybchik if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 298227569Sphilip goto done; 299227569Sphilip 300272328Sgnn stop = (id + 1) & txq->ptr_mask; 301272328Sgnn id = txq->pending & txq->ptr_mask; 302227569Sphilip 303272328Sgnn delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 304227569Sphilip txq->pending += delta; 305227569Sphilip 306227569Sphilip evq->tx_done++; 307227569Sphilip 308227569Sphilip if (txq->next == NULL && 309227569Sphilip evq->txqs != &(txq->next)) { 310227569Sphilip *(evq->txqs) = txq; 311227569Sphilip evq->txqs = &(txq->next); 312227569Sphilip } 313227569Sphilip 314227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 315277889Sarybchik sfxge_tx_qcomplete(txq, evq); 316227569Sphilip 317227569Sphilipdone: 318227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 319227569Sphilip} 320227569Sphilip 321227569Sphilipstatic boolean_t 322264461Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 323227569Sphilip{ 324227569Sphilip struct sfxge_evq *evq; 325227569Sphilip struct sfxge_softc *sc; 326227569Sphilip struct sfxge_txq *txq; 327227569Sphilip uint16_t magic; 328227569Sphilip 329227569Sphilip evq = (struct sfxge_evq *)arg; 330279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 331279177Sarybchik 332227569Sphilip sc = evq->sc; 333264461Sgnn txq = sc->txq[txq_index]; 334227569Sphilip 335227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 336227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 337227569Sphilip ("txq not initialized")); 338227569Sphilip 339283514Sarybchik if (txq->evq_index == evq->index) { 340283514Sarybchik sfxge_tx_qflush_done(txq); 341283514Sarybchik return (B_FALSE); 342283514Sarybchik } 343283514Sarybchik 344227569Sphilip /* Resend a software event on the correct queue */ 345227569Sphilip evq = sc->evq[txq->evq_index]; 346301105Sarybchik magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 347227569Sphilip 348227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 349227569Sphilip ("evq not started")); 350227569Sphilip efx_ev_qpost(evq->common, magic); 351227569Sphilip 352227569Sphilip return (B_FALSE); 353227569Sphilip} 354227569Sphilip 355227569Sphilipstatic boolean_t 356227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 357227569Sphilip{ 358227569Sphilip struct sfxge_evq *evq; 359227569Sphilip struct sfxge_softc *sc; 360227569Sphilip unsigned int label; 361227569Sphilip 362227569Sphilip evq = (struct sfxge_evq *)arg; 363279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 364279177Sarybchik 365227569Sphilip sc = evq->sc; 366227569Sphilip 367227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 368227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 369227569Sphilip 370227569Sphilip switch (magic) { 371301075Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 372298735Sarybchik sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 373298735Sarybchik break; 374227569Sphilip 375301075Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 376298735Sarybchik sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 377227569Sphilip break; 378227569Sphilip 379301075Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 380298735Sarybchik sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 381227569Sphilip break; 382227569Sphilip 383301075Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 384264461Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 385227569Sphilip 386227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 387227569Sphilip KASSERT(evq->index == txq->evq_index, 388227569Sphilip ("evq->index != txq->evq_index")); 389227569Sphilip 390227569Sphilip sfxge_tx_qflush_done(txq); 391227569Sphilip break; 392227569Sphilip } 393227569Sphilip default: 394227569Sphilip break; 395227569Sphilip } 396227569Sphilip 397227569Sphilip return (B_FALSE); 398227569Sphilip} 399227569Sphilip 400227569Sphilipstatic boolean_t 401227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 402227569Sphilip{ 403227569Sphilip (void)arg; 404227569Sphilip (void)code; 405227569Sphilip 406227569Sphilip switch (code) { 407227569Sphilip case EFX_SRAM_UPDATE: 408227569Sphilip EFSYS_PROBE(sram_update); 409227569Sphilip break; 410227569Sphilip 411227569Sphilip case EFX_SRAM_CLEAR: 412227569Sphilip EFSYS_PROBE(sram_clear); 413227569Sphilip break; 414227569Sphilip 415227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 416227569Sphilip EFSYS_PROBE(sram_illegal_clear); 417227569Sphilip break; 418227569Sphilip 419227569Sphilip default: 420227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 421227569Sphilip break; 422227569Sphilip } 423227569Sphilip 424227569Sphilip return (B_FALSE); 425227569Sphilip} 426227569Sphilip 427227569Sphilipstatic boolean_t 428227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 429227569Sphilip{ 430227569Sphilip (void)arg; 431227569Sphilip (void)index; 432227569Sphilip 433227569Sphilip return (B_FALSE); 434227569Sphilip} 435227569Sphilip 436227569Sphilipstatic boolean_t 437227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 438227569Sphilip{ 439227569Sphilip (void)arg; 440227569Sphilip (void)index; 441227569Sphilip 442227569Sphilip return (B_FALSE); 443227569Sphilip} 444227569Sphilip 445277886Sarybchik#if EFSYS_OPT_QSTATS 446277886Sarybchik 447227569Sphilipstatic void 448342452Sarybchiksfxge_evq_stat_update(struct sfxge_evq *evq) 449342452Sarybchik{ 450342452Sarybchik clock_t now; 451342452Sarybchik 452342452Sarybchik SFXGE_EVQ_LOCK(evq); 453342452Sarybchik 454342452Sarybchik if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 455342452Sarybchik goto out; 456342452Sarybchik 457342452Sarybchik now = ticks; 458342452Sarybchik if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 459342452Sarybchik goto out; 460342452Sarybchik 461342452Sarybchik evq->stats_update_time = now; 462342452Sarybchik efx_ev_qstats_update(evq->common, evq->stats); 463342452Sarybchik 464342452Sarybchikout: 465342452Sarybchik SFXGE_EVQ_UNLOCK(evq); 466342452Sarybchik} 467342452Sarybchik 468342452Sarybchikstatic int 469342452Sarybchiksfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 470342452Sarybchik{ 471342452Sarybchik struct sfxge_evq *evq = arg1; 472342452Sarybchik struct sfxge_softc *sc = evq->sc; 473342452Sarybchik unsigned int id = arg2; 474342452Sarybchik 475342452Sarybchik SFXGE_ADAPTER_LOCK(sc); 476342452Sarybchik 477342452Sarybchik sfxge_evq_stat_update(evq); 478342452Sarybchik 479342452Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 480342452Sarybchik 481342452Sarybchik return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 482342452Sarybchik} 483342452Sarybchik 484342452Sarybchikstatic int 485342452Sarybchiksfxge_evq_stat_init(struct sfxge_evq *evq) 486342452Sarybchik{ 487342452Sarybchik struct sfxge_softc *sc = evq->sc; 488342452Sarybchik struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 489342452Sarybchik char name[16]; 490342452Sarybchik struct sysctl_oid *evq_stats_node; 491342452Sarybchik unsigned int id; 492342452Sarybchik 493342452Sarybchik snprintf(name, sizeof(name), "%u", evq->index); 494342452Sarybchik evq_stats_node = SYSCTL_ADD_NODE(ctx, 495342452Sarybchik SYSCTL_CHILDREN(sc->evqs_stats_node), 496342452Sarybchik OID_AUTO, name, CTLFLAG_RD, NULL, ""); 497342452Sarybchik if (evq_stats_node == NULL) 498342452Sarybchik return (ENOMEM); 499342452Sarybchik 500342452Sarybchik for (id = 0; id < EV_NQSTATS; id++) { 501342452Sarybchik SYSCTL_ADD_PROC( 502342452Sarybchik ctx, SYSCTL_CHILDREN(evq_stats_node), 503342452Sarybchik OID_AUTO, efx_ev_qstat_name(sc->enp, id), 504342452Sarybchik CTLTYPE_U64|CTLFLAG_RD, 505342452Sarybchik evq, id, sfxge_evq_stat_handler, "Q", 506342452Sarybchik ""); 507342452Sarybchik } 508342452Sarybchik 509342452Sarybchik return (0); 510342452Sarybchik} 511342452Sarybchik 512342452Sarybchikstatic void 513227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 514227569Sphilip{ 515227569Sphilip struct sfxge_evq *evq; 516227569Sphilip unsigned int index; 517227569Sphilip clock_t now; 518342452Sarybchik unsigned int id; 519227569Sphilip 520278221Sarybchik SFXGE_ADAPTER_LOCK(sc); 521227569Sphilip 522227569Sphilip now = ticks; 523301724Sarybchik if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 524227569Sphilip goto out; 525227569Sphilip 526227569Sphilip sc->ev_stats_update_time = now; 527227569Sphilip 528342452Sarybchik memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 529342452Sarybchik 530342452Sarybchik /* Update and add event counts from each event queue in turn */ 531278940Sarybchik for (index = 0; index < sc->evq_count; index++) { 532227569Sphilip evq = sc->evq[index]; 533342452Sarybchik sfxge_evq_stat_update(evq); 534342452Sarybchik for (id = 0; id < EV_NQSTATS; id++) 535342452Sarybchik sc->ev_stats[id] += evq->stats[id]; 536227569Sphilip } 537227569Sphilipout: 538278221Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 539227569Sphilip} 540227569Sphilip 541227569Sphilipstatic int 542227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 543227569Sphilip{ 544227569Sphilip struct sfxge_softc *sc = arg1; 545227569Sphilip unsigned int id = arg2; 546227569Sphilip 547227569Sphilip sfxge_ev_stat_update(sc); 548227569Sphilip 549272325Sgnn return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 550227569Sphilip} 551227569Sphilip 552227569Sphilipstatic void 553227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 554227569Sphilip{ 555227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 556227569Sphilip struct sysctl_oid_list *stat_list; 557227569Sphilip unsigned int id; 558227569Sphilip char name[40]; 559227569Sphilip 560227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 561227569Sphilip 562227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 563227569Sphilip snprintf(name, sizeof(name), "ev_%s", 564227569Sphilip efx_ev_qstat_name(sc->enp, id)); 565227569Sphilip SYSCTL_ADD_PROC( 566227569Sphilip ctx, stat_list, 567227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 568227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 569227569Sphilip ""); 570227569Sphilip } 571227569Sphilip} 572227569Sphilip 573277886Sarybchik#endif /* EFSYS_OPT_QSTATS */ 574277886Sarybchik 575227569Sphilipstatic void 576227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 577227569Sphilip{ 578227569Sphilip struct sfxge_evq *evq; 579227569Sphilip efx_evq_t *eep; 580227569Sphilip 581227569Sphilip evq = sc->evq[idx]; 582227569Sphilip eep = evq->common; 583227569Sphilip 584227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 585227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 586227569Sphilip 587227569Sphilip (void)efx_ev_qmoderate(eep, us); 588227569Sphilip} 589227569Sphilip 590227569Sphilipstatic int 591227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 592227569Sphilip{ 593227569Sphilip struct sfxge_softc *sc = arg1; 594227569Sphilip struct sfxge_intr *intr = &sc->intr; 595227569Sphilip unsigned int moderation; 596227569Sphilip int error; 597278940Sarybchik unsigned int index; 598227569Sphilip 599278221Sarybchik SFXGE_ADAPTER_LOCK(sc); 600227569Sphilip 601272325Sgnn if (req->newptr != NULL) { 602227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 603227569Sphilip != 0) 604227569Sphilip goto out; 605227569Sphilip 606227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 607227569Sphilip * so we have to range-check the value ourselves. 608227569Sphilip */ 609227569Sphilip if (moderation > 610279182Sarybchik efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 611227569Sphilip error = EINVAL; 612227569Sphilip goto out; 613227569Sphilip } 614227569Sphilip 615227569Sphilip sc->ev_moderation = moderation; 616227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 617278940Sarybchik for (index = 0; index < sc->evq_count; index++) 618227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 619227569Sphilip } 620227569Sphilip } else { 621227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 622227569Sphilip sizeof(sc->ev_moderation)); 623227569Sphilip } 624227569Sphilip 625227569Sphilipout: 626278221Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 627227569Sphilip 628272325Sgnn return (error); 629227569Sphilip} 630227569Sphilip 631227569Sphilipstatic boolean_t 632227569Sphilipsfxge_ev_initialized(void *arg) 633227569Sphilip{ 634227569Sphilip struct sfxge_evq *evq; 635272325Sgnn 636227569Sphilip evq = (struct sfxge_evq *)arg; 637279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 638227569Sphilip 639283514Sarybchik /* Init done events may be duplicated on 7xxx */ 640283514Sarybchik KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 641283514Sarybchik evq->init_state == SFXGE_EVQ_STARTED, 642227569Sphilip ("evq not starting")); 643227569Sphilip 644227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 645227569Sphilip 646227569Sphilip return (0); 647227569Sphilip} 648227569Sphilip 649227569Sphilipstatic boolean_t 650227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 651227569Sphilip{ 652227569Sphilip struct sfxge_evq *evq; 653227569Sphilip struct sfxge_softc *sc; 654227569Sphilip 655227569Sphilip evq = (struct sfxge_evq *)arg; 656279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 657279177Sarybchik 658227569Sphilip sc = evq->sc; 659227569Sphilip 660227569Sphilip sfxge_mac_link_update(sc, link_mode); 661227569Sphilip 662227569Sphilip return (0); 663227569Sphilip} 664227569Sphilip 665227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 666227569Sphilip .eec_initialized = sfxge_ev_initialized, 667227569Sphilip .eec_rx = sfxge_ev_rx, 668227569Sphilip .eec_tx = sfxge_ev_tx, 669227569Sphilip .eec_exception = sfxge_ev_exception, 670227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 671227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 672227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 673227569Sphilip .eec_software = sfxge_ev_software, 674227569Sphilip .eec_sram = sfxge_ev_sram, 675227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 676227569Sphilip .eec_timer = sfxge_ev_timer, 677227569Sphilip .eec_link_change = sfxge_ev_link_change, 678227569Sphilip}; 679227569Sphilip 680227569Sphilip 681227569Sphilipint 682277884Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 683227569Sphilip{ 684227569Sphilip int rc; 685227569Sphilip 686278221Sarybchik SFXGE_EVQ_LOCK(evq); 687227569Sphilip 688279351Sarybchik if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 689279351Sarybchik evq->init_state != SFXGE_EVQ_STARTED)) { 690227569Sphilip rc = EINVAL; 691227569Sphilip goto fail; 692227569Sphilip } 693227569Sphilip 694227569Sphilip /* Synchronize the DMA memory for reading */ 695227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 696227569Sphilip BUS_DMASYNC_POSTREAD); 697227569Sphilip 698227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 699227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 700227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 701227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 702227569Sphilip 703227569Sphilip /* Poll the queue */ 704227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 705227569Sphilip 706227569Sphilip evq->rx_done = 0; 707227569Sphilip evq->tx_done = 0; 708227569Sphilip 709227569Sphilip /* Perform any pending completion processing */ 710227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 711227569Sphilip 712227569Sphilip /* Re-prime the event queue for interrupts */ 713227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 714227569Sphilip goto fail; 715227569Sphilip 716278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 717227569Sphilip 718227569Sphilip return (0); 719227569Sphilip 720227569Sphilipfail: 721278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 722227569Sphilip return (rc); 723227569Sphilip} 724227569Sphilip 725227569Sphilipstatic void 726227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 727227569Sphilip{ 728227569Sphilip struct sfxge_evq *evq; 729227569Sphilip 730227569Sphilip evq = sc->evq[index]; 731227569Sphilip 732227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 733227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 734227569Sphilip 735278221Sarybchik SFXGE_EVQ_LOCK(evq); 736227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 737227569Sphilip evq->read_ptr = 0; 738227569Sphilip evq->exception = B_FALSE; 739227569Sphilip 740277886Sarybchik#if EFSYS_OPT_QSTATS 741227569Sphilip /* Add event counts before discarding the common evq state */ 742342452Sarybchik efx_ev_qstats_update(evq->common, evq->stats); 743277886Sarybchik#endif 744227569Sphilip 745227569Sphilip efx_ev_qdestroy(evq->common); 746227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 747272328Sgnn EFX_EVQ_NBUFS(evq->entries)); 748278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 749227569Sphilip} 750227569Sphilip 751227569Sphilipstatic int 752227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 753227569Sphilip{ 754227569Sphilip struct sfxge_evq *evq; 755227569Sphilip efsys_mem_t *esmp; 756227569Sphilip int count; 757227569Sphilip int rc; 758227569Sphilip 759227569Sphilip evq = sc->evq[index]; 760227569Sphilip esmp = &evq->mem; 761227569Sphilip 762227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 763227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 764227569Sphilip 765227569Sphilip /* Clear all events. */ 766272328Sgnn (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 767227569Sphilip 768227569Sphilip /* Program the buffer table. */ 769227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 770272328Sgnn EFX_EVQ_NBUFS(evq->entries))) != 0) 771272328Sgnn return (rc); 772227569Sphilip 773227569Sphilip /* Create the common code event queue. */ 774272328Sgnn if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 775310939Sarybchik evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 776310939Sarybchik &evq->common)) != 0) 777227569Sphilip goto fail; 778227569Sphilip 779278221Sarybchik SFXGE_EVQ_LOCK(evq); 780227569Sphilip 781227569Sphilip /* Prime the event queue for interrupts */ 782227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 783227569Sphilip goto fail2; 784227569Sphilip 785227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 786227569Sphilip 787278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 788227569Sphilip 789227569Sphilip /* Wait for the initialization event */ 790227569Sphilip count = 0; 791227569Sphilip do { 792227569Sphilip /* Pause for 100 ms */ 793227569Sphilip pause("sfxge evq init", hz / 10); 794227569Sphilip 795227569Sphilip /* Check to see if the test event has been processed */ 796227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 797227569Sphilip goto done; 798227569Sphilip 799227569Sphilip } while (++count < 20); 800227569Sphilip 801227569Sphilip rc = ETIMEDOUT; 802227569Sphilip goto fail3; 803227569Sphilip 804227569Sphilipdone: 805227569Sphilip return (0); 806227569Sphilip 807227569Sphilipfail3: 808278221Sarybchik SFXGE_EVQ_LOCK(evq); 809227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 810227569Sphilipfail2: 811278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 812227569Sphilip efx_ev_qdestroy(evq->common); 813227569Sphilipfail: 814227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 815272328Sgnn EFX_EVQ_NBUFS(evq->entries)); 816227569Sphilip 817227569Sphilip return (rc); 818227569Sphilip} 819227569Sphilip 820227569Sphilipvoid 821227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 822227569Sphilip{ 823227569Sphilip struct sfxge_intr *intr; 824227569Sphilip efx_nic_t *enp; 825227569Sphilip int index; 826227569Sphilip 827227569Sphilip intr = &sc->intr; 828227569Sphilip enp = sc->enp; 829227569Sphilip 830227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 831227569Sphilip ("Interrupts not started")); 832227569Sphilip 833227569Sphilip /* Stop the event queue(s) */ 834278940Sarybchik index = sc->evq_count; 835227569Sphilip while (--index >= 0) 836227569Sphilip sfxge_ev_qstop(sc, index); 837227569Sphilip 838227569Sphilip /* Tear down the event module */ 839227569Sphilip efx_ev_fini(enp); 840227569Sphilip} 841227569Sphilip 842227569Sphilipint 843227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 844227569Sphilip{ 845227569Sphilip struct sfxge_intr *intr; 846227569Sphilip int index; 847227569Sphilip int rc; 848227569Sphilip 849227569Sphilip intr = &sc->intr; 850227569Sphilip 851227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 852227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 853227569Sphilip 854227569Sphilip /* Initialize the event module */ 855227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 856272325Sgnn return (rc); 857227569Sphilip 858227569Sphilip /* Start the event queues */ 859278940Sarybchik for (index = 0; index < sc->evq_count; index++) { 860227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 861227569Sphilip goto fail; 862227569Sphilip } 863227569Sphilip 864227569Sphilip return (0); 865227569Sphilip 866227569Sphilipfail: 867227569Sphilip /* Stop the event queue(s) */ 868227569Sphilip while (--index >= 0) 869227569Sphilip sfxge_ev_qstop(sc, index); 870227569Sphilip 871227569Sphilip /* Tear down the event module */ 872227569Sphilip efx_ev_fini(sc->enp); 873227569Sphilip 874227569Sphilip return (rc); 875227569Sphilip} 876227569Sphilip 877227569Sphilipstatic void 878227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 879227569Sphilip{ 880227569Sphilip struct sfxge_evq *evq; 881227569Sphilip 882227569Sphilip evq = sc->evq[index]; 883227569Sphilip 884227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 885227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 886227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 887227569Sphilip 888227569Sphilip sfxge_dma_free(&evq->mem); 889227569Sphilip 890227569Sphilip sc->evq[index] = NULL; 891227569Sphilip 892278221Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 893227569Sphilip 894227569Sphilip free(evq, M_SFXGE); 895227569Sphilip} 896227569Sphilip 897227569Sphilipstatic int 898227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 899227569Sphilip{ 900227569Sphilip struct sfxge_evq *evq; 901227569Sphilip efsys_mem_t *esmp; 902227569Sphilip int rc; 903227569Sphilip 904227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 905227569Sphilip 906227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 907227569Sphilip evq->sc = sc; 908227569Sphilip evq->index = index; 909227569Sphilip sc->evq[index] = evq; 910227569Sphilip esmp = &evq->mem; 911227569Sphilip 912272328Sgnn /* Build an event queue with room for one event per tx and rx buffer, 913272328Sgnn * plus some extra for link state events and MCDI completions. 914272328Sgnn * There are three tx queues in the first event queue and one in 915272328Sgnn * other. 916272328Sgnn */ 917272328Sgnn if (index == 0) 918272328Sgnn evq->entries = 919272328Sgnn ROUNDUP_POW_OF_TWO(sc->rxq_entries + 920272328Sgnn 3 * sc->txq_entries + 921272328Sgnn 128); 922272328Sgnn else 923272328Sgnn evq->entries = 924272328Sgnn ROUNDUP_POW_OF_TWO(sc->rxq_entries + 925272328Sgnn sc->txq_entries + 926272328Sgnn 128); 927272328Sgnn 928227569Sphilip /* Initialise TX completion list */ 929227569Sphilip evq->txqs = &evq->txq; 930227569Sphilip 931227569Sphilip /* Allocate DMA space. */ 932272328Sgnn if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 933227569Sphilip return (rc); 934227569Sphilip 935227569Sphilip /* Allocate buffer table entries. */ 936272328Sgnn sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 937227569Sphilip &evq->buf_base_id); 938227569Sphilip 939278250Sarybchik SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 940227569Sphilip 941227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 942227569Sphilip 943342452Sarybchik#if EFSYS_OPT_QSTATS 944342452Sarybchik rc = sfxge_evq_stat_init(evq); 945342452Sarybchik if (rc != 0) 946342452Sarybchik goto fail_evq_stat_init; 947342452Sarybchik#endif 948342452Sarybchik 949227569Sphilip return (0); 950342452Sarybchik 951342452Sarybchik#if EFSYS_OPT_QSTATS 952342452Sarybchikfail_evq_stat_init: 953342452Sarybchik evq->init_state = SFXGE_EVQ_UNINITIALIZED; 954342452Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 955342452Sarybchik sfxge_dma_free(esmp); 956342452Sarybchik sc->evq[index] = NULL; 957342452Sarybchik free(evq, M_SFXGE); 958342452Sarybchik 959342452Sarybchik return (rc); 960342452Sarybchik#endif 961227569Sphilip} 962227569Sphilip 963227569Sphilipvoid 964227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 965227569Sphilip{ 966227569Sphilip struct sfxge_intr *intr; 967227569Sphilip int index; 968227569Sphilip 969227569Sphilip intr = &sc->intr; 970227569Sphilip 971227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 972227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 973227569Sphilip 974227569Sphilip sc->ev_moderation = 0; 975227569Sphilip 976227569Sphilip /* Tear down the event queue(s). */ 977278940Sarybchik index = sc->evq_count; 978227569Sphilip while (--index >= 0) 979227569Sphilip sfxge_ev_qfini(sc, index); 980278940Sarybchik 981278940Sarybchik sc->evq_count = 0; 982227569Sphilip} 983227569Sphilip 984227569Sphilipint 985227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 986227569Sphilip{ 987227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 988227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 989227569Sphilip struct sfxge_intr *intr; 990227569Sphilip int index; 991227569Sphilip int rc; 992227569Sphilip 993227569Sphilip intr = &sc->intr; 994227569Sphilip 995278940Sarybchik sc->evq_count = intr->n_alloc; 996278940Sarybchik 997227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 998227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 999227569Sphilip 1000227569Sphilip /* Set default interrupt moderation; add a sysctl to 1001227569Sphilip * read and change it. 1002227569Sphilip */ 1003277893Sarybchik sc->ev_moderation = SFXGE_MODERATION; 1004227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1005227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 1006227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 1007227569Sphilip "sfxge interrupt moderation (us)"); 1008227569Sphilip 1009342452Sarybchik#if EFSYS_OPT_QSTATS 1010342452Sarybchik sc->evqs_stats_node = SYSCTL_ADD_NODE( 1011342452Sarybchik device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1012342452Sarybchik OID_AUTO, "evq", CTLFLAG_RD, NULL, "Event queues stats"); 1013342452Sarybchik if (sc->evqs_stats_node == NULL) { 1014342452Sarybchik rc = ENOMEM; 1015342452Sarybchik goto fail_evqs_stats_node; 1016342452Sarybchik } 1017342452Sarybchik#endif 1018342452Sarybchik 1019227569Sphilip /* 1020227569Sphilip * Initialize the event queue(s) - one per interrupt. 1021227569Sphilip */ 1022278940Sarybchik for (index = 0; index < sc->evq_count; index++) { 1023227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1024227569Sphilip goto fail; 1025227569Sphilip } 1026227569Sphilip 1027277886Sarybchik#if EFSYS_OPT_QSTATS 1028227569Sphilip sfxge_ev_stat_init(sc); 1029277886Sarybchik#endif 1030227569Sphilip 1031227569Sphilip return (0); 1032227569Sphilip 1033227569Sphilipfail: 1034227569Sphilip while (--index >= 0) 1035227569Sphilip sfxge_ev_qfini(sc, index); 1036227569Sphilip 1037342452Sarybchik#if EFSYS_OPT_QSTATS 1038342452Sarybchikfail_evqs_stats_node: 1039342452Sarybchik#endif 1040278940Sarybchik sc->evq_count = 0; 1041227569Sphilip return (rc); 1042227569Sphilip} 1043