1227569Sphilip/*- 2301388Sarybchik * Copyright (c) 2010-2016 Solarflare Communications Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9284555Sarybchik * modification, are permitted provided that the following conditions are met: 10227569Sphilip * 11284555Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 12284555Sarybchik * this list of conditions and the following disclaimer. 13284555Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 14284555Sarybchik * this list of conditions and the following disclaimer in the documentation 15284555Sarybchik * and/or other materials provided with the distribution. 16284555Sarybchik * 17284555Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18284555Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19284555Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20284555Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21284555Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22284555Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23284555Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24284555Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25284555Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26284555Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27284555Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28284555Sarybchik * 29284555Sarybchik * The views and conclusions contained in the software and documentation are 30284555Sarybchik * those of the authors and should not be interpreted as representing official 31284555Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 32227569Sphilip */ 33227569Sphilip 34227569Sphilip#include <sys/cdefs.h> 35227569Sphilip__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_ev.c 342529 2018-12-26 10:39:34Z arybchik $"); 36227569Sphilip 37227569Sphilip#include <sys/param.h> 38227569Sphilip#include <sys/systm.h> 39227569Sphilip 40227569Sphilip#include "common/efx.h" 41227569Sphilip 42227569Sphilip#include "sfxge.h" 43227569Sphilip 44227569Sphilipstatic void 45227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 46227569Sphilip{ 47227569Sphilip struct sfxge_softc *sc; 48227569Sphilip unsigned int index; 49227569Sphilip struct sfxge_rxq *rxq; 50227569Sphilip struct sfxge_txq *txq; 51227569Sphilip 52280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 53280582Sarybchik 54227569Sphilip sc = evq->sc; 55227569Sphilip index = evq->index; 56227569Sphilip rxq = sc->rxq[index]; 57227569Sphilip 58227569Sphilip if ((txq = evq->txq) != NULL) { 59227569Sphilip evq->txq = NULL; 60227569Sphilip evq->txqs = &(evq->txq); 61227569Sphilip 62227569Sphilip do { 63227569Sphilip struct sfxge_txq *next; 64227569Sphilip 65227569Sphilip next = txq->next; 66227569Sphilip txq->next = NULL; 67227569Sphilip 68227569Sphilip KASSERT(txq->evq_index == index, 69227569Sphilip ("txq->evq_index != index")); 70227569Sphilip 71227569Sphilip if (txq->pending != txq->completed) 72280513Sarybchik sfxge_tx_qcomplete(txq, evq); 73227569Sphilip 74227569Sphilip txq = next; 75227569Sphilip } while (txq != NULL); 76227569Sphilip } 77227569Sphilip 78227569Sphilip if (rxq->pending != rxq->completed) 79227569Sphilip sfxge_rx_qcomplete(rxq, eop); 80227569Sphilip} 81227569Sphilip 82298836Sarybchikstatic struct sfxge_rxq * 83298836Sarybchiksfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 84298836Sarybchik{ 85298836Sarybchik struct sfxge_rxq *rxq; 86298836Sarybchik 87298836Sarybchik KASSERT(label == 0, ("unexpected rxq label != 0")); 88298836Sarybchik 89298836Sarybchik rxq = evq->sc->rxq[evq->index]; 90298836Sarybchik 91298836Sarybchik KASSERT(rxq != NULL, ("rxq == NULL")); 92298836Sarybchik KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 93298836Sarybchik 94298836Sarybchik return (rxq); 95298836Sarybchik} 96298836Sarybchik 97227569Sphilipstatic boolean_t 98227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 99284555Sarybchik uint16_t flags) 100227569Sphilip{ 101227569Sphilip struct sfxge_evq *evq; 102227569Sphilip struct sfxge_softc *sc; 103227569Sphilip struct sfxge_rxq *rxq; 104284555Sarybchik unsigned int stop; 105284555Sarybchik unsigned int delta; 106227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 107227569Sphilip 108227569Sphilip evq = arg; 109280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 110280582Sarybchik 111227569Sphilip sc = evq->sc; 112227569Sphilip 113227569Sphilip if (evq->exception) 114227569Sphilip goto done; 115227569Sphilip 116298836Sarybchik rxq = sfxge_get_rxq_by_label(evq, label); 117280596Sarybchik if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 118227569Sphilip goto done; 119227569Sphilip 120284555Sarybchik stop = (id + 1) & rxq->ptr_mask; 121284555Sarybchik id = rxq->pending & rxq->ptr_mask; 122284555Sarybchik delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 123284555Sarybchik rxq->pending += delta; 124227569Sphilip 125284555Sarybchik if (delta != 1) { 126301985Sarybchik if ((delta <= 0) || 127284555Sarybchik (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 128284555Sarybchik evq->exception = B_TRUE; 129227569Sphilip 130284555Sarybchik device_printf(sc->dev, "RX completion out of order" 131284555Sarybchik " (id=%#x delta=%u flags=%#x); resetting\n", 132284555Sarybchik id, delta, flags); 133284555Sarybchik sfxge_schedule_reset(sc); 134284555Sarybchik 135284555Sarybchik goto done; 136284555Sarybchik } 137227569Sphilip } 138227569Sphilip 139227569Sphilip rx_desc = &rxq->queue[id]; 140227569Sphilip 141227569Sphilip prefetch_read_many(rx_desc->mbuf); 142227569Sphilip 143284555Sarybchik for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 144284555Sarybchik rx_desc = &rxq->queue[id]; 145284555Sarybchik KASSERT(rx_desc->flags == EFX_DISCARD, 146284555Sarybchik ("rx_desc->flags != EFX_DISCARD")); 147284555Sarybchik rx_desc->flags = flags; 148284555Sarybchik 149284555Sarybchik KASSERT(size < (1 << 16), ("size > (1 << 16)")); 150284555Sarybchik rx_desc->size = (uint16_t)size; 151284555Sarybchik } 152284555Sarybchik 153227569Sphilip evq->rx_done++; 154227569Sphilip 155227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 156227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 157227569Sphilip 158227569Sphilipdone: 159227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 160227569Sphilip} 161227569Sphilip 162227569Sphilipstatic boolean_t 163227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 164227569Sphilip{ 165227569Sphilip struct sfxge_evq *evq; 166227569Sphilip struct sfxge_softc *sc; 167227569Sphilip 168227569Sphilip evq = (struct sfxge_evq *)arg; 169280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 170280582Sarybchik 171227569Sphilip sc = evq->sc; 172227569Sphilip 173284555Sarybchik DBGPRINT(sc->dev, "[%d] %s", evq->index, 174284555Sarybchik (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 175284555Sarybchik (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 176284555Sarybchik (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 177284555Sarybchik (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 178284555Sarybchik (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 179284555Sarybchik (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 180284555Sarybchik (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 181284555Sarybchik (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 182284555Sarybchik (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 183284555Sarybchik "UNKNOWN"); 184284555Sarybchik 185227569Sphilip evq->exception = B_TRUE; 186227569Sphilip 187227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 188227569Sphilip device_printf(sc->dev, 189227569Sphilip "hardware exception (code=%u); resetting\n", 190227569Sphilip code); 191227569Sphilip sfxge_schedule_reset(sc); 192227569Sphilip } 193227569Sphilip 194227569Sphilip return (B_FALSE); 195227569Sphilip} 196227569Sphilip 197227569Sphilipstatic boolean_t 198265884Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 199227569Sphilip{ 200227569Sphilip struct sfxge_evq *evq; 201227569Sphilip struct sfxge_softc *sc; 202227569Sphilip struct sfxge_rxq *rxq; 203227569Sphilip unsigned int index; 204227569Sphilip uint16_t magic; 205227569Sphilip 206227569Sphilip evq = (struct sfxge_evq *)arg; 207280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 208280582Sarybchik 209227569Sphilip sc = evq->sc; 210265884Sgnn rxq = sc->rxq[rxq_index]; 211227569Sphilip 212227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 213227569Sphilip 214227569Sphilip /* Resend a software event on the correct queue */ 215227569Sphilip index = rxq->index; 216284555Sarybchik if (index == evq->index) { 217284555Sarybchik sfxge_rx_qflush_done(rxq); 218284555Sarybchik return (B_FALSE); 219284555Sarybchik } 220284555Sarybchik 221227569Sphilip evq = sc->evq[index]; 222301315Sarybchik magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 223227569Sphilip 224227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 225227569Sphilip ("evq not started")); 226227569Sphilip efx_ev_qpost(evq->common, magic); 227227569Sphilip 228227569Sphilip return (B_FALSE); 229227569Sphilip} 230227569Sphilip 231227569Sphilipstatic boolean_t 232265884Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 233227569Sphilip{ 234227569Sphilip struct sfxge_evq *evq; 235227569Sphilip struct sfxge_softc *sc; 236227569Sphilip struct sfxge_rxq *rxq; 237227569Sphilip unsigned int index; 238227569Sphilip uint16_t magic; 239227569Sphilip 240227569Sphilip evq = (struct sfxge_evq *)arg; 241280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 242280582Sarybchik 243227569Sphilip sc = evq->sc; 244265884Sgnn rxq = sc->rxq[rxq_index]; 245227569Sphilip 246227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 247227569Sphilip 248227569Sphilip /* Resend a software event on the correct queue */ 249227569Sphilip index = rxq->index; 250227569Sphilip evq = sc->evq[index]; 251301315Sarybchik magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 252227569Sphilip 253227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 254227569Sphilip ("evq not started")); 255227569Sphilip efx_ev_qpost(evq->common, magic); 256227569Sphilip 257227569Sphilip return (B_FALSE); 258227569Sphilip} 259227569Sphilip 260265884Sgnnstatic struct sfxge_txq * 261265884Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 262265884Sgnn{ 263265884Sgnn unsigned int index; 264265884Sgnn 265342529Sarybchik KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 266342529Sarybchik ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 267342529Sarybchik (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 268342529Sarybchik ("unexpected txq label")); 269342529Sarybchik 270342527Sarybchik index = (evq->index == 0) ? 271342527Sarybchik label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 272280501Sarybchik return (evq->sc->txq[index]); 273265884Sgnn} 274265884Sgnn 275227569Sphilipstatic boolean_t 276227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 277227569Sphilip{ 278227569Sphilip struct sfxge_evq *evq; 279227569Sphilip struct sfxge_txq *txq; 280227569Sphilip unsigned int stop; 281227569Sphilip unsigned int delta; 282227569Sphilip 283227569Sphilip evq = (struct sfxge_evq *)arg; 284280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 285280582Sarybchik 286265884Sgnn txq = sfxge_get_txq_by_label(evq, label); 287227569Sphilip 288227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 289227569Sphilip KASSERT(evq->index == txq->evq_index, 290227569Sphilip ("evq->index != txq->evq_index")); 291227569Sphilip 292280596Sarybchik if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 293227569Sphilip goto done; 294227569Sphilip 295280502Sarybchik stop = (id + 1) & txq->ptr_mask; 296280502Sarybchik id = txq->pending & txq->ptr_mask; 297227569Sphilip 298280502Sarybchik delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 299227569Sphilip txq->pending += delta; 300227569Sphilip 301227569Sphilip evq->tx_done++; 302227569Sphilip 303227569Sphilip if (txq->next == NULL && 304227569Sphilip evq->txqs != &(txq->next)) { 305227569Sphilip *(evq->txqs) = txq; 306227569Sphilip evq->txqs = &(txq->next); 307227569Sphilip } 308227569Sphilip 309227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 310280513Sarybchik sfxge_tx_qcomplete(txq, evq); 311227569Sphilip 312227569Sphilipdone: 313227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 314227569Sphilip} 315227569Sphilip 316227569Sphilipstatic boolean_t 317265884Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 318227569Sphilip{ 319227569Sphilip struct sfxge_evq *evq; 320227569Sphilip struct sfxge_softc *sc; 321227569Sphilip struct sfxge_txq *txq; 322227569Sphilip uint16_t magic; 323227569Sphilip 324227569Sphilip evq = (struct sfxge_evq *)arg; 325280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 326280582Sarybchik 327227569Sphilip sc = evq->sc; 328265884Sgnn txq = sc->txq[txq_index]; 329227569Sphilip 330227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 331227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 332227569Sphilip ("txq not initialized")); 333227569Sphilip 334284555Sarybchik if (txq->evq_index == evq->index) { 335284555Sarybchik sfxge_tx_qflush_done(txq); 336284555Sarybchik return (B_FALSE); 337284555Sarybchik } 338284555Sarybchik 339227569Sphilip /* Resend a software event on the correct queue */ 340227569Sphilip evq = sc->evq[txq->evq_index]; 341301315Sarybchik magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 342227569Sphilip 343227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 344227569Sphilip ("evq not started")); 345227569Sphilip efx_ev_qpost(evq->common, magic); 346227569Sphilip 347227569Sphilip return (B_FALSE); 348227569Sphilip} 349227569Sphilip 350227569Sphilipstatic boolean_t 351227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 352227569Sphilip{ 353227569Sphilip struct sfxge_evq *evq; 354227569Sphilip struct sfxge_softc *sc; 355227569Sphilip unsigned int label; 356227569Sphilip 357227569Sphilip evq = (struct sfxge_evq *)arg; 358280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 359280582Sarybchik 360227569Sphilip sc = evq->sc; 361227569Sphilip 362227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 363227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 364227569Sphilip 365227569Sphilip switch (magic) { 366301314Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 367298836Sarybchik sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 368298836Sarybchik break; 369227569Sphilip 370301314Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 371298836Sarybchik sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 372227569Sphilip break; 373227569Sphilip 374301314Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 375298836Sarybchik sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 376227569Sphilip break; 377227569Sphilip 378301314Sarybchik case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 379265884Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 380227569Sphilip 381227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 382227569Sphilip KASSERT(evq->index == txq->evq_index, 383227569Sphilip ("evq->index != txq->evq_index")); 384227569Sphilip 385227569Sphilip sfxge_tx_qflush_done(txq); 386227569Sphilip break; 387227569Sphilip } 388227569Sphilip default: 389227569Sphilip break; 390227569Sphilip } 391227569Sphilip 392227569Sphilip return (B_FALSE); 393227569Sphilip} 394227569Sphilip 395227569Sphilipstatic boolean_t 396227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 397227569Sphilip{ 398227569Sphilip (void)arg; 399227569Sphilip (void)code; 400227569Sphilip 401227569Sphilip switch (code) { 402227569Sphilip case EFX_SRAM_UPDATE: 403227569Sphilip EFSYS_PROBE(sram_update); 404227569Sphilip break; 405227569Sphilip 406227569Sphilip case EFX_SRAM_CLEAR: 407227569Sphilip EFSYS_PROBE(sram_clear); 408227569Sphilip break; 409227569Sphilip 410227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 411227569Sphilip EFSYS_PROBE(sram_illegal_clear); 412227569Sphilip break; 413227569Sphilip 414227569Sphilip default: 415227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 416227569Sphilip break; 417227569Sphilip } 418227569Sphilip 419227569Sphilip return (B_FALSE); 420227569Sphilip} 421227569Sphilip 422227569Sphilipstatic boolean_t 423227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 424227569Sphilip{ 425227569Sphilip (void)arg; 426227569Sphilip (void)index; 427227569Sphilip 428227569Sphilip return (B_FALSE); 429227569Sphilip} 430227569Sphilip 431227569Sphilipstatic boolean_t 432227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 433227569Sphilip{ 434227569Sphilip (void)arg; 435227569Sphilip (void)index; 436227569Sphilip 437227569Sphilip return (B_FALSE); 438227569Sphilip} 439227569Sphilip 440280510Sarybchik#if EFSYS_OPT_QSTATS 441280510Sarybchik 442227569Sphilipstatic void 443342523Sarybchiksfxge_evq_stat_update(struct sfxge_evq *evq) 444342523Sarybchik{ 445342523Sarybchik clock_t now; 446342523Sarybchik 447342523Sarybchik SFXGE_EVQ_LOCK(evq); 448342523Sarybchik 449342523Sarybchik if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 450342523Sarybchik goto out; 451342523Sarybchik 452342523Sarybchik now = ticks; 453342523Sarybchik if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 454342523Sarybchik goto out; 455342523Sarybchik 456342523Sarybchik evq->stats_update_time = now; 457342523Sarybchik efx_ev_qstats_update(evq->common, evq->stats); 458342523Sarybchik 459342523Sarybchikout: 460342523Sarybchik SFXGE_EVQ_UNLOCK(evq); 461342523Sarybchik} 462342523Sarybchik 463342523Sarybchikstatic int 464342523Sarybchiksfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 465342523Sarybchik{ 466342523Sarybchik struct sfxge_evq *evq = arg1; 467342523Sarybchik struct sfxge_softc *sc = evq->sc; 468342523Sarybchik unsigned int id = arg2; 469342523Sarybchik 470342523Sarybchik SFXGE_ADAPTER_LOCK(sc); 471342523Sarybchik 472342523Sarybchik sfxge_evq_stat_update(evq); 473342523Sarybchik 474342523Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 475342523Sarybchik 476342523Sarybchik return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 477342523Sarybchik} 478342523Sarybchik 479342523Sarybchikstatic int 480342523Sarybchiksfxge_evq_stat_init(struct sfxge_evq *evq) 481342523Sarybchik{ 482342523Sarybchik struct sfxge_softc *sc = evq->sc; 483342523Sarybchik struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 484342523Sarybchik char name[16]; 485342523Sarybchik struct sysctl_oid *evq_stats_node; 486342523Sarybchik unsigned int id; 487342523Sarybchik 488342523Sarybchik snprintf(name, sizeof(name), "%u", evq->index); 489342523Sarybchik evq_stats_node = SYSCTL_ADD_NODE(ctx, 490342523Sarybchik SYSCTL_CHILDREN(sc->evqs_stats_node), 491342523Sarybchik OID_AUTO, name, CTLFLAG_RD, NULL, ""); 492342523Sarybchik if (evq_stats_node == NULL) 493342523Sarybchik return (ENOMEM); 494342523Sarybchik 495342523Sarybchik for (id = 0; id < EV_NQSTATS; id++) { 496342523Sarybchik SYSCTL_ADD_PROC( 497342523Sarybchik ctx, SYSCTL_CHILDREN(evq_stats_node), 498342523Sarybchik OID_AUTO, efx_ev_qstat_name(sc->enp, id), 499342523Sarybchik CTLTYPE_U64|CTLFLAG_RD, 500342523Sarybchik evq, id, sfxge_evq_stat_handler, "Q", 501342523Sarybchik ""); 502342523Sarybchik } 503342523Sarybchik 504342523Sarybchik return (0); 505342523Sarybchik} 506342523Sarybchik 507342523Sarybchikstatic void 508227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 509227569Sphilip{ 510227569Sphilip struct sfxge_evq *evq; 511227569Sphilip unsigned int index; 512227569Sphilip clock_t now; 513342523Sarybchik unsigned int id; 514227569Sphilip 515280522Sarybchik SFXGE_ADAPTER_LOCK(sc); 516227569Sphilip 517227569Sphilip now = ticks; 518304679Sarybchik if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 519227569Sphilip goto out; 520227569Sphilip 521227569Sphilip sc->ev_stats_update_time = now; 522227569Sphilip 523342523Sarybchik memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 524342523Sarybchik 525342523Sarybchik /* Update and add event counts from each event queue in turn */ 526280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 527227569Sphilip evq = sc->evq[index]; 528342523Sarybchik sfxge_evq_stat_update(evq); 529342523Sarybchik for (id = 0; id < EV_NQSTATS; id++) 530342523Sarybchik sc->ev_stats[id] += evq->stats[id]; 531227569Sphilip } 532227569Sphilipout: 533280522Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 534227569Sphilip} 535227569Sphilip 536227569Sphilipstatic int 537227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 538227569Sphilip{ 539227569Sphilip struct sfxge_softc *sc = arg1; 540227569Sphilip unsigned int id = arg2; 541227569Sphilip 542227569Sphilip sfxge_ev_stat_update(sc); 543227569Sphilip 544280501Sarybchik return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 545227569Sphilip} 546227569Sphilip 547227569Sphilipstatic void 548227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 549227569Sphilip{ 550227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 551227569Sphilip struct sysctl_oid_list *stat_list; 552227569Sphilip unsigned int id; 553227569Sphilip char name[40]; 554227569Sphilip 555227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 556227569Sphilip 557227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 558227569Sphilip snprintf(name, sizeof(name), "ev_%s", 559227569Sphilip efx_ev_qstat_name(sc->enp, id)); 560227569Sphilip SYSCTL_ADD_PROC( 561227569Sphilip ctx, stat_list, 562227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 563227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 564227569Sphilip ""); 565227569Sphilip } 566227569Sphilip} 567227569Sphilip 568280510Sarybchik#endif /* EFSYS_OPT_QSTATS */ 569280510Sarybchik 570227569Sphilipstatic void 571227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 572227569Sphilip{ 573227569Sphilip struct sfxge_evq *evq; 574227569Sphilip efx_evq_t *eep; 575227569Sphilip 576227569Sphilip evq = sc->evq[idx]; 577227569Sphilip eep = evq->common; 578227569Sphilip 579227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 580227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 581227569Sphilip 582227569Sphilip (void)efx_ev_qmoderate(eep, us); 583227569Sphilip} 584227569Sphilip 585227569Sphilipstatic int 586227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 587227569Sphilip{ 588227569Sphilip struct sfxge_softc *sc = arg1; 589227569Sphilip struct sfxge_intr *intr = &sc->intr; 590227569Sphilip unsigned int moderation; 591227569Sphilip int error; 592280541Sarybchik unsigned int index; 593227569Sphilip 594280522Sarybchik SFXGE_ADAPTER_LOCK(sc); 595227569Sphilip 596280501Sarybchik if (req->newptr != NULL) { 597227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 598227569Sphilip != 0) 599227569Sphilip goto out; 600227569Sphilip 601227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 602227569Sphilip * so we have to range-check the value ourselves. 603227569Sphilip */ 604227569Sphilip if (moderation > 605280588Sarybchik efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 606227569Sphilip error = EINVAL; 607227569Sphilip goto out; 608227569Sphilip } 609227569Sphilip 610227569Sphilip sc->ev_moderation = moderation; 611227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 612280541Sarybchik for (index = 0; index < sc->evq_count; index++) 613227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 614227569Sphilip } 615227569Sphilip } else { 616227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 617227569Sphilip sizeof(sc->ev_moderation)); 618227569Sphilip } 619227569Sphilip 620227569Sphilipout: 621280522Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 622227569Sphilip 623280501Sarybchik return (error); 624227569Sphilip} 625227569Sphilip 626227569Sphilipstatic boolean_t 627227569Sphilipsfxge_ev_initialized(void *arg) 628227569Sphilip{ 629227569Sphilip struct sfxge_evq *evq; 630280501Sarybchik 631227569Sphilip evq = (struct sfxge_evq *)arg; 632280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 633227569Sphilip 634284555Sarybchik /* Init done events may be duplicated on 7xxx */ 635284555Sarybchik KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 636284555Sarybchik evq->init_state == SFXGE_EVQ_STARTED, 637227569Sphilip ("evq not starting")); 638227569Sphilip 639227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 640227569Sphilip 641227569Sphilip return (0); 642227569Sphilip} 643227569Sphilip 644227569Sphilipstatic boolean_t 645227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 646227569Sphilip{ 647227569Sphilip struct sfxge_evq *evq; 648227569Sphilip struct sfxge_softc *sc; 649227569Sphilip 650227569Sphilip evq = (struct sfxge_evq *)arg; 651280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 652280582Sarybchik 653227569Sphilip sc = evq->sc; 654227569Sphilip 655227569Sphilip sfxge_mac_link_update(sc, link_mode); 656227569Sphilip 657227569Sphilip return (0); 658227569Sphilip} 659227569Sphilip 660227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 661227569Sphilip .eec_initialized = sfxge_ev_initialized, 662227569Sphilip .eec_rx = sfxge_ev_rx, 663227569Sphilip .eec_tx = sfxge_ev_tx, 664227569Sphilip .eec_exception = sfxge_ev_exception, 665227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 666227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 667227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 668227569Sphilip .eec_software = sfxge_ev_software, 669227569Sphilip .eec_sram = sfxge_ev_sram, 670227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 671227569Sphilip .eec_timer = sfxge_ev_timer, 672227569Sphilip .eec_link_change = sfxge_ev_link_change, 673227569Sphilip}; 674227569Sphilip 675227569Sphilip 676227569Sphilipint 677280508Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 678227569Sphilip{ 679227569Sphilip int rc; 680227569Sphilip 681280522Sarybchik SFXGE_EVQ_LOCK(evq); 682227569Sphilip 683280596Sarybchik if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 684280596Sarybchik evq->init_state != SFXGE_EVQ_STARTED)) { 685227569Sphilip rc = EINVAL; 686227569Sphilip goto fail; 687227569Sphilip } 688227569Sphilip 689227569Sphilip /* Synchronize the DMA memory for reading */ 690227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 691227569Sphilip BUS_DMASYNC_POSTREAD); 692227569Sphilip 693227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 694227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 695227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 696227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 697227569Sphilip 698227569Sphilip /* Poll the queue */ 699227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 700227569Sphilip 701227569Sphilip evq->rx_done = 0; 702227569Sphilip evq->tx_done = 0; 703227569Sphilip 704227569Sphilip /* Perform any pending completion processing */ 705227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 706227569Sphilip 707227569Sphilip /* Re-prime the event queue for interrupts */ 708227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 709227569Sphilip goto fail; 710227569Sphilip 711280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 712227569Sphilip 713227569Sphilip return (0); 714227569Sphilip 715227569Sphilipfail: 716280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 717227569Sphilip return (rc); 718227569Sphilip} 719227569Sphilip 720227569Sphilipstatic void 721227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 722227569Sphilip{ 723227569Sphilip struct sfxge_evq *evq; 724227569Sphilip 725227569Sphilip evq = sc->evq[index]; 726227569Sphilip 727227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 728227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 729227569Sphilip 730280522Sarybchik SFXGE_EVQ_LOCK(evq); 731227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 732227569Sphilip evq->read_ptr = 0; 733227569Sphilip evq->exception = B_FALSE; 734227569Sphilip 735280510Sarybchik#if EFSYS_OPT_QSTATS 736227569Sphilip /* Add event counts before discarding the common evq state */ 737342523Sarybchik efx_ev_qstats_update(evq->common, evq->stats); 738280510Sarybchik#endif 739227569Sphilip 740227569Sphilip efx_ev_qdestroy(evq->common); 741227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 742280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 743280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 744227569Sphilip} 745227569Sphilip 746227569Sphilipstatic int 747227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 748227569Sphilip{ 749227569Sphilip struct sfxge_evq *evq; 750227569Sphilip efsys_mem_t *esmp; 751227569Sphilip int count; 752227569Sphilip int rc; 753227569Sphilip 754227569Sphilip evq = sc->evq[index]; 755227569Sphilip esmp = &evq->mem; 756227569Sphilip 757227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 758227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 759227569Sphilip 760227569Sphilip /* Clear all events. */ 761280502Sarybchik (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 762227569Sphilip 763227569Sphilip /* Program the buffer table. */ 764227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 765280502Sarybchik EFX_EVQ_NBUFS(evq->entries))) != 0) 766280502Sarybchik return (rc); 767227569Sphilip 768227569Sphilip /* Create the common code event queue. */ 769280502Sarybchik if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 770311070Sarybchik evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 771311070Sarybchik &evq->common)) != 0) 772227569Sphilip goto fail; 773227569Sphilip 774280522Sarybchik SFXGE_EVQ_LOCK(evq); 775227569Sphilip 776227569Sphilip /* Prime the event queue for interrupts */ 777227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 778227569Sphilip goto fail2; 779227569Sphilip 780227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 781227569Sphilip 782280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 783227569Sphilip 784227569Sphilip /* Wait for the initialization event */ 785227569Sphilip count = 0; 786227569Sphilip do { 787227569Sphilip /* Pause for 100 ms */ 788227569Sphilip pause("sfxge evq init", hz / 10); 789227569Sphilip 790227569Sphilip /* Check to see if the test event has been processed */ 791227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 792227569Sphilip goto done; 793227569Sphilip 794227569Sphilip } while (++count < 20); 795227569Sphilip 796227569Sphilip rc = ETIMEDOUT; 797227569Sphilip goto fail3; 798227569Sphilip 799227569Sphilipdone: 800227569Sphilip return (0); 801227569Sphilip 802227569Sphilipfail3: 803280522Sarybchik SFXGE_EVQ_LOCK(evq); 804227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 805227569Sphilipfail2: 806280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 807227569Sphilip efx_ev_qdestroy(evq->common); 808227569Sphilipfail: 809227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 810280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 811227569Sphilip 812227569Sphilip return (rc); 813227569Sphilip} 814227569Sphilip 815227569Sphilipvoid 816227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 817227569Sphilip{ 818227569Sphilip struct sfxge_intr *intr; 819227569Sphilip efx_nic_t *enp; 820227569Sphilip int index; 821227569Sphilip 822227569Sphilip intr = &sc->intr; 823227569Sphilip enp = sc->enp; 824227569Sphilip 825227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 826227569Sphilip ("Interrupts not started")); 827227569Sphilip 828227569Sphilip /* Stop the event queue(s) */ 829280541Sarybchik index = sc->evq_count; 830227569Sphilip while (--index >= 0) 831227569Sphilip sfxge_ev_qstop(sc, index); 832227569Sphilip 833227569Sphilip /* Tear down the event module */ 834227569Sphilip efx_ev_fini(enp); 835227569Sphilip} 836227569Sphilip 837227569Sphilipint 838227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 839227569Sphilip{ 840227569Sphilip struct sfxge_intr *intr; 841227569Sphilip int index; 842227569Sphilip int rc; 843227569Sphilip 844227569Sphilip intr = &sc->intr; 845227569Sphilip 846227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 847227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 848227569Sphilip 849227569Sphilip /* Initialize the event module */ 850227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 851280501Sarybchik return (rc); 852227569Sphilip 853227569Sphilip /* Start the event queues */ 854280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 855227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 856227569Sphilip goto fail; 857227569Sphilip } 858227569Sphilip 859227569Sphilip return (0); 860227569Sphilip 861227569Sphilipfail: 862227569Sphilip /* Stop the event queue(s) */ 863227569Sphilip while (--index >= 0) 864227569Sphilip sfxge_ev_qstop(sc, index); 865227569Sphilip 866227569Sphilip /* Tear down the event module */ 867227569Sphilip efx_ev_fini(sc->enp); 868227569Sphilip 869227569Sphilip return (rc); 870227569Sphilip} 871227569Sphilip 872227569Sphilipstatic void 873227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 874227569Sphilip{ 875227569Sphilip struct sfxge_evq *evq; 876227569Sphilip 877227569Sphilip evq = sc->evq[index]; 878227569Sphilip 879227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 880227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 881227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 882227569Sphilip 883227569Sphilip sfxge_dma_free(&evq->mem); 884227569Sphilip 885227569Sphilip sc->evq[index] = NULL; 886227569Sphilip 887280522Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 888227569Sphilip 889227569Sphilip free(evq, M_SFXGE); 890227569Sphilip} 891227569Sphilip 892227569Sphilipstatic int 893227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 894227569Sphilip{ 895227569Sphilip struct sfxge_evq *evq; 896227569Sphilip efsys_mem_t *esmp; 897227569Sphilip int rc; 898227569Sphilip 899227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 900227569Sphilip 901227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 902227569Sphilip evq->sc = sc; 903227569Sphilip evq->index = index; 904227569Sphilip sc->evq[index] = evq; 905227569Sphilip esmp = &evq->mem; 906227569Sphilip 907280502Sarybchik /* Build an event queue with room for one event per tx and rx buffer, 908280502Sarybchik * plus some extra for link state events and MCDI completions. 909280502Sarybchik * There are three tx queues in the first event queue and one in 910280502Sarybchik * other. 911280502Sarybchik */ 912280502Sarybchik if (index == 0) 913280502Sarybchik evq->entries = 914280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 915280502Sarybchik 3 * sc->txq_entries + 916280502Sarybchik 128); 917280502Sarybchik else 918280502Sarybchik evq->entries = 919280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 920280502Sarybchik sc->txq_entries + 921280502Sarybchik 128); 922280502Sarybchik 923227569Sphilip /* Initialise TX completion list */ 924227569Sphilip evq->txqs = &evq->txq; 925227569Sphilip 926227569Sphilip /* Allocate DMA space. */ 927280502Sarybchik if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 928227569Sphilip return (rc); 929227569Sphilip 930227569Sphilip /* Allocate buffer table entries. */ 931280502Sarybchik sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 932227569Sphilip &evq->buf_base_id); 933227569Sphilip 934280524Sarybchik SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 935227569Sphilip 936227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 937227569Sphilip 938342523Sarybchik#if EFSYS_OPT_QSTATS 939342523Sarybchik rc = sfxge_evq_stat_init(evq); 940342523Sarybchik if (rc != 0) 941342523Sarybchik goto fail_evq_stat_init; 942342523Sarybchik#endif 943342523Sarybchik 944227569Sphilip return (0); 945342523Sarybchik 946342523Sarybchik#if EFSYS_OPT_QSTATS 947342523Sarybchikfail_evq_stat_init: 948342523Sarybchik evq->init_state = SFXGE_EVQ_UNINITIALIZED; 949342523Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 950342523Sarybchik sfxge_dma_free(esmp); 951342523Sarybchik sc->evq[index] = NULL; 952342523Sarybchik free(evq, M_SFXGE); 953342523Sarybchik 954342523Sarybchik return (rc); 955342523Sarybchik#endif 956227569Sphilip} 957227569Sphilip 958227569Sphilipvoid 959227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 960227569Sphilip{ 961227569Sphilip struct sfxge_intr *intr; 962227569Sphilip int index; 963227569Sphilip 964227569Sphilip intr = &sc->intr; 965227569Sphilip 966227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 967227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 968227569Sphilip 969227569Sphilip sc->ev_moderation = 0; 970227569Sphilip 971227569Sphilip /* Tear down the event queue(s). */ 972280541Sarybchik index = sc->evq_count; 973227569Sphilip while (--index >= 0) 974227569Sphilip sfxge_ev_qfini(sc, index); 975280541Sarybchik 976280541Sarybchik sc->evq_count = 0; 977227569Sphilip} 978227569Sphilip 979227569Sphilipint 980227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 981227569Sphilip{ 982227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 983227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 984227569Sphilip struct sfxge_intr *intr; 985227569Sphilip int index; 986227569Sphilip int rc; 987227569Sphilip 988227569Sphilip intr = &sc->intr; 989227569Sphilip 990280541Sarybchik sc->evq_count = intr->n_alloc; 991280541Sarybchik 992227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 993227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 994227569Sphilip 995227569Sphilip /* Set default interrupt moderation; add a sysctl to 996227569Sphilip * read and change it. 997227569Sphilip */ 998280517Sarybchik sc->ev_moderation = SFXGE_MODERATION; 999227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1000227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 1001227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 1002227569Sphilip "sfxge interrupt moderation (us)"); 1003227569Sphilip 1004342523Sarybchik#if EFSYS_OPT_QSTATS 1005342523Sarybchik sc->evqs_stats_node = SYSCTL_ADD_NODE( 1006342523Sarybchik device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1007342523Sarybchik OID_AUTO, "evq", CTLFLAG_RD, NULL, "Event queues stats"); 1008342523Sarybchik if (sc->evqs_stats_node == NULL) { 1009342523Sarybchik rc = ENOMEM; 1010342523Sarybchik goto fail_evqs_stats_node; 1011342523Sarybchik } 1012342523Sarybchik#endif 1013342523Sarybchik 1014227569Sphilip /* 1015227569Sphilip * Initialize the event queue(s) - one per interrupt. 1016227569Sphilip */ 1017280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 1018227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1019227569Sphilip goto fail; 1020227569Sphilip } 1021227569Sphilip 1022280510Sarybchik#if EFSYS_OPT_QSTATS 1023227569Sphilip sfxge_ev_stat_init(sc); 1024280510Sarybchik#endif 1025227569Sphilip 1026227569Sphilip return (0); 1027227569Sphilip 1028227569Sphilipfail: 1029227569Sphilip while (--index >= 0) 1030227569Sphilip sfxge_ev_qfini(sc, index); 1031227569Sphilip 1032342523Sarybchik#if EFSYS_OPT_QSTATS 1033342523Sarybchikfail_evqs_stats_node: 1034342523Sarybchik#endif 1035280541Sarybchik sc->evq_count = 0; 1036227569Sphilip return (rc); 1037227569Sphilip} 1038