sfxge_ev.c revision 257179
1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_ev.c 257179 2013-10-26 18:18:50Z glebius $"); 32227569Sphilip 33227569Sphilip#include <sys/param.h> 34257179Sglebius#include <sys/kernel.h> 35257179Sglebius#include <sys/malloc.h> 36257179Sglebius#include <sys/param.h> 37257179Sglebius#include <sys/queue.h> 38227569Sphilip#include <sys/systm.h> 39257179Sglebius#include <sys/taskqueue.h> 40227569Sphilip 41227569Sphilip#include "common/efx.h" 42227569Sphilip 43227569Sphilip#include "sfxge.h" 44227569Sphilip 45227569Sphilipstatic void 46227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 47227569Sphilip{ 48227569Sphilip struct sfxge_softc *sc; 49227569Sphilip unsigned int index; 50227569Sphilip struct sfxge_rxq *rxq; 51227569Sphilip struct sfxge_txq *txq; 52227569Sphilip 53227569Sphilip sc = evq->sc; 54227569Sphilip index = evq->index; 55227569Sphilip rxq = sc->rxq[index]; 56227569Sphilip 57227569Sphilip if ((txq = evq->txq) != NULL) { 58227569Sphilip evq->txq = NULL; 59227569Sphilip evq->txqs = &(evq->txq); 60227569Sphilip 61227569Sphilip do { 62227569Sphilip struct sfxge_txq *next; 63227569Sphilip 64227569Sphilip next = txq->next; 65227569Sphilip txq->next = NULL; 66227569Sphilip 67227569Sphilip KASSERT(txq->evq_index == index, 68227569Sphilip ("txq->evq_index != index")); 69227569Sphilip 70227569Sphilip if (txq->pending != txq->completed) 71227569Sphilip sfxge_tx_qcomplete(txq); 72227569Sphilip 73227569Sphilip txq = next; 74227569Sphilip } while (txq != NULL); 75227569Sphilip } 76227569Sphilip 77227569Sphilip if (rxq->pending != rxq->completed) 78227569Sphilip sfxge_rx_qcomplete(rxq, eop); 79227569Sphilip} 80227569Sphilip 81227569Sphilipstatic boolean_t 82227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 83227569Sphilip uint16_t flags) 84227569Sphilip{ 85227569Sphilip struct sfxge_evq *evq; 86227569Sphilip struct sfxge_softc *sc; 87227569Sphilip struct sfxge_rxq *rxq; 88227569Sphilip unsigned int expected; 89227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 90227569Sphilip 91227569Sphilip evq = arg; 92227569Sphilip sc = evq->sc; 93227569Sphilip 94227569Sphilip if (evq->exception) 95227569Sphilip goto done; 96227569Sphilip 97227569Sphilip rxq = sc->rxq[label]; 98227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 99227569Sphilip KASSERT(evq->index == rxq->index, 100227569Sphilip ("evq->index != rxq->index")); 101227569Sphilip 102227569Sphilip if (rxq->init_state != SFXGE_RXQ_STARTED) 103227569Sphilip goto done; 104227569Sphilip 105227569Sphilip expected = rxq->pending++ & (SFXGE_NDESCS - 1); 106227569Sphilip if (id != expected) { 107227569Sphilip evq->exception = B_TRUE; 108227569Sphilip 109227569Sphilip device_printf(sc->dev, "RX completion out of order" 110227569Sphilip " (id=%#x expected=%#x flags=%#x); resetting\n", 111227569Sphilip id, expected, flags); 112227569Sphilip sfxge_schedule_reset(sc); 113227569Sphilip 114227569Sphilip goto done; 115227569Sphilip } 116227569Sphilip 117227569Sphilip rx_desc = &rxq->queue[id]; 118227569Sphilip 119227569Sphilip KASSERT(rx_desc->flags == EFX_DISCARD, 120227569Sphilip ("rx_desc->flags != EFX_DISCARD")); 121227569Sphilip rx_desc->flags = flags; 122227569Sphilip 123227569Sphilip KASSERT(size < (1 << 16), ("size > (1 << 16)")); 124227569Sphilip rx_desc->size = (uint16_t)size; 125227569Sphilip prefetch_read_many(rx_desc->mbuf); 126227569Sphilip 127227569Sphilip evq->rx_done++; 128227569Sphilip 129227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 130227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 131227569Sphilip 132227569Sphilipdone: 133227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 134227569Sphilip} 135227569Sphilip 136227569Sphilipstatic boolean_t 137227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 138227569Sphilip{ 139227569Sphilip struct sfxge_evq *evq; 140227569Sphilip struct sfxge_softc *sc; 141227569Sphilip 142227569Sphilip evq = (struct sfxge_evq *)arg; 143227569Sphilip sc = evq->sc; 144227569Sphilip 145227569Sphilip evq->exception = B_TRUE; 146227569Sphilip 147227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 148227569Sphilip device_printf(sc->dev, 149227569Sphilip "hardware exception (code=%u); resetting\n", 150227569Sphilip code); 151227569Sphilip sfxge_schedule_reset(sc); 152227569Sphilip } 153227569Sphilip 154227569Sphilip return (B_FALSE); 155227569Sphilip} 156227569Sphilip 157227569Sphilipstatic boolean_t 158227569Sphilipsfxge_ev_rxq_flush_done(void *arg, uint32_t label) 159227569Sphilip{ 160227569Sphilip struct sfxge_evq *evq; 161227569Sphilip struct sfxge_softc *sc; 162227569Sphilip struct sfxge_rxq *rxq; 163227569Sphilip unsigned int index; 164227569Sphilip uint16_t magic; 165227569Sphilip 166227569Sphilip evq = (struct sfxge_evq *)arg; 167227569Sphilip sc = evq->sc; 168227569Sphilip rxq = sc->rxq[label]; 169227569Sphilip 170227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 171227569Sphilip 172227569Sphilip /* Resend a software event on the correct queue */ 173227569Sphilip index = rxq->index; 174227569Sphilip evq = sc->evq[index]; 175227569Sphilip 176227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 177227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 178227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 179227569Sphilip 180227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 181227569Sphilip ("evq not started")); 182227569Sphilip efx_ev_qpost(evq->common, magic); 183227569Sphilip 184227569Sphilip return (B_FALSE); 185227569Sphilip} 186227569Sphilip 187227569Sphilipstatic boolean_t 188227569Sphilipsfxge_ev_rxq_flush_failed(void *arg, uint32_t label) 189227569Sphilip{ 190227569Sphilip struct sfxge_evq *evq; 191227569Sphilip struct sfxge_softc *sc; 192227569Sphilip struct sfxge_rxq *rxq; 193227569Sphilip unsigned int index; 194227569Sphilip uint16_t magic; 195227569Sphilip 196227569Sphilip evq = (struct sfxge_evq *)arg; 197227569Sphilip sc = evq->sc; 198227569Sphilip rxq = sc->rxq[label]; 199227569Sphilip 200227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 201227569Sphilip 202227569Sphilip /* Resend a software event on the correct queue */ 203227569Sphilip index = rxq->index; 204227569Sphilip evq = sc->evq[index]; 205227569Sphilip 206227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 207227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 208227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 209227569Sphilip 210227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 211227569Sphilip ("evq not started")); 212227569Sphilip efx_ev_qpost(evq->common, magic); 213227569Sphilip 214227569Sphilip return (B_FALSE); 215227569Sphilip} 216227569Sphilip 217227569Sphilipstatic boolean_t 218227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 219227569Sphilip{ 220227569Sphilip struct sfxge_evq *evq; 221227569Sphilip struct sfxge_softc *sc; 222227569Sphilip struct sfxge_txq *txq; 223227569Sphilip unsigned int stop; 224227569Sphilip unsigned int delta; 225227569Sphilip 226227569Sphilip evq = (struct sfxge_evq *)arg; 227227569Sphilip sc = evq->sc; 228227569Sphilip txq = sc->txq[label]; 229227569Sphilip 230227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 231227569Sphilip KASSERT(evq->index == txq->evq_index, 232227569Sphilip ("evq->index != txq->evq_index")); 233227569Sphilip 234227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 235227569Sphilip goto done; 236227569Sphilip 237227569Sphilip stop = (id + 1) & (SFXGE_NDESCS - 1); 238227569Sphilip id = txq->pending & (SFXGE_NDESCS - 1); 239227569Sphilip 240227569Sphilip delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop); 241227569Sphilip txq->pending += delta; 242227569Sphilip 243227569Sphilip evq->tx_done++; 244227569Sphilip 245227569Sphilip if (txq->next == NULL && 246227569Sphilip evq->txqs != &(txq->next)) { 247227569Sphilip *(evq->txqs) = txq; 248227569Sphilip evq->txqs = &(txq->next); 249227569Sphilip } 250227569Sphilip 251227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 252227569Sphilip sfxge_tx_qcomplete(txq); 253227569Sphilip 254227569Sphilipdone: 255227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 256227569Sphilip} 257227569Sphilip 258227569Sphilipstatic boolean_t 259227569Sphilipsfxge_ev_txq_flush_done(void *arg, uint32_t label) 260227569Sphilip{ 261227569Sphilip struct sfxge_evq *evq; 262227569Sphilip struct sfxge_softc *sc; 263227569Sphilip struct sfxge_txq *txq; 264227569Sphilip uint16_t magic; 265227569Sphilip 266227569Sphilip evq = (struct sfxge_evq *)arg; 267227569Sphilip sc = evq->sc; 268227569Sphilip txq = sc->txq[label]; 269227569Sphilip 270227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 271227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 272227569Sphilip ("txq not initialized")); 273227569Sphilip 274227569Sphilip /* Resend a software event on the correct queue */ 275227569Sphilip evq = sc->evq[txq->evq_index]; 276227569Sphilip 277227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 278227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 279227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 280227569Sphilip 281227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 282227569Sphilip ("evq not started")); 283227569Sphilip efx_ev_qpost(evq->common, magic); 284227569Sphilip 285227569Sphilip return (B_FALSE); 286227569Sphilip} 287227569Sphilip 288227569Sphilipstatic boolean_t 289227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 290227569Sphilip{ 291227569Sphilip struct sfxge_evq *evq; 292227569Sphilip struct sfxge_softc *sc; 293227569Sphilip unsigned int label; 294227569Sphilip 295227569Sphilip evq = (struct sfxge_evq *)arg; 296227569Sphilip sc = evq->sc; 297227569Sphilip 298227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 299227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 300227569Sphilip 301227569Sphilip switch (magic) { 302227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 303227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 304227569Sphilip 305227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 306227569Sphilip KASSERT(evq->index == rxq->index, 307227569Sphilip ("evq->index != rxq->index")); 308227569Sphilip 309227569Sphilip sfxge_rx_qflush_done(rxq); 310227569Sphilip break; 311227569Sphilip } 312227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 313227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 314227569Sphilip 315227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 316227569Sphilip KASSERT(evq->index == rxq->index, 317227569Sphilip ("evq->index != rxq->index")); 318227569Sphilip 319227569Sphilip sfxge_rx_qflush_failed(rxq); 320227569Sphilip break; 321227569Sphilip } 322227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 323227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 324227569Sphilip 325227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 326227569Sphilip KASSERT(evq->index == rxq->index, 327227569Sphilip ("evq->index != rxq->index")); 328227569Sphilip 329227569Sphilip sfxge_rx_qrefill(rxq); 330227569Sphilip break; 331227569Sphilip } 332227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 333227569Sphilip struct sfxge_txq *txq = sc->txq[label]; 334227569Sphilip 335227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 336227569Sphilip KASSERT(evq->index == txq->evq_index, 337227569Sphilip ("evq->index != txq->evq_index")); 338227569Sphilip 339227569Sphilip sfxge_tx_qflush_done(txq); 340227569Sphilip break; 341227569Sphilip } 342227569Sphilip default: 343227569Sphilip break; 344227569Sphilip } 345227569Sphilip 346227569Sphilip return (B_FALSE); 347227569Sphilip} 348227569Sphilip 349227569Sphilipstatic boolean_t 350227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 351227569Sphilip{ 352227569Sphilip (void)arg; 353227569Sphilip (void)code; 354227569Sphilip 355227569Sphilip switch (code) { 356227569Sphilip case EFX_SRAM_UPDATE: 357227569Sphilip EFSYS_PROBE(sram_update); 358227569Sphilip break; 359227569Sphilip 360227569Sphilip case EFX_SRAM_CLEAR: 361227569Sphilip EFSYS_PROBE(sram_clear); 362227569Sphilip break; 363227569Sphilip 364227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 365227569Sphilip EFSYS_PROBE(sram_illegal_clear); 366227569Sphilip break; 367227569Sphilip 368227569Sphilip default: 369227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 370227569Sphilip break; 371227569Sphilip } 372227569Sphilip 373227569Sphilip return (B_FALSE); 374227569Sphilip} 375227569Sphilip 376227569Sphilipstatic boolean_t 377227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 378227569Sphilip{ 379227569Sphilip (void)arg; 380227569Sphilip (void)index; 381227569Sphilip 382227569Sphilip return (B_FALSE); 383227569Sphilip} 384227569Sphilip 385227569Sphilipstatic boolean_t 386227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 387227569Sphilip{ 388227569Sphilip (void)arg; 389227569Sphilip (void)index; 390227569Sphilip 391227569Sphilip return (B_FALSE); 392227569Sphilip} 393227569Sphilip 394227569Sphilipstatic void 395227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 396227569Sphilip{ 397227569Sphilip struct sfxge_evq *evq; 398227569Sphilip unsigned int index; 399227569Sphilip clock_t now; 400227569Sphilip 401227569Sphilip sx_xlock(&sc->softc_lock); 402227569Sphilip 403227569Sphilip if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 404227569Sphilip goto out; 405227569Sphilip 406227569Sphilip now = ticks; 407227569Sphilip if (now - sc->ev_stats_update_time < hz) 408227569Sphilip goto out; 409227569Sphilip 410227569Sphilip sc->ev_stats_update_time = now; 411227569Sphilip 412227569Sphilip /* Add event counts from each event queue in turn */ 413227569Sphilip for (index = 0; index < sc->intr.n_alloc; index++) { 414227569Sphilip evq = sc->evq[index]; 415227569Sphilip mtx_lock(&evq->lock); 416227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 417227569Sphilip mtx_unlock(&evq->lock); 418227569Sphilip } 419227569Sphilipout: 420227569Sphilip sx_xunlock(&sc->softc_lock); 421227569Sphilip} 422227569Sphilip 423227569Sphilipstatic int 424227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 425227569Sphilip{ 426227569Sphilip struct sfxge_softc *sc = arg1; 427227569Sphilip unsigned int id = arg2; 428227569Sphilip 429227569Sphilip sfxge_ev_stat_update(sc); 430227569Sphilip 431227569Sphilip return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])); 432227569Sphilip} 433227569Sphilip 434227569Sphilipstatic void 435227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 436227569Sphilip{ 437227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 438227569Sphilip struct sysctl_oid_list *stat_list; 439227569Sphilip unsigned int id; 440227569Sphilip char name[40]; 441227569Sphilip 442227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 443227569Sphilip 444227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 445227569Sphilip snprintf(name, sizeof(name), "ev_%s", 446227569Sphilip efx_ev_qstat_name(sc->enp, id)); 447227569Sphilip SYSCTL_ADD_PROC( 448227569Sphilip ctx, stat_list, 449227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 450227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 451227569Sphilip ""); 452227569Sphilip } 453227569Sphilip} 454227569Sphilip 455227569Sphilipstatic void 456227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 457227569Sphilip{ 458227569Sphilip struct sfxge_evq *evq; 459227569Sphilip efx_evq_t *eep; 460227569Sphilip 461227569Sphilip evq = sc->evq[idx]; 462227569Sphilip eep = evq->common; 463227569Sphilip 464227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 465227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 466227569Sphilip 467227569Sphilip (void)efx_ev_qmoderate(eep, us); 468227569Sphilip} 469227569Sphilip 470227569Sphilipstatic int 471227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 472227569Sphilip{ 473227569Sphilip struct sfxge_softc *sc = arg1; 474227569Sphilip struct sfxge_intr *intr = &sc->intr; 475227569Sphilip unsigned int moderation; 476227569Sphilip int error; 477227569Sphilip int index; 478227569Sphilip 479227569Sphilip sx_xlock(&sc->softc_lock); 480227569Sphilip 481227569Sphilip if (req->newptr) { 482227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 483227569Sphilip != 0) 484227569Sphilip goto out; 485227569Sphilip 486227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 487227569Sphilip * so we have to range-check the value ourselves. 488227569Sphilip */ 489227569Sphilip if (moderation > 490227569Sphilip efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 491227569Sphilip error = EINVAL; 492227569Sphilip goto out; 493227569Sphilip } 494227569Sphilip 495227569Sphilip sc->ev_moderation = moderation; 496227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 497227569Sphilip for (index = 0; index < intr->n_alloc; index++) 498227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 499227569Sphilip } 500227569Sphilip } else { 501227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 502227569Sphilip sizeof(sc->ev_moderation)); 503227569Sphilip } 504227569Sphilip 505227569Sphilipout: 506227569Sphilip sx_xunlock(&sc->softc_lock); 507227569Sphilip 508227569Sphilip return error; 509227569Sphilip} 510227569Sphilip 511227569Sphilipstatic boolean_t 512227569Sphilipsfxge_ev_initialized(void *arg) 513227569Sphilip{ 514227569Sphilip struct sfxge_evq *evq; 515227569Sphilip 516227569Sphilip evq = (struct sfxge_evq *)arg; 517227569Sphilip 518227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 519227569Sphilip ("evq not starting")); 520227569Sphilip 521227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 522227569Sphilip 523227569Sphilip return (0); 524227569Sphilip} 525227569Sphilip 526227569Sphilipstatic boolean_t 527227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 528227569Sphilip{ 529227569Sphilip struct sfxge_evq *evq; 530227569Sphilip struct sfxge_softc *sc; 531227569Sphilip 532227569Sphilip evq = (struct sfxge_evq *)arg; 533227569Sphilip sc = evq->sc; 534227569Sphilip 535227569Sphilip sfxge_mac_link_update(sc, link_mode); 536227569Sphilip 537227569Sphilip return (0); 538227569Sphilip} 539227569Sphilip 540227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 541227569Sphilip .eec_initialized = sfxge_ev_initialized, 542227569Sphilip .eec_rx = sfxge_ev_rx, 543227569Sphilip .eec_tx = sfxge_ev_tx, 544227569Sphilip .eec_exception = sfxge_ev_exception, 545227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 546227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 547227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 548227569Sphilip .eec_software = sfxge_ev_software, 549227569Sphilip .eec_sram = sfxge_ev_sram, 550227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 551227569Sphilip .eec_timer = sfxge_ev_timer, 552227569Sphilip .eec_link_change = sfxge_ev_link_change, 553227569Sphilip}; 554227569Sphilip 555227569Sphilip 556227569Sphilipint 557227569Sphilipsfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index) 558227569Sphilip{ 559227569Sphilip struct sfxge_evq *evq; 560227569Sphilip int rc; 561227569Sphilip 562227569Sphilip evq = sc->evq[index]; 563227569Sphilip 564227569Sphilip mtx_lock(&evq->lock); 565227569Sphilip 566227569Sphilip if (evq->init_state != SFXGE_EVQ_STARTING && 567227569Sphilip evq->init_state != SFXGE_EVQ_STARTED) { 568227569Sphilip rc = EINVAL; 569227569Sphilip goto fail; 570227569Sphilip } 571227569Sphilip 572227569Sphilip /* Synchronize the DMA memory for reading */ 573227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 574227569Sphilip BUS_DMASYNC_POSTREAD); 575227569Sphilip 576227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 577227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 578227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 579227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 580227569Sphilip 581227569Sphilip /* Poll the queue */ 582227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 583227569Sphilip 584227569Sphilip evq->rx_done = 0; 585227569Sphilip evq->tx_done = 0; 586227569Sphilip 587227569Sphilip /* Perform any pending completion processing */ 588227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 589227569Sphilip 590227569Sphilip /* Re-prime the event queue for interrupts */ 591227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 592227569Sphilip goto fail; 593227569Sphilip 594227569Sphilip mtx_unlock(&evq->lock); 595227569Sphilip 596227569Sphilip return (0); 597227569Sphilip 598227569Sphilipfail: 599227569Sphilip mtx_unlock(&(evq->lock)); 600227569Sphilip return (rc); 601227569Sphilip} 602227569Sphilip 603227569Sphilipstatic void 604227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 605227569Sphilip{ 606227569Sphilip struct sfxge_evq *evq; 607227569Sphilip 608227569Sphilip evq = sc->evq[index]; 609227569Sphilip 610227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 611227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 612227569Sphilip 613227569Sphilip mtx_lock(&evq->lock); 614227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 615227569Sphilip evq->read_ptr = 0; 616227569Sphilip evq->exception = B_FALSE; 617227569Sphilip 618227569Sphilip /* Add event counts before discarding the common evq state */ 619227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 620227569Sphilip 621227569Sphilip efx_ev_qdestroy(evq->common); 622227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 623227569Sphilip EFX_EVQ_NBUFS(SFXGE_NEVS)); 624227569Sphilip mtx_unlock(&evq->lock); 625227569Sphilip} 626227569Sphilip 627227569Sphilipstatic int 628227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 629227569Sphilip{ 630227569Sphilip struct sfxge_evq *evq; 631227569Sphilip efsys_mem_t *esmp; 632227569Sphilip int count; 633227569Sphilip int rc; 634227569Sphilip 635227569Sphilip evq = sc->evq[index]; 636227569Sphilip esmp = &evq->mem; 637227569Sphilip 638227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 639227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 640227569Sphilip 641227569Sphilip /* Clear all events. */ 642227569Sphilip (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS)); 643227569Sphilip 644227569Sphilip /* Program the buffer table. */ 645227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 646227569Sphilip EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0) 647227569Sphilip return rc; 648227569Sphilip 649227569Sphilip /* Create the common code event queue. */ 650227569Sphilip if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS, 651227569Sphilip evq->buf_base_id, &evq->common)) != 0) 652227569Sphilip goto fail; 653227569Sphilip 654227569Sphilip mtx_lock(&evq->lock); 655227569Sphilip 656227569Sphilip /* Set the default moderation */ 657227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 658227569Sphilip 659227569Sphilip /* Prime the event queue for interrupts */ 660227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 661227569Sphilip goto fail2; 662227569Sphilip 663227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 664227569Sphilip 665227569Sphilip mtx_unlock(&evq->lock); 666227569Sphilip 667227569Sphilip /* Wait for the initialization event */ 668227569Sphilip count = 0; 669227569Sphilip do { 670227569Sphilip /* Pause for 100 ms */ 671227569Sphilip pause("sfxge evq init", hz / 10); 672227569Sphilip 673227569Sphilip /* Check to see if the test event has been processed */ 674227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 675227569Sphilip goto done; 676227569Sphilip 677227569Sphilip } while (++count < 20); 678227569Sphilip 679227569Sphilip rc = ETIMEDOUT; 680227569Sphilip goto fail3; 681227569Sphilip 682227569Sphilipdone: 683227569Sphilip return (0); 684227569Sphilip 685227569Sphilipfail3: 686227569Sphilip mtx_lock(&evq->lock); 687227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 688227569Sphilipfail2: 689227569Sphilip mtx_unlock(&evq->lock); 690227569Sphilip efx_ev_qdestroy(evq->common); 691227569Sphilipfail: 692227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 693227569Sphilip EFX_EVQ_NBUFS(SFXGE_NEVS)); 694227569Sphilip 695227569Sphilip return (rc); 696227569Sphilip} 697227569Sphilip 698227569Sphilipvoid 699227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 700227569Sphilip{ 701227569Sphilip struct sfxge_intr *intr; 702227569Sphilip efx_nic_t *enp; 703227569Sphilip int index; 704227569Sphilip 705227569Sphilip intr = &sc->intr; 706227569Sphilip enp = sc->enp; 707227569Sphilip 708227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 709227569Sphilip ("Interrupts not started")); 710227569Sphilip 711227569Sphilip /* Stop the event queue(s) */ 712227569Sphilip index = intr->n_alloc; 713227569Sphilip while (--index >= 0) 714227569Sphilip sfxge_ev_qstop(sc, index); 715227569Sphilip 716227569Sphilip /* Tear down the event module */ 717227569Sphilip efx_ev_fini(enp); 718227569Sphilip} 719227569Sphilip 720227569Sphilipint 721227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 722227569Sphilip{ 723227569Sphilip struct sfxge_intr *intr; 724227569Sphilip int index; 725227569Sphilip int rc; 726227569Sphilip 727227569Sphilip intr = &sc->intr; 728227569Sphilip 729227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 730227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 731227569Sphilip 732227569Sphilip /* Initialize the event module */ 733227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 734227569Sphilip return rc; 735227569Sphilip 736227569Sphilip /* Start the event queues */ 737227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 738227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 739227569Sphilip goto fail; 740227569Sphilip } 741227569Sphilip 742227569Sphilip return (0); 743227569Sphilip 744227569Sphilipfail: 745227569Sphilip /* Stop the event queue(s) */ 746227569Sphilip while (--index >= 0) 747227569Sphilip sfxge_ev_qstop(sc, index); 748227569Sphilip 749227569Sphilip /* Tear down the event module */ 750227569Sphilip efx_ev_fini(sc->enp); 751227569Sphilip 752227569Sphilip return (rc); 753227569Sphilip} 754227569Sphilip 755227569Sphilipstatic void 756227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 757227569Sphilip{ 758227569Sphilip struct sfxge_evq *evq; 759227569Sphilip 760227569Sphilip evq = sc->evq[index]; 761227569Sphilip 762227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 763227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 764227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 765227569Sphilip 766227569Sphilip sfxge_dma_free(&evq->mem); 767227569Sphilip 768227569Sphilip sc->evq[index] = NULL; 769227569Sphilip 770227569Sphilip mtx_destroy(&evq->lock); 771227569Sphilip 772227569Sphilip free(evq, M_SFXGE); 773227569Sphilip} 774227569Sphilip 775227569Sphilipstatic int 776227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 777227569Sphilip{ 778227569Sphilip struct sfxge_evq *evq; 779227569Sphilip efsys_mem_t *esmp; 780227569Sphilip int rc; 781227569Sphilip 782227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 783227569Sphilip 784227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 785227569Sphilip evq->sc = sc; 786227569Sphilip evq->index = index; 787227569Sphilip sc->evq[index] = evq; 788227569Sphilip esmp = &evq->mem; 789227569Sphilip 790227569Sphilip /* Initialise TX completion list */ 791227569Sphilip evq->txqs = &evq->txq; 792227569Sphilip 793227569Sphilip /* Allocate DMA space. */ 794227569Sphilip if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0) 795227569Sphilip return (rc); 796227569Sphilip 797227569Sphilip /* Allocate buffer table entries. */ 798227569Sphilip sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS), 799227569Sphilip &evq->buf_base_id); 800227569Sphilip 801227569Sphilip mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 802227569Sphilip 803227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 804227569Sphilip 805227569Sphilip return (0); 806227569Sphilip} 807227569Sphilip 808227569Sphilipvoid 809227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 810227569Sphilip{ 811227569Sphilip struct sfxge_intr *intr; 812227569Sphilip int index; 813227569Sphilip 814227569Sphilip intr = &sc->intr; 815227569Sphilip 816227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 817227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 818227569Sphilip 819227569Sphilip sc->ev_moderation = 0; 820227569Sphilip 821227569Sphilip /* Tear down the event queue(s). */ 822227569Sphilip index = intr->n_alloc; 823227569Sphilip while (--index >= 0) 824227569Sphilip sfxge_ev_qfini(sc, index); 825227569Sphilip} 826227569Sphilip 827227569Sphilipint 828227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 829227569Sphilip{ 830227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 831227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 832227569Sphilip struct sfxge_intr *intr; 833227569Sphilip int index; 834227569Sphilip int rc; 835227569Sphilip 836227569Sphilip intr = &sc->intr; 837227569Sphilip 838227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 839227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 840227569Sphilip 841227569Sphilip /* Set default interrupt moderation; add a sysctl to 842227569Sphilip * read and change it. 843227569Sphilip */ 844227569Sphilip sc->ev_moderation = 30; 845227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 846227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 847227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 848227569Sphilip "sfxge interrupt moderation (us)"); 849227569Sphilip 850227569Sphilip /* 851227569Sphilip * Initialize the event queue(s) - one per interrupt. 852227569Sphilip */ 853227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 854227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 855227569Sphilip goto fail; 856227569Sphilip } 857227569Sphilip 858227569Sphilip sfxge_ev_stat_init(sc); 859227569Sphilip 860227569Sphilip return (0); 861227569Sphilip 862227569Sphilipfail: 863227569Sphilip while (--index >= 0) 864227569Sphilip sfxge_ev_qfini(sc, index); 865227569Sphilip 866227569Sphilip return (rc); 867227569Sphilip} 868