1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD$"); 32227569Sphilip 33227569Sphilip#include <sys/param.h> 34227569Sphilip#include <sys/systm.h> 35227569Sphilip 36227569Sphilip#include "common/efx.h" 37227569Sphilip 38227569Sphilip#include "sfxge.h" 39227569Sphilip 40227569Sphilipstatic void 41227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 42227569Sphilip{ 43227569Sphilip struct sfxge_softc *sc; 44227569Sphilip unsigned int index; 45227569Sphilip struct sfxge_rxq *rxq; 46227569Sphilip struct sfxge_txq *txq; 47227569Sphilip 48227569Sphilip sc = evq->sc; 49227569Sphilip index = evq->index; 50227569Sphilip rxq = sc->rxq[index]; 51227569Sphilip 52227569Sphilip if ((txq = evq->txq) != NULL) { 53227569Sphilip evq->txq = NULL; 54227569Sphilip evq->txqs = &(evq->txq); 55227569Sphilip 56227569Sphilip do { 57227569Sphilip struct sfxge_txq *next; 58227569Sphilip 59227569Sphilip next = txq->next; 60227569Sphilip txq->next = NULL; 61227569Sphilip 62227569Sphilip KASSERT(txq->evq_index == index, 63227569Sphilip ("txq->evq_index != index")); 64227569Sphilip 65227569Sphilip if (txq->pending != txq->completed) 66227569Sphilip sfxge_tx_qcomplete(txq); 67227569Sphilip 68227569Sphilip txq = next; 69227569Sphilip } while (txq != NULL); 70227569Sphilip } 71227569Sphilip 72227569Sphilip if (rxq->pending != rxq->completed) 73227569Sphilip sfxge_rx_qcomplete(rxq, eop); 74227569Sphilip} 75227569Sphilip 76227569Sphilipstatic boolean_t 77227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 78227569Sphilip uint16_t flags) 79227569Sphilip{ 80227569Sphilip struct sfxge_evq *evq; 81227569Sphilip struct sfxge_softc *sc; 82227569Sphilip struct sfxge_rxq *rxq; 83227569Sphilip unsigned int expected; 84227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 85227569Sphilip 86227569Sphilip evq = arg; 87227569Sphilip sc = evq->sc; 88227569Sphilip 89227569Sphilip if (evq->exception) 90227569Sphilip goto done; 91227569Sphilip 92227569Sphilip rxq = sc->rxq[label]; 93227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 94227569Sphilip KASSERT(evq->index == rxq->index, 95227569Sphilip ("evq->index != rxq->index")); 96227569Sphilip 97227569Sphilip if (rxq->init_state != SFXGE_RXQ_STARTED) 98227569Sphilip goto done; 99227569Sphilip 100227569Sphilip expected = rxq->pending++ & (SFXGE_NDESCS - 1); 101227569Sphilip if (id != expected) { 102227569Sphilip evq->exception = B_TRUE; 103227569Sphilip 104227569Sphilip device_printf(sc->dev, "RX completion out of order" 105227569Sphilip " (id=%#x expected=%#x flags=%#x); resetting\n", 106227569Sphilip id, expected, flags); 107227569Sphilip sfxge_schedule_reset(sc); 108227569Sphilip 109227569Sphilip goto done; 110227569Sphilip } 111227569Sphilip 112227569Sphilip rx_desc = &rxq->queue[id]; 113227569Sphilip 114227569Sphilip KASSERT(rx_desc->flags == EFX_DISCARD, 115227569Sphilip ("rx_desc->flags != EFX_DISCARD")); 116227569Sphilip rx_desc->flags = flags; 117227569Sphilip 118227569Sphilip KASSERT(size < (1 << 16), ("size > (1 << 16)")); 119227569Sphilip rx_desc->size = (uint16_t)size; 120227569Sphilip prefetch_read_many(rx_desc->mbuf); 121227569Sphilip 122227569Sphilip evq->rx_done++; 123227569Sphilip 124227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 125227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 126227569Sphilip 127227569Sphilipdone: 128227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 129227569Sphilip} 130227569Sphilip 131227569Sphilipstatic boolean_t 132227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 133227569Sphilip{ 134227569Sphilip struct sfxge_evq *evq; 135227569Sphilip struct sfxge_softc *sc; 136227569Sphilip 137227569Sphilip evq = (struct sfxge_evq *)arg; 138227569Sphilip sc = evq->sc; 139227569Sphilip 140227569Sphilip evq->exception = B_TRUE; 141227569Sphilip 142227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 143227569Sphilip device_printf(sc->dev, 144227569Sphilip "hardware exception (code=%u); resetting\n", 145227569Sphilip code); 146227569Sphilip sfxge_schedule_reset(sc); 147227569Sphilip } 148227569Sphilip 149227569Sphilip return (B_FALSE); 150227569Sphilip} 151227569Sphilip 152227569Sphilipstatic boolean_t 153227569Sphilipsfxge_ev_rxq_flush_done(void *arg, uint32_t label) 154227569Sphilip{ 155227569Sphilip struct sfxge_evq *evq; 156227569Sphilip struct sfxge_softc *sc; 157227569Sphilip struct sfxge_rxq *rxq; 158227569Sphilip unsigned int index; 159227569Sphilip uint16_t magic; 160227569Sphilip 161227569Sphilip evq = (struct sfxge_evq *)arg; 162227569Sphilip sc = evq->sc; 163227569Sphilip rxq = sc->rxq[label]; 164227569Sphilip 165227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 166227569Sphilip 167227569Sphilip /* Resend a software event on the correct queue */ 168227569Sphilip index = rxq->index; 169227569Sphilip evq = sc->evq[index]; 170227569Sphilip 171227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 172227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 173227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 174227569Sphilip 175227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 176227569Sphilip ("evq not started")); 177227569Sphilip efx_ev_qpost(evq->common, magic); 178227569Sphilip 179227569Sphilip return (B_FALSE); 180227569Sphilip} 181227569Sphilip 182227569Sphilipstatic boolean_t 183227569Sphilipsfxge_ev_rxq_flush_failed(void *arg, uint32_t label) 184227569Sphilip{ 185227569Sphilip struct sfxge_evq *evq; 186227569Sphilip struct sfxge_softc *sc; 187227569Sphilip struct sfxge_rxq *rxq; 188227569Sphilip unsigned int index; 189227569Sphilip uint16_t magic; 190227569Sphilip 191227569Sphilip evq = (struct sfxge_evq *)arg; 192227569Sphilip sc = evq->sc; 193227569Sphilip rxq = sc->rxq[label]; 194227569Sphilip 195227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 196227569Sphilip 197227569Sphilip /* Resend a software event on the correct queue */ 198227569Sphilip index = rxq->index; 199227569Sphilip evq = sc->evq[index]; 200227569Sphilip 201227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 202227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 203227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 204227569Sphilip 205227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 206227569Sphilip ("evq not started")); 207227569Sphilip efx_ev_qpost(evq->common, magic); 208227569Sphilip 209227569Sphilip return (B_FALSE); 210227569Sphilip} 211227569Sphilip 212227569Sphilipstatic boolean_t 213227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 214227569Sphilip{ 215227569Sphilip struct sfxge_evq *evq; 216227569Sphilip struct sfxge_softc *sc; 217227569Sphilip struct sfxge_txq *txq; 218227569Sphilip unsigned int stop; 219227569Sphilip unsigned int delta; 220227569Sphilip 221227569Sphilip evq = (struct sfxge_evq *)arg; 222227569Sphilip sc = evq->sc; 223227569Sphilip txq = sc->txq[label]; 224227569Sphilip 225227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 226227569Sphilip KASSERT(evq->index == txq->evq_index, 227227569Sphilip ("evq->index != txq->evq_index")); 228227569Sphilip 229227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 230227569Sphilip goto done; 231227569Sphilip 232227569Sphilip stop = (id + 1) & (SFXGE_NDESCS - 1); 233227569Sphilip id = txq->pending & (SFXGE_NDESCS - 1); 234227569Sphilip 235227569Sphilip delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop); 236227569Sphilip txq->pending += delta; 237227569Sphilip 238227569Sphilip evq->tx_done++; 239227569Sphilip 240227569Sphilip if (txq->next == NULL && 241227569Sphilip evq->txqs != &(txq->next)) { 242227569Sphilip *(evq->txqs) = txq; 243227569Sphilip evq->txqs = &(txq->next); 244227569Sphilip } 245227569Sphilip 246227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 247227569Sphilip sfxge_tx_qcomplete(txq); 248227569Sphilip 249227569Sphilipdone: 250227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 251227569Sphilip} 252227569Sphilip 253227569Sphilipstatic boolean_t 254227569Sphilipsfxge_ev_txq_flush_done(void *arg, uint32_t label) 255227569Sphilip{ 256227569Sphilip struct sfxge_evq *evq; 257227569Sphilip struct sfxge_softc *sc; 258227569Sphilip struct sfxge_txq *txq; 259227569Sphilip uint16_t magic; 260227569Sphilip 261227569Sphilip evq = (struct sfxge_evq *)arg; 262227569Sphilip sc = evq->sc; 263227569Sphilip txq = sc->txq[label]; 264227569Sphilip 265227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 266227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 267227569Sphilip ("txq not initialized")); 268227569Sphilip 269227569Sphilip /* Resend a software event on the correct queue */ 270227569Sphilip evq = sc->evq[txq->evq_index]; 271227569Sphilip 272227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 273227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 274227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 275227569Sphilip 276227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 277227569Sphilip ("evq not started")); 278227569Sphilip efx_ev_qpost(evq->common, magic); 279227569Sphilip 280227569Sphilip return (B_FALSE); 281227569Sphilip} 282227569Sphilip 283227569Sphilipstatic boolean_t 284227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 285227569Sphilip{ 286227569Sphilip struct sfxge_evq *evq; 287227569Sphilip struct sfxge_softc *sc; 288227569Sphilip unsigned int label; 289227569Sphilip 290227569Sphilip evq = (struct sfxge_evq *)arg; 291227569Sphilip sc = evq->sc; 292227569Sphilip 293227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 294227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 295227569Sphilip 296227569Sphilip switch (magic) { 297227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 298227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 299227569Sphilip 300227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 301227569Sphilip KASSERT(evq->index == rxq->index, 302227569Sphilip ("evq->index != rxq->index")); 303227569Sphilip 304227569Sphilip sfxge_rx_qflush_done(rxq); 305227569Sphilip break; 306227569Sphilip } 307227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 308227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 309227569Sphilip 310227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 311227569Sphilip KASSERT(evq->index == rxq->index, 312227569Sphilip ("evq->index != rxq->index")); 313227569Sphilip 314227569Sphilip sfxge_rx_qflush_failed(rxq); 315227569Sphilip break; 316227569Sphilip } 317227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 318227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 319227569Sphilip 320227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 321227569Sphilip KASSERT(evq->index == rxq->index, 322227569Sphilip ("evq->index != rxq->index")); 323227569Sphilip 324227569Sphilip sfxge_rx_qrefill(rxq); 325227569Sphilip break; 326227569Sphilip } 327227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 328227569Sphilip struct sfxge_txq *txq = sc->txq[label]; 329227569Sphilip 330227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 331227569Sphilip KASSERT(evq->index == txq->evq_index, 332227569Sphilip ("evq->index != txq->evq_index")); 333227569Sphilip 334227569Sphilip sfxge_tx_qflush_done(txq); 335227569Sphilip break; 336227569Sphilip } 337227569Sphilip default: 338227569Sphilip break; 339227569Sphilip } 340227569Sphilip 341227569Sphilip return (B_FALSE); 342227569Sphilip} 343227569Sphilip 344227569Sphilipstatic boolean_t 345227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 346227569Sphilip{ 347227569Sphilip (void)arg; 348227569Sphilip (void)code; 349227569Sphilip 350227569Sphilip switch (code) { 351227569Sphilip case EFX_SRAM_UPDATE: 352227569Sphilip EFSYS_PROBE(sram_update); 353227569Sphilip break; 354227569Sphilip 355227569Sphilip case EFX_SRAM_CLEAR: 356227569Sphilip EFSYS_PROBE(sram_clear); 357227569Sphilip break; 358227569Sphilip 359227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 360227569Sphilip EFSYS_PROBE(sram_illegal_clear); 361227569Sphilip break; 362227569Sphilip 363227569Sphilip default: 364227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 365227569Sphilip break; 366227569Sphilip } 367227569Sphilip 368227569Sphilip return (B_FALSE); 369227569Sphilip} 370227569Sphilip 371227569Sphilipstatic boolean_t 372227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 373227569Sphilip{ 374227569Sphilip (void)arg; 375227569Sphilip (void)index; 376227569Sphilip 377227569Sphilip return (B_FALSE); 378227569Sphilip} 379227569Sphilip 380227569Sphilipstatic boolean_t 381227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 382227569Sphilip{ 383227569Sphilip (void)arg; 384227569Sphilip (void)index; 385227569Sphilip 386227569Sphilip return (B_FALSE); 387227569Sphilip} 388227569Sphilip 389227569Sphilipstatic void 390227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 391227569Sphilip{ 392227569Sphilip struct sfxge_evq *evq; 393227569Sphilip unsigned int index; 394227569Sphilip clock_t now; 395227569Sphilip 396227569Sphilip sx_xlock(&sc->softc_lock); 397227569Sphilip 398227569Sphilip if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 399227569Sphilip goto out; 400227569Sphilip 401227569Sphilip now = ticks; 402227569Sphilip if (now - sc->ev_stats_update_time < hz) 403227569Sphilip goto out; 404227569Sphilip 405227569Sphilip sc->ev_stats_update_time = now; 406227569Sphilip 407227569Sphilip /* Add event counts from each event queue in turn */ 408227569Sphilip for (index = 0; index < sc->intr.n_alloc; index++) { 409227569Sphilip evq = sc->evq[index]; 410227569Sphilip mtx_lock(&evq->lock); 411227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 412227569Sphilip mtx_unlock(&evq->lock); 413227569Sphilip } 414227569Sphilipout: 415227569Sphilip sx_xunlock(&sc->softc_lock); 416227569Sphilip} 417227569Sphilip 418227569Sphilipstatic int 419227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 420227569Sphilip{ 421227569Sphilip struct sfxge_softc *sc = arg1; 422227569Sphilip unsigned int id = arg2; 423227569Sphilip 424227569Sphilip sfxge_ev_stat_update(sc); 425227569Sphilip 426227569Sphilip return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])); 427227569Sphilip} 428227569Sphilip 429227569Sphilipstatic void 430227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 431227569Sphilip{ 432227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 433227569Sphilip struct sysctl_oid_list *stat_list; 434227569Sphilip unsigned int id; 435227569Sphilip char name[40]; 436227569Sphilip 437227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 438227569Sphilip 439227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 440227569Sphilip snprintf(name, sizeof(name), "ev_%s", 441227569Sphilip efx_ev_qstat_name(sc->enp, id)); 442227569Sphilip SYSCTL_ADD_PROC( 443227569Sphilip ctx, stat_list, 444227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 445227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 446227569Sphilip ""); 447227569Sphilip } 448227569Sphilip} 449227569Sphilip 450227569Sphilipstatic void 451227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 452227569Sphilip{ 453227569Sphilip struct sfxge_evq *evq; 454227569Sphilip efx_evq_t *eep; 455227569Sphilip 456227569Sphilip evq = sc->evq[idx]; 457227569Sphilip eep = evq->common; 458227569Sphilip 459227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 460227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 461227569Sphilip 462227569Sphilip (void)efx_ev_qmoderate(eep, us); 463227569Sphilip} 464227569Sphilip 465227569Sphilipstatic int 466227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 467227569Sphilip{ 468227569Sphilip struct sfxge_softc *sc = arg1; 469227569Sphilip struct sfxge_intr *intr = &sc->intr; 470227569Sphilip unsigned int moderation; 471227569Sphilip int error; 472227569Sphilip int index; 473227569Sphilip 474227569Sphilip sx_xlock(&sc->softc_lock); 475227569Sphilip 476227569Sphilip if (req->newptr) { 477227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 478227569Sphilip != 0) 479227569Sphilip goto out; 480227569Sphilip 481227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 482227569Sphilip * so we have to range-check the value ourselves. 483227569Sphilip */ 484227569Sphilip if (moderation > 485227569Sphilip efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 486227569Sphilip error = EINVAL; 487227569Sphilip goto out; 488227569Sphilip } 489227569Sphilip 490227569Sphilip sc->ev_moderation = moderation; 491227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 492227569Sphilip for (index = 0; index < intr->n_alloc; index++) 493227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 494227569Sphilip } 495227569Sphilip } else { 496227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 497227569Sphilip sizeof(sc->ev_moderation)); 498227569Sphilip } 499227569Sphilip 500227569Sphilipout: 501227569Sphilip sx_xunlock(&sc->softc_lock); 502227569Sphilip 503227569Sphilip return error; 504227569Sphilip} 505227569Sphilip 506227569Sphilipstatic boolean_t 507227569Sphilipsfxge_ev_initialized(void *arg) 508227569Sphilip{ 509227569Sphilip struct sfxge_evq *evq; 510227569Sphilip 511227569Sphilip evq = (struct sfxge_evq *)arg; 512227569Sphilip 513227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 514227569Sphilip ("evq not starting")); 515227569Sphilip 516227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 517227569Sphilip 518227569Sphilip return (0); 519227569Sphilip} 520227569Sphilip 521227569Sphilipstatic boolean_t 522227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 523227569Sphilip{ 524227569Sphilip struct sfxge_evq *evq; 525227569Sphilip struct sfxge_softc *sc; 526227569Sphilip 527227569Sphilip evq = (struct sfxge_evq *)arg; 528227569Sphilip sc = evq->sc; 529227569Sphilip 530227569Sphilip sfxge_mac_link_update(sc, link_mode); 531227569Sphilip 532227569Sphilip return (0); 533227569Sphilip} 534227569Sphilip 535227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 536227569Sphilip .eec_initialized = sfxge_ev_initialized, 537227569Sphilip .eec_rx = sfxge_ev_rx, 538227569Sphilip .eec_tx = sfxge_ev_tx, 539227569Sphilip .eec_exception = sfxge_ev_exception, 540227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 541227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 542227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 543227569Sphilip .eec_software = sfxge_ev_software, 544227569Sphilip .eec_sram = sfxge_ev_sram, 545227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 546227569Sphilip .eec_timer = sfxge_ev_timer, 547227569Sphilip .eec_link_change = sfxge_ev_link_change, 548227569Sphilip}; 549227569Sphilip 550227569Sphilip 551227569Sphilipint 552227569Sphilipsfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index) 553227569Sphilip{ 554227569Sphilip struct sfxge_evq *evq; 555227569Sphilip int rc; 556227569Sphilip 557227569Sphilip evq = sc->evq[index]; 558227569Sphilip 559227569Sphilip mtx_lock(&evq->lock); 560227569Sphilip 561227569Sphilip if (evq->init_state != SFXGE_EVQ_STARTING && 562227569Sphilip evq->init_state != SFXGE_EVQ_STARTED) { 563227569Sphilip rc = EINVAL; 564227569Sphilip goto fail; 565227569Sphilip } 566227569Sphilip 567227569Sphilip /* Synchronize the DMA memory for reading */ 568227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 569227569Sphilip BUS_DMASYNC_POSTREAD); 570227569Sphilip 571227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 572227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 573227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 574227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 575227569Sphilip 576227569Sphilip /* Poll the queue */ 577227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 578227569Sphilip 579227569Sphilip evq->rx_done = 0; 580227569Sphilip evq->tx_done = 0; 581227569Sphilip 582227569Sphilip /* Perform any pending completion processing */ 583227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 584227569Sphilip 585227569Sphilip /* Re-prime the event queue for interrupts */ 586227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 587227569Sphilip goto fail; 588227569Sphilip 589227569Sphilip mtx_unlock(&evq->lock); 590227569Sphilip 591227569Sphilip return (0); 592227569Sphilip 593227569Sphilipfail: 594227569Sphilip mtx_unlock(&(evq->lock)); 595227569Sphilip return (rc); 596227569Sphilip} 597227569Sphilip 598227569Sphilipstatic void 599227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 600227569Sphilip{ 601227569Sphilip struct sfxge_evq *evq; 602227569Sphilip 603227569Sphilip evq = sc->evq[index]; 604227569Sphilip 605227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 606227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 607227569Sphilip 608227569Sphilip mtx_lock(&evq->lock); 609227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 610227569Sphilip evq->read_ptr = 0; 611227569Sphilip evq->exception = B_FALSE; 612227569Sphilip 613227569Sphilip /* Add event counts before discarding the common evq state */ 614227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 615227569Sphilip 616227569Sphilip efx_ev_qdestroy(evq->common); 617227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 618227569Sphilip EFX_EVQ_NBUFS(SFXGE_NEVS)); 619227569Sphilip mtx_unlock(&evq->lock); 620227569Sphilip} 621227569Sphilip 622227569Sphilipstatic int 623227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 624227569Sphilip{ 625227569Sphilip struct sfxge_evq *evq; 626227569Sphilip efsys_mem_t *esmp; 627227569Sphilip int count; 628227569Sphilip int rc; 629227569Sphilip 630227569Sphilip evq = sc->evq[index]; 631227569Sphilip esmp = &evq->mem; 632227569Sphilip 633227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 634227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 635227569Sphilip 636227569Sphilip /* Clear all events. */ 637227569Sphilip (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS)); 638227569Sphilip 639227569Sphilip /* Program the buffer table. */ 640227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 641227569Sphilip EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0) 642227569Sphilip return rc; 643227569Sphilip 644227569Sphilip /* Create the common code event queue. */ 645227569Sphilip if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS, 646227569Sphilip evq->buf_base_id, &evq->common)) != 0) 647227569Sphilip goto fail; 648227569Sphilip 649227569Sphilip mtx_lock(&evq->lock); 650227569Sphilip 651227569Sphilip /* Set the default moderation */ 652227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 653227569Sphilip 654227569Sphilip /* Prime the event queue for interrupts */ 655227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 656227569Sphilip goto fail2; 657227569Sphilip 658227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 659227569Sphilip 660227569Sphilip mtx_unlock(&evq->lock); 661227569Sphilip 662227569Sphilip /* Wait for the initialization event */ 663227569Sphilip count = 0; 664227569Sphilip do { 665227569Sphilip /* Pause for 100 ms */ 666227569Sphilip pause("sfxge evq init", hz / 10); 667227569Sphilip 668227569Sphilip /* Check to see if the test event has been processed */ 669227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 670227569Sphilip goto done; 671227569Sphilip 672227569Sphilip } while (++count < 20); 673227569Sphilip 674227569Sphilip rc = ETIMEDOUT; 675227569Sphilip goto fail3; 676227569Sphilip 677227569Sphilipdone: 678227569Sphilip return (0); 679227569Sphilip 680227569Sphilipfail3: 681227569Sphilip mtx_lock(&evq->lock); 682227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 683227569Sphilipfail2: 684227569Sphilip mtx_unlock(&evq->lock); 685227569Sphilip efx_ev_qdestroy(evq->common); 686227569Sphilipfail: 687227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 688227569Sphilip EFX_EVQ_NBUFS(SFXGE_NEVS)); 689227569Sphilip 690227569Sphilip return (rc); 691227569Sphilip} 692227569Sphilip 693227569Sphilipvoid 694227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 695227569Sphilip{ 696227569Sphilip struct sfxge_intr *intr; 697227569Sphilip efx_nic_t *enp; 698227569Sphilip int index; 699227569Sphilip 700227569Sphilip intr = &sc->intr; 701227569Sphilip enp = sc->enp; 702227569Sphilip 703227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 704227569Sphilip ("Interrupts not started")); 705227569Sphilip 706227569Sphilip /* Stop the event queue(s) */ 707227569Sphilip index = intr->n_alloc; 708227569Sphilip while (--index >= 0) 709227569Sphilip sfxge_ev_qstop(sc, index); 710227569Sphilip 711227569Sphilip /* Tear down the event module */ 712227569Sphilip efx_ev_fini(enp); 713227569Sphilip} 714227569Sphilip 715227569Sphilipint 716227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 717227569Sphilip{ 718227569Sphilip struct sfxge_intr *intr; 719227569Sphilip int index; 720227569Sphilip int rc; 721227569Sphilip 722227569Sphilip intr = &sc->intr; 723227569Sphilip 724227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 725227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 726227569Sphilip 727227569Sphilip /* Initialize the event module */ 728227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 729227569Sphilip return rc; 730227569Sphilip 731227569Sphilip /* Start the event queues */ 732227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 733227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 734227569Sphilip goto fail; 735227569Sphilip } 736227569Sphilip 737227569Sphilip return (0); 738227569Sphilip 739227569Sphilipfail: 740227569Sphilip /* Stop the event queue(s) */ 741227569Sphilip while (--index >= 0) 742227569Sphilip sfxge_ev_qstop(sc, index); 743227569Sphilip 744227569Sphilip /* Tear down the event module */ 745227569Sphilip efx_ev_fini(sc->enp); 746227569Sphilip 747227569Sphilip return (rc); 748227569Sphilip} 749227569Sphilip 750227569Sphilipstatic void 751227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 752227569Sphilip{ 753227569Sphilip struct sfxge_evq *evq; 754227569Sphilip 755227569Sphilip evq = sc->evq[index]; 756227569Sphilip 757227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 758227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 759227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 760227569Sphilip 761227569Sphilip sfxge_dma_free(&evq->mem); 762227569Sphilip 763227569Sphilip sc->evq[index] = NULL; 764227569Sphilip 765227569Sphilip mtx_destroy(&evq->lock); 766227569Sphilip 767227569Sphilip free(evq, M_SFXGE); 768227569Sphilip} 769227569Sphilip 770227569Sphilipstatic int 771227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 772227569Sphilip{ 773227569Sphilip struct sfxge_evq *evq; 774227569Sphilip efsys_mem_t *esmp; 775227569Sphilip int rc; 776227569Sphilip 777227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 778227569Sphilip 779227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 780227569Sphilip evq->sc = sc; 781227569Sphilip evq->index = index; 782227569Sphilip sc->evq[index] = evq; 783227569Sphilip esmp = &evq->mem; 784227569Sphilip 785227569Sphilip /* Initialise TX completion list */ 786227569Sphilip evq->txqs = &evq->txq; 787227569Sphilip 788227569Sphilip /* Allocate DMA space. */ 789227569Sphilip if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0) 790227569Sphilip return (rc); 791227569Sphilip 792227569Sphilip /* Allocate buffer table entries. */ 793227569Sphilip sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS), 794227569Sphilip &evq->buf_base_id); 795227569Sphilip 796227569Sphilip mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 797227569Sphilip 798227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 799227569Sphilip 800227569Sphilip return (0); 801227569Sphilip} 802227569Sphilip 803227569Sphilipvoid 804227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 805227569Sphilip{ 806227569Sphilip struct sfxge_intr *intr; 807227569Sphilip int index; 808227569Sphilip 809227569Sphilip intr = &sc->intr; 810227569Sphilip 811227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 812227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 813227569Sphilip 814227569Sphilip sc->ev_moderation = 0; 815227569Sphilip 816227569Sphilip /* Tear down the event queue(s). */ 817227569Sphilip index = intr->n_alloc; 818227569Sphilip while (--index >= 0) 819227569Sphilip sfxge_ev_qfini(sc, index); 820227569Sphilip} 821227569Sphilip 822227569Sphilipint 823227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 824227569Sphilip{ 825227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 826227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 827227569Sphilip struct sfxge_intr *intr; 828227569Sphilip int index; 829227569Sphilip int rc; 830227569Sphilip 831227569Sphilip intr = &sc->intr; 832227569Sphilip 833227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 834227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 835227569Sphilip 836227569Sphilip /* Set default interrupt moderation; add a sysctl to 837227569Sphilip * read and change it. 838227569Sphilip */ 839227569Sphilip sc->ev_moderation = 30; 840227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 841227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 842227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 843227569Sphilip "sfxge interrupt moderation (us)"); 844227569Sphilip 845227569Sphilip /* 846227569Sphilip * Initialize the event queue(s) - one per interrupt. 847227569Sphilip */ 848227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 849227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 850227569Sphilip goto fail; 851227569Sphilip } 852227569Sphilip 853227569Sphilip sfxge_ev_stat_init(sc); 854227569Sphilip 855227569Sphilip return (0); 856227569Sphilip 857227569Sphilipfail: 858227569Sphilip while (--index >= 0) 859227569Sphilip sfxge_ev_qfini(sc, index); 860227569Sphilip 861227569Sphilip return (rc); 862227569Sphilip} 863