sfxge_ev.c revision 277893
1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_ev.c 277893 2015-01-29 19:06:14Z arybchik $"); 32227569Sphilip 33227569Sphilip#include <sys/param.h> 34257179Sglebius#include <sys/kernel.h> 35257179Sglebius#include <sys/malloc.h> 36257179Sglebius#include <sys/param.h> 37257179Sglebius#include <sys/queue.h> 38227569Sphilip#include <sys/systm.h> 39257179Sglebius#include <sys/taskqueue.h> 40227569Sphilip 41227569Sphilip#include "common/efx.h" 42227569Sphilip 43227569Sphilip#include "sfxge.h" 44227569Sphilip 45227569Sphilipstatic void 46227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 47227569Sphilip{ 48227569Sphilip struct sfxge_softc *sc; 49227569Sphilip unsigned int index; 50227569Sphilip struct sfxge_rxq *rxq; 51227569Sphilip struct sfxge_txq *txq; 52227569Sphilip 53227569Sphilip sc = evq->sc; 54227569Sphilip index = evq->index; 55227569Sphilip rxq = sc->rxq[index]; 56227569Sphilip 57227569Sphilip if ((txq = evq->txq) != NULL) { 58227569Sphilip evq->txq = NULL; 59227569Sphilip evq->txqs = &(evq->txq); 60227569Sphilip 61227569Sphilip do { 62227569Sphilip struct sfxge_txq *next; 63227569Sphilip 64227569Sphilip next = txq->next; 65227569Sphilip txq->next = NULL; 66227569Sphilip 67227569Sphilip KASSERT(txq->evq_index == index, 68227569Sphilip ("txq->evq_index != index")); 69227569Sphilip 70227569Sphilip if (txq->pending != txq->completed) 71277889Sarybchik sfxge_tx_qcomplete(txq, evq); 72227569Sphilip 73227569Sphilip txq = next; 74227569Sphilip } while (txq != NULL); 75227569Sphilip } 76227569Sphilip 77227569Sphilip if (rxq->pending != rxq->completed) 78227569Sphilip sfxge_rx_qcomplete(rxq, eop); 79227569Sphilip} 80227569Sphilip 81227569Sphilipstatic boolean_t 82227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 83227569Sphilip uint16_t flags) 84227569Sphilip{ 85227569Sphilip struct sfxge_evq *evq; 86227569Sphilip struct sfxge_softc *sc; 87227569Sphilip struct sfxge_rxq *rxq; 88227569Sphilip unsigned int expected; 89227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 90227569Sphilip 91227569Sphilip evq = arg; 92227569Sphilip sc = evq->sc; 93227569Sphilip 94227569Sphilip if (evq->exception) 95227569Sphilip goto done; 96227569Sphilip 97227569Sphilip rxq = sc->rxq[label]; 98227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 99227569Sphilip KASSERT(evq->index == rxq->index, 100227569Sphilip ("evq->index != rxq->index")); 101227569Sphilip 102227569Sphilip if (rxq->init_state != SFXGE_RXQ_STARTED) 103227569Sphilip goto done; 104227569Sphilip 105272328Sgnn expected = rxq->pending++ & rxq->ptr_mask; 106227569Sphilip if (id != expected) { 107227569Sphilip evq->exception = B_TRUE; 108227569Sphilip 109227569Sphilip device_printf(sc->dev, "RX completion out of order" 110227569Sphilip " (id=%#x expected=%#x flags=%#x); resetting\n", 111227569Sphilip id, expected, flags); 112227569Sphilip sfxge_schedule_reset(sc); 113227569Sphilip 114227569Sphilip goto done; 115227569Sphilip } 116227569Sphilip 117227569Sphilip rx_desc = &rxq->queue[id]; 118227569Sphilip 119227569Sphilip KASSERT(rx_desc->flags == EFX_DISCARD, 120227569Sphilip ("rx_desc->flags != EFX_DISCARD")); 121227569Sphilip rx_desc->flags = flags; 122227569Sphilip 123227569Sphilip KASSERT(size < (1 << 16), ("size > (1 << 16)")); 124227569Sphilip rx_desc->size = (uint16_t)size; 125227569Sphilip prefetch_read_many(rx_desc->mbuf); 126227569Sphilip 127227569Sphilip evq->rx_done++; 128227569Sphilip 129227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 130227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 131227569Sphilip 132227569Sphilipdone: 133227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 134227569Sphilip} 135227569Sphilip 136227569Sphilipstatic boolean_t 137227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 138227569Sphilip{ 139227569Sphilip struct sfxge_evq *evq; 140227569Sphilip struct sfxge_softc *sc; 141227569Sphilip 142227569Sphilip evq = (struct sfxge_evq *)arg; 143227569Sphilip sc = evq->sc; 144227569Sphilip 145227569Sphilip evq->exception = B_TRUE; 146227569Sphilip 147227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 148227569Sphilip device_printf(sc->dev, 149227569Sphilip "hardware exception (code=%u); resetting\n", 150227569Sphilip code); 151227569Sphilip sfxge_schedule_reset(sc); 152227569Sphilip } 153227569Sphilip 154227569Sphilip return (B_FALSE); 155227569Sphilip} 156227569Sphilip 157227569Sphilipstatic boolean_t 158264461Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 159227569Sphilip{ 160227569Sphilip struct sfxge_evq *evq; 161227569Sphilip struct sfxge_softc *sc; 162227569Sphilip struct sfxge_rxq *rxq; 163227569Sphilip unsigned int index; 164264461Sgnn unsigned int label; 165227569Sphilip uint16_t magic; 166227569Sphilip 167227569Sphilip evq = (struct sfxge_evq *)arg; 168227569Sphilip sc = evq->sc; 169264461Sgnn rxq = sc->rxq[rxq_index]; 170227569Sphilip 171227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 172227569Sphilip 173227569Sphilip /* Resend a software event on the correct queue */ 174227569Sphilip index = rxq->index; 175227569Sphilip evq = sc->evq[index]; 176227569Sphilip 177264461Sgnn label = rxq_index; 178227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 179227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 180227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 181227569Sphilip 182227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 183227569Sphilip ("evq not started")); 184227569Sphilip efx_ev_qpost(evq->common, magic); 185227569Sphilip 186227569Sphilip return (B_FALSE); 187227569Sphilip} 188227569Sphilip 189227569Sphilipstatic boolean_t 190264461Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 191227569Sphilip{ 192227569Sphilip struct sfxge_evq *evq; 193227569Sphilip struct sfxge_softc *sc; 194227569Sphilip struct sfxge_rxq *rxq; 195227569Sphilip unsigned int index; 196264461Sgnn unsigned int label; 197227569Sphilip uint16_t magic; 198227569Sphilip 199227569Sphilip evq = (struct sfxge_evq *)arg; 200227569Sphilip sc = evq->sc; 201264461Sgnn rxq = sc->rxq[rxq_index]; 202227569Sphilip 203227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 204227569Sphilip 205227569Sphilip /* Resend a software event on the correct queue */ 206227569Sphilip index = rxq->index; 207227569Sphilip evq = sc->evq[index]; 208227569Sphilip 209264461Sgnn label = rxq_index; 210227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 211227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 212227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 213227569Sphilip 214227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 215227569Sphilip ("evq not started")); 216227569Sphilip efx_ev_qpost(evq->common, magic); 217227569Sphilip 218227569Sphilip return (B_FALSE); 219227569Sphilip} 220227569Sphilip 221264461Sgnnstatic struct sfxge_txq * 222264461Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 223264461Sgnn{ 224264461Sgnn unsigned int index; 225264461Sgnn 226264461Sgnn KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 227264461Sgnn (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 228264461Sgnn index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 229272325Sgnn return (evq->sc->txq[index]); 230264461Sgnn} 231264461Sgnn 232227569Sphilipstatic boolean_t 233227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 234227569Sphilip{ 235227569Sphilip struct sfxge_evq *evq; 236227569Sphilip struct sfxge_txq *txq; 237227569Sphilip unsigned int stop; 238227569Sphilip unsigned int delta; 239227569Sphilip 240227569Sphilip evq = (struct sfxge_evq *)arg; 241264461Sgnn txq = sfxge_get_txq_by_label(evq, label); 242227569Sphilip 243227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 244227569Sphilip KASSERT(evq->index == txq->evq_index, 245227569Sphilip ("evq->index != txq->evq_index")); 246227569Sphilip 247227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 248227569Sphilip goto done; 249227569Sphilip 250272328Sgnn stop = (id + 1) & txq->ptr_mask; 251272328Sgnn id = txq->pending & txq->ptr_mask; 252227569Sphilip 253272328Sgnn delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 254227569Sphilip txq->pending += delta; 255227569Sphilip 256227569Sphilip evq->tx_done++; 257227569Sphilip 258227569Sphilip if (txq->next == NULL && 259227569Sphilip evq->txqs != &(txq->next)) { 260227569Sphilip *(evq->txqs) = txq; 261227569Sphilip evq->txqs = &(txq->next); 262227569Sphilip } 263227569Sphilip 264227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 265277889Sarybchik sfxge_tx_qcomplete(txq, evq); 266227569Sphilip 267227569Sphilipdone: 268227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 269227569Sphilip} 270227569Sphilip 271227569Sphilipstatic boolean_t 272264461Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 273227569Sphilip{ 274227569Sphilip struct sfxge_evq *evq; 275227569Sphilip struct sfxge_softc *sc; 276227569Sphilip struct sfxge_txq *txq; 277264461Sgnn unsigned int label; 278227569Sphilip uint16_t magic; 279227569Sphilip 280227569Sphilip evq = (struct sfxge_evq *)arg; 281227569Sphilip sc = evq->sc; 282264461Sgnn txq = sc->txq[txq_index]; 283227569Sphilip 284227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 285227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 286227569Sphilip ("txq not initialized")); 287227569Sphilip 288227569Sphilip /* Resend a software event on the correct queue */ 289227569Sphilip evq = sc->evq[txq->evq_index]; 290227569Sphilip 291264461Sgnn label = txq->type; 292227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 293227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 294227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 295227569Sphilip 296227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 297227569Sphilip ("evq not started")); 298227569Sphilip efx_ev_qpost(evq->common, magic); 299227569Sphilip 300227569Sphilip return (B_FALSE); 301227569Sphilip} 302227569Sphilip 303227569Sphilipstatic boolean_t 304227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 305227569Sphilip{ 306227569Sphilip struct sfxge_evq *evq; 307227569Sphilip struct sfxge_softc *sc; 308227569Sphilip unsigned int label; 309227569Sphilip 310227569Sphilip evq = (struct sfxge_evq *)arg; 311227569Sphilip sc = evq->sc; 312227569Sphilip 313227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 314227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 315227569Sphilip 316227569Sphilip switch (magic) { 317227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 318227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 319227569Sphilip 320227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 321227569Sphilip KASSERT(evq->index == rxq->index, 322227569Sphilip ("evq->index != rxq->index")); 323227569Sphilip 324227569Sphilip sfxge_rx_qflush_done(rxq); 325227569Sphilip break; 326227569Sphilip } 327227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 328227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 329227569Sphilip 330227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 331227569Sphilip KASSERT(evq->index == rxq->index, 332227569Sphilip ("evq->index != rxq->index")); 333227569Sphilip 334227569Sphilip sfxge_rx_qflush_failed(rxq); 335227569Sphilip break; 336227569Sphilip } 337227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 338227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 339227569Sphilip 340227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 341227569Sphilip KASSERT(evq->index == rxq->index, 342227569Sphilip ("evq->index != rxq->index")); 343227569Sphilip 344227569Sphilip sfxge_rx_qrefill(rxq); 345227569Sphilip break; 346227569Sphilip } 347227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 348264461Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 349227569Sphilip 350227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 351227569Sphilip KASSERT(evq->index == txq->evq_index, 352227569Sphilip ("evq->index != txq->evq_index")); 353227569Sphilip 354227569Sphilip sfxge_tx_qflush_done(txq); 355227569Sphilip break; 356227569Sphilip } 357227569Sphilip default: 358227569Sphilip break; 359227569Sphilip } 360227569Sphilip 361227569Sphilip return (B_FALSE); 362227569Sphilip} 363227569Sphilip 364227569Sphilipstatic boolean_t 365227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 366227569Sphilip{ 367227569Sphilip (void)arg; 368227569Sphilip (void)code; 369227569Sphilip 370227569Sphilip switch (code) { 371227569Sphilip case EFX_SRAM_UPDATE: 372227569Sphilip EFSYS_PROBE(sram_update); 373227569Sphilip break; 374227569Sphilip 375227569Sphilip case EFX_SRAM_CLEAR: 376227569Sphilip EFSYS_PROBE(sram_clear); 377227569Sphilip break; 378227569Sphilip 379227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 380227569Sphilip EFSYS_PROBE(sram_illegal_clear); 381227569Sphilip break; 382227569Sphilip 383227569Sphilip default: 384227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 385227569Sphilip break; 386227569Sphilip } 387227569Sphilip 388227569Sphilip return (B_FALSE); 389227569Sphilip} 390227569Sphilip 391227569Sphilipstatic boolean_t 392227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 393227569Sphilip{ 394227569Sphilip (void)arg; 395227569Sphilip (void)index; 396227569Sphilip 397227569Sphilip return (B_FALSE); 398227569Sphilip} 399227569Sphilip 400227569Sphilipstatic boolean_t 401227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 402227569Sphilip{ 403227569Sphilip (void)arg; 404227569Sphilip (void)index; 405227569Sphilip 406227569Sphilip return (B_FALSE); 407227569Sphilip} 408227569Sphilip 409277886Sarybchik#if EFSYS_OPT_QSTATS 410277886Sarybchik 411227569Sphilipstatic void 412227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 413227569Sphilip{ 414227569Sphilip struct sfxge_evq *evq; 415227569Sphilip unsigned int index; 416227569Sphilip clock_t now; 417227569Sphilip 418227569Sphilip sx_xlock(&sc->softc_lock); 419227569Sphilip 420227569Sphilip if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 421227569Sphilip goto out; 422227569Sphilip 423227569Sphilip now = ticks; 424227569Sphilip if (now - sc->ev_stats_update_time < hz) 425227569Sphilip goto out; 426227569Sphilip 427227569Sphilip sc->ev_stats_update_time = now; 428227569Sphilip 429227569Sphilip /* Add event counts from each event queue in turn */ 430227569Sphilip for (index = 0; index < sc->intr.n_alloc; index++) { 431227569Sphilip evq = sc->evq[index]; 432227569Sphilip mtx_lock(&evq->lock); 433227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 434227569Sphilip mtx_unlock(&evq->lock); 435227569Sphilip } 436227569Sphilipout: 437227569Sphilip sx_xunlock(&sc->softc_lock); 438227569Sphilip} 439227569Sphilip 440227569Sphilipstatic int 441227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 442227569Sphilip{ 443227569Sphilip struct sfxge_softc *sc = arg1; 444227569Sphilip unsigned int id = arg2; 445227569Sphilip 446227569Sphilip sfxge_ev_stat_update(sc); 447227569Sphilip 448272325Sgnn return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 449227569Sphilip} 450227569Sphilip 451227569Sphilipstatic void 452227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 453227569Sphilip{ 454227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 455227569Sphilip struct sysctl_oid_list *stat_list; 456227569Sphilip unsigned int id; 457227569Sphilip char name[40]; 458227569Sphilip 459227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 460227569Sphilip 461227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 462227569Sphilip snprintf(name, sizeof(name), "ev_%s", 463227569Sphilip efx_ev_qstat_name(sc->enp, id)); 464227569Sphilip SYSCTL_ADD_PROC( 465227569Sphilip ctx, stat_list, 466227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 467227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 468227569Sphilip ""); 469227569Sphilip } 470227569Sphilip} 471227569Sphilip 472277886Sarybchik#endif /* EFSYS_OPT_QSTATS */ 473277886Sarybchik 474227569Sphilipstatic void 475227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 476227569Sphilip{ 477227569Sphilip struct sfxge_evq *evq; 478227569Sphilip efx_evq_t *eep; 479227569Sphilip 480227569Sphilip evq = sc->evq[idx]; 481227569Sphilip eep = evq->common; 482227569Sphilip 483227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 484227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 485227569Sphilip 486227569Sphilip (void)efx_ev_qmoderate(eep, us); 487227569Sphilip} 488227569Sphilip 489227569Sphilipstatic int 490227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 491227569Sphilip{ 492227569Sphilip struct sfxge_softc *sc = arg1; 493227569Sphilip struct sfxge_intr *intr = &sc->intr; 494227569Sphilip unsigned int moderation; 495227569Sphilip int error; 496227569Sphilip int index; 497227569Sphilip 498227569Sphilip sx_xlock(&sc->softc_lock); 499227569Sphilip 500272325Sgnn if (req->newptr != NULL) { 501227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 502227569Sphilip != 0) 503227569Sphilip goto out; 504227569Sphilip 505227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 506227569Sphilip * so we have to range-check the value ourselves. 507227569Sphilip */ 508227569Sphilip if (moderation > 509227569Sphilip efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 510227569Sphilip error = EINVAL; 511227569Sphilip goto out; 512227569Sphilip } 513227569Sphilip 514227569Sphilip sc->ev_moderation = moderation; 515227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 516227569Sphilip for (index = 0; index < intr->n_alloc; index++) 517227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 518227569Sphilip } 519227569Sphilip } else { 520227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 521227569Sphilip sizeof(sc->ev_moderation)); 522227569Sphilip } 523227569Sphilip 524227569Sphilipout: 525227569Sphilip sx_xunlock(&sc->softc_lock); 526227569Sphilip 527272325Sgnn return (error); 528227569Sphilip} 529227569Sphilip 530227569Sphilipstatic boolean_t 531227569Sphilipsfxge_ev_initialized(void *arg) 532227569Sphilip{ 533227569Sphilip struct sfxge_evq *evq; 534272325Sgnn 535227569Sphilip evq = (struct sfxge_evq *)arg; 536227569Sphilip 537227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 538227569Sphilip ("evq not starting")); 539227569Sphilip 540227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 541227569Sphilip 542227569Sphilip return (0); 543227569Sphilip} 544227569Sphilip 545227569Sphilipstatic boolean_t 546227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 547227569Sphilip{ 548227569Sphilip struct sfxge_evq *evq; 549227569Sphilip struct sfxge_softc *sc; 550227569Sphilip 551227569Sphilip evq = (struct sfxge_evq *)arg; 552227569Sphilip sc = evq->sc; 553227569Sphilip 554227569Sphilip sfxge_mac_link_update(sc, link_mode); 555227569Sphilip 556227569Sphilip return (0); 557227569Sphilip} 558227569Sphilip 559227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 560227569Sphilip .eec_initialized = sfxge_ev_initialized, 561227569Sphilip .eec_rx = sfxge_ev_rx, 562227569Sphilip .eec_tx = sfxge_ev_tx, 563227569Sphilip .eec_exception = sfxge_ev_exception, 564227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 565227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 566227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 567227569Sphilip .eec_software = sfxge_ev_software, 568227569Sphilip .eec_sram = sfxge_ev_sram, 569227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 570227569Sphilip .eec_timer = sfxge_ev_timer, 571227569Sphilip .eec_link_change = sfxge_ev_link_change, 572227569Sphilip}; 573227569Sphilip 574227569Sphilip 575227569Sphilipint 576277884Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 577227569Sphilip{ 578227569Sphilip int rc; 579227569Sphilip 580227569Sphilip mtx_lock(&evq->lock); 581227569Sphilip 582227569Sphilip if (evq->init_state != SFXGE_EVQ_STARTING && 583227569Sphilip evq->init_state != SFXGE_EVQ_STARTED) { 584227569Sphilip rc = EINVAL; 585227569Sphilip goto fail; 586227569Sphilip } 587227569Sphilip 588227569Sphilip /* Synchronize the DMA memory for reading */ 589227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 590227569Sphilip BUS_DMASYNC_POSTREAD); 591227569Sphilip 592227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 593227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 594227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 595227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 596227569Sphilip 597227569Sphilip /* Poll the queue */ 598227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 599227569Sphilip 600227569Sphilip evq->rx_done = 0; 601227569Sphilip evq->tx_done = 0; 602227569Sphilip 603227569Sphilip /* Perform any pending completion processing */ 604227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 605227569Sphilip 606227569Sphilip /* Re-prime the event queue for interrupts */ 607227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 608227569Sphilip goto fail; 609227569Sphilip 610227569Sphilip mtx_unlock(&evq->lock); 611227569Sphilip 612227569Sphilip return (0); 613227569Sphilip 614227569Sphilipfail: 615227569Sphilip mtx_unlock(&(evq->lock)); 616227569Sphilip return (rc); 617227569Sphilip} 618227569Sphilip 619227569Sphilipstatic void 620227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 621227569Sphilip{ 622227569Sphilip struct sfxge_evq *evq; 623227569Sphilip 624227569Sphilip evq = sc->evq[index]; 625227569Sphilip 626227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 627227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 628227569Sphilip 629227569Sphilip mtx_lock(&evq->lock); 630227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 631227569Sphilip evq->read_ptr = 0; 632227569Sphilip evq->exception = B_FALSE; 633227569Sphilip 634277886Sarybchik#if EFSYS_OPT_QSTATS 635227569Sphilip /* Add event counts before discarding the common evq state */ 636227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 637277886Sarybchik#endif 638227569Sphilip 639227569Sphilip efx_ev_qdestroy(evq->common); 640227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 641272328Sgnn EFX_EVQ_NBUFS(evq->entries)); 642227569Sphilip mtx_unlock(&evq->lock); 643227569Sphilip} 644227569Sphilip 645227569Sphilipstatic int 646227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 647227569Sphilip{ 648227569Sphilip struct sfxge_evq *evq; 649227569Sphilip efsys_mem_t *esmp; 650227569Sphilip int count; 651227569Sphilip int rc; 652227569Sphilip 653227569Sphilip evq = sc->evq[index]; 654227569Sphilip esmp = &evq->mem; 655227569Sphilip 656227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 657227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 658227569Sphilip 659227569Sphilip /* Clear all events. */ 660272328Sgnn (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 661227569Sphilip 662227569Sphilip /* Program the buffer table. */ 663227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 664272328Sgnn EFX_EVQ_NBUFS(evq->entries))) != 0) 665272328Sgnn return (rc); 666227569Sphilip 667227569Sphilip /* Create the common code event queue. */ 668272328Sgnn if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 669227569Sphilip evq->buf_base_id, &evq->common)) != 0) 670227569Sphilip goto fail; 671227569Sphilip 672227569Sphilip mtx_lock(&evq->lock); 673227569Sphilip 674227569Sphilip /* Set the default moderation */ 675227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 676227569Sphilip 677227569Sphilip /* Prime the event queue for interrupts */ 678227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 679227569Sphilip goto fail2; 680227569Sphilip 681227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 682227569Sphilip 683227569Sphilip mtx_unlock(&evq->lock); 684227569Sphilip 685227569Sphilip /* Wait for the initialization event */ 686227569Sphilip count = 0; 687227569Sphilip do { 688227569Sphilip /* Pause for 100 ms */ 689227569Sphilip pause("sfxge evq init", hz / 10); 690227569Sphilip 691227569Sphilip /* Check to see if the test event has been processed */ 692227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 693227569Sphilip goto done; 694227569Sphilip 695227569Sphilip } while (++count < 20); 696227569Sphilip 697227569Sphilip rc = ETIMEDOUT; 698227569Sphilip goto fail3; 699227569Sphilip 700227569Sphilipdone: 701227569Sphilip return (0); 702227569Sphilip 703227569Sphilipfail3: 704227569Sphilip mtx_lock(&evq->lock); 705227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 706227569Sphilipfail2: 707227569Sphilip mtx_unlock(&evq->lock); 708227569Sphilip efx_ev_qdestroy(evq->common); 709227569Sphilipfail: 710227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 711272328Sgnn EFX_EVQ_NBUFS(evq->entries)); 712227569Sphilip 713227569Sphilip return (rc); 714227569Sphilip} 715227569Sphilip 716227569Sphilipvoid 717227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 718227569Sphilip{ 719227569Sphilip struct sfxge_intr *intr; 720227569Sphilip efx_nic_t *enp; 721227569Sphilip int index; 722227569Sphilip 723227569Sphilip intr = &sc->intr; 724227569Sphilip enp = sc->enp; 725227569Sphilip 726227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 727227569Sphilip ("Interrupts not started")); 728227569Sphilip 729227569Sphilip /* Stop the event queue(s) */ 730227569Sphilip index = intr->n_alloc; 731227569Sphilip while (--index >= 0) 732227569Sphilip sfxge_ev_qstop(sc, index); 733227569Sphilip 734227569Sphilip /* Tear down the event module */ 735227569Sphilip efx_ev_fini(enp); 736227569Sphilip} 737227569Sphilip 738227569Sphilipint 739227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 740227569Sphilip{ 741227569Sphilip struct sfxge_intr *intr; 742227569Sphilip int index; 743227569Sphilip int rc; 744227569Sphilip 745227569Sphilip intr = &sc->intr; 746227569Sphilip 747227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 748227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 749227569Sphilip 750227569Sphilip /* Initialize the event module */ 751227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 752272325Sgnn return (rc); 753227569Sphilip 754227569Sphilip /* Start the event queues */ 755227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 756227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 757227569Sphilip goto fail; 758227569Sphilip } 759227569Sphilip 760227569Sphilip return (0); 761227569Sphilip 762227569Sphilipfail: 763227569Sphilip /* Stop the event queue(s) */ 764227569Sphilip while (--index >= 0) 765227569Sphilip sfxge_ev_qstop(sc, index); 766227569Sphilip 767227569Sphilip /* Tear down the event module */ 768227569Sphilip efx_ev_fini(sc->enp); 769227569Sphilip 770227569Sphilip return (rc); 771227569Sphilip} 772227569Sphilip 773227569Sphilipstatic void 774227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 775227569Sphilip{ 776227569Sphilip struct sfxge_evq *evq; 777227569Sphilip 778227569Sphilip evq = sc->evq[index]; 779227569Sphilip 780227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 781227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 782227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 783227569Sphilip 784227569Sphilip sfxge_dma_free(&evq->mem); 785227569Sphilip 786227569Sphilip sc->evq[index] = NULL; 787227569Sphilip 788227569Sphilip mtx_destroy(&evq->lock); 789227569Sphilip 790227569Sphilip free(evq, M_SFXGE); 791227569Sphilip} 792227569Sphilip 793227569Sphilipstatic int 794227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 795227569Sphilip{ 796227569Sphilip struct sfxge_evq *evq; 797227569Sphilip efsys_mem_t *esmp; 798227569Sphilip int rc; 799227569Sphilip 800227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 801227569Sphilip 802227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 803227569Sphilip evq->sc = sc; 804227569Sphilip evq->index = index; 805227569Sphilip sc->evq[index] = evq; 806227569Sphilip esmp = &evq->mem; 807227569Sphilip 808272328Sgnn /* Build an event queue with room for one event per tx and rx buffer, 809272328Sgnn * plus some extra for link state events and MCDI completions. 810272328Sgnn * There are three tx queues in the first event queue and one in 811272328Sgnn * other. 812272328Sgnn */ 813272328Sgnn if (index == 0) 814272328Sgnn evq->entries = 815272328Sgnn ROUNDUP_POW_OF_TWO(sc->rxq_entries + 816272328Sgnn 3 * sc->txq_entries + 817272328Sgnn 128); 818272328Sgnn else 819272328Sgnn evq->entries = 820272328Sgnn ROUNDUP_POW_OF_TWO(sc->rxq_entries + 821272328Sgnn sc->txq_entries + 822272328Sgnn 128); 823272328Sgnn 824227569Sphilip /* Initialise TX completion list */ 825227569Sphilip evq->txqs = &evq->txq; 826227569Sphilip 827227569Sphilip /* Allocate DMA space. */ 828272328Sgnn if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 829227569Sphilip return (rc); 830227569Sphilip 831227569Sphilip /* Allocate buffer table entries. */ 832272328Sgnn sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 833227569Sphilip &evq->buf_base_id); 834227569Sphilip 835227569Sphilip mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 836227569Sphilip 837227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 838227569Sphilip 839227569Sphilip return (0); 840227569Sphilip} 841227569Sphilip 842227569Sphilipvoid 843227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 844227569Sphilip{ 845227569Sphilip struct sfxge_intr *intr; 846227569Sphilip int index; 847227569Sphilip 848227569Sphilip intr = &sc->intr; 849227569Sphilip 850227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 851227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 852227569Sphilip 853227569Sphilip sc->ev_moderation = 0; 854227569Sphilip 855227569Sphilip /* Tear down the event queue(s). */ 856227569Sphilip index = intr->n_alloc; 857227569Sphilip while (--index >= 0) 858227569Sphilip sfxge_ev_qfini(sc, index); 859227569Sphilip} 860227569Sphilip 861227569Sphilipint 862227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 863227569Sphilip{ 864227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 865227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 866227569Sphilip struct sfxge_intr *intr; 867227569Sphilip int index; 868227569Sphilip int rc; 869227569Sphilip 870227569Sphilip intr = &sc->intr; 871227569Sphilip 872227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 873227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 874227569Sphilip 875227569Sphilip /* Set default interrupt moderation; add a sysctl to 876227569Sphilip * read and change it. 877227569Sphilip */ 878277893Sarybchik sc->ev_moderation = SFXGE_MODERATION; 879227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 880227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 881227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 882227569Sphilip "sfxge interrupt moderation (us)"); 883227569Sphilip 884227569Sphilip /* 885227569Sphilip * Initialize the event queue(s) - one per interrupt. 886227569Sphilip */ 887227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 888227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 889227569Sphilip goto fail; 890227569Sphilip } 891227569Sphilip 892277886Sarybchik#if EFSYS_OPT_QSTATS 893227569Sphilip sfxge_ev_stat_init(sc); 894277886Sarybchik#endif 895227569Sphilip 896227569Sphilip return (0); 897227569Sphilip 898227569Sphilipfail: 899227569Sphilip while (--index >= 0) 900227569Sphilip sfxge_ev_qfini(sc, index); 901227569Sphilip 902227569Sphilip return (rc); 903227569Sphilip} 904