sfxge_ev.c revision 280517
1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_ev.c 280517 2015-03-25 10:21:42Z arybchik $"); 32227569Sphilip 33227569Sphilip#include <sys/param.h> 34227569Sphilip#include <sys/systm.h> 35227569Sphilip 36227569Sphilip#include "common/efx.h" 37227569Sphilip 38227569Sphilip#include "sfxge.h" 39227569Sphilip 40227569Sphilipstatic void 41227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 42227569Sphilip{ 43227569Sphilip struct sfxge_softc *sc; 44227569Sphilip unsigned int index; 45227569Sphilip struct sfxge_rxq *rxq; 46227569Sphilip struct sfxge_txq *txq; 47227569Sphilip 48227569Sphilip sc = evq->sc; 49227569Sphilip index = evq->index; 50227569Sphilip rxq = sc->rxq[index]; 51227569Sphilip 52227569Sphilip if ((txq = evq->txq) != NULL) { 53227569Sphilip evq->txq = NULL; 54227569Sphilip evq->txqs = &(evq->txq); 55227569Sphilip 56227569Sphilip do { 57227569Sphilip struct sfxge_txq *next; 58227569Sphilip 59227569Sphilip next = txq->next; 60227569Sphilip txq->next = NULL; 61227569Sphilip 62227569Sphilip KASSERT(txq->evq_index == index, 63227569Sphilip ("txq->evq_index != index")); 64227569Sphilip 65227569Sphilip if (txq->pending != txq->completed) 66280513Sarybchik sfxge_tx_qcomplete(txq, evq); 67227569Sphilip 68227569Sphilip txq = next; 69227569Sphilip } while (txq != NULL); 70227569Sphilip } 71227569Sphilip 72227569Sphilip if (rxq->pending != rxq->completed) 73227569Sphilip sfxge_rx_qcomplete(rxq, eop); 74227569Sphilip} 75227569Sphilip 76227569Sphilipstatic boolean_t 77227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 78227569Sphilip uint16_t flags) 79227569Sphilip{ 80227569Sphilip struct sfxge_evq *evq; 81227569Sphilip struct sfxge_softc *sc; 82227569Sphilip struct sfxge_rxq *rxq; 83227569Sphilip unsigned int expected; 84227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 85227569Sphilip 86227569Sphilip evq = arg; 87227569Sphilip sc = evq->sc; 88227569Sphilip 89227569Sphilip if (evq->exception) 90227569Sphilip goto done; 91227569Sphilip 92227569Sphilip rxq = sc->rxq[label]; 93227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 94227569Sphilip KASSERT(evq->index == rxq->index, 95227569Sphilip ("evq->index != rxq->index")); 96227569Sphilip 97227569Sphilip if (rxq->init_state != SFXGE_RXQ_STARTED) 98227569Sphilip goto done; 99227569Sphilip 100280502Sarybchik expected = rxq->pending++ & rxq->ptr_mask; 101227569Sphilip if (id != expected) { 102227569Sphilip evq->exception = B_TRUE; 103227569Sphilip 104227569Sphilip device_printf(sc->dev, "RX completion out of order" 105227569Sphilip " (id=%#x expected=%#x flags=%#x); resetting\n", 106227569Sphilip id, expected, flags); 107227569Sphilip sfxge_schedule_reset(sc); 108227569Sphilip 109227569Sphilip goto done; 110227569Sphilip } 111227569Sphilip 112227569Sphilip rx_desc = &rxq->queue[id]; 113227569Sphilip 114227569Sphilip KASSERT(rx_desc->flags == EFX_DISCARD, 115227569Sphilip ("rx_desc->flags != EFX_DISCARD")); 116227569Sphilip rx_desc->flags = flags; 117227569Sphilip 118227569Sphilip KASSERT(size < (1 << 16), ("size > (1 << 16)")); 119227569Sphilip rx_desc->size = (uint16_t)size; 120227569Sphilip prefetch_read_many(rx_desc->mbuf); 121227569Sphilip 122227569Sphilip evq->rx_done++; 123227569Sphilip 124227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 125227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 126227569Sphilip 127227569Sphilipdone: 128227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 129227569Sphilip} 130227569Sphilip 131227569Sphilipstatic boolean_t 132227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 133227569Sphilip{ 134227569Sphilip struct sfxge_evq *evq; 135227569Sphilip struct sfxge_softc *sc; 136227569Sphilip 137227569Sphilip evq = (struct sfxge_evq *)arg; 138227569Sphilip sc = evq->sc; 139227569Sphilip 140227569Sphilip evq->exception = B_TRUE; 141227569Sphilip 142227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 143227569Sphilip device_printf(sc->dev, 144227569Sphilip "hardware exception (code=%u); resetting\n", 145227569Sphilip code); 146227569Sphilip sfxge_schedule_reset(sc); 147227569Sphilip } 148227569Sphilip 149227569Sphilip return (B_FALSE); 150227569Sphilip} 151227569Sphilip 152227569Sphilipstatic boolean_t 153265884Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 154227569Sphilip{ 155227569Sphilip struct sfxge_evq *evq; 156227569Sphilip struct sfxge_softc *sc; 157227569Sphilip struct sfxge_rxq *rxq; 158227569Sphilip unsigned int index; 159265884Sgnn unsigned int label; 160227569Sphilip uint16_t magic; 161227569Sphilip 162227569Sphilip evq = (struct sfxge_evq *)arg; 163227569Sphilip sc = evq->sc; 164265884Sgnn rxq = sc->rxq[rxq_index]; 165227569Sphilip 166227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 167227569Sphilip 168227569Sphilip /* Resend a software event on the correct queue */ 169227569Sphilip index = rxq->index; 170227569Sphilip evq = sc->evq[index]; 171227569Sphilip 172265884Sgnn label = rxq_index; 173227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 174227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 175227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 176227569Sphilip 177227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 178227569Sphilip ("evq not started")); 179227569Sphilip efx_ev_qpost(evq->common, magic); 180227569Sphilip 181227569Sphilip return (B_FALSE); 182227569Sphilip} 183227569Sphilip 184227569Sphilipstatic boolean_t 185265884Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 186227569Sphilip{ 187227569Sphilip struct sfxge_evq *evq; 188227569Sphilip struct sfxge_softc *sc; 189227569Sphilip struct sfxge_rxq *rxq; 190227569Sphilip unsigned int index; 191265884Sgnn unsigned int label; 192227569Sphilip uint16_t magic; 193227569Sphilip 194227569Sphilip evq = (struct sfxge_evq *)arg; 195227569Sphilip sc = evq->sc; 196265884Sgnn rxq = sc->rxq[rxq_index]; 197227569Sphilip 198227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 199227569Sphilip 200227569Sphilip /* Resend a software event on the correct queue */ 201227569Sphilip index = rxq->index; 202227569Sphilip evq = sc->evq[index]; 203227569Sphilip 204265884Sgnn label = rxq_index; 205227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 206227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 207227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 208227569Sphilip 209227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 210227569Sphilip ("evq not started")); 211227569Sphilip efx_ev_qpost(evq->common, magic); 212227569Sphilip 213227569Sphilip return (B_FALSE); 214227569Sphilip} 215227569Sphilip 216265884Sgnnstatic struct sfxge_txq * 217265884Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 218265884Sgnn{ 219265884Sgnn unsigned int index; 220265884Sgnn 221265884Sgnn KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 222265884Sgnn (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 223265884Sgnn index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 224280501Sarybchik return (evq->sc->txq[index]); 225265884Sgnn} 226265884Sgnn 227227569Sphilipstatic boolean_t 228227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 229227569Sphilip{ 230227569Sphilip struct sfxge_evq *evq; 231227569Sphilip struct sfxge_txq *txq; 232227569Sphilip unsigned int stop; 233227569Sphilip unsigned int delta; 234227569Sphilip 235227569Sphilip evq = (struct sfxge_evq *)arg; 236265884Sgnn txq = sfxge_get_txq_by_label(evq, label); 237227569Sphilip 238227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 239227569Sphilip KASSERT(evq->index == txq->evq_index, 240227569Sphilip ("evq->index != txq->evq_index")); 241227569Sphilip 242227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 243227569Sphilip goto done; 244227569Sphilip 245280502Sarybchik stop = (id + 1) & txq->ptr_mask; 246280502Sarybchik id = txq->pending & txq->ptr_mask; 247227569Sphilip 248280502Sarybchik delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 249227569Sphilip txq->pending += delta; 250227569Sphilip 251227569Sphilip evq->tx_done++; 252227569Sphilip 253227569Sphilip if (txq->next == NULL && 254227569Sphilip evq->txqs != &(txq->next)) { 255227569Sphilip *(evq->txqs) = txq; 256227569Sphilip evq->txqs = &(txq->next); 257227569Sphilip } 258227569Sphilip 259227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 260280513Sarybchik sfxge_tx_qcomplete(txq, evq); 261227569Sphilip 262227569Sphilipdone: 263227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 264227569Sphilip} 265227569Sphilip 266227569Sphilipstatic boolean_t 267265884Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 268227569Sphilip{ 269227569Sphilip struct sfxge_evq *evq; 270227569Sphilip struct sfxge_softc *sc; 271227569Sphilip struct sfxge_txq *txq; 272265884Sgnn unsigned int label; 273227569Sphilip uint16_t magic; 274227569Sphilip 275227569Sphilip evq = (struct sfxge_evq *)arg; 276227569Sphilip sc = evq->sc; 277265884Sgnn txq = sc->txq[txq_index]; 278227569Sphilip 279227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 280227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 281227569Sphilip ("txq not initialized")); 282227569Sphilip 283227569Sphilip /* Resend a software event on the correct queue */ 284227569Sphilip evq = sc->evq[txq->evq_index]; 285227569Sphilip 286265884Sgnn label = txq->type; 287227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 288227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 289227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 290227569Sphilip 291227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 292227569Sphilip ("evq not started")); 293227569Sphilip efx_ev_qpost(evq->common, magic); 294227569Sphilip 295227569Sphilip return (B_FALSE); 296227569Sphilip} 297227569Sphilip 298227569Sphilipstatic boolean_t 299227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 300227569Sphilip{ 301227569Sphilip struct sfxge_evq *evq; 302227569Sphilip struct sfxge_softc *sc; 303227569Sphilip unsigned int label; 304227569Sphilip 305227569Sphilip evq = (struct sfxge_evq *)arg; 306227569Sphilip sc = evq->sc; 307227569Sphilip 308227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 309227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 310227569Sphilip 311227569Sphilip switch (magic) { 312227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 313227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 314227569Sphilip 315227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 316227569Sphilip KASSERT(evq->index == rxq->index, 317227569Sphilip ("evq->index != rxq->index")); 318227569Sphilip 319227569Sphilip sfxge_rx_qflush_done(rxq); 320227569Sphilip break; 321227569Sphilip } 322227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 323227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 324227569Sphilip 325227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 326227569Sphilip KASSERT(evq->index == rxq->index, 327227569Sphilip ("evq->index != rxq->index")); 328227569Sphilip 329227569Sphilip sfxge_rx_qflush_failed(rxq); 330227569Sphilip break; 331227569Sphilip } 332227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 333227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 334227569Sphilip 335227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 336227569Sphilip KASSERT(evq->index == rxq->index, 337227569Sphilip ("evq->index != rxq->index")); 338227569Sphilip 339227569Sphilip sfxge_rx_qrefill(rxq); 340227569Sphilip break; 341227569Sphilip } 342227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 343265884Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 344227569Sphilip 345227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 346227569Sphilip KASSERT(evq->index == txq->evq_index, 347227569Sphilip ("evq->index != txq->evq_index")); 348227569Sphilip 349227569Sphilip sfxge_tx_qflush_done(txq); 350227569Sphilip break; 351227569Sphilip } 352227569Sphilip default: 353227569Sphilip break; 354227569Sphilip } 355227569Sphilip 356227569Sphilip return (B_FALSE); 357227569Sphilip} 358227569Sphilip 359227569Sphilipstatic boolean_t 360227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 361227569Sphilip{ 362227569Sphilip (void)arg; 363227569Sphilip (void)code; 364227569Sphilip 365227569Sphilip switch (code) { 366227569Sphilip case EFX_SRAM_UPDATE: 367227569Sphilip EFSYS_PROBE(sram_update); 368227569Sphilip break; 369227569Sphilip 370227569Sphilip case EFX_SRAM_CLEAR: 371227569Sphilip EFSYS_PROBE(sram_clear); 372227569Sphilip break; 373227569Sphilip 374227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 375227569Sphilip EFSYS_PROBE(sram_illegal_clear); 376227569Sphilip break; 377227569Sphilip 378227569Sphilip default: 379227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 380227569Sphilip break; 381227569Sphilip } 382227569Sphilip 383227569Sphilip return (B_FALSE); 384227569Sphilip} 385227569Sphilip 386227569Sphilipstatic boolean_t 387227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 388227569Sphilip{ 389227569Sphilip (void)arg; 390227569Sphilip (void)index; 391227569Sphilip 392227569Sphilip return (B_FALSE); 393227569Sphilip} 394227569Sphilip 395227569Sphilipstatic boolean_t 396227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 397227569Sphilip{ 398227569Sphilip (void)arg; 399227569Sphilip (void)index; 400227569Sphilip 401227569Sphilip return (B_FALSE); 402227569Sphilip} 403227569Sphilip 404280510Sarybchik#if EFSYS_OPT_QSTATS 405280510Sarybchik 406227569Sphilipstatic void 407227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 408227569Sphilip{ 409227569Sphilip struct sfxge_evq *evq; 410227569Sphilip unsigned int index; 411227569Sphilip clock_t now; 412227569Sphilip 413227569Sphilip sx_xlock(&sc->softc_lock); 414227569Sphilip 415227569Sphilip if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 416227569Sphilip goto out; 417227569Sphilip 418227569Sphilip now = ticks; 419227569Sphilip if (now - sc->ev_stats_update_time < hz) 420227569Sphilip goto out; 421227569Sphilip 422227569Sphilip sc->ev_stats_update_time = now; 423227569Sphilip 424227569Sphilip /* Add event counts from each event queue in turn */ 425227569Sphilip for (index = 0; index < sc->intr.n_alloc; index++) { 426227569Sphilip evq = sc->evq[index]; 427227569Sphilip mtx_lock(&evq->lock); 428227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 429227569Sphilip mtx_unlock(&evq->lock); 430227569Sphilip } 431227569Sphilipout: 432227569Sphilip sx_xunlock(&sc->softc_lock); 433227569Sphilip} 434227569Sphilip 435227569Sphilipstatic int 436227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 437227569Sphilip{ 438227569Sphilip struct sfxge_softc *sc = arg1; 439227569Sphilip unsigned int id = arg2; 440227569Sphilip 441227569Sphilip sfxge_ev_stat_update(sc); 442227569Sphilip 443280501Sarybchik return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 444227569Sphilip} 445227569Sphilip 446227569Sphilipstatic void 447227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 448227569Sphilip{ 449227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 450227569Sphilip struct sysctl_oid_list *stat_list; 451227569Sphilip unsigned int id; 452227569Sphilip char name[40]; 453227569Sphilip 454227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 455227569Sphilip 456227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 457227569Sphilip snprintf(name, sizeof(name), "ev_%s", 458227569Sphilip efx_ev_qstat_name(sc->enp, id)); 459227569Sphilip SYSCTL_ADD_PROC( 460227569Sphilip ctx, stat_list, 461227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 462227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 463227569Sphilip ""); 464227569Sphilip } 465227569Sphilip} 466227569Sphilip 467280510Sarybchik#endif /* EFSYS_OPT_QSTATS */ 468280510Sarybchik 469227569Sphilipstatic void 470227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 471227569Sphilip{ 472227569Sphilip struct sfxge_evq *evq; 473227569Sphilip efx_evq_t *eep; 474227569Sphilip 475227569Sphilip evq = sc->evq[idx]; 476227569Sphilip eep = evq->common; 477227569Sphilip 478227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 479227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 480227569Sphilip 481227569Sphilip (void)efx_ev_qmoderate(eep, us); 482227569Sphilip} 483227569Sphilip 484227569Sphilipstatic int 485227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 486227569Sphilip{ 487227569Sphilip struct sfxge_softc *sc = arg1; 488227569Sphilip struct sfxge_intr *intr = &sc->intr; 489227569Sphilip unsigned int moderation; 490227569Sphilip int error; 491227569Sphilip int index; 492227569Sphilip 493227569Sphilip sx_xlock(&sc->softc_lock); 494227569Sphilip 495280501Sarybchik if (req->newptr != NULL) { 496227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 497227569Sphilip != 0) 498227569Sphilip goto out; 499227569Sphilip 500227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 501227569Sphilip * so we have to range-check the value ourselves. 502227569Sphilip */ 503227569Sphilip if (moderation > 504227569Sphilip efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 505227569Sphilip error = EINVAL; 506227569Sphilip goto out; 507227569Sphilip } 508227569Sphilip 509227569Sphilip sc->ev_moderation = moderation; 510227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 511227569Sphilip for (index = 0; index < intr->n_alloc; index++) 512227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 513227569Sphilip } 514227569Sphilip } else { 515227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 516227569Sphilip sizeof(sc->ev_moderation)); 517227569Sphilip } 518227569Sphilip 519227569Sphilipout: 520227569Sphilip sx_xunlock(&sc->softc_lock); 521227569Sphilip 522280501Sarybchik return (error); 523227569Sphilip} 524227569Sphilip 525227569Sphilipstatic boolean_t 526227569Sphilipsfxge_ev_initialized(void *arg) 527227569Sphilip{ 528227569Sphilip struct sfxge_evq *evq; 529280501Sarybchik 530227569Sphilip evq = (struct sfxge_evq *)arg; 531227569Sphilip 532227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 533227569Sphilip ("evq not starting")); 534227569Sphilip 535227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 536227569Sphilip 537227569Sphilip return (0); 538227569Sphilip} 539227569Sphilip 540227569Sphilipstatic boolean_t 541227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 542227569Sphilip{ 543227569Sphilip struct sfxge_evq *evq; 544227569Sphilip struct sfxge_softc *sc; 545227569Sphilip 546227569Sphilip evq = (struct sfxge_evq *)arg; 547227569Sphilip sc = evq->sc; 548227569Sphilip 549227569Sphilip sfxge_mac_link_update(sc, link_mode); 550227569Sphilip 551227569Sphilip return (0); 552227569Sphilip} 553227569Sphilip 554227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 555227569Sphilip .eec_initialized = sfxge_ev_initialized, 556227569Sphilip .eec_rx = sfxge_ev_rx, 557227569Sphilip .eec_tx = sfxge_ev_tx, 558227569Sphilip .eec_exception = sfxge_ev_exception, 559227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 560227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 561227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 562227569Sphilip .eec_software = sfxge_ev_software, 563227569Sphilip .eec_sram = sfxge_ev_sram, 564227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 565227569Sphilip .eec_timer = sfxge_ev_timer, 566227569Sphilip .eec_link_change = sfxge_ev_link_change, 567227569Sphilip}; 568227569Sphilip 569227569Sphilip 570227569Sphilipint 571280508Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 572227569Sphilip{ 573227569Sphilip int rc; 574227569Sphilip 575227569Sphilip mtx_lock(&evq->lock); 576227569Sphilip 577227569Sphilip if (evq->init_state != SFXGE_EVQ_STARTING && 578227569Sphilip evq->init_state != SFXGE_EVQ_STARTED) { 579227569Sphilip rc = EINVAL; 580227569Sphilip goto fail; 581227569Sphilip } 582227569Sphilip 583227569Sphilip /* Synchronize the DMA memory for reading */ 584227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 585227569Sphilip BUS_DMASYNC_POSTREAD); 586227569Sphilip 587227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 588227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 589227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 590227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 591227569Sphilip 592227569Sphilip /* Poll the queue */ 593227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 594227569Sphilip 595227569Sphilip evq->rx_done = 0; 596227569Sphilip evq->tx_done = 0; 597227569Sphilip 598227569Sphilip /* Perform any pending completion processing */ 599227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 600227569Sphilip 601227569Sphilip /* Re-prime the event queue for interrupts */ 602227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 603227569Sphilip goto fail; 604227569Sphilip 605227569Sphilip mtx_unlock(&evq->lock); 606227569Sphilip 607227569Sphilip return (0); 608227569Sphilip 609227569Sphilipfail: 610227569Sphilip mtx_unlock(&(evq->lock)); 611227569Sphilip return (rc); 612227569Sphilip} 613227569Sphilip 614227569Sphilipstatic void 615227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 616227569Sphilip{ 617227569Sphilip struct sfxge_evq *evq; 618227569Sphilip 619227569Sphilip evq = sc->evq[index]; 620227569Sphilip 621227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 622227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 623227569Sphilip 624227569Sphilip mtx_lock(&evq->lock); 625227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 626227569Sphilip evq->read_ptr = 0; 627227569Sphilip evq->exception = B_FALSE; 628227569Sphilip 629280510Sarybchik#if EFSYS_OPT_QSTATS 630227569Sphilip /* Add event counts before discarding the common evq state */ 631227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 632280510Sarybchik#endif 633227569Sphilip 634227569Sphilip efx_ev_qdestroy(evq->common); 635227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 636280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 637227569Sphilip mtx_unlock(&evq->lock); 638227569Sphilip} 639227569Sphilip 640227569Sphilipstatic int 641227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 642227569Sphilip{ 643227569Sphilip struct sfxge_evq *evq; 644227569Sphilip efsys_mem_t *esmp; 645227569Sphilip int count; 646227569Sphilip int rc; 647227569Sphilip 648227569Sphilip evq = sc->evq[index]; 649227569Sphilip esmp = &evq->mem; 650227569Sphilip 651227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 652227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 653227569Sphilip 654227569Sphilip /* Clear all events. */ 655280502Sarybchik (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 656227569Sphilip 657227569Sphilip /* Program the buffer table. */ 658227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 659280502Sarybchik EFX_EVQ_NBUFS(evq->entries))) != 0) 660280502Sarybchik return (rc); 661227569Sphilip 662227569Sphilip /* Create the common code event queue. */ 663280502Sarybchik if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 664227569Sphilip evq->buf_base_id, &evq->common)) != 0) 665227569Sphilip goto fail; 666227569Sphilip 667227569Sphilip mtx_lock(&evq->lock); 668227569Sphilip 669227569Sphilip /* Set the default moderation */ 670227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 671227569Sphilip 672227569Sphilip /* Prime the event queue for interrupts */ 673227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 674227569Sphilip goto fail2; 675227569Sphilip 676227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 677227569Sphilip 678227569Sphilip mtx_unlock(&evq->lock); 679227569Sphilip 680227569Sphilip /* Wait for the initialization event */ 681227569Sphilip count = 0; 682227569Sphilip do { 683227569Sphilip /* Pause for 100 ms */ 684227569Sphilip pause("sfxge evq init", hz / 10); 685227569Sphilip 686227569Sphilip /* Check to see if the test event has been processed */ 687227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 688227569Sphilip goto done; 689227569Sphilip 690227569Sphilip } while (++count < 20); 691227569Sphilip 692227569Sphilip rc = ETIMEDOUT; 693227569Sphilip goto fail3; 694227569Sphilip 695227569Sphilipdone: 696227569Sphilip return (0); 697227569Sphilip 698227569Sphilipfail3: 699227569Sphilip mtx_lock(&evq->lock); 700227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 701227569Sphilipfail2: 702227569Sphilip mtx_unlock(&evq->lock); 703227569Sphilip efx_ev_qdestroy(evq->common); 704227569Sphilipfail: 705227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 706280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 707227569Sphilip 708227569Sphilip return (rc); 709227569Sphilip} 710227569Sphilip 711227569Sphilipvoid 712227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 713227569Sphilip{ 714227569Sphilip struct sfxge_intr *intr; 715227569Sphilip efx_nic_t *enp; 716227569Sphilip int index; 717227569Sphilip 718227569Sphilip intr = &sc->intr; 719227569Sphilip enp = sc->enp; 720227569Sphilip 721227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 722227569Sphilip ("Interrupts not started")); 723227569Sphilip 724227569Sphilip /* Stop the event queue(s) */ 725227569Sphilip index = intr->n_alloc; 726227569Sphilip while (--index >= 0) 727227569Sphilip sfxge_ev_qstop(sc, index); 728227569Sphilip 729227569Sphilip /* Tear down the event module */ 730227569Sphilip efx_ev_fini(enp); 731227569Sphilip} 732227569Sphilip 733227569Sphilipint 734227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 735227569Sphilip{ 736227569Sphilip struct sfxge_intr *intr; 737227569Sphilip int index; 738227569Sphilip int rc; 739227569Sphilip 740227569Sphilip intr = &sc->intr; 741227569Sphilip 742227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 743227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 744227569Sphilip 745227569Sphilip /* Initialize the event module */ 746227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 747280501Sarybchik return (rc); 748227569Sphilip 749227569Sphilip /* Start the event queues */ 750227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 751227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 752227569Sphilip goto fail; 753227569Sphilip } 754227569Sphilip 755227569Sphilip return (0); 756227569Sphilip 757227569Sphilipfail: 758227569Sphilip /* Stop the event queue(s) */ 759227569Sphilip while (--index >= 0) 760227569Sphilip sfxge_ev_qstop(sc, index); 761227569Sphilip 762227569Sphilip /* Tear down the event module */ 763227569Sphilip efx_ev_fini(sc->enp); 764227569Sphilip 765227569Sphilip return (rc); 766227569Sphilip} 767227569Sphilip 768227569Sphilipstatic void 769227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 770227569Sphilip{ 771227569Sphilip struct sfxge_evq *evq; 772227569Sphilip 773227569Sphilip evq = sc->evq[index]; 774227569Sphilip 775227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 776227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 777227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 778227569Sphilip 779227569Sphilip sfxge_dma_free(&evq->mem); 780227569Sphilip 781227569Sphilip sc->evq[index] = NULL; 782227569Sphilip 783227569Sphilip mtx_destroy(&evq->lock); 784227569Sphilip 785227569Sphilip free(evq, M_SFXGE); 786227569Sphilip} 787227569Sphilip 788227569Sphilipstatic int 789227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 790227569Sphilip{ 791227569Sphilip struct sfxge_evq *evq; 792227569Sphilip efsys_mem_t *esmp; 793227569Sphilip int rc; 794227569Sphilip 795227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 796227569Sphilip 797227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 798227569Sphilip evq->sc = sc; 799227569Sphilip evq->index = index; 800227569Sphilip sc->evq[index] = evq; 801227569Sphilip esmp = &evq->mem; 802227569Sphilip 803280502Sarybchik /* Build an event queue with room for one event per tx and rx buffer, 804280502Sarybchik * plus some extra for link state events and MCDI completions. 805280502Sarybchik * There are three tx queues in the first event queue and one in 806280502Sarybchik * other. 807280502Sarybchik */ 808280502Sarybchik if (index == 0) 809280502Sarybchik evq->entries = 810280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 811280502Sarybchik 3 * sc->txq_entries + 812280502Sarybchik 128); 813280502Sarybchik else 814280502Sarybchik evq->entries = 815280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 816280502Sarybchik sc->txq_entries + 817280502Sarybchik 128); 818280502Sarybchik 819227569Sphilip /* Initialise TX completion list */ 820227569Sphilip evq->txqs = &evq->txq; 821227569Sphilip 822227569Sphilip /* Allocate DMA space. */ 823280502Sarybchik if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 824227569Sphilip return (rc); 825227569Sphilip 826227569Sphilip /* Allocate buffer table entries. */ 827280502Sarybchik sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 828227569Sphilip &evq->buf_base_id); 829227569Sphilip 830227569Sphilip mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 831227569Sphilip 832227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 833227569Sphilip 834227569Sphilip return (0); 835227569Sphilip} 836227569Sphilip 837227569Sphilipvoid 838227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 839227569Sphilip{ 840227569Sphilip struct sfxge_intr *intr; 841227569Sphilip int index; 842227569Sphilip 843227569Sphilip intr = &sc->intr; 844227569Sphilip 845227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 846227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 847227569Sphilip 848227569Sphilip sc->ev_moderation = 0; 849227569Sphilip 850227569Sphilip /* Tear down the event queue(s). */ 851227569Sphilip index = intr->n_alloc; 852227569Sphilip while (--index >= 0) 853227569Sphilip sfxge_ev_qfini(sc, index); 854227569Sphilip} 855227569Sphilip 856227569Sphilipint 857227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 858227569Sphilip{ 859227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 860227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 861227569Sphilip struct sfxge_intr *intr; 862227569Sphilip int index; 863227569Sphilip int rc; 864227569Sphilip 865227569Sphilip intr = &sc->intr; 866227569Sphilip 867227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 868227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 869227569Sphilip 870227569Sphilip /* Set default interrupt moderation; add a sysctl to 871227569Sphilip * read and change it. 872227569Sphilip */ 873280517Sarybchik sc->ev_moderation = SFXGE_MODERATION; 874227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 875227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 876227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 877227569Sphilip "sfxge interrupt moderation (us)"); 878227569Sphilip 879227569Sphilip /* 880227569Sphilip * Initialize the event queue(s) - one per interrupt. 881227569Sphilip */ 882227569Sphilip for (index = 0; index < intr->n_alloc; index++) { 883227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 884227569Sphilip goto fail; 885227569Sphilip } 886227569Sphilip 887280510Sarybchik#if EFSYS_OPT_QSTATS 888227569Sphilip sfxge_ev_stat_init(sc); 889280510Sarybchik#endif 890227569Sphilip 891227569Sphilip return (0); 892227569Sphilip 893227569Sphilipfail: 894227569Sphilip while (--index >= 0) 895227569Sphilip sfxge_ev_qfini(sc, index); 896227569Sphilip 897227569Sphilip return (rc); 898227569Sphilip} 899