sfxge_ev.c revision 280582
1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_ev.c 280582 2015-03-25 13:04:28Z arybchik $"); 32227569Sphilip 33227569Sphilip#include <sys/param.h> 34227569Sphilip#include <sys/systm.h> 35227569Sphilip 36227569Sphilip#include "common/efx.h" 37227569Sphilip 38227569Sphilip#include "sfxge.h" 39227569Sphilip 40227569Sphilipstatic void 41227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 42227569Sphilip{ 43227569Sphilip struct sfxge_softc *sc; 44227569Sphilip unsigned int index; 45227569Sphilip struct sfxge_rxq *rxq; 46227569Sphilip struct sfxge_txq *txq; 47227569Sphilip 48280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 49280582Sarybchik 50227569Sphilip sc = evq->sc; 51227569Sphilip index = evq->index; 52227569Sphilip rxq = sc->rxq[index]; 53227569Sphilip 54227569Sphilip if ((txq = evq->txq) != NULL) { 55227569Sphilip evq->txq = NULL; 56227569Sphilip evq->txqs = &(evq->txq); 57227569Sphilip 58227569Sphilip do { 59227569Sphilip struct sfxge_txq *next; 60227569Sphilip 61227569Sphilip next = txq->next; 62227569Sphilip txq->next = NULL; 63227569Sphilip 64227569Sphilip KASSERT(txq->evq_index == index, 65227569Sphilip ("txq->evq_index != index")); 66227569Sphilip 67227569Sphilip if (txq->pending != txq->completed) 68280513Sarybchik sfxge_tx_qcomplete(txq, evq); 69227569Sphilip 70227569Sphilip txq = next; 71227569Sphilip } while (txq != NULL); 72227569Sphilip } 73227569Sphilip 74227569Sphilip if (rxq->pending != rxq->completed) 75227569Sphilip sfxge_rx_qcomplete(rxq, eop); 76227569Sphilip} 77227569Sphilip 78227569Sphilipstatic boolean_t 79227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 80227569Sphilip uint16_t flags) 81227569Sphilip{ 82227569Sphilip struct sfxge_evq *evq; 83227569Sphilip struct sfxge_softc *sc; 84227569Sphilip struct sfxge_rxq *rxq; 85227569Sphilip unsigned int expected; 86227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 87227569Sphilip 88227569Sphilip evq = arg; 89280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 90280582Sarybchik 91227569Sphilip sc = evq->sc; 92227569Sphilip 93227569Sphilip if (evq->exception) 94227569Sphilip goto done; 95227569Sphilip 96227569Sphilip rxq = sc->rxq[label]; 97227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 98227569Sphilip KASSERT(evq->index == rxq->index, 99227569Sphilip ("evq->index != rxq->index")); 100227569Sphilip 101227569Sphilip if (rxq->init_state != SFXGE_RXQ_STARTED) 102227569Sphilip goto done; 103227569Sphilip 104280502Sarybchik expected = rxq->pending++ & rxq->ptr_mask; 105227569Sphilip if (id != expected) { 106227569Sphilip evq->exception = B_TRUE; 107227569Sphilip 108227569Sphilip device_printf(sc->dev, "RX completion out of order" 109227569Sphilip " (id=%#x expected=%#x flags=%#x); resetting\n", 110227569Sphilip id, expected, flags); 111227569Sphilip sfxge_schedule_reset(sc); 112227569Sphilip 113227569Sphilip goto done; 114227569Sphilip } 115227569Sphilip 116227569Sphilip rx_desc = &rxq->queue[id]; 117227569Sphilip 118227569Sphilip KASSERT(rx_desc->flags == EFX_DISCARD, 119227569Sphilip ("rx_desc->flags != EFX_DISCARD")); 120227569Sphilip rx_desc->flags = flags; 121227569Sphilip 122227569Sphilip KASSERT(size < (1 << 16), ("size > (1 << 16)")); 123227569Sphilip rx_desc->size = (uint16_t)size; 124227569Sphilip prefetch_read_many(rx_desc->mbuf); 125227569Sphilip 126227569Sphilip evq->rx_done++; 127227569Sphilip 128227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 129227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 130227569Sphilip 131227569Sphilipdone: 132227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 133227569Sphilip} 134227569Sphilip 135227569Sphilipstatic boolean_t 136227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 137227569Sphilip{ 138227569Sphilip struct sfxge_evq *evq; 139227569Sphilip struct sfxge_softc *sc; 140227569Sphilip 141227569Sphilip evq = (struct sfxge_evq *)arg; 142280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 143280582Sarybchik 144227569Sphilip sc = evq->sc; 145227569Sphilip 146227569Sphilip evq->exception = B_TRUE; 147227569Sphilip 148227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 149227569Sphilip device_printf(sc->dev, 150227569Sphilip "hardware exception (code=%u); resetting\n", 151227569Sphilip code); 152227569Sphilip sfxge_schedule_reset(sc); 153227569Sphilip } 154227569Sphilip 155227569Sphilip return (B_FALSE); 156227569Sphilip} 157227569Sphilip 158227569Sphilipstatic boolean_t 159265884Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 160227569Sphilip{ 161227569Sphilip struct sfxge_evq *evq; 162227569Sphilip struct sfxge_softc *sc; 163227569Sphilip struct sfxge_rxq *rxq; 164227569Sphilip unsigned int index; 165265884Sgnn unsigned int label; 166227569Sphilip uint16_t magic; 167227569Sphilip 168227569Sphilip evq = (struct sfxge_evq *)arg; 169280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 170280582Sarybchik 171227569Sphilip sc = evq->sc; 172265884Sgnn rxq = sc->rxq[rxq_index]; 173227569Sphilip 174227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 175227569Sphilip 176227569Sphilip /* Resend a software event on the correct queue */ 177227569Sphilip index = rxq->index; 178227569Sphilip evq = sc->evq[index]; 179227569Sphilip 180265884Sgnn label = rxq_index; 181227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 182227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 183227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 184227569Sphilip 185227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 186227569Sphilip ("evq not started")); 187227569Sphilip efx_ev_qpost(evq->common, magic); 188227569Sphilip 189227569Sphilip return (B_FALSE); 190227569Sphilip} 191227569Sphilip 192227569Sphilipstatic boolean_t 193265884Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 194227569Sphilip{ 195227569Sphilip struct sfxge_evq *evq; 196227569Sphilip struct sfxge_softc *sc; 197227569Sphilip struct sfxge_rxq *rxq; 198227569Sphilip unsigned int index; 199265884Sgnn unsigned int label; 200227569Sphilip uint16_t magic; 201227569Sphilip 202227569Sphilip evq = (struct sfxge_evq *)arg; 203280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 204280582Sarybchik 205227569Sphilip sc = evq->sc; 206265884Sgnn rxq = sc->rxq[rxq_index]; 207227569Sphilip 208227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 209227569Sphilip 210227569Sphilip /* Resend a software event on the correct queue */ 211227569Sphilip index = rxq->index; 212227569Sphilip evq = sc->evq[index]; 213227569Sphilip 214265884Sgnn label = rxq_index; 215227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 216227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 217227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 218227569Sphilip 219227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 220227569Sphilip ("evq not started")); 221227569Sphilip efx_ev_qpost(evq->common, magic); 222227569Sphilip 223227569Sphilip return (B_FALSE); 224227569Sphilip} 225227569Sphilip 226265884Sgnnstatic struct sfxge_txq * 227265884Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 228265884Sgnn{ 229265884Sgnn unsigned int index; 230265884Sgnn 231265884Sgnn KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 232265884Sgnn (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 233265884Sgnn index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 234280501Sarybchik return (evq->sc->txq[index]); 235265884Sgnn} 236265884Sgnn 237227569Sphilipstatic boolean_t 238227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 239227569Sphilip{ 240227569Sphilip struct sfxge_evq *evq; 241227569Sphilip struct sfxge_txq *txq; 242227569Sphilip unsigned int stop; 243227569Sphilip unsigned int delta; 244227569Sphilip 245227569Sphilip evq = (struct sfxge_evq *)arg; 246280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 247280582Sarybchik 248265884Sgnn txq = sfxge_get_txq_by_label(evq, label); 249227569Sphilip 250227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 251227569Sphilip KASSERT(evq->index == txq->evq_index, 252227569Sphilip ("evq->index != txq->evq_index")); 253227569Sphilip 254227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 255227569Sphilip goto done; 256227569Sphilip 257280502Sarybchik stop = (id + 1) & txq->ptr_mask; 258280502Sarybchik id = txq->pending & txq->ptr_mask; 259227569Sphilip 260280502Sarybchik delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 261227569Sphilip txq->pending += delta; 262227569Sphilip 263227569Sphilip evq->tx_done++; 264227569Sphilip 265227569Sphilip if (txq->next == NULL && 266227569Sphilip evq->txqs != &(txq->next)) { 267227569Sphilip *(evq->txqs) = txq; 268227569Sphilip evq->txqs = &(txq->next); 269227569Sphilip } 270227569Sphilip 271227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 272280513Sarybchik sfxge_tx_qcomplete(txq, evq); 273227569Sphilip 274227569Sphilipdone: 275227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 276227569Sphilip} 277227569Sphilip 278227569Sphilipstatic boolean_t 279265884Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 280227569Sphilip{ 281227569Sphilip struct sfxge_evq *evq; 282227569Sphilip struct sfxge_softc *sc; 283227569Sphilip struct sfxge_txq *txq; 284265884Sgnn unsigned int label; 285227569Sphilip uint16_t magic; 286227569Sphilip 287227569Sphilip evq = (struct sfxge_evq *)arg; 288280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 289280582Sarybchik 290227569Sphilip sc = evq->sc; 291265884Sgnn txq = sc->txq[txq_index]; 292227569Sphilip 293227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 294227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 295227569Sphilip ("txq not initialized")); 296227569Sphilip 297227569Sphilip /* Resend a software event on the correct queue */ 298227569Sphilip evq = sc->evq[txq->evq_index]; 299227569Sphilip 300265884Sgnn label = txq->type; 301227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 302227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 303227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 304227569Sphilip 305227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 306227569Sphilip ("evq not started")); 307227569Sphilip efx_ev_qpost(evq->common, magic); 308227569Sphilip 309227569Sphilip return (B_FALSE); 310227569Sphilip} 311227569Sphilip 312227569Sphilipstatic boolean_t 313227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 314227569Sphilip{ 315227569Sphilip struct sfxge_evq *evq; 316227569Sphilip struct sfxge_softc *sc; 317227569Sphilip unsigned int label; 318227569Sphilip 319227569Sphilip evq = (struct sfxge_evq *)arg; 320280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 321280582Sarybchik 322227569Sphilip sc = evq->sc; 323227569Sphilip 324227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 325227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 326227569Sphilip 327227569Sphilip switch (magic) { 328227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 329227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 330227569Sphilip 331227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 332227569Sphilip KASSERT(evq->index == rxq->index, 333227569Sphilip ("evq->index != rxq->index")); 334227569Sphilip 335227569Sphilip sfxge_rx_qflush_done(rxq); 336227569Sphilip break; 337227569Sphilip } 338227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 339227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 340227569Sphilip 341227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 342227569Sphilip KASSERT(evq->index == rxq->index, 343227569Sphilip ("evq->index != rxq->index")); 344227569Sphilip 345227569Sphilip sfxge_rx_qflush_failed(rxq); 346227569Sphilip break; 347227569Sphilip } 348227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 349227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 350227569Sphilip 351227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 352227569Sphilip KASSERT(evq->index == rxq->index, 353227569Sphilip ("evq->index != rxq->index")); 354227569Sphilip 355227569Sphilip sfxge_rx_qrefill(rxq); 356227569Sphilip break; 357227569Sphilip } 358227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 359265884Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 360227569Sphilip 361227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 362227569Sphilip KASSERT(evq->index == txq->evq_index, 363227569Sphilip ("evq->index != txq->evq_index")); 364227569Sphilip 365227569Sphilip sfxge_tx_qflush_done(txq); 366227569Sphilip break; 367227569Sphilip } 368227569Sphilip default: 369227569Sphilip break; 370227569Sphilip } 371227569Sphilip 372227569Sphilip return (B_FALSE); 373227569Sphilip} 374227569Sphilip 375227569Sphilipstatic boolean_t 376227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 377227569Sphilip{ 378227569Sphilip (void)arg; 379227569Sphilip (void)code; 380227569Sphilip 381227569Sphilip switch (code) { 382227569Sphilip case EFX_SRAM_UPDATE: 383227569Sphilip EFSYS_PROBE(sram_update); 384227569Sphilip break; 385227569Sphilip 386227569Sphilip case EFX_SRAM_CLEAR: 387227569Sphilip EFSYS_PROBE(sram_clear); 388227569Sphilip break; 389227569Sphilip 390227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 391227569Sphilip EFSYS_PROBE(sram_illegal_clear); 392227569Sphilip break; 393227569Sphilip 394227569Sphilip default: 395227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 396227569Sphilip break; 397227569Sphilip } 398227569Sphilip 399227569Sphilip return (B_FALSE); 400227569Sphilip} 401227569Sphilip 402227569Sphilipstatic boolean_t 403227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 404227569Sphilip{ 405227569Sphilip (void)arg; 406227569Sphilip (void)index; 407227569Sphilip 408227569Sphilip return (B_FALSE); 409227569Sphilip} 410227569Sphilip 411227569Sphilipstatic boolean_t 412227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 413227569Sphilip{ 414227569Sphilip (void)arg; 415227569Sphilip (void)index; 416227569Sphilip 417227569Sphilip return (B_FALSE); 418227569Sphilip} 419227569Sphilip 420280510Sarybchik#if EFSYS_OPT_QSTATS 421280510Sarybchik 422227569Sphilipstatic void 423227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 424227569Sphilip{ 425227569Sphilip struct sfxge_evq *evq; 426227569Sphilip unsigned int index; 427227569Sphilip clock_t now; 428227569Sphilip 429280522Sarybchik SFXGE_ADAPTER_LOCK(sc); 430227569Sphilip 431227569Sphilip if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 432227569Sphilip goto out; 433227569Sphilip 434227569Sphilip now = ticks; 435227569Sphilip if (now - sc->ev_stats_update_time < hz) 436227569Sphilip goto out; 437227569Sphilip 438227569Sphilip sc->ev_stats_update_time = now; 439227569Sphilip 440227569Sphilip /* Add event counts from each event queue in turn */ 441280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 442227569Sphilip evq = sc->evq[index]; 443280522Sarybchik SFXGE_EVQ_LOCK(evq); 444227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 445280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 446227569Sphilip } 447227569Sphilipout: 448280522Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 449227569Sphilip} 450227569Sphilip 451227569Sphilipstatic int 452227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 453227569Sphilip{ 454227569Sphilip struct sfxge_softc *sc = arg1; 455227569Sphilip unsigned int id = arg2; 456227569Sphilip 457227569Sphilip sfxge_ev_stat_update(sc); 458227569Sphilip 459280501Sarybchik return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 460227569Sphilip} 461227569Sphilip 462227569Sphilipstatic void 463227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 464227569Sphilip{ 465227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 466227569Sphilip struct sysctl_oid_list *stat_list; 467227569Sphilip unsigned int id; 468227569Sphilip char name[40]; 469227569Sphilip 470227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 471227569Sphilip 472227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 473227569Sphilip snprintf(name, sizeof(name), "ev_%s", 474227569Sphilip efx_ev_qstat_name(sc->enp, id)); 475227569Sphilip SYSCTL_ADD_PROC( 476227569Sphilip ctx, stat_list, 477227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 478227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 479227569Sphilip ""); 480227569Sphilip } 481227569Sphilip} 482227569Sphilip 483280510Sarybchik#endif /* EFSYS_OPT_QSTATS */ 484280510Sarybchik 485227569Sphilipstatic void 486227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 487227569Sphilip{ 488227569Sphilip struct sfxge_evq *evq; 489227569Sphilip efx_evq_t *eep; 490227569Sphilip 491227569Sphilip evq = sc->evq[idx]; 492227569Sphilip eep = evq->common; 493227569Sphilip 494227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 495227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 496227569Sphilip 497227569Sphilip (void)efx_ev_qmoderate(eep, us); 498227569Sphilip} 499227569Sphilip 500227569Sphilipstatic int 501227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 502227569Sphilip{ 503227569Sphilip struct sfxge_softc *sc = arg1; 504227569Sphilip struct sfxge_intr *intr = &sc->intr; 505227569Sphilip unsigned int moderation; 506227569Sphilip int error; 507280541Sarybchik unsigned int index; 508227569Sphilip 509280522Sarybchik SFXGE_ADAPTER_LOCK(sc); 510227569Sphilip 511280501Sarybchik if (req->newptr != NULL) { 512227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 513227569Sphilip != 0) 514227569Sphilip goto out; 515227569Sphilip 516227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 517227569Sphilip * so we have to range-check the value ourselves. 518227569Sphilip */ 519227569Sphilip if (moderation > 520227569Sphilip efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 521227569Sphilip error = EINVAL; 522227569Sphilip goto out; 523227569Sphilip } 524227569Sphilip 525227569Sphilip sc->ev_moderation = moderation; 526227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 527280541Sarybchik for (index = 0; index < sc->evq_count; index++) 528227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 529227569Sphilip } 530227569Sphilip } else { 531227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 532227569Sphilip sizeof(sc->ev_moderation)); 533227569Sphilip } 534227569Sphilip 535227569Sphilipout: 536280522Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 537227569Sphilip 538280501Sarybchik return (error); 539227569Sphilip} 540227569Sphilip 541227569Sphilipstatic boolean_t 542227569Sphilipsfxge_ev_initialized(void *arg) 543227569Sphilip{ 544227569Sphilip struct sfxge_evq *evq; 545280501Sarybchik 546227569Sphilip evq = (struct sfxge_evq *)arg; 547280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 548227569Sphilip 549227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 550227569Sphilip ("evq not starting")); 551227569Sphilip 552227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 553227569Sphilip 554227569Sphilip return (0); 555227569Sphilip} 556227569Sphilip 557227569Sphilipstatic boolean_t 558227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 559227569Sphilip{ 560227569Sphilip struct sfxge_evq *evq; 561227569Sphilip struct sfxge_softc *sc; 562227569Sphilip 563227569Sphilip evq = (struct sfxge_evq *)arg; 564280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 565280582Sarybchik 566227569Sphilip sc = evq->sc; 567227569Sphilip 568227569Sphilip sfxge_mac_link_update(sc, link_mode); 569227569Sphilip 570227569Sphilip return (0); 571227569Sphilip} 572227569Sphilip 573227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 574227569Sphilip .eec_initialized = sfxge_ev_initialized, 575227569Sphilip .eec_rx = sfxge_ev_rx, 576227569Sphilip .eec_tx = sfxge_ev_tx, 577227569Sphilip .eec_exception = sfxge_ev_exception, 578227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 579227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 580227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 581227569Sphilip .eec_software = sfxge_ev_software, 582227569Sphilip .eec_sram = sfxge_ev_sram, 583227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 584227569Sphilip .eec_timer = sfxge_ev_timer, 585227569Sphilip .eec_link_change = sfxge_ev_link_change, 586227569Sphilip}; 587227569Sphilip 588227569Sphilip 589227569Sphilipint 590280508Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 591227569Sphilip{ 592227569Sphilip int rc; 593227569Sphilip 594280522Sarybchik SFXGE_EVQ_LOCK(evq); 595227569Sphilip 596227569Sphilip if (evq->init_state != SFXGE_EVQ_STARTING && 597227569Sphilip evq->init_state != SFXGE_EVQ_STARTED) { 598227569Sphilip rc = EINVAL; 599227569Sphilip goto fail; 600227569Sphilip } 601227569Sphilip 602227569Sphilip /* Synchronize the DMA memory for reading */ 603227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 604227569Sphilip BUS_DMASYNC_POSTREAD); 605227569Sphilip 606227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 607227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 608227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 609227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 610227569Sphilip 611227569Sphilip /* Poll the queue */ 612227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 613227569Sphilip 614227569Sphilip evq->rx_done = 0; 615227569Sphilip evq->tx_done = 0; 616227569Sphilip 617227569Sphilip /* Perform any pending completion processing */ 618227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 619227569Sphilip 620227569Sphilip /* Re-prime the event queue for interrupts */ 621227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 622227569Sphilip goto fail; 623227569Sphilip 624280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 625227569Sphilip 626227569Sphilip return (0); 627227569Sphilip 628227569Sphilipfail: 629280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 630227569Sphilip return (rc); 631227569Sphilip} 632227569Sphilip 633227569Sphilipstatic void 634227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 635227569Sphilip{ 636227569Sphilip struct sfxge_evq *evq; 637227569Sphilip 638227569Sphilip evq = sc->evq[index]; 639227569Sphilip 640227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 641227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 642227569Sphilip 643280522Sarybchik SFXGE_EVQ_LOCK(evq); 644227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 645227569Sphilip evq->read_ptr = 0; 646227569Sphilip evq->exception = B_FALSE; 647227569Sphilip 648280510Sarybchik#if EFSYS_OPT_QSTATS 649227569Sphilip /* Add event counts before discarding the common evq state */ 650227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 651280510Sarybchik#endif 652227569Sphilip 653227569Sphilip efx_ev_qdestroy(evq->common); 654227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 655280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 656280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 657227569Sphilip} 658227569Sphilip 659227569Sphilipstatic int 660227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 661227569Sphilip{ 662227569Sphilip struct sfxge_evq *evq; 663227569Sphilip efsys_mem_t *esmp; 664227569Sphilip int count; 665227569Sphilip int rc; 666227569Sphilip 667227569Sphilip evq = sc->evq[index]; 668227569Sphilip esmp = &evq->mem; 669227569Sphilip 670227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 671227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 672227569Sphilip 673227569Sphilip /* Clear all events. */ 674280502Sarybchik (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 675227569Sphilip 676227569Sphilip /* Program the buffer table. */ 677227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 678280502Sarybchik EFX_EVQ_NBUFS(evq->entries))) != 0) 679280502Sarybchik return (rc); 680227569Sphilip 681227569Sphilip /* Create the common code event queue. */ 682280502Sarybchik if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 683227569Sphilip evq->buf_base_id, &evq->common)) != 0) 684227569Sphilip goto fail; 685227569Sphilip 686280522Sarybchik SFXGE_EVQ_LOCK(evq); 687227569Sphilip 688227569Sphilip /* Set the default moderation */ 689227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 690227569Sphilip 691227569Sphilip /* Prime the event queue for interrupts */ 692227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 693227569Sphilip goto fail2; 694227569Sphilip 695227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 696227569Sphilip 697280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 698227569Sphilip 699227569Sphilip /* Wait for the initialization event */ 700227569Sphilip count = 0; 701227569Sphilip do { 702227569Sphilip /* Pause for 100 ms */ 703227569Sphilip pause("sfxge evq init", hz / 10); 704227569Sphilip 705227569Sphilip /* Check to see if the test event has been processed */ 706227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 707227569Sphilip goto done; 708227569Sphilip 709227569Sphilip } while (++count < 20); 710227569Sphilip 711227569Sphilip rc = ETIMEDOUT; 712227569Sphilip goto fail3; 713227569Sphilip 714227569Sphilipdone: 715227569Sphilip return (0); 716227569Sphilip 717227569Sphilipfail3: 718280522Sarybchik SFXGE_EVQ_LOCK(evq); 719227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 720227569Sphilipfail2: 721280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 722227569Sphilip efx_ev_qdestroy(evq->common); 723227569Sphilipfail: 724227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 725280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 726227569Sphilip 727227569Sphilip return (rc); 728227569Sphilip} 729227569Sphilip 730227569Sphilipvoid 731227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 732227569Sphilip{ 733227569Sphilip struct sfxge_intr *intr; 734227569Sphilip efx_nic_t *enp; 735227569Sphilip int index; 736227569Sphilip 737227569Sphilip intr = &sc->intr; 738227569Sphilip enp = sc->enp; 739227569Sphilip 740227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 741227569Sphilip ("Interrupts not started")); 742227569Sphilip 743227569Sphilip /* Stop the event queue(s) */ 744280541Sarybchik index = sc->evq_count; 745227569Sphilip while (--index >= 0) 746227569Sphilip sfxge_ev_qstop(sc, index); 747227569Sphilip 748227569Sphilip /* Tear down the event module */ 749227569Sphilip efx_ev_fini(enp); 750227569Sphilip} 751227569Sphilip 752227569Sphilipint 753227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 754227569Sphilip{ 755227569Sphilip struct sfxge_intr *intr; 756227569Sphilip int index; 757227569Sphilip int rc; 758227569Sphilip 759227569Sphilip intr = &sc->intr; 760227569Sphilip 761227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 762227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 763227569Sphilip 764227569Sphilip /* Initialize the event module */ 765227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 766280501Sarybchik return (rc); 767227569Sphilip 768227569Sphilip /* Start the event queues */ 769280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 770227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 771227569Sphilip goto fail; 772227569Sphilip } 773227569Sphilip 774227569Sphilip return (0); 775227569Sphilip 776227569Sphilipfail: 777227569Sphilip /* Stop the event queue(s) */ 778227569Sphilip while (--index >= 0) 779227569Sphilip sfxge_ev_qstop(sc, index); 780227569Sphilip 781227569Sphilip /* Tear down the event module */ 782227569Sphilip efx_ev_fini(sc->enp); 783227569Sphilip 784227569Sphilip return (rc); 785227569Sphilip} 786227569Sphilip 787227569Sphilipstatic void 788227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 789227569Sphilip{ 790227569Sphilip struct sfxge_evq *evq; 791227569Sphilip 792227569Sphilip evq = sc->evq[index]; 793227569Sphilip 794227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 795227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 796227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 797227569Sphilip 798227569Sphilip sfxge_dma_free(&evq->mem); 799227569Sphilip 800227569Sphilip sc->evq[index] = NULL; 801227569Sphilip 802280522Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 803227569Sphilip 804227569Sphilip free(evq, M_SFXGE); 805227569Sphilip} 806227569Sphilip 807227569Sphilipstatic int 808227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 809227569Sphilip{ 810227569Sphilip struct sfxge_evq *evq; 811227569Sphilip efsys_mem_t *esmp; 812227569Sphilip int rc; 813227569Sphilip 814227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 815227569Sphilip 816227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 817227569Sphilip evq->sc = sc; 818227569Sphilip evq->index = index; 819227569Sphilip sc->evq[index] = evq; 820227569Sphilip esmp = &evq->mem; 821227569Sphilip 822280502Sarybchik /* Build an event queue with room for one event per tx and rx buffer, 823280502Sarybchik * plus some extra for link state events and MCDI completions. 824280502Sarybchik * There are three tx queues in the first event queue and one in 825280502Sarybchik * other. 826280502Sarybchik */ 827280502Sarybchik if (index == 0) 828280502Sarybchik evq->entries = 829280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 830280502Sarybchik 3 * sc->txq_entries + 831280502Sarybchik 128); 832280502Sarybchik else 833280502Sarybchik evq->entries = 834280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 835280502Sarybchik sc->txq_entries + 836280502Sarybchik 128); 837280502Sarybchik 838227569Sphilip /* Initialise TX completion list */ 839227569Sphilip evq->txqs = &evq->txq; 840227569Sphilip 841227569Sphilip /* Allocate DMA space. */ 842280502Sarybchik if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 843227569Sphilip return (rc); 844227569Sphilip 845227569Sphilip /* Allocate buffer table entries. */ 846280502Sarybchik sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 847227569Sphilip &evq->buf_base_id); 848227569Sphilip 849280524Sarybchik SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 850227569Sphilip 851227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 852227569Sphilip 853227569Sphilip return (0); 854227569Sphilip} 855227569Sphilip 856227569Sphilipvoid 857227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 858227569Sphilip{ 859227569Sphilip struct sfxge_intr *intr; 860227569Sphilip int index; 861227569Sphilip 862227569Sphilip intr = &sc->intr; 863227569Sphilip 864227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 865227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 866227569Sphilip 867227569Sphilip sc->ev_moderation = 0; 868227569Sphilip 869227569Sphilip /* Tear down the event queue(s). */ 870280541Sarybchik index = sc->evq_count; 871227569Sphilip while (--index >= 0) 872227569Sphilip sfxge_ev_qfini(sc, index); 873280541Sarybchik 874280541Sarybchik sc->evq_count = 0; 875227569Sphilip} 876227569Sphilip 877227569Sphilipint 878227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 879227569Sphilip{ 880227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 881227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 882227569Sphilip struct sfxge_intr *intr; 883227569Sphilip int index; 884227569Sphilip int rc; 885227569Sphilip 886227569Sphilip intr = &sc->intr; 887227569Sphilip 888280541Sarybchik sc->evq_count = intr->n_alloc; 889280541Sarybchik 890227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 891227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 892227569Sphilip 893227569Sphilip /* Set default interrupt moderation; add a sysctl to 894227569Sphilip * read and change it. 895227569Sphilip */ 896280517Sarybchik sc->ev_moderation = SFXGE_MODERATION; 897227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 898227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 899227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 900227569Sphilip "sfxge interrupt moderation (us)"); 901227569Sphilip 902227569Sphilip /* 903227569Sphilip * Initialize the event queue(s) - one per interrupt. 904227569Sphilip */ 905280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 906227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 907227569Sphilip goto fail; 908227569Sphilip } 909227569Sphilip 910280510Sarybchik#if EFSYS_OPT_QSTATS 911227569Sphilip sfxge_ev_stat_init(sc); 912280510Sarybchik#endif 913227569Sphilip 914227569Sphilip return (0); 915227569Sphilip 916227569Sphilipfail: 917227569Sphilip while (--index >= 0) 918227569Sphilip sfxge_ev_qfini(sc, index); 919227569Sphilip 920280541Sarybchik sc->evq_count = 0; 921227569Sphilip return (rc); 922227569Sphilip} 923