sfxge_ev.c revision 279177
1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_ev.c 279177 2015-02-22 18:52:15Z arybchik $"); 32227569Sphilip 33227569Sphilip#include <sys/param.h> 34257179Sglebius#include <sys/kernel.h> 35257179Sglebius#include <sys/malloc.h> 36257179Sglebius#include <sys/param.h> 37257179Sglebius#include <sys/queue.h> 38227569Sphilip#include <sys/systm.h> 39257179Sglebius#include <sys/taskqueue.h> 40227569Sphilip 41227569Sphilip#include "common/efx.h" 42227569Sphilip 43227569Sphilip#include "sfxge.h" 44227569Sphilip 45227569Sphilipstatic void 46227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 47227569Sphilip{ 48227569Sphilip struct sfxge_softc *sc; 49227569Sphilip unsigned int index; 50227569Sphilip struct sfxge_rxq *rxq; 51227569Sphilip struct sfxge_txq *txq; 52227569Sphilip 53279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 54279177Sarybchik 55227569Sphilip sc = evq->sc; 56227569Sphilip index = evq->index; 57227569Sphilip rxq = sc->rxq[index]; 58227569Sphilip 59227569Sphilip if ((txq = evq->txq) != NULL) { 60227569Sphilip evq->txq = NULL; 61227569Sphilip evq->txqs = &(evq->txq); 62227569Sphilip 63227569Sphilip do { 64227569Sphilip struct sfxge_txq *next; 65227569Sphilip 66227569Sphilip next = txq->next; 67227569Sphilip txq->next = NULL; 68227569Sphilip 69227569Sphilip KASSERT(txq->evq_index == index, 70227569Sphilip ("txq->evq_index != index")); 71227569Sphilip 72227569Sphilip if (txq->pending != txq->completed) 73277889Sarybchik sfxge_tx_qcomplete(txq, evq); 74227569Sphilip 75227569Sphilip txq = next; 76227569Sphilip } while (txq != NULL); 77227569Sphilip } 78227569Sphilip 79227569Sphilip if (rxq->pending != rxq->completed) 80227569Sphilip sfxge_rx_qcomplete(rxq, eop); 81227569Sphilip} 82227569Sphilip 83227569Sphilipstatic boolean_t 84227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 85227569Sphilip uint16_t flags) 86227569Sphilip{ 87227569Sphilip struct sfxge_evq *evq; 88227569Sphilip struct sfxge_softc *sc; 89227569Sphilip struct sfxge_rxq *rxq; 90227569Sphilip unsigned int expected; 91227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 92227569Sphilip 93227569Sphilip evq = arg; 94279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 95279177Sarybchik 96227569Sphilip sc = evq->sc; 97227569Sphilip 98227569Sphilip if (evq->exception) 99227569Sphilip goto done; 100227569Sphilip 101227569Sphilip rxq = sc->rxq[label]; 102227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 103227569Sphilip KASSERT(evq->index == rxq->index, 104227569Sphilip ("evq->index != rxq->index")); 105227569Sphilip 106227569Sphilip if (rxq->init_state != SFXGE_RXQ_STARTED) 107227569Sphilip goto done; 108227569Sphilip 109272328Sgnn expected = rxq->pending++ & rxq->ptr_mask; 110227569Sphilip if (id != expected) { 111227569Sphilip evq->exception = B_TRUE; 112227569Sphilip 113227569Sphilip device_printf(sc->dev, "RX completion out of order" 114227569Sphilip " (id=%#x expected=%#x flags=%#x); resetting\n", 115227569Sphilip id, expected, flags); 116227569Sphilip sfxge_schedule_reset(sc); 117227569Sphilip 118227569Sphilip goto done; 119227569Sphilip } 120227569Sphilip 121227569Sphilip rx_desc = &rxq->queue[id]; 122227569Sphilip 123227569Sphilip KASSERT(rx_desc->flags == EFX_DISCARD, 124227569Sphilip ("rx_desc->flags != EFX_DISCARD")); 125227569Sphilip rx_desc->flags = flags; 126227569Sphilip 127227569Sphilip KASSERT(size < (1 << 16), ("size > (1 << 16)")); 128227569Sphilip rx_desc->size = (uint16_t)size; 129227569Sphilip prefetch_read_many(rx_desc->mbuf); 130227569Sphilip 131227569Sphilip evq->rx_done++; 132227569Sphilip 133227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 134227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 135227569Sphilip 136227569Sphilipdone: 137227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 138227569Sphilip} 139227569Sphilip 140227569Sphilipstatic boolean_t 141227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 142227569Sphilip{ 143227569Sphilip struct sfxge_evq *evq; 144227569Sphilip struct sfxge_softc *sc; 145227569Sphilip 146227569Sphilip evq = (struct sfxge_evq *)arg; 147279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 148279177Sarybchik 149227569Sphilip sc = evq->sc; 150227569Sphilip 151227569Sphilip evq->exception = B_TRUE; 152227569Sphilip 153227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 154227569Sphilip device_printf(sc->dev, 155227569Sphilip "hardware exception (code=%u); resetting\n", 156227569Sphilip code); 157227569Sphilip sfxge_schedule_reset(sc); 158227569Sphilip } 159227569Sphilip 160227569Sphilip return (B_FALSE); 161227569Sphilip} 162227569Sphilip 163227569Sphilipstatic boolean_t 164264461Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 165227569Sphilip{ 166227569Sphilip struct sfxge_evq *evq; 167227569Sphilip struct sfxge_softc *sc; 168227569Sphilip struct sfxge_rxq *rxq; 169227569Sphilip unsigned int index; 170264461Sgnn unsigned int label; 171227569Sphilip uint16_t magic; 172227569Sphilip 173227569Sphilip evq = (struct sfxge_evq *)arg; 174279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 175279177Sarybchik 176227569Sphilip sc = evq->sc; 177264461Sgnn rxq = sc->rxq[rxq_index]; 178227569Sphilip 179227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 180227569Sphilip 181227569Sphilip /* Resend a software event on the correct queue */ 182227569Sphilip index = rxq->index; 183227569Sphilip evq = sc->evq[index]; 184227569Sphilip 185264461Sgnn label = rxq_index; 186227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 187227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 188227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 189227569Sphilip 190227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 191227569Sphilip ("evq not started")); 192227569Sphilip efx_ev_qpost(evq->common, magic); 193227569Sphilip 194227569Sphilip return (B_FALSE); 195227569Sphilip} 196227569Sphilip 197227569Sphilipstatic boolean_t 198264461Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 199227569Sphilip{ 200227569Sphilip struct sfxge_evq *evq; 201227569Sphilip struct sfxge_softc *sc; 202227569Sphilip struct sfxge_rxq *rxq; 203227569Sphilip unsigned int index; 204264461Sgnn unsigned int label; 205227569Sphilip uint16_t magic; 206227569Sphilip 207227569Sphilip evq = (struct sfxge_evq *)arg; 208279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 209279177Sarybchik 210227569Sphilip sc = evq->sc; 211264461Sgnn rxq = sc->rxq[rxq_index]; 212227569Sphilip 213227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 214227569Sphilip 215227569Sphilip /* Resend a software event on the correct queue */ 216227569Sphilip index = rxq->index; 217227569Sphilip evq = sc->evq[index]; 218227569Sphilip 219264461Sgnn label = rxq_index; 220227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 221227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 222227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 223227569Sphilip 224227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 225227569Sphilip ("evq not started")); 226227569Sphilip efx_ev_qpost(evq->common, magic); 227227569Sphilip 228227569Sphilip return (B_FALSE); 229227569Sphilip} 230227569Sphilip 231264461Sgnnstatic struct sfxge_txq * 232264461Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 233264461Sgnn{ 234264461Sgnn unsigned int index; 235264461Sgnn 236264461Sgnn KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 237264461Sgnn (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 238264461Sgnn index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 239272325Sgnn return (evq->sc->txq[index]); 240264461Sgnn} 241264461Sgnn 242227569Sphilipstatic boolean_t 243227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 244227569Sphilip{ 245227569Sphilip struct sfxge_evq *evq; 246227569Sphilip struct sfxge_txq *txq; 247227569Sphilip unsigned int stop; 248227569Sphilip unsigned int delta; 249227569Sphilip 250227569Sphilip evq = (struct sfxge_evq *)arg; 251279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 252279177Sarybchik 253264461Sgnn txq = sfxge_get_txq_by_label(evq, label); 254227569Sphilip 255227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 256227569Sphilip KASSERT(evq->index == txq->evq_index, 257227569Sphilip ("evq->index != txq->evq_index")); 258227569Sphilip 259227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 260227569Sphilip goto done; 261227569Sphilip 262272328Sgnn stop = (id + 1) & txq->ptr_mask; 263272328Sgnn id = txq->pending & txq->ptr_mask; 264227569Sphilip 265272328Sgnn delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 266227569Sphilip txq->pending += delta; 267227569Sphilip 268227569Sphilip evq->tx_done++; 269227569Sphilip 270227569Sphilip if (txq->next == NULL && 271227569Sphilip evq->txqs != &(txq->next)) { 272227569Sphilip *(evq->txqs) = txq; 273227569Sphilip evq->txqs = &(txq->next); 274227569Sphilip } 275227569Sphilip 276227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 277277889Sarybchik sfxge_tx_qcomplete(txq, evq); 278227569Sphilip 279227569Sphilipdone: 280227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 281227569Sphilip} 282227569Sphilip 283227569Sphilipstatic boolean_t 284264461Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 285227569Sphilip{ 286227569Sphilip struct sfxge_evq *evq; 287227569Sphilip struct sfxge_softc *sc; 288227569Sphilip struct sfxge_txq *txq; 289264461Sgnn unsigned int label; 290227569Sphilip uint16_t magic; 291227569Sphilip 292227569Sphilip evq = (struct sfxge_evq *)arg; 293279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 294279177Sarybchik 295227569Sphilip sc = evq->sc; 296264461Sgnn txq = sc->txq[txq_index]; 297227569Sphilip 298227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 299227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 300227569Sphilip ("txq not initialized")); 301227569Sphilip 302227569Sphilip /* Resend a software event on the correct queue */ 303227569Sphilip evq = sc->evq[txq->evq_index]; 304227569Sphilip 305264461Sgnn label = txq->type; 306227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 307227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 308227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 309227569Sphilip 310227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 311227569Sphilip ("evq not started")); 312227569Sphilip efx_ev_qpost(evq->common, magic); 313227569Sphilip 314227569Sphilip return (B_FALSE); 315227569Sphilip} 316227569Sphilip 317227569Sphilipstatic boolean_t 318227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 319227569Sphilip{ 320227569Sphilip struct sfxge_evq *evq; 321227569Sphilip struct sfxge_softc *sc; 322227569Sphilip unsigned int label; 323227569Sphilip 324227569Sphilip evq = (struct sfxge_evq *)arg; 325279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 326279177Sarybchik 327227569Sphilip sc = evq->sc; 328227569Sphilip 329227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 330227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 331227569Sphilip 332227569Sphilip switch (magic) { 333227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 334227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 335227569Sphilip 336227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 337227569Sphilip KASSERT(evq->index == rxq->index, 338227569Sphilip ("evq->index != rxq->index")); 339227569Sphilip 340227569Sphilip sfxge_rx_qflush_done(rxq); 341227569Sphilip break; 342227569Sphilip } 343227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 344227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 345227569Sphilip 346227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 347227569Sphilip KASSERT(evq->index == rxq->index, 348227569Sphilip ("evq->index != rxq->index")); 349227569Sphilip 350227569Sphilip sfxge_rx_qflush_failed(rxq); 351227569Sphilip break; 352227569Sphilip } 353227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 354227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 355227569Sphilip 356227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 357227569Sphilip KASSERT(evq->index == rxq->index, 358227569Sphilip ("evq->index != rxq->index")); 359227569Sphilip 360227569Sphilip sfxge_rx_qrefill(rxq); 361227569Sphilip break; 362227569Sphilip } 363227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 364264461Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 365227569Sphilip 366227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 367227569Sphilip KASSERT(evq->index == txq->evq_index, 368227569Sphilip ("evq->index != txq->evq_index")); 369227569Sphilip 370227569Sphilip sfxge_tx_qflush_done(txq); 371227569Sphilip break; 372227569Sphilip } 373227569Sphilip default: 374227569Sphilip break; 375227569Sphilip } 376227569Sphilip 377227569Sphilip return (B_FALSE); 378227569Sphilip} 379227569Sphilip 380227569Sphilipstatic boolean_t 381227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 382227569Sphilip{ 383227569Sphilip (void)arg; 384227569Sphilip (void)code; 385227569Sphilip 386227569Sphilip switch (code) { 387227569Sphilip case EFX_SRAM_UPDATE: 388227569Sphilip EFSYS_PROBE(sram_update); 389227569Sphilip break; 390227569Sphilip 391227569Sphilip case EFX_SRAM_CLEAR: 392227569Sphilip EFSYS_PROBE(sram_clear); 393227569Sphilip break; 394227569Sphilip 395227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 396227569Sphilip EFSYS_PROBE(sram_illegal_clear); 397227569Sphilip break; 398227569Sphilip 399227569Sphilip default: 400227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 401227569Sphilip break; 402227569Sphilip } 403227569Sphilip 404227569Sphilip return (B_FALSE); 405227569Sphilip} 406227569Sphilip 407227569Sphilipstatic boolean_t 408227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 409227569Sphilip{ 410227569Sphilip (void)arg; 411227569Sphilip (void)index; 412227569Sphilip 413227569Sphilip return (B_FALSE); 414227569Sphilip} 415227569Sphilip 416227569Sphilipstatic boolean_t 417227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 418227569Sphilip{ 419227569Sphilip (void)arg; 420227569Sphilip (void)index; 421227569Sphilip 422227569Sphilip return (B_FALSE); 423227569Sphilip} 424227569Sphilip 425277886Sarybchik#if EFSYS_OPT_QSTATS 426277886Sarybchik 427227569Sphilipstatic void 428227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 429227569Sphilip{ 430227569Sphilip struct sfxge_evq *evq; 431227569Sphilip unsigned int index; 432227569Sphilip clock_t now; 433227569Sphilip 434278221Sarybchik SFXGE_ADAPTER_LOCK(sc); 435227569Sphilip 436227569Sphilip if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 437227569Sphilip goto out; 438227569Sphilip 439227569Sphilip now = ticks; 440227569Sphilip if (now - sc->ev_stats_update_time < hz) 441227569Sphilip goto out; 442227569Sphilip 443227569Sphilip sc->ev_stats_update_time = now; 444227569Sphilip 445227569Sphilip /* Add event counts from each event queue in turn */ 446278940Sarybchik for (index = 0; index < sc->evq_count; index++) { 447227569Sphilip evq = sc->evq[index]; 448278221Sarybchik SFXGE_EVQ_LOCK(evq); 449227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 450278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 451227569Sphilip } 452227569Sphilipout: 453278221Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 454227569Sphilip} 455227569Sphilip 456227569Sphilipstatic int 457227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 458227569Sphilip{ 459227569Sphilip struct sfxge_softc *sc = arg1; 460227569Sphilip unsigned int id = arg2; 461227569Sphilip 462227569Sphilip sfxge_ev_stat_update(sc); 463227569Sphilip 464272325Sgnn return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 465227569Sphilip} 466227569Sphilip 467227569Sphilipstatic void 468227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 469227569Sphilip{ 470227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 471227569Sphilip struct sysctl_oid_list *stat_list; 472227569Sphilip unsigned int id; 473227569Sphilip char name[40]; 474227569Sphilip 475227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 476227569Sphilip 477227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 478227569Sphilip snprintf(name, sizeof(name), "ev_%s", 479227569Sphilip efx_ev_qstat_name(sc->enp, id)); 480227569Sphilip SYSCTL_ADD_PROC( 481227569Sphilip ctx, stat_list, 482227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 483227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 484227569Sphilip ""); 485227569Sphilip } 486227569Sphilip} 487227569Sphilip 488277886Sarybchik#endif /* EFSYS_OPT_QSTATS */ 489277886Sarybchik 490227569Sphilipstatic void 491227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 492227569Sphilip{ 493227569Sphilip struct sfxge_evq *evq; 494227569Sphilip efx_evq_t *eep; 495227569Sphilip 496227569Sphilip evq = sc->evq[idx]; 497227569Sphilip eep = evq->common; 498227569Sphilip 499227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 500227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 501227569Sphilip 502227569Sphilip (void)efx_ev_qmoderate(eep, us); 503227569Sphilip} 504227569Sphilip 505227569Sphilipstatic int 506227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 507227569Sphilip{ 508227569Sphilip struct sfxge_softc *sc = arg1; 509227569Sphilip struct sfxge_intr *intr = &sc->intr; 510227569Sphilip unsigned int moderation; 511227569Sphilip int error; 512278940Sarybchik unsigned int index; 513227569Sphilip 514278221Sarybchik SFXGE_ADAPTER_LOCK(sc); 515227569Sphilip 516272325Sgnn if (req->newptr != NULL) { 517227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 518227569Sphilip != 0) 519227569Sphilip goto out; 520227569Sphilip 521227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 522227569Sphilip * so we have to range-check the value ourselves. 523227569Sphilip */ 524227569Sphilip if (moderation > 525227569Sphilip efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 526227569Sphilip error = EINVAL; 527227569Sphilip goto out; 528227569Sphilip } 529227569Sphilip 530227569Sphilip sc->ev_moderation = moderation; 531227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 532278940Sarybchik for (index = 0; index < sc->evq_count; index++) 533227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 534227569Sphilip } 535227569Sphilip } else { 536227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 537227569Sphilip sizeof(sc->ev_moderation)); 538227569Sphilip } 539227569Sphilip 540227569Sphilipout: 541278221Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 542227569Sphilip 543272325Sgnn return (error); 544227569Sphilip} 545227569Sphilip 546227569Sphilipstatic boolean_t 547227569Sphilipsfxge_ev_initialized(void *arg) 548227569Sphilip{ 549227569Sphilip struct sfxge_evq *evq; 550272325Sgnn 551227569Sphilip evq = (struct sfxge_evq *)arg; 552279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 553227569Sphilip 554227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 555227569Sphilip ("evq not starting")); 556227569Sphilip 557227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 558227569Sphilip 559227569Sphilip return (0); 560227569Sphilip} 561227569Sphilip 562227569Sphilipstatic boolean_t 563227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 564227569Sphilip{ 565227569Sphilip struct sfxge_evq *evq; 566227569Sphilip struct sfxge_softc *sc; 567227569Sphilip 568227569Sphilip evq = (struct sfxge_evq *)arg; 569279177Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 570279177Sarybchik 571227569Sphilip sc = evq->sc; 572227569Sphilip 573227569Sphilip sfxge_mac_link_update(sc, link_mode); 574227569Sphilip 575227569Sphilip return (0); 576227569Sphilip} 577227569Sphilip 578227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 579227569Sphilip .eec_initialized = sfxge_ev_initialized, 580227569Sphilip .eec_rx = sfxge_ev_rx, 581227569Sphilip .eec_tx = sfxge_ev_tx, 582227569Sphilip .eec_exception = sfxge_ev_exception, 583227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 584227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 585227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 586227569Sphilip .eec_software = sfxge_ev_software, 587227569Sphilip .eec_sram = sfxge_ev_sram, 588227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 589227569Sphilip .eec_timer = sfxge_ev_timer, 590227569Sphilip .eec_link_change = sfxge_ev_link_change, 591227569Sphilip}; 592227569Sphilip 593227569Sphilip 594227569Sphilipint 595277884Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 596227569Sphilip{ 597227569Sphilip int rc; 598227569Sphilip 599278221Sarybchik SFXGE_EVQ_LOCK(evq); 600227569Sphilip 601227569Sphilip if (evq->init_state != SFXGE_EVQ_STARTING && 602227569Sphilip evq->init_state != SFXGE_EVQ_STARTED) { 603227569Sphilip rc = EINVAL; 604227569Sphilip goto fail; 605227569Sphilip } 606227569Sphilip 607227569Sphilip /* Synchronize the DMA memory for reading */ 608227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 609227569Sphilip BUS_DMASYNC_POSTREAD); 610227569Sphilip 611227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 612227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 613227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 614227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 615227569Sphilip 616227569Sphilip /* Poll the queue */ 617227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 618227569Sphilip 619227569Sphilip evq->rx_done = 0; 620227569Sphilip evq->tx_done = 0; 621227569Sphilip 622227569Sphilip /* Perform any pending completion processing */ 623227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 624227569Sphilip 625227569Sphilip /* Re-prime the event queue for interrupts */ 626227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 627227569Sphilip goto fail; 628227569Sphilip 629278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 630227569Sphilip 631227569Sphilip return (0); 632227569Sphilip 633227569Sphilipfail: 634278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 635227569Sphilip return (rc); 636227569Sphilip} 637227569Sphilip 638227569Sphilipstatic void 639227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 640227569Sphilip{ 641227569Sphilip struct sfxge_evq *evq; 642227569Sphilip 643227569Sphilip evq = sc->evq[index]; 644227569Sphilip 645227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 646227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 647227569Sphilip 648278221Sarybchik SFXGE_EVQ_LOCK(evq); 649227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 650227569Sphilip evq->read_ptr = 0; 651227569Sphilip evq->exception = B_FALSE; 652227569Sphilip 653277886Sarybchik#if EFSYS_OPT_QSTATS 654227569Sphilip /* Add event counts before discarding the common evq state */ 655227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 656277886Sarybchik#endif 657227569Sphilip 658227569Sphilip efx_ev_qdestroy(evq->common); 659227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 660272328Sgnn EFX_EVQ_NBUFS(evq->entries)); 661278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 662227569Sphilip} 663227569Sphilip 664227569Sphilipstatic int 665227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 666227569Sphilip{ 667227569Sphilip struct sfxge_evq *evq; 668227569Sphilip efsys_mem_t *esmp; 669227569Sphilip int count; 670227569Sphilip int rc; 671227569Sphilip 672227569Sphilip evq = sc->evq[index]; 673227569Sphilip esmp = &evq->mem; 674227569Sphilip 675227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 676227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 677227569Sphilip 678227569Sphilip /* Clear all events. */ 679272328Sgnn (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 680227569Sphilip 681227569Sphilip /* Program the buffer table. */ 682227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 683272328Sgnn EFX_EVQ_NBUFS(evq->entries))) != 0) 684272328Sgnn return (rc); 685227569Sphilip 686227569Sphilip /* Create the common code event queue. */ 687272328Sgnn if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 688227569Sphilip evq->buf_base_id, &evq->common)) != 0) 689227569Sphilip goto fail; 690227569Sphilip 691278221Sarybchik SFXGE_EVQ_LOCK(evq); 692227569Sphilip 693227569Sphilip /* Set the default moderation */ 694227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 695227569Sphilip 696227569Sphilip /* Prime the event queue for interrupts */ 697227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 698227569Sphilip goto fail2; 699227569Sphilip 700227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 701227569Sphilip 702278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 703227569Sphilip 704227569Sphilip /* Wait for the initialization event */ 705227569Sphilip count = 0; 706227569Sphilip do { 707227569Sphilip /* Pause for 100 ms */ 708227569Sphilip pause("sfxge evq init", hz / 10); 709227569Sphilip 710227569Sphilip /* Check to see if the test event has been processed */ 711227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 712227569Sphilip goto done; 713227569Sphilip 714227569Sphilip } while (++count < 20); 715227569Sphilip 716227569Sphilip rc = ETIMEDOUT; 717227569Sphilip goto fail3; 718227569Sphilip 719227569Sphilipdone: 720227569Sphilip return (0); 721227569Sphilip 722227569Sphilipfail3: 723278221Sarybchik SFXGE_EVQ_LOCK(evq); 724227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 725227569Sphilipfail2: 726278221Sarybchik SFXGE_EVQ_UNLOCK(evq); 727227569Sphilip efx_ev_qdestroy(evq->common); 728227569Sphilipfail: 729227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 730272328Sgnn EFX_EVQ_NBUFS(evq->entries)); 731227569Sphilip 732227569Sphilip return (rc); 733227569Sphilip} 734227569Sphilip 735227569Sphilipvoid 736227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 737227569Sphilip{ 738227569Sphilip struct sfxge_intr *intr; 739227569Sphilip efx_nic_t *enp; 740227569Sphilip int index; 741227569Sphilip 742227569Sphilip intr = &sc->intr; 743227569Sphilip enp = sc->enp; 744227569Sphilip 745227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 746227569Sphilip ("Interrupts not started")); 747227569Sphilip 748227569Sphilip /* Stop the event queue(s) */ 749278940Sarybchik index = sc->evq_count; 750227569Sphilip while (--index >= 0) 751227569Sphilip sfxge_ev_qstop(sc, index); 752227569Sphilip 753227569Sphilip /* Tear down the event module */ 754227569Sphilip efx_ev_fini(enp); 755227569Sphilip} 756227569Sphilip 757227569Sphilipint 758227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 759227569Sphilip{ 760227569Sphilip struct sfxge_intr *intr; 761227569Sphilip int index; 762227569Sphilip int rc; 763227569Sphilip 764227569Sphilip intr = &sc->intr; 765227569Sphilip 766227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 767227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 768227569Sphilip 769227569Sphilip /* Initialize the event module */ 770227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 771272325Sgnn return (rc); 772227569Sphilip 773227569Sphilip /* Start the event queues */ 774278940Sarybchik for (index = 0; index < sc->evq_count; index++) { 775227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 776227569Sphilip goto fail; 777227569Sphilip } 778227569Sphilip 779227569Sphilip return (0); 780227569Sphilip 781227569Sphilipfail: 782227569Sphilip /* Stop the event queue(s) */ 783227569Sphilip while (--index >= 0) 784227569Sphilip sfxge_ev_qstop(sc, index); 785227569Sphilip 786227569Sphilip /* Tear down the event module */ 787227569Sphilip efx_ev_fini(sc->enp); 788227569Sphilip 789227569Sphilip return (rc); 790227569Sphilip} 791227569Sphilip 792227569Sphilipstatic void 793227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 794227569Sphilip{ 795227569Sphilip struct sfxge_evq *evq; 796227569Sphilip 797227569Sphilip evq = sc->evq[index]; 798227569Sphilip 799227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 800227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 801227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 802227569Sphilip 803227569Sphilip sfxge_dma_free(&evq->mem); 804227569Sphilip 805227569Sphilip sc->evq[index] = NULL; 806227569Sphilip 807278221Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 808227569Sphilip 809227569Sphilip free(evq, M_SFXGE); 810227569Sphilip} 811227569Sphilip 812227569Sphilipstatic int 813227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 814227569Sphilip{ 815227569Sphilip struct sfxge_evq *evq; 816227569Sphilip efsys_mem_t *esmp; 817227569Sphilip int rc; 818227569Sphilip 819227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 820227569Sphilip 821227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 822227569Sphilip evq->sc = sc; 823227569Sphilip evq->index = index; 824227569Sphilip sc->evq[index] = evq; 825227569Sphilip esmp = &evq->mem; 826227569Sphilip 827272328Sgnn /* Build an event queue with room for one event per tx and rx buffer, 828272328Sgnn * plus some extra for link state events and MCDI completions. 829272328Sgnn * There are three tx queues in the first event queue and one in 830272328Sgnn * other. 831272328Sgnn */ 832272328Sgnn if (index == 0) 833272328Sgnn evq->entries = 834272328Sgnn ROUNDUP_POW_OF_TWO(sc->rxq_entries + 835272328Sgnn 3 * sc->txq_entries + 836272328Sgnn 128); 837272328Sgnn else 838272328Sgnn evq->entries = 839272328Sgnn ROUNDUP_POW_OF_TWO(sc->rxq_entries + 840272328Sgnn sc->txq_entries + 841272328Sgnn 128); 842272328Sgnn 843227569Sphilip /* Initialise TX completion list */ 844227569Sphilip evq->txqs = &evq->txq; 845227569Sphilip 846227569Sphilip /* Allocate DMA space. */ 847272328Sgnn if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 848227569Sphilip return (rc); 849227569Sphilip 850227569Sphilip /* Allocate buffer table entries. */ 851272328Sgnn sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 852227569Sphilip &evq->buf_base_id); 853227569Sphilip 854278250Sarybchik SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 855227569Sphilip 856227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 857227569Sphilip 858227569Sphilip return (0); 859227569Sphilip} 860227569Sphilip 861227569Sphilipvoid 862227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 863227569Sphilip{ 864227569Sphilip struct sfxge_intr *intr; 865227569Sphilip int index; 866227569Sphilip 867227569Sphilip intr = &sc->intr; 868227569Sphilip 869227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 870227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 871227569Sphilip 872227569Sphilip sc->ev_moderation = 0; 873227569Sphilip 874227569Sphilip /* Tear down the event queue(s). */ 875278940Sarybchik index = sc->evq_count; 876227569Sphilip while (--index >= 0) 877227569Sphilip sfxge_ev_qfini(sc, index); 878278940Sarybchik 879278940Sarybchik sc->evq_count = 0; 880227569Sphilip} 881227569Sphilip 882227569Sphilipint 883227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 884227569Sphilip{ 885227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 886227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 887227569Sphilip struct sfxge_intr *intr; 888227569Sphilip int index; 889227569Sphilip int rc; 890227569Sphilip 891227569Sphilip intr = &sc->intr; 892227569Sphilip 893278940Sarybchik sc->evq_count = intr->n_alloc; 894278940Sarybchik 895227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 896227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 897227569Sphilip 898227569Sphilip /* Set default interrupt moderation; add a sysctl to 899227569Sphilip * read and change it. 900227569Sphilip */ 901277893Sarybchik sc->ev_moderation = SFXGE_MODERATION; 902227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 903227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 904227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 905227569Sphilip "sfxge interrupt moderation (us)"); 906227569Sphilip 907227569Sphilip /* 908227569Sphilip * Initialize the event queue(s) - one per interrupt. 909227569Sphilip */ 910278940Sarybchik for (index = 0; index < sc->evq_count; index++) { 911227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 912227569Sphilip goto fail; 913227569Sphilip } 914227569Sphilip 915277886Sarybchik#if EFSYS_OPT_QSTATS 916227569Sphilip sfxge_ev_stat_init(sc); 917277886Sarybchik#endif 918227569Sphilip 919227569Sphilip return (0); 920227569Sphilip 921227569Sphilipfail: 922227569Sphilip while (--index >= 0) 923227569Sphilip sfxge_ev_qfini(sc, index); 924227569Sphilip 925278940Sarybchik sc->evq_count = 0; 926227569Sphilip return (rc); 927227569Sphilip} 928