1227569Sphilip/*- 2284555Sarybchik * Copyright (c) 2010-2015 Solarflare Communications Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9284555Sarybchik * modification, are permitted provided that the following conditions are met: 10227569Sphilip * 11284555Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 12284555Sarybchik * this list of conditions and the following disclaimer. 13284555Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 14284555Sarybchik * this list of conditions and the following disclaimer in the documentation 15284555Sarybchik * and/or other materials provided with the distribution. 16284555Sarybchik * 17284555Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18284555Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19284555Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20284555Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21284555Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22284555Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23284555Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24284555Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25284555Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26284555Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27284555Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28284555Sarybchik * 29284555Sarybchik * The views and conclusions contained in the software and documentation are 30284555Sarybchik * those of the authors and should not be interpreted as representing official 31284555Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 32227569Sphilip */ 33227569Sphilip 34227569Sphilip#include <sys/cdefs.h> 35227569Sphilip__FBSDID("$FreeBSD: releng/10.2/sys/dev/sfxge/sfxge_ev.c 284555 2015-06-18 15:46:39Z arybchik $"); 36227569Sphilip 37227569Sphilip#include <sys/param.h> 38227569Sphilip#include <sys/systm.h> 39227569Sphilip 40227569Sphilip#include "common/efx.h" 41227569Sphilip 42227569Sphilip#include "sfxge.h" 43227569Sphilip 44227569Sphilipstatic void 45227569Sphilipsfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 46227569Sphilip{ 47227569Sphilip struct sfxge_softc *sc; 48227569Sphilip unsigned int index; 49227569Sphilip struct sfxge_rxq *rxq; 50227569Sphilip struct sfxge_txq *txq; 51227569Sphilip 52280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 53280582Sarybchik 54227569Sphilip sc = evq->sc; 55227569Sphilip index = evq->index; 56227569Sphilip rxq = sc->rxq[index]; 57227569Sphilip 58227569Sphilip if ((txq = evq->txq) != NULL) { 59227569Sphilip evq->txq = NULL; 60227569Sphilip evq->txqs = &(evq->txq); 61227569Sphilip 62227569Sphilip do { 63227569Sphilip struct sfxge_txq *next; 64227569Sphilip 65227569Sphilip next = txq->next; 66227569Sphilip txq->next = NULL; 67227569Sphilip 68227569Sphilip KASSERT(txq->evq_index == index, 69227569Sphilip ("txq->evq_index != index")); 70227569Sphilip 71227569Sphilip if (txq->pending != txq->completed) 72280513Sarybchik sfxge_tx_qcomplete(txq, evq); 73227569Sphilip 74227569Sphilip txq = next; 75227569Sphilip } while (txq != NULL); 76227569Sphilip } 77227569Sphilip 78227569Sphilip if (rxq->pending != rxq->completed) 79227569Sphilip sfxge_rx_qcomplete(rxq, eop); 80227569Sphilip} 81227569Sphilip 82227569Sphilipstatic boolean_t 83227569Sphilipsfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 84284555Sarybchik uint16_t flags) 85227569Sphilip{ 86227569Sphilip struct sfxge_evq *evq; 87227569Sphilip struct sfxge_softc *sc; 88227569Sphilip struct sfxge_rxq *rxq; 89284555Sarybchik unsigned int stop; 90284555Sarybchik unsigned int delta; 91227569Sphilip struct sfxge_rx_sw_desc *rx_desc; 92227569Sphilip 93227569Sphilip evq = arg; 94280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 95280582Sarybchik 96227569Sphilip sc = evq->sc; 97227569Sphilip 98227569Sphilip if (evq->exception) 99227569Sphilip goto done; 100227569Sphilip 101227569Sphilip rxq = sc->rxq[label]; 102227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 103227569Sphilip KASSERT(evq->index == rxq->index, 104227569Sphilip ("evq->index != rxq->index")); 105227569Sphilip 106280596Sarybchik if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 107227569Sphilip goto done; 108227569Sphilip 109284555Sarybchik stop = (id + 1) & rxq->ptr_mask; 110284555Sarybchik id = rxq->pending & rxq->ptr_mask; 111284555Sarybchik delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 112284555Sarybchik rxq->pending += delta; 113227569Sphilip 114284555Sarybchik if (delta != 1) { 115284555Sarybchik if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) || 116284555Sarybchik (delta <= 0) || 117284555Sarybchik (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 118284555Sarybchik evq->exception = B_TRUE; 119227569Sphilip 120284555Sarybchik device_printf(sc->dev, "RX completion out of order" 121284555Sarybchik " (id=%#x delta=%u flags=%#x); resetting\n", 122284555Sarybchik id, delta, flags); 123284555Sarybchik sfxge_schedule_reset(sc); 124284555Sarybchik 125284555Sarybchik goto done; 126284555Sarybchik } 127227569Sphilip } 128227569Sphilip 129227569Sphilip rx_desc = &rxq->queue[id]; 130227569Sphilip 131227569Sphilip prefetch_read_many(rx_desc->mbuf); 132227569Sphilip 133284555Sarybchik for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 134284555Sarybchik rx_desc = &rxq->queue[id]; 135284555Sarybchik KASSERT(rx_desc->flags == EFX_DISCARD, 136284555Sarybchik ("rx_desc->flags != EFX_DISCARD")); 137284555Sarybchik rx_desc->flags = flags; 138284555Sarybchik 139284555Sarybchik KASSERT(size < (1 << 16), ("size > (1 << 16)")); 140284555Sarybchik rx_desc->size = (uint16_t)size; 141284555Sarybchik } 142284555Sarybchik 143227569Sphilip evq->rx_done++; 144227569Sphilip 145227569Sphilip if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 146227569Sphilip sfxge_ev_qcomplete(evq, B_FALSE); 147227569Sphilip 148227569Sphilipdone: 149227569Sphilip return (evq->rx_done >= SFXGE_EV_BATCH); 150227569Sphilip} 151227569Sphilip 152227569Sphilipstatic boolean_t 153227569Sphilipsfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 154227569Sphilip{ 155227569Sphilip struct sfxge_evq *evq; 156227569Sphilip struct sfxge_softc *sc; 157227569Sphilip 158227569Sphilip evq = (struct sfxge_evq *)arg; 159280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 160280582Sarybchik 161227569Sphilip sc = evq->sc; 162227569Sphilip 163284555Sarybchik DBGPRINT(sc->dev, "[%d] %s", evq->index, 164284555Sarybchik (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 165284555Sarybchik (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 166284555Sarybchik (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 167284555Sarybchik (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 168284555Sarybchik (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 169284555Sarybchik (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 170284555Sarybchik (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 171284555Sarybchik (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 172284555Sarybchik (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 173284555Sarybchik "UNKNOWN"); 174284555Sarybchik 175227569Sphilip evq->exception = B_TRUE; 176227569Sphilip 177227569Sphilip if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 178227569Sphilip device_printf(sc->dev, 179227569Sphilip "hardware exception (code=%u); resetting\n", 180227569Sphilip code); 181227569Sphilip sfxge_schedule_reset(sc); 182227569Sphilip } 183227569Sphilip 184227569Sphilip return (B_FALSE); 185227569Sphilip} 186227569Sphilip 187227569Sphilipstatic boolean_t 188265884Sgnnsfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 189227569Sphilip{ 190227569Sphilip struct sfxge_evq *evq; 191227569Sphilip struct sfxge_softc *sc; 192227569Sphilip struct sfxge_rxq *rxq; 193227569Sphilip unsigned int index; 194265884Sgnn unsigned int label; 195227569Sphilip uint16_t magic; 196227569Sphilip 197227569Sphilip evq = (struct sfxge_evq *)arg; 198280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 199280582Sarybchik 200227569Sphilip sc = evq->sc; 201265884Sgnn rxq = sc->rxq[rxq_index]; 202227569Sphilip 203227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 204227569Sphilip 205227569Sphilip /* Resend a software event on the correct queue */ 206227569Sphilip index = rxq->index; 207284555Sarybchik if (index == evq->index) { 208284555Sarybchik sfxge_rx_qflush_done(rxq); 209284555Sarybchik return (B_FALSE); 210284555Sarybchik } 211284555Sarybchik 212227569Sphilip evq = sc->evq[index]; 213227569Sphilip 214265884Sgnn label = rxq_index; 215227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 216227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 217227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 218227569Sphilip 219227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 220227569Sphilip ("evq not started")); 221227569Sphilip efx_ev_qpost(evq->common, magic); 222227569Sphilip 223227569Sphilip return (B_FALSE); 224227569Sphilip} 225227569Sphilip 226227569Sphilipstatic boolean_t 227265884Sgnnsfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 228227569Sphilip{ 229227569Sphilip struct sfxge_evq *evq; 230227569Sphilip struct sfxge_softc *sc; 231227569Sphilip struct sfxge_rxq *rxq; 232227569Sphilip unsigned int index; 233265884Sgnn unsigned int label; 234227569Sphilip uint16_t magic; 235227569Sphilip 236227569Sphilip evq = (struct sfxge_evq *)arg; 237280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 238280582Sarybchik 239227569Sphilip sc = evq->sc; 240265884Sgnn rxq = sc->rxq[rxq_index]; 241227569Sphilip 242227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 243227569Sphilip 244227569Sphilip /* Resend a software event on the correct queue */ 245227569Sphilip index = rxq->index; 246227569Sphilip evq = sc->evq[index]; 247227569Sphilip 248265884Sgnn label = rxq_index; 249227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 250227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 251227569Sphilip magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 252227569Sphilip 253227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 254227569Sphilip ("evq not started")); 255227569Sphilip efx_ev_qpost(evq->common, magic); 256227569Sphilip 257227569Sphilip return (B_FALSE); 258227569Sphilip} 259227569Sphilip 260265884Sgnnstatic struct sfxge_txq * 261265884Sgnnsfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 262265884Sgnn{ 263265884Sgnn unsigned int index; 264265884Sgnn 265265884Sgnn KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 266265884Sgnn (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 267265884Sgnn index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 268280501Sarybchik return (evq->sc->txq[index]); 269265884Sgnn} 270265884Sgnn 271227569Sphilipstatic boolean_t 272227569Sphilipsfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 273227569Sphilip{ 274227569Sphilip struct sfxge_evq *evq; 275227569Sphilip struct sfxge_txq *txq; 276227569Sphilip unsigned int stop; 277227569Sphilip unsigned int delta; 278227569Sphilip 279227569Sphilip evq = (struct sfxge_evq *)arg; 280280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 281280582Sarybchik 282265884Sgnn txq = sfxge_get_txq_by_label(evq, label); 283227569Sphilip 284227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 285227569Sphilip KASSERT(evq->index == txq->evq_index, 286227569Sphilip ("evq->index != txq->evq_index")); 287227569Sphilip 288280596Sarybchik if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 289227569Sphilip goto done; 290227569Sphilip 291280502Sarybchik stop = (id + 1) & txq->ptr_mask; 292280502Sarybchik id = txq->pending & txq->ptr_mask; 293227569Sphilip 294280502Sarybchik delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 295227569Sphilip txq->pending += delta; 296227569Sphilip 297227569Sphilip evq->tx_done++; 298227569Sphilip 299227569Sphilip if (txq->next == NULL && 300227569Sphilip evq->txqs != &(txq->next)) { 301227569Sphilip *(evq->txqs) = txq; 302227569Sphilip evq->txqs = &(txq->next); 303227569Sphilip } 304227569Sphilip 305227569Sphilip if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 306280513Sarybchik sfxge_tx_qcomplete(txq, evq); 307227569Sphilip 308227569Sphilipdone: 309227569Sphilip return (evq->tx_done >= SFXGE_EV_BATCH); 310227569Sphilip} 311227569Sphilip 312227569Sphilipstatic boolean_t 313265884Sgnnsfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 314227569Sphilip{ 315227569Sphilip struct sfxge_evq *evq; 316227569Sphilip struct sfxge_softc *sc; 317227569Sphilip struct sfxge_txq *txq; 318265884Sgnn unsigned int label; 319227569Sphilip uint16_t magic; 320227569Sphilip 321227569Sphilip evq = (struct sfxge_evq *)arg; 322280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 323280582Sarybchik 324227569Sphilip sc = evq->sc; 325265884Sgnn txq = sc->txq[txq_index]; 326227569Sphilip 327227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 328227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 329227569Sphilip ("txq not initialized")); 330227569Sphilip 331284555Sarybchik if (txq->evq_index == evq->index) { 332284555Sarybchik sfxge_tx_qflush_done(txq); 333284555Sarybchik return (B_FALSE); 334284555Sarybchik } 335284555Sarybchik 336227569Sphilip /* Resend a software event on the correct queue */ 337227569Sphilip evq = sc->evq[txq->evq_index]; 338227569Sphilip 339265884Sgnn label = txq->type; 340227569Sphilip KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 341227569Sphilip ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 342227569Sphilip magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 343227569Sphilip 344227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 345227569Sphilip ("evq not started")); 346227569Sphilip efx_ev_qpost(evq->common, magic); 347227569Sphilip 348227569Sphilip return (B_FALSE); 349227569Sphilip} 350227569Sphilip 351227569Sphilipstatic boolean_t 352227569Sphilipsfxge_ev_software(void *arg, uint16_t magic) 353227569Sphilip{ 354227569Sphilip struct sfxge_evq *evq; 355227569Sphilip struct sfxge_softc *sc; 356227569Sphilip unsigned int label; 357227569Sphilip 358227569Sphilip evq = (struct sfxge_evq *)arg; 359280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 360280582Sarybchik 361227569Sphilip sc = evq->sc; 362227569Sphilip 363227569Sphilip label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 364227569Sphilip magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 365227569Sphilip 366227569Sphilip switch (magic) { 367227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_DONE: { 368227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 369227569Sphilip 370227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 371227569Sphilip KASSERT(evq->index == rxq->index, 372227569Sphilip ("evq->index != rxq->index")); 373227569Sphilip 374227569Sphilip sfxge_rx_qflush_done(rxq); 375227569Sphilip break; 376227569Sphilip } 377227569Sphilip case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 378227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 379227569Sphilip 380227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 381227569Sphilip KASSERT(evq->index == rxq->index, 382227569Sphilip ("evq->index != rxq->index")); 383227569Sphilip 384227569Sphilip sfxge_rx_qflush_failed(rxq); 385227569Sphilip break; 386227569Sphilip } 387227569Sphilip case SFXGE_MAGIC_RX_QREFILL: { 388227569Sphilip struct sfxge_rxq *rxq = sc->rxq[label]; 389227569Sphilip 390227569Sphilip KASSERT(rxq != NULL, ("rxq == NULL")); 391227569Sphilip KASSERT(evq->index == rxq->index, 392227569Sphilip ("evq->index != rxq->index")); 393227569Sphilip 394227569Sphilip sfxge_rx_qrefill(rxq); 395227569Sphilip break; 396227569Sphilip } 397227569Sphilip case SFXGE_MAGIC_TX_QFLUSH_DONE: { 398265884Sgnn struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 399227569Sphilip 400227569Sphilip KASSERT(txq != NULL, ("txq == NULL")); 401227569Sphilip KASSERT(evq->index == txq->evq_index, 402227569Sphilip ("evq->index != txq->evq_index")); 403227569Sphilip 404227569Sphilip sfxge_tx_qflush_done(txq); 405227569Sphilip break; 406227569Sphilip } 407227569Sphilip default: 408227569Sphilip break; 409227569Sphilip } 410227569Sphilip 411227569Sphilip return (B_FALSE); 412227569Sphilip} 413227569Sphilip 414227569Sphilipstatic boolean_t 415227569Sphilipsfxge_ev_sram(void *arg, uint32_t code) 416227569Sphilip{ 417227569Sphilip (void)arg; 418227569Sphilip (void)code; 419227569Sphilip 420227569Sphilip switch (code) { 421227569Sphilip case EFX_SRAM_UPDATE: 422227569Sphilip EFSYS_PROBE(sram_update); 423227569Sphilip break; 424227569Sphilip 425227569Sphilip case EFX_SRAM_CLEAR: 426227569Sphilip EFSYS_PROBE(sram_clear); 427227569Sphilip break; 428227569Sphilip 429227569Sphilip case EFX_SRAM_ILLEGAL_CLEAR: 430227569Sphilip EFSYS_PROBE(sram_illegal_clear); 431227569Sphilip break; 432227569Sphilip 433227569Sphilip default: 434227569Sphilip KASSERT(B_FALSE, ("Impossible SRAM event")); 435227569Sphilip break; 436227569Sphilip } 437227569Sphilip 438227569Sphilip return (B_FALSE); 439227569Sphilip} 440227569Sphilip 441227569Sphilipstatic boolean_t 442227569Sphilipsfxge_ev_timer(void *arg, uint32_t index) 443227569Sphilip{ 444227569Sphilip (void)arg; 445227569Sphilip (void)index; 446227569Sphilip 447227569Sphilip return (B_FALSE); 448227569Sphilip} 449227569Sphilip 450227569Sphilipstatic boolean_t 451227569Sphilipsfxge_ev_wake_up(void *arg, uint32_t index) 452227569Sphilip{ 453227569Sphilip (void)arg; 454227569Sphilip (void)index; 455227569Sphilip 456227569Sphilip return (B_FALSE); 457227569Sphilip} 458227569Sphilip 459280510Sarybchik#if EFSYS_OPT_QSTATS 460280510Sarybchik 461227569Sphilipstatic void 462227569Sphilipsfxge_ev_stat_update(struct sfxge_softc *sc) 463227569Sphilip{ 464227569Sphilip struct sfxge_evq *evq; 465227569Sphilip unsigned int index; 466227569Sphilip clock_t now; 467227569Sphilip 468280522Sarybchik SFXGE_ADAPTER_LOCK(sc); 469227569Sphilip 470280596Sarybchik if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) 471227569Sphilip goto out; 472227569Sphilip 473227569Sphilip now = ticks; 474227569Sphilip if (now - sc->ev_stats_update_time < hz) 475227569Sphilip goto out; 476227569Sphilip 477227569Sphilip sc->ev_stats_update_time = now; 478227569Sphilip 479227569Sphilip /* Add event counts from each event queue in turn */ 480280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 481227569Sphilip evq = sc->evq[index]; 482280522Sarybchik SFXGE_EVQ_LOCK(evq); 483227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 484280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 485227569Sphilip } 486227569Sphilipout: 487280522Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 488227569Sphilip} 489227569Sphilip 490227569Sphilipstatic int 491227569Sphilipsfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 492227569Sphilip{ 493227569Sphilip struct sfxge_softc *sc = arg1; 494227569Sphilip unsigned int id = arg2; 495227569Sphilip 496227569Sphilip sfxge_ev_stat_update(sc); 497227569Sphilip 498280501Sarybchik return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 499227569Sphilip} 500227569Sphilip 501227569Sphilipstatic void 502227569Sphilipsfxge_ev_stat_init(struct sfxge_softc *sc) 503227569Sphilip{ 504227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 505227569Sphilip struct sysctl_oid_list *stat_list; 506227569Sphilip unsigned int id; 507227569Sphilip char name[40]; 508227569Sphilip 509227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 510227569Sphilip 511227569Sphilip for (id = 0; id < EV_NQSTATS; id++) { 512227569Sphilip snprintf(name, sizeof(name), "ev_%s", 513227569Sphilip efx_ev_qstat_name(sc->enp, id)); 514227569Sphilip SYSCTL_ADD_PROC( 515227569Sphilip ctx, stat_list, 516227569Sphilip OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 517227569Sphilip sc, id, sfxge_ev_stat_handler, "Q", 518227569Sphilip ""); 519227569Sphilip } 520227569Sphilip} 521227569Sphilip 522280510Sarybchik#endif /* EFSYS_OPT_QSTATS */ 523280510Sarybchik 524227569Sphilipstatic void 525227569Sphilipsfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 526227569Sphilip{ 527227569Sphilip struct sfxge_evq *evq; 528227569Sphilip efx_evq_t *eep; 529227569Sphilip 530227569Sphilip evq = sc->evq[idx]; 531227569Sphilip eep = evq->common; 532227569Sphilip 533227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 534227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 535227569Sphilip 536227569Sphilip (void)efx_ev_qmoderate(eep, us); 537227569Sphilip} 538227569Sphilip 539227569Sphilipstatic int 540227569Sphilipsfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 541227569Sphilip{ 542227569Sphilip struct sfxge_softc *sc = arg1; 543227569Sphilip struct sfxge_intr *intr = &sc->intr; 544227569Sphilip unsigned int moderation; 545227569Sphilip int error; 546280541Sarybchik unsigned int index; 547227569Sphilip 548280522Sarybchik SFXGE_ADAPTER_LOCK(sc); 549227569Sphilip 550280501Sarybchik if (req->newptr != NULL) { 551227569Sphilip if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 552227569Sphilip != 0) 553227569Sphilip goto out; 554227569Sphilip 555227569Sphilip /* We may not be calling efx_ev_qmoderate() now, 556227569Sphilip * so we have to range-check the value ourselves. 557227569Sphilip */ 558227569Sphilip if (moderation > 559280588Sarybchik efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 560227569Sphilip error = EINVAL; 561227569Sphilip goto out; 562227569Sphilip } 563227569Sphilip 564227569Sphilip sc->ev_moderation = moderation; 565227569Sphilip if (intr->state == SFXGE_INTR_STARTED) { 566280541Sarybchik for (index = 0; index < sc->evq_count; index++) 567227569Sphilip sfxge_ev_qmoderate(sc, index, moderation); 568227569Sphilip } 569227569Sphilip } else { 570227569Sphilip error = SYSCTL_OUT(req, &sc->ev_moderation, 571227569Sphilip sizeof(sc->ev_moderation)); 572227569Sphilip } 573227569Sphilip 574227569Sphilipout: 575280522Sarybchik SFXGE_ADAPTER_UNLOCK(sc); 576227569Sphilip 577280501Sarybchik return (error); 578227569Sphilip} 579227569Sphilip 580227569Sphilipstatic boolean_t 581227569Sphilipsfxge_ev_initialized(void *arg) 582227569Sphilip{ 583227569Sphilip struct sfxge_evq *evq; 584280501Sarybchik 585227569Sphilip evq = (struct sfxge_evq *)arg; 586280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 587227569Sphilip 588284555Sarybchik /* Init done events may be duplicated on 7xxx */ 589284555Sarybchik KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 590284555Sarybchik evq->init_state == SFXGE_EVQ_STARTED, 591227569Sphilip ("evq not starting")); 592227569Sphilip 593227569Sphilip evq->init_state = SFXGE_EVQ_STARTED; 594227569Sphilip 595227569Sphilip return (0); 596227569Sphilip} 597227569Sphilip 598227569Sphilipstatic boolean_t 599227569Sphilipsfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 600227569Sphilip{ 601227569Sphilip struct sfxge_evq *evq; 602227569Sphilip struct sfxge_softc *sc; 603227569Sphilip 604227569Sphilip evq = (struct sfxge_evq *)arg; 605280582Sarybchik SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 606280582Sarybchik 607227569Sphilip sc = evq->sc; 608227569Sphilip 609227569Sphilip sfxge_mac_link_update(sc, link_mode); 610227569Sphilip 611227569Sphilip return (0); 612227569Sphilip} 613227569Sphilip 614227569Sphilipstatic const efx_ev_callbacks_t sfxge_ev_callbacks = { 615227569Sphilip .eec_initialized = sfxge_ev_initialized, 616227569Sphilip .eec_rx = sfxge_ev_rx, 617227569Sphilip .eec_tx = sfxge_ev_tx, 618227569Sphilip .eec_exception = sfxge_ev_exception, 619227569Sphilip .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 620227569Sphilip .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 621227569Sphilip .eec_txq_flush_done = sfxge_ev_txq_flush_done, 622227569Sphilip .eec_software = sfxge_ev_software, 623227569Sphilip .eec_sram = sfxge_ev_sram, 624227569Sphilip .eec_wake_up = sfxge_ev_wake_up, 625227569Sphilip .eec_timer = sfxge_ev_timer, 626227569Sphilip .eec_link_change = sfxge_ev_link_change, 627227569Sphilip}; 628227569Sphilip 629227569Sphilip 630227569Sphilipint 631280508Sarybchiksfxge_ev_qpoll(struct sfxge_evq *evq) 632227569Sphilip{ 633227569Sphilip int rc; 634227569Sphilip 635280522Sarybchik SFXGE_EVQ_LOCK(evq); 636227569Sphilip 637280596Sarybchik if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 638280596Sarybchik evq->init_state != SFXGE_EVQ_STARTED)) { 639227569Sphilip rc = EINVAL; 640227569Sphilip goto fail; 641227569Sphilip } 642227569Sphilip 643227569Sphilip /* Synchronize the DMA memory for reading */ 644227569Sphilip bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 645227569Sphilip BUS_DMASYNC_POSTREAD); 646227569Sphilip 647227569Sphilip KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 648227569Sphilip KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 649227569Sphilip KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 650227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 651227569Sphilip 652227569Sphilip /* Poll the queue */ 653227569Sphilip efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 654227569Sphilip 655227569Sphilip evq->rx_done = 0; 656227569Sphilip evq->tx_done = 0; 657227569Sphilip 658227569Sphilip /* Perform any pending completion processing */ 659227569Sphilip sfxge_ev_qcomplete(evq, B_TRUE); 660227569Sphilip 661227569Sphilip /* Re-prime the event queue for interrupts */ 662227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 663227569Sphilip goto fail; 664227569Sphilip 665280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 666227569Sphilip 667227569Sphilip return (0); 668227569Sphilip 669227569Sphilipfail: 670280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 671227569Sphilip return (rc); 672227569Sphilip} 673227569Sphilip 674227569Sphilipstatic void 675227569Sphilipsfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 676227569Sphilip{ 677227569Sphilip struct sfxge_evq *evq; 678227569Sphilip 679227569Sphilip evq = sc->evq[index]; 680227569Sphilip 681227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 682227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 683227569Sphilip 684280522Sarybchik SFXGE_EVQ_LOCK(evq); 685227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 686227569Sphilip evq->read_ptr = 0; 687227569Sphilip evq->exception = B_FALSE; 688227569Sphilip 689280510Sarybchik#if EFSYS_OPT_QSTATS 690227569Sphilip /* Add event counts before discarding the common evq state */ 691227569Sphilip efx_ev_qstats_update(evq->common, sc->ev_stats); 692280510Sarybchik#endif 693227569Sphilip 694227569Sphilip efx_ev_qdestroy(evq->common); 695227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 696280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 697280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 698227569Sphilip} 699227569Sphilip 700227569Sphilipstatic int 701227569Sphilipsfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 702227569Sphilip{ 703227569Sphilip struct sfxge_evq *evq; 704227569Sphilip efsys_mem_t *esmp; 705227569Sphilip int count; 706227569Sphilip int rc; 707227569Sphilip 708227569Sphilip evq = sc->evq[index]; 709227569Sphilip esmp = &evq->mem; 710227569Sphilip 711227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 712227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 713227569Sphilip 714227569Sphilip /* Clear all events. */ 715280502Sarybchik (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 716227569Sphilip 717227569Sphilip /* Program the buffer table. */ 718227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 719280502Sarybchik EFX_EVQ_NBUFS(evq->entries))) != 0) 720280502Sarybchik return (rc); 721227569Sphilip 722227569Sphilip /* Create the common code event queue. */ 723280502Sarybchik if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 724227569Sphilip evq->buf_base_id, &evq->common)) != 0) 725227569Sphilip goto fail; 726227569Sphilip 727280522Sarybchik SFXGE_EVQ_LOCK(evq); 728227569Sphilip 729227569Sphilip /* Set the default moderation */ 730227569Sphilip (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 731227569Sphilip 732227569Sphilip /* Prime the event queue for interrupts */ 733227569Sphilip if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 734227569Sphilip goto fail2; 735227569Sphilip 736227569Sphilip evq->init_state = SFXGE_EVQ_STARTING; 737227569Sphilip 738280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 739227569Sphilip 740227569Sphilip /* Wait for the initialization event */ 741227569Sphilip count = 0; 742227569Sphilip do { 743227569Sphilip /* Pause for 100 ms */ 744227569Sphilip pause("sfxge evq init", hz / 10); 745227569Sphilip 746227569Sphilip /* Check to see if the test event has been processed */ 747227569Sphilip if (evq->init_state == SFXGE_EVQ_STARTED) 748227569Sphilip goto done; 749227569Sphilip 750227569Sphilip } while (++count < 20); 751227569Sphilip 752227569Sphilip rc = ETIMEDOUT; 753227569Sphilip goto fail3; 754227569Sphilip 755227569Sphilipdone: 756227569Sphilip return (0); 757227569Sphilip 758227569Sphilipfail3: 759280522Sarybchik SFXGE_EVQ_LOCK(evq); 760227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 761227569Sphilipfail2: 762280522Sarybchik SFXGE_EVQ_UNLOCK(evq); 763227569Sphilip efx_ev_qdestroy(evq->common); 764227569Sphilipfail: 765227569Sphilip efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 766280502Sarybchik EFX_EVQ_NBUFS(evq->entries)); 767227569Sphilip 768227569Sphilip return (rc); 769227569Sphilip} 770227569Sphilip 771227569Sphilipvoid 772227569Sphilipsfxge_ev_stop(struct sfxge_softc *sc) 773227569Sphilip{ 774227569Sphilip struct sfxge_intr *intr; 775227569Sphilip efx_nic_t *enp; 776227569Sphilip int index; 777227569Sphilip 778227569Sphilip intr = &sc->intr; 779227569Sphilip enp = sc->enp; 780227569Sphilip 781227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 782227569Sphilip ("Interrupts not started")); 783227569Sphilip 784227569Sphilip /* Stop the event queue(s) */ 785280541Sarybchik index = sc->evq_count; 786227569Sphilip while (--index >= 0) 787227569Sphilip sfxge_ev_qstop(sc, index); 788227569Sphilip 789227569Sphilip /* Tear down the event module */ 790227569Sphilip efx_ev_fini(enp); 791227569Sphilip} 792227569Sphilip 793227569Sphilipint 794227569Sphilipsfxge_ev_start(struct sfxge_softc *sc) 795227569Sphilip{ 796227569Sphilip struct sfxge_intr *intr; 797227569Sphilip int index; 798227569Sphilip int rc; 799227569Sphilip 800227569Sphilip intr = &sc->intr; 801227569Sphilip 802227569Sphilip KASSERT(intr->state == SFXGE_INTR_STARTED, 803227569Sphilip ("intr->state != SFXGE_INTR_STARTED")); 804227569Sphilip 805227569Sphilip /* Initialize the event module */ 806227569Sphilip if ((rc = efx_ev_init(sc->enp)) != 0) 807280501Sarybchik return (rc); 808227569Sphilip 809227569Sphilip /* Start the event queues */ 810280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 811227569Sphilip if ((rc = sfxge_ev_qstart(sc, index)) != 0) 812227569Sphilip goto fail; 813227569Sphilip } 814227569Sphilip 815227569Sphilip return (0); 816227569Sphilip 817227569Sphilipfail: 818227569Sphilip /* Stop the event queue(s) */ 819227569Sphilip while (--index >= 0) 820227569Sphilip sfxge_ev_qstop(sc, index); 821227569Sphilip 822227569Sphilip /* Tear down the event module */ 823227569Sphilip efx_ev_fini(sc->enp); 824227569Sphilip 825227569Sphilip return (rc); 826227569Sphilip} 827227569Sphilip 828227569Sphilipstatic void 829227569Sphilipsfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 830227569Sphilip{ 831227569Sphilip struct sfxge_evq *evq; 832227569Sphilip 833227569Sphilip evq = sc->evq[index]; 834227569Sphilip 835227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 836227569Sphilip ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 837227569Sphilip KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 838227569Sphilip 839227569Sphilip sfxge_dma_free(&evq->mem); 840227569Sphilip 841227569Sphilip sc->evq[index] = NULL; 842227569Sphilip 843280522Sarybchik SFXGE_EVQ_LOCK_DESTROY(evq); 844227569Sphilip 845227569Sphilip free(evq, M_SFXGE); 846227569Sphilip} 847227569Sphilip 848227569Sphilipstatic int 849227569Sphilipsfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 850227569Sphilip{ 851227569Sphilip struct sfxge_evq *evq; 852227569Sphilip efsys_mem_t *esmp; 853227569Sphilip int rc; 854227569Sphilip 855227569Sphilip KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 856227569Sphilip 857227569Sphilip evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 858227569Sphilip evq->sc = sc; 859227569Sphilip evq->index = index; 860227569Sphilip sc->evq[index] = evq; 861227569Sphilip esmp = &evq->mem; 862227569Sphilip 863280502Sarybchik /* Build an event queue with room for one event per tx and rx buffer, 864280502Sarybchik * plus some extra for link state events and MCDI completions. 865280502Sarybchik * There are three tx queues in the first event queue and one in 866280502Sarybchik * other. 867280502Sarybchik */ 868280502Sarybchik if (index == 0) 869280502Sarybchik evq->entries = 870280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 871280502Sarybchik 3 * sc->txq_entries + 872280502Sarybchik 128); 873280502Sarybchik else 874280502Sarybchik evq->entries = 875280502Sarybchik ROUNDUP_POW_OF_TWO(sc->rxq_entries + 876280502Sarybchik sc->txq_entries + 877280502Sarybchik 128); 878280502Sarybchik 879227569Sphilip /* Initialise TX completion list */ 880227569Sphilip evq->txqs = &evq->txq; 881227569Sphilip 882227569Sphilip /* Allocate DMA space. */ 883280502Sarybchik if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 884227569Sphilip return (rc); 885227569Sphilip 886227569Sphilip /* Allocate buffer table entries. */ 887280502Sarybchik sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 888227569Sphilip &evq->buf_base_id); 889227569Sphilip 890280524Sarybchik SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 891227569Sphilip 892227569Sphilip evq->init_state = SFXGE_EVQ_INITIALIZED; 893227569Sphilip 894227569Sphilip return (0); 895227569Sphilip} 896227569Sphilip 897227569Sphilipvoid 898227569Sphilipsfxge_ev_fini(struct sfxge_softc *sc) 899227569Sphilip{ 900227569Sphilip struct sfxge_intr *intr; 901227569Sphilip int index; 902227569Sphilip 903227569Sphilip intr = &sc->intr; 904227569Sphilip 905227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 906227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 907227569Sphilip 908227569Sphilip sc->ev_moderation = 0; 909227569Sphilip 910227569Sphilip /* Tear down the event queue(s). */ 911280541Sarybchik index = sc->evq_count; 912227569Sphilip while (--index >= 0) 913227569Sphilip sfxge_ev_qfini(sc, index); 914280541Sarybchik 915280541Sarybchik sc->evq_count = 0; 916227569Sphilip} 917227569Sphilip 918227569Sphilipint 919227569Sphilipsfxge_ev_init(struct sfxge_softc *sc) 920227569Sphilip{ 921227569Sphilip struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 922227569Sphilip struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 923227569Sphilip struct sfxge_intr *intr; 924227569Sphilip int index; 925227569Sphilip int rc; 926227569Sphilip 927227569Sphilip intr = &sc->intr; 928227569Sphilip 929280541Sarybchik sc->evq_count = intr->n_alloc; 930280541Sarybchik 931227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 932227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 933227569Sphilip 934227569Sphilip /* Set default interrupt moderation; add a sysctl to 935227569Sphilip * read and change it. 936227569Sphilip */ 937280517Sarybchik sc->ev_moderation = SFXGE_MODERATION; 938227569Sphilip SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 939227569Sphilip OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 940227569Sphilip sc, 0, sfxge_int_mod_handler, "IU", 941227569Sphilip "sfxge interrupt moderation (us)"); 942227569Sphilip 943227569Sphilip /* 944227569Sphilip * Initialize the event queue(s) - one per interrupt. 945227569Sphilip */ 946280541Sarybchik for (index = 0; index < sc->evq_count; index++) { 947227569Sphilip if ((rc = sfxge_ev_qinit(sc, index)) != 0) 948227569Sphilip goto fail; 949227569Sphilip } 950227569Sphilip 951280510Sarybchik#if EFSYS_OPT_QSTATS 952227569Sphilip sfxge_ev_stat_init(sc); 953280510Sarybchik#endif 954227569Sphilip 955227569Sphilip return (0); 956227569Sphilip 957227569Sphilipfail: 958227569Sphilip while (--index >= 0) 959227569Sphilip sfxge_ev_qfini(sc, index); 960227569Sphilip 961280541Sarybchik sc->evq_count = 0; 962227569Sphilip return (rc); 963227569Sphilip} 964