cxgb_sge.c revision 205949
1167514Skmacy/************************************************************************** 2167514Skmacy 3189643SgnnCopyright (c) 2007-2009, Chelsio Inc. 4167514SkmacyAll rights reserved. 5167514Skmacy 6167514SkmacyRedistribution and use in source and binary forms, with or without 7167514Skmacymodification, are permitted provided that the following conditions are met: 8167514Skmacy 9167514Skmacy 1. Redistributions of source code must retain the above copyright notice, 10167514Skmacy this list of conditions and the following disclaimer. 11167514Skmacy 12169978Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13167514Skmacy contributors may be used to endorse or promote products derived from 14167514Skmacy this software without specific prior written permission. 15169978Skmacy 16167514SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17167514SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18167514SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19167514SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20167514SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21167514SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22167514SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23167514SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24167514SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25167514SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26167514SkmacyPOSSIBILITY OF SUCH DAMAGE. 27167514Skmacy 28167514Skmacy***************************************************************************/ 29167514Skmacy 30167514Skmacy#include <sys/cdefs.h> 31167514Skmacy__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_sge.c 205949 2010-03-31 00:26:35Z np $"); 32167514Skmacy 33205947Snp#include "opt_inet.h" 34205947Snp 35167514Skmacy#include <sys/param.h> 36167514Skmacy#include <sys/systm.h> 37167514Skmacy#include <sys/kernel.h> 38167514Skmacy#include <sys/module.h> 39167514Skmacy#include <sys/bus.h> 40167514Skmacy#include <sys/conf.h> 41167514Skmacy#include <machine/bus.h> 42167514Skmacy#include <machine/resource.h> 43167514Skmacy#include <sys/bus_dma.h> 44167514Skmacy#include <sys/rman.h> 45167514Skmacy#include <sys/queue.h> 46167514Skmacy#include <sys/sysctl.h> 47167514Skmacy#include <sys/taskqueue.h> 48167514Skmacy 49167514Skmacy#include <sys/proc.h> 50175200Skmacy#include <sys/sbuf.h> 51167514Skmacy#include <sys/sched.h> 52167514Skmacy#include <sys/smp.h> 53167760Skmacy#include <sys/systm.h> 54174708Skmacy#include <sys/syslog.h> 55204348Snp#include <sys/socket.h> 56167514Skmacy 57194521Skmacy#include <net/bpf.h> 58204348Snp#include <net/ethernet.h> 59204348Snp#include <net/if.h> 60204348Snp#include <net/if_vlan_var.h> 61194521Skmacy 62167514Skmacy#include <netinet/in_systm.h> 63167514Skmacy#include <netinet/in.h> 64167514Skmacy#include <netinet/ip.h> 65167514Skmacy#include <netinet/tcp.h> 66167514Skmacy 67167514Skmacy#include <dev/pci/pcireg.h> 68167514Skmacy#include <dev/pci/pcivar.h> 69167514Skmacy 70174670Skmacy#include <vm/vm.h> 71174708Skmacy#include <vm/pmap.h> 72174672Skmacy 73170076Skmacy#include <cxgb_include.h> 74174670Skmacy#include <sys/mvec.h> 75168491Skmacy 76194521Skmacyint txq_fills = 0; 77194521Skmacyint multiq_tx_enable = 1; 78194521Skmacy 79194521Skmacyextern struct sysctl_oid_list sysctl__hw_cxgb_children; 80194521Skmacyint cxgb_txq_buf_ring_size = TX_ETH_Q_SIZE; 81194521SkmacyTUNABLE_INT("hw.cxgb.txq_mr_size", &cxgb_txq_buf_ring_size); 82194521SkmacySYSCTL_UINT(_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_buf_ring_size, 0, 83194521Skmacy "size of per-queue mbuf ring"); 84194521Skmacy 85194521Skmacystatic int cxgb_tx_coalesce_force = 0; 86194521SkmacyTUNABLE_INT("hw.cxgb.tx_coalesce_force", &cxgb_tx_coalesce_force); 87194521SkmacySYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce_force, CTLFLAG_RW, 88194521Skmacy &cxgb_tx_coalesce_force, 0, 89194521Skmacy "coalesce small packets into a single work request regardless of ring state"); 90194521Skmacy 91194521Skmacy#define COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1 92194521Skmacy#define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3)) 93194521Skmacy#define COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2 94194521Skmacy#define COALESCE_STOP_MIN TX_ETH_Q_SIZE>>5 95194521Skmacy#define TX_RECLAIM_DEFAULT TX_ETH_Q_SIZE>>5 96194521Skmacy#define TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2 97194521Skmacy#define TX_RECLAIM_MIN TX_ETH_Q_SIZE>>6 98194521Skmacy 99194521Skmacy 100194521Skmacystatic int cxgb_tx_coalesce_enable_start = COALESCE_START_DEFAULT; 101194521SkmacyTUNABLE_INT("hw.cxgb.tx_coalesce_enable_start", 102194521Skmacy &cxgb_tx_coalesce_enable_start); 103194521SkmacySYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_start, CTLFLAG_RW, 104194521Skmacy &cxgb_tx_coalesce_enable_start, 0, 105194521Skmacy "coalesce enable threshold"); 106194521Skmacystatic int cxgb_tx_coalesce_enable_stop = COALESCE_STOP_DEFAULT; 107194521SkmacyTUNABLE_INT("hw.cxgb.tx_coalesce_enable_stop", &cxgb_tx_coalesce_enable_stop); 108194521SkmacySYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_stop, CTLFLAG_RW, 109194521Skmacy &cxgb_tx_coalesce_enable_stop, 0, 110194521Skmacy "coalesce disable threshold"); 111194521Skmacystatic int cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT; 112194521SkmacyTUNABLE_INT("hw.cxgb.tx_reclaim_threshold", &cxgb_tx_reclaim_threshold); 113194521SkmacySYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_reclaim_threshold, CTLFLAG_RW, 114194521Skmacy &cxgb_tx_reclaim_threshold, 0, 115194521Skmacy "tx cleaning minimum threshold"); 116194521Skmacy 117176472Skmacy/* 118176472Skmacy * XXX don't re-enable this until TOE stops assuming 119176472Skmacy * we have an m_ext 120176472Skmacy */ 121176472Skmacystatic int recycle_enable = 0; 122176472Skmacyint cxgb_ext_freed = 0; 123176472Skmacyint cxgb_ext_inited = 0; 124177464Skmacyint fl_q_size = 0; 125177464Skmacyint jumbo_q_size = 0; 126177464Skmacy 127175200Skmacyextern int cxgb_use_16k_clusters; 128177464Skmacyextern int nmbjumbo4; 129177464Skmacyextern int nmbjumbo9; 130177464Skmacyextern int nmbjumbo16; 131168737Skmacy 132167514Skmacy#define USE_GTS 0 133167514Skmacy 134167514Skmacy#define SGE_RX_SM_BUF_SIZE 1536 135167514Skmacy#define SGE_RX_DROP_THRES 16 136169978Skmacy#define SGE_RX_COPY_THRES 128 137167514Skmacy 138167514Skmacy/* 139167514Skmacy * Period of the Tx buffer reclaim timer. This timer does not need to run 140167514Skmacy * frequently as Tx buffers are usually reclaimed by new Tx packets. 141167514Skmacy */ 142170038Skmacy#define TX_RECLAIM_PERIOD (hz >> 1) 143167514Skmacy 144167514Skmacy/* 145167514Skmacy * Values for sge_txq.flags 146167514Skmacy */ 147167514Skmacyenum { 148167514Skmacy TXQ_RUNNING = 1 << 0, /* fetch engine is running */ 149167514Skmacy TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */ 150167514Skmacy}; 151167514Skmacy 152167514Skmacystruct tx_desc { 153167514Skmacy uint64_t flit[TX_DESC_FLITS]; 154167514Skmacy} __packed; 155167514Skmacy 156167514Skmacystruct rx_desc { 157167514Skmacy uint32_t addr_lo; 158167514Skmacy uint32_t len_gen; 159167514Skmacy uint32_t gen2; 160167514Skmacy uint32_t addr_hi; 161201758Smbr} __packed; 162167514Skmacy 163167514Skmacystruct rsp_desc { /* response queue descriptor */ 164167514Skmacy struct rss_header rss_hdr; 165167514Skmacy uint32_t flags; 166167514Skmacy uint32_t len_cq; 167167514Skmacy uint8_t imm_data[47]; 168167514Skmacy uint8_t intr_gen; 169167514Skmacy} __packed; 170167514Skmacy 171167514Skmacy#define RX_SW_DESC_MAP_CREATED (1 << 0) 172168737Skmacy#define TX_SW_DESC_MAP_CREATED (1 << 1) 173167514Skmacy#define RX_SW_DESC_INUSE (1 << 3) 174167514Skmacy#define TX_SW_DESC_MAPPED (1 << 4) 175167514Skmacy 176167514Skmacy#define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0) 177167514Skmacy#define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP) 178167514Skmacy#define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP) 179167514Skmacy#define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP) 180167514Skmacy 181167514Skmacystruct tx_sw_desc { /* SW state per Tx descriptor */ 182194521Skmacy struct mbuf *m; 183167514Skmacy bus_dmamap_t map; 184167514Skmacy int flags; 185167514Skmacy}; 186167514Skmacy 187167514Skmacystruct rx_sw_desc { /* SW state per Rx descriptor */ 188194521Skmacy caddr_t rxsd_cl; 189194521Skmacy struct mbuf *m; 190194521Skmacy bus_dmamap_t map; 191194521Skmacy int flags; 192167514Skmacy}; 193167514Skmacy 194167514Skmacystruct txq_state { 195194521Skmacy unsigned int compl; 196194521Skmacy unsigned int gen; 197194521Skmacy unsigned int pidx; 198167514Skmacy}; 199167514Skmacy 200168351Skmacystruct refill_fl_cb_arg { 201168351Skmacy int error; 202168351Skmacy bus_dma_segment_t seg; 203168351Skmacy int nseg; 204168351Skmacy}; 205168351Skmacy 206194521Skmacy 207167514Skmacy/* 208167514Skmacy * Maps a number of flits to the number of Tx descriptors that can hold them. 209167514Skmacy * The formula is 210167514Skmacy * 211167514Skmacy * desc = 1 + (flits - 2) / (WR_FLITS - 1). 212167514Skmacy * 213167514Skmacy * HW allows up to 4 descriptors to be combined into a WR. 214167514Skmacy */ 215167514Skmacystatic uint8_t flit_desc_map[] = { 216167514Skmacy 0, 217167514Skmacy#if SGE_NUM_GENBITS == 1 218167514Skmacy 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 219167514Skmacy 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 220167514Skmacy 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 221167514Skmacy 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 222167514Skmacy#elif SGE_NUM_GENBITS == 2 223167514Skmacy 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 224167514Skmacy 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 225167514Skmacy 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 226167514Skmacy 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 227167514Skmacy#else 228167514Skmacy# error "SGE_NUM_GENBITS must be 1 or 2" 229167514Skmacy#endif 230167514Skmacy}; 231167514Skmacy 232194521Skmacy#define TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED) 233194521Skmacy#define TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock) 234194521Skmacy#define TXQ_LOCK(qs) mtx_lock(&(qs)->lock) 235194521Skmacy#define TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock) 236194521Skmacy#define TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) 237203834Smlaier#define TXQ_RING_NEEDS_ENQUEUE(qs) \ 238203834Smlaier drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) 239194521Skmacy#define TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) 240194521Skmacy#define TXQ_RING_DEQUEUE_COND(qs, func, arg) \ 241194521Skmacy drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg) 242194521Skmacy#define TXQ_RING_DEQUEUE(qs) \ 243194521Skmacy drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) 244167514Skmacy 245167514Skmacyint cxgb_debug = 0; 246167514Skmacy 247167514Skmacystatic void sge_timer_cb(void *arg); 248167514Skmacystatic void sge_timer_reclaim(void *arg, int ncount); 249171335Skmacystatic void sge_txq_reclaim_handler(void *arg, int ncount); 250194521Skmacystatic void cxgb_start_locked(struct sge_qset *qs); 251167514Skmacy 252194521Skmacy/* 253194521Skmacy * XXX need to cope with bursty scheduling by looking at a wider 254194521Skmacy * window than we are now for determining the need for coalescing 255194521Skmacy * 256194521Skmacy */ 257194521Skmacystatic __inline uint64_t 258194521Skmacycheck_pkt_coalesce(struct sge_qset *qs) 259194521Skmacy{ 260194521Skmacy struct adapter *sc; 261194521Skmacy struct sge_txq *txq; 262194521Skmacy uint8_t *fill; 263194521Skmacy 264194521Skmacy if (__predict_false(cxgb_tx_coalesce_force)) 265194521Skmacy return (1); 266194521Skmacy txq = &qs->txq[TXQ_ETH]; 267194521Skmacy sc = qs->port->adapter; 268194521Skmacy fill = &sc->tunq_fill[qs->idx]; 269194521Skmacy 270194521Skmacy if (cxgb_tx_coalesce_enable_start > COALESCE_START_MAX) 271194521Skmacy cxgb_tx_coalesce_enable_start = COALESCE_START_MAX; 272194521Skmacy if (cxgb_tx_coalesce_enable_stop < COALESCE_STOP_MIN) 273194521Skmacy cxgb_tx_coalesce_enable_start = COALESCE_STOP_MIN; 274194521Skmacy /* 275194521Skmacy * if the hardware transmit queue is more than 1/8 full 276194521Skmacy * we mark it as coalescing - we drop back from coalescing 277194521Skmacy * when we go below 1/32 full and there are no packets enqueued, 278194521Skmacy * this provides us with some degree of hysteresis 279194521Skmacy */ 280194521Skmacy if (*fill != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) && 281194521Skmacy TXQ_RING_EMPTY(qs) && (qs->coalescing == 0)) 282194521Skmacy *fill = 0; 283194521Skmacy else if (*fill == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start)) 284194521Skmacy *fill = 1; 285194521Skmacy 286194521Skmacy return (sc->tunq_coalesce); 287194521Skmacy} 288194521Skmacy 289194521Skmacy#ifdef __LP64__ 290194521Skmacystatic void 291194521Skmacyset_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo) 292194521Skmacy{ 293194521Skmacy uint64_t wr_hilo; 294194521Skmacy#if _BYTE_ORDER == _LITTLE_ENDIAN 295194521Skmacy wr_hilo = wr_hi; 296194521Skmacy wr_hilo |= (((uint64_t)wr_lo)<<32); 297194521Skmacy#else 298194521Skmacy wr_hilo = wr_lo; 299194521Skmacy wr_hilo |= (((uint64_t)wr_hi)<<32); 300194521Skmacy#endif 301194521Skmacy wrp->wrh_hilo = wr_hilo; 302194521Skmacy} 303194521Skmacy#else 304194521Skmacystatic void 305194521Skmacyset_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo) 306194521Skmacy{ 307194521Skmacy 308194521Skmacy wrp->wrh_hi = wr_hi; 309194521Skmacy wmb(); 310194521Skmacy wrp->wrh_lo = wr_lo; 311194521Skmacy} 312194521Skmacy#endif 313194521Skmacy 314194521Skmacystruct coalesce_info { 315194521Skmacy int count; 316194521Skmacy int nbytes; 317194521Skmacy}; 318194521Skmacy 319194521Skmacystatic int 320194521Skmacycoalesce_check(struct mbuf *m, void *arg) 321194521Skmacy{ 322194521Skmacy struct coalesce_info *ci = arg; 323194521Skmacy int *count = &ci->count; 324194521Skmacy int *nbytes = &ci->nbytes; 325194521Skmacy 326194521Skmacy if ((*nbytes == 0) || ((*nbytes + m->m_len <= 10500) && 327194521Skmacy (*count < 7) && (m->m_next == NULL))) { 328194521Skmacy *count += 1; 329194521Skmacy *nbytes += m->m_len; 330194521Skmacy return (1); 331194521Skmacy } 332194521Skmacy return (0); 333194521Skmacy} 334194521Skmacy 335194521Skmacystatic struct mbuf * 336194521Skmacycxgb_dequeue(struct sge_qset *qs) 337194521Skmacy{ 338194521Skmacy struct mbuf *m, *m_head, *m_tail; 339194521Skmacy struct coalesce_info ci; 340194521Skmacy 341194521Skmacy 342194521Skmacy if (check_pkt_coalesce(qs) == 0) 343194521Skmacy return TXQ_RING_DEQUEUE(qs); 344194521Skmacy 345194521Skmacy m_head = m_tail = NULL; 346194521Skmacy ci.count = ci.nbytes = 0; 347194521Skmacy do { 348194521Skmacy m = TXQ_RING_DEQUEUE_COND(qs, coalesce_check, &ci); 349194521Skmacy if (m_head == NULL) { 350194521Skmacy m_tail = m_head = m; 351194521Skmacy } else if (m != NULL) { 352194521Skmacy m_tail->m_nextpkt = m; 353194521Skmacy m_tail = m; 354194521Skmacy } 355194521Skmacy } while (m != NULL); 356194521Skmacy if (ci.count > 7) 357194521Skmacy panic("trying to coalesce %d packets in to one WR", ci.count); 358194521Skmacy return (m_head); 359194521Skmacy} 360194521Skmacy 361167514Skmacy/** 362167514Skmacy * reclaim_completed_tx - reclaims completed Tx descriptors 363167514Skmacy * @adapter: the adapter 364167514Skmacy * @q: the Tx queue to reclaim completed descriptors from 365167514Skmacy * 366167514Skmacy * Reclaims Tx descriptors that the SGE has indicated it has processed, 367167514Skmacy * and frees the associated buffers if possible. Called with the Tx 368167514Skmacy * queue's lock held. 369167514Skmacy */ 370167514Skmacystatic __inline int 371194521Skmacyreclaim_completed_tx(struct sge_qset *qs, int reclaim_min, int queue) 372167514Skmacy{ 373194521Skmacy struct sge_txq *q = &qs->txq[queue]; 374174708Skmacy int reclaim = desc_reclaimable(q); 375167514Skmacy 376194521Skmacy if ((cxgb_tx_reclaim_threshold > TX_RECLAIM_MAX) || 377194521Skmacy (cxgb_tx_reclaim_threshold < TX_RECLAIM_MIN)) 378194521Skmacy cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT; 379194521Skmacy 380175224Skmacy if (reclaim < reclaim_min) 381175224Skmacy return (0); 382194521Skmacy 383194521Skmacy mtx_assert(&qs->lock, MA_OWNED); 384167514Skmacy if (reclaim > 0) { 385194521Skmacy t3_free_tx_desc(qs, reclaim, queue); 386174708Skmacy q->cleaned += reclaim; 387174708Skmacy q->in_use -= reclaim; 388194521Skmacy } 389194521Skmacy if (isset(&qs->txq_stopped, TXQ_ETH)) 390194521Skmacy clrbit(&qs->txq_stopped, TXQ_ETH); 391194521Skmacy 392174708Skmacy return (reclaim); 393167514Skmacy} 394167514Skmacy 395167514Skmacy/** 396169978Skmacy * should_restart_tx - are there enough resources to restart a Tx queue? 397169978Skmacy * @q: the Tx queue 398169978Skmacy * 399169978Skmacy * Checks if there are enough descriptors to restart a suspended Tx queue. 400169978Skmacy */ 401169978Skmacystatic __inline int 402169978Skmacyshould_restart_tx(const struct sge_txq *q) 403169978Skmacy{ 404169978Skmacy unsigned int r = q->processed - q->cleaned; 405169978Skmacy 406169978Skmacy return q->in_use - r < (q->size >> 1); 407169978Skmacy} 408169978Skmacy 409169978Skmacy/** 410167514Skmacy * t3_sge_init - initialize SGE 411167514Skmacy * @adap: the adapter 412167514Skmacy * @p: the SGE parameters 413167514Skmacy * 414167514Skmacy * Performs SGE initialization needed every time after a chip reset. 415167514Skmacy * We do not initialize any of the queue sets here, instead the driver 416167514Skmacy * top-level must request those individually. We also do not enable DMA 417167514Skmacy * here, that should be done after the queues have been set up. 418167514Skmacy */ 419167514Skmacyvoid 420167514Skmacyt3_sge_init(adapter_t *adap, struct sge_params *p) 421167514Skmacy{ 422167514Skmacy u_int ctrl, ups; 423167514Skmacy 424167514Skmacy ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ 425167514Skmacy 426167514Skmacy ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | 427176472Skmacy F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN | 428167514Skmacy V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | 429167514Skmacy V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; 430167514Skmacy#if SGE_NUM_GENBITS == 1 431167514Skmacy ctrl |= F_EGRGENCTRL; 432167514Skmacy#endif 433167514Skmacy if (adap->params.rev > 0) { 434167514Skmacy if (!(adap->flags & (USING_MSIX | USING_MSI))) 435167514Skmacy ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ; 436167514Skmacy } 437167514Skmacy t3_write_reg(adap, A_SG_CONTROL, ctrl); 438167514Skmacy t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) | 439167514Skmacy V_LORCQDRBTHRSH(512)); 440167514Skmacy t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10); 441167514Skmacy t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) | 442167514Skmacy V_TIMEOUT(200 * core_ticks_per_usec(adap))); 443176472Skmacy t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 444176472Skmacy adap->params.rev < T3_REV_C ? 1000 : 500); 445167514Skmacy t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256); 446167514Skmacy t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000); 447167514Skmacy t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256); 448167514Skmacy t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff)); 449167514Skmacy t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024); 450167514Skmacy} 451167514Skmacy 452167514Skmacy 453167514Skmacy/** 454167514Skmacy * sgl_len - calculates the size of an SGL of the given capacity 455167514Skmacy * @n: the number of SGL entries 456167514Skmacy * 457167514Skmacy * Calculates the number of flits needed for a scatter/gather list that 458167514Skmacy * can hold the given number of entries. 459167514Skmacy */ 460167514Skmacystatic __inline unsigned int 461167514Skmacysgl_len(unsigned int n) 462167514Skmacy{ 463167514Skmacy return ((3 * n) / 2 + (n & 1)); 464167514Skmacy} 465167514Skmacy 466167514Skmacy/** 467167514Skmacy * get_imm_packet - return the next ingress packet buffer from a response 468167514Skmacy * @resp: the response descriptor containing the packet data 469167514Skmacy * 470167514Skmacy * Return a packet containing the immediate data of the given response. 471167514Skmacy */ 472171471Skmacystatic int 473176472Skmacyget_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m) 474167514Skmacy{ 475174708Skmacy 476176472Skmacy m->m_len = m->m_pkthdr.len = IMMED_PKT_SIZE; 477176472Skmacy m->m_ext.ext_buf = NULL; 478176472Skmacy m->m_ext.ext_type = 0; 479174708Skmacy memcpy(mtod(m, uint8_t *), resp->imm_data, IMMED_PKT_SIZE); 480176472Skmacy return (0); 481167514Skmacy} 482167514Skmacy 483167514Skmacystatic __inline u_int 484167514Skmacyflits_to_desc(u_int n) 485167514Skmacy{ 486167514Skmacy return (flit_desc_map[n]); 487167514Skmacy} 488167514Skmacy 489176472Skmacy#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \ 490176472Skmacy F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \ 491176472Skmacy V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \ 492176472Skmacy F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \ 493176472Skmacy F_HIRCQPARITYERROR) 494176472Skmacy#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR) 495176472Skmacy#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \ 496176472Skmacy F_RSPQDISABLED) 497176472Skmacy 498176472Skmacy/** 499176472Skmacy * t3_sge_err_intr_handler - SGE async event interrupt handler 500176472Skmacy * @adapter: the adapter 501176472Skmacy * 502176472Skmacy * Interrupt handler for SGE asynchronous (non-data) events. 503176472Skmacy */ 504167514Skmacyvoid 505167514Skmacyt3_sge_err_intr_handler(adapter_t *adapter) 506167514Skmacy{ 507167514Skmacy unsigned int v, status; 508167514Skmacy 509167514Skmacy status = t3_read_reg(adapter, A_SG_INT_CAUSE); 510176472Skmacy if (status & SGE_PARERR) 511176472Skmacy CH_ALERT(adapter, "SGE parity error (0x%x)\n", 512176472Skmacy status & SGE_PARERR); 513176472Skmacy if (status & SGE_FRAMINGERR) 514176472Skmacy CH_ALERT(adapter, "SGE framing error (0x%x)\n", 515176472Skmacy status & SGE_FRAMINGERR); 516167514Skmacy if (status & F_RSPQCREDITOVERFOW) 517167514Skmacy CH_ALERT(adapter, "SGE response queue credit overflow\n"); 518167514Skmacy 519167514Skmacy if (status & F_RSPQDISABLED) { 520167514Skmacy v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS); 521167514Skmacy 522167514Skmacy CH_ALERT(adapter, 523167514Skmacy "packet delivered to disabled response queue (0x%x)\n", 524167514Skmacy (v >> S_RSPQ0DISABLED) & 0xff); 525167514Skmacy } 526167514Skmacy 527167514Skmacy t3_write_reg(adapter, A_SG_INT_CAUSE, status); 528176472Skmacy if (status & SGE_FATALERR) 529167514Skmacy t3_fatal_err(adapter); 530167514Skmacy} 531167514Skmacy 532167514Skmacyvoid 533167514Skmacyt3_sge_prep(adapter_t *adap, struct sge_params *p) 534167514Skmacy{ 535177464Skmacy int i, nqsets; 536167514Skmacy 537177464Skmacy nqsets = min(SGE_QSETS, mp_ncpus*4); 538177464Skmacy 539177464Skmacy fl_q_size = min(nmbclusters/(3*nqsets), FL_Q_SIZE); 540177464Skmacy 541177464Skmacy while (!powerof2(fl_q_size)) 542177464Skmacy fl_q_size--; 543183059Skmacy#if __FreeBSD_version >= 700111 544177464Skmacy if (cxgb_use_16k_clusters) 545177464Skmacy jumbo_q_size = min(nmbjumbo16/(3*nqsets), JUMBO_Q_SIZE); 546177464Skmacy else 547177464Skmacy jumbo_q_size = min(nmbjumbo9/(3*nqsets), JUMBO_Q_SIZE); 548177464Skmacy#else 549177464Skmacy jumbo_q_size = min(nmbjumbo4/(3*nqsets), JUMBO_Q_SIZE); 550177464Skmacy#endif 551177464Skmacy while (!powerof2(jumbo_q_size)) 552202678Snp jumbo_q_size--; 553202678Snp 554202678Snp if (fl_q_size < (FL_Q_SIZE / 4) || jumbo_q_size < (JUMBO_Q_SIZE / 2)) 555202678Snp device_printf(adap->dev, 556202678Snp "Insufficient clusters and/or jumbo buffers.\n"); 557202678Snp 558167514Skmacy /* XXX Does ETHER_ALIGN need to be accounted for here? */ 559175200Skmacy p->max_pkt_size = adap->sge.qs[0].fl[1].buf_size - sizeof(struct cpl_rx_data); 560167514Skmacy 561167514Skmacy for (i = 0; i < SGE_QSETS; ++i) { 562167514Skmacy struct qset_params *q = p->qset + i; 563167514Skmacy 564174708Skmacy if (adap->params.nports > 2) { 565180583Skmacy q->coalesce_usecs = 50; 566174708Skmacy } else { 567174708Skmacy#ifdef INVARIANTS 568180583Skmacy q->coalesce_usecs = 10; 569174708Skmacy#else 570180583Skmacy q->coalesce_usecs = 5; 571174708Skmacy#endif 572174708Skmacy } 573182679Skmacy q->polling = 0; 574167514Skmacy q->rspq_size = RSPQ_Q_SIZE; 575177464Skmacy q->fl_size = fl_q_size; 576177464Skmacy q->jumbo_size = jumbo_q_size; 577167514Skmacy q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; 578167514Skmacy q->txq_size[TXQ_OFLD] = 1024; 579167514Skmacy q->txq_size[TXQ_CTRL] = 256; 580167514Skmacy q->cong_thres = 0; 581167514Skmacy } 582167514Skmacy} 583167514Skmacy 584167514Skmacyint 585167514Skmacyt3_sge_alloc(adapter_t *sc) 586167514Skmacy{ 587167514Skmacy 588167514Skmacy /* The parent tag. */ 589167514Skmacy if (bus_dma_tag_create( NULL, /* parent */ 590167514Skmacy 1, 0, /* algnmnt, boundary */ 591167514Skmacy BUS_SPACE_MAXADDR, /* lowaddr */ 592167514Skmacy BUS_SPACE_MAXADDR, /* highaddr */ 593167514Skmacy NULL, NULL, /* filter, filterarg */ 594167514Skmacy BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 595167514Skmacy BUS_SPACE_UNRESTRICTED, /* nsegments */ 596167514Skmacy BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 597167514Skmacy 0, /* flags */ 598167514Skmacy NULL, NULL, /* lock, lockarg */ 599167514Skmacy &sc->parent_dmat)) { 600167514Skmacy device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); 601167514Skmacy return (ENOMEM); 602167514Skmacy } 603167514Skmacy 604167514Skmacy /* 605167514Skmacy * DMA tag for normal sized RX frames 606167514Skmacy */ 607167514Skmacy if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, 608167514Skmacy BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, 609167514Skmacy MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { 610167514Skmacy device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); 611167514Skmacy return (ENOMEM); 612167514Skmacy } 613167514Skmacy 614167514Skmacy /* 615167514Skmacy * DMA tag for jumbo sized RX frames. 616167514Skmacy */ 617175200Skmacy if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR, 618175200Skmacy BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES, 619167514Skmacy BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { 620167514Skmacy device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); 621167514Skmacy return (ENOMEM); 622167514Skmacy } 623167514Skmacy 624167514Skmacy /* 625167514Skmacy * DMA tag for TX frames. 626167514Skmacy */ 627167514Skmacy if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, 628167514Skmacy BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, 629167514Skmacy TX_MAX_SIZE, BUS_DMA_ALLOCNOW, 630167514Skmacy NULL, NULL, &sc->tx_dmat)) { 631167514Skmacy device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); 632167514Skmacy return (ENOMEM); 633167514Skmacy } 634167514Skmacy 635167514Skmacy return (0); 636167514Skmacy} 637167514Skmacy 638167514Skmacyint 639167514Skmacyt3_sge_free(struct adapter * sc) 640167514Skmacy{ 641167514Skmacy 642167514Skmacy if (sc->tx_dmat != NULL) 643167514Skmacy bus_dma_tag_destroy(sc->tx_dmat); 644167514Skmacy 645167514Skmacy if (sc->rx_jumbo_dmat != NULL) 646167514Skmacy bus_dma_tag_destroy(sc->rx_jumbo_dmat); 647167514Skmacy 648167514Skmacy if (sc->rx_dmat != NULL) 649167514Skmacy bus_dma_tag_destroy(sc->rx_dmat); 650167514Skmacy 651167514Skmacy if (sc->parent_dmat != NULL) 652167514Skmacy bus_dma_tag_destroy(sc->parent_dmat); 653167514Skmacy 654167514Skmacy return (0); 655167514Skmacy} 656167514Skmacy 657167514Skmacyvoid 658167514Skmacyt3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) 659167514Skmacy{ 660167514Skmacy 661180583Skmacy qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U); 662167514Skmacy qs->rspq.polling = 0 /* p->polling */; 663167514Skmacy} 664167514Skmacy 665174708Skmacy#if !defined(__i386__) && !defined(__amd64__) 666168351Skmacystatic void 667168351Skmacyrefill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 668168351Skmacy{ 669168351Skmacy struct refill_fl_cb_arg *cb_arg = arg; 670168351Skmacy 671168351Skmacy cb_arg->error = error; 672168351Skmacy cb_arg->seg = segs[0]; 673168351Skmacy cb_arg->nseg = nseg; 674167514Skmacy 675168351Skmacy} 676174708Skmacy#endif 677167514Skmacy/** 678167514Skmacy * refill_fl - refill an SGE free-buffer list 679167514Skmacy * @sc: the controller softc 680167514Skmacy * @q: the free-list to refill 681167514Skmacy * @n: the number of new buffers to allocate 682167514Skmacy * 683167514Skmacy * (Re)populate an SGE free-buffer list with up to @n new packet buffers. 684167514Skmacy * The caller must assure that @n does not exceed the queue's capacity. 685167514Skmacy */ 686167514Skmacystatic void 687167514Skmacyrefill_fl(adapter_t *sc, struct sge_fl *q, int n) 688167514Skmacy{ 689167514Skmacy struct rx_sw_desc *sd = &q->sdesc[q->pidx]; 690167514Skmacy struct rx_desc *d = &q->desc[q->pidx]; 691168351Skmacy struct refill_fl_cb_arg cb_arg; 692194521Skmacy struct mbuf *m; 693174708Skmacy caddr_t cl; 694176472Skmacy int err, count = 0; 695175200Skmacy 696168351Skmacy cb_arg.error = 0; 697167514Skmacy while (n--) { 698168351Skmacy /* 699168351Skmacy * We only allocate a cluster, mbuf allocation happens after rx 700168351Skmacy */ 701194521Skmacy if (q->zone == zone_pack) { 702194521Skmacy if ((m = m_getcl(M_NOWAIT, MT_NOINIT, M_PKTHDR)) == NULL) 703194521Skmacy break; 704194521Skmacy cl = m->m_ext.ext_buf; 705194521Skmacy } else { 706194521Skmacy if ((cl = m_cljget(NULL, M_NOWAIT, q->buf_size)) == NULL) 707194521Skmacy break; 708194521Skmacy if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { 709194521Skmacy uma_zfree(q->zone, cl); 710194521Skmacy break; 711194521Skmacy } 712167514Skmacy } 713167514Skmacy if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { 714168351Skmacy if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { 715167848Skmacy log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); 716168890Skmacy uma_zfree(q->zone, cl); 717167848Skmacy goto done; 718167848Skmacy } 719167514Skmacy sd->flags |= RX_SW_DESC_MAP_CREATED; 720167514Skmacy } 721174708Skmacy#if !defined(__i386__) && !defined(__amd64__) 722174708Skmacy err = bus_dmamap_load(q->entry_tag, sd->map, 723194521Skmacy cl, q->buf_size, refill_fl_cb, &cb_arg, 0); 724167514Skmacy 725168351Skmacy if (err != 0 || cb_arg.error) { 726194554Skmacy if (q->zone == zone_pack) 727194521Skmacy uma_zfree(q->zone, cl); 728194521Skmacy m_free(m); 729194553Skmacy goto done; 730167514Skmacy } 731174708Skmacy#else 732194521Skmacy cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)cl); 733174708Skmacy#endif 734168351Skmacy sd->flags |= RX_SW_DESC_INUSE; 735174708Skmacy sd->rxsd_cl = cl; 736194521Skmacy sd->m = m; 737168351Skmacy d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); 738168351Skmacy d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); 739167514Skmacy d->len_gen = htobe32(V_FLD_GEN1(q->gen)); 740167514Skmacy d->gen2 = htobe32(V_FLD_GEN2(q->gen)); 741167514Skmacy 742167514Skmacy d++; 743167514Skmacy sd++; 744167514Skmacy 745167514Skmacy if (++q->pidx == q->size) { 746167514Skmacy q->pidx = 0; 747167514Skmacy q->gen ^= 1; 748167514Skmacy sd = q->sdesc; 749167514Skmacy d = q->desc; 750167514Skmacy } 751167514Skmacy q->credits++; 752176472Skmacy count++; 753167514Skmacy } 754167514Skmacy 755167514Skmacydone: 756176472Skmacy if (count) 757176472Skmacy t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 758167514Skmacy} 759167514Skmacy 760167514Skmacy 761167514Skmacy/** 762167514Skmacy * free_rx_bufs - free the Rx buffers on an SGE free list 763167514Skmacy * @sc: the controle softc 764167514Skmacy * @q: the SGE free list to clean up 765167514Skmacy * 766167514Skmacy * Release the buffers on an SGE free-buffer Rx queue. HW fetching from 767167514Skmacy * this queue should be stopped before calling this function. 768167514Skmacy */ 769167514Skmacystatic void 770167514Skmacyfree_rx_bufs(adapter_t *sc, struct sge_fl *q) 771167514Skmacy{ 772167514Skmacy u_int cidx = q->cidx; 773167514Skmacy 774167514Skmacy while (q->credits--) { 775167514Skmacy struct rx_sw_desc *d = &q->sdesc[cidx]; 776167514Skmacy 777167514Skmacy if (d->flags & RX_SW_DESC_INUSE) { 778168351Skmacy bus_dmamap_unload(q->entry_tag, d->map); 779168351Skmacy bus_dmamap_destroy(q->entry_tag, d->map); 780194521Skmacy if (q->zone == zone_pack) { 781194521Skmacy m_init(d->m, zone_pack, MCLBYTES, 782194521Skmacy M_NOWAIT, MT_DATA, M_EXT); 783194521Skmacy uma_zfree(zone_pack, d->m); 784194521Skmacy } else { 785194521Skmacy m_init(d->m, zone_mbuf, MLEN, 786194521Skmacy M_NOWAIT, MT_DATA, 0); 787194521Skmacy uma_zfree(zone_mbuf, d->m); 788194521Skmacy uma_zfree(q->zone, d->rxsd_cl); 789194521Skmacy } 790167514Skmacy } 791194521Skmacy 792174708Skmacy d->rxsd_cl = NULL; 793194521Skmacy d->m = NULL; 794167514Skmacy if (++cidx == q->size) 795167514Skmacy cidx = 0; 796167514Skmacy } 797167514Skmacy} 798167514Skmacy 799167514Skmacystatic __inline void 800167514Skmacy__refill_fl(adapter_t *adap, struct sge_fl *fl) 801167514Skmacy{ 802167514Skmacy refill_fl(adap, fl, min(16U, fl->size - fl->credits)); 803167514Skmacy} 804167514Skmacy 805174708Skmacystatic __inline void 806174708Skmacy__refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max) 807174708Skmacy{ 808174708Skmacy if ((fl->size - fl->credits) < max) 809174708Skmacy refill_fl(adap, fl, min(max, fl->size - fl->credits)); 810174708Skmacy} 811174708Skmacy 812169978Skmacy/** 813169978Skmacy * recycle_rx_buf - recycle a receive buffer 814169978Skmacy * @adapter: the adapter 815169978Skmacy * @q: the SGE free list 816169978Skmacy * @idx: index of buffer to recycle 817169978Skmacy * 818169978Skmacy * Recycles the specified buffer on the given free list by adding it at 819169978Skmacy * the next available slot on the list. 820169978Skmacy */ 821167514Skmacystatic void 822169978Skmacyrecycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx) 823169978Skmacy{ 824169978Skmacy struct rx_desc *from = &q->desc[idx]; 825169978Skmacy struct rx_desc *to = &q->desc[q->pidx]; 826169978Skmacy 827169978Skmacy q->sdesc[q->pidx] = q->sdesc[idx]; 828169978Skmacy to->addr_lo = from->addr_lo; // already big endian 829169978Skmacy to->addr_hi = from->addr_hi; // likewise 830194521Skmacy wmb(); /* necessary ? */ 831169978Skmacy to->len_gen = htobe32(V_FLD_GEN1(q->gen)); 832169978Skmacy to->gen2 = htobe32(V_FLD_GEN2(q->gen)); 833169978Skmacy q->credits++; 834169978Skmacy 835169978Skmacy if (++q->pidx == q->size) { 836169978Skmacy q->pidx = 0; 837169978Skmacy q->gen ^= 1; 838169978Skmacy } 839169978Skmacy t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 840169978Skmacy} 841169978Skmacy 842169978Skmacystatic void 843167514Skmacyalloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 844167514Skmacy{ 845167514Skmacy uint32_t *addr; 846167514Skmacy 847167514Skmacy addr = arg; 848167514Skmacy *addr = segs[0].ds_addr; 849167514Skmacy} 850167514Skmacy 851167514Skmacystatic int 852167514Skmacyalloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, 853168351Skmacy bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag, 854168351Skmacy bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag) 855167514Skmacy{ 856167514Skmacy size_t len = nelem * elem_size; 857167514Skmacy void *s = NULL; 858167514Skmacy void *p = NULL; 859167514Skmacy int err; 860167514Skmacy 861167514Skmacy if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, 862167514Skmacy BUS_SPACE_MAXADDR_32BIT, 863167514Skmacy BUS_SPACE_MAXADDR, NULL, NULL, len, 1, 864167514Skmacy len, 0, NULL, NULL, tag)) != 0) { 865167514Skmacy device_printf(sc->dev, "Cannot allocate descriptor tag\n"); 866167514Skmacy return (ENOMEM); 867167514Skmacy } 868167514Skmacy 869167514Skmacy if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT, 870167514Skmacy map)) != 0) { 871167514Skmacy device_printf(sc->dev, "Cannot allocate descriptor memory\n"); 872167514Skmacy return (ENOMEM); 873167514Skmacy } 874167514Skmacy 875167514Skmacy bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0); 876167514Skmacy bzero(p, len); 877167514Skmacy *(void **)desc = p; 878167514Skmacy 879167514Skmacy if (sw_size) { 880167514Skmacy len = nelem * sw_size; 881175340Skmacy s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); 882167514Skmacy *(void **)sdesc = s; 883167514Skmacy } 884168351Skmacy if (parent_entry_tag == NULL) 885168351Skmacy return (0); 886168351Skmacy 887168646Skmacy if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0, 888168351Skmacy BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 889168646Skmacy NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, 890168646Skmacy TX_MAX_SIZE, BUS_DMA_ALLOCNOW, 891168351Skmacy NULL, NULL, entry_tag)) != 0) { 892168351Skmacy device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); 893168351Skmacy return (ENOMEM); 894168351Skmacy } 895167514Skmacy return (0); 896167514Skmacy} 897167514Skmacy 898167514Skmacystatic void 899167514Skmacysge_slow_intr_handler(void *arg, int ncount) 900167514Skmacy{ 901167514Skmacy adapter_t *sc = arg; 902167760Skmacy 903167514Skmacy t3_slow_intr_handler(sc); 904167514Skmacy} 905167514Skmacy 906171471Skmacy/** 907171471Skmacy * sge_timer_cb - perform periodic maintenance of an SGE qset 908171471Skmacy * @data: the SGE queue set to maintain 909171471Skmacy * 910171471Skmacy * Runs periodically from a timer to perform maintenance of an SGE queue 911171471Skmacy * set. It performs two tasks: 912171471Skmacy * 913171471Skmacy * a) Cleans up any completed Tx descriptors that may still be pending. 914171471Skmacy * Normal descriptor cleanup happens when new packets are added to a Tx 915171471Skmacy * queue so this timer is relatively infrequent and does any cleanup only 916171471Skmacy * if the Tx queue has not seen any new packets in a while. We make a 917171471Skmacy * best effort attempt to reclaim descriptors, in that we don't wait 918171471Skmacy * around if we cannot get a queue's lock (which most likely is because 919171471Skmacy * someone else is queueing new packets and so will also handle the clean 920171471Skmacy * up). Since control queues use immediate data exclusively we don't 921171471Skmacy * bother cleaning them up here. 922171471Skmacy * 923171471Skmacy * b) Replenishes Rx queues that have run out due to memory shortage. 924171471Skmacy * Normally new Rx buffers are added when existing ones are consumed but 925171471Skmacy * when out of memory a queue can become empty. We try to add only a few 926171471Skmacy * buffers here, the queue will be replenished fully as these new buffers 927171471Skmacy * are used up if memory shortage has subsided. 928171471Skmacy * 929171471Skmacy * c) Return coalesced response queue credits in case a response queue is 930171471Skmacy * starved. 931171471Skmacy * 932171471Skmacy * d) Ring doorbells for T304 tunnel queues since we have seen doorbell 933171471Skmacy * fifo overflows and the FW doesn't implement any recovery scheme yet. 934171471Skmacy */ 935167514Skmacystatic void 936167514Skmacysge_timer_cb(void *arg) 937167514Skmacy{ 938167514Skmacy adapter_t *sc = arg; 939194521Skmacy if ((sc->flags & USING_MSIX) == 0) { 940194521Skmacy 941194521Skmacy struct port_info *pi; 942194521Skmacy struct sge_qset *qs; 943194521Skmacy struct sge_txq *txq; 944194521Skmacy int i, j; 945194521Skmacy int reclaim_ofl, refill_rx; 946174708Skmacy 947194521Skmacy if (sc->open_device_map == 0) 948194521Skmacy return; 949194521Skmacy 950194521Skmacy for (i = 0; i < sc->params.nports; i++) { 951194521Skmacy pi = &sc->port[i]; 952194521Skmacy for (j = 0; j < pi->nqsets; j++) { 953194521Skmacy qs = &sc->sge.qs[pi->first_qset + j]; 954194521Skmacy txq = &qs->txq[0]; 955194521Skmacy reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; 956194521Skmacy refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || 957194521Skmacy (qs->fl[1].credits < qs->fl[1].size)); 958194521Skmacy if (reclaim_ofl || refill_rx) { 959194521Skmacy taskqueue_enqueue(sc->tq, &pi->timer_reclaim_task); 960194521Skmacy break; 961194521Skmacy } 962167514Skmacy } 963167514Skmacy } 964182882Skmacy } 965194521Skmacy 966171471Skmacy if (sc->params.nports > 2) { 967171471Skmacy int i; 968171471Skmacy 969171471Skmacy for_each_port(sc, i) { 970171471Skmacy struct port_info *pi = &sc->port[i]; 971171471Skmacy 972171471Skmacy t3_write_reg(sc, A_SG_KDOORBELL, 973171471Skmacy F_SELEGRCNTX | 974171471Skmacy (FW_TUNNEL_SGEEC_START + pi->first_qset)); 975171471Skmacy } 976171471Skmacy } 977194521Skmacy if (((sc->flags & USING_MSIX) == 0 || sc->params.nports > 2) && 978194521Skmacy sc->open_device_map != 0) 979170869Skmacy callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 980167514Skmacy} 981167514Skmacy 982167514Skmacy/* 983167514Skmacy * This is meant to be a catch-all function to keep sge state private 984167514Skmacy * to sge.c 985167514Skmacy * 986167514Skmacy */ 987167514Skmacyint 988170654Skmacyt3_sge_init_adapter(adapter_t *sc) 989167514Skmacy{ 990167514Skmacy callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE); 991167514Skmacy callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 992167514Skmacy TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); 993167514Skmacy return (0); 994167514Skmacy} 995167514Skmacy 996170654Skmacyint 997175224Skmacyt3_sge_reset_adapter(adapter_t *sc) 998175224Skmacy{ 999175224Skmacy callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 1000175224Skmacy return (0); 1001175224Skmacy} 1002175224Skmacy 1003175224Skmacyint 1004174708Skmacyt3_sge_init_port(struct port_info *pi) 1005170654Skmacy{ 1006174708Skmacy TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); 1007170789Skmacy return (0); 1008170654Skmacy} 1009170654Skmacy 1010167655Skmacy/** 1011167655Skmacy * refill_rspq - replenish an SGE response queue 1012167655Skmacy * @adapter: the adapter 1013167655Skmacy * @q: the response queue to replenish 1014167655Skmacy * @credits: how many new responses to make available 1015167655Skmacy * 1016167655Skmacy * Replenishes a response queue by making the supplied number of responses 1017167655Skmacy * available to HW. 1018167655Skmacy */ 1019167655Skmacystatic __inline void 1020167655Skmacyrefill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) 1021167655Skmacy{ 1022167514Skmacy 1023167655Skmacy /* mbufs are allocated on demand when a rspq entry is processed. */ 1024167655Skmacy t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN, 1025167655Skmacy V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); 1026167655Skmacy} 1027167655Skmacy 1028167514Skmacystatic void 1029171335Skmacysge_txq_reclaim_handler(void *arg, int ncount) 1030171335Skmacy{ 1031194521Skmacy struct sge_qset *qs = arg; 1032194521Skmacy int i; 1033171335Skmacy 1034194521Skmacy for (i = 0; i < 3; i++) 1035194521Skmacy reclaim_completed_tx(qs, 16, i); 1036171335Skmacy} 1037171335Skmacy 1038171335Skmacystatic void 1039167514Skmacysge_timer_reclaim(void *arg, int ncount) 1040167514Skmacy{ 1041174708Skmacy struct port_info *pi = arg; 1042174708Skmacy int i, nqsets = pi->nqsets; 1043174708Skmacy adapter_t *sc = pi->adapter; 1044167514Skmacy struct sge_qset *qs; 1045167514Skmacy struct mtx *lock; 1046194521Skmacy 1047194521Skmacy KASSERT((sc->flags & USING_MSIX) == 0, 1048194521Skmacy ("can't call timer reclaim for msi-x")); 1049167760Skmacy 1050167514Skmacy for (i = 0; i < nqsets; i++) { 1051182882Skmacy qs = &sc->sge.qs[pi->first_qset + i]; 1052169978Skmacy 1053194521Skmacy reclaim_completed_tx(qs, 16, TXQ_OFLD); 1054167514Skmacy lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 1055167514Skmacy &sc->sge.qs[0].rspq.lock; 1056167514Skmacy 1057167514Skmacy if (mtx_trylock(lock)) { 1058167514Skmacy /* XXX currently assume that we are *NOT* polling */ 1059167514Skmacy uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS); 1060167760Skmacy 1061167760Skmacy if (qs->fl[0].credits < qs->fl[0].size - 16) 1062167514Skmacy __refill_fl(sc, &qs->fl[0]); 1063167760Skmacy if (qs->fl[1].credits < qs->fl[1].size - 16) 1064167514Skmacy __refill_fl(sc, &qs->fl[1]); 1065167514Skmacy 1066167514Skmacy if (status & (1 << qs->rspq.cntxt_id)) { 1067167514Skmacy if (qs->rspq.credits) { 1068167514Skmacy refill_rspq(sc, &qs->rspq, 1); 1069167514Skmacy qs->rspq.credits--; 1070167514Skmacy t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, 1071167514Skmacy 1 << qs->rspq.cntxt_id); 1072167514Skmacy } 1073167514Skmacy } 1074167514Skmacy mtx_unlock(lock); 1075167514Skmacy } 1076167514Skmacy } 1077167514Skmacy} 1078167514Skmacy 1079167514Skmacy/** 1080167514Skmacy * init_qset_cntxt - initialize an SGE queue set context info 1081167514Skmacy * @qs: the queue set 1082167514Skmacy * @id: the queue set id 1083167514Skmacy * 1084167514Skmacy * Initializes the TIDs and context ids for the queues of a queue set. 1085167514Skmacy */ 1086167514Skmacystatic void 1087167514Skmacyinit_qset_cntxt(struct sge_qset *qs, u_int id) 1088167514Skmacy{ 1089167514Skmacy 1090167514Skmacy qs->rspq.cntxt_id = id; 1091167514Skmacy qs->fl[0].cntxt_id = 2 * id; 1092167514Skmacy qs->fl[1].cntxt_id = 2 * id + 1; 1093167514Skmacy qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; 1094167514Skmacy qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; 1095167514Skmacy qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; 1096167514Skmacy qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; 1097167514Skmacy qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; 1098174708Skmacy 1099174708Skmacy mbufq_init(&qs->txq[TXQ_ETH].sendq); 1100174708Skmacy mbufq_init(&qs->txq[TXQ_OFLD].sendq); 1101174708Skmacy mbufq_init(&qs->txq[TXQ_CTRL].sendq); 1102167514Skmacy} 1103167514Skmacy 1104167514Skmacy 1105167514Skmacystatic void 1106167514Skmacytxq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs) 1107167514Skmacy{ 1108167514Skmacy txq->in_use += ndesc; 1109167514Skmacy /* 1110167514Skmacy * XXX we don't handle stopping of queue 1111167514Skmacy * presumably start handles this when we bump against the end 1112167514Skmacy */ 1113167514Skmacy txqs->gen = txq->gen; 1114167514Skmacy txq->unacked += ndesc; 1115176472Skmacy txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5); 1116176472Skmacy txq->unacked &= 31; 1117167514Skmacy txqs->pidx = txq->pidx; 1118167514Skmacy txq->pidx += ndesc; 1119175347Skmacy#ifdef INVARIANTS 1120175347Skmacy if (((txqs->pidx > txq->cidx) && 1121175347Skmacy (txq->pidx < txqs->pidx) && 1122175347Skmacy (txq->pidx >= txq->cidx)) || 1123175347Skmacy ((txqs->pidx < txq->cidx) && 1124175347Skmacy (txq->pidx >= txq-> cidx)) || 1125175347Skmacy ((txqs->pidx < txq->cidx) && 1126175347Skmacy (txq->cidx < txqs->pidx))) 1127175347Skmacy panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d", 1128175347Skmacy txqs->pidx, txq->pidx, txq->cidx); 1129175347Skmacy#endif 1130167514Skmacy if (txq->pidx >= txq->size) { 1131167514Skmacy txq->pidx -= txq->size; 1132167514Skmacy txq->gen ^= 1; 1133167514Skmacy } 1134167514Skmacy 1135167514Skmacy} 1136167514Skmacy 1137167514Skmacy/** 1138167514Skmacy * calc_tx_descs - calculate the number of Tx descriptors for a packet 1139167514Skmacy * @m: the packet mbufs 1140167514Skmacy * @nsegs: the number of segments 1141167514Skmacy * 1142167514Skmacy * Returns the number of Tx descriptors needed for the given Ethernet 1143167514Skmacy * packet. Ethernet packets require addition of WR and CPL headers. 1144167514Skmacy */ 1145167514Skmacystatic __inline unsigned int 1146167514Skmacycalc_tx_descs(const struct mbuf *m, int nsegs) 1147167514Skmacy{ 1148167514Skmacy unsigned int flits; 1149167514Skmacy 1150194521Skmacy if (m->m_pkthdr.len <= PIO_LEN) 1151167514Skmacy return 1; 1152167514Skmacy 1153167514Skmacy flits = sgl_len(nsegs) + 2; 1154174708Skmacy if (m->m_pkthdr.csum_flags & CSUM_TSO) 1155167514Skmacy flits++; 1156204274Snp 1157167514Skmacy return flits_to_desc(flits); 1158167514Skmacy} 1159167514Skmacy 1160168646Skmacystatic unsigned int 1161168646Skmacybusdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, 1162174708Skmacy struct tx_sw_desc *txsd, bus_dma_segment_t *segs, int *nsegs) 1163167514Skmacy{ 1164168646Skmacy struct mbuf *m0; 1165174708Skmacy int err, pktlen, pass = 0; 1166195006Snp bus_dma_tag_t tag = txq->entry_tag; 1167194521Skmacy 1168174708Skmacyretry: 1169174708Skmacy err = 0; 1170167514Skmacy m0 = *m; 1171167514Skmacy pktlen = m0->m_pkthdr.len; 1172174708Skmacy#if defined(__i386__) || defined(__amd64__) 1173195006Snp if (busdma_map_sg_collapse(tag, txsd->map, m, segs, nsegs) == 0) { 1174174708Skmacy goto done; 1175174708Skmacy } else 1176174708Skmacy#endif 1177195006Snp err = bus_dmamap_load_mbuf_sg(tag, txsd->map, m0, segs, nsegs, 0); 1178168737Skmacy 1179174708Skmacy if (err == 0) { 1180174708Skmacy goto done; 1181168737Skmacy } 1182174708Skmacy if (err == EFBIG && pass == 0) { 1183174708Skmacy pass = 1; 1184167514Skmacy /* Too many segments, try to defrag */ 1185171471Skmacy m0 = m_defrag(m0, M_DONTWAIT); 1186167514Skmacy if (m0 == NULL) { 1187167514Skmacy m_freem(*m); 1188167514Skmacy *m = NULL; 1189167514Skmacy return (ENOBUFS); 1190167514Skmacy } 1191167514Skmacy *m = m0; 1192174708Skmacy goto retry; 1193174708Skmacy } else if (err == ENOMEM) { 1194167514Skmacy return (err); 1195174708Skmacy } if (err) { 1196167514Skmacy if (cxgb_debug) 1197167514Skmacy printf("map failure err=%d pktlen=%d\n", err, pktlen); 1198174639Skmacy m_freem(m0); 1199167514Skmacy *m = NULL; 1200167514Skmacy return (err); 1201167514Skmacy } 1202174708Skmacydone: 1203174708Skmacy#if !defined(__i386__) && !defined(__amd64__) 1204195006Snp bus_dmamap_sync(tag, txsd->map, BUS_DMASYNC_PREWRITE); 1205174708Skmacy#endif 1206174708Skmacy txsd->flags |= TX_SW_DESC_MAPPED; 1207167514Skmacy 1208167514Skmacy return (0); 1209167514Skmacy} 1210167514Skmacy 1211167514Skmacy/** 1212167514Skmacy * make_sgl - populate a scatter/gather list for a packet 1213167514Skmacy * @sgp: the SGL to populate 1214167514Skmacy * @segs: the packet dma segments 1215167514Skmacy * @nsegs: the number of segments 1216167514Skmacy * 1217167514Skmacy * Generates a scatter/gather list for the buffers that make up a packet 1218167514Skmacy * and returns the SGL size in 8-byte words. The caller must size the SGL 1219167514Skmacy * appropriately. 1220167514Skmacy */ 1221167514Skmacystatic __inline void 1222167514Skmacymake_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs) 1223167514Skmacy{ 1224167514Skmacy int i, idx; 1225167514Skmacy 1226174708Skmacy for (idx = 0, i = 0; i < nsegs; i++) { 1227174708Skmacy /* 1228174708Skmacy * firmware doesn't like empty segments 1229174708Skmacy */ 1230174708Skmacy if (segs[i].ds_len == 0) 1231174708Skmacy continue; 1232167514Skmacy if (i && idx == 0) 1233167514Skmacy ++sgp; 1234174708Skmacy 1235167514Skmacy sgp->len[idx] = htobe32(segs[i].ds_len); 1236167514Skmacy sgp->addr[idx] = htobe64(segs[i].ds_addr); 1237174708Skmacy idx ^= 1; 1238167514Skmacy } 1239167514Skmacy 1240175200Skmacy if (idx) { 1241167514Skmacy sgp->len[idx] = 0; 1242175200Skmacy sgp->addr[idx] = 0; 1243175200Skmacy } 1244167514Skmacy} 1245167514Skmacy 1246167514Skmacy/** 1247167514Skmacy * check_ring_tx_db - check and potentially ring a Tx queue's doorbell 1248167514Skmacy * @adap: the adapter 1249167514Skmacy * @q: the Tx queue 1250167514Skmacy * 1251194521Skmacy * Ring the doorbell if a Tx queue is asleep. There is a natural race, 1252167514Skmacy * where the HW is going to sleep just after we checked, however, 1253167514Skmacy * then the interrupt handler will detect the outstanding TX packet 1254167514Skmacy * and ring the doorbell for us. 1255167514Skmacy * 1256167514Skmacy * When GTS is disabled we unconditionally ring the doorbell. 1257167514Skmacy */ 1258167514Skmacystatic __inline void 1259167514Skmacycheck_ring_tx_db(adapter_t *adap, struct sge_txq *q) 1260167514Skmacy{ 1261167514Skmacy#if USE_GTS 1262167514Skmacy clear_bit(TXQ_LAST_PKT_DB, &q->flags); 1263167514Skmacy if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { 1264167514Skmacy set_bit(TXQ_LAST_PKT_DB, &q->flags); 1265167514Skmacy#ifdef T3_TRACE 1266167514Skmacy T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", 1267167514Skmacy q->cntxt_id); 1268167514Skmacy#endif 1269167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1270167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1271167514Skmacy } 1272167514Skmacy#else 1273167514Skmacy wmb(); /* write descriptors before telling HW */ 1274167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1275167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1276167514Skmacy#endif 1277167514Skmacy} 1278167514Skmacy 1279167514Skmacystatic __inline void 1280167514Skmacywr_gen2(struct tx_desc *d, unsigned int gen) 1281167514Skmacy{ 1282167514Skmacy#if SGE_NUM_GENBITS == 2 1283167514Skmacy d->flit[TX_DESC_FLITS - 1] = htobe64(gen); 1284167514Skmacy#endif 1285167514Skmacy} 1286167514Skmacy 1287169978Skmacy/** 1288169978Skmacy * write_wr_hdr_sgl - write a WR header and, optionally, SGL 1289169978Skmacy * @ndesc: number of Tx descriptors spanned by the SGL 1290169978Skmacy * @txd: first Tx descriptor to be written 1291169978Skmacy * @txqs: txq state (generation and producer index) 1292169978Skmacy * @txq: the SGE Tx queue 1293169978Skmacy * @sgl: the SGL 1294169978Skmacy * @flits: number of flits to the start of the SGL in the first descriptor 1295169978Skmacy * @sgl_flits: the SGL size in flits 1296169978Skmacy * @wr_hi: top 32 bits of WR header based on WR type (big endian) 1297169978Skmacy * @wr_lo: low 32 bits of WR header based on WR type (big endian) 1298169978Skmacy * 1299169978Skmacy * Write a work request header and an associated SGL. If the SGL is 1300169978Skmacy * small enough to fit into one Tx descriptor it has already been written 1301169978Skmacy * and we just need to write the WR header. Otherwise we distribute the 1302169978Skmacy * SGL across the number of descriptors it spans. 1303169978Skmacy */ 1304169978Skmacystatic void 1305169978Skmacywrite_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, 1306169978Skmacy const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, 1307169978Skmacy unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo) 1308169978Skmacy{ 1309169978Skmacy 1310169978Skmacy struct work_request_hdr *wrp = (struct work_request_hdr *)txd; 1311169978Skmacy struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx]; 1312169978Skmacy 1313169978Skmacy if (__predict_true(ndesc == 1)) { 1314194521Skmacy set_wr_hdr(wrp, htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1315194521Skmacy V_WR_SGLSFLT(flits)) | wr_hi, 1316194521Skmacy htonl(V_WR_LEN(flits + sgl_flits) | 1317194521Skmacy V_WR_GEN(txqs->gen)) | wr_lo); 1318169978Skmacy /* XXX gen? */ 1319169978Skmacy wr_gen2(txd, txqs->gen); 1320174708Skmacy 1321169978Skmacy } else { 1322169978Skmacy unsigned int ogen = txqs->gen; 1323169978Skmacy const uint64_t *fp = (const uint64_t *)sgl; 1324169978Skmacy struct work_request_hdr *wp = wrp; 1325169978Skmacy 1326194521Skmacy wrp->wrh_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | 1327169978Skmacy V_WR_SGLSFLT(flits)) | wr_hi; 1328169978Skmacy 1329169978Skmacy while (sgl_flits) { 1330169978Skmacy unsigned int avail = WR_FLITS - flits; 1331169978Skmacy 1332169978Skmacy if (avail > sgl_flits) 1333169978Skmacy avail = sgl_flits; 1334169978Skmacy memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); 1335169978Skmacy sgl_flits -= avail; 1336169978Skmacy ndesc--; 1337169978Skmacy if (!sgl_flits) 1338169978Skmacy break; 1339169978Skmacy 1340169978Skmacy fp += avail; 1341169978Skmacy txd++; 1342169978Skmacy txsd++; 1343169978Skmacy if (++txqs->pidx == txq->size) { 1344169978Skmacy txqs->pidx = 0; 1345169978Skmacy txqs->gen ^= 1; 1346169978Skmacy txd = txq->desc; 1347169978Skmacy txsd = txq->sdesc; 1348169978Skmacy } 1349194521Skmacy 1350169978Skmacy /* 1351169978Skmacy * when the head of the mbuf chain 1352169978Skmacy * is freed all clusters will be freed 1353169978Skmacy * with it 1354169978Skmacy */ 1355169978Skmacy wrp = (struct work_request_hdr *)txd; 1356194521Skmacy wrp->wrh_hi = htonl(V_WR_DATATYPE(1) | 1357169978Skmacy V_WR_SGLSFLT(1)) | wr_hi; 1358194521Skmacy wrp->wrh_lo = htonl(V_WR_LEN(min(WR_FLITS, 1359169978Skmacy sgl_flits + 1)) | 1360169978Skmacy V_WR_GEN(txqs->gen)) | wr_lo; 1361169978Skmacy wr_gen2(txd, txqs->gen); 1362169978Skmacy flits = 1; 1363169978Skmacy } 1364194521Skmacy wrp->wrh_hi |= htonl(F_WR_EOP); 1365169978Skmacy wmb(); 1366194521Skmacy wp->wrh_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; 1367169978Skmacy wr_gen2((struct tx_desc *)wp, ogen); 1368169978Skmacy } 1369169978Skmacy} 1370169978Skmacy 1371204348Snp/* sizeof(*eh) + sizeof(*ip) + sizeof(*tcp) */ 1372204348Snp#define TCPPKTHDRSIZE (ETHER_HDR_LEN + 20 + 20) 1373167514Skmacy 1374174708Skmacy#define GET_VTAG(cntrl, m) \ 1375174708Skmacydo { \ 1376174708Skmacy if ((m)->m_flags & M_VLANTAG) \ 1377174708Skmacy cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \ 1378174708Skmacy} while (0) 1379174708Skmacy 1380194521Skmacystatic int 1381194521Skmacyt3_encap(struct sge_qset *qs, struct mbuf **m) 1382167514Skmacy{ 1383167514Skmacy adapter_t *sc; 1384167514Skmacy struct mbuf *m0; 1385167514Skmacy struct sge_txq *txq; 1386167514Skmacy struct txq_state txqs; 1387174708Skmacy struct port_info *pi; 1388171868Skmacy unsigned int ndesc, flits, cntrl, mlen; 1389172096Skmacy int err, nsegs, tso_info = 0; 1390167514Skmacy 1391167514Skmacy struct work_request_hdr *wrp; 1392167514Skmacy struct tx_sw_desc *txsd; 1393174708Skmacy struct sg_ent *sgp, *sgl; 1394167514Skmacy uint32_t wr_hi, wr_lo, sgl_flits; 1395175347Skmacy bus_dma_segment_t segs[TX_MAX_SEGS]; 1396167514Skmacy 1397167514Skmacy struct tx_desc *txd; 1398174708Skmacy 1399174708Skmacy pi = qs->port; 1400174708Skmacy sc = pi->adapter; 1401167514Skmacy txq = &qs->txq[TXQ_ETH]; 1402175347Skmacy txd = &txq->desc[txq->pidx]; 1403174708Skmacy txsd = &txq->sdesc[txq->pidx]; 1404174708Skmacy sgl = txq->txq_sgl; 1405194521Skmacy 1406194521Skmacy prefetch(txd); 1407174708Skmacy m0 = *m; 1408204348Snp 1409194521Skmacy mtx_assert(&qs->lock, MA_OWNED); 1410174708Skmacy cntrl = V_TXPKT_INTF(pi->txpkt_intf); 1411194521Skmacy KASSERT(m0->m_flags & M_PKTHDR, ("not packet header\n")); 1412194521Skmacy 1413194521Skmacy if (m0->m_nextpkt == NULL && m0->m_next != NULL && 1414194521Skmacy m0->m_pkthdr.csum_flags & (CSUM_TSO)) 1415168644Skmacy tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); 1416204274Snp 1417194521Skmacy if (m0->m_nextpkt != NULL) { 1418195006Snp busdma_map_sg_vec(txq->entry_tag, txsd->map, m0, segs, &nsegs); 1419194521Skmacy ndesc = 1; 1420194521Skmacy mlen = 0; 1421194521Skmacy } else { 1422195006Snp if ((err = busdma_map_sg_collapse(txq->entry_tag, txsd->map, 1423195006Snp &m0, segs, &nsegs))) { 1424194521Skmacy if (cxgb_debug) 1425194521Skmacy printf("failed ... err=%d\n", err); 1426174708Skmacy return (err); 1427194521Skmacy } 1428194521Skmacy mlen = m0->m_pkthdr.len; 1429194521Skmacy ndesc = calc_tx_descs(m0, nsegs); 1430194521Skmacy } 1431194521Skmacy txq_prod(txq, ndesc, &txqs); 1432174708Skmacy 1433194521Skmacy KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d", nsegs)); 1434194521Skmacy txsd->m = m0; 1435194521Skmacy 1436194521Skmacy if (m0->m_nextpkt != NULL) { 1437174708Skmacy struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; 1438174708Skmacy int i, fidx; 1439174708Skmacy 1440194521Skmacy if (nsegs > 7) 1441194521Skmacy panic("trying to coalesce %d packets in to one WR", nsegs); 1442194521Skmacy txq->txq_coalesced += nsegs; 1443174708Skmacy wrp = (struct work_request_hdr *)txd; 1444194521Skmacy flits = nsegs*2 + 1; 1445174708Skmacy 1446194521Skmacy for (fidx = 1, i = 0; i < nsegs; i++, fidx += 2) { 1447194521Skmacy struct cpl_tx_pkt_batch_entry *cbe; 1448194521Skmacy uint64_t flit; 1449194521Skmacy uint32_t *hflit = (uint32_t *)&flit; 1450194521Skmacy int cflags = m0->m_pkthdr.csum_flags; 1451174708Skmacy 1452174708Skmacy cntrl = V_TXPKT_INTF(pi->txpkt_intf); 1453194521Skmacy GET_VTAG(cntrl, m0); 1454174708Skmacy cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1455194521Skmacy if (__predict_false(!(cflags & CSUM_IP))) 1456180583Skmacy cntrl |= F_TXPKT_IPCSUM_DIS; 1457194521Skmacy if (__predict_false(!(cflags & (CSUM_TCP | CSUM_UDP)))) 1458180583Skmacy cntrl |= F_TXPKT_L4CSUM_DIS; 1459194521Skmacy 1460194521Skmacy hflit[0] = htonl(cntrl); 1461194521Skmacy hflit[1] = htonl(segs[i].ds_len | 0x80000000); 1462194521Skmacy flit |= htobe64(1 << 24); 1463194521Skmacy cbe = &cpl_batch->pkt_entry[i]; 1464194521Skmacy cbe->cntrl = hflit[0]; 1465194521Skmacy cbe->len = hflit[1]; 1466174708Skmacy cbe->addr = htobe64(segs[i].ds_addr); 1467174708Skmacy } 1468174708Skmacy 1469194521Skmacy wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1470194521Skmacy V_WR_SGLSFLT(flits)) | 1471194521Skmacy htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1472194521Skmacy wr_lo = htonl(V_WR_LEN(flits) | 1473194521Skmacy V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token)); 1474194521Skmacy set_wr_hdr(wrp, wr_hi, wr_lo); 1475174708Skmacy wmb(); 1476204271Snp ETHER_BPF_MTAP(pi->ifp, m0); 1477174708Skmacy wr_gen2(txd, txqs.gen); 1478174708Skmacy check_ring_tx_db(sc, txq); 1479174708Skmacy return (0); 1480174708Skmacy } else if (tso_info) { 1481204348Snp int eth_type; 1482174708Skmacy struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd; 1483204348Snp struct ether_header *eh; 1484167514Skmacy struct ip *ip; 1485167514Skmacy struct tcphdr *tcp; 1486174708Skmacy 1487167514Skmacy txd->flit[2] = 0; 1488180583Skmacy GET_VTAG(cntrl, m0); 1489167514Skmacy cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); 1490167514Skmacy hdr->cntrl = htonl(cntrl); 1491174708Skmacy hdr->len = htonl(mlen | 0x80000000); 1492167514Skmacy 1493204348Snp if (__predict_false(mlen < TCPPKTHDRSIZE)) { 1494180583Skmacy printf("mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%#x,flags=%#x", 1495181653Skmacy m0, mlen, m0->m_pkthdr.tso_segsz, 1496181653Skmacy m0->m_pkthdr.csum_flags, m0->m_flags); 1497181653Skmacy panic("tx tso packet too small"); 1498181653Skmacy } 1499174708Skmacy 1500181653Skmacy /* Make sure that ether, ip, tcp headers are all in m0 */ 1501204348Snp if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) { 1502204348Snp m0 = m_pullup(m0, TCPPKTHDRSIZE); 1503181653Skmacy if (__predict_false(m0 == NULL)) { 1504181653Skmacy /* XXX panic probably an overreaction */ 1505181653Skmacy panic("couldn't fit header into mbuf"); 1506181653Skmacy } 1507181653Skmacy } 1508181653Skmacy 1509204348Snp eh = mtod(m0, struct ether_header *); 1510204348Snp if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1511167514Skmacy eth_type = CPL_ETH_II_VLAN; 1512204348Snp ip = (struct ip *)((struct ether_vlan_header *)eh + 1); 1513167514Skmacy } else { 1514167514Skmacy eth_type = CPL_ETH_II; 1515204348Snp ip = (struct ip *)(eh + 1); 1516167514Skmacy } 1517204348Snp tcp = (struct tcphdr *)(ip + 1); 1518168737Skmacy 1519167514Skmacy tso_info |= V_LSO_ETH_TYPE(eth_type) | 1520167514Skmacy V_LSO_IPHDR_WORDS(ip->ip_hl) | 1521167514Skmacy V_LSO_TCPHDR_WORDS(tcp->th_off); 1522167514Skmacy hdr->lso_info = htonl(tso_info); 1523180583Skmacy 1524180583Skmacy if (__predict_false(mlen <= PIO_LEN)) { 1525204348Snp /* 1526204348Snp * pkt not undersized but fits in PIO_LEN 1527183062Skmacy * Indicates a TSO bug at the higher levels. 1528183059Skmacy */ 1529194521Skmacy txsd->m = NULL; 1530180583Skmacy m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]); 1531180583Skmacy flits = (mlen + 7) / 8 + 3; 1532194521Skmacy wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | 1533180583Skmacy V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | 1534180583Skmacy F_WR_SOP | F_WR_EOP | txqs.compl); 1535194521Skmacy wr_lo = htonl(V_WR_LEN(flits) | 1536194521Skmacy V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); 1537194521Skmacy set_wr_hdr(&hdr->wr, wr_hi, wr_lo); 1538180583Skmacy wmb(); 1539204271Snp ETHER_BPF_MTAP(pi->ifp, m0); 1540180583Skmacy wr_gen2(txd, txqs.gen); 1541180583Skmacy check_ring_tx_db(sc, txq); 1542204271Snp m_freem(m0); 1543180583Skmacy return (0); 1544180583Skmacy } 1545167514Skmacy flits = 3; 1546167514Skmacy } else { 1547174708Skmacy struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd; 1548194521Skmacy 1549174708Skmacy GET_VTAG(cntrl, m0); 1550167514Skmacy cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1551180583Skmacy if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP))) 1552180583Skmacy cntrl |= F_TXPKT_IPCSUM_DIS; 1553180583Skmacy if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)))) 1554180583Skmacy cntrl |= F_TXPKT_L4CSUM_DIS; 1555167514Skmacy cpl->cntrl = htonl(cntrl); 1556174708Skmacy cpl->len = htonl(mlen | 0x80000000); 1557174708Skmacy 1558175340Skmacy if (mlen <= PIO_LEN) { 1559194521Skmacy txsd->m = NULL; 1560175340Skmacy m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); 1561167514Skmacy flits = (mlen + 7) / 8 + 2; 1562194521Skmacy 1563194521Skmacy wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | 1564194521Skmacy V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | 1565167514Skmacy F_WR_SOP | F_WR_EOP | txqs.compl); 1566194521Skmacy wr_lo = htonl(V_WR_LEN(flits) | 1567194521Skmacy V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); 1568194521Skmacy set_wr_hdr(&cpl->wr, wr_hi, wr_lo); 1569167514Skmacy wmb(); 1570204271Snp ETHER_BPF_MTAP(pi->ifp, m0); 1571167514Skmacy wr_gen2(txd, txqs.gen); 1572167514Skmacy check_ring_tx_db(sc, txq); 1573204271Snp m_freem(m0); 1574167514Skmacy return (0); 1575167514Skmacy } 1576167514Skmacy flits = 2; 1577167514Skmacy } 1578174708Skmacy wrp = (struct work_request_hdr *)txd; 1579169978Skmacy sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; 1580167514Skmacy make_sgl(sgp, segs, nsegs); 1581167514Skmacy 1582167514Skmacy sgl_flits = sgl_len(nsegs); 1583167514Skmacy 1584204271Snp ETHER_BPF_MTAP(pi->ifp, m0); 1585204271Snp 1586194521Skmacy KASSERT(ndesc <= 4, ("ndesc too large %d", ndesc)); 1587167514Skmacy wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1588167514Skmacy wr_lo = htonl(V_WR_TID(txq->token)); 1589194521Skmacy write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, 1590194521Skmacy sgl_flits, wr_hi, wr_lo); 1591204271Snp check_ring_tx_db(sc, txq); 1592167514Skmacy 1593194521Skmacy return (0); 1594194521Skmacy} 1595194521Skmacy 1596194521Skmacyvoid 1597194521Skmacycxgb_tx_watchdog(void *arg) 1598194521Skmacy{ 1599194521Skmacy struct sge_qset *qs = arg; 1600194521Skmacy struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1601194521Skmacy 1602194521Skmacy if (qs->coalescing != 0 && 1603194521Skmacy (txq->in_use <= cxgb_tx_coalesce_enable_stop) && 1604194521Skmacy TXQ_RING_EMPTY(qs)) 1605194521Skmacy qs->coalescing = 0; 1606194521Skmacy else if (qs->coalescing == 0 && 1607194521Skmacy (txq->in_use >= cxgb_tx_coalesce_enable_start)) 1608194521Skmacy qs->coalescing = 1; 1609194521Skmacy if (TXQ_TRYLOCK(qs)) { 1610194521Skmacy qs->qs_flags |= QS_FLUSHING; 1611194521Skmacy cxgb_start_locked(qs); 1612194521Skmacy qs->qs_flags &= ~QS_FLUSHING; 1613194521Skmacy TXQ_UNLOCK(qs); 1614174708Skmacy } 1615194521Skmacy if (qs->port->ifp->if_drv_flags & IFF_DRV_RUNNING) 1616194521Skmacy callout_reset_on(&txq->txq_watchdog, hz/4, cxgb_tx_watchdog, 1617194521Skmacy qs, txq->txq_watchdog.c_cpu); 1618194521Skmacy} 1619194521Skmacy 1620194521Skmacystatic void 1621194521Skmacycxgb_tx_timeout(void *arg) 1622194521Skmacy{ 1623194521Skmacy struct sge_qset *qs = arg; 1624194521Skmacy struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1625194521Skmacy 1626194521Skmacy if (qs->coalescing == 0 && (txq->in_use >= (txq->size>>3))) 1627194521Skmacy qs->coalescing = 1; 1628194521Skmacy if (TXQ_TRYLOCK(qs)) { 1629194521Skmacy qs->qs_flags |= QS_TIMEOUT; 1630194521Skmacy cxgb_start_locked(qs); 1631194521Skmacy qs->qs_flags &= ~QS_TIMEOUT; 1632194521Skmacy TXQ_UNLOCK(qs); 1633194521Skmacy } 1634194521Skmacy} 1635194521Skmacy 1636194521Skmacystatic void 1637194521Skmacycxgb_start_locked(struct sge_qset *qs) 1638194521Skmacy{ 1639194521Skmacy struct mbuf *m_head = NULL; 1640194521Skmacy struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1641194521Skmacy int in_use_init = txq->in_use; 1642194521Skmacy struct port_info *pi = qs->port; 1643194521Skmacy struct ifnet *ifp = pi->ifp; 1644194521Skmacy 1645194521Skmacy if (qs->qs_flags & (QS_FLUSHING|QS_TIMEOUT)) 1646194521Skmacy reclaim_completed_tx(qs, 0, TXQ_ETH); 1647194521Skmacy 1648194521Skmacy if (!pi->link_config.link_ok) { 1649194521Skmacy TXQ_RING_FLUSH(qs); 1650194521Skmacy return; 1651194521Skmacy } 1652194521Skmacy TXQ_LOCK_ASSERT(qs); 1653205949Snp while ((txq->in_use - in_use_init < TX_START_MAX_DESC) && 1654205949Snp !TXQ_RING_EMPTY(qs) && (ifp->if_drv_flags & IFF_DRV_RUNNING) && 1655194521Skmacy pi->link_config.link_ok) { 1656194521Skmacy reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH); 1657194521Skmacy 1658205949Snp if (txq->size - txq->in_use <= TX_MAX_DESC) 1659205949Snp break; 1660205949Snp 1661194521Skmacy if ((m_head = cxgb_dequeue(qs)) == NULL) 1662194521Skmacy break; 1663194521Skmacy /* 1664194521Skmacy * Encapsulation can modify our pointer, and or make it 1665194521Skmacy * NULL on failure. In that event, we can't requeue. 1666194521Skmacy */ 1667194521Skmacy if (t3_encap(qs, &m_head) || m_head == NULL) 1668194521Skmacy break; 1669194521Skmacy 1670194521Skmacy m_head = NULL; 1671194521Skmacy } 1672194521Skmacy if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 && 1673194521Skmacy pi->link_config.link_ok) 1674194521Skmacy callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, 1675194521Skmacy qs, txq->txq_timer.c_cpu); 1676194521Skmacy if (m_head != NULL) 1677194521Skmacy m_freem(m_head); 1678194521Skmacy} 1679194521Skmacy 1680194521Skmacystatic int 1681194521Skmacycxgb_transmit_locked(struct ifnet *ifp, struct sge_qset *qs, struct mbuf *m) 1682194521Skmacy{ 1683194521Skmacy struct port_info *pi = qs->port; 1684194521Skmacy struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1685194521Skmacy struct buf_ring *br = txq->txq_mr; 1686194521Skmacy int error, avail; 1687194521Skmacy 1688194521Skmacy avail = txq->size - txq->in_use; 1689194521Skmacy TXQ_LOCK_ASSERT(qs); 1690194521Skmacy 1691194521Skmacy /* 1692194521Skmacy * We can only do a direct transmit if the following are true: 1693194521Skmacy * - we aren't coalescing (ring < 3/4 full) 1694194521Skmacy * - the link is up -- checked in caller 1695194521Skmacy * - there are no packets enqueued already 1696194521Skmacy * - there is space in hardware transmit queue 1697194521Skmacy */ 1698194521Skmacy if (check_pkt_coalesce(qs) == 0 && 1699205949Snp !TXQ_RING_NEEDS_ENQUEUE(qs) && avail > TX_MAX_DESC) { 1700194521Skmacy if (t3_encap(qs, &m)) { 1701194521Skmacy if (m != NULL && 1702194521Skmacy (error = drbr_enqueue(ifp, br, m)) != 0) 1703194521Skmacy return (error); 1704194521Skmacy } else { 1705194521Skmacy /* 1706194521Skmacy * We've bypassed the buf ring so we need to update 1707194521Skmacy * the stats directly 1708194521Skmacy */ 1709194521Skmacy txq->txq_direct_packets++; 1710194521Skmacy txq->txq_direct_bytes += m->m_pkthdr.len; 1711194521Skmacy } 1712194521Skmacy } else if ((error = drbr_enqueue(ifp, br, m)) != 0) 1713194521Skmacy return (error); 1714194521Skmacy 1715194521Skmacy reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH); 1716194521Skmacy if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok && 1717194521Skmacy (!check_pkt_coalesce(qs) || (drbr_inuse(ifp, br) >= 7))) 1718194521Skmacy cxgb_start_locked(qs); 1719194521Skmacy else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer)) 1720194521Skmacy callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, 1721194521Skmacy qs, txq->txq_timer.c_cpu); 1722167514Skmacy return (0); 1723167514Skmacy} 1724167514Skmacy 1725194521Skmacyint 1726194521Skmacycxgb_transmit(struct ifnet *ifp, struct mbuf *m) 1727194521Skmacy{ 1728194521Skmacy struct sge_qset *qs; 1729194521Skmacy struct port_info *pi = ifp->if_softc; 1730194521Skmacy int error, qidx = pi->first_qset; 1731167514Skmacy 1732194521Skmacy if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 1733194521Skmacy ||(!pi->link_config.link_ok)) { 1734194521Skmacy m_freem(m); 1735194521Skmacy return (0); 1736194521Skmacy } 1737194521Skmacy 1738194521Skmacy if (m->m_flags & M_FLOWID) 1739194521Skmacy qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset; 1740194521Skmacy 1741194521Skmacy qs = &pi->adapter->sge.qs[qidx]; 1742194521Skmacy 1743194521Skmacy if (TXQ_TRYLOCK(qs)) { 1744194521Skmacy /* XXX running */ 1745194521Skmacy error = cxgb_transmit_locked(ifp, qs, m); 1746194521Skmacy TXQ_UNLOCK(qs); 1747194521Skmacy } else 1748194521Skmacy error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m); 1749194521Skmacy return (error); 1750194521Skmacy} 1751194521Skmacyvoid 1752194521Skmacycxgb_start(struct ifnet *ifp) 1753194521Skmacy{ 1754194521Skmacy struct port_info *pi = ifp->if_softc; 1755194521Skmacy struct sge_qset *qs = &pi->adapter->sge.qs[pi->first_qset]; 1756194521Skmacy 1757194521Skmacy if (!pi->link_config.link_ok) 1758194521Skmacy return; 1759194521Skmacy 1760194521Skmacy TXQ_LOCK(qs); 1761194521Skmacy cxgb_start_locked(qs); 1762194521Skmacy TXQ_UNLOCK(qs); 1763194521Skmacy} 1764194521Skmacy 1765194521Skmacyvoid 1766194521Skmacycxgb_qflush(struct ifnet *ifp) 1767194521Skmacy{ 1768194521Skmacy /* 1769194521Skmacy * flush any enqueued mbufs in the buf_rings 1770194521Skmacy * and in the transmit queues 1771194521Skmacy * no-op for now 1772194521Skmacy */ 1773194521Skmacy return; 1774194521Skmacy} 1775194521Skmacy 1776167514Skmacy/** 1777167514Skmacy * write_imm - write a packet into a Tx descriptor as immediate data 1778167514Skmacy * @d: the Tx descriptor to write 1779167514Skmacy * @m: the packet 1780167514Skmacy * @len: the length of packet data to write as immediate data 1781167514Skmacy * @gen: the generation bit value to write 1782167514Skmacy * 1783167514Skmacy * Writes a packet as immediate data into a Tx descriptor. The packet 1784167514Skmacy * contains a work request at its beginning. We must write the packet 1785167514Skmacy * carefully so the SGE doesn't read accidentally before it's written in 1786167514Skmacy * its entirety. 1787167514Skmacy */ 1788169978Skmacystatic __inline void 1789169978Skmacywrite_imm(struct tx_desc *d, struct mbuf *m, 1790169978Skmacy unsigned int len, unsigned int gen) 1791167514Skmacy{ 1792169978Skmacy struct work_request_hdr *from = mtod(m, struct work_request_hdr *); 1793167514Skmacy struct work_request_hdr *to = (struct work_request_hdr *)d; 1794194521Skmacy uint32_t wr_hi, wr_lo; 1795167514Skmacy 1796174708Skmacy if (len > WR_LEN) 1797174708Skmacy panic("len too big %d\n", len); 1798174708Skmacy if (len < sizeof(*from)) 1799174708Skmacy panic("len too small %d", len); 1800174708Skmacy 1801167514Skmacy memcpy(&to[1], &from[1], len - sizeof(*from)); 1802194521Skmacy wr_hi = from->wrh_hi | htonl(F_WR_SOP | F_WR_EOP | 1803167514Skmacy V_WR_BCNTLFLT(len & 7)); 1804194521Skmacy wr_lo = from->wrh_lo | htonl(V_WR_GEN(gen) | 1805194521Skmacy V_WR_LEN((len + 7) / 8)); 1806194521Skmacy set_wr_hdr(to, wr_hi, wr_lo); 1807167514Skmacy wmb(); 1808167514Skmacy wr_gen2(d, gen); 1809174708Skmacy 1810174708Skmacy /* 1811174708Skmacy * This check is a hack we should really fix the logic so 1812174708Skmacy * that this can't happen 1813174708Skmacy */ 1814174708Skmacy if (m->m_type != MT_DONTFREE) 1815174708Skmacy m_freem(m); 1816174708Skmacy 1817167514Skmacy} 1818167514Skmacy 1819167514Skmacy/** 1820167514Skmacy * check_desc_avail - check descriptor availability on a send queue 1821167514Skmacy * @adap: the adapter 1822167514Skmacy * @q: the TX queue 1823167514Skmacy * @m: the packet needing the descriptors 1824167514Skmacy * @ndesc: the number of Tx descriptors needed 1825167514Skmacy * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) 1826167514Skmacy * 1827167514Skmacy * Checks if the requested number of Tx descriptors is available on an 1828167514Skmacy * SGE send queue. If the queue is already suspended or not enough 1829167514Skmacy * descriptors are available the packet is queued for later transmission. 1830167514Skmacy * Must be called with the Tx queue locked. 1831167514Skmacy * 1832167514Skmacy * Returns 0 if enough descriptors are available, 1 if there aren't 1833167514Skmacy * enough descriptors and the packet has been queued, and 2 if the caller 1834167514Skmacy * needs to retry because there weren't enough descriptors at the 1835167514Skmacy * beginning of the call but some freed up in the mean time. 1836167514Skmacy */ 1837167514Skmacystatic __inline int 1838167514Skmacycheck_desc_avail(adapter_t *adap, struct sge_txq *q, 1839169978Skmacy struct mbuf *m, unsigned int ndesc, 1840169978Skmacy unsigned int qid) 1841167514Skmacy{ 1842167514Skmacy /* 1843167514Skmacy * XXX We currently only use this for checking the control queue 1844167514Skmacy * the control queue is only used for binding qsets which happens 1845167514Skmacy * at init time so we are guaranteed enough descriptors 1846167514Skmacy */ 1847169978Skmacy if (__predict_false(!mbufq_empty(&q->sendq))) { 1848169978Skmacyaddq_exit: mbufq_tail(&q->sendq, m); 1849167514Skmacy return 1; 1850167514Skmacy } 1851167514Skmacy if (__predict_false(q->size - q->in_use < ndesc)) { 1852167514Skmacy 1853167514Skmacy struct sge_qset *qs = txq_to_qset(q, qid); 1854167514Skmacy 1855169978Skmacy setbit(&qs->txq_stopped, qid); 1856167514Skmacy if (should_restart_tx(q) && 1857167514Skmacy test_and_clear_bit(qid, &qs->txq_stopped)) 1858167514Skmacy return 2; 1859167514Skmacy 1860167514Skmacy q->stops++; 1861167514Skmacy goto addq_exit; 1862167514Skmacy } 1863167514Skmacy return 0; 1864167514Skmacy} 1865167514Skmacy 1866167514Skmacy 1867167514Skmacy/** 1868167514Skmacy * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs 1869167514Skmacy * @q: the SGE control Tx queue 1870167514Skmacy * 1871167514Skmacy * This is a variant of reclaim_completed_tx() that is used for Tx queues 1872167514Skmacy * that send only immediate data (presently just the control queues) and 1873169978Skmacy * thus do not have any mbufs 1874167514Skmacy */ 1875167514Skmacystatic __inline void 1876167514Skmacyreclaim_completed_tx_imm(struct sge_txq *q) 1877167514Skmacy{ 1878167514Skmacy unsigned int reclaim = q->processed - q->cleaned; 1879167514Skmacy 1880167514Skmacy q->in_use -= reclaim; 1881167514Skmacy q->cleaned += reclaim; 1882167514Skmacy} 1883167514Skmacy 1884167514Skmacystatic __inline int 1885167514Skmacyimmediate(const struct mbuf *m) 1886167514Skmacy{ 1887167514Skmacy return m->m_len <= WR_LEN && m->m_pkthdr.len <= WR_LEN ; 1888167514Skmacy} 1889167514Skmacy 1890167514Skmacy/** 1891167514Skmacy * ctrl_xmit - send a packet through an SGE control Tx queue 1892167514Skmacy * @adap: the adapter 1893167514Skmacy * @q: the control queue 1894167514Skmacy * @m: the packet 1895167514Skmacy * 1896167514Skmacy * Send a packet through an SGE control Tx queue. Packets sent through 1897167514Skmacy * a control queue must fit entirely as immediate data in a single Tx 1898167514Skmacy * descriptor and have no page fragments. 1899167514Skmacy */ 1900167514Skmacystatic int 1901194521Skmacyctrl_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m) 1902167514Skmacy{ 1903167514Skmacy int ret; 1904170654Skmacy struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *); 1905194521Skmacy struct sge_txq *q = &qs->txq[TXQ_CTRL]; 1906194521Skmacy 1907167514Skmacy if (__predict_false(!immediate(m))) { 1908167514Skmacy m_freem(m); 1909167514Skmacy return 0; 1910167514Skmacy } 1911174708Skmacy 1912194521Skmacy wrp->wrh_hi |= htonl(F_WR_SOP | F_WR_EOP); 1913194521Skmacy wrp->wrh_lo = htonl(V_WR_TID(q->token)); 1914167514Skmacy 1915194521Skmacy TXQ_LOCK(qs); 1916167514Skmacyagain: reclaim_completed_tx_imm(q); 1917167514Skmacy 1918167514Skmacy ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); 1919167514Skmacy if (__predict_false(ret)) { 1920167514Skmacy if (ret == 1) { 1921194521Skmacy TXQ_UNLOCK(qs); 1922174708Skmacy return (ENOSPC); 1923167514Skmacy } 1924167514Skmacy goto again; 1925167514Skmacy } 1926167514Skmacy write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1927174708Skmacy 1928167514Skmacy q->in_use++; 1929167514Skmacy if (++q->pidx >= q->size) { 1930167514Skmacy q->pidx = 0; 1931167514Skmacy q->gen ^= 1; 1932167514Skmacy } 1933194521Skmacy TXQ_UNLOCK(qs); 1934197043Snp wmb(); 1935167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1936167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1937167514Skmacy return (0); 1938167514Skmacy} 1939167514Skmacy 1940169978Skmacy 1941167514Skmacy/** 1942167514Skmacy * restart_ctrlq - restart a suspended control queue 1943167514Skmacy * @qs: the queue set cotaining the control queue 1944167514Skmacy * 1945167514Skmacy * Resumes transmission on a suspended Tx control queue. 1946167514Skmacy */ 1947167514Skmacystatic void 1948169978Skmacyrestart_ctrlq(void *data, int npending) 1949167514Skmacy{ 1950167514Skmacy struct mbuf *m; 1951167514Skmacy struct sge_qset *qs = (struct sge_qset *)data; 1952167514Skmacy struct sge_txq *q = &qs->txq[TXQ_CTRL]; 1953167514Skmacy adapter_t *adap = qs->port->adapter; 1954167514Skmacy 1955194521Skmacy TXQ_LOCK(qs); 1956167514Skmacyagain: reclaim_completed_tx_imm(q); 1957169978Skmacy 1958167514Skmacy while (q->in_use < q->size && 1959169978Skmacy (m = mbufq_dequeue(&q->sendq)) != NULL) { 1960167514Skmacy 1961169978Skmacy write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1962167514Skmacy 1963167514Skmacy if (++q->pidx >= q->size) { 1964167514Skmacy q->pidx = 0; 1965167514Skmacy q->gen ^= 1; 1966167514Skmacy } 1967167514Skmacy q->in_use++; 1968167514Skmacy } 1969169978Skmacy if (!mbufq_empty(&q->sendq)) { 1970169978Skmacy setbit(&qs->txq_stopped, TXQ_CTRL); 1971167514Skmacy 1972167514Skmacy if (should_restart_tx(q) && 1973167514Skmacy test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) 1974167514Skmacy goto again; 1975167514Skmacy q->stops++; 1976167514Skmacy } 1977194521Skmacy TXQ_UNLOCK(qs); 1978167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1979167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1980167514Skmacy} 1981167514Skmacy 1982169978Skmacy 1983167514Skmacy/* 1984167514Skmacy * Send a management message through control queue 0 1985167514Skmacy */ 1986167514Skmacyint 1987167514Skmacyt3_mgmt_tx(struct adapter *adap, struct mbuf *m) 1988167514Skmacy{ 1989194521Skmacy return ctrl_xmit(adap, &adap->sge.qs[0], m); 1990167514Skmacy} 1991167514Skmacy 1992167514Skmacy/** 1993167514Skmacy * free_qset - free the resources of an SGE queue set 1994167514Skmacy * @sc: the controller owning the queue set 1995167514Skmacy * @q: the queue set 1996167514Skmacy * 1997167514Skmacy * Release the HW and SW resources associated with an SGE queue set, such 1998167514Skmacy * as HW contexts, packet buffers, and descriptor rings. Traffic to the 1999167514Skmacy * queue set must be quiesced prior to calling this. 2000167514Skmacy */ 2001194521Skmacystatic void 2002167514Skmacyt3_free_qset(adapter_t *sc, struct sge_qset *q) 2003167514Skmacy{ 2004167514Skmacy int i; 2005174708Skmacy 2006194521Skmacy reclaim_completed_tx(q, 0, TXQ_ETH); 2007185165Skmacy for (i = 0; i < SGE_TXQ_PER_SET; i++) { 2008185162Skmacy if (q->txq[i].txq_mr != NULL) 2009185162Skmacy buf_ring_free(q->txq[i].txq_mr, M_DEVBUF); 2010185165Skmacy if (q->txq[i].txq_ifq != NULL) { 2011194259Ssam ifq_delete(q->txq[i].txq_ifq); 2012185165Skmacy free(q->txq[i].txq_ifq, M_DEVBUF); 2013185165Skmacy } 2014185165Skmacy } 2015185165Skmacy 2016167514Skmacy for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 2017167514Skmacy if (q->fl[i].desc) { 2018176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 2019167514Skmacy t3_sge_disable_fl(sc, q->fl[i].cntxt_id); 2020176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2021167514Skmacy bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); 2022167514Skmacy bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, 2023167514Skmacy q->fl[i].desc_map); 2024167514Skmacy bus_dma_tag_destroy(q->fl[i].desc_tag); 2025168351Skmacy bus_dma_tag_destroy(q->fl[i].entry_tag); 2026167514Skmacy } 2027167514Skmacy if (q->fl[i].sdesc) { 2028167514Skmacy free_rx_bufs(sc, &q->fl[i]); 2029167514Skmacy free(q->fl[i].sdesc, M_DEVBUF); 2030167514Skmacy } 2031167514Skmacy } 2032167514Skmacy 2033194521Skmacy mtx_unlock(&q->lock); 2034194521Skmacy MTX_DESTROY(&q->lock); 2035170869Skmacy for (i = 0; i < SGE_TXQ_PER_SET; i++) { 2036167514Skmacy if (q->txq[i].desc) { 2037176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 2038167514Skmacy t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); 2039176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2040167514Skmacy bus_dmamap_unload(q->txq[i].desc_tag, 2041167514Skmacy q->txq[i].desc_map); 2042167514Skmacy bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, 2043167514Skmacy q->txq[i].desc_map); 2044167514Skmacy bus_dma_tag_destroy(q->txq[i].desc_tag); 2045168351Skmacy bus_dma_tag_destroy(q->txq[i].entry_tag); 2046167514Skmacy } 2047167514Skmacy if (q->txq[i].sdesc) { 2048167514Skmacy free(q->txq[i].sdesc, M_DEVBUF); 2049167514Skmacy } 2050167514Skmacy } 2051167514Skmacy 2052167514Skmacy if (q->rspq.desc) { 2053176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 2054167514Skmacy t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); 2055176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2056167514Skmacy 2057167514Skmacy bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); 2058167514Skmacy bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, 2059167514Skmacy q->rspq.desc_map); 2060167514Skmacy bus_dma_tag_destroy(q->rspq.desc_tag); 2061170869Skmacy MTX_DESTROY(&q->rspq.lock); 2062167514Skmacy } 2063168351Skmacy 2064205947Snp#ifdef INET 2065181616Skmacy tcp_lro_free(&q->lro.ctrl); 2066205947Snp#endif 2067181616Skmacy 2068167514Skmacy bzero(q, sizeof(*q)); 2069167514Skmacy} 2070167514Skmacy 2071167514Skmacy/** 2072167514Skmacy * t3_free_sge_resources - free SGE resources 2073167514Skmacy * @sc: the adapter softc 2074167514Skmacy * 2075167514Skmacy * Frees resources used by the SGE queue sets. 2076167514Skmacy */ 2077167514Skmacyvoid 2078167514Skmacyt3_free_sge_resources(adapter_t *sc) 2079167514Skmacy{ 2080170869Skmacy int i, nqsets; 2081174708Skmacy 2082170869Skmacy for (nqsets = i = 0; i < (sc)->params.nports; i++) 2083170869Skmacy nqsets += sc->port[i].nqsets; 2084174708Skmacy 2085194521Skmacy for (i = 0; i < nqsets; ++i) { 2086194521Skmacy TXQ_LOCK(&sc->sge.qs[i]); 2087167514Skmacy t3_free_qset(sc, &sc->sge.qs[i]); 2088194521Skmacy } 2089194521Skmacy 2090167514Skmacy} 2091167514Skmacy 2092167514Skmacy/** 2093167514Skmacy * t3_sge_start - enable SGE 2094167514Skmacy * @sc: the controller softc 2095167514Skmacy * 2096167514Skmacy * Enables the SGE for DMAs. This is the last step in starting packet 2097167514Skmacy * transfers. 2098167514Skmacy */ 2099167514Skmacyvoid 2100167514Skmacyt3_sge_start(adapter_t *sc) 2101167514Skmacy{ 2102167514Skmacy t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); 2103167514Skmacy} 2104167514Skmacy 2105169978Skmacy/** 2106169978Skmacy * t3_sge_stop - disable SGE operation 2107169978Skmacy * @sc: the adapter 2108169978Skmacy * 2109169978Skmacy * Disables the DMA engine. This can be called in emeregencies (e.g., 2110169978Skmacy * from error interrupts) or from normal process context. In the latter 2111169978Skmacy * case it also disables any pending queue restart tasklets. Note that 2112169978Skmacy * if it is called in interrupt context it cannot disable the restart 2113169978Skmacy * tasklets as it cannot wait, however the tasklets will have no effect 2114169978Skmacy * since the doorbells are disabled and the driver will call this again 2115169978Skmacy * later from process context, at which time the tasklets will be stopped 2116169978Skmacy * if they are still running. 2117169978Skmacy */ 2118169978Skmacyvoid 2119169978Skmacyt3_sge_stop(adapter_t *sc) 2120169978Skmacy{ 2121170869Skmacy int i, nqsets; 2122170869Skmacy 2123169978Skmacy t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0); 2124167514Skmacy 2125170654Skmacy if (sc->tq == NULL) 2126170654Skmacy return; 2127170654Skmacy 2128170869Skmacy for (nqsets = i = 0; i < (sc)->params.nports; i++) 2129170869Skmacy nqsets += sc->port[i].nqsets; 2130175340Skmacy#ifdef notyet 2131175340Skmacy /* 2132175340Skmacy * 2133175340Skmacy * XXX 2134175340Skmacy */ 2135170869Skmacy for (i = 0; i < nqsets; ++i) { 2136169978Skmacy struct sge_qset *qs = &sc->sge.qs[i]; 2137169978Skmacy 2138171335Skmacy taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); 2139171335Skmacy taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); 2140169978Skmacy } 2141175340Skmacy#endif 2142169978Skmacy} 2143169978Skmacy 2144167514Skmacy/** 2145174708Skmacy * t3_free_tx_desc - reclaims Tx descriptors and their buffers 2146167514Skmacy * @adapter: the adapter 2147167514Skmacy * @q: the Tx queue to reclaim descriptors from 2148174708Skmacy * @reclaimable: the number of descriptors to reclaim 2149174708Skmacy * @m_vec_size: maximum number of buffers to reclaim 2150174708Skmacy * @desc_reclaimed: returns the number of descriptors reclaimed 2151167514Skmacy * 2152167514Skmacy * Reclaims Tx descriptors from an SGE Tx queue and frees the associated 2153167514Skmacy * Tx buffers. Called with the Tx queue lock held. 2154174708Skmacy * 2155174708Skmacy * Returns number of buffers of reclaimed 2156167514Skmacy */ 2157174708Skmacyvoid 2158194521Skmacyt3_free_tx_desc(struct sge_qset *qs, int reclaimable, int queue) 2159167514Skmacy{ 2160174708Skmacy struct tx_sw_desc *txsd; 2161194521Skmacy unsigned int cidx, mask; 2162194521Skmacy struct sge_txq *q = &qs->txq[queue]; 2163194521Skmacy 2164167514Skmacy#ifdef T3_TRACE 2165167514Skmacy T3_TRACE2(sc->tb[q->cntxt_id & 7], 2166174708Skmacy "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx); 2167167514Skmacy#endif 2168174708Skmacy cidx = q->cidx; 2169194521Skmacy mask = q->size - 1; 2170174708Skmacy txsd = &q->sdesc[cidx]; 2171194521Skmacy 2172194521Skmacy mtx_assert(&qs->lock, MA_OWNED); 2173174708Skmacy while (reclaimable--) { 2174194521Skmacy prefetch(q->sdesc[(cidx + 1) & mask].m); 2175194521Skmacy prefetch(q->sdesc[(cidx + 2) & mask].m); 2176194521Skmacy 2177194521Skmacy if (txsd->m != NULL) { 2178174708Skmacy if (txsd->flags & TX_SW_DESC_MAPPED) { 2179174708Skmacy bus_dmamap_unload(q->entry_tag, txsd->map); 2180174708Skmacy txsd->flags &= ~TX_SW_DESC_MAPPED; 2181167514Skmacy } 2182194521Skmacy m_freem_list(txsd->m); 2183194521Skmacy txsd->m = NULL; 2184174708Skmacy } else 2185174708Skmacy q->txq_skipped++; 2186174708Skmacy 2187174708Skmacy ++txsd; 2188167514Skmacy if (++cidx == q->size) { 2189167514Skmacy cidx = 0; 2190174708Skmacy txsd = q->sdesc; 2191167514Skmacy } 2192167514Skmacy } 2193167514Skmacy q->cidx = cidx; 2194167514Skmacy 2195167514Skmacy} 2196167514Skmacy 2197167514Skmacy/** 2198167514Skmacy * is_new_response - check if a response is newly written 2199167514Skmacy * @r: the response descriptor 2200167514Skmacy * @q: the response queue 2201167514Skmacy * 2202167514Skmacy * Returns true if a response descriptor contains a yet unprocessed 2203167514Skmacy * response. 2204167514Skmacy */ 2205167514Skmacystatic __inline int 2206167514Skmacyis_new_response(const struct rsp_desc *r, 2207167514Skmacy const struct sge_rspq *q) 2208167514Skmacy{ 2209167514Skmacy return (r->intr_gen & F_RSPD_GEN2) == q->gen; 2210167514Skmacy} 2211167514Skmacy 2212167514Skmacy#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) 2213167514Skmacy#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ 2214167514Skmacy V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ 2215167514Skmacy V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ 2216167514Skmacy V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) 2217167514Skmacy 2218167514Skmacy/* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ 2219167514Skmacy#define NOMEM_INTR_DELAY 2500 2220167514Skmacy 2221169978Skmacy/** 2222169978Skmacy * write_ofld_wr - write an offload work request 2223169978Skmacy * @adap: the adapter 2224169978Skmacy * @m: the packet to send 2225169978Skmacy * @q: the Tx queue 2226169978Skmacy * @pidx: index of the first Tx descriptor to write 2227169978Skmacy * @gen: the generation value to use 2228169978Skmacy * @ndesc: number of descriptors the packet will occupy 2229169978Skmacy * 2230169978Skmacy * Write an offload work request to send the supplied packet. The packet 2231169978Skmacy * data already carry the work request with most fields populated. 2232169978Skmacy */ 2233169978Skmacystatic void 2234169978Skmacywrite_ofld_wr(adapter_t *adap, struct mbuf *m, 2235169978Skmacy struct sge_txq *q, unsigned int pidx, 2236169978Skmacy unsigned int gen, unsigned int ndesc, 2237169978Skmacy bus_dma_segment_t *segs, unsigned int nsegs) 2238167514Skmacy{ 2239169978Skmacy unsigned int sgl_flits, flits; 2240169978Skmacy struct work_request_hdr *from; 2241169978Skmacy struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; 2242169978Skmacy struct tx_desc *d = &q->desc[pidx]; 2243169978Skmacy struct txq_state txqs; 2244169978Skmacy 2245176472Skmacy if (immediate(m) && nsegs == 0) { 2246169978Skmacy write_imm(d, m, m->m_len, gen); 2247169978Skmacy return; 2248169978Skmacy } 2249169978Skmacy 2250169978Skmacy /* Only TX_DATA builds SGLs */ 2251169978Skmacy from = mtod(m, struct work_request_hdr *); 2252174708Skmacy memcpy(&d->flit[1], &from[1], m->m_len - sizeof(*from)); 2253169978Skmacy 2254174708Skmacy flits = m->m_len / 8; 2255169978Skmacy sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl; 2256169978Skmacy 2257169978Skmacy make_sgl(sgp, segs, nsegs); 2258169978Skmacy sgl_flits = sgl_len(nsegs); 2259169978Skmacy 2260174708Skmacy txqs.gen = gen; 2261174708Skmacy txqs.pidx = pidx; 2262174708Skmacy txqs.compl = 0; 2263174708Skmacy 2264169978Skmacy write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits, 2265194521Skmacy from->wrh_hi, from->wrh_lo); 2266167514Skmacy} 2267167514Skmacy 2268169978Skmacy/** 2269169978Skmacy * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet 2270169978Skmacy * @m: the packet 2271169978Skmacy * 2272169978Skmacy * Returns the number of Tx descriptors needed for the given offload 2273169978Skmacy * packet. These packets are already fully constructed. 2274169978Skmacy */ 2275169978Skmacystatic __inline unsigned int 2276169978Skmacycalc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs) 2277167514Skmacy{ 2278169978Skmacy unsigned int flits, cnt = 0; 2279176472Skmacy int ndescs; 2280169978Skmacy 2281176472Skmacy if (m->m_len <= WR_LEN && nsegs == 0) 2282176472Skmacy return (1); /* packet fits as immediate data */ 2283169978Skmacy 2284194521Skmacy /* 2285194521Skmacy * This needs to be re-visited for TOE 2286194521Skmacy */ 2287169978Skmacy 2288194521Skmacy cnt = nsegs; 2289194521Skmacy 2290175200Skmacy /* headers */ 2291176472Skmacy flits = m->m_len / 8; 2292169978Skmacy 2293176472Skmacy ndescs = flits_to_desc(flits + sgl_len(cnt)); 2294176472Skmacy 2295176472Skmacy return (ndescs); 2296169978Skmacy} 2297169978Skmacy 2298169978Skmacy/** 2299169978Skmacy * ofld_xmit - send a packet through an offload queue 2300169978Skmacy * @adap: the adapter 2301169978Skmacy * @q: the Tx offload queue 2302169978Skmacy * @m: the packet 2303169978Skmacy * 2304169978Skmacy * Send an offload packet through an SGE offload queue. 2305169978Skmacy */ 2306169978Skmacystatic int 2307194521Skmacyofld_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m) 2308169978Skmacy{ 2309171978Skmacy int ret, nsegs; 2310171978Skmacy unsigned int ndesc; 2311171978Skmacy unsigned int pidx, gen; 2312194521Skmacy struct sge_txq *q = &qs->txq[TXQ_OFLD]; 2313174708Skmacy bus_dma_segment_t segs[TX_MAX_SEGS], *vsegs; 2314174708Skmacy struct tx_sw_desc *stx; 2315169978Skmacy 2316174708Skmacy nsegs = m_get_sgllen(m); 2317174708Skmacy vsegs = m_get_sgl(m); 2318169978Skmacy ndesc = calc_tx_descs_ofld(m, nsegs); 2319174708Skmacy busdma_map_sgl(vsegs, segs, nsegs); 2320169978Skmacy 2321174708Skmacy stx = &q->sdesc[q->pidx]; 2322174708Skmacy 2323194521Skmacy TXQ_LOCK(qs); 2324194521Skmacyagain: reclaim_completed_tx(qs, 16, TXQ_OFLD); 2325169978Skmacy ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD); 2326169978Skmacy if (__predict_false(ret)) { 2327169978Skmacy if (ret == 1) { 2328174708Skmacy printf("no ofld desc avail\n"); 2329174708Skmacy 2330169978Skmacy m_set_priority(m, ndesc); /* save for restart */ 2331194521Skmacy TXQ_UNLOCK(qs); 2332174708Skmacy return (EINTR); 2333167514Skmacy } 2334169978Skmacy goto again; 2335169978Skmacy } 2336169978Skmacy 2337169978Skmacy gen = q->gen; 2338169978Skmacy q->in_use += ndesc; 2339169978Skmacy pidx = q->pidx; 2340169978Skmacy q->pidx += ndesc; 2341169978Skmacy if (q->pidx >= q->size) { 2342169978Skmacy q->pidx -= q->size; 2343169978Skmacy q->gen ^= 1; 2344169978Skmacy } 2345169978Skmacy#ifdef T3_TRACE 2346169978Skmacy T3_TRACE5(adap->tb[q->cntxt_id & 7], 2347169978Skmacy "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u", 2348169978Skmacy ndesc, pidx, skb->len, skb->len - skb->data_len, 2349169978Skmacy skb_shinfo(skb)->nr_frags); 2350167514Skmacy#endif 2351194521Skmacy TXQ_UNLOCK(qs); 2352169978Skmacy 2353169978Skmacy write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 2354169978Skmacy check_ring_tx_db(adap, q); 2355172101Skmacy return (0); 2356169978Skmacy} 2357167514Skmacy 2358169978Skmacy/** 2359169978Skmacy * restart_offloadq - restart a suspended offload queue 2360169978Skmacy * @qs: the queue set cotaining the offload queue 2361169978Skmacy * 2362169978Skmacy * Resumes transmission on a suspended Tx offload queue. 2363169978Skmacy */ 2364169978Skmacystatic void 2365169978Skmacyrestart_offloadq(void *data, int npending) 2366169978Skmacy{ 2367169978Skmacy struct mbuf *m; 2368169978Skmacy struct sge_qset *qs = data; 2369169978Skmacy struct sge_txq *q = &qs->txq[TXQ_OFLD]; 2370169978Skmacy adapter_t *adap = qs->port->adapter; 2371169978Skmacy bus_dma_segment_t segs[TX_MAX_SEGS]; 2372169978Skmacy struct tx_sw_desc *stx = &q->sdesc[q->pidx]; 2373174708Skmacy int nsegs, cleaned; 2374169978Skmacy 2375194521Skmacy TXQ_LOCK(qs); 2376194521Skmacyagain: cleaned = reclaim_completed_tx(qs, 16, TXQ_OFLD); 2377169978Skmacy 2378169978Skmacy while ((m = mbufq_peek(&q->sendq)) != NULL) { 2379169978Skmacy unsigned int gen, pidx; 2380169978Skmacy unsigned int ndesc = m_get_priority(m); 2381169978Skmacy 2382169978Skmacy if (__predict_false(q->size - q->in_use < ndesc)) { 2383169978Skmacy setbit(&qs->txq_stopped, TXQ_OFLD); 2384169978Skmacy if (should_restart_tx(q) && 2385169978Skmacy test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) 2386169978Skmacy goto again; 2387169978Skmacy q->stops++; 2388169978Skmacy break; 2389169978Skmacy } 2390169978Skmacy 2391169978Skmacy gen = q->gen; 2392169978Skmacy q->in_use += ndesc; 2393169978Skmacy pidx = q->pidx; 2394169978Skmacy q->pidx += ndesc; 2395169978Skmacy if (q->pidx >= q->size) { 2396169978Skmacy q->pidx -= q->size; 2397169978Skmacy q->gen ^= 1; 2398169978Skmacy } 2399169978Skmacy 2400169978Skmacy (void)mbufq_dequeue(&q->sendq); 2401169978Skmacy busdma_map_mbufs(&m, q, stx, segs, &nsegs); 2402194521Skmacy TXQ_UNLOCK(qs); 2403169978Skmacy write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 2404194521Skmacy TXQ_LOCK(qs); 2405169978Skmacy } 2406169978Skmacy#if USE_GTS 2407169978Skmacy set_bit(TXQ_RUNNING, &q->flags); 2408169978Skmacy set_bit(TXQ_LAST_PKT_DB, &q->flags); 2409169978Skmacy#endif 2410194521Skmacy TXQ_UNLOCK(qs); 2411176472Skmacy wmb(); 2412169978Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 2413169978Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 2414167514Skmacy} 2415167514Skmacy 2416169978Skmacy/** 2417169978Skmacy * queue_set - return the queue set a packet should use 2418169978Skmacy * @m: the packet 2419169978Skmacy * 2420169978Skmacy * Maps a packet to the SGE queue set it should use. The desired queue 2421169978Skmacy * set is carried in bits 1-3 in the packet's priority. 2422169978Skmacy */ 2423169978Skmacystatic __inline int 2424169978Skmacyqueue_set(const struct mbuf *m) 2425169978Skmacy{ 2426169978Skmacy return m_get_priority(m) >> 1; 2427169978Skmacy} 2428169978Skmacy 2429169978Skmacy/** 2430169978Skmacy * is_ctrl_pkt - return whether an offload packet is a control packet 2431169978Skmacy * @m: the packet 2432169978Skmacy * 2433169978Skmacy * Determines whether an offload packet should use an OFLD or a CTRL 2434169978Skmacy * Tx queue. This is indicated by bit 0 in the packet's priority. 2435169978Skmacy */ 2436169978Skmacystatic __inline int 2437169978Skmacyis_ctrl_pkt(const struct mbuf *m) 2438169978Skmacy{ 2439169978Skmacy return m_get_priority(m) & 1; 2440169978Skmacy} 2441169978Skmacy 2442169978Skmacy/** 2443169978Skmacy * t3_offload_tx - send an offload packet 2444169978Skmacy * @tdev: the offload device to send to 2445169978Skmacy * @m: the packet 2446169978Skmacy * 2447169978Skmacy * Sends an offload packet. We use the packet priority to select the 2448169978Skmacy * appropriate Tx queue as follows: bit 0 indicates whether the packet 2449169978Skmacy * should be sent as regular or control, bits 1-3 select the queue set. 2450169978Skmacy */ 2451169978Skmacyint 2452174626Skmacyt3_offload_tx(struct t3cdev *tdev, struct mbuf *m) 2453169978Skmacy{ 2454169978Skmacy adapter_t *adap = tdev2adap(tdev); 2455169978Skmacy struct sge_qset *qs = &adap->sge.qs[queue_set(m)]; 2456169978Skmacy 2457174708Skmacy if (__predict_false(is_ctrl_pkt(m))) 2458194521Skmacy return ctrl_xmit(adap, qs, m); 2459169978Skmacy 2460194521Skmacy return ofld_xmit(adap, qs, m); 2461169978Skmacy} 2462169978Skmacy 2463169978Skmacy/** 2464169978Skmacy * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts 2465169978Skmacy * @tdev: the offload device that will be receiving the packets 2466169978Skmacy * @q: the SGE response queue that assembled the bundle 2467169978Skmacy * @m: the partial bundle 2468169978Skmacy * @n: the number of packets in the bundle 2469169978Skmacy * 2470169978Skmacy * Delivers a (partial) bundle of Rx offload packets to an offload device. 2471169978Skmacy */ 2472169978Skmacystatic __inline void 2473174626Skmacydeliver_partial_bundle(struct t3cdev *tdev, 2474169978Skmacy struct sge_rspq *q, 2475169978Skmacy struct mbuf *mbufs[], int n) 2476169978Skmacy{ 2477169978Skmacy if (n) { 2478169978Skmacy q->offload_bundles++; 2479169978Skmacy cxgb_ofld_recv(tdev, mbufs, n); 2480169978Skmacy } 2481169978Skmacy} 2482169978Skmacy 2483169978Skmacystatic __inline int 2484174626Skmacyrx_offload(struct t3cdev *tdev, struct sge_rspq *rq, 2485169978Skmacy struct mbuf *m, struct mbuf *rx_gather[], 2486169978Skmacy unsigned int gather_idx) 2487169978Skmacy{ 2488174708Skmacy 2489169978Skmacy rq->offload_pkts++; 2490169978Skmacy m->m_pkthdr.header = mtod(m, void *); 2491169978Skmacy rx_gather[gather_idx++] = m; 2492169978Skmacy if (gather_idx == RX_BUNDLE_SIZE) { 2493169978Skmacy cxgb_ofld_recv(tdev, rx_gather, RX_BUNDLE_SIZE); 2494169978Skmacy gather_idx = 0; 2495169978Skmacy rq->offload_bundles++; 2496169978Skmacy } 2497169978Skmacy return (gather_idx); 2498169978Skmacy} 2499169978Skmacy 2500167514Skmacystatic void 2501167514Skmacyrestart_tx(struct sge_qset *qs) 2502167514Skmacy{ 2503169978Skmacy struct adapter *sc = qs->port->adapter; 2504169978Skmacy 2505174708Skmacy 2506169978Skmacy if (isset(&qs->txq_stopped, TXQ_OFLD) && 2507169978Skmacy should_restart_tx(&qs->txq[TXQ_OFLD]) && 2508169978Skmacy test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { 2509169978Skmacy qs->txq[TXQ_OFLD].restarts++; 2510175200Skmacy DPRINTF("restarting TXQ_OFLD\n"); 2511171335Skmacy taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); 2512169978Skmacy } 2513175200Skmacy DPRINTF("stopped=0x%x restart=%d processed=%d cleaned=%d in_use=%d\n", 2514174708Skmacy qs->txq_stopped, should_restart_tx(&qs->txq[TXQ_CTRL]), 2515174708Skmacy qs->txq[TXQ_CTRL].processed, qs->txq[TXQ_CTRL].cleaned, 2516174708Skmacy qs->txq[TXQ_CTRL].in_use); 2517174708Skmacy 2518169978Skmacy if (isset(&qs->txq_stopped, TXQ_CTRL) && 2519169978Skmacy should_restart_tx(&qs->txq[TXQ_CTRL]) && 2520169978Skmacy test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { 2521169978Skmacy qs->txq[TXQ_CTRL].restarts++; 2522175200Skmacy DPRINTF("restarting TXQ_CTRL\n"); 2523171335Skmacy taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); 2524169978Skmacy } 2525167514Skmacy} 2526167514Skmacy 2527169978Skmacy/** 2528169978Skmacy * t3_sge_alloc_qset - initialize an SGE queue set 2529169978Skmacy * @sc: the controller softc 2530169978Skmacy * @id: the queue set id 2531169978Skmacy * @nports: how many Ethernet ports will be using this queue set 2532169978Skmacy * @irq_vec_idx: the IRQ vector index for response queue interrupts 2533169978Skmacy * @p: configuration parameters for this queue set 2534169978Skmacy * @ntxq: number of Tx queues for the queue set 2535169978Skmacy * @pi: port info for queue set 2536169978Skmacy * 2537169978Skmacy * Allocate resources and initialize an SGE queue set. A queue set 2538169978Skmacy * comprises a response queue, two Rx free-buffer queues, and up to 3 2539169978Skmacy * Tx queues. The Tx queues are assigned roles in the order Ethernet 2540169978Skmacy * queue, offload queue, and control queue. 2541169978Skmacy */ 2542169978Skmacyint 2543169978Skmacyt3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, 2544169978Skmacy const struct qset_params *p, int ntxq, struct port_info *pi) 2545169978Skmacy{ 2546169978Skmacy struct sge_qset *q = &sc->sge.qs[id]; 2547194521Skmacy int i, ret = 0; 2548169978Skmacy 2549194521Skmacy MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF); 2550194521Skmacy q->port = pi; 2551194521Skmacy 2552174708Skmacy for (i = 0; i < SGE_TXQ_PER_SET; i++) { 2553185162Skmacy 2554185162Skmacy if ((q->txq[i].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, 2555194521Skmacy M_DEVBUF, M_WAITOK, &q->lock)) == NULL) { 2556174708Skmacy device_printf(sc->dev, "failed to allocate mbuf ring\n"); 2557174708Skmacy goto err; 2558174708Skmacy } 2559185165Skmacy if ((q->txq[i].txq_ifq = 2560185165Skmacy malloc(sizeof(struct ifaltq), M_DEVBUF, M_NOWAIT|M_ZERO)) 2561185165Skmacy == NULL) { 2562185165Skmacy device_printf(sc->dev, "failed to allocate ifq\n"); 2563185165Skmacy goto err; 2564185165Skmacy } 2565194521Skmacy ifq_init(q->txq[i].txq_ifq, pi->ifp); 2566194521Skmacy callout_init(&q->txq[i].txq_timer, 1); 2567194521Skmacy callout_init(&q->txq[i].txq_watchdog, 1); 2568194521Skmacy q->txq[i].txq_timer.c_cpu = id % mp_ncpus; 2569194521Skmacy q->txq[i].txq_watchdog.c_cpu = id % mp_ncpus; 2570174708Skmacy } 2571169978Skmacy init_qset_cntxt(q, id); 2572175347Skmacy q->idx = id; 2573169978Skmacy if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), 2574169978Skmacy sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, 2575169978Skmacy &q->fl[0].desc, &q->fl[0].sdesc, 2576169978Skmacy &q->fl[0].desc_tag, &q->fl[0].desc_map, 2577169978Skmacy sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { 2578169978Skmacy printf("error %d from alloc ring fl0\n", ret); 2579169978Skmacy goto err; 2580169978Skmacy } 2581169978Skmacy 2582169978Skmacy if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), 2583169978Skmacy sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, 2584169978Skmacy &q->fl[1].desc, &q->fl[1].sdesc, 2585169978Skmacy &q->fl[1].desc_tag, &q->fl[1].desc_map, 2586169978Skmacy sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { 2587169978Skmacy printf("error %d from alloc ring fl1\n", ret); 2588169978Skmacy goto err; 2589169978Skmacy } 2590169978Skmacy 2591169978Skmacy if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, 2592169978Skmacy &q->rspq.phys_addr, &q->rspq.desc, NULL, 2593169978Skmacy &q->rspq.desc_tag, &q->rspq.desc_map, 2594169978Skmacy NULL, NULL)) != 0) { 2595169978Skmacy printf("error %d from alloc ring rspq\n", ret); 2596169978Skmacy goto err; 2597169978Skmacy } 2598169978Skmacy 2599169978Skmacy for (i = 0; i < ntxq; ++i) { 2600169978Skmacy size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); 2601169978Skmacy 2602169978Skmacy if ((ret = alloc_ring(sc, p->txq_size[i], 2603169978Skmacy sizeof(struct tx_desc), sz, 2604169978Skmacy &q->txq[i].phys_addr, &q->txq[i].desc, 2605169978Skmacy &q->txq[i].sdesc, &q->txq[i].desc_tag, 2606169978Skmacy &q->txq[i].desc_map, 2607169978Skmacy sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { 2608169978Skmacy printf("error %d from alloc ring tx %i\n", ret, i); 2609169978Skmacy goto err; 2610169978Skmacy } 2611169978Skmacy mbufq_init(&q->txq[i].sendq); 2612169978Skmacy q->txq[i].gen = 1; 2613169978Skmacy q->txq[i].size = p->txq_size[i]; 2614169978Skmacy } 2615169978Skmacy 2616171335Skmacy TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q); 2617171335Skmacy TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q); 2618194521Skmacy TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, q); 2619194521Skmacy TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, q); 2620171335Skmacy 2621169978Skmacy q->fl[0].gen = q->fl[1].gen = 1; 2622169978Skmacy q->fl[0].size = p->fl_size; 2623169978Skmacy q->fl[1].size = p->jumbo_size; 2624169978Skmacy 2625169978Skmacy q->rspq.gen = 1; 2626171471Skmacy q->rspq.cidx = 0; 2627169978Skmacy q->rspq.size = p->rspq_size; 2628170869Skmacy 2629169978Skmacy q->txq[TXQ_ETH].stop_thres = nports * 2630169978Skmacy flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); 2631169978Skmacy 2632194521Skmacy q->fl[0].buf_size = MCLBYTES; 2633194521Skmacy q->fl[0].zone = zone_pack; 2634194521Skmacy q->fl[0].type = EXT_PACKET; 2635174708Skmacy#if __FreeBSD_version > 800000 2636175200Skmacy if (cxgb_use_16k_clusters) { 2637194521Skmacy q->fl[1].buf_size = MJUM16BYTES; 2638174708Skmacy q->fl[1].zone = zone_jumbo16; 2639174708Skmacy q->fl[1].type = EXT_JUMBO16; 2640175200Skmacy } else { 2641194521Skmacy q->fl[1].buf_size = MJUM9BYTES; 2642175200Skmacy q->fl[1].zone = zone_jumbo9; 2643175200Skmacy q->fl[1].type = EXT_JUMBO9; 2644175200Skmacy } 2645174708Skmacy#else 2646194521Skmacy q->fl[1].buf_size = MJUMPAGESIZE; 2647175200Skmacy q->fl[1].zone = zone_jumbop; 2648175200Skmacy q->fl[1].type = EXT_JUMBOP; 2649174708Skmacy#endif 2650171978Skmacy 2651183289Skmacy /* Allocate and setup the lro_ctrl structure */ 2652181616Skmacy q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO); 2653205947Snp#ifdef INET 2654181616Skmacy ret = tcp_lro_init(&q->lro.ctrl); 2655181616Skmacy if (ret) { 2656181616Skmacy printf("error %d from tcp_lro_init\n", ret); 2657181616Skmacy goto err; 2658181616Skmacy } 2659205947Snp#endif 2660181616Skmacy q->lro.ctrl.ifp = pi->ifp; 2661181616Skmacy 2662176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 2663169978Skmacy ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, 2664169978Skmacy q->rspq.phys_addr, q->rspq.size, 2665169978Skmacy q->fl[0].buf_size, 1, 0); 2666169978Skmacy if (ret) { 2667169978Skmacy printf("error %d from t3_sge_init_rspcntxt\n", ret); 2668169978Skmacy goto err_unlock; 2669169978Skmacy } 2670169978Skmacy 2671169978Skmacy for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 2672169978Skmacy ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, 2673169978Skmacy q->fl[i].phys_addr, q->fl[i].size, 2674169978Skmacy q->fl[i].buf_size, p->cong_thres, 1, 2675169978Skmacy 0); 2676169978Skmacy if (ret) { 2677169978Skmacy printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); 2678169978Skmacy goto err_unlock; 2679169978Skmacy } 2680169978Skmacy } 2681169978Skmacy 2682169978Skmacy ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, 2683169978Skmacy SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, 2684169978Skmacy q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, 2685169978Skmacy 1, 0); 2686169978Skmacy if (ret) { 2687169978Skmacy printf("error %d from t3_sge_init_ecntxt\n", ret); 2688169978Skmacy goto err_unlock; 2689169978Skmacy } 2690169978Skmacy 2691169978Skmacy if (ntxq > 1) { 2692169978Skmacy ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, 2693169978Skmacy USE_GTS, SGE_CNTXT_OFLD, id, 2694169978Skmacy q->txq[TXQ_OFLD].phys_addr, 2695169978Skmacy q->txq[TXQ_OFLD].size, 0, 1, 0); 2696169978Skmacy if (ret) { 2697169978Skmacy printf("error %d from t3_sge_init_ecntxt\n", ret); 2698169978Skmacy goto err_unlock; 2699169978Skmacy } 2700169978Skmacy } 2701169978Skmacy 2702169978Skmacy if (ntxq > 2) { 2703169978Skmacy ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, 2704169978Skmacy SGE_CNTXT_CTRL, id, 2705169978Skmacy q->txq[TXQ_CTRL].phys_addr, 2706169978Skmacy q->txq[TXQ_CTRL].size, 2707169978Skmacy q->txq[TXQ_CTRL].token, 1, 0); 2708169978Skmacy if (ret) { 2709169978Skmacy printf("error %d from t3_sge_init_ecntxt\n", ret); 2710169978Skmacy goto err_unlock; 2711169978Skmacy } 2712169978Skmacy } 2713169978Skmacy 2714170869Skmacy snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", 2715170869Skmacy device_get_unit(sc->dev), irq_vec_idx); 2716170869Skmacy MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); 2717170869Skmacy 2718176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2719169978Skmacy t3_update_qset_coalesce(q, p); 2720169978Skmacy q->port = pi; 2721169978Skmacy 2722169978Skmacy refill_fl(sc, &q->fl[0], q->fl[0].size); 2723169978Skmacy refill_fl(sc, &q->fl[1], q->fl[1].size); 2724169978Skmacy refill_rspq(sc, &q->rspq, q->rspq.size - 1); 2725169978Skmacy 2726169978Skmacy t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | 2727169978Skmacy V_NEWTIMER(q->rspq.holdoff_tmr)); 2728169978Skmacy 2729169978Skmacy return (0); 2730169978Skmacy 2731169978Skmacyerr_unlock: 2732176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2733169978Skmacyerr: 2734194521Skmacy TXQ_LOCK(q); 2735169978Skmacy t3_free_qset(sc, q); 2736169978Skmacy 2737169978Skmacy return (ret); 2738169978Skmacy} 2739169978Skmacy 2740181616Skmacy/* 2741181616Skmacy * Remove CPL_RX_PKT headers from the mbuf and reduce it to a regular mbuf with 2742181616Skmacy * ethernet data. Hardware assistance with various checksums and any vlan tag 2743181616Skmacy * will also be taken into account here. 2744181616Skmacy */ 2745167514Skmacyvoid 2746171978Skmacyt3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad) 2747167514Skmacy{ 2748170654Skmacy struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad); 2749171978Skmacy struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; 2750167514Skmacy struct ifnet *ifp = pi->ifp; 2751167514Skmacy 2752170654Skmacy DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff); 2753167514Skmacy 2754167514Skmacy if ((ifp->if_capenable & IFCAP_RXCSUM) && !cpl->fragment && 2755167514Skmacy cpl->csum_valid && cpl->csum == 0xffff) { 2756167514Skmacy m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); 2757167514Skmacy rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; 2758167514Skmacy m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID|CSUM_DATA_VALID|CSUM_PSEUDO_HDR); 2759167514Skmacy m->m_pkthdr.csum_data = 0xffff; 2760167514Skmacy } 2761204274Snp 2762204274Snp if (cpl->vlan_valid) { 2763167514Skmacy m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); 2764167514Skmacy m->m_flags |= M_VLANTAG; 2765167514Skmacy } 2766204274Snp 2767167514Skmacy m->m_pkthdr.rcvif = ifp; 2768170654Skmacy m->m_pkthdr.header = mtod(m, uint8_t *) + sizeof(*cpl) + ethpad; 2769168737Skmacy /* 2770168737Skmacy * adjust after conversion to mbuf chain 2771168737Skmacy */ 2772174708Skmacy m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); 2773174708Skmacy m->m_len -= (sizeof(*cpl) + ethpad); 2774174708Skmacy m->m_data += (sizeof(*cpl) + ethpad); 2775167514Skmacy} 2776167514Skmacy 2777167514Skmacy/** 2778167514Skmacy * get_packet - return the next ingress packet buffer from a free list 2779167514Skmacy * @adap: the adapter that received the packet 2780167514Skmacy * @drop_thres: # of remaining buffers before we start dropping packets 2781167514Skmacy * @qs: the qset that the SGE free list holding the packet belongs to 2782167514Skmacy * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain 2783167514Skmacy * @r: response descriptor 2784167514Skmacy * 2785167514Skmacy * Get the next packet from a free list and complete setup of the 2786167514Skmacy * sk_buff. If the packet is small we make a copy and recycle the 2787167514Skmacy * original buffer, otherwise we use the original buffer itself. If a 2788167514Skmacy * positive drop threshold is supplied packets are dropped and their 2789167514Skmacy * buffers recycled if (a) the number of remaining buffers is under the 2790167514Skmacy * threshold and the packet is too big to copy, or (b) the packet should 2791167514Skmacy * be copied but there is no memory for the copy. 2792167514Skmacy */ 2793167514Skmacystatic int 2794167514Skmacyget_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2795175340Skmacy struct t3_mbuf_hdr *mh, struct rsp_desc *r) 2796172101Skmacy{ 2797172101Skmacy 2798172101Skmacy unsigned int len_cq = ntohl(r->len_cq); 2799172101Skmacy struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2800194521Skmacy int mask, cidx = fl->cidx; 2801194521Skmacy struct rx_sw_desc *sd = &fl->sdesc[cidx]; 2802172101Skmacy uint32_t len = G_RSPD_LEN(len_cq); 2803194521Skmacy uint32_t flags = M_EXT; 2804194521Skmacy uint8_t sopeop = G_RSPD_SOP_EOP(ntohl(r->flags)); 2805175711Skmacy caddr_t cl; 2806194521Skmacy struct mbuf *m; 2807172101Skmacy int ret = 0; 2808172101Skmacy 2809194521Skmacy mask = fl->size - 1; 2810194521Skmacy prefetch(fl->sdesc[(cidx + 1) & mask].m); 2811194521Skmacy prefetch(fl->sdesc[(cidx + 2) & mask].m); 2812194521Skmacy prefetch(fl->sdesc[(cidx + 1) & mask].rxsd_cl); 2813194521Skmacy prefetch(fl->sdesc[(cidx + 2) & mask].rxsd_cl); 2814194521Skmacy 2815172101Skmacy fl->credits--; 2816172101Skmacy bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); 2817175200Skmacy 2818194521Skmacy if (recycle_enable && len <= SGE_RX_COPY_THRES && 2819194521Skmacy sopeop == RSPQ_SOP_EOP) { 2820194521Skmacy if ((m = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) 2821175200Skmacy goto skip_recycle; 2822194521Skmacy cl = mtod(m, void *); 2823194521Skmacy memcpy(cl, sd->rxsd_cl, len); 2824175200Skmacy recycle_rx_buf(adap, fl, fl->cidx); 2825194521Skmacy m->m_pkthdr.len = m->m_len = len; 2826194521Skmacy m->m_flags = 0; 2827194521Skmacy mh->mh_head = mh->mh_tail = m; 2828194521Skmacy ret = 1; 2829194521Skmacy goto done; 2830175200Skmacy } else { 2831175200Skmacy skip_recycle: 2832175200Skmacy bus_dmamap_unload(fl->entry_tag, sd->map); 2833175200Skmacy cl = sd->rxsd_cl; 2834194521Skmacy m = sd->m; 2835172101Skmacy 2836175200Skmacy if ((sopeop == RSPQ_SOP_EOP) || 2837175200Skmacy (sopeop == RSPQ_SOP)) 2838194521Skmacy flags |= M_PKTHDR; 2839195512Snp m_init(m, fl->zone, fl->buf_size, M_NOWAIT, MT_DATA, flags); 2840194521Skmacy if (fl->zone == zone_pack) { 2841194521Skmacy /* 2842194521Skmacy * restore clobbered data pointer 2843194521Skmacy */ 2844194521Skmacy m->m_data = m->m_ext.ext_buf; 2845194521Skmacy } else { 2846194521Skmacy m_cljset(m, cl, fl->type); 2847194521Skmacy } 2848194521Skmacy m->m_len = len; 2849175200Skmacy } 2850172101Skmacy switch(sopeop) { 2851172101Skmacy case RSPQ_SOP_EOP: 2852194521Skmacy ret = 1; 2853194521Skmacy /* FALLTHROUGH */ 2854194521Skmacy case RSPQ_SOP: 2855172101Skmacy mh->mh_head = mh->mh_tail = m; 2856172101Skmacy m->m_pkthdr.len = len; 2857194521Skmacy break; 2858194521Skmacy case RSPQ_EOP: 2859172101Skmacy ret = 1; 2860194521Skmacy /* FALLTHROUGH */ 2861172101Skmacy case RSPQ_NSOP_NEOP: 2862172101Skmacy if (mh->mh_tail == NULL) { 2863175711Skmacy log(LOG_ERR, "discarding intermediate descriptor entry\n"); 2864172101Skmacy m_freem(m); 2865172101Skmacy break; 2866172101Skmacy } 2867172101Skmacy mh->mh_tail->m_next = m; 2868172101Skmacy mh->mh_tail = m; 2869172101Skmacy mh->mh_head->m_pkthdr.len += len; 2870172101Skmacy break; 2871172101Skmacy } 2872194521Skmacy if (cxgb_debug) 2873194521Skmacy printf("len=%d pktlen=%d\n", m->m_len, m->m_pkthdr.len); 2874194521Skmacydone: 2875172101Skmacy if (++fl->cidx == fl->size) 2876172101Skmacy fl->cidx = 0; 2877172101Skmacy 2878172101Skmacy return (ret); 2879172101Skmacy} 2880172101Skmacy 2881167514Skmacy/** 2882167514Skmacy * handle_rsp_cntrl_info - handles control information in a response 2883167514Skmacy * @qs: the queue set corresponding to the response 2884167514Skmacy * @flags: the response control flags 2885167514Skmacy * 2886167514Skmacy * Handles the control information of an SGE response, such as GTS 2887167514Skmacy * indications and completion credits for the queue set's Tx queues. 2888167514Skmacy * HW coalesces credits, we don't do any extra SW coalescing. 2889167514Skmacy */ 2890167514Skmacystatic __inline void 2891167514Skmacyhandle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags) 2892167514Skmacy{ 2893167514Skmacy unsigned int credits; 2894167514Skmacy 2895167514Skmacy#if USE_GTS 2896167514Skmacy if (flags & F_RSPD_TXQ0_GTS) 2897167514Skmacy clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); 2898167514Skmacy#endif 2899167514Skmacy credits = G_RSPD_TXQ0_CR(flags); 2900175200Skmacy if (credits) 2901167514Skmacy qs->txq[TXQ_ETH].processed += credits; 2902197043Snp 2903167514Skmacy credits = G_RSPD_TXQ2_CR(flags); 2904197043Snp if (credits) 2905167514Skmacy qs->txq[TXQ_CTRL].processed += credits; 2906167514Skmacy 2907167514Skmacy# if USE_GTS 2908167514Skmacy if (flags & F_RSPD_TXQ1_GTS) 2909167514Skmacy clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); 2910167514Skmacy# endif 2911167514Skmacy credits = G_RSPD_TXQ1_CR(flags); 2912167514Skmacy if (credits) 2913167514Skmacy qs->txq[TXQ_OFLD].processed += credits; 2914174708Skmacy 2915167514Skmacy} 2916167514Skmacy 2917167514Skmacystatic void 2918167514Skmacycheck_ring_db(adapter_t *adap, struct sge_qset *qs, 2919167514Skmacy unsigned int sleeping) 2920167514Skmacy{ 2921167514Skmacy ; 2922167514Skmacy} 2923167514Skmacy 2924167514Skmacy/** 2925167514Skmacy * process_responses - process responses from an SGE response queue 2926167514Skmacy * @adap: the adapter 2927167514Skmacy * @qs: the queue set to which the response queue belongs 2928167514Skmacy * @budget: how many responses can be processed in this round 2929167514Skmacy * 2930167514Skmacy * Process responses from an SGE response queue up to the supplied budget. 2931167514Skmacy * Responses include received packets as well as credits and other events 2932167514Skmacy * for the queues that belong to the response queue's queue set. 2933167514Skmacy * A negative budget is effectively unlimited. 2934167514Skmacy * 2935167514Skmacy * Additionally choose the interrupt holdoff time for the next interrupt 2936167514Skmacy * on this queue. If the system is under memory shortage use a fairly 2937167514Skmacy * long delay to help recovery. 2938167514Skmacy */ 2939194521Skmacystatic int 2940167514Skmacyprocess_responses(adapter_t *adap, struct sge_qset *qs, int budget) 2941167514Skmacy{ 2942167514Skmacy struct sge_rspq *rspq = &qs->rspq; 2943167514Skmacy struct rsp_desc *r = &rspq->desc[rspq->cidx]; 2944167514Skmacy int budget_left = budget; 2945167514Skmacy unsigned int sleeping = 0; 2946181616Skmacy int lro_enabled = qs->lro.enabled; 2947183559Skmacy int skip_lro; 2948181616Skmacy struct lro_ctrl *lro_ctrl = &qs->lro.ctrl; 2949169978Skmacy struct mbuf *offload_mbufs[RX_BUNDLE_SIZE]; 2950169978Skmacy int ngathered = 0; 2951167514Skmacy#ifdef DEBUG 2952167514Skmacy static int last_holdoff = 0; 2953171471Skmacy if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { 2954167514Skmacy printf("next_holdoff=%d\n", rspq->holdoff_tmr); 2955167514Skmacy last_holdoff = rspq->holdoff_tmr; 2956167514Skmacy } 2957169978Skmacy#endif 2958167514Skmacy rspq->next_holdoff = rspq->holdoff_tmr; 2959167514Skmacy 2960167514Skmacy while (__predict_true(budget_left && is_new_response(r, rspq))) { 2961167514Skmacy int eth, eop = 0, ethpad = 0; 2962167514Skmacy uint32_t flags = ntohl(r->flags); 2963167514Skmacy uint32_t rss_csum = *(const uint32_t *)r; 2964174708Skmacy uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val); 2965167514Skmacy 2966167514Skmacy eth = (r->rss_hdr.opcode == CPL_RX_PKT); 2967167514Skmacy 2968167514Skmacy if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) { 2969176472Skmacy struct mbuf *m; 2970167514Skmacy 2971176472Skmacy if (cxgb_debug) 2972176472Skmacy printf("async notification\n"); 2973176472Skmacy 2974176472Skmacy if (rspq->rspq_mh.mh_head == NULL) { 2975176472Skmacy rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); 2976176472Skmacy m = rspq->rspq_mh.mh_head; 2977176472Skmacy } else { 2978176472Skmacy m = m_gethdr(M_DONTWAIT, MT_DATA); 2979176472Skmacy } 2980176472Skmacy if (m == NULL) 2981176472Skmacy goto no_mem; 2982176472Skmacy 2983176472Skmacy memcpy(mtod(m, char *), r, AN_PKT_SIZE); 2984176472Skmacy m->m_len = m->m_pkthdr.len = AN_PKT_SIZE; 2985176472Skmacy *mtod(m, char *) = CPL_ASYNC_NOTIF; 2986176472Skmacy rss_csum = htonl(CPL_ASYNC_NOTIF << 24); 2987176472Skmacy eop = 1; 2988176472Skmacy rspq->async_notif++; 2989176472Skmacy goto skip; 2990167514Skmacy } else if (flags & F_RSPD_IMM_DATA_VALID) { 2991175200Skmacy struct mbuf *m = NULL; 2992175200Skmacy 2993175200Skmacy DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", 2994175200Skmacy r->rss_hdr.opcode, rspq->cidx); 2995175711Skmacy if (rspq->rspq_mh.mh_head == NULL) 2996175711Skmacy rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); 2997174708Skmacy else 2998171471Skmacy m = m_gethdr(M_DONTWAIT, MT_DATA); 2999168491Skmacy 3000176472Skmacy if (rspq->rspq_mh.mh_head == NULL && m == NULL) { 3001176472Skmacy no_mem: 3002167514Skmacy rspq->next_holdoff = NOMEM_INTR_DELAY; 3003167514Skmacy budget_left--; 3004167514Skmacy break; 3005167514Skmacy } 3006176472Skmacy get_imm_packet(adap, r, rspq->rspq_mh.mh_head); 3007168491Skmacy eop = 1; 3008174708Skmacy rspq->imm_data++; 3009176472Skmacy } else if (r->len_cq) { 3010167514Skmacy int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; 3011172101Skmacy 3012175340Skmacy eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r); 3013194521Skmacy if (eop) { 3014194521Skmacy rspq->rspq_mh.mh_head->m_flags |= M_FLOWID; 3015194521Skmacy rspq->rspq_mh.mh_head->m_pkthdr.flowid = rss_hash; 3016194521Skmacy } 3017194521Skmacy 3018167514Skmacy ethpad = 2; 3019167514Skmacy } else { 3020167514Skmacy rspq->pure_rsps++; 3021167514Skmacy } 3022176472Skmacy skip: 3023167514Skmacy if (flags & RSPD_CTRL_MASK) { 3024167514Skmacy sleeping |= flags & RSPD_GTS_MASK; 3025167514Skmacy handle_rsp_cntrl_info(qs, flags); 3026167514Skmacy } 3027174708Skmacy 3028167514Skmacy r++; 3029167514Skmacy if (__predict_false(++rspq->cidx == rspq->size)) { 3030167514Skmacy rspq->cidx = 0; 3031167514Skmacy rspq->gen ^= 1; 3032167514Skmacy r = rspq->desc; 3033167514Skmacy } 3034194521Skmacy 3035167514Skmacy if (++rspq->credits >= (rspq->size / 4)) { 3036167514Skmacy refill_rspq(adap, rspq, rspq->credits); 3037167514Skmacy rspq->credits = 0; 3038167514Skmacy } 3039174708Skmacy if (!eth && eop) { 3040174708Skmacy rspq->rspq_mh.mh_head->m_pkthdr.csum_data = rss_csum; 3041174708Skmacy /* 3042174708Skmacy * XXX size mismatch 3043174708Skmacy */ 3044174708Skmacy m_set_priority(rspq->rspq_mh.mh_head, rss_hash); 3045176472Skmacy 3046176472Skmacy 3047174708Skmacy ngathered = rx_offload(&adap->tdev, rspq, 3048174708Skmacy rspq->rspq_mh.mh_head, offload_mbufs, ngathered); 3049174708Skmacy rspq->rspq_mh.mh_head = NULL; 3050174708Skmacy DPRINTF("received offload packet\n"); 3051174708Skmacy 3052174708Skmacy } else if (eth && eop) { 3053181616Skmacy struct mbuf *m = rspq->rspq_mh.mh_head; 3054167514Skmacy 3055181616Skmacy t3_rx_eth(adap, rspq, m, ethpad); 3056183559Skmacy 3057183559Skmacy /* 3058183559Skmacy * The T304 sends incoming packets on any qset. If LRO 3059183559Skmacy * is also enabled, we could end up sending packet up 3060183559Skmacy * lro_ctrl->ifp's input. That is incorrect. 3061183559Skmacy * 3062183559Skmacy * The mbuf's rcvif was derived from the cpl header and 3063183559Skmacy * is accurate. Skip LRO and just use that. 3064183559Skmacy */ 3065183559Skmacy skip_lro = __predict_false(qs->port->ifp != m->m_pkthdr.rcvif); 3066183559Skmacy 3067205947Snp if (lro_enabled && lro_ctrl->lro_cnt && !skip_lro 3068205947Snp#ifdef INET 3069205947Snp && (tcp_lro_rx(lro_ctrl, m, 0) == 0) 3070205947Snp#endif 3071205947Snp ) { 3072181616Skmacy /* successfully queue'd for LRO */ 3073204274Snp } else { 3074181616Skmacy /* 3075181616Skmacy * LRO not enabled, packet unsuitable for LRO, 3076181616Skmacy * or unable to queue. Pass it up right now in 3077181616Skmacy * either case. 3078181616Skmacy */ 3079181616Skmacy struct ifnet *ifp = m->m_pkthdr.rcvif; 3080181616Skmacy (*ifp->if_input)(ifp, m); 3081181616Skmacy } 3082181616Skmacy rspq->rspq_mh.mh_head = NULL; 3083171469Skmacy 3084167514Skmacy } 3085174708Skmacy __refill_fl_lt(adap, &qs->fl[0], 32); 3086174708Skmacy __refill_fl_lt(adap, &qs->fl[1], 32); 3087167514Skmacy --budget_left; 3088167514Skmacy } 3089167514Skmacy 3090169978Skmacy deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered); 3091181616Skmacy 3092205947Snp#ifdef INET 3093181616Skmacy /* Flush LRO */ 3094181616Skmacy while (!SLIST_EMPTY(&lro_ctrl->lro_active)) { 3095181616Skmacy struct lro_entry *queued = SLIST_FIRST(&lro_ctrl->lro_active); 3096181616Skmacy SLIST_REMOVE_HEAD(&lro_ctrl->lro_active, next); 3097181616Skmacy tcp_lro_flush(lro_ctrl, queued); 3098181616Skmacy } 3099205947Snp#endif 3100181616Skmacy 3101167514Skmacy if (sleeping) 3102167514Skmacy check_ring_db(adap, qs, sleeping); 3103167514Skmacy 3104194521Skmacy mb(); /* commit Tx queue processed updates */ 3105197043Snp if (__predict_false(qs->txq_stopped > 1)) 3106167514Skmacy restart_tx(qs); 3107197043Snp 3108174708Skmacy __refill_fl_lt(adap, &qs->fl[0], 512); 3109174708Skmacy __refill_fl_lt(adap, &qs->fl[1], 512); 3110167514Skmacy budget -= budget_left; 3111167514Skmacy return (budget); 3112167514Skmacy} 3113167514Skmacy 3114167514Skmacy/* 3115167514Skmacy * A helper function that processes responses and issues GTS. 3116167514Skmacy */ 3117167514Skmacystatic __inline int 3118167514Skmacyprocess_responses_gts(adapter_t *adap, struct sge_rspq *rq) 3119167514Skmacy{ 3120167514Skmacy int work; 3121167514Skmacy static int last_holdoff = 0; 3122167514Skmacy 3123167514Skmacy work = process_responses(adap, rspq_to_qset(rq), -1); 3124167514Skmacy 3125167514Skmacy if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { 3126167514Skmacy printf("next_holdoff=%d\n", rq->next_holdoff); 3127167514Skmacy last_holdoff = rq->next_holdoff; 3128167514Skmacy } 3129175223Skmacy t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | 3130175223Skmacy V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); 3131175223Skmacy 3132175223Skmacy return (work); 3133167514Skmacy} 3134167514Skmacy 3135167514Skmacy 3136167514Skmacy/* 3137167514Skmacy * Interrupt handler for legacy INTx interrupts for T3B-based cards. 3138167514Skmacy * Handles data events from SGE response queues as well as error and other 3139167514Skmacy * async events as they all use the same interrupt pin. We use one SGE 3140167514Skmacy * response queue per port in this mode and protect all response queues with 3141167514Skmacy * queue 0's lock. 3142167514Skmacy */ 3143167514Skmacyvoid 3144167514Skmacyt3b_intr(void *data) 3145167514Skmacy{ 3146171978Skmacy uint32_t i, map; 3147167514Skmacy adapter_t *adap = data; 3148167514Skmacy struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 3149167514Skmacy 3150167514Skmacy t3_write_reg(adap, A_PL_CLI, 0); 3151167514Skmacy map = t3_read_reg(adap, A_SG_DATA_INTR); 3152167514Skmacy 3153167514Skmacy if (!map) 3154167514Skmacy return; 3155167514Skmacy 3156167514Skmacy if (__predict_false(map & F_ERRINTR)) 3157167514Skmacy taskqueue_enqueue(adap->tq, &adap->slow_intr_task); 3158172096Skmacy 3159167514Skmacy mtx_lock(&q0->lock); 3160171978Skmacy for_each_port(adap, i) 3161171978Skmacy if (map & (1 << i)) 3162171978Skmacy process_responses_gts(adap, &adap->sge.qs[i].rspq); 3163167514Skmacy mtx_unlock(&q0->lock); 3164167514Skmacy} 3165167514Skmacy 3166167514Skmacy/* 3167167514Skmacy * The MSI interrupt handler. This needs to handle data events from SGE 3168167514Skmacy * response queues as well as error and other async events as they all use 3169167514Skmacy * the same MSI vector. We use one SGE response queue per port in this mode 3170167514Skmacy * and protect all response queues with queue 0's lock. 3171167514Skmacy */ 3172167514Skmacyvoid 3173167514Skmacyt3_intr_msi(void *data) 3174167514Skmacy{ 3175167514Skmacy adapter_t *adap = data; 3176167514Skmacy struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 3177171978Skmacy int i, new_packets = 0; 3178172096Skmacy 3179167514Skmacy mtx_lock(&q0->lock); 3180167514Skmacy 3181171978Skmacy for_each_port(adap, i) 3182171978Skmacy if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) 3183171978Skmacy new_packets = 1; 3184167514Skmacy mtx_unlock(&q0->lock); 3185167514Skmacy if (new_packets == 0) 3186167514Skmacy taskqueue_enqueue(adap->tq, &adap->slow_intr_task); 3187167514Skmacy} 3188167514Skmacy 3189167514Skmacyvoid 3190167514Skmacyt3_intr_msix(void *data) 3191167514Skmacy{ 3192167514Skmacy struct sge_qset *qs = data; 3193167514Skmacy adapter_t *adap = qs->port->adapter; 3194167514Skmacy struct sge_rspq *rspq = &qs->rspq; 3195194521Skmacy 3196194521Skmacy if (process_responses_gts(adap, rspq) == 0) 3197194521Skmacy rspq->unhandled_irqs++; 3198167514Skmacy} 3199167514Skmacy 3200175200Skmacy#define QDUMP_SBUF_SIZE 32 * 400 3201175209Skmacystatic int 3202175209Skmacyt3_dump_rspq(SYSCTL_HANDLER_ARGS) 3203175209Skmacy{ 3204175209Skmacy struct sge_rspq *rspq; 3205175223Skmacy struct sge_qset *qs; 3206175209Skmacy int i, err, dump_end, idx; 3207175209Skmacy static int multiplier = 1; 3208175209Skmacy struct sbuf *sb; 3209175209Skmacy struct rsp_desc *rspd; 3210175223Skmacy uint32_t data[4]; 3211175209Skmacy 3212175209Skmacy rspq = arg1; 3213175223Skmacy qs = rspq_to_qset(rspq); 3214175209Skmacy if (rspq->rspq_dump_count == 0) 3215175209Skmacy return (0); 3216175209Skmacy if (rspq->rspq_dump_count > RSPQ_Q_SIZE) { 3217175209Skmacy log(LOG_WARNING, 3218175209Skmacy "dump count is too large %d\n", rspq->rspq_dump_count); 3219175209Skmacy rspq->rspq_dump_count = 0; 3220175209Skmacy return (EINVAL); 3221175209Skmacy } 3222175209Skmacy if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) { 3223175209Skmacy log(LOG_WARNING, 3224175209Skmacy "dump start of %d is greater than queue size\n", 3225175209Skmacy rspq->rspq_dump_start); 3226175209Skmacy rspq->rspq_dump_start = 0; 3227175209Skmacy return (EINVAL); 3228175209Skmacy } 3229175223Skmacy err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data); 3230175223Skmacy if (err) 3231175223Skmacy return (err); 3232175209Skmacyretry_sbufops: 3233175209Skmacy sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3234175223Skmacy 3235175223Skmacy sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n", 3236175223Skmacy (data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f), 3237175223Skmacy ((data[2] >> 26) & 1), ((data[2] >> 27) & 1)); 3238175223Skmacy sbuf_printf(sb, " generation=%u CQ mode=%u FL threshold=%u\n", 3239175223Skmacy ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]); 3240175209Skmacy 3241175223Skmacy sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start, 3242175209Skmacy (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1)); 3243175223Skmacy 3244175209Skmacy dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count; 3245175209Skmacy for (i = rspq->rspq_dump_start; i < dump_end; i++) { 3246175209Skmacy idx = i & (RSPQ_Q_SIZE-1); 3247175209Skmacy 3248175209Skmacy rspd = &rspq->desc[idx]; 3249175209Skmacy sbuf_printf(sb, "\tidx=%04d opcode=%02x cpu_idx=%x hash_type=%x cq_idx=%x\n", 3250175209Skmacy idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx, 3251175209Skmacy rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx)); 3252175209Skmacy sbuf_printf(sb, "\trss_hash_val=%x flags=%08x len_cq=%x intr_gen=%x\n", 3253175209Skmacy rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags), 3254175209Skmacy be32toh(rspd->len_cq), rspd->intr_gen); 3255175209Skmacy } 3256175209Skmacy if (sbuf_overflowed(sb)) { 3257175209Skmacy sbuf_delete(sb); 3258175209Skmacy multiplier++; 3259175209Skmacy goto retry_sbufops; 3260175209Skmacy } 3261175209Skmacy sbuf_finish(sb); 3262175209Skmacy err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3263175209Skmacy sbuf_delete(sb); 3264175209Skmacy return (err); 3265175209Skmacy} 3266175209Skmacy 3267167514Skmacystatic int 3268176472Skmacyt3_dump_txq_eth(SYSCTL_HANDLER_ARGS) 3269175200Skmacy{ 3270175200Skmacy struct sge_txq *txq; 3271175200Skmacy struct sge_qset *qs; 3272175200Skmacy int i, j, err, dump_end; 3273175200Skmacy static int multiplier = 1; 3274175200Skmacy struct sbuf *sb; 3275175200Skmacy struct tx_desc *txd; 3276175200Skmacy uint32_t *WR, wr_hi, wr_lo, gen; 3277175223Skmacy uint32_t data[4]; 3278175200Skmacy 3279175200Skmacy txq = arg1; 3280175200Skmacy qs = txq_to_qset(txq, TXQ_ETH); 3281175200Skmacy if (txq->txq_dump_count == 0) { 3282175200Skmacy return (0); 3283175200Skmacy } 3284175200Skmacy if (txq->txq_dump_count > TX_ETH_Q_SIZE) { 3285175200Skmacy log(LOG_WARNING, 3286175200Skmacy "dump count is too large %d\n", txq->txq_dump_count); 3287175200Skmacy txq->txq_dump_count = 1; 3288175200Skmacy return (EINVAL); 3289175200Skmacy } 3290175200Skmacy if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) { 3291175200Skmacy log(LOG_WARNING, 3292175200Skmacy "dump start of %d is greater than queue size\n", 3293175200Skmacy txq->txq_dump_start); 3294175200Skmacy txq->txq_dump_start = 0; 3295175200Skmacy return (EINVAL); 3296175200Skmacy } 3297176472Skmacy err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data); 3298175223Skmacy if (err) 3299175223Skmacy return (err); 3300175223Skmacy 3301175223Skmacy 3302175200Skmacyretry_sbufops: 3303175200Skmacy sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3304175223Skmacy 3305175223Skmacy sbuf_printf(sb, " \n credits=%u GTS=%u index=%u size=%u rspq#=%u cmdq#=%u\n", 3306175223Skmacy (data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16), 3307175223Skmacy (data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1)); 3308175223Skmacy sbuf_printf(sb, " TUN=%u TOE=%u generation%u uP token=%u valid=%u\n", 3309175223Skmacy ((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1), 3310175223Skmacy ((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1)); 3311175223Skmacy sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, 3312175200Skmacy txq->txq_dump_start, 3313175200Skmacy (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1)); 3314175200Skmacy 3315175200Skmacy dump_end = txq->txq_dump_start + txq->txq_dump_count; 3316175200Skmacy for (i = txq->txq_dump_start; i < dump_end; i++) { 3317175200Skmacy txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)]; 3318175200Skmacy WR = (uint32_t *)txd->flit; 3319175200Skmacy wr_hi = ntohl(WR[0]); 3320175200Skmacy wr_lo = ntohl(WR[1]); 3321175200Skmacy gen = G_WR_GEN(wr_lo); 3322175200Skmacy 3323175200Skmacy sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", 3324175200Skmacy wr_hi, wr_lo, gen); 3325175200Skmacy for (j = 2; j < 30; j += 4) 3326175200Skmacy sbuf_printf(sb, "\t%08x %08x %08x %08x \n", 3327175200Skmacy WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); 3328175200Skmacy 3329175200Skmacy } 3330175200Skmacy if (sbuf_overflowed(sb)) { 3331175200Skmacy sbuf_delete(sb); 3332175200Skmacy multiplier++; 3333175200Skmacy goto retry_sbufops; 3334175200Skmacy } 3335175200Skmacy sbuf_finish(sb); 3336175200Skmacy err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3337175200Skmacy sbuf_delete(sb); 3338175200Skmacy return (err); 3339175200Skmacy} 3340175200Skmacy 3341176472Skmacystatic int 3342176472Skmacyt3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS) 3343176472Skmacy{ 3344176472Skmacy struct sge_txq *txq; 3345176472Skmacy struct sge_qset *qs; 3346176472Skmacy int i, j, err, dump_end; 3347176472Skmacy static int multiplier = 1; 3348176472Skmacy struct sbuf *sb; 3349176472Skmacy struct tx_desc *txd; 3350176472Skmacy uint32_t *WR, wr_hi, wr_lo, gen; 3351176472Skmacy 3352176472Skmacy txq = arg1; 3353176472Skmacy qs = txq_to_qset(txq, TXQ_CTRL); 3354176472Skmacy if (txq->txq_dump_count == 0) { 3355176472Skmacy return (0); 3356176472Skmacy } 3357176472Skmacy if (txq->txq_dump_count > 256) { 3358176472Skmacy log(LOG_WARNING, 3359176472Skmacy "dump count is too large %d\n", txq->txq_dump_count); 3360176472Skmacy txq->txq_dump_count = 1; 3361176472Skmacy return (EINVAL); 3362176472Skmacy } 3363176472Skmacy if (txq->txq_dump_start > 255) { 3364176472Skmacy log(LOG_WARNING, 3365176472Skmacy "dump start of %d is greater than queue size\n", 3366176472Skmacy txq->txq_dump_start); 3367176472Skmacy txq->txq_dump_start = 0; 3368176472Skmacy return (EINVAL); 3369176472Skmacy } 3370175200Skmacy 3371176472Skmacyretry_sbufops: 3372176472Skmacy sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3373176472Skmacy sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, 3374176472Skmacy txq->txq_dump_start, 3375176472Skmacy (txq->txq_dump_start + txq->txq_dump_count) & 255); 3376176472Skmacy 3377176472Skmacy dump_end = txq->txq_dump_start + txq->txq_dump_count; 3378176472Skmacy for (i = txq->txq_dump_start; i < dump_end; i++) { 3379176472Skmacy txd = &txq->desc[i & (255)]; 3380176472Skmacy WR = (uint32_t *)txd->flit; 3381176472Skmacy wr_hi = ntohl(WR[0]); 3382176472Skmacy wr_lo = ntohl(WR[1]); 3383176472Skmacy gen = G_WR_GEN(wr_lo); 3384176472Skmacy 3385176472Skmacy sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", 3386176472Skmacy wr_hi, wr_lo, gen); 3387176472Skmacy for (j = 2; j < 30; j += 4) 3388176472Skmacy sbuf_printf(sb, "\t%08x %08x %08x %08x \n", 3389176472Skmacy WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); 3390176472Skmacy 3391176472Skmacy } 3392176472Skmacy if (sbuf_overflowed(sb)) { 3393176472Skmacy sbuf_delete(sb); 3394176472Skmacy multiplier++; 3395176472Skmacy goto retry_sbufops; 3396176472Skmacy } 3397176472Skmacy sbuf_finish(sb); 3398176472Skmacy err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3399176472Skmacy sbuf_delete(sb); 3400176472Skmacy return (err); 3401176472Skmacy} 3402176472Skmacy 3403175200Skmacystatic int 3404180583Skmacyt3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS) 3405167514Skmacy{ 3406167514Skmacy adapter_t *sc = arg1; 3407167514Skmacy struct qset_params *qsp = &sc->params.sge.qset[0]; 3408180583Skmacy int coalesce_usecs; 3409167514Skmacy struct sge_qset *qs; 3410167514Skmacy int i, j, err, nqsets = 0; 3411167514Skmacy struct mtx *lock; 3412174708Skmacy 3413174708Skmacy if ((sc->flags & FULL_INIT_DONE) == 0) 3414174708Skmacy return (ENXIO); 3415174708Skmacy 3416180583Skmacy coalesce_usecs = qsp->coalesce_usecs; 3417180583Skmacy err = sysctl_handle_int(oidp, &coalesce_usecs, arg2, req); 3418167514Skmacy 3419167514Skmacy if (err != 0) { 3420167514Skmacy return (err); 3421167514Skmacy } 3422180583Skmacy if (coalesce_usecs == qsp->coalesce_usecs) 3423167514Skmacy return (0); 3424167514Skmacy 3425167514Skmacy for (i = 0; i < sc->params.nports; i++) 3426167514Skmacy for (j = 0; j < sc->port[i].nqsets; j++) 3427167514Skmacy nqsets++; 3428167514Skmacy 3429180583Skmacy coalesce_usecs = max(1, coalesce_usecs); 3430167514Skmacy 3431167514Skmacy for (i = 0; i < nqsets; i++) { 3432167514Skmacy qs = &sc->sge.qs[i]; 3433167514Skmacy qsp = &sc->params.sge.qset[i]; 3434180583Skmacy qsp->coalesce_usecs = coalesce_usecs; 3435167514Skmacy 3436167514Skmacy lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 3437167514Skmacy &sc->sge.qs[0].rspq.lock; 3438167514Skmacy 3439167514Skmacy mtx_lock(lock); 3440167514Skmacy t3_update_qset_coalesce(qs, qsp); 3441167514Skmacy t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | 3442167514Skmacy V_NEWTIMER(qs->rspq.holdoff_tmr)); 3443167514Skmacy mtx_unlock(lock); 3444167514Skmacy } 3445167514Skmacy 3446167514Skmacy return (0); 3447167514Skmacy} 3448167514Skmacy 3449167514Skmacy 3450167514Skmacyvoid 3451174708Skmacyt3_add_attach_sysctls(adapter_t *sc) 3452167514Skmacy{ 3453167514Skmacy struct sysctl_ctx_list *ctx; 3454167514Skmacy struct sysctl_oid_list *children; 3455174708Skmacy 3456167514Skmacy ctx = device_get_sysctl_ctx(sc->dev); 3457167514Skmacy children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 3458167514Skmacy 3459167514Skmacy /* random information */ 3460167514Skmacy SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 3461167514Skmacy "firmware_version", 3462167514Skmacy CTLFLAG_RD, &sc->fw_version, 3463167514Skmacy 0, "firmware version"); 3464167514Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3465176472Skmacy "hw_revision", 3466176472Skmacy CTLFLAG_RD, &sc->params.rev, 3467176472Skmacy 0, "chip model"); 3468192540Sgnn SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 3469192540Sgnn "port_types", 3470192540Sgnn CTLFLAG_RD, &sc->port_types, 3471192540Sgnn 0, "type of ports"); 3472176472Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3473167514Skmacy "enable_debug", 3474167514Skmacy CTLFLAG_RW, &cxgb_debug, 3475167514Skmacy 0, "enable verbose debugging output"); 3476194521Skmacy SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tunq_coalesce", 3477174708Skmacy CTLFLAG_RD, &sc->tunq_coalesce, 3478174708Skmacy "#tunneled packets freed"); 3479168737Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3480171335Skmacy "txq_overrun", 3481171335Skmacy CTLFLAG_RD, &txq_fills, 3482171335Skmacy 0, "#times txq overrun"); 3483167514Skmacy} 3484167514Skmacy 3485175209Skmacy 3486175209Skmacystatic const char *rspq_name = "rspq"; 3487175209Skmacystatic const char *txq_names[] = 3488175209Skmacy{ 3489175209Skmacy "txq_eth", 3490175209Skmacy "txq_ofld", 3491175209Skmacy "txq_ctrl" 3492181652Skmacy}; 3493175209Skmacy 3494181652Skmacystatic int 3495181652Skmacysysctl_handle_macstat(SYSCTL_HANDLER_ARGS) 3496181652Skmacy{ 3497181652Skmacy struct port_info *p = arg1; 3498181652Skmacy uint64_t *parg; 3499181652Skmacy 3500181652Skmacy if (!p) 3501181652Skmacy return (EINVAL); 3502181652Skmacy 3503181652Skmacy parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2); 3504181652Skmacy PORT_LOCK(p); 3505181652Skmacy t3_mac_update_stats(&p->mac); 3506181652Skmacy PORT_UNLOCK(p); 3507181652Skmacy 3508181652Skmacy return (sysctl_handle_quad(oidp, parg, 0, req)); 3509181652Skmacy} 3510181652Skmacy 3511174708Skmacyvoid 3512174708Skmacyt3_add_configured_sysctls(adapter_t *sc) 3513174708Skmacy{ 3514174708Skmacy struct sysctl_ctx_list *ctx; 3515174708Skmacy struct sysctl_oid_list *children; 3516174708Skmacy int i, j; 3517174708Skmacy 3518174708Skmacy ctx = device_get_sysctl_ctx(sc->dev); 3519174708Skmacy children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 3520174708Skmacy 3521174708Skmacy SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 3522174708Skmacy "intr_coal", 3523174708Skmacy CTLTYPE_INT|CTLFLAG_RW, sc, 3524180583Skmacy 0, t3_set_coalesce_usecs, 3525180583Skmacy "I", "interrupt coalescing timer (us)"); 3526174708Skmacy 3527174708Skmacy for (i = 0; i < sc->params.nports; i++) { 3528174708Skmacy struct port_info *pi = &sc->port[i]; 3529174708Skmacy struct sysctl_oid *poid; 3530174708Skmacy struct sysctl_oid_list *poidlist; 3531181652Skmacy struct mac_stats *mstats = &pi->mac.stats; 3532174708Skmacy 3533174708Skmacy snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i); 3534174708Skmacy poid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, 3535174708Skmacy pi->namebuf, CTLFLAG_RD, NULL, "port statistics"); 3536174708Skmacy poidlist = SYSCTL_CHILDREN(poid); 3537174708Skmacy SYSCTL_ADD_INT(ctx, poidlist, OID_AUTO, 3538174708Skmacy "nqsets", CTLFLAG_RD, &pi->nqsets, 3539174708Skmacy 0, "#queue sets"); 3540181652Skmacy 3541174708Skmacy for (j = 0; j < pi->nqsets; j++) { 3542174708Skmacy struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j]; 3543189643Sgnn struct sysctl_oid *qspoid, *rspqpoid, *txqpoid, 3544189643Sgnn *ctrlqpoid, *lropoid; 3545189643Sgnn struct sysctl_oid_list *qspoidlist, *rspqpoidlist, 3546189643Sgnn *txqpoidlist, *ctrlqpoidlist, 3547189643Sgnn *lropoidlist; 3548174708Skmacy struct sge_txq *txq = &qs->txq[TXQ_ETH]; 3549174708Skmacy 3550174708Skmacy snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j); 3551174708Skmacy 3552174708Skmacy qspoid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, 3553174708Skmacy qs->namebuf, CTLFLAG_RD, NULL, "qset statistics"); 3554174708Skmacy qspoidlist = SYSCTL_CHILDREN(qspoid); 3555189643Sgnn 3556189643Sgnn SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl0_empty", 3557189643Sgnn CTLFLAG_RD, &qs->fl[0].empty, 0, 3558189643Sgnn "freelist #0 empty"); 3559189643Sgnn SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl1_empty", 3560189643Sgnn CTLFLAG_RD, &qs->fl[1].empty, 0, 3561189643Sgnn "freelist #1 empty"); 3562189643Sgnn 3563175209Skmacy rspqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3564175209Skmacy rspq_name, CTLFLAG_RD, NULL, "rspq statistics"); 3565175209Skmacy rspqpoidlist = SYSCTL_CHILDREN(rspqpoid); 3566175209Skmacy 3567175209Skmacy txqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3568175209Skmacy txq_names[0], CTLFLAG_RD, NULL, "txq statistics"); 3569175209Skmacy txqpoidlist = SYSCTL_CHILDREN(txqpoid); 3570175209Skmacy 3571176472Skmacy ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3572176472Skmacy txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics"); 3573176472Skmacy ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid); 3574176472Skmacy 3575181652Skmacy lropoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3576181652Skmacy "lro_stats", CTLFLAG_RD, NULL, "LRO statistics"); 3577181652Skmacy lropoidlist = SYSCTL_CHILDREN(lropoid); 3578181652Skmacy 3579175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size", 3580175209Skmacy CTLFLAG_RD, &qs->rspq.size, 3581175209Skmacy 0, "#entries in response queue"); 3582175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "cidx", 3583175209Skmacy CTLFLAG_RD, &qs->rspq.cidx, 3584175209Skmacy 0, "consumer index"); 3585175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "credits", 3586175209Skmacy CTLFLAG_RD, &qs->rspq.credits, 3587175209Skmacy 0, "#credits"); 3588175209Skmacy SYSCTL_ADD_XLONG(ctx, rspqpoidlist, OID_AUTO, "phys_addr", 3589175209Skmacy CTLFLAG_RD, &qs->rspq.phys_addr, 3590175209Skmacy "physical_address_of the queue"); 3591175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_start", 3592175209Skmacy CTLFLAG_RW, &qs->rspq.rspq_dump_start, 3593175209Skmacy 0, "start rspq dump entry"); 3594175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_count", 3595175209Skmacy CTLFLAG_RW, &qs->rspq.rspq_dump_count, 3596175209Skmacy 0, "#rspq entries to dump"); 3597175209Skmacy SYSCTL_ADD_PROC(ctx, rspqpoidlist, OID_AUTO, "qdump", 3598175209Skmacy CTLTYPE_STRING | CTLFLAG_RD, &qs->rspq, 3599175209Skmacy 0, t3_dump_rspq, "A", "dump of the response queue"); 3600175209Skmacy 3601205948Snp SYSCTL_ADD_QUAD(ctx, txqpoidlist, OID_AUTO, "dropped", 3602205948Snp CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_mr->br_drops, 3603205948Snp "#tunneled packets dropped"); 3604175209Skmacy SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "sendqlen", 3605174708Skmacy CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.qlen, 3606174708Skmacy 0, "#tunneled packets waiting to be sent"); 3607185162Skmacy#if 0 3608175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx", 3609174708Skmacy CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, 3610174708Skmacy 0, "#tunneled packets queue producer index"); 3611175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx", 3612174708Skmacy CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, 3613174708Skmacy 0, "#tunneled packets queue consumer index"); 3614185162Skmacy#endif 3615175209Skmacy SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "processed", 3616174708Skmacy CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, 3617174708Skmacy 0, "#tunneled packets processed by the card"); 3618175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "cleaned", 3619174708Skmacy CTLFLAG_RD, &txq->cleaned, 3620174708Skmacy 0, "#tunneled packets cleaned"); 3621175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "in_use", 3622174708Skmacy CTLFLAG_RD, &txq->in_use, 3623174708Skmacy 0, "#tunneled packet slots in use"); 3624175209Skmacy SYSCTL_ADD_ULONG(ctx, txqpoidlist, OID_AUTO, "frees", 3625174708Skmacy CTLFLAG_RD, &txq->txq_frees, 3626174708Skmacy "#tunneled packets freed"); 3627175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "skipped", 3628174708Skmacy CTLFLAG_RD, &txq->txq_skipped, 3629174708Skmacy 0, "#tunneled packet descriptors skipped"); 3630194521Skmacy SYSCTL_ADD_QUAD(ctx, txqpoidlist, OID_AUTO, "coalesced", 3631174708Skmacy CTLFLAG_RD, &txq->txq_coalesced, 3632194521Skmacy "#tunneled packets coalesced"); 3633175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "enqueued", 3634174708Skmacy CTLFLAG_RD, &txq->txq_enqueued, 3635174708Skmacy 0, "#tunneled packets enqueued to hardware"); 3636175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "stopped_flags", 3637174708Skmacy CTLFLAG_RD, &qs->txq_stopped, 3638174708Skmacy 0, "tx queues stopped"); 3639175209Skmacy SYSCTL_ADD_XLONG(ctx, txqpoidlist, OID_AUTO, "phys_addr", 3640175200Skmacy CTLFLAG_RD, &txq->phys_addr, 3641175200Skmacy "physical_address_of the queue"); 3642175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "qgen", 3643175200Skmacy CTLFLAG_RW, &qs->txq[TXQ_ETH].gen, 3644175200Skmacy 0, "txq generation"); 3645175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_cidx", 3646175200Skmacy CTLFLAG_RD, &txq->cidx, 3647175200Skmacy 0, "hardware queue cidx"); 3648175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_pidx", 3649175200Skmacy CTLFLAG_RD, &txq->pidx, 3650175200Skmacy 0, "hardware queue pidx"); 3651175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_start", 3652175200Skmacy CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start, 3653175200Skmacy 0, "txq start idx for dump"); 3654175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_count", 3655175200Skmacy CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count, 3656175200Skmacy 0, "txq #entries to dump"); 3657175209Skmacy SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump", 3658175200Skmacy CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH], 3659176472Skmacy 0, t3_dump_txq_eth, "A", "dump of the transmit queue"); 3660176472Skmacy 3661176472Skmacy SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start", 3662176472Skmacy CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start, 3663176472Skmacy 0, "ctrlq start idx for dump"); 3664176472Skmacy SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count", 3665176472Skmacy CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count, 3666176472Skmacy 0, "ctrl #entries to dump"); 3667176472Skmacy SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump", 3668176472Skmacy CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL], 3669176472Skmacy 0, t3_dump_txq_ctrl, "A", "dump of the transmit queue"); 3670176472Skmacy 3671181652Skmacy SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_queued", 3672181652Skmacy CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL); 3673181652Skmacy SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_flushed", 3674181652Skmacy CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL); 3675181652Skmacy SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_bad_csum", 3676181652Skmacy CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL); 3677181652Skmacy SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_cnt", 3678181652Skmacy CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL); 3679181652Skmacy } 3680176472Skmacy 3681181652Skmacy /* Now add a node for mac stats. */ 3682181652Skmacy poid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, "mac_stats", 3683181652Skmacy CTLFLAG_RD, NULL, "MAC statistics"); 3684181652Skmacy poidlist = SYSCTL_CHILDREN(poid); 3685176472Skmacy 3686181652Skmacy /* 3687181652Skmacy * We (ab)use the length argument (arg2) to pass on the offset 3688181652Skmacy * of the data that we are interested in. This is only required 3689181652Skmacy * for the quad counters that are updated from the hardware (we 3690181652Skmacy * make sure that we return the latest value). 3691181652Skmacy * sysctl_handle_macstat first updates *all* the counters from 3692181652Skmacy * the hardware, and then returns the latest value of the 3693181652Skmacy * requested counter. Best would be to update only the 3694181652Skmacy * requested counter from hardware, but t3_mac_update_stats() 3695181652Skmacy * hides all the register details and we don't want to dive into 3696181652Skmacy * all that here. 3697181652Skmacy */ 3698181652Skmacy#define CXGB_SYSCTL_ADD_QUAD(a) SYSCTL_ADD_OID(ctx, poidlist, OID_AUTO, #a, \ 3699181652Skmacy (CTLTYPE_QUAD | CTLFLAG_RD), pi, offsetof(struct mac_stats, a), \ 3700181652Skmacy sysctl_handle_macstat, "QU", 0) 3701181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_octets); 3702181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_octets_bad); 3703181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames); 3704181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_mcast_frames); 3705181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_bcast_frames); 3706181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_pause); 3707181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_deferred); 3708181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_late_collisions); 3709181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_total_collisions); 3710181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_excess_collisions); 3711181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_underrun); 3712181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_len_errs); 3713181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_mac_internal_errs); 3714181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_excess_deferral); 3715181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_fcs_errs); 3716181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_64); 3717181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_65_127); 3718181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_128_255); 3719181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_256_511); 3720181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_512_1023); 3721181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_1024_1518); 3722181652Skmacy CXGB_SYSCTL_ADD_QUAD(tx_frames_1519_max); 3723181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_octets); 3724181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_octets_bad); 3725181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames); 3726181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_mcast_frames); 3727181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_bcast_frames); 3728181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_pause); 3729193925Sgnn CXGB_SYSCTL_ADD_QUAD(rx_fcs_errs); 3730181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_align_errs); 3731181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_symbol_errs); 3732181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_data_errs); 3733181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_sequence_errs); 3734181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_runt); 3735181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_jabber); 3736181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_short); 3737181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_too_long); 3738181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_mac_internal_errs); 3739181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_cong_drops); 3740181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_64); 3741181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_65_127); 3742181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_128_255); 3743181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_256_511); 3744181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_512_1023); 3745181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_1024_1518); 3746181652Skmacy CXGB_SYSCTL_ADD_QUAD(rx_frames_1519_max); 3747181652Skmacy#undef CXGB_SYSCTL_ADD_QUAD 3748181652Skmacy 3749181652Skmacy#define CXGB_SYSCTL_ADD_ULONG(a) SYSCTL_ADD_ULONG(ctx, poidlist, OID_AUTO, #a, \ 3750181652Skmacy CTLFLAG_RD, &mstats->a, 0) 3751181652Skmacy CXGB_SYSCTL_ADD_ULONG(tx_fifo_parity_err); 3752181652Skmacy CXGB_SYSCTL_ADD_ULONG(rx_fifo_parity_err); 3753181652Skmacy CXGB_SYSCTL_ADD_ULONG(tx_fifo_urun); 3754181652Skmacy CXGB_SYSCTL_ADD_ULONG(rx_fifo_ovfl); 3755181652Skmacy CXGB_SYSCTL_ADD_ULONG(serdes_signal_loss); 3756181652Skmacy CXGB_SYSCTL_ADD_ULONG(xaui_pcs_ctc_err); 3757181652Skmacy CXGB_SYSCTL_ADD_ULONG(xaui_pcs_align_change); 3758181652Skmacy CXGB_SYSCTL_ADD_ULONG(num_toggled); 3759181652Skmacy CXGB_SYSCTL_ADD_ULONG(num_resets); 3760192540Sgnn CXGB_SYSCTL_ADD_ULONG(link_faults); 3761181652Skmacy#undef CXGB_SYSCTL_ADD_ULONG 3762174708Skmacy } 3763174708Skmacy} 3764174708Skmacy 3765167514Skmacy/** 3766167514Skmacy * t3_get_desc - dump an SGE descriptor for debugging purposes 3767167514Skmacy * @qs: the queue set 3768167514Skmacy * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) 3769167514Skmacy * @idx: the descriptor index in the queue 3770167514Skmacy * @data: where to dump the descriptor contents 3771167514Skmacy * 3772167514Skmacy * Dumps the contents of a HW descriptor of an SGE queue. Returns the 3773167514Skmacy * size of the descriptor. 3774167514Skmacy */ 3775167514Skmacyint 3776167514Skmacyt3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, 3777167514Skmacy unsigned char *data) 3778167514Skmacy{ 3779167514Skmacy if (qnum >= 6) 3780167514Skmacy return (EINVAL); 3781167514Skmacy 3782167514Skmacy if (qnum < 3) { 3783167514Skmacy if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) 3784167514Skmacy return -EINVAL; 3785167514Skmacy memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); 3786167514Skmacy return sizeof(struct tx_desc); 3787167514Skmacy } 3788167514Skmacy 3789167514Skmacy if (qnum == 3) { 3790167514Skmacy if (!qs->rspq.desc || idx >= qs->rspq.size) 3791167514Skmacy return (EINVAL); 3792167514Skmacy memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); 3793167514Skmacy return sizeof(struct rsp_desc); 3794167514Skmacy } 3795167514Skmacy 3796167514Skmacy qnum -= 4; 3797167514Skmacy if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) 3798167514Skmacy return (EINVAL); 3799167514Skmacy memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); 3800167514Skmacy return sizeof(struct rx_desc); 3801167514Skmacy} 3802