cxgb_sge.c revision 176472
1167514Skmacy/************************************************************************** 2167514Skmacy 3167514SkmacyCopyright (c) 2007, Chelsio Inc. 4167514SkmacyAll rights reserved. 5167514Skmacy 6167514SkmacyRedistribution and use in source and binary forms, with or without 7167514Skmacymodification, are permitted provided that the following conditions are met: 8167514Skmacy 9167514Skmacy 1. Redistributions of source code must retain the above copyright notice, 10167514Skmacy this list of conditions and the following disclaimer. 11167514Skmacy 12169978Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13167514Skmacy contributors may be used to endorse or promote products derived from 14167514Skmacy this software without specific prior written permission. 15169978Skmacy 16167514SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17167514SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18167514SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19167514SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20167514SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21167514SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22167514SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23167514SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24167514SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25167514SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26167514SkmacyPOSSIBILITY OF SUCH DAMAGE. 27167514Skmacy 28167514Skmacy***************************************************************************/ 29175340Skmacy#define DEBUG_BUFRING 30167514Skmacy 31175340Skmacy 32167514Skmacy#include <sys/cdefs.h> 33167514Skmacy__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_sge.c 176472 2008-02-23 01:06:17Z kmacy $"); 34167514Skmacy 35167514Skmacy#include <sys/param.h> 36167514Skmacy#include <sys/systm.h> 37167514Skmacy#include <sys/kernel.h> 38167514Skmacy#include <sys/module.h> 39167514Skmacy#include <sys/bus.h> 40167514Skmacy#include <sys/conf.h> 41167514Skmacy#include <machine/bus.h> 42167514Skmacy#include <machine/resource.h> 43167514Skmacy#include <sys/bus_dma.h> 44167514Skmacy#include <sys/rman.h> 45167514Skmacy#include <sys/queue.h> 46167514Skmacy#include <sys/sysctl.h> 47167514Skmacy#include <sys/taskqueue.h> 48167514Skmacy 49167514Skmacy#include <sys/proc.h> 50175200Skmacy#include <sys/sbuf.h> 51167514Skmacy#include <sys/sched.h> 52167514Skmacy#include <sys/smp.h> 53167760Skmacy#include <sys/systm.h> 54174708Skmacy#include <sys/syslog.h> 55167514Skmacy 56167514Skmacy#include <netinet/in_systm.h> 57167514Skmacy#include <netinet/in.h> 58167514Skmacy#include <netinet/ip.h> 59167514Skmacy#include <netinet/tcp.h> 60167514Skmacy 61167514Skmacy#include <dev/pci/pcireg.h> 62167514Skmacy#include <dev/pci/pcivar.h> 63167514Skmacy 64174670Skmacy#include <vm/vm.h> 65174708Skmacy#include <vm/pmap.h> 66174672Skmacy 67170076Skmacy#ifdef CONFIG_DEFINED 68170076Skmacy#include <cxgb_include.h> 69174670Skmacy#include <sys/mvec.h> 70170076Skmacy#else 71170076Skmacy#include <dev/cxgb/cxgb_include.h> 72174670Skmacy#include <dev/cxgb/sys/mvec.h> 73170076Skmacy#endif 74168491Skmacy 75171335Skmacyint txq_fills = 0; 76176472Skmacy/* 77176472Skmacy * XXX don't re-enable this until TOE stops assuming 78176472Skmacy * we have an m_ext 79176472Skmacy */ 80176472Skmacystatic int recycle_enable = 0; 81174708Skmacyextern int cxgb_txq_buf_ring_size; 82174708Skmacyint cxgb_cached_allocations; 83174708Skmacyint cxgb_cached; 84176472Skmacyint cxgb_ext_freed = 0; 85176472Skmacyint cxgb_ext_inited = 0; 86175200Skmacyextern int cxgb_use_16k_clusters; 87175415Skmacyextern int cxgb_pcpu_cache_enable; 88168737Skmacy 89175200Skmacy 90167514Skmacy#define USE_GTS 0 91167514Skmacy 92167514Skmacy#define SGE_RX_SM_BUF_SIZE 1536 93167514Skmacy#define SGE_RX_DROP_THRES 16 94169978Skmacy#define SGE_RX_COPY_THRES 128 95167514Skmacy 96167514Skmacy/* 97167514Skmacy * Period of the Tx buffer reclaim timer. This timer does not need to run 98167514Skmacy * frequently as Tx buffers are usually reclaimed by new Tx packets. 99167514Skmacy */ 100170038Skmacy#define TX_RECLAIM_PERIOD (hz >> 1) 101167514Skmacy 102167514Skmacy/* 103167514Skmacy * Values for sge_txq.flags 104167514Skmacy */ 105167514Skmacyenum { 106167514Skmacy TXQ_RUNNING = 1 << 0, /* fetch engine is running */ 107167514Skmacy TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */ 108167514Skmacy}; 109167514Skmacy 110167514Skmacystruct tx_desc { 111167514Skmacy uint64_t flit[TX_DESC_FLITS]; 112167514Skmacy} __packed; 113167514Skmacy 114167514Skmacystruct rx_desc { 115167514Skmacy uint32_t addr_lo; 116167514Skmacy uint32_t len_gen; 117167514Skmacy uint32_t gen2; 118167514Skmacy uint32_t addr_hi; 119167514Skmacy} __packed;; 120167514Skmacy 121167514Skmacystruct rsp_desc { /* response queue descriptor */ 122167514Skmacy struct rss_header rss_hdr; 123167514Skmacy uint32_t flags; 124167514Skmacy uint32_t len_cq; 125167514Skmacy uint8_t imm_data[47]; 126167514Skmacy uint8_t intr_gen; 127167514Skmacy} __packed; 128167514Skmacy 129167514Skmacy#define RX_SW_DESC_MAP_CREATED (1 << 0) 130168737Skmacy#define TX_SW_DESC_MAP_CREATED (1 << 1) 131167514Skmacy#define RX_SW_DESC_INUSE (1 << 3) 132167514Skmacy#define TX_SW_DESC_MAPPED (1 << 4) 133167514Skmacy 134167514Skmacy#define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0) 135167514Skmacy#define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP) 136167514Skmacy#define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP) 137167514Skmacy#define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP) 138167514Skmacy 139167514Skmacystruct tx_sw_desc { /* SW state per Tx descriptor */ 140174708Skmacy struct mbuf_iovec mi; 141167514Skmacy bus_dmamap_t map; 142167514Skmacy int flags; 143167514Skmacy}; 144167514Skmacy 145167514Skmacystruct rx_sw_desc { /* SW state per Rx descriptor */ 146174708Skmacy caddr_t rxsd_cl; 147174708Skmacy caddr_t data; 148174708Skmacy bus_dmamap_t map; 149174708Skmacy int flags; 150167514Skmacy}; 151167514Skmacy 152167514Skmacystruct txq_state { 153167514Skmacy unsigned int compl; 154167514Skmacy unsigned int gen; 155167514Skmacy unsigned int pidx; 156167514Skmacy}; 157167514Skmacy 158168351Skmacystruct refill_fl_cb_arg { 159168351Skmacy int error; 160168351Skmacy bus_dma_segment_t seg; 161168351Skmacy int nseg; 162168351Skmacy}; 163168351Skmacy 164167514Skmacy/* 165167514Skmacy * Maps a number of flits to the number of Tx descriptors that can hold them. 166167514Skmacy * The formula is 167167514Skmacy * 168167514Skmacy * desc = 1 + (flits - 2) / (WR_FLITS - 1). 169167514Skmacy * 170167514Skmacy * HW allows up to 4 descriptors to be combined into a WR. 171167514Skmacy */ 172167514Skmacystatic uint8_t flit_desc_map[] = { 173167514Skmacy 0, 174167514Skmacy#if SGE_NUM_GENBITS == 1 175167514Skmacy 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 176167514Skmacy 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 177167514Skmacy 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 178167514Skmacy 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 179167514Skmacy#elif SGE_NUM_GENBITS == 2 180167514Skmacy 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 181167514Skmacy 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 182167514Skmacy 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 183167514Skmacy 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 184167514Skmacy#else 185167514Skmacy# error "SGE_NUM_GENBITS must be 1 or 2" 186167514Skmacy#endif 187167514Skmacy}; 188167514Skmacy 189167514Skmacy 190167514Skmacystatic int lro_default = 0; 191167514Skmacyint cxgb_debug = 0; 192167514Skmacy 193167514Skmacystatic void sge_timer_cb(void *arg); 194167514Skmacystatic void sge_timer_reclaim(void *arg, int ncount); 195171335Skmacystatic void sge_txq_reclaim_handler(void *arg, int ncount); 196167514Skmacy 197167514Skmacy/** 198167514Skmacy * reclaim_completed_tx - reclaims completed Tx descriptors 199167514Skmacy * @adapter: the adapter 200167514Skmacy * @q: the Tx queue to reclaim completed descriptors from 201167514Skmacy * 202167514Skmacy * Reclaims Tx descriptors that the SGE has indicated it has processed, 203167514Skmacy * and frees the associated buffers if possible. Called with the Tx 204167514Skmacy * queue's lock held. 205167514Skmacy */ 206167514Skmacystatic __inline int 207175224Skmacyreclaim_completed_tx_(struct sge_txq *q, int reclaim_min) 208167514Skmacy{ 209174708Skmacy int reclaim = desc_reclaimable(q); 210167514Skmacy 211175224Skmacy if (reclaim < reclaim_min) 212175224Skmacy return (0); 213175224Skmacy 214167514Skmacy mtx_assert(&q->lock, MA_OWNED); 215167514Skmacy if (reclaim > 0) { 216174708Skmacy t3_free_tx_desc(q, reclaim); 217174708Skmacy q->cleaned += reclaim; 218174708Skmacy q->in_use -= reclaim; 219167514Skmacy } 220174708Skmacy return (reclaim); 221167514Skmacy} 222167514Skmacy 223167514Skmacy/** 224169978Skmacy * should_restart_tx - are there enough resources to restart a Tx queue? 225169978Skmacy * @q: the Tx queue 226169978Skmacy * 227169978Skmacy * Checks if there are enough descriptors to restart a suspended Tx queue. 228169978Skmacy */ 229169978Skmacystatic __inline int 230169978Skmacyshould_restart_tx(const struct sge_txq *q) 231169978Skmacy{ 232169978Skmacy unsigned int r = q->processed - q->cleaned; 233169978Skmacy 234169978Skmacy return q->in_use - r < (q->size >> 1); 235169978Skmacy} 236169978Skmacy 237169978Skmacy/** 238167514Skmacy * t3_sge_init - initialize SGE 239167514Skmacy * @adap: the adapter 240167514Skmacy * @p: the SGE parameters 241167514Skmacy * 242167514Skmacy * Performs SGE initialization needed every time after a chip reset. 243167514Skmacy * We do not initialize any of the queue sets here, instead the driver 244167514Skmacy * top-level must request those individually. We also do not enable DMA 245167514Skmacy * here, that should be done after the queues have been set up. 246167514Skmacy */ 247167514Skmacyvoid 248167514Skmacyt3_sge_init(adapter_t *adap, struct sge_params *p) 249167514Skmacy{ 250167514Skmacy u_int ctrl, ups; 251167514Skmacy 252167514Skmacy ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ 253167514Skmacy 254167514Skmacy ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | 255176472Skmacy F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN | 256167514Skmacy V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | 257167514Skmacy V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; 258167514Skmacy#if SGE_NUM_GENBITS == 1 259167514Skmacy ctrl |= F_EGRGENCTRL; 260167514Skmacy#endif 261167514Skmacy if (adap->params.rev > 0) { 262167514Skmacy if (!(adap->flags & (USING_MSIX | USING_MSI))) 263167514Skmacy ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ; 264167514Skmacy } 265167514Skmacy t3_write_reg(adap, A_SG_CONTROL, ctrl); 266167514Skmacy t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) | 267167514Skmacy V_LORCQDRBTHRSH(512)); 268167514Skmacy t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10); 269167514Skmacy t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) | 270167514Skmacy V_TIMEOUT(200 * core_ticks_per_usec(adap))); 271176472Skmacy t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 272176472Skmacy adap->params.rev < T3_REV_C ? 1000 : 500); 273167514Skmacy t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256); 274167514Skmacy t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000); 275167514Skmacy t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256); 276167514Skmacy t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff)); 277167514Skmacy t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024); 278167514Skmacy} 279167514Skmacy 280167514Skmacy 281167514Skmacy/** 282167514Skmacy * sgl_len - calculates the size of an SGL of the given capacity 283167514Skmacy * @n: the number of SGL entries 284167514Skmacy * 285167514Skmacy * Calculates the number of flits needed for a scatter/gather list that 286167514Skmacy * can hold the given number of entries. 287167514Skmacy */ 288167514Skmacystatic __inline unsigned int 289167514Skmacysgl_len(unsigned int n) 290167514Skmacy{ 291167514Skmacy return ((3 * n) / 2 + (n & 1)); 292167514Skmacy} 293167514Skmacy 294167514Skmacy/** 295167514Skmacy * get_imm_packet - return the next ingress packet buffer from a response 296167514Skmacy * @resp: the response descriptor containing the packet data 297167514Skmacy * 298167514Skmacy * Return a packet containing the immediate data of the given response. 299167514Skmacy */ 300171471Skmacystatic int 301176472Skmacyget_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m) 302167514Skmacy{ 303174708Skmacy 304176472Skmacy m->m_len = m->m_pkthdr.len = IMMED_PKT_SIZE; 305176472Skmacy m->m_ext.ext_buf = NULL; 306176472Skmacy m->m_ext.ext_type = 0; 307174708Skmacy memcpy(mtod(m, uint8_t *), resp->imm_data, IMMED_PKT_SIZE); 308176472Skmacy return (0); 309167514Skmacy} 310167514Skmacy 311167514Skmacystatic __inline u_int 312167514Skmacyflits_to_desc(u_int n) 313167514Skmacy{ 314167514Skmacy return (flit_desc_map[n]); 315167514Skmacy} 316167514Skmacy 317176472Skmacy#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \ 318176472Skmacy F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \ 319176472Skmacy V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \ 320176472Skmacy F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \ 321176472Skmacy F_HIRCQPARITYERROR) 322176472Skmacy#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR) 323176472Skmacy#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \ 324176472Skmacy F_RSPQDISABLED) 325176472Skmacy 326176472Skmacy/** 327176472Skmacy * t3_sge_err_intr_handler - SGE async event interrupt handler 328176472Skmacy * @adapter: the adapter 329176472Skmacy * 330176472Skmacy * Interrupt handler for SGE asynchronous (non-data) events. 331176472Skmacy */ 332167514Skmacyvoid 333167514Skmacyt3_sge_err_intr_handler(adapter_t *adapter) 334167514Skmacy{ 335167514Skmacy unsigned int v, status; 336167514Skmacy 337167514Skmacy status = t3_read_reg(adapter, A_SG_INT_CAUSE); 338176472Skmacy if (status & SGE_PARERR) 339176472Skmacy CH_ALERT(adapter, "SGE parity error (0x%x)\n", 340176472Skmacy status & SGE_PARERR); 341176472Skmacy if (status & SGE_FRAMINGERR) 342176472Skmacy CH_ALERT(adapter, "SGE framing error (0x%x)\n", 343176472Skmacy status & SGE_FRAMINGERR); 344167514Skmacy if (status & F_RSPQCREDITOVERFOW) 345167514Skmacy CH_ALERT(adapter, "SGE response queue credit overflow\n"); 346167514Skmacy 347167514Skmacy if (status & F_RSPQDISABLED) { 348167514Skmacy v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS); 349167514Skmacy 350167514Skmacy CH_ALERT(adapter, 351167514Skmacy "packet delivered to disabled response queue (0x%x)\n", 352167514Skmacy (v >> S_RSPQ0DISABLED) & 0xff); 353167514Skmacy } 354167514Skmacy 355167514Skmacy t3_write_reg(adapter, A_SG_INT_CAUSE, status); 356176472Skmacy if (status & SGE_FATALERR) 357167514Skmacy t3_fatal_err(adapter); 358167514Skmacy} 359167514Skmacy 360167514Skmacyvoid 361167514Skmacyt3_sge_prep(adapter_t *adap, struct sge_params *p) 362167514Skmacy{ 363167514Skmacy int i; 364167514Skmacy 365167514Skmacy /* XXX Does ETHER_ALIGN need to be accounted for here? */ 366175200Skmacy p->max_pkt_size = adap->sge.qs[0].fl[1].buf_size - sizeof(struct cpl_rx_data); 367167514Skmacy 368167514Skmacy for (i = 0; i < SGE_QSETS; ++i) { 369167514Skmacy struct qset_params *q = p->qset + i; 370167514Skmacy 371174708Skmacy if (adap->params.nports > 2) { 372171471Skmacy q->coalesce_nsecs = 50000; 373174708Skmacy } else { 374174708Skmacy#ifdef INVARIANTS 375175200Skmacy q->coalesce_nsecs = 10000; 376174708Skmacy#else 377171471Skmacy q->coalesce_nsecs = 5000; 378174708Skmacy#endif 379174708Skmacy } 380176472Skmacy q->polling = adap->params.rev > 0; 381167514Skmacy q->rspq_size = RSPQ_Q_SIZE; 382167514Skmacy q->fl_size = FL_Q_SIZE; 383167514Skmacy q->jumbo_size = JUMBO_Q_SIZE; 384167514Skmacy q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; 385167514Skmacy q->txq_size[TXQ_OFLD] = 1024; 386167514Skmacy q->txq_size[TXQ_CTRL] = 256; 387167514Skmacy q->cong_thres = 0; 388167514Skmacy } 389167514Skmacy} 390167514Skmacy 391167514Skmacyint 392167514Skmacyt3_sge_alloc(adapter_t *sc) 393167514Skmacy{ 394167514Skmacy 395167514Skmacy /* The parent tag. */ 396167514Skmacy if (bus_dma_tag_create( NULL, /* parent */ 397167514Skmacy 1, 0, /* algnmnt, boundary */ 398167514Skmacy BUS_SPACE_MAXADDR, /* lowaddr */ 399167514Skmacy BUS_SPACE_MAXADDR, /* highaddr */ 400167514Skmacy NULL, NULL, /* filter, filterarg */ 401167514Skmacy BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 402167514Skmacy BUS_SPACE_UNRESTRICTED, /* nsegments */ 403167514Skmacy BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 404167514Skmacy 0, /* flags */ 405167514Skmacy NULL, NULL, /* lock, lockarg */ 406167514Skmacy &sc->parent_dmat)) { 407167514Skmacy device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); 408167514Skmacy return (ENOMEM); 409167514Skmacy } 410167514Skmacy 411167514Skmacy /* 412167514Skmacy * DMA tag for normal sized RX frames 413167514Skmacy */ 414167514Skmacy if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, 415167514Skmacy BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, 416167514Skmacy MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { 417167514Skmacy device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); 418167514Skmacy return (ENOMEM); 419167514Skmacy } 420167514Skmacy 421167514Skmacy /* 422167514Skmacy * DMA tag for jumbo sized RX frames. 423167514Skmacy */ 424175200Skmacy if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR, 425175200Skmacy BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES, 426167514Skmacy BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { 427167514Skmacy device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); 428167514Skmacy return (ENOMEM); 429167514Skmacy } 430167514Skmacy 431167514Skmacy /* 432167514Skmacy * DMA tag for TX frames. 433167514Skmacy */ 434167514Skmacy if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, 435167514Skmacy BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, 436167514Skmacy TX_MAX_SIZE, BUS_DMA_ALLOCNOW, 437167514Skmacy NULL, NULL, &sc->tx_dmat)) { 438167514Skmacy device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); 439167514Skmacy return (ENOMEM); 440167514Skmacy } 441167514Skmacy 442167514Skmacy return (0); 443167514Skmacy} 444167514Skmacy 445167514Skmacyint 446167514Skmacyt3_sge_free(struct adapter * sc) 447167514Skmacy{ 448167514Skmacy 449167514Skmacy if (sc->tx_dmat != NULL) 450167514Skmacy bus_dma_tag_destroy(sc->tx_dmat); 451167514Skmacy 452167514Skmacy if (sc->rx_jumbo_dmat != NULL) 453167514Skmacy bus_dma_tag_destroy(sc->rx_jumbo_dmat); 454167514Skmacy 455167514Skmacy if (sc->rx_dmat != NULL) 456167514Skmacy bus_dma_tag_destroy(sc->rx_dmat); 457167514Skmacy 458167514Skmacy if (sc->parent_dmat != NULL) 459167514Skmacy bus_dma_tag_destroy(sc->parent_dmat); 460167514Skmacy 461167514Skmacy return (0); 462167514Skmacy} 463167514Skmacy 464167514Skmacyvoid 465167514Skmacyt3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) 466167514Skmacy{ 467167514Skmacy 468167514Skmacy qs->rspq.holdoff_tmr = max(p->coalesce_nsecs/100, 1U); 469167514Skmacy qs->rspq.polling = 0 /* p->polling */; 470167514Skmacy} 471167514Skmacy 472174708Skmacy#if !defined(__i386__) && !defined(__amd64__) 473168351Skmacystatic void 474168351Skmacyrefill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 475168351Skmacy{ 476168351Skmacy struct refill_fl_cb_arg *cb_arg = arg; 477168351Skmacy 478168351Skmacy cb_arg->error = error; 479168351Skmacy cb_arg->seg = segs[0]; 480168351Skmacy cb_arg->nseg = nseg; 481167514Skmacy 482168351Skmacy} 483174708Skmacy#endif 484167514Skmacy/** 485167514Skmacy * refill_fl - refill an SGE free-buffer list 486167514Skmacy * @sc: the controller softc 487167514Skmacy * @q: the free-list to refill 488167514Skmacy * @n: the number of new buffers to allocate 489167514Skmacy * 490167514Skmacy * (Re)populate an SGE free-buffer list with up to @n new packet buffers. 491167514Skmacy * The caller must assure that @n does not exceed the queue's capacity. 492167514Skmacy */ 493167514Skmacystatic void 494167514Skmacyrefill_fl(adapter_t *sc, struct sge_fl *q, int n) 495167514Skmacy{ 496167514Skmacy struct rx_sw_desc *sd = &q->sdesc[q->pidx]; 497167514Skmacy struct rx_desc *d = &q->desc[q->pidx]; 498168351Skmacy struct refill_fl_cb_arg cb_arg; 499174708Skmacy caddr_t cl; 500176472Skmacy int err, count = 0; 501175200Skmacy int header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t); 502175200Skmacy 503168351Skmacy cb_arg.error = 0; 504167514Skmacy while (n--) { 505168351Skmacy /* 506168351Skmacy * We only allocate a cluster, mbuf allocation happens after rx 507168351Skmacy */ 508174708Skmacy if ((cl = cxgb_cache_get(q->zone)) == NULL) { 509168351Skmacy log(LOG_WARNING, "Failed to allocate cluster\n"); 510167514Skmacy goto done; 511167514Skmacy } 512174708Skmacy 513167514Skmacy if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { 514168351Skmacy if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { 515167848Skmacy log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); 516168890Skmacy uma_zfree(q->zone, cl); 517167848Skmacy goto done; 518167848Skmacy } 519167514Skmacy sd->flags |= RX_SW_DESC_MAP_CREATED; 520167514Skmacy } 521174708Skmacy#if !defined(__i386__) && !defined(__amd64__) 522174708Skmacy err = bus_dmamap_load(q->entry_tag, sd->map, 523175200Skmacy cl + header_size, q->buf_size, 524168351Skmacy refill_fl_cb, &cb_arg, 0); 525167514Skmacy 526168351Skmacy if (err != 0 || cb_arg.error) { 527168351Skmacy log(LOG_WARNING, "failure in refill_fl %d\n", cb_arg.error); 528168351Skmacy /* 529168351Skmacy * XXX free cluster 530168351Skmacy */ 531167514Skmacy return; 532167514Skmacy } 533174708Skmacy#else 534175200Skmacy cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)(cl + header_size)); 535174708Skmacy#endif 536168351Skmacy sd->flags |= RX_SW_DESC_INUSE; 537174708Skmacy sd->rxsd_cl = cl; 538175200Skmacy sd->data = cl + header_size; 539168351Skmacy d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); 540168351Skmacy d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); 541167514Skmacy d->len_gen = htobe32(V_FLD_GEN1(q->gen)); 542167514Skmacy d->gen2 = htobe32(V_FLD_GEN2(q->gen)); 543167514Skmacy 544167514Skmacy d++; 545167514Skmacy sd++; 546167514Skmacy 547167514Skmacy if (++q->pidx == q->size) { 548167514Skmacy q->pidx = 0; 549167514Skmacy q->gen ^= 1; 550167514Skmacy sd = q->sdesc; 551167514Skmacy d = q->desc; 552167514Skmacy } 553167514Skmacy q->credits++; 554176472Skmacy count++; 555167514Skmacy } 556167514Skmacy 557167514Skmacydone: 558176472Skmacy if (count) 559176472Skmacy t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 560167514Skmacy} 561167514Skmacy 562167514Skmacy 563167514Skmacy/** 564167514Skmacy * free_rx_bufs - free the Rx buffers on an SGE free list 565167514Skmacy * @sc: the controle softc 566167514Skmacy * @q: the SGE free list to clean up 567167514Skmacy * 568167514Skmacy * Release the buffers on an SGE free-buffer Rx queue. HW fetching from 569167514Skmacy * this queue should be stopped before calling this function. 570167514Skmacy */ 571167514Skmacystatic void 572167514Skmacyfree_rx_bufs(adapter_t *sc, struct sge_fl *q) 573167514Skmacy{ 574167514Skmacy u_int cidx = q->cidx; 575167514Skmacy 576167514Skmacy while (q->credits--) { 577167514Skmacy struct rx_sw_desc *d = &q->sdesc[cidx]; 578167514Skmacy 579167514Skmacy if (d->flags & RX_SW_DESC_INUSE) { 580168351Skmacy bus_dmamap_unload(q->entry_tag, d->map); 581168351Skmacy bus_dmamap_destroy(q->entry_tag, d->map); 582174708Skmacy uma_zfree(q->zone, d->rxsd_cl); 583167514Skmacy } 584174708Skmacy d->rxsd_cl = NULL; 585167514Skmacy if (++cidx == q->size) 586167514Skmacy cidx = 0; 587167514Skmacy } 588167514Skmacy} 589167514Skmacy 590167514Skmacystatic __inline void 591167514Skmacy__refill_fl(adapter_t *adap, struct sge_fl *fl) 592167514Skmacy{ 593167514Skmacy refill_fl(adap, fl, min(16U, fl->size - fl->credits)); 594167514Skmacy} 595167514Skmacy 596174708Skmacystatic __inline void 597174708Skmacy__refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max) 598174708Skmacy{ 599174708Skmacy if ((fl->size - fl->credits) < max) 600174708Skmacy refill_fl(adap, fl, min(max, fl->size - fl->credits)); 601174708Skmacy} 602174708Skmacy 603174708Skmacyvoid 604174708Skmacyrefill_fl_service(adapter_t *adap, struct sge_fl *fl) 605174708Skmacy{ 606174708Skmacy __refill_fl_lt(adap, fl, 512); 607174708Skmacy} 608174708Skmacy 609169978Skmacy/** 610169978Skmacy * recycle_rx_buf - recycle a receive buffer 611169978Skmacy * @adapter: the adapter 612169978Skmacy * @q: the SGE free list 613169978Skmacy * @idx: index of buffer to recycle 614169978Skmacy * 615169978Skmacy * Recycles the specified buffer on the given free list by adding it at 616169978Skmacy * the next available slot on the list. 617169978Skmacy */ 618167514Skmacystatic void 619169978Skmacyrecycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx) 620169978Skmacy{ 621169978Skmacy struct rx_desc *from = &q->desc[idx]; 622169978Skmacy struct rx_desc *to = &q->desc[q->pidx]; 623169978Skmacy 624169978Skmacy q->sdesc[q->pidx] = q->sdesc[idx]; 625169978Skmacy to->addr_lo = from->addr_lo; // already big endian 626169978Skmacy to->addr_hi = from->addr_hi; // likewise 627169978Skmacy wmb(); 628169978Skmacy to->len_gen = htobe32(V_FLD_GEN1(q->gen)); 629169978Skmacy to->gen2 = htobe32(V_FLD_GEN2(q->gen)); 630169978Skmacy q->credits++; 631169978Skmacy 632169978Skmacy if (++q->pidx == q->size) { 633169978Skmacy q->pidx = 0; 634169978Skmacy q->gen ^= 1; 635169978Skmacy } 636169978Skmacy t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 637169978Skmacy} 638169978Skmacy 639169978Skmacystatic void 640167514Skmacyalloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 641167514Skmacy{ 642167514Skmacy uint32_t *addr; 643167514Skmacy 644167514Skmacy addr = arg; 645167514Skmacy *addr = segs[0].ds_addr; 646167514Skmacy} 647167514Skmacy 648167514Skmacystatic int 649167514Skmacyalloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, 650168351Skmacy bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag, 651168351Skmacy bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag) 652167514Skmacy{ 653167514Skmacy size_t len = nelem * elem_size; 654167514Skmacy void *s = NULL; 655167514Skmacy void *p = NULL; 656167514Skmacy int err; 657167514Skmacy 658167514Skmacy if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, 659167514Skmacy BUS_SPACE_MAXADDR_32BIT, 660167514Skmacy BUS_SPACE_MAXADDR, NULL, NULL, len, 1, 661167514Skmacy len, 0, NULL, NULL, tag)) != 0) { 662167514Skmacy device_printf(sc->dev, "Cannot allocate descriptor tag\n"); 663167514Skmacy return (ENOMEM); 664167514Skmacy } 665167514Skmacy 666167514Skmacy if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT, 667167514Skmacy map)) != 0) { 668167514Skmacy device_printf(sc->dev, "Cannot allocate descriptor memory\n"); 669167514Skmacy return (ENOMEM); 670167514Skmacy } 671167514Skmacy 672167514Skmacy bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0); 673167514Skmacy bzero(p, len); 674167514Skmacy *(void **)desc = p; 675167514Skmacy 676167514Skmacy if (sw_size) { 677167514Skmacy len = nelem * sw_size; 678175340Skmacy s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); 679167514Skmacy *(void **)sdesc = s; 680167514Skmacy } 681168351Skmacy if (parent_entry_tag == NULL) 682168351Skmacy return (0); 683168351Skmacy 684168646Skmacy if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0, 685168351Skmacy BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 686168646Skmacy NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, 687168646Skmacy TX_MAX_SIZE, BUS_DMA_ALLOCNOW, 688168351Skmacy NULL, NULL, entry_tag)) != 0) { 689168351Skmacy device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); 690168351Skmacy return (ENOMEM); 691168351Skmacy } 692167514Skmacy return (0); 693167514Skmacy} 694167514Skmacy 695167514Skmacystatic void 696167514Skmacysge_slow_intr_handler(void *arg, int ncount) 697167514Skmacy{ 698167514Skmacy adapter_t *sc = arg; 699167760Skmacy 700167514Skmacy t3_slow_intr_handler(sc); 701167514Skmacy} 702167514Skmacy 703171471Skmacy/** 704171471Skmacy * sge_timer_cb - perform periodic maintenance of an SGE qset 705171471Skmacy * @data: the SGE queue set to maintain 706171471Skmacy * 707171471Skmacy * Runs periodically from a timer to perform maintenance of an SGE queue 708171471Skmacy * set. It performs two tasks: 709171471Skmacy * 710171471Skmacy * a) Cleans up any completed Tx descriptors that may still be pending. 711171471Skmacy * Normal descriptor cleanup happens when new packets are added to a Tx 712171471Skmacy * queue so this timer is relatively infrequent and does any cleanup only 713171471Skmacy * if the Tx queue has not seen any new packets in a while. We make a 714171471Skmacy * best effort attempt to reclaim descriptors, in that we don't wait 715171471Skmacy * around if we cannot get a queue's lock (which most likely is because 716171471Skmacy * someone else is queueing new packets and so will also handle the clean 717171471Skmacy * up). Since control queues use immediate data exclusively we don't 718171471Skmacy * bother cleaning them up here. 719171471Skmacy * 720171471Skmacy * b) Replenishes Rx queues that have run out due to memory shortage. 721171471Skmacy * Normally new Rx buffers are added when existing ones are consumed but 722171471Skmacy * when out of memory a queue can become empty. We try to add only a few 723171471Skmacy * buffers here, the queue will be replenished fully as these new buffers 724171471Skmacy * are used up if memory shortage has subsided. 725171471Skmacy * 726171471Skmacy * c) Return coalesced response queue credits in case a response queue is 727171471Skmacy * starved. 728171471Skmacy * 729171471Skmacy * d) Ring doorbells for T304 tunnel queues since we have seen doorbell 730171471Skmacy * fifo overflows and the FW doesn't implement any recovery scheme yet. 731171471Skmacy */ 732167514Skmacystatic void 733167514Skmacysge_timer_cb(void *arg) 734167514Skmacy{ 735167514Skmacy adapter_t *sc = arg; 736174708Skmacy#ifndef IFNET_MULTIQUEUE 737174708Skmacy struct port_info *pi; 738167514Skmacy struct sge_qset *qs; 739167514Skmacy struct sge_txq *txq; 740167514Skmacy int i, j; 741175340Skmacy int reclaim_ofl, refill_rx; 742174708Skmacy 743167514Skmacy for (i = 0; i < sc->params.nports; i++) 744167514Skmacy for (j = 0; j < sc->port[i].nqsets; j++) { 745167514Skmacy qs = &sc->sge.qs[i + j]; 746167514Skmacy txq = &qs->txq[0]; 747167514Skmacy reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; 748167514Skmacy refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || 749167514Skmacy (qs->fl[1].credits < qs->fl[1].size)); 750175340Skmacy if (reclaim_ofl || refill_rx) { 751174708Skmacy pi = &sc->port[i]; 752174708Skmacy taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task); 753170654Skmacy break; 754167514Skmacy } 755167514Skmacy } 756174708Skmacy#endif 757171471Skmacy if (sc->params.nports > 2) { 758171471Skmacy int i; 759171471Skmacy 760171471Skmacy for_each_port(sc, i) { 761171471Skmacy struct port_info *pi = &sc->port[i]; 762171471Skmacy 763171471Skmacy t3_write_reg(sc, A_SG_KDOORBELL, 764171471Skmacy F_SELEGRCNTX | 765171471Skmacy (FW_TUNNEL_SGEEC_START + pi->first_qset)); 766171471Skmacy } 767171471Skmacy } 768170869Skmacy if (sc->open_device_map != 0) 769170869Skmacy callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 770167514Skmacy} 771167514Skmacy 772167514Skmacy/* 773167514Skmacy * This is meant to be a catch-all function to keep sge state private 774167514Skmacy * to sge.c 775167514Skmacy * 776167514Skmacy */ 777167514Skmacyint 778170654Skmacyt3_sge_init_adapter(adapter_t *sc) 779167514Skmacy{ 780167514Skmacy callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE); 781167514Skmacy callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 782167514Skmacy TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); 783174708Skmacy mi_init(); 784174708Skmacy cxgb_cache_init(); 785167514Skmacy return (0); 786167514Skmacy} 787167514Skmacy 788170654Skmacyint 789175224Skmacyt3_sge_reset_adapter(adapter_t *sc) 790175224Skmacy{ 791175224Skmacy callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 792175224Skmacy return (0); 793175224Skmacy} 794175224Skmacy 795175224Skmacyint 796174708Skmacyt3_sge_init_port(struct port_info *pi) 797170654Skmacy{ 798174708Skmacy TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); 799170789Skmacy return (0); 800170654Skmacy} 801170654Skmacy 802167514Skmacyvoid 803167514Skmacyt3_sge_deinit_sw(adapter_t *sc) 804167514Skmacy{ 805174708Skmacy 806174708Skmacy mi_deinit(); 807167514Skmacy} 808167514Skmacy 809167655Skmacy/** 810167655Skmacy * refill_rspq - replenish an SGE response queue 811167655Skmacy * @adapter: the adapter 812167655Skmacy * @q: the response queue to replenish 813167655Skmacy * @credits: how many new responses to make available 814167655Skmacy * 815167655Skmacy * Replenishes a response queue by making the supplied number of responses 816167655Skmacy * available to HW. 817167655Skmacy */ 818167655Skmacystatic __inline void 819167655Skmacyrefill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) 820167655Skmacy{ 821167514Skmacy 822167655Skmacy /* mbufs are allocated on demand when a rspq entry is processed. */ 823167655Skmacy t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN, 824167655Skmacy V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); 825167655Skmacy} 826167655Skmacy 827171335Skmacystatic __inline void 828175224Skmacysge_txq_reclaim_(struct sge_txq *txq, int force) 829171335Skmacy{ 830167655Skmacy 831175340Skmacy if (desc_reclaimable(txq) < 16) 832175224Skmacy return; 833175224Skmacy if (mtx_trylock(&txq->lock) == 0) 834175224Skmacy return; 835175340Skmacy reclaim_completed_tx_(txq, 16); 836175224Skmacy mtx_unlock(&txq->lock); 837175224Skmacy 838171335Skmacy} 839171335Skmacy 840167514Skmacystatic void 841171335Skmacysge_txq_reclaim_handler(void *arg, int ncount) 842171335Skmacy{ 843171335Skmacy struct sge_txq *q = arg; 844171335Skmacy 845175224Skmacy sge_txq_reclaim_(q, TRUE); 846171335Skmacy} 847171335Skmacy 848175224Skmacy 849175224Skmacy 850171335Skmacystatic void 851167514Skmacysge_timer_reclaim(void *arg, int ncount) 852167514Skmacy{ 853174708Skmacy struct port_info *pi = arg; 854174708Skmacy int i, nqsets = pi->nqsets; 855174708Skmacy adapter_t *sc = pi->adapter; 856167514Skmacy struct sge_qset *qs; 857167514Skmacy struct sge_txq *txq; 858167514Skmacy struct mtx *lock; 859167760Skmacy 860174708Skmacy#ifdef IFNET_MULTIQUEUE 861174708Skmacy panic("%s should not be called with multiqueue support\n", __FUNCTION__); 862174708Skmacy#endif 863167514Skmacy for (i = 0; i < nqsets; i++) { 864167514Skmacy qs = &sc->sge.qs[i]; 865169978Skmacy 866167514Skmacy txq = &qs->txq[TXQ_OFLD]; 867175224Skmacy sge_txq_reclaim_(txq, FALSE); 868171335Skmacy 869167514Skmacy lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 870167514Skmacy &sc->sge.qs[0].rspq.lock; 871167514Skmacy 872167514Skmacy if (mtx_trylock(lock)) { 873167514Skmacy /* XXX currently assume that we are *NOT* polling */ 874167514Skmacy uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS); 875167760Skmacy 876167760Skmacy if (qs->fl[0].credits < qs->fl[0].size - 16) 877167514Skmacy __refill_fl(sc, &qs->fl[0]); 878167760Skmacy if (qs->fl[1].credits < qs->fl[1].size - 16) 879167514Skmacy __refill_fl(sc, &qs->fl[1]); 880167514Skmacy 881167514Skmacy if (status & (1 << qs->rspq.cntxt_id)) { 882167514Skmacy if (qs->rspq.credits) { 883167514Skmacy refill_rspq(sc, &qs->rspq, 1); 884167514Skmacy qs->rspq.credits--; 885167514Skmacy t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, 886167514Skmacy 1 << qs->rspq.cntxt_id); 887167514Skmacy } 888167514Skmacy } 889167514Skmacy mtx_unlock(lock); 890167514Skmacy } 891167514Skmacy } 892167514Skmacy} 893167514Skmacy 894167514Skmacy/** 895167514Skmacy * init_qset_cntxt - initialize an SGE queue set context info 896167514Skmacy * @qs: the queue set 897167514Skmacy * @id: the queue set id 898167514Skmacy * 899167514Skmacy * Initializes the TIDs and context ids for the queues of a queue set. 900167514Skmacy */ 901167514Skmacystatic void 902167514Skmacyinit_qset_cntxt(struct sge_qset *qs, u_int id) 903167514Skmacy{ 904167514Skmacy 905167514Skmacy qs->rspq.cntxt_id = id; 906167514Skmacy qs->fl[0].cntxt_id = 2 * id; 907167514Skmacy qs->fl[1].cntxt_id = 2 * id + 1; 908167514Skmacy qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; 909167514Skmacy qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; 910167514Skmacy qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; 911167514Skmacy qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; 912167514Skmacy qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; 913174708Skmacy 914174708Skmacy mbufq_init(&qs->txq[TXQ_ETH].sendq); 915174708Skmacy mbufq_init(&qs->txq[TXQ_OFLD].sendq); 916174708Skmacy mbufq_init(&qs->txq[TXQ_CTRL].sendq); 917167514Skmacy} 918167514Skmacy 919167514Skmacy 920167514Skmacystatic void 921167514Skmacytxq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs) 922167514Skmacy{ 923167514Skmacy txq->in_use += ndesc; 924167514Skmacy /* 925167514Skmacy * XXX we don't handle stopping of queue 926167514Skmacy * presumably start handles this when we bump against the end 927167514Skmacy */ 928167514Skmacy txqs->gen = txq->gen; 929167514Skmacy txq->unacked += ndesc; 930176472Skmacy txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5); 931176472Skmacy txq->unacked &= 31; 932167514Skmacy txqs->pidx = txq->pidx; 933167514Skmacy txq->pidx += ndesc; 934175347Skmacy#ifdef INVARIANTS 935175347Skmacy if (((txqs->pidx > txq->cidx) && 936175347Skmacy (txq->pidx < txqs->pidx) && 937175347Skmacy (txq->pidx >= txq->cidx)) || 938175347Skmacy ((txqs->pidx < txq->cidx) && 939175347Skmacy (txq->pidx >= txq-> cidx)) || 940175347Skmacy ((txqs->pidx < txq->cidx) && 941175347Skmacy (txq->cidx < txqs->pidx))) 942175347Skmacy panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d", 943175347Skmacy txqs->pidx, txq->pidx, txq->cidx); 944175347Skmacy#endif 945167514Skmacy if (txq->pidx >= txq->size) { 946167514Skmacy txq->pidx -= txq->size; 947167514Skmacy txq->gen ^= 1; 948167514Skmacy } 949167514Skmacy 950167514Skmacy} 951167514Skmacy 952167514Skmacy/** 953167514Skmacy * calc_tx_descs - calculate the number of Tx descriptors for a packet 954167514Skmacy * @m: the packet mbufs 955167514Skmacy * @nsegs: the number of segments 956167514Skmacy * 957167514Skmacy * Returns the number of Tx descriptors needed for the given Ethernet 958167514Skmacy * packet. Ethernet packets require addition of WR and CPL headers. 959167514Skmacy */ 960167514Skmacystatic __inline unsigned int 961167514Skmacycalc_tx_descs(const struct mbuf *m, int nsegs) 962167514Skmacy{ 963167514Skmacy unsigned int flits; 964167514Skmacy 965167514Skmacy if (m->m_pkthdr.len <= WR_LEN - sizeof(struct cpl_tx_pkt)) 966167514Skmacy return 1; 967167514Skmacy 968167514Skmacy flits = sgl_len(nsegs) + 2; 969167514Skmacy#ifdef TSO_SUPPORTED 970174708Skmacy if (m->m_pkthdr.csum_flags & CSUM_TSO) 971167514Skmacy flits++; 972167514Skmacy#endif 973167514Skmacy return flits_to_desc(flits); 974167514Skmacy} 975167514Skmacy 976168646Skmacystatic unsigned int 977168646Skmacybusdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, 978174708Skmacy struct tx_sw_desc *txsd, bus_dma_segment_t *segs, int *nsegs) 979167514Skmacy{ 980168646Skmacy struct mbuf *m0; 981174708Skmacy int err, pktlen, pass = 0; 982167514Skmacy 983174708Skmacyretry: 984174708Skmacy err = 0; 985167514Skmacy m0 = *m; 986167514Skmacy pktlen = m0->m_pkthdr.len; 987174708Skmacy#if defined(__i386__) || defined(__amd64__) 988174708Skmacy if (busdma_map_sg_collapse(m, segs, nsegs) == 0) { 989174708Skmacy goto done; 990174708Skmacy } else 991174708Skmacy#endif 992174708Skmacy err = bus_dmamap_load_mbuf_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0); 993168737Skmacy 994174708Skmacy if (err == 0) { 995174708Skmacy goto done; 996168737Skmacy } 997174708Skmacy if (err == EFBIG && pass == 0) { 998174708Skmacy pass = 1; 999167514Skmacy /* Too many segments, try to defrag */ 1000171471Skmacy m0 = m_defrag(m0, M_DONTWAIT); 1001167514Skmacy if (m0 == NULL) { 1002167514Skmacy m_freem(*m); 1003167514Skmacy *m = NULL; 1004167514Skmacy return (ENOBUFS); 1005167514Skmacy } 1006167514Skmacy *m = m0; 1007174708Skmacy goto retry; 1008174708Skmacy } else if (err == ENOMEM) { 1009167514Skmacy return (err); 1010174708Skmacy } if (err) { 1011167514Skmacy if (cxgb_debug) 1012167514Skmacy printf("map failure err=%d pktlen=%d\n", err, pktlen); 1013174639Skmacy m_freem(m0); 1014167514Skmacy *m = NULL; 1015167514Skmacy return (err); 1016167514Skmacy } 1017174708Skmacydone: 1018174708Skmacy#if !defined(__i386__) && !defined(__amd64__) 1019174708Skmacy bus_dmamap_sync(txq->entry_tag, txsd->map, BUS_DMASYNC_PREWRITE); 1020174708Skmacy#endif 1021174708Skmacy txsd->flags |= TX_SW_DESC_MAPPED; 1022167514Skmacy 1023167514Skmacy return (0); 1024167514Skmacy} 1025167514Skmacy 1026167514Skmacy/** 1027167514Skmacy * make_sgl - populate a scatter/gather list for a packet 1028167514Skmacy * @sgp: the SGL to populate 1029167514Skmacy * @segs: the packet dma segments 1030167514Skmacy * @nsegs: the number of segments 1031167514Skmacy * 1032167514Skmacy * Generates a scatter/gather list for the buffers that make up a packet 1033167514Skmacy * and returns the SGL size in 8-byte words. The caller must size the SGL 1034167514Skmacy * appropriately. 1035167514Skmacy */ 1036167514Skmacystatic __inline void 1037167514Skmacymake_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs) 1038167514Skmacy{ 1039167514Skmacy int i, idx; 1040167514Skmacy 1041174708Skmacy for (idx = 0, i = 0; i < nsegs; i++) { 1042174708Skmacy /* 1043174708Skmacy * firmware doesn't like empty segments 1044174708Skmacy */ 1045174708Skmacy if (segs[i].ds_len == 0) 1046174708Skmacy continue; 1047167514Skmacy if (i && idx == 0) 1048167514Skmacy ++sgp; 1049174708Skmacy 1050167514Skmacy sgp->len[idx] = htobe32(segs[i].ds_len); 1051167514Skmacy sgp->addr[idx] = htobe64(segs[i].ds_addr); 1052174708Skmacy idx ^= 1; 1053167514Skmacy } 1054167514Skmacy 1055175200Skmacy if (idx) { 1056167514Skmacy sgp->len[idx] = 0; 1057175200Skmacy sgp->addr[idx] = 0; 1058175200Skmacy } 1059167514Skmacy} 1060167514Skmacy 1061167514Skmacy/** 1062167514Skmacy * check_ring_tx_db - check and potentially ring a Tx queue's doorbell 1063167514Skmacy * @adap: the adapter 1064167514Skmacy * @q: the Tx queue 1065167514Skmacy * 1066167514Skmacy * Ring the doorbel if a Tx queue is asleep. There is a natural race, 1067167514Skmacy * where the HW is going to sleep just after we checked, however, 1068167514Skmacy * then the interrupt handler will detect the outstanding TX packet 1069167514Skmacy * and ring the doorbell for us. 1070167514Skmacy * 1071167514Skmacy * When GTS is disabled we unconditionally ring the doorbell. 1072167514Skmacy */ 1073167514Skmacystatic __inline void 1074167514Skmacycheck_ring_tx_db(adapter_t *adap, struct sge_txq *q) 1075167514Skmacy{ 1076167514Skmacy#if USE_GTS 1077167514Skmacy clear_bit(TXQ_LAST_PKT_DB, &q->flags); 1078167514Skmacy if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { 1079167514Skmacy set_bit(TXQ_LAST_PKT_DB, &q->flags); 1080167514Skmacy#ifdef T3_TRACE 1081167514Skmacy T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", 1082167514Skmacy q->cntxt_id); 1083167514Skmacy#endif 1084167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1085167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1086167514Skmacy } 1087167514Skmacy#else 1088167514Skmacy wmb(); /* write descriptors before telling HW */ 1089167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1090167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1091167514Skmacy#endif 1092167514Skmacy} 1093167514Skmacy 1094167514Skmacystatic __inline void 1095167514Skmacywr_gen2(struct tx_desc *d, unsigned int gen) 1096167514Skmacy{ 1097167514Skmacy#if SGE_NUM_GENBITS == 2 1098167514Skmacy d->flit[TX_DESC_FLITS - 1] = htobe64(gen); 1099167514Skmacy#endif 1100167514Skmacy} 1101167514Skmacy 1102169978Skmacy/** 1103169978Skmacy * write_wr_hdr_sgl - write a WR header and, optionally, SGL 1104169978Skmacy * @ndesc: number of Tx descriptors spanned by the SGL 1105169978Skmacy * @txd: first Tx descriptor to be written 1106169978Skmacy * @txqs: txq state (generation and producer index) 1107169978Skmacy * @txq: the SGE Tx queue 1108169978Skmacy * @sgl: the SGL 1109169978Skmacy * @flits: number of flits to the start of the SGL in the first descriptor 1110169978Skmacy * @sgl_flits: the SGL size in flits 1111169978Skmacy * @wr_hi: top 32 bits of WR header based on WR type (big endian) 1112169978Skmacy * @wr_lo: low 32 bits of WR header based on WR type (big endian) 1113169978Skmacy * 1114169978Skmacy * Write a work request header and an associated SGL. If the SGL is 1115169978Skmacy * small enough to fit into one Tx descriptor it has already been written 1116169978Skmacy * and we just need to write the WR header. Otherwise we distribute the 1117169978Skmacy * SGL across the number of descriptors it spans. 1118169978Skmacy */ 1119169978Skmacystatic void 1120169978Skmacywrite_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, 1121169978Skmacy const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, 1122169978Skmacy unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo) 1123169978Skmacy{ 1124169978Skmacy 1125169978Skmacy struct work_request_hdr *wrp = (struct work_request_hdr *)txd; 1126169978Skmacy struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx]; 1127169978Skmacy 1128169978Skmacy if (__predict_true(ndesc == 1)) { 1129169978Skmacy wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1130169978Skmacy V_WR_SGLSFLT(flits)) | wr_hi; 1131169978Skmacy wmb(); 1132169978Skmacy wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) | 1133169978Skmacy V_WR_GEN(txqs->gen)) | wr_lo; 1134169978Skmacy /* XXX gen? */ 1135169978Skmacy wr_gen2(txd, txqs->gen); 1136174708Skmacy 1137169978Skmacy } else { 1138169978Skmacy unsigned int ogen = txqs->gen; 1139169978Skmacy const uint64_t *fp = (const uint64_t *)sgl; 1140169978Skmacy struct work_request_hdr *wp = wrp; 1141169978Skmacy 1142169978Skmacy wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | 1143169978Skmacy V_WR_SGLSFLT(flits)) | wr_hi; 1144169978Skmacy 1145169978Skmacy while (sgl_flits) { 1146169978Skmacy unsigned int avail = WR_FLITS - flits; 1147169978Skmacy 1148169978Skmacy if (avail > sgl_flits) 1149169978Skmacy avail = sgl_flits; 1150169978Skmacy memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); 1151169978Skmacy sgl_flits -= avail; 1152169978Skmacy ndesc--; 1153169978Skmacy if (!sgl_flits) 1154169978Skmacy break; 1155169978Skmacy 1156169978Skmacy fp += avail; 1157169978Skmacy txd++; 1158169978Skmacy txsd++; 1159169978Skmacy if (++txqs->pidx == txq->size) { 1160169978Skmacy txqs->pidx = 0; 1161169978Skmacy txqs->gen ^= 1; 1162169978Skmacy txd = txq->desc; 1163169978Skmacy txsd = txq->sdesc; 1164169978Skmacy } 1165169978Skmacy 1166169978Skmacy /* 1167169978Skmacy * when the head of the mbuf chain 1168169978Skmacy * is freed all clusters will be freed 1169169978Skmacy * with it 1170169978Skmacy */ 1171175340Skmacy KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p", txsd->mi.mi_base)); 1172169978Skmacy wrp = (struct work_request_hdr *)txd; 1173169978Skmacy wrp->wr_hi = htonl(V_WR_DATATYPE(1) | 1174169978Skmacy V_WR_SGLSFLT(1)) | wr_hi; 1175169978Skmacy wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS, 1176169978Skmacy sgl_flits + 1)) | 1177169978Skmacy V_WR_GEN(txqs->gen)) | wr_lo; 1178169978Skmacy wr_gen2(txd, txqs->gen); 1179169978Skmacy flits = 1; 1180169978Skmacy } 1181169978Skmacy wrp->wr_hi |= htonl(F_WR_EOP); 1182169978Skmacy wmb(); 1183169978Skmacy wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; 1184169978Skmacy wr_gen2((struct tx_desc *)wp, ogen); 1185169978Skmacy } 1186169978Skmacy} 1187169978Skmacy 1188167514Skmacy/* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ 1189167514Skmacy#define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) 1190167514Skmacy 1191174708Skmacy#ifdef VLAN_SUPPORTED 1192174708Skmacy#define GET_VTAG(cntrl, m) \ 1193174708Skmacydo { \ 1194174708Skmacy if ((m)->m_flags & M_VLANTAG) \ 1195174708Skmacy cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \ 1196174708Skmacy} while (0) 1197174708Skmacy 1198174708Skmacy#define GET_VTAG_MI(cntrl, mi) \ 1199174708Skmacydo { \ 1200174708Skmacy if ((mi)->mi_flags & M_VLANTAG) \ 1201174708Skmacy cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((mi)->mi_ether_vtag); \ 1202174708Skmacy} while (0) 1203174708Skmacy#else 1204174708Skmacy#define GET_VTAG(cntrl, m) 1205174708Skmacy#define GET_VTAG_MI(cntrl, m) 1206174708Skmacy#endif 1207174708Skmacy 1208167514Skmacyint 1209174708Skmacyt3_encap(struct sge_qset *qs, struct mbuf **m, int count) 1210167514Skmacy{ 1211167514Skmacy adapter_t *sc; 1212167514Skmacy struct mbuf *m0; 1213167514Skmacy struct sge_txq *txq; 1214167514Skmacy struct txq_state txqs; 1215174708Skmacy struct port_info *pi; 1216171868Skmacy unsigned int ndesc, flits, cntrl, mlen; 1217172096Skmacy int err, nsegs, tso_info = 0; 1218167514Skmacy 1219167514Skmacy struct work_request_hdr *wrp; 1220167514Skmacy struct tx_sw_desc *txsd; 1221174708Skmacy struct sg_ent *sgp, *sgl; 1222167514Skmacy uint32_t wr_hi, wr_lo, sgl_flits; 1223175347Skmacy bus_dma_segment_t segs[TX_MAX_SEGS]; 1224167514Skmacy 1225167514Skmacy struct tx_desc *txd; 1226174708Skmacy struct mbuf_vec *mv; 1227174708Skmacy struct mbuf_iovec *mi; 1228174708Skmacy 1229174708Skmacy DPRINTF("t3_encap cpu=%d ", curcpu); 1230171978Skmacy 1231175340Skmacy mi = NULL; 1232174708Skmacy pi = qs->port; 1233174708Skmacy sc = pi->adapter; 1234167514Skmacy txq = &qs->txq[TXQ_ETH]; 1235175347Skmacy txd = &txq->desc[txq->pidx]; 1236174708Skmacy txsd = &txq->sdesc[txq->pidx]; 1237174708Skmacy sgl = txq->txq_sgl; 1238174708Skmacy m0 = *m; 1239175340Skmacy 1240174708Skmacy DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset); 1241174708Skmacy DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan); 1242175340Skmacy if (cxgb_debug) 1243175340Skmacy printf("mi_base=%p cidx=%d pidx=%d\n\n", txsd->mi.mi_base, txq->cidx, txq->pidx); 1244175340Skmacy 1245175340Skmacy mtx_assert(&txq->lock, MA_OWNED); 1246174708Skmacy cntrl = V_TXPKT_INTF(pi->txpkt_intf); 1247174708Skmacy/* 1248174708Skmacy * XXX need to add VLAN support for 6.x 1249174708Skmacy */ 1250167514Skmacy#ifdef VLAN_SUPPORTED 1251168644Skmacy if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) 1252168644Skmacy tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); 1253174708Skmacy#endif 1254175340Skmacy KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p", 1255175340Skmacy txsd->mi.mi_base)); 1256174708Skmacy if (count > 1) { 1257175200Skmacy panic("count > 1 not support in CVS\n"); 1258174708Skmacy if ((err = busdma_map_sg_vec(m, &m0, segs, count))) 1259174708Skmacy return (err); 1260174708Skmacy nsegs = count; 1261174708Skmacy } else if ((err = busdma_map_sg_collapse(&m0, segs, &nsegs))) { 1262174708Skmacy if (cxgb_debug) 1263174708Skmacy printf("failed ... err=%d\n", err); 1264174708Skmacy return (err); 1265174708Skmacy } 1266174708Skmacy KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d count=%d", nsegs, count)); 1267174708Skmacy 1268175347Skmacy if (!(m0->m_pkthdr.len <= PIO_LEN)) { 1269175340Skmacy mi_collapse_mbuf(&txsd->mi, m0); 1270175340Skmacy mi = &txsd->mi; 1271175340Skmacy } 1272174708Skmacy if (count > 1) { 1273174708Skmacy struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; 1274174708Skmacy int i, fidx; 1275174708Skmacy struct mbuf_iovec *batchmi; 1276174708Skmacy 1277174708Skmacy mv = mtomv(m0); 1278174708Skmacy batchmi = mv->mv_vec; 1279174708Skmacy 1280174708Skmacy wrp = (struct work_request_hdr *)txd; 1281174708Skmacy 1282174708Skmacy flits = count*2 + 1; 1283174708Skmacy txq_prod(txq, 1, &txqs); 1284174708Skmacy 1285174708Skmacy for (fidx = 1, i = 0; i < count; i++, batchmi++, fidx += 2) { 1286174708Skmacy struct cpl_tx_pkt_batch_entry *cbe = &cpl_batch->pkt_entry[i]; 1287174708Skmacy 1288174708Skmacy cntrl = V_TXPKT_INTF(pi->txpkt_intf); 1289174708Skmacy GET_VTAG_MI(cntrl, batchmi); 1290174708Skmacy cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1291174708Skmacy cbe->cntrl = htonl(cntrl); 1292174708Skmacy cbe->len = htonl(batchmi->mi_len | 0x80000000); 1293174708Skmacy cbe->addr = htobe64(segs[i].ds_addr); 1294174708Skmacy txd->flit[fidx] |= htobe64(1 << 24); 1295174708Skmacy } 1296174708Skmacy 1297174708Skmacy wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1298174708Skmacy V_WR_SGLSFLT(flits)) | htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1299174708Skmacy wmb(); 1300174708Skmacy wrp->wr_lo = htonl(V_WR_LEN(flits) | 1301174708Skmacy V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token)); 1302174708Skmacy /* XXX gen? */ 1303174708Skmacy wr_gen2(txd, txqs.gen); 1304174708Skmacy check_ring_tx_db(sc, txq); 1305174708Skmacy 1306174708Skmacy return (0); 1307174708Skmacy } else if (tso_info) { 1308174708Skmacy int undersized, eth_type; 1309174708Skmacy struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd; 1310167514Skmacy struct ip *ip; 1311167514Skmacy struct tcphdr *tcp; 1312174708Skmacy char *pkthdr, tmp[TCPPKTHDRSIZE]; 1313174708Skmacy struct mbuf_vec *mv; 1314174708Skmacy struct mbuf_iovec *tmpmi; 1315174708Skmacy 1316174708Skmacy mv = mtomv(m0); 1317174708Skmacy tmpmi = mv->mv_vec; 1318168737Skmacy 1319167514Skmacy txd->flit[2] = 0; 1320174708Skmacy GET_VTAG_MI(cntrl, mi); 1321167514Skmacy cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); 1322167514Skmacy hdr->cntrl = htonl(cntrl); 1323174708Skmacy mlen = m0->m_pkthdr.len; 1324174708Skmacy hdr->len = htonl(mlen | 0x80000000); 1325167514Skmacy 1326174708Skmacy DPRINTF("tso buf len=%d\n", mlen); 1327174708Skmacy undersized = (((tmpmi->mi_len < TCPPKTHDRSIZE) && 1328174708Skmacy (m0->m_flags & M_VLANTAG)) || 1329174708Skmacy (tmpmi->mi_len < TCPPKTHDRSIZE - ETHER_VLAN_ENCAP_LEN)); 1330176472Skmacy 1331174708Skmacy if (__predict_false(undersized)) { 1332174708Skmacy pkthdr = tmp; 1333174708Skmacy dump_mi(mi); 1334174708Skmacy panic("discontig packet - fixxorz"); 1335174708Skmacy } else 1336174708Skmacy pkthdr = m0->m_data; 1337174708Skmacy 1338167514Skmacy if (__predict_false(m0->m_flags & M_VLANTAG)) { 1339167514Skmacy eth_type = CPL_ETH_II_VLAN; 1340167514Skmacy ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + 1341167514Skmacy ETHER_VLAN_ENCAP_LEN); 1342167514Skmacy } else { 1343167514Skmacy eth_type = CPL_ETH_II; 1344167514Skmacy ip = (struct ip *)(pkthdr + ETHER_HDR_LEN); 1345167514Skmacy } 1346167514Skmacy tcp = (struct tcphdr *)((uint8_t *)ip + 1347167514Skmacy sizeof(*ip)); 1348168737Skmacy 1349167514Skmacy tso_info |= V_LSO_ETH_TYPE(eth_type) | 1350167514Skmacy V_LSO_IPHDR_WORDS(ip->ip_hl) | 1351167514Skmacy V_LSO_TCPHDR_WORDS(tcp->th_off); 1352167514Skmacy hdr->lso_info = htonl(tso_info); 1353167514Skmacy flits = 3; 1354167514Skmacy } else { 1355174708Skmacy struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd; 1356174708Skmacy 1357174708Skmacy GET_VTAG(cntrl, m0); 1358167514Skmacy cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1359167514Skmacy cpl->cntrl = htonl(cntrl); 1360174708Skmacy mlen = m0->m_pkthdr.len; 1361174708Skmacy cpl->len = htonl(mlen | 0x80000000); 1362174708Skmacy 1363175340Skmacy if (mlen <= PIO_LEN) { 1364167514Skmacy txq_prod(txq, 1, &txqs); 1365175340Skmacy m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); 1366175340Skmacy m_freem(m0); 1367175340Skmacy m0 = NULL; 1368167514Skmacy flits = (mlen + 7) / 8 + 2; 1369167514Skmacy cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | 1370167514Skmacy V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | 1371167514Skmacy F_WR_SOP | F_WR_EOP | txqs.compl); 1372167514Skmacy wmb(); 1373167514Skmacy cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | 1374167514Skmacy V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); 1375167514Skmacy 1376167514Skmacy wr_gen2(txd, txqs.gen); 1377167514Skmacy check_ring_tx_db(sc, txq); 1378174708Skmacy DPRINTF("pio buf\n"); 1379167514Skmacy return (0); 1380167514Skmacy } 1381174708Skmacy DPRINTF("regular buf\n"); 1382167514Skmacy flits = 2; 1383167514Skmacy } 1384174708Skmacy wrp = (struct work_request_hdr *)txd; 1385167514Skmacy 1386174708Skmacy#ifdef nomore 1387174708Skmacy /* 1388174708Skmacy * XXX need to move into one of the helper routines above 1389174708Skmacy * 1390174708Skmacy */ 1391174708Skmacy if ((err = busdma_map_mbufs(m, txq, txsd, segs, &nsegs)) != 0) 1392167514Skmacy return (err); 1393167514Skmacy m0 = *m; 1394174708Skmacy#endif 1395167514Skmacy ndesc = calc_tx_descs(m0, nsegs); 1396167514Skmacy 1397169978Skmacy sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; 1398167514Skmacy make_sgl(sgp, segs, nsegs); 1399167514Skmacy 1400167514Skmacy sgl_flits = sgl_len(nsegs); 1401167514Skmacy 1402167514Skmacy DPRINTF("make_sgl success nsegs==%d ndesc==%d\n", nsegs, ndesc); 1403167514Skmacy txq_prod(txq, ndesc, &txqs); 1404167514Skmacy wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1405167514Skmacy wr_lo = htonl(V_WR_TID(txq->token)); 1406169978Skmacy write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); 1407174708Skmacy check_ring_tx_db(pi->adapter, txq); 1408167514Skmacy 1409175347Skmacy if ((m0->m_type == MT_DATA) && 1410175347Skmacy ((m0->m_flags & (M_EXT|M_NOFREE)) == M_EXT) && 1411175347Skmacy (m0->m_ext.ext_type != EXT_PACKET)) { 1412174708Skmacy m0->m_flags &= ~M_EXT ; 1413175375Skmacy cxgb_mbufs_outstanding--; 1414174708Skmacy m_free(m0); 1415174708Skmacy } 1416174708Skmacy 1417167514Skmacy return (0); 1418167514Skmacy} 1419167514Skmacy 1420167514Skmacy 1421167514Skmacy/** 1422167514Skmacy * write_imm - write a packet into a Tx descriptor as immediate data 1423167514Skmacy * @d: the Tx descriptor to write 1424167514Skmacy * @m: the packet 1425167514Skmacy * @len: the length of packet data to write as immediate data 1426167514Skmacy * @gen: the generation bit value to write 1427167514Skmacy * 1428167514Skmacy * Writes a packet as immediate data into a Tx descriptor. The packet 1429167514Skmacy * contains a work request at its beginning. We must write the packet 1430167514Skmacy * carefully so the SGE doesn't read accidentally before it's written in 1431167514Skmacy * its entirety. 1432167514Skmacy */ 1433169978Skmacystatic __inline void 1434169978Skmacywrite_imm(struct tx_desc *d, struct mbuf *m, 1435169978Skmacy unsigned int len, unsigned int gen) 1436167514Skmacy{ 1437169978Skmacy struct work_request_hdr *from = mtod(m, struct work_request_hdr *); 1438167514Skmacy struct work_request_hdr *to = (struct work_request_hdr *)d; 1439167514Skmacy 1440174708Skmacy if (len > WR_LEN) 1441174708Skmacy panic("len too big %d\n", len); 1442174708Skmacy if (len < sizeof(*from)) 1443174708Skmacy panic("len too small %d", len); 1444174708Skmacy 1445167514Skmacy memcpy(&to[1], &from[1], len - sizeof(*from)); 1446167514Skmacy to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP | 1447167514Skmacy V_WR_BCNTLFLT(len & 7)); 1448167514Skmacy wmb(); 1449167514Skmacy to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) | 1450167514Skmacy V_WR_LEN((len + 7) / 8)); 1451167514Skmacy wr_gen2(d, gen); 1452174708Skmacy 1453174708Skmacy /* 1454174708Skmacy * This check is a hack we should really fix the logic so 1455174708Skmacy * that this can't happen 1456174708Skmacy */ 1457174708Skmacy if (m->m_type != MT_DONTFREE) 1458174708Skmacy m_freem(m); 1459174708Skmacy 1460167514Skmacy} 1461167514Skmacy 1462167514Skmacy/** 1463167514Skmacy * check_desc_avail - check descriptor availability on a send queue 1464167514Skmacy * @adap: the adapter 1465167514Skmacy * @q: the TX queue 1466167514Skmacy * @m: the packet needing the descriptors 1467167514Skmacy * @ndesc: the number of Tx descriptors needed 1468167514Skmacy * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) 1469167514Skmacy * 1470167514Skmacy * Checks if the requested number of Tx descriptors is available on an 1471167514Skmacy * SGE send queue. If the queue is already suspended or not enough 1472167514Skmacy * descriptors are available the packet is queued for later transmission. 1473167514Skmacy * Must be called with the Tx queue locked. 1474167514Skmacy * 1475167514Skmacy * Returns 0 if enough descriptors are available, 1 if there aren't 1476167514Skmacy * enough descriptors and the packet has been queued, and 2 if the caller 1477167514Skmacy * needs to retry because there weren't enough descriptors at the 1478167514Skmacy * beginning of the call but some freed up in the mean time. 1479167514Skmacy */ 1480167514Skmacystatic __inline int 1481167514Skmacycheck_desc_avail(adapter_t *adap, struct sge_txq *q, 1482169978Skmacy struct mbuf *m, unsigned int ndesc, 1483169978Skmacy unsigned int qid) 1484167514Skmacy{ 1485167514Skmacy /* 1486167514Skmacy * XXX We currently only use this for checking the control queue 1487167514Skmacy * the control queue is only used for binding qsets which happens 1488167514Skmacy * at init time so we are guaranteed enough descriptors 1489167514Skmacy */ 1490169978Skmacy if (__predict_false(!mbufq_empty(&q->sendq))) { 1491169978Skmacyaddq_exit: mbufq_tail(&q->sendq, m); 1492167514Skmacy return 1; 1493167514Skmacy } 1494167514Skmacy if (__predict_false(q->size - q->in_use < ndesc)) { 1495167514Skmacy 1496167514Skmacy struct sge_qset *qs = txq_to_qset(q, qid); 1497167514Skmacy 1498174708Skmacy printf("stopping q\n"); 1499174708Skmacy 1500169978Skmacy setbit(&qs->txq_stopped, qid); 1501169978Skmacy smp_mb(); 1502167514Skmacy 1503167514Skmacy if (should_restart_tx(q) && 1504167514Skmacy test_and_clear_bit(qid, &qs->txq_stopped)) 1505167514Skmacy return 2; 1506167514Skmacy 1507167514Skmacy q->stops++; 1508167514Skmacy goto addq_exit; 1509167514Skmacy } 1510167514Skmacy return 0; 1511167514Skmacy} 1512167514Skmacy 1513167514Skmacy 1514167514Skmacy/** 1515167514Skmacy * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs 1516167514Skmacy * @q: the SGE control Tx queue 1517167514Skmacy * 1518167514Skmacy * This is a variant of reclaim_completed_tx() that is used for Tx queues 1519167514Skmacy * that send only immediate data (presently just the control queues) and 1520169978Skmacy * thus do not have any mbufs 1521167514Skmacy */ 1522167514Skmacystatic __inline void 1523167514Skmacyreclaim_completed_tx_imm(struct sge_txq *q) 1524167514Skmacy{ 1525167514Skmacy unsigned int reclaim = q->processed - q->cleaned; 1526167514Skmacy 1527167514Skmacy mtx_assert(&q->lock, MA_OWNED); 1528167514Skmacy 1529167514Skmacy q->in_use -= reclaim; 1530167514Skmacy q->cleaned += reclaim; 1531167514Skmacy} 1532167514Skmacy 1533167514Skmacystatic __inline int 1534167514Skmacyimmediate(const struct mbuf *m) 1535167514Skmacy{ 1536167514Skmacy return m->m_len <= WR_LEN && m->m_pkthdr.len <= WR_LEN ; 1537167514Skmacy} 1538167514Skmacy 1539167514Skmacy/** 1540167514Skmacy * ctrl_xmit - send a packet through an SGE control Tx queue 1541167514Skmacy * @adap: the adapter 1542167514Skmacy * @q: the control queue 1543167514Skmacy * @m: the packet 1544167514Skmacy * 1545167514Skmacy * Send a packet through an SGE control Tx queue. Packets sent through 1546167514Skmacy * a control queue must fit entirely as immediate data in a single Tx 1547167514Skmacy * descriptor and have no page fragments. 1548167514Skmacy */ 1549167514Skmacystatic int 1550167514Skmacyctrl_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) 1551167514Skmacy{ 1552167514Skmacy int ret; 1553170654Skmacy struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *); 1554167514Skmacy 1555167514Skmacy if (__predict_false(!immediate(m))) { 1556167514Skmacy m_freem(m); 1557167514Skmacy return 0; 1558167514Skmacy } 1559174708Skmacy 1560167514Skmacy wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP); 1561167514Skmacy wrp->wr_lo = htonl(V_WR_TID(q->token)); 1562167514Skmacy 1563167514Skmacy mtx_lock(&q->lock); 1564167514Skmacyagain: reclaim_completed_tx_imm(q); 1565167514Skmacy 1566167514Skmacy ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); 1567167514Skmacy if (__predict_false(ret)) { 1568167514Skmacy if (ret == 1) { 1569167514Skmacy mtx_unlock(&q->lock); 1570174708Skmacy log(LOG_ERR, "no desc available\n"); 1571174708Skmacy return (ENOSPC); 1572167514Skmacy } 1573167514Skmacy goto again; 1574167514Skmacy } 1575167514Skmacy write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1576174708Skmacy 1577167514Skmacy q->in_use++; 1578167514Skmacy if (++q->pidx >= q->size) { 1579167514Skmacy q->pidx = 0; 1580167514Skmacy q->gen ^= 1; 1581167514Skmacy } 1582167514Skmacy mtx_unlock(&q->lock); 1583167514Skmacy wmb(); 1584167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1585167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1586167514Skmacy return (0); 1587167514Skmacy} 1588167514Skmacy 1589169978Skmacy 1590167514Skmacy/** 1591167514Skmacy * restart_ctrlq - restart a suspended control queue 1592167514Skmacy * @qs: the queue set cotaining the control queue 1593167514Skmacy * 1594167514Skmacy * Resumes transmission on a suspended Tx control queue. 1595167514Skmacy */ 1596167514Skmacystatic void 1597169978Skmacyrestart_ctrlq(void *data, int npending) 1598167514Skmacy{ 1599167514Skmacy struct mbuf *m; 1600167514Skmacy struct sge_qset *qs = (struct sge_qset *)data; 1601167514Skmacy struct sge_txq *q = &qs->txq[TXQ_CTRL]; 1602167514Skmacy adapter_t *adap = qs->port->adapter; 1603167514Skmacy 1604174708Skmacy log(LOG_WARNING, "Restart_ctrlq in_use=%d\n", q->in_use); 1605174708Skmacy 1606167514Skmacy mtx_lock(&q->lock); 1607167514Skmacyagain: reclaim_completed_tx_imm(q); 1608169978Skmacy 1609167514Skmacy while (q->in_use < q->size && 1610169978Skmacy (m = mbufq_dequeue(&q->sendq)) != NULL) { 1611167514Skmacy 1612169978Skmacy write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1613167514Skmacy 1614167514Skmacy if (++q->pidx >= q->size) { 1615167514Skmacy q->pidx = 0; 1616167514Skmacy q->gen ^= 1; 1617167514Skmacy } 1618167514Skmacy q->in_use++; 1619167514Skmacy } 1620169978Skmacy if (!mbufq_empty(&q->sendq)) { 1621169978Skmacy setbit(&qs->txq_stopped, TXQ_CTRL); 1622169978Skmacy smp_mb(); 1623167514Skmacy 1624167514Skmacy if (should_restart_tx(q) && 1625167514Skmacy test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) 1626167514Skmacy goto again; 1627167514Skmacy q->stops++; 1628167514Skmacy } 1629167514Skmacy mtx_unlock(&q->lock); 1630176472Skmacy wmb(); 1631167514Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 1632167514Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1633167514Skmacy} 1634167514Skmacy 1635169978Skmacy 1636167514Skmacy/* 1637167514Skmacy * Send a management message through control queue 0 1638167514Skmacy */ 1639167514Skmacyint 1640167514Skmacyt3_mgmt_tx(struct adapter *adap, struct mbuf *m) 1641167514Skmacy{ 1642167514Skmacy return ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], m); 1643167514Skmacy} 1644167514Skmacy 1645174708Skmacy 1646167514Skmacy/** 1647167514Skmacy * free_qset - free the resources of an SGE queue set 1648167514Skmacy * @sc: the controller owning the queue set 1649167514Skmacy * @q: the queue set 1650167514Skmacy * 1651167514Skmacy * Release the HW and SW resources associated with an SGE queue set, such 1652167514Skmacy * as HW contexts, packet buffers, and descriptor rings. Traffic to the 1653167514Skmacy * queue set must be quiesced prior to calling this. 1654167514Skmacy */ 1655174708Skmacyvoid 1656167514Skmacyt3_free_qset(adapter_t *sc, struct sge_qset *q) 1657167514Skmacy{ 1658167514Skmacy int i; 1659174708Skmacy 1660174708Skmacy t3_free_tx_desc_all(&q->txq[TXQ_ETH]); 1661174708Skmacy 1662174708Skmacy for (i = 0; i < SGE_TXQ_PER_SET; i++) 1663174708Skmacy if (q->txq[i].txq_mr.br_ring != NULL) { 1664174708Skmacy free(q->txq[i].txq_mr.br_ring, M_DEVBUF); 1665174708Skmacy mtx_destroy(&q->txq[i].txq_mr.br_lock); 1666174708Skmacy } 1667167514Skmacy for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 1668167514Skmacy if (q->fl[i].desc) { 1669176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 1670167514Skmacy t3_sge_disable_fl(sc, q->fl[i].cntxt_id); 1671176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 1672167514Skmacy bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); 1673167514Skmacy bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, 1674167514Skmacy q->fl[i].desc_map); 1675167514Skmacy bus_dma_tag_destroy(q->fl[i].desc_tag); 1676168351Skmacy bus_dma_tag_destroy(q->fl[i].entry_tag); 1677167514Skmacy } 1678167514Skmacy if (q->fl[i].sdesc) { 1679167514Skmacy free_rx_bufs(sc, &q->fl[i]); 1680167514Skmacy free(q->fl[i].sdesc, M_DEVBUF); 1681167514Skmacy } 1682167514Skmacy } 1683167514Skmacy 1684170869Skmacy for (i = 0; i < SGE_TXQ_PER_SET; i++) { 1685167514Skmacy if (q->txq[i].desc) { 1686176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 1687167514Skmacy t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); 1688176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 1689167514Skmacy bus_dmamap_unload(q->txq[i].desc_tag, 1690167514Skmacy q->txq[i].desc_map); 1691167514Skmacy bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, 1692167514Skmacy q->txq[i].desc_map); 1693167514Skmacy bus_dma_tag_destroy(q->txq[i].desc_tag); 1694168351Skmacy bus_dma_tag_destroy(q->txq[i].entry_tag); 1695170869Skmacy MTX_DESTROY(&q->txq[i].lock); 1696167514Skmacy } 1697167514Skmacy if (q->txq[i].sdesc) { 1698167514Skmacy free(q->txq[i].sdesc, M_DEVBUF); 1699167514Skmacy } 1700167514Skmacy } 1701167514Skmacy 1702167514Skmacy if (q->rspq.desc) { 1703176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 1704167514Skmacy t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); 1705176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 1706167514Skmacy 1707167514Skmacy bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); 1708167514Skmacy bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, 1709167514Skmacy q->rspq.desc_map); 1710167514Skmacy bus_dma_tag_destroy(q->rspq.desc_tag); 1711170869Skmacy MTX_DESTROY(&q->rspq.lock); 1712167514Skmacy } 1713168351Skmacy 1714167514Skmacy bzero(q, sizeof(*q)); 1715167514Skmacy} 1716167514Skmacy 1717167514Skmacy/** 1718167514Skmacy * t3_free_sge_resources - free SGE resources 1719167514Skmacy * @sc: the adapter softc 1720167514Skmacy * 1721167514Skmacy * Frees resources used by the SGE queue sets. 1722167514Skmacy */ 1723167514Skmacyvoid 1724167514Skmacyt3_free_sge_resources(adapter_t *sc) 1725167514Skmacy{ 1726170869Skmacy int i, nqsets; 1727174708Skmacy 1728174708Skmacy#ifdef IFNET_MULTIQUEUE 1729174708Skmacy panic("%s should not be called when IFNET_MULTIQUEUE is defined", __FUNCTION__); 1730174708Skmacy#endif 1731170869Skmacy for (nqsets = i = 0; i < (sc)->params.nports; i++) 1732170869Skmacy nqsets += sc->port[i].nqsets; 1733174708Skmacy 1734170869Skmacy for (i = 0; i < nqsets; ++i) 1735167514Skmacy t3_free_qset(sc, &sc->sge.qs[i]); 1736167514Skmacy} 1737167514Skmacy 1738167514Skmacy/** 1739167514Skmacy * t3_sge_start - enable SGE 1740167514Skmacy * @sc: the controller softc 1741167514Skmacy * 1742167514Skmacy * Enables the SGE for DMAs. This is the last step in starting packet 1743167514Skmacy * transfers. 1744167514Skmacy */ 1745167514Skmacyvoid 1746167514Skmacyt3_sge_start(adapter_t *sc) 1747167514Skmacy{ 1748167514Skmacy t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); 1749167514Skmacy} 1750167514Skmacy 1751169978Skmacy/** 1752169978Skmacy * t3_sge_stop - disable SGE operation 1753169978Skmacy * @sc: the adapter 1754169978Skmacy * 1755169978Skmacy * Disables the DMA engine. This can be called in emeregencies (e.g., 1756169978Skmacy * from error interrupts) or from normal process context. In the latter 1757169978Skmacy * case it also disables any pending queue restart tasklets. Note that 1758169978Skmacy * if it is called in interrupt context it cannot disable the restart 1759169978Skmacy * tasklets as it cannot wait, however the tasklets will have no effect 1760169978Skmacy * since the doorbells are disabled and the driver will call this again 1761169978Skmacy * later from process context, at which time the tasklets will be stopped 1762169978Skmacy * if they are still running. 1763169978Skmacy */ 1764169978Skmacyvoid 1765169978Skmacyt3_sge_stop(adapter_t *sc) 1766169978Skmacy{ 1767170869Skmacy int i, nqsets; 1768170869Skmacy 1769169978Skmacy t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0); 1770167514Skmacy 1771170654Skmacy if (sc->tq == NULL) 1772170654Skmacy return; 1773170654Skmacy 1774170869Skmacy for (nqsets = i = 0; i < (sc)->params.nports; i++) 1775170869Skmacy nqsets += sc->port[i].nqsets; 1776175340Skmacy#ifdef notyet 1777175340Skmacy /* 1778175340Skmacy * 1779175340Skmacy * XXX 1780175340Skmacy */ 1781170869Skmacy for (i = 0; i < nqsets; ++i) { 1782169978Skmacy struct sge_qset *qs = &sc->sge.qs[i]; 1783169978Skmacy 1784171335Skmacy taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); 1785171335Skmacy taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); 1786169978Skmacy } 1787175340Skmacy#endif 1788169978Skmacy} 1789169978Skmacy 1790167514Skmacy/** 1791174708Skmacy * t3_free_tx_desc - reclaims Tx descriptors and their buffers 1792167514Skmacy * @adapter: the adapter 1793167514Skmacy * @q: the Tx queue to reclaim descriptors from 1794174708Skmacy * @reclaimable: the number of descriptors to reclaim 1795174708Skmacy * @m_vec_size: maximum number of buffers to reclaim 1796174708Skmacy * @desc_reclaimed: returns the number of descriptors reclaimed 1797167514Skmacy * 1798167514Skmacy * Reclaims Tx descriptors from an SGE Tx queue and frees the associated 1799167514Skmacy * Tx buffers. Called with the Tx queue lock held. 1800174708Skmacy * 1801174708Skmacy * Returns number of buffers of reclaimed 1802167514Skmacy */ 1803174708Skmacyvoid 1804174708Skmacyt3_free_tx_desc(struct sge_txq *q, int reclaimable) 1805167514Skmacy{ 1806174708Skmacy struct tx_sw_desc *txsd; 1807174708Skmacy unsigned int cidx; 1808167514Skmacy 1809167514Skmacy#ifdef T3_TRACE 1810167514Skmacy T3_TRACE2(sc->tb[q->cntxt_id & 7], 1811174708Skmacy "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx); 1812167514Skmacy#endif 1813174708Skmacy cidx = q->cidx; 1814174708Skmacy txsd = &q->sdesc[cidx]; 1815174708Skmacy DPRINTF("reclaiming %d WR\n", reclaimable); 1816175347Skmacy mtx_assert(&q->lock, MA_OWNED); 1817174708Skmacy while (reclaimable--) { 1818174708Skmacy DPRINTF("cidx=%d d=%p\n", cidx, txsd); 1819174708Skmacy if (txsd->mi.mi_base != NULL) { 1820174708Skmacy if (txsd->flags & TX_SW_DESC_MAPPED) { 1821174708Skmacy bus_dmamap_unload(q->entry_tag, txsd->map); 1822174708Skmacy txsd->flags &= ~TX_SW_DESC_MAPPED; 1823167514Skmacy } 1824175340Skmacy m_freem_iovec(&txsd->mi); 1825175340Skmacy buf_ring_scan(&q->txq_mr, txsd->mi.mi_base, __FILE__, __LINE__); 1826174708Skmacy txsd->mi.mi_base = NULL; 1827174708Skmacy 1828174708Skmacy#if defined(DIAGNOSTIC) && 0 1829174708Skmacy if (m_get_priority(txsd->m[0]) != cidx) 1830175200Skmacy printf("pri=%d cidx=%d\n", 1831175200Skmacy (int)m_get_priority(txsd->m[0]), cidx); 1832174708Skmacy#endif 1833174708Skmacy 1834174708Skmacy } else 1835174708Skmacy q->txq_skipped++; 1836174708Skmacy 1837174708Skmacy ++txsd; 1838167514Skmacy if (++cidx == q->size) { 1839167514Skmacy cidx = 0; 1840174708Skmacy txsd = q->sdesc; 1841167514Skmacy } 1842167514Skmacy } 1843167514Skmacy q->cidx = cidx; 1844167514Skmacy 1845167514Skmacy} 1846167514Skmacy 1847174708Skmacyvoid 1848174708Skmacyt3_free_tx_desc_all(struct sge_txq *q) 1849174708Skmacy{ 1850174708Skmacy int i; 1851174708Skmacy struct tx_sw_desc *txsd; 1852174708Skmacy 1853174708Skmacy for (i = 0; i < q->size; i++) { 1854174708Skmacy txsd = &q->sdesc[i]; 1855174708Skmacy if (txsd->mi.mi_base != NULL) { 1856174708Skmacy if (txsd->flags & TX_SW_DESC_MAPPED) { 1857174708Skmacy bus_dmamap_unload(q->entry_tag, txsd->map); 1858174708Skmacy txsd->flags &= ~TX_SW_DESC_MAPPED; 1859174708Skmacy } 1860174708Skmacy m_freem_iovec(&txsd->mi); 1861174708Skmacy bzero(&txsd->mi, sizeof(txsd->mi)); 1862174708Skmacy } 1863174708Skmacy } 1864174708Skmacy} 1865174708Skmacy 1866167514Skmacy/** 1867167514Skmacy * is_new_response - check if a response is newly written 1868167514Skmacy * @r: the response descriptor 1869167514Skmacy * @q: the response queue 1870167514Skmacy * 1871167514Skmacy * Returns true if a response descriptor contains a yet unprocessed 1872167514Skmacy * response. 1873167514Skmacy */ 1874167514Skmacystatic __inline int 1875167514Skmacyis_new_response(const struct rsp_desc *r, 1876167514Skmacy const struct sge_rspq *q) 1877167514Skmacy{ 1878167514Skmacy return (r->intr_gen & F_RSPD_GEN2) == q->gen; 1879167514Skmacy} 1880167514Skmacy 1881167514Skmacy#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) 1882167514Skmacy#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ 1883167514Skmacy V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ 1884167514Skmacy V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ 1885167514Skmacy V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) 1886167514Skmacy 1887167514Skmacy/* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ 1888167514Skmacy#define NOMEM_INTR_DELAY 2500 1889167514Skmacy 1890169978Skmacy/** 1891169978Skmacy * write_ofld_wr - write an offload work request 1892169978Skmacy * @adap: the adapter 1893169978Skmacy * @m: the packet to send 1894169978Skmacy * @q: the Tx queue 1895169978Skmacy * @pidx: index of the first Tx descriptor to write 1896169978Skmacy * @gen: the generation value to use 1897169978Skmacy * @ndesc: number of descriptors the packet will occupy 1898169978Skmacy * 1899169978Skmacy * Write an offload work request to send the supplied packet. The packet 1900169978Skmacy * data already carry the work request with most fields populated. 1901169978Skmacy */ 1902169978Skmacystatic void 1903169978Skmacywrite_ofld_wr(adapter_t *adap, struct mbuf *m, 1904169978Skmacy struct sge_txq *q, unsigned int pidx, 1905169978Skmacy unsigned int gen, unsigned int ndesc, 1906169978Skmacy bus_dma_segment_t *segs, unsigned int nsegs) 1907167514Skmacy{ 1908169978Skmacy unsigned int sgl_flits, flits; 1909169978Skmacy struct work_request_hdr *from; 1910169978Skmacy struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; 1911169978Skmacy struct tx_desc *d = &q->desc[pidx]; 1912169978Skmacy struct txq_state txqs; 1913169978Skmacy 1914176472Skmacy if (immediate(m) && nsegs == 0) { 1915169978Skmacy write_imm(d, m, m->m_len, gen); 1916169978Skmacy return; 1917169978Skmacy } 1918169978Skmacy 1919169978Skmacy /* Only TX_DATA builds SGLs */ 1920169978Skmacy from = mtod(m, struct work_request_hdr *); 1921174708Skmacy memcpy(&d->flit[1], &from[1], m->m_len - sizeof(*from)); 1922169978Skmacy 1923174708Skmacy flits = m->m_len / 8; 1924169978Skmacy sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl; 1925169978Skmacy 1926169978Skmacy make_sgl(sgp, segs, nsegs); 1927169978Skmacy sgl_flits = sgl_len(nsegs); 1928169978Skmacy 1929174708Skmacy txqs.gen = gen; 1930174708Skmacy txqs.pidx = pidx; 1931174708Skmacy txqs.compl = 0; 1932174708Skmacy 1933169978Skmacy write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits, 1934169978Skmacy from->wr_hi, from->wr_lo); 1935167514Skmacy} 1936167514Skmacy 1937169978Skmacy/** 1938169978Skmacy * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet 1939169978Skmacy * @m: the packet 1940169978Skmacy * 1941169978Skmacy * Returns the number of Tx descriptors needed for the given offload 1942169978Skmacy * packet. These packets are already fully constructed. 1943169978Skmacy */ 1944169978Skmacystatic __inline unsigned int 1945169978Skmacycalc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs) 1946167514Skmacy{ 1947169978Skmacy unsigned int flits, cnt = 0; 1948176472Skmacy int ndescs; 1949169978Skmacy 1950176472Skmacy if (m->m_len <= WR_LEN && nsegs == 0) 1951176472Skmacy return (1); /* packet fits as immediate data */ 1952169978Skmacy 1953169978Skmacy if (m->m_flags & M_IOVEC) 1954169978Skmacy cnt = mtomv(m)->mv_count; 1955176472Skmacy else 1956176472Skmacy cnt = nsegs; 1957169978Skmacy 1958175200Skmacy /* headers */ 1959176472Skmacy flits = m->m_len / 8; 1960169978Skmacy 1961176472Skmacy ndescs = flits_to_desc(flits + sgl_len(cnt)); 1962176472Skmacy 1963176472Skmacy CTR4(KTR_CXGB, "flits=%d sgl_len=%d nsegs=%d ndescs=%d", 1964176472Skmacy flits, sgl_len(cnt), nsegs, ndescs); 1965176472Skmacy 1966176472Skmacy return (ndescs); 1967169978Skmacy} 1968169978Skmacy 1969169978Skmacy/** 1970169978Skmacy * ofld_xmit - send a packet through an offload queue 1971169978Skmacy * @adap: the adapter 1972169978Skmacy * @q: the Tx offload queue 1973169978Skmacy * @m: the packet 1974169978Skmacy * 1975169978Skmacy * Send an offload packet through an SGE offload queue. 1976169978Skmacy */ 1977169978Skmacystatic int 1978169978Skmacyofld_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) 1979169978Skmacy{ 1980171978Skmacy int ret, nsegs; 1981171978Skmacy unsigned int ndesc; 1982171978Skmacy unsigned int pidx, gen; 1983174708Skmacy bus_dma_segment_t segs[TX_MAX_SEGS], *vsegs; 1984174708Skmacy struct tx_sw_desc *stx; 1985169978Skmacy 1986174708Skmacy nsegs = m_get_sgllen(m); 1987174708Skmacy vsegs = m_get_sgl(m); 1988169978Skmacy ndesc = calc_tx_descs_ofld(m, nsegs); 1989174708Skmacy busdma_map_sgl(vsegs, segs, nsegs); 1990169978Skmacy 1991174708Skmacy stx = &q->sdesc[q->pidx]; 1992174708Skmacy KASSERT(stx->mi.mi_base == NULL, ("mi_base set")); 1993174708Skmacy 1994174708Skmacy mtx_lock(&q->lock); 1995175224Skmacyagain: reclaim_completed_tx_(q, 16); 1996169978Skmacy ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD); 1997169978Skmacy if (__predict_false(ret)) { 1998169978Skmacy if (ret == 1) { 1999174708Skmacy printf("no ofld desc avail\n"); 2000174708Skmacy 2001169978Skmacy m_set_priority(m, ndesc); /* save for restart */ 2002169978Skmacy mtx_unlock(&q->lock); 2003174708Skmacy return (EINTR); 2004167514Skmacy } 2005169978Skmacy goto again; 2006169978Skmacy } 2007169978Skmacy 2008169978Skmacy gen = q->gen; 2009169978Skmacy q->in_use += ndesc; 2010169978Skmacy pidx = q->pidx; 2011169978Skmacy q->pidx += ndesc; 2012169978Skmacy if (q->pidx >= q->size) { 2013169978Skmacy q->pidx -= q->size; 2014169978Skmacy q->gen ^= 1; 2015169978Skmacy } 2016169978Skmacy#ifdef T3_TRACE 2017169978Skmacy T3_TRACE5(adap->tb[q->cntxt_id & 7], 2018169978Skmacy "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u", 2019169978Skmacy ndesc, pidx, skb->len, skb->len - skb->data_len, 2020169978Skmacy skb_shinfo(skb)->nr_frags); 2021167514Skmacy#endif 2022169978Skmacy mtx_unlock(&q->lock); 2023169978Skmacy 2024169978Skmacy write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 2025169978Skmacy check_ring_tx_db(adap, q); 2026172101Skmacy return (0); 2027169978Skmacy} 2028167514Skmacy 2029169978Skmacy/** 2030169978Skmacy * restart_offloadq - restart a suspended offload queue 2031169978Skmacy * @qs: the queue set cotaining the offload queue 2032169978Skmacy * 2033169978Skmacy * Resumes transmission on a suspended Tx offload queue. 2034169978Skmacy */ 2035169978Skmacystatic void 2036169978Skmacyrestart_offloadq(void *data, int npending) 2037169978Skmacy{ 2038169978Skmacy struct mbuf *m; 2039169978Skmacy struct sge_qset *qs = data; 2040169978Skmacy struct sge_txq *q = &qs->txq[TXQ_OFLD]; 2041169978Skmacy adapter_t *adap = qs->port->adapter; 2042169978Skmacy bus_dma_segment_t segs[TX_MAX_SEGS]; 2043169978Skmacy struct tx_sw_desc *stx = &q->sdesc[q->pidx]; 2044174708Skmacy int nsegs, cleaned; 2045169978Skmacy 2046169978Skmacy mtx_lock(&q->lock); 2047175224Skmacyagain: cleaned = reclaim_completed_tx_(q, 16); 2048169978Skmacy 2049169978Skmacy while ((m = mbufq_peek(&q->sendq)) != NULL) { 2050169978Skmacy unsigned int gen, pidx; 2051169978Skmacy unsigned int ndesc = m_get_priority(m); 2052169978Skmacy 2053169978Skmacy if (__predict_false(q->size - q->in_use < ndesc)) { 2054169978Skmacy setbit(&qs->txq_stopped, TXQ_OFLD); 2055169978Skmacy smp_mb(); 2056169978Skmacy 2057169978Skmacy if (should_restart_tx(q) && 2058169978Skmacy test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) 2059169978Skmacy goto again; 2060169978Skmacy q->stops++; 2061169978Skmacy break; 2062169978Skmacy } 2063169978Skmacy 2064169978Skmacy gen = q->gen; 2065169978Skmacy q->in_use += ndesc; 2066169978Skmacy pidx = q->pidx; 2067169978Skmacy q->pidx += ndesc; 2068169978Skmacy if (q->pidx >= q->size) { 2069169978Skmacy q->pidx -= q->size; 2070169978Skmacy q->gen ^= 1; 2071169978Skmacy } 2072169978Skmacy 2073169978Skmacy (void)mbufq_dequeue(&q->sendq); 2074169978Skmacy busdma_map_mbufs(&m, q, stx, segs, &nsegs); 2075169978Skmacy mtx_unlock(&q->lock); 2076169978Skmacy write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 2077169978Skmacy mtx_lock(&q->lock); 2078169978Skmacy } 2079169978Skmacy mtx_unlock(&q->lock); 2080169978Skmacy 2081169978Skmacy#if USE_GTS 2082169978Skmacy set_bit(TXQ_RUNNING, &q->flags); 2083169978Skmacy set_bit(TXQ_LAST_PKT_DB, &q->flags); 2084169978Skmacy#endif 2085176472Skmacy wmb(); 2086169978Skmacy t3_write_reg(adap, A_SG_KDOORBELL, 2087169978Skmacy F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 2088167514Skmacy} 2089167514Skmacy 2090169978Skmacy/** 2091169978Skmacy * queue_set - return the queue set a packet should use 2092169978Skmacy * @m: the packet 2093169978Skmacy * 2094169978Skmacy * Maps a packet to the SGE queue set it should use. The desired queue 2095169978Skmacy * set is carried in bits 1-3 in the packet's priority. 2096169978Skmacy */ 2097169978Skmacystatic __inline int 2098169978Skmacyqueue_set(const struct mbuf *m) 2099169978Skmacy{ 2100169978Skmacy return m_get_priority(m) >> 1; 2101169978Skmacy} 2102169978Skmacy 2103169978Skmacy/** 2104169978Skmacy * is_ctrl_pkt - return whether an offload packet is a control packet 2105169978Skmacy * @m: the packet 2106169978Skmacy * 2107169978Skmacy * Determines whether an offload packet should use an OFLD or a CTRL 2108169978Skmacy * Tx queue. This is indicated by bit 0 in the packet's priority. 2109169978Skmacy */ 2110169978Skmacystatic __inline int 2111169978Skmacyis_ctrl_pkt(const struct mbuf *m) 2112169978Skmacy{ 2113169978Skmacy return m_get_priority(m) & 1; 2114169978Skmacy} 2115169978Skmacy 2116169978Skmacy/** 2117169978Skmacy * t3_offload_tx - send an offload packet 2118169978Skmacy * @tdev: the offload device to send to 2119169978Skmacy * @m: the packet 2120169978Skmacy * 2121169978Skmacy * Sends an offload packet. We use the packet priority to select the 2122169978Skmacy * appropriate Tx queue as follows: bit 0 indicates whether the packet 2123169978Skmacy * should be sent as regular or control, bits 1-3 select the queue set. 2124169978Skmacy */ 2125169978Skmacyint 2126174626Skmacyt3_offload_tx(struct t3cdev *tdev, struct mbuf *m) 2127169978Skmacy{ 2128169978Skmacy adapter_t *adap = tdev2adap(tdev); 2129169978Skmacy struct sge_qset *qs = &adap->sge.qs[queue_set(m)]; 2130169978Skmacy 2131174708Skmacy if (__predict_false(is_ctrl_pkt(m))) 2132169978Skmacy return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], m); 2133169978Skmacy 2134169978Skmacy return ofld_xmit(adap, &qs->txq[TXQ_OFLD], m); 2135169978Skmacy} 2136169978Skmacy 2137169978Skmacy/** 2138169978Skmacy * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts 2139169978Skmacy * @tdev: the offload device that will be receiving the packets 2140169978Skmacy * @q: the SGE response queue that assembled the bundle 2141169978Skmacy * @m: the partial bundle 2142169978Skmacy * @n: the number of packets in the bundle 2143169978Skmacy * 2144169978Skmacy * Delivers a (partial) bundle of Rx offload packets to an offload device. 2145169978Skmacy */ 2146169978Skmacystatic __inline void 2147174626Skmacydeliver_partial_bundle(struct t3cdev *tdev, 2148169978Skmacy struct sge_rspq *q, 2149169978Skmacy struct mbuf *mbufs[], int n) 2150169978Skmacy{ 2151169978Skmacy if (n) { 2152169978Skmacy q->offload_bundles++; 2153169978Skmacy cxgb_ofld_recv(tdev, mbufs, n); 2154169978Skmacy } 2155169978Skmacy} 2156169978Skmacy 2157169978Skmacystatic __inline int 2158174626Skmacyrx_offload(struct t3cdev *tdev, struct sge_rspq *rq, 2159169978Skmacy struct mbuf *m, struct mbuf *rx_gather[], 2160169978Skmacy unsigned int gather_idx) 2161169978Skmacy{ 2162174708Skmacy 2163169978Skmacy rq->offload_pkts++; 2164169978Skmacy m->m_pkthdr.header = mtod(m, void *); 2165169978Skmacy rx_gather[gather_idx++] = m; 2166169978Skmacy if (gather_idx == RX_BUNDLE_SIZE) { 2167169978Skmacy cxgb_ofld_recv(tdev, rx_gather, RX_BUNDLE_SIZE); 2168169978Skmacy gather_idx = 0; 2169169978Skmacy rq->offload_bundles++; 2170169978Skmacy } 2171169978Skmacy return (gather_idx); 2172169978Skmacy} 2173169978Skmacy 2174167514Skmacystatic void 2175167514Skmacyrestart_tx(struct sge_qset *qs) 2176167514Skmacy{ 2177169978Skmacy struct adapter *sc = qs->port->adapter; 2178169978Skmacy 2179174708Skmacy 2180169978Skmacy if (isset(&qs->txq_stopped, TXQ_OFLD) && 2181169978Skmacy should_restart_tx(&qs->txq[TXQ_OFLD]) && 2182169978Skmacy test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { 2183169978Skmacy qs->txq[TXQ_OFLD].restarts++; 2184175200Skmacy DPRINTF("restarting TXQ_OFLD\n"); 2185171335Skmacy taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); 2186169978Skmacy } 2187175200Skmacy DPRINTF("stopped=0x%x restart=%d processed=%d cleaned=%d in_use=%d\n", 2188174708Skmacy qs->txq_stopped, should_restart_tx(&qs->txq[TXQ_CTRL]), 2189174708Skmacy qs->txq[TXQ_CTRL].processed, qs->txq[TXQ_CTRL].cleaned, 2190174708Skmacy qs->txq[TXQ_CTRL].in_use); 2191174708Skmacy 2192169978Skmacy if (isset(&qs->txq_stopped, TXQ_CTRL) && 2193169978Skmacy should_restart_tx(&qs->txq[TXQ_CTRL]) && 2194169978Skmacy test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { 2195169978Skmacy qs->txq[TXQ_CTRL].restarts++; 2196175200Skmacy DPRINTF("restarting TXQ_CTRL\n"); 2197171335Skmacy taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); 2198169978Skmacy } 2199167514Skmacy} 2200167514Skmacy 2201169978Skmacy/** 2202169978Skmacy * t3_sge_alloc_qset - initialize an SGE queue set 2203169978Skmacy * @sc: the controller softc 2204169978Skmacy * @id: the queue set id 2205169978Skmacy * @nports: how many Ethernet ports will be using this queue set 2206169978Skmacy * @irq_vec_idx: the IRQ vector index for response queue interrupts 2207169978Skmacy * @p: configuration parameters for this queue set 2208169978Skmacy * @ntxq: number of Tx queues for the queue set 2209169978Skmacy * @pi: port info for queue set 2210169978Skmacy * 2211169978Skmacy * Allocate resources and initialize an SGE queue set. A queue set 2212169978Skmacy * comprises a response queue, two Rx free-buffer queues, and up to 3 2213169978Skmacy * Tx queues. The Tx queues are assigned roles in the order Ethernet 2214169978Skmacy * queue, offload queue, and control queue. 2215169978Skmacy */ 2216169978Skmacyint 2217169978Skmacyt3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, 2218169978Skmacy const struct qset_params *p, int ntxq, struct port_info *pi) 2219169978Skmacy{ 2220169978Skmacy struct sge_qset *q = &sc->sge.qs[id]; 2221175200Skmacy int i, header_size, ret = 0; 2222169978Skmacy 2223174708Skmacy for (i = 0; i < SGE_TXQ_PER_SET; i++) { 2224174708Skmacy if ((q->txq[i].txq_mr.br_ring = malloc(cxgb_txq_buf_ring_size*sizeof(struct mbuf *), 2225174708Skmacy M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { 2226174708Skmacy device_printf(sc->dev, "failed to allocate mbuf ring\n"); 2227174708Skmacy goto err; 2228174708Skmacy } 2229174708Skmacy q->txq[i].txq_mr.br_prod = q->txq[i].txq_mr.br_cons = 0; 2230174708Skmacy q->txq[i].txq_mr.br_size = cxgb_txq_buf_ring_size; 2231174708Skmacy mtx_init(&q->txq[i].txq_mr.br_lock, "txq mbuf ring", NULL, MTX_DEF); 2232174708Skmacy } 2233174708Skmacy 2234169978Skmacy init_qset_cntxt(q, id); 2235175347Skmacy q->idx = id; 2236169978Skmacy 2237169978Skmacy if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), 2238169978Skmacy sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, 2239169978Skmacy &q->fl[0].desc, &q->fl[0].sdesc, 2240169978Skmacy &q->fl[0].desc_tag, &q->fl[0].desc_map, 2241169978Skmacy sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { 2242169978Skmacy printf("error %d from alloc ring fl0\n", ret); 2243169978Skmacy goto err; 2244169978Skmacy } 2245169978Skmacy 2246169978Skmacy if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), 2247169978Skmacy sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, 2248169978Skmacy &q->fl[1].desc, &q->fl[1].sdesc, 2249169978Skmacy &q->fl[1].desc_tag, &q->fl[1].desc_map, 2250169978Skmacy sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { 2251169978Skmacy printf("error %d from alloc ring fl1\n", ret); 2252169978Skmacy goto err; 2253169978Skmacy } 2254169978Skmacy 2255169978Skmacy if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, 2256169978Skmacy &q->rspq.phys_addr, &q->rspq.desc, NULL, 2257169978Skmacy &q->rspq.desc_tag, &q->rspq.desc_map, 2258169978Skmacy NULL, NULL)) != 0) { 2259169978Skmacy printf("error %d from alloc ring rspq\n", ret); 2260169978Skmacy goto err; 2261169978Skmacy } 2262169978Skmacy 2263169978Skmacy for (i = 0; i < ntxq; ++i) { 2264169978Skmacy /* 2265169978Skmacy * The control queue always uses immediate data so does not 2266169978Skmacy * need to keep track of any mbufs. 2267169978Skmacy * XXX Placeholder for future TOE support. 2268169978Skmacy */ 2269169978Skmacy size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); 2270169978Skmacy 2271169978Skmacy if ((ret = alloc_ring(sc, p->txq_size[i], 2272169978Skmacy sizeof(struct tx_desc), sz, 2273169978Skmacy &q->txq[i].phys_addr, &q->txq[i].desc, 2274169978Skmacy &q->txq[i].sdesc, &q->txq[i].desc_tag, 2275169978Skmacy &q->txq[i].desc_map, 2276169978Skmacy sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { 2277169978Skmacy printf("error %d from alloc ring tx %i\n", ret, i); 2278169978Skmacy goto err; 2279169978Skmacy } 2280169978Skmacy mbufq_init(&q->txq[i].sendq); 2281169978Skmacy q->txq[i].gen = 1; 2282169978Skmacy q->txq[i].size = p->txq_size[i]; 2283170869Skmacy snprintf(q->txq[i].lockbuf, TXQ_NAME_LEN, "t3 txq lock %d:%d:%d", 2284170869Skmacy device_get_unit(sc->dev), irq_vec_idx, i); 2285170869Skmacy MTX_INIT(&q->txq[i].lock, q->txq[i].lockbuf, NULL, MTX_DEF); 2286169978Skmacy } 2287169978Skmacy 2288171335Skmacy q->txq[TXQ_ETH].port = pi; 2289169978Skmacy 2290171335Skmacy TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q); 2291171335Skmacy TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q); 2292171469Skmacy TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, &q->txq[TXQ_ETH]); 2293171469Skmacy TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, &q->txq[TXQ_OFLD]); 2294171335Skmacy 2295169978Skmacy q->fl[0].gen = q->fl[1].gen = 1; 2296169978Skmacy q->fl[0].size = p->fl_size; 2297169978Skmacy q->fl[1].size = p->jumbo_size; 2298169978Skmacy 2299169978Skmacy q->rspq.gen = 1; 2300171471Skmacy q->rspq.cidx = 0; 2301169978Skmacy q->rspq.size = p->rspq_size; 2302170869Skmacy 2303175200Skmacy 2304175200Skmacy header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t); 2305169978Skmacy q->txq[TXQ_ETH].stop_thres = nports * 2306169978Skmacy flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); 2307169978Skmacy 2308175200Skmacy q->fl[0].buf_size = (MCLBYTES - header_size); 2309169978Skmacy q->fl[0].zone = zone_clust; 2310169978Skmacy q->fl[0].type = EXT_CLUSTER; 2311174708Skmacy#if __FreeBSD_version > 800000 2312175200Skmacy if (cxgb_use_16k_clusters) { 2313175200Skmacy q->fl[1].buf_size = MJUM16BYTES - header_size; 2314174708Skmacy q->fl[1].zone = zone_jumbo16; 2315174708Skmacy q->fl[1].type = EXT_JUMBO16; 2316175200Skmacy } else { 2317175200Skmacy q->fl[1].buf_size = MJUM9BYTES - header_size; 2318175200Skmacy q->fl[1].zone = zone_jumbo9; 2319175200Skmacy q->fl[1].type = EXT_JUMBO9; 2320175200Skmacy } 2321174708Skmacy#else 2322175200Skmacy q->fl[1].buf_size = MJUMPAGESIZE - header_size; 2323175200Skmacy q->fl[1].zone = zone_jumbop; 2324175200Skmacy q->fl[1].type = EXT_JUMBOP; 2325174708Skmacy#endif 2326169978Skmacy q->lro.enabled = lro_default; 2327171978Skmacy 2328176472Skmacy mtx_lock_spin(&sc->sge.reg_lock); 2329169978Skmacy ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, 2330169978Skmacy q->rspq.phys_addr, q->rspq.size, 2331169978Skmacy q->fl[0].buf_size, 1, 0); 2332169978Skmacy if (ret) { 2333169978Skmacy printf("error %d from t3_sge_init_rspcntxt\n", ret); 2334169978Skmacy goto err_unlock; 2335169978Skmacy } 2336169978Skmacy 2337169978Skmacy for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 2338169978Skmacy ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, 2339169978Skmacy q->fl[i].phys_addr, q->fl[i].size, 2340169978Skmacy q->fl[i].buf_size, p->cong_thres, 1, 2341169978Skmacy 0); 2342169978Skmacy if (ret) { 2343169978Skmacy printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); 2344169978Skmacy goto err_unlock; 2345169978Skmacy } 2346169978Skmacy } 2347169978Skmacy 2348169978Skmacy ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, 2349169978Skmacy SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, 2350169978Skmacy q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, 2351169978Skmacy 1, 0); 2352169978Skmacy if (ret) { 2353169978Skmacy printf("error %d from t3_sge_init_ecntxt\n", ret); 2354169978Skmacy goto err_unlock; 2355169978Skmacy } 2356169978Skmacy 2357169978Skmacy if (ntxq > 1) { 2358169978Skmacy ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, 2359169978Skmacy USE_GTS, SGE_CNTXT_OFLD, id, 2360169978Skmacy q->txq[TXQ_OFLD].phys_addr, 2361169978Skmacy q->txq[TXQ_OFLD].size, 0, 1, 0); 2362169978Skmacy if (ret) { 2363169978Skmacy printf("error %d from t3_sge_init_ecntxt\n", ret); 2364169978Skmacy goto err_unlock; 2365169978Skmacy } 2366169978Skmacy } 2367169978Skmacy 2368169978Skmacy if (ntxq > 2) { 2369169978Skmacy ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, 2370169978Skmacy SGE_CNTXT_CTRL, id, 2371169978Skmacy q->txq[TXQ_CTRL].phys_addr, 2372169978Skmacy q->txq[TXQ_CTRL].size, 2373169978Skmacy q->txq[TXQ_CTRL].token, 1, 0); 2374169978Skmacy if (ret) { 2375169978Skmacy printf("error %d from t3_sge_init_ecntxt\n", ret); 2376169978Skmacy goto err_unlock; 2377169978Skmacy } 2378169978Skmacy } 2379169978Skmacy 2380170869Skmacy snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", 2381170869Skmacy device_get_unit(sc->dev), irq_vec_idx); 2382170869Skmacy MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); 2383170869Skmacy 2384176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2385169978Skmacy t3_update_qset_coalesce(q, p); 2386169978Skmacy q->port = pi; 2387169978Skmacy 2388169978Skmacy refill_fl(sc, &q->fl[0], q->fl[0].size); 2389169978Skmacy refill_fl(sc, &q->fl[1], q->fl[1].size); 2390169978Skmacy refill_rspq(sc, &q->rspq, q->rspq.size - 1); 2391169978Skmacy 2392169978Skmacy t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | 2393169978Skmacy V_NEWTIMER(q->rspq.holdoff_tmr)); 2394169978Skmacy 2395169978Skmacy return (0); 2396169978Skmacy 2397169978Skmacyerr_unlock: 2398176472Skmacy mtx_unlock_spin(&sc->sge.reg_lock); 2399169978Skmacyerr: 2400169978Skmacy t3_free_qset(sc, q); 2401169978Skmacy 2402169978Skmacy return (ret); 2403169978Skmacy} 2404169978Skmacy 2405167514Skmacyvoid 2406171978Skmacyt3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad) 2407167514Skmacy{ 2408170654Skmacy struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad); 2409171978Skmacy struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; 2410167514Skmacy struct ifnet *ifp = pi->ifp; 2411167514Skmacy 2412170654Skmacy DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff); 2413167514Skmacy 2414167514Skmacy if ((ifp->if_capenable & IFCAP_RXCSUM) && !cpl->fragment && 2415167514Skmacy cpl->csum_valid && cpl->csum == 0xffff) { 2416167514Skmacy m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); 2417167514Skmacy rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; 2418167514Skmacy m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID|CSUM_DATA_VALID|CSUM_PSEUDO_HDR); 2419167514Skmacy m->m_pkthdr.csum_data = 0xffff; 2420167514Skmacy } 2421167514Skmacy /* 2422167514Skmacy * XXX need to add VLAN support for 6.x 2423167514Skmacy */ 2424167514Skmacy#ifdef VLAN_SUPPORTED 2425167514Skmacy if (__predict_false(cpl->vlan_valid)) { 2426167514Skmacy m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); 2427167514Skmacy m->m_flags |= M_VLANTAG; 2428167514Skmacy } 2429167514Skmacy#endif 2430168737Skmacy 2431167514Skmacy m->m_pkthdr.rcvif = ifp; 2432170654Skmacy m->m_pkthdr.header = mtod(m, uint8_t *) + sizeof(*cpl) + ethpad; 2433174708Skmacy#ifndef DISABLE_MBUF_IOVEC 2434168491Skmacy m_explode(m); 2435174708Skmacy#endif 2436168737Skmacy /* 2437168737Skmacy * adjust after conversion to mbuf chain 2438168737Skmacy */ 2439174708Skmacy m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); 2440174708Skmacy m->m_len -= (sizeof(*cpl) + ethpad); 2441174708Skmacy m->m_data += (sizeof(*cpl) + ethpad); 2442168351Skmacy 2443167514Skmacy (*ifp->if_input)(ifp, m); 2444167514Skmacy} 2445167514Skmacy 2446175200Skmacystatic void 2447176472Skmacyext_free_handler(void *arg1, void * arg2) 2448175200Skmacy{ 2449176472Skmacy uintptr_t type = (uintptr_t)arg2; 2450175200Skmacy uma_zone_t zone; 2451175200Skmacy struct mbuf *m; 2452175200Skmacy 2453176472Skmacy m = arg1; 2454175200Skmacy zone = m_getzonefromtype(type); 2455175200Skmacy m->m_ext.ext_type = (int)type; 2456175200Skmacy cxgb_ext_freed++; 2457176472Skmacy cxgb_cache_put(zone, m); 2458175200Skmacy} 2459175200Skmacy 2460175200Skmacystatic void 2461175200Skmacyinit_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone) 2462175200Skmacy{ 2463175200Skmacy struct mbuf *m; 2464175200Skmacy int header_size; 2465175200Skmacy 2466175302Skmacy header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + 2467175302Skmacy sizeof(struct m_ext_) + sizeof(uint32_t); 2468175200Skmacy 2469175200Skmacy bzero(cl, header_size); 2470175200Skmacy m = (struct mbuf *)cl; 2471176472Skmacy 2472176472Skmacy cxgb_ext_inited++; 2473175200Skmacy SLIST_INIT(&m->m_pkthdr.tags); 2474175200Skmacy m->m_type = MT_DATA; 2475175200Skmacy m->m_flags = flags | M_NOFREE | M_EXT; 2476175200Skmacy m->m_data = cl + header_size; 2477175200Skmacy m->m_ext.ext_buf = cl; 2478175200Skmacy m->m_ext.ref_cnt = (uint32_t *)(cl + header_size - sizeof(uint32_t)); 2479175200Skmacy m->m_ext.ext_size = m_getsizefromtype(type); 2480175200Skmacy m->m_ext.ext_free = ext_free_handler; 2481175872Sphk m->m_ext.ext_arg1 = cl; 2482175872Sphk m->m_ext.ext_arg2 = (void *)(uintptr_t)type; 2483175200Skmacy m->m_ext.ext_type = EXT_EXTREF; 2484175200Skmacy *(m->m_ext.ref_cnt) = 1; 2485175200Skmacy DPRINTF("data=%p ref_cnt=%p\n", m->m_data, m->m_ext.ref_cnt); 2486175200Skmacy} 2487175200Skmacy 2488175200Skmacy 2489167514Skmacy/** 2490167514Skmacy * get_packet - return the next ingress packet buffer from a free list 2491167514Skmacy * @adap: the adapter that received the packet 2492167514Skmacy * @drop_thres: # of remaining buffers before we start dropping packets 2493167514Skmacy * @qs: the qset that the SGE free list holding the packet belongs to 2494167514Skmacy * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain 2495167514Skmacy * @r: response descriptor 2496167514Skmacy * 2497167514Skmacy * Get the next packet from a free list and complete setup of the 2498167514Skmacy * sk_buff. If the packet is small we make a copy and recycle the 2499167514Skmacy * original buffer, otherwise we use the original buffer itself. If a 2500167514Skmacy * positive drop threshold is supplied packets are dropped and their 2501167514Skmacy * buffers recycled if (a) the number of remaining buffers is under the 2502167514Skmacy * threshold and the packet is too big to copy, or (b) the packet should 2503167514Skmacy * be copied but there is no memory for the copy. 2504167514Skmacy */ 2505172101Skmacy#ifdef DISABLE_MBUF_IOVEC 2506172101Skmacy 2507167514Skmacystatic int 2508167514Skmacyget_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2509175340Skmacy struct t3_mbuf_hdr *mh, struct rsp_desc *r) 2510172101Skmacy{ 2511172101Skmacy 2512172101Skmacy unsigned int len_cq = ntohl(r->len_cq); 2513172101Skmacy struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2514172101Skmacy struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; 2515172101Skmacy uint32_t len = G_RSPD_LEN(len_cq); 2516172101Skmacy uint32_t flags = ntohl(r->flags); 2517172101Skmacy uint8_t sopeop = G_RSPD_SOP_EOP(flags); 2518175711Skmacy caddr_t cl; 2519175711Skmacy struct mbuf *m, *m0; 2520172101Skmacy int ret = 0; 2521175711Skmacy 2522174708Skmacy prefetch(sd->rxsd_cl); 2523172101Skmacy 2524172101Skmacy fl->credits--; 2525172101Skmacy bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); 2526175200Skmacy 2527175200Skmacy if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { 2528175200Skmacy if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) 2529175200Skmacy goto skip_recycle; 2530175200Skmacy cl = mtod(m0, void *); 2531175200Skmacy memcpy(cl, sd->data, len); 2532175200Skmacy recycle_rx_buf(adap, fl, fl->cidx); 2533175340Skmacy m = m0; 2534175711Skmacy m0->m_len = len; 2535175200Skmacy } else { 2536175200Skmacy skip_recycle: 2537175711Skmacy 2538175200Skmacy bus_dmamap_unload(fl->entry_tag, sd->map); 2539175200Skmacy cl = sd->rxsd_cl; 2540175340Skmacy m = m0 = (struct mbuf *)cl; 2541172101Skmacy 2542175200Skmacy if ((sopeop == RSPQ_SOP_EOP) || 2543175200Skmacy (sopeop == RSPQ_SOP)) 2544175200Skmacy flags = M_PKTHDR; 2545175200Skmacy init_cluster_mbuf(cl, flags, fl->type, fl->zone); 2546175711Skmacy m0->m_len = len; 2547175200Skmacy } 2548172101Skmacy switch(sopeop) { 2549172101Skmacy case RSPQ_SOP_EOP: 2550172101Skmacy DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); 2551172101Skmacy mh->mh_head = mh->mh_tail = m; 2552172101Skmacy m->m_pkthdr.len = len; 2553172101Skmacy ret = 1; 2554172101Skmacy break; 2555172101Skmacy case RSPQ_NSOP_NEOP: 2556172101Skmacy DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); 2557172101Skmacy if (mh->mh_tail == NULL) { 2558175711Skmacy log(LOG_ERR, "discarding intermediate descriptor entry\n"); 2559172101Skmacy m_freem(m); 2560172101Skmacy break; 2561172101Skmacy } 2562172101Skmacy mh->mh_tail->m_next = m; 2563172101Skmacy mh->mh_tail = m; 2564172101Skmacy mh->mh_head->m_pkthdr.len += len; 2565172101Skmacy ret = 0; 2566172101Skmacy break; 2567172101Skmacy case RSPQ_SOP: 2568172101Skmacy DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); 2569172101Skmacy m->m_pkthdr.len = len; 2570172101Skmacy mh->mh_head = mh->mh_tail = m; 2571172101Skmacy ret = 0; 2572172101Skmacy break; 2573172101Skmacy case RSPQ_EOP: 2574172101Skmacy DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); 2575172101Skmacy mh->mh_head->m_pkthdr.len += len; 2576172101Skmacy mh->mh_tail->m_next = m; 2577172101Skmacy mh->mh_tail = m; 2578172101Skmacy ret = 1; 2579172101Skmacy break; 2580172101Skmacy } 2581172101Skmacy if (++fl->cidx == fl->size) 2582172101Skmacy fl->cidx = 0; 2583172101Skmacy 2584172101Skmacy return (ret); 2585172101Skmacy} 2586172101Skmacy 2587172101Skmacy#else 2588174708Skmacy 2589172101Skmacystatic int 2590172101Skmacyget_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2591174708Skmacy struct mbuf **m, struct rsp_desc *r) 2592167514Skmacy{ 2593167514Skmacy 2594167514Skmacy unsigned int len_cq = ntohl(r->len_cq); 2595167514Skmacy struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2596167514Skmacy struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; 2597167514Skmacy uint32_t len = G_RSPD_LEN(len_cq); 2598167514Skmacy uint32_t flags = ntohl(r->flags); 2599167514Skmacy uint8_t sopeop = G_RSPD_SOP_EOP(flags); 2600169978Skmacy void *cl; 2601167514Skmacy int ret = 0; 2602174708Skmacy struct mbuf *m0; 2603174708Skmacy#if 0 2604174708Skmacy if ((sd + 1 )->rxsd_cl) 2605174708Skmacy prefetch((sd + 1)->rxsd_cl); 2606174708Skmacy if ((sd + 2)->rxsd_cl) 2607174708Skmacy prefetch((sd + 2)->rxsd_cl); 2608174708Skmacy#endif 2609174708Skmacy DPRINTF("rx cpu=%d\n", curcpu); 2610167514Skmacy fl->credits--; 2611168351Skmacy bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); 2612168351Skmacy 2613169978Skmacy if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { 2614174708Skmacy if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) 2615174708Skmacy goto skip_recycle; 2616174708Skmacy cl = mtod(m0, void *); 2617174708Skmacy memcpy(cl, sd->data, len); 2618169978Skmacy recycle_rx_buf(adap, fl, fl->cidx); 2619174708Skmacy *m = m0; 2620169978Skmacy } else { 2621174708Skmacy skip_recycle: 2622169978Skmacy bus_dmamap_unload(fl->entry_tag, sd->map); 2623174708Skmacy cl = sd->rxsd_cl; 2624174708Skmacy *m = m0 = (struct mbuf *)cl; 2625169978Skmacy } 2626174708Skmacy 2627167514Skmacy switch(sopeop) { 2628167514Skmacy case RSPQ_SOP_EOP: 2629167514Skmacy DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); 2630174708Skmacy if (cl == sd->rxsd_cl) 2631174708Skmacy init_cluster_mbuf(cl, M_PKTHDR, fl->type, fl->zone); 2632174708Skmacy m0->m_len = m0->m_pkthdr.len = len; 2633167514Skmacy ret = 1; 2634168491Skmacy goto done; 2635167514Skmacy break; 2636167514Skmacy case RSPQ_NSOP_NEOP: 2637167514Skmacy DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); 2638174708Skmacy panic("chaining unsupported"); 2639167514Skmacy ret = 0; 2640167514Skmacy break; 2641167514Skmacy case RSPQ_SOP: 2642167514Skmacy DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); 2643174708Skmacy panic("chaining unsupported"); 2644174708Skmacy m_iovinit(m0); 2645167514Skmacy ret = 0; 2646167514Skmacy break; 2647167514Skmacy case RSPQ_EOP: 2648167514Skmacy DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); 2649174708Skmacy panic("chaining unsupported"); 2650167514Skmacy ret = 1; 2651167514Skmacy break; 2652167514Skmacy } 2653174708Skmacy panic("append not supported"); 2654174708Skmacy#if 0 2655174708Skmacy m_iovappend(m0, cl, fl->buf_size, len, sizeof(uint32_t), sd->rxsd_ref); 2656174708Skmacy#endif 2657168491Skmacydone: 2658167514Skmacy if (++fl->cidx == fl->size) 2659167514Skmacy fl->cidx = 0; 2660167514Skmacy 2661167514Skmacy return (ret); 2662167514Skmacy} 2663172101Skmacy#endif 2664167514Skmacy/** 2665167514Skmacy * handle_rsp_cntrl_info - handles control information in a response 2666167514Skmacy * @qs: the queue set corresponding to the response 2667167514Skmacy * @flags: the response control flags 2668167514Skmacy * 2669167514Skmacy * Handles the control information of an SGE response, such as GTS 2670167514Skmacy * indications and completion credits for the queue set's Tx queues. 2671167514Skmacy * HW coalesces credits, we don't do any extra SW coalescing. 2672167514Skmacy */ 2673167514Skmacystatic __inline void 2674167514Skmacyhandle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags) 2675167514Skmacy{ 2676167514Skmacy unsigned int credits; 2677167514Skmacy 2678167514Skmacy#if USE_GTS 2679167514Skmacy if (flags & F_RSPD_TXQ0_GTS) 2680167514Skmacy clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); 2681167514Skmacy#endif 2682167514Skmacy credits = G_RSPD_TXQ0_CR(flags); 2683175200Skmacy if (credits) 2684167514Skmacy qs->txq[TXQ_ETH].processed += credits; 2685167527Skmacy 2686167514Skmacy credits = G_RSPD_TXQ2_CR(flags); 2687167527Skmacy if (credits) 2688167514Skmacy qs->txq[TXQ_CTRL].processed += credits; 2689167514Skmacy 2690167514Skmacy# if USE_GTS 2691167514Skmacy if (flags & F_RSPD_TXQ1_GTS) 2692167514Skmacy clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); 2693167514Skmacy# endif 2694167514Skmacy credits = G_RSPD_TXQ1_CR(flags); 2695167514Skmacy if (credits) 2696167514Skmacy qs->txq[TXQ_OFLD].processed += credits; 2697174708Skmacy 2698167514Skmacy} 2699167514Skmacy 2700167514Skmacystatic void 2701167514Skmacycheck_ring_db(adapter_t *adap, struct sge_qset *qs, 2702167514Skmacy unsigned int sleeping) 2703167514Skmacy{ 2704167514Skmacy ; 2705167514Skmacy} 2706167514Skmacy 2707167514Skmacy/** 2708167514Skmacy * process_responses - process responses from an SGE response queue 2709167514Skmacy * @adap: the adapter 2710167514Skmacy * @qs: the queue set to which the response queue belongs 2711167514Skmacy * @budget: how many responses can be processed in this round 2712167514Skmacy * 2713167514Skmacy * Process responses from an SGE response queue up to the supplied budget. 2714167514Skmacy * Responses include received packets as well as credits and other events 2715167514Skmacy * for the queues that belong to the response queue's queue set. 2716167514Skmacy * A negative budget is effectively unlimited. 2717167514Skmacy * 2718167514Skmacy * Additionally choose the interrupt holdoff time for the next interrupt 2719167514Skmacy * on this queue. If the system is under memory shortage use a fairly 2720167514Skmacy * long delay to help recovery. 2721167514Skmacy */ 2722174708Skmacyint 2723167514Skmacyprocess_responses(adapter_t *adap, struct sge_qset *qs, int budget) 2724167514Skmacy{ 2725167514Skmacy struct sge_rspq *rspq = &qs->rspq; 2726167514Skmacy struct rsp_desc *r = &rspq->desc[rspq->cidx]; 2727167514Skmacy int budget_left = budget; 2728167514Skmacy unsigned int sleeping = 0; 2729167514Skmacy int lro = qs->lro.enabled; 2730169978Skmacy struct mbuf *offload_mbufs[RX_BUNDLE_SIZE]; 2731169978Skmacy int ngathered = 0; 2732167514Skmacy#ifdef DEBUG 2733167514Skmacy static int last_holdoff = 0; 2734171471Skmacy if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { 2735167514Skmacy printf("next_holdoff=%d\n", rspq->holdoff_tmr); 2736167514Skmacy last_holdoff = rspq->holdoff_tmr; 2737167514Skmacy } 2738169978Skmacy#endif 2739167514Skmacy rspq->next_holdoff = rspq->holdoff_tmr; 2740167514Skmacy 2741167514Skmacy while (__predict_true(budget_left && is_new_response(r, rspq))) { 2742167514Skmacy int eth, eop = 0, ethpad = 0; 2743167514Skmacy uint32_t flags = ntohl(r->flags); 2744167514Skmacy uint32_t rss_csum = *(const uint32_t *)r; 2745174708Skmacy uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val); 2746167514Skmacy 2747167514Skmacy eth = (r->rss_hdr.opcode == CPL_RX_PKT); 2748167514Skmacy 2749167514Skmacy if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) { 2750176472Skmacy struct mbuf *m; 2751167514Skmacy 2752176472Skmacy if (cxgb_debug) 2753176472Skmacy printf("async notification\n"); 2754176472Skmacy 2755176472Skmacy if (rspq->rspq_mh.mh_head == NULL) { 2756176472Skmacy rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); 2757176472Skmacy m = rspq->rspq_mh.mh_head; 2758176472Skmacy } else { 2759176472Skmacy m = m_gethdr(M_DONTWAIT, MT_DATA); 2760176472Skmacy } 2761176472Skmacy 2762176472Skmacy /* XXX m is lost here if rspq->rspq_mbuf is not NULL */ 2763176472Skmacy 2764176472Skmacy if (m == NULL) 2765176472Skmacy goto no_mem; 2766176472Skmacy 2767176472Skmacy memcpy(mtod(m, char *), r, AN_PKT_SIZE); 2768176472Skmacy m->m_len = m->m_pkthdr.len = AN_PKT_SIZE; 2769176472Skmacy *mtod(m, char *) = CPL_ASYNC_NOTIF; 2770176472Skmacy rss_csum = htonl(CPL_ASYNC_NOTIF << 24); 2771176472Skmacy eop = 1; 2772176472Skmacy rspq->async_notif++; 2773176472Skmacy goto skip; 2774167514Skmacy } else if (flags & F_RSPD_IMM_DATA_VALID) { 2775175200Skmacy struct mbuf *m = NULL; 2776175200Skmacy 2777175200Skmacy DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", 2778175200Skmacy r->rss_hdr.opcode, rspq->cidx); 2779175711Skmacy if (rspq->rspq_mh.mh_head == NULL) 2780175711Skmacy rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); 2781174708Skmacy else 2782171471Skmacy m = m_gethdr(M_DONTWAIT, MT_DATA); 2783168491Skmacy 2784176472Skmacy if (rspq->rspq_mh.mh_head == NULL && m == NULL) { 2785176472Skmacy no_mem: 2786167514Skmacy rspq->next_holdoff = NOMEM_INTR_DELAY; 2787167514Skmacy budget_left--; 2788167514Skmacy break; 2789167514Skmacy } 2790176472Skmacy get_imm_packet(adap, r, rspq->rspq_mh.mh_head); 2791168491Skmacy eop = 1; 2792174708Skmacy rspq->imm_data++; 2793176472Skmacy } else if (r->len_cq) { 2794167514Skmacy int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; 2795172101Skmacy 2796172101Skmacy#ifdef DISABLE_MBUF_IOVEC 2797175340Skmacy eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r); 2798172101Skmacy#else 2799174708Skmacy eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r); 2800176472Skmacy#endif 2801174708Skmacy#ifdef IFNET_MULTIQUEUE 2802176472Skmacy rspq->rspq_mh.mh_head->m_pkthdr.rss_hash = rss_hash; 2803174708Skmacy#endif 2804167514Skmacy ethpad = 2; 2805167514Skmacy } else { 2806167514Skmacy DPRINTF("pure response\n"); 2807167514Skmacy rspq->pure_rsps++; 2808167514Skmacy } 2809176472Skmacy skip: 2810167514Skmacy if (flags & RSPD_CTRL_MASK) { 2811167514Skmacy sleeping |= flags & RSPD_GTS_MASK; 2812167514Skmacy handle_rsp_cntrl_info(qs, flags); 2813167514Skmacy } 2814174708Skmacy 2815167514Skmacy r++; 2816167514Skmacy if (__predict_false(++rspq->cidx == rspq->size)) { 2817167514Skmacy rspq->cidx = 0; 2818167514Skmacy rspq->gen ^= 1; 2819167514Skmacy r = rspq->desc; 2820167514Skmacy } 2821167514Skmacy prefetch(r); 2822167514Skmacy if (++rspq->credits >= (rspq->size / 4)) { 2823167514Skmacy refill_rspq(adap, rspq, rspq->credits); 2824167514Skmacy rspq->credits = 0; 2825167514Skmacy } 2826174708Skmacy DPRINTF("eth=%d eop=%d flags=0x%x\n", eth, eop, flags); 2827174708Skmacy 2828174708Skmacy if (!eth && eop) { 2829174708Skmacy rspq->rspq_mh.mh_head->m_pkthdr.csum_data = rss_csum; 2830174708Skmacy /* 2831174708Skmacy * XXX size mismatch 2832174708Skmacy */ 2833174708Skmacy m_set_priority(rspq->rspq_mh.mh_head, rss_hash); 2834176472Skmacy 2835176472Skmacy 2836174708Skmacy ngathered = rx_offload(&adap->tdev, rspq, 2837174708Skmacy rspq->rspq_mh.mh_head, offload_mbufs, ngathered); 2838174708Skmacy rspq->rspq_mh.mh_head = NULL; 2839174708Skmacy DPRINTF("received offload packet\n"); 2840174708Skmacy 2841174708Skmacy } else if (eth && eop) { 2842172101Skmacy prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *)); 2843174708Skmacy prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES); 2844167514Skmacy 2845174708Skmacy t3_rx_eth_lro(adap, rspq, rspq->rspq_mh.mh_head, ethpad, 2846174708Skmacy rss_hash, rss_csum, lro); 2847174708Skmacy DPRINTF("received tunnel packet\n"); 2848172101Skmacy rspq->rspq_mh.mh_head = NULL; 2849171469Skmacy 2850167514Skmacy } 2851174708Skmacy __refill_fl_lt(adap, &qs->fl[0], 32); 2852174708Skmacy __refill_fl_lt(adap, &qs->fl[1], 32); 2853167514Skmacy --budget_left; 2854167514Skmacy } 2855167514Skmacy 2856169978Skmacy deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered); 2857169978Skmacy t3_lro_flush(adap, qs, &qs->lro); 2858169978Skmacy 2859167514Skmacy if (sleeping) 2860167514Skmacy check_ring_db(adap, qs, sleeping); 2861167514Skmacy 2862167514Skmacy smp_mb(); /* commit Tx queue processed updates */ 2863175200Skmacy if (__predict_false(qs->txq_stopped > 1)) { 2864174708Skmacy printf("restarting tx on %p\n", qs); 2865174708Skmacy 2866167514Skmacy restart_tx(qs); 2867174708Skmacy } 2868174708Skmacy 2869174708Skmacy __refill_fl_lt(adap, &qs->fl[0], 512); 2870174708Skmacy __refill_fl_lt(adap, &qs->fl[1], 512); 2871167514Skmacy budget -= budget_left; 2872167514Skmacy return (budget); 2873167514Skmacy} 2874167514Skmacy 2875167514Skmacy/* 2876167514Skmacy * A helper function that processes responses and issues GTS. 2877167514Skmacy */ 2878167514Skmacystatic __inline int 2879167514Skmacyprocess_responses_gts(adapter_t *adap, struct sge_rspq *rq) 2880167514Skmacy{ 2881167514Skmacy int work; 2882167514Skmacy static int last_holdoff = 0; 2883167514Skmacy 2884167514Skmacy work = process_responses(adap, rspq_to_qset(rq), -1); 2885167514Skmacy 2886167514Skmacy if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { 2887167514Skmacy printf("next_holdoff=%d\n", rq->next_holdoff); 2888167514Skmacy last_holdoff = rq->next_holdoff; 2889167514Skmacy } 2890175223Skmacy t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | 2891175223Skmacy V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); 2892175223Skmacy 2893175223Skmacy return (work); 2894167514Skmacy} 2895167514Skmacy 2896167514Skmacy 2897167514Skmacy/* 2898167514Skmacy * Interrupt handler for legacy INTx interrupts for T3B-based cards. 2899167514Skmacy * Handles data events from SGE response queues as well as error and other 2900167514Skmacy * async events as they all use the same interrupt pin. We use one SGE 2901167514Skmacy * response queue per port in this mode and protect all response queues with 2902167514Skmacy * queue 0's lock. 2903167514Skmacy */ 2904167514Skmacyvoid 2905167514Skmacyt3b_intr(void *data) 2906167514Skmacy{ 2907171978Skmacy uint32_t i, map; 2908167514Skmacy adapter_t *adap = data; 2909167514Skmacy struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 2910167514Skmacy 2911167514Skmacy t3_write_reg(adap, A_PL_CLI, 0); 2912167514Skmacy map = t3_read_reg(adap, A_SG_DATA_INTR); 2913167514Skmacy 2914167514Skmacy if (!map) 2915167514Skmacy return; 2916167514Skmacy 2917167514Skmacy if (__predict_false(map & F_ERRINTR)) 2918167514Skmacy taskqueue_enqueue(adap->tq, &adap->slow_intr_task); 2919172096Skmacy 2920167514Skmacy mtx_lock(&q0->lock); 2921171978Skmacy for_each_port(adap, i) 2922171978Skmacy if (map & (1 << i)) 2923171978Skmacy process_responses_gts(adap, &adap->sge.qs[i].rspq); 2924167514Skmacy mtx_unlock(&q0->lock); 2925167514Skmacy} 2926167514Skmacy 2927167514Skmacy/* 2928167514Skmacy * The MSI interrupt handler. This needs to handle data events from SGE 2929167514Skmacy * response queues as well as error and other async events as they all use 2930167514Skmacy * the same MSI vector. We use one SGE response queue per port in this mode 2931167514Skmacy * and protect all response queues with queue 0's lock. 2932167514Skmacy */ 2933167514Skmacyvoid 2934167514Skmacyt3_intr_msi(void *data) 2935167514Skmacy{ 2936167514Skmacy adapter_t *adap = data; 2937167514Skmacy struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 2938171978Skmacy int i, new_packets = 0; 2939172096Skmacy 2940167514Skmacy mtx_lock(&q0->lock); 2941167514Skmacy 2942171978Skmacy for_each_port(adap, i) 2943171978Skmacy if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) 2944171978Skmacy new_packets = 1; 2945167514Skmacy mtx_unlock(&q0->lock); 2946167514Skmacy if (new_packets == 0) 2947167514Skmacy taskqueue_enqueue(adap->tq, &adap->slow_intr_task); 2948167514Skmacy} 2949167514Skmacy 2950167514Skmacyvoid 2951167514Skmacyt3_intr_msix(void *data) 2952167514Skmacy{ 2953167514Skmacy struct sge_qset *qs = data; 2954167514Skmacy adapter_t *adap = qs->port->adapter; 2955167514Skmacy struct sge_rspq *rspq = &qs->rspq; 2956175223Skmacy#ifndef IFNET_MULTIQUEUE 2957175223Skmacy mtx_lock(&rspq->lock); 2958175223Skmacy#else 2959175223Skmacy if (mtx_trylock(&rspq->lock)) 2960175223Skmacy#endif 2961175223Skmacy { 2962175223Skmacy 2963174708Skmacy if (process_responses_gts(adap, rspq) == 0) 2964174708Skmacy rspq->unhandled_irqs++; 2965174708Skmacy mtx_unlock(&rspq->lock); 2966174708Skmacy } 2967167514Skmacy} 2968167514Skmacy 2969175200Skmacy#define QDUMP_SBUF_SIZE 32 * 400 2970175209Skmacystatic int 2971175209Skmacyt3_dump_rspq(SYSCTL_HANDLER_ARGS) 2972175209Skmacy{ 2973175209Skmacy struct sge_rspq *rspq; 2974175223Skmacy struct sge_qset *qs; 2975175209Skmacy int i, err, dump_end, idx; 2976175209Skmacy static int multiplier = 1; 2977175209Skmacy struct sbuf *sb; 2978175209Skmacy struct rsp_desc *rspd; 2979175223Skmacy uint32_t data[4]; 2980175209Skmacy 2981175209Skmacy rspq = arg1; 2982175223Skmacy qs = rspq_to_qset(rspq); 2983175209Skmacy if (rspq->rspq_dump_count == 0) 2984175209Skmacy return (0); 2985175209Skmacy if (rspq->rspq_dump_count > RSPQ_Q_SIZE) { 2986175209Skmacy log(LOG_WARNING, 2987175209Skmacy "dump count is too large %d\n", rspq->rspq_dump_count); 2988175209Skmacy rspq->rspq_dump_count = 0; 2989175209Skmacy return (EINVAL); 2990175209Skmacy } 2991175209Skmacy if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) { 2992175209Skmacy log(LOG_WARNING, 2993175209Skmacy "dump start of %d is greater than queue size\n", 2994175209Skmacy rspq->rspq_dump_start); 2995175209Skmacy rspq->rspq_dump_start = 0; 2996175209Skmacy return (EINVAL); 2997175209Skmacy } 2998175223Skmacy err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data); 2999175223Skmacy if (err) 3000175223Skmacy return (err); 3001175209Skmacyretry_sbufops: 3002175209Skmacy sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3003175223Skmacy 3004175223Skmacy sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n", 3005175223Skmacy (data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f), 3006175223Skmacy ((data[2] >> 26) & 1), ((data[2] >> 27) & 1)); 3007175223Skmacy sbuf_printf(sb, " generation=%u CQ mode=%u FL threshold=%u\n", 3008175223Skmacy ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]); 3009175209Skmacy 3010175223Skmacy sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start, 3011175209Skmacy (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1)); 3012175223Skmacy 3013175209Skmacy dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count; 3014175209Skmacy for (i = rspq->rspq_dump_start; i < dump_end; i++) { 3015175209Skmacy idx = i & (RSPQ_Q_SIZE-1); 3016175209Skmacy 3017175209Skmacy rspd = &rspq->desc[idx]; 3018175209Skmacy sbuf_printf(sb, "\tidx=%04d opcode=%02x cpu_idx=%x hash_type=%x cq_idx=%x\n", 3019175209Skmacy idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx, 3020175209Skmacy rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx)); 3021175209Skmacy sbuf_printf(sb, "\trss_hash_val=%x flags=%08x len_cq=%x intr_gen=%x\n", 3022175209Skmacy rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags), 3023175209Skmacy be32toh(rspd->len_cq), rspd->intr_gen); 3024175209Skmacy } 3025175209Skmacy if (sbuf_overflowed(sb)) { 3026175209Skmacy sbuf_delete(sb); 3027175209Skmacy multiplier++; 3028175209Skmacy goto retry_sbufops; 3029175209Skmacy } 3030175209Skmacy sbuf_finish(sb); 3031175209Skmacy err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3032175209Skmacy sbuf_delete(sb); 3033175209Skmacy return (err); 3034175209Skmacy} 3035175209Skmacy 3036167514Skmacystatic int 3037176472Skmacyt3_dump_txq_eth(SYSCTL_HANDLER_ARGS) 3038175200Skmacy{ 3039175200Skmacy struct sge_txq *txq; 3040175200Skmacy struct sge_qset *qs; 3041175200Skmacy int i, j, err, dump_end; 3042175200Skmacy static int multiplier = 1; 3043175200Skmacy struct sbuf *sb; 3044175200Skmacy struct tx_desc *txd; 3045175200Skmacy uint32_t *WR, wr_hi, wr_lo, gen; 3046175223Skmacy uint32_t data[4]; 3047175200Skmacy 3048175200Skmacy txq = arg1; 3049175200Skmacy qs = txq_to_qset(txq, TXQ_ETH); 3050175200Skmacy if (txq->txq_dump_count == 0) { 3051175200Skmacy return (0); 3052175200Skmacy } 3053175200Skmacy if (txq->txq_dump_count > TX_ETH_Q_SIZE) { 3054175200Skmacy log(LOG_WARNING, 3055175200Skmacy "dump count is too large %d\n", txq->txq_dump_count); 3056175200Skmacy txq->txq_dump_count = 1; 3057175200Skmacy return (EINVAL); 3058175200Skmacy } 3059175200Skmacy if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) { 3060175200Skmacy log(LOG_WARNING, 3061175200Skmacy "dump start of %d is greater than queue size\n", 3062175200Skmacy txq->txq_dump_start); 3063175200Skmacy txq->txq_dump_start = 0; 3064175200Skmacy return (EINVAL); 3065175200Skmacy } 3066176472Skmacy err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data); 3067175223Skmacy if (err) 3068175223Skmacy return (err); 3069175223Skmacy 3070175223Skmacy 3071175200Skmacyretry_sbufops: 3072175200Skmacy sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3073175223Skmacy 3074175223Skmacy sbuf_printf(sb, " \n credits=%u GTS=%u index=%u size=%u rspq#=%u cmdq#=%u\n", 3075175223Skmacy (data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16), 3076175223Skmacy (data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1)); 3077175223Skmacy sbuf_printf(sb, " TUN=%u TOE=%u generation%u uP token=%u valid=%u\n", 3078175223Skmacy ((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1), 3079175223Skmacy ((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1)); 3080175223Skmacy sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, 3081175200Skmacy txq->txq_dump_start, 3082175200Skmacy (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1)); 3083175200Skmacy 3084175200Skmacy dump_end = txq->txq_dump_start + txq->txq_dump_count; 3085175200Skmacy for (i = txq->txq_dump_start; i < dump_end; i++) { 3086175200Skmacy txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)]; 3087175200Skmacy WR = (uint32_t *)txd->flit; 3088175200Skmacy wr_hi = ntohl(WR[0]); 3089175200Skmacy wr_lo = ntohl(WR[1]); 3090175200Skmacy gen = G_WR_GEN(wr_lo); 3091175200Skmacy 3092175200Skmacy sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", 3093175200Skmacy wr_hi, wr_lo, gen); 3094175200Skmacy for (j = 2; j < 30; j += 4) 3095175200Skmacy sbuf_printf(sb, "\t%08x %08x %08x %08x \n", 3096175200Skmacy WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); 3097175200Skmacy 3098175200Skmacy } 3099175200Skmacy if (sbuf_overflowed(sb)) { 3100175200Skmacy sbuf_delete(sb); 3101175200Skmacy multiplier++; 3102175200Skmacy goto retry_sbufops; 3103175200Skmacy } 3104175200Skmacy sbuf_finish(sb); 3105175200Skmacy err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3106175200Skmacy sbuf_delete(sb); 3107175200Skmacy return (err); 3108175200Skmacy} 3109175200Skmacy 3110176472Skmacystatic int 3111176472Skmacyt3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS) 3112176472Skmacy{ 3113176472Skmacy struct sge_txq *txq; 3114176472Skmacy struct sge_qset *qs; 3115176472Skmacy int i, j, err, dump_end; 3116176472Skmacy static int multiplier = 1; 3117176472Skmacy struct sbuf *sb; 3118176472Skmacy struct tx_desc *txd; 3119176472Skmacy uint32_t *WR, wr_hi, wr_lo, gen; 3120176472Skmacy 3121176472Skmacy txq = arg1; 3122176472Skmacy qs = txq_to_qset(txq, TXQ_CTRL); 3123176472Skmacy if (txq->txq_dump_count == 0) { 3124176472Skmacy return (0); 3125176472Skmacy } 3126176472Skmacy if (txq->txq_dump_count > 256) { 3127176472Skmacy log(LOG_WARNING, 3128176472Skmacy "dump count is too large %d\n", txq->txq_dump_count); 3129176472Skmacy txq->txq_dump_count = 1; 3130176472Skmacy return (EINVAL); 3131176472Skmacy } 3132176472Skmacy if (txq->txq_dump_start > 255) { 3133176472Skmacy log(LOG_WARNING, 3134176472Skmacy "dump start of %d is greater than queue size\n", 3135176472Skmacy txq->txq_dump_start); 3136176472Skmacy txq->txq_dump_start = 0; 3137176472Skmacy return (EINVAL); 3138176472Skmacy } 3139175200Skmacy 3140176472Skmacyretry_sbufops: 3141176472Skmacy sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3142176472Skmacy sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, 3143176472Skmacy txq->txq_dump_start, 3144176472Skmacy (txq->txq_dump_start + txq->txq_dump_count) & 255); 3145176472Skmacy 3146176472Skmacy dump_end = txq->txq_dump_start + txq->txq_dump_count; 3147176472Skmacy for (i = txq->txq_dump_start; i < dump_end; i++) { 3148176472Skmacy txd = &txq->desc[i & (255)]; 3149176472Skmacy WR = (uint32_t *)txd->flit; 3150176472Skmacy wr_hi = ntohl(WR[0]); 3151176472Skmacy wr_lo = ntohl(WR[1]); 3152176472Skmacy gen = G_WR_GEN(wr_lo); 3153176472Skmacy 3154176472Skmacy sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", 3155176472Skmacy wr_hi, wr_lo, gen); 3156176472Skmacy for (j = 2; j < 30; j += 4) 3157176472Skmacy sbuf_printf(sb, "\t%08x %08x %08x %08x \n", 3158176472Skmacy WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); 3159176472Skmacy 3160176472Skmacy } 3161176472Skmacy if (sbuf_overflowed(sb)) { 3162176472Skmacy sbuf_delete(sb); 3163176472Skmacy multiplier++; 3164176472Skmacy goto retry_sbufops; 3165176472Skmacy } 3166176472Skmacy sbuf_finish(sb); 3167176472Skmacy err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3168176472Skmacy sbuf_delete(sb); 3169176472Skmacy return (err); 3170176472Skmacy} 3171176472Skmacy 3172175200Skmacystatic int 3173167514Skmacyt3_lro_enable(SYSCTL_HANDLER_ARGS) 3174167514Skmacy{ 3175167514Skmacy adapter_t *sc; 3176167514Skmacy int i, j, enabled, err, nqsets = 0; 3177168491Skmacy 3178168491Skmacy#ifndef LRO_WORKING 3179168491Skmacy return (0); 3180168491Skmacy#endif 3181167514Skmacy sc = arg1; 3182167514Skmacy enabled = sc->sge.qs[0].lro.enabled; 3183167514Skmacy err = sysctl_handle_int(oidp, &enabled, arg2, req); 3184167514Skmacy 3185170654Skmacy if (err != 0) 3186167514Skmacy return (err); 3187167514Skmacy if (enabled == sc->sge.qs[0].lro.enabled) 3188167514Skmacy return (0); 3189167514Skmacy 3190167514Skmacy for (i = 0; i < sc->params.nports; i++) 3191167514Skmacy for (j = 0; j < sc->port[i].nqsets; j++) 3192167514Skmacy nqsets++; 3193167514Skmacy 3194170654Skmacy for (i = 0; i < nqsets; i++) 3195167514Skmacy sc->sge.qs[i].lro.enabled = enabled; 3196167514Skmacy 3197167514Skmacy return (0); 3198167514Skmacy} 3199167514Skmacy 3200167514Skmacystatic int 3201167514Skmacyt3_set_coalesce_nsecs(SYSCTL_HANDLER_ARGS) 3202167514Skmacy{ 3203167514Skmacy adapter_t *sc = arg1; 3204167514Skmacy struct qset_params *qsp = &sc->params.sge.qset[0]; 3205167514Skmacy int coalesce_nsecs; 3206167514Skmacy struct sge_qset *qs; 3207167514Skmacy int i, j, err, nqsets = 0; 3208167514Skmacy struct mtx *lock; 3209174708Skmacy 3210174708Skmacy if ((sc->flags & FULL_INIT_DONE) == 0) 3211174708Skmacy return (ENXIO); 3212174708Skmacy 3213167514Skmacy coalesce_nsecs = qsp->coalesce_nsecs; 3214167514Skmacy err = sysctl_handle_int(oidp, &coalesce_nsecs, arg2, req); 3215167514Skmacy 3216167514Skmacy if (err != 0) { 3217167514Skmacy return (err); 3218167514Skmacy } 3219167514Skmacy if (coalesce_nsecs == qsp->coalesce_nsecs) 3220167514Skmacy return (0); 3221167514Skmacy 3222167514Skmacy for (i = 0; i < sc->params.nports; i++) 3223167514Skmacy for (j = 0; j < sc->port[i].nqsets; j++) 3224167514Skmacy nqsets++; 3225167514Skmacy 3226167514Skmacy coalesce_nsecs = max(100, coalesce_nsecs); 3227167514Skmacy 3228167514Skmacy for (i = 0; i < nqsets; i++) { 3229167514Skmacy qs = &sc->sge.qs[i]; 3230167514Skmacy qsp = &sc->params.sge.qset[i]; 3231167514Skmacy qsp->coalesce_nsecs = coalesce_nsecs; 3232167514Skmacy 3233167514Skmacy lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 3234167514Skmacy &sc->sge.qs[0].rspq.lock; 3235167514Skmacy 3236167514Skmacy mtx_lock(lock); 3237167514Skmacy t3_update_qset_coalesce(qs, qsp); 3238167514Skmacy t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | 3239167514Skmacy V_NEWTIMER(qs->rspq.holdoff_tmr)); 3240167514Skmacy mtx_unlock(lock); 3241167514Skmacy } 3242167514Skmacy 3243167514Skmacy return (0); 3244167514Skmacy} 3245167514Skmacy 3246167514Skmacy 3247167514Skmacyvoid 3248174708Skmacyt3_add_attach_sysctls(adapter_t *sc) 3249167514Skmacy{ 3250167514Skmacy struct sysctl_ctx_list *ctx; 3251167514Skmacy struct sysctl_oid_list *children; 3252174708Skmacy 3253167514Skmacy ctx = device_get_sysctl_ctx(sc->dev); 3254167514Skmacy children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 3255167514Skmacy 3256167514Skmacy /* random information */ 3257167514Skmacy SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 3258167514Skmacy "firmware_version", 3259167514Skmacy CTLFLAG_RD, &sc->fw_version, 3260167514Skmacy 0, "firmware version"); 3261167514Skmacy 3262167514Skmacy SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 3263167514Skmacy "enable_lro", 3264167514Skmacy CTLTYPE_INT|CTLFLAG_RW, sc, 3265167514Skmacy 0, t3_lro_enable, 3266167514Skmacy "I", "enable large receive offload"); 3267167514Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3268176472Skmacy "hw_revision", 3269176472Skmacy CTLFLAG_RD, &sc->params.rev, 3270176472Skmacy 0, "chip model"); 3271176472Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3272167514Skmacy "enable_debug", 3273167514Skmacy CTLFLAG_RW, &cxgb_debug, 3274167514Skmacy 0, "enable verbose debugging output"); 3275174708Skmacy SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tunq_coalesce", 3276174708Skmacy CTLFLAG_RD, &sc->tunq_coalesce, 3277174708Skmacy "#tunneled packets freed"); 3278168737Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3279171335Skmacy "txq_overrun", 3280171335Skmacy CTLFLAG_RD, &txq_fills, 3281171335Skmacy 0, "#times txq overrun"); 3282171471Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3283175415Skmacy "pcpu_cache_enable", 3284175504Skmacy CTLFLAG_RW, &cxgb_pcpu_cache_enable, 3285175415Skmacy 0, "#enable driver local pcpu caches"); 3286174708Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3287174708Skmacy "cache_alloc", 3288174708Skmacy CTLFLAG_RD, &cxgb_cached_allocations, 3289174708Skmacy 0, "#times a cluster was allocated from cache"); 3290174708Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3291174708Skmacy "cached", 3292174708Skmacy CTLFLAG_RD, &cxgb_cached, 3293174708Skmacy 0, "#times a cluster was cached"); 3294174708Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3295174708Skmacy "ext_freed", 3296174708Skmacy CTLFLAG_RD, &cxgb_ext_freed, 3297175347Skmacy 0, "#times a cluster was freed through ext_free"); 3298175347Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3299176472Skmacy "ext_inited", 3300176472Skmacy CTLFLAG_RD, &cxgb_ext_inited, 3301176472Skmacy 0, "#times a cluster was initialized for ext_free"); 3302176472Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3303175347Skmacy "mbufs_outstanding", 3304175375Skmacy CTLFLAG_RD, &cxgb_mbufs_outstanding, 3305175375Skmacy 0, "#mbufs in flight in the driver"); 3306175375Skmacy SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3307175375Skmacy "pack_outstanding", 3308175375Skmacy CTLFLAG_RD, &cxgb_pack_outstanding, 3309175375Skmacy 0, "#packet in flight in the driver"); 3310167514Skmacy} 3311167514Skmacy 3312175209Skmacy 3313175209Skmacystatic const char *rspq_name = "rspq"; 3314175209Skmacystatic const char *txq_names[] = 3315175209Skmacy{ 3316175209Skmacy "txq_eth", 3317175209Skmacy "txq_ofld", 3318175209Skmacy "txq_ctrl" 3319175209Skmacy}; 3320175209Skmacy 3321174708Skmacyvoid 3322174708Skmacyt3_add_configured_sysctls(adapter_t *sc) 3323174708Skmacy{ 3324174708Skmacy struct sysctl_ctx_list *ctx; 3325174708Skmacy struct sysctl_oid_list *children; 3326174708Skmacy int i, j; 3327174708Skmacy 3328174708Skmacy ctx = device_get_sysctl_ctx(sc->dev); 3329174708Skmacy children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 3330174708Skmacy 3331174708Skmacy SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 3332174708Skmacy "intr_coal", 3333174708Skmacy CTLTYPE_INT|CTLFLAG_RW, sc, 3334174708Skmacy 0, t3_set_coalesce_nsecs, 3335174708Skmacy "I", "interrupt coalescing timer (ns)"); 3336174708Skmacy 3337174708Skmacy for (i = 0; i < sc->params.nports; i++) { 3338174708Skmacy struct port_info *pi = &sc->port[i]; 3339174708Skmacy struct sysctl_oid *poid; 3340174708Skmacy struct sysctl_oid_list *poidlist; 3341174708Skmacy 3342174708Skmacy snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i); 3343174708Skmacy poid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, 3344174708Skmacy pi->namebuf, CTLFLAG_RD, NULL, "port statistics"); 3345174708Skmacy poidlist = SYSCTL_CHILDREN(poid); 3346174708Skmacy SYSCTL_ADD_INT(ctx, poidlist, OID_AUTO, 3347174708Skmacy "nqsets", CTLFLAG_RD, &pi->nqsets, 3348174708Skmacy 0, "#queue sets"); 3349174708Skmacy 3350174708Skmacy for (j = 0; j < pi->nqsets; j++) { 3351174708Skmacy struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j]; 3352176472Skmacy struct sysctl_oid *qspoid, *rspqpoid, *txqpoid, *ctrlqpoid; 3353176472Skmacy struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist, *ctrlqpoidlist; 3354174708Skmacy struct sge_txq *txq = &qs->txq[TXQ_ETH]; 3355174708Skmacy 3356174708Skmacy snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j); 3357174708Skmacy 3358174708Skmacy qspoid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, 3359174708Skmacy qs->namebuf, CTLFLAG_RD, NULL, "qset statistics"); 3360174708Skmacy qspoidlist = SYSCTL_CHILDREN(qspoid); 3361174708Skmacy 3362175209Skmacy rspqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3363175209Skmacy rspq_name, CTLFLAG_RD, NULL, "rspq statistics"); 3364175209Skmacy rspqpoidlist = SYSCTL_CHILDREN(rspqpoid); 3365175209Skmacy 3366175209Skmacy txqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3367175209Skmacy txq_names[0], CTLFLAG_RD, NULL, "txq statistics"); 3368175209Skmacy txqpoidlist = SYSCTL_CHILDREN(txqpoid); 3369175209Skmacy 3370176472Skmacy ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3371176472Skmacy txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics"); 3372176472Skmacy ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid); 3373176472Skmacy 3374175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size", 3375175209Skmacy CTLFLAG_RD, &qs->rspq.size, 3376175209Skmacy 0, "#entries in response queue"); 3377175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "cidx", 3378175209Skmacy CTLFLAG_RD, &qs->rspq.cidx, 3379175209Skmacy 0, "consumer index"); 3380175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "credits", 3381175209Skmacy CTLFLAG_RD, &qs->rspq.credits, 3382175209Skmacy 0, "#credits"); 3383175209Skmacy SYSCTL_ADD_XLONG(ctx, rspqpoidlist, OID_AUTO, "phys_addr", 3384175209Skmacy CTLFLAG_RD, &qs->rspq.phys_addr, 3385175209Skmacy "physical_address_of the queue"); 3386175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_start", 3387175209Skmacy CTLFLAG_RW, &qs->rspq.rspq_dump_start, 3388175209Skmacy 0, "start rspq dump entry"); 3389175209Skmacy SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_count", 3390175209Skmacy CTLFLAG_RW, &qs->rspq.rspq_dump_count, 3391175209Skmacy 0, "#rspq entries to dump"); 3392175209Skmacy SYSCTL_ADD_PROC(ctx, rspqpoidlist, OID_AUTO, "qdump", 3393175209Skmacy CTLTYPE_STRING | CTLFLAG_RD, &qs->rspq, 3394175209Skmacy 0, t3_dump_rspq, "A", "dump of the response queue"); 3395175209Skmacy 3396176472Skmacy 3397175209Skmacy SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "dropped", 3398174708Skmacy CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_drops, 3399174708Skmacy 0, "#tunneled packets dropped"); 3400175209Skmacy SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "sendqlen", 3401174708Skmacy CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.qlen, 3402174708Skmacy 0, "#tunneled packets waiting to be sent"); 3403175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx", 3404174708Skmacy CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, 3405174708Skmacy 0, "#tunneled packets queue producer index"); 3406175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx", 3407174708Skmacy CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, 3408174708Skmacy 0, "#tunneled packets queue consumer index"); 3409175209Skmacy SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "processed", 3410174708Skmacy CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, 3411174708Skmacy 0, "#tunneled packets processed by the card"); 3412175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "cleaned", 3413174708Skmacy CTLFLAG_RD, &txq->cleaned, 3414174708Skmacy 0, "#tunneled packets cleaned"); 3415175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "in_use", 3416174708Skmacy CTLFLAG_RD, &txq->in_use, 3417174708Skmacy 0, "#tunneled packet slots in use"); 3418175209Skmacy SYSCTL_ADD_ULONG(ctx, txqpoidlist, OID_AUTO, "frees", 3419174708Skmacy CTLFLAG_RD, &txq->txq_frees, 3420174708Skmacy "#tunneled packets freed"); 3421175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "skipped", 3422174708Skmacy CTLFLAG_RD, &txq->txq_skipped, 3423174708Skmacy 0, "#tunneled packet descriptors skipped"); 3424175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "coalesced", 3425174708Skmacy CTLFLAG_RD, &txq->txq_coalesced, 3426174708Skmacy 0, "#tunneled packets coalesced"); 3427175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "enqueued", 3428174708Skmacy CTLFLAG_RD, &txq->txq_enqueued, 3429174708Skmacy 0, "#tunneled packets enqueued to hardware"); 3430175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "stopped_flags", 3431174708Skmacy CTLFLAG_RD, &qs->txq_stopped, 3432174708Skmacy 0, "tx queues stopped"); 3433175209Skmacy SYSCTL_ADD_XLONG(ctx, txqpoidlist, OID_AUTO, "phys_addr", 3434175200Skmacy CTLFLAG_RD, &txq->phys_addr, 3435175200Skmacy "physical_address_of the queue"); 3436175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "qgen", 3437175200Skmacy CTLFLAG_RW, &qs->txq[TXQ_ETH].gen, 3438175200Skmacy 0, "txq generation"); 3439175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_cidx", 3440175200Skmacy CTLFLAG_RD, &txq->cidx, 3441175200Skmacy 0, "hardware queue cidx"); 3442175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_pidx", 3443175200Skmacy CTLFLAG_RD, &txq->pidx, 3444175200Skmacy 0, "hardware queue pidx"); 3445175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_start", 3446175200Skmacy CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start, 3447175200Skmacy 0, "txq start idx for dump"); 3448175209Skmacy SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_count", 3449175200Skmacy CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count, 3450175200Skmacy 0, "txq #entries to dump"); 3451175209Skmacy SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump", 3452175200Skmacy CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH], 3453176472Skmacy 0, t3_dump_txq_eth, "A", "dump of the transmit queue"); 3454176472Skmacy 3455176472Skmacy SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start", 3456176472Skmacy CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start, 3457176472Skmacy 0, "ctrlq start idx for dump"); 3458176472Skmacy SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count", 3459176472Skmacy CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count, 3460176472Skmacy 0, "ctrl #entries to dump"); 3461176472Skmacy SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump", 3462176472Skmacy CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL], 3463176472Skmacy 0, t3_dump_txq_ctrl, "A", "dump of the transmit queue"); 3464176472Skmacy 3465176472Skmacy 3466176472Skmacy 3467176472Skmacy 3468176472Skmacy 3469174708Skmacy } 3470174708Skmacy } 3471174708Skmacy} 3472174708Skmacy 3473167514Skmacy/** 3474167514Skmacy * t3_get_desc - dump an SGE descriptor for debugging purposes 3475167514Skmacy * @qs: the queue set 3476167514Skmacy * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) 3477167514Skmacy * @idx: the descriptor index in the queue 3478167514Skmacy * @data: where to dump the descriptor contents 3479167514Skmacy * 3480167514Skmacy * Dumps the contents of a HW descriptor of an SGE queue. Returns the 3481167514Skmacy * size of the descriptor. 3482167514Skmacy */ 3483167514Skmacyint 3484167514Skmacyt3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, 3485167514Skmacy unsigned char *data) 3486167514Skmacy{ 3487167514Skmacy if (qnum >= 6) 3488167514Skmacy return (EINVAL); 3489167514Skmacy 3490167514Skmacy if (qnum < 3) { 3491167514Skmacy if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) 3492167514Skmacy return -EINVAL; 3493167514Skmacy memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); 3494167514Skmacy return sizeof(struct tx_desc); 3495167514Skmacy } 3496167514Skmacy 3497167514Skmacy if (qnum == 3) { 3498167514Skmacy if (!qs->rspq.desc || idx >= qs->rspq.size) 3499167514Skmacy return (EINVAL); 3500167514Skmacy memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); 3501167514Skmacy return sizeof(struct rsp_desc); 3502167514Skmacy } 3503167514Skmacy 3504167514Skmacy qnum -= 4; 3505167514Skmacy if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) 3506167514Skmacy return (EINVAL); 3507167514Skmacy memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); 3508167514Skmacy return sizeof(struct rx_desc); 3509167514Skmacy} 3510