adapter.h revision 255015
1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD: head/sys/dev/cxgbe/adapter.h 255015 2013-08-29 06:26:22Z np $ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34228561Snp#include <sys/kernel.h> 35218792Snp#include <sys/bus.h> 36218792Snp#include <sys/rman.h> 37218792Snp#include <sys/types.h> 38218792Snp#include <sys/malloc.h> 39218792Snp#include <dev/pci/pcivar.h> 40218792Snp#include <dev/pci/pcireg.h> 41218792Snp#include <machine/bus.h> 42218792Snp#include <sys/socket.h> 43218792Snp#include <sys/sysctl.h> 44218792Snp#include <net/ethernet.h> 45218792Snp#include <net/if.h> 46218792Snp#include <net/if_media.h> 47235944Sbz#include <netinet/in.h> 48218792Snp#include <netinet/tcp_lro.h> 49218792Snp 50218792Snp#include "offload.h" 51228561Snp#include "firmware/t4fw_interface.h" 52218792Snp 53218792SnpMALLOC_DECLARE(M_CXGBE); 54218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 55218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 56218792Snp 57218792Snp#if defined(__i386__) || defined(__amd64__) 58218792Snpstatic __inline void 59218792Snpprefetch(void *x) 60218792Snp{ 61218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 62218792Snp} 63218792Snp#else 64218792Snp#define prefetch(x) 65218792Snp#endif 66218792Snp 67231115Snp#ifndef SYSCTL_ADD_UQUAD 68231115Snp#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 69231115Snp#define sysctl_handle_64 sysctl_handle_quad 70231115Snp#define CTLTYPE_U64 CTLTYPE_QUAD 71231115Snp#endif 72231115Snp 73231115Snp#if (__FreeBSD_version >= 900030) || \ 74231115Snp ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000)) 75231115Snp#define SBUF_DRAIN 1 76231115Snp#endif 77231115Snp 78218792Snp#ifdef __amd64__ 79218792Snp/* XXX: need systemwide bus_space_read_8/bus_space_write_8 */ 80218792Snpstatic __inline uint64_t 81218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 82218792Snp bus_size_t offset) 83218792Snp{ 84219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 85219285Snp ("%s: can only handle mem space", __func__)); 86218792Snp 87218792Snp return (*(volatile uint64_t *)(handle + offset)); 88218792Snp} 89218792Snp 90218792Snpstatic __inline void 91218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 92218792Snp bus_size_t offset, uint64_t value) 93218792Snp{ 94219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 95219285Snp ("%s: can only handle mem space", __func__)); 96219285Snp 97218792Snp *(volatile uint64_t *)(bsh + offset) = value; 98218792Snp} 99218792Snp#else 100218792Snpstatic __inline uint64_t 101218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 102218792Snp bus_size_t offset) 103218792Snp{ 104218792Snp return (uint64_t)bus_space_read_4(tag, handle, offset) + 105218792Snp ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32); 106218792Snp} 107218792Snp 108218792Snpstatic __inline void 109218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 110218792Snp bus_size_t offset, uint64_t value) 111218792Snp{ 112218792Snp bus_space_write_4(tag, bsh, offset, value); 113218792Snp bus_space_write_4(tag, bsh, offset + 4, value >> 32); 114218792Snp} 115218792Snp#endif 116218792Snp 117218792Snpstruct adapter; 118218792Snptypedef struct adapter adapter_t; 119218792Snp 120218792Snpenum { 121218792Snp FW_IQ_QSIZE = 256, 122218792Snp FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */ 123218792Snp 124218792Snp RX_IQ_QSIZE = 1024, 125218792Snp RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */ 126218792Snp 127228561Snp EQ_ESIZE = 64, /* All egress queues use this entry size */ 128218792Snp 129228561Snp RX_FL_ESIZE = EQ_ESIZE, /* 8 64bit addresses */ 130219392Snp#if MJUMPAGESIZE != MCLBYTES 131219392Snp FL_BUF_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ 132219392Snp#else 133219392Snp FL_BUF_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ 134219392Snp#endif 135218792Snp 136228561Snp CTRL_EQ_QSIZE = 128, 137228561Snp 138218792Snp TX_EQ_QSIZE = 1024, 139218792Snp TX_SGL_SEGS = 36, 140218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 141218792Snp}; 142218792Snp 143218792Snpenum { 144219944Snp /* adapter intr_type */ 145219944Snp INTR_INTX = (1 << 0), 146219944Snp INTR_MSI = (1 << 1), 147219944Snp INTR_MSIX = (1 << 2) 148219944Snp}; 149219944Snp 150219944Snpenum { 151245274Snp /* flags understood by begin_synchronized_op */ 152245274Snp HOLD_LOCK = (1 << 0), 153245274Snp SLEEP_OK = (1 << 1), 154245274Snp INTR_OK = (1 << 2), 155245274Snp 156245274Snp /* flags understood by end_synchronized_op */ 157245274Snp LOCK_HELD = HOLD_LOCK, 158245274Snp}; 159245274Snp 160245274Snpenum { 161218792Snp /* adapter flags */ 162218792Snp FULL_INIT_DONE = (1 << 0), 163218792Snp FW_OK = (1 << 1), 164228561Snp INTR_DIRECT = (1 << 2), /* direct interrupts for everything */ 165228561Snp MASTER_PF = (1 << 3), 166228561Snp ADAP_SYSCTL_CTX = (1 << 4), 167237263Snp TOM_INIT_DONE = (1 << 5), 168218792Snp 169218792Snp CXGBE_BUSY = (1 << 9), 170218792Snp 171218792Snp /* port flags */ 172218792Snp DOOMED = (1 << 0), 173228561Snp PORT_INIT_DONE = (1 << 1), 174228561Snp PORT_SYSCTL_CTX = (1 << 2), 175253691Snp HAS_TRACEQ = (1 << 3), 176218792Snp}; 177218792Snp 178245274Snp#define IS_DOOMED(pi) ((pi)->flags & DOOMED) 179245274Snp#define SET_DOOMED(pi) do {(pi)->flags |= DOOMED;} while (0) 180245274Snp#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) 181245274Snp#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) 182245274Snp#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) 183218792Snp 184218792Snpstruct port_info { 185218792Snp device_t dev; 186218792Snp struct adapter *adapter; 187218792Snp 188218792Snp struct ifnet *ifp; 189218792Snp struct ifmedia media; 190218792Snp 191218792Snp struct mtx pi_lock; 192218792Snp char lockname[16]; 193218792Snp unsigned long flags; 194218792Snp int if_flags; 195218792Snp 196218792Snp uint16_t viid; 197218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 198218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 199218792Snp uint8_t lport; /* associated offload logical port */ 200218792Snp int8_t mdio_addr; 201218792Snp uint8_t port_type; 202218792Snp uint8_t mod_type; 203218792Snp uint8_t port_id; 204218792Snp uint8_t tx_chan; 205218792Snp 206218792Snp /* These need to be int as they are used in sysctl */ 207218792Snp int ntxq; /* # of tx queues */ 208218792Snp int first_txq; /* index of first tx queue */ 209218792Snp int nrxq; /* # of rx queues */ 210218792Snp int first_rxq; /* index of first rx queue */ 211237263Snp#ifdef TCP_OFFLOAD 212228561Snp int nofldtxq; /* # of offload tx queues */ 213228561Snp int first_ofld_txq; /* index of first offload tx queue */ 214228561Snp int nofldrxq; /* # of offload rx queues */ 215228561Snp int first_ofld_rxq; /* index of first offload rx queue */ 216228561Snp#endif 217218792Snp int tmr_idx; 218218792Snp int pktc_idx; 219218792Snp int qsize_rxq; 220218792Snp int qsize_txq; 221218792Snp 222252747Snp int linkdnrc; 223218792Snp struct link_config link_cfg; 224218792Snp struct port_stats stats; 225218792Snp 226237263Snp eventhandler_tag vlan_c; 227237263Snp 228218792Snp struct callout tick; 229228561Snp struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ 230218792Snp 231218792Snp uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 232218792Snp}; 233218792Snp 234218792Snpstruct fl_sdesc { 235218792Snp struct mbuf *m; 236218792Snp bus_dmamap_t map; 237218792Snp caddr_t cl; 238218792Snp uint8_t tag_idx; /* the sc->fl_tag this map comes from */ 239218792Snp#ifdef INVARIANTS 240218792Snp __be64 ba_tag; 241218792Snp#endif 242218792Snp}; 243218792Snp 244218792Snpstruct tx_desc { 245218792Snp __be64 flit[8]; 246218792Snp}; 247218792Snp 248218792Snpstruct tx_map { 249218792Snp struct mbuf *m; 250218792Snp bus_dmamap_t map; 251218792Snp}; 252218792Snp 253228561Snp/* DMA maps used for tx */ 254228561Snpstruct tx_maps { 255228561Snp struct tx_map *maps; 256228561Snp uint32_t map_total; /* # of DMA maps */ 257228561Snp uint32_t map_pidx; /* next map to be used */ 258228561Snp uint32_t map_cidx; /* reclaimed up to this index */ 259228561Snp uint32_t map_avail; /* # of available maps */ 260228561Snp}; 261228561Snp 262218792Snpstruct tx_sdesc { 263218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 264220873Snp uint8_t credits; /* NIC txq: # of frames sent out in the WR */ 265218792Snp}; 266218792Snp 267218792Snpenum { 268218792Snp /* iq flags */ 269228561Snp IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ 270228561Snp IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ 271228561Snp IQ_INTR = (1 << 2), /* iq takes direct interrupt */ 272228561Snp IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ 273220649Snp 274220649Snp /* iq state */ 275220649Snp IQS_DISABLED = 0, 276220649Snp IQS_BUSY = 1, 277220649Snp IQS_IDLE = 2, 278218792Snp}; 279218792Snp 280218792Snp/* 281218792Snp * Ingress Queue: T4 is producer, driver is consumer. 282218792Snp */ 283218792Snpstruct sge_iq { 284218792Snp bus_dma_tag_t desc_tag; 285218792Snp bus_dmamap_t desc_map; 286219290Snp bus_addr_t ba; /* bus address of descriptor ring */ 287219290Snp uint32_t flags; 288219290Snp uint16_t abs_id; /* absolute SGE id for the iq */ 289219290Snp int8_t intr_pktc_idx; /* packet count threshold index */ 290219290Snp int8_t pad0; 291219290Snp __be64 *desc; /* KVA of descriptor ring */ 292219290Snp 293228561Snp volatile int state; 294218792Snp struct adapter *adapter; 295218792Snp const __be64 *cdesc; /* current descriptor */ 296218792Snp uint8_t gen; /* generation bit */ 297218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 298228561Snp uint8_t intr_next; /* XXX: holdoff for next interrupt */ 299218792Snp uint8_t esize; /* size (bytes) of each entry in the queue */ 300218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 301218792Snp uint16_t cidx; /* consumer index */ 302228561Snp uint16_t cntxt_id; /* SGE context id for the iq */ 303228561Snp 304228561Snp STAILQ_ENTRY(sge_iq) link; 305218792Snp}; 306218792Snp 307218792Snpenum { 308228561Snp EQ_CTRL = 1, 309228561Snp EQ_ETH = 2, 310237263Snp#ifdef TCP_OFFLOAD 311228561Snp EQ_OFLD = 3, 312228561Snp#endif 313228561Snp 314218792Snp /* eq flags */ 315228561Snp EQ_TYPEMASK = 7, /* 3 lsbits hold the type */ 316228561Snp EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */ 317228561Snp EQ_DOOMED = (1 << 4), /* about to be destroyed */ 318228561Snp EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */ 319228561Snp EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */ 320218792Snp}; 321218792Snp 322248925Snp/* Listed in order of preference. Update t4_sysctls too if you change these */ 323249392Snpenum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; 324248925Snp 325218792Snp/* 326218792Snp * Egress Queue: driver is producer, T4 is consumer. 327218792Snp * 328218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 329218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 330218792Snp */ 331218792Snpstruct sge_eq { 332228561Snp unsigned int flags; /* MUST be first */ 333228561Snp unsigned int cntxt_id; /* SGE context id for the eq */ 334218792Snp bus_dma_tag_t desc_tag; 335218792Snp bus_dmamap_t desc_map; 336218792Snp char lockname[16]; 337218792Snp struct mtx eq_lock; 338218792Snp 339218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 340218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 341218792Snp struct sge_qstat *spg; /* status page, for convenience */ 342248925Snp int doorbells; 343248925Snp volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ 344248925Snp u_int udb_qid; /* relative qid within the doorbell page */ 345218792Snp uint16_t cap; /* max # of desc, for convenience */ 346218792Snp uint16_t avail; /* available descriptors, for convenience */ 347218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 348218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 349218792Snp uint16_t pidx; /* producer idx (desc idx) */ 350218792Snp uint16_t pending; /* # of descriptors used since last doorbell */ 351219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 352228561Snp uint8_t tx_chan; /* tx channel used by the eq */ 353228561Snp struct task tx_task; 354228561Snp struct callout tx_callout; 355228561Snp 356228561Snp /* stats */ 357228561Snp 358228561Snp uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for eq */ 359228561Snp uint32_t unstalled; /* recovered from stall */ 360220873Snp}; 361218792Snp 362228561Snpenum { 363228561Snp FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ 364228561Snp FL_DOOMED = (1 << 1), /* about to be destroyed */ 365228561Snp}; 366228561Snp 367228561Snp#define FL_RUNNING_LOW(fl) (fl->cap - fl->needed <= fl->lowat) 368228561Snp#define FL_NOT_RUNNING_LOW(fl) (fl->cap - fl->needed >= 2 * fl->lowat) 369228561Snp 370218792Snpstruct sge_fl { 371218792Snp bus_dma_tag_t desc_tag; 372218792Snp bus_dmamap_t desc_map; 373218792Snp bus_dma_tag_t tag[FL_BUF_SIZES]; 374218792Snp uint8_t tag_idx; 375218792Snp struct mtx fl_lock; 376218792Snp char lockname[16]; 377228561Snp int flags; 378218792Snp 379218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 380218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 381218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 382218792Snp uint32_t cap; /* max # of buffers, for convenience */ 383218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 384218792Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 385218792Snp uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */ 386218792Snp uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */ 387218792Snp uint32_t needed; /* # of buffers needed to fill up fl. */ 388228561Snp uint32_t lowat; /* # of buffers <= this means fl needs help */ 389218792Snp uint32_t pending; /* # of bufs allocated since last doorbell */ 390218792Snp unsigned int dmamap_failed; 391228561Snp TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ 392218792Snp}; 393218792Snp 394220873Snp/* txq: SGE egress queue + what's needed for Ethernet NIC */ 395218792Snpstruct sge_txq { 396218792Snp struct sge_eq eq; /* MUST be first */ 397220873Snp 398220873Snp struct ifnet *ifp; /* the interface this txq belongs to */ 399220873Snp bus_dma_tag_t tx_tag; /* tag for transmit buffers */ 400220873Snp struct buf_ring *br; /* tx buffer ring */ 401220873Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 402218792Snp struct mbuf *m; /* held up due to temporary resource shortage */ 403218792Snp 404228561Snp struct tx_maps txmaps; 405219286Snp 406218792Snp /* stats for common events first */ 407218792Snp 408218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 409237819Snp uint64_t tso_wrs; /* # of TSO work requests */ 410218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 411218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 412218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 413218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 414218792Snp uint64_t txpkts_wrs; /* # of coalesced tx work requests */ 415218792Snp uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */ 416218792Snp 417218792Snp /* stats for not-that-common events */ 418218792Snp 419218792Snp uint32_t no_dmamap; /* no DMA map to load the mbuf */ 420218792Snp uint32_t no_desc; /* out of hardware descriptors */ 421220873Snp} __aligned(CACHE_LINE_SIZE); 422218792Snp 423218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 424218792Snpstruct sge_rxq { 425218792Snp struct sge_iq iq; /* MUST be first */ 426228561Snp struct sge_fl fl; /* MUST follow iq */ 427218792Snp 428219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 429237819Snp#if defined(INET) || defined(INET6) 430218792Snp struct lro_ctrl lro; /* LRO state */ 431219290Snp#endif 432218792Snp 433218792Snp /* stats for common events first */ 434218792Snp 435218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 436218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 437218792Snp 438218792Snp /* stats for not-that-common events */ 439218792Snp 440218792Snp} __aligned(CACHE_LINE_SIZE); 441218792Snp 442237263Snpstatic inline struct sge_rxq * 443237263Snpiq_to_rxq(struct sge_iq *iq) 444237263Snp{ 445237263Snp 446241733Sed return (__containerof(iq, struct sge_rxq, iq)); 447237263Snp} 448237263Snp 449237263Snp 450237263Snp#ifdef TCP_OFFLOAD 451228561Snp/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ 452228561Snpstruct sge_ofld_rxq { 453228561Snp struct sge_iq iq; /* MUST be first */ 454228561Snp struct sge_fl fl; /* MUST follow iq */ 455228561Snp} __aligned(CACHE_LINE_SIZE); 456237263Snp 457237263Snpstatic inline struct sge_ofld_rxq * 458237263Snpiq_to_ofld_rxq(struct sge_iq *iq) 459237263Snp{ 460237263Snp 461241733Sed return (__containerof(iq, struct sge_ofld_rxq, iq)); 462237263Snp} 463228561Snp#endif 464228561Snp 465237263Snpstruct wrqe { 466237263Snp STAILQ_ENTRY(wrqe) link; 467237263Snp struct sge_wrq *wrq; 468237263Snp int wr_len; 469237263Snp uint64_t wr[] __aligned(16); 470237263Snp}; 471237263Snp 472228561Snp/* 473228561Snp * wrq: SGE egress queue that is given prebuilt work requests. Both the control 474228561Snp * and offload tx queues are of this type. 475228561Snp */ 476228561Snpstruct sge_wrq { 477220873Snp struct sge_eq eq; /* MUST be first */ 478220873Snp 479228561Snp struct adapter *adapter; 480228561Snp 481237263Snp /* List of WRs held up due to lack of tx descriptors */ 482237263Snp STAILQ_HEAD(, wrqe) wr_list; 483237263Snp 484220873Snp /* stats for common events first */ 485220873Snp 486228561Snp uint64_t tx_wrs; /* # of tx work requests */ 487220873Snp 488220873Snp /* stats for not-that-common events */ 489220873Snp 490220873Snp uint32_t no_desc; /* out of hardware descriptors */ 491220873Snp} __aligned(CACHE_LINE_SIZE); 492220873Snp 493218792Snpstruct sge { 494228561Snp int timer_val[SGE_NTIMERS]; 495228561Snp int counter_val[SGE_NCOUNTERS]; 496222701Snp int fl_starve_threshold; 497248925Snp int s_qpp; 498218792Snp 499228561Snp int nrxq; /* total # of Ethernet rx queues */ 500228561Snp int ntxq; /* total # of Ethernet tx tx queues */ 501237263Snp#ifdef TCP_OFFLOAD 502228561Snp int nofldrxq; /* total # of TOE rx queues */ 503228561Snp int nofldtxq; /* total # of TOE tx queues */ 504228561Snp#endif 505228561Snp int niq; /* total # of ingress queues */ 506228561Snp int neq; /* total # of egress queues */ 507218792Snp 508218792Snp struct sge_iq fwq; /* Firmware event queue */ 509228561Snp struct sge_wrq mgmtq; /* Management queue (control queue) */ 510228561Snp struct sge_wrq *ctrlq; /* Control queues */ 511218792Snp struct sge_txq *txq; /* NIC tx queues */ 512218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 513237263Snp#ifdef TCP_OFFLOAD 514228561Snp struct sge_wrq *ofld_txq; /* TOE tx queues */ 515228561Snp struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ 516228561Snp#endif 517218792Snp 518218792Snp uint16_t iq_start; 519218792Snp int eq_start; 520218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 521218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 522218792Snp}; 523218792Snp 524228561Snpstruct rss_header; 525228561Snptypedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, 526228561Snp struct mbuf *); 527237263Snptypedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); 528239336Snptypedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); 529228561Snp 530218792Snpstruct adapter { 531228561Snp SLIST_ENTRY(adapter) link; 532218792Snp device_t dev; 533218792Snp struct cdev *cdev; 534218792Snp 535218792Snp /* PCIe register resources */ 536218792Snp int regs_rid; 537218792Snp struct resource *regs_res; 538218792Snp int msix_rid; 539218792Snp struct resource *msix_res; 540218792Snp bus_space_handle_t bh; 541218792Snp bus_space_tag_t bt; 542218792Snp bus_size_t mmio_len; 543248925Snp int udbs_rid; 544248925Snp struct resource *udbs_res; 545248925Snp volatile uint8_t *udbs_base; 546218792Snp 547218792Snp unsigned int pf; 548218792Snp unsigned int mbox; 549218792Snp 550218792Snp /* Interrupt information */ 551218792Snp int intr_type; 552218792Snp int intr_count; 553218792Snp struct irq { 554218792Snp struct resource *res; 555218792Snp int rid; 556218792Snp void *tag; 557218792Snp } *irq; 558218792Snp 559218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 560218792Snp 561218792Snp struct sge sge; 562255015Snp int lro_timeout; 563218792Snp 564228561Snp struct taskqueue *tq[NCHAN]; /* taskqueues that flush data out */ 565218792Snp struct port_info *port[MAX_NPORTS]; 566218792Snp uint8_t chan_map[NCHAN]; 567218792Snp 568237263Snp#ifdef TCP_OFFLOAD 569237263Snp void *tom_softc; /* (struct tom_data *) */ 570228561Snp struct tom_tunables tt; 571255005Snp void *iwarp_softc; /* (struct c4iw_dev *) */ 572228561Snp#endif 573222509Snp struct l2t_data *l2t; /* L2 table */ 574218792Snp struct tid_info tids; 575218792Snp 576248925Snp int doorbells; 577218792Snp int open_device_map; 578237263Snp#ifdef TCP_OFFLOAD 579228561Snp int offload_map; 580228561Snp#endif 581218792Snp int flags; 582218792Snp 583253691Snp char ifp_lockname[16]; 584253691Snp struct mtx ifp_lock; 585253691Snp struct ifnet *ifp; /* tracer ifp */ 586253691Snp struct ifmedia media; 587253691Snp int traceq; /* iq used by all tracers, -1 if none */ 588253691Snp int tracer_valid; /* bitmap of valid tracers */ 589253691Snp int tracer_enabled; /* bitmap of enabled tracers */ 590253691Snp 591218792Snp char fw_version[32]; 592245936Snp char cfg_file[32]; 593245936Snp u_int cfcsum; 594218792Snp struct adapter_params params; 595218792Snp struct t4_virt_res vres; 596218792Snp 597228561Snp uint16_t linkcaps; 598228561Snp uint16_t niccaps; 599228561Snp uint16_t toecaps; 600228561Snp uint16_t rdmacaps; 601228561Snp uint16_t iscsicaps; 602228561Snp uint16_t fcoecaps; 603220873Snp 604228561Snp struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ 605228561Snp 606218792Snp struct mtx sc_lock; 607218792Snp char lockname[16]; 608228561Snp 609228561Snp /* Starving free lists */ 610228561Snp struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ 611228561Snp TAILQ_HEAD(, sge_fl) sfl; 612228561Snp struct callout sfl_callout; 613228561Snp 614237263Snp an_handler_t an_handler __aligned(CACHE_LINE_SIZE); 615247291Snp fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */ 616239336Snp cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */ 617245274Snp 618245274Snp#ifdef INVARIANTS 619245274Snp const char *last_op; 620245274Snp const void *last_op_thr; 621245274Snp#endif 622218792Snp}; 623218792Snp 624218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 625218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 626218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 627218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 628218792Snp 629245274Snp/* XXX: not bulletproof, but much better than nothing */ 630245274Snp#define ASSERT_SYNCHRONIZED_OP(sc) \ 631245274Snp KASSERT(IS_BUSY(sc) && \ 632245274Snp (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ 633245274Snp ("%s: operation not synchronized.", __func__)) 634245274Snp 635218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 636218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 637218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 638218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 639218792Snp 640218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 641218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 642218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 643218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 644218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 645218792Snp 646218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 647218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 648218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 649218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 650218792Snp 651218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 652218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 653218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 654218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 655218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 656218792Snp 657218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 658218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 659218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 660218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 661218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 662218792Snp 663245517Snp#define for_each_txq(pi, iter, q) \ 664245567Snp for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \ 665245567Snp iter < pi->ntxq; ++iter, ++q) 666245517Snp#define for_each_rxq(pi, iter, q) \ 667245567Snp for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \ 668245567Snp iter < pi->nrxq; ++iter, ++q) 669245517Snp#define for_each_ofld_txq(pi, iter, q) \ 670245567Snp for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \ 671245567Snp iter < pi->nofldtxq; ++iter, ++q) 672245517Snp#define for_each_ofld_rxq(pi, iter, q) \ 673245567Snp for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \ 674245567Snp iter < pi->nofldrxq; ++iter, ++q) 675218792Snp 676222510Snp/* One for errors, one for firmware events */ 677222510Snp#define T4_EXTRA_INTR 2 678218792Snp 679218792Snpstatic inline uint32_t 680218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 681218792Snp{ 682237263Snp 683218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 684218792Snp} 685218792Snp 686218792Snpstatic inline void 687218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 688218792Snp{ 689237263Snp 690218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 691218792Snp} 692218792Snp 693218792Snpstatic inline uint64_t 694218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 695218792Snp{ 696237263Snp 697218792Snp return t4_bus_space_read_8(sc->bt, sc->bh, reg); 698218792Snp} 699218792Snp 700218792Snpstatic inline void 701218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 702218792Snp{ 703237263Snp 704218792Snp t4_bus_space_write_8(sc->bt, sc->bh, reg, val); 705218792Snp} 706218792Snp 707218792Snpstatic inline void 708218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 709218792Snp{ 710237263Snp 711218792Snp *val = pci_read_config(sc->dev, reg, 1); 712218792Snp} 713218792Snp 714218792Snpstatic inline void 715218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 716218792Snp{ 717237263Snp 718218792Snp pci_write_config(sc->dev, reg, val, 1); 719218792Snp} 720218792Snp 721218792Snpstatic inline void 722218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 723218792Snp{ 724237263Snp 725218792Snp *val = pci_read_config(sc->dev, reg, 2); 726218792Snp} 727218792Snp 728218792Snpstatic inline void 729218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 730218792Snp{ 731237263Snp 732218792Snp pci_write_config(sc->dev, reg, val, 2); 733218792Snp} 734218792Snp 735218792Snpstatic inline void 736218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 737218792Snp{ 738237263Snp 739218792Snp *val = pci_read_config(sc->dev, reg, 4); 740218792Snp} 741218792Snp 742218792Snpstatic inline void 743218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 744218792Snp{ 745237263Snp 746218792Snp pci_write_config(sc->dev, reg, val, 4); 747218792Snp} 748218792Snp 749218792Snpstatic inline struct port_info * 750218792Snpadap2pinfo(struct adapter *sc, int idx) 751218792Snp{ 752237263Snp 753218792Snp return (sc->port[idx]); 754218792Snp} 755218792Snp 756218792Snpstatic inline void 757218792Snpt4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[]) 758218792Snp{ 759237263Snp 760218792Snp bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN); 761218792Snp} 762218792Snp 763248925Snpstatic inline bool 764248925Snpis_10G_port(const struct port_info *pi) 765218792Snp{ 766237263Snp 767218792Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0); 768218792Snp} 769218792Snp 770250092Snpstatic inline bool 771250092Snpis_40G_port(const struct port_info *pi) 772250092Snp{ 773250092Snp 774250092Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0); 775250092Snp} 776250092Snp 777248925Snpstatic inline int 778248925Snptx_resume_threshold(struct sge_eq *eq) 779228561Snp{ 780237263Snp 781228561Snp return (eq->qsize / 4); 782228561Snp} 783228561Snp 784219286Snp/* t4_main.c */ 785228561Snpvoid t4_tx_task(void *, int); 786228561Snpvoid t4_tx_callout(void *); 787218792Snpint t4_os_find_pci_capability(struct adapter *, int); 788218792Snpint t4_os_pci_save_state(struct adapter *); 789218792Snpint t4_os_pci_restore_state(struct adapter *); 790218792Snpvoid t4_os_portmod_changed(const struct adapter *, int); 791252747Snpvoid t4_os_link_changed(struct adapter *, int, int, int); 792228561Snpvoid t4_iterate(void (*)(struct adapter *, void *), void *); 793228561Snpint t4_register_cpl_handler(struct adapter *, int, cpl_handler_t); 794237263Snpint t4_register_an_handler(struct adapter *, an_handler_t); 795239336Snpint t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t); 796239338Snpint t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 797245274Snpint begin_synchronized_op(struct adapter *, struct port_info *, int, char *); 798245274Snpvoid end_synchronized_op(struct adapter *, int); 799218792Snp 800219286Snp/* t4_sge.c */ 801219392Snpvoid t4_sge_modload(void); 802248925Snpvoid t4_init_sge_cpl_handlers(struct adapter *); 803248925Snpvoid t4_tweak_chip_settings(struct adapter *); 804248925Snpint t4_read_chip_settings(struct adapter *); 805218792Snpint t4_create_dma_tag(struct adapter *); 806253829Snpvoid t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, 807253829Snp struct sysctl_oid_list *); 808218792Snpint t4_destroy_dma_tag(struct adapter *); 809220873Snpint t4_setup_adapter_queues(struct adapter *); 810220873Snpint t4_teardown_adapter_queues(struct adapter *); 811228561Snpint t4_setup_port_queues(struct port_info *); 812228561Snpint t4_teardown_port_queues(struct port_info *); 813228561Snpint t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int); 814228561Snpvoid t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t); 815218792Snpvoid t4_intr_all(void *); 816222510Snpvoid t4_intr(void *); 817218792Snpvoid t4_intr_err(void *); 818218792Snpvoid t4_intr_evt(void *); 819237263Snpvoid t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); 820218792Snpint t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *); 821218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 822228561Snpint can_resume_tx(struct sge_eq *); 823218792Snp 824253691Snp/* t4_tracer.c */ 825253691Snpstruct t4_tracer; 826253691Snpvoid t4_tracer_modload(void); 827253691Snpvoid t4_tracer_modunload(void); 828253691Snpvoid t4_tracer_port_detach(struct adapter *); 829253691Snpint t4_get_tracer(struct adapter *, struct t4_tracer *); 830253691Snpint t4_set_tracer(struct adapter *, struct t4_tracer *); 831253691Snpint t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 832253691Snpint t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 833253691Snp 834237263Snpstatic inline struct wrqe * 835237263Snpalloc_wrqe(int wr_len, struct sge_wrq *wrq) 836228561Snp{ 837237263Snp int len = offsetof(struct wrqe, wr) + wr_len; 838237263Snp struct wrqe *wr; 839228561Snp 840237263Snp wr = malloc(len, M_CXGBE, M_NOWAIT); 841237263Snp if (__predict_false(wr == NULL)) 842237263Snp return (NULL); 843237263Snp wr->wr_len = wr_len; 844237263Snp wr->wrq = wrq; 845237263Snp return (wr); 846237263Snp} 847237263Snp 848237263Snpstatic inline void * 849237263Snpwrtod(struct wrqe *wr) 850237263Snp{ 851237263Snp return (&wr->wr[0]); 852237263Snp} 853237263Snp 854237263Snpstatic inline void 855237263Snpfree_wrqe(struct wrqe *wr) 856237263Snp{ 857237263Snp free(wr, M_CXGBE); 858237263Snp} 859237263Snp 860237263Snpstatic inline void 861237263Snpt4_wrq_tx(struct adapter *sc, struct wrqe *wr) 862237263Snp{ 863237263Snp struct sge_wrq *wrq = wr->wrq; 864237263Snp 865228561Snp TXQ_LOCK(wrq); 866237263Snp t4_wrq_tx_locked(sc, wrq, wr); 867228561Snp TXQ_UNLOCK(wrq); 868228561Snp} 869228561Snp 870218792Snp#endif 871