adapter.h revision 269356
1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD: stable/10/sys/dev/cxgbe/adapter.h 269356 2014-07-31 23:04:41Z np $ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34228561Snp#include <sys/kernel.h> 35218792Snp#include <sys/bus.h> 36218792Snp#include <sys/rman.h> 37218792Snp#include <sys/types.h> 38218792Snp#include <sys/malloc.h> 39218792Snp#include <dev/pci/pcivar.h> 40218792Snp#include <dev/pci/pcireg.h> 41218792Snp#include <machine/bus.h> 42218792Snp#include <sys/socket.h> 43218792Snp#include <sys/sysctl.h> 44218792Snp#include <net/ethernet.h> 45218792Snp#include <net/if.h> 46218792Snp#include <net/if_media.h> 47235944Sbz#include <netinet/in.h> 48218792Snp#include <netinet/tcp_lro.h> 49218792Snp 50218792Snp#include "offload.h" 51228561Snp#include "firmware/t4fw_interface.h" 52218792Snp 53218792SnpMALLOC_DECLARE(M_CXGBE); 54218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 55218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 56218792Snp 57218792Snp#if defined(__i386__) || defined(__amd64__) 58218792Snpstatic __inline void 59218792Snpprefetch(void *x) 60218792Snp{ 61218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 62218792Snp} 63218792Snp#else 64218792Snp#define prefetch(x) 65218792Snp#endif 66218792Snp 67231115Snp#ifndef SYSCTL_ADD_UQUAD 68231115Snp#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 69231115Snp#define sysctl_handle_64 sysctl_handle_quad 70231115Snp#define CTLTYPE_U64 CTLTYPE_QUAD 71231115Snp#endif 72231115Snp 73231115Snp#if (__FreeBSD_version >= 900030) || \ 74231115Snp ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000)) 75231115Snp#define SBUF_DRAIN 1 76231115Snp#endif 77231115Snp 78218792Snp#ifdef __amd64__ 79218792Snp/* XXX: need systemwide bus_space_read_8/bus_space_write_8 */ 80218792Snpstatic __inline uint64_t 81218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 82218792Snp bus_size_t offset) 83218792Snp{ 84219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 85219285Snp ("%s: can only handle mem space", __func__)); 86218792Snp 87218792Snp return (*(volatile uint64_t *)(handle + offset)); 88218792Snp} 89218792Snp 90218792Snpstatic __inline void 91218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 92218792Snp bus_size_t offset, uint64_t value) 93218792Snp{ 94219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 95219285Snp ("%s: can only handle mem space", __func__)); 96219285Snp 97218792Snp *(volatile uint64_t *)(bsh + offset) = value; 98218792Snp} 99218792Snp#else 100218792Snpstatic __inline uint64_t 101218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 102218792Snp bus_size_t offset) 103218792Snp{ 104218792Snp return (uint64_t)bus_space_read_4(tag, handle, offset) + 105218792Snp ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32); 106218792Snp} 107218792Snp 108218792Snpstatic __inline void 109218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 110218792Snp bus_size_t offset, uint64_t value) 111218792Snp{ 112218792Snp bus_space_write_4(tag, bsh, offset, value); 113218792Snp bus_space_write_4(tag, bsh, offset + 4, value >> 32); 114218792Snp} 115218792Snp#endif 116218792Snp 117218792Snpstruct adapter; 118218792Snptypedef struct adapter adapter_t; 119218792Snp 120218792Snpenum { 121218792Snp FW_IQ_QSIZE = 256, 122218792Snp FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */ 123218792Snp 124218792Snp RX_IQ_QSIZE = 1024, 125218792Snp RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */ 126218792Snp 127228561Snp EQ_ESIZE = 64, /* All egress queues use this entry size */ 128218792Snp 129228561Snp RX_FL_ESIZE = EQ_ESIZE, /* 8 64bit addresses */ 130219392Snp#if MJUMPAGESIZE != MCLBYTES 131265425Snp SW_ZONE_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ 132219392Snp#else 133265425Snp SW_ZONE_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ 134219392Snp#endif 135265425Snp CL_METADATA_SIZE = CACHE_LINE_SIZE, 136218792Snp 137228561Snp CTRL_EQ_QSIZE = 128, 138228561Snp 139218792Snp TX_EQ_QSIZE = 1024, 140218792Snp TX_SGL_SEGS = 36, 141218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 142218792Snp}; 143218792Snp 144218792Snpenum { 145219944Snp /* adapter intr_type */ 146219944Snp INTR_INTX = (1 << 0), 147219944Snp INTR_MSI = (1 << 1), 148219944Snp INTR_MSIX = (1 << 2) 149219944Snp}; 150219944Snp 151219944Snpenum { 152245274Snp /* flags understood by begin_synchronized_op */ 153245274Snp HOLD_LOCK = (1 << 0), 154245274Snp SLEEP_OK = (1 << 1), 155245274Snp INTR_OK = (1 << 2), 156245274Snp 157245274Snp /* flags understood by end_synchronized_op */ 158245274Snp LOCK_HELD = HOLD_LOCK, 159245274Snp}; 160245274Snp 161245274Snpenum { 162218792Snp /* adapter flags */ 163218792Snp FULL_INIT_DONE = (1 << 0), 164218792Snp FW_OK = (1 << 1), 165228561Snp INTR_DIRECT = (1 << 2), /* direct interrupts for everything */ 166228561Snp MASTER_PF = (1 << 3), 167228561Snp ADAP_SYSCTL_CTX = (1 << 4), 168237263Snp TOM_INIT_DONE = (1 << 5), 169255050Snp BUF_PACKING_OK = (1 << 6), 170218792Snp 171218792Snp CXGBE_BUSY = (1 << 9), 172218792Snp 173218792Snp /* port flags */ 174218792Snp DOOMED = (1 << 0), 175228561Snp PORT_INIT_DONE = (1 << 1), 176228561Snp PORT_SYSCTL_CTX = (1 << 2), 177253691Snp HAS_TRACEQ = (1 << 3), 178218792Snp}; 179218792Snp 180245274Snp#define IS_DOOMED(pi) ((pi)->flags & DOOMED) 181245274Snp#define SET_DOOMED(pi) do {(pi)->flags |= DOOMED;} while (0) 182245274Snp#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) 183245274Snp#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) 184245274Snp#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) 185218792Snp 186218792Snpstruct port_info { 187218792Snp device_t dev; 188218792Snp struct adapter *adapter; 189218792Snp 190218792Snp struct ifnet *ifp; 191218792Snp struct ifmedia media; 192218792Snp 193218792Snp struct mtx pi_lock; 194218792Snp char lockname[16]; 195218792Snp unsigned long flags; 196218792Snp int if_flags; 197218792Snp 198259142Snp uint16_t *rss; 199218792Snp uint16_t viid; 200218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 201218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 202218792Snp uint8_t lport; /* associated offload logical port */ 203218792Snp int8_t mdio_addr; 204218792Snp uint8_t port_type; 205218792Snp uint8_t mod_type; 206218792Snp uint8_t port_id; 207218792Snp uint8_t tx_chan; 208265410Snp uint8_t rx_chan_map; /* rx MPS channel bitmap */ 209218792Snp 210218792Snp /* These need to be int as they are used in sysctl */ 211218792Snp int ntxq; /* # of tx queues */ 212218792Snp int first_txq; /* index of first tx queue */ 213264493Sscottl int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ 214218792Snp int nrxq; /* # of rx queues */ 215218792Snp int first_rxq; /* index of first rx queue */ 216237263Snp#ifdef TCP_OFFLOAD 217228561Snp int nofldtxq; /* # of offload tx queues */ 218228561Snp int first_ofld_txq; /* index of first offload tx queue */ 219228561Snp int nofldrxq; /* # of offload rx queues */ 220228561Snp int first_ofld_rxq; /* index of first offload rx queue */ 221228561Snp#endif 222218792Snp int tmr_idx; 223218792Snp int pktc_idx; 224218792Snp int qsize_rxq; 225218792Snp int qsize_txq; 226218792Snp 227252747Snp int linkdnrc; 228218792Snp struct link_config link_cfg; 229218792Snp struct port_stats stats; 230218792Snp 231237263Snp eventhandler_tag vlan_c; 232237263Snp 233218792Snp struct callout tick; 234228561Snp struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ 235218792Snp 236218792Snp uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 237218792Snp}; 238218792Snp 239265425Snp/* Where the cluster came from, how it has been carved up. */ 240265425Snpstruct cluster_layout { 241265425Snp int8_t zidx; 242265425Snp int8_t hwidx; 243265425Snp uint16_t region1; /* mbufs laid out within this region */ 244265425Snp /* region2 is the DMA region */ 245265425Snp uint16_t region3; /* cluster_metadata within this region */ 246265425Snp}; 247265425Snp 248265425Snpstruct cluster_metadata { 249265425Snp u_int refcount; 250218792Snp#ifdef INVARIANTS 251265425Snp struct fl_sdesc *sd; /* For debug only. Could easily be stale */ 252218792Snp#endif 253218792Snp}; 254218792Snp 255265425Snpstruct fl_sdesc { 256265425Snp caddr_t cl; 257269356Snp uint16_t nmbuf; /* # of driver originated mbufs with ref on cluster */ 258265425Snp struct cluster_layout cll; 259265425Snp}; 260265425Snp 261218792Snpstruct tx_desc { 262218792Snp __be64 flit[8]; 263218792Snp}; 264218792Snp 265218792Snpstruct tx_map { 266218792Snp struct mbuf *m; 267218792Snp bus_dmamap_t map; 268218792Snp}; 269218792Snp 270228561Snp/* DMA maps used for tx */ 271228561Snpstruct tx_maps { 272228561Snp struct tx_map *maps; 273228561Snp uint32_t map_total; /* # of DMA maps */ 274228561Snp uint32_t map_pidx; /* next map to be used */ 275228561Snp uint32_t map_cidx; /* reclaimed up to this index */ 276228561Snp uint32_t map_avail; /* # of available maps */ 277228561Snp}; 278228561Snp 279218792Snpstruct tx_sdesc { 280218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 281220873Snp uint8_t credits; /* NIC txq: # of frames sent out in the WR */ 282218792Snp}; 283218792Snp 284218792Snpenum { 285218792Snp /* iq flags */ 286228561Snp IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ 287228561Snp IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ 288228561Snp IQ_INTR = (1 << 2), /* iq takes direct interrupt */ 289228561Snp IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ 290220649Snp 291220649Snp /* iq state */ 292220649Snp IQS_DISABLED = 0, 293220649Snp IQS_BUSY = 1, 294220649Snp IQS_IDLE = 2, 295218792Snp}; 296218792Snp 297218792Snp/* 298218792Snp * Ingress Queue: T4 is producer, driver is consumer. 299218792Snp */ 300218792Snpstruct sge_iq { 301218792Snp bus_dma_tag_t desc_tag; 302218792Snp bus_dmamap_t desc_map; 303219290Snp bus_addr_t ba; /* bus address of descriptor ring */ 304219290Snp uint32_t flags; 305219290Snp uint16_t abs_id; /* absolute SGE id for the iq */ 306219290Snp int8_t intr_pktc_idx; /* packet count threshold index */ 307219290Snp int8_t pad0; 308219290Snp __be64 *desc; /* KVA of descriptor ring */ 309219290Snp 310228561Snp volatile int state; 311218792Snp struct adapter *adapter; 312218792Snp const __be64 *cdesc; /* current descriptor */ 313218792Snp uint8_t gen; /* generation bit */ 314218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 315228561Snp uint8_t intr_next; /* XXX: holdoff for next interrupt */ 316218792Snp uint8_t esize; /* size (bytes) of each entry in the queue */ 317218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 318218792Snp uint16_t cidx; /* consumer index */ 319228561Snp uint16_t cntxt_id; /* SGE context id for the iq */ 320228561Snp 321228561Snp STAILQ_ENTRY(sge_iq) link; 322218792Snp}; 323218792Snp 324218792Snpenum { 325228561Snp EQ_CTRL = 1, 326228561Snp EQ_ETH = 2, 327237263Snp#ifdef TCP_OFFLOAD 328228561Snp EQ_OFLD = 3, 329228561Snp#endif 330228561Snp 331218792Snp /* eq flags */ 332228561Snp EQ_TYPEMASK = 7, /* 3 lsbits hold the type */ 333228561Snp EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */ 334228561Snp EQ_DOOMED = (1 << 4), /* about to be destroyed */ 335228561Snp EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */ 336228561Snp EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */ 337218792Snp}; 338218792Snp 339248925Snp/* Listed in order of preference. Update t4_sysctls too if you change these */ 340249392Snpenum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; 341248925Snp 342218792Snp/* 343218792Snp * Egress Queue: driver is producer, T4 is consumer. 344218792Snp * 345218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 346218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 347218792Snp */ 348218792Snpstruct sge_eq { 349228561Snp unsigned int flags; /* MUST be first */ 350228561Snp unsigned int cntxt_id; /* SGE context id for the eq */ 351218792Snp bus_dma_tag_t desc_tag; 352218792Snp bus_dmamap_t desc_map; 353218792Snp char lockname[16]; 354218792Snp struct mtx eq_lock; 355218792Snp 356218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 357218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 358218792Snp struct sge_qstat *spg; /* status page, for convenience */ 359248925Snp int doorbells; 360248925Snp volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ 361248925Snp u_int udb_qid; /* relative qid within the doorbell page */ 362218792Snp uint16_t cap; /* max # of desc, for convenience */ 363218792Snp uint16_t avail; /* available descriptors, for convenience */ 364218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 365218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 366218792Snp uint16_t pidx; /* producer idx (desc idx) */ 367218792Snp uint16_t pending; /* # of descriptors used since last doorbell */ 368219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 369228561Snp uint8_t tx_chan; /* tx channel used by the eq */ 370228561Snp struct task tx_task; 371228561Snp struct callout tx_callout; 372228561Snp 373228561Snp /* stats */ 374228561Snp 375228561Snp uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for eq */ 376228561Snp uint32_t unstalled; /* recovered from stall */ 377220873Snp}; 378218792Snp 379265425Snpstruct sw_zone_info { 380265425Snp uma_zone_t zone; /* zone that this cluster comes from */ 381265425Snp int size; /* size of cluster: 2K, 4K, 9K, 16K, etc. */ 382265425Snp int type; /* EXT_xxx type of the cluster */ 383265425Snp int8_t head_hwidx; 384265425Snp int8_t tail_hwidx; 385255050Snp}; 386255050Snp 387265425Snpstruct hw_buf_info { 388265425Snp int8_t zidx; /* backpointer to zone; -ve means unused */ 389265425Snp int8_t next; /* next hwidx for this zone; -1 means no more */ 390265425Snp int size; 391265425Snp}; 392265425Snp 393228561Snpenum { 394228561Snp FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ 395228561Snp FL_DOOMED = (1 << 1), /* about to be destroyed */ 396255050Snp FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ 397228561Snp}; 398228561Snp 399228561Snp#define FL_RUNNING_LOW(fl) (fl->cap - fl->needed <= fl->lowat) 400228561Snp#define FL_NOT_RUNNING_LOW(fl) (fl->cap - fl->needed >= 2 * fl->lowat) 401228561Snp 402218792Snpstruct sge_fl { 403218792Snp bus_dma_tag_t desc_tag; 404218792Snp bus_dmamap_t desc_map; 405265425Snp struct cluster_layout cll_def; /* default refill zone, layout */ 406265425Snp struct cluster_layout cll_alt; /* alternate refill zone, layout */ 407218792Snp struct mtx fl_lock; 408218792Snp char lockname[16]; 409228561Snp int flags; 410218792Snp 411218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 412218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 413218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 414218792Snp uint32_t cap; /* max # of buffers, for convenience */ 415218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 416218792Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 417218792Snp uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */ 418255050Snp uint32_t rx_offset; /* offset in fl buf (when buffer packing) */ 419218792Snp uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */ 420218792Snp uint32_t needed; /* # of buffers needed to fill up fl. */ 421228561Snp uint32_t lowat; /* # of buffers <= this means fl needs help */ 422218792Snp uint32_t pending; /* # of bufs allocated since last doorbell */ 423228561Snp TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ 424265425Snp 425265425Snp struct mbuf *m0; 426265425Snp struct mbuf **pnext; 427265425Snp u_int remaining; 428265425Snp 429265425Snp uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */ 430265425Snp uint64_t mbuf_inlined; /* # of mbuf created within clusters */ 431265425Snp uint64_t cl_allocated; /* # of clusters allocated */ 432265425Snp uint64_t cl_recycled; /* # of clusters recycled */ 433265425Snp uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */ 434218792Snp}; 435218792Snp 436220873Snp/* txq: SGE egress queue + what's needed for Ethernet NIC */ 437218792Snpstruct sge_txq { 438218792Snp struct sge_eq eq; /* MUST be first */ 439220873Snp 440220873Snp struct ifnet *ifp; /* the interface this txq belongs to */ 441220873Snp bus_dma_tag_t tx_tag; /* tag for transmit buffers */ 442220873Snp struct buf_ring *br; /* tx buffer ring */ 443220873Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 444218792Snp struct mbuf *m; /* held up due to temporary resource shortage */ 445218792Snp 446228561Snp struct tx_maps txmaps; 447219286Snp 448218792Snp /* stats for common events first */ 449218792Snp 450218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 451237819Snp uint64_t tso_wrs; /* # of TSO work requests */ 452218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 453218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 454218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 455218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 456218792Snp uint64_t txpkts_wrs; /* # of coalesced tx work requests */ 457218792Snp uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */ 458218792Snp 459218792Snp /* stats for not-that-common events */ 460218792Snp 461218792Snp uint32_t no_dmamap; /* no DMA map to load the mbuf */ 462218792Snp uint32_t no_desc; /* out of hardware descriptors */ 463220873Snp} __aligned(CACHE_LINE_SIZE); 464218792Snp 465218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 466218792Snpstruct sge_rxq { 467218792Snp struct sge_iq iq; /* MUST be first */ 468228561Snp struct sge_fl fl; /* MUST follow iq */ 469218792Snp 470219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 471237819Snp#if defined(INET) || defined(INET6) 472218792Snp struct lro_ctrl lro; /* LRO state */ 473219290Snp#endif 474218792Snp 475218792Snp /* stats for common events first */ 476218792Snp 477218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 478218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 479218792Snp 480218792Snp /* stats for not-that-common events */ 481218792Snp 482218792Snp} __aligned(CACHE_LINE_SIZE); 483218792Snp 484237263Snpstatic inline struct sge_rxq * 485237263Snpiq_to_rxq(struct sge_iq *iq) 486237263Snp{ 487237263Snp 488241733Sed return (__containerof(iq, struct sge_rxq, iq)); 489237263Snp} 490237263Snp 491237263Snp 492237263Snp#ifdef TCP_OFFLOAD 493228561Snp/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ 494228561Snpstruct sge_ofld_rxq { 495228561Snp struct sge_iq iq; /* MUST be first */ 496228561Snp struct sge_fl fl; /* MUST follow iq */ 497228561Snp} __aligned(CACHE_LINE_SIZE); 498237263Snp 499237263Snpstatic inline struct sge_ofld_rxq * 500237263Snpiq_to_ofld_rxq(struct sge_iq *iq) 501237263Snp{ 502237263Snp 503241733Sed return (__containerof(iq, struct sge_ofld_rxq, iq)); 504237263Snp} 505228561Snp#endif 506228561Snp 507237263Snpstruct wrqe { 508237263Snp STAILQ_ENTRY(wrqe) link; 509237263Snp struct sge_wrq *wrq; 510237263Snp int wr_len; 511237263Snp uint64_t wr[] __aligned(16); 512237263Snp}; 513237263Snp 514228561Snp/* 515228561Snp * wrq: SGE egress queue that is given prebuilt work requests. Both the control 516228561Snp * and offload tx queues are of this type. 517228561Snp */ 518228561Snpstruct sge_wrq { 519220873Snp struct sge_eq eq; /* MUST be first */ 520220873Snp 521228561Snp struct adapter *adapter; 522228561Snp 523237263Snp /* List of WRs held up due to lack of tx descriptors */ 524237263Snp STAILQ_HEAD(, wrqe) wr_list; 525237263Snp 526220873Snp /* stats for common events first */ 527220873Snp 528228561Snp uint64_t tx_wrs; /* # of tx work requests */ 529220873Snp 530220873Snp /* stats for not-that-common events */ 531220873Snp 532220873Snp uint32_t no_desc; /* out of hardware descriptors */ 533220873Snp} __aligned(CACHE_LINE_SIZE); 534220873Snp 535218792Snpstruct sge { 536228561Snp int timer_val[SGE_NTIMERS]; 537228561Snp int counter_val[SGE_NCOUNTERS]; 538222701Snp int fl_starve_threshold; 539265410Snp int fl_starve_threshold2; 540256794Snp int eq_s_qpp; 541256794Snp int iq_s_qpp; 542218792Snp 543228561Snp int nrxq; /* total # of Ethernet rx queues */ 544228561Snp int ntxq; /* total # of Ethernet tx tx queues */ 545237263Snp#ifdef TCP_OFFLOAD 546228561Snp int nofldrxq; /* total # of TOE rx queues */ 547228561Snp int nofldtxq; /* total # of TOE tx queues */ 548228561Snp#endif 549228561Snp int niq; /* total # of ingress queues */ 550228561Snp int neq; /* total # of egress queues */ 551218792Snp 552218792Snp struct sge_iq fwq; /* Firmware event queue */ 553228561Snp struct sge_wrq mgmtq; /* Management queue (control queue) */ 554228561Snp struct sge_wrq *ctrlq; /* Control queues */ 555218792Snp struct sge_txq *txq; /* NIC tx queues */ 556218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 557237263Snp#ifdef TCP_OFFLOAD 558228561Snp struct sge_wrq *ofld_txq; /* TOE tx queues */ 559228561Snp struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ 560228561Snp#endif 561218792Snp 562218792Snp uint16_t iq_start; 563218792Snp int eq_start; 564218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 565218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 566255050Snp 567265425Snp int pack_boundary; 568265425Snp int8_t safe_hwidx1; /* may not have room for metadata */ 569265425Snp int8_t safe_hwidx2; /* with room for metadata and maybe more */ 570265425Snp struct sw_zone_info sw_zone_info[SW_ZONE_SIZES]; 571265425Snp struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES]; 572218792Snp}; 573218792Snp 574228561Snpstruct rss_header; 575228561Snptypedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, 576228561Snp struct mbuf *); 577237263Snptypedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); 578239336Snptypedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); 579228561Snp 580218792Snpstruct adapter { 581228561Snp SLIST_ENTRY(adapter) link; 582218792Snp device_t dev; 583218792Snp struct cdev *cdev; 584218792Snp 585218792Snp /* PCIe register resources */ 586218792Snp int regs_rid; 587218792Snp struct resource *regs_res; 588218792Snp int msix_rid; 589218792Snp struct resource *msix_res; 590218792Snp bus_space_handle_t bh; 591218792Snp bus_space_tag_t bt; 592218792Snp bus_size_t mmio_len; 593248925Snp int udbs_rid; 594248925Snp struct resource *udbs_res; 595248925Snp volatile uint8_t *udbs_base; 596218792Snp 597218792Snp unsigned int pf; 598218792Snp unsigned int mbox; 599218792Snp 600218792Snp /* Interrupt information */ 601218792Snp int intr_type; 602218792Snp int intr_count; 603218792Snp struct irq { 604218792Snp struct resource *res; 605218792Snp int rid; 606218792Snp void *tag; 607218792Snp } *irq; 608218792Snp 609218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 610218792Snp 611218792Snp struct sge sge; 612255015Snp int lro_timeout; 613218792Snp 614228561Snp struct taskqueue *tq[NCHAN]; /* taskqueues that flush data out */ 615218792Snp struct port_info *port[MAX_NPORTS]; 616218792Snp uint8_t chan_map[NCHAN]; 617218792Snp 618237263Snp#ifdef TCP_OFFLOAD 619237263Snp void *tom_softc; /* (struct tom_data *) */ 620228561Snp struct tom_tunables tt; 621255005Snp void *iwarp_softc; /* (struct c4iw_dev *) */ 622228561Snp#endif 623222509Snp struct l2t_data *l2t; /* L2 table */ 624218792Snp struct tid_info tids; 625218792Snp 626248925Snp int doorbells; 627218792Snp int open_device_map; 628237263Snp#ifdef TCP_OFFLOAD 629228561Snp int offload_map; 630228561Snp#endif 631218792Snp int flags; 632218792Snp 633253691Snp char ifp_lockname[16]; 634253691Snp struct mtx ifp_lock; 635253691Snp struct ifnet *ifp; /* tracer ifp */ 636253691Snp struct ifmedia media; 637253691Snp int traceq; /* iq used by all tracers, -1 if none */ 638253691Snp int tracer_valid; /* bitmap of valid tracers */ 639253691Snp int tracer_enabled; /* bitmap of enabled tracers */ 640253691Snp 641218792Snp char fw_version[32]; 642245936Snp char cfg_file[32]; 643245936Snp u_int cfcsum; 644218792Snp struct adapter_params params; 645218792Snp struct t4_virt_res vres; 646218792Snp 647228561Snp uint16_t linkcaps; 648228561Snp uint16_t niccaps; 649228561Snp uint16_t toecaps; 650228561Snp uint16_t rdmacaps; 651228561Snp uint16_t iscsicaps; 652228561Snp uint16_t fcoecaps; 653220873Snp 654228561Snp struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ 655228561Snp 656218792Snp struct mtx sc_lock; 657218792Snp char lockname[16]; 658228561Snp 659228561Snp /* Starving free lists */ 660228561Snp struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ 661228561Snp TAILQ_HEAD(, sge_fl) sfl; 662228561Snp struct callout sfl_callout; 663228561Snp 664237263Snp an_handler_t an_handler __aligned(CACHE_LINE_SIZE); 665247291Snp fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */ 666239336Snp cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */ 667245274Snp 668245274Snp#ifdef INVARIANTS 669245274Snp const char *last_op; 670245274Snp const void *last_op_thr; 671245274Snp#endif 672265421Snp 673265421Snp int sc_do_rxcopy; 674218792Snp}; 675218792Snp 676218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 677218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 678218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 679218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 680218792Snp 681245274Snp/* XXX: not bulletproof, but much better than nothing */ 682245274Snp#define ASSERT_SYNCHRONIZED_OP(sc) \ 683245274Snp KASSERT(IS_BUSY(sc) && \ 684245274Snp (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ 685245274Snp ("%s: operation not synchronized.", __func__)) 686245274Snp 687218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 688218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 689218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 690218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 691218792Snp 692218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 693218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 694218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 695218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 696218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 697218792Snp 698218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 699218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 700218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 701218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 702218792Snp 703218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 704218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 705218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 706218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 707218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 708218792Snp 709218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 710218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 711218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 712218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 713218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 714218792Snp 715245517Snp#define for_each_txq(pi, iter, q) \ 716245567Snp for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \ 717245567Snp iter < pi->ntxq; ++iter, ++q) 718245517Snp#define for_each_rxq(pi, iter, q) \ 719245567Snp for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \ 720245567Snp iter < pi->nrxq; ++iter, ++q) 721245517Snp#define for_each_ofld_txq(pi, iter, q) \ 722245567Snp for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \ 723245567Snp iter < pi->nofldtxq; ++iter, ++q) 724245517Snp#define for_each_ofld_rxq(pi, iter, q) \ 725245567Snp for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \ 726245567Snp iter < pi->nofldrxq; ++iter, ++q) 727218792Snp 728222510Snp/* One for errors, one for firmware events */ 729222510Snp#define T4_EXTRA_INTR 2 730218792Snp 731218792Snpstatic inline uint32_t 732218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 733218792Snp{ 734237263Snp 735218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 736218792Snp} 737218792Snp 738218792Snpstatic inline void 739218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 740218792Snp{ 741237263Snp 742218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 743218792Snp} 744218792Snp 745218792Snpstatic inline uint64_t 746218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 747218792Snp{ 748237263Snp 749218792Snp return t4_bus_space_read_8(sc->bt, sc->bh, reg); 750218792Snp} 751218792Snp 752218792Snpstatic inline void 753218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 754218792Snp{ 755237263Snp 756218792Snp t4_bus_space_write_8(sc->bt, sc->bh, reg, val); 757218792Snp} 758218792Snp 759218792Snpstatic inline void 760218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 761218792Snp{ 762237263Snp 763218792Snp *val = pci_read_config(sc->dev, reg, 1); 764218792Snp} 765218792Snp 766218792Snpstatic inline void 767218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 768218792Snp{ 769237263Snp 770218792Snp pci_write_config(sc->dev, reg, val, 1); 771218792Snp} 772218792Snp 773218792Snpstatic inline void 774218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 775218792Snp{ 776237263Snp 777218792Snp *val = pci_read_config(sc->dev, reg, 2); 778218792Snp} 779218792Snp 780218792Snpstatic inline void 781218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 782218792Snp{ 783237263Snp 784218792Snp pci_write_config(sc->dev, reg, val, 2); 785218792Snp} 786218792Snp 787218792Snpstatic inline void 788218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 789218792Snp{ 790237263Snp 791218792Snp *val = pci_read_config(sc->dev, reg, 4); 792218792Snp} 793218792Snp 794218792Snpstatic inline void 795218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 796218792Snp{ 797237263Snp 798218792Snp pci_write_config(sc->dev, reg, val, 4); 799218792Snp} 800218792Snp 801218792Snpstatic inline struct port_info * 802218792Snpadap2pinfo(struct adapter *sc, int idx) 803218792Snp{ 804237263Snp 805218792Snp return (sc->port[idx]); 806218792Snp} 807218792Snp 808218792Snpstatic inline void 809218792Snpt4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[]) 810218792Snp{ 811237263Snp 812218792Snp bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN); 813218792Snp} 814218792Snp 815248925Snpstatic inline bool 816248925Snpis_10G_port(const struct port_info *pi) 817218792Snp{ 818237263Snp 819218792Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0); 820218792Snp} 821218792Snp 822250092Snpstatic inline bool 823250092Snpis_40G_port(const struct port_info *pi) 824250092Snp{ 825250092Snp 826250092Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0); 827250092Snp} 828250092Snp 829248925Snpstatic inline int 830248925Snptx_resume_threshold(struct sge_eq *eq) 831228561Snp{ 832237263Snp 833228561Snp return (eq->qsize / 4); 834228561Snp} 835228561Snp 836219286Snp/* t4_main.c */ 837228561Snpvoid t4_tx_task(void *, int); 838228561Snpvoid t4_tx_callout(void *); 839218792Snpint t4_os_find_pci_capability(struct adapter *, int); 840218792Snpint t4_os_pci_save_state(struct adapter *); 841218792Snpint t4_os_pci_restore_state(struct adapter *); 842218792Snpvoid t4_os_portmod_changed(const struct adapter *, int); 843252747Snpvoid t4_os_link_changed(struct adapter *, int, int, int); 844228561Snpvoid t4_iterate(void (*)(struct adapter *, void *), void *); 845228561Snpint t4_register_cpl_handler(struct adapter *, int, cpl_handler_t); 846237263Snpint t4_register_an_handler(struct adapter *, an_handler_t); 847239336Snpint t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t); 848239338Snpint t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 849245274Snpint begin_synchronized_op(struct adapter *, struct port_info *, int, char *); 850245274Snpvoid end_synchronized_op(struct adapter *, int); 851218792Snp 852219286Snp/* t4_sge.c */ 853219392Snpvoid t4_sge_modload(void); 854269356Snpvoid t4_sge_modunload(void); 855269356Snpuint64_t t4_sge_extfree_refs(void); 856248925Snpvoid t4_init_sge_cpl_handlers(struct adapter *); 857248925Snpvoid t4_tweak_chip_settings(struct adapter *); 858248925Snpint t4_read_chip_settings(struct adapter *); 859218792Snpint t4_create_dma_tag(struct adapter *); 860253829Snpvoid t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, 861253829Snp struct sysctl_oid_list *); 862218792Snpint t4_destroy_dma_tag(struct adapter *); 863220873Snpint t4_setup_adapter_queues(struct adapter *); 864220873Snpint t4_teardown_adapter_queues(struct adapter *); 865228561Snpint t4_setup_port_queues(struct port_info *); 866228561Snpint t4_teardown_port_queues(struct port_info *); 867228561Snpint t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int); 868228561Snpvoid t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t); 869218792Snpvoid t4_intr_all(void *); 870222510Snpvoid t4_intr(void *); 871218792Snpvoid t4_intr_err(void *); 872218792Snpvoid t4_intr_evt(void *); 873237263Snpvoid t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); 874218792Snpint t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *); 875218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 876228561Snpint can_resume_tx(struct sge_eq *); 877218792Snp 878253691Snp/* t4_tracer.c */ 879253691Snpstruct t4_tracer; 880253691Snpvoid t4_tracer_modload(void); 881253691Snpvoid t4_tracer_modunload(void); 882253691Snpvoid t4_tracer_port_detach(struct adapter *); 883253691Snpint t4_get_tracer(struct adapter *, struct t4_tracer *); 884253691Snpint t4_set_tracer(struct adapter *, struct t4_tracer *); 885253691Snpint t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 886253691Snpint t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 887253691Snp 888237263Snpstatic inline struct wrqe * 889237263Snpalloc_wrqe(int wr_len, struct sge_wrq *wrq) 890228561Snp{ 891237263Snp int len = offsetof(struct wrqe, wr) + wr_len; 892237263Snp struct wrqe *wr; 893228561Snp 894237263Snp wr = malloc(len, M_CXGBE, M_NOWAIT); 895237263Snp if (__predict_false(wr == NULL)) 896237263Snp return (NULL); 897237263Snp wr->wr_len = wr_len; 898237263Snp wr->wrq = wrq; 899237263Snp return (wr); 900237263Snp} 901237263Snp 902237263Snpstatic inline void * 903237263Snpwrtod(struct wrqe *wr) 904237263Snp{ 905237263Snp return (&wr->wr[0]); 906237263Snp} 907237263Snp 908237263Snpstatic inline void 909237263Snpfree_wrqe(struct wrqe *wr) 910237263Snp{ 911237263Snp free(wr, M_CXGBE); 912237263Snp} 913237263Snp 914237263Snpstatic inline void 915237263Snpt4_wrq_tx(struct adapter *sc, struct wrqe *wr) 916237263Snp{ 917237263Snp struct sge_wrq *wrq = wr->wrq; 918237263Snp 919228561Snp TXQ_LOCK(wrq); 920237263Snp t4_wrq_tx_locked(sc, wrq, wr); 921228561Snp TXQ_UNLOCK(wrq); 922228561Snp} 923228561Snp 924218792Snp#endif 925