1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD$ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34228561Snp#include <sys/kernel.h> 35218792Snp#include <sys/bus.h> 36218792Snp#include <sys/rman.h> 37218792Snp#include <sys/types.h> 38218792Snp#include <sys/malloc.h> 39218792Snp#include <dev/pci/pcivar.h> 40218792Snp#include <dev/pci/pcireg.h> 41218792Snp#include <machine/bus.h> 42218792Snp#include <sys/socket.h> 43218792Snp#include <sys/sysctl.h> 44218792Snp#include <net/ethernet.h> 45218792Snp#include <net/if.h> 46218792Snp#include <net/if_media.h> 47235944Sbz#include <netinet/in.h> 48218792Snp#include <netinet/tcp_lro.h> 49218792Snp 50218792Snp#include "offload.h" 51270297Snp#include "common/t4_msg.h" 52228561Snp#include "firmware/t4fw_interface.h" 53218792Snp 54218792SnpMALLOC_DECLARE(M_CXGBE); 55218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 56218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 57218792Snp 58218792Snp#if defined(__i386__) || defined(__amd64__) 59218792Snpstatic __inline void 60218792Snpprefetch(void *x) 61218792Snp{ 62218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 63218792Snp} 64218792Snp#else 65218792Snp#define prefetch(x) 66218792Snp#endif 67218792Snp 68231115Snp#ifndef SYSCTL_ADD_UQUAD 69231115Snp#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 70231115Snp#define sysctl_handle_64 sysctl_handle_quad 71231115Snp#define CTLTYPE_U64 CTLTYPE_QUAD 72231115Snp#endif 73231115Snp 74231115Snp#if (__FreeBSD_version >= 900030) || \ 75231115Snp ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000)) 76231115Snp#define SBUF_DRAIN 1 77231115Snp#endif 78231115Snp 79218792Snp#ifdef __amd64__ 80218792Snp/* XXX: need systemwide bus_space_read_8/bus_space_write_8 */ 81218792Snpstatic __inline uint64_t 82218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 83218792Snp bus_size_t offset) 84218792Snp{ 85219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 86219285Snp ("%s: can only handle mem space", __func__)); 87218792Snp 88218792Snp return (*(volatile uint64_t *)(handle + offset)); 89218792Snp} 90218792Snp 91218792Snpstatic __inline void 92218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 93218792Snp bus_size_t offset, uint64_t value) 94218792Snp{ 95219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 96219285Snp ("%s: can only handle mem space", __func__)); 97219285Snp 98218792Snp *(volatile uint64_t *)(bsh + offset) = value; 99218792Snp} 100218792Snp#else 101218792Snpstatic __inline uint64_t 102218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 103218792Snp bus_size_t offset) 104218792Snp{ 105218792Snp return (uint64_t)bus_space_read_4(tag, handle, offset) + 106218792Snp ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32); 107218792Snp} 108218792Snp 109218792Snpstatic __inline void 110218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 111218792Snp bus_size_t offset, uint64_t value) 112218792Snp{ 113218792Snp bus_space_write_4(tag, bsh, offset, value); 114218792Snp bus_space_write_4(tag, bsh, offset + 4, value >> 32); 115218792Snp} 116218792Snp#endif 117218792Snp 118218792Snpstruct adapter; 119218792Snptypedef struct adapter adapter_t; 120218792Snp 121218792Snpenum { 122270297Snp /* 123270297Snp * All ingress queues use this entry size. Note that the firmware event 124270297Snp * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to 125270297Snp * be at least 64. 126270297Snp */ 127270297Snp IQ_ESIZE = 64, 128270297Snp 129270297Snp /* Default queue sizes for all kinds of ingress queues */ 130218792Snp FW_IQ_QSIZE = 256, 131218792Snp RX_IQ_QSIZE = 1024, 132218792Snp 133270297Snp /* All egress queues use this entry size */ 134270297Snp EQ_ESIZE = 64, 135218792Snp 136270297Snp /* Default queue sizes for all kinds of egress queues */ 137270297Snp CTRL_EQ_QSIZE = 128, 138270297Snp TX_EQ_QSIZE = 1024, 139270297Snp 140219392Snp#if MJUMPAGESIZE != MCLBYTES 141265425Snp SW_ZONE_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ 142219392Snp#else 143265425Snp SW_ZONE_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ 144219392Snp#endif 145265425Snp CL_METADATA_SIZE = CACHE_LINE_SIZE, 146218792Snp 147270297Snp SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */ 148218792Snp TX_SGL_SEGS = 36, 149218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 150218792Snp}; 151218792Snp 152218792Snpenum { 153219944Snp /* adapter intr_type */ 154219944Snp INTR_INTX = (1 << 0), 155219944Snp INTR_MSI = (1 << 1), 156219944Snp INTR_MSIX = (1 << 2) 157219944Snp}; 158219944Snp 159219944Snpenum { 160270297Snp XGMAC_MTU = (1 << 0), 161270297Snp XGMAC_PROMISC = (1 << 1), 162270297Snp XGMAC_ALLMULTI = (1 << 2), 163270297Snp XGMAC_VLANEX = (1 << 3), 164270297Snp XGMAC_UCADDR = (1 << 4), 165270297Snp XGMAC_MCADDRS = (1 << 5), 166270297Snp 167270297Snp XGMAC_ALL = 0xffff 168270297Snp}; 169270297Snp 170270297Snpenum { 171245274Snp /* flags understood by begin_synchronized_op */ 172245274Snp HOLD_LOCK = (1 << 0), 173245274Snp SLEEP_OK = (1 << 1), 174245274Snp INTR_OK = (1 << 2), 175245274Snp 176245274Snp /* flags understood by end_synchronized_op */ 177245274Snp LOCK_HELD = HOLD_LOCK, 178245274Snp}; 179245274Snp 180245274Snpenum { 181218792Snp /* adapter flags */ 182218792Snp FULL_INIT_DONE = (1 << 0), 183218792Snp FW_OK = (1 << 1), 184270297Snp /* INTR_DIRECT = (1 << 2), No longer used. */ 185228561Snp MASTER_PF = (1 << 3), 186228561Snp ADAP_SYSCTL_CTX = (1 << 4), 187237263Snp TOM_INIT_DONE = (1 << 5), 188255050Snp BUF_PACKING_OK = (1 << 6), 189218792Snp 190218792Snp CXGBE_BUSY = (1 << 9), 191218792Snp 192218792Snp /* port flags */ 193218792Snp DOOMED = (1 << 0), 194228561Snp PORT_INIT_DONE = (1 << 1), 195228561Snp PORT_SYSCTL_CTX = (1 << 2), 196253691Snp HAS_TRACEQ = (1 << 3), 197270297Snp INTR_RXQ = (1 << 4), /* All NIC rxq's take interrupts */ 198270297Snp INTR_OFLD_RXQ = (1 << 5), /* All TOE rxq's take interrupts */ 199270297Snp INTR_NM_RXQ = (1 << 6), /* All netmap rxq's take interrupts */ 200270297Snp INTR_ALL = (INTR_RXQ | INTR_OFLD_RXQ | INTR_NM_RXQ), 201218792Snp}; 202218792Snp 203245274Snp#define IS_DOOMED(pi) ((pi)->flags & DOOMED) 204245274Snp#define SET_DOOMED(pi) do {(pi)->flags |= DOOMED;} while (0) 205245274Snp#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) 206245274Snp#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) 207245274Snp#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) 208218792Snp 209218792Snpstruct port_info { 210218792Snp device_t dev; 211218792Snp struct adapter *adapter; 212218792Snp 213218792Snp struct ifnet *ifp; 214218792Snp struct ifmedia media; 215218792Snp 216218792Snp struct mtx pi_lock; 217218792Snp char lockname[16]; 218218792Snp unsigned long flags; 219218792Snp int if_flags; 220218792Snp 221259142Snp uint16_t *rss; 222218792Snp uint16_t viid; 223218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 224218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 225218792Snp uint8_t lport; /* associated offload logical port */ 226218792Snp int8_t mdio_addr; 227218792Snp uint8_t port_type; 228218792Snp uint8_t mod_type; 229218792Snp uint8_t port_id; 230218792Snp uint8_t tx_chan; 231265410Snp uint8_t rx_chan_map; /* rx MPS channel bitmap */ 232218792Snp 233218792Snp /* These need to be int as they are used in sysctl */ 234218792Snp int ntxq; /* # of tx queues */ 235218792Snp int first_txq; /* index of first tx queue */ 236264493Sscottl int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ 237218792Snp int nrxq; /* # of rx queues */ 238218792Snp int first_rxq; /* index of first rx queue */ 239237263Snp#ifdef TCP_OFFLOAD 240228561Snp int nofldtxq; /* # of offload tx queues */ 241228561Snp int first_ofld_txq; /* index of first offload tx queue */ 242228561Snp int nofldrxq; /* # of offload rx queues */ 243228561Snp int first_ofld_rxq; /* index of first offload rx queue */ 244228561Snp#endif 245270297Snp#ifdef DEV_NETMAP 246270297Snp int nnmtxq; /* # of netmap tx queues */ 247270297Snp int first_nm_txq; /* index of first netmap tx queue */ 248270297Snp int nnmrxq; /* # of netmap rx queues */ 249270297Snp int first_nm_rxq; /* index of first netmap rx queue */ 250270297Snp 251270297Snp struct ifnet *nm_ifp; 252270297Snp struct ifmedia nm_media; 253270297Snp int nmif_flags; 254270297Snp uint16_t nm_viid; 255270297Snp int16_t nm_xact_addr_filt; 256270297Snp uint16_t nm_rss_size; /* size of netmap VI's RSS table slice */ 257270297Snp#endif 258218792Snp int tmr_idx; 259218792Snp int pktc_idx; 260218792Snp int qsize_rxq; 261218792Snp int qsize_txq; 262218792Snp 263252747Snp int linkdnrc; 264218792Snp struct link_config link_cfg; 265218792Snp struct port_stats stats; 266218792Snp 267237263Snp eventhandler_tag vlan_c; 268237263Snp 269218792Snp struct callout tick; 270228561Snp struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ 271218792Snp 272218792Snp uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 273218792Snp}; 274218792Snp 275265425Snp/* Where the cluster came from, how it has been carved up. */ 276265425Snpstruct cluster_layout { 277265425Snp int8_t zidx; 278265425Snp int8_t hwidx; 279265425Snp uint16_t region1; /* mbufs laid out within this region */ 280265425Snp /* region2 is the DMA region */ 281265425Snp uint16_t region3; /* cluster_metadata within this region */ 282265425Snp}; 283265425Snp 284265425Snpstruct cluster_metadata { 285265425Snp u_int refcount; 286218792Snp#ifdef INVARIANTS 287265425Snp struct fl_sdesc *sd; /* For debug only. Could easily be stale */ 288218792Snp#endif 289218792Snp}; 290218792Snp 291265425Snpstruct fl_sdesc { 292265425Snp caddr_t cl; 293269356Snp uint16_t nmbuf; /* # of driver originated mbufs with ref on cluster */ 294265425Snp struct cluster_layout cll; 295265425Snp}; 296265425Snp 297218792Snpstruct tx_desc { 298218792Snp __be64 flit[8]; 299218792Snp}; 300218792Snp 301218792Snpstruct tx_map { 302218792Snp struct mbuf *m; 303218792Snp bus_dmamap_t map; 304218792Snp}; 305218792Snp 306228561Snp/* DMA maps used for tx */ 307228561Snpstruct tx_maps { 308228561Snp struct tx_map *maps; 309228561Snp uint32_t map_total; /* # of DMA maps */ 310228561Snp uint32_t map_pidx; /* next map to be used */ 311228561Snp uint32_t map_cidx; /* reclaimed up to this index */ 312228561Snp uint32_t map_avail; /* # of available maps */ 313228561Snp}; 314228561Snp 315218792Snpstruct tx_sdesc { 316218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 317220873Snp uint8_t credits; /* NIC txq: # of frames sent out in the WR */ 318218792Snp}; 319218792Snp 320270297Snp 321270297Snp#define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header)) 322270297Snpstruct iq_desc { 323270297Snp struct rss_header rss; 324270297Snp uint8_t cpl[IQ_PAD]; 325270297Snp struct rsp_ctrl rsp; 326270297Snp}; 327270297Snp#undef IQ_PAD 328270297SnpCTASSERT(sizeof(struct iq_desc) == IQ_ESIZE); 329270297Snp 330218792Snpenum { 331218792Snp /* iq flags */ 332228561Snp IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ 333228561Snp IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ 334228561Snp IQ_INTR = (1 << 2), /* iq takes direct interrupt */ 335228561Snp IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ 336220649Snp 337220649Snp /* iq state */ 338220649Snp IQS_DISABLED = 0, 339220649Snp IQS_BUSY = 1, 340220649Snp IQS_IDLE = 2, 341218792Snp}; 342218792Snp 343218792Snp/* 344218792Snp * Ingress Queue: T4 is producer, driver is consumer. 345218792Snp */ 346218792Snpstruct sge_iq { 347219290Snp uint32_t flags; 348228561Snp volatile int state; 349218792Snp struct adapter *adapter; 350270297Snp struct iq_desc *desc; /* KVA of descriptor ring */ 351270297Snp int8_t intr_pktc_idx; /* packet count threshold index */ 352218792Snp uint8_t gen; /* generation bit */ 353218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 354228561Snp uint8_t intr_next; /* XXX: holdoff for next interrupt */ 355218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 356270297Snp uint16_t sidx; /* index of the entry with the status page */ 357218792Snp uint16_t cidx; /* consumer index */ 358228561Snp uint16_t cntxt_id; /* SGE context id for the iq */ 359270297Snp uint16_t abs_id; /* absolute SGE id for the iq */ 360228561Snp 361228561Snp STAILQ_ENTRY(sge_iq) link; 362270297Snp 363270297Snp bus_dma_tag_t desc_tag; 364270297Snp bus_dmamap_t desc_map; 365270297Snp bus_addr_t ba; /* bus address of descriptor ring */ 366218792Snp}; 367218792Snp 368218792Snpenum { 369228561Snp EQ_CTRL = 1, 370228561Snp EQ_ETH = 2, 371237263Snp#ifdef TCP_OFFLOAD 372228561Snp EQ_OFLD = 3, 373228561Snp#endif 374228561Snp 375218792Snp /* eq flags */ 376228561Snp EQ_TYPEMASK = 7, /* 3 lsbits hold the type */ 377228561Snp EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */ 378228561Snp EQ_DOOMED = (1 << 4), /* about to be destroyed */ 379228561Snp EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */ 380228561Snp EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */ 381218792Snp}; 382218792Snp 383248925Snp/* Listed in order of preference. Update t4_sysctls too if you change these */ 384249392Snpenum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; 385248925Snp 386218792Snp/* 387218792Snp * Egress Queue: driver is producer, T4 is consumer. 388218792Snp * 389218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 390218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 391218792Snp */ 392218792Snpstruct sge_eq { 393228561Snp unsigned int flags; /* MUST be first */ 394228561Snp unsigned int cntxt_id; /* SGE context id for the eq */ 395218792Snp bus_dma_tag_t desc_tag; 396218792Snp bus_dmamap_t desc_map; 397218792Snp char lockname[16]; 398218792Snp struct mtx eq_lock; 399218792Snp 400218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 401218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 402218792Snp struct sge_qstat *spg; /* status page, for convenience */ 403270297Snp uint16_t doorbells; 404248925Snp volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ 405248925Snp u_int udb_qid; /* relative qid within the doorbell page */ 406218792Snp uint16_t cap; /* max # of desc, for convenience */ 407218792Snp uint16_t avail; /* available descriptors, for convenience */ 408218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 409218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 410218792Snp uint16_t pidx; /* producer idx (desc idx) */ 411218792Snp uint16_t pending; /* # of descriptors used since last doorbell */ 412219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 413228561Snp uint8_t tx_chan; /* tx channel used by the eq */ 414228561Snp struct task tx_task; 415228561Snp struct callout tx_callout; 416228561Snp 417228561Snp /* stats */ 418228561Snp 419228561Snp uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for eq */ 420228561Snp uint32_t unstalled; /* recovered from stall */ 421220873Snp}; 422218792Snp 423265425Snpstruct sw_zone_info { 424265425Snp uma_zone_t zone; /* zone that this cluster comes from */ 425265425Snp int size; /* size of cluster: 2K, 4K, 9K, 16K, etc. */ 426265425Snp int type; /* EXT_xxx type of the cluster */ 427265425Snp int8_t head_hwidx; 428265425Snp int8_t tail_hwidx; 429255050Snp}; 430255050Snp 431265425Snpstruct hw_buf_info { 432265425Snp int8_t zidx; /* backpointer to zone; -ve means unused */ 433265425Snp int8_t next; /* next hwidx for this zone; -1 means no more */ 434265425Snp int size; 435265425Snp}; 436265425Snp 437228561Snpenum { 438228561Snp FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ 439228561Snp FL_DOOMED = (1 << 1), /* about to be destroyed */ 440255050Snp FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ 441270297Snp FL_BUF_RESUME = (1 << 3), /* resume from the middle of the frame */ 442228561Snp}; 443228561Snp 444270297Snp#define FL_RUNNING_LOW(fl) \ 445270297Snp (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat) 446270297Snp#define FL_NOT_RUNNING_LOW(fl) \ 447270297Snp (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat) 448228561Snp 449218792Snpstruct sge_fl { 450218792Snp struct mtx fl_lock; 451218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 452218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 453270297Snp struct cluster_layout cll_def; /* default refill zone, layout */ 454270297Snp uint16_t lowat; /* # of buffers <= this means fl needs help */ 455270297Snp int flags; 456270297Snp uint16_t buf_boundary; 457265425Snp 458270297Snp /* The 16b idx all deal with hw descriptors */ 459270297Snp uint16_t dbidx; /* hw pidx after last doorbell */ 460270297Snp uint16_t sidx; /* index of status page */ 461270297Snp volatile uint16_t hw_cidx; 462265425Snp 463270297Snp /* The 32b idx are all buffer idx, not hardware descriptor idx */ 464270297Snp uint32_t cidx; /* consumer index */ 465270297Snp uint32_t pidx; /* producer index */ 466270297Snp 467270297Snp uint32_t dbval; 468270297Snp u_int rx_offset; /* offset in fl buf (when buffer packing) */ 469270297Snp volatile uint32_t *udb; 470270297Snp 471265425Snp uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */ 472265425Snp uint64_t mbuf_inlined; /* # of mbuf created within clusters */ 473265425Snp uint64_t cl_allocated; /* # of clusters allocated */ 474265425Snp uint64_t cl_recycled; /* # of clusters recycled */ 475265425Snp uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */ 476270297Snp 477270297Snp /* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */ 478270297Snp struct mbuf *m0; 479270297Snp struct mbuf **pnext; 480270297Snp u_int remaining; 481270297Snp 482270297Snp uint16_t qsize; /* # of hw descriptors (status page included) */ 483270297Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 484270297Snp TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ 485270297Snp bus_dma_tag_t desc_tag; 486270297Snp bus_dmamap_t desc_map; 487270297Snp char lockname[16]; 488270297Snp bus_addr_t ba; /* bus address of descriptor ring */ 489270297Snp struct cluster_layout cll_alt; /* alternate refill zone, layout */ 490218792Snp}; 491218792Snp 492220873Snp/* txq: SGE egress queue + what's needed for Ethernet NIC */ 493218792Snpstruct sge_txq { 494218792Snp struct sge_eq eq; /* MUST be first */ 495220873Snp 496220873Snp struct ifnet *ifp; /* the interface this txq belongs to */ 497220873Snp bus_dma_tag_t tx_tag; /* tag for transmit buffers */ 498220873Snp struct buf_ring *br; /* tx buffer ring */ 499220873Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 500218792Snp struct mbuf *m; /* held up due to temporary resource shortage */ 501218792Snp 502228561Snp struct tx_maps txmaps; 503219286Snp 504218792Snp /* stats for common events first */ 505218792Snp 506218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 507237819Snp uint64_t tso_wrs; /* # of TSO work requests */ 508218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 509218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 510218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 511218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 512218792Snp uint64_t txpkts_wrs; /* # of coalesced tx work requests */ 513218792Snp uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */ 514218792Snp 515218792Snp /* stats for not-that-common events */ 516218792Snp 517218792Snp uint32_t no_dmamap; /* no DMA map to load the mbuf */ 518218792Snp uint32_t no_desc; /* out of hardware descriptors */ 519220873Snp} __aligned(CACHE_LINE_SIZE); 520218792Snp 521218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 522218792Snpstruct sge_rxq { 523218792Snp struct sge_iq iq; /* MUST be first */ 524228561Snp struct sge_fl fl; /* MUST follow iq */ 525218792Snp 526219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 527237819Snp#if defined(INET) || defined(INET6) 528218792Snp struct lro_ctrl lro; /* LRO state */ 529219290Snp#endif 530218792Snp 531218792Snp /* stats for common events first */ 532218792Snp 533218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 534218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 535218792Snp 536218792Snp /* stats for not-that-common events */ 537218792Snp 538218792Snp} __aligned(CACHE_LINE_SIZE); 539218792Snp 540237263Snpstatic inline struct sge_rxq * 541237263Snpiq_to_rxq(struct sge_iq *iq) 542237263Snp{ 543237263Snp 544241733Sed return (__containerof(iq, struct sge_rxq, iq)); 545237263Snp} 546237263Snp 547237263Snp 548237263Snp#ifdef TCP_OFFLOAD 549228561Snp/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ 550228561Snpstruct sge_ofld_rxq { 551228561Snp struct sge_iq iq; /* MUST be first */ 552228561Snp struct sge_fl fl; /* MUST follow iq */ 553228561Snp} __aligned(CACHE_LINE_SIZE); 554237263Snp 555237263Snpstatic inline struct sge_ofld_rxq * 556237263Snpiq_to_ofld_rxq(struct sge_iq *iq) 557237263Snp{ 558237263Snp 559241733Sed return (__containerof(iq, struct sge_ofld_rxq, iq)); 560237263Snp} 561228561Snp#endif 562228561Snp 563237263Snpstruct wrqe { 564237263Snp STAILQ_ENTRY(wrqe) link; 565237263Snp struct sge_wrq *wrq; 566237263Snp int wr_len; 567237263Snp uint64_t wr[] __aligned(16); 568237263Snp}; 569237263Snp 570228561Snp/* 571228561Snp * wrq: SGE egress queue that is given prebuilt work requests. Both the control 572228561Snp * and offload tx queues are of this type. 573228561Snp */ 574228561Snpstruct sge_wrq { 575220873Snp struct sge_eq eq; /* MUST be first */ 576220873Snp 577228561Snp struct adapter *adapter; 578228561Snp 579237263Snp /* List of WRs held up due to lack of tx descriptors */ 580237263Snp STAILQ_HEAD(, wrqe) wr_list; 581237263Snp 582220873Snp /* stats for common events first */ 583220873Snp 584228561Snp uint64_t tx_wrs; /* # of tx work requests */ 585220873Snp 586220873Snp /* stats for not-that-common events */ 587220873Snp 588220873Snp uint32_t no_desc; /* out of hardware descriptors */ 589220873Snp} __aligned(CACHE_LINE_SIZE); 590220873Snp 591270297Snp 592270297Snp#ifdef DEV_NETMAP 593270297Snpstruct sge_nm_rxq { 594270297Snp struct port_info *pi; 595270297Snp 596270297Snp struct iq_desc *iq_desc; 597270297Snp uint16_t iq_abs_id; 598270297Snp uint16_t iq_cntxt_id; 599270297Snp uint16_t iq_cidx; 600270297Snp uint16_t iq_sidx; 601270297Snp uint8_t iq_gen; 602270297Snp 603270297Snp __be64 *fl_desc; 604270297Snp uint16_t fl_cntxt_id; 605270297Snp uint32_t fl_cidx; 606270297Snp uint32_t fl_pidx; 607270297Snp uint32_t fl_sidx; 608270297Snp uint32_t fl_db_val; 609270297Snp u_int fl_hwidx:4; 610270297Snp 611270297Snp u_int nid; /* netmap ring # for this queue */ 612270297Snp 613270297Snp /* infrequently used items after this */ 614270297Snp 615270297Snp bus_dma_tag_t iq_desc_tag; 616270297Snp bus_dmamap_t iq_desc_map; 617270297Snp bus_addr_t iq_ba; 618270297Snp int intr_idx; 619270297Snp 620270297Snp bus_dma_tag_t fl_desc_tag; 621270297Snp bus_dmamap_t fl_desc_map; 622270297Snp bus_addr_t fl_ba; 623270297Snp} __aligned(CACHE_LINE_SIZE); 624270297Snp 625270297Snpstruct sge_nm_txq { 626270297Snp struct tx_desc *desc; 627270297Snp uint16_t cidx; 628270297Snp uint16_t pidx; 629270297Snp uint16_t sidx; 630270297Snp uint16_t equiqidx; /* EQUIQ last requested at this pidx */ 631270297Snp uint16_t equeqidx; /* EQUEQ last requested at this pidx */ 632270297Snp uint16_t dbidx; /* pidx of the most recent doorbell */ 633270297Snp uint16_t doorbells; 634270297Snp volatile uint32_t *udb; 635270297Snp u_int udb_qid; 636270297Snp u_int cntxt_id; 637270297Snp __be32 cpl_ctrl0; /* for convenience */ 638270297Snp u_int nid; /* netmap ring # for this queue */ 639270297Snp 640270297Snp /* infrequently used items after this */ 641270297Snp 642270297Snp bus_dma_tag_t desc_tag; 643270297Snp bus_dmamap_t desc_map; 644270297Snp bus_addr_t ba; 645270297Snp int iqidx; 646270297Snp} __aligned(CACHE_LINE_SIZE); 647270297Snp#endif 648270297Snp 649218792Snpstruct sge { 650228561Snp int timer_val[SGE_NTIMERS]; 651228561Snp int counter_val[SGE_NCOUNTERS]; 652222701Snp int fl_starve_threshold; 653265410Snp int fl_starve_threshold2; 654256794Snp int eq_s_qpp; 655256794Snp int iq_s_qpp; 656218792Snp 657228561Snp int nrxq; /* total # of Ethernet rx queues */ 658228561Snp int ntxq; /* total # of Ethernet tx tx queues */ 659237263Snp#ifdef TCP_OFFLOAD 660228561Snp int nofldrxq; /* total # of TOE rx queues */ 661228561Snp int nofldtxq; /* total # of TOE tx queues */ 662228561Snp#endif 663270297Snp#ifdef DEV_NETMAP 664270297Snp int nnmrxq; /* total # of netmap rx queues */ 665270297Snp int nnmtxq; /* total # of netmap tx queues */ 666270297Snp#endif 667228561Snp int niq; /* total # of ingress queues */ 668228561Snp int neq; /* total # of egress queues */ 669218792Snp 670218792Snp struct sge_iq fwq; /* Firmware event queue */ 671228561Snp struct sge_wrq mgmtq; /* Management queue (control queue) */ 672228561Snp struct sge_wrq *ctrlq; /* Control queues */ 673218792Snp struct sge_txq *txq; /* NIC tx queues */ 674218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 675237263Snp#ifdef TCP_OFFLOAD 676228561Snp struct sge_wrq *ofld_txq; /* TOE tx queues */ 677228561Snp struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ 678228561Snp#endif 679270297Snp#ifdef DEV_NETMAP 680270297Snp struct sge_nm_txq *nm_txq; /* netmap tx queues */ 681270297Snp struct sge_nm_rxq *nm_rxq; /* netmap rx queues */ 682270297Snp#endif 683218792Snp 684218792Snp uint16_t iq_start; 685218792Snp int eq_start; 686218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 687218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 688255050Snp 689265425Snp int pack_boundary; 690265425Snp int8_t safe_hwidx1; /* may not have room for metadata */ 691265425Snp int8_t safe_hwidx2; /* with room for metadata and maybe more */ 692265425Snp struct sw_zone_info sw_zone_info[SW_ZONE_SIZES]; 693265425Snp struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES]; 694218792Snp}; 695218792Snp 696228561Snpstruct rss_header; 697228561Snptypedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, 698228561Snp struct mbuf *); 699237263Snptypedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); 700239336Snptypedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); 701228561Snp 702218792Snpstruct adapter { 703228561Snp SLIST_ENTRY(adapter) link; 704218792Snp device_t dev; 705218792Snp struct cdev *cdev; 706218792Snp 707218792Snp /* PCIe register resources */ 708218792Snp int regs_rid; 709218792Snp struct resource *regs_res; 710218792Snp int msix_rid; 711218792Snp struct resource *msix_res; 712218792Snp bus_space_handle_t bh; 713218792Snp bus_space_tag_t bt; 714218792Snp bus_size_t mmio_len; 715248925Snp int udbs_rid; 716248925Snp struct resource *udbs_res; 717248925Snp volatile uint8_t *udbs_base; 718218792Snp 719218792Snp unsigned int pf; 720218792Snp unsigned int mbox; 721218792Snp 722218792Snp /* Interrupt information */ 723218792Snp int intr_type; 724218792Snp int intr_count; 725218792Snp struct irq { 726218792Snp struct resource *res; 727218792Snp int rid; 728218792Snp void *tag; 729218792Snp } *irq; 730218792Snp 731218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 732218792Snp 733218792Snp struct sge sge; 734255015Snp int lro_timeout; 735218792Snp 736228561Snp struct taskqueue *tq[NCHAN]; /* taskqueues that flush data out */ 737218792Snp struct port_info *port[MAX_NPORTS]; 738218792Snp uint8_t chan_map[NCHAN]; 739218792Snp 740237263Snp#ifdef TCP_OFFLOAD 741237263Snp void *tom_softc; /* (struct tom_data *) */ 742228561Snp struct tom_tunables tt; 743255005Snp void *iwarp_softc; /* (struct c4iw_dev *) */ 744270297Snp void *iscsi_softc; 745228561Snp#endif 746222509Snp struct l2t_data *l2t; /* L2 table */ 747218792Snp struct tid_info tids; 748218792Snp 749270297Snp uint16_t doorbells; 750218792Snp int open_device_map; 751237263Snp#ifdef TCP_OFFLOAD 752228561Snp int offload_map; 753228561Snp#endif 754218792Snp int flags; 755218792Snp 756253691Snp char ifp_lockname[16]; 757253691Snp struct mtx ifp_lock; 758253691Snp struct ifnet *ifp; /* tracer ifp */ 759253691Snp struct ifmedia media; 760253691Snp int traceq; /* iq used by all tracers, -1 if none */ 761253691Snp int tracer_valid; /* bitmap of valid tracers */ 762253691Snp int tracer_enabled; /* bitmap of enabled tracers */ 763253691Snp 764218792Snp char fw_version[32]; 765245936Snp char cfg_file[32]; 766245936Snp u_int cfcsum; 767218792Snp struct adapter_params params; 768218792Snp struct t4_virt_res vres; 769218792Snp 770228561Snp uint16_t linkcaps; 771228561Snp uint16_t niccaps; 772228561Snp uint16_t toecaps; 773228561Snp uint16_t rdmacaps; 774228561Snp uint16_t iscsicaps; 775228561Snp uint16_t fcoecaps; 776220873Snp 777228561Snp struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ 778228561Snp 779218792Snp struct mtx sc_lock; 780218792Snp char lockname[16]; 781228561Snp 782228561Snp /* Starving free lists */ 783228561Snp struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ 784228561Snp TAILQ_HEAD(, sge_fl) sfl; 785228561Snp struct callout sfl_callout; 786228561Snp 787237263Snp an_handler_t an_handler __aligned(CACHE_LINE_SIZE); 788247291Snp fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */ 789239336Snp cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */ 790245274Snp 791245274Snp#ifdef INVARIANTS 792245274Snp const char *last_op; 793245274Snp const void *last_op_thr; 794245274Snp#endif 795265421Snp 796265421Snp int sc_do_rxcopy; 797218792Snp}; 798218792Snp 799218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 800218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 801218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 802218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 803218792Snp 804245274Snp/* XXX: not bulletproof, but much better than nothing */ 805245274Snp#define ASSERT_SYNCHRONIZED_OP(sc) \ 806245274Snp KASSERT(IS_BUSY(sc) && \ 807245274Snp (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ 808245274Snp ("%s: operation not synchronized.", __func__)) 809245274Snp 810218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 811218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 812218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 813218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 814218792Snp 815218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 816218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 817218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 818218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 819218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 820218792Snp 821218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 822218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 823218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 824218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 825218792Snp 826218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 827218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 828218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 829218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 830218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 831218792Snp 832218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 833218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 834218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 835218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 836218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 837218792Snp 838245517Snp#define for_each_txq(pi, iter, q) \ 839245567Snp for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \ 840245567Snp iter < pi->ntxq; ++iter, ++q) 841245517Snp#define for_each_rxq(pi, iter, q) \ 842245567Snp for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \ 843245567Snp iter < pi->nrxq; ++iter, ++q) 844245517Snp#define for_each_ofld_txq(pi, iter, q) \ 845245567Snp for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \ 846245567Snp iter < pi->nofldtxq; ++iter, ++q) 847245517Snp#define for_each_ofld_rxq(pi, iter, q) \ 848245567Snp for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \ 849245567Snp iter < pi->nofldrxq; ++iter, ++q) 850270297Snp#define for_each_nm_txq(pi, iter, q) \ 851270297Snp for (q = &pi->adapter->sge.nm_txq[pi->first_nm_txq], iter = 0; \ 852270297Snp iter < pi->nnmtxq; ++iter, ++q) 853270297Snp#define for_each_nm_rxq(pi, iter, q) \ 854270297Snp for (q = &pi->adapter->sge.nm_rxq[pi->first_nm_rxq], iter = 0; \ 855270297Snp iter < pi->nnmrxq; ++iter, ++q) 856218792Snp 857270297Snp#define IDXINCR(idx, incr, wrap) do { \ 858270297Snp idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \ 859270297Snp} while (0) 860270297Snp#define IDXDIFF(head, tail, wrap) \ 861270297Snp ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head)) 862270297Snp 863222510Snp/* One for errors, one for firmware events */ 864222510Snp#define T4_EXTRA_INTR 2 865218792Snp 866218792Snpstatic inline uint32_t 867218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 868218792Snp{ 869237263Snp 870218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 871218792Snp} 872218792Snp 873218792Snpstatic inline void 874218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 875218792Snp{ 876237263Snp 877218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 878218792Snp} 879218792Snp 880218792Snpstatic inline uint64_t 881218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 882218792Snp{ 883237263Snp 884218792Snp return t4_bus_space_read_8(sc->bt, sc->bh, reg); 885218792Snp} 886218792Snp 887218792Snpstatic inline void 888218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 889218792Snp{ 890237263Snp 891218792Snp t4_bus_space_write_8(sc->bt, sc->bh, reg, val); 892218792Snp} 893218792Snp 894218792Snpstatic inline void 895218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 896218792Snp{ 897237263Snp 898218792Snp *val = pci_read_config(sc->dev, reg, 1); 899218792Snp} 900218792Snp 901218792Snpstatic inline void 902218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 903218792Snp{ 904237263Snp 905218792Snp pci_write_config(sc->dev, reg, val, 1); 906218792Snp} 907218792Snp 908218792Snpstatic inline void 909218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 910218792Snp{ 911237263Snp 912218792Snp *val = pci_read_config(sc->dev, reg, 2); 913218792Snp} 914218792Snp 915218792Snpstatic inline void 916218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 917218792Snp{ 918237263Snp 919218792Snp pci_write_config(sc->dev, reg, val, 2); 920218792Snp} 921218792Snp 922218792Snpstatic inline void 923218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 924218792Snp{ 925237263Snp 926218792Snp *val = pci_read_config(sc->dev, reg, 4); 927218792Snp} 928218792Snp 929218792Snpstatic inline void 930218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 931218792Snp{ 932237263Snp 933218792Snp pci_write_config(sc->dev, reg, val, 4); 934218792Snp} 935218792Snp 936218792Snpstatic inline struct port_info * 937218792Snpadap2pinfo(struct adapter *sc, int idx) 938218792Snp{ 939237263Snp 940218792Snp return (sc->port[idx]); 941218792Snp} 942218792Snp 943218792Snpstatic inline void 944218792Snpt4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[]) 945218792Snp{ 946237263Snp 947218792Snp bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN); 948218792Snp} 949218792Snp 950248925Snpstatic inline bool 951248925Snpis_10G_port(const struct port_info *pi) 952218792Snp{ 953237263Snp 954218792Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0); 955218792Snp} 956218792Snp 957250092Snpstatic inline bool 958250092Snpis_40G_port(const struct port_info *pi) 959250092Snp{ 960250092Snp 961250092Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0); 962250092Snp} 963250092Snp 964248925Snpstatic inline int 965248925Snptx_resume_threshold(struct sge_eq *eq) 966228561Snp{ 967237263Snp 968228561Snp return (eq->qsize / 4); 969228561Snp} 970228561Snp 971219286Snp/* t4_main.c */ 972228561Snpvoid t4_tx_task(void *, int); 973228561Snpvoid t4_tx_callout(void *); 974218792Snpint t4_os_find_pci_capability(struct adapter *, int); 975218792Snpint t4_os_pci_save_state(struct adapter *); 976218792Snpint t4_os_pci_restore_state(struct adapter *); 977218792Snpvoid t4_os_portmod_changed(const struct adapter *, int); 978252747Snpvoid t4_os_link_changed(struct adapter *, int, int, int); 979228561Snpvoid t4_iterate(void (*)(struct adapter *, void *), void *); 980228561Snpint t4_register_cpl_handler(struct adapter *, int, cpl_handler_t); 981237263Snpint t4_register_an_handler(struct adapter *, an_handler_t); 982239336Snpint t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t); 983239338Snpint t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 984245274Snpint begin_synchronized_op(struct adapter *, struct port_info *, int, char *); 985245274Snpvoid end_synchronized_op(struct adapter *, int); 986270297Snpint update_mac_settings(struct ifnet *, int); 987270297Snpint adapter_full_init(struct adapter *); 988270297Snpint adapter_full_uninit(struct adapter *); 989270297Snpint port_full_init(struct port_info *); 990270297Snpint port_full_uninit(struct port_info *); 991218792Snp 992270297Snp#ifdef DEV_NETMAP 993270297Snp/* t4_netmap.c */ 994270297Snpint create_netmap_ifnet(struct port_info *); 995270297Snpint destroy_netmap_ifnet(struct port_info *); 996270297Snpvoid t4_nm_intr(void *); 997270297Snp#endif 998270297Snp 999219286Snp/* t4_sge.c */ 1000219392Snpvoid t4_sge_modload(void); 1001269356Snpvoid t4_sge_modunload(void); 1002269356Snpuint64_t t4_sge_extfree_refs(void); 1003248925Snpvoid t4_init_sge_cpl_handlers(struct adapter *); 1004248925Snpvoid t4_tweak_chip_settings(struct adapter *); 1005248925Snpint t4_read_chip_settings(struct adapter *); 1006218792Snpint t4_create_dma_tag(struct adapter *); 1007253829Snpvoid t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, 1008253829Snp struct sysctl_oid_list *); 1009218792Snpint t4_destroy_dma_tag(struct adapter *); 1010220873Snpint t4_setup_adapter_queues(struct adapter *); 1011220873Snpint t4_teardown_adapter_queues(struct adapter *); 1012228561Snpint t4_setup_port_queues(struct port_info *); 1013228561Snpint t4_teardown_port_queues(struct port_info *); 1014228561Snpint t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int); 1015228561Snpvoid t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t); 1016218792Snpvoid t4_intr_all(void *); 1017222510Snpvoid t4_intr(void *); 1018218792Snpvoid t4_intr_err(void *); 1019218792Snpvoid t4_intr_evt(void *); 1020237263Snpvoid t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); 1021218792Snpint t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *); 1022218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 1023228561Snpint can_resume_tx(struct sge_eq *); 1024218792Snp 1025253691Snp/* t4_tracer.c */ 1026253691Snpstruct t4_tracer; 1027253691Snpvoid t4_tracer_modload(void); 1028253691Snpvoid t4_tracer_modunload(void); 1029253691Snpvoid t4_tracer_port_detach(struct adapter *); 1030253691Snpint t4_get_tracer(struct adapter *, struct t4_tracer *); 1031253691Snpint t4_set_tracer(struct adapter *, struct t4_tracer *); 1032253691Snpint t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 1033253691Snpint t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 1034253691Snp 1035237263Snpstatic inline struct wrqe * 1036237263Snpalloc_wrqe(int wr_len, struct sge_wrq *wrq) 1037228561Snp{ 1038237263Snp int len = offsetof(struct wrqe, wr) + wr_len; 1039237263Snp struct wrqe *wr; 1040228561Snp 1041237263Snp wr = malloc(len, M_CXGBE, M_NOWAIT); 1042237263Snp if (__predict_false(wr == NULL)) 1043237263Snp return (NULL); 1044237263Snp wr->wr_len = wr_len; 1045237263Snp wr->wrq = wrq; 1046237263Snp return (wr); 1047237263Snp} 1048237263Snp 1049237263Snpstatic inline void * 1050237263Snpwrtod(struct wrqe *wr) 1051237263Snp{ 1052237263Snp return (&wr->wr[0]); 1053237263Snp} 1054237263Snp 1055237263Snpstatic inline void 1056237263Snpfree_wrqe(struct wrqe *wr) 1057237263Snp{ 1058237263Snp free(wr, M_CXGBE); 1059237263Snp} 1060237263Snp 1061237263Snpstatic inline void 1062237263Snpt4_wrq_tx(struct adapter *sc, struct wrqe *wr) 1063237263Snp{ 1064237263Snp struct sge_wrq *wrq = wr->wrq; 1065237263Snp 1066228561Snp TXQ_LOCK(wrq); 1067237263Snp t4_wrq_tx_locked(sc, wrq, wr); 1068228561Snp TXQ_UNLOCK(wrq); 1069228561Snp} 1070228561Snp 1071218792Snp#endif 1072