1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD: stable/11/sys/dev/cxgbe/adapter.h 346967 2019-04-30 17:30:37Z np $ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34228561Snp#include <sys/kernel.h> 35218792Snp#include <sys/bus.h> 36218792Snp#include <sys/rman.h> 37218792Snp#include <sys/types.h> 38257176Sglebius#include <sys/lock.h> 39218792Snp#include <sys/malloc.h> 40257176Sglebius#include <sys/rwlock.h> 41257176Sglebius#include <sys/sx.h> 42345664Sjhb#include <sys/vmem.h> 43257176Sglebius#include <vm/uma.h> 44257176Sglebius 45218792Snp#include <dev/pci/pcivar.h> 46218792Snp#include <dev/pci/pcireg.h> 47218792Snp#include <machine/bus.h> 48218792Snp#include <sys/socket.h> 49218792Snp#include <sys/sysctl.h> 50218792Snp#include <net/ethernet.h> 51218792Snp#include <net/if.h> 52257176Sglebius#include <net/if_var.h> 53218792Snp#include <net/if_media.h> 54235944Sbz#include <netinet/in.h> 55218792Snp#include <netinet/tcp_lro.h> 56218792Snp 57218792Snp#include "offload.h" 58301535Snp#include "t4_ioctl.h" 59266757Snp#include "common/t4_msg.h" 60228561Snp#include "firmware/t4fw_interface.h" 61218792Snp 62275733Snp#define KTR_CXGBE KTR_SPARE3 63218792SnpMALLOC_DECLARE(M_CXGBE); 64218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 65218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 66218792Snp 67218792Snp#if defined(__i386__) || defined(__amd64__) 68218792Snpstatic __inline void 69218792Snpprefetch(void *x) 70218792Snp{ 71218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 72218792Snp} 73218792Snp#else 74339396Snp#define prefetch(x) __builtin_prefetch(x) 75218792Snp#endif 76218792Snp 77231115Snp#ifndef SYSCTL_ADD_UQUAD 78231115Snp#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 79231115Snp#define sysctl_handle_64 sysctl_handle_quad 80231115Snp#define CTLTYPE_U64 CTLTYPE_QUAD 81231115Snp#endif 82231115Snp 83344858SjhbSYSCTL_DECL(_hw_cxgbe); 84344858Sjhb 85218792Snpstruct adapter; 86218792Snptypedef struct adapter adapter_t; 87218792Snp 88218792Snpenum { 89269411Snp /* 90269411Snp * All ingress queues use this entry size. Note that the firmware event 91269411Snp * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to 92269411Snp * be at least 64. 93269411Snp */ 94269411Snp IQ_ESIZE = 64, 95269411Snp 96269411Snp /* Default queue sizes for all kinds of ingress queues */ 97218792Snp FW_IQ_QSIZE = 256, 98218792Snp RX_IQ_QSIZE = 1024, 99218792Snp 100269411Snp /* All egress queues use this entry size */ 101269411Snp EQ_ESIZE = 64, 102218792Snp 103269411Snp /* Default queue sizes for all kinds of egress queues */ 104346876Snp CTRL_EQ_QSIZE = 1024, 105269411Snp TX_EQ_QSIZE = 1024, 106269411Snp 107219392Snp#if MJUMPAGESIZE != MCLBYTES 108263317Snp SW_ZONE_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ 109219392Snp#else 110263317Snp SW_ZONE_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ 111219392Snp#endif 112275554Snp CL_METADATA_SIZE = CACHE_LINE_SIZE, 113218792Snp 114269411Snp SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */ 115276485Snp TX_SGL_SEGS = 39, 116276485Snp TX_SGL_SEGS_TSO = 38, 117346928Snp TX_SGL_SEGS_EO_TSO = 30, /* XXX: lower for IPv6. */ 118218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 119218792Snp}; 120218792Snp 121218792Snpenum { 122219944Snp /* adapter intr_type */ 123219944Snp INTR_INTX = (1 << 0), 124219944Snp INTR_MSI = (1 << 1), 125219944Snp INTR_MSIX = (1 << 2) 126219944Snp}; 127219944Snp 128219944Snpenum { 129266757Snp XGMAC_MTU = (1 << 0), 130266757Snp XGMAC_PROMISC = (1 << 1), 131266757Snp XGMAC_ALLMULTI = (1 << 2), 132266757Snp XGMAC_VLANEX = (1 << 3), 133266757Snp XGMAC_UCADDR = (1 << 4), 134266757Snp XGMAC_MCADDRS = (1 << 5), 135266757Snp 136266757Snp XGMAC_ALL = 0xffff 137266757Snp}; 138266757Snp 139266757Snpenum { 140245274Snp /* flags understood by begin_synchronized_op */ 141245274Snp HOLD_LOCK = (1 << 0), 142245274Snp SLEEP_OK = (1 << 1), 143245274Snp INTR_OK = (1 << 2), 144245274Snp 145245274Snp /* flags understood by end_synchronized_op */ 146245274Snp LOCK_HELD = HOLD_LOCK, 147245274Snp}; 148245274Snp 149245274Snpenum { 150218792Snp /* adapter flags */ 151218792Snp FULL_INIT_DONE = (1 << 0), 152218792Snp FW_OK = (1 << 1), 153330307Snp CHK_MBOX_ACCESS = (1 << 2), 154228561Snp MASTER_PF = (1 << 3), 155228561Snp ADAP_SYSCTL_CTX = (1 << 4), 156346963Snp ADAP_ERR = (1 << 5), 157255050Snp BUF_PACKING_OK = (1 << 6), 158306664Sjhb IS_VF = (1 << 7), 159218792Snp 160218792Snp CXGBE_BUSY = (1 << 9), 161218792Snp 162218792Snp /* port flags */ 163291665Sjhb HAS_TRACEQ = (1 << 3), 164334562Snp FIXED_IFMEDIA = (1 << 4), /* ifmedia list doesn't change. */ 165291665Sjhb 166291665Sjhb /* VI flags */ 167218792Snp DOOMED = (1 << 0), 168291665Sjhb VI_INIT_DONE = (1 << 1), 169291665Sjhb VI_SYSCTL_CTX = (1 << 2), 170284445Snp 171284445Snp /* adapter debug_flags */ 172330307Snp DF_DUMP_MBOX = (1 << 0), /* Log all mbox cmd/rpl. */ 173330307Snp DF_LOAD_FW_ANYTIME = (1 << 1), /* Allow LOAD_FW after init */ 174330307Snp DF_DISABLE_TCB_CACHE = (1 << 2), /* Disable TCB cache (T6+) */ 175346942Snp DF_DISABLE_CFG_RETRY = (1 << 3), /* Disable fallback config */ 176346963Snp DF_VERBOSE_SLOWINTR = (1 << 4), /* Chatty slow intr handler */ 177218792Snp}; 178218792Snp 179291665Sjhb#define IS_DOOMED(vi) ((vi)->flags & DOOMED) 180291665Sjhb#define SET_DOOMED(vi) do {(vi)->flags |= DOOMED;} while (0) 181245274Snp#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) 182245274Snp#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) 183245274Snp#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) 184218792Snp 185291665Sjhbstruct vi_info { 186218792Snp device_t dev; 187291665Sjhb struct port_info *pi; 188218792Snp 189218792Snp struct ifnet *ifp; 190218792Snp 191218792Snp unsigned long flags; 192218792Snp int if_flags; 193218792Snp 194302110Snp uint16_t *rss, *nm_rss; 195346967Snp uint16_t viid; /* opaque VI identifier */ 196346967Snp uint16_t smt_idx; 197346967Snp uint16_t vin; 198346967Snp uint8_t vfvld; 199218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 200218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 201285648Snp uint16_t rss_base; /* start of VI's RSS table slice */ 202218792Snp 203291665Sjhb int nintr; 204291665Sjhb int first_intr; 205291665Sjhb 206218792Snp /* These need to be int as they are used in sysctl */ 207318854Snp int ntxq; /* # of tx queues */ 208318854Snp int first_txq; /* index of first tx queue */ 209318854Snp int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ 210318854Snp int nrxq; /* # of rx queues */ 211318854Snp int first_rxq; /* index of first rx queue */ 212228561Snp int nofldtxq; /* # of offload tx queues */ 213228561Snp int first_ofld_txq; /* index of first offload tx queue */ 214228561Snp int nofldrxq; /* # of offload rx queues */ 215228561Snp int first_ofld_rxq; /* index of first offload rx queue */ 216302110Snp int nnmtxq; 217302110Snp int first_nm_txq; 218302110Snp int nnmrxq; 219302110Snp int first_nm_rxq; 220218792Snp int tmr_idx; 221330307Snp int ofld_tmr_idx; 222218792Snp int pktc_idx; 223330307Snp int ofld_pktc_idx; 224218792Snp int qsize_rxq; 225218792Snp int qsize_txq; 226218792Snp 227291665Sjhb struct timeval last_refreshed; 228291665Sjhb struct fw_vi_stats_vf stats; 229291665Sjhb 230291665Sjhb struct callout tick; 231291665Sjhb struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ 232291665Sjhb 233291665Sjhb uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 234291665Sjhb}; 235291665Sjhb 236318850Snpstruct tx_ch_rl_params { 237318850Snp enum fw_sched_params_rate ratemode; /* %port (REL) or kbps (ABS) */ 238318850Snp uint32_t maxrate; 239318850Snp}; 240318850Snp 241301535Snpenum { 242346871Snp CLRL_USER = (1 << 0), /* allocated manually. */ 243346871Snp CLRL_SYNC = (1 << 1), /* sync hw update in progress. */ 244346871Snp CLRL_ASYNC = (1 << 2), /* async hw update requested. */ 245346871Snp CLRL_ERR = (1 << 3), /* last hw setup ended in error. */ 246301535Snp}; 247301535Snp 248318850Snpstruct tx_cl_rl_params { 249301535Snp int refcount; 250346871Snp uint8_t flags; 251318850Snp enum fw_sched_params_rate ratemode; /* %port REL or ABS value */ 252318850Snp enum fw_sched_params_unit rateunit; /* kbps or pps (when ABS) */ 253318850Snp enum fw_sched_params_mode mode; /* aggr or per-flow */ 254318850Snp uint32_t maxrate; 255318850Snp uint16_t pktsize; 256346871Snp uint16_t burstsize; 257301535Snp}; 258301535Snp 259318850Snp/* Tx scheduler parameters for a channel/port */ 260318850Snpstruct tx_sched_params { 261318850Snp /* Channel Rate Limiter */ 262318850Snp struct tx_ch_rl_params ch_rl; 263318850Snp 264318850Snp /* Class WRR */ 265318850Snp /* XXX */ 266318850Snp 267346871Snp /* Class Rate Limiter (including the default pktsize and burstsize). */ 268346871Snp int pktsize; 269346871Snp int burstsize; 270318850Snp struct tx_cl_rl_params cl_rl[]; 271318850Snp}; 272318850Snp 273291665Sjhbstruct port_info { 274291665Sjhb device_t dev; 275291665Sjhb struct adapter *adapter; 276291665Sjhb 277291665Sjhb struct vi_info *vi; 278291665Sjhb int nvi; 279291665Sjhb int up_vis; 280291665Sjhb int uld_vis; 281291665Sjhb 282318850Snp struct tx_sched_params *sched_params; 283301535Snp 284291665Sjhb struct mtx pi_lock; 285291665Sjhb char lockname[16]; 286291665Sjhb unsigned long flags; 287291665Sjhb 288291665Sjhb uint8_t lport; /* associated offload logical port */ 289291665Sjhb int8_t mdio_addr; 290291665Sjhb uint8_t port_type; 291291665Sjhb uint8_t mod_type; 292291665Sjhb uint8_t port_id; 293291665Sjhb uint8_t tx_chan; 294330307Snp uint8_t mps_bg_map; /* rx MPS buffer group bitmap */ 295330307Snp uint8_t rx_e_chan_map; /* rx TP e-channel bitmap */ 296291665Sjhb 297218792Snp struct link_config link_cfg; 298330307Snp struct ifmedia media; 299218792Snp 300272200Snp struct timeval last_refreshed; 301272200Snp struct port_stats stats; 302272200Snp u_int tnl_cong_drops; 303276485Snp u_int tx_parse_error; 304345664Sjhb u_long tx_tls_records; 305345664Sjhb u_long tx_tls_octets; 306345664Sjhb u_long rx_tls_records; 307345664Sjhb u_long rx_tls_octets; 308272200Snp 309218792Snp struct callout tick; 310218792Snp}; 311218792Snp 312291665Sjhb#define IS_MAIN_VI(vi) ((vi) == &((vi)->pi->vi[0])) 313291665Sjhb 314263317Snp/* Where the cluster came from, how it has been carved up. */ 315263317Snpstruct cluster_layout { 316263317Snp int8_t zidx; 317263317Snp int8_t hwidx; 318263317Snp uint16_t region1; /* mbufs laid out within this region */ 319263317Snp /* region2 is the DMA region */ 320263317Snp uint16_t region3; /* cluster_metadata within this region */ 321263317Snp}; 322263317Snp 323263317Snpstruct cluster_metadata { 324263317Snp u_int refcount; 325263317Snp struct fl_sdesc *sd; /* For debug only. Could easily be stale */ 326218792Snp}; 327218792Snp 328263317Snpstruct fl_sdesc { 329263317Snp caddr_t cl; 330268971Snp uint16_t nmbuf; /* # of driver originated mbufs with ref on cluster */ 331263317Snp struct cluster_layout cll; 332263317Snp}; 333263317Snp 334218792Snpstruct tx_desc { 335218792Snp __be64 flit[8]; 336218792Snp}; 337218792Snp 338218792Snpstruct tx_sdesc { 339276485Snp struct mbuf *m; /* m_nextpkt linked chain of frames */ 340218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 341218792Snp}; 342218792Snp 343269411Snp 344269411Snp#define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header)) 345269411Snpstruct iq_desc { 346269411Snp struct rss_header rss; 347269411Snp uint8_t cpl[IQ_PAD]; 348269411Snp struct rsp_ctrl rsp; 349269411Snp}; 350269411Snp#undef IQ_PAD 351269411SnpCTASSERT(sizeof(struct iq_desc) == IQ_ESIZE); 352269411Snp 353218792Snpenum { 354218792Snp /* iq flags */ 355228561Snp IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ 356228561Snp IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ 357330307Snp /* 1 << 2 Used to be IQ_INTR */ 358228561Snp IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ 359318842Snp IQ_ADJ_CREDIT = (1 << 4), /* hw is off by 1 credit for this iq */ 360220649Snp 361220649Snp /* iq state */ 362220649Snp IQS_DISABLED = 0, 363220649Snp IQS_BUSY = 1, 364220649Snp IQS_IDLE = 2, 365302110Snp 366302110Snp /* netmap related flags */ 367302110Snp NM_OFF = 0, 368302110Snp NM_ON = 1, 369302110Snp NM_BUSY = 2, 370218792Snp}; 371218792Snp 372346852Snpenum { 373346852Snp CPL_COOKIE_RESERVED = 0, 374346852Snp CPL_COOKIE_FILTER, 375346852Snp CPL_COOKIE_DDP0, 376346852Snp CPL_COOKIE_DDP1, 377346852Snp CPL_COOKIE_TOM, 378346855Snp CPL_COOKIE_HASHFILTER, 379346863Snp CPL_COOKIE_ETHOFLD, 380346852Snp CPL_COOKIE_AVAILABLE3, 381346852Snp 382346852Snp NUM_CPL_COOKIES = 8 /* Limited by M_COOKIE. Do not increase. */ 383346852Snp}; 384346852Snp 385302339Snpstruct sge_iq; 386302339Snpstruct rss_header; 387302339Snptypedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, 388302339Snp struct mbuf *); 389302339Snptypedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); 390302339Snptypedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); 391302339Snp 392218792Snp/* 393218792Snp * Ingress Queue: T4 is producer, driver is consumer. 394218792Snp */ 395218792Snpstruct sge_iq { 396219290Snp uint32_t flags; 397228561Snp volatile int state; 398218792Snp struct adapter *adapter; 399269411Snp struct iq_desc *desc; /* KVA of descriptor ring */ 400269411Snp int8_t intr_pktc_idx; /* packet count threshold index */ 401218792Snp uint8_t gen; /* generation bit */ 402218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 403228561Snp uint8_t intr_next; /* XXX: holdoff for next interrupt */ 404218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 405269411Snp uint16_t sidx; /* index of the entry with the status page */ 406218792Snp uint16_t cidx; /* consumer index */ 407228561Snp uint16_t cntxt_id; /* SGE context id for the iq */ 408269411Snp uint16_t abs_id; /* absolute SGE id for the iq */ 409228561Snp 410228561Snp STAILQ_ENTRY(sge_iq) link; 411269411Snp 412269411Snp bus_dma_tag_t desc_tag; 413269411Snp bus_dmamap_t desc_map; 414269411Snp bus_addr_t ba; /* bus address of descriptor ring */ 415218792Snp}; 416218792Snp 417218792Snpenum { 418228561Snp EQ_CTRL = 1, 419228561Snp EQ_ETH = 2, 420228561Snp EQ_OFLD = 3, 421228561Snp 422218792Snp /* eq flags */ 423276485Snp EQ_TYPEMASK = 0x3, /* 2 lsbits hold the type (see above) */ 424276485Snp EQ_ALLOCATED = (1 << 2), /* firmware resources allocated */ 425276485Snp EQ_ENABLED = (1 << 3), /* open for business */ 426318854Snp EQ_QFLUSH = (1 << 4), /* if_qflush in progress */ 427218792Snp}; 428218792Snp 429248925Snp/* Listed in order of preference. Update t4_sysctls too if you change these */ 430249392Snpenum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; 431248925Snp 432218792Snp/* 433218792Snp * Egress Queue: driver is producer, T4 is consumer. 434218792Snp * 435218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 436218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 437218792Snp */ 438218792Snpstruct sge_eq { 439228561Snp unsigned int flags; /* MUST be first */ 440228561Snp unsigned int cntxt_id; /* SGE context id for the eq */ 441306664Sjhb unsigned int abs_id; /* absolute SGE id for the eq */ 442218792Snp struct mtx eq_lock; 443218792Snp 444218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 445339396Snp uint8_t doorbells; 446248925Snp volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ 447248925Snp u_int udb_qid; /* relative qid within the doorbell page */ 448276485Snp uint16_t sidx; /* index of the entry with the status page */ 449218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 450218792Snp uint16_t pidx; /* producer idx (desc idx) */ 451276485Snp uint16_t equeqidx; /* EQUEQ last requested at this pidx */ 452276485Snp uint16_t dbidx; /* pidx of the most recent doorbell */ 453219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 454228561Snp uint8_t tx_chan; /* tx channel used by the eq */ 455276485Snp volatile u_int equiq; /* EQUIQ outstanding */ 456228561Snp 457276485Snp bus_dma_tag_t desc_tag; 458276485Snp bus_dmamap_t desc_map; 459276485Snp bus_addr_t ba; /* bus address of descriptor ring */ 460276485Snp char lockname[16]; 461220873Snp}; 462218792Snp 463263317Snpstruct sw_zone_info { 464263317Snp uma_zone_t zone; /* zone that this cluster comes from */ 465263317Snp int size; /* size of cluster: 2K, 4K, 9K, 16K, etc. */ 466263317Snp int type; /* EXT_xxx type of the cluster */ 467263317Snp int8_t head_hwidx; 468263317Snp int8_t tail_hwidx; 469255050Snp}; 470255050Snp 471263317Snpstruct hw_buf_info { 472263317Snp int8_t zidx; /* backpointer to zone; -ve means unused */ 473263317Snp int8_t next; /* next hwidx for this zone; -1 means no more */ 474263317Snp int size; 475263317Snp}; 476263317Snp 477228561Snpenum { 478296603Snp NUM_MEMWIN = 3, 479296603Snp 480296603Snp MEMWIN0_APERTURE = 2048, 481296603Snp MEMWIN0_BASE = 0x1b800, 482296603Snp 483296603Snp MEMWIN1_APERTURE = 32768, 484296603Snp MEMWIN1_BASE = 0x28000, 485296603Snp 486296603Snp MEMWIN2_APERTURE_T4 = 65536, 487296603Snp MEMWIN2_BASE_T4 = 0x30000, 488296603Snp 489296603Snp MEMWIN2_APERTURE_T5 = 128 * 1024, 490296603Snp MEMWIN2_BASE_T5 = 0x60000, 491296603Snp}; 492296603Snp 493296603Snpstruct memwin { 494296603Snp struct rwlock mw_lock __aligned(CACHE_LINE_SIZE); 495296603Snp uint32_t mw_base; /* constant after setup_memwin */ 496296603Snp uint32_t mw_aperture; /* ditto */ 497296603Snp uint32_t mw_curpos; /* protected by mw_lock */ 498296603Snp}; 499296603Snp 500296603Snpenum { 501228561Snp FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ 502228561Snp FL_DOOMED = (1 << 1), /* about to be destroyed */ 503255050Snp FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ 504269428Snp FL_BUF_RESUME = (1 << 3), /* resume from the middle of the frame */ 505228561Snp}; 506228561Snp 507269428Snp#define FL_RUNNING_LOW(fl) \ 508269428Snp (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat) 509269428Snp#define FL_NOT_RUNNING_LOW(fl) \ 510269428Snp (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat) 511228561Snp 512218792Snpstruct sge_fl { 513218792Snp struct mtx fl_lock; 514218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 515218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 516269428Snp struct cluster_layout cll_def; /* default refill zone, layout */ 517269428Snp uint16_t lowat; /* # of buffers <= this means fl needs help */ 518269428Snp int flags; 519269428Snp uint16_t buf_boundary; 520263317Snp 521269428Snp /* The 16b idx all deal with hw descriptors */ 522269428Snp uint16_t dbidx; /* hw pidx after last doorbell */ 523269428Snp uint16_t sidx; /* index of status page */ 524269428Snp volatile uint16_t hw_cidx; 525263317Snp 526269428Snp /* The 32b idx are all buffer idx, not hardware descriptor idx */ 527269428Snp uint32_t cidx; /* consumer index */ 528269428Snp uint32_t pidx; /* producer index */ 529269428Snp 530269428Snp uint32_t dbval; 531269428Snp u_int rx_offset; /* offset in fl buf (when buffer packing) */ 532269428Snp volatile uint32_t *udb; 533269428Snp 534263317Snp uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */ 535263317Snp uint64_t mbuf_inlined; /* # of mbuf created within clusters */ 536263317Snp uint64_t cl_allocated; /* # of clusters allocated */ 537263317Snp uint64_t cl_recycled; /* # of clusters recycled */ 538263317Snp uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */ 539269428Snp 540269428Snp /* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */ 541269428Snp struct mbuf *m0; 542269428Snp struct mbuf **pnext; 543269428Snp u_int remaining; 544269428Snp 545269428Snp uint16_t qsize; /* # of hw descriptors (status page included) */ 546269428Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 547269428Snp TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ 548269428Snp bus_dma_tag_t desc_tag; 549269428Snp bus_dmamap_t desc_map; 550269428Snp char lockname[16]; 551269428Snp bus_addr_t ba; /* bus address of descriptor ring */ 552269428Snp struct cluster_layout cll_alt; /* alternate refill zone, layout */ 553218792Snp}; 554218792Snp 555276485Snpstruct mp_ring; 556276485Snp 557220873Snp/* txq: SGE egress queue + what's needed for Ethernet NIC */ 558218792Snpstruct sge_txq { 559218792Snp struct sge_eq eq; /* MUST be first */ 560220873Snp 561220873Snp struct ifnet *ifp; /* the interface this txq belongs to */ 562276485Snp struct mp_ring *r; /* tx software ring */ 563220873Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 564276485Snp struct sglist *gl; 565276485Snp __be32 cpl_ctrl0; /* for convenience */ 566301628Snp int tc_idx; /* traffic class */ 567218792Snp 568276485Snp struct task tx_reclaim_task; 569218792Snp /* stats for common events first */ 570218792Snp 571218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 572237819Snp uint64_t tso_wrs; /* # of TSO work requests */ 573218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 574218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 575218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 576218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 577276485Snp uint64_t txpkts0_wrs; /* # of type0 coalesced tx work requests */ 578276485Snp uint64_t txpkts1_wrs; /* # of type1 coalesced tx work requests */ 579276485Snp uint64_t txpkts0_pkts; /* # of frames in type0 coalesced tx WRs */ 580276485Snp uint64_t txpkts1_pkts; /* # of frames in type1 coalesced tx WRs */ 581218792Snp 582218792Snp /* stats for not-that-common events */ 583220873Snp} __aligned(CACHE_LINE_SIZE); 584218792Snp 585218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 586218792Snpstruct sge_rxq { 587218792Snp struct sge_iq iq; /* MUST be first */ 588228561Snp struct sge_fl fl; /* MUST follow iq */ 589218792Snp 590219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 591237819Snp#if defined(INET) || defined(INET6) 592218792Snp struct lro_ctrl lro; /* LRO state */ 593219290Snp#endif 594218792Snp 595218792Snp /* stats for common events first */ 596218792Snp 597218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 598218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 599218792Snp 600218792Snp /* stats for not-that-common events */ 601218792Snp 602218792Snp} __aligned(CACHE_LINE_SIZE); 603218792Snp 604237263Snpstatic inline struct sge_rxq * 605237263Snpiq_to_rxq(struct sge_iq *iq) 606237263Snp{ 607237263Snp 608241733Sed return (__containerof(iq, struct sge_rxq, iq)); 609237263Snp} 610237263Snp 611237263Snp 612228561Snp/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ 613228561Snpstruct sge_ofld_rxq { 614228561Snp struct sge_iq iq; /* MUST be first */ 615228561Snp struct sge_fl fl; /* MUST follow iq */ 616228561Snp} __aligned(CACHE_LINE_SIZE); 617237263Snp 618237263Snpstatic inline struct sge_ofld_rxq * 619237263Snpiq_to_ofld_rxq(struct sge_iq *iq) 620237263Snp{ 621237263Snp 622241733Sed return (__containerof(iq, struct sge_ofld_rxq, iq)); 623237263Snp} 624228561Snp 625237263Snpstruct wrqe { 626237263Snp STAILQ_ENTRY(wrqe) link; 627237263Snp struct sge_wrq *wrq; 628237263Snp int wr_len; 629276485Snp char wr[] __aligned(16); 630237263Snp}; 631237263Snp 632276485Snpstruct wrq_cookie { 633276485Snp TAILQ_ENTRY(wrq_cookie) link; 634276485Snp int ndesc; 635276485Snp int pidx; 636276485Snp}; 637276485Snp 638228561Snp/* 639228561Snp * wrq: SGE egress queue that is given prebuilt work requests. Both the control 640228561Snp * and offload tx queues are of this type. 641228561Snp */ 642228561Snpstruct sge_wrq { 643220873Snp struct sge_eq eq; /* MUST be first */ 644220873Snp 645228561Snp struct adapter *adapter; 646276485Snp struct task wrq_tx_task; 647228561Snp 648276485Snp /* Tx desc reserved but WR not "committed" yet. */ 649276485Snp TAILQ_HEAD(wrq_incomplete_wrs , wrq_cookie) incomplete_wrs; 650276485Snp 651276485Snp /* List of WRs ready to go out as soon as descriptors are available. */ 652237263Snp STAILQ_HEAD(, wrqe) wr_list; 653276485Snp u_int nwr_pending; 654276485Snp u_int ndesc_needed; 655237263Snp 656220873Snp /* stats for common events first */ 657220873Snp 658276485Snp uint64_t tx_wrs_direct; /* # of WRs written directly to desc ring. */ 659276485Snp uint64_t tx_wrs_ss; /* # of WRs copied from scratch space. */ 660276485Snp uint64_t tx_wrs_copied; /* # of WRs queued and copied to desc ring. */ 661220873Snp 662220873Snp /* stats for not-that-common events */ 663220873Snp 664276485Snp /* 665276485Snp * Scratch space for work requests that wrap around after reaching the 666298955Spfg * status page, and some information about the last WR that used it. 667276485Snp */ 668276485Snp uint16_t ss_pidx; 669276485Snp uint16_t ss_len; 670276485Snp uint8_t ss[SGE_MAX_WR_LEN]; 671276485Snp 672220873Snp} __aligned(CACHE_LINE_SIZE); 673220873Snp 674266757Snp 675266757Snpstruct sge_nm_rxq { 676346875Snp volatile int nm_state; /* NM_OFF, NM_ON, or NM_BUSY */ 677291665Sjhb struct vi_info *vi; 678266757Snp 679269411Snp struct iq_desc *iq_desc; 680266757Snp uint16_t iq_abs_id; 681266757Snp uint16_t iq_cntxt_id; 682266757Snp uint16_t iq_cidx; 683266757Snp uint16_t iq_sidx; 684266757Snp uint8_t iq_gen; 685266757Snp 686266757Snp __be64 *fl_desc; 687266757Snp uint16_t fl_cntxt_id; 688266757Snp uint32_t fl_cidx; 689266757Snp uint32_t fl_pidx; 690266757Snp uint32_t fl_sidx; 691266757Snp uint32_t fl_db_val; 692266757Snp u_int fl_hwidx:4; 693266757Snp 694266757Snp u_int nid; /* netmap ring # for this queue */ 695266757Snp 696266757Snp /* infrequently used items after this */ 697266757Snp 698266757Snp bus_dma_tag_t iq_desc_tag; 699266757Snp bus_dmamap_t iq_desc_map; 700266757Snp bus_addr_t iq_ba; 701266757Snp int intr_idx; 702266757Snp 703266757Snp bus_dma_tag_t fl_desc_tag; 704266757Snp bus_dmamap_t fl_desc_map; 705266757Snp bus_addr_t fl_ba; 706266757Snp} __aligned(CACHE_LINE_SIZE); 707266757Snp 708266757Snpstruct sge_nm_txq { 709266757Snp struct tx_desc *desc; 710266757Snp uint16_t cidx; 711266757Snp uint16_t pidx; 712266757Snp uint16_t sidx; 713266757Snp uint16_t equiqidx; /* EQUIQ last requested at this pidx */ 714266757Snp uint16_t equeqidx; /* EQUEQ last requested at this pidx */ 715266757Snp uint16_t dbidx; /* pidx of the most recent doorbell */ 716339396Snp uint8_t doorbells; 717266757Snp volatile uint32_t *udb; 718266757Snp u_int udb_qid; 719266757Snp u_int cntxt_id; 720266757Snp __be32 cpl_ctrl0; /* for convenience */ 721266757Snp u_int nid; /* netmap ring # for this queue */ 722266757Snp 723266757Snp /* infrequently used items after this */ 724266757Snp 725266757Snp bus_dma_tag_t desc_tag; 726266757Snp bus_dmamap_t desc_map; 727266757Snp bus_addr_t ba; 728266757Snp int iqidx; 729266757Snp} __aligned(CACHE_LINE_SIZE); 730266757Snp 731218792Snpstruct sge { 732228561Snp int nrxq; /* total # of Ethernet rx queues */ 733318854Snp int ntxq; /* total # of Ethernet tx queues */ 734228561Snp int nofldrxq; /* total # of TOE rx queues */ 735228561Snp int nofldtxq; /* total # of TOE tx queues */ 736266757Snp int nnmrxq; /* total # of netmap rx queues */ 737266757Snp int nnmtxq; /* total # of netmap tx queues */ 738228561Snp int niq; /* total # of ingress queues */ 739228561Snp int neq; /* total # of egress queues */ 740218792Snp 741218792Snp struct sge_iq fwq; /* Firmware event queue */ 742228561Snp struct sge_wrq *ctrlq; /* Control queues */ 743218792Snp struct sge_txq *txq; /* NIC tx queues */ 744218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 745228561Snp struct sge_wrq *ofld_txq; /* TOE tx queues */ 746228561Snp struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ 747266757Snp struct sge_nm_txq *nm_txq; /* netmap tx queues */ 748266757Snp struct sge_nm_rxq *nm_rxq; /* netmap rx queues */ 749218792Snp 750306664Sjhb uint16_t iq_start; /* first cntxt_id */ 751306664Sjhb uint16_t iq_base; /* first abs_id */ 752306664Sjhb int eq_start; /* first cntxt_id */ 753306664Sjhb int eq_base; /* first abs_id */ 754218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 755218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 756255050Snp 757263317Snp int8_t safe_hwidx1; /* may not have room for metadata */ 758263317Snp int8_t safe_hwidx2; /* with room for metadata and maybe more */ 759263317Snp struct sw_zone_info sw_zone_info[SW_ZONE_SIZES]; 760263317Snp struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES]; 761218792Snp}; 762218792Snp 763309560Sjhbstruct devnames { 764309560Sjhb const char *nexus_name; 765309560Sjhb const char *ifnet_name; 766309560Sjhb const char *vi_ifnet_name; 767309560Sjhb const char *pf03_drv_name; 768309560Sjhb const char *vf_nexus_name; 769309560Sjhb const char *vf_ifnet_name; 770309560Sjhb}; 771309560Sjhb 772346934Snpstruct clip_entry; 773346934Snp 774218792Snpstruct adapter { 775228561Snp SLIST_ENTRY(adapter) link; 776218792Snp device_t dev; 777218792Snp struct cdev *cdev; 778309560Sjhb const struct devnames *names; 779218792Snp 780218792Snp /* PCIe register resources */ 781218792Snp int regs_rid; 782218792Snp struct resource *regs_res; 783218792Snp int msix_rid; 784218792Snp struct resource *msix_res; 785218792Snp bus_space_handle_t bh; 786218792Snp bus_space_tag_t bt; 787218792Snp bus_size_t mmio_len; 788248925Snp int udbs_rid; 789248925Snp struct resource *udbs_res; 790248925Snp volatile uint8_t *udbs_base; 791218792Snp 792218792Snp unsigned int pf; 793218792Snp unsigned int mbox; 794296489Snp unsigned int vpd_busy; 795296489Snp unsigned int vpd_flag; 796218792Snp 797218792Snp /* Interrupt information */ 798218792Snp int intr_type; 799218792Snp int intr_count; 800218792Snp struct irq { 801218792Snp struct resource *res; 802218792Snp int rid; 803218792Snp void *tag; 804302110Snp struct sge_rxq *rxq; 805302110Snp struct sge_nm_rxq *nm_rxq; 806302110Snp } __aligned(CACHE_LINE_SIZE) *irq; 807306664Sjhb int sge_gts_reg; 808306664Sjhb int sge_kdoorbell_reg; 809218792Snp 810218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 811218792Snp 812218792Snp struct sge sge; 813255015Snp int lro_timeout; 814302339Snp int sc_do_rxcopy; 815218792Snp 816296383Snp struct taskqueue *tq[MAX_NCHAN]; /* General purpose taskqueues */ 817218792Snp struct port_info *port[MAX_NPORTS]; 818330307Snp uint8_t chan_map[MAX_NCHAN]; /* channel -> port */ 819218792Snp 820346934Snp struct mtx clip_table_lock; 821346934Snp TAILQ_HEAD(, clip_entry) clip_table; 822346934Snp int clip_gen; 823346934Snp 824237263Snp void *tom_softc; /* (struct tom_data *) */ 825228561Snp struct tom_tunables tt; 826346805Snp struct t4_offload_policy *policy; 827346805Snp struct rwlock policy_lock; 828346805Snp 829346805Snp void *iwarp_softc; /* (struct c4iw_dev *) */ 830331769Shselasky struct iw_tunables iwt; 831292736Snp void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ 832345040Sjhb void *ccr_softc; /* (struct ccr_softc *) */ 833222509Snp struct l2t_data *l2t; /* L2 table */ 834346855Snp struct smt_data *smt; /* Source MAC Table */ 835218792Snp struct tid_info tids; 836345664Sjhb vmem_t *key_map; 837218792Snp 838339396Snp uint8_t doorbells; 839278374Snp int offload_map; /* ports with IFCAP_TOE enabled */ 840278374Snp int active_ulds; /* ULDs activated on this adapter */ 841218792Snp int flags; 842284445Snp int debug_flags; 843218792Snp 844253691Snp char ifp_lockname[16]; 845253691Snp struct mtx ifp_lock; 846253691Snp struct ifnet *ifp; /* tracer ifp */ 847253691Snp struct ifmedia media; 848253691Snp int traceq; /* iq used by all tracers, -1 if none */ 849253691Snp int tracer_valid; /* bitmap of valid tracers */ 850253691Snp int tracer_enabled; /* bitmap of enabled tracers */ 851253691Snp 852296641Snp char fw_version[16]; 853296641Snp char tp_version[16]; 854309458Sjhb char er_version[16]; 855309458Sjhb char bs_version[16]; 856245936Snp char cfg_file[32]; 857245936Snp u_int cfcsum; 858218792Snp struct adapter_params params; 859296383Snp const struct chip_params *chip_params; 860218792Snp struct t4_virt_res vres; 861218792Snp 862296710Snp uint16_t nbmcaps; 863228561Snp uint16_t linkcaps; 864296710Snp uint16_t switchcaps; 865228561Snp uint16_t niccaps; 866228561Snp uint16_t toecaps; 867228561Snp uint16_t rdmacaps; 868309560Sjhb uint16_t cryptocaps; 869228561Snp uint16_t iscsicaps; 870228561Snp uint16_t fcoecaps; 871220873Snp 872228561Snp struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ 873228561Snp 874218792Snp struct mtx sc_lock; 875218792Snp char lockname[16]; 876228561Snp 877228561Snp /* Starving free lists */ 878228561Snp struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ 879228561Snp TAILQ_HEAD(, sge_fl) sfl; 880228561Snp struct callout sfl_callout; 881228561Snp 882296552Snp struct mtx reg_lock; /* for indirect register access */ 883272200Snp 884296603Snp struct memwin memwin[NUM_MEMWIN]; /* memory windows */ 885296603Snp 886318850Snp struct mtx tc_lock; 887318850Snp struct task tc_task; 888318850Snp 889245274Snp const char *last_op; 890245274Snp const void *last_op_thr; 891286926Snp int last_op_flags; 892346963Snp 893346963Snp int swintr; 894218792Snp}; 895218792Snp 896218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 897218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 898218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 899218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 900218792Snp 901245274Snp#define ASSERT_SYNCHRONIZED_OP(sc) \ 902245274Snp KASSERT(IS_BUSY(sc) && \ 903245274Snp (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ 904245274Snp ("%s: operation not synchronized.", __func__)) 905245274Snp 906218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 907218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 908218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 909218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 910218792Snp 911218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 912218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 913218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 914218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 915218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 916218792Snp 917218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 918218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 919218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 920218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 921218792Snp 922218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 923218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 924218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 925218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 926218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 927218792Snp 928218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 929218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 930218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 931218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 932218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 933218792Snp 934291665Sjhb#define for_each_txq(vi, iter, q) \ 935291665Sjhb for (q = &vi->pi->adapter->sge.txq[vi->first_txq], iter = 0; \ 936291665Sjhb iter < vi->ntxq; ++iter, ++q) 937291665Sjhb#define for_each_rxq(vi, iter, q) \ 938291665Sjhb for (q = &vi->pi->adapter->sge.rxq[vi->first_rxq], iter = 0; \ 939291665Sjhb iter < vi->nrxq; ++iter, ++q) 940291665Sjhb#define for_each_ofld_txq(vi, iter, q) \ 941291665Sjhb for (q = &vi->pi->adapter->sge.ofld_txq[vi->first_ofld_txq], iter = 0; \ 942291665Sjhb iter < vi->nofldtxq; ++iter, ++q) 943291665Sjhb#define for_each_ofld_rxq(vi, iter, q) \ 944291665Sjhb for (q = &vi->pi->adapter->sge.ofld_rxq[vi->first_ofld_rxq], iter = 0; \ 945291665Sjhb iter < vi->nofldrxq; ++iter, ++q) 946291665Sjhb#define for_each_nm_txq(vi, iter, q) \ 947302110Snp for (q = &vi->pi->adapter->sge.nm_txq[vi->first_nm_txq], iter = 0; \ 948302110Snp iter < vi->nnmtxq; ++iter, ++q) 949291665Sjhb#define for_each_nm_rxq(vi, iter, q) \ 950302110Snp for (q = &vi->pi->adapter->sge.nm_rxq[vi->first_nm_rxq], iter = 0; \ 951302110Snp iter < vi->nnmrxq; ++iter, ++q) 952291665Sjhb#define for_each_vi(_pi, _iter, _vi) \ 953291665Sjhb for ((_vi) = (_pi)->vi, (_iter) = 0; (_iter) < (_pi)->nvi; \ 954291665Sjhb ++(_iter), ++(_vi)) 955218792Snp 956269428Snp#define IDXINCR(idx, incr, wrap) do { \ 957269428Snp idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \ 958269411Snp} while (0) 959269411Snp#define IDXDIFF(head, tail, wrap) \ 960269428Snp ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head)) 961269411Snp 962222510Snp/* One for errors, one for firmware events */ 963222510Snp#define T4_EXTRA_INTR 2 964218792Snp 965306664Sjhb/* One for firmware events */ 966306664Sjhb#define T4VF_EXTRA_INTR 1 967306664Sjhb 968330307Snpstatic inline int 969330307Snpforwarding_intr_to_fwq(struct adapter *sc) 970330307Snp{ 971330307Snp 972330307Snp return (sc->intr_count == 1); 973330307Snp} 974330307Snp 975218792Snpstatic inline uint32_t 976218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 977218792Snp{ 978237263Snp 979218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 980218792Snp} 981218792Snp 982218792Snpstatic inline void 983218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 984218792Snp{ 985237263Snp 986218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 987218792Snp} 988218792Snp 989218792Snpstatic inline uint64_t 990218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 991218792Snp{ 992237263Snp 993311260Snp#ifdef __LP64__ 994311260Snp return bus_space_read_8(sc->bt, sc->bh, reg); 995311260Snp#else 996311260Snp return (uint64_t)bus_space_read_4(sc->bt, sc->bh, reg) + 997311260Snp ((uint64_t)bus_space_read_4(sc->bt, sc->bh, reg + 4) << 32); 998311260Snp 999311260Snp#endif 1000218792Snp} 1001218792Snp 1002218792Snpstatic inline void 1003218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 1004218792Snp{ 1005237263Snp 1006311260Snp#ifdef __LP64__ 1007311260Snp bus_space_write_8(sc->bt, sc->bh, reg, val); 1008311260Snp#else 1009311260Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 1010311260Snp bus_space_write_4(sc->bt, sc->bh, reg + 4, val>> 32); 1011311260Snp#endif 1012218792Snp} 1013218792Snp 1014218792Snpstatic inline void 1015218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 1016218792Snp{ 1017237263Snp 1018218792Snp *val = pci_read_config(sc->dev, reg, 1); 1019218792Snp} 1020218792Snp 1021218792Snpstatic inline void 1022218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 1023218792Snp{ 1024237263Snp 1025218792Snp pci_write_config(sc->dev, reg, val, 1); 1026218792Snp} 1027218792Snp 1028218792Snpstatic inline void 1029218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 1030218792Snp{ 1031237263Snp 1032218792Snp *val = pci_read_config(sc->dev, reg, 2); 1033218792Snp} 1034218792Snp 1035218792Snpstatic inline void 1036218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 1037218792Snp{ 1038237263Snp 1039218792Snp pci_write_config(sc->dev, reg, val, 2); 1040218792Snp} 1041218792Snp 1042218792Snpstatic inline void 1043218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 1044218792Snp{ 1045237263Snp 1046218792Snp *val = pci_read_config(sc->dev, reg, 4); 1047218792Snp} 1048218792Snp 1049218792Snpstatic inline void 1050218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 1051218792Snp{ 1052237263Snp 1053218792Snp pci_write_config(sc->dev, reg, val, 4); 1054218792Snp} 1055218792Snp 1056218792Snpstatic inline struct port_info * 1057218792Snpadap2pinfo(struct adapter *sc, int idx) 1058218792Snp{ 1059237263Snp 1060218792Snp return (sc->port[idx]); 1061218792Snp} 1062218792Snp 1063218792Snpstatic inline void 1064330307Snpt4_os_set_hw_addr(struct port_info *pi, uint8_t hw_addr[]) 1065218792Snp{ 1066237263Snp 1067330307Snp bcopy(hw_addr, pi->vi[0].hw_addr, ETHER_ADDR_LEN); 1068218792Snp} 1069218792Snp 1070248925Snpstatic inline int 1071248925Snptx_resume_threshold(struct sge_eq *eq) 1072228561Snp{ 1073237263Snp 1074276485Snp /* not quite the same as qsize / 4, but this will do. */ 1075276485Snp return (eq->sidx / 4); 1076228561Snp} 1077228561Snp 1078296481Snpstatic inline int 1079296481Snpt4_use_ldst(struct adapter *sc) 1080296481Snp{ 1081296481Snp 1082296481Snp#ifdef notyet 1083296481Snp return (sc->flags & FW_OK || !sc->use_bd); 1084296481Snp#else 1085296481Snp return (0); 1086296481Snp#endif 1087296481Snp} 1088296481Snp 1089346963Snpstatic inline void 1090346963SnpCH_DUMP_MBOX(struct adapter *sc, int mbox, const int reg, 1091346963Snp const char *msg, const __be64 *const p, const bool err) 1092346963Snp{ 1093346963Snp 1094346963Snp if (!(sc->debug_flags & DF_DUMP_MBOX) && !err) 1095346963Snp return; 1096346963Snp if (p != NULL) { 1097346963Snp log(err ? LOG_ERR : LOG_DEBUG, 1098346963Snp "%s: mbox %u %s %016llx %016llx %016llx %016llx " 1099346963Snp "%016llx %016llx %016llx %016llx\n", 1100346963Snp device_get_nameunit(sc->dev), mbox, msg, 1101346963Snp (long long)be64_to_cpu(p[0]), (long long)be64_to_cpu(p[1]), 1102346963Snp (long long)be64_to_cpu(p[2]), (long long)be64_to_cpu(p[3]), 1103346963Snp (long long)be64_to_cpu(p[4]), (long long)be64_to_cpu(p[5]), 1104346963Snp (long long)be64_to_cpu(p[6]), (long long)be64_to_cpu(p[7])); 1105346963Snp } else { 1106346963Snp log(err ? LOG_ERR : LOG_DEBUG, 1107346963Snp "%s: mbox %u %s %016llx %016llx %016llx %016llx " 1108346963Snp "%016llx %016llx %016llx %016llx\n", 1109346963Snp device_get_nameunit(sc->dev), mbox, msg, 1110346963Snp (long long)t4_read_reg64(sc, reg), 1111346963Snp (long long)t4_read_reg64(sc, reg + 8), 1112346963Snp (long long)t4_read_reg64(sc, reg + 16), 1113346963Snp (long long)t4_read_reg64(sc, reg + 24), 1114346963Snp (long long)t4_read_reg64(sc, reg + 32), 1115346963Snp (long long)t4_read_reg64(sc, reg + 40), 1116346963Snp (long long)t4_read_reg64(sc, reg + 48), 1117346963Snp (long long)t4_read_reg64(sc, reg + 56)); 1118346963Snp } 1119346963Snp} 1120346963Snp 1121219286Snp/* t4_main.c */ 1122330307Snpextern int t4_ntxq; 1123330307Snpextern int t4_nrxq; 1124306664Sjhbextern int t4_intr_types; 1125330307Snpextern int t4_tmr_idx; 1126330307Snpextern int t4_pktc_idx; 1127306664Sjhbextern unsigned int t4_qsize_rxq; 1128306664Sjhbextern unsigned int t4_qsize_txq; 1129306664Sjhbextern device_method_t cxgbe_methods[]; 1130306664Sjhb 1131218792Snpint t4_os_find_pci_capability(struct adapter *, int); 1132218792Snpint t4_os_pci_save_state(struct adapter *); 1133218792Snpint t4_os_pci_restore_state(struct adapter *); 1134330307Snpvoid t4_os_portmod_changed(struct port_info *); 1135330307Snpvoid t4_os_link_changed(struct port_info *); 1136228561Snpvoid t4_iterate(void (*)(struct adapter *, void *), void *); 1137309560Sjhbvoid t4_init_devnames(struct adapter *); 1138306664Sjhbvoid t4_add_adapter(struct adapter *); 1139345664Sjhbvoid t4_aes_getdeckey(void *, const void *, unsigned int); 1140306664Sjhbint t4_detach_common(device_t); 1141306664Sjhbint t4_map_bars_0_and_4(struct adapter *); 1142306664Sjhbint t4_map_bar_2(struct adapter *); 1143306664Sjhbint t4_setup_intr_handlers(struct adapter *); 1144306664Sjhbvoid t4_sysctls(struct adapter *); 1145291665Sjhbint begin_synchronized_op(struct adapter *, struct vi_info *, int, char *); 1146291665Sjhbvoid doom_vi(struct adapter *, struct vi_info *); 1147245274Snpvoid end_synchronized_op(struct adapter *, int); 1148266757Snpint update_mac_settings(struct ifnet *, int); 1149266757Snpint adapter_full_init(struct adapter *); 1150266757Snpint adapter_full_uninit(struct adapter *); 1151291665Sjhbuint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 1152291665Sjhbint vi_full_init(struct vi_info *); 1153291665Sjhbint vi_full_uninit(struct vi_info *); 1154291665Sjhbvoid vi_sysctls(struct vi_info *); 1155291665Sjhbvoid vi_tick(void *); 1156346848Snpint rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int); 1157346849Snpint alloc_atid_tab(struct tid_info *, int); 1158346849Snpvoid free_atid_tab(struct tid_info *); 1159346849Snpint alloc_atid(struct adapter *, void *); 1160346849Snpvoid *lookup_atid(struct adapter *, int); 1161346849Snpvoid free_atid(struct adapter *, int); 1162346850Snpvoid release_tid(struct adapter *, int, struct sge_wrq *); 1163346883Snpint cxgbe_media_change(struct ifnet *); 1164346883Snpvoid cxgbe_media_status(struct ifnet *, struct ifmediareq *); 1165346963Snpbool t4_os_dump_cimla(struct adapter *, int, bool); 1166346963Snpvoid t4_os_dump_devlog(struct adapter *); 1167218792Snp 1168266757Snp#ifdef DEV_NETMAP 1169266757Snp/* t4_netmap.c */ 1170346875Snpstruct sge_nm_rxq; 1171302110Snpvoid cxgbe_nm_attach(struct vi_info *); 1172302110Snpvoid cxgbe_nm_detach(struct vi_info *); 1173346875Snpvoid service_nm_rxq(struct sge_nm_rxq *); 1174266757Snp#endif 1175266757Snp 1176219286Snp/* t4_sge.c */ 1177219392Snpvoid t4_sge_modload(void); 1178269032Snpvoid t4_sge_modunload(void); 1179269032Snpuint64_t t4_sge_extfree_refs(void); 1180248925Snpvoid t4_tweak_chip_settings(struct adapter *); 1181248925Snpint t4_read_chip_settings(struct adapter *); 1182218792Snpint t4_create_dma_tag(struct adapter *); 1183253829Snpvoid t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, 1184253829Snp struct sysctl_oid_list *); 1185218792Snpint t4_destroy_dma_tag(struct adapter *); 1186220873Snpint t4_setup_adapter_queues(struct adapter *); 1187220873Snpint t4_teardown_adapter_queues(struct adapter *); 1188291665Sjhbint t4_setup_vi_queues(struct vi_info *); 1189291665Sjhbint t4_teardown_vi_queues(struct vi_info *); 1190218792Snpvoid t4_intr_all(void *); 1191222510Snpvoid t4_intr(void *); 1192346875Snp#ifdef DEV_NETMAP 1193346875Snpvoid t4_nm_intr(void *); 1194302110Snpvoid t4_vi_intr(void *); 1195346875Snp#endif 1196218792Snpvoid t4_intr_err(void *); 1197218792Snpvoid t4_intr_evt(void *); 1198237263Snpvoid t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); 1199218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 1200306664Sjhbint parse_pkt(struct adapter *, struct mbuf **); 1201276485Snpvoid *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *); 1202276485Snpvoid commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *); 1203285221Snpint tnl_cong(struct port_info *, int); 1204346852Snpvoid t4_register_an_handler(an_handler_t); 1205346852Snpvoid t4_register_fw_msg_handler(int, fw_msg_handler_t); 1206346852Snpvoid t4_register_cpl_handler(int, cpl_handler_t); 1207346852Snpvoid t4_register_shared_cpl_handler(int, cpl_handler_t, int); 1208218792Snp 1209253691Snp/* t4_tracer.c */ 1210253691Snpstruct t4_tracer; 1211253691Snpvoid t4_tracer_modload(void); 1212253691Snpvoid t4_tracer_modunload(void); 1213253691Snpvoid t4_tracer_port_detach(struct adapter *); 1214253691Snpint t4_get_tracer(struct adapter *, struct t4_tracer *); 1215253691Snpint t4_set_tracer(struct adapter *, struct t4_tracer *); 1216253691Snpint t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 1217253691Snpint t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 1218253691Snp 1219318850Snp/* t4_sched.c */ 1220318850Snpint t4_set_sched_class(struct adapter *, struct t4_sched_params *); 1221318850Snpint t4_set_sched_queue(struct adapter *, struct t4_sched_queue *); 1222318850Snpint t4_init_tx_sched(struct adapter *); 1223318850Snpint t4_free_tx_sched(struct adapter *); 1224318850Snpvoid t4_update_tx_sched(struct adapter *); 1225318850Snpint t4_reserve_cl_rl_kbps(struct adapter *, int, u_int, int *); 1226346871Snpvoid t4_release_cl_rl(struct adapter *, int, int); 1227346871Snpint sysctl_tc(SYSCTL_HANDLER_ARGS); 1228346871Snpint sysctl_tc_params(SYSCTL_HANDLER_ARGS); 1229318850Snp 1230346855Snp/* t4_filter.c */ 1231346855Snpint get_filter_mode(struct adapter *, uint32_t *); 1232346855Snpint set_filter_mode(struct adapter *, uint32_t); 1233346855Snpint get_filter(struct adapter *, struct t4_filter *); 1234346855Snpint set_filter(struct adapter *, struct t4_filter *); 1235346855Snpint del_filter(struct adapter *, struct t4_filter *); 1236346855Snpint t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 1237346855Snpint t4_hashfilter_ao_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 1238346855Snpint t4_hashfilter_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 1239346855Snpint t4_del_hashfilter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 1240346877Snpvoid free_hftid_hash(struct tid_info *); 1241346855Snp 1242237263Snpstatic inline struct wrqe * 1243237263Snpalloc_wrqe(int wr_len, struct sge_wrq *wrq) 1244228561Snp{ 1245237263Snp int len = offsetof(struct wrqe, wr) + wr_len; 1246237263Snp struct wrqe *wr; 1247228561Snp 1248237263Snp wr = malloc(len, M_CXGBE, M_NOWAIT); 1249237263Snp if (__predict_false(wr == NULL)) 1250237263Snp return (NULL); 1251237263Snp wr->wr_len = wr_len; 1252237263Snp wr->wrq = wrq; 1253237263Snp return (wr); 1254237263Snp} 1255237263Snp 1256237263Snpstatic inline void * 1257237263Snpwrtod(struct wrqe *wr) 1258237263Snp{ 1259237263Snp return (&wr->wr[0]); 1260237263Snp} 1261237263Snp 1262237263Snpstatic inline void 1263237263Snpfree_wrqe(struct wrqe *wr) 1264237263Snp{ 1265237263Snp free(wr, M_CXGBE); 1266237263Snp} 1267237263Snp 1268237263Snpstatic inline void 1269237263Snpt4_wrq_tx(struct adapter *sc, struct wrqe *wr) 1270237263Snp{ 1271237263Snp struct sge_wrq *wrq = wr->wrq; 1272237263Snp 1273228561Snp TXQ_LOCK(wrq); 1274237263Snp t4_wrq_tx_locked(sc, wrq, wr); 1275228561Snp TXQ_UNLOCK(wrq); 1276228561Snp} 1277228561Snp 1278346848Snpstatic inline int 1279346848Snpread_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, 1280346848Snp int len) 1281346848Snp{ 1282346848Snp 1283346848Snp return (rw_via_memwin(sc, idx, addr, val, len, 0)); 1284346848Snp} 1285346848Snp 1286346848Snpstatic inline int 1287346848Snpwrite_via_memwin(struct adapter *sc, int idx, uint32_t addr, 1288346848Snp const uint32_t *val, int len) 1289346848Snp{ 1290346848Snp 1291346848Snp return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1)); 1292346848Snp} 1293218792Snp#endif 1294