adapter.h revision 296018
1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD: head/sys/dev/cxgbe/adapter.h 296018 2016-02-25 01:10:56Z np $ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34228561Snp#include <sys/kernel.h> 35218792Snp#include <sys/bus.h> 36218792Snp#include <sys/rman.h> 37218792Snp#include <sys/types.h> 38257176Sglebius#include <sys/lock.h> 39218792Snp#include <sys/malloc.h> 40257176Sglebius#include <sys/rwlock.h> 41257176Sglebius#include <sys/sx.h> 42257176Sglebius#include <vm/uma.h> 43257176Sglebius 44218792Snp#include <dev/pci/pcivar.h> 45218792Snp#include <dev/pci/pcireg.h> 46218792Snp#include <machine/bus.h> 47218792Snp#include <sys/socket.h> 48218792Snp#include <sys/sysctl.h> 49218792Snp#include <net/ethernet.h> 50218792Snp#include <net/if.h> 51257176Sglebius#include <net/if_var.h> 52218792Snp#include <net/if_media.h> 53235944Sbz#include <netinet/in.h> 54218792Snp#include <netinet/tcp_lro.h> 55218792Snp 56218792Snp#include "offload.h" 57266757Snp#include "common/t4_msg.h" 58228561Snp#include "firmware/t4fw_interface.h" 59218792Snp 60275733Snp#define KTR_CXGBE KTR_SPARE3 61218792SnpMALLOC_DECLARE(M_CXGBE); 62218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 63218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 64218792Snp 65218792Snp#if defined(__i386__) || defined(__amd64__) 66218792Snpstatic __inline void 67218792Snpprefetch(void *x) 68218792Snp{ 69218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 70218792Snp} 71218792Snp#else 72218792Snp#define prefetch(x) 73218792Snp#endif 74218792Snp 75231115Snp#ifndef SYSCTL_ADD_UQUAD 76231115Snp#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 77231115Snp#define sysctl_handle_64 sysctl_handle_quad 78231115Snp#define CTLTYPE_U64 CTLTYPE_QUAD 79231115Snp#endif 80231115Snp 81231115Snp#if (__FreeBSD_version >= 900030) || \ 82231115Snp ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000)) 83231115Snp#define SBUF_DRAIN 1 84231115Snp#endif 85231115Snp 86218792Snp#ifdef __amd64__ 87218792Snp/* XXX: need systemwide bus_space_read_8/bus_space_write_8 */ 88218792Snpstatic __inline uint64_t 89218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 90218792Snp bus_size_t offset) 91218792Snp{ 92219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 93219285Snp ("%s: can only handle mem space", __func__)); 94218792Snp 95218792Snp return (*(volatile uint64_t *)(handle + offset)); 96218792Snp} 97218792Snp 98218792Snpstatic __inline void 99218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 100218792Snp bus_size_t offset, uint64_t value) 101218792Snp{ 102219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 103219285Snp ("%s: can only handle mem space", __func__)); 104219285Snp 105218792Snp *(volatile uint64_t *)(bsh + offset) = value; 106218792Snp} 107218792Snp#else 108218792Snpstatic __inline uint64_t 109218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 110218792Snp bus_size_t offset) 111218792Snp{ 112218792Snp return (uint64_t)bus_space_read_4(tag, handle, offset) + 113218792Snp ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32); 114218792Snp} 115218792Snp 116218792Snpstatic __inline void 117218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 118218792Snp bus_size_t offset, uint64_t value) 119218792Snp{ 120218792Snp bus_space_write_4(tag, bsh, offset, value); 121218792Snp bus_space_write_4(tag, bsh, offset + 4, value >> 32); 122218792Snp} 123218792Snp#endif 124218792Snp 125218792Snpstruct adapter; 126218792Snptypedef struct adapter adapter_t; 127218792Snp 128218792Snpenum { 129269411Snp /* 130269411Snp * All ingress queues use this entry size. Note that the firmware event 131269411Snp * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to 132269411Snp * be at least 64. 133269411Snp */ 134269411Snp IQ_ESIZE = 64, 135269411Snp 136269411Snp /* Default queue sizes for all kinds of ingress queues */ 137218792Snp FW_IQ_QSIZE = 256, 138218792Snp RX_IQ_QSIZE = 1024, 139218792Snp 140269411Snp /* All egress queues use this entry size */ 141269411Snp EQ_ESIZE = 64, 142218792Snp 143269411Snp /* Default queue sizes for all kinds of egress queues */ 144269411Snp CTRL_EQ_QSIZE = 128, 145269411Snp TX_EQ_QSIZE = 1024, 146269411Snp 147219392Snp#if MJUMPAGESIZE != MCLBYTES 148263317Snp SW_ZONE_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ 149219392Snp#else 150263317Snp SW_ZONE_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ 151219392Snp#endif 152275554Snp CL_METADATA_SIZE = CACHE_LINE_SIZE, 153218792Snp 154269411Snp SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */ 155276485Snp TX_SGL_SEGS = 39, 156276485Snp TX_SGL_SEGS_TSO = 38, 157218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 158218792Snp}; 159218792Snp 160218792Snpenum { 161219944Snp /* adapter intr_type */ 162219944Snp INTR_INTX = (1 << 0), 163219944Snp INTR_MSI = (1 << 1), 164219944Snp INTR_MSIX = (1 << 2) 165219944Snp}; 166219944Snp 167219944Snpenum { 168266757Snp XGMAC_MTU = (1 << 0), 169266757Snp XGMAC_PROMISC = (1 << 1), 170266757Snp XGMAC_ALLMULTI = (1 << 2), 171266757Snp XGMAC_VLANEX = (1 << 3), 172266757Snp XGMAC_UCADDR = (1 << 4), 173266757Snp XGMAC_MCADDRS = (1 << 5), 174266757Snp 175266757Snp XGMAC_ALL = 0xffff 176266757Snp}; 177266757Snp 178266757Snpenum { 179245274Snp /* flags understood by begin_synchronized_op */ 180245274Snp HOLD_LOCK = (1 << 0), 181245274Snp SLEEP_OK = (1 << 1), 182245274Snp INTR_OK = (1 << 2), 183245274Snp 184245274Snp /* flags understood by end_synchronized_op */ 185245274Snp LOCK_HELD = HOLD_LOCK, 186245274Snp}; 187245274Snp 188245274Snpenum { 189218792Snp /* adapter flags */ 190218792Snp FULL_INIT_DONE = (1 << 0), 191218792Snp FW_OK = (1 << 1), 192266757Snp /* INTR_DIRECT = (1 << 2), No longer used. */ 193228561Snp MASTER_PF = (1 << 3), 194228561Snp ADAP_SYSCTL_CTX = (1 << 4), 195278374Snp /* TOM_INIT_DONE= (1 << 5), No longer used */ 196255050Snp BUF_PACKING_OK = (1 << 6), 197218792Snp 198218792Snp CXGBE_BUSY = (1 << 9), 199218792Snp 200218792Snp /* port flags */ 201291665Sjhb HAS_TRACEQ = (1 << 3), 202291665Sjhb 203291665Sjhb /* VI flags */ 204218792Snp DOOMED = (1 << 0), 205291665Sjhb VI_INIT_DONE = (1 << 1), 206291665Sjhb VI_SYSCTL_CTX = (1 << 2), 207266757Snp INTR_RXQ = (1 << 4), /* All NIC rxq's take interrupts */ 208266757Snp INTR_OFLD_RXQ = (1 << 5), /* All TOE rxq's take interrupts */ 209291665Sjhb INTR_ALL = (INTR_RXQ | INTR_OFLD_RXQ), 210291665Sjhb VI_NETMAP = (1 << 6), 211284445Snp 212284445Snp /* adapter debug_flags */ 213284445Snp DF_DUMP_MBOX = (1 << 0), 214218792Snp}; 215218792Snp 216291665Sjhb#define IS_DOOMED(vi) ((vi)->flags & DOOMED) 217291665Sjhb#define SET_DOOMED(vi) do {(vi)->flags |= DOOMED;} while (0) 218245274Snp#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) 219245274Snp#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) 220245274Snp#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) 221218792Snp 222291665Sjhbstruct vi_info { 223218792Snp device_t dev; 224291665Sjhb struct port_info *pi; 225218792Snp 226218792Snp struct ifnet *ifp; 227218792Snp struct ifmedia media; 228218792Snp 229218792Snp unsigned long flags; 230218792Snp int if_flags; 231218792Snp 232259103Snp uint16_t *rss; 233218792Snp uint16_t viid; 234218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 235218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 236285648Snp uint16_t rss_base; /* start of VI's RSS table slice */ 237218792Snp 238291665Sjhb eventhandler_tag vlan_c; 239291665Sjhb 240291665Sjhb int nintr; 241291665Sjhb int first_intr; 242291665Sjhb 243218792Snp /* These need to be int as they are used in sysctl */ 244218792Snp int ntxq; /* # of tx queues */ 245218792Snp int first_txq; /* index of first tx queue */ 246261558Sscottl int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */ 247218792Snp int nrxq; /* # of rx queues */ 248218792Snp int first_rxq; /* index of first rx queue */ 249237263Snp#ifdef TCP_OFFLOAD 250228561Snp int nofldtxq; /* # of offload tx queues */ 251228561Snp int first_ofld_txq; /* index of first offload tx queue */ 252228561Snp int nofldrxq; /* # of offload rx queues */ 253228561Snp int first_ofld_rxq; /* index of first offload rx queue */ 254228561Snp#endif 255218792Snp int tmr_idx; 256218792Snp int pktc_idx; 257218792Snp int qsize_rxq; 258218792Snp int qsize_txq; 259218792Snp 260291665Sjhb struct timeval last_refreshed; 261291665Sjhb struct fw_vi_stats_vf stats; 262291665Sjhb 263291665Sjhb struct callout tick; 264291665Sjhb struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ 265291665Sjhb 266291665Sjhb uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 267291665Sjhb}; 268291665Sjhb 269291665Sjhbstruct port_info { 270291665Sjhb device_t dev; 271291665Sjhb struct adapter *adapter; 272291665Sjhb 273291665Sjhb struct vi_info *vi; 274291665Sjhb int nvi; 275291665Sjhb int up_vis; 276291665Sjhb int uld_vis; 277291665Sjhb 278291665Sjhb struct mtx pi_lock; 279291665Sjhb char lockname[16]; 280291665Sjhb unsigned long flags; 281291665Sjhb 282291665Sjhb uint8_t lport; /* associated offload logical port */ 283291665Sjhb int8_t mdio_addr; 284291665Sjhb uint8_t port_type; 285291665Sjhb uint8_t mod_type; 286291665Sjhb uint8_t port_id; 287291665Sjhb uint8_t tx_chan; 288291665Sjhb uint8_t rx_chan_map; /* rx MPS channel bitmap */ 289291665Sjhb 290252747Snp int linkdnrc; 291218792Snp struct link_config link_cfg; 292218792Snp 293272200Snp struct timeval last_refreshed; 294272200Snp struct port_stats stats; 295272200Snp u_int tnl_cong_drops; 296276485Snp u_int tx_parse_error; 297272200Snp 298218792Snp struct callout tick; 299218792Snp}; 300218792Snp 301291665Sjhb#define IS_MAIN_VI(vi) ((vi) == &((vi)->pi->vi[0])) 302291665Sjhb 303263317Snp/* Where the cluster came from, how it has been carved up. */ 304263317Snpstruct cluster_layout { 305263317Snp int8_t zidx; 306263317Snp int8_t hwidx; 307263317Snp uint16_t region1; /* mbufs laid out within this region */ 308263317Snp /* region2 is the DMA region */ 309263317Snp uint16_t region3; /* cluster_metadata within this region */ 310263317Snp}; 311263317Snp 312263317Snpstruct cluster_metadata { 313263317Snp u_int refcount; 314218792Snp#ifdef INVARIANTS 315263317Snp struct fl_sdesc *sd; /* For debug only. Could easily be stale */ 316218792Snp#endif 317218792Snp}; 318218792Snp 319263317Snpstruct fl_sdesc { 320263317Snp caddr_t cl; 321268971Snp uint16_t nmbuf; /* # of driver originated mbufs with ref on cluster */ 322263317Snp struct cluster_layout cll; 323263317Snp}; 324263317Snp 325218792Snpstruct tx_desc { 326218792Snp __be64 flit[8]; 327218792Snp}; 328218792Snp 329218792Snpstruct tx_sdesc { 330276485Snp struct mbuf *m; /* m_nextpkt linked chain of frames */ 331218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 332218792Snp}; 333218792Snp 334269411Snp 335269411Snp#define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header)) 336269411Snpstruct iq_desc { 337269411Snp struct rss_header rss; 338269411Snp uint8_t cpl[IQ_PAD]; 339269411Snp struct rsp_ctrl rsp; 340269411Snp}; 341269411Snp#undef IQ_PAD 342269411SnpCTASSERT(sizeof(struct iq_desc) == IQ_ESIZE); 343269411Snp 344218792Snpenum { 345218792Snp /* iq flags */ 346228561Snp IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ 347228561Snp IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ 348228561Snp IQ_INTR = (1 << 2), /* iq takes direct interrupt */ 349228561Snp IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ 350220649Snp 351220649Snp /* iq state */ 352220649Snp IQS_DISABLED = 0, 353220649Snp IQS_BUSY = 1, 354220649Snp IQS_IDLE = 2, 355218792Snp}; 356218792Snp 357218792Snp/* 358218792Snp * Ingress Queue: T4 is producer, driver is consumer. 359218792Snp */ 360218792Snpstruct sge_iq { 361219290Snp uint32_t flags; 362228561Snp volatile int state; 363218792Snp struct adapter *adapter; 364269411Snp struct iq_desc *desc; /* KVA of descriptor ring */ 365269411Snp int8_t intr_pktc_idx; /* packet count threshold index */ 366218792Snp uint8_t gen; /* generation bit */ 367218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 368228561Snp uint8_t intr_next; /* XXX: holdoff for next interrupt */ 369218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 370269411Snp uint16_t sidx; /* index of the entry with the status page */ 371218792Snp uint16_t cidx; /* consumer index */ 372228561Snp uint16_t cntxt_id; /* SGE context id for the iq */ 373269411Snp uint16_t abs_id; /* absolute SGE id for the iq */ 374228561Snp 375228561Snp STAILQ_ENTRY(sge_iq) link; 376269411Snp 377269411Snp bus_dma_tag_t desc_tag; 378269411Snp bus_dmamap_t desc_map; 379269411Snp bus_addr_t ba; /* bus address of descriptor ring */ 380218792Snp}; 381218792Snp 382218792Snpenum { 383228561Snp EQ_CTRL = 1, 384228561Snp EQ_ETH = 2, 385228561Snp EQ_OFLD = 3, 386228561Snp 387218792Snp /* eq flags */ 388276485Snp EQ_TYPEMASK = 0x3, /* 2 lsbits hold the type (see above) */ 389276485Snp EQ_ALLOCATED = (1 << 2), /* firmware resources allocated */ 390276485Snp EQ_ENABLED = (1 << 3), /* open for business */ 391218792Snp}; 392218792Snp 393248925Snp/* Listed in order of preference. Update t4_sysctls too if you change these */ 394249392Snpenum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; 395248925Snp 396218792Snp/* 397218792Snp * Egress Queue: driver is producer, T4 is consumer. 398218792Snp * 399218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 400218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 401218792Snp */ 402218792Snpstruct sge_eq { 403228561Snp unsigned int flags; /* MUST be first */ 404228561Snp unsigned int cntxt_id; /* SGE context id for the eq */ 405218792Snp struct mtx eq_lock; 406218792Snp 407218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 408266757Snp uint16_t doorbells; 409248925Snp volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ 410248925Snp u_int udb_qid; /* relative qid within the doorbell page */ 411276485Snp uint16_t sidx; /* index of the entry with the status page */ 412218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 413218792Snp uint16_t pidx; /* producer idx (desc idx) */ 414276485Snp uint16_t equeqidx; /* EQUEQ last requested at this pidx */ 415276485Snp uint16_t dbidx; /* pidx of the most recent doorbell */ 416219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 417228561Snp uint8_t tx_chan; /* tx channel used by the eq */ 418276485Snp volatile u_int equiq; /* EQUIQ outstanding */ 419228561Snp 420276485Snp bus_dma_tag_t desc_tag; 421276485Snp bus_dmamap_t desc_map; 422276485Snp bus_addr_t ba; /* bus address of descriptor ring */ 423276485Snp char lockname[16]; 424220873Snp}; 425218792Snp 426263317Snpstruct sw_zone_info { 427263317Snp uma_zone_t zone; /* zone that this cluster comes from */ 428263317Snp int size; /* size of cluster: 2K, 4K, 9K, 16K, etc. */ 429263317Snp int type; /* EXT_xxx type of the cluster */ 430263317Snp int8_t head_hwidx; 431263317Snp int8_t tail_hwidx; 432255050Snp}; 433255050Snp 434263317Snpstruct hw_buf_info { 435263317Snp int8_t zidx; /* backpointer to zone; -ve means unused */ 436263317Snp int8_t next; /* next hwidx for this zone; -1 means no more */ 437263317Snp int size; 438263317Snp}; 439263317Snp 440228561Snpenum { 441228561Snp FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ 442228561Snp FL_DOOMED = (1 << 1), /* about to be destroyed */ 443255050Snp FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ 444269428Snp FL_BUF_RESUME = (1 << 3), /* resume from the middle of the frame */ 445228561Snp}; 446228561Snp 447269428Snp#define FL_RUNNING_LOW(fl) \ 448269428Snp (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat) 449269428Snp#define FL_NOT_RUNNING_LOW(fl) \ 450269428Snp (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat) 451228561Snp 452218792Snpstruct sge_fl { 453218792Snp struct mtx fl_lock; 454218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 455218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 456269428Snp struct cluster_layout cll_def; /* default refill zone, layout */ 457269428Snp uint16_t lowat; /* # of buffers <= this means fl needs help */ 458269428Snp int flags; 459269428Snp uint16_t buf_boundary; 460263317Snp 461269428Snp /* The 16b idx all deal with hw descriptors */ 462269428Snp uint16_t dbidx; /* hw pidx after last doorbell */ 463269428Snp uint16_t sidx; /* index of status page */ 464269428Snp volatile uint16_t hw_cidx; 465263317Snp 466269428Snp /* The 32b idx are all buffer idx, not hardware descriptor idx */ 467269428Snp uint32_t cidx; /* consumer index */ 468269428Snp uint32_t pidx; /* producer index */ 469269428Snp 470269428Snp uint32_t dbval; 471269428Snp u_int rx_offset; /* offset in fl buf (when buffer packing) */ 472269428Snp volatile uint32_t *udb; 473269428Snp 474263317Snp uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */ 475263317Snp uint64_t mbuf_inlined; /* # of mbuf created within clusters */ 476263317Snp uint64_t cl_allocated; /* # of clusters allocated */ 477263317Snp uint64_t cl_recycled; /* # of clusters recycled */ 478263317Snp uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */ 479269428Snp 480269428Snp /* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */ 481269428Snp struct mbuf *m0; 482269428Snp struct mbuf **pnext; 483269428Snp u_int remaining; 484269428Snp 485269428Snp uint16_t qsize; /* # of hw descriptors (status page included) */ 486269428Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 487269428Snp TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ 488269428Snp bus_dma_tag_t desc_tag; 489269428Snp bus_dmamap_t desc_map; 490269428Snp char lockname[16]; 491269428Snp bus_addr_t ba; /* bus address of descriptor ring */ 492269428Snp struct cluster_layout cll_alt; /* alternate refill zone, layout */ 493218792Snp}; 494218792Snp 495276485Snpstruct mp_ring; 496276485Snp 497220873Snp/* txq: SGE egress queue + what's needed for Ethernet NIC */ 498218792Snpstruct sge_txq { 499218792Snp struct sge_eq eq; /* MUST be first */ 500220873Snp 501220873Snp struct ifnet *ifp; /* the interface this txq belongs to */ 502276485Snp struct mp_ring *r; /* tx software ring */ 503220873Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 504276485Snp struct sglist *gl; 505276485Snp __be32 cpl_ctrl0; /* for convenience */ 506218792Snp 507276485Snp struct task tx_reclaim_task; 508218792Snp /* stats for common events first */ 509218792Snp 510218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 511237819Snp uint64_t tso_wrs; /* # of TSO work requests */ 512218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 513218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 514218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 515218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 516276485Snp uint64_t txpkts0_wrs; /* # of type0 coalesced tx work requests */ 517276485Snp uint64_t txpkts1_wrs; /* # of type1 coalesced tx work requests */ 518276485Snp uint64_t txpkts0_pkts; /* # of frames in type0 coalesced tx WRs */ 519276485Snp uint64_t txpkts1_pkts; /* # of frames in type1 coalesced tx WRs */ 520218792Snp 521218792Snp /* stats for not-that-common events */ 522220873Snp} __aligned(CACHE_LINE_SIZE); 523218792Snp 524218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 525218792Snpstruct sge_rxq { 526218792Snp struct sge_iq iq; /* MUST be first */ 527228561Snp struct sge_fl fl; /* MUST follow iq */ 528218792Snp 529219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 530237819Snp#if defined(INET) || defined(INET6) 531218792Snp struct lro_ctrl lro; /* LRO state */ 532219290Snp#endif 533218792Snp 534218792Snp /* stats for common events first */ 535218792Snp 536218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 537218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 538218792Snp 539218792Snp /* stats for not-that-common events */ 540218792Snp 541218792Snp} __aligned(CACHE_LINE_SIZE); 542218792Snp 543237263Snpstatic inline struct sge_rxq * 544237263Snpiq_to_rxq(struct sge_iq *iq) 545237263Snp{ 546237263Snp 547241733Sed return (__containerof(iq, struct sge_rxq, iq)); 548237263Snp} 549237263Snp 550237263Snp 551237263Snp#ifdef TCP_OFFLOAD 552228561Snp/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ 553228561Snpstruct sge_ofld_rxq { 554228561Snp struct sge_iq iq; /* MUST be first */ 555228561Snp struct sge_fl fl; /* MUST follow iq */ 556228561Snp} __aligned(CACHE_LINE_SIZE); 557237263Snp 558237263Snpstatic inline struct sge_ofld_rxq * 559237263Snpiq_to_ofld_rxq(struct sge_iq *iq) 560237263Snp{ 561237263Snp 562241733Sed return (__containerof(iq, struct sge_ofld_rxq, iq)); 563237263Snp} 564228561Snp#endif 565228561Snp 566237263Snpstruct wrqe { 567237263Snp STAILQ_ENTRY(wrqe) link; 568237263Snp struct sge_wrq *wrq; 569237263Snp int wr_len; 570276485Snp char wr[] __aligned(16); 571237263Snp}; 572237263Snp 573276485Snpstruct wrq_cookie { 574276485Snp TAILQ_ENTRY(wrq_cookie) link; 575276485Snp int ndesc; 576276485Snp int pidx; 577276485Snp}; 578276485Snp 579228561Snp/* 580228561Snp * wrq: SGE egress queue that is given prebuilt work requests. Both the control 581228561Snp * and offload tx queues are of this type. 582228561Snp */ 583228561Snpstruct sge_wrq { 584220873Snp struct sge_eq eq; /* MUST be first */ 585220873Snp 586228561Snp struct adapter *adapter; 587276485Snp struct task wrq_tx_task; 588228561Snp 589276485Snp /* Tx desc reserved but WR not "committed" yet. */ 590276485Snp TAILQ_HEAD(wrq_incomplete_wrs , wrq_cookie) incomplete_wrs; 591276485Snp 592276485Snp /* List of WRs ready to go out as soon as descriptors are available. */ 593237263Snp STAILQ_HEAD(, wrqe) wr_list; 594276485Snp u_int nwr_pending; 595276485Snp u_int ndesc_needed; 596237263Snp 597220873Snp /* stats for common events first */ 598220873Snp 599276485Snp uint64_t tx_wrs_direct; /* # of WRs written directly to desc ring. */ 600276485Snp uint64_t tx_wrs_ss; /* # of WRs copied from scratch space. */ 601276485Snp uint64_t tx_wrs_copied; /* # of WRs queued and copied to desc ring. */ 602220873Snp 603220873Snp /* stats for not-that-common events */ 604220873Snp 605276485Snp /* 606276485Snp * Scratch space for work requests that wrap around after reaching the 607276485Snp * status page, and some infomation about the last WR that used it. 608276485Snp */ 609276485Snp uint16_t ss_pidx; 610276485Snp uint16_t ss_len; 611276485Snp uint8_t ss[SGE_MAX_WR_LEN]; 612276485Snp 613220873Snp} __aligned(CACHE_LINE_SIZE); 614220873Snp 615266757Snp 616266757Snp#ifdef DEV_NETMAP 617266757Snpstruct sge_nm_rxq { 618291665Sjhb struct vi_info *vi; 619266757Snp 620269411Snp struct iq_desc *iq_desc; 621266757Snp uint16_t iq_abs_id; 622266757Snp uint16_t iq_cntxt_id; 623266757Snp uint16_t iq_cidx; 624266757Snp uint16_t iq_sidx; 625266757Snp uint8_t iq_gen; 626266757Snp 627266757Snp __be64 *fl_desc; 628266757Snp uint16_t fl_cntxt_id; 629266757Snp uint32_t fl_cidx; 630266757Snp uint32_t fl_pidx; 631266757Snp uint32_t fl_sidx; 632266757Snp uint32_t fl_db_val; 633266757Snp u_int fl_hwidx:4; 634266757Snp 635266757Snp u_int nid; /* netmap ring # for this queue */ 636266757Snp 637266757Snp /* infrequently used items after this */ 638266757Snp 639266757Snp bus_dma_tag_t iq_desc_tag; 640266757Snp bus_dmamap_t iq_desc_map; 641266757Snp bus_addr_t iq_ba; 642266757Snp int intr_idx; 643266757Snp 644266757Snp bus_dma_tag_t fl_desc_tag; 645266757Snp bus_dmamap_t fl_desc_map; 646266757Snp bus_addr_t fl_ba; 647266757Snp} __aligned(CACHE_LINE_SIZE); 648266757Snp 649266757Snpstruct sge_nm_txq { 650266757Snp struct tx_desc *desc; 651266757Snp uint16_t cidx; 652266757Snp uint16_t pidx; 653266757Snp uint16_t sidx; 654266757Snp uint16_t equiqidx; /* EQUIQ last requested at this pidx */ 655266757Snp uint16_t equeqidx; /* EQUEQ last requested at this pidx */ 656266757Snp uint16_t dbidx; /* pidx of the most recent doorbell */ 657266757Snp uint16_t doorbells; 658266757Snp volatile uint32_t *udb; 659266757Snp u_int udb_qid; 660266757Snp u_int cntxt_id; 661266757Snp __be32 cpl_ctrl0; /* for convenience */ 662266757Snp u_int nid; /* netmap ring # for this queue */ 663266757Snp 664266757Snp /* infrequently used items after this */ 665266757Snp 666266757Snp bus_dma_tag_t desc_tag; 667266757Snp bus_dmamap_t desc_map; 668266757Snp bus_addr_t ba; 669266757Snp int iqidx; 670266757Snp} __aligned(CACHE_LINE_SIZE); 671266757Snp#endif 672266757Snp 673218792Snpstruct sge { 674228561Snp int timer_val[SGE_NTIMERS]; 675228561Snp int counter_val[SGE_NCOUNTERS]; 676222701Snp int fl_starve_threshold; 677261536Snp int fl_starve_threshold2; 678256477Snp int eq_s_qpp; 679256477Snp int iq_s_qpp; 680218792Snp 681228561Snp int nrxq; /* total # of Ethernet rx queues */ 682228561Snp int ntxq; /* total # of Ethernet tx tx queues */ 683237263Snp#ifdef TCP_OFFLOAD 684228561Snp int nofldrxq; /* total # of TOE rx queues */ 685228561Snp int nofldtxq; /* total # of TOE tx queues */ 686228561Snp#endif 687266757Snp#ifdef DEV_NETMAP 688266757Snp int nnmrxq; /* total # of netmap rx queues */ 689266757Snp int nnmtxq; /* total # of netmap tx queues */ 690266757Snp#endif 691228561Snp int niq; /* total # of ingress queues */ 692228561Snp int neq; /* total # of egress queues */ 693218792Snp 694218792Snp struct sge_iq fwq; /* Firmware event queue */ 695228561Snp struct sge_wrq mgmtq; /* Management queue (control queue) */ 696228561Snp struct sge_wrq *ctrlq; /* Control queues */ 697218792Snp struct sge_txq *txq; /* NIC tx queues */ 698218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 699237263Snp#ifdef TCP_OFFLOAD 700228561Snp struct sge_wrq *ofld_txq; /* TOE tx queues */ 701228561Snp struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ 702228561Snp#endif 703266757Snp#ifdef DEV_NETMAP 704266757Snp struct sge_nm_txq *nm_txq; /* netmap tx queues */ 705266757Snp struct sge_nm_rxq *nm_rxq; /* netmap rx queues */ 706266757Snp#endif 707218792Snp 708218792Snp uint16_t iq_start; 709218792Snp int eq_start; 710218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 711218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 712255050Snp 713275539Snp int pad_boundary; 714263317Snp int pack_boundary; 715263317Snp int8_t safe_hwidx1; /* may not have room for metadata */ 716263317Snp int8_t safe_hwidx2; /* with room for metadata and maybe more */ 717263317Snp struct sw_zone_info sw_zone_info[SW_ZONE_SIZES]; 718263317Snp struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES]; 719218792Snp}; 720218792Snp 721228561Snpstruct rss_header; 722228561Snptypedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, 723228561Snp struct mbuf *); 724237263Snptypedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); 725239336Snptypedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); 726228561Snp 727218792Snpstruct adapter { 728228561Snp SLIST_ENTRY(adapter) link; 729218792Snp device_t dev; 730218792Snp struct cdev *cdev; 731218792Snp 732218792Snp /* PCIe register resources */ 733218792Snp int regs_rid; 734218792Snp struct resource *regs_res; 735218792Snp int msix_rid; 736218792Snp struct resource *msix_res; 737218792Snp bus_space_handle_t bh; 738218792Snp bus_space_tag_t bt; 739218792Snp bus_size_t mmio_len; 740248925Snp int udbs_rid; 741248925Snp struct resource *udbs_res; 742248925Snp volatile uint8_t *udbs_base; 743218792Snp 744218792Snp unsigned int pf; 745218792Snp unsigned int mbox; 746218792Snp 747218792Snp /* Interrupt information */ 748218792Snp int intr_type; 749218792Snp int intr_count; 750218792Snp struct irq { 751218792Snp struct resource *res; 752218792Snp int rid; 753218792Snp void *tag; 754218792Snp } *irq; 755218792Snp 756218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 757218792Snp 758218792Snp struct sge sge; 759255015Snp int lro_timeout; 760218792Snp 761276485Snp struct taskqueue *tq[NCHAN]; /* General purpose taskqueues */ 762218792Snp struct port_info *port[MAX_NPORTS]; 763218792Snp uint8_t chan_map[NCHAN]; 764218792Snp 765237263Snp#ifdef TCP_OFFLOAD 766237263Snp void *tom_softc; /* (struct tom_data *) */ 767228561Snp struct tom_tunables tt; 768255005Snp void *iwarp_softc; /* (struct c4iw_dev *) */ 769292736Snp void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ 770228561Snp#endif 771222509Snp struct l2t_data *l2t; /* L2 table */ 772218792Snp struct tid_info tids; 773218792Snp 774266757Snp uint16_t doorbells; 775237263Snp#ifdef TCP_OFFLOAD 776278374Snp int offload_map; /* ports with IFCAP_TOE enabled */ 777278374Snp int active_ulds; /* ULDs activated on this adapter */ 778228561Snp#endif 779218792Snp int flags; 780284445Snp int debug_flags; 781218792Snp 782253691Snp char ifp_lockname[16]; 783253691Snp struct mtx ifp_lock; 784253691Snp struct ifnet *ifp; /* tracer ifp */ 785253691Snp struct ifmedia media; 786253691Snp int traceq; /* iq used by all tracers, -1 if none */ 787253691Snp int tracer_valid; /* bitmap of valid tracers */ 788253691Snp int tracer_enabled; /* bitmap of enabled tracers */ 789253691Snp 790218792Snp char fw_version[32]; 791245936Snp char cfg_file[32]; 792245936Snp u_int cfcsum; 793218792Snp struct adapter_params params; 794218792Snp struct t4_virt_res vres; 795218792Snp 796228561Snp uint16_t linkcaps; 797228561Snp uint16_t niccaps; 798228561Snp uint16_t toecaps; 799228561Snp uint16_t rdmacaps; 800228561Snp uint16_t iscsicaps; 801228561Snp uint16_t fcoecaps; 802220873Snp 803228561Snp struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ 804228561Snp 805218792Snp struct mtx sc_lock; 806218792Snp char lockname[16]; 807228561Snp 808228561Snp /* Starving free lists */ 809228561Snp struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ 810228561Snp TAILQ_HEAD(, sge_fl) sfl; 811228561Snp struct callout sfl_callout; 812228561Snp 813272200Snp struct mtx regwin_lock; /* for indirect reads and memory windows */ 814272200Snp 815237263Snp an_handler_t an_handler __aligned(CACHE_LINE_SIZE); 816295778Snp fw_msg_handler_t fw_msg_handler[7]; /* NUM_FW6_TYPES */ 817239336Snp cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */ 818245274Snp 819245274Snp#ifdef INVARIANTS 820245274Snp const char *last_op; 821245274Snp const void *last_op_thr; 822286926Snp int last_op_flags; 823245274Snp#endif 824260210Sadrian 825260210Sadrian int sc_do_rxcopy; 826218792Snp}; 827218792Snp 828218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 829218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 830218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 831218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 832218792Snp 833245274Snp#define ASSERT_SYNCHRONIZED_OP(sc) \ 834245274Snp KASSERT(IS_BUSY(sc) && \ 835245274Snp (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ 836245274Snp ("%s: operation not synchronized.", __func__)) 837245274Snp 838218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 839218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 840218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 841218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 842218792Snp 843218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 844218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 845218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 846218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 847218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 848218792Snp 849218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 850218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 851218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 852218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 853218792Snp 854218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 855218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 856218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 857218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 858218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 859218792Snp 860218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 861218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 862218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 863218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 864218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 865218792Snp 866284445Snp#define CH_DUMP_MBOX(sc, mbox, data_reg) \ 867284445Snp do { \ 868284445Snp if (sc->debug_flags & DF_DUMP_MBOX) { \ 869284445Snp log(LOG_NOTICE, \ 870284445Snp "%s mbox %u: %016llx %016llx %016llx %016llx " \ 871284445Snp "%016llx %016llx %016llx %016llx\n", \ 872284445Snp device_get_nameunit(sc->dev), mbox, \ 873284445Snp (unsigned long long)t4_read_reg64(sc, data_reg), \ 874284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 8), \ 875284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 16), \ 876284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 24), \ 877284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 32), \ 878284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 40), \ 879284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 48), \ 880284445Snp (unsigned long long)t4_read_reg64(sc, data_reg + 56)); \ 881284445Snp } \ 882284445Snp } while (0) 883284445Snp 884291665Sjhb#define for_each_txq(vi, iter, q) \ 885291665Sjhb for (q = &vi->pi->adapter->sge.txq[vi->first_txq], iter = 0; \ 886291665Sjhb iter < vi->ntxq; ++iter, ++q) 887291665Sjhb#define for_each_rxq(vi, iter, q) \ 888291665Sjhb for (q = &vi->pi->adapter->sge.rxq[vi->first_rxq], iter = 0; \ 889291665Sjhb iter < vi->nrxq; ++iter, ++q) 890291665Sjhb#define for_each_ofld_txq(vi, iter, q) \ 891291665Sjhb for (q = &vi->pi->adapter->sge.ofld_txq[vi->first_ofld_txq], iter = 0; \ 892291665Sjhb iter < vi->nofldtxq; ++iter, ++q) 893291665Sjhb#define for_each_ofld_rxq(vi, iter, q) \ 894291665Sjhb for (q = &vi->pi->adapter->sge.ofld_rxq[vi->first_ofld_rxq], iter = 0; \ 895291665Sjhb iter < vi->nofldrxq; ++iter, ++q) 896291665Sjhb#define for_each_nm_txq(vi, iter, q) \ 897291665Sjhb for (q = &vi->pi->adapter->sge.nm_txq[vi->first_txq], iter = 0; \ 898291665Sjhb iter < vi->ntxq; ++iter, ++q) 899291665Sjhb#define for_each_nm_rxq(vi, iter, q) \ 900291665Sjhb for (q = &vi->pi->adapter->sge.nm_rxq[vi->first_rxq], iter = 0; \ 901291665Sjhb iter < vi->nrxq; ++iter, ++q) 902291665Sjhb#define for_each_vi(_pi, _iter, _vi) \ 903291665Sjhb for ((_vi) = (_pi)->vi, (_iter) = 0; (_iter) < (_pi)->nvi; \ 904291665Sjhb ++(_iter), ++(_vi)) 905218792Snp 906269428Snp#define IDXINCR(idx, incr, wrap) do { \ 907269428Snp idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \ 908269411Snp} while (0) 909269411Snp#define IDXDIFF(head, tail, wrap) \ 910269428Snp ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head)) 911269411Snp 912222510Snp/* One for errors, one for firmware events */ 913222510Snp#define T4_EXTRA_INTR 2 914218792Snp 915218792Snpstatic inline uint32_t 916218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 917218792Snp{ 918237263Snp 919218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 920218792Snp} 921218792Snp 922218792Snpstatic inline void 923218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 924218792Snp{ 925237263Snp 926218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 927218792Snp} 928218792Snp 929218792Snpstatic inline uint64_t 930218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 931218792Snp{ 932237263Snp 933218792Snp return t4_bus_space_read_8(sc->bt, sc->bh, reg); 934218792Snp} 935218792Snp 936218792Snpstatic inline void 937218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 938218792Snp{ 939237263Snp 940218792Snp t4_bus_space_write_8(sc->bt, sc->bh, reg, val); 941218792Snp} 942218792Snp 943218792Snpstatic inline void 944218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 945218792Snp{ 946237263Snp 947218792Snp *val = pci_read_config(sc->dev, reg, 1); 948218792Snp} 949218792Snp 950218792Snpstatic inline void 951218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 952218792Snp{ 953237263Snp 954218792Snp pci_write_config(sc->dev, reg, val, 1); 955218792Snp} 956218792Snp 957218792Snpstatic inline void 958218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 959218792Snp{ 960237263Snp 961218792Snp *val = pci_read_config(sc->dev, reg, 2); 962218792Snp} 963218792Snp 964218792Snpstatic inline void 965218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 966218792Snp{ 967237263Snp 968218792Snp pci_write_config(sc->dev, reg, val, 2); 969218792Snp} 970218792Snp 971218792Snpstatic inline void 972218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 973218792Snp{ 974237263Snp 975218792Snp *val = pci_read_config(sc->dev, reg, 4); 976218792Snp} 977218792Snp 978218792Snpstatic inline void 979218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 980218792Snp{ 981237263Snp 982218792Snp pci_write_config(sc->dev, reg, val, 4); 983218792Snp} 984218792Snp 985218792Snpstatic inline struct port_info * 986218792Snpadap2pinfo(struct adapter *sc, int idx) 987218792Snp{ 988237263Snp 989218792Snp return (sc->port[idx]); 990218792Snp} 991218792Snp 992218792Snpstatic inline void 993218792Snpt4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[]) 994218792Snp{ 995237263Snp 996291665Sjhb bcopy(hw_addr, sc->port[idx]->vi[0].hw_addr, ETHER_ADDR_LEN); 997218792Snp} 998218792Snp 999248925Snpstatic inline bool 1000248925Snpis_10G_port(const struct port_info *pi) 1001218792Snp{ 1002237263Snp 1003218792Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0); 1004218792Snp} 1005218792Snp 1006250092Snpstatic inline bool 1007250092Snpis_40G_port(const struct port_info *pi) 1008250092Snp{ 1009250092Snp 1010250092Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0); 1011250092Snp} 1012250092Snp 1013248925Snpstatic inline int 1014296018Snpport_top_speed(const struct port_info *pi) 1015296018Snp{ 1016296018Snp 1017296018Snp if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G) 1018296018Snp return (100); 1019296018Snp if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) 1020296018Snp return (40); 1021296018Snp if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) 1022296018Snp return (10); 1023296018Snp if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G) 1024296018Snp return (1); 1025296018Snp 1026296018Snp return (0); 1027296018Snp} 1028296018Snp 1029296018Snpstatic inline int 1030248925Snptx_resume_threshold(struct sge_eq *eq) 1031228561Snp{ 1032237263Snp 1033276485Snp /* not quite the same as qsize / 4, but this will do. */ 1034276485Snp return (eq->sidx / 4); 1035228561Snp} 1036228561Snp 1037219286Snp/* t4_main.c */ 1038218792Snpint t4_os_find_pci_capability(struct adapter *, int); 1039218792Snpint t4_os_pci_save_state(struct adapter *); 1040218792Snpint t4_os_pci_restore_state(struct adapter *); 1041218792Snpvoid t4_os_portmod_changed(const struct adapter *, int); 1042252747Snpvoid t4_os_link_changed(struct adapter *, int, int, int); 1043228561Snpvoid t4_iterate(void (*)(struct adapter *, void *), void *); 1044228561Snpint t4_register_cpl_handler(struct adapter *, int, cpl_handler_t); 1045237263Snpint t4_register_an_handler(struct adapter *, an_handler_t); 1046239336Snpint t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t); 1047239338Snpint t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 1048291665Sjhbint begin_synchronized_op(struct adapter *, struct vi_info *, int, char *); 1049291665Sjhbvoid doom_vi(struct adapter *, struct vi_info *); 1050245274Snpvoid end_synchronized_op(struct adapter *, int); 1051266757Snpint update_mac_settings(struct ifnet *, int); 1052266757Snpint adapter_full_init(struct adapter *); 1053266757Snpint adapter_full_uninit(struct adapter *); 1054291665Sjhbuint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 1055291665Sjhbint vi_full_init(struct vi_info *); 1056291665Sjhbint vi_full_uninit(struct vi_info *); 1057291665Sjhbvoid vi_sysctls(struct vi_info *); 1058291665Sjhbvoid vi_tick(void *); 1059218792Snp 1060266757Snp#ifdef DEV_NETMAP 1061266757Snp/* t4_netmap.c */ 1062266757Snpint create_netmap_ifnet(struct port_info *); 1063266757Snpint destroy_netmap_ifnet(struct port_info *); 1064266757Snpvoid t4_nm_intr(void *); 1065266757Snp#endif 1066266757Snp 1067219286Snp/* t4_sge.c */ 1068219392Snpvoid t4_sge_modload(void); 1069269032Snpvoid t4_sge_modunload(void); 1070269032Snpuint64_t t4_sge_extfree_refs(void); 1071248925Snpvoid t4_init_sge_cpl_handlers(struct adapter *); 1072248925Snpvoid t4_tweak_chip_settings(struct adapter *); 1073248925Snpint t4_read_chip_settings(struct adapter *); 1074218792Snpint t4_create_dma_tag(struct adapter *); 1075253829Snpvoid t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, 1076253829Snp struct sysctl_oid_list *); 1077218792Snpint t4_destroy_dma_tag(struct adapter *); 1078220873Snpint t4_setup_adapter_queues(struct adapter *); 1079220873Snpint t4_teardown_adapter_queues(struct adapter *); 1080291665Sjhbint t4_setup_vi_queues(struct vi_info *); 1081291665Sjhbint t4_teardown_vi_queues(struct vi_info *); 1082218792Snpvoid t4_intr_all(void *); 1083222510Snpvoid t4_intr(void *); 1084218792Snpvoid t4_intr_err(void *); 1085218792Snpvoid t4_intr_evt(void *); 1086237263Snpvoid t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); 1087218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 1088276485Snpint parse_pkt(struct mbuf **); 1089276485Snpvoid *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *); 1090276485Snpvoid commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *); 1091285221Snpint tnl_cong(struct port_info *, int); 1092218792Snp 1093253691Snp/* t4_tracer.c */ 1094253691Snpstruct t4_tracer; 1095253691Snpvoid t4_tracer_modload(void); 1096253691Snpvoid t4_tracer_modunload(void); 1097253691Snpvoid t4_tracer_port_detach(struct adapter *); 1098253691Snpint t4_get_tracer(struct adapter *, struct t4_tracer *); 1099253691Snpint t4_set_tracer(struct adapter *, struct t4_tracer *); 1100253691Snpint t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 1101253691Snpint t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 1102253691Snp 1103237263Snpstatic inline struct wrqe * 1104237263Snpalloc_wrqe(int wr_len, struct sge_wrq *wrq) 1105228561Snp{ 1106237263Snp int len = offsetof(struct wrqe, wr) + wr_len; 1107237263Snp struct wrqe *wr; 1108228561Snp 1109237263Snp wr = malloc(len, M_CXGBE, M_NOWAIT); 1110237263Snp if (__predict_false(wr == NULL)) 1111237263Snp return (NULL); 1112237263Snp wr->wr_len = wr_len; 1113237263Snp wr->wrq = wrq; 1114237263Snp return (wr); 1115237263Snp} 1116237263Snp 1117237263Snpstatic inline void * 1118237263Snpwrtod(struct wrqe *wr) 1119237263Snp{ 1120237263Snp return (&wr->wr[0]); 1121237263Snp} 1122237263Snp 1123237263Snpstatic inline void 1124237263Snpfree_wrqe(struct wrqe *wr) 1125237263Snp{ 1126237263Snp free(wr, M_CXGBE); 1127237263Snp} 1128237263Snp 1129237263Snpstatic inline void 1130237263Snpt4_wrq_tx(struct adapter *sc, struct wrqe *wr) 1131237263Snp{ 1132237263Snp struct sge_wrq *wrq = wr->wrq; 1133237263Snp 1134228561Snp TXQ_LOCK(wrq); 1135237263Snp t4_wrq_tx_locked(sc, wrq, wr); 1136228561Snp TXQ_UNLOCK(wrq); 1137228561Snp} 1138228561Snp 1139218792Snp#endif 1140