adapter.h revision 219944
1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD: head/sys/dev/cxgbe/adapter.h 219944 2011-03-24 01:03:01Z np $ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34218792Snp#include <sys/bus.h> 35218792Snp#include <sys/rman.h> 36218792Snp#include <sys/types.h> 37218792Snp#include <sys/malloc.h> 38218792Snp#include <dev/pci/pcivar.h> 39218792Snp#include <dev/pci/pcireg.h> 40218792Snp#include <machine/bus.h> 41218792Snp#include <sys/socket.h> 42218792Snp#include <sys/sysctl.h> 43218792Snp#include <net/ethernet.h> 44218792Snp#include <net/if.h> 45218792Snp#include <net/if_media.h> 46218792Snp#include <netinet/tcp_lro.h> 47218792Snp 48218792Snp#include "offload.h" 49218792Snp#include "common/t4fw_interface.h" 50218792Snp 51218792Snp#define T4_FWNAME "t4fw" 52218792Snp 53218792SnpMALLOC_DECLARE(M_CXGBE); 54218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 55218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 56218792Snp 57218792Snp#if defined(__i386__) || defined(__amd64__) 58218792Snpstatic __inline void 59218792Snpprefetch(void *x) 60218792Snp{ 61218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 62218792Snp} 63218792Snp#else 64218792Snp#define prefetch(x) 65218792Snp#endif 66218792Snp 67218792Snp#ifdef __amd64__ 68218792Snp/* XXX: need systemwide bus_space_read_8/bus_space_write_8 */ 69218792Snpstatic __inline uint64_t 70218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 71218792Snp bus_size_t offset) 72218792Snp{ 73219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 74219285Snp ("%s: can only handle mem space", __func__)); 75218792Snp 76218792Snp return (*(volatile uint64_t *)(handle + offset)); 77218792Snp} 78218792Snp 79218792Snpstatic __inline void 80218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 81218792Snp bus_size_t offset, uint64_t value) 82218792Snp{ 83219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 84219285Snp ("%s: can only handle mem space", __func__)); 85219285Snp 86218792Snp *(volatile uint64_t *)(bsh + offset) = value; 87218792Snp} 88218792Snp#else 89218792Snpstatic __inline uint64_t 90218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 91218792Snp bus_size_t offset) 92218792Snp{ 93218792Snp return (uint64_t)bus_space_read_4(tag, handle, offset) + 94218792Snp ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32); 95218792Snp} 96218792Snp 97218792Snpstatic __inline void 98218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 99218792Snp bus_size_t offset, uint64_t value) 100218792Snp{ 101218792Snp bus_space_write_4(tag, bsh, offset, value); 102218792Snp bus_space_write_4(tag, bsh, offset + 4, value >> 32); 103218792Snp} 104218792Snp#endif 105218792Snp 106218792Snpstruct adapter; 107218792Snptypedef struct adapter adapter_t; 108218792Snp 109218792Snpenum { 110218792Snp FW_IQ_QSIZE = 256, 111218792Snp FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */ 112218792Snp 113218792Snp RX_IQ_QSIZE = 1024, 114218792Snp RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */ 115218792Snp 116218792Snp RX_FL_ESIZE = 64, /* 8 64bit addresses */ 117218792Snp 118219392Snp#if MJUMPAGESIZE != MCLBYTES 119219392Snp FL_BUF_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */ 120219392Snp#else 121219392Snp FL_BUF_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ 122219392Snp#endif 123218792Snp 124218792Snp TX_EQ_QSIZE = 1024, 125218792Snp TX_EQ_ESIZE = 64, 126218792Snp TX_SGL_SEGS = 36, 127218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 128218792Snp}; 129218792Snp 130218792Snpenum { 131219944Snp /* adapter intr_type */ 132219944Snp INTR_INTX = (1 << 0), 133219944Snp INTR_MSI = (1 << 1), 134219944Snp INTR_MSIX = (1 << 2) 135219944Snp}; 136219944Snp 137219944Snpenum { 138218792Snp /* adapter flags */ 139218792Snp FULL_INIT_DONE = (1 << 0), 140218792Snp FW_OK = (1 << 1), 141218792Snp INTR_FWD = (1 << 2), 142218792Snp 143218792Snp CXGBE_BUSY = (1 << 9), 144218792Snp 145218792Snp /* port flags */ 146218792Snp DOOMED = (1 << 0), 147218792Snp VI_ENABLED = (1 << 1), 148218792Snp}; 149218792Snp 150218792Snp#define IS_DOOMED(pi) (pi->flags & DOOMED) 151218792Snp#define SET_DOOMED(pi) do {pi->flags |= DOOMED;} while (0) 152218792Snp#define IS_BUSY(sc) (sc->flags & CXGBE_BUSY) 153218792Snp#define SET_BUSY(sc) do {sc->flags |= CXGBE_BUSY;} while (0) 154218792Snp#define CLR_BUSY(sc) do {sc->flags &= ~CXGBE_BUSY;} while (0) 155218792Snp 156218792Snpstruct port_info { 157218792Snp device_t dev; 158218792Snp struct adapter *adapter; 159218792Snp 160218792Snp struct ifnet *ifp; 161218792Snp struct ifmedia media; 162218792Snp 163218792Snp struct mtx pi_lock; 164218792Snp char lockname[16]; 165218792Snp unsigned long flags; 166218792Snp int if_flags; 167218792Snp 168218792Snp uint16_t viid; 169218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 170218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 171218792Snp uint8_t lport; /* associated offload logical port */ 172218792Snp int8_t mdio_addr; 173218792Snp uint8_t port_type; 174218792Snp uint8_t mod_type; 175218792Snp uint8_t port_id; 176218792Snp uint8_t tx_chan; 177218792Snp 178218792Snp /* These need to be int as they are used in sysctl */ 179218792Snp int ntxq; /* # of tx queues */ 180218792Snp int first_txq; /* index of first tx queue */ 181218792Snp int nrxq; /* # of rx queues */ 182218792Snp int first_rxq; /* index of first rx queue */ 183218792Snp int tmr_idx; 184218792Snp int pktc_idx; 185218792Snp int qsize_rxq; 186218792Snp int qsize_txq; 187218792Snp 188218792Snp struct link_config link_cfg; 189218792Snp struct port_stats stats; 190218792Snp 191219286Snp struct taskqueue *tq; 192218792Snp struct callout tick; 193218792Snp struct sysctl_ctx_list ctx; /* lives from ifconfig up to down */ 194218792Snp struct sysctl_oid *oid_rxq; 195218792Snp struct sysctl_oid *oid_txq; 196218792Snp 197218792Snp uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 198218792Snp}; 199218792Snp 200218792Snpstruct fl_sdesc { 201218792Snp struct mbuf *m; 202218792Snp bus_dmamap_t map; 203218792Snp caddr_t cl; 204218792Snp uint8_t tag_idx; /* the sc->fl_tag this map comes from */ 205218792Snp#ifdef INVARIANTS 206218792Snp __be64 ba_tag; 207218792Snp#endif 208218792Snp}; 209218792Snp 210218792Snpstruct tx_desc { 211218792Snp __be64 flit[8]; 212218792Snp}; 213218792Snp 214218792Snpstruct tx_map { 215218792Snp struct mbuf *m; 216218792Snp bus_dmamap_t map; 217218792Snp}; 218218792Snp 219218792Snpstruct tx_sdesc { 220218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 221218792Snp uint8_t map_used; /* # of frames sent out in the WR */ 222218792Snp}; 223218792Snp 224218792Snptypedef void (iq_intr_handler_t)(void *); 225218792Snp 226218792Snpenum { 227218792Snp /* iq flags */ 228218792Snp IQ_ALLOCATED = (1 << 1), /* firmware resources allocated */ 229218792Snp IQ_STARTED = (1 << 2), /* started */ 230218792Snp}; 231218792Snp 232218792Snp/* 233218792Snp * Ingress Queue: T4 is producer, driver is consumer. 234218792Snp */ 235218792Snpstruct sge_iq { 236218792Snp bus_dma_tag_t desc_tag; 237218792Snp bus_dmamap_t desc_map; 238219290Snp bus_addr_t ba; /* bus address of descriptor ring */ 239219290Snp char lockname[16]; 240219290Snp uint32_t flags; 241219290Snp uint16_t abs_id; /* absolute SGE id for the iq */ 242219290Snp int8_t intr_pktc_idx; /* packet count threshold index */ 243219290Snp int8_t pad0; 244219290Snp iq_intr_handler_t *handler; 245219290Snp __be64 *desc; /* KVA of descriptor ring */ 246219290Snp 247218792Snp struct mtx iq_lock; 248218792Snp struct adapter *adapter; 249218792Snp const __be64 *cdesc; /* current descriptor */ 250218792Snp uint8_t gen; /* generation bit */ 251218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 252218792Snp uint8_t intr_next; /* holdoff for next interrupt */ 253218792Snp uint8_t esize; /* size (bytes) of each entry in the queue */ 254218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 255218792Snp uint16_t cidx; /* consumer index */ 256218792Snp uint16_t cntxt_id; /* SGE context id for the iq */ 257218792Snp}; 258218792Snp 259218792Snpenum { 260218792Snp /* eq flags */ 261218792Snp EQ_ALLOCATED = (1 << 1), /* firmware resources allocated */ 262218792Snp EQ_STARTED = (1 << 2), /* started */ 263218792Snp EQ_STALLED = (1 << 3), /* currently stalled */ 264218792Snp}; 265218792Snp 266218792Snp/* 267218792Snp * Egress Queue: driver is producer, T4 is consumer. 268218792Snp * 269218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 270218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 271218792Snp */ 272218792Snpstruct sge_eq { 273218792Snp bus_dma_tag_t tx_tag; /* tag for transmit buffers */ 274218792Snp bus_dma_tag_t desc_tag; 275218792Snp bus_dmamap_t desc_map; 276218792Snp char lockname[16]; 277218792Snp unsigned int flags; 278218792Snp struct mtx eq_lock; 279218792Snp 280218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 281218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 282218792Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 283218792Snp struct buf_ring *br; /* tx buffer ring */ 284218792Snp struct sge_qstat *spg; /* status page, for convenience */ 285218792Snp uint16_t cap; /* max # of desc, for convenience */ 286218792Snp uint16_t avail; /* available descriptors, for convenience */ 287218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 288218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 289218792Snp uint16_t pidx; /* producer idx (desc idx) */ 290218792Snp uint16_t pending; /* # of descriptors used since last doorbell */ 291219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 292218792Snp uint32_t cntxt_id; /* SGE context id for the eq */ 293218792Snp 294218792Snp /* DMA maps used for tx */ 295218792Snp struct tx_map *maps; 296218792Snp uint32_t map_total; /* # of DMA maps */ 297218792Snp uint32_t map_pidx; /* next map to be used */ 298218792Snp uint32_t map_cidx; /* reclaimed up to this index */ 299218792Snp uint32_t map_avail; /* # of available maps */ 300218792Snp} __aligned(CACHE_LINE_SIZE); 301218792Snp 302218792Snpstruct sge_fl { 303218792Snp bus_dma_tag_t desc_tag; 304218792Snp bus_dmamap_t desc_map; 305218792Snp bus_dma_tag_t tag[FL_BUF_SIZES]; 306218792Snp uint8_t tag_idx; 307218792Snp struct mtx fl_lock; 308218792Snp char lockname[16]; 309218792Snp 310218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 311218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 312218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 313218792Snp uint32_t cap; /* max # of buffers, for convenience */ 314218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 315218792Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 316218792Snp uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */ 317218792Snp uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */ 318218792Snp uint32_t needed; /* # of buffers needed to fill up fl. */ 319218792Snp uint32_t pending; /* # of bufs allocated since last doorbell */ 320218792Snp unsigned int dmamap_failed; 321218792Snp}; 322218792Snp 323218792Snp/* txq: SGE egress queue + miscellaneous items */ 324218792Snpstruct sge_txq { 325218792Snp struct sge_eq eq; /* MUST be first */ 326218792Snp struct mbuf *m; /* held up due to temporary resource shortage */ 327219286Snp struct task resume_tx; 328218792Snp 329219289Snp struct ifnet *ifp; /* the interface this txq belongs to */ 330219286Snp 331218792Snp /* stats for common events first */ 332218792Snp 333218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 334218792Snp uint64_t tso_wrs; /* # of IPv4 TSO work requests */ 335218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 336218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 337218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 338218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 339218792Snp uint64_t txpkts_wrs; /* # of coalesced tx work requests */ 340218792Snp uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */ 341218792Snp 342218792Snp /* stats for not-that-common events */ 343218792Snp 344218792Snp uint32_t no_dmamap; /* no DMA map to load the mbuf */ 345218792Snp uint32_t no_desc; /* out of hardware descriptors */ 346218792Snp uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for txq */ 347218792Snp}; 348218792Snp 349218792Snpenum { 350218792Snp RXQ_LRO_ENABLED = (1 << 0) 351218792Snp}; 352218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 353218792Snpstruct sge_rxq { 354218792Snp struct sge_iq iq; /* MUST be first */ 355218792Snp struct sge_fl fl; 356218792Snp 357219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 358218792Snp unsigned int flags; 359219290Snp#ifdef INET 360218792Snp struct lro_ctrl lro; /* LRO state */ 361219290Snp#endif 362218792Snp 363218792Snp /* stats for common events first */ 364218792Snp 365218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 366218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 367218792Snp 368218792Snp /* stats for not-that-common events */ 369218792Snp 370218792Snp} __aligned(CACHE_LINE_SIZE); 371218792Snp 372218792Snpstruct sge { 373218792Snp uint16_t timer_val[SGE_NTIMERS]; 374218792Snp uint8_t counter_val[SGE_NCOUNTERS]; 375218792Snp 376218792Snp int nrxq; /* total rx queues (all ports and the rest) */ 377218792Snp int ntxq; /* total tx queues (all ports and the rest) */ 378218792Snp int niq; /* total ingress queues */ 379218792Snp int neq; /* total egress queues */ 380218792Snp 381218792Snp struct sge_iq fwq; /* Firmware event queue */ 382218792Snp struct sge_iq *fiq; /* Forwarded interrupt queues (INTR_FWD) */ 383218792Snp struct sge_txq *txq; /* NIC tx queues */ 384218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 385218792Snp 386218792Snp uint16_t iq_start; 387218792Snp int eq_start; 388218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 389218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 390218792Snp}; 391218792Snp 392218792Snpstruct adapter { 393218792Snp device_t dev; 394218792Snp struct cdev *cdev; 395218792Snp 396218792Snp /* PCIe register resources */ 397218792Snp int regs_rid; 398218792Snp struct resource *regs_res; 399218792Snp int msix_rid; 400218792Snp struct resource *msix_res; 401218792Snp bus_space_handle_t bh; 402218792Snp bus_space_tag_t bt; 403218792Snp bus_size_t mmio_len; 404218792Snp 405218792Snp unsigned int pf; 406218792Snp unsigned int mbox; 407218792Snp 408218792Snp /* Interrupt information */ 409218792Snp int intr_type; 410218792Snp int intr_count; 411218792Snp struct irq { 412218792Snp struct resource *res; 413218792Snp int rid; 414218792Snp void *tag; 415218792Snp } *irq; 416218792Snp 417218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 418218792Snp 419218792Snp struct sge sge; 420218792Snp 421218792Snp struct port_info *port[MAX_NPORTS]; 422218792Snp uint8_t chan_map[NCHAN]; 423218792Snp 424218792Snp struct tid_info tids; 425218792Snp 426218792Snp int registered_device_map; 427218792Snp int open_device_map; 428218792Snp int flags; 429218792Snp 430218792Snp char fw_version[32]; 431218792Snp struct adapter_params params; 432218792Snp struct t4_virt_res vres; 433218792Snp 434218792Snp struct mtx sc_lock; 435218792Snp char lockname[16]; 436218792Snp}; 437218792Snp 438218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 439218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 440218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 441218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 442218792Snp 443218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 444218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 445218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 446218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 447218792Snp 448218792Snp#define IQ_LOCK(iq) mtx_lock(&(iq)->iq_lock) 449218792Snp#define IQ_UNLOCK(iq) mtx_unlock(&(iq)->iq_lock) 450218792Snp#define IQ_LOCK_ASSERT_OWNED(iq) mtx_assert(&(iq)->iq_lock, MA_OWNED) 451218792Snp#define IQ_LOCK_ASSERT_NOTOWNED(iq) mtx_assert(&(iq)->iq_lock, MA_NOTOWNED) 452218792Snp 453218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 454218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 455218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 456218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 457218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 458218792Snp 459218792Snp#define RXQ_LOCK(rxq) IQ_LOCK(&(rxq)->iq) 460218792Snp#define RXQ_UNLOCK(rxq) IQ_UNLOCK(&(rxq)->iq) 461218792Snp#define RXQ_LOCK_ASSERT_OWNED(rxq) IQ_LOCK_ASSERT_OWNED(&(rxq)->iq) 462218792Snp#define RXQ_LOCK_ASSERT_NOTOWNED(rxq) IQ_LOCK_ASSERT_NOTOWNED(&(rxq)->iq) 463218792Snp 464218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 465218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 466218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 467218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 468218792Snp 469218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 470218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 471218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 472218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 473218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 474218792Snp 475218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 476218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 477218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 478218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 479218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 480218792Snp 481218792Snp#define for_each_txq(pi, iter, txq) \ 482218792Snp txq = &pi->adapter->sge.txq[pi->first_txq]; \ 483218792Snp for (iter = 0; iter < pi->ntxq; ++iter, ++txq) 484218792Snp#define for_each_rxq(pi, iter, rxq) \ 485218792Snp rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \ 486218792Snp for (iter = 0; iter < pi->nrxq; ++iter, ++rxq) 487218792Snp 488218792Snp#define NFIQ(sc) ((sc)->intr_count > 1 ? (sc)->intr_count - 1 : 1) 489218792Snp 490218792Snpstatic inline uint32_t 491218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 492218792Snp{ 493218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 494218792Snp} 495218792Snp 496218792Snpstatic inline void 497218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 498218792Snp{ 499218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 500218792Snp} 501218792Snp 502218792Snpstatic inline uint64_t 503218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 504218792Snp{ 505218792Snp return t4_bus_space_read_8(sc->bt, sc->bh, reg); 506218792Snp} 507218792Snp 508218792Snpstatic inline void 509218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 510218792Snp{ 511218792Snp t4_bus_space_write_8(sc->bt, sc->bh, reg, val); 512218792Snp} 513218792Snp 514218792Snpstatic inline void 515218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 516218792Snp{ 517218792Snp *val = pci_read_config(sc->dev, reg, 1); 518218792Snp} 519218792Snp 520218792Snpstatic inline void 521218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 522218792Snp{ 523218792Snp pci_write_config(sc->dev, reg, val, 1); 524218792Snp} 525218792Snp 526218792Snpstatic inline void 527218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 528218792Snp{ 529218792Snp *val = pci_read_config(sc->dev, reg, 2); 530218792Snp} 531218792Snp 532218792Snpstatic inline void 533218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 534218792Snp{ 535218792Snp pci_write_config(sc->dev, reg, val, 2); 536218792Snp} 537218792Snp 538218792Snpstatic inline void 539218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 540218792Snp{ 541218792Snp *val = pci_read_config(sc->dev, reg, 4); 542218792Snp} 543218792Snp 544218792Snpstatic inline void 545218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 546218792Snp{ 547218792Snp pci_write_config(sc->dev, reg, val, 4); 548218792Snp} 549218792Snp 550218792Snpstatic inline struct port_info * 551218792Snpadap2pinfo(struct adapter *sc, int idx) 552218792Snp{ 553218792Snp return (sc->port[idx]); 554218792Snp} 555218792Snp 556218792Snpstatic inline void 557218792Snpt4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[]) 558218792Snp{ 559218792Snp bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN); 560218792Snp} 561218792Snp 562218792Snpstatic inline bool is_10G_port(const struct port_info *pi) 563218792Snp{ 564218792Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0); 565218792Snp} 566218792Snp 567219286Snp/* t4_main.c */ 568219286Snpvoid cxgbe_txq_start(void *, int); 569218792Snpint t4_os_find_pci_capability(struct adapter *, int); 570218792Snpint t4_os_pci_save_state(struct adapter *); 571218792Snpint t4_os_pci_restore_state(struct adapter *); 572218792Snpvoid t4_os_portmod_changed(const struct adapter *, int); 573218792Snpvoid t4_os_link_changed(struct adapter *, int, int); 574218792Snp 575219286Snp/* t4_sge.c */ 576219392Snpvoid t4_sge_modload(void); 577218792Snpvoid t4_sge_init(struct adapter *); 578218792Snpint t4_create_dma_tag(struct adapter *); 579218792Snpint t4_destroy_dma_tag(struct adapter *); 580218792Snpint t4_setup_adapter_iqs(struct adapter *); 581218792Snpint t4_teardown_adapter_iqs(struct adapter *); 582218792Snpint t4_setup_eth_queues(struct port_info *); 583218792Snpint t4_teardown_eth_queues(struct port_info *); 584218792Snpvoid t4_intr_all(void *); 585218792Snpvoid t4_intr_fwd(void *); 586218792Snpvoid t4_intr_err(void *); 587218792Snpvoid t4_intr_evt(void *); 588218792Snpvoid t4_intr_data(void *); 589218792Snpint t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *); 590218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 591218792Snp 592218792Snp#endif 593