1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp */ 27218792Snp 28218792Snp#include <sys/cdefs.h> 29218792Snp__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 353418 2019-10-10 23:27:02Z np $"); 30218792Snp 31218792Snp#include "opt_inet.h" 32237819Snp#include "opt_inet6.h" 33218792Snp 34218792Snp#include <sys/types.h> 35218792Snp#include <sys/mbuf.h> 36218792Snp#include <sys/socket.h> 37218792Snp#include <sys/kernel.h> 38219286Snp#include <sys/malloc.h> 39219286Snp#include <sys/queue.h> 40265425Snp#include <sys/sbuf.h> 41219286Snp#include <sys/taskqueue.h> 42255015Snp#include <sys/time.h> 43284052Snp#include <sys/sglist.h> 44218792Snp#include <sys/sysctl.h> 45228561Snp#include <sys/smp.h> 46269356Snp#include <sys/counter.h> 47218792Snp#include <net/bpf.h> 48218792Snp#include <net/ethernet.h> 49218792Snp#include <net/if.h> 50218792Snp#include <net/if_vlan_var.h> 51218792Snp#include <netinet/in.h> 52218792Snp#include <netinet/ip.h> 53237819Snp#include <netinet/ip6.h> 54218792Snp#include <netinet/tcp.h> 55309447Sjhb#include <machine/in_cksum.h> 56256131Sdim#include <machine/md_var.h> 57265425Snp#include <vm/vm.h> 58265425Snp#include <vm/pmap.h> 59270297Snp#ifdef DEV_NETMAP 60270297Snp#include <machine/bus.h> 61270297Snp#include <sys/selinfo.h> 62270297Snp#include <net/if_var.h> 63270297Snp#include <net/netmap.h> 64270297Snp#include <dev/netmap/netmap_kern.h> 65270297Snp#endif 66218792Snp 67218792Snp#include "common/common.h" 68218792Snp#include "common/t4_regs.h" 69218792Snp#include "common/t4_regs_values.h" 70218792Snp#include "common/t4_msg.h" 71309442Sjhb#include "t4_l2t.h" 72284052Snp#include "t4_mp_ring.h" 73218792Snp 74248925Snp#ifdef T4_PKT_TIMESTAMP 75248925Snp#define RX_COPY_THRESHOLD (MINCLSIZE - 8) 76248925Snp#else 77248925Snp#define RX_COPY_THRESHOLD MINCLSIZE 78248925Snp#endif 79248925Snp 80239258Snp/* 81239258Snp * Ethernet frames are DMA'd at this byte offset into the freelist buffer. 82239258Snp * 0-7 are valid values. 83239258Snp */ 84309448Sjhbstatic int fl_pktshift = 2; 85239258SnpTUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift); 86218792Snp 87239258Snp/* 88239258Snp * Pad ethernet payload up to this boundary. 89239258Snp * -1: driver should figure out a good value. 90255050Snp * 0: disable padding. 91255050Snp * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value. 92239258Snp */ 93270297Snpint fl_pad = -1; 94239258SnpTUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad); 95218792Snp 96239258Snp/* 97239258Snp * Status page length. 98239258Snp * -1: driver should figure out a good value. 99239258Snp * 64 or 128 are the only other valid values. 100239258Snp */ 101309448Sjhbstatic int spg_len = -1; 102239258SnpTUNABLE_INT("hw.cxgbe.spg_len", &spg_len); 103239258Snp 104239258Snp/* 105239258Snp * Congestion drops. 106239258Snp * -1: no congestion feedback (not recommended). 107239258Snp * 0: backpressure the channel instead of dropping packets right away. 108239258Snp * 1: no backpressure, drop packets for the congested queue immediately. 109239258Snp */ 110239258Snpstatic int cong_drop = 0; 111239258SnpTUNABLE_INT("hw.cxgbe.cong_drop", &cong_drop); 112239258Snp 113255050Snp/* 114255050Snp * Deliver multiple frames in the same free list buffer if they fit. 115255050Snp * -1: let the driver decide whether to enable buffer packing or not. 116255050Snp * 0: disable buffer packing. 117255050Snp * 1: enable buffer packing. 118255050Snp */ 119255050Snpstatic int buffer_packing = -1; 120255050SnpTUNABLE_INT("hw.cxgbe.buffer_packing", &buffer_packing); 121255050Snp 122255050Snp/* 123255050Snp * Start next frame in a packed buffer at this boundary. 124255050Snp * -1: driver should figure out a good value. 125281212Snp * T4: driver will ignore this and use the same value as fl_pad above. 126281212Snp * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value. 127255050Snp */ 128255050Snpstatic int fl_pack = -1; 129255050SnpTUNABLE_INT("hw.cxgbe.fl_pack", &fl_pack); 130255050Snp 131265425Snp/* 132265425Snp * Allow the driver to create mbuf(s) in a cluster allocated for rx. 133265425Snp * 0: never; always allocate mbufs from the zone_mbuf UMA zone. 134265425Snp * 1: ok to create mbuf(s) within a cluster if there is room. 135265425Snp */ 136265425Snpstatic int allow_mbufs_in_cluster = 1; 137265425SnpTUNABLE_INT("hw.cxgbe.allow_mbufs_in_cluster", &allow_mbufs_in_cluster); 138265425Snp 139265425Snp/* 140265425Snp * Largest rx cluster size that the driver is allowed to allocate. 141265425Snp */ 142265425Snpstatic int largest_rx_cluster = MJUM16BYTES; 143265425SnpTUNABLE_INT("hw.cxgbe.largest_rx_cluster", &largest_rx_cluster); 144265425Snp 145265425Snp/* 146265425Snp * Size of cluster allocation that's most likely to succeed. The driver will 147265425Snp * fall back to this size if it fails to allocate clusters larger than this. 148265425Snp */ 149265425Snpstatic int safest_rx_cluster = PAGE_SIZE; 150265425SnpTUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster); 151265425Snp 152318840Snp/* 153318840Snp * The interrupt holdoff timers are multiplied by this value on T6+. 154318840Snp * 1 and 3-17 (both inclusive) are legal values. 155318840Snp */ 156318840Snpstatic int tscale = 1; 157318840SnpTUNABLE_INT("hw.cxgbe.tscale", &tscale); 158318840Snp 159218792Snpstruct txpkts { 160284052Snp u_int wr_type; /* type 0 or type 1 */ 161284052Snp u_int npkt; /* # of packets in this work request */ 162284052Snp u_int plen; /* total payload (sum of all packets) */ 163284052Snp u_int len16; /* # of 16B pieces used by this work request */ 164218792Snp}; 165218792Snp 166218792Snp/* A packet's SGL. This + m_pkthdr has all info needed for tx */ 167218792Snpstruct sgl { 168284052Snp struct sglist sg; 169284052Snp struct sglist_seg seg[TX_SGL_SEGS]; 170218792Snp}; 171218792Snp 172228561Snpstatic int service_iq(struct sge_iq *, int); 173270297Snpstatic struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); 174228561Snpstatic int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *); 175270297Snpstatic inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); 176281212Snpstatic inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); 177308304Sjhbstatic inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, 178308304Sjhb uint16_t, char *); 179218792Snpstatic int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, 180218792Snp bus_addr_t *, void **); 181218792Snpstatic int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 182218792Snp void *); 183308154Sjhbstatic int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, 184222085Snp int, int); 185308154Sjhbstatic int free_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *); 186309564Sjhbstatic void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, 187309564Sjhb struct sysctl_oid *, struct sge_fl *); 188228561Snpstatic int alloc_fwq(struct adapter *); 189228561Snpstatic int free_fwq(struct adapter *); 190228561Snpstatic int alloc_mgmtq(struct adapter *); 191228561Snpstatic int free_mgmtq(struct adapter *); 192308154Sjhbstatic int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, 193228561Snp struct sysctl_oid *); 194308154Sjhbstatic int free_rxq(struct vi_info *, struct sge_rxq *); 195237263Snp#ifdef TCP_OFFLOAD 196308154Sjhbstatic int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int, 197228561Snp struct sysctl_oid *); 198308154Sjhbstatic int free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *); 199228561Snp#endif 200270297Snp#ifdef DEV_NETMAP 201308154Sjhbstatic int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int, 202270297Snp struct sysctl_oid *); 203308154Sjhbstatic int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *); 204308154Sjhbstatic int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int, 205270297Snp struct sysctl_oid *); 206308154Sjhbstatic int free_nm_txq(struct vi_info *, struct sge_nm_txq *); 207270297Snp#endif 208228561Snpstatic int ctrl_eq_alloc(struct adapter *, struct sge_eq *); 209308154Sjhbstatic int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 210237263Snp#ifdef TCP_OFFLOAD 211308154Sjhbstatic int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 212228561Snp#endif 213308154Sjhbstatic int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); 214228561Snpstatic int free_eq(struct adapter *, struct sge_eq *); 215308154Sjhbstatic int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, 216228561Snp struct sysctl_oid *); 217228561Snpstatic int free_wrq(struct adapter *, struct sge_wrq *); 218308154Sjhbstatic int alloc_txq(struct vi_info *, struct sge_txq *, int, 219228561Snp struct sysctl_oid *); 220308154Sjhbstatic int free_txq(struct vi_info *, struct sge_txq *); 221218792Snpstatic void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); 222218792Snpstatic inline void ring_fl_db(struct adapter *, struct sge_fl *); 223228561Snpstatic int refill_fl(struct adapter *, struct sge_fl *, int); 224228561Snpstatic void refill_sfl(void *); 225218792Snpstatic int alloc_fl_sdesc(struct sge_fl *); 226255050Snpstatic void free_fl_sdesc(struct adapter *, struct sge_fl *); 227265425Snpstatic void find_best_refill_source(struct adapter *, struct sge_fl *, int); 228265425Snpstatic void find_safe_refill_source(struct adapter *, struct sge_fl *); 229228561Snpstatic void add_fl_to_sfl(struct adapter *, struct sge_fl *); 230218792Snp 231284052Snpstatic inline void get_pkt_gl(struct mbuf *, struct sglist *); 232284052Snpstatic inline u_int txpkt_len16(u_int, u_int); 233309447Sjhbstatic inline u_int txpkt_vm_len16(u_int, u_int); 234284052Snpstatic inline u_int txpkts0_len16(u_int); 235284052Snpstatic inline u_int txpkts1_len16(void); 236284052Snpstatic u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *, 237284052Snp struct mbuf *, u_int); 238309560Sjhbstatic u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, 239309560Sjhb struct fw_eth_tx_pkt_vm_wr *, struct mbuf *, u_int); 240284052Snpstatic int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int); 241284052Snpstatic int add_to_txpkts(struct mbuf *, struct txpkts *, u_int); 242284052Snpstatic u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *, 243284052Snp struct mbuf *, const struct txpkts *, u_int); 244284052Snpstatic void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); 245218792Snpstatic inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); 246284052Snpstatic inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); 247284052Snpstatic inline uint16_t read_hw_cidx(struct sge_eq *); 248284052Snpstatic inline u_int reclaimable_tx_desc(struct sge_eq *); 249284052Snpstatic inline u_int total_available_tx_desc(struct sge_eq *); 250284052Snpstatic u_int reclaim_tx_descs(struct sge_txq *, u_int); 251284052Snpstatic void tx_reclaim(void *, int); 252284052Snpstatic __be64 get_flit(struct sglist_seg *, int, int); 253228561Snpstatic int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 254228561Snp struct mbuf *); 255239336Snpstatic int handle_fw_msg(struct sge_iq *, const struct rss_header *, 256228561Snp struct mbuf *); 257309444Sjhbstatic int t4_handle_wrerr_rpl(struct adapter *, const __be64 *); 258284052Snpstatic void wrq_tx_drain(void *, int); 259284052Snpstatic void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); 260218792Snp 261222510Snpstatic int sysctl_uint16(SYSCTL_HANDLER_ARGS); 262265425Snpstatic int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); 263308321Sjhbstatic int sysctl_tc(SYSCTL_HANDLER_ARGS); 264220873Snp 265269356Snpstatic counter_u64_t extfree_refs; 266269356Snpstatic counter_u64_t extfree_rels; 267269356Snp 268309442Sjhban_handler_t t4_an_handler; 269309442Sjhbfw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; 270309442Sjhbcpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; 271309442Sjhb 272309442Sjhb 273309442Sjhbstatic int 274309442Sjhban_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 275309442Sjhb{ 276309442Sjhb 277309442Sjhb#ifdef INVARIANTS 278309442Sjhb panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 279309442Sjhb#else 280309442Sjhb log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 281309442Sjhb __func__, iq, ctrl); 282309442Sjhb#endif 283309442Sjhb return (EDOOFUS); 284309442Sjhb} 285309442Sjhb 286309442Sjhbint 287309442Sjhbt4_register_an_handler(an_handler_t h) 288309442Sjhb{ 289309442Sjhb uintptr_t *loc, new; 290309442Sjhb 291309442Sjhb new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 292309442Sjhb loc = (uintptr_t *) &t4_an_handler; 293309442Sjhb atomic_store_rel_ptr(loc, new); 294309442Sjhb 295309442Sjhb return (0); 296309442Sjhb} 297309442Sjhb 298309442Sjhbstatic int 299309442Sjhbfw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 300309442Sjhb{ 301309442Sjhb const struct cpl_fw6_msg *cpl = 302309442Sjhb __containerof(rpl, struct cpl_fw6_msg, data[0]); 303309442Sjhb 304309442Sjhb#ifdef INVARIANTS 305309442Sjhb panic("%s: fw_msg type %d", __func__, cpl->type); 306309442Sjhb#else 307309442Sjhb log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 308309442Sjhb#endif 309309442Sjhb return (EDOOFUS); 310309442Sjhb} 311309442Sjhb 312309442Sjhbint 313309442Sjhbt4_register_fw_msg_handler(int type, fw_msg_handler_t h) 314309442Sjhb{ 315309442Sjhb uintptr_t *loc, new; 316309442Sjhb 317309442Sjhb if (type >= nitems(t4_fw_msg_handler)) 318309442Sjhb return (EINVAL); 319309442Sjhb 320309442Sjhb /* 321309442Sjhb * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 322309442Sjhb * handler dispatch table. Reject any attempt to install a handler for 323309442Sjhb * this subtype. 324309442Sjhb */ 325309442Sjhb if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 326309442Sjhb return (EINVAL); 327309442Sjhb 328309442Sjhb new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 329309442Sjhb loc = (uintptr_t *) &t4_fw_msg_handler[type]; 330309442Sjhb atomic_store_rel_ptr(loc, new); 331309442Sjhb 332309442Sjhb return (0); 333309442Sjhb} 334309442Sjhb 335309442Sjhbstatic int 336309442Sjhbcpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 337309442Sjhb{ 338309442Sjhb 339309442Sjhb#ifdef INVARIANTS 340309442Sjhb panic("%s: opcode 0x%02x on iq %p with payload %p", 341309442Sjhb __func__, rss->opcode, iq, m); 342309442Sjhb#else 343309442Sjhb log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 344309442Sjhb __func__, rss->opcode, iq, m); 345309442Sjhb m_freem(m); 346309442Sjhb#endif 347309442Sjhb return (EDOOFUS); 348309442Sjhb} 349309442Sjhb 350309442Sjhbint 351309442Sjhbt4_register_cpl_handler(int opcode, cpl_handler_t h) 352309442Sjhb{ 353309442Sjhb uintptr_t *loc, new; 354309442Sjhb 355309442Sjhb if (opcode >= nitems(t4_cpl_handler)) 356309442Sjhb return (EINVAL); 357309442Sjhb 358309442Sjhb new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 359309442Sjhb loc = (uintptr_t *) &t4_cpl_handler[opcode]; 360309442Sjhb atomic_store_rel_ptr(loc, new); 361309442Sjhb 362309442Sjhb return (0); 363309442Sjhb} 364309442Sjhb 365219392Snp/* 366255050Snp * Called on MOD_LOAD. Validates and calculates the SGE tunables. 367219392Snp */ 368219392Snpvoid 369219392Snpt4_sge_modload(void) 370219392Snp{ 371309442Sjhb int i; 372255050Snp 373239258Snp if (fl_pktshift < 0 || fl_pktshift > 7) { 374239258Snp printf("Invalid hw.cxgbe.fl_pktshift value (%d)," 375239258Snp " using 2 instead.\n", fl_pktshift); 376239258Snp fl_pktshift = 2; 377239258Snp } 378239258Snp 379239258Snp if (spg_len != 64 && spg_len != 128) { 380239258Snp int len; 381239258Snp 382239258Snp#if defined(__i386__) || defined(__amd64__) 383239258Snp len = cpu_clflush_line_size > 64 ? 128 : 64; 384239258Snp#else 385239258Snp len = 64; 386239258Snp#endif 387239258Snp if (spg_len != -1) { 388239258Snp printf("Invalid hw.cxgbe.spg_len value (%d)," 389239258Snp " using %d instead.\n", spg_len, len); 390239258Snp } 391239258Snp spg_len = len; 392239258Snp } 393239258Snp 394239258Snp if (cong_drop < -1 || cong_drop > 1) { 395239258Snp printf("Invalid hw.cxgbe.cong_drop value (%d)," 396239258Snp " using 0 instead.\n", cong_drop); 397239258Snp cong_drop = 0; 398239258Snp } 399269356Snp 400318840Snp if (tscale != 1 && (tscale < 3 || tscale > 17)) { 401318840Snp printf("Invalid hw.cxgbe.tscale value (%d)," 402318840Snp " using 1 instead.\n", tscale); 403318840Snp tscale = 1; 404318840Snp } 405318840Snp 406269356Snp extfree_refs = counter_u64_alloc(M_WAITOK); 407269356Snp extfree_rels = counter_u64_alloc(M_WAITOK); 408269356Snp counter_u64_zero(extfree_refs); 409269356Snp counter_u64_zero(extfree_rels); 410309442Sjhb 411309442Sjhb t4_an_handler = an_not_handled; 412309442Sjhb for (i = 0; i < nitems(t4_fw_msg_handler); i++) 413309442Sjhb t4_fw_msg_handler[i] = fw_msg_not_handled; 414309442Sjhb for (i = 0; i < nitems(t4_cpl_handler); i++) 415309442Sjhb t4_cpl_handler[i] = cpl_not_handled; 416309442Sjhb 417309442Sjhb t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); 418309442Sjhb t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); 419309442Sjhb t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 420309442Sjhb t4_register_cpl_handler(CPL_RX_PKT, t4_eth_rx); 421309442Sjhb t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl); 422309444Sjhb t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl); 423219392Snp} 424219392Snp 425248925Snpvoid 426269356Snpt4_sge_modunload(void) 427269356Snp{ 428269356Snp 429269356Snp counter_u64_free(extfree_refs); 430269356Snp counter_u64_free(extfree_rels); 431269356Snp} 432269356Snp 433269356Snpuint64_t 434269356Snpt4_sge_extfree_refs(void) 435269356Snp{ 436269356Snp uint64_t refs, rels; 437269356Snp 438269356Snp rels = counter_u64_fetch(extfree_rels); 439269356Snp refs = counter_u64_fetch(extfree_refs); 440269356Snp 441269356Snp return (refs - rels); 442269356Snp} 443269356Snp 444281212Snpstatic inline void 445281212Snpsetup_pad_and_pack_boundaries(struct adapter *sc) 446281212Snp{ 447281212Snp uint32_t v, m; 448309560Sjhb int pad, pack, pad_shift; 449281212Snp 450309560Sjhb pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT : 451309560Sjhb X_INGPADBOUNDARY_SHIFT; 452281212Snp pad = fl_pad; 453309560Sjhb if (fl_pad < (1 << pad_shift) || 454309560Sjhb fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) || 455309560Sjhb !powerof2(fl_pad)) { 456281212Snp /* 457281212Snp * If there is any chance that we might use buffer packing and 458281212Snp * the chip is a T4, then pick 64 as the pad/pack boundary. Set 459309560Sjhb * it to the minimum allowed in all other cases. 460281212Snp */ 461309560Sjhb pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift; 462281212Snp 463281212Snp /* 464281212Snp * For fl_pad = 0 we'll still write a reasonable value to the 465281212Snp * register but all the freelists will opt out of padding. 466281212Snp * We'll complain here only if the user tried to set it to a 467281212Snp * value greater than 0 that was invalid. 468281212Snp */ 469281212Snp if (fl_pad > 0) { 470281212Snp device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value" 471281212Snp " (%d), using %d instead.\n", fl_pad, pad); 472281212Snp } 473281212Snp } 474281212Snp m = V_INGPADBOUNDARY(M_INGPADBOUNDARY); 475309560Sjhb v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift); 476281212Snp t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 477281212Snp 478281212Snp if (is_t4(sc)) { 479281212Snp if (fl_pack != -1 && fl_pack != pad) { 480281212Snp /* Complain but carry on. */ 481281212Snp device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored," 482281212Snp " using %d instead.\n", fl_pack, pad); 483281212Snp } 484281212Snp return; 485281212Snp } 486281212Snp 487281212Snp pack = fl_pack; 488281212Snp if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 || 489281212Snp !powerof2(fl_pack)) { 490281212Snp pack = max(sc->params.pci.mps, CACHE_LINE_SIZE); 491281212Snp MPASS(powerof2(pack)); 492281212Snp if (pack < 16) 493281212Snp pack = 16; 494281212Snp if (pack == 32) 495281212Snp pack = 64; 496281212Snp if (pack > 4096) 497281212Snp pack = 4096; 498281212Snp if (fl_pack != -1) { 499281212Snp device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value" 500281212Snp " (%d), using %d instead.\n", fl_pack, pack); 501281212Snp } 502281212Snp } 503281212Snp m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY); 504281212Snp if (pack == 16) 505281212Snp v = V_INGPACKBOUNDARY(0); 506281212Snp else 507281212Snp v = V_INGPACKBOUNDARY(ilog2(pack) - 5); 508281212Snp 509281212Snp MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */ 510281212Snp t4_set_reg_field(sc, A_SGE_CONTROL2, m, v); 511281212Snp} 512281212Snp 513249391Snp/* 514249391Snp * adap->params.vpd.cclk must be set up before this is called. 515249391Snp */ 516248925Snpvoid 517248925Snpt4_tweak_chip_settings(struct adapter *sc) 518248925Snp{ 519248925Snp int i; 520248925Snp uint32_t v, m; 521248925Snp int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; 522249391Snp int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk; 523248925Snp int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 524248925Snp uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 525265425Snp static int sge_flbuf_sizes[] = { 526255050Snp MCLBYTES, 527255050Snp#if MJUMPAGESIZE != MCLBYTES 528255050Snp MJUMPAGESIZE, 529265425Snp MJUMPAGESIZE - CL_METADATA_SIZE, 530265425Snp MJUMPAGESIZE - 2 * MSIZE - CL_METADATA_SIZE, 531255050Snp#endif 532255050Snp MJUM9BYTES, 533255050Snp MJUM16BYTES, 534265425Snp MCLBYTES - MSIZE - CL_METADATA_SIZE, 535265425Snp MJUM9BYTES - CL_METADATA_SIZE, 536265425Snp MJUM16BYTES - CL_METADATA_SIZE, 537255050Snp }; 538248925Snp 539248925Snp KASSERT(sc->flags & MASTER_PF, 540248925Snp ("%s: trying to change chip settings when not master.", __func__)); 541248925Snp 542255050Snp m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE; 543248925Snp v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE | 544237512Snp V_EGRSTATUSPAGESIZE(spg_len == 128); 545248925Snp t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 546218792Snp 547281212Snp setup_pad_and_pack_boundaries(sc); 548255050Snp 549248925Snp v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) | 550228561Snp V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) | 551228561Snp V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) | 552228561Snp V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) | 553228561Snp V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) | 554228561Snp V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) | 555228561Snp V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) | 556228561Snp V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10); 557248925Snp t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v); 558228561Snp 559265425Snp KASSERT(nitems(sge_flbuf_sizes) <= SGE_FLBUF_SIZES, 560265425Snp ("%s: hw buffer size table too big", __func__)); 561265425Snp for (i = 0; i < min(nitems(sge_flbuf_sizes), SGE_FLBUF_SIZES); i++) { 562248925Snp t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i), 563265425Snp sge_flbuf_sizes[i]); 564248925Snp } 565228561Snp 566248925Snp v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) | 567248925Snp V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]); 568248925Snp t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v); 569228561Snp 570249391Snp KASSERT(intr_timer[0] <= timer_max, 571249391Snp ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0], 572249391Snp timer_max)); 573249391Snp for (i = 1; i < nitems(intr_timer); i++) { 574249391Snp KASSERT(intr_timer[i] >= intr_timer[i - 1], 575249391Snp ("%s: timers not listed in increasing order (%d)", 576249391Snp __func__, i)); 577249391Snp 578249391Snp while (intr_timer[i] > timer_max) { 579249391Snp if (i == nitems(intr_timer) - 1) { 580249391Snp intr_timer[i] = timer_max; 581249391Snp break; 582249391Snp } 583249391Snp intr_timer[i] += intr_timer[i - 1]; 584249391Snp intr_timer[i] /= 2; 585249391Snp } 586249391Snp } 587249391Snp 588248925Snp v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) | 589248925Snp V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])); 590248925Snp t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v); 591248925Snp v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) | 592248925Snp V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])); 593248925Snp t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v); 594248925Snp v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) | 595248925Snp V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); 596248925Snp t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); 597228561Snp 598318840Snp if (chip_id(sc) >= CHELSIO_T6) { 599318840Snp m = V_TSCALE(M_TSCALE); 600318840Snp if (tscale == 1) 601318840Snp v = 0; 602318840Snp else 603318840Snp v = V_TSCALE(tscale - 2); 604318840Snp t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); 605318840Snp } 606318840Snp 607248925Snp /* 4K, 16K, 64K, 256K DDP "page sizes" */ 608248925Snp v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 609248925Snp t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); 610248925Snp 611248925Snp m = v = F_TDDPTAGTCB; 612248925Snp t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); 613248925Snp 614248925Snp m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 615248925Snp F_RESETDDPOFFSET; 616248925Snp v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 617248925Snp t4_set_reg_field(sc, A_TP_PARA_REG5, m, v); 618248925Snp} 619248925Snp 620248925Snp/* 621281212Snp * SGE wants the buffer to be at least 64B and then a multiple of 16. If 622309449Sjhb * padding is in use, the buffer's start and end need to be aligned to the pad 623281212Snp * boundary as well. We'll just make sure that the size is a multiple of the 624281212Snp * boundary here, it is up to the buffer allocation code to make sure the start 625281212Snp * of the buffer is aligned as well. 626265425Snp */ 627265425Snpstatic inline int 628281212Snphwsz_ok(struct adapter *sc, int hwsz) 629265425Snp{ 630308304Sjhb int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1; 631265425Snp 632265425Snp return (hwsz >= 64 && (hwsz & mask) == 0); 633265425Snp} 634265425Snp 635265425Snp/* 636248925Snp * XXX: driver really should be able to deal with unexpected settings. 637248925Snp */ 638248925Snpint 639248925Snpt4_read_chip_settings(struct adapter *sc) 640248925Snp{ 641248925Snp struct sge *s = &sc->sge; 642308304Sjhb struct sge_params *sp = &sc->params.sge; 643255050Snp int i, j, n, rc = 0; 644248925Snp uint32_t m, v, r; 645248925Snp uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 646265425Snp static int sw_buf_sizes[] = { /* Sorted by size */ 647255050Snp MCLBYTES, 648255050Snp#if MJUMPAGESIZE != MCLBYTES 649255050Snp MJUMPAGESIZE, 650255050Snp#endif 651255050Snp MJUM9BYTES, 652255050Snp MJUM16BYTES 653255050Snp }; 654265425Snp struct sw_zone_info *swz, *safe_swz; 655265425Snp struct hw_buf_info *hwb; 656248925Snp 657308304Sjhb m = F_RXPKTCPLMODE; 658308304Sjhb v = F_RXPKTCPLMODE; 659309447Sjhb r = sc->params.sge.sge_control; 660248925Snp if ((r & m) != v) { 661248925Snp device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); 662228561Snp rc = EINVAL; 663228561Snp } 664228561Snp 665308304Sjhb /* 666308304Sjhb * If this changes then every single use of PAGE_SHIFT in the driver 667308304Sjhb * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift. 668308304Sjhb */ 669308304Sjhb if (sp->page_shift != PAGE_SHIFT) { 670248925Snp device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r); 671228561Snp rc = EINVAL; 672228561Snp } 673228561Snp 674265425Snp /* Filter out unusable hw buffer sizes entirely (mark with -2). */ 675265425Snp hwb = &s->hw_buf_info[0]; 676265425Snp for (i = 0; i < nitems(s->hw_buf_info); i++, hwb++) { 677309447Sjhb r = sc->params.sge.sge_fl_buffer_size[i]; 678265425Snp hwb->size = r; 679281212Snp hwb->zidx = hwsz_ok(sc, r) ? -1 : -2; 680265425Snp hwb->next = -1; 681265425Snp } 682265425Snp 683255050Snp /* 684265425Snp * Create a sorted list in decreasing order of hw buffer sizes (and so 685265425Snp * increasing order of spare area) for each software zone. 686281212Snp * 687281212Snp * If padding is enabled then the start and end of the buffer must align 688281212Snp * to the pad boundary; if packing is enabled then they must align with 689281212Snp * the pack boundary as well. Allocations from the cluster zones are 690281212Snp * aligned to min(size, 4K), so the buffer starts at that alignment and 691281212Snp * ends at hwb->size alignment. If mbuf inlining is allowed the 692281212Snp * starting alignment will be reduced to MSIZE and the driver will 693281212Snp * exercise appropriate caution when deciding on the best buffer layout 694281212Snp * to use. 695255050Snp */ 696265425Snp n = 0; /* no usable buffer size to begin with */ 697265425Snp swz = &s->sw_zone_info[0]; 698265425Snp safe_swz = NULL; 699265425Snp for (i = 0; i < SW_ZONE_SIZES; i++, swz++) { 700265425Snp int8_t head = -1, tail = -1; 701265425Snp 702265425Snp swz->size = sw_buf_sizes[i]; 703265425Snp swz->zone = m_getzone(swz->size); 704265425Snp swz->type = m_gettype(swz->size); 705265425Snp 706281212Snp if (swz->size < PAGE_SIZE) { 707281212Snp MPASS(powerof2(swz->size)); 708308304Sjhb if (fl_pad && (swz->size % sp->pad_boundary != 0)) 709281212Snp continue; 710281212Snp } 711281212Snp 712265425Snp if (swz->size == safest_rx_cluster) 713265425Snp safe_swz = swz; 714265425Snp 715265425Snp hwb = &s->hw_buf_info[0]; 716265425Snp for (j = 0; j < SGE_FLBUF_SIZES; j++, hwb++) { 717265425Snp if (hwb->zidx != -1 || hwb->size > swz->size) 718265425Snp continue; 719281212Snp#ifdef INVARIANTS 720281212Snp if (fl_pad) 721308304Sjhb MPASS(hwb->size % sp->pad_boundary == 0); 722281212Snp#endif 723265425Snp hwb->zidx = i; 724265425Snp if (head == -1) 725265425Snp head = tail = j; 726265425Snp else if (hwb->size < s->hw_buf_info[tail].size) { 727265425Snp s->hw_buf_info[tail].next = j; 728265425Snp tail = j; 729265425Snp } else { 730265425Snp int8_t *cur; 731265425Snp struct hw_buf_info *t; 732265425Snp 733265425Snp for (cur = &head; *cur != -1; cur = &t->next) { 734265425Snp t = &s->hw_buf_info[*cur]; 735265425Snp if (hwb->size == t->size) { 736265425Snp hwb->zidx = -2; 737265425Snp break; 738265425Snp } 739265425Snp if (hwb->size > t->size) { 740265425Snp hwb->next = *cur; 741265425Snp *cur = j; 742265425Snp break; 743265425Snp } 744265425Snp } 745265425Snp } 746228561Snp } 747265425Snp swz->head_hwidx = head; 748265425Snp swz->tail_hwidx = tail; 749265425Snp 750265425Snp if (tail != -1) { 751255050Snp n++; 752265425Snp if (swz->size - s->hw_buf_info[tail].size >= 753265425Snp CL_METADATA_SIZE) 754265425Snp sc->flags |= BUF_PACKING_OK; 755255050Snp } 756255050Snp } 757255050Snp if (n == 0) { 758255050Snp device_printf(sc->dev, "no usable SGE FL buffer size.\n"); 759255050Snp rc = EINVAL; 760255050Snp } 761218792Snp 762265425Snp s->safe_hwidx1 = -1; 763265425Snp s->safe_hwidx2 = -1; 764265425Snp if (safe_swz != NULL) { 765265425Snp s->safe_hwidx1 = safe_swz->head_hwidx; 766265425Snp for (i = safe_swz->head_hwidx; i != -1; i = hwb->next) { 767265425Snp int spare; 768265425Snp 769265425Snp hwb = &s->hw_buf_info[i]; 770281212Snp#ifdef INVARIANTS 771281212Snp if (fl_pad) 772308304Sjhb MPASS(hwb->size % sp->pad_boundary == 0); 773281212Snp#endif 774265425Snp spare = safe_swz->size - hwb->size; 775281212Snp if (spare >= CL_METADATA_SIZE) { 776265425Snp s->safe_hwidx2 = i; 777265425Snp break; 778281212Snp } 779265425Snp } 780265425Snp } 781265425Snp 782309447Sjhb if (sc->flags & IS_VF) 783309447Sjhb return (0); 784309447Sjhb 785248925Snp v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 786248925Snp r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); 787248925Snp if (r != v) { 788248925Snp device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r); 789248925Snp rc = EINVAL; 790248925Snp } 791228561Snp 792248925Snp m = v = F_TDDPTAGTCB; 793248925Snp r = t4_read_reg(sc, A_ULP_RX_CTL); 794248925Snp if ((r & m) != v) { 795248925Snp device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r); 796248925Snp rc = EINVAL; 797248925Snp } 798239336Snp 799248925Snp m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 800248925Snp F_RESETDDPOFFSET; 801248925Snp v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 802248925Snp r = t4_read_reg(sc, A_TP_PARA_REG5); 803248925Snp if ((r & m) != v) { 804248925Snp device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r); 805248925Snp rc = EINVAL; 806248925Snp } 807248925Snp 808353418Snp t4_init_tp_params(sc, 1); 809248925Snp 810248925Snp t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 811248925Snp t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); 812248925Snp 813228561Snp return (rc); 814218792Snp} 815218792Snp 816218792Snpint 817218792Snpt4_create_dma_tag(struct adapter *sc) 818218792Snp{ 819218792Snp int rc; 820218792Snp 821218792Snp rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, 822218792Snp BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, 823218792Snp BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, 824218792Snp NULL, &sc->dmat); 825218792Snp if (rc != 0) { 826218792Snp device_printf(sc->dev, 827218792Snp "failed to create main DMA tag: %d\n", rc); 828218792Snp } 829218792Snp 830218792Snp return (rc); 831218792Snp} 832218792Snp 833253829Snpvoid 834253829Snpt4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 835253829Snp struct sysctl_oid_list *children) 836253829Snp{ 837308304Sjhb struct sge_params *sp = &sc->params.sge; 838253829Snp 839265425Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes", 840265425Snp CTLTYPE_STRING | CTLFLAG_RD, &sc->sge, 0, sysctl_bufsizes, "A", 841265425Snp "freelist buffer sizes"); 842265425Snp 843253829Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD, 844308304Sjhb NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)"); 845253829Snp 846253829Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD, 847308304Sjhb NULL, sp->pad_boundary, "payload pad boundary (bytes)"); 848253829Snp 849253829Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD, 850308304Sjhb NULL, sp->spg_len, "status page size (bytes)"); 851253829Snp 852253829Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD, 853253829Snp NULL, cong_drop, "congestion drop setting"); 854255050Snp 855255050Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD, 856308304Sjhb NULL, sp->pack_boundary, "payload pack boundary (bytes)"); 857253829Snp} 858253829Snp 859218792Snpint 860218792Snpt4_destroy_dma_tag(struct adapter *sc) 861218792Snp{ 862218792Snp if (sc->dmat) 863218792Snp bus_dma_tag_destroy(sc->dmat); 864218792Snp 865218792Snp return (0); 866218792Snp} 867218792Snp 868218792Snp/* 869228561Snp * Allocate and initialize the firmware event queue and the management queue. 870218792Snp * 871218792Snp * Returns errno on failure. Resources allocated up to that point may still be 872218792Snp * allocated. Caller is responsible for cleanup in case this function fails. 873218792Snp */ 874218792Snpint 875220873Snpt4_setup_adapter_queues(struct adapter *sc) 876218792Snp{ 877228561Snp int rc; 878218792Snp 879218792Snp ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 880218792Snp 881228561Snp sysctl_ctx_init(&sc->ctx); 882228561Snp sc->flags |= ADAP_SYSCTL_CTX; 883220873Snp 884222510Snp /* 885222510Snp * Firmware event queue 886222510Snp */ 887228561Snp rc = alloc_fwq(sc); 888241398Snp if (rc != 0) 889220873Snp return (rc); 890218792Snp 891220873Snp /* 892228561Snp * Management queue. This is just a control queue that uses the fwq as 893228561Snp * its associated iq. 894220873Snp */ 895309447Sjhb if (!(sc->flags & IS_VF)) 896309447Sjhb rc = alloc_mgmtq(sc); 897220873Snp 898218792Snp return (rc); 899218792Snp} 900218792Snp 901218792Snp/* 902218792Snp * Idempotent 903218792Snp */ 904218792Snpint 905220873Snpt4_teardown_adapter_queues(struct adapter *sc) 906218792Snp{ 907218792Snp 908218792Snp ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 909218792Snp 910228561Snp /* Do this before freeing the queue */ 911228561Snp if (sc->flags & ADAP_SYSCTL_CTX) { 912220873Snp sysctl_ctx_free(&sc->ctx); 913228561Snp sc->flags &= ~ADAP_SYSCTL_CTX; 914220873Snp } 915220873Snp 916228561Snp free_mgmtq(sc); 917228561Snp free_fwq(sc); 918220873Snp 919228561Snp return (0); 920228561Snp} 921222510Snp 922228561Snpstatic inline int 923308154Sjhbfirst_vector(struct vi_info *vi) 924270297Snp{ 925308154Sjhb struct adapter *sc = vi->pi->adapter; 926270297Snp 927228561Snp if (sc->intr_count == 1) 928228561Snp return (0); 929228561Snp 930308154Sjhb return (vi->first_intr); 931218792Snp} 932218792Snp 933228561Snp/* 934228561Snp * Given an arbitrary "index," come up with an iq that can be used by other 935308154Sjhb * queues (of this VI) for interrupt forwarding, SGE egress updates, etc. 936228561Snp * The iq returned is guaranteed to be something that takes direct interrupts. 937228561Snp */ 938228561Snpstatic struct sge_iq * 939308154Sjhbvi_intr_iq(struct vi_info *vi, int idx) 940228561Snp{ 941308154Sjhb struct adapter *sc = vi->pi->adapter; 942228561Snp struct sge *s = &sc->sge; 943228561Snp struct sge_iq *iq = NULL; 944270297Snp int nintr, i; 945228561Snp 946228561Snp if (sc->intr_count == 1) 947228561Snp return (&sc->sge.fwq); 948228561Snp 949308154Sjhb nintr = vi->nintr; 950318855Snp#ifdef DEV_NETMAP 951318855Snp /* Do not consider any netmap-only interrupts */ 952318855Snp if (vi->flags & INTR_RXQ && vi->nnmrxq > vi->nrxq) 953318855Snp nintr -= vi->nnmrxq - vi->nrxq; 954318855Snp#endif 955270297Snp KASSERT(nintr != 0, 956308154Sjhb ("%s: vi %p has no exclusive interrupts, total interrupts = %d", 957308154Sjhb __func__, vi, sc->intr_count)); 958270297Snp i = idx % nintr; 959265425Snp 960308154Sjhb if (vi->flags & INTR_RXQ) { 961308154Sjhb if (i < vi->nrxq) { 962308154Sjhb iq = &s->rxq[vi->first_rxq + i].iq; 963270297Snp goto done; 964270297Snp } 965308154Sjhb i -= vi->nrxq; 966228561Snp } 967270297Snp#ifdef TCP_OFFLOAD 968308154Sjhb if (vi->flags & INTR_OFLD_RXQ) { 969308154Sjhb if (i < vi->nofldrxq) { 970308154Sjhb iq = &s->ofld_rxq[vi->first_ofld_rxq + i].iq; 971270297Snp goto done; 972270297Snp } 973308154Sjhb i -= vi->nofldrxq; 974270297Snp } 975228561Snp#endif 976308154Sjhb panic("%s: vi %p, intr_flags 0x%lx, idx %d, total intr %d\n", __func__, 977308154Sjhb vi, vi->flags & INTR_ALL, idx, nintr); 978270297Snpdone: 979270297Snp MPASS(iq != NULL); 980270297Snp KASSERT(iq->flags & IQ_INTR, 981308154Sjhb ("%s: iq %p (vi %p, intr_flags 0x%lx, idx %d)", __func__, iq, vi, 982308154Sjhb vi->flags & INTR_ALL, idx)); 983228561Snp return (iq); 984228561Snp} 985228561Snp 986265425Snp/* Maximum payload that can be delivered with a single iq descriptor */ 987239266Snpstatic inline int 988265425Snpmtu_to_max_payload(struct adapter *sc, int mtu, const int toe) 989239266Snp{ 990265425Snp int payload; 991239266Snp 992252728Snp#ifdef TCP_OFFLOAD 993265425Snp if (toe) { 994265425Snp payload = sc->tt.rx_coalesce ? 995265425Snp G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)) : mtu; 996265425Snp } else { 997265425Snp#endif 998265425Snp /* large enough even when hw VLAN extraction is disabled */ 999308304Sjhb payload = sc->params.sge.fl_pktshift + ETHER_HDR_LEN + 1000308304Sjhb ETHER_VLAN_ENCAP_LEN + mtu; 1001265425Snp#ifdef TCP_OFFLOAD 1002265425Snp } 1003265425Snp#endif 1004252728Snp 1005265425Snp return (payload); 1006252728Snp} 1007252728Snp 1008218792Snpint 1009308154Sjhbt4_setup_vi_queues(struct vi_info *vi) 1010218792Snp{ 1011228561Snp int rc = 0, i, j, intr_idx, iqid; 1012218792Snp struct sge_rxq *rxq; 1013218792Snp struct sge_txq *txq; 1014228561Snp struct sge_wrq *ctrlq; 1015237263Snp#ifdef TCP_OFFLOAD 1016228561Snp struct sge_ofld_rxq *ofld_rxq; 1017228561Snp struct sge_wrq *ofld_txq; 1018228561Snp#endif 1019270297Snp#ifdef DEV_NETMAP 1020308154Sjhb int saved_idx; 1021270297Snp struct sge_nm_rxq *nm_rxq; 1022270297Snp struct sge_nm_txq *nm_txq; 1023270297Snp#endif 1024218792Snp char name[16]; 1025308154Sjhb struct port_info *pi = vi->pi; 1026218792Snp struct adapter *sc = pi->adapter; 1027308154Sjhb struct ifnet *ifp = vi->ifp; 1028308154Sjhb struct sysctl_oid *oid = device_get_sysctl_tree(vi->dev); 1029228561Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 1030281212Snp int maxp, mtu = ifp->if_mtu; 1031218792Snp 1032228561Snp /* Interrupt vector to start from (when using multiple vectors) */ 1033308154Sjhb intr_idx = first_vector(vi); 1034228561Snp 1035308154Sjhb#ifdef DEV_NETMAP 1036308154Sjhb saved_idx = intr_idx; 1037308154Sjhb if (ifp->if_capabilities & IFCAP_NETMAP) { 1038308154Sjhb 1039308154Sjhb /* netmap is supported with direct interrupts only. */ 1040308154Sjhb MPASS(vi->flags & INTR_RXQ); 1041308154Sjhb 1042308154Sjhb /* 1043308154Sjhb * We don't have buffers to back the netmap rx queues 1044308154Sjhb * right now so we create the queues in a way that 1045308154Sjhb * doesn't set off any congestion signal in the chip. 1046308154Sjhb */ 1047308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_rxq", 1048308154Sjhb CTLFLAG_RD, NULL, "rx queues"); 1049308154Sjhb for_each_nm_rxq(vi, i, nm_rxq) { 1050308154Sjhb rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i, oid); 1051308154Sjhb if (rc != 0) 1052308154Sjhb goto done; 1053308154Sjhb intr_idx++; 1054308154Sjhb } 1055308154Sjhb 1056308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_txq", 1057308154Sjhb CTLFLAG_RD, NULL, "tx queues"); 1058308154Sjhb for_each_nm_txq(vi, i, nm_txq) { 1059308154Sjhb iqid = vi->first_nm_rxq + (i % vi->nnmrxq); 1060308154Sjhb rc = alloc_nm_txq(vi, nm_txq, iqid, i, oid); 1061308154Sjhb if (rc != 0) 1062308154Sjhb goto done; 1063308154Sjhb } 1064308154Sjhb } 1065308154Sjhb 1066308154Sjhb /* Normal rx queues and netmap rx queues share the same interrupts. */ 1067308154Sjhb intr_idx = saved_idx; 1068308154Sjhb#endif 1069308154Sjhb 1070228561Snp /* 1071270297Snp * First pass over all NIC and TOE rx queues: 1072228561Snp * a) initialize iq and fl 1073228561Snp * b) allocate queue iff it will take direct interrupts. 1074228561Snp */ 1075265425Snp maxp = mtu_to_max_payload(sc, mtu, 0); 1076308154Sjhb if (vi->flags & INTR_RXQ) { 1077308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", 1078270297Snp CTLFLAG_RD, NULL, "rx queues"); 1079270297Snp } 1080308154Sjhb for_each_rxq(vi, i, rxq) { 1081218792Snp 1082308154Sjhb init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq); 1083218792Snp 1084218792Snp snprintf(name, sizeof(name), "%s rxq%d-fl", 1085308154Sjhb device_get_nameunit(vi->dev), i); 1086308154Sjhb init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name); 1087218792Snp 1088308154Sjhb if (vi->flags & INTR_RXQ) { 1089228561Snp rxq->iq.flags |= IQ_INTR; 1090308154Sjhb rc = alloc_rxq(vi, rxq, intr_idx, i, oid); 1091228561Snp if (rc != 0) 1092228561Snp goto done; 1093228561Snp intr_idx++; 1094228561Snp } 1095228561Snp } 1096308154Sjhb#ifdef DEV_NETMAP 1097308154Sjhb if (ifp->if_capabilities & IFCAP_NETMAP) 1098308154Sjhb intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq); 1099308154Sjhb#endif 1100237263Snp#ifdef TCP_OFFLOAD 1101265425Snp maxp = mtu_to_max_payload(sc, mtu, 1); 1102308154Sjhb if (vi->flags & INTR_OFLD_RXQ) { 1103308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", 1104270297Snp CTLFLAG_RD, NULL, 1105270297Snp "rx queues for offloaded TCP connections"); 1106270297Snp } 1107308154Sjhb for_each_ofld_rxq(vi, i, ofld_rxq) { 1108228561Snp 1109308154Sjhb init_iq(&ofld_rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, 1110308154Sjhb vi->qsize_rxq); 1111228561Snp 1112228561Snp snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", 1113308154Sjhb device_get_nameunit(vi->dev), i); 1114308154Sjhb init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name); 1115228561Snp 1116308154Sjhb if (vi->flags & INTR_OFLD_RXQ) { 1117228561Snp ofld_rxq->iq.flags |= IQ_INTR; 1118308154Sjhb rc = alloc_ofld_rxq(vi, ofld_rxq, intr_idx, i, oid); 1119228561Snp if (rc != 0) 1120228561Snp goto done; 1121228561Snp intr_idx++; 1122228561Snp } 1123228561Snp } 1124228561Snp#endif 1125228561Snp 1126228561Snp /* 1127270297Snp * Second pass over all NIC and TOE rx queues. The queues forwarding 1128228561Snp * their interrupts are allocated now. 1129228561Snp */ 1130228561Snp j = 0; 1131308154Sjhb if (!(vi->flags & INTR_RXQ)) { 1132308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", 1133270297Snp CTLFLAG_RD, NULL, "rx queues"); 1134308154Sjhb for_each_rxq(vi, i, rxq) { 1135270297Snp MPASS(!(rxq->iq.flags & IQ_INTR)); 1136228561Snp 1137308154Sjhb intr_idx = vi_intr_iq(vi, j)->abs_id; 1138228561Snp 1139308154Sjhb rc = alloc_rxq(vi, rxq, intr_idx, i, oid); 1140270297Snp if (rc != 0) 1141270297Snp goto done; 1142270297Snp j++; 1143270297Snp } 1144218792Snp } 1145237263Snp#ifdef TCP_OFFLOAD 1146308154Sjhb if (vi->nofldrxq != 0 && !(vi->flags & INTR_OFLD_RXQ)) { 1147308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", 1148270297Snp CTLFLAG_RD, NULL, 1149270297Snp "rx queues for offloaded TCP connections"); 1150308154Sjhb for_each_ofld_rxq(vi, i, ofld_rxq) { 1151270297Snp MPASS(!(ofld_rxq->iq.flags & IQ_INTR)); 1152228561Snp 1153308154Sjhb intr_idx = vi_intr_iq(vi, j)->abs_id; 1154228561Snp 1155308154Sjhb rc = alloc_ofld_rxq(vi, ofld_rxq, intr_idx, i, oid); 1156270297Snp if (rc != 0) 1157270297Snp goto done; 1158270297Snp j++; 1159270297Snp } 1160228561Snp } 1161228561Snp#endif 1162228561Snp 1163228561Snp /* 1164228561Snp * Now the tx queues. Only one pass needed. 1165228561Snp */ 1166308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD, 1167228561Snp NULL, "tx queues"); 1168228561Snp j = 0; 1169308154Sjhb for_each_txq(vi, i, txq) { 1170308154Sjhb iqid = vi_intr_iq(vi, j)->cntxt_id; 1171218792Snp snprintf(name, sizeof(name), "%s txq%d", 1172308154Sjhb device_get_nameunit(vi->dev), i); 1173308304Sjhb init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, iqid, 1174228561Snp name); 1175218792Snp 1176308154Sjhb rc = alloc_txq(vi, txq, i, oid); 1177218792Snp if (rc != 0) 1178218792Snp goto done; 1179228561Snp j++; 1180218792Snp } 1181237263Snp#ifdef TCP_OFFLOAD 1182308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", 1183228561Snp CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections"); 1184308154Sjhb for_each_ofld_txq(vi, i, ofld_txq) { 1185270297Snp struct sysctl_oid *oid2; 1186228561Snp 1187308154Sjhb iqid = vi_intr_iq(vi, j)->cntxt_id; 1188228561Snp snprintf(name, sizeof(name), "%s ofld_txq%d", 1189308154Sjhb device_get_nameunit(vi->dev), i); 1190308304Sjhb init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan, 1191228561Snp iqid, name); 1192228561Snp 1193228561Snp snprintf(name, sizeof(name), "%d", i); 1194308154Sjhb oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO, 1195228561Snp name, CTLFLAG_RD, NULL, "offload tx queue"); 1196228561Snp 1197308154Sjhb rc = alloc_wrq(sc, vi, ofld_txq, oid2); 1198228561Snp if (rc != 0) 1199228561Snp goto done; 1200228561Snp j++; 1201228561Snp } 1202228561Snp#endif 1203228561Snp 1204228561Snp /* 1205228561Snp * Finally, the control queue. 1206228561Snp */ 1207309447Sjhb if (!IS_MAIN_VI(vi) || sc->flags & IS_VF) 1208308154Sjhb goto done; 1209308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD, 1210228561Snp NULL, "ctrl queue"); 1211228561Snp ctrlq = &sc->sge.ctrlq[pi->port_id]; 1212308154Sjhb iqid = vi_intr_iq(vi, 0)->cntxt_id; 1213308154Sjhb snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(vi->dev)); 1214308304Sjhb init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid, 1215308304Sjhb name); 1216308154Sjhb rc = alloc_wrq(sc, vi, ctrlq, oid); 1217228561Snp 1218218792Snpdone: 1219218792Snp if (rc) 1220308154Sjhb t4_teardown_vi_queues(vi); 1221218792Snp 1222218792Snp return (rc); 1223218792Snp} 1224218792Snp 1225218792Snp/* 1226218792Snp * Idempotent 1227218792Snp */ 1228218792Snpint 1229308154Sjhbt4_teardown_vi_queues(struct vi_info *vi) 1230218792Snp{ 1231218792Snp int i; 1232308154Sjhb struct port_info *pi = vi->pi; 1233228561Snp struct adapter *sc = pi->adapter; 1234218792Snp struct sge_rxq *rxq; 1235218792Snp struct sge_txq *txq; 1236237263Snp#ifdef TCP_OFFLOAD 1237228561Snp struct sge_ofld_rxq *ofld_rxq; 1238228561Snp struct sge_wrq *ofld_txq; 1239228561Snp#endif 1240270297Snp#ifdef DEV_NETMAP 1241270297Snp struct sge_nm_rxq *nm_rxq; 1242270297Snp struct sge_nm_txq *nm_txq; 1243270297Snp#endif 1244218792Snp 1245218792Snp /* Do this before freeing the queues */ 1246308154Sjhb if (vi->flags & VI_SYSCTL_CTX) { 1247308154Sjhb sysctl_ctx_free(&vi->ctx); 1248308154Sjhb vi->flags &= ~VI_SYSCTL_CTX; 1249218792Snp } 1250218792Snp 1251308154Sjhb#ifdef DEV_NETMAP 1252308154Sjhb if (vi->ifp->if_capabilities & IFCAP_NETMAP) { 1253308154Sjhb for_each_nm_txq(vi, i, nm_txq) { 1254308154Sjhb free_nm_txq(vi, nm_txq); 1255308154Sjhb } 1256308154Sjhb 1257308154Sjhb for_each_nm_rxq(vi, i, nm_rxq) { 1258308154Sjhb free_nm_rxq(vi, nm_rxq); 1259308154Sjhb } 1260308154Sjhb } 1261308154Sjhb#endif 1262308154Sjhb 1263228561Snp /* 1264228561Snp * Take down all the tx queues first, as they reference the rx queues 1265228561Snp * (for egress updates, etc.). 1266228561Snp */ 1267228561Snp 1268309447Sjhb if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF)) 1269308154Sjhb free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 1270228561Snp 1271308154Sjhb for_each_txq(vi, i, txq) { 1272308154Sjhb free_txq(vi, txq); 1273218792Snp } 1274237263Snp#ifdef TCP_OFFLOAD 1275308154Sjhb for_each_ofld_txq(vi, i, ofld_txq) { 1276228561Snp free_wrq(sc, ofld_txq); 1277228561Snp } 1278228561Snp#endif 1279228561Snp 1280228561Snp /* 1281228561Snp * Then take down the rx queues that forward their interrupts, as they 1282228561Snp * reference other rx queues. 1283228561Snp */ 1284228561Snp 1285308154Sjhb for_each_rxq(vi, i, rxq) { 1286228561Snp if ((rxq->iq.flags & IQ_INTR) == 0) 1287308154Sjhb free_rxq(vi, rxq); 1288218792Snp } 1289237263Snp#ifdef TCP_OFFLOAD 1290308154Sjhb for_each_ofld_rxq(vi, i, ofld_rxq) { 1291228561Snp if ((ofld_rxq->iq.flags & IQ_INTR) == 0) 1292308154Sjhb free_ofld_rxq(vi, ofld_rxq); 1293228561Snp } 1294228561Snp#endif 1295228561Snp 1296228561Snp /* 1297228561Snp * Then take down the rx queues that take direct interrupts. 1298228561Snp */ 1299228561Snp 1300308154Sjhb for_each_rxq(vi, i, rxq) { 1301228561Snp if (rxq->iq.flags & IQ_INTR) 1302308154Sjhb free_rxq(vi, rxq); 1303228561Snp } 1304237263Snp#ifdef TCP_OFFLOAD 1305308154Sjhb for_each_ofld_rxq(vi, i, ofld_rxq) { 1306228561Snp if (ofld_rxq->iq.flags & IQ_INTR) 1307308154Sjhb free_ofld_rxq(vi, ofld_rxq); 1308228561Snp } 1309228561Snp#endif 1310228561Snp 1311218792Snp return (0); 1312218792Snp} 1313218792Snp 1314228561Snp/* 1315228561Snp * Deals with errors and the firmware event queue. All data rx queues forward 1316228561Snp * their interrupt to the firmware event queue. 1317228561Snp */ 1318218792Snpvoid 1319218792Snpt4_intr_all(void *arg) 1320218792Snp{ 1321218792Snp struct adapter *sc = arg; 1322228561Snp struct sge_iq *fwq = &sc->sge.fwq; 1323218792Snp 1324218792Snp t4_intr_err(arg); 1325228561Snp if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) { 1326228561Snp service_iq(fwq, 0); 1327228561Snp atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE); 1328218792Snp } 1329218792Snp} 1330218792Snp 1331218792Snp/* Deals with error interrupts */ 1332218792Snpvoid 1333218792Snpt4_intr_err(void *arg) 1334218792Snp{ 1335218792Snp struct adapter *sc = arg; 1336218792Snp 1337222510Snp t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 1338218792Snp t4_slow_intr_handler(sc); 1339218792Snp} 1340218792Snp 1341218792Snpvoid 1342218792Snpt4_intr_evt(void *arg) 1343218792Snp{ 1344218792Snp struct sge_iq *iq = arg; 1345220649Snp 1346228561Snp if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1347228561Snp service_iq(iq, 0); 1348228561Snp atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1349222510Snp } 1350220649Snp} 1351220649Snp 1352228561Snpvoid 1353228561Snpt4_intr(void *arg) 1354220649Snp{ 1355220649Snp struct sge_iq *iq = arg; 1356228561Snp 1357228561Snp if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1358228561Snp service_iq(iq, 0); 1359228561Snp atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1360228561Snp } 1361228561Snp} 1362228561Snp 1363308154Sjhbvoid 1364308154Sjhbt4_vi_intr(void *arg) 1365308154Sjhb{ 1366308154Sjhb struct irq *irq = arg; 1367308154Sjhb 1368308154Sjhb#ifdef DEV_NETMAP 1369308154Sjhb if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) { 1370308154Sjhb t4_nm_intr(irq->nm_rxq); 1371308154Sjhb atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON); 1372308154Sjhb } 1373308154Sjhb#endif 1374308154Sjhb if (irq->rxq != NULL) 1375308154Sjhb t4_intr(irq->rxq); 1376308154Sjhb} 1377308154Sjhb 1378228561Snp/* 1379228561Snp * Deals with anything and everything on the given ingress queue. 1380228561Snp */ 1381228561Snpstatic int 1382228561Snpservice_iq(struct sge_iq *iq, int budget) 1383228561Snp{ 1384228561Snp struct sge_iq *q; 1385237263Snp struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 1386270297Snp struct sge_fl *fl; /* Use iff IQ_HAS_FL */ 1387218792Snp struct adapter *sc = iq->adapter; 1388270297Snp struct iq_desc *d = &iq->desc[iq->cidx]; 1389270297Snp int ndescs = 0, limit; 1390270297Snp int rsp_type, refill; 1391228561Snp uint32_t lq; 1392270297Snp uint16_t fl_hw_cidx; 1393228561Snp struct mbuf *m0; 1394228561Snp STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 1395255015Snp#if defined(INET) || defined(INET6) 1396255015Snp const struct timeval lro_timeout = {0, sc->lro_timeout}; 1397255015Snp#endif 1398218792Snp 1399228561Snp KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 1400218792Snp 1401270297Snp limit = budget ? budget : iq->qsize / 16; 1402270297Snp 1403270297Snp if (iq->flags & IQ_HAS_FL) { 1404270297Snp fl = &rxq->fl; 1405270297Snp fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ 1406270297Snp } else { 1407270297Snp fl = NULL; 1408270297Snp fl_hw_cidx = 0; /* to silence gcc warning */ 1409270297Snp } 1410270297Snp 1411228561Snp /* 1412228561Snp * We always come back and check the descriptor ring for new indirect 1413228561Snp * interrupts and other responses after running a single handler. 1414228561Snp */ 1415228561Snp for (;;) { 1416270297Snp while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 1417218792Snp 1418228561Snp rmb(); 1419218792Snp 1420270297Snp refill = 0; 1421228561Snp m0 = NULL; 1422270297Snp rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 1423270297Snp lq = be32toh(d->rsp.pldbuflen_qid); 1424218792Snp 1425228561Snp switch (rsp_type) { 1426228561Snp case X_RSPD_TYPE_FLBUF: 1427228561Snp 1428228561Snp KASSERT(iq->flags & IQ_HAS_FL, 1429228561Snp ("%s: data for an iq (%p) with no freelist", 1430228561Snp __func__, iq)); 1431228561Snp 1432270297Snp m0 = get_fl_payload(sc, fl, lq); 1433255050Snp if (__predict_false(m0 == NULL)) 1434255050Snp goto process_iql; 1435270297Snp refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2; 1436228561Snp#ifdef T4_PKT_TIMESTAMP 1437228561Snp /* 1438228561Snp * 60 bit timestamp for the payload is 1439228561Snp * *(uint64_t *)m0->m_pktdat. Note that it is 1440228561Snp * in the leading free-space in the mbuf. The 1441228561Snp * kernel can clobber it during a pullup, 1442228561Snp * m_copymdata, etc. You need to make sure that 1443228561Snp * the mbuf reaches you unmolested if you care 1444228561Snp * about the timestamp. 1445228561Snp */ 1446228561Snp *(uint64_t *)m0->m_pktdat = 1447228561Snp be64toh(ctrl->u.last_flit) & 1448228561Snp 0xfffffffffffffff; 1449228561Snp#endif 1450228561Snp 1451228561Snp /* fall through */ 1452228561Snp 1453228561Snp case X_RSPD_TYPE_CPL: 1454270297Snp KASSERT(d->rss.opcode < NUM_CPL_CMDS, 1455228561Snp ("%s: bad opcode %02x.", __func__, 1456270297Snp d->rss.opcode)); 1457309442Sjhb t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); 1458228561Snp break; 1459228561Snp 1460228561Snp case X_RSPD_TYPE_INTR: 1461228561Snp 1462228561Snp /* 1463228561Snp * Interrupts should be forwarded only to queues 1464228561Snp * that are not forwarding their interrupts. 1465228561Snp * This means service_iq can recurse but only 1 1466228561Snp * level deep. 1467228561Snp */ 1468228561Snp KASSERT(budget == 0, 1469228561Snp ("%s: budget %u, rsp_type %u", __func__, 1470228561Snp budget, rsp_type)); 1471228561Snp 1472255005Snp /* 1473255005Snp * There are 1K interrupt-capable queues (qids 0 1474255005Snp * through 1023). A response type indicating a 1475255005Snp * forwarded interrupt with a qid >= 1K is an 1476255005Snp * iWARP async notification. 1477255005Snp */ 1478255005Snp if (lq >= 1024) { 1479309442Sjhb t4_an_handler(iq, &d->rsp); 1480255005Snp break; 1481255005Snp } 1482255005Snp 1483309447Sjhb q = sc->sge.iqmap[lq - sc->sge.iq_start - 1484309447Sjhb sc->sge.iq_base]; 1485228561Snp if (atomic_cmpset_int(&q->state, IQS_IDLE, 1486228561Snp IQS_BUSY)) { 1487270297Snp if (service_iq(q, q->qsize / 16) == 0) { 1488228561Snp atomic_cmpset_int(&q->state, 1489228561Snp IQS_BUSY, IQS_IDLE); 1490228561Snp } else { 1491228561Snp STAILQ_INSERT_TAIL(&iql, q, 1492228561Snp link); 1493228561Snp } 1494228561Snp } 1495228561Snp break; 1496228561Snp 1497228561Snp default: 1498255005Snp KASSERT(0, 1499255005Snp ("%s: illegal response type %d on iq %p", 1500255005Snp __func__, rsp_type, iq)); 1501255005Snp log(LOG_ERR, 1502255005Snp "%s: illegal response type %d on iq %p", 1503255005Snp device_get_nameunit(sc->dev), rsp_type, iq); 1504237263Snp break; 1505228561Snp } 1506228561Snp 1507270297Snp d++; 1508270297Snp if (__predict_false(++iq->cidx == iq->sidx)) { 1509270297Snp iq->cidx = 0; 1510270297Snp iq->gen ^= F_RSPD_GEN; 1511270297Snp d = &iq->desc[0]; 1512265425Snp } 1513270297Snp if (__predict_false(++ndescs == limit)) { 1514309447Sjhb t4_write_reg(sc, sc->sge_gts_reg, 1515228561Snp V_CIDXINC(ndescs) | 1516228561Snp V_INGRESSQID(iq->cntxt_id) | 1517228561Snp V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 1518228561Snp ndescs = 0; 1519228561Snp 1520255015Snp#if defined(INET) || defined(INET6) 1521255015Snp if (iq->flags & IQ_LRO_ENABLED && 1522255015Snp sc->lro_timeout != 0) { 1523255015Snp tcp_lro_flush_inactive(&rxq->lro, 1524255015Snp &lro_timeout); 1525255015Snp } 1526255015Snp#endif 1527255015Snp 1528267244Snp if (budget) { 1529270297Snp if (iq->flags & IQ_HAS_FL) { 1530267244Snp FL_LOCK(fl); 1531267244Snp refill_fl(sc, fl, 32); 1532267244Snp FL_UNLOCK(fl); 1533267244Snp } 1534228561Snp return (EINPROGRESS); 1535267244Snp } 1536228561Snp } 1537270297Snp if (refill) { 1538270297Snp FL_LOCK(fl); 1539270297Snp refill_fl(sc, fl, 32); 1540270297Snp FL_UNLOCK(fl); 1541270297Snp fl_hw_cidx = fl->hw_cidx; 1542270297Snp } 1543218792Snp } 1544222510Snp 1545255050Snpprocess_iql: 1546228561Snp if (STAILQ_EMPTY(&iql)) 1547228561Snp break; 1548228561Snp 1549228561Snp /* 1550228561Snp * Process the head only, and send it to the back of the list if 1551228561Snp * it's still not done. 1552228561Snp */ 1553228561Snp q = STAILQ_FIRST(&iql); 1554228561Snp STAILQ_REMOVE_HEAD(&iql, link); 1555228561Snp if (service_iq(q, q->qsize / 8) == 0) 1556228561Snp atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); 1557228561Snp else 1558228561Snp STAILQ_INSERT_TAIL(&iql, q, link); 1559218792Snp } 1560218792Snp 1561237819Snp#if defined(INET) || defined(INET6) 1562228561Snp if (iq->flags & IQ_LRO_ENABLED) { 1563228561Snp struct lro_ctrl *lro = &rxq->lro; 1564228561Snp struct lro_entry *l; 1565228561Snp 1566228561Snp while (!SLIST_EMPTY(&lro->lro_active)) { 1567228561Snp l = SLIST_FIRST(&lro->lro_active); 1568228561Snp SLIST_REMOVE_HEAD(&lro->lro_active, next); 1569228561Snp tcp_lro_flush(lro, l); 1570228561Snp } 1571228561Snp } 1572228561Snp#endif 1573228561Snp 1574309447Sjhb t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1575228561Snp V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 1576228561Snp 1577228561Snp if (iq->flags & IQ_HAS_FL) { 1578228561Snp int starved; 1579228561Snp 1580228561Snp FL_LOCK(fl); 1581265425Snp starved = refill_fl(sc, fl, 64); 1582228561Snp FL_UNLOCK(fl); 1583228561Snp if (__predict_false(starved != 0)) 1584228561Snp add_fl_to_sfl(sc, fl); 1585228561Snp } 1586228561Snp 1587228561Snp return (0); 1588218792Snp} 1589218792Snp 1590265425Snpstatic inline int 1591265425Snpcl_has_metadata(struct sge_fl *fl, struct cluster_layout *cll) 1592255050Snp{ 1593265425Snp int rc = fl->flags & FL_BUF_PACKING || cll->region1 > 0; 1594255050Snp 1595265425Snp if (rc) 1596265425Snp MPASS(cll->region3 >= CL_METADATA_SIZE); 1597255050Snp 1598265425Snp return (rc); 1599255050Snp} 1600255050Snp 1601265425Snpstatic inline struct cluster_metadata * 1602265425Snpcl_metadata(struct adapter *sc, struct sge_fl *fl, struct cluster_layout *cll, 1603265425Snp caddr_t cl) 1604255050Snp{ 1605255050Snp 1606265425Snp if (cl_has_metadata(fl, cll)) { 1607265425Snp struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx]; 1608255050Snp 1609265425Snp return ((struct cluster_metadata *)(cl + swz->size) - 1); 1610255050Snp } 1611265425Snp return (NULL); 1612255050Snp} 1613255050Snp 1614255050Snpstatic int 1615255050Snprxb_free(struct mbuf *m, void *arg1, void *arg2) 1616255050Snp{ 1617255050Snp uma_zone_t zone = arg1; 1618255050Snp caddr_t cl = arg2; 1619255050Snp 1620255050Snp uma_zfree(zone, cl); 1621269356Snp counter_u64_add(extfree_rels, 1); 1622255050Snp 1623255050Snp return (EXT_FREE_OK); 1624255050Snp} 1625255050Snp 1626265425Snp/* 1627265425Snp * The mbuf returned by this function could be allocated from zone_mbuf or 1628265425Snp * constructed in spare room in the cluster. 1629265425Snp * 1630265425Snp * The mbuf carries the payload in one of these ways 1631265425Snp * a) frame inside the mbuf (mbuf from zone_mbuf) 1632265425Snp * b) m_cljset (for clusters without metadata) zone_mbuf 1633265425Snp * c) m_extaddref (cluster with metadata) inline mbuf 1634265425Snp * d) m_extaddref (cluster with metadata) zone_mbuf 1635265425Snp */ 1636255050Snpstatic struct mbuf * 1637281212Snpget_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 1638281212Snp int remaining) 1639218792Snp{ 1640265425Snp struct mbuf *m; 1641228561Snp struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1642265425Snp struct cluster_layout *cll = &sd->cll; 1643265425Snp struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx]; 1644265425Snp struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx]; 1645265425Snp struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl); 1646281212Snp int len, blen; 1647265425Snp caddr_t payload; 1648218792Snp 1649281212Snp blen = hwb->size - fl->rx_offset; /* max possible in this buf */ 1650281212Snp len = min(remaining, blen); 1651265425Snp payload = sd->cl + cll->region1 + fl->rx_offset; 1652281212Snp if (fl->flags & FL_BUF_PACKING) { 1653281212Snp const u_int l = fr_offset + len; 1654281212Snp const u_int pad = roundup2(l, fl->buf_boundary) - l; 1655219290Snp 1656281212Snp if (fl->rx_offset + len + pad < hwb->size) 1657281212Snp blen = len + pad; 1658281212Snp MPASS(fl->rx_offset + blen <= hwb->size); 1659281212Snp } else { 1660281212Snp MPASS(fl->rx_offset == 0); /* not packing */ 1661281212Snp } 1662281212Snp 1663281212Snp 1664265425Snp if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) { 1665255050Snp 1666265425Snp /* 1667265425Snp * Copy payload into a freshly allocated mbuf. 1668265425Snp */ 1669255050Snp 1670281212Snp m = fr_offset == 0 ? 1671265425Snp m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA); 1672265425Snp if (m == NULL) 1673255050Snp return (NULL); 1674265425Snp fl->mbuf_allocated++; 1675255050Snp#ifdef T4_PKT_TIMESTAMP 1676265425Snp /* Leave room for a timestamp */ 1677265425Snp m->m_data += 8; 1678255050Snp#endif 1679265425Snp /* copy data to mbuf */ 1680265425Snp bcopy(payload, mtod(m, caddr_t), len); 1681255050Snp 1682269356Snp } else if (sd->nmbuf * MSIZE < cll->region1) { 1683255050Snp 1684265425Snp /* 1685265425Snp * There's spare room in the cluster for an mbuf. Create one 1686267694Snp * and associate it with the payload that's in the cluster. 1687265425Snp */ 1688255050Snp 1689265425Snp MPASS(clm != NULL); 1690269356Snp m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE); 1691265425Snp /* No bzero required */ 1692281212Snp if (m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 1693281212Snp fr_offset == 0 ? M_PKTHDR | M_NOFREE : M_NOFREE)) 1694265425Snp return (NULL); 1695265425Snp fl->mbuf_inlined++; 1696281212Snp m_extaddref(m, payload, blen, &clm->refcount, rxb_free, 1697265425Snp swz->zone, sd->cl); 1698269356Snp if (sd->nmbuf++ == 0) 1699269356Snp counter_u64_add(extfree_refs, 1); 1700255050Snp 1701265425Snp } else { 1702255050Snp 1703265425Snp /* 1704265425Snp * Grab an mbuf from zone_mbuf and associate it with the 1705265425Snp * payload in the cluster. 1706265425Snp */ 1707255050Snp 1708281212Snp m = fr_offset == 0 ? 1709265425Snp m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA); 1710265425Snp if (m == NULL) 1711265425Snp return (NULL); 1712265425Snp fl->mbuf_allocated++; 1713267694Snp if (clm != NULL) { 1714281212Snp m_extaddref(m, payload, blen, &clm->refcount, 1715265425Snp rxb_free, swz->zone, sd->cl); 1716269356Snp if (sd->nmbuf++ == 0) 1717269356Snp counter_u64_add(extfree_refs, 1); 1718267694Snp } else { 1719265425Snp m_cljset(m, sd->cl, swz->type); 1720265425Snp sd->cl = NULL; /* consumed, not a recycle candidate */ 1721255050Snp } 1722255050Snp } 1723281212Snp if (fr_offset == 0) 1724281212Snp m->m_pkthdr.len = remaining; 1725265425Snp m->m_len = len; 1726255050Snp 1727265425Snp if (fl->flags & FL_BUF_PACKING) { 1728281212Snp fl->rx_offset += blen; 1729265425Snp MPASS(fl->rx_offset <= hwb->size); 1730265425Snp if (fl->rx_offset < hwb->size) 1731265425Snp return (m); /* without advancing the cidx */ 1732265425Snp } 1733255050Snp 1734270297Snp if (__predict_false(++fl->cidx % 8 == 0)) { 1735270297Snp uint16_t cidx = fl->cidx / 8; 1736270297Snp 1737270297Snp if (__predict_false(cidx == fl->sidx)) 1738270297Snp fl->cidx = cidx = 0; 1739270297Snp fl->hw_cidx = cidx; 1740270297Snp } 1741265425Snp fl->rx_offset = 0; 1742255050Snp 1743265425Snp return (m); 1744255050Snp} 1745255050Snp 1746255050Snpstatic struct mbuf * 1747270297Snpget_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf) 1748255050Snp{ 1749265425Snp struct mbuf *m0, *m, **pnext; 1750281212Snp u_int remaining; 1751281212Snp const u_int total = G_RSPD_LEN(len_newbuf); 1752255050Snp 1753270297Snp if (__predict_false(fl->flags & FL_BUF_RESUME)) { 1754266965Snp M_ASSERTPKTHDR(fl->m0); 1755281212Snp MPASS(fl->m0->m_pkthdr.len == total); 1756281212Snp MPASS(fl->remaining < total); 1757218792Snp 1758265425Snp m0 = fl->m0; 1759265425Snp pnext = fl->pnext; 1760281212Snp remaining = fl->remaining; 1761270297Snp fl->flags &= ~FL_BUF_RESUME; 1762265425Snp goto get_segment; 1763255050Snp } 1764255050Snp 1765265425Snp if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 1766265425Snp fl->rx_offset = 0; 1767270297Snp if (__predict_false(++fl->cidx % 8 == 0)) { 1768270297Snp uint16_t cidx = fl->cidx / 8; 1769270297Snp 1770270297Snp if (__predict_false(cidx == fl->sidx)) 1771270297Snp fl->cidx = cidx = 0; 1772270297Snp fl->hw_cidx = cidx; 1773270297Snp } 1774228561Snp } 1775218792Snp 1776265425Snp /* 1777265425Snp * Payload starts at rx_offset in the current hw buffer. Its length is 1778265425Snp * 'len' and it may span multiple hw buffers. 1779265425Snp */ 1780218792Snp 1781281212Snp m0 = get_scatter_segment(sc, fl, 0, total); 1782266965Snp if (m0 == NULL) 1783270297Snp return (NULL); 1784281212Snp remaining = total - m0->m_len; 1785265425Snp pnext = &m0->m_next; 1786281212Snp while (remaining > 0) { 1787265425Snpget_segment: 1788265425Snp MPASS(fl->rx_offset == 0); 1789281212Snp m = get_scatter_segment(sc, fl, total - remaining, remaining); 1790270297Snp if (__predict_false(m == NULL)) { 1791265425Snp fl->m0 = m0; 1792265425Snp fl->pnext = pnext; 1793281212Snp fl->remaining = remaining; 1794270297Snp fl->flags |= FL_BUF_RESUME; 1795270297Snp return (NULL); 1796218792Snp } 1797265425Snp *pnext = m; 1798265425Snp pnext = &m->m_next; 1799281212Snp remaining -= m->m_len; 1800265425Snp } 1801265425Snp *pnext = NULL; 1802270297Snp 1803286273Snp M_ASSERTPKTHDR(m0); 1804228561Snp return (m0); 1805228561Snp} 1806218792Snp 1807228561Snpstatic int 1808228561Snpt4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) 1809228561Snp{ 1810237463Snp struct sge_rxq *rxq = iq_to_rxq(iq); 1811228561Snp struct ifnet *ifp = rxq->ifp; 1812308304Sjhb struct adapter *sc = iq->adapter; 1813228561Snp const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 1814237819Snp#if defined(INET) || defined(INET6) 1815228561Snp struct lro_ctrl *lro = &rxq->lro; 1816228561Snp#endif 1817219290Snp 1818228561Snp KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__, 1819228561Snp rss->opcode)); 1820219290Snp 1821308304Sjhb m0->m_pkthdr.len -= sc->params.sge.fl_pktshift; 1822308304Sjhb m0->m_len -= sc->params.sge.fl_pktshift; 1823308304Sjhb m0->m_data += sc->params.sge.fl_pktshift; 1824219290Snp 1825228561Snp m0->m_pkthdr.rcvif = ifp; 1826281955Shiren M_HASHTYPE_SET(m0, M_HASHTYPE_OPAQUE); 1827259142Snp m0->m_pkthdr.flowid = be32toh(rss->hash_val); 1828219290Snp 1829311261Snp if (cpl->csum_calc && !(cpl->err_vec & sc->params.tp.err_vec_mask)) { 1830237799Snp if (ifp->if_capenable & IFCAP_RXCSUM && 1831237799Snp cpl->l2info & htobe32(F_RXF_IP)) { 1832237831Snp m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | 1833237799Snp CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1834237799Snp rxq->rxcsum++; 1835237799Snp } else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 && 1836237799Snp cpl->l2info & htobe32(F_RXF_IP6)) { 1837237831Snp m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 | 1838237799Snp CSUM_PSEUDO_HDR); 1839237799Snp rxq->rxcsum++; 1840237799Snp } 1841237799Snp 1842237799Snp if (__predict_false(cpl->ip_frag)) 1843228561Snp m0->m_pkthdr.csum_data = be16toh(cpl->csum); 1844228561Snp else 1845228561Snp m0->m_pkthdr.csum_data = 0xffff; 1846228561Snp } 1847219290Snp 1848228561Snp if (cpl->vlan_ex) { 1849228561Snp m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan); 1850228561Snp m0->m_flags |= M_VLANTAG; 1851228561Snp rxq->vlan_extraction++; 1852228561Snp } 1853219290Snp 1854237819Snp#if defined(INET) || defined(INET6) 1855309558Sjhb if (iq->flags & IQ_LRO_ENABLED && 1856228561Snp tcp_lro_rx(lro, m0, 0) == 0) { 1857228561Snp /* queued for LRO */ 1858228561Snp } else 1859218792Snp#endif 1860228561Snp ifp->if_input(ifp, m0); 1861218792Snp 1862228561Snp return (0); 1863228561Snp} 1864218792Snp 1865228561Snp/* 1866284052Snp * Must drain the wrq or make sure that someone else will. 1867284052Snp */ 1868284052Snpstatic void 1869284052Snpwrq_tx_drain(void *arg, int n) 1870284052Snp{ 1871284052Snp struct sge_wrq *wrq = arg; 1872284052Snp struct sge_eq *eq = &wrq->eq; 1873284052Snp 1874284052Snp EQ_LOCK(eq); 1875284052Snp if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 1876284052Snp drain_wrq_wr_list(wrq->adapter, wrq); 1877284052Snp EQ_UNLOCK(eq); 1878284052Snp} 1879284052Snp 1880284052Snpstatic void 1881284052Snpdrain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) 1882284052Snp{ 1883284052Snp struct sge_eq *eq = &wrq->eq; 1884284052Snp u_int available, dbdiff; /* # of hardware descriptors */ 1885284052Snp u_int n; 1886284052Snp struct wrqe *wr; 1887284052Snp struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 1888284052Snp 1889284052Snp EQ_LOCK_ASSERT_OWNED(eq); 1890284052Snp MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); 1891284052Snp wr = STAILQ_FIRST(&wrq->wr_list); 1892284052Snp MPASS(wr != NULL); /* Must be called with something useful to do */ 1893308320Sjhb MPASS(eq->pidx == eq->dbidx); 1894308320Sjhb dbdiff = 0; 1895284052Snp 1896284052Snp do { 1897284052Snp eq->cidx = read_hw_cidx(eq); 1898284052Snp if (eq->pidx == eq->cidx) 1899284052Snp available = eq->sidx - 1; 1900284052Snp else 1901284052Snp available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 1902284052Snp 1903284052Snp MPASS(wr->wrq == wrq); 1904284052Snp n = howmany(wr->wr_len, EQ_ESIZE); 1905284052Snp if (available < n) 1906308320Sjhb break; 1907284052Snp 1908284052Snp dst = (void *)&eq->desc[eq->pidx]; 1909284052Snp if (__predict_true(eq->sidx - eq->pidx > n)) { 1910284052Snp /* Won't wrap, won't end exactly at the status page. */ 1911284052Snp bcopy(&wr->wr[0], dst, wr->wr_len); 1912284052Snp eq->pidx += n; 1913284052Snp } else { 1914284052Snp int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; 1915284052Snp 1916284052Snp bcopy(&wr->wr[0], dst, first_portion); 1917284052Snp if (wr->wr_len > first_portion) { 1918284052Snp bcopy(&wr->wr[first_portion], &eq->desc[0], 1919284052Snp wr->wr_len - first_portion); 1920284052Snp } 1921284052Snp eq->pidx = n - (eq->sidx - eq->pidx); 1922284052Snp } 1923309458Sjhb wrq->tx_wrs_copied++; 1924284052Snp 1925284052Snp if (available < eq->sidx / 4 && 1926284052Snp atomic_cmpset_int(&eq->equiq, 0, 1)) { 1927284052Snp dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 1928284052Snp F_FW_WR_EQUEQ); 1929284052Snp eq->equeqidx = eq->pidx; 1930284052Snp } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 1931284052Snp dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 1932284052Snp eq->equeqidx = eq->pidx; 1933284052Snp } 1934284052Snp 1935284052Snp dbdiff += n; 1936284052Snp if (dbdiff >= 16) { 1937284052Snp ring_eq_db(sc, eq, dbdiff); 1938284052Snp dbdiff = 0; 1939284052Snp } 1940284052Snp 1941284052Snp STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 1942284052Snp free_wrqe(wr); 1943284052Snp MPASS(wrq->nwr_pending > 0); 1944284052Snp wrq->nwr_pending--; 1945284052Snp MPASS(wrq->ndesc_needed >= n); 1946284052Snp wrq->ndesc_needed -= n; 1947284052Snp } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); 1948284052Snp 1949284052Snp if (dbdiff) 1950284052Snp ring_eq_db(sc, eq, dbdiff); 1951284052Snp} 1952284052Snp 1953284052Snp/* 1954228561Snp * Doesn't fail. Holds on to work requests it can't send right away. 1955228561Snp */ 1956237263Snpvoid 1957237263Snpt4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) 1958228561Snp{ 1959284052Snp#ifdef INVARIANTS 1960228561Snp struct sge_eq *eq = &wrq->eq; 1961284052Snp#endif 1962228561Snp 1963284052Snp EQ_LOCK_ASSERT_OWNED(eq); 1964284052Snp MPASS(wr != NULL); 1965284052Snp MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); 1966284052Snp MPASS((wr->wr_len & 0x7) == 0); 1967284052Snp 1968284052Snp STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); 1969284052Snp wrq->nwr_pending++; 1970284052Snp wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); 1971284052Snp 1972284052Snp if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) 1973284052Snp return; /* commit_wrq_wr will drain wr_list as well. */ 1974284052Snp 1975284052Snp drain_wrq_wr_list(sc, wrq); 1976284052Snp 1977284052Snp /* Doorbell must have caught up to the pidx. */ 1978284052Snp MPASS(eq->pidx == eq->dbidx); 1979284052Snp} 1980284052Snp 1981284052Snpvoid 1982284052Snpt4_update_fl_bufsize(struct ifnet *ifp) 1983284052Snp{ 1984308154Sjhb struct vi_info *vi = ifp->if_softc; 1985308154Sjhb struct adapter *sc = vi->pi->adapter; 1986284052Snp struct sge_rxq *rxq; 1987237263Snp#ifdef TCP_OFFLOAD 1988284052Snp struct sge_ofld_rxq *ofld_rxq; 1989237263Snp#endif 1990284052Snp struct sge_fl *fl; 1991284052Snp int i, maxp, mtu = ifp->if_mtu; 1992228561Snp 1993284052Snp maxp = mtu_to_max_payload(sc, mtu, 0); 1994308154Sjhb for_each_rxq(vi, i, rxq) { 1995284052Snp fl = &rxq->fl; 1996218792Snp 1997284052Snp FL_LOCK(fl); 1998284052Snp find_best_refill_source(sc, fl, maxp); 1999284052Snp FL_UNLOCK(fl); 2000218792Snp } 2001284052Snp#ifdef TCP_OFFLOAD 2002284052Snp maxp = mtu_to_max_payload(sc, mtu, 1); 2003308154Sjhb for_each_ofld_rxq(vi, i, ofld_rxq) { 2004284052Snp fl = &ofld_rxq->fl; 2005228561Snp 2006284052Snp FL_LOCK(fl); 2007284052Snp find_best_refill_source(sc, fl, maxp); 2008284052Snp FL_UNLOCK(fl); 2009284052Snp } 2010284052Snp#endif 2011284052Snp} 2012228561Snp 2013284052Snpstatic inline int 2014284052Snpmbuf_nsegs(struct mbuf *m) 2015284052Snp{ 2016228561Snp 2017284052Snp M_ASSERTPKTHDR(m); 2018284052Snp KASSERT(m->m_pkthdr.l5hlen > 0, 2019284052Snp ("%s: mbuf %p missing information on # of segments.", __func__, m)); 2020218792Snp 2021284052Snp return (m->m_pkthdr.l5hlen); 2022284052Snp} 2023218792Snp 2024284052Snpstatic inline void 2025284052Snpset_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) 2026284052Snp{ 2027218792Snp 2028284052Snp M_ASSERTPKTHDR(m); 2029284052Snp m->m_pkthdr.l5hlen = nsegs; 2030284052Snp} 2031228561Snp 2032284052Snpstatic inline int 2033284052Snpmbuf_len16(struct mbuf *m) 2034284052Snp{ 2035284052Snp int n; 2036228561Snp 2037284052Snp M_ASSERTPKTHDR(m); 2038284052Snp n = m->m_pkthdr.PH_loc.eigth[0]; 2039284052Snp MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 2040228561Snp 2041284052Snp return (n); 2042284052Snp} 2043228561Snp 2044284052Snpstatic inline void 2045284052Snpset_mbuf_len16(struct mbuf *m, uint8_t len16) 2046284052Snp{ 2047228561Snp 2048284052Snp M_ASSERTPKTHDR(m); 2049284052Snp m->m_pkthdr.PH_loc.eigth[0] = len16; 2050220873Snp} 2051220873Snp 2052284052Snpstatic inline int 2053284052Snpneeds_tso(struct mbuf *m) 2054284052Snp{ 2055218792Snp 2056284052Snp M_ASSERTPKTHDR(m); 2057218792Snp 2058284052Snp if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2059284052Snp KASSERT(m->m_pkthdr.tso_segsz > 0, 2060284052Snp ("%s: TSO requested in mbuf %p but MSS not provided", 2061284052Snp __func__, m)); 2062284052Snp return (1); 2063284052Snp } 2064218792Snp 2065284052Snp return (0); 2066284052Snp} 2067218792Snp 2068284052Snpstatic inline int 2069284052Snpneeds_l3_csum(struct mbuf *m) 2070218792Snp{ 2071218792Snp 2072284052Snp M_ASSERTPKTHDR(m); 2073218792Snp 2074284052Snp if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) 2075284052Snp return (1); 2076284052Snp return (0); 2077284052Snp} 2078219292Snp 2079284052Snpstatic inline int 2080284052Snpneeds_l4_csum(struct mbuf *m) 2081284052Snp{ 2082218792Snp 2083284052Snp M_ASSERTPKTHDR(m); 2084218792Snp 2085284052Snp if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | 2086284052Snp CSUM_TCP_IPV6 | CSUM_TSO)) 2087284052Snp return (1); 2088284052Snp return (0); 2089284052Snp} 2090284052Snp 2091284052Snpstatic inline int 2092284052Snpneeds_vlan_insertion(struct mbuf *m) 2093284052Snp{ 2094284052Snp 2095284052Snp M_ASSERTPKTHDR(m); 2096284052Snp 2097284052Snp if (m->m_flags & M_VLANTAG) { 2098284052Snp KASSERT(m->m_pkthdr.ether_vtag != 0, 2099284052Snp ("%s: HWVLAN requested in mbuf %p but tag not provided", 2100284052Snp __func__, m)); 2101284052Snp return (1); 2102228561Snp } 2103284052Snp return (0); 2104284052Snp} 2105228561Snp 2106284052Snpstatic void * 2107284052Snpm_advance(struct mbuf **pm, int *poffset, int len) 2108284052Snp{ 2109284052Snp struct mbuf *m = *pm; 2110284052Snp int offset = *poffset; 2111284052Snp uintptr_t p = 0; 2112228561Snp 2113284052Snp MPASS(len > 0); 2114218792Snp 2115309447Sjhb for (;;) { 2116284052Snp if (offset + len < m->m_len) { 2117284052Snp offset += len; 2118284052Snp p = mtod(m, uintptr_t) + offset; 2119218792Snp break; 2120284052Snp } 2121284052Snp len -= m->m_len - offset; 2122284052Snp m = m->m_next; 2123284052Snp offset = 0; 2124284052Snp MPASS(m != NULL); 2125284052Snp } 2126284052Snp *poffset = offset; 2127284052Snp *pm = m; 2128284052Snp return ((void *)p); 2129284052Snp} 2130218792Snp 2131284052Snp/* 2132284052Snp * Can deal with empty mbufs in the chain that have m_len = 0, but the chain 2133284052Snp * must have at least one mbuf that's not empty. 2134284052Snp */ 2135284052Snpstatic inline int 2136284052Snpcount_mbuf_nsegs(struct mbuf *m) 2137284052Snp{ 2138309579Sjhb vm_paddr_t lastb, next; 2139309579Sjhb vm_offset_t va; 2140284052Snp int len, nsegs; 2141218792Snp 2142284052Snp MPASS(m != NULL); 2143218792Snp 2144284052Snp nsegs = 0; 2145309579Sjhb lastb = 0; 2146284052Snp for (; m; m = m->m_next) { 2147284052Snp 2148284052Snp len = m->m_len; 2149284052Snp if (__predict_false(len == 0)) 2150218792Snp continue; 2151309579Sjhb va = mtod(m, vm_offset_t); 2152309579Sjhb next = pmap_kextract(va); 2153309579Sjhb nsegs += sglist_count(m->m_data, len); 2154309579Sjhb if (lastb + 1 == next) 2155284052Snp nsegs--; 2156309579Sjhb lastb = pmap_kextract(va + len - 1); 2157284052Snp } 2158218792Snp 2159284052Snp MPASS(nsegs > 0); 2160284052Snp return (nsegs); 2161284052Snp} 2162218792Snp 2163284052Snp/* 2164284052Snp * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: 2165284052Snp * a) caller can assume it's been freed if this function returns with an error. 2166284052Snp * b) it may get defragged up if the gather list is too long for the hardware. 2167284052Snp */ 2168284052Snpint 2169309447Sjhbparse_pkt(struct adapter *sc, struct mbuf **mp) 2170284052Snp{ 2171284052Snp struct mbuf *m0 = *mp, *m; 2172284052Snp int rc, nsegs, defragged = 0, offset; 2173284052Snp struct ether_header *eh; 2174284052Snp void *l3hdr; 2175284052Snp#if defined(INET) || defined(INET6) 2176284052Snp struct tcphdr *tcp; 2177284052Snp#endif 2178284052Snp uint16_t eh_type; 2179284052Snp 2180284052Snp M_ASSERTPKTHDR(m0); 2181284052Snp if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { 2182284052Snp rc = EINVAL; 2183284052Snpfail: 2184284052Snp m_freem(m0); 2185284052Snp *mp = NULL; 2186284052Snp return (rc); 2187284052Snp } 2188284052Snprestart: 2189284052Snp /* 2190284052Snp * First count the number of gather list segments in the payload. 2191284052Snp * Defrag the mbuf if nsegs exceeds the hardware limit. 2192284052Snp */ 2193284052Snp M_ASSERTPKTHDR(m0); 2194284052Snp MPASS(m0->m_pkthdr.len > 0); 2195284052Snp nsegs = count_mbuf_nsegs(m0); 2196284052Snp if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { 2197284052Snp if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { 2198284052Snp rc = EFBIG; 2199284052Snp goto fail; 2200218792Snp } 2201284052Snp *mp = m0 = m; /* update caller's copy after defrag */ 2202284052Snp goto restart; 2203284052Snp } 2204218792Snp 2205284052Snp if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) { 2206284052Snp m0 = m_pullup(m0, m0->m_pkthdr.len); 2207284052Snp if (m0 == NULL) { 2208284052Snp /* Should have left well enough alone. */ 2209284052Snp rc = EFBIG; 2210284052Snp goto fail; 2211284052Snp } 2212284052Snp *mp = m0; /* update caller's copy after pullup */ 2213284052Snp goto restart; 2214284052Snp } 2215284052Snp set_mbuf_nsegs(m0, nsegs); 2216309447Sjhb if (sc->flags & IS_VF) 2217309447Sjhb set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0))); 2218309447Sjhb else 2219309447Sjhb set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); 2220218792Snp 2221309447Sjhb if (!needs_tso(m0) && 2222309447Sjhb !(sc->flags & IS_VF && (needs_l3_csum(m0) || needs_l4_csum(m0)))) 2223284052Snp return (0); 2224218792Snp 2225284052Snp m = m0; 2226284052Snp eh = mtod(m, struct ether_header *); 2227284052Snp eh_type = ntohs(eh->ether_type); 2228284052Snp if (eh_type == ETHERTYPE_VLAN) { 2229284052Snp struct ether_vlan_header *evh = (void *)eh; 2230218792Snp 2231284052Snp eh_type = ntohs(evh->evl_proto); 2232284052Snp m0->m_pkthdr.l2hlen = sizeof(*evh); 2233284052Snp } else 2234284052Snp m0->m_pkthdr.l2hlen = sizeof(*eh); 2235218792Snp 2236284052Snp offset = 0; 2237284052Snp l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); 2238218792Snp 2239284052Snp switch (eh_type) { 2240284052Snp#ifdef INET6 2241284052Snp case ETHERTYPE_IPV6: 2242284052Snp { 2243284052Snp struct ip6_hdr *ip6 = l3hdr; 2244218792Snp 2245309447Sjhb MPASS(!needs_tso(m0) || ip6->ip6_nxt == IPPROTO_TCP); 2246219292Snp 2247284052Snp m0->m_pkthdr.l3hlen = sizeof(*ip6); 2248284052Snp break; 2249218792Snp } 2250284052Snp#endif 2251284052Snp#ifdef INET 2252284052Snp case ETHERTYPE_IP: 2253284052Snp { 2254284052Snp struct ip *ip = l3hdr; 2255218792Snp 2256284052Snp m0->m_pkthdr.l3hlen = ip->ip_hl * 4; 2257284052Snp break; 2258284052Snp } 2259284052Snp#endif 2260284052Snp default: 2261284052Snp panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled" 2262284052Snp " with the same INET/INET6 options as the kernel.", 2263284052Snp __func__, eh_type); 2264284052Snp } 2265218792Snp 2266284052Snp#if defined(INET) || defined(INET6) 2267309447Sjhb if (needs_tso(m0)) { 2268309447Sjhb tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); 2269309447Sjhb m0->m_pkthdr.l4hlen = tcp->th_off * 4; 2270309447Sjhb } 2271284052Snp#endif 2272284052Snp MPASS(m0 == *mp); 2273284052Snp return (0); 2274284052Snp} 2275220873Snp 2276284052Snpvoid * 2277284052Snpstart_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) 2278284052Snp{ 2279284052Snp struct sge_eq *eq = &wrq->eq; 2280284052Snp struct adapter *sc = wrq->adapter; 2281284052Snp int ndesc, available; 2282284052Snp struct wrqe *wr; 2283284052Snp void *w; 2284228561Snp 2285284052Snp MPASS(len16 > 0); 2286284052Snp ndesc = howmany(len16, EQ_ESIZE / 16); 2287284052Snp MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); 2288284052Snp 2289284052Snp EQ_LOCK(eq); 2290284052Snp 2291284052Snp if (!STAILQ_EMPTY(&wrq->wr_list)) 2292284052Snp drain_wrq_wr_list(sc, wrq); 2293284052Snp 2294284052Snp if (!STAILQ_EMPTY(&wrq->wr_list)) { 2295284052Snpslowpath: 2296284052Snp EQ_UNLOCK(eq); 2297284052Snp wr = alloc_wrqe(len16 * 16, wrq); 2298284052Snp if (__predict_false(wr == NULL)) 2299284052Snp return (NULL); 2300284052Snp cookie->pidx = -1; 2301284052Snp cookie->ndesc = ndesc; 2302284052Snp return (&wr->wr); 2303220873Snp } 2304218792Snp 2305284052Snp eq->cidx = read_hw_cidx(eq); 2306284052Snp if (eq->pidx == eq->cidx) 2307284052Snp available = eq->sidx - 1; 2308284052Snp else 2309284052Snp available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2310284052Snp if (available < ndesc) 2311284052Snp goto slowpath; 2312218792Snp 2313284052Snp cookie->pidx = eq->pidx; 2314284052Snp cookie->ndesc = ndesc; 2315284052Snp TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); 2316218792Snp 2317284052Snp w = &eq->desc[eq->pidx]; 2318284052Snp IDXINCR(eq->pidx, ndesc, eq->sidx); 2319312186Snp if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { 2320284052Snp w = &wrq->ss[0]; 2321284052Snp wrq->ss_pidx = cookie->pidx; 2322284052Snp wrq->ss_len = len16 * 16; 2323284052Snp } 2324228561Snp 2325284052Snp EQ_UNLOCK(eq); 2326284052Snp 2327284052Snp return (w); 2328218792Snp} 2329218792Snp 2330218792Snpvoid 2331284052Snpcommit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) 2332218792Snp{ 2333284052Snp struct sge_eq *eq = &wrq->eq; 2334284052Snp struct adapter *sc = wrq->adapter; 2335284052Snp int ndesc, pidx; 2336284052Snp struct wrq_cookie *prev, *next; 2337218792Snp 2338284052Snp if (cookie->pidx == -1) { 2339284052Snp struct wrqe *wr = __containerof(w, struct wrqe, wr); 2340218792Snp 2341284052Snp t4_wrq_tx(sc, wr); 2342284052Snp return; 2343218792Snp } 2344252728Snp 2345284052Snp ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ 2346284052Snp pidx = cookie->pidx; 2347284052Snp MPASS(pidx >= 0 && pidx < eq->sidx); 2348284052Snp if (__predict_false(w == &wrq->ss[0])) { 2349284052Snp int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; 2350284052Snp 2351284052Snp MPASS(wrq->ss_len > n); /* WR had better wrap around. */ 2352284052Snp bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); 2353284052Snp bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); 2354284052Snp wrq->tx_wrs_ss++; 2355284052Snp } else 2356284052Snp wrq->tx_wrs_direct++; 2357284052Snp 2358284052Snp EQ_LOCK(eq); 2359284052Snp prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); 2360284052Snp next = TAILQ_NEXT(cookie, link); 2361284052Snp if (prev == NULL) { 2362284052Snp MPASS(pidx == eq->dbidx); 2363284052Snp if (next == NULL || ndesc >= 16) 2364284052Snp ring_eq_db(wrq->adapter, eq, ndesc); 2365284052Snp else { 2366284052Snp MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); 2367284052Snp next->pidx = pidx; 2368284052Snp next->ndesc += ndesc; 2369284052Snp } 2370284052Snp } else { 2371284052Snp MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); 2372284052Snp prev->ndesc += ndesc; 2373252728Snp } 2374284052Snp TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); 2375284052Snp 2376284052Snp if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2377284052Snp drain_wrq_wr_list(sc, wrq); 2378284052Snp 2379284052Snp#ifdef INVARIANTS 2380284052Snp if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { 2381284052Snp /* Doorbell must have caught up to the pidx. */ 2382284052Snp MPASS(wrq->eq.pidx == wrq->eq.dbidx); 2383284052Snp } 2384252728Snp#endif 2385284052Snp EQ_UNLOCK(eq); 2386218792Snp} 2387218792Snp 2388284052Snpstatic u_int 2389284052Snpcan_resume_eth_tx(struct mp_ring *r) 2390228561Snp{ 2391284052Snp struct sge_eq *eq = r->cookie; 2392267764Snp 2393284052Snp return (total_available_tx_desc(eq) > eq->sidx / 8); 2394228561Snp} 2395228561Snp 2396284052Snpstatic inline int 2397284052Snpcannot_use_txpkts(struct mbuf *m) 2398284052Snp{ 2399284052Snp /* maybe put a GL limit too, to avoid silliness? */ 2400284052Snp 2401284052Snp return (needs_tso(m)); 2402284052Snp} 2403284052Snp 2404318855Snpstatic inline int 2405318855Snpdiscard_tx(struct sge_eq *eq) 2406318855Snp{ 2407318855Snp 2408318855Snp return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); 2409318855Snp} 2410318855Snp 2411284052Snp/* 2412284052Snp * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to 2413284052Snp * be consumed. Return the actual number consumed. 0 indicates a stall. 2414284052Snp */ 2415284052Snpstatic u_int 2416284052Snpeth_tx(struct mp_ring *r, u_int cidx, u_int pidx) 2417284052Snp{ 2418284052Snp struct sge_txq *txq = r->cookie; 2419284052Snp struct sge_eq *eq = &txq->eq; 2420284052Snp struct ifnet *ifp = txq->ifp; 2421308154Sjhb struct vi_info *vi = ifp->if_softc; 2422308154Sjhb struct port_info *pi = vi->pi; 2423284052Snp struct adapter *sc = pi->adapter; 2424284052Snp u_int total, remaining; /* # of packets */ 2425284052Snp u_int available, dbdiff; /* # of hardware descriptors */ 2426284052Snp u_int n, next_cidx; 2427284052Snp struct mbuf *m0, *tail; 2428284052Snp struct txpkts txp; 2429284052Snp struct fw_eth_tx_pkts_wr *wr; /* any fw WR struct will do */ 2430284052Snp 2431284052Snp remaining = IDXDIFF(pidx, cidx, r->size); 2432284052Snp MPASS(remaining > 0); /* Must not be called without work to do. */ 2433284052Snp total = 0; 2434284052Snp 2435284052Snp TXQ_LOCK(txq); 2436318855Snp if (__predict_false(discard_tx(eq))) { 2437284052Snp while (cidx != pidx) { 2438284052Snp m0 = r->items[cidx]; 2439284052Snp m_freem(m0); 2440284052Snp if (++cidx == r->size) 2441284052Snp cidx = 0; 2442284052Snp } 2443284052Snp reclaim_tx_descs(txq, 2048); 2444284052Snp total = remaining; 2445284052Snp goto done; 2446284052Snp } 2447284052Snp 2448284052Snp /* How many hardware descriptors do we have readily available. */ 2449284052Snp if (eq->pidx == eq->cidx) 2450284052Snp available = eq->sidx - 1; 2451284052Snp else 2452284052Snp available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2453284052Snp dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); 2454284052Snp 2455284052Snp while (remaining > 0) { 2456284052Snp 2457284052Snp m0 = r->items[cidx]; 2458284052Snp M_ASSERTPKTHDR(m0); 2459284052Snp MPASS(m0->m_nextpkt == NULL); 2460284052Snp 2461284052Snp if (available < SGE_MAX_WR_NDESC) { 2462284052Snp available += reclaim_tx_descs(txq, 64); 2463284052Snp if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) 2464284052Snp break; /* out of descriptors */ 2465284052Snp } 2466284052Snp 2467284052Snp next_cidx = cidx + 1; 2468284052Snp if (__predict_false(next_cidx == r->size)) 2469284052Snp next_cidx = 0; 2470284052Snp 2471284052Snp wr = (void *)&eq->desc[eq->pidx]; 2472309447Sjhb if (sc->flags & IS_VF) { 2473309447Sjhb total++; 2474309447Sjhb remaining--; 2475309447Sjhb ETHER_BPF_MTAP(ifp, m0); 2476309560Sjhb n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0, 2477309560Sjhb available); 2478309447Sjhb } else if (remaining > 1 && 2479284052Snp try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) { 2480284052Snp 2481284052Snp /* pkts at cidx, next_cidx should both be in txp. */ 2482284052Snp MPASS(txp.npkt == 2); 2483284052Snp tail = r->items[next_cidx]; 2484284052Snp MPASS(tail->m_nextpkt == NULL); 2485284052Snp ETHER_BPF_MTAP(ifp, m0); 2486284052Snp ETHER_BPF_MTAP(ifp, tail); 2487284052Snp m0->m_nextpkt = tail; 2488284052Snp 2489284052Snp if (__predict_false(++next_cidx == r->size)) 2490284052Snp next_cidx = 0; 2491284052Snp 2492284052Snp while (next_cidx != pidx) { 2493284052Snp if (add_to_txpkts(r->items[next_cidx], &txp, 2494284052Snp available) != 0) 2495284052Snp break; 2496284052Snp tail->m_nextpkt = r->items[next_cidx]; 2497284052Snp tail = tail->m_nextpkt; 2498284052Snp ETHER_BPF_MTAP(ifp, tail); 2499284052Snp if (__predict_false(++next_cidx == r->size)) 2500284052Snp next_cidx = 0; 2501284052Snp } 2502284052Snp 2503284052Snp n = write_txpkts_wr(txq, wr, m0, &txp, available); 2504284052Snp total += txp.npkt; 2505284052Snp remaining -= txp.npkt; 2506284052Snp } else { 2507284052Snp total++; 2508284052Snp remaining--; 2509308316Sjhb ETHER_BPF_MTAP(ifp, m0); 2510284052Snp n = write_txpkt_wr(txq, (void *)wr, m0, available); 2511284052Snp } 2512284052Snp MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC); 2513284052Snp 2514284052Snp available -= n; 2515284052Snp dbdiff += n; 2516284052Snp IDXINCR(eq->pidx, n, eq->sidx); 2517284052Snp 2518284052Snp if (total_available_tx_desc(eq) < eq->sidx / 4 && 2519284052Snp atomic_cmpset_int(&eq->equiq, 0, 1)) { 2520284052Snp wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 2521284052Snp F_FW_WR_EQUEQ); 2522284052Snp eq->equeqidx = eq->pidx; 2523284052Snp } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 2524284052Snp wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 2525284052Snp eq->equeqidx = eq->pidx; 2526284052Snp } 2527284052Snp 2528284052Snp if (dbdiff >= 16 && remaining >= 4) { 2529284052Snp ring_eq_db(sc, eq, dbdiff); 2530284052Snp available += reclaim_tx_descs(txq, 4 * dbdiff); 2531284052Snp dbdiff = 0; 2532284052Snp } 2533284052Snp 2534284052Snp cidx = next_cidx; 2535284052Snp } 2536284052Snp if (dbdiff != 0) { 2537284052Snp ring_eq_db(sc, eq, dbdiff); 2538284052Snp reclaim_tx_descs(txq, 32); 2539284052Snp } 2540284052Snpdone: 2541284052Snp TXQ_UNLOCK(txq); 2542284052Snp 2543284052Snp return (total); 2544284052Snp} 2545284052Snp 2546218792Snpstatic inline void 2547218792Snpinit_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, 2548270297Snp int qsize) 2549218792Snp{ 2550270297Snp 2551218792Snp KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, 2552218792Snp ("%s: bad tmr_idx %d", __func__, tmr_idx)); 2553218792Snp KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ 2554218792Snp ("%s: bad pktc_idx %d", __func__, pktc_idx)); 2555218792Snp 2556218792Snp iq->flags = 0; 2557218792Snp iq->adapter = sc; 2558234833Snp iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 2559234833Snp iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 2560234833Snp if (pktc_idx >= 0) { 2561234833Snp iq->intr_params |= F_QINTR_CNT_EN; 2562234833Snp iq->intr_pktc_idx = pktc_idx; 2563234833Snp } 2564248925Snp iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ 2565308304Sjhb iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; 2566218792Snp} 2567218792Snp 2568218792Snpstatic inline void 2569281212Snpinit_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name) 2570218792Snp{ 2571255050Snp 2572218792Snp fl->qsize = qsize; 2573308304Sjhb fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 2574218792Snp strlcpy(fl->lockname, name, sizeof(fl->lockname)); 2575281212Snp if (sc->flags & BUF_PACKING_OK && 2576281212Snp ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */ 2577281212Snp (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */ 2578255050Snp fl->flags |= FL_BUF_PACKING; 2579265425Snp find_best_refill_source(sc, fl, maxp); 2580265425Snp find_safe_refill_source(sc, fl); 2581218792Snp} 2582218792Snp 2583218792Snpstatic inline void 2584308304Sjhbinit_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, 2585308304Sjhb uint8_t tx_chan, uint16_t iqid, char *name) 2586218792Snp{ 2587228561Snp KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); 2588228561Snp 2589228561Snp eq->flags = eqtype & EQ_TYPEMASK; 2590228561Snp eq->tx_chan = tx_chan; 2591228561Snp eq->iqid = iqid; 2592308304Sjhb eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 2593220873Snp strlcpy(eq->lockname, name, sizeof(eq->lockname)); 2594218792Snp} 2595218792Snp 2596218792Snpstatic int 2597218792Snpalloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, 2598218792Snp bus_dmamap_t *map, bus_addr_t *pa, void **va) 2599218792Snp{ 2600218792Snp int rc; 2601218792Snp 2602218792Snp rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR, 2603218792Snp BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag); 2604218792Snp if (rc != 0) { 2605218792Snp device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc); 2606218792Snp goto done; 2607218792Snp } 2608218792Snp 2609218792Snp rc = bus_dmamem_alloc(*tag, va, 2610218792Snp BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map); 2611218792Snp if (rc != 0) { 2612218792Snp device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc); 2613218792Snp goto done; 2614218792Snp } 2615218792Snp 2616218792Snp rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0); 2617218792Snp if (rc != 0) { 2618218792Snp device_printf(sc->dev, "cannot load DMA map: %d\n", rc); 2619218792Snp goto done; 2620218792Snp } 2621218792Snpdone: 2622218792Snp if (rc) 2623218792Snp free_ring(sc, *tag, *map, *pa, *va); 2624218792Snp 2625218792Snp return (rc); 2626218792Snp} 2627218792Snp 2628218792Snpstatic int 2629218792Snpfree_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, 2630218792Snp bus_addr_t pa, void *va) 2631218792Snp{ 2632218792Snp if (pa) 2633218792Snp bus_dmamap_unload(tag, map); 2634218792Snp if (va) 2635218792Snp bus_dmamem_free(tag, va, map); 2636218792Snp if (tag) 2637218792Snp bus_dma_tag_destroy(tag); 2638218792Snp 2639218792Snp return (0); 2640218792Snp} 2641218792Snp 2642218792Snp/* 2643218792Snp * Allocates the ring for an ingress queue and an optional freelist. If the 2644218792Snp * freelist is specified it will be allocated and then associated with the 2645218792Snp * ingress queue. 2646218792Snp * 2647218792Snp * Returns errno on failure. Resources allocated up to that point may still be 2648218792Snp * allocated. Caller is responsible for cleanup in case this function fails. 2649218792Snp * 2650228561Snp * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 2651218792Snp * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 2652228561Snp * the abs_id of the ingress queue to which its interrupts should be forwarded. 2653218792Snp */ 2654218792Snpstatic int 2655308154Sjhballoc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, 2656222085Snp int intr_idx, int cong) 2657218792Snp{ 2658218792Snp int rc, i, cntxt_id; 2659218792Snp size_t len; 2660218792Snp struct fw_iq_cmd c; 2661308154Sjhb struct port_info *pi = vi->pi; 2662218792Snp struct adapter *sc = iq->adapter; 2663308304Sjhb struct sge_params *sp = &sc->params.sge; 2664218792Snp __be32 v = 0; 2665218792Snp 2666270297Snp len = iq->qsize * IQ_ESIZE; 2667218792Snp rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, 2668218792Snp (void **)&iq->desc); 2669218792Snp if (rc != 0) 2670218792Snp return (rc); 2671218792Snp 2672218792Snp bzero(&c, sizeof(c)); 2673218792Snp c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 2674218792Snp F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 2675218792Snp V_FW_IQ_CMD_VFN(0)); 2676218792Snp 2677218792Snp c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 2678218792Snp FW_LEN16(c)); 2679218792Snp 2680218792Snp /* Special handling for firmware event queue */ 2681218792Snp if (iq == &sc->sge.fwq) 2682218792Snp v |= F_FW_IQ_CMD_IQASYNCH; 2683218792Snp 2684228561Snp if (iq->flags & IQ_INTR) { 2685218792Snp KASSERT(intr_idx < sc->intr_count, 2686218792Snp ("%s: invalid direct intr_idx %d", __func__, intr_idx)); 2687228561Snp } else 2688228561Snp v |= F_FW_IQ_CMD_IQANDST; 2689228561Snp v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 2690218792Snp 2691218792Snp c.type_to_iqandstindex = htobe32(v | 2692218792Snp V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 2693308154Sjhb V_FW_IQ_CMD_VIID(vi->viid) | 2694218792Snp V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 2695218792Snp c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 2696218792Snp F_FW_IQ_CMD_IQGTSMODE | 2697218792Snp V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 2698270297Snp V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); 2699218792Snp c.iqsize = htobe16(iq->qsize); 2700218792Snp c.iqaddr = htobe64(iq->ba); 2701222085Snp if (cong >= 0) 2702222085Snp c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN); 2703218792Snp 2704218792Snp if (fl) { 2705218792Snp mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF); 2706218792Snp 2707270297Snp len = fl->qsize * EQ_ESIZE; 2708218792Snp rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map, 2709218792Snp &fl->ba, (void **)&fl->desc); 2710218792Snp if (rc) 2711218792Snp return (rc); 2712218792Snp 2713218792Snp /* Allocate space for one software descriptor per buffer. */ 2714218792Snp rc = alloc_fl_sdesc(fl); 2715218792Snp if (rc != 0) { 2716218792Snp device_printf(sc->dev, 2717218792Snp "failed to setup fl software descriptors: %d\n", 2718218792Snp rc); 2719218792Snp return (rc); 2720218792Snp } 2721218792Snp 2722270297Snp if (fl->flags & FL_BUF_PACKING) { 2723308304Sjhb fl->lowat = roundup2(sp->fl_starve_threshold2, 8); 2724308304Sjhb fl->buf_boundary = sp->pack_boundary; 2725270297Snp } else { 2726308304Sjhb fl->lowat = roundup2(sp->fl_starve_threshold, 8); 2727281212Snp fl->buf_boundary = 16; 2728270297Snp } 2729308304Sjhb if (fl_pad && fl->buf_boundary < sp->pad_boundary) 2730308304Sjhb fl->buf_boundary = sp->pad_boundary; 2731270297Snp 2732228491Snp c.iqns_to_fl0congen |= 2733222085Snp htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 2734222085Snp F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | 2735255050Snp (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) | 2736255050Snp (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN : 2737255050Snp 0)); 2738222085Snp if (cong >= 0) { 2739222085Snp c.iqns_to_fl0congen |= 2740222085Snp htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 2741222085Snp F_FW_IQ_CMD_FL0CONGCIF | 2742222085Snp F_FW_IQ_CMD_FL0CONGEN); 2743222085Snp } 2744218792Snp c.fl0dcaen_to_fl0cidxfthresh = 2745309560Sjhb htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ? 2746309560Sjhb X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) | 2747309560Sjhb V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ? 2748309560Sjhb X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); 2749218792Snp c.fl0size = htobe16(fl->qsize); 2750218792Snp c.fl0addr = htobe64(fl->ba); 2751218792Snp } 2752218792Snp 2753218792Snp rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 2754218792Snp if (rc != 0) { 2755218792Snp device_printf(sc->dev, 2756218792Snp "failed to create ingress queue: %d\n", rc); 2757218792Snp return (rc); 2758218792Snp } 2759218792Snp 2760218792Snp iq->cidx = 0; 2761270297Snp iq->gen = F_RSPD_GEN; 2762218792Snp iq->intr_next = iq->intr_params; 2763218792Snp iq->cntxt_id = be16toh(c.iqid); 2764218792Snp iq->abs_id = be16toh(c.physiqid); 2765228561Snp iq->flags |= IQ_ALLOCATED; 2766218792Snp 2767218792Snp cntxt_id = iq->cntxt_id - sc->sge.iq_start; 2768228561Snp if (cntxt_id >= sc->sge.niq) { 2769228561Snp panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 2770228561Snp cntxt_id, sc->sge.niq - 1); 2771228561Snp } 2772218792Snp sc->sge.iqmap[cntxt_id] = iq; 2773218792Snp 2774218792Snp if (fl) { 2775270297Snp u_int qid; 2776270297Snp 2777270297Snp iq->flags |= IQ_HAS_FL; 2778218792Snp fl->cntxt_id = be16toh(c.fl0id); 2779218792Snp fl->pidx = fl->cidx = 0; 2780218792Snp 2781219883Snp cntxt_id = fl->cntxt_id - sc->sge.eq_start; 2782228561Snp if (cntxt_id >= sc->sge.neq) { 2783228561Snp panic("%s: fl->cntxt_id (%d) more than the max (%d)", 2784228561Snp __func__, cntxt_id, sc->sge.neq - 1); 2785228561Snp } 2786218792Snp sc->sge.eqmap[cntxt_id] = (void *)fl; 2787218792Snp 2788270297Snp qid = fl->cntxt_id; 2789270297Snp if (isset(&sc->doorbells, DOORBELL_UDB)) { 2790308304Sjhb uint32_t s_qpp = sc->params.sge.eq_s_qpp; 2791270297Snp uint32_t mask = (1 << s_qpp) - 1; 2792270297Snp volatile uint8_t *udb; 2793270297Snp 2794270297Snp udb = sc->udbs_base + UDBS_DB_OFFSET; 2795270297Snp udb += (qid >> s_qpp) << PAGE_SHIFT; 2796270297Snp qid &= mask; 2797270297Snp if (qid < PAGE_SIZE / UDBS_SEG_SIZE) { 2798270297Snp udb += qid << UDBS_SEG_SHIFT; 2799270297Snp qid = 0; 2800270297Snp } 2801270297Snp fl->udb = (volatile void *)udb; 2802270297Snp } 2803308304Sjhb fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db; 2804270297Snp 2805218792Snp FL_LOCK(fl); 2806228561Snp /* Enough to make sure the SGE doesn't think it's starved */ 2807228561Snp refill_fl(sc, fl, fl->lowat); 2808218792Snp FL_UNLOCK(fl); 2809218792Snp } 2810218792Snp 2811309560Sjhb if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && cong >= 0) { 2812253873Snp uint32_t param, val; 2813253873Snp 2814253873Snp param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 2815253873Snp V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 2816253873Snp V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 2817253889Snp if (cong == 0) 2818253889Snp val = 1 << 19; 2819253889Snp else { 2820253889Snp val = 2 << 19; 2821253889Snp for (i = 0; i < 4; i++) { 2822253889Snp if (cong & (1 << i)) 2823253889Snp val |= 1 << (i << 2); 2824253889Snp } 2825253889Snp } 2826253889Snp 2827253873Snp rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2828253873Snp if (rc != 0) { 2829253873Snp /* report error but carry on */ 2830253873Snp device_printf(sc->dev, 2831253873Snp "failed to set congestion manager context for " 2832253873Snp "ingress queue %d: %d\n", iq->cntxt_id, rc); 2833253873Snp } 2834253873Snp } 2835253873Snp 2836218792Snp /* Enable IQ interrupts */ 2837228561Snp atomic_store_rel_int(&iq->state, IQS_IDLE); 2838309447Sjhb t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | 2839218792Snp V_INGRESSQID(iq->cntxt_id)); 2840218792Snp 2841218792Snp return (0); 2842218792Snp} 2843218792Snp 2844218792Snpstatic int 2845308154Sjhbfree_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) 2846218792Snp{ 2847265425Snp int rc; 2848218792Snp struct adapter *sc = iq->adapter; 2849218792Snp device_t dev; 2850218792Snp 2851218792Snp if (sc == NULL) 2852218792Snp return (0); /* nothing to do */ 2853218792Snp 2854308154Sjhb dev = vi ? vi->dev : sc->dev; 2855218792Snp 2856218792Snp if (iq->flags & IQ_ALLOCATED) { 2857218792Snp rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 2858218792Snp FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 2859218792Snp fl ? fl->cntxt_id : 0xffff, 0xffff); 2860218792Snp if (rc != 0) { 2861218792Snp device_printf(dev, 2862218792Snp "failed to free queue %p: %d\n", iq, rc); 2863218792Snp return (rc); 2864218792Snp } 2865218792Snp iq->flags &= ~IQ_ALLOCATED; 2866218792Snp } 2867218792Snp 2868218792Snp free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); 2869218792Snp 2870218792Snp bzero(iq, sizeof(*iq)); 2871218792Snp 2872218792Snp if (fl) { 2873218792Snp free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, 2874218792Snp fl->desc); 2875218792Snp 2876254727Snp if (fl->sdesc) 2877255050Snp free_fl_sdesc(sc, fl); 2878218792Snp 2879218792Snp if (mtx_initialized(&fl->fl_lock)) 2880218792Snp mtx_destroy(&fl->fl_lock); 2881218792Snp 2882218792Snp bzero(fl, sizeof(*fl)); 2883218792Snp } 2884218792Snp 2885218792Snp return (0); 2886218792Snp} 2887218792Snp 2888265425Snpstatic void 2889309564Sjhbadd_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 2890309564Sjhb struct sysctl_oid *oid, struct sge_fl *fl) 2891265425Snp{ 2892265425Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 2893265425Snp 2894265425Snp oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL, 2895265425Snp "freelist"); 2896265425Snp children = SYSCTL_CHILDREN(oid); 2897265425Snp 2898309564Sjhb SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 2899309564Sjhb &fl->ba, "bus address of descriptor ring"); 2900309564Sjhb SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 2901309564Sjhb fl->sidx * EQ_ESIZE + sc->params.sge.spg_len, 2902309564Sjhb "desc ring size in bytes"); 2903265425Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 2904265425Snp CTLTYPE_INT | CTLFLAG_RD, &fl->cntxt_id, 0, sysctl_uint16, "I", 2905265425Snp "SGE context id of the freelist"); 2906281212Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL, 2907281212Snp fl_pad ? 1 : 0, "padding enabled"); 2908281212Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL, 2909281212Snp fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled"); 2910265425Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx, 2911265425Snp 0, "consumer index"); 2912265425Snp if (fl->flags & FL_BUF_PACKING) { 2913265425Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset", 2914265425Snp CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset"); 2915265425Snp } 2916265425Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx, 2917265425Snp 0, "producer index"); 2918265425Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_allocated", 2919265425Snp CTLFLAG_RD, &fl->mbuf_allocated, "# of mbuf allocated"); 2920265425Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_inlined", 2921265425Snp CTLFLAG_RD, &fl->mbuf_inlined, "# of mbuf inlined in clusters"); 2922265425Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated", 2923265425Snp CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated"); 2924265425Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled", 2925265425Snp CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled"); 2926265425Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled", 2927265425Snp CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)"); 2928265425Snp} 2929265425Snp 2930218792Snpstatic int 2931228561Snpalloc_fwq(struct adapter *sc) 2932218792Snp{ 2933228561Snp int rc, intr_idx; 2934228561Snp struct sge_iq *fwq = &sc->sge.fwq; 2935228561Snp struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); 2936228561Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 2937222510Snp 2938270297Snp init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE); 2939228561Snp fwq->flags |= IQ_INTR; /* always */ 2940309447Sjhb if (sc->flags & IS_VF) 2941309447Sjhb intr_idx = 0; 2942309447Sjhb else { 2943309447Sjhb intr_idx = sc->intr_count > 1 ? 1 : 0; 2944309447Sjhb fwq->set_tcb_rpl = t4_filter_rpl; 2945309447Sjhb fwq->l2t_write_rpl = do_l2t_write_rpl; 2946309447Sjhb } 2947308154Sjhb rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1); 2948228561Snp if (rc != 0) { 2949228561Snp device_printf(sc->dev, 2950228561Snp "failed to create firmware event queue: %d\n", rc); 2951222510Snp return (rc); 2952228561Snp } 2953222510Snp 2954228561Snp oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD, 2955228561Snp NULL, "firmware event queue"); 2956222510Snp children = SYSCTL_CHILDREN(oid); 2957222510Snp 2958309564Sjhb SYSCTL_ADD_UAUTO(&sc->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 2959309564Sjhb &fwq->ba, "bus address of descriptor ring"); 2960309564Sjhb SYSCTL_ADD_INT(&sc->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 2961309564Sjhb fwq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); 2962228561Snp SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id", 2963228561Snp CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I", 2964228561Snp "absolute id of the queue"); 2965228561Snp SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id", 2966228561Snp CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I", 2967228561Snp "SGE context id of the queue"); 2968222510Snp SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx", 2969228561Snp CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I", 2970222510Snp "consumer index"); 2971222510Snp 2972228561Snp return (0); 2973218792Snp} 2974218792Snp 2975218792Snpstatic int 2976228561Snpfree_fwq(struct adapter *sc) 2977218792Snp{ 2978228561Snp return free_iq_fl(NULL, &sc->sge.fwq, NULL); 2979218792Snp} 2980218792Snp 2981218792Snpstatic int 2982228561Snpalloc_mgmtq(struct adapter *sc) 2983222510Snp{ 2984222510Snp int rc; 2985228561Snp struct sge_wrq *mgmtq = &sc->sge.mgmtq; 2986228561Snp char name[16]; 2987228561Snp struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); 2988228561Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 2989222510Snp 2990228561Snp oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD, 2991228561Snp NULL, "management queue"); 2992228561Snp 2993228561Snp snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev)); 2994308304Sjhb init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, 2995228561Snp sc->sge.fwq.cntxt_id, name); 2996228561Snp rc = alloc_wrq(sc, NULL, mgmtq, oid); 2997228561Snp if (rc != 0) { 2998228561Snp device_printf(sc->dev, 2999228561Snp "failed to create management queue: %d\n", rc); 3000222510Snp return (rc); 3001228561Snp } 3002222510Snp 3003228561Snp return (0); 3004222510Snp} 3005222510Snp 3006222510Snpstatic int 3007228561Snpfree_mgmtq(struct adapter *sc) 3008222510Snp{ 3009237263Snp 3010228561Snp return free_wrq(sc, &sc->sge.mgmtq); 3011222510Snp} 3012222510Snp 3013281253Snpint 3014286274Snptnl_cong(struct port_info *pi, int drop) 3015239258Snp{ 3016239258Snp 3017286274Snp if (drop == -1) 3018239258Snp return (-1); 3019286274Snp else if (drop == 1) 3020239258Snp return (0); 3021239258Snp else 3022265410Snp return (pi->rx_chan_map); 3023239258Snp} 3024239258Snp 3025222510Snpstatic int 3026308154Sjhballoc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, 3027228561Snp struct sysctl_oid *oid) 3028218792Snp{ 3029218792Snp int rc; 3030309447Sjhb struct adapter *sc = vi->pi->adapter; 3031218792Snp struct sysctl_oid_list *children; 3032218792Snp char name[16]; 3033218792Snp 3034308154Sjhb rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx, 3035308154Sjhb tnl_cong(vi->pi, cong_drop)); 3036218792Snp if (rc != 0) 3037218792Snp return (rc); 3038218792Snp 3039309447Sjhb if (idx == 0) 3040309447Sjhb sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; 3041309447Sjhb else 3042309447Sjhb KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, 3043309447Sjhb ("iq_base mismatch")); 3044309447Sjhb KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, 3045309447Sjhb ("PF with non-zero iq_base")); 3046309447Sjhb 3047270297Snp /* 3048270297Snp * The freelist is just barely above the starvation threshold right now, 3049270297Snp * fill it up a bit more. 3050270297Snp */ 3051222701Snp FL_LOCK(&rxq->fl); 3052309447Sjhb refill_fl(sc, &rxq->fl, 128); 3053222701Snp FL_UNLOCK(&rxq->fl); 3054222701Snp 3055237819Snp#if defined(INET) || defined(INET6) 3056218792Snp rc = tcp_lro_init(&rxq->lro); 3057218792Snp if (rc != 0) 3058218792Snp return (rc); 3059308154Sjhb rxq->lro.ifp = vi->ifp; /* also indicates LRO init'ed */ 3060218792Snp 3061308154Sjhb if (vi->ifp->if_capenable & IFCAP_LRO) 3062228561Snp rxq->iq.flags |= IQ_LRO_ENABLED; 3063218792Snp#endif 3064308154Sjhb rxq->ifp = vi->ifp; 3065218792Snp 3066228561Snp children = SYSCTL_CHILDREN(oid); 3067218792Snp 3068218792Snp snprintf(name, sizeof(name), "%d", idx); 3069308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, 3070218792Snp NULL, "rx queue"); 3071218792Snp children = SYSCTL_CHILDREN(oid); 3072218792Snp 3073309564Sjhb SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3074309564Sjhb &rxq->iq.ba, "bus address of descriptor ring"); 3075309564Sjhb SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3076309564Sjhb rxq->iq.qsize * IQ_ESIZE, "descriptor ring size in bytes"); 3077308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "abs_id", 3078222510Snp CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_uint16, "I", 3079221911Snp "absolute id of the queue"); 3080308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cntxt_id", 3081222973Snp CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cntxt_id, 0, sysctl_uint16, "I", 3082222973Snp "SGE context id of the queue"); 3083308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 3084222973Snp CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cidx, 0, sysctl_uint16, "I", 3085222973Snp "consumer index"); 3086237819Snp#if defined(INET) || defined(INET6) 3087308154Sjhb SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, 3088218792Snp &rxq->lro.lro_queued, 0, NULL); 3089308154Sjhb SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, 3090218792Snp &rxq->lro.lro_flushed, 0, NULL); 3091219290Snp#endif 3092308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, 3093218792Snp &rxq->rxcsum, "# of times hardware assisted with checksum"); 3094308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_extraction", 3095218792Snp CTLFLAG_RD, &rxq->vlan_extraction, 3096218792Snp "# of times hardware extracted 802.1Q tag"); 3097218792Snp 3098309564Sjhb add_fl_sysctls(sc, &vi->ctx, oid, &rxq->fl); 3099222973Snp 3100218792Snp return (rc); 3101218792Snp} 3102218792Snp 3103218792Snpstatic int 3104308154Sjhbfree_rxq(struct vi_info *vi, struct sge_rxq *rxq) 3105218792Snp{ 3106218792Snp int rc; 3107218792Snp 3108237819Snp#if defined(INET) || defined(INET6) 3109218792Snp if (rxq->lro.ifp) { 3110218792Snp tcp_lro_free(&rxq->lro); 3111218792Snp rxq->lro.ifp = NULL; 3112218792Snp } 3113218792Snp#endif 3114218792Snp 3115308154Sjhb rc = free_iq_fl(vi, &rxq->iq, &rxq->fl); 3116218792Snp if (rc == 0) 3117218792Snp bzero(rxq, sizeof(*rxq)); 3118218792Snp 3119218792Snp return (rc); 3120218792Snp} 3121218792Snp 3122237263Snp#ifdef TCP_OFFLOAD 3123218792Snpstatic int 3124308154Sjhballoc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, 3125228561Snp int intr_idx, int idx, struct sysctl_oid *oid) 3126220873Snp{ 3127309564Sjhb struct port_info *pi = vi->pi; 3128228561Snp int rc; 3129228561Snp struct sysctl_oid_list *children; 3130220873Snp char name[16]; 3131220873Snp 3132308154Sjhb rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 3133309564Sjhb pi->rx_chan_map); 3134228561Snp if (rc != 0) 3135220873Snp return (rc); 3136220873Snp 3137228561Snp children = SYSCTL_CHILDREN(oid); 3138220873Snp 3139228561Snp snprintf(name, sizeof(name), "%d", idx); 3140308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, 3141228561Snp NULL, "rx queue"); 3142228561Snp children = SYSCTL_CHILDREN(oid); 3143228561Snp 3144309564Sjhb SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3145309564Sjhb &ofld_rxq->iq.ba, "bus address of descriptor ring"); 3146309564Sjhb SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3147309564Sjhb ofld_rxq->iq.qsize * IQ_ESIZE, "descriptor ring size in bytes"); 3148308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "abs_id", 3149228561Snp CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.abs_id, 0, sysctl_uint16, 3150228561Snp "I", "absolute id of the queue"); 3151308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cntxt_id", 3152228561Snp CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cntxt_id, 0, sysctl_uint16, 3153228561Snp "I", "SGE context id of the queue"); 3154308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 3155228561Snp CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cidx, 0, sysctl_uint16, "I", 3156228561Snp "consumer index"); 3157228561Snp 3158309564Sjhb add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl); 3159228561Snp 3160228561Snp return (rc); 3161228561Snp} 3162228561Snp 3163228561Snpstatic int 3164308154Sjhbfree_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) 3165228561Snp{ 3166228561Snp int rc; 3167228561Snp 3168308154Sjhb rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl); 3169228561Snp if (rc == 0) 3170228561Snp bzero(ofld_rxq, sizeof(*ofld_rxq)); 3171228561Snp 3172228561Snp return (rc); 3173228561Snp} 3174228561Snp#endif 3175228561Snp 3176270297Snp#ifdef DEV_NETMAP 3177228561Snpstatic int 3178308154Sjhballoc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx, 3179270297Snp int idx, struct sysctl_oid *oid) 3180270297Snp{ 3181270297Snp int rc; 3182270297Snp struct sysctl_oid_list *children; 3183270297Snp struct sysctl_ctx_list *ctx; 3184270297Snp char name[16]; 3185270297Snp size_t len; 3186308154Sjhb struct adapter *sc = vi->pi->adapter; 3187308154Sjhb struct netmap_adapter *na = NA(vi->ifp); 3188270297Snp 3189270297Snp MPASS(na != NULL); 3190270297Snp 3191308154Sjhb len = vi->qsize_rxq * IQ_ESIZE; 3192270297Snp rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map, 3193270297Snp &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc); 3194270297Snp if (rc != 0) 3195270297Snp return (rc); 3196270297Snp 3197308304Sjhb len = na->num_rx_desc * EQ_ESIZE + sc->params.sge.spg_len; 3198270297Snp rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map, 3199270297Snp &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc); 3200270297Snp if (rc != 0) 3201270297Snp return (rc); 3202270297Snp 3203308154Sjhb nm_rxq->vi = vi; 3204270297Snp nm_rxq->nid = idx; 3205270297Snp nm_rxq->iq_cidx = 0; 3206308304Sjhb nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE; 3207270297Snp nm_rxq->iq_gen = F_RSPD_GEN; 3208270297Snp nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0; 3209270297Snp nm_rxq->fl_sidx = na->num_rx_desc; 3210270297Snp nm_rxq->intr_idx = intr_idx; 3211270297Snp 3212308154Sjhb ctx = &vi->ctx; 3213270297Snp children = SYSCTL_CHILDREN(oid); 3214270297Snp 3215270297Snp snprintf(name, sizeof(name), "%d", idx); 3216270297Snp oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL, 3217270297Snp "rx queue"); 3218270297Snp children = SYSCTL_CHILDREN(oid); 3219270297Snp 3220270297Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", 3221270297Snp CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16, 3222270297Snp "I", "absolute id of the queue"); 3223270297Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3224270297Snp CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16, 3225270297Snp "I", "SGE context id of the queue"); 3226270297Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3227270297Snp CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I", 3228270297Snp "consumer index"); 3229270297Snp 3230270297Snp children = SYSCTL_CHILDREN(oid); 3231270297Snp oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL, 3232270297Snp "freelist"); 3233270297Snp children = SYSCTL_CHILDREN(oid); 3234270297Snp 3235270297Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3236270297Snp CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16, 3237270297Snp "I", "SGE context id of the freelist"); 3238270297Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, 3239270297Snp &nm_rxq->fl_cidx, 0, "consumer index"); 3240270297Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, 3241270297Snp &nm_rxq->fl_pidx, 0, "producer index"); 3242270297Snp 3243270297Snp return (rc); 3244270297Snp} 3245270297Snp 3246270297Snp 3247270297Snpstatic int 3248308154Sjhbfree_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq) 3249270297Snp{ 3250308154Sjhb struct adapter *sc = vi->pi->adapter; 3251270297Snp 3252270297Snp free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba, 3253270297Snp nm_rxq->iq_desc); 3254270297Snp free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba, 3255270297Snp nm_rxq->fl_desc); 3256270297Snp 3257270297Snp return (0); 3258270297Snp} 3259270297Snp 3260270297Snpstatic int 3261308154Sjhballoc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx, 3262270297Snp struct sysctl_oid *oid) 3263270297Snp{ 3264270297Snp int rc; 3265270297Snp size_t len; 3266308154Sjhb struct port_info *pi = vi->pi; 3267270297Snp struct adapter *sc = pi->adapter; 3268308154Sjhb struct netmap_adapter *na = NA(vi->ifp); 3269270297Snp char name[16]; 3270270297Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3271270297Snp 3272308304Sjhb len = na->num_tx_desc * EQ_ESIZE + sc->params.sge.spg_len; 3273270297Snp rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map, 3274270297Snp &nm_txq->ba, (void **)&nm_txq->desc); 3275270297Snp if (rc) 3276270297Snp return (rc); 3277270297Snp 3278270297Snp nm_txq->pidx = nm_txq->cidx = 0; 3279270297Snp nm_txq->sidx = na->num_tx_desc; 3280270297Snp nm_txq->nid = idx; 3281270297Snp nm_txq->iqidx = iqidx; 3282270297Snp nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3283309560Sjhb V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) | 3284309560Sjhb V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) | 3285309560Sjhb V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid))); 3286270297Snp 3287270297Snp snprintf(name, sizeof(name), "%d", idx); 3288308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, 3289270297Snp NULL, "netmap tx queue"); 3290270297Snp children = SYSCTL_CHILDREN(oid); 3291270297Snp 3292308154Sjhb SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3293270297Snp &nm_txq->cntxt_id, 0, "SGE context id of the queue"); 3294308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 3295270297Snp CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I", 3296270297Snp "consumer index"); 3297308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", 3298270297Snp CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I", 3299270297Snp "producer index"); 3300270297Snp 3301270297Snp return (rc); 3302270297Snp} 3303270297Snp 3304270297Snpstatic int 3305308154Sjhbfree_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq) 3306270297Snp{ 3307308154Sjhb struct adapter *sc = vi->pi->adapter; 3308270297Snp 3309270297Snp free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba, 3310270297Snp nm_txq->desc); 3311270297Snp 3312270297Snp return (0); 3313270297Snp} 3314270297Snp#endif 3315270297Snp 3316270297Snpstatic int 3317228561Snpctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 3318228561Snp{ 3319228561Snp int rc, cntxt_id; 3320228561Snp struct fw_eq_ctrl_cmd c; 3321308304Sjhb int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3322228561Snp 3323220873Snp bzero(&c, sizeof(c)); 3324220873Snp 3325220873Snp c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 3326220873Snp F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 3327220873Snp V_FW_EQ_CTRL_CMD_VFN(0)); 3328220873Snp c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | 3329220873Snp F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 3330284052Snp c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); 3331220873Snp c.physeqid_pkd = htobe32(0); 3332220873Snp c.fetchszm_to_iqid = 3333312186Snp htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 3334228561Snp V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 3335222510Snp F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 3336220873Snp c.dcaen_to_eqsize = 3337220873Snp htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 3338220873Snp V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3339312186Snp V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 3340284052Snp V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); 3341220873Snp c.eqaddr = htobe64(eq->ba); 3342220873Snp 3343220873Snp rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3344220873Snp if (rc != 0) { 3345220873Snp device_printf(sc->dev, 3346228561Snp "failed to create control queue %d: %d\n", eq->tx_chan, rc); 3347220873Snp return (rc); 3348220873Snp } 3349228561Snp eq->flags |= EQ_ALLOCATED; 3350220873Snp 3351220873Snp eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid)); 3352228561Snp cntxt_id = eq->cntxt_id - sc->sge.eq_start; 3353228561Snp if (cntxt_id >= sc->sge.neq) 3354228561Snp panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 3355228561Snp cntxt_id, sc->sge.neq - 1); 3356228561Snp sc->sge.eqmap[cntxt_id] = eq; 3357220873Snp 3358228561Snp return (rc); 3359228561Snp} 3360228561Snp 3361228561Snpstatic int 3362308154Sjhbeth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 3363228561Snp{ 3364228561Snp int rc, cntxt_id; 3365228561Snp struct fw_eq_eth_cmd c; 3366308304Sjhb int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3367228561Snp 3368228561Snp bzero(&c, sizeof(c)); 3369228561Snp 3370228561Snp c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 3371228561Snp F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 3372228561Snp V_FW_EQ_ETH_CMD_VFN(0)); 3373228561Snp c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | 3374228561Snp F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 3375284052Snp c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 3376308154Sjhb F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); 3377228561Snp c.fetchszm_to_iqid = 3378284052Snp htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | 3379228561Snp V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 3380228561Snp V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 3381228561Snp c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 3382284052Snp V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3383284052Snp V_FW_EQ_ETH_CMD_EQSIZE(qsize)); 3384228561Snp c.eqaddr = htobe64(eq->ba); 3385228561Snp 3386228561Snp rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3387228561Snp if (rc != 0) { 3388308154Sjhb device_printf(vi->dev, 3389228561Snp "failed to create Ethernet egress queue: %d\n", rc); 3390228561Snp return (rc); 3391228561Snp } 3392228561Snp eq->flags |= EQ_ALLOCATED; 3393228561Snp 3394228561Snp eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); 3395309447Sjhb eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); 3396220873Snp cntxt_id = eq->cntxt_id - sc->sge.eq_start; 3397228561Snp if (cntxt_id >= sc->sge.neq) 3398228561Snp panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 3399228561Snp cntxt_id, sc->sge.neq - 1); 3400220873Snp sc->sge.eqmap[cntxt_id] = eq; 3401220873Snp 3402228561Snp return (rc); 3403228561Snp} 3404220873Snp 3405237263Snp#ifdef TCP_OFFLOAD 3406228561Snpstatic int 3407308154Sjhbofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 3408228561Snp{ 3409228561Snp int rc, cntxt_id; 3410228561Snp struct fw_eq_ofld_cmd c; 3411308304Sjhb int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3412220873Snp 3413228561Snp bzero(&c, sizeof(c)); 3414220873Snp 3415228561Snp c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 3416228561Snp F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 3417228561Snp V_FW_EQ_OFLD_CMD_VFN(0)); 3418228561Snp c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 3419228561Snp F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 3420228561Snp c.fetchszm_to_iqid = 3421284052Snp htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | 3422228561Snp V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 3423228561Snp F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 3424228561Snp c.dcaen_to_eqsize = 3425228561Snp htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 3426228561Snp V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3427284052Snp V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); 3428228561Snp c.eqaddr = htobe64(eq->ba); 3429228561Snp 3430228561Snp rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3431228561Snp if (rc != 0) { 3432308154Sjhb device_printf(vi->dev, 3433228561Snp "failed to create egress queue for TCP offload: %d\n", rc); 3434228561Snp return (rc); 3435228561Snp } 3436228561Snp eq->flags |= EQ_ALLOCATED; 3437228561Snp 3438228561Snp eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd)); 3439228561Snp cntxt_id = eq->cntxt_id - sc->sge.eq_start; 3440228561Snp if (cntxt_id >= sc->sge.neq) 3441228561Snp panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 3442228561Snp cntxt_id, sc->sge.neq - 1); 3443228561Snp sc->sge.eqmap[cntxt_id] = eq; 3444228561Snp 3445220873Snp return (rc); 3446220873Snp} 3447228561Snp#endif 3448220873Snp 3449220873Snpstatic int 3450308154Sjhballoc_eq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 3451220873Snp{ 3452284052Snp int rc, qsize; 3453228561Snp size_t len; 3454220873Snp 3455228561Snp mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); 3456228561Snp 3457308304Sjhb qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 3458284052Snp len = qsize * EQ_ESIZE; 3459228561Snp rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, 3460228561Snp &eq->ba, (void **)&eq->desc); 3461228561Snp if (rc) 3462228561Snp return (rc); 3463228561Snp 3464228561Snp eq->pidx = eq->cidx = 0; 3465284052Snp eq->equeqidx = eq->dbidx = 0; 3466248925Snp eq->doorbells = sc->doorbells; 3467228561Snp 3468228561Snp switch (eq->flags & EQ_TYPEMASK) { 3469228561Snp case EQ_CTRL: 3470228561Snp rc = ctrl_eq_alloc(sc, eq); 3471228561Snp break; 3472228561Snp 3473228561Snp case EQ_ETH: 3474308154Sjhb rc = eth_eq_alloc(sc, vi, eq); 3475228561Snp break; 3476228561Snp 3477237263Snp#ifdef TCP_OFFLOAD 3478228561Snp case EQ_OFLD: 3479308154Sjhb rc = ofld_eq_alloc(sc, vi, eq); 3480228561Snp break; 3481228561Snp#endif 3482228561Snp 3483228561Snp default: 3484228561Snp panic("%s: invalid eq type %d.", __func__, 3485228561Snp eq->flags & EQ_TYPEMASK); 3486228561Snp } 3487228561Snp if (rc != 0) { 3488228561Snp device_printf(sc->dev, 3489269082Snp "failed to allocate egress queue(%d): %d\n", 3490228561Snp eq->flags & EQ_TYPEMASK, rc); 3491228561Snp } 3492228561Snp 3493248925Snp if (isset(&eq->doorbells, DOORBELL_UDB) || 3494248925Snp isset(&eq->doorbells, DOORBELL_UDBWC) || 3495249392Snp isset(&eq->doorbells, DOORBELL_WCWR)) { 3496308304Sjhb uint32_t s_qpp = sc->params.sge.eq_s_qpp; 3497248925Snp uint32_t mask = (1 << s_qpp) - 1; 3498248925Snp volatile uint8_t *udb; 3499248925Snp 3500248925Snp udb = sc->udbs_base + UDBS_DB_OFFSET; 3501248925Snp udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 3502248925Snp eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 3503270297Snp if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE) 3504249392Snp clrbit(&eq->doorbells, DOORBELL_WCWR); 3505248925Snp else { 3506248925Snp udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 3507248925Snp eq->udb_qid = 0; 3508248925Snp } 3509248925Snp eq->udb = (volatile void *)udb; 3510248925Snp } 3511248925Snp 3512228561Snp return (rc); 3513228561Snp} 3514228561Snp 3515228561Snpstatic int 3516228561Snpfree_eq(struct adapter *sc, struct sge_eq *eq) 3517228561Snp{ 3518228561Snp int rc; 3519228561Snp 3520228561Snp if (eq->flags & EQ_ALLOCATED) { 3521228561Snp switch (eq->flags & EQ_TYPEMASK) { 3522228561Snp case EQ_CTRL: 3523228561Snp rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 3524228561Snp eq->cntxt_id); 3525228561Snp break; 3526228561Snp 3527228561Snp case EQ_ETH: 3528228561Snp rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 3529228561Snp eq->cntxt_id); 3530228561Snp break; 3531228561Snp 3532237263Snp#ifdef TCP_OFFLOAD 3533228561Snp case EQ_OFLD: 3534228561Snp rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 3535228561Snp eq->cntxt_id); 3536228561Snp break; 3537228561Snp#endif 3538228561Snp 3539228561Snp default: 3540228561Snp panic("%s: invalid eq type %d.", __func__, 3541228561Snp eq->flags & EQ_TYPEMASK); 3542228561Snp } 3543220873Snp if (rc != 0) { 3544220873Snp device_printf(sc->dev, 3545228561Snp "failed to free egress queue (%d): %d\n", 3546228561Snp eq->flags & EQ_TYPEMASK, rc); 3547220873Snp return (rc); 3548220873Snp } 3549228561Snp eq->flags &= ~EQ_ALLOCATED; 3550220873Snp } 3551220873Snp 3552220873Snp free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc); 3553220873Snp 3554220873Snp if (mtx_initialized(&eq->eq_lock)) 3555220873Snp mtx_destroy(&eq->eq_lock); 3556220873Snp 3557228561Snp bzero(eq, sizeof(*eq)); 3558220873Snp return (0); 3559220873Snp} 3560220873Snp 3561220873Snpstatic int 3562308154Sjhballoc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq, 3563228561Snp struct sysctl_oid *oid) 3564218792Snp{ 3565228561Snp int rc; 3566308154Sjhb struct sysctl_ctx_list *ctx = vi ? &vi->ctx : &sc->ctx; 3567228561Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3568228561Snp 3569308154Sjhb rc = alloc_eq(sc, vi, &wrq->eq); 3570228561Snp if (rc) 3571228561Snp return (rc); 3572228561Snp 3573228561Snp wrq->adapter = sc; 3574284052Snp TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); 3575284052Snp TAILQ_INIT(&wrq->incomplete_wrs); 3576237263Snp STAILQ_INIT(&wrq->wr_list); 3577284052Snp wrq->nwr_pending = 0; 3578284052Snp wrq->ndesc_needed = 0; 3579228561Snp 3580309564Sjhb SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3581309564Sjhb &wrq->eq.ba, "bus address of descriptor ring"); 3582309564Sjhb SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3583309564Sjhb wrq->eq.sidx * EQ_ESIZE + sc->params.sge.spg_len, 3584309564Sjhb "desc ring size in bytes"); 3585228561Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3586228561Snp &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); 3587228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3588228561Snp CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I", 3589228561Snp "consumer index"); 3590228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", 3591228561Snp CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I", 3592228561Snp "producer index"); 3593309564Sjhb SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 3594309564Sjhb wrq->eq.sidx, "status page index"); 3595284052Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, 3596284052Snp &wrq->tx_wrs_direct, "# of work requests (direct)"); 3597284052Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, 3598284052Snp &wrq->tx_wrs_copied, "# of work requests (copied)"); 3599309458Sjhb SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD, 3600309458Sjhb &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)"); 3601228561Snp 3602228561Snp return (rc); 3603228561Snp} 3604228561Snp 3605228561Snpstatic int 3606228561Snpfree_wrq(struct adapter *sc, struct sge_wrq *wrq) 3607228561Snp{ 3608228561Snp int rc; 3609228561Snp 3610228561Snp rc = free_eq(sc, &wrq->eq); 3611228561Snp if (rc) 3612228561Snp return (rc); 3613228561Snp 3614228561Snp bzero(wrq, sizeof(*wrq)); 3615228561Snp return (0); 3616228561Snp} 3617228561Snp 3618228561Snpstatic int 3619308154Sjhballoc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, 3620228561Snp struct sysctl_oid *oid) 3621228561Snp{ 3622228561Snp int rc; 3623308154Sjhb struct port_info *pi = vi->pi; 3624218792Snp struct adapter *sc = pi->adapter; 3625218792Snp struct sge_eq *eq = &txq->eq; 3626218792Snp char name[16]; 3627228561Snp struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3628218792Snp 3629284052Snp rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx, 3630284052Snp M_CXGBE, M_WAITOK); 3631218792Snp if (rc != 0) { 3632284052Snp device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc); 3633218792Snp return (rc); 3634218792Snp } 3635218792Snp 3636308154Sjhb rc = alloc_eq(sc, vi, eq); 3637218792Snp if (rc != 0) { 3638284052Snp mp_ring_free(txq->r); 3639284052Snp txq->r = NULL; 3640218792Snp return (rc); 3641218792Snp } 3642218792Snp 3643284052Snp /* Can't fail after this point. */ 3644284052Snp 3645309447Sjhb if (idx == 0) 3646309447Sjhb sc->sge.eq_base = eq->abs_id - eq->cntxt_id; 3647309447Sjhb else 3648309447Sjhb KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, 3649309447Sjhb ("eq_base mismatch")); 3650309447Sjhb KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, 3651309447Sjhb ("PF with non-zero eq_base")); 3652309447Sjhb 3653284052Snp TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); 3654308154Sjhb txq->ifp = vi->ifp; 3655284052Snp txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); 3656309447Sjhb if (sc->flags & IS_VF) 3657309447Sjhb txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 3658309447Sjhb V_TXPKT_INTF(pi->tx_chan)); 3659309447Sjhb else 3660309447Sjhb txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3661309560Sjhb V_TXPKT_INTF(pi->tx_chan) | 3662309560Sjhb V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) | 3663309560Sjhb V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) | 3664309560Sjhb V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid))); 3665308321Sjhb txq->tc_idx = -1; 3666284052Snp txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, 3667284052Snp M_ZERO | M_WAITOK); 3668284052Snp 3669218792Snp snprintf(name, sizeof(name), "%d", idx); 3670308154Sjhb oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD, 3671218792Snp NULL, "tx queue"); 3672218792Snp children = SYSCTL_CHILDREN(oid); 3673218792Snp 3674309564Sjhb SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3675309564Sjhb &eq->ba, "bus address of descriptor ring"); 3676309564Sjhb SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3677309564Sjhb eq->sidx * EQ_ESIZE + sc->params.sge.spg_len, 3678309564Sjhb "desc ring size in bytes"); 3679309447Sjhb SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, 3680309447Sjhb &eq->abs_id, 0, "absolute id of the queue"); 3681308154Sjhb SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3682222973Snp &eq->cntxt_id, 0, "SGE context id of the queue"); 3683308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 3684222973Snp CTLTYPE_INT | CTLFLAG_RD, &eq->cidx, 0, sysctl_uint16, "I", 3685222973Snp "consumer index"); 3686308154Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", 3687222973Snp CTLTYPE_INT | CTLFLAG_RD, &eq->pidx, 0, sysctl_uint16, "I", 3688222973Snp "producer index"); 3689309564Sjhb SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 3690309564Sjhb eq->sidx, "status page index"); 3691222973Snp 3692308321Sjhb SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "tc", 3693308321Sjhb CTLTYPE_INT | CTLFLAG_RW, vi, idx, sysctl_tc, "I", 3694308321Sjhb "traffic class (-1 means none)"); 3695308321Sjhb 3696308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD, 3697218792Snp &txq->txcsum, "# of times hardware assisted with checksum"); 3698308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_insertion", 3699218792Snp CTLFLAG_RD, &txq->vlan_insertion, 3700218792Snp "# of times hardware inserted 802.1Q tag"); 3701308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, 3702237819Snp &txq->tso_wrs, "# of TSO work requests"); 3703308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, 3704218792Snp &txq->imm_wrs, "# of work requests with immediate data"); 3705308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, 3706218792Snp &txq->sgl_wrs, "# of work requests with direct SGL"); 3707308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, 3708218792Snp &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); 3709308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_wrs", 3710284052Snp CTLFLAG_RD, &txq->txpkts0_wrs, 3711284052Snp "# of txpkts (type 0) work requests"); 3712308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_wrs", 3713284052Snp CTLFLAG_RD, &txq->txpkts1_wrs, 3714284052Snp "# of txpkts (type 1) work requests"); 3715308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", 3716284052Snp CTLFLAG_RD, &txq->txpkts0_pkts, 3717284052Snp "# of frames tx'd using type0 txpkts work requests"); 3718308154Sjhb SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", 3719284052Snp CTLFLAG_RD, &txq->txpkts1_pkts, 3720284052Snp "# of frames tx'd using type1 txpkts work requests"); 3721218792Snp 3722308154Sjhb SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_enqueues", 3723284052Snp CTLFLAG_RD, &txq->r->enqueues, 3724284052Snp "# of enqueues to the mp_ring for this queue"); 3725308154Sjhb SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_drops", 3726284052Snp CTLFLAG_RD, &txq->r->drops, 3727284052Snp "# of drops in the mp_ring for this queue"); 3728308154Sjhb SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_starts", 3729284052Snp CTLFLAG_RD, &txq->r->starts, 3730284052Snp "# of normal consumer starts in the mp_ring for this queue"); 3731308154Sjhb SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_stalls", 3732284052Snp CTLFLAG_RD, &txq->r->stalls, 3733284052Snp "# of consumer stalls in the mp_ring for this queue"); 3734308154Sjhb SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_restarts", 3735284052Snp CTLFLAG_RD, &txq->r->restarts, 3736284052Snp "# of consumer restarts in the mp_ring for this queue"); 3737308154Sjhb SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_abdications", 3738284052Snp CTLFLAG_RD, &txq->r->abdications, 3739284052Snp "# of consumer abdications in the mp_ring for this queue"); 3740218792Snp 3741284052Snp return (0); 3742218792Snp} 3743218792Snp 3744218792Snpstatic int 3745308154Sjhbfree_txq(struct vi_info *vi, struct sge_txq *txq) 3746218792Snp{ 3747218792Snp int rc; 3748308154Sjhb struct adapter *sc = vi->pi->adapter; 3749218792Snp struct sge_eq *eq = &txq->eq; 3750218792Snp 3751228561Snp rc = free_eq(sc, eq); 3752228561Snp if (rc) 3753228561Snp return (rc); 3754220649Snp 3755284052Snp sglist_free(txq->gl); 3756220873Snp free(txq->sdesc, M_CXGBE); 3757284052Snp mp_ring_free(txq->r); 3758218792Snp 3759218792Snp bzero(txq, sizeof(*txq)); 3760218792Snp return (0); 3761218792Snp} 3762218792Snp 3763218792Snpstatic void 3764218792Snponeseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3765218792Snp{ 3766218792Snp bus_addr_t *ba = arg; 3767218792Snp 3768218792Snp KASSERT(nseg == 1, 3769218792Snp ("%s meant for single segment mappings only.", __func__)); 3770218792Snp 3771218792Snp *ba = error ? 0 : segs->ds_addr; 3772218792Snp} 3773218792Snp 3774218792Snpstatic inline void 3775218792Snpring_fl_db(struct adapter *sc, struct sge_fl *fl) 3776218792Snp{ 3777270297Snp uint32_t n, v; 3778218792Snp 3779270297Snp n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx); 3780270297Snp MPASS(n > 0); 3781218792Snp 3782218792Snp wmb(); 3783270297Snp v = fl->dbval | V_PIDX(n); 3784270297Snp if (fl->udb) 3785270297Snp *fl->udb = htole32(v); 3786270297Snp else 3787309447Sjhb t4_write_reg(sc, sc->sge_kdoorbell_reg, v); 3788270297Snp IDXINCR(fl->dbidx, n, fl->sidx); 3789218792Snp} 3790218792Snp 3791220905Snp/* 3792270297Snp * Fills up the freelist by allocating upto 'n' buffers. Buffers that are 3793270297Snp * recycled do not count towards this allocation budget. 3794228561Snp * 3795270297Snp * Returns non-zero to indicate that this freelist should be added to the list 3796270297Snp * of starving freelists. 3797220905Snp */ 3798228561Snpstatic int 3799270297Snprefill_fl(struct adapter *sc, struct sge_fl *fl, int n) 3800218792Snp{ 3801270297Snp __be64 *d; 3802270297Snp struct fl_sdesc *sd; 3803265425Snp uintptr_t pa; 3804218792Snp caddr_t cl; 3805270297Snp struct cluster_layout *cll; 3806270297Snp struct sw_zone_info *swz; 3807265425Snp struct cluster_metadata *clm; 3808270297Snp uint16_t max_pidx; 3809270297Snp uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */ 3810218792Snp 3811218792Snp FL_LOCK_ASSERT_OWNED(fl); 3812218792Snp 3813270297Snp /* 3814270297Snp * We always stop at the begining of the hardware descriptor that's just 3815270297Snp * before the one with the hw cidx. This is to avoid hw pidx = hw cidx, 3816270297Snp * which would mean an empty freelist to the chip. 3817270297Snp */ 3818270297Snp max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1; 3819270297Snp if (fl->pidx == max_pidx * 8) 3820270297Snp return (0); 3821218792Snp 3822270297Snp d = &fl->desc[fl->pidx]; 3823270297Snp sd = &fl->sdesc[fl->pidx]; 3824270297Snp cll = &fl->cll_def; /* default layout */ 3825270297Snp swz = &sc->sge.sw_zone_info[cll->zidx]; 3826218792Snp 3827270297Snp while (n > 0) { 3828270297Snp 3829218792Snp if (sd->cl != NULL) { 3830218792Snp 3831269356Snp if (sd->nmbuf == 0) { 3832255050Snp /* 3833265425Snp * Fast recycle without involving any atomics on 3834265425Snp * the cluster's metadata (if the cluster has 3835265425Snp * metadata). This happens when all frames 3836265425Snp * received in the cluster were small enough to 3837265425Snp * fit within a single mbuf each. 3838255050Snp */ 3839265425Snp fl->cl_fast_recycled++; 3840267694Snp#ifdef INVARIANTS 3841267694Snp clm = cl_metadata(sc, fl, &sd->cll, sd->cl); 3842267694Snp if (clm != NULL) 3843267694Snp MPASS(clm->refcount == 1); 3844267694Snp#endif 3845265425Snp goto recycled_fast; 3846255050Snp } 3847218792Snp 3848218792Snp /* 3849265425Snp * Cluster is guaranteed to have metadata. Clusters 3850265425Snp * without metadata always take the fast recycle path 3851265425Snp * when they're recycled. 3852218792Snp */ 3853265425Snp clm = cl_metadata(sc, fl, &sd->cll, sd->cl); 3854265425Snp MPASS(clm != NULL); 3855265425Snp 3856265425Snp if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 3857265425Snp fl->cl_recycled++; 3858269356Snp counter_u64_add(extfree_rels, 1); 3859265425Snp goto recycled; 3860218792Snp } 3861265425Snp sd->cl = NULL; /* gave up my reference */ 3862218792Snp } 3863265425Snp MPASS(sd->cl == NULL); 3864265425Snpalloc: 3865265425Snp cl = uma_zalloc(swz->zone, M_NOWAIT); 3866265425Snp if (__predict_false(cl == NULL)) { 3867265425Snp if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 || 3868265425Snp fl->cll_def.zidx == fl->cll_alt.zidx) 3869265425Snp break; 3870218792Snp 3871265425Snp /* fall back to the safe zone */ 3872265425Snp cll = &fl->cll_alt; 3873265425Snp swz = &sc->sge.sw_zone_info[cll->zidx]; 3874265425Snp goto alloc; 3875255050Snp } 3876265425Snp fl->cl_allocated++; 3877270297Snp n--; 3878218792Snp 3879265425Snp pa = pmap_kextract((vm_offset_t)cl); 3880265425Snp pa += cll->region1; 3881218792Snp sd->cl = cl; 3882265425Snp sd->cll = *cll; 3883265425Snp *d = htobe64(pa | cll->hwidx); 3884265425Snp clm = cl_metadata(sc, fl, cll, cl); 3885265425Snp if (clm != NULL) { 3886265425Snprecycled: 3887218792Snp#ifdef INVARIANTS 3888265425Snp clm->sd = sd; 3889218792Snp#endif 3890265425Snp clm->refcount = 1; 3891265425Snp } 3892269356Snp sd->nmbuf = 0; 3893265425Snprecycled_fast: 3894265425Snp d++; 3895218792Snp sd++; 3896270297Snp if (__predict_false(++fl->pidx % 8 == 0)) { 3897270297Snp uint16_t pidx = fl->pidx / 8; 3898270297Snp 3899270297Snp if (__predict_false(pidx == fl->sidx)) { 3900270297Snp fl->pidx = 0; 3901270297Snp pidx = 0; 3902270297Snp sd = fl->sdesc; 3903270297Snp d = fl->desc; 3904270297Snp } 3905270297Snp if (pidx == max_pidx) 3906270297Snp break; 3907270297Snp 3908270297Snp if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4) 3909270297Snp ring_fl_db(sc, fl); 3910218792Snp } 3911218792Snp } 3912220905Snp 3913270297Snp if (fl->pidx / 8 != fl->dbidx) 3914220905Snp ring_fl_db(sc, fl); 3915228561Snp 3916228561Snp return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 3917218792Snp} 3918218792Snp 3919228561Snp/* 3920228561Snp * Attempt to refill all starving freelists. 3921228561Snp */ 3922228561Snpstatic void 3923228561Snprefill_sfl(void *arg) 3924228561Snp{ 3925228561Snp struct adapter *sc = arg; 3926228561Snp struct sge_fl *fl, *fl_temp; 3927228561Snp 3928308154Sjhb mtx_assert(&sc->sfl_lock, MA_OWNED); 3929228561Snp TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 3930228561Snp FL_LOCK(fl); 3931228561Snp refill_fl(sc, fl, 64); 3932228561Snp if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 3933228561Snp TAILQ_REMOVE(&sc->sfl, fl, link); 3934228561Snp fl->flags &= ~FL_STARVING; 3935228561Snp } 3936228561Snp FL_UNLOCK(fl); 3937228561Snp } 3938228561Snp 3939228561Snp if (!TAILQ_EMPTY(&sc->sfl)) 3940228561Snp callout_schedule(&sc->sfl_callout, hz / 5); 3941228561Snp} 3942228561Snp 3943218792Snpstatic int 3944218792Snpalloc_fl_sdesc(struct sge_fl *fl) 3945218792Snp{ 3946218792Snp 3947270297Snp fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE, 3948218792Snp M_ZERO | M_WAITOK); 3949218792Snp 3950218792Snp return (0); 3951218792Snp} 3952218792Snp 3953218792Snpstatic void 3954255050Snpfree_fl_sdesc(struct adapter *sc, struct sge_fl *fl) 3955218792Snp{ 3956218792Snp struct fl_sdesc *sd; 3957265425Snp struct cluster_metadata *clm; 3958265425Snp struct cluster_layout *cll; 3959218792Snp int i; 3960218792Snp 3961218792Snp sd = fl->sdesc; 3962270297Snp for (i = 0; i < fl->sidx * 8; i++, sd++) { 3963265425Snp if (sd->cl == NULL) 3964265425Snp continue; 3965218792Snp 3966265425Snp cll = &sd->cll; 3967265425Snp clm = cl_metadata(sc, fl, cll, sd->cl); 3968269356Snp if (sd->nmbuf == 0) 3969265425Snp uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl); 3970269356Snp else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) { 3971269356Snp uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl); 3972269356Snp counter_u64_add(extfree_rels, 1); 3973218792Snp } 3974265425Snp sd->cl = NULL; 3975218792Snp } 3976218792Snp 3977218792Snp free(fl->sdesc, M_CXGBE); 3978218792Snp fl->sdesc = NULL; 3979218792Snp} 3980218792Snp 3981284052Snpstatic inline void 3982284052Snpget_pkt_gl(struct mbuf *m, struct sglist *gl) 3983218792Snp{ 3984284052Snp int rc; 3985218792Snp 3986284052Snp M_ASSERTPKTHDR(m); 3987218792Snp 3988284052Snp sglist_reset(gl); 3989284052Snp rc = sglist_append_mbuf(gl, m); 3990284052Snp if (__predict_false(rc != 0)) { 3991284052Snp panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " 3992284052Snp "with %d.", __func__, m, mbuf_nsegs(m), rc); 3993218792Snp } 3994218792Snp 3995284052Snp KASSERT(gl->sg_nseg == mbuf_nsegs(m), 3996284052Snp ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, 3997284052Snp mbuf_nsegs(m), gl->sg_nseg)); 3998284052Snp KASSERT(gl->sg_nseg > 0 && 3999284052Snp gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS), 4000284052Snp ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, 4001284052Snp gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)); 4002218792Snp} 4003218792Snp 4004284052Snp/* 4005284052Snp * len16 for a txpkt WR with a GL. Includes the firmware work request header. 4006284052Snp */ 4007284052Snpstatic inline u_int 4008284052Snptxpkt_len16(u_int nsegs, u_int tso) 4009218792Snp{ 4010284052Snp u_int n; 4011218792Snp 4012284052Snp MPASS(nsegs > 0); 4013218792Snp 4014284052Snp nsegs--; /* first segment is part of ulptx_sgl */ 4015284052Snp n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + 4016284052Snp sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4017284052Snp if (tso) 4018284052Snp n += sizeof(struct cpl_tx_pkt_lso_core); 4019218792Snp 4020284052Snp return (howmany(n, 16)); 4021218792Snp} 4022218792Snp 4023218792Snp/* 4024309447Sjhb * len16 for a txpkt_vm WR with a GL. Includes the firmware work 4025309447Sjhb * request header. 4026309447Sjhb */ 4027309447Sjhbstatic inline u_int 4028309447Sjhbtxpkt_vm_len16(u_int nsegs, u_int tso) 4029309447Sjhb{ 4030309447Sjhb u_int n; 4031309447Sjhb 4032309447Sjhb MPASS(nsegs > 0); 4033309447Sjhb 4034309447Sjhb nsegs--; /* first segment is part of ulptx_sgl */ 4035309447Sjhb n = sizeof(struct fw_eth_tx_pkt_vm_wr) + 4036309447Sjhb sizeof(struct cpl_tx_pkt_core) + 4037309447Sjhb sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4038309447Sjhb if (tso) 4039309447Sjhb n += sizeof(struct cpl_tx_pkt_lso_core); 4040309447Sjhb 4041309447Sjhb return (howmany(n, 16)); 4042309447Sjhb} 4043309447Sjhb 4044309447Sjhb/* 4045284052Snp * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work 4046284052Snp * request header. 4047218792Snp */ 4048284052Snpstatic inline u_int 4049284052Snptxpkts0_len16(u_int nsegs) 4050218792Snp{ 4051284052Snp u_int n; 4052218792Snp 4053284052Snp MPASS(nsegs > 0); 4054218792Snp 4055284052Snp nsegs--; /* first segment is part of ulptx_sgl */ 4056284052Snp n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + 4057284052Snp sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 4058284052Snp 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4059218792Snp 4060284052Snp return (howmany(n, 16)); 4061218792Snp} 4062218792Snp 4063218792Snp/* 4064284052Snp * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work 4065284052Snp * request header. 4066218792Snp */ 4067284052Snpstatic inline u_int 4068284052Snptxpkts1_len16(void) 4069218792Snp{ 4070284052Snp u_int n; 4071218792Snp 4072284052Snp n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); 4073218792Snp 4074284052Snp return (howmany(n, 16)); 4075284052Snp} 4076218792Snp 4077284052Snpstatic inline u_int 4078284052Snpimm_payload(u_int ndesc) 4079284052Snp{ 4080284052Snp u_int n; 4081228561Snp 4082284052Snp n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - 4083284052Snp sizeof(struct cpl_tx_pkt_core); 4084218792Snp 4085284052Snp return (n); 4086218792Snp} 4087218792Snp 4088284052Snp/* 4089309447Sjhb * Write a VM txpkt WR for this packet to the hardware descriptors, update the 4090309447Sjhb * software descriptor, and advance the pidx. It is guaranteed that enough 4091309447Sjhb * descriptors are available. 4092309447Sjhb * 4093309447Sjhb * The return value is the # of hardware descriptors used. 4094309447Sjhb */ 4095309447Sjhbstatic u_int 4096309560Sjhbwrite_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, 4097309560Sjhb struct fw_eth_tx_pkt_vm_wr *wr, struct mbuf *m0, u_int available) 4098309447Sjhb{ 4099309447Sjhb struct sge_eq *eq = &txq->eq; 4100309447Sjhb struct tx_sdesc *txsd; 4101309447Sjhb struct cpl_tx_pkt_core *cpl; 4102309447Sjhb uint32_t ctrl; /* used in many unrelated places */ 4103309447Sjhb uint64_t ctrl1; 4104309447Sjhb int csum_type, len16, ndesc, pktlen, nsegs; 4105309447Sjhb caddr_t dst; 4106309447Sjhb 4107309447Sjhb TXQ_LOCK_ASSERT_OWNED(txq); 4108309447Sjhb M_ASSERTPKTHDR(m0); 4109309447Sjhb MPASS(available > 0 && available < eq->sidx); 4110309447Sjhb 4111309447Sjhb len16 = mbuf_len16(m0); 4112309447Sjhb nsegs = mbuf_nsegs(m0); 4113309447Sjhb pktlen = m0->m_pkthdr.len; 4114309447Sjhb ctrl = sizeof(struct cpl_tx_pkt_core); 4115309447Sjhb if (needs_tso(m0)) 4116309447Sjhb ctrl += sizeof(struct cpl_tx_pkt_lso_core); 4117309447Sjhb ndesc = howmany(len16, EQ_ESIZE / 16); 4118309447Sjhb MPASS(ndesc <= available); 4119309447Sjhb 4120309447Sjhb /* Firmware work request header */ 4121309447Sjhb MPASS(wr == (void *)&eq->desc[eq->pidx]); 4122309447Sjhb wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | 4123309447Sjhb V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 4124309447Sjhb 4125309447Sjhb ctrl = V_FW_WR_LEN16(len16); 4126309447Sjhb wr->equiq_to_len16 = htobe32(ctrl); 4127309447Sjhb wr->r3[0] = 0; 4128309447Sjhb wr->r3[1] = 0; 4129309447Sjhb 4130309447Sjhb /* 4131309447Sjhb * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci. 4132309447Sjhb * vlantci is ignored unless the ethtype is 0x8100, so it's 4133309447Sjhb * simpler to always copy it rather than making it 4134309447Sjhb * conditional. Also, it seems that we do not have to set 4135309447Sjhb * vlantci or fake the ethtype when doing VLAN tag insertion. 4136309447Sjhb */ 4137309447Sjhb m_copydata(m0, 0, sizeof(struct ether_header) + 2, wr->ethmacdst); 4138309447Sjhb 4139309447Sjhb csum_type = -1; 4140309447Sjhb if (needs_tso(m0)) { 4141309447Sjhb struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 4142309447Sjhb 4143309447Sjhb KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 4144309447Sjhb m0->m_pkthdr.l4hlen > 0, 4145309447Sjhb ("%s: mbuf %p needs TSO but missing header lengths", 4146309447Sjhb __func__, m0)); 4147309447Sjhb 4148309447Sjhb ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 4149309447Sjhb F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) 4150309447Sjhb | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 4151309447Sjhb if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) 4152309447Sjhb ctrl |= V_LSO_ETHHDR_LEN(1); 4153309447Sjhb if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 4154309447Sjhb ctrl |= F_LSO_IPV6; 4155309447Sjhb 4156309447Sjhb lso->lso_ctrl = htobe32(ctrl); 4157309447Sjhb lso->ipid_ofst = htobe16(0); 4158309447Sjhb lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 4159309447Sjhb lso->seqno_offset = htobe32(0); 4160309447Sjhb lso->len = htobe32(pktlen); 4161309447Sjhb 4162309447Sjhb if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 4163309447Sjhb csum_type = TX_CSUM_TCPIP6; 4164309447Sjhb else 4165309447Sjhb csum_type = TX_CSUM_TCPIP; 4166309447Sjhb 4167309447Sjhb cpl = (void *)(lso + 1); 4168309447Sjhb 4169309447Sjhb txq->tso_wrs++; 4170309447Sjhb } else { 4171309447Sjhb if (m0->m_pkthdr.csum_flags & CSUM_IP_TCP) 4172309447Sjhb csum_type = TX_CSUM_TCPIP; 4173309447Sjhb else if (m0->m_pkthdr.csum_flags & CSUM_IP_UDP) 4174309447Sjhb csum_type = TX_CSUM_UDPIP; 4175309447Sjhb else if (m0->m_pkthdr.csum_flags & CSUM_IP6_TCP) 4176309447Sjhb csum_type = TX_CSUM_TCPIP6; 4177309447Sjhb else if (m0->m_pkthdr.csum_flags & CSUM_IP6_UDP) 4178309447Sjhb csum_type = TX_CSUM_UDPIP6; 4179309447Sjhb#if defined(INET) 4180309447Sjhb else if (m0->m_pkthdr.csum_flags & CSUM_IP) { 4181309447Sjhb /* 4182309447Sjhb * XXX: The firmware appears to stomp on the 4183309447Sjhb * fragment/flags field of the IP header when 4184309447Sjhb * using TX_CSUM_IP. Fall back to doing 4185309447Sjhb * software checksums. 4186309447Sjhb */ 4187309447Sjhb u_short *sump; 4188309447Sjhb struct mbuf *m; 4189309447Sjhb int offset; 4190309447Sjhb 4191309447Sjhb m = m0; 4192309447Sjhb offset = 0; 4193309447Sjhb sump = m_advance(&m, &offset, m0->m_pkthdr.l2hlen + 4194309447Sjhb offsetof(struct ip, ip_sum)); 4195309447Sjhb *sump = in_cksum_skip(m0, m0->m_pkthdr.l2hlen + 4196309447Sjhb m0->m_pkthdr.l3hlen, m0->m_pkthdr.l2hlen); 4197309447Sjhb m0->m_pkthdr.csum_flags &= ~CSUM_IP; 4198309447Sjhb } 4199309447Sjhb#endif 4200309447Sjhb 4201309447Sjhb cpl = (void *)(wr + 1); 4202309447Sjhb } 4203309447Sjhb 4204309447Sjhb /* Checksum offload */ 4205309447Sjhb ctrl1 = 0; 4206309447Sjhb if (needs_l3_csum(m0) == 0) 4207309447Sjhb ctrl1 |= F_TXPKT_IPCSUM_DIS; 4208309447Sjhb if (csum_type >= 0) { 4209309447Sjhb KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0, 4210309447Sjhb ("%s: mbuf %p needs checksum offload but missing header lengths", 4211309447Sjhb __func__, m0)); 4212309447Sjhb 4213309560Sjhb if (chip_id(sc) <= CHELSIO_T5) { 4214309560Sjhb ctrl1 |= V_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen - 4215309560Sjhb ETHER_HDR_LEN); 4216309560Sjhb } else { 4217309560Sjhb ctrl1 |= V_T6_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen - 4218309560Sjhb ETHER_HDR_LEN); 4219309560Sjhb } 4220309447Sjhb ctrl1 |= V_TXPKT_IPHDR_LEN(m0->m_pkthdr.l3hlen); 4221309447Sjhb ctrl1 |= V_TXPKT_CSUM_TYPE(csum_type); 4222309447Sjhb } else 4223309447Sjhb ctrl1 |= F_TXPKT_L4CSUM_DIS; 4224309447Sjhb if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | 4225309447Sjhb CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 4226309447Sjhb txq->txcsum++; /* some hardware assistance provided */ 4227309447Sjhb 4228309447Sjhb /* VLAN tag insertion */ 4229309447Sjhb if (needs_vlan_insertion(m0)) { 4230309447Sjhb ctrl1 |= F_TXPKT_VLAN_VLD | 4231309447Sjhb V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 4232309447Sjhb txq->vlan_insertion++; 4233309447Sjhb } 4234309447Sjhb 4235309447Sjhb /* CPL header */ 4236309447Sjhb cpl->ctrl0 = txq->cpl_ctrl0; 4237309447Sjhb cpl->pack = 0; 4238309447Sjhb cpl->len = htobe16(pktlen); 4239309447Sjhb cpl->ctrl1 = htobe64(ctrl1); 4240309447Sjhb 4241309447Sjhb /* SGL */ 4242309447Sjhb dst = (void *)(cpl + 1); 4243309447Sjhb 4244309447Sjhb /* 4245309447Sjhb * A packet using TSO will use up an entire descriptor for the 4246309447Sjhb * firmware work request header, LSO CPL, and TX_PKT_XT CPL. 4247309447Sjhb * If this descriptor is the last descriptor in the ring, wrap 4248309447Sjhb * around to the front of the ring explicitly for the start of 4249309447Sjhb * the sgl. 4250309447Sjhb */ 4251309447Sjhb if (dst == (void *)&eq->desc[eq->sidx]) { 4252309447Sjhb dst = (void *)&eq->desc[0]; 4253309447Sjhb write_gl_to_txd(txq, m0, &dst, 0); 4254309447Sjhb } else 4255309447Sjhb write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 4256309447Sjhb txq->sgl_wrs++; 4257309447Sjhb 4258309447Sjhb txq->txpkt_wrs++; 4259309447Sjhb 4260309447Sjhb txsd = &txq->sdesc[eq->pidx]; 4261309447Sjhb txsd->m = m0; 4262309447Sjhb txsd->desc_used = ndesc; 4263309447Sjhb 4264309447Sjhb return (ndesc); 4265309447Sjhb} 4266309447Sjhb 4267309447Sjhb/* 4268284052Snp * Write a txpkt WR for this packet to the hardware descriptors, update the 4269284052Snp * software descriptor, and advance the pidx. It is guaranteed that enough 4270284052Snp * descriptors are available. 4271284052Snp * 4272284052Snp * The return value is the # of hardware descriptors used. 4273284052Snp */ 4274284052Snpstatic u_int 4275284052Snpwrite_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr, 4276284052Snp struct mbuf *m0, u_int available) 4277218792Snp{ 4278218792Snp struct sge_eq *eq = &txq->eq; 4279284052Snp struct tx_sdesc *txsd; 4280218792Snp struct cpl_tx_pkt_core *cpl; 4281218792Snp uint32_t ctrl; /* used in many unrelated places */ 4282218792Snp uint64_t ctrl1; 4283284052Snp int len16, ndesc, pktlen, nsegs; 4284218792Snp caddr_t dst; 4285218792Snp 4286218792Snp TXQ_LOCK_ASSERT_OWNED(txq); 4287284052Snp M_ASSERTPKTHDR(m0); 4288284052Snp MPASS(available > 0 && available < eq->sidx); 4289218792Snp 4290284052Snp len16 = mbuf_len16(m0); 4291284052Snp nsegs = mbuf_nsegs(m0); 4292284052Snp pktlen = m0->m_pkthdr.len; 4293218792Snp ctrl = sizeof(struct cpl_tx_pkt_core); 4294284052Snp if (needs_tso(m0)) 4295237436Snp ctrl += sizeof(struct cpl_tx_pkt_lso_core); 4296284052Snp else if (pktlen <= imm_payload(2) && available >= 2) { 4297284052Snp /* Immediate data. Recalculate len16 and set nsegs to 0. */ 4298219286Snp ctrl += pktlen; 4299284052Snp len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + 4300284052Snp sizeof(struct cpl_tx_pkt_core) + pktlen, 16); 4301284052Snp nsegs = 0; 4302218792Snp } 4303284052Snp ndesc = howmany(len16, EQ_ESIZE / 16); 4304284052Snp MPASS(ndesc <= available); 4305218792Snp 4306218792Snp /* Firmware work request header */ 4307284052Snp MPASS(wr == (void *)&eq->desc[eq->pidx]); 4308218792Snp wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 4309228561Snp V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 4310220643Snp 4311284052Snp ctrl = V_FW_WR_LEN16(len16); 4312218792Snp wr->equiq_to_len16 = htobe32(ctrl); 4313218792Snp wr->r3 = 0; 4314218792Snp 4315284052Snp if (needs_tso(m0)) { 4316237436Snp struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 4317218792Snp 4318284052Snp KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 4319284052Snp m0->m_pkthdr.l4hlen > 0, 4320284052Snp ("%s: mbuf %p needs TSO but missing header lengths", 4321284052Snp __func__, m0)); 4322284052Snp 4323218792Snp ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 4324284052Snp F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) 4325284052Snp | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 4326284052Snp if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) 4327218792Snp ctrl |= V_LSO_ETHHDR_LEN(1); 4328284052Snp if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 4329237819Snp ctrl |= F_LSO_IPV6; 4330237819Snp 4331218792Snp lso->lso_ctrl = htobe32(ctrl); 4332218792Snp lso->ipid_ofst = htobe16(0); 4333284052Snp lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 4334218792Snp lso->seqno_offset = htobe32(0); 4335219286Snp lso->len = htobe32(pktlen); 4336218792Snp 4337218792Snp cpl = (void *)(lso + 1); 4338218792Snp 4339218792Snp txq->tso_wrs++; 4340218792Snp } else 4341218792Snp cpl = (void *)(wr + 1); 4342218792Snp 4343218792Snp /* Checksum offload */ 4344218792Snp ctrl1 = 0; 4345284052Snp if (needs_l3_csum(m0) == 0) 4346218792Snp ctrl1 |= F_TXPKT_IPCSUM_DIS; 4347284052Snp if (needs_l4_csum(m0) == 0) 4348218792Snp ctrl1 |= F_TXPKT_L4CSUM_DIS; 4349284052Snp if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | 4350247062Snp CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 4351218792Snp txq->txcsum++; /* some hardware assistance provided */ 4352218792Snp 4353218792Snp /* VLAN tag insertion */ 4354284052Snp if (needs_vlan_insertion(m0)) { 4355284052Snp ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 4356218792Snp txq->vlan_insertion++; 4357218792Snp } 4358218792Snp 4359218792Snp /* CPL header */ 4360284052Snp cpl->ctrl0 = txq->cpl_ctrl0; 4361218792Snp cpl->pack = 0; 4362219286Snp cpl->len = htobe16(pktlen); 4363218792Snp cpl->ctrl1 = htobe64(ctrl1); 4364218792Snp 4365218792Snp /* SGL */ 4366218792Snp dst = (void *)(cpl + 1); 4367284052Snp if (nsegs > 0) { 4368284052Snp 4369284052Snp write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 4370218792Snp txq->sgl_wrs++; 4371218792Snp } else { 4372284052Snp struct mbuf *m; 4373284052Snp 4374284052Snp for (m = m0; m != NULL; m = m->m_next) { 4375218792Snp copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 4376219286Snp#ifdef INVARIANTS 4377219286Snp pktlen -= m->m_len; 4378219286Snp#endif 4379218792Snp } 4380219286Snp#ifdef INVARIANTS 4381219286Snp KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); 4382219286Snp#endif 4383284052Snp txq->imm_wrs++; 4384218792Snp } 4385218792Snp 4386218792Snp txq->txpkt_wrs++; 4387284052Snp 4388284052Snp txsd = &txq->sdesc[eq->pidx]; 4389284052Snp txsd->m = m0; 4390284052Snp txsd->desc_used = ndesc; 4391284052Snp 4392284052Snp return (ndesc); 4393218792Snp} 4394218792Snp 4395218792Snpstatic int 4396284052Snptry_txpkts(struct mbuf *m, struct mbuf *n, struct txpkts *txp, u_int available) 4397218792Snp{ 4398284052Snp u_int needed, nsegs1, nsegs2, l1, l2; 4399218792Snp 4400284052Snp if (cannot_use_txpkts(m) || cannot_use_txpkts(n)) 4401284052Snp return (1); 4402218792Snp 4403284052Snp nsegs1 = mbuf_nsegs(m); 4404284052Snp nsegs2 = mbuf_nsegs(n); 4405284052Snp if (nsegs1 + nsegs2 == 2) { 4406284052Snp txp->wr_type = 1; 4407284052Snp l1 = l2 = txpkts1_len16(); 4408284052Snp } else { 4409284052Snp txp->wr_type = 0; 4410284052Snp l1 = txpkts0_len16(nsegs1); 4411284052Snp l2 = txpkts0_len16(nsegs2); 4412284052Snp } 4413284052Snp txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + l1 + l2; 4414284052Snp needed = howmany(txp->len16, EQ_ESIZE / 16); 4415284052Snp if (needed > SGE_MAX_WR_NDESC || needed > available) 4416284052Snp return (1); 4417228561Snp 4418284052Snp txp->plen = m->m_pkthdr.len + n->m_pkthdr.len; 4419284052Snp if (txp->plen > 65535) 4420284052Snp return (1); 4421218792Snp 4422284052Snp txp->npkt = 2; 4423284052Snp set_mbuf_len16(m, l1); 4424284052Snp set_mbuf_len16(n, l2); 4425218792Snp 4426284052Snp return (0); 4427284052Snp} 4428218792Snp 4429284052Snpstatic int 4430284052Snpadd_to_txpkts(struct mbuf *m, struct txpkts *txp, u_int available) 4431284052Snp{ 4432284052Snp u_int plen, len16, needed, nsegs; 4433218792Snp 4434284052Snp MPASS(txp->wr_type == 0 || txp->wr_type == 1); 4435218792Snp 4436284052Snp nsegs = mbuf_nsegs(m); 4437284052Snp if (needs_tso(m) || (txp->wr_type == 1 && nsegs != 1)) 4438284052Snp return (1); 4439218792Snp 4440284052Snp plen = txp->plen + m->m_pkthdr.len; 4441284052Snp if (plen > 65535) 4442284052Snp return (1); 4443218792Snp 4444284052Snp if (txp->wr_type == 0) 4445284052Snp len16 = txpkts0_len16(nsegs); 4446284052Snp else 4447284052Snp len16 = txpkts1_len16(); 4448284052Snp needed = howmany(txp->len16 + len16, EQ_ESIZE / 16); 4449284052Snp if (needed > SGE_MAX_WR_NDESC || needed > available) 4450284052Snp return (1); 4451218792Snp 4452284052Snp txp->npkt++; 4453284052Snp txp->plen = plen; 4454284052Snp txp->len16 += len16; 4455284052Snp set_mbuf_len16(m, len16); 4456218792Snp 4457218792Snp return (0); 4458218792Snp} 4459218792Snp 4460218792Snp/* 4461284052Snp * Write a txpkts WR for the packets in txp to the hardware descriptors, update 4462284052Snp * the software descriptor, and advance the pidx. It is guaranteed that enough 4463284052Snp * descriptors are available. 4464284052Snp * 4465284052Snp * The return value is the # of hardware descriptors used. 4466218792Snp */ 4467284052Snpstatic u_int 4468284052Snpwrite_txpkts_wr(struct sge_txq *txq, struct fw_eth_tx_pkts_wr *wr, 4469284052Snp struct mbuf *m0, const struct txpkts *txp, u_int available) 4470218792Snp{ 4471218792Snp struct sge_eq *eq = &txq->eq; 4472218792Snp struct tx_sdesc *txsd; 4473284052Snp struct cpl_tx_pkt_core *cpl; 4474218792Snp uint32_t ctrl; 4475284052Snp uint64_t ctrl1; 4476284052Snp int ndesc, checkwrap; 4477284052Snp struct mbuf *m; 4478284052Snp void *flitp; 4479218792Snp 4480218792Snp TXQ_LOCK_ASSERT_OWNED(txq); 4481284052Snp MPASS(txp->npkt > 0); 4482284052Snp MPASS(txp->plen < 65536); 4483284052Snp MPASS(m0 != NULL); 4484284052Snp MPASS(m0->m_nextpkt != NULL); 4485284052Snp MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 4486284052Snp MPASS(available > 0 && available < eq->sidx); 4487218792Snp 4488284052Snp ndesc = howmany(txp->len16, EQ_ESIZE / 16); 4489284052Snp MPASS(ndesc <= available); 4490218792Snp 4491284052Snp MPASS(wr == (void *)&eq->desc[eq->pidx]); 4492228561Snp wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); 4493284052Snp ctrl = V_FW_WR_LEN16(txp->len16); 4494218792Snp wr->equiq_to_len16 = htobe32(ctrl); 4495284052Snp wr->plen = htobe16(txp->plen); 4496284052Snp wr->npkt = txp->npkt; 4497284052Snp wr->r3 = 0; 4498284052Snp wr->type = txp->wr_type; 4499284052Snp flitp = wr + 1; 4500218792Snp 4501284052Snp /* 4502284052Snp * At this point we are 16B into a hardware descriptor. If checkwrap is 4503284052Snp * set then we know the WR is going to wrap around somewhere. We'll 4504284052Snp * check for that at appropriate points. 4505284052Snp */ 4506284052Snp checkwrap = eq->sidx - ndesc < eq->pidx; 4507284052Snp for (m = m0; m != NULL; m = m->m_nextpkt) { 4508284052Snp if (txp->wr_type == 0) { 4509284052Snp struct ulp_txpkt *ulpmc; 4510284052Snp struct ulptx_idata *ulpsc; 4511218792Snp 4512284052Snp /* ULP master command */ 4513284052Snp ulpmc = flitp; 4514284052Snp ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 4515284052Snp V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); 4516284052Snp ulpmc->len = htobe32(mbuf_len16(m)); 4517218792Snp 4518284052Snp /* ULP subcommand */ 4519284052Snp ulpsc = (void *)(ulpmc + 1); 4520284052Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 4521284052Snp F_ULP_TX_SC_MORE); 4522284052Snp ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); 4523218792Snp 4524284052Snp cpl = (void *)(ulpsc + 1); 4525284052Snp if (checkwrap && 4526284052Snp (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) 4527284052Snp cpl = (void *)&eq->desc[0]; 4528284052Snp txq->txpkts0_pkts += txp->npkt; 4529284052Snp txq->txpkts0_wrs++; 4530284052Snp } else { 4531284052Snp cpl = flitp; 4532284052Snp txq->txpkts1_pkts += txp->npkt; 4533284052Snp txq->txpkts1_wrs++; 4534284052Snp } 4535218792Snp 4536284052Snp /* Checksum offload */ 4537284052Snp ctrl1 = 0; 4538284052Snp if (needs_l3_csum(m) == 0) 4539284052Snp ctrl1 |= F_TXPKT_IPCSUM_DIS; 4540284052Snp if (needs_l4_csum(m) == 0) 4541284052Snp ctrl1 |= F_TXPKT_L4CSUM_DIS; 4542284052Snp if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | 4543284052Snp CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 4544284052Snp txq->txcsum++; /* some hardware assistance provided */ 4545218792Snp 4546284052Snp /* VLAN tag insertion */ 4547284052Snp if (needs_vlan_insertion(m)) { 4548284052Snp ctrl1 |= F_TXPKT_VLAN_VLD | 4549284052Snp V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 4550284052Snp txq->vlan_insertion++; 4551284052Snp } 4552218792Snp 4553284052Snp /* CPL header */ 4554284052Snp cpl->ctrl0 = txq->cpl_ctrl0; 4555284052Snp cpl->pack = 0; 4556284052Snp cpl->len = htobe16(m->m_pkthdr.len); 4557284052Snp cpl->ctrl1 = htobe64(ctrl1); 4558218792Snp 4559284052Snp flitp = cpl + 1; 4560284052Snp if (checkwrap && 4561284052Snp (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 4562284052Snp flitp = (void *)&eq->desc[0]; 4563218792Snp 4564284052Snp write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); 4565218792Snp 4566218792Snp } 4567218792Snp 4568284052Snp txsd = &txq->sdesc[eq->pidx]; 4569284052Snp txsd->m = m0; 4570284052Snp txsd->desc_used = ndesc; 4571218792Snp 4572284052Snp return (ndesc); 4573218792Snp} 4574218792Snp 4575218792Snp/* 4576218792Snp * If the SGL ends on an address that is not 16 byte aligned, this function will 4577284052Snp * add a 0 filled flit at the end. 4578218792Snp */ 4579284052Snpstatic void 4580284052Snpwrite_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) 4581218792Snp{ 4582284052Snp struct sge_eq *eq = &txq->eq; 4583284052Snp struct sglist *gl = txq->gl; 4584284052Snp struct sglist_seg *seg; 4585284052Snp __be64 *flitp, *wrap; 4586218792Snp struct ulptx_sgl *usgl; 4587284052Snp int i, nflits, nsegs; 4588218792Snp 4589218792Snp KASSERT(((uintptr_t)(*to) & 0xf) == 0, 4590218792Snp ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); 4591284052Snp MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 4592284052Snp MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 4593218792Snp 4594284052Snp get_pkt_gl(m, gl); 4595284052Snp nsegs = gl->sg_nseg; 4596284052Snp MPASS(nsegs > 0); 4597284052Snp 4598284052Snp nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; 4599218792Snp flitp = (__be64 *)(*to); 4600284052Snp wrap = (__be64 *)(&eq->desc[eq->sidx]); 4601284052Snp seg = &gl->sg_segs[0]; 4602218792Snp usgl = (void *)flitp; 4603218792Snp 4604218792Snp /* 4605218792Snp * We start at a 16 byte boundary somewhere inside the tx descriptor 4606218792Snp * ring, so we're at least 16 bytes away from the status page. There is 4607218792Snp * no chance of a wrap around in the middle of usgl (which is 16 bytes). 4608218792Snp */ 4609218792Snp 4610218792Snp usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 4611284052Snp V_ULPTX_NSGE(nsegs)); 4612284052Snp usgl->len0 = htobe32(seg->ss_len); 4613284052Snp usgl->addr0 = htobe64(seg->ss_paddr); 4614218792Snp seg++; 4615218792Snp 4616284052Snp if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { 4617218792Snp 4618218792Snp /* Won't wrap around at all */ 4619218792Snp 4620284052Snp for (i = 0; i < nsegs - 1; i++, seg++) { 4621284052Snp usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); 4622284052Snp usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); 4623218792Snp } 4624218792Snp if (i & 1) 4625218792Snp usgl->sge[i / 2].len[1] = htobe32(0); 4626284052Snp flitp += nflits; 4627218792Snp } else { 4628218792Snp 4629218792Snp /* Will wrap somewhere in the rest of the SGL */ 4630218792Snp 4631218792Snp /* 2 flits already written, write the rest flit by flit */ 4632218792Snp flitp = (void *)(usgl + 1); 4633284052Snp for (i = 0; i < nflits - 2; i++) { 4634284052Snp if (flitp == wrap) 4635218792Snp flitp = (void *)eq->desc; 4636284052Snp *flitp++ = get_flit(seg, nsegs - 1, i); 4637218792Snp } 4638218792Snp } 4639218792Snp 4640284052Snp if (nflits & 1) { 4641284052Snp MPASS(((uintptr_t)flitp) & 0xf); 4642284052Snp *flitp++ = 0; 4643284052Snp } 4644218792Snp 4645284052Snp MPASS((((uintptr_t)flitp) & 0xf) == 0); 4646284052Snp if (__predict_false(flitp == wrap)) 4647218792Snp *to = (void *)eq->desc; 4648218792Snp else 4649284052Snp *to = (void *)flitp; 4650218792Snp} 4651218792Snp 4652218792Snpstatic inline void 4653218792Snpcopy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 4654218792Snp{ 4655284052Snp 4656284052Snp MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 4657284052Snp MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 4658284052Snp 4659284052Snp if (__predict_true((uintptr_t)(*to) + len <= 4660284052Snp (uintptr_t)&eq->desc[eq->sidx])) { 4661218792Snp bcopy(from, *to, len); 4662218792Snp (*to) += len; 4663218792Snp } else { 4664284052Snp int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); 4665218792Snp 4666218792Snp bcopy(from, *to, portion); 4667218792Snp from += portion; 4668218792Snp portion = len - portion; /* remaining */ 4669218792Snp bcopy(from, (void *)eq->desc, portion); 4670218792Snp (*to) = (caddr_t)eq->desc + portion; 4671218792Snp } 4672218792Snp} 4673218792Snp 4674218792Snpstatic inline void 4675284052Snpring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) 4676218792Snp{ 4677284052Snp u_int db; 4678248925Snp 4679284052Snp MPASS(n > 0); 4680284052Snp 4681248925Snp db = eq->doorbells; 4682284052Snp if (n > 1) 4683249392Snp clrbit(&db, DOORBELL_WCWR); 4684218792Snp wmb(); 4685248925Snp 4686248925Snp switch (ffs(db) - 1) { 4687248925Snp case DOORBELL_UDB: 4688284052Snp *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 4689284052Snp break; 4690248925Snp 4691249392Snp case DOORBELL_WCWR: { 4692248925Snp volatile uint64_t *dst, *src; 4693248925Snp int i; 4694248925Snp 4695248925Snp /* 4696248925Snp * Queues whose 128B doorbell segment fits in the page do not 4697248925Snp * use relative qid (udb_qid is always 0). Only queues with 4698249392Snp * doorbell segments can do WCWR. 4699248925Snp */ 4700284052Snp KASSERT(eq->udb_qid == 0 && n == 1, 4701248925Snp ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", 4702284052Snp __func__, eq->doorbells, n, eq->dbidx, eq)); 4703248925Snp 4704248925Snp dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - 4705248925Snp UDBS_DB_OFFSET); 4706284052Snp i = eq->dbidx; 4707248925Snp src = (void *)&eq->desc[i]; 4708248925Snp while (src != (void *)&eq->desc[i + 1]) 4709248925Snp *dst++ = *src++; 4710248925Snp wmb(); 4711284052Snp break; 4712248925Snp } 4713248925Snp 4714248925Snp case DOORBELL_UDBWC: 4715284052Snp *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 4716248925Snp wmb(); 4717284052Snp break; 4718248925Snp 4719248925Snp case DOORBELL_KDB: 4720309447Sjhb t4_write_reg(sc, sc->sge_kdoorbell_reg, 4721284052Snp V_QID(eq->cntxt_id) | V_PIDX(n)); 4722284052Snp break; 4723248925Snp } 4724284052Snp 4725284052Snp IDXINCR(eq->dbidx, n, eq->sidx); 4726218792Snp} 4727218792Snp 4728284052Snpstatic inline u_int 4729284052Snpreclaimable_tx_desc(struct sge_eq *eq) 4730218792Snp{ 4731284052Snp uint16_t hw_cidx; 4732218792Snp 4733284052Snp hw_cidx = read_hw_cidx(eq); 4734284052Snp return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); 4735284052Snp} 4736218792Snp 4737284052Snpstatic inline u_int 4738284052Snptotal_available_tx_desc(struct sge_eq *eq) 4739284052Snp{ 4740284052Snp uint16_t hw_cidx, pidx; 4741284052Snp 4742284052Snp hw_cidx = read_hw_cidx(eq); 4743284052Snp pidx = eq->pidx; 4744284052Snp 4745284052Snp if (pidx == hw_cidx) 4746284052Snp return (eq->sidx - 1); 4747218792Snp else 4748284052Snp return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); 4749219292Snp} 4750218792Snp 4751284052Snpstatic inline uint16_t 4752284052Snpread_hw_cidx(struct sge_eq *eq) 4753284052Snp{ 4754284052Snp struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4755284052Snp uint16_t cidx = spg->cidx; /* stable snapshot */ 4756284052Snp 4757284052Snp return (be16toh(cidx)); 4758284052Snp} 4759284052Snp 4760219292Snp/* 4761284052Snp * Reclaim 'n' descriptors approximately. 4762219292Snp */ 4763284052Snpstatic u_int 4764284052Snpreclaim_tx_descs(struct sge_txq *txq, u_int n) 4765219292Snp{ 4766219292Snp struct tx_sdesc *txsd; 4767220873Snp struct sge_eq *eq = &txq->eq; 4768284052Snp u_int can_reclaim, reclaimed; 4769218792Snp 4770228561Snp TXQ_LOCK_ASSERT_OWNED(txq); 4771284052Snp MPASS(n > 0); 4772218792Snp 4773284052Snp reclaimed = 0; 4774284052Snp can_reclaim = reclaimable_tx_desc(eq); 4775284052Snp while (can_reclaim && reclaimed < n) { 4776218792Snp int ndesc; 4777284052Snp struct mbuf *m, *nextpkt; 4778218792Snp 4779220873Snp txsd = &txq->sdesc[eq->cidx]; 4780218792Snp ndesc = txsd->desc_used; 4781218792Snp 4782218792Snp /* Firmware doesn't return "partial" credits. */ 4783218792Snp KASSERT(can_reclaim >= ndesc, 4784218792Snp ("%s: unexpected number of credits: %d, %d", 4785218792Snp __func__, can_reclaim, ndesc)); 4786218792Snp 4787284052Snp for (m = txsd->m; m != NULL; m = nextpkt) { 4788284052Snp nextpkt = m->m_nextpkt; 4789284052Snp m->m_nextpkt = NULL; 4790284052Snp m_freem(m); 4791284052Snp } 4792218792Snp reclaimed += ndesc; 4793219292Snp can_reclaim -= ndesc; 4794284052Snp IDXINCR(eq->cidx, ndesc, eq->sidx); 4795219292Snp } 4796218792Snp 4797218792Snp return (reclaimed); 4798218792Snp} 4799218792Snp 4800218792Snpstatic void 4801284052Snptx_reclaim(void *arg, int n) 4802218792Snp{ 4803284052Snp struct sge_txq *txq = arg; 4804284052Snp struct sge_eq *eq = &txq->eq; 4805218792Snp 4806284052Snp do { 4807284052Snp if (TXQ_TRYLOCK(txq) == 0) 4808284052Snp break; 4809284052Snp n = reclaim_tx_descs(txq, 32); 4810284052Snp if (eq->cidx == eq->pidx) 4811284052Snp eq->equeqidx = eq->pidx; 4812284052Snp TXQ_UNLOCK(txq); 4813284052Snp } while (n > 0); 4814218792Snp} 4815218792Snp 4816218792Snpstatic __be64 4817284052Snpget_flit(struct sglist_seg *segs, int nsegs, int idx) 4818218792Snp{ 4819218792Snp int i = (idx / 3) * 2; 4820218792Snp 4821218792Snp switch (idx % 3) { 4822218792Snp case 0: { 4823218792Snp __be64 rc; 4824218792Snp 4825284052Snp rc = htobe32(segs[i].ss_len); 4826218792Snp if (i + 1 < nsegs) 4827284052Snp rc |= (uint64_t)htobe32(segs[i + 1].ss_len) << 32; 4828218792Snp 4829218792Snp return (rc); 4830218792Snp } 4831218792Snp case 1: 4832284052Snp return (htobe64(segs[i].ss_paddr)); 4833218792Snp case 2: 4834284052Snp return (htobe64(segs[i + 1].ss_paddr)); 4835218792Snp } 4836218792Snp 4837218792Snp return (0); 4838218792Snp} 4839218792Snp 4840218792Snpstatic void 4841265425Snpfind_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp) 4842218792Snp{ 4843265425Snp int8_t zidx, hwidx, idx; 4844265425Snp uint16_t region1, region3; 4845265425Snp int spare, spare_needed, n; 4846265425Snp struct sw_zone_info *swz; 4847265425Snp struct hw_buf_info *hwb, *hwb_list = &sc->sge.hw_buf_info[0]; 4848218792Snp 4849265425Snp /* 4850265425Snp * Buffer Packing: Look for PAGE_SIZE or larger zone which has a bufsize 4851265425Snp * large enough for the max payload and cluster metadata. Otherwise 4852265425Snp * settle for the largest bufsize that leaves enough room in the cluster 4853265425Snp * for metadata. 4854265425Snp * 4855265425Snp * Without buffer packing: Look for the smallest zone which has a 4856265425Snp * bufsize large enough for the max payload. Settle for the largest 4857265425Snp * bufsize available if there's nothing big enough for max payload. 4858265425Snp */ 4859265425Snp spare_needed = fl->flags & FL_BUF_PACKING ? CL_METADATA_SIZE : 0; 4860265425Snp swz = &sc->sge.sw_zone_info[0]; 4861265425Snp hwidx = -1; 4862265425Snp for (zidx = 0; zidx < SW_ZONE_SIZES; zidx++, swz++) { 4863265425Snp if (swz->size > largest_rx_cluster) { 4864265425Snp if (__predict_true(hwidx != -1)) 4865265425Snp break; 4866218792Snp 4867265425Snp /* 4868265425Snp * This is a misconfiguration. largest_rx_cluster is 4869265425Snp * preventing us from finding a refill source. See 4870265425Snp * dev.t5nex.<n>.buffer_sizes to figure out why. 4871265425Snp */ 4872265425Snp device_printf(sc->dev, "largest_rx_cluster=%u leaves no" 4873265425Snp " refill source for fl %p (dma %u). Ignored.\n", 4874265425Snp largest_rx_cluster, fl, maxp); 4875265425Snp } 4876265425Snp for (idx = swz->head_hwidx; idx != -1; idx = hwb->next) { 4877265425Snp hwb = &hwb_list[idx]; 4878265425Snp spare = swz->size - hwb->size; 4879265425Snp if (spare < spare_needed) 4880265425Snp continue; 4881265425Snp 4882265425Snp hwidx = idx; /* best option so far */ 4883265425Snp if (hwb->size >= maxp) { 4884265425Snp 4885265425Snp if ((fl->flags & FL_BUF_PACKING) == 0) 4886265425Snp goto done; /* stop looking (not packing) */ 4887265425Snp 4888265425Snp if (swz->size >= safest_rx_cluster) 4889265425Snp goto done; /* stop looking (packing) */ 4890265425Snp } 4891265425Snp break; /* keep looking, next zone */ 4892265425Snp } 4893255050Snp } 4894265425Snpdone: 4895265425Snp /* A usable hwidx has been located. */ 4896265425Snp MPASS(hwidx != -1); 4897265425Snp hwb = &hwb_list[hwidx]; 4898265425Snp zidx = hwb->zidx; 4899265425Snp swz = &sc->sge.sw_zone_info[zidx]; 4900265425Snp region1 = 0; 4901265425Snp region3 = swz->size - hwb->size; 4902255050Snp 4903265425Snp /* 4904265425Snp * Stay within this zone and see if there is a better match when mbuf 4905265425Snp * inlining is allowed. Remember that the hwidx's are sorted in 4906265425Snp * decreasing order of size (so in increasing order of spare area). 4907265425Snp */ 4908265425Snp for (idx = hwidx; idx != -1; idx = hwb->next) { 4909265425Snp hwb = &hwb_list[idx]; 4910265425Snp spare = swz->size - hwb->size; 4911255050Snp 4912265425Snp if (allow_mbufs_in_cluster == 0 || hwb->size < maxp) 4913265425Snp break; 4914281212Snp 4915281212Snp /* 4916281212Snp * Do not inline mbufs if doing so would violate the pad/pack 4917281212Snp * boundary alignment requirement. 4918281212Snp */ 4919308304Sjhb if (fl_pad && (MSIZE % sc->params.sge.pad_boundary) != 0) 4920281212Snp continue; 4921281212Snp if (fl->flags & FL_BUF_PACKING && 4922308304Sjhb (MSIZE % sc->params.sge.pack_boundary) != 0) 4923281212Snp continue; 4924281212Snp 4925265425Snp if (spare < CL_METADATA_SIZE + MSIZE) 4926265425Snp continue; 4927265425Snp n = (spare - CL_METADATA_SIZE) / MSIZE; 4928265425Snp if (n > howmany(hwb->size, maxp)) 4929265425Snp break; 4930255050Snp 4931265425Snp hwidx = idx; 4932265425Snp if (fl->flags & FL_BUF_PACKING) { 4933265425Snp region1 = n * MSIZE; 4934265425Snp region3 = spare - region1; 4935265425Snp } else { 4936265425Snp region1 = MSIZE; 4937265425Snp region3 = spare - region1; 4938265425Snp break; 4939255050Snp } 4940255050Snp } 4941255050Snp 4942265425Snp KASSERT(zidx >= 0 && zidx < SW_ZONE_SIZES, 4943265425Snp ("%s: bad zone %d for fl %p, maxp %d", __func__, zidx, fl, maxp)); 4944265425Snp KASSERT(hwidx >= 0 && hwidx <= SGE_FLBUF_SIZES, 4945265425Snp ("%s: bad hwidx %d for fl %p, maxp %d", __func__, hwidx, fl, maxp)); 4946265425Snp KASSERT(region1 + sc->sge.hw_buf_info[hwidx].size + region3 == 4947265425Snp sc->sge.sw_zone_info[zidx].size, 4948265425Snp ("%s: bad buffer layout for fl %p, maxp %d. " 4949265425Snp "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, 4950265425Snp sc->sge.sw_zone_info[zidx].size, region1, 4951265425Snp sc->sge.hw_buf_info[hwidx].size, region3)); 4952265425Snp if (fl->flags & FL_BUF_PACKING || region1 > 0) { 4953265425Snp KASSERT(region3 >= CL_METADATA_SIZE, 4954265425Snp ("%s: no room for metadata. fl %p, maxp %d; " 4955265425Snp "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, 4956265425Snp sc->sge.sw_zone_info[zidx].size, region1, 4957265425Snp sc->sge.hw_buf_info[hwidx].size, region3)); 4958265425Snp KASSERT(region1 % MSIZE == 0, 4959265425Snp ("%s: bad mbuf region for fl %p, maxp %d. " 4960265425Snp "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp, 4961265425Snp sc->sge.sw_zone_info[zidx].size, region1, 4962265425Snp sc->sge.hw_buf_info[hwidx].size, region3)); 4963265425Snp } 4964265425Snp 4965265425Snp fl->cll_def.zidx = zidx; 4966265425Snp fl->cll_def.hwidx = hwidx; 4967265425Snp fl->cll_def.region1 = region1; 4968265425Snp fl->cll_def.region3 = region3; 4969265425Snp} 4970265425Snp 4971265425Snpstatic void 4972265425Snpfind_safe_refill_source(struct adapter *sc, struct sge_fl *fl) 4973265425Snp{ 4974265425Snp struct sge *s = &sc->sge; 4975265425Snp struct hw_buf_info *hwb; 4976265425Snp struct sw_zone_info *swz; 4977265425Snp int spare; 4978265425Snp int8_t hwidx; 4979265425Snp 4980265425Snp if (fl->flags & FL_BUF_PACKING) 4981265425Snp hwidx = s->safe_hwidx2; /* with room for metadata */ 4982265425Snp else if (allow_mbufs_in_cluster && s->safe_hwidx2 != -1) { 4983265425Snp hwidx = s->safe_hwidx2; 4984265425Snp hwb = &s->hw_buf_info[hwidx]; 4985265425Snp swz = &s->sw_zone_info[hwb->zidx]; 4986265425Snp spare = swz->size - hwb->size; 4987265425Snp 4988265425Snp /* no good if there isn't room for an mbuf as well */ 4989265425Snp if (spare < CL_METADATA_SIZE + MSIZE) 4990265425Snp hwidx = s->safe_hwidx1; 4991265425Snp } else 4992265425Snp hwidx = s->safe_hwidx1; 4993265425Snp 4994265425Snp if (hwidx == -1) { 4995265425Snp /* No fallback source */ 4996265425Snp fl->cll_alt.hwidx = -1; 4997265425Snp fl->cll_alt.zidx = -1; 4998265425Snp 4999265425Snp return; 5000265425Snp } 5001265425Snp 5002265425Snp hwb = &s->hw_buf_info[hwidx]; 5003265425Snp swz = &s->sw_zone_info[hwb->zidx]; 5004265425Snp spare = swz->size - hwb->size; 5005265425Snp fl->cll_alt.hwidx = hwidx; 5006265425Snp fl->cll_alt.zidx = hwb->zidx; 5007281212Snp if (allow_mbufs_in_cluster && 5008308304Sjhb (fl_pad == 0 || (MSIZE % sc->params.sge.pad_boundary) == 0)) 5009265425Snp fl->cll_alt.region1 = ((spare - CL_METADATA_SIZE) / MSIZE) * MSIZE; 5010255050Snp else 5011265425Snp fl->cll_alt.region1 = 0; 5012265425Snp fl->cll_alt.region3 = spare - fl->cll_alt.region1; 5013218792Snp} 5014219286Snp 5015222510Snpstatic void 5016228561Snpadd_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 5017222510Snp{ 5018228561Snp mtx_lock(&sc->sfl_lock); 5019228561Snp FL_LOCK(fl); 5020228561Snp if ((fl->flags & FL_DOOMED) == 0) { 5021228561Snp fl->flags |= FL_STARVING; 5022228561Snp TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 5023228561Snp callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); 5024222510Snp } 5025228561Snp FL_UNLOCK(fl); 5026228561Snp mtx_unlock(&sc->sfl_lock); 5027222510Snp} 5028222510Snp 5029284052Snpstatic void 5030284052Snphandle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) 5031284052Snp{ 5032284052Snp struct sge_wrq *wrq = (void *)eq; 5033284052Snp 5034284052Snp atomic_readandclear_int(&eq->equiq); 5035284052Snp taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); 5036284052Snp} 5037284052Snp 5038284052Snpstatic void 5039284052Snphandle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) 5040284052Snp{ 5041284052Snp struct sge_txq *txq = (void *)eq; 5042284052Snp 5043284052Snp MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH); 5044284052Snp 5045284052Snp atomic_readandclear_int(&eq->equiq); 5046284052Snp mp_ring_check_drainage(txq->r, 0); 5047284052Snp taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); 5048284052Snp} 5049284052Snp 5050220873Snpstatic int 5051228561Snphandle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 5052228561Snp struct mbuf *m) 5053220873Snp{ 5054228561Snp const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 5055228561Snp unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 5056228561Snp struct adapter *sc = iq->adapter; 5057228561Snp struct sge *s = &sc->sge; 5058228561Snp struct sge_eq *eq; 5059284052Snp static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, 5060284052Snp &handle_wrq_egr_update, &handle_eth_egr_update, 5061284052Snp &handle_wrq_egr_update}; 5062220873Snp 5063228561Snp KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5064228561Snp rss->opcode)); 5065220873Snp 5066309447Sjhb eq = s->eqmap[qid - s->eq_start - s->eq_base]; 5067284052Snp (*h[eq->flags & EQ_TYPEMASK])(sc, eq); 5068220873Snp 5069228561Snp return (0); 5070228561Snp} 5071220873Snp 5072247291Snp/* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ 5073247291SnpCTASSERT(offsetof(struct cpl_fw4_msg, data) == \ 5074247291Snp offsetof(struct cpl_fw6_msg, data)); 5075247291Snp 5076228561Snpstatic int 5077239336Snphandle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5078228561Snp{ 5079239336Snp struct adapter *sc = iq->adapter; 5080228561Snp const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 5081220873Snp 5082228561Snp KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5083228561Snp rss->opcode)); 5084220873Snp 5085247291Snp if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 5086247291Snp const struct rss_header *rss2; 5087247291Snp 5088247291Snp rss2 = (const struct rss_header *)&cpl->data[0]; 5089309442Sjhb return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); 5090247291Snp } 5091247291Snp 5092309442Sjhb return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0])); 5093220873Snp} 5094221911Snp 5095309444Sjhb/** 5096309444Sjhb * t4_handle_wrerr_rpl - process a FW work request error message 5097309444Sjhb * @adap: the adapter 5098309444Sjhb * @rpl: start of the FW message 5099309444Sjhb */ 5100221911Snpstatic int 5101309444Sjhbt4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl) 5102309444Sjhb{ 5103309444Sjhb u8 opcode = *(const u8 *)rpl; 5104309444Sjhb const struct fw_error_cmd *e = (const void *)rpl; 5105309444Sjhb unsigned int i; 5106309444Sjhb 5107309444Sjhb if (opcode != FW_ERROR_CMD) { 5108309444Sjhb log(LOG_ERR, 5109309444Sjhb "%s: Received WRERR_RPL message with opcode %#x\n", 5110309444Sjhb device_get_nameunit(adap->dev), opcode); 5111309444Sjhb return (EINVAL); 5112309444Sjhb } 5113309444Sjhb log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev), 5114309444Sjhb G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" : 5115309444Sjhb "non-fatal"); 5116309444Sjhb switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) { 5117309444Sjhb case FW_ERROR_TYPE_EXCEPTION: 5118309444Sjhb log(LOG_ERR, "exception info:\n"); 5119309444Sjhb for (i = 0; i < nitems(e->u.exception.info); i++) 5120309444Sjhb log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ", 5121309444Sjhb be32toh(e->u.exception.info[i])); 5122309444Sjhb log(LOG_ERR, "\n"); 5123309444Sjhb break; 5124309444Sjhb case FW_ERROR_TYPE_HWMODULE: 5125309444Sjhb log(LOG_ERR, "HW module regaddr %08x regval %08x\n", 5126309444Sjhb be32toh(e->u.hwmodule.regaddr), 5127309444Sjhb be32toh(e->u.hwmodule.regval)); 5128309444Sjhb break; 5129309444Sjhb case FW_ERROR_TYPE_WR: 5130309444Sjhb log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n", 5131309444Sjhb be16toh(e->u.wr.cidx), 5132309444Sjhb G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)), 5133309444Sjhb G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)), 5134309444Sjhb be32toh(e->u.wr.eqid)); 5135309444Sjhb for (i = 0; i < nitems(e->u.wr.wrhdr); i++) 5136309444Sjhb log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ", 5137309444Sjhb e->u.wr.wrhdr[i]); 5138309444Sjhb log(LOG_ERR, "\n"); 5139309444Sjhb break; 5140309444Sjhb case FW_ERROR_TYPE_ACL: 5141309444Sjhb log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s", 5142309444Sjhb be16toh(e->u.acl.cidx), 5143309444Sjhb G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)), 5144309444Sjhb G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)), 5145309444Sjhb be32toh(e->u.acl.eqid), 5146309444Sjhb G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" : 5147309444Sjhb "MAC"); 5148309444Sjhb for (i = 0; i < nitems(e->u.acl.val); i++) 5149309444Sjhb log(LOG_ERR, " %02x", e->u.acl.val[i]); 5150309444Sjhb log(LOG_ERR, "\n"); 5151309444Sjhb break; 5152309444Sjhb default: 5153309444Sjhb log(LOG_ERR, "type %#x\n", 5154309444Sjhb G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))); 5155309444Sjhb return (EINVAL); 5156309444Sjhb } 5157309444Sjhb return (0); 5158309444Sjhb} 5159309444Sjhb 5160309444Sjhbstatic int 5161222510Snpsysctl_uint16(SYSCTL_HANDLER_ARGS) 5162221911Snp{ 5163221911Snp uint16_t *id = arg1; 5164221911Snp int i = *id; 5165221911Snp 5166221911Snp return sysctl_handle_int(oidp, &i, 0, req); 5167221911Snp} 5168265425Snp 5169265425Snpstatic int 5170265425Snpsysctl_bufsizes(SYSCTL_HANDLER_ARGS) 5171265425Snp{ 5172265425Snp struct sge *s = arg1; 5173265425Snp struct hw_buf_info *hwb = &s->hw_buf_info[0]; 5174265425Snp struct sw_zone_info *swz = &s->sw_zone_info[0]; 5175265425Snp int i, rc; 5176265425Snp struct sbuf sb; 5177265425Snp char c; 5178265425Snp 5179265425Snp sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5180265425Snp for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) { 5181265425Snp if (hwb->zidx >= 0 && swz[hwb->zidx].size <= largest_rx_cluster) 5182265425Snp c = '*'; 5183265425Snp else 5184265425Snp c = '\0'; 5185265425Snp 5186265425Snp sbuf_printf(&sb, "%u%c ", hwb->size, c); 5187265425Snp } 5188265425Snp sbuf_trim(&sb); 5189265425Snp sbuf_finish(&sb); 5190265425Snp rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5191265425Snp sbuf_delete(&sb); 5192265425Snp return (rc); 5193265425Snp} 5194308321Sjhb 5195308321Sjhbstatic int 5196308321Sjhbsysctl_tc(SYSCTL_HANDLER_ARGS) 5197308321Sjhb{ 5198308321Sjhb struct vi_info *vi = arg1; 5199308321Sjhb struct port_info *pi; 5200308321Sjhb struct adapter *sc; 5201308321Sjhb struct sge_txq *txq; 5202318851Snp struct tx_cl_rl_params *tc; 5203308321Sjhb int qidx = arg2, rc, tc_idx; 5204308321Sjhb uint32_t fw_queue, fw_class; 5205308321Sjhb 5206308321Sjhb MPASS(qidx >= 0 && qidx < vi->ntxq); 5207308321Sjhb pi = vi->pi; 5208308321Sjhb sc = pi->adapter; 5209308321Sjhb txq = &sc->sge.txq[vi->first_txq + qidx]; 5210308321Sjhb 5211308321Sjhb tc_idx = txq->tc_idx; 5212308321Sjhb rc = sysctl_handle_int(oidp, &tc_idx, 0, req); 5213308321Sjhb if (rc != 0 || req->newptr == NULL) 5214308321Sjhb return (rc); 5215308321Sjhb 5216318851Snp if (sc->flags & IS_VF) 5217318851Snp return (EPERM); 5218318851Snp 5219308321Sjhb /* Note that -1 is legitimate input (it means unbind). */ 5220308321Sjhb if (tc_idx < -1 || tc_idx >= sc->chip_params->nsched_cls) 5221308321Sjhb return (EINVAL); 5222308321Sjhb 5223318851Snp mtx_lock(&sc->tc_lock); 5224308321Sjhb if (tc_idx == txq->tc_idx) { 5225308321Sjhb rc = 0; /* No change, nothing to do. */ 5226308321Sjhb goto done; 5227308321Sjhb } 5228308321Sjhb 5229308321Sjhb fw_queue = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 5230308321Sjhb V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH) | 5231308321Sjhb V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id); 5232308321Sjhb 5233308321Sjhb if (tc_idx == -1) 5234308321Sjhb fw_class = 0xffffffff; /* Unbind. */ 5235308321Sjhb else { 5236308321Sjhb /* 5237318851Snp * Bind to a different class. 5238308321Sjhb */ 5239318851Snp tc = &pi->sched_params->cl_rl[tc_idx]; 5240318851Snp if (tc->flags & TX_CLRL_ERROR) { 5241318851Snp /* Previous attempt to set the cl-rl params failed. */ 5242318851Snp rc = EIO; 5243318851Snp goto done; 5244318851Snp } else { 5245318851Snp /* 5246318851Snp * Ok to proceed. Place a reference on the new class 5247318851Snp * while still holding on to the reference on the 5248318851Snp * previous class, if any. 5249318851Snp */ 5250308321Sjhb fw_class = tc_idx; 5251318851Snp tc->refcount++; 5252308321Sjhb } 5253308321Sjhb } 5254318851Snp mtx_unlock(&sc->tc_lock); 5255308321Sjhb 5256318851Snp rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4stc"); 5257318851Snp if (rc) 5258318851Snp return (rc); 5259308321Sjhb rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, &fw_class); 5260318851Snp end_synchronized_op(sc, 0); 5261318851Snp 5262318851Snp mtx_lock(&sc->tc_lock); 5263308321Sjhb if (rc == 0) { 5264308321Sjhb if (txq->tc_idx != -1) { 5265318851Snp tc = &pi->sched_params->cl_rl[txq->tc_idx]; 5266308321Sjhb MPASS(tc->refcount > 0); 5267308321Sjhb tc->refcount--; 5268308321Sjhb } 5269308321Sjhb txq->tc_idx = tc_idx; 5270318851Snp } else if (tc_idx != -1) { 5271318851Snp tc = &pi->sched_params->cl_rl[tc_idx]; 5272318851Snp MPASS(tc->refcount > 0); 5273318851Snp tc->refcount--; 5274308321Sjhb } 5275308321Sjhbdone: 5276318851Snp mtx_unlock(&sc->tc_lock); 5277308321Sjhb return (rc); 5278308321Sjhb} 5279