sfxge_tx.c revision 227569
1227569Sphilip/*- 2227569Sphilip * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9227569Sphilip * modification, are permitted provided that the following conditions 10227569Sphilip * are met: 11227569Sphilip * 1. Redistributions of source code must retain the above copyright 12227569Sphilip * notice, this list of conditions and the following disclaimer. 13227569Sphilip * 2. Redistributions in binary form must reproduce the above copyright 14227569Sphilip * notice, this list of conditions and the following disclaimer in the 15227569Sphilip * documentation and/or other materials provided with the distribution. 16227569Sphilip * 17227569Sphilip * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18227569Sphilip * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19227569Sphilip * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20227569Sphilip * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21227569Sphilip * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22227569Sphilip * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23227569Sphilip * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24227569Sphilip * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25227569Sphilip * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26227569Sphilip * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27227569Sphilip * SUCH DAMAGE. 28227569Sphilip */ 29227569Sphilip 30227569Sphilip#include <sys/cdefs.h> 31227569Sphilip__FBSDID("$FreeBSD: head/sys/dev/sfxge/sfxge_tx.c 227569 2011-11-16 17:11:13Z philip $"); 32227569Sphilip 33227569Sphilip#include <sys/types.h> 34227569Sphilip#include <sys/mbuf.h> 35227569Sphilip#include <sys/smp.h> 36227569Sphilip#include <sys/socket.h> 37227569Sphilip#include <sys/sysctl.h> 38227569Sphilip 39227569Sphilip#include <net/bpf.h> 40227569Sphilip#include <net/ethernet.h> 41227569Sphilip#include <net/if.h> 42227569Sphilip#include <net/if_vlan_var.h> 43227569Sphilip 44227569Sphilip#include <netinet/in.h> 45227569Sphilip#include <netinet/ip.h> 46227569Sphilip#include <netinet/ip6.h> 47227569Sphilip#include <netinet/tcp.h> 48227569Sphilip 49227569Sphilip#include "common/efx.h" 50227569Sphilip 51227569Sphilip#include "sfxge.h" 52227569Sphilip#include "sfxge_tx.h" 53227569Sphilip 54227569Sphilip/* Set the block level to ensure there is space to generate a 55227569Sphilip * large number of descriptors for TSO. With minimum MSS and 56227569Sphilip * maximum mbuf length we might need more than a ring-ful of 57227569Sphilip * descriptors, but this should not happen in practice except 58227569Sphilip * due to deliberate attack. In that case we will truncate 59227569Sphilip * the output at a packet boundary. Allow for a reasonable 60227569Sphilip * minimum MSS of 512. 61227569Sphilip */ 62227569Sphilip#define SFXGE_TSO_MAX_DESC ((65535 / 512) * 2 + SFXGE_TX_MAPPING_MAX_SEG - 1) 63227569Sphilip#define SFXGE_TXQ_BLOCK_LEVEL (SFXGE_NDESCS - SFXGE_TSO_MAX_DESC) 64227569Sphilip 65227569Sphilip/* Forward declarations. */ 66227569Sphilipstatic inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq); 67227569Sphilipstatic void sfxge_tx_qlist_post(struct sfxge_txq *txq); 68227569Sphilipstatic void sfxge_tx_qunblock(struct sfxge_txq *txq); 69227569Sphilipstatic int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, 70227569Sphilip const bus_dma_segment_t *dma_seg, int n_dma_seg); 71227569Sphilip 72227569Sphilipvoid 73227569Sphilipsfxge_tx_qcomplete(struct sfxge_txq *txq) 74227569Sphilip{ 75227569Sphilip struct sfxge_softc *sc; 76227569Sphilip struct sfxge_evq *evq; 77227569Sphilip unsigned int completed; 78227569Sphilip 79227569Sphilip sc = txq->sc; 80227569Sphilip evq = sc->evq[txq->evq_index]; 81227569Sphilip 82227569Sphilip mtx_assert(&evq->lock, MA_OWNED); 83227569Sphilip 84227569Sphilip completed = txq->completed; 85227569Sphilip while (completed != txq->pending) { 86227569Sphilip struct sfxge_tx_mapping *stmp; 87227569Sphilip unsigned int id; 88227569Sphilip 89227569Sphilip id = completed++ & (SFXGE_NDESCS - 1); 90227569Sphilip 91227569Sphilip stmp = &txq->stmp[id]; 92227569Sphilip if (stmp->flags & TX_BUF_UNMAP) { 93227569Sphilip bus_dmamap_unload(txq->packet_dma_tag, stmp->map); 94227569Sphilip if (stmp->flags & TX_BUF_MBUF) { 95227569Sphilip struct mbuf *m = stmp->u.mbuf; 96227569Sphilip do 97227569Sphilip m = m_free(m); 98227569Sphilip while (m != NULL); 99227569Sphilip } else { 100227569Sphilip free(stmp->u.heap_buf, M_SFXGE); 101227569Sphilip } 102227569Sphilip stmp->flags = 0; 103227569Sphilip } 104227569Sphilip } 105227569Sphilip txq->completed = completed; 106227569Sphilip 107227569Sphilip /* Check whether we need to unblock the queue. */ 108227569Sphilip mb(); 109227569Sphilip if (txq->blocked) { 110227569Sphilip unsigned int level; 111227569Sphilip 112227569Sphilip level = txq->added - txq->completed; 113227569Sphilip if (level <= SFXGE_TXQ_UNBLOCK_LEVEL) 114227569Sphilip sfxge_tx_qunblock(txq); 115227569Sphilip } 116227569Sphilip} 117227569Sphilip 118227569Sphilip#ifdef SFXGE_HAVE_MQ 119227569Sphilip 120227569Sphilip/* 121227569Sphilip * Reorder the put list and append it to the get list. 122227569Sphilip */ 123227569Sphilipstatic void 124227569Sphilipsfxge_tx_qdpl_swizzle(struct sfxge_txq *txq) 125227569Sphilip{ 126227569Sphilip struct sfxge_tx_dpl *stdp; 127227569Sphilip struct mbuf *mbuf, *get_next, **get_tailp; 128227569Sphilip volatile uintptr_t *putp; 129227569Sphilip uintptr_t put; 130227569Sphilip unsigned int count; 131227569Sphilip 132227569Sphilip mtx_assert(&txq->lock, MA_OWNED); 133227569Sphilip 134227569Sphilip stdp = &txq->dpl; 135227569Sphilip 136227569Sphilip /* Acquire the put list. */ 137227569Sphilip putp = &stdp->std_put; 138227569Sphilip put = atomic_readandclear_long(putp); 139227569Sphilip mbuf = (void *)put; 140227569Sphilip 141227569Sphilip if (mbuf == NULL) 142227569Sphilip return; 143227569Sphilip 144227569Sphilip /* Reverse the put list. */ 145227569Sphilip get_tailp = &mbuf->m_nextpkt; 146227569Sphilip get_next = NULL; 147227569Sphilip 148227569Sphilip count = 0; 149227569Sphilip do { 150227569Sphilip struct mbuf *put_next; 151227569Sphilip 152227569Sphilip put_next = mbuf->m_nextpkt; 153227569Sphilip mbuf->m_nextpkt = get_next; 154227569Sphilip get_next = mbuf; 155227569Sphilip mbuf = put_next; 156227569Sphilip 157227569Sphilip count++; 158227569Sphilip } while (mbuf != NULL); 159227569Sphilip 160227569Sphilip /* Append the reversed put list to the get list. */ 161227569Sphilip KASSERT(*get_tailp == NULL, ("*get_tailp != NULL")); 162227569Sphilip *stdp->std_getp = get_next; 163227569Sphilip stdp->std_getp = get_tailp; 164227569Sphilip stdp->std_count += count; 165227569Sphilip} 166227569Sphilip 167227569Sphilip#endif /* SFXGE_HAVE_MQ */ 168227569Sphilip 169227569Sphilipstatic void 170227569Sphilipsfxge_tx_qreap(struct sfxge_txq *txq) 171227569Sphilip{ 172227569Sphilip mtx_assert(SFXGE_TXQ_LOCK(txq), MA_OWNED); 173227569Sphilip 174227569Sphilip txq->reaped = txq->completed; 175227569Sphilip} 176227569Sphilip 177227569Sphilipstatic void 178227569Sphilipsfxge_tx_qlist_post(struct sfxge_txq *txq) 179227569Sphilip{ 180227569Sphilip unsigned int old_added; 181227569Sphilip unsigned int level; 182227569Sphilip int rc; 183227569Sphilip 184227569Sphilip mtx_assert(SFXGE_TXQ_LOCK(txq), MA_OWNED); 185227569Sphilip 186227569Sphilip KASSERT(txq->n_pend_desc != 0, ("txq->n_pend_desc == 0")); 187227569Sphilip KASSERT(txq->n_pend_desc <= SFXGE_TSO_MAX_DESC, 188227569Sphilip ("txq->n_pend_desc too large")); 189227569Sphilip KASSERT(!txq->blocked, ("txq->blocked")); 190227569Sphilip 191227569Sphilip old_added = txq->added; 192227569Sphilip 193227569Sphilip /* Post the fragment list. */ 194227569Sphilip rc = efx_tx_qpost(txq->common, txq->pend_desc, txq->n_pend_desc, 195227569Sphilip txq->reaped, &txq->added); 196227569Sphilip KASSERT(rc == 0, ("efx_tx_qpost() failed")); 197227569Sphilip 198227569Sphilip /* If efx_tx_qpost() had to refragment, our information about 199227569Sphilip * buffers to free may be associated with the wrong 200227569Sphilip * descriptors. 201227569Sphilip */ 202227569Sphilip KASSERT(txq->added - old_added == txq->n_pend_desc, 203227569Sphilip ("efx_tx_qpost() refragmented descriptors")); 204227569Sphilip 205227569Sphilip level = txq->added - txq->reaped; 206227569Sphilip KASSERT(level <= SFXGE_NDESCS, ("overfilled TX queue")); 207227569Sphilip 208227569Sphilip /* Clear the fragment list. */ 209227569Sphilip txq->n_pend_desc = 0; 210227569Sphilip 211227569Sphilip /* Have we reached the block level? */ 212227569Sphilip if (level < SFXGE_TXQ_BLOCK_LEVEL) 213227569Sphilip return; 214227569Sphilip 215227569Sphilip /* Reap, and check again */ 216227569Sphilip sfxge_tx_qreap(txq); 217227569Sphilip level = txq->added - txq->reaped; 218227569Sphilip if (level < SFXGE_TXQ_BLOCK_LEVEL) 219227569Sphilip return; 220227569Sphilip 221227569Sphilip txq->blocked = 1; 222227569Sphilip 223227569Sphilip /* 224227569Sphilip * Avoid a race with completion interrupt handling that could leave 225227569Sphilip * the queue blocked. 226227569Sphilip */ 227227569Sphilip mb(); 228227569Sphilip sfxge_tx_qreap(txq); 229227569Sphilip level = txq->added - txq->reaped; 230227569Sphilip if (level < SFXGE_TXQ_BLOCK_LEVEL) { 231227569Sphilip mb(); 232227569Sphilip txq->blocked = 0; 233227569Sphilip } 234227569Sphilip} 235227569Sphilip 236227569Sphilipstatic int sfxge_tx_queue_mbuf(struct sfxge_txq *txq, struct mbuf *mbuf) 237227569Sphilip{ 238227569Sphilip bus_dmamap_t *used_map; 239227569Sphilip bus_dmamap_t map; 240227569Sphilip bus_dma_segment_t dma_seg[SFXGE_TX_MAPPING_MAX_SEG]; 241227569Sphilip unsigned int id; 242227569Sphilip struct sfxge_tx_mapping *stmp; 243227569Sphilip efx_buffer_t *desc; 244227569Sphilip int n_dma_seg; 245227569Sphilip int rc; 246227569Sphilip int i; 247227569Sphilip 248227569Sphilip KASSERT(!txq->blocked, ("txq->blocked")); 249227569Sphilip 250227569Sphilip if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) 251227569Sphilip prefetch_read_many(mbuf->m_data); 252227569Sphilip 253227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) { 254227569Sphilip rc = EINTR; 255227569Sphilip goto reject; 256227569Sphilip } 257227569Sphilip 258227569Sphilip /* Load the packet for DMA. */ 259227569Sphilip id = txq->added & (SFXGE_NDESCS - 1); 260227569Sphilip stmp = &txq->stmp[id]; 261227569Sphilip rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag, stmp->map, 262227569Sphilip mbuf, dma_seg, &n_dma_seg, 0); 263227569Sphilip if (rc == EFBIG) { 264227569Sphilip /* Try again. */ 265227569Sphilip struct mbuf *new_mbuf = m_collapse(mbuf, M_DONTWAIT, 266227569Sphilip SFXGE_TX_MAPPING_MAX_SEG); 267227569Sphilip if (new_mbuf == NULL) 268227569Sphilip goto reject; 269227569Sphilip ++txq->collapses; 270227569Sphilip mbuf = new_mbuf; 271227569Sphilip rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag, 272227569Sphilip stmp->map, mbuf, 273227569Sphilip dma_seg, &n_dma_seg, 0); 274227569Sphilip } 275227569Sphilip if (rc != 0) 276227569Sphilip goto reject; 277227569Sphilip 278227569Sphilip /* Make the packet visible to the hardware. */ 279227569Sphilip bus_dmamap_sync(txq->packet_dma_tag, stmp->map, BUS_DMASYNC_PREWRITE); 280227569Sphilip 281227569Sphilip used_map = &stmp->map; 282227569Sphilip 283227569Sphilip if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) { 284227569Sphilip rc = sfxge_tx_queue_tso(txq, mbuf, dma_seg, n_dma_seg); 285227569Sphilip if (rc < 0) 286227569Sphilip goto reject_mapped; 287227569Sphilip stmp = &txq->stmp[rc]; 288227569Sphilip } else { 289227569Sphilip /* Add the mapping to the fragment list, and set flags 290227569Sphilip * for the buffer. 291227569Sphilip */ 292227569Sphilip i = 0; 293227569Sphilip for (;;) { 294227569Sphilip desc = &txq->pend_desc[i]; 295227569Sphilip desc->eb_addr = dma_seg[i].ds_addr; 296227569Sphilip desc->eb_size = dma_seg[i].ds_len; 297227569Sphilip if (i == n_dma_seg - 1) { 298227569Sphilip desc->eb_eop = 1; 299227569Sphilip break; 300227569Sphilip } 301227569Sphilip desc->eb_eop = 0; 302227569Sphilip i++; 303227569Sphilip 304227569Sphilip stmp->flags = 0; 305227569Sphilip if (__predict_false(stmp == 306227569Sphilip &txq->stmp[SFXGE_NDESCS - 1])) 307227569Sphilip stmp = &txq->stmp[0]; 308227569Sphilip else 309227569Sphilip stmp++; 310227569Sphilip } 311227569Sphilip txq->n_pend_desc = n_dma_seg; 312227569Sphilip } 313227569Sphilip 314227569Sphilip /* 315227569Sphilip * If the mapping required more than one descriptor 316227569Sphilip * then we need to associate the DMA map with the last 317227569Sphilip * descriptor, not the first. 318227569Sphilip */ 319227569Sphilip if (used_map != &stmp->map) { 320227569Sphilip map = stmp->map; 321227569Sphilip stmp->map = *used_map; 322227569Sphilip *used_map = map; 323227569Sphilip } 324227569Sphilip 325227569Sphilip stmp->u.mbuf = mbuf; 326227569Sphilip stmp->flags = TX_BUF_UNMAP | TX_BUF_MBUF; 327227569Sphilip 328227569Sphilip /* Post the fragment list. */ 329227569Sphilip sfxge_tx_qlist_post(txq); 330227569Sphilip 331227569Sphilip return 0; 332227569Sphilip 333227569Sphilipreject_mapped: 334227569Sphilip bus_dmamap_unload(txq->packet_dma_tag, *used_map); 335227569Sphilipreject: 336227569Sphilip /* Drop the packet on the floor. */ 337227569Sphilip m_freem(mbuf); 338227569Sphilip ++txq->drops; 339227569Sphilip 340227569Sphilip return rc; 341227569Sphilip} 342227569Sphilip 343227569Sphilip#ifdef SFXGE_HAVE_MQ 344227569Sphilip 345227569Sphilip/* 346227569Sphilip * Drain the deferred packet list into the transmit queue. 347227569Sphilip */ 348227569Sphilipstatic void 349227569Sphilipsfxge_tx_qdpl_drain(struct sfxge_txq *txq) 350227569Sphilip{ 351227569Sphilip struct sfxge_softc *sc; 352227569Sphilip struct sfxge_tx_dpl *stdp; 353227569Sphilip struct mbuf *mbuf, *next; 354227569Sphilip unsigned int count; 355227569Sphilip unsigned int pushed; 356227569Sphilip int rc; 357227569Sphilip 358227569Sphilip mtx_assert(&txq->lock, MA_OWNED); 359227569Sphilip 360227569Sphilip sc = txq->sc; 361227569Sphilip stdp = &txq->dpl; 362227569Sphilip pushed = txq->added; 363227569Sphilip 364227569Sphilip prefetch_read_many(sc->enp); 365227569Sphilip prefetch_read_many(txq->common); 366227569Sphilip 367227569Sphilip mbuf = stdp->std_get; 368227569Sphilip count = stdp->std_count; 369227569Sphilip 370227569Sphilip while (count != 0) { 371227569Sphilip KASSERT(mbuf != NULL, ("mbuf == NULL")); 372227569Sphilip 373227569Sphilip next = mbuf->m_nextpkt; 374227569Sphilip mbuf->m_nextpkt = NULL; 375227569Sphilip 376227569Sphilip ETHER_BPF_MTAP(sc->ifnet, mbuf); /* packet capture */ 377227569Sphilip 378227569Sphilip if (next != NULL) 379227569Sphilip prefetch_read_many(next); 380227569Sphilip 381227569Sphilip rc = sfxge_tx_queue_mbuf(txq, mbuf); 382227569Sphilip --count; 383227569Sphilip mbuf = next; 384227569Sphilip if (rc != 0) 385227569Sphilip continue; 386227569Sphilip 387227569Sphilip if (txq->blocked) 388227569Sphilip break; 389227569Sphilip 390227569Sphilip /* Push the fragments to the hardware in batches. */ 391227569Sphilip if (txq->added - pushed >= SFXGE_TX_BATCH) { 392227569Sphilip efx_tx_qpush(txq->common, txq->added); 393227569Sphilip pushed = txq->added; 394227569Sphilip } 395227569Sphilip } 396227569Sphilip 397227569Sphilip if (count == 0) { 398227569Sphilip KASSERT(mbuf == NULL, ("mbuf != NULL")); 399227569Sphilip stdp->std_get = NULL; 400227569Sphilip stdp->std_count = 0; 401227569Sphilip stdp->std_getp = &stdp->std_get; 402227569Sphilip } else { 403227569Sphilip stdp->std_get = mbuf; 404227569Sphilip stdp->std_count = count; 405227569Sphilip } 406227569Sphilip 407227569Sphilip if (txq->added != pushed) 408227569Sphilip efx_tx_qpush(txq->common, txq->added); 409227569Sphilip 410227569Sphilip KASSERT(txq->blocked || stdp->std_count == 0, 411227569Sphilip ("queue unblocked but count is non-zero")); 412227569Sphilip} 413227569Sphilip 414227569Sphilip#define SFXGE_TX_QDPL_PENDING(_txq) \ 415227569Sphilip ((_txq)->dpl.std_put != 0) 416227569Sphilip 417227569Sphilip/* 418227569Sphilip * Service the deferred packet list. 419227569Sphilip * 420227569Sphilip * NOTE: drops the txq mutex! 421227569Sphilip */ 422227569Sphilipstatic inline void 423227569Sphilipsfxge_tx_qdpl_service(struct sfxge_txq *txq) 424227569Sphilip{ 425227569Sphilip mtx_assert(&txq->lock, MA_OWNED); 426227569Sphilip 427227569Sphilip do { 428227569Sphilip if (SFXGE_TX_QDPL_PENDING(txq)) 429227569Sphilip sfxge_tx_qdpl_swizzle(txq); 430227569Sphilip 431227569Sphilip if (!txq->blocked) 432227569Sphilip sfxge_tx_qdpl_drain(txq); 433227569Sphilip 434227569Sphilip mtx_unlock(&txq->lock); 435227569Sphilip } while (SFXGE_TX_QDPL_PENDING(txq) && 436227569Sphilip mtx_trylock(&txq->lock)); 437227569Sphilip} 438227569Sphilip 439227569Sphilip/* 440227569Sphilip * Put a packet on the deferred packet list. 441227569Sphilip * 442227569Sphilip * If we are called with the txq lock held, we put the packet on the "get 443227569Sphilip * list", otherwise we atomically push it on the "put list". The swizzle 444227569Sphilip * function takes care of ordering. 445227569Sphilip * 446227569Sphilip * The length of the put list is bounded by SFXGE_TX_MAX_DEFFERED. We 447227569Sphilip * overload the csum_data field in the mbuf to keep track of this length 448227569Sphilip * because there is no cheap alternative to avoid races. 449227569Sphilip */ 450227569Sphilipstatic inline int 451227569Sphilipsfxge_tx_qdpl_put(struct sfxge_txq *txq, struct mbuf *mbuf, int locked) 452227569Sphilip{ 453227569Sphilip struct sfxge_tx_dpl *stdp; 454227569Sphilip 455227569Sphilip stdp = &txq->dpl; 456227569Sphilip 457227569Sphilip KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL")); 458227569Sphilip 459227569Sphilip if (locked) { 460227569Sphilip mtx_assert(&txq->lock, MA_OWNED); 461227569Sphilip 462227569Sphilip sfxge_tx_qdpl_swizzle(txq); 463227569Sphilip 464227569Sphilip *(stdp->std_getp) = mbuf; 465227569Sphilip stdp->std_getp = &mbuf->m_nextpkt; 466227569Sphilip stdp->std_count++; 467227569Sphilip } else { 468227569Sphilip volatile uintptr_t *putp; 469227569Sphilip uintptr_t old; 470227569Sphilip uintptr_t new; 471227569Sphilip unsigned old_len; 472227569Sphilip 473227569Sphilip putp = &stdp->std_put; 474227569Sphilip new = (uintptr_t)mbuf; 475227569Sphilip 476227569Sphilip do { 477227569Sphilip old = *putp; 478227569Sphilip if (old) { 479227569Sphilip struct mbuf *mp = (struct mbuf *)old; 480227569Sphilip old_len = mp->m_pkthdr.csum_data; 481227569Sphilip } else 482227569Sphilip old_len = 0; 483227569Sphilip if (old_len >= SFXGE_TX_MAX_DEFERRED) 484227569Sphilip return ENOBUFS; 485227569Sphilip mbuf->m_pkthdr.csum_data = old_len + 1; 486227569Sphilip mbuf->m_nextpkt = (void *)old; 487227569Sphilip } while (atomic_cmpset_long(putp, old, new) == 0); 488227569Sphilip } 489227569Sphilip 490227569Sphilip return (0); 491227569Sphilip} 492227569Sphilip 493227569Sphilip/* 494227569Sphilip * Called from if_transmit - will try to grab the txq lock and enqueue to the 495227569Sphilip * put list if it succeeds, otherwise will push onto the defer list. 496227569Sphilip */ 497227569Sphilipint 498227569Sphilipsfxge_tx_packet_add(struct sfxge_txq *txq, struct mbuf *m) 499227569Sphilip{ 500227569Sphilip int locked; 501227569Sphilip int rc; 502227569Sphilip 503227569Sphilip /* 504227569Sphilip * Try to grab the txq lock. If we are able to get the lock, 505227569Sphilip * the packet will be appended to the "get list" of the deferred 506227569Sphilip * packet list. Otherwise, it will be pushed on the "put list". 507227569Sphilip */ 508227569Sphilip locked = mtx_trylock(&txq->lock); 509227569Sphilip 510227569Sphilip /* 511227569Sphilip * Can only fail if we weren't able to get the lock. 512227569Sphilip */ 513227569Sphilip if (sfxge_tx_qdpl_put(txq, m, locked) != 0) { 514227569Sphilip KASSERT(!locked, 515227569Sphilip ("sfxge_tx_qdpl_put() failed locked")); 516227569Sphilip rc = ENOBUFS; 517227569Sphilip goto fail; 518227569Sphilip } 519227569Sphilip 520227569Sphilip /* 521227569Sphilip * Try to grab the lock again. 522227569Sphilip * 523227569Sphilip * If we are able to get the lock, we need to process the deferred 524227569Sphilip * packet list. If we are not able to get the lock, another thread 525227569Sphilip * is processing the list. 526227569Sphilip */ 527227569Sphilip if (!locked) 528227569Sphilip locked = mtx_trylock(&txq->lock); 529227569Sphilip 530227569Sphilip if (locked) { 531227569Sphilip /* Try to service the list. */ 532227569Sphilip sfxge_tx_qdpl_service(txq); 533227569Sphilip /* Lock has been dropped. */ 534227569Sphilip } 535227569Sphilip 536227569Sphilip return (0); 537227569Sphilip 538227569Sphilipfail: 539227569Sphilip return (rc); 540227569Sphilip 541227569Sphilip} 542227569Sphilip 543227569Sphilipstatic void 544227569Sphilipsfxge_tx_qdpl_flush(struct sfxge_txq *txq) 545227569Sphilip{ 546227569Sphilip struct sfxge_tx_dpl *stdp = &txq->dpl; 547227569Sphilip struct mbuf *mbuf, *next; 548227569Sphilip 549227569Sphilip mtx_lock(&txq->lock); 550227569Sphilip 551227569Sphilip sfxge_tx_qdpl_swizzle(txq); 552227569Sphilip for (mbuf = stdp->std_get; mbuf != NULL; mbuf = next) { 553227569Sphilip next = mbuf->m_nextpkt; 554227569Sphilip m_freem(mbuf); 555227569Sphilip } 556227569Sphilip stdp->std_get = NULL; 557227569Sphilip stdp->std_count = 0; 558227569Sphilip stdp->std_getp = &stdp->std_get; 559227569Sphilip 560227569Sphilip mtx_unlock(&txq->lock); 561227569Sphilip} 562227569Sphilip 563227569Sphilipvoid 564227569Sphilipsfxge_if_qflush(struct ifnet *ifp) 565227569Sphilip{ 566227569Sphilip struct sfxge_softc *sc; 567227569Sphilip int i; 568227569Sphilip 569227569Sphilip sc = ifp->if_softc; 570227569Sphilip 571227569Sphilip for (i = 0; i < SFXGE_TX_SCALE(sc); i++) 572227569Sphilip sfxge_tx_qdpl_flush(sc->txq[i]); 573227569Sphilip} 574227569Sphilip 575227569Sphilip/* 576227569Sphilip * TX start -- called by the stack. 577227569Sphilip */ 578227569Sphilipint 579227569Sphilipsfxge_if_transmit(struct ifnet *ifp, struct mbuf *m) 580227569Sphilip{ 581227569Sphilip struct sfxge_softc *sc; 582227569Sphilip struct sfxge_txq *txq; 583227569Sphilip int rc; 584227569Sphilip 585227569Sphilip sc = (struct sfxge_softc *)ifp->if_softc; 586227569Sphilip 587227569Sphilip KASSERT(ifp->if_flags & IFF_UP, ("interface not up")); 588227569Sphilip 589227569Sphilip if (!SFXGE_LINK_UP(sc)) { 590227569Sphilip m_freem(m); 591227569Sphilip return (0); 592227569Sphilip } 593227569Sphilip 594227569Sphilip /* Pick the desired transmit queue. */ 595227569Sphilip if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO)) { 596227569Sphilip int index = 0; 597227569Sphilip 598227569Sphilip if (m->m_flags & M_FLOWID) { 599227569Sphilip uint32_t hash = m->m_pkthdr.flowid; 600227569Sphilip 601227569Sphilip index = sc->rx_indir_table[hash % SFXGE_RX_SCALE_MAX]; 602227569Sphilip } 603227569Sphilip txq = sc->txq[SFXGE_TXQ_IP_TCP_UDP_CKSUM + index]; 604227569Sphilip } else if (m->m_pkthdr.csum_flags & CSUM_DELAY_IP) { 605227569Sphilip txq = sc->txq[SFXGE_TXQ_IP_CKSUM]; 606227569Sphilip } else { 607227569Sphilip txq = sc->txq[SFXGE_TXQ_NON_CKSUM]; 608227569Sphilip } 609227569Sphilip 610227569Sphilip rc = sfxge_tx_packet_add(txq, m); 611227569Sphilip 612227569Sphilip return (rc); 613227569Sphilip} 614227569Sphilip 615227569Sphilip#else /* !SFXGE_HAVE_MQ */ 616227569Sphilip 617227569Sphilipstatic void sfxge_if_start_locked(struct ifnet *ifp) 618227569Sphilip{ 619227569Sphilip struct sfxge_softc *sc = ifp->if_softc; 620227569Sphilip struct sfxge_txq *txq; 621227569Sphilip struct mbuf *mbuf; 622227569Sphilip unsigned int pushed[SFXGE_TXQ_NTYPES]; 623227569Sphilip unsigned int q_index; 624227569Sphilip 625227569Sphilip if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 626227569Sphilip IFF_DRV_RUNNING) 627227569Sphilip return; 628227569Sphilip 629227569Sphilip if (!sc->port.link_up) 630227569Sphilip return; 631227569Sphilip 632227569Sphilip for (q_index = 0; q_index < SFXGE_TXQ_NTYPES; q_index++) { 633227569Sphilip txq = sc->txq[q_index]; 634227569Sphilip pushed[q_index] = txq->added; 635227569Sphilip } 636227569Sphilip 637227569Sphilip while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 638227569Sphilip IFQ_DRV_DEQUEUE(&ifp->if_snd, mbuf); 639227569Sphilip if (mbuf == NULL) 640227569Sphilip break; 641227569Sphilip 642227569Sphilip ETHER_BPF_MTAP(ifp, mbuf); /* packet capture */ 643227569Sphilip 644227569Sphilip /* Pick the desired transmit queue. */ 645227569Sphilip if (mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO)) 646227569Sphilip q_index = SFXGE_TXQ_IP_TCP_UDP_CKSUM; 647227569Sphilip else if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) 648227569Sphilip q_index = SFXGE_TXQ_IP_CKSUM; 649227569Sphilip else 650227569Sphilip q_index = SFXGE_TXQ_NON_CKSUM; 651227569Sphilip txq = sc->txq[q_index]; 652227569Sphilip 653227569Sphilip if (sfxge_tx_queue_mbuf(txq, mbuf) != 0) 654227569Sphilip continue; 655227569Sphilip 656227569Sphilip if (txq->blocked) { 657227569Sphilip ifp->if_drv_flags |= IFF_DRV_OACTIVE; 658227569Sphilip break; 659227569Sphilip } 660227569Sphilip 661227569Sphilip /* Push the fragments to the hardware in batches. */ 662227569Sphilip if (txq->added - pushed[q_index] >= SFXGE_TX_BATCH) { 663227569Sphilip efx_tx_qpush(txq->common, txq->added); 664227569Sphilip pushed[q_index] = txq->added; 665227569Sphilip } 666227569Sphilip } 667227569Sphilip 668227569Sphilip for (q_index = 0; q_index < SFXGE_TXQ_NTYPES; q_index++) { 669227569Sphilip txq = sc->txq[q_index]; 670227569Sphilip if (txq->added != pushed[q_index]) 671227569Sphilip efx_tx_qpush(txq->common, txq->added); 672227569Sphilip } 673227569Sphilip} 674227569Sphilip 675227569Sphilipvoid sfxge_if_start(struct ifnet *ifp) 676227569Sphilip{ 677227569Sphilip struct sfxge_softc *sc = ifp->if_softc; 678227569Sphilip 679227569Sphilip mtx_lock(&sc->tx_lock); 680227569Sphilip sfxge_if_start_locked(ifp); 681227569Sphilip mtx_unlock(&sc->tx_lock); 682227569Sphilip} 683227569Sphilip 684227569Sphilipstatic inline void 685227569Sphilipsfxge_tx_qdpl_service(struct sfxge_txq *txq) 686227569Sphilip{ 687227569Sphilip struct sfxge_softc *sc = txq->sc; 688227569Sphilip struct ifnet *ifp = sc->ifnet; 689227569Sphilip 690227569Sphilip mtx_assert(&sc->tx_lock, MA_OWNED); 691227569Sphilip ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 692227569Sphilip sfxge_if_start_locked(ifp); 693227569Sphilip mtx_unlock(&sc->tx_lock); 694227569Sphilip} 695227569Sphilip 696227569Sphilip#endif /* SFXGE_HAVE_MQ */ 697227569Sphilip 698227569Sphilip/* 699227569Sphilip * Software "TSO". Not quite as good as doing it in hardware, but 700227569Sphilip * still faster than segmenting in the stack. 701227569Sphilip */ 702227569Sphilip 703227569Sphilipstruct sfxge_tso_state { 704227569Sphilip /* Output position */ 705227569Sphilip unsigned out_len; /* Remaining length in current segment */ 706227569Sphilip unsigned seqnum; /* Current sequence number */ 707227569Sphilip unsigned packet_space; /* Remaining space in current packet */ 708227569Sphilip 709227569Sphilip /* Input position */ 710227569Sphilip unsigned dma_seg_i; /* Current DMA segment number */ 711227569Sphilip uint64_t dma_addr; /* DMA address of current position */ 712227569Sphilip unsigned in_len; /* Remaining length in current mbuf */ 713227569Sphilip 714227569Sphilip const struct mbuf *mbuf; /* Input mbuf (head of chain) */ 715227569Sphilip u_short protocol; /* Network protocol (after VLAN decap) */ 716227569Sphilip ssize_t nh_off; /* Offset of network header */ 717227569Sphilip ssize_t tcph_off; /* Offset of TCP header */ 718227569Sphilip unsigned header_len; /* Number of bytes of header */ 719227569Sphilip int full_packet_size; /* Number of bytes to put in each outgoing 720227569Sphilip * segment */ 721227569Sphilip}; 722227569Sphilip 723227569Sphilipstatic inline const struct ip *tso_iph(const struct sfxge_tso_state *tso) 724227569Sphilip{ 725227569Sphilip KASSERT(tso->protocol == htons(ETHERTYPE_IP), 726227569Sphilip ("tso_iph() in non-IPv4 state")); 727227569Sphilip return (const struct ip *)(tso->mbuf->m_data + tso->nh_off); 728227569Sphilip} 729227569Sphilipstatic inline const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso) 730227569Sphilip{ 731227569Sphilip KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), 732227569Sphilip ("tso_ip6h() in non-IPv6 state")); 733227569Sphilip return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off); 734227569Sphilip} 735227569Sphilipstatic inline const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso) 736227569Sphilip{ 737227569Sphilip return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off); 738227569Sphilip} 739227569Sphilip 740227569Sphilip/* Size of preallocated TSO header buffers. Larger blocks must be 741227569Sphilip * allocated from the heap. 742227569Sphilip */ 743227569Sphilip#define TSOH_STD_SIZE 128 744227569Sphilip 745227569Sphilip/* At most half the descriptors in the queue at any time will refer to 746227569Sphilip * a TSO header buffer, since they must always be followed by a 747227569Sphilip * payload descriptor referring to an mbuf. 748227569Sphilip */ 749227569Sphilip#define TSOH_COUNT (SFXGE_NDESCS / 2u) 750227569Sphilip#define TSOH_PER_PAGE (PAGE_SIZE / TSOH_STD_SIZE) 751227569Sphilip#define TSOH_PAGE_COUNT ((TSOH_COUNT + TSOH_PER_PAGE - 1) / TSOH_PER_PAGE) 752227569Sphilip 753227569Sphilipstatic int tso_init(struct sfxge_txq *txq) 754227569Sphilip{ 755227569Sphilip struct sfxge_softc *sc = txq->sc; 756227569Sphilip int i, rc; 757227569Sphilip 758227569Sphilip /* Allocate TSO header buffers */ 759227569Sphilip txq->tsoh_buffer = malloc(TSOH_PAGE_COUNT * sizeof(txq->tsoh_buffer[0]), 760227569Sphilip M_SFXGE, M_WAITOK); 761227569Sphilip 762227569Sphilip for (i = 0; i < TSOH_PAGE_COUNT; i++) { 763227569Sphilip rc = sfxge_dma_alloc(sc, PAGE_SIZE, &txq->tsoh_buffer[i]); 764227569Sphilip if (rc) 765227569Sphilip goto fail; 766227569Sphilip } 767227569Sphilip 768227569Sphilip return 0; 769227569Sphilip 770227569Sphilipfail: 771227569Sphilip while (i-- > 0) 772227569Sphilip sfxge_dma_free(&txq->tsoh_buffer[i]); 773227569Sphilip free(txq->tsoh_buffer, M_SFXGE); 774227569Sphilip txq->tsoh_buffer = NULL; 775227569Sphilip return rc; 776227569Sphilip} 777227569Sphilip 778227569Sphilipstatic void tso_fini(struct sfxge_txq *txq) 779227569Sphilip{ 780227569Sphilip int i; 781227569Sphilip 782227569Sphilip if (txq->tsoh_buffer) { 783227569Sphilip for (i = 0; i < TSOH_PAGE_COUNT; i++) 784227569Sphilip sfxge_dma_free(&txq->tsoh_buffer[i]); 785227569Sphilip free(txq->tsoh_buffer, M_SFXGE); 786227569Sphilip } 787227569Sphilip} 788227569Sphilip 789227569Sphilipstatic void tso_start(struct sfxge_tso_state *tso, struct mbuf *mbuf) 790227569Sphilip{ 791227569Sphilip struct ether_header *eh = mtod(mbuf, struct ether_header *); 792227569Sphilip 793227569Sphilip tso->mbuf = mbuf; 794227569Sphilip 795227569Sphilip /* Find network protocol and header */ 796227569Sphilip tso->protocol = eh->ether_type; 797227569Sphilip if (tso->protocol == htons(ETHERTYPE_VLAN)) { 798227569Sphilip struct ether_vlan_header *veh = 799227569Sphilip mtod(mbuf, struct ether_vlan_header *); 800227569Sphilip tso->protocol = veh->evl_proto; 801227569Sphilip tso->nh_off = sizeof(*veh); 802227569Sphilip } else { 803227569Sphilip tso->nh_off = sizeof(*eh); 804227569Sphilip } 805227569Sphilip 806227569Sphilip /* Find TCP header */ 807227569Sphilip if (tso->protocol == htons(ETHERTYPE_IP)) { 808227569Sphilip KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP, 809227569Sphilip ("TSO required on non-TCP packet")); 810227569Sphilip tso->tcph_off = tso->nh_off + 4 * tso_iph(tso)->ip_hl; 811227569Sphilip } else { 812227569Sphilip KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), 813227569Sphilip ("TSO required on non-IP packet")); 814227569Sphilip KASSERT(tso_ip6h(tso)->ip6_nxt == IPPROTO_TCP, 815227569Sphilip ("TSO required on non-TCP packet")); 816227569Sphilip tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr); 817227569Sphilip } 818227569Sphilip 819227569Sphilip /* We assume all headers are linear in the head mbuf */ 820227569Sphilip tso->header_len = tso->tcph_off + 4 * tso_tcph(tso)->th_off; 821227569Sphilip KASSERT(tso->header_len <= mbuf->m_len, ("packet headers fragmented")); 822227569Sphilip tso->full_packet_size = tso->header_len + mbuf->m_pkthdr.tso_segsz; 823227569Sphilip 824227569Sphilip tso->seqnum = ntohl(tso_tcph(tso)->th_seq); 825227569Sphilip 826227569Sphilip /* These flags must not be duplicated */ 827227569Sphilip KASSERT(!(tso_tcph(tso)->th_flags & (TH_URG | TH_SYN | TH_RST)), 828227569Sphilip ("incompatible TCP flag on TSO packet")); 829227569Sphilip 830227569Sphilip tso->out_len = mbuf->m_pkthdr.len - tso->header_len; 831227569Sphilip} 832227569Sphilip 833227569Sphilip/* 834227569Sphilip * tso_fill_packet_with_fragment - form descriptors for the current fragment 835227569Sphilip * 836227569Sphilip * Form descriptors for the current fragment, until we reach the end 837227569Sphilip * of fragment or end-of-packet. Return 0 on success, 1 if not enough 838227569Sphilip * space. 839227569Sphilip */ 840227569Sphilipstatic void tso_fill_packet_with_fragment(struct sfxge_txq *txq, 841227569Sphilip struct sfxge_tso_state *tso) 842227569Sphilip{ 843227569Sphilip efx_buffer_t *desc; 844227569Sphilip int n; 845227569Sphilip 846227569Sphilip if (tso->in_len == 0 || tso->packet_space == 0) 847227569Sphilip return; 848227569Sphilip 849227569Sphilip KASSERT(tso->in_len > 0, ("TSO input length went negative")); 850227569Sphilip KASSERT(tso->packet_space > 0, ("TSO packet space went negative")); 851227569Sphilip 852227569Sphilip n = min(tso->in_len, tso->packet_space); 853227569Sphilip 854227569Sphilip tso->packet_space -= n; 855227569Sphilip tso->out_len -= n; 856227569Sphilip tso->in_len -= n; 857227569Sphilip 858227569Sphilip desc = &txq->pend_desc[txq->n_pend_desc++]; 859227569Sphilip desc->eb_addr = tso->dma_addr; 860227569Sphilip desc->eb_size = n; 861227569Sphilip desc->eb_eop = tso->out_len == 0 || tso->packet_space == 0; 862227569Sphilip 863227569Sphilip tso->dma_addr += n; 864227569Sphilip} 865227569Sphilip 866227569Sphilip/* Callback from bus_dmamap_load() for long TSO headers. */ 867227569Sphilipstatic void tso_map_long_header(void *dma_addr_ret, 868227569Sphilip bus_dma_segment_t *segs, int nseg, 869227569Sphilip int error) 870227569Sphilip{ 871227569Sphilip *(uint64_t *)dma_addr_ret = ((__predict_true(error == 0) && 872227569Sphilip __predict_true(nseg == 1)) ? 873227569Sphilip segs->ds_addr : 0); 874227569Sphilip} 875227569Sphilip 876227569Sphilip/* 877227569Sphilip * tso_start_new_packet - generate a new header and prepare for the new packet 878227569Sphilip * 879227569Sphilip * Generate a new header and prepare for the new packet. Return 0 on 880227569Sphilip * success, or an error code if failed to alloc header. 881227569Sphilip */ 882227569Sphilipstatic int tso_start_new_packet(struct sfxge_txq *txq, 883227569Sphilip struct sfxge_tso_state *tso, 884227569Sphilip unsigned int id) 885227569Sphilip{ 886227569Sphilip struct sfxge_tx_mapping *stmp = &txq->stmp[id]; 887227569Sphilip struct tcphdr *tsoh_th; 888227569Sphilip unsigned ip_length; 889227569Sphilip caddr_t header; 890227569Sphilip uint64_t dma_addr; 891227569Sphilip bus_dmamap_t map; 892227569Sphilip efx_buffer_t *desc; 893227569Sphilip int rc; 894227569Sphilip 895227569Sphilip /* Allocate a DMA-mapped header buffer. */ 896227569Sphilip if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) { 897227569Sphilip unsigned int page_index = (id / 2) / TSOH_PER_PAGE; 898227569Sphilip unsigned int buf_index = (id / 2) % TSOH_PER_PAGE; 899227569Sphilip 900227569Sphilip header = (txq->tsoh_buffer[page_index].esm_base + 901227569Sphilip buf_index * TSOH_STD_SIZE); 902227569Sphilip dma_addr = (txq->tsoh_buffer[page_index].esm_addr + 903227569Sphilip buf_index * TSOH_STD_SIZE); 904227569Sphilip map = txq->tsoh_buffer[page_index].esm_map; 905227569Sphilip 906227569Sphilip stmp->flags = 0; 907227569Sphilip } else { 908227569Sphilip /* We cannot use bus_dmamem_alloc() as that may sleep */ 909227569Sphilip header = malloc(tso->header_len, M_SFXGE, M_NOWAIT); 910227569Sphilip if (__predict_false(!header)) 911227569Sphilip return ENOMEM; 912227569Sphilip rc = bus_dmamap_load(txq->packet_dma_tag, stmp->map, 913227569Sphilip header, tso->header_len, 914227569Sphilip tso_map_long_header, &dma_addr, 915227569Sphilip BUS_DMA_NOWAIT); 916227569Sphilip if (__predict_false(dma_addr == 0)) { 917227569Sphilip if (rc == 0) { 918227569Sphilip /* Succeeded but got >1 segment */ 919227569Sphilip bus_dmamap_unload(txq->packet_dma_tag, 920227569Sphilip stmp->map); 921227569Sphilip rc = EINVAL; 922227569Sphilip } 923227569Sphilip free(header, M_SFXGE); 924227569Sphilip return rc; 925227569Sphilip } 926227569Sphilip map = stmp->map; 927227569Sphilip 928227569Sphilip txq->tso_long_headers++; 929227569Sphilip stmp->u.heap_buf = header; 930227569Sphilip stmp->flags = TX_BUF_UNMAP; 931227569Sphilip } 932227569Sphilip 933227569Sphilip tsoh_th = (struct tcphdr *)(header + tso->tcph_off); 934227569Sphilip 935227569Sphilip /* Copy and update the headers. */ 936227569Sphilip memcpy(header, tso->mbuf->m_data, tso->header_len); 937227569Sphilip 938227569Sphilip tsoh_th->th_seq = htonl(tso->seqnum); 939227569Sphilip tso->seqnum += tso->mbuf->m_pkthdr.tso_segsz; 940227569Sphilip if (tso->out_len > tso->mbuf->m_pkthdr.tso_segsz) { 941227569Sphilip /* This packet will not finish the TSO burst. */ 942227569Sphilip ip_length = tso->full_packet_size - tso->nh_off; 943227569Sphilip tsoh_th->th_flags &= ~(TH_FIN | TH_PUSH); 944227569Sphilip } else { 945227569Sphilip /* This packet will be the last in the TSO burst. */ 946227569Sphilip ip_length = tso->header_len - tso->nh_off + tso->out_len; 947227569Sphilip } 948227569Sphilip 949227569Sphilip if (tso->protocol == htons(ETHERTYPE_IP)) { 950227569Sphilip struct ip *tsoh_iph = (struct ip *)(header + tso->nh_off); 951227569Sphilip tsoh_iph->ip_len = htons(ip_length); 952227569Sphilip /* XXX We should increment ip_id, but FreeBSD doesn't 953227569Sphilip * currently allocate extra IDs for multiple segments. 954227569Sphilip */ 955227569Sphilip } else { 956227569Sphilip struct ip6_hdr *tsoh_iph = 957227569Sphilip (struct ip6_hdr *)(header + tso->nh_off); 958227569Sphilip tsoh_iph->ip6_plen = htons(ip_length - sizeof(*tsoh_iph)); 959227569Sphilip } 960227569Sphilip 961227569Sphilip /* Make the header visible to the hardware. */ 962227569Sphilip bus_dmamap_sync(txq->packet_dma_tag, map, BUS_DMASYNC_PREWRITE); 963227569Sphilip 964227569Sphilip tso->packet_space = tso->mbuf->m_pkthdr.tso_segsz; 965227569Sphilip txq->tso_packets++; 966227569Sphilip 967227569Sphilip /* Form a descriptor for this header. */ 968227569Sphilip desc = &txq->pend_desc[txq->n_pend_desc++]; 969227569Sphilip desc->eb_addr = dma_addr; 970227569Sphilip desc->eb_size = tso->header_len; 971227569Sphilip desc->eb_eop = 0; 972227569Sphilip 973227569Sphilip return 0; 974227569Sphilip} 975227569Sphilip 976227569Sphilipstatic int 977227569Sphilipsfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, 978227569Sphilip const bus_dma_segment_t *dma_seg, int n_dma_seg) 979227569Sphilip{ 980227569Sphilip struct sfxge_tso_state tso; 981227569Sphilip unsigned int id, next_id; 982227569Sphilip 983227569Sphilip tso_start(&tso, mbuf); 984227569Sphilip 985227569Sphilip /* Grab the first payload fragment. */ 986227569Sphilip if (dma_seg->ds_len == tso.header_len) { 987227569Sphilip --n_dma_seg; 988227569Sphilip KASSERT(n_dma_seg, ("no payload found in TSO packet")); 989227569Sphilip ++dma_seg; 990227569Sphilip tso.in_len = dma_seg->ds_len; 991227569Sphilip tso.dma_addr = dma_seg->ds_addr; 992227569Sphilip } else { 993227569Sphilip tso.in_len = dma_seg->ds_len - tso.header_len; 994227569Sphilip tso.dma_addr = dma_seg->ds_addr + tso.header_len; 995227569Sphilip } 996227569Sphilip 997227569Sphilip id = txq->added & (SFXGE_NDESCS - 1); 998227569Sphilip if (__predict_false(tso_start_new_packet(txq, &tso, id))) 999227569Sphilip return -1; 1000227569Sphilip 1001227569Sphilip while (1) { 1002227569Sphilip id = (id + 1) & (SFXGE_NDESCS - 1); 1003227569Sphilip tso_fill_packet_with_fragment(txq, &tso); 1004227569Sphilip 1005227569Sphilip /* Move onto the next fragment? */ 1006227569Sphilip if (tso.in_len == 0) { 1007227569Sphilip --n_dma_seg; 1008227569Sphilip if (n_dma_seg == 0) 1009227569Sphilip break; 1010227569Sphilip ++dma_seg; 1011227569Sphilip tso.in_len = dma_seg->ds_len; 1012227569Sphilip tso.dma_addr = dma_seg->ds_addr; 1013227569Sphilip } 1014227569Sphilip 1015227569Sphilip /* End of packet? */ 1016227569Sphilip if (tso.packet_space == 0) { 1017227569Sphilip /* If the queue is now full due to tiny MSS, 1018227569Sphilip * or we can't create another header, discard 1019227569Sphilip * the remainder of the input mbuf but do not 1020227569Sphilip * roll back the work we have done. 1021227569Sphilip */ 1022227569Sphilip if (txq->n_pend_desc > 1023227569Sphilip SFXGE_TSO_MAX_DESC - (1 + SFXGE_TX_MAPPING_MAX_SEG)) 1024227569Sphilip break; 1025227569Sphilip next_id = (id + 1) & (SFXGE_NDESCS - 1); 1026227569Sphilip if (__predict_false(tso_start_new_packet(txq, &tso, 1027227569Sphilip next_id))) 1028227569Sphilip break; 1029227569Sphilip id = next_id; 1030227569Sphilip } 1031227569Sphilip } 1032227569Sphilip 1033227569Sphilip txq->tso_bursts++; 1034227569Sphilip return id; 1035227569Sphilip} 1036227569Sphilip 1037227569Sphilipstatic void 1038227569Sphilipsfxge_tx_qunblock(struct sfxge_txq *txq) 1039227569Sphilip{ 1040227569Sphilip struct sfxge_softc *sc; 1041227569Sphilip struct sfxge_evq *evq; 1042227569Sphilip 1043227569Sphilip sc = txq->sc; 1044227569Sphilip evq = sc->evq[txq->evq_index]; 1045227569Sphilip 1046227569Sphilip mtx_assert(&evq->lock, MA_OWNED); 1047227569Sphilip 1048227569Sphilip if (txq->init_state != SFXGE_TXQ_STARTED) 1049227569Sphilip return; 1050227569Sphilip 1051227569Sphilip mtx_lock(SFXGE_TXQ_LOCK(txq)); 1052227569Sphilip 1053227569Sphilip if (txq->blocked) { 1054227569Sphilip unsigned int level; 1055227569Sphilip 1056227569Sphilip level = txq->added - txq->completed; 1057227569Sphilip if (level <= SFXGE_TXQ_UNBLOCK_LEVEL) 1058227569Sphilip txq->blocked = 0; 1059227569Sphilip } 1060227569Sphilip 1061227569Sphilip sfxge_tx_qdpl_service(txq); 1062227569Sphilip /* note: lock has been dropped */ 1063227569Sphilip} 1064227569Sphilip 1065227569Sphilipvoid 1066227569Sphilipsfxge_tx_qflush_done(struct sfxge_txq *txq) 1067227569Sphilip{ 1068227569Sphilip 1069227569Sphilip txq->flush_state = SFXGE_FLUSH_DONE; 1070227569Sphilip} 1071227569Sphilip 1072227569Sphilipstatic void 1073227569Sphilipsfxge_tx_qstop(struct sfxge_softc *sc, unsigned int index) 1074227569Sphilip{ 1075227569Sphilip struct sfxge_txq *txq; 1076227569Sphilip struct sfxge_evq *evq; 1077227569Sphilip unsigned int count; 1078227569Sphilip 1079227569Sphilip txq = sc->txq[index]; 1080227569Sphilip evq = sc->evq[txq->evq_index]; 1081227569Sphilip 1082227569Sphilip mtx_lock(SFXGE_TXQ_LOCK(txq)); 1083227569Sphilip 1084227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_STARTED, 1085227569Sphilip ("txq->init_state != SFXGE_TXQ_STARTED")); 1086227569Sphilip 1087227569Sphilip txq->init_state = SFXGE_TXQ_INITIALIZED; 1088227569Sphilip txq->flush_state = SFXGE_FLUSH_PENDING; 1089227569Sphilip 1090227569Sphilip /* Flush the transmit queue. */ 1091227569Sphilip efx_tx_qflush(txq->common); 1092227569Sphilip 1093227569Sphilip mtx_unlock(SFXGE_TXQ_LOCK(txq)); 1094227569Sphilip 1095227569Sphilip count = 0; 1096227569Sphilip do { 1097227569Sphilip /* Spin for 100ms. */ 1098227569Sphilip DELAY(100000); 1099227569Sphilip 1100227569Sphilip if (txq->flush_state != SFXGE_FLUSH_PENDING) 1101227569Sphilip break; 1102227569Sphilip } while (++count < 20); 1103227569Sphilip 1104227569Sphilip mtx_lock(&evq->lock); 1105227569Sphilip mtx_lock(SFXGE_TXQ_LOCK(txq)); 1106227569Sphilip 1107227569Sphilip KASSERT(txq->flush_state != SFXGE_FLUSH_FAILED, 1108227569Sphilip ("txq->flush_state == SFXGE_FLUSH_FAILED")); 1109227569Sphilip 1110227569Sphilip txq->flush_state = SFXGE_FLUSH_DONE; 1111227569Sphilip 1112227569Sphilip txq->blocked = 0; 1113227569Sphilip txq->pending = txq->added; 1114227569Sphilip 1115227569Sphilip sfxge_tx_qcomplete(txq); 1116227569Sphilip KASSERT(txq->completed == txq->added, 1117227569Sphilip ("txq->completed != txq->added")); 1118227569Sphilip 1119227569Sphilip sfxge_tx_qreap(txq); 1120227569Sphilip KASSERT(txq->reaped == txq->completed, 1121227569Sphilip ("txq->reaped != txq->completed")); 1122227569Sphilip 1123227569Sphilip txq->added = 0; 1124227569Sphilip txq->pending = 0; 1125227569Sphilip txq->completed = 0; 1126227569Sphilip txq->reaped = 0; 1127227569Sphilip 1128227569Sphilip /* Destroy the common code transmit queue. */ 1129227569Sphilip efx_tx_qdestroy(txq->common); 1130227569Sphilip txq->common = NULL; 1131227569Sphilip 1132227569Sphilip efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id, 1133227569Sphilip EFX_TXQ_NBUFS(SFXGE_NDESCS)); 1134227569Sphilip 1135227569Sphilip mtx_unlock(&evq->lock); 1136227569Sphilip mtx_unlock(SFXGE_TXQ_LOCK(txq)); 1137227569Sphilip} 1138227569Sphilip 1139227569Sphilipstatic int 1140227569Sphilipsfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) 1141227569Sphilip{ 1142227569Sphilip struct sfxge_txq *txq; 1143227569Sphilip efsys_mem_t *esmp; 1144227569Sphilip uint16_t flags; 1145227569Sphilip struct sfxge_evq *evq; 1146227569Sphilip int rc; 1147227569Sphilip 1148227569Sphilip txq = sc->txq[index]; 1149227569Sphilip esmp = &txq->mem; 1150227569Sphilip evq = sc->evq[txq->evq_index]; 1151227569Sphilip 1152227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 1153227569Sphilip ("txq->init_state != SFXGE_TXQ_INITIALIZED")); 1154227569Sphilip KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 1155227569Sphilip ("evq->init_state != SFXGE_EVQ_STARTED")); 1156227569Sphilip 1157227569Sphilip /* Program the buffer table. */ 1158227569Sphilip if ((rc = efx_sram_buf_tbl_set(sc->enp, txq->buf_base_id, esmp, 1159227569Sphilip EFX_TXQ_NBUFS(SFXGE_NDESCS))) != 0) 1160227569Sphilip return rc; 1161227569Sphilip 1162227569Sphilip /* Determine the kind of queue we are creating. */ 1163227569Sphilip switch (txq->type) { 1164227569Sphilip case SFXGE_TXQ_NON_CKSUM: 1165227569Sphilip flags = 0; 1166227569Sphilip break; 1167227569Sphilip case SFXGE_TXQ_IP_CKSUM: 1168227569Sphilip flags = EFX_CKSUM_IPV4; 1169227569Sphilip break; 1170227569Sphilip case SFXGE_TXQ_IP_TCP_UDP_CKSUM: 1171227569Sphilip flags = EFX_CKSUM_IPV4 | EFX_CKSUM_TCPUDP; 1172227569Sphilip break; 1173227569Sphilip default: 1174227569Sphilip KASSERT(0, ("Impossible TX queue")); 1175227569Sphilip flags = 0; 1176227569Sphilip break; 1177227569Sphilip } 1178227569Sphilip 1179227569Sphilip /* Create the common code transmit queue. */ 1180227569Sphilip if ((rc = efx_tx_qcreate(sc->enp, index, index, esmp, 1181227569Sphilip SFXGE_NDESCS, txq->buf_base_id, flags, evq->common, 1182227569Sphilip &txq->common)) != 0) 1183227569Sphilip goto fail; 1184227569Sphilip 1185227569Sphilip mtx_lock(SFXGE_TXQ_LOCK(txq)); 1186227569Sphilip 1187227569Sphilip /* Enable the transmit queue. */ 1188227569Sphilip efx_tx_qenable(txq->common); 1189227569Sphilip 1190227569Sphilip txq->init_state = SFXGE_TXQ_STARTED; 1191227569Sphilip 1192227569Sphilip mtx_unlock(SFXGE_TXQ_LOCK(txq)); 1193227569Sphilip 1194227569Sphilip return (0); 1195227569Sphilip 1196227569Sphilipfail: 1197227569Sphilip efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id, 1198227569Sphilip EFX_TXQ_NBUFS(SFXGE_NDESCS)); 1199227569Sphilip return rc; 1200227569Sphilip} 1201227569Sphilip 1202227569Sphilipvoid 1203227569Sphilipsfxge_tx_stop(struct sfxge_softc *sc) 1204227569Sphilip{ 1205227569Sphilip const efx_nic_cfg_t *encp; 1206227569Sphilip int index; 1207227569Sphilip 1208227569Sphilip index = SFXGE_TX_SCALE(sc); 1209227569Sphilip while (--index >= 0) 1210227569Sphilip sfxge_tx_qstop(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); 1211227569Sphilip 1212227569Sphilip sfxge_tx_qstop(sc, SFXGE_TXQ_IP_CKSUM); 1213227569Sphilip 1214227569Sphilip encp = efx_nic_cfg_get(sc->enp); 1215227569Sphilip sfxge_tx_qstop(sc, SFXGE_TXQ_NON_CKSUM); 1216227569Sphilip 1217227569Sphilip /* Tear down the transmit module */ 1218227569Sphilip efx_tx_fini(sc->enp); 1219227569Sphilip} 1220227569Sphilip 1221227569Sphilipint 1222227569Sphilipsfxge_tx_start(struct sfxge_softc *sc) 1223227569Sphilip{ 1224227569Sphilip int index; 1225227569Sphilip int rc; 1226227569Sphilip 1227227569Sphilip /* Initialize the common code transmit module. */ 1228227569Sphilip if ((rc = efx_tx_init(sc->enp)) != 0) 1229227569Sphilip return (rc); 1230227569Sphilip 1231227569Sphilip if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_NON_CKSUM)) != 0) 1232227569Sphilip goto fail; 1233227569Sphilip 1234227569Sphilip if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_IP_CKSUM)) != 0) 1235227569Sphilip goto fail2; 1236227569Sphilip 1237227569Sphilip for (index = 0; index < SFXGE_TX_SCALE(sc); index++) { 1238227569Sphilip if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + 1239227569Sphilip index)) != 0) 1240227569Sphilip goto fail3; 1241227569Sphilip } 1242227569Sphilip 1243227569Sphilip return (0); 1244227569Sphilip 1245227569Sphilipfail3: 1246227569Sphilip while (--index >= 0) 1247227569Sphilip sfxge_tx_qstop(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); 1248227569Sphilip 1249227569Sphilip sfxge_tx_qstop(sc, SFXGE_TXQ_IP_CKSUM); 1250227569Sphilip 1251227569Sphilipfail2: 1252227569Sphilip sfxge_tx_qstop(sc, SFXGE_TXQ_NON_CKSUM); 1253227569Sphilip 1254227569Sphilipfail: 1255227569Sphilip efx_tx_fini(sc->enp); 1256227569Sphilip 1257227569Sphilip return (rc); 1258227569Sphilip} 1259227569Sphilip 1260227569Sphilip/** 1261227569Sphilip * Destroy a transmit queue. 1262227569Sphilip */ 1263227569Sphilipstatic void 1264227569Sphilipsfxge_tx_qfini(struct sfxge_softc *sc, unsigned int index) 1265227569Sphilip{ 1266227569Sphilip struct sfxge_txq *txq; 1267227569Sphilip unsigned int nmaps = SFXGE_NDESCS; 1268227569Sphilip 1269227569Sphilip txq = sc->txq[index]; 1270227569Sphilip 1271227569Sphilip KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 1272227569Sphilip ("txq->init_state != SFXGE_TXQ_INITIALIZED")); 1273227569Sphilip 1274227569Sphilip if (txq->type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) 1275227569Sphilip tso_fini(txq); 1276227569Sphilip 1277227569Sphilip /* Free the context arrays. */ 1278227569Sphilip free(txq->pend_desc, M_SFXGE); 1279227569Sphilip while (nmaps--) 1280227569Sphilip bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map); 1281227569Sphilip free(txq->stmp, M_SFXGE); 1282227569Sphilip 1283227569Sphilip /* Release DMA memory mapping. */ 1284227569Sphilip sfxge_dma_free(&txq->mem); 1285227569Sphilip 1286227569Sphilip sc->txq[index] = NULL; 1287227569Sphilip 1288227569Sphilip#ifdef SFXGE_HAVE_MQ 1289227569Sphilip mtx_destroy(&txq->lock); 1290227569Sphilip#endif 1291227569Sphilip 1292227569Sphilip free(txq, M_SFXGE); 1293227569Sphilip} 1294227569Sphilip 1295227569Sphilipstatic int 1296227569Sphilipsfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index, 1297227569Sphilip enum sfxge_txq_type type, unsigned int evq_index) 1298227569Sphilip{ 1299227569Sphilip struct sfxge_txq *txq; 1300227569Sphilip struct sfxge_evq *evq; 1301227569Sphilip#ifdef SFXGE_HAVE_MQ 1302227569Sphilip struct sfxge_tx_dpl *stdp; 1303227569Sphilip#endif 1304227569Sphilip efsys_mem_t *esmp; 1305227569Sphilip unsigned int nmaps; 1306227569Sphilip int rc; 1307227569Sphilip 1308227569Sphilip txq = malloc(sizeof(struct sfxge_txq), M_SFXGE, M_ZERO | M_WAITOK); 1309227569Sphilip txq->sc = sc; 1310227569Sphilip 1311227569Sphilip sc->txq[txq_index] = txq; 1312227569Sphilip esmp = &txq->mem; 1313227569Sphilip 1314227569Sphilip evq = sc->evq[evq_index]; 1315227569Sphilip 1316227569Sphilip /* Allocate and zero DMA space for the descriptor ring. */ 1317227569Sphilip if ((rc = sfxge_dma_alloc(sc, EFX_TXQ_SIZE(SFXGE_NDESCS), esmp)) != 0) 1318227569Sphilip return (rc); 1319227569Sphilip (void)memset(esmp->esm_base, 0, EFX_TXQ_SIZE(SFXGE_NDESCS)); 1320227569Sphilip 1321227569Sphilip /* Allocate buffer table entries. */ 1322227569Sphilip sfxge_sram_buf_tbl_alloc(sc, EFX_TXQ_NBUFS(SFXGE_NDESCS), 1323227569Sphilip &txq->buf_base_id); 1324227569Sphilip 1325227569Sphilip /* Create a DMA tag for packet mappings. */ 1326227569Sphilip if (bus_dma_tag_create(sc->parent_dma_tag, 1, 0x1000, 0x3FFFFFFFFFFFULL, 1327227569Sphilip BUS_SPACE_MAXADDR, NULL, NULL, 0x11000, 1328227569Sphilip SFXGE_TX_MAPPING_MAX_SEG, 0x1000, 0, NULL, NULL, 1329227569Sphilip &txq->packet_dma_tag) != 0) { 1330227569Sphilip device_printf(sc->dev, "Couldn't allocate txq DMA tag\n"); 1331227569Sphilip rc = ENOMEM; 1332227569Sphilip goto fail; 1333227569Sphilip } 1334227569Sphilip 1335227569Sphilip /* Allocate pending descriptor array for batching writes. */ 1336227569Sphilip txq->pend_desc = malloc(sizeof(efx_buffer_t) * SFXGE_NDESCS, 1337227569Sphilip M_SFXGE, M_ZERO | M_WAITOK); 1338227569Sphilip 1339227569Sphilip /* Allocate and initialise mbuf DMA mapping array. */ 1340227569Sphilip txq->stmp = malloc(sizeof(struct sfxge_tx_mapping) * SFXGE_NDESCS, 1341227569Sphilip M_SFXGE, M_ZERO | M_WAITOK); 1342227569Sphilip for (nmaps = 0; nmaps < SFXGE_NDESCS; nmaps++) { 1343227569Sphilip rc = bus_dmamap_create(txq->packet_dma_tag, 0, 1344227569Sphilip &txq->stmp[nmaps].map); 1345227569Sphilip if (rc != 0) 1346227569Sphilip goto fail2; 1347227569Sphilip } 1348227569Sphilip 1349227569Sphilip if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM && 1350227569Sphilip (rc = tso_init(txq)) != 0) 1351227569Sphilip goto fail3; 1352227569Sphilip 1353227569Sphilip#ifdef SFXGE_HAVE_MQ 1354227569Sphilip /* Initialize the deferred packet list. */ 1355227569Sphilip stdp = &txq->dpl; 1356227569Sphilip stdp->std_getp = &stdp->std_get; 1357227569Sphilip 1358227569Sphilip mtx_init(&txq->lock, "txq", NULL, MTX_DEF); 1359227569Sphilip#endif 1360227569Sphilip 1361227569Sphilip txq->type = type; 1362227569Sphilip txq->evq_index = evq_index; 1363227569Sphilip txq->txq_index = txq_index; 1364227569Sphilip txq->init_state = SFXGE_TXQ_INITIALIZED; 1365227569Sphilip 1366227569Sphilip return (0); 1367227569Sphilip 1368227569Sphilipfail3: 1369227569Sphilip free(txq->pend_desc, M_SFXGE); 1370227569Sphilipfail2: 1371227569Sphilip while (nmaps--) 1372227569Sphilip bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map); 1373227569Sphilip free(txq->stmp, M_SFXGE); 1374227569Sphilip bus_dma_tag_destroy(txq->packet_dma_tag); 1375227569Sphilip 1376227569Sphilipfail: 1377227569Sphilip sfxge_dma_free(esmp); 1378227569Sphilip 1379227569Sphilip return (rc); 1380227569Sphilip} 1381227569Sphilip 1382227569Sphilipstatic const struct { 1383227569Sphilip const char *name; 1384227569Sphilip size_t offset; 1385227569Sphilip} sfxge_tx_stats[] = { 1386227569Sphilip#define SFXGE_TX_STAT(name, member) \ 1387227569Sphilip { #name, offsetof(struct sfxge_txq, member) } 1388227569Sphilip SFXGE_TX_STAT(tso_bursts, tso_bursts), 1389227569Sphilip SFXGE_TX_STAT(tso_packets, tso_packets), 1390227569Sphilip SFXGE_TX_STAT(tso_long_headers, tso_long_headers), 1391227569Sphilip SFXGE_TX_STAT(tx_collapses, collapses), 1392227569Sphilip SFXGE_TX_STAT(tx_drops, drops), 1393227569Sphilip}; 1394227569Sphilip 1395227569Sphilipstatic int 1396227569Sphilipsfxge_tx_stat_handler(SYSCTL_HANDLER_ARGS) 1397227569Sphilip{ 1398227569Sphilip struct sfxge_softc *sc = arg1; 1399227569Sphilip unsigned int id = arg2; 1400227569Sphilip unsigned long sum; 1401227569Sphilip unsigned int index; 1402227569Sphilip 1403227569Sphilip /* Sum across all TX queues */ 1404227569Sphilip sum = 0; 1405227569Sphilip for (index = 0; 1406227569Sphilip index < SFXGE_TXQ_IP_TCP_UDP_CKSUM + SFXGE_TX_SCALE(sc); 1407227569Sphilip index++) 1408227569Sphilip sum += *(unsigned long *)((caddr_t)sc->txq[index] + 1409227569Sphilip sfxge_tx_stats[id].offset); 1410227569Sphilip 1411227569Sphilip return SYSCTL_OUT(req, &sum, sizeof(sum)); 1412227569Sphilip} 1413227569Sphilip 1414227569Sphilipstatic void 1415227569Sphilipsfxge_tx_stat_init(struct sfxge_softc *sc) 1416227569Sphilip{ 1417227569Sphilip struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 1418227569Sphilip struct sysctl_oid_list *stat_list; 1419227569Sphilip unsigned int id; 1420227569Sphilip 1421227569Sphilip stat_list = SYSCTL_CHILDREN(sc->stats_node); 1422227569Sphilip 1423227569Sphilip for (id = 0; 1424227569Sphilip id < sizeof(sfxge_tx_stats) / sizeof(sfxge_tx_stats[0]); 1425227569Sphilip id++) { 1426227569Sphilip SYSCTL_ADD_PROC( 1427227569Sphilip ctx, stat_list, 1428227569Sphilip OID_AUTO, sfxge_tx_stats[id].name, 1429227569Sphilip CTLTYPE_ULONG|CTLFLAG_RD, 1430227569Sphilip sc, id, sfxge_tx_stat_handler, "LU", 1431227569Sphilip ""); 1432227569Sphilip } 1433227569Sphilip} 1434227569Sphilip 1435227569Sphilipvoid 1436227569Sphilipsfxge_tx_fini(struct sfxge_softc *sc) 1437227569Sphilip{ 1438227569Sphilip int index; 1439227569Sphilip 1440227569Sphilip index = SFXGE_TX_SCALE(sc); 1441227569Sphilip while (--index >= 0) 1442227569Sphilip sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); 1443227569Sphilip 1444227569Sphilip sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM); 1445227569Sphilip sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM); 1446227569Sphilip} 1447227569Sphilip 1448227569Sphilip 1449227569Sphilipint 1450227569Sphilipsfxge_tx_init(struct sfxge_softc *sc) 1451227569Sphilip{ 1452227569Sphilip struct sfxge_intr *intr; 1453227569Sphilip int index; 1454227569Sphilip int rc; 1455227569Sphilip 1456227569Sphilip intr = &sc->intr; 1457227569Sphilip 1458227569Sphilip KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 1459227569Sphilip ("intr->state != SFXGE_INTR_INITIALIZED")); 1460227569Sphilip 1461227569Sphilip /* Initialize the transmit queues */ 1462227569Sphilip if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_NON_CKSUM, 1463227569Sphilip SFXGE_TXQ_NON_CKSUM, 0)) != 0) 1464227569Sphilip goto fail; 1465227569Sphilip 1466227569Sphilip if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_CKSUM, 1467227569Sphilip SFXGE_TXQ_IP_CKSUM, 0)) != 0) 1468227569Sphilip goto fail2; 1469227569Sphilip 1470227569Sphilip for (index = 0; index < SFXGE_TX_SCALE(sc); index++) { 1471227569Sphilip if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index, 1472227569Sphilip SFXGE_TXQ_IP_TCP_UDP_CKSUM, index)) != 0) 1473227569Sphilip goto fail3; 1474227569Sphilip } 1475227569Sphilip 1476227569Sphilip sfxge_tx_stat_init(sc); 1477227569Sphilip 1478227569Sphilip return (0); 1479227569Sphilip 1480227569Sphilipfail3: 1481227569Sphilip sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM); 1482227569Sphilip 1483227569Sphilip while (--index >= 0) 1484227569Sphilip sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); 1485227569Sphilip 1486227569Sphilipfail2: 1487227569Sphilip sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM); 1488227569Sphilip 1489227569Sphilipfail: 1490227569Sphilip return (rc); 1491227569Sphilip} 1492