| sfxge_tx.c (280530) | sfxge_tx.c (280532) |
|---|---|
| 1/*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 29 unchanged lines hidden (view full) --- 38 * 39 * So, event queue plus label mapping to Tx queue index is: 40 * if event queue index is 0, TxQ-index = TxQ-label * [0..SFXGE_TXQ_NTYPES) 41 * else TxQ-index = SFXGE_TXQ_NTYPES + EvQ-index - 1 42 * See sfxge_get_txq_by_label() sfxge_ev.c 43 */ 44 45#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 29 unchanged lines hidden (view full) --- 38 * 39 * So, event queue plus label mapping to Tx queue index is: 40 * if event queue index is 0, TxQ-index = TxQ-label * [0..SFXGE_TXQ_NTYPES) 41 * else TxQ-index = SFXGE_TXQ_NTYPES + EvQ-index - 1 42 * See sfxge_get_txq_by_label() sfxge_ev.c 43 */ 44 45#include <sys/cdefs.h> |
| 46__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_tx.c 280530 2015-03-25 10:36:37Z arybchik $"); | 46__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/sfxge_tx.c 280532 2015-03-25 10:37:36Z arybchik $"); |
| 47 48#include <sys/types.h> 49#include <sys/mbuf.h> 50#include <sys/smp.h> 51#include <sys/socket.h> 52#include <sys/sysctl.h> 53#include <sys/syslog.h> 54 --- 47 unchanged lines hidden (view full) --- 102SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN, 103 &sfxge_tx_dpl_put_max, 0, 104 "Maximum number of any packets in deferred packet put-list"); 105 106#endif 107 108 109/* Forward declarations. */ | 47 48#include <sys/types.h> 49#include <sys/mbuf.h> 50#include <sys/smp.h> 51#include <sys/socket.h> 52#include <sys/sysctl.h> 53#include <sys/syslog.h> 54 --- 47 unchanged lines hidden (view full) --- 102SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN, 103 &sfxge_tx_dpl_put_max, 0, 104 "Maximum number of any packets in deferred packet put-list"); 105 106#endif 107 108 109/* Forward declarations. */ |
| 110static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq); | 110static void sfxge_tx_qdpl_service(struct sfxge_txq *txq); |
| 111static void sfxge_tx_qlist_post(struct sfxge_txq *txq); 112static void sfxge_tx_qunblock(struct sfxge_txq *txq); 113static int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, 114 const bus_dma_segment_t *dma_seg, int n_dma_seg); 115 116void 117sfxge_tx_qcomplete(struct sfxge_txq *txq, struct sfxge_evq *evq) 118{ --- 32 unchanged lines hidden (view full) --- 151 level = txq->added - txq->completed; 152 if (level <= SFXGE_TXQ_UNBLOCK_LEVEL(txq->entries)) 153 sfxge_tx_qunblock(txq); 154 } 155} 156 157#ifdef SFXGE_HAVE_MQ 158 | 111static void sfxge_tx_qlist_post(struct sfxge_txq *txq); 112static void sfxge_tx_qunblock(struct sfxge_txq *txq); 113static int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, 114 const bus_dma_segment_t *dma_seg, int n_dma_seg); 115 116void 117sfxge_tx_qcomplete(struct sfxge_txq *txq, struct sfxge_evq *evq) 118{ --- 32 unchanged lines hidden (view full) --- 151 level = txq->added - txq->completed; 152 if (level <= SFXGE_TXQ_UNBLOCK_LEVEL(txq->entries)) 153 sfxge_tx_qunblock(txq); 154 } 155} 156 157#ifdef SFXGE_HAVE_MQ 158 |
| 159static inline unsigned int | 159static unsigned int |
| 160sfxge_is_mbuf_non_tcp(struct mbuf *mbuf) 161{ 162 /* Absense of TCP checksum flags does not mean that it is non-TCP 163 * but it should be true if user wants to achieve high throughput. 164 */ 165 return (!(mbuf->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))); 166} 167 --- 308 unchanged lines hidden (view full) --- 476#define SFXGE_TX_QDPL_PENDING(_txq) \ 477 ((_txq)->dpl.std_put != 0) 478 479/* 480 * Service the deferred packet list. 481 * 482 * NOTE: drops the txq mutex! 483 */ | 160sfxge_is_mbuf_non_tcp(struct mbuf *mbuf) 161{ 162 /* Absense of TCP checksum flags does not mean that it is non-TCP 163 * but it should be true if user wants to achieve high throughput. 164 */ 165 return (!(mbuf->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))); 166} 167 --- 308 unchanged lines hidden (view full) --- 476#define SFXGE_TX_QDPL_PENDING(_txq) \ 477 ((_txq)->dpl.std_put != 0) 478 479/* 480 * Service the deferred packet list. 481 * 482 * NOTE: drops the txq mutex! 483 */ |
| 484static inline void | 484static void |
| 485sfxge_tx_qdpl_service(struct sfxge_txq *txq) 486{ 487 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); 488 489 do { 490 if (SFXGE_TX_QDPL_PENDING(txq)) 491 sfxge_tx_qdpl_swizzle(txq); 492 --- 11 unchanged lines hidden (view full) --- 504 * If we are called with the txq lock held, we put the packet on the "get 505 * list", otherwise we atomically push it on the "put list". The swizzle 506 * function takes care of ordering. 507 * 508 * The length of the put list is bounded by SFXGE_TX_MAX_DEFFERED. We 509 * overload the csum_data field in the mbuf to keep track of this length 510 * because there is no cheap alternative to avoid races. 511 */ | 485sfxge_tx_qdpl_service(struct sfxge_txq *txq) 486{ 487 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); 488 489 do { 490 if (SFXGE_TX_QDPL_PENDING(txq)) 491 sfxge_tx_qdpl_swizzle(txq); 492 --- 11 unchanged lines hidden (view full) --- 504 * If we are called with the txq lock held, we put the packet on the "get 505 * list", otherwise we atomically push it on the "put list". The swizzle 506 * function takes care of ordering. 507 * 508 * The length of the put list is bounded by SFXGE_TX_MAX_DEFFERED. We 509 * overload the csum_data field in the mbuf to keep track of this length 510 * because there is no cheap alternative to avoid races. 511 */ |
| 512static inline int | 512static int |
| 513sfxge_tx_qdpl_put(struct sfxge_txq *txq, struct mbuf *mbuf, int locked) 514{ 515 struct sfxge_tx_dpl *stdp; 516 517 stdp = &txq->dpl; 518 519 KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL")); 520 --- 231 unchanged lines hidden (view full) --- 752{ 753 struct sfxge_softc *sc = ifp->if_softc; 754 755 SFXGE_TXQ_LOCK(sc->txq[0]); 756 sfxge_if_start_locked(ifp); 757 SFXGE_TXQ_UNLOCK(sc->txq[0]); 758} 759 | 513sfxge_tx_qdpl_put(struct sfxge_txq *txq, struct mbuf *mbuf, int locked) 514{ 515 struct sfxge_tx_dpl *stdp; 516 517 stdp = &txq->dpl; 518 519 KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL")); 520 --- 231 unchanged lines hidden (view full) --- 752{ 753 struct sfxge_softc *sc = ifp->if_softc; 754 755 SFXGE_TXQ_LOCK(sc->txq[0]); 756 sfxge_if_start_locked(ifp); 757 SFXGE_TXQ_UNLOCK(sc->txq[0]); 758} 759 |
| 760static inline void | 760static void |
| 761sfxge_tx_qdpl_service(struct sfxge_txq *txq) 762{ 763 struct ifnet *ifp = txq->sc->ifnet; 764 765 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); 766 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 767 sfxge_if_start_locked(ifp); 768 SFXGE_TXQ_UNLOCK(txq); --- 18 unchanged lines hidden (view full) --- 787 788 const struct mbuf *mbuf; /* Input mbuf (head of chain) */ 789 u_short protocol; /* Network protocol (after VLAN decap) */ 790 ssize_t nh_off; /* Offset of network header */ 791 ssize_t tcph_off; /* Offset of TCP header */ 792 unsigned header_len; /* Number of bytes of header */ 793}; 794 | 761sfxge_tx_qdpl_service(struct sfxge_txq *txq) 762{ 763 struct ifnet *ifp = txq->sc->ifnet; 764 765 SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); 766 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 767 sfxge_if_start_locked(ifp); 768 SFXGE_TXQ_UNLOCK(txq); --- 18 unchanged lines hidden (view full) --- 787 788 const struct mbuf *mbuf; /* Input mbuf (head of chain) */ 789 u_short protocol; /* Network protocol (after VLAN decap) */ 790 ssize_t nh_off; /* Offset of network header */ 791 ssize_t tcph_off; /* Offset of TCP header */ 792 unsigned header_len; /* Number of bytes of header */ 793}; 794 |
| 795static inline const struct ip *tso_iph(const struct sfxge_tso_state *tso) | 795static const struct ip *tso_iph(const struct sfxge_tso_state *tso) |
| 796{ 797 KASSERT(tso->protocol == htons(ETHERTYPE_IP), 798 ("tso_iph() in non-IPv4 state")); 799 return (const struct ip *)(tso->mbuf->m_data + tso->nh_off); 800} | 796{ 797 KASSERT(tso->protocol == htons(ETHERTYPE_IP), 798 ("tso_iph() in non-IPv4 state")); 799 return (const struct ip *)(tso->mbuf->m_data + tso->nh_off); 800} |
| 801static inline const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso) | 801static __unused const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso) |
| 802{ 803 KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), 804 ("tso_ip6h() in non-IPv6 state")); 805 return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off); 806} | 802{ 803 KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), 804 ("tso_ip6h() in non-IPv6 state")); 805 return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off); 806} |
| 807static inline const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso) | 807static const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso) |
| 808{ 809 return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off); 810} 811 812/* Size of preallocated TSO header buffers. Larger blocks must be 813 * allocated from the heap. 814 */ 815#define TSOH_STD_SIZE 128 --- 818 unchanged lines hidden --- | 808{ 809 return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off); 810} 811 812/* Size of preallocated TSO header buffers. Larger blocks must be 813 * allocated from the heap. 814 */ 815#define TSOH_STD_SIZE 128 --- 818 unchanged lines hidden --- |