t4_sge.c (281955) | t4_sge.c (284052) |
---|---|
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 281955 2015-04-24 23:26:44Z hiren $"); | 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 284052 2015-06-06 09:28:40Z np $"); |
30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/types.h> 35#include <sys/mbuf.h> 36#include <sys/socket.h> 37#include <sys/kernel.h> | 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/types.h> 35#include <sys/mbuf.h> 36#include <sys/socket.h> 37#include <sys/kernel.h> |
38#include <sys/kdb.h> | |
39#include <sys/malloc.h> 40#include <sys/queue.h> 41#include <sys/sbuf.h> 42#include <sys/taskqueue.h> 43#include <sys/time.h> | 38#include <sys/malloc.h> 39#include <sys/queue.h> 40#include <sys/sbuf.h> 41#include <sys/taskqueue.h> 42#include <sys/time.h> |
43#include <sys/sglist.h> |
|
44#include <sys/sysctl.h> 45#include <sys/smp.h> 46#include <sys/counter.h> 47#include <net/bpf.h> 48#include <net/ethernet.h> 49#include <net/if.h> 50#include <net/if_vlan_var.h> 51#include <netinet/in.h> --- 10 unchanged lines hidden (view full) --- 62#include <net/netmap.h> 63#include <dev/netmap/netmap_kern.h> 64#endif 65 66#include "common/common.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "common/t4_msg.h" | 44#include <sys/sysctl.h> 45#include <sys/smp.h> 46#include <sys/counter.h> 47#include <net/bpf.h> 48#include <net/ethernet.h> 49#include <net/if.h> 50#include <net/if_vlan_var.h> 51#include <netinet/in.h> --- 10 unchanged lines hidden (view full) --- 62#include <net/netmap.h> 63#include <dev/netmap/netmap_kern.h> 64#endif 65 66#include "common/common.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "common/t4_msg.h" |
70#include "t4_mp_ring.h" |
|
70 71#ifdef T4_PKT_TIMESTAMP 72#define RX_COPY_THRESHOLD (MINCLSIZE - 8) 73#else 74#define RX_COPY_THRESHOLD MINCLSIZE 75#endif 76 77/* --- 63 unchanged lines hidden (view full) --- 141 142/* 143 * Size of cluster allocation that's most likely to succeed. The driver will 144 * fall back to this size if it fails to allocate clusters larger than this. 145 */ 146static int safest_rx_cluster = PAGE_SIZE; 147TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster); 148 | 71 72#ifdef T4_PKT_TIMESTAMP 73#define RX_COPY_THRESHOLD (MINCLSIZE - 8) 74#else 75#define RX_COPY_THRESHOLD MINCLSIZE 76#endif 77 78/* --- 63 unchanged lines hidden (view full) --- 142 143/* 144 * Size of cluster allocation that's most likely to succeed. The driver will 145 * fall back to this size if it fails to allocate clusters larger than this. 146 */ 147static int safest_rx_cluster = PAGE_SIZE; 148TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster); 149 |
149/* Used to track coalesced tx work request */ | |
150struct txpkts { | 150struct txpkts { |
151 uint64_t *flitp; /* ptr to flit where next pkt should start */ 152 uint8_t npkt; /* # of packets in this work request */ 153 uint8_t nflits; /* # of flits used by this work request */ 154 uint16_t plen; /* total payload (sum of all packets) */ | 151 u_int wr_type; /* type 0 or type 1 */ 152 u_int npkt; /* # of packets in this work request */ 153 u_int plen; /* total payload (sum of all packets) */ 154 u_int len16; /* # of 16B pieces used by this work request */ |
155}; 156 157/* A packet's SGL. This + m_pkthdr has all info needed for tx */ 158struct sgl { | 155}; 156 157/* A packet's SGL. This + m_pkthdr has all info needed for tx */ 158struct sgl { |
159 int nsegs; /* # of segments in the SGL, 0 means imm. tx */ 160 int nflits; /* # of flits needed for the SGL */ 161 bus_dma_segment_t seg[TX_SGL_SEGS]; | 159 struct sglist sg; 160 struct sglist_seg seg[TX_SGL_SEGS]; |
162}; 163 164static int service_iq(struct sge_iq *, int); 165static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); 166static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *); 167static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); 168static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); 169static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t, --- 45 unchanged lines hidden (view full) --- 215static int refill_fl(struct adapter *, struct sge_fl *, int); 216static void refill_sfl(void *); 217static int alloc_fl_sdesc(struct sge_fl *); 218static void free_fl_sdesc(struct adapter *, struct sge_fl *); 219static void find_best_refill_source(struct adapter *, struct sge_fl *, int); 220static void find_safe_refill_source(struct adapter *, struct sge_fl *); 221static void add_fl_to_sfl(struct adapter *, struct sge_fl *); 222 | 161}; 162 163static int service_iq(struct sge_iq *, int); 164static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); 165static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *); 166static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); 167static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); 168static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t, --- 45 unchanged lines hidden (view full) --- 214static int refill_fl(struct adapter *, struct sge_fl *, int); 215static void refill_sfl(void *); 216static int alloc_fl_sdesc(struct sge_fl *); 217static void free_fl_sdesc(struct adapter *, struct sge_fl *); 218static void find_best_refill_source(struct adapter *, struct sge_fl *, int); 219static void find_safe_refill_source(struct adapter *, struct sge_fl *); 220static void add_fl_to_sfl(struct adapter *, struct sge_fl *); 221 |
223static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int); 224static int free_pkt_sgl(struct sge_txq *, struct sgl *); 225static int write_txpkt_wr(struct port_info *, struct sge_txq *, struct mbuf *, 226 struct sgl *); 227static int add_to_txpkts(struct port_info *, struct sge_txq *, struct txpkts *, 228 struct mbuf *, struct sgl *); 229static void write_txpkts_wr(struct sge_txq *, struct txpkts *); 230static inline void write_ulp_cpl_sgl(struct port_info *, struct sge_txq *, 231 struct txpkts *, struct mbuf *, struct sgl *); 232static int write_sgl_to_txd(struct sge_eq *, struct sgl *, caddr_t *); | 222static inline void get_pkt_gl(struct mbuf *, struct sglist *); 223static inline u_int txpkt_len16(u_int, u_int); 224static inline u_int txpkts0_len16(u_int); 225static inline u_int txpkts1_len16(void); 226static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *, 227 struct mbuf *, u_int); 228static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int); 229static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int); 230static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *, 231 struct mbuf *, const struct txpkts *, u_int); 232static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); |
233static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); | 233static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); |
234static inline void ring_eq_db(struct adapter *, struct sge_eq *); 235static inline int reclaimable(struct sge_eq *); 236static int reclaim_tx_descs(struct sge_txq *, int, int); 237static void write_eqflush_wr(struct sge_eq *); 238static __be64 get_flit(bus_dma_segment_t *, int, int); | 234static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); 235static inline uint16_t read_hw_cidx(struct sge_eq *); 236static inline u_int reclaimable_tx_desc(struct sge_eq *); 237static inline u_int total_available_tx_desc(struct sge_eq *); 238static u_int reclaim_tx_descs(struct sge_txq *, u_int); 239static void tx_reclaim(void *, int); 240static __be64 get_flit(struct sglist_seg *, int, int); |
239static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 240 struct mbuf *); 241static int handle_fw_msg(struct sge_iq *, const struct rss_header *, 242 struct mbuf *); | 241static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 242 struct mbuf *); 243static int handle_fw_msg(struct sge_iq *, const struct rss_header *, 244 struct mbuf *); |
245static void wrq_tx_drain(void *, int); 246static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); |
|
243 244static int sysctl_uint16(SYSCTL_HANDLER_ARGS); 245static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); 246 247static counter_u64_t extfree_refs; 248static counter_u64_t extfree_rels; 249 250/* --- 1527 unchanged lines hidden (view full) --- 1778 } else 1779#endif 1780 ifp->if_input(ifp, m0); 1781 1782 return (0); 1783} 1784 1785/* | 247 248static int sysctl_uint16(SYSCTL_HANDLER_ARGS); 249static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); 250 251static counter_u64_t extfree_refs; 252static counter_u64_t extfree_rels; 253 254/* --- 1527 unchanged lines hidden (view full) --- 1782 } else 1783#endif 1784 ifp->if_input(ifp, m0); 1785 1786 return (0); 1787} 1788 1789/* |
1790 * Must drain the wrq or make sure that someone else will. 1791 */ 1792static void 1793wrq_tx_drain(void *arg, int n) 1794{ 1795 struct sge_wrq *wrq = arg; 1796 struct sge_eq *eq = &wrq->eq; 1797 1798 EQ_LOCK(eq); 1799 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 1800 drain_wrq_wr_list(wrq->adapter, wrq); 1801 EQ_UNLOCK(eq); 1802} 1803 1804static void 1805drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) 1806{ 1807 struct sge_eq *eq = &wrq->eq; 1808 u_int available, dbdiff; /* # of hardware descriptors */ 1809 u_int n; 1810 struct wrqe *wr; 1811 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 1812 1813 EQ_LOCK_ASSERT_OWNED(eq); 1814 MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); 1815 wr = STAILQ_FIRST(&wrq->wr_list); 1816 MPASS(wr != NULL); /* Must be called with something useful to do */ 1817 dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); 1818 1819 do { 1820 eq->cidx = read_hw_cidx(eq); 1821 if (eq->pidx == eq->cidx) 1822 available = eq->sidx - 1; 1823 else 1824 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 1825 1826 MPASS(wr->wrq == wrq); 1827 n = howmany(wr->wr_len, EQ_ESIZE); 1828 if (available < n) 1829 return; 1830 1831 dst = (void *)&eq->desc[eq->pidx]; 1832 if (__predict_true(eq->sidx - eq->pidx > n)) { 1833 /* Won't wrap, won't end exactly at the status page. */ 1834 bcopy(&wr->wr[0], dst, wr->wr_len); 1835 eq->pidx += n; 1836 } else { 1837 int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; 1838 1839 bcopy(&wr->wr[0], dst, first_portion); 1840 if (wr->wr_len > first_portion) { 1841 bcopy(&wr->wr[first_portion], &eq->desc[0], 1842 wr->wr_len - first_portion); 1843 } 1844 eq->pidx = n - (eq->sidx - eq->pidx); 1845 } 1846 1847 if (available < eq->sidx / 4 && 1848 atomic_cmpset_int(&eq->equiq, 0, 1)) { 1849 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 1850 F_FW_WR_EQUEQ); 1851 eq->equeqidx = eq->pidx; 1852 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 1853 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 1854 eq->equeqidx = eq->pidx; 1855 } 1856 1857 dbdiff += n; 1858 if (dbdiff >= 16) { 1859 ring_eq_db(sc, eq, dbdiff); 1860 dbdiff = 0; 1861 } 1862 1863 STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 1864 free_wrqe(wr); 1865 MPASS(wrq->nwr_pending > 0); 1866 wrq->nwr_pending--; 1867 MPASS(wrq->ndesc_needed >= n); 1868 wrq->ndesc_needed -= n; 1869 } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); 1870 1871 if (dbdiff) 1872 ring_eq_db(sc, eq, dbdiff); 1873} 1874 1875/* |
|
1786 * Doesn't fail. Holds on to work requests it can't send right away. 1787 */ 1788void 1789t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) 1790{ | 1876 * Doesn't fail. Holds on to work requests it can't send right away. 1877 */ 1878void 1879t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) 1880{ |
1881#ifdef INVARIANTS |
|
1791 struct sge_eq *eq = &wrq->eq; | 1882 struct sge_eq *eq = &wrq->eq; |
1792 int can_reclaim; 1793 caddr_t dst; | 1883#endif |
1794 | 1884 |
1795 TXQ_LOCK_ASSERT_OWNED(wrq); | 1885 EQ_LOCK_ASSERT_OWNED(eq); 1886 MPASS(wr != NULL); 1887 MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); 1888 MPASS((wr->wr_len & 0x7) == 0); 1889 1890 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); 1891 wrq->nwr_pending++; 1892 wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); 1893 1894 if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) 1895 return; /* commit_wrq_wr will drain wr_list as well. */ 1896 1897 drain_wrq_wr_list(sc, wrq); 1898 1899 /* Doorbell must have caught up to the pidx. */ 1900 MPASS(eq->pidx == eq->dbidx); 1901} 1902 1903void 1904t4_update_fl_bufsize(struct ifnet *ifp) 1905{ 1906 struct port_info *pi = ifp->if_softc; 1907 struct adapter *sc = pi->adapter; 1908 struct sge_rxq *rxq; |
1796#ifdef TCP_OFFLOAD | 1909#ifdef TCP_OFFLOAD |
1797 KASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD || 1798 (eq->flags & EQ_TYPEMASK) == EQ_CTRL, 1799 ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK)); 1800#else 1801 KASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL, 1802 ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK)); | 1910 struct sge_ofld_rxq *ofld_rxq; |
1803#endif | 1911#endif |
1912 struct sge_fl *fl; 1913 int i, maxp, mtu = ifp->if_mtu; |
|
1804 | 1914 |
1805 if (__predict_true(wr != NULL)) 1806 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); | 1915 maxp = mtu_to_max_payload(sc, mtu, 0); 1916 for_each_rxq(pi, i, rxq) { 1917 fl = &rxq->fl; |
1807 | 1918 |
1808 can_reclaim = reclaimable(eq); 1809 if (__predict_false(eq->flags & EQ_STALLED)) { 1810 if (eq->avail + can_reclaim < tx_resume_threshold(eq)) 1811 return; 1812 eq->flags &= ~EQ_STALLED; 1813 eq->unstalled++; | 1919 FL_LOCK(fl); 1920 find_best_refill_source(sc, fl, maxp); 1921 FL_UNLOCK(fl); |
1814 } | 1922 } |
1815 eq->cidx += can_reclaim; 1816 eq->avail += can_reclaim; 1817 if (__predict_false(eq->cidx >= eq->cap)) 1818 eq->cidx -= eq->cap; | 1923#ifdef TCP_OFFLOAD 1924 maxp = mtu_to_max_payload(sc, mtu, 1); 1925 for_each_ofld_rxq(pi, i, ofld_rxq) { 1926 fl = &ofld_rxq->fl; |
1819 | 1927 |
1820 while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) { 1821 int ndesc; | 1928 FL_LOCK(fl); 1929 find_best_refill_source(sc, fl, maxp); 1930 FL_UNLOCK(fl); 1931 } 1932#endif 1933} |
1822 | 1934 |
1823 if (__predict_false(wr->wr_len < 0 || 1824 wr->wr_len > SGE_MAX_WR_LEN || (wr->wr_len & 0x7))) { | 1935static inline int 1936mbuf_nsegs(struct mbuf *m) 1937{ |
1825 | 1938 |
1826#ifdef INVARIANTS 1827 panic("%s: work request with length %d", __func__, 1828 wr->wr_len); 1829#endif 1830#ifdef KDB 1831 kdb_backtrace(); 1832#endif 1833 log(LOG_ERR, "%s: %s work request with length %d", 1834 device_get_nameunit(sc->dev), __func__, wr->wr_len); 1835 STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 1836 free_wrqe(wr); 1837 continue; 1838 } | 1939 M_ASSERTPKTHDR(m); 1940 KASSERT(m->m_pkthdr.l5hlen > 0, 1941 ("%s: mbuf %p missing information on # of segments.", __func__, m)); |
1839 | 1942 |
1840 ndesc = howmany(wr->wr_len, EQ_ESIZE); 1841 if (eq->avail < ndesc) { 1842 wrq->no_desc++; 1843 break; 1844 } | 1943 return (m->m_pkthdr.l5hlen); 1944} |
1845 | 1945 |
1846 dst = (void *)&eq->desc[eq->pidx]; 1847 copy_to_txd(eq, wrtod(wr), &dst, wr->wr_len); | 1946static inline void 1947set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) 1948{ |
1848 | 1949 |
1849 eq->pidx += ndesc; 1850 eq->avail -= ndesc; 1851 if (__predict_false(eq->pidx >= eq->cap)) 1852 eq->pidx -= eq->cap; | 1950 M_ASSERTPKTHDR(m); 1951 m->m_pkthdr.l5hlen = nsegs; 1952} |
1853 | 1953 |
1854 eq->pending += ndesc; 1855 if (eq->pending >= 8) 1856 ring_eq_db(sc, eq); | 1954static inline int 1955mbuf_len16(struct mbuf *m) 1956{ 1957 int n; |
1857 | 1958 |
1858 wrq->tx_wrs++; 1859 STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 1860 free_wrqe(wr); | 1959 M_ASSERTPKTHDR(m); 1960 n = m->m_pkthdr.PH_loc.eigth[0]; 1961 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); |
1861 | 1962 |
1862 if (eq->avail < 8) { 1863 can_reclaim = reclaimable(eq); 1864 eq->cidx += can_reclaim; 1865 eq->avail += can_reclaim; 1866 if (__predict_false(eq->cidx >= eq->cap)) 1867 eq->cidx -= eq->cap; 1868 } 1869 } | 1963 return (n); 1964} |
1870 | 1965 |
1871 if (eq->pending) 1872 ring_eq_db(sc, eq); | 1966static inline void 1967set_mbuf_len16(struct mbuf *m, uint8_t len16) 1968{ |
1873 | 1969 |
1874 if (wr != NULL) { 1875 eq->flags |= EQ_STALLED; 1876 if (callout_pending(&eq->tx_callout) == 0) 1877 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1878 } | 1970 M_ASSERTPKTHDR(m); 1971 m->m_pkthdr.PH_loc.eigth[0] = len16; |
1879} 1880 | 1972} 1973 |
1881/* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 1882#define TXPKTS_PKT_HDR ((\ 1883 sizeof(struct ulp_txpkt) + \ 1884 sizeof(struct ulptx_idata) + \ 1885 sizeof(struct cpl_tx_pkt_core) \ 1886 ) / 8) | 1974static inline int 1975needs_tso(struct mbuf *m) 1976{ |
1887 | 1977 |
1888/* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 1889#define TXPKTS_WR_HDR (\ 1890 sizeof(struct fw_eth_tx_pkts_wr) / 8 + \ 1891 TXPKTS_PKT_HDR) | 1978 M_ASSERTPKTHDR(m); |
1892 | 1979 |
1893/* Header of a tx WR, before SGL of first packet (in flits) */ 1894#define TXPKT_WR_HDR ((\ 1895 sizeof(struct fw_eth_tx_pkt_wr) + \ 1896 sizeof(struct cpl_tx_pkt_core) \ 1897 ) / 8 ) | 1980 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1981 KASSERT(m->m_pkthdr.tso_segsz > 0, 1982 ("%s: TSO requested in mbuf %p but MSS not provided", 1983 __func__, m)); 1984 return (1); 1985 } |
1898 | 1986 |
1899/* Header of a tx LSO WR, before SGL of first packet (in flits) */ 1900#define TXPKT_LSO_WR_HDR ((\ 1901 sizeof(struct fw_eth_tx_pkt_wr) + \ 1902 sizeof(struct cpl_tx_pkt_lso_core) + \ 1903 sizeof(struct cpl_tx_pkt_core) \ 1904 ) / 8 ) | 1987 return (0); 1988} |
1905 | 1989 |
1906int 1907t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m) | 1990static inline int 1991needs_l3_csum(struct mbuf *m) |
1908{ | 1992{ |
1909 struct port_info *pi = (void *)ifp->if_softc; 1910 struct adapter *sc = pi->adapter; 1911 struct sge_eq *eq = &txq->eq; 1912 struct buf_ring *br = txq->br; 1913 struct mbuf *next; 1914 int rc, coalescing, can_reclaim; 1915 struct txpkts txpkts; 1916 struct sgl sgl; | |
1917 | 1993 |
1918 TXQ_LOCK_ASSERT_OWNED(txq); 1919 KASSERT(m, ("%s: called with nothing to do.", __func__)); 1920 KASSERT((eq->flags & EQ_TYPEMASK) == EQ_ETH, 1921 ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK)); | 1994 M_ASSERTPKTHDR(m); |
1922 | 1995 |
1923 prefetch(&eq->desc[eq->pidx]); 1924 prefetch(&txq->sdesc[eq->pidx]); | 1996 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) 1997 return (1); 1998 return (0); 1999} |
1925 | 2000 |
1926 txpkts.npkt = 0;/* indicates there's nothing in txpkts */ 1927 coalescing = 0; | 2001static inline int 2002needs_l4_csum(struct mbuf *m) 2003{ |
1928 | 2004 |
1929 can_reclaim = reclaimable(eq); 1930 if (__predict_false(eq->flags & EQ_STALLED)) { 1931 if (eq->avail + can_reclaim < tx_resume_threshold(eq)) { 1932 txq->m = m; 1933 return (0); 1934 } 1935 eq->flags &= ~EQ_STALLED; 1936 eq->unstalled++; 1937 } | 2005 M_ASSERTPKTHDR(m); |
1938 | 2006 |
1939 if (__predict_false(eq->flags & EQ_DOOMED)) { 1940 m_freem(m); 1941 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1942 m_freem(m); 1943 return (ENETDOWN); | 2007 if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | 2008 CSUM_TCP_IPV6 | CSUM_TSO)) 2009 return (1); 2010 return (0); 2011} 2012 2013static inline int 2014needs_vlan_insertion(struct mbuf *m) 2015{ 2016 2017 M_ASSERTPKTHDR(m); 2018 2019 if (m->m_flags & M_VLANTAG) { 2020 KASSERT(m->m_pkthdr.ether_vtag != 0, 2021 ("%s: HWVLAN requested in mbuf %p but tag not provided", 2022 __func__, m)); 2023 return (1); |
1944 } | 2024 } |
2025 return (0); 2026} |
|
1945 | 2027 |
1946 if (eq->avail < 8 && can_reclaim) 1947 reclaim_tx_descs(txq, can_reclaim, 32); | 2028static void * 2029m_advance(struct mbuf **pm, int *poffset, int len) 2030{ 2031 struct mbuf *m = *pm; 2032 int offset = *poffset; 2033 uintptr_t p = 0; |
1948 | 2034 |
1949 for (; m; m = next ? next : drbr_dequeue(ifp, br)) { | 2035 MPASS(len > 0); |
1950 | 2036 |
1951 if (eq->avail < 8) | 2037 while (len) { 2038 if (offset + len < m->m_len) { 2039 offset += len; 2040 p = mtod(m, uintptr_t) + offset; |
1952 break; | 2041 break; |
2042 } 2043 len -= m->m_len - offset; 2044 m = m->m_next; 2045 offset = 0; 2046 MPASS(m != NULL); 2047 } 2048 *poffset = offset; 2049 *pm = m; 2050 return ((void *)p); 2051} |
|
1953 | 2052 |
1954 next = m->m_nextpkt; 1955 m->m_nextpkt = NULL; | 2053static inline int 2054same_paddr(char *a, char *b) 2055{ |
1956 | 2056 |
1957 if (next || buf_ring_peek(br)) 1958 coalescing = 1; | 2057 if (a == b) 2058 return (1); 2059 else if (a != NULL && b != NULL) { 2060 vm_offset_t x = (vm_offset_t)a; 2061 vm_offset_t y = (vm_offset_t)b; |
1959 | 2062 |
1960 rc = get_pkt_sgl(txq, &m, &sgl, coalescing); 1961 if (rc != 0) { 1962 if (rc == ENOMEM) { | 2063 if ((x & PAGE_MASK) == (y & PAGE_MASK) && 2064 pmap_kextract(x) == pmap_kextract(y)) 2065 return (1); 2066 } |
1963 | 2067 |
1964 /* Short of resources, suspend tx */ | 2068 return (0); 2069} |
1965 | 2070 |
1966 m->m_nextpkt = next; 1967 break; 1968 } | 2071/* 2072 * Can deal with empty mbufs in the chain that have m_len = 0, but the chain 2073 * must have at least one mbuf that's not empty. 2074 */ 2075static inline int 2076count_mbuf_nsegs(struct mbuf *m) 2077{ 2078 char *prev_end, *start; 2079 int len, nsegs; |
1969 | 2080 |
1970 /* 1971 * Unrecoverable error for this packet, throw it away 1972 * and move on to the next. get_pkt_sgl may already 1973 * have freed m (it will be NULL in that case and the 1974 * m_freem here is still safe). 1975 */ | 2081 MPASS(m != NULL); |
1976 | 2082 |
1977 m_freem(m); | 2083 nsegs = 0; 2084 prev_end = NULL; 2085 for (; m; m = m->m_next) { 2086 2087 len = m->m_len; 2088 if (__predict_false(len == 0)) |
1978 continue; | 2089 continue; |
1979 } | 2090 start = mtod(m, char *); |
1980 | 2091 |
1981 if (coalescing && 1982 add_to_txpkts(pi, txq, &txpkts, m, &sgl) == 0) { | 2092 nsegs += sglist_count(start, len); 2093 if (same_paddr(prev_end, start)) 2094 nsegs--; 2095 prev_end = start + len; 2096 } |
1983 | 2097 |
1984 /* Successfully absorbed into txpkts */ | 2098 MPASS(nsegs > 0); 2099 return (nsegs); 2100} |
1985 | 2101 |
1986 write_ulp_cpl_sgl(pi, txq, &txpkts, m, &sgl); 1987 goto doorbell; | 2102/* 2103 * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: 2104 * a) caller can assume it's been freed if this function returns with an error. 2105 * b) it may get defragged up if the gather list is too long for the hardware. 2106 */ 2107int 2108parse_pkt(struct mbuf **mp) 2109{ 2110 struct mbuf *m0 = *mp, *m; 2111 int rc, nsegs, defragged = 0, offset; 2112 struct ether_header *eh; 2113 void *l3hdr; 2114#if defined(INET) || defined(INET6) 2115 struct tcphdr *tcp; 2116#endif 2117 uint16_t eh_type; 2118 2119 M_ASSERTPKTHDR(m0); 2120 if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { 2121 rc = EINVAL; 2122fail: 2123 m_freem(m0); 2124 *mp = NULL; 2125 return (rc); 2126 } 2127restart: 2128 /* 2129 * First count the number of gather list segments in the payload. 2130 * Defrag the mbuf if nsegs exceeds the hardware limit. 2131 */ 2132 M_ASSERTPKTHDR(m0); 2133 MPASS(m0->m_pkthdr.len > 0); 2134 nsegs = count_mbuf_nsegs(m0); 2135 if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { 2136 if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { 2137 rc = EFBIG; 2138 goto fail; |
1988 } | 2139 } |
2140 *mp = m0 = m; /* update caller's copy after defrag */ 2141 goto restart; 2142 } |
|
1989 | 2143 |
1990 /* 1991 * We weren't coalescing to begin with, or current frame could 1992 * not be coalesced (add_to_txpkts flushes txpkts if a frame 1993 * given to it can't be coalesced). Either way there should be 1994 * nothing in txpkts. 1995 */ 1996 KASSERT(txpkts.npkt == 0, 1997 ("%s: txpkts not empty: %d", __func__, txpkts.npkt)); | 2144 if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) { 2145 m0 = m_pullup(m0, m0->m_pkthdr.len); 2146 if (m0 == NULL) { 2147 /* Should have left well enough alone. */ 2148 rc = EFBIG; 2149 goto fail; 2150 } 2151 *mp = m0; /* update caller's copy after pullup */ 2152 goto restart; 2153 } 2154 set_mbuf_nsegs(m0, nsegs); 2155 set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); |
1998 | 2156 |
1999 /* We're sending out individual packets now */ 2000 coalescing = 0; | 2157 if (!needs_tso(m0)) 2158 return (0); |
2001 | 2159 |
2002 if (eq->avail < 8) 2003 reclaim_tx_descs(txq, 0, 8); 2004 rc = write_txpkt_wr(pi, txq, m, &sgl); 2005 if (rc != 0) { | 2160 m = m0; 2161 eh = mtod(m, struct ether_header *); 2162 eh_type = ntohs(eh->ether_type); 2163 if (eh_type == ETHERTYPE_VLAN) { 2164 struct ether_vlan_header *evh = (void *)eh; |
2006 | 2165 |
2007 /* Short of hardware descriptors, suspend tx */ | 2166 eh_type = ntohs(evh->evl_proto); 2167 m0->m_pkthdr.l2hlen = sizeof(*evh); 2168 } else 2169 m0->m_pkthdr.l2hlen = sizeof(*eh); |
2008 | 2170 |
2009 /* 2010 * This is an unlikely but expensive failure. We've 2011 * done all the hard work (DMA mappings etc.) and now we 2012 * can't send out the packet. What's worse, we have to 2013 * spend even more time freeing up everything in sgl. 2014 */ 2015 txq->no_desc++; 2016 free_pkt_sgl(txq, &sgl); | 2171 offset = 0; 2172 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); |
2017 | 2173 |
2018 m->m_nextpkt = next; 2019 break; 2020 } | 2174 switch (eh_type) { 2175#ifdef INET6 2176 case ETHERTYPE_IPV6: 2177 { 2178 struct ip6_hdr *ip6 = l3hdr; |
2021 | 2179 |
2022 ETHER_BPF_MTAP(ifp, m); 2023 if (sgl.nsegs == 0) 2024 m_freem(m); 2025doorbell: 2026 if (eq->pending >= 8) 2027 ring_eq_db(sc, eq); | 2180 MPASS(ip6->ip6_nxt == IPPROTO_TCP); |
2028 | 2181 |
2029 can_reclaim = reclaimable(eq); 2030 if (can_reclaim >= 32) 2031 reclaim_tx_descs(txq, can_reclaim, 64); | 2182 m0->m_pkthdr.l3hlen = sizeof(*ip6); 2183 break; |
2032 } | 2184 } |
2185#endif 2186#ifdef INET 2187 case ETHERTYPE_IP: 2188 { 2189 struct ip *ip = l3hdr; |
|
2033 | 2190 |
2034 if (txpkts.npkt > 0) 2035 write_txpkts_wr(txq, &txpkts); | 2191 m0->m_pkthdr.l3hlen = ip->ip_hl * 4; 2192 break; 2193 } 2194#endif 2195 default: 2196 panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled" 2197 " with the same INET/INET6 options as the kernel.", 2198 __func__, eh_type); 2199 } |
2036 | 2200 |
2037 /* 2038 * m not NULL means there was an error but we haven't thrown it away. 2039 * This can happen when we're short of tx descriptors (no_desc) or maybe 2040 * even DMA maps (no_dmamap). Either way, a credit flush and reclaim 2041 * will get things going again. 2042 */ 2043 if (m && !(eq->flags & EQ_CRFLUSHED)) { 2044 struct tx_sdesc *txsd = &txq->sdesc[eq->pidx]; | 2201#if defined(INET) || defined(INET6) 2202 tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); 2203 m0->m_pkthdr.l4hlen = tcp->th_off * 4; 2204#endif 2205 MPASS(m0 == *mp); 2206 return (0); 2207} |
2045 | 2208 |
2046 /* 2047 * If EQ_CRFLUSHED is not set then we know we have at least one 2048 * available descriptor because any WR that reduces eq->avail to 2049 * 0 also sets EQ_CRFLUSHED. 2050 */ 2051 KASSERT(eq->avail > 0, ("%s: no space for eqflush.", __func__)); | 2209void * 2210start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) 2211{ 2212 struct sge_eq *eq = &wrq->eq; 2213 struct adapter *sc = wrq->adapter; 2214 int ndesc, available; 2215 struct wrqe *wr; 2216 void *w; |
2052 | 2217 |
2053 txsd->desc_used = 1; 2054 txsd->credits = 0; 2055 write_eqflush_wr(eq); | 2218 MPASS(len16 > 0); 2219 ndesc = howmany(len16, EQ_ESIZE / 16); 2220 MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); 2221 2222 EQ_LOCK(eq); 2223 2224 if (!STAILQ_EMPTY(&wrq->wr_list)) 2225 drain_wrq_wr_list(sc, wrq); 2226 2227 if (!STAILQ_EMPTY(&wrq->wr_list)) { 2228slowpath: 2229 EQ_UNLOCK(eq); 2230 wr = alloc_wrqe(len16 * 16, wrq); 2231 if (__predict_false(wr == NULL)) 2232 return (NULL); 2233 cookie->pidx = -1; 2234 cookie->ndesc = ndesc; 2235 return (&wr->wr); |
2056 } | 2236 } |
2057 txq->m = m; | |
2058 | 2237 |
2059 if (eq->pending) 2060 ring_eq_db(sc, eq); | 2238 eq->cidx = read_hw_cidx(eq); 2239 if (eq->pidx == eq->cidx) 2240 available = eq->sidx - 1; 2241 else 2242 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2243 if (available < ndesc) 2244 goto slowpath; |
2061 | 2245 |
2062 reclaim_tx_descs(txq, 0, 128); | 2246 cookie->pidx = eq->pidx; 2247 cookie->ndesc = ndesc; 2248 TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); |
2063 | 2249 |
2064 if (eq->flags & EQ_STALLED && callout_pending(&eq->tx_callout) == 0) 2065 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); | 2250 w = &eq->desc[eq->pidx]; 2251 IDXINCR(eq->pidx, ndesc, eq->sidx); 2252 if (__predict_false(eq->pidx < ndesc - 1)) { 2253 w = &wrq->ss[0]; 2254 wrq->ss_pidx = cookie->pidx; 2255 wrq->ss_len = len16 * 16; 2256 } |
2066 | 2257 |
2067 return (0); | 2258 EQ_UNLOCK(eq); 2259 2260 return (w); |
2068} 2069 2070void | 2261} 2262 2263void |
2071t4_update_fl_bufsize(struct ifnet *ifp) | 2264commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) |
2072{ | 2265{ |
2073 struct port_info *pi = ifp->if_softc; 2074 struct adapter *sc = pi->adapter; 2075 struct sge_rxq *rxq; 2076#ifdef TCP_OFFLOAD 2077 struct sge_ofld_rxq *ofld_rxq; 2078#endif 2079 struct sge_fl *fl; 2080 int i, maxp, mtu = ifp->if_mtu; | 2266 struct sge_eq *eq = &wrq->eq; 2267 struct adapter *sc = wrq->adapter; 2268 int ndesc, pidx; 2269 struct wrq_cookie *prev, *next; |
2081 | 2270 |
2082 maxp = mtu_to_max_payload(sc, mtu, 0); 2083 for_each_rxq(pi, i, rxq) { 2084 fl = &rxq->fl; | 2271 if (cookie->pidx == -1) { 2272 struct wrqe *wr = __containerof(w, struct wrqe, wr); |
2085 | 2273 |
2086 FL_LOCK(fl); 2087 find_best_refill_source(sc, fl, maxp); 2088 FL_UNLOCK(fl); | 2274 t4_wrq_tx(sc, wr); 2275 return; |
2089 } | 2276 } |
2090#ifdef TCP_OFFLOAD 2091 maxp = mtu_to_max_payload(sc, mtu, 1); 2092 for_each_ofld_rxq(pi, i, ofld_rxq) { 2093 fl = &ofld_rxq->fl; | |
2094 | 2277 |
2095 FL_LOCK(fl); 2096 find_best_refill_source(sc, fl, maxp); 2097 FL_UNLOCK(fl); | 2278 ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ 2279 pidx = cookie->pidx; 2280 MPASS(pidx >= 0 && pidx < eq->sidx); 2281 if (__predict_false(w == &wrq->ss[0])) { 2282 int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; 2283 2284 MPASS(wrq->ss_len > n); /* WR had better wrap around. */ 2285 bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); 2286 bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); 2287 wrq->tx_wrs_ss++; 2288 } else 2289 wrq->tx_wrs_direct++; 2290 2291 EQ_LOCK(eq); 2292 prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); 2293 next = TAILQ_NEXT(cookie, link); 2294 if (prev == NULL) { 2295 MPASS(pidx == eq->dbidx); 2296 if (next == NULL || ndesc >= 16) 2297 ring_eq_db(wrq->adapter, eq, ndesc); 2298 else { 2299 MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); 2300 next->pidx = pidx; 2301 next->ndesc += ndesc; 2302 } 2303 } else { 2304 MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); 2305 prev->ndesc += ndesc; |
2098 } | 2306 } |
2307 TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); 2308 2309 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2310 drain_wrq_wr_list(sc, wrq); 2311 2312#ifdef INVARIANTS 2313 if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { 2314 /* Doorbell must have caught up to the pidx. */ 2315 MPASS(wrq->eq.pidx == wrq->eq.dbidx); 2316 } |
|
2099#endif | 2317#endif |
2318 EQ_UNLOCK(eq); |
|
2100} 2101 | 2319} 2320 |
2102int 2103can_resume_tx(struct sge_eq *eq) | 2321static u_int 2322can_resume_eth_tx(struct mp_ring *r) |
2104{ | 2323{ |
2324 struct sge_eq *eq = r->cookie; |
|
2105 | 2325 |
2106 return (eq->avail + reclaimable(eq) >= tx_resume_threshold(eq)); | 2326 return (total_available_tx_desc(eq) > eq->sidx / 8); |
2107} 2108 | 2327} 2328 |
2329static inline int 2330cannot_use_txpkts(struct mbuf *m) 2331{ 2332 /* maybe put a GL limit too, to avoid silliness? */ 2333 2334 return (needs_tso(m)); 2335} 2336 2337/* 2338 * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to 2339 * be consumed. Return the actual number consumed. 0 indicates a stall. 2340 */ 2341static u_int 2342eth_tx(struct mp_ring *r, u_int cidx, u_int pidx) 2343{ 2344 struct sge_txq *txq = r->cookie; 2345 struct sge_eq *eq = &txq->eq; 2346 struct ifnet *ifp = txq->ifp; 2347 struct port_info *pi = (void *)ifp->if_softc; 2348 struct adapter *sc = pi->adapter; 2349 u_int total, remaining; /* # of packets */ 2350 u_int available, dbdiff; /* # of hardware descriptors */ 2351 u_int n, next_cidx; 2352 struct mbuf *m0, *tail; 2353 struct txpkts txp; 2354 struct fw_eth_tx_pkts_wr *wr; /* any fw WR struct will do */ 2355 2356 remaining = IDXDIFF(pidx, cidx, r->size); 2357 MPASS(remaining > 0); /* Must not be called without work to do. */ 2358 total = 0; 2359 2360 TXQ_LOCK(txq); 2361 if (__predict_false((eq->flags & EQ_ENABLED) == 0)) { 2362 while (cidx != pidx) { 2363 m0 = r->items[cidx]; 2364 m_freem(m0); 2365 if (++cidx == r->size) 2366 cidx = 0; 2367 } 2368 reclaim_tx_descs(txq, 2048); 2369 total = remaining; 2370 goto done; 2371 } 2372 2373 /* How many hardware descriptors do we have readily available. */ 2374 if (eq->pidx == eq->cidx) 2375 available = eq->sidx - 1; 2376 else 2377 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2378 dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); 2379 2380 while (remaining > 0) { 2381 2382 m0 = r->items[cidx]; 2383 M_ASSERTPKTHDR(m0); 2384 MPASS(m0->m_nextpkt == NULL); 2385 2386 if (available < SGE_MAX_WR_NDESC) { 2387 available += reclaim_tx_descs(txq, 64); 2388 if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) 2389 break; /* out of descriptors */ 2390 } 2391 2392 next_cidx = cidx + 1; 2393 if (__predict_false(next_cidx == r->size)) 2394 next_cidx = 0; 2395 2396 wr = (void *)&eq->desc[eq->pidx]; 2397 if (remaining > 1 && 2398 try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) { 2399 2400 /* pkts at cidx, next_cidx should both be in txp. */ 2401 MPASS(txp.npkt == 2); 2402 tail = r->items[next_cidx]; 2403 MPASS(tail->m_nextpkt == NULL); 2404 ETHER_BPF_MTAP(ifp, m0); 2405 ETHER_BPF_MTAP(ifp, tail); 2406 m0->m_nextpkt = tail; 2407 2408 if (__predict_false(++next_cidx == r->size)) 2409 next_cidx = 0; 2410 2411 while (next_cidx != pidx) { 2412 if (add_to_txpkts(r->items[next_cidx], &txp, 2413 available) != 0) 2414 break; 2415 tail->m_nextpkt = r->items[next_cidx]; 2416 tail = tail->m_nextpkt; 2417 ETHER_BPF_MTAP(ifp, tail); 2418 if (__predict_false(++next_cidx == r->size)) 2419 next_cidx = 0; 2420 } 2421 2422 n = write_txpkts_wr(txq, wr, m0, &txp, available); 2423 total += txp.npkt; 2424 remaining -= txp.npkt; 2425 } else { 2426 total++; 2427 remaining--; 2428 n = write_txpkt_wr(txq, (void *)wr, m0, available); 2429 ETHER_BPF_MTAP(ifp, m0); 2430 } 2431 MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC); 2432 2433 available -= n; 2434 dbdiff += n; 2435 IDXINCR(eq->pidx, n, eq->sidx); 2436 2437 if (total_available_tx_desc(eq) < eq->sidx / 4 && 2438 atomic_cmpset_int(&eq->equiq, 0, 1)) { 2439 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 2440 F_FW_WR_EQUEQ); 2441 eq->equeqidx = eq->pidx; 2442 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 2443 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 2444 eq->equeqidx = eq->pidx; 2445 } 2446 2447 if (dbdiff >= 16 && remaining >= 4) { 2448 ring_eq_db(sc, eq, dbdiff); 2449 available += reclaim_tx_descs(txq, 4 * dbdiff); 2450 dbdiff = 0; 2451 } 2452 2453 cidx = next_cidx; 2454 } 2455 if (dbdiff != 0) { 2456 ring_eq_db(sc, eq, dbdiff); 2457 reclaim_tx_descs(txq, 32); 2458 } 2459done: 2460 TXQ_UNLOCK(txq); 2461 2462 return (total); 2463} 2464 |
|
2109static inline void 2110init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, 2111 int qsize) 2112{ 2113 2114 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, 2115 ("%s: bad tmr_idx %d", __func__, tmr_idx)); 2116 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ --- 31 unchanged lines hidden (view full) --- 2148 uint16_t iqid, char *name) 2149{ 2150 KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan)); 2151 KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); 2152 2153 eq->flags = eqtype & EQ_TYPEMASK; 2154 eq->tx_chan = tx_chan; 2155 eq->iqid = iqid; | 2465static inline void 2466init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, 2467 int qsize) 2468{ 2469 2470 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, 2471 ("%s: bad tmr_idx %d", __func__, tmr_idx)); 2472 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ --- 31 unchanged lines hidden (view full) --- 2504 uint16_t iqid, char *name) 2505{ 2506 KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan)); 2507 KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); 2508 2509 eq->flags = eqtype & EQ_TYPEMASK; 2510 eq->tx_chan = tx_chan; 2511 eq->iqid = iqid; |
2156 eq->qsize = qsize; | 2512 eq->sidx = qsize - spg_len / EQ_ESIZE; |
2157 strlcpy(eq->lockname, name, sizeof(eq->lockname)); | 2513 strlcpy(eq->lockname, name, sizeof(eq->lockname)); |
2158 2159 TASK_INIT(&eq->tx_task, 0, t4_tx_task, eq); 2160 callout_init(&eq->tx_callout, CALLOUT_MPSAFE); | |
2161} 2162 2163static int 2164alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, 2165 bus_dmamap_t *map, bus_addr_t *pa, void **va) 2166{ 2167 int rc; 2168 --- 672 unchanged lines hidden (view full) --- 2841} 2842#endif 2843 2844static int 2845ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 2846{ 2847 int rc, cntxt_id; 2848 struct fw_eq_ctrl_cmd c; | 2514} 2515 2516static int 2517alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, 2518 bus_dmamap_t *map, bus_addr_t *pa, void **va) 2519{ 2520 int rc; 2521 --- 672 unchanged lines hidden (view full) --- 3194} 3195#endif 3196 3197static int 3198ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 3199{ 3200 int rc, cntxt_id; 3201 struct fw_eq_ctrl_cmd c; |
3202 int qsize = eq->sidx + spg_len / EQ_ESIZE; |
|
2849 2850 bzero(&c, sizeof(c)); 2851 2852 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 2853 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 2854 V_FW_EQ_CTRL_CMD_VFN(0)); 2855 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | 2856 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); | 3203 3204 bzero(&c, sizeof(c)); 3205 3206 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 3207 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 3208 V_FW_EQ_CTRL_CMD_VFN(0)); 3209 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | 3210 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); |
2857 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* XXX */ | 3211 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); |
2858 c.physeqid_pkd = htobe32(0); 2859 c.fetchszm_to_iqid = | 3212 c.physeqid_pkd = htobe32(0); 3213 c.fetchszm_to_iqid = |
2860 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | | 3214 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | |
2861 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 2862 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 2863 c.dcaen_to_eqsize = 2864 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 2865 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | | 3215 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 3216 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 3217 c.dcaen_to_eqsize = 3218 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 3219 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | |
2866 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 2867 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); | 3220 V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); |
2868 c.eqaddr = htobe64(eq->ba); 2869 2870 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 2871 if (rc != 0) { 2872 device_printf(sc->dev, 2873 "failed to create control queue %d: %d\n", eq->tx_chan, rc); 2874 return (rc); 2875 } --- 9 unchanged lines hidden (view full) --- 2885 return (rc); 2886} 2887 2888static int 2889eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 2890{ 2891 int rc, cntxt_id; 2892 struct fw_eq_eth_cmd c; | 3221 c.eqaddr = htobe64(eq->ba); 3222 3223 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3224 if (rc != 0) { 3225 device_printf(sc->dev, 3226 "failed to create control queue %d: %d\n", eq->tx_chan, rc); 3227 return (rc); 3228 } --- 9 unchanged lines hidden (view full) --- 3238 return (rc); 3239} 3240 3241static int 3242eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 3243{ 3244 int rc, cntxt_id; 3245 struct fw_eq_eth_cmd c; |
3246 int qsize = eq->sidx + spg_len / EQ_ESIZE; |
|
2893 2894 bzero(&c, sizeof(c)); 2895 2896 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 2897 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 2898 V_FW_EQ_ETH_CMD_VFN(0)); 2899 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | 2900 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); | 3247 3248 bzero(&c, sizeof(c)); 3249 3250 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 3251 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 3252 V_FW_EQ_ETH_CMD_VFN(0)); 3253 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | 3254 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); |
2901 c.autoequiqe_to_viid = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->viid)); | 3255 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 3256 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid)); |
2902 c.fetchszm_to_iqid = | 3257 c.fetchszm_to_iqid = |
2903 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | | 3258 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | |
2904 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 2905 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 2906 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | | 3259 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 3260 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 3261 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | |
2907 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 2908 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 2909 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); | 3262 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 3263 V_FW_EQ_ETH_CMD_EQSIZE(qsize)); |
2910 c.eqaddr = htobe64(eq->ba); 2911 2912 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 2913 if (rc != 0) { 2914 device_printf(pi->dev, 2915 "failed to create Ethernet egress queue: %d\n", rc); 2916 return (rc); 2917 } --- 10 unchanged lines hidden (view full) --- 2928} 2929 2930#ifdef TCP_OFFLOAD 2931static int 2932ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 2933{ 2934 int rc, cntxt_id; 2935 struct fw_eq_ofld_cmd c; | 3264 c.eqaddr = htobe64(eq->ba); 3265 3266 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3267 if (rc != 0) { 3268 device_printf(pi->dev, 3269 "failed to create Ethernet egress queue: %d\n", rc); 3270 return (rc); 3271 } --- 10 unchanged lines hidden (view full) --- 3282} 3283 3284#ifdef TCP_OFFLOAD 3285static int 3286ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 3287{ 3288 int rc, cntxt_id; 3289 struct fw_eq_ofld_cmd c; |
3290 int qsize = eq->sidx + spg_len / EQ_ESIZE; |
|
2936 2937 bzero(&c, sizeof(c)); 2938 2939 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 2940 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 2941 V_FW_EQ_OFLD_CMD_VFN(0)); 2942 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 2943 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 2944 c.fetchszm_to_iqid = | 3291 3292 bzero(&c, sizeof(c)); 3293 3294 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 3295 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 3296 V_FW_EQ_OFLD_CMD_VFN(0)); 3297 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 3298 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 3299 c.fetchszm_to_iqid = |
2945 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | | 3300 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | |
2946 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 2947 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 2948 c.dcaen_to_eqsize = 2949 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 2950 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | | 3301 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 3302 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 3303 c.dcaen_to_eqsize = 3304 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 3305 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | |
2951 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 2952 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize)); | 3306 V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); |
2953 c.eqaddr = htobe64(eq->ba); 2954 2955 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 2956 if (rc != 0) { 2957 device_printf(pi->dev, 2958 "failed to create egress queue for TCP offload: %d\n", rc); 2959 return (rc); 2960 } --- 8 unchanged lines hidden (view full) --- 2969 2970 return (rc); 2971} 2972#endif 2973 2974static int 2975alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 2976{ | 3307 c.eqaddr = htobe64(eq->ba); 3308 3309 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3310 if (rc != 0) { 3311 device_printf(pi->dev, 3312 "failed to create egress queue for TCP offload: %d\n", rc); 3313 return (rc); 3314 } --- 8 unchanged lines hidden (view full) --- 3323 3324 return (rc); 3325} 3326#endif 3327 3328static int 3329alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 3330{ |
2977 int rc; | 3331 int rc, qsize; |
2978 size_t len; 2979 2980 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); 2981 | 3332 size_t len; 3333 3334 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); 3335 |
2982 len = eq->qsize * EQ_ESIZE; | 3336 qsize = eq->sidx + spg_len / EQ_ESIZE; 3337 len = qsize * EQ_ESIZE; |
2983 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, 2984 &eq->ba, (void **)&eq->desc); 2985 if (rc) 2986 return (rc); 2987 | 3338 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, 3339 &eq->ba, (void **)&eq->desc); 3340 if (rc) 3341 return (rc); 3342 |
2988 eq->cap = eq->qsize - spg_len / EQ_ESIZE; 2989 eq->spg = (void *)&eq->desc[eq->cap]; 2990 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ | |
2991 eq->pidx = eq->cidx = 0; | 3343 eq->pidx = eq->cidx = 0; |
3344 eq->equeqidx = eq->dbidx = 0; |
|
2992 eq->doorbells = sc->doorbells; 2993 2994 switch (eq->flags & EQ_TYPEMASK) { 2995 case EQ_CTRL: 2996 rc = ctrl_eq_alloc(sc, eq); 2997 break; 2998 2999 case EQ_ETH: --- 11 unchanged lines hidden (view full) --- 3011 eq->flags & EQ_TYPEMASK); 3012 } 3013 if (rc != 0) { 3014 device_printf(sc->dev, 3015 "failed to allocate egress queue(%d): %d\n", 3016 eq->flags & EQ_TYPEMASK, rc); 3017 } 3018 | 3345 eq->doorbells = sc->doorbells; 3346 3347 switch (eq->flags & EQ_TYPEMASK) { 3348 case EQ_CTRL: 3349 rc = ctrl_eq_alloc(sc, eq); 3350 break; 3351 3352 case EQ_ETH: --- 11 unchanged lines hidden (view full) --- 3364 eq->flags & EQ_TYPEMASK); 3365 } 3366 if (rc != 0) { 3367 device_printf(sc->dev, 3368 "failed to allocate egress queue(%d): %d\n", 3369 eq->flags & EQ_TYPEMASK, rc); 3370 } 3371 |
3019 eq->tx_callout.c_cpu = eq->cntxt_id % mp_ncpus; 3020 | |
3021 if (isset(&eq->doorbells, DOORBELL_UDB) || 3022 isset(&eq->doorbells, DOORBELL_UDBWC) || 3023 isset(&eq->doorbells, DOORBELL_WCWR)) { 3024 uint32_t s_qpp = sc->sge.eq_s_qpp; 3025 uint32_t mask = (1 << s_qpp) - 1; 3026 volatile uint8_t *udb; 3027 3028 udb = sc->udbs_base + UDBS_DB_OFFSET; --- 65 unchanged lines hidden (view full) --- 3094 struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx; 3095 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3096 3097 rc = alloc_eq(sc, pi, &wrq->eq); 3098 if (rc) 3099 return (rc); 3100 3101 wrq->adapter = sc; | 3372 if (isset(&eq->doorbells, DOORBELL_UDB) || 3373 isset(&eq->doorbells, DOORBELL_UDBWC) || 3374 isset(&eq->doorbells, DOORBELL_WCWR)) { 3375 uint32_t s_qpp = sc->sge.eq_s_qpp; 3376 uint32_t mask = (1 << s_qpp) - 1; 3377 volatile uint8_t *udb; 3378 3379 udb = sc->udbs_base + UDBS_DB_OFFSET; --- 65 unchanged lines hidden (view full) --- 3445 struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx; 3446 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3447 3448 rc = alloc_eq(sc, pi, &wrq->eq); 3449 if (rc) 3450 return (rc); 3451 3452 wrq->adapter = sc; |
3453 TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); 3454 TAILQ_INIT(&wrq->incomplete_wrs); |
|
3102 STAILQ_INIT(&wrq->wr_list); | 3455 STAILQ_INIT(&wrq->wr_list); |
3456 wrq->nwr_pending = 0; 3457 wrq->ndesc_needed = 0; |
|
3103 3104 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3105 &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); 3106 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3107 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I", 3108 "consumer index"); 3109 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", 3110 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I", 3111 "producer index"); | 3458 3459 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3460 &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); 3461 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3462 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I", 3463 "consumer index"); 3464 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", 3465 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I", 3466 "producer index"); |
3112 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs", CTLFLAG_RD, 3113 &wrq->tx_wrs, "# of work requests"); 3114 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD, 3115 &wrq->no_desc, 0, 3116 "# of times queue ran out of hardware descriptors"); 3117 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD, 3118 &wrq->eq.unstalled, 0, "# of times queue recovered after stall"); | 3467 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, 3468 &wrq->tx_wrs_direct, "# of work requests (direct)"); 3469 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, 3470 &wrq->tx_wrs_copied, "# of work requests (copied)"); |
3119 3120 return (rc); 3121} 3122 3123static int 3124free_wrq(struct adapter *sc, struct sge_wrq *wrq) 3125{ 3126 int rc; --- 11 unchanged lines hidden (view full) --- 3138 struct sysctl_oid *oid) 3139{ 3140 int rc; 3141 struct adapter *sc = pi->adapter; 3142 struct sge_eq *eq = &txq->eq; 3143 char name[16]; 3144 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3145 | 3471 3472 return (rc); 3473} 3474 3475static int 3476free_wrq(struct adapter *sc, struct sge_wrq *wrq) 3477{ 3478 int rc; --- 11 unchanged lines hidden (view full) --- 3490 struct sysctl_oid *oid) 3491{ 3492 int rc; 3493 struct adapter *sc = pi->adapter; 3494 struct sge_eq *eq = &txq->eq; 3495 char name[16]; 3496 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3497 |
3146 rc = alloc_eq(sc, pi, eq); 3147 if (rc) 3148 return (rc); 3149 3150 txq->ifp = pi->ifp; 3151 3152 txq->sdesc = malloc(eq->cap * sizeof(struct tx_sdesc), M_CXGBE, 3153 M_ZERO | M_WAITOK); 3154 txq->br = buf_ring_alloc(eq->qsize, M_CXGBE, M_WAITOK, &eq->eq_lock); 3155 3156 rc = bus_dma_tag_create(sc->dmat, 1, 0, BUS_SPACE_MAXADDR, 3157 BUS_SPACE_MAXADDR, NULL, NULL, 64 * 1024, TX_SGL_SEGS, 3158 BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &txq->tx_tag); | 3498 rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx, 3499 M_CXGBE, M_WAITOK); |
3159 if (rc != 0) { | 3500 if (rc != 0) { |
3160 device_printf(sc->dev, 3161 "failed to create tx DMA tag: %d\n", rc); | 3501 device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc); |
3162 return (rc); 3163 } 3164 | 3502 return (rc); 3503 } 3504 |
3165 /* 3166 * We can stuff ~10 frames in an 8-descriptor txpkts WR (8 is the SGE 3167 * limit for any WR). txq->no_dmamap events shouldn't occur if maps is 3168 * sized for the worst case. 3169 */ 3170 rc = t4_alloc_tx_maps(&txq->txmaps, txq->tx_tag, eq->qsize * 10 / 8, 3171 M_WAITOK); | 3505 rc = alloc_eq(sc, pi, eq); |
3172 if (rc != 0) { | 3506 if (rc != 0) { |
3173 device_printf(sc->dev, "failed to setup tx DMA maps: %d\n", rc); | 3507 mp_ring_free(txq->r); 3508 txq->r = NULL; |
3174 return (rc); 3175 } 3176 | 3509 return (rc); 3510 } 3511 |
3512 /* Can't fail after this point. */ 3513 3514 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); 3515 txq->ifp = pi->ifp; 3516 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); 3517 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3518 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf)); 3519 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, 3520 M_ZERO | M_WAITOK); 3521 |
|
3177 snprintf(name, sizeof(name), "%d", idx); 3178 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD, 3179 NULL, "tx queue"); 3180 children = SYSCTL_CHILDREN(oid); 3181 3182 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3183 &eq->cntxt_id, 0, "SGE context id of the queue"); 3184 SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx", --- 11 unchanged lines hidden (view full) --- 3196 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, 3197 &txq->tso_wrs, "# of TSO work requests"); 3198 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, 3199 &txq->imm_wrs, "# of work requests with immediate data"); 3200 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, 3201 &txq->sgl_wrs, "# of work requests with direct SGL"); 3202 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, 3203 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); | 3522 snprintf(name, sizeof(name), "%d", idx); 3523 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD, 3524 NULL, "tx queue"); 3525 children = SYSCTL_CHILDREN(oid); 3526 3527 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3528 &eq->cntxt_id, 0, "SGE context id of the queue"); 3529 SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx", --- 11 unchanged lines hidden (view full) --- 3541 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, 3542 &txq->tso_wrs, "# of TSO work requests"); 3543 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, 3544 &txq->imm_wrs, "# of work requests with immediate data"); 3545 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, 3546 &txq->sgl_wrs, "# of work requests with direct SGL"); 3547 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, 3548 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); |
3204 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_wrs", CTLFLAG_RD, 3205 &txq->txpkts_wrs, "# of txpkts work requests (multiple pkts/WR)"); 3206 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_pkts", CTLFLAG_RD, 3207 &txq->txpkts_pkts, "# of frames tx'd using txpkts work requests"); | 3549 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts0_wrs", 3550 CTLFLAG_RD, &txq->txpkts0_wrs, 3551 "# of txpkts (type 0) work requests"); 3552 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts1_wrs", 3553 CTLFLAG_RD, &txq->txpkts1_wrs, 3554 "# of txpkts (type 1) work requests"); 3555 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts0_pkts", 3556 CTLFLAG_RD, &txq->txpkts0_pkts, 3557 "# of frames tx'd using type0 txpkts work requests"); 3558 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts1_pkts", 3559 CTLFLAG_RD, &txq->txpkts1_pkts, 3560 "# of frames tx'd using type1 txpkts work requests"); |
3208 | 3561 |
3209 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "br_drops", CTLFLAG_RD, 3210 &txq->br->br_drops, "# of drops in the buf_ring for this queue"); 3211 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_dmamap", CTLFLAG_RD, 3212 &txq->no_dmamap, 0, "# of times txq ran out of DMA maps"); 3213 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD, 3214 &txq->no_desc, 0, "# of times txq ran out of hardware descriptors"); 3215 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "egr_update", CTLFLAG_RD, 3216 &eq->egr_update, 0, "egress update notifications from the SGE"); 3217 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD, 3218 &eq->unstalled, 0, "# of times txq recovered after stall"); | 3562 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_enqueues", 3563 CTLFLAG_RD, &txq->r->enqueues, 3564 "# of enqueues to the mp_ring for this queue"); 3565 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_drops", 3566 CTLFLAG_RD, &txq->r->drops, 3567 "# of drops in the mp_ring for this queue"); 3568 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_starts", 3569 CTLFLAG_RD, &txq->r->starts, 3570 "# of normal consumer starts in the mp_ring for this queue"); 3571 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_stalls", 3572 CTLFLAG_RD, &txq->r->stalls, 3573 "# of consumer stalls in the mp_ring for this queue"); 3574 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_restarts", 3575 CTLFLAG_RD, &txq->r->restarts, 3576 "# of consumer restarts in the mp_ring for this queue"); 3577 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_abdications", 3578 CTLFLAG_RD, &txq->r->abdications, 3579 "# of consumer abdications in the mp_ring for this queue"); |
3219 | 3580 |
3220 return (rc); | 3581 return (0); |
3221} 3222 3223static int 3224free_txq(struct port_info *pi, struct sge_txq *txq) 3225{ 3226 int rc; 3227 struct adapter *sc = pi->adapter; 3228 struct sge_eq *eq = &txq->eq; 3229 3230 rc = free_eq(sc, eq); 3231 if (rc) 3232 return (rc); 3233 | 3582} 3583 3584static int 3585free_txq(struct port_info *pi, struct sge_txq *txq) 3586{ 3587 int rc; 3588 struct adapter *sc = pi->adapter; 3589 struct sge_eq *eq = &txq->eq; 3590 3591 rc = free_eq(sc, eq); 3592 if (rc) 3593 return (rc); 3594 |
3595 sglist_free(txq->gl); |
|
3234 free(txq->sdesc, M_CXGBE); | 3596 free(txq->sdesc, M_CXGBE); |
3597 mp_ring_free(txq->r); |
|
3235 | 3598 |
3236 if (txq->txmaps.maps) 3237 t4_free_tx_maps(&txq->txmaps, txq->tx_tag); 3238 3239 buf_ring_free(txq->br, M_CXGBE); 3240 3241 if (txq->tx_tag) 3242 bus_dma_tag_destroy(txq->tx_tag); 3243 | |
3244 bzero(txq, sizeof(*txq)); 3245 return (0); 3246} 3247 3248static void 3249oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3250{ 3251 bus_addr_t *ba = arg; --- 207 unchanged lines hidden (view full) --- 3459 } 3460 sd->cl = NULL; 3461 } 3462 3463 free(fl->sdesc, M_CXGBE); 3464 fl->sdesc = NULL; 3465} 3466 | 3599 bzero(txq, sizeof(*txq)); 3600 return (0); 3601} 3602 3603static void 3604oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3605{ 3606 bus_addr_t *ba = arg; --- 207 unchanged lines hidden (view full) --- 3814 } 3815 sd->cl = NULL; 3816 } 3817 3818 free(fl->sdesc, M_CXGBE); 3819 fl->sdesc = NULL; 3820} 3821 |
3467int 3468t4_alloc_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag, int count, 3469 int flags) | 3822static inline void 3823get_pkt_gl(struct mbuf *m, struct sglist *gl) |
3470{ | 3824{ |
3471 struct tx_map *txm; 3472 int i, rc; | 3825 int rc; |
3473 | 3826 |
3474 txmaps->map_total = txmaps->map_avail = count; 3475 txmaps->map_cidx = txmaps->map_pidx = 0; | 3827 M_ASSERTPKTHDR(m); |
3476 | 3828 |
3477 txmaps->maps = malloc(count * sizeof(struct tx_map), M_CXGBE, 3478 M_ZERO | flags); 3479 3480 txm = txmaps->maps; 3481 for (i = 0; i < count; i++, txm++) { 3482 rc = bus_dmamap_create(tx_tag, 0, &txm->map); 3483 if (rc != 0) 3484 goto failed; | 3829 sglist_reset(gl); 3830 rc = sglist_append_mbuf(gl, m); 3831 if (__predict_false(rc != 0)) { 3832 panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " 3833 "with %d.", __func__, m, mbuf_nsegs(m), rc); |
3485 } 3486 | 3834 } 3835 |
3487 return (0); 3488failed: 3489 while (--i >= 0) { 3490 txm--; 3491 bus_dmamap_destroy(tx_tag, txm->map); 3492 } 3493 KASSERT(txm == txmaps->maps, ("%s: EDOOFUS", __func__)); 3494 3495 free(txmaps->maps, M_CXGBE); 3496 txmaps->maps = NULL; 3497 3498 return (rc); | 3836 KASSERT(gl->sg_nseg == mbuf_nsegs(m), 3837 ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, 3838 mbuf_nsegs(m), gl->sg_nseg)); 3839 KASSERT(gl->sg_nseg > 0 && 3840 gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS), 3841 ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, 3842 gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)); |
3499} 3500 | 3843} 3844 |
3501void 3502t4_free_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag) | 3845/* 3846 * len16 for a txpkt WR with a GL. Includes the firmware work request header. 3847 */ 3848static inline u_int 3849txpkt_len16(u_int nsegs, u_int tso) |
3503{ | 3850{ |
3504 struct tx_map *txm; 3505 int i; | 3851 u_int n; |
3506 | 3852 |
3507 txm = txmaps->maps; 3508 for (i = 0; i < txmaps->map_total; i++, txm++) { | 3853 MPASS(nsegs > 0); |
3509 | 3854 |
3510 if (txm->m) { 3511 bus_dmamap_unload(tx_tag, txm->map); 3512 m_freem(txm->m); 3513 txm->m = NULL; 3514 } | 3855 nsegs--; /* first segment is part of ulptx_sgl */ 3856 n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) + 3857 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 3858 if (tso) 3859 n += sizeof(struct cpl_tx_pkt_lso_core); |
3515 | 3860 |
3516 bus_dmamap_destroy(tx_tag, txm->map); 3517 } 3518 3519 free(txmaps->maps, M_CXGBE); 3520 txmaps->maps = NULL; | 3861 return (howmany(n, 16)); |
3521} 3522 3523/* | 3862} 3863 3864/* |
3524 * We'll do immediate data tx for non-TSO, but only when not coalescing. We're 3525 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 3526 * of immediate data. | 3865 * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work 3866 * request header. |
3527 */ | 3867 */ |
3528#define IMM_LEN ( \ 3529 2 * EQ_ESIZE \ 3530 - sizeof(struct fw_eth_tx_pkt_wr) \ 3531 - sizeof(struct cpl_tx_pkt_core)) 3532 3533/* 3534 * Returns non-zero on failure, no need to cleanup anything in that case. 3535 * 3536 * Note 1: We always try to defrag the mbuf if required and return EFBIG only 3537 * if the resulting chain still won't fit in a tx descriptor. 3538 * 3539 * Note 2: We'll pullup the mbuf chain if TSO is requested and the first mbuf 3540 * does not have the TCP header in it. 3541 */ 3542static int 3543get_pkt_sgl(struct sge_txq *txq, struct mbuf **fp, struct sgl *sgl, 3544 int sgl_only) | 3868static inline u_int 3869txpkts0_len16(u_int nsegs) |
3545{ | 3870{ |
3546 struct mbuf *m = *fp; 3547 struct tx_maps *txmaps; 3548 struct tx_map *txm; 3549 int rc, defragged = 0, n; | 3871 u_int n; |
3550 | 3872 |
3551 TXQ_LOCK_ASSERT_OWNED(txq); | 3873 MPASS(nsegs > 0); |
3552 | 3874 |
3553 if (m->m_pkthdr.tso_segsz) 3554 sgl_only = 1; /* Do not allow immediate data with LSO */ | 3875 nsegs--; /* first segment is part of ulptx_sgl */ 3876 n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + 3877 sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 3878 8 * ((3 * nsegs) / 2 + (nsegs & 1)); |
3555 | 3879 |
3556start: sgl->nsegs = 0; 3557 3558 if (m->m_pkthdr.len <= IMM_LEN && !sgl_only) 3559 return (0); /* nsegs = 0 tells caller to use imm. tx */ 3560 3561 txmaps = &txq->txmaps; 3562 if (txmaps->map_avail == 0) { 3563 txq->no_dmamap++; 3564 return (ENOMEM); 3565 } 3566 txm = &txmaps->maps[txmaps->map_pidx]; 3567 3568 if (m->m_pkthdr.tso_segsz && m->m_len < 50) { 3569 *fp = m_pullup(m, 50); 3570 m = *fp; 3571 if (m == NULL) 3572 return (ENOBUFS); 3573 } 3574 3575 rc = bus_dmamap_load_mbuf_sg(txq->tx_tag, txm->map, m, sgl->seg, 3576 &sgl->nsegs, BUS_DMA_NOWAIT); 3577 if (rc == EFBIG && defragged == 0) { 3578 m = m_defrag(m, M_NOWAIT); 3579 if (m == NULL) 3580 return (EFBIG); 3581 3582 defragged = 1; 3583 *fp = m; 3584 goto start; 3585 } 3586 if (rc != 0) 3587 return (rc); 3588 3589 txm->m = m; 3590 txmaps->map_avail--; 3591 if (++txmaps->map_pidx == txmaps->map_total) 3592 txmaps->map_pidx = 0; 3593 3594 KASSERT(sgl->nsegs > 0 && sgl->nsegs <= TX_SGL_SEGS, 3595 ("%s: bad DMA mapping (%d segments)", __func__, sgl->nsegs)); 3596 3597 /* 3598 * Store the # of flits required to hold this frame's SGL in nflits. An 3599 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 3600 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 3601 * then len1 must be set to 0. 3602 */ 3603 n = sgl->nsegs - 1; 3604 sgl->nflits = (3 * n) / 2 + (n & 1) + 2; 3605 3606 return (0); | 3880 return (howmany(n, 16)); |
3607} 3608 | 3881} 3882 |
3609 | |
3610/* | 3883/* |
3611 * Releases all the txq resources used up in the specified sgl. | 3884 * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work 3885 * request header. |
3612 */ | 3886 */ |
3613static int 3614free_pkt_sgl(struct sge_txq *txq, struct sgl *sgl) | 3887static inline u_int 3888txpkts1_len16(void) |
3615{ | 3889{ |
3616 struct tx_maps *txmaps; 3617 struct tx_map *txm; | 3890 u_int n; |
3618 | 3891 |
3619 TXQ_LOCK_ASSERT_OWNED(txq); | 3892 n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); |
3620 | 3893 |
3621 if (sgl->nsegs == 0) 3622 return (0); /* didn't use any map */ | 3894 return (howmany(n, 16)); 3895} |
3623 | 3896 |
3624 txmaps = &txq->txmaps; | 3897static inline u_int 3898imm_payload(u_int ndesc) 3899{ 3900 u_int n; |
3625 | 3901 |
3626 /* 1 pkt uses exactly 1 map, back it out */ | 3902 n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - 3903 sizeof(struct cpl_tx_pkt_core); |
3627 | 3904 |
3628 txmaps->map_avail++; 3629 if (txmaps->map_pidx > 0) 3630 txmaps->map_pidx--; 3631 else 3632 txmaps->map_pidx = txmaps->map_total - 1; 3633 3634 txm = &txmaps->maps[txmaps->map_pidx]; 3635 bus_dmamap_unload(txq->tx_tag, txm->map); 3636 txm->m = NULL; 3637 3638 return (0); | 3905 return (n); |
3639} 3640 | 3906} 3907 |
3641static int 3642write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, struct mbuf *m, 3643 struct sgl *sgl) | 3908/* 3909 * Write a txpkt WR for this packet to the hardware descriptors, update the 3910 * software descriptor, and advance the pidx. It is guaranteed that enough 3911 * descriptors are available. 3912 * 3913 * The return value is the # of hardware descriptors used. 3914 */ 3915static u_int 3916write_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr, 3917 struct mbuf *m0, u_int available) |
3644{ 3645 struct sge_eq *eq = &txq->eq; | 3918{ 3919 struct sge_eq *eq = &txq->eq; |
3646 struct fw_eth_tx_pkt_wr *wr; | 3920 struct tx_sdesc *txsd; |
3647 struct cpl_tx_pkt_core *cpl; 3648 uint32_t ctrl; /* used in many unrelated places */ 3649 uint64_t ctrl1; | 3921 struct cpl_tx_pkt_core *cpl; 3922 uint32_t ctrl; /* used in many unrelated places */ 3923 uint64_t ctrl1; |
3650 int nflits, ndesc, pktlen; 3651 struct tx_sdesc *txsd; | 3924 int len16, ndesc, pktlen, nsegs; |
3652 caddr_t dst; 3653 3654 TXQ_LOCK_ASSERT_OWNED(txq); | 3925 caddr_t dst; 3926 3927 TXQ_LOCK_ASSERT_OWNED(txq); |
3928 M_ASSERTPKTHDR(m0); 3929 MPASS(available > 0 && available < eq->sidx); |
|
3655 | 3930 |
3656 pktlen = m->m_pkthdr.len; 3657 3658 /* 3659 * Do we have enough flits to send this frame out? 3660 */ | 3931 len16 = mbuf_len16(m0); 3932 nsegs = mbuf_nsegs(m0); 3933 pktlen = m0->m_pkthdr.len; |
3661 ctrl = sizeof(struct cpl_tx_pkt_core); | 3934 ctrl = sizeof(struct cpl_tx_pkt_core); |
3662 if (m->m_pkthdr.tso_segsz) { 3663 nflits = TXPKT_LSO_WR_HDR; | 3935 if (needs_tso(m0)) |
3664 ctrl += sizeof(struct cpl_tx_pkt_lso_core); | 3936 ctrl += sizeof(struct cpl_tx_pkt_lso_core); |
3665 } else 3666 nflits = TXPKT_WR_HDR; 3667 if (sgl->nsegs > 0) 3668 nflits += sgl->nflits; 3669 else { 3670 nflits += howmany(pktlen, 8); | 3937 else if (pktlen <= imm_payload(2) && available >= 2) { 3938 /* Immediate data. Recalculate len16 and set nsegs to 0. */ |
3671 ctrl += pktlen; | 3939 ctrl += pktlen; |
3940 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + 3941 sizeof(struct cpl_tx_pkt_core) + pktlen, 16); 3942 nsegs = 0; |
|
3672 } | 3943 } |
3673 ndesc = howmany(nflits, 8); 3674 if (ndesc > eq->avail) 3675 return (ENOMEM); | 3944 ndesc = howmany(len16, EQ_ESIZE / 16); 3945 MPASS(ndesc <= available); |
3676 3677 /* Firmware work request header */ | 3946 3947 /* Firmware work request header */ |
3678 wr = (void *)&eq->desc[eq->pidx]; | 3948 MPASS(wr == (void *)&eq->desc[eq->pidx]); |
3679 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 3680 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); | 3949 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 3950 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); |
3681 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 3682 if (eq->avail == ndesc) { 3683 if (!(eq->flags & EQ_CRFLUSHED)) { 3684 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 3685 eq->flags |= EQ_CRFLUSHED; 3686 } 3687 eq->flags |= EQ_STALLED; 3688 } | |
3689 | 3951 |
3952 ctrl = V_FW_WR_LEN16(len16); |
|
3690 wr->equiq_to_len16 = htobe32(ctrl); 3691 wr->r3 = 0; 3692 | 3953 wr->equiq_to_len16 = htobe32(ctrl); 3954 wr->r3 = 0; 3955 |
3693 if (m->m_pkthdr.tso_segsz) { | 3956 if (needs_tso(m0)) { |
3694 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); | 3957 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); |
3695 struct ether_header *eh; 3696 void *l3hdr; 3697#if defined(INET) || defined(INET6) 3698 struct tcphdr *tcp; 3699#endif 3700 uint16_t eh_type; | |
3701 | 3958 |
3702 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 3703 F_LSO_LAST_SLICE; | 3959 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 3960 m0->m_pkthdr.l4hlen > 0, 3961 ("%s: mbuf %p needs TSO but missing header lengths", 3962 __func__, m0)); |
3704 | 3963 |
3705 eh = mtod(m, struct ether_header *); 3706 eh_type = ntohs(eh->ether_type); 3707 if (eh_type == ETHERTYPE_VLAN) { 3708 struct ether_vlan_header *evh = (void *)eh; 3709 | 3964 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 3965 F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) 3966 | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 3967 if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header)) |
3710 ctrl |= V_LSO_ETHHDR_LEN(1); | 3968 ctrl |= V_LSO_ETHHDR_LEN(1); |
3711 l3hdr = evh + 1; 3712 eh_type = ntohs(evh->evl_proto); 3713 } else 3714 l3hdr = eh + 1; 3715 3716 switch (eh_type) { 3717#ifdef INET6 3718 case ETHERTYPE_IPV6: 3719 { 3720 struct ip6_hdr *ip6 = l3hdr; 3721 3722 /* 3723 * XXX-BZ For now we do not pretend to support 3724 * IPv6 extension headers. 3725 */ 3726 KASSERT(ip6->ip6_nxt == IPPROTO_TCP, ("%s: CSUM_TSO " 3727 "with ip6_nxt != TCP: %u", __func__, ip6->ip6_nxt)); 3728 tcp = (struct tcphdr *)(ip6 + 1); | 3969 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) |
3729 ctrl |= F_LSO_IPV6; | 3970 ctrl |= F_LSO_IPV6; |
3730 ctrl |= V_LSO_IPHDR_LEN(sizeof(*ip6) >> 2) | 3731 V_LSO_TCPHDR_LEN(tcp->th_off); 3732 break; 3733 } 3734#endif 3735#ifdef INET 3736 case ETHERTYPE_IP: 3737 { 3738 struct ip *ip = l3hdr; | |
3739 | 3971 |
3740 tcp = (void *)((uintptr_t)ip + ip->ip_hl * 4); 3741 ctrl |= V_LSO_IPHDR_LEN(ip->ip_hl) | 3742 V_LSO_TCPHDR_LEN(tcp->th_off); 3743 break; 3744 } 3745#endif 3746 default: 3747 panic("%s: CSUM_TSO but no supported IP version " 3748 "(0x%04x)", __func__, eh_type); 3749 } 3750 | |
3751 lso->lso_ctrl = htobe32(ctrl); 3752 lso->ipid_ofst = htobe16(0); | 3972 lso->lso_ctrl = htobe32(ctrl); 3973 lso->ipid_ofst = htobe16(0); |
3753 lso->mss = htobe16(m->m_pkthdr.tso_segsz); | 3974 lso->mss = htobe16(m0->m_pkthdr.tso_segsz); |
3754 lso->seqno_offset = htobe32(0); 3755 lso->len = htobe32(pktlen); 3756 3757 cpl = (void *)(lso + 1); 3758 3759 txq->tso_wrs++; 3760 } else 3761 cpl = (void *)(wr + 1); 3762 3763 /* Checksum offload */ 3764 ctrl1 = 0; | 3975 lso->seqno_offset = htobe32(0); 3976 lso->len = htobe32(pktlen); 3977 3978 cpl = (void *)(lso + 1); 3979 3980 txq->tso_wrs++; 3981 } else 3982 cpl = (void *)(wr + 1); 3983 3984 /* Checksum offload */ 3985 ctrl1 = 0; |
3765 if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO))) | 3986 if (needs_l3_csum(m0) == 0) |
3766 ctrl1 |= F_TXPKT_IPCSUM_DIS; | 3987 ctrl1 |= F_TXPKT_IPCSUM_DIS; |
3767 if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | 3768 CSUM_TCP_IPV6 | CSUM_TSO))) | 3988 if (needs_l4_csum(m0) == 0) |
3769 ctrl1 |= F_TXPKT_L4CSUM_DIS; | 3989 ctrl1 |= F_TXPKT_L4CSUM_DIS; |
3770 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | | 3990 if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | |
3771 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 3772 txq->txcsum++; /* some hardware assistance provided */ 3773 3774 /* VLAN tag insertion */ | 3991 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 3992 txq->txcsum++; /* some hardware assistance provided */ 3993 3994 /* VLAN tag insertion */ |
3775 if (m->m_flags & M_VLANTAG) { 3776 ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); | 3995 if (needs_vlan_insertion(m0)) { 3996 ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); |
3777 txq->vlan_insertion++; 3778 } 3779 3780 /* CPL header */ | 3997 txq->vlan_insertion++; 3998 } 3999 4000 /* CPL header */ |
3781 cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3782 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); | 4001 cpl->ctrl0 = txq->cpl_ctrl0; |
3783 cpl->pack = 0; 3784 cpl->len = htobe16(pktlen); 3785 cpl->ctrl1 = htobe64(ctrl1); 3786 | 4002 cpl->pack = 0; 4003 cpl->len = htobe16(pktlen); 4004 cpl->ctrl1 = htobe64(ctrl1); 4005 |
3787 /* Software descriptor */ 3788 txsd = &txq->sdesc[eq->pidx]; 3789 txsd->desc_used = ndesc; 3790 3791 eq->pending += ndesc; 3792 eq->avail -= ndesc; 3793 eq->pidx += ndesc; 3794 if (eq->pidx >= eq->cap) 3795 eq->pidx -= eq->cap; 3796 | |
3797 /* SGL */ 3798 dst = (void *)(cpl + 1); | 4006 /* SGL */ 4007 dst = (void *)(cpl + 1); |
3799 if (sgl->nsegs > 0) { 3800 txsd->credits = 1; | 4008 if (nsegs > 0) { 4009 4010 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); |
3801 txq->sgl_wrs++; | 4011 txq->sgl_wrs++; |
3802 write_sgl_to_txd(eq, sgl, &dst); | |
3803 } else { | 4012 } else { |
3804 txsd->credits = 0; 3805 txq->imm_wrs++; 3806 for (; m; m = m->m_next) { | 4013 struct mbuf *m; 4014 4015 for (m = m0; m != NULL; m = m->m_next) { |
3807 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 3808#ifdef INVARIANTS 3809 pktlen -= m->m_len; 3810#endif 3811 } 3812#ifdef INVARIANTS 3813 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); 3814#endif | 4016 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 4017#ifdef INVARIANTS 4018 pktlen -= m->m_len; 4019#endif 4020 } 4021#ifdef INVARIANTS 4022 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); 4023#endif |
3815 | 4024 txq->imm_wrs++; |
3816 } 3817 3818 txq->txpkt_wrs++; | 4025 } 4026 4027 txq->txpkt_wrs++; |
3819 return (0); | 4028 4029 txsd = &txq->sdesc[eq->pidx]; 4030 txsd->m = m0; 4031 txsd->desc_used = ndesc; 4032 4033 return (ndesc); |
3820} 3821 | 4034} 4035 |
3822/* 3823 * Returns 0 to indicate that m has been accepted into a coalesced tx work 3824 * request. It has either been folded into txpkts or txpkts was flushed and m 3825 * has started a new coalesced work request (as the first frame in a fresh 3826 * txpkts). 3827 * 3828 * Returns non-zero to indicate a failure - caller is responsible for 3829 * transmitting m, if there was anything in txpkts it has been flushed. 3830 */ | |
3831static int | 4036static int |
3832add_to_txpkts(struct port_info *pi, struct sge_txq *txq, struct txpkts *txpkts, 3833 struct mbuf *m, struct sgl *sgl) | 4037try_txpkts(struct mbuf *m, struct mbuf *n, struct txpkts *txp, u_int available) |
3834{ | 4038{ |
3835 struct sge_eq *eq = &txq->eq; 3836 int can_coalesce; 3837 struct tx_sdesc *txsd; 3838 int flits; | 4039 u_int needed, nsegs1, nsegs2, l1, l2; |
3839 | 4040 |
3840 TXQ_LOCK_ASSERT_OWNED(txq); | 4041 if (cannot_use_txpkts(m) || cannot_use_txpkts(n)) 4042 return (1); |
3841 | 4043 |
3842 KASSERT(sgl->nsegs, ("%s: can't coalesce imm data", __func__)); | 4044 nsegs1 = mbuf_nsegs(m); 4045 nsegs2 = mbuf_nsegs(n); 4046 if (nsegs1 + nsegs2 == 2) { 4047 txp->wr_type = 1; 4048 l1 = l2 = txpkts1_len16(); 4049 } else { 4050 txp->wr_type = 0; 4051 l1 = txpkts0_len16(nsegs1); 4052 l2 = txpkts0_len16(nsegs2); 4053 } 4054 txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + l1 + l2; 4055 needed = howmany(txp->len16, EQ_ESIZE / 16); 4056 if (needed > SGE_MAX_WR_NDESC || needed > available) 4057 return (1); |
3843 | 4058 |
3844 if (txpkts->npkt > 0) { 3845 flits = TXPKTS_PKT_HDR + sgl->nflits; 3846 can_coalesce = m->m_pkthdr.tso_segsz == 0 && 3847 txpkts->nflits + flits <= TX_WR_FLITS && 3848 txpkts->nflits + flits <= eq->avail * 8 && 3849 txpkts->plen + m->m_pkthdr.len < 65536; | 4059 txp->plen = m->m_pkthdr.len + n->m_pkthdr.len; 4060 if (txp->plen > 65535) 4061 return (1); |
3850 | 4062 |
3851 if (can_coalesce) { 3852 txpkts->npkt++; 3853 txpkts->nflits += flits; 3854 txpkts->plen += m->m_pkthdr.len; | 4063 txp->npkt = 2; 4064 set_mbuf_len16(m, l1); 4065 set_mbuf_len16(n, l2); |
3855 | 4066 |
3856 txsd = &txq->sdesc[eq->pidx]; 3857 txsd->credits++; | 4067 return (0); 4068} |
3858 | 4069 |
3859 return (0); 3860 } | 4070static int 4071add_to_txpkts(struct mbuf *m, struct txpkts *txp, u_int available) 4072{ 4073 u_int plen, len16, needed, nsegs; |
3861 | 4074 |
3862 /* 3863 * Couldn't coalesce m into txpkts. The first order of business 3864 * is to send txpkts on its way. Then we'll revisit m. 3865 */ 3866 write_txpkts_wr(txq, txpkts); 3867 } | 4075 MPASS(txp->wr_type == 0 || txp->wr_type == 1); |
3868 | 4076 |
3869 /* 3870 * Check if we can start a new coalesced tx work request with m as 3871 * the first packet in it. 3872 */ | 4077 nsegs = mbuf_nsegs(m); 4078 if (needs_tso(m) || (txp->wr_type == 1 && nsegs != 1)) 4079 return (1); |
3873 | 4080 |
3874 KASSERT(txpkts->npkt == 0, ("%s: txpkts not empty", __func__)); | 4081 plen = txp->plen + m->m_pkthdr.len; 4082 if (plen > 65535) 4083 return (1); |
3875 | 4084 |
3876 flits = TXPKTS_WR_HDR + sgl->nflits; 3877 can_coalesce = m->m_pkthdr.tso_segsz == 0 && 3878 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; | 4085 if (txp->wr_type == 0) 4086 len16 = txpkts0_len16(nsegs); 4087 else 4088 len16 = txpkts1_len16(); 4089 needed = howmany(txp->len16 + len16, EQ_ESIZE / 16); 4090 if (needed > SGE_MAX_WR_NDESC || needed > available) 4091 return (1); |
3879 | 4092 |
3880 if (can_coalesce == 0) 3881 return (EINVAL); | 4093 txp->npkt++; 4094 txp->plen = plen; 4095 txp->len16 += len16; 4096 set_mbuf_len16(m, len16); |
3882 | 4097 |
3883 /* 3884 * Start a fresh coalesced tx WR with m as the first frame in it. 3885 */ 3886 txpkts->npkt = 1; 3887 txpkts->nflits = flits; 3888 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 3889 txpkts->plen = m->m_pkthdr.len; 3890 3891 txsd = &txq->sdesc[eq->pidx]; 3892 txsd->credits = 1; 3893 | |
3894 return (0); 3895} 3896 3897/* | 4098 return (0); 4099} 4100 4101/* |
3898 * Note that write_txpkts_wr can never run out of hardware descriptors (but 3899 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 3900 * coalescing only if sufficient hardware descriptors are available. | 4102 * Write a txpkts WR for the packets in txp to the hardware descriptors, update 4103 * the software descriptor, and advance the pidx. It is guaranteed that enough 4104 * descriptors are available. 4105 * 4106 * The return value is the # of hardware descriptors used. |
3901 */ | 4107 */ |
3902static void 3903write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) | 4108static u_int 4109write_txpkts_wr(struct sge_txq *txq, struct fw_eth_tx_pkts_wr *wr, 4110 struct mbuf *m0, const struct txpkts *txp, u_int available) |
3904{ 3905 struct sge_eq *eq = &txq->eq; | 4111{ 4112 struct sge_eq *eq = &txq->eq; |
3906 struct fw_eth_tx_pkts_wr *wr; | |
3907 struct tx_sdesc *txsd; | 4113 struct tx_sdesc *txsd; |
4114 struct cpl_tx_pkt_core *cpl; |
|
3908 uint32_t ctrl; | 4115 uint32_t ctrl; |
3909 int ndesc; | 4116 uint64_t ctrl1; 4117 int ndesc, checkwrap; 4118 struct mbuf *m; 4119 void *flitp; |
3910 3911 TXQ_LOCK_ASSERT_OWNED(txq); | 4120 4121 TXQ_LOCK_ASSERT_OWNED(txq); |
4122 MPASS(txp->npkt > 0); 4123 MPASS(txp->plen < 65536); 4124 MPASS(m0 != NULL); 4125 MPASS(m0->m_nextpkt != NULL); 4126 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 4127 MPASS(available > 0 && available < eq->sidx); |
|
3912 | 4128 |
3913 ndesc = howmany(txpkts->nflits, 8); | 4129 ndesc = howmany(txp->len16, EQ_ESIZE / 16); 4130 MPASS(ndesc <= available); |
3914 | 4131 |
3915 wr = (void *)&eq->desc[eq->pidx]; | 4132 MPASS(wr == (void *)&eq->desc[eq->pidx]); |
3916 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); | 4133 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); |
3917 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 3918 if (eq->avail == ndesc) { 3919 if (!(eq->flags & EQ_CRFLUSHED)) { 3920 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 3921 eq->flags |= EQ_CRFLUSHED; 3922 } 3923 eq->flags |= EQ_STALLED; 3924 } | 4134 ctrl = V_FW_WR_LEN16(txp->len16); |
3925 wr->equiq_to_len16 = htobe32(ctrl); | 4135 wr->equiq_to_len16 = htobe32(ctrl); |
3926 wr->plen = htobe16(txpkts->plen); 3927 wr->npkt = txpkts->npkt; 3928 wr->r3 = wr->type = 0; | 4136 wr->plen = htobe16(txp->plen); 4137 wr->npkt = txp->npkt; 4138 wr->r3 = 0; 4139 wr->type = txp->wr_type; 4140 flitp = wr + 1; |
3929 | 4141 |
3930 /* Everything else already written */ | 4142 /* 4143 * At this point we are 16B into a hardware descriptor. If checkwrap is 4144 * set then we know the WR is going to wrap around somewhere. We'll 4145 * check for that at appropriate points. 4146 */ 4147 checkwrap = eq->sidx - ndesc < eq->pidx; 4148 for (m = m0; m != NULL; m = m->m_nextpkt) { 4149 if (txp->wr_type == 0) { 4150 struct ulp_txpkt *ulpmc; 4151 struct ulptx_idata *ulpsc; |
3931 | 4152 |
3932 txsd = &txq->sdesc[eq->pidx]; 3933 txsd->desc_used = ndesc; | 4153 /* ULP master command */ 4154 ulpmc = flitp; 4155 ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 4156 V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); 4157 ulpmc->len = htobe32(mbuf_len16(m)); |
3934 | 4158 |
3935 KASSERT(eq->avail >= ndesc, ("%s: out of descriptors", __func__)); | 4159 /* ULP subcommand */ 4160 ulpsc = (void *)(ulpmc + 1); 4161 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 4162 F_ULP_TX_SC_MORE); 4163 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); |
3936 | 4164 |
3937 eq->pending += ndesc; 3938 eq->avail -= ndesc; 3939 eq->pidx += ndesc; 3940 if (eq->pidx >= eq->cap) 3941 eq->pidx -= eq->cap; | 4165 cpl = (void *)(ulpsc + 1); 4166 if (checkwrap && 4167 (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) 4168 cpl = (void *)&eq->desc[0]; 4169 txq->txpkts0_pkts += txp->npkt; 4170 txq->txpkts0_wrs++; 4171 } else { 4172 cpl = flitp; 4173 txq->txpkts1_pkts += txp->npkt; 4174 txq->txpkts1_wrs++; 4175 } |
3942 | 4176 |
3943 txq->txpkts_pkts += txpkts->npkt; 3944 txq->txpkts_wrs++; 3945 txpkts->npkt = 0; /* emptied */ 3946} | 4177 /* Checksum offload */ 4178 ctrl1 = 0; 4179 if (needs_l3_csum(m) == 0) 4180 ctrl1 |= F_TXPKT_IPCSUM_DIS; 4181 if (needs_l4_csum(m) == 0) 4182 ctrl1 |= F_TXPKT_L4CSUM_DIS; 4183 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | 4184 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 4185 txq->txcsum++; /* some hardware assistance provided */ |
3947 | 4186 |
3948static inline void 3949write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 3950 struct txpkts *txpkts, struct mbuf *m, struct sgl *sgl) 3951{ 3952 struct ulp_txpkt *ulpmc; 3953 struct ulptx_idata *ulpsc; 3954 struct cpl_tx_pkt_core *cpl; 3955 struct sge_eq *eq = &txq->eq; 3956 uintptr_t flitp, start, end; 3957 uint64_t ctrl; 3958 caddr_t dst; | 4187 /* VLAN tag insertion */ 4188 if (needs_vlan_insertion(m)) { 4189 ctrl1 |= F_TXPKT_VLAN_VLD | 4190 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 4191 txq->vlan_insertion++; 4192 } |
3959 | 4193 |
3960 KASSERT(txpkts->npkt > 0, ("%s: txpkts is empty", __func__)); | 4194 /* CPL header */ 4195 cpl->ctrl0 = txq->cpl_ctrl0; 4196 cpl->pack = 0; 4197 cpl->len = htobe16(m->m_pkthdr.len); 4198 cpl->ctrl1 = htobe64(ctrl1); |
3961 | 4199 |
3962 start = (uintptr_t)eq->desc; 3963 end = (uintptr_t)eq->spg; | 4200 flitp = cpl + 1; 4201 if (checkwrap && 4202 (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 4203 flitp = (void *)&eq->desc[0]; |
3964 | 4204 |
3965 /* Checksum offload */ 3966 ctrl = 0; 3967 if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO))) 3968 ctrl |= F_TXPKT_IPCSUM_DIS; 3969 if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | 3970 CSUM_TCP_IPV6 | CSUM_TSO))) 3971 ctrl |= F_TXPKT_L4CSUM_DIS; 3972 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | 3973 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 3974 txq->txcsum++; /* some hardware assistance provided */ | 4205 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); |
3975 | 4206 |
3976 /* VLAN tag insertion */ 3977 if (m->m_flags & M_VLANTAG) { 3978 ctrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 3979 txq->vlan_insertion++; | |
3980 } 3981 | 4207 } 4208 |
3982 /* 3983 * The previous packet's SGL must have ended at a 16 byte boundary (this 3984 * is required by the firmware/hardware). It follows that flitp cannot 3985 * wrap around between the ULPTX master command and ULPTX subcommand (8 3986 * bytes each), and that it can not wrap around in the middle of the 3987 * cpl_tx_pkt_core either. 3988 */ 3989 flitp = (uintptr_t)txpkts->flitp; 3990 KASSERT((flitp & 0xf) == 0, 3991 ("%s: last SGL did not end at 16 byte boundary: %p", 3992 __func__, txpkts->flitp)); | 4209 txsd = &txq->sdesc[eq->pidx]; 4210 txsd->m = m0; 4211 txsd->desc_used = ndesc; |
3993 | 4212 |
3994 /* ULP master command */ 3995 ulpmc = (void *)flitp; 3996 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0) | 3997 V_ULP_TXPKT_FID(eq->iqid)); 3998 ulpmc->len = htonl(howmany(sizeof(*ulpmc) + sizeof(*ulpsc) + 3999 sizeof(*cpl) + 8 * sgl->nflits, 16)); 4000 4001 /* ULP subcommand */ 4002 ulpsc = (void *)(ulpmc + 1); 4003 ulpsc->cmd_more = htobe32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 4004 F_ULP_TX_SC_MORE); 4005 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); 4006 4007 flitp += sizeof(*ulpmc) + sizeof(*ulpsc); 4008 if (flitp == end) 4009 flitp = start; 4010 4011 /* CPL_TX_PKT */ 4012 cpl = (void *)flitp; 4013 cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 4014 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 4015 cpl->pack = 0; 4016 cpl->len = htobe16(m->m_pkthdr.len); 4017 cpl->ctrl1 = htobe64(ctrl); 4018 4019 flitp += sizeof(*cpl); 4020 if (flitp == end) 4021 flitp = start; 4022 4023 /* SGL for this frame */ 4024 dst = (caddr_t)flitp; 4025 txpkts->nflits += write_sgl_to_txd(eq, sgl, &dst); 4026 txpkts->flitp = (void *)dst; 4027 4028 KASSERT(((uintptr_t)dst & 0xf) == 0, 4029 ("%s: SGL ends at %p (not a 16 byte boundary)", __func__, dst)); | 4213 return (ndesc); |
4030} 4031 4032/* 4033 * If the SGL ends on an address that is not 16 byte aligned, this function will | 4214} 4215 4216/* 4217 * If the SGL ends on an address that is not 16 byte aligned, this function will |
4034 * add a 0 filled flit at the end. It returns 1 in that case. | 4218 * add a 0 filled flit at the end. |
4035 */ | 4219 */ |
4036static int 4037write_sgl_to_txd(struct sge_eq *eq, struct sgl *sgl, caddr_t *to) | 4220static void 4221write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) |
4038{ | 4222{ |
4039 __be64 *flitp, *end; | 4223 struct sge_eq *eq = &txq->eq; 4224 struct sglist *gl = txq->gl; 4225 struct sglist_seg *seg; 4226 __be64 *flitp, *wrap; |
4040 struct ulptx_sgl *usgl; | 4227 struct ulptx_sgl *usgl; |
4041 bus_dma_segment_t *seg; 4042 int i, padded; | 4228 int i, nflits, nsegs; |
4043 | 4229 |
4044 KASSERT(sgl->nsegs > 0 && sgl->nflits > 0, 4045 ("%s: bad SGL - nsegs=%d, nflits=%d", 4046 __func__, sgl->nsegs, sgl->nflits)); 4047 | |
4048 KASSERT(((uintptr_t)(*to) & 0xf) == 0, 4049 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); | 4230 KASSERT(((uintptr_t)(*to) & 0xf) == 0, 4231 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); |
4232 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 4233 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); |
|
4050 | 4234 |
4235 get_pkt_gl(m, gl); 4236 nsegs = gl->sg_nseg; 4237 MPASS(nsegs > 0); 4238 4239 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; |
|
4051 flitp = (__be64 *)(*to); | 4240 flitp = (__be64 *)(*to); |
4052 end = flitp + sgl->nflits; 4053 seg = &sgl->seg[0]; | 4241 wrap = (__be64 *)(&eq->desc[eq->sidx]); 4242 seg = &gl->sg_segs[0]; |
4054 usgl = (void *)flitp; 4055 4056 /* 4057 * We start at a 16 byte boundary somewhere inside the tx descriptor 4058 * ring, so we're at least 16 bytes away from the status page. There is 4059 * no chance of a wrap around in the middle of usgl (which is 16 bytes). 4060 */ 4061 4062 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | | 4243 usgl = (void *)flitp; 4244 4245 /* 4246 * We start at a 16 byte boundary somewhere inside the tx descriptor 4247 * ring, so we're at least 16 bytes away from the status page. There is 4248 * no chance of a wrap around in the middle of usgl (which is 16 bytes). 4249 */ 4250 4251 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | |
4063 V_ULPTX_NSGE(sgl->nsegs)); 4064 usgl->len0 = htobe32(seg->ds_len); 4065 usgl->addr0 = htobe64(seg->ds_addr); | 4252 V_ULPTX_NSGE(nsegs)); 4253 usgl->len0 = htobe32(seg->ss_len); 4254 usgl->addr0 = htobe64(seg->ss_paddr); |
4066 seg++; 4067 | 4255 seg++; 4256 |
4068 if ((uintptr_t)end <= (uintptr_t)eq->spg) { | 4257 if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { |
4069 4070 /* Won't wrap around at all */ 4071 | 4258 4259 /* Won't wrap around at all */ 4260 |
4072 for (i = 0; i < sgl->nsegs - 1; i++, seg++) { 4073 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ds_len); 4074 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ds_addr); | 4261 for (i = 0; i < nsegs - 1; i++, seg++) { 4262 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); 4263 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); |
4075 } 4076 if (i & 1) 4077 usgl->sge[i / 2].len[1] = htobe32(0); | 4264 } 4265 if (i & 1) 4266 usgl->sge[i / 2].len[1] = htobe32(0); |
4267 flitp += nflits; |
|
4078 } else { 4079 4080 /* Will wrap somewhere in the rest of the SGL */ 4081 4082 /* 2 flits already written, write the rest flit by flit */ 4083 flitp = (void *)(usgl + 1); | 4268 } else { 4269 4270 /* Will wrap somewhere in the rest of the SGL */ 4271 4272 /* 2 flits already written, write the rest flit by flit */ 4273 flitp = (void *)(usgl + 1); |
4084 for (i = 0; i < sgl->nflits - 2; i++) { 4085 if ((uintptr_t)flitp == (uintptr_t)eq->spg) | 4274 for (i = 0; i < nflits - 2; i++) { 4275 if (flitp == wrap) |
4086 flitp = (void *)eq->desc; | 4276 flitp = (void *)eq->desc; |
4087 *flitp++ = get_flit(seg, sgl->nsegs - 1, i); | 4277 *flitp++ = get_flit(seg, nsegs - 1, i); |
4088 } | 4278 } |
4089 end = flitp; | |
4090 } 4091 | 4279 } 4280 |
4092 if ((uintptr_t)end & 0xf) { 4093 *(uint64_t *)end = 0; 4094 end++; 4095 padded = 1; 4096 } else 4097 padded = 0; | 4281 if (nflits & 1) { 4282 MPASS(((uintptr_t)flitp) & 0xf); 4283 *flitp++ = 0; 4284 } |
4098 | 4285 |
4099 if ((uintptr_t)end == (uintptr_t)eq->spg) | 4286 MPASS((((uintptr_t)flitp) & 0xf) == 0); 4287 if (__predict_false(flitp == wrap)) |
4100 *to = (void *)eq->desc; 4101 else | 4288 *to = (void *)eq->desc; 4289 else |
4102 *to = (void *)end; 4103 4104 return (padded); | 4290 *to = (void *)flitp; |
4105} 4106 4107static inline void 4108copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 4109{ | 4291} 4292 4293static inline void 4294copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 4295{ |
4110 if (__predict_true((uintptr_t)(*to) + len <= (uintptr_t)eq->spg)) { | 4296 4297 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 4298 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 4299 4300 if (__predict_true((uintptr_t)(*to) + len <= 4301 (uintptr_t)&eq->desc[eq->sidx])) { |
4111 bcopy(from, *to, len); 4112 (*to) += len; 4113 } else { | 4302 bcopy(from, *to, len); 4303 (*to) += len; 4304 } else { |
4114 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); | 4305 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); |
4115 4116 bcopy(from, *to, portion); 4117 from += portion; 4118 portion = len - portion; /* remaining */ 4119 bcopy(from, (void *)eq->desc, portion); 4120 (*to) = (caddr_t)eq->desc + portion; 4121 } 4122} 4123 4124static inline void | 4306 4307 bcopy(from, *to, portion); 4308 from += portion; 4309 portion = len - portion; /* remaining */ 4310 bcopy(from, (void *)eq->desc, portion); 4311 (*to) = (caddr_t)eq->desc + portion; 4312 } 4313} 4314 4315static inline void |
4125ring_eq_db(struct adapter *sc, struct sge_eq *eq) | 4316ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) |
4126{ | 4317{ |
4127 u_int db, pending; | 4318 u_int db; |
4128 | 4319 |
4320 MPASS(n > 0); 4321 |
|
4129 db = eq->doorbells; | 4322 db = eq->doorbells; |
4130 pending = eq->pending; 4131 if (pending > 1) | 4323 if (n > 1) |
4132 clrbit(&db, DOORBELL_WCWR); | 4324 clrbit(&db, DOORBELL_WCWR); |
4133 eq->pending = 0; | |
4134 wmb(); 4135 4136 switch (ffs(db) - 1) { 4137 case DOORBELL_UDB: | 4325 wmb(); 4326 4327 switch (ffs(db) - 1) { 4328 case DOORBELL_UDB: |
4138 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending)); 4139 return; | 4329 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 4330 break; |
4140 4141 case DOORBELL_WCWR: { 4142 volatile uint64_t *dst, *src; 4143 int i; 4144 4145 /* 4146 * Queues whose 128B doorbell segment fits in the page do not 4147 * use relative qid (udb_qid is always 0). Only queues with 4148 * doorbell segments can do WCWR. 4149 */ | 4331 4332 case DOORBELL_WCWR: { 4333 volatile uint64_t *dst, *src; 4334 int i; 4335 4336 /* 4337 * Queues whose 128B doorbell segment fits in the page do not 4338 * use relative qid (udb_qid is always 0). Only queues with 4339 * doorbell segments can do WCWR. 4340 */ |
4150 KASSERT(eq->udb_qid == 0 && pending == 1, | 4341 KASSERT(eq->udb_qid == 0 && n == 1, |
4151 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", | 4342 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", |
4152 __func__, eq->doorbells, pending, eq->pidx, eq)); | 4343 __func__, eq->doorbells, n, eq->dbidx, eq)); |
4153 4154 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - 4155 UDBS_DB_OFFSET); | 4344 4345 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - 4346 UDBS_DB_OFFSET); |
4156 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; | 4347 i = eq->dbidx; |
4157 src = (void *)&eq->desc[i]; 4158 while (src != (void *)&eq->desc[i + 1]) 4159 *dst++ = *src++; 4160 wmb(); | 4348 src = (void *)&eq->desc[i]; 4349 while (src != (void *)&eq->desc[i + 1]) 4350 *dst++ = *src++; 4351 wmb(); |
4161 return; | 4352 break; |
4162 } 4163 4164 case DOORBELL_UDBWC: | 4353 } 4354 4355 case DOORBELL_UDBWC: |
4165 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending)); | 4356 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); |
4166 wmb(); | 4357 wmb(); |
4167 return; | 4358 break; |
4168 4169 case DOORBELL_KDB: 4170 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), | 4359 4360 case DOORBELL_KDB: 4361 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), |
4171 V_QID(eq->cntxt_id) | V_PIDX(pending)); 4172 return; | 4362 V_QID(eq->cntxt_id) | V_PIDX(n)); 4363 break; |
4173 } | 4364 } |
4365 4366 IDXINCR(eq->dbidx, n, eq->sidx); |
|
4174} 4175 | 4367} 4368 |
4176static inline int 4177reclaimable(struct sge_eq *eq) | 4369static inline u_int 4370reclaimable_tx_desc(struct sge_eq *eq) |
4178{ | 4371{ |
4179 unsigned int cidx; | 4372 uint16_t hw_cidx; |
4180 | 4373 |
4181 cidx = eq->spg->cidx; /* stable snapshot */ 4182 cidx = be16toh(cidx); | 4374 hw_cidx = read_hw_cidx(eq); 4375 return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); 4376} |
4183 | 4377 |
4184 if (cidx >= eq->cidx) 4185 return (cidx - eq->cidx); | 4378static inline u_int 4379total_available_tx_desc(struct sge_eq *eq) 4380{ 4381 uint16_t hw_cidx, pidx; 4382 4383 hw_cidx = read_hw_cidx(eq); 4384 pidx = eq->pidx; 4385 4386 if (pidx == hw_cidx) 4387 return (eq->sidx - 1); |
4186 else | 4388 else |
4187 return (cidx + eq->cap - eq->cidx); | 4389 return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); |
4188} 4189 | 4390} 4391 |
4392static inline uint16_t 4393read_hw_cidx(struct sge_eq *eq) 4394{ 4395 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4396 uint16_t cidx = spg->cidx; /* stable snapshot */ 4397 4398 return (be16toh(cidx)); 4399} 4400 |
|
4190/* | 4401/* |
4191 * There are "can_reclaim" tx descriptors ready to be reclaimed. Reclaim as 4192 * many as possible but stop when there are around "n" mbufs to free. 4193 * 4194 * The actual number reclaimed is provided as the return value. | 4402 * Reclaim 'n' descriptors approximately. |
4195 */ | 4403 */ |
4196static int 4197reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n) | 4404static u_int 4405reclaim_tx_descs(struct sge_txq *txq, u_int n) |
4198{ 4199 struct tx_sdesc *txsd; | 4406{ 4407 struct tx_sdesc *txsd; |
4200 struct tx_maps *txmaps; 4201 struct tx_map *txm; 4202 unsigned int reclaimed, maps; | |
4203 struct sge_eq *eq = &txq->eq; | 4408 struct sge_eq *eq = &txq->eq; |
4409 u_int can_reclaim, reclaimed; |
|
4204 4205 TXQ_LOCK_ASSERT_OWNED(txq); | 4410 4411 TXQ_LOCK_ASSERT_OWNED(txq); |
4412 MPASS(n > 0); |
|
4206 | 4413 |
4207 if (can_reclaim == 0) 4208 can_reclaim = reclaimable(eq); 4209 4210 maps = reclaimed = 0; 4211 while (can_reclaim && maps < n) { | 4414 reclaimed = 0; 4415 can_reclaim = reclaimable_tx_desc(eq); 4416 while (can_reclaim && reclaimed < n) { |
4212 int ndesc; | 4417 int ndesc; |
4418 struct mbuf *m, *nextpkt; |
|
4213 4214 txsd = &txq->sdesc[eq->cidx]; 4215 ndesc = txsd->desc_used; 4216 4217 /* Firmware doesn't return "partial" credits. */ 4218 KASSERT(can_reclaim >= ndesc, 4219 ("%s: unexpected number of credits: %d, %d", 4220 __func__, can_reclaim, ndesc)); 4221 | 4419 4420 txsd = &txq->sdesc[eq->cidx]; 4421 ndesc = txsd->desc_used; 4422 4423 /* Firmware doesn't return "partial" credits. */ 4424 KASSERT(can_reclaim >= ndesc, 4425 ("%s: unexpected number of credits: %d, %d", 4426 __func__, can_reclaim, ndesc)); 4427 |
4222 maps += txsd->credits; 4223 | 4428 for (m = txsd->m; m != NULL; m = nextpkt) { 4429 nextpkt = m->m_nextpkt; 4430 m->m_nextpkt = NULL; 4431 m_freem(m); 4432 } |
4224 reclaimed += ndesc; 4225 can_reclaim -= ndesc; | 4433 reclaimed += ndesc; 4434 can_reclaim -= ndesc; |
4226 4227 eq->cidx += ndesc; 4228 if (__predict_false(eq->cidx >= eq->cap)) 4229 eq->cidx -= eq->cap; | 4435 IDXINCR(eq->cidx, ndesc, eq->sidx); |
4230 } 4231 | 4436 } 4437 |
4232 txmaps = &txq->txmaps; 4233 txm = &txmaps->maps[txmaps->map_cidx]; 4234 if (maps) 4235 prefetch(txm->m); 4236 4237 eq->avail += reclaimed; 4238 KASSERT(eq->avail < eq->cap, /* avail tops out at (cap - 1) */ 4239 ("%s: too many descriptors available", __func__)); 4240 4241 txmaps->map_avail += maps; 4242 KASSERT(txmaps->map_avail <= txmaps->map_total, 4243 ("%s: too many maps available", __func__)); 4244 4245 while (maps--) { 4246 struct tx_map *next; 4247 4248 next = txm + 1; 4249 if (__predict_false(txmaps->map_cidx + 1 == txmaps->map_total)) 4250 next = txmaps->maps; 4251 prefetch(next->m); 4252 4253 bus_dmamap_unload(txq->tx_tag, txm->map); 4254 m_freem(txm->m); 4255 txm->m = NULL; 4256 4257 txm = next; 4258 if (__predict_false(++txmaps->map_cidx == txmaps->map_total)) 4259 txmaps->map_cidx = 0; 4260 } 4261 | |
4262 return (reclaimed); 4263} 4264 4265static void | 4438 return (reclaimed); 4439} 4440 4441static void |
4266write_eqflush_wr(struct sge_eq *eq) | 4442tx_reclaim(void *arg, int n) |
4267{ | 4443{ |
4268 struct fw_eq_flush_wr *wr; | 4444 struct sge_txq *txq = arg; 4445 struct sge_eq *eq = &txq->eq; |
4269 | 4446 |
4270 EQ_LOCK_ASSERT_OWNED(eq); 4271 KASSERT(eq->avail > 0, ("%s: no descriptors left.", __func__)); 4272 KASSERT(!(eq->flags & EQ_CRFLUSHED), ("%s: flushed already", __func__)); 4273 4274 wr = (void *)&eq->desc[eq->pidx]; 4275 bzero(wr, sizeof(*wr)); 4276 wr->opcode = FW_EQ_FLUSH_WR; 4277 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(sizeof(*wr) / 16) | 4278 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 4279 4280 eq->flags |= (EQ_CRFLUSHED | EQ_STALLED); 4281 eq->pending++; 4282 eq->avail--; 4283 if (++eq->pidx == eq->cap) 4284 eq->pidx = 0; | 4447 do { 4448 if (TXQ_TRYLOCK(txq) == 0) 4449 break; 4450 n = reclaim_tx_descs(txq, 32); 4451 if (eq->cidx == eq->pidx) 4452 eq->equeqidx = eq->pidx; 4453 TXQ_UNLOCK(txq); 4454 } while (n > 0); |
4285} 4286 4287static __be64 | 4455} 4456 4457static __be64 |
4288get_flit(bus_dma_segment_t *sgl, int nsegs, int idx) | 4458get_flit(struct sglist_seg *segs, int nsegs, int idx) |
4289{ 4290 int i = (idx / 3) * 2; 4291 4292 switch (idx % 3) { 4293 case 0: { 4294 __be64 rc; 4295 | 4459{ 4460 int i = (idx / 3) * 2; 4461 4462 switch (idx % 3) { 4463 case 0: { 4464 __be64 rc; 4465 |
4296 rc = htobe32(sgl[i].ds_len); | 4466 rc = htobe32(segs[i].ss_len); |
4297 if (i + 1 < nsegs) | 4467 if (i + 1 < nsegs) |
4298 rc |= (uint64_t)htobe32(sgl[i + 1].ds_len) << 32; | 4468 rc |= (uint64_t)htobe32(segs[i + 1].ss_len) << 32; |
4299 4300 return (rc); 4301 } 4302 case 1: | 4469 4470 return (rc); 4471 } 4472 case 1: |
4303 return htobe64(sgl[i].ds_addr); | 4473 return (htobe64(segs[i].ss_paddr)); |
4304 case 2: | 4474 case 2: |
4305 return htobe64(sgl[i + 1].ds_addr); | 4475 return (htobe64(segs[i + 1].ss_paddr)); |
4306 } 4307 4308 return (0); 4309} 4310 4311static void 4312find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp) 4313{ --- 178 unchanged lines hidden (view full) --- 4492 fl->flags |= FL_STARVING; 4493 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 4494 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); 4495 } 4496 FL_UNLOCK(fl); 4497 mtx_unlock(&sc->sfl_lock); 4498} 4499 | 4476 } 4477 4478 return (0); 4479} 4480 4481static void 4482find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp) 4483{ --- 178 unchanged lines hidden (view full) --- 4662 fl->flags |= FL_STARVING; 4663 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 4664 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); 4665 } 4666 FL_UNLOCK(fl); 4667 mtx_unlock(&sc->sfl_lock); 4668} 4669 |
4670static void 4671handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) 4672{ 4673 struct sge_wrq *wrq = (void *)eq; 4674 4675 atomic_readandclear_int(&eq->equiq); 4676 taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); 4677} 4678 4679static void 4680handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) 4681{ 4682 struct sge_txq *txq = (void *)eq; 4683 4684 MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH); 4685 4686 atomic_readandclear_int(&eq->equiq); 4687 mp_ring_check_drainage(txq->r, 0); 4688 taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); 4689} 4690 |
|
4500static int 4501handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 4502 struct mbuf *m) 4503{ 4504 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 4505 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 4506 struct adapter *sc = iq->adapter; 4507 struct sge *s = &sc->sge; 4508 struct sge_eq *eq; | 4691static int 4692handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 4693 struct mbuf *m) 4694{ 4695 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 4696 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 4697 struct adapter *sc = iq->adapter; 4698 struct sge *s = &sc->sge; 4699 struct sge_eq *eq; |
4700 static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, 4701 &handle_wrq_egr_update, &handle_eth_egr_update, 4702 &handle_wrq_egr_update}; |
|
4509 4510 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 4511 rss->opcode)); 4512 4513 eq = s->eqmap[qid - s->eq_start]; | 4703 4704 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 4705 rss->opcode)); 4706 4707 eq = s->eqmap[qid - s->eq_start]; |
4514 EQ_LOCK(eq); 4515 KASSERT(eq->flags & EQ_CRFLUSHED, 4516 ("%s: unsolicited egress update", __func__)); 4517 eq->flags &= ~EQ_CRFLUSHED; 4518 eq->egr_update++; | 4708 (*h[eq->flags & EQ_TYPEMASK])(sc, eq); |
4519 | 4709 |
4520 if (__predict_false(eq->flags & EQ_DOOMED)) 4521 wakeup_one(eq); 4522 else if (eq->flags & EQ_STALLED && can_resume_tx(eq)) 4523 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4524 EQ_UNLOCK(eq); 4525 | |
4526 return (0); 4527} 4528 4529/* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ 4530CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ 4531 offsetof(struct cpl_fw6_msg, data)); 4532 4533static int --- 52 unchanged lines hidden --- | 4710 return (0); 4711} 4712 4713/* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ 4714CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ 4715 offsetof(struct cpl_fw6_msg, data)); 4716 4717static int --- 52 unchanged lines hidden --- |