Deleted Added
sdiff udiff text old ( 281955 ) new ( 284052 )
full compact
t4_sge.c (281955) t4_sge.c (284052)
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 281955 2015-04-24 23:26:44Z hiren $");
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_sge.c 284052 2015-06-06 09:28:40Z np $");
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/types.h>
35#include <sys/mbuf.h>
36#include <sys/socket.h>
37#include <sys/kernel.h>
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/types.h>
35#include <sys/mbuf.h>
36#include <sys/socket.h>
37#include <sys/kernel.h>
38#include <sys/kdb.h>
39#include <sys/malloc.h>
40#include <sys/queue.h>
41#include <sys/sbuf.h>
42#include <sys/taskqueue.h>
43#include <sys/time.h>
38#include <sys/malloc.h>
39#include <sys/queue.h>
40#include <sys/sbuf.h>
41#include <sys/taskqueue.h>
42#include <sys/time.h>
43#include <sys/sglist.h>
44#include <sys/sysctl.h>
45#include <sys/smp.h>
46#include <sys/counter.h>
47#include <net/bpf.h>
48#include <net/ethernet.h>
49#include <net/if.h>
50#include <net/if_vlan_var.h>
51#include <netinet/in.h>

--- 10 unchanged lines hidden (view full) ---

62#include <net/netmap.h>
63#include <dev/netmap/netmap_kern.h>
64#endif
65
66#include "common/common.h"
67#include "common/t4_regs.h"
68#include "common/t4_regs_values.h"
69#include "common/t4_msg.h"
44#include <sys/sysctl.h>
45#include <sys/smp.h>
46#include <sys/counter.h>
47#include <net/bpf.h>
48#include <net/ethernet.h>
49#include <net/if.h>
50#include <net/if_vlan_var.h>
51#include <netinet/in.h>

--- 10 unchanged lines hidden (view full) ---

62#include <net/netmap.h>
63#include <dev/netmap/netmap_kern.h>
64#endif
65
66#include "common/common.h"
67#include "common/t4_regs.h"
68#include "common/t4_regs_values.h"
69#include "common/t4_msg.h"
70#include "t4_mp_ring.h"
70
71#ifdef T4_PKT_TIMESTAMP
72#define RX_COPY_THRESHOLD (MINCLSIZE - 8)
73#else
74#define RX_COPY_THRESHOLD MINCLSIZE
75#endif
76
77/*

--- 63 unchanged lines hidden (view full) ---

141
142/*
143 * Size of cluster allocation that's most likely to succeed. The driver will
144 * fall back to this size if it fails to allocate clusters larger than this.
145 */
146static int safest_rx_cluster = PAGE_SIZE;
147TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster);
148
71
72#ifdef T4_PKT_TIMESTAMP
73#define RX_COPY_THRESHOLD (MINCLSIZE - 8)
74#else
75#define RX_COPY_THRESHOLD MINCLSIZE
76#endif
77
78/*

--- 63 unchanged lines hidden (view full) ---

142
143/*
144 * Size of cluster allocation that's most likely to succeed. The driver will
145 * fall back to this size if it fails to allocate clusters larger than this.
146 */
147static int safest_rx_cluster = PAGE_SIZE;
148TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster);
149
149/* Used to track coalesced tx work request */
150struct txpkts {
150struct txpkts {
151 uint64_t *flitp; /* ptr to flit where next pkt should start */
152 uint8_t npkt; /* # of packets in this work request */
153 uint8_t nflits; /* # of flits used by this work request */
154 uint16_t plen; /* total payload (sum of all packets) */
151 u_int wr_type; /* type 0 or type 1 */
152 u_int npkt; /* # of packets in this work request */
153 u_int plen; /* total payload (sum of all packets) */
154 u_int len16; /* # of 16B pieces used by this work request */
155};
156
157/* A packet's SGL. This + m_pkthdr has all info needed for tx */
158struct sgl {
155};
156
157/* A packet's SGL. This + m_pkthdr has all info needed for tx */
158struct sgl {
159 int nsegs; /* # of segments in the SGL, 0 means imm. tx */
160 int nflits; /* # of flits needed for the SGL */
161 bus_dma_segment_t seg[TX_SGL_SEGS];
159 struct sglist sg;
160 struct sglist_seg seg[TX_SGL_SEGS];
162};
163
164static int service_iq(struct sge_iq *, int);
165static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
166static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
167static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
168static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *);
169static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,

--- 45 unchanged lines hidden (view full) ---

215static int refill_fl(struct adapter *, struct sge_fl *, int);
216static void refill_sfl(void *);
217static int alloc_fl_sdesc(struct sge_fl *);
218static void free_fl_sdesc(struct adapter *, struct sge_fl *);
219static void find_best_refill_source(struct adapter *, struct sge_fl *, int);
220static void find_safe_refill_source(struct adapter *, struct sge_fl *);
221static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
222
161};
162
163static int service_iq(struct sge_iq *, int);
164static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
165static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
166static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
167static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *);
168static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,

--- 45 unchanged lines hidden (view full) ---

214static int refill_fl(struct adapter *, struct sge_fl *, int);
215static void refill_sfl(void *);
216static int alloc_fl_sdesc(struct sge_fl *);
217static void free_fl_sdesc(struct adapter *, struct sge_fl *);
218static void find_best_refill_source(struct adapter *, struct sge_fl *, int);
219static void find_safe_refill_source(struct adapter *, struct sge_fl *);
220static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
221
223static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
224static int free_pkt_sgl(struct sge_txq *, struct sgl *);
225static int write_txpkt_wr(struct port_info *, struct sge_txq *, struct mbuf *,
226 struct sgl *);
227static int add_to_txpkts(struct port_info *, struct sge_txq *, struct txpkts *,
228 struct mbuf *, struct sgl *);
229static void write_txpkts_wr(struct sge_txq *, struct txpkts *);
230static inline void write_ulp_cpl_sgl(struct port_info *, struct sge_txq *,
231 struct txpkts *, struct mbuf *, struct sgl *);
232static int write_sgl_to_txd(struct sge_eq *, struct sgl *, caddr_t *);
222static inline void get_pkt_gl(struct mbuf *, struct sglist *);
223static inline u_int txpkt_len16(u_int, u_int);
224static inline u_int txpkts0_len16(u_int);
225static inline u_int txpkts1_len16(void);
226static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *,
227 struct mbuf *, u_int);
228static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int);
229static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int);
230static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *,
231 struct mbuf *, const struct txpkts *, u_int);
232static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
233static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
233static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
234static inline void ring_eq_db(struct adapter *, struct sge_eq *);
235static inline int reclaimable(struct sge_eq *);
236static int reclaim_tx_descs(struct sge_txq *, int, int);
237static void write_eqflush_wr(struct sge_eq *);
238static __be64 get_flit(bus_dma_segment_t *, int, int);
234static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
235static inline uint16_t read_hw_cidx(struct sge_eq *);
236static inline u_int reclaimable_tx_desc(struct sge_eq *);
237static inline u_int total_available_tx_desc(struct sge_eq *);
238static u_int reclaim_tx_descs(struct sge_txq *, u_int);
239static void tx_reclaim(void *, int);
240static __be64 get_flit(struct sglist_seg *, int, int);
239static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
240 struct mbuf *);
241static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
242 struct mbuf *);
241static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
242 struct mbuf *);
243static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
244 struct mbuf *);
245static void wrq_tx_drain(void *, int);
246static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *);
243
244static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
245static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
246
247static counter_u64_t extfree_refs;
248static counter_u64_t extfree_rels;
249
250/*

--- 1527 unchanged lines hidden (view full) ---

1778 } else
1779#endif
1780 ifp->if_input(ifp, m0);
1781
1782 return (0);
1783}
1784
1785/*
247
248static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
249static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
250
251static counter_u64_t extfree_refs;
252static counter_u64_t extfree_rels;
253
254/*

--- 1527 unchanged lines hidden (view full) ---

1782 } else
1783#endif
1784 ifp->if_input(ifp, m0);
1785
1786 return (0);
1787}
1788
1789/*
1790 * Must drain the wrq or make sure that someone else will.
1791 */
1792static void
1793wrq_tx_drain(void *arg, int n)
1794{
1795 struct sge_wrq *wrq = arg;
1796 struct sge_eq *eq = &wrq->eq;
1797
1798 EQ_LOCK(eq);
1799 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
1800 drain_wrq_wr_list(wrq->adapter, wrq);
1801 EQ_UNLOCK(eq);
1802}
1803
1804static void
1805drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq)
1806{
1807 struct sge_eq *eq = &wrq->eq;
1808 u_int available, dbdiff; /* # of hardware descriptors */
1809 u_int n;
1810 struct wrqe *wr;
1811 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */
1812
1813 EQ_LOCK_ASSERT_OWNED(eq);
1814 MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
1815 wr = STAILQ_FIRST(&wrq->wr_list);
1816 MPASS(wr != NULL); /* Must be called with something useful to do */
1817 dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx);
1818
1819 do {
1820 eq->cidx = read_hw_cidx(eq);
1821 if (eq->pidx == eq->cidx)
1822 available = eq->sidx - 1;
1823 else
1824 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
1825
1826 MPASS(wr->wrq == wrq);
1827 n = howmany(wr->wr_len, EQ_ESIZE);
1828 if (available < n)
1829 return;
1830
1831 dst = (void *)&eq->desc[eq->pidx];
1832 if (__predict_true(eq->sidx - eq->pidx > n)) {
1833 /* Won't wrap, won't end exactly at the status page. */
1834 bcopy(&wr->wr[0], dst, wr->wr_len);
1835 eq->pidx += n;
1836 } else {
1837 int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE;
1838
1839 bcopy(&wr->wr[0], dst, first_portion);
1840 if (wr->wr_len > first_portion) {
1841 bcopy(&wr->wr[first_portion], &eq->desc[0],
1842 wr->wr_len - first_portion);
1843 }
1844 eq->pidx = n - (eq->sidx - eq->pidx);
1845 }
1846
1847 if (available < eq->sidx / 4 &&
1848 atomic_cmpset_int(&eq->equiq, 0, 1)) {
1849 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
1850 F_FW_WR_EQUEQ);
1851 eq->equeqidx = eq->pidx;
1852 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
1853 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
1854 eq->equeqidx = eq->pidx;
1855 }
1856
1857 dbdiff += n;
1858 if (dbdiff >= 16) {
1859 ring_eq_db(sc, eq, dbdiff);
1860 dbdiff = 0;
1861 }
1862
1863 STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
1864 free_wrqe(wr);
1865 MPASS(wrq->nwr_pending > 0);
1866 wrq->nwr_pending--;
1867 MPASS(wrq->ndesc_needed >= n);
1868 wrq->ndesc_needed -= n;
1869 } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL);
1870
1871 if (dbdiff)
1872 ring_eq_db(sc, eq, dbdiff);
1873}
1874
1875/*
1786 * Doesn't fail. Holds on to work requests it can't send right away.
1787 */
1788void
1789t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
1790{
1876 * Doesn't fail. Holds on to work requests it can't send right away.
1877 */
1878void
1879t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
1880{
1881#ifdef INVARIANTS
1791 struct sge_eq *eq = &wrq->eq;
1882 struct sge_eq *eq = &wrq->eq;
1792 int can_reclaim;
1793 caddr_t dst;
1883#endif
1794
1884
1795 TXQ_LOCK_ASSERT_OWNED(wrq);
1885 EQ_LOCK_ASSERT_OWNED(eq);
1886 MPASS(wr != NULL);
1887 MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN);
1888 MPASS((wr->wr_len & 0x7) == 0);
1889
1890 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
1891 wrq->nwr_pending++;
1892 wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE);
1893
1894 if (!TAILQ_EMPTY(&wrq->incomplete_wrs))
1895 return; /* commit_wrq_wr will drain wr_list as well. */
1896
1897 drain_wrq_wr_list(sc, wrq);
1898
1899 /* Doorbell must have caught up to the pidx. */
1900 MPASS(eq->pidx == eq->dbidx);
1901}
1902
1903void
1904t4_update_fl_bufsize(struct ifnet *ifp)
1905{
1906 struct port_info *pi = ifp->if_softc;
1907 struct adapter *sc = pi->adapter;
1908 struct sge_rxq *rxq;
1796#ifdef TCP_OFFLOAD
1909#ifdef TCP_OFFLOAD
1797 KASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1798 (eq->flags & EQ_TYPEMASK) == EQ_CTRL,
1799 ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
1800#else
1801 KASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL,
1802 ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
1910 struct sge_ofld_rxq *ofld_rxq;
1803#endif
1911#endif
1912 struct sge_fl *fl;
1913 int i, maxp, mtu = ifp->if_mtu;
1804
1914
1805 if (__predict_true(wr != NULL))
1806 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
1915 maxp = mtu_to_max_payload(sc, mtu, 0);
1916 for_each_rxq(pi, i, rxq) {
1917 fl = &rxq->fl;
1807
1918
1808 can_reclaim = reclaimable(eq);
1809 if (__predict_false(eq->flags & EQ_STALLED)) {
1810 if (eq->avail + can_reclaim < tx_resume_threshold(eq))
1811 return;
1812 eq->flags &= ~EQ_STALLED;
1813 eq->unstalled++;
1919 FL_LOCK(fl);
1920 find_best_refill_source(sc, fl, maxp);
1921 FL_UNLOCK(fl);
1814 }
1922 }
1815 eq->cidx += can_reclaim;
1816 eq->avail += can_reclaim;
1817 if (__predict_false(eq->cidx >= eq->cap))
1818 eq->cidx -= eq->cap;
1923#ifdef TCP_OFFLOAD
1924 maxp = mtu_to_max_payload(sc, mtu, 1);
1925 for_each_ofld_rxq(pi, i, ofld_rxq) {
1926 fl = &ofld_rxq->fl;
1819
1927
1820 while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) {
1821 int ndesc;
1928 FL_LOCK(fl);
1929 find_best_refill_source(sc, fl, maxp);
1930 FL_UNLOCK(fl);
1931 }
1932#endif
1933}
1822
1934
1823 if (__predict_false(wr->wr_len < 0 ||
1824 wr->wr_len > SGE_MAX_WR_LEN || (wr->wr_len & 0x7))) {
1935static inline int
1936mbuf_nsegs(struct mbuf *m)
1937{
1825
1938
1826#ifdef INVARIANTS
1827 panic("%s: work request with length %d", __func__,
1828 wr->wr_len);
1829#endif
1830#ifdef KDB
1831 kdb_backtrace();
1832#endif
1833 log(LOG_ERR, "%s: %s work request with length %d",
1834 device_get_nameunit(sc->dev), __func__, wr->wr_len);
1835 STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
1836 free_wrqe(wr);
1837 continue;
1838 }
1939 M_ASSERTPKTHDR(m);
1940 KASSERT(m->m_pkthdr.l5hlen > 0,
1941 ("%s: mbuf %p missing information on # of segments.", __func__, m));
1839
1942
1840 ndesc = howmany(wr->wr_len, EQ_ESIZE);
1841 if (eq->avail < ndesc) {
1842 wrq->no_desc++;
1843 break;
1844 }
1943 return (m->m_pkthdr.l5hlen);
1944}
1845
1945
1846 dst = (void *)&eq->desc[eq->pidx];
1847 copy_to_txd(eq, wrtod(wr), &dst, wr->wr_len);
1946static inline void
1947set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs)
1948{
1848
1949
1849 eq->pidx += ndesc;
1850 eq->avail -= ndesc;
1851 if (__predict_false(eq->pidx >= eq->cap))
1852 eq->pidx -= eq->cap;
1950 M_ASSERTPKTHDR(m);
1951 m->m_pkthdr.l5hlen = nsegs;
1952}
1853
1953
1854 eq->pending += ndesc;
1855 if (eq->pending >= 8)
1856 ring_eq_db(sc, eq);
1954static inline int
1955mbuf_len16(struct mbuf *m)
1956{
1957 int n;
1857
1958
1858 wrq->tx_wrs++;
1859 STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
1860 free_wrqe(wr);
1959 M_ASSERTPKTHDR(m);
1960 n = m->m_pkthdr.PH_loc.eigth[0];
1961 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
1861
1962
1862 if (eq->avail < 8) {
1863 can_reclaim = reclaimable(eq);
1864 eq->cidx += can_reclaim;
1865 eq->avail += can_reclaim;
1866 if (__predict_false(eq->cidx >= eq->cap))
1867 eq->cidx -= eq->cap;
1868 }
1869 }
1963 return (n);
1964}
1870
1965
1871 if (eq->pending)
1872 ring_eq_db(sc, eq);
1966static inline void
1967set_mbuf_len16(struct mbuf *m, uint8_t len16)
1968{
1873
1969
1874 if (wr != NULL) {
1875 eq->flags |= EQ_STALLED;
1876 if (callout_pending(&eq->tx_callout) == 0)
1877 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1878 }
1970 M_ASSERTPKTHDR(m);
1971 m->m_pkthdr.PH_loc.eigth[0] = len16;
1879}
1880
1972}
1973
1881/* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1882#define TXPKTS_PKT_HDR ((\
1883 sizeof(struct ulp_txpkt) + \
1884 sizeof(struct ulptx_idata) + \
1885 sizeof(struct cpl_tx_pkt_core) \
1886 ) / 8)
1974static inline int
1975needs_tso(struct mbuf *m)
1976{
1887
1977
1888/* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1889#define TXPKTS_WR_HDR (\
1890 sizeof(struct fw_eth_tx_pkts_wr) / 8 + \
1891 TXPKTS_PKT_HDR)
1978 M_ASSERTPKTHDR(m);
1892
1979
1893/* Header of a tx WR, before SGL of first packet (in flits) */
1894#define TXPKT_WR_HDR ((\
1895 sizeof(struct fw_eth_tx_pkt_wr) + \
1896 sizeof(struct cpl_tx_pkt_core) \
1897 ) / 8 )
1980 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1981 KASSERT(m->m_pkthdr.tso_segsz > 0,
1982 ("%s: TSO requested in mbuf %p but MSS not provided",
1983 __func__, m));
1984 return (1);
1985 }
1898
1986
1899/* Header of a tx LSO WR, before SGL of first packet (in flits) */
1900#define TXPKT_LSO_WR_HDR ((\
1901 sizeof(struct fw_eth_tx_pkt_wr) + \
1902 sizeof(struct cpl_tx_pkt_lso_core) + \
1903 sizeof(struct cpl_tx_pkt_core) \
1904 ) / 8 )
1987 return (0);
1988}
1905
1989
1906int
1907t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m)
1990static inline int
1991needs_l3_csum(struct mbuf *m)
1908{
1992{
1909 struct port_info *pi = (void *)ifp->if_softc;
1910 struct adapter *sc = pi->adapter;
1911 struct sge_eq *eq = &txq->eq;
1912 struct buf_ring *br = txq->br;
1913 struct mbuf *next;
1914 int rc, coalescing, can_reclaim;
1915 struct txpkts txpkts;
1916 struct sgl sgl;
1917
1993
1918 TXQ_LOCK_ASSERT_OWNED(txq);
1919 KASSERT(m, ("%s: called with nothing to do.", __func__));
1920 KASSERT((eq->flags & EQ_TYPEMASK) == EQ_ETH,
1921 ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
1994 M_ASSERTPKTHDR(m);
1922
1995
1923 prefetch(&eq->desc[eq->pidx]);
1924 prefetch(&txq->sdesc[eq->pidx]);
1996 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO))
1997 return (1);
1998 return (0);
1999}
1925
2000
1926 txpkts.npkt = 0;/* indicates there's nothing in txpkts */
1927 coalescing = 0;
2001static inline int
2002needs_l4_csum(struct mbuf *m)
2003{
1928
2004
1929 can_reclaim = reclaimable(eq);
1930 if (__predict_false(eq->flags & EQ_STALLED)) {
1931 if (eq->avail + can_reclaim < tx_resume_threshold(eq)) {
1932 txq->m = m;
1933 return (0);
1934 }
1935 eq->flags &= ~EQ_STALLED;
1936 eq->unstalled++;
1937 }
2005 M_ASSERTPKTHDR(m);
1938
2006
1939 if (__predict_false(eq->flags & EQ_DOOMED)) {
1940 m_freem(m);
1941 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1942 m_freem(m);
1943 return (ENETDOWN);
2007 if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
2008 CSUM_TCP_IPV6 | CSUM_TSO))
2009 return (1);
2010 return (0);
2011}
2012
2013static inline int
2014needs_vlan_insertion(struct mbuf *m)
2015{
2016
2017 M_ASSERTPKTHDR(m);
2018
2019 if (m->m_flags & M_VLANTAG) {
2020 KASSERT(m->m_pkthdr.ether_vtag != 0,
2021 ("%s: HWVLAN requested in mbuf %p but tag not provided",
2022 __func__, m));
2023 return (1);
1944 }
2024 }
2025 return (0);
2026}
1945
2027
1946 if (eq->avail < 8 && can_reclaim)
1947 reclaim_tx_descs(txq, can_reclaim, 32);
2028static void *
2029m_advance(struct mbuf **pm, int *poffset, int len)
2030{
2031 struct mbuf *m = *pm;
2032 int offset = *poffset;
2033 uintptr_t p = 0;
1948
2034
1949 for (; m; m = next ? next : drbr_dequeue(ifp, br)) {
2035 MPASS(len > 0);
1950
2036
1951 if (eq->avail < 8)
2037 while (len) {
2038 if (offset + len < m->m_len) {
2039 offset += len;
2040 p = mtod(m, uintptr_t) + offset;
1952 break;
2041 break;
2042 }
2043 len -= m->m_len - offset;
2044 m = m->m_next;
2045 offset = 0;
2046 MPASS(m != NULL);
2047 }
2048 *poffset = offset;
2049 *pm = m;
2050 return ((void *)p);
2051}
1953
2052
1954 next = m->m_nextpkt;
1955 m->m_nextpkt = NULL;
2053static inline int
2054same_paddr(char *a, char *b)
2055{
1956
2056
1957 if (next || buf_ring_peek(br))
1958 coalescing = 1;
2057 if (a == b)
2058 return (1);
2059 else if (a != NULL && b != NULL) {
2060 vm_offset_t x = (vm_offset_t)a;
2061 vm_offset_t y = (vm_offset_t)b;
1959
2062
1960 rc = get_pkt_sgl(txq, &m, &sgl, coalescing);
1961 if (rc != 0) {
1962 if (rc == ENOMEM) {
2063 if ((x & PAGE_MASK) == (y & PAGE_MASK) &&
2064 pmap_kextract(x) == pmap_kextract(y))
2065 return (1);
2066 }
1963
2067
1964 /* Short of resources, suspend tx */
2068 return (0);
2069}
1965
2070
1966 m->m_nextpkt = next;
1967 break;
1968 }
2071/*
2072 * Can deal with empty mbufs in the chain that have m_len = 0, but the chain
2073 * must have at least one mbuf that's not empty.
2074 */
2075static inline int
2076count_mbuf_nsegs(struct mbuf *m)
2077{
2078 char *prev_end, *start;
2079 int len, nsegs;
1969
2080
1970 /*
1971 * Unrecoverable error for this packet, throw it away
1972 * and move on to the next. get_pkt_sgl may already
1973 * have freed m (it will be NULL in that case and the
1974 * m_freem here is still safe).
1975 */
2081 MPASS(m != NULL);
1976
2082
1977 m_freem(m);
2083 nsegs = 0;
2084 prev_end = NULL;
2085 for (; m; m = m->m_next) {
2086
2087 len = m->m_len;
2088 if (__predict_false(len == 0))
1978 continue;
2089 continue;
1979 }
2090 start = mtod(m, char *);
1980
2091
1981 if (coalescing &&
1982 add_to_txpkts(pi, txq, &txpkts, m, &sgl) == 0) {
2092 nsegs += sglist_count(start, len);
2093 if (same_paddr(prev_end, start))
2094 nsegs--;
2095 prev_end = start + len;
2096 }
1983
2097
1984 /* Successfully absorbed into txpkts */
2098 MPASS(nsegs > 0);
2099 return (nsegs);
2100}
1985
2101
1986 write_ulp_cpl_sgl(pi, txq, &txpkts, m, &sgl);
1987 goto doorbell;
2102/*
2103 * Analyze the mbuf to determine its tx needs. The mbuf passed in may change:
2104 * a) caller can assume it's been freed if this function returns with an error.
2105 * b) it may get defragged up if the gather list is too long for the hardware.
2106 */
2107int
2108parse_pkt(struct mbuf **mp)
2109{
2110 struct mbuf *m0 = *mp, *m;
2111 int rc, nsegs, defragged = 0, offset;
2112 struct ether_header *eh;
2113 void *l3hdr;
2114#if defined(INET) || defined(INET6)
2115 struct tcphdr *tcp;
2116#endif
2117 uint16_t eh_type;
2118
2119 M_ASSERTPKTHDR(m0);
2120 if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) {
2121 rc = EINVAL;
2122fail:
2123 m_freem(m0);
2124 *mp = NULL;
2125 return (rc);
2126 }
2127restart:
2128 /*
2129 * First count the number of gather list segments in the payload.
2130 * Defrag the mbuf if nsegs exceeds the hardware limit.
2131 */
2132 M_ASSERTPKTHDR(m0);
2133 MPASS(m0->m_pkthdr.len > 0);
2134 nsegs = count_mbuf_nsegs(m0);
2135 if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) {
2136 if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) {
2137 rc = EFBIG;
2138 goto fail;
1988 }
2139 }
2140 *mp = m0 = m; /* update caller's copy after defrag */
2141 goto restart;
2142 }
1989
2143
1990 /*
1991 * We weren't coalescing to begin with, or current frame could
1992 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1993 * given to it can't be coalesced). Either way there should be
1994 * nothing in txpkts.
1995 */
1996 KASSERT(txpkts.npkt == 0,
1997 ("%s: txpkts not empty: %d", __func__, txpkts.npkt));
2144 if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) {
2145 m0 = m_pullup(m0, m0->m_pkthdr.len);
2146 if (m0 == NULL) {
2147 /* Should have left well enough alone. */
2148 rc = EFBIG;
2149 goto fail;
2150 }
2151 *mp = m0; /* update caller's copy after pullup */
2152 goto restart;
2153 }
2154 set_mbuf_nsegs(m0, nsegs);
2155 set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0)));
1998
2156
1999 /* We're sending out individual packets now */
2000 coalescing = 0;
2157 if (!needs_tso(m0))
2158 return (0);
2001
2159
2002 if (eq->avail < 8)
2003 reclaim_tx_descs(txq, 0, 8);
2004 rc = write_txpkt_wr(pi, txq, m, &sgl);
2005 if (rc != 0) {
2160 m = m0;
2161 eh = mtod(m, struct ether_header *);
2162 eh_type = ntohs(eh->ether_type);
2163 if (eh_type == ETHERTYPE_VLAN) {
2164 struct ether_vlan_header *evh = (void *)eh;
2006
2165
2007 /* Short of hardware descriptors, suspend tx */
2166 eh_type = ntohs(evh->evl_proto);
2167 m0->m_pkthdr.l2hlen = sizeof(*evh);
2168 } else
2169 m0->m_pkthdr.l2hlen = sizeof(*eh);
2008
2170
2009 /*
2010 * This is an unlikely but expensive failure. We've
2011 * done all the hard work (DMA mappings etc.) and now we
2012 * can't send out the packet. What's worse, we have to
2013 * spend even more time freeing up everything in sgl.
2014 */
2015 txq->no_desc++;
2016 free_pkt_sgl(txq, &sgl);
2171 offset = 0;
2172 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
2017
2173
2018 m->m_nextpkt = next;
2019 break;
2020 }
2174 switch (eh_type) {
2175#ifdef INET6
2176 case ETHERTYPE_IPV6:
2177 {
2178 struct ip6_hdr *ip6 = l3hdr;
2021
2179
2022 ETHER_BPF_MTAP(ifp, m);
2023 if (sgl.nsegs == 0)
2024 m_freem(m);
2025doorbell:
2026 if (eq->pending >= 8)
2027 ring_eq_db(sc, eq);
2180 MPASS(ip6->ip6_nxt == IPPROTO_TCP);
2028
2181
2029 can_reclaim = reclaimable(eq);
2030 if (can_reclaim >= 32)
2031 reclaim_tx_descs(txq, can_reclaim, 64);
2182 m0->m_pkthdr.l3hlen = sizeof(*ip6);
2183 break;
2032 }
2184 }
2185#endif
2186#ifdef INET
2187 case ETHERTYPE_IP:
2188 {
2189 struct ip *ip = l3hdr;
2033
2190
2034 if (txpkts.npkt > 0)
2035 write_txpkts_wr(txq, &txpkts);
2191 m0->m_pkthdr.l3hlen = ip->ip_hl * 4;
2192 break;
2193 }
2194#endif
2195 default:
2196 panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled"
2197 " with the same INET/INET6 options as the kernel.",
2198 __func__, eh_type);
2199 }
2036
2200
2037 /*
2038 * m not NULL means there was an error but we haven't thrown it away.
2039 * This can happen when we're short of tx descriptors (no_desc) or maybe
2040 * even DMA maps (no_dmamap). Either way, a credit flush and reclaim
2041 * will get things going again.
2042 */
2043 if (m && !(eq->flags & EQ_CRFLUSHED)) {
2044 struct tx_sdesc *txsd = &txq->sdesc[eq->pidx];
2201#if defined(INET) || defined(INET6)
2202 tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen);
2203 m0->m_pkthdr.l4hlen = tcp->th_off * 4;
2204#endif
2205 MPASS(m0 == *mp);
2206 return (0);
2207}
2045
2208
2046 /*
2047 * If EQ_CRFLUSHED is not set then we know we have at least one
2048 * available descriptor because any WR that reduces eq->avail to
2049 * 0 also sets EQ_CRFLUSHED.
2050 */
2051 KASSERT(eq->avail > 0, ("%s: no space for eqflush.", __func__));
2209void *
2210start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie)
2211{
2212 struct sge_eq *eq = &wrq->eq;
2213 struct adapter *sc = wrq->adapter;
2214 int ndesc, available;
2215 struct wrqe *wr;
2216 void *w;
2052
2217
2053 txsd->desc_used = 1;
2054 txsd->credits = 0;
2055 write_eqflush_wr(eq);
2218 MPASS(len16 > 0);
2219 ndesc = howmany(len16, EQ_ESIZE / 16);
2220 MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC);
2221
2222 EQ_LOCK(eq);
2223
2224 if (!STAILQ_EMPTY(&wrq->wr_list))
2225 drain_wrq_wr_list(sc, wrq);
2226
2227 if (!STAILQ_EMPTY(&wrq->wr_list)) {
2228slowpath:
2229 EQ_UNLOCK(eq);
2230 wr = alloc_wrqe(len16 * 16, wrq);
2231 if (__predict_false(wr == NULL))
2232 return (NULL);
2233 cookie->pidx = -1;
2234 cookie->ndesc = ndesc;
2235 return (&wr->wr);
2056 }
2236 }
2057 txq->m = m;
2058
2237
2059 if (eq->pending)
2060 ring_eq_db(sc, eq);
2238 eq->cidx = read_hw_cidx(eq);
2239 if (eq->pidx == eq->cidx)
2240 available = eq->sidx - 1;
2241 else
2242 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2243 if (available < ndesc)
2244 goto slowpath;
2061
2245
2062 reclaim_tx_descs(txq, 0, 128);
2246 cookie->pidx = eq->pidx;
2247 cookie->ndesc = ndesc;
2248 TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link);
2063
2249
2064 if (eq->flags & EQ_STALLED && callout_pending(&eq->tx_callout) == 0)
2065 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
2250 w = &eq->desc[eq->pidx];
2251 IDXINCR(eq->pidx, ndesc, eq->sidx);
2252 if (__predict_false(eq->pidx < ndesc - 1)) {
2253 w = &wrq->ss[0];
2254 wrq->ss_pidx = cookie->pidx;
2255 wrq->ss_len = len16 * 16;
2256 }
2066
2257
2067 return (0);
2258 EQ_UNLOCK(eq);
2259
2260 return (w);
2068}
2069
2070void
2261}
2262
2263void
2071t4_update_fl_bufsize(struct ifnet *ifp)
2264commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie)
2072{
2265{
2073 struct port_info *pi = ifp->if_softc;
2074 struct adapter *sc = pi->adapter;
2075 struct sge_rxq *rxq;
2076#ifdef TCP_OFFLOAD
2077 struct sge_ofld_rxq *ofld_rxq;
2078#endif
2079 struct sge_fl *fl;
2080 int i, maxp, mtu = ifp->if_mtu;
2266 struct sge_eq *eq = &wrq->eq;
2267 struct adapter *sc = wrq->adapter;
2268 int ndesc, pidx;
2269 struct wrq_cookie *prev, *next;
2081
2270
2082 maxp = mtu_to_max_payload(sc, mtu, 0);
2083 for_each_rxq(pi, i, rxq) {
2084 fl = &rxq->fl;
2271 if (cookie->pidx == -1) {
2272 struct wrqe *wr = __containerof(w, struct wrqe, wr);
2085
2273
2086 FL_LOCK(fl);
2087 find_best_refill_source(sc, fl, maxp);
2088 FL_UNLOCK(fl);
2274 t4_wrq_tx(sc, wr);
2275 return;
2089 }
2276 }
2090#ifdef TCP_OFFLOAD
2091 maxp = mtu_to_max_payload(sc, mtu, 1);
2092 for_each_ofld_rxq(pi, i, ofld_rxq) {
2093 fl = &ofld_rxq->fl;
2094
2277
2095 FL_LOCK(fl);
2096 find_best_refill_source(sc, fl, maxp);
2097 FL_UNLOCK(fl);
2278 ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */
2279 pidx = cookie->pidx;
2280 MPASS(pidx >= 0 && pidx < eq->sidx);
2281 if (__predict_false(w == &wrq->ss[0])) {
2282 int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE;
2283
2284 MPASS(wrq->ss_len > n); /* WR had better wrap around. */
2285 bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n);
2286 bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n);
2287 wrq->tx_wrs_ss++;
2288 } else
2289 wrq->tx_wrs_direct++;
2290
2291 EQ_LOCK(eq);
2292 prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link);
2293 next = TAILQ_NEXT(cookie, link);
2294 if (prev == NULL) {
2295 MPASS(pidx == eq->dbidx);
2296 if (next == NULL || ndesc >= 16)
2297 ring_eq_db(wrq->adapter, eq, ndesc);
2298 else {
2299 MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc);
2300 next->pidx = pidx;
2301 next->ndesc += ndesc;
2302 }
2303 } else {
2304 MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc);
2305 prev->ndesc += ndesc;
2098 }
2306 }
2307 TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link);
2308
2309 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
2310 drain_wrq_wr_list(sc, wrq);
2311
2312#ifdef INVARIANTS
2313 if (TAILQ_EMPTY(&wrq->incomplete_wrs)) {
2314 /* Doorbell must have caught up to the pidx. */
2315 MPASS(wrq->eq.pidx == wrq->eq.dbidx);
2316 }
2099#endif
2317#endif
2318 EQ_UNLOCK(eq);
2100}
2101
2319}
2320
2102int
2103can_resume_tx(struct sge_eq *eq)
2321static u_int
2322can_resume_eth_tx(struct mp_ring *r)
2104{
2323{
2324 struct sge_eq *eq = r->cookie;
2105
2325
2106 return (eq->avail + reclaimable(eq) >= tx_resume_threshold(eq));
2326 return (total_available_tx_desc(eq) > eq->sidx / 8);
2107}
2108
2327}
2328
2329static inline int
2330cannot_use_txpkts(struct mbuf *m)
2331{
2332 /* maybe put a GL limit too, to avoid silliness? */
2333
2334 return (needs_tso(m));
2335}
2336
2337/*
2338 * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to
2339 * be consumed. Return the actual number consumed. 0 indicates a stall.
2340 */
2341static u_int
2342eth_tx(struct mp_ring *r, u_int cidx, u_int pidx)
2343{
2344 struct sge_txq *txq = r->cookie;
2345 struct sge_eq *eq = &txq->eq;
2346 struct ifnet *ifp = txq->ifp;
2347 struct port_info *pi = (void *)ifp->if_softc;
2348 struct adapter *sc = pi->adapter;
2349 u_int total, remaining; /* # of packets */
2350 u_int available, dbdiff; /* # of hardware descriptors */
2351 u_int n, next_cidx;
2352 struct mbuf *m0, *tail;
2353 struct txpkts txp;
2354 struct fw_eth_tx_pkts_wr *wr; /* any fw WR struct will do */
2355
2356 remaining = IDXDIFF(pidx, cidx, r->size);
2357 MPASS(remaining > 0); /* Must not be called without work to do. */
2358 total = 0;
2359
2360 TXQ_LOCK(txq);
2361 if (__predict_false((eq->flags & EQ_ENABLED) == 0)) {
2362 while (cidx != pidx) {
2363 m0 = r->items[cidx];
2364 m_freem(m0);
2365 if (++cidx == r->size)
2366 cidx = 0;
2367 }
2368 reclaim_tx_descs(txq, 2048);
2369 total = remaining;
2370 goto done;
2371 }
2372
2373 /* How many hardware descriptors do we have readily available. */
2374 if (eq->pidx == eq->cidx)
2375 available = eq->sidx - 1;
2376 else
2377 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2378 dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx);
2379
2380 while (remaining > 0) {
2381
2382 m0 = r->items[cidx];
2383 M_ASSERTPKTHDR(m0);
2384 MPASS(m0->m_nextpkt == NULL);
2385
2386 if (available < SGE_MAX_WR_NDESC) {
2387 available += reclaim_tx_descs(txq, 64);
2388 if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16))
2389 break; /* out of descriptors */
2390 }
2391
2392 next_cidx = cidx + 1;
2393 if (__predict_false(next_cidx == r->size))
2394 next_cidx = 0;
2395
2396 wr = (void *)&eq->desc[eq->pidx];
2397 if (remaining > 1 &&
2398 try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) {
2399
2400 /* pkts at cidx, next_cidx should both be in txp. */
2401 MPASS(txp.npkt == 2);
2402 tail = r->items[next_cidx];
2403 MPASS(tail->m_nextpkt == NULL);
2404 ETHER_BPF_MTAP(ifp, m0);
2405 ETHER_BPF_MTAP(ifp, tail);
2406 m0->m_nextpkt = tail;
2407
2408 if (__predict_false(++next_cidx == r->size))
2409 next_cidx = 0;
2410
2411 while (next_cidx != pidx) {
2412 if (add_to_txpkts(r->items[next_cidx], &txp,
2413 available) != 0)
2414 break;
2415 tail->m_nextpkt = r->items[next_cidx];
2416 tail = tail->m_nextpkt;
2417 ETHER_BPF_MTAP(ifp, tail);
2418 if (__predict_false(++next_cidx == r->size))
2419 next_cidx = 0;
2420 }
2421
2422 n = write_txpkts_wr(txq, wr, m0, &txp, available);
2423 total += txp.npkt;
2424 remaining -= txp.npkt;
2425 } else {
2426 total++;
2427 remaining--;
2428 n = write_txpkt_wr(txq, (void *)wr, m0, available);
2429 ETHER_BPF_MTAP(ifp, m0);
2430 }
2431 MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC);
2432
2433 available -= n;
2434 dbdiff += n;
2435 IDXINCR(eq->pidx, n, eq->sidx);
2436
2437 if (total_available_tx_desc(eq) < eq->sidx / 4 &&
2438 atomic_cmpset_int(&eq->equiq, 0, 1)) {
2439 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
2440 F_FW_WR_EQUEQ);
2441 eq->equeqidx = eq->pidx;
2442 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
2443 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
2444 eq->equeqidx = eq->pidx;
2445 }
2446
2447 if (dbdiff >= 16 && remaining >= 4) {
2448 ring_eq_db(sc, eq, dbdiff);
2449 available += reclaim_tx_descs(txq, 4 * dbdiff);
2450 dbdiff = 0;
2451 }
2452
2453 cidx = next_cidx;
2454 }
2455 if (dbdiff != 0) {
2456 ring_eq_db(sc, eq, dbdiff);
2457 reclaim_tx_descs(txq, 32);
2458 }
2459done:
2460 TXQ_UNLOCK(txq);
2461
2462 return (total);
2463}
2464
2109static inline void
2110init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
2111 int qsize)
2112{
2113
2114 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
2115 ("%s: bad tmr_idx %d", __func__, tmr_idx));
2116 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */

--- 31 unchanged lines hidden (view full) ---

2148 uint16_t iqid, char *name)
2149{
2150 KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan));
2151 KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype));
2152
2153 eq->flags = eqtype & EQ_TYPEMASK;
2154 eq->tx_chan = tx_chan;
2155 eq->iqid = iqid;
2465static inline void
2466init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
2467 int qsize)
2468{
2469
2470 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
2471 ("%s: bad tmr_idx %d", __func__, tmr_idx));
2472 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */

--- 31 unchanged lines hidden (view full) ---

2504 uint16_t iqid, char *name)
2505{
2506 KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan));
2507 KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype));
2508
2509 eq->flags = eqtype & EQ_TYPEMASK;
2510 eq->tx_chan = tx_chan;
2511 eq->iqid = iqid;
2156 eq->qsize = qsize;
2512 eq->sidx = qsize - spg_len / EQ_ESIZE;
2157 strlcpy(eq->lockname, name, sizeof(eq->lockname));
2513 strlcpy(eq->lockname, name, sizeof(eq->lockname));
2158
2159 TASK_INIT(&eq->tx_task, 0, t4_tx_task, eq);
2160 callout_init(&eq->tx_callout, CALLOUT_MPSAFE);
2161}
2162
2163static int
2164alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
2165 bus_dmamap_t *map, bus_addr_t *pa, void **va)
2166{
2167 int rc;
2168

--- 672 unchanged lines hidden (view full) ---

2841}
2842#endif
2843
2844static int
2845ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
2846{
2847 int rc, cntxt_id;
2848 struct fw_eq_ctrl_cmd c;
2514}
2515
2516static int
2517alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
2518 bus_dmamap_t *map, bus_addr_t *pa, void **va)
2519{
2520 int rc;
2521

--- 672 unchanged lines hidden (view full) ---

3194}
3195#endif
3196
3197static int
3198ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
3199{
3200 int rc, cntxt_id;
3201 struct fw_eq_ctrl_cmd c;
3202 int qsize = eq->sidx + spg_len / EQ_ESIZE;
2849
2850 bzero(&c, sizeof(c));
2851
2852 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
2853 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
2854 V_FW_EQ_CTRL_CMD_VFN(0));
2855 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
2856 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
3203
3204 bzero(&c, sizeof(c));
3205
3206 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
3207 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
3208 V_FW_EQ_CTRL_CMD_VFN(0));
3209 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
3210 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
2857 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* XXX */
3211 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid));
2858 c.physeqid_pkd = htobe32(0);
2859 c.fetchszm_to_iqid =
3212 c.physeqid_pkd = htobe32(0);
3213 c.fetchszm_to_iqid =
2860 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
3214 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
2861 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
2862 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
2863 c.dcaen_to_eqsize =
2864 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
2865 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
3215 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
3216 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
3217 c.dcaen_to_eqsize =
3218 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
3219 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
2866 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
2867 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
3220 V_FW_EQ_CTRL_CMD_EQSIZE(qsize));
2868 c.eqaddr = htobe64(eq->ba);
2869
2870 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2871 if (rc != 0) {
2872 device_printf(sc->dev,
2873 "failed to create control queue %d: %d\n", eq->tx_chan, rc);
2874 return (rc);
2875 }

--- 9 unchanged lines hidden (view full) ---

2885 return (rc);
2886}
2887
2888static int
2889eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
2890{
2891 int rc, cntxt_id;
2892 struct fw_eq_eth_cmd c;
3221 c.eqaddr = htobe64(eq->ba);
3222
3223 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3224 if (rc != 0) {
3225 device_printf(sc->dev,
3226 "failed to create control queue %d: %d\n", eq->tx_chan, rc);
3227 return (rc);
3228 }

--- 9 unchanged lines hidden (view full) ---

3238 return (rc);
3239}
3240
3241static int
3242eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
3243{
3244 int rc, cntxt_id;
3245 struct fw_eq_eth_cmd c;
3246 int qsize = eq->sidx + spg_len / EQ_ESIZE;
2893
2894 bzero(&c, sizeof(c));
2895
2896 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
2897 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
2898 V_FW_EQ_ETH_CMD_VFN(0));
2899 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
2900 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
3247
3248 bzero(&c, sizeof(c));
3249
3250 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
3251 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
3252 V_FW_EQ_ETH_CMD_VFN(0));
3253 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
3254 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
2901 c.autoequiqe_to_viid = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
3255 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
3256 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
2902 c.fetchszm_to_iqid =
3257 c.fetchszm_to_iqid =
2903 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
3258 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
2904 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
2905 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
2906 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
3259 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
3260 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
3261 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
2907 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
2908 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
2909 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
3262 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
3263 V_FW_EQ_ETH_CMD_EQSIZE(qsize));
2910 c.eqaddr = htobe64(eq->ba);
2911
2912 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2913 if (rc != 0) {
2914 device_printf(pi->dev,
2915 "failed to create Ethernet egress queue: %d\n", rc);
2916 return (rc);
2917 }

--- 10 unchanged lines hidden (view full) ---

2928}
2929
2930#ifdef TCP_OFFLOAD
2931static int
2932ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
2933{
2934 int rc, cntxt_id;
2935 struct fw_eq_ofld_cmd c;
3264 c.eqaddr = htobe64(eq->ba);
3265
3266 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3267 if (rc != 0) {
3268 device_printf(pi->dev,
3269 "failed to create Ethernet egress queue: %d\n", rc);
3270 return (rc);
3271 }

--- 10 unchanged lines hidden (view full) ---

3282}
3283
3284#ifdef TCP_OFFLOAD
3285static int
3286ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
3287{
3288 int rc, cntxt_id;
3289 struct fw_eq_ofld_cmd c;
3290 int qsize = eq->sidx + spg_len / EQ_ESIZE;
2936
2937 bzero(&c, sizeof(c));
2938
2939 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
2940 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
2941 V_FW_EQ_OFLD_CMD_VFN(0));
2942 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
2943 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
2944 c.fetchszm_to_iqid =
3291
3292 bzero(&c, sizeof(c));
3293
3294 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
3295 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
3296 V_FW_EQ_OFLD_CMD_VFN(0));
3297 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
3298 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
3299 c.fetchszm_to_iqid =
2945 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
3300 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
2946 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
2947 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
2948 c.dcaen_to_eqsize =
2949 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
2950 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
3301 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
3302 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
3303 c.dcaen_to_eqsize =
3304 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
3305 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
2951 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
2952 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
3306 V_FW_EQ_OFLD_CMD_EQSIZE(qsize));
2953 c.eqaddr = htobe64(eq->ba);
2954
2955 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2956 if (rc != 0) {
2957 device_printf(pi->dev,
2958 "failed to create egress queue for TCP offload: %d\n", rc);
2959 return (rc);
2960 }

--- 8 unchanged lines hidden (view full) ---

2969
2970 return (rc);
2971}
2972#endif
2973
2974static int
2975alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
2976{
3307 c.eqaddr = htobe64(eq->ba);
3308
3309 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3310 if (rc != 0) {
3311 device_printf(pi->dev,
3312 "failed to create egress queue for TCP offload: %d\n", rc);
3313 return (rc);
3314 }

--- 8 unchanged lines hidden (view full) ---

3323
3324 return (rc);
3325}
3326#endif
3327
3328static int
3329alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
3330{
2977 int rc;
3331 int rc, qsize;
2978 size_t len;
2979
2980 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
2981
3332 size_t len;
3333
3334 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
3335
2982 len = eq->qsize * EQ_ESIZE;
3336 qsize = eq->sidx + spg_len / EQ_ESIZE;
3337 len = qsize * EQ_ESIZE;
2983 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
2984 &eq->ba, (void **)&eq->desc);
2985 if (rc)
2986 return (rc);
2987
3338 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
3339 &eq->ba, (void **)&eq->desc);
3340 if (rc)
3341 return (rc);
3342
2988 eq->cap = eq->qsize - spg_len / EQ_ESIZE;
2989 eq->spg = (void *)&eq->desc[eq->cap];
2990 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
2991 eq->pidx = eq->cidx = 0;
3343 eq->pidx = eq->cidx = 0;
3344 eq->equeqidx = eq->dbidx = 0;
2992 eq->doorbells = sc->doorbells;
2993
2994 switch (eq->flags & EQ_TYPEMASK) {
2995 case EQ_CTRL:
2996 rc = ctrl_eq_alloc(sc, eq);
2997 break;
2998
2999 case EQ_ETH:

--- 11 unchanged lines hidden (view full) ---

3011 eq->flags & EQ_TYPEMASK);
3012 }
3013 if (rc != 0) {
3014 device_printf(sc->dev,
3015 "failed to allocate egress queue(%d): %d\n",
3016 eq->flags & EQ_TYPEMASK, rc);
3017 }
3018
3345 eq->doorbells = sc->doorbells;
3346
3347 switch (eq->flags & EQ_TYPEMASK) {
3348 case EQ_CTRL:
3349 rc = ctrl_eq_alloc(sc, eq);
3350 break;
3351
3352 case EQ_ETH:

--- 11 unchanged lines hidden (view full) ---

3364 eq->flags & EQ_TYPEMASK);
3365 }
3366 if (rc != 0) {
3367 device_printf(sc->dev,
3368 "failed to allocate egress queue(%d): %d\n",
3369 eq->flags & EQ_TYPEMASK, rc);
3370 }
3371
3019 eq->tx_callout.c_cpu = eq->cntxt_id % mp_ncpus;
3020
3021 if (isset(&eq->doorbells, DOORBELL_UDB) ||
3022 isset(&eq->doorbells, DOORBELL_UDBWC) ||
3023 isset(&eq->doorbells, DOORBELL_WCWR)) {
3024 uint32_t s_qpp = sc->sge.eq_s_qpp;
3025 uint32_t mask = (1 << s_qpp) - 1;
3026 volatile uint8_t *udb;
3027
3028 udb = sc->udbs_base + UDBS_DB_OFFSET;

--- 65 unchanged lines hidden (view full) ---

3094 struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx;
3095 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3096
3097 rc = alloc_eq(sc, pi, &wrq->eq);
3098 if (rc)
3099 return (rc);
3100
3101 wrq->adapter = sc;
3372 if (isset(&eq->doorbells, DOORBELL_UDB) ||
3373 isset(&eq->doorbells, DOORBELL_UDBWC) ||
3374 isset(&eq->doorbells, DOORBELL_WCWR)) {
3375 uint32_t s_qpp = sc->sge.eq_s_qpp;
3376 uint32_t mask = (1 << s_qpp) - 1;
3377 volatile uint8_t *udb;
3378
3379 udb = sc->udbs_base + UDBS_DB_OFFSET;

--- 65 unchanged lines hidden (view full) ---

3445 struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx;
3446 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3447
3448 rc = alloc_eq(sc, pi, &wrq->eq);
3449 if (rc)
3450 return (rc);
3451
3452 wrq->adapter = sc;
3453 TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq);
3454 TAILQ_INIT(&wrq->incomplete_wrs);
3102 STAILQ_INIT(&wrq->wr_list);
3455 STAILQ_INIT(&wrq->wr_list);
3456 wrq->nwr_pending = 0;
3457 wrq->ndesc_needed = 0;
3103
3104 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3105 &wrq->eq.cntxt_id, 0, "SGE context id of the queue");
3106 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
3107 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I",
3108 "consumer index");
3109 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx",
3110 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I",
3111 "producer index");
3458
3459 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3460 &wrq->eq.cntxt_id, 0, "SGE context id of the queue");
3461 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
3462 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I",
3463 "consumer index");
3464 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx",
3465 CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I",
3466 "producer index");
3112 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs", CTLFLAG_RD,
3113 &wrq->tx_wrs, "# of work requests");
3114 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
3115 &wrq->no_desc, 0,
3116 "# of times queue ran out of hardware descriptors");
3117 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
3118 &wrq->eq.unstalled, 0, "# of times queue recovered after stall");
3467 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD,
3468 &wrq->tx_wrs_direct, "# of work requests (direct)");
3469 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD,
3470 &wrq->tx_wrs_copied, "# of work requests (copied)");
3119
3120 return (rc);
3121}
3122
3123static int
3124free_wrq(struct adapter *sc, struct sge_wrq *wrq)
3125{
3126 int rc;

--- 11 unchanged lines hidden (view full) ---

3138 struct sysctl_oid *oid)
3139{
3140 int rc;
3141 struct adapter *sc = pi->adapter;
3142 struct sge_eq *eq = &txq->eq;
3143 char name[16];
3144 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3145
3471
3472 return (rc);
3473}
3474
3475static int
3476free_wrq(struct adapter *sc, struct sge_wrq *wrq)
3477{
3478 int rc;

--- 11 unchanged lines hidden (view full) ---

3490 struct sysctl_oid *oid)
3491{
3492 int rc;
3493 struct adapter *sc = pi->adapter;
3494 struct sge_eq *eq = &txq->eq;
3495 char name[16];
3496 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3497
3146 rc = alloc_eq(sc, pi, eq);
3147 if (rc)
3148 return (rc);
3149
3150 txq->ifp = pi->ifp;
3151
3152 txq->sdesc = malloc(eq->cap * sizeof(struct tx_sdesc), M_CXGBE,
3153 M_ZERO | M_WAITOK);
3154 txq->br = buf_ring_alloc(eq->qsize, M_CXGBE, M_WAITOK, &eq->eq_lock);
3155
3156 rc = bus_dma_tag_create(sc->dmat, 1, 0, BUS_SPACE_MAXADDR,
3157 BUS_SPACE_MAXADDR, NULL, NULL, 64 * 1024, TX_SGL_SEGS,
3158 BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &txq->tx_tag);
3498 rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx,
3499 M_CXGBE, M_WAITOK);
3159 if (rc != 0) {
3500 if (rc != 0) {
3160 device_printf(sc->dev,
3161 "failed to create tx DMA tag: %d\n", rc);
3501 device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc);
3162 return (rc);
3163 }
3164
3502 return (rc);
3503 }
3504
3165 /*
3166 * We can stuff ~10 frames in an 8-descriptor txpkts WR (8 is the SGE
3167 * limit for any WR). txq->no_dmamap events shouldn't occur if maps is
3168 * sized for the worst case.
3169 */
3170 rc = t4_alloc_tx_maps(&txq->txmaps, txq->tx_tag, eq->qsize * 10 / 8,
3171 M_WAITOK);
3505 rc = alloc_eq(sc, pi, eq);
3172 if (rc != 0) {
3506 if (rc != 0) {
3173 device_printf(sc->dev, "failed to setup tx DMA maps: %d\n", rc);
3507 mp_ring_free(txq->r);
3508 txq->r = NULL;
3174 return (rc);
3175 }
3176
3509 return (rc);
3510 }
3511
3512 /* Can't fail after this point. */
3513
3514 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
3515 txq->ifp = pi->ifp;
3516 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
3517 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3518 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf));
3519 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
3520 M_ZERO | M_WAITOK);
3521
3177 snprintf(name, sizeof(name), "%d", idx);
3178 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3179 NULL, "tx queue");
3180 children = SYSCTL_CHILDREN(oid);
3181
3182 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3183 &eq->cntxt_id, 0, "SGE context id of the queue");
3184 SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",

--- 11 unchanged lines hidden (view full) ---

3196 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
3197 &txq->tso_wrs, "# of TSO work requests");
3198 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
3199 &txq->imm_wrs, "# of work requests with immediate data");
3200 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
3201 &txq->sgl_wrs, "# of work requests with direct SGL");
3202 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
3203 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
3522 snprintf(name, sizeof(name), "%d", idx);
3523 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3524 NULL, "tx queue");
3525 children = SYSCTL_CHILDREN(oid);
3526
3527 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3528 &eq->cntxt_id, 0, "SGE context id of the queue");
3529 SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",

--- 11 unchanged lines hidden (view full) ---

3541 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
3542 &txq->tso_wrs, "# of TSO work requests");
3543 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
3544 &txq->imm_wrs, "# of work requests with immediate data");
3545 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
3546 &txq->sgl_wrs, "# of work requests with direct SGL");
3547 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
3548 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
3204 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_wrs", CTLFLAG_RD,
3205 &txq->txpkts_wrs, "# of txpkts work requests (multiple pkts/WR)");
3206 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_pkts", CTLFLAG_RD,
3207 &txq->txpkts_pkts, "# of frames tx'd using txpkts work requests");
3549 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts0_wrs",
3550 CTLFLAG_RD, &txq->txpkts0_wrs,
3551 "# of txpkts (type 0) work requests");
3552 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts1_wrs",
3553 CTLFLAG_RD, &txq->txpkts1_wrs,
3554 "# of txpkts (type 1) work requests");
3555 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts0_pkts",
3556 CTLFLAG_RD, &txq->txpkts0_pkts,
3557 "# of frames tx'd using type0 txpkts work requests");
3558 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts1_pkts",
3559 CTLFLAG_RD, &txq->txpkts1_pkts,
3560 "# of frames tx'd using type1 txpkts work requests");
3208
3561
3209 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "br_drops", CTLFLAG_RD,
3210 &txq->br->br_drops, "# of drops in the buf_ring for this queue");
3211 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_dmamap", CTLFLAG_RD,
3212 &txq->no_dmamap, 0, "# of times txq ran out of DMA maps");
3213 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
3214 &txq->no_desc, 0, "# of times txq ran out of hardware descriptors");
3215 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "egr_update", CTLFLAG_RD,
3216 &eq->egr_update, 0, "egress update notifications from the SGE");
3217 SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
3218 &eq->unstalled, 0, "# of times txq recovered after stall");
3562 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_enqueues",
3563 CTLFLAG_RD, &txq->r->enqueues,
3564 "# of enqueues to the mp_ring for this queue");
3565 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_drops",
3566 CTLFLAG_RD, &txq->r->drops,
3567 "# of drops in the mp_ring for this queue");
3568 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_starts",
3569 CTLFLAG_RD, &txq->r->starts,
3570 "# of normal consumer starts in the mp_ring for this queue");
3571 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_stalls",
3572 CTLFLAG_RD, &txq->r->stalls,
3573 "# of consumer stalls in the mp_ring for this queue");
3574 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_restarts",
3575 CTLFLAG_RD, &txq->r->restarts,
3576 "# of consumer restarts in the mp_ring for this queue");
3577 SYSCTL_ADD_COUNTER_U64(&pi->ctx, children, OID_AUTO, "r_abdications",
3578 CTLFLAG_RD, &txq->r->abdications,
3579 "# of consumer abdications in the mp_ring for this queue");
3219
3580
3220 return (rc);
3581 return (0);
3221}
3222
3223static int
3224free_txq(struct port_info *pi, struct sge_txq *txq)
3225{
3226 int rc;
3227 struct adapter *sc = pi->adapter;
3228 struct sge_eq *eq = &txq->eq;
3229
3230 rc = free_eq(sc, eq);
3231 if (rc)
3232 return (rc);
3233
3582}
3583
3584static int
3585free_txq(struct port_info *pi, struct sge_txq *txq)
3586{
3587 int rc;
3588 struct adapter *sc = pi->adapter;
3589 struct sge_eq *eq = &txq->eq;
3590
3591 rc = free_eq(sc, eq);
3592 if (rc)
3593 return (rc);
3594
3595 sglist_free(txq->gl);
3234 free(txq->sdesc, M_CXGBE);
3596 free(txq->sdesc, M_CXGBE);
3597 mp_ring_free(txq->r);
3235
3598
3236 if (txq->txmaps.maps)
3237 t4_free_tx_maps(&txq->txmaps, txq->tx_tag);
3238
3239 buf_ring_free(txq->br, M_CXGBE);
3240
3241 if (txq->tx_tag)
3242 bus_dma_tag_destroy(txq->tx_tag);
3243
3244 bzero(txq, sizeof(*txq));
3245 return (0);
3246}
3247
3248static void
3249oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3250{
3251 bus_addr_t *ba = arg;

--- 207 unchanged lines hidden (view full) ---

3459 }
3460 sd->cl = NULL;
3461 }
3462
3463 free(fl->sdesc, M_CXGBE);
3464 fl->sdesc = NULL;
3465}
3466
3599 bzero(txq, sizeof(*txq));
3600 return (0);
3601}
3602
3603static void
3604oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3605{
3606 bus_addr_t *ba = arg;

--- 207 unchanged lines hidden (view full) ---

3814 }
3815 sd->cl = NULL;
3816 }
3817
3818 free(fl->sdesc, M_CXGBE);
3819 fl->sdesc = NULL;
3820}
3821
3467int
3468t4_alloc_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag, int count,
3469 int flags)
3822static inline void
3823get_pkt_gl(struct mbuf *m, struct sglist *gl)
3470{
3824{
3471 struct tx_map *txm;
3472 int i, rc;
3825 int rc;
3473
3826
3474 txmaps->map_total = txmaps->map_avail = count;
3475 txmaps->map_cidx = txmaps->map_pidx = 0;
3827 M_ASSERTPKTHDR(m);
3476
3828
3477 txmaps->maps = malloc(count * sizeof(struct tx_map), M_CXGBE,
3478 M_ZERO | flags);
3479
3480 txm = txmaps->maps;
3481 for (i = 0; i < count; i++, txm++) {
3482 rc = bus_dmamap_create(tx_tag, 0, &txm->map);
3483 if (rc != 0)
3484 goto failed;
3829 sglist_reset(gl);
3830 rc = sglist_append_mbuf(gl, m);
3831 if (__predict_false(rc != 0)) {
3832 panic("%s: mbuf %p (%d segs) was vetted earlier but now fails "
3833 "with %d.", __func__, m, mbuf_nsegs(m), rc);
3485 }
3486
3834 }
3835
3487 return (0);
3488failed:
3489 while (--i >= 0) {
3490 txm--;
3491 bus_dmamap_destroy(tx_tag, txm->map);
3492 }
3493 KASSERT(txm == txmaps->maps, ("%s: EDOOFUS", __func__));
3494
3495 free(txmaps->maps, M_CXGBE);
3496 txmaps->maps = NULL;
3497
3498 return (rc);
3836 KASSERT(gl->sg_nseg == mbuf_nsegs(m),
3837 ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m,
3838 mbuf_nsegs(m), gl->sg_nseg));
3839 KASSERT(gl->sg_nseg > 0 &&
3840 gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS),
3841 ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__,
3842 gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS));
3499}
3500
3843}
3844
3501void
3502t4_free_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag)
3845/*
3846 * len16 for a txpkt WR with a GL. Includes the firmware work request header.
3847 */
3848static inline u_int
3849txpkt_len16(u_int nsegs, u_int tso)
3503{
3850{
3504 struct tx_map *txm;
3505 int i;
3851 u_int n;
3506
3852
3507 txm = txmaps->maps;
3508 for (i = 0; i < txmaps->map_total; i++, txm++) {
3853 MPASS(nsegs > 0);
3509
3854
3510 if (txm->m) {
3511 bus_dmamap_unload(tx_tag, txm->map);
3512 m_freem(txm->m);
3513 txm->m = NULL;
3514 }
3855 nsegs--; /* first segment is part of ulptx_sgl */
3856 n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) +
3857 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
3858 if (tso)
3859 n += sizeof(struct cpl_tx_pkt_lso_core);
3515
3860
3516 bus_dmamap_destroy(tx_tag, txm->map);
3517 }
3518
3519 free(txmaps->maps, M_CXGBE);
3520 txmaps->maps = NULL;
3861 return (howmany(n, 16));
3521}
3522
3523/*
3862}
3863
3864/*
3524 * We'll do immediate data tx for non-TSO, but only when not coalescing. We're
3525 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
3526 * of immediate data.
3865 * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work
3866 * request header.
3527 */
3867 */
3528#define IMM_LEN ( \
3529 2 * EQ_ESIZE \
3530 - sizeof(struct fw_eth_tx_pkt_wr) \
3531 - sizeof(struct cpl_tx_pkt_core))
3532
3533/*
3534 * Returns non-zero on failure, no need to cleanup anything in that case.
3535 *
3536 * Note 1: We always try to defrag the mbuf if required and return EFBIG only
3537 * if the resulting chain still won't fit in a tx descriptor.
3538 *
3539 * Note 2: We'll pullup the mbuf chain if TSO is requested and the first mbuf
3540 * does not have the TCP header in it.
3541 */
3542static int
3543get_pkt_sgl(struct sge_txq *txq, struct mbuf **fp, struct sgl *sgl,
3544 int sgl_only)
3868static inline u_int
3869txpkts0_len16(u_int nsegs)
3545{
3870{
3546 struct mbuf *m = *fp;
3547 struct tx_maps *txmaps;
3548 struct tx_map *txm;
3549 int rc, defragged = 0, n;
3871 u_int n;
3550
3872
3551 TXQ_LOCK_ASSERT_OWNED(txq);
3873 MPASS(nsegs > 0);
3552
3874
3553 if (m->m_pkthdr.tso_segsz)
3554 sgl_only = 1; /* Do not allow immediate data with LSO */
3875 nsegs--; /* first segment is part of ulptx_sgl */
3876 n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) +
3877 sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) +
3878 8 * ((3 * nsegs) / 2 + (nsegs & 1));
3555
3879
3556start: sgl->nsegs = 0;
3557
3558 if (m->m_pkthdr.len <= IMM_LEN && !sgl_only)
3559 return (0); /* nsegs = 0 tells caller to use imm. tx */
3560
3561 txmaps = &txq->txmaps;
3562 if (txmaps->map_avail == 0) {
3563 txq->no_dmamap++;
3564 return (ENOMEM);
3565 }
3566 txm = &txmaps->maps[txmaps->map_pidx];
3567
3568 if (m->m_pkthdr.tso_segsz && m->m_len < 50) {
3569 *fp = m_pullup(m, 50);
3570 m = *fp;
3571 if (m == NULL)
3572 return (ENOBUFS);
3573 }
3574
3575 rc = bus_dmamap_load_mbuf_sg(txq->tx_tag, txm->map, m, sgl->seg,
3576 &sgl->nsegs, BUS_DMA_NOWAIT);
3577 if (rc == EFBIG && defragged == 0) {
3578 m = m_defrag(m, M_NOWAIT);
3579 if (m == NULL)
3580 return (EFBIG);
3581
3582 defragged = 1;
3583 *fp = m;
3584 goto start;
3585 }
3586 if (rc != 0)
3587 return (rc);
3588
3589 txm->m = m;
3590 txmaps->map_avail--;
3591 if (++txmaps->map_pidx == txmaps->map_total)
3592 txmaps->map_pidx = 0;
3593
3594 KASSERT(sgl->nsegs > 0 && sgl->nsegs <= TX_SGL_SEGS,
3595 ("%s: bad DMA mapping (%d segments)", __func__, sgl->nsegs));
3596
3597 /*
3598 * Store the # of flits required to hold this frame's SGL in nflits. An
3599 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
3600 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
3601 * then len1 must be set to 0.
3602 */
3603 n = sgl->nsegs - 1;
3604 sgl->nflits = (3 * n) / 2 + (n & 1) + 2;
3605
3606 return (0);
3880 return (howmany(n, 16));
3607}
3608
3881}
3882
3609
3610/*
3883/*
3611 * Releases all the txq resources used up in the specified sgl.
3884 * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work
3885 * request header.
3612 */
3886 */
3613static int
3614free_pkt_sgl(struct sge_txq *txq, struct sgl *sgl)
3887static inline u_int
3888txpkts1_len16(void)
3615{
3889{
3616 struct tx_maps *txmaps;
3617 struct tx_map *txm;
3890 u_int n;
3618
3891
3619 TXQ_LOCK_ASSERT_OWNED(txq);
3892 n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl);
3620
3893
3621 if (sgl->nsegs == 0)
3622 return (0); /* didn't use any map */
3894 return (howmany(n, 16));
3895}
3623
3896
3624 txmaps = &txq->txmaps;
3897static inline u_int
3898imm_payload(u_int ndesc)
3899{
3900 u_int n;
3625
3901
3626 /* 1 pkt uses exactly 1 map, back it out */
3902 n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) -
3903 sizeof(struct cpl_tx_pkt_core);
3627
3904
3628 txmaps->map_avail++;
3629 if (txmaps->map_pidx > 0)
3630 txmaps->map_pidx--;
3631 else
3632 txmaps->map_pidx = txmaps->map_total - 1;
3633
3634 txm = &txmaps->maps[txmaps->map_pidx];
3635 bus_dmamap_unload(txq->tx_tag, txm->map);
3636 txm->m = NULL;
3637
3638 return (0);
3905 return (n);
3639}
3640
3906}
3907
3641static int
3642write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, struct mbuf *m,
3643 struct sgl *sgl)
3908/*
3909 * Write a txpkt WR for this packet to the hardware descriptors, update the
3910 * software descriptor, and advance the pidx. It is guaranteed that enough
3911 * descriptors are available.
3912 *
3913 * The return value is the # of hardware descriptors used.
3914 */
3915static u_int
3916write_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr,
3917 struct mbuf *m0, u_int available)
3644{
3645 struct sge_eq *eq = &txq->eq;
3918{
3919 struct sge_eq *eq = &txq->eq;
3646 struct fw_eth_tx_pkt_wr *wr;
3920 struct tx_sdesc *txsd;
3647 struct cpl_tx_pkt_core *cpl;
3648 uint32_t ctrl; /* used in many unrelated places */
3649 uint64_t ctrl1;
3921 struct cpl_tx_pkt_core *cpl;
3922 uint32_t ctrl; /* used in many unrelated places */
3923 uint64_t ctrl1;
3650 int nflits, ndesc, pktlen;
3651 struct tx_sdesc *txsd;
3924 int len16, ndesc, pktlen, nsegs;
3652 caddr_t dst;
3653
3654 TXQ_LOCK_ASSERT_OWNED(txq);
3925 caddr_t dst;
3926
3927 TXQ_LOCK_ASSERT_OWNED(txq);
3928 M_ASSERTPKTHDR(m0);
3929 MPASS(available > 0 && available < eq->sidx);
3655
3930
3656 pktlen = m->m_pkthdr.len;
3657
3658 /*
3659 * Do we have enough flits to send this frame out?
3660 */
3931 len16 = mbuf_len16(m0);
3932 nsegs = mbuf_nsegs(m0);
3933 pktlen = m0->m_pkthdr.len;
3661 ctrl = sizeof(struct cpl_tx_pkt_core);
3934 ctrl = sizeof(struct cpl_tx_pkt_core);
3662 if (m->m_pkthdr.tso_segsz) {
3663 nflits = TXPKT_LSO_WR_HDR;
3935 if (needs_tso(m0))
3664 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
3936 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
3665 } else
3666 nflits = TXPKT_WR_HDR;
3667 if (sgl->nsegs > 0)
3668 nflits += sgl->nflits;
3669 else {
3670 nflits += howmany(pktlen, 8);
3937 else if (pktlen <= imm_payload(2) && available >= 2) {
3938 /* Immediate data. Recalculate len16 and set nsegs to 0. */
3671 ctrl += pktlen;
3939 ctrl += pktlen;
3940 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) +
3941 sizeof(struct cpl_tx_pkt_core) + pktlen, 16);
3942 nsegs = 0;
3672 }
3943 }
3673 ndesc = howmany(nflits, 8);
3674 if (ndesc > eq->avail)
3675 return (ENOMEM);
3944 ndesc = howmany(len16, EQ_ESIZE / 16);
3945 MPASS(ndesc <= available);
3676
3677 /* Firmware work request header */
3946
3947 /* Firmware work request header */
3678 wr = (void *)&eq->desc[eq->pidx];
3948 MPASS(wr == (void *)&eq->desc[eq->pidx]);
3679 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
3680 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
3949 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
3950 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
3681 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
3682 if (eq->avail == ndesc) {
3683 if (!(eq->flags & EQ_CRFLUSHED)) {
3684 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
3685 eq->flags |= EQ_CRFLUSHED;
3686 }
3687 eq->flags |= EQ_STALLED;
3688 }
3689
3951
3952 ctrl = V_FW_WR_LEN16(len16);
3690 wr->equiq_to_len16 = htobe32(ctrl);
3691 wr->r3 = 0;
3692
3953 wr->equiq_to_len16 = htobe32(ctrl);
3954 wr->r3 = 0;
3955
3693 if (m->m_pkthdr.tso_segsz) {
3956 if (needs_tso(m0)) {
3694 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
3957 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
3695 struct ether_header *eh;
3696 void *l3hdr;
3697#if defined(INET) || defined(INET6)
3698 struct tcphdr *tcp;
3699#endif
3700 uint16_t eh_type;
3701
3958
3702 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
3703 F_LSO_LAST_SLICE;
3959 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
3960 m0->m_pkthdr.l4hlen > 0,
3961 ("%s: mbuf %p needs TSO but missing header lengths",
3962 __func__, m0));
3704
3963
3705 eh = mtod(m, struct ether_header *);
3706 eh_type = ntohs(eh->ether_type);
3707 if (eh_type == ETHERTYPE_VLAN) {
3708 struct ether_vlan_header *evh = (void *)eh;
3709
3964 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
3965 F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2)
3966 | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
3967 if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header))
3710 ctrl |= V_LSO_ETHHDR_LEN(1);
3968 ctrl |= V_LSO_ETHHDR_LEN(1);
3711 l3hdr = evh + 1;
3712 eh_type = ntohs(evh->evl_proto);
3713 } else
3714 l3hdr = eh + 1;
3715
3716 switch (eh_type) {
3717#ifdef INET6
3718 case ETHERTYPE_IPV6:
3719 {
3720 struct ip6_hdr *ip6 = l3hdr;
3721
3722 /*
3723 * XXX-BZ For now we do not pretend to support
3724 * IPv6 extension headers.
3725 */
3726 KASSERT(ip6->ip6_nxt == IPPROTO_TCP, ("%s: CSUM_TSO "
3727 "with ip6_nxt != TCP: %u", __func__, ip6->ip6_nxt));
3728 tcp = (struct tcphdr *)(ip6 + 1);
3969 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
3729 ctrl |= F_LSO_IPV6;
3970 ctrl |= F_LSO_IPV6;
3730 ctrl |= V_LSO_IPHDR_LEN(sizeof(*ip6) >> 2) |
3731 V_LSO_TCPHDR_LEN(tcp->th_off);
3732 break;
3733 }
3734#endif
3735#ifdef INET
3736 case ETHERTYPE_IP:
3737 {
3738 struct ip *ip = l3hdr;
3739
3971
3740 tcp = (void *)((uintptr_t)ip + ip->ip_hl * 4);
3741 ctrl |= V_LSO_IPHDR_LEN(ip->ip_hl) |
3742 V_LSO_TCPHDR_LEN(tcp->th_off);
3743 break;
3744 }
3745#endif
3746 default:
3747 panic("%s: CSUM_TSO but no supported IP version "
3748 "(0x%04x)", __func__, eh_type);
3749 }
3750
3751 lso->lso_ctrl = htobe32(ctrl);
3752 lso->ipid_ofst = htobe16(0);
3972 lso->lso_ctrl = htobe32(ctrl);
3973 lso->ipid_ofst = htobe16(0);
3753 lso->mss = htobe16(m->m_pkthdr.tso_segsz);
3974 lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
3754 lso->seqno_offset = htobe32(0);
3755 lso->len = htobe32(pktlen);
3756
3757 cpl = (void *)(lso + 1);
3758
3759 txq->tso_wrs++;
3760 } else
3761 cpl = (void *)(wr + 1);
3762
3763 /* Checksum offload */
3764 ctrl1 = 0;
3975 lso->seqno_offset = htobe32(0);
3976 lso->len = htobe32(pktlen);
3977
3978 cpl = (void *)(lso + 1);
3979
3980 txq->tso_wrs++;
3981 } else
3982 cpl = (void *)(wr + 1);
3983
3984 /* Checksum offload */
3985 ctrl1 = 0;
3765 if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)))
3986 if (needs_l3_csum(m0) == 0)
3766 ctrl1 |= F_TXPKT_IPCSUM_DIS;
3987 ctrl1 |= F_TXPKT_IPCSUM_DIS;
3767 if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
3768 CSUM_TCP_IPV6 | CSUM_TSO)))
3988 if (needs_l4_csum(m0) == 0)
3769 ctrl1 |= F_TXPKT_L4CSUM_DIS;
3989 ctrl1 |= F_TXPKT_L4CSUM_DIS;
3770 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
3990 if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
3771 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
3772 txq->txcsum++; /* some hardware assistance provided */
3773
3774 /* VLAN tag insertion */
3991 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
3992 txq->txcsum++; /* some hardware assistance provided */
3993
3994 /* VLAN tag insertion */
3775 if (m->m_flags & M_VLANTAG) {
3776 ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
3995 if (needs_vlan_insertion(m0)) {
3996 ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
3777 txq->vlan_insertion++;
3778 }
3779
3780 /* CPL header */
3997 txq->vlan_insertion++;
3998 }
3999
4000 /* CPL header */
3781 cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3782 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
4001 cpl->ctrl0 = txq->cpl_ctrl0;
3783 cpl->pack = 0;
3784 cpl->len = htobe16(pktlen);
3785 cpl->ctrl1 = htobe64(ctrl1);
3786
4002 cpl->pack = 0;
4003 cpl->len = htobe16(pktlen);
4004 cpl->ctrl1 = htobe64(ctrl1);
4005
3787 /* Software descriptor */
3788 txsd = &txq->sdesc[eq->pidx];
3789 txsd->desc_used = ndesc;
3790
3791 eq->pending += ndesc;
3792 eq->avail -= ndesc;
3793 eq->pidx += ndesc;
3794 if (eq->pidx >= eq->cap)
3795 eq->pidx -= eq->cap;
3796
3797 /* SGL */
3798 dst = (void *)(cpl + 1);
4006 /* SGL */
4007 dst = (void *)(cpl + 1);
3799 if (sgl->nsegs > 0) {
3800 txsd->credits = 1;
4008 if (nsegs > 0) {
4009
4010 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
3801 txq->sgl_wrs++;
4011 txq->sgl_wrs++;
3802 write_sgl_to_txd(eq, sgl, &dst);
3803 } else {
4012 } else {
3804 txsd->credits = 0;
3805 txq->imm_wrs++;
3806 for (; m; m = m->m_next) {
4013 struct mbuf *m;
4014
4015 for (m = m0; m != NULL; m = m->m_next) {
3807 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
3808#ifdef INVARIANTS
3809 pktlen -= m->m_len;
3810#endif
3811 }
3812#ifdef INVARIANTS
3813 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
3814#endif
4016 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
4017#ifdef INVARIANTS
4018 pktlen -= m->m_len;
4019#endif
4020 }
4021#ifdef INVARIANTS
4022 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
4023#endif
3815
4024 txq->imm_wrs++;
3816 }
3817
3818 txq->txpkt_wrs++;
4025 }
4026
4027 txq->txpkt_wrs++;
3819 return (0);
4028
4029 txsd = &txq->sdesc[eq->pidx];
4030 txsd->m = m0;
4031 txsd->desc_used = ndesc;
4032
4033 return (ndesc);
3820}
3821
4034}
4035
3822/*
3823 * Returns 0 to indicate that m has been accepted into a coalesced tx work
3824 * request. It has either been folded into txpkts or txpkts was flushed and m
3825 * has started a new coalesced work request (as the first frame in a fresh
3826 * txpkts).
3827 *
3828 * Returns non-zero to indicate a failure - caller is responsible for
3829 * transmitting m, if there was anything in txpkts it has been flushed.
3830 */
3831static int
4036static int
3832add_to_txpkts(struct port_info *pi, struct sge_txq *txq, struct txpkts *txpkts,
3833 struct mbuf *m, struct sgl *sgl)
4037try_txpkts(struct mbuf *m, struct mbuf *n, struct txpkts *txp, u_int available)
3834{
4038{
3835 struct sge_eq *eq = &txq->eq;
3836 int can_coalesce;
3837 struct tx_sdesc *txsd;
3838 int flits;
4039 u_int needed, nsegs1, nsegs2, l1, l2;
3839
4040
3840 TXQ_LOCK_ASSERT_OWNED(txq);
4041 if (cannot_use_txpkts(m) || cannot_use_txpkts(n))
4042 return (1);
3841
4043
3842 KASSERT(sgl->nsegs, ("%s: can't coalesce imm data", __func__));
4044 nsegs1 = mbuf_nsegs(m);
4045 nsegs2 = mbuf_nsegs(n);
4046 if (nsegs1 + nsegs2 == 2) {
4047 txp->wr_type = 1;
4048 l1 = l2 = txpkts1_len16();
4049 } else {
4050 txp->wr_type = 0;
4051 l1 = txpkts0_len16(nsegs1);
4052 l2 = txpkts0_len16(nsegs2);
4053 }
4054 txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + l1 + l2;
4055 needed = howmany(txp->len16, EQ_ESIZE / 16);
4056 if (needed > SGE_MAX_WR_NDESC || needed > available)
4057 return (1);
3843
4058
3844 if (txpkts->npkt > 0) {
3845 flits = TXPKTS_PKT_HDR + sgl->nflits;
3846 can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
3847 txpkts->nflits + flits <= TX_WR_FLITS &&
3848 txpkts->nflits + flits <= eq->avail * 8 &&
3849 txpkts->plen + m->m_pkthdr.len < 65536;
4059 txp->plen = m->m_pkthdr.len + n->m_pkthdr.len;
4060 if (txp->plen > 65535)
4061 return (1);
3850
4062
3851 if (can_coalesce) {
3852 txpkts->npkt++;
3853 txpkts->nflits += flits;
3854 txpkts->plen += m->m_pkthdr.len;
4063 txp->npkt = 2;
4064 set_mbuf_len16(m, l1);
4065 set_mbuf_len16(n, l2);
3855
4066
3856 txsd = &txq->sdesc[eq->pidx];
3857 txsd->credits++;
4067 return (0);
4068}
3858
4069
3859 return (0);
3860 }
4070static int
4071add_to_txpkts(struct mbuf *m, struct txpkts *txp, u_int available)
4072{
4073 u_int plen, len16, needed, nsegs;
3861
4074
3862 /*
3863 * Couldn't coalesce m into txpkts. The first order of business
3864 * is to send txpkts on its way. Then we'll revisit m.
3865 */
3866 write_txpkts_wr(txq, txpkts);
3867 }
4075 MPASS(txp->wr_type == 0 || txp->wr_type == 1);
3868
4076
3869 /*
3870 * Check if we can start a new coalesced tx work request with m as
3871 * the first packet in it.
3872 */
4077 nsegs = mbuf_nsegs(m);
4078 if (needs_tso(m) || (txp->wr_type == 1 && nsegs != 1))
4079 return (1);
3873
4080
3874 KASSERT(txpkts->npkt == 0, ("%s: txpkts not empty", __func__));
4081 plen = txp->plen + m->m_pkthdr.len;
4082 if (plen > 65535)
4083 return (1);
3875
4084
3876 flits = TXPKTS_WR_HDR + sgl->nflits;
3877 can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
3878 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
4085 if (txp->wr_type == 0)
4086 len16 = txpkts0_len16(nsegs);
4087 else
4088 len16 = txpkts1_len16();
4089 needed = howmany(txp->len16 + len16, EQ_ESIZE / 16);
4090 if (needed > SGE_MAX_WR_NDESC || needed > available)
4091 return (1);
3879
4092
3880 if (can_coalesce == 0)
3881 return (EINVAL);
4093 txp->npkt++;
4094 txp->plen = plen;
4095 txp->len16 += len16;
4096 set_mbuf_len16(m, len16);
3882
4097
3883 /*
3884 * Start a fresh coalesced tx WR with m as the first frame in it.
3885 */
3886 txpkts->npkt = 1;
3887 txpkts->nflits = flits;
3888 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
3889 txpkts->plen = m->m_pkthdr.len;
3890
3891 txsd = &txq->sdesc[eq->pidx];
3892 txsd->credits = 1;
3893
3894 return (0);
3895}
3896
3897/*
4098 return (0);
4099}
4100
4101/*
3898 * Note that write_txpkts_wr can never run out of hardware descriptors (but
3899 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
3900 * coalescing only if sufficient hardware descriptors are available.
4102 * Write a txpkts WR for the packets in txp to the hardware descriptors, update
4103 * the software descriptor, and advance the pidx. It is guaranteed that enough
4104 * descriptors are available.
4105 *
4106 * The return value is the # of hardware descriptors used.
3901 */
4107 */
3902static void
3903write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
4108static u_int
4109write_txpkts_wr(struct sge_txq *txq, struct fw_eth_tx_pkts_wr *wr,
4110 struct mbuf *m0, const struct txpkts *txp, u_int available)
3904{
3905 struct sge_eq *eq = &txq->eq;
4111{
4112 struct sge_eq *eq = &txq->eq;
3906 struct fw_eth_tx_pkts_wr *wr;
3907 struct tx_sdesc *txsd;
4113 struct tx_sdesc *txsd;
4114 struct cpl_tx_pkt_core *cpl;
3908 uint32_t ctrl;
4115 uint32_t ctrl;
3909 int ndesc;
4116 uint64_t ctrl1;
4117 int ndesc, checkwrap;
4118 struct mbuf *m;
4119 void *flitp;
3910
3911 TXQ_LOCK_ASSERT_OWNED(txq);
4120
4121 TXQ_LOCK_ASSERT_OWNED(txq);
4122 MPASS(txp->npkt > 0);
4123 MPASS(txp->plen < 65536);
4124 MPASS(m0 != NULL);
4125 MPASS(m0->m_nextpkt != NULL);
4126 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
4127 MPASS(available > 0 && available < eq->sidx);
3912
4128
3913 ndesc = howmany(txpkts->nflits, 8);
4129 ndesc = howmany(txp->len16, EQ_ESIZE / 16);
4130 MPASS(ndesc <= available);
3914
4131
3915 wr = (void *)&eq->desc[eq->pidx];
4132 MPASS(wr == (void *)&eq->desc[eq->pidx]);
3916 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
4133 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
3917 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
3918 if (eq->avail == ndesc) {
3919 if (!(eq->flags & EQ_CRFLUSHED)) {
3920 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
3921 eq->flags |= EQ_CRFLUSHED;
3922 }
3923 eq->flags |= EQ_STALLED;
3924 }
4134 ctrl = V_FW_WR_LEN16(txp->len16);
3925 wr->equiq_to_len16 = htobe32(ctrl);
4135 wr->equiq_to_len16 = htobe32(ctrl);
3926 wr->plen = htobe16(txpkts->plen);
3927 wr->npkt = txpkts->npkt;
3928 wr->r3 = wr->type = 0;
4136 wr->plen = htobe16(txp->plen);
4137 wr->npkt = txp->npkt;
4138 wr->r3 = 0;
4139 wr->type = txp->wr_type;
4140 flitp = wr + 1;
3929
4141
3930 /* Everything else already written */
4142 /*
4143 * At this point we are 16B into a hardware descriptor. If checkwrap is
4144 * set then we know the WR is going to wrap around somewhere. We'll
4145 * check for that at appropriate points.
4146 */
4147 checkwrap = eq->sidx - ndesc < eq->pidx;
4148 for (m = m0; m != NULL; m = m->m_nextpkt) {
4149 if (txp->wr_type == 0) {
4150 struct ulp_txpkt *ulpmc;
4151 struct ulptx_idata *ulpsc;
3931
4152
3932 txsd = &txq->sdesc[eq->pidx];
3933 txsd->desc_used = ndesc;
4153 /* ULP master command */
4154 ulpmc = flitp;
4155 ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
4156 V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid));
4157 ulpmc->len = htobe32(mbuf_len16(m));
3934
4158
3935 KASSERT(eq->avail >= ndesc, ("%s: out of descriptors", __func__));
4159 /* ULP subcommand */
4160 ulpsc = (void *)(ulpmc + 1);
4161 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
4162 F_ULP_TX_SC_MORE);
4163 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
3936
4164
3937 eq->pending += ndesc;
3938 eq->avail -= ndesc;
3939 eq->pidx += ndesc;
3940 if (eq->pidx >= eq->cap)
3941 eq->pidx -= eq->cap;
4165 cpl = (void *)(ulpsc + 1);
4166 if (checkwrap &&
4167 (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx])
4168 cpl = (void *)&eq->desc[0];
4169 txq->txpkts0_pkts += txp->npkt;
4170 txq->txpkts0_wrs++;
4171 } else {
4172 cpl = flitp;
4173 txq->txpkts1_pkts += txp->npkt;
4174 txq->txpkts1_wrs++;
4175 }
3942
4176
3943 txq->txpkts_pkts += txpkts->npkt;
3944 txq->txpkts_wrs++;
3945 txpkts->npkt = 0; /* emptied */
3946}
4177 /* Checksum offload */
4178 ctrl1 = 0;
4179 if (needs_l3_csum(m) == 0)
4180 ctrl1 |= F_TXPKT_IPCSUM_DIS;
4181 if (needs_l4_csum(m) == 0)
4182 ctrl1 |= F_TXPKT_L4CSUM_DIS;
4183 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
4184 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
4185 txq->txcsum++; /* some hardware assistance provided */
3947
4186
3948static inline void
3949write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
3950 struct txpkts *txpkts, struct mbuf *m, struct sgl *sgl)
3951{
3952 struct ulp_txpkt *ulpmc;
3953 struct ulptx_idata *ulpsc;
3954 struct cpl_tx_pkt_core *cpl;
3955 struct sge_eq *eq = &txq->eq;
3956 uintptr_t flitp, start, end;
3957 uint64_t ctrl;
3958 caddr_t dst;
4187 /* VLAN tag insertion */
4188 if (needs_vlan_insertion(m)) {
4189 ctrl1 |= F_TXPKT_VLAN_VLD |
4190 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
4191 txq->vlan_insertion++;
4192 }
3959
4193
3960 KASSERT(txpkts->npkt > 0, ("%s: txpkts is empty", __func__));
4194 /* CPL header */
4195 cpl->ctrl0 = txq->cpl_ctrl0;
4196 cpl->pack = 0;
4197 cpl->len = htobe16(m->m_pkthdr.len);
4198 cpl->ctrl1 = htobe64(ctrl1);
3961
4199
3962 start = (uintptr_t)eq->desc;
3963 end = (uintptr_t)eq->spg;
4200 flitp = cpl + 1;
4201 if (checkwrap &&
4202 (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
4203 flitp = (void *)&eq->desc[0];
3964
4204
3965 /* Checksum offload */
3966 ctrl = 0;
3967 if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)))
3968 ctrl |= F_TXPKT_IPCSUM_DIS;
3969 if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
3970 CSUM_TCP_IPV6 | CSUM_TSO)))
3971 ctrl |= F_TXPKT_L4CSUM_DIS;
3972 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
3973 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
3974 txq->txcsum++; /* some hardware assistance provided */
4205 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap);
3975
4206
3976 /* VLAN tag insertion */
3977 if (m->m_flags & M_VLANTAG) {
3978 ctrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
3979 txq->vlan_insertion++;
3980 }
3981
4207 }
4208
3982 /*
3983 * The previous packet's SGL must have ended at a 16 byte boundary (this
3984 * is required by the firmware/hardware). It follows that flitp cannot
3985 * wrap around between the ULPTX master command and ULPTX subcommand (8
3986 * bytes each), and that it can not wrap around in the middle of the
3987 * cpl_tx_pkt_core either.
3988 */
3989 flitp = (uintptr_t)txpkts->flitp;
3990 KASSERT((flitp & 0xf) == 0,
3991 ("%s: last SGL did not end at 16 byte boundary: %p",
3992 __func__, txpkts->flitp));
4209 txsd = &txq->sdesc[eq->pidx];
4210 txsd->m = m0;
4211 txsd->desc_used = ndesc;
3993
4212
3994 /* ULP master command */
3995 ulpmc = (void *)flitp;
3996 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0) |
3997 V_ULP_TXPKT_FID(eq->iqid));
3998 ulpmc->len = htonl(howmany(sizeof(*ulpmc) + sizeof(*ulpsc) +
3999 sizeof(*cpl) + 8 * sgl->nflits, 16));
4000
4001 /* ULP subcommand */
4002 ulpsc = (void *)(ulpmc + 1);
4003 ulpsc->cmd_more = htobe32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
4004 F_ULP_TX_SC_MORE);
4005 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
4006
4007 flitp += sizeof(*ulpmc) + sizeof(*ulpsc);
4008 if (flitp == end)
4009 flitp = start;
4010
4011 /* CPL_TX_PKT */
4012 cpl = (void *)flitp;
4013 cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
4014 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
4015 cpl->pack = 0;
4016 cpl->len = htobe16(m->m_pkthdr.len);
4017 cpl->ctrl1 = htobe64(ctrl);
4018
4019 flitp += sizeof(*cpl);
4020 if (flitp == end)
4021 flitp = start;
4022
4023 /* SGL for this frame */
4024 dst = (caddr_t)flitp;
4025 txpkts->nflits += write_sgl_to_txd(eq, sgl, &dst);
4026 txpkts->flitp = (void *)dst;
4027
4028 KASSERT(((uintptr_t)dst & 0xf) == 0,
4029 ("%s: SGL ends at %p (not a 16 byte boundary)", __func__, dst));
4213 return (ndesc);
4030}
4031
4032/*
4033 * If the SGL ends on an address that is not 16 byte aligned, this function will
4214}
4215
4216/*
4217 * If the SGL ends on an address that is not 16 byte aligned, this function will
4034 * add a 0 filled flit at the end. It returns 1 in that case.
4218 * add a 0 filled flit at the end.
4035 */
4219 */
4036static int
4037write_sgl_to_txd(struct sge_eq *eq, struct sgl *sgl, caddr_t *to)
4220static void
4221write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap)
4038{
4222{
4039 __be64 *flitp, *end;
4223 struct sge_eq *eq = &txq->eq;
4224 struct sglist *gl = txq->gl;
4225 struct sglist_seg *seg;
4226 __be64 *flitp, *wrap;
4040 struct ulptx_sgl *usgl;
4227 struct ulptx_sgl *usgl;
4041 bus_dma_segment_t *seg;
4042 int i, padded;
4228 int i, nflits, nsegs;
4043
4229
4044 KASSERT(sgl->nsegs > 0 && sgl->nflits > 0,
4045 ("%s: bad SGL - nsegs=%d, nflits=%d",
4046 __func__, sgl->nsegs, sgl->nflits));
4047
4048 KASSERT(((uintptr_t)(*to) & 0xf) == 0,
4049 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
4230 KASSERT(((uintptr_t)(*to) & 0xf) == 0,
4231 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
4232 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
4233 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
4050
4234
4235 get_pkt_gl(m, gl);
4236 nsegs = gl->sg_nseg;
4237 MPASS(nsegs > 0);
4238
4239 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
4051 flitp = (__be64 *)(*to);
4240 flitp = (__be64 *)(*to);
4052 end = flitp + sgl->nflits;
4053 seg = &sgl->seg[0];
4241 wrap = (__be64 *)(&eq->desc[eq->sidx]);
4242 seg = &gl->sg_segs[0];
4054 usgl = (void *)flitp;
4055
4056 /*
4057 * We start at a 16 byte boundary somewhere inside the tx descriptor
4058 * ring, so we're at least 16 bytes away from the status page. There is
4059 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
4060 */
4061
4062 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
4243 usgl = (void *)flitp;
4244
4245 /*
4246 * We start at a 16 byte boundary somewhere inside the tx descriptor
4247 * ring, so we're at least 16 bytes away from the status page. There is
4248 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
4249 */
4250
4251 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
4063 V_ULPTX_NSGE(sgl->nsegs));
4064 usgl->len0 = htobe32(seg->ds_len);
4065 usgl->addr0 = htobe64(seg->ds_addr);
4252 V_ULPTX_NSGE(nsegs));
4253 usgl->len0 = htobe32(seg->ss_len);
4254 usgl->addr0 = htobe64(seg->ss_paddr);
4066 seg++;
4067
4255 seg++;
4256
4068 if ((uintptr_t)end <= (uintptr_t)eq->spg) {
4257 if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) {
4069
4070 /* Won't wrap around at all */
4071
4258
4259 /* Won't wrap around at all */
4260
4072 for (i = 0; i < sgl->nsegs - 1; i++, seg++) {
4073 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ds_len);
4074 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ds_addr);
4261 for (i = 0; i < nsegs - 1; i++, seg++) {
4262 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
4263 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
4075 }
4076 if (i & 1)
4077 usgl->sge[i / 2].len[1] = htobe32(0);
4264 }
4265 if (i & 1)
4266 usgl->sge[i / 2].len[1] = htobe32(0);
4267 flitp += nflits;
4078 } else {
4079
4080 /* Will wrap somewhere in the rest of the SGL */
4081
4082 /* 2 flits already written, write the rest flit by flit */
4083 flitp = (void *)(usgl + 1);
4268 } else {
4269
4270 /* Will wrap somewhere in the rest of the SGL */
4271
4272 /* 2 flits already written, write the rest flit by flit */
4273 flitp = (void *)(usgl + 1);
4084 for (i = 0; i < sgl->nflits - 2; i++) {
4085 if ((uintptr_t)flitp == (uintptr_t)eq->spg)
4274 for (i = 0; i < nflits - 2; i++) {
4275 if (flitp == wrap)
4086 flitp = (void *)eq->desc;
4276 flitp = (void *)eq->desc;
4087 *flitp++ = get_flit(seg, sgl->nsegs - 1, i);
4277 *flitp++ = get_flit(seg, nsegs - 1, i);
4088 }
4278 }
4089 end = flitp;
4090 }
4091
4279 }
4280
4092 if ((uintptr_t)end & 0xf) {
4093 *(uint64_t *)end = 0;
4094 end++;
4095 padded = 1;
4096 } else
4097 padded = 0;
4281 if (nflits & 1) {
4282 MPASS(((uintptr_t)flitp) & 0xf);
4283 *flitp++ = 0;
4284 }
4098
4285
4099 if ((uintptr_t)end == (uintptr_t)eq->spg)
4286 MPASS((((uintptr_t)flitp) & 0xf) == 0);
4287 if (__predict_false(flitp == wrap))
4100 *to = (void *)eq->desc;
4101 else
4288 *to = (void *)eq->desc;
4289 else
4102 *to = (void *)end;
4103
4104 return (padded);
4290 *to = (void *)flitp;
4105}
4106
4107static inline void
4108copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
4109{
4291}
4292
4293static inline void
4294copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
4295{
4110 if (__predict_true((uintptr_t)(*to) + len <= (uintptr_t)eq->spg)) {
4296
4297 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
4298 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
4299
4300 if (__predict_true((uintptr_t)(*to) + len <=
4301 (uintptr_t)&eq->desc[eq->sidx])) {
4111 bcopy(from, *to, len);
4112 (*to) += len;
4113 } else {
4302 bcopy(from, *to, len);
4303 (*to) += len;
4304 } else {
4114 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
4305 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
4115
4116 bcopy(from, *to, portion);
4117 from += portion;
4118 portion = len - portion; /* remaining */
4119 bcopy(from, (void *)eq->desc, portion);
4120 (*to) = (caddr_t)eq->desc + portion;
4121 }
4122}
4123
4124static inline void
4306
4307 bcopy(from, *to, portion);
4308 from += portion;
4309 portion = len - portion; /* remaining */
4310 bcopy(from, (void *)eq->desc, portion);
4311 (*to) = (caddr_t)eq->desc + portion;
4312 }
4313}
4314
4315static inline void
4125ring_eq_db(struct adapter *sc, struct sge_eq *eq)
4316ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n)
4126{
4317{
4127 u_int db, pending;
4318 u_int db;
4128
4319
4320 MPASS(n > 0);
4321
4129 db = eq->doorbells;
4322 db = eq->doorbells;
4130 pending = eq->pending;
4131 if (pending > 1)
4323 if (n > 1)
4132 clrbit(&db, DOORBELL_WCWR);
4324 clrbit(&db, DOORBELL_WCWR);
4133 eq->pending = 0;
4134 wmb();
4135
4136 switch (ffs(db) - 1) {
4137 case DOORBELL_UDB:
4325 wmb();
4326
4327 switch (ffs(db) - 1) {
4328 case DOORBELL_UDB:
4138 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending));
4139 return;
4329 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
4330 break;
4140
4141 case DOORBELL_WCWR: {
4142 volatile uint64_t *dst, *src;
4143 int i;
4144
4145 /*
4146 * Queues whose 128B doorbell segment fits in the page do not
4147 * use relative qid (udb_qid is always 0). Only queues with
4148 * doorbell segments can do WCWR.
4149 */
4331
4332 case DOORBELL_WCWR: {
4333 volatile uint64_t *dst, *src;
4334 int i;
4335
4336 /*
4337 * Queues whose 128B doorbell segment fits in the page do not
4338 * use relative qid (udb_qid is always 0). Only queues with
4339 * doorbell segments can do WCWR.
4340 */
4150 KASSERT(eq->udb_qid == 0 && pending == 1,
4341 KASSERT(eq->udb_qid == 0 && n == 1,
4151 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
4342 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
4152 __func__, eq->doorbells, pending, eq->pidx, eq));
4343 __func__, eq->doorbells, n, eq->dbidx, eq));
4153
4154 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
4155 UDBS_DB_OFFSET);
4344
4345 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
4346 UDBS_DB_OFFSET);
4156 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
4347 i = eq->dbidx;
4157 src = (void *)&eq->desc[i];
4158 while (src != (void *)&eq->desc[i + 1])
4159 *dst++ = *src++;
4160 wmb();
4348 src = (void *)&eq->desc[i];
4349 while (src != (void *)&eq->desc[i + 1])
4350 *dst++ = *src++;
4351 wmb();
4161 return;
4352 break;
4162 }
4163
4164 case DOORBELL_UDBWC:
4353 }
4354
4355 case DOORBELL_UDBWC:
4165 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending));
4356 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
4166 wmb();
4357 wmb();
4167 return;
4358 break;
4168
4169 case DOORBELL_KDB:
4170 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
4359
4360 case DOORBELL_KDB:
4361 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
4171 V_QID(eq->cntxt_id) | V_PIDX(pending));
4172 return;
4362 V_QID(eq->cntxt_id) | V_PIDX(n));
4363 break;
4173 }
4364 }
4365
4366 IDXINCR(eq->dbidx, n, eq->sidx);
4174}
4175
4367}
4368
4176static inline int
4177reclaimable(struct sge_eq *eq)
4369static inline u_int
4370reclaimable_tx_desc(struct sge_eq *eq)
4178{
4371{
4179 unsigned int cidx;
4372 uint16_t hw_cidx;
4180
4373
4181 cidx = eq->spg->cidx; /* stable snapshot */
4182 cidx = be16toh(cidx);
4374 hw_cidx = read_hw_cidx(eq);
4375 return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx));
4376}
4183
4377
4184 if (cidx >= eq->cidx)
4185 return (cidx - eq->cidx);
4378static inline u_int
4379total_available_tx_desc(struct sge_eq *eq)
4380{
4381 uint16_t hw_cidx, pidx;
4382
4383 hw_cidx = read_hw_cidx(eq);
4384 pidx = eq->pidx;
4385
4386 if (pidx == hw_cidx)
4387 return (eq->sidx - 1);
4186 else
4388 else
4187 return (cidx + eq->cap - eq->cidx);
4389 return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1);
4188}
4189
4390}
4391
4392static inline uint16_t
4393read_hw_cidx(struct sge_eq *eq)
4394{
4395 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
4396 uint16_t cidx = spg->cidx; /* stable snapshot */
4397
4398 return (be16toh(cidx));
4399}
4400
4190/*
4401/*
4191 * There are "can_reclaim" tx descriptors ready to be reclaimed. Reclaim as
4192 * many as possible but stop when there are around "n" mbufs to free.
4193 *
4194 * The actual number reclaimed is provided as the return value.
4402 * Reclaim 'n' descriptors approximately.
4195 */
4403 */
4196static int
4197reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
4404static u_int
4405reclaim_tx_descs(struct sge_txq *txq, u_int n)
4198{
4199 struct tx_sdesc *txsd;
4406{
4407 struct tx_sdesc *txsd;
4200 struct tx_maps *txmaps;
4201 struct tx_map *txm;
4202 unsigned int reclaimed, maps;
4203 struct sge_eq *eq = &txq->eq;
4408 struct sge_eq *eq = &txq->eq;
4409 u_int can_reclaim, reclaimed;
4204
4205 TXQ_LOCK_ASSERT_OWNED(txq);
4410
4411 TXQ_LOCK_ASSERT_OWNED(txq);
4412 MPASS(n > 0);
4206
4413
4207 if (can_reclaim == 0)
4208 can_reclaim = reclaimable(eq);
4209
4210 maps = reclaimed = 0;
4211 while (can_reclaim && maps < n) {
4414 reclaimed = 0;
4415 can_reclaim = reclaimable_tx_desc(eq);
4416 while (can_reclaim && reclaimed < n) {
4212 int ndesc;
4417 int ndesc;
4418 struct mbuf *m, *nextpkt;
4213
4214 txsd = &txq->sdesc[eq->cidx];
4215 ndesc = txsd->desc_used;
4216
4217 /* Firmware doesn't return "partial" credits. */
4218 KASSERT(can_reclaim >= ndesc,
4219 ("%s: unexpected number of credits: %d, %d",
4220 __func__, can_reclaim, ndesc));
4221
4419
4420 txsd = &txq->sdesc[eq->cidx];
4421 ndesc = txsd->desc_used;
4422
4423 /* Firmware doesn't return "partial" credits. */
4424 KASSERT(can_reclaim >= ndesc,
4425 ("%s: unexpected number of credits: %d, %d",
4426 __func__, can_reclaim, ndesc));
4427
4222 maps += txsd->credits;
4223
4428 for (m = txsd->m; m != NULL; m = nextpkt) {
4429 nextpkt = m->m_nextpkt;
4430 m->m_nextpkt = NULL;
4431 m_freem(m);
4432 }
4224 reclaimed += ndesc;
4225 can_reclaim -= ndesc;
4433 reclaimed += ndesc;
4434 can_reclaim -= ndesc;
4226
4227 eq->cidx += ndesc;
4228 if (__predict_false(eq->cidx >= eq->cap))
4229 eq->cidx -= eq->cap;
4435 IDXINCR(eq->cidx, ndesc, eq->sidx);
4230 }
4231
4436 }
4437
4232 txmaps = &txq->txmaps;
4233 txm = &txmaps->maps[txmaps->map_cidx];
4234 if (maps)
4235 prefetch(txm->m);
4236
4237 eq->avail += reclaimed;
4238 KASSERT(eq->avail < eq->cap, /* avail tops out at (cap - 1) */
4239 ("%s: too many descriptors available", __func__));
4240
4241 txmaps->map_avail += maps;
4242 KASSERT(txmaps->map_avail <= txmaps->map_total,
4243 ("%s: too many maps available", __func__));
4244
4245 while (maps--) {
4246 struct tx_map *next;
4247
4248 next = txm + 1;
4249 if (__predict_false(txmaps->map_cidx + 1 == txmaps->map_total))
4250 next = txmaps->maps;
4251 prefetch(next->m);
4252
4253 bus_dmamap_unload(txq->tx_tag, txm->map);
4254 m_freem(txm->m);
4255 txm->m = NULL;
4256
4257 txm = next;
4258 if (__predict_false(++txmaps->map_cidx == txmaps->map_total))
4259 txmaps->map_cidx = 0;
4260 }
4261
4262 return (reclaimed);
4263}
4264
4265static void
4438 return (reclaimed);
4439}
4440
4441static void
4266write_eqflush_wr(struct sge_eq *eq)
4442tx_reclaim(void *arg, int n)
4267{
4443{
4268 struct fw_eq_flush_wr *wr;
4444 struct sge_txq *txq = arg;
4445 struct sge_eq *eq = &txq->eq;
4269
4446
4270 EQ_LOCK_ASSERT_OWNED(eq);
4271 KASSERT(eq->avail > 0, ("%s: no descriptors left.", __func__));
4272 KASSERT(!(eq->flags & EQ_CRFLUSHED), ("%s: flushed already", __func__));
4273
4274 wr = (void *)&eq->desc[eq->pidx];
4275 bzero(wr, sizeof(*wr));
4276 wr->opcode = FW_EQ_FLUSH_WR;
4277 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(sizeof(*wr) / 16) |
4278 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
4279
4280 eq->flags |= (EQ_CRFLUSHED | EQ_STALLED);
4281 eq->pending++;
4282 eq->avail--;
4283 if (++eq->pidx == eq->cap)
4284 eq->pidx = 0;
4447 do {
4448 if (TXQ_TRYLOCK(txq) == 0)
4449 break;
4450 n = reclaim_tx_descs(txq, 32);
4451 if (eq->cidx == eq->pidx)
4452 eq->equeqidx = eq->pidx;
4453 TXQ_UNLOCK(txq);
4454 } while (n > 0);
4285}
4286
4287static __be64
4455}
4456
4457static __be64
4288get_flit(bus_dma_segment_t *sgl, int nsegs, int idx)
4458get_flit(struct sglist_seg *segs, int nsegs, int idx)
4289{
4290 int i = (idx / 3) * 2;
4291
4292 switch (idx % 3) {
4293 case 0: {
4294 __be64 rc;
4295
4459{
4460 int i = (idx / 3) * 2;
4461
4462 switch (idx % 3) {
4463 case 0: {
4464 __be64 rc;
4465
4296 rc = htobe32(sgl[i].ds_len);
4466 rc = htobe32(segs[i].ss_len);
4297 if (i + 1 < nsegs)
4467 if (i + 1 < nsegs)
4298 rc |= (uint64_t)htobe32(sgl[i + 1].ds_len) << 32;
4468 rc |= (uint64_t)htobe32(segs[i + 1].ss_len) << 32;
4299
4300 return (rc);
4301 }
4302 case 1:
4469
4470 return (rc);
4471 }
4472 case 1:
4303 return htobe64(sgl[i].ds_addr);
4473 return (htobe64(segs[i].ss_paddr));
4304 case 2:
4474 case 2:
4305 return htobe64(sgl[i + 1].ds_addr);
4475 return (htobe64(segs[i + 1].ss_paddr));
4306 }
4307
4308 return (0);
4309}
4310
4311static void
4312find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp)
4313{

--- 178 unchanged lines hidden (view full) ---

4492 fl->flags |= FL_STARVING;
4493 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
4494 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
4495 }
4496 FL_UNLOCK(fl);
4497 mtx_unlock(&sc->sfl_lock);
4498}
4499
4476 }
4477
4478 return (0);
4479}
4480
4481static void
4482find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp)
4483{

--- 178 unchanged lines hidden (view full) ---

4662 fl->flags |= FL_STARVING;
4663 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
4664 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
4665 }
4666 FL_UNLOCK(fl);
4667 mtx_unlock(&sc->sfl_lock);
4668}
4669
4670static void
4671handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
4672{
4673 struct sge_wrq *wrq = (void *)eq;
4674
4675 atomic_readandclear_int(&eq->equiq);
4676 taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task);
4677}
4678
4679static void
4680handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
4681{
4682 struct sge_txq *txq = (void *)eq;
4683
4684 MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH);
4685
4686 atomic_readandclear_int(&eq->equiq);
4687 mp_ring_check_drainage(txq->r, 0);
4688 taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task);
4689}
4690
4500static int
4501handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
4502 struct mbuf *m)
4503{
4504 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
4505 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
4506 struct adapter *sc = iq->adapter;
4507 struct sge *s = &sc->sge;
4508 struct sge_eq *eq;
4691static int
4692handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
4693 struct mbuf *m)
4694{
4695 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
4696 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
4697 struct adapter *sc = iq->adapter;
4698 struct sge *s = &sc->sge;
4699 struct sge_eq *eq;
4700 static void (*h[])(struct adapter *, struct sge_eq *) = {NULL,
4701 &handle_wrq_egr_update, &handle_eth_egr_update,
4702 &handle_wrq_egr_update};
4509
4510 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
4511 rss->opcode));
4512
4513 eq = s->eqmap[qid - s->eq_start];
4703
4704 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
4705 rss->opcode));
4706
4707 eq = s->eqmap[qid - s->eq_start];
4514 EQ_LOCK(eq);
4515 KASSERT(eq->flags & EQ_CRFLUSHED,
4516 ("%s: unsolicited egress update", __func__));
4517 eq->flags &= ~EQ_CRFLUSHED;
4518 eq->egr_update++;
4708 (*h[eq->flags & EQ_TYPEMASK])(sc, eq);
4519
4709
4520 if (__predict_false(eq->flags & EQ_DOOMED))
4521 wakeup_one(eq);
4522 else if (eq->flags & EQ_STALLED && can_resume_tx(eq))
4523 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4524 EQ_UNLOCK(eq);
4525
4526 return (0);
4527}
4528
4529/* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
4530CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
4531 offsetof(struct cpl_fw6_msg, data));
4532
4533static int

--- 52 unchanged lines hidden --- 		4710 return (0);
4711}
4712
4713/* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
4714CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
4715 offsetof(struct cpl_fw6_msg, data));
4716
4717static int

--- 52 unchanged lines hidden ---