Deleted Added
sdiff udiff text old ( 170869 ) new ( 171335 )
full compact
cxgb_sge.c (170869) cxgb_sge.c (171335)
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 14 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 14 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_sge.c 170869 2007-06-17 04:33:38Z kmacy $");
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_sge.c 171335 2007-07-10 06:01:45Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/bus.h>
38#include <sys/conf.h>
39#include <machine/bus.h>

--- 21 unchanged lines hidden (view full) ---

61#ifdef CONFIG_DEFINED
62#include <cxgb_include.h>
63#else
64#include <dev/cxgb/cxgb_include.h>
65#endif
66
67uint32_t collapse_free = 0;
68uint32_t mb_free_vec_free = 0;
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/bus.h>
38#include <sys/conf.h>
39#include <machine/bus.h>

--- 21 unchanged lines hidden (view full) ---

61#ifdef CONFIG_DEFINED
62#include <cxgb_include.h>
63#else
64#include <dev/cxgb/cxgb_include.h>
65#endif
66
67uint32_t collapse_free = 0;
68uint32_t mb_free_vec_free = 0;
69int txq_fills = 0;
69int collapse_mbufs = 0;
70static int recycle_enable = 1;
71
72
73/*
74 * XXX GC
75 */
76#define NET_XMIT_CN 2

--- 104 unchanged lines hidden (view full) ---

181
182
183static int lro_default = 0;
184int cxgb_debug = 0;
185
186static void t3_free_qset(adapter_t *sc, struct sge_qset *q);
187static void sge_timer_cb(void *arg);
188static void sge_timer_reclaim(void *arg, int ncount);
70int collapse_mbufs = 0;
71static int recycle_enable = 1;
72
73
74/*
75 * XXX GC
76 */
77#define NET_XMIT_CN 2

--- 104 unchanged lines hidden (view full) ---

182
183
184static int lro_default = 0;
185int cxgb_debug = 0;
186
187static void t3_free_qset(adapter_t *sc, struct sge_qset *q);
188static void sge_timer_cb(void *arg);
189static void sge_timer_reclaim(void *arg, int ncount);
189static int free_tx_desc(adapter_t *sc, struct sge_txq *q, int n, struct mbuf **m_vec);
190static void sge_txq_reclaim_handler(void *arg, int ncount);
191static int free_tx_desc(struct sge_txq *q, int n, struct mbuf **m_vec);
190
191/**
192 * reclaim_completed_tx - reclaims completed Tx descriptors
193 * @adapter: the adapter
194 * @q: the Tx queue to reclaim completed descriptors from
195 *
196 * Reclaims Tx descriptors that the SGE has indicated it has processed,
197 * and frees the associated buffers if possible. Called with the Tx
198 * queue's lock held.
199 */
200static __inline int
192
193/**
194 * reclaim_completed_tx - reclaims completed Tx descriptors
195 * @adapter: the adapter
196 * @q: the Tx queue to reclaim completed descriptors from
197 *
198 * Reclaims Tx descriptors that the SGE has indicated it has processed,
199 * and frees the associated buffers if possible. Called with the Tx
200 * queue's lock held.
201 */
202static __inline int
201reclaim_completed_tx(adapter_t *adapter, struct sge_txq *q, int nbufs, struct mbuf **mvec)
203reclaim_completed_tx(struct sge_txq *q, int nbufs, struct mbuf **mvec)
202{
203 int reclaimed, reclaim = desc_reclaimable(q);
204 int n = 0;
205
206 mtx_assert(&q->lock, MA_OWNED);
207 if (reclaim > 0) {
204{
205 int reclaimed, reclaim = desc_reclaimable(q);
206 int n = 0;
207
208 mtx_assert(&q->lock, MA_OWNED);
209 if (reclaim > 0) {
208 n = free_tx_desc(adapter, q, min(reclaim, nbufs), mvec);
210 n = free_tx_desc(q, min(reclaim, nbufs), mvec);
209 reclaimed = min(reclaim, nbufs);
210 q->cleaned += reclaimed;
211 q->in_use -= reclaimed;
212 }
213 return (n);
214}
215
216/**

--- 520 unchanged lines hidden (view full) ---

737refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits)
738{
739
740 /* mbufs are allocated on demand when a rspq entry is processed. */
741 t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN,
742 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
743}
744
211 reclaimed = min(reclaim, nbufs);
212 q->cleaned += reclaimed;
213 q->in_use -= reclaimed;
214 }
215 return (n);
216}
217
218/**

--- 520 unchanged lines hidden (view full) ---

739refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits)
740{
741
742 /* mbufs are allocated on demand when a rspq entry is processed. */
743 t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN,
744 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
745}
746
747static __inline void
748sge_txq_reclaim_(struct sge_txq *txq)
749{
750 int reclaimable, i, n;
751 struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
752 struct port_info *p;
745
753
754 p = txq->port;
755reclaim_more:
756 n = 0;
757 reclaimable = desc_reclaimable(txq);
758 if (reclaimable > 0 && mtx_trylock(&txq->lock)) {
759 n = reclaim_completed_tx(txq, TX_CLEAN_MAX_DESC, m_vec);
760 mtx_unlock(&txq->lock);
761 }
762 if (n == 0)
763 return;
764
765 for (i = 0; i < n; i++) {
766 m_freem_vec(m_vec[i]);
767 }
768 if (p && p->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
769 txq->size - txq->in_use >= TX_START_MAX_DESC) {
770 txq_fills++;
771 p->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
772 taskqueue_enqueue(p->tq, &p->start_task);
773 }
774
775 if (n)
776 goto reclaim_more;
777}
778
746static void
779static void
780sge_txq_reclaim_handler(void *arg, int ncount)
781{
782 struct sge_txq *q = arg;
783
784 sge_txq_reclaim_(q);
785}
786
787static void
747sge_timer_reclaim(void *arg, int ncount)
748{
749 struct port_info *p = arg;
750 int i, nqsets = p->nqsets;
751 adapter_t *sc = p->adapter;
752 struct sge_qset *qs;
753 struct sge_txq *txq;
754 struct mtx *lock;
788sge_timer_reclaim(void *arg, int ncount)
789{
790 struct port_info *p = arg;
791 int i, nqsets = p->nqsets;
792 adapter_t *sc = p->adapter;
793 struct sge_qset *qs;
794 struct sge_txq *txq;
795 struct mtx *lock;
755 struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
756 int n, reclaimable;
757
758 for (i = 0; i < nqsets; i++) {
759 qs = &sc->sge.qs[i];
760 txq = &qs->txq[TXQ_ETH];
796
797 for (i = 0; i < nqsets; i++) {
798 qs = &sc->sge.qs[i];
799 txq = &qs->txq[TXQ_ETH];
761 reclaimable = desc_reclaimable(txq);
762 if (reclaimable > 0) {
763 mtx_lock(&txq->lock);
764 n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec);
765 mtx_unlock(&txq->lock);
800 sge_txq_reclaim_(txq);
766
801
767 for (i = 0; i < n; i++)
768 m_freem_vec(m_vec[i]);
769
770 if (p->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
771 txq->size - txq->in_use >= TX_START_MAX_DESC) {
772 p->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
773 taskqueue_enqueue(p->tq, &p->start_task);
774 }
775 }
776
777 txq = &qs->txq[TXQ_OFLD];
802 txq = &qs->txq[TXQ_OFLD];
778 reclaimable = desc_reclaimable(txq);
779 if (reclaimable > 0) {
780 mtx_lock(&txq->lock);
781 n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec);
782 mtx_unlock(&txq->lock);
783
784 for (i = 0; i < n; i++)
785 m_freem_vec(m_vec[i]);
786 }
787
803 sge_txq_reclaim_(txq);
804
788 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
789 &sc->sge.qs[0].rspq.lock;
790
791 if (mtx_trylock(lock)) {
792 /* XXX currently assume that we are *NOT* polling */
793 uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS);
794
795 if (qs->fl[0].credits < qs->fl[0].size - 16)

--- 760 unchanged lines hidden (view full) ---

1556 return;
1557
1558 for (nqsets = i = 0; i < (sc)->params.nports; i++)
1559 nqsets += sc->port[i].nqsets;
1560
1561 for (i = 0; i < nqsets; ++i) {
1562 struct sge_qset *qs = &sc->sge.qs[i];
1563
805 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock :
806 &sc->sge.qs[0].rspq.lock;
807
808 if (mtx_trylock(lock)) {
809 /* XXX currently assume that we are *NOT* polling */
810 uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS);
811
812 if (qs->fl[0].credits < qs->fl[0].size - 16)

--- 760 unchanged lines hidden (view full) ---

1573 return;
1574
1575 for (nqsets = i = 0; i < (sc)->params.nports; i++)
1576 nqsets += sc->port[i].nqsets;
1577
1578 for (i = 0; i < nqsets; ++i) {
1579 struct sge_qset *qs = &sc->sge.qs[i];
1580
1564 taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_tsk);
1565 taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_tsk);
1581 taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
1582 taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
1566 }
1567}
1568
1569
1570/**
1571 * free_tx_desc - reclaims Tx descriptors and their buffers
1572 * @adapter: the adapter
1573 * @q: the Tx queue to reclaim descriptors from
1574 * @n: the number of descriptors to reclaim
1575 *
1576 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
1577 * Tx buffers. Called with the Tx queue lock held.
1578 */
1579int
1583 }
1584}
1585
1586
1587/**
1588 * free_tx_desc - reclaims Tx descriptors and their buffers
1589 * @adapter: the adapter
1590 * @q: the Tx queue to reclaim descriptors from
1591 * @n: the number of descriptors to reclaim
1592 *
1593 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
1594 * Tx buffers. Called with the Tx queue lock held.
1595 */
1596int
1580free_tx_desc(adapter_t *sc, struct sge_txq *q, int n, struct mbuf **m_vec)
1597free_tx_desc(struct sge_txq *q, int n, struct mbuf **m_vec)
1581{
1582 struct tx_sw_desc *d;
1583 unsigned int cidx = q->cidx;
1584 int nbufs = 0;
1585
1586#ifdef T3_TRACE
1587 T3_TRACE2(sc->tb[q->cntxt_id & 7],
1588 "reclaiming %u Tx descriptors at cidx %u", n, cidx);

--- 144 unchanged lines hidden (view full) ---

1733 struct tx_sw_desc *stx = &q->sdesc[q->pidx];
1734
1735 mtx_lock(&q->lock);
1736 if ((ret = busdma_map_mbufs(&m, q, stx, segs, &nsegs)) != 0) {
1737 mtx_unlock(&q->lock);
1738 return (ret);
1739 }
1740 ndesc = calc_tx_descs_ofld(m, nsegs);
1598{
1599 struct tx_sw_desc *d;
1600 unsigned int cidx = q->cidx;
1601 int nbufs = 0;
1602
1603#ifdef T3_TRACE
1604 T3_TRACE2(sc->tb[q->cntxt_id & 7],
1605 "reclaiming %u Tx descriptors at cidx %u", n, cidx);

--- 144 unchanged lines hidden (view full) ---

1750 struct tx_sw_desc *stx = &q->sdesc[q->pidx];
1751
1752 mtx_lock(&q->lock);
1753 if ((ret = busdma_map_mbufs(&m, q, stx, segs, &nsegs)) != 0) {
1754 mtx_unlock(&q->lock);
1755 return (ret);
1756 }
1757 ndesc = calc_tx_descs_ofld(m, nsegs);
1741again: cleaned = reclaim_completed_tx(adap, q, TX_CLEAN_MAX_DESC, m_vec);
1758again: cleaned = reclaim_completed_tx(q, TX_CLEAN_MAX_DESC, m_vec);
1742
1743 ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD);
1744 if (__predict_false(ret)) {
1745 if (ret == 1) {
1746 m_set_priority(m, ndesc); /* save for restart */
1747 mtx_unlock(&q->lock);
1748 return NET_XMIT_CN;
1749 }

--- 40 unchanged lines hidden (view full) ---

1790 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1791 adapter_t *adap = qs->port->adapter;
1792 struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
1793 bus_dma_segment_t segs[TX_MAX_SEGS];
1794 int nsegs, i, cleaned;
1795 struct tx_sw_desc *stx = &q->sdesc[q->pidx];
1796
1797 mtx_lock(&q->lock);
1759
1760 ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD);
1761 if (__predict_false(ret)) {
1762 if (ret == 1) {
1763 m_set_priority(m, ndesc); /* save for restart */
1764 mtx_unlock(&q->lock);
1765 return NET_XMIT_CN;
1766 }

--- 40 unchanged lines hidden (view full) ---

1807 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1808 adapter_t *adap = qs->port->adapter;
1809 struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
1810 bus_dma_segment_t segs[TX_MAX_SEGS];
1811 int nsegs, i, cleaned;
1812 struct tx_sw_desc *stx = &q->sdesc[q->pidx];
1813
1814 mtx_lock(&q->lock);
1798again: cleaned = reclaim_completed_tx(adap, q, TX_CLEAN_MAX_DESC, m_vec);
1815again: cleaned = reclaim_completed_tx(q, TX_CLEAN_MAX_DESC, m_vec);
1799
1800 while ((m = mbufq_peek(&q->sendq)) != NULL) {
1801 unsigned int gen, pidx;
1802 unsigned int ndesc = m_get_priority(m);
1803
1804 if (__predict_false(q->size - q->in_use < ndesc)) {
1805 setbit(&qs->txq_stopped, TXQ_OFLD);
1806 smp_mb();

--- 122 unchanged lines hidden (view full) ---

1929restart_tx(struct sge_qset *qs)
1930{
1931 struct adapter *sc = qs->port->adapter;
1932
1933 if (isset(&qs->txq_stopped, TXQ_OFLD) &&
1934 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1935 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1936 qs->txq[TXQ_OFLD].restarts++;
1816
1817 while ((m = mbufq_peek(&q->sendq)) != NULL) {
1818 unsigned int gen, pidx;
1819 unsigned int ndesc = m_get_priority(m);
1820
1821 if (__predict_false(q->size - q->in_use < ndesc)) {
1822 setbit(&qs->txq_stopped, TXQ_OFLD);
1823 smp_mb();

--- 122 unchanged lines hidden (view full) ---

1946restart_tx(struct sge_qset *qs)
1947{
1948 struct adapter *sc = qs->port->adapter;
1949
1950 if (isset(&qs->txq_stopped, TXQ_OFLD) &&
1951 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1952 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1953 qs->txq[TXQ_OFLD].restarts++;
1937 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_tsk);
1954 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
1938 }
1939 if (isset(&qs->txq_stopped, TXQ_CTRL) &&
1940 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
1941 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
1942 qs->txq[TXQ_CTRL].restarts++;
1955 }
1956 if (isset(&qs->txq_stopped, TXQ_CTRL) &&
1957 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
1958 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
1959 qs->txq[TXQ_CTRL].restarts++;
1943 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_tsk);
1960 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
1944 }
1945}
1946
1947/**
1948 * t3_sge_alloc_qset - initialize an SGE queue set
1949 * @sc: the controller softc
1950 * @id: the queue set id
1951 * @nports: how many Ethernet ports will be using this queue set

--- 62 unchanged lines hidden (view full) ---

2014 mbufq_init(&q->txq[i].sendq);
2015 q->txq[i].gen = 1;
2016 q->txq[i].size = p->txq_size[i];
2017 snprintf(q->txq[i].lockbuf, TXQ_NAME_LEN, "t3 txq lock %d:%d:%d",
2018 device_get_unit(sc->dev), irq_vec_idx, i);
2019 MTX_INIT(&q->txq[i].lock, q->txq[i].lockbuf, NULL, MTX_DEF);
2020 }
2021
1961 }
1962}
1963
1964/**
1965 * t3_sge_alloc_qset - initialize an SGE queue set
1966 * @sc: the controller softc
1967 * @id: the queue set id
1968 * @nports: how many Ethernet ports will be using this queue set

--- 62 unchanged lines hidden (view full) ---

2031 mbufq_init(&q->txq[i].sendq);
2032 q->txq[i].gen = 1;
2033 q->txq[i].size = p->txq_size[i];
2034 snprintf(q->txq[i].lockbuf, TXQ_NAME_LEN, "t3 txq lock %d:%d:%d",
2035 device_get_unit(sc->dev), irq_vec_idx, i);
2036 MTX_INIT(&q->txq[i].lock, q->txq[i].lockbuf, NULL, MTX_DEF);
2037 }
2038
2022 TASK_INIT(&q->txq[TXQ_OFLD].qresume_tsk, 0, restart_offloadq, q);
2023 TASK_INIT(&q->txq[TXQ_CTRL].qresume_tsk, 0, restart_ctrlq, q);
2039 q->txq[TXQ_ETH].port = pi;
2024
2040
2041 TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q);
2042 TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q);
2043 TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, &q->txq[TXQ_ETH]);
2044 TASK_INIT(&q->txq[TXQ_CTRL].qreclaim_task, 0, sge_txq_reclaim_handler, &q->txq[TXQ_OFLD]);
2045
2046
2047
2048
2025 q->fl[0].gen = q->fl[1].gen = 1;
2026 q->fl[0].size = p->fl_size;
2027 q->fl[1].size = p->jumbo_size;
2028
2029 q->rspq.gen = 1;
2030 q->rspq.size = p->rspq_size;
2031
2032 q->txq[TXQ_ETH].stop_thres = nports *

--- 598 unchanged lines hidden (view full) ---

2631 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2632 "mb_free_vec_free",
2633 CTLFLAG_RD, &mb_free_vec_free,
2634 0, "frees during mb_free_vec");
2635 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2636 "collapse_mbufs",
2637 CTLFLAG_RW, &collapse_mbufs,
2638 0, "collapse mbuf chains into iovecs");
2049 q->fl[0].gen = q->fl[1].gen = 1;
2050 q->fl[0].size = p->fl_size;
2051 q->fl[1].size = p->jumbo_size;
2052
2053 q->rspq.gen = 1;
2054 q->rspq.size = p->rspq_size;
2055
2056 q->txq[TXQ_ETH].stop_thres = nports *

--- 598 unchanged lines hidden (view full) ---

2655 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2656 "mb_free_vec_free",
2657 CTLFLAG_RD, &mb_free_vec_free,
2658 0, "frees during mb_free_vec");
2659 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2660 "collapse_mbufs",
2661 CTLFLAG_RW, &collapse_mbufs,
2662 0, "collapse mbuf chains into iovecs");
2663 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
2664 "txq_overrun",
2665 CTLFLAG_RD, &txq_fills,
2666 0, "#times txq overrun");
2639}
2640
2641/**
2642 * t3_get_desc - dump an SGE descriptor for debugging purposes
2643 * @qs: the queue set
2644 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
2645 * @idx: the descriptor index in the queue
2646 * @data: where to dump the descriptor contents

--- 31 unchanged lines hidden --- 		2667}
2668
2669/**
2670 * t3_get_desc - dump an SGE descriptor for debugging purposes
2671 * @qs: the queue set
2672 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
2673 * @idx: the descriptor index in the queue
2674 * @data: where to dump the descriptor contents

--- 31 unchanged lines hidden ---