Deleted Added
sdiff udiff text old ( 282486 ) new ( 284052 )
full compact
t4_main.c (282486) t4_main.c (284052)
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 282486 2015-05-05 15:13:59Z np $");
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 284052 2015-06-06 09:28:40Z np $");
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/param.h>
35#include <sys/conf.h>
36#include <sys/priv.h>
37#include <sys/kernel.h>
38#include <sys/bus.h>
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/param.h>
35#include <sys/conf.h>
36#include <sys/priv.h>
37#include <sys/kernel.h>
38#include <sys/bus.h>
39#include <sys/systm.h>
40#include <sys/counter.h>
39#include <sys/module.h>
40#include <sys/malloc.h>
41#include <sys/queue.h>
42#include <sys/taskqueue.h>
43#include <sys/pciio.h>
44#include <dev/pci/pcireg.h>
45#include <dev/pci/pcivar.h>
46#include <dev/pci/pci_private.h>

--- 14 unchanged lines hidden (view full) ---

61#endif
62
63#include "common/common.h"
64#include "common/t4_msg.h"
65#include "common/t4_regs.h"
66#include "common/t4_regs_values.h"
67#include "t4_ioctl.h"
68#include "t4_l2t.h"
41#include <sys/module.h>
42#include <sys/malloc.h>
43#include <sys/queue.h>
44#include <sys/taskqueue.h>
45#include <sys/pciio.h>
46#include <dev/pci/pcireg.h>
47#include <dev/pci/pcivar.h>
48#include <dev/pci/pci_private.h>

--- 14 unchanged lines hidden (view full) ---

63#endif
64
65#include "common/common.h"
66#include "common/t4_msg.h"
67#include "common/t4_regs.h"
68#include "common/t4_regs_values.h"
69#include "t4_ioctl.h"
70#include "t4_l2t.h"
71#include "t4_mp_ring.h"
69
70/* T4 bus driver interface */
71static int t4_probe(device_t);
72static int t4_attach(device_t);
73static int t4_detach(device_t);
74static device_method_t t4_methods[] = {
75 DEVMETHOD(device_probe, t4_probe),
76 DEVMETHOD(device_attach, t4_attach),

--- 295 unchanged lines hidden (view full) ---

372static int get_params__pre_init(struct adapter *);
373static int get_params__post_init(struct adapter *);
374static int set_params__post_init(struct adapter *);
375static void t4_set_desc(struct adapter *);
376static void build_medialist(struct port_info *, struct ifmedia *);
377static int cxgbe_init_synchronized(struct port_info *);
378static int cxgbe_uninit_synchronized(struct port_info *);
379static int setup_intr_handlers(struct adapter *);
72
73/* T4 bus driver interface */
74static int t4_probe(device_t);
75static int t4_attach(device_t);
76static int t4_detach(device_t);
77static device_method_t t4_methods[] = {
78 DEVMETHOD(device_probe, t4_probe),
79 DEVMETHOD(device_attach, t4_attach),

--- 295 unchanged lines hidden (view full) ---

375static int get_params__pre_init(struct adapter *);
376static int get_params__post_init(struct adapter *);
377static int set_params__post_init(struct adapter *);
378static void t4_set_desc(struct adapter *);
379static void build_medialist(struct port_info *, struct ifmedia *);
380static int cxgbe_init_synchronized(struct port_info *);
381static int cxgbe_uninit_synchronized(struct port_info *);
382static int setup_intr_handlers(struct adapter *);
380static void quiesce_eq(struct adapter *, struct sge_eq *);
383static void quiesce_txq(struct adapter *, struct sge_txq *);
384static void quiesce_wrq(struct adapter *, struct sge_wrq *);
381static void quiesce_iq(struct adapter *, struct sge_iq *);
382static void quiesce_fl(struct adapter *, struct sge_fl *);
383static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
384 driver_intr_t *, void *, char *);
385static int t4_free_irq(struct adapter *, struct irq *);
386static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
387 unsigned int);
388static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);

--- 39 unchanged lines hidden (view full) ---

428static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
429static int sysctl_tids(SYSCTL_HANDLER_ARGS);
430static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
431static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
432static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
433static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
434static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
435#endif
385static void quiesce_iq(struct adapter *, struct sge_iq *);
386static void quiesce_fl(struct adapter *, struct sge_fl *);
387static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
388 driver_intr_t *, void *, char *);
389static int t4_free_irq(struct adapter *, struct irq *);
390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
391 unsigned int);
392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);

--- 39 unchanged lines hidden (view full) ---

432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
433static int sysctl_tids(SYSCTL_HANDLER_ARGS);
434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
439#endif
436static inline void txq_start(struct ifnet *, struct sge_txq *);
437static uint32_t fconf_to_mode(uint32_t);
438static uint32_t mode_to_fconf(uint32_t);
439static uint32_t fspec_to_fconf(struct t4_filter_specification *);
440static int get_filter_mode(struct adapter *, uint32_t *);
441static int set_filter_mode(struct adapter *, uint32_t);
442static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
443static int get_filter(struct adapter *, struct t4_filter *);
444static int set_filter(struct adapter *, struct t4_filter *);

--- 215 unchanged lines hidden (view full) ---

660 sc->cdev->si_drv1 = sc;
661
662 /* Go no further if recovery mode has been requested. */
663 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
664 device_printf(dev, "recovery mode.\n");
665 goto done;
666 }
667
440static uint32_t fconf_to_mode(uint32_t);
441static uint32_t mode_to_fconf(uint32_t);
442static uint32_t fspec_to_fconf(struct t4_filter_specification *);
443static int get_filter_mode(struct adapter *, uint32_t *);
444static int set_filter_mode(struct adapter *, uint32_t);
445static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
446static int get_filter(struct adapter *, struct t4_filter *);
447static int set_filter(struct adapter *, struct t4_filter *);

--- 215 unchanged lines hidden (view full) ---

663 sc->cdev->si_drv1 = sc;
664
665 /* Go no further if recovery mode has been requested. */
666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
667 device_printf(dev, "recovery mode.\n");
668 goto done;
669 }
670
671#if defined(__i386__)
672 if ((cpu_feature & CPUID_CX8) == 0) {
673 device_printf(dev, "64 bit atomics not available.\n");
674 rc = ENOTSUP;
675 goto done;
676 }
677#endif
678
668 /* Prepare the firmware for operation */
669 rc = prep_firmware(sc);
670 if (rc != 0)
671 goto done; /* error message displayed already */
672
673 rc = get_params__post_init(sc);
674 if (rc != 0)
675 goto done; /* error message displayed already */

--- 705 unchanged lines hidden (view full) ---

1381 return (rc);
1382}
1383
1384static int
1385cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1386{
1387 struct port_info *pi = ifp->if_softc;
1388 struct adapter *sc = pi->adapter;
679 /* Prepare the firmware for operation */
680 rc = prep_firmware(sc);
681 if (rc != 0)
682 goto done; /* error message displayed already */
683
684 rc = get_params__post_init(sc);
685 if (rc != 0)
686 goto done; /* error message displayed already */

--- 705 unchanged lines hidden (view full) ---

1392 return (rc);
1393}
1394
1395static int
1396cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1397{
1398 struct port_info *pi = ifp->if_softc;
1399 struct adapter *sc = pi->adapter;
1389 struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1390 struct buf_ring *br;
1400 struct sge_txq *txq;
1401 void *items[1];
1391 int rc;
1392
1393 M_ASSERTPKTHDR(m);
1402 int rc;
1403
1404 M_ASSERTPKTHDR(m);
1405 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
1394
1395 if (__predict_false(pi->link_cfg.link_ok == 0)) {
1396 m_freem(m);
1397 return (ENETDOWN);
1398 }
1399
1406
1407 if (__predict_false(pi->link_cfg.link_ok == 0)) {
1408 m_freem(m);
1409 return (ENETDOWN);
1410 }
1411
1400 /* check if flowid is set */
1401 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1402 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1403 + pi->rsrv_noflowq);
1404 br = txq->br;
1405
1406 if (TXQ_TRYLOCK(txq) == 0) {
1407 struct sge_eq *eq = &txq->eq;
1408
1409 /*
1410 * It is possible that t4_eth_tx finishes up and releases the
1411 * lock between the TRYLOCK above and the drbr_enqueue here. We
1412 * need to make sure that this mbuf doesn't just sit there in
1413 * the drbr.
1414 */
1415
1416 rc = drbr_enqueue(ifp, br, m);
1417 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1418 !(eq->flags & EQ_DOOMED))
1419 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1412 rc = parse_pkt(&m);
1413 if (__predict_false(rc != 0)) {
1414 MPASS(m == NULL); /* was freed already */
1415 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1420 return (rc);
1421 }
1422
1416 return (rc);
1417 }
1418
1423 /*
1424 * txq->m is the mbuf that is held up due to a temporary shortage of
1425 * resources and it should be put on the wire first. Then what's in
1426 * drbr and finally the mbuf that was just passed in to us.
1427 *
1428 * Return code should indicate the fate of the mbuf that was passed in
1429 * this time.
1430 */
1419 /* Select a txq. */
1420 txq = &sc->sge.txq[pi->first_txq];
1421 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1422 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1423 pi->rsrv_noflowq);
1431
1424
1432 TXQ_LOCK_ASSERT_OWNED(txq);
1433 if (drbr_needs_enqueue(ifp, br) || txq->m) {
1425 items[0] = m;
1426 rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1427 if (__predict_false(rc != 0))
1428 m_freem(m);
1434
1429
1435 /* Queued for transmission. */
1436
1437 rc = drbr_enqueue(ifp, br, m);
1438 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1439 (void) t4_eth_tx(ifp, txq, m);
1440 TXQ_UNLOCK(txq);
1441 return (rc);
1442 }
1443
1444 /* Direct transmission. */
1445 rc = t4_eth_tx(ifp, txq, m);
1446 if (rc != 0 && txq->m)
1447 rc = 0; /* held, will be transmitted soon (hopefully) */
1448
1449 TXQ_UNLOCK(txq);
1450 return (rc);
1451}
1452
1453static void
1454cxgbe_qflush(struct ifnet *ifp)
1455{
1456 struct port_info *pi = ifp->if_softc;
1457 struct sge_txq *txq;
1458 int i;
1430 return (rc);
1431}
1432
1433static void
1434cxgbe_qflush(struct ifnet *ifp)
1435{
1436 struct port_info *pi = ifp->if_softc;
1437 struct sge_txq *txq;
1438 int i;
1459 struct mbuf *m;
1460
1461 /* queues do not exist if !PORT_INIT_DONE. */
1462 if (pi->flags & PORT_INIT_DONE) {
1463 for_each_txq(pi, i, txq) {
1464 TXQ_LOCK(txq);
1439
1440 /* queues do not exist if !PORT_INIT_DONE. */
1441 if (pi->flags & PORT_INIT_DONE) {
1442 for_each_txq(pi, i, txq) {
1443 TXQ_LOCK(txq);
1465 m_freem(txq->m);
1466 txq->m = NULL;
1467 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1468 m_freem(m);
1444 txq->eq.flags &= ~EQ_ENABLED;
1469 TXQ_UNLOCK(txq);
1445 TXQ_UNLOCK(txq);
1446 while (!mp_ring_is_idle(txq->r)) {
1447 mp_ring_check_drainage(txq->r, 0);
1448 pause("qflush", 1);
1449 }
1470 }
1471 }
1472 if_qflush(ifp);
1473}
1474
1475static int
1476cxgbe_media_change(struct ifnet *ifp)
1477{

--- 1652 unchanged lines hidden (view full) ---

3130 ADAPTER_UNLOCK(sc);
3131}
3132
3133static int
3134cxgbe_init_synchronized(struct port_info *pi)
3135{
3136 struct adapter *sc = pi->adapter;
3137 struct ifnet *ifp = pi->ifp;
1450 }
1451 }
1452 if_qflush(ifp);
1453}
1454
1455static int
1456cxgbe_media_change(struct ifnet *ifp)
1457{

--- 1652 unchanged lines hidden (view full) ---

3110 ADAPTER_UNLOCK(sc);
3111}
3112
3113static int
3114cxgbe_init_synchronized(struct port_info *pi)
3115{
3116 struct adapter *sc = pi->adapter;
3117 struct ifnet *ifp = pi->ifp;
3138 int rc = 0;
3118 int rc = 0, i;
3119 struct sge_txq *txq;
3139
3140 ASSERT_SYNCHRONIZED_OP(sc);
3141
3142 if (isset(&sc->open_device_map, pi->port_id)) {
3143 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3144 ("mismatch between open_device_map and if_drv_flags"));
3145 return (0); /* already running */
3146 }

--- 12 unchanged lines hidden (view full) ---

3159
3160 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3161 if (rc != 0) {
3162 if_printf(ifp, "enable_vi failed: %d\n", rc);
3163 goto done;
3164 }
3165
3166 /*
3120
3121 ASSERT_SYNCHRONIZED_OP(sc);
3122
3123 if (isset(&sc->open_device_map, pi->port_id)) {
3124 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3125 ("mismatch between open_device_map and if_drv_flags"));
3126 return (0); /* already running */
3127 }

--- 12 unchanged lines hidden (view full) ---

3140
3141 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3142 if (rc != 0) {
3143 if_printf(ifp, "enable_vi failed: %d\n", rc);
3144 goto done;
3145 }
3146
3147 /*
3148 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized
3149 * if this changes.
3150 */
3151
3152 for_each_txq(pi, i, txq) {
3153 TXQ_LOCK(txq);
3154 txq->eq.flags |= EQ_ENABLED;
3155 TXQ_UNLOCK(txq);
3156 }
3157
3158 /*
3167 * The first iq of the first port to come up is used for tracing.
3168 */
3169 if (sc->traceq < 0) {
3170 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3171 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
3172 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3173 V_QUEUENUMBER(sc->traceq));
3174 pi->flags |= HAS_TRACEQ;

--- 16 unchanged lines hidden (view full) ---

3191/*
3192 * Idempotent.
3193 */
3194static int
3195cxgbe_uninit_synchronized(struct port_info *pi)
3196{
3197 struct adapter *sc = pi->adapter;
3198 struct ifnet *ifp = pi->ifp;
3159 * The first iq of the first port to come up is used for tracing.
3160 */
3161 if (sc->traceq < 0) {
3162 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3163 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
3164 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3165 V_QUEUENUMBER(sc->traceq));
3166 pi->flags |= HAS_TRACEQ;

--- 16 unchanged lines hidden (view full) ---

3183/*
3184 * Idempotent.
3185 */
3186static int
3187cxgbe_uninit_synchronized(struct port_info *pi)
3188{
3189 struct adapter *sc = pi->adapter;
3190 struct ifnet *ifp = pi->ifp;
3199 int rc;
3191 int rc, i;
3192 struct sge_txq *txq;
3200
3201 ASSERT_SYNCHRONIZED_OP(sc);
3202
3203 /*
3204 * Disable the VI so that all its data in either direction is discarded
3205 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz
3206 * tick) intact as the TP can deliver negative advice or data that it's
3207 * holding in its RAM (for an offloaded connection) even after the VI is
3208 * disabled.
3209 */
3210 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3211 if (rc) {
3212 if_printf(ifp, "disable_vi failed: %d\n", rc);
3213 return (rc);
3214 }
3215
3193
3194 ASSERT_SYNCHRONIZED_OP(sc);
3195
3196 /*
3197 * Disable the VI so that all its data in either direction is discarded
3198 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz
3199 * tick) intact as the TP can deliver negative advice or data that it's
3200 * holding in its RAM (for an offloaded connection) even after the VI is
3201 * disabled.
3202 */
3203 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3204 if (rc) {
3205 if_printf(ifp, "disable_vi failed: %d\n", rc);
3206 return (rc);
3207 }
3208
3209 for_each_txq(pi, i, txq) {
3210 TXQ_LOCK(txq);
3211 txq->eq.flags &= ~EQ_ENABLED;
3212 TXQ_UNLOCK(txq);
3213 }
3214
3216 clrbit(&sc->open_device_map, pi->port_id);
3217 PORT_LOCK(pi);
3218 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3219 PORT_UNLOCK(pi);
3220
3221 pi->link_cfg.link_ok = 0;
3222 pi->link_cfg.speed = 0;
3223 pi->linkdnrc = -1;

--- 214 unchanged lines hidden (view full) ---

3438 struct sge_txq *txq;
3439#ifdef TCP_OFFLOAD
3440 struct sge_ofld_rxq *ofld_rxq;
3441 struct sge_wrq *ofld_txq;
3442#endif
3443
3444 if (pi->flags & PORT_INIT_DONE) {
3445
3215 clrbit(&sc->open_device_map, pi->port_id);
3216 PORT_LOCK(pi);
3217 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3218 PORT_UNLOCK(pi);
3219
3220 pi->link_cfg.link_ok = 0;
3221 pi->link_cfg.speed = 0;
3222 pi->linkdnrc = -1;

--- 214 unchanged lines hidden (view full) ---

3437 struct sge_txq *txq;
3438#ifdef TCP_OFFLOAD
3439 struct sge_ofld_rxq *ofld_rxq;
3440 struct sge_wrq *ofld_txq;
3441#endif
3442
3443 if (pi->flags & PORT_INIT_DONE) {
3444
3446 /* Need to quiesce queues. XXX: ctrl queues? */
3445 /* Need to quiesce queues. */
3447
3446
3447 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3448
3448 for_each_txq(pi, i, txq) {
3449 for_each_txq(pi, i, txq) {
3449 quiesce_eq(sc, &txq->eq);
3450 quiesce_txq(sc, txq);
3450 }
3451
3452#ifdef TCP_OFFLOAD
3453 for_each_ofld_txq(pi, i, ofld_txq) {
3451 }
3452
3453#ifdef TCP_OFFLOAD
3454 for_each_ofld_txq(pi, i, ofld_txq) {
3454 quiesce_eq(sc, &ofld_txq->eq);
3455 quiesce_wrq(sc, ofld_txq);
3455 }
3456#endif
3457
3458 for_each_rxq(pi, i, rxq) {
3459 quiesce_iq(sc, &rxq->iq);
3460 quiesce_fl(sc, &rxq->fl);
3461 }
3462

--- 8 unchanged lines hidden (view full) ---

3471
3472 t4_teardown_port_queues(pi);
3473 pi->flags &= ~PORT_INIT_DONE;
3474
3475 return (0);
3476}
3477
3478static void
3456 }
3457#endif
3458
3459 for_each_rxq(pi, i, rxq) {
3460 quiesce_iq(sc, &rxq->iq);
3461 quiesce_fl(sc, &rxq->fl);
3462 }
3463

--- 8 unchanged lines hidden (view full) ---

3472
3473 t4_teardown_port_queues(pi);
3474 pi->flags &= ~PORT_INIT_DONE;
3475
3476 return (0);
3477}
3478
3479static void
3479quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3480quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3480{
3481{
3481 EQ_LOCK(eq);
3482 eq->flags |= EQ_DOOMED;
3482 struct sge_eq *eq = &txq->eq;
3483 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3483
3484
3484 /*
3485 * Wait for the response to a credit flush if one's
3486 * pending.
3487 */
3488 while (eq->flags & EQ_CRFLUSHED)
3489 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3490 EQ_UNLOCK(eq);
3485 (void) sc; /* unused */
3491
3486
3492 callout_drain(&eq->tx_callout); /* XXX: iffy */
3493 pause("callout", 10); /* Still iffy */
3487#ifdef INVARIANTS
3488 TXQ_LOCK(txq);
3489 MPASS((eq->flags & EQ_ENABLED) == 0);
3490 TXQ_UNLOCK(txq);
3491#endif
3494
3492
3495 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3493 /* Wait for the mp_ring to empty. */
3494 while (!mp_ring_is_idle(txq->r)) {
3495 mp_ring_check_drainage(txq->r, 0);
3496 pause("rquiesce", 1);
3497 }
3498
3499 /* Then wait for the hardware to finish. */
3500 while (spg->cidx != htobe16(eq->pidx))
3501 pause("equiesce", 1);
3502
3503 /* Finally, wait for the driver to reclaim all descriptors. */
3504 while (eq->cidx != eq->pidx)
3505 pause("dquiesce", 1);
3496}
3497
3498static void
3506}
3507
3508static void
3509quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3510{
3511
3512 /* XXXTX */
3513}
3514
3515static void
3499quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3500{
3501 (void) sc; /* unused */
3502
3503 /* Synchronize with the interrupt handler */
3504 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3505 pause("iqfree", 1);
3506}

--- 775 unchanged lines hidden (view full) ---

4282 1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4283 mtx_unlock(&sc->regwin_lock);
4284 ifp->if_iqdrops += v;
4285 }
4286 }
4287
4288 drops = s->tx_drop;
4289 for_each_txq(pi, i, txq)
3516quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3517{
3518 (void) sc; /* unused */
3519
3520 /* Synchronize with the interrupt handler */
3521 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3522 pause("iqfree", 1);
3523}

--- 775 unchanged lines hidden (view full) ---

4299 1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4300 mtx_unlock(&sc->regwin_lock);
4301 ifp->if_iqdrops += v;
4302 }
4303 }
4304
4305 drops = s->tx_drop;
4306 for_each_txq(pi, i, txq)
4290 drops += txq->br->br_drops;
4307 drops += counter_u64_fetch(txq->r->drops);
4291 ifp->if_snd.ifq_drops = drops;
4292
4293 ifp->if_oerrors = s->tx_error_frames;
4294 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4295 s->rx_fcs_err + s->rx_len_err;
4296
4297 getmicrotime(&pi->last_refreshed);
4298}

--- 514 unchanged lines hidden (view full) ---

4813 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4814
4815 /*
4816 * dev.cxgbe.X.stats.
4817 */
4818 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4819 NULL, "port statistics");
4820 children = SYSCTL_CHILDREN(oid);
4308 ifp->if_snd.ifq_drops = drops;
4309
4310 ifp->if_oerrors = s->tx_error_frames;
4311 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4312 s->rx_fcs_err + s->rx_len_err;
4313
4314 getmicrotime(&pi->last_refreshed);
4315}

--- 514 unchanged lines hidden (view full) ---

4830 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4831
4832 /*
4833 * dev.cxgbe.X.stats.
4834 */
4835 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4836 NULL, "port statistics");
4837 children = SYSCTL_CHILDREN(oid);
4838 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4839 &pi->tx_parse_error, 0,
4840 "# of tx packets with invalid length or # of segments");
4821
4822#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4823 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4824 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4825 sysctl_handle_t4_reg64, "QU", desc)
4826
4827 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4828 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));

--- 343 unchanged lines hidden (view full) ---

5172 int qsize, rc;
5173
5174 qsize = pi->qsize_txq;
5175
5176 rc = sysctl_handle_int(oidp, &qsize, 0, req);
5177 if (rc != 0 || req->newptr == NULL)
5178 return (rc);
5179
4841
4842#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4843 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4844 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4845 sysctl_handle_t4_reg64, "QU", desc)
4846
4847 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4848 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));

--- 343 unchanged lines hidden (view full) ---

5192 int qsize, rc;
5193
5194 qsize = pi->qsize_txq;
5195
5196 rc = sysctl_handle_int(oidp, &qsize, 0, req);
5197 if (rc != 0 || req->newptr == NULL)
5198 return (rc);
5199
5180 /* bufring size must be powerof2 */
5181 if (qsize < 128 || !powerof2(qsize))
5200 if (qsize < 128 || qsize > 65536)
5182 return (EINVAL);
5183
5184 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5185 "t4txqs");
5186 if (rc)
5187 return (rc);
5188
5189 if (pi->flags & PORT_INIT_DONE)

--- 1681 unchanged lines hidden (view full) ---

6871 }
6872 rc = sbuf_finish(sb);
6873 sbuf_delete(sb);
6874
6875 return (rc);
6876}
6877#endif
6878
5201 return (EINVAL);
5202
5203 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5204 "t4txqs");
5205 if (rc)
5206 return (rc);
5207
5208 if (pi->flags & PORT_INIT_DONE)

--- 1681 unchanged lines hidden (view full) ---

6890 }
6891 rc = sbuf_finish(sb);
6892 sbuf_delete(sb);
6893
6894 return (rc);
6895}
6896#endif
6897
6879static inline void
6880txq_start(struct ifnet *ifp, struct sge_txq *txq)
6881{
6882 struct buf_ring *br;
6883 struct mbuf *m;
6884
6885 TXQ_LOCK_ASSERT_OWNED(txq);
6886
6887 br = txq->br;
6888 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6889 if (m)
6890 t4_eth_tx(ifp, txq, m);
6891}
6892
6893void
6894t4_tx_callout(void *arg)
6895{
6896 struct sge_eq *eq = arg;
6897 struct adapter *sc;
6898
6899 if (EQ_TRYLOCK(eq) == 0)
6900 goto reschedule;
6901
6902 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6903 EQ_UNLOCK(eq);
6904reschedule:
6905 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6906 callout_schedule(&eq->tx_callout, 1);
6907 return;
6908 }
6909
6910 EQ_LOCK_ASSERT_OWNED(eq);
6911
6912 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6913
6914 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6915 struct sge_txq *txq = arg;
6916 struct port_info *pi = txq->ifp->if_softc;
6917
6918 sc = pi->adapter;
6919 } else {
6920 struct sge_wrq *wrq = arg;
6921
6922 sc = wrq->adapter;
6923 }
6924
6925 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6926 }
6927
6928 EQ_UNLOCK(eq);
6929}
6930
6931void
6932t4_tx_task(void *arg, int count)
6933{
6934 struct sge_eq *eq = arg;
6935
6936 EQ_LOCK(eq);
6937 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6938 struct sge_txq *txq = arg;
6939 txq_start(txq->ifp, txq);
6940 } else {
6941 struct sge_wrq *wrq = arg;
6942 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6943 }
6944 EQ_UNLOCK(eq);
6945}
6946
6947static uint32_t
6948fconf_to_mode(uint32_t fconf)
6949{
6950 uint32_t mode;
6951
6952 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6953 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6954

--- 413 unchanged lines hidden (view full) ---

7368
7369 bzero(f, sizeof (*f));
7370}
7371
7372static int
7373set_filter_wr(struct adapter *sc, int fidx)
7374{
7375 struct filter_entry *f = &sc->tids.ftid_tab[fidx];
6898static uint32_t
6899fconf_to_mode(uint32_t fconf)
6900{
6901 uint32_t mode;
6902
6903 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6904 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6905

--- 413 unchanged lines hidden (view full) ---

7319
7320 bzero(f, sizeof (*f));
7321}
7322
7323static int
7324set_filter_wr(struct adapter *sc, int fidx)
7325{
7326 struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7376 struct wrqe *wr;
7377 struct fw_filter_wr *fwr;
7378 unsigned int ftid;
7327 struct fw_filter_wr *fwr;
7328 unsigned int ftid;
7329 struct wrq_cookie cookie;
7379
7380 ASSERT_SYNCHRONIZED_OP(sc);
7381
7382 if (f->fs.newdmac || f->fs.newvlan) {
7383 /* This filter needs an L2T entry; allocate one. */
7384 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7385 if (f->l2t == NULL)
7386 return (EAGAIN);
7387 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7388 f->fs.dmac)) {
7389 t4_l2t_release(f->l2t);
7390 f->l2t = NULL;
7391 return (ENOMEM);
7392 }
7393 }
7394
7395 ftid = sc->tids.ftid_base + fidx;
7396
7330
7331 ASSERT_SYNCHRONIZED_OP(sc);
7332
7333 if (f->fs.newdmac || f->fs.newvlan) {
7334 /* This filter needs an L2T entry; allocate one. */
7335 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7336 if (f->l2t == NULL)
7337 return (EAGAIN);
7338 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7339 f->fs.dmac)) {
7340 t4_l2t_release(f->l2t);
7341 f->l2t = NULL;
7342 return (ENOMEM);
7343 }
7344 }
7345
7346 ftid = sc->tids.ftid_base + fidx;
7347
7397 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7398 if (wr == NULL)
7348 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7349 if (fwr == NULL)
7399 return (ENOMEM);
7350 return (ENOMEM);
7351 bzero(fwr, sizeof(*fwr));
7400
7352
7401 fwr = wrtod(wr);
7402 bzero(fwr, sizeof (*fwr));
7403
7404 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7405 fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7406 fwr->tid_to_iq =
7407 htobe32(V_FW_FILTER_WR_TID(ftid) |
7408 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7409 V_FW_FILTER_WR_NOREPLY(0) |
7410 V_FW_FILTER_WR_IQ(f->fs.iq));
7411 fwr->del_filter_to_l2tix =

--- 51 unchanged lines hidden (view full) ---

7463 fwr->fp = htobe16(f->fs.val.sport);
7464 fwr->fpm = htobe16(f->fs.mask.sport);
7465 if (f->fs.newsmac)
7466 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7467
7468 f->pending = 1;
7469 sc->tids.ftids_in_use++;
7470
7353 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7354 fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7355 fwr->tid_to_iq =
7356 htobe32(V_FW_FILTER_WR_TID(ftid) |
7357 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7358 V_FW_FILTER_WR_NOREPLY(0) |
7359 V_FW_FILTER_WR_IQ(f->fs.iq));
7360 fwr->del_filter_to_l2tix =

--- 51 unchanged lines hidden (view full) ---

7412 fwr->fp = htobe16(f->fs.val.sport);
7413 fwr->fpm = htobe16(f->fs.mask.sport);
7414 if (f->fs.newsmac)
7415 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7416
7417 f->pending = 1;
7418 sc->tids.ftids_in_use++;
7419
7471 t4_wrq_tx(sc, wr);
7420 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7472 return (0);
7473}
7474
7475static int
7476del_filter_wr(struct adapter *sc, int fidx)
7477{
7478 struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7421 return (0);
7422}
7423
7424static int
7425del_filter_wr(struct adapter *sc, int fidx)
7426{
7427 struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7479 struct wrqe *wr;
7480 struct fw_filter_wr *fwr;
7481 unsigned int ftid;
7428 struct fw_filter_wr *fwr;
7429 unsigned int ftid;
7430 struct wrq_cookie cookie;
7482
7483 ftid = sc->tids.ftid_base + fidx;
7484
7431
7432 ftid = sc->tids.ftid_base + fidx;
7433
7485 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7486 if (wr == NULL)
7434 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7435 if (fwr == NULL)
7487 return (ENOMEM);
7436 return (ENOMEM);
7488 fwr = wrtod(wr);
7489 bzero(fwr, sizeof (*fwr));
7490
7491 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7492
7493 f->pending = 1;
7437 bzero(fwr, sizeof (*fwr));
7438
7439 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7440
7441 f->pending = 1;
7494 t4_wrq_tx(sc, wr);
7442 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7495 return (0);
7496}
7497
7498int
7499t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7500{
7501 struct adapter *sc = iq->adapter;
7502 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);

--- 583 unchanged lines hidden (view full) ---

8086 struct port_info *pi;
8087
8088 if (port_id >= sc->params.nports)
8089 return (EINVAL);
8090 pi = sc->port[port_id];
8091
8092 /* MAC stats */
8093 t4_clr_port_stats(sc, pi->tx_chan);
7443 return (0);
7444}
7445
7446int
7447t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7448{
7449 struct adapter *sc = iq->adapter;
7450 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);

--- 583 unchanged lines hidden (view full) ---

8034 struct port_info *pi;
8035
8036 if (port_id >= sc->params.nports)
8037 return (EINVAL);
8038 pi = sc->port[port_id];
8039
8040 /* MAC stats */
8041 t4_clr_port_stats(sc, pi->tx_chan);
8042 pi->tx_parse_error = 0;
8094
8095 if (pi->flags & PORT_INIT_DONE) {
8096 struct sge_rxq *rxq;
8097 struct sge_txq *txq;
8098 struct sge_wrq *wrq;
8099
8100 for_each_rxq(pi, i, rxq) {
8101#if defined(INET) || defined(INET6)

--- 6 unchanged lines hidden (view full) ---

8108
8109 for_each_txq(pi, i, txq) {
8110 txq->txcsum = 0;
8111 txq->tso_wrs = 0;
8112 txq->vlan_insertion = 0;
8113 txq->imm_wrs = 0;
8114 txq->sgl_wrs = 0;
8115 txq->txpkt_wrs = 0;
8043
8044 if (pi->flags & PORT_INIT_DONE) {
8045 struct sge_rxq *rxq;
8046 struct sge_txq *txq;
8047 struct sge_wrq *wrq;
8048
8049 for_each_rxq(pi, i, rxq) {
8050#if defined(INET) || defined(INET6)

--- 6 unchanged lines hidden (view full) ---

8057
8058 for_each_txq(pi, i, txq) {
8059 txq->txcsum = 0;
8060 txq->tso_wrs = 0;
8061 txq->vlan_insertion = 0;
8062 txq->imm_wrs = 0;
8063 txq->sgl_wrs = 0;
8064 txq->txpkt_wrs = 0;
8116 txq->txpkts_wrs = 0;
8117 txq->txpkts_pkts = 0;
8118 txq->br->br_drops = 0;
8119 txq->no_dmamap = 0;
8120 txq->no_desc = 0;
8065 txq->txpkts0_wrs = 0;
8066 txq->txpkts1_wrs = 0;
8067 txq->txpkts0_pkts = 0;
8068 txq->txpkts1_pkts = 0;
8069 mp_ring_reset_stats(txq->r);
8121 }
8122
8123#ifdef TCP_OFFLOAD
8124 /* nothing to clear for each ofld_rxq */
8125
8126 for_each_ofld_txq(pi, i, wrq) {
8070 }
8071
8072#ifdef TCP_OFFLOAD
8073 /* nothing to clear for each ofld_rxq */
8074
8075 for_each_ofld_txq(pi, i, wrq) {
8127 wrq->tx_wrs = 0;
8128 wrq->no_desc = 0;
8076 wrq->tx_wrs_direct = 0;
8077 wrq->tx_wrs_copied = 0;
8129 }
8130#endif
8131 wrq = &sc->sge.ctrlq[pi->port_id];
8078 }
8079#endif
8080 wrq = &sc->sge.ctrlq[pi->port_id];
8132 wrq->tx_wrs = 0;
8133 wrq->no_desc = 0;
8081 wrq->tx_wrs_direct = 0;
8082 wrq->tx_wrs_copied = 0;
8134 }
8135 break;
8136 }
8137 case CHELSIO_T4_SCHED_CLASS:
8138 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8139 break;
8140 case CHELSIO_T4_SCHED_QUEUE:
8141 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);

--- 358 unchanged lines hidden --- 		8083 }
8084 break;
8085 }
8086 case CHELSIO_T4_SCHED_CLASS:
8087 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8088 break;
8089 case CHELSIO_T4_SCHED_QUEUE:
8090 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);

--- 358 unchanged lines hidden ---