t4_main.c (282486) | t4_main.c (284052) |
---|---|
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 282486 2015-05-05 15:13:59Z np $"); | 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 284052 2015-06-06 09:28:40Z np $"); |
30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> | 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> |
39#include <sys/systm.h> 40#include <sys/counter.h> |
|
39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> --- 14 unchanged lines hidden (view full) --- 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" | 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> --- 14 unchanged lines hidden (view full) --- 63#endif 64 65#include "common/common.h" 66#include "common/t4_msg.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "t4_ioctl.h" 70#include "t4_l2t.h" |
71#include "t4_mp_ring.h" |
|
69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), --- 295 unchanged lines hidden (view full) --- 372static int get_params__pre_init(struct adapter *); 373static int get_params__post_init(struct adapter *); 374static int set_params__post_init(struct adapter *); 375static void t4_set_desc(struct adapter *); 376static void build_medialist(struct port_info *, struct ifmedia *); 377static int cxgbe_init_synchronized(struct port_info *); 378static int cxgbe_uninit_synchronized(struct port_info *); 379static int setup_intr_handlers(struct adapter *); | 72 73/* T4 bus driver interface */ 74static int t4_probe(device_t); 75static int t4_attach(device_t); 76static int t4_detach(device_t); 77static device_method_t t4_methods[] = { 78 DEVMETHOD(device_probe, t4_probe), 79 DEVMETHOD(device_attach, t4_attach), --- 295 unchanged lines hidden (view full) --- 375static int get_params__pre_init(struct adapter *); 376static int get_params__post_init(struct adapter *); 377static int set_params__post_init(struct adapter *); 378static void t4_set_desc(struct adapter *); 379static void build_medialist(struct port_info *, struct ifmedia *); 380static int cxgbe_init_synchronized(struct port_info *); 381static int cxgbe_uninit_synchronized(struct port_info *); 382static int setup_intr_handlers(struct adapter *); |
380static void quiesce_eq(struct adapter *, struct sge_eq *); | 383static void quiesce_txq(struct adapter *, struct sge_txq *); 384static void quiesce_wrq(struct adapter *, struct sge_wrq *); |
381static void quiesce_iq(struct adapter *, struct sge_iq *); 382static void quiesce_fl(struct adapter *, struct sge_fl *); 383static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 384 driver_intr_t *, void *, char *); 385static int t4_free_irq(struct adapter *, struct irq *); 386static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 387 unsigned int); 388static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); --- 39 unchanged lines hidden (view full) --- 428static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 429static int sysctl_tids(SYSCTL_HANDLER_ARGS); 430static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 431static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 432static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 433static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 434static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 435#endif | 385static void quiesce_iq(struct adapter *, struct sge_iq *); 386static void quiesce_fl(struct adapter *, struct sge_fl *); 387static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 388 driver_intr_t *, void *, char *); 389static int t4_free_irq(struct adapter *, struct irq *); 390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 391 unsigned int); 392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); --- 39 unchanged lines hidden (view full) --- 432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 433static int sysctl_tids(SYSCTL_HANDLER_ARGS); 434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 439#endif |
436static inline void txq_start(struct ifnet *, struct sge_txq *); | |
437static uint32_t fconf_to_mode(uint32_t); 438static uint32_t mode_to_fconf(uint32_t); 439static uint32_t fspec_to_fconf(struct t4_filter_specification *); 440static int get_filter_mode(struct adapter *, uint32_t *); 441static int set_filter_mode(struct adapter *, uint32_t); 442static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 443static int get_filter(struct adapter *, struct t4_filter *); 444static int set_filter(struct adapter *, struct t4_filter *); --- 215 unchanged lines hidden (view full) --- 660 sc->cdev->si_drv1 = sc; 661 662 /* Go no further if recovery mode has been requested. */ 663 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 664 device_printf(dev, "recovery mode.\n"); 665 goto done; 666 } 667 | 440static uint32_t fconf_to_mode(uint32_t); 441static uint32_t mode_to_fconf(uint32_t); 442static uint32_t fspec_to_fconf(struct t4_filter_specification *); 443static int get_filter_mode(struct adapter *, uint32_t *); 444static int set_filter_mode(struct adapter *, uint32_t); 445static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 446static int get_filter(struct adapter *, struct t4_filter *); 447static int set_filter(struct adapter *, struct t4_filter *); --- 215 unchanged lines hidden (view full) --- 663 sc->cdev->si_drv1 = sc; 664 665 /* Go no further if recovery mode has been requested. */ 666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 667 device_printf(dev, "recovery mode.\n"); 668 goto done; 669 } 670 |
671#if defined(__i386__) 672 if ((cpu_feature & CPUID_CX8) == 0) { 673 device_printf(dev, "64 bit atomics not available.\n"); 674 rc = ENOTSUP; 675 goto done; 676 } 677#endif 678 |
|
668 /* Prepare the firmware for operation */ 669 rc = prep_firmware(sc); 670 if (rc != 0) 671 goto done; /* error message displayed already */ 672 673 rc = get_params__post_init(sc); 674 if (rc != 0) 675 goto done; /* error message displayed already */ --- 705 unchanged lines hidden (view full) --- 1381 return (rc); 1382} 1383 1384static int 1385cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1386{ 1387 struct port_info *pi = ifp->if_softc; 1388 struct adapter *sc = pi->adapter; | 679 /* Prepare the firmware for operation */ 680 rc = prep_firmware(sc); 681 if (rc != 0) 682 goto done; /* error message displayed already */ 683 684 rc = get_params__post_init(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ --- 705 unchanged lines hidden (view full) --- 1392 return (rc); 1393} 1394 1395static int 1396cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1397{ 1398 struct port_info *pi = ifp->if_softc; 1399 struct adapter *sc = pi->adapter; |
1389 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1390 struct buf_ring *br; | 1400 struct sge_txq *txq; 1401 void *items[1]; |
1391 int rc; 1392 1393 M_ASSERTPKTHDR(m); | 1402 int rc; 1403 1404 M_ASSERTPKTHDR(m); |
1405 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ |
|
1394 1395 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1396 m_freem(m); 1397 return (ENETDOWN); 1398 } 1399 | 1406 1407 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1408 m_freem(m); 1409 return (ENETDOWN); 1410 } 1411 |
1400 /* check if flowid is set */ 1401 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1402 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1403 + pi->rsrv_noflowq); 1404 br = txq->br; 1405 1406 if (TXQ_TRYLOCK(txq) == 0) { 1407 struct sge_eq *eq = &txq->eq; 1408 1409 /* 1410 * It is possible that t4_eth_tx finishes up and releases the 1411 * lock between the TRYLOCK above and the drbr_enqueue here. We 1412 * need to make sure that this mbuf doesn't just sit there in 1413 * the drbr. 1414 */ 1415 1416 rc = drbr_enqueue(ifp, br, m); 1417 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1418 !(eq->flags & EQ_DOOMED)) 1419 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); | 1412 rc = parse_pkt(&m); 1413 if (__predict_false(rc != 0)) { 1414 MPASS(m == NULL); /* was freed already */ 1415 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ |
1420 return (rc); 1421 } 1422 | 1416 return (rc); 1417 } 1418 |
1423 /* 1424 * txq->m is the mbuf that is held up due to a temporary shortage of 1425 * resources and it should be put on the wire first. Then what's in 1426 * drbr and finally the mbuf that was just passed in to us. 1427 * 1428 * Return code should indicate the fate of the mbuf that was passed in 1429 * this time. 1430 */ | 1419 /* Select a txq. */ 1420 txq = &sc->sge.txq[pi->first_txq]; 1421 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1422 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1423 pi->rsrv_noflowq); |
1431 | 1424 |
1432 TXQ_LOCK_ASSERT_OWNED(txq); 1433 if (drbr_needs_enqueue(ifp, br) || txq->m) { | 1425 items[0] = m; 1426 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1427 if (__predict_false(rc != 0)) 1428 m_freem(m); |
1434 | 1429 |
1435 /* Queued for transmission. */ 1436 1437 rc = drbr_enqueue(ifp, br, m); 1438 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1439 (void) t4_eth_tx(ifp, txq, m); 1440 TXQ_UNLOCK(txq); 1441 return (rc); 1442 } 1443 1444 /* Direct transmission. */ 1445 rc = t4_eth_tx(ifp, txq, m); 1446 if (rc != 0 && txq->m) 1447 rc = 0; /* held, will be transmitted soon (hopefully) */ 1448 1449 TXQ_UNLOCK(txq); | |
1450 return (rc); 1451} 1452 1453static void 1454cxgbe_qflush(struct ifnet *ifp) 1455{ 1456 struct port_info *pi = ifp->if_softc; 1457 struct sge_txq *txq; 1458 int i; | 1430 return (rc); 1431} 1432 1433static void 1434cxgbe_qflush(struct ifnet *ifp) 1435{ 1436 struct port_info *pi = ifp->if_softc; 1437 struct sge_txq *txq; 1438 int i; |
1459 struct mbuf *m; | |
1460 1461 /* queues do not exist if !PORT_INIT_DONE. */ 1462 if (pi->flags & PORT_INIT_DONE) { 1463 for_each_txq(pi, i, txq) { 1464 TXQ_LOCK(txq); | 1439 1440 /* queues do not exist if !PORT_INIT_DONE. */ 1441 if (pi->flags & PORT_INIT_DONE) { 1442 for_each_txq(pi, i, txq) { 1443 TXQ_LOCK(txq); |
1465 m_freem(txq->m); 1466 txq->m = NULL; 1467 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1468 m_freem(m); | 1444 txq->eq.flags &= ~EQ_ENABLED; |
1469 TXQ_UNLOCK(txq); | 1445 TXQ_UNLOCK(txq); |
1446 while (!mp_ring_is_idle(txq->r)) { 1447 mp_ring_check_drainage(txq->r, 0); 1448 pause("qflush", 1); 1449 } |
|
1470 } 1471 } 1472 if_qflush(ifp); 1473} 1474 1475static int 1476cxgbe_media_change(struct ifnet *ifp) 1477{ --- 1652 unchanged lines hidden (view full) --- 3130 ADAPTER_UNLOCK(sc); 3131} 3132 3133static int 3134cxgbe_init_synchronized(struct port_info *pi) 3135{ 3136 struct adapter *sc = pi->adapter; 3137 struct ifnet *ifp = pi->ifp; | 1450 } 1451 } 1452 if_qflush(ifp); 1453} 1454 1455static int 1456cxgbe_media_change(struct ifnet *ifp) 1457{ --- 1652 unchanged lines hidden (view full) --- 3110 ADAPTER_UNLOCK(sc); 3111} 3112 3113static int 3114cxgbe_init_synchronized(struct port_info *pi) 3115{ 3116 struct adapter *sc = pi->adapter; 3117 struct ifnet *ifp = pi->ifp; |
3138 int rc = 0; | 3118 int rc = 0, i; 3119 struct sge_txq *txq; |
3139 3140 ASSERT_SYNCHRONIZED_OP(sc); 3141 3142 if (isset(&sc->open_device_map, pi->port_id)) { 3143 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3144 ("mismatch between open_device_map and if_drv_flags")); 3145 return (0); /* already running */ 3146 } --- 12 unchanged lines hidden (view full) --- 3159 3160 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3161 if (rc != 0) { 3162 if_printf(ifp, "enable_vi failed: %d\n", rc); 3163 goto done; 3164 } 3165 3166 /* | 3120 3121 ASSERT_SYNCHRONIZED_OP(sc); 3122 3123 if (isset(&sc->open_device_map, pi->port_id)) { 3124 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3125 ("mismatch between open_device_map and if_drv_flags")); 3126 return (0); /* already running */ 3127 } --- 12 unchanged lines hidden (view full) --- 3140 3141 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3142 if (rc != 0) { 3143 if_printf(ifp, "enable_vi failed: %d\n", rc); 3144 goto done; 3145 } 3146 3147 /* |
3148 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3149 * if this changes. 3150 */ 3151 3152 for_each_txq(pi, i, txq) { 3153 TXQ_LOCK(txq); 3154 txq->eq.flags |= EQ_ENABLED; 3155 TXQ_UNLOCK(txq); 3156 } 3157 3158 /* |
|
3167 * The first iq of the first port to come up is used for tracing. 3168 */ 3169 if (sc->traceq < 0) { 3170 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3171 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3172 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3173 V_QUEUENUMBER(sc->traceq)); 3174 pi->flags |= HAS_TRACEQ; --- 16 unchanged lines hidden (view full) --- 3191/* 3192 * Idempotent. 3193 */ 3194static int 3195cxgbe_uninit_synchronized(struct port_info *pi) 3196{ 3197 struct adapter *sc = pi->adapter; 3198 struct ifnet *ifp = pi->ifp; | 3159 * The first iq of the first port to come up is used for tracing. 3160 */ 3161 if (sc->traceq < 0) { 3162 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3163 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3164 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3165 V_QUEUENUMBER(sc->traceq)); 3166 pi->flags |= HAS_TRACEQ; --- 16 unchanged lines hidden (view full) --- 3183/* 3184 * Idempotent. 3185 */ 3186static int 3187cxgbe_uninit_synchronized(struct port_info *pi) 3188{ 3189 struct adapter *sc = pi->adapter; 3190 struct ifnet *ifp = pi->ifp; |
3199 int rc; | 3191 int rc, i; 3192 struct sge_txq *txq; |
3200 3201 ASSERT_SYNCHRONIZED_OP(sc); 3202 3203 /* 3204 * Disable the VI so that all its data in either direction is discarded 3205 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3206 * tick) intact as the TP can deliver negative advice or data that it's 3207 * holding in its RAM (for an offloaded connection) even after the VI is 3208 * disabled. 3209 */ 3210 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3211 if (rc) { 3212 if_printf(ifp, "disable_vi failed: %d\n", rc); 3213 return (rc); 3214 } 3215 | 3193 3194 ASSERT_SYNCHRONIZED_OP(sc); 3195 3196 /* 3197 * Disable the VI so that all its data in either direction is discarded 3198 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3199 * tick) intact as the TP can deliver negative advice or data that it's 3200 * holding in its RAM (for an offloaded connection) even after the VI is 3201 * disabled. 3202 */ 3203 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3204 if (rc) { 3205 if_printf(ifp, "disable_vi failed: %d\n", rc); 3206 return (rc); 3207 } 3208 |
3209 for_each_txq(pi, i, txq) { 3210 TXQ_LOCK(txq); 3211 txq->eq.flags &= ~EQ_ENABLED; 3212 TXQ_UNLOCK(txq); 3213 } 3214 |
|
3216 clrbit(&sc->open_device_map, pi->port_id); 3217 PORT_LOCK(pi); 3218 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3219 PORT_UNLOCK(pi); 3220 3221 pi->link_cfg.link_ok = 0; 3222 pi->link_cfg.speed = 0; 3223 pi->linkdnrc = -1; --- 214 unchanged lines hidden (view full) --- 3438 struct sge_txq *txq; 3439#ifdef TCP_OFFLOAD 3440 struct sge_ofld_rxq *ofld_rxq; 3441 struct sge_wrq *ofld_txq; 3442#endif 3443 3444 if (pi->flags & PORT_INIT_DONE) { 3445 | 3215 clrbit(&sc->open_device_map, pi->port_id); 3216 PORT_LOCK(pi); 3217 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3218 PORT_UNLOCK(pi); 3219 3220 pi->link_cfg.link_ok = 0; 3221 pi->link_cfg.speed = 0; 3222 pi->linkdnrc = -1; --- 214 unchanged lines hidden (view full) --- 3437 struct sge_txq *txq; 3438#ifdef TCP_OFFLOAD 3439 struct sge_ofld_rxq *ofld_rxq; 3440 struct sge_wrq *ofld_txq; 3441#endif 3442 3443 if (pi->flags & PORT_INIT_DONE) { 3444 |
3446 /* Need to quiesce queues. XXX: ctrl queues? */ | 3445 /* Need to quiesce queues. */ |
3447 | 3446 |
3447 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3448 |
|
3448 for_each_txq(pi, i, txq) { | 3449 for_each_txq(pi, i, txq) { |
3449 quiesce_eq(sc, &txq->eq); | 3450 quiesce_txq(sc, txq); |
3450 } 3451 3452#ifdef TCP_OFFLOAD 3453 for_each_ofld_txq(pi, i, ofld_txq) { | 3451 } 3452 3453#ifdef TCP_OFFLOAD 3454 for_each_ofld_txq(pi, i, ofld_txq) { |
3454 quiesce_eq(sc, &ofld_txq->eq); | 3455 quiesce_wrq(sc, ofld_txq); |
3455 } 3456#endif 3457 3458 for_each_rxq(pi, i, rxq) { 3459 quiesce_iq(sc, &rxq->iq); 3460 quiesce_fl(sc, &rxq->fl); 3461 } 3462 --- 8 unchanged lines hidden (view full) --- 3471 3472 t4_teardown_port_queues(pi); 3473 pi->flags &= ~PORT_INIT_DONE; 3474 3475 return (0); 3476} 3477 3478static void | 3456 } 3457#endif 3458 3459 for_each_rxq(pi, i, rxq) { 3460 quiesce_iq(sc, &rxq->iq); 3461 quiesce_fl(sc, &rxq->fl); 3462 } 3463 --- 8 unchanged lines hidden (view full) --- 3472 3473 t4_teardown_port_queues(pi); 3474 pi->flags &= ~PORT_INIT_DONE; 3475 3476 return (0); 3477} 3478 3479static void |
3479quiesce_eq(struct adapter *sc, struct sge_eq *eq) | 3480quiesce_txq(struct adapter *sc, struct sge_txq *txq) |
3480{ | 3481{ |
3481 EQ_LOCK(eq); 3482 eq->flags |= EQ_DOOMED; | 3482 struct sge_eq *eq = &txq->eq; 3483 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; |
3483 | 3484 |
3484 /* 3485 * Wait for the response to a credit flush if one's 3486 * pending. 3487 */ 3488 while (eq->flags & EQ_CRFLUSHED) 3489 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3490 EQ_UNLOCK(eq); | 3485 (void) sc; /* unused */ |
3491 | 3486 |
3492 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3493 pause("callout", 10); /* Still iffy */ | 3487#ifdef INVARIANTS 3488 TXQ_LOCK(txq); 3489 MPASS((eq->flags & EQ_ENABLED) == 0); 3490 TXQ_UNLOCK(txq); 3491#endif |
3494 | 3492 |
3495 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); | 3493 /* Wait for the mp_ring to empty. */ 3494 while (!mp_ring_is_idle(txq->r)) { 3495 mp_ring_check_drainage(txq->r, 0); 3496 pause("rquiesce", 1); 3497 } 3498 3499 /* Then wait for the hardware to finish. */ 3500 while (spg->cidx != htobe16(eq->pidx)) 3501 pause("equiesce", 1); 3502 3503 /* Finally, wait for the driver to reclaim all descriptors. */ 3504 while (eq->cidx != eq->pidx) 3505 pause("dquiesce", 1); |
3496} 3497 3498static void | 3506} 3507 3508static void |
3509quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3510{ 3511 3512 /* XXXTX */ 3513} 3514 3515static void |
|
3499quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3500{ 3501 (void) sc; /* unused */ 3502 3503 /* Synchronize with the interrupt handler */ 3504 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3505 pause("iqfree", 1); 3506} --- 775 unchanged lines hidden (view full) --- 4282 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4283 mtx_unlock(&sc->regwin_lock); 4284 ifp->if_iqdrops += v; 4285 } 4286 } 4287 4288 drops = s->tx_drop; 4289 for_each_txq(pi, i, txq) | 3516quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3517{ 3518 (void) sc; /* unused */ 3519 3520 /* Synchronize with the interrupt handler */ 3521 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3522 pause("iqfree", 1); 3523} --- 775 unchanged lines hidden (view full) --- 4299 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4300 mtx_unlock(&sc->regwin_lock); 4301 ifp->if_iqdrops += v; 4302 } 4303 } 4304 4305 drops = s->tx_drop; 4306 for_each_txq(pi, i, txq) |
4290 drops += txq->br->br_drops; | 4307 drops += counter_u64_fetch(txq->r->drops); |
4291 ifp->if_snd.ifq_drops = drops; 4292 4293 ifp->if_oerrors = s->tx_error_frames; 4294 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4295 s->rx_fcs_err + s->rx_len_err; 4296 4297 getmicrotime(&pi->last_refreshed); 4298} --- 514 unchanged lines hidden (view full) --- 4813 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4814 4815 /* 4816 * dev.cxgbe.X.stats. 4817 */ 4818 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4819 NULL, "port statistics"); 4820 children = SYSCTL_CHILDREN(oid); | 4308 ifp->if_snd.ifq_drops = drops; 4309 4310 ifp->if_oerrors = s->tx_error_frames; 4311 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4312 s->rx_fcs_err + s->rx_len_err; 4313 4314 getmicrotime(&pi->last_refreshed); 4315} --- 514 unchanged lines hidden (view full) --- 4830 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4831 4832 /* 4833 * dev.cxgbe.X.stats. 4834 */ 4835 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4836 NULL, "port statistics"); 4837 children = SYSCTL_CHILDREN(oid); |
4838 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4839 &pi->tx_parse_error, 0, 4840 "# of tx packets with invalid length or # of segments"); |
|
4821 4822#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4823 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4824 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4825 sysctl_handle_t4_reg64, "QU", desc) 4826 4827 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4828 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); --- 343 unchanged lines hidden (view full) --- 5172 int qsize, rc; 5173 5174 qsize = pi->qsize_txq; 5175 5176 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5177 if (rc != 0 || req->newptr == NULL) 5178 return (rc); 5179 | 4841 4842#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4843 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4844 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4845 sysctl_handle_t4_reg64, "QU", desc) 4846 4847 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4848 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); --- 343 unchanged lines hidden (view full) --- 5192 int qsize, rc; 5193 5194 qsize = pi->qsize_txq; 5195 5196 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5197 if (rc != 0 || req->newptr == NULL) 5198 return (rc); 5199 |
5180 /* bufring size must be powerof2 */ 5181 if (qsize < 128 || !powerof2(qsize)) | 5200 if (qsize < 128 || qsize > 65536) |
5182 return (EINVAL); 5183 5184 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5185 "t4txqs"); 5186 if (rc) 5187 return (rc); 5188 5189 if (pi->flags & PORT_INIT_DONE) --- 1681 unchanged lines hidden (view full) --- 6871 } 6872 rc = sbuf_finish(sb); 6873 sbuf_delete(sb); 6874 6875 return (rc); 6876} 6877#endif 6878 | 5201 return (EINVAL); 5202 5203 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5204 "t4txqs"); 5205 if (rc) 5206 return (rc); 5207 5208 if (pi->flags & PORT_INIT_DONE) --- 1681 unchanged lines hidden (view full) --- 6890 } 6891 rc = sbuf_finish(sb); 6892 sbuf_delete(sb); 6893 6894 return (rc); 6895} 6896#endif 6897 |
6879static inline void 6880txq_start(struct ifnet *ifp, struct sge_txq *txq) 6881{ 6882 struct buf_ring *br; 6883 struct mbuf *m; 6884 6885 TXQ_LOCK_ASSERT_OWNED(txq); 6886 6887 br = txq->br; 6888 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6889 if (m) 6890 t4_eth_tx(ifp, txq, m); 6891} 6892 6893void 6894t4_tx_callout(void *arg) 6895{ 6896 struct sge_eq *eq = arg; 6897 struct adapter *sc; 6898 6899 if (EQ_TRYLOCK(eq) == 0) 6900 goto reschedule; 6901 6902 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6903 EQ_UNLOCK(eq); 6904reschedule: 6905 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6906 callout_schedule(&eq->tx_callout, 1); 6907 return; 6908 } 6909 6910 EQ_LOCK_ASSERT_OWNED(eq); 6911 6912 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6913 6914 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6915 struct sge_txq *txq = arg; 6916 struct port_info *pi = txq->ifp->if_softc; 6917 6918 sc = pi->adapter; 6919 } else { 6920 struct sge_wrq *wrq = arg; 6921 6922 sc = wrq->adapter; 6923 } 6924 6925 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6926 } 6927 6928 EQ_UNLOCK(eq); 6929} 6930 6931void 6932t4_tx_task(void *arg, int count) 6933{ 6934 struct sge_eq *eq = arg; 6935 6936 EQ_LOCK(eq); 6937 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6938 struct sge_txq *txq = arg; 6939 txq_start(txq->ifp, txq); 6940 } else { 6941 struct sge_wrq *wrq = arg; 6942 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6943 } 6944 EQ_UNLOCK(eq); 6945} 6946 | |
6947static uint32_t 6948fconf_to_mode(uint32_t fconf) 6949{ 6950 uint32_t mode; 6951 6952 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6953 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6954 --- 413 unchanged lines hidden (view full) --- 7368 7369 bzero(f, sizeof (*f)); 7370} 7371 7372static int 7373set_filter_wr(struct adapter *sc, int fidx) 7374{ 7375 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; | 6898static uint32_t 6899fconf_to_mode(uint32_t fconf) 6900{ 6901 uint32_t mode; 6902 6903 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6904 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6905 --- 413 unchanged lines hidden (view full) --- 7319 7320 bzero(f, sizeof (*f)); 7321} 7322 7323static int 7324set_filter_wr(struct adapter *sc, int fidx) 7325{ 7326 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; |
7376 struct wrqe *wr; | |
7377 struct fw_filter_wr *fwr; 7378 unsigned int ftid; | 7327 struct fw_filter_wr *fwr; 7328 unsigned int ftid; |
7329 struct wrq_cookie cookie; |
|
7379 7380 ASSERT_SYNCHRONIZED_OP(sc); 7381 7382 if (f->fs.newdmac || f->fs.newvlan) { 7383 /* This filter needs an L2T entry; allocate one. */ 7384 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7385 if (f->l2t == NULL) 7386 return (EAGAIN); 7387 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7388 f->fs.dmac)) { 7389 t4_l2t_release(f->l2t); 7390 f->l2t = NULL; 7391 return (ENOMEM); 7392 } 7393 } 7394 7395 ftid = sc->tids.ftid_base + fidx; 7396 | 7330 7331 ASSERT_SYNCHRONIZED_OP(sc); 7332 7333 if (f->fs.newdmac || f->fs.newvlan) { 7334 /* This filter needs an L2T entry; allocate one. */ 7335 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7336 if (f->l2t == NULL) 7337 return (EAGAIN); 7338 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7339 f->fs.dmac)) { 7340 t4_l2t_release(f->l2t); 7341 f->l2t = NULL; 7342 return (ENOMEM); 7343 } 7344 } 7345 7346 ftid = sc->tids.ftid_base + fidx; 7347 |
7397 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7398 if (wr == NULL) | 7348 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7349 if (fwr == NULL) |
7399 return (ENOMEM); | 7350 return (ENOMEM); |
7351 bzero(fwr, sizeof(*fwr)); |
|
7400 | 7352 |
7401 fwr = wrtod(wr); 7402 bzero(fwr, sizeof (*fwr)); 7403 | |
7404 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7405 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7406 fwr->tid_to_iq = 7407 htobe32(V_FW_FILTER_WR_TID(ftid) | 7408 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7409 V_FW_FILTER_WR_NOREPLY(0) | 7410 V_FW_FILTER_WR_IQ(f->fs.iq)); 7411 fwr->del_filter_to_l2tix = --- 51 unchanged lines hidden (view full) --- 7463 fwr->fp = htobe16(f->fs.val.sport); 7464 fwr->fpm = htobe16(f->fs.mask.sport); 7465 if (f->fs.newsmac) 7466 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7467 7468 f->pending = 1; 7469 sc->tids.ftids_in_use++; 7470 | 7353 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7354 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7355 fwr->tid_to_iq = 7356 htobe32(V_FW_FILTER_WR_TID(ftid) | 7357 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7358 V_FW_FILTER_WR_NOREPLY(0) | 7359 V_FW_FILTER_WR_IQ(f->fs.iq)); 7360 fwr->del_filter_to_l2tix = --- 51 unchanged lines hidden (view full) --- 7412 fwr->fp = htobe16(f->fs.val.sport); 7413 fwr->fpm = htobe16(f->fs.mask.sport); 7414 if (f->fs.newsmac) 7415 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7416 7417 f->pending = 1; 7418 sc->tids.ftids_in_use++; 7419 |
7471 t4_wrq_tx(sc, wr); | 7420 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); |
7472 return (0); 7473} 7474 7475static int 7476del_filter_wr(struct adapter *sc, int fidx) 7477{ 7478 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; | 7421 return (0); 7422} 7423 7424static int 7425del_filter_wr(struct adapter *sc, int fidx) 7426{ 7427 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; |
7479 struct wrqe *wr; | |
7480 struct fw_filter_wr *fwr; 7481 unsigned int ftid; | 7428 struct fw_filter_wr *fwr; 7429 unsigned int ftid; |
7430 struct wrq_cookie cookie; |
|
7482 7483 ftid = sc->tids.ftid_base + fidx; 7484 | 7431 7432 ftid = sc->tids.ftid_base + fidx; 7433 |
7485 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7486 if (wr == NULL) | 7434 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7435 if (fwr == NULL) |
7487 return (ENOMEM); | 7436 return (ENOMEM); |
7488 fwr = wrtod(wr); | |
7489 bzero(fwr, sizeof (*fwr)); 7490 7491 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7492 7493 f->pending = 1; | 7437 bzero(fwr, sizeof (*fwr)); 7438 7439 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7440 7441 f->pending = 1; |
7494 t4_wrq_tx(sc, wr); | 7442 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); |
7495 return (0); 7496} 7497 7498int 7499t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7500{ 7501 struct adapter *sc = iq->adapter; 7502 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); --- 583 unchanged lines hidden (view full) --- 8086 struct port_info *pi; 8087 8088 if (port_id >= sc->params.nports) 8089 return (EINVAL); 8090 pi = sc->port[port_id]; 8091 8092 /* MAC stats */ 8093 t4_clr_port_stats(sc, pi->tx_chan); | 7443 return (0); 7444} 7445 7446int 7447t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7448{ 7449 struct adapter *sc = iq->adapter; 7450 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); --- 583 unchanged lines hidden (view full) --- 8034 struct port_info *pi; 8035 8036 if (port_id >= sc->params.nports) 8037 return (EINVAL); 8038 pi = sc->port[port_id]; 8039 8040 /* MAC stats */ 8041 t4_clr_port_stats(sc, pi->tx_chan); |
8042 pi->tx_parse_error = 0; |
|
8094 8095 if (pi->flags & PORT_INIT_DONE) { 8096 struct sge_rxq *rxq; 8097 struct sge_txq *txq; 8098 struct sge_wrq *wrq; 8099 8100 for_each_rxq(pi, i, rxq) { 8101#if defined(INET) || defined(INET6) --- 6 unchanged lines hidden (view full) --- 8108 8109 for_each_txq(pi, i, txq) { 8110 txq->txcsum = 0; 8111 txq->tso_wrs = 0; 8112 txq->vlan_insertion = 0; 8113 txq->imm_wrs = 0; 8114 txq->sgl_wrs = 0; 8115 txq->txpkt_wrs = 0; | 8043 8044 if (pi->flags & PORT_INIT_DONE) { 8045 struct sge_rxq *rxq; 8046 struct sge_txq *txq; 8047 struct sge_wrq *wrq; 8048 8049 for_each_rxq(pi, i, rxq) { 8050#if defined(INET) || defined(INET6) --- 6 unchanged lines hidden (view full) --- 8057 8058 for_each_txq(pi, i, txq) { 8059 txq->txcsum = 0; 8060 txq->tso_wrs = 0; 8061 txq->vlan_insertion = 0; 8062 txq->imm_wrs = 0; 8063 txq->sgl_wrs = 0; 8064 txq->txpkt_wrs = 0; |
8116 txq->txpkts_wrs = 0; 8117 txq->txpkts_pkts = 0; 8118 txq->br->br_drops = 0; 8119 txq->no_dmamap = 0; 8120 txq->no_desc = 0; | 8065 txq->txpkts0_wrs = 0; 8066 txq->txpkts1_wrs = 0; 8067 txq->txpkts0_pkts = 0; 8068 txq->txpkts1_pkts = 0; 8069 mp_ring_reset_stats(txq->r); |
8121 } 8122 8123#ifdef TCP_OFFLOAD 8124 /* nothing to clear for each ofld_rxq */ 8125 8126 for_each_ofld_txq(pi, i, wrq) { | 8070 } 8071 8072#ifdef TCP_OFFLOAD 8073 /* nothing to clear for each ofld_rxq */ 8074 8075 for_each_ofld_txq(pi, i, wrq) { |
8127 wrq->tx_wrs = 0; 8128 wrq->no_desc = 0; | 8076 wrq->tx_wrs_direct = 0; 8077 wrq->tx_wrs_copied = 0; |
8129 } 8130#endif 8131 wrq = &sc->sge.ctrlq[pi->port_id]; | 8078 } 8079#endif 8080 wrq = &sc->sge.ctrlq[pi->port_id]; |
8132 wrq->tx_wrs = 0; 8133 wrq->no_desc = 0; | 8081 wrq->tx_wrs_direct = 0; 8082 wrq->tx_wrs_copied = 0; |
8134 } 8135 break; 8136 } 8137 case CHELSIO_T4_SCHED_CLASS: 8138 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8139 break; 8140 case CHELSIO_T4_SCHED_QUEUE: 8141 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); --- 358 unchanged lines hidden --- | 8083 } 8084 break; 8085 } 8086 case CHELSIO_T4_SCHED_CLASS: 8087 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8088 break; 8089 case CHELSIO_T4_SCHED_QUEUE: 8090 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); --- 358 unchanged lines hidden --- |