1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
--- 12 unchanged lines hidden (view full) ---
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_sge.c 219289 2011-03-05 03:27:14Z np $");
30
31#include "opt_inet.h"
32
33#include <sys/types.h>
34#include <sys/mbuf.h>
35#include <sys/socket.h>
36#include <sys/kernel.h>
37#include <sys/malloc.h>
--- 78 unchanged lines hidden (view full) ---
116static inline bool is_new_response(const struct sge_iq *, struct rsp_ctrl **);
117static inline void iq_next(struct sge_iq *);
118static inline void ring_fl_db(struct adapter *, struct sge_fl *);
119static void refill_fl(struct sge_fl *, int);
120static int alloc_fl_sdesc(struct sge_fl *);
121static void free_fl_sdesc(struct sge_fl *);
122static int alloc_eq_maps(struct sge_eq *);
123static void free_eq_maps(struct sge_eq *);
124static struct mbuf *get_fl_sdesc_data(struct sge_fl *, int, int);
125static void set_fl_tag_idx(struct sge_fl *, int);
126
127static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
128static int free_pkt_sgl(struct sge_txq *, struct sgl *);
129static int write_txpkt_wr(struct port_info *, struct sge_txq *, struct mbuf *,
130 struct sgl *);
131static int add_to_txpkts(struct port_info *, struct sge_txq *, struct txpkts *,
132 struct mbuf *, struct sgl *);
--- 395 unchanged lines hidden (view full) ---
528
529void
530t4_intr_data(void *arg)
531{
532 struct sge_rxq *rxq = arg;
533 struct sge_iq *iq = arg;
534 struct adapter *sc = iq->adapter;
535 struct rsp_ctrl *ctrl;
536 struct sge_fl *fl = &rxq->fl;
537 struct ifnet *ifp = rxq->ifp;
538 const struct rss_header *rss;
539 const struct cpl_rx_pkt *cpl;
540 int ndescs = 0, rsp_type;
541 uint32_t len;
542 struct mbuf *m0, *m;
543#ifdef INET
544 struct lro_ctrl *lro = &rxq->lro;
545 struct lro_entry *l;
546#endif
547
548 IQ_LOCK(iq);
549 iq->intr_next = iq->intr_params;
550 while (is_new_response(iq, &ctrl)) {
551
552 rmb();
553
554 rss = (const void *)iq->cdesc;
555 cpl = (const void *)(rss + 1);
556
557 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
558
559 if (__predict_false(rsp_type == X_RSPD_TYPE_CPL)) {
560 /* Can't be anything except an egress update */
561 handle_sge_egr_update(sc, (const void *)cpl);
562 goto nextdesc;
563 }
564
565 KASSERT(G_RSPD_TYPE(ctrl->u.type_gen) == X_RSPD_TYPE_FLBUF,
566 ("unexpected event on data ingress queue: %x",
567 G_RSPD_TYPE(ctrl->u.type_gen)));
568
569 len = be32toh(ctrl->pldbuflen_qid);
570
571 KASSERT(len & F_RSPD_NEWBUF,
572 ("%s: T4 misconfigured to pack buffers.", __func__));
573
574 len = G_RSPD_LEN(len);
575 m0 = get_fl_sdesc_data(fl, len, M_PKTHDR);
576 if (m0 == NULL) {
577 iq->intr_next = V_QINTR_TIMER_IDX(SGE_NTIMERS - 1);
578 break;
579 }
580
581 len -= FL_PKTSHIFT;
582 m0->m_len -= FL_PKTSHIFT;
583 m0->m_data += FL_PKTSHIFT;
584
585 m0->m_pkthdr.len = len;
586 m0->m_pkthdr.rcvif = ifp;
--- 12 unchanged lines hidden (view full) ---
599 }
600
601 if (cpl->vlan_ex) {
602 m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
603 m0->m_flags |= M_VLANTAG;
604 rxq->vlan_extraction++;
605 }
606
607 len -= m0->m_len;
608 m = m0;
609 while (len) {
610 m->m_next = get_fl_sdesc_data(fl, len, 0);
611 if (m->m_next == NULL)
612 CXGBE_UNIMPLEMENTED("mbuf recovery");
613
614 m = m->m_next;
615 len -= m->m_len;
616 }
617#ifdef INET
618 if (cpl->l2info & htobe32(F_RXF_LRO) &&
619 rxq->flags & RXQ_LRO_ENABLED &&
620 tcp_lro_rx(lro, m0, 0) == 0) {
621 /* queued for LRO */
622 } else
623#endif
624 (*ifp->if_input)(ifp, m0);
625
626 FL_LOCK(fl);
627 if (fl->needed >= 32) {
628 refill_fl(fl, 64);
629 if (fl->pending >= 32)
630 ring_fl_db(sc, fl);
631 }
632 FL_UNLOCK(fl);
633
634nextdesc: ndescs++;
635 iq_next(iq);
636
637 if (ndescs > 32) {
638 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
639 V_CIDXINC(ndescs) |
640 V_INGRESSQID((u32)iq->cntxt_id) |
641 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
642 ndescs = 0;
643 }
644 }
645
646#ifdef INET
647 while (!SLIST_EMPTY(&lro->lro_active)) {
648 l = SLIST_FIRST(&lro->lro_active);
649 SLIST_REMOVE_HEAD(&lro->lro_active, next);
650 tcp_lro_flush(lro, l);
651 }
652#endif
653
654 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
655 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
656
657 IQ_UNLOCK(iq);
658
659 FL_LOCK(fl);
660 if (fl->needed) {
661 refill_fl(fl, -1);
662 if (fl->pending >= 8)
663 ring_fl_db(sc, fl);
664 }
665 FL_UNLOCK(fl);
666}
667
668/* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
669#define TXPKTS_PKT_HDR ((\
670 sizeof(struct ulp_txpkt) + \
671 sizeof(struct ulptx_idata) + \
672 sizeof(struct cpl_tx_pkt_core) \
--- 523 unchanged lines hidden (view full) ---
1196
1197 children = SYSCTL_CHILDREN(pi->oid_rxq);
1198
1199 snprintf(name, sizeof(name), "%d", idx);
1200 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
1201 NULL, "rx queue");
1202 children = SYSCTL_CHILDREN(oid);
1203
1204 SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
1205 &rxq->lro.lro_queued, 0, NULL);
1206 SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
1207 &rxq->lro.lro_flushed, 0, NULL);
1208 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
1209 &rxq->rxcsum, "# of times hardware assisted with checksum");
1210 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_extraction",
1211 CTLFLAG_RD, &rxq->vlan_extraction,
1212 "# of times hardware extracted 802.1Q tag");
1213
1214 return (rc);
1215}
--- 271 unchanged lines hidden (view full) ---
1487 }
1488
1489 tag = fl->tag[sd->tag_idx];
1490
1491 cl = m_cljget(NULL, M_NOWAIT, FL_BUF_SIZE(sd->tag_idx));
1492 if (cl == NULL)
1493 break;
1494
1495 rc = bus_dmamap_load(tag, sd->map, cl,
1496 FL_BUF_SIZE(sd->tag_idx), oneseg_dma_callback,
1497 &pa, 0);
1498 if (rc != 0 || pa == 0) {
1499 fl->dmamap_failed++;
1500 uma_zfree(FL_BUF_ZONE(sd->tag_idx), cl);
1501 break;
1502 }
1503
1504 sd->cl = cl;
1505 *d++ = htobe64(pa | sd->tag_idx);
1506
1507#ifdef INVARIANTS
1508 sd->ba_tag = htobe64(pa | sd->tag_idx);
1509#endif
1510
1511recycled: fl->pending++;
1512 fl->needed--;
1513 sd++;
1514 if (++fl->pidx == fl->cap) {
1515 fl->pidx = 0;
1516 sd = fl->sdesc;
1517 d = fl->desc;
1518 }
1519
1520 /* No harm if gethdr fails, we'll retry after rx */
1521 if (sd->m == NULL)
1522 sd->m = m_gethdr(M_NOWAIT, MT_NOINIT);
1523 }
1524}
1525
1526static int
1527alloc_fl_sdesc(struct sge_fl *fl)
1528{
1529 struct fl_sdesc *sd;
1530 bus_dma_tag_t tag;
--- 799 unchanged lines hidden (view full) ---
2330 return htobe64(sgl[i].ds_addr);
2331 case 2:
2332 return htobe64(sgl[i + 1].ds_addr);
2333 }
2334
2335 return (0);
2336}
2337
2338static struct mbuf *
2339get_fl_sdesc_data(struct sge_fl *fl, int len, int flags)
2340{
2341 struct fl_sdesc *sd;
2342 struct mbuf *m;
2343
2344 sd = &fl->sdesc[fl->cidx];
2345 FL_LOCK(fl);
2346 if (++fl->cidx == fl->cap)
2347 fl->cidx = 0;
2348 fl->needed++;
2349 FL_UNLOCK(fl);
2350
2351 m = sd->m;
2352 if (m == NULL) {
2353 m = m_gethdr(M_NOWAIT, MT_NOINIT);
2354 if (m == NULL)
2355 return (NULL);
2356 }
2357 sd->m = NULL; /* consumed */
2358
2359 bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD);
2360 m_init(m, zone_mbuf, MLEN, M_NOWAIT, MT_DATA, flags);
2361 if ((flags && len < MINCLSIZE) || (!flags && len <= MLEN))
2362 bcopy(sd->cl, mtod(m, caddr_t), len);
2363 else {
2364 bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
2365 m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx));
2366 sd->cl = NULL; /* consumed */
2367 }
2368
2369 m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
2370
2371 return (m);
2372}
2373
2374static void
2375set_fl_tag_idx(struct sge_fl *fl, int mtu)
2376{
2377 int i;
2378
2379 FL_LOCK_ASSERT_OWNED(fl);
2380
2381 for (i = 0; i < FL_BUF_SIZES - 1; i++) {
--- 22 unchanged lines hidden ---
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
--- 12 unchanged lines hidden (view full) ---
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_sge.c 219289 2011-03-05 03:27:14Z np $");
30
31#include "opt_inet.h"
32
33#include <sys/types.h>
34#include <sys/mbuf.h>
35#include <sys/socket.h>
36#include <sys/kernel.h>
37#include <sys/malloc.h>
--- 78 unchanged lines hidden (view full) ---
116static inline bool is_new_response(const struct sge_iq *, struct rsp_ctrl **);
117static inline void iq_next(struct sge_iq *);
118static inline void ring_fl_db(struct adapter *, struct sge_fl *);
119static void refill_fl(struct sge_fl *, int);
120static int alloc_fl_sdesc(struct sge_fl *);
121static void free_fl_sdesc(struct sge_fl *);
122static int alloc_eq_maps(struct sge_eq *);
123static void free_eq_maps(struct sge_eq *);
124static struct mbuf *get_fl_sdesc_data(struct sge_fl *, int, int);
125static void set_fl_tag_idx(struct sge_fl *, int);
126
127static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
128static int free_pkt_sgl(struct sge_txq *, struct sgl *);
129static int write_txpkt_wr(struct port_info *, struct sge_txq *, struct mbuf *,
130 struct sgl *);
131static int add_to_txpkts(struct port_info *, struct sge_txq *, struct txpkts *,
132 struct mbuf *, struct sgl *);
--- 395 unchanged lines hidden (view full) ---
528
529void
530t4_intr_data(void *arg)
531{
532 struct sge_rxq *rxq = arg;
533 struct sge_iq *iq = arg;
534 struct adapter *sc = iq->adapter;
535 struct rsp_ctrl *ctrl;
536 struct sge_fl *fl = &rxq->fl;
537 struct ifnet *ifp = rxq->ifp;
538 const struct rss_header *rss;
539 const struct cpl_rx_pkt *cpl;
540 int ndescs = 0, rsp_type;
541 uint32_t len;
542 struct mbuf *m0, *m;
543#ifdef INET
544 struct lro_ctrl *lro = &rxq->lro;
545 struct lro_entry *l;
546#endif
547
548 IQ_LOCK(iq);
549 iq->intr_next = iq->intr_params;
550 while (is_new_response(iq, &ctrl)) {
551
552 rmb();
553
554 rss = (const void *)iq->cdesc;
555 cpl = (const void *)(rss + 1);
556
557 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
558
559 if (__predict_false(rsp_type == X_RSPD_TYPE_CPL)) {
560 /* Can't be anything except an egress update */
561 handle_sge_egr_update(sc, (const void *)cpl);
562 goto nextdesc;
563 }
564
565 KASSERT(G_RSPD_TYPE(ctrl->u.type_gen) == X_RSPD_TYPE_FLBUF,
566 ("unexpected event on data ingress queue: %x",
567 G_RSPD_TYPE(ctrl->u.type_gen)));
568
569 len = be32toh(ctrl->pldbuflen_qid);
570
571 KASSERT(len & F_RSPD_NEWBUF,
572 ("%s: T4 misconfigured to pack buffers.", __func__));
573
574 len = G_RSPD_LEN(len);
575 m0 = get_fl_sdesc_data(fl, len, M_PKTHDR);
576 if (m0 == NULL) {
577 iq->intr_next = V_QINTR_TIMER_IDX(SGE_NTIMERS - 1);
578 break;
579 }
580
581 len -= FL_PKTSHIFT;
582 m0->m_len -= FL_PKTSHIFT;
583 m0->m_data += FL_PKTSHIFT;
584
585 m0->m_pkthdr.len = len;
586 m0->m_pkthdr.rcvif = ifp;
--- 12 unchanged lines hidden (view full) ---
599 }
600
601 if (cpl->vlan_ex) {
602 m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
603 m0->m_flags |= M_VLANTAG;
604 rxq->vlan_extraction++;
605 }
606
607 len -= m0->m_len;
608 m = m0;
609 while (len) {
610 m->m_next = get_fl_sdesc_data(fl, len, 0);
611 if (m->m_next == NULL)
612 CXGBE_UNIMPLEMENTED("mbuf recovery");
613
614 m = m->m_next;
615 len -= m->m_len;
616 }
617#ifdef INET
618 if (cpl->l2info & htobe32(F_RXF_LRO) &&
619 rxq->flags & RXQ_LRO_ENABLED &&
620 tcp_lro_rx(lro, m0, 0) == 0) {
621 /* queued for LRO */
622 } else
623#endif
624 (*ifp->if_input)(ifp, m0);
625
626 FL_LOCK(fl);
627 if (fl->needed >= 32) {
628 refill_fl(fl, 64);
629 if (fl->pending >= 32)
630 ring_fl_db(sc, fl);
631 }
632 FL_UNLOCK(fl);
633
634nextdesc: ndescs++;
635 iq_next(iq);
636
637 if (ndescs > 32) {
638 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
639 V_CIDXINC(ndescs) |
640 V_INGRESSQID((u32)iq->cntxt_id) |
641 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
642 ndescs = 0;
643 }
644 }
645
646#ifdef INET
647 while (!SLIST_EMPTY(&lro->lro_active)) {
648 l = SLIST_FIRST(&lro->lro_active);
649 SLIST_REMOVE_HEAD(&lro->lro_active, next);
650 tcp_lro_flush(lro, l);
651 }
652#endif
653
654 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
655 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
656
657 IQ_UNLOCK(iq);
658
659 FL_LOCK(fl);
660 if (fl->needed) {
661 refill_fl(fl, -1);
662 if (fl->pending >= 8)
663 ring_fl_db(sc, fl);
664 }
665 FL_UNLOCK(fl);
666}
667
668/* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
669#define TXPKTS_PKT_HDR ((\
670 sizeof(struct ulp_txpkt) + \
671 sizeof(struct ulptx_idata) + \
672 sizeof(struct cpl_tx_pkt_core) \
--- 523 unchanged lines hidden (view full) ---
1196
1197 children = SYSCTL_CHILDREN(pi->oid_rxq);
1198
1199 snprintf(name, sizeof(name), "%d", idx);
1200 oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
1201 NULL, "rx queue");
1202 children = SYSCTL_CHILDREN(oid);
1203
1204 SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
1205 &rxq->lro.lro_queued, 0, NULL);
1206 SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
1207 &rxq->lro.lro_flushed, 0, NULL);
1208 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
1209 &rxq->rxcsum, "# of times hardware assisted with checksum");
1210 SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_extraction",
1211 CTLFLAG_RD, &rxq->vlan_extraction,
1212 "# of times hardware extracted 802.1Q tag");
1213
1214 return (rc);
1215}
--- 271 unchanged lines hidden (view full) ---
1487 }
1488
1489 tag = fl->tag[sd->tag_idx];
1490
1491 cl = m_cljget(NULL, M_NOWAIT, FL_BUF_SIZE(sd->tag_idx));
1492 if (cl == NULL)
1493 break;
1494
1495 rc = bus_dmamap_load(tag, sd->map, cl,
1496 FL_BUF_SIZE(sd->tag_idx), oneseg_dma_callback,
1497 &pa, 0);
1498 if (rc != 0 || pa == 0) {
1499 fl->dmamap_failed++;
1500 uma_zfree(FL_BUF_ZONE(sd->tag_idx), cl);
1501 break;
1502 }
1503
1504 sd->cl = cl;
1505 *d++ = htobe64(pa | sd->tag_idx);
1506
1507#ifdef INVARIANTS
1508 sd->ba_tag = htobe64(pa | sd->tag_idx);
1509#endif
1510
1511recycled: fl->pending++;
1512 fl->needed--;
1513 sd++;
1514 if (++fl->pidx == fl->cap) {
1515 fl->pidx = 0;
1516 sd = fl->sdesc;
1517 d = fl->desc;
1518 }
1519
1520 /* No harm if gethdr fails, we'll retry after rx */
1521 if (sd->m == NULL)
1522 sd->m = m_gethdr(M_NOWAIT, MT_NOINIT);
1523 }
1524}
1525
1526static int
1527alloc_fl_sdesc(struct sge_fl *fl)
1528{
1529 struct fl_sdesc *sd;
1530 bus_dma_tag_t tag;
--- 799 unchanged lines hidden (view full) ---
2330 return htobe64(sgl[i].ds_addr);
2331 case 2:
2332 return htobe64(sgl[i + 1].ds_addr);
2333 }
2334
2335 return (0);
2336}
2337
2338static struct mbuf *
2339get_fl_sdesc_data(struct sge_fl *fl, int len, int flags)
2340{
2341 struct fl_sdesc *sd;
2342 struct mbuf *m;
2343
2344 sd = &fl->sdesc[fl->cidx];
2345 FL_LOCK(fl);
2346 if (++fl->cidx == fl->cap)
2347 fl->cidx = 0;
2348 fl->needed++;
2349 FL_UNLOCK(fl);
2350
2351 m = sd->m;
2352 if (m == NULL) {
2353 m = m_gethdr(M_NOWAIT, MT_NOINIT);
2354 if (m == NULL)
2355 return (NULL);
2356 }
2357 sd->m = NULL; /* consumed */
2358
2359 bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD);
2360 m_init(m, zone_mbuf, MLEN, M_NOWAIT, MT_DATA, flags);
2361 if ((flags && len < MINCLSIZE) || (!flags && len <= MLEN))
2362 bcopy(sd->cl, mtod(m, caddr_t), len);
2363 else {
2364 bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
2365 m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx));
2366 sd->cl = NULL; /* consumed */
2367 }
2368
2369 m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
2370
2371 return (m);
2372}
2373
2374static void
2375set_fl_tag_idx(struct sge_fl *fl, int mtu)
2376{
2377 int i;
2378
2379 FL_LOCK_ASSERT_OWNED(fl);
2380
2381 for (i = 0; i < FL_BUF_SIZES - 1; i++) {
--- 22 unchanged lines hidden ---