Deleted Added
sdiff udiff text old ( 309558 ) new ( 309560 )
full compact
t4_sge.c (309558) t4_sge.c (309560)
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/t4_sge.c 309558 2016-12-05 19:34:52Z jhb $");
29__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/t4_sge.c 309560 2016-12-05 20:43:25Z jhb $");
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/types.h>
35#include <sys/eventhandler.h>
36#include <sys/mbuf.h>
37#include <sys/socket.h>

--- 186 unchanged lines hidden (view full) ---

224
225static inline void get_pkt_gl(struct mbuf *, struct sglist *);
226static inline u_int txpkt_len16(u_int, u_int);
227static inline u_int txpkt_vm_len16(u_int, u_int);
228static inline u_int txpkts0_len16(u_int);
229static inline u_int txpkts1_len16(void);
230static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *,
231 struct mbuf *, u_int);
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#include <sys/types.h>
35#include <sys/eventhandler.h>
36#include <sys/mbuf.h>
37#include <sys/socket.h>

--- 186 unchanged lines hidden (view full) ---

224
225static inline void get_pkt_gl(struct mbuf *, struct sglist *);
226static inline u_int txpkt_len16(u_int, u_int);
227static inline u_int txpkt_vm_len16(u_int, u_int);
228static inline u_int txpkts0_len16(u_int);
229static inline u_int txpkts1_len16(void);
230static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *,
231 struct mbuf *, u_int);
232static u_int write_txpkt_vm_wr(struct sge_txq *, struct fw_eth_tx_pkt_vm_wr *,
233 struct mbuf *, u_int);
232static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *,
233 struct fw_eth_tx_pkt_vm_wr *, struct mbuf *, u_int);
234static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int);
235static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int);
236static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *,
237 struct mbuf *, const struct txpkts *, u_int);
238static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
239static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
240static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
241static inline uint16_t read_hw_cidx(struct sge_eq *);

--- 186 unchanged lines hidden (view full) ---

428
429 return (refs - rels);
430}
431
432static inline void
433setup_pad_and_pack_boundaries(struct adapter *sc)
434{
435 uint32_t v, m;
234static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int);
235static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int);
236static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *,
237 struct mbuf *, const struct txpkts *, u_int);
238static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
239static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
240static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
241static inline uint16_t read_hw_cidx(struct sge_eq *);

--- 186 unchanged lines hidden (view full) ---

428
429 return (refs - rels);
430}
431
432static inline void
433setup_pad_and_pack_boundaries(struct adapter *sc)
434{
435 uint32_t v, m;
436 int pad, pack;
436 int pad, pack, pad_shift;
437
437
438 pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT :
439 X_INGPADBOUNDARY_SHIFT;
438 pad = fl_pad;
440 pad = fl_pad;
439 if (fl_pad < 32 || fl_pad > 4096 || !powerof2(fl_pad)) {
441 if (fl_pad < (1 << pad_shift) ||
442 fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) ||
443 !powerof2(fl_pad)) {
440 /*
441 * If there is any chance that we might use buffer packing and
442 * the chip is a T4, then pick 64 as the pad/pack boundary. Set
444 /*
445 * If there is any chance that we might use buffer packing and
446 * the chip is a T4, then pick 64 as the pad/pack boundary. Set
443 * it to 32 in all other cases.
447 * it to the minimum allowed in all other cases.
444 */
448 */
445 pad = is_t4(sc) && buffer_packing ? 64 : 32;
449 pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift;
446
447 /*
448 * For fl_pad = 0 we'll still write a reasonable value to the
449 * register but all the freelists will opt out of padding.
450 * We'll complain here only if the user tried to set it to a
451 * value greater than 0 that was invalid.
452 */
453 if (fl_pad > 0) {
454 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value"
455 " (%d), using %d instead.\n", fl_pad, pad);
456 }
457 }
458 m = V_INGPADBOUNDARY(M_INGPADBOUNDARY);
450
451 /*
452 * For fl_pad = 0 we'll still write a reasonable value to the
453 * register but all the freelists will opt out of padding.
454 * We'll complain here only if the user tried to set it to a
455 * value greater than 0 that was invalid.
456 */
457 if (fl_pad > 0) {
458 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value"
459 " (%d), using %d instead.\n", fl_pad, pad);
460 }
461 }
462 m = V_INGPADBOUNDARY(M_INGPADBOUNDARY);
459 v = V_INGPADBOUNDARY(ilog2(pad) - 5);
463 v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift);
460 t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
461
462 if (is_t4(sc)) {
463 if (fl_pack != -1 && fl_pack != pad) {
464 /* Complain but carry on. */
465 device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored,"
466 " using %d instead.\n", fl_pack, pad);
467 }

--- 1979 unchanged lines hidden (view full) ---

2447 if (__predict_false(next_cidx == r->size))
2448 next_cidx = 0;
2449
2450 wr = (void *)&eq->desc[eq->pidx];
2451 if (sc->flags & IS_VF) {
2452 total++;
2453 remaining--;
2454 ETHER_BPF_MTAP(ifp, m0);
464 t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
465
466 if (is_t4(sc)) {
467 if (fl_pack != -1 && fl_pack != pad) {
468 /* Complain but carry on. */
469 device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored,"
470 " using %d instead.\n", fl_pack, pad);
471 }

--- 1979 unchanged lines hidden (view full) ---

2451 if (__predict_false(next_cidx == r->size))
2452 next_cidx = 0;
2453
2454 wr = (void *)&eq->desc[eq->pidx];
2455 if (sc->flags & IS_VF) {
2456 total++;
2457 remaining--;
2458 ETHER_BPF_MTAP(ifp, m0);
2455 n = write_txpkt_vm_wr(txq, (void *)wr, m0, available);
2459 n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0,
2460 available);
2456 } else if (remaining > 1 &&
2457 try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) {
2458
2459 /* pkts at cidx, next_cidx should both be in txp. */
2460 MPASS(txp.npkt == 2);
2461 tail = r->items[next_cidx];
2462 MPASS(tail->m_nextpkt == NULL);
2463 ETHER_BPF_MTAP(ifp, m0);

--- 251 unchanged lines hidden (view full) ---

2715 0));
2716 if (cong >= 0) {
2717 c.iqns_to_fl0congen |=
2718 htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
2719 F_FW_IQ_CMD_FL0CONGCIF |
2720 F_FW_IQ_CMD_FL0CONGEN);
2721 }
2722 c.fl0dcaen_to_fl0cidxfthresh =
2461 } else if (remaining > 1 &&
2462 try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) {
2463
2464 /* pkts at cidx, next_cidx should both be in txp. */
2465 MPASS(txp.npkt == 2);
2466 tail = r->items[next_cidx];
2467 MPASS(tail->m_nextpkt == NULL);
2468 ETHER_BPF_MTAP(ifp, m0);

--- 251 unchanged lines hidden (view full) ---

2720 0));
2721 if (cong >= 0) {
2722 c.iqns_to_fl0congen |=
2723 htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
2724 F_FW_IQ_CMD_FL0CONGCIF |
2725 F_FW_IQ_CMD_FL0CONGEN);
2726 }
2727 c.fl0dcaen_to_fl0cidxfthresh =
2723 htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_128B) |
2724 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
2728 htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ?
2729 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) |
2730 V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ?
2731 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
2725 c.fl0size = htobe16(fl->qsize);
2726 c.fl0addr = htobe64(fl->ba);
2727 }
2728
2729 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2730 if (rc != 0) {
2731 device_printf(sc->dev,
2732 "failed to create ingress queue: %d\n", rc);

--- 46 unchanged lines hidden (view full) ---

2779 fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
2780
2781 FL_LOCK(fl);
2782 /* Enough to make sure the SGE doesn't think it's starved */
2783 refill_fl(sc, fl, fl->lowat);
2784 FL_UNLOCK(fl);
2785 }
2786
2732 c.fl0size = htobe16(fl->qsize);
2733 c.fl0addr = htobe64(fl->ba);
2734 }
2735
2736 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2737 if (rc != 0) {
2738 device_printf(sc->dev,
2739 "failed to create ingress queue: %d\n", rc);

--- 46 unchanged lines hidden (view full) ---

2786 fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
2787
2788 FL_LOCK(fl);
2789 /* Enough to make sure the SGE doesn't think it's starved */
2790 refill_fl(sc, fl, fl->lowat);
2791 FL_UNLOCK(fl);
2792 }
2793
2787 if (is_t5(sc) && !(sc->flags & IS_VF) && cong >= 0) {
2794 if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && cong >= 0) {
2788 uint32_t param, val;
2789
2790 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2791 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
2792 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
2793 if (cong == 0)
2794 val = 1 << 19;
2795 else {

--- 437 unchanged lines hidden (view full) ---

3233 if (rc)
3234 return (rc);
3235
3236 nm_txq->pidx = nm_txq->cidx = 0;
3237 nm_txq->sidx = na->num_tx_desc;
3238 nm_txq->nid = idx;
3239 nm_txq->iqidx = iqidx;
3240 nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2795 uint32_t param, val;
2796
2797 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2798 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
2799 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
2800 if (cong == 0)
2801 val = 1 << 19;
2802 else {

--- 437 unchanged lines hidden (view full) ---

3240 if (rc)
3241 return (rc);
3242
3243 nm_txq->pidx = nm_txq->cidx = 0;
3244 nm_txq->sidx = na->num_tx_desc;
3245 nm_txq->nid = idx;
3246 nm_txq->iqidx = iqidx;
3247 nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3241 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_VF_VLD(1) |
3242 V_TXPKT_VF(vi->viid));
3248 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) |
3249 V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) |
3250 V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid)));
3243
3244 snprintf(name, sizeof(name), "%d", idx);
3245 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3246 NULL, "netmap tx queue");
3247 children = SYSCTL_CHILDREN(oid);
3248
3249 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3250 &nm_txq->cntxt_id, 0, "SGE context id of the queue");

--- 351 unchanged lines hidden (view full) ---

3602 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
3603 txq->ifp = vi->ifp;
3604 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
3605 if (sc->flags & IS_VF)
3606 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
3607 V_TXPKT_INTF(pi->tx_chan));
3608 else
3609 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3251
3252 snprintf(name, sizeof(name), "%d", idx);
3253 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3254 NULL, "netmap tx queue");
3255 children = SYSCTL_CHILDREN(oid);
3256
3257 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3258 &nm_txq->cntxt_id, 0, "SGE context id of the queue");

--- 351 unchanged lines hidden (view full) ---

3610 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
3611 txq->ifp = vi->ifp;
3612 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
3613 if (sc->flags & IS_VF)
3614 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
3615 V_TXPKT_INTF(pi->tx_chan));
3616 else
3617 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3610 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_VF_VLD(1) |
3611 V_TXPKT_VF(vi->viid));
3618 V_TXPKT_INTF(pi->tx_chan) |
3619 V_TXPKT_PF(G_FW_VIID_PFN(vi->viid)) |
3620 V_TXPKT_VF(G_FW_VIID_VIN(vi->viid)) |
3621 V_TXPKT_VF_VLD(G_FW_VIID_VIVLD(vi->viid)));
3612 txq->tc_idx = -1;
3613 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
3614 M_ZERO | M_WAITOK);
3615
3616 snprintf(name, sizeof(name), "%d", idx);
3617 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3618 NULL, "tx queue");
3619 children = SYSCTL_CHILDREN(oid);

--- 408 unchanged lines hidden (view full) ---

4028/*
4029 * Write a VM txpkt WR for this packet to the hardware descriptors, update the
4030 * software descriptor, and advance the pidx. It is guaranteed that enough
4031 * descriptors are available.
4032 *
4033 * The return value is the # of hardware descriptors used.
4034 */
4035static u_int
3622 txq->tc_idx = -1;
3623 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
3624 M_ZERO | M_WAITOK);
3625
3626 snprintf(name, sizeof(name), "%d", idx);
3627 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3628 NULL, "tx queue");
3629 children = SYSCTL_CHILDREN(oid);

--- 408 unchanged lines hidden (view full) ---

4038/*
4039 * Write a VM txpkt WR for this packet to the hardware descriptors, update the
4040 * software descriptor, and advance the pidx. It is guaranteed that enough
4041 * descriptors are available.
4042 *
4043 * The return value is the # of hardware descriptors used.
4044 */
4045static u_int
4036write_txpkt_vm_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_vm_wr *wr,
4037 struct mbuf *m0, u_int available)
4046write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq,
4047 struct fw_eth_tx_pkt_vm_wr *wr, struct mbuf *m0, u_int available)
4038{
4039 struct sge_eq *eq = &txq->eq;
4040 struct tx_sdesc *txsd;
4041 struct cpl_tx_pkt_core *cpl;
4042 uint32_t ctrl; /* used in many unrelated places */
4043 uint64_t ctrl1;
4044 int csum_type, len16, ndesc, pktlen, nsegs;
4045 caddr_t dst;

--- 99 unchanged lines hidden (view full) ---

4145 ctrl1 = 0;
4146 if (needs_l3_csum(m0) == 0)
4147 ctrl1 |= F_TXPKT_IPCSUM_DIS;
4148 if (csum_type >= 0) {
4149 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0,
4150 ("%s: mbuf %p needs checksum offload but missing header lengths",
4151 __func__, m0));
4152
4048{
4049 struct sge_eq *eq = &txq->eq;
4050 struct tx_sdesc *txsd;
4051 struct cpl_tx_pkt_core *cpl;
4052 uint32_t ctrl; /* used in many unrelated places */
4053 uint64_t ctrl1;
4054 int csum_type, len16, ndesc, pktlen, nsegs;
4055 caddr_t dst;

--- 99 unchanged lines hidden (view full) ---

4155 ctrl1 = 0;
4156 if (needs_l3_csum(m0) == 0)
4157 ctrl1 |= F_TXPKT_IPCSUM_DIS;
4158 if (csum_type >= 0) {
4159 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0,
4160 ("%s: mbuf %p needs checksum offload but missing header lengths",
4161 __func__, m0));
4162
4153 /* XXX: T6 */
4154 ctrl1 |= V_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen -
4155 ETHER_HDR_LEN);
4163 if (chip_id(sc) <= CHELSIO_T5) {
4164 ctrl1 |= V_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen -
4165 ETHER_HDR_LEN);
4166 } else {
4167 ctrl1 |= V_T6_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen -
4168 ETHER_HDR_LEN);
4169 }
4156 ctrl1 |= V_TXPKT_IPHDR_LEN(m0->m_pkthdr.l3hlen);
4157 ctrl1 |= V_TXPKT_CSUM_TYPE(csum_type);
4158 } else
4159 ctrl1 |= F_TXPKT_L4CSUM_DIS;
4160 if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
4161 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
4162 txq->txcsum++; /* some hardware assistance provided */
4163

--- 1041 unchanged lines hidden --- 		4170 ctrl1 |= V_TXPKT_IPHDR_LEN(m0->m_pkthdr.l3hlen);
4171 ctrl1 |= V_TXPKT_CSUM_TYPE(csum_type);
4172 } else
4173 ctrl1 |= F_TXPKT_L4CSUM_DIS;
4174 if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
4175 CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
4176 txq->txcsum++; /* some hardware assistance provided */
4177

--- 1041 unchanged lines hidden ---