Deleted Added
sdiff udiff text old ( 274449 ) new ( 281253 )
full compact
t4_netmap.c (274449) t4_netmap.c (281253)
1/*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_netmap.c 274449 2014-11-12 20:37:23Z np $");
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_netmap.c 281253 2015-04-08 01:43:29Z np $");
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#ifdef DEV_NETMAP
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/lock.h>

--- 15 unchanged lines hidden (view full) ---

53#include "common/common.h"
54#include "common/t4_regs.h"
55#include "common/t4_regs_values.h"
56
57extern int fl_pad; /* XXXNM */
58extern int spg_len; /* XXXNM */
59extern int fl_pktshift; /* XXXNM */
60
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#ifdef DEV_NETMAP
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/lock.h>

--- 15 unchanged lines hidden (view full) ---

53#include "common/common.h"
54#include "common/t4_regs.h"
55#include "common/t4_regs_values.h"
56
57extern int fl_pad; /* XXXNM */
58extern int spg_len; /* XXXNM */
59extern int fl_pktshift; /* XXXNM */
60
61SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0, "cxgbe netmap parameters");
62
63/*
64 * 0 = normal netmap rx
65 * 1 = black hole
66 * 2 = supermassive black hole (buffer packing enabled)
67 */
68int black_hole = 0;
69SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_black_hole, CTLFLAG_RDTUN, &black_hole, 0,
70 "Sink incoming packets.");
71
72int rx_ndesc = 256;
73SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_rx_ndesc, CTLFLAG_RWTUN,
74 &rx_ndesc, 0, "# of rx descriptors after which the hw cidx is updated.");
75
76int holdoff_tmr_idx = 2;
77SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_holdoff_tmr_idx, CTLFLAG_RWTUN,
78 &holdoff_tmr_idx, 0, "Holdoff timer index for netmap rx queues.");
79
61/* netmap ifnet routines */
62static void cxgbe_nm_init(void *);
63static int cxgbe_nm_ioctl(struct ifnet *, unsigned long, caddr_t);
64static int cxgbe_nm_transmit(struct ifnet *, struct mbuf *);
65static void cxgbe_nm_qflush(struct ifnet *);
66
67static int cxgbe_nm_init_synchronized(struct port_info *);
68static int cxgbe_nm_uninit_synchronized(struct port_info *);

--- 152 unchanged lines hidden (view full) ---

221static void
222cxgbe_nm_qflush(struct ifnet *ifp)
223{
224
225 return;
226}
227
228static int
80/* netmap ifnet routines */
81static void cxgbe_nm_init(void *);
82static int cxgbe_nm_ioctl(struct ifnet *, unsigned long, caddr_t);
83static int cxgbe_nm_transmit(struct ifnet *, struct mbuf *);
84static void cxgbe_nm_qflush(struct ifnet *);
85
86static int cxgbe_nm_init_synchronized(struct port_info *);
87static int cxgbe_nm_uninit_synchronized(struct port_info *);

--- 152 unchanged lines hidden (view full) ---

240static void
241cxgbe_nm_qflush(struct ifnet *ifp)
242{
243
244 return;
245}
246
247static int
229alloc_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
248alloc_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq, int cong)
230{
249{
231 int rc, cntxt_id;
250 int rc, cntxt_id, i;
232 __be32 v;
233 struct adapter *sc = pi->adapter;
234 struct netmap_adapter *na = NA(pi->nm_ifp);
235 struct fw_iq_cmd c;
236
237 MPASS(na != NULL);
238 MPASS(nm_rxq->iq_desc != NULL);
239 MPASS(nm_rxq->fl_desc != NULL);

--- 22 unchanged lines hidden (view full) ---

262 V_FW_IQ_CMD_VIID(pi->nm_viid) |
263 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
264 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
265 F_FW_IQ_CMD_IQGTSMODE |
266 V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
267 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
268 c.iqsize = htobe16(pi->qsize_rxq);
269 c.iqaddr = htobe64(nm_rxq->iq_ba);
251 __be32 v;
252 struct adapter *sc = pi->adapter;
253 struct netmap_adapter *na = NA(pi->nm_ifp);
254 struct fw_iq_cmd c;
255
256 MPASS(na != NULL);
257 MPASS(nm_rxq->iq_desc != NULL);
258 MPASS(nm_rxq->fl_desc != NULL);

--- 22 unchanged lines hidden (view full) ---

281 V_FW_IQ_CMD_VIID(pi->nm_viid) |
282 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
283 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
284 F_FW_IQ_CMD_IQGTSMODE |
285 V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
286 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
287 c.iqsize = htobe16(pi->qsize_rxq);
288 c.iqaddr = htobe64(nm_rxq->iq_ba);
289 if (cong >= 0) {
290 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN |
291 V_FW_IQ_CMD_FL0CNGCHMAP(cong) | F_FW_IQ_CMD_FL0CONGCIF |
292 F_FW_IQ_CMD_FL0CONGEN);
293 }
270 c.iqns_to_fl0congen |=
271 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
272 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
294 c.iqns_to_fl0congen |=
295 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
296 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
273 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0));
297 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
298 (black_hole == 2 ? F_FW_IQ_CMD_FL0PACKEN : 0));
274 c.fl0dcaen_to_fl0cidxfthresh =
275 htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
276 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
299 c.fl0dcaen_to_fl0cidxfthresh =
300 htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
301 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
277 c.fl0size = htobe16(na->num_rx_desc + spg_len / EQ_ESIZE);
302 c.fl0size = htobe16(na->num_rx_desc / 8 + spg_len / EQ_ESIZE);
278 c.fl0addr = htobe64(nm_rxq->fl_ba);
279
280 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
281 if (rc != 0) {
282 device_printf(sc->dev,
283 "failed to create netmap ingress queue: %d\n", rc);
284 return (rc);
285 }

--- 19 unchanged lines hidden (view full) ---

305 __func__, cntxt_id, sc->sge.neq - 1);
306 }
307 sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;
308
309 nm_rxq->fl_db_val = F_DBPRIO | V_QID(nm_rxq->fl_cntxt_id) | V_PIDX(0);
310 if (is_t5(sc))
311 nm_rxq->fl_db_val |= F_DBTYPE;
312
303 c.fl0addr = htobe64(nm_rxq->fl_ba);
304
305 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
306 if (rc != 0) {
307 device_printf(sc->dev,
308 "failed to create netmap ingress queue: %d\n", rc);
309 return (rc);
310 }

--- 19 unchanged lines hidden (view full) ---

330 __func__, cntxt_id, sc->sge.neq - 1);
331 }
332 sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;
333
334 nm_rxq->fl_db_val = F_DBPRIO | V_QID(nm_rxq->fl_cntxt_id) | V_PIDX(0);
335 if (is_t5(sc))
336 nm_rxq->fl_db_val |= F_DBTYPE;
337
313 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(F_QINTR_CNT_EN) |
314 V_INGRESSQID(nm_rxq->iq_cntxt_id));
338 if (is_t5(sc) && cong >= 0) {
339 uint32_t param, val;
315
340
341 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
342 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
343 V_FW_PARAMS_PARAM_YZ(nm_rxq->iq_cntxt_id);
344 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
345 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
346 V_FW_PARAMS_PARAM_YZ(nm_rxq->iq_cntxt_id);
347 if (cong == 0)
348 val = 1 << 19;
349 else {
350 val = 2 << 19;
351 for (i = 0; i < 4; i++) {
352 if (cong & (1 << i))
353 val |= 1 << (i << 2);
354 }
355 }
356
357 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
358 if (rc != 0) {
359 /* report error but carry on */
360 device_printf(sc->dev,
361 "failed to set congestion manager context for "
362 "ingress queue %d: %d\n", nm_rxq->iq_cntxt_id, rc);
363 }
364 }
365
366 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
367 V_INGRESSQID(nm_rxq->iq_cntxt_id) |
368 V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx)));
369
316 return (rc);
317}
318
319static int
320free_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
321{
322 struct adapter *sc = pi->adapter;
323 int rc;

--- 22 unchanged lines hidden (view full) ---

346 bzero(nm_txq->desc, len);
347
348 bzero(&c, sizeof(c));
349 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
350 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
351 V_FW_EQ_ETH_CMD_VFN(0));
352 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
353 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
370 return (rc);
371}
372
373static int
374free_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
375{
376 struct adapter *sc = pi->adapter;
377 int rc;

--- 22 unchanged lines hidden (view full) ---

400 bzero(nm_txq->desc, len);
401
402 bzero(&c, sizeof(c));
403 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
404 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
405 V_FW_EQ_ETH_CMD_VFN(0));
406 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
407 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
354 c.autoequiqe_to_viid = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->nm_viid));
408 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUEQE |
409 V_FW_EQ_ETH_CMD_VIID(pi->nm_viid));
355 c.fetchszm_to_iqid =
356 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
357 V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
358 V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
359 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
360 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
361 V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
362 c.eqaddr = htobe64(nm_txq->ba);

--- 80 unchanged lines hidden (view full) ---

443 return (ENXIO);
444 }
445 hwidx = i;
446
447 /* Must set caps before calling netmap_reset */
448 nm_set_native_flags(na);
449
450 for_each_nm_rxq(pi, i, nm_rxq) {
410 c.fetchszm_to_iqid =
411 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
412 V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
413 V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
414 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
415 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
416 V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
417 c.eqaddr = htobe64(nm_txq->ba);

--- 80 unchanged lines hidden (view full) ---

498 return (ENXIO);
499 }
500 hwidx = i;
501
502 /* Must set caps before calling netmap_reset */
503 nm_set_native_flags(na);
504
505 for_each_nm_rxq(pi, i, nm_rxq) {
451 alloc_nm_rxq_hwq(pi, nm_rxq);
506 alloc_nm_rxq_hwq(pi, nm_rxq, tnl_cong(pi));
452 nm_rxq->fl_hwidx = hwidx;
453 slot = netmap_reset(na, NR_RX, i, 0);
454 MPASS(slot != NULL); /* XXXNM: error check, not assert */
455
456 /* We deal with 8 bufs at a time */
457 MPASS((na->num_rx_desc & 7) == 0);
458 MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
507 nm_rxq->fl_hwidx = hwidx;
508 slot = netmap_reset(na, NR_RX, i, 0);
509 MPASS(slot != NULL); /* XXXNM: error check, not assert */
510
511 /* We deal with 8 bufs at a time */
512 MPASS((na->num_rx_desc & 7) == 0);
513 MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
459 for (j = 0; j < nm_rxq->fl_sidx - 8; j++) {
514 for (j = 0; j < nm_rxq->fl_sidx; j++) {
460 uint64_t ba;
461
462 PNMB(na, &slot[j], &ba);
515 uint64_t ba;
516
517 PNMB(na, &slot[j], &ba);
518 MPASS(ba != 0);
463 nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
464 }
519 nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
520 }
465 nm_rxq->fl_pidx = j;
521 j = nm_rxq->fl_pidx = nm_rxq->fl_sidx - 8;
466 MPASS((j & 7) == 0);
467 j /= 8; /* driver pidx to hardware pidx */
468 wmb();
469 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
470 nm_rxq->fl_db_val | V_PIDX(j));
471 }
472
473 for_each_nm_txq(pi, i, nm_txq) {

--- 34 unchanged lines hidden (view full) ---

508
509 ASSERT_SYNCHRONIZED_OP(sc);
510
511 rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, false, false);
512 if (rc != 0)
513 if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
514 nm_clear_native_flags(na);
515
522 MPASS((j & 7) == 0);
523 j /= 8; /* driver pidx to hardware pidx */
524 wmb();
525 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
526 nm_rxq->fl_db_val | V_PIDX(j));
527 }
528
529 for_each_nm_txq(pi, i, nm_txq) {

--- 34 unchanged lines hidden (view full) ---

564
565 ASSERT_SYNCHRONIZED_OP(sc);
566
567 rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, false, false);
568 if (rc != 0)
569 if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
570 nm_clear_native_flags(na);
571
516 /*
517 * XXXNM: We need to make sure that the tx queues are quiet and won't
518 * request any more SGE_EGR_UPDATEs.
519 */
520
521 for_each_nm_txq(pi, i, nm_txq) {
572 for_each_nm_txq(pi, i, nm_txq) {
573 struct sge_qstat *spg = (void *)&nm_txq->desc[nm_txq->sidx];
574
575 /* Wait for hw pidx to catch up ... */
576 while (be16toh(nm_txq->pidx) != spg->pidx)
577 pause("nmpidx", 1);
578
579 /* ... and then for the cidx. */
580 while (spg->pidx != spg->cidx)
581 pause("nmcidx", 1);
582
522 free_nm_txq_hwq(pi, nm_txq);
523 }
524 for_each_nm_rxq(pi, i, nm_rxq) {
525 free_nm_rxq_hwq(pi, nm_rxq);
526 }
527
528 return (rc);
529}

--- 106 unchanged lines hidden (view full) ---

636int lazy_tx_credit_flush = 1;
637
638/*
639 * Write work requests to send 'npkt' frames and ring the doorbell to send them
640 * on their way. No need to check for wraparound.
641 */
642static void
643cxgbe_nm_tx(struct adapter *sc, struct sge_nm_txq *nm_txq,
583 free_nm_txq_hwq(pi, nm_txq);
584 }
585 for_each_nm_rxq(pi, i, nm_rxq) {
586 free_nm_rxq_hwq(pi, nm_rxq);
587 }
588
589 return (rc);
590}

--- 106 unchanged lines hidden (view full) ---

697int lazy_tx_credit_flush = 1;
698
699/*
700 * Write work requests to send 'npkt' frames and ring the doorbell to send them
701 * on their way. No need to check for wraparound.
702 */
703static void
704cxgbe_nm_tx(struct adapter *sc, struct sge_nm_txq *nm_txq,
644 struct netmap_kring *kring, int npkt, int npkt_remaining)
705 struct netmap_kring *kring, int npkt, int npkt_remaining, int txcsum)
645{
646 struct netmap_ring *ring = kring->ring;
647 struct netmap_slot *slot;
648 const u_int lim = kring->nkr_num_slots - 1;
649 struct fw_eth_tx_pkts_wr *wr = (void *)&nm_txq->desc[nm_txq->pidx];
650 uint16_t len;
651 uint64_t ba;
652 struct cpl_tx_pkt_core *cpl;

--- 10 unchanged lines hidden (view full) ---

663 wr->npkt = n;
664 wr->r3 = 0;
665 wr->type = 1;
666 cpl = (void *)(wr + 1);
667
668 for (i = 0; i < n; i++) {
669 slot = &ring->slot[kring->nr_hwcur];
670 PNMB(kring->na, slot, &ba);
706{
707 struct netmap_ring *ring = kring->ring;
708 struct netmap_slot *slot;
709 const u_int lim = kring->nkr_num_slots - 1;
710 struct fw_eth_tx_pkts_wr *wr = (void *)&nm_txq->desc[nm_txq->pidx];
711 uint16_t len;
712 uint64_t ba;
713 struct cpl_tx_pkt_core *cpl;

--- 10 unchanged lines hidden (view full) ---

724 wr->npkt = n;
725 wr->r3 = 0;
726 wr->type = 1;
727 cpl = (void *)(wr + 1);
728
729 for (i = 0; i < n; i++) {
730 slot = &ring->slot[kring->nr_hwcur];
731 PNMB(kring->na, slot, &ba);
732 MPASS(ba != 0);
671
672 cpl->ctrl0 = nm_txq->cpl_ctrl0;
673 cpl->pack = 0;
674 cpl->len = htobe16(slot->len);
675 /*
676 * netmap(4) says "netmap does not use features such as
677 * checksum offloading, TCP segmentation offloading,
678 * encryption, VLAN encapsulation/decapsulation, etc."
679 *
733
734 cpl->ctrl0 = nm_txq->cpl_ctrl0;
735 cpl->pack = 0;
736 cpl->len = htobe16(slot->len);
737 /*
738 * netmap(4) says "netmap does not use features such as
739 * checksum offloading, TCP segmentation offloading,
740 * encryption, VLAN encapsulation/decapsulation, etc."
741 *
680 * XXXNM: it makes sense to enable checksum offload.
742 * So the ncxl interfaces have tx hardware checksumming
743 * disabled by default. But you can override netmap by
744 * enabling IFCAP_TXCSUM on the interface manully.
681 */
745 */
682 cpl->ctrl1 = htobe64(F_TXPKT_IPCSUM_DIS |
683 F_TXPKT_L4CSUM_DIS);
746 cpl->ctrl1 = txcsum ? 0 :
747 htobe64(F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS);
684
685 usgl = (void *)(cpl + 1);
686 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
687 V_ULPTX_NSGE(1));
688 usgl->len0 = htobe32(slot->len);
689 usgl->addr0 = htobe64(ba);
690
691 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);

--- 96 unchanged lines hidden (view full) ---

788{
789 struct netmap_adapter *na = kring->na;
790 struct ifnet *ifp = na->ifp;
791 struct port_info *pi = ifp->if_softc;
792 struct adapter *sc = pi->adapter;
793 struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[pi->first_nm_txq + kring->ring_id];
794 const u_int head = kring->rhead;
795 u_int reclaimed = 0;
748
749 usgl = (void *)(cpl + 1);
750 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
751 V_ULPTX_NSGE(1));
752 usgl->len0 = htobe32(slot->len);
753 usgl->addr0 = htobe64(ba);
754
755 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);

--- 96 unchanged lines hidden (view full) ---

852{
853 struct netmap_adapter *na = kring->na;
854 struct ifnet *ifp = na->ifp;
855 struct port_info *pi = ifp->if_softc;
856 struct adapter *sc = pi->adapter;
857 struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[pi->first_nm_txq + kring->ring_id];
858 const u_int head = kring->rhead;
859 u_int reclaimed = 0;
796 int n, d, npkt_remaining, ndesc_remaining;
860 int n, d, npkt_remaining, ndesc_remaining, txcsum;
797
798 /*
799 * Tx was at kring->nr_hwcur last time around and now we need to advance
800 * to kring->rhead. Note that the driver's pidx moves independent of
801 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation
802 * between descriptors and frames isn't 1:1).
803 */
804
805 npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
806 kring->nkr_num_slots - kring->nr_hwcur + head;
861
862 /*
863 * Tx was at kring->nr_hwcur last time around and now we need to advance
864 * to kring->rhead. Note that the driver's pidx moves independent of
865 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation
866 * between descriptors and frames isn't 1:1).
867 */
868
869 npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
870 kring->nkr_num_slots - kring->nr_hwcur + head;
871 txcsum = ifp->if_capenable & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6);
807 while (npkt_remaining) {
808 reclaimed += reclaim_nm_tx_desc(nm_txq);
809 ndesc_remaining = contiguous_ndesc_available(nm_txq);
810 /* Can't run out of descriptors with packets still remaining */
811 MPASS(ndesc_remaining > 0);
812
813 /* # of desc needed to tx all remaining packets */
814 d = (npkt_remaining / MAX_NPKT_IN_TYPE1_WR) * SGE_MAX_WR_NDESC;

--- 7 unchanged lines hidden (view full) ---

822 n = (ndesc_remaining / SGE_MAX_WR_NDESC) *
823 MAX_NPKT_IN_TYPE1_WR;
824 if (ndesc_remaining % SGE_MAX_WR_NDESC)
825 n += ndesc_to_npkt(ndesc_remaining % SGE_MAX_WR_NDESC);
826 }
827
828 /* Send n packets and update nm_txq->pidx and kring->nr_hwcur */
829 npkt_remaining -= n;
872 while (npkt_remaining) {
873 reclaimed += reclaim_nm_tx_desc(nm_txq);
874 ndesc_remaining = contiguous_ndesc_available(nm_txq);
875 /* Can't run out of descriptors with packets still remaining */
876 MPASS(ndesc_remaining > 0);
877
878 /* # of desc needed to tx all remaining packets */
879 d = (npkt_remaining / MAX_NPKT_IN_TYPE1_WR) * SGE_MAX_WR_NDESC;

--- 7 unchanged lines hidden (view full) ---

887 n = (ndesc_remaining / SGE_MAX_WR_NDESC) *
888 MAX_NPKT_IN_TYPE1_WR;
889 if (ndesc_remaining % SGE_MAX_WR_NDESC)
890 n += ndesc_to_npkt(ndesc_remaining % SGE_MAX_WR_NDESC);
891 }
892
893 /* Send n packets and update nm_txq->pidx and kring->nr_hwcur */
894 npkt_remaining -= n;
830 cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining);
895 cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining, txcsum);
831 }
832 MPASS(npkt_remaining == 0);
833 MPASS(kring->nr_hwcur == head);
834 MPASS(nm_txq->dbidx == nm_txq->pidx);
835
836 /*
837 * Second part: reclaim buffers for completed transmissions.
838 */

--- 17 unchanged lines hidden (view full) ---

856 struct ifnet *ifp = na->ifp;
857 struct port_info *pi = ifp->if_softc;
858 struct adapter *sc = pi->adapter;
859 struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[pi->first_nm_rxq + kring->ring_id];
860 u_int const head = nm_rxsync_prologue(kring);
861 u_int n;
862 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
863
896 }
897 MPASS(npkt_remaining == 0);
898 MPASS(kring->nr_hwcur == head);
899 MPASS(nm_txq->dbidx == nm_txq->pidx);
900
901 /*
902 * Second part: reclaim buffers for completed transmissions.
903 */

--- 17 unchanged lines hidden (view full) ---

921 struct ifnet *ifp = na->ifp;
922 struct port_info *pi = ifp->if_softc;
923 struct adapter *sc = pi->adapter;
924 struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[pi->first_nm_rxq + kring->ring_id];
925 u_int const head = nm_rxsync_prologue(kring);
926 u_int n;
927 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
928
929 if (black_hole)
930 return (0); /* No updates ever. */
931
864 if (netmap_no_pendintr || force_update) {
865 kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
866 kring->nr_kflags &= ~NKR_PENDINTR;
867 }
868
869 /* Userspace done with buffers from kring->nr_hwcur to head */
870 n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
871 kring->nkr_num_slots - kring->nr_hwcur + head;

--- 13 unchanged lines hidden (view full) ---

885 MPASS((n & 7) == 0);
886
887 IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);
888 IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx);
889
890 while (n > 0) {
891 for (i = 0; i < 8; i++, fl_pidx++, slot++) {
892 PNMB(na, slot, &ba);
932 if (netmap_no_pendintr || force_update) {
933 kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
934 kring->nr_kflags &= ~NKR_PENDINTR;
935 }
936
937 /* Userspace done with buffers from kring->nr_hwcur to head */
938 n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
939 kring->nkr_num_slots - kring->nr_hwcur + head;

--- 13 unchanged lines hidden (view full) ---

953 MPASS((n & 7) == 0);
954
955 IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);
956 IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx);
957
958 while (n > 0) {
959 for (i = 0; i < 8; i++, fl_pidx++, slot++) {
960 PNMB(na, slot, &ba);
961 MPASS(ba != 0);
893 nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
894 slot->flags &= ~NS_BUF_CHANGED;
895 MPASS(fl_pidx <= nm_rxq->fl_sidx);
896 }
897 n -= 8;
898 if (fl_pidx == nm_rxq->fl_sidx) {
899 fl_pidx = 0;
900 slot = &ring->slot[0];

--- 158 unchanged lines hidden (view full) ---

1059 struct port_info *pi = nm_rxq->pi;
1060 struct adapter *sc = pi->adapter;
1061 struct ifnet *ifp = pi->nm_ifp;
1062 struct netmap_adapter *na = NA(ifp);
1063 struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
1064 struct netmap_ring *ring = kring->ring;
1065 struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
1066 uint32_t lq;
962 nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
963 slot->flags &= ~NS_BUF_CHANGED;
964 MPASS(fl_pidx <= nm_rxq->fl_sidx);
965 }
966 n -= 8;
967 if (fl_pidx == nm_rxq->fl_sidx) {
968 fl_pidx = 0;
969 slot = &ring->slot[0];

--- 158 unchanged lines hidden (view full) ---

1128 struct port_info *pi = nm_rxq->pi;
1129 struct adapter *sc = pi->adapter;
1130 struct ifnet *ifp = pi->nm_ifp;
1131 struct netmap_adapter *na = NA(ifp);
1132 struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
1133 struct netmap_ring *ring = kring->ring;
1134 struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
1135 uint32_t lq;
1067 u_int n = 0;
1068 int processed = 0;
1136 u_int n = 0, work = 0;
1069 uint8_t opcode;
1070 uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
1137 uint8_t opcode;
1138 uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
1139 u_int fl_credits = fl_cidx & 7;
1071
1072 while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
1073
1074 rmb();
1075
1076 lq = be32toh(d->rsp.pldbuflen_qid);
1077 opcode = d->rss.opcode;
1078
1079 switch (G_RSPD_TYPE(d->rsp.u.type_gen)) {
1080 case X_RSPD_TYPE_FLBUF:
1140
1141 while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
1142
1143 rmb();
1144
1145 lq = be32toh(d->rsp.pldbuflen_qid);
1146 opcode = d->rss.opcode;
1147
1148 switch (G_RSPD_TYPE(d->rsp.u.type_gen)) {
1149 case X_RSPD_TYPE_FLBUF:
1081 /* No buffer packing so new buf every time */
1082 MPASS(lq & F_RSPD_NEWBUF);
1150 if (black_hole != 2) {
1151 /* No buffer packing so new buf every time */
1152 MPASS(lq & F_RSPD_NEWBUF);
1153 }
1083
1084 /* fall through */
1085
1086 case X_RSPD_TYPE_CPL:
1087 MPASS(opcode < NUM_CPL_CMDS);
1088
1089 switch (opcode) {
1090 case CPL_FW4_MSG:
1091 case CPL_FW6_MSG:
1092 handle_nm_fw6_msg(sc, ifp,
1093 (const void *)&d->cpl[0]);
1094 break;
1095 case CPL_RX_PKT:
1096 ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
1097 ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
1154
1155 /* fall through */
1156
1157 case X_RSPD_TYPE_CPL:
1158 MPASS(opcode < NUM_CPL_CMDS);
1159
1160 switch (opcode) {
1161 case CPL_FW4_MSG:
1162 case CPL_FW6_MSG:
1163 handle_nm_fw6_msg(sc, ifp,
1164 (const void *)&d->cpl[0]);
1165 break;
1166 case CPL_RX_PKT:
1167 ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
1168 ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
1098 if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))
1169 fl_cidx += (lq & F_RSPD_NEWBUF) ? 1 : 0;
1170 fl_credits += (lq & F_RSPD_NEWBUF) ? 1 : 0;
1171 if (__predict_false(fl_cidx == nm_rxq->fl_sidx))
1099 fl_cidx = 0;
1100 break;
1101 default:
1102 panic("%s: unexpected opcode 0x%x on nm_rxq %p",
1103 __func__, opcode, nm_rxq);
1104 }
1105 break;
1106

--- 9 unchanged lines hidden (view full) ---

1116
1117 d++;
1118 if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {
1119 nm_rxq->iq_cidx = 0;
1120 d = &nm_rxq->iq_desc[0];
1121 nm_rxq->iq_gen ^= F_RSPD_GEN;
1122 }
1123
1172 fl_cidx = 0;
1173 break;
1174 default:
1175 panic("%s: unexpected opcode 0x%x on nm_rxq %p",
1176 __func__, opcode, nm_rxq);
1177 }
1178 break;
1179

--- 9 unchanged lines hidden (view full) ---

1189
1190 d++;
1191 if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {
1192 nm_rxq->iq_cidx = 0;
1193 d = &nm_rxq->iq_desc[0];
1194 nm_rxq->iq_gen ^= F_RSPD_GEN;
1195 }
1196
1124 if (__predict_false(++n == 64)) { /* XXXNM: tune */
1197 if (__predict_false(++n == rx_ndesc)) {
1198 atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
1199 if (black_hole && fl_credits >= 8) {
1200 fl_credits /= 8;
1201 IDXINCR(nm_rxq->fl_pidx, fl_credits * 8,
1202 nm_rxq->fl_sidx);
1203 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
1204 nm_rxq->fl_db_val | V_PIDX(fl_credits));
1205 fl_credits = fl_cidx & 7;
1206 } else if (!black_hole) {
1207 netmap_rx_irq(ifp, nm_rxq->nid, &work);
1208 MPASS(work != 0);
1209 }
1125 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
1126 V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) |
1127 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1128 n = 0;
1129 }
1130 }
1210 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
1211 V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) |
1212 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1213 n = 0;
1214 }
1215 }
1131 if (fl_cidx != nm_rxq->fl_cidx) {
1132 atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
1133 netmap_rx_irq(ifp, nm_rxq->nid, &processed);
1134 }
1216
1217 atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
1218 if (black_hole) {
1219 fl_credits /= 8;
1220 IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx);
1221 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
1222 nm_rxq->fl_db_val | V_PIDX(fl_credits));
1223 } else
1224 netmap_rx_irq(ifp, nm_rxq->nid, &work);
1225
1135 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) |
1226 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) |
1136 V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) | V_SEINTARM(F_QINTR_CNT_EN));
1227 V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) |
1228 V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx)));
1137}
1138#endif
1229}
1230#endif