Deleted Added
sdiff udiff text old ( 266757 ) new ( 270297 )
full compact
t4_netmap.c (266757) t4_netmap.c (270297)
1/*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_netmap.c 266757 2014-05-27 18:18:41Z np $");
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_netmap.c 270297 2014-08-21 19:54:02Z np $");
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#ifdef DEV_NETMAP
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/lock.h>

--- 195 unchanged lines hidden (view full) ---

233 struct adapter *sc = pi->adapter;
234 struct netmap_adapter *na = NA(pi->nm_ifp);
235 struct fw_iq_cmd c;
236
237 MPASS(na != NULL);
238 MPASS(nm_rxq->iq_desc != NULL);
239 MPASS(nm_rxq->fl_desc != NULL);
240
30
31#include "opt_inet.h"
32#include "opt_inet6.h"
33
34#ifdef DEV_NETMAP
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/lock.h>

--- 195 unchanged lines hidden (view full) ---

233 struct adapter *sc = pi->adapter;
234 struct netmap_adapter *na = NA(pi->nm_ifp);
235 struct fw_iq_cmd c;
236
237 MPASS(na != NULL);
238 MPASS(nm_rxq->iq_desc != NULL);
239 MPASS(nm_rxq->fl_desc != NULL);
240
241 bzero(nm_rxq->iq_desc, pi->qsize_rxq * RX_IQ_ESIZE);
242 bzero(nm_rxq->fl_desc, na->num_rx_desc * RX_FL_ESIZE + spg_len);
241 bzero(nm_rxq->iq_desc, pi->qsize_rxq * IQ_ESIZE);
242 bzero(nm_rxq->fl_desc, na->num_rx_desc * EQ_ESIZE + spg_len);
243
244 bzero(&c, sizeof(c));
245 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
246 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
247 V_FW_IQ_CMD_VFN(0));
248 c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
249 FW_LEN16(c));
250 if (pi->flags & INTR_NM_RXQ) {

--- 8 unchanged lines hidden (view full) ---

259 }
260 c.type_to_iqandstindex = htobe32(v |
261 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
262 V_FW_IQ_CMD_VIID(pi->nm_viid) |
263 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
264 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
265 F_FW_IQ_CMD_IQGTSMODE |
266 V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
243
244 bzero(&c, sizeof(c));
245 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
246 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
247 V_FW_IQ_CMD_VFN(0));
248 c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
249 FW_LEN16(c));
250 if (pi->flags & INTR_NM_RXQ) {

--- 8 unchanged lines hidden (view full) ---

259 }
260 c.type_to_iqandstindex = htobe32(v |
261 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
262 V_FW_IQ_CMD_VIID(pi->nm_viid) |
263 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
264 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
265 F_FW_IQ_CMD_IQGTSMODE |
266 V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
267 V_FW_IQ_CMD_IQESIZE(ilog2(RX_IQ_ESIZE) - 4));
267 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
268 c.iqsize = htobe16(pi->qsize_rxq);
269 c.iqaddr = htobe64(nm_rxq->iq_ba);
270 c.iqns_to_fl0congen |=
271 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
272 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
273 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0));
274 c.fl0dcaen_to_fl0cidxfthresh =
275 htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
276 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
268 c.iqsize = htobe16(pi->qsize_rxq);
269 c.iqaddr = htobe64(nm_rxq->iq_ba);
270 c.iqns_to_fl0congen |=
271 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
272 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
273 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0));
274 c.fl0dcaen_to_fl0cidxfthresh =
275 htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
276 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
277 c.fl0size = htobe16(na->num_rx_desc + spg_len / RX_FL_ESIZE);
277 c.fl0size = htobe16(na->num_rx_desc + spg_len / EQ_ESIZE);
278 c.fl0addr = htobe64(nm_rxq->fl_ba);
279
280 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
281 if (rc != 0) {
282 device_printf(sc->dev,
283 "failed to create netmap ingress queue: %d\n", rc);
284 return (rc);
285 }
286
287 nm_rxq->iq_cidx = 0;
278 c.fl0addr = htobe64(nm_rxq->fl_ba);
279
280 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
281 if (rc != 0) {
282 device_printf(sc->dev,
283 "failed to create netmap ingress queue: %d\n", rc);
284 return (rc);
285 }
286
287 nm_rxq->iq_cidx = 0;
288 MPASS(nm_rxq->iq_sidx == pi->qsize_rxq - spg_len / RX_IQ_ESIZE);
288 MPASS(nm_rxq->iq_sidx == pi->qsize_rxq - spg_len / IQ_ESIZE);
289 nm_rxq->iq_gen = F_RSPD_GEN;
290 nm_rxq->iq_cntxt_id = be16toh(c.iqid);
291 nm_rxq->iq_abs_id = be16toh(c.physiqid);
292 cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;
293 if (cntxt_id >= sc->sge.niq) {
294 panic ("%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
295 __func__, cntxt_id, sc->sge.niq - 1);
296 }

--- 49 unchanged lines hidden (view full) ---

346 bzero(nm_txq->desc, len);
347
348 bzero(&c, sizeof(c));
349 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
350 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
351 V_FW_EQ_ETH_CMD_VFN(0));
352 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
353 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
289 nm_rxq->iq_gen = F_RSPD_GEN;
290 nm_rxq->iq_cntxt_id = be16toh(c.iqid);
291 nm_rxq->iq_abs_id = be16toh(c.physiqid);
292 cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;
293 if (cntxt_id >= sc->sge.niq) {
294 panic ("%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
295 __func__, cntxt_id, sc->sge.niq - 1);
296 }

--- 49 unchanged lines hidden (view full) ---

346 bzero(nm_txq->desc, len);
347
348 bzero(&c, sizeof(c));
349 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
350 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
351 V_FW_EQ_ETH_CMD_VFN(0));
352 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
353 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
354 c.viid_pkd = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->nm_viid));
354 c.autoequiqe_to_viid = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->nm_viid));
355 c.fetchszm_to_iqid =
356 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
357 V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
358 V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
359 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
360 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
361 V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
362 c.eqaddr = htobe64(nm_txq->ba);

--- 22 unchanged lines hidden (view full) ---

385 isset(&nm_txq->doorbells, DOORBELL_WCWR)) {
386 uint32_t s_qpp = sc->sge.eq_s_qpp;
387 uint32_t mask = (1 << s_qpp) - 1;
388 volatile uint8_t *udb;
389
390 udb = sc->udbs_base + UDBS_DB_OFFSET;
391 udb += (nm_txq->cntxt_id >> s_qpp) << PAGE_SHIFT;
392 nm_txq->udb_qid = nm_txq->cntxt_id & mask;
355 c.fetchszm_to_iqid =
356 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
357 V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
358 V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
359 c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
360 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
361 V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
362 c.eqaddr = htobe64(nm_txq->ba);

--- 22 unchanged lines hidden (view full) ---

385 isset(&nm_txq->doorbells, DOORBELL_WCWR)) {
386 uint32_t s_qpp = sc->sge.eq_s_qpp;
387 uint32_t mask = (1 << s_qpp) - 1;
388 volatile uint8_t *udb;
389
390 udb = sc->udbs_base + UDBS_DB_OFFSET;
391 udb += (nm_txq->cntxt_id >> s_qpp) << PAGE_SHIFT;
392 nm_txq->udb_qid = nm_txq->cntxt_id & mask;
393 if (nm_txq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
393 if (nm_txq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
394 clrbit(&nm_txq->doorbells, DOORBELL_WCWR);
395 else {
396 udb += nm_txq->udb_qid << UDBS_SEG_SHIFT;
397 nm_txq->udb_qid = 0;
398 }
399 nm_txq->udb = (volatile void *)udb;
400 }
401

--- 27 unchanged lines hidden (view full) ---

429 ASSERT_SYNCHRONIZED_OP(sc);
430
431 if ((pi->flags & PORT_INIT_DONE) == 0 ||
432 (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
433 return (EAGAIN);
434
435 hwb = &sc->sge.hw_buf_info[0];
436 for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
394 clrbit(&nm_txq->doorbells, DOORBELL_WCWR);
395 else {
396 udb += nm_txq->udb_qid << UDBS_SEG_SHIFT;
397 nm_txq->udb_qid = 0;
398 }
399 nm_txq->udb = (volatile void *)udb;
400 }
401

--- 27 unchanged lines hidden (view full) ---

429 ASSERT_SYNCHRONIZED_OP(sc);
430
431 if ((pi->flags & PORT_INIT_DONE) == 0 ||
432 (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
433 return (EAGAIN);
434
435 hwb = &sc->sge.hw_buf_info[0];
436 for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
437 if (hwb->size == NETMAP_BUF_SIZE)
437 if (hwb->size == NETMAP_BUF_SIZE(na))
438 break;
439 }
440 if (i >= SGE_FLBUF_SIZES) {
441 if_printf(ifp, "no hwidx for netmap buffer size %d.\n",
438 break;
439 }
440 if (i >= SGE_FLBUF_SIZES) {
441 if_printf(ifp, "no hwidx for netmap buffer size %d.\n",
442 NETMAP_BUF_SIZE);
442 NETMAP_BUF_SIZE(na));
443 return (ENXIO);
444 }
445 hwidx = i;
446
447 /* Must set caps before calling netmap_reset */
443 return (ENXIO);
444 }
445 hwidx = i;
446
447 /* Must set caps before calling netmap_reset */
448 na->na_flags |= (NAF_NATIVE_ON | NAF_NETMAP_ON);
449 ifp->if_capenable |= IFCAP_NETMAP;
448 nm_set_native_flags(na);
450
451 for_each_nm_rxq(pi, i, nm_rxq) {
452 alloc_nm_rxq_hwq(pi, nm_rxq);
453 nm_rxq->fl_hwidx = hwidx;
454 slot = netmap_reset(na, NR_RX, i, 0);
455 MPASS(slot != NULL); /* XXXNM: error check, not assert */
456
457 /* We deal with 8 bufs at a time */
458 MPASS((na->num_rx_desc & 7) == 0);
459 MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
460 for (j = 0; j < nm_rxq->fl_sidx - 8; j++) {
461 uint64_t ba;
462
449
450 for_each_nm_rxq(pi, i, nm_rxq) {
451 alloc_nm_rxq_hwq(pi, nm_rxq);
452 nm_rxq->fl_hwidx = hwidx;
453 slot = netmap_reset(na, NR_RX, i, 0);
454 MPASS(slot != NULL); /* XXXNM: error check, not assert */
455
456 /* We deal with 8 bufs at a time */
457 MPASS((na->num_rx_desc & 7) == 0);
458 MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
459 for (j = 0; j < nm_rxq->fl_sidx - 8; j++) {
460 uint64_t ba;
461
463 PNMB(&slot[j], &ba);
462 PNMB(na, &slot[j], &ba);
464 nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
465 }
466 nm_rxq->fl_pidx = j;
467 MPASS((j & 7) == 0);
468 j /= 8; /* driver pidx to hardware pidx */
469 wmb();
470 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
471 nm_rxq->fl_db_val | V_PIDX(j));

--- 35 unchanged lines hidden (view full) ---

507 struct sge_nm_txq *nm_txq;
508 struct sge_nm_rxq *nm_rxq;
509
510 ASSERT_SYNCHRONIZED_OP(sc);
511
512 rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, false, false);
513 if (rc != 0)
514 if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
463 nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
464 }
465 nm_rxq->fl_pidx = j;
466 MPASS((j & 7) == 0);
467 j /= 8; /* driver pidx to hardware pidx */
468 wmb();
469 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
470 nm_rxq->fl_db_val | V_PIDX(j));

--- 35 unchanged lines hidden (view full) ---

506 struct sge_nm_txq *nm_txq;
507 struct sge_nm_rxq *nm_rxq;
508
509 ASSERT_SYNCHRONIZED_OP(sc);
510
511 rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, false, false);
512 if (rc != 0)
513 if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
515 na->na_flags &= ~(NAF_NATIVE_ON | NAF_NETMAP_ON);
516 ifp->if_capenable &= ~IFCAP_NETMAP;
514 nm_clear_native_flags(na);
517
518 /*
519 * XXXNM: We need to make sure that the tx queues are quiet and won't
520 * request any more SGE_EGR_UPDATEs.
521 */
522
523 for_each_nm_txq(pi, i, nm_txq) {
524 free_nm_txq_hwq(pi, nm_txq);

--- 51 unchanged lines hidden (view full) ---

576npkt_to_len16(const int n)
577{
578
579 MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
580
581 return (n * 2 + 1);
582}
583
515
516 /*
517 * XXXNM: We need to make sure that the tx queues are quiet and won't
518 * request any more SGE_EGR_UPDATEs.
519 */
520
521 for_each_nm_txq(pi, i, nm_txq) {
522 free_nm_txq_hwq(pi, nm_txq);

--- 51 unchanged lines hidden (view full) ---

574npkt_to_len16(const int n)
575{
576
577 MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
578
579 return (n * 2 + 1);
580}
581
584static inline uint16_t
585idxdiff(uint16_t head, uint16_t tail, uint16_t wrap)
586{
587 MPASS(wrap > head);
588 MPASS(wrap > tail);
582#define NMIDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->sidx)
589
583
590 if (head >= tail)
591 return (head - tail);
592 else
593 return (wrap - tail + head);
594}
595#define IDXDIFF(q, idx) idxdiff((q)->pidx, (q)->idx, (q)->sidx)
596
597static void
598ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq)
599{
600 int n;
601 u_int db = nm_txq->doorbells;
602
603 MPASS(nm_txq->pidx != nm_txq->dbidx);
604
584static void
585ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq)
586{
587 int n;
588 u_int db = nm_txq->doorbells;
589
590 MPASS(nm_txq->pidx != nm_txq->dbidx);
591
605 n = IDXDIFF(nm_txq, dbidx);
592 n = NMIDXDIFF(nm_txq, dbidx);
606 if (n > 1)
607 clrbit(&db, DOORBELL_WCWR);
608 wmb();
609
610 switch (ffs(db) - 1) {
611 case DOORBELL_UDB:
612 *nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
613 break;

--- 61 unchanged lines hidden (view full) ---

675 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(npkt_to_len16(n)));
676 wr->npkt = n;
677 wr->r3 = 0;
678 wr->type = 1;
679 cpl = (void *)(wr + 1);
680
681 for (i = 0; i < n; i++) {
682 slot = &ring->slot[kring->nr_hwcur];
593 if (n > 1)
594 clrbit(&db, DOORBELL_WCWR);
595 wmb();
596
597 switch (ffs(db) - 1) {
598 case DOORBELL_UDB:
599 *nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
600 break;

--- 61 unchanged lines hidden (view full) ---

662 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(npkt_to_len16(n)));
663 wr->npkt = n;
664 wr->r3 = 0;
665 wr->type = 1;
666 cpl = (void *)(wr + 1);
667
668 for (i = 0; i < n; i++) {
669 slot = &ring->slot[kring->nr_hwcur];
683 PNMB(slot, &ba);
670 PNMB(kring->na, slot, &ba);
684
685 cpl->ctrl0 = nm_txq->cpl_ctrl0;
686 cpl->pack = 0;
687 cpl->len = htobe16(slot->len);
688 /*
689 * netmap(4) says "netmap does not use features such as
690 * checksum offloading, TCP segmentation offloading,
691 * encryption, VLAN encapsulation/decapsulation, etc."

--- 36 unchanged lines hidden (view full) ---

728 F_FW_WR_EQUIQ);
729 nm_txq->equeqidx = nm_txq->pidx;
730 nm_txq->equiqidx = nm_txq->pidx;
731 }
732 ring_nm_txq_db(sc, nm_txq);
733 return;
734 }
735
671
672 cpl->ctrl0 = nm_txq->cpl_ctrl0;
673 cpl->pack = 0;
674 cpl->len = htobe16(slot->len);
675 /*
676 * netmap(4) says "netmap does not use features such as
677 * checksum offloading, TCP segmentation offloading,
678 * encryption, VLAN encapsulation/decapsulation, etc."

--- 36 unchanged lines hidden (view full) ---

715 F_FW_WR_EQUIQ);
716 nm_txq->equeqidx = nm_txq->pidx;
717 nm_txq->equiqidx = nm_txq->pidx;
718 }
719 ring_nm_txq_db(sc, nm_txq);
720 return;
721 }
722
736 if (IDXDIFF(nm_txq, equiqidx) >= nm_txq->sidx / 2) {
723 if (NMIDXDIFF(nm_txq, equiqidx) >= nm_txq->sidx / 2) {
737 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
738 F_FW_WR_EQUIQ);
739 nm_txq->equeqidx = nm_txq->pidx;
740 nm_txq->equiqidx = nm_txq->pidx;
724 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
725 F_FW_WR_EQUIQ);
726 nm_txq->equeqidx = nm_txq->pidx;
727 nm_txq->equiqidx = nm_txq->pidx;
741 } else if (IDXDIFF(nm_txq, equeqidx) >= 64) {
728 } else if (NMIDXDIFF(nm_txq, equeqidx) >= 64) {
742 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
743 nm_txq->equeqidx = nm_txq->pidx;
744 }
729 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
730 nm_txq->equeqidx = nm_txq->pidx;
731 }
745 if (IDXDIFF(nm_txq, dbidx) >= 2 * SGE_MAX_WR_NDESC)
732 if (NMIDXDIFF(nm_txq, dbidx) >= 2 * SGE_MAX_WR_NDESC)
746 ring_nm_txq_db(sc, nm_txq);
747 }
748
749 /* Will get called again. */
750 MPASS(npkt_remaining);
751}
752
753/* How many contiguous free descriptors starting at pidx */

--- 23 unchanged lines hidden (view full) ---

777 wr = (void *)&nm_txq->desc[nm_txq->cidx];
778
779 MPASS(wr->op_pkd == htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)));
780 MPASS(wr->type == 1);
781 MPASS(wr->npkt > 0 && wr->npkt <= MAX_NPKT_IN_TYPE1_WR);
782
783 n += wr->npkt;
784 nm_txq->cidx += npkt_to_ndesc(wr->npkt);
733 ring_nm_txq_db(sc, nm_txq);
734 }
735
736 /* Will get called again. */
737 MPASS(npkt_remaining);
738}
739
740/* How many contiguous free descriptors starting at pidx */

--- 23 unchanged lines hidden (view full) ---

764 wr = (void *)&nm_txq->desc[nm_txq->cidx];
765
766 MPASS(wr->op_pkd == htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)));
767 MPASS(wr->type == 1);
768 MPASS(wr->npkt > 0 && wr->npkt <= MAX_NPKT_IN_TYPE1_WR);
769
770 n += wr->npkt;
771 nm_txq->cidx += npkt_to_ndesc(wr->npkt);
785 if (__predict_false(nm_txq->cidx >= nm_txq->sidx))
786 nm_txq->cidx -= nm_txq->sidx;
772
773 /*
774 * We never sent a WR that wrapped around so the credits coming
775 * back, WR by WR, should never cause the cidx to wrap around
776 * either.
777 */
778 MPASS(nm_txq->cidx <= nm_txq->sidx);
779 if (__predict_false(nm_txq->cidx == nm_txq->sidx))
780 nm_txq->cidx = 0;
787 }
788
789 return (n);
790}
791
792static int
781 }
782
783 return (n);
784}
785
786static int
793cxgbe_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags)
787cxgbe_netmap_txsync(struct netmap_kring *kring, int flags)
794{
788{
795 struct netmap_kring *kring = &na->tx_rings[ring_nr];
789 struct netmap_adapter *na = kring->na;
796 struct ifnet *ifp = na->ifp;
797 struct port_info *pi = ifp->if_softc;
798 struct adapter *sc = pi->adapter;
790 struct ifnet *ifp = na->ifp;
791 struct port_info *pi = ifp->if_softc;
792 struct adapter *sc = pi->adapter;
799 struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[pi->first_nm_txq + ring_nr];
793 struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[pi->first_nm_txq + kring->ring_id];
800 const u_int head = kring->rhead;
801 u_int reclaimed = 0;
802 int n, d, npkt_remaining, ndesc_remaining;
803
804 /*
805 * Tx was at kring->nr_hwcur last time around and now we need to advance
806 * to kring->rhead. Note that the driver's pidx moves independent of
807 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation

--- 42 unchanged lines hidden (view full) ---

850 }
851
852 nm_txsync_finalize(kring);
853
854 return (0);
855}
856
857static int
794 const u_int head = kring->rhead;
795 u_int reclaimed = 0;
796 int n, d, npkt_remaining, ndesc_remaining;
797
798 /*
799 * Tx was at kring->nr_hwcur last time around and now we need to advance
800 * to kring->rhead. Note that the driver's pidx moves independent of
801 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation

--- 42 unchanged lines hidden (view full) ---

844 }
845
846 nm_txsync_finalize(kring);
847
848 return (0);
849}
850
851static int
858cxgbe_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags)
852cxgbe_netmap_rxsync(struct netmap_kring *kring, int flags)
859{
853{
860 struct netmap_kring *kring = &na->rx_rings[ring_nr];
854 struct netmap_adapter *na = kring->na;
861 struct netmap_ring *ring = kring->ring;
862 struct ifnet *ifp = na->ifp;
863 struct port_info *pi = ifp->if_softc;
864 struct adapter *sc = pi->adapter;
855 struct netmap_ring *ring = kring->ring;
856 struct ifnet *ifp = na->ifp;
857 struct port_info *pi = ifp->if_softc;
858 struct adapter *sc = pi->adapter;
865 struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[pi->first_nm_rxq + ring_nr];
859 struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[pi->first_nm_rxq + kring->ring_id];
866 u_int const head = nm_rxsync_prologue(kring);
867 u_int n;
868 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
869
870 if (netmap_no_pendintr || force_update) {
871 kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
872 kring->nr_kflags &= ~NKR_PENDINTR;
873 }

--- 11 unchanged lines hidden (view full) ---

885 /*
886 * We always deal with 8 buffers at a time. We must have
887 * stopped at an 8B boundary (fl_pidx) last time around and we
888 * must have a multiple of 8B buffers to give to the freelist.
889 */
890 MPASS((fl_pidx & 7) == 0);
891 MPASS((n & 7) == 0);
892
860 u_int const head = nm_rxsync_prologue(kring);
861 u_int n;
862 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
863
864 if (netmap_no_pendintr || force_update) {
865 kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
866 kring->nr_kflags &= ~NKR_PENDINTR;
867 }

--- 11 unchanged lines hidden (view full) ---

879 /*
880 * We always deal with 8 buffers at a time. We must have
881 * stopped at an 8B boundary (fl_pidx) last time around and we
882 * must have a multiple of 8B buffers to give to the freelist.
883 */
884 MPASS((fl_pidx & 7) == 0);
885 MPASS((n & 7) == 0);
886
893 kring->nr_hwcur += n;
894 if (kring->nr_hwcur >= kring->nkr_num_slots)
895 kring->nr_hwcur -= kring->nkr_num_slots;
887 IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);
888 IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx);
896
889
897 nm_rxq->fl_pidx += n;
898 if (nm_rxq->fl_pidx >= nm_rxq->fl_sidx)
899 nm_rxq->fl_pidx -= nm_rxq->fl_sidx;
900
901 while (n > 0) {
902 for (i = 0; i < 8; i++, fl_pidx++, slot++) {
890 while (n > 0) {
891 for (i = 0; i < 8; i++, fl_pidx++, slot++) {
903 PNMB(slot, &ba);
892 PNMB(na, slot, &ba);
904 nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
905 slot->flags &= ~NS_BUF_CHANGED;
906 MPASS(fl_pidx <= nm_rxq->fl_sidx);
907 }
908 n -= 8;
909 if (fl_pidx == nm_rxq->fl_sidx) {
910 fl_pidx = 0;
911 slot = &ring->slot[0];

--- 156 unchanged lines hidden (view full) ---

1068{
1069 struct sge_nm_rxq *nm_rxq = arg;
1070 struct port_info *pi = nm_rxq->pi;
1071 struct adapter *sc = pi->adapter;
1072 struct ifnet *ifp = pi->nm_ifp;
1073 struct netmap_adapter *na = NA(ifp);
1074 struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
1075 struct netmap_ring *ring = kring->ring;
893 nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
894 slot->flags &= ~NS_BUF_CHANGED;
895 MPASS(fl_pidx <= nm_rxq->fl_sidx);
896 }
897 n -= 8;
898 if (fl_pidx == nm_rxq->fl_sidx) {
899 fl_pidx = 0;
900 slot = &ring->slot[0];

--- 156 unchanged lines hidden (view full) ---

1057{
1058 struct sge_nm_rxq *nm_rxq = arg;
1059 struct port_info *pi = nm_rxq->pi;
1060 struct adapter *sc = pi->adapter;
1061 struct ifnet *ifp = pi->nm_ifp;
1062 struct netmap_adapter *na = NA(ifp);
1063 struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
1064 struct netmap_ring *ring = kring->ring;
1076 struct nm_iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
1065 struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
1077 uint32_t lq;
1078 u_int n = 0;
1079 int processed = 0;
1080 uint8_t opcode;
1081 uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
1082
1083 while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
1084

--- 10 unchanged lines hidden (view full) ---

1095 /* fall through */
1096
1097 case X_RSPD_TYPE_CPL:
1098 MPASS(opcode < NUM_CPL_CMDS);
1099
1100 switch (opcode) {
1101 case CPL_FW4_MSG:
1102 case CPL_FW6_MSG:
1066 uint32_t lq;
1067 u_int n = 0;
1068 int processed = 0;
1069 uint8_t opcode;
1070 uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
1071
1072 while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
1073

--- 10 unchanged lines hidden (view full) ---

1084 /* fall through */
1085
1086 case X_RSPD_TYPE_CPL:
1087 MPASS(opcode < NUM_CPL_CMDS);
1088
1089 switch (opcode) {
1090 case CPL_FW4_MSG:
1091 case CPL_FW6_MSG:
1103 handle_nm_fw6_msg(sc, ifp, &d->u.fw6_msg);
1092 handle_nm_fw6_msg(sc, ifp,
1093 (const void *)&d->cpl[0]);
1104 break;
1105 case CPL_RX_PKT:
1106 ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
1107 ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
1108 if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))
1109 fl_cidx = 0;
1110 break;
1111 default:

--- 37 unchanged lines hidden --- 		1094 break;
1095 case CPL_RX_PKT:
1096 ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
1097 ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
1098 if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))
1099 fl_cidx = 0;
1100 break;
1101 default:

--- 37 unchanged lines hidden ---