| nicvf_queues.c (296030) | nicvf_queues.c (296031) |
|---|---|
| 1/* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 9 unchanged lines hidden (view full) --- 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * | 1/* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 9 unchanged lines hidden (view full) --- 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * |
| 26 * $FreeBSD: head/sys/dev/vnic/nicvf_queues.c 296030 2016-02-25 14:12:51Z zbb $ | 26 * $FreeBSD: head/sys/dev/vnic/nicvf_queues.c 296031 2016-02-25 14:14:46Z zbb $ |
| 27 * 28 */ 29#include <sys/cdefs.h> | 27 * 28 */ 29#include <sys/cdefs.h> |
| 30__FBSDID("$FreeBSD: head/sys/dev/vnic/nicvf_queues.c 296030 2016-02-25 14:12:51Z zbb $"); | 30__FBSDID("$FreeBSD: head/sys/dev/vnic/nicvf_queues.c 296031 2016-02-25 14:14:46Z zbb $"); |
| 31 32#include "opt_inet.h" 33#include "opt_inet6.h" 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/bitset.h> 38#include <sys/bitstring.h> --- 593 unchanged lines hidden (view full) --- 632 } 633} 634 635static int 636nicvf_rcv_pkt_handler(struct nicvf *nic, struct cmp_queue *cq, 637 struct cqe_rx_t *cqe_rx, int cqe_type) 638{ 639 struct mbuf *mbuf; | 31 32#include "opt_inet.h" 33#include "opt_inet6.h" 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/bitset.h> 38#include <sys/bitstring.h> --- 593 unchanged lines hidden (view full) --- 632 } 633} 634 635static int 636nicvf_rcv_pkt_handler(struct nicvf *nic, struct cmp_queue *cq, 637 struct cqe_rx_t *cqe_rx, int cqe_type) 638{ 639 struct mbuf *mbuf; |
| 640 struct rcv_queue *rq; |
|
| 640 int rq_idx; 641 int err = 0; 642 643 rq_idx = cqe_rx->rq_idx; | 641 int rq_idx; 642 int err = 0; 643 644 rq_idx = cqe_rx->rq_idx; |
| 645 rq = &nic->qs->rq[rq_idx]; |
|
| 644 645 /* Check for errors */ 646 err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx); 647 if (err && !cqe_rx->rb_cnt) 648 return (0); 649 650 mbuf = nicvf_get_rcv_mbuf(nic, cqe_rx); 651 if (mbuf == NULL) { 652 dprintf(nic->dev, "Packet not received\n"); 653 return (0); 654 } 655 656 /* If error packet */ 657 if (err != 0) { 658 m_freem(mbuf); 659 return (0); 660 } 661 | 646 647 /* Check for errors */ 648 err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx); 649 if (err && !cqe_rx->rb_cnt) 650 return (0); 651 652 mbuf = nicvf_get_rcv_mbuf(nic, cqe_rx); 653 if (mbuf == NULL) { 654 dprintf(nic->dev, "Packet not received\n"); 655 return (0); 656 } 657 658 /* If error packet */ 659 if (err != 0) { 660 m_freem(mbuf); 661 return (0); 662 } 663 |
| 664 if (rq->lro_enabled && 665 ((cqe_rx->l3_type == L3TYPE_IPV4) && (cqe_rx->l4_type == L4TYPE_TCP)) && 666 (mbuf->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 667 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 668 /* 669 * At this point it is known that there are no errors in the 670 * packet. Attempt to LRO enqueue. Send to stack if no resources 671 * or enqueue error. 672 */ 673 if ((rq->lro.lro_cnt != 0) && 674 (tcp_lro_rx(&rq->lro, mbuf, 0) == 0)) 675 return (0); 676 } |
|
| 662 /* 663 * Push this packet to the stack later to avoid 664 * unlocking completion task in the middle of work. 665 */ 666 err = buf_ring_enqueue(cq->rx_br, mbuf); 667 if (err != 0) { 668 /* 669 * Failed to enqueue this mbuf. --- 51 unchanged lines hidden (view full) --- 721nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx) 722{ 723 struct mbuf *mbuf; 724 struct ifnet *ifp; 725 int processed_cqe, work_done = 0, tx_done = 0; 726 int cqe_count, cqe_head; 727 struct queue_set *qs = nic->qs; 728 struct cmp_queue *cq = &qs->cq[cq_idx]; | 677 /* 678 * Push this packet to the stack later to avoid 679 * unlocking completion task in the middle of work. 680 */ 681 err = buf_ring_enqueue(cq->rx_br, mbuf); 682 if (err != 0) { 683 /* 684 * Failed to enqueue this mbuf. --- 51 unchanged lines hidden (view full) --- 736nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx) 737{ 738 struct mbuf *mbuf; 739 struct ifnet *ifp; 740 int processed_cqe, work_done = 0, tx_done = 0; 741 int cqe_count, cqe_head; 742 struct queue_set *qs = nic->qs; 743 struct cmp_queue *cq = &qs->cq[cq_idx]; |
| 744 struct rcv_queue *rq; |
|
| 729 struct cqe_rx_t *cq_desc; | 745 struct cqe_rx_t *cq_desc; |
| 746 struct lro_ctrl *lro; 747 struct lro_entry *queued; 748 int rq_idx; |
|
| 730 int cmp_err; 731 732 NICVF_CMP_LOCK(cq); 733 cmp_err = 0; 734 processed_cqe = 0; 735 /* Get no of valid CQ entries to process */ 736 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); 737 cqe_count &= CQ_CQE_COUNT; --- 58 unchanged lines hidden (view full) --- 796 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, cq_idx, processed_cqe); 797 798 if ((tx_done > 0) && 799 ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0)) { 800 /* Reenable TXQ if its stopped earlier due to SQ full */ 801 if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 802 } 803out: | 749 int cmp_err; 750 751 NICVF_CMP_LOCK(cq); 752 cmp_err = 0; 753 processed_cqe = 0; 754 /* Get no of valid CQ entries to process */ 755 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); 756 cqe_count &= CQ_CQE_COUNT; --- 58 unchanged lines hidden (view full) --- 815 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, cq_idx, processed_cqe); 816 817 if ((tx_done > 0) && 818 ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0)) { 819 /* Reenable TXQ if its stopped earlier due to SQ full */ 820 if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 821 } 822out: |
| 823 /* 824 * Flush any outstanding LRO work 825 */ 826 rq_idx = cq_idx; 827 rq = &nic->qs->rq[rq_idx]; 828 lro = &rq->lro; 829 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 830 SLIST_REMOVE_HEAD(&lro->lro_active, next); 831 tcp_lro_flush(lro, queued); 832 } 833 |
|
| 804 NICVF_CMP_UNLOCK(cq); 805 806 ifp = nic->ifp; 807 /* Push received MBUFs to the stack */ 808 while (!buf_ring_empty(cq->rx_br)) { 809 mbuf = buf_ring_dequeue_mc(cq->rx_br); 810 if (__predict_true(mbuf != NULL)) 811 (*ifp->if_input)(ifp, mbuf); --- 424 unchanged lines hidden (view full) --- 1236/* Configures receive queue */ 1237static void 1238nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, 1239 int qidx, bool enable) 1240{ 1241 union nic_mbx mbx = {}; 1242 struct rcv_queue *rq; 1243 struct rq_cfg rq_cfg; | 834 NICVF_CMP_UNLOCK(cq); 835 836 ifp = nic->ifp; 837 /* Push received MBUFs to the stack */ 838 while (!buf_ring_empty(cq->rx_br)) { 839 mbuf = buf_ring_dequeue_mc(cq->rx_br); 840 if (__predict_true(mbuf != NULL)) 841 (*ifp->if_input)(ifp, mbuf); --- 424 unchanged lines hidden (view full) --- 1266/* Configures receive queue */ 1267static void 1268nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, 1269 int qidx, bool enable) 1270{ 1271 union nic_mbx mbx = {}; 1272 struct rcv_queue *rq; 1273 struct rq_cfg rq_cfg; |
| 1274 struct ifnet *ifp; 1275 struct lro_ctrl *lro; |
|
| 1244 | 1276 |
| 1277 ifp = nic->ifp; 1278 |
|
| 1245 rq = &qs->rq[qidx]; 1246 rq->enable = enable; 1247 | 1279 rq = &qs->rq[qidx]; 1280 rq->enable = enable; 1281 |
| 1282 lro = &rq->lro; 1283 |
|
| 1248 /* Disable receive queue */ 1249 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); 1250 1251 if (!rq->enable) { 1252 nicvf_reclaim_rcv_queue(nic, qs, qidx); | 1284 /* Disable receive queue */ 1285 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); 1286 1287 if (!rq->enable) { 1288 nicvf_reclaim_rcv_queue(nic, qs, qidx); |
| 1289 /* Free LRO memory */ 1290 tcp_lro_free(lro); 1291 rq->lro_enabled = FALSE; |
|
| 1253 return; 1254 } 1255 | 1292 return; 1293 } 1294 |
| 1295 /* Configure LRO if enabled */ 1296 rq->lro_enabled = FALSE; 1297 if ((if_getcapenable(ifp) & IFCAP_LRO) != 0) { 1298 if (tcp_lro_init(lro) != 0) { 1299 device_printf(nic->dev, 1300 "Failed to initialize LRO for RXQ%d\n", qidx); 1301 } else { 1302 rq->lro_enabled = TRUE; 1303 lro->ifp = nic->ifp; 1304 } 1305 } 1306 |
|
| 1256 rq->cq_qs = qs->vnic_id; 1257 rq->cq_idx = qidx; 1258 rq->start_rbdr_qs = qs->vnic_id; 1259 rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1; 1260 rq->cont_rbdr_qs = qs->vnic_id; 1261 rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1; 1262 /* all writes of RBDR data to be loaded into L2 Cache as well*/ 1263 rq->caching = 1; --- 1004 unchanged lines hidden --- | 1307 rq->cq_qs = qs->vnic_id; 1308 rq->cq_idx = qidx; 1309 rq->start_rbdr_qs = qs->vnic_id; 1310 rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1; 1311 rq->cont_rbdr_qs = qs->vnic_id; 1312 rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1; 1313 /* all writes of RBDR data to be loaded into L2 Cache as well*/ 1314 rq->caching = 1; --- 1004 unchanged lines hidden --- |