dev/ixgbe/ixgbe.c

1512 	struct rx_ring	*rxr = que->rxr;
1553         if ((txr->bytes == 0) && (rxr->bytes == 0))
1558 	if ((rxr->bytes) && (rxr->packets))
1560 		    (rxr->bytes / rxr->packets));
1583         rxr->bytes = 0;
1584         rxr->packets = 0;
2822 	struct rx_ring	*rxr;
2910 		rxr = &adapter->rx_rings[i];
2912 		rxr->adapter = adapter;
2913 		rxr->me = i;
2914 		rxr->num_desc = adapter->num_rx_desc;
2917 		snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2918 		    device_get_nameunit(dev), rxr->me);
2919 		mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2922 			&rxr->rxdma, BUS_DMA_NOWAIT)) {
2928 		rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2929 		bzero((void *)rxr->rx_base, rsize);
2932 		if (ixgbe_allocate_receive_buffers(rxr)) {
2947 		que->rxr = &adapter->rx_rings[i];
2953 	for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2954 		ixgbe_dma_free(adapter, &rxr->rxdma);
3745 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3747 	struct adapter		*adapter = rxr->adapter;
3754 	i = j = rxr->next_to_refresh;
3756 	if (++j == rxr->num_desc)
3760 		rxbuf = &rxr->rx_buffers[i];
3763 			    M_PKTHDR, rxr->mbuf_sz);
3771 		mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3778 			error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3788 			bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3790 			rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3793 			rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3800 		rxr->next_to_refresh = i;
3801 		if (++j == rxr->num_desc)
3807 		    IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3820 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3822 	struct	adapter 	*adapter = rxr->adapter;
3827 	bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3828 	if (!(rxr->rx_buffers =
3847 				   &rxr->ptag))) {
3852 	for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3853 		rxbuf = &rxr->rx_buffers[i];
3854 		error = bus_dmamap_create(rxr->ptag,
3893 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3895 	struct	adapter 	*adapter = rxr->adapter;
3901 		rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3915 	rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3921 	if (rxr->mbuf_sz == MCLBYTES)
3923 	else if (rxr->mbuf_sz == MJUMPAGESIZE)
3925 	else if (rxr->mbuf_sz == MJUM9BYTES)
3930 	IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3941 	rxr->hw_rsc = TRUE;
3946 ixgbe_free_receive_ring(struct rx_ring *rxr)
3951 	for (i = 0; i < rxr->num_desc; i++) {
3952 		rxbuf = &rxr->rx_buffers[i];
3954 			bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3956 			bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3972 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3979 	struct lro_ctrl		*lro = &rxr->lro;
3982 	struct netmap_adapter *na = NA(rxr->adapter->ifp);
3986 	adapter = rxr->adapter;
3991 	IXGBE_RX_LOCK(rxr);
3994 	slot = netmap_reset(na, NR_RX, rxr->me, 0);
3998 	bzero((void *)rxr->rx_base, rsize);
4000 	rxr->mbuf_sz = adapter->rx_mbuf_sz;
4003 	ixgbe_free_receive_ring(rxr);
4006 	for (int j = 0; j != rxr->num_desc; ++j) {
4009 		rxbuf = &rxr->rx_buffers[j];
4019 			int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4024 			netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4026 			rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4039 		mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4041 		error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4046 		bus_dmamap_sync(rxr->ptag,
4049 		rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4055 	rxr->next_to_check = 0;
4056 	rxr->next_to_refresh = 0;
4057 	rxr->lro_enabled = FALSE;
4058 	rxr->rx_copies = 0;
4059 	rxr->rx_bytes = 0;
4060 	rxr->discard = FALSE;
4061 	rxr->vtag_strip = FALSE;
4063 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4070 		ixgbe_setup_hw_rsc(rxr);
4078 		rxr->lro_enabled = TRUE;
4082 	IXGBE_RX_UNLOCK(rxr);
4086 	ixgbe_free_receive_ring(rxr);
4087 	IXGBE_RX_UNLOCK(rxr);
4099 	struct rx_ring *rxr = adapter->rx_rings;
4102 	for (j = 0; j < adapter->num_queues; j++, rxr++)
4103 		if (ixgbe_setup_receive_ring(rxr))
4114 		rxr = &adapter->rx_rings[i];
4115 		ixgbe_free_receive_ring(rxr);
4133 	struct	rx_ring	*rxr = adapter->rx_rings;
4173 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4174 		u64 rdba = rxr->rxdma.dma_paddr;
4196 		rxr->process_limit = ixgbe_rx_process_limit;
4265 	struct rx_ring *rxr = adapter->rx_rings;
4269 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4270 		struct lro_ctrl		*lro = &rxr->lro;
4271 		ixgbe_free_receive_buffers(rxr);
4275 		ixgbe_dma_free(adapter, &rxr->rxdma);
4288 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4290 	struct adapter		*adapter = rxr->adapter;
4296 	if (rxr->rx_buffers != NULL) {
4298 			rxbuf = &rxr->rx_buffers[i];
4300 				bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4302 				bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4308 				bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4312 		if (rxr->rx_buffers != NULL) {
4313 			free(rxr->rx_buffers, M_DEVBUF);
4314 			rxr->rx_buffers = NULL;
4318 	if (rxr->ptag != NULL) {
4319 		bus_dma_tag_destroy(rxr->ptag);
4320 		rxr->ptag = NULL;
4327 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4335         if (rxr->lro_enabled &&
4350                 if (rxr->lro.lro_cnt != 0)
4351                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4354 	IXGBE_RX_UNLOCK(rxr);
4356 	IXGBE_RX_LOCK(rxr);
4360 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4364 	rbuf = &rxr->rx_buffers[i];
4405 	struct rx_ring		*rxr = que->rxr;
4407 	struct lro_ctrl		*lro = &rxr->lro;
4411 	u16			count = rxr->process_limit;
4415 	IXGBE_RX_LOCK(rxr);
4419 	if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4420 		IXGBE_RX_UNLOCK(rxr);
4425 	for (i = rxr->next_to_check; count != 0;) {
4433 		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4436 		cur = &rxr->rx_base[i];
4449 		rbuf = &rxr->rx_buffers[i];
4459 		    (rxr->discard)) {
4460 			rxr->rx_discarded++;
4462 				rxr->discard = FALSE;
4464 				rxr->discard = TRUE;
4465 			ixgbe_rx_discard(rxr, i);
4486 			if (rxr->hw_rsc == TRUE) {
4488 				rxr->rsc_num += (rsc - 1);
4499 			nbuf = &rxr->rx_buffers[nextp];
4533 					rxr->rx_copies++;
4556 			rxr->rx_packets++;
4558 			rxr->bytes += sendmp->m_pkthdr.len;
4559 			rxr->rx_bytes += sendmp->m_pkthdr.len;
4561 			if ((rxr->vtag_strip) &&
4576 		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4580 		if (++i == rxr->num_desc)
4585 			rxr->next_to_check = i;
4586 			ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4587 			i = rxr->next_to_check;
4592 			ixgbe_refresh_mbufs(rxr, i);
4598 	if (ixgbe_rx_unrefreshed(rxr))
4599 		ixgbe_refresh_mbufs(rxr, i);
4601 	rxr->next_to_check = i;
4611 	IXGBE_RX_UNLOCK(rxr);
4726 	struct rx_ring	*rxr;
4760 		rxr = &adapter->rx_rings[i];
4767 		rxr->vtag_strip = TRUE;
5363 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5364 	if (!rxr) return 0;
5366 	unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5381 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5382 	if (!rxr) return 0;
5384 	unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5429 	struct rx_ring *rxr = adapter->rx_rings;
5492 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5498 		struct lro_ctrl *lro = &rxr->lro;
5506 				CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5510 				CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5514 				CTLFLAG_RD, &rxr->rx_packets,
5517 				CTLFLAG_RD, &rxr->rx_bytes,
5520 				CTLFLAG_RD, &rxr->rx_copies,