dev/e1000/if_igb.c

1597 	struct rx_ring *rxr = que->rxr;
1638         if ((txr->bytes == 0) && (rxr->bytes == 0))
1647 		if ((rxr->bytes) && (rxr->packets))
1649 			    (rxr->bytes / rxr->packets));
1671         rxr->bytes = 0;
1672         rxr->packets = 0;
3296 	struct rx_ring		*rxr = NULL;
3376 		rxr = &adapter->rx_rings[i];
3377 		rxr->adapter = adapter;
3378 		rxr->me = i;
3381 		snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3383 		mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3386 			&rxr->rxdma, BUS_DMA_NOWAIT)) {
3392 		rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3393 		bzero((void *)rxr->rx_base, rsize);
3396 		if (igb_allocate_receive_buffers(rxr)) {
3411 		que->rxr = &adapter->rx_rings[i];
3417 	for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3418 		igb_dma_free(adapter, &rxr->rxdma);
4084 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
4086 	struct adapter		*adapter = rxr->adapter;
4094 	i = j = rxr->next_to_refresh;
4104 		rxbuf = &rxr->rx_buffers[i];
4106 		if (rxr->hdr_split == FALSE)
4119 		error = bus_dmamap_load_mbuf_sg(rxr->htag,
4129 		bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4131 		rxr->rx_base[i].read.hdr_addr =
4144 		error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4154 		bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4156 		rxr->rx_base[i].read.pkt_addr =
4161 		rxr->next_to_refresh = i;
4168 		    E1000_RDT(rxr->me), rxr->next_to_refresh);
4182 igb_allocate_receive_buffers(struct rx_ring *rxr)
4184 	struct	adapter 	*adapter = rxr->adapter;
4190 	if (!(rxr->rx_buffers =
4209 				   &rxr->htag))) {
4225 				   &rxr->ptag))) {
4231 		rxbuf = &rxr->rx_buffers[i];
4232 		error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
4238 		error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
4256 igb_free_receive_ring(struct rx_ring *rxr)
4258 	struct	adapter		*adapter = rxr->adapter;
4263 		rxbuf = &rxr->rx_buffers[i];
4265 			bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4267 			bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4272 			bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4274 			bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4290 igb_setup_receive_ring(struct rx_ring *rxr)
4297 	struct lro_ctrl		*lro = &rxr->lro;
4300 	struct netmap_adapter *na = NA(rxr->adapter->ifp);
4304 	adapter = rxr->adapter;
4309 	IGB_RX_LOCK(rxr);
4311 	slot = netmap_reset(na, NR_RX, rxr->me, 0);
4315 	bzero((void *)rxr->rx_base, rsize);
4320 	igb_free_receive_ring(rxr);
4324 		rxr->hdr_split = TRUE;
4330 		rxbuf = &rxr->rx_buffers[j];
4334 			int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4339 			netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
4341 			rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4345 		if (rxr->hdr_split == FALSE)
4359 		error = bus_dmamap_load_mbuf_sg(rxr->htag,
4364 		bus_dmamap_sync(rxr->htag,
4367 		rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4380 		error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4385 		bus_dmamap_sync(rxr->ptag,
4388 		rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4392 	rxr->next_to_check = 0;
4393 	rxr->next_to_refresh = adapter->num_rx_desc - 1;
4394 	rxr->lro_enabled = FALSE;
4395 	rxr->rx_split_packets = 0;
4396 	rxr->rx_bytes = 0;
4398 	rxr->fmp = NULL;
4399 	rxr->lmp = NULL;
4400 	rxr->discard = FALSE;
4402 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4418 		rxr->lro_enabled = TRUE;
4422 	IGB_RX_UNLOCK(rxr);
4426 	igb_free_receive_ring(rxr);
4427 	IGB_RX_UNLOCK(rxr);
4440 	struct rx_ring *rxr = adapter->rx_rings;
4443 	for (i = 0; i < adapter->num_queues; i++, rxr++)
4444 		if (igb_setup_receive_ring(rxr))
4455 		rxr = &adapter->rx_rings[j];
4456 		IGB_RX_LOCK(rxr);
4457 		igb_free_receive_ring(rxr);
4458 		IGB_RX_UNLOCK(rxr);
4472 	struct rx_ring	*rxr = adapter->rx_rings;
4521 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4522 		u64 bus_addr = rxr->rxdma.dma_paddr;
4627 		rxr = &adapter->rx_rings[i];
4628 		E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4640 			int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
4649 		E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4662 	struct rx_ring *rxr = adapter->rx_rings;
4664 	for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4665 		struct lro_ctrl	*lro = &rxr->lro;
4666 		igb_free_receive_buffers(rxr);
4668 		igb_dma_free(adapter, &rxr->rxdma);
4680 igb_free_receive_buffers(struct rx_ring *rxr)
4682 	struct adapter		*adapter = rxr->adapter;
4689 	if (rxr->rx_buffers != NULL) {
4691 			rxbuf = &rxr->rx_buffers[i];
4693 				bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4695 				bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4700 				bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4702 				bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4709 				bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4713 				bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4717 		if (rxr->rx_buffers != NULL) {
4718 			free(rxr->rx_buffers, M_DEVBUF);
4719 			rxr->rx_buffers = NULL;
4723 	if (rxr->htag != NULL) {
4724 		bus_dma_tag_destroy(rxr->htag);
4725 		rxr->htag = NULL;
4727 	if (rxr->ptag != NULL) {
4728 		bus_dma_tag_destroy(rxr->ptag);
4729 		rxr->ptag = NULL;
4734 igb_rx_discard(struct rx_ring *rxr, int i)
4738 	rbuf = &rxr->rx_buffers[i];
4741 	if (rxr->fmp != NULL) {
4742 		rxr->fmp->m_flags |= M_PKTHDR;
4743 		m_freem(rxr->fmp);
4744 		rxr->fmp = NULL;
4745 		rxr->lmp = NULL;
4758 		bus_dmamap_unload(rxr->htag, rbuf->hmap);
4764 		bus_dmamap_unload(rxr->ptag, rbuf->pmap);
4771 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4779 	if (rxr->lro_enabled &&
4792 		if (rxr->lro.lro_cnt != 0)
4793 			if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4796 	IGB_RX_UNLOCK(rxr);
4798 	IGB_RX_LOCK(rxr);
4816 	struct rx_ring		*rxr = que->rxr;
4818 	struct lro_ctrl		*lro = &rxr->lro;
4824 	IGB_RX_LOCK(rxr);
4826 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4830 	if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4831 		IGB_RX_UNLOCK(rxr);
4837 	for (i = rxr->next_to_check; count != 0;) {
4843 		cur = &rxr->rx_base[i];
4852 		rxbuf = &rxr->rx_buffers[i];
4866 		    (rxr->discard)) {
4868 			++rxr->rx_discarded;
4870 				rxr->discard = TRUE;
4872 				rxr->discard = FALSE;
4873 			igb_rx_discard(rxr, i);
4888 		if (rxr->hdr_split && rxr->fmp == NULL) {
4889 			bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4894 			mh = rxr->rx_buffers[i].m_head;
4904 				mp = rxr->rx_buffers[i].m_pack;
4909 				rxr->rx_split_packets++;
4917 			mh = rxr->rx_buffers[i].m_pack;
4922 		bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4927 		if (rxr->fmp == NULL) {
4930 			rxr->fmp = mh;
4931 			rxr->lmp = mh;
4935 				rxr->lmp = mh->m_next;
4939 			rxr->lmp->m_next = mh;
4940 			rxr->lmp = rxr->lmp->m_next;
4941 			rxr->fmp->m_pkthdr.len += mh->m_len;
4945 			rxr->fmp->m_pkthdr.rcvif = ifp;
4947 			rxr->rx_packets++;
4949 			rxr->packets++;
4950 			rxr->bytes += rxr->fmp->m_pkthdr.len;
4951 			rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4954 				igb_rx_checksum(staterr, rxr->fmp, ptype);
4958 				rxr->fmp->m_pkthdr.ether_vtag = vtag;
4959 				rxr->fmp->m_flags |= M_VLANTAG;
4962 			rxr->fmp->m_pkthdr.flowid = que->msix;
4963 			rxr->fmp->m_flags |= M_FLOWID;
4965 			sendmp = rxr->fmp;
4968 			rxr->fmp = NULL;
4969 			rxr->lmp = NULL;
4973 		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4983 			rxr->next_to_check = i;
4984 			igb_rx_input(rxr, ifp, sendmp, ptype);
4985 			i = rxr->next_to_check;
4991                         igb_refresh_mbufs(rxr, i);
4997 	if (igb_rx_unrefreshed(rxr))
4998 		igb_refresh_mbufs(rxr, i);
5000 	rxr->next_to_check = i;
5013 	IGB_RX_UNLOCK(rxr);
5579 	struct rx_ring *rxr = adapter->rx_rings;
5634 	for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5635 		struct lro_ctrl *lro = &rxr->lro;
5664 				CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5668 				CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5672 				CTLFLAG_RD, &rxr->rx_packets,
5675 				CTLFLAG_RD, &rxr->rx_bytes,