virtio/network/if_vtnet.c

691 	struct vtnet_rxq *rxq;
693 	rxq = &sc->vtnet_rxqs[id];
695 	snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d",
697 	mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF);
699 	rxq->vtnrx_sc = sc;
700 	rxq->vtnrx_id = id;
702 	rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT);
703 	if (rxq->vtnrx_sg == NULL)
706 	TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);
707 	rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT,
708 	    taskqueue_thread_enqueue, &rxq->vtnrx_tq);
710 	return (rxq->vtnrx_tq == NULL ? ENOMEM : 0);
777 vtnet_destroy_rxq(struct vtnet_rxq *rxq)
780 	rxq->vtnrx_sc = NULL;
781 	rxq->vtnrx_id = -1;
783 	if (rxq->vtnrx_sg != NULL) {
784 		sglist_free(rxq->vtnrx_sg);
785 		rxq->vtnrx_sg = NULL;
788 	if (mtx_initialized(&rxq->vtnrx_mtx) != 0)
789 		mtx_destroy(&rxq->vtnrx_mtx);
876 	struct vtnet_rxq *rxq;
892 		rxq = &sc->vtnet_rxqs[i];
894 		    vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq,
895 		    "%s-%d rx", device_get_nameunit(dev), rxq->vtnrx_id);
1183 vtnet_rxq_populate(struct vtnet_rxq *rxq)
1189 	error = vtnet_netmap_rxq_populate(rxq);
1194 	vq = rxq->vtnrx_vq;
1198 		error = vtnet_rxq_new_buf(rxq);
1218 vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq)
1224 	int netmap_bufs = vtnet_netmap_queue_on(rxq->vtnrx_sc, NR_RX,
1225 						rxq->vtnrx_id);
1230 	vq = rxq->vtnrx_vq;
1239 	    ("%s: mbufs remaining in rx queue %p", __func__, rxq));
1287 vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *rxq, struct mbuf *m0,
1295 	sc = rxq->vtnrx_sc;
1353 	error = vtnet_rxq_enqueue_buf(rxq, m_new);
1378 vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len)
1384 	sc = rxq->vtnrx_sc;
1398 		error = vtnet_rxq_enqueue_buf(rxq, m_new);
1410 		error = vtnet_rxq_replace_lro_nomgr_buf(rxq, m, len);
1416 vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)
1424 	sc = rxq->vtnrx_sc;
1425 	sg = rxq->vtnrx_sg;
1428 	VTNET_RXQ_LOCK_ASSERT(rxq);
1450 	error = virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg);
1456 vtnet_rxq_new_buf(struct vtnet_rxq *rxq)
1462 	sc = rxq->vtnrx_sc;
1468 	error = vtnet_rxq_enqueue_buf(rxq, m);
1480 vtnet_rxq_csum_by_offset(struct vtnet_rxq *rxq, struct mbuf *m,
1488 	sc = rxq->vtnrx_sc;
1530 vtnet_rxq_csum_by_parse(struct vtnet_rxq *rxq, struct mbuf *m,
1536 	sc = rxq->vtnrx_sc;
1606 vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,
1625 		error = vtnet_rxq_csum_by_offset(rxq, m, eth_type, offset, hdr);
1627 		error = vtnet_rxq_csum_by_parse(rxq, m, eth_type, offset, hdr);
1633 vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs)
1638 		m = virtqueue_dequeue(rxq->vtnrx_vq, NULL);
1641 		vtnet_rxq_discard_buf(rxq, m);
1646 vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m)
1654 	error = vtnet_rxq_enqueue_buf(rxq, m);
1660 vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
1667 	sc = rxq->vtnrx_sc;
1668 	vq = rxq->vtnrx_vq;
1674 			rxq->vtnrx_stats.vrxs_ierrors++;
1678 		if (vtnet_rxq_new_buf(rxq) != 0) {
1679 			rxq->vtnrx_stats.vrxs_iqdrops++;
1680 			vtnet_rxq_discard_buf(rxq, m);
1682 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);
1707 vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
1714 	sc = rxq->vtnrx_sc;
1730 	m->m_pkthdr.flowid = rxq->vtnrx_id;
1740 		if (vtnet_rxq_csum(rxq, m, hdr) == 0)
1741 			rxq->vtnrx_stats.vrxs_csum++;
1743 			rxq->vtnrx_stats.vrxs_csum_failed++;
1746 	rxq->vtnrx_stats.vrxs_ipackets++;
1747 	rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;
1749 	VTNET_RXQ_UNLOCK(rxq);
1751 	VTNET_RXQ_LOCK(rxq);
1755 vtnet_rxq_eof(struct vtnet_rxq *rxq)
1765 	sc = rxq->vtnrx_sc;
1766 	vq = rxq->vtnrx_vq;
1772 	VTNET_RXQ_LOCK_ASSERT(rxq);
1781 			rxq->vtnrx_stats.vrxs_ierrors++;
1782 			vtnet_rxq_discard_buf(rxq, m);
1800 		if (vtnet_rxq_replace_buf(rxq, m, len) != 0) {
1801 			rxq->vtnrx_stats.vrxs_iqdrops++;
1802 			vtnet_rxq_discard_buf(rxq, m);
1804 				vtnet_rxq_discard_merged_bufs(rxq, nbufs);
1814 			if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0)
1830 		vtnet_rxq_input(rxq, m, hdr);
1847 	struct vtnet_rxq *rxq;
1851 	rxq = xrxq;
1852 	sc = rxq->vtnrx_sc;
1856 	if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) {
1863 		vtnet_rxq_disable_intr(rxq);
1868 	if (netmap_rx_irq(ifp, rxq->vtnrx_id, &more) != NM_IRQ_PASS)
1872 	VTNET_RXQ_LOCK(rxq);
1876 		VTNET_RXQ_UNLOCK(rxq);
1880 	more = vtnet_rxq_eof(rxq);
1881 	if (more || vtnet_rxq_enable_intr(rxq) != 0) {
1883 			vtnet_rxq_disable_intr(rxq);
1891 		VTNET_RXQ_UNLOCK(rxq);
1892 		rxq->vtnrx_stats.vrxs_rescheduled++;
1893 		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
1895 		VTNET_RXQ_UNLOCK(rxq);
1902 	struct vtnet_rxq *rxq;
1906 	rxq = xrxq;
1907 	sc = rxq->vtnrx_sc;
1910 	VTNET_RXQ_LOCK(rxq);
1913 		VTNET_RXQ_UNLOCK(rxq);
1917 	more = vtnet_rxq_eof(rxq);
1918 	if (more || vtnet_rxq_enable_intr(rxq) != 0) {
1920 			vtnet_rxq_disable_intr(rxq);
1921 		rxq->vtnrx_stats.vrxs_rescheduled++;
1922 		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
1925 	VTNET_RXQ_UNLOCK(rxq);
2691 	struct vtnet_rxq *rxq;
2707 		rxq = &sc->vtnet_rxqs[i];
2708 		error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET,
2709 		    "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id);
2712 			    rxq->vtnrx_id);
2728 	struct vtnet_rxq *rxq;
2733 		rxq = &sc->vtnet_rxqs[i];
2734 		if (rxq->vtnrx_tq != NULL) {
2735 			taskqueue_free(rxq->vtnrx_tq);
2736 			rxq->vtnrx_tq = NULL;
2750 	struct vtnet_rxq *rxq;
2755 		rxq = &sc->vtnet_rxqs[i];
2756 		if (rxq->vtnrx_tq != NULL)
2757 			taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
2772 	struct vtnet_rxq *rxq;
2777 		rxq = &sc->vtnet_rxqs[i];
2778 		vtnet_rxq_free_mbufs(rxq);
2788 	struct vtnet_rxq *rxq;
2799 		rxq = &sc->vtnet_rxqs[i];
2800 		VTNET_RXQ_LOCK(rxq);
2801 		VTNET_RXQ_UNLOCK(rxq);
2914 	struct vtnet_rxq *rxq;
2941 		rxq = &sc->vtnet_rxqs[i];
2944 		VTNET_RXQ_LOCK(rxq);
2945 		error = vtnet_rxq_populate(rxq);
2946 		VTNET_RXQ_UNLOCK(rxq);
3679     struct sysctl_oid_list *child, struct vtnet_rxq *rxq)
3686 	snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id);
3691 	stats = &rxq->vtnrx_stats;
3869 vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)
3872 	return (virtqueue_enable_intr(rxq->vtnrx_vq));
3876 vtnet_rxq_disable_intr(struct vtnet_rxq *rxq)
3879 	virtqueue_disable_intr(rxq->vtnrx_vq);