virtio/network/if_vtnet.c

716 	struct vtnet_txq *txq;
718 	txq = &sc->vtnet_txqs[id];
720 	snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d",
722 	mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF);
724 	txq->vtntx_sc = sc;
725 	txq->vtntx_id = id;
727 	txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT);
728 	if (txq->vtntx_sg == NULL)
732 	txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF,
733 	    M_NOWAIT, &txq->vtntx_mtx);
734 	if (txq->vtntx_br == NULL)
737 	TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq);
739 	TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq);
740 	txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT,
741 	    taskqueue_thread_enqueue, &txq->vtntx_tq);
742 	if (txq->vtntx_tq == NULL)
793 vtnet_destroy_txq(struct vtnet_txq *txq)
796 	txq->vtntx_sc = NULL;
797 	txq->vtntx_id = -1;
799 	if (txq->vtntx_sg != NULL) {
800 		sglist_free(txq->vtntx_sg);
801 		txq->vtntx_sg = NULL;
805 	if (txq->vtntx_br != NULL) {
806 		buf_ring_free(txq->vtntx_br, M_DEVBUF);
807 		txq->vtntx_br = NULL;
811 	if (mtx_initialized(&txq->vtntx_mtx) != 0)
812 		mtx_destroy(&txq->vtntx_mtx);
877 	struct vtnet_txq *txq;
897 		txq = &sc->vtnet_txqs[i];
899 		    vtnet_tx_vq_intr, txq, &txq->vtntx_vq,
900 		    "%s-%d tx", device_get_nameunit(dev), txq->vtntx_id);
1929 vtnet_txq_below_threshold(struct vtnet_txq *txq)
1934 	sc = txq->vtntx_sc;
1935 	vq = txq->vtntx_vq;
1941 vtnet_txq_notify(struct vtnet_txq *txq)
1945 	vq = txq->vtntx_vq;
1947 	txq->vtntx_watchdog = VTNET_TX_TIMEOUT;
1950 	if (vtnet_txq_enable_intr(txq) == 0)
1958 	if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) {
1967 vtnet_txq_free_mbufs(struct vtnet_txq *txq)
1973 	int netmap_bufs = vtnet_netmap_queue_on(txq->vtntx_sc, NR_TX,
1974 						txq->vtntx_id);
1979 	vq = txq->vtntx_vq;
1990 	    ("%s: mbufs remaining in tx queue %p", __func__, txq));
1998 vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m,
2005 	sc = txq->vtntx_sc;
2051 vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type,
2059 	sc = txq->vtntx_sc;
2087 	txq->vtntx_stats.vtxs_tso++;
2093 vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m,
2099 	sc = txq->vtntx_sc;
2102 	error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start);
2115 		txq->vtntx_stats.vtxs_csum++;
2129 		error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr);
2142 vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
2151 	sc = txq->vtntx_sc;
2152 	vq = txq->vtntx_vq;
2153 	sg = txq->vtntx_sg;
2189 vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head)
2223 		m = vtnet_txq_offload(txq, m, hdr);
2230 	error = vtnet_txq_enqueue_buf(txq, m_head, txhdr);
2243 vtnet_start_locked(struct vtnet_txq *txq, struct ifnet *ifp)
2250 	sc = txq->vtntx_sc;
2251 	vq = txq->vtntx_vq;
2254 	VTNET_TXQ_LOCK_ASSERT(txq);
2260 	vtnet_txq_eof(txq);
2273 		if (vtnet_txq_encap(txq, &m0) != 0) {
2283 	if (enq > 0 && vtnet_txq_notify(txq) != 0) {
2287 		txq->vtntx_stats.vtxs_rescheduled++;
2288 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
2296 	struct vtnet_txq *txq;
2299 	txq = &sc->vtnet_txqs[0];
2301 	VTNET_TXQ_LOCK(txq);
2302 	vtnet_start_locked(txq, ifp);
2303 	VTNET_TXQ_UNLOCK(txq);
2309 vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m)
2317 	sc = txq->vtntx_sc;
2318 	vq = txq->vtntx_vq;
2319 	br = txq->vtntx_br;
2324 	VTNET_TXQ_LOCK_ASSERT(txq);
2339 	vtnet_txq_eof(txq);
2350 		if (vtnet_txq_encap(txq, &m) != 0) {
2363 	if (enq > 0 && vtnet_txq_notify(txq) != 0) {
2367 		txq->vtntx_stats.vtxs_rescheduled++;
2368 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask);
2378 	struct vtnet_txq *txq;
2390 	txq = &sc->vtnet_txqs[i];
2392 	if (VTNET_TXQ_TRYLOCK(txq) != 0) {
2393 		error = vtnet_txq_mq_start_locked(txq, m);
2394 		VTNET_TXQ_UNLOCK(txq);
2396 		error = drbr_enqueue(ifp, txq->vtntx_br, m);
2397 		taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask);
2407 	struct vtnet_txq *txq;
2409 	txq = xtxq;
2410 	sc = txq->vtntx_sc;
2412 	VTNET_TXQ_LOCK(txq);
2413 	if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br))
2414 		vtnet_txq_mq_start_locked(txq, NULL);
2415 	VTNET_TXQ_UNLOCK(txq);
2421 vtnet_txq_start(struct vtnet_txq *txq)
2426 	sc = txq->vtntx_sc;
2431 		vtnet_start_locked(txq, ifp);
2433 	if (!drbr_empty(ifp, txq->vtntx_br))
2434 		vtnet_txq_mq_start_locked(txq, NULL);
2442 	struct vtnet_txq *txq;
2445 	txq = xtxq;
2446 	sc = txq->vtntx_sc;
2449 	VTNET_TXQ_LOCK(txq);
2452 		VTNET_TXQ_UNLOCK(txq);
2456 	vtnet_txq_eof(txq);
2457 	vtnet_txq_start(txq);
2459 	VTNET_TXQ_UNLOCK(txq);
2463 vtnet_txq_eof(struct vtnet_txq *txq)
2470 	vq = txq->vtntx_vq;
2472 	VTNET_TXQ_LOCK_ASSERT(txq);
2478 		txq->vtntx_stats.vtxs_opackets++;
2479 		txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len;
2481 			txq->vtntx_stats.vtxs_omcasts++;
2488 		txq->vtntx_watchdog = 0;
2497 	struct vtnet_txq *txq;
2500 	txq = xtxq;
2501 	sc = txq->vtntx_sc;
2504 	if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) {
2511 		vtnet_txq_disable_intr(txq);
2516 	if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS)
2520 	VTNET_TXQ_LOCK(txq);
2523 		VTNET_TXQ_UNLOCK(txq);
2527 	vtnet_txq_eof(txq);
2528 	vtnet_txq_start(txq);
2530 	VTNET_TXQ_UNLOCK(txq);
2536 	struct vtnet_txq *txq;
2542 		txq = &sc->vtnet_txqs[i];
2544 		VTNET_TXQ_LOCK(txq);
2545 		vtnet_txq_start(txq);
2546 		VTNET_TXQ_UNLOCK(txq);
2555 	struct vtnet_txq *txq;
2562 		txq = &sc->vtnet_txqs[i];
2564 		VTNET_TXQ_LOCK(txq);
2565 		while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL)
2567 		VTNET_TXQ_UNLOCK(txq);
2575 vtnet_watchdog(struct vtnet_txq *txq)
2579 	ifp = txq->vtntx_sc->vtnet_ifp;
2581 	VTNET_TXQ_LOCK(txq);
2582 	if (txq->vtntx_watchdog == 1) {
2591 		if (vtnet_txq_eof(txq) != 0)
2592 			vtnet_txq_start(txq);
2595 	if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) {
2596 		VTNET_TXQ_UNLOCK(txq);
2599 	VTNET_TXQ_UNLOCK(txq);
2601 	if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id);
2692 	struct vtnet_txq *txq;
2715 		txq = &sc->vtnet_txqs[i];
2716 		error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET,
2717 		    "%s txq %d", device_get_nameunit(dev), txq->vtntx_id);
2720 			    txq->vtntx_id);
2729 	struct vtnet_txq *txq;
2739 		txq = &sc->vtnet_txqs[i];
2740 		if (txq->vtntx_tq != NULL) {
2741 			taskqueue_free(txq->vtntx_tq);
2742 			txq->vtntx_tq = NULL;
2751 	struct vtnet_txq *txq;
2759 		txq = &sc->vtnet_txqs[i];
2760 		if (txq->vtntx_tq != NULL) {
2761 			taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask);
2763 			taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask);
2773 	struct vtnet_txq *txq;
2780 		txq = &sc->vtnet_txqs[i];
2781 		vtnet_txq_free_mbufs(txq);
2789 	struct vtnet_txq *txq;
2803 		txq = &sc->vtnet_txqs[i];
2804 		VTNET_TXQ_LOCK(txq);
2805 		VTNET_TXQ_UNLOCK(txq);
2961 	struct vtnet_txq *txq;
2965 		txq = &sc->vtnet_txqs[i];
2966 		txq->vtntx_watchdog = 0;
3712     struct sysctl_oid_list *child, struct vtnet_txq *txq)
3719 	snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id);
3724 	stats = &txq->vtntx_stats;
3883 vtnet_txq_enable_intr(struct vtnet_txq *txq)
3887 	vq = txq->vtntx_vq;
3889 	if (vtnet_txq_below_threshold(txq) != 0)
3900 vtnet_txq_disable_intr(struct vtnet_txq *txq)
3903 	virtqueue_disable_intr(txq->vtntx_vq);