Lines Matching refs:txq
441 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
570 #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
571 #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)
672 &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
729 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
734 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
747 static void iflib_ifmp_purge(iflib_txq_t txq);
1035 iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
1037 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
1067 pi.ipi_segs = txq->ift_segs;
1072 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
1073 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
1121 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
1122 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
1128 netmap_reload_map(na, txq->ift_buf_tag,
1129 txq->ift_sds.ifsd_map[nic_i], addr);
1132 bus_dmamap_sync(txq->ift_buf_tag,
1133 txq->ift_sds.ifsd_map[nic_i],
1143 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
1147 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
1158 if (iflib_tx_credits_update(ctx, txq)) {
1160 nic_i = txq->ift_cidx_processed;
1167 callout_reset_sbt_on(&txq->ift_netmap_timer,
1169 iflib_netmap_timer, txq,
1170 txq->ift_netmap_timer.c_cpu, 0);
1360 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
1365 slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
1376 int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i);
1377 netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i],
1401 iflib_txq_t txq = arg;
1402 if_ctx_t ctx = txq->ift_ctx;
1408 netmap_tx_irq(ctx->ifc_ifp, txq->ift_id);
1414 #define iflib_netmap_txq_init(ctx, txq) (0)
1601 iflib_txq_t txq;
1621 txq = &ctx->ifc_txqs[txqid];
1622 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
1631 GROUPTASK_ENQUEUE(&txq->ift_task);
1716 iflib_txsd_alloc(iflib_txq_t txq)
1718 if_ctx_t ctx = txq->ift_ctx;
1732 MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0);
1753 &txq->ift_buf_tag))) {
1771 &txq->ift_tso_buf_tag))) {
1778 if (!(txq->ift_sds.ifsd_m =
1780 scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
1789 if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc(
1790 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
1797 if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc(
1798 sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
1805 for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
1806 err = bus_dmamap_create(txq->ift_buf_tag, 0,
1807 &txq->ift_sds.ifsd_map[i]);
1814 err = bus_dmamap_create(txq->ift_tso_buf_tag, 0,
1815 &txq->ift_sds.ifsd_tso_map[i]);
1829 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
1833 if (txq->ift_sds.ifsd_map != NULL) {
1834 map = txq->ift_sds.ifsd_map[i];
1835 bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE);
1836 bus_dmamap_unload(txq->ift_buf_tag, map);
1837 bus_dmamap_destroy(txq->ift_buf_tag, map);
1838 txq->ift_sds.ifsd_map[i] = NULL;
1841 if (txq->ift_sds.ifsd_tso_map != NULL) {
1842 map = txq->ift_sds.ifsd_tso_map[i];
1843 bus_dmamap_sync(txq->ift_tso_buf_tag, map,
1845 bus_dmamap_unload(txq->ift_tso_buf_tag, map);
1846 bus_dmamap_destroy(txq->ift_tso_buf_tag, map);
1847 txq->ift_sds.ifsd_tso_map[i] = NULL;
1852 iflib_txq_destroy(iflib_txq_t txq)
1854 if_ctx_t ctx = txq->ift_ctx;
1856 for (int i = 0; i < txq->ift_size; i++)
1857 iflib_txsd_destroy(ctx, txq, i);
1859 if (txq->ift_br != NULL) {
1860 ifmp_ring_free(txq->ift_br);
1861 txq->ift_br = NULL;
1864 mtx_destroy(&txq->ift_mtx);
1866 if (txq->ift_sds.ifsd_map != NULL) {
1867 free(txq->ift_sds.ifsd_map, M_IFLIB);
1868 txq->ift_sds.ifsd_map = NULL;
1870 if (txq->ift_sds.ifsd_tso_map != NULL) {
1871 free(txq->ift_sds.ifsd_tso_map, M_IFLIB);
1872 txq->ift_sds.ifsd_tso_map = NULL;
1874 if (txq->ift_sds.ifsd_m != NULL) {
1875 free(txq->ift_sds.ifsd_m, M_IFLIB);
1876 txq->ift_sds.ifsd_m = NULL;
1878 if (txq->ift_buf_tag != NULL) {
1879 bus_dma_tag_destroy(txq->ift_buf_tag);
1880 txq->ift_buf_tag = NULL;
1882 if (txq->ift_tso_buf_tag != NULL) {
1883 bus_dma_tag_destroy(txq->ift_tso_buf_tag);
1884 txq->ift_tso_buf_tag = NULL;
1886 if (txq->ift_ifdi != NULL) {
1887 free(txq->ift_ifdi, M_IFLIB);
1892 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
1896 mp = &txq->ift_sds.ifsd_m[i];
1900 if (txq->ift_sds.ifsd_map != NULL) {
1901 bus_dmamap_sync(txq->ift_buf_tag,
1902 txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE);
1903 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]);
1905 if (txq->ift_sds.ifsd_tso_map != NULL) {
1906 bus_dmamap_sync(txq->ift_tso_buf_tag,
1907 txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE);
1908 bus_dmamap_unload(txq->ift_tso_buf_tag,
1909 txq->ift_sds.ifsd_tso_map[i]);
1917 iflib_txq_setup(iflib_txq_t txq)
1919 if_ctx_t ctx = txq->ift_ctx;
1926 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
1928 txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;
1931 txq->ift_cidx_processed = 0;
1932 txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
1933 txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];
1935 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
1938 IFDI_TXQ_SETUP(ctx, txq->ift_id);
1939 for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
2429 iflib_txq_t txq = arg;
2430 if_ctx_t ctx = txq->ift_ctx;
2442 if (this_tick - txq->ift_last_timer_tick >= iflib_timer_default) {
2443 txq->ift_last_timer_tick = this_tick;
2444 IFDI_TIMER(ctx, txq->ift_id);
2445 if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
2446 ((txq->ift_cleaned_prev == txq->ift_cleaned) ||
2450 if (txq->ift_qstatus != IFLIB_QUEUE_IDLE &&
2451 ifmp_ring_is_stalled(txq->ift_br)) {
2454 txq->ift_qstatus = IFLIB_QUEUE_HUNG;
2456 txq->ift_cleaned_prev = txq->ift_cleaned;
2459 if (txq->ift_db_pending)
2460 GROUPTASK_ENQUEUE(&txq->ift_task);
2464 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer,
2465 txq, txq->ift_timer.c_cpu);
2471 txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
2516 iflib_txq_t txq;
2542 for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
2543 CALLOUT_LOCK(txq);
2544 callout_stop(&txq->ift_timer);
2546 callout_stop(&txq->ift_netmap_timer);
2548 CALLOUT_UNLOCK(txq);
2549 iflib_netmap_txq_init(ctx, txq);
2581 txq = ctx->ifc_txqs;
2582 for (i = 0; i < sctx->isc_ntxqsets; i++, txq++)
2583 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq,
2584 txq->ift_timer.c_cpu);
2617 iflib_txq_t txq = ctx->ifc_txqs;
2642 for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
2645 CALLOUT_LOCK(txq);
2646 callout_stop(&txq->ift_timer);
2648 callout_stop(&txq->ift_netmap_timer);
2650 CALLOUT_UNLOCK(txq);
2653 iflib_ifmp_purge(txq);
2655 for (j = 0; j < txq->ift_size; j++) {
2656 iflib_txsd_free(ctx, txq, j);
2658 txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
2659 txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
2660 txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
2661 txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
2662 txq->ift_pullups = 0;
2663 ifmp_ring_reset_stats(txq->ift_br);
2664 for (j = 0, di = txq->ift_ifdi; j < sctx->isc_ntxqs; j++, di++)
3148 #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)
3150 txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
3152 qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
3153 qidx_t minthresh = txq->ift_size / 8;
3164 txq_max_rs_deferred(iflib_txq_t txq)
3166 qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
3167 qidx_t minthresh = txq->ift_size / 8;
3168 if (txq->ift_in_use > 4*minthresh)
3170 if (txq->ift_in_use > 2*minthresh)
3172 if (txq->ift_in_use > minthresh)
3180 #define TXQ_MAX_DB_DEFERRED(txq, in_use) txq_max_db_deferred((txq), (in_use))
3181 #define TXQ_MAX_RS_DEFERRED(txq) txq_max_rs_deferred(txq)
3197 iflib_txd_db_check(iflib_txq_t txq, int ring)
3199 if_ctx_t ctx = txq->ift_ctx;
3202 max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use);
3205 if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2)) {
3212 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
3213 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
3215 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
3220 txq->ift_db_pending = txq->ift_npending = 0;
3245 iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
3247 if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
3269 txq->ift_pullups++;
3305 txq->ift_pullups++;
3311 txq->ift_pullups++;
3331 txq->ift_pullups++;
3370 txq->ift_pullups++;
3384 txq->ift_pullups++;
3422 iflib_remove_mbuf(iflib_txq_t txq)
3427 ifsd_m = txq->ift_sds.ifsd_m;
3428 ntxd = txq->ift_size;
3429 pidx = txq->ift_pidx & (ntxd - 1);
3430 ifsd_m = txq->ift_sds.ifsd_m;
3433 bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]);
3434 if (txq->ift_sds.ifsd_tso_map != NULL)
3435 bus_dmamap_unload(txq->ift_tso_buf_tag,
3436 txq->ift_sds.ifsd_tso_map[pidx]);
3438 txq->ift_dequeued++;
3444 calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid)
3450 ntxd = txq->ift_size;
3451 size = txq->ift_txd_size[qid];
3452 start = txq->ift_ifdi[qid].idi_vaddr;
3507 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
3521 ctx = txq->ift_ctx;
3524 segs = txq->ift_segs;
3525 ntxd = txq->ift_size;
3532 cidx = txq->ift_cidx;
3533 pidx = txq->ift_pidx;
3537 next_txd = calc_next_txd(txq, cidx, 0);
3542 prefetch(&txq->ift_sds.ifsd_m[next]);
3543 prefetch(&txq->ift_sds.ifsd_map[next]);
3546 map = txq->ift_sds.ifsd_map[pidx];
3547 ifsd_m = txq->ift_sds.ifsd_m;
3550 buf_tag = txq->ift_tso_buf_tag;
3552 map = txq->ift_sds.ifsd_tso_map[pidx];
3556 buf_tag = txq->ift_buf_tag;
3558 map = txq->ift_sds.ifsd_map[pidx];
3573 pi.ipi_qsidx = txq->ift_id;
3580 if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) {
3602 txq->ift_mbuf_defrag++;
3617 txq->ift_no_tx_dma_setup++;
3620 txq->ift_no_tx_dma_setup++;
3626 txq->ift_map_failed++;
3637 if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
3638 txq->ift_no_desc_avail++;
3642 if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)
3643 GROUPTASK_ENQUEUE(&txq->ift_task);
3652 txq->ift_rs_pending += nsegs + 1;
3653 if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) ||
3654 iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) {
3656 txq->ift_rs_pending = 0;
3662 MPASS(pidx >= 0 && pidx < txq->ift_size);
3669 MPASS(pi.ipi_new_pidx < txq->ift_size);
3673 ndesc += txq->ift_size;
3674 txq->ift_gen = 1;
3683 txq->ift_in_use += ndesc;
3684 txq->ift_db_pending += ndesc;
3690 txq->ift_pidx = pi.ipi_new_pidx;
3691 txq->ift_npending += pi.ipi_ndescs;
3693 *m_headp = m_head = iflib_remove_mbuf(txq);
3695 txq->ift_txd_encap_efbig++;
3710 txq->ift_mbuf_defrag_failed++;
3711 txq->ift_map_failed++;
3720 iflib_tx_desc_free(iflib_txq_t txq, int n)
3726 cidx = txq->ift_cidx;
3727 gen = txq->ift_gen;
3728 qsize = txq->ift_size;
3730 ifsd_m = txq->ift_sds.ifsd_m;
3731 do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);
3741 bus_dmamap_sync(txq->ift_tso_buf_tag,
3742 txq->ift_sds.ifsd_tso_map[cidx],
3744 bus_dmamap_unload(txq->ift_tso_buf_tag,
3745 txq->ift_sds.ifsd_tso_map[cidx]);
3747 bus_dmamap_sync(txq->ift_buf_tag,
3748 txq->ift_sds.ifsd_map[cidx],
3750 bus_dmamap_unload(txq->ift_buf_tag,
3751 txq->ift_sds.ifsd_map[cidx]);
3758 txq->ift_dequeued++;
3767 txq->ift_cidx = cidx;
3768 txq->ift_gen = gen;
3772 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
3775 if_ctx_t ctx = txq->ift_ctx;
3778 MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
3783 iflib_tx_credits_update(ctx, txq);
3784 reclaim = DESC_RECLAIMABLE(txq);
3786 if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
3790 txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
3796 iflib_tx_desc_free(txq, reclaim);
3797 txq->ift_cleaned += reclaim;
3798 txq->ift_in_use -= reclaim;
3824 iflib_txq_check_drain(iflib_txq_t txq, int budget)
3827 ifmp_ring_check_drainage(txq->ift_br, budget);
3833 iflib_txq_t txq = r->cookie;
3834 if_ctx_t ctx = txq->ift_ctx;
3836 if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2)
3838 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
3840 return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id,
3847 iflib_txq_t txq = r->cookie;
3848 if_ctx_t ctx = txq->ift_ctx;
3860 reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
3861 rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending);
3870 if (__predict_true(r->items[(cidx + i) & (r->size-1)] != (void *)txq))
3878 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3879 CALLOUT_LOCK(txq);
3880 callout_stop(&txq->ift_timer);
3881 CALLOUT_UNLOCK(txq);
3890 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3896 avail, ctx->ifc_flags, TXQ_AVAIL(txq));
3900 for (i = 0; i < count && TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx) + 2; i++) {
3907 * Completion interrupts will use the address of the txq
3913 if (__predict_false(*mp == (struct mbuf *)txq)) {
3917 err = iflib_encap(txq, mp);
3935 rang = iflib_txd_db_check(txq, false);
3940 iflib_txd_db_check(txq, ring);
3963 iflib_txq_t txq;
3965 txq = r->cookie;
3967 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3968 CALLOUT_LOCK(txq);
3969 callout_stop(&txq->ift_timer);
3970 CALLOUT_UNLOCK(txq);
3975 if (__predict_false(*mp == (struct mbuf *)txq))
3985 iflib_ifmp_purge(iflib_txq_t txq)
3989 r = txq->ift_br;
4002 iflib_txq_t txq = context;
4003 if_ctx_t ctx = txq->ift_ctx;
4008 txq->ift_cpu_exec_count[curcpu]++;
4014 netmap_tx_irq(ifp, txq->ift_id))
4021 if (txq->ift_db_pending)
4022 ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate);
4024 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
4029 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
4036 IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
4096 iflib_txq_t txq;
4115 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
4116 CALLOUT_LOCK(txq);
4117 callout_stop(&txq->ift_timer);
4118 CALLOUT_UNLOCK(txq);
4125 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
4126 callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq,
4127 txq->ift_timer.c_cpu);
4136 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
4137 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
4199 iflib_txq_t txq;
4217 txq = &ctx->ifc_txqs[qidx];
4220 if (txq->ift_closed) {
4255 err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate);
4258 GROUPTASK_ENQUEUE(&txq->ift_task);
4261 GROUPTASK_ENQUEUE(&txq->ift_task);
4264 txq->ift_closed = TRUE;
4266 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
4331 iflib_txq_t txq = ctx->ifc_txqs;
4337 for (i = 0; i < NTXQSETS(ctx); i++, txq++)
4338 while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br)))
4339 iflib_txq_check_drain(txq, 0);
5676 iflib_txq_t txq;
5683 for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
5684 callout_drain(&txq->ift_timer);
5686 callout_drain(&txq->ift_netmap_timer);
5688 if (txq->ift_task.gt_uniq != NULL)
5689 taskqgroup_detach(tqg, &txq->ift_task);
5754 iflib_txq_t txq = ctx->ifc_txqs;
5760 for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
5761 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
6013 iflib_txq_t txq;
6050 txq = ctx->ifc_txqs;
6056 for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) {
6066 txq->ift_ifdi = ifdip;
6074 txq->ift_txd_size[j] = scctx->isc_txd_size[j];
6077 txq->ift_ctx = ctx;
6078 txq->ift_id = i;
6080 txq->ift_br_offset = 1;
6082 txq->ift_br_offset = 0;
6085 if (iflib_txsd_alloc(txq)) {
6092 snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:TX(%d):callout",
6093 device_get_nameunit(dev), txq->ift_id);
6094 mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
6095 callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
6097 txq->ift_timer.c_cpu = cpu;
6100 callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0);
6101 txq->ift_netmap_timer.c_cpu = cpu;
6104 err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain,
6231 iflib_txq_t txq = ctx->ifc_txqs;
6234 for (i = 0; i < NTXQSETS(ctx); i++, txq++)
6235 iflib_txq_setup(txq);
6243 iflib_txq_t txq = ctx->ifc_txqs;
6247 for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
6249 iflib_dma_free(&txq->ift_ifdi[j]);
6250 iflib_txq_destroy(txq);
6531 iflib_txq_t txq = ctx->ifc_txqs;
6566 GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
6567 taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res,
6638 iflib_txq_t txq = ctx->ifc_txqs;
6648 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
6649 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
6656 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
6660 int credits_pre = txq->ift_cidx_processed;
6663 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
6665 if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
6668 txq->ift_processed += credits;
6669 txq->ift_cidx_processed += credits;
6671 MPASS(credits_pre + credits == txq->ift_cidx_processed);
6672 if (txq->ift_cidx_processed >= txq->ift_size)
6673 txq->ift_cidx_processed -= txq->ift_size;
6982 "permit #txq != #rxq");
7025 iflib_txq_t txq;
7038 qfmt = "txq%03d";
7040 qfmt = "txq%02d";
7042 qfmt = "txq%d";
7043 for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
7050 &txq->ift_task.gt_cpu, 0, "cpu this queue is bound to");
7054 &txq->ift_dequeued, "total mbufs freed");
7057 &txq->ift_enqueued, "total mbufs enqueued");
7061 &txq->ift_mbuf_defrag, "# of times m_defrag was called");
7064 &txq->ift_pullups, "# of times m_pullup was called");
7067 &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
7070 &txq->ift_no_desc_avail, "# of times no descriptors were available");
7073 &txq->ift_map_failed, "# of times DMA map failed");
7076 &txq->ift_txd_encap_efbig, "# of times txd_encap returned EFBIG");
7079 &txq->ift_no_tx_dma_setup, "# of times map failed for other than EFBIG");
7082 &txq->ift_pidx, 1, "Producer Index");
7085 &txq->ift_cidx, 1, "Consumer Index");
7088 &txq->ift_cidx_processed, 1, "Consumer Index seen by credit update");
7091 &txq->ift_in_use, 1, "descriptors in use");
7094 &txq->ift_processed, "descriptors procesed for clean");
7097 &txq->ift_cleaned, "total cleaned");
7100 __DEVOLATILE(uint64_t *, &txq->ift_br->state), 0,
7103 CTLFLAG_RD, &txq->ift_br->enqueues,
7106 CTLFLAG_RD, &txq->ift_br->drops,
7109 CTLFLAG_RD, &txq->ift_br->starts,
7112 CTLFLAG_RD, &txq->ift_br->stalls,
7115 CTLFLAG_RD, &txq->ift_br->restarts,
7118 CTLFLAG_RD, &txq->ift_br->abdications,
7262 iflib_txq_t txq;
7270 txq = &ctx->ifc_txqs[0];
7271 error = iflib_encap(txq, &m);
7273 (void)iflib_txd_db_check(txq, true);
7283 iflib_txq_t txq;
7293 txq = &ctx->ifc_txqs[0];
7294 (void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));