Lines Matching defs:fl
873 iflib_fl_t fl = &rxq->ifr_fl[0];
879 info->rx_buf_maxsize = fl->ifl_buf_size;
896 iflib_fl_t fl = &rxq->ifr_fl[0];
908 * such a way to keep fl->ifl_pidx and kring->nr_hwcur in sync
916 * (fl->ifl_pidx - 1) % N (included), to avoid the NIC tail/prod
931 map = fl->ifl_sds.ifsd_map;
932 nic_i = fl->ifl_pidx;
952 void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[i]);
959 fl->ifl_rxd_idxs[i] = nic_i;
962 netmap_load_map(na, fl->ifl_buf_tag,
966 netmap_reload_map(na, fl->ifl_buf_tag,
969 bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i],
981 fl->ifl_pidx = nic_i;
989 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
991 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id,
1209 iflib_fl_t fl = &rxq->ifr_fl[0];
1218 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
1230 * nic_i = fl->ifl_cidx;
1235 * fl->ifl_cidx is set to 0 on a ring reinit
1250 cidxp = &fl->ifl_cidx;
1254 nic_i = fl->ifl_cidx;
1282 bus_dmamap_sync(fl->ifl_buf_tag,
1283 fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD);
1285 fl->ifl_cidx = nic_i = nm_next(nic_i, lim);
1446 iflib_fl_t fl;
1448 fl = &rxq->ifr_fl[flid];
1449 iru->iru_paddrs = fl->ifl_bus_addrs;
1450 iru->iru_idxs = fl->ifl_rxd_idxs;
1452 iru->iru_buf_size = fl->ifl_buf_size;
1453 iru->iru_flidx = fl->ifl_id;
1963 iflib_fl_t fl;
1969 fl = rxq->ifr_fl;
1970 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) {
1971 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */
1984 &fl->ifl_buf_tag);
1992 if (!(fl->ifl_sds.ifsd_m =
2002 if (!(fl->ifl_sds.ifsd_cl =
2012 if (!(fl->ifl_sds.ifsd_ba =
2024 if (!(fl->ifl_sds.ifsd_map =
2032 err = bus_dmamap_create(fl->ifl_buf_tag, 0,
2033 &fl->ifl_sds.ifsd_map[i]);
2070 * @fl: the free list to refill
2078 iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
2090 MPASS(count <= fl->ifl_size - fl->ifl_credits - 1);
2092 sd_m = fl->ifl_sds.ifsd_m;
2093 sd_map = fl->ifl_sds.ifsd_map;
2094 sd_cl = fl->ifl_sds.ifsd_cl;
2095 sd_ba = fl->ifl_sds.ifsd_ba;
2096 pidx = fl->ifl_pidx;
2098 frag_idx = fl->ifl_fragidx;
2099 credits = fl->ifl_credits;
2104 MPASS(credits + n <= fl->ifl_size);
2106 if (pidx < fl->ifl_cidx)
2107 MPASS(pidx + n <= fl->ifl_cidx);
2108 if (pidx == fl->ifl_cidx && (credits < fl->ifl_size))
2109 MPASS(fl->ifl_gen == 0);
2110 if (pidx > fl->ifl_cidx)
2111 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
2116 iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
2125 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size,
2128 bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
2132 cl = uma_zalloc(fl->ifl_zone, M_NOWAIT);
2135 if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
2141 err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx],
2142 cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg,
2146 uma_zfree(fl->ifl_zone, cl);
2156 fl->ifl_cl_enqueued++;
2161 bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx],
2170 bit_set(fl->ifl_rx_bitmap, frag_idx);
2172 fl->ifl_m_enqueued++;
2176 fl->ifl_rxd_idxs[i] = frag_idx;
2177 fl->ifl_bus_addrs[i] = bus_addr;
2180 MPASS(credits <= fl->ifl_size);
2181 if (++idx == fl->ifl_size) {
2183 fl->ifl_gen = 1;
2191 fl->ifl_pidx = idx;
2192 fl->ifl_credits = credits;
2203 fl->ifl_pidx = idx;
2204 fl->ifl_credits = credits;
2207 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
2209 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id,
2210 fl->ifl_id, fl->ifl_pidx);
2211 if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) {
2212 fl->ifl_fragidx = frag_idx + 1;
2213 if (fl->ifl_fragidx == fl->ifl_size)
2214 fl->ifl_fragidx = 0;
2216 fl->ifl_fragidx = frag_idx;
2224 iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl)
2237 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
2239 int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
2242 MPASS(fl->ifl_credits <= fl->ifl_size);
2246 return (iflib_fl_refill(ctx, fl, reclaimable));
2262 iflib_fl_bufs_free(iflib_fl_t fl)
2264 iflib_dma_info_t idi = fl->ifl_ifdi;
2268 for (i = 0; i < fl->ifl_size; i++) {
2269 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
2270 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];
2273 sd_map = fl->ifl_sds.ifsd_map[i];
2274 bus_dmamap_sync(fl->ifl_buf_tag, sd_map,
2276 bus_dmamap_unload(fl->ifl_buf_tag, sd_map);
2278 uma_zfree(fl->ifl_zone, *sd_cl);
2281 m_cljset(mb, *sd_cl, fl->ifl_cltype);
2298 fl->ifl_m_dequeued++;
2299 fl->ifl_cl_dequeued++;
2303 for (i = 0; i < fl->ifl_size; i++) {
2304 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL);
2305 MPASS(fl->ifl_sds.ifsd_m[i] == NULL);
2311 fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0;
2321 iflib_fl_setup(iflib_fl_t fl)
2323 iflib_rxq_t rxq = fl->ifl_rxq;
2328 bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1);
2332 iflib_fl_bufs_free(fl);
2334 MPASS(fl->ifl_credits == 0);
2335 qidx = rxq->ifr_fl_offset + fl->ifl_id;
2337 fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx];
2339 fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz;
2344 fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size);
2345 if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
2346 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
2347 fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
2349 fl->ifl_zone = m_getzone(fl->ifl_buf_size);
2358 MPASS(fl->ifl_size > 0);
2359 (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1));
2360 if (min(128, fl->ifl_size - 1) != fl->ifl_credits)
2366 MPASS(fl->ifl_ifdi != NULL);
2367 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
2380 iflib_fl_t fl;
2385 fl = &rxq->ifr_fl[i];
2386 if (fl->ifl_buf_tag != NULL) {
2387 if (fl->ifl_sds.ifsd_map != NULL) {
2388 for (j = 0; j < fl->ifl_size; j++) {
2390 fl->ifl_buf_tag,
2391 fl->ifl_sds.ifsd_map[j],
2394 fl->ifl_buf_tag,
2395 fl->ifl_sds.ifsd_map[j]);
2397 fl->ifl_buf_tag,
2398 fl->ifl_sds.ifsd_map[j]);
2401 bus_dma_tag_destroy(fl->ifl_buf_tag);
2402 fl->ifl_buf_tag = NULL;
2404 free(fl->ifl_sds.ifsd_m, M_IFLIB);
2405 free(fl->ifl_sds.ifsd_cl, M_IFLIB);
2406 free(fl->ifl_sds.ifsd_ba, M_IFLIB);
2407 free(fl->ifl_sds.ifsd_map, M_IFLIB);
2408 free(fl->ifl_rx_bitmap, M_IFLIB);
2409 fl->ifl_sds.ifsd_m = NULL;
2410 fl->ifl_sds.ifsd_cl = NULL;
2411 fl->ifl_sds.ifsd_ba = NULL;
2412 fl->ifl_sds.ifsd_map = NULL;
2413 fl->ifl_rx_bitmap = NULL;
2515 iflib_fl_t fl;
2569 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
2570 if (iflib_fl_setup(fl)) {
2622 iflib_fl_t fl;
2674 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
2675 iflib_fl_bufs_free(fl);
2680 calc_next_rxd(iflib_fl_t fl, int cidx)
2686 nrxd = fl->ifl_size;
2687 size = fl->ifl_rxd_size;
2688 start = fl->ifl_ifdi->idi_vaddr;
2699 prefetch_pkts(iflib_fl_t fl, int cidx)
2702 int nrxd = fl->ifl_size;
2706 prefetch(&fl->ifl_sds.ifsd_m[nextptr]);
2707 prefetch(&fl->ifl_sds.ifsd_cl[nextptr]);
2708 next_rxd = calc_next_rxd(fl, cidx);
2710 prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]);
2711 prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]);
2712 prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]);
2713 prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]);
2714 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]);
2715 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]);
2716 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]);
2717 prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]);
2725 iflib_fl_t fl;
2733 fl = &rxq->ifr_fl[flid];
2734 sd->ifsd_fl = fl;
2735 m = fl->ifl_sds.ifsd_m[cidx];
2736 sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx];
2737 fl->ifl_credits--;
2739 fl->ifl_m_dequeued++;
2742 prefetch_pkts(fl, cidx);
2743 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1);
2744 prefetch(&fl->ifl_sds.ifsd_map[next]);
2745 map = fl->ifl_sds.ifsd_map[cidx];
2747 bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD);
2778 fl->ifl_sds.ifsd_m[cidx] = NULL;
2786 fl->ifl_sds.ifsd_m[cidx] = NULL;
2792 bus_dmamap_unload(fl->ifl_buf_tag, map);
2793 fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1);
2794 if (__predict_false(fl->ifl_cidx == 0))
2795 fl->ifl_gen = 0;
2796 bit_clear(fl->ifl_rx_bitmap, cidx);
2988 iflib_fl_t fl;
3013 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
3014 retval |= iflib_fl_refill_all(ctx, fl);
3073 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
3074 retval |= iflib_fl_refill_all(ctx, fl);
6015 iflib_fl_t fl = NULL;
6142 if (!(fl =
6148 rxq->ifr_fl = fl;
6150 fl[j].ifl_rxq = rxq;
6151 fl[j].ifl_id = j;
6152 fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
6153 fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];
6163 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
6164 fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB,
6680 iflib_fl_t fl;
6683 for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++)
6684 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
7024 iflib_fl_t fl;
7142 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
7149 &fl->ifl_pidx, 1, "Producer Index");
7152 &fl->ifl_cidx, 1, "Consumer Index");
7155 &fl->ifl_credits, 1, "credits available");
7158 &fl->ifl_buf_size, 1, "buffer size");
7162 &fl->ifl_m_enqueued, "mbufs allocated");
7165 &fl->ifl_m_dequeued, "mbufs freed");
7168 &fl->ifl_cl_enqueued, "clusters allocated");
7171 &fl->ifl_cl_dequeued, "clusters freed");
7233 iflib_fl_t fl;
7246 fl = rxq->ifr_fl;
7247 fl->ifl_zone = m_getzone(fl->ifl_buf_size);