Lines Matching refs:fl
1163 snprintf(name, sizeof(name), "%s rxq%d-fl",
1165 init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name);
1186 snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
1188 init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name);
1374 * Interrupt handler for iq+fl queues.
1434 ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
1551 * Deals with interrupts on an iq+fl queue.
1557 struct sge_fl *fl;
1574 fl = &rxq->fl;
1575 fl_hw_cidx = fl->hw_cidx; /* stable snapshot */
1606 m0 = get_fl_payload(sc, fl, lq);
1609 refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
1677 FL_LOCK(fl);
1678 refill_fl(sc, fl, 32);
1679 FL_UNLOCK(fl);
1685 FL_LOCK(fl);
1686 refill_fl(sc, fl, 32);
1687 FL_UNLOCK(fl);
1688 fl_hw_cidx = fl->hw_cidx;
1708 FL_LOCK(fl);
1709 starved = refill_fl(sc, fl, 64);
1710 FL_UNLOCK(fl);
1712 add_fl_to_sfl(sc, fl);
1718 cl_has_metadata(struct sge_fl *fl, struct cluster_layout *cll)
1720 int rc = fl->flags & FL_BUF_PACKING || cll->region1 > 0;
1729 cl_metadata(struct adapter *sc, struct sge_fl *fl, struct cluster_layout *cll,
1733 if (cl_has_metadata(fl, cll)) {
1762 get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1766 struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1770 struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl);
1774 blen = hwb->size - fl->rx_offset; /* max possible in this buf */
1776 payload = sd->cl + cll->region1 + fl->rx_offset;
1777 if (fl->flags & FL_BUF_PACKING) {
1779 const u_int pad = roundup2(l, fl->buf_boundary) - l;
1781 if (fl->rx_offset + len + pad < hwb->size)
1783 MPASS(fl->rx_offset + blen <= hwb->size);
1785 MPASS(fl->rx_offset == 0); /* not packing */
1799 fl->mbuf_allocated++;
1820 fl->mbuf_inlined++;
1837 fl->mbuf_allocated++;
1852 if (fl->flags & FL_BUF_PACKING) {
1853 fl->rx_offset += blen;
1854 MPASS(fl->rx_offset <= hwb->size);
1855 if (fl->rx_offset < hwb->size)
1859 if (__predict_false(++fl->cidx % 8 == 0)) {
1860 uint16_t cidx = fl->cidx / 8;
1862 if (__predict_false(cidx == fl->sidx))
1863 fl->cidx = cidx = 0;
1864 fl->hw_cidx = cidx;
1866 fl->rx_offset = 0;
1872 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf)
1878 if (__predict_false(fl->flags & FL_BUF_RESUME)) {
1879 M_ASSERTPKTHDR(fl->m0);
1880 MPASS(fl->m0->m_pkthdr.len == total);
1881 MPASS(fl->remaining < total);
1883 m0 = fl->m0;
1884 pnext = fl->pnext;
1885 remaining = fl->remaining;
1886 fl->flags &= ~FL_BUF_RESUME;
1890 if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
1891 fl->rx_offset = 0;
1892 if (__predict_false(++fl->cidx % 8 == 0)) {
1893 uint16_t cidx = fl->cidx / 8;
1895 if (__predict_false(cidx == fl->sidx))
1896 fl->cidx = cidx = 0;
1897 fl->hw_cidx = cidx;
1906 m0 = get_scatter_segment(sc, fl, 0, total);
1913 MPASS(fl->rx_offset == 0);
1914 m = get_scatter_segment(sc, fl, total - remaining, remaining);
1916 fl->m0 = m0;
1917 fl->pnext = pnext;
1918 fl->remaining = remaining;
1919 fl->flags |= FL_BUF_RESUME;
2126 struct sge_fl *fl;
2131 fl = &rxq->fl;
2133 FL_LOCK(fl);
2134 find_best_refill_source(sc, fl, maxp);
2135 FL_UNLOCK(fl);
2140 fl = &ofld_rxq->fl;
2142 FL_LOCK(fl);
2143 find_best_refill_source(sc, fl, maxp);
2144 FL_UNLOCK(fl);
2709 init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name)
2712 fl->qsize = qsize;
2713 fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
2714 strlcpy(fl->lockname, name, sizeof(fl->lockname));
2718 fl->flags |= FL_BUF_PACKING;
2719 find_best_refill_source(sc, fl, maxp);
2720 find_safe_refill_source(sc, fl);
2795 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl,
2847 if (fl) {
2848 mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
2850 len = fl->qsize * EQ_ESIZE;
2851 rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
2852 &fl->ba, (void **)&fl->desc);
2857 rc = alloc_fl_sdesc(fl);
2860 "failed to setup fl software descriptors: %d\n",
2865 if (fl->flags & FL_BUF_PACKING) {
2866 fl->lowat = roundup2(sp->fl_starve_threshold2, 8);
2867 fl->buf_boundary = sp->pack_boundary;
2869 fl->lowat = roundup2(sp->fl_starve_threshold, 8);
2870 fl->buf_boundary = 16;
2872 if (fl_pad && fl->buf_boundary < sp->pad_boundary)
2873 fl->buf_boundary = sp->pad_boundary;
2879 (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
2892 c.fl0size = htobe16(fl->qsize);
2893 c.fl0addr = htobe64(fl->ba);
2917 if (fl) {
2921 fl->cntxt_id = be16toh(c.fl0id);
2922 fl->pidx = fl->cidx = 0;
2924 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
2926 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
2929 sc->sge.eqmap[cntxt_id] = (void *)fl;
2931 qid = fl->cntxt_id;
2944 fl->udb = (volatile void *)udb;
2946 fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
2948 FL_LOCK(fl);
2950 refill_fl(sc, fl, fl->lowat);
2951 FL_UNLOCK(fl);
2988 free_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl)
3002 fl ? fl->cntxt_id : 0xffff, 0xffff);
3015 if (fl) {
3016 free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba,
3017 fl->desc);
3019 if (fl->sdesc)
3020 free_fl_sdesc(sc, fl);
3022 if (mtx_initialized(&fl->fl_lock))
3023 mtx_destroy(&fl->fl_lock);
3025 bzero(fl, sizeof(*fl));
3054 struct sysctl_oid *oid, struct sge_fl *fl)
3058 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
3063 &fl->ba, "bus address of descriptor ring");
3065 fl->sidx * EQ_ESIZE + sc->params.sge.spg_len,
3068 CTLTYPE_INT | CTLFLAG_RD, &fl->cntxt_id, 0, sysctl_uint16, "I",
3073 fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled");
3074 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx,
3076 if (fl->flags & FL_BUF_PACKING) {
3078 CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset");
3080 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx,
3083 CTLFLAG_RD, &fl->mbuf_allocated, "# of mbuf allocated");
3085 CTLFLAG_RD, &fl->mbuf_inlined, "# of mbuf inlined in clusters");
3087 CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated");
3089 CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled");
3091 CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)");
3170 rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx,
3187 FL_LOCK(&rxq->fl);
3188 refill_fl(sc, &rxq->fl, 128);
3189 FL_UNLOCK(&rxq->fl);
3222 add_fl_sysctls(sc, &vi->ctx, oid, &rxq->fl);
3239 rc = free_iq_fl(vi, &rxq->iq, &rxq->fl);
3256 rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 0);
3266 add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl);
3276 rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl);
3339 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
3903 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3907 n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx);
3911 v = fl->dbval | V_PIDX(n);
3912 if (fl->udb)
3913 *fl->udb = htole32(v);
3916 IDXINCR(fl->dbidx, n, fl->sidx);
3927 refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
3937 uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */
3939 FL_LOCK_ASSERT_OWNED(fl);
3946 max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
3947 if (fl->pidx == max_pidx * 8)
3950 d = &fl->desc[fl->pidx];
3951 sd = &fl->sdesc[fl->pidx];
3952 cll = &fl->cll_def; /* default layout */
3967 fl->cl_fast_recycled++;
3969 clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
3981 clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
3985 fl->cl_recycled++;
3995 if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 ||
3996 fl->cll_def.zidx == fl->cll_alt.zidx)
4000 cll = &fl->cll_alt;
4004 fl->cl_allocated++;
4012 clm = cl_metadata(sc, fl, cll, cl);
4024 if (__predict_false(++fl->pidx % 8 == 0)) {
4025 uint16_t pidx = fl->pidx / 8;
4027 if (__predict_false(pidx == fl->sidx)) {
4028 fl->pidx = 0;
4030 sd = fl->sdesc;
4031 d = fl->desc;
4036 if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
4037 ring_fl_db(sc, fl);
4041 if (fl->pidx / 8 != fl->dbidx)
4042 ring_fl_db(sc, fl);
4044 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
4054 struct sge_fl *fl, *fl_temp;
4057 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
4058 FL_LOCK(fl);
4059 refill_fl(sc, fl, 64);
4060 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
4061 TAILQ_REMOVE(&sc->sfl, fl, link);
4062 fl->flags &= ~FL_STARVING;
4064 FL_UNLOCK(fl);
4072 alloc_fl_sdesc(struct sge_fl *fl)
4075 fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE,
4082 free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
4089 sd = fl->sdesc;
4090 for (i = 0; i < fl->sidx * 8; i++, sd++) {
4095 clm = cl_metadata(sc, fl, cll, sd->cl);
4105 free(fl->sdesc, M_CXGBE);
4106 fl->sdesc = NULL;
4976 find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp)
4994 spare_needed = fl->flags & FL_BUF_PACKING ? CL_METADATA_SIZE : 0;
5008 " refill source for fl %p (dma %u). Ignored.\n",
5009 largest_rx_cluster, fl, maxp);
5020 if ((fl->flags & FL_BUF_PACKING) == 0)
5056 if (fl->flags & FL_BUF_PACKING &&
5067 if (fl->flags & FL_BUF_PACKING) {
5078 ("%s: bad zone %d for fl %p, maxp %d", __func__, zidx, fl, maxp));
5080 ("%s: bad hwidx %d for fl %p, maxp %d", __func__, hwidx, fl, maxp));
5083 ("%s: bad buffer layout for fl %p, maxp %d. "
5084 "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
5087 if (fl->flags & FL_BUF_PACKING || region1 > 0) {
5089 ("%s: no room for metadata. fl %p, maxp %d; "
5090 "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
5094 ("%s: bad mbuf region for fl %p, maxp %d. "
5095 "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
5100 fl->cll_def.zidx = zidx;
5101 fl->cll_def.hwidx = hwidx;
5102 fl->cll_def.region1 = region1;
5103 fl->cll_def.region3 = region3;
5107 find_safe_refill_source(struct adapter *sc, struct sge_fl *fl)
5115 if (fl->flags & FL_BUF_PACKING)
5131 fl->cll_alt.hwidx = -1;
5132 fl->cll_alt.zidx = -1;
5140 fl->cll_alt.hwidx = hwidx;
5141 fl->cll_alt.zidx = hwb->zidx;
5144 fl->cll_alt.region1 = ((spare - CL_METADATA_SIZE) / MSIZE) * MSIZE;
5146 fl->cll_alt.region1 = 0;
5147 fl->cll_alt.region3 = spare - fl->cll_alt.region1;
5151 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
5154 FL_LOCK(fl);
5155 if ((fl->flags & FL_DOOMED) == 0) {
5156 fl->flags |= FL_STARVING;
5157 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
5160 FL_UNLOCK(fl);