net/pktsched/pktsched_qfq.c

301 	struct qfq_class *cl;
308 		if ((cl = qif->qif_class_tbl[i]) != NULL)
309 			qfq_class_destroy(qif, cl);
318 	struct qfq_class *cl;
324 		if ((cl = qif->qif_class_tbl[i]) != NULL)
325 			qfq_purgeq(qif, cl, 0, NULL, NULL);
353 		qfq_purgeq(qif, ifq->ifcq_disc_slots[i].cl,
362 			qfq_purgeq(qif, ifq->ifcq_disc_slots[i].cl,
373 	struct qfq_class *cl;
379 		if ((cl = qif->qif_class_tbl[i]) != NULL)
380 			qfq_updateq(qif, cl, ev);
387 	struct qfq_class *cl;
406 	cl = qfq_class_create(qif, weight, qlimit, flags, maxsz, qid);
407 	if (cl == NULL)
411 		*clp = cl;
423 	struct qfq_class *cl;
478 	cl = zalloc(qfq_cl_zone);
479 	if (cl == NULL)
482 	bzero(cl, qfq_cl_size);
489 	_qinit(&cl->cl_q, Q_DROPTAIL, qlimit);
490 	cl->cl_qif = qif;
491 	cl->cl_flags = flags;
492 	cl->cl_handle = qid;
501 		qif->qif_class_tbl[i] = cl;
505 				qif->qif_class_tbl[i] = cl;
510 			zfree(qfq_cl_zone, cl);
517 	cl->cl_lmax = maxsz;
518 	cl->cl_inv_w = (QFQ_ONE_FP / w);
519 	w = (QFQ_ONE_FP / cl->cl_inv_w);
522 	i = qfq_calc_index(cl, cl->cl_inv_w, cl->cl_lmax);
541 			    cl->cl_handle);
547 			zfree(qfq_cl_zone, cl);
553 	cl->cl_grp = grp;
555 	/* XXX cl->cl_S = qif->qif_V; ? */
561 		qif->qif_default = cl;
569 		cl->cl_qflags = 0;
572 				cl->cl_qflags |= BLUEF_ECN;
574 				cl->cl_qflags |= SFBF_ECN;
576 				cl->cl_qflags |= REDF_ECN;
578 				cl->cl_qflags |= RIOF_ECN;
582 				cl->cl_qflags |= SFBF_FLOWCTL;
586 				cl->cl_qflags |= RIOF_CLEARDSCP;
602 			cl->cl_red = red_alloc(ifp, 0, 0,
603 			    qlimit(&cl->cl_q) * 10/100,
604 			    qlimit(&cl->cl_q) * 30/100,
605 			    cl->cl_qflags, pkttime);
606 			if (cl->cl_red != NULL)
607 				qtype(&cl->cl_q) = Q_RED;
612 			cl->cl_rio =
613 			    rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime);
614 			if (cl->cl_rio != NULL)
615 				qtype(&cl->cl_q) = Q_RIO;
621 			cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags);
622 			if (cl->cl_blue != NULL)
623 				qtype(&cl->cl_q) = Q_BLUE;
627 			if (!(cl->cl_flags & QFCF_LAZY))
628 				cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle,
629 				    qlimit(&cl->cl_q), cl->cl_qflags);
630 			if (cl->cl_sfb != NULL || (cl->cl_flags & QFCF_LAZY))
631 				qtype(&cl->cl_q) = Q_SFB;
638 		    cl->cl_handle, cl->cl_grp->qfg_index, weight, qlimit,
642 	return (cl);
648 	struct qfq_class *cl;
652 	if ((cl = qfq_clh_to_clp(qif, qid)) == NULL)
655 	return (qfq_class_destroy(qif, cl));
659 qfq_class_destroy(struct qfq_if *qif, struct qfq_class *cl)
666 	qfq_purgeq(qif, cl, 0, NULL, NULL);
668 	if (cl->cl_inv_w != 0) {
669 		qif->qif_wsum -= (QFQ_ONE_FP / cl->cl_inv_w);
670 		cl->cl_inv_w = 0;	/* reset weight to avoid run twice */
674 		if (qif->qif_class_tbl[i] == cl) {
681 	if (cl->cl_qalg.ptr != NULL) {
683 		if (q_is_rio(&cl->cl_q))
684 			rio_destroy(cl->cl_rio);
687 		if (q_is_red(&cl->cl_q))
688 			red_destroy(cl->cl_red);
691 		if (q_is_blue(&cl->cl_q))
692 			blue_destroy(cl->cl_blue);
694 		if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
695 			sfb_destroy(cl->cl_sfb);
696 		cl->cl_qalg.ptr = NULL;
697 		qtype(&cl->cl_q) = Q_DROPTAIL;
698 		qstate(&cl->cl_q) = QS_RUNNING;
701 	if (qif->qif_default == cl)
706 		    if_name(QFQIF_IFP(qif)), qfq_style(qif), cl->cl_handle);
709 	zfree(qfq_cl_zone, cl);
803  * roundedS is always cl->qfg_S rounded on grp->qfg_slot_shift bits.
807     struct qfq_class *cl, u_int64_t roundedS)
812 	cl->cl_next = grp->qfg_slots[i];
813 	grp->qfg_slots[i] = cl;
899     struct qfq_class *cl)
902 	cl->cl_S = cl->cl_F;
903 	if (qempty(&cl->cl_q))  {
909 		len = m_pktlen(qhead(&cl->cl_q));
910 		cl->cl_F = cl->cl_S + (u_int64_t)len * cl->cl_inv_w;
911 		roundedS = qfq_round_down(cl->cl_S, grp->qfg_slot_shift);
916 		qfq_slot_insert(qif, grp, cl, roundedS);
933 	struct qfq_class *cl;
960 	cl = grp->qfg_slots[grp->qfg_front];
961 	VERIFY(cl != NULL && !qempty(&cl->cl_q));
964 		return (qfq_pollq(cl));
966 	m = qfq_getq(cl);
975 	if (qempty(&cl->cl_q))
976 		cl->cl_period++;
977 	PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len);
985 		    if_name(QFQIF_IFP(qif)), qfq_style(qif), cl->cl_handle,
986 		    m, cl->cl_F, qif->qif_V);
989 	if (qfq_update_class(qif, grp, cl)) {
992 		cl = qfq_slot_scan(qif, grp);
993 		if (!cl) { /* group gone, remove from ER */
998 			    qfq_round_down(cl->cl_S, grp->qfg_slot_shift);
1040 qfq_update_start(struct qfq_if *qif, struct qfq_class *cl)
1044 	int slot_shift = cl->cl_grp->qfg_slot_shift;
1046 	roundedF = qfq_round_down(cl->cl_F, slot_shift);
1049 	if (!qfq_gt(cl->cl_F, qif->qif_V) || qfq_gt(roundedF, limit)) {
1051 		mask = mask_from(qif->qif_bitmaps[ER], cl->cl_grp->qfg_index);
1055 				cl->cl_S = next->qfg_F;
1059 		cl->cl_S = qif->qif_V;
1061 		cl->cl_S = cl->cl_F;
1066 qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m,
1075 	VERIFY(cl == NULL || cl->cl_qif == qif);
1077 	if (cl == NULL) {
1078 		cl = qfq_clh_to_clp(qif, t->pftag_qid);
1079 		if (cl == NULL) {
1080 			cl = qif->qif_default;
1081 			if (cl == NULL) {
1091 	ret = qfq_addq(cl, m, t);
1101 			PKTCNTR_ADD(&cl->cl_dropcnt, 1, len);
1121 	if (qlen(&cl->cl_q) > 1)
1125 	grp = cl->cl_grp;
1126 	qfq_update_start(qif, cl);	/* adjust start time */
1129 	cl->cl_F = cl->cl_S + (u_int64_t)len * cl->cl_inv_w;
1130 	roundedS = qfq_round_down(cl->cl_S, grp->qfg_slot_shift);
1133 	 * Insert cl in the correct bucket.
1135 	 * If cl->cl_S >= grp->qfg_S we don't need to adjust the bucket list
1142 		if (!qfq_gt(grp->qfg_S, cl->cl_S))
1145 		/* create a slot for this cl->cl_S */
1164 		    qfq_style(qif), cl->cl_handle, m, qfq_state2str(s),
1165 		    qif->qif_bitmaps[s], cl->cl_S, cl->cl_F, qif->qif_V);
1169 	qfq_slot_insert(qif, grp, cl, roundedS);
1178     struct qfq_class *cl)
1185 	roundedS = qfq_round_down(cl->cl_S, grp->qfg_slot_shift);
1190 	while (*pprev && *pprev != cl)
1193 	*pprev = cl->cl_next;
1207 qfq_deactivate_class(struct qfq_if *qif, struct qfq_class *cl)
1209 	struct qfq_group *grp = cl->cl_grp;
1218 		    if_name(QFQIF_IFP(cl->cl_qif)), qfq_style(cl->cl_qif),
1219 		    cl->cl_handle, grp->qfg_index, grp->qfg_full_slots,
1228 	cl->cl_F = cl->cl_S;	/* not needed if the class goes away */
1229 	qfq_slot_remove(qif, grp, cl);
1254 		cl = qfq_slot_scan(qif, grp);
1255 		roundedS = qfq_round_down(cl->cl_S, grp->qfg_slot_shift);
1301 qfq_addq(struct qfq_class *cl, struct mbuf *m, struct pf_mtag *t)
1303 	struct qfq_if	*qif = cl->cl_qif;
1309 	if (q_is_rio(&cl->cl_q))
1310 		return (rio_addq(cl->cl_rio, &cl->cl_q, m, t));
1314 	if (q_is_red(&cl->cl_q))
1315 		return (red_addq(cl->cl_red, &cl->cl_q, m, t));
1319 	if (q_is_blue(&cl->cl_q))
1320 		return (blue_addq(cl->cl_blue, &cl->cl_q, m, t));
1323 	if (q_is_sfb(&cl->cl_q)) {
1324 		if (cl->cl_sfb == NULL) {
1327 			VERIFY(cl->cl_flags & QFCF_LAZY);
1328 			cl->cl_flags &= ~QFCF_LAZY;
1331 			cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle,
1332 			    qlimit(&cl->cl_q), cl->cl_qflags);
1333 			if (cl->cl_sfb == NULL) {
1335 				qtype(&cl->cl_q) = Q_DROPTAIL;
1336 				cl->cl_flags &= ~QFCF_SFB;
1337 				cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL);
1342 				    qfq_style(qif), cl->cl_handle,
1343 				    cl->cl_grp->qfg_index);
1357 		if (cl->cl_sfb != NULL)
1358 			return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t));
1359 	} else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) {
1365 	if (cl->cl_flags & QFCF_CLEARDSCP)
1368 	_addq(&cl->cl_q, m);
1374 qfq_getq(struct qfq_class *cl)
1376 	IFCQ_LOCK_ASSERT_HELD(cl->cl_qif->qif_ifq);
1379 	if (q_is_rio(&cl->cl_q))
1380 		return (rio_getq(cl->cl_rio, &cl->cl_q));
1384 	if (q_is_red(&cl->cl_q))
1385 		return (red_getq(cl->cl_red, &cl->cl_q));
1389 	if (q_is_blue(&cl->cl_q))
1390 		return (blue_getq(cl->cl_blue, &cl->cl_q));
1393 	if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1394 		return (sfb_getq(cl->cl_sfb, &cl->cl_q));
1396 	return (_getq(&cl->cl_q));
1400 qfq_pollq(struct qfq_class *cl)
1402 	IFCQ_LOCK_ASSERT_HELD(cl->cl_qif->qif_ifq);
1404 	return (qhead(&cl->cl_q));
1408 qfq_purgeq(struct qfq_if *qif, struct qfq_class *cl, u_int32_t flow,
1416 	if ((qlen = qlen(&cl->cl_q)) == 0)
1423 	if (q_is_rio(&cl->cl_q))
1424 		rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len);
1428 	if (q_is_red(&cl->cl_q))
1429 		red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len);
1433 	if (q_is_blue(&cl->cl_q))
1434 		blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len);
1437 	if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1438 		sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len);
1440 		_flushq_flow(&cl->cl_q, flow, &cnt, &len);
1443 		VERIFY(qlen(&cl->cl_q) == (qlen - cnt));
1449 		PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len);
1455 		if (qempty(&cl->cl_q))
1456 			qfq_deactivate_class(qif, cl);
1462 			    qfq_style(qif), cl->cl_handle,
1463 			    (u_int32_t)(QFQ_ONE_FP / cl->cl_inv_w), qlen,
1464 			    qlen(&cl->cl_q), cnt, len, flow);
1475 qfq_updateq(struct qfq_if *qif, struct qfq_class *cl, cqev_t ev)
1482 		    cl->cl_handle, (u_int32_t)(QFQ_ONE_FP / cl->cl_inv_w),
1487 	if (q_is_rio(&cl->cl_q))
1488 		return (rio_updateq(cl->cl_rio, ev));
1491 	if (q_is_red(&cl->cl_q))
1492 		return (red_updateq(cl->cl_red, ev));
1495 	if (q_is_blue(&cl->cl_q))
1496 		return (blue_updateq(cl->cl_blue, ev));
1498 	if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1499 		return (sfb_updateq(cl->cl_sfb, ev));
1506 	struct qfq_class *cl;
1510 	if ((cl = qfq_clh_to_clp(qif, qid)) == NULL)
1513 	sp->class_handle = cl->cl_handle;
1514 	sp->index = cl->cl_grp->qfg_index;
1515 	sp->weight = (QFQ_ONE_FP / cl->cl_inv_w);
1516 	sp->lmax = cl->cl_lmax;
1517 	sp->qlength = qlen(&cl->cl_q);
1518 	sp->qlimit = qlimit(&cl->cl_q);
1519 	sp->period = cl->cl_period;
1520 	sp->xmitcnt = cl->cl_xmitcnt;
1521 	sp->dropcnt = cl->cl_dropcnt;
1523 	sp->qtype = qtype(&cl->cl_q);
1524 	sp->qstate = qstate(&cl->cl_q);
1526 	if (q_is_red(&cl->cl_q))
1527 		red_getstats(cl->cl_red, &sp->red[0]);
1530 	if (q_is_rio(&cl->cl_q))
1531 		rio_getstats(cl->cl_rio, &sp->red[0]);
1534 	if (q_is_blue(&cl->cl_q))
1535 		blue_getstats(cl->cl_blue, &sp->blue);
1537 	if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1538 		sfb_getstats(cl->cl_sfb, &sp->sfb);
1547 	struct qfq_class *cl;
1557 	if ((cl = qif->qif_class_tbl[i]) != NULL && cl->cl_handle == chandle)
1558 		return (cl);
1560 		if ((cl = qif->qif_class_tbl[i]) != NULL &&
1561 		    cl->cl_handle == chandle)
1562 			return (cl);
1609 qfq_calc_index(struct qfq_class *cl, u_int32_t inv_w, u_int32_t maxlen)
1627 		    if_name(QFQIF_IFP(cl->cl_qif)), qfq_style(cl->cl_qif),
1628 		    cl->cl_handle, index, (u_int32_t)(QFQ_ONE_FP/inv_w),
1710 	    ifq->ifcq_disc_slots[i].cl, m, m_pftag(m)));
1837 		ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl = cl0;
1840 		ifq->ifcq_disc_slots[SCIDX_BK].cl = cl1;
1843 		ifq->ifcq_disc_slots[SCIDX_BE].cl = cl2;
1846 		ifq->ifcq_disc_slots[SCIDX_RD].cl = cl3;
1849 		ifq->ifcq_disc_slots[SCIDX_OAM].cl = cl4;
1852 		ifq->ifcq_disc_slots[SCIDX_AV].cl = cl5;
1855 		ifq->ifcq_disc_slots[SCIDX_RV].cl = cl6;
1858 		ifq->ifcq_disc_slots[SCIDX_VI].cl = cl7;
1861 		ifq->ifcq_disc_slots[SCIDX_VO].cl = cl8;
1864 		ifq->ifcq_disc_slots[SCIDX_CTL].cl = cl9;
1888 		ifq->ifcq_disc_slots[i].cl = NULL;
1914 	struct qfq_class *cl;
1929 	cl = ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl;
1933 		err = qfq_resumeq(qif, cl);
1937 		err = qfq_suspendq(qif, cl);
1956 			qfq_purgeq(qif, cl, 0, NULL, NULL);
1967 qfq_resumeq(struct qfq_if *qif, struct qfq_class *cl)
1975 	if (q_is_rio(&cl->cl_q))
1976 		err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE);
1980 	if (q_is_red(&cl->cl_q))
1981 		err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE);
1985 	if (q_is_blue(&cl->cl_q))
1986 		err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE);
1989 	if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1990 		err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE);
1993 		qstate(&cl->cl_q) = QS_RUNNING;
1999 qfq_suspendq(struct qfq_if *qif, struct qfq_class *cl)
2007 	if (q_is_rio(&cl->cl_q))
2008 		err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE);
2012 	if (q_is_red(&cl->cl_q))
2013 		err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE);
2017 	if (q_is_blue(&cl->cl_q))
2018 		err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE);
2021 	if (q_is_sfb(&cl->cl_q)) {
2022 		if (cl->cl_sfb != NULL) {
2023 			err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE);
2025 			VERIFY(cl->cl_flags & QFCF_LAZY);
2031 		qstate(&cl->cl_q) = QS_SUSPENDED;