net/classq/classq_sfb.c

132 #define	SFB_RANDOM(sp, tmin, tmax)	((sfb_random(sp) % (tmax)) + (tmin))
171 #define	SFB_HINTERVAL(sp) ((sfb_random(sp) % HINTERVAL_MAX) + HINTERVAL_MIN)
212 static void sfb_fclists_clean(struct sfb *sp);
281 sfb_random(struct sfb *sp)
283 	IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd);
288 sfb_calc_holdtime(struct sfb *sp, u_int64_t outbw)
295 		holdtime = SFB_RANDOM(sp, HOLDTIME_MIN, HOLDTIME_MAX);
307 	net_nsectimer(&holdtime, &sp->sfb_holdtime);
311 sfb_calc_pboxtime(struct sfb *sp, u_int64_t outbw)
318 		pboxtime = SFB_RANDOM(sp, PBOXTIME_MIN, PBOXTIME_MAX);
330 	net_nsectimer(&pboxtime, &sp->sfb_pboxtime);
331 	net_timerclear(&sp->sfb_pboxfreeze);
335 sfb_calc_hinterval(struct sfb *sp, u_int64_t *t)
350 		hinterval = ((u_int64_t)SFB_HINTERVAL(sp) * NSEC_PER_SEC);
352 	net_nsectimer(&hinterval, &sp->sfb_hinterval);
355 	net_timeradd(&now, &sp->sfb_hinterval, &sp->sfb_nextreset);
364 	struct sfb *sp;
368 	sp = zalloc(sfb_zone);
369 	if (sp == NULL) {
374 	bzero(sp, sfb_size);
375 	if ((sp->sfb_bins = _MALLOC(sizeof (*sp->sfb_bins), M_DEVBUF,
378 		sfb_destroy(sp);
382 	if ((sp->sfb_fc_lists = _MALLOC(sizeof (*sp->sfb_fc_lists), M_DEVBUF,
386 		sfb_destroy(sp);
390 	sp->sfb_flags = (flags & SFBF_USERFLAGS);
391 	sp->sfb_ifp = ifp;
392 	sp->sfb_qlim = qlim;
393 	sp->sfb_qid = qid;
395 	sfb_resetq(sp, -1);
397 	return (sp);
401 sfb_fclist_append(struct sfb *sp, struct sfb_fc_list *fcl)
403 	IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd);
404 	ifnet_fclist_append(sp, fcl);
408 sfb_fclists_clean(struct sfb *sp)
414 		struct sfb_fc_list *fcl = SFB_FC_LIST(sp, i);
416 			sfb_fclist_append(sp, fcl);
421 sfb_destroy(struct sfb *sp)
423 	sfb_fclists_clean(sp);
424 	if (sp->sfb_bins != NULL) {
425 		_FREE(sp->sfb_bins, M_DEVBUF);
426 		sp->sfb_bins = NULL;
428 	if (sp->sfb_fc_lists != NULL) {
429 		_FREE(sp->sfb_fc_lists, M_DEVBUF);
430 		sp->sfb_fc_lists = NULL;
432 	zfree(sfb_zone, sp);
436 sfb_resetq(struct sfb *sp, cqev_t ev)
438 	struct ifnet *ifp = sp->sfb_ifp;
444 		(*sp->sfb_bins)[0].fudge = sfb_random(sp);
445 		(*sp->sfb_bins)[1].fudge = sfb_random(sp);
446 		sp->sfb_allocation = ((sfb_allocation == 0) ?
447 		    (sp->sfb_qlim / 3) : sfb_allocation);
448 		sp->sfb_drop_thresh = sp->sfb_allocation +
449 		    (sp->sfb_allocation >> 1);
452 	sp->sfb_clearpkts = 0;
453 	sp->sfb_current = 0;
456 	sp->sfb_eff_rate = eff_rate;
458 	sfb_calc_holdtime(sp, eff_rate);
459 	sfb_calc_pboxtime(sp, eff_rate);
460 	sfb_calc_hinterval(sp, NULL);
464 		sfb_fclists_clean(sp);
466 	bzero(sp->sfb_bins, sizeof (*sp->sfb_bins));
467 	bzero(&sp->sfb_stats, sizeof (sp->sfb_stats));
475 	    if_name(ifp), sp->sfb_qid, (u_int64_t)sp->sfb_holdtime.tv_nsec,
476 	    (u_int64_t)sp->sfb_pboxtime.tv_nsec,
477 	    (u_int32_t)sp->sfb_allocation, (u_int32_t)sp->sfb_drop_thresh,
478 	    (int)sp->sfb_hinterval.tv_sec, (int)sizeof (*sp->sfb_bins),
483 sfb_getstats(struct sfb *sp, struct sfb_stats *sps)
485 	sps->allocation = sp->sfb_allocation;
486 	sps->dropthresh = sp->sfb_drop_thresh;
487 	sps->clearpkts = sp->sfb_clearpkts;
488 	sps->current = sp->sfb_current;
490 	net_timernsec(&sp->sfb_holdtime, &sp->sfb_stats.hold_time);
491 	net_timernsec(&sp->sfb_pboxtime, &sp->sfb_stats.pbox_time);
492 	net_timernsec(&sp->sfb_hinterval, &sp->sfb_stats.rehash_intval);
493 	*(&(sps->sfbstats)) = *(&(sp->sfb_stats));
495 	_CASSERT(sizeof ((*sp->sfb_bins)[0].stats) ==
498 	bcopy(&(*sp->sfb_bins)[0].stats, &sps->binstats[0].stats,
500 	bcopy(&(*sp->sfb_bins)[1].stats, &sps->binstats[1].stats,
505 sfb_swap_bins(struct sfb *sp, u_int32_t len)
509 	if (sp->sfb_flags & SFBF_SUSPENDED)
512 	s = sp->sfb_current;
515 	(*sp->sfb_bins)[s].fudge = sfb_random(sp); /* recompute perturbation */
516 	sp->sfb_clearpkts = len;
517 	sp->sfb_stats.num_rehash++;
519 	s = (sp->sfb_current ^= 1);	/* flip the bit (swap current) */
523 		    "qlen=%d\n", if_name(sp->sfb_ifp), sp->sfb_qid, s, len);
527 	bzero(&(*sp->sfb_bins)[s].freezetime,
528 	    sizeof ((*sp->sfb_bins)[s].freezetime));
532 		struct sfb_fc_list *fcl = SFB_FC_LIST(sp, i);
535 			sfb_fclist_append(sp, fcl);
540 			cbin = SFB_BINST(sp, j, i, s);		/* current */
541 			wbin = SFB_BINST(sp, j, i, s ^ 1);	/* warm-up */
560 sfb_pcheck(struct sfb *sp, struct pf_mtag *t)
567 	s = sp->sfb_current;
579 	if (SFB_BINST(sp, 0, SFB_BINMASK(t->pftag_qpriv8[(s << 1)]),
581 	    SFB_BINST(sp, 1, SFB_BINMASK(t->pftag_qpriv8[(s << 1) + 1]),
591 		if (SFB_BINST(sp, i, n, s)->pmark < SFB_PMARK_TH)
599 sfb_penalize(struct sfb *sp, struct pf_mtag *t, struct timespec *now)
604 	if (!sfb_ratelimit || !sfb_pcheck(sp, t))
607 	net_timersub(now, &sp->sfb_pboxfreeze, &delta);
608 	if (net_timercmp(&delta, &sp->sfb_pboxtime, <)) {
615 		w = sp->sfb_current ^ 1;
624 		bin = SFB_BINST(sp, 0, n, w);
625 		if (bin->pkts >= sp->sfb_allocation)
626 			sfb_increment_bin(sp, bin, SFB_BINFT(sp, 0, n, w), now);
630 		bin = SFB_BINST(sp, 1, n, w);
631 		if (bin->pkts >= sp->sfb_allocation)
632 			sfb_increment_bin(sp, bin, SFB_BINFT(sp, 1, n, w), now);
640 			bin = SFB_BINST(sp, i, n, w);
641 			if (bin->pkts >= sp->sfb_allocation) {
642 				sfb_increment_bin(sp, bin,
643 				    SFB_BINFT(sp, i, n, w), now);
652 	*(&sp->sfb_pboxfreeze) = *now;
658 sfb_adjust_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft,
664 	if (net_timercmp(&delta, &sp->sfb_holdtime, <)) {
667 			    "(delta=%llu nsec)\n", if_name(sp->sfb_ifp),
668 			    sp->sfb_qid, inc ?  "increment" : "decrement",
683 sfb_decrement_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft,
686 	return (sfb_adjust_bin(sp, bin, ft, now, FALSE));
690 sfb_increment_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft,
693 	return (sfb_adjust_bin(sp, bin, ft, now, TRUE));
697 sfb_dq_update_bins(struct sfb *sp, struct pf_mtag *t, struct timespec *now)
706 	s = sp->sfb_current;
715 	bin = SFB_BINST(sp, 0, n, s);
719 		sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 0, n, s), now);
721 	if (bin->pkts <= (sp->sfb_allocation >> 2)) {
723 		fcl = SFB_FC_LIST(sp, n);
725 			sfb_fclist_append(sp, fcl);
730 	bin = SFB_BINST(sp, 1, n, s);
734 		sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 1, n, s), now);
742 		bin = SFB_BINST(sp, i, n, s);
746 			sfb_decrement_bin(sp, bin,
747 			    SFB_BINFT(sp, i, n, s), now);
749 		if (bin->pkts <= (sp->sfb_allocation >> 2)) {
752 				fcl = SFB_FC_LIST(sp, n);
754 					sfb_fclist_append(sp, fcl);
762 sfb_eq_update_bins(struct sfb *sp, struct pf_mtag *t)
769 	s = sp->sfb_current;
777 	SFB_BINST(sp, 0, SFB_BINMASK(t->pftag_qpriv8[(s << 1)]), s)->pkts++;
780 	SFB_BINST(sp, 1, SFB_BINMASK(t->pftag_qpriv8[(s << 1) + 1]), s)->pkts++;
788 		SFB_BINST(sp, i, n, s)->pkts++;
794 sfb_bin_addfcentry(struct sfb *sp, struct pf_mtag *t)
801 	s = sp->sfb_current;
807 		sp->sfb_stats.null_flowhash++;
815 	fcl = SFB_FC_LIST(sp, SFB_BINMASK(t->pftag_qpriv8[(s << 1)]));
823 	IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd);
828 		sp->sfb_stats.flow_controlled++;
838 sfb_drop_early(struct sfb *sp, struct pf_mtag *t, u_int16_t *pmin,
847 	s = sp->sfb_current;
858 	bin = SFB_BINST(sp, 0, n, s);
862 	if (bin->pkts >= sp->sfb_allocation) {
863 		if (bin->pkts >= sp->sfb_drop_thresh)
865 		sfb_increment_bin(sp, bin, SFB_BINFT(sp, 0, n, s), now);
870 	bin = SFB_BINST(sp, 1, n, s);
874 	if (bin->pkts >= sp->sfb_allocation) {
875 		if (bin->pkts >= sp->sfb_drop_thresh)
877 		sfb_increment_bin(sp, bin, SFB_BINFT(sp, 1, n, s), now);
886 		bin = SFB_BINST(sp, i, n, s);
890 		if (bin->pkts >= sp->sfb_allocation) {
891 			if (bin->pkts >= sp->sfb_drop_thresh)
893 			sfb_increment_bin(sp, bin,
894 			    SFB_BINFT(sp, i, n, s), now);
899 	if (sp->sfb_flags & SFBF_SUSPENDED)
910 sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t)
920 	s = sp->sfb_current;
924 	if (net_timercmp(&now, &sp->sfb_nextreset, >=)) {
925 		net_timeradd(&now, &sp->sfb_hinterval, &sp->sfb_nextreset);
926 		sfb_swap_bins(sp, qlen(q));
927 		s = sp->sfb_current;
934 	    (*sp->sfb_bins)[s].fudge) & SFB_HASHMASK);
937 	    (*sp->sfb_bins)[s ^ 1].fudge) & SFB_HASHMASK);
941 	if (sfb_drop_early(sp, t, &pmin, &now)) {
943 		if ((sp->sfb_flags & SFBF_FLOWCTL) &&
947 			if ((sp->sfb_flags & SFBF_SUSPENDED) ||
950 				sp->sfb_stats.drop_early++;
952 		} else if ((sp->sfb_flags & SFBF_ECN) &&
954 		    ((sfb_random(sp) & SFB_MAX_PMARK) <= pmin) &&
955 		    mark_ecn(m, t, sp->sfb_flags) &&
956 		    !(sp->sfb_flags & SFBF_SUSPENDED)) {
958 			sp->sfb_stats.marked_packets++;
962 			sp->sfb_stats.drop_early++;
967 	if (droptype == DTYPE_NODROP && sfb_penalize(sp, t, &now)) {
969 		sp->sfb_stats.drop_pbox++;
975 		sp->sfb_stats.drop_queue++;
979 	    sfb_bin_addfcentry(sp, t)) {
983 			VERIFY(!(sp->sfb_flags & SFBF_SUSPENDED));
984 		} else if (sp->sfb_flags & SFBF_SUSPENDED) {
997 		IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd);
1003 		sfb_eq_update_bins(sp, t);
1005 		sp->sfb_stats.pbox_packets++;
1012 sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge)
1018 	if (!purge && (sp->sfb_flags & SFBF_SUSPENDED))
1026 			net_timerclear(&sp->sfb_getqtime);
1036 		if (net_timerisset(&sp->sfb_getqtime)) {
1040 			net_timersub(&now, &sp->sfb_getqtime, &delta);
1042 			avg = sp->sfb_stats.dequeue_avg;
1057 			sp->sfb_stats.dequeue_avg = avg;
1059 		*(&sp->sfb_getqtime) = *(&now);
1073 		if (sp->sfb_clearpkts > 0)
1074 			sp->sfb_clearpkts--;
1075 	} else if (sp->sfb_clearpkts > 0) {
1076 		sp->sfb_clearpkts--;
1078 		sfb_dq_update_bins(sp, t, &now);
1085 sfb_getq(struct sfb *sp, class_queue_t *q)
1087 	return (sfb_getq_flow(sp, q, 0, FALSE));
1091 sfb_purgeq(struct sfb *sp, class_queue_t *q, u_int32_t flow, u_int32_t *packets,
1097 	IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd);
1099 	while ((m = sfb_getq_flow(sp, q, flow, TRUE)) != NULL) {
1112 sfb_updateq(struct sfb *sp, cqev_t ev)
1114 	struct ifnet *ifp = sp->sfb_ifp;
1123 		if (eff_rate == sp->sfb_eff_rate)
1128 			    "eff_rate=%llu bps\n", if_name(ifp), sp->sfb_qid,
1131 		sfb_calc_holdtime(sp, eff_rate);
1132 		sfb_calc_pboxtime(sp, eff_rate);
1140 			    "link %s\n", if_name(ifp), sp->sfb_qid,
1143 		sfb_resetq(sp, ev);
1153 sfb_suspendq(struct sfb *sp, class_queue_t *q, boolean_t on)
1156 	struct ifnet *ifp = sp->sfb_ifp;
1160 	if ((on && (sp->sfb_flags & SFBF_SUSPENDED)) ||
1161 	    (!on && !(sp->sfb_flags & SFBF_SUSPENDED)))
1164 	if (!(sp->sfb_flags & SFBF_FLOWCTL)) {
1166 		    "flow-control is not enabled", if_name(ifp), sp->sfb_qid,
1173 		    if_name(ifp), sp->sfb_qid, (on ? "SUSPENDED" : "RUNNING"));
1177 		sp->sfb_flags |= SFBF_SUSPENDED;
1179 		sp->sfb_flags &= ~SFBF_SUSPENDED;
1180 		sfb_swap_bins(sp, qlen(q));