net/ipv4/tcp_bbr.c

44  * we estimated that we reached the full bw of the pipe then we enter PROBE_BW;
84 	BBR_PROBE_BW,	/* discover, share bw: pace around estimated bw */
93 	struct minmax bw;	/* Max recent delivery rate in pkts/uS << 24 */
106 		lt_use_bw:1;	     /* use lt_bw as our bw estimate? */
113 		full_bw_reached:1,   /* reached full bw in Startup? */
114 		full_bw_cnt:2,	/* number of rounds without large bw gains */
119 	u32	full_bw;	/* recent bw, to estimate if pipe is full */
132 /* Window length of bw filter (in rounds): */
141 /* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck.
161 /* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw: */
163 	BBR_UNIT * 5 / 4,	/* probe for more available bw */
164 	BBR_UNIT * 3 / 4,	/* drain queue and/or yield bw to other flows */
165 	BBR_UNIT, BBR_UNIT, BBR_UNIT,	/* cruise at 1.0*bw to utilize pipe, */
178 /* If bw has increased significantly (1.25x), there may be more bw available: */
180 /* But after 3 rounds w/o significant bw growth, estimate pipe is full: */
184 /* The minimum number of rounds in an LT bw sampling interval: */
188 /* If 2 intervals have a bw ratio <= 1/8, their bw is "consistent": */
190 /* If 2 intervals have a bw diff <= 4 Kbit/sec their bw is "consistent": */
219 	return minmax_get(&bbr->bw);
255 /* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */
256 static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain)
258 	u64 rate = bw;
270 	u64 bw;
279 	bw = (u64)tcp_snd_cwnd(tp) * BW_UNIT;
280 	do_div(bw, rtt_us);
282 		   bbr_bw_to_pacing_rate(sk, bw, bbr_high_gain));
285 /* Pace using current bw estimate and a gain factor. */
286 static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)
290 	unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain);
341 		/* Avoid pointless buffer overflows: pace at est. bw if we don't
353  * bdp = ceil(bw * min_rtt * gain)
360 static u32 bbr_bdp(struct sock *sk, u32 bw, int gain)
375 	w = (u64)bw * bbr->min_rtt_us;
413 static u32 bbr_inflight(struct sock *sk, u32 bw, int gain)
417 	inflight = bbr_bdp(sk, bw, gain);
431  *   in_network_at_edt = inflight_at_edt - (EDT - now) * bw
516 /* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss
520 			 u32 acked, u32 bw, int gain)
532 	target_cwnd = bbr_bdp(sk, bw, gain);
562 	u32 inflight, bw;
564 	/* The pacing_gain of 1.0 paces at the estimated bw to try to fully
571 	bw = bbr_max_bw(sk);
573 	/* A pacing_gain > 1.0 probes for bw by trying to raise inflight to at
581 			 inflight >= bbr_inflight(sk, bw, bbr->pacing_gain));
583 	/* A pacing_gain < 1.0 tries to drain extra queue we added if bw
584 	 * probing didn't find more bw. If inflight falls to match BDP then we
588 		inflight <= bbr_inflight(sk, bw, BBR_UNIT);
600 /* Gain cycling: cycle pacing gain to converge to fair share of available bw. */
657 /* Long-term bw sampling interval is done. Estimate whether we're policed. */
658 static void bbr_lt_bw_interval_done(struct sock *sk, u32 bw)
663 	if (bbr->lt_bw) {  /* do we have bw from a previous interval? */
664 		/* Is new bw close to the lt_bw from the previous interval? */
665 		diff = abs(bw - bbr->lt_bw);
670 			bbr->lt_bw = (bw + bbr->lt_bw) >> 1;  /* avg 2 intvls */
677 	bbr->lt_bw = bw;
693 	u64 bw;
707 	 * Starting samples earlier includes bursts that over-estimate the bw.
755 	bw = (u64)delivered * BW_UNIT;
756 	do_div(bw, t);
757 	bbr_lt_bw_interval_done(sk, bw);
765 	u64 bw;
785 	bw = div64_long((u64)rs->delivered * BW_UNIT, rs->interval_us);
790 	 * bw, causing needless slow-down. Thus, to continue to send at the
792 	 * they describe the path bw at least as well as our bw model.
798 	if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) {
799 		/* Incorporate new sample into our max bw filter. */
800 		minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw);
813  * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms).
866  * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by
1030 	u32 bw;
1034 	bw = bbr_bw(sk);
1035 	bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);
1036 	bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain);
1056 	minmax_reset(&bbr->bw, bbr->rtt_cnt, 0);  /* init max bw to 0 */
1114 		u64 bw = bbr_bw(sk);
1116 		bw = bw * tp->mss_cache * USEC_PER_SEC >> BW_SCALE;
1118 		info->bbr.bbr_bw_lo		= (u32)bw;
1119 		info->bbr.bbr_bw_hi		= (u32)(bw >> 32);