Lines Matching refs:tp
265 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line);
269 struct socket *so, struct tcpcb *tp, struct tcpopt *to,
273 struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
276 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack,
282 rack_check_recovery_mode(struct tcpcb *tp,
285 rack_cong_signal(struct tcpcb *tp, struct tcphdr *th,
290 struct inpcb *inp, struct tcpcb *tp);
294 struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
298 rack_earlier_retran(struct tcpcb *tp, struct rack_sendmap *rsm,
305 static void rack_fini(struct tcpcb *tp, int32_t tcb_is_purged);
308 struct inpcb *inp, struct tcpcb *tp, struct tcp_rack *rack);
309 static int32_t rack_handoff_ok(struct tcpcb *tp);
310 static int32_t rack_init(struct tcpcb *tp);
313 rack_log_ack(struct tcpcb *tp, struct tcpopt *to,
316 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len,
320 rack_log_sack_passed(struct tcpcb *tp, struct tcp_rack *rack,
323 static int32_t rack_output(struct tcpcb *tp);
326 struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
330 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack,
333 static void rack_post_recovery(struct tcpcb *tp, struct tcphdr *th);
334 static void rack_remxt_tmr(struct tcpcb *tp);
337 struct inpcb *inp, struct tcpcb *tp, struct tcp_rack *rack);
338 static void rack_set_state(struct tcpcb *tp, struct tcp_rack *rack);
339 static int32_t rack_stopall(struct tcpcb *tp);
341 rack_timer_activate(struct tcpcb *tp, uint32_t timer_type,
343 static int32_t rack_timer_active(struct tcpcb *tp, uint32_t timer_type);
344 static void rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line);
345 static void rack_timer_stop(struct tcpcb *tp, uint32_t timer_type);
347 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack,
350 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
353 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack,
358 struct tcpcb *tp, int32_t * ret_val);
361 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
365 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
367 static void rack_do_drop(struct mbuf *m, struct tcpcb *tp);
369 rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp,
372 rack_do_dropwithreset(struct mbuf *m, struct tcpcb *tp,
376 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
380 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
384 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
388 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
392 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
396 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
400 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
404 struct tcphdr *th, struct tcpcb *tp, int32_t * tlenp, int32_t * thf,
408 struct socket *so, struct tcpcb *tp);
410 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack,
414 tcp_rack_partialack(struct tcpcb *tp, struct tcphdr *th);
418 struct tcpcb *tp, int32_t tlen, int32_t thflags, int32_t * ret_val);
886 rack_progress_timeout_check(struct tcpcb *tp)
889 if (tp->t_maxunacktime && tp->t_acktime && TSTMP_GT(ticks, tp->t_acktime)) {
890 if ((ticks - tp->t_acktime) >= tp->t_maxunacktime) {
897 rack = (struct tcp_rack *)tp->t_fb_ptr;
900 rack_log_progress_event(rack, tp, ticks, PROGRESS_DROP, __LINE__);
954 rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, int32_t t,
957 if (tp->t_logstate != TCP_LOG_STATE_OFF) {
971 TCP_LOG_EVENT(tp, NULL,
1005 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line)
1007 if (rack_verbose_logging && (tp->t_logstate != TCP_LOG_STATE_OFF)) {
1015 log.u_bbr.flex3 = tp->t_maxunacktime;
1016 log.u_bbr.flex4 = tp->t_acktime;
1018 TCP_LOG_EVENT(tp, NULL,
1265 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack, struct tcphdr *th, uint16_t nsegs,
1272 INP_WLOCK_ASSERT(tp->t_inpcb);
1274 tp->ccv->nsegs = nsegs;
1275 tp->ccv->bytes_this_ack = BYTES_THIS_ACK(tp, th);
1279 max = rack->r_ctl.rc_early_recovery_segs * tp->t_maxseg;
1280 if (tp->ccv->bytes_this_ack > max) {
1281 tp->ccv->bytes_this_ack = max;
1284 if (tp->snd_cwnd <= tp->snd_wnd)
1285 tp->ccv->flags |= CCF_CWND_LIMITED;
1287 tp->ccv->flags &= ~CCF_CWND_LIMITED;
1291 stats_voi_update_abs_s32(tp->t_stats, VOI_TCP_CALCFRWINDIFF,
1292 ((int32_t) tp->snd_cwnd) - tp->snd_wnd);
1293 if ((tp->t_flags & TF_GPUTINPROG) &&
1294 SEQ_GEQ(th->th_ack, tp->gput_ack)) {
1295 gput = (((int64_t) (th->th_ack - tp->gput_seq)) << 3) /
1296 max(1, tcp_ts_getticks() - tp->gput_ts);
1297 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
1304 if (tp->t_stats_gput_prev > 0)
1305 stats_voi_update_abs_s32(tp->t_stats,
1307 ((gput - tp->t_stats_gput_prev) * 100) /
1308 tp->t_stats_gput_prev);
1309 tp->t_flags &= ~TF_GPUTINPROG;
1310 tp->t_stats_gput_prev = gput;
1311 if (tp->t_maxpeakrate) {
1316 tcp_update_peakrate_thr(tp);
1320 if (tp->snd_cwnd > tp->snd_ssthresh) {
1321 tp->t_bytes_acked += tp->ccv->bytes_this_ack;
1322 if (tp->t_bytes_acked >= tp->snd_cwnd) {
1323 tp->t_bytes_acked -= tp->snd_cwnd;
1324 tp->ccv->flags |= CCF_ABC_SENTAWND;
1327 tp->ccv->flags &= ~CCF_ABC_SENTAWND;
1328 tp->t_bytes_acked = 0;
1331 if (CC_ALGO(tp)->ack_received != NULL) {
1333 tp->ccv->curack = th->th_ack;
1334 CC_ALGO(tp)->ack_received(tp->ccv, type);
1337 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_LCWIN, tp->snd_cwnd);
1339 if (rack->r_ctl.rc_rack_largest_cwnd < tp->snd_cwnd) {
1340 rack->r_ctl.rc_rack_largest_cwnd = tp->snd_cwnd;
1343 if (tp->t_peakrate_thr && tp->snd_cwnd > tp->t_peakrate_thr) {
1344 tp->snd_cwnd = tp->t_peakrate_thr;
1349 tcp_rack_partialack(struct tcpcb *tp, struct tcphdr *th)
1353 rack = (struct tcp_rack *)tp->t_fb_ptr;
1354 INP_WLOCK_ASSERT(tp->t_inpcb);
1360 rack_post_recovery(struct tcpcb *tp, struct tcphdr *th)
1364 INP_WLOCK_ASSERT(tp->t_inpcb);
1365 rack = (struct tcp_rack *)tp->t_fb_ptr;
1366 if (CC_ALGO(tp)->post_recovery != NULL) {
1367 tp->ccv->curack = th->th_ack;
1368 CC_ALGO(tp)->post_recovery(tp->ccv);
1382 tp->snd_cwnd -= ((reduce * tp->snd_cwnd) / 100);
1384 if (tp->snd_cwnd > tp->snd_ssthresh) {
1386 tp->snd_cwnd = tp->snd_ssthresh;
1391 tp->snd_cwnd += rack->r_ctl.rc_prr_sndcnt;
1394 tp->snd_recover = tp->snd_una;
1395 EXIT_RECOVERY(tp->t_flags);
1399 rack_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type)
1403 INP_WLOCK_ASSERT(tp->t_inpcb);
1405 rack = (struct tcp_rack *)tp->t_fb_ptr;
1409 if (!IN_FASTRECOVERY(tp->t_flags)) {
1414 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
1415 rack->r_ctl.rc_prr_recovery_fs = tp->snd_max - tp->snd_una;
1416 tp->snd_recover = tp->snd_max;
1417 if (tp->t_flags & TF_ECN_PERMIT)
1418 tp->t_flags |= TF_ECN_SND_CWR;
1422 if (!IN_CONGRECOVERY(tp->t_flags) ||
1427 SEQ_GEQ(th->th_ack, tp->snd_recover)) {
1428 EXIT_CONGRECOVERY(tp->t_flags);
1430 tp->snd_recover = tp->snd_max + 1;
1431 if (tp->t_flags & TF_ECN_PERMIT)
1432 tp->t_flags |= TF_ECN_SND_CWR;
1436 tp->t_dupacks = 0;
1437 tp->t_bytes_acked = 0;
1438 EXIT_RECOVERY(tp->t_flags);
1439 tp->snd_ssthresh = max(2, min(tp->snd_wnd, tp->snd_cwnd) / 2 /
1440 tp->t_maxseg) * tp->t_maxseg;
1441 tp->snd_cwnd = tp->t_maxseg;
1442 if (tp->t_flags & TF_ECN_PERMIT)
1443 tp->t_flags |= TF_ECN_SND_CWR;
1448 tp->snd_cwnd = tp->snd_cwnd_prev;
1449 tp->snd_ssthresh = tp->snd_ssthresh_prev;
1450 tp->snd_recover = tp->snd_recover_prev;
1451 if (tp->t_flags & TF_WASFRECOVERY)
1452 ENTER_FASTRECOVERY(tp->t_flags);
1453 if (tp->t_flags & TF_WASCRECOVERY)
1454 ENTER_CONGRECOVERY(tp->t_flags);
1455 tp->snd_nxt = tp->snd_max;
1456 tp->t_badrxtwin = 0;
1460 if (CC_ALGO(tp)->cong_signal != NULL) {
1462 tp->ccv->curack = th->th_ack;
1463 CC_ALGO(tp)->cong_signal(tp->ccv, type);
1470 rack_cc_after_idle(struct tcpcb *tp, int reduce_largest)
1474 INP_WLOCK_ASSERT(tp->t_inpcb);
1478 if (tp->t_state == TCPS_ESTABLISHED)
1481 if (CC_ALGO(tp)->after_idle != NULL)
1482 CC_ALGO(tp)->after_idle(tp->ccv);
1484 if (tp->snd_cwnd == 1)
1485 i_cwnd = tp->t_maxseg; /* SYN(-ACK) lost */
1487 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),
1488 max(2 * tp->t_maxseg, V_tcp_initcwnd_segments * 1460));
1490 i_cwnd = min(4 * tp->t_maxseg,
1491 max(2 * tp->t_maxseg, 4380));
1494 if (tp->t_maxseg > 2190)
1495 i_cwnd = 2 * tp->t_maxseg;
1496 else if (tp->t_maxseg > 1095)
1497 i_cwnd = 3 * tp->t_maxseg;
1499 i_cwnd = 4 * tp->t_maxseg;
1506 if (((struct tcp_rack *)tp->t_fb_ptr)->r_ctl.rc_rack_largest_cwnd > i_cwnd)
1507 ((struct tcp_rack *)tp->t_fb_ptr)->r_ctl.rc_rack_largest_cwnd = i_cwnd;
1514 if (tp->snd_cwnd < i_cwnd) {
1515 tp->snd_cwnd = i_cwnd;
1531 #define DELAY_ACK(tp, tlen) \
1532 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
1533 ((tp->t_flags & TF_DELACK) == 0) && \
1534 (tlen <= tp->t_maxseg) && \
1535 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
1538 rack_calc_rwin(struct socket *so, struct tcpcb *tp)
1550 tp->rcv_wnd = imax(win, (int)(tp->rcv_adv - tp->rcv_nxt));
1554 rack_do_drop(struct mbuf *m, struct tcpcb *tp)
1559 if (tp != NULL)
1560 INP_WUNLOCK(tp->t_inpcb);
1566 rack_do_dropwithreset(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t rstreason, int32_t tlen)
1568 if (tp != NULL) {
1569 tcp_dropwithreset(m, th, tp, tlen, rstreason);
1570 INP_WUNLOCK(tp->t_inpcb);
1582 rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t thflags, int32_t tlen, int32_t * ret_val)
1600 if (tp->t_state == TCPS_SYN_RECEIVED && (thflags & TH_ACK) &&
1601 (SEQ_GT(tp->snd_una, th->th_ack) ||
1602 SEQ_GT(th->th_ack, tp->snd_max))) {
1604 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
1608 rack = (struct tcp_rack *)tp->t_fb_ptr;
1610 tp->t_flags |= TF_ACKNOW;
1617 rack_process_rst(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp)
1631 if ((SEQ_GEQ(th->th_seq, (tp->last_ack_sent - 1)) &&
1632 SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) ||
1633 (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) {
1636 KASSERT(tp->t_state != TCPS_SYN_SENT,
1637 ("%s: TH_RST for TCPS_SYN_SENT th %p tp %p",
1638 __func__, th, tp));
1641 (tp->last_ack_sent == th->th_seq) ||
1642 (tp->rcv_nxt == th->th_seq) ||
1643 ((tp->last_ack_sent - 1) == th->th_seq)) {
1646 switch (tp->t_state) {
1658 tcp_state_change(tp, TCPS_CLOSED);
1661 tp = tcp_close(tp);
1664 rack_do_drop(m, tp);
1668 tcp_respond(tp, mtod(m, void *), th, m,
1669 tp->rcv_nxt, tp->snd_nxt, TH_ACK);
1670 tp->last_ack_sent = tp->rcv_nxt;
1685 rack_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * ret_val)
1691 SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&
1692 SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) {
1693 tp = tcp_drop(tp, ECONNRESET);
1695 rack_do_drop(m, tp);
1698 tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt,
1699 tp->snd_nxt, TH_ACK);
1700 tp->last_ack_sent = tp->rcv_nxt;
1714 rack_ts_check(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t tlen, int32_t thflags, int32_t * ret_val)
1718 if (tcp_ts_getticks() - tp->ts_recent_age > TCP_PAWS_IDLE) {
1729 tp->ts_recent = 0;
1736 rack_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
1752 rack_drop_checks(struct tcpopt *to, struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * tlenp, int32_t * thf, int32_t * drop_hdrlen, int32_t * ret_val)
1760 todrop = tp->rcv_nxt - th->th_seq;
1786 tp->t_flags |= TF_ACKNOW;
1797 if (tp->t_flags & TF_SACK_PERMIT) {
1798 tcp_update_sack_list(tp, th->th_seq,
1804 tp->t_flags |= TF_ACKNOW;
1809 tp->t_flags |= TF_ACKNOW;
1825 todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd);
1837 if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) {
1838 tp->t_flags |= TF_ACKNOW;
1841 rack_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
1971 rack_calc_thresh_tlp(struct tcpcb *tp, struct tcp_rack *rack,
1986 maxseg = tcp_maxseg(tp);
1991 if (((tp->snd_max - tp->snd_una) - rack->r_ctl.rc_sacked + rack->r_ctl.rc_holes_rxt) <= maxseg) {
2045 if (thresh > TICKS_2_MSEC(tp->t_rxtcur)) {
2046 thresh = TICKS_2_MSEC(tp->t_rxtcur);
2060 rack_check_recovery_mode(struct tcpcb *tp, uint32_t tsused)
2072 rack = (struct tcp_rack *)tp->t_fb_ptr;
2076 srtt_cur = tp->t_srtt >> TCP_RTT_SHIFT;
2100 rack->r_ctl.rc_cwnd_at = tp->snd_cwnd;
2101 rack->r_ctl.rc_ssthresh_at = tp->snd_ssthresh;
2102 rack_cong_signal(tp, NULL, CC_NDUPACK);
2107 rack_get_persists_timer_val(struct tcpcb *tp, struct tcp_rack *rack)
2113 t = TICKS_2_MSEC((tp->t_srtt >> TCP_RTT_SHIFT) + ((tp->t_rttvar * 4) >> TCP_RTT_SHIFT));
2114 TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift],
2116 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
2117 tp->t_rxtshift++;
2124 rack_timer_start(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2144 return (rack_get_persists_timer_val(tp, rack));
2150 if (SEQ_LT(tp->snd_una, tp->snd_max) || sbavail(&(tp->t_inpcb->inp_socket->so_snd))) {
2152 to = TICKS_2_MSEC(tp->t_rxtcur);
2168 if ((tp->t_flags & TF_SENTFIN) &&
2169 ((tp->snd_max - tp->snd_una) == 1) &&
2177 if (tp->t_srtt) {
2178 srtt_cur = (tp->t_srtt >> TCP_RTT_SHIFT);
2204 if ((tp->snd_max - tp->snd_una) > tp->snd_wnd) {
2226 if (tp->t_srtt) {
2227 srtt_cur = (tp->t_srtt >> TCP_RTT_SHIFT);
2231 thresh = rack_calc_thresh_tlp(tp, rack, rsm, srtt);
2272 rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2275 if (((tp->t_flags & TF_SENTFIN) == 0) &&
2276 (tp->snd_max - tp->snd_una) >= sbavail(&rack->rc_inp->inp_socket->so_snd))
2280 rack_timer_cancel(tp, rack, cts, __LINE__);
2281 tp->t_rxtshift = 0;
2287 rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack)
2295 tp->t_flags &= ~TF_FORCEDATA;
2296 tp->t_rxtshift = 0;
2300 rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts, int32_t line,
2309 inp = tp->t_inpcb;
2315 if ((tp->t_state == TCPS_CLOSED) ||
2316 (tp->t_state == TCPS_LISTEN)) {
2340 if ((tp->snd_wnd == 0) && TCPS_HAVEESTABLISHED(tp->t_state)) {
2342 rack_enter_persist(tp, rack, cts);
2344 (frm_out_sbavail > (tp->snd_max - tp->snd_una)) &&
2345 (tp->snd_wnd < tp->t_maxseg)) &&
2346 TCPS_HAVEESTABLISHED(tp->t_state)) {
2354 rack_enter_persist(tp, rack, cts);
2356 hpts_timeout = rack_timer_start(tp, rack, cts);
2357 if (tp->t_flags & TF_DELACK) {
2373 (tp->t_state <= TCPS_CLOSING)) {
2379 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
2381 hpts_timeout = TP_KEEPIDLE(tp);
2384 hpts_timeout = TP_KEEPINIT(tp);
2416 tcp_hpts_insert(tp->t_inpcb, HPTS_MS_TO_SLOTS(slot));
2424 tcp_hpts_insert(tp->t_inpcb, HPTS_MS_TO_SLOTS(hpts_timeout));
2429 tcp_hpts_insert(tp->t_inpcb, HPTS_MS_TO_SLOTS(hpts_timeout));
2434 if (SEQ_GT(tp->snd_max, tp->snd_una)) {
2435 panic("tp:%p rack:%p tlts:%d cts:%u slot:%u pto:%u -- no timer started?",
2436 tp, rack, tot_len_this_send, cts, slot, hpts_timeout);
2453 rack_timeout_rack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2465 if (tp->t_timers->tt_flags & TT_STOPPED) {
2473 recovery = IN_RECOVERY(tp->t_flags);
2475 if (rack->r_state && (rack->r_state != tp->t_state))
2476 rack_set_state(tp, rack);
2477 rsm = rack_check_recovery_mode(tp, cts);
2485 (rack->r_ctl.rc_prr_sndcnt < tp->t_maxseg)) {
2492 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
2493 } else if ((rack->r_ctl.rc_prr_sndcnt < tp->t_maxseg) &&
2501 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
2507 tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true);
2561 rack_timeout_tlp(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2571 if (tp->t_timers->tt_flags & TT_STOPPED) {
2578 if (rack_progress_timeout_check(tp)) {
2579 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
2588 if (rack->r_state && (rack->r_state != tp->t_state))
2589 rack_set_state(tp, rack);
2590 so = tp->t_inpcb->inp_socket;
2592 out = tp->snd_max - tp->snd_una;
2602 if (amm > tp->t_maxseg) {
2603 amm = tp->t_maxseg;
2604 } else if ((amm < tp->t_maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {
2608 if (IN_RECOVERY(tp->t_flags)) {
2611 if (out + amm <= tp->snd_wnd)
2617 if (out + amm <= tp->snd_wnd)
2623 rack->r_ctl.rc_last_tlp_seq = tp->snd_max;
2644 tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true);
2648 if ((rsm->r_end - rsm->r_start) > tp->t_maxseg) {
2664 nrsm->r_start = (rsm->r_end - tp->t_maxseg);
2686 tp->t_rxtshift++;
2737 rack_timeout_delack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2739 if (tp->t_timers->tt_flags & TT_STOPPED) {
2743 tp->t_flags &= ~TF_DELACK;
2744 tp->t_flags |= TF_ACKNOW;
2759 rack_timeout_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2764 inp = tp->t_inpcb;
2766 if (tp->t_timers->tt_flags & TT_STOPPED) {
2771 if (rack_progress_timeout_check(tp)) {
2775 KASSERT(inp != NULL, ("%s: tp %p tp->t_inpcb == NULL", __func__, tp));
2787 if (tp->t_rxtshift == TCP_MAXRXTSHIFT &&
2788 (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
2789 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {
2796 tp->snd_una == tp->snd_max)
2797 rack_exit_persist(tp, rack);
2803 if (tp->t_state > TCPS_CLOSE_WAIT &&
2804 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
2810 tp->t_flags |= TF_FORCEDATA;
2823 rack_timeout_keepalive(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2828 if (tp->t_timers->tt_flags & TT_STOPPED) {
2832 inp = tp->t_inpcb;
2839 if (tp->t_state < TCPS_ESTABLISHED)
2842 tp->t_state <= TCPS_CLOSING) {
2843 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
2850 * number tp->snd_una-1 causes the transmitted zero-length
2858 tcp_respond(tp, t_template->tt_ipgen,
2860 tp->rcv_nxt, tp->snd_una - 1, 0);
2864 rack_start_hpts_timer(rack, tp, cts, __LINE__, 0, 0, 0);
2877 rack_remxt_tmr(struct tcpcb *tp)
2887 rack = (struct tcp_rack *)tp->t_fb_ptr;
2888 rack_timer_cancel(tp, rack, tcp_ts_getticks(), __LINE__);
2890 if (rack->r_state && (rack->r_state != tp->t_state))
2891 rack_set_state(tp, rack);
2924 if (rack->r_ctl.rc_prr_sndcnt < tp->t_maxseg)
2925 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
2934 rack_timeout_rxt(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
2941 inp = tp->t_inpcb;
2942 if (tp->t_timers->tt_flags & TT_STOPPED) {
2945 if (rack_progress_timeout_check(tp)) {
2950 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
2951 (tp->snd_una == tp->snd_max)) {
2960 if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) {
2961 tp->t_rxtshift = TCP_MAXRXTSHIFT;
2965 (tp->t_softerror ? (uint16_t) tp->t_softerror : ETIMEDOUT));
2968 rack_remxt_tmr(tp);
2969 if (tp->t_state == TCPS_SYN_SENT) {
2974 tp->snd_cwnd = 1;
2975 } else if (tp->t_rxtshift == 1) {
2985 tp->snd_cwnd_prev = tp->snd_cwnd;
2986 tp->snd_ssthresh_prev = tp->snd_ssthresh;
2987 tp->snd_recover_prev = tp->snd_recover;
2988 if (IN_FASTRECOVERY(tp->t_flags))
2989 tp->t_flags |= TF_WASFRECOVERY;
2991 tp->t_flags &= ~TF_WASFRECOVERY;
2992 if (IN_CONGRECOVERY(tp->t_flags))
2993 tp->t_flags |= TF_WASCRECOVERY;
2995 tp->t_flags &= ~TF_WASCRECOVERY;
2996 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
2997 tp->t_flags |= TF_PREVVALID;
2999 tp->t_flags &= ~TF_PREVVALID;
3001 if ((tp->t_state == TCPS_SYN_SENT) ||
3002 (tp->t_state == TCPS_SYN_RECEIVED))
3003 rexmt = MSEC_2_TICKS(RACK_INITIAL_RTO * tcp_syn_backoff[tp->t_rxtshift]);
3005 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
3006 TCPT_RANGESET(tp->t_rxtcur, rexmt,
3017 isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) ? true : false;
3024 ((tp->t_state == TCPS_ESTABLISHED) ||
3025 (tp->t_state == TCPS_FIN_WAIT_1))) {
3030 * before further clamping down. 'tp->t_rxtshift % 2 == 0'
3033 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
3035 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
3036 tp->t_rxtshift % 2 == 0)) {
3043 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
3045 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
3047 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
3056 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
3058 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
3062 tp->t_maxseg = V_tcp_v6mssdflt;
3067 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
3075 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
3077 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
3081 tp->t_maxseg = V_tcp_mssdflt;
3086 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
3099 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
3100 (tp->t_rxtshift >= 6)) {
3101 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
3102 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
3103 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
3115 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
3116 (tp->t_rxtshift == 3))
3117 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);
3124 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
3126 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0)
3127 in6_losing(tp->t_inpcb);
3130 in_losing(tp->t_inpcb);
3131 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
3132 tp->t_srtt = 0;
3135 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
3136 tp->snd_recover = tp->snd_max;
3137 tp->t_flags |= TF_ACKNOW;
3138 tp->t_rtttime = 0;
3139 rack_cong_signal(tp, NULL, CC_RTO);
3145 rack_process_timers(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t hpts_calling)
3153 if (tp->t_state == TCPS_LISTEN) {
3178 tcp_hpts_insert(tp->t_inpcb, HPTS_MS_TO_SLOTS(left));
3186 ret = rack_timeout_delack(tp, rack, cts);
3188 ret = rack_timeout_rack(tp, rack, cts);
3190 ret = rack_timeout_tlp(tp, rack, cts);
3192 ret = rack_timeout_rxt(tp, rack, cts);
3194 ret = rack_timeout_persist(tp, rack, cts);
3196 ret = rack_timeout_keepalive(tp, rack, cts);
3203 rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line)
3230 rack_timer_stop(struct tcpcb *tp, uint32_t timer_type)
3236 rack_stopall(struct tcpcb *tp)
3239 rack = (struct tcp_rack *)tp->t_fb_ptr;
3245 rack_timer_activate(struct tcpcb *tp, uint32_t timer_type, uint32_t delta)
3251 rack_timer_active(struct tcpcb *tp, uint32_t timer_type)
3257 rack_stop_all_timers(struct tcpcb *tp)
3264 if (tcp_timer_active(tp, TT_PERSIST)) {
3266 rack = (struct tcp_rack *)tp->t_fb_ptr;
3269 tcp_timer_suspend(tp, TT_PERSIST);
3270 tcp_timer_suspend(tp, TT_REXMT);
3271 tcp_timer_suspend(tp, TT_KEEP);
3272 tcp_timer_suspend(tp, TT_DELACK);
3276 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
3314 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack,
3333 rack_update_rsm(tp, rack, rsm, ts);
3382 rack_update_rsm(tp, rack, rsm, ts);
3389 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len,
3419 INP_WLOCK_ASSERT(tp->t_inpcb);
3434 rack = (struct tcp_rack *)tp->t_fb_ptr;
3435 snd_una = tp->snd_una;
3448 snd_max = tp->snd_max;
3460 if (SEQ_LT(snd_max, tp->snd_nxt)) {
3465 snd_max = tp->snd_nxt;
3473 if (IN_RECOVERY(tp->t_flags)) {
3522 seq_out = rack_update_entry(tp, rack, rsm, ts, &len);
3532 seq_out = rack_update_entry(tp, rack, rsm, ts, &len);
3548 rack_update_rsm(tp, rack, rsm, ts);
3571 seq_out = rack_update_entry(tp, rack, nrsm, ts, &len);
3581 if (seq_out == tp->snd_max) {
3583 } else if (SEQ_LT(seq_out, tp->snd_max)) {
3586 seq_out, len, tp->snd_una, tp->snd_max);
3593 panic("seq_out not found rack:%p tp:%p",
3594 rack, tp);
3602 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p",
3603 seq_out, len, tp->snd_max, tp);
3633 tcp_rack_xmit_timer_commit(struct tcp_rack *rack, struct tcpcb *tp)
3661 o_srtt = tp->t_srtt;
3662 o_var = tp->t_rttvar;
3663 rack = (struct tcp_rack *)tp->t_fb_ptr;
3664 if (tp->t_srtt != 0) {
3673 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));
3675 tp->t_srtt += delta;
3676 if (tp->t_srtt <= 0)
3677 tp->t_srtt = 1;
3691 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
3692 tp->t_rttvar += delta;
3693 if (tp->t_rttvar <= 0)
3694 tp->t_rttvar = 1;
3695 if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
3696 tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
3703 tp->t_srtt = rtt << TCP_RTT_SHIFT;
3704 tp->t_rttvar = rtt << (TCP_RTTVAR_SHIFT - 1);
3705 tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
3708 rack_log_rtt_upd(tp, rack, rtt, o_srtt, o_var);
3709 tp->t_rttupdated++;
3711 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt));
3713 tp->t_rxtshift = 0;
3725 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
3727 tp->t_softerror = 0;
3731 rack_earlier_retran(struct tcpcb *tp, struct rack_sendmap *rsm,
3755 rack = (struct tcp_rack *)tp->t_fb_ptr;
3757 if (IN_RECOVERY(tp->t_flags)) {
3763 EXIT_RECOVERY(tp->t_flags);
3764 tp->snd_recover = tp->snd_una;
3765 if (rack->r_ctl.rc_cwnd_at > tp->snd_cwnd)
3766 tp->snd_cwnd = rack->r_ctl.rc_cwnd_at;
3767 if (rack->r_ctl.rc_ssthresh_at > tp->snd_ssthresh)
3768 tp->snd_ssthresh = rack->r_ctl.rc_ssthresh_at;
3785 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack,
3810 if (!tp->t_rttlow || tp->t_rttlow > t)
3811 tp->t_rttlow = t;
3821 (!IN_RECOVERY(tp->t_flags))) {
3824 rack->r_ctl.rc_rsm_start = tp->snd_max;
3825 rack->r_ctl.rc_cwnd_at = tp->snd_cwnd;
3826 rack->r_ctl.rc_ssthresh_at = tp->snd_ssthresh;
3827 rack_cong_signal(tp, NULL, CC_NDUPACK);
3832 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
3848 tp->t_rxtshift = 0;
3849 tp->t_softerror = 0;
3865 rack_earlier_retran(tp, rsm, t, cts);
3867 if (!tp->t_rttlow || tp->t_rttlow > t)
3868 tp->t_rttlow = t;
3918 rack_earlier_retran(tp, rsm, t, cts);
3946 rack_log_sack_passed(struct tcpcb *tp,
4003 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack, struct sackblk *sack,
4043 if (tp->t_flags & TF_SENTFIN) {
4049 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {
4060 panic("tp:%p rack:%p sack:%p to:%p prsm:%p",
4061 tp, rack, sack, to, prsm);
4108 rack_update_rtt(tp, rack, rsm, to, cts, SACKED);
4111 rack_log_sack_passed(tp, rack, rsm);
4163 rack_update_rtt(tp, rack, rsm, to, cts, SACKED);
4166 rack_log_sack_passed(tp, rack, rsm);
4250 rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th)
4260 INP_WLOCK_ASSERT(tp->t_inpcb);
4265 rack = (struct tcp_rack *)tp->t_fb_ptr;
4271 if (SEQ_GT(th_ack, tp->snd_una)) {
4272 rack_log_progress_event(rack, tp, ticks, PROGRESS_UPDATE, __LINE__);
4273 tp->t_acktime = ticks;
4287 if ((th_ack - 1) == tp->iss) {
4296 if (tp->t_flags & TF_SENTFIN) {
4301 panic("No rack map tp:%p for th:%p state:%d rack:%p snd_una:%u snd_max:%u snd_nxt:%u chg:%d\n",
4302 tp,
4303 th, tp->t_state, rack,
4304 tp->snd_una, tp->snd_max, tp->snd_nxt, changed);
4313 th_ack, tp->t_state, rack->r_state);
4317 rack_update_rtt(tp, rack, rsm, to, cts, CUM_ACKED);
4409 last_seq = tp->snd_max;
4412 if (SEQ_GT(th_ack, tp->snd_una))
4415 ack_point = tp->snd_una;
4423 SEQ_LT(sack.start, tp->snd_max) &&
4425 SEQ_LEQ(sack.end, tp->snd_max)) {
4428 ((sack.end - sack.start) < (tp->t_maxseg / 8))) {
4510 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[i], to, &rsm, cts);
4520 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
4522 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {
4531 rsm = tcp_rack_output(tp, rack, tsused);
4535 rack->r_ctl.rc_cwnd_at = tp->snd_cwnd;
4536 rack->r_ctl.rc_ssthresh_at = tp->snd_ssthresh;
4538 rack_cong_signal(tp, NULL, CC_NDUPACK);
4543 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
4547 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {
4553 if (SEQ_GT(tp->snd_una, th_ack)) {
4554 snd_una = tp->snd_una;
4558 pipe = ((tp->snd_max - snd_una) - rack->r_ctl.rc_sacked) + rack->r_ctl.rc_holes_rxt;
4559 if (pipe > tp->snd_ssthresh) {
4562 sndcnt = rack->r_ctl.rc_prr_delivered * tp->snd_ssthresh;
4584 limit += tp->t_maxseg;
4585 if (tp->snd_ssthresh > pipe) {
4586 rack->r_ctl.rc_prr_sndcnt = min((tp->snd_ssthresh - pipe), limit);
4591 if (rack->r_ctl.rc_prr_sndcnt >= tp->t_maxseg) {
4605 struct tcpcb *tp, struct tcpopt *to,
4616 rack = (struct tcp_rack *)tp->t_fb_ptr;
4617 if (SEQ_GT(th->th_ack, tp->snd_max)) {
4618 rack_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
4621 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
4622 rack_log_ack(tp, to, th);
4624 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
4638 if (tp->t_flags & TF_NEEDSYN) {
4645 tp->t_flags &= ~TF_NEEDSYN;
4646 tp->snd_una++;
4648 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
4650 tp->rcv_scale = tp->request_r_scale;
4655 INP_WLOCK_ASSERT(tp->t_inpcb);
4657 acked = BYTES_THIS_ACK(tp, th);
4667 if (tp->t_flags & TF_PREVVALID) {
4668 tp->t_flags &= ~TF_PREVVALID;
4669 if (tp->t_rxtshift == 1 &&
4670 (int)(ticks - tp->t_badrxtwin) < 0)
4671 rack_cong_signal(tp, th, CC_RTO_ERR);
4691 if (th->th_ack == tp->snd_max) {
4692 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
4704 if (IN_RECOVERY(tp->t_flags)) {
4705 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
4706 (SEQ_LT(th->th_ack, tp->snd_max))) {
4707 tcp_rack_partialack(tp, th);
4709 rack_post_recovery(tp, th);
4719 rack_ack_received(tp, rack, th, nsegs, CC_ACK, recovery);
4722 tp->snd_wnd -= acked_amount;
4726 (tp->t_state >= TCPS_FIN_WAIT_1)) {
4733 if (IN_RECOVERY(tp->t_flags)) {
4734 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
4735 (SEQ_LT(th->th_ack, tp->snd_max))) {
4736 tcp_rack_partialack(tp, th);
4738 rack_post_recovery(tp, th);
4742 tp->snd_una = th->th_ack;
4743 if (SEQ_GT(tp->snd_una, tp->snd_recover))
4744 tp->snd_recover = tp->snd_una;
4746 if (SEQ_LT(tp->snd_nxt, tp->snd_una)) {
4747 tp->snd_nxt = tp->snd_una;
4749 if (tp->snd_una == tp->snd_max) {
4751 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
4752 tp->t_acktime = 0;
4753 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
4757 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
4758 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
4760 (tp->t_flags2 & TF2_DROP_AF_DATA)) {
4767 tp = tcp_close(tp);
4768 rack_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen);
4785 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
4800 rack = (struct tcp_rack *)tp->t_fb_ptr;
4801 INP_WLOCK_ASSERT(tp->t_inpcb);
4804 (SEQ_LT(tp->snd_wl1, th->th_seq) ||
4805 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
4806 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
4809 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
4811 tp->snd_wnd = tiwin;
4812 tp->snd_wl1 = th->th_seq;
4813 tp->snd_wl2 = th->th_ack;
4814 if (tp->snd_wnd > tp->max_sndwnd)
4815 tp->max_sndwnd = tp->snd_wnd;
4818 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
4819 tp->snd_wnd = tiwin;
4820 tp->snd_wl1 = th->th_seq;
4821 tp->snd_wl2 = th->th_ack;
4825 if ((rack->rc_in_persist != 0) && tp->snd_wnd) {
4826 rack_exit_persist(tp, rack);
4827 tp->snd_nxt = tp->snd_max;
4831 if (tp->t_flags2 & TF2_DROP_AF_DATA) {
4839 TCPS_HAVERCVDFIN(tp->t_state) == 0) {
4866 if (SEQ_GT(th->th_seq + th->th_urp, tp->rcv_up)) {
4867 tp->rcv_up = th->th_seq + th->th_urp;
4869 (tp->rcv_up - tp->rcv_nxt) - 1;
4873 tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA);
4892 if (SEQ_GT(tp->rcv_nxt, tp->rcv_up))
4893 tp->rcv_up = tp->rcv_nxt;
4896 INP_WLOCK_ASSERT(tp->t_inpcb);
4901 * This process logically involves adjusting tp->rcv_wnd as data is
4907 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
4908 (tp->t_flags & TF_FASTOPEN));
4911 TCPS_HAVERCVDFIN(tp->t_state) == 0) {
4913 tcp_seq save_rnxt = tp->rcv_nxt;
4919 * queue with control block tp. Set thflags to whether
4928 if (th->th_seq == tp->rcv_nxt &&
4929 SEGQ_EMPTY(tp) &&
4930 (TCPS_HAVEESTABLISHED(tp->t_state) ||
4932 if (DELAY_ACK(tp, tlen) || tfo_syn) {
4933 rack_timer_cancel(tp, rack,
4935 tp->t_flags |= TF_DELACK;
4938 tp->t_flags |= TF_ACKNOW;
4940 tp->rcv_nxt += tlen;
4959 thflags = tcp_reass(tp, th, &temp, &tlen, m);
4960 tp->t_flags |= TF_ACKNOW;
4962 if ((tp->t_flags & TF_SACK_PERMIT) && (save_tlen > 0)) {
4968 tcp_update_sack_list(tp, save_start,
4970 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
4971 if ((tp->rcv_numsacks >= 1) &&
4972 (tp->sackblks[0].end == save_start)) {
4977 tcp_update_sack_list(tp,
4978 tp->sackblks[0].start,
4979 tp->sackblks[0].end);
4981 tcp_update_dsack_list(tp, save_start,
4986 tcp_update_dsack_list(tp, save_start,
4989 tcp_update_dsack_list(tp, save_start,
5003 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
5012 if (tp->t_flags & TF_NEEDSYN) {
5013 rack_timer_cancel(tp, rack,
5015 tp->t_flags |= TF_DELACK;
5017 tp->t_flags |= TF_ACKNOW;
5019 tp->rcv_nxt++;
5021 switch (tp->t_state) {
5028 tp->t_starttime = ticks;
5031 rack_timer_cancel(tp, rack,
5033 tcp_state_change(tp, TCPS_CLOSE_WAIT);
5041 rack_timer_cancel(tp, rack,
5043 tcp_state_change(tp, TCPS_CLOSING);
5052 rack_timer_cancel(tp, rack,
5055 tcp_twstart(tp);
5062 if ((tp->t_flags & TF_ACKNOW) ||
5063 (sbavail(&so->so_snd) > (tp->snd_max - tp->snd_una))) {
5066 INP_WLOCK_ASSERT(tp->t_inpcb);
5077 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5098 if (__predict_false(th->th_seq != tp->rcv_nxt)) {
5101 if (__predict_false(tp->snd_nxt != tp->snd_max)) {
5104 if (tiwin && tiwin != tp->snd_wnd) {
5107 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
5111 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
5114 if (__predict_false((th->th_ack != tp->snd_una))) {
5121 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
5122 tp->ts_recent_age = tcp_ts_getticks();
5123 tp->ts_recent = to->to_tsval;
5125 rack = (struct tcp_rack *)tp->t_fb_ptr;
5134 /* if (tp->rcv_numsacks)
5135 tcp_clean_sackreport(tp);
5139 tp->rcv_nxt += tlen;
5143 tp->snd_wl1 = th->th_seq;
5147 tp->rcv_up = tp->rcv_nxt;
5152 tcp_trace(TA_INPUT, ostate, tp,
5155 newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
5172 rack_calc_rwin(so, tp);
5176 if (DELAY_ACK(tp, tlen)) {
5177 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
5178 tp->t_flags |= TF_DELACK;
5180 tp->t_flags |= TF_ACKNOW;
5183 if ((tp->snd_una == tp->snd_max) && rack_use_sack_filter)
5184 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
5199 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5217 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
5221 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
5225 if (__predict_false(tp->snd_nxt != tp->snd_max)) {
5233 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
5237 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
5241 if (__predict_false(IN_RECOVERY(tp->t_flags))) {
5245 rack = (struct tcp_rack *)tp->t_fb_ptr;
5252 rack_log_ack(tp, to, th);
5254 if (tiwin != tp->snd_wnd) {
5255 tp->snd_wnd = tiwin;
5256 tp->snd_wl1 = th->th_seq;
5257 if (tp->snd_wnd > tp->max_sndwnd)
5258 tp->max_sndwnd = tp->snd_wnd;
5260 if ((rack->rc_in_persist != 0) && (tp->snd_wnd >= tp->t_maxseg)) {
5261 rack_exit_persist(tp, rack);
5269 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
5270 tp->ts_recent_age = tcp_ts_getticks();
5271 tp->ts_recent = to->to_tsval;
5281 if (tp->t_flags & TF_PREVVALID) {
5282 tp->t_flags &= ~TF_PREVVALID;
5283 if (tp->t_rxtshift == 1 &&
5284 (int)(ticks - tp->t_badrxtwin) < 0)
5285 rack_cong_signal(tp, th, CC_RTO_ERR);
5294 acked = BYTES_THIS_ACK(tp, th);
5298 hhook_run_tcp_est_in(tp, th, to);
5309 rack_ack_received(tp, rack, th, nsegs, CC_ACK, 0);
5311 tp->snd_una = th->th_ack;
5315 tp->snd_wl2 = th->th_ack;
5316 tp->t_dupacks = 0;
5318 /* ND6_HINT(tp); *//* Some progress has been made. */
5329 tcp_trace(TA_INPUT, ostate, tp,
5333 if (tp->snd_una == tp->snd_max) {
5334 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
5335 tp->t_acktime = 0;
5336 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
5353 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5360 rack_calc_rwin(so, tp);
5365 * this is an acceptable SYN segment initialize tp->rcv_nxt and
5366 * tp->irs if seg contains ack then advance tp->snd_una if seg
5373 (SEQ_LEQ(th->th_ack, tp->iss) ||
5374 SEQ_GT(th->th_ack, tp->snd_max))) {
5375 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
5379 TCP_PROBE5(connect__refused, NULL, tp,
5380 mtod(m, const char *), tp, th);
5381 tp = tcp_drop(tp, ECONNREFUSED);
5382 rack_do_drop(m, tp);
5386 rack_do_drop(m, tp);
5390 rack_do_drop(m, tp);
5393 tp->irs = th->th_seq;
5394 tcp_rcvseqinit(tp);
5402 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
5404 tp->rcv_scale = tp->request_r_scale;
5406 tp->rcv_adv += min(tp->rcv_wnd,
5407 TCP_MAXWIN << tp->rcv_scale);
5412 if (DELAY_ACK(tp, tlen) && tlen != 0) {
5413 rack_timer_cancel(tp, (struct tcp_rack *)tp->t_fb_ptr,
5414 ((struct tcp_rack *)tp->t_fb_ptr)->r_ctl.rc_rcvtime, __LINE__);
5415 tp->t_flags |= TF_DELACK;
5417 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output++;
5418 tp->t_flags |= TF_ACKNOW;
5423 tp->t_flags |= TF_ECN_PERMIT;
5430 tp->t_starttime = ticks;
5431 if (tp->t_flags & TF_NEEDFIN) {
5432 tcp_state_change(tp, TCPS_FIN_WAIT_1);
5433 tp->t_flags &= ~TF_NEEDFIN;
5436 tcp_state_change(tp, TCPS_ESTABLISHED);
5437 TCP_PROBE5(connect__established, NULL, tp,
5438 mtod(m, const char *), tp, th);
5439 cc_conn_init(tp);
5450 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN);
5451 tcp_state_change(tp, TCPS_SYN_RECEIVED);
5454 INP_WLOCK_ASSERT(tp->t_inpcb);
5460 if (tlen > tp->rcv_wnd) {
5461 todrop = tlen - tp->rcv_wnd;
5463 tlen = tp->rcv_wnd;
5468 tp->snd_wl1 = th->th_seq - 1;
5469 tp->rcv_up = th->th_seq;
5477 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val))
5480 if (tp->t_state == TCPS_FIN_WAIT_1) {
5494 * XXXjl: we should release the tp also, and
5499 tcp_timer_activate(tp, TT_2MSL,
5502 TP_MAXIDLE(tp)));
5504 tcp_state_change(tp, TCPS_FIN_WAIT_2);
5508 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5519 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5525 rack_calc_rwin(so, tp);
5528 (SEQ_LEQ(th->th_ack, tp->snd_una) ||
5529 SEQ_GT(th->th_ack, tp->snd_max))) {
5530 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
5534 if (tp->t_flags & TF_FASTOPEN) {
5542 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
5548 rack = (struct tcp_rack *)tp->t_fb_ptr;
5562 return (rack_process_rst(m, th, so, tp));
5568 rack_challenge_ack(m, th, tp, &ret_val);
5575 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
5576 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
5577 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
5587 if (SEQ_LT(th->th_seq, tp->irs)) {
5588 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
5591 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
5609 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
5610 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
5612 tp->ts_recent_age = tcp_ts_getticks();
5613 tp->ts_recent = to->to_tsval;
5622 if (tp->t_flags & TF_FASTOPEN) {
5623 tp->snd_wnd = tiwin;
5624 cc_conn_init(tp);
5627 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5633 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
5635 tp->rcv_scale = tp->request_r_scale;
5636 tp->snd_wnd = tiwin;
5642 tp->t_starttime = ticks;
5643 if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) {
5644 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
5645 tp->t_tfo_pending = NULL;
5647 if (tp->t_flags & TF_NEEDFIN) {
5648 tcp_state_change(tp, TCPS_FIN_WAIT_1);
5649 tp->t_flags &= ~TF_NEEDFIN;
5651 tcp_state_change(tp, TCPS_ESTABLISHED);
5652 TCP_PROBE5(accept__established, NULL, tp,
5653 mtod(m, const char *), tp, th);
5655 if (tp->t_tfo_pending) {
5656 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
5657 tp->t_tfo_pending = NULL;
5663 tp->snd_una++;
5671 if (!(tp->t_flags & TF_FASTOPEN))
5673 cc_conn_init(tp);
5680 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))
5681 tp->snd_una++;
5687 (void)tcp_reass(tp, (struct tcphdr *)0, NULL, 0,
5689 tp->snd_wl1 = th->th_seq - 1;
5690 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
5693 if (tp->t_state == TCPS_FIN_WAIT_1) {
5707 * XXXjl: we should release the tp also, and use a
5712 tcp_timer_activate(tp, TT_2MSL,
5715 TP_MAXIDLE(tp)));
5717 tcp_state_change(tp, TCPS_FIN_WAIT_2);
5720 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5731 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5752 __predict_true(SEGQ_EMPTY(tp)) &&
5753 __predict_true(th->th_seq == tp->rcv_nxt)) {
5756 rack = (struct tcp_rack *)tp->t_fb_ptr;
5758 if (rack_fastack(m, th, so, tp, to, drop_hdrlen, tlen,
5763 if (rack_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen,
5769 rack_calc_rwin(so, tp);
5772 return (rack_process_rst(m, th, so, tp));
5779 rack_challenge_ack(m, th, tp, &ret_val);
5786 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
5787 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
5788 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
5791 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
5809 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
5810 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
5812 tp->ts_recent_age = tcp_ts_getticks();
5813 tp->ts_recent = to->to_tsval;
5821 if (tp->t_flags & TF_NEEDSYN) {
5823 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5826 } else if (tp->t_flags & TF_ACKNOW) {
5827 rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
5837 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
5841 if (rack_progress_timeout_check(tp)) {
5842 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
5843 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
5848 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5859 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5864 rack_calc_rwin(so, tp);
5866 return (rack_process_rst(m, th, so, tp));
5872 rack_challenge_ack(m, th, tp, &ret_val);
5879 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
5880 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
5881 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
5884 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
5902 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
5903 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
5905 tp->ts_recent_age = tcp_ts_getticks();
5906 tp->ts_recent = to->to_tsval;
5914 if (tp->t_flags & TF_NEEDSYN) {
5915 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5918 } else if (tp->t_flags & TF_ACKNOW) {
5919 rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
5929 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
5933 if (rack_progress_timeout_check(tp)) {
5934 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
5935 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
5939 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
5945 struct tcpcb *tp, int32_t *tlen, struct tcphdr *th, struct socket *so)
5950 rack = (struct tcp_rack *)tp->t_fb_ptr;
5953 tp = tcp_close(tp);
5955 rack_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen));
5961 tp->rcv_nxt = th->th_seq + *tlen;
5962 tp->t_flags2 |= TF2_DROP_AF_DATA;
5975 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
5981 rack_calc_rwin(so, tp);
5984 return (rack_process_rst(m, th, so, tp));
5990 rack_challenge_ack(m, th, tp, &ret_val);
5997 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
5998 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
5999 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
6002 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
6010 if (rack_check_data_after_close(m, tp, &tlen, th, so))
6028 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
6029 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
6031 tp->ts_recent_age = tcp_ts_getticks();
6032 tp->ts_recent = to->to_tsval;
6040 if (tp->t_flags & TF_NEEDSYN) {
6041 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6043 } else if (tp->t_flags & TF_ACKNOW) {
6044 rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
6054 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
6064 * XXXjl: we should release the tp also, and use a
6069 tcp_timer_activate(tp, TT_2MSL,
6072 TP_MAXIDLE(tp)));
6074 tcp_state_change(tp, TCPS_FIN_WAIT_2);
6077 if (rack_progress_timeout_check(tp)) {
6078 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
6079 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
6083 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6094 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
6100 rack_calc_rwin(so, tp);
6103 return (rack_process_rst(m, th, so, tp));
6109 rack_challenge_ack(m, th, tp, &ret_val);
6116 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
6117 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
6118 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
6121 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
6129 if (rack_check_data_after_close(m, tp, &tlen, th, so))
6147 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
6148 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
6150 tp->ts_recent_age = tcp_ts_getticks();
6151 tp->ts_recent = to->to_tsval;
6159 if (tp->t_flags & TF_NEEDSYN) {
6160 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6162 } else if (tp->t_flags & TF_ACKNOW) {
6163 rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
6173 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
6178 tcp_twstart(tp);
6183 if (rack_progress_timeout_check(tp)) {
6184 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
6185 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
6189 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6200 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
6206 rack_calc_rwin(so, tp);
6209 return (rack_process_rst(m, th, so, tp));
6215 rack_challenge_ack(m, th, tp, &ret_val);
6222 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
6223 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
6224 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
6227 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
6235 if (rack_check_data_after_close(m, tp, &tlen, th, so))
6253 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
6254 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
6256 tp->ts_recent_age = tcp_ts_getticks();
6257 tp->ts_recent = to->to_tsval;
6265 if (tp->t_flags & TF_NEEDSYN) {
6266 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6268 } else if (tp->t_flags & TF_ACKNOW) {
6269 rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
6279 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
6284 tp = tcp_close(tp);
6285 rack_do_drop(m, tp);
6289 if (rack_progress_timeout_check(tp)) {
6290 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
6291 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
6295 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6307 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
6313 rack_calc_rwin(so, tp);
6317 return (rack_process_rst(m, th, so, tp));
6323 rack_challenge_ack(m, th, tp, &ret_val);
6330 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
6331 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
6332 if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val))
6335 if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
6344 if (rack_check_data_after_close(m, tp, &tlen, th, so))
6362 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
6363 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
6365 tp->ts_recent_age = tcp_ts_getticks();
6366 tp->ts_recent = to->to_tsval;
6374 if (tp->t_flags & TF_NEEDSYN) {
6375 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6377 } else if (tp->t_flags & TF_ACKNOW) {
6378 rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
6388 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
6392 if (rack_progress_timeout_check(tp)) {
6393 tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT);
6394 rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
6398 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
6412 rack_init(struct tcpcb *tp)
6416 tp->t_fb_ptr = uma_zalloc(rack_pcb_zone, M_NOWAIT);
6417 if (tp->t_fb_ptr == NULL) {
6426 memset(tp->t_fb_ptr, 0, sizeof(struct tcp_rack));
6428 rack = (struct tcp_rack *)tp->t_fb_ptr;
6432 rack->rc_tp = tp;
6433 if (tp->t_inpcb) {
6434 rack->rc_inp = tp->t_inpcb;
6444 tp->t_delayed_ack = 1;
6446 tp->t_delayed_ack = 0;
6464 if (tp->snd_una != tp->snd_max) {
6470 uma_zfree(rack_pcb_zone, tp->t_fb_ptr);
6471 tp->t_fb_ptr = NULL;
6478 rsm->r_start = tp->snd_una;
6479 rsm->r_end = tp->snd_max;
6485 rack_stop_all_timers(tp);
6486 rack_start_hpts_timer(rack, tp, tcp_ts_getticks(), __LINE__, 0, 0, 0);
6491 rack_handoff_ok(struct tcpcb *tp)
6493 if ((tp->t_state == TCPS_CLOSED) ||
6494 (tp->t_state == TCPS_LISTEN)) {
6498 if ((tp->t_state == TCPS_SYN_SENT) ||
6499 (tp->t_state == TCPS_SYN_RECEIVED)) {
6506 if (tp->t_flags & TF_SACK_PERMIT) {
6517 rack_fini(struct tcpcb *tp, int32_t tcb_is_purged)
6519 if (tp->t_fb_ptr) {
6523 rack = (struct tcp_rack *)tp->t_fb_ptr;
6525 tcp_log_flowend(tp);
6540 uma_zfree(rack_pcb_zone, tp->t_fb_ptr);
6541 tp->t_fb_ptr = NULL;
6544 tp->snd_nxt = tp->snd_max;
6548 rack_set_state(struct tcpcb *tp, struct tcp_rack *rack)
6550 switch (tp->t_state) {
6593 rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb)
6608 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
6615 if (tp->t_flags & TF_DELACK) {
6619 } else if (sbavail(&tp->t_inpcb->inp_socket->so_snd) && (tmr_up == PACE_TMR_RXT)) {
6627 (tp->t_state <= TCPS_CLOSING)) &&
6629 (tp->snd_max == tp->snd_una)) {
6635 if ((tp->t_flags & TF_SENTFIN) &&
6636 ((tp->snd_max - tp->snd_una) == 1) &&
6643 } else if (SEQ_GT(tp->snd_max,tp->snd_una) &&
6666 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
6667 rack_start_hpts_timer(rack, tp, tcp_ts_getticks(), __LINE__, 0, 0, 0);
6672 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos,
6685 rack = (struct tcp_rack *)tp->t_fb_ptr;
6696 INP_WLOCK_ASSERT(tp->t_inpcb);
6697 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
6699 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
6708 TCP_LOG_EVENT(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0,
6726 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&
6738 if (tp->t_idle_reduce && (tp->snd_max == tp->snd_una)) {
6739 if ((ticks - tp->t_rcvtime) >= tp->t_rxtcur) {
6741 rack_cc_after_idle(tp,
6746 tp->t_rcvtime = ticks;
6752 tiwin = th->th_win << tp->snd_scale;
6754 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
6760 if (tp->t_flags & TF_ECN_PERMIT) {
6762 tp->t_flags &= ~TF_ECN_SND_ECE;
6763 tp->t_flags |= TF_ACKNOW;
6767 tp->t_flags |= TF_ECN_SND_ECE;
6779 cc_ecnpkt_handler(tp, th, iptos);
6783 rack_cong_signal(tp, th, CC_ECN);
6793 to.to_tsecr -= tp->ts_offset;
6806 rack->rc_inp = tp->t_inpcb;
6816 rack->r_cpu = inp_to_cpuid(tp->t_inpcb);
6817 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
6819 (tp->t_flags & TF_REQ_SCALE)) {
6820 tp->t_flags |= TF_RCVD_SCALE;
6821 tp->snd_scale = to.to_wscale;
6823 tp->t_flags &= ~TF_REQ_SCALE;
6828 tp->snd_wnd = th->th_win;
6830 (tp->t_flags & TF_REQ_TSTMP)) {
6831 tp->t_flags |= TF_RCVD_TSTMP;
6832 tp->ts_recent = to.to_tsval;
6833 tp->ts_recent_age = cts;
6835 tp->t_flags &= ~TF_REQ_TSTMP;
6837 tcp_mss(tp, to.to_mss);
6838 if ((tp->t_flags & TF_SACK_PERMIT) &&
6840 tp->t_flags &= ~TF_SACK_PERMIT;
6848 if ((tp->t_flags & TF_SACK_PERMIT) == 0) {
6849 tcp_switch_back_to_default(tp);
6850 (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen,
6855 rack->r_is_v6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
6856 tcp_set_hpts(tp->t_inpcb);
6864 if (rack->r_state != tp->t_state)
6865 rack_set_state(tp, rack);
6866 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&rack->r_ctl.rc_map)) != NULL)
6872 tp, &to, drop_hdrlen,
6876 (tp->t_inpcb == NULL)) {
6877 panic("retval:%d tp:%p t_inpcb:NULL state:%d",
6878 retval, tp, prev_state);
6883 * If retval is 1 the tcb is unlocked and most likely the tp
6886 INP_WLOCK_ASSERT(tp->t_inpcb);
6887 tcp_rack_xmit_timer_commit(rack, tp);
6891 (void)tp->t_fb->tfb_tcp_output(tp);
6893 rack_start_hpts_timer(rack, tp, cts, __LINE__, 0, 0, 0);
6896 (SEQ_GT(tp->snd_max, tp->snd_una) ||
6897 (tp->t_flags & TF_DELACK) ||
6899 (tp->t_state <= TCPS_CLOSING)))) {
6901 if ((tp->snd_max == tp->snd_una) &&
6902 ((tp->t_flags & TF_DELACK) == 0) &&
6909 rack_start_hpts_timer(rack, tp, tcp_ts_getticks(), __LINE__, 0, 0, 0);
6914 rack_timer_audit(tp, rack, &so->so_snd);
6922 if (tp->t_inpcb == NULL) {
6923 panic("OP:%d retval:%d tp:%p t_inpcb:NULL state:%d",
6925 retval, tp, prev_state);
6928 INP_WUNLOCK(tp->t_inpcb);
6934 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
6942 rack = (struct tcp_rack *)tp->t_fb_ptr;
6949 rack_hpts_do_segment(m, th, so, tp, drop_hdrlen,
6953 tcp_queue_to_input(tp, m, th, tlen, drop_hdrlen, iptos);
6954 INP_WUNLOCK(tp->t_inpcb);
6957 rack_hpts_do_segment(m, th, so, tp, drop_hdrlen,
6963 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tsused)
6973 if (tp->t_flags & TF_SENTFIN) {
6987 srtt_cur = tp->t_srtt >> TCP_RTT_SHIFT;
7011 rack_output(struct tcpcb *tp)
7063 rack = (struct tcp_rack *)tp->t_fb_ptr;
7072 if (tp->t_flags & TF_TOE)
7073 return (tcp_offload_output(tp));
7081 if ((tp->t_flags & TF_FASTOPEN) &&
7082 (tp->t_state == TCPS_SYN_RECEIVED) &&
7083 SEQ_GT(tp->snd_max, tp->snd_una) && /* inital SYN|ACK sent */
7084 (tp->snd_nxt != tp->snd_una)) /* not a retransmit */
7102 rack_timer_cancel(tp, rack, cts, __LINE__);
7106 (tp->t_flags & TF_FORCEDATA) ||
7107 (tp->t_state < TCPS_ESTABLISHED)) {
7108 if (tp->t_inpcb->inp_in_hpts)
7109 tcp_hpts_remove(tp->t_inpcb, HPTS_REMOVE_OUTPUT);
7110 } else if (tp->t_inpcb->inp_in_hpts) {
7121 if (rack_process_timers(tp, rack, cts, hpts_calling)) {
7134 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
7135 if (tp->t_idle_reduce) {
7136 if (idle && ((ticks - tp->t_rcvtime) >= tp->t_rxtcur))
7137 rack_cc_after_idle(tp,
7140 tp->t_flags &= ~TF_LASTIDLE;
7142 if (tp->t_flags & TF_MORETOCOME) {
7143 tp->t_flags |= TF_LASTIDLE;
7157 sb_offset = tp->snd_max - tp->snd_una;
7158 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
7160 flags = tcp_outflags[tp->t_state];
7203 if (tlen > tp->t_maxseg)
7204 tlen = tp->t_maxseg;
7206 if (SEQ_GT(tp->snd_una, rsm->r_start)) {
7207 panic("tp:%p rack:%p snd_una:%u rsm:%p r_start:%u",
7208 tp, rack, tp->snd_una, rsm, rsm->r_start);
7211 sb_offset = rsm->r_start - tp->snd_una;
7212 cwin = min(tp->snd_wnd, tlen);
7221 sb_offset = rsm->r_start - tp->snd_una;
7222 if (len >= tp->t_maxseg) {
7223 len = tp->t_maxseg;
7228 ((rsm = tcp_rack_output(tp, rack, cts)) != NULL)) {
7231 if ((!IN_RECOVERY(tp->t_flags)) &&
7232 ((tp->t_flags & (TF_WASFRECOVERY | TF_WASCRECOVERY)) == 0)) {
7235 rack->r_ctl.rc_cwnd_at = tp->snd_cwnd;
7236 rack->r_ctl.rc_ssthresh_at = tp->snd_ssthresh;
7237 rack_cong_signal(tp, NULL, CC_NDUPACK);
7242 rack->r_ctl.rc_prr_sndcnt = tp->t_maxseg;
7245 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
7246 panic("Huh, tp:%p rack:%p rsm:%p start:%u < snd_una:%u\n",
7247 tp, rack, rsm, rsm->r_start, tp->snd_una);
7251 sb_offset = rsm->r_start - tp->snd_una;
7257 if (len >= tp->t_maxseg) {
7259 len = tp->t_maxseg;
7279 min(len, tp->t_maxseg));
7316 if (tp->t_flags & TF_NEEDFIN)
7318 if (tp->t_flags & TF_NEEDSYN)
7333 if (tp->t_flags & TF_FORCEDATA) {
7354 rack_exit_persist(tp, rack);
7359 tp->snd_nxt = tp->snd_max;
7360 sb_offset = tp->snd_nxt - tp->snd_una;
7382 if (SEQ_GT(tp->snd_nxt, tp->snd_una))
7383 sb_offset = tp->snd_nxt - tp->snd_una;
7386 if (IN_FASTRECOVERY(tp->t_flags) == 0) {
7392 if (rack->r_ctl.rc_tlp_new_data > tp->snd_wnd)
7393 len = tp->snd_wnd;
7422 outstanding = tp->snd_max - tp->snd_una;
7423 if ((rack->r_ctl.rc_prr_sndcnt + outstanding) > tp->snd_wnd) {
7424 if (tp->snd_wnd > outstanding) {
7425 len = tp->snd_wnd - outstanding;
7447 if (len > tp->t_maxseg) {
7456 len = tp->t_maxseg;
7457 } else if (len < tp->t_maxseg) {
7483 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
7484 if ((tp->t_state != TCPS_SYN_RECEIVED) &&
7485 (tp->t_state != TCPS_SYN_SENT))
7492 if ((tp->t_flags & TF_FASTOPEN) &&
7493 (tp->t_state == TCPS_SYN_RECEIVED))
7502 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
7512 if ((tp->t_flags & TF_FASTOPEN) &&
7513 ((tp->t_state == TCPS_SYN_RECEIVED) && (tp->t_rxtshift > 0)))
7533 if ((tp->snd_wnd == 0) &&
7534 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
7536 tp->snd_nxt = tp->snd_una;
7537 rack_enter_persist(tp, rack, cts);
7542 tcp_sndbuf_autoscale(tp, so, sendwin);
7562 ipoptlen = ip6_optlen(tp->t_inpcb);
7565 if (tp->t_inpcb->inp_options)
7566 ipoptlen = tp->t_inpcb->inp_options->m_len -
7577 ipsec_optlen = IPSEC_HDRSIZE(ipv6, tp->t_inpcb);
7584 ipsec_optlen = IPSEC_HDRSIZE(ipv4, tp->t_inpcb);
7591 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > tp->t_maxseg &&
7593 (tp->t_port == 0) &&
7595 ((tp->t_flags & TF_SIGNATURE) == 0) &&
7596 tp->rcv_numsacks == 0 && sack_rxmit == 0 &&
7602 outstanding = tp->snd_max - tp->snd_una;
7603 if (tp->t_flags & TF_SENTFIN) {
7623 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una +
7641 if (len >= tp->t_maxseg) {
7651 if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
7652 (idle || (tp->t_flags & TF_NODELAY)) &&
7654 (tp->t_flags & TF_NOPUSH) == 0) {
7658 if (tp->t_flags & TF_FORCEDATA) { /* typ. timeout case */
7662 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */
7665 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
7669 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { /* retransmit case */
7706 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
7707 !(tp->t_flags & TF_DELACK) &&
7708 !TCPS_HAVERCVDFIN(tp->t_state)) {
7712 * tp->rcv_scale.
7717 adv = min(recwin, (long)TCP_MAXWIN << tp->rcv_scale);
7718 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
7719 oldwin = (tp->rcv_adv - tp->rcv_nxt);
7728 if (oldwin >> tp->rcv_scale == (adv + oldwin) >> tp->rcv_scale)
7731 if (adv >= (int32_t)(2 * tp->t_maxseg) &&
7734 so->so_rcv.sb_hiwat <= 8 * tp->t_maxseg)) {
7747 if (tp->t_flags & TF_ACKNOW) {
7751 if (((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0)) {
7755 if (SEQ_GT(tp->snd_up, tp->snd_una)) {
7764 if ((tp->t_flags & TF_SENTFIN) ||
7765 (((tp->t_flags & TF_SENTFIN) == 0) &&
7766 (tp->snd_nxt == tp->snd_una))) {
7779 rack_start_hpts_timer(rack, tp, cts, __LINE__, slot, tot_len_this_send, 1);
7781 tp->t_flags &= ~TF_FORCEDATA;
7797 if (len >= tp->t_maxseg)
7798 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
7800 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
7823 if ((tp->t_flags & TF_NOOPT) == 0) {
7826 tp->snd_nxt = tp->iss;
7829 if (tp->t_port)
7841 if ((tp->t_flags & TF_FASTOPEN) &&
7842 (tp->t_state == TCPS_SYN_RECEIVED) &&
7843 (tp->t_rxtshift == 0)) {
7845 to.to_tfo_cookie = (u_char *)&tp->t_tfo_cookie;
7851 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
7852 to.to_wscale = tp->request_r_scale;
7856 if ((tp->t_flags & TF_RCVD_TSTMP) ||
7857 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
7858 to.to_tsval = cts + tp->ts_offset;
7859 to.to_tsecr = tp->ts_recent;
7863 if (tp->rfbuf_ts == 0 &&
7865 tp->rfbuf_ts = tcp_ts_getticks();
7869 else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
7870 tp->rcv_numsacks > 0) {
7872 to.to_nsacks = tp->rcv_numsacks;
7873 to.to_sacks = (u_char *)tp->sackblks;
7877 if (tp->t_flags & TF_SIGNATURE)
7885 if (tp->t_port) {
7896 ipoptlen = ip6_optlen(tp->t_inpcb);
7899 if (tp->t_inpcb->inp_options)
7900 ipoptlen = tp->t_inpcb->inp_options->m_len -
7913 if (len + optlen + ipoptlen > tp->t_maxseg) {
7924 if_hw_tsomax = tp->t_tsomax;
7925 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
7926 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
7949 max_len = (tp->t_maxseg - optlen);
7972 if (tp->t_flags & TF_NEEDFIN)
7976 if (optlen + ipoptlen >= tp->t_maxseg) {
7990 len = tp->t_maxseg - optlen - ipoptlen;
8030 max_val = rack->rc_pace_max_segs * tp->t_maxseg;
8063 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
8069 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
8073 m->m_next = tcp_m_copym(/*tp, */ mb, moff, &len,
8075 if (len <= (tp->t_maxseg - optlen)) {
8092 if ((tp->t_flags & TF_FORCEDATA) && len == 1) {
8095 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
8096 stats_voi_update_abs_u32(tp->t_stats,
8099 stats_voi_update_abs_u64(tp->t_stats,
8102 } else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
8108 /* tp->t_sndtlppack++;*/
8109 /* tp->t_sndtlpbyte += len;*/
8113 tp->t_sndrexmitpack++;
8118 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
8125 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
8144 if (((tp->t_state == TCPS_ESTABLISHED) ||
8145 (tp->t_state == TCPS_CLOSE_WAIT) ||
8146 ((tp->t_state == TCPS_FIN_WAIT_1) &&
8147 ((tp->t_flags & TF_SENTFIN) == 0) &&
8163 srtt = TICKS_2_MSEC((tp->t_srtt >> TCP_RTT_SHIFT));
8167 cwnd = tp->snd_cwnd;
8170 tr_perms = tp->t_maxseg;
8195 (tot_len_this_send >= (rack->r_min_pace_seg_thresh * tp->t_maxseg))) {
8203 if (tp->t_flags & TF_ACKNOW)
8207 else if (SEQ_GT(tp->snd_up, tp->snd_una))
8236 if (tp->t_port) {
8239 udp->uh_dport = tp->t_port;
8246 tcpip_fillheaders(inp, /*tp->t_port, */ ip6, th);
8255 if (tp->t_port) {
8258 udp->uh_dport = tp->t_port;
8265 tcpip_fillheaders(inp,/*tp->t_port, */ ip, th);
8272 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN &&
8273 tp->snd_nxt == tp->snd_max)
8274 tp->snd_nxt--;
8280 if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn == 1) {
8281 if (tp->t_rxtshift >= 1) {
8282 if (tp->t_rxtshift <= V_tcp_ecn_maxretries)
8287 if (tp->t_state == TCPS_ESTABLISHED &&
8288 (tp->t_flags & TF_ECN_PERMIT)) {
8294 if (len > 0 && SEQ_GEQ(tp->snd_nxt, tp->snd_max) &&
8296 !((tp->t_flags & TF_FORCEDATA) && len == 1)) {
8308 if (tp->t_flags & TF_ECN_SND_CWR) {
8310 tp->t_flags &= ~TF_ECN_SND_CWR;
8313 if (tp->t_flags & TF_ECN_SND_ECE)
8330 th->th_seq = htonl(tp->snd_nxt);
8331 rack_seq = tp->snd_nxt;
8339 th->th_seq = htonl(tp->snd_una);
8340 rack_seq = tp->snd_una;
8342 th->th_seq = htonl(tp->snd_max);
8343 rack_seq = tp->snd_max;
8349 th->th_ack = htonl(tp->rcv_nxt);
8360 recwin < (long)tp->t_maxseg)
8362 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
8363 recwin < (long)(tp->rcv_adv - tp->rcv_nxt))
8364 recwin = (long)(tp->rcv_adv - tp->rcv_nxt);
8365 if (recwin > (long)TCP_MAXWIN << tp->rcv_scale)
8366 recwin = (long)TCP_MAXWIN << tp->rcv_scale;
8378 recwin = roundup2(recwin, 1 << tp->rcv_scale);
8379 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
8390 tp->t_sndzerowin++;
8391 tp->t_flags |= TF_RXWIN0SENT;
8393 tp->t_flags &= ~TF_RXWIN0SENT;
8394 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
8395 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
8404 tp->snd_up = tp->snd_una; /* drag it along */
8437 if (tp->t_port) {
8461 if (tp->t_port) {
8490 KASSERT(len > tp->t_maxseg - optlen,
8493 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
8501 hhook_run_tcp_est_out(tp, th, &to, len, tso);
8519 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
8528 if (tp->t_logstate != TCP_LOG_STATE_OFF) {
8540 lgb = tcp_log_event_(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
8571 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
8572 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
8574 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
8576 if (tp->t_state == TCPS_SYN_SENT)
8577 TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
8579 TCP_PROBE5(send, NULL, tp, ip6, tp, th);
8581 error = ip6_output(m, tp->t_inpcb->in6p_outputopts,
8610 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
8611 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
8612 if (tp->t_port == 0 || len < V_tcp_minmss) {
8616 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
8619 if (tp->t_state == TCPS_SYN_SENT)
8620 TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
8622 TCP_PROBE5(send, NULL, tp, ip, tp, th);
8624 error = ip_output(m, tp->t_inpcb->inp_options, &inp->inp_route,
8642 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
8643 (tp->t_flags & TF_SACK_PERMIT) &&
8644 tp->rcv_numsacks > 0)
8645 tcp_clean_dsack_blocks(tp);
8653 idx = (len / tp->t_maxseg) + 3;
8667 rack_log_output(tp, &to, len, rack_seq, (uint8_t) flags, error, cts,
8669 if ((tp->t_flags & TF_FORCEDATA) == 0 ||
8672 tcp_seq startseq = tp->snd_nxt;
8683 tp->snd_nxt++;
8685 tp->snd_nxt++;
8686 tp->t_flags |= TF_SENTFIN;
8693 tp->snd_nxt += len;
8694 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
8695 if (tp->snd_una == tp->snd_max) {
8700 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__);
8701 tp->t_acktime = ticks;
8703 tp->snd_max = tp->snd_nxt;
8705 if (!(tp->t_flags & TF_GPUTINPROG) && len) {
8706 tp->t_flags |= TF_GPUTINPROG;
8707 tp->gput_seq = startseq;
8708 tp->gput_ack = startseq +
8710 tp->gput_ts = tcp_ts_getticks();
8722 if ((tp->snd_wnd == 0) &&
8723 TCPS_HAVEESTABLISHED(tp->t_state)) {
8731 rack_enter_persist(tp, rack, cts);
8748 tp->t_flags |= TF_SENTFIN;
8751 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) {
8752 if (tp->snd_una == tp->snd_max) {
8757 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__);
8758 tp->t_acktime = ticks;
8760 tp->snd_max = tp->snd_nxt + len;
8778 tp->t_flags &= ~TF_FORCEDATA;
8779 tp->t_softerror = error;
8809 tp->t_flags &= ~TF_TSO;
8811 tcp_mss_update(tp, -1, mtu, NULL, NULL);
8815 rack_start_hpts_timer(rack, tp, cts, __LINE__, slot, 0, 1);
8816 tp->t_flags &= ~TF_FORCEDATA;
8823 if (TCPS_HAVERCVDSYN(tp->t_state)) {
8824 tp->t_softerror = error;
8829 rack_start_hpts_timer(rack, tp, cts, __LINE__, slot, 0, 1);
8830 tp->t_flags &= ~TF_FORCEDATA;
8843 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
8844 tp->rcv_adv = tp->rcv_nxt + recwin;
8845 tp->last_ack_sent = tp->rcv_nxt;
8846 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
8865 tp->t_flags &= ~TF_FORCEDATA;
8870 tp->t_flags &= ~TF_FORCEDATA;
8871 rack_start_hpts_timer(rack, tp, cts, __LINE__, slot, tot_len_this_send, 1);
8883 struct inpcb *inp, struct tcpcb *tp, struct tcp_rack *rack)
8912 return (tcp_default_ctloutput(so, sopt, inp, tp));
8924 tp = intotcpcb(inp);
8925 rack = (struct tcp_rack *)tp->t_fb_ptr;
9035 tp->t_delayed_ack = 0;
9037 tp->t_delayed_ack = 1;
9038 if (tp->t_flags & TF_DELACK) {
9039 tp->t_flags &= ~TF_DELACK;
9040 tp->t_flags |= TF_ACKNOW;
9041 rack_output(tp);
9073 return (tcp_default_ctloutput(so, sopt, inp, tp));
9076 /* tcp_log_socket_option(tp, sopt->sopt_name, optval, error);*/
9083 struct inpcb *inp, struct tcpcb *tp, struct tcp_rack *rack)
9169 optval = tp->t_delayed_ack;
9175 return (tcp_default_ctloutput(so, sopt, inp, tp));
9184 rack_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp)
9189 rack = (struct tcp_rack *)tp->t_fb_ptr;
9195 return (rack_set_sockopt(so, sopt, inp, tp, rack));
9197 return (rack_get_sockopt(so, sopt, inp, tp, rack));