netinet/tcp_stacks/fastpath.c

22  * contract to Juniper Networks, Inc.
36  *    may be used to endorse or promote products derived from this software
146  *	- Our last ack wasn't a 0-sized window. We never want to delay
159  * It basically allows us to also stay in the fastpath
162  * due to the fact that along with moving forward
167 	       struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen,
207 	 * NOTE that the test is modified according to the latest
210 	if ((to->to_flags & TOF_TS) != 0 &&
213 		tp->ts_recent = to->to_tsval;
242 	if ((to->to_flags & TOF_TS) != 0 &&
243 	    to->to_tsecr) {
246 		t = tcp_ts_getticks() - to->to_tsecr;
263 		hhook_run_tcp_est_in(tp, th, to);
281 		 * Pull snd_wl2 up to prevent seq wrap relative
282 		 * to th_ack.
293 		 * are ready to send, let tcp_output
338 		   struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen,
354 	 * NOTE that the test is modified according to the latest
357 	if ((to->to_flags & TOF_TS) != 0 &&
360 		tp->ts_recent = to->to_tsval;
366 	 * buffer space to take it.
379 	 * Pull snd_wl1 up to prevent seq wrap relative to
384 	 * Pull rcv_up up to prevent seq wrap relative to
398 	/* Add data to socket buffer. */
442 		struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen,
495 	 *	if SYN has been acked change to ESTABLISHED else SYN_RCVD state
496 	 *	arrange for segment to be acked (eventually)
579 		 * Advance th->th_seq to correspond to first data byte.
580 		 * If data, trim to stay within window,
596 		 * If the remote host used T/TCP to validate the SYN,
610 	 * NB: Leftover from RFC1644 T/TCP.  Cases to be reused later.
623 	 * to fall though and be processed (which gives forged reset
639 		 * Note: to take into account delayed ACKs, we should
716 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
717 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {
719 		/* Check to see if ts_recent is over 24 days old.  */
725 			 * ts_recent to zero will at least satisfy the
729 			 * because we don't want out-of-order segments to be
744 	 * In the SYN-RECEIVED state, validate that the packet belongs to
745 	 * this connection before trimming the data to fit the receive
772 			 * Any valid FIN must be to the left of the window.
779 			 * Send an ACK to resynchronize and drop any data.
835 			 * window edge, and have to drop data and PUSH from
837 			 * remember to ack.  Otherwise, drop segment
861 	 * 3) That we modify the segment boundary check to be
865 	 *    This modified check allows us to overcome RFC1323's
870 	if ((to->to_flags & TOF_TS) != 0 &&
875 		tp->ts_recent = to->to_tsval;
931 		 * later; if not, do so now to pass queued data to user.
943 	 * then advance tp->snd_una to th->th_ack and drop
945 	 * more up to date window information we update our window information.
958 		    ((to->to_flags & TOF_SACK) ||
960 			tcp_sack_doack(tp, to, th->th_ack);
969 		hhook_run_tcp_est_in(tp, th, to);
976 				 * duplicate and it needs to be processed
1001 				 * (set ssthresh to half the current window
1002 				 * and pull our congestion window back to
1008 				 * to keep a constant cwnd packets in the
1049 					 * If we're doing sack, check to
1052 					 * check to see if we're in newreno
1121 					 * is new data available to be sent.
1153 		 * If the congestion window was inflated to account
1174 			 * synchronized).  Go to non-starred state,
1198 		 * to do the retransmit in the first place.  Recover our
1199 		 * original cwnd and ssthresh, and proceed to transmit where
1220 		if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) {
1223 			t = tcp_ts_getticks() - to->to_tsecr;
1235 		 * timer and remember to restart (more output or persist).
1236 		 * If there is more data to be acked, restart retransmit
1294 		 * In FIN_WAIT_1 STATE in addition to the processing
1303 				 * Starting the timer is contrary to the
1323 		 * In CLOSING STATE in addition to the processing for
1339 		 * In LAST_ACK, we may still be waiting for data to drain
1340 		 * and/or to be acked, as well as for the ack of our FIN.
1385 		 * soreceive.  It's hard to imagine someone
1386 		 * actually wanting to send this much urgent data.
1402 		 * According to RFC961 (Assigned Protocols),
1403 		 * the urgent pointer points to the last octet
1405 		 * to consider it to indicate the first octet
1420 		 * Remove out of band data so doesn't get presented to user.
1446 	 * is presented to the user (this happens in tcp_usrreq.c,
1456 		 * with control block tp.  Set thflags to whether reassembly now
1458 		 * inline (segment is the next to be received on an established
1486 			 * XXX: Due to the header drop above "th" is
1499 		 * our window, in order to estimate the sender's
1606 	 * paths to this code happen after packets containing
1694 	struct tcpopt to;
1701 	 * allow the tcbinfo to be in either alocked or unlocked, as the
1702 	 * caller may have unnecessarily acquired a write lock due to a race.
1761 	 * validation to ignore broken/spoofed segs.
1800 	tcp_dooptions(&to, (u_char *)(th + 1),
1806 	 * fall back to non RFC1323 RTT calculation.  Normalize
1810 	if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
1811 		to.to_tsecr -= tp->ts_offset;
1812 		if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks()))
1813 			to.to_tsecr = 0;
1819 	 * According to RFC1323 the window field in a SYN (i.e., a <SYN>
1821 	 * XXX this is traditional behavior, may need to be cleaned up.
1824 		if ((to.to_flags & TOF_SCALE) &&
1827 			tp->snd_scale = to.to_wscale;
1831 		 * the next incoming segment to the scaled value.
1834 		if (to.to_flags & TOF_TS) {
1836 			tp->ts_recent = to.to_tsval;
1839 		if (to.to_flags & TOF_MSS)
1840 			tcp_mss(tp, to.to_mss);
1842 		    (to.to_flags & TOF_SACKPERM) == 0)
1850 	if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) {
1857 	if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) {
1896 	 * (the reassembly queue is empty), add the data to
1909 	    ((to.to_flags & TOF_TS) == 0 ||
1910 	     TSTMP_GEQ(to.to_tsval, tp->ts_recent)))) {
1914 		     (to.to_flags & TOF_SACK) == 0 &&
1917 			tcp_do_fastack(m, th, so, tp, &to, drop_hdrlen, tlen,
1923 			tcp_do_fastnewdata(m, th, so, tp, &to, drop_hdrlen, tlen,
1929 	tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen,
1935  * This subfunction is used to try to highly optimize the
1937  * in sequence to remain in the fast-path. We also add
1938  * in the __predict's to attempt to help the compiler.
1945 	       struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen,
1962 		/* Old ack, behind (or duplicate to) the last one rcv'd */
1986 	if((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
1994 	if (__predict_false(to->to_flags & TOF_SACK)) {
2017 	 * Pull snd_wl2 up to prevent seq wrap relative
2018 	 * to th_ack.
2024 	 * NOTE that the test is modified according to the latest
2027 	if ((to->to_flags & TOF_TS) != 0 &&
2030 		tp->ts_recent = to->to_tsval;
2059 	if ((to->to_flags & TOF_TS) != 0 &&
2060 	    to->to_tsecr) {
2063 		t = tcp_ts_getticks() - to->to_tsecr;
2080 		hhook_run_tcp_est_in(tp, th, to);
2105 		 * are ready to send, let tcp_output
2121 		/* Wake up the socket if we have room to write more */
2154  * output to the peer as quickly as possible. This routine
2155  * seems to be about an overall 3% faster then the old
2157  * much more (by allowing window updates to also stay in the fastpath).
2168 	struct tcpopt to;
2175 	 * allow the tcbinfo to be in either alocked or unlocked, as the
2176 	 * caller may have unnecessarily acquired a write lock due to a race.
2235 	 * validation to ignore broken/spoofed segs.
2274 	tcp_dooptions(&to, (u_char *)(th + 1),
2280 	 * fall back to non RFC1323 RTT calculation.  Normalize
2284 	if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
2285 		to.to_tsecr -= tp->ts_offset;
2286 		if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks()))
2287 			to.to_tsecr = 0;
2293 	 * According to RFC1323 the window field in a SYN (i.e., a <SYN>
2295 	 * XXX this is traditional behavior, may need to be cleaned up.
2298 		if ((to.to_flags & TOF_SCALE) &&
2301 			tp->snd_scale = to.to_wscale;
2305 		 * the next incoming segment to the scaled value.
2308 		if (to.to_flags & TOF_TS) {
2310 			tp->ts_recent = to.to_tsval;
2313 		if (to.to_flags & TOF_MSS)
2314 			tcp_mss(tp, to.to_mss);
2316 		    (to.to_flags & TOF_SACKPERM) == 0)
2324 	if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) {
2331 	if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) {
2350 	 * (the reassembly queue is empty), add the data to
2357 	    __predict_true(((to.to_flags & TOF_SACK) == 0)) &&
2362 		    if (tcp_fastack(m, th, so, tp, &to, drop_hdrlen, tlen,
2367 	tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen,
2394 			printf("Failed to register fastack module -- err:%d\n", err);
2399 			printf("Failed to register fastslow module -- err:%d\n", err);