1/*
2 * Copyright (c) 2013-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/param.h>
30#include <sys/systm.h>
31#include <sys/kernel.h>
32#include <sys/syslog.h>
33#include <sys/protosw.h>
34#include <sys/socketvar.h>
35#include <sys/kern_control.h>
36#include <sys/domain.h>
37
38#include <netinet/in.h>
39#include <netinet/tcp.h>
40#include <netinet/tcp_var.h>
41#include <netinet/tcp_cc.h>
42#include <mach/sdt.h>
43#include <libkern/OSAtomic.h>
44
45struct tcp_cc_debug_state {
46	u_int64_t ccd_tsns;
47	char ccd_srcaddr[INET6_ADDRSTRLEN];
48	uint16_t ccd_srcport;
49	char ccd_destaddr[INET6_ADDRSTRLEN];
50	uint16_t ccd_destport;
51	uint32_t ccd_snd_cwnd;
52	uint32_t ccd_snd_wnd;
53	uint32_t ccd_snd_ssthresh;
54	uint32_t ccd_rttcur;
55	uint32_t ccd_rxtcur;
56	uint32_t ccd_srtt;
57	uint32_t ccd_event;
58	uint32_t ccd_sndcc;
59	uint32_t ccd_sndhiwat;
60	uint32_t ccd_bytes_acked;
61	union {
62		struct {
63			uint32_t ccd_last_max;
64			uint32_t ccd_tcp_win;
65			uint32_t ccd_target_win;
66			uint32_t ccd_avg_lastmax;
67			uint32_t ccd_mean_deviation;
68		} cubic_state;
69	} u;
70};
71
72int tcp_cc_debug = 0;
73SYSCTL_INT(_net_inet_tcp, OID_AUTO, cc_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
74	&tcp_cc_debug, 0, "Enable debug data collection");
75
76extern struct tcp_cc_algo tcp_cc_newreno;
77SYSCTL_INT(_net_inet_tcp, OID_AUTO, newreno_sockets,
78	CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_cc_newreno.num_sockets,
79	0, "Number of sockets using newreno");
80
81extern struct tcp_cc_algo tcp_cc_ledbat;
82SYSCTL_INT(_net_inet_tcp, OID_AUTO, background_sockets,
83	CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_cc_ledbat.num_sockets,
84	0, "Number of sockets using background transport");
85
86extern struct tcp_cc_algo tcp_cc_cubic;
87SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_sockets,
88	CTLFLAG_RD | CTLFLAG_LOCKED,&tcp_cc_cubic.num_sockets,
89	0, "Number of sockets using cubic");
90
91int tcp_use_newreno = 0;
92SYSCTL_INT(_net_inet_tcp, OID_AUTO, use_newreno,
93	CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_newreno, 0,
94	"Use TCP NewReno by default");
95
96 #define SET_SNDSB_IDEAL_SIZE(sndsb, size) \
97	sndsb->sb_idealsize = min(max(tcp_sendspace, tp->snd_ssthresh), \
98	tcp_autosndbuf_max);
99
100/* Array containing pointers to currently implemented TCP CC algorithms */
101struct tcp_cc_algo* tcp_cc_algo_list[TCP_CC_ALGO_COUNT];
102struct zone *tcp_cc_zone;
103
104/* Information for colelcting TCP debug information using control socket */
105#define TCP_CCDEBUG_CONTROL_NAME "com.apple.network.tcp_ccdebug"
106#define TCP_CCDBG_NOUNIT 0xffffffff
107static kern_ctl_ref tcp_ccdbg_ctlref = NULL;
108volatile UInt32 tcp_ccdbg_unit = TCP_CCDBG_NOUNIT;
109
110void tcp_cc_init(void);
111static void tcp_cc_control_register(void);
112static errno_t tcp_ccdbg_control_connect(kern_ctl_ref kctl,
113	struct sockaddr_ctl *sac, void **uinfo);
114static errno_t tcp_ccdbg_control_disconnect(kern_ctl_ref kctl,
115	u_int32_t unit, void *uinfo);
116static struct tcp_cc_algo tcp_cc_algo_none;
117/*
118 * Initialize TCP congestion control algorithms.
119 */
120
121void
122tcp_cc_init(void)
123{
124	bzero(&tcp_cc_algo_list, sizeof(tcp_cc_algo_list));
125	bzero(&tcp_cc_algo_none, sizeof(tcp_cc_algo_none));
126
127	tcp_cc_algo_list[TCP_CC_ALGO_NONE] = &tcp_cc_algo_none;
128	tcp_cc_algo_list[TCP_CC_ALGO_NEWRENO_INDEX] = &tcp_cc_newreno;
129	tcp_cc_algo_list[TCP_CC_ALGO_BACKGROUND_INDEX] = &tcp_cc_ledbat;
130	tcp_cc_algo_list[TCP_CC_ALGO_CUBIC_INDEX] = &tcp_cc_cubic;
131
132	tcp_cc_control_register();
133}
134
135static void
136tcp_cc_control_register(void)
137{
138	struct kern_ctl_reg ccdbg_control;
139	errno_t err;
140
141	bzero(&ccdbg_control, sizeof(ccdbg_control));
142	strlcpy(ccdbg_control.ctl_name, TCP_CCDEBUG_CONTROL_NAME,
143	    sizeof(ccdbg_control.ctl_name));
144	ccdbg_control.ctl_connect = tcp_ccdbg_control_connect;
145	ccdbg_control.ctl_disconnect = tcp_ccdbg_control_disconnect;
146	ccdbg_control.ctl_flags |= CTL_FLAG_PRIVILEGED;
147	ccdbg_control.ctl_flags |= CTL_FLAG_REG_SOCK_STREAM;
148
149	err = ctl_register(&ccdbg_control, &tcp_ccdbg_ctlref);
150	if (err != 0) {
151		log(LOG_ERR, "failed to register tcp_cc debug control");
152	}
153}
154
155/* Allow only one socket to connect at any time for debugging */
156static errno_t
157tcp_ccdbg_control_connect(kern_ctl_ref kctl, struct sockaddr_ctl *sac,
158	void **uinfo)
159{
160#pragma unused(kctl)
161#pragma unused(uinfo)
162
163	UInt32 old_value = TCP_CCDBG_NOUNIT;
164	UInt32 new_value = sac->sc_unit;
165
166	if (tcp_ccdbg_unit != old_value)
167		return (EALREADY);
168
169	if (OSCompareAndSwap(old_value, new_value, &tcp_ccdbg_unit))
170		return (0);
171	else
172		return (EALREADY);
173}
174
175static errno_t
176tcp_ccdbg_control_disconnect(kern_ctl_ref kctl, u_int32_t unit, void *uinfo)
177{
178#pragma unused(kctl, unit, uinfo)
179
180	if (unit == tcp_ccdbg_unit) {
181		UInt32 old_value = tcp_ccdbg_unit;
182		UInt32 new_value = TCP_CCDBG_NOUNIT;
183		if (tcp_ccdbg_unit == new_value)
184			return (0);
185
186		if (!OSCompareAndSwap(old_value, new_value,
187			&tcp_ccdbg_unit))
188			log(LOG_DEBUG,
189			    "failed to disconnect tcp_cc debug control");
190	}
191	return (0);
192}
193
194inline void
195tcp_ccdbg_trace(struct tcpcb *tp, struct tcphdr *th, int32_t event)
196{
197#if !CONFIG_DTRACE
198#pragma unused(th)
199#endif /* !CONFIG_DTRACE */
200	struct inpcb *inp = tp->t_inpcb;
201
202	if (tcp_cc_debug && tcp_ccdbg_unit > 0) {
203		struct tcp_cc_debug_state dbg_state;
204		struct timespec tv;
205
206		bzero(&dbg_state, sizeof(dbg_state));
207
208		nanotime(&tv);
209		/* Take time in seconds */
210		dbg_state.ccd_tsns = (tv.tv_sec * 1000000000) + tv.tv_nsec;
211		inet_ntop(SOCK_DOM(inp->inp_socket),
212		    ((SOCK_DOM(inp->inp_socket) == PF_INET) ?
213		    (void *)&inp->inp_laddr.s_addr :
214		    (void *)&inp->in6p_laddr), dbg_state.ccd_srcaddr,
215		    sizeof(dbg_state.ccd_srcaddr));
216		dbg_state.ccd_srcport = ntohs(inp->inp_lport);
217		inet_ntop(SOCK_DOM(inp->inp_socket),
218		    ((SOCK_DOM(inp->inp_socket) == PF_INET) ?
219		    (void *)&inp->inp_faddr.s_addr :
220		    (void *)&inp->in6p_faddr), dbg_state.ccd_destaddr,
221		    sizeof(dbg_state.ccd_destaddr));
222		dbg_state.ccd_destport = ntohs(inp->inp_fport);
223
224		dbg_state.ccd_snd_cwnd = tp->snd_cwnd;
225		dbg_state.ccd_snd_wnd = tp->snd_wnd;
226		dbg_state.ccd_snd_ssthresh = tp->snd_ssthresh;
227		dbg_state.ccd_rttcur = tp->t_rttcur;
228		dbg_state.ccd_rxtcur = tp->t_rxtcur;
229		dbg_state.ccd_srtt = tp->t_srtt >> TCP_RTT_SHIFT;
230		dbg_state.ccd_event = event;
231		dbg_state.ccd_sndcc = inp->inp_socket->so_snd.sb_cc;
232		dbg_state.ccd_sndhiwat = inp->inp_socket->so_snd.sb_hiwat;
233		dbg_state.ccd_bytes_acked = tp->t_bytes_acked;
234		switch (tp->tcp_cc_index) {
235		    case TCP_CC_ALGO_CUBIC_INDEX:
236			dbg_state.u.cubic_state.ccd_last_max =
237			    tp->t_ccstate->cub_last_max;
238			dbg_state.u.cubic_state.ccd_tcp_win =
239			    tp->t_ccstate->cub_tcp_win;
240			dbg_state.u.cubic_state.ccd_target_win =
241			    tp->t_ccstate->cub_target_win;
242			dbg_state.u.cubic_state.ccd_avg_lastmax =
243			    tp->t_ccstate->cub_avg_lastmax;
244			dbg_state.u.cubic_state.ccd_mean_deviation =
245			    tp->t_ccstate->cub_mean_dev;
246			break;
247		    default:
248			break;
249		}
250
251		ctl_enqueuedata(tcp_ccdbg_ctlref, tcp_ccdbg_unit,
252			&dbg_state, sizeof(dbg_state), 0);
253	}
254	DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp,
255		struct tcpcb *, tp, struct tcphdr *, th, int32_t, event);
256}
257
258void tcp_cc_resize_sndbuf(struct tcpcb *tp)
259{
260	struct sockbuf *sb;
261	/*
262	 * If the send socket buffer size is bigger than ssthresh,
263	 * it is time to trim it because we do not want to hold
264	 * too many mbufs in the socket buffer
265	 */
266	sb = &tp->t_inpcb->inp_socket->so_snd;
267	if (sb->sb_hiwat > tp->snd_ssthresh &&
268		(sb->sb_flags & SB_AUTOSIZE)) {
269		if (sb->sb_idealsize > tp->snd_ssthresh) {
270			SET_SNDSB_IDEAL_SIZE(sb, tp->snd_ssthresh);
271		}
272		sb->sb_flags |= SB_TRIM;
273	}
274}
275
276void tcp_bad_rexmt_fix_sndbuf(struct tcpcb *tp)
277{
278	struct sockbuf *sb;
279	sb = &tp->t_inpcb->inp_socket->so_snd;
280	if ((sb->sb_flags & (SB_TRIM|SB_AUTOSIZE)) == (SB_TRIM|SB_AUTOSIZE)) {
281		/*
282		 * If there was a retransmission that was not necessary
283		 * then the size of socket buffer can be restored to
284		 * what it was before
285		 */
286		SET_SNDSB_IDEAL_SIZE(sb, tp->snd_ssthresh);
287		if (sb->sb_hiwat <= sb->sb_idealsize) {
288			sbreserve(sb, sb->sb_idealsize);
289			sb->sb_flags &= ~SB_TRIM;
290		}
291	}
292}
293
294/*
295 * Calculate initial cwnd according to RFC3390.
296 *
297 * Keep the old ss_fltsz sysctl for ABI compabitility issues.
298 * but it will be overriden if tcp_do_rfc3390 sysctl when it is set.
299 */
300void
301tcp_cc_cwnd_init_or_reset(struct tcpcb *tp)
302{
303	if (tp->t_flags & TF_LOCAL) {
304		tp->snd_cwnd = tp->t_maxseg * ss_fltsz_local;
305	} else {
306		/* initial congestion window according to RFC 3390 */
307		if (tcp_do_rfc3390)
308			tp->snd_cwnd = min(4 * tp->t_maxseg,
309				max(2 * tp->t_maxseg, TCP_CC_CWND_INIT_BYTES));
310		else
311			tp->snd_cwnd = tp->t_maxseg * ss_fltsz;
312	}
313}
314
315/*
316 * Indicate whether this ack should be delayed.
317 * Here is the explanation for different settings of tcp_delack_enabled:
318 *  - when set to 1, the bhavior is same as when set to 2. We kept this
319 *    for binary compatibility.
320 *  - when set to 2, will "ack every other packet"
321 *      - if our last ack wasn't a 0-sized window.
322 *      - if the peer hasn't sent us a TH_PUSH data packet (radar 3649245).
323 *              If TH_PUSH is set, take this as a clue that we need to ACK
324 *              with no delay. This helps higher level protocols who
325 *              won't send us more data even if the window is open
326 *              because their last "segment" hasn't been ACKed
327 *  - when set to 3,  will do "streaming detection"
328 *      - if we receive more than "maxseg_unacked" full packets
329 *        in the last 100ms
330 *      - if the connection is not in slow-start or idle or
331 *        loss/recovery states
332 *      - if those criteria aren't met, it will ack every other packet.
333 */
334int
335tcp_cc_delay_ack(struct tcpcb *tp, struct tcphdr *th)
336{
337	/* If any flags other than TH_ACK is set, set "end-of-write" bit */
338	if ((th->th_flags & ~TH_ACK))
339		tp->t_flagsext |= TF_STREAMEOW;
340	else
341		tp->t_flagsext &= ~(TF_STREAMEOW);
342
343	switch (tcp_delack_enabled) {
344	    case 1:
345	    case 2:
346		if ((tp->t_flags & TF_RXWIN0SENT) == 0 &&
347		    (th->th_flags & TH_PUSH) == 0 &&
348		    (tp->t_unacksegs == 1))
349			return(1);
350		break;
351	    case 3:
352		if ((tp->t_flags & TF_RXWIN0SENT) == 0 &&
353		    (th->th_flags & TH_PUSH) == 0 &&
354		    ((tp->t_unacksegs == 1) ||
355		    ((tp->t_flags & TF_STRETCHACK) != 0 &&
356			tp->t_unacksegs < (maxseg_unacked))))
357			return(1);
358		break;
359	}
360	return(0);
361}
362
363void
364tcp_cc_allocate_state(struct tcpcb *tp)
365{
366	if (tp->tcp_cc_index == TCP_CC_ALGO_CUBIC_INDEX &&
367		tp->t_ccstate == NULL) {
368		tp->t_ccstate = (struct tcp_ccstate *)zalloc(tcp_cc_zone);
369
370		/*
371		 * If we could not allocate memory for congestion control
372		 * state, revert to using TCP NewReno as it does not
373		 * require any state
374		 */
375		if (tp->t_ccstate == NULL)
376			tp->tcp_cc_index = TCP_CC_ALGO_NEWRENO_INDEX;
377		else
378			bzero(tp->t_ccstate, sizeof(*tp->t_ccstate));
379	}
380}
381
382/*
383 * If stretch ack was disabled automatically on long standing connections,
384 * re-evaluate the situation after 15 minutes to enable it.
385 */
386#define	TCP_STRETCHACK_DISABLE_WIN	(15 * 60 * TCP_RETRANSHZ)
387void
388tcp_cc_after_idle_stretchack(struct tcpcb *tp)
389{
390	int32_t tdiff;
391
392	if (!(tp->t_flagsext & TF_DISABLE_STRETCHACK))
393		return;
394
395	tdiff = timer_diff(tcp_now, 0, tp->rcv_nostrack_ts, 0);
396	if (tdiff < 0)
397		tdiff = -tdiff;
398
399	if (tdiff > TCP_STRETCHACK_DISABLE_WIN) {
400		tp->t_flagsext &= ~TF_DISABLE_STRETCHACK;
401		tp->t_stretchack_delayed = 0;
402
403		tcp_reset_stretch_ack(tp);
404	}
405}
406