1/* 2 * Copyright (c) 2013-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <sys/param.h> 30#include <sys/systm.h> 31#include <sys/kernel.h> 32#include <sys/syslog.h> 33#include <sys/protosw.h> 34#include <sys/socketvar.h> 35#include <sys/kern_control.h> 36#include <sys/domain.h> 37 38#include <netinet/in.h> 39#include <netinet/tcp.h> 40#include <netinet/tcp_var.h> 41#include <netinet/tcp_cc.h> 42#include <mach/sdt.h> 43#include <libkern/OSAtomic.h> 44 45struct tcp_cc_debug_state { 46 u_int64_t ccd_tsns; 47 char ccd_srcaddr[INET6_ADDRSTRLEN]; 48 uint16_t ccd_srcport; 49 char ccd_destaddr[INET6_ADDRSTRLEN]; 50 uint16_t ccd_destport; 51 uint32_t ccd_snd_cwnd; 52 uint32_t ccd_snd_wnd; 53 uint32_t ccd_snd_ssthresh; 54 uint32_t ccd_rttcur; 55 uint32_t ccd_rxtcur; 56 uint32_t ccd_srtt; 57 uint32_t ccd_event; 58 uint32_t ccd_sndcc; 59 uint32_t ccd_sndhiwat; 60 uint32_t ccd_bytes_acked; 61 union { 62 struct { 63 uint32_t ccd_last_max; 64 uint32_t ccd_tcp_win; 65 uint32_t ccd_target_win; 66 uint32_t ccd_avg_lastmax; 67 uint32_t ccd_mean_deviation; 68 } cubic_state; 69 } u; 70}; 71 72int tcp_cc_debug = 0; 73SYSCTL_INT(_net_inet_tcp, OID_AUTO, cc_debug, CTLFLAG_RW | CTLFLAG_LOCKED, 74 &tcp_cc_debug, 0, "Enable debug data collection"); 75 76extern struct tcp_cc_algo tcp_cc_newreno; 77SYSCTL_INT(_net_inet_tcp, OID_AUTO, newreno_sockets, 78 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_cc_newreno.num_sockets, 79 0, "Number of sockets using newreno"); 80 81extern struct tcp_cc_algo tcp_cc_ledbat; 82SYSCTL_INT(_net_inet_tcp, OID_AUTO, background_sockets, 83 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_cc_ledbat.num_sockets, 84 0, "Number of sockets using background transport"); 85 86extern struct tcp_cc_algo tcp_cc_cubic; 87SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_sockets, 88 CTLFLAG_RD | CTLFLAG_LOCKED,&tcp_cc_cubic.num_sockets, 89 0, "Number of sockets using cubic"); 90 91int tcp_use_newreno = 0; 92SYSCTL_INT(_net_inet_tcp, OID_AUTO, use_newreno, 93 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_newreno, 0, 94 "Use TCP NewReno by default"); 95 96 #define SET_SNDSB_IDEAL_SIZE(sndsb, size) \ 97 sndsb->sb_idealsize = min(max(tcp_sendspace, tp->snd_ssthresh), \ 98 tcp_autosndbuf_max); 99 100/* Array containing pointers to currently implemented TCP CC algorithms */ 101struct tcp_cc_algo* tcp_cc_algo_list[TCP_CC_ALGO_COUNT]; 102struct zone *tcp_cc_zone; 103 104/* Information for colelcting TCP debug information using control socket */ 105#define TCP_CCDEBUG_CONTROL_NAME "com.apple.network.tcp_ccdebug" 106#define TCP_CCDBG_NOUNIT 0xffffffff 107static kern_ctl_ref tcp_ccdbg_ctlref = NULL; 108volatile UInt32 tcp_ccdbg_unit = TCP_CCDBG_NOUNIT; 109 110void tcp_cc_init(void); 111static void tcp_cc_control_register(void); 112static errno_t tcp_ccdbg_control_connect(kern_ctl_ref kctl, 113 struct sockaddr_ctl *sac, void **uinfo); 114static errno_t tcp_ccdbg_control_disconnect(kern_ctl_ref kctl, 115 u_int32_t unit, void *uinfo); 116static struct tcp_cc_algo tcp_cc_algo_none; 117/* 118 * Initialize TCP congestion control algorithms. 119 */ 120 121void 122tcp_cc_init(void) 123{ 124 bzero(&tcp_cc_algo_list, sizeof(tcp_cc_algo_list)); 125 bzero(&tcp_cc_algo_none, sizeof(tcp_cc_algo_none)); 126 127 tcp_cc_algo_list[TCP_CC_ALGO_NONE] = &tcp_cc_algo_none; 128 tcp_cc_algo_list[TCP_CC_ALGO_NEWRENO_INDEX] = &tcp_cc_newreno; 129 tcp_cc_algo_list[TCP_CC_ALGO_BACKGROUND_INDEX] = &tcp_cc_ledbat; 130 tcp_cc_algo_list[TCP_CC_ALGO_CUBIC_INDEX] = &tcp_cc_cubic; 131 132 tcp_cc_control_register(); 133} 134 135static void 136tcp_cc_control_register(void) 137{ 138 struct kern_ctl_reg ccdbg_control; 139 errno_t err; 140 141 bzero(&ccdbg_control, sizeof(ccdbg_control)); 142 strlcpy(ccdbg_control.ctl_name, TCP_CCDEBUG_CONTROL_NAME, 143 sizeof(ccdbg_control.ctl_name)); 144 ccdbg_control.ctl_connect = tcp_ccdbg_control_connect; 145 ccdbg_control.ctl_disconnect = tcp_ccdbg_control_disconnect; 146 ccdbg_control.ctl_flags |= CTL_FLAG_PRIVILEGED; 147 ccdbg_control.ctl_flags |= CTL_FLAG_REG_SOCK_STREAM; 148 149 err = ctl_register(&ccdbg_control, &tcp_ccdbg_ctlref); 150 if (err != 0) { 151 log(LOG_ERR, "failed to register tcp_cc debug control"); 152 } 153} 154 155/* Allow only one socket to connect at any time for debugging */ 156static errno_t 157tcp_ccdbg_control_connect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, 158 void **uinfo) 159{ 160#pragma unused(kctl) 161#pragma unused(uinfo) 162 163 UInt32 old_value = TCP_CCDBG_NOUNIT; 164 UInt32 new_value = sac->sc_unit; 165 166 if (tcp_ccdbg_unit != old_value) 167 return (EALREADY); 168 169 if (OSCompareAndSwap(old_value, new_value, &tcp_ccdbg_unit)) 170 return (0); 171 else 172 return (EALREADY); 173} 174 175static errno_t 176tcp_ccdbg_control_disconnect(kern_ctl_ref kctl, u_int32_t unit, void *uinfo) 177{ 178#pragma unused(kctl, unit, uinfo) 179 180 if (unit == tcp_ccdbg_unit) { 181 UInt32 old_value = tcp_ccdbg_unit; 182 UInt32 new_value = TCP_CCDBG_NOUNIT; 183 if (tcp_ccdbg_unit == new_value) 184 return (0); 185 186 if (!OSCompareAndSwap(old_value, new_value, 187 &tcp_ccdbg_unit)) 188 log(LOG_DEBUG, 189 "failed to disconnect tcp_cc debug control"); 190 } 191 return (0); 192} 193 194inline void 195tcp_ccdbg_trace(struct tcpcb *tp, struct tcphdr *th, int32_t event) 196{ 197#if !CONFIG_DTRACE 198#pragma unused(th) 199#endif /* !CONFIG_DTRACE */ 200 struct inpcb *inp = tp->t_inpcb; 201 202 if (tcp_cc_debug && tcp_ccdbg_unit > 0) { 203 struct tcp_cc_debug_state dbg_state; 204 struct timespec tv; 205 206 bzero(&dbg_state, sizeof(dbg_state)); 207 208 nanotime(&tv); 209 /* Take time in seconds */ 210 dbg_state.ccd_tsns = (tv.tv_sec * 1000000000) + tv.tv_nsec; 211 inet_ntop(SOCK_DOM(inp->inp_socket), 212 ((SOCK_DOM(inp->inp_socket) == PF_INET) ? 213 (void *)&inp->inp_laddr.s_addr : 214 (void *)&inp->in6p_laddr), dbg_state.ccd_srcaddr, 215 sizeof(dbg_state.ccd_srcaddr)); 216 dbg_state.ccd_srcport = ntohs(inp->inp_lport); 217 inet_ntop(SOCK_DOM(inp->inp_socket), 218 ((SOCK_DOM(inp->inp_socket) == PF_INET) ? 219 (void *)&inp->inp_faddr.s_addr : 220 (void *)&inp->in6p_faddr), dbg_state.ccd_destaddr, 221 sizeof(dbg_state.ccd_destaddr)); 222 dbg_state.ccd_destport = ntohs(inp->inp_fport); 223 224 dbg_state.ccd_snd_cwnd = tp->snd_cwnd; 225 dbg_state.ccd_snd_wnd = tp->snd_wnd; 226 dbg_state.ccd_snd_ssthresh = tp->snd_ssthresh; 227 dbg_state.ccd_rttcur = tp->t_rttcur; 228 dbg_state.ccd_rxtcur = tp->t_rxtcur; 229 dbg_state.ccd_srtt = tp->t_srtt >> TCP_RTT_SHIFT; 230 dbg_state.ccd_event = event; 231 dbg_state.ccd_sndcc = inp->inp_socket->so_snd.sb_cc; 232 dbg_state.ccd_sndhiwat = inp->inp_socket->so_snd.sb_hiwat; 233 dbg_state.ccd_bytes_acked = tp->t_bytes_acked; 234 switch (tp->tcp_cc_index) { 235 case TCP_CC_ALGO_CUBIC_INDEX: 236 dbg_state.u.cubic_state.ccd_last_max = 237 tp->t_ccstate->cub_last_max; 238 dbg_state.u.cubic_state.ccd_tcp_win = 239 tp->t_ccstate->cub_tcp_win; 240 dbg_state.u.cubic_state.ccd_target_win = 241 tp->t_ccstate->cub_target_win; 242 dbg_state.u.cubic_state.ccd_avg_lastmax = 243 tp->t_ccstate->cub_avg_lastmax; 244 dbg_state.u.cubic_state.ccd_mean_deviation = 245 tp->t_ccstate->cub_mean_dev; 246 break; 247 default: 248 break; 249 } 250 251 ctl_enqueuedata(tcp_ccdbg_ctlref, tcp_ccdbg_unit, 252 &dbg_state, sizeof(dbg_state), 0); 253 } 254 DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp, 255 struct tcpcb *, tp, struct tcphdr *, th, int32_t, event); 256} 257 258void tcp_cc_resize_sndbuf(struct tcpcb *tp) 259{ 260 struct sockbuf *sb; 261 /* 262 * If the send socket buffer size is bigger than ssthresh, 263 * it is time to trim it because we do not want to hold 264 * too many mbufs in the socket buffer 265 */ 266 sb = &tp->t_inpcb->inp_socket->so_snd; 267 if (sb->sb_hiwat > tp->snd_ssthresh && 268 (sb->sb_flags & SB_AUTOSIZE)) { 269 if (sb->sb_idealsize > tp->snd_ssthresh) { 270 SET_SNDSB_IDEAL_SIZE(sb, tp->snd_ssthresh); 271 } 272 sb->sb_flags |= SB_TRIM; 273 } 274} 275 276void tcp_bad_rexmt_fix_sndbuf(struct tcpcb *tp) 277{ 278 struct sockbuf *sb; 279 sb = &tp->t_inpcb->inp_socket->so_snd; 280 if ((sb->sb_flags & (SB_TRIM|SB_AUTOSIZE)) == (SB_TRIM|SB_AUTOSIZE)) { 281 /* 282 * If there was a retransmission that was not necessary 283 * then the size of socket buffer can be restored to 284 * what it was before 285 */ 286 SET_SNDSB_IDEAL_SIZE(sb, tp->snd_ssthresh); 287 if (sb->sb_hiwat <= sb->sb_idealsize) { 288 sbreserve(sb, sb->sb_idealsize); 289 sb->sb_flags &= ~SB_TRIM; 290 } 291 } 292} 293 294/* 295 * Calculate initial cwnd according to RFC3390. 296 * 297 * Keep the old ss_fltsz sysctl for ABI compabitility issues. 298 * but it will be overriden if tcp_do_rfc3390 sysctl when it is set. 299 */ 300void 301tcp_cc_cwnd_init_or_reset(struct tcpcb *tp) 302{ 303 if (tp->t_flags & TF_LOCAL) { 304 tp->snd_cwnd = tp->t_maxseg * ss_fltsz_local; 305 } else { 306 /* initial congestion window according to RFC 3390 */ 307 if (tcp_do_rfc3390) 308 tp->snd_cwnd = min(4 * tp->t_maxseg, 309 max(2 * tp->t_maxseg, TCP_CC_CWND_INIT_BYTES)); 310 else 311 tp->snd_cwnd = tp->t_maxseg * ss_fltsz; 312 } 313} 314 315/* 316 * Indicate whether this ack should be delayed. 317 * Here is the explanation for different settings of tcp_delack_enabled: 318 * - when set to 1, the bhavior is same as when set to 2. We kept this 319 * for binary compatibility. 320 * - when set to 2, will "ack every other packet" 321 * - if our last ack wasn't a 0-sized window. 322 * - if the peer hasn't sent us a TH_PUSH data packet (radar 3649245). 323 * If TH_PUSH is set, take this as a clue that we need to ACK 324 * with no delay. This helps higher level protocols who 325 * won't send us more data even if the window is open 326 * because their last "segment" hasn't been ACKed 327 * - when set to 3, will do "streaming detection" 328 * - if we receive more than "maxseg_unacked" full packets 329 * in the last 100ms 330 * - if the connection is not in slow-start or idle or 331 * loss/recovery states 332 * - if those criteria aren't met, it will ack every other packet. 333 */ 334int 335tcp_cc_delay_ack(struct tcpcb *tp, struct tcphdr *th) 336{ 337 /* If any flags other than TH_ACK is set, set "end-of-write" bit */ 338 if ((th->th_flags & ~TH_ACK)) 339 tp->t_flagsext |= TF_STREAMEOW; 340 else 341 tp->t_flagsext &= ~(TF_STREAMEOW); 342 343 switch (tcp_delack_enabled) { 344 case 1: 345 case 2: 346 if ((tp->t_flags & TF_RXWIN0SENT) == 0 && 347 (th->th_flags & TH_PUSH) == 0 && 348 (tp->t_unacksegs == 1)) 349 return(1); 350 break; 351 case 3: 352 if ((tp->t_flags & TF_RXWIN0SENT) == 0 && 353 (th->th_flags & TH_PUSH) == 0 && 354 ((tp->t_unacksegs == 1) || 355 ((tp->t_flags & TF_STRETCHACK) != 0 && 356 tp->t_unacksegs < (maxseg_unacked)))) 357 return(1); 358 break; 359 } 360 return(0); 361} 362 363void 364tcp_cc_allocate_state(struct tcpcb *tp) 365{ 366 if (tp->tcp_cc_index == TCP_CC_ALGO_CUBIC_INDEX && 367 tp->t_ccstate == NULL) { 368 tp->t_ccstate = (struct tcp_ccstate *)zalloc(tcp_cc_zone); 369 370 /* 371 * If we could not allocate memory for congestion control 372 * state, revert to using TCP NewReno as it does not 373 * require any state 374 */ 375 if (tp->t_ccstate == NULL) 376 tp->tcp_cc_index = TCP_CC_ALGO_NEWRENO_INDEX; 377 else 378 bzero(tp->t_ccstate, sizeof(*tp->t_ccstate)); 379 } 380} 381 382/* 383 * If stretch ack was disabled automatically on long standing connections, 384 * re-evaluate the situation after 15 minutes to enable it. 385 */ 386#define TCP_STRETCHACK_DISABLE_WIN (15 * 60 * TCP_RETRANSHZ) 387void 388tcp_cc_after_idle_stretchack(struct tcpcb *tp) 389{ 390 int32_t tdiff; 391 392 if (!(tp->t_flagsext & TF_DISABLE_STRETCHACK)) 393 return; 394 395 tdiff = timer_diff(tcp_now, 0, tp->rcv_nostrack_ts, 0); 396 if (tdiff < 0) 397 tdiff = -tdiff; 398 399 if (tdiff > TCP_STRETCHACK_DISABLE_WIN) { 400 tp->t_flagsext &= ~TF_DISABLE_STRETCHACK; 401 tp->t_stretchack_delayed = 0; 402 403 tcp_reset_stretch_ack(tp); 404 } 405} 406