1/* 2 * Copyright (c) 2012-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <string.h> 30#include <sys/types.h> 31#include <sys/syslog.h> 32#include <sys/queue.h> 33#include <sys/malloc.h> 34#include <sys/socket.h> 35#include <sys/kpi_mbuf.h> 36#include <sys/mbuf.h> 37#include <sys/domain.h> 38#include <sys/protosw.h> 39#include <sys/socketvar.h> 40#include <sys/kernel.h> 41#include <sys/systm.h> 42#include <sys/kern_control.h> 43#include <sys/ubc.h> 44#include <sys/codesign.h> 45#include <libkern/tree.h> 46#include <kern/locks.h> 47#include <kern/debug.h> 48#include <net/if_var.h> 49#include <net/route.h> 50#include <net/flowhash.h> 51#include <net/ntstat.h> 52#include <netinet/in.h> 53#include <netinet/in_var.h> 54#include <netinet/tcp.h> 55#include <netinet/tcp_var.h> 56#include <netinet/tcp_fsm.h> 57#include <netinet/flow_divert.h> 58#include <netinet/flow_divert_proto.h> 59#if INET6 60#include <netinet6/ip6protosw.h> 61#endif /* INET6 */ 62#include <dev/random/randomdev.h> 63#include <libkern/crypto/sha1.h> 64#include <libkern/crypto/crypto_internal.h> 65 66#define FLOW_DIVERT_CONNECT_STARTED 0x00000001 67#define FLOW_DIVERT_READ_CLOSED 0x00000002 68#define FLOW_DIVERT_WRITE_CLOSED 0x00000004 69#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008 70#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010 71#define FLOW_DIVERT_TRANSFERRED 0x00000020 72 73#define FDLOG(level, pcb, format, ...) do { \ 74 if (level <= (pcb)->log_level) { \ 75 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): " format "\n", __FUNCTION__, (pcb)->hash, __VA_ARGS__); \ 76 } \ 77} while (0) 78 79#define FDLOG0(level, pcb, msg) do { \ 80 if (level <= (pcb)->log_level) { \ 81 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): %s\n", __FUNCTION__, (pcb)->hash, msg); \ 82 } \ 83} while (0) 84 85#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count) 86#define FDRELEASE(pcb) \ 87 do { \ 88 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \ 89 flow_divert_pcb_destroy(pcb); \ 90 } \ 91 } while (0) 92 93#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx) 94#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx) 95 96#define FD_CTL_SENDBUFF_SIZE (2 * FLOW_DIVERT_CHUNK_SIZE) 97#define FD_CTL_RCVBUFF_SIZE (128 * 1024) 98 99#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0 100 101#define GROUP_COUNT_MAX 32 102#define FLOW_DIVERT_MAX_NAME_SIZE 4096 103#define FLOW_DIVERT_MAX_KEY_SIZE 1024 104 105#define DNS_SERVICE_GROUP_UNIT (GROUP_COUNT_MAX + 1) 106 107struct flow_divert_trie_node 108{ 109 uint16_t start; 110 uint16_t length; 111 uint16_t child_map; 112 uint32_t group_unit; 113}; 114 115struct flow_divert_trie 116{ 117 struct flow_divert_trie_node *nodes; 118 uint16_t *child_maps; 119 uint8_t *bytes; 120 void *memory; 121 size_t nodes_count; 122 size_t child_maps_count; 123 size_t bytes_count; 124 size_t nodes_free_next; 125 size_t child_maps_free_next; 126 size_t bytes_free_next; 127 uint16_t root; 128}; 129 130#define CHILD_MAP_SIZE 256 131#define NULL_TRIE_IDX 0xffff 132#define TRIE_NODE(t, i) ((t)->nodes[(i)]) 133#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)]) 134#define TRIE_BYTE(t, i) ((t)->bytes[(i)]) 135 136static struct flow_divert_pcb nil_pcb; 137 138decl_lck_rw_data(static, g_flow_divert_group_lck); 139static struct flow_divert_group **g_flow_divert_groups = NULL; 140static uint32_t g_active_group_count = 0; 141static struct flow_divert_trie g_signing_id_trie; 142 143static lck_grp_attr_t *flow_divert_grp_attr = NULL; 144static lck_attr_t *flow_divert_mtx_attr = NULL; 145static lck_grp_t *flow_divert_mtx_grp = NULL; 146static errno_t g_init_result = 0; 147 148static kern_ctl_ref g_flow_divert_kctl_ref = NULL; 149 150static struct protosw g_flow_divert_in_protosw; 151static struct pr_usrreqs g_flow_divert_in_usrreqs; 152#if INET6 153static struct ip6protosw g_flow_divert_in6_protosw; 154static struct pr_usrreqs g_flow_divert_in6_usrreqs; 155#endif /* INET6 */ 156 157static struct protosw *g_tcp_protosw = NULL; 158static struct ip6protosw *g_tcp6_protosw = NULL; 159 160static inline int 161flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b) 162{ 163 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash)); 164} 165 166RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp); 167RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp); 168 169static const char * 170flow_divert_packet_type2str(uint8_t packet_type) 171{ 172 switch (packet_type) { 173 case FLOW_DIVERT_PKT_CONNECT: 174 return "connect"; 175 case FLOW_DIVERT_PKT_CONNECT_RESULT: 176 return "connect result"; 177 case FLOW_DIVERT_PKT_DATA: 178 return "data"; 179 case FLOW_DIVERT_PKT_CLOSE: 180 return "close"; 181 case FLOW_DIVERT_PKT_READ_NOTIFY: 182 return "read notification"; 183 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE: 184 return "properties update"; 185 case FLOW_DIVERT_PKT_APP_MAP_UPDATE: 186 return "app map update"; 187 case FLOW_DIVERT_PKT_APP_MAP_CREATE: 188 return "app map create"; 189 default: 190 return "unknown"; 191 } 192} 193 194static struct flow_divert_pcb * 195flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group) 196{ 197 struct flow_divert_pcb key_item; 198 struct flow_divert_pcb *fd_cb = NULL; 199 200 key_item.hash = hash; 201 202 lck_rw_lock_shared(&group->lck); 203 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item); 204 FDRETAIN(fd_cb); 205 lck_rw_done(&group->lck); 206 207 return fd_cb; 208} 209 210static errno_t 211flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit) 212{ 213 int error = 0; 214 struct flow_divert_pcb *exist = NULL; 215 struct flow_divert_group *group; 216 static uint32_t g_nextkey = 1; 217 static uint32_t g_hash_seed = 0; 218 errno_t result = 0; 219 int try_count = 0; 220 221 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) { 222 return EINVAL; 223 } 224 225 socket_unlock(fd_cb->so, 0); 226 lck_rw_lock_shared(&g_flow_divert_group_lck); 227 228 if (g_flow_divert_groups == NULL || g_active_group_count == 0) { 229 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket"); 230 error = ENETUNREACH; 231 goto done; 232 } 233 234 group = g_flow_divert_groups[ctl_unit]; 235 if (group == NULL) { 236 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit); 237 error = ENETUNREACH; 238 goto done; 239 } 240 241 socket_lock(fd_cb->so, 0); 242 243 do { 244 uint32_t key[2]; 245 uint32_t idx; 246 247 key[0] = g_nextkey++; 248 key[1] = RandomULong(); 249 250 if (g_hash_seed == 0) { 251 g_hash_seed = RandomULong(); 252 } 253 254 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed); 255 256 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) { 257 struct flow_divert_group *curr_group = g_flow_divert_groups[idx]; 258 if (curr_group != NULL && curr_group != group) { 259 lck_rw_lock_shared(&curr_group->lck); 260 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb); 261 lck_rw_done(&curr_group->lck); 262 if (exist != NULL) { 263 break; 264 } 265 } 266 } 267 268 if (exist == NULL) { 269 lck_rw_lock_exclusive(&group->lck); 270 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb); 271 lck_rw_done(&group->lck); 272 } 273 } while (exist != NULL && try_count++ < 3); 274 275 if (exist == NULL) { 276 fd_cb->group = group; 277 FDRETAIN(fd_cb); /* The group now has a reference */ 278 } else { 279 fd_cb->hash = 0; 280 result = EEXIST; 281 } 282 283 socket_unlock(fd_cb->so, 0); 284 285done: 286 lck_rw_done(&g_flow_divert_group_lck); 287 socket_lock(fd_cb->so, 0); 288 289 return result; 290} 291 292static struct flow_divert_pcb * 293flow_divert_pcb_create(socket_t so) 294{ 295 struct flow_divert_pcb *new_pcb = NULL; 296 297 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK); 298 if (new_pcb == NULL) { 299 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb"); 300 return NULL; 301 } 302 303 memset(new_pcb, 0, sizeof(*new_pcb)); 304 305 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr); 306 new_pcb->so = so; 307 new_pcb->log_level = nil_pcb.log_level; 308 309 FDRETAIN(new_pcb); /* Represents the socket's reference */ 310 311 return new_pcb; 312} 313 314static void 315flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb) 316{ 317 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u", 318 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received); 319 320 if (fd_cb->local_address != NULL) { 321 FREE(fd_cb->local_address, M_SONAME); 322 } 323 if (fd_cb->remote_address != NULL) { 324 FREE(fd_cb->remote_address, M_SONAME); 325 } 326 if (fd_cb->connect_token != NULL) { 327 mbuf_freem(fd_cb->connect_token); 328 } 329 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB); 330} 331 332static void 333flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb) 334{ 335 if (fd_cb->group != NULL) { 336 struct flow_divert_group *group = fd_cb->group; 337 lck_rw_lock_exclusive(&group->lck); 338 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count); 339 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb); 340 fd_cb->group = NULL; 341 FDRELEASE(fd_cb); /* Release the group's reference */ 342 lck_rw_done(&group->lck); 343 } 344} 345 346static int 347flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet) 348{ 349 struct flow_divert_packet_header hdr; 350 int error = 0; 351 352 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet); 353 if (error) { 354 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); 355 return error; 356 } 357 358 hdr.packet_type = packet_type; 359 hdr.conn_id = htonl(fd_cb->hash); 360 361 /* Lay down the header */ 362 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT); 363 if (error) { 364 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error); 365 mbuf_freem(*packet); 366 *packet = NULL; 367 return error; 368 } 369 370 return 0; 371} 372 373static int 374flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, size_t length, const void *value) 375{ 376 size_t net_length = htonl(length); 377 int error = 0; 378 379 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT); 380 if (error) { 381 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type); 382 return error; 383 } 384 385 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT); 386 if (error) { 387 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%lu)", length); 388 return error; 389 } 390 391 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT); 392 if (error) { 393 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value"); 394 return error; 395 } 396 397 return error; 398} 399 400static int 401flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next) 402{ 403 size_t cursor = offset; 404 int error = 0; 405 size_t curr_length; 406 uint8_t curr_type; 407 408 *err = 0; 409 410 do { 411 if (!next) { 412 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type); 413 if (error) { 414 *err = ENOENT; 415 return -1; 416 } 417 } else { 418 next = 0; 419 curr_type = FLOW_DIVERT_TLV_NIL; 420 } 421 422 if (curr_type != type) { 423 cursor += sizeof(curr_type); 424 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length); 425 if (error) { 426 *err = error; 427 return -1; 428 } 429 430 cursor += (sizeof(curr_length) + ntohl(curr_length)); 431 } 432 } while (curr_type != type); 433 434 return cursor; 435} 436 437static int 438flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, size_t *val_size) 439{ 440 int error = 0; 441 size_t length; 442 int tlv_offset; 443 444 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0); 445 if (tlv_offset < 0) { 446 return error; 447 } 448 449 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length); 450 if (error) { 451 return error; 452 } 453 454 length = ntohl(length); 455 456 if (val_size != NULL) { 457 *val_size = length; 458 } 459 460 if (buff != NULL && buff_len > 0) { 461 size_t to_copy = (length < buff_len) ? length : buff_len; 462 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff); 463 if (error) { 464 return error; 465 } 466 } 467 468 return 0; 469} 470 471static int 472flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac) 473{ 474 mbuf_t curr_mbuf = packet; 475 476 if (g_crypto_funcs == NULL || group->token_key == NULL) { 477 return ENOPROTOOPT; 478 } 479 480 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx); 481 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key); 482 483 while (curr_mbuf != NULL) { 484 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf)); 485 curr_mbuf = mbuf_next(curr_mbuf); 486 } 487 488 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac); 489 490 return 0; 491} 492 493static int 494flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit) 495{ 496 int error = 0; 497 struct flow_divert_group *group = NULL; 498 int hmac_offset; 499 uint8_t packet_hmac[SHA_DIGEST_LENGTH]; 500 uint8_t computed_hmac[SHA_DIGEST_LENGTH]; 501 mbuf_t tail; 502 503 lck_rw_lock_shared(&g_flow_divert_group_lck); 504 505 if (g_flow_divert_groups != NULL && g_active_group_count > 0) { 506 group = g_flow_divert_groups[ctl_unit]; 507 } 508 509 if (group == NULL) { 510 lck_rw_done(&g_flow_divert_group_lck); 511 return ENOPROTOOPT; 512 } 513 514 lck_rw_lock_shared(&group->lck); 515 516 if (group->token_key == NULL) { 517 error = ENOPROTOOPT; 518 goto done; 519 } 520 521 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0); 522 if (hmac_offset < 0) { 523 goto done; 524 } 525 526 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL); 527 if (error) { 528 goto done; 529 } 530 531 /* Chop off the HMAC TLV */ 532 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail); 533 if (error) { 534 goto done; 535 } 536 537 mbuf_free(tail); 538 539 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac); 540 if (error) { 541 goto done; 542 } 543 544 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) { 545 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC"); 546 error = EINVAL; 547 goto done; 548 } 549 550done: 551 lck_rw_done(&group->lck); 552 lck_rw_done(&g_flow_divert_group_lck); 553 return error; 554} 555 556static void 557flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send) 558{ 559 struct inpcb *inp = NULL; 560 struct ifnet *ifp = NULL; 561 Boolean cell = FALSE; 562 Boolean wifi = FALSE; 563 Boolean wired = FALSE; 564 565 inp = sotoinpcb(fd_cb->so); 566 if (inp == NULL) { 567 return; 568 } 569 570 ifp = inp->inp_last_outifp; 571 if (ifp != NULL) { 572 cell = IFNET_IS_CELLULAR(ifp); 573 wifi = (!cell && IFNET_IS_WIFI(ifp)); 574 wired = (!wifi && IFNET_IS_WIRED(ifp)); 575 } 576 577 if (send) { 578 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1); 579 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len); 580 } else { 581 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); 582 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len); 583 } 584} 585 586static errno_t 587flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb) 588{ 589 struct inpcb *inp = NULL; 590 591 inp = sotoinpcb(fd_cb->so); 592 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp && 593 IFNET_IS_CELLULAR(inp->inp_last_outifp)) 594 return EHOSTUNREACH; 595 596 return 0; 597} 598 599static errno_t 600flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb) 601{ 602 struct inpcb *inp = NULL; 603 604 inp = sotoinpcb(fd_cb->so); 605 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp && 606 IFNET_IS_EXPENSIVE(inp->inp_last_outifp)) 607 return EHOSTUNREACH; 608 609 return 0; 610} 611 612static void 613flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel) 614{ 615 if (how != SHUT_RD) { 616 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED; 617 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { 618 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; 619 /* If the tunnel is not accepting writes any more, then flush the send buffer */ 620 sbflush(&fd_cb->so->so_snd); 621 } 622 } 623 if (how != SHUT_WR) { 624 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED; 625 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { 626 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; 627 } 628 } 629} 630 631static uint16_t 632trie_node_alloc(struct flow_divert_trie *trie) 633{ 634 if (trie->nodes_free_next < trie->nodes_count) { 635 uint16_t node_idx = trie->nodes_free_next++; 636 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX; 637 return node_idx; 638 } else { 639 return NULL_TRIE_IDX; 640 } 641} 642 643static uint16_t 644trie_child_map_alloc(struct flow_divert_trie *trie) 645{ 646 if (trie->child_maps_free_next < trie->child_maps_count) { 647 return trie->child_maps_free_next++; 648 } else { 649 return NULL_TRIE_IDX; 650 } 651} 652 653static uint16_t 654trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size) 655{ 656 uint16_t start = trie->bytes_free_next; 657 if (start + bytes_size <= trie->bytes_count) { 658 if (start != bytes_idx) { 659 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size); 660 } 661 trie->bytes_free_next += bytes_size; 662 return start; 663 } else { 664 return NULL_TRIE_IDX; 665 } 666} 667 668static uint16_t 669flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len) 670{ 671 uint16_t current = trie->root; 672 uint16_t child = trie->root; 673 uint16_t string_end = string_start + string_len; 674 uint16_t string_idx = string_start; 675 uint16_t string_remainder = string_len; 676 677 while (child != NULL_TRIE_IDX) { 678 uint16_t parent = current; 679 uint16_t node_idx; 680 uint16_t current_end; 681 682 current = child; 683 child = NULL_TRIE_IDX; 684 685 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length; 686 687 for (node_idx = TRIE_NODE(trie, current).start; 688 node_idx < current_end && 689 string_idx < string_end && 690 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx); 691 node_idx++, string_idx++); 692 693 string_remainder = string_end - string_idx; 694 695 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) { 696 /* 697 * We did not reach the end of the current node's string. 698 * We need to split the current node into two: 699 * 1. A new node that contains the prefix of the node that matches 700 * the prefix of the string being inserted. 701 * 2. The current node modified to point to the remainder 702 * of the current node's string. 703 */ 704 uint16_t prefix = trie_node_alloc(trie); 705 if (prefix == NULL_TRIE_IDX) { 706 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node"); 707 return NULL_TRIE_IDX; 708 } 709 710 /* 711 * Prefix points to the portion of the current nodes's string that has matched 712 * the input string thus far. 713 */ 714 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start; 715 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start); 716 717 /* 718 * Prefix has the current node as the child corresponding to the first byte 719 * after the split. 720 */ 721 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie); 722 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) { 723 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node"); 724 return NULL_TRIE_IDX; 725 } 726 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current; 727 728 /* Parent has the prefix as the child correspoding to the first byte in the prefix */ 729 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix; 730 731 /* Current node is adjusted to point to the remainder */ 732 TRIE_NODE(trie, current).start = node_idx; 733 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length; 734 735 /* We want to insert the new leaf (if any) as a child of the prefix */ 736 current = prefix; 737 } 738 739 if (string_remainder > 0) { 740 /* 741 * We still have bytes in the string that have not been matched yet. 742 * If the current node has children, iterate to the child corresponding 743 * to the next byte in the string. 744 */ 745 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) { 746 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx)); 747 } 748 } 749 } /* while (child != NULL_TRIE_IDX) */ 750 751 if (string_remainder > 0) { 752 /* Add a new leaf containing the remainder of the string */ 753 uint16_t leaf = trie_node_alloc(trie); 754 if (leaf == NULL_TRIE_IDX) { 755 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf"); 756 return NULL_TRIE_IDX; 757 } 758 759 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder); 760 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) { 761 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf"); 762 return NULL_TRIE_IDX; 763 } 764 TRIE_NODE(trie, leaf).length = string_remainder; 765 766 /* Set the new leaf as the child of the current node */ 767 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) { 768 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie); 769 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) { 770 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf"); 771 return NULL_TRIE_IDX; 772 } 773 } 774 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf; 775 current = leaf; 776 } /* else duplicate or this string is a prefix of one of the existing strings */ 777 778 return current; 779} 780 781static uint16_t 782flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes) 783{ 784 uint16_t current = trie->root; 785 uint16_t string_idx = 0; 786 787 while (current != NULL_TRIE_IDX) { 788 uint16_t next = NULL_TRIE_IDX; 789 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length; 790 uint16_t node_idx; 791 792 for (node_idx = TRIE_NODE(trie, current).start; 793 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx); 794 node_idx++, string_idx++); 795 796 if (node_idx == node_end) { 797 if (string_bytes[string_idx] == '\0') { 798 return current; /* Got an exact match */ 799 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) { 800 next = TRIE_CHILD(trie, current, string_bytes[string_idx]); 801 } 802 } 803 current = next; 804 } 805 806 return NULL_TRIE_IDX; 807} 808 809static int 810flow_divert_get_src_proc(struct socket *so, proc_t *proc, boolean_t match_delegate) 811{ 812 int release = 0; 813 814 if (!match_delegate && 815 (so->so_flags & SOF_DELEGATED) && 816 (*proc == PROC_NULL || (*proc)->p_pid != so->e_pid)) 817 { 818 *proc = proc_find(so->e_pid); 819 release = 1; 820 } else if (*proc == PROC_NULL) { 821 *proc = current_proc(); 822 } 823 824 if (*proc != PROC_NULL) { 825 if ((*proc)->p_pid == 0) { 826 if (release) { 827 proc_rele(*proc); 828 } 829 release = 0; 830 *proc = PROC_NULL; 831 } 832 } 833 834 return release; 835} 836 837static int 838flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue) 839{ 840 int error; 841 842 if (fd_cb->group == NULL) { 843 fd_cb->so->so_error = ECONNABORTED; 844 soisdisconnected(fd_cb->so); 845 return ECONNABORTED; 846 } 847 848 lck_rw_lock_shared(&fd_cb->group->lck); 849 850 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) { 851 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR); 852 } else { 853 error = ENOBUFS; 854 } 855 856 if (error == ENOBUFS) { 857 if (enqueue) { 858 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) { 859 lck_rw_lock_exclusive(&fd_cb->group->lck); 860 } 861 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet); 862 error = 0; 863 } 864 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits); 865 } 866 867 lck_rw_done(&fd_cb->group->lck); 868 869 return error; 870} 871 872static int 873flow_divert_send_connect(struct flow_divert_pcb *fd_cb, struct sockaddr *to, mbuf_t connect_packet) 874{ 875 int error = 0; 876 877 error = flow_divert_packet_append_tlv(connect_packet, 878 FLOW_DIVERT_TLV_TRAFFIC_CLASS, 879 sizeof(fd_cb->so->so_traffic_class), 880 &fd_cb->so->so_traffic_class); 881 if (error) { 882 goto done; 883 } 884 885 if (fd_cb->so->so_flags & SOF_DELEGATED) { 886 error = flow_divert_packet_append_tlv(connect_packet, 887 FLOW_DIVERT_TLV_PID, 888 sizeof(fd_cb->so->e_pid), 889 &fd_cb->so->e_pid); 890 if (error) { 891 goto done; 892 } 893 894 error = flow_divert_packet_append_tlv(connect_packet, 895 FLOW_DIVERT_TLV_UUID, 896 sizeof(fd_cb->so->e_uuid), 897 &fd_cb->so->e_uuid); 898 if (error) { 899 goto done; 900 } 901 } else { 902 error = flow_divert_packet_append_tlv(connect_packet, 903 FLOW_DIVERT_TLV_PID, 904 sizeof(fd_cb->so->e_pid), 905 &fd_cb->so->last_pid); 906 if (error) { 907 goto done; 908 } 909 910 error = flow_divert_packet_append_tlv(connect_packet, 911 FLOW_DIVERT_TLV_UUID, 912 sizeof(fd_cb->so->e_uuid), 913 &fd_cb->so->last_uuid); 914 if (error) { 915 goto done; 916 } 917 } 918 919 if (fd_cb->connect_token != NULL) { 920 unsigned int token_len = m_length(fd_cb->connect_token); 921 mbuf_concatenate(connect_packet, fd_cb->connect_token); 922 mbuf_pkthdr_adjustlen(connect_packet, token_len); 923 fd_cb->connect_token = NULL; 924 } else { 925 uint32_t ctl_unit = htonl(fd_cb->control_group_unit); 926 int port; 927 928 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit); 929 if (error) { 930 goto done; 931 } 932 933 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, to->sa_len, to); 934 if (error) { 935 goto done; 936 } 937 938 if (to->sa_family == AF_INET) { 939 port = ntohs((satosin(to))->sin_port); 940 } 941#if INET6 942 else { 943 port = ntohs((satosin6(to))->sin6_port); 944 } 945#endif 946 947 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port); 948 if (error) { 949 goto done; 950 } 951 } 952 953 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE); 954 if (error) { 955 goto done; 956 } 957 958done: 959 return error; 960} 961 962static int 963flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb) 964{ 965 int error = 0; 966 mbuf_t packet = NULL; 967 int rbuff_space = 0; 968 969 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet); 970 if (error) { 971 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error); 972 goto done; 973 } 974 975 rbuff_space = sbspace(&fd_cb->so->so_rcv); 976 if (rbuff_space < 0) { 977 rbuff_space = 0; 978 } 979 rbuff_space = htonl(rbuff_space); 980 error = flow_divert_packet_append_tlv(packet, 981 FLOW_DIVERT_TLV_SPACE_AVAILABLE, 982 sizeof(rbuff_space), 983 &rbuff_space); 984 if (error) { 985 goto done; 986 } 987 988 error = flow_divert_send_packet(fd_cb, packet, TRUE); 989 if (error) { 990 goto done; 991 } 992 993done: 994 if (error && packet != NULL) { 995 mbuf_free(packet); 996 } 997 998 return error; 999} 1000 1001static int 1002flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how) 1003{ 1004 int error = 0; 1005 mbuf_t packet = NULL; 1006 uint32_t zero = 0; 1007 1008 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet); 1009 if (error) { 1010 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error); 1011 goto done; 1012 } 1013 1014 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero); 1015 if (error) { 1016 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error); 1017 goto done; 1018 } 1019 1020 how = htonl(how); 1021 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how); 1022 if (error) { 1023 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error); 1024 goto done; 1025 } 1026 1027 error = flow_divert_send_packet(fd_cb, packet, TRUE); 1028 if (error) { 1029 goto done; 1030 } 1031 1032done: 1033 if (error && packet != NULL) { 1034 mbuf_free(packet); 1035 } 1036 1037 return error; 1038} 1039 1040static int 1041flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb) 1042{ 1043 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == 1044 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) 1045 { 1046 return SHUT_RDWR; 1047 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) { 1048 return SHUT_RD; 1049 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) { 1050 return SHUT_WR; 1051 } 1052 1053 return -1; 1054} 1055 1056/* 1057 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and 1058 * writes. Returns FALSE otherwise. 1059 */ 1060static void 1061flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb) 1062{ 1063 int how = -1; 1064 1065 /* Do not send any close messages if there is still data in the send buffer */ 1066 if (fd_cb->so->so_snd.sb_cc == 0) { 1067 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) { 1068 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */ 1069 how = SHUT_RD; 1070 } 1071 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) { 1072 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */ 1073 if (how == SHUT_RD) { 1074 how = SHUT_RDWR; 1075 } else { 1076 how = SHUT_WR; 1077 } 1078 } 1079 } 1080 1081 if (how != -1) { 1082 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how); 1083 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) { 1084 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */ 1085 if (how != SHUT_RD) { 1086 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED; 1087 } 1088 if (how != SHUT_WR) { 1089 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED; 1090 } 1091 } 1092 } 1093 1094 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) { 1095 soisdisconnected(fd_cb->so); 1096 } 1097} 1098 1099static errno_t 1100flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, Boolean force) 1101{ 1102 mbuf_t packet; 1103 mbuf_t last; 1104 int error = 0; 1105 1106 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet); 1107 if (error) { 1108 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error); 1109 return error; 1110 } 1111 1112 last = m_last(packet); 1113 mbuf_setnext(last, data); 1114 mbuf_pkthdr_adjustlen(packet, data_len); 1115 1116 error = flow_divert_send_packet(fd_cb, packet, force); 1117 1118 if (error) { 1119 mbuf_setnext(last, NULL); 1120 mbuf_free(packet); 1121 } else { 1122 fd_cb->bytes_sent += data_len; 1123 flow_divert_add_data_statistics(fd_cb, data_len, TRUE); 1124 } 1125 1126 return error; 1127} 1128 1129static void 1130flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) 1131{ 1132 size_t to_send; 1133 size_t sent = 0; 1134 int error = 0; 1135 mbuf_t buffer; 1136 1137 to_send = fd_cb->so->so_snd.sb_cc; 1138 buffer = fd_cb->so->so_snd.sb_mb; 1139 1140 if (buffer == NULL && to_send > 0) { 1141 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send); 1142 return; 1143 } 1144 1145 /* Ignore the send window if force is enabled */ 1146 if (!force && (to_send > fd_cb->send_window)) { 1147 to_send = fd_cb->send_window; 1148 } 1149 1150 while (sent < to_send) { 1151 mbuf_t data; 1152 size_t data_len; 1153 1154 data_len = to_send - sent; 1155 if (data_len > FLOW_DIVERT_CHUNK_SIZE) { 1156 data_len = FLOW_DIVERT_CHUNK_SIZE; 1157 } 1158 1159 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data); 1160 if (error) { 1161 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); 1162 break; 1163 } 1164 1165 error = flow_divert_send_data_packet(fd_cb, data, data_len, force); 1166 if (error) { 1167 mbuf_free(data); 1168 break; 1169 } 1170 1171 sent += data_len; 1172 } 1173 1174 if (sent > 0) { 1175 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent); 1176 if (fd_cb->send_window >= sent) { 1177 fd_cb->send_window -= sent; 1178 } else { 1179 fd_cb->send_window = 0; 1180 } 1181 sbdrop(&fd_cb->so->so_snd, sent); 1182 sowwakeup(fd_cb->so); 1183 } 1184} 1185 1186static int 1187flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data) 1188{ 1189 size_t to_send = mbuf_pkthdr_len(data); 1190 size_t sent = 0; 1191 int error = 0; 1192 mbuf_t remaining_data = data; 1193 mbuf_t pkt_data = NULL; 1194 1195 if (to_send > fd_cb->send_window) { 1196 to_send = fd_cb->send_window; 1197 } 1198 1199 if (fd_cb->so->so_snd.sb_cc > 0) { 1200 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */ 1201 } 1202 1203 while (sent < to_send) { 1204 size_t pkt_data_len; 1205 1206 pkt_data = remaining_data; 1207 1208 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) { 1209 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE; 1210 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data); 1211 if (error) { 1212 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); 1213 pkt_data = NULL; 1214 break; 1215 } 1216 } else { 1217 pkt_data_len = to_send - sent; 1218 remaining_data = NULL; 1219 } 1220 1221 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, FALSE); 1222 1223 if (error) { 1224 break; 1225 } 1226 1227 pkt_data = NULL; 1228 sent += pkt_data_len; 1229 } 1230 1231 fd_cb->send_window -= sent; 1232 1233 error = 0; 1234 1235 if (pkt_data != NULL) { 1236 if (sbspace(&fd_cb->so->so_snd) > 0) { 1237 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) { 1238 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n", 1239 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); 1240 } 1241 } else { 1242 error = ENOBUFS; 1243 } 1244 } 1245 1246 if (remaining_data != NULL) { 1247 if (sbspace(&fd_cb->so->so_snd) > 0) { 1248 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) { 1249 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n", 1250 fd_cb->so->so_snd.sb_cc, fd_cb->send_window); 1251 } 1252 } else { 1253 error = ENOBUFS; 1254 } 1255 } 1256 1257 return error; 1258} 1259 1260static int 1261flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count) 1262{ 1263 int error = 0; 1264 mbuf_t packet = NULL; 1265 uint32_t net_read_count = htonl(read_count); 1266 1267 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet); 1268 if (error) { 1269 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error); 1270 goto done; 1271 } 1272 1273 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count); 1274 if (error) { 1275 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error); 1276 goto done; 1277 } 1278 1279 error = flow_divert_send_packet(fd_cb, packet, TRUE); 1280 if (error) { 1281 goto done; 1282 } 1283 1284done: 1285 if (error && packet != NULL) { 1286 mbuf_free(packet); 1287 } 1288 1289 return error; 1290} 1291 1292static int 1293flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class) 1294{ 1295 int error = 0; 1296 mbuf_t packet = NULL; 1297 1298 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet); 1299 if (error) { 1300 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error); 1301 goto done; 1302 } 1303 1304 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class); 1305 if (error) { 1306 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error); 1307 goto done; 1308 } 1309 1310 error = flow_divert_send_packet(fd_cb, packet, TRUE); 1311 if (error) { 1312 goto done; 1313 } 1314 1315done: 1316 if (error && packet != NULL) { 1317 mbuf_free(packet); 1318 } 1319 1320 return error; 1321} 1322 1323static void 1324flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) 1325{ 1326 uint32_t connect_error; 1327 uint32_t ctl_unit = 0; 1328 int error = 0; 1329 struct flow_divert_group *grp = NULL; 1330 struct sockaddr_storage local_address; 1331 int out_if_index = 0; 1332 struct sockaddr_storage remote_address; 1333 uint32_t send_window; 1334 1335 memset(&local_address, 0, sizeof(local_address)); 1336 memset(&remote_address, 0, sizeof(remote_address)); 1337 1338 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL); 1339 if (error) { 1340 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error); 1341 return; 1342 } 1343 1344 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error); 1345 1346 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL); 1347 if (error) { 1348 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error); 1349 return; 1350 } 1351 1352 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL); 1353 if (error) { 1354 FDLOG(LOG_ERR, fd_cb, "failed to get the control unit: %d", error); 1355 return; 1356 } 1357 1358 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL); 1359 if (error) { 1360 FDLOG0(LOG_NOTICE, fd_cb, "No local address provided"); 1361 } 1362 1363 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL); 1364 if (error) { 1365 FDLOG0(LOG_NOTICE, fd_cb, "No remote address provided"); 1366 } 1367 1368 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL); 1369 if (error) { 1370 FDLOG0(LOG_NOTICE, fd_cb, "No output if index provided"); 1371 } 1372 1373 connect_error = ntohl(connect_error); 1374 ctl_unit = ntohl(ctl_unit); 1375 1376 lck_rw_lock_shared(&g_flow_divert_group_lck); 1377 1378 if (connect_error == 0) { 1379 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) { 1380 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit); 1381 error = EINVAL; 1382 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) { 1383 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection"); 1384 error = EINVAL; 1385 } else { 1386 grp = g_flow_divert_groups[ctl_unit]; 1387 if (grp == NULL) { 1388 error = ECONNRESET; 1389 } 1390 } 1391 } 1392 1393 FDLOCK(fd_cb); 1394 if (fd_cb->so != NULL) { 1395 struct inpcb *inp = NULL; 1396 struct ifnet *ifp = NULL; 1397 struct flow_divert_group *old_group; 1398 1399 socket_lock(fd_cb->so, 0); 1400 1401 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) { 1402 goto done; 1403 } 1404 1405 inp = sotoinpcb(fd_cb->so); 1406 1407 if (connect_error || error) { 1408 goto set_socket_state; 1409 } 1410 1411 if (local_address.ss_family != 0) { 1412 if (local_address.ss_len > sizeof(local_address)) { 1413 local_address.ss_len = sizeof(local_address); 1414 } 1415 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1); 1416 } else { 1417 error = EINVAL; 1418 goto set_socket_state; 1419 } 1420 1421 if (remote_address.ss_family != 0) { 1422 if (remote_address.ss_len > sizeof(remote_address)) { 1423 remote_address.ss_len = sizeof(remote_address); 1424 } 1425 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1); 1426 } else { 1427 error = EINVAL; 1428 goto set_socket_state; 1429 } 1430 1431 ifnet_head_lock_shared(); 1432 if (out_if_index > 0 && out_if_index <= if_index) { 1433 ifp = ifindex2ifnet[out_if_index]; 1434 } 1435 1436 if (ifp != NULL) { 1437 inp->inp_last_outifp = ifp; 1438 } else { 1439 error = EINVAL; 1440 } 1441 ifnet_head_done(); 1442 1443 if (error) { 1444 goto set_socket_state; 1445 } 1446 1447 if (fd_cb->group == NULL) { 1448 error = EINVAL; 1449 goto set_socket_state; 1450 } 1451 1452 old_group = fd_cb->group; 1453 1454 lck_rw_lock_exclusive(&old_group->lck); 1455 lck_rw_lock_exclusive(&grp->lck); 1456 1457 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb); 1458 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) { 1459 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash); 1460 } 1461 1462 fd_cb->group = grp; 1463 1464 lck_rw_done(&grp->lck); 1465 lck_rw_done(&old_group->lck); 1466 1467 fd_cb->send_window = ntohl(send_window); 1468 flow_divert_send_buffered_data(fd_cb, FALSE); 1469 1470set_socket_state: 1471 if (!connect_error && !error) { 1472 FDLOG0(LOG_INFO, fd_cb, "sending connect result"); 1473 error = flow_divert_send_connect_result(fd_cb); 1474 } 1475 1476 if (connect_error || error) { 1477 if (!connect_error) { 1478 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE); 1479 fd_cb->so->so_error = error; 1480 flow_divert_send_close_if_needed(fd_cb); 1481 } else { 1482 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE); 1483 fd_cb->so->so_error = connect_error; 1484 } 1485 soisdisconnected(fd_cb->so); 1486 } else { 1487 soisconnected(fd_cb->so); 1488 } 1489 1490done: 1491 socket_unlock(fd_cb->so, 0); 1492 } 1493 FDUNLOCK(fd_cb); 1494 1495 lck_rw_done(&g_flow_divert_group_lck); 1496} 1497 1498static void 1499flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) 1500{ 1501 uint32_t close_error; 1502 int error = 0; 1503 int how; 1504 1505 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL); 1506 if (error) { 1507 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error); 1508 return; 1509 } 1510 1511 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL); 1512 if (error) { 1513 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error); 1514 return; 1515 } 1516 1517 how = ntohl(how); 1518 1519 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how); 1520 1521 FDLOCK(fd_cb); 1522 if (fd_cb->so != NULL) { 1523 socket_lock(fd_cb->so, 0); 1524 1525 fd_cb->so->so_error = ntohl(close_error); 1526 1527 flow_divert_update_closed_state(fd_cb, how, TRUE); 1528 1529 how = flow_divert_tunnel_how_closed(fd_cb); 1530 if (how == SHUT_RDWR) { 1531 soisdisconnected(fd_cb->so); 1532 } else if (how == SHUT_RD) { 1533 socantrcvmore(fd_cb->so); 1534 } else if (how == SHUT_WR) { 1535 socantsendmore(fd_cb->so); 1536 } 1537 1538 socket_unlock(fd_cb->so, 0); 1539 } 1540 FDUNLOCK(fd_cb); 1541} 1542 1543static void 1544flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset) 1545{ 1546 int error = 0; 1547 mbuf_t data = NULL; 1548 size_t data_size; 1549 1550 data_size = (mbuf_pkthdr_len(packet) - offset); 1551 1552 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size); 1553 1554 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data); 1555 if (error || data == NULL) { 1556 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error); 1557 return; 1558 } 1559 1560 FDLOCK(fd_cb); 1561 if (fd_cb->so != NULL) { 1562 socket_lock(fd_cb->so, 0); 1563 if (flow_divert_check_no_cellular(fd_cb) || 1564 flow_divert_check_no_expensive(fd_cb)) { 1565 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE); 1566 flow_divert_send_close(fd_cb, SHUT_RDWR); 1567 soisdisconnected(fd_cb->so); 1568 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) { 1569 if (sbappendstream(&fd_cb->so->so_rcv, data)) { 1570 fd_cb->bytes_received += data_size; 1571 flow_divert_add_data_statistics(fd_cb, data_size, FALSE); 1572 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc; 1573 sorwakeup(fd_cb->so); 1574 data = NULL; 1575 } else { 1576 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed"); 1577 } 1578 } 1579 socket_unlock(fd_cb->so, 0); 1580 } 1581 FDUNLOCK(fd_cb); 1582 1583 if (data != NULL) { 1584 mbuf_free(data); 1585 } 1586} 1587 1588static void 1589flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) 1590{ 1591 uint32_t read_count; 1592 int error = 0; 1593 1594 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL); 1595 if (error) { 1596 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error); 1597 return; 1598 } 1599 1600 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", read_count); 1601 1602 FDLOCK(fd_cb); 1603 if (fd_cb->so != NULL) { 1604 socket_lock(fd_cb->so, 0); 1605 fd_cb->send_window += ntohl(read_count); 1606 flow_divert_send_buffered_data(fd_cb, FALSE); 1607 socket_unlock(fd_cb->so, 0); 1608 } 1609 FDUNLOCK(fd_cb); 1610} 1611 1612static void 1613flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset) 1614{ 1615 int error = 0; 1616 size_t key_size = 0; 1617 int log_level; 1618 1619 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size); 1620 if (error) { 1621 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error); 1622 return; 1623 } 1624 1625 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) { 1626 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %lu", key_size); 1627 return; 1628 } 1629 1630 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL); 1631 if (!error) { 1632 nil_pcb.log_level = log_level; 1633 } 1634 1635 lck_rw_lock_exclusive(&group->lck); 1636 1637 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK); 1638 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL); 1639 if (error) { 1640 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error); 1641 FREE(group->token_key, M_TEMP); 1642 group->token_key = NULL; 1643 lck_rw_done(&group->lck); 1644 return; 1645 } 1646 1647 group->token_key_size = key_size; 1648 1649 lck_rw_done(&group->lck); 1650} 1651 1652static void 1653flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset) 1654{ 1655 int error = 0; 1656 struct sockaddr_storage local_address; 1657 int out_if_index = 0; 1658 struct sockaddr_storage remote_address; 1659 1660 FDLOG0(LOG_INFO, fd_cb, "received a properties update"); 1661 1662 memset(&local_address, 0, sizeof(local_address)); 1663 memset(&remote_address, 0, sizeof(remote_address)); 1664 1665 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL); 1666 if (error) { 1667 FDLOG0(LOG_INFO, fd_cb, "No local address provided"); 1668 } 1669 1670 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL); 1671 if (error) { 1672 FDLOG0(LOG_INFO, fd_cb, "No remote address provided"); 1673 } 1674 1675 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL); 1676 if (error) { 1677 FDLOG0(LOG_INFO, fd_cb, "No output if index provided"); 1678 } 1679 1680 FDLOCK(fd_cb); 1681 if (fd_cb->so != NULL) { 1682 struct inpcb *inp = NULL; 1683 struct ifnet *ifp = NULL; 1684 1685 socket_lock(fd_cb->so, 0); 1686 1687 inp = sotoinpcb(fd_cb->so); 1688 1689 if (local_address.ss_family != 0) { 1690 if (local_address.ss_len > sizeof(local_address)) { 1691 local_address.ss_len = sizeof(local_address); 1692 } 1693 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1); 1694 } 1695 1696 if (remote_address.ss_family != 0) { 1697 if (remote_address.ss_len > sizeof(remote_address)) { 1698 remote_address.ss_len = sizeof(remote_address); 1699 } 1700 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1); 1701 } 1702 1703 ifnet_head_lock_shared(); 1704 if (out_if_index > 0 && out_if_index <= if_index) { 1705 ifp = ifindex2ifnet[out_if_index]; 1706 } 1707 1708 if (ifp != NULL) { 1709 inp->inp_last_outifp = ifp; 1710 } 1711 ifnet_head_done(); 1712 1713 socket_unlock(fd_cb->so, 0); 1714 } 1715 FDUNLOCK(fd_cb); 1716} 1717 1718static void 1719flow_divert_handle_app_map_create(mbuf_t packet, int offset) 1720{ 1721 size_t bytes_mem_size; 1722 size_t child_maps_mem_size; 1723 int cursor; 1724 int error = 0; 1725 struct flow_divert_trie new_trie; 1726 int insert_error = 0; 1727 size_t nodes_mem_size; 1728 int prefix_count = 0; 1729 int signing_id_count = 0; 1730 1731 lck_rw_lock_exclusive(&g_flow_divert_group_lck); 1732 1733 /* Re-set the current trie */ 1734 if (g_signing_id_trie.memory != NULL) { 1735 FREE(g_signing_id_trie.memory, M_TEMP); 1736 } 1737 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie)); 1738 g_signing_id_trie.root = NULL_TRIE_IDX; 1739 1740 memset(&new_trie, 0, sizeof(new_trie)); 1741 1742 /* Get the number of shared prefixes in the new set of signing ID strings */ 1743 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL); 1744 1745 /* Compute the number of signing IDs and the total amount of bytes needed to store them */ 1746 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0); 1747 cursor >= 0; 1748 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) 1749 { 1750 size_t sid_size = 0; 1751 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); 1752 new_trie.bytes_count += sid_size; 1753 signing_id_count++; 1754 } 1755 1756 if (signing_id_count == 0) { 1757 lck_rw_done(&g_flow_divert_group_lck); 1758 return; 1759 } 1760 1761 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */ 1762 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */ 1763 1764 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu", 1765 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count); 1766 1767 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count); 1768 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count); 1769 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count); 1770 1771 MALLOC(new_trie.memory, void *, nodes_mem_size + child_maps_mem_size + bytes_mem_size, M_TEMP, M_WAITOK); 1772 if (new_trie.memory == NULL) { 1773 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie", 1774 nodes_mem_size + child_maps_mem_size + bytes_mem_size); 1775 return; 1776 } 1777 1778 /* Initialize the free lists */ 1779 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory; 1780 new_trie.nodes_free_next = 0; 1781 memset(new_trie.nodes, 0, nodes_mem_size); 1782 1783 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size); 1784 new_trie.child_maps_free_next = 0; 1785 memset(new_trie.child_maps, 0xff, child_maps_mem_size); 1786 1787 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size); 1788 new_trie.bytes_free_next = 0; 1789 1790 /* The root is an empty node */ 1791 new_trie.root = trie_node_alloc(&new_trie); 1792 1793 /* Add each signing ID to the trie */ 1794 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0); 1795 cursor >= 0; 1796 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) 1797 { 1798 size_t sid_size = 0; 1799 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); 1800 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) { 1801 boolean_t is_dns; 1802 uint16_t new_node_idx; 1803 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL); 1804 is_dns = (sid_size == sizeof(FLOW_DIVERT_DNS_SERVICE_SIGNING_ID) - 1 && 1805 !memcmp(&TRIE_BYTE(&new_trie, new_trie.bytes_free_next), 1806 FLOW_DIVERT_DNS_SERVICE_SIGNING_ID, 1807 sid_size)); 1808 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size); 1809 if (new_node_idx != NULL_TRIE_IDX) { 1810 if (is_dns) { 1811 FDLOG(LOG_INFO, &nil_pcb, "Setting group unit for %s to %d", FLOW_DIVERT_DNS_SERVICE_SIGNING_ID, DNS_SERVICE_GROUP_UNIT); 1812 TRIE_NODE(&new_trie, new_node_idx).group_unit = DNS_SERVICE_GROUP_UNIT; 1813 } 1814 } else { 1815 insert_error = EINVAL; 1816 break; 1817 } 1818 } else { 1819 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion"); 1820 insert_error = ENOBUFS; 1821 break; 1822 } 1823 } 1824 1825 if (!insert_error) { 1826 g_signing_id_trie = new_trie; 1827 } else { 1828 FREE(new_trie.memory, M_TEMP); 1829 } 1830 1831 lck_rw_done(&g_flow_divert_group_lck); 1832} 1833 1834static void 1835flow_divert_handle_app_map_update(struct flow_divert_group *group, mbuf_t packet, int offset) 1836{ 1837 int error = 0; 1838 int cursor; 1839 size_t max_size = 0; 1840 uint8_t *signing_id; 1841 uint32_t ctl_unit; 1842 1843 lck_rw_lock_shared(&group->lck); 1844 ctl_unit = group->ctl_unit; 1845 lck_rw_done(&group->lck); 1846 1847 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0); 1848 cursor >= 0; 1849 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) 1850 { 1851 size_t sid_size = 0; 1852 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); 1853 if (sid_size > max_size) { 1854 max_size = sid_size; 1855 } 1856 } 1857 1858 MALLOC(signing_id, uint8_t *, max_size + 1, M_TEMP, M_WAITOK); 1859 if (signing_id == NULL) { 1860 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate a string to hold the signing ID (size %lu)", max_size); 1861 return; 1862 } 1863 1864 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0); 1865 cursor >= 0; 1866 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) 1867 { 1868 size_t signing_id_len = 0; 1869 uint16_t node; 1870 1871 flow_divert_packet_get_tlv(packet, 1872 cursor, FLOW_DIVERT_TLV_SIGNING_ID, max_size, signing_id, &signing_id_len); 1873 1874 signing_id[signing_id_len] = '\0'; 1875 1876 lck_rw_lock_exclusive(&g_flow_divert_group_lck); 1877 1878 node = flow_divert_trie_search(&g_signing_id_trie, signing_id); 1879 if (node != NULL_TRIE_IDX) { 1880 if (TRIE_NODE(&g_signing_id_trie, node).group_unit != DNS_SERVICE_GROUP_UNIT) { 1881 FDLOG(LOG_INFO, &nil_pcb, "Setting %s to ctl unit %u", signing_id, group->ctl_unit); 1882 TRIE_NODE(&g_signing_id_trie, node).group_unit = ctl_unit; 1883 } 1884 } else { 1885 FDLOG(LOG_ERR, &nil_pcb, "Failed to find signing ID %s", signing_id); 1886 } 1887 1888 lck_rw_done(&g_flow_divert_group_lck); 1889 } 1890 1891 FREE(signing_id, M_TEMP); 1892} 1893 1894static int 1895flow_divert_input(mbuf_t packet, struct flow_divert_group *group) 1896{ 1897 struct flow_divert_packet_header hdr; 1898 int error = 0; 1899 struct flow_divert_pcb *fd_cb; 1900 1901 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) { 1902 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr)); 1903 error = EINVAL; 1904 goto done; 1905 } 1906 1907 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr); 1908 if (error) { 1909 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error); 1910 error = ENOBUFS; 1911 goto done; 1912 } 1913 1914 hdr.conn_id = ntohl(hdr.conn_id); 1915 1916 if (hdr.conn_id == 0) { 1917 switch (hdr.packet_type) { 1918 case FLOW_DIVERT_PKT_GROUP_INIT: 1919 flow_divert_handle_group_init(group, packet, sizeof(hdr)); 1920 break; 1921 case FLOW_DIVERT_PKT_APP_MAP_CREATE: 1922 flow_divert_handle_app_map_create(packet, sizeof(hdr)); 1923 break; 1924 case FLOW_DIVERT_PKT_APP_MAP_UPDATE: 1925 flow_divert_handle_app_map_update(group, packet, sizeof(hdr)); 1926 break; 1927 default: 1928 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type); 1929 break; 1930 } 1931 goto done; 1932 } 1933 1934 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */ 1935 if (fd_cb == NULL) { 1936 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) { 1937 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id); 1938 } 1939 goto done; 1940 } 1941 1942 switch (hdr.packet_type) { 1943 case FLOW_DIVERT_PKT_CONNECT_RESULT: 1944 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr)); 1945 break; 1946 case FLOW_DIVERT_PKT_CLOSE: 1947 flow_divert_handle_close(fd_cb, packet, sizeof(hdr)); 1948 break; 1949 case FLOW_DIVERT_PKT_DATA: 1950 flow_divert_handle_data(fd_cb, packet, sizeof(hdr)); 1951 break; 1952 case FLOW_DIVERT_PKT_READ_NOTIFY: 1953 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr)); 1954 break; 1955 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE: 1956 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr)); 1957 break; 1958 default: 1959 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type); 1960 break; 1961 } 1962 1963 FDRELEASE(fd_cb); 1964 1965done: 1966 mbuf_free(packet); 1967 return error; 1968} 1969 1970static void 1971flow_divert_close_all(struct flow_divert_group *group) 1972{ 1973 struct flow_divert_pcb *fd_cb; 1974 SLIST_HEAD(, flow_divert_pcb) tmp_list; 1975 1976 SLIST_INIT(&tmp_list); 1977 1978 lck_rw_lock_exclusive(&group->lck); 1979 1980 MBUFQ_DRAIN(&group->send_queue); 1981 1982 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { 1983 FDRETAIN(fd_cb); 1984 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); 1985 } 1986 1987 lck_rw_done(&group->lck); 1988 1989 while (!SLIST_EMPTY(&tmp_list)) { 1990 fd_cb = SLIST_FIRST(&tmp_list); 1991 FDLOCK(fd_cb); 1992 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry); 1993 if (fd_cb->so != NULL) { 1994 socket_lock(fd_cb->so, 0); 1995 flow_divert_pcb_remove(fd_cb); 1996 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE); 1997 fd_cb->so->so_error = ECONNABORTED; 1998 socket_unlock(fd_cb->so, 0); 1999 } 2000 FDUNLOCK(fd_cb); 2001 FDRELEASE(fd_cb); 2002 } 2003} 2004 2005void 2006flow_divert_detach(struct socket *so) 2007{ 2008 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2009 2010 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2011 2012 so->so_flags &= ~SOF_FLOW_DIVERT; 2013 so->so_fd_pcb = NULL; 2014 2015 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count); 2016 2017 if (fd_cb->group != NULL) { 2018 /* Last-ditch effort to send any buffered data */ 2019 flow_divert_send_buffered_data(fd_cb, TRUE); 2020 2021 /* Remove from the group */ 2022 flow_divert_pcb_remove(fd_cb); 2023 } 2024 2025 socket_unlock(so, 0); 2026 FDLOCK(fd_cb); 2027 fd_cb->so = NULL; 2028 FDUNLOCK(fd_cb); 2029 socket_lock(so, 0); 2030 2031 FDRELEASE(fd_cb); /* Release the socket's reference */ 2032} 2033 2034static int 2035flow_divert_close(struct socket *so) 2036{ 2037 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2038 2039 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2040 2041 FDLOG0(LOG_INFO, fd_cb, "Closing"); 2042 2043 soisdisconnecting(so); 2044 sbflush(&so->so_rcv); 2045 2046 flow_divert_send_buffered_data(fd_cb, TRUE); 2047 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE); 2048 flow_divert_send_close_if_needed(fd_cb); 2049 2050 /* Remove from the group */ 2051 flow_divert_pcb_remove(fd_cb); 2052 2053 return 0; 2054} 2055 2056static int 2057flow_divert_disconnectx(struct socket *so, associd_t aid, connid_t cid __unused) 2058{ 2059 if (aid != ASSOCID_ANY && aid != ASSOCID_ALL) { 2060 return (EINVAL); 2061 } 2062 2063 return (flow_divert_close(so)); 2064} 2065 2066static int 2067flow_divert_shutdown(struct socket *so) 2068{ 2069 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2070 2071 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2072 2073 FDLOG0(LOG_INFO, fd_cb, "Can't send more"); 2074 2075 socantsendmore(so); 2076 2077 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE); 2078 flow_divert_send_close_if_needed(fd_cb); 2079 2080 return 0; 2081} 2082 2083static int 2084flow_divert_rcvd(struct socket *so, int flags __unused) 2085{ 2086 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2087 uint32_t latest_sb_size; 2088 uint32_t read_count; 2089 2090 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2091 2092 latest_sb_size = fd_cb->so->so_rcv.sb_cc; 2093 2094 if (fd_cb->sb_size < latest_sb_size) { 2095 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)", 2096 fd_cb->hash, fd_cb->sb_size, latest_sb_size); 2097 } 2098 2099 read_count = fd_cb->sb_size - latest_sb_size; 2100 2101 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count); 2102 2103 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) { 2104 fd_cb->bytes_read_by_app += read_count; 2105 fd_cb->sb_size = latest_sb_size; 2106 } 2107 2108 return 0; 2109} 2110 2111static errno_t 2112flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, 2113 struct sockaddr **dup) 2114{ 2115 int error = 0; 2116 struct sockaddr *result; 2117 struct sockaddr_storage ss; 2118 2119 if (addr != NULL) { 2120 result = addr; 2121 } else { 2122 memset(&ss, 0, sizeof(ss)); 2123 ss.ss_family = family; 2124 if (ss.ss_family == AF_INET) { 2125 ss.ss_len = sizeof(struct sockaddr_in); 2126 } 2127#if INET6 2128 else if (ss.ss_family == AF_INET6) { 2129 ss.ss_len = sizeof(struct sockaddr_in6); 2130 } 2131#endif /* INET6 */ 2132 else { 2133 error = EINVAL; 2134 } 2135 result = (struct sockaddr *)&ss; 2136 } 2137 2138 if (!error) { 2139 *dup = dup_sockaddr(result, 1); 2140 if (*dup == NULL) { 2141 error = ENOBUFS; 2142 } 2143 } 2144 2145 return error; 2146} 2147 2148static errno_t 2149flow_divert_getpeername(struct socket *so, struct sockaddr **sa) 2150{ 2151 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2152 2153 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2154 2155 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family, 2156 fd_cb->remote_address, 2157 sa); 2158} 2159 2160static errno_t 2161flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa) 2162{ 2163 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2164 2165 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2166 2167 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family, 2168 fd_cb->local_address, 2169 sa); 2170} 2171 2172static errno_t 2173flow_divert_ctloutput(struct socket *so, struct sockopt *sopt) 2174{ 2175 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2176 2177 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2178 2179 if (sopt->sopt_name == SO_TRAFFIC_CLASS) { 2180 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) { 2181 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class); 2182 } 2183 } 2184 2185 if (SOCK_DOM(so) == PF_INET) { 2186 return g_tcp_protosw->pr_ctloutput(so, sopt); 2187 } 2188#if INET6 2189 else if (SOCK_DOM(so) == PF_INET6) { 2190 return g_tcp6_protosw->pr_ctloutput(so, sopt); 2191 } 2192#endif 2193 return 0; 2194} 2195 2196errno_t 2197flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p) 2198{ 2199 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2200 int error = 0; 2201 struct inpcb *inp = sotoinpcb(so); 2202 struct sockaddr_in *sinp; 2203 mbuf_t connect_packet = NULL; 2204 char *signing_id = NULL; 2205 int free_signing_id = 0; 2206 2207 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2208 2209 if (fd_cb->group == NULL) { 2210 error = ENETUNREACH; 2211 goto done; 2212 } 2213 2214 if (inp == NULL) { 2215 error = EINVAL; 2216 goto done; 2217 } else if (inp->inp_state == INPCB_STATE_DEAD) { 2218 if (so->so_error) { 2219 error = so->so_error; 2220 so->so_error = 0; 2221 } else { 2222 error = EINVAL; 2223 } 2224 goto done; 2225 } 2226 2227 sinp = (struct sockaddr_in *)(void *)to; 2228 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) { 2229 error = EAFNOSUPPORT; 2230 goto done; 2231 } 2232 2233 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) { 2234 error = EALREADY; 2235 goto done; 2236 } 2237 2238 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) { 2239 FDLOG0(LOG_INFO, fd_cb, "fully transferred"); 2240 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED; 2241 if (fd_cb->remote_address != NULL) { 2242 soisconnected(fd_cb->so); 2243 goto done; 2244 } 2245 } 2246 2247 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet); 2248 if (error) { 2249 goto done; 2250 } 2251 2252 error = EPERM; 2253 2254 if (fd_cb->connect_token != NULL) { 2255 size_t sid_size = 0; 2256 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size); 2257 if (find_error == 0 && sid_size > 0) { 2258 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO); 2259 if (signing_id != NULL) { 2260 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL); 2261 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id); 2262 free_signing_id = 1; 2263 } 2264 } 2265 } 2266 2267 socket_unlock(so, 0); 2268 if (g_signing_id_trie.root != NULL_TRIE_IDX) { 2269 proc_t src_proc = p; 2270 int release_proc = 0; 2271 2272 if (signing_id == NULL) { 2273 release_proc = flow_divert_get_src_proc(so, &src_proc, FALSE); 2274 if (src_proc != PROC_NULL) { 2275 proc_lock(src_proc); 2276 if (src_proc->p_csflags & CS_VALID) { 2277 signing_id = (char *)cs_identity_get(src_proc); 2278 } else { 2279 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid"); 2280 } 2281 } else { 2282 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc"); 2283 } 2284 } else { 2285 src_proc = PROC_NULL; 2286 } 2287 2288 if (signing_id != NULL) { 2289 uint16_t result = NULL_TRIE_IDX; 2290 lck_rw_lock_shared(&g_flow_divert_group_lck); 2291 result = flow_divert_trie_search(&g_signing_id_trie, (const uint8_t *)signing_id); 2292 lck_rw_done(&g_flow_divert_group_lck); 2293 if (result != NULL_TRIE_IDX) { 2294 error = 0; 2295 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id); 2296 2297 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id); 2298 if (error == 0) { 2299 if (src_proc != PROC_NULL) { 2300 unsigned char cdhash[SHA1_RESULTLEN]; 2301 error = proc_getcdhash(src_proc, cdhash); 2302 if (error == 0) { 2303 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash); 2304 if (error) { 2305 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error); 2306 } 2307 } else { 2308 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error); 2309 } 2310 } 2311 } else { 2312 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error); 2313 } 2314 } else { 2315 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id); 2316 } 2317 } else { 2318 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity"); 2319 } 2320 2321 if (src_proc != PROC_NULL) { 2322 proc_unlock(src_proc); 2323 if (release_proc) { 2324 proc_rele(src_proc); 2325 } 2326 } 2327 } else { 2328 FDLOG0(LOG_WARNING, fd_cb, "The signing ID trie is empty"); 2329 } 2330 socket_lock(so, 0); 2331 2332 if (free_signing_id) { 2333 FREE(signing_id, M_TEMP); 2334 } 2335 2336 if (error) { 2337 goto done; 2338 } 2339 2340 FDLOG0(LOG_INFO, fd_cb, "Connecting"); 2341 2342 error = flow_divert_send_connect(fd_cb, to, connect_packet); 2343 if (error) { 2344 goto done; 2345 } 2346 2347 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; 2348 2349 soisconnecting(so); 2350 2351done: 2352 if (error && connect_packet != NULL) { 2353 mbuf_free(connect_packet); 2354 } 2355 return error; 2356} 2357 2358static int 2359flow_divert_connectx_out_common(struct socket *so, int af, 2360 struct sockaddr_list **src_sl, struct sockaddr_list **dst_sl, 2361 struct proc *p, uint32_t ifscope __unused, associd_t aid __unused, 2362 connid_t *pcid, uint32_t flags __unused, void *arg __unused, 2363 uint32_t arglen __unused) 2364{ 2365 struct sockaddr_entry *src_se = NULL, *dst_se = NULL; 2366 struct inpcb *inp = sotoinpcb(so); 2367 int error; 2368 2369 if (inp == NULL) { 2370 return (EINVAL); 2371 } 2372 2373 VERIFY(dst_sl != NULL); 2374 2375 /* select source (if specified) and destination addresses */ 2376 error = in_selectaddrs(af, src_sl, &src_se, dst_sl, &dst_se); 2377 if (error != 0) { 2378 return (error); 2379 } 2380 2381 VERIFY(*dst_sl != NULL && dst_se != NULL); 2382 VERIFY(src_se == NULL || *src_sl != NULL); 2383 VERIFY(dst_se->se_addr->sa_family == af); 2384 VERIFY(src_se == NULL || src_se->se_addr->sa_family == af); 2385 2386 error = flow_divert_connect_out(so, dst_se->se_addr, p); 2387 2388 if (error == 0 && pcid != NULL) { 2389 *pcid = 1; /* there is only 1 connection for a TCP */ 2390 } 2391 2392 return (error); 2393} 2394 2395static int 2396flow_divert_connectx_out(struct socket *so, struct sockaddr_list **src_sl, 2397 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope, 2398 associd_t aid, connid_t *pcid, uint32_t flags, void *arg, 2399 uint32_t arglen) 2400{ 2401 return (flow_divert_connectx_out_common(so, AF_INET, src_sl, dst_sl, 2402 p, ifscope, aid, pcid, flags, arg, arglen)); 2403} 2404 2405#if INET6 2406static int 2407flow_divert_connectx6_out(struct socket *so, struct sockaddr_list **src_sl, 2408 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope, 2409 associd_t aid, connid_t *pcid, uint32_t flags, void *arg, 2410 uint32_t arglen) 2411{ 2412 return (flow_divert_connectx_out_common(so, AF_INET6, src_sl, dst_sl, 2413 p, ifscope, aid, pcid, flags, arg, arglen)); 2414} 2415#endif /* INET6 */ 2416 2417static int 2418flow_divert_getconninfo(struct socket *so, connid_t cid, uint32_t *flags, 2419 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len, 2420 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, 2421 user_addr_t aux_data __unused, uint32_t *aux_len) 2422{ 2423 int error = 0; 2424 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2425 struct ifnet *ifp = NULL; 2426 struct inpcb *inp = sotoinpcb(so); 2427 2428 VERIFY((so->so_flags & SOF_FLOW_DIVERT)); 2429 2430 if (so->so_fd_pcb == NULL || inp == NULL) { 2431 error = EINVAL; 2432 goto out; 2433 } 2434 2435 if (cid != CONNID_ANY && cid != CONNID_ALL && cid != 1) { 2436 error = EINVAL; 2437 goto out; 2438 } 2439 2440 ifp = inp->inp_last_outifp; 2441 *ifindex = ((ifp != NULL) ? ifp->if_index : 0); 2442 *soerror = so->so_error; 2443 *flags = 0; 2444 2445 if (so->so_state & SS_ISCONNECTED) { 2446 *flags |= (CIF_CONNECTED | CIF_PREFERRED); 2447 } 2448 2449 if (fd_cb->local_address == NULL) { 2450 struct sockaddr_in sin; 2451 bzero(&sin, sizeof(sin)); 2452 sin.sin_len = sizeof(sin); 2453 sin.sin_family = AF_INET; 2454 *src_len = sin.sin_len; 2455 if (src != USER_ADDR_NULL) { 2456 error = copyout(&sin, src, sin.sin_len); 2457 if (error != 0) { 2458 goto out; 2459 } 2460 } 2461 } else { 2462 *src_len = fd_cb->local_address->sa_len; 2463 if (src != USER_ADDR_NULL) { 2464 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len); 2465 if (error != 0) { 2466 goto out; 2467 } 2468 } 2469 } 2470 2471 if (fd_cb->remote_address == NULL) { 2472 struct sockaddr_in sin; 2473 bzero(&sin, sizeof(sin)); 2474 sin.sin_len = sizeof(sin); 2475 sin.sin_family = AF_INET; 2476 *dst_len = sin.sin_len; 2477 if (dst != USER_ADDR_NULL) { 2478 error = copyout(&sin, dst, sin.sin_len); 2479 if (error != 0) { 2480 goto out; 2481 } 2482 } 2483 } else { 2484 *dst_len = fd_cb->remote_address->sa_len; 2485 if (dst != USER_ADDR_NULL) { 2486 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len); 2487 if (error != 0) { 2488 goto out; 2489 } 2490 } 2491 } 2492 2493 *aux_type = 0; 2494 *aux_len = 0; 2495 2496out: 2497 return error; 2498} 2499 2500static int 2501flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused) 2502{ 2503 int error = 0; 2504 2505 switch (cmd) { 2506 case SIOCGCONNINFO32: { 2507 struct so_cinforeq32 cifr; 2508 bcopy(data, &cifr, sizeof (cifr)); 2509 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags, 2510 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src, 2511 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len, 2512 &cifr.scir_aux_type, cifr.scir_aux_data, 2513 &cifr.scir_aux_len); 2514 if (error == 0) { 2515 bcopy(&cifr, data, sizeof (cifr)); 2516 } 2517 break; 2518 } 2519 2520 case SIOCGCONNINFO64: { 2521 struct so_cinforeq64 cifr; 2522 bcopy(data, &cifr, sizeof (cifr)); 2523 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags, 2524 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src, 2525 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len, 2526 &cifr.scir_aux_type, cifr.scir_aux_data, 2527 &cifr.scir_aux_len); 2528 if (error == 0) { 2529 bcopy(&cifr, data, sizeof (cifr)); 2530 } 2531 break; 2532 } 2533 2534 default: 2535 error = EOPNOTSUPP; 2536 } 2537 2538 return error; 2539} 2540 2541static int 2542flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p) 2543{ 2544 int error = flow_divert_control(so, cmd, data, ifp, p); 2545 2546 if (error == EOPNOTSUPP) { 2547 error = in_control(so, cmd, data, ifp, p); 2548 } 2549 2550 return error; 2551} 2552 2553static int 2554flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p) 2555{ 2556 int error = flow_divert_control(so, cmd, data, ifp, p); 2557 2558 if (error == EOPNOTSUPP) { 2559 error = in6_control(so, cmd, data, ifp, p); 2560 } 2561 2562 return error; 2563} 2564 2565static errno_t 2566flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p __unused) 2567{ 2568 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2569 int error = 0; 2570 struct inpcb *inp; 2571 2572 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2573 2574 inp = sotoinpcb(so); 2575 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) { 2576 error = ECONNRESET; 2577 goto done; 2578 } 2579 2580 if (control && mbuf_len(control) > 0) { 2581 error = EINVAL; 2582 goto done; 2583 } 2584 2585 if (flags & MSG_OOB) { 2586 error = EINVAL; 2587 goto done; /* We don't support OOB data */ 2588 } 2589 2590 error = flow_divert_check_no_cellular(fd_cb) || 2591 flow_divert_check_no_expensive(fd_cb); 2592 if (error) { 2593 goto done; 2594 } 2595 2596 /* Implicit connect */ 2597 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) { 2598 FDLOG0(LOG_INFO, fd_cb, "implicit connect"); 2599 error = flow_divert_connect_out(so, to, NULL); 2600 if (error) { 2601 goto done; 2602 } 2603 } 2604 2605 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data)); 2606 2607 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data); 2608 error = flow_divert_send_app_data(fd_cb, data); 2609 if (error) { 2610 goto done; 2611 } 2612 2613 data = NULL; 2614 2615 if (flags & PRUS_EOF) { 2616 flow_divert_shutdown(so); 2617 } 2618 2619done: 2620 if (data) { 2621 mbuf_free(data); 2622 } 2623 if (control) { 2624 mbuf_free(control); 2625 } 2626 return error; 2627} 2628 2629static void 2630flow_divert_set_protosw(struct socket *so) 2631{ 2632 so->so_flags |= SOF_FLOW_DIVERT; 2633 if (SOCK_DOM(so) == PF_INET) { 2634 so->so_proto = &g_flow_divert_in_protosw; 2635 } 2636#if INET6 2637 else { 2638 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw; 2639 } 2640#endif /* INET6 */ 2641} 2642 2643static errno_t 2644flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit) 2645{ 2646 int error = 0; 2647 struct flow_divert_pcb *fd_cb = NULL; 2648 struct ifnet *ifp = NULL; 2649 struct inpcb *inp = NULL; 2650 struct socket *old_so; 2651 mbuf_t recv_data = NULL; 2652 2653 socket_unlock(so, 0); 2654 2655 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id); 2656 2657 /* Find the flow divert control block */ 2658 lck_rw_lock_shared(&g_flow_divert_group_lck); 2659 if (g_flow_divert_groups != NULL && g_active_group_count > 0) { 2660 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit]; 2661 if (group != NULL) { 2662 fd_cb = flow_divert_pcb_lookup(flow_id, group); 2663 } 2664 } 2665 lck_rw_done(&g_flow_divert_group_lck); 2666 2667 if (fd_cb == NULL) { 2668 error = ENOENT; 2669 goto done; 2670 } 2671 2672 FDLOCK(fd_cb); 2673 2674 /* Dis-associate the flow divert control block from its current socket */ 2675 old_so = fd_cb->so; 2676 2677 inp = sotoinpcb(old_so); 2678 2679 VERIFY(inp != NULL); 2680 2681 socket_lock(old_so, 0); 2682 soisdisconnected(old_so); 2683 old_so->so_flags &= ~SOF_FLOW_DIVERT; 2684 old_so->so_fd_pcb = NULL; 2685 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM); 2686 fd_cb->so = NULL; 2687 /* Save the output interface */ 2688 ifp = inp->inp_last_outifp; 2689 if (old_so->so_rcv.sb_cc > 0) { 2690 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data); 2691 sbflush(&old_so->so_rcv); 2692 } 2693 socket_unlock(old_so, 0); 2694 2695 /* Associate the new socket with the flow divert control block */ 2696 socket_lock(so, 0); 2697 so->so_fd_pcb = fd_cb; 2698 inp = sotoinpcb(so); 2699 inp->inp_last_outifp = ifp; 2700 if (recv_data != NULL) { 2701 if (sbappendstream(&so->so_rcv, recv_data)) { 2702 sorwakeup(so); 2703 } 2704 } 2705 flow_divert_set_protosw(so); 2706 socket_unlock(so, 0); 2707 2708 fd_cb->so = so; 2709 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED; 2710 2711 FDUNLOCK(fd_cb); 2712 2713done: 2714 socket_lock(so, 0); 2715 2716 if (fd_cb != NULL) { 2717 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */ 2718 } 2719 2720 return error; 2721} 2722 2723errno_t 2724flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit) 2725{ 2726 errno_t error = 0; 2727 struct flow_divert_pcb *fd_cb; 2728 2729 if (so->so_flags & SOF_FLOW_DIVERT) { 2730 return EALREADY; 2731 } 2732 2733 fd_cb = flow_divert_pcb_create(so); 2734 if (fd_cb != NULL) { 2735 error = flow_divert_pcb_insert(fd_cb, ctl_unit); 2736 if (error) { 2737 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error); 2738 FDRELEASE(fd_cb); 2739 } else { 2740 fd_cb->log_level = LOG_NOTICE; 2741 fd_cb->control_group_unit = ctl_unit; 2742 so->so_fd_pcb = fd_cb; 2743 2744 flow_divert_set_protosw(so); 2745 2746 FDLOG0(LOG_INFO, fd_cb, "Created"); 2747 } 2748 } else { 2749 error = ENOMEM; 2750 } 2751 2752 return error; 2753} 2754 2755errno_t 2756flow_divert_token_set(struct socket *so, struct sockopt *sopt) 2757{ 2758 uint32_t ctl_unit = 0; 2759 uint32_t key_unit = 0; 2760 uint32_t flow_id = 0; 2761 int error = 0; 2762 mbuf_t token = NULL; 2763 2764 if (so->so_flags & SOF_FLOW_DIVERT) { 2765 error = EALREADY; 2766 goto done; 2767 } 2768 2769 if (g_init_result) { 2770 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result); 2771 error = ENOPROTOOPT; 2772 goto done; 2773 } 2774 2775 if (SOCK_TYPE(so) != SOCK_STREAM || 2776 SOCK_PROTO(so) != IPPROTO_TCP || 2777 (SOCK_DOM(so) != PF_INET 2778#if INET6 2779 && SOCK_DOM(so) != PF_INET6 2780#endif 2781 )) 2782 { 2783 error = EINVAL; 2784 goto done; 2785 } else { 2786 struct tcpcb *tp = sototcpcb(so); 2787 if (tp == NULL || tp->t_state != TCPS_CLOSED) { 2788 error = EINVAL; 2789 goto done; 2790 } 2791 } 2792 2793 error = soopt_getm(sopt, &token); 2794 if (error) { 2795 goto done; 2796 } 2797 2798 error = soopt_mcopyin(sopt, token); 2799 if (error) { 2800 goto done; 2801 } 2802 2803 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL); 2804 if (!error) { 2805 key_unit = ntohl(key_unit); 2806 } else if (error != ENOENT) { 2807 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error); 2808 goto done; 2809 } else { 2810 key_unit = 0; 2811 } 2812 2813 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL); 2814 if (error) { 2815 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error); 2816 goto done; 2817 } 2818 2819 /* A valid kernel control unit is required */ 2820 ctl_unit = ntohl(ctl_unit); 2821 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) { 2822 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit); 2823 error = EINVAL; 2824 goto done; 2825 } 2826 2827 socket_unlock(so, 0); 2828 error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit)); 2829 socket_lock(so, 0); 2830 2831 if (error) { 2832 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", error); 2833 goto done; 2834 } 2835 2836 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL); 2837 if (error && error != ENOENT) { 2838 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error); 2839 goto done; 2840 } 2841 2842 if (flow_id == 0) { 2843 error = flow_divert_pcb_init(so, ctl_unit); 2844 if (error == 0) { 2845 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2846 int log_level = LOG_NOTICE; 2847 2848 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, 2849 sizeof(log_level), &log_level, NULL); 2850 if (error == 0) { 2851 fd_cb->log_level = log_level; 2852 } 2853 error = 0; 2854 2855 fd_cb->connect_token = token; 2856 token = NULL; 2857 } 2858 } else { 2859 error = flow_divert_attach(so, flow_id, ctl_unit); 2860 } 2861 2862done: 2863 if (token != NULL) { 2864 mbuf_freem(token); 2865 } 2866 2867 return error; 2868} 2869 2870errno_t 2871flow_divert_token_get(struct socket *so, struct sockopt *sopt) 2872{ 2873 uint32_t ctl_unit; 2874 int error = 0; 2875 uint8_t hmac[SHA_DIGEST_LENGTH]; 2876 struct flow_divert_pcb *fd_cb = so->so_fd_pcb; 2877 mbuf_t token = NULL; 2878 struct flow_divert_group *control_group = NULL; 2879 2880 if (!(so->so_flags & SOF_FLOW_DIVERT)) { 2881 error = EINVAL; 2882 goto done; 2883 } 2884 2885 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL); 2886 2887 if (fd_cb->group == NULL) { 2888 error = EINVAL; 2889 goto done; 2890 } 2891 2892 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token); 2893 if (error) { 2894 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error); 2895 goto done; 2896 } 2897 2898 ctl_unit = htonl(fd_cb->group->ctl_unit); 2899 2900 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit); 2901 if (error) { 2902 goto done; 2903 } 2904 2905 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash); 2906 if (error) { 2907 goto done; 2908 } 2909 2910 socket_unlock(so, 0); 2911 lck_rw_lock_shared(&g_flow_divert_group_lck); 2912 2913 if (g_flow_divert_groups != NULL && g_active_group_count > 0 && 2914 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) 2915 { 2916 control_group = g_flow_divert_groups[fd_cb->control_group_unit]; 2917 } 2918 2919 if (control_group != NULL) { 2920 lck_rw_lock_shared(&control_group->lck); 2921 ctl_unit = htonl(control_group->ctl_unit); 2922 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit); 2923 if (!error) { 2924 error = flow_divert_packet_compute_hmac(token, control_group, hmac); 2925 } 2926 lck_rw_done(&control_group->lck); 2927 } else { 2928 error = ENOPROTOOPT; 2929 } 2930 2931 lck_rw_done(&g_flow_divert_group_lck); 2932 socket_lock(so, 0); 2933 2934 if (error) { 2935 goto done; 2936 } 2937 2938 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac); 2939 if (error) { 2940 goto done; 2941 } 2942 2943 error = soopt_mcopyout(sopt, token); 2944 if (error) { 2945 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */ 2946 goto done; 2947 } 2948 2949done: 2950 if (token != NULL) { 2951 mbuf_freem(token); 2952 } 2953 2954 return error; 2955} 2956 2957static errno_t 2958flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo) 2959{ 2960 struct flow_divert_group *new_group; 2961 int error = 0; 2962 2963 if (sac->sc_unit >= GROUP_COUNT_MAX) { 2964 error = EINVAL; 2965 goto done; 2966 } 2967 2968 *unitinfo = NULL; 2969 2970 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK); 2971 if (new_group == NULL) { 2972 error = ENOBUFS; 2973 goto done; 2974 } 2975 2976 memset(new_group, 0, sizeof(*new_group)); 2977 2978 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr); 2979 RB_INIT(&new_group->pcb_tree); 2980 new_group->ctl_unit = sac->sc_unit; 2981 MBUFQ_INIT(&new_group->send_queue); 2982 2983 lck_rw_lock_exclusive(&g_flow_divert_group_lck); 2984 2985 if (g_flow_divert_groups == NULL) { 2986 MALLOC(g_flow_divert_groups, 2987 struct flow_divert_group **, 2988 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *), 2989 M_TEMP, 2990 M_WAITOK | M_ZERO); 2991 } 2992 2993 if (g_flow_divert_groups == NULL) { 2994 error = ENOBUFS; 2995 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) { 2996 error = EALREADY; 2997 } else { 2998 g_flow_divert_groups[sac->sc_unit] = new_group; 2999 g_active_group_count++; 3000 } 3001 3002 lck_rw_done(&g_flow_divert_group_lck); 3003 3004 *unitinfo = new_group; 3005 3006done: 3007 if (error != 0 && new_group != NULL) { 3008 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP); 3009 } 3010 return error; 3011} 3012 3013static errno_t 3014flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo) 3015{ 3016 struct flow_divert_group *group = NULL; 3017 errno_t error = 0; 3018 uint16_t node = 0; 3019 3020 if (unit >= GROUP_COUNT_MAX) { 3021 return EINVAL; 3022 } 3023 3024 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit); 3025 3026 lck_rw_lock_exclusive(&g_flow_divert_group_lck); 3027 3028 if (g_flow_divert_groups == NULL || g_active_group_count == 0) { 3029 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit, 3030 g_flow_divert_groups, g_active_group_count); 3031 } 3032 3033 group = g_flow_divert_groups[unit]; 3034 3035 if (group != (struct flow_divert_group *)unitinfo) { 3036 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo); 3037 } 3038 3039 if (group != NULL) { 3040 flow_divert_close_all(group); 3041 if (group->token_key != NULL) { 3042 memset(group->token_key, 0, group->token_key_size); 3043 FREE(group->token_key, M_TEMP); 3044 group->token_key = NULL; 3045 group->token_key_size = 0; 3046 } 3047 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP); 3048 g_flow_divert_groups[unit] = NULL; 3049 g_active_group_count--; 3050 } else { 3051 error = EINVAL; 3052 } 3053 3054 if (g_active_group_count == 0) { 3055 FREE(g_flow_divert_groups, M_TEMP); 3056 g_flow_divert_groups = NULL; 3057 } 3058 3059 /* Remove all signing IDs that point to this unit */ 3060 for (node = 0; node < g_signing_id_trie.nodes_count; node++) { 3061 if (TRIE_NODE(&g_signing_id_trie, node).group_unit == unit) { 3062 TRIE_NODE(&g_signing_id_trie, node).group_unit = 0; 3063 } 3064 } 3065 3066 lck_rw_done(&g_flow_divert_group_lck); 3067 3068 return error; 3069} 3070 3071static errno_t 3072flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused) 3073{ 3074 return flow_divert_input(m, (struct flow_divert_group *)unitinfo); 3075} 3076 3077static void 3078flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused) 3079{ 3080 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo; 3081 3082 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) { 3083 struct flow_divert_pcb *fd_cb; 3084 SLIST_HEAD(, flow_divert_pcb) tmp_list; 3085 3086 lck_rw_lock_shared(&g_flow_divert_group_lck); 3087 lck_rw_lock_exclusive(&group->lck); 3088 3089 while (!MBUFQ_EMPTY(&group->send_queue)) { 3090 mbuf_t next_packet; 3091 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again"); 3092 next_packet = MBUFQ_FIRST(&group->send_queue); 3093 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR); 3094 if (error) { 3095 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error); 3096 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits); 3097 lck_rw_done(&group->lck); 3098 lck_rw_done(&g_flow_divert_group_lck); 3099 return; 3100 } 3101 MBUFQ_DEQUEUE(&group->send_queue, next_packet); 3102 } 3103 3104 SLIST_INIT(&tmp_list); 3105 3106 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) { 3107 FDRETAIN(fd_cb); 3108 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry); 3109 } 3110 3111 lck_rw_done(&group->lck); 3112 3113 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) { 3114 FDLOCK(fd_cb); 3115 if (fd_cb->so != NULL) { 3116 socket_lock(fd_cb->so, 0); 3117 if (fd_cb->group != NULL) { 3118 flow_divert_send_buffered_data(fd_cb, FALSE); 3119 } 3120 socket_unlock(fd_cb->so, 0); 3121 } 3122 FDUNLOCK(fd_cb); 3123 FDRELEASE(fd_cb); 3124 } 3125 3126 lck_rw_done(&g_flow_divert_group_lck); 3127 } 3128} 3129 3130static int 3131flow_divert_kctl_init(void) 3132{ 3133 struct kern_ctl_reg ctl_reg; 3134 int result; 3135 3136 memset(&ctl_reg, 0, sizeof(ctl_reg)); 3137 3138 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name)); 3139 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0'; 3140 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; 3141 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE; 3142 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE; 3143 3144 ctl_reg.ctl_connect = flow_divert_kctl_connect; 3145 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect; 3146 ctl_reg.ctl_send = flow_divert_kctl_send; 3147 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd; 3148 3149 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref); 3150 3151 if (result) { 3152 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result); 3153 return result; 3154 } 3155 3156 return 0; 3157} 3158 3159void 3160flow_divert_init(void) 3161{ 3162 memset(&nil_pcb, 0, sizeof(nil_pcb)); 3163 nil_pcb.log_level = LOG_NOTICE; 3164 3165 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM); 3166 3167 VERIFY(g_tcp_protosw != NULL); 3168 3169 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw)); 3170 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs)); 3171 3172 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out; 3173 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out; 3174 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control; 3175 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close; 3176 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx; 3177 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername; 3178 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd; 3179 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out; 3180 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown; 3181 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr; 3182 3183 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs; 3184 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput; 3185 3186 /* 3187 * Socket filters shouldn't attach/detach to/from this protosw 3188 * since pr_protosw is to be used instead, which points to the 3189 * real protocol; if they do, it is a bug and we should panic. 3190 */ 3191 g_flow_divert_in_protosw.pr_filter_head.tqh_first = 3192 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef; 3193 g_flow_divert_in_protosw.pr_filter_head.tqh_last = 3194 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef; 3195 3196#if INET6 3197 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM); 3198 3199 VERIFY(g_tcp6_protosw != NULL); 3200 3201 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw)); 3202 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs)); 3203 3204 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out; 3205 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out; 3206 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control; 3207 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close; 3208 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx; 3209 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername; 3210 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd; 3211 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out; 3212 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown; 3213 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr; 3214 3215 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs; 3216 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput; 3217 /* 3218 * Socket filters shouldn't attach/detach to/from this protosw 3219 * since pr_protosw is to be used instead, which points to the 3220 * real protocol; if they do, it is a bug and we should panic. 3221 */ 3222 g_flow_divert_in6_protosw.pr_filter_head.tqh_first = 3223 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef; 3224 g_flow_divert_in6_protosw.pr_filter_head.tqh_last = 3225 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef; 3226#endif /* INET6 */ 3227 3228 flow_divert_grp_attr = lck_grp_attr_alloc_init(); 3229 if (flow_divert_grp_attr == NULL) { 3230 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed"); 3231 g_init_result = ENOMEM; 3232 goto done; 3233 } 3234 3235 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr); 3236 if (flow_divert_mtx_grp == NULL) { 3237 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed"); 3238 g_init_result = ENOMEM; 3239 goto done; 3240 } 3241 3242 flow_divert_mtx_attr = lck_attr_alloc_init(); 3243 if (flow_divert_mtx_attr == NULL) { 3244 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed"); 3245 g_init_result = ENOMEM; 3246 goto done; 3247 } 3248 3249 g_init_result = flow_divert_kctl_init(); 3250 if (g_init_result) { 3251 goto done; 3252 } 3253 3254 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr); 3255 3256 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie)); 3257 g_signing_id_trie.root = NULL_TRIE_IDX; 3258 3259done: 3260 if (g_init_result != 0) { 3261 if (flow_divert_mtx_attr != NULL) { 3262 lck_attr_free(flow_divert_mtx_attr); 3263 flow_divert_mtx_attr = NULL; 3264 } 3265 if (flow_divert_mtx_grp != NULL) { 3266 lck_grp_free(flow_divert_mtx_grp); 3267 flow_divert_mtx_grp = NULL; 3268 } 3269 if (flow_divert_grp_attr != NULL) { 3270 lck_grp_attr_free(flow_divert_grp_attr); 3271 flow_divert_grp_attr = NULL; 3272 } 3273 3274 if (g_flow_divert_kctl_ref != NULL) { 3275 ctl_deregister(g_flow_divert_kctl_ref); 3276 g_flow_divert_kctl_ref = NULL; 3277 } 3278 } 3279} 3280