1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce 58 * support for mandatory and extensible security protections. This notice 59 * is included in support of clause 2.2 (b) of the Apple Public License, 60 * Version 2.0. 61 * Copyright (c) 2005 SPARTA, Inc. 62 */ 63/* 64 */ 65/* 66 * File: ipc/ipc_kmsg.c 67 * Author: Rich Draves 68 * Date: 1989 69 * 70 * Operations on kernel messages. 71 */ 72 73 74#include <mach/mach_types.h> 75#include <mach/boolean.h> 76#include <mach/kern_return.h> 77#include <mach/message.h> 78#include <mach/port.h> 79#include <mach/vm_map.h> 80#include <mach/mach_vm.h> 81#include <mach/vm_statistics.h> 82 83#include <kern/kern_types.h> 84#include <kern/assert.h> 85#include <kern/debug.h> 86#include <kern/ipc_kobject.h> 87#include <kern/kalloc.h> 88#include <kern/zalloc.h> 89#include <kern/processor.h> 90#include <kern/thread.h> 91#include <kern/sched_prim.h> 92#include <kern/spl.h> 93#include <kern/misc_protos.h> 94#include <kern/counters.h> 95#include <kern/cpu_data.h> 96 97#include <machine/machlimits.h> 98 99#include <vm/vm_map.h> 100#include <vm/vm_object.h> 101#include <vm/vm_kern.h> 102 103#include <ipc/port.h> 104#include <ipc/ipc_types.h> 105#include <ipc/ipc_entry.h> 106#include <ipc/ipc_kmsg.h> 107#include <ipc/ipc_notify.h> 108#include <ipc/ipc_object.h> 109#include <ipc/ipc_space.h> 110#include <ipc/ipc_port.h> 111#include <ipc/ipc_right.h> 112#include <ipc/ipc_hash.h> 113#include <ipc/ipc_table.h> 114#include <ipc/ipc_importance.h> 115 116#include <security/mac_mach_internal.h> 117 118#include <string.h> 119 120#ifdef ppc 121#include <ppc/Firmware.h> 122#include <ppc/low_trace.h> 123#endif 124 125#if DEBUG 126#define DEBUG_MSGS_K64 1 127#endif 128 129#include <sys/kdebug.h> 130#include <libkern/OSAtomic.h> 131 132#pragma pack(4) 133 134typedef struct 135{ 136 mach_msg_bits_t msgh_bits; 137 mach_msg_size_t msgh_size; 138 mach_port_name_t msgh_remote_port; 139 mach_port_name_t msgh_local_port; 140 mach_port_name_t msgh_voucher_port; 141 mach_msg_id_t msgh_id; 142} mach_msg_legacy_header_t; 143 144typedef struct 145{ 146 mach_msg_legacy_header_t header; 147 mach_msg_body_t body; 148} mach_msg_legacy_base_t; 149 150typedef struct 151{ 152 mach_port_name_t name; 153 mach_msg_size_t pad1; 154 uint32_t pad2 : 16; 155 mach_msg_type_name_t disposition : 8; 156 mach_msg_descriptor_type_t type : 8; 157} mach_msg_legacy_port_descriptor_t; 158 159 160typedef union 161{ 162 mach_msg_legacy_port_descriptor_t port; 163 mach_msg_ool_descriptor32_t out_of_line32; 164 mach_msg_ool_ports_descriptor32_t ool_ports32; 165 mach_msg_type_descriptor_t type; 166} mach_msg_legacy_descriptor_t; 167 168#pragma pack() 169 170#define LEGACY_HEADER_SIZE_DELTA ((mach_msg_size_t)(sizeof(mach_msg_header_t) - sizeof(mach_msg_legacy_header_t))) 171 172// END LP64 fixes 173 174 175#if DEBUG_MSGS_K64 176extern void ipc_pset_print64( 177 ipc_pset_t pset); 178 179extern void ipc_kmsg_print64( 180 ipc_kmsg_t kmsg, 181 const char *str); 182 183extern void ipc_msg_print64( 184 mach_msg_header_t *msgh); 185 186extern ipc_port_t ipc_name_to_data64( 187 task_t task, 188 mach_port_name_t name); 189 190/* 191 * Forward declarations 192 */ 193void ipc_msg_print_untyped64( 194 mach_msg_body_t *body); 195 196const char * ipc_type_name64( 197 int type_name, 198 boolean_t received); 199 200void ipc_print_type_name64( 201 int type_name); 202 203const char * 204msgh_bit_decode64( 205 mach_msg_bits_t bit); 206 207const char * 208mm_copy_options_string64( 209 mach_msg_copy_options_t option); 210 211void db_print_msg_uid64(mach_msg_header_t *); 212 213static void 214ipc_msg_body_print64(void *body, int size) 215{ 216 uint32_t *word = (uint32_t *) body; 217 uint32_t *end = (uint32_t *)(((uintptr_t) body) + size 218 - sizeof(mach_msg_header_t)); 219 int i; 220 221 kprintf(" body(%p-%p):\n %p: ", body, end, word); 222 for (;;) { 223 for (i = 0; i < 8; i++, word++) { 224 if (word >= end) { 225 kprintf("\n"); 226 return; 227 } 228 kprintf("%08x ", *word); 229 } 230 kprintf("\n %p: ", word); 231 } 232} 233 234 235const char * 236ipc_type_name64( 237 int type_name, 238 boolean_t received) 239{ 240 switch (type_name) { 241 case MACH_MSG_TYPE_PORT_NAME: 242 return "port_name"; 243 244 case MACH_MSG_TYPE_MOVE_RECEIVE: 245 if (received) { 246 return "port_receive"; 247 } else { 248 return "move_receive"; 249 } 250 251 case MACH_MSG_TYPE_MOVE_SEND: 252 if (received) { 253 return "port_send"; 254 } else { 255 return "move_send"; 256 } 257 258 case MACH_MSG_TYPE_MOVE_SEND_ONCE: 259 if (received) { 260 return "port_send_once"; 261 } else { 262 return "move_send_once"; 263 } 264 265 case MACH_MSG_TYPE_COPY_SEND: 266 return "copy_send"; 267 268 case MACH_MSG_TYPE_MAKE_SEND: 269 return "make_send"; 270 271 case MACH_MSG_TYPE_MAKE_SEND_ONCE: 272 return "make_send_once"; 273 274 default: 275 return (char *) 0; 276 } 277} 278 279void 280ipc_print_type_name64( 281 int type_name) 282{ 283 const char *name = ipc_type_name64(type_name, TRUE); 284 if (name) { 285 kprintf("%s", name); 286 } else { 287 kprintf("type%d", type_name); 288 } 289} 290 291/* 292 * ipc_kmsg_print64 [ debug ] 293 */ 294void 295ipc_kmsg_print64( 296 ipc_kmsg_t kmsg, 297 const char *str) 298{ 299 kprintf("%s kmsg=%p:\n", str, kmsg); 300 kprintf(" next=%p, prev=%p, size=%d", 301 kmsg->ikm_next, 302 kmsg->ikm_prev, 303 kmsg->ikm_size); 304 kprintf("\n"); 305 ipc_msg_print64(kmsg->ikm_header); 306} 307 308const char * 309msgh_bit_decode64( 310 mach_msg_bits_t bit) 311{ 312 switch (bit) { 313 case MACH_MSGH_BITS_COMPLEX: return "complex"; 314 case MACH_MSGH_BITS_CIRCULAR: return "circular"; 315 default: return (char *) 0; 316 } 317} 318 319/* 320 * ipc_msg_print64 [ debug ] 321 */ 322void 323ipc_msg_print64( 324 mach_msg_header_t *msgh) 325{ 326 mach_msg_bits_t mbits; 327 unsigned int bit, i; 328 const char *bit_name; 329 int needs_comma; 330 331 mbits = msgh->msgh_bits; 332 kprintf(" msgh_bits=0x%x: l=0x%x,r=0x%x\n", 333 mbits, 334 MACH_MSGH_BITS_LOCAL(msgh->msgh_bits), 335 MACH_MSGH_BITS_REMOTE(msgh->msgh_bits)); 336 337 mbits = MACH_MSGH_BITS_OTHER(mbits) & MACH_MSGH_BITS_USED; 338 kprintf(" decoded bits: "); 339 needs_comma = 0; 340 for (i = 0, bit = 1; i < sizeof(mbits) * 8; ++i, bit <<= 1) { 341 if ((mbits & bit) == 0) 342 continue; 343 bit_name = msgh_bit_decode64((mach_msg_bits_t)bit); 344 if (bit_name) 345 kprintf("%s%s", needs_comma ? "," : "", bit_name); 346 else 347 kprintf("%sunknown(0x%x),", needs_comma ? "," : "", bit); 348 ++needs_comma; 349 } 350 if (msgh->msgh_bits & ~MACH_MSGH_BITS_USED) { 351 kprintf("%sunused=0x%x,", needs_comma ? "," : "", 352 msgh->msgh_bits & ~MACH_MSGH_BITS_USED); 353 } 354 kprintf("\n"); 355 356 needs_comma = 1; 357 if (msgh->msgh_remote_port) { 358 kprintf(" remote=%p(", msgh->msgh_remote_port); 359 ipc_print_type_name64(MACH_MSGH_BITS_REMOTE(msgh->msgh_bits)); 360 kprintf(")"); 361 } else { 362 kprintf(" remote=null"); 363 } 364 365 if (msgh->msgh_local_port) { 366 kprintf("%slocal=%p(", needs_comma ? "," : "", 367 msgh->msgh_local_port); 368 ipc_print_type_name64(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits)); 369 kprintf(")\n"); 370 } else { 371 kprintf("local=null\n"); 372 } 373 374 kprintf(" msgh_id=%d, size=%d\n", 375 msgh->msgh_id, 376 msgh->msgh_size); 377 378 if (mbits & MACH_MSGH_BITS_COMPLEX) { 379 ipc_msg_print_untyped64((mach_msg_body_t *) (msgh + 1)); 380 } 381 382 ipc_msg_body_print64((void *)(msgh + 1), msgh->msgh_size); 383} 384 385 386const char * 387mm_copy_options_string64( 388 mach_msg_copy_options_t option) 389{ 390 const char *name; 391 392 switch (option) { 393 case MACH_MSG_PHYSICAL_COPY: 394 name = "PHYSICAL"; 395 break; 396 case MACH_MSG_VIRTUAL_COPY: 397 name = "VIRTUAL"; 398 break; 399 case MACH_MSG_OVERWRITE: 400 name = "OVERWRITE"; 401 break; 402 case MACH_MSG_ALLOCATE: 403 name = "ALLOCATE"; 404 break; 405 case MACH_MSG_KALLOC_COPY_T: 406 name = "KALLOC_COPY_T"; 407 break; 408 default: 409 name = "unknown"; 410 break; 411 } 412 return name; 413} 414 415void 416ipc_msg_print_untyped64( 417 mach_msg_body_t *body) 418{ 419 mach_msg_descriptor_t *saddr, *send; 420 mach_msg_descriptor_type_t type; 421 422 kprintf(" %d descriptors: \n", body->msgh_descriptor_count); 423 424 saddr = (mach_msg_descriptor_t *) (body + 1); 425 send = saddr + body->msgh_descriptor_count; 426 427 for ( ; saddr < send; saddr++ ) { 428 429 type = saddr->type.type; 430 431 switch (type) { 432 433 case MACH_MSG_PORT_DESCRIPTOR: { 434 mach_msg_port_descriptor_t *dsc; 435 436 dsc = &saddr->port; 437 kprintf(" PORT name = %p disp = ", dsc->name); 438 ipc_print_type_name64(dsc->disposition); 439 kprintf("\n"); 440 break; 441 } 442 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 443 case MACH_MSG_OOL_DESCRIPTOR: { 444 mach_msg_ool_descriptor_t *dsc; 445 446 dsc = (mach_msg_ool_descriptor_t *) &saddr->out_of_line; 447 kprintf(" OOL%s addr = %p size = 0x%x copy = %s %s\n", 448 type == MACH_MSG_OOL_DESCRIPTOR ? "" : " VOLATILE", 449 dsc->address, dsc->size, 450 mm_copy_options_string64(dsc->copy), 451 dsc->deallocate ? "DEALLOC" : ""); 452 break; 453 } 454 case MACH_MSG_OOL_PORTS_DESCRIPTOR : { 455 mach_msg_ool_ports_descriptor_t *dsc; 456 457 dsc = (mach_msg_ool_ports_descriptor_t *) &saddr->ool_ports; 458 459 kprintf(" OOL_PORTS addr = %p count = 0x%x ", 460 dsc->address, dsc->count); 461 kprintf("disp = "); 462 ipc_print_type_name64(dsc->disposition); 463 kprintf(" copy = %s %s\n", 464 mm_copy_options_string64(dsc->copy), 465 dsc->deallocate ? "DEALLOC" : ""); 466 break; 467 } 468 469 default: { 470 kprintf(" UNKNOWN DESCRIPTOR 0x%x\n", type); 471 break; 472 } 473 } 474 } 475} 476 477#define DEBUG_IPC_KMSG_PRINT(kmsg,string) \ 478 __unreachable_ok_push \ 479 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ 480 ipc_kmsg_print64(kmsg, string); \ 481 } \ 482 __unreachable_ok_pop 483 484#define DEBUG_IPC_MSG_BODY_PRINT(body,size) \ 485 __unreachable_ok_push \ 486 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { \ 487 ipc_msg_body_print64(body,size);\ 488 } \ 489 __unreachable_ok_pop 490#else /* !DEBUG_MSGS_K64 */ 491#define DEBUG_IPC_KMSG_PRINT(kmsg,string) 492#define DEBUG_IPC_MSG_BODY_PRINT(body,size) 493#endif /* !DEBUG_MSGS_K64 */ 494 495extern vm_map_t ipc_kernel_copy_map; 496extern vm_size_t ipc_kmsg_max_space; 497extern vm_size_t ipc_kmsg_max_vm_space; 498extern vm_size_t ipc_kmsg_max_body_space; 499extern vm_size_t msg_ool_size_small; 500 501#define MSG_OOL_SIZE_SMALL msg_ool_size_small 502 503#if defined(__LP64__) 504#define MAP_SIZE_DIFFERS(map) (map->max_offset < MACH_VM_MAX_ADDRESS) 505#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor32_t 506#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor32_t 507#else 508#define MAP_SIZE_DIFFERS(map) (map->max_offset > VM_MAX_ADDRESS) 509#define OTHER_OOL_DESCRIPTOR mach_msg_ool_descriptor64_t 510#define OTHER_OOL_PORTS_DESCRIPTOR mach_msg_ool_ports_descriptor64_t 511#endif 512 513#define DESC_SIZE_ADJUSTMENT ((mach_msg_size_t)(sizeof(mach_msg_ool_descriptor64_t) - \ 514 sizeof(mach_msg_ool_descriptor32_t))) 515 516/* scatter list macros */ 517 518#define SKIP_PORT_DESCRIPTORS(s, c) \ 519MACRO_BEGIN \ 520 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \ 521 while ((c) > 0) { \ 522 if ((s)->type.type != MACH_MSG_PORT_DESCRIPTOR) \ 523 break; \ 524 (s)++; (c)--; \ 525 } \ 526 if (c == 0) \ 527 (s) = MACH_MSG_DESCRIPTOR_NULL; \ 528 } \ 529MACRO_END 530 531#define INCREMENT_SCATTER(s, c, d) \ 532MACRO_BEGIN \ 533 if ((s) != MACH_MSG_DESCRIPTOR_NULL) { \ 534 s = (d) ? (mach_msg_descriptor_t *) \ 535 ((OTHER_OOL_DESCRIPTOR *)(s) + 1) : \ 536 (s + 1); \ 537 (c)--; \ 538 } \ 539MACRO_END 540 541/* zone for cached ipc_kmsg_t structures */ 542zone_t ipc_kmsg_zone; 543 544/* 545 * Forward declarations 546 */ 547 548void ipc_kmsg_clean( 549 ipc_kmsg_t kmsg); 550 551void ipc_kmsg_clean_body( 552 ipc_kmsg_t kmsg, 553 mach_msg_type_number_t number, 554 mach_msg_descriptor_t *desc); 555 556void ipc_kmsg_clean_partial( 557 ipc_kmsg_t kmsg, 558 mach_msg_type_number_t number, 559 mach_msg_descriptor_t *desc, 560 vm_offset_t paddr, 561 vm_size_t length); 562 563mach_msg_return_t ipc_kmsg_copyin_body( 564 ipc_kmsg_t kmsg, 565 ipc_space_t space, 566 vm_map_t map); 567 568/* 569 * We keep a per-processor cache of kernel message buffers. 570 * The cache saves the overhead/locking of using kalloc/kfree. 571 * The per-processor cache seems to miss less than a per-thread cache, 572 * and it also uses less memory. Access to the cache doesn't 573 * require locking. 574 */ 575 576/* 577 * Routine: ipc_kmsg_alloc 578 * Purpose: 579 * Allocate a kernel message structure. If we can get one from 580 * the cache, that is best. Otherwise, allocate a new one. 581 * Conditions: 582 * Nothing locked. 583 */ 584ipc_kmsg_t 585ipc_kmsg_alloc( 586 mach_msg_size_t msg_and_trailer_size) 587{ 588 mach_msg_size_t max_expanded_size; 589 ipc_kmsg_t kmsg; 590 591 /* 592 * LP64support - 593 * Pad the allocation in case we need to expand the 594 * message descrptors for user spaces with pointers larger than 595 * the kernel's own, or vice versa. We don't know how many descriptors 596 * there are yet, so just assume the whole body could be 597 * descriptors (if there could be any at all). 598 * 599 * The expansion space is left in front of the header, 600 * because it is easier to pull the header and descriptors 601 * forward as we process them than it is to push all the 602 * data backwards. 603 */ 604 mach_msg_size_t size = msg_and_trailer_size - MAX_TRAILER_SIZE; 605 606 /* compare against implementation upper limit for the body */ 607 if (size > ipc_kmsg_max_body_space) 608 return IKM_NULL; 609 610 if (size > sizeof(mach_msg_base_t)) { 611 mach_msg_size_t max_desc = (mach_msg_size_t)(((size - sizeof(mach_msg_base_t)) / 612 sizeof(mach_msg_ool_descriptor32_t)) * 613 DESC_SIZE_ADJUSTMENT); 614 615 /* make sure expansion won't cause wrap */ 616 if (msg_and_trailer_size > MACH_MSG_SIZE_MAX - max_desc) 617 return IKM_NULL; 618 619 max_expanded_size = msg_and_trailer_size + max_desc; 620 } else 621 max_expanded_size = msg_and_trailer_size; 622 623 if (max_expanded_size < IKM_SAVED_MSG_SIZE) 624 max_expanded_size = IKM_SAVED_MSG_SIZE; /* round up for ikm_cache */ 625 626 if (max_expanded_size == IKM_SAVED_MSG_SIZE) { 627 struct ikm_cache *cache; 628 unsigned int i; 629 630 disable_preemption(); 631 cache = &PROCESSOR_DATA(current_processor(), ikm_cache); 632 if ((i = cache->avail) > 0) { 633 assert(i <= IKM_STASH); 634 kmsg = cache->entries[--i]; 635 cache->avail = i; 636 enable_preemption(); 637 ikm_check_init(kmsg, max_expanded_size); 638 ikm_set_header(kmsg, msg_and_trailer_size); 639 return (kmsg); 640 } 641 enable_preemption(); 642 kmsg = (ipc_kmsg_t)zalloc(ipc_kmsg_zone); 643 } else { 644 kmsg = (ipc_kmsg_t)kalloc(ikm_plus_overhead(max_expanded_size)); 645 } 646 647 if (kmsg != IKM_NULL) { 648 ikm_init(kmsg, max_expanded_size); 649 ikm_set_header(kmsg, msg_and_trailer_size); 650 } 651 652 return(kmsg); 653} 654 655/* 656 * Routine: ipc_kmsg_free 657 * Purpose: 658 * Free a kernel message buffer. If the kms is preallocated 659 * to a port, just "put it back (marked unused)." We have to 660 * do this with the port locked. The port may have its hold 661 * on our message released. In that case, we have to just 662 * revert the message to a traditional one and free it normally. 663 * Conditions: 664 * Nothing locked. 665 */ 666 667void 668ipc_kmsg_free( 669 ipc_kmsg_t kmsg) 670{ 671 mach_msg_size_t size = kmsg->ikm_size; 672 ipc_port_t port; 673 674 assert(!IP_VALID(kmsg->ikm_voucher)); 675 676 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_KMSG_FREE) | DBG_FUNC_NONE, 677 VM_KERNEL_ADDRPERM((uintptr_t)kmsg), 678 0, 0, 0, 0); 679 680 /* 681 * Check to see if the message is bound to the port. If so, 682 * mark it not in use. If the port isn't already dead, then 683 * leave the message associated with it. Otherwise, free it. 684 */ 685 port = ikm_prealloc_inuse_port(kmsg); 686 if (port != IP_NULL) { 687 ip_lock(port); 688 ikm_prealloc_clear_inuse(kmsg, port); 689 if (ip_active(port) && (port->ip_premsg == kmsg)) { 690 assert(IP_PREALLOC(port)); 691 ip_unlock(port); 692 ip_release(port); 693 return; 694 } 695 ip_unlock(port); 696 ip_release(port); /* May be last reference */ 697 } 698 699 /* 700 * Peek and see if it has to go back in the cache. 701 */ 702 if (kmsg->ikm_size == IKM_SAVED_MSG_SIZE) { 703 struct ikm_cache *cache; 704 unsigned int i; 705 706 disable_preemption(); 707 cache = &PROCESSOR_DATA(current_processor(), ikm_cache); 708 if ((i = cache->avail) < IKM_STASH) { 709 cache->entries[i] = kmsg; 710 cache->avail = i + 1; 711 enable_preemption(); 712 return; 713 } 714 enable_preemption(); 715 zfree(ipc_kmsg_zone, kmsg); 716 return; 717 } 718 kfree(kmsg, ikm_plus_overhead(size)); 719} 720 721 722/* 723 * Routine: ipc_kmsg_enqueue 724 * Purpose: 725 * Enqueue a kmsg. 726 */ 727 728void 729ipc_kmsg_enqueue( 730 ipc_kmsg_queue_t queue, 731 ipc_kmsg_t kmsg) 732{ 733 ipc_kmsg_enqueue_macro(queue, kmsg); 734} 735 736/* 737 * Routine: ipc_kmsg_dequeue 738 * Purpose: 739 * Dequeue and return a kmsg. 740 */ 741 742ipc_kmsg_t 743ipc_kmsg_dequeue( 744 ipc_kmsg_queue_t queue) 745{ 746 ipc_kmsg_t first; 747 748 first = ipc_kmsg_queue_first(queue); 749 750 if (first != IKM_NULL) 751 ipc_kmsg_rmqueue_first_macro(queue, first); 752 753 return first; 754} 755 756/* 757 * Routine: ipc_kmsg_rmqueue 758 * Purpose: 759 * Pull a kmsg out of a queue. 760 */ 761 762void 763ipc_kmsg_rmqueue( 764 ipc_kmsg_queue_t queue, 765 ipc_kmsg_t kmsg) 766{ 767 ipc_kmsg_t next, prev; 768 769 assert(queue->ikmq_base != IKM_NULL); 770 771 next = kmsg->ikm_next; 772 prev = kmsg->ikm_prev; 773 774 if (next == kmsg) { 775 assert(prev == kmsg); 776 assert(queue->ikmq_base == kmsg); 777 778 queue->ikmq_base = IKM_NULL; 779 } else { 780 if (queue->ikmq_base == kmsg) 781 queue->ikmq_base = next; 782 783 next->ikm_prev = prev; 784 prev->ikm_next = next; 785 } 786 /* XXX Temporary debug logic */ 787 assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS); 788 assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS); 789} 790 791/* 792 * Routine: ipc_kmsg_queue_next 793 * Purpose: 794 * Return the kmsg following the given kmsg. 795 * (Or IKM_NULL if it is the last one in the queue.) 796 */ 797 798ipc_kmsg_t 799ipc_kmsg_queue_next( 800 ipc_kmsg_queue_t queue, 801 ipc_kmsg_t kmsg) 802{ 803 ipc_kmsg_t next; 804 805 assert(queue->ikmq_base != IKM_NULL); 806 807 next = kmsg->ikm_next; 808 if (queue->ikmq_base == next) 809 next = IKM_NULL; 810 811 return next; 812} 813 814/* 815 * Routine: ipc_kmsg_destroy 816 * Purpose: 817 * Destroys a kernel message. Releases all rights, 818 * references, and memory held by the message. 819 * Frees the message. 820 * Conditions: 821 * No locks held. 822 */ 823 824void 825ipc_kmsg_destroy( 826 ipc_kmsg_t kmsg) 827{ 828 /* 829 * Destroying a message can cause more messages to be destroyed. 830 * Curtail recursion by putting messages on the deferred 831 * destruction queue. If this was the first message on the 832 * queue, this instance must process the full queue. 833 */ 834 if (ipc_kmsg_delayed_destroy(kmsg)) 835 ipc_kmsg_reap_delayed(); 836} 837 838/* 839 * Routine: ipc_kmsg_delayed_destroy 840 * Purpose: 841 * Enqueues a kernel message for deferred destruction. 842 * Returns: 843 * Boolean indicator that the caller is responsible to reap 844 * deferred messages. 845 */ 846 847boolean_t ipc_kmsg_delayed_destroy( 848 ipc_kmsg_t kmsg) 849{ 850 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages); 851 boolean_t first = ipc_kmsg_queue_empty(queue); 852 853 ipc_kmsg_enqueue(queue, kmsg); 854 return first; 855} 856 857/* 858 * Routine: ipc_kmsg_destroy_queue 859 * Purpose: 860 * Destroys messages from the per-thread 861 * deferred reaping queue. 862 * Conditions: 863 * No locks held. 864 */ 865 866void 867ipc_kmsg_reap_delayed(void) 868{ 869 ipc_kmsg_queue_t queue = &(current_thread()->ith_messages); 870 ipc_kmsg_t kmsg; 871 872 /* 873 * must leave kmsg in queue while cleaning it to assure 874 * no nested calls recurse into here. 875 */ 876 while ((kmsg = ipc_kmsg_queue_first(queue)) != IKM_NULL) { 877 ipc_kmsg_clean(kmsg); 878 ipc_kmsg_rmqueue(queue, kmsg); 879 ipc_kmsg_free(kmsg); 880 } 881} 882 883/* 884 * Routine: ipc_kmsg_clean_body 885 * Purpose: 886 * Cleans the body of a kernel message. 887 * Releases all rights, references, and memory. 888 * 889 * Conditions: 890 * No locks held. 891 */ 892static unsigned int _ipc_kmsg_clean_invalid_desc = 0; 893void 894ipc_kmsg_clean_body( 895 __unused ipc_kmsg_t kmsg, 896 mach_msg_type_number_t number, 897 mach_msg_descriptor_t *saddr) 898{ 899 mach_msg_type_number_t i; 900 901 if ( number == 0 ) 902 return; 903 904 for (i = 0 ; i < number; i++, saddr++ ) { 905 906 switch (saddr->type.type) { 907 908 case MACH_MSG_PORT_DESCRIPTOR: { 909 mach_msg_port_descriptor_t *dsc; 910 911 dsc = &saddr->port; 912 913 /* 914 * Destroy port rights carried in the message 915 */ 916 if (!IO_VALID((ipc_object_t) dsc->name)) 917 continue; 918 ipc_object_destroy((ipc_object_t) dsc->name, dsc->disposition); 919 break; 920 } 921 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 922 case MACH_MSG_OOL_DESCRIPTOR : { 923 mach_msg_ool_descriptor_t *dsc; 924 925 dsc = (mach_msg_ool_descriptor_t *)&saddr->out_of_line; 926 927 /* 928 * Destroy memory carried in the message 929 */ 930 if (dsc->size == 0) { 931 assert(dsc->address == (void *) 0); 932 } else { 933 vm_map_copy_discard((vm_map_copy_t) dsc->address); 934 } 935 break; 936 } 937 case MACH_MSG_OOL_PORTS_DESCRIPTOR : { 938 ipc_object_t *objects; 939 mach_msg_type_number_t j; 940 mach_msg_ool_ports_descriptor_t *dsc; 941 942 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; 943 objects = (ipc_object_t *) dsc->address; 944 945 if (dsc->count == 0) { 946 break; 947 } 948 949 assert(objects != (ipc_object_t *) 0); 950 951 /* destroy port rights carried in the message */ 952 953 for (j = 0; j < dsc->count; j++) { 954 ipc_object_t object = objects[j]; 955 956 if (!IO_VALID(object)) 957 continue; 958 959 ipc_object_destroy(object, dsc->disposition); 960 } 961 962 /* destroy memory carried in the message */ 963 964 assert(dsc->count != 0); 965 966 kfree(dsc->address, 967 (vm_size_t) dsc->count * sizeof(mach_port_t)); 968 break; 969 } 970 default : { 971 _ipc_kmsg_clean_invalid_desc++; /* don't understand this type of descriptor */ 972 } 973 } 974 } 975} 976 977/* 978 * Routine: ipc_kmsg_clean_partial 979 * Purpose: 980 * Cleans a partially-acquired kernel message. 981 * number is the index of the type descriptor 982 * in the body of the message that contained the error. 983 * If dolast, the memory and port rights in this last 984 * type spec are also cleaned. In that case, number 985 * specifies the number of port rights to clean. 986 * Conditions: 987 * Nothing locked. 988 */ 989 990void 991ipc_kmsg_clean_partial( 992 ipc_kmsg_t kmsg, 993 mach_msg_type_number_t number, 994 mach_msg_descriptor_t *desc, 995 vm_offset_t paddr, 996 vm_size_t length) 997{ 998 ipc_object_t object; 999 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits; 1000 1001 /* deal with importance chain while we still have dest and voucher references */ 1002 ipc_importance_clean(kmsg); 1003 1004 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 1005 assert(IO_VALID(object)); 1006 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits)); 1007 1008 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 1009 if (IO_VALID(object)) 1010 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits)); 1011 1012 object = (ipc_object_t) kmsg->ikm_voucher; 1013 if (IO_VALID(object)) { 1014 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND); 1015 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND); 1016 kmsg->ikm_voucher = IP_NULL; 1017 } 1018 1019 if (paddr) { 1020 (void) vm_deallocate(ipc_kernel_copy_map, paddr, length); 1021 } 1022 1023 ipc_kmsg_clean_body(kmsg, number, desc); 1024} 1025 1026/* 1027 * Routine: ipc_kmsg_clean 1028 * Purpose: 1029 * Cleans a kernel message. Releases all rights, 1030 * references, and memory held by the message. 1031 * Conditions: 1032 * No locks held. 1033 */ 1034 1035void 1036ipc_kmsg_clean( 1037 ipc_kmsg_t kmsg) 1038{ 1039 ipc_object_t object; 1040 mach_msg_bits_t mbits; 1041 1042 /* deal with importance chain while we still have dest and voucher references */ 1043 ipc_importance_clean(kmsg); 1044 1045 mbits = kmsg->ikm_header->msgh_bits; 1046 object = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 1047 if (IO_VALID(object)) 1048 ipc_object_destroy_dest(object, MACH_MSGH_BITS_REMOTE(mbits)); 1049 1050 object = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 1051 if (IO_VALID(object)) 1052 ipc_object_destroy(object, MACH_MSGH_BITS_LOCAL(mbits)); 1053 1054 object = (ipc_object_t) kmsg->ikm_voucher; 1055 if (IO_VALID(object)) { 1056 assert(MACH_MSGH_BITS_VOUCHER(mbits) == MACH_MSG_TYPE_MOVE_SEND); 1057 ipc_object_destroy(object, MACH_MSG_TYPE_PORT_SEND); 1058 kmsg->ikm_voucher = IP_NULL; 1059 } 1060 1061 if (mbits & MACH_MSGH_BITS_COMPLEX) { 1062 mach_msg_body_t *body; 1063 1064 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 1065 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count, 1066 (mach_msg_descriptor_t *)(body + 1)); 1067 } 1068} 1069 1070/* 1071 * Routine: ipc_kmsg_set_prealloc 1072 * Purpose: 1073 * Assign a kmsg as a preallocated message buffer to a port. 1074 * Conditions: 1075 * port locked. 1076 */ 1077 1078void 1079ipc_kmsg_set_prealloc( 1080 ipc_kmsg_t kmsg, 1081 ipc_port_t port) 1082{ 1083 assert(kmsg->ikm_prealloc == IP_NULL); 1084 1085 kmsg->ikm_prealloc = IP_NULL; 1086 IP_SET_PREALLOC(port, kmsg); 1087} 1088 1089/* 1090 * Routine: ipc_kmsg_clear_prealloc 1091 * Purpose: 1092 * Release the Assignment of a preallocated message buffer from a port. 1093 * Conditions: 1094 * port locked. 1095 */ 1096void 1097ipc_kmsg_clear_prealloc( 1098 ipc_kmsg_t kmsg, 1099 ipc_port_t port) 1100{ 1101 assert(kmsg->ikm_prealloc == port); 1102 1103 kmsg->ikm_prealloc = IP_NULL; 1104 IP_CLEAR_PREALLOC(port, kmsg); 1105} 1106 1107/* 1108 * Routine: ipc_kmsg_prealloc 1109 * Purpose: 1110 * Wraper to ipc_kmsg_alloc() to account for 1111 * header expansion requirements. 1112 */ 1113ipc_kmsg_t 1114ipc_kmsg_prealloc(mach_msg_size_t size) 1115{ 1116#if defined(__LP64__) 1117 if (size > MACH_MSG_SIZE_MAX - LEGACY_HEADER_SIZE_DELTA) 1118 return IKM_NULL; 1119 1120 size += LEGACY_HEADER_SIZE_DELTA; 1121#endif 1122 return ipc_kmsg_alloc(size); 1123} 1124 1125 1126/* 1127 * Routine: ipc_kmsg_get 1128 * Purpose: 1129 * Allocates a kernel message buffer. 1130 * Copies a user message to the message buffer. 1131 * Conditions: 1132 * Nothing locked. 1133 * Returns: 1134 * MACH_MSG_SUCCESS Acquired a message buffer. 1135 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header. 1136 * MACH_SEND_MSG_TOO_SMALL Message size not long-word multiple. 1137 * MACH_SEND_TOO_LARGE Message too large to ever be sent. 1138 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer. 1139 * MACH_SEND_INVALID_DATA Couldn't copy message data. 1140 */ 1141 1142mach_msg_return_t 1143ipc_kmsg_get( 1144 mach_vm_address_t msg_addr, 1145 mach_msg_size_t size, 1146 ipc_kmsg_t *kmsgp) 1147{ 1148 mach_msg_size_t msg_and_trailer_size; 1149 ipc_kmsg_t kmsg; 1150 mach_msg_max_trailer_t *trailer; 1151 mach_msg_legacy_base_t legacy_base; 1152 mach_msg_size_t len_copied; 1153 legacy_base.body.msgh_descriptor_count = 0; 1154 1155 if ((size < sizeof(mach_msg_legacy_header_t)) || (size & 3)) 1156 return MACH_SEND_MSG_TOO_SMALL; 1157 1158 if (size > ipc_kmsg_max_body_space) 1159 return MACH_SEND_TOO_LARGE; 1160 1161 if(size == sizeof(mach_msg_legacy_header_t)) 1162 len_copied = sizeof(mach_msg_legacy_header_t); 1163 else 1164 len_copied = sizeof(mach_msg_legacy_base_t); 1165 1166 if (copyinmsg(msg_addr, (char *)&legacy_base, len_copied)) 1167 return MACH_SEND_INVALID_DATA; 1168 1169 msg_addr += sizeof(legacy_base.header); 1170#if defined(__LP64__) 1171 size += LEGACY_HEADER_SIZE_DELTA; 1172#endif 1173 /* unreachable if !DEBUG */ 1174 __unreachable_ok_push 1175 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { 1176 unsigned int j; 1177 for (j=0; j<sizeof(legacy_base.header); j++) { 1178 kprintf("%02x\n", ((unsigned char*)&legacy_base.header)[j]); 1179 } 1180 } 1181 __unreachable_ok_pop 1182 1183 msg_and_trailer_size = size + MAX_TRAILER_SIZE; 1184 kmsg = ipc_kmsg_alloc(msg_and_trailer_size); 1185 if (kmsg == IKM_NULL) 1186 return MACH_SEND_NO_BUFFER; 1187 1188 kmsg->ikm_header->msgh_size = size; 1189 kmsg->ikm_header->msgh_bits = legacy_base.header.msgh_bits; 1190 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_remote_port); 1191 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(legacy_base.header.msgh_local_port); 1192 kmsg->ikm_header->msgh_voucher_port = legacy_base.header.msgh_voucher_port; 1193 kmsg->ikm_header->msgh_id = legacy_base.header.msgh_id; 1194 1195 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_get header:\n" 1196 " size: 0x%.8x\n" 1197 " bits: 0x%.8x\n" 1198 " remote_port: %p\n" 1199 " local_port: %p\n" 1200 " voucher_port: 0x%.8x\n" 1201 " id: %.8d\n", 1202 kmsg->ikm_header->msgh_size, 1203 kmsg->ikm_header->msgh_bits, 1204 kmsg->ikm_header->msgh_remote_port, 1205 kmsg->ikm_header->msgh_local_port, 1206 kmsg->ikm_header->msgh_voucher_port, 1207 kmsg->ikm_header->msgh_id); 1208 1209 if (copyinmsg(msg_addr, (char *)(kmsg->ikm_header + 1), size - (mach_msg_size_t)sizeof(mach_msg_header_t))) { 1210 ipc_kmsg_free(kmsg); 1211 return MACH_SEND_INVALID_DATA; 1212 } 1213 1214 /* unreachable if !DEBUG */ 1215 __unreachable_ok_push 1216 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) 1217 { 1218 kprintf("body: size: %lu\n", (size - sizeof(mach_msg_header_t))); 1219 uint32_t i; 1220 for(i=0;i*4 < (size - sizeof(mach_msg_header_t));i++) 1221 { 1222 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]); 1223 } 1224 } 1225 __unreachable_ok_pop 1226 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_get()"); 1227 1228 /* 1229 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE) 1230 * However, the internal size field of the trailer (msgh_trailer_size) 1231 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize 1232 * the cases where no implicit data is requested. 1233 */ 1234 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + size); 1235 trailer->msgh_sender = current_thread()->task->sec_token; 1236 trailer->msgh_audit = current_thread()->task->audit_token; 1237 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0; 1238 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE; 1239 1240#ifdef ppc 1241 if(trcWork.traceMask) dbgTrace(0x1100, (unsigned int)kmsg->ikm_header->msgh_id, 1242 (unsigned int)kmsg->ikm_header->msgh_remote_port, 1243 (unsigned int)kmsg->ikm_header->msgh_local_port, 0); 1244#endif 1245 1246 trailer->msgh_labels.sender = 0; 1247 *kmsgp = kmsg; 1248 return MACH_MSG_SUCCESS; 1249} 1250 1251/* 1252 * Routine: ipc_kmsg_get_from_kernel 1253 * Purpose: 1254 * First checks for a preallocated message 1255 * reserved for kernel clients. If not found - 1256 * allocates a new kernel message buffer. 1257 * Copies a kernel message to the message buffer. 1258 * Only resource errors are allowed. 1259 * Conditions: 1260 * Nothing locked. 1261 * Ports in header are ipc_port_t. 1262 * Returns: 1263 * MACH_MSG_SUCCESS Acquired a message buffer. 1264 * MACH_SEND_NO_BUFFER Couldn't allocate a message buffer. 1265 */ 1266 1267mach_msg_return_t 1268ipc_kmsg_get_from_kernel( 1269 mach_msg_header_t *msg, 1270 mach_msg_size_t size, 1271 ipc_kmsg_t *kmsgp) 1272{ 1273 ipc_kmsg_t kmsg; 1274 mach_msg_size_t msg_and_trailer_size; 1275 mach_msg_max_trailer_t *trailer; 1276 ipc_port_t dest_port; 1277 1278 assert(size >= sizeof(mach_msg_header_t)); 1279 assert((size & 3) == 0); 1280 1281 dest_port = (ipc_port_t)msg->msgh_remote_port; 1282 1283 msg_and_trailer_size = size + MAX_TRAILER_SIZE; 1284 1285 /* 1286 * See if the port has a pre-allocated kmsg for kernel 1287 * clients. These are set up for those kernel clients 1288 * which cannot afford to wait. 1289 */ 1290 if (IP_VALID(dest_port) && IP_PREALLOC(dest_port)) { 1291 mach_msg_size_t max_desc = 0; 1292 1293 ip_lock(dest_port); 1294 if (!ip_active(dest_port)) { 1295 ip_unlock(dest_port); 1296 return MACH_SEND_NO_BUFFER; 1297 } 1298 assert(IP_PREALLOC(dest_port)); 1299 kmsg = dest_port->ip_premsg; 1300 if (ikm_prealloc_inuse(kmsg)) { 1301 ip_unlock(dest_port); 1302 return MACH_SEND_NO_BUFFER; 1303 } 1304#if !defined(__LP64__) 1305 if (msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) { 1306 assert(size > sizeof(mach_msg_base_t)); 1307 max_desc = ((mach_msg_base_t *)msg)->body.msgh_descriptor_count * 1308 DESC_SIZE_ADJUSTMENT; 1309 } 1310#endif 1311 if (msg_and_trailer_size > kmsg->ikm_size - max_desc) { 1312 ip_unlock(dest_port); 1313 return MACH_SEND_TOO_LARGE; 1314 } 1315 ikm_prealloc_set_inuse(kmsg, dest_port); 1316 ikm_set_header(kmsg, msg_and_trailer_size); 1317 ip_unlock(dest_port); 1318 } 1319 else 1320 { 1321 kmsg = ipc_kmsg_alloc(msg_and_trailer_size); 1322 if (kmsg == IKM_NULL) 1323 return MACH_SEND_NO_BUFFER; 1324 } 1325 1326 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, size); 1327 1328 kmsg->ikm_header->msgh_size = size; 1329 1330 /* 1331 * I reserve for the trailer the largest space (MAX_TRAILER_SIZE) 1332 * However, the internal size field of the trailer (msgh_trailer_size) 1333 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to 1334 * optimize the cases where no implicit data is requested. 1335 */ 1336 trailer = (mach_msg_max_trailer_t *) 1337 ((vm_offset_t)kmsg->ikm_header + size); 1338 trailer->msgh_sender = KERNEL_SECURITY_TOKEN; 1339 trailer->msgh_audit = KERNEL_AUDIT_TOKEN; 1340 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0; 1341 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE; 1342 1343 trailer->msgh_labels.sender = 0; 1344 1345 *kmsgp = kmsg; 1346 return MACH_MSG_SUCCESS; 1347} 1348 1349/* 1350 * Routine: ipc_kmsg_send 1351 * Purpose: 1352 * Send a message. The message holds a reference 1353 * for the destination port in the msgh_remote_port field. 1354 * 1355 * If unsuccessful, the caller still has possession of 1356 * the message and must do something with it. If successful, 1357 * the message is queued, given to a receiver, destroyed, 1358 * or handled directly by the kernel via mach_msg. 1359 * Conditions: 1360 * Nothing locked. 1361 * Returns: 1362 * MACH_MSG_SUCCESS The message was accepted. 1363 * MACH_SEND_TIMED_OUT Caller still has message. 1364 * MACH_SEND_INTERRUPTED Caller still has message. 1365 * MACH_SEND_INVALID_DEST Caller still has message. 1366 */ 1367 1368 1369mach_msg_return_t 1370ipc_kmsg_send( 1371 ipc_kmsg_t kmsg, 1372 mach_msg_option_t option, 1373 mach_msg_timeout_t send_timeout) 1374{ 1375 ipc_port_t port; 1376 mach_msg_return_t error = MACH_MSG_SUCCESS; 1377 spl_t s; 1378 1379#if IMPORTANCE_INHERITANCE 1380 boolean_t did_importance = FALSE; 1381#if IMPORTANCE_DEBUG 1382 mach_msg_id_t imp_msgh_id = -1; 1383 int sender_pid = -1; 1384#endif /* IMPORTANCE_DEBUG */ 1385#endif /* IMPORTANCE_INHERITANCE */ 1386 1387 /* don't allow the creation of a circular loop */ 1388 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_CIRCULAR) { 1389 ipc_kmsg_destroy(kmsg); 1390 return MACH_MSG_SUCCESS; 1391 } 1392 1393 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; 1394 assert(IP_VALID(port)); 1395 ip_lock(port); 1396 1397#if IMPORTANCE_INHERITANCE 1398retry: 1399#endif /* IMPORTANCE_INHERITANCE */ 1400 /* 1401 * Can't deliver to a dead port. 1402 * However, we can pretend it got sent 1403 * and was then immediately destroyed. 1404 */ 1405 if (!ip_active(port)) { 1406 ip_unlock(port); 1407 ip_release(port); /* JMM - Future: release right, not just ref */ 1408 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; 1409 ipc_kmsg_destroy(kmsg); 1410 return MACH_MSG_SUCCESS; 1411 } 1412 1413 if (port->ip_receiver == ipc_space_kernel) { 1414 1415 /* 1416 * We can check ip_receiver == ipc_space_kernel 1417 * before checking that the port is active because 1418 * ipc_port_dealloc_kernel clears ip_receiver 1419 * before destroying a kernel port. 1420 */ 1421 assert(ip_active(port)); 1422 port->ip_messages.imq_seqno++; 1423 ip_unlock(port); 1424 1425 current_task()->messages_sent++; 1426 1427 /* 1428 * Call the server routine, and get the reply message to send. 1429 */ 1430 kmsg = ipc_kobject_server(kmsg); 1431 if (kmsg == IKM_NULL) 1432 return MACH_MSG_SUCCESS; 1433 1434 port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port; 1435 assert(IP_VALID(port)); 1436 ip_lock(port); 1437 /* fall thru with reply - same options */ 1438 } 1439 1440#if IMPORTANCE_INHERITANCE 1441 /* 1442 * Need to see if this message needs importance donation and/or 1443 * propagation. That routine can drop the port lock temporarily. 1444 * If it does we'll have to revalidate the destination. 1445 */ 1446 if (did_importance == FALSE) { 1447 did_importance = TRUE; 1448 if (ipc_importance_send(kmsg, option)) 1449 goto retry; 1450 } 1451#endif /* IMPORTANCE_INHERITANCE */ 1452 1453 /* 1454 * We have a valid message and a valid reference on the port. 1455 * we can unlock the port and call mqueue_send() on its message 1456 * queue. Lock message queue while port is locked. 1457 */ 1458 s = splsched(); 1459 imq_lock(&port->ip_messages); 1460 ip_unlock(port); 1461 1462 error = ipc_mqueue_send(&port->ip_messages, kmsg, option, 1463 send_timeout, s); 1464 1465#if IMPORTANCE_INHERITANCE 1466 if (did_importance == TRUE) { 1467 __unused int importance_cleared = 0; 1468 switch (error) { 1469 case MACH_SEND_TIMED_OUT: 1470 case MACH_SEND_NO_BUFFER: 1471 case MACH_SEND_INTERRUPTED: 1472 case MACH_SEND_INVALID_DEST: 1473 /* 1474 * We still have the kmsg and its 1475 * reference on the port. But we 1476 * have to back out the importance 1477 * boost. 1478 * 1479 * The port could have changed hands, 1480 * be inflight to another destination, 1481 * etc... But in those cases our 1482 * back-out will find the new owner 1483 * (and all the operations that 1484 * transferred the right should have 1485 * applied their own boost adjustments 1486 * to the old owner(s)). 1487 */ 1488 importance_cleared = 1; 1489 ipc_importance_clean(kmsg); 1490 break; 1491 1492 case MACH_MSG_SUCCESS: 1493 default: 1494 break; 1495 } 1496#if IMPORTANCE_DEBUG 1497 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (IMPORTANCE_CODE(IMP_MSG, IMP_MSG_SEND)) | DBG_FUNC_END, 1498 audit_token_pid_from_task(current_task()), sender_pid, imp_msgh_id, importance_cleared, 0); 1499#endif /* IMPORTANCE_DEBUG */ 1500 } 1501#endif /* IMPORTANCE_INHERITANCE */ 1502 1503 /* 1504 * If the port has been destroyed while we wait, treat the message 1505 * as a successful delivery (like we do for an inactive port). 1506 */ 1507 if (error == MACH_SEND_INVALID_DEST) { 1508 ip_release(port); /* JMM - Future: release right, not just ref */ 1509 kmsg->ikm_header->msgh_remote_port = MACH_PORT_NULL; 1510 ipc_kmsg_destroy(kmsg); 1511 return MACH_MSG_SUCCESS; 1512 } 1513 return error; 1514} 1515 1516/* 1517 * Routine: ipc_kmsg_put 1518 * Purpose: 1519 * Copies a message buffer to a user message. 1520 * Copies only the specified number of bytes. 1521 * Frees the message buffer. 1522 * Conditions: 1523 * Nothing locked. The message buffer must have clean 1524 * header fields. 1525 * Returns: 1526 * MACH_MSG_SUCCESS Copied data out of message buffer. 1527 * MACH_RCV_INVALID_DATA Couldn't copy to user message. 1528 */ 1529 1530mach_msg_return_t 1531ipc_kmsg_put( 1532 mach_vm_address_t msg_addr, 1533 ipc_kmsg_t kmsg, 1534 mach_msg_size_t size) 1535{ 1536 mach_msg_return_t mr; 1537 1538 DEBUG_IPC_KMSG_PRINT(kmsg, "ipc_kmsg_put()"); 1539 1540 1541 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_put header:\n" 1542 " size: 0x%.8x\n" 1543 " bits: 0x%.8x\n" 1544 " remote_port: %p\n" 1545 " local_port: %p\n" 1546 " voucher_port: 0x%.8x\n" 1547 " id: %.8d\n", 1548 kmsg->ikm_header->msgh_size, 1549 kmsg->ikm_header->msgh_bits, 1550 kmsg->ikm_header->msgh_remote_port, 1551 kmsg->ikm_header->msgh_local_port, 1552 kmsg->ikm_header->msgh_voucher_port, 1553 kmsg->ikm_header->msgh_id); 1554 1555#if defined(__LP64__) 1556 if (current_task() != kernel_task) { /* don't if receiver expects fully-cooked in-kernel msg; ux_exception */ 1557 mach_msg_legacy_header_t *legacy_header = 1558 (mach_msg_legacy_header_t *)((vm_offset_t)(kmsg->ikm_header) + LEGACY_HEADER_SIZE_DELTA); 1559 1560 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; 1561 mach_msg_size_t msg_size = kmsg->ikm_header->msgh_size; 1562 mach_port_name_t remote_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_remote_port); 1563 mach_port_name_t local_port = CAST_MACH_PORT_TO_NAME(kmsg->ikm_header->msgh_local_port); 1564 mach_port_name_t voucher_port = kmsg->ikm_header->msgh_voucher_port; 1565 mach_msg_id_t id = kmsg->ikm_header->msgh_id; 1566 1567 legacy_header->msgh_id = id; 1568 legacy_header->msgh_local_port = local_port; 1569 legacy_header->msgh_remote_port = remote_port; 1570 legacy_header->msgh_voucher_port = voucher_port; 1571 legacy_header->msgh_size = msg_size - LEGACY_HEADER_SIZE_DELTA; 1572 legacy_header->msgh_bits = bits; 1573 1574 size -= LEGACY_HEADER_SIZE_DELTA; 1575 kmsg->ikm_header = (mach_msg_header_t *)legacy_header; 1576 } 1577#endif 1578 1579 /* unreachable if !DEBUG */ 1580 __unreachable_ok_push 1581 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) { 1582 kprintf("ipc_kmsg_put header+body: %d\n", (size)); 1583 uint32_t i; 1584 for(i=0;i*4 < size;i++) 1585 { 1586 kprintf("%.4x\n",((uint32_t *)kmsg->ikm_header)[i]); 1587 } 1588 kprintf("type: %d\n", ((mach_msg_type_descriptor_t *)(((mach_msg_base_t *)kmsg->ikm_header)+1))->type); 1589 } 1590 __unreachable_ok_pop 1591 if (copyoutmsg((const char *) kmsg->ikm_header, msg_addr, size)) 1592 mr = MACH_RCV_INVALID_DATA; 1593 else 1594 mr = MACH_MSG_SUCCESS; 1595 1596 ipc_kmsg_free(kmsg); 1597 return mr; 1598} 1599 1600/* 1601 * Routine: ipc_kmsg_put_to_kernel 1602 * Purpose: 1603 * Copies a message buffer to a kernel message. 1604 * Frees the message buffer. 1605 * No errors allowed. 1606 * Conditions: 1607 * Nothing locked. 1608 */ 1609 1610void 1611ipc_kmsg_put_to_kernel( 1612 mach_msg_header_t *msg, 1613 ipc_kmsg_t kmsg, 1614 mach_msg_size_t size) 1615{ 1616 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, size); 1617 1618 ipc_kmsg_free(kmsg); 1619} 1620 1621/* 1622 * Routine: ipc_kmsg_copyin_header 1623 * Purpose: 1624 * "Copy-in" port rights in the header of a message. 1625 * Operates atomically; if it doesn't succeed the 1626 * message header and the space are left untouched. 1627 * If it does succeed the remote/local port fields 1628 * contain object pointers instead of port names, 1629 * and the bits field is updated. The destination port 1630 * will be a valid port pointer. 1631 * 1632 * Conditions: 1633 * Nothing locked. 1634 * Returns: 1635 * MACH_MSG_SUCCESS Successful copyin. 1636 * MACH_SEND_INVALID_HEADER 1637 * Illegal value in the message header bits. 1638 * MACH_SEND_INVALID_DEST The space is dead. 1639 * MACH_SEND_INVALID_DEST Can't copyin destination port. 1640 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) 1641 * MACH_SEND_INVALID_REPLY Can't copyin reply port. 1642 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) 1643 */ 1644 1645mach_msg_return_t 1646ipc_kmsg_copyin_header( 1647 ipc_kmsg_t kmsg, 1648 ipc_space_t space, 1649 mach_msg_option_t *optionp) 1650{ 1651 mach_msg_header_t *msg = kmsg->ikm_header; 1652 mach_msg_bits_t mbits = msg->msgh_bits & MACH_MSGH_BITS_USER; 1653 mach_port_name_t dest_name = CAST_MACH_PORT_TO_NAME(msg->msgh_remote_port); 1654 mach_port_name_t reply_name = CAST_MACH_PORT_TO_NAME(msg->msgh_local_port); 1655 mach_port_name_t voucher_name = MACH_PORT_NULL; 1656 kern_return_t kr; 1657 1658 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); 1659 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); 1660 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); 1661 ipc_object_t dest_port = IO_NULL; 1662 ipc_object_t reply_port = IO_NULL; 1663 ipc_port_t dest_soright = IP_NULL; 1664 ipc_port_t reply_soright = IP_NULL; 1665 ipc_port_t voucher_soright = IP_NULL; 1666 ipc_port_t release_port = IP_NULL; 1667 ipc_port_t voucher_port = IP_NULL; 1668 ipc_port_t voucher_release_port = IP_NULL; 1669 ipc_entry_t dest_entry = IE_NULL; 1670 ipc_entry_t reply_entry = IE_NULL; 1671 ipc_entry_t voucher_entry = IE_NULL; 1672 1673#if IMPORTANCE_INHERITANCE 1674 int assertcnt = 0; 1675 boolean_t needboost = FALSE; 1676#endif /* IMPORTANCE_INHERITANCE */ 1677 1678 queue_head_t links_data; 1679 queue_t links = &links_data; 1680 wait_queue_link_t wql; 1681 1682 queue_init(links); 1683 1684 if ((mbits != msg->msgh_bits) || 1685 (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) || 1686 ((reply_type == 0) ? 1687 (reply_name != MACH_PORT_NULL) : 1688 !MACH_MSG_TYPE_PORT_ANY_SEND(reply_type))) 1689 return MACH_SEND_INVALID_HEADER; 1690 1691 if (!MACH_PORT_VALID(dest_name)) 1692 return MACH_SEND_INVALID_DEST; 1693 1694 is_write_lock(space); 1695 if (!is_active(space)) { 1696 is_write_unlock(space); 1697 return MACH_SEND_INVALID_DEST; 1698 } 1699 /* space locked and active */ 1700 1701 /* 1702 * If there is a voucher specified, make sure the disposition is 1703 * valid and the entry actually refers to a voucher port. Don't 1704 * actually copy in until we validate destination and reply. 1705 */ 1706 if (voucher_type != MACH_MSGH_BITS_ZERO) { 1707 1708 voucher_name = msg->msgh_voucher_port; 1709 1710 if (voucher_name == MACH_PORT_DEAD || 1711 (voucher_type != MACH_MSG_TYPE_MOVE_SEND && 1712 voucher_type != MACH_MSG_TYPE_COPY_SEND)) { 1713 is_write_unlock(space); 1714 return MACH_SEND_INVALID_VOUCHER; 1715 } 1716 1717 if (voucher_name != MACH_PORT_NULL) { 1718 voucher_entry = ipc_entry_lookup(space, voucher_name); 1719 if (voucher_entry == IE_NULL || 1720 (voucher_entry->ie_bits & MACH_PORT_TYPE_SEND) == 0 || 1721 io_kotype(voucher_entry->ie_object) != IKOT_VOUCHER) { 1722 is_write_unlock(space); 1723 return MACH_SEND_INVALID_VOUCHER; 1724 } 1725 } else { 1726 voucher_type = MACH_MSG_TYPE_MOVE_SEND; 1727 } 1728 } 1729 1730 /* 1731 * Handle combinations of validating destination and reply; along 1732 * with copying in destination, reply, and voucher in an atomic way. 1733 */ 1734 1735 if (dest_name == voucher_name) { 1736 1737 /* 1738 * If the destination name is the same as the voucher name, 1739 * the voucher_entry must already be known. Either that or 1740 * the destination name is MACH_PORT_NULL (i.e. invalid). 1741 */ 1742 dest_entry = voucher_entry; 1743 if (dest_entry == IE_NULL) { 1744 goto invalid_dest; 1745 } 1746 1747 /* 1748 * Make sure a future copyin of the reply port will succeed. 1749 * Once we start copying in the dest/voucher pair, we can't 1750 * back out. 1751 */ 1752 if (MACH_PORT_VALID(reply_name)) { 1753 assert(reply_type != 0); /* because reply_name not null */ 1754 1755 /* It is just WRONG if dest, voucher, and reply are all the same. */ 1756 if (voucher_name == reply_name) { 1757 goto invalid_reply; 1758 } 1759 reply_entry = ipc_entry_lookup(space, reply_name); 1760 if (reply_entry == IE_NULL) { 1761 goto invalid_reply; 1762 } 1763 assert(dest_entry != reply_entry); /* names are not equal */ 1764 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) { 1765 goto invalid_reply; 1766 } 1767 } 1768 1769 /* 1770 * Do the joint copyin of the dest disposition and 1771 * voucher disposition from the one entry/port. We 1772 * already validated that the voucher copyin would 1773 * succeed (above). So, any failure in combining 1774 * the copyins can be blamed on the destination. 1775 */ 1776 kr = ipc_right_copyin_two(space, dest_name, dest_entry, 1777 dest_type, voucher_type, 1778 &dest_port, &dest_soright, 1779 &release_port); 1780 if (kr != KERN_SUCCESS) { 1781 assert(kr != KERN_INVALID_CAPABILITY); 1782 goto invalid_dest; 1783 } 1784 voucher_port = (ipc_port_t)dest_port; 1785 1786 /* 1787 * could not have been one of these dispositions, 1788 * validated the port was a true kernel voucher port above, 1789 * AND was successfully able to copyin both dest and voucher. 1790 */ 1791 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND); 1792 assert(dest_type != MACH_MSG_TYPE_MAKE_SEND_ONCE); 1793 assert(dest_type != MACH_MSG_TYPE_MOVE_SEND_ONCE); 1794 1795 /* 1796 * Perform the delayed reply right copyin (guaranteed success). 1797 */ 1798 if (reply_entry != IE_NULL) { 1799#if IMPORTANCE_INHERITANCE 1800 kr = ipc_right_copyin(space, reply_name, reply_entry, 1801 reply_type, TRUE, 1802 &reply_port, &reply_soright, 1803 &release_port, 1804 &assertcnt, 1805 links); 1806 assert(assertcnt == 0); 1807#else 1808 kr = ipc_right_copyin(space, reply_name, reply_entry, 1809 reply_type, TRUE, 1810 &reply_port, &reply_soright, 1811 &release_port, 1812 links); 1813#endif /* IMPORTANCE_INHERITANCE */ 1814 assert(kr == KERN_SUCCESS); 1815 } 1816 1817 } else { 1818 if (dest_name == reply_name) { 1819 /* 1820 * Destination and reply ports are the same! 1821 * This is very similar to the case where the 1822 * destination and voucher ports were the same 1823 * (except the reply port disposition is not 1824 * previously validated). 1825 */ 1826 dest_entry = ipc_entry_lookup(space, dest_name); 1827 if (dest_entry == IE_NULL) { 1828 goto invalid_dest; 1829 } 1830 reply_entry = dest_entry; 1831 assert(reply_type != 0); /* because name not null */ 1832 1833 /* 1834 * Do the joint copyin of the dest disposition and 1835 * reply disposition from the one entry/port. 1836 */ 1837 kr = ipc_right_copyin_two(space, dest_name, dest_entry, 1838 dest_type, reply_type, 1839 &dest_port, &dest_soright, 1840 &release_port); 1841 if (kr == KERN_INVALID_CAPABILITY) { 1842 goto invalid_reply; 1843 } else if (kr != KERN_SUCCESS) { 1844 goto invalid_dest; 1845 } 1846 reply_port = dest_port; 1847 1848 1849 } else { 1850 /* 1851 * Handle destination and reply independently, as 1852 * they are independent entries (even if the entries 1853 * refer to the same port). 1854 * 1855 * This can be the tough case to make atomic. 1856 * 1857 * The difficult problem is serializing with port death. 1858 * The bad case is when dest_port dies after its copyin, 1859 * reply_port dies before its copyin, and dest_port dies before 1860 * reply_port. Then the copyins operated as if dest_port was 1861 * alive and reply_port was dead, which shouldn't have happened 1862 * because they died in the other order. 1863 * 1864 * Note that it is easy for a user task to tell if 1865 * a copyin happened before or after a port died. 1866 * If a port dies before copyin, a dead-name notification 1867 * is generated and the dead name's urefs are incremented, 1868 * and if the copyin happens first, a port-deleted 1869 * notification is generated. 1870 * 1871 * Even so, avoiding that potentially detectable race is too 1872 * expensive - and no known code cares about it. So, we just 1873 * do the expedient thing and copy them in one after the other. 1874 */ 1875 1876 dest_entry = ipc_entry_lookup(space, dest_name); 1877 if (dest_entry == IE_NULL) { 1878 goto invalid_dest; 1879 } 1880 assert(dest_entry != voucher_entry); 1881 1882 /* 1883 * Make sure reply port entry is valid before dest copyin. 1884 */ 1885 if (MACH_PORT_VALID(reply_name)) { 1886 if (reply_name == voucher_name) { 1887 goto invalid_reply; 1888 } 1889 reply_entry = ipc_entry_lookup(space, reply_name); 1890 if (reply_entry == IE_NULL) { 1891 goto invalid_reply; 1892 } 1893 assert(dest_entry != reply_entry); /* names are not equal */ 1894 assert(reply_type != 0); /* because reply_name not null */ 1895 1896 if (!ipc_right_copyin_check(space, reply_name, reply_entry, reply_type)) { 1897 goto invalid_reply; 1898 } 1899 } 1900 1901 /* 1902 * copyin the destination. 1903 */ 1904#if IMPORTANCE_INHERITANCE 1905 kr = ipc_right_copyin(space, dest_name, dest_entry, 1906 dest_type, FALSE, 1907 &dest_port, &dest_soright, 1908 &release_port, 1909 &assertcnt, 1910 links); 1911 assert(assertcnt == 0); 1912#else 1913 kr = ipc_right_copyin(space, dest_name, dest_entry, 1914 dest_type, FALSE, 1915 &dest_port, &dest_soright, 1916 &release_port, 1917 links); 1918#endif /* IMPORTANCE_INHERITANCE */ 1919 if (kr != KERN_SUCCESS) { 1920 goto invalid_dest; 1921 } 1922 assert(IO_VALID(dest_port)); 1923 assert(!IP_VALID(release_port)); 1924 1925 /* 1926 * Copyin the pre-validated reply right. 1927 * It's OK if the reply right has gone dead in the meantime. 1928 */ 1929 if (MACH_PORT_VALID(reply_name)) { 1930#if IMPORTANCE_INHERITANCE 1931 kr = ipc_right_copyin(space, reply_name, reply_entry, 1932 reply_type, TRUE, 1933 &reply_port, &reply_soright, 1934 &release_port, 1935 &assertcnt, 1936 links); 1937 assert(assertcnt == 0); 1938#else 1939 kr = ipc_right_copyin(space, reply_name, reply_entry, 1940 reply_type, TRUE, 1941 &reply_port, &reply_soright, 1942 &release_port, 1943 links); 1944#endif /* IMPORTANCE_INHERITANCE */ 1945 assert(kr == KERN_SUCCESS); 1946 } else { 1947 /* convert invalid name to equivalent ipc_object type */ 1948 reply_port = (ipc_object_t)CAST_MACH_NAME_TO_PORT(reply_name); 1949 } 1950 } 1951 1952 /* 1953 * Finally can copyin the voucher right now that dest and reply 1954 * are fully copied in (guaranteed success). 1955 */ 1956 if (IE_NULL != voucher_entry) { 1957#if IMPORTANCE_INHERITANCE 1958 kr = ipc_right_copyin(space, voucher_name, voucher_entry, 1959 voucher_type, FALSE, 1960 (ipc_object_t *)&voucher_port, 1961 &voucher_soright, 1962 &voucher_release_port, 1963 &assertcnt, 1964 links); 1965 assert(assertcnt == 0); 1966#else 1967 kr = ipc_right_copyin(space, voucher_name, voucher_entry, 1968 voucher_type, FALSE, 1969 (ipc_object_t *)&voucher_port, 1970 &voucher_soright, 1971 &voucher_release_port, 1972 links); 1973#endif /* IMPORTANCE_INHERITANCE */ 1974 assert(KERN_SUCCESS == kr); 1975 assert(IP_VALID(voucher_port)); 1976 assert(ip_active(voucher_port)); 1977 } 1978 } 1979 1980 /* the entry(s) might need to be deallocated */ 1981 assert(IE_NULL != dest_entry); 1982 if (IE_BITS_TYPE(dest_entry->ie_bits) == MACH_PORT_TYPE_NONE) { 1983 ipc_entry_dealloc(space, dest_name, dest_entry); 1984 dest_entry = IE_NULL; 1985 } 1986 if (dest_entry != reply_entry && IE_NULL != reply_entry && 1987 IE_BITS_TYPE(reply_entry->ie_bits) == MACH_PORT_TYPE_NONE) { 1988 ipc_entry_dealloc(space, reply_name, reply_entry); 1989 reply_entry = IE_NULL; 1990 } 1991 if (dest_entry != voucher_entry && IE_NULL != voucher_entry && 1992 IE_BITS_TYPE(voucher_entry->ie_bits) == MACH_PORT_TYPE_NONE) { 1993 ipc_entry_dealloc(space, voucher_name, voucher_entry); 1994 voucher_entry = IE_NULL; 1995 } 1996 1997 /* 1998 * No room to store voucher port in in-kernel msg header, 1999 * so we store it back in the kmsg itself. 2000 */ 2001 if (IP_VALID(voucher_port)) { 2002 assert(ip_active(voucher_port)); 2003 kmsg->ikm_voucher = voucher_port; 2004 voucher_type = MACH_MSG_TYPE_MOVE_SEND; 2005 } 2006 2007 dest_type = ipc_object_copyin_type(dest_type); 2008 reply_type = ipc_object_copyin_type(reply_type); 2009 2010 /* 2011 * JMM - Without rdar://problem/6275821, this is the last place we can 2012 * re-arm the send-possible notifications. It may trigger unexpectedly 2013 * early (send may NOT have failed), but better than missing. We assure 2014 * we won't miss by forcing MACH_SEND_ALWAYS if we got past arming. 2015 */ 2016 if (((*optionp & MACH_SEND_NOTIFY) != 0) && 2017 dest_type != MACH_MSG_TYPE_PORT_SEND_ONCE && 2018 dest_entry != IE_NULL && dest_entry->ie_request != IE_REQ_NONE) { 2019 ipc_port_t dport = (ipc_port_t)dest_port; 2020 2021 assert(dport != IP_NULL); 2022 ip_lock(dport); 2023 if (ip_active(dport) && dport->ip_receiver != ipc_space_kernel) { 2024 if (ip_full(dport)) { 2025#if IMPORTANCE_INHERITANCE 2026 needboost = ipc_port_request_sparm(dport, dest_name, 2027 dest_entry->ie_request, 2028 (*optionp & MACH_SEND_NOIMPORTANCE)); 2029 if (needboost == FALSE) 2030 ip_unlock(dport); 2031#else 2032 ipc_port_request_sparm(dport, dest_name, dest_entry->ie_request); 2033 ip_unlock(dport); 2034#endif /* IMPORTANCE_INHERITANCE */ 2035 } else { 2036 *optionp |= MACH_SEND_ALWAYS; 2037 ip_unlock(dport); 2038 } 2039 } else { 2040 ip_unlock(dport); 2041 } 2042 } 2043 2044 is_write_unlock(space); 2045 2046#if IMPORTANCE_INHERITANCE 2047 /* 2048 * If our request is the first boosting send-possible 2049 * notification this cycle, push the boost down the 2050 * destination port. 2051 */ 2052 if (needboost == TRUE) { 2053 ipc_port_t dport = (ipc_port_t)dest_port; 2054 2055 /* dport still locked from above */ 2056 if (ipc_port_importance_delta(dport, 1) == FALSE) { 2057 ip_unlock(dport); 2058 } 2059 } 2060#endif /* IMPORTANCE_INHERITANCE */ 2061 2062 if (dest_soright != IP_NULL) { 2063 ipc_notify_port_deleted(dest_soright, dest_name); 2064 } 2065 if (reply_soright != IP_NULL) { 2066 ipc_notify_port_deleted(reply_soright, reply_name); 2067 } 2068 if (voucher_soright != IP_NULL) { 2069 ipc_notify_port_deleted(voucher_soright, voucher_name); 2070 } 2071 msg->msgh_bits = MACH_MSGH_BITS_SET(dest_type, reply_type, voucher_type, mbits); 2072 msg->msgh_remote_port = (ipc_port_t)dest_port; 2073 msg->msgh_local_port = (ipc_port_t)reply_port; 2074 2075 while(!queue_empty(links)) { 2076 wql = (wait_queue_link_t) dequeue(links); 2077 wait_queue_link_free(wql); 2078 } 2079 2080 if (release_port != IP_NULL) 2081 ip_release(release_port); 2082 2083 if (voucher_release_port != IP_NULL) 2084 ip_release(voucher_release_port); 2085 2086 return MACH_MSG_SUCCESS; 2087 2088invalid_reply: 2089 is_write_unlock(space); 2090 2091 while(!queue_empty(links)) { 2092 wql = (wait_queue_link_t) dequeue(links); 2093 wait_queue_link_free(wql); 2094 } 2095 2096 if (release_port != IP_NULL) 2097 ip_release(release_port); 2098 2099 assert(voucher_port == IP_NULL); 2100 assert(voucher_soright == IP_NULL); 2101 2102 return MACH_SEND_INVALID_REPLY; 2103 2104invalid_dest: 2105 is_write_unlock(space); 2106 2107 while(!queue_empty(links)) { 2108 wql = (wait_queue_link_t) dequeue(links); 2109 wait_queue_link_free(wql); 2110 } 2111 2112 if (release_port != IP_NULL) 2113 ip_release(release_port); 2114 2115 if (reply_soright != IP_NULL) 2116 ipc_notify_port_deleted(reply_soright, reply_name); 2117 2118 assert(voucher_port == IP_NULL); 2119 assert(voucher_soright == IP_NULL); 2120 2121 return MACH_SEND_INVALID_DEST; 2122} 2123 2124mach_msg_descriptor_t *ipc_kmsg_copyin_port_descriptor( 2125 volatile mach_msg_port_descriptor_t *dsc, 2126 mach_msg_legacy_port_descriptor_t *user_dsc, 2127 ipc_space_t space, 2128 ipc_object_t dest, 2129 ipc_kmsg_t kmsg, 2130 mach_msg_return_t *mr); 2131 2132void ipc_print_type_name( 2133 int type_name); 2134mach_msg_descriptor_t * 2135ipc_kmsg_copyin_port_descriptor( 2136 volatile mach_msg_port_descriptor_t *dsc, 2137 mach_msg_legacy_port_descriptor_t *user_dsc_in, 2138 ipc_space_t space, 2139 ipc_object_t dest, 2140 ipc_kmsg_t kmsg, 2141 mach_msg_return_t *mr) 2142{ 2143 volatile mach_msg_legacy_port_descriptor_t *user_dsc = user_dsc_in; 2144 mach_msg_type_name_t user_disp; 2145 mach_msg_type_name_t result_disp; 2146 mach_port_name_t name; 2147 ipc_object_t object; 2148 2149 user_disp = user_dsc->disposition; 2150 result_disp = ipc_object_copyin_type(user_disp); 2151 2152 name = (mach_port_name_t)user_dsc->name; 2153 if (MACH_PORT_VALID(name)) { 2154 2155 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object); 2156 if (kr != KERN_SUCCESS) { 2157 *mr = MACH_SEND_INVALID_RIGHT; 2158 return NULL; 2159 } 2160 2161 if ((result_disp == MACH_MSG_TYPE_PORT_RECEIVE) && 2162 ipc_port_check_circularity((ipc_port_t) object, 2163 (ipc_port_t) dest)) { 2164 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; 2165 } 2166 dsc->name = (ipc_port_t) object; 2167 } else { 2168 dsc->name = CAST_MACH_NAME_TO_PORT(name); 2169 } 2170 dsc->disposition = result_disp; 2171 dsc->type = MACH_MSG_PORT_DESCRIPTOR; 2172 2173 dsc->pad_end = 0; // debug, unnecessary 2174 2175 return (mach_msg_descriptor_t *)(user_dsc_in+1); 2176} 2177 2178mach_msg_descriptor_t * ipc_kmsg_copyin_ool_descriptor( 2179 mach_msg_ool_descriptor_t *dsc, 2180 mach_msg_descriptor_t *user_dsc, 2181 int is_64bit, 2182 vm_offset_t *paddr, 2183 vm_map_copy_t *copy, 2184 vm_size_t *space_needed, 2185 vm_map_t map, 2186 mach_msg_return_t *mr); 2187mach_msg_descriptor_t * 2188ipc_kmsg_copyin_ool_descriptor( 2189 mach_msg_ool_descriptor_t *dsc, 2190 mach_msg_descriptor_t *user_dsc, 2191 int is_64bit, 2192 vm_offset_t *paddr, 2193 vm_map_copy_t *copy, 2194 vm_size_t *space_needed, 2195 vm_map_t map, 2196 mach_msg_return_t *mr) 2197{ 2198 vm_size_t length; 2199 boolean_t dealloc; 2200 mach_msg_copy_options_t copy_options; 2201 mach_vm_offset_t addr; 2202 mach_msg_descriptor_type_t dsc_type; 2203 2204 if (is_64bit) { 2205 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 2206 2207 addr = (mach_vm_offset_t) user_ool_dsc->address; 2208 length = user_ool_dsc->size; 2209 dealloc = user_ool_dsc->deallocate; 2210 copy_options = user_ool_dsc->copy; 2211 dsc_type = user_ool_dsc->type; 2212 2213 user_dsc = (typeof(user_dsc))(user_ool_dsc+1); 2214 } else { 2215 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 2216 2217 addr = CAST_USER_ADDR_T(user_ool_dsc->address); 2218 dealloc = user_ool_dsc->deallocate; 2219 copy_options = user_ool_dsc->copy; 2220 dsc_type = user_ool_dsc->type; 2221 length = user_ool_dsc->size; 2222 2223 user_dsc = (typeof(user_dsc))(user_ool_dsc+1); 2224 } 2225 2226 dsc->size = (mach_msg_size_t)length; 2227 dsc->deallocate = dealloc; 2228 dsc->copy = copy_options; 2229 dsc->type = dsc_type; 2230 2231 if (length == 0) { 2232 dsc->address = NULL; 2233 } else if ((length >= MSG_OOL_SIZE_SMALL) && 2234 (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) { 2235 2236 /* 2237 * If the request is a physical copy and the source 2238 * is not being deallocated, then allocate space 2239 * in the kernel's pageable ipc copy map and copy 2240 * the data in. The semantics guarantee that the 2241 * data will have been physically copied before 2242 * the send operation terminates. Thus if the data 2243 * is not being deallocated, we must be prepared 2244 * to page if the region is sufficiently large. 2245 */ 2246 if (copyin(addr, (char *)*paddr, length)) { 2247 *mr = MACH_SEND_INVALID_MEMORY; 2248 return NULL; 2249 } 2250 2251 /* 2252 * The kernel ipc copy map is marked no_zero_fill. 2253 * If the transfer is not a page multiple, we need 2254 * to zero fill the balance. 2255 */ 2256 if (!page_aligned(length)) { 2257 (void) memset((void *) (*paddr + length), 0, 2258 round_page(length) - length); 2259 } 2260 if (vm_map_copyin(ipc_kernel_copy_map, (vm_map_address_t)*paddr, 2261 (vm_map_size_t)length, TRUE, copy) != KERN_SUCCESS) { 2262 *mr = MACH_MSG_VM_KERNEL; 2263 return NULL; 2264 } 2265 dsc->address = (void *)*copy; 2266 *paddr += round_page(length); 2267 *space_needed -= round_page(length); 2268 } else { 2269 2270 /* 2271 * Make a vm_map_copy_t of the of the data. If the 2272 * data is small, this will do an optimized physical 2273 * copy. Otherwise, it will do a virtual copy. 2274 * 2275 * NOTE: A virtual copy is OK if the original is being 2276 * deallocted, even if a physical copy was requested. 2277 */ 2278 kern_return_t kr = vm_map_copyin(map, addr, 2279 (vm_map_size_t)length, dealloc, copy); 2280 if (kr != KERN_SUCCESS) { 2281 *mr = (kr == KERN_RESOURCE_SHORTAGE) ? 2282 MACH_MSG_VM_KERNEL : 2283 MACH_SEND_INVALID_MEMORY; 2284 return NULL; 2285 } 2286 dsc->address = (void *)*copy; 2287 } 2288 return user_dsc; 2289} 2290 2291mach_msg_descriptor_t * ipc_kmsg_copyin_ool_ports_descriptor( 2292 mach_msg_ool_ports_descriptor_t *dsc, 2293 mach_msg_descriptor_t *user_dsc, 2294 int is_64bit, 2295 vm_map_t map, 2296 ipc_space_t space, 2297 ipc_object_t dest, 2298 ipc_kmsg_t kmsg, 2299 mach_msg_return_t *mr); 2300mach_msg_descriptor_t * 2301ipc_kmsg_copyin_ool_ports_descriptor( 2302 mach_msg_ool_ports_descriptor_t *dsc, 2303 mach_msg_descriptor_t *user_dsc, 2304 int is_64bit, 2305 vm_map_t map, 2306 ipc_space_t space, 2307 ipc_object_t dest, 2308 ipc_kmsg_t kmsg, 2309 mach_msg_return_t *mr) 2310{ 2311 void *data; 2312 ipc_object_t *objects; 2313 unsigned int i; 2314 mach_vm_offset_t addr; 2315 mach_msg_type_name_t user_disp; 2316 mach_msg_type_name_t result_disp; 2317 mach_msg_type_number_t count; 2318 mach_msg_copy_options_t copy_option; 2319 boolean_t deallocate; 2320 mach_msg_descriptor_type_t type; 2321 vm_size_t ports_length, names_length; 2322 2323 if (is_64bit) { 2324 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 2325 2326 addr = (mach_vm_offset_t)user_ool_dsc->address; 2327 count = user_ool_dsc->count; 2328 deallocate = user_ool_dsc->deallocate; 2329 copy_option = user_ool_dsc->copy; 2330 user_disp = user_ool_dsc->disposition; 2331 type = user_ool_dsc->type; 2332 2333 user_dsc = (typeof(user_dsc))(user_ool_dsc+1); 2334 } else { 2335 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 2336 2337 addr = CAST_USER_ADDR_T(user_ool_dsc->address); 2338 count = user_ool_dsc->count; 2339 deallocate = user_ool_dsc->deallocate; 2340 copy_option = user_ool_dsc->copy; 2341 user_disp = user_ool_dsc->disposition; 2342 type = user_ool_dsc->type; 2343 2344 user_dsc = (typeof(user_dsc))(user_ool_dsc+1); 2345 } 2346 2347 dsc->deallocate = deallocate; 2348 dsc->copy = copy_option; 2349 dsc->type = type; 2350 dsc->count = count; 2351 dsc->address = NULL; /* for now */ 2352 2353 result_disp = ipc_object_copyin_type(user_disp); 2354 dsc->disposition = result_disp; 2355 2356 if (count > (INT_MAX / sizeof(mach_port_t))) { 2357 *mr = MACH_SEND_TOO_LARGE; 2358 return NULL; 2359 } 2360 2361 /* calculate length of data in bytes, rounding up */ 2362 ports_length = count * sizeof(mach_port_t); 2363 names_length = count * sizeof(mach_port_name_t); 2364 2365 if (ports_length == 0) { 2366 return user_dsc; 2367 } 2368 2369 data = kalloc(ports_length); 2370 2371 if (data == NULL) { 2372 *mr = MACH_SEND_NO_BUFFER; 2373 return NULL; 2374 } 2375 2376#ifdef __LP64__ 2377 mach_port_name_t *names = &((mach_port_name_t *)data)[count]; 2378#else 2379 mach_port_name_t *names = ((mach_port_name_t *)data); 2380#endif 2381 2382 if (copyinmap(map, addr, names, names_length) != KERN_SUCCESS) { 2383 kfree(data, ports_length); 2384 *mr = MACH_SEND_INVALID_MEMORY; 2385 return NULL; 2386 } 2387 2388 if (deallocate) { 2389 (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)ports_length); 2390 } 2391 2392 objects = (ipc_object_t *) data; 2393 dsc->address = data; 2394 2395 for ( i = 0; i < count; i++) { 2396 mach_port_name_t name = names[i]; 2397 ipc_object_t object; 2398 2399 if (!MACH_PORT_VALID(name)) { 2400 objects[i] = (ipc_object_t)CAST_MACH_NAME_TO_PORT(name); 2401 continue; 2402 } 2403 2404 kern_return_t kr = ipc_object_copyin(space, name, user_disp, &object); 2405 2406 if (kr != KERN_SUCCESS) { 2407 unsigned int j; 2408 2409 for(j = 0; j < i; j++) { 2410 object = objects[j]; 2411 if (IPC_OBJECT_VALID(object)) 2412 ipc_object_destroy(object, result_disp); 2413 } 2414 kfree(data, ports_length); 2415 dsc->address = NULL; 2416 *mr = MACH_SEND_INVALID_RIGHT; 2417 return NULL; 2418 } 2419 2420 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && 2421 ipc_port_check_circularity( 2422 (ipc_port_t) object, 2423 (ipc_port_t) dest)) 2424 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; 2425 2426 objects[i] = object; 2427 } 2428 2429 return user_dsc; 2430} 2431 2432/* 2433 * Routine: ipc_kmsg_copyin_body 2434 * Purpose: 2435 * "Copy-in" port rights and out-of-line memory 2436 * in the message body. 2437 * 2438 * In all failure cases, the message is left holding 2439 * no rights or memory. However, the message buffer 2440 * is not deallocated. If successful, the message 2441 * contains a valid destination port. 2442 * Conditions: 2443 * Nothing locked. 2444 * Returns: 2445 * MACH_MSG_SUCCESS Successful copyin. 2446 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory. 2447 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body. 2448 * MACH_SEND_INVALID_TYPE Bad type specification. 2449 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data. 2450 * MACH_SEND_INVALID_RT_OOL_SIZE OOL Buffer too large for RT 2451 * MACH_MSG_INVALID_RT_DESCRIPTOR Dealloc and RT are incompatible 2452 */ 2453 2454mach_msg_return_t 2455ipc_kmsg_copyin_body( 2456 ipc_kmsg_t kmsg, 2457 ipc_space_t space, 2458 vm_map_t map) 2459{ 2460 ipc_object_t dest; 2461 mach_msg_body_t *body; 2462 mach_msg_descriptor_t *daddr, *naddr; 2463 mach_msg_descriptor_t *user_addr, *kern_addr; 2464 mach_msg_type_number_t dsc_count; 2465 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); 2466 boolean_t complex = FALSE; 2467 vm_size_t space_needed = 0; 2468 vm_offset_t paddr = 0; 2469 vm_map_copy_t copy = VM_MAP_COPY_NULL; 2470 mach_msg_type_number_t i; 2471 mach_msg_return_t mr = MACH_MSG_SUCCESS; 2472 2473 vm_size_t descriptor_size = 0; 2474 2475 /* 2476 * Determine if the target is a kernel port. 2477 */ 2478 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 2479 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 2480 naddr = (mach_msg_descriptor_t *) (body + 1); 2481 2482 dsc_count = body->msgh_descriptor_count; 2483 if (dsc_count == 0) 2484 return MACH_MSG_SUCCESS; 2485 2486 /* 2487 * Make an initial pass to determine kernal VM space requirements for 2488 * physical copies and possible contraction of the descriptors from 2489 * processes with pointers larger than the kernel's. 2490 */ 2491 daddr = NULL; 2492 for (i = 0; i < dsc_count; i++) { 2493 mach_msg_size_t size; 2494 2495 daddr = naddr; 2496 2497 /* make sure the descriptor fits in the message */ 2498 if (is_task_64bit) { 2499 switch (daddr->type.type) { 2500 case MACH_MSG_OOL_DESCRIPTOR: 2501 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 2502 case MACH_MSG_OOL_PORTS_DESCRIPTOR: 2503 descriptor_size += 16; 2504 naddr = (typeof(naddr))((vm_offset_t)daddr + 16); 2505 break; 2506 default: 2507 descriptor_size += 12; 2508 naddr = (typeof(naddr))((vm_offset_t)daddr + 12); 2509 break; 2510 } 2511 } else { 2512 descriptor_size += 12; 2513 naddr = (typeof(naddr))((vm_offset_t)daddr + 12); 2514 } 2515 2516 if (naddr > (mach_msg_descriptor_t *) 2517 ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) { 2518 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); 2519 mr = MACH_SEND_MSG_TOO_SMALL; 2520 goto out; 2521 } 2522 2523 switch (daddr->type.type) { 2524 case MACH_MSG_OOL_DESCRIPTOR: 2525 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 2526 size = (is_task_64bit) ? 2527 ((mach_msg_ool_descriptor64_t *)daddr)->size : 2528 daddr->out_of_line.size; 2529 2530 if (daddr->out_of_line.copy != MACH_MSG_PHYSICAL_COPY && 2531 daddr->out_of_line.copy != MACH_MSG_VIRTUAL_COPY) { 2532 /* 2533 * Invalid copy option 2534 */ 2535 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); 2536 mr = MACH_SEND_INVALID_TYPE; 2537 goto out; 2538 } 2539 2540 if ((size >= MSG_OOL_SIZE_SMALL) && 2541 (daddr->out_of_line.copy == MACH_MSG_PHYSICAL_COPY) && 2542 !(daddr->out_of_line.deallocate)) { 2543 2544 /* 2545 * Out-of-line memory descriptor, accumulate kernel 2546 * memory requirements 2547 */ 2548 if (space_needed + round_page(size) <= space_needed) { 2549 /* Overflow dectected */ 2550 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); 2551 mr = MACH_MSG_VM_KERNEL; 2552 goto out; 2553 } 2554 2555 space_needed += round_page(size); 2556 if (space_needed > ipc_kmsg_max_vm_space) { 2557 2558 /* 2559 * Per message kernel memory limit exceeded 2560 */ 2561 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); 2562 mr = MACH_MSG_VM_KERNEL; 2563 goto out; 2564 } 2565 } 2566 } 2567 } 2568 2569 /* 2570 * Allocate space in the pageable kernel ipc copy map for all the 2571 * ool data that is to be physically copied. Map is marked wait for 2572 * space. 2573 */ 2574 if (space_needed) { 2575 if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed, 2576 VM_FLAGS_ANYWHERE) != KERN_SUCCESS) { 2577 ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0); 2578 mr = MACH_MSG_VM_KERNEL; 2579 goto out; 2580 } 2581 } 2582 2583 /* user_addr = just after base as it was copied in */ 2584 user_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); 2585 2586 /* Shift the mach_msg_base_t down to make room for dsc_count*16bytes of descriptors */ 2587 if(descriptor_size != 16*dsc_count) { 2588 vm_offset_t dsc_adjust = 16*dsc_count - descriptor_size; 2589 2590 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); 2591 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust); 2592 2593 /* Update the message size for the larger in-kernel representation */ 2594 kmsg->ikm_header->msgh_size += (mach_msg_size_t)dsc_adjust; 2595 } 2596 2597 2598 /* kern_addr = just after base after it has been (conditionally) moved */ 2599 kern_addr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); 2600 2601 /* handle the OOL regions and port descriptors. */ 2602 for(i=0;i<dsc_count;i++) { 2603 switch (user_addr->type.type) { 2604 case MACH_MSG_PORT_DESCRIPTOR: 2605 user_addr = ipc_kmsg_copyin_port_descriptor((mach_msg_port_descriptor_t *)kern_addr, 2606 (mach_msg_legacy_port_descriptor_t *)user_addr, space, dest, kmsg, &mr); 2607 kern_addr++; 2608 complex = TRUE; 2609 break; 2610 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 2611 case MACH_MSG_OOL_DESCRIPTOR: 2612 user_addr = ipc_kmsg_copyin_ool_descriptor((mach_msg_ool_descriptor_t *)kern_addr, 2613 user_addr, is_task_64bit, &paddr, ©, &space_needed, map, &mr); 2614 kern_addr++; 2615 complex = TRUE; 2616 break; 2617 case MACH_MSG_OOL_PORTS_DESCRIPTOR: 2618 user_addr = ipc_kmsg_copyin_ool_ports_descriptor((mach_msg_ool_ports_descriptor_t *)kern_addr, 2619 user_addr, is_task_64bit, map, space, dest, kmsg, &mr); 2620 kern_addr++; 2621 complex = TRUE; 2622 break; 2623 default: 2624 /* Invalid descriptor */ 2625 mr = MACH_SEND_INVALID_TYPE; 2626 break; 2627 } 2628 2629 if (MACH_MSG_SUCCESS != mr) { 2630 /* clean from start of message descriptors to i */ 2631 ipc_kmsg_clean_partial(kmsg, i, 2632 (mach_msg_descriptor_t *)((mach_msg_base_t *)kmsg->ikm_header + 1), 2633 paddr, space_needed); 2634 goto out; 2635 } 2636 } /* End of loop */ 2637 2638 if (!complex) { 2639 kmsg->ikm_header->msgh_bits &= ~MACH_MSGH_BITS_COMPLEX; 2640 } 2641 out: 2642 return mr; 2643} 2644 2645 2646/* 2647 * Routine: ipc_kmsg_copyin 2648 * Purpose: 2649 * "Copy-in" port rights and out-of-line memory 2650 * in the message. 2651 * 2652 * In all failure cases, the message is left holding 2653 * no rights or memory. However, the message buffer 2654 * is not deallocated. If successful, the message 2655 * contains a valid destination port. 2656 * Conditions: 2657 * Nothing locked. 2658 * Returns: 2659 * MACH_MSG_SUCCESS Successful copyin. 2660 * MACH_SEND_INVALID_HEADER 2661 * Illegal value in the message header bits. 2662 * MACH_SEND_INVALID_DEST Can't copyin destination port. 2663 * MACH_SEND_INVALID_REPLY Can't copyin reply port. 2664 * MACH_SEND_INVALID_MEMORY Can't grab out-of-line memory. 2665 * MACH_SEND_INVALID_RIGHT Can't copyin port right in body. 2666 * MACH_SEND_INVALID_TYPE Bad type specification. 2667 * MACH_SEND_MSG_TOO_SMALL Body is too small for types/data. 2668 */ 2669 2670mach_msg_return_t 2671ipc_kmsg_copyin( 2672 ipc_kmsg_t kmsg, 2673 ipc_space_t space, 2674 vm_map_t map, 2675 mach_msg_option_t *optionp) 2676{ 2677 mach_msg_return_t mr; 2678 2679 kmsg->ikm_header->msgh_bits &= MACH_MSGH_BITS_USER; 2680 2681 mr = ipc_kmsg_copyin_header(kmsg, space, optionp); 2682 2683 if (mr != MACH_MSG_SUCCESS) 2684 return mr; 2685 2686 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC,MACH_IPC_MSG_SEND) | DBG_FUNC_NONE, 2687 VM_KERNEL_ADDRPERM((uintptr_t)kmsg), 2688 (uintptr_t)kmsg->ikm_header->msgh_bits, 2689 (uintptr_t)kmsg->ikm_header->msgh_id, 2690 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(kmsg->ikm_voucher)), 2691 0); 2692 2693 DEBUG_KPRINT_SYSCALL_IPC("ipc_kmsg_copyin header:\n%.8x\n%.8x\n%p\n%p\n%p\n%.8x\n", 2694 kmsg->ikm_header->msgh_size, 2695 kmsg->ikm_header->msgh_bits, 2696 kmsg->ikm_header->msgh_remote_port, 2697 kmsg->ikm_header->msgh_local_port, 2698 kmsg->ikm_voucher, 2699 kmsg->ikm_header->msgh_id); 2700 2701 if ((kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) == 0) 2702 return MACH_MSG_SUCCESS; 2703 2704 mr = ipc_kmsg_copyin_body( kmsg, space, map); 2705 2706 /* unreachable if !DEBUG */ 2707 __unreachable_ok_push 2708 if (DEBUG_KPRINT_SYSCALL_PREDICATE(DEBUG_KPRINT_SYSCALL_IPC_MASK)) 2709 { 2710 kprintf("body:\n"); 2711 uint32_t i; 2712 for(i=0;i*4 < (kmsg->ikm_header->msgh_size - sizeof(mach_msg_header_t));i++) 2713 { 2714 kprintf("%.4x\n",((uint32_t *)(kmsg->ikm_header + 1))[i]); 2715 } 2716 } 2717 __unreachable_ok_pop 2718 2719 return mr; 2720} 2721 2722/* 2723 * Routine: ipc_kmsg_copyin_from_kernel 2724 * Purpose: 2725 * "Copy-in" port rights and out-of-line memory 2726 * in a message sent from the kernel. 2727 * 2728 * Because the message comes from the kernel, 2729 * the implementation assumes there are no errors 2730 * or peculiarities in the message. 2731 * Conditions: 2732 * Nothing locked. 2733 */ 2734 2735mach_msg_return_t 2736ipc_kmsg_copyin_from_kernel( 2737 ipc_kmsg_t kmsg) 2738{ 2739 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; 2740 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits); 2741 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits); 2742 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 2743 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 2744 2745 /* translate the destination and reply ports */ 2746 if (!IO_VALID(remote)) 2747 return MACH_SEND_INVALID_DEST; 2748 2749 ipc_object_copyin_from_kernel(remote, rname); 2750 if (IO_VALID(local)) 2751 ipc_object_copyin_from_kernel(local, lname); 2752 2753 /* 2754 * The common case is a complex message with no reply port, 2755 * because that is what the memory_object interface uses. 2756 */ 2757 2758 if (bits == (MACH_MSGH_BITS_COMPLEX | 2759 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { 2760 bits = (MACH_MSGH_BITS_COMPLEX | 2761 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); 2762 2763 kmsg->ikm_header->msgh_bits = bits; 2764 } else { 2765 bits = (MACH_MSGH_BITS_OTHER(bits) | 2766 MACH_MSGH_BITS(ipc_object_copyin_type(rname), 2767 ipc_object_copyin_type(lname))); 2768 2769 kmsg->ikm_header->msgh_bits = bits; 2770 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) 2771 return MACH_MSG_SUCCESS; 2772 } 2773 { 2774 mach_msg_descriptor_t *saddr; 2775 mach_msg_body_t *body; 2776 mach_msg_type_number_t i, count; 2777 2778 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 2779 saddr = (mach_msg_descriptor_t *) (body + 1); 2780 count = body->msgh_descriptor_count; 2781 2782 for (i = 0; i < count; i++, saddr++) { 2783 2784 switch (saddr->type.type) { 2785 2786 case MACH_MSG_PORT_DESCRIPTOR: { 2787 mach_msg_type_name_t name; 2788 ipc_object_t object; 2789 mach_msg_port_descriptor_t *dsc; 2790 2791 dsc = &saddr->port; 2792 2793 /* this is really the type SEND, SEND_ONCE, etc. */ 2794 name = dsc->disposition; 2795 object = (ipc_object_t) dsc->name; 2796 dsc->disposition = ipc_object_copyin_type(name); 2797 2798 if (!IO_VALID(object)) { 2799 break; 2800 } 2801 2802 ipc_object_copyin_from_kernel(object, name); 2803 2804 /* CDY avoid circularity when the destination is also */ 2805 /* the kernel. This check should be changed into an */ 2806 /* assert when the new kobject model is in place since*/ 2807 /* ports will not be used in kernel to kernel chats */ 2808 2809 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) { 2810 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && 2811 ipc_port_check_circularity((ipc_port_t) object, 2812 (ipc_port_t) remote)) { 2813 kmsg->ikm_header->msgh_bits |= 2814 MACH_MSGH_BITS_CIRCULAR; 2815 } 2816 } 2817 break; 2818 } 2819 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 2820 case MACH_MSG_OOL_DESCRIPTOR: { 2821 /* 2822 * The sender should supply ready-made memory, i.e. 2823 * a vm_map_copy_t, so we don't need to do anything. 2824 */ 2825 break; 2826 } 2827 case MACH_MSG_OOL_PORTS_DESCRIPTOR: { 2828 ipc_object_t *objects; 2829 unsigned int j; 2830 mach_msg_type_name_t name; 2831 mach_msg_ool_ports_descriptor_t *dsc; 2832 2833 dsc = (mach_msg_ool_ports_descriptor_t *)&saddr->ool_ports; 2834 2835 /* this is really the type SEND, SEND_ONCE, etc. */ 2836 name = dsc->disposition; 2837 dsc->disposition = ipc_object_copyin_type(name); 2838 2839 objects = (ipc_object_t *) dsc->address; 2840 2841 for ( j = 0; j < dsc->count; j++) { 2842 ipc_object_t object = objects[j]; 2843 2844 if (!IO_VALID(object)) 2845 continue; 2846 2847 ipc_object_copyin_from_kernel(object, name); 2848 2849 if ((dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && 2850 ipc_port_check_circularity( 2851 (ipc_port_t) object, 2852 (ipc_port_t) remote)) 2853 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; 2854 } 2855 break; 2856 } 2857 default: { 2858#if MACH_ASSERT 2859 panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); 2860#endif /* MACH_ASSERT */ 2861 } 2862 } 2863 } 2864 } 2865 return MACH_MSG_SUCCESS; 2866} 2867 2868#if IKM_SUPPORT_LEGACY 2869mach_msg_return_t 2870ipc_kmsg_copyin_from_kernel_legacy( 2871 ipc_kmsg_t kmsg) 2872{ 2873 mach_msg_bits_t bits = kmsg->ikm_header->msgh_bits; 2874 mach_msg_type_name_t rname = MACH_MSGH_BITS_REMOTE(bits); 2875 mach_msg_type_name_t lname = MACH_MSGH_BITS_LOCAL(bits); 2876 ipc_object_t remote = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 2877 ipc_object_t local = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 2878 2879 /* translate the destination and reply ports */ 2880 if (!IO_VALID(remote)) 2881 return MACH_SEND_INVALID_DEST; 2882 2883 ipc_object_copyin_from_kernel(remote, rname); 2884 if (IO_VALID(local)) 2885 ipc_object_copyin_from_kernel(local, lname); 2886 2887 /* 2888 * The common case is a complex message with no reply port, 2889 * because that is what the memory_object interface uses. 2890 */ 2891 2892 if (bits == (MACH_MSGH_BITS_COMPLEX | 2893 MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0))) { 2894 bits = (MACH_MSGH_BITS_COMPLEX | 2895 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 0)); 2896 2897 kmsg->ikm_header->msgh_bits = bits; 2898 } else { 2899 bits = (MACH_MSGH_BITS_OTHER(bits) | 2900 MACH_MSGH_BITS(ipc_object_copyin_type(rname), 2901 ipc_object_copyin_type(lname))); 2902 2903 kmsg->ikm_header->msgh_bits = bits; 2904 if ((bits & MACH_MSGH_BITS_COMPLEX) == 0) 2905 return MACH_MSG_SUCCESS; 2906 } 2907 { 2908 mach_msg_legacy_descriptor_t *saddr; 2909 mach_msg_descriptor_t *daddr; 2910 mach_msg_body_t *body; 2911 mach_msg_type_number_t i, count; 2912 2913 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 2914 saddr = (typeof(saddr)) (body + 1); 2915 count = body->msgh_descriptor_count; 2916 2917 if(count) { 2918 vm_offset_t dsc_adjust = 4*count; 2919 memmove((char *)(((vm_offset_t)kmsg->ikm_header) - dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); 2920 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header - dsc_adjust); 2921 /* Update the message size for the larger in-kernel representation */ 2922 kmsg->ikm_header->msgh_size += dsc_adjust; 2923 } 2924 daddr = (mach_msg_descriptor_t *)((vm_offset_t)kmsg->ikm_header + sizeof(mach_msg_base_t)); 2925 2926 for (i = 0; i < count; i++, saddr++, daddr++) { 2927 switch (saddr->type.type) { 2928 2929 case MACH_MSG_PORT_DESCRIPTOR: { 2930 mach_msg_type_name_t name; 2931 ipc_object_t object; 2932 mach_msg_legacy_port_descriptor_t *dsc; 2933 mach_msg_port_descriptor_t *dest_dsc; 2934 2935 dsc = (typeof(dsc))&saddr->port; 2936 dest_dsc = &daddr->port; 2937 2938 /* this is really the type SEND, SEND_ONCE, etc. */ 2939 name = dsc->disposition; 2940 object = (ipc_object_t) CAST_MACH_NAME_TO_PORT(dsc->name); 2941 dest_dsc->disposition = ipc_object_copyin_type(name); 2942 dest_dsc->name = (mach_port_t)object; 2943 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR; 2944 2945 if (!IO_VALID(object)) { 2946 break; 2947 } 2948 2949 ipc_object_copyin_from_kernel(object, name); 2950 2951 /* CDY avoid circularity when the destination is also */ 2952 /* the kernel. This check should be changed into an */ 2953 /* assert when the new kobject model is in place since*/ 2954 /* ports will not be used in kernel to kernel chats */ 2955 2956 if (((ipc_port_t)remote)->ip_receiver != ipc_space_kernel) { 2957 if ((dest_dsc->disposition == MACH_MSG_TYPE_PORT_RECEIVE) && 2958 ipc_port_check_circularity((ipc_port_t) object, 2959 (ipc_port_t) remote)) { 2960 kmsg->ikm_header->msgh_bits |= 2961 MACH_MSGH_BITS_CIRCULAR; 2962 } 2963 } 2964 break; 2965 } 2966 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 2967 case MACH_MSG_OOL_DESCRIPTOR: { 2968 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t 2969 * so we don't need to do anything special. */ 2970 2971 mach_msg_ool_descriptor32_t *source_dsc = &saddr->out_of_line32; 2972 mach_msg_ool_descriptor_t *dest_dsc = (typeof(dest_dsc))&daddr->out_of_line; 2973 2974 vm_offset_t address = source_dsc->address; 2975 vm_size_t size = source_dsc->size; 2976 boolean_t deallocate = source_dsc->deallocate; 2977 mach_msg_copy_options_t copy = source_dsc->copy; 2978 mach_msg_descriptor_type_t type = source_dsc->type; 2979 2980 dest_dsc->address = (void *)address; 2981 dest_dsc->size = size; 2982 dest_dsc->deallocate = deallocate; 2983 dest_dsc->copy = copy; 2984 dest_dsc->type = type; 2985 break; 2986 } 2987 case MACH_MSG_OOL_PORTS_DESCRIPTOR: { 2988 ipc_object_t *objects; 2989 unsigned int j; 2990 mach_msg_type_name_t name; 2991 mach_msg_ool_ports_descriptor_t *dest_dsc; 2992 2993 mach_msg_ool_ports_descriptor32_t *source_dsc = &saddr->ool_ports32; 2994 dest_dsc = (typeof(dest_dsc))&daddr->ool_ports; 2995 2996 boolean_t deallocate = source_dsc->deallocate; 2997 mach_msg_copy_options_t copy = source_dsc->copy; 2998 mach_msg_size_t port_count = source_dsc->count; 2999 mach_msg_type_name_t disposition = source_dsc->disposition; 3000 3001 /* this is really the type SEND, SEND_ONCE, etc. */ 3002 name = disposition; 3003 disposition = ipc_object_copyin_type(name); 3004 3005 objects = (ipc_object_t *) (uintptr_t)source_dsc->address; 3006 3007 for ( j = 0; j < port_count; j++) { 3008 ipc_object_t object = objects[j]; 3009 3010 if (!IO_VALID(object)) 3011 continue; 3012 3013 ipc_object_copyin_from_kernel(object, name); 3014 3015 if ((disposition == MACH_MSG_TYPE_PORT_RECEIVE) && 3016 ipc_port_check_circularity( 3017 (ipc_port_t) object, 3018 (ipc_port_t) remote)) 3019 kmsg->ikm_header->msgh_bits |= MACH_MSGH_BITS_CIRCULAR; 3020 } 3021 3022 dest_dsc->address = objects; 3023 dest_dsc->deallocate = deallocate; 3024 dest_dsc->copy = copy; 3025 dest_dsc->disposition = disposition; 3026 dest_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; 3027 dest_dsc->count = port_count; 3028 break; 3029 } 3030 default: { 3031#if MACH_ASSERT 3032 panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); 3033#endif /* MACH_ASSERT */ 3034 } 3035 } 3036 } 3037 } 3038 return MACH_MSG_SUCCESS; 3039} 3040#endif /* IKM_SUPPORT_LEGACY */ 3041 3042/* 3043 * Routine: ipc_kmsg_copyout_header 3044 * Purpose: 3045 * "Copy-out" port rights in the header of a message. 3046 * Operates atomically; if it doesn't succeed the 3047 * message header and the space are left untouched. 3048 * If it does succeed the remote/local port fields 3049 * contain port names instead of object pointers, 3050 * and the bits field is updated. 3051 * Conditions: 3052 * Nothing locked. 3053 * Returns: 3054 * MACH_MSG_SUCCESS Copied out port rights. 3055 * MACH_RCV_INVALID_NOTIFY 3056 * Notify is non-null and doesn't name a receive right. 3057 * (Either KERN_INVALID_NAME or KERN_INVALID_RIGHT.) 3058 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE 3059 * The space is dead. 3060 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE 3061 * No room in space for another name. 3062 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL 3063 * Couldn't allocate memory for the reply port. 3064 * MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_KERNEL 3065 * Couldn't allocate memory for the dead-name request. 3066 */ 3067 3068mach_msg_return_t 3069ipc_kmsg_copyout_header( 3070 ipc_kmsg_t kmsg, 3071 ipc_space_t space, 3072 mach_msg_option_t option) 3073{ 3074 mach_msg_header_t *msg = kmsg->ikm_header; 3075 mach_msg_bits_t mbits = msg->msgh_bits; 3076 ipc_port_t dest = (ipc_port_t) msg->msgh_remote_port; 3077 3078 assert(IP_VALID(dest)); 3079 3080 /* 3081 * While we still hold a reference on the received-from port, 3082 * process all send-possible notfications we received along with 3083 * the message. 3084 */ 3085 ipc_port_spnotify(dest); 3086 3087 { 3088 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); 3089 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); 3090 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); 3091 ipc_port_t reply = msg->msgh_local_port; 3092 ipc_port_t release_reply_port = IP_NULL; 3093 mach_port_name_t dest_name, reply_name; 3094 3095 ipc_port_t voucher = kmsg->ikm_voucher; 3096 ipc_port_t release_voucher_port = IP_NULL; 3097 mach_port_name_t voucher_name; 3098 3099 uint32_t entries_held = 0; 3100 boolean_t need_write_lock = FALSE; 3101 kern_return_t kr; 3102 3103 /* 3104 * Reserve any potentially needed entries in the target space. 3105 * We'll free any unused before unlocking the space. 3106 */ 3107 if (IP_VALID(reply)) { 3108 entries_held++; 3109 need_write_lock = TRUE; 3110 } 3111 if (IP_VALID(voucher)) { 3112 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); 3113 3114 if ((option & MACH_RCV_VOUCHER) != 0) 3115 entries_held++; 3116 need_write_lock = TRUE; 3117 } 3118 3119 if (need_write_lock) { 3120 3121 is_write_lock(space); 3122 3123 while(entries_held) { 3124 if (!is_active(space)) { 3125 is_write_unlock(space); 3126 return (MACH_RCV_HEADER_ERROR| 3127 MACH_MSG_IPC_SPACE); 3128 } 3129 3130 kr = ipc_entries_hold(space, entries_held); 3131 if (KERN_SUCCESS == kr) 3132 break; 3133 3134 kr = ipc_entry_grow_table(space, ITS_SIZE_NONE); 3135 if (KERN_SUCCESS != kr) 3136 return(MACH_RCV_HEADER_ERROR| 3137 MACH_MSG_IPC_SPACE); 3138 /* space was unlocked and relocked - retry */ 3139 } 3140 3141 /* Handle reply port. */ 3142 if (IP_VALID(reply)) { 3143 ipc_entry_t entry; 3144 3145 /* Is there already an entry we can use? */ 3146 if ((reply_type != MACH_MSG_TYPE_PORT_SEND_ONCE) && 3147 ipc_right_reverse(space, (ipc_object_t) reply, &reply_name, &entry)) { 3148 /* reply port is locked and active */ 3149 assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE); 3150 } else { 3151 ip_lock(reply); 3152 if (!ip_active(reply)) { 3153 ip_unlock(reply); 3154 3155 release_reply_port = reply; 3156 reply = IP_DEAD; 3157 reply_name = MACH_PORT_DEAD; 3158 goto done_with_reply; 3159 } 3160 3161 /* claim a held entry for the reply port */ 3162 assert(entries_held > 0); 3163 entries_held--; 3164 ipc_entry_claim(space, &reply_name, &entry); 3165 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE); 3166 assert(entry->ie_object == IO_NULL); 3167 entry->ie_object = (ipc_object_t) reply; 3168 } 3169 3170 /* space and reply port are locked and active */ 3171 ip_reference(reply); /* hold onto the reply port */ 3172 3173 kr = ipc_right_copyout(space, reply_name, entry, 3174 reply_type, TRUE, (ipc_object_t) reply); 3175 assert(kr == KERN_SUCCESS); 3176 /* reply port is unlocked */ 3177 } else 3178 reply_name = CAST_MACH_PORT_TO_NAME(reply); 3179 3180 done_with_reply: 3181 3182 /* Handle voucher port. */ 3183 if (voucher_type != MACH_MSGH_BITS_ZERO) { 3184 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); 3185 3186 if (!IP_VALID(voucher)) { 3187 if ((option & MACH_RCV_VOUCHER) == 0) { 3188 voucher_type = MACH_MSGH_BITS_ZERO; 3189 } 3190 voucher_name = MACH_PORT_NULL; 3191 goto done_with_voucher; 3192 } 3193 3194 /* clear voucher from its hiding place back in the kmsg */ 3195 kmsg->ikm_voucher = IP_NULL; 3196 3197 if ((option & MACH_RCV_VOUCHER) != 0) { 3198 ipc_entry_t entry; 3199 3200 if (ipc_right_reverse(space, (ipc_object_t) voucher, 3201 &voucher_name, &entry)) { 3202 /* voucher port locked */ 3203 assert(entry->ie_bits & MACH_PORT_TYPE_SEND); 3204 } else { 3205 assert(entries_held > 0); 3206 entries_held--; 3207 ipc_entry_claim(space, &voucher_name, &entry); 3208 assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE); 3209 assert(entry->ie_object == IO_NULL); 3210 entry->ie_object = (ipc_object_t) voucher; 3211 ip_lock(voucher); 3212 } 3213 /* space is locked and active */ 3214 3215 assert(ip_active(voucher)); 3216 assert(ip_kotype(voucher) == IKOT_VOUCHER); 3217 kr = ipc_right_copyout(space, voucher_name, entry, 3218 MACH_MSG_TYPE_MOVE_SEND, TRUE, 3219 (ipc_object_t) voucher); 3220 /* voucher port is unlocked */ 3221 } else { 3222 voucher_type = MACH_MSGH_BITS_ZERO; 3223 release_voucher_port = voucher; 3224 voucher_name = MACH_PORT_NULL; 3225 } 3226 } else { 3227 voucher_name = msg->msgh_voucher_port; 3228 } 3229 3230 done_with_voucher: 3231 3232 ip_lock(dest); 3233 is_write_unlock(space); 3234 3235 } else { 3236 /* 3237 * No reply or voucher port! This is an easy case. 3238 * We only need to have the space locked 3239 * when locking the destination. 3240 */ 3241 3242 is_read_lock(space); 3243 if (!is_active(space)) { 3244 is_read_unlock(space); 3245 return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE; 3246 } 3247 3248 ip_lock(dest); 3249 is_read_unlock(space); 3250 3251 reply_name = CAST_MACH_PORT_TO_NAME(reply); 3252 3253 if (voucher_type != MACH_MSGH_BITS_ZERO) { 3254 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); 3255 if ((option & MACH_RCV_VOUCHER) == 0) { 3256 voucher_type = MACH_MSGH_BITS_ZERO; 3257 } 3258 voucher_name = MACH_PORT_NULL; 3259 } else { 3260 voucher_name = msg->msgh_voucher_port; 3261 } 3262 } 3263 3264 /* 3265 * At this point, the space is unlocked and the destination 3266 * port is locked. (Lock taken while space was locked.) 3267 * reply_name is taken care of; we still need dest_name. 3268 * We still hold a ref for reply (if it is valid). 3269 * 3270 * If the space holds receive rights for the destination, 3271 * we return its name for the right. Otherwise the task 3272 * managed to destroy or give away the receive right between 3273 * receiving the message and this copyout. If the destination 3274 * is dead, return MACH_PORT_DEAD, and if the receive right 3275 * exists somewhere else (another space, in transit) 3276 * return MACH_PORT_NULL. 3277 * 3278 * Making this copyout operation atomic with the previous 3279 * copyout of the reply port is a bit tricky. If there was 3280 * no real reply port (it wasn't IP_VALID) then this isn't 3281 * an issue. If the reply port was dead at copyout time, 3282 * then we are OK, because if dest is dead we serialize 3283 * after the death of both ports and if dest is alive 3284 * we serialize after reply died but before dest's (later) death. 3285 * So assume reply was alive when we copied it out. If dest 3286 * is alive, then we are OK because we serialize before 3287 * the ports' deaths. So assume dest is dead when we look at it. 3288 * If reply dies/died after dest, then we are OK because 3289 * we serialize after dest died but before reply dies. 3290 * So the hard case is when reply is alive at copyout, 3291 * dest is dead at copyout, and reply died before dest died. 3292 * In this case pretend that dest is still alive, so 3293 * we serialize while both ports are alive. 3294 * 3295 * Because the space lock is held across the copyout of reply 3296 * and locking dest, the receive right for dest can't move 3297 * in or out of the space while the copyouts happen, so 3298 * that isn't an atomicity problem. In the last hard case 3299 * above, this implies that when dest is dead that the 3300 * space couldn't have had receive rights for dest at 3301 * the time reply was copied-out, so when we pretend 3302 * that dest is still alive, we can return MACH_PORT_NULL. 3303 * 3304 * If dest == reply, then we have to make it look like 3305 * either both copyouts happened before the port died, 3306 * or both happened after the port died. This special 3307 * case works naturally if the timestamp comparison 3308 * is done correctly. 3309 */ 3310 3311 if (ip_active(dest)) { 3312 ipc_object_copyout_dest(space, (ipc_object_t) dest, 3313 dest_type, &dest_name); 3314 /* dest is unlocked */ 3315 3316 } else { 3317 ipc_port_timestamp_t timestamp; 3318 3319 timestamp = dest->ip_timestamp; 3320 ip_unlock(dest); 3321 ip_release(dest); 3322 3323 if (IP_VALID(reply)) { 3324 ip_lock(reply); 3325 if (ip_active(reply) || 3326 IP_TIMESTAMP_ORDER(timestamp, 3327 reply->ip_timestamp)) 3328 dest_name = MACH_PORT_DEAD; 3329 else 3330 dest_name = MACH_PORT_NULL; 3331 ip_unlock(reply); 3332 } else 3333 dest_name = MACH_PORT_DEAD; 3334 } 3335 3336 if (IP_VALID(reply)) 3337 ip_release(reply); 3338 3339 if (IP_VALID(release_reply_port)) { 3340 if (reply_type == MACH_MSG_TYPE_PORT_SEND_ONCE) 3341 ipc_port_release_sonce(release_reply_port); 3342 else 3343 ipc_port_release_send(release_reply_port); 3344 } 3345 3346 if (IP_VALID(release_voucher_port)) 3347 ipc_port_release_send(release_voucher_port); 3348 3349 3350 if ((option & MACH_RCV_VOUCHER) != 0) { 3351 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV) | DBG_FUNC_NONE, 3352 VM_KERNEL_ADDRPERM((uintptr_t)kmsg), 3353 (uintptr_t)kmsg->ikm_header->msgh_bits, 3354 (uintptr_t)kmsg->ikm_header->msgh_id, 3355 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)), 3356 0); 3357 } else { 3358 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_IPC, MACH_IPC_MSG_RECV_VOUCHER_REFUSED) | DBG_FUNC_NONE, 3359 VM_KERNEL_ADDRPERM((uintptr_t)kmsg), 3360 (uintptr_t)kmsg->ikm_header->msgh_bits, 3361 (uintptr_t)kmsg->ikm_header->msgh_id, 3362 VM_KERNEL_ADDRPERM((uintptr_t)unsafe_convert_port_to_voucher(voucher)), 3363 0); 3364 } 3365 3366 msg->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, 3367 voucher_type, mbits); 3368 msg->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); 3369 msg->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); 3370 msg->msgh_voucher_port = voucher_name; 3371 } 3372 3373 return MACH_MSG_SUCCESS; 3374} 3375 3376/* 3377 * Routine: ipc_kmsg_copyout_object 3378 * Purpose: 3379 * Copy-out a port right. Always returns a name, 3380 * even for unsuccessful return codes. Always 3381 * consumes the supplied object. 3382 * Conditions: 3383 * Nothing locked. 3384 * Returns: 3385 * MACH_MSG_SUCCESS The space acquired the right 3386 * (name is valid) or the object is dead (MACH_PORT_DEAD). 3387 * MACH_MSG_IPC_SPACE No room in space for the right, 3388 * or the space is dead. (Name is MACH_PORT_NULL.) 3389 * MACH_MSG_IPC_KERNEL Kernel resource shortage. 3390 * (Name is MACH_PORT_NULL.) 3391 */ 3392 3393mach_msg_return_t 3394ipc_kmsg_copyout_object( 3395 ipc_space_t space, 3396 ipc_object_t object, 3397 mach_msg_type_name_t msgt_name, 3398 mach_port_name_t *namep) 3399{ 3400 kern_return_t kr; 3401 3402 if (!IO_VALID(object)) { 3403 *namep = CAST_MACH_PORT_TO_NAME(object); 3404 return MACH_MSG_SUCCESS; 3405 } 3406 3407 kr = ipc_object_copyout(space, object, msgt_name, TRUE, namep); 3408 if (kr != KERN_SUCCESS) { 3409 ipc_object_destroy(object, msgt_name); 3410 3411 if (kr == KERN_INVALID_CAPABILITY) 3412 *namep = MACH_PORT_DEAD; 3413 else { 3414 *namep = MACH_PORT_NULL; 3415 3416 if (kr == KERN_RESOURCE_SHORTAGE) 3417 return MACH_MSG_IPC_KERNEL; 3418 else 3419 return MACH_MSG_IPC_SPACE; 3420 } 3421 } 3422 3423 return MACH_MSG_SUCCESS; 3424} 3425 3426mach_msg_descriptor_t * 3427ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc, 3428 mach_msg_descriptor_t *user_dsc, 3429 ipc_space_t space, 3430 kern_return_t *mr); 3431mach_msg_descriptor_t * 3432ipc_kmsg_copyout_port_descriptor(mach_msg_descriptor_t *dsc, 3433 mach_msg_descriptor_t *dest_dsc, 3434 ipc_space_t space, 3435 kern_return_t *mr) 3436{ 3437 mach_port_t port; 3438 mach_port_name_t name; 3439 mach_msg_type_name_t disp; 3440 3441 3442 /* Copyout port right carried in the message */ 3443 port = dsc->port.name; 3444 disp = dsc->port.disposition; 3445 *mr |= ipc_kmsg_copyout_object(space, 3446 (ipc_object_t)port, 3447 disp, 3448 &name); 3449 3450 if(current_task() == kernel_task) 3451 { 3452 mach_msg_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; 3453 user_dsc--; // point to the start of this port descriptor 3454 user_dsc->name = CAST_MACH_NAME_TO_PORT(name); 3455 user_dsc->disposition = disp; 3456 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; 3457 dest_dsc = (typeof(dest_dsc))user_dsc; 3458 } else { 3459 mach_msg_legacy_port_descriptor_t *user_dsc = (typeof(user_dsc))dest_dsc; 3460 user_dsc--; // point to the start of this port descriptor 3461 user_dsc->name = CAST_MACH_PORT_TO_NAME(name); 3462 user_dsc->disposition = disp; 3463 user_dsc->type = MACH_MSG_PORT_DESCRIPTOR; 3464 dest_dsc = (typeof(dest_dsc))user_dsc; 3465 } 3466 3467 return (mach_msg_descriptor_t *)dest_dsc; 3468} 3469 3470mach_msg_descriptor_t * 3471ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr); 3472mach_msg_descriptor_t * 3473ipc_kmsg_copyout_ool_descriptor(mach_msg_ool_descriptor_t *dsc, mach_msg_descriptor_t *user_dsc, int is_64bit, vm_map_t map, mach_msg_return_t *mr) 3474{ 3475 vm_map_copy_t copy; 3476 vm_map_address_t rcv_addr; 3477 mach_msg_copy_options_t copy_options; 3478 mach_msg_size_t size; 3479 mach_msg_descriptor_type_t dsc_type; 3480 3481 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); 3482 3483 copy = (vm_map_copy_t) dsc->address; 3484 size = dsc->size; 3485 copy_options = dsc->copy; 3486 assert(copy_options != MACH_MSG_KALLOC_COPY_T); 3487 dsc_type = dsc->type; 3488 rcv_addr = 0; 3489 3490 if (copy != VM_MAP_COPY_NULL) { 3491 /* 3492 * Check to see if there is an overwrite descriptor 3493 * specified in the scatter list for this ool data. 3494 * The descriptor has already been verified. 3495 */ 3496#if 0 3497 if (saddr != MACH_MSG_DESCRIPTOR_NULL) { 3498 if (differs) { 3499 OTHER_OOL_DESCRIPTOR *scatter_dsc; 3500 3501 scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr; 3502 if (scatter_dsc->copy == MACH_MSG_OVERWRITE) { 3503 rcv_addr = (mach_vm_offset_t) scatter_dsc->address; 3504 copy_options = MACH_MSG_OVERWRITE; 3505 } else { 3506 copy_options = MACH_MSG_VIRTUAL_COPY; 3507 } 3508 } else { 3509 mach_msg_ool_descriptor_t *scatter_dsc; 3510 3511 scatter_dsc = &saddr->out_of_line; 3512 if (scatter_dsc->copy == MACH_MSG_OVERWRITE) { 3513 rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address); 3514 copy_options = MACH_MSG_OVERWRITE; 3515 } else { 3516 copy_options = MACH_MSG_VIRTUAL_COPY; 3517 } 3518 } 3519 INCREMENT_SCATTER(saddr, sdsc_count, differs); 3520 } 3521#endif 3522 3523 3524 /* 3525 * Whether the data was virtually or physically 3526 * copied we have a vm_map_copy_t for it. 3527 * If there's an overwrite region specified 3528 * overwrite it, otherwise do a virtual copy out. 3529 */ 3530 kern_return_t kr; 3531 if (copy_options == MACH_MSG_OVERWRITE && rcv_addr != 0) { 3532 kr = vm_map_copy_overwrite(map, rcv_addr, 3533 copy, TRUE); 3534 } else { 3535 kr = vm_map_copyout(map, &rcv_addr, copy); 3536 } 3537 if (kr != KERN_SUCCESS) { 3538 if (kr == KERN_RESOURCE_SHORTAGE) 3539 *mr |= MACH_MSG_VM_KERNEL; 3540 else 3541 *mr |= MACH_MSG_VM_SPACE; 3542 vm_map_copy_discard(copy); 3543 rcv_addr = 0; 3544 size = 0; 3545 } 3546 } else { 3547 rcv_addr = 0; 3548 size = 0; 3549 } 3550 3551 /* 3552 * Now update the descriptor as the user would see it. 3553 * This may require expanding the descriptor to the user 3554 * visible size. There is already space allocated for 3555 * this in what naddr points to. 3556 */ 3557 if(current_task() == kernel_task) 3558 { 3559 mach_msg_ool_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 3560 user_ool_dsc--; 3561 3562 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; 3563 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? 3564 TRUE : FALSE; 3565 user_ool_dsc->copy = copy_options; 3566 user_ool_dsc->type = dsc_type; 3567 user_ool_dsc->size = size; 3568 3569 user_dsc = (typeof(user_dsc))user_ool_dsc; 3570 } else if (is_64bit) { 3571 mach_msg_ool_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 3572 user_ool_dsc--; 3573 3574 user_ool_dsc->address = rcv_addr; 3575 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? 3576 TRUE : FALSE; 3577 user_ool_dsc->copy = copy_options; 3578 user_ool_dsc->type = dsc_type; 3579 user_ool_dsc->size = size; 3580 3581 user_dsc = (typeof(user_dsc))user_ool_dsc; 3582 } else { 3583 mach_msg_ool_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 3584 user_ool_dsc--; 3585 3586 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); 3587 user_ool_dsc->size = size; 3588 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? 3589 TRUE : FALSE; 3590 user_ool_dsc->copy = copy_options; 3591 user_ool_dsc->type = dsc_type; 3592 3593 user_dsc = (typeof(user_dsc))user_ool_dsc; 3594 } 3595 return user_dsc; 3596} 3597 3598mach_msg_descriptor_t * 3599ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, 3600 mach_msg_descriptor_t *user_dsc, 3601 int is_64bit, 3602 vm_map_t map, 3603 ipc_space_t space, 3604 ipc_kmsg_t kmsg, 3605 mach_msg_return_t *mr); 3606mach_msg_descriptor_t * 3607ipc_kmsg_copyout_ool_ports_descriptor(mach_msg_ool_ports_descriptor_t *dsc, 3608 mach_msg_descriptor_t *user_dsc, 3609 int is_64bit, 3610 vm_map_t map, 3611 ipc_space_t space, 3612 ipc_kmsg_t kmsg, 3613 mach_msg_return_t *mr) 3614{ 3615 mach_vm_offset_t rcv_addr = 0; 3616 mach_msg_type_name_t disp; 3617 mach_msg_type_number_t count, i; 3618 vm_size_t ports_length, names_length; 3619 3620 mach_msg_copy_options_t copy_options = MACH_MSG_VIRTUAL_COPY; 3621 3622 //SKIP_PORT_DESCRIPTORS(saddr, sdsc_count); 3623 3624 count = dsc->count; 3625 disp = dsc->disposition; 3626 ports_length = count * sizeof(mach_port_t); 3627 names_length = count * sizeof(mach_port_name_t); 3628 3629 if (ports_length != 0 && dsc->address != 0) { 3630 3631 /* 3632 * Check to see if there is an overwrite descriptor 3633 * specified in the scatter list for this ool data. 3634 * The descriptor has already been verified. 3635 */ 3636#if 0 3637 if (saddr != MACH_MSG_DESCRIPTOR_NULL) { 3638 if (differs) { 3639 OTHER_OOL_DESCRIPTOR *scatter_dsc; 3640 3641 scatter_dsc = (OTHER_OOL_DESCRIPTOR *)saddr; 3642 rcv_addr = (mach_vm_offset_t) scatter_dsc->address; 3643 copy_options = scatter_dsc->copy; 3644 } else { 3645 mach_msg_ool_descriptor_t *scatter_dsc; 3646 3647 scatter_dsc = &saddr->out_of_line; 3648 rcv_addr = CAST_USER_ADDR_T(scatter_dsc->address); 3649 copy_options = scatter_dsc->copy; 3650 } 3651 INCREMENT_SCATTER(saddr, sdsc_count, differs); 3652 } 3653#endif 3654 3655 if (copy_options == MACH_MSG_VIRTUAL_COPY) { 3656 /* 3657 * Dynamically allocate the region 3658 */ 3659 int anywhere = VM_MAKE_TAG(VM_MEMORY_MACH_MSG)| 3660 VM_FLAGS_ANYWHERE; 3661 3662 kern_return_t kr; 3663 if ((kr = mach_vm_allocate(map, &rcv_addr, 3664 (mach_vm_size_t)names_length, 3665 anywhere)) != KERN_SUCCESS) { 3666 ipc_kmsg_clean_body(kmsg, 1, (mach_msg_descriptor_t *)dsc); 3667 rcv_addr = 0; 3668 3669 if (kr == KERN_RESOURCE_SHORTAGE){ 3670 *mr |= MACH_MSG_VM_KERNEL; 3671 } else { 3672 *mr |= MACH_MSG_VM_SPACE; 3673 } 3674 } 3675 } 3676 3677 /* 3678 * Handle the port rights and copy out the names 3679 * for those rights out to user-space. 3680 */ 3681 if (rcv_addr != 0) { 3682 mach_port_t *objects = (mach_port_t *) dsc->address; 3683 mach_port_name_t *names = (mach_port_name_t *) dsc->address; 3684 3685 /* copyout port rights carried in the message */ 3686 3687 for ( i = 0; i < count ; i++) { 3688 ipc_object_t object = (ipc_object_t)objects[i]; 3689 3690 *mr |= ipc_kmsg_copyout_object(space, object, 3691 disp, &names[i]); 3692 } 3693 3694 /* copyout to memory allocated above */ 3695 void *data = dsc->address; 3696 if (copyoutmap(map, data, rcv_addr, names_length) != KERN_SUCCESS) 3697 *mr |= MACH_MSG_VM_SPACE; 3698 kfree(data, ports_length); 3699 } 3700 } else { 3701 rcv_addr = 0; 3702 } 3703 3704 /* 3705 * Now update the descriptor based on the information 3706 * calculated above. 3707 */ 3708 if(current_task() == kernel_task) { 3709 mach_msg_ool_ports_descriptor_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 3710 user_ool_dsc--; 3711 3712 user_ool_dsc->address = (void *)(uintptr_t)rcv_addr; 3713 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? 3714 TRUE : FALSE; 3715 user_ool_dsc->copy = copy_options; 3716 user_ool_dsc->disposition = disp; 3717 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; 3718 user_ool_dsc->count = count; 3719 3720 user_dsc = (typeof(user_dsc))user_ool_dsc; 3721 } if (is_64bit) { 3722 mach_msg_ool_ports_descriptor64_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 3723 user_ool_dsc--; 3724 3725 user_ool_dsc->address = rcv_addr; 3726 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? 3727 TRUE : FALSE; 3728 user_ool_dsc->copy = copy_options; 3729 user_ool_dsc->disposition = disp; 3730 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; 3731 user_ool_dsc->count = count; 3732 3733 user_dsc = (typeof(user_dsc))user_ool_dsc; 3734 } else { 3735 mach_msg_ool_ports_descriptor32_t *user_ool_dsc = (typeof(user_ool_dsc))user_dsc; 3736 user_ool_dsc--; 3737 3738 user_ool_dsc->address = CAST_DOWN_EXPLICIT(uint32_t, rcv_addr); 3739 user_ool_dsc->count = count; 3740 user_ool_dsc->deallocate = (copy_options == MACH_MSG_VIRTUAL_COPY) ? 3741 TRUE : FALSE; 3742 user_ool_dsc->copy = copy_options; 3743 user_ool_dsc->disposition = disp; 3744 user_ool_dsc->type = MACH_MSG_OOL_PORTS_DESCRIPTOR; 3745 3746 user_dsc = (typeof(user_dsc))user_ool_dsc; 3747 } 3748 return user_dsc; 3749} 3750 3751/* 3752 * Routine: ipc_kmsg_copyout_body 3753 * Purpose: 3754 * "Copy-out" port rights and out-of-line memory 3755 * in the body of a message. 3756 * 3757 * The error codes are a combination of special bits. 3758 * The copyout proceeds despite errors. 3759 * Conditions: 3760 * Nothing locked. 3761 * Returns: 3762 * MACH_MSG_SUCCESS Successful copyout. 3763 * MACH_MSG_IPC_SPACE No room for port right in name space. 3764 * MACH_MSG_VM_SPACE No room for memory in address space. 3765 * MACH_MSG_IPC_KERNEL Resource shortage handling port right. 3766 * MACH_MSG_VM_KERNEL Resource shortage handling memory. 3767 * MACH_MSG_INVALID_RT_DESCRIPTOR Descriptor incompatible with RT 3768 */ 3769 3770mach_msg_return_t 3771ipc_kmsg_copyout_body( 3772 ipc_kmsg_t kmsg, 3773 ipc_space_t space, 3774 vm_map_t map, 3775 mach_msg_body_t *slist) 3776{ 3777 mach_msg_body_t *body; 3778 mach_msg_descriptor_t *kern_dsc, *user_dsc; 3779 mach_msg_descriptor_t *saddr; 3780 mach_msg_type_number_t dsc_count, sdsc_count; 3781 int i; 3782 mach_msg_return_t mr = MACH_MSG_SUCCESS; 3783 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); 3784 3785 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 3786 dsc_count = body->msgh_descriptor_count; 3787 kern_dsc = (mach_msg_descriptor_t *) (body + 1); 3788 /* Point user_dsc just after the end of all the descriptors */ 3789 user_dsc = &kern_dsc[dsc_count]; 3790 3791 /* Do scatter list setup */ 3792 if (slist != MACH_MSG_BODY_NULL) { 3793 panic("Scatter lists disabled"); 3794 saddr = (mach_msg_descriptor_t *) (slist + 1); 3795 sdsc_count = slist->msgh_descriptor_count; 3796 } 3797 else { 3798 saddr = MACH_MSG_DESCRIPTOR_NULL; 3799 sdsc_count = 0; 3800 } 3801 3802 /* Now process the descriptors */ 3803 for (i = dsc_count-1; i >= 0; i--) { 3804 switch (kern_dsc[i].type.type) { 3805 3806 case MACH_MSG_PORT_DESCRIPTOR: 3807 user_dsc = ipc_kmsg_copyout_port_descriptor(&kern_dsc[i], user_dsc, space, &mr); 3808 break; 3809 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 3810 case MACH_MSG_OOL_DESCRIPTOR : 3811 user_dsc = ipc_kmsg_copyout_ool_descriptor( 3812 (mach_msg_ool_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, &mr); 3813 break; 3814 case MACH_MSG_OOL_PORTS_DESCRIPTOR : 3815 user_dsc = ipc_kmsg_copyout_ool_ports_descriptor( 3816 (mach_msg_ool_ports_descriptor_t *)&kern_dsc[i], user_dsc, is_task_64bit, map, space, kmsg, &mr); 3817 break; 3818 default : { 3819 panic("untyped IPC copyout body: invalid message descriptor"); 3820 } 3821 } 3822 } 3823 3824 if(user_dsc != kern_dsc) { 3825 vm_offset_t dsc_adjust = (vm_offset_t)user_dsc - (vm_offset_t)kern_dsc; 3826 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); 3827 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust); 3828 /* Update the message size for the smaller user representation */ 3829 kmsg->ikm_header->msgh_size -= (mach_msg_size_t)dsc_adjust; 3830 } 3831 3832 return mr; 3833} 3834 3835/* 3836 * Routine: ipc_kmsg_copyout_size 3837 * Purpose: 3838 * Compute the size of the message as copied out to the given 3839 * map. If the destination map's pointers are a different size 3840 * than the kernel's, we have to allow for expansion/ 3841 * contraction of the descriptors as appropriate. 3842 * Conditions: 3843 * Nothing locked. 3844 * Returns: 3845 * size of the message as it would be received. 3846 */ 3847 3848mach_msg_size_t 3849ipc_kmsg_copyout_size( 3850 ipc_kmsg_t kmsg, 3851 vm_map_t map) 3852{ 3853 mach_msg_size_t send_size; 3854 3855 send_size = kmsg->ikm_header->msgh_size; 3856 3857 boolean_t is_task_64bit = (map->max_offset > VM_MAX_ADDRESS); 3858 3859#if defined(__LP64__) 3860 send_size -= LEGACY_HEADER_SIZE_DELTA; 3861#endif 3862 3863 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) { 3864 3865 mach_msg_body_t *body; 3866 mach_msg_descriptor_t *saddr, *eaddr; 3867 3868 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 3869 saddr = (mach_msg_descriptor_t *) (body + 1); 3870 eaddr = saddr + body->msgh_descriptor_count; 3871 3872 for ( ; saddr < eaddr; saddr++ ) { 3873 switch (saddr->type.type) { 3874 case MACH_MSG_OOL_DESCRIPTOR: 3875 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 3876 case MACH_MSG_OOL_PORTS_DESCRIPTOR: 3877 if(!is_task_64bit) 3878 send_size -= DESC_SIZE_ADJUSTMENT; 3879 break; 3880 case MACH_MSG_PORT_DESCRIPTOR: 3881 send_size -= DESC_SIZE_ADJUSTMENT; 3882 break; 3883 default: 3884 break; 3885 } 3886 } 3887 } 3888 return send_size; 3889} 3890 3891/* 3892 * Routine: ipc_kmsg_copyout 3893 * Purpose: 3894 * "Copy-out" port rights and out-of-line memory 3895 * in the message. 3896 * Conditions: 3897 * Nothing locked. 3898 * Returns: 3899 * MACH_MSG_SUCCESS Copied out all rights and memory. 3900 * MACH_RCV_HEADER_ERROR + special bits 3901 * Rights and memory in the message are intact. 3902 * MACH_RCV_BODY_ERROR + special bits 3903 * The message header was successfully copied out. 3904 * As much of the body was handled as possible. 3905 */ 3906 3907mach_msg_return_t 3908ipc_kmsg_copyout( 3909 ipc_kmsg_t kmsg, 3910 ipc_space_t space, 3911 vm_map_t map, 3912 mach_msg_body_t *slist, 3913 mach_msg_option_t option) 3914{ 3915 mach_msg_return_t mr; 3916 3917 mr = ipc_kmsg_copyout_header(kmsg, space, option); 3918 if (mr != MACH_MSG_SUCCESS) { 3919 return mr; 3920 } 3921 3922 if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) { 3923 mr = ipc_kmsg_copyout_body(kmsg, space, map, slist); 3924 3925 if (mr != MACH_MSG_SUCCESS) 3926 mr |= MACH_RCV_BODY_ERROR; 3927 } 3928 3929 return mr; 3930} 3931 3932/* 3933 * Routine: ipc_kmsg_copyout_pseudo 3934 * Purpose: 3935 * Does a pseudo-copyout of the message. 3936 * This is like a regular copyout, except 3937 * that the ports in the header are handled 3938 * as if they are in the body. They aren't reversed. 3939 * 3940 * The error codes are a combination of special bits. 3941 * The copyout proceeds despite errors. 3942 * Conditions: 3943 * Nothing locked. 3944 * Returns: 3945 * MACH_MSG_SUCCESS Successful copyout. 3946 * MACH_MSG_IPC_SPACE No room for port right in name space. 3947 * MACH_MSG_VM_SPACE No room for memory in address space. 3948 * MACH_MSG_IPC_KERNEL Resource shortage handling port right. 3949 * MACH_MSG_VM_KERNEL Resource shortage handling memory. 3950 */ 3951 3952mach_msg_return_t 3953ipc_kmsg_copyout_pseudo( 3954 ipc_kmsg_t kmsg, 3955 ipc_space_t space, 3956 vm_map_t map, 3957 mach_msg_body_t *slist) 3958{ 3959 mach_msg_bits_t mbits = kmsg->ikm_header->msgh_bits; 3960 ipc_object_t dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 3961 ipc_object_t reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 3962 ipc_object_t voucher = (ipc_object_t) kmsg->ikm_voucher; 3963 mach_msg_type_name_t dest_type = MACH_MSGH_BITS_REMOTE(mbits); 3964 mach_msg_type_name_t reply_type = MACH_MSGH_BITS_LOCAL(mbits); 3965 mach_msg_type_name_t voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); 3966 mach_port_name_t voucher_name = kmsg->ikm_header->msgh_voucher_port; 3967 mach_port_name_t dest_name, reply_name; 3968 mach_msg_return_t mr; 3969 3970 assert(IO_VALID(dest)); 3971 3972#if 0 3973 /* 3974 * If we did this here, it looks like we wouldn't need the undo logic 3975 * at the end of ipc_kmsg_send() in the error cases. Not sure which 3976 * would be more elegant to keep. 3977 */ 3978 ipc_importance_clean(kmsg); 3979#else 3980 /* just assert it is already clean */ 3981 ipc_importance_assert_clean(kmsg); 3982#endif 3983 3984 mr = (ipc_kmsg_copyout_object(space, dest, dest_type, &dest_name) | 3985 ipc_kmsg_copyout_object(space, reply, reply_type, &reply_name)); 3986 3987 kmsg->ikm_header->msgh_bits = mbits & MACH_MSGH_BITS_USER; 3988 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(dest_name); 3989 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(reply_name); 3990 3991 if (IO_VALID(voucher)) { 3992 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); 3993 3994 kmsg->ikm_voucher = IP_NULL; 3995 mr |= ipc_kmsg_copyout_object(space, voucher, voucher_type, &voucher_name); 3996 kmsg->ikm_header->msgh_voucher_port = voucher_name; 3997 } 3998 3999 if (mbits & MACH_MSGH_BITS_COMPLEX) { 4000 mr |= ipc_kmsg_copyout_body(kmsg, space, map, slist); 4001 } 4002 4003 return mr; 4004} 4005 4006/* 4007 * Routine: ipc_kmsg_copyout_dest 4008 * Purpose: 4009 * Copies out the destination port in the message. 4010 * Destroys all other rights and memory in the message. 4011 * Conditions: 4012 * Nothing locked. 4013 */ 4014 4015void 4016ipc_kmsg_copyout_dest( 4017 ipc_kmsg_t kmsg, 4018 ipc_space_t space) 4019{ 4020 mach_msg_bits_t mbits; 4021 ipc_object_t dest; 4022 ipc_object_t reply; 4023 ipc_object_t voucher; 4024 mach_msg_type_name_t dest_type; 4025 mach_msg_type_name_t reply_type; 4026 mach_msg_type_name_t voucher_type; 4027 mach_port_name_t dest_name, reply_name, voucher_name; 4028 4029 mbits = kmsg->ikm_header->msgh_bits; 4030 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 4031 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 4032 voucher = (ipc_object_t) kmsg->ikm_voucher; 4033 voucher_name = kmsg->ikm_header->msgh_voucher_port; 4034 dest_type = MACH_MSGH_BITS_REMOTE(mbits); 4035 reply_type = MACH_MSGH_BITS_LOCAL(mbits); 4036 voucher_type = MACH_MSGH_BITS_VOUCHER(mbits); 4037 4038 assert(IO_VALID(dest)); 4039 4040 ipc_importance_assert_clean(kmsg); 4041 4042 io_lock(dest); 4043 if (io_active(dest)) { 4044 ipc_object_copyout_dest(space, dest, dest_type, &dest_name); 4045 /* dest is unlocked */ 4046 } else { 4047 io_unlock(dest); 4048 io_release(dest); 4049 dest_name = MACH_PORT_DEAD; 4050 } 4051 4052 if (IO_VALID(reply)) { 4053 ipc_object_destroy(reply, reply_type); 4054 reply_name = MACH_PORT_NULL; 4055 } else 4056 reply_name = CAST_MACH_PORT_TO_NAME(reply); 4057 4058 if (IO_VALID(voucher)) { 4059 assert(voucher_type == MACH_MSG_TYPE_MOVE_SEND); 4060 4061 kmsg->ikm_voucher = IP_NULL; 4062 ipc_object_destroy((ipc_object_t)voucher, voucher_type); 4063 voucher_name = MACH_PORT_NULL; 4064 } 4065 4066 kmsg->ikm_header->msgh_bits = MACH_MSGH_BITS_SET(reply_type, dest_type, 4067 voucher_type, mbits); 4068 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); 4069 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); 4070 kmsg->ikm_header->msgh_voucher_port = voucher_name; 4071 4072 if (mbits & MACH_MSGH_BITS_COMPLEX) { 4073 mach_msg_body_t *body; 4074 4075 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 4076 ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count, 4077 (mach_msg_descriptor_t *)(body + 1)); 4078 } 4079} 4080 4081/* 4082 * Routine: ipc_kmsg_copyin_scatter 4083 * Purpose: 4084 * allocate and copyin a scatter list 4085 * Algorithm: 4086 * The gather (kmsg) is valid since it has been copied in. 4087 * Gather list descriptors are sequentially paired with scatter 4088 * list descriptors, with port descriptors in either list ignored. 4089 * Descriptors are consistent if the type fileds match and size 4090 * of the scatter descriptor is less than or equal to the 4091 * size of the gather descriptor. A MACH_MSG_ALLOCATE copy 4092 * strategy in a scatter descriptor matches any size in the 4093 * corresponding gather descriptor assuming they are the same type. 4094 * Either list may be larger than the other. During the 4095 * subsequent copy out, excess scatter descriptors are ignored 4096 * and excess gather descriptors default to dynamic allocation. 4097 * 4098 * In the case of a size error, the scatter list is released. 4099 * Conditions: 4100 * Nothing locked. 4101 * Returns: 4102 * the allocated message body containing the scatter list. 4103 */ 4104 4105mach_msg_body_t * 4106ipc_kmsg_get_scatter( 4107 mach_vm_address_t msg_addr, 4108 mach_msg_size_t slist_size, 4109 ipc_kmsg_t kmsg) 4110{ 4111 mach_msg_body_t *slist; 4112 mach_msg_body_t *body; 4113 mach_msg_descriptor_t *gstart, *gend; 4114 mach_msg_descriptor_t *sstart, *send; 4115 4116#if defined(__LP64__) 4117 panic("ipc_kmsg_get_scatter called!"); 4118#endif 4119 4120 if (slist_size < sizeof(mach_msg_base_t)) 4121 return MACH_MSG_BODY_NULL; 4122 4123 slist_size -= (mach_msg_size_t)sizeof(mach_msg_header_t); 4124 slist = (mach_msg_body_t *)kalloc(slist_size); 4125 if (slist == MACH_MSG_BODY_NULL) 4126 return slist; 4127 4128 if (copyin(msg_addr + sizeof(mach_msg_header_t), (char *)slist, slist_size)) { 4129 kfree(slist, slist_size); 4130 return MACH_MSG_BODY_NULL; 4131 } 4132 4133 if ((slist->msgh_descriptor_count* sizeof(mach_msg_descriptor_t) 4134 + sizeof(mach_msg_size_t)) > slist_size) { 4135 kfree(slist, slist_size); 4136 return MACH_MSG_BODY_NULL; 4137 } 4138 4139 body = (mach_msg_body_t *) (kmsg->ikm_header + 1); 4140 gstart = (mach_msg_descriptor_t *) (body + 1); 4141 gend = gstart + body->msgh_descriptor_count; 4142 4143 sstart = (mach_msg_descriptor_t *) (slist + 1); 4144 send = sstart + slist->msgh_descriptor_count; 4145 4146 while (gstart < gend) { 4147 mach_msg_descriptor_type_t g_type; 4148 4149 /* 4150 * Skip port descriptors in gather list. 4151 */ 4152 g_type = gstart->type.type; 4153 4154 if (g_type != MACH_MSG_PORT_DESCRIPTOR) { 4155 4156 /* 4157 * A scatter list with a 0 descriptor count is treated as an 4158 * automatic size mismatch. 4159 */ 4160 if (slist->msgh_descriptor_count == 0) { 4161 kfree(slist, slist_size); 4162 return MACH_MSG_BODY_NULL; 4163 } 4164 4165 /* 4166 * Skip port descriptors in scatter list. 4167 */ 4168 while (sstart < send) { 4169 if (sstart->type.type != MACH_MSG_PORT_DESCRIPTOR) 4170 break; 4171 sstart++; 4172 } 4173 4174 /* 4175 * No more scatter descriptors, we're done 4176 */ 4177 if (sstart >= send) { 4178 break; 4179 } 4180 4181 /* 4182 * Check type, copy and size fields 4183 */ 4184 if (g_type == MACH_MSG_OOL_DESCRIPTOR || 4185 g_type == MACH_MSG_OOL_VOLATILE_DESCRIPTOR) { 4186 if (sstart->type.type != MACH_MSG_OOL_DESCRIPTOR && 4187 sstart->type.type != MACH_MSG_OOL_VOLATILE_DESCRIPTOR) { 4188 kfree(slist, slist_size); 4189 return MACH_MSG_BODY_NULL; 4190 } 4191 if (sstart->out_of_line.copy == MACH_MSG_OVERWRITE && 4192 gstart->out_of_line.size > sstart->out_of_line.size) { 4193 kfree(slist, slist_size); 4194 return MACH_MSG_BODY_NULL; 4195 } 4196 } 4197 else { 4198 if (sstart->type.type != MACH_MSG_OOL_PORTS_DESCRIPTOR) { 4199 kfree(slist, slist_size); 4200 return MACH_MSG_BODY_NULL; 4201 } 4202 if (sstart->ool_ports.copy == MACH_MSG_OVERWRITE && 4203 gstart->ool_ports.count > sstart->ool_ports.count) { 4204 kfree(slist, slist_size); 4205 return MACH_MSG_BODY_NULL; 4206 } 4207 } 4208 sstart++; 4209 } 4210 gstart++; 4211 } 4212 return slist; 4213} 4214 4215 4216/* 4217 * Routine: ipc_kmsg_free_scatter 4218 * Purpose: 4219 * Deallocate a scatter list. Since we actually allocated 4220 * a body without a header, and since the header was originally 4221 * accounted for in slist_size, we have to ajust it down 4222 * before freeing the scatter list. 4223 */ 4224void 4225ipc_kmsg_free_scatter( 4226 mach_msg_body_t *slist, 4227 mach_msg_size_t slist_size) 4228{ 4229#if defined(__LP64__) 4230 panic("%s called; halting!", __func__); 4231#endif 4232 4233 slist_size -= (mach_msg_size_t)sizeof(mach_msg_header_t); 4234 kfree(slist, slist_size); 4235} 4236 4237 4238/* 4239 * Routine: ipc_kmsg_copyout_to_kernel 4240 * Purpose: 4241 * Copies out the destination and reply ports in the message. 4242 * Leaves all other rights and memory in the message alone. 4243 * Conditions: 4244 * Nothing locked. 4245 * 4246 * Derived from ipc_kmsg_copyout_dest. 4247 * Use by mach_msg_rpc_from_kernel (which used to use copyout_dest). 4248 * We really do want to save rights and memory. 4249 */ 4250 4251void 4252ipc_kmsg_copyout_to_kernel( 4253 ipc_kmsg_t kmsg, 4254 ipc_space_t space) 4255{ 4256 ipc_object_t dest; 4257 ipc_object_t reply; 4258 mach_msg_type_name_t dest_type; 4259 mach_msg_type_name_t reply_type; 4260 mach_port_name_t dest_name, reply_name; 4261 4262 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 4263 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 4264 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits); 4265 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits); 4266 4267 assert(IO_VALID(dest)); 4268 4269 io_lock(dest); 4270 if (io_active(dest)) { 4271 ipc_object_copyout_dest(space, dest, dest_type, &dest_name); 4272 /* dest is unlocked */ 4273 } else { 4274 io_unlock(dest); 4275 io_release(dest); 4276 dest_name = MACH_PORT_DEAD; 4277 } 4278 4279 reply_name = CAST_MACH_PORT_TO_NAME(reply); 4280 4281 kmsg->ikm_header->msgh_bits = 4282 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | 4283 MACH_MSGH_BITS(reply_type, dest_type)); 4284 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); 4285 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); 4286} 4287 4288#if IKM_SUPPORT_LEGACY 4289void 4290ipc_kmsg_copyout_to_kernel_legacy( 4291 ipc_kmsg_t kmsg, 4292 ipc_space_t space) 4293{ 4294 ipc_object_t dest; 4295 ipc_object_t reply; 4296 mach_msg_type_name_t dest_type; 4297 mach_msg_type_name_t reply_type; 4298 mach_port_name_t dest_name, reply_name; 4299 4300 dest = (ipc_object_t) kmsg->ikm_header->msgh_remote_port; 4301 reply = (ipc_object_t) kmsg->ikm_header->msgh_local_port; 4302 dest_type = MACH_MSGH_BITS_REMOTE(kmsg->ikm_header->msgh_bits); 4303 reply_type = MACH_MSGH_BITS_LOCAL(kmsg->ikm_header->msgh_bits); 4304 4305 assert(IO_VALID(dest)); 4306 4307 io_lock(dest); 4308 if (io_active(dest)) { 4309 ipc_object_copyout_dest(space, dest, dest_type, &dest_name); 4310 /* dest is unlocked */ 4311 } else { 4312 io_unlock(dest); 4313 io_release(dest); 4314 dest_name = MACH_PORT_DEAD; 4315 } 4316 4317 reply_name = CAST_MACH_PORT_TO_NAME(reply); 4318 4319 kmsg->ikm_header->msgh_bits = 4320 (MACH_MSGH_BITS_OTHER(kmsg->ikm_header->msgh_bits) | 4321 MACH_MSGH_BITS(reply_type, dest_type)); 4322 kmsg->ikm_header->msgh_local_port = CAST_MACH_NAME_TO_PORT(dest_name); 4323 kmsg->ikm_header->msgh_remote_port = CAST_MACH_NAME_TO_PORT(reply_name); 4324 4325 mach_msg_descriptor_t *saddr; 4326 mach_msg_legacy_descriptor_t *daddr; 4327 mach_msg_type_number_t i, count = ((mach_msg_base_t *)kmsg->ikm_header)->body.msgh_descriptor_count; 4328 saddr = (mach_msg_descriptor_t *) (((mach_msg_base_t *)kmsg->ikm_header) + 1); 4329 saddr = &saddr[count-1]; 4330 daddr = (mach_msg_legacy_descriptor_t *)&saddr[count]; 4331 daddr--; 4332 4333 vm_offset_t dsc_adjust = 0; 4334 4335 for (i = 0; i < count; i++, saddr--, daddr--) { 4336 switch (saddr->type.type) { 4337 case MACH_MSG_PORT_DESCRIPTOR: { 4338 mach_msg_port_descriptor_t *dsc = &saddr->port; 4339 mach_msg_legacy_port_descriptor_t *dest_dsc = &daddr->port; 4340 4341 mach_port_t name = dsc->name; 4342 mach_msg_type_name_t disposition = dsc->disposition; 4343 4344 dest_dsc->name = CAST_MACH_PORT_TO_NAME(name); 4345 dest_dsc->disposition = disposition; 4346 dest_dsc->type = MACH_MSG_PORT_DESCRIPTOR; 4347 break; 4348 } 4349 case MACH_MSG_OOL_VOLATILE_DESCRIPTOR: 4350 case MACH_MSG_OOL_DESCRIPTOR: { 4351 /* The sender should supply ready-made memory, i.e. a vm_map_copy_t 4352 * so we don't need to do anything special. */ 4353 4354 mach_msg_ool_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->out_of_line; 4355 4356 mach_msg_ool_descriptor32_t *dest_dsc = &daddr->out_of_line32; 4357 4358 vm_offset_t address = (vm_offset_t)source_dsc->address; 4359 vm_size_t size = source_dsc->size; 4360 boolean_t deallocate = source_dsc->deallocate; 4361 mach_msg_copy_options_t copy = source_dsc->copy; 4362 mach_msg_descriptor_type_t type = source_dsc->type; 4363 4364 dest_dsc->address = address; 4365 dest_dsc->size = size; 4366 dest_dsc->deallocate = deallocate; 4367 dest_dsc->copy = copy; 4368 dest_dsc->type = type; 4369 break; 4370 } 4371 case MACH_MSG_OOL_PORTS_DESCRIPTOR: { 4372 mach_msg_ool_ports_descriptor_t *source_dsc = (typeof(source_dsc))&saddr->ool_ports; 4373 4374 mach_msg_ool_ports_descriptor32_t *dest_dsc = &daddr->ool_ports32; 4375 4376 vm_offset_t address = (vm_offset_t)source_dsc->address; 4377 vm_size_t port_count = source_dsc->count; 4378 boolean_t deallocate = source_dsc->deallocate; 4379 mach_msg_copy_options_t copy = source_dsc->copy; 4380 mach_msg_descriptor_type_t type = source_dsc->type; 4381 4382 dest_dsc->address = address; 4383 dest_dsc->count = port_count; 4384 dest_dsc->deallocate = deallocate; 4385 dest_dsc->copy = copy; 4386 dest_dsc->type = type; 4387 break; 4388 } 4389 default: { 4390#if MACH_ASSERT 4391 panic("ipc_kmsg_copyin_from_kernel: bad descriptor"); 4392#endif /* MACH_ASSERT */ 4393 } 4394 } 4395 } 4396 4397 if(count) { 4398 dsc_adjust = 4*count; 4399 memmove((char *)((vm_offset_t)kmsg->ikm_header + dsc_adjust), kmsg->ikm_header, sizeof(mach_msg_base_t)); 4400 kmsg->ikm_header = (mach_msg_header_t *)((vm_offset_t)kmsg->ikm_header + dsc_adjust); 4401 /* Update the message size for the smaller user representation */ 4402 kmsg->ikm_header->msgh_size -= dsc_adjust; 4403 } 4404} 4405#endif /* IKM_SUPPORT_LEGACY */ 4406 4407 4408mach_msg_trailer_size_t 4409ipc_kmsg_add_trailer(ipc_kmsg_t kmsg, ipc_space_t space __unused, 4410 mach_msg_option_t option, thread_t thread, 4411 mach_port_seqno_t seqno, boolean_t minimal_trailer, 4412 mach_vm_offset_t context) 4413{ 4414 mach_msg_max_trailer_t *trailer; 4415 4416 (void)thread; 4417 trailer = (mach_msg_max_trailer_t *) 4418 ((vm_offset_t)kmsg->ikm_header + 4419 round_msg(kmsg->ikm_header->msgh_size)); 4420 4421 if (!(option & MACH_RCV_TRAILER_MASK)) { 4422 return trailer->msgh_trailer_size; 4423 } 4424 4425 trailer->msgh_seqno = seqno; 4426 trailer->msgh_context = context; 4427 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(thread_is_64bit(thread), option); 4428 4429 if (minimal_trailer) { 4430 goto done; 4431 } 4432 4433 if (MACH_RCV_TRAILER_ELEMENTS(option) >= 4434 MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_AV)){ 4435 trailer->msgh_ad = 0; 4436 } 4437 4438 /* 4439 * The ipc_kmsg_t holds a reference to the label of a label 4440 * handle, not the port. We must get a reference to the port 4441 * and a send right to copyout to the receiver. 4442 */ 4443 4444 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) { 4445 trailer->msgh_labels.sender = 0; 4446 } 4447 4448done: 4449 4450 return trailer->msgh_trailer_size; 4451} 4452