1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce 58 * support for mandatory and extensible security protections. This notice 59 * is included in support of clause 2.2 (b) of the Apple Public License, 60 * Version 2.0. 61 * Copyright (c) 2005 SPARTA, Inc. 62 */ 63/* 64 */ 65/* 66 * File: ipc/mach_msg.c 67 * Author: Rich Draves 68 * Date: 1989 69 * 70 * Exported message traps. See mach/message.h. 71 */ 72 73#include <mach/mach_types.h> 74#include <mach/kern_return.h> 75#include <mach/port.h> 76#include <mach/message.h> 77#include <mach/mig_errors.h> 78#include <mach/mach_traps.h> 79 80#include <kern/kern_types.h> 81#include <kern/assert.h> 82#include <kern/counters.h> 83#include <kern/cpu_number.h> 84#include <kern/ipc_kobject.h> 85#include <kern/ipc_mig.h> 86#include <kern/task.h> 87#include <kern/thread.h> 88#include <kern/lock.h> 89#include <kern/sched_prim.h> 90#include <kern/exception.h> 91#include <kern/misc_protos.h> 92#include <kern/kalloc.h> 93#include <kern/processor.h> 94#include <kern/syscall_subr.h> 95 96#include <vm/vm_map.h> 97 98#include <ipc/ipc_types.h> 99#include <ipc/ipc_kmsg.h> 100#include <ipc/ipc_mqueue.h> 101#include <ipc/ipc_object.h> 102#include <ipc/ipc_notify.h> 103#include <ipc/ipc_port.h> 104#include <ipc/ipc_pset.h> 105#include <ipc/ipc_space.h> 106#include <ipc/ipc_entry.h> 107 108#include <machine/machine_routines.h> 109#include <security/mac_mach_internal.h> 110 111#include <sys/kdebug.h> 112 113 114#ifndef offsetof 115#define offsetof(type, member) ((size_t)(&((type *)0)->member)) 116#endif /* offsetof */ 117 118/* 119 * Forward declarations - kernel internal routines 120 */ 121 122mach_msg_return_t mach_msg_send( 123 mach_msg_header_t *msg, 124 mach_msg_option_t option, 125 mach_msg_size_t send_size, 126 mach_msg_timeout_t send_timeout, 127 mach_port_name_t notify); 128 129mach_msg_return_t mach_msg_receive( 130 mach_msg_header_t *msg, 131 mach_msg_option_t option, 132 mach_msg_size_t rcv_size, 133 mach_port_name_t rcv_name, 134 mach_msg_timeout_t rcv_timeout, 135 void (*continuation)(mach_msg_return_t), 136 mach_msg_size_t slist_size); 137 138 139mach_msg_return_t mach_msg_receive_results(void); 140 141mach_msg_return_t msg_receive_error( 142 ipc_kmsg_t kmsg, 143 mach_vm_address_t msg_addr, 144 mach_msg_option_t option, 145 mach_port_seqno_t seqno, 146 ipc_space_t space); 147 148security_token_t KERNEL_SECURITY_TOKEN = KERNEL_SECURITY_TOKEN_VALUE; 149audit_token_t KERNEL_AUDIT_TOKEN = KERNEL_AUDIT_TOKEN_VALUE; 150 151mach_msg_format_0_trailer_t trailer_template = { 152 /* mach_msg_trailer_type_t */ MACH_MSG_TRAILER_FORMAT_0, 153 /* mach_msg_trailer_size_t */ MACH_MSG_TRAILER_MINIMUM_SIZE, 154 /* mach_port_seqno_t */ 0, 155 /* security_token_t */ KERNEL_SECURITY_TOKEN_VALUE 156}; 157 158/* 159 * Routine: mach_msg_send 160 * Purpose: 161 * Send a message. 162 * Conditions: 163 * Nothing locked. 164 * Returns: 165 * MACH_MSG_SUCCESS Sent the message. 166 * MACH_SEND_MSG_TOO_SMALL Message smaller than a header. 167 * MACH_SEND_NO_BUFFER Couldn't allocate buffer. 168 * MACH_SEND_INVALID_DATA Couldn't copy message data. 169 * MACH_SEND_INVALID_HEADER 170 * Illegal value in the message header bits. 171 * MACH_SEND_INVALID_DEST The space is dead. 172 * MACH_SEND_INVALID_NOTIFY Bad notify port. 173 * MACH_SEND_INVALID_DEST Can't copyin destination port. 174 * MACH_SEND_INVALID_REPLY Can't copyin reply port. 175 * MACH_SEND_TIMED_OUT Timeout expired without delivery. 176 * MACH_SEND_INTERRUPTED Delivery interrupted. 177 * MACH_SEND_NO_NOTIFY Can't allocate a msg-accepted request. 178 * MACH_SEND_WILL_NOTIFY Msg-accepted notif. requested. 179 * MACH_SEND_NOTIFY_IN_PROGRESS 180 * This space has already forced a message to this port. 181 */ 182 183mach_msg_return_t 184mach_msg_send( 185 mach_msg_header_t *msg, 186 mach_msg_option_t option, 187 mach_msg_size_t send_size, 188 mach_msg_timeout_t send_timeout, 189 mach_port_name_t notify) 190{ 191 ipc_space_t space = current_space(); 192 vm_map_t map = current_map(); 193 ipc_kmsg_t kmsg; 194 mach_msg_return_t mr; 195 mach_msg_size_t msg_and_trailer_size; 196 mach_msg_max_trailer_t *trailer; 197 198 if ((send_size < sizeof(mach_msg_header_t)) || (send_size & 3)) 199 return MACH_SEND_MSG_TOO_SMALL; 200 201 if (send_size > MACH_MSG_SIZE_MAX - MAX_TRAILER_SIZE) 202 return MACH_SEND_TOO_LARGE; 203 204 msg_and_trailer_size = send_size + MAX_TRAILER_SIZE; 205 206 kmsg = ipc_kmsg_alloc(msg_and_trailer_size); 207 208 if (kmsg == IKM_NULL) 209 return MACH_SEND_NO_BUFFER; 210 211 (void) memcpy((void *) kmsg->ikm_header, (const void *) msg, send_size); 212 213 kmsg->ikm_header->msgh_size = send_size; 214 215 /* 216 * reserve for the trailer the largest space (MAX_TRAILER_SIZE) 217 * However, the internal size field of the trailer (msgh_trailer_size) 218 * is initialized to the minimum (sizeof(mach_msg_trailer_t)), to optimize 219 * the cases where no implicit data is requested. 220 */ 221 trailer = (mach_msg_max_trailer_t *) ((vm_offset_t)kmsg->ikm_header + send_size); 222 trailer->msgh_sender = current_thread()->task->sec_token; 223 trailer->msgh_audit = current_thread()->task->audit_token; 224 trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0; 225 trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE; 226 227 if (option & MACH_SEND_CANCEL) { 228 if (notify == MACH_PORT_NULL) 229 mr = MACH_SEND_INVALID_NOTIFY; 230 else 231 mr = ipc_kmsg_copyin(kmsg, space, map, notify); 232 } else 233 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL); 234 if (mr != MACH_MSG_SUCCESS) { 235 ipc_kmsg_free(kmsg); 236 return mr; 237 } 238 239 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, send_timeout); 240 241 if (mr != MACH_MSG_SUCCESS) { 242 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL); 243 (void) memcpy((void *) msg, (const void *) kmsg->ikm_header, 244 kmsg->ikm_header->msgh_size); 245 ipc_kmsg_free(kmsg); 246 } 247 248 return mr; 249} 250 251/* 252 * Routine: mach_msg_receive 253 * Purpose: 254 * Receive a message. 255 * Conditions: 256 * Nothing locked. 257 * Returns: 258 * MACH_MSG_SUCCESS Received a message. 259 * MACH_RCV_INVALID_NAME The name doesn't denote a right, 260 * or the denoted right is not receive or port set. 261 * MACH_RCV_IN_SET Receive right is a member of a set. 262 * MACH_RCV_TOO_LARGE Message wouldn't fit into buffer. 263 * MACH_RCV_TIMED_OUT Timeout expired without a message. 264 * MACH_RCV_INTERRUPTED Reception interrupted. 265 * MACH_RCV_PORT_DIED Port/set died while receiving. 266 * MACH_RCV_PORT_CHANGED Port moved into set while receiving. 267 * MACH_RCV_INVALID_DATA Couldn't copy to user buffer. 268 * MACH_RCV_INVALID_NOTIFY Bad notify port. 269 * MACH_RCV_HEADER_ERROR 270 */ 271 272mach_msg_return_t 273mach_msg_receive_results(void) 274{ 275 thread_t self = current_thread(); 276 ipc_space_t space = current_space(); 277 vm_map_t map = current_map(); 278 279 ipc_object_t object = self->ith_object; 280 mach_msg_return_t mr = self->ith_state; 281 mach_vm_address_t msg_addr = self->ith_msg_addr; 282 mach_msg_option_t option = self->ith_option; 283 ipc_kmsg_t kmsg = self->ith_kmsg; 284 mach_port_seqno_t seqno = self->ith_seqno; 285 286 mach_msg_max_trailer_t *trailer; 287 288 ipc_object_release(object); 289 290 if (mr != MACH_MSG_SUCCESS) { 291 292 if (mr == MACH_RCV_TOO_LARGE ) { 293 if (option & MACH_RCV_LARGE) { 294 /* 295 * We need to inform the user-level code that it needs more 296 * space. The value for how much space was returned in the 297 * msize save area instead of the message (which was left on 298 * the queue). 299 */ 300 if (copyout((char *) &self->ith_msize, 301 msg_addr + offsetof(mach_msg_header_t, msgh_size), 302 sizeof(mach_msg_size_t))) 303 mr = MACH_RCV_INVALID_DATA; 304 goto out; 305 } 306 307 if (msg_receive_error(kmsg, msg_addr, option, seqno, space) 308 == MACH_RCV_INVALID_DATA) 309 mr = MACH_RCV_INVALID_DATA; 310 } 311 goto out; 312 } 313 314 trailer = (mach_msg_max_trailer_t *) 315 ((vm_offset_t)kmsg->ikm_header + 316 round_msg(kmsg->ikm_header->msgh_size)); 317 if (option & MACH_RCV_TRAILER_MASK) { 318 trailer->msgh_seqno = seqno; 319 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option); 320 321 322 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_AV)) { 323#if CONFIG_MACF_MACH 324 if (kmsg->ikm_sender != NULL && 325 IP_VALID(kmsg->ikm_header->msgh_remote_port) && 326 mac_port_check_method(kmsg->ikm_sender, 327 &kmsg->ikm_sender->maclabel, 328 &((ipc_port_t)kmsg->ikm_header->msgh_remote_port)->ip_label, 329 kmsg->ikm_header->msgh_id) == 0) 330 trailer->msgh_ad = 1; 331 else 332#endif 333 trailer->msgh_ad = 0; 334 } 335 336 /* 337 * The ipc_kmsg_t holds a reference to the label of a label 338 * handle, not the port. We must get a reference to the port 339 * and a send right to copyout to the receiver. 340 */ 341 342 if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) { 343#if CONFIG_MACF_MACH 344 if (kmsg->ikm_sender != NULL) { 345 ipc_labelh_t lh = kmsg->ikm_sender->label; 346 kern_return_t kr; 347 348 ip_lock(lh->lh_port); 349 lh->lh_port->ip_mscount++; 350 lh->lh_port->ip_srights++; 351 ip_reference(lh->lh_port); 352 ip_unlock(lh->lh_port); 353 354 kr = ipc_object_copyout(space, (ipc_object_t)lh->lh_port, 355 MACH_MSG_TYPE_PORT_SEND, 0, 356 &trailer->msgh_labels.sender); 357 if (kr != KERN_SUCCESS) { 358 ip_lock(lh->lh_port); 359 ip_release(lh->lh_port); 360 ip_check_unlock(lh->lh_port); 361 362 trailer->msgh_labels.sender = 0; 363 } 364 } else { 365 trailer->msgh_labels.sender = 0; 366 } 367#else 368 trailer->msgh_labels.sender = 0; 369#endif 370 } 371 } 372 373 /* 374 * If MACH_RCV_OVERWRITE was specified, try to get the scatter 375 * list and verify it against the contents of the message. If 376 * there is any problem with it, we will continue without it as 377 * normal. 378 */ 379 if (option & MACH_RCV_OVERWRITE) { 380 mach_msg_size_t slist_size = self->ith_scatter_list_size; 381 mach_msg_body_t *slist; 382 383 slist = ipc_kmsg_get_scatter(msg_addr, slist_size, kmsg); 384 mr = ipc_kmsg_copyout(kmsg, space, map, MACH_PORT_NULL, slist); 385 ipc_kmsg_free_scatter(slist, slist_size); 386 } else { 387 mr = ipc_kmsg_copyout(kmsg, space, map, 388 MACH_PORT_NULL, MACH_MSG_BODY_NULL); 389 } 390 391 if (mr != MACH_MSG_SUCCESS) { 392 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { 393 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size + 394 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA) 395 mr = MACH_RCV_INVALID_DATA; 396 } 397 else { 398 if (msg_receive_error(kmsg, msg_addr, option, seqno, space) 399 == MACH_RCV_INVALID_DATA) 400 mr = MACH_RCV_INVALID_DATA; 401 } 402 goto out; 403 } 404 mr = ipc_kmsg_put(msg_addr, 405 kmsg, 406 kmsg->ikm_header->msgh_size + 407 trailer->msgh_trailer_size); 408 out: 409 return mr; 410} 411 412mach_msg_return_t 413mach_msg_receive( 414 mach_msg_header_t *msg, 415 mach_msg_option_t option, 416 mach_msg_size_t rcv_size, 417 mach_port_name_t rcv_name, 418 mach_msg_timeout_t rcv_timeout, 419 void (*continuation)(mach_msg_return_t), 420 mach_msg_size_t slist_size) 421{ 422 thread_t self = current_thread(); 423 ipc_space_t space = current_space(); 424 ipc_object_t object; 425 ipc_mqueue_t mqueue; 426 mach_msg_return_t mr; 427 428 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object); 429 if (mr != MACH_MSG_SUCCESS) { 430 return mr; 431 } 432 /* hold ref for object */ 433 434 self->ith_msg_addr = CAST_DOWN(mach_vm_address_t, msg); 435 self->ith_object = object; 436 self->ith_msize = rcv_size; 437 self->ith_option = option; 438 self->ith_scatter_list_size = slist_size; 439 self->ith_continuation = continuation; 440 441 ipc_mqueue_receive(mqueue, option, rcv_size, rcv_timeout, THREAD_ABORTSAFE); 442 if ((option & MACH_RCV_TIMEOUT) && rcv_timeout == 0) 443 thread_poll_yield(self); 444 return mach_msg_receive_results(); 445} 446 447void 448mach_msg_receive_continue(void) 449{ 450 thread_t self = current_thread(); 451 452 (*self->ith_continuation)(mach_msg_receive_results()); 453} 454 455/* 456 * Toggle this to compile the hotpath in/out 457 * If compiled in, the run-time toggle "enable_hotpath" below 458 * eases testing & debugging 459 */ 460#define ENABLE_HOTPATH 1 /* Hacked on for now */ 461 462#if ENABLE_HOTPATH 463/* 464 * These counters allow tracing of hotpath behavior under test loads. 465 * A couple key counters are unconditional (see below). 466 */ 467#define HOTPATH_DEBUG 0 /* Toggle to include lots of counters */ 468#if HOTPATH_DEBUG 469#define HOT(expr) expr 470 471unsigned int c_mmot_FIRST = 0; /* Unused First Counter */ 472unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */ 473unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */ 474unsigned int c_mmot_kernel_send = 0; /* kernel server */ 475unsigned int c_mmot_cold_000 = 0; /* see below ... */ 476unsigned int c_mmot_smallsendsize = 0; 477unsigned int c_mmot_oddsendsize = 0; 478unsigned int c_mmot_bigsendsize = 0; 479unsigned int c_mmot_copyinmsg_fail = 0; 480unsigned int c_mmot_g_slow_copyin3 = 0; 481unsigned int c_mmot_cold_006 = 0; 482unsigned int c_mmot_cold_007 = 0; 483unsigned int c_mmot_cold_008 = 0; 484unsigned int c_mmot_cold_009 = 0; 485unsigned int c_mmot_cold_010 = 0; 486unsigned int c_mmot_cold_012 = 0; 487unsigned int c_mmot_cold_013 = 0; 488unsigned int c_mmot_cold_014 = 0; 489unsigned int c_mmot_cold_016 = 0; 490unsigned int c_mmot_cold_018 = 0; 491unsigned int c_mmot_cold_019 = 0; 492unsigned int c_mmot_cold_020 = 0; 493unsigned int c_mmot_cold_021 = 0; 494unsigned int c_mmot_cold_022 = 0; 495unsigned int c_mmot_cold_023 = 0; 496unsigned int c_mmot_cold_024 = 0; 497unsigned int c_mmot_cold_025 = 0; 498unsigned int c_mmot_cold_026 = 0; 499unsigned int c_mmot_cold_027 = 0; 500unsigned int c_mmot_hot_fSR_ok = 0; 501unsigned int c_mmot_cold_029 = 0; 502unsigned int c_mmot_cold_030 = 0; 503unsigned int c_mmot_cold_031 = 0; 504unsigned int c_mmot_cold_032 = 0; 505unsigned int c_mmot_cold_033 = 0; 506unsigned int c_mmot_bad_rcvr = 0; 507unsigned int c_mmot_rcvr_swapped = 0; 508unsigned int c_mmot_rcvr_locked = 0; 509unsigned int c_mmot_rcvr_tswapped = 0; 510unsigned int c_mmot_rcvr_freed = 0; 511unsigned int c_mmot_g_slow_copyout6 = 0; 512unsigned int c_mmot_g_slow_copyout5 = 0; 513unsigned int c_mmot_cold_037 = 0; 514unsigned int c_mmot_cold_038 = 0; 515unsigned int c_mmot_cold_039 = 0; 516unsigned int c_mmot_g_slow_copyout4 = 0; 517unsigned int c_mmot_g_slow_copyout3 = 0; 518unsigned int c_mmot_hot_ok1 = 0; 519unsigned int c_mmot_hot_ok2 = 0; 520unsigned int c_mmot_hot_ok3 = 0; 521unsigned int c_mmot_g_slow_copyout1 = 0; 522unsigned int c_mmot_g_slow_copyout2 = 0; 523unsigned int c_mmot_getback_fast_copyin = 0; 524unsigned int c_mmot_cold_048 = 0; 525unsigned int c_mmot_getback_FastSR = 0; 526unsigned int c_mmot_cold_050 = 0; 527unsigned int c_mmot_cold_051 = 0; 528unsigned int c_mmot_cold_052 = 0; 529unsigned int c_mmot_cold_053 = 0; 530unsigned int c_mmot_fastkernelreply = 0; 531unsigned int c_mmot_cold_055 = 0; 532unsigned int c_mmot_getback_fast_put = 0; 533unsigned int c_mmot_LAST = 0; /* End Marker - Unused */ 534 535void db_mmot_zero_counters(void); /* forward; */ 536void db_mmot_show_counters(void); /* forward; */ 537 538void /* Call from the debugger to clear all counters */ 539db_mmot_zero_counters(void) 540{ 541 register unsigned int *ip = &c_mmot_FIRST; 542 while (ip <= &c_mmot_LAST) 543 *ip++ = 0; 544} 545 546void /* Call from the debugger to show all counters */ 547db_mmot_show_counters(void) 548{ 549#define xx(str) printf("%s: %d\n", # str, str); 550 551 xx(c_mmot_combined_S_R); 552 xx(c_mach_msg_trap_switch_fast); 553 xx(c_mmot_kernel_send); 554 xx(c_mmot_cold_000); 555 xx(c_mmot_smallsendsize); 556 xx(c_mmot_oddsendsize); 557 xx(c_mmot_bigsendsize); 558 xx(c_mmot_copyinmsg_fail); 559 xx(c_mmot_g_slow_copyin3); 560 xx(c_mmot_cold_006); 561 xx(c_mmot_cold_007); 562 xx(c_mmot_cold_008); 563 xx(c_mmot_cold_009); 564 xx(c_mmot_cold_010); 565 xx(c_mmot_cold_012); 566 xx(c_mmot_cold_013); 567 xx(c_mmot_cold_014); 568 xx(c_mmot_cold_016); 569 xx(c_mmot_cold_018); 570 xx(c_mmot_cold_019); 571 xx(c_mmot_cold_020); 572 xx(c_mmot_cold_021); 573 xx(c_mmot_cold_022); 574 xx(c_mmot_cold_023); 575 xx(c_mmot_cold_024); 576 xx(c_mmot_cold_025); 577 xx(c_mmot_cold_026); 578 xx(c_mmot_cold_027); 579 xx(c_mmot_hot_fSR_ok); 580 xx(c_mmot_cold_029); 581 xx(c_mmot_cold_030); 582 xx(c_mmot_cold_031); 583 xx(c_mmot_cold_032); 584 xx(c_mmot_cold_033); 585 xx(c_mmot_bad_rcvr); 586 xx(c_mmot_rcvr_swapped); 587 xx(c_mmot_rcvr_locked); 588 xx(c_mmot_rcvr_tswapped); 589 xx(c_mmot_rcvr_freed); 590 xx(c_mmot_g_slow_copyout6); 591 xx(c_mmot_g_slow_copyout5); 592 xx(c_mmot_cold_037); 593 xx(c_mmot_cold_038); 594 xx(c_mmot_cold_039); 595 xx(c_mmot_g_slow_copyout4); 596 xx(c_mmot_g_slow_copyout3); 597 xx(c_mmot_g_slow_copyout1); 598 xx(c_mmot_hot_ok3); 599 xx(c_mmot_hot_ok2); 600 xx(c_mmot_hot_ok1); 601 xx(c_mmot_g_slow_copyout2); 602 xx(c_mmot_getback_fast_copyin); 603 xx(c_mmot_cold_048); 604 xx(c_mmot_getback_FastSR); 605 xx(c_mmot_cold_050); 606 xx(c_mmot_cold_051); 607 xx(c_mmot_cold_052); 608 xx(c_mmot_cold_053); 609 xx(c_mmot_fastkernelreply); 610 xx(c_mmot_cold_055); 611 xx(c_mmot_getback_fast_put); 612 613#undef xx 614} 615 616#else /* !HOTPATH_DEBUG */ 617 618/* 619 * Duplicate just these few so we can always do a quick sanity check 620 */ 621unsigned int c_mmot_combined_S_R = 0; /* hotpath candidates */ 622unsigned int c_mach_msg_trap_switch_fast = 0; /* hotpath successes */ 623unsigned int c_mmot_kernel_send = 0; /* kernel server calls */ 624#define HOT(expr) /* no optional counters */ 625 626#endif /* !HOTPATH_DEBUG */ 627 628#if CONFIG_MACF_MACH 629boolean_t enable_hotpath = FALSE; /* XXX - push MAC into HOTPATH too */ 630#else 631boolean_t enable_hotpath = TRUE; /* Patchable, just in case ... */ 632#endif 633#endif /* HOTPATH_ENABLE */ 634 635/* 636 * Routine: mach_msg_overwrite_trap [mach trap] 637 * Purpose: 638 * Possibly send a message; possibly receive a message. 639 * Conditions: 640 * Nothing locked. 641 * Returns: 642 * All of mach_msg_send and mach_msg_receive error codes. 643 */ 644 645mach_msg_return_t 646mach_msg_overwrite_trap( 647 struct mach_msg_overwrite_trap_args *args) 648{ 649 mach_vm_address_t msg_addr = args->msg; 650 mach_msg_option_t option = args->option; 651 mach_msg_size_t send_size = args->send_size; 652 mach_msg_size_t rcv_size = args->rcv_size; 653 mach_port_name_t rcv_name = args->rcv_name; 654 mach_msg_timeout_t msg_timeout = args->timeout; 655 mach_port_name_t notify = args->notify; 656 mach_vm_address_t rcv_msg_addr = args->rcv_msg; 657 mach_msg_size_t scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */ 658 __unused mach_port_seqno_t temp_seqno = 0; 659 660 mach_msg_return_t mr = MACH_MSG_SUCCESS; 661#if ENABLE_HOTPATH 662 /* mask out some of the options before entering the hot path */ 663 mach_msg_option_t masked_option = 664 option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE); 665 register mach_msg_header_t *hdr; 666 667 if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) { 668 thread_t self = current_thread(); 669 mach_msg_format_0_trailer_t *trailer; 670 ipc_space_t space = self->task->itk_space; 671 ipc_kmsg_t kmsg; 672 register ipc_port_t dest_port; 673 ipc_object_t rcv_object; 674 ipc_mqueue_t rcv_mqueue; 675 mach_msg_size_t reply_size; 676 677 c_mmot_combined_S_R++; 678 679 /* 680 * This case is divided into ten sections, each 681 * with a label. There are five optimized 682 * sections and six unoptimized sections, which 683 * do the same thing but handle all possible 684 * cases and are slower. 685 * 686 * The five sections for an RPC are 687 * 1) Get request message into a buffer. 688 * 2) Copyin request message and rcv_name. 689 * (fast_copyin or slow_copyin) 690 * 3) Enqueue request and dequeue reply. 691 * (fast_send_receive or 692 * slow_send and slow_receive) 693 * 4) Copyout reply message. 694 * (fast_copyout or slow_copyout) 695 * 5) Put reply message to user's buffer. 696 * 697 * Keep the locking hierarchy firmly in mind. 698 * (First spaces, then ports, then port sets, 699 * then message queues.) Only a non-blocking 700 * attempt can be made to acquire locks out of 701 * order, or acquire two locks on the same level. 702 * Acquiring two locks on the same level will 703 * fail if the objects are really the same, 704 * unless simple locking is disabled. This is OK, 705 * because then the extra unlock does nothing. 706 * 707 * There are two major reasons these RPCs can't use 708 * ipc_thread_switch, and use slow_send/slow_receive: 709 * 1) Kernel RPCs. 710 * 2) Servers fall behind clients, so 711 * client doesn't find a blocked server thread and 712 * server finds waiting messages and can't block. 713 */ 714 715 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg); 716 if (mr != KERN_SUCCESS) { 717 return mr; 718 } 719 hdr = kmsg->ikm_header; 720 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr + 721 send_size); 722 723 /* 724 * fast_copyin: 725 * 726 * optimized ipc_kmsg_copyin/ipc_mqueue_copyin 727 * 728 * We have the request message data in kmsg. 729 * Must still do copyin, send, receive, etc. 730 * 731 * If the message isn't simple, we can't combine 732 * ipc_kmsg_copyin_header and ipc_mqueue_copyin, 733 * because copyin of the message body might 734 * affect rcv_name. 735 */ 736 737 switch (hdr->msgh_bits) { 738 case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 739 MACH_MSG_TYPE_MAKE_SEND_ONCE): { 740 register ipc_entry_t table; 741 register ipc_entry_num_t size; 742 register ipc_port_t reply_port; 743 744 /* sending a request message */ 745 746 { 747 register mach_port_index_t index; 748 register mach_port_gen_t gen; 749 750 { 751 register mach_port_name_t reply_name = 752 (mach_port_name_t)hdr->msgh_local_port; 753 754 if (reply_name != rcv_name) { 755 HOT(c_mmot_g_slow_copyin3++); 756 goto slow_copyin; 757 } 758 759 /* optimized ipc_entry_lookup of reply_name */ 760 761 index = MACH_PORT_INDEX(reply_name); 762 gen = MACH_PORT_GEN(reply_name); 763 764 is_read_lock(space); 765 assert(space->is_active); 766 767 size = space->is_table_size; 768 table = space->is_table; 769 770 { 771 register ipc_entry_t entry; 772 register ipc_entry_bits_t bits; 773 774 if (index < size) { 775 entry = &table[index]; 776 bits = entry->ie_bits; 777 if (IE_BITS_GEN(bits) != gen || 778 (bits & IE_BITS_COLLISION)) { 779 entry = IE_NULL; 780 } 781 } else { 782 entry = IE_NULL; 783 bits = 0; 784 } 785 if (entry == IE_NULL) { 786 entry = ipc_entry_lookup(space, reply_name); 787 if (entry == IE_NULL) { 788 HOT(c_mmot_cold_006++); 789 goto abort_request_copyin; 790 } 791 bits = entry->ie_bits; 792 } 793 794 /* check type bit */ 795 796 if (! (bits & MACH_PORT_TYPE_RECEIVE)) { 797 HOT(c_mmot_cold_007++); 798 goto abort_request_copyin; 799 } 800 801 reply_port = (ipc_port_t) entry->ie_object; 802 assert(reply_port != IP_NULL); 803 } 804 } 805 } 806 807 /* optimized ipc_entry_lookup of dest_name */ 808 809 { 810 register mach_port_index_t index; 811 register mach_port_gen_t gen; 812 813 { 814 register mach_port_name_t dest_name = 815 (mach_port_name_t)hdr->msgh_remote_port; 816 817 index = MACH_PORT_INDEX(dest_name); 818 gen = MACH_PORT_GEN(dest_name); 819 820 { 821 register ipc_entry_t entry; 822 register ipc_entry_bits_t bits; 823 824 if (index < size) { 825 entry = &table[index]; 826 bits = entry->ie_bits; 827 if (IE_BITS_GEN(bits) != gen || 828 (bits & IE_BITS_COLLISION)) { 829 entry = IE_NULL; 830 } 831 } else { 832 entry = IE_NULL; 833 bits = 0; 834 } 835 if (entry == IE_NULL) { 836 entry = ipc_entry_lookup(space, dest_name); 837 if (entry == IE_NULL) { 838 HOT(c_mmot_cold_008++); 839 goto abort_request_copyin; 840 } 841 bits = entry->ie_bits; 842 } 843 844 /* check type bit */ 845 846 if (! (bits & MACH_PORT_TYPE_SEND)) { 847 HOT(c_mmot_cold_009++); 848 goto abort_request_copyin; 849 } 850 851 assert(IE_BITS_UREFS(bits) > 0); 852 853 dest_port = (ipc_port_t) entry->ie_object; 854 assert(dest_port != IP_NULL); 855 } 856 } 857 } 858 859 /* 860 * To do an atomic copyin, need simultaneous 861 * locks on both ports and the space. If 862 * dest_port == reply_port, and simple locking is 863 * enabled, then we will abort. Otherwise it's 864 * OK to unlock twice. 865 */ 866 867 ip_lock(dest_port); 868 if (!ip_active(dest_port) || 869 !ip_lock_try(reply_port)) { 870 ip_unlock(dest_port); 871 HOT(c_mmot_cold_010++); 872 goto abort_request_copyin; 873 } 874 is_read_unlock(space); 875 876 assert(dest_port->ip_srights > 0); 877 dest_port->ip_srights++; 878 ip_reference(dest_port); 879 880 assert(ip_active(reply_port)); 881 assert(reply_port->ip_receiver_name == 882 (mach_port_name_t)hdr->msgh_local_port); 883 assert(reply_port->ip_receiver == space); 884 885 reply_port->ip_sorights++; 886 ip_reference(reply_port); 887 888 hdr->msgh_bits = 889 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 890 MACH_MSG_TYPE_PORT_SEND_ONCE); 891 hdr->msgh_remote_port = dest_port; 892 hdr->msgh_local_port = reply_port; 893 894 /* make sure we can queue to the destination */ 895 896 if (dest_port->ip_receiver == ipc_space_kernel) { 897 /* 898 * The kernel server has a reference to 899 * the reply port, which it hands back 900 * to us in the reply message. We do 901 * not need to keep another reference to 902 * it. 903 */ 904 ip_unlock(reply_port); 905 906 assert(ip_active(dest_port)); 907 dest_port->ip_messages.imq_seqno++; 908 ip_unlock(dest_port); 909 goto kernel_send; 910 } 911 912 if (imq_full(&dest_port->ip_messages)) { 913 HOT(c_mmot_cold_013++); 914 goto abort_request_send_receive; 915 } 916 917 /* optimized ipc_mqueue_copyin */ 918 919 rcv_object = (ipc_object_t) reply_port; 920 io_reference(rcv_object); 921 rcv_mqueue = &reply_port->ip_messages; 922 io_unlock(rcv_object); 923 HOT(c_mmot_hot_fSR_ok++); 924 goto fast_send_receive; 925 926 abort_request_copyin: 927 is_read_unlock(space); 928 goto slow_copyin; 929 930 abort_request_send_receive: 931 ip_unlock(dest_port); 932 ip_unlock(reply_port); 933 goto slow_send; 934 } 935 936 case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): { 937 register ipc_entry_num_t size; 938 register ipc_entry_t table; 939 940 /* sending a reply message */ 941 942 { 943 register mach_port_name_t reply_name = 944 (mach_port_name_t)hdr->msgh_local_port; 945 946 if (reply_name != MACH_PORT_NULL) { 947 HOT(c_mmot_cold_018++); 948 goto slow_copyin; 949 } 950 } 951 952 is_write_lock(space); 953 assert(space->is_active); 954 955 /* optimized ipc_entry_lookup */ 956 957 size = space->is_table_size; 958 table = space->is_table; 959 960 { 961 register ipc_entry_t entry; 962 register mach_port_gen_t gen; 963 register mach_port_index_t index; 964 965 { 966 register mach_port_name_t dest_name = 967 (mach_port_name_t)hdr->msgh_remote_port; 968 969 index = MACH_PORT_INDEX(dest_name); 970 gen = MACH_PORT_GEN(dest_name); 971 } 972 973 if (index >= size) { 974 HOT(c_mmot_cold_019++); 975 goto abort_reply_dest_copyin; 976 } 977 978 entry = &table[index]; 979 980 /* check generation, collision bit, and type bit */ 981 982 if ((entry->ie_bits & (IE_BITS_GEN_MASK| 983 IE_BITS_COLLISION| 984 MACH_PORT_TYPE_SEND_ONCE)) != 985 (gen | MACH_PORT_TYPE_SEND_ONCE)) { 986 HOT(c_mmot_cold_020++); 987 goto abort_reply_dest_copyin; 988 } 989 990 /* optimized ipc_right_copyin */ 991 992 assert(IE_BITS_TYPE(entry->ie_bits) == 993 MACH_PORT_TYPE_SEND_ONCE); 994 assert(IE_BITS_UREFS(entry->ie_bits) == 1); 995 996 if (entry->ie_request != 0) { 997 HOT(c_mmot_cold_021++); 998 goto abort_reply_dest_copyin; 999 } 1000 1001 dest_port = (ipc_port_t) entry->ie_object; 1002 assert(dest_port != IP_NULL); 1003 1004 ip_lock(dest_port); 1005 if (!ip_active(dest_port)) { 1006 ip_unlock(dest_port); 1007 HOT(c_mmot_cold_022++); 1008 goto abort_reply_dest_copyin; 1009 } 1010 1011 assert(dest_port->ip_sorights > 0); 1012 1013 /* optimized ipc_entry_dealloc */ 1014 1015 1016 entry->ie_bits = gen; 1017 entry->ie_next = table->ie_next; 1018 table->ie_next = index; 1019 entry->ie_object = IO_NULL; 1020 } 1021 1022 hdr->msgh_bits = 1023 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 1024 0); 1025 hdr->msgh_remote_port = dest_port; 1026 1027 /* make sure we can queue to the destination */ 1028 1029 assert(dest_port->ip_receiver != ipc_space_kernel); 1030 1031 /* optimized ipc_entry_lookup/ipc_mqueue_copyin */ 1032 1033 { 1034 register ipc_entry_t entry; 1035 register ipc_entry_bits_t bits; 1036 1037 { 1038 register mach_port_index_t index; 1039 register mach_port_gen_t gen; 1040 1041 index = MACH_PORT_INDEX(rcv_name); 1042 gen = MACH_PORT_GEN(rcv_name); 1043 1044 if (index < size) { 1045 entry = &table[index]; 1046 bits = entry->ie_bits; 1047 if (IE_BITS_GEN(bits) != gen || 1048 (bits & IE_BITS_COLLISION)) { 1049 entry = IE_NULL; 1050 } 1051 } else { 1052 entry = IE_NULL; 1053 bits = 0; 1054 } 1055 if (entry == IE_NULL) { 1056 entry = ipc_entry_lookup(space, rcv_name); 1057 if (entry == IE_NULL) { 1058 HOT(c_mmot_cold_024++); 1059 goto abort_reply_rcv_copyin; 1060 } 1061 bits = entry->ie_bits; 1062 } 1063 1064 } 1065 1066 /* check type bits; looking for receive or set */ 1067#if 0 1068 /* 1069 * JMM - The check below for messages in the receive 1070 * mqueue is insufficient to work with port sets, since 1071 * the messages stay in the port queues. For now, don't 1072 * allow portsets (but receiving on portsets when sending 1073 * a message to a send-once right is actually a very 1074 * common case (so we should re-enable). 1075 */ 1076 if (bits & MACH_PORT_TYPE_PORT_SET) { 1077 register ipc_pset_t rcv_pset; 1078 1079 rcv_pset = (ipc_pset_t) entry->ie_object; 1080 assert(rcv_pset != IPS_NULL); 1081 1082 ips_lock(rcv_pset); 1083 assert(ips_active(rcv_pset)); 1084 1085 rcv_object = (ipc_object_t) rcv_pset; 1086 rcv_mqueue = &rcv_pset->ips_messages; 1087 } else 1088#endif /* 0 */ 1089 if (bits & MACH_PORT_TYPE_RECEIVE) { 1090 register ipc_port_t rcv_port; 1091 1092 rcv_port = (ipc_port_t) entry->ie_object; 1093 assert(rcv_port != IP_NULL); 1094 1095 if (!ip_lock_try(rcv_port)) { 1096 HOT(c_mmot_cold_025++); 1097 goto abort_reply_rcv_copyin; 1098 } 1099 assert(ip_active(rcv_port)); 1100 1101 if (rcv_port->ip_pset_count != 0) { 1102 ip_unlock(rcv_port); 1103 HOT(c_mmot_cold_026++); 1104 goto abort_reply_rcv_copyin; 1105 } 1106 1107 rcv_object = (ipc_object_t) rcv_port; 1108 rcv_mqueue = &rcv_port->ip_messages; 1109 } else { 1110 HOT(c_mmot_cold_027++); 1111 goto abort_reply_rcv_copyin; 1112 } 1113 } 1114 1115 is_write_unlock(space); 1116 io_reference(rcv_object); 1117 io_unlock(rcv_object); 1118 HOT(c_mmot_hot_fSR_ok++); 1119 goto fast_send_receive; 1120 1121 abort_reply_dest_copyin: 1122 is_write_unlock(space); 1123 HOT(c_mmot_cold_029++); 1124 goto slow_copyin; 1125 1126 abort_reply_rcv_copyin: 1127 ip_unlock(dest_port); 1128 is_write_unlock(space); 1129 HOT(c_mmot_cold_030++); 1130 goto slow_send; 1131 } 1132 1133 default: 1134 HOT(c_mmot_cold_031++); 1135 goto slow_copyin; 1136 } 1137 /*NOTREACHED*/ 1138 1139 fast_send_receive: 1140 /* 1141 * optimized ipc_mqueue_send/ipc_mqueue_receive 1142 * 1143 * Finished get/copyin of kmsg and copyin of rcv_name. 1144 * space is unlocked, dest_port is locked, 1145 * we can queue kmsg to dest_port, 1146 * rcv_mqueue is set, and rcv_object holds a ref 1147 * so the mqueue cannot go away. 1148 * 1149 * JMM - For now, rcv_object is just a port. Portsets 1150 * are disabled for the time being. 1151 */ 1152 1153 assert(ip_active(dest_port)); 1154 assert(dest_port->ip_receiver != ipc_space_kernel); 1155// assert(!imq_full(&dest_port->ip_messages) || 1156// (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) == 1157// MACH_MSG_TYPE_PORT_SEND_ONCE)); 1158 assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0); 1159 1160 { 1161 register ipc_mqueue_t dest_mqueue; 1162 wait_queue_t waitq; 1163 thread_t receiver; 1164 spl_t s; 1165 1166 s = splsched(); 1167 dest_mqueue = &dest_port->ip_messages; 1168 waitq = &dest_mqueue->imq_wait_queue; 1169 imq_lock(dest_mqueue); 1170 1171 get_next_receiver: 1172 receiver = wait_queue_wakeup64_identity_locked(waitq, 1173 IPC_MQUEUE_RECEIVE, 1174 THREAD_AWAKENED, 1175 FALSE); 1176 /* queue still locked, receiver thread locked (if any) */ 1177 1178 if ( receiver == THREAD_NULL ) { 1179 imq_unlock(dest_mqueue); 1180 splx(s); 1181 1182 ip_unlock(dest_port); 1183 ipc_object_release(rcv_object); 1184 HOT(c_mmot_cold_032++); 1185 goto slow_send; 1186 } 1187 1188 /* 1189 * Check that the receiver can handle the size of the message. 1190 * If not, and the receiver just wants to be informed of that 1191 * fact, set it running and try to find another thread. 1192 * 1193 * If he didn't want the "too large" message left on the queue, 1194 * give it to him anyway, he'll consume it as part of his receive 1195 * processing. 1196 */ 1197 if (receiver->ith_msize < 1198 ipc_kmsg_copyout_size(kmsg, receiver->map) + 1199 REQUESTED_TRAILER_SIZE(receiver->ith_option)) 1200 { 1201 receiver->ith_msize = kmsg->ikm_header->msgh_size; 1202 receiver->ith_state = MACH_RCV_TOO_LARGE; 1203 1204 if ((receiver->ith_option & MACH_RCV_LARGE) != 0) { 1205 receiver->ith_kmsg = IKM_NULL; 1206 receiver->ith_seqno = 0; 1207 thread_unlock(receiver); 1208 HOT(c_mmot_bad_rcvr++); 1209 goto get_next_receiver; 1210 } 1211 } else { 1212 receiver->ith_state = MACH_MSG_SUCCESS; 1213 } 1214 1215 /* At this point we are committed to do the message handoff. */ 1216 c_mach_msg_trap_switch_fast++; 1217 1218 /* 1219 * Store the kmsg and seqno where the receiver can pick it up. 1220 * and set it running. 1221 */ 1222 receiver->ith_kmsg = kmsg; 1223 receiver->ith_seqno = dest_mqueue->imq_seqno++; 1224 thread_unlock(receiver); 1225 1226 imq_unlock(dest_mqueue); 1227 ip_unlock(dest_port); 1228 current_task()->messages_sent++; 1229 1230 /* 1231 * Now prepare to wait on our receive queue. But we have to make 1232 * sure the queue doesn't already have messages. If it does, we'll 1233 * have to do a slow receive. 1234 * 1235 * JMM - Need to make this check appropriate for portsets as 1236 * well before re-enabling them. 1237 */ 1238 imq_lock(rcv_mqueue); 1239 1240 if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) { 1241 imq_unlock(rcv_mqueue); 1242 splx(s); 1243 HOT(c_mmot_cold_033++); 1244 goto slow_receive; 1245 } 1246 1247 /* 1248 * Put self on receive port's queue. 1249 * Also save state that the sender of 1250 * our reply message needs to determine if it 1251 * can hand off directly back to us. 1252 */ 1253 thread_lock(self); 1254 self->ith_msg_addr = (rcv_msg_addr) ? rcv_msg_addr : msg_addr; 1255 self->ith_object = rcv_object; /* still holds reference */ 1256 self->ith_msize = rcv_size; 1257 self->ith_option = option; 1258 self->ith_scatter_list_size = scatter_list_size; 1259 self->ith_continuation = thread_syscall_return; 1260 1261 waitq = &rcv_mqueue->imq_wait_queue; 1262 (void)wait_queue_assert_wait64_locked(waitq, 1263 IPC_MQUEUE_RECEIVE, 1264 THREAD_ABORTSAFE, 0, 1265 self); 1266 thread_unlock(self); 1267 imq_unlock(rcv_mqueue); 1268 splx(s); 1269 thread_block(ipc_mqueue_receive_continue); 1270 /* NOTREACHED */ 1271 } 1272 1273 fast_copyout: 1274 /* 1275 * Nothing locked and no references held, except 1276 * we have kmsg with msgh_seqno filled in. Must 1277 * still check against rcv_size and do 1278 * ipc_kmsg_copyout/ipc_kmsg_put. 1279 */ 1280 1281 reply_size = send_size + trailer->msgh_trailer_size; 1282 if (rcv_size < reply_size) { 1283 HOT(c_mmot_g_slow_copyout6++); 1284 goto slow_copyout; 1285 } 1286 1287 /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */ 1288 1289 switch (hdr->msgh_bits) { 1290 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND, 1291 MACH_MSG_TYPE_PORT_SEND_ONCE): { 1292 ipc_port_t reply_port = 1293 (ipc_port_t) hdr->msgh_local_port; 1294 mach_port_name_t dest_name, reply_name; 1295 1296 /* receiving a request message */ 1297 1298 if (!IP_VALID(reply_port)) { 1299 HOT(c_mmot_g_slow_copyout5++); 1300 goto slow_copyout; 1301 } 1302 1303 is_write_lock(space); 1304 assert(space->is_active); 1305 1306 /* 1307 * To do an atomic copyout, need simultaneous 1308 * locks on both ports and the space. If 1309 * dest_port == reply_port, and simple locking is 1310 * enabled, then we will abort. Otherwise it's 1311 * OK to unlock twice. 1312 */ 1313 1314 ip_lock(dest_port); 1315 if (!ip_active(dest_port) || 1316 !ip_lock_try(reply_port)) { 1317 HOT(c_mmot_cold_037++); 1318 goto abort_request_copyout; 1319 } 1320 1321 if (!ip_active(reply_port)) { 1322 ip_unlock(reply_port); 1323 HOT(c_mmot_cold_038++); 1324 goto abort_request_copyout; 1325 } 1326 1327 assert(reply_port->ip_sorights > 0); 1328 ip_unlock(reply_port); 1329 1330 { 1331 register ipc_entry_t table; 1332 register ipc_entry_t entry; 1333 register mach_port_index_t index; 1334 1335 /* optimized ipc_entry_get */ 1336 1337 table = space->is_table; 1338 index = table->ie_next; 1339 1340 if (index == 0) { 1341 HOT(c_mmot_cold_039++); 1342 goto abort_request_copyout; 1343 } 1344 1345 entry = &table[index]; 1346 table->ie_next = entry->ie_next; 1347 entry->ie_request = 0; 1348 1349 { 1350 register mach_port_gen_t gen; 1351 1352 assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0); 1353 gen = IE_BITS_NEW_GEN(entry->ie_bits); 1354 1355 reply_name = MACH_PORT_MAKE(index, gen); 1356 1357 /* optimized ipc_right_copyout */ 1358 1359 entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1); 1360 } 1361 1362 assert(MACH_PORT_VALID(reply_name)); 1363 entry->ie_object = (ipc_object_t) reply_port; 1364 is_write_unlock(space); 1365 } 1366 1367 /* optimized ipc_object_copyout_dest */ 1368 1369 assert(dest_port->ip_srights > 0); 1370 ip_release(dest_port); 1371 1372 if (dest_port->ip_receiver == space) 1373 dest_name = dest_port->ip_receiver_name; 1374 else 1375 dest_name = MACH_PORT_NULL; 1376 1377 if ((--dest_port->ip_srights == 0) && 1378 (dest_port->ip_nsrequest != IP_NULL)) { 1379 ipc_port_t nsrequest; 1380 mach_port_mscount_t mscount; 1381 1382 /* a rather rare case */ 1383 1384 nsrequest = dest_port->ip_nsrequest; 1385 mscount = dest_port->ip_mscount; 1386 dest_port->ip_nsrequest = IP_NULL; 1387 ip_unlock(dest_port); 1388 ipc_notify_no_senders(nsrequest, mscount); 1389 } else 1390 ip_unlock(dest_port); 1391 1392 hdr->msgh_bits = 1393 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 1394 MACH_MSG_TYPE_PORT_SEND); 1395 hdr->msgh_remote_port = (mach_port_t)reply_name; 1396 hdr->msgh_local_port = (mach_port_t)dest_name; 1397 HOT(c_mmot_hot_ok1++); 1398 goto fast_put; 1399 1400 abort_request_copyout: 1401 ip_unlock(dest_port); 1402 is_write_unlock(space); 1403 HOT(c_mmot_g_slow_copyout4++); 1404 goto slow_copyout; 1405 } 1406 1407 case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): { 1408 register mach_port_name_t dest_name; 1409 1410 /* receiving a reply message */ 1411 1412 ip_lock(dest_port); 1413 if (!ip_active(dest_port)) { 1414 ip_unlock(dest_port); 1415 HOT(c_mmot_g_slow_copyout3++); 1416 goto slow_copyout; 1417 } 1418 1419 /* optimized ipc_object_copyout_dest */ 1420 1421 assert(dest_port->ip_sorights > 0); 1422 1423 if (dest_port->ip_receiver == space) { 1424 ip_release(dest_port); 1425 dest_port->ip_sorights--; 1426 dest_name = dest_port->ip_receiver_name; 1427 ip_unlock(dest_port); 1428 } else { 1429 ip_unlock(dest_port); 1430 1431 ipc_notify_send_once(dest_port); 1432 dest_name = MACH_PORT_NULL; 1433 } 1434 1435 hdr->msgh_bits = MACH_MSGH_BITS(0, 1436 MACH_MSG_TYPE_PORT_SEND_ONCE); 1437 hdr->msgh_remote_port = MACH_PORT_NULL; 1438 hdr->msgh_local_port = (ipc_port_t)dest_name; 1439 HOT(c_mmot_hot_ok2++); 1440 goto fast_put; 1441 } 1442 1443 case MACH_MSGH_BITS_COMPLEX| 1444 MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): { 1445 register mach_port_name_t dest_name; 1446 1447 /* receiving a complex reply message */ 1448 1449 ip_lock(dest_port); 1450 if (!ip_active(dest_port)) { 1451 ip_unlock(dest_port); 1452 HOT(c_mmot_g_slow_copyout1++); 1453 goto slow_copyout; 1454 } 1455 1456 /* optimized ipc_object_copyout_dest */ 1457 1458 assert(dest_port->ip_sorights > 0); 1459 1460 if (dest_port->ip_receiver == space) { 1461 ip_release(dest_port); 1462 dest_port->ip_sorights--; 1463 dest_name = dest_port->ip_receiver_name; 1464 ip_unlock(dest_port); 1465 } else { 1466 ip_unlock(dest_port); 1467 1468 ipc_notify_send_once(dest_port); 1469 dest_name = MACH_PORT_NULL; 1470 } 1471 1472 hdr->msgh_bits = 1473 MACH_MSGH_BITS_COMPLEX | 1474 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE); 1475 hdr->msgh_remote_port = MACH_PORT_NULL; 1476 hdr->msgh_local_port = (mach_port_t)dest_name; 1477 1478 mr = ipc_kmsg_copyout_body(kmsg, space, 1479 current_map(), 1480 MACH_MSG_BODY_NULL); 1481 /* hdr and send_size may be invalid now - done use */ 1482 if (mr != MACH_MSG_SUCCESS) { 1483 if (ipc_kmsg_put(msg_addr, kmsg, 1484 kmsg->ikm_header->msgh_size + 1485 trailer->msgh_trailer_size) == 1486 MACH_RCV_INVALID_DATA) 1487 return MACH_RCV_INVALID_DATA; 1488 else 1489 return mr | MACH_RCV_BODY_ERROR; 1490 } 1491 HOT(c_mmot_hot_ok3++); 1492 goto fast_put; 1493 } 1494 1495 default: 1496 HOT(c_mmot_g_slow_copyout2++); 1497 goto slow_copyout; 1498 } 1499 /*NOTREACHED*/ 1500 1501 fast_put: 1502 mr = ipc_kmsg_put(rcv_msg_addr ? rcv_msg_addr : msg_addr, 1503 kmsg, 1504 kmsg->ikm_header->msgh_size + 1505 trailer->msgh_trailer_size); 1506 if (mr != MACH_MSG_SUCCESS) { 1507 return MACH_RCV_INVALID_DATA; 1508 } 1509 current_task()->messages_received++; 1510 return mr; 1511 1512 1513 /* BEGINNING OF WARM PATH */ 1514 1515 /* 1516 * The slow path has a few non-register temporary 1517 * variables used only for call-by-reference. 1518 */ 1519 1520 slow_copyin: 1521 { 1522 register mach_port_name_t reply_name = 1523 (mach_port_name_t)hdr->msgh_local_port; 1524 1525 1526 /* 1527 * We have the message data in kmsg, but 1528 * we still need to copyin, send it, 1529 * receive a reply, and do copyout. 1530 */ 1531 1532 mr = ipc_kmsg_copyin(kmsg, space, current_map(), 1533 MACH_PORT_NULL); 1534 if (mr != MACH_MSG_SUCCESS) { 1535 ipc_kmsg_free(kmsg); 1536 return(mr); 1537 } 1538 1539 /* 1540 * LP64support - We have to recompute the header pointer 1541 * and send_size - as they could have changed during the 1542 * complex copyin. 1543 */ 1544 hdr = kmsg->ikm_header; 1545 send_size = hdr->msgh_size; 1546 1547 /* try to get back on optimized path */ 1548 if ((reply_name != rcv_name) || 1549 (hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR)) { 1550 HOT(c_mmot_cold_048++); 1551 goto slow_send; 1552 } 1553 1554 dest_port = (ipc_port_t) hdr->msgh_remote_port; 1555 assert(IP_VALID(dest_port)); 1556 1557 ip_lock(dest_port); 1558 if (!ip_active(dest_port)) { 1559 ip_unlock(dest_port); 1560 goto slow_send; 1561 } 1562 1563 if (dest_port->ip_receiver == ipc_space_kernel) { 1564 dest_port->ip_messages.imq_seqno++; 1565 ip_unlock(dest_port); 1566 goto kernel_send; 1567 } 1568 1569 if (!imq_full(&dest_port->ip_messages) || 1570 (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) == 1571 MACH_MSG_TYPE_PORT_SEND_ONCE)) 1572 { 1573 /* 1574 * Try an optimized ipc_mqueue_copyin. 1575 * It will work if this is a request message. 1576 */ 1577 1578 register ipc_port_t reply_port; 1579 1580 reply_port = (ipc_port_t) hdr->msgh_local_port; 1581 if (IP_VALID(reply_port)) { 1582 if (ip_lock_try(reply_port)) { 1583 if (ip_active(reply_port) && 1584 reply_port->ip_receiver == space && 1585 reply_port->ip_receiver_name == rcv_name && 1586 reply_port->ip_pset_count == 0) 1587 { 1588 /* Grab a reference to the reply port. */ 1589 rcv_object = (ipc_object_t) reply_port; 1590 io_reference(rcv_object); 1591 rcv_mqueue = &reply_port->ip_messages; 1592 io_unlock(rcv_object); 1593 HOT(c_mmot_getback_FastSR++); 1594 goto fast_send_receive; 1595 } 1596 ip_unlock(reply_port); 1597 } 1598 } 1599 } 1600 1601 ip_unlock(dest_port); 1602 HOT(c_mmot_cold_050++); 1603 goto slow_send; 1604 1605 kernel_send: 1606 /* 1607 * Special case: send message to kernel services. 1608 * The request message has been copied into the 1609 * kmsg. Nothing is locked. 1610 */ 1611 1612 { 1613 register ipc_port_t reply_port; 1614 spl_t s; 1615 1616 /* 1617 * Perform the kernel function. 1618 */ 1619 c_mmot_kernel_send++; 1620 1621 current_task()->messages_sent++; 1622 1623 kmsg = ipc_kobject_server(kmsg); 1624 if (kmsg == IKM_NULL) { 1625 /* 1626 * No reply. Take the 1627 * slow receive path. 1628 */ 1629 HOT(c_mmot_cold_051++); 1630 goto slow_get_rcv_port; 1631 } 1632 1633 /* 1634 * Check that: 1635 * the reply port is alive 1636 * we hold the receive right 1637 * the name has not changed. 1638 * the port is not in a set 1639 * If any of these are not true, 1640 * we cannot directly receive the reply 1641 * message. 1642 */ 1643 hdr = kmsg->ikm_header; 1644 send_size = hdr->msgh_size; 1645 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr + 1646 round_msg(send_size)); 1647 reply_port = (ipc_port_t) hdr->msgh_remote_port; 1648 ip_lock(reply_port); 1649 1650 if ((!ip_active(reply_port)) || 1651 (reply_port->ip_receiver != space) || 1652 (reply_port->ip_receiver_name != rcv_name) || 1653 (reply_port->ip_pset_count != 0)) 1654 { 1655 /* try to enqueue by sending with an immediate timeout */ 1656 ip_unlock(reply_port); 1657 mr = ipc_kmsg_send(kmsg, MACH_SEND_TIMEOUT, 0); 1658 if (mr != MACH_MSG_SUCCESS) { 1659 ipc_kmsg_destroy(kmsg); 1660 } 1661 HOT(c_mmot_cold_052++); 1662 goto slow_get_rcv_port; 1663 } 1664 1665 s = splsched(); 1666 rcv_mqueue = &reply_port->ip_messages; 1667 imq_lock(rcv_mqueue); 1668 1669 /* keep port locked, and don`t change ref count yet */ 1670 1671 /* 1672 * If there are messages on the port 1673 * or other threads waiting for a message, 1674 * we cannot directly receive the reply. 1675 * Try to enqueue it by sending with an 1676 * immediate timeout. 1677 */ 1678 if (!wait_queue_empty(&rcv_mqueue->imq_wait_queue) || 1679 (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL)) 1680 { 1681 imq_unlock(rcv_mqueue); 1682 splx(s); 1683 ip_unlock(reply_port); 1684 mr = ipc_kmsg_send(kmsg, MACH_SEND_TIMEOUT, 0); 1685 if (mr != MACH_MSG_SUCCESS) { 1686 ipc_kmsg_destroy(kmsg); 1687 } 1688 HOT(c_mmot_cold_053++); 1689 goto slow_get_rcv_port; 1690 } 1691 1692 /* 1693 * We can directly receive this reply. 1694 * Since there were no messages queued 1695 * on the reply port, there should be 1696 * no threads blocked waiting to send. 1697 */ 1698 dest_port = reply_port; 1699 temp_seqno = rcv_mqueue->imq_seqno++; 1700 imq_unlock(rcv_mqueue); 1701 splx(s); 1702 1703 /* 1704 * inline ipc_object_release. 1705 * Port is still locked. 1706 * Reference count was not incremented. 1707 */ 1708 ip_check_unlock(reply_port); 1709 1710 if (option & MACH_RCV_TRAILER_MASK) { 1711 trailer->msgh_seqno = temp_seqno; 1712 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option); 1713 } 1714 /* copy out the kernel reply */ 1715 HOT(c_mmot_fastkernelreply++); 1716 goto fast_copyout; 1717 } 1718 1719 slow_send: 1720 /* 1721 * Nothing is locked. We have acquired kmsg, but 1722 * we still need to send it and receive a reply. 1723 */ 1724 1725 mr = ipc_kmsg_send(kmsg, MACH_MSG_OPTION_NONE, 1726 MACH_MSG_TIMEOUT_NONE); 1727 if (mr != MACH_MSG_SUCCESS) { 1728 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, 1729 current_map(), 1730 MACH_MSG_BODY_NULL); 1731 1732 (void) ipc_kmsg_put(msg_addr, kmsg, 1733 kmsg->ikm_header->msgh_size); 1734 return(mr); 1735 } 1736 1737 slow_get_rcv_port: 1738 /* 1739 * We have sent the message. Copy in the receive port. 1740 */ 1741 mr = ipc_mqueue_copyin(space, rcv_name, 1742 &rcv_mqueue, &rcv_object); 1743 if (mr != MACH_MSG_SUCCESS) { 1744 return(mr); 1745 } 1746 /* hold ref for rcv_object */ 1747 1748 /* 1749 * 1750 * Now we have sent the request and copied in rcv_name, 1751 * and hold ref for rcv_object (to keep mqueue alive). 1752 * Just receive a reply and try to get back to fast path. 1753 */ 1754 1755 slow_receive: 1756 self->ith_continuation = (void (*)(mach_msg_return_t))0; 1757 ipc_mqueue_receive(rcv_mqueue, 1758 MACH_MSG_OPTION_NONE, 1759 MACH_MSG_SIZE_MAX, 1760 MACH_MSG_TIMEOUT_NONE, 1761 THREAD_ABORTSAFE); 1762 1763 mr = self->ith_state; 1764 temp_seqno = self->ith_seqno; 1765 1766 ipc_object_release(rcv_object); 1767 1768 if (mr != MACH_MSG_SUCCESS) { 1769 return(mr); 1770 } 1771 1772 kmsg = self->ith_kmsg; 1773 hdr = kmsg->ikm_header; 1774 send_size = hdr->msgh_size; 1775 trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr + 1776 round_msg(send_size)); 1777 if (option & MACH_RCV_TRAILER_MASK) { 1778 trailer->msgh_seqno = temp_seqno; 1779 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option); 1780 } 1781 dest_port = (ipc_port_t) hdr->msgh_remote_port; 1782 HOT(c_mmot_cold_055++); 1783 goto fast_copyout; 1784 1785 slow_copyout: 1786 /* 1787 * Nothing locked and no references held, except 1788 * we have kmsg with msgh_seqno filled in. Must 1789 * still check against rcv_size and do 1790 * ipc_kmsg_copyout/ipc_kmsg_put. 1791 */ 1792 1793 /* LP64support - have to compute real size as it would be received */ 1794 reply_size = ipc_kmsg_copyout_size(kmsg, current_map()) + 1795 REQUESTED_TRAILER_SIZE(option); 1796 temp_seqno = trailer->msgh_seqno; 1797 if (rcv_size < reply_size) { 1798 if (msg_receive_error(kmsg, msg_addr, option, temp_seqno, 1799 space) == MACH_RCV_INVALID_DATA) { 1800 mr = MACH_RCV_INVALID_DATA; 1801 return(mr); 1802 } 1803 else { 1804 mr = MACH_RCV_TOO_LARGE; 1805 return(mr); 1806 } 1807 } 1808 1809 mr = ipc_kmsg_copyout(kmsg, space, current_map(), 1810 MACH_PORT_NULL, MACH_MSG_BODY_NULL); 1811 if (mr != MACH_MSG_SUCCESS) { 1812 if ((mr &~ MACH_MSG_MASK) == MACH_RCV_BODY_ERROR) { 1813 if (ipc_kmsg_put(msg_addr, kmsg, reply_size) == 1814 MACH_RCV_INVALID_DATA) 1815 mr = MACH_RCV_INVALID_DATA; 1816 } 1817 else { 1818 if (msg_receive_error(kmsg, msg_addr, option, 1819 temp_seqno, space) == MACH_RCV_INVALID_DATA) 1820 mr = MACH_RCV_INVALID_DATA; 1821 } 1822 1823 return(mr); 1824 } 1825 1826 /* try to get back on optimized path */ 1827 HOT(c_mmot_getback_fast_put++); 1828 goto fast_put; 1829 1830 /*NOTREACHED*/ 1831 } 1832 } /* END OF HOT PATH */ 1833#endif /* ENABLE_HOTPATH */ 1834 1835 if (option & MACH_SEND_MSG) { 1836 ipc_space_t space = current_space(); 1837 vm_map_t map = current_map(); 1838 ipc_kmsg_t kmsg; 1839 1840 mr = ipc_kmsg_get(msg_addr, send_size, &kmsg); 1841 1842 if (mr != MACH_MSG_SUCCESS) 1843 return mr; 1844 1845 if (option & MACH_SEND_CANCEL) { 1846 if (notify == MACH_PORT_NULL) 1847 mr = MACH_SEND_INVALID_NOTIFY; 1848 else 1849 mr = ipc_kmsg_copyin(kmsg, space, map, notify); 1850 } else 1851 mr = ipc_kmsg_copyin(kmsg, space, map, MACH_PORT_NULL); 1852 if (mr != MACH_MSG_SUCCESS) { 1853 ipc_kmsg_free(kmsg); 1854 return mr; 1855 } 1856 1857 mr = ipc_kmsg_send(kmsg, option & MACH_SEND_TIMEOUT, msg_timeout); 1858 1859 if (mr != MACH_MSG_SUCCESS) { 1860 mr |= ipc_kmsg_copyout_pseudo(kmsg, space, map, MACH_MSG_BODY_NULL); 1861 (void) ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size); 1862 return mr; 1863 } 1864 1865 } 1866 1867 if (option & MACH_RCV_MSG) { 1868 thread_t self = current_thread(); 1869 ipc_space_t space = current_space(); 1870 ipc_object_t object; 1871 ipc_mqueue_t mqueue; 1872 1873 mr = ipc_mqueue_copyin(space, rcv_name, &mqueue, &object); 1874 if (mr != MACH_MSG_SUCCESS) { 1875 return mr; 1876 } 1877 /* hold ref for object */ 1878 1879 /* 1880 * 1. MACH_RCV_OVERWRITE is on, and rcv_msg is our scatter list 1881 * and receive buffer 1882 * 2. MACH_RCV_OVERWRITE is off, and rcv_msg might be the 1883 * alternate receive buffer (separate send and receive buffers). 1884 */ 1885 if (option & MACH_RCV_OVERWRITE) 1886 self->ith_msg_addr = rcv_msg_addr; 1887 else if (rcv_msg_addr != (mach_vm_address_t)0) 1888 self->ith_msg_addr = rcv_msg_addr; 1889 else 1890 self->ith_msg_addr = msg_addr; 1891 self->ith_object = object; 1892 self->ith_msize = rcv_size; 1893 self->ith_option = option; 1894 self->ith_scatter_list_size = scatter_list_size; 1895 self->ith_continuation = thread_syscall_return; 1896 1897 ipc_mqueue_receive(mqueue, option, rcv_size, msg_timeout, THREAD_ABORTSAFE); 1898 if ((option & MACH_RCV_TIMEOUT) && msg_timeout == 0) 1899 thread_poll_yield(self); 1900 return mach_msg_receive_results(); 1901 } 1902 1903 return MACH_MSG_SUCCESS; 1904} 1905 1906/* 1907 * Routine: mach_msg_trap [mach trap] 1908 * Purpose: 1909 * Possibly send a message; possibly receive a message. 1910 * Conditions: 1911 * Nothing locked. 1912 * Returns: 1913 * All of mach_msg_send and mach_msg_receive error codes. 1914 */ 1915 1916mach_msg_return_t 1917mach_msg_trap( 1918 struct mach_msg_overwrite_trap_args *args) 1919{ 1920 kern_return_t kr; 1921 args->rcv_msg = (mach_vm_address_t)0; 1922 1923 kr = mach_msg_overwrite_trap(args); 1924 return kr; 1925} 1926 1927 1928/* 1929 * Routine: msg_receive_error [internal] 1930 * Purpose: 1931 * Builds a minimal header/trailer and copies it to 1932 * the user message buffer. Invoked when in the case of a 1933 * MACH_RCV_TOO_LARGE or MACH_RCV_BODY_ERROR error. 1934 * Conditions: 1935 * Nothing locked. 1936 * Returns: 1937 * MACH_MSG_SUCCESS minimal header/trailer copied 1938 * MACH_RCV_INVALID_DATA copyout to user buffer failed 1939 */ 1940 1941mach_msg_return_t 1942msg_receive_error( 1943 ipc_kmsg_t kmsg, 1944 mach_vm_address_t msg_addr, 1945 mach_msg_option_t option, 1946 mach_port_seqno_t seqno, 1947 ipc_space_t space) 1948{ 1949 mach_msg_format_0_trailer_t *trailer; 1950 1951 /* 1952 * Copy out the destination port in the message. 1953 * Destroy all other rights and memory in the message. 1954 */ 1955 ipc_kmsg_copyout_dest(kmsg, space); 1956 1957 /* 1958 * Build a minimal message with the requested trailer. 1959 */ 1960 trailer = (mach_msg_format_0_trailer_t *) 1961 ((vm_offset_t)kmsg->ikm_header + 1962 round_msg(sizeof(mach_msg_header_t))); 1963 kmsg->ikm_header->msgh_size = sizeof(mach_msg_header_t); 1964 bcopy( (char *)&trailer_template, 1965 (char *)trailer, 1966 sizeof(trailer_template)); 1967 if (option & MACH_RCV_TRAILER_MASK) { 1968 trailer->msgh_seqno = seqno; 1969 trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option); 1970 } 1971 1972 /* 1973 * Copy the message to user space 1974 */ 1975 if (ipc_kmsg_put(msg_addr, kmsg, kmsg->ikm_header->msgh_size + 1976 trailer->msgh_trailer_size) == MACH_RCV_INVALID_DATA) 1977 return(MACH_RCV_INVALID_DATA); 1978 else 1979 return(MACH_MSG_SUCCESS); 1980} 1981