1/* 2 * Copyright (c) 2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. Please obtain a copy of the License at 10 * http://www.opensource.apple.com/apsl/ and read it before using this 11 * file. 12 * 13 * The Original Code and all software distributed under the License are 14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 18 * Please see the License for the specific language governing rights and 19 * limitations under the License. 20 * 21 * @APPLE_LICENSE_HEADER_END@ 22 */ 23 24/* 25 * Shared region (... and comm page) 26 * 27 * This file handles the VM shared region and comm page. 28 * 29 */ 30/* 31 * SHARED REGIONS 32 * -------------- 33 * 34 * A shared region is a submap that contains the most common system shared 35 * libraries for a given environment. 36 * An environment is defined by (cpu-type, 64-bitness, root directory). 37 * 38 * The point of a shared region is to reduce the setup overhead when exec'ing 39 * a new process. 40 * A shared region uses a shared VM submap that gets mapped automatically 41 * at exec() time (see vm_map_exec()). The first process of a given 42 * environment sets up the shared region and all further processes in that 43 * environment can re-use that shared region without having to re-create 44 * the same mappings in their VM map. All they need is contained in the shared 45 * region. 46 * It can also shared a pmap (mostly for read-only parts but also for the 47 * initial version of some writable parts), which gets "nested" into the 48 * process's pmap. This reduces the number of soft faults: once one process 49 * brings in a page in the shared region, all the other processes can access 50 * it without having to enter it in their own pmap. 51 * 52 * 53 * When a process is being exec'ed, vm_map_exec() calls vm_shared_region_enter() 54 * to map the appropriate shared region in the process's address space. 55 * We look up the appropriate shared region for the process's environment. 56 * If we can't find one, we create a new (empty) one and add it to the list. 57 * Otherwise, we just take an extra reference on the shared region we found. 58 * 59 * The "dyld" runtime (mapped into the process's address space at exec() time) 60 * will then use the shared_region_check_np() and shared_region_map_np() 61 * system call to validate and/or populate the shared region with the 62 * appropriate dyld_shared_cache file. 63 * 64 * The shared region is inherited on fork() and the child simply takes an 65 * extra reference on its parent's shared region. 66 * 67 * When the task terminates, we release a reference on its shared region. 68 * When the last reference is released, we destroy the shared region. 69 * 70 * After a chroot(), the calling process keeps using its original shared region, 71 * since that's what was mapped when it was started. But its children 72 * will use a different shared region, because they need to use the shared 73 * cache that's relative to the new root directory. 74 */ 75/* 76 * COMM PAGE 77 * 78 * A "comm page" is an area of memory that is populated by the kernel with 79 * the appropriate platform-specific version of some commonly used code. 80 * There is one "comm page" per platform (cpu-type, 64-bitness) but only 81 * for the native cpu-type. No need to overly optimize translated code 82 * for hardware that is not really there ! 83 * 84 * The comm pages are created and populated at boot time. 85 * 86 * The appropriate comm page is mapped into a process's address space 87 * at exec() time, in vm_map_exec(). 88 * It is then inherited on fork(). 89 * 90 * The comm page is shared between the kernel and all applications of 91 * a given platform. Only the kernel can modify it. 92 * 93 * Applications just branch to fixed addresses in the comm page and find 94 * the right version of the code for the platform. There is also some 95 * data provided and updated by the kernel for processes to retrieve easily 96 * without having to do a system call. 97 */ 98 99#include <debug.h> 100 101#include <kern/ipc_tt.h> 102#include <kern/kalloc.h> 103#include <kern/thread_call.h> 104 105#include <mach/mach_vm.h> 106 107#include <vm/vm_map.h> 108#include <vm/vm_shared_region.h> 109 110#include <vm/vm_protos.h> 111 112#include <machine/commpage.h> 113#include <machine/cpu_capabilities.h> 114 115/* "dyld" uses this to figure out what the kernel supports */ 116int shared_region_version = 3; 117 118/* trace level, output is sent to the system log file */ 119int shared_region_trace_level = SHARED_REGION_TRACE_ERROR_LVL; 120 121/* should local (non-chroot) shared regions persist when no task uses them ? */ 122int shared_region_persistence = 0; /* no by default */ 123 124/* delay before reclaiming an unused shared region */ 125int shared_region_destroy_delay = 120; /* in seconds */ 126 127/* indicate if the shared region has been slid. Only one region can be slid */ 128boolean_t shared_region_completed_slide = FALSE; 129 130/* this lock protects all the shared region data structures */ 131lck_grp_t *vm_shared_region_lck_grp; 132lck_mtx_t vm_shared_region_lock; 133 134#define vm_shared_region_lock() lck_mtx_lock(&vm_shared_region_lock) 135#define vm_shared_region_unlock() lck_mtx_unlock(&vm_shared_region_lock) 136#define vm_shared_region_sleep(event, interruptible) \ 137 lck_mtx_sleep(&vm_shared_region_lock, \ 138 LCK_SLEEP_DEFAULT, \ 139 (event_t) (event), \ 140 (interruptible)) 141 142/* the list of currently available shared regions (one per environment) */ 143queue_head_t vm_shared_region_queue; 144 145static void vm_shared_region_reference_locked(vm_shared_region_t shared_region); 146static vm_shared_region_t vm_shared_region_create( 147 void *root_dir, 148 cpu_type_t cputype, 149 boolean_t is_64bit); 150static void vm_shared_region_destroy(vm_shared_region_t shared_region); 151 152static void vm_shared_region_timeout(thread_call_param_t param0, 153 thread_call_param_t param1); 154 155static int __commpage_setup = 0; 156#if defined(__i386__) || defined(__x86_64__) 157static int __system_power_source = 1; /* init to extrnal power source */ 158static void post_sys_powersource_internal(int i, int internal); 159#endif /* __i386__ || __x86_64__ */ 160 161 162/* 163 * Initialize the module... 164 */ 165void 166vm_shared_region_init(void) 167{ 168 SHARED_REGION_TRACE_DEBUG( 169 ("shared_region: -> init\n")); 170 171 vm_shared_region_lck_grp = lck_grp_alloc_init("vm shared region", 172 LCK_GRP_ATTR_NULL); 173 lck_mtx_init(&vm_shared_region_lock, 174 vm_shared_region_lck_grp, 175 LCK_ATTR_NULL); 176 177 queue_init(&vm_shared_region_queue); 178 179 SHARED_REGION_TRACE_DEBUG( 180 ("shared_region: <- init\n")); 181} 182 183/* 184 * Retrieve a task's shared region and grab an extra reference to 185 * make sure it doesn't disappear while the caller is using it. 186 * The caller is responsible for consuming that extra reference if 187 * necessary. 188 */ 189vm_shared_region_t 190vm_shared_region_get( 191 task_t task) 192{ 193 vm_shared_region_t shared_region; 194 195 SHARED_REGION_TRACE_DEBUG( 196 ("shared_region: -> get(%p)\n", 197 task)); 198 199 task_lock(task); 200 vm_shared_region_lock(); 201 shared_region = task->shared_region; 202 if (shared_region) { 203 assert(shared_region->sr_ref_count > 0); 204 vm_shared_region_reference_locked(shared_region); 205 } 206 vm_shared_region_unlock(); 207 task_unlock(task); 208 209 SHARED_REGION_TRACE_DEBUG( 210 ("shared_region: get(%p) <- %p\n", 211 task, shared_region)); 212 213 return shared_region; 214} 215 216/* 217 * Get the base address of the shared region. 218 * That's the address at which it needs to be mapped in the process's address 219 * space. 220 * No need to lock since this data is set when the shared region is 221 * created and is never modified after that. The caller must hold an extra 222 * reference on the shared region to prevent it from being destroyed. 223 */ 224mach_vm_offset_t 225vm_shared_region_base_address( 226 vm_shared_region_t shared_region) 227{ 228 SHARED_REGION_TRACE_DEBUG( 229 ("shared_region: -> base_address(%p)\n", 230 shared_region)); 231 assert(shared_region->sr_ref_count > 1); 232 SHARED_REGION_TRACE_DEBUG( 233 ("shared_region: base_address(%p) <- 0x%llx\n", 234 shared_region, (long long)shared_region->sr_base_address)); 235 return shared_region->sr_base_address; 236} 237 238/* 239 * Get the size of the shared region. 240 * That's the size that needs to be mapped in the process's address 241 * space. 242 * No need to lock since this data is set when the shared region is 243 * created and is never modified after that. The caller must hold an extra 244 * reference on the shared region to prevent it from being destroyed. 245 */ 246mach_vm_size_t 247vm_shared_region_size( 248 vm_shared_region_t shared_region) 249{ 250 SHARED_REGION_TRACE_DEBUG( 251 ("shared_region: -> size(%p)\n", 252 shared_region)); 253 assert(shared_region->sr_ref_count > 1); 254 SHARED_REGION_TRACE_DEBUG( 255 ("shared_region: size(%p) <- 0x%llx\n", 256 shared_region, (long long)shared_region->sr_size)); 257 return shared_region->sr_size; 258} 259 260/* 261 * Get the memory entry of the shared region. 262 * That's the "memory object" that needs to be mapped in the process's address 263 * space. 264 * No need to lock since this data is set when the shared region is 265 * created and is never modified after that. The caller must hold an extra 266 * reference on the shared region to prevent it from being destroyed. 267 */ 268ipc_port_t 269vm_shared_region_mem_entry( 270 vm_shared_region_t shared_region) 271{ 272 SHARED_REGION_TRACE_DEBUG( 273 ("shared_region: -> mem_entry(%p)\n", 274 shared_region)); 275 assert(shared_region->sr_ref_count > 1); 276 SHARED_REGION_TRACE_DEBUG( 277 ("shared_region: mem_entry(%p) <- %p\n", 278 shared_region, shared_region->sr_mem_entry)); 279 return shared_region->sr_mem_entry; 280} 281 282/* 283 * Set the shared region the process should use. 284 * A NULL new shared region means that we just want to release the old 285 * shared region. 286 * The caller should already have an extra reference on the new shared region 287 * (if any). We release a reference on the old shared region (if any). 288 */ 289void 290vm_shared_region_set( 291 task_t task, 292 vm_shared_region_t new_shared_region) 293{ 294 vm_shared_region_t old_shared_region; 295 296 SHARED_REGION_TRACE_DEBUG( 297 ("shared_region: -> set(%p, %p)\n", 298 task, new_shared_region)); 299 300 task_lock(task); 301 vm_shared_region_lock(); 302 303 old_shared_region = task->shared_region; 304 if (new_shared_region) { 305 assert(new_shared_region->sr_ref_count > 0); 306 } 307 308 task->shared_region = new_shared_region; 309 310 vm_shared_region_unlock(); 311 task_unlock(task); 312 313 if (old_shared_region) { 314 assert(old_shared_region->sr_ref_count > 0); 315 vm_shared_region_deallocate(old_shared_region); 316 } 317 318 SHARED_REGION_TRACE_DEBUG( 319 ("shared_region: set(%p) <- old=%p new=%p\n", 320 task, old_shared_region, new_shared_region)); 321} 322 323/* 324 * Lookup up the shared region for the desired environment. 325 * If none is found, create a new (empty) one. 326 * Grab an extra reference on the returned shared region, to make sure 327 * it doesn't get destroyed before the caller is done with it. The caller 328 * is responsible for consuming that extra reference if necessary. 329 */ 330vm_shared_region_t 331vm_shared_region_lookup( 332 void *root_dir, 333 cpu_type_t cputype, 334 boolean_t is_64bit) 335{ 336 vm_shared_region_t shared_region; 337 vm_shared_region_t new_shared_region; 338 339 SHARED_REGION_TRACE_DEBUG( 340 ("shared_region: -> lookup(root=%p,cpu=%d,64bit=%d)\n", 341 root_dir, cputype, is_64bit)); 342 343 shared_region = NULL; 344 new_shared_region = NULL; 345 346 vm_shared_region_lock(); 347 for (;;) { 348 queue_iterate(&vm_shared_region_queue, 349 shared_region, 350 vm_shared_region_t, 351 sr_q) { 352 assert(shared_region->sr_ref_count > 0); 353 if (shared_region->sr_cpu_type == cputype && 354 shared_region->sr_root_dir == root_dir && 355 shared_region->sr_64bit == is_64bit) { 356 /* found a match ! */ 357 vm_shared_region_reference_locked(shared_region); 358 goto done; 359 } 360 } 361 if (new_shared_region == NULL) { 362 /* no match: create a new one */ 363 vm_shared_region_unlock(); 364 new_shared_region = vm_shared_region_create(root_dir, 365 cputype, 366 is_64bit); 367 /* do the lookup again, in case we lost a race */ 368 vm_shared_region_lock(); 369 continue; 370 } 371 /* still no match: use our new one */ 372 shared_region = new_shared_region; 373 new_shared_region = NULL; 374 queue_enter(&vm_shared_region_queue, 375 shared_region, 376 vm_shared_region_t, 377 sr_q); 378 break; 379 } 380 381done: 382 vm_shared_region_unlock(); 383 384 if (new_shared_region) { 385 /* 386 * We lost a race with someone else to create a new shared 387 * region for that environment. Get rid of our unused one. 388 */ 389 assert(new_shared_region->sr_ref_count == 1); 390 new_shared_region->sr_ref_count--; 391 vm_shared_region_destroy(new_shared_region); 392 new_shared_region = NULL; 393 } 394 395 SHARED_REGION_TRACE_DEBUG( 396 ("shared_region: lookup(root=%p,cpu=%d,64bit=%d) <- %p\n", 397 root_dir, cputype, is_64bit, shared_region)); 398 399 assert(shared_region->sr_ref_count > 0); 400 return shared_region; 401} 402 403/* 404 * Take an extra reference on a shared region. 405 * The vm_shared_region_lock should already be held by the caller. 406 */ 407static void 408vm_shared_region_reference_locked( 409 vm_shared_region_t shared_region) 410{ 411#if DEBUG 412 lck_mtx_assert(&vm_shared_region_lock, LCK_MTX_ASSERT_OWNED); 413#endif 414 415 SHARED_REGION_TRACE_DEBUG( 416 ("shared_region: -> reference_locked(%p)\n", 417 shared_region)); 418 assert(shared_region->sr_ref_count > 0); 419 shared_region->sr_ref_count++; 420 421 if (shared_region->sr_timer_call != NULL) { 422 boolean_t cancelled; 423 424 /* cancel and free any pending timeout */ 425 cancelled = thread_call_cancel(shared_region->sr_timer_call); 426 if (cancelled) { 427 thread_call_free(shared_region->sr_timer_call); 428 shared_region->sr_timer_call = NULL; 429 /* release the reference held by the cancelled timer */ 430 shared_region->sr_ref_count--; 431 } else { 432 /* the timer will drop the reference and free itself */ 433 } 434 } 435 436 SHARED_REGION_TRACE_DEBUG( 437 ("shared_region: reference_locked(%p) <- %d\n", 438 shared_region, shared_region->sr_ref_count)); 439} 440 441/* 442 * Release a reference on the shared region. 443 * Destroy it if there are no references left. 444 */ 445void 446vm_shared_region_deallocate( 447 vm_shared_region_t shared_region) 448{ 449 SHARED_REGION_TRACE_DEBUG( 450 ("shared_region: -> deallocate(%p)\n", 451 shared_region)); 452 453 vm_shared_region_lock(); 454 455 assert(shared_region->sr_ref_count > 0); 456 457 if (shared_region->sr_root_dir == NULL) { 458 /* 459 * Local (i.e. based on the boot volume) shared regions 460 * can persist or not based on the "shared_region_persistence" 461 * sysctl. 462 * Make sure that this one complies. 463 */ 464 if (shared_region_persistence && 465 !shared_region->sr_persists) { 466 /* make this one persistent */ 467 shared_region->sr_ref_count++; 468 shared_region->sr_persists = TRUE; 469 } else if (!shared_region_persistence && 470 shared_region->sr_persists) { 471 /* make this one no longer persistent */ 472 assert(shared_region->sr_ref_count > 1); 473 shared_region->sr_ref_count--; 474 shared_region->sr_persists = FALSE; 475 } 476 } 477 478 assert(shared_region->sr_ref_count > 0); 479 shared_region->sr_ref_count--; 480 SHARED_REGION_TRACE_DEBUG( 481 ("shared_region: deallocate(%p): ref now %d\n", 482 shared_region, shared_region->sr_ref_count)); 483 484 if (shared_region->sr_ref_count == 0) { 485 uint64_t deadline; 486 487 if (shared_region->sr_timer_call == NULL) { 488 /* hold one reference for the timer */ 489 assert(! shared_region->sr_mapping_in_progress); 490 shared_region->sr_ref_count++; 491 492 /* set up the timer */ 493 shared_region->sr_timer_call = thread_call_allocate( 494 (thread_call_func_t) vm_shared_region_timeout, 495 (thread_call_param_t) shared_region); 496 497 /* schedule the timer */ 498 clock_interval_to_deadline(shared_region_destroy_delay, 499 1000 * 1000 * 1000, 500 &deadline); 501 thread_call_enter_delayed(shared_region->sr_timer_call, 502 deadline); 503 504 SHARED_REGION_TRACE_DEBUG( 505 ("shared_region: deallocate(%p): armed timer\n", 506 shared_region)); 507 508 vm_shared_region_unlock(); 509 } else { 510 /* timer expired: let go of this shared region */ 511 512 /* 513 * Remove it from the queue first, so no one can find 514 * it... 515 */ 516 queue_remove(&vm_shared_region_queue, 517 shared_region, 518 vm_shared_region_t, 519 sr_q); 520 vm_shared_region_unlock(); 521 /* ... and destroy it */ 522 vm_shared_region_destroy(shared_region); 523 shared_region = NULL; 524 } 525 } else { 526 vm_shared_region_unlock(); 527 } 528 529 SHARED_REGION_TRACE_DEBUG( 530 ("shared_region: deallocate(%p) <-\n", 531 shared_region)); 532} 533 534void 535vm_shared_region_timeout( 536 thread_call_param_t param0, 537 __unused thread_call_param_t param1) 538{ 539 vm_shared_region_t shared_region; 540 541 shared_region = (vm_shared_region_t) param0; 542 543 vm_shared_region_deallocate(shared_region); 544} 545 546/* 547 * Create a new (empty) shared region for a new environment. 548 */ 549static vm_shared_region_t 550vm_shared_region_create( 551 void *root_dir, 552 cpu_type_t cputype, 553 boolean_t is_64bit) 554{ 555 kern_return_t kr; 556 vm_named_entry_t mem_entry; 557 ipc_port_t mem_entry_port; 558 vm_shared_region_t shared_region; 559 vm_map_t sub_map; 560 mach_vm_offset_t base_address, pmap_nesting_start; 561 mach_vm_size_t size, pmap_nesting_size; 562 563 SHARED_REGION_TRACE_DEBUG( 564 ("shared_region: -> create(root=%p,cpu=%d,64bit=%d)\n", 565 root_dir, cputype, is_64bit)); 566 567 base_address = 0; 568 size = 0; 569 mem_entry = NULL; 570 mem_entry_port = IPC_PORT_NULL; 571 sub_map = VM_MAP_NULL; 572 573 /* create a new shared region structure... */ 574 shared_region = kalloc(sizeof (*shared_region)); 575 if (shared_region == NULL) { 576 SHARED_REGION_TRACE_ERROR( 577 ("shared_region: create: couldn't allocate\n")); 578 goto done; 579 } 580 581 /* figure out the correct settings for the desired environment */ 582 if (is_64bit) { 583 switch (cputype) { 584 case CPU_TYPE_I386: 585 base_address = SHARED_REGION_BASE_X86_64; 586 size = SHARED_REGION_SIZE_X86_64; 587 pmap_nesting_start = SHARED_REGION_NESTING_BASE_X86_64; 588 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_X86_64; 589 break; 590 case CPU_TYPE_POWERPC: 591 base_address = SHARED_REGION_BASE_PPC64; 592 size = SHARED_REGION_SIZE_PPC64; 593 pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC64; 594 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC64; 595 break; 596 default: 597 SHARED_REGION_TRACE_ERROR( 598 ("shared_region: create: unknown cpu type %d\n", 599 cputype)); 600 kfree(shared_region, sizeof (*shared_region)); 601 shared_region = NULL; 602 goto done; 603 } 604 } else { 605 switch (cputype) { 606 case CPU_TYPE_I386: 607 base_address = SHARED_REGION_BASE_I386; 608 size = SHARED_REGION_SIZE_I386; 609 pmap_nesting_start = SHARED_REGION_NESTING_BASE_I386; 610 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_I386; 611 break; 612 case CPU_TYPE_POWERPC: 613 base_address = SHARED_REGION_BASE_PPC; 614 size = SHARED_REGION_SIZE_PPC; 615 pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC; 616 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC; 617 break; 618#ifdef CPU_TYPE_ARM 619 case CPU_TYPE_ARM: 620 base_address = SHARED_REGION_BASE_ARM; 621 size = SHARED_REGION_SIZE_ARM; 622 pmap_nesting_start = SHARED_REGION_NESTING_BASE_ARM; 623 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_ARM; 624 break; 625#endif /* CPU_TYPE_ARM */ 626 default: 627 SHARED_REGION_TRACE_ERROR( 628 ("shared_region: create: unknown cpu type %d\n", 629 cputype)); 630 kfree(shared_region, sizeof (*shared_region)); 631 shared_region = NULL; 632 goto done; 633 634 } 635 } 636 637 /* create a memory entry structure and a Mach port handle */ 638 kr = mach_memory_entry_allocate(&mem_entry, 639 &mem_entry_port); 640 if (kr != KERN_SUCCESS) { 641 kfree(shared_region, sizeof (*shared_region)); 642 shared_region = NULL; 643 SHARED_REGION_TRACE_ERROR( 644 ("shared_region: create: " 645 "couldn't allocate mem_entry\n")); 646 goto done; 647 } 648 649 /* create a VM sub map and its pmap */ 650 sub_map = vm_map_create(pmap_create(NULL, 0, is_64bit), 651 0, size, 652 TRUE); 653 if (sub_map == VM_MAP_NULL) { 654 ipc_port_release_send(mem_entry_port); 655 kfree(shared_region, sizeof (*shared_region)); 656 shared_region = NULL; 657 SHARED_REGION_TRACE_ERROR( 658 ("shared_region: create: " 659 "couldn't allocate map\n")); 660 goto done; 661 } 662 663 /* make the memory entry point to the VM sub map */ 664 mem_entry->is_sub_map = TRUE; 665 mem_entry->backing.map = sub_map; 666 mem_entry->size = size; 667 mem_entry->protection = VM_PROT_ALL; 668 669 /* make the shared region point at the memory entry */ 670 shared_region->sr_mem_entry = mem_entry_port; 671 672 /* fill in the shared region's environment and settings */ 673 shared_region->sr_base_address = base_address; 674 shared_region->sr_size = size; 675 shared_region->sr_pmap_nesting_start = pmap_nesting_start; 676 shared_region->sr_pmap_nesting_size = pmap_nesting_size; 677 shared_region->sr_cpu_type = cputype; 678 shared_region->sr_64bit = is_64bit; 679 shared_region->sr_root_dir = root_dir; 680 681 queue_init(&shared_region->sr_q); 682 shared_region->sr_mapping_in_progress = FALSE; 683 shared_region->sr_persists = FALSE; 684 shared_region->sr_timer_call = NULL; 685 shared_region->sr_first_mapping = (mach_vm_offset_t) -1; 686 687 /* grab a reference for the caller */ 688 shared_region->sr_ref_count = 1; 689 690done: 691 if (shared_region) { 692 SHARED_REGION_TRACE_INFO( 693 ("shared_region: create(root=%p,cpu=%d,64bit=%d," 694 "base=0x%llx,size=0x%llx) <- " 695 "%p mem=(%p,%p) map=%p pmap=%p\n", 696 root_dir, cputype, is_64bit, (long long)base_address, 697 (long long)size, shared_region, 698 mem_entry_port, mem_entry, sub_map, sub_map->pmap)); 699 } else { 700 SHARED_REGION_TRACE_INFO( 701 ("shared_region: create(root=%p,cpu=%d,64bit=%d," 702 "base=0x%llx,size=0x%llx) <- NULL", 703 root_dir, cputype, is_64bit, (long long)base_address, 704 (long long)size)); 705 } 706 return shared_region; 707} 708 709/* 710 * Destroy a now-unused shared region. 711 * The shared region is no longer in the queue and can not be looked up. 712 */ 713static void 714vm_shared_region_destroy( 715 vm_shared_region_t shared_region) 716{ 717 vm_named_entry_t mem_entry; 718 vm_map_t map; 719 720 SHARED_REGION_TRACE_INFO( 721 ("shared_region: -> destroy(%p) (root=%p,cpu=%d,64bit=%d)\n", 722 shared_region, 723 shared_region->sr_root_dir, 724 shared_region->sr_cpu_type, 725 shared_region->sr_64bit)); 726 727 assert(shared_region->sr_ref_count == 0); 728 assert(!shared_region->sr_persists); 729 730 mem_entry = (vm_named_entry_t) shared_region->sr_mem_entry->ip_kobject; 731 assert(mem_entry->is_sub_map); 732 assert(!mem_entry->internal); 733 assert(!mem_entry->is_pager); 734 map = mem_entry->backing.map; 735 736 /* 737 * Clean up the pmap first. The virtual addresses that were 738 * entered in this possibly "nested" pmap may have different values 739 * than the VM map's min and max offsets, if the VM sub map was 740 * mapped at a non-zero offset in the processes' main VM maps, which 741 * is usually the case, so the clean-up we do in vm_map_destroy() would 742 * not be enough. 743 */ 744 if (map->pmap) { 745 pmap_remove(map->pmap, 746 shared_region->sr_base_address, 747 (shared_region->sr_base_address + 748 shared_region->sr_size)); 749 } 750 751 /* 752 * Release our (one and only) handle on the memory entry. 753 * This will generate a no-senders notification, which will be processed 754 * by ipc_kobject_notify(), which will release the one and only 755 * reference on the memory entry and cause it to be destroyed, along 756 * with the VM sub map and its pmap. 757 */ 758 mach_memory_entry_port_release(shared_region->sr_mem_entry); 759 mem_entry = NULL; 760 shared_region->sr_mem_entry = IPC_PORT_NULL; 761 762 if (shared_region->sr_timer_call) { 763 thread_call_free(shared_region->sr_timer_call); 764 } 765 766 if ((slide_info.slide_info_entry != NULL) && (slide_info.sr == shared_region)) { 767 kmem_free(kernel_map, 768 (vm_offset_t) slide_info.slide_info_entry, 769 (vm_size_t) slide_info.slide_info_size); 770 vm_object_deallocate(slide_info.slide_object); 771 slide_info.slide_object = NULL; 772 slide_info.start = 0; 773 slide_info.end = 0; 774 slide_info.slide = 0; 775 slide_info.sr = NULL; 776 slide_info.slide_info_entry = NULL; 777 slide_info.slide_info_size = 0; 778 shared_region_completed_slide = FALSE; 779 } 780 781 /* release the shared region structure... */ 782 kfree(shared_region, sizeof (*shared_region)); 783 784 SHARED_REGION_TRACE_DEBUG( 785 ("shared_region: destroy(%p) <-\n", 786 shared_region)); 787 shared_region = NULL; 788 789} 790 791/* 792 * Gets the address of the first (in time) mapping in the shared region. 793 */ 794kern_return_t 795vm_shared_region_start_address( 796 vm_shared_region_t shared_region, 797 mach_vm_offset_t *start_address) 798{ 799 kern_return_t kr; 800 mach_vm_offset_t sr_base_address; 801 mach_vm_offset_t sr_first_mapping; 802 803 SHARED_REGION_TRACE_DEBUG( 804 ("shared_region: -> start_address(%p)\n", 805 shared_region)); 806 assert(shared_region->sr_ref_count > 1); 807 808 vm_shared_region_lock(); 809 810 /* 811 * Wait if there's another thread establishing a mapping 812 * in this shared region right when we're looking at it. 813 * We want a consistent view of the map... 814 */ 815 while (shared_region->sr_mapping_in_progress) { 816 /* wait for our turn... */ 817 assert(shared_region->sr_ref_count > 1); 818 vm_shared_region_sleep(&shared_region->sr_mapping_in_progress, 819 THREAD_UNINT); 820 } 821 assert(! shared_region->sr_mapping_in_progress); 822 assert(shared_region->sr_ref_count > 1); 823 824 sr_base_address = shared_region->sr_base_address; 825 sr_first_mapping = shared_region->sr_first_mapping; 826 827 if (sr_first_mapping == (mach_vm_offset_t) -1) { 828 /* shared region is empty */ 829 kr = KERN_INVALID_ADDRESS; 830 } else { 831 kr = KERN_SUCCESS; 832 *start_address = sr_base_address + sr_first_mapping; 833 } 834 835 vm_shared_region_unlock(); 836 837 SHARED_REGION_TRACE_DEBUG( 838 ("shared_region: start_address(%p) <- 0x%llx\n", 839 shared_region, (long long)shared_region->sr_base_address)); 840 841 return kr; 842} 843 844void 845vm_shared_region_undo_mappings( 846 vm_map_t sr_map, 847 mach_vm_offset_t sr_base_address, 848 struct shared_file_mapping_np *mappings, 849 unsigned int mappings_count) 850{ 851 unsigned int j = 0; 852 vm_shared_region_t shared_region = NULL; 853 boolean_t reset_shared_region_state = FALSE; 854 855 shared_region = vm_shared_region_get(current_task()); 856 if (shared_region == NULL) { 857 printf("Failed to undo mappings because of NULL shared region.\n"); 858 return; 859 } 860 861 862 if (sr_map == NULL) { 863 ipc_port_t sr_handle; 864 vm_named_entry_t sr_mem_entry; 865 866 vm_shared_region_lock(); 867 assert(shared_region->sr_ref_count > 1); 868 869 while (shared_region->sr_mapping_in_progress) { 870 /* wait for our turn... */ 871 vm_shared_region_sleep(&shared_region->sr_mapping_in_progress, 872 THREAD_UNINT); 873 } 874 assert(! shared_region->sr_mapping_in_progress); 875 assert(shared_region->sr_ref_count > 1); 876 /* let others know we're working in this shared region */ 877 shared_region->sr_mapping_in_progress = TRUE; 878 879 vm_shared_region_unlock(); 880 881 reset_shared_region_state = TRUE; 882 883 /* no need to lock because this data is never modified... */ 884 sr_handle = shared_region->sr_mem_entry; 885 sr_mem_entry = (vm_named_entry_t) sr_handle->ip_kobject; 886 sr_map = sr_mem_entry->backing.map; 887 sr_base_address = shared_region->sr_base_address; 888 } 889 /* 890 * Undo the mappings we've established so far. 891 */ 892 for (j = 0; j < mappings_count; j++) { 893 kern_return_t kr2; 894 895 if (mappings[j].sfm_size == 0) { 896 /* 897 * We didn't establish this 898 * mapping, so nothing to undo. 899 */ 900 continue; 901 } 902 SHARED_REGION_TRACE_INFO( 903 ("shared_region: mapping[%d]: " 904 "address:0x%016llx " 905 "size:0x%016llx " 906 "offset:0x%016llx " 907 "maxprot:0x%x prot:0x%x: " 908 "undoing...\n", 909 j, 910 (long long)mappings[j].sfm_address, 911 (long long)mappings[j].sfm_size, 912 (long long)mappings[j].sfm_file_offset, 913 mappings[j].sfm_max_prot, 914 mappings[j].sfm_init_prot)); 915 kr2 = mach_vm_deallocate( 916 sr_map, 917 (mappings[j].sfm_address - 918 sr_base_address), 919 mappings[j].sfm_size); 920 assert(kr2 == KERN_SUCCESS); 921 } 922 923 /* 924 * This is how check_np() knows if the shared region 925 * is mapped. So clear it here. 926 */ 927 shared_region->sr_first_mapping = (mach_vm_offset_t) -1; 928 929 if (reset_shared_region_state) { 930 vm_shared_region_lock(); 931 assert(shared_region->sr_ref_count > 1); 932 assert(shared_region->sr_mapping_in_progress); 933 /* we're done working on that shared region */ 934 shared_region->sr_mapping_in_progress = FALSE; 935 thread_wakeup((event_t) &shared_region->sr_mapping_in_progress); 936 vm_shared_region_unlock(); 937 reset_shared_region_state = FALSE; 938 } 939 940 vm_shared_region_deallocate(shared_region); 941} 942 943/* 944 * Establish some mappings of a file in the shared region. 945 * This is used by "dyld" via the shared_region_map_np() system call 946 * to populate the shared region with the appropriate shared cache. 947 * 948 * One could also call it several times to incrementally load several 949 * libraries, as long as they do not overlap. 950 * It will return KERN_SUCCESS if the mappings were successfully established 951 * or if they were already established identically by another process. 952 */ 953kern_return_t 954vm_shared_region_map_file( 955 vm_shared_region_t shared_region, 956 unsigned int mappings_count, 957 struct shared_file_mapping_np *mappings, 958 memory_object_control_t file_control, 959 memory_object_size_t file_size, 960 void *root_dir, 961 struct shared_file_mapping_np *mapping_to_slide) 962{ 963 kern_return_t kr; 964 vm_object_t file_object; 965 ipc_port_t sr_handle; 966 vm_named_entry_t sr_mem_entry; 967 vm_map_t sr_map; 968 mach_vm_offset_t sr_base_address; 969 unsigned int i; 970 mach_port_t map_port; 971 vm_map_offset_t target_address; 972 vm_object_t object; 973 vm_object_size_t obj_size; 974 boolean_t found_mapping_to_slide = FALSE; 975 976 977 kr = KERN_SUCCESS; 978 979 vm_shared_region_lock(); 980 assert(shared_region->sr_ref_count > 1); 981 982 if (shared_region->sr_root_dir != root_dir) { 983 /* 984 * This shared region doesn't match the current root 985 * directory of this process. Deny the mapping to 986 * avoid tainting the shared region with something that 987 * doesn't quite belong into it. 988 */ 989 vm_shared_region_unlock(); 990 kr = KERN_PROTECTION_FAILURE; 991 goto done; 992 } 993 994 /* 995 * Make sure we handle only one mapping at a time in a given 996 * shared region, to avoid race conditions. This should not 997 * happen frequently... 998 */ 999 while (shared_region->sr_mapping_in_progress) { 1000 /* wait for our turn... */ 1001 vm_shared_region_sleep(&shared_region->sr_mapping_in_progress, 1002 THREAD_UNINT); 1003 } 1004 assert(! shared_region->sr_mapping_in_progress); 1005 assert(shared_region->sr_ref_count > 1); 1006 /* let others know we're working in this shared region */ 1007 shared_region->sr_mapping_in_progress = TRUE; 1008 1009 vm_shared_region_unlock(); 1010 1011 /* no need to lock because this data is never modified... */ 1012 sr_handle = shared_region->sr_mem_entry; 1013 sr_mem_entry = (vm_named_entry_t) sr_handle->ip_kobject; 1014 sr_map = sr_mem_entry->backing.map; 1015 sr_base_address = shared_region->sr_base_address; 1016 1017 SHARED_REGION_TRACE_DEBUG( 1018 ("shared_region: -> map(%p,%d,%p,%p,0x%llx)\n", 1019 shared_region, mappings_count, mappings, 1020 file_control, file_size)); 1021 1022 /* get the VM object associated with the file to be mapped */ 1023 file_object = memory_object_control_to_vm_object(file_control); 1024 1025 /* establish the mappings */ 1026 for (i = 0; i < mappings_count; i++) { 1027 SHARED_REGION_TRACE_INFO( 1028 ("shared_region: mapping[%d]: " 1029 "address:0x%016llx size:0x%016llx offset:0x%016llx " 1030 "maxprot:0x%x prot:0x%x\n", 1031 i, 1032 (long long)mappings[i].sfm_address, 1033 (long long)mappings[i].sfm_size, 1034 (long long)mappings[i].sfm_file_offset, 1035 mappings[i].sfm_max_prot, 1036 mappings[i].sfm_init_prot)); 1037 1038 if (mappings[i].sfm_init_prot & VM_PROT_ZF) { 1039 /* zero-filled memory */ 1040 map_port = MACH_PORT_NULL; 1041 } else { 1042 /* file-backed memory */ 1043 map_port = (ipc_port_t) file_object->pager; 1044 } 1045 1046 if (mappings[i].sfm_init_prot & VM_PROT_SLIDE) { 1047 /* 1048 * This is the mapping that needs to be slid. 1049 */ 1050 if (found_mapping_to_slide == TRUE) { 1051 SHARED_REGION_TRACE_INFO( 1052 ("shared_region: mapping[%d]: " 1053 "address:0x%016llx size:0x%016llx " 1054 "offset:0x%016llx " 1055 "maxprot:0x%x prot:0x%x " 1056 "will not be slid as only one such mapping is allowed...\n", 1057 i, 1058 (long long)mappings[i].sfm_address, 1059 (long long)mappings[i].sfm_size, 1060 (long long)mappings[i].sfm_file_offset, 1061 mappings[i].sfm_max_prot, 1062 mappings[i].sfm_init_prot)); 1063 } else { 1064 if (mapping_to_slide != NULL) { 1065 mapping_to_slide->sfm_file_offset = mappings[i].sfm_file_offset; 1066 mapping_to_slide->sfm_size = mappings[i].sfm_size; 1067 found_mapping_to_slide = TRUE; 1068 } 1069 } 1070 } 1071 1072 /* mapping's address is relative to the shared region base */ 1073 target_address = 1074 mappings[i].sfm_address - sr_base_address; 1075 1076 /* establish that mapping, OK if it's "already" there */ 1077 if (map_port == MACH_PORT_NULL) { 1078 /* 1079 * We want to map some anonymous memory in a 1080 * shared region. 1081 * We have to create the VM object now, so that it 1082 * can be mapped "copy-on-write". 1083 */ 1084 obj_size = vm_map_round_page(mappings[i].sfm_size); 1085 object = vm_object_allocate(obj_size); 1086 if (object == VM_OBJECT_NULL) { 1087 kr = KERN_RESOURCE_SHORTAGE; 1088 } else { 1089 kr = vm_map_enter( 1090 sr_map, 1091 &target_address, 1092 vm_map_round_page(mappings[i].sfm_size), 1093 0, 1094 VM_FLAGS_FIXED | VM_FLAGS_ALREADY, 1095 object, 1096 0, 1097 TRUE, 1098 mappings[i].sfm_init_prot & VM_PROT_ALL, 1099 mappings[i].sfm_max_prot & VM_PROT_ALL, 1100 VM_INHERIT_DEFAULT); 1101 } 1102 } else { 1103 object = VM_OBJECT_NULL; /* no anonymous memory here */ 1104 kr = vm_map_enter_mem_object( 1105 sr_map, 1106 &target_address, 1107 vm_map_round_page(mappings[i].sfm_size), 1108 0, 1109 VM_FLAGS_FIXED | VM_FLAGS_ALREADY, 1110 map_port, 1111 mappings[i].sfm_file_offset, 1112 TRUE, 1113 mappings[i].sfm_init_prot & VM_PROT_ALL, 1114 mappings[i].sfm_max_prot & VM_PROT_ALL, 1115 VM_INHERIT_DEFAULT); 1116 } 1117 1118 if (kr != KERN_SUCCESS) { 1119 if (map_port == MACH_PORT_NULL) { 1120 /* 1121 * Get rid of the VM object we just created 1122 * but failed to map. 1123 */ 1124 vm_object_deallocate(object); 1125 object = VM_OBJECT_NULL; 1126 } 1127 if (kr == KERN_MEMORY_PRESENT) { 1128 /* 1129 * This exact mapping was already there: 1130 * that's fine. 1131 */ 1132 SHARED_REGION_TRACE_INFO( 1133 ("shared_region: mapping[%d]: " 1134 "address:0x%016llx size:0x%016llx " 1135 "offset:0x%016llx " 1136 "maxprot:0x%x prot:0x%x " 1137 "already mapped...\n", 1138 i, 1139 (long long)mappings[i].sfm_address, 1140 (long long)mappings[i].sfm_size, 1141 (long long)mappings[i].sfm_file_offset, 1142 mappings[i].sfm_max_prot, 1143 mappings[i].sfm_init_prot)); 1144 /* 1145 * We didn't establish this mapping ourselves; 1146 * let's reset its size, so that we do not 1147 * attempt to undo it if an error occurs later. 1148 */ 1149 mappings[i].sfm_size = 0; 1150 kr = KERN_SUCCESS; 1151 } else { 1152 /* this mapping failed ! */ 1153 SHARED_REGION_TRACE_ERROR( 1154 ("shared_region: mapping[%d]: " 1155 "address:0x%016llx size:0x%016llx " 1156 "offset:0x%016llx " 1157 "maxprot:0x%x prot:0x%x failed 0x%x\n", 1158 i, 1159 (long long)mappings[i].sfm_address, 1160 (long long)mappings[i].sfm_size, 1161 (long long)mappings[i].sfm_file_offset, 1162 mappings[i].sfm_max_prot, 1163 mappings[i].sfm_init_prot, 1164 kr)); 1165 1166 vm_shared_region_undo_mappings(sr_map, sr_base_address, mappings, i); 1167 break; 1168 } 1169 1170 } 1171 1172 /* 1173 * Record the first (chronologically) mapping in 1174 * this shared region. 1175 * We're protected by "sr_mapping_in_progress" here, 1176 * so no need to lock "shared_region". 1177 */ 1178 if (shared_region->sr_first_mapping == (mach_vm_offset_t) -1) { 1179 shared_region->sr_first_mapping = target_address; 1180 } 1181 } 1182 1183 vm_shared_region_lock(); 1184 assert(shared_region->sr_ref_count > 1); 1185 assert(shared_region->sr_mapping_in_progress); 1186 /* we're done working on that shared region */ 1187 shared_region->sr_mapping_in_progress = FALSE; 1188 thread_wakeup((event_t) &shared_region->sr_mapping_in_progress); 1189 vm_shared_region_unlock(); 1190 1191done: 1192 SHARED_REGION_TRACE_DEBUG( 1193 ("shared_region: map(%p,%d,%p,%p,0x%llx) <- 0x%x \n", 1194 shared_region, mappings_count, mappings, 1195 file_control, file_size, kr)); 1196 return kr; 1197} 1198 1199/* 1200 * Enter the appropriate shared region into "map" for "task". 1201 * This involves looking up the shared region (and possibly creating a new 1202 * one) for the desired environment, then mapping the VM sub map into the 1203 * task's VM "map", with the appropriate level of pmap-nesting. 1204 */ 1205kern_return_t 1206vm_shared_region_enter( 1207 struct _vm_map *map, 1208 struct task *task, 1209 void *fsroot, 1210 cpu_type_t cpu) 1211{ 1212 kern_return_t kr; 1213 vm_shared_region_t shared_region; 1214 vm_map_offset_t sr_address, sr_offset, target_address; 1215 vm_map_size_t sr_size, mapping_size; 1216 vm_map_offset_t sr_pmap_nesting_start; 1217 vm_map_size_t sr_pmap_nesting_size; 1218 ipc_port_t sr_handle; 1219 boolean_t is_64bit; 1220 1221 is_64bit = task_has_64BitAddr(task); 1222 1223 SHARED_REGION_TRACE_DEBUG( 1224 ("shared_region: -> " 1225 "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d)\n", 1226 map, task, fsroot, cpu, is_64bit)); 1227 1228 /* lookup (create if needed) the shared region for this environment */ 1229 shared_region = vm_shared_region_lookup(fsroot, cpu, is_64bit); 1230 if (shared_region == NULL) { 1231 /* this should not happen ! */ 1232 SHARED_REGION_TRACE_ERROR( 1233 ("shared_region: -> " 1234 "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d): " 1235 "lookup failed !\n", 1236 map, task, fsroot, cpu, is_64bit)); 1237 //panic("shared_region_enter: lookup failed\n"); 1238 return KERN_FAILURE; 1239 } 1240 1241 /* let the task use that shared region */ 1242 vm_shared_region_set(task, shared_region); 1243 1244 kr = KERN_SUCCESS; 1245 /* no need to lock since this data is never modified */ 1246 sr_address = shared_region->sr_base_address; 1247 sr_size = shared_region->sr_size; 1248 sr_handle = shared_region->sr_mem_entry; 1249 sr_pmap_nesting_start = shared_region->sr_pmap_nesting_start; 1250 sr_pmap_nesting_size = shared_region->sr_pmap_nesting_size; 1251 1252 /* 1253 * Start mapping the shared region's VM sub map into the task's VM map. 1254 */ 1255 sr_offset = 0; 1256 1257 if (sr_pmap_nesting_start > sr_address) { 1258 /* we need to map a range without pmap-nesting first */ 1259 target_address = sr_address; 1260 mapping_size = sr_pmap_nesting_start - sr_address; 1261 kr = vm_map_enter_mem_object( 1262 map, 1263 &target_address, 1264 mapping_size, 1265 0, 1266 VM_FLAGS_FIXED, 1267 sr_handle, 1268 sr_offset, 1269 TRUE, 1270 VM_PROT_READ, 1271 VM_PROT_ALL, 1272 VM_INHERIT_SHARE); 1273 if (kr != KERN_SUCCESS) { 1274 SHARED_REGION_TRACE_ERROR( 1275 ("shared_region: enter(%p,%p,%p,%d,%d): " 1276 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", 1277 map, task, fsroot, cpu, is_64bit, 1278 (long long)target_address, 1279 (long long)mapping_size, sr_handle, kr)); 1280 goto done; 1281 } 1282 SHARED_REGION_TRACE_DEBUG( 1283 ("shared_region: enter(%p,%p,%p,%d,%d): " 1284 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", 1285 map, task, fsroot, cpu, is_64bit, 1286 (long long)target_address, (long long)mapping_size, 1287 sr_handle, kr)); 1288 sr_offset += mapping_size; 1289 sr_size -= mapping_size; 1290 } 1291 /* 1292 * We may need to map several pmap-nested portions, due to platform 1293 * specific restrictions on pmap nesting. 1294 * The pmap-nesting is triggered by the "VM_MEMORY_SHARED_PMAP" alias... 1295 */ 1296 for (; 1297 sr_pmap_nesting_size > 0; 1298 sr_offset += mapping_size, 1299 sr_size -= mapping_size, 1300 sr_pmap_nesting_size -= mapping_size) { 1301 target_address = sr_address + sr_offset; 1302 mapping_size = sr_pmap_nesting_size; 1303 if (mapping_size > pmap_nesting_size_max) { 1304 mapping_size = (vm_map_offset_t) pmap_nesting_size_max; 1305 } 1306 kr = vm_map_enter_mem_object( 1307 map, 1308 &target_address, 1309 mapping_size, 1310 0, 1311 (VM_FLAGS_FIXED | VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP)), 1312 sr_handle, 1313 sr_offset, 1314 TRUE, 1315 VM_PROT_READ, 1316 VM_PROT_ALL, 1317 VM_INHERIT_SHARE); 1318 if (kr != KERN_SUCCESS) { 1319 SHARED_REGION_TRACE_ERROR( 1320 ("shared_region: enter(%p,%p,%p,%d,%d): " 1321 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", 1322 map, task, fsroot, cpu, is_64bit, 1323 (long long)target_address, 1324 (long long)mapping_size, sr_handle, kr)); 1325 goto done; 1326 } 1327 SHARED_REGION_TRACE_DEBUG( 1328 ("shared_region: enter(%p,%p,%p,%d,%d): " 1329 "nested vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", 1330 map, task, fsroot, cpu, is_64bit, 1331 (long long)target_address, (long long)mapping_size, 1332 sr_handle, kr)); 1333 } 1334 if (sr_size > 0) { 1335 /* and there's some left to be mapped without pmap-nesting */ 1336 target_address = sr_address + sr_offset; 1337 mapping_size = sr_size; 1338 kr = vm_map_enter_mem_object( 1339 map, 1340 &target_address, 1341 mapping_size, 1342 0, 1343 VM_FLAGS_FIXED, 1344 sr_handle, 1345 sr_offset, 1346 TRUE, 1347 VM_PROT_READ, 1348 VM_PROT_ALL, 1349 VM_INHERIT_SHARE); 1350 if (kr != KERN_SUCCESS) { 1351 SHARED_REGION_TRACE_ERROR( 1352 ("shared_region: enter(%p,%p,%p,%d,%d): " 1353 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", 1354 map, task, fsroot, cpu, is_64bit, 1355 (long long)target_address, 1356 (long long)mapping_size, sr_handle, kr)); 1357 goto done; 1358 } 1359 SHARED_REGION_TRACE_DEBUG( 1360 ("shared_region: enter(%p,%p,%p,%d,%d): " 1361 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", 1362 map, task, fsroot, cpu, is_64bit, 1363 (long long)target_address, (long long)mapping_size, 1364 sr_handle, kr)); 1365 sr_offset += mapping_size; 1366 sr_size -= mapping_size; 1367 } 1368 assert(sr_size == 0); 1369 1370done: 1371 SHARED_REGION_TRACE_DEBUG( 1372 ("shared_region: enter(%p,%p,%p,%d,%d) <- 0x%x\n", 1373 map, task, fsroot, cpu, is_64bit, kr)); 1374 return kr; 1375} 1376 1377#define SANE_SLIDE_INFO_SIZE (1024*1024) /*Can be changed if needed*/ 1378struct vm_shared_region_slide_info slide_info; 1379 1380kern_return_t 1381vm_shared_region_sliding_valid(uint32_t slide) { 1382 1383 kern_return_t kr = KERN_SUCCESS; 1384 1385 if ((shared_region_completed_slide == TRUE) && slide) { 1386 if (slide != slide_info.slide) { 1387 printf("Only one shared region can be slid\n"); 1388 kr = KERN_FAILURE; 1389 } else if (slide == slide_info.slide) { 1390 /* 1391 * Request for sliding when we've 1392 * already done it with exactly the 1393 * same slide value before. 1394 * This isn't wrong technically but 1395 * we don't want to slide again and 1396 * so we return this value. 1397 */ 1398 kr = KERN_INVALID_ARGUMENT; 1399 } 1400 } 1401 return kr; 1402} 1403 1404kern_return_t 1405vm_shared_region_slide_init( 1406 mach_vm_size_t slide_info_size, 1407 mach_vm_offset_t start, 1408 mach_vm_size_t size, 1409 uint32_t slide, 1410 memory_object_control_t sr_file_control) 1411{ 1412 kern_return_t kr = KERN_SUCCESS; 1413 vm_object_t object = VM_OBJECT_NULL; 1414 vm_object_offset_t offset = 0; 1415 1416 vm_map_t map =NULL, cur_map = NULL; 1417 boolean_t is_map_locked = FALSE; 1418 1419 if ((kr = vm_shared_region_sliding_valid(slide)) != KERN_SUCCESS) { 1420 if (kr == KERN_INVALID_ARGUMENT) { 1421 /* 1422 * This will happen if we request sliding again 1423 * with the same slide value that was used earlier 1424 * for the very first sliding. 1425 */ 1426 kr = KERN_SUCCESS; 1427 } 1428 return kr; 1429 } 1430 1431 if (slide_info_size > SANE_SLIDE_INFO_SIZE) { 1432 printf("Slide_info_size too large: %lx\n", (uintptr_t)slide_info_size); 1433 kr = KERN_FAILURE; 1434 return kr; 1435 } 1436 1437 if (sr_file_control != MEMORY_OBJECT_CONTROL_NULL) { 1438 1439 object = memory_object_control_to_vm_object(sr_file_control); 1440 vm_object_reference(object); 1441 offset = start; 1442 1443 vm_object_lock_shared(object); 1444 1445 } else { 1446 /* 1447 * Remove this entire "else" block and all "map" references 1448 * once we get rid of the shared_region_slide_np() 1449 * system call. 1450 */ 1451 vm_map_entry_t entry = VM_MAP_ENTRY_NULL; 1452 map = current_map(); 1453 vm_map_lock_read(map); 1454 is_map_locked = TRUE; 1455 Retry: 1456 cur_map = map; 1457 if(!vm_map_lookup_entry(map, start, &entry)) { 1458 kr = KERN_INVALID_ARGUMENT; 1459 } else { 1460 vm_object_t shadow_obj = VM_OBJECT_NULL; 1461 1462 if (entry->is_sub_map == TRUE) { 1463 map = entry->object.sub_map; 1464 start -= entry->vme_start; 1465 start += entry->offset; 1466 vm_map_lock_read(map); 1467 vm_map_unlock_read(cur_map); 1468 goto Retry; 1469 } else { 1470 object = entry->object.vm_object; 1471 offset = (start - entry->vme_start) + entry->offset; 1472 } 1473 1474 vm_object_lock_shared(object); 1475 while (object->shadow != VM_OBJECT_NULL) { 1476 shadow_obj = object->shadow; 1477 vm_object_lock_shared(shadow_obj); 1478 vm_object_unlock(object); 1479 object = shadow_obj; 1480 } 1481 } 1482 } 1483 1484 if (object->internal == TRUE) { 1485 kr = KERN_INVALID_ADDRESS; 1486 } else { 1487 kr = kmem_alloc(kernel_map, 1488 (vm_offset_t *) &slide_info.slide_info_entry, 1489 (vm_size_t) slide_info_size); 1490 if (kr == KERN_SUCCESS) { 1491 slide_info.slide_info_size = slide_info_size; 1492 slide_info.slide_object = object; 1493 slide_info.start = offset; 1494 slide_info.end = slide_info.start + size; 1495 slide_info.slide = slide; 1496 slide_info.sr = vm_shared_region_get(current_task()); 1497 /* 1498 * We want to keep the above reference on the shared region 1499 * because we have a pointer to it in the slide_info. 1500 * 1501 * If we want to have this region get deallocated/freed 1502 * then we will have to make sure that we msync(..MS_INVALIDATE..) 1503 * the pages associated with this shared region. Those pages would 1504 * have been slid with an older slide value. 1505 * 1506 * vm_shared_region_deallocate(slide_info.sr); 1507 */ 1508 shared_region_completed_slide = TRUE; 1509 } else { 1510 kr = KERN_FAILURE; 1511 } 1512 } 1513 vm_object_unlock(object); 1514 1515 if (is_map_locked == TRUE) { 1516 vm_map_unlock_read(map); 1517 } 1518 return kr; 1519} 1520 1521void* 1522vm_shared_region_get_slide_info(void) { 1523 return (void*)&slide_info; 1524} 1525 1526void* 1527vm_shared_region_get_slide_info_entry(void) { 1528 return (void*)slide_info.slide_info_entry; 1529} 1530 1531 1532kern_return_t 1533vm_shared_region_slide_sanity_check(void) 1534{ 1535 uint32_t pageIndex=0; 1536 uint16_t entryIndex=0; 1537 uint16_t *toc = NULL; 1538 vm_shared_region_slide_info_entry_t s_info; 1539 kern_return_t kr; 1540 1541 s_info = vm_shared_region_get_slide_info_entry(); 1542 toc = (uint16_t*)((uintptr_t)s_info + s_info->toc_offset); 1543 1544 kr = mach_vm_protect(kernel_map, 1545 (mach_vm_offset_t)(vm_offset_t) slide_info.slide_info_entry, 1546 (mach_vm_size_t) slide_info.slide_info_size, 1547 VM_PROT_READ, TRUE); 1548 if (kr != KERN_SUCCESS) { 1549 panic("vm_shared_region_slide_sanity_check: vm_protect() error 0x%x\n", kr); 1550 } 1551 1552 for (;pageIndex < s_info->toc_count; pageIndex++) { 1553 1554 entryIndex = (uint16_t)(toc[pageIndex]); 1555 1556 if (entryIndex >= s_info->entry_count) { 1557 printf("No sliding bitmap entry for pageIndex: %d at entryIndex: %d amongst %d entries\n", pageIndex, entryIndex, s_info->entry_count); 1558 goto fail; 1559 } 1560 1561 } 1562 return KERN_SUCCESS; 1563fail: 1564 if (slide_info.slide_info_entry != NULL) { 1565 kmem_free(kernel_map, 1566 (vm_offset_t) slide_info.slide_info_entry, 1567 (vm_size_t) slide_info.slide_info_size); 1568 vm_object_deallocate(slide_info.slide_object); 1569 slide_info.slide_object = NULL; 1570 slide_info.start = 0; 1571 slide_info.end = 0; 1572 slide_info.slide = 0; 1573 slide_info.slide_info_entry = NULL; 1574 slide_info.slide_info_size = 0; 1575 shared_region_completed_slide = FALSE; 1576 } 1577 return KERN_FAILURE; 1578} 1579 1580kern_return_t 1581vm_shared_region_slide(vm_offset_t vaddr, uint32_t pageIndex) 1582{ 1583 uint16_t *toc = NULL; 1584 slide_info_entry_toc_t bitmap = NULL; 1585 uint32_t i=0, j=0; 1586 uint8_t b = 0; 1587 uint32_t slide = slide_info.slide; 1588 int is_64 = task_has_64BitAddr(current_task()); 1589 1590 vm_shared_region_slide_info_entry_t s_info = vm_shared_region_get_slide_info_entry(); 1591 toc = (uint16_t*)((uintptr_t)s_info + s_info->toc_offset); 1592 1593 if (pageIndex >= s_info->toc_count) { 1594 printf("No slide entry for this page in toc. PageIndex: %d Toc Count: %d\n", pageIndex, s_info->toc_count); 1595 } else { 1596 uint16_t entryIndex = (uint16_t)(toc[pageIndex]); 1597 slide_info_entry_toc_t slide_info_entries = (slide_info_entry_toc_t)((uintptr_t)s_info + s_info->entry_offset); 1598 1599 if (entryIndex >= s_info->entry_count) { 1600 printf("No sliding bitmap entry for entryIndex: %d amongst %d entries\n", entryIndex, s_info->entry_count); 1601 } else { 1602 bitmap = &slide_info_entries[entryIndex]; 1603 1604 for(i=0; i < NUM_SLIDING_BITMAPS_PER_PAGE; ++i) { 1605 b = bitmap->entry[i]; 1606 if (b!=0) { 1607 for (j=0; j <8; ++j) { 1608 if (b & (1 <<j)){ 1609 uint32_t *ptr_to_slide; 1610 uint32_t old_value; 1611 1612 ptr_to_slide = (uint32_t*)((uintptr_t)(vaddr)+(sizeof(uint32_t)*(i*8 +j))); 1613 old_value = *ptr_to_slide; 1614 *ptr_to_slide += slide; 1615 if (is_64 && *ptr_to_slide < old_value) { 1616 /* 1617 * We just slid the low 32 bits of a 64-bit pointer 1618 * and it looks like there should have been a carry-over 1619 * to the upper 32 bits. 1620 * The sliding failed... 1621 */ 1622 printf("vm_shared_region_slide() carry over: i=%d j=%d b=0x%x slide=0x%x old=0x%x new=0x%x\n", 1623 i, j, b, slide, old_value, *ptr_to_slide); 1624 return KERN_FAILURE; 1625 } 1626 } 1627 } 1628 } 1629 } 1630 } 1631 } 1632 1633 return KERN_SUCCESS; 1634} 1635 1636/******************************************************************************/ 1637/* Comm page support */ 1638/******************************************************************************/ 1639 1640ipc_port_t commpage32_handle = IPC_PORT_NULL; 1641ipc_port_t commpage64_handle = IPC_PORT_NULL; 1642vm_named_entry_t commpage32_entry = NULL; 1643vm_named_entry_t commpage64_entry = NULL; 1644vm_map_t commpage32_map = VM_MAP_NULL; 1645vm_map_t commpage64_map = VM_MAP_NULL; 1646 1647ipc_port_t commpage_text32_handle = IPC_PORT_NULL; 1648ipc_port_t commpage_text64_handle = IPC_PORT_NULL; 1649vm_named_entry_t commpage_text32_entry = NULL; 1650vm_named_entry_t commpage_text64_entry = NULL; 1651vm_map_t commpage_text32_map = VM_MAP_NULL; 1652vm_map_t commpage_text64_map = VM_MAP_NULL; 1653 1654user32_addr_t commpage_text32_location = (user32_addr_t) _COMM_PAGE32_TEXT_START; 1655user64_addr_t commpage_text64_location = (user64_addr_t) _COMM_PAGE64_TEXT_START; 1656 1657#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) 1658/* 1659 * Create a memory entry, VM submap and pmap for one commpage. 1660 */ 1661static void 1662_vm_commpage_init( 1663 ipc_port_t *handlep, 1664 vm_map_size_t size) 1665{ 1666 kern_return_t kr; 1667 vm_named_entry_t mem_entry; 1668 vm_map_t new_map; 1669 1670 SHARED_REGION_TRACE_DEBUG( 1671 ("commpage: -> _init(0x%llx)\n", 1672 (long long)size)); 1673 1674 kr = mach_memory_entry_allocate(&mem_entry, 1675 handlep); 1676 if (kr != KERN_SUCCESS) { 1677 panic("_vm_commpage_init: could not allocate mem_entry"); 1678 } 1679 new_map = vm_map_create(pmap_create(NULL, 0, FALSE), 0, size, TRUE); 1680 if (new_map == VM_MAP_NULL) { 1681 panic("_vm_commpage_init: could not allocate VM map"); 1682 } 1683 mem_entry->backing.map = new_map; 1684 mem_entry->internal = TRUE; 1685 mem_entry->is_sub_map = TRUE; 1686 mem_entry->offset = 0; 1687 mem_entry->protection = VM_PROT_ALL; 1688 mem_entry->size = size; 1689 1690 SHARED_REGION_TRACE_DEBUG( 1691 ("commpage: _init(0x%llx) <- %p\n", 1692 (long long)size, *handlep)); 1693} 1694#endif 1695 1696 1697/* 1698 *Initialize the comm text pages at boot time 1699 */ 1700 extern u_int32_t random(void); 1701 void 1702vm_commpage_text_init(void) 1703{ 1704 SHARED_REGION_TRACE_DEBUG( 1705 ("commpage text: ->init()\n")); 1706#if defined(__i386__) || defined(__x86_64__) 1707 /* create the 32 bit comm text page */ 1708 unsigned int offset = (random() % _PFZ32_SLIDE_RANGE) << PAGE_SHIFT; /* restricting to 32bMAX-2PAGE */ 1709 _vm_commpage_init(&commpage_text32_handle, _COMM_PAGE_TEXT_AREA_LENGTH); 1710 commpage_text32_entry = (vm_named_entry_t) commpage_text32_handle->ip_kobject; 1711 commpage_text32_map = commpage_text32_entry->backing.map; 1712 commpage_text32_location = (user32_addr_t) (_COMM_PAGE32_TEXT_START + offset); 1713 /* XXX if (cpu_is_64bit_capable()) ? */ 1714 /* create the 64-bit comm page */ 1715 offset = (random() % _PFZ64_SLIDE_RANGE) << PAGE_SHIFT; /* restricting sliding upto 2Mb range */ 1716 _vm_commpage_init(&commpage_text64_handle, _COMM_PAGE_TEXT_AREA_LENGTH); 1717 commpage_text64_entry = (vm_named_entry_t) commpage_text64_handle->ip_kobject; 1718 commpage_text64_map = commpage_text64_entry->backing.map; 1719 commpage_text64_location = (user64_addr_t) (_COMM_PAGE64_TEXT_START + offset); 1720 1721 commpage_text_populate(); 1722#elif defined(__arm__) 1723 _vm_commpage_init(&commpage_text32_handle, _COMM_PAGE_TEXT_AREA_LENGTH); 1724 commpage_text32_entry = (vm_named_entry_t) commpage_text32_handle->ip_kobject; 1725 commpage_text32_map = commpage_text32_entry->backing.map; 1726 commpage_text32_location = (user32_addr_t) (_COMM_PAGE32_TEXT_START + 0x100); 1727#else 1728#error Unknown architecture. 1729#endif /* __i386__ || __x86_64__ */ 1730 /* populate the routines in here */ 1731 SHARED_REGION_TRACE_DEBUG( 1732 ("commpage text: init() <-\n")); 1733 1734} 1735 1736/* 1737 * Initialize the comm pages at boot time. 1738 */ 1739void 1740vm_commpage_init(void) 1741{ 1742 SHARED_REGION_TRACE_DEBUG( 1743 ("commpage: -> init()\n")); 1744 1745#if defined(__i386__) || defined(__x86_64__) 1746 /* create the 32-bit comm page */ 1747 _vm_commpage_init(&commpage32_handle, _COMM_PAGE32_AREA_LENGTH); 1748 commpage32_entry = (vm_named_entry_t) commpage32_handle->ip_kobject; 1749 commpage32_map = commpage32_entry->backing.map; 1750 1751 /* XXX if (cpu_is_64bit_capable()) ? */ 1752 /* create the 64-bit comm page */ 1753 _vm_commpage_init(&commpage64_handle, _COMM_PAGE64_AREA_LENGTH); 1754 commpage64_entry = (vm_named_entry_t) commpage64_handle->ip_kobject; 1755 commpage64_map = commpage64_entry->backing.map; 1756#endif /* __i386__ || __x86_64__ */ 1757 1758#if defined(__arm__) 1759 /* Create a 32-bit commpage. */ 1760 _vm_commpage_init(&commpage32_handle, _COMM_PAGE32_AREA_LENGTH); 1761 commpage32_entry = (vm_named_entry_t) commpage32_handle->ip_kobject; 1762 commpage32_map = commpage32_entry->backing.map; 1763#endif 1764 1765 /* populate them according to this specific platform */ 1766 commpage_populate(); 1767 __commpage_setup = 1; 1768#if defined(__i386__) || defined(__x86_64__) 1769 if (__system_power_source == 0) { 1770 post_sys_powersource_internal(0, 1); 1771 } 1772#endif /* __i386__ || __x86_64__ */ 1773 1774 SHARED_REGION_TRACE_DEBUG( 1775 ("commpage: init() <-\n")); 1776} 1777 1778/* 1779 * Enter the appropriate comm page into the task's address space. 1780 * This is called at exec() time via vm_map_exec(). 1781 */ 1782kern_return_t 1783vm_commpage_enter( 1784 vm_map_t map, 1785 task_t task) 1786{ 1787 ipc_port_t commpage_handle, commpage_text_handle; 1788 vm_map_offset_t commpage_address, objc_address, commpage_text_address; 1789 vm_map_size_t commpage_size, objc_size, commpage_text_size; 1790 int vm_flags; 1791 kern_return_t kr; 1792 1793 SHARED_REGION_TRACE_DEBUG( 1794 ("commpage: -> enter(%p,%p)\n", 1795 map, task)); 1796 1797 commpage_text_size = _COMM_PAGE_TEXT_AREA_LENGTH; 1798 /* the comm page is likely to be beyond the actual end of the VM map */ 1799 vm_flags = VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX; 1800 1801 /* select the appropriate comm page for this task */ 1802 assert(! (task_has_64BitAddr(task) ^ vm_map_is_64bit(map))); 1803 if (task_has_64BitAddr(task)) { 1804 commpage_handle = commpage64_handle; 1805 commpage_address = (vm_map_offset_t) _COMM_PAGE64_BASE_ADDRESS; 1806 commpage_size = _COMM_PAGE64_AREA_LENGTH; 1807 objc_size = _COMM_PAGE64_OBJC_SIZE; 1808 objc_address = _COMM_PAGE64_OBJC_BASE; 1809 commpage_text_handle = commpage_text64_handle; 1810 commpage_text_address = (vm_map_offset_t) commpage_text64_location; 1811 } else { 1812 commpage_handle = commpage32_handle; 1813 commpage_address = 1814 (vm_map_offset_t)(unsigned) _COMM_PAGE32_BASE_ADDRESS; 1815 commpage_size = _COMM_PAGE32_AREA_LENGTH; 1816 objc_size = _COMM_PAGE32_OBJC_SIZE; 1817 objc_address = _COMM_PAGE32_OBJC_BASE; 1818 commpage_text_handle = commpage_text32_handle; 1819 commpage_text_address = (vm_map_offset_t) commpage_text32_location; 1820 } 1821 1822 if ((commpage_address & (pmap_nesting_size_min - 1)) == 0 && 1823 (commpage_size & (pmap_nesting_size_min - 1)) == 0) { 1824 /* the commpage is properly aligned or sized for pmap-nesting */ 1825 vm_flags |= VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP); 1826 } 1827 /* map the comm page in the task's address space */ 1828 assert(commpage_handle != IPC_PORT_NULL); 1829 kr = vm_map_enter_mem_object( 1830 map, 1831 &commpage_address, 1832 commpage_size, 1833 0, 1834 vm_flags, 1835 commpage_handle, 1836 0, 1837 FALSE, 1838 VM_PROT_READ, 1839 VM_PROT_READ, 1840 VM_INHERIT_SHARE); 1841 if (kr != KERN_SUCCESS) { 1842 SHARED_REGION_TRACE_ERROR( 1843 ("commpage: enter(%p,0x%llx,0x%llx) " 1844 "commpage %p mapping failed 0x%x\n", 1845 map, (long long)commpage_address, 1846 (long long)commpage_size, commpage_handle, kr)); 1847 } 1848 1849 /* map the comm text page in the task's address space */ 1850 assert(commpage_text_handle != IPC_PORT_NULL); 1851 kr = vm_map_enter_mem_object( 1852 map, 1853 &commpage_text_address, 1854 commpage_text_size, 1855 0, 1856 vm_flags, 1857 commpage_text_handle, 1858 0, 1859 FALSE, 1860 VM_PROT_READ|VM_PROT_EXECUTE, 1861 VM_PROT_READ|VM_PROT_EXECUTE, 1862 VM_INHERIT_SHARE); 1863 if (kr != KERN_SUCCESS) { 1864 SHARED_REGION_TRACE_ERROR( 1865 ("commpage text: enter(%p,0x%llx,0x%llx) " 1866 "commpage text %p mapping failed 0x%x\n", 1867 map, (long long)commpage_text_address, 1868 (long long)commpage_text_size, commpage_text_handle, kr)); 1869 } 1870 1871 /* 1872 * Since we're here, we also pre-allocate some virtual space for the 1873 * Objective-C run-time, if needed... 1874 */ 1875 if (objc_size != 0) { 1876 kr = vm_map_enter_mem_object( 1877 map, 1878 &objc_address, 1879 objc_size, 1880 0, 1881 VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX, 1882 IPC_PORT_NULL, 1883 0, 1884 FALSE, 1885 VM_PROT_ALL, 1886 VM_PROT_ALL, 1887 VM_INHERIT_DEFAULT); 1888 if (kr != KERN_SUCCESS) { 1889 SHARED_REGION_TRACE_ERROR( 1890 ("commpage: enter(%p,0x%llx,0x%llx) " 1891 "objc mapping failed 0x%x\n", 1892 map, (long long)objc_address, 1893 (long long)objc_size, kr)); 1894 } 1895 } 1896 1897 SHARED_REGION_TRACE_DEBUG( 1898 ("commpage: enter(%p,%p) <- 0x%x\n", 1899 map, task, kr)); 1900 return kr; 1901} 1902 1903 1904/* 1905 * This is called from powermanagement code to let kernel know the current source of power. 1906 * 0 if it is external source (connected to power ) 1907 * 1 if it is internal power source ie battery 1908 */ 1909void 1910#if defined(__i386__) || defined(__x86_64__) 1911post_sys_powersource(int i) 1912#else 1913post_sys_powersource(__unused int i) 1914#endif 1915{ 1916#if defined(__i386__) || defined(__x86_64__) 1917 post_sys_powersource_internal(i, 0); 1918#endif /* __i386__ || __x86_64__ */ 1919} 1920 1921 1922#if defined(__i386__) || defined(__x86_64__) 1923static void 1924post_sys_powersource_internal(int i, int internal) 1925{ 1926 if (internal == 0) 1927 __system_power_source = i; 1928 1929 if (__commpage_setup != 0) { 1930 if (__system_power_source != 0) 1931 commpage_set_spin_count(0); 1932 else 1933 commpage_set_spin_count(MP_SPIN_TRIES); 1934 } 1935} 1936#endif /* __i386__ || __x86_64__ */ 1937