1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58/* 59 * File: vm/vm_user.c 60 * Author: Avadis Tevanian, Jr., Michael Wayne Young 61 * 62 * User-exported virtual memory functions. 63 */ 64 65/* 66 * There are three implementations of the "XXX_allocate" functionality in 67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate 68 * (for a task with the same address space size, especially the current task), 69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate 70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only 71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel 72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred 73 * for new code. 74 * 75 * The entrypoints into the kernel are more complex. All platforms support a 76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest 77 * size types for the platform. On platforms that only support U32/K32, 78 * subsystem 4800 is all you need. On platforms that support both U32 and U64, 79 * subsystem 3800 is used disambiguate the size of parameters, and they will 80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms, 81 * the MIG glue should never call into vm_allocate directly, because the calling 82 * task and kernel_task are unlikely to use the same size parameters 83 * 84 * New VM call implementations should be added here and to mach_vm.defs 85 * (subsystem 4800), and use mach_vm_* "wide" types. 86 */ 87 88#include <debug.h> 89 90#include <vm_cpm.h> 91#include <mach/boolean.h> 92#include <mach/kern_return.h> 93#include <mach/mach_types.h> /* to get vm_address_t */ 94#include <mach/memory_object.h> 95#include <mach/std_types.h> /* to get pointer_t */ 96#include <mach/upl.h> 97#include <mach/vm_attributes.h> 98#include <mach/vm_param.h> 99#include <mach/vm_statistics.h> 100#include <mach/mach_syscalls.h> 101 102#include <mach/host_priv_server.h> 103#include <mach/mach_vm_server.h> 104#include <mach/vm_map_server.h> 105 106#include <kern/host.h> 107#include <kern/kalloc.h> 108#include <kern/task.h> 109#include <kern/misc_protos.h> 110#include <vm/vm_fault.h> 111#include <vm/vm_map.h> 112#include <vm/vm_object.h> 113#include <vm/vm_page.h> 114#include <vm/memory_object.h> 115#include <vm/vm_pageout.h> 116#include <vm/vm_protos.h> 117#include <vm/vm_purgeable_internal.h> 118 119vm_size_t upl_offset_to_pagelist = 0; 120 121#if VM_CPM 122#include <vm/cpm.h> 123#endif /* VM_CPM */ 124 125ipc_port_t dynamic_pager_control_port=NULL; 126 127/* 128 * mach_vm_allocate allocates "zero fill" memory in the specfied 129 * map. 130 */ 131kern_return_t 132mach_vm_allocate( 133 vm_map_t map, 134 mach_vm_offset_t *addr, 135 mach_vm_size_t size, 136 int flags) 137{ 138 vm_map_offset_t map_addr; 139 vm_map_size_t map_size; 140 kern_return_t result; 141 boolean_t anywhere; 142 143 /* filter out any kernel-only flags */ 144 if (flags & ~VM_FLAGS_USER_ALLOCATE) 145 return KERN_INVALID_ARGUMENT; 146 147 if (map == VM_MAP_NULL) 148 return(KERN_INVALID_ARGUMENT); 149 if (size == 0) { 150 *addr = 0; 151 return(KERN_SUCCESS); 152 } 153 154 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0); 155 if (anywhere) { 156 /* 157 * No specific address requested, so start candidate address 158 * search at the minimum address in the map. However, if that 159 * minimum is 0, bump it up by PAGE_SIZE. We want to limit 160 * allocations of PAGEZERO to explicit requests since its 161 * normal use is to catch dereferences of NULL and many 162 * applications also treat pointers with a value of 0 as 163 * special and suddenly having address 0 contain useable 164 * memory would tend to confuse those applications. 165 */ 166 map_addr = vm_map_min(map); 167 if (map_addr == 0) 168 map_addr += VM_MAP_PAGE_SIZE(map); 169 } else 170 map_addr = vm_map_trunc_page(*addr, 171 VM_MAP_PAGE_MASK(map)); 172 map_size = vm_map_round_page(size, 173 VM_MAP_PAGE_MASK(map)); 174 if (map_size == 0) { 175 return(KERN_INVALID_ARGUMENT); 176 } 177 178 result = vm_map_enter( 179 map, 180 &map_addr, 181 map_size, 182 (vm_map_offset_t)0, 183 flags, 184 VM_OBJECT_NULL, 185 (vm_object_offset_t)0, 186 FALSE, 187 VM_PROT_DEFAULT, 188 VM_PROT_ALL, 189 VM_INHERIT_DEFAULT); 190 191 *addr = map_addr; 192 return(result); 193} 194 195/* 196 * vm_allocate 197 * Legacy routine that allocates "zero fill" memory in the specfied 198 * map (which is limited to the same size as the kernel). 199 */ 200kern_return_t 201vm_allocate( 202 vm_map_t map, 203 vm_offset_t *addr, 204 vm_size_t size, 205 int flags) 206{ 207 vm_map_offset_t map_addr; 208 vm_map_size_t map_size; 209 kern_return_t result; 210 boolean_t anywhere; 211 212 /* filter out any kernel-only flags */ 213 if (flags & ~VM_FLAGS_USER_ALLOCATE) 214 return KERN_INVALID_ARGUMENT; 215 216 if (map == VM_MAP_NULL) 217 return(KERN_INVALID_ARGUMENT); 218 if (size == 0) { 219 *addr = 0; 220 return(KERN_SUCCESS); 221 } 222 223 anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0); 224 if (anywhere) { 225 /* 226 * No specific address requested, so start candidate address 227 * search at the minimum address in the map. However, if that 228 * minimum is 0, bump it up by PAGE_SIZE. We want to limit 229 * allocations of PAGEZERO to explicit requests since its 230 * normal use is to catch dereferences of NULL and many 231 * applications also treat pointers with a value of 0 as 232 * special and suddenly having address 0 contain useable 233 * memory would tend to confuse those applications. 234 */ 235 map_addr = vm_map_min(map); 236 if (map_addr == 0) 237 map_addr += VM_MAP_PAGE_SIZE(map); 238 } else 239 map_addr = vm_map_trunc_page(*addr, 240 VM_MAP_PAGE_MASK(map)); 241 map_size = vm_map_round_page(size, 242 VM_MAP_PAGE_MASK(map)); 243 if (map_size == 0) { 244 return(KERN_INVALID_ARGUMENT); 245 } 246 247 result = vm_map_enter( 248 map, 249 &map_addr, 250 map_size, 251 (vm_map_offset_t)0, 252 flags, 253 VM_OBJECT_NULL, 254 (vm_object_offset_t)0, 255 FALSE, 256 VM_PROT_DEFAULT, 257 VM_PROT_ALL, 258 VM_INHERIT_DEFAULT); 259 260 *addr = CAST_DOWN(vm_offset_t, map_addr); 261 return(result); 262} 263 264/* 265 * mach_vm_deallocate - 266 * deallocates the specified range of addresses in the 267 * specified address map. 268 */ 269kern_return_t 270mach_vm_deallocate( 271 vm_map_t map, 272 mach_vm_offset_t start, 273 mach_vm_size_t size) 274{ 275 if ((map == VM_MAP_NULL) || (start + size < start)) 276 return(KERN_INVALID_ARGUMENT); 277 278 if (size == (mach_vm_offset_t) 0) 279 return(KERN_SUCCESS); 280 281 return(vm_map_remove(map, 282 vm_map_trunc_page(start, 283 VM_MAP_PAGE_MASK(map)), 284 vm_map_round_page(start+size, 285 VM_MAP_PAGE_MASK(map)), 286 VM_MAP_NO_FLAGS)); 287} 288 289/* 290 * vm_deallocate - 291 * deallocates the specified range of addresses in the 292 * specified address map (limited to addresses the same 293 * size as the kernel). 294 */ 295kern_return_t 296vm_deallocate( 297 register vm_map_t map, 298 vm_offset_t start, 299 vm_size_t size) 300{ 301 if ((map == VM_MAP_NULL) || (start + size < start)) 302 return(KERN_INVALID_ARGUMENT); 303 304 if (size == (vm_offset_t) 0) 305 return(KERN_SUCCESS); 306 307 return(vm_map_remove(map, 308 vm_map_trunc_page(start, 309 VM_MAP_PAGE_MASK(map)), 310 vm_map_round_page(start+size, 311 VM_MAP_PAGE_MASK(map)), 312 VM_MAP_NO_FLAGS)); 313} 314 315/* 316 * mach_vm_inherit - 317 * Sets the inheritance of the specified range in the 318 * specified map. 319 */ 320kern_return_t 321mach_vm_inherit( 322 vm_map_t map, 323 mach_vm_offset_t start, 324 mach_vm_size_t size, 325 vm_inherit_t new_inheritance) 326{ 327 if ((map == VM_MAP_NULL) || (start + size < start) || 328 (new_inheritance > VM_INHERIT_LAST_VALID)) 329 return(KERN_INVALID_ARGUMENT); 330 331 if (size == 0) 332 return KERN_SUCCESS; 333 334 return(vm_map_inherit(map, 335 vm_map_trunc_page(start, 336 VM_MAP_PAGE_MASK(map)), 337 vm_map_round_page(start+size, 338 VM_MAP_PAGE_MASK(map)), 339 new_inheritance)); 340} 341 342/* 343 * vm_inherit - 344 * Sets the inheritance of the specified range in the 345 * specified map (range limited to addresses 346 */ 347kern_return_t 348vm_inherit( 349 register vm_map_t map, 350 vm_offset_t start, 351 vm_size_t size, 352 vm_inherit_t new_inheritance) 353{ 354 if ((map == VM_MAP_NULL) || (start + size < start) || 355 (new_inheritance > VM_INHERIT_LAST_VALID)) 356 return(KERN_INVALID_ARGUMENT); 357 358 if (size == 0) 359 return KERN_SUCCESS; 360 361 return(vm_map_inherit(map, 362 vm_map_trunc_page(start, 363 VM_MAP_PAGE_MASK(map)), 364 vm_map_round_page(start+size, 365 VM_MAP_PAGE_MASK(map)), 366 new_inheritance)); 367} 368 369/* 370 * mach_vm_protect - 371 * Sets the protection of the specified range in the 372 * specified map. 373 */ 374 375kern_return_t 376mach_vm_protect( 377 vm_map_t map, 378 mach_vm_offset_t start, 379 mach_vm_size_t size, 380 boolean_t set_maximum, 381 vm_prot_t new_protection) 382{ 383 if ((map == VM_MAP_NULL) || (start + size < start) || 384 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) 385 return(KERN_INVALID_ARGUMENT); 386 387 if (size == 0) 388 return KERN_SUCCESS; 389 390 return(vm_map_protect(map, 391 vm_map_trunc_page(start, 392 VM_MAP_PAGE_MASK(map)), 393 vm_map_round_page(start+size, 394 VM_MAP_PAGE_MASK(map)), 395 new_protection, 396 set_maximum)); 397} 398 399/* 400 * vm_protect - 401 * Sets the protection of the specified range in the 402 * specified map. Addressability of the range limited 403 * to the same size as the kernel. 404 */ 405 406kern_return_t 407vm_protect( 408 vm_map_t map, 409 vm_offset_t start, 410 vm_size_t size, 411 boolean_t set_maximum, 412 vm_prot_t new_protection) 413{ 414 if ((map == VM_MAP_NULL) || (start + size < start) || 415 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) 416 return(KERN_INVALID_ARGUMENT); 417 418 if (size == 0) 419 return KERN_SUCCESS; 420 421 return(vm_map_protect(map, 422 vm_map_trunc_page(start, 423 VM_MAP_PAGE_MASK(map)), 424 vm_map_round_page(start+size, 425 VM_MAP_PAGE_MASK(map)), 426 new_protection, 427 set_maximum)); 428} 429 430/* 431 * mach_vm_machine_attributes - 432 * Handle machine-specific attributes for a mapping, such 433 * as cachability, migrability, etc. 434 */ 435kern_return_t 436mach_vm_machine_attribute( 437 vm_map_t map, 438 mach_vm_address_t addr, 439 mach_vm_size_t size, 440 vm_machine_attribute_t attribute, 441 vm_machine_attribute_val_t* value) /* IN/OUT */ 442{ 443 if ((map == VM_MAP_NULL) || (addr + size < addr)) 444 return(KERN_INVALID_ARGUMENT); 445 446 if (size == 0) 447 return KERN_SUCCESS; 448 449 return vm_map_machine_attribute( 450 map, 451 vm_map_trunc_page(addr, 452 VM_MAP_PAGE_MASK(map)), 453 vm_map_round_page(addr+size, 454 VM_MAP_PAGE_MASK(map)), 455 attribute, 456 value); 457} 458 459/* 460 * vm_machine_attribute - 461 * Handle machine-specific attributes for a mapping, such 462 * as cachability, migrability, etc. Limited addressability 463 * (same range limits as for the native kernel map). 464 */ 465kern_return_t 466vm_machine_attribute( 467 vm_map_t map, 468 vm_address_t addr, 469 vm_size_t size, 470 vm_machine_attribute_t attribute, 471 vm_machine_attribute_val_t* value) /* IN/OUT */ 472{ 473 if ((map == VM_MAP_NULL) || (addr + size < addr)) 474 return(KERN_INVALID_ARGUMENT); 475 476 if (size == 0) 477 return KERN_SUCCESS; 478 479 return vm_map_machine_attribute( 480 map, 481 vm_map_trunc_page(addr, 482 VM_MAP_PAGE_MASK(map)), 483 vm_map_round_page(addr+size, 484 VM_MAP_PAGE_MASK(map)), 485 attribute, 486 value); 487} 488 489/* 490 * mach_vm_read - 491 * Read/copy a range from one address space and return it to the caller. 492 * 493 * It is assumed that the address for the returned memory is selected by 494 * the IPC implementation as part of receiving the reply to this call. 495 * If IPC isn't used, the caller must deal with the vm_map_copy_t object 496 * that gets returned. 497 * 498 * JMM - because of mach_msg_type_number_t, this call is limited to a 499 * single 4GB region at this time. 500 * 501 */ 502kern_return_t 503mach_vm_read( 504 vm_map_t map, 505 mach_vm_address_t addr, 506 mach_vm_size_t size, 507 pointer_t *data, 508 mach_msg_type_number_t *data_size) 509{ 510 kern_return_t error; 511 vm_map_copy_t ipc_address; 512 513 if (map == VM_MAP_NULL) 514 return(KERN_INVALID_ARGUMENT); 515 516 if ((mach_msg_type_number_t) size != size) 517 return KERN_INVALID_ARGUMENT; 518 519 error = vm_map_copyin(map, 520 (vm_map_address_t)addr, 521 (vm_map_size_t)size, 522 FALSE, /* src_destroy */ 523 &ipc_address); 524 525 if (KERN_SUCCESS == error) { 526 *data = (pointer_t) ipc_address; 527 *data_size = (mach_msg_type_number_t) size; 528 assert(*data_size == size); 529 } 530 return(error); 531} 532 533/* 534 * vm_read - 535 * Read/copy a range from one address space and return it to the caller. 536 * Limited addressability (same range limits as for the native kernel map). 537 * 538 * It is assumed that the address for the returned memory is selected by 539 * the IPC implementation as part of receiving the reply to this call. 540 * If IPC isn't used, the caller must deal with the vm_map_copy_t object 541 * that gets returned. 542 */ 543kern_return_t 544vm_read( 545 vm_map_t map, 546 vm_address_t addr, 547 vm_size_t size, 548 pointer_t *data, 549 mach_msg_type_number_t *data_size) 550{ 551 kern_return_t error; 552 vm_map_copy_t ipc_address; 553 554 if (map == VM_MAP_NULL) 555 return(KERN_INVALID_ARGUMENT); 556 557 if (size > (unsigned)(mach_msg_type_number_t) -1) { 558 /* 559 * The kernel could handle a 64-bit "size" value, but 560 * it could not return the size of the data in "*data_size" 561 * without overflowing. 562 * Let's reject this "size" as invalid. 563 */ 564 return KERN_INVALID_ARGUMENT; 565 } 566 567 error = vm_map_copyin(map, 568 (vm_map_address_t)addr, 569 (vm_map_size_t)size, 570 FALSE, /* src_destroy */ 571 &ipc_address); 572 573 if (KERN_SUCCESS == error) { 574 *data = (pointer_t) ipc_address; 575 *data_size = (mach_msg_type_number_t) size; 576 assert(*data_size == size); 577 } 578 return(error); 579} 580 581/* 582 * mach_vm_read_list - 583 * Read/copy a list of address ranges from specified map. 584 * 585 * MIG does not know how to deal with a returned array of 586 * vm_map_copy_t structures, so we have to do the copyout 587 * manually here. 588 */ 589kern_return_t 590mach_vm_read_list( 591 vm_map_t map, 592 mach_vm_read_entry_t data_list, 593 natural_t count) 594{ 595 mach_msg_type_number_t i; 596 kern_return_t error; 597 vm_map_copy_t copy; 598 599 if (map == VM_MAP_NULL || 600 count > VM_MAP_ENTRY_MAX) 601 return(KERN_INVALID_ARGUMENT); 602 603 error = KERN_SUCCESS; 604 for(i=0; i<count; i++) { 605 vm_map_address_t map_addr; 606 vm_map_size_t map_size; 607 608 map_addr = (vm_map_address_t)(data_list[i].address); 609 map_size = (vm_map_size_t)(data_list[i].size); 610 611 if(map_size != 0) { 612 error = vm_map_copyin(map, 613 map_addr, 614 map_size, 615 FALSE, /* src_destroy */ 616 ©); 617 if (KERN_SUCCESS == error) { 618 error = vm_map_copyout( 619 current_task()->map, 620 &map_addr, 621 copy); 622 if (KERN_SUCCESS == error) { 623 data_list[i].address = map_addr; 624 continue; 625 } 626 vm_map_copy_discard(copy); 627 } 628 } 629 data_list[i].address = (mach_vm_address_t)0; 630 data_list[i].size = (mach_vm_size_t)0; 631 } 632 return(error); 633} 634 635/* 636 * vm_read_list - 637 * Read/copy a list of address ranges from specified map. 638 * 639 * MIG does not know how to deal with a returned array of 640 * vm_map_copy_t structures, so we have to do the copyout 641 * manually here. 642 * 643 * The source and destination ranges are limited to those 644 * that can be described with a vm_address_t (i.e. same 645 * size map as the kernel). 646 * 647 * JMM - If the result of the copyout is an address range 648 * that cannot be described with a vm_address_t (i.e. the 649 * caller had a larger address space but used this call 650 * anyway), it will result in a truncated address being 651 * returned (and a likely confused caller). 652 */ 653 654kern_return_t 655vm_read_list( 656 vm_map_t map, 657 vm_read_entry_t data_list, 658 natural_t count) 659{ 660 mach_msg_type_number_t i; 661 kern_return_t error; 662 vm_map_copy_t copy; 663 664 if (map == VM_MAP_NULL || 665 count > VM_MAP_ENTRY_MAX) 666 return(KERN_INVALID_ARGUMENT); 667 668 error = KERN_SUCCESS; 669 for(i=0; i<count; i++) { 670 vm_map_address_t map_addr; 671 vm_map_size_t map_size; 672 673 map_addr = (vm_map_address_t)(data_list[i].address); 674 map_size = (vm_map_size_t)(data_list[i].size); 675 676 if(map_size != 0) { 677 error = vm_map_copyin(map, 678 map_addr, 679 map_size, 680 FALSE, /* src_destroy */ 681 ©); 682 if (KERN_SUCCESS == error) { 683 error = vm_map_copyout(current_task()->map, 684 &map_addr, 685 copy); 686 if (KERN_SUCCESS == error) { 687 data_list[i].address = 688 CAST_DOWN(vm_offset_t, map_addr); 689 continue; 690 } 691 vm_map_copy_discard(copy); 692 } 693 } 694 data_list[i].address = (mach_vm_address_t)0; 695 data_list[i].size = (mach_vm_size_t)0; 696 } 697 return(error); 698} 699 700/* 701 * mach_vm_read_overwrite - 702 * Overwrite a range of the current map with data from the specified 703 * map/address range. 704 * 705 * In making an assumption that the current thread is local, it is 706 * no longer cluster-safe without a fully supportive local proxy 707 * thread/task (but we don't support cluster's anymore so this is moot). 708 */ 709 710kern_return_t 711mach_vm_read_overwrite( 712 vm_map_t map, 713 mach_vm_address_t address, 714 mach_vm_size_t size, 715 mach_vm_address_t data, 716 mach_vm_size_t *data_size) 717{ 718 kern_return_t error; 719 vm_map_copy_t copy; 720 721 if (map == VM_MAP_NULL) 722 return(KERN_INVALID_ARGUMENT); 723 724 error = vm_map_copyin(map, (vm_map_address_t)address, 725 (vm_map_size_t)size, FALSE, ©); 726 727 if (KERN_SUCCESS == error) { 728 error = vm_map_copy_overwrite(current_thread()->map, 729 (vm_map_address_t)data, 730 copy, FALSE); 731 if (KERN_SUCCESS == error) { 732 *data_size = size; 733 return error; 734 } 735 vm_map_copy_discard(copy); 736 } 737 return(error); 738} 739 740/* 741 * vm_read_overwrite - 742 * Overwrite a range of the current map with data from the specified 743 * map/address range. 744 * 745 * This routine adds the additional limitation that the source and 746 * destination ranges must be describable with vm_address_t values 747 * (i.e. the same size address spaces as the kernel, or at least the 748 * the ranges are in that first portion of the respective address 749 * spaces). 750 */ 751 752kern_return_t 753vm_read_overwrite( 754 vm_map_t map, 755 vm_address_t address, 756 vm_size_t size, 757 vm_address_t data, 758 vm_size_t *data_size) 759{ 760 kern_return_t error; 761 vm_map_copy_t copy; 762 763 if (map == VM_MAP_NULL) 764 return(KERN_INVALID_ARGUMENT); 765 766 error = vm_map_copyin(map, (vm_map_address_t)address, 767 (vm_map_size_t)size, FALSE, ©); 768 769 if (KERN_SUCCESS == error) { 770 error = vm_map_copy_overwrite(current_thread()->map, 771 (vm_map_address_t)data, 772 copy, FALSE); 773 if (KERN_SUCCESS == error) { 774 *data_size = size; 775 return error; 776 } 777 vm_map_copy_discard(copy); 778 } 779 return(error); 780} 781 782 783/* 784 * mach_vm_write - 785 * Overwrite the specified address range with the data provided 786 * (from the current map). 787 */ 788kern_return_t 789mach_vm_write( 790 vm_map_t map, 791 mach_vm_address_t address, 792 pointer_t data, 793 __unused mach_msg_type_number_t size) 794{ 795 if (map == VM_MAP_NULL) 796 return KERN_INVALID_ARGUMENT; 797 798 return vm_map_copy_overwrite(map, (vm_map_address_t)address, 799 (vm_map_copy_t) data, FALSE /* interruptible XXX */); 800} 801 802/* 803 * vm_write - 804 * Overwrite the specified address range with the data provided 805 * (from the current map). 806 * 807 * The addressability of the range of addresses to overwrite is 808 * limited bu the use of a vm_address_t (same size as kernel map). 809 * Either the target map is also small, or the range is in the 810 * low addresses within it. 811 */ 812kern_return_t 813vm_write( 814 vm_map_t map, 815 vm_address_t address, 816 pointer_t data, 817 __unused mach_msg_type_number_t size) 818{ 819 if (map == VM_MAP_NULL) 820 return KERN_INVALID_ARGUMENT; 821 822 return vm_map_copy_overwrite(map, (vm_map_address_t)address, 823 (vm_map_copy_t) data, FALSE /* interruptible XXX */); 824} 825 826/* 827 * mach_vm_copy - 828 * Overwrite one range of the specified map with the contents of 829 * another range within that same map (i.e. both address ranges 830 * are "over there"). 831 */ 832kern_return_t 833mach_vm_copy( 834 vm_map_t map, 835 mach_vm_address_t source_address, 836 mach_vm_size_t size, 837 mach_vm_address_t dest_address) 838{ 839 vm_map_copy_t copy; 840 kern_return_t kr; 841 842 if (map == VM_MAP_NULL) 843 return KERN_INVALID_ARGUMENT; 844 845 kr = vm_map_copyin(map, (vm_map_address_t)source_address, 846 (vm_map_size_t)size, FALSE, ©); 847 848 if (KERN_SUCCESS == kr) { 849 kr = vm_map_copy_overwrite(map, 850 (vm_map_address_t)dest_address, 851 copy, FALSE /* interruptible XXX */); 852 853 if (KERN_SUCCESS != kr) 854 vm_map_copy_discard(copy); 855 } 856 return kr; 857} 858 859kern_return_t 860vm_copy( 861 vm_map_t map, 862 vm_address_t source_address, 863 vm_size_t size, 864 vm_address_t dest_address) 865{ 866 vm_map_copy_t copy; 867 kern_return_t kr; 868 869 if (map == VM_MAP_NULL) 870 return KERN_INVALID_ARGUMENT; 871 872 kr = vm_map_copyin(map, (vm_map_address_t)source_address, 873 (vm_map_size_t)size, FALSE, ©); 874 875 if (KERN_SUCCESS == kr) { 876 kr = vm_map_copy_overwrite(map, 877 (vm_map_address_t)dest_address, 878 copy, FALSE /* interruptible XXX */); 879 880 if (KERN_SUCCESS != kr) 881 vm_map_copy_discard(copy); 882 } 883 return kr; 884} 885 886/* 887 * mach_vm_map - 888 * Map some range of an object into an address space. 889 * 890 * The object can be one of several types of objects: 891 * NULL - anonymous memory 892 * a named entry - a range within another address space 893 * or a range within a memory object 894 * a whole memory object 895 * 896 */ 897kern_return_t 898mach_vm_map( 899 vm_map_t target_map, 900 mach_vm_offset_t *address, 901 mach_vm_size_t initial_size, 902 mach_vm_offset_t mask, 903 int flags, 904 ipc_port_t port, 905 vm_object_offset_t offset, 906 boolean_t copy, 907 vm_prot_t cur_protection, 908 vm_prot_t max_protection, 909 vm_inherit_t inheritance) 910{ 911 kern_return_t kr; 912 vm_map_offset_t vmmaddr; 913 914 vmmaddr = (vm_map_offset_t) *address; 915 916 /* filter out any kernel-only flags */ 917 if (flags & ~VM_FLAGS_USER_MAP) 918 return KERN_INVALID_ARGUMENT; 919 920 kr = vm_map_enter_mem_object(target_map, 921 &vmmaddr, 922 initial_size, 923 mask, 924 flags, 925 port, 926 offset, 927 copy, 928 cur_protection, 929 max_protection, 930 inheritance); 931 932 *address = vmmaddr; 933 return kr; 934} 935 936 937/* legacy interface */ 938kern_return_t 939vm_map_64( 940 vm_map_t target_map, 941 vm_offset_t *address, 942 vm_size_t size, 943 vm_offset_t mask, 944 int flags, 945 ipc_port_t port, 946 vm_object_offset_t offset, 947 boolean_t copy, 948 vm_prot_t cur_protection, 949 vm_prot_t max_protection, 950 vm_inherit_t inheritance) 951{ 952 mach_vm_address_t map_addr; 953 mach_vm_size_t map_size; 954 mach_vm_offset_t map_mask; 955 kern_return_t kr; 956 957 map_addr = (mach_vm_address_t)*address; 958 map_size = (mach_vm_size_t)size; 959 map_mask = (mach_vm_offset_t)mask; 960 961 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags, 962 port, offset, copy, 963 cur_protection, max_protection, inheritance); 964 *address = CAST_DOWN(vm_offset_t, map_addr); 965 return kr; 966} 967 968/* temporary, until world build */ 969kern_return_t 970vm_map( 971 vm_map_t target_map, 972 vm_offset_t *address, 973 vm_size_t size, 974 vm_offset_t mask, 975 int flags, 976 ipc_port_t port, 977 vm_offset_t offset, 978 boolean_t copy, 979 vm_prot_t cur_protection, 980 vm_prot_t max_protection, 981 vm_inherit_t inheritance) 982{ 983 mach_vm_address_t map_addr; 984 mach_vm_size_t map_size; 985 mach_vm_offset_t map_mask; 986 vm_object_offset_t obj_offset; 987 kern_return_t kr; 988 989 map_addr = (mach_vm_address_t)*address; 990 map_size = (mach_vm_size_t)size; 991 map_mask = (mach_vm_offset_t)mask; 992 obj_offset = (vm_object_offset_t)offset; 993 994 kr = mach_vm_map(target_map, &map_addr, map_size, map_mask, flags, 995 port, obj_offset, copy, 996 cur_protection, max_protection, inheritance); 997 *address = CAST_DOWN(vm_offset_t, map_addr); 998 return kr; 999} 1000 1001/* 1002 * mach_vm_remap - 1003 * Remap a range of memory from one task into another, 1004 * to another address range within the same task, or 1005 * over top of itself (with altered permissions and/or 1006 * as an in-place copy of itself). 1007 */ 1008 1009kern_return_t 1010mach_vm_remap( 1011 vm_map_t target_map, 1012 mach_vm_offset_t *address, 1013 mach_vm_size_t size, 1014 mach_vm_offset_t mask, 1015 int flags, 1016 vm_map_t src_map, 1017 mach_vm_offset_t memory_address, 1018 boolean_t copy, 1019 vm_prot_t *cur_protection, 1020 vm_prot_t *max_protection, 1021 vm_inherit_t inheritance) 1022{ 1023 vm_map_offset_t map_addr; 1024 kern_return_t kr; 1025 1026 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) 1027 return KERN_INVALID_ARGUMENT; 1028 1029 /* filter out any kernel-only flags */ 1030 if (flags & ~VM_FLAGS_USER_REMAP) 1031 return KERN_INVALID_ARGUMENT; 1032 1033 map_addr = (vm_map_offset_t)*address; 1034 1035 kr = vm_map_remap(target_map, 1036 &map_addr, 1037 size, 1038 mask, 1039 flags, 1040 src_map, 1041 memory_address, 1042 copy, 1043 cur_protection, 1044 max_protection, 1045 inheritance); 1046 *address = map_addr; 1047 return kr; 1048} 1049 1050/* 1051 * vm_remap - 1052 * Remap a range of memory from one task into another, 1053 * to another address range within the same task, or 1054 * over top of itself (with altered permissions and/or 1055 * as an in-place copy of itself). 1056 * 1057 * The addressability of the source and target address 1058 * range is limited by the size of vm_address_t (in the 1059 * kernel context). 1060 */ 1061kern_return_t 1062vm_remap( 1063 vm_map_t target_map, 1064 vm_offset_t *address, 1065 vm_size_t size, 1066 vm_offset_t mask, 1067 int flags, 1068 vm_map_t src_map, 1069 vm_offset_t memory_address, 1070 boolean_t copy, 1071 vm_prot_t *cur_protection, 1072 vm_prot_t *max_protection, 1073 vm_inherit_t inheritance) 1074{ 1075 vm_map_offset_t map_addr; 1076 kern_return_t kr; 1077 1078 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) 1079 return KERN_INVALID_ARGUMENT; 1080 1081 /* filter out any kernel-only flags */ 1082 if (flags & ~VM_FLAGS_USER_REMAP) 1083 return KERN_INVALID_ARGUMENT; 1084 1085 map_addr = (vm_map_offset_t)*address; 1086 1087 kr = vm_map_remap(target_map, 1088 &map_addr, 1089 size, 1090 mask, 1091 flags, 1092 src_map, 1093 memory_address, 1094 copy, 1095 cur_protection, 1096 max_protection, 1097 inheritance); 1098 *address = CAST_DOWN(vm_offset_t, map_addr); 1099 return kr; 1100} 1101 1102/* 1103 * NOTE: these routine (and this file) will no longer require mach_host_server.h 1104 * when mach_vm_wire and vm_wire are changed to use ledgers. 1105 */ 1106#include <mach/mach_host_server.h> 1107/* 1108 * mach_vm_wire 1109 * Specify that the range of the virtual address space 1110 * of the target task must not cause page faults for 1111 * the indicated accesses. 1112 * 1113 * [ To unwire the pages, specify VM_PROT_NONE. ] 1114 */ 1115kern_return_t 1116mach_vm_wire( 1117 host_priv_t host_priv, 1118 vm_map_t map, 1119 mach_vm_offset_t start, 1120 mach_vm_size_t size, 1121 vm_prot_t access) 1122{ 1123 kern_return_t rc; 1124 1125 if (host_priv == HOST_PRIV_NULL) 1126 return KERN_INVALID_HOST; 1127 1128 assert(host_priv == &realhost); 1129 1130 if (map == VM_MAP_NULL) 1131 return KERN_INVALID_TASK; 1132 1133 if (access & ~VM_PROT_ALL || (start + size < start)) 1134 return KERN_INVALID_ARGUMENT; 1135 1136 if (access != VM_PROT_NONE) { 1137 rc = vm_map_wire(map, 1138 vm_map_trunc_page(start, 1139 VM_MAP_PAGE_MASK(map)), 1140 vm_map_round_page(start+size, 1141 VM_MAP_PAGE_MASK(map)), 1142 access, 1143 TRUE); 1144 } else { 1145 rc = vm_map_unwire(map, 1146 vm_map_trunc_page(start, 1147 VM_MAP_PAGE_MASK(map)), 1148 vm_map_round_page(start+size, 1149 VM_MAP_PAGE_MASK(map)), 1150 TRUE); 1151 } 1152 return rc; 1153} 1154 1155/* 1156 * vm_wire - 1157 * Specify that the range of the virtual address space 1158 * of the target task must not cause page faults for 1159 * the indicated accesses. 1160 * 1161 * [ To unwire the pages, specify VM_PROT_NONE. ] 1162 */ 1163kern_return_t 1164vm_wire( 1165 host_priv_t host_priv, 1166 register vm_map_t map, 1167 vm_offset_t start, 1168 vm_size_t size, 1169 vm_prot_t access) 1170{ 1171 kern_return_t rc; 1172 1173 if (host_priv == HOST_PRIV_NULL) 1174 return KERN_INVALID_HOST; 1175 1176 assert(host_priv == &realhost); 1177 1178 if (map == VM_MAP_NULL) 1179 return KERN_INVALID_TASK; 1180 1181 if ((access & ~VM_PROT_ALL) || (start + size < start)) 1182 return KERN_INVALID_ARGUMENT; 1183 1184 if (size == 0) { 1185 rc = KERN_SUCCESS; 1186 } else if (access != VM_PROT_NONE) { 1187 rc = vm_map_wire(map, 1188 vm_map_trunc_page(start, 1189 VM_MAP_PAGE_MASK(map)), 1190 vm_map_round_page(start+size, 1191 VM_MAP_PAGE_MASK(map)), 1192 access, 1193 TRUE); 1194 } else { 1195 rc = vm_map_unwire(map, 1196 vm_map_trunc_page(start, 1197 VM_MAP_PAGE_MASK(map)), 1198 vm_map_round_page(start+size, 1199 VM_MAP_PAGE_MASK(map)), 1200 TRUE); 1201 } 1202 return rc; 1203} 1204 1205/* 1206 * vm_msync 1207 * 1208 * Synchronises the memory range specified with its backing store 1209 * image by either flushing or cleaning the contents to the appropriate 1210 * memory manager. 1211 * 1212 * interpretation of sync_flags 1213 * VM_SYNC_INVALIDATE - discard pages, only return precious 1214 * pages to manager. 1215 * 1216 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS) 1217 * - discard pages, write dirty or precious 1218 * pages back to memory manager. 1219 * 1220 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS 1221 * - write dirty or precious pages back to 1222 * the memory manager. 1223 * 1224 * VM_SYNC_CONTIGUOUS - does everything normally, but if there 1225 * is a hole in the region, and we would 1226 * have returned KERN_SUCCESS, return 1227 * KERN_INVALID_ADDRESS instead. 1228 * 1229 * RETURNS 1230 * KERN_INVALID_TASK Bad task parameter 1231 * KERN_INVALID_ARGUMENT both sync and async were specified. 1232 * KERN_SUCCESS The usual. 1233 * KERN_INVALID_ADDRESS There was a hole in the region. 1234 */ 1235 1236kern_return_t 1237mach_vm_msync( 1238 vm_map_t map, 1239 mach_vm_address_t address, 1240 mach_vm_size_t size, 1241 vm_sync_t sync_flags) 1242{ 1243 1244 if (map == VM_MAP_NULL) 1245 return(KERN_INVALID_TASK); 1246 1247 return vm_map_msync(map, (vm_map_address_t)address, 1248 (vm_map_size_t)size, sync_flags); 1249} 1250 1251/* 1252 * vm_msync 1253 * 1254 * Synchronises the memory range specified with its backing store 1255 * image by either flushing or cleaning the contents to the appropriate 1256 * memory manager. 1257 * 1258 * interpretation of sync_flags 1259 * VM_SYNC_INVALIDATE - discard pages, only return precious 1260 * pages to manager. 1261 * 1262 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS) 1263 * - discard pages, write dirty or precious 1264 * pages back to memory manager. 1265 * 1266 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS 1267 * - write dirty or precious pages back to 1268 * the memory manager. 1269 * 1270 * VM_SYNC_CONTIGUOUS - does everything normally, but if there 1271 * is a hole in the region, and we would 1272 * have returned KERN_SUCCESS, return 1273 * KERN_INVALID_ADDRESS instead. 1274 * 1275 * The addressability of the range is limited to that which can 1276 * be described by a vm_address_t. 1277 * 1278 * RETURNS 1279 * KERN_INVALID_TASK Bad task parameter 1280 * KERN_INVALID_ARGUMENT both sync and async were specified. 1281 * KERN_SUCCESS The usual. 1282 * KERN_INVALID_ADDRESS There was a hole in the region. 1283 */ 1284 1285kern_return_t 1286vm_msync( 1287 vm_map_t map, 1288 vm_address_t address, 1289 vm_size_t size, 1290 vm_sync_t sync_flags) 1291{ 1292 1293 if (map == VM_MAP_NULL) 1294 return(KERN_INVALID_TASK); 1295 1296 return vm_map_msync(map, (vm_map_address_t)address, 1297 (vm_map_size_t)size, sync_flags); 1298} 1299 1300 1301int 1302vm_toggle_entry_reuse(int toggle, int *old_value) 1303{ 1304 vm_map_t map = current_map(); 1305 1306 if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){ 1307 *old_value = map->disable_vmentry_reuse; 1308 } else if(toggle == VM_TOGGLE_SET){ 1309 vm_map_lock(map); 1310 map->disable_vmentry_reuse = TRUE; 1311 if (map->first_free == vm_map_to_entry(map)) { 1312 map->highest_entry_end = vm_map_min(map); 1313 } else { 1314 map->highest_entry_end = map->first_free->vme_end; 1315 } 1316 vm_map_unlock(map); 1317 } else if (toggle == VM_TOGGLE_CLEAR){ 1318 vm_map_lock(map); 1319 map->disable_vmentry_reuse = FALSE; 1320 vm_map_unlock(map); 1321 } else 1322 return KERN_INVALID_ARGUMENT; 1323 1324 return KERN_SUCCESS; 1325} 1326 1327/* 1328 * mach_vm_behavior_set 1329 * 1330 * Sets the paging behavior attribute for the specified range 1331 * in the specified map. 1332 * 1333 * This routine will fail with KERN_INVALID_ADDRESS if any address 1334 * in [start,start+size) is not a valid allocated memory region. 1335 */ 1336kern_return_t 1337mach_vm_behavior_set( 1338 vm_map_t map, 1339 mach_vm_offset_t start, 1340 mach_vm_size_t size, 1341 vm_behavior_t new_behavior) 1342{ 1343 if ((map == VM_MAP_NULL) || (start + size < start)) 1344 return(KERN_INVALID_ARGUMENT); 1345 1346 if (size == 0) 1347 return KERN_SUCCESS; 1348 1349 return(vm_map_behavior_set(map, 1350 vm_map_trunc_page(start, 1351 VM_MAP_PAGE_MASK(map)), 1352 vm_map_round_page(start+size, 1353 VM_MAP_PAGE_MASK(map)), 1354 new_behavior)); 1355} 1356 1357/* 1358 * vm_behavior_set 1359 * 1360 * Sets the paging behavior attribute for the specified range 1361 * in the specified map. 1362 * 1363 * This routine will fail with KERN_INVALID_ADDRESS if any address 1364 * in [start,start+size) is not a valid allocated memory region. 1365 * 1366 * This routine is potentially limited in addressibility by the 1367 * use of vm_offset_t (if the map provided is larger than the 1368 * kernel's). 1369 */ 1370kern_return_t 1371vm_behavior_set( 1372 vm_map_t map, 1373 vm_offset_t start, 1374 vm_size_t size, 1375 vm_behavior_t new_behavior) 1376{ 1377 if ((map == VM_MAP_NULL) || (start + size < start)) 1378 return(KERN_INVALID_ARGUMENT); 1379 1380 if (size == 0) 1381 return KERN_SUCCESS; 1382 1383 return(vm_map_behavior_set(map, 1384 vm_map_trunc_page(start, 1385 VM_MAP_PAGE_MASK(map)), 1386 vm_map_round_page(start+size, 1387 VM_MAP_PAGE_MASK(map)), 1388 new_behavior)); 1389} 1390 1391/* 1392 * mach_vm_region: 1393 * 1394 * User call to obtain information about a region in 1395 * a task's address map. Currently, only one flavor is 1396 * supported. 1397 * 1398 * XXX The reserved and behavior fields cannot be filled 1399 * in until the vm merge from the IK is completed, and 1400 * vm_reserve is implemented. 1401 * 1402 * XXX Dependency: syscall_vm_region() also supports only one flavor. 1403 */ 1404 1405kern_return_t 1406mach_vm_region( 1407 vm_map_t map, 1408 mach_vm_offset_t *address, /* IN/OUT */ 1409 mach_vm_size_t *size, /* OUT */ 1410 vm_region_flavor_t flavor, /* IN */ 1411 vm_region_info_t info, /* OUT */ 1412 mach_msg_type_number_t *count, /* IN/OUT */ 1413 mach_port_t *object_name) /* OUT */ 1414{ 1415 vm_map_offset_t map_addr; 1416 vm_map_size_t map_size; 1417 kern_return_t kr; 1418 1419 if (VM_MAP_NULL == map) 1420 return KERN_INVALID_ARGUMENT; 1421 1422 map_addr = (vm_map_offset_t)*address; 1423 map_size = (vm_map_size_t)*size; 1424 1425 /* legacy conversion */ 1426 if (VM_REGION_BASIC_INFO == flavor) 1427 flavor = VM_REGION_BASIC_INFO_64; 1428 1429 kr = vm_map_region(map, 1430 &map_addr, &map_size, 1431 flavor, info, count, 1432 object_name); 1433 1434 *address = map_addr; 1435 *size = map_size; 1436 return kr; 1437} 1438 1439/* 1440 * vm_region_64 and vm_region: 1441 * 1442 * User call to obtain information about a region in 1443 * a task's address map. Currently, only one flavor is 1444 * supported. 1445 * 1446 * XXX The reserved and behavior fields cannot be filled 1447 * in until the vm merge from the IK is completed, and 1448 * vm_reserve is implemented. 1449 * 1450 * XXX Dependency: syscall_vm_region() also supports only one flavor. 1451 */ 1452 1453kern_return_t 1454vm_region_64( 1455 vm_map_t map, 1456 vm_offset_t *address, /* IN/OUT */ 1457 vm_size_t *size, /* OUT */ 1458 vm_region_flavor_t flavor, /* IN */ 1459 vm_region_info_t info, /* OUT */ 1460 mach_msg_type_number_t *count, /* IN/OUT */ 1461 mach_port_t *object_name) /* OUT */ 1462{ 1463 vm_map_offset_t map_addr; 1464 vm_map_size_t map_size; 1465 kern_return_t kr; 1466 1467 if (VM_MAP_NULL == map) 1468 return KERN_INVALID_ARGUMENT; 1469 1470 map_addr = (vm_map_offset_t)*address; 1471 map_size = (vm_map_size_t)*size; 1472 1473 /* legacy conversion */ 1474 if (VM_REGION_BASIC_INFO == flavor) 1475 flavor = VM_REGION_BASIC_INFO_64; 1476 1477 kr = vm_map_region(map, 1478 &map_addr, &map_size, 1479 flavor, info, count, 1480 object_name); 1481 1482 *address = CAST_DOWN(vm_offset_t, map_addr); 1483 *size = CAST_DOWN(vm_size_t, map_size); 1484 1485 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) 1486 return KERN_INVALID_ADDRESS; 1487 return kr; 1488} 1489 1490kern_return_t 1491vm_region( 1492 vm_map_t map, 1493 vm_address_t *address, /* IN/OUT */ 1494 vm_size_t *size, /* OUT */ 1495 vm_region_flavor_t flavor, /* IN */ 1496 vm_region_info_t info, /* OUT */ 1497 mach_msg_type_number_t *count, /* IN/OUT */ 1498 mach_port_t *object_name) /* OUT */ 1499{ 1500 vm_map_address_t map_addr; 1501 vm_map_size_t map_size; 1502 kern_return_t kr; 1503 1504 if (VM_MAP_NULL == map) 1505 return KERN_INVALID_ARGUMENT; 1506 1507 map_addr = (vm_map_address_t)*address; 1508 map_size = (vm_map_size_t)*size; 1509 1510 kr = vm_map_region(map, 1511 &map_addr, &map_size, 1512 flavor, info, count, 1513 object_name); 1514 1515 *address = CAST_DOWN(vm_address_t, map_addr); 1516 *size = CAST_DOWN(vm_size_t, map_size); 1517 1518 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) 1519 return KERN_INVALID_ADDRESS; 1520 return kr; 1521} 1522 1523/* 1524 * vm_region_recurse: A form of vm_region which follows the 1525 * submaps in a target map 1526 * 1527 */ 1528kern_return_t 1529mach_vm_region_recurse( 1530 vm_map_t map, 1531 mach_vm_address_t *address, 1532 mach_vm_size_t *size, 1533 uint32_t *depth, 1534 vm_region_recurse_info_t info, 1535 mach_msg_type_number_t *infoCnt) 1536{ 1537 vm_map_address_t map_addr; 1538 vm_map_size_t map_size; 1539 kern_return_t kr; 1540 1541 if (VM_MAP_NULL == map) 1542 return KERN_INVALID_ARGUMENT; 1543 1544 map_addr = (vm_map_address_t)*address; 1545 map_size = (vm_map_size_t)*size; 1546 1547 kr = vm_map_region_recurse_64( 1548 map, 1549 &map_addr, 1550 &map_size, 1551 depth, 1552 (vm_region_submap_info_64_t)info, 1553 infoCnt); 1554 1555 *address = map_addr; 1556 *size = map_size; 1557 return kr; 1558} 1559 1560/* 1561 * vm_region_recurse: A form of vm_region which follows the 1562 * submaps in a target map 1563 * 1564 */ 1565kern_return_t 1566vm_region_recurse_64( 1567 vm_map_t map, 1568 vm_address_t *address, 1569 vm_size_t *size, 1570 uint32_t *depth, 1571 vm_region_recurse_info_64_t info, 1572 mach_msg_type_number_t *infoCnt) 1573{ 1574 vm_map_address_t map_addr; 1575 vm_map_size_t map_size; 1576 kern_return_t kr; 1577 1578 if (VM_MAP_NULL == map) 1579 return KERN_INVALID_ARGUMENT; 1580 1581 map_addr = (vm_map_address_t)*address; 1582 map_size = (vm_map_size_t)*size; 1583 1584 kr = vm_map_region_recurse_64( 1585 map, 1586 &map_addr, 1587 &map_size, 1588 depth, 1589 (vm_region_submap_info_64_t)info, 1590 infoCnt); 1591 1592 *address = CAST_DOWN(vm_address_t, map_addr); 1593 *size = CAST_DOWN(vm_size_t, map_size); 1594 1595 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) 1596 return KERN_INVALID_ADDRESS; 1597 return kr; 1598} 1599 1600kern_return_t 1601vm_region_recurse( 1602 vm_map_t map, 1603 vm_offset_t *address, /* IN/OUT */ 1604 vm_size_t *size, /* OUT */ 1605 natural_t *depth, /* IN/OUT */ 1606 vm_region_recurse_info_t info32, /* IN/OUT */ 1607 mach_msg_type_number_t *infoCnt) /* IN/OUT */ 1608{ 1609 vm_region_submap_info_data_64_t info64; 1610 vm_region_submap_info_t info; 1611 vm_map_address_t map_addr; 1612 vm_map_size_t map_size; 1613 kern_return_t kr; 1614 1615 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) 1616 return KERN_INVALID_ARGUMENT; 1617 1618 1619 map_addr = (vm_map_address_t)*address; 1620 map_size = (vm_map_size_t)*size; 1621 info = (vm_region_submap_info_t)info32; 1622 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64; 1623 1624 kr = vm_map_region_recurse_64(map, &map_addr,&map_size, 1625 depth, &info64, infoCnt); 1626 1627 info->protection = info64.protection; 1628 info->max_protection = info64.max_protection; 1629 info->inheritance = info64.inheritance; 1630 info->offset = (uint32_t)info64.offset; /* trouble-maker */ 1631 info->user_tag = info64.user_tag; 1632 info->pages_resident = info64.pages_resident; 1633 info->pages_shared_now_private = info64.pages_shared_now_private; 1634 info->pages_swapped_out = info64.pages_swapped_out; 1635 info->pages_dirtied = info64.pages_dirtied; 1636 info->ref_count = info64.ref_count; 1637 info->shadow_depth = info64.shadow_depth; 1638 info->external_pager = info64.external_pager; 1639 info->share_mode = info64.share_mode; 1640 info->is_submap = info64.is_submap; 1641 info->behavior = info64.behavior; 1642 info->object_id = info64.object_id; 1643 info->user_wired_count = info64.user_wired_count; 1644 1645 *address = CAST_DOWN(vm_address_t, map_addr); 1646 *size = CAST_DOWN(vm_size_t, map_size); 1647 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT; 1648 1649 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) 1650 return KERN_INVALID_ADDRESS; 1651 return kr; 1652} 1653 1654kern_return_t 1655mach_vm_purgable_control( 1656 vm_map_t map, 1657 mach_vm_offset_t address, 1658 vm_purgable_t control, 1659 int *state) 1660{ 1661 if (VM_MAP_NULL == map) 1662 return KERN_INVALID_ARGUMENT; 1663 1664 return vm_map_purgable_control(map, 1665 vm_map_trunc_page(address, PAGE_MASK), 1666 control, 1667 state); 1668} 1669 1670kern_return_t 1671vm_purgable_control( 1672 vm_map_t map, 1673 vm_offset_t address, 1674 vm_purgable_t control, 1675 int *state) 1676{ 1677 if (VM_MAP_NULL == map) 1678 return KERN_INVALID_ARGUMENT; 1679 1680 return vm_map_purgable_control(map, 1681 vm_map_trunc_page(address, PAGE_MASK), 1682 control, 1683 state); 1684} 1685 1686 1687/* 1688 * Ordinarily, the right to allocate CPM is restricted 1689 * to privileged applications (those that can gain access 1690 * to the host priv port). Set this variable to zero if 1691 * you want to let any application allocate CPM. 1692 */ 1693unsigned int vm_allocate_cpm_privileged = 0; 1694 1695/* 1696 * Allocate memory in the specified map, with the caveat that 1697 * the memory is physically contiguous. This call may fail 1698 * if the system can't find sufficient contiguous memory. 1699 * This call may cause or lead to heart-stopping amounts of 1700 * paging activity. 1701 * 1702 * Memory obtained from this call should be freed in the 1703 * normal way, viz., via vm_deallocate. 1704 */ 1705kern_return_t 1706vm_allocate_cpm( 1707 host_priv_t host_priv, 1708 vm_map_t map, 1709 vm_address_t *addr, 1710 vm_size_t size, 1711 int flags) 1712{ 1713 vm_map_address_t map_addr; 1714 vm_map_size_t map_size; 1715 kern_return_t kr; 1716 1717 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv) 1718 return KERN_INVALID_HOST; 1719 1720 if (VM_MAP_NULL == map) 1721 return KERN_INVALID_ARGUMENT; 1722 1723 map_addr = (vm_map_address_t)*addr; 1724 map_size = (vm_map_size_t)size; 1725 1726 kr = vm_map_enter_cpm(map, 1727 &map_addr, 1728 map_size, 1729 flags); 1730 1731 *addr = CAST_DOWN(vm_address_t, map_addr); 1732 return kr; 1733} 1734 1735 1736kern_return_t 1737mach_vm_page_query( 1738 vm_map_t map, 1739 mach_vm_offset_t offset, 1740 int *disposition, 1741 int *ref_count) 1742{ 1743 if (VM_MAP_NULL == map) 1744 return KERN_INVALID_ARGUMENT; 1745 1746 return vm_map_page_query_internal( 1747 map, 1748 vm_map_trunc_page(offset, PAGE_MASK), 1749 disposition, ref_count); 1750} 1751 1752kern_return_t 1753vm_map_page_query( 1754 vm_map_t map, 1755 vm_offset_t offset, 1756 int *disposition, 1757 int *ref_count) 1758{ 1759 if (VM_MAP_NULL == map) 1760 return KERN_INVALID_ARGUMENT; 1761 1762 return vm_map_page_query_internal( 1763 map, 1764 vm_map_trunc_page(offset, PAGE_MASK), 1765 disposition, ref_count); 1766} 1767 1768kern_return_t 1769mach_vm_page_info( 1770 vm_map_t map, 1771 mach_vm_address_t address, 1772 vm_page_info_flavor_t flavor, 1773 vm_page_info_t info, 1774 mach_msg_type_number_t *count) 1775{ 1776 kern_return_t kr; 1777 1778 if (map == VM_MAP_NULL) { 1779 return KERN_INVALID_ARGUMENT; 1780 } 1781 1782 kr = vm_map_page_info(map, address, flavor, info, count); 1783 return kr; 1784} 1785 1786/* map a (whole) upl into an address space */ 1787kern_return_t 1788vm_upl_map( 1789 vm_map_t map, 1790 upl_t upl, 1791 vm_address_t *dst_addr) 1792{ 1793 vm_map_offset_t map_addr; 1794 kern_return_t kr; 1795 1796 if (VM_MAP_NULL == map) 1797 return KERN_INVALID_ARGUMENT; 1798 1799 kr = vm_map_enter_upl(map, upl, &map_addr); 1800 *dst_addr = CAST_DOWN(vm_address_t, map_addr); 1801 return kr; 1802} 1803 1804kern_return_t 1805vm_upl_unmap( 1806 vm_map_t map, 1807 upl_t upl) 1808{ 1809 if (VM_MAP_NULL == map) 1810 return KERN_INVALID_ARGUMENT; 1811 1812 return (vm_map_remove_upl(map, upl)); 1813} 1814 1815/* Retrieve a upl for an object underlying an address range in a map */ 1816 1817kern_return_t 1818vm_map_get_upl( 1819 vm_map_t map, 1820 vm_map_offset_t map_offset, 1821 upl_size_t *upl_size, 1822 upl_t *upl, 1823 upl_page_info_array_t page_list, 1824 unsigned int *count, 1825 int *flags, 1826 int force_data_sync) 1827{ 1828 int map_flags; 1829 kern_return_t kr; 1830 1831 if (VM_MAP_NULL == map) 1832 return KERN_INVALID_ARGUMENT; 1833 1834 map_flags = *flags & ~UPL_NOZEROFILL; 1835 if (force_data_sync) 1836 map_flags |= UPL_FORCE_DATA_SYNC; 1837 1838 kr = vm_map_create_upl(map, 1839 map_offset, 1840 upl_size, 1841 upl, 1842 page_list, 1843 count, 1844 &map_flags); 1845 1846 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC); 1847 return kr; 1848} 1849 1850/* 1851 * mach_make_memory_entry_64 1852 * 1853 * Think of it as a two-stage vm_remap() operation. First 1854 * you get a handle. Second, you get map that handle in 1855 * somewhere else. Rather than doing it all at once (and 1856 * without needing access to the other whole map). 1857 */ 1858 1859kern_return_t 1860mach_make_memory_entry_64( 1861 vm_map_t target_map, 1862 memory_object_size_t *size, 1863 memory_object_offset_t offset, 1864 vm_prot_t permission, 1865 ipc_port_t *object_handle, 1866 ipc_port_t parent_handle) 1867{ 1868 vm_map_version_t version; 1869 vm_named_entry_t parent_entry; 1870 vm_named_entry_t user_entry; 1871 ipc_port_t user_handle; 1872 kern_return_t kr; 1873 vm_map_t real_map; 1874 1875 /* needed for call to vm_map_lookup_locked */ 1876 boolean_t wired; 1877 vm_object_offset_t obj_off; 1878 vm_prot_t prot; 1879 struct vm_object_fault_info fault_info; 1880 vm_object_t object; 1881 vm_object_t shadow_object; 1882 1883 /* needed for direct map entry manipulation */ 1884 vm_map_entry_t map_entry; 1885 vm_map_entry_t next_entry; 1886 vm_map_t local_map; 1887 vm_map_t original_map = target_map; 1888 vm_map_size_t total_size; 1889 vm_map_size_t map_size; 1890 vm_map_offset_t map_offset; 1891 vm_map_offset_t local_offset; 1892 vm_object_size_t mappable_size; 1893 1894 /* 1895 * Stash the offset in the page for use by vm_map_enter_mem_object() 1896 * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case. 1897 */ 1898 vm_object_offset_t offset_in_page; 1899 1900 unsigned int access; 1901 vm_prot_t protections; 1902 vm_prot_t original_protections, mask_protections; 1903 unsigned int wimg_mode; 1904 1905 boolean_t force_shadow = FALSE; 1906 boolean_t use_data_addr; 1907 1908 if (((permission & 0x00FF0000) & 1909 ~(MAP_MEM_ONLY | 1910 MAP_MEM_NAMED_CREATE | 1911 MAP_MEM_PURGABLE | 1912 MAP_MEM_NAMED_REUSE | 1913 MAP_MEM_USE_DATA_ADDR | 1914 MAP_MEM_VM_COPY | 1915 MAP_MEM_VM_SHARE))) { 1916 /* 1917 * Unknown flag: reject for forward compatibility. 1918 */ 1919 return KERN_INVALID_VALUE; 1920 } 1921 1922 if (parent_handle != IP_NULL && 1923 ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) { 1924 parent_entry = (vm_named_entry_t) parent_handle->ip_kobject; 1925 } else { 1926 parent_entry = NULL; 1927 } 1928 1929 if (parent_entry && parent_entry->is_copy) { 1930 return KERN_INVALID_ARGUMENT; 1931 } 1932 1933 original_protections = permission & VM_PROT_ALL; 1934 protections = original_protections; 1935 mask_protections = permission & VM_PROT_IS_MASK; 1936 access = GET_MAP_MEM(permission); 1937 use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0); 1938 1939 user_handle = IP_NULL; 1940 user_entry = NULL; 1941 1942 map_offset = vm_map_trunc_page(offset, PAGE_MASK); 1943 1944 if (permission & MAP_MEM_ONLY) { 1945 boolean_t parent_is_object; 1946 1947 map_size = vm_map_round_page(*size, PAGE_MASK); 1948 1949 if (use_data_addr || parent_entry == NULL) { 1950 return KERN_INVALID_ARGUMENT; 1951 } 1952 1953 parent_is_object = !(parent_entry->is_sub_map || 1954 parent_entry->is_pager); 1955 object = parent_entry->backing.object; 1956 if(parent_is_object && object != VM_OBJECT_NULL) 1957 wimg_mode = object->wimg_bits; 1958 else 1959 wimg_mode = VM_WIMG_USE_DEFAULT; 1960 if((access != GET_MAP_MEM(parent_entry->protection)) && 1961 !(parent_entry->protection & VM_PROT_WRITE)) { 1962 return KERN_INVALID_RIGHT; 1963 } 1964 if(access == MAP_MEM_IO) { 1965 SET_MAP_MEM(access, parent_entry->protection); 1966 wimg_mode = VM_WIMG_IO; 1967 } else if (access == MAP_MEM_COPYBACK) { 1968 SET_MAP_MEM(access, parent_entry->protection); 1969 wimg_mode = VM_WIMG_USE_DEFAULT; 1970 } else if (access == MAP_MEM_INNERWBACK) { 1971 SET_MAP_MEM(access, parent_entry->protection); 1972 wimg_mode = VM_WIMG_INNERWBACK; 1973 } else if (access == MAP_MEM_WTHRU) { 1974 SET_MAP_MEM(access, parent_entry->protection); 1975 wimg_mode = VM_WIMG_WTHRU; 1976 } else if (access == MAP_MEM_WCOMB) { 1977 SET_MAP_MEM(access, parent_entry->protection); 1978 wimg_mode = VM_WIMG_WCOMB; 1979 } 1980 if (parent_is_object && object && 1981 (access != MAP_MEM_NOOP) && 1982 (!(object->nophyscache))) { 1983 1984 if (object->wimg_bits != wimg_mode) { 1985 vm_object_lock(object); 1986 vm_object_change_wimg_mode(object, wimg_mode); 1987 vm_object_unlock(object); 1988 } 1989 } 1990 if (object_handle) 1991 *object_handle = IP_NULL; 1992 return KERN_SUCCESS; 1993 } else if (permission & MAP_MEM_NAMED_CREATE) { 1994 map_size = vm_map_round_page(*size, PAGE_MASK); 1995 1996 if (use_data_addr) { 1997 return KERN_INVALID_ARGUMENT; 1998 } 1999 2000 kr = mach_memory_entry_allocate(&user_entry, &user_handle); 2001 if (kr != KERN_SUCCESS) { 2002 return KERN_FAILURE; 2003 } 2004 2005 /* 2006 * Force the creation of the VM object now. 2007 */ 2008 if (map_size > (vm_map_size_t) ANON_MAX_SIZE) { 2009 /* 2010 * LP64todo - for now, we can only allocate 4GB-4096 2011 * internal objects because the default pager can't 2012 * page bigger ones. Remove this when it can. 2013 */ 2014 kr = KERN_FAILURE; 2015 goto make_mem_done; 2016 } 2017 2018 object = vm_object_allocate(map_size); 2019 assert(object != VM_OBJECT_NULL); 2020 2021 if (permission & MAP_MEM_PURGABLE) { 2022 if (! (permission & VM_PROT_WRITE)) { 2023 /* if we can't write, we can't purge */ 2024 vm_object_deallocate(object); 2025 kr = KERN_INVALID_ARGUMENT; 2026 goto make_mem_done; 2027 } 2028 object->purgable = VM_PURGABLE_NONVOLATILE; 2029 assert(object->vo_purgeable_owner == NULL); 2030 assert(object->resident_page_count == 0); 2031 assert(object->wired_page_count == 0); 2032 vm_object_lock(object); 2033 vm_purgeable_nonvolatile_enqueue(object, 2034 current_task()); 2035 vm_object_unlock(object); 2036 } 2037 2038 /* 2039 * The VM object is brand new and nobody else knows about it, 2040 * so we don't need to lock it. 2041 */ 2042 2043 wimg_mode = object->wimg_bits; 2044 if (access == MAP_MEM_IO) { 2045 wimg_mode = VM_WIMG_IO; 2046 } else if (access == MAP_MEM_COPYBACK) { 2047 wimg_mode = VM_WIMG_USE_DEFAULT; 2048 } else if (access == MAP_MEM_INNERWBACK) { 2049 wimg_mode = VM_WIMG_INNERWBACK; 2050 } else if (access == MAP_MEM_WTHRU) { 2051 wimg_mode = VM_WIMG_WTHRU; 2052 } else if (access == MAP_MEM_WCOMB) { 2053 wimg_mode = VM_WIMG_WCOMB; 2054 } 2055 if (access != MAP_MEM_NOOP) { 2056 object->wimg_bits = wimg_mode; 2057 } 2058 /* the object has no pages, so no WIMG bits to update here */ 2059 2060 /* 2061 * XXX 2062 * We use this path when we want to make sure that 2063 * nobody messes with the object (coalesce, for 2064 * example) before we map it. 2065 * We might want to use these objects for transposition via 2066 * vm_object_transpose() too, so we don't want any copy or 2067 * shadow objects either... 2068 */ 2069 object->copy_strategy = MEMORY_OBJECT_COPY_NONE; 2070 object->true_share = TRUE; 2071 2072 user_entry->backing.object = object; 2073 user_entry->internal = TRUE; 2074 user_entry->is_sub_map = FALSE; 2075 user_entry->is_pager = FALSE; 2076 user_entry->offset = 0; 2077 user_entry->data_offset = 0; 2078 user_entry->protection = protections; 2079 SET_MAP_MEM(access, user_entry->protection); 2080 user_entry->size = map_size; 2081 2082 /* user_object pager and internal fields are not used */ 2083 /* when the object field is filled in. */ 2084 2085 *size = CAST_DOWN(vm_size_t, map_size); 2086 *object_handle = user_handle; 2087 return KERN_SUCCESS; 2088 } 2089 2090 if (permission & MAP_MEM_VM_COPY) { 2091 vm_map_copy_t copy; 2092 2093 if (target_map == VM_MAP_NULL) { 2094 return KERN_INVALID_TASK; 2095 } 2096 2097 if (use_data_addr) { 2098 map_size = (vm_map_round_page(offset + *size, 2099 PAGE_MASK) - 2100 map_offset); 2101 offset_in_page = offset - map_offset; 2102 } else { 2103 map_size = vm_map_round_page(*size, PAGE_MASK); 2104 offset_in_page = 0; 2105 } 2106 2107 kr = vm_map_copyin(target_map, 2108 map_offset, 2109 map_size, 2110 FALSE, 2111 ©); 2112 if (kr != KERN_SUCCESS) { 2113 return kr; 2114 } 2115 2116 kr = mach_memory_entry_allocate(&user_entry, &user_handle); 2117 if (kr != KERN_SUCCESS) { 2118 vm_map_copy_discard(copy); 2119 return KERN_FAILURE; 2120 } 2121 2122 user_entry->backing.copy = copy; 2123 user_entry->internal = FALSE; 2124 user_entry->is_sub_map = FALSE; 2125 user_entry->is_pager = FALSE; 2126 user_entry->is_copy = TRUE; 2127 user_entry->offset = 0; 2128 user_entry->protection = protections; 2129 user_entry->size = map_size; 2130 user_entry->data_offset = offset_in_page; 2131 2132 *size = CAST_DOWN(vm_size_t, map_size); 2133 *object_handle = user_handle; 2134 return KERN_SUCCESS; 2135 } 2136 2137 if (permission & MAP_MEM_VM_SHARE) { 2138 vm_map_copy_t copy; 2139 vm_prot_t cur_prot, max_prot; 2140 2141 if (target_map == VM_MAP_NULL) { 2142 return KERN_INVALID_TASK; 2143 } 2144 2145 if (use_data_addr) { 2146 map_size = (vm_map_round_page(offset + *size, 2147 PAGE_MASK) - 2148 map_offset); 2149 offset_in_page = offset - map_offset; 2150 } else { 2151 map_size = vm_map_round_page(*size, PAGE_MASK); 2152 offset_in_page = 0; 2153 } 2154 2155 kr = vm_map_copy_extract(target_map, 2156 map_offset, 2157 map_size, 2158 ©, 2159 &cur_prot, 2160 &max_prot); 2161 if (kr != KERN_SUCCESS) { 2162 return kr; 2163 } 2164 2165 if (mask_protections) { 2166 /* 2167 * We just want as much of "original_protections" 2168 * as we can get out of the actual "cur_prot". 2169 */ 2170 protections &= cur_prot; 2171 if (protections == VM_PROT_NONE) { 2172 /* no access at all: fail */ 2173 vm_map_copy_discard(copy); 2174 return KERN_PROTECTION_FAILURE; 2175 } 2176 } else { 2177 /* 2178 * We want exactly "original_protections" 2179 * out of "cur_prot". 2180 */ 2181 if ((cur_prot & protections) != protections) { 2182 vm_map_copy_discard(copy); 2183 return KERN_PROTECTION_FAILURE; 2184 } 2185 } 2186 2187 kr = mach_memory_entry_allocate(&user_entry, &user_handle); 2188 if (kr != KERN_SUCCESS) { 2189 vm_map_copy_discard(copy); 2190 return KERN_FAILURE; 2191 } 2192 2193 user_entry->backing.copy = copy; 2194 user_entry->internal = FALSE; 2195 user_entry->is_sub_map = FALSE; 2196 user_entry->is_pager = FALSE; 2197 user_entry->is_copy = TRUE; 2198 user_entry->offset = 0; 2199 user_entry->protection = protections; 2200 user_entry->size = map_size; 2201 user_entry->data_offset = offset_in_page; 2202 2203 *size = CAST_DOWN(vm_size_t, map_size); 2204 *object_handle = user_handle; 2205 return KERN_SUCCESS; 2206 } 2207 2208 if (parent_entry == NULL || 2209 (permission & MAP_MEM_NAMED_REUSE)) { 2210 2211 if (use_data_addr) { 2212 map_size = vm_map_round_page(offset + *size, PAGE_MASK) - map_offset; 2213 offset_in_page = offset - map_offset; 2214 } else { 2215 map_size = vm_map_round_page(*size, PAGE_MASK); 2216 offset_in_page = 0; 2217 } 2218 2219 /* Create a named object based on address range within the task map */ 2220 /* Go find the object at given address */ 2221 2222 if (target_map == VM_MAP_NULL) { 2223 return KERN_INVALID_TASK; 2224 } 2225 2226redo_lookup: 2227 protections = original_protections; 2228 vm_map_lock_read(target_map); 2229 2230 /* get the object associated with the target address */ 2231 /* note we check the permission of the range against */ 2232 /* that requested by the caller */ 2233 2234 kr = vm_map_lookup_locked(&target_map, map_offset, 2235 protections | mask_protections, 2236 OBJECT_LOCK_EXCLUSIVE, &version, 2237 &object, &obj_off, &prot, &wired, 2238 &fault_info, 2239 &real_map); 2240 if (kr != KERN_SUCCESS) { 2241 vm_map_unlock_read(target_map); 2242 goto make_mem_done; 2243 } 2244 if (mask_protections) { 2245 /* 2246 * The caller asked us to use the "protections" as 2247 * a mask, so restrict "protections" to what this 2248 * mapping actually allows. 2249 */ 2250 protections &= prot; 2251 } 2252 if (((prot & protections) != protections) 2253 || (object == kernel_object)) { 2254 kr = KERN_INVALID_RIGHT; 2255 vm_object_unlock(object); 2256 vm_map_unlock_read(target_map); 2257 if(real_map != target_map) 2258 vm_map_unlock_read(real_map); 2259 if(object == kernel_object) { 2260 printf("Warning: Attempt to create a named" 2261 " entry from the kernel_object\n"); 2262 } 2263 goto make_mem_done; 2264 } 2265 2266 /* We have an object, now check to see if this object */ 2267 /* is suitable. If not, create a shadow and share that */ 2268 2269 /* 2270 * We have to unlock the VM object to avoid deadlocking with 2271 * a VM map lock (the lock ordering is map, the object), if we 2272 * need to modify the VM map to create a shadow object. Since 2273 * we might release the VM map lock below anyway, we have 2274 * to release the VM map lock now. 2275 * XXX FBDP There must be a way to avoid this double lookup... 2276 * 2277 * Take an extra reference on the VM object to make sure it's 2278 * not going to disappear. 2279 */ 2280 vm_object_reference_locked(object); /* extra ref to hold obj */ 2281 vm_object_unlock(object); 2282 2283 local_map = original_map; 2284 local_offset = map_offset; 2285 if(target_map != local_map) { 2286 vm_map_unlock_read(target_map); 2287 if(real_map != target_map) 2288 vm_map_unlock_read(real_map); 2289 vm_map_lock_read(local_map); 2290 target_map = local_map; 2291 real_map = local_map; 2292 } 2293 while(TRUE) { 2294 if(!vm_map_lookup_entry(local_map, 2295 local_offset, &map_entry)) { 2296 kr = KERN_INVALID_ARGUMENT; 2297 vm_map_unlock_read(target_map); 2298 if(real_map != target_map) 2299 vm_map_unlock_read(real_map); 2300 vm_object_deallocate(object); /* release extra ref */ 2301 object = VM_OBJECT_NULL; 2302 goto make_mem_done; 2303 } 2304 if(!(map_entry->is_sub_map)) { 2305 if(map_entry->object.vm_object != object) { 2306 kr = KERN_INVALID_ARGUMENT; 2307 vm_map_unlock_read(target_map); 2308 if(real_map != target_map) 2309 vm_map_unlock_read(real_map); 2310 vm_object_deallocate(object); /* release extra ref */ 2311 object = VM_OBJECT_NULL; 2312 goto make_mem_done; 2313 } 2314 break; 2315 } else { 2316 vm_map_t tmap; 2317 tmap = local_map; 2318 local_map = map_entry->object.sub_map; 2319 2320 vm_map_lock_read(local_map); 2321 vm_map_unlock_read(tmap); 2322 target_map = local_map; 2323 real_map = local_map; 2324 local_offset = local_offset - map_entry->vme_start; 2325 local_offset += map_entry->offset; 2326 } 2327 } 2328 2329 /* 2330 * We found the VM map entry, lock the VM object again. 2331 */ 2332 vm_object_lock(object); 2333 if(map_entry->wired_count) { 2334 /* JMM - The check below should be reworked instead. */ 2335 object->true_share = TRUE; 2336 } 2337 if (mask_protections) { 2338 /* 2339 * The caller asked us to use the "protections" as 2340 * a mask, so restrict "protections" to what this 2341 * mapping actually allows. 2342 */ 2343 protections &= map_entry->max_protection; 2344 } 2345 if(((map_entry->max_protection) & protections) != protections) { 2346 kr = KERN_INVALID_RIGHT; 2347 vm_object_unlock(object); 2348 vm_map_unlock_read(target_map); 2349 if(real_map != target_map) 2350 vm_map_unlock_read(real_map); 2351 vm_object_deallocate(object); 2352 object = VM_OBJECT_NULL; 2353 goto make_mem_done; 2354 } 2355 2356 mappable_size = fault_info.hi_offset - obj_off; 2357 total_size = map_entry->vme_end - map_entry->vme_start; 2358 if(map_size > mappable_size) { 2359 /* try to extend mappable size if the entries */ 2360 /* following are from the same object and are */ 2361 /* compatible */ 2362 next_entry = map_entry->vme_next; 2363 /* lets see if the next map entry is still */ 2364 /* pointing at this object and is contiguous */ 2365 while(map_size > mappable_size) { 2366 if((next_entry->object.vm_object == object) && 2367 (next_entry->vme_start == 2368 next_entry->vme_prev->vme_end) && 2369 (next_entry->offset == 2370 next_entry->vme_prev->offset + 2371 (next_entry->vme_prev->vme_end - 2372 next_entry->vme_prev->vme_start))) { 2373 if (mask_protections) { 2374 /* 2375 * The caller asked us to use 2376 * the "protections" as a mask, 2377 * so restrict "protections" to 2378 * what this mapping actually 2379 * allows. 2380 */ 2381 protections &= next_entry->max_protection; 2382 } 2383 if ((next_entry->wired_count) && 2384 (map_entry->wired_count == 0)) { 2385 break; 2386 } 2387 if(((next_entry->max_protection) 2388 & protections) != protections) { 2389 break; 2390 } 2391 if (next_entry->needs_copy != 2392 map_entry->needs_copy) 2393 break; 2394 mappable_size += next_entry->vme_end 2395 - next_entry->vme_start; 2396 total_size += next_entry->vme_end 2397 - next_entry->vme_start; 2398 next_entry = next_entry->vme_next; 2399 } else { 2400 break; 2401 } 2402 2403 } 2404 } 2405 2406 if (vm_map_entry_should_cow_for_true_share(map_entry) && 2407 object->vo_size > map_size && 2408 map_size != 0) { 2409 /* 2410 * Set up the targeted range for copy-on-write to 2411 * limit the impact of "true_share"/"copy_delay" to 2412 * that range instead of the entire VM object... 2413 */ 2414 2415 vm_object_unlock(object); 2416 if (vm_map_lock_read_to_write(target_map)) { 2417 vm_object_deallocate(object); 2418 target_map = original_map; 2419 goto redo_lookup; 2420 } 2421 2422 vm_map_clip_start(target_map, 2423 map_entry, 2424 vm_map_trunc_page(offset, 2425 VM_MAP_PAGE_MASK(target_map))); 2426 vm_map_clip_end(target_map, 2427 map_entry, 2428 (vm_map_round_page(offset + map_size, 2429 VM_MAP_PAGE_MASK(target_map)))); 2430 force_shadow = TRUE; 2431 2432 if ((map_entry->vme_end - offset) < map_size) { 2433 map_size = map_entry->vme_end - offset; 2434 } 2435 total_size = map_entry->vme_end - map_entry->vme_start; 2436 2437 vm_map_lock_write_to_read(target_map); 2438 vm_object_lock(object); 2439 } 2440 2441 if (object->internal) { 2442 /* vm_map_lookup_locked will create a shadow if */ 2443 /* needs_copy is set but does not check for the */ 2444 /* other two conditions shown. It is important to */ 2445 /* set up an object which will not be pulled from */ 2446 /* under us. */ 2447 2448 if (force_shadow || 2449 ((map_entry->needs_copy || 2450 object->shadowed || 2451 (object->vo_size > total_size && 2452 (map_entry->offset != 0 || 2453 object->vo_size > 2454 vm_map_round_page(total_size, 2455 VM_MAP_PAGE_MASK(target_map))))) 2456 && !object->true_share)) { 2457 /* 2458 * We have to unlock the VM object before 2459 * trying to upgrade the VM map lock, to 2460 * honor lock ordering (map then object). 2461 * Otherwise, we would deadlock if another 2462 * thread holds a read lock on the VM map and 2463 * is trying to acquire the VM object's lock. 2464 * We still hold an extra reference on the 2465 * VM object, guaranteeing that it won't 2466 * disappear. 2467 */ 2468 vm_object_unlock(object); 2469 2470 if (vm_map_lock_read_to_write(target_map)) { 2471 /* 2472 * We couldn't upgrade our VM map lock 2473 * from "read" to "write" and we lost 2474 * our "read" lock. 2475 * Start all over again... 2476 */ 2477 vm_object_deallocate(object); /* extra ref */ 2478 target_map = original_map; 2479 goto redo_lookup; 2480 } 2481#if 00 2482 vm_object_lock(object); 2483#endif 2484 2485 /* 2486 * JMM - We need to avoid coming here when the object 2487 * is wired by anybody, not just the current map. Why 2488 * couldn't we use the standard vm_object_copy_quickly() 2489 * approach here? 2490 */ 2491 2492 /* create a shadow object */ 2493 vm_object_shadow(&map_entry->object.vm_object, 2494 &map_entry->offset, total_size); 2495 shadow_object = map_entry->object.vm_object; 2496#if 00 2497 vm_object_unlock(object); 2498#endif 2499 2500 prot = map_entry->protection & ~VM_PROT_WRITE; 2501 2502 if (override_nx(target_map, map_entry->alias) && prot) 2503 prot |= VM_PROT_EXECUTE; 2504 2505 vm_object_pmap_protect( 2506 object, map_entry->offset, 2507 total_size, 2508 ((map_entry->is_shared 2509 || target_map->mapped_in_other_pmaps) 2510 ? PMAP_NULL : 2511 target_map->pmap), 2512 map_entry->vme_start, 2513 prot); 2514 total_size -= (map_entry->vme_end 2515 - map_entry->vme_start); 2516 next_entry = map_entry->vme_next; 2517 map_entry->needs_copy = FALSE; 2518 2519 vm_object_lock(shadow_object); 2520 while (total_size) { 2521 assert((next_entry->wired_count == 0) || 2522 (map_entry->wired_count)); 2523 2524 if(next_entry->object.vm_object == object) { 2525 vm_object_reference_locked(shadow_object); 2526 next_entry->object.vm_object 2527 = shadow_object; 2528 vm_object_deallocate(object); 2529 next_entry->offset 2530 = next_entry->vme_prev->offset + 2531 (next_entry->vme_prev->vme_end 2532 - next_entry->vme_prev->vme_start); 2533 next_entry->needs_copy = FALSE; 2534 } else { 2535 panic("mach_make_memory_entry_64:" 2536 " map entries out of sync\n"); 2537 } 2538 total_size -= 2539 next_entry->vme_end 2540 - next_entry->vme_start; 2541 next_entry = next_entry->vme_next; 2542 } 2543 2544 /* 2545 * Transfer our extra reference to the 2546 * shadow object. 2547 */ 2548 vm_object_reference_locked(shadow_object); 2549 vm_object_deallocate(object); /* extra ref */ 2550 object = shadow_object; 2551 2552 obj_off = (local_offset - map_entry->vme_start) 2553 + map_entry->offset; 2554 2555 vm_map_lock_write_to_read(target_map); 2556 } 2557 } 2558 2559 /* note: in the future we can (if necessary) allow for */ 2560 /* memory object lists, this will better support */ 2561 /* fragmentation, but is it necessary? The user should */ 2562 /* be encouraged to create address space oriented */ 2563 /* shared objects from CLEAN memory regions which have */ 2564 /* a known and defined history. i.e. no inheritence */ 2565 /* share, make this call before making the region the */ 2566 /* target of ipc's, etc. The code above, protecting */ 2567 /* against delayed copy, etc. is mostly defensive. */ 2568 2569 wimg_mode = object->wimg_bits; 2570 if(!(object->nophyscache)) { 2571 if(access == MAP_MEM_IO) { 2572 wimg_mode = VM_WIMG_IO; 2573 } else if (access == MAP_MEM_COPYBACK) { 2574 wimg_mode = VM_WIMG_USE_DEFAULT; 2575 } else if (access == MAP_MEM_INNERWBACK) { 2576 wimg_mode = VM_WIMG_INNERWBACK; 2577 } else if (access == MAP_MEM_WTHRU) { 2578 wimg_mode = VM_WIMG_WTHRU; 2579 } else if (access == MAP_MEM_WCOMB) { 2580 wimg_mode = VM_WIMG_WCOMB; 2581 } 2582 } 2583 2584#if VM_OBJECT_TRACKING_OP_TRUESHARE 2585 if (!object->true_share && 2586 vm_object_tracking_inited) { 2587 void *bt[VM_OBJECT_TRACKING_BTDEPTH]; 2588 int num = 0; 2589 2590 num = OSBacktrace(bt, 2591 VM_OBJECT_TRACKING_BTDEPTH); 2592 btlog_add_entry(vm_object_tracking_btlog, 2593 object, 2594 VM_OBJECT_TRACKING_OP_TRUESHARE, 2595 bt, 2596 num); 2597 } 2598#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ 2599 2600 object->true_share = TRUE; 2601 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) 2602 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; 2603 2604 /* 2605 * The memory entry now points to this VM object and we 2606 * need to hold a reference on the VM object. Use the extra 2607 * reference we took earlier to keep the object alive when we 2608 * had to unlock it. 2609 */ 2610 2611 vm_map_unlock_read(target_map); 2612 if(real_map != target_map) 2613 vm_map_unlock_read(real_map); 2614 2615 if (object->wimg_bits != wimg_mode) 2616 vm_object_change_wimg_mode(object, wimg_mode); 2617 2618 /* the size of mapped entry that overlaps with our region */ 2619 /* which is targeted for share. */ 2620 /* (entry_end - entry_start) - */ 2621 /* offset of our beg addr within entry */ 2622 /* it corresponds to this: */ 2623 2624 if(map_size > mappable_size) 2625 map_size = mappable_size; 2626 2627 if (permission & MAP_MEM_NAMED_REUSE) { 2628 /* 2629 * Compare what we got with the "parent_entry". 2630 * If they match, re-use the "parent_entry" instead 2631 * of creating a new one. 2632 */ 2633 if (parent_entry != NULL && 2634 parent_entry->backing.object == object && 2635 parent_entry->internal == object->internal && 2636 parent_entry->is_sub_map == FALSE && 2637 parent_entry->is_pager == FALSE && 2638 parent_entry->offset == obj_off && 2639 parent_entry->protection == protections && 2640 parent_entry->size == map_size && 2641 ((!use_data_addr && (parent_entry->data_offset == 0)) || 2642 (use_data_addr && (parent_entry->data_offset == offset_in_page)))) { 2643 /* 2644 * We have a match: re-use "parent_entry". 2645 */ 2646 /* release our extra reference on object */ 2647 vm_object_unlock(object); 2648 vm_object_deallocate(object); 2649 /* parent_entry->ref_count++; XXX ? */ 2650 /* Get an extra send-right on handle */ 2651 ipc_port_copy_send(parent_handle); 2652 2653 *size = CAST_DOWN(vm_size_t, map_size); 2654 *object_handle = parent_handle; 2655 return KERN_SUCCESS; 2656 } else { 2657 /* 2658 * No match: we need to create a new entry. 2659 * fall through... 2660 */ 2661 } 2662 } 2663 2664 vm_object_unlock(object); 2665 if (mach_memory_entry_allocate(&user_entry, &user_handle) 2666 != KERN_SUCCESS) { 2667 /* release our unused reference on the object */ 2668 vm_object_deallocate(object); 2669 return KERN_FAILURE; 2670 } 2671 2672 user_entry->backing.object = object; 2673 user_entry->internal = object->internal; 2674 user_entry->is_sub_map = FALSE; 2675 user_entry->is_pager = FALSE; 2676 user_entry->offset = obj_off; 2677 user_entry->data_offset = offset_in_page; 2678 user_entry->protection = protections; 2679 SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection); 2680 user_entry->size = map_size; 2681 2682 /* user_object pager and internal fields are not used */ 2683 /* when the object field is filled in. */ 2684 2685 *size = CAST_DOWN(vm_size_t, map_size); 2686 *object_handle = user_handle; 2687 return KERN_SUCCESS; 2688 2689 } else { 2690 /* The new object will be base on an existing named object */ 2691 if (parent_entry == NULL) { 2692 kr = KERN_INVALID_ARGUMENT; 2693 goto make_mem_done; 2694 } 2695 2696 if (use_data_addr) { 2697 /* 2698 * submaps and pagers should only be accessible from within 2699 * the kernel, which shouldn't use the data address flag, so can fail here. 2700 */ 2701 if (parent_entry->is_pager || parent_entry->is_sub_map) { 2702 panic("Shouldn't be using data address with a parent entry that is a submap or pager."); 2703 } 2704 /* 2705 * Account for offset to data in parent entry and 2706 * compute our own offset to data. 2707 */ 2708 if((offset + *size + parent_entry->data_offset) > parent_entry->size) { 2709 kr = KERN_INVALID_ARGUMENT; 2710 goto make_mem_done; 2711 } 2712 2713 map_offset = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK); 2714 offset_in_page = (offset + parent_entry->data_offset) - map_offset; 2715 map_size = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK) - map_offset; 2716 } else { 2717 map_size = vm_map_round_page(*size, PAGE_MASK); 2718 offset_in_page = 0; 2719 2720 if((offset + map_size) > parent_entry->size) { 2721 kr = KERN_INVALID_ARGUMENT; 2722 goto make_mem_done; 2723 } 2724 } 2725 2726 if (mask_protections) { 2727 /* 2728 * The caller asked us to use the "protections" as 2729 * a mask, so restrict "protections" to what this 2730 * mapping actually allows. 2731 */ 2732 protections &= parent_entry->protection; 2733 } 2734 if((protections & parent_entry->protection) != protections) { 2735 kr = KERN_PROTECTION_FAILURE; 2736 goto make_mem_done; 2737 } 2738 2739 if (mach_memory_entry_allocate(&user_entry, &user_handle) 2740 != KERN_SUCCESS) { 2741 kr = KERN_FAILURE; 2742 goto make_mem_done; 2743 } 2744 2745 user_entry->size = map_size; 2746 user_entry->offset = parent_entry->offset + map_offset; 2747 user_entry->data_offset = offset_in_page; 2748 user_entry->is_sub_map = parent_entry->is_sub_map; 2749 user_entry->is_pager = parent_entry->is_pager; 2750 user_entry->is_copy = parent_entry->is_copy; 2751 user_entry->internal = parent_entry->internal; 2752 user_entry->protection = protections; 2753 2754 if(access != MAP_MEM_NOOP) { 2755 SET_MAP_MEM(access, user_entry->protection); 2756 } 2757 2758 if(parent_entry->is_sub_map) { 2759 user_entry->backing.map = parent_entry->backing.map; 2760 vm_map_lock(user_entry->backing.map); 2761 user_entry->backing.map->ref_count++; 2762 vm_map_unlock(user_entry->backing.map); 2763 } 2764 else if (parent_entry->is_pager) { 2765 user_entry->backing.pager = parent_entry->backing.pager; 2766 /* JMM - don't we need a reference here? */ 2767 } else { 2768 object = parent_entry->backing.object; 2769 assert(object != VM_OBJECT_NULL); 2770 user_entry->backing.object = object; 2771 /* we now point to this object, hold on */ 2772 vm_object_reference(object); 2773 vm_object_lock(object); 2774#if VM_OBJECT_TRACKING_OP_TRUESHARE 2775 if (!object->true_share && 2776 vm_object_tracking_inited) { 2777 void *bt[VM_OBJECT_TRACKING_BTDEPTH]; 2778 int num = 0; 2779 2780 num = OSBacktrace(bt, 2781 VM_OBJECT_TRACKING_BTDEPTH); 2782 btlog_add_entry(vm_object_tracking_btlog, 2783 object, 2784 VM_OBJECT_TRACKING_OP_TRUESHARE, 2785 bt, 2786 num); 2787 } 2788#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */ 2789 2790 object->true_share = TRUE; 2791 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) 2792 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; 2793 vm_object_unlock(object); 2794 } 2795 *size = CAST_DOWN(vm_size_t, map_size); 2796 *object_handle = user_handle; 2797 return KERN_SUCCESS; 2798 } 2799 2800make_mem_done: 2801 if (user_handle != IP_NULL) { 2802 /* 2803 * Releasing "user_handle" causes the kernel object 2804 * associated with it ("user_entry" here) to also be 2805 * released and freed. 2806 */ 2807 mach_memory_entry_port_release(user_handle); 2808 } 2809 return kr; 2810} 2811 2812kern_return_t 2813_mach_make_memory_entry( 2814 vm_map_t target_map, 2815 memory_object_size_t *size, 2816 memory_object_offset_t offset, 2817 vm_prot_t permission, 2818 ipc_port_t *object_handle, 2819 ipc_port_t parent_entry) 2820{ 2821 memory_object_size_t mo_size; 2822 kern_return_t kr; 2823 2824 mo_size = (memory_object_size_t)*size; 2825 kr = mach_make_memory_entry_64(target_map, &mo_size, 2826 (memory_object_offset_t)offset, permission, object_handle, 2827 parent_entry); 2828 *size = mo_size; 2829 return kr; 2830} 2831 2832kern_return_t 2833mach_make_memory_entry( 2834 vm_map_t target_map, 2835 vm_size_t *size, 2836 vm_offset_t offset, 2837 vm_prot_t permission, 2838 ipc_port_t *object_handle, 2839 ipc_port_t parent_entry) 2840{ 2841 memory_object_size_t mo_size; 2842 kern_return_t kr; 2843 2844 mo_size = (memory_object_size_t)*size; 2845 kr = mach_make_memory_entry_64(target_map, &mo_size, 2846 (memory_object_offset_t)offset, permission, object_handle, 2847 parent_entry); 2848 *size = CAST_DOWN(vm_size_t, mo_size); 2849 return kr; 2850} 2851 2852/* 2853 * task_wire 2854 * 2855 * Set or clear the map's wiring_required flag. This flag, if set, 2856 * will cause all future virtual memory allocation to allocate 2857 * user wired memory. Unwiring pages wired down as a result of 2858 * this routine is done with the vm_wire interface. 2859 */ 2860kern_return_t 2861task_wire( 2862 vm_map_t map, 2863 boolean_t must_wire) 2864{ 2865 if (map == VM_MAP_NULL) 2866 return(KERN_INVALID_ARGUMENT); 2867 2868 if (must_wire) 2869 map->wiring_required = TRUE; 2870 else 2871 map->wiring_required = FALSE; 2872 2873 return(KERN_SUCCESS); 2874} 2875 2876__private_extern__ kern_return_t 2877mach_memory_entry_allocate( 2878 vm_named_entry_t *user_entry_p, 2879 ipc_port_t *user_handle_p) 2880{ 2881 vm_named_entry_t user_entry; 2882 ipc_port_t user_handle; 2883 ipc_port_t previous; 2884 2885 user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry); 2886 if (user_entry == NULL) 2887 return KERN_FAILURE; 2888 2889 named_entry_lock_init(user_entry); 2890 2891 user_handle = ipc_port_alloc_kernel(); 2892 if (user_handle == IP_NULL) { 2893 kfree(user_entry, sizeof *user_entry); 2894 return KERN_FAILURE; 2895 } 2896 ip_lock(user_handle); 2897 2898 /* make a sonce right */ 2899 user_handle->ip_sorights++; 2900 ip_reference(user_handle); 2901 2902 user_handle->ip_destination = IP_NULL; 2903 user_handle->ip_receiver_name = MACH_PORT_NULL; 2904 user_handle->ip_receiver = ipc_space_kernel; 2905 2906 /* make a send right */ 2907 user_handle->ip_mscount++; 2908 user_handle->ip_srights++; 2909 ip_reference(user_handle); 2910 2911 ipc_port_nsrequest(user_handle, 1, user_handle, &previous); 2912 /* nsrequest unlocks user_handle */ 2913 2914 user_entry->backing.pager = NULL; 2915 user_entry->is_sub_map = FALSE; 2916 user_entry->is_pager = FALSE; 2917 user_entry->is_copy = FALSE; 2918 user_entry->internal = FALSE; 2919 user_entry->size = 0; 2920 user_entry->offset = 0; 2921 user_entry->data_offset = 0; 2922 user_entry->protection = VM_PROT_NONE; 2923 user_entry->ref_count = 1; 2924 2925 ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry, 2926 IKOT_NAMED_ENTRY); 2927 2928 *user_entry_p = user_entry; 2929 *user_handle_p = user_handle; 2930 2931 return KERN_SUCCESS; 2932} 2933 2934/* 2935 * mach_memory_object_memory_entry_64 2936 * 2937 * Create a named entry backed by the provided pager. 2938 * 2939 * JMM - we need to hold a reference on the pager - 2940 * and release it when the named entry is destroyed. 2941 */ 2942kern_return_t 2943mach_memory_object_memory_entry_64( 2944 host_t host, 2945 boolean_t internal, 2946 vm_object_offset_t size, 2947 vm_prot_t permission, 2948 memory_object_t pager, 2949 ipc_port_t *entry_handle) 2950{ 2951 unsigned int access; 2952 vm_named_entry_t user_entry; 2953 ipc_port_t user_handle; 2954 2955 if (host == HOST_NULL) 2956 return(KERN_INVALID_HOST); 2957 2958 if (mach_memory_entry_allocate(&user_entry, &user_handle) 2959 != KERN_SUCCESS) { 2960 return KERN_FAILURE; 2961 } 2962 2963 user_entry->backing.pager = pager; 2964 user_entry->size = size; 2965 user_entry->offset = 0; 2966 user_entry->protection = permission & VM_PROT_ALL; 2967 access = GET_MAP_MEM(permission); 2968 SET_MAP_MEM(access, user_entry->protection); 2969 user_entry->internal = internal; 2970 user_entry->is_sub_map = FALSE; 2971 user_entry->is_pager = TRUE; 2972 assert(user_entry->ref_count == 1); 2973 2974 *entry_handle = user_handle; 2975 return KERN_SUCCESS; 2976} 2977 2978kern_return_t 2979mach_memory_object_memory_entry( 2980 host_t host, 2981 boolean_t internal, 2982 vm_size_t size, 2983 vm_prot_t permission, 2984 memory_object_t pager, 2985 ipc_port_t *entry_handle) 2986{ 2987 return mach_memory_object_memory_entry_64( host, internal, 2988 (vm_object_offset_t)size, permission, pager, entry_handle); 2989} 2990 2991 2992kern_return_t 2993mach_memory_entry_purgable_control( 2994 ipc_port_t entry_port, 2995 vm_purgable_t control, 2996 int *state) 2997{ 2998 kern_return_t kr; 2999 vm_named_entry_t mem_entry; 3000 vm_object_t object; 3001 3002 if (entry_port == IP_NULL || 3003 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { 3004 return KERN_INVALID_ARGUMENT; 3005 } 3006 if (control != VM_PURGABLE_SET_STATE && 3007 control != VM_PURGABLE_GET_STATE) 3008 return(KERN_INVALID_ARGUMENT); 3009 3010 if (control == VM_PURGABLE_SET_STATE && 3011 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) || 3012 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) 3013 return(KERN_INVALID_ARGUMENT); 3014 3015 mem_entry = (vm_named_entry_t) entry_port->ip_kobject; 3016 3017 named_entry_lock(mem_entry); 3018 3019 if (mem_entry->is_sub_map || 3020 mem_entry->is_pager || 3021 mem_entry->is_copy) { 3022 named_entry_unlock(mem_entry); 3023 return KERN_INVALID_ARGUMENT; 3024 } 3025 3026 object = mem_entry->backing.object; 3027 if (object == VM_OBJECT_NULL) { 3028 named_entry_unlock(mem_entry); 3029 return KERN_INVALID_ARGUMENT; 3030 } 3031 3032 vm_object_lock(object); 3033 3034 /* check that named entry covers entire object ? */ 3035 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) { 3036 vm_object_unlock(object); 3037 named_entry_unlock(mem_entry); 3038 return KERN_INVALID_ARGUMENT; 3039 } 3040 3041 named_entry_unlock(mem_entry); 3042 3043 kr = vm_object_purgable_control(object, control, state); 3044 3045 vm_object_unlock(object); 3046 3047 return kr; 3048} 3049 3050kern_return_t 3051mach_memory_entry_get_page_counts( 3052 ipc_port_t entry_port, 3053 unsigned int *resident_page_count, 3054 unsigned int *dirty_page_count) 3055{ 3056 kern_return_t kr; 3057 vm_named_entry_t mem_entry; 3058 vm_object_t object; 3059 vm_object_offset_t offset; 3060 vm_object_size_t size; 3061 3062 if (entry_port == IP_NULL || 3063 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { 3064 return KERN_INVALID_ARGUMENT; 3065 } 3066 3067 mem_entry = (vm_named_entry_t) entry_port->ip_kobject; 3068 3069 named_entry_lock(mem_entry); 3070 3071 if (mem_entry->is_sub_map || 3072 mem_entry->is_pager || 3073 mem_entry->is_copy) { 3074 named_entry_unlock(mem_entry); 3075 return KERN_INVALID_ARGUMENT; 3076 } 3077 3078 object = mem_entry->backing.object; 3079 if (object == VM_OBJECT_NULL) { 3080 named_entry_unlock(mem_entry); 3081 return KERN_INVALID_ARGUMENT; 3082 } 3083 3084 vm_object_lock(object); 3085 3086 offset = mem_entry->offset; 3087 size = mem_entry->size; 3088 3089 named_entry_unlock(mem_entry); 3090 3091 kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count); 3092 3093 vm_object_unlock(object); 3094 3095 return kr; 3096} 3097 3098/* 3099 * mach_memory_entry_port_release: 3100 * 3101 * Release a send right on a named entry port. This is the correct 3102 * way to destroy a named entry. When the last right on the port is 3103 * released, ipc_kobject_destroy() will call mach_destroy_memory_entry(). 3104 */ 3105void 3106mach_memory_entry_port_release( 3107 ipc_port_t port) 3108{ 3109 assert(ip_kotype(port) == IKOT_NAMED_ENTRY); 3110 ipc_port_release_send(port); 3111} 3112 3113/* 3114 * mach_destroy_memory_entry: 3115 * 3116 * Drops a reference on a memory entry and destroys the memory entry if 3117 * there are no more references on it. 3118 * NOTE: This routine should not be called to destroy a memory entry from the 3119 * kernel, as it will not release the Mach port associated with the memory 3120 * entry. The proper way to destroy a memory entry in the kernel is to 3121 * call mach_memort_entry_port_release() to release the kernel's send-right on 3122 * the memory entry's port. When the last send right is released, the memory 3123 * entry will be destroyed via ipc_kobject_destroy(). 3124 */ 3125void 3126mach_destroy_memory_entry( 3127 ipc_port_t port) 3128{ 3129 vm_named_entry_t named_entry; 3130#if MACH_ASSERT 3131 assert(ip_kotype(port) == IKOT_NAMED_ENTRY); 3132#endif /* MACH_ASSERT */ 3133 named_entry = (vm_named_entry_t)port->ip_kobject; 3134 3135 named_entry_lock(named_entry); 3136 named_entry->ref_count -= 1; 3137 3138 if(named_entry->ref_count == 0) { 3139 if (named_entry->is_sub_map) { 3140 vm_map_deallocate(named_entry->backing.map); 3141 } else if (named_entry->is_pager) { 3142 /* JMM - need to drop reference on pager in that case */ 3143 } else if (named_entry->is_copy) { 3144 vm_map_copy_discard(named_entry->backing.copy); 3145 } else { 3146 /* release the VM object we've been pointing to */ 3147 vm_object_deallocate(named_entry->backing.object); 3148 } 3149 3150 named_entry_unlock(named_entry); 3151 named_entry_lock_destroy(named_entry); 3152 3153 kfree((void *) port->ip_kobject, 3154 sizeof (struct vm_named_entry)); 3155 } else 3156 named_entry_unlock(named_entry); 3157} 3158 3159/* Allow manipulation of individual page state. This is actually part of */ 3160/* the UPL regimen but takes place on the memory entry rather than on a UPL */ 3161 3162kern_return_t 3163mach_memory_entry_page_op( 3164 ipc_port_t entry_port, 3165 vm_object_offset_t offset, 3166 int ops, 3167 ppnum_t *phys_entry, 3168 int *flags) 3169{ 3170 vm_named_entry_t mem_entry; 3171 vm_object_t object; 3172 kern_return_t kr; 3173 3174 if (entry_port == IP_NULL || 3175 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { 3176 return KERN_INVALID_ARGUMENT; 3177 } 3178 3179 mem_entry = (vm_named_entry_t) entry_port->ip_kobject; 3180 3181 named_entry_lock(mem_entry); 3182 3183 if (mem_entry->is_sub_map || 3184 mem_entry->is_pager || 3185 mem_entry->is_copy) { 3186 named_entry_unlock(mem_entry); 3187 return KERN_INVALID_ARGUMENT; 3188 } 3189 3190 object = mem_entry->backing.object; 3191 if (object == VM_OBJECT_NULL) { 3192 named_entry_unlock(mem_entry); 3193 return KERN_INVALID_ARGUMENT; 3194 } 3195 3196 vm_object_reference(object); 3197 named_entry_unlock(mem_entry); 3198 3199 kr = vm_object_page_op(object, offset, ops, phys_entry, flags); 3200 3201 vm_object_deallocate(object); 3202 3203 return kr; 3204} 3205 3206/* 3207 * mach_memory_entry_range_op offers performance enhancement over 3208 * mach_memory_entry_page_op for page_op functions which do not require page 3209 * level state to be returned from the call. Page_op was created to provide 3210 * a low-cost alternative to page manipulation via UPLs when only a single 3211 * page was involved. The range_op call establishes the ability in the _op 3212 * family of functions to work on multiple pages where the lack of page level 3213 * state handling allows the caller to avoid the overhead of the upl structures. 3214 */ 3215 3216kern_return_t 3217mach_memory_entry_range_op( 3218 ipc_port_t entry_port, 3219 vm_object_offset_t offset_beg, 3220 vm_object_offset_t offset_end, 3221 int ops, 3222 int *range) 3223{ 3224 vm_named_entry_t mem_entry; 3225 vm_object_t object; 3226 kern_return_t kr; 3227 3228 if (entry_port == IP_NULL || 3229 ip_kotype(entry_port) != IKOT_NAMED_ENTRY) { 3230 return KERN_INVALID_ARGUMENT; 3231 } 3232 3233 mem_entry = (vm_named_entry_t) entry_port->ip_kobject; 3234 3235 named_entry_lock(mem_entry); 3236 3237 if (mem_entry->is_sub_map || 3238 mem_entry->is_pager || 3239 mem_entry->is_copy) { 3240 named_entry_unlock(mem_entry); 3241 return KERN_INVALID_ARGUMENT; 3242 } 3243 3244 object = mem_entry->backing.object; 3245 if (object == VM_OBJECT_NULL) { 3246 named_entry_unlock(mem_entry); 3247 return KERN_INVALID_ARGUMENT; 3248 } 3249 3250 vm_object_reference(object); 3251 named_entry_unlock(mem_entry); 3252 3253 kr = vm_object_range_op(object, 3254 offset_beg, 3255 offset_end, 3256 ops, 3257 (uint32_t *) range); 3258 3259 vm_object_deallocate(object); 3260 3261 return kr; 3262} 3263 3264 3265kern_return_t 3266set_dp_control_port( 3267 host_priv_t host_priv, 3268 ipc_port_t control_port) 3269{ 3270 if (host_priv == HOST_PRIV_NULL) 3271 return (KERN_INVALID_HOST); 3272 3273 if (IP_VALID(dynamic_pager_control_port)) 3274 ipc_port_release_send(dynamic_pager_control_port); 3275 3276 dynamic_pager_control_port = control_port; 3277 return KERN_SUCCESS; 3278} 3279 3280kern_return_t 3281get_dp_control_port( 3282 host_priv_t host_priv, 3283 ipc_port_t *control_port) 3284{ 3285 if (host_priv == HOST_PRIV_NULL) 3286 return (KERN_INVALID_HOST); 3287 3288 *control_port = ipc_port_copy_send(dynamic_pager_control_port); 3289 return KERN_SUCCESS; 3290 3291} 3292 3293/* ******* Temporary Internal calls to UPL for BSD ***** */ 3294 3295extern int kernel_upl_map( 3296 vm_map_t map, 3297 upl_t upl, 3298 vm_offset_t *dst_addr); 3299 3300extern int kernel_upl_unmap( 3301 vm_map_t map, 3302 upl_t upl); 3303 3304extern int kernel_upl_commit( 3305 upl_t upl, 3306 upl_page_info_t *pl, 3307 mach_msg_type_number_t count); 3308 3309extern int kernel_upl_commit_range( 3310 upl_t upl, 3311 upl_offset_t offset, 3312 upl_size_t size, 3313 int flags, 3314 upl_page_info_array_t pl, 3315 mach_msg_type_number_t count); 3316 3317extern int kernel_upl_abort( 3318 upl_t upl, 3319 int abort_type); 3320 3321extern int kernel_upl_abort_range( 3322 upl_t upl, 3323 upl_offset_t offset, 3324 upl_size_t size, 3325 int abort_flags); 3326 3327 3328kern_return_t 3329kernel_upl_map( 3330 vm_map_t map, 3331 upl_t upl, 3332 vm_offset_t *dst_addr) 3333{ 3334 return vm_upl_map(map, upl, dst_addr); 3335} 3336 3337 3338kern_return_t 3339kernel_upl_unmap( 3340 vm_map_t map, 3341 upl_t upl) 3342{ 3343 return vm_upl_unmap(map, upl); 3344} 3345 3346kern_return_t 3347kernel_upl_commit( 3348 upl_t upl, 3349 upl_page_info_t *pl, 3350 mach_msg_type_number_t count) 3351{ 3352 kern_return_t kr; 3353 3354 kr = upl_commit(upl, pl, count); 3355 upl_deallocate(upl); 3356 return kr; 3357} 3358 3359 3360kern_return_t 3361kernel_upl_commit_range( 3362 upl_t upl, 3363 upl_offset_t offset, 3364 upl_size_t size, 3365 int flags, 3366 upl_page_info_array_t pl, 3367 mach_msg_type_number_t count) 3368{ 3369 boolean_t finished = FALSE; 3370 kern_return_t kr; 3371 3372 if (flags & UPL_COMMIT_FREE_ON_EMPTY) 3373 flags |= UPL_COMMIT_NOTIFY_EMPTY; 3374 3375 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) { 3376 return KERN_INVALID_ARGUMENT; 3377 } 3378 3379 kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished); 3380 3381 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished) 3382 upl_deallocate(upl); 3383 3384 return kr; 3385} 3386 3387kern_return_t 3388kernel_upl_abort_range( 3389 upl_t upl, 3390 upl_offset_t offset, 3391 upl_size_t size, 3392 int abort_flags) 3393{ 3394 kern_return_t kr; 3395 boolean_t finished = FALSE; 3396 3397 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY) 3398 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY; 3399 3400 kr = upl_abort_range(upl, offset, size, abort_flags, &finished); 3401 3402 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished) 3403 upl_deallocate(upl); 3404 3405 return kr; 3406} 3407 3408kern_return_t 3409kernel_upl_abort( 3410 upl_t upl, 3411 int abort_type) 3412{ 3413 kern_return_t kr; 3414 3415 kr = upl_abort(upl, abort_type); 3416 upl_deallocate(upl); 3417 return kr; 3418} 3419 3420/* 3421 * Now a kernel-private interface (for BootCache 3422 * use only). Need a cleaner way to create an 3423 * empty vm_map() and return a handle to it. 3424 */ 3425 3426kern_return_t 3427vm_region_object_create( 3428 __unused vm_map_t target_map, 3429 vm_size_t size, 3430 ipc_port_t *object_handle) 3431{ 3432 vm_named_entry_t user_entry; 3433 ipc_port_t user_handle; 3434 3435 vm_map_t new_map; 3436 3437 if (mach_memory_entry_allocate(&user_entry, &user_handle) 3438 != KERN_SUCCESS) { 3439 return KERN_FAILURE; 3440 } 3441 3442 /* Create a named object based on a submap of specified size */ 3443 3444 new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS, 3445 vm_map_round_page(size, 3446 VM_MAP_PAGE_MASK(target_map)), 3447 TRUE); 3448 vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map)); 3449 3450 user_entry->backing.map = new_map; 3451 user_entry->internal = TRUE; 3452 user_entry->is_sub_map = TRUE; 3453 user_entry->offset = 0; 3454 user_entry->protection = VM_PROT_ALL; 3455 user_entry->size = size; 3456 assert(user_entry->ref_count == 1); 3457 3458 *object_handle = user_handle; 3459 return KERN_SUCCESS; 3460 3461} 3462 3463ppnum_t vm_map_get_phys_page( /* forward */ 3464 vm_map_t map, 3465 vm_offset_t offset); 3466 3467ppnum_t 3468vm_map_get_phys_page( 3469 vm_map_t map, 3470 vm_offset_t addr) 3471{ 3472 vm_object_offset_t offset; 3473 vm_object_t object; 3474 vm_map_offset_t map_offset; 3475 vm_map_entry_t entry; 3476 ppnum_t phys_page = 0; 3477 3478 map_offset = vm_map_trunc_page(addr, PAGE_MASK); 3479 3480 vm_map_lock(map); 3481 while (vm_map_lookup_entry(map, map_offset, &entry)) { 3482 3483 if (entry->object.vm_object == VM_OBJECT_NULL) { 3484 vm_map_unlock(map); 3485 return (ppnum_t) 0; 3486 } 3487 if (entry->is_sub_map) { 3488 vm_map_t old_map; 3489 vm_map_lock(entry->object.sub_map); 3490 old_map = map; 3491 map = entry->object.sub_map; 3492 map_offset = entry->offset + (map_offset - entry->vme_start); 3493 vm_map_unlock(old_map); 3494 continue; 3495 } 3496 if (entry->object.vm_object->phys_contiguous) { 3497 /* These are not standard pageable memory mappings */ 3498 /* If they are not present in the object they will */ 3499 /* have to be picked up from the pager through the */ 3500 /* fault mechanism. */ 3501 if(entry->object.vm_object->vo_shadow_offset == 0) { 3502 /* need to call vm_fault */ 3503 vm_map_unlock(map); 3504 vm_fault(map, map_offset, VM_PROT_NONE, 3505 FALSE, THREAD_UNINT, NULL, 0); 3506 vm_map_lock(map); 3507 continue; 3508 } 3509 offset = entry->offset + (map_offset - entry->vme_start); 3510 phys_page = (ppnum_t) 3511 ((entry->object.vm_object->vo_shadow_offset 3512 + offset) >> PAGE_SHIFT); 3513 break; 3514 3515 } 3516 offset = entry->offset + (map_offset - entry->vme_start); 3517 object = entry->object.vm_object; 3518 vm_object_lock(object); 3519 while (TRUE) { 3520 vm_page_t dst_page = vm_page_lookup(object,offset); 3521 if(dst_page == VM_PAGE_NULL) { 3522 if(object->shadow) { 3523 vm_object_t old_object; 3524 vm_object_lock(object->shadow); 3525 old_object = object; 3526 offset = offset + object->vo_shadow_offset; 3527 object = object->shadow; 3528 vm_object_unlock(old_object); 3529 } else { 3530 vm_object_unlock(object); 3531 break; 3532 } 3533 } else { 3534 phys_page = (ppnum_t)(dst_page->phys_page); 3535 vm_object_unlock(object); 3536 break; 3537 } 3538 } 3539 break; 3540 3541 } 3542 3543 vm_map_unlock(map); 3544 return phys_page; 3545} 3546 3547 3548 3549kern_return_t kernel_object_iopl_request( /* forward */ 3550 vm_named_entry_t named_entry, 3551 memory_object_offset_t offset, 3552 upl_size_t *upl_size, 3553 upl_t *upl_ptr, 3554 upl_page_info_array_t user_page_list, 3555 unsigned int *page_list_count, 3556 int *flags); 3557 3558kern_return_t 3559kernel_object_iopl_request( 3560 vm_named_entry_t named_entry, 3561 memory_object_offset_t offset, 3562 upl_size_t *upl_size, 3563 upl_t *upl_ptr, 3564 upl_page_info_array_t user_page_list, 3565 unsigned int *page_list_count, 3566 int *flags) 3567{ 3568 vm_object_t object; 3569 kern_return_t ret; 3570 3571 int caller_flags; 3572 3573 caller_flags = *flags; 3574 3575 if (caller_flags & ~UPL_VALID_FLAGS) { 3576 /* 3577 * For forward compatibility's sake, 3578 * reject any unknown flag. 3579 */ 3580 return KERN_INVALID_VALUE; 3581 } 3582 3583 /* a few checks to make sure user is obeying rules */ 3584 if(*upl_size == 0) { 3585 if(offset >= named_entry->size) 3586 return(KERN_INVALID_RIGHT); 3587 *upl_size = (upl_size_t) (named_entry->size - offset); 3588 if (*upl_size != named_entry->size - offset) 3589 return KERN_INVALID_ARGUMENT; 3590 } 3591 if(caller_flags & UPL_COPYOUT_FROM) { 3592 if((named_entry->protection & VM_PROT_READ) 3593 != VM_PROT_READ) { 3594 return(KERN_INVALID_RIGHT); 3595 } 3596 } else { 3597 if((named_entry->protection & 3598 (VM_PROT_READ | VM_PROT_WRITE)) 3599 != (VM_PROT_READ | VM_PROT_WRITE)) { 3600 return(KERN_INVALID_RIGHT); 3601 } 3602 } 3603 if(named_entry->size < (offset + *upl_size)) 3604 return(KERN_INVALID_ARGUMENT); 3605 3606 /* the callers parameter offset is defined to be the */ 3607 /* offset from beginning of named entry offset in object */ 3608 offset = offset + named_entry->offset; 3609 3610 if (named_entry->is_sub_map || 3611 named_entry->is_copy) 3612 return KERN_INVALID_ARGUMENT; 3613 3614 named_entry_lock(named_entry); 3615 3616 if (named_entry->is_pager) { 3617 object = vm_object_enter(named_entry->backing.pager, 3618 named_entry->offset + named_entry->size, 3619 named_entry->internal, 3620 FALSE, 3621 FALSE); 3622 if (object == VM_OBJECT_NULL) { 3623 named_entry_unlock(named_entry); 3624 return(KERN_INVALID_OBJECT); 3625 } 3626 3627 /* JMM - drop reference on the pager here? */ 3628 3629 /* create an extra reference for the object */ 3630 vm_object_lock(object); 3631 vm_object_reference_locked(object); 3632 named_entry->backing.object = object; 3633 named_entry->is_pager = FALSE; 3634 named_entry_unlock(named_entry); 3635 3636 /* wait for object (if any) to be ready */ 3637 if (!named_entry->internal) { 3638 while (!object->pager_ready) { 3639 vm_object_wait(object, 3640 VM_OBJECT_EVENT_PAGER_READY, 3641 THREAD_UNINT); 3642 vm_object_lock(object); 3643 } 3644 } 3645 vm_object_unlock(object); 3646 3647 } else { 3648 /* This is the case where we are going to operate */ 3649 /* an an already known object. If the object is */ 3650 /* not ready it is internal. An external */ 3651 /* object cannot be mapped until it is ready */ 3652 /* we can therefore avoid the ready check */ 3653 /* in this case. */ 3654 object = named_entry->backing.object; 3655 vm_object_reference(object); 3656 named_entry_unlock(named_entry); 3657 } 3658 3659 if (!object->private) { 3660 if (*upl_size > MAX_UPL_TRANSFER_BYTES) 3661 *upl_size = MAX_UPL_TRANSFER_BYTES; 3662 if (object->phys_contiguous) { 3663 *flags = UPL_PHYS_CONTIG; 3664 } else { 3665 *flags = 0; 3666 } 3667 } else { 3668 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG; 3669 } 3670 3671 ret = vm_object_iopl_request(object, 3672 offset, 3673 *upl_size, 3674 upl_ptr, 3675 user_page_list, 3676 page_list_count, 3677 caller_flags); 3678 vm_object_deallocate(object); 3679 return ret; 3680} 3681