1/* 2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58/* 59 * File: vm/memory_object.c 60 * Author: Michael Wayne Young 61 * 62 * External memory management interface control functions. 63 */ 64 65/* 66 * Interface dependencies: 67 */ 68 69#include <mach/std_types.h> /* For pointer_t */ 70#include <mach/mach_types.h> 71 72#include <mach/mig.h> 73#include <mach/kern_return.h> 74#include <mach/memory_object.h> 75#include <mach/memory_object_default.h> 76#include <mach/memory_object_control_server.h> 77#include <mach/host_priv_server.h> 78#include <mach/boolean.h> 79#include <mach/vm_prot.h> 80#include <mach/message.h> 81 82/* 83 * Implementation dependencies: 84 */ 85#include <string.h> /* For memcpy() */ 86 87#include <kern/xpr.h> 88#include <kern/host.h> 89#include <kern/thread.h> /* For current_thread() */ 90#include <kern/ipc_mig.h> 91#include <kern/misc_protos.h> 92 93#include <vm/vm_object.h> 94#include <vm/vm_fault.h> 95#include <vm/memory_object.h> 96#include <vm/vm_page.h> 97#include <vm/vm_pageout.h> 98#include <vm/pmap.h> /* For pmap_clear_modify */ 99#include <vm/vm_kern.h> /* For kernel_map, vm_move */ 100#include <vm/vm_map.h> /* For vm_map_pageable */ 101#include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */ 102#include <vm/vm_shared_region.h> 103 104#include <vm/vm_external.h> 105 106#include <vm/vm_protos.h> 107 108 109memory_object_default_t memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; 110decl_lck_mtx_data(, memory_manager_default_lock) 111 112 113/* 114 * Routine: memory_object_should_return_page 115 * 116 * Description: 117 * Determine whether the given page should be returned, 118 * based on the page's state and on the given return policy. 119 * 120 * We should return the page if one of the following is true: 121 * 122 * 1. Page is dirty and should_return is not RETURN_NONE. 123 * 2. Page is precious and should_return is RETURN_ALL. 124 * 3. Should_return is RETURN_ANYTHING. 125 * 126 * As a side effect, m->dirty will be made consistent 127 * with pmap_is_modified(m), if should_return is not 128 * MEMORY_OBJECT_RETURN_NONE. 129 */ 130 131#define memory_object_should_return_page(m, should_return) \ 132 (should_return != MEMORY_OBJECT_RETURN_NONE && \ 133 (((m)->dirty || ((m)->dirty = pmap_is_modified((m)->phys_page))) || \ 134 ((m)->precious && (should_return) == MEMORY_OBJECT_RETURN_ALL) || \ 135 (should_return) == MEMORY_OBJECT_RETURN_ANYTHING)) 136 137typedef int memory_object_lock_result_t; 138 139#define MEMORY_OBJECT_LOCK_RESULT_DONE 0 140#define MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK 1 141#define MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN 2 142#define MEMORY_OBJECT_LOCK_RESULT_MUST_FREE 3 143 144memory_object_lock_result_t memory_object_lock_page( 145 vm_page_t m, 146 memory_object_return_t should_return, 147 boolean_t should_flush, 148 vm_prot_t prot); 149 150/* 151 * Routine: memory_object_lock_page 152 * 153 * Description: 154 * Perform the appropriate lock operations on the 155 * given page. See the description of 156 * "memory_object_lock_request" for the meanings 157 * of the arguments. 158 * 159 * Returns an indication that the operation 160 * completed, blocked, or that the page must 161 * be cleaned. 162 */ 163memory_object_lock_result_t 164memory_object_lock_page( 165 vm_page_t m, 166 memory_object_return_t should_return, 167 boolean_t should_flush, 168 vm_prot_t prot) 169{ 170 XPR(XPR_MEMORY_OBJECT, 171 "m_o_lock_page, page 0x%X rtn %d flush %d prot %d\n", 172 m, should_return, should_flush, prot, 0); 173 174 175 if (m->busy || m->cleaning) 176 return (MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK); 177 178 if (m->laundry) 179 vm_pageout_steal_laundry(m, FALSE); 180 181 /* 182 * Don't worry about pages for which the kernel 183 * does not have any data. 184 */ 185 if (m->absent || m->error || m->restart) { 186 if (m->error && should_flush && !VM_PAGE_WIRED(m)) { 187 /* 188 * dump the page, pager wants us to 189 * clean it up and there is no 190 * relevant data to return 191 */ 192 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE); 193 } 194 return (MEMORY_OBJECT_LOCK_RESULT_DONE); 195 } 196 assert(!m->fictitious); 197 198 if (VM_PAGE_WIRED(m)) { 199 /* 200 * The page is wired... just clean or return the page if needed. 201 * Wired pages don't get flushed or disconnected from the pmap. 202 */ 203 if (memory_object_should_return_page(m, should_return)) 204 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); 205 206 return (MEMORY_OBJECT_LOCK_RESULT_DONE); 207 } 208 209 if (should_flush) { 210 /* 211 * must do the pmap_disconnect before determining the 212 * need to return the page... otherwise it's possible 213 * for the page to go from the clean to the dirty state 214 * after we've made our decision 215 */ 216 if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED) { 217 SET_PAGE_DIRTY(m, FALSE); 218 } 219 } else { 220 /* 221 * If we are decreasing permission, do it now; 222 * let the fault handler take care of increases 223 * (pmap_page_protect may not increase protection). 224 */ 225 if (prot != VM_PROT_NO_CHANGE) 226 pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot); 227 } 228 /* 229 * Handle returning dirty or precious pages 230 */ 231 if (memory_object_should_return_page(m, should_return)) { 232 /* 233 * we use to do a pmap_disconnect here in support 234 * of memory_object_lock_request, but that routine 235 * no longer requires this... in any event, in 236 * our world, it would turn into a big noop since 237 * we don't lock the page in any way and as soon 238 * as we drop the object lock, the page can be 239 * faulted back into an address space 240 * 241 * if (!should_flush) 242 * pmap_disconnect(m->phys_page); 243 */ 244 return (MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN); 245 } 246 247 /* 248 * Handle flushing clean pages 249 */ 250 if (should_flush) 251 return (MEMORY_OBJECT_LOCK_RESULT_MUST_FREE); 252 253 /* 254 * we use to deactivate clean pages at this point, 255 * but we do not believe that an msync should change 256 * the 'age' of a page in the cache... here is the 257 * original comment and code concerning this... 258 * 259 * XXX Make clean but not flush a paging hint, 260 * and deactivate the pages. This is a hack 261 * because it overloads flush/clean with 262 * implementation-dependent meaning. This only 263 * happens to pages that are already clean. 264 * 265 * if (vm_page_deactivate_hint && (should_return != MEMORY_OBJECT_RETURN_NONE)) 266 * return (MEMORY_OBJECT_LOCK_RESULT_MUST_DEACTIVATE); 267 */ 268 269 return (MEMORY_OBJECT_LOCK_RESULT_DONE); 270} 271 272 273 274/* 275 * Routine: memory_object_lock_request [user interface] 276 * 277 * Description: 278 * Control use of the data associated with the given 279 * memory object. For each page in the given range, 280 * perform the following operations, in order: 281 * 1) restrict access to the page (disallow 282 * forms specified by "prot"); 283 * 2) return data to the manager (if "should_return" 284 * is RETURN_DIRTY and the page is dirty, or 285 * "should_return" is RETURN_ALL and the page 286 * is either dirty or precious); and, 287 * 3) flush the cached copy (if "should_flush" 288 * is asserted). 289 * The set of pages is defined by a starting offset 290 * ("offset") and size ("size"). Only pages with the 291 * same page alignment as the starting offset are 292 * considered. 293 * 294 * A single acknowledgement is sent (to the "reply_to" 295 * port) when these actions are complete. If successful, 296 * the naked send right for reply_to is consumed. 297 */ 298 299kern_return_t 300memory_object_lock_request( 301 memory_object_control_t control, 302 memory_object_offset_t offset, 303 memory_object_size_t size, 304 memory_object_offset_t * resid_offset, 305 int * io_errno, 306 memory_object_return_t should_return, 307 int flags, 308 vm_prot_t prot) 309{ 310 vm_object_t object; 311 312 /* 313 * Check for bogus arguments. 314 */ 315 object = memory_object_control_to_vm_object(control); 316 if (object == VM_OBJECT_NULL) 317 return (KERN_INVALID_ARGUMENT); 318 319 if ((prot & ~VM_PROT_ALL) != 0 && prot != VM_PROT_NO_CHANGE) 320 return (KERN_INVALID_ARGUMENT); 321 322 size = round_page_64(size); 323 324 /* 325 * Lock the object, and acquire a paging reference to 326 * prevent the memory_object reference from being released. 327 */ 328 vm_object_lock(object); 329 vm_object_paging_begin(object); 330 331 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) { 332 if ((should_return != MEMORY_OBJECT_RETURN_NONE) || offset || object->copy) { 333 flags &= ~MEMORY_OBJECT_DATA_FLUSH_ALL; 334 flags |= MEMORY_OBJECT_DATA_FLUSH; 335 } 336 } 337 offset -= object->paging_offset; 338 339 if (flags & MEMORY_OBJECT_DATA_FLUSH_ALL) 340 vm_object_reap_pages(object, REAP_DATA_FLUSH); 341 else 342 (void)vm_object_update(object, offset, size, resid_offset, 343 io_errno, should_return, flags, prot); 344 345 vm_object_paging_end(object); 346 vm_object_unlock(object); 347 348 return (KERN_SUCCESS); 349} 350 351/* 352 * memory_object_release_name: [interface] 353 * 354 * Enforces name semantic on memory_object reference count decrement 355 * This routine should not be called unless the caller holds a name 356 * reference gained through the memory_object_named_create or the 357 * memory_object_rename call. 358 * If the TERMINATE_IDLE flag is set, the call will return if the 359 * reference count is not 1. i.e. idle with the only remaining reference 360 * being the name. 361 * If the decision is made to proceed the name field flag is set to 362 * false and the reference count is decremented. If the RESPECT_CACHE 363 * flag is set and the reference count has gone to zero, the 364 * memory_object is checked to see if it is cacheable otherwise when 365 * the reference count is zero, it is simply terminated. 366 */ 367 368kern_return_t 369memory_object_release_name( 370 memory_object_control_t control, 371 int flags) 372{ 373 vm_object_t object; 374 375 object = memory_object_control_to_vm_object(control); 376 if (object == VM_OBJECT_NULL) 377 return (KERN_INVALID_ARGUMENT); 378 379 return vm_object_release_name(object, flags); 380} 381 382 383 384/* 385 * Routine: memory_object_destroy [user interface] 386 * Purpose: 387 * Shut down a memory object, despite the 388 * presence of address map (or other) references 389 * to the vm_object. 390 */ 391kern_return_t 392memory_object_destroy( 393 memory_object_control_t control, 394 kern_return_t reason) 395{ 396 vm_object_t object; 397 398 object = memory_object_control_to_vm_object(control); 399 if (object == VM_OBJECT_NULL) 400 return (KERN_INVALID_ARGUMENT); 401 402 return (vm_object_destroy(object, reason)); 403} 404 405/* 406 * Routine: vm_object_sync 407 * 408 * Kernel internal function to synch out pages in a given 409 * range within an object to its memory manager. Much the 410 * same as memory_object_lock_request but page protection 411 * is not changed. 412 * 413 * If the should_flush and should_return flags are true pages 414 * are flushed, that is dirty & precious pages are written to 415 * the memory manager and then discarded. If should_return 416 * is false, only precious pages are returned to the memory 417 * manager. 418 * 419 * If should flush is false and should_return true, the memory 420 * manager's copy of the pages is updated. If should_return 421 * is also false, only the precious pages are updated. This 422 * last option is of limited utility. 423 * 424 * Returns: 425 * FALSE if no pages were returned to the pager 426 * TRUE otherwise. 427 */ 428 429boolean_t 430vm_object_sync( 431 vm_object_t object, 432 vm_object_offset_t offset, 433 vm_object_size_t size, 434 boolean_t should_flush, 435 boolean_t should_return, 436 boolean_t should_iosync) 437{ 438 boolean_t rv; 439 int flags; 440 441 XPR(XPR_VM_OBJECT, 442 "vm_o_sync, object 0x%X, offset 0x%X size 0x%x flush %d rtn %d\n", 443 object, offset, size, should_flush, should_return); 444 445 /* 446 * Lock the object, and acquire a paging reference to 447 * prevent the memory_object and control ports from 448 * being destroyed. 449 */ 450 vm_object_lock(object); 451 vm_object_paging_begin(object); 452 453 if (should_flush) { 454 flags = MEMORY_OBJECT_DATA_FLUSH; 455 /* 456 * This flush is from an msync(), not a truncate(), so the 457 * contents of the file are not affected. 458 * MEMORY_OBECT_DATA_NO_CHANGE lets vm_object_update() know 459 * that the data is not changed and that there's no need to 460 * push the old contents to a copy object. 461 */ 462 flags |= MEMORY_OBJECT_DATA_NO_CHANGE; 463 } else 464 flags = 0; 465 466 if (should_iosync) 467 flags |= MEMORY_OBJECT_IO_SYNC; 468 469 rv = vm_object_update(object, offset, (vm_object_size_t)size, NULL, NULL, 470 (should_return) ? 471 MEMORY_OBJECT_RETURN_ALL : 472 MEMORY_OBJECT_RETURN_NONE, 473 flags, 474 VM_PROT_NO_CHANGE); 475 476 477 vm_object_paging_end(object); 478 vm_object_unlock(object); 479 return rv; 480} 481 482 483 484#define LIST_REQ_PAGEOUT_PAGES(object, data_cnt, po, ro, ioerr, iosync) \ 485MACRO_BEGIN \ 486 \ 487 int upl_flags; \ 488 memory_object_t pager; \ 489 \ 490 if (object->object_slid) { \ 491 panic("Objects with slid pages not allowed\n"); \ 492 } \ 493 \ 494 if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) { \ 495 vm_object_paging_begin(object); \ 496 vm_object_unlock(object); \ 497 \ 498 if (iosync) \ 499 upl_flags = UPL_MSYNC | UPL_IOSYNC; \ 500 else \ 501 upl_flags = UPL_MSYNC; \ 502 \ 503 (void) memory_object_data_return(pager, \ 504 po, \ 505 (memory_object_cluster_size_t)data_cnt, \ 506 ro, \ 507 ioerr, \ 508 FALSE, \ 509 FALSE, \ 510 upl_flags); \ 511 \ 512 vm_object_lock(object); \ 513 vm_object_paging_end(object); \ 514 } \ 515MACRO_END 516 517 518 519static int 520vm_object_update_extent( 521 vm_object_t object, 522 vm_object_offset_t offset, 523 vm_object_offset_t offset_end, 524 vm_object_offset_t *offset_resid, 525 int *io_errno, 526 boolean_t should_flush, 527 memory_object_return_t should_return, 528 boolean_t should_iosync, 529 vm_prot_t prot) 530{ 531 vm_page_t m; 532 int retval = 0; 533 vm_object_offset_t paging_offset = 0; 534 vm_object_offset_t next_offset = offset; 535 memory_object_lock_result_t page_lock_result; 536 memory_object_cluster_size_t data_cnt = 0; 537 struct vm_page_delayed_work dw_array[DEFAULT_DELAYED_WORK_LIMIT]; 538 struct vm_page_delayed_work *dwp; 539 int dw_count; 540 int dw_limit; 541 542 dwp = &dw_array[0]; 543 dw_count = 0; 544 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT); 545 546 for (; 547 offset < offset_end && object->resident_page_count; 548 offset += PAGE_SIZE_64) { 549 550 /* 551 * Limit the number of pages to be cleaned at once to a contiguous 552 * run, or at most MAX_UPL_TRANSFER_BYTES 553 */ 554 if (data_cnt) { 555 if ((data_cnt >= MAX_UPL_TRANSFER_BYTES) || (next_offset != offset)) { 556 557 if (dw_count) { 558 vm_page_do_delayed_work(object, &dw_array[0], dw_count); 559 dwp = &dw_array[0]; 560 dw_count = 0; 561 } 562 LIST_REQ_PAGEOUT_PAGES(object, data_cnt, 563 paging_offset, offset_resid, io_errno, should_iosync); 564 data_cnt = 0; 565 } 566 } 567 while ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { 568 569 dwp->dw_mask = 0; 570 571 page_lock_result = memory_object_lock_page(m, should_return, should_flush, prot); 572 573 if (data_cnt && page_lock_result != MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN) { 574 /* 575 * End of a run of dirty/precious pages. 576 */ 577 if (dw_count) { 578 vm_page_do_delayed_work(object, &dw_array[0], dw_count); 579 dwp = &dw_array[0]; 580 dw_count = 0; 581 } 582 LIST_REQ_PAGEOUT_PAGES(object, data_cnt, 583 paging_offset, offset_resid, io_errno, should_iosync); 584 /* 585 * LIST_REQ_PAGEOUT_PAGES will drop the object lock which will 586 * allow the state of page 'm' to change... we need to re-lookup 587 * the current offset 588 */ 589 data_cnt = 0; 590 continue; 591 } 592 593 switch (page_lock_result) { 594 595 case MEMORY_OBJECT_LOCK_RESULT_DONE: 596 break; 597 598 case MEMORY_OBJECT_LOCK_RESULT_MUST_FREE: 599 dwp->dw_mask |= DW_vm_page_free; 600 break; 601 602 case MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK: 603 PAGE_SLEEP(object, m, THREAD_UNINT); 604 continue; 605 606 case MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN: 607 if (data_cnt == 0) 608 paging_offset = offset; 609 610 data_cnt += PAGE_SIZE; 611 next_offset = offset + PAGE_SIZE_64; 612 613 /* 614 * wired pages shouldn't be flushed and 615 * since they aren't on any queue, 616 * no need to remove them 617 */ 618 if (!VM_PAGE_WIRED(m)) { 619 620 if (should_flush) { 621 /* 622 * add additional state for the flush 623 */ 624 m->pageout = TRUE; 625 } 626 /* 627 * we use to remove the page from the queues at this 628 * point, but we do not believe that an msync 629 * should cause the 'age' of a page to be changed 630 * 631 * else 632 * dwp->dw_mask |= DW_VM_PAGE_QUEUES_REMOVE; 633 */ 634 } 635 retval = 1; 636 break; 637 } 638 if (dwp->dw_mask) { 639 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count); 640 641 if (dw_count >= dw_limit) { 642 vm_page_do_delayed_work(object, &dw_array[0], dw_count); 643 dwp = &dw_array[0]; 644 dw_count = 0; 645 } 646 } 647 break; 648 } 649 } 650 /* 651 * We have completed the scan for applicable pages. 652 * Clean any pages that have been saved. 653 */ 654 if (dw_count) 655 vm_page_do_delayed_work(object, &dw_array[0], dw_count); 656 657 if (data_cnt) { 658 LIST_REQ_PAGEOUT_PAGES(object, data_cnt, 659 paging_offset, offset_resid, io_errno, should_iosync); 660 } 661 return (retval); 662} 663 664 665 666/* 667 * Routine: vm_object_update 668 * Description: 669 * Work function for m_o_lock_request(), vm_o_sync(). 670 * 671 * Called with object locked and paging ref taken. 672 */ 673kern_return_t 674vm_object_update( 675 vm_object_t object, 676 vm_object_offset_t offset, 677 vm_object_size_t size, 678 vm_object_offset_t *resid_offset, 679 int *io_errno, 680 memory_object_return_t should_return, 681 int flags, 682 vm_prot_t protection) 683{ 684 vm_object_t copy_object = VM_OBJECT_NULL; 685 boolean_t data_returned = FALSE; 686 boolean_t update_cow; 687 boolean_t should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE; 688 boolean_t should_iosync = (flags & MEMORY_OBJECT_IO_SYNC) ? TRUE : FALSE; 689 vm_fault_return_t result; 690 int num_of_extents; 691 int n; 692#define MAX_EXTENTS 8 693#define EXTENT_SIZE (1024 * 1024 * 256) 694#define RESIDENT_LIMIT (1024 * 32) 695 struct extent { 696 vm_object_offset_t e_base; 697 vm_object_offset_t e_min; 698 vm_object_offset_t e_max; 699 } extents[MAX_EXTENTS]; 700 701 /* 702 * To avoid blocking while scanning for pages, save 703 * dirty pages to be cleaned all at once. 704 * 705 * XXXO A similar strategy could be used to limit the 706 * number of times that a scan must be restarted for 707 * other reasons. Those pages that would require blocking 708 * could be temporarily collected in another list, or 709 * their offsets could be recorded in a small array. 710 */ 711 712 /* 713 * XXX NOTE: May want to consider converting this to a page list 714 * XXX vm_map_copy interface. Need to understand object 715 * XXX coalescing implications before doing so. 716 */ 717 718 update_cow = ((flags & MEMORY_OBJECT_DATA_FLUSH) 719 && (!(flags & MEMORY_OBJECT_DATA_NO_CHANGE) && 720 !(flags & MEMORY_OBJECT_DATA_PURGE))) 721 || (flags & MEMORY_OBJECT_COPY_SYNC); 722 723 if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) { 724 int collisions = 0; 725 726 while ((copy_object = object->copy) != VM_OBJECT_NULL) { 727 /* 728 * need to do a try here since we're swimming upstream 729 * against the normal lock ordering... however, we need 730 * to hold the object stable until we gain control of the 731 * copy object so we have to be careful how we approach this 732 */ 733 if (vm_object_lock_try(copy_object)) { 734 /* 735 * we 'won' the lock on the copy object... 736 * no need to hold the object lock any longer... 737 * take a real reference on the copy object because 738 * we're going to call vm_fault_page on it which may 739 * under certain conditions drop the lock and the paging 740 * reference we're about to take... the reference 741 * will keep the copy object from going away if that happens 742 */ 743 vm_object_unlock(object); 744 vm_object_reference_locked(copy_object); 745 break; 746 } 747 vm_object_unlock(object); 748 749 collisions++; 750 mutex_pause(collisions); 751 752 vm_object_lock(object); 753 } 754 } 755 if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) { 756 vm_map_size_t i; 757 vm_map_size_t copy_size; 758 vm_map_offset_t copy_offset; 759 vm_prot_t prot; 760 vm_page_t page; 761 vm_page_t top_page; 762 kern_return_t error = 0; 763 struct vm_object_fault_info fault_info; 764 765 if (copy_object != VM_OBJECT_NULL) { 766 /* 767 * translate offset with respect to shadow's offset 768 */ 769 copy_offset = (offset >= copy_object->vo_shadow_offset) ? 770 (vm_map_offset_t)(offset - copy_object->vo_shadow_offset) : 771 (vm_map_offset_t) 0; 772 773 if (copy_offset > copy_object->vo_size) 774 copy_offset = copy_object->vo_size; 775 776 /* 777 * clip size with respect to shadow offset 778 */ 779 if (offset >= copy_object->vo_shadow_offset) { 780 copy_size = size; 781 } else if (size >= copy_object->vo_shadow_offset - offset) { 782 copy_size = size - (copy_object->vo_shadow_offset - offset); 783 } else { 784 copy_size = 0; 785 } 786 787 if (copy_offset + copy_size > copy_object->vo_size) { 788 if (copy_object->vo_size >= copy_offset) { 789 copy_size = copy_object->vo_size - copy_offset; 790 } else { 791 copy_size = 0; 792 } 793 } 794 copy_size+=copy_offset; 795 796 } else { 797 copy_object = object; 798 799 copy_size = offset + size; 800 copy_offset = offset; 801 } 802 fault_info.interruptible = THREAD_UNINT; 803 fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL; 804 fault_info.user_tag = 0; 805 fault_info.pmap_options = 0; 806 fault_info.lo_offset = copy_offset; 807 fault_info.hi_offset = copy_size; 808 fault_info.no_cache = FALSE; 809 fault_info.stealth = TRUE; 810 fault_info.io_sync = FALSE; 811 fault_info.cs_bypass = FALSE; 812 fault_info.mark_zf_absent = FALSE; 813 fault_info.batch_pmap_op = FALSE; 814 815 vm_object_paging_begin(copy_object); 816 817 for (i = copy_offset; i < copy_size; i += PAGE_SIZE) { 818 RETRY_COW_OF_LOCK_REQUEST: 819 fault_info.cluster_size = (vm_size_t) (copy_size - i); 820 assert(fault_info.cluster_size == copy_size - i); 821 822 prot = VM_PROT_WRITE|VM_PROT_READ; 823 page = VM_PAGE_NULL; 824 result = vm_fault_page(copy_object, i, 825 VM_PROT_WRITE|VM_PROT_READ, 826 FALSE, 827 FALSE, /* page not looked up */ 828 &prot, 829 &page, 830 &top_page, 831 (int *)0, 832 &error, 833 FALSE, 834 FALSE, &fault_info); 835 836 switch (result) { 837 case VM_FAULT_SUCCESS: 838 if (top_page) { 839 vm_fault_cleanup( 840 page->object, top_page); 841 vm_object_lock(copy_object); 842 vm_object_paging_begin(copy_object); 843 } 844 if (!page->active && 845 !page->inactive && 846 !page->throttled) { 847 vm_page_lockspin_queues(); 848 if (!page->active && 849 !page->inactive && 850 !page->throttled) 851 vm_page_deactivate(page); 852 vm_page_unlock_queues(); 853 } 854 PAGE_WAKEUP_DONE(page); 855 break; 856 case VM_FAULT_RETRY: 857 prot = VM_PROT_WRITE|VM_PROT_READ; 858 vm_object_lock(copy_object); 859 vm_object_paging_begin(copy_object); 860 goto RETRY_COW_OF_LOCK_REQUEST; 861 case VM_FAULT_INTERRUPTED: 862 prot = VM_PROT_WRITE|VM_PROT_READ; 863 vm_object_lock(copy_object); 864 vm_object_paging_begin(copy_object); 865 goto RETRY_COW_OF_LOCK_REQUEST; 866 case VM_FAULT_MEMORY_SHORTAGE: 867 VM_PAGE_WAIT(); 868 prot = VM_PROT_WRITE|VM_PROT_READ; 869 vm_object_lock(copy_object); 870 vm_object_paging_begin(copy_object); 871 goto RETRY_COW_OF_LOCK_REQUEST; 872 case VM_FAULT_SUCCESS_NO_VM_PAGE: 873 /* success but no VM page: fail */ 874 vm_object_paging_end(copy_object); 875 vm_object_unlock(copy_object); 876 /*FALLTHROUGH*/ 877 case VM_FAULT_MEMORY_ERROR: 878 if (object != copy_object) 879 vm_object_deallocate(copy_object); 880 vm_object_lock(object); 881 goto BYPASS_COW_COPYIN; 882 default: 883 panic("vm_object_update: unexpected error 0x%x" 884 " from vm_fault_page()\n", result); 885 } 886 887 } 888 vm_object_paging_end(copy_object); 889 } 890 if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) { 891 if (copy_object != VM_OBJECT_NULL && copy_object != object) { 892 vm_object_unlock(copy_object); 893 vm_object_deallocate(copy_object); 894 vm_object_lock(object); 895 } 896 return KERN_SUCCESS; 897 } 898 if (copy_object != VM_OBJECT_NULL && copy_object != object) { 899 if ((flags & MEMORY_OBJECT_DATA_PURGE)) { 900 copy_object->shadow_severed = TRUE; 901 copy_object->shadowed = FALSE; 902 copy_object->shadow = NULL; 903 /* 904 * delete the ref the COW was holding on the target object 905 */ 906 vm_object_deallocate(object); 907 } 908 vm_object_unlock(copy_object); 909 vm_object_deallocate(copy_object); 910 vm_object_lock(object); 911 } 912BYPASS_COW_COPYIN: 913 914 /* 915 * when we have a really large range to check relative 916 * to the number of actual resident pages, we'd like 917 * to use the resident page list to drive our checks 918 * however, the object lock will get dropped while processing 919 * the page which means the resident queue can change which 920 * means we can't walk the queue as we process the pages 921 * we also want to do the processing in offset order to allow 922 * 'runs' of pages to be collected if we're being told to 923 * flush to disk... the resident page queue is NOT ordered. 924 * 925 * a temporary solution (until we figure out how to deal with 926 * large address spaces more generically) is to pre-flight 927 * the resident page queue (if it's small enough) and develop 928 * a collection of extents (that encompass actual resident pages) 929 * to visit. This will at least allow us to deal with some of the 930 * more pathological cases in a more efficient manner. The current 931 * worst case (a single resident page at the end of an extremely large 932 * range) can take minutes to complete for ranges in the terrabyte 933 * category... since this routine is called when truncating a file, 934 * and we currently support files up to 16 Tbytes in size, this 935 * is not a theoretical problem 936 */ 937 938 if ((object->resident_page_count < RESIDENT_LIMIT) && 939 (atop_64(size) > (unsigned)(object->resident_page_count/(8 * MAX_EXTENTS)))) { 940 vm_page_t next; 941 vm_object_offset_t start; 942 vm_object_offset_t end; 943 vm_object_size_t e_mask; 944 vm_page_t m; 945 946 start = offset; 947 end = offset + size; 948 num_of_extents = 0; 949 e_mask = ~((vm_object_size_t)(EXTENT_SIZE - 1)); 950 951 m = (vm_page_t) queue_first(&object->memq); 952 953 while (!queue_end(&object->memq, (queue_entry_t) m)) { 954 next = (vm_page_t) queue_next(&m->listq); 955 956 if ((m->offset >= start) && (m->offset < end)) { 957 /* 958 * this is a page we're interested in 959 * try to fit it into a current extent 960 */ 961 for (n = 0; n < num_of_extents; n++) { 962 if ((m->offset & e_mask) == extents[n].e_base) { 963 /* 964 * use (PAGE_SIZE - 1) to determine the 965 * max offset so that we don't wrap if 966 * we're at the last page of the space 967 */ 968 if (m->offset < extents[n].e_min) 969 extents[n].e_min = m->offset; 970 else if ((m->offset + (PAGE_SIZE - 1)) > extents[n].e_max) 971 extents[n].e_max = m->offset + (PAGE_SIZE - 1); 972 break; 973 } 974 } 975 if (n == num_of_extents) { 976 /* 977 * didn't find a current extent that can encompass 978 * this page 979 */ 980 if (n < MAX_EXTENTS) { 981 /* 982 * if we still have room, 983 * create a new extent 984 */ 985 extents[n].e_base = m->offset & e_mask; 986 extents[n].e_min = m->offset; 987 extents[n].e_max = m->offset + (PAGE_SIZE - 1); 988 989 num_of_extents++; 990 } else { 991 /* 992 * no room to create a new extent... 993 * fall back to a single extent based 994 * on the min and max page offsets 995 * we find in the range we're interested in... 996 * first, look through the extent list and 997 * develop the overall min and max for the 998 * pages we've looked at up to this point 999 */ 1000 for (n = 1; n < num_of_extents; n++) { 1001 if (extents[n].e_min < extents[0].e_min) 1002 extents[0].e_min = extents[n].e_min; 1003 if (extents[n].e_max > extents[0].e_max) 1004 extents[0].e_max = extents[n].e_max; 1005 } 1006 /* 1007 * now setup to run through the remaining pages 1008 * to determine the overall min and max 1009 * offset for the specified range 1010 */ 1011 extents[0].e_base = 0; 1012 e_mask = 0; 1013 num_of_extents = 1; 1014 1015 /* 1016 * by continuing, we'll reprocess the 1017 * page that forced us to abandon trying 1018 * to develop multiple extents 1019 */ 1020 continue; 1021 } 1022 } 1023 } 1024 m = next; 1025 } 1026 } else { 1027 extents[0].e_min = offset; 1028 extents[0].e_max = offset + (size - 1); 1029 1030 num_of_extents = 1; 1031 } 1032 for (n = 0; n < num_of_extents; n++) { 1033 if (vm_object_update_extent(object, extents[n].e_min, extents[n].e_max, resid_offset, io_errno, 1034 should_flush, should_return, should_iosync, protection)) 1035 data_returned = TRUE; 1036 } 1037 return (data_returned); 1038} 1039 1040 1041/* 1042 * Routine: memory_object_synchronize_completed [user interface] 1043 * 1044 * Tell kernel that previously synchronized data 1045 * (memory_object_synchronize) has been queue or placed on the 1046 * backing storage. 1047 * 1048 * Note: there may be multiple synchronize requests for a given 1049 * memory object outstanding but they will not overlap. 1050 */ 1051 1052kern_return_t 1053memory_object_synchronize_completed( 1054 memory_object_control_t control, 1055 memory_object_offset_t offset, 1056 memory_object_size_t length) 1057{ 1058 vm_object_t object; 1059 msync_req_t msr; 1060 1061 object = memory_object_control_to_vm_object(control); 1062 1063 XPR(XPR_MEMORY_OBJECT, 1064 "m_o_sync_completed, object 0x%X, offset 0x%X length 0x%X\n", 1065 object, offset, length, 0, 0); 1066 1067 /* 1068 * Look for bogus arguments 1069 */ 1070 1071 if (object == VM_OBJECT_NULL) 1072 return (KERN_INVALID_ARGUMENT); 1073 1074 vm_object_lock(object); 1075 1076/* 1077 * search for sync request structure 1078 */ 1079 queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) { 1080 if (msr->offset == offset && msr->length == length) { 1081 queue_remove(&object->msr_q, msr, msync_req_t, msr_q); 1082 break; 1083 } 1084 }/* queue_iterate */ 1085 1086 if (queue_end(&object->msr_q, (queue_entry_t)msr)) { 1087 vm_object_unlock(object); 1088 return KERN_INVALID_ARGUMENT; 1089 } 1090 1091 msr_lock(msr); 1092 vm_object_unlock(object); 1093 msr->flag = VM_MSYNC_DONE; 1094 msr_unlock(msr); 1095 thread_wakeup((event_t) msr); 1096 1097 return KERN_SUCCESS; 1098}/* memory_object_synchronize_completed */ 1099 1100static kern_return_t 1101vm_object_set_attributes_common( 1102 vm_object_t object, 1103 boolean_t may_cache, 1104 memory_object_copy_strategy_t copy_strategy, 1105 boolean_t temporary, 1106 __unused boolean_t silent_overwrite, 1107 boolean_t advisory_pageout) 1108{ 1109 boolean_t object_became_ready; 1110 1111 XPR(XPR_MEMORY_OBJECT, 1112 "m_o_set_attr_com, object 0x%X flg %x strat %d\n", 1113 object, (may_cache&1)|((temporary&1)<1), copy_strategy, 0, 0); 1114 1115 if (object == VM_OBJECT_NULL) 1116 return(KERN_INVALID_ARGUMENT); 1117 1118 /* 1119 * Verify the attributes of importance 1120 */ 1121 1122 switch(copy_strategy) { 1123 case MEMORY_OBJECT_COPY_NONE: 1124 case MEMORY_OBJECT_COPY_DELAY: 1125 break; 1126 default: 1127 return(KERN_INVALID_ARGUMENT); 1128 } 1129 1130 if (may_cache) 1131 may_cache = TRUE; 1132 if (temporary) 1133 temporary = TRUE; 1134 1135 vm_object_lock(object); 1136 1137 /* 1138 * Copy the attributes 1139 */ 1140 assert(!object->internal); 1141 object_became_ready = !object->pager_ready; 1142 object->copy_strategy = copy_strategy; 1143 object->can_persist = may_cache; 1144 object->temporary = temporary; 1145// object->silent_overwrite = silent_overwrite; 1146 object->advisory_pageout = advisory_pageout; 1147 1148 /* 1149 * Wake up anyone waiting for the ready attribute 1150 * to become asserted. 1151 */ 1152 1153 if (object_became_ready) { 1154 object->pager_ready = TRUE; 1155 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY); 1156 } 1157 1158 vm_object_unlock(object); 1159 1160 return(KERN_SUCCESS); 1161} 1162 1163/* 1164 * Set the memory object attribute as provided. 1165 * 1166 * XXX This routine cannot be completed until the vm_msync, clean 1167 * in place, and cluster work is completed. See ifdef notyet 1168 * below and note that vm_object_set_attributes_common() 1169 * may have to be expanded. 1170 */ 1171kern_return_t 1172memory_object_change_attributes( 1173 memory_object_control_t control, 1174 memory_object_flavor_t flavor, 1175 memory_object_info_t attributes, 1176 mach_msg_type_number_t count) 1177{ 1178 vm_object_t object; 1179 kern_return_t result = KERN_SUCCESS; 1180 boolean_t temporary; 1181 boolean_t may_cache; 1182 boolean_t invalidate; 1183 memory_object_copy_strategy_t copy_strategy; 1184 boolean_t silent_overwrite; 1185 boolean_t advisory_pageout; 1186 1187 object = memory_object_control_to_vm_object(control); 1188 if (object == VM_OBJECT_NULL) 1189 return (KERN_INVALID_ARGUMENT); 1190 1191 vm_object_lock(object); 1192 1193 temporary = object->temporary; 1194 may_cache = object->can_persist; 1195 copy_strategy = object->copy_strategy; 1196// silent_overwrite = object->silent_overwrite; 1197 silent_overwrite = FALSE; 1198 advisory_pageout = object->advisory_pageout; 1199#if notyet 1200 invalidate = object->invalidate; 1201#endif 1202 vm_object_unlock(object); 1203 1204 switch (flavor) { 1205 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: 1206 { 1207 old_memory_object_behave_info_t behave; 1208 1209 if (count != OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { 1210 result = KERN_INVALID_ARGUMENT; 1211 break; 1212 } 1213 1214 behave = (old_memory_object_behave_info_t) attributes; 1215 1216 temporary = behave->temporary; 1217 invalidate = behave->invalidate; 1218 copy_strategy = behave->copy_strategy; 1219 1220 break; 1221 } 1222 1223 case MEMORY_OBJECT_BEHAVIOR_INFO: 1224 { 1225 memory_object_behave_info_t behave; 1226 1227 if (count != MEMORY_OBJECT_BEHAVE_INFO_COUNT) { 1228 result = KERN_INVALID_ARGUMENT; 1229 break; 1230 } 1231 1232 behave = (memory_object_behave_info_t) attributes; 1233 1234 temporary = behave->temporary; 1235 invalidate = behave->invalidate; 1236 copy_strategy = behave->copy_strategy; 1237 silent_overwrite = behave->silent_overwrite; 1238 advisory_pageout = behave->advisory_pageout; 1239 break; 1240 } 1241 1242 case MEMORY_OBJECT_PERFORMANCE_INFO: 1243 { 1244 memory_object_perf_info_t perf; 1245 1246 if (count != MEMORY_OBJECT_PERF_INFO_COUNT) { 1247 result = KERN_INVALID_ARGUMENT; 1248 break; 1249 } 1250 1251 perf = (memory_object_perf_info_t) attributes; 1252 1253 may_cache = perf->may_cache; 1254 1255 break; 1256 } 1257 1258 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: 1259 { 1260 old_memory_object_attr_info_t attr; 1261 1262 if (count != OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { 1263 result = KERN_INVALID_ARGUMENT; 1264 break; 1265 } 1266 1267 attr = (old_memory_object_attr_info_t) attributes; 1268 1269 may_cache = attr->may_cache; 1270 copy_strategy = attr->copy_strategy; 1271 1272 break; 1273 } 1274 1275 case MEMORY_OBJECT_ATTRIBUTE_INFO: 1276 { 1277 memory_object_attr_info_t attr; 1278 1279 if (count != MEMORY_OBJECT_ATTR_INFO_COUNT) { 1280 result = KERN_INVALID_ARGUMENT; 1281 break; 1282 } 1283 1284 attr = (memory_object_attr_info_t) attributes; 1285 1286 copy_strategy = attr->copy_strategy; 1287 may_cache = attr->may_cache_object; 1288 temporary = attr->temporary; 1289 1290 break; 1291 } 1292 1293 default: 1294 result = KERN_INVALID_ARGUMENT; 1295 break; 1296 } 1297 1298 if (result != KERN_SUCCESS) 1299 return(result); 1300 1301 if (copy_strategy == MEMORY_OBJECT_COPY_TEMPORARY) { 1302 copy_strategy = MEMORY_OBJECT_COPY_DELAY; 1303 temporary = TRUE; 1304 } else { 1305 temporary = FALSE; 1306 } 1307 1308 /* 1309 * XXX may_cache may become a tri-valued variable to handle 1310 * XXX uncache if not in use. 1311 */ 1312 return (vm_object_set_attributes_common(object, 1313 may_cache, 1314 copy_strategy, 1315 temporary, 1316 silent_overwrite, 1317 advisory_pageout)); 1318} 1319 1320kern_return_t 1321memory_object_get_attributes( 1322 memory_object_control_t control, 1323 memory_object_flavor_t flavor, 1324 memory_object_info_t attributes, /* pointer to OUT array */ 1325 mach_msg_type_number_t *count) /* IN/OUT */ 1326{ 1327 kern_return_t ret = KERN_SUCCESS; 1328 vm_object_t object; 1329 1330 object = memory_object_control_to_vm_object(control); 1331 if (object == VM_OBJECT_NULL) 1332 return (KERN_INVALID_ARGUMENT); 1333 1334 vm_object_lock(object); 1335 1336 switch (flavor) { 1337 case OLD_MEMORY_OBJECT_BEHAVIOR_INFO: 1338 { 1339 old_memory_object_behave_info_t behave; 1340 1341 if (*count < OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT) { 1342 ret = KERN_INVALID_ARGUMENT; 1343 break; 1344 } 1345 1346 behave = (old_memory_object_behave_info_t) attributes; 1347 behave->copy_strategy = object->copy_strategy; 1348 behave->temporary = object->temporary; 1349#if notyet /* remove when vm_msync complies and clean in place fini */ 1350 behave->invalidate = object->invalidate; 1351#else 1352 behave->invalidate = FALSE; 1353#endif 1354 1355 *count = OLD_MEMORY_OBJECT_BEHAVE_INFO_COUNT; 1356 break; 1357 } 1358 1359 case MEMORY_OBJECT_BEHAVIOR_INFO: 1360 { 1361 memory_object_behave_info_t behave; 1362 1363 if (*count < MEMORY_OBJECT_BEHAVE_INFO_COUNT) { 1364 ret = KERN_INVALID_ARGUMENT; 1365 break; 1366 } 1367 1368 behave = (memory_object_behave_info_t) attributes; 1369 behave->copy_strategy = object->copy_strategy; 1370 behave->temporary = object->temporary; 1371#if notyet /* remove when vm_msync complies and clean in place fini */ 1372 behave->invalidate = object->invalidate; 1373#else 1374 behave->invalidate = FALSE; 1375#endif 1376 behave->advisory_pageout = object->advisory_pageout; 1377// behave->silent_overwrite = object->silent_overwrite; 1378 behave->silent_overwrite = FALSE; 1379 *count = MEMORY_OBJECT_BEHAVE_INFO_COUNT; 1380 break; 1381 } 1382 1383 case MEMORY_OBJECT_PERFORMANCE_INFO: 1384 { 1385 memory_object_perf_info_t perf; 1386 1387 if (*count < MEMORY_OBJECT_PERF_INFO_COUNT) { 1388 ret = KERN_INVALID_ARGUMENT; 1389 break; 1390 } 1391 1392 perf = (memory_object_perf_info_t) attributes; 1393 perf->cluster_size = PAGE_SIZE; 1394 perf->may_cache = object->can_persist; 1395 1396 *count = MEMORY_OBJECT_PERF_INFO_COUNT; 1397 break; 1398 } 1399 1400 case OLD_MEMORY_OBJECT_ATTRIBUTE_INFO: 1401 { 1402 old_memory_object_attr_info_t attr; 1403 1404 if (*count < OLD_MEMORY_OBJECT_ATTR_INFO_COUNT) { 1405 ret = KERN_INVALID_ARGUMENT; 1406 break; 1407 } 1408 1409 attr = (old_memory_object_attr_info_t) attributes; 1410 attr->may_cache = object->can_persist; 1411 attr->copy_strategy = object->copy_strategy; 1412 1413 *count = OLD_MEMORY_OBJECT_ATTR_INFO_COUNT; 1414 break; 1415 } 1416 1417 case MEMORY_OBJECT_ATTRIBUTE_INFO: 1418 { 1419 memory_object_attr_info_t attr; 1420 1421 if (*count < MEMORY_OBJECT_ATTR_INFO_COUNT) { 1422 ret = KERN_INVALID_ARGUMENT; 1423 break; 1424 } 1425 1426 attr = (memory_object_attr_info_t) attributes; 1427 attr->copy_strategy = object->copy_strategy; 1428 attr->cluster_size = PAGE_SIZE; 1429 attr->may_cache_object = object->can_persist; 1430 attr->temporary = object->temporary; 1431 1432 *count = MEMORY_OBJECT_ATTR_INFO_COUNT; 1433 break; 1434 } 1435 1436 default: 1437 ret = KERN_INVALID_ARGUMENT; 1438 break; 1439 } 1440 1441 vm_object_unlock(object); 1442 1443 return(ret); 1444} 1445 1446 1447kern_return_t 1448memory_object_iopl_request( 1449 ipc_port_t port, 1450 memory_object_offset_t offset, 1451 upl_size_t *upl_size, 1452 upl_t *upl_ptr, 1453 upl_page_info_array_t user_page_list, 1454 unsigned int *page_list_count, 1455 int *flags) 1456{ 1457 vm_object_t object; 1458 kern_return_t ret; 1459 int caller_flags; 1460 1461 caller_flags = *flags; 1462 1463 if (caller_flags & ~UPL_VALID_FLAGS) { 1464 /* 1465 * For forward compatibility's sake, 1466 * reject any unknown flag. 1467 */ 1468 return KERN_INVALID_VALUE; 1469 } 1470 1471 if (ip_kotype(port) == IKOT_NAMED_ENTRY) { 1472 vm_named_entry_t named_entry; 1473 1474 named_entry = (vm_named_entry_t)port->ip_kobject; 1475 /* a few checks to make sure user is obeying rules */ 1476 if(*upl_size == 0) { 1477 if(offset >= named_entry->size) 1478 return(KERN_INVALID_RIGHT); 1479 *upl_size = (upl_size_t)(named_entry->size - offset); 1480 if (*upl_size != named_entry->size - offset) 1481 return KERN_INVALID_ARGUMENT; 1482 } 1483 if(caller_flags & UPL_COPYOUT_FROM) { 1484 if((named_entry->protection & VM_PROT_READ) 1485 != VM_PROT_READ) { 1486 return(KERN_INVALID_RIGHT); 1487 } 1488 } else { 1489 if((named_entry->protection & 1490 (VM_PROT_READ | VM_PROT_WRITE)) 1491 != (VM_PROT_READ | VM_PROT_WRITE)) { 1492 return(KERN_INVALID_RIGHT); 1493 } 1494 } 1495 if(named_entry->size < (offset + *upl_size)) 1496 return(KERN_INVALID_ARGUMENT); 1497 1498 /* the callers parameter offset is defined to be the */ 1499 /* offset from beginning of named entry offset in object */ 1500 offset = offset + named_entry->offset; 1501 1502 if (named_entry->is_sub_map || 1503 named_entry->is_copy) 1504 return KERN_INVALID_ARGUMENT; 1505 1506 named_entry_lock(named_entry); 1507 1508 if (named_entry->is_pager) { 1509 object = vm_object_enter(named_entry->backing.pager, 1510 named_entry->offset + named_entry->size, 1511 named_entry->internal, 1512 FALSE, 1513 FALSE); 1514 if (object == VM_OBJECT_NULL) { 1515 named_entry_unlock(named_entry); 1516 return(KERN_INVALID_OBJECT); 1517 } 1518 1519 /* JMM - drop reference on pager here? */ 1520 1521 /* create an extra reference for the named entry */ 1522 vm_object_lock(object); 1523 vm_object_reference_locked(object); 1524 named_entry->backing.object = object; 1525 named_entry->is_pager = FALSE; 1526 named_entry_unlock(named_entry); 1527 1528 /* wait for object to be ready */ 1529 while (!object->pager_ready) { 1530 vm_object_wait(object, 1531 VM_OBJECT_EVENT_PAGER_READY, 1532 THREAD_UNINT); 1533 vm_object_lock(object); 1534 } 1535 vm_object_unlock(object); 1536 } else { 1537 /* This is the case where we are going to map */ 1538 /* an already mapped object. If the object is */ 1539 /* not ready it is internal. An external */ 1540 /* object cannot be mapped until it is ready */ 1541 /* we can therefore avoid the ready check */ 1542 /* in this case. */ 1543 object = named_entry->backing.object; 1544 vm_object_reference(object); 1545 named_entry_unlock(named_entry); 1546 } 1547 } else if (ip_kotype(port) == IKOT_MEM_OBJ_CONTROL) { 1548 memory_object_control_t control; 1549 control = (memory_object_control_t) port; 1550 if (control == NULL) 1551 return (KERN_INVALID_ARGUMENT); 1552 object = memory_object_control_to_vm_object(control); 1553 if (object == VM_OBJECT_NULL) 1554 return (KERN_INVALID_ARGUMENT); 1555 vm_object_reference(object); 1556 } else { 1557 return KERN_INVALID_ARGUMENT; 1558 } 1559 if (object == VM_OBJECT_NULL) 1560 return (KERN_INVALID_ARGUMENT); 1561 1562 if (!object->private) { 1563 if (object->phys_contiguous) { 1564 *flags = UPL_PHYS_CONTIG; 1565 } else { 1566 *flags = 0; 1567 } 1568 } else { 1569 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG; 1570 } 1571 1572 ret = vm_object_iopl_request(object, 1573 offset, 1574 *upl_size, 1575 upl_ptr, 1576 user_page_list, 1577 page_list_count, 1578 caller_flags); 1579 vm_object_deallocate(object); 1580 return ret; 1581} 1582 1583/* 1584 * Routine: memory_object_upl_request [interface] 1585 * Purpose: 1586 * Cause the population of a portion of a vm_object. 1587 * Depending on the nature of the request, the pages 1588 * returned may be contain valid data or be uninitialized. 1589 * 1590 */ 1591 1592kern_return_t 1593memory_object_upl_request( 1594 memory_object_control_t control, 1595 memory_object_offset_t offset, 1596 upl_size_t size, 1597 upl_t *upl_ptr, 1598 upl_page_info_array_t user_page_list, 1599 unsigned int *page_list_count, 1600 int cntrl_flags) 1601{ 1602 vm_object_t object; 1603 1604 object = memory_object_control_to_vm_object(control); 1605 if (object == VM_OBJECT_NULL) 1606 return (KERN_TERMINATED); 1607 1608 return vm_object_upl_request(object, 1609 offset, 1610 size, 1611 upl_ptr, 1612 user_page_list, 1613 page_list_count, 1614 cntrl_flags); 1615} 1616 1617/* 1618 * Routine: memory_object_super_upl_request [interface] 1619 * Purpose: 1620 * Cause the population of a portion of a vm_object 1621 * in much the same way as memory_object_upl_request. 1622 * Depending on the nature of the request, the pages 1623 * returned may be contain valid data or be uninitialized. 1624 * However, the region may be expanded up to the super 1625 * cluster size provided. 1626 */ 1627 1628kern_return_t 1629memory_object_super_upl_request( 1630 memory_object_control_t control, 1631 memory_object_offset_t offset, 1632 upl_size_t size, 1633 upl_size_t super_cluster, 1634 upl_t *upl, 1635 upl_page_info_t *user_page_list, 1636 unsigned int *page_list_count, 1637 int cntrl_flags) 1638{ 1639 vm_object_t object; 1640 1641 object = memory_object_control_to_vm_object(control); 1642 if (object == VM_OBJECT_NULL) 1643 return (KERN_INVALID_ARGUMENT); 1644 1645 return vm_object_super_upl_request(object, 1646 offset, 1647 size, 1648 super_cluster, 1649 upl, 1650 user_page_list, 1651 page_list_count, 1652 cntrl_flags); 1653} 1654 1655kern_return_t 1656memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start, 1657 vm_size_t *length, uint32_t *io_streaming, memory_object_fault_info_t fault_info) 1658{ 1659 vm_object_t object; 1660 1661 object = memory_object_control_to_vm_object(control); 1662 1663 if (object == VM_OBJECT_NULL || object->paging_offset > *start) 1664 return (KERN_INVALID_ARGUMENT); 1665 1666 *start -= object->paging_offset; 1667 1668 vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info, io_streaming); 1669 1670 *start += object->paging_offset; 1671 1672 return (KERN_SUCCESS); 1673} 1674 1675 1676int vm_stat_discard_cleared_reply = 0; 1677int vm_stat_discard_cleared_unset = 0; 1678int vm_stat_discard_cleared_too_late = 0; 1679 1680 1681 1682/* 1683 * Routine: host_default_memory_manager [interface] 1684 * Purpose: 1685 * set/get the default memory manager port and default cluster 1686 * size. 1687 * 1688 * If successful, consumes the supplied naked send right. 1689 */ 1690kern_return_t 1691host_default_memory_manager( 1692 host_priv_t host_priv, 1693 memory_object_default_t *default_manager, 1694 __unused memory_object_cluster_size_t cluster_size) 1695{ 1696 memory_object_default_t current_manager; 1697 memory_object_default_t new_manager; 1698 memory_object_default_t returned_manager; 1699 kern_return_t result = KERN_SUCCESS; 1700 1701 if (host_priv == HOST_PRIV_NULL) 1702 return(KERN_INVALID_HOST); 1703 1704 assert(host_priv == &realhost); 1705 1706 new_manager = *default_manager; 1707 lck_mtx_lock(&memory_manager_default_lock); 1708 current_manager = memory_manager_default; 1709 returned_manager = MEMORY_OBJECT_DEFAULT_NULL; 1710 1711 if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) { 1712 /* 1713 * Retrieve the current value. 1714 */ 1715 returned_manager = current_manager; 1716 memory_object_default_reference(returned_manager); 1717 } else { 1718 1719 /* 1720 * If this is the first non-null manager, start 1721 * up the internal pager support. 1722 */ 1723 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { 1724 result = vm_pageout_internal_start(); 1725 if (result != KERN_SUCCESS) 1726 goto out; 1727 } 1728 1729 /* 1730 * Retrieve the current value, 1731 * and replace it with the supplied value. 1732 * We return the old reference to the caller 1733 * but we have to take a reference on the new 1734 * one. 1735 */ 1736 returned_manager = current_manager; 1737 memory_manager_default = new_manager; 1738 memory_object_default_reference(new_manager); 1739 1740 /* 1741 * In case anyone's been waiting for a memory 1742 * manager to be established, wake them up. 1743 */ 1744 1745 thread_wakeup((event_t) &memory_manager_default); 1746 1747 /* 1748 * Now that we have a default pager for anonymous memory, 1749 * reactivate all the throttled pages (i.e. dirty pages with 1750 * no pager). 1751 */ 1752 if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) 1753 { 1754 vm_page_reactivate_all_throttled(); 1755 } 1756 } 1757 out: 1758 lck_mtx_unlock(&memory_manager_default_lock); 1759 1760 *default_manager = returned_manager; 1761 return(result); 1762} 1763 1764/* 1765 * Routine: memory_manager_default_reference 1766 * Purpose: 1767 * Returns a naked send right for the default 1768 * memory manager. The returned right is always 1769 * valid (not IP_NULL or IP_DEAD). 1770 */ 1771 1772__private_extern__ memory_object_default_t 1773memory_manager_default_reference(void) 1774{ 1775 memory_object_default_t current_manager; 1776 1777 lck_mtx_lock(&memory_manager_default_lock); 1778 current_manager = memory_manager_default; 1779 while (current_manager == MEMORY_OBJECT_DEFAULT_NULL) { 1780 wait_result_t res; 1781 1782 res = lck_mtx_sleep(&memory_manager_default_lock, 1783 LCK_SLEEP_DEFAULT, 1784 (event_t) &memory_manager_default, 1785 THREAD_UNINT); 1786 assert(res == THREAD_AWAKENED); 1787 current_manager = memory_manager_default; 1788 } 1789 memory_object_default_reference(current_manager); 1790 lck_mtx_unlock(&memory_manager_default_lock); 1791 1792 return current_manager; 1793} 1794 1795/* 1796 * Routine: memory_manager_default_check 1797 * 1798 * Purpose: 1799 * Check whether a default memory manager has been set 1800 * up yet, or not. Returns KERN_SUCCESS if dmm exists, 1801 * and KERN_FAILURE if dmm does not exist. 1802 * 1803 * If there is no default memory manager, log an error, 1804 * but only the first time. 1805 * 1806 */ 1807__private_extern__ kern_return_t 1808memory_manager_default_check(void) 1809{ 1810 memory_object_default_t current; 1811 1812 lck_mtx_lock(&memory_manager_default_lock); 1813 current = memory_manager_default; 1814 if (current == MEMORY_OBJECT_DEFAULT_NULL) { 1815 static boolean_t logged; /* initialized to 0 */ 1816 boolean_t complain = !logged; 1817 logged = TRUE; 1818 lck_mtx_unlock(&memory_manager_default_lock); 1819 if (complain) 1820 printf("Warning: No default memory manager\n"); 1821 return(KERN_FAILURE); 1822 } else { 1823 lck_mtx_unlock(&memory_manager_default_lock); 1824 return(KERN_SUCCESS); 1825 } 1826} 1827 1828__private_extern__ void 1829memory_manager_default_init(void) 1830{ 1831 memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL; 1832 lck_mtx_init(&memory_manager_default_lock, &vm_object_lck_grp, &vm_object_lck_attr); 1833} 1834 1835 1836 1837/* Allow manipulation of individual page state. This is actually part of */ 1838/* the UPL regimen but takes place on the object rather than on a UPL */ 1839 1840kern_return_t 1841memory_object_page_op( 1842 memory_object_control_t control, 1843 memory_object_offset_t offset, 1844 int ops, 1845 ppnum_t *phys_entry, 1846 int *flags) 1847{ 1848 vm_object_t object; 1849 1850 object = memory_object_control_to_vm_object(control); 1851 if (object == VM_OBJECT_NULL) 1852 return (KERN_INVALID_ARGUMENT); 1853 1854 return vm_object_page_op(object, offset, ops, phys_entry, flags); 1855} 1856 1857/* 1858 * memory_object_range_op offers performance enhancement over 1859 * memory_object_page_op for page_op functions which do not require page 1860 * level state to be returned from the call. Page_op was created to provide 1861 * a low-cost alternative to page manipulation via UPLs when only a single 1862 * page was involved. The range_op call establishes the ability in the _op 1863 * family of functions to work on multiple pages where the lack of page level 1864 * state handling allows the caller to avoid the overhead of the upl structures. 1865 */ 1866 1867kern_return_t 1868memory_object_range_op( 1869 memory_object_control_t control, 1870 memory_object_offset_t offset_beg, 1871 memory_object_offset_t offset_end, 1872 int ops, 1873 int *range) 1874{ 1875 vm_object_t object; 1876 1877 object = memory_object_control_to_vm_object(control); 1878 if (object == VM_OBJECT_NULL) 1879 return (KERN_INVALID_ARGUMENT); 1880 1881 return vm_object_range_op(object, 1882 offset_beg, 1883 offset_end, 1884 ops, 1885 (uint32_t *) range); 1886} 1887 1888 1889void 1890memory_object_mark_used( 1891 memory_object_control_t control) 1892{ 1893 vm_object_t object; 1894 1895 if (control == NULL) 1896 return; 1897 1898 object = memory_object_control_to_vm_object(control); 1899 1900 if (object != VM_OBJECT_NULL) 1901 vm_object_cache_remove(object); 1902} 1903 1904 1905void 1906memory_object_mark_unused( 1907 memory_object_control_t control, 1908 __unused boolean_t rage) 1909{ 1910 vm_object_t object; 1911 1912 if (control == NULL) 1913 return; 1914 1915 object = memory_object_control_to_vm_object(control); 1916 1917 if (object != VM_OBJECT_NULL) 1918 vm_object_cache_add(object); 1919} 1920 1921void 1922memory_object_mark_io_tracking( 1923 memory_object_control_t control) 1924{ 1925 vm_object_t object; 1926 1927 if (control == NULL) 1928 return; 1929 object = memory_object_control_to_vm_object(control); 1930 1931 if (object != VM_OBJECT_NULL) { 1932 vm_object_lock(object); 1933 object->io_tracking = TRUE; 1934 vm_object_unlock(object); 1935 } 1936} 1937 1938kern_return_t 1939memory_object_pages_resident( 1940 memory_object_control_t control, 1941 boolean_t * has_pages_resident) 1942{ 1943 vm_object_t object; 1944 1945 *has_pages_resident = FALSE; 1946 1947 object = memory_object_control_to_vm_object(control); 1948 if (object == VM_OBJECT_NULL) 1949 return (KERN_INVALID_ARGUMENT); 1950 1951 if (object->resident_page_count) 1952 *has_pages_resident = TRUE; 1953 1954 return (KERN_SUCCESS); 1955} 1956 1957kern_return_t 1958memory_object_signed( 1959 memory_object_control_t control, 1960 boolean_t is_signed) 1961{ 1962 vm_object_t object; 1963 1964 object = memory_object_control_to_vm_object(control); 1965 if (object == VM_OBJECT_NULL) 1966 return KERN_INVALID_ARGUMENT; 1967 1968 vm_object_lock(object); 1969 object->code_signed = is_signed; 1970 vm_object_unlock(object); 1971 1972 return KERN_SUCCESS; 1973} 1974 1975boolean_t 1976memory_object_is_signed( 1977 memory_object_control_t control) 1978{ 1979 boolean_t is_signed; 1980 vm_object_t object; 1981 1982 object = memory_object_control_to_vm_object(control); 1983 if (object == VM_OBJECT_NULL) 1984 return FALSE; 1985 1986 vm_object_lock_shared(object); 1987 is_signed = object->code_signed; 1988 vm_object_unlock(object); 1989 1990 return is_signed; 1991} 1992 1993boolean_t 1994memory_object_is_slid( 1995 memory_object_control_t control) 1996{ 1997 vm_object_t object = VM_OBJECT_NULL; 1998 1999 object = memory_object_control_to_vm_object(control); 2000 if (object == VM_OBJECT_NULL) 2001 return FALSE; 2002 2003 return object->object_slid; 2004} 2005 2006static zone_t mem_obj_control_zone; 2007 2008__private_extern__ void 2009memory_object_control_bootstrap(void) 2010{ 2011 int i; 2012 2013 i = (vm_size_t) sizeof (struct memory_object_control); 2014 mem_obj_control_zone = zinit (i, 8192*i, 4096, "mem_obj_control"); 2015 zone_change(mem_obj_control_zone, Z_CALLERACCT, FALSE); 2016 zone_change(mem_obj_control_zone, Z_NOENCRYPT, TRUE); 2017 return; 2018} 2019 2020__private_extern__ memory_object_control_t 2021memory_object_control_allocate( 2022 vm_object_t object) 2023{ 2024 memory_object_control_t control; 2025 2026 control = (memory_object_control_t)zalloc(mem_obj_control_zone); 2027 if (control != MEMORY_OBJECT_CONTROL_NULL) { 2028 control->moc_object = object; 2029 control->moc_ikot = IKOT_MEM_OBJ_CONTROL; /* fake ip_kotype */ 2030 } 2031 return (control); 2032} 2033 2034__private_extern__ void 2035memory_object_control_collapse( 2036 memory_object_control_t control, 2037 vm_object_t object) 2038{ 2039 assert((control->moc_object != VM_OBJECT_NULL) && 2040 (control->moc_object != object)); 2041 control->moc_object = object; 2042} 2043 2044__private_extern__ vm_object_t 2045memory_object_control_to_vm_object( 2046 memory_object_control_t control) 2047{ 2048 if (control == MEMORY_OBJECT_CONTROL_NULL || 2049 control->moc_ikot != IKOT_MEM_OBJ_CONTROL) 2050 return VM_OBJECT_NULL; 2051 2052 return (control->moc_object); 2053} 2054 2055memory_object_control_t 2056convert_port_to_mo_control( 2057 __unused mach_port_t port) 2058{ 2059 return MEMORY_OBJECT_CONTROL_NULL; 2060} 2061 2062 2063mach_port_t 2064convert_mo_control_to_port( 2065 __unused memory_object_control_t control) 2066{ 2067 return MACH_PORT_NULL; 2068} 2069 2070void 2071memory_object_control_reference( 2072 __unused memory_object_control_t control) 2073{ 2074 return; 2075} 2076 2077/* 2078 * We only every issue one of these references, so kill it 2079 * when that gets released (should switch the real reference 2080 * counting in true port-less EMMI). 2081 */ 2082void 2083memory_object_control_deallocate( 2084 memory_object_control_t control) 2085{ 2086 zfree(mem_obj_control_zone, control); 2087} 2088 2089void 2090memory_object_control_disable( 2091 memory_object_control_t control) 2092{ 2093 assert(control->moc_object != VM_OBJECT_NULL); 2094 control->moc_object = VM_OBJECT_NULL; 2095} 2096 2097void 2098memory_object_default_reference( 2099 memory_object_default_t dmm) 2100{ 2101 ipc_port_make_send(dmm); 2102} 2103 2104void 2105memory_object_default_deallocate( 2106 memory_object_default_t dmm) 2107{ 2108 ipc_port_release_send(dmm); 2109} 2110 2111memory_object_t 2112convert_port_to_memory_object( 2113 __unused mach_port_t port) 2114{ 2115 return (MEMORY_OBJECT_NULL); 2116} 2117 2118 2119mach_port_t 2120convert_memory_object_to_port( 2121 __unused memory_object_t object) 2122{ 2123 return (MACH_PORT_NULL); 2124} 2125 2126 2127/* Routine memory_object_reference */ 2128void memory_object_reference( 2129 memory_object_t memory_object) 2130{ 2131 (memory_object->mo_pager_ops->memory_object_reference)( 2132 memory_object); 2133} 2134 2135/* Routine memory_object_deallocate */ 2136void memory_object_deallocate( 2137 memory_object_t memory_object) 2138{ 2139 (memory_object->mo_pager_ops->memory_object_deallocate)( 2140 memory_object); 2141} 2142 2143 2144/* Routine memory_object_init */ 2145kern_return_t memory_object_init 2146( 2147 memory_object_t memory_object, 2148 memory_object_control_t memory_control, 2149 memory_object_cluster_size_t memory_object_page_size 2150) 2151{ 2152 return (memory_object->mo_pager_ops->memory_object_init)( 2153 memory_object, 2154 memory_control, 2155 memory_object_page_size); 2156} 2157 2158/* Routine memory_object_terminate */ 2159kern_return_t memory_object_terminate 2160( 2161 memory_object_t memory_object 2162) 2163{ 2164 return (memory_object->mo_pager_ops->memory_object_terminate)( 2165 memory_object); 2166} 2167 2168/* Routine memory_object_data_request */ 2169kern_return_t memory_object_data_request 2170( 2171 memory_object_t memory_object, 2172 memory_object_offset_t offset, 2173 memory_object_cluster_size_t length, 2174 vm_prot_t desired_access, 2175 memory_object_fault_info_t fault_info 2176) 2177{ 2178 return (memory_object->mo_pager_ops->memory_object_data_request)( 2179 memory_object, 2180 offset, 2181 length, 2182 desired_access, 2183 fault_info); 2184} 2185 2186/* Routine memory_object_data_return */ 2187kern_return_t memory_object_data_return 2188( 2189 memory_object_t memory_object, 2190 memory_object_offset_t offset, 2191 memory_object_cluster_size_t size, 2192 memory_object_offset_t *resid_offset, 2193 int *io_error, 2194 boolean_t dirty, 2195 boolean_t kernel_copy, 2196 int upl_flags 2197) 2198{ 2199 return (memory_object->mo_pager_ops->memory_object_data_return)( 2200 memory_object, 2201 offset, 2202 size, 2203 resid_offset, 2204 io_error, 2205 dirty, 2206 kernel_copy, 2207 upl_flags); 2208} 2209 2210/* Routine memory_object_data_initialize */ 2211kern_return_t memory_object_data_initialize 2212( 2213 memory_object_t memory_object, 2214 memory_object_offset_t offset, 2215 memory_object_cluster_size_t size 2216) 2217{ 2218 return (memory_object->mo_pager_ops->memory_object_data_initialize)( 2219 memory_object, 2220 offset, 2221 size); 2222} 2223 2224/* Routine memory_object_data_unlock */ 2225kern_return_t memory_object_data_unlock 2226( 2227 memory_object_t memory_object, 2228 memory_object_offset_t offset, 2229 memory_object_size_t size, 2230 vm_prot_t desired_access 2231) 2232{ 2233 return (memory_object->mo_pager_ops->memory_object_data_unlock)( 2234 memory_object, 2235 offset, 2236 size, 2237 desired_access); 2238} 2239 2240/* Routine memory_object_synchronize */ 2241kern_return_t memory_object_synchronize 2242( 2243 memory_object_t memory_object, 2244 memory_object_offset_t offset, 2245 memory_object_size_t size, 2246 vm_sync_t sync_flags 2247) 2248{ 2249 return (memory_object->mo_pager_ops->memory_object_synchronize)( 2250 memory_object, 2251 offset, 2252 size, 2253 sync_flags); 2254} 2255 2256 2257/* 2258 * memory_object_map() is called by VM (in vm_map_enter() and its variants) 2259 * each time a "named" VM object gets mapped directly or indirectly 2260 * (copy-on-write mapping). A "named" VM object has an extra reference held 2261 * by the pager to keep it alive until the pager decides that the 2262 * memory object (and its VM object) can be reclaimed. 2263 * VM calls memory_object_last_unmap() (in vm_object_deallocate()) when all 2264 * the mappings of that memory object have been removed. 2265 * 2266 * For a given VM object, calls to memory_object_map() and memory_object_unmap() 2267 * are serialized (through object->mapping_in_progress), to ensure that the 2268 * pager gets a consistent view of the mapping status of the memory object. 2269 * 2270 * This allows the pager to keep track of how many times a memory object 2271 * has been mapped and with which protections, to decide when it can be 2272 * reclaimed. 2273 */ 2274 2275/* Routine memory_object_map */ 2276kern_return_t memory_object_map 2277( 2278 memory_object_t memory_object, 2279 vm_prot_t prot 2280) 2281{ 2282 return (memory_object->mo_pager_ops->memory_object_map)( 2283 memory_object, 2284 prot); 2285} 2286 2287/* Routine memory_object_last_unmap */ 2288kern_return_t memory_object_last_unmap 2289( 2290 memory_object_t memory_object 2291) 2292{ 2293 return (memory_object->mo_pager_ops->memory_object_last_unmap)( 2294 memory_object); 2295} 2296 2297/* Routine memory_object_data_reclaim */ 2298kern_return_t memory_object_data_reclaim 2299( 2300 memory_object_t memory_object, 2301 boolean_t reclaim_backing_store 2302) 2303{ 2304 if (memory_object->mo_pager_ops->memory_object_data_reclaim == NULL) 2305 return KERN_NOT_SUPPORTED; 2306 return (memory_object->mo_pager_ops->memory_object_data_reclaim)( 2307 memory_object, 2308 reclaim_backing_store); 2309} 2310 2311/* Routine memory_object_create */ 2312kern_return_t memory_object_create 2313( 2314 memory_object_default_t default_memory_manager, 2315 vm_size_t new_memory_object_size, 2316 memory_object_t *new_memory_object 2317) 2318{ 2319 return default_pager_memory_object_create(default_memory_manager, 2320 new_memory_object_size, 2321 new_memory_object); 2322} 2323 2324upl_t 2325convert_port_to_upl( 2326 ipc_port_t port) 2327{ 2328 upl_t upl; 2329 2330 ip_lock(port); 2331 if (!ip_active(port) || (ip_kotype(port) != IKOT_UPL)) { 2332 ip_unlock(port); 2333 return (upl_t)NULL; 2334 } 2335 upl = (upl_t) port->ip_kobject; 2336 ip_unlock(port); 2337 upl_lock(upl); 2338 upl->ref_count+=1; 2339 upl_unlock(upl); 2340 return upl; 2341} 2342 2343mach_port_t 2344convert_upl_to_port( 2345 __unused upl_t upl) 2346{ 2347 return MACH_PORT_NULL; 2348} 2349 2350__private_extern__ void 2351upl_no_senders( 2352 __unused ipc_port_t port, 2353 __unused mach_port_mscount_t mscount) 2354{ 2355 return; 2356} 2357