1/* 2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <sys/errno.h> 30 31#include <mach/mach_types.h> 32#include <mach/mach_traps.h> 33#include <mach/host_priv.h> 34#include <mach/kern_return.h> 35#include <mach/memory_object_control.h> 36#include <mach/memory_object_types.h> 37#include <mach/port.h> 38#include <mach/policy.h> 39#include <mach/upl.h> 40#include <mach/thread_act.h> 41#include <mach/mach_vm.h> 42 43#include <kern/host.h> 44#include <kern/kalloc.h> 45#include <kern/page_decrypt.h> 46#include <kern/queue.h> 47#include <kern/thread.h> 48 49#include <ipc/ipc_port.h> 50#include <ipc/ipc_space.h> 51 52#include <default_pager/default_pager_types.h> 53#include <default_pager/default_pager_object_server.h> 54 55#include <vm/vm_fault.h> 56#include <vm/vm_map.h> 57#include <vm/vm_pageout.h> 58#include <vm/memory_object.h> 59#include <vm/vm_pageout.h> 60#include <vm/vm_protos.h> 61 62 63/* 64 * APPLE PROTECT MEMORY PAGER 65 * 66 * This external memory manager (EMM) handles memory from the encrypted 67 * sections of some executables protected by the DSMOS kernel extension. 68 * 69 * It mostly handles page-in requests (from memory_object_data_request()) by 70 * getting the encrypted data from its backing VM object, itself backed by 71 * the encrypted file, decrypting it and providing it to VM. 72 * 73 * The decrypted pages will never be dirtied, so the memory manager doesn't 74 * need to handle page-out requests (from memory_object_data_return()). The 75 * pages need to be mapped copy-on-write, so that the originals stay clean. 76 * 77 * We don't expect to have to handle a large number of apple-protected 78 * binaries, so the data structures are very simple (simple linked list) 79 * for now. 80 */ 81 82/* forward declarations */ 83void apple_protect_pager_reference(memory_object_t mem_obj); 84void apple_protect_pager_deallocate(memory_object_t mem_obj); 85kern_return_t apple_protect_pager_init(memory_object_t mem_obj, 86 memory_object_control_t control, 87 memory_object_cluster_size_t pg_size); 88kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj); 89kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj, 90 memory_object_offset_t offset, 91 memory_object_cluster_size_t length, 92 vm_prot_t protection_required, 93 memory_object_fault_info_t fault_info); 94kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj, 95 memory_object_offset_t offset, 96 memory_object_cluster_size_t data_cnt, 97 memory_object_offset_t *resid_offset, 98 int *io_error, 99 boolean_t dirty, 100 boolean_t kernel_copy, 101 int upl_flags); 102kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj, 103 memory_object_offset_t offset, 104 memory_object_cluster_size_t data_cnt); 105kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj, 106 memory_object_offset_t offset, 107 memory_object_size_t size, 108 vm_prot_t desired_access); 109kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj, 110 memory_object_offset_t offset, 111 memory_object_size_t length, 112 vm_sync_t sync_flags); 113kern_return_t apple_protect_pager_map(memory_object_t mem_obj, 114 vm_prot_t prot); 115kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj); 116 117/* 118 * Vector of VM operations for this EMM. 119 * These routines are invoked by VM via the memory_object_*() interfaces. 120 */ 121const struct memory_object_pager_ops apple_protect_pager_ops = { 122 apple_protect_pager_reference, 123 apple_protect_pager_deallocate, 124 apple_protect_pager_init, 125 apple_protect_pager_terminate, 126 apple_protect_pager_data_request, 127 apple_protect_pager_data_return, 128 apple_protect_pager_data_initialize, 129 apple_protect_pager_data_unlock, 130 apple_protect_pager_synchronize, 131 apple_protect_pager_map, 132 apple_protect_pager_last_unmap, 133 NULL, /* data_reclaim */ 134 "apple protect pager" 135}; 136 137/* 138 * The "apple_protect_pager" describes a memory object backed by 139 * the "apple protect" EMM. 140 */ 141typedef struct apple_protect_pager { 142 struct ipc_object_header pager_header; /* fake ip_kotype() */ 143 memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */ 144 queue_chain_t pager_queue; /* next & prev pagers */ 145 unsigned int ref_count; /* reference count */ 146 boolean_t is_ready; /* is this pager ready ? */ 147 boolean_t is_mapped; /* is this mem_obj mapped ? */ 148 memory_object_control_t pager_control; /* mem object control handle */ 149 vm_object_t backing_object; /* VM obj w/ encrypted data */ 150 struct pager_crypt_info crypt; 151} *apple_protect_pager_t; 152#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) 153#define pager_ikot pager_header.io_bits 154 155/* 156 * List of memory objects managed by this EMM. 157 * The list is protected by the "apple_protect_pager_lock" lock. 158 */ 159int apple_protect_pager_count = 0; /* number of pagers */ 160int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ 161queue_head_t apple_protect_pager_queue; 162decl_lck_mtx_data(,apple_protect_pager_lock) 163 164/* 165 * Maximum number of unmapped pagers we're willing to keep around. 166 */ 167int apple_protect_pager_cache_limit = 10; 168 169/* 170 * Statistics & counters. 171 */ 172int apple_protect_pager_count_max = 0; 173int apple_protect_pager_count_unmapped_max = 0; 174int apple_protect_pager_num_trim_max = 0; 175int apple_protect_pager_num_trim_total = 0; 176 177 178lck_grp_t apple_protect_pager_lck_grp; 179lck_grp_attr_t apple_protect_pager_lck_grp_attr; 180lck_attr_t apple_protect_pager_lck_attr; 181 182 183/* internal prototypes */ 184apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object, 185 struct pager_crypt_info *crypt_info); 186apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj); 187void apple_protect_pager_dequeue(apple_protect_pager_t pager); 188void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager, 189 boolean_t locked); 190void apple_protect_pager_terminate_internal(apple_protect_pager_t pager); 191void apple_protect_pager_trim(void); 192 193 194#if DEBUG 195int apple_protect_pagerdebug = 0; 196#define PAGER_ALL 0xffffffff 197#define PAGER_INIT 0x00000001 198#define PAGER_PAGEIN 0x00000002 199 200#define PAGER_DEBUG(LEVEL, A) \ 201 MACRO_BEGIN \ 202 if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ 203 printf A; \ 204 } \ 205 MACRO_END 206#else 207#define PAGER_DEBUG(LEVEL, A) 208#endif 209 210 211void 212apple_protect_pager_bootstrap(void) 213{ 214 lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr); 215 lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr); 216 lck_attr_setdefault(&apple_protect_pager_lck_attr); 217 lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr); 218 queue_init(&apple_protect_pager_queue); 219} 220 221/* 222 * apple_protect_pager_init() 223 * 224 * Initialize the memory object and makes it ready to be used and mapped. 225 */ 226kern_return_t 227apple_protect_pager_init( 228 memory_object_t mem_obj, 229 memory_object_control_t control, 230#if !DEBUG 231 __unused 232#endif 233 memory_object_cluster_size_t pg_size) 234{ 235 apple_protect_pager_t pager; 236 kern_return_t kr; 237 memory_object_attr_info_data_t attributes; 238 239 PAGER_DEBUG(PAGER_ALL, 240 ("apple_protect_pager_init: %p, %p, %x\n", 241 mem_obj, control, pg_size)); 242 243 if (control == MEMORY_OBJECT_CONTROL_NULL) 244 return KERN_INVALID_ARGUMENT; 245 246 pager = apple_protect_pager_lookup(mem_obj); 247 248 memory_object_control_reference(control); 249 250 pager->pager_control = control; 251 252 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; 253 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ 254 attributes.cluster_size = (1 << (PAGE_SHIFT)); 255 attributes.may_cache_object = FALSE; 256 attributes.temporary = TRUE; 257 258 kr = memory_object_change_attributes( 259 control, 260 MEMORY_OBJECT_ATTRIBUTE_INFO, 261 (memory_object_info_t) &attributes, 262 MEMORY_OBJECT_ATTR_INFO_COUNT); 263 if (kr != KERN_SUCCESS) 264 panic("apple_protect_pager_init: " 265 "memory_object_change_attributes() failed"); 266 267 return KERN_SUCCESS; 268} 269 270/* 271 * apple_protect_data_return() 272 * 273 * Handles page-out requests from VM. This should never happen since 274 * the pages provided by this EMM are not supposed to be dirty or dirtied 275 * and VM should simply discard the contents and reclaim the pages if it 276 * needs to. 277 */ 278kern_return_t 279apple_protect_pager_data_return( 280 __unused memory_object_t mem_obj, 281 __unused memory_object_offset_t offset, 282 __unused memory_object_cluster_size_t data_cnt, 283 __unused memory_object_offset_t *resid_offset, 284 __unused int *io_error, 285 __unused boolean_t dirty, 286 __unused boolean_t kernel_copy, 287 __unused int upl_flags) 288{ 289 panic("apple_protect_pager_data_return: should never get called"); 290 return KERN_FAILURE; 291} 292 293kern_return_t 294apple_protect_pager_data_initialize( 295 __unused memory_object_t mem_obj, 296 __unused memory_object_offset_t offset, 297 __unused memory_object_cluster_size_t data_cnt) 298{ 299 panic("apple_protect_pager_data_initialize: should never get called"); 300 return KERN_FAILURE; 301} 302 303kern_return_t 304apple_protect_pager_data_unlock( 305 __unused memory_object_t mem_obj, 306 __unused memory_object_offset_t offset, 307 __unused memory_object_size_t size, 308 __unused vm_prot_t desired_access) 309{ 310 return KERN_FAILURE; 311} 312 313/* 314 * apple_protect_pager_data_request() 315 * 316 * Handles page-in requests from VM. 317 */ 318kern_return_t 319apple_protect_pager_data_request( 320 memory_object_t mem_obj, 321 memory_object_offset_t offset, 322 memory_object_cluster_size_t length, 323#if !DEBUG 324 __unused 325#endif 326 vm_prot_t protection_required, 327 memory_object_fault_info_t mo_fault_info) 328{ 329 apple_protect_pager_t pager; 330 memory_object_control_t mo_control; 331 upl_t upl; 332 int upl_flags; 333 upl_size_t upl_size; 334 upl_page_info_t *upl_pl; 335 unsigned int pl_count; 336 vm_object_t src_object, dst_object; 337 kern_return_t kr, retval; 338 vm_map_offset_t kernel_mapping; 339 vm_offset_t src_vaddr, dst_vaddr; 340 vm_offset_t cur_offset; 341 vm_offset_t offset_in_page; 342 vm_map_entry_t map_entry; 343 kern_return_t error_code; 344 vm_prot_t prot; 345 vm_page_t src_page, top_page; 346 int interruptible; 347 struct vm_object_fault_info fault_info; 348 int ret; 349 350 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); 351 352 retval = KERN_SUCCESS; 353 src_object = VM_OBJECT_NULL; 354 kernel_mapping = 0; 355 upl = NULL; 356 upl_pl = NULL; 357 fault_info = *((struct vm_object_fault_info *) mo_fault_info); 358 fault_info.stealth = TRUE; 359 fault_info.io_sync = FALSE; 360 fault_info.mark_zf_absent = FALSE; 361 fault_info.batch_pmap_op = FALSE; 362 interruptible = fault_info.interruptible; 363 364 pager = apple_protect_pager_lookup(mem_obj); 365 assert(pager->is_ready); 366 assert(pager->ref_count > 1); /* pager is alive and mapped */ 367 368 PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); 369 370 /* 371 * Gather in a UPL all the VM pages requested by VM. 372 */ 373 mo_control = pager->pager_control; 374 375 upl_size = length; 376 upl_flags = 377 UPL_RET_ONLY_ABSENT | 378 UPL_SET_LITE | 379 UPL_NO_SYNC | 380 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ 381 UPL_SET_INTERNAL; 382 pl_count = 0; 383 kr = memory_object_upl_request(mo_control, 384 offset, upl_size, 385 &upl, NULL, NULL, upl_flags); 386 if (kr != KERN_SUCCESS) { 387 retval = kr; 388 goto done; 389 } 390 dst_object = mo_control->moc_object; 391 assert(dst_object != VM_OBJECT_NULL); 392 393 394 /* 395 * Reserve 2 virtual pages in the kernel address space to map each 396 * source and destination physical pages when it's their turn to 397 * be processed. 398 */ 399 vm_object_reference(kernel_object); /* ref. for mapping */ 400 kr = vm_map_find_space(kernel_map, 401 &kernel_mapping, 402 2 * PAGE_SIZE_64, 403 0, 404 0, 405 &map_entry); 406 if (kr != KERN_SUCCESS) { 407 vm_object_deallocate(kernel_object); 408 retval = kr; 409 goto done; 410 } 411 map_entry->object.vm_object = kernel_object; 412 map_entry->offset = kernel_mapping; 413 vm_map_unlock(kernel_map); 414 src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); 415 dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64); 416 417 /* 418 * We'll map the encrypted data in the kernel address space from the 419 * backing VM object (itself backed by the encrypted file via 420 * the vnode pager). 421 */ 422 src_object = pager->backing_object; 423 assert(src_object != VM_OBJECT_NULL); 424 vm_object_reference(src_object); /* to keep the source object alive */ 425 426 /* 427 * Fill in the contents of the pages requested by VM. 428 */ 429 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); 430 pl_count = length / PAGE_SIZE; 431 for (cur_offset = 0; 432 retval == KERN_SUCCESS && cur_offset < length; 433 cur_offset += PAGE_SIZE) { 434 ppnum_t dst_pnum; 435 436 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { 437 /* this page is not in the UPL: skip it */ 438 continue; 439 } 440 441 /* 442 * Map the source (encrypted) page in the kernel's 443 * virtual address space. 444 * We already hold a reference on the src_object. 445 */ 446 retry_src_fault: 447 vm_object_lock(src_object); 448 vm_object_paging_begin(src_object); 449 error_code = 0; 450 prot = VM_PROT_READ; 451 src_page = VM_PAGE_NULL; 452 kr = vm_fault_page(src_object, 453 offset + cur_offset, 454 VM_PROT_READ, 455 FALSE, 456 FALSE, /* src_page not looked up */ 457 &prot, 458 &src_page, 459 &top_page, 460 NULL, 461 &error_code, 462 FALSE, 463 FALSE, 464 &fault_info); 465 switch (kr) { 466 case VM_FAULT_SUCCESS: 467 break; 468 case VM_FAULT_RETRY: 469 goto retry_src_fault; 470 case VM_FAULT_MEMORY_SHORTAGE: 471 if (vm_page_wait(interruptible)) { 472 goto retry_src_fault; 473 } 474 /* fall thru */ 475 case VM_FAULT_INTERRUPTED: 476 retval = MACH_SEND_INTERRUPTED; 477 goto done; 478 case VM_FAULT_SUCCESS_NO_VM_PAGE: 479 /* success but no VM page: fail */ 480 vm_object_paging_end(src_object); 481 vm_object_unlock(src_object); 482 /*FALLTHROUGH*/ 483 case VM_FAULT_MEMORY_ERROR: 484 /* the page is not there ! */ 485 if (error_code) { 486 retval = error_code; 487 } else { 488 retval = KERN_MEMORY_ERROR; 489 } 490 goto done; 491 default: 492 panic("apple_protect_pager_data_request: " 493 "vm_fault_page() unexpected error 0x%x\n", 494 kr); 495 } 496 assert(src_page != VM_PAGE_NULL); 497 assert(src_page->busy); 498 499 if (!src_page->active && 500 !src_page->inactive && 501 !src_page->throttled) { 502 vm_page_lockspin_queues(); 503 if (!src_page->active && 504 !src_page->inactive && 505 !src_page->throttled) { 506 vm_page_deactivate(src_page); 507 } 508 vm_page_unlock_queues(); 509 } 510 511 /* 512 * Establish an explicit mapping of the source 513 * physical page. 514 */ 515 pmap_enter(kernel_pmap, 516 kernel_mapping, 517 src_page->phys_page, 518 VM_PROT_READ, 519 VM_PROT_NONE, 520 0, 521 TRUE); 522 /* 523 * Establish an explicit pmap mapping of the destination 524 * physical page. 525 * We can't do a regular VM mapping because the VM page 526 * is "busy". 527 */ 528 dst_pnum = (ppnum_t) 529 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); 530 assert(dst_pnum != 0); 531 pmap_enter(kernel_pmap, 532 kernel_mapping + PAGE_SIZE_64, 533 dst_pnum, 534 VM_PROT_READ | VM_PROT_WRITE, 535 VM_PROT_NONE, 536 0, 537 TRUE); 538 539 /* 540 * Decrypt the encrypted contents of the source page 541 * into the destination page. 542 */ 543 for (offset_in_page = 0; 544 offset_in_page < PAGE_SIZE; 545 offset_in_page += 4096) { 546 ret = pager->crypt.page_decrypt((const void *) 547 (src_vaddr + 548 offset_in_page), 549 (void *) 550 (dst_vaddr + 551 offset_in_page), 552 (offset + 553 cur_offset + 554 offset_in_page), 555 pager->crypt.crypt_ops); 556 if (ret) { 557 break; 558 } 559 } 560 if (ret) { 561 /* 562 * Decryption failed. Abort the fault. 563 */ 564 retval = KERN_ABORTED; 565 } else { 566 /* 567 * Validate the original page... 568 */ 569 if (src_page->object->code_signed) { 570 vm_page_validate_cs_mapped( 571 src_page, 572 (const void *) src_vaddr); 573 } 574 /* 575 * ... and transfer the results to the destination page. 576 */ 577 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, 578 src_page->cs_validated); 579 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, 580 src_page->cs_tainted); 581 } 582 583 /* 584 * Remove the pmap mapping of the source and destination pages 585 * in the kernel. 586 */ 587 pmap_remove(kernel_pmap, 588 (addr64_t) kernel_mapping, 589 (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64))); 590 591 /* 592 * Cleanup the result of vm_fault_page() of the source page. 593 */ 594 PAGE_WAKEUP_DONE(src_page); 595 vm_object_paging_end(src_page->object); 596 vm_object_unlock(src_page->object); 597 if (top_page != VM_PAGE_NULL) { 598 vm_object_t top_object; 599 600 top_object = top_page->object; 601 vm_object_lock(top_object); 602 VM_PAGE_FREE(top_page); 603 vm_object_paging_end(top_object); 604 vm_object_unlock(top_object); 605 } 606 } 607 608done: 609 if (upl != NULL) { 610 /* clean up the UPL */ 611 612 /* 613 * The pages are currently dirty because we've just been 614 * writing on them, but as far as we're concerned, they're 615 * clean since they contain their "original" contents as 616 * provided by us, the pager. 617 * Tell the UPL to mark them "clean". 618 */ 619 upl_clear_dirty(upl, TRUE); 620 621 /* abort or commit the UPL */ 622 if (retval != KERN_SUCCESS) { 623 upl_abort(upl, 0); 624 if (retval == KERN_ABORTED) { 625 wait_result_t wait_result; 626 627 /* 628 * We aborted the fault and did not provide 629 * any contents for the requested pages but 630 * the pages themselves are not invalid, so 631 * let's return success and let the caller 632 * retry the fault, in case it might succeed 633 * later (when the decryption code is up and 634 * running in the kernel, for example). 635 */ 636 retval = KERN_SUCCESS; 637 /* 638 * Wait a little bit first to avoid using 639 * too much CPU time retrying and failing 640 * the same fault over and over again. 641 */ 642 wait_result = assert_wait_timeout( 643 (event_t) apple_protect_pager_data_request, 644 THREAD_UNINT, 645 10000, /* 10ms */ 646 NSEC_PER_USEC); 647 assert(wait_result == THREAD_WAITING); 648 wait_result = thread_block(THREAD_CONTINUE_NULL); 649 assert(wait_result == THREAD_TIMED_OUT); 650 } 651 } else { 652 boolean_t empty; 653 upl_commit_range(upl, 0, upl->size, 654 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, 655 upl_pl, pl_count, &empty); 656 } 657 658 /* and deallocate the UPL */ 659 upl_deallocate(upl); 660 upl = NULL; 661 } 662 if (kernel_mapping != 0) { 663 /* clean up the mapping of the source and destination pages */ 664 kr = vm_map_remove(kernel_map, 665 kernel_mapping, 666 kernel_mapping + (2 * PAGE_SIZE_64), 667 VM_MAP_NO_FLAGS); 668 assert(kr == KERN_SUCCESS); 669 kernel_mapping = 0; 670 src_vaddr = 0; 671 dst_vaddr = 0; 672 } 673 if (src_object != VM_OBJECT_NULL) { 674 vm_object_deallocate(src_object); 675 } 676 677 return retval; 678} 679 680/* 681 * apple_protect_pager_reference() 682 * 683 * Get a reference on this memory object. 684 * For external usage only. Assumes that the initial reference count is not 0, 685 * i.e one should not "revive" a dead pager this way. 686 */ 687void 688apple_protect_pager_reference( 689 memory_object_t mem_obj) 690{ 691 apple_protect_pager_t pager; 692 693 pager = apple_protect_pager_lookup(mem_obj); 694 695 lck_mtx_lock(&apple_protect_pager_lock); 696 assert(pager->ref_count > 0); 697 pager->ref_count++; 698 lck_mtx_unlock(&apple_protect_pager_lock); 699} 700 701 702/* 703 * apple_protect_pager_dequeue: 704 * 705 * Removes a pager from the list of pagers. 706 * 707 * The caller must hold "apple_protect_pager_lock". 708 */ 709void 710apple_protect_pager_dequeue( 711 apple_protect_pager_t pager) 712{ 713 assert(!pager->is_mapped); 714 715 queue_remove(&apple_protect_pager_queue, 716 pager, 717 apple_protect_pager_t, 718 pager_queue); 719 pager->pager_queue.next = NULL; 720 pager->pager_queue.prev = NULL; 721 722 apple_protect_pager_count--; 723} 724 725/* 726 * apple_protect_pager_terminate_internal: 727 * 728 * Trigger the asynchronous termination of the memory object associated 729 * with this pager. 730 * When the memory object is terminated, there will be one more call 731 * to memory_object_deallocate() (i.e. apple_protect_pager_deallocate()) 732 * to finish the clean up. 733 * 734 * "apple_protect_pager_lock" should not be held by the caller. 735 * We don't need the lock because the pager has already been removed from 736 * the pagers' list and is now ours exclusively. 737 */ 738void 739apple_protect_pager_terminate_internal( 740 apple_protect_pager_t pager) 741{ 742 assert(pager->is_ready); 743 assert(!pager->is_mapped); 744 745 if (pager->backing_object != VM_OBJECT_NULL) { 746 vm_object_deallocate(pager->backing_object); 747 pager->backing_object = VM_OBJECT_NULL; 748 } 749 750 /* deallocate any crypt module data */ 751 if(pager->crypt.crypt_end) 752 pager->crypt.crypt_end(pager->crypt.crypt_ops); 753 754 /* trigger the destruction of the memory object */ 755 memory_object_destroy(pager->pager_control, 0); 756} 757 758/* 759 * apple_protect_pager_deallocate_internal() 760 * 761 * Release a reference on this pager and free it when the last 762 * reference goes away. 763 * Can be called with apple_protect_pager_lock held or not but always returns 764 * with it unlocked. 765 */ 766void 767apple_protect_pager_deallocate_internal( 768 apple_protect_pager_t pager, 769 boolean_t locked) 770{ 771 boolean_t needs_trimming; 772 int count_unmapped; 773 774 if (! locked) { 775 lck_mtx_lock(&apple_protect_pager_lock); 776 } 777 778 count_unmapped = (apple_protect_pager_count - 779 apple_protect_pager_count_mapped); 780 if (count_unmapped > apple_protect_pager_cache_limit) { 781 /* we have too many unmapped pagers: trim some */ 782 needs_trimming = TRUE; 783 } else { 784 needs_trimming = FALSE; 785 } 786 787 /* drop a reference on this pager */ 788 pager->ref_count--; 789 790 if (pager->ref_count == 1) { 791 /* 792 * Only the "named" reference is left, which means that 793 * no one is really holding on to this pager anymore. 794 * Terminate it. 795 */ 796 apple_protect_pager_dequeue(pager); 797 /* the pager is all ours: no need for the lock now */ 798 lck_mtx_unlock(&apple_protect_pager_lock); 799 apple_protect_pager_terminate_internal(pager); 800 } else if (pager->ref_count == 0) { 801 /* 802 * Dropped the existence reference; the memory object has 803 * been terminated. Do some final cleanup and release the 804 * pager structure. 805 */ 806 lck_mtx_unlock(&apple_protect_pager_lock); 807 if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { 808 memory_object_control_deallocate(pager->pager_control); 809 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; 810 } 811 kfree(pager, sizeof (*pager)); 812 pager = APPLE_PROTECT_PAGER_NULL; 813 } else { 814 /* there are still plenty of references: keep going... */ 815 lck_mtx_unlock(&apple_protect_pager_lock); 816 } 817 818 if (needs_trimming) { 819 apple_protect_pager_trim(); 820 } 821 /* caution: lock is not held on return... */ 822} 823 824/* 825 * apple_protect_pager_deallocate() 826 * 827 * Release a reference on this pager and free it when the last 828 * reference goes away. 829 */ 830void 831apple_protect_pager_deallocate( 832 memory_object_t mem_obj) 833{ 834 apple_protect_pager_t pager; 835 836 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj)); 837 pager = apple_protect_pager_lookup(mem_obj); 838 apple_protect_pager_deallocate_internal(pager, FALSE); 839} 840 841/* 842 * 843 */ 844kern_return_t 845apple_protect_pager_terminate( 846#if !DEBUG 847 __unused 848#endif 849 memory_object_t mem_obj) 850{ 851 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj)); 852 853 return KERN_SUCCESS; 854} 855 856/* 857 * 858 */ 859kern_return_t 860apple_protect_pager_synchronize( 861 memory_object_t mem_obj, 862 memory_object_offset_t offset, 863 memory_object_size_t length, 864 __unused vm_sync_t sync_flags) 865{ 866 apple_protect_pager_t pager; 867 868 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj)); 869 870 pager = apple_protect_pager_lookup(mem_obj); 871 872 memory_object_synchronize_completed(pager->pager_control, 873 offset, length); 874 875 return KERN_SUCCESS; 876} 877 878/* 879 * apple_protect_pager_map() 880 * 881 * This allows VM to let us, the EMM, know that this memory object 882 * is currently mapped one or more times. This is called by VM each time 883 * the memory object gets mapped and we take one extra reference on the 884 * memory object to account for all its mappings. 885 */ 886kern_return_t 887apple_protect_pager_map( 888 memory_object_t mem_obj, 889 __unused vm_prot_t prot) 890{ 891 apple_protect_pager_t pager; 892 893 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj)); 894 895 pager = apple_protect_pager_lookup(mem_obj); 896 897 lck_mtx_lock(&apple_protect_pager_lock); 898 assert(pager->is_ready); 899 assert(pager->ref_count > 0); /* pager is alive */ 900 if (pager->is_mapped == FALSE) { 901 /* 902 * First mapping of this pager: take an extra reference 903 * that will remain until all the mappings of this pager 904 * are removed. 905 */ 906 pager->is_mapped = TRUE; 907 pager->ref_count++; 908 apple_protect_pager_count_mapped++; 909 } 910 lck_mtx_unlock(&apple_protect_pager_lock); 911 912 return KERN_SUCCESS; 913} 914 915/* 916 * apple_protect_pager_last_unmap() 917 * 918 * This is called by VM when this memory object is no longer mapped anywhere. 919 */ 920kern_return_t 921apple_protect_pager_last_unmap( 922 memory_object_t mem_obj) 923{ 924 apple_protect_pager_t pager; 925 int count_unmapped; 926 927 PAGER_DEBUG(PAGER_ALL, 928 ("apple_protect_pager_last_unmap: %p\n", mem_obj)); 929 930 pager = apple_protect_pager_lookup(mem_obj); 931 932 lck_mtx_lock(&apple_protect_pager_lock); 933 if (pager->is_mapped) { 934 /* 935 * All the mappings are gone, so let go of the one extra 936 * reference that represents all the mappings of this pager. 937 */ 938 apple_protect_pager_count_mapped--; 939 count_unmapped = (apple_protect_pager_count - 940 apple_protect_pager_count_mapped); 941 if (count_unmapped > apple_protect_pager_count_unmapped_max) { 942 apple_protect_pager_count_unmapped_max = count_unmapped; 943 } 944 pager->is_mapped = FALSE; 945 apple_protect_pager_deallocate_internal(pager, TRUE); 946 /* caution: deallocate_internal() released the lock ! */ 947 } else { 948 lck_mtx_unlock(&apple_protect_pager_lock); 949 } 950 951 return KERN_SUCCESS; 952} 953 954 955/* 956 * 957 */ 958apple_protect_pager_t 959apple_protect_pager_lookup( 960 memory_object_t mem_obj) 961{ 962 apple_protect_pager_t pager; 963 964 pager = (apple_protect_pager_t) mem_obj; 965 assert(pager->pager_ops == &apple_protect_pager_ops); 966 assert(pager->ref_count > 0); 967 return pager; 968} 969 970apple_protect_pager_t 971apple_protect_pager_create( 972 vm_object_t backing_object, 973 struct pager_crypt_info *crypt_info) 974{ 975 apple_protect_pager_t pager, pager2; 976 memory_object_control_t control; 977 kern_return_t kr; 978 979 pager = (apple_protect_pager_t) kalloc(sizeof (*pager)); 980 if (pager == APPLE_PROTECT_PAGER_NULL) { 981 return APPLE_PROTECT_PAGER_NULL; 982 } 983 984 /* 985 * The vm_map call takes both named entry ports and raw memory 986 * objects in the same parameter. We need to make sure that 987 * vm_map does not see this object as a named entry port. So, 988 * we reserve the first word in the object for a fake ip_kotype 989 * setting - that will tell vm_map to use it as a memory object. 990 */ 991 pager->pager_ops = &apple_protect_pager_ops; 992 pager->pager_ikot = IKOT_MEMORY_OBJECT; 993 pager->is_ready = FALSE;/* not ready until it has a "name" */ 994 pager->ref_count = 2; /* existence + setup reference */ 995 pager->is_mapped = FALSE; 996 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; 997 pager->backing_object = backing_object; 998 pager->crypt = *crypt_info; 999 1000 vm_object_reference(backing_object); 1001 1002 lck_mtx_lock(&apple_protect_pager_lock); 1003 /* see if anyone raced us to create a pager for the same object */ 1004 queue_iterate(&apple_protect_pager_queue, 1005 pager2, 1006 apple_protect_pager_t, 1007 pager_queue) { 1008 if (pager2->backing_object == backing_object) { 1009 break; 1010 } 1011 } 1012 if (! queue_end(&apple_protect_pager_queue, 1013 (queue_entry_t) pager2)) { 1014 /* while we hold the lock, transfer our setup ref to winner */ 1015 pager2->ref_count++; 1016 /* we lost the race, down with the loser... */ 1017 lck_mtx_unlock(&apple_protect_pager_lock); 1018 vm_object_deallocate(pager->backing_object); 1019 pager->backing_object = VM_OBJECT_NULL; 1020 kfree(pager, sizeof (*pager)); 1021 /* ... and go with the winner */ 1022 pager = pager2; 1023 /* let the winner make sure the pager gets ready */ 1024 return pager; 1025 } 1026 1027 /* enter new pager at the head of our list of pagers */ 1028 queue_enter_first(&apple_protect_pager_queue, 1029 pager, 1030 apple_protect_pager_t, 1031 pager_queue); 1032 apple_protect_pager_count++; 1033 if (apple_protect_pager_count > apple_protect_pager_count_max) { 1034 apple_protect_pager_count_max = apple_protect_pager_count; 1035 } 1036 lck_mtx_unlock(&apple_protect_pager_lock); 1037 1038 kr = memory_object_create_named((memory_object_t) pager, 1039 0, 1040 &control); 1041 assert(kr == KERN_SUCCESS); 1042 1043 lck_mtx_lock(&apple_protect_pager_lock); 1044 /* the new pager is now ready to be used */ 1045 pager->is_ready = TRUE; 1046 lck_mtx_unlock(&apple_protect_pager_lock); 1047 1048 /* wakeup anyone waiting for this pager to be ready */ 1049 thread_wakeup(&pager->is_ready); 1050 1051 return pager; 1052} 1053 1054/* 1055 * apple_protect_pager_setup() 1056 * 1057 * Provide the caller with a memory object backed by the provided 1058 * "backing_object" VM object. If such a memory object already exists, 1059 * re-use it, otherwise create a new memory object. 1060 */ 1061memory_object_t 1062apple_protect_pager_setup( 1063 vm_object_t backing_object, 1064 struct pager_crypt_info *crypt_info) 1065{ 1066 apple_protect_pager_t pager; 1067 1068 lck_mtx_lock(&apple_protect_pager_lock); 1069 1070 queue_iterate(&apple_protect_pager_queue, 1071 pager, 1072 apple_protect_pager_t, 1073 pager_queue) { 1074 if (pager->backing_object == backing_object) { 1075 /* For the same object we must always use the same protection options */ 1076 if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) && 1077 (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) { 1078 lck_mtx_unlock(&apple_protect_pager_lock); 1079 return MEMORY_OBJECT_NULL; 1080 } 1081 break; 1082 } 1083 } 1084 if (queue_end(&apple_protect_pager_queue, 1085 (queue_entry_t) pager)) { 1086 /* no existing pager for this backing object */ 1087 pager = APPLE_PROTECT_PAGER_NULL; 1088 } else { 1089 /* make sure pager doesn't disappear */ 1090 pager->ref_count++; 1091 } 1092 1093 lck_mtx_unlock(&apple_protect_pager_lock); 1094 1095 if (pager == APPLE_PROTECT_PAGER_NULL) { 1096 pager = apple_protect_pager_create(backing_object, crypt_info); 1097 if (pager == APPLE_PROTECT_PAGER_NULL) { 1098 return MEMORY_OBJECT_NULL; 1099 } 1100 } 1101 1102 lck_mtx_lock(&apple_protect_pager_lock); 1103 while (!pager->is_ready) { 1104 lck_mtx_sleep(&apple_protect_pager_lock, 1105 LCK_SLEEP_DEFAULT, 1106 &pager->is_ready, 1107 THREAD_UNINT); 1108 } 1109 lck_mtx_unlock(&apple_protect_pager_lock); 1110 1111 return (memory_object_t) pager; 1112} 1113 1114void 1115apple_protect_pager_trim(void) 1116{ 1117 apple_protect_pager_t pager, prev_pager; 1118 queue_head_t trim_queue; 1119 int num_trim; 1120 int count_unmapped; 1121 1122 lck_mtx_lock(&apple_protect_pager_lock); 1123 1124 /* 1125 * We have too many pagers, try and trim some unused ones, 1126 * starting with the oldest pager at the end of the queue. 1127 */ 1128 queue_init(&trim_queue); 1129 num_trim = 0; 1130 1131 for (pager = (apple_protect_pager_t) 1132 queue_last(&apple_protect_pager_queue); 1133 !queue_end(&apple_protect_pager_queue, 1134 (queue_entry_t) pager); 1135 pager = prev_pager) { 1136 /* get prev elt before we dequeue */ 1137 prev_pager = (apple_protect_pager_t) 1138 queue_prev(&pager->pager_queue); 1139 1140 if (pager->ref_count == 2 && 1141 pager->is_ready && 1142 !pager->is_mapped) { 1143 /* this pager can be trimmed */ 1144 num_trim++; 1145 /* remove this pager from the main list ... */ 1146 apple_protect_pager_dequeue(pager); 1147 /* ... and add it to our trim queue */ 1148 queue_enter_first(&trim_queue, 1149 pager, 1150 apple_protect_pager_t, 1151 pager_queue); 1152 1153 count_unmapped = (apple_protect_pager_count - 1154 apple_protect_pager_count_mapped); 1155 if (count_unmapped <= apple_protect_pager_cache_limit) { 1156 /* we have enough pagers to trim */ 1157 break; 1158 } 1159 } 1160 } 1161 if (num_trim > apple_protect_pager_num_trim_max) { 1162 apple_protect_pager_num_trim_max = num_trim; 1163 } 1164 apple_protect_pager_num_trim_total += num_trim; 1165 1166 lck_mtx_unlock(&apple_protect_pager_lock); 1167 1168 /* terminate the trimmed pagers */ 1169 while (!queue_empty(&trim_queue)) { 1170 queue_remove_first(&trim_queue, 1171 pager, 1172 apple_protect_pager_t, 1173 pager_queue); 1174 pager->pager_queue.next = NULL; 1175 pager->pager_queue.prev = NULL; 1176 assert(pager->ref_count == 2); 1177 /* 1178 * We can't call deallocate_internal() because the pager 1179 * has already been dequeued, but we still need to remove 1180 * a reference. 1181 */ 1182 pager->ref_count--; 1183 apple_protect_pager_terminate_internal(pager); 1184 } 1185} 1186