1/* 2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <sys/errno.h> 30 31#include <mach/mach_types.h> 32#include <mach/mach_traps.h> 33#include <mach/host_priv.h> 34#include <mach/kern_return.h> 35#include <mach/memory_object_control.h> 36#include <mach/memory_object_types.h> 37#include <mach/port.h> 38#include <mach/policy.h> 39#include <mach/upl.h> 40#include <mach/thread_act.h> 41#include <mach/mach_vm.h> 42 43#include <kern/host.h> 44#include <kern/kalloc.h> 45#include <kern/page_decrypt.h> 46#include <kern/queue.h> 47#include <kern/thread.h> 48 49#include <ipc/ipc_port.h> 50#include <ipc/ipc_space.h> 51 52#include <default_pager/default_pager_types.h> 53#include <default_pager/default_pager_object_server.h> 54 55#include <vm/vm_fault.h> 56#include <vm/vm_map.h> 57#include <vm/vm_pageout.h> 58#include <vm/memory_object.h> 59#include <vm/vm_pageout.h> 60#include <vm/vm_protos.h> 61 62 63/* 64 * APPLE PROTECT MEMORY PAGER 65 * 66 * This external memory manager (EMM) handles memory from the encrypted 67 * sections of some executables protected by the DSMOS kernel extension. 68 * 69 * It mostly handles page-in requests (from memory_object_data_request()) by 70 * getting the encrypted data from its backing VM object, itself backed by 71 * the encrypted file, decrypting it and providing it to VM. 72 * 73 * The decrypted pages will never be dirtied, so the memory manager doesn't 74 * need to handle page-out requests (from memory_object_data_return()). The 75 * pages need to be mapped copy-on-write, so that the originals stay clean. 76 * 77 * We don't expect to have to handle a large number of apple-protected 78 * binaries, so the data structures are very simple (simple linked list) 79 * for now. 80 */ 81 82/* forward declarations */ 83void apple_protect_pager_reference(memory_object_t mem_obj); 84void apple_protect_pager_deallocate(memory_object_t mem_obj); 85kern_return_t apple_protect_pager_init(memory_object_t mem_obj, 86 memory_object_control_t control, 87 memory_object_cluster_size_t pg_size); 88kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj); 89kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj, 90 memory_object_offset_t offset, 91 memory_object_cluster_size_t length, 92 vm_prot_t protection_required, 93 memory_object_fault_info_t fault_info); 94kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj, 95 memory_object_offset_t offset, 96 memory_object_cluster_size_t data_cnt, 97 memory_object_offset_t *resid_offset, 98 int *io_error, 99 boolean_t dirty, 100 boolean_t kernel_copy, 101 int upl_flags); 102kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj, 103 memory_object_offset_t offset, 104 memory_object_cluster_size_t data_cnt); 105kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj, 106 memory_object_offset_t offset, 107 memory_object_size_t size, 108 vm_prot_t desired_access); 109kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj, 110 memory_object_offset_t offset, 111 memory_object_size_t length, 112 vm_sync_t sync_flags); 113kern_return_t apple_protect_pager_map(memory_object_t mem_obj, 114 vm_prot_t prot); 115kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj); 116 117/* 118 * Vector of VM operations for this EMM. 119 * These routines are invoked by VM via the memory_object_*() interfaces. 120 */ 121const struct memory_object_pager_ops apple_protect_pager_ops = { 122 apple_protect_pager_reference, 123 apple_protect_pager_deallocate, 124 apple_protect_pager_init, 125 apple_protect_pager_terminate, 126 apple_protect_pager_data_request, 127 apple_protect_pager_data_return, 128 apple_protect_pager_data_initialize, 129 apple_protect_pager_data_unlock, 130 apple_protect_pager_synchronize, 131 apple_protect_pager_map, 132 apple_protect_pager_last_unmap, 133 NULL, /* data_reclaim */ 134 "apple protect pager" 135}; 136 137/* 138 * The "apple_protect_pager" describes a memory object backed by 139 * the "apple protect" EMM. 140 */ 141typedef struct apple_protect_pager { 142 struct ipc_object_header pager_header; /* fake ip_kotype() */ 143 memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */ 144 queue_chain_t pager_queue; /* next & prev pagers */ 145 unsigned int ref_count; /* reference count */ 146 boolean_t is_ready; /* is this pager ready ? */ 147 boolean_t is_mapped; /* is this mem_obj mapped ? */ 148 memory_object_control_t pager_control; /* mem object control handle */ 149 vm_object_t backing_object; /* VM obj w/ encrypted data */ 150 struct pager_crypt_info crypt; 151} *apple_protect_pager_t; 152#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) 153#define pager_ikot pager_header.io_bits 154 155/* 156 * List of memory objects managed by this EMM. 157 * The list is protected by the "apple_protect_pager_lock" lock. 158 */ 159int apple_protect_pager_count = 0; /* number of pagers */ 160int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ 161queue_head_t apple_protect_pager_queue; 162decl_lck_mtx_data(,apple_protect_pager_lock) 163 164/* 165 * Maximum number of unmapped pagers we're willing to keep around. 166 */ 167int apple_protect_pager_cache_limit = 10; 168 169/* 170 * Statistics & counters. 171 */ 172int apple_protect_pager_count_max = 0; 173int apple_protect_pager_count_unmapped_max = 0; 174int apple_protect_pager_num_trim_max = 0; 175int apple_protect_pager_num_trim_total = 0; 176 177 178lck_grp_t apple_protect_pager_lck_grp; 179lck_grp_attr_t apple_protect_pager_lck_grp_attr; 180lck_attr_t apple_protect_pager_lck_attr; 181 182 183/* internal prototypes */ 184apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object, 185 struct pager_crypt_info *crypt_info); 186apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj); 187void apple_protect_pager_dequeue(apple_protect_pager_t pager); 188void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager, 189 boolean_t locked); 190void apple_protect_pager_terminate_internal(apple_protect_pager_t pager); 191void apple_protect_pager_trim(void); 192 193 194#if DEBUG 195int apple_protect_pagerdebug = 0; 196#define PAGER_ALL 0xffffffff 197#define PAGER_INIT 0x00000001 198#define PAGER_PAGEIN 0x00000002 199 200#define PAGER_DEBUG(LEVEL, A) \ 201 MACRO_BEGIN \ 202 if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ 203 printf A; \ 204 } \ 205 MACRO_END 206#else 207#define PAGER_DEBUG(LEVEL, A) 208#endif 209 210 211void 212apple_protect_pager_bootstrap(void) 213{ 214 lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr); 215 lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr); 216 lck_attr_setdefault(&apple_protect_pager_lck_attr); 217 lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr); 218 queue_init(&apple_protect_pager_queue); 219} 220 221/* 222 * apple_protect_pager_init() 223 * 224 * Initialize the memory object and makes it ready to be used and mapped. 225 */ 226kern_return_t 227apple_protect_pager_init( 228 memory_object_t mem_obj, 229 memory_object_control_t control, 230#if !DEBUG 231 __unused 232#endif 233 memory_object_cluster_size_t pg_size) 234{ 235 apple_protect_pager_t pager; 236 kern_return_t kr; 237 memory_object_attr_info_data_t attributes; 238 239 PAGER_DEBUG(PAGER_ALL, 240 ("apple_protect_pager_init: %p, %p, %x\n", 241 mem_obj, control, pg_size)); 242 243 if (control == MEMORY_OBJECT_CONTROL_NULL) 244 return KERN_INVALID_ARGUMENT; 245 246 pager = apple_protect_pager_lookup(mem_obj); 247 248 memory_object_control_reference(control); 249 250 pager->pager_control = control; 251 252 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; 253 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ 254 attributes.cluster_size = (1 << (PAGE_SHIFT)); 255 attributes.may_cache_object = FALSE; 256 attributes.temporary = TRUE; 257 258 kr = memory_object_change_attributes( 259 control, 260 MEMORY_OBJECT_ATTRIBUTE_INFO, 261 (memory_object_info_t) &attributes, 262 MEMORY_OBJECT_ATTR_INFO_COUNT); 263 if (kr != KERN_SUCCESS) 264 panic("apple_protect_pager_init: " 265 "memory_object_change_attributes() failed"); 266 267 return KERN_SUCCESS; 268} 269 270/* 271 * apple_protect_data_return() 272 * 273 * Handles page-out requests from VM. This should never happen since 274 * the pages provided by this EMM are not supposed to be dirty or dirtied 275 * and VM should simply discard the contents and reclaim the pages if it 276 * needs to. 277 */ 278kern_return_t 279apple_protect_pager_data_return( 280 __unused memory_object_t mem_obj, 281 __unused memory_object_offset_t offset, 282 __unused memory_object_cluster_size_t data_cnt, 283 __unused memory_object_offset_t *resid_offset, 284 __unused int *io_error, 285 __unused boolean_t dirty, 286 __unused boolean_t kernel_copy, 287 __unused int upl_flags) 288{ 289 panic("apple_protect_pager_data_return: should never get called"); 290 return KERN_FAILURE; 291} 292 293kern_return_t 294apple_protect_pager_data_initialize( 295 __unused memory_object_t mem_obj, 296 __unused memory_object_offset_t offset, 297 __unused memory_object_cluster_size_t data_cnt) 298{ 299 panic("apple_protect_pager_data_initialize: should never get called"); 300 return KERN_FAILURE; 301} 302 303kern_return_t 304apple_protect_pager_data_unlock( 305 __unused memory_object_t mem_obj, 306 __unused memory_object_offset_t offset, 307 __unused memory_object_size_t size, 308 __unused vm_prot_t desired_access) 309{ 310 return KERN_FAILURE; 311} 312 313/* 314 * apple_protect_pager_data_request() 315 * 316 * Handles page-in requests from VM. 317 */ 318kern_return_t 319apple_protect_pager_data_request( 320 memory_object_t mem_obj, 321 memory_object_offset_t offset, 322 memory_object_cluster_size_t length, 323#if !DEBUG 324 __unused 325#endif 326 vm_prot_t protection_required, 327 memory_object_fault_info_t mo_fault_info) 328{ 329 apple_protect_pager_t pager; 330 memory_object_control_t mo_control; 331 upl_t upl; 332 int upl_flags; 333 upl_size_t upl_size; 334 upl_page_info_t *upl_pl; 335 unsigned int pl_count; 336 vm_object_t src_object, dst_object; 337 kern_return_t kr, retval; 338 vm_map_offset_t kernel_mapping; 339 vm_offset_t src_vaddr, dst_vaddr; 340 vm_offset_t cur_offset; 341 vm_map_entry_t map_entry; 342 kern_return_t error_code; 343 vm_prot_t prot; 344 vm_page_t src_page, top_page; 345 int interruptible; 346 struct vm_object_fault_info fault_info; 347 int ret; 348 349 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); 350 351 retval = KERN_SUCCESS; 352 src_object = VM_OBJECT_NULL; 353 kernel_mapping = 0; 354 upl = NULL; 355 upl_pl = NULL; 356 fault_info = *((struct vm_object_fault_info *) mo_fault_info); 357 fault_info.stealth = TRUE; 358 fault_info.io_sync = FALSE; 359 fault_info.mark_zf_absent = FALSE; 360 fault_info.batch_pmap_op = FALSE; 361 interruptible = fault_info.interruptible; 362 363 pager = apple_protect_pager_lookup(mem_obj); 364 assert(pager->is_ready); 365 assert(pager->ref_count > 1); /* pager is alive and mapped */ 366 367 PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); 368 369 /* 370 * Gather in a UPL all the VM pages requested by VM. 371 */ 372 mo_control = pager->pager_control; 373 374 upl_size = length; 375 upl_flags = 376 UPL_RET_ONLY_ABSENT | 377 UPL_SET_LITE | 378 UPL_NO_SYNC | 379 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ 380 UPL_SET_INTERNAL; 381 pl_count = 0; 382 kr = memory_object_upl_request(mo_control, 383 offset, upl_size, 384 &upl, NULL, NULL, upl_flags); 385 if (kr != KERN_SUCCESS) { 386 retval = kr; 387 goto done; 388 } 389 dst_object = mo_control->moc_object; 390 assert(dst_object != VM_OBJECT_NULL); 391 392 393 /* 394 * Reserve 2 virtual pages in the kernel address space to map each 395 * source and destination physical pages when it's their turn to 396 * be processed. 397 */ 398 vm_object_reference(kernel_object); /* ref. for mapping */ 399 kr = vm_map_find_space(kernel_map, 400 &kernel_mapping, 401 2 * PAGE_SIZE_64, 402 0, 403 0, 404 &map_entry); 405 if (kr != KERN_SUCCESS) { 406 vm_object_deallocate(kernel_object); 407 retval = kr; 408 goto done; 409 } 410 map_entry->object.vm_object = kernel_object; 411 map_entry->offset = kernel_mapping; 412 vm_map_unlock(kernel_map); 413 src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); 414 dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64); 415 416 /* 417 * We'll map the encrypted data in the kernel address space from the 418 * backing VM object (itself backed by the encrypted file via 419 * the vnode pager). 420 */ 421 src_object = pager->backing_object; 422 assert(src_object != VM_OBJECT_NULL); 423 vm_object_reference(src_object); /* to keep the source object alive */ 424 425 /* 426 * Fill in the contents of the pages requested by VM. 427 */ 428 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); 429 pl_count = length / PAGE_SIZE; 430 for (cur_offset = 0; 431 retval == KERN_SUCCESS && cur_offset < length; 432 cur_offset += PAGE_SIZE) { 433 ppnum_t dst_pnum; 434 435 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { 436 /* this page is not in the UPL: skip it */ 437 continue; 438 } 439 440 /* 441 * Map the source (encrypted) page in the kernel's 442 * virtual address space. 443 * We already hold a reference on the src_object. 444 */ 445 retry_src_fault: 446 vm_object_lock(src_object); 447 vm_object_paging_begin(src_object); 448 error_code = 0; 449 prot = VM_PROT_READ; 450 src_page = VM_PAGE_NULL; 451 kr = vm_fault_page(src_object, 452 offset + cur_offset, 453 VM_PROT_READ, 454 FALSE, 455 FALSE, /* src_page not looked up */ 456 &prot, 457 &src_page, 458 &top_page, 459 NULL, 460 &error_code, 461 FALSE, 462 FALSE, 463 &fault_info); 464 switch (kr) { 465 case VM_FAULT_SUCCESS: 466 break; 467 case VM_FAULT_RETRY: 468 goto retry_src_fault; 469 case VM_FAULT_MEMORY_SHORTAGE: 470 if (vm_page_wait(interruptible)) { 471 goto retry_src_fault; 472 } 473 /* fall thru */ 474 case VM_FAULT_INTERRUPTED: 475 retval = MACH_SEND_INTERRUPTED; 476 goto done; 477 case VM_FAULT_SUCCESS_NO_VM_PAGE: 478 /* success but no VM page: fail */ 479 vm_object_paging_end(src_object); 480 vm_object_unlock(src_object); 481 /*FALLTHROUGH*/ 482 case VM_FAULT_MEMORY_ERROR: 483 /* the page is not there ! */ 484 if (error_code) { 485 retval = error_code; 486 } else { 487 retval = KERN_MEMORY_ERROR; 488 } 489 goto done; 490 default: 491 panic("apple_protect_pager_data_request: " 492 "vm_fault_page() unexpected error 0x%x\n", 493 kr); 494 } 495 assert(src_page != VM_PAGE_NULL); 496 assert(src_page->busy); 497 498 if (!src_page->active && 499 !src_page->inactive && 500 !src_page->throttled) { 501 vm_page_lockspin_queues(); 502 if (!src_page->active && 503 !src_page->inactive && 504 !src_page->throttled) { 505 vm_page_deactivate(src_page); 506 } 507 vm_page_unlock_queues(); 508 } 509 510 /* 511 * Establish an explicit mapping of the source 512 * physical page. 513 */ 514 pmap_enter(kernel_pmap, 515 kernel_mapping, 516 src_page->phys_page, 517 VM_PROT_READ, 518 VM_PROT_NONE, 519 0, 520 TRUE); 521 /* 522 * Establish an explicit pmap mapping of the destination 523 * physical page. 524 * We can't do a regular VM mapping because the VM page 525 * is "busy". 526 */ 527 dst_pnum = (ppnum_t) 528 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); 529 assert(dst_pnum != 0); 530 pmap_enter(kernel_pmap, 531 kernel_mapping + PAGE_SIZE_64, 532 dst_pnum, 533 VM_PROT_READ | VM_PROT_WRITE, 534 VM_PROT_NONE, 535 0, 536 TRUE); 537 538 /* 539 * Decrypt the encrypted contents of the source page 540 * into the destination page. 541 */ 542 ret = pager->crypt.page_decrypt((const void *) src_vaddr, 543 (void *) dst_vaddr, 544 offset+cur_offset, 545 pager->crypt.crypt_ops); 546 if (ret) { 547 /* 548 * Decryption failed. Abort the fault. 549 */ 550 retval = KERN_ABORTED; 551 } else { 552 /* 553 * Validate the original page... 554 */ 555 if (src_page->object->code_signed) { 556 vm_page_validate_cs_mapped( 557 src_page, 558 (const void *) src_vaddr); 559 } 560 /* 561 * ... and transfer the results to the destination page. 562 */ 563 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, 564 src_page->cs_validated); 565 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, 566 src_page->cs_tainted); 567 } 568 569 /* 570 * Remove the pmap mapping of the source and destination pages 571 * in the kernel. 572 */ 573 pmap_remove(kernel_pmap, 574 (addr64_t) kernel_mapping, 575 (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64))); 576 577 /* 578 * Cleanup the result of vm_fault_page() of the source page. 579 */ 580 PAGE_WAKEUP_DONE(src_page); 581 vm_object_paging_end(src_page->object); 582 vm_object_unlock(src_page->object); 583 if (top_page != VM_PAGE_NULL) { 584 vm_object_t top_object; 585 586 top_object = top_page->object; 587 vm_object_lock(top_object); 588 VM_PAGE_FREE(top_page); 589 vm_object_paging_end(top_object); 590 vm_object_unlock(top_object); 591 } 592 } 593 594done: 595 if (upl != NULL) { 596 /* clean up the UPL */ 597 598 /* 599 * The pages are currently dirty because we've just been 600 * writing on them, but as far as we're concerned, they're 601 * clean since they contain their "original" contents as 602 * provided by us, the pager. 603 * Tell the UPL to mark them "clean". 604 */ 605 upl_clear_dirty(upl, TRUE); 606 607 /* abort or commit the UPL */ 608 if (retval != KERN_SUCCESS) { 609 upl_abort(upl, 0); 610 if (retval == KERN_ABORTED) { 611 wait_result_t wait_result; 612 613 /* 614 * We aborted the fault and did not provide 615 * any contents for the requested pages but 616 * the pages themselves are not invalid, so 617 * let's return success and let the caller 618 * retry the fault, in case it might succeed 619 * later (when the decryption code is up and 620 * running in the kernel, for example). 621 */ 622 retval = KERN_SUCCESS; 623 /* 624 * Wait a little bit first to avoid using 625 * too much CPU time retrying and failing 626 * the same fault over and over again. 627 */ 628 wait_result = assert_wait_timeout( 629 (event_t) apple_protect_pager_data_request, 630 THREAD_UNINT, 631 10000, /* 10ms */ 632 NSEC_PER_USEC); 633 assert(wait_result == THREAD_WAITING); 634 wait_result = thread_block(THREAD_CONTINUE_NULL); 635 assert(wait_result == THREAD_TIMED_OUT); 636 } 637 } else { 638 boolean_t empty; 639 upl_commit_range(upl, 0, upl->size, 640 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, 641 upl_pl, pl_count, &empty); 642 } 643 644 /* and deallocate the UPL */ 645 upl_deallocate(upl); 646 upl = NULL; 647 } 648 if (kernel_mapping != 0) { 649 /* clean up the mapping of the source and destination pages */ 650 kr = vm_map_remove(kernel_map, 651 kernel_mapping, 652 kernel_mapping + (2 * PAGE_SIZE_64), 653 VM_MAP_NO_FLAGS); 654 assert(kr == KERN_SUCCESS); 655 kernel_mapping = 0; 656 src_vaddr = 0; 657 dst_vaddr = 0; 658 } 659 if (src_object != VM_OBJECT_NULL) { 660 vm_object_deallocate(src_object); 661 } 662 663 return retval; 664} 665 666/* 667 * apple_protect_pager_reference() 668 * 669 * Get a reference on this memory object. 670 * For external usage only. Assumes that the initial reference count is not 0, 671 * i.e one should not "revive" a dead pager this way. 672 */ 673void 674apple_protect_pager_reference( 675 memory_object_t mem_obj) 676{ 677 apple_protect_pager_t pager; 678 679 pager = apple_protect_pager_lookup(mem_obj); 680 681 lck_mtx_lock(&apple_protect_pager_lock); 682 assert(pager->ref_count > 0); 683 pager->ref_count++; 684 lck_mtx_unlock(&apple_protect_pager_lock); 685} 686 687 688/* 689 * apple_protect_pager_dequeue: 690 * 691 * Removes a pager from the list of pagers. 692 * 693 * The caller must hold "apple_protect_pager_lock". 694 */ 695void 696apple_protect_pager_dequeue( 697 apple_protect_pager_t pager) 698{ 699 assert(!pager->is_mapped); 700 701 queue_remove(&apple_protect_pager_queue, 702 pager, 703 apple_protect_pager_t, 704 pager_queue); 705 pager->pager_queue.next = NULL; 706 pager->pager_queue.prev = NULL; 707 708 apple_protect_pager_count--; 709} 710 711/* 712 * apple_protect_pager_terminate_internal: 713 * 714 * Trigger the asynchronous termination of the memory object associated 715 * with this pager. 716 * When the memory object is terminated, there will be one more call 717 * to memory_object_deallocate() (i.e. apple_protect_pager_deallocate()) 718 * to finish the clean up. 719 * 720 * "apple_protect_pager_lock" should not be held by the caller. 721 * We don't need the lock because the pager has already been removed from 722 * the pagers' list and is now ours exclusively. 723 */ 724void 725apple_protect_pager_terminate_internal( 726 apple_protect_pager_t pager) 727{ 728 assert(pager->is_ready); 729 assert(!pager->is_mapped); 730 731 if (pager->backing_object != VM_OBJECT_NULL) { 732 vm_object_deallocate(pager->backing_object); 733 pager->backing_object = VM_OBJECT_NULL; 734 } 735 736 /* deallocate any crypt module data */ 737 if(pager->crypt.crypt_end) 738 pager->crypt.crypt_end(pager->crypt.crypt_ops); 739 740 /* trigger the destruction of the memory object */ 741 memory_object_destroy(pager->pager_control, 0); 742} 743 744/* 745 * apple_protect_pager_deallocate_internal() 746 * 747 * Release a reference on this pager and free it when the last 748 * reference goes away. 749 * Can be called with apple_protect_pager_lock held or not but always returns 750 * with it unlocked. 751 */ 752void 753apple_protect_pager_deallocate_internal( 754 apple_protect_pager_t pager, 755 boolean_t locked) 756{ 757 boolean_t needs_trimming; 758 int count_unmapped; 759 760 if (! locked) { 761 lck_mtx_lock(&apple_protect_pager_lock); 762 } 763 764 count_unmapped = (apple_protect_pager_count - 765 apple_protect_pager_count_mapped); 766 if (count_unmapped > apple_protect_pager_cache_limit) { 767 /* we have too many unmapped pagers: trim some */ 768 needs_trimming = TRUE; 769 } else { 770 needs_trimming = FALSE; 771 } 772 773 /* drop a reference on this pager */ 774 pager->ref_count--; 775 776 if (pager->ref_count == 1) { 777 /* 778 * Only the "named" reference is left, which means that 779 * no one is really holding on to this pager anymore. 780 * Terminate it. 781 */ 782 apple_protect_pager_dequeue(pager); 783 /* the pager is all ours: no need for the lock now */ 784 lck_mtx_unlock(&apple_protect_pager_lock); 785 apple_protect_pager_terminate_internal(pager); 786 } else if (pager->ref_count == 0) { 787 /* 788 * Dropped the existence reference; the memory object has 789 * been terminated. Do some final cleanup and release the 790 * pager structure. 791 */ 792 lck_mtx_unlock(&apple_protect_pager_lock); 793 if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { 794 memory_object_control_deallocate(pager->pager_control); 795 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; 796 } 797 kfree(pager, sizeof (*pager)); 798 pager = APPLE_PROTECT_PAGER_NULL; 799 } else { 800 /* there are still plenty of references: keep going... */ 801 lck_mtx_unlock(&apple_protect_pager_lock); 802 } 803 804 if (needs_trimming) { 805 apple_protect_pager_trim(); 806 } 807 /* caution: lock is not held on return... */ 808} 809 810/* 811 * apple_protect_pager_deallocate() 812 * 813 * Release a reference on this pager and free it when the last 814 * reference goes away. 815 */ 816void 817apple_protect_pager_deallocate( 818 memory_object_t mem_obj) 819{ 820 apple_protect_pager_t pager; 821 822 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj)); 823 pager = apple_protect_pager_lookup(mem_obj); 824 apple_protect_pager_deallocate_internal(pager, FALSE); 825} 826 827/* 828 * 829 */ 830kern_return_t 831apple_protect_pager_terminate( 832#if !DEBUG 833 __unused 834#endif 835 memory_object_t mem_obj) 836{ 837 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj)); 838 839 return KERN_SUCCESS; 840} 841 842/* 843 * 844 */ 845kern_return_t 846apple_protect_pager_synchronize( 847 memory_object_t mem_obj, 848 memory_object_offset_t offset, 849 memory_object_size_t length, 850 __unused vm_sync_t sync_flags) 851{ 852 apple_protect_pager_t pager; 853 854 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj)); 855 856 pager = apple_protect_pager_lookup(mem_obj); 857 858 memory_object_synchronize_completed(pager->pager_control, 859 offset, length); 860 861 return KERN_SUCCESS; 862} 863 864/* 865 * apple_protect_pager_map() 866 * 867 * This allows VM to let us, the EMM, know that this memory object 868 * is currently mapped one or more times. This is called by VM each time 869 * the memory object gets mapped and we take one extra reference on the 870 * memory object to account for all its mappings. 871 */ 872kern_return_t 873apple_protect_pager_map( 874 memory_object_t mem_obj, 875 __unused vm_prot_t prot) 876{ 877 apple_protect_pager_t pager; 878 879 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj)); 880 881 pager = apple_protect_pager_lookup(mem_obj); 882 883 lck_mtx_lock(&apple_protect_pager_lock); 884 assert(pager->is_ready); 885 assert(pager->ref_count > 0); /* pager is alive */ 886 if (pager->is_mapped == FALSE) { 887 /* 888 * First mapping of this pager: take an extra reference 889 * that will remain until all the mappings of this pager 890 * are removed. 891 */ 892 pager->is_mapped = TRUE; 893 pager->ref_count++; 894 apple_protect_pager_count_mapped++; 895 } 896 lck_mtx_unlock(&apple_protect_pager_lock); 897 898 return KERN_SUCCESS; 899} 900 901/* 902 * apple_protect_pager_last_unmap() 903 * 904 * This is called by VM when this memory object is no longer mapped anywhere. 905 */ 906kern_return_t 907apple_protect_pager_last_unmap( 908 memory_object_t mem_obj) 909{ 910 apple_protect_pager_t pager; 911 int count_unmapped; 912 913 PAGER_DEBUG(PAGER_ALL, 914 ("apple_protect_pager_last_unmap: %p\n", mem_obj)); 915 916 pager = apple_protect_pager_lookup(mem_obj); 917 918 lck_mtx_lock(&apple_protect_pager_lock); 919 if (pager->is_mapped) { 920 /* 921 * All the mappings are gone, so let go of the one extra 922 * reference that represents all the mappings of this pager. 923 */ 924 apple_protect_pager_count_mapped--; 925 count_unmapped = (apple_protect_pager_count - 926 apple_protect_pager_count_mapped); 927 if (count_unmapped > apple_protect_pager_count_unmapped_max) { 928 apple_protect_pager_count_unmapped_max = count_unmapped; 929 } 930 pager->is_mapped = FALSE; 931 apple_protect_pager_deallocate_internal(pager, TRUE); 932 /* caution: deallocate_internal() released the lock ! */ 933 } else { 934 lck_mtx_unlock(&apple_protect_pager_lock); 935 } 936 937 return KERN_SUCCESS; 938} 939 940 941/* 942 * 943 */ 944apple_protect_pager_t 945apple_protect_pager_lookup( 946 memory_object_t mem_obj) 947{ 948 apple_protect_pager_t pager; 949 950 pager = (apple_protect_pager_t) mem_obj; 951 assert(pager->pager_ops == &apple_protect_pager_ops); 952 assert(pager->ref_count > 0); 953 return pager; 954} 955 956apple_protect_pager_t 957apple_protect_pager_create( 958 vm_object_t backing_object, 959 struct pager_crypt_info *crypt_info) 960{ 961 apple_protect_pager_t pager, pager2; 962 memory_object_control_t control; 963 kern_return_t kr; 964 965 pager = (apple_protect_pager_t) kalloc(sizeof (*pager)); 966 if (pager == APPLE_PROTECT_PAGER_NULL) { 967 return APPLE_PROTECT_PAGER_NULL; 968 } 969 970 /* 971 * The vm_map call takes both named entry ports and raw memory 972 * objects in the same parameter. We need to make sure that 973 * vm_map does not see this object as a named entry port. So, 974 * we reserve the first word in the object for a fake ip_kotype 975 * setting - that will tell vm_map to use it as a memory object. 976 */ 977 pager->pager_ops = &apple_protect_pager_ops; 978 pager->pager_ikot = IKOT_MEMORY_OBJECT; 979 pager->is_ready = FALSE;/* not ready until it has a "name" */ 980 pager->ref_count = 2; /* existence + setup reference */ 981 pager->is_mapped = FALSE; 982 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; 983 pager->backing_object = backing_object; 984 pager->crypt = *crypt_info; 985 986 vm_object_reference(backing_object); 987 988 lck_mtx_lock(&apple_protect_pager_lock); 989 /* see if anyone raced us to create a pager for the same object */ 990 queue_iterate(&apple_protect_pager_queue, 991 pager2, 992 apple_protect_pager_t, 993 pager_queue) { 994 if (pager2->backing_object == backing_object) { 995 break; 996 } 997 } 998 if (! queue_end(&apple_protect_pager_queue, 999 (queue_entry_t) pager2)) { 1000 /* while we hold the lock, transfer our setup ref to winner */ 1001 pager2->ref_count++; 1002 /* we lost the race, down with the loser... */ 1003 lck_mtx_unlock(&apple_protect_pager_lock); 1004 vm_object_deallocate(pager->backing_object); 1005 pager->backing_object = VM_OBJECT_NULL; 1006 kfree(pager, sizeof (*pager)); 1007 /* ... and go with the winner */ 1008 pager = pager2; 1009 /* let the winner make sure the pager gets ready */ 1010 return pager; 1011 } 1012 1013 /* enter new pager at the head of our list of pagers */ 1014 queue_enter_first(&apple_protect_pager_queue, 1015 pager, 1016 apple_protect_pager_t, 1017 pager_queue); 1018 apple_protect_pager_count++; 1019 if (apple_protect_pager_count > apple_protect_pager_count_max) { 1020 apple_protect_pager_count_max = apple_protect_pager_count; 1021 } 1022 lck_mtx_unlock(&apple_protect_pager_lock); 1023 1024 kr = memory_object_create_named((memory_object_t) pager, 1025 0, 1026 &control); 1027 assert(kr == KERN_SUCCESS); 1028 1029 lck_mtx_lock(&apple_protect_pager_lock); 1030 /* the new pager is now ready to be used */ 1031 pager->is_ready = TRUE; 1032 lck_mtx_unlock(&apple_protect_pager_lock); 1033 1034 /* wakeup anyone waiting for this pager to be ready */ 1035 thread_wakeup(&pager->is_ready); 1036 1037 return pager; 1038} 1039 1040/* 1041 * apple_protect_pager_setup() 1042 * 1043 * Provide the caller with a memory object backed by the provided 1044 * "backing_object" VM object. If such a memory object already exists, 1045 * re-use it, otherwise create a new memory object. 1046 */ 1047memory_object_t 1048apple_protect_pager_setup( 1049 vm_object_t backing_object, 1050 struct pager_crypt_info *crypt_info) 1051{ 1052 apple_protect_pager_t pager; 1053 1054 lck_mtx_lock(&apple_protect_pager_lock); 1055 1056 queue_iterate(&apple_protect_pager_queue, 1057 pager, 1058 apple_protect_pager_t, 1059 pager_queue) { 1060 if (pager->backing_object == backing_object) { 1061 /* For the same object we must always use the same protection options */ 1062 if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) && 1063 (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) { 1064 lck_mtx_unlock(&apple_protect_pager_lock); 1065 return MEMORY_OBJECT_NULL; 1066 } 1067 break; 1068 } 1069 } 1070 if (queue_end(&apple_protect_pager_queue, 1071 (queue_entry_t) pager)) { 1072 /* no existing pager for this backing object */ 1073 pager = APPLE_PROTECT_PAGER_NULL; 1074 } else { 1075 /* make sure pager doesn't disappear */ 1076 pager->ref_count++; 1077 } 1078 1079 lck_mtx_unlock(&apple_protect_pager_lock); 1080 1081 if (pager == APPLE_PROTECT_PAGER_NULL) { 1082 pager = apple_protect_pager_create(backing_object, crypt_info); 1083 if (pager == APPLE_PROTECT_PAGER_NULL) { 1084 return MEMORY_OBJECT_NULL; 1085 } 1086 } 1087 1088 lck_mtx_lock(&apple_protect_pager_lock); 1089 while (!pager->is_ready) { 1090 lck_mtx_sleep(&apple_protect_pager_lock, 1091 LCK_SLEEP_DEFAULT, 1092 &pager->is_ready, 1093 THREAD_UNINT); 1094 } 1095 lck_mtx_unlock(&apple_protect_pager_lock); 1096 1097 return (memory_object_t) pager; 1098} 1099 1100void 1101apple_protect_pager_trim(void) 1102{ 1103 apple_protect_pager_t pager, prev_pager; 1104 queue_head_t trim_queue; 1105 int num_trim; 1106 int count_unmapped; 1107 1108 lck_mtx_lock(&apple_protect_pager_lock); 1109 1110 /* 1111 * We have too many pagers, try and trim some unused ones, 1112 * starting with the oldest pager at the end of the queue. 1113 */ 1114 queue_init(&trim_queue); 1115 num_trim = 0; 1116 1117 for (pager = (apple_protect_pager_t) 1118 queue_last(&apple_protect_pager_queue); 1119 !queue_end(&apple_protect_pager_queue, 1120 (queue_entry_t) pager); 1121 pager = prev_pager) { 1122 /* get prev elt before we dequeue */ 1123 prev_pager = (apple_protect_pager_t) 1124 queue_prev(&pager->pager_queue); 1125 1126 if (pager->ref_count == 2 && 1127 pager->is_ready && 1128 !pager->is_mapped) { 1129 /* this pager can be trimmed */ 1130 num_trim++; 1131 /* remove this pager from the main list ... */ 1132 apple_protect_pager_dequeue(pager); 1133 /* ... and add it to our trim queue */ 1134 queue_enter_first(&trim_queue, 1135 pager, 1136 apple_protect_pager_t, 1137 pager_queue); 1138 1139 count_unmapped = (apple_protect_pager_count - 1140 apple_protect_pager_count_mapped); 1141 if (count_unmapped <= apple_protect_pager_cache_limit) { 1142 /* we have enough pagers to trim */ 1143 break; 1144 } 1145 } 1146 } 1147 if (num_trim > apple_protect_pager_num_trim_max) { 1148 apple_protect_pager_num_trim_max = num_trim; 1149 } 1150 apple_protect_pager_num_trim_total += num_trim; 1151 1152 lck_mtx_unlock(&apple_protect_pager_lock); 1153 1154 /* terminate the trimmed pagers */ 1155 while (!queue_empty(&trim_queue)) { 1156 queue_remove_first(&trim_queue, 1157 pager, 1158 apple_protect_pager_t, 1159 pager_queue); 1160 pager->pager_queue.next = NULL; 1161 pager->pager_queue.prev = NULL; 1162 assert(pager->ref_count == 2); 1163 /* 1164 * We can't call deallocate_internal() because the pager 1165 * has already been dequeued, but we still need to remove 1166 * a reference. 1167 */ 1168 pager->ref_count--; 1169 apple_protect_pager_terminate_internal(pager); 1170 } 1171} 1172