1/* 2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <sys/errno.h> 30 31#include <mach/mach_types.h> 32#include <mach/mach_traps.h> 33#include <mach/host_priv.h> 34#include <mach/kern_return.h> 35#include <mach/memory_object_control.h> 36#include <mach/memory_object_types.h> 37#include <mach/port.h> 38#include <mach/policy.h> 39#include <mach/upl.h> 40#include <mach/thread_act.h> 41#include <mach/mach_vm.h> 42 43#include <kern/host.h> 44#include <kern/kalloc.h> 45#include <kern/page_decrypt.h> 46#include <kern/queue.h> 47#include <kern/thread.h> 48 49#include <ipc/ipc_port.h> 50#include <ipc/ipc_space.h> 51 52#include <default_pager/default_pager_types.h> 53#include <default_pager/default_pager_object_server.h> 54 55#include <vm/vm_fault.h> 56#include <vm/vm_map.h> 57#include <vm/vm_pageout.h> 58#include <vm/memory_object.h> 59#include <vm/vm_pageout.h> 60#include <vm/vm_protos.h> 61 62 63/* 64 * APPLE PROTECT MEMORY PAGER 65 * 66 * This external memory manager (EMM) handles memory from the encrypted 67 * sections of some executables protected by the DSMOS kernel extension. 68 * 69 * It mostly handles page-in requests (from memory_object_data_request()) by 70 * getting the encrypted data from its backing VM object, itself backed by 71 * the encrypted file, decrypting it and providing it to VM. 72 * 73 * The decrypted pages will never be dirtied, so the memory manager doesn't 74 * need to handle page-out requests (from memory_object_data_return()). The 75 * pages need to be mapped copy-on-write, so that the originals stay clean. 76 * 77 * We don't expect to have to handle a large number of apple-protected 78 * binaries, so the data structures are very simple (simple linked list) 79 * for now. 80 */ 81 82/* forward declarations */ 83void apple_protect_pager_reference(memory_object_t mem_obj); 84void apple_protect_pager_deallocate(memory_object_t mem_obj); 85kern_return_t apple_protect_pager_init(memory_object_t mem_obj, 86 memory_object_control_t control, 87 memory_object_cluster_size_t pg_size); 88kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj); 89kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj, 90 memory_object_offset_t offset, 91 memory_object_cluster_size_t length, 92 vm_prot_t protection_required, 93 memory_object_fault_info_t fault_info); 94kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj, 95 memory_object_offset_t offset, 96 memory_object_cluster_size_t data_cnt, 97 memory_object_offset_t *resid_offset, 98 int *io_error, 99 boolean_t dirty, 100 boolean_t kernel_copy, 101 int upl_flags); 102kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj, 103 memory_object_offset_t offset, 104 memory_object_cluster_size_t data_cnt); 105kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj, 106 memory_object_offset_t offset, 107 memory_object_size_t size, 108 vm_prot_t desired_access); 109kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj, 110 memory_object_offset_t offset, 111 memory_object_size_t length, 112 vm_sync_t sync_flags); 113kern_return_t apple_protect_pager_map(memory_object_t mem_obj, 114 vm_prot_t prot); 115kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj); 116 117/* 118 * Vector of VM operations for this EMM. 119 * These routines are invoked by VM via the memory_object_*() interfaces. 120 */ 121const struct memory_object_pager_ops apple_protect_pager_ops = { 122 apple_protect_pager_reference, 123 apple_protect_pager_deallocate, 124 apple_protect_pager_init, 125 apple_protect_pager_terminate, 126 apple_protect_pager_data_request, 127 apple_protect_pager_data_return, 128 apple_protect_pager_data_initialize, 129 apple_protect_pager_data_unlock, 130 apple_protect_pager_synchronize, 131 apple_protect_pager_map, 132 apple_protect_pager_last_unmap, 133 NULL, /* data_reclaim */ 134 "apple protect pager" 135}; 136 137/* 138 * The "apple_protect_pager" describes a memory object backed by 139 * the "apple protect" EMM. 140 */ 141typedef struct apple_protect_pager { 142 struct ipc_object_header pager_header; /* fake ip_kotype() */ 143 memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */ 144 queue_chain_t pager_queue; /* next & prev pagers */ 145 unsigned int ref_count; /* reference count */ 146 boolean_t is_ready; /* is this pager ready ? */ 147 boolean_t is_mapped; /* is this mem_obj mapped ? */ 148 memory_object_control_t pager_control; /* mem object control handle */ 149 vm_object_t backing_object; /* VM obj w/ encrypted data */ 150 struct pager_crypt_info crypt; 151} *apple_protect_pager_t; 152#define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL) 153#define pager_ikot pager_header.io_bits 154 155/* 156 * List of memory objects managed by this EMM. 157 * The list is protected by the "apple_protect_pager_lock" lock. 158 */ 159int apple_protect_pager_count = 0; /* number of pagers */ 160int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */ 161queue_head_t apple_protect_pager_queue; 162decl_lck_mtx_data(,apple_protect_pager_lock) 163 164/* 165 * Maximum number of unmapped pagers we're willing to keep around. 166 */ 167int apple_protect_pager_cache_limit = 10; 168 169/* 170 * Statistics & counters. 171 */ 172int apple_protect_pager_count_max = 0; 173int apple_protect_pager_count_unmapped_max = 0; 174int apple_protect_pager_num_trim_max = 0; 175int apple_protect_pager_num_trim_total = 0; 176 177 178lck_grp_t apple_protect_pager_lck_grp; 179lck_grp_attr_t apple_protect_pager_lck_grp_attr; 180lck_attr_t apple_protect_pager_lck_attr; 181 182 183/* internal prototypes */ 184apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object, 185 struct pager_crypt_info *crypt_info); 186apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj); 187void apple_protect_pager_dequeue(apple_protect_pager_t pager); 188void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager, 189 boolean_t locked); 190void apple_protect_pager_terminate_internal(apple_protect_pager_t pager); 191void apple_protect_pager_trim(void); 192 193 194#if DEBUG 195int apple_protect_pagerdebug = 0; 196#define PAGER_ALL 0xffffffff 197#define PAGER_INIT 0x00000001 198#define PAGER_PAGEIN 0x00000002 199 200#define PAGER_DEBUG(LEVEL, A) \ 201 MACRO_BEGIN \ 202 if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \ 203 printf A; \ 204 } \ 205 MACRO_END 206#else 207#define PAGER_DEBUG(LEVEL, A) 208#endif 209 210 211void 212apple_protect_pager_bootstrap(void) 213{ 214 lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr); 215 lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr); 216 lck_attr_setdefault(&apple_protect_pager_lck_attr); 217 lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr); 218 queue_init(&apple_protect_pager_queue); 219} 220 221/* 222 * apple_protect_pager_init() 223 * 224 * Initialize the memory object and makes it ready to be used and mapped. 225 */ 226kern_return_t 227apple_protect_pager_init( 228 memory_object_t mem_obj, 229 memory_object_control_t control, 230#if !DEBUG 231 __unused 232#endif 233 memory_object_cluster_size_t pg_size) 234{ 235 apple_protect_pager_t pager; 236 kern_return_t kr; 237 memory_object_attr_info_data_t attributes; 238 239 PAGER_DEBUG(PAGER_ALL, 240 ("apple_protect_pager_init: %p, %p, %x\n", 241 mem_obj, control, pg_size)); 242 243 if (control == MEMORY_OBJECT_CONTROL_NULL) 244 return KERN_INVALID_ARGUMENT; 245 246 pager = apple_protect_pager_lookup(mem_obj); 247 248 memory_object_control_reference(control); 249 250 pager->pager_control = control; 251 252 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; 253 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ 254 attributes.cluster_size = (1 << (PAGE_SHIFT)); 255 attributes.may_cache_object = FALSE; 256 attributes.temporary = TRUE; 257 258 kr = memory_object_change_attributes( 259 control, 260 MEMORY_OBJECT_ATTRIBUTE_INFO, 261 (memory_object_info_t) &attributes, 262 MEMORY_OBJECT_ATTR_INFO_COUNT); 263 if (kr != KERN_SUCCESS) 264 panic("apple_protect_pager_init: " 265 "memory_object_change_attributes() failed"); 266 267 return KERN_SUCCESS; 268} 269 270/* 271 * apple_protect_data_return() 272 * 273 * Handles page-out requests from VM. This should never happen since 274 * the pages provided by this EMM are not supposed to be dirty or dirtied 275 * and VM should simply discard the contents and reclaim the pages if it 276 * needs to. 277 */ 278kern_return_t 279apple_protect_pager_data_return( 280 __unused memory_object_t mem_obj, 281 __unused memory_object_offset_t offset, 282 __unused memory_object_cluster_size_t data_cnt, 283 __unused memory_object_offset_t *resid_offset, 284 __unused int *io_error, 285 __unused boolean_t dirty, 286 __unused boolean_t kernel_copy, 287 __unused int upl_flags) 288{ 289 panic("apple_protect_pager_data_return: should never get called"); 290 return KERN_FAILURE; 291} 292 293kern_return_t 294apple_protect_pager_data_initialize( 295 __unused memory_object_t mem_obj, 296 __unused memory_object_offset_t offset, 297 __unused memory_object_cluster_size_t data_cnt) 298{ 299 panic("apple_protect_pager_data_initialize: should never get called"); 300 return KERN_FAILURE; 301} 302 303kern_return_t 304apple_protect_pager_data_unlock( 305 __unused memory_object_t mem_obj, 306 __unused memory_object_offset_t offset, 307 __unused memory_object_size_t size, 308 __unused vm_prot_t desired_access) 309{ 310 return KERN_FAILURE; 311} 312 313/* 314 * apple_protect_pager_data_request() 315 * 316 * Handles page-in requests from VM. 317 */ 318kern_return_t 319apple_protect_pager_data_request( 320 memory_object_t mem_obj, 321 memory_object_offset_t offset, 322 memory_object_cluster_size_t length, 323#if !DEBUG 324 __unused 325#endif 326 vm_prot_t protection_required, 327 memory_object_fault_info_t mo_fault_info) 328{ 329 apple_protect_pager_t pager; 330 memory_object_control_t mo_control; 331 upl_t upl; 332 int upl_flags; 333 upl_size_t upl_size; 334 upl_page_info_t *upl_pl; 335 unsigned int pl_count; 336 vm_object_t src_object, dst_object; 337 kern_return_t kr, retval; 338 vm_map_offset_t kernel_mapping; 339 vm_offset_t src_vaddr, dst_vaddr; 340 vm_offset_t cur_offset; 341 vm_map_entry_t map_entry; 342 kern_return_t error_code; 343 vm_prot_t prot; 344 vm_page_t src_page, top_page; 345 int interruptible; 346 struct vm_object_fault_info fault_info; 347 int ret; 348 349 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); 350 351 retval = KERN_SUCCESS; 352 src_object = VM_OBJECT_NULL; 353 kernel_mapping = 0; 354 upl = NULL; 355 upl_pl = NULL; 356 fault_info = *((struct vm_object_fault_info *) mo_fault_info); 357 fault_info.stealth = TRUE; 358 fault_info.io_sync = FALSE; 359 fault_info.mark_zf_absent = FALSE; 360 fault_info.batch_pmap_op = FALSE; 361 interruptible = fault_info.interruptible; 362 363 pager = apple_protect_pager_lookup(mem_obj); 364 assert(pager->is_ready); 365 assert(pager->ref_count > 1); /* pager is alive and mapped */ 366 367 PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); 368 369 /* 370 * Gather in a UPL all the VM pages requested by VM. 371 */ 372 mo_control = pager->pager_control; 373 374 upl_size = length; 375 upl_flags = 376 UPL_RET_ONLY_ABSENT | 377 UPL_SET_LITE | 378 UPL_NO_SYNC | 379 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ 380 UPL_SET_INTERNAL; 381 pl_count = 0; 382 kr = memory_object_upl_request(mo_control, 383 offset, upl_size, 384 &upl, NULL, NULL, upl_flags); 385 if (kr != KERN_SUCCESS) { 386 retval = kr; 387 goto done; 388 } 389 dst_object = mo_control->moc_object; 390 assert(dst_object != VM_OBJECT_NULL); 391 392 393 /* 394 * Reserve 2 virtual pages in the kernel address space to map each 395 * source and destination physical pages when it's their turn to 396 * be processed. 397 */ 398 vm_object_reference(kernel_object); /* ref. for mapping */ 399 kr = vm_map_find_space(kernel_map, 400 &kernel_mapping, 401 2 * PAGE_SIZE_64, 402 0, 403 0, 404 &map_entry); 405 if (kr != KERN_SUCCESS) { 406 vm_object_deallocate(kernel_object); 407 retval = kr; 408 goto done; 409 } 410 map_entry->object.vm_object = kernel_object; 411 map_entry->offset = kernel_mapping; 412 vm_map_unlock(kernel_map); 413 src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); 414 dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64); 415 416 /* 417 * We'll map the encrypted data in the kernel address space from the 418 * backing VM object (itself backed by the encrypted file via 419 * the vnode pager). 420 */ 421 src_object = pager->backing_object; 422 assert(src_object != VM_OBJECT_NULL); 423 vm_object_reference(src_object); /* to keep the source object alive */ 424 425 /* 426 * Fill in the contents of the pages requested by VM. 427 */ 428 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); 429 pl_count = length / PAGE_SIZE; 430 for (cur_offset = 0; 431 retval == KERN_SUCCESS && cur_offset < length; 432 cur_offset += PAGE_SIZE) { 433 ppnum_t dst_pnum; 434 435 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { 436 /* this page is not in the UPL: skip it */ 437 continue; 438 } 439 440 /* 441 * Map the source (encrypted) page in the kernel's 442 * virtual address space. 443 * We already hold a reference on the src_object. 444 */ 445 retry_src_fault: 446 vm_object_lock(src_object); 447 vm_object_paging_begin(src_object); 448 error_code = 0; 449 prot = VM_PROT_READ; 450 kr = vm_fault_page(src_object, 451 offset + cur_offset, 452 VM_PROT_READ, 453 FALSE, 454 &prot, 455 &src_page, 456 &top_page, 457 NULL, 458 &error_code, 459 FALSE, 460 FALSE, 461 &fault_info); 462 switch (kr) { 463 case VM_FAULT_SUCCESS: 464 break; 465 case VM_FAULT_RETRY: 466 goto retry_src_fault; 467 case VM_FAULT_MEMORY_SHORTAGE: 468 if (vm_page_wait(interruptible)) { 469 goto retry_src_fault; 470 } 471 /* fall thru */ 472 case VM_FAULT_INTERRUPTED: 473 retval = MACH_SEND_INTERRUPTED; 474 goto done; 475 case VM_FAULT_SUCCESS_NO_VM_PAGE: 476 /* success but no VM page: fail */ 477 vm_object_paging_end(src_object); 478 vm_object_unlock(src_object); 479 /*FALLTHROUGH*/ 480 case VM_FAULT_MEMORY_ERROR: 481 /* the page is not there ! */ 482 if (error_code) { 483 retval = error_code; 484 } else { 485 retval = KERN_MEMORY_ERROR; 486 } 487 goto done; 488 default: 489 panic("apple_protect_pager_data_request: " 490 "vm_fault_page() unexpected error 0x%x\n", 491 kr); 492 } 493 assert(src_page != VM_PAGE_NULL); 494 assert(src_page->busy); 495 496 if (!src_page->active && 497 !src_page->inactive && 498 !src_page->throttled) { 499 vm_page_lockspin_queues(); 500 if (!src_page->active && 501 !src_page->inactive && 502 !src_page->throttled) { 503 vm_page_deactivate(src_page); 504 } 505 vm_page_unlock_queues(); 506 } 507 508 /* 509 * Establish an explicit mapping of the source 510 * physical page. 511 */ 512 pmap_enter(kernel_pmap, 513 kernel_mapping, 514 src_page->phys_page, 515 VM_PROT_READ, 516 VM_PROT_NONE, 517 0, 518 TRUE); 519 /* 520 * Establish an explicit pmap mapping of the destination 521 * physical page. 522 * We can't do a regular VM mapping because the VM page 523 * is "busy". 524 */ 525 dst_pnum = (ppnum_t) 526 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); 527 assert(dst_pnum != 0); 528 pmap_enter(kernel_pmap, 529 kernel_mapping + PAGE_SIZE_64, 530 dst_pnum, 531 VM_PROT_READ | VM_PROT_WRITE, 532 VM_PROT_NONE, 533 0, 534 TRUE); 535 536 /* 537 * Decrypt the encrypted contents of the source page 538 * into the destination page. 539 */ 540 ret = pager->crypt.page_decrypt((const void *) src_vaddr, 541 (void *) dst_vaddr, 542 offset+cur_offset, 543 pager->crypt.crypt_ops); 544 if (ret) { 545 /* 546 * Decryption failed. Abort the fault. 547 */ 548 retval = KERN_ABORTED; 549 } else { 550 /* 551 * Validate the original page... 552 */ 553 if (src_page->object->code_signed) { 554 vm_page_validate_cs_mapped( 555 src_page, 556 (const void *) src_vaddr); 557 } 558 /* 559 * ... and transfer the results to the destination page. 560 */ 561 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, 562 src_page->cs_validated); 563 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, 564 src_page->cs_tainted); 565 } 566 567 /* 568 * Remove the pmap mapping of the source and destination pages 569 * in the kernel. 570 */ 571 pmap_remove(kernel_pmap, 572 (addr64_t) kernel_mapping, 573 (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64))); 574 575 /* 576 * Cleanup the result of vm_fault_page() of the source page. 577 */ 578 PAGE_WAKEUP_DONE(src_page); 579 vm_object_paging_end(src_page->object); 580 vm_object_unlock(src_page->object); 581 if (top_page != VM_PAGE_NULL) { 582 vm_object_t top_object; 583 584 top_object = top_page->object; 585 vm_object_lock(top_object); 586 VM_PAGE_FREE(top_page); 587 vm_object_paging_end(top_object); 588 vm_object_unlock(top_object); 589 } 590 } 591 592done: 593 if (upl != NULL) { 594 /* clean up the UPL */ 595 596 /* 597 * The pages are currently dirty because we've just been 598 * writing on them, but as far as we're concerned, they're 599 * clean since they contain their "original" contents as 600 * provided by us, the pager. 601 * Tell the UPL to mark them "clean". 602 */ 603 upl_clear_dirty(upl, TRUE); 604 605 /* abort or commit the UPL */ 606 if (retval != KERN_SUCCESS) { 607 upl_abort(upl, 0); 608 if (retval == KERN_ABORTED) { 609 wait_result_t wait_result; 610 611 /* 612 * We aborted the fault and did not provide 613 * any contents for the requested pages but 614 * the pages themselves are not invalid, so 615 * let's return success and let the caller 616 * retry the fault, in case it might succeed 617 * later (when the decryption code is up and 618 * running in the kernel, for example). 619 */ 620 retval = KERN_SUCCESS; 621 /* 622 * Wait a little bit first to avoid using 623 * too much CPU time retrying and failing 624 * the same fault over and over again. 625 */ 626 wait_result = assert_wait_timeout( 627 (event_t) apple_protect_pager_data_request, 628 THREAD_UNINT, 629 10000, /* 10ms */ 630 NSEC_PER_USEC); 631 assert(wait_result == THREAD_WAITING); 632 wait_result = thread_block(THREAD_CONTINUE_NULL); 633 assert(wait_result == THREAD_TIMED_OUT); 634 } 635 } else { 636 boolean_t empty; 637 upl_commit_range(upl, 0, upl->size, 638 UPL_COMMIT_CS_VALIDATED, 639 upl_pl, pl_count, &empty); 640 } 641 642 /* and deallocate the UPL */ 643 upl_deallocate(upl); 644 upl = NULL; 645 } 646 if (kernel_mapping != 0) { 647 /* clean up the mapping of the source and destination pages */ 648 kr = vm_map_remove(kernel_map, 649 kernel_mapping, 650 kernel_mapping + (2 * PAGE_SIZE_64), 651 VM_MAP_NO_FLAGS); 652 assert(kr == KERN_SUCCESS); 653 kernel_mapping = 0; 654 src_vaddr = 0; 655 dst_vaddr = 0; 656 } 657 if (src_object != VM_OBJECT_NULL) { 658 vm_object_deallocate(src_object); 659 } 660 661 return retval; 662} 663 664/* 665 * apple_protect_pager_reference() 666 * 667 * Get a reference on this memory object. 668 * For external usage only. Assumes that the initial reference count is not 0, 669 * i.e one should not "revive" a dead pager this way. 670 */ 671void 672apple_protect_pager_reference( 673 memory_object_t mem_obj) 674{ 675 apple_protect_pager_t pager; 676 677 pager = apple_protect_pager_lookup(mem_obj); 678 679 lck_mtx_lock(&apple_protect_pager_lock); 680 assert(pager->ref_count > 0); 681 pager->ref_count++; 682 lck_mtx_unlock(&apple_protect_pager_lock); 683} 684 685 686/* 687 * apple_protect_pager_dequeue: 688 * 689 * Removes a pager from the list of pagers. 690 * 691 * The caller must hold "apple_protect_pager_lock". 692 */ 693void 694apple_protect_pager_dequeue( 695 apple_protect_pager_t pager) 696{ 697 assert(!pager->is_mapped); 698 699 queue_remove(&apple_protect_pager_queue, 700 pager, 701 apple_protect_pager_t, 702 pager_queue); 703 pager->pager_queue.next = NULL; 704 pager->pager_queue.prev = NULL; 705 706 apple_protect_pager_count--; 707} 708 709/* 710 * apple_protect_pager_terminate_internal: 711 * 712 * Trigger the asynchronous termination of the memory object associated 713 * with this pager. 714 * When the memory object is terminated, there will be one more call 715 * to memory_object_deallocate() (i.e. apple_protect_pager_deallocate()) 716 * to finish the clean up. 717 * 718 * "apple_protect_pager_lock" should not be held by the caller. 719 * We don't need the lock because the pager has already been removed from 720 * the pagers' list and is now ours exclusively. 721 */ 722void 723apple_protect_pager_terminate_internal( 724 apple_protect_pager_t pager) 725{ 726 assert(pager->is_ready); 727 assert(!pager->is_mapped); 728 729 if (pager->backing_object != VM_OBJECT_NULL) { 730 vm_object_deallocate(pager->backing_object); 731 pager->backing_object = VM_OBJECT_NULL; 732 } 733 734 /* deallocate any crypt module data */ 735 if(pager->crypt.crypt_end) 736 pager->crypt.crypt_end(pager->crypt.crypt_ops); 737 738 /* trigger the destruction of the memory object */ 739 memory_object_destroy(pager->pager_control, 0); 740} 741 742/* 743 * apple_protect_pager_deallocate_internal() 744 * 745 * Release a reference on this pager and free it when the last 746 * reference goes away. 747 * Can be called with apple_protect_pager_lock held or not but always returns 748 * with it unlocked. 749 */ 750void 751apple_protect_pager_deallocate_internal( 752 apple_protect_pager_t pager, 753 boolean_t locked) 754{ 755 boolean_t needs_trimming; 756 int count_unmapped; 757 758 if (! locked) { 759 lck_mtx_lock(&apple_protect_pager_lock); 760 } 761 762 count_unmapped = (apple_protect_pager_count - 763 apple_protect_pager_count_mapped); 764 if (count_unmapped > apple_protect_pager_cache_limit) { 765 /* we have too many unmapped pagers: trim some */ 766 needs_trimming = TRUE; 767 } else { 768 needs_trimming = FALSE; 769 } 770 771 /* drop a reference on this pager */ 772 pager->ref_count--; 773 774 if (pager->ref_count == 1) { 775 /* 776 * Only the "named" reference is left, which means that 777 * no one is really holding on to this pager anymore. 778 * Terminate it. 779 */ 780 apple_protect_pager_dequeue(pager); 781 /* the pager is all ours: no need for the lock now */ 782 lck_mtx_unlock(&apple_protect_pager_lock); 783 apple_protect_pager_terminate_internal(pager); 784 } else if (pager->ref_count == 0) { 785 /* 786 * Dropped the existence reference; the memory object has 787 * been terminated. Do some final cleanup and release the 788 * pager structure. 789 */ 790 lck_mtx_unlock(&apple_protect_pager_lock); 791 if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) { 792 memory_object_control_deallocate(pager->pager_control); 793 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; 794 } 795 kfree(pager, sizeof (*pager)); 796 pager = APPLE_PROTECT_PAGER_NULL; 797 } else { 798 /* there are still plenty of references: keep going... */ 799 lck_mtx_unlock(&apple_protect_pager_lock); 800 } 801 802 if (needs_trimming) { 803 apple_protect_pager_trim(); 804 } 805 /* caution: lock is not held on return... */ 806} 807 808/* 809 * apple_protect_pager_deallocate() 810 * 811 * Release a reference on this pager and free it when the last 812 * reference goes away. 813 */ 814void 815apple_protect_pager_deallocate( 816 memory_object_t mem_obj) 817{ 818 apple_protect_pager_t pager; 819 820 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj)); 821 pager = apple_protect_pager_lookup(mem_obj); 822 apple_protect_pager_deallocate_internal(pager, FALSE); 823} 824 825/* 826 * 827 */ 828kern_return_t 829apple_protect_pager_terminate( 830#if !DEBUG 831 __unused 832#endif 833 memory_object_t mem_obj) 834{ 835 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj)); 836 837 return KERN_SUCCESS; 838} 839 840/* 841 * 842 */ 843kern_return_t 844apple_protect_pager_synchronize( 845 memory_object_t mem_obj, 846 memory_object_offset_t offset, 847 memory_object_size_t length, 848 __unused vm_sync_t sync_flags) 849{ 850 apple_protect_pager_t pager; 851 852 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj)); 853 854 pager = apple_protect_pager_lookup(mem_obj); 855 856 memory_object_synchronize_completed(pager->pager_control, 857 offset, length); 858 859 return KERN_SUCCESS; 860} 861 862/* 863 * apple_protect_pager_map() 864 * 865 * This allows VM to let us, the EMM, know that this memory object 866 * is currently mapped one or more times. This is called by VM each time 867 * the memory object gets mapped and we take one extra reference on the 868 * memory object to account for all its mappings. 869 */ 870kern_return_t 871apple_protect_pager_map( 872 memory_object_t mem_obj, 873 __unused vm_prot_t prot) 874{ 875 apple_protect_pager_t pager; 876 877 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj)); 878 879 pager = apple_protect_pager_lookup(mem_obj); 880 881 lck_mtx_lock(&apple_protect_pager_lock); 882 assert(pager->is_ready); 883 assert(pager->ref_count > 0); /* pager is alive */ 884 if (pager->is_mapped == FALSE) { 885 /* 886 * First mapping of this pager: take an extra reference 887 * that will remain until all the mappings of this pager 888 * are removed. 889 */ 890 pager->is_mapped = TRUE; 891 pager->ref_count++; 892 apple_protect_pager_count_mapped++; 893 } 894 lck_mtx_unlock(&apple_protect_pager_lock); 895 896 return KERN_SUCCESS; 897} 898 899/* 900 * apple_protect_pager_last_unmap() 901 * 902 * This is called by VM when this memory object is no longer mapped anywhere. 903 */ 904kern_return_t 905apple_protect_pager_last_unmap( 906 memory_object_t mem_obj) 907{ 908 apple_protect_pager_t pager; 909 int count_unmapped; 910 911 PAGER_DEBUG(PAGER_ALL, 912 ("apple_protect_pager_last_unmap: %p\n", mem_obj)); 913 914 pager = apple_protect_pager_lookup(mem_obj); 915 916 lck_mtx_lock(&apple_protect_pager_lock); 917 if (pager->is_mapped) { 918 /* 919 * All the mappings are gone, so let go of the one extra 920 * reference that represents all the mappings of this pager. 921 */ 922 apple_protect_pager_count_mapped--; 923 count_unmapped = (apple_protect_pager_count - 924 apple_protect_pager_count_mapped); 925 if (count_unmapped > apple_protect_pager_count_unmapped_max) { 926 apple_protect_pager_count_unmapped_max = count_unmapped; 927 } 928 pager->is_mapped = FALSE; 929 apple_protect_pager_deallocate_internal(pager, TRUE); 930 /* caution: deallocate_internal() released the lock ! */ 931 } else { 932 lck_mtx_unlock(&apple_protect_pager_lock); 933 } 934 935 return KERN_SUCCESS; 936} 937 938 939/* 940 * 941 */ 942apple_protect_pager_t 943apple_protect_pager_lookup( 944 memory_object_t mem_obj) 945{ 946 apple_protect_pager_t pager; 947 948 pager = (apple_protect_pager_t) mem_obj; 949 assert(pager->pager_ops == &apple_protect_pager_ops); 950 assert(pager->ref_count > 0); 951 return pager; 952} 953 954apple_protect_pager_t 955apple_protect_pager_create( 956 vm_object_t backing_object, 957 struct pager_crypt_info *crypt_info) 958{ 959 apple_protect_pager_t pager, pager2; 960 memory_object_control_t control; 961 kern_return_t kr; 962 963 pager = (apple_protect_pager_t) kalloc(sizeof (*pager)); 964 if (pager == APPLE_PROTECT_PAGER_NULL) { 965 return APPLE_PROTECT_PAGER_NULL; 966 } 967 968 /* 969 * The vm_map call takes both named entry ports and raw memory 970 * objects in the same parameter. We need to make sure that 971 * vm_map does not see this object as a named entry port. So, 972 * we reserve the first word in the object for a fake ip_kotype 973 * setting - that will tell vm_map to use it as a memory object. 974 */ 975 pager->pager_ops = &apple_protect_pager_ops; 976 pager->pager_ikot = IKOT_MEMORY_OBJECT; 977 pager->is_ready = FALSE;/* not ready until it has a "name" */ 978 pager->ref_count = 2; /* existence + setup reference */ 979 pager->is_mapped = FALSE; 980 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL; 981 pager->backing_object = backing_object; 982 pager->crypt = *crypt_info; 983 984 vm_object_reference(backing_object); 985 986 lck_mtx_lock(&apple_protect_pager_lock); 987 /* see if anyone raced us to create a pager for the same object */ 988 queue_iterate(&apple_protect_pager_queue, 989 pager2, 990 apple_protect_pager_t, 991 pager_queue) { 992 if (pager2->backing_object == backing_object) { 993 break; 994 } 995 } 996 if (! queue_end(&apple_protect_pager_queue, 997 (queue_entry_t) pager2)) { 998 /* while we hold the lock, transfer our setup ref to winner */ 999 pager2->ref_count++; 1000 /* we lost the race, down with the loser... */ 1001 lck_mtx_unlock(&apple_protect_pager_lock); 1002 vm_object_deallocate(pager->backing_object); 1003 pager->backing_object = VM_OBJECT_NULL; 1004 kfree(pager, sizeof (*pager)); 1005 /* ... and go with the winner */ 1006 pager = pager2; 1007 /* let the winner make sure the pager gets ready */ 1008 return pager; 1009 } 1010 1011 /* enter new pager at the head of our list of pagers */ 1012 queue_enter_first(&apple_protect_pager_queue, 1013 pager, 1014 apple_protect_pager_t, 1015 pager_queue); 1016 apple_protect_pager_count++; 1017 if (apple_protect_pager_count > apple_protect_pager_count_max) { 1018 apple_protect_pager_count_max = apple_protect_pager_count; 1019 } 1020 lck_mtx_unlock(&apple_protect_pager_lock); 1021 1022 kr = memory_object_create_named((memory_object_t) pager, 1023 0, 1024 &control); 1025 assert(kr == KERN_SUCCESS); 1026 1027 lck_mtx_lock(&apple_protect_pager_lock); 1028 /* the new pager is now ready to be used */ 1029 pager->is_ready = TRUE; 1030 lck_mtx_unlock(&apple_protect_pager_lock); 1031 1032 /* wakeup anyone waiting for this pager to be ready */ 1033 thread_wakeup(&pager->is_ready); 1034 1035 return pager; 1036} 1037 1038/* 1039 * apple_protect_pager_setup() 1040 * 1041 * Provide the caller with a memory object backed by the provided 1042 * "backing_object" VM object. If such a memory object already exists, 1043 * re-use it, otherwise create a new memory object. 1044 */ 1045memory_object_t 1046apple_protect_pager_setup( 1047 vm_object_t backing_object, 1048 struct pager_crypt_info *crypt_info) 1049{ 1050 apple_protect_pager_t pager; 1051 1052 lck_mtx_lock(&apple_protect_pager_lock); 1053 1054 queue_iterate(&apple_protect_pager_queue, 1055 pager, 1056 apple_protect_pager_t, 1057 pager_queue) { 1058 if (pager->backing_object == backing_object) { 1059 /* For the same object we must always use the same protection options */ 1060 if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) && 1061 (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) { 1062 lck_mtx_unlock(&apple_protect_pager_lock); 1063 return MEMORY_OBJECT_NULL; 1064 } 1065 break; 1066 } 1067 } 1068 if (queue_end(&apple_protect_pager_queue, 1069 (queue_entry_t) pager)) { 1070 /* no existing pager for this backing object */ 1071 pager = APPLE_PROTECT_PAGER_NULL; 1072 } else { 1073 /* make sure pager doesn't disappear */ 1074 pager->ref_count++; 1075 } 1076 1077 lck_mtx_unlock(&apple_protect_pager_lock); 1078 1079 if (pager == APPLE_PROTECT_PAGER_NULL) { 1080 pager = apple_protect_pager_create(backing_object, crypt_info); 1081 if (pager == APPLE_PROTECT_PAGER_NULL) { 1082 return MEMORY_OBJECT_NULL; 1083 } 1084 } 1085 1086 lck_mtx_lock(&apple_protect_pager_lock); 1087 while (!pager->is_ready) { 1088 lck_mtx_sleep(&apple_protect_pager_lock, 1089 LCK_SLEEP_DEFAULT, 1090 &pager->is_ready, 1091 THREAD_UNINT); 1092 } 1093 lck_mtx_unlock(&apple_protect_pager_lock); 1094 1095 return (memory_object_t) pager; 1096} 1097 1098void 1099apple_protect_pager_trim(void) 1100{ 1101 apple_protect_pager_t pager, prev_pager; 1102 queue_head_t trim_queue; 1103 int num_trim; 1104 int count_unmapped; 1105 1106 lck_mtx_lock(&apple_protect_pager_lock); 1107 1108 /* 1109 * We have too many pagers, try and trim some unused ones, 1110 * starting with the oldest pager at the end of the queue. 1111 */ 1112 queue_init(&trim_queue); 1113 num_trim = 0; 1114 1115 for (pager = (apple_protect_pager_t) 1116 queue_last(&apple_protect_pager_queue); 1117 !queue_end(&apple_protect_pager_queue, 1118 (queue_entry_t) pager); 1119 pager = prev_pager) { 1120 /* get prev elt before we dequeue */ 1121 prev_pager = (apple_protect_pager_t) 1122 queue_prev(&pager->pager_queue); 1123 1124 if (pager->ref_count == 2 && 1125 pager->is_ready && 1126 !pager->is_mapped) { 1127 /* this pager can be trimmed */ 1128 num_trim++; 1129 /* remove this pager from the main list ... */ 1130 apple_protect_pager_dequeue(pager); 1131 /* ... and add it to our trim queue */ 1132 queue_enter_first(&trim_queue, 1133 pager, 1134 apple_protect_pager_t, 1135 pager_queue); 1136 1137 count_unmapped = (apple_protect_pager_count - 1138 apple_protect_pager_count_mapped); 1139 if (count_unmapped <= apple_protect_pager_cache_limit) { 1140 /* we have enough pagers to trim */ 1141 break; 1142 } 1143 } 1144 } 1145 if (num_trim > apple_protect_pager_num_trim_max) { 1146 apple_protect_pager_num_trim_max = num_trim; 1147 } 1148 apple_protect_pager_num_trim_total += num_trim; 1149 1150 lck_mtx_unlock(&apple_protect_pager_lock); 1151 1152 /* terminate the trimmed pagers */ 1153 while (!queue_empty(&trim_queue)) { 1154 queue_remove_first(&trim_queue, 1155 pager, 1156 apple_protect_pager_t, 1157 pager_queue); 1158 pager->pager_queue.next = NULL; 1159 pager->pager_queue.prev = NULL; 1160 assert(pager->ref_count == 2); 1161 /* 1162 * We can't call deallocate_internal() because the pager 1163 * has already been dequeued, but we still need to remove 1164 * a reference. 1165 */ 1166 pager->ref_count--; 1167 apple_protect_pager_terminate_internal(pager); 1168 } 1169} 1170