1/* 2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 57/* 58 * Default Pager. 59 * Memory Object Management. 60 */ 61 62#include "default_pager_internal.h" 63#include <default_pager/default_pager_object_server.h> 64#include <mach/memory_object_default_server.h> 65#include <mach/memory_object_control.h> 66#include <mach/memory_object_types.h> 67#include <mach/memory_object_server.h> 68#include <mach/upl.h> 69#include <mach/vm_map.h> 70#include <vm/memory_object.h> 71#include <vm/vm_pageout.h> 72#include <vm/vm_map.h> 73#include <vm/vm_protos.h> 74 75/* forward declaration */ 76vstruct_t vs_object_create(vm_size_t size); 77 78/* 79 * List of all vstructs. A specific vstruct is 80 * found directly via its port, this list is 81 * only used for monitoring purposes by the 82 * default_pager_object* calls and by ps_delete 83 * when abstract memory objects must be scanned 84 * to remove any live storage on a segment which 85 * is to be removed. 86 */ 87struct vstruct_list_head vstruct_list; 88 89__private_extern__ void 90vstruct_list_insert( 91 vstruct_t vs) 92{ 93 VSL_LOCK(); 94 queue_enter(&vstruct_list.vsl_queue, vs, vstruct_t, vs_links); 95 vstruct_list.vsl_count++; 96 VSL_UNLOCK(); 97} 98 99 100__private_extern__ void 101vstruct_list_delete( 102 vstruct_t vs) 103{ 104 queue_remove(&vstruct_list.vsl_queue, vs, vstruct_t, vs_links); 105 vstruct_list.vsl_count--; 106} 107 108/* 109 * We use the sequence numbers on requests to regulate 110 * our parallelism. In general, we allow multiple reads and writes 111 * to proceed in parallel, with the exception that reads must 112 * wait for previous writes to finish. (Because the kernel might 113 * generate a data-request for a page on the heels of a data-write 114 * for the same page, and we must avoid returning stale data.) 115 * terminate requests wait for proceeding reads and writes to finish. 116 */ 117 118static unsigned int default_pager_total = 0; /* debugging */ 119static unsigned int default_pager_wait_seqno = 0; /* debugging */ 120static unsigned int default_pager_wait_read = 0; /* debugging */ 121static unsigned int default_pager_wait_write = 0; /* debugging */ 122 123__private_extern__ void 124vs_async_wait( 125 vstruct_t vs) 126{ 127 128 ASSERT(vs->vs_async_pending >= 0); 129 while (vs->vs_async_pending > 0) { 130 vs->vs_waiting_async = TRUE; 131 assert_wait(&vs->vs_async_pending, THREAD_UNINT); 132 VS_UNLOCK(vs); 133 thread_block(THREAD_CONTINUE_NULL); 134 VS_LOCK(vs); 135 } 136 ASSERT(vs->vs_async_pending == 0); 137} 138 139 140#if PARALLEL 141/* 142 * Waits for correct sequence number. Leaves pager locked. 143 * 144 * JMM - Sequence numbers guarantee ordering of requests generated 145 * by a single thread if the receiver is multithreaded and 146 * the interfaces are asynchronous (i.e. sender can generate 147 * more than one request before the first is received in the 148 * pager). Normally, IPC would generate these number in that 149 * case. But we are trying to avoid using IPC for the in-kernel 150 * scenario. Since these are actually invoked synchronously 151 * anyway (in-kernel), we can just fake the sequence number 152 * generation here (thus avoiding the dependence on IPC). 153 */ 154__private_extern__ void 155vs_lock( 156 vstruct_t vs) 157{ 158 mach_port_seqno_t seqno; 159 160 default_pager_total++; 161 VS_LOCK(vs); 162 163 seqno = vs->vs_next_seqno++; 164 165 while (vs->vs_seqno != seqno) { 166 default_pager_wait_seqno++; 167 vs->vs_waiting_seqno = TRUE; 168 assert_wait(&vs->vs_seqno, THREAD_UNINT); 169 VS_UNLOCK(vs); 170 thread_block(THREAD_CONTINUE_NULL); 171 VS_LOCK(vs); 172 } 173} 174 175/* 176 * Increments sequence number and unlocks pager. 177 */ 178__private_extern__ void 179vs_unlock(vstruct_t vs) 180{ 181 vs->vs_seqno++; 182 if (vs->vs_waiting_seqno) { 183 vs->vs_waiting_seqno = FALSE; 184 VS_UNLOCK(vs); 185 thread_wakeup(&vs->vs_seqno); 186 return; 187 } 188 VS_UNLOCK(vs); 189} 190 191/* 192 * Start a read - one more reader. Pager must be locked. 193 */ 194__private_extern__ void 195vs_start_read( 196 vstruct_t vs) 197{ 198 vs->vs_readers++; 199} 200 201/* 202 * Wait for readers. Unlocks and relocks pager if wait needed. 203 */ 204__private_extern__ void 205vs_wait_for_readers( 206 vstruct_t vs) 207{ 208 while (vs->vs_readers != 0) { 209 default_pager_wait_read++; 210 vs->vs_waiting_read = TRUE; 211 assert_wait(&vs->vs_readers, THREAD_UNINT); 212 VS_UNLOCK(vs); 213 thread_block(THREAD_CONTINUE_NULL); 214 VS_LOCK(vs); 215 } 216} 217 218/* 219 * Finish a read. Pager is unlocked and returns unlocked. 220 */ 221__private_extern__ void 222vs_finish_read( 223 vstruct_t vs) 224{ 225 VS_LOCK(vs); 226 if (--vs->vs_readers == 0 && vs->vs_waiting_read) { 227 vs->vs_waiting_read = FALSE; 228 VS_UNLOCK(vs); 229 thread_wakeup(&vs->vs_readers); 230 return; 231 } 232 VS_UNLOCK(vs); 233} 234 235/* 236 * Start a write - one more writer. Pager must be locked. 237 */ 238__private_extern__ void 239vs_start_write( 240 vstruct_t vs) 241{ 242 vs->vs_writers++; 243} 244 245/* 246 * Wait for writers. Unlocks and relocks pager if wait needed. 247 */ 248__private_extern__ void 249vs_wait_for_writers( 250 vstruct_t vs) 251{ 252 while (vs->vs_writers != 0) { 253 default_pager_wait_write++; 254 vs->vs_waiting_write = TRUE; 255 assert_wait(&vs->vs_writers, THREAD_UNINT); 256 VS_UNLOCK(vs); 257 thread_block(THREAD_CONTINUE_NULL); 258 VS_LOCK(vs); 259 } 260 vs_async_wait(vs); 261} 262 263/* This is to be used for the transfer from segment code ONLY */ 264/* The transfer code holds off vs destruction by keeping the */ 265/* vs_async_wait count non-zero. It will not ocnflict with */ 266/* other writers on an async basis because it only writes on */ 267/* a cluster basis into fresh (as of sync time) cluster locations */ 268 269__private_extern__ void 270vs_wait_for_sync_writers( 271 vstruct_t vs) 272{ 273 while (vs->vs_writers != 0) { 274 default_pager_wait_write++; 275 vs->vs_waiting_write = TRUE; 276 assert_wait(&vs->vs_writers, THREAD_UNINT); 277 VS_UNLOCK(vs); 278 thread_block(THREAD_CONTINUE_NULL); 279 VS_LOCK(vs); 280 } 281} 282 283 284/* 285 * Finish a write. Pager is unlocked and returns unlocked. 286 */ 287__private_extern__ void 288vs_finish_write( 289 vstruct_t vs) 290{ 291 VS_LOCK(vs); 292 if (--vs->vs_writers == 0 && vs->vs_waiting_write) { 293 vs->vs_waiting_write = FALSE; 294 VS_UNLOCK(vs); 295 thread_wakeup(&vs->vs_writers); 296 return; 297 } 298 VS_UNLOCK(vs); 299} 300#endif /* PARALLEL */ 301 302vstruct_t 303vs_object_create( 304 vm_size_t size) 305{ 306 vstruct_t vs; 307 308 /* 309 * Allocate a vstruct. If there are any problems, then report them 310 * to the console. 311 */ 312 vs = ps_vstruct_create(size); 313 if (vs == VSTRUCT_NULL) { 314 dprintf(("vs_object_create: unable to allocate %s\n", 315 "-- either run swapon command or reboot")); 316 return VSTRUCT_NULL; 317 } 318 319 return vs; 320} 321 322#if 0 323void default_pager_add(vstruct_t, boolean_t); /* forward */ 324 325void 326default_pager_add( 327 vstruct_t vs, 328 boolean_t internal) 329{ 330 memory_object_t mem_obj = vs->vs_mem_obj; 331 mach_port_t pset; 332 mach_port_mscount_t sync; 333 mach_port_t previous; 334 kern_return_t kr; 335 static char here[] = "default_pager_add"; 336 337 /* 338 * The port currently has a make-send count of zero, 339 * because either we just created the port or we just 340 * received the port in a memory_object_create request. 341 */ 342 343 if (internal) { 344 /* possibly generate an immediate no-senders notification */ 345 sync = 0; 346 pset = default_pager_internal_set; 347 } else { 348 /* delay notification till send right is created */ 349 sync = 1; 350 pset = default_pager_external_set; 351 } 352 353 ipc_port_make_sonce(mem_obj); 354 ip_lock(mem_obj); /* unlocked in nsrequest below */ 355 ipc_port_nsrequest(mem_obj, sync, mem_obj, &previous); 356} 357 358#endif 359 360const struct memory_object_pager_ops default_pager_ops = { 361 dp_memory_object_reference, 362 dp_memory_object_deallocate, 363 dp_memory_object_init, 364 dp_memory_object_terminate, 365 dp_memory_object_data_request, 366 dp_memory_object_data_return, 367 dp_memory_object_data_initialize, 368 dp_memory_object_data_unlock, 369 dp_memory_object_synchronize, 370 dp_memory_object_map, 371 dp_memory_object_last_unmap, 372 "default pager" 373}; 374 375kern_return_t 376dp_memory_object_init( 377 memory_object_t mem_obj, 378 memory_object_control_t control, 379 __unused vm_size_t pager_page_size) 380{ 381 vstruct_t vs; 382 383 assert(pager_page_size == vm_page_size); 384 385 memory_object_control_reference(control); 386 387 vs_lookup(mem_obj, vs); 388 vs_lock(vs); 389 390 if (vs->vs_control != MEMORY_OBJECT_CONTROL_NULL) 391 Panic("bad request"); 392 393 vs->vs_control = control; 394 vs_unlock(vs); 395 396 return KERN_SUCCESS; 397} 398 399kern_return_t 400dp_memory_object_synchronize( 401 memory_object_t mem_obj, 402 memory_object_offset_t offset, 403 vm_size_t length, 404 __unused vm_sync_t flags) 405{ 406 vstruct_t vs; 407 408 vs_lookup(mem_obj, vs); 409 vs_lock(vs); 410 vs_unlock(vs); 411 412 memory_object_synchronize_completed(vs->vs_control, offset, length); 413 414 return KERN_SUCCESS; 415} 416 417kern_return_t 418dp_memory_object_map( 419 __unused memory_object_t mem_obj, 420 __unused vm_prot_t prot) 421{ 422 panic("dp_memory_object_map"); 423 return KERN_FAILURE; 424} 425 426kern_return_t 427dp_memory_object_last_unmap( 428 __unused memory_object_t mem_obj) 429{ 430 panic("dp_memory_object_last_unmap"); 431 return KERN_FAILURE; 432} 433 434kern_return_t 435dp_memory_object_terminate( 436 memory_object_t mem_obj) 437{ 438 memory_object_control_t control; 439 vstruct_t vs; 440 441 /* 442 * control port is a receive right, not a send right. 443 */ 444 445 vs_lookup(mem_obj, vs); 446 vs_lock(vs); 447 448 /* 449 * Wait for read and write requests to terminate. 450 */ 451 452 vs_wait_for_readers(vs); 453 vs_wait_for_writers(vs); 454 455 /* 456 * After memory_object_terminate both memory_object_init 457 * and a no-senders notification are possible, so we need 458 * to clean up our reference to the memory_object_control 459 * to prepare for a new init. 460 */ 461 462 control = vs->vs_control; 463 vs->vs_control = MEMORY_OBJECT_CONTROL_NULL; 464 465 /* a bit of special case ugliness here. Wakeup any waiting reads */ 466 /* these data requests had to be removed from the seqno traffic */ 467 /* based on a performance bottleneck with large memory objects */ 468 /* the problem will right itself with the new component based */ 469 /* synchronous interface. The new async will be able to return */ 470 /* failure during its sync phase. In the mean time ... */ 471 472 thread_wakeup(&vs->vs_writers); 473 thread_wakeup(&vs->vs_async_pending); 474 475 vs_unlock(vs); 476 477 /* 478 * Now we deallocate our reference on the control. 479 */ 480 memory_object_control_deallocate(control); 481 return KERN_SUCCESS; 482} 483 484void 485dp_memory_object_reference( 486 memory_object_t mem_obj) 487{ 488 vstruct_t vs; 489 490 vs_lookup_safe(mem_obj, vs); 491 if (vs == VSTRUCT_NULL) 492 return; 493 494 VS_LOCK(vs); 495 assert(vs->vs_references > 0); 496 vs->vs_references++; 497 VS_UNLOCK(vs); 498} 499 500void 501dp_memory_object_deallocate( 502 memory_object_t mem_obj) 503{ 504 vstruct_t vs; 505 mach_port_seqno_t seqno; 506 507 /* 508 * Because we don't give out multiple first references 509 * for a memory object, there can't be a race 510 * between getting a deallocate call and creating 511 * a new reference for the object. 512 */ 513 514 vs_lookup_safe(mem_obj, vs); 515 if (vs == VSTRUCT_NULL) 516 return; 517 518 VS_LOCK(vs); 519 if (--vs->vs_references > 0) { 520 VS_UNLOCK(vs); 521 return; 522 } 523 524 seqno = vs->vs_next_seqno++; 525 while (vs->vs_seqno != seqno) { 526 default_pager_wait_seqno++; 527 vs->vs_waiting_seqno = TRUE; 528 assert_wait(&vs->vs_seqno, THREAD_UNINT); 529 VS_UNLOCK(vs); 530 thread_block(THREAD_CONTINUE_NULL); 531 VS_LOCK(vs); 532 } 533 534 vs_async_wait(vs); /* wait for pending async IO */ 535 536 /* do not delete the vs structure until the referencing pointers */ 537 /* in the vstruct list have been expunged */ 538 539 /* get VSL_LOCK out of order by using TRY mechanism */ 540 while(!VSL_LOCK_TRY()) { 541 VS_UNLOCK(vs); 542 VSL_LOCK(); 543 VSL_UNLOCK(); 544 VS_LOCK(vs); 545 vs_async_wait(vs); /* wait for pending async IO */ 546 } 547 548 549 /* 550 * We shouldn't get a deallocation call 551 * when the kernel has the object cached. 552 */ 553 if (vs->vs_control != MEMORY_OBJECT_CONTROL_NULL) 554 Panic("bad request"); 555 556 /* 557 * Unlock the pager (though there should be no one 558 * waiting for it). 559 */ 560 VS_UNLOCK(vs); 561 562 /* Lock out paging segment removal for the duration of this */ 563 /* call. We are vulnerable to losing a paging segment we rely */ 564 /* on as soon as we remove ourselves from the VSL and unlock */ 565 566 /* Keep our thread from blocking on attempt to trigger backing */ 567 /* store release */ 568 backing_store_release_trigger_disable += 1; 569 570 /* 571 * Remove the memory object port association, and then 572 * the destroy the port itself. We must remove the object 573 * from the port list before deallocating the pager, 574 * because of default_pager_objects. 575 */ 576 vstruct_list_delete(vs); 577 VSL_UNLOCK(); 578 579 ps_vstruct_dealloc(vs); 580 581 VSL_LOCK(); 582 backing_store_release_trigger_disable -= 1; 583 if(backing_store_release_trigger_disable == 0) { 584 thread_wakeup((event_t)&backing_store_release_trigger_disable); 585 } 586 VSL_UNLOCK(); 587} 588 589kern_return_t 590dp_memory_object_data_request( 591 memory_object_t mem_obj, 592 memory_object_offset_t offset, 593 vm_size_t length, 594 __unused vm_prot_t protection_required, 595 memory_object_fault_info_t fault_info) 596{ 597 vstruct_t vs; 598 599 GSTAT(global_stats.gs_pagein_calls++); 600 601 602 /* CDY at this moment vs_lookup panics when presented with the wrong */ 603 /* port. As we are expanding this pager to support user interfaces */ 604 /* this should be changed to return kern_failure */ 605 vs_lookup(mem_obj, vs); 606 vs_lock(vs); 607 608 /* We are going to relax the strict sequencing here for performance */ 609 /* reasons. We can do this because we know that the read and */ 610 /* write threads are different and we rely on synchronization */ 611 /* of read and write requests at the cache memory_object level */ 612 /* break out wait_for_writers, all of this goes away when */ 613 /* we get real control of seqno with the new component interface */ 614 615 if (vs->vs_writers != 0) { 616 /* you can't hold on to the seqno and go */ 617 /* to sleep like that */ 618 vs_unlock(vs); /* bump internal count of seqno */ 619 VS_LOCK(vs); 620 while (vs->vs_writers != 0) { 621 default_pager_wait_write++; 622 vs->vs_waiting_write = TRUE; 623 assert_wait(&vs->vs_writers, THREAD_UNINT); 624 VS_UNLOCK(vs); 625 thread_block(THREAD_CONTINUE_NULL); 626 VS_LOCK(vs); 627 vs_async_wait(vs); 628 } 629 if(vs->vs_control == MEMORY_OBJECT_CONTROL_NULL) { 630 VS_UNLOCK(vs); 631 return KERN_FAILURE; 632 } 633 vs_start_read(vs); 634 VS_UNLOCK(vs); 635 } else { 636 vs_start_read(vs); 637 vs_unlock(vs); 638 } 639 640 /* 641 * Request must be on a page boundary and a multiple of pages. 642 */ 643 if ((offset & vm_page_mask) != 0 || (length & vm_page_mask) != 0) 644 Panic("bad alignment"); 645 646 pvs_cluster_read(vs, (vm_offset_t)offset, length, fault_info); 647 648 vs_finish_read(vs); 649 650 return KERN_SUCCESS; 651} 652 653/* 654 * memory_object_data_initialize: check whether we already have each page, and 655 * write it if we do not. The implementation is far from optimized, and 656 * also assumes that the default_pager is single-threaded. 657 */ 658/* It is questionable whether or not a pager should decide what is relevant */ 659/* and what is not in data sent from the kernel. Data initialize has been */ 660/* changed to copy back all data sent to it in preparation for its eventual */ 661/* merge with data return. It is the kernel that should decide what pages */ 662/* to write back. As of the writing of this note, this is indeed the case */ 663/* the kernel writes back one page at a time through this interface */ 664 665kern_return_t 666dp_memory_object_data_initialize( 667 memory_object_t mem_obj, 668 memory_object_offset_t offset, 669 vm_size_t size) 670{ 671 vstruct_t vs; 672 673 DP_DEBUG(DEBUG_MO_EXTERNAL, 674 ("mem_obj=0x%x,offset=0x%x,cnt=0x%x\n", 675 (int)mem_obj, (int)offset, (int)size)); 676 GSTAT(global_stats.gs_pages_init += atop_32(size)); 677 678 vs_lookup(mem_obj, vs); 679 vs_lock(vs); 680 vs_start_write(vs); 681 vs_unlock(vs); 682 683 /* 684 * Write the data via clustered writes. vs_cluster_write will 685 * loop if the address range specified crosses cluster 686 * boundaries. 687 */ 688 vs_cluster_write(vs, 0, (vm_offset_t)offset, size, FALSE, 0); 689 690 vs_finish_write(vs); 691 692 return KERN_SUCCESS; 693} 694 695kern_return_t 696dp_memory_object_data_unlock( 697 __unused memory_object_t mem_obj, 698 __unused memory_object_offset_t offset, 699 __unused vm_size_t size, 700 __unused vm_prot_t desired_access) 701{ 702 Panic("dp_memory_object_data_unlock: illegal"); 703 return KERN_FAILURE; 704} 705 706 707/*ARGSUSED8*/ 708kern_return_t 709dp_memory_object_data_return( 710 memory_object_t mem_obj, 711 memory_object_offset_t offset, 712 vm_size_t size, 713 __unused memory_object_offset_t *resid_offset, 714 __unused int *io_error, 715 __unused boolean_t dirty, 716 __unused boolean_t kernel_copy, 717 __unused int upl_flags) 718{ 719 vstruct_t vs; 720 721 DP_DEBUG(DEBUG_MO_EXTERNAL, 722 ("mem_obj=0x%x,offset=0x%x,size=0x%x\n", 723 (int)mem_obj, (int)offset, (int)size)); 724 GSTAT(global_stats.gs_pageout_calls++); 725 726 /* This routine is called by the pageout thread. The pageout thread */ 727 /* cannot be blocked by read activities unless the read activities */ 728 /* Therefore the grant of vs lock must be done on a try versus a */ 729 /* blocking basis. The code below relies on the fact that the */ 730 /* interface is synchronous. Should this interface be again async */ 731 /* for some type of pager in the future the pages will have to be */ 732 /* returned through a separate, asynchronous path. */ 733 734 vs_lookup(mem_obj, vs); 735 736 default_pager_total++; 737 if(!VS_TRY_LOCK(vs)) { 738 /* the call below will not be done by caller when we have */ 739 /* a synchronous interface */ 740 /* return KERN_LOCK_OWNED; */ 741 upl_t upl; 742 unsigned int page_list_count = 0; 743 memory_object_super_upl_request(vs->vs_control, 744 (memory_object_offset_t)offset, 745 size, size, 746 &upl, NULL, &page_list_count, 747 UPL_NOBLOCK | UPL_CLEAN_IN_PLACE 748 | UPL_NO_SYNC | UPL_COPYOUT_FROM); 749 upl_abort(upl,0); 750 upl_deallocate(upl); 751 return KERN_SUCCESS; 752 } 753 754 if ((vs->vs_seqno != vs->vs_next_seqno++) 755 || (vs->vs_readers) 756 || (vs->vs_xfer_pending)) { 757 upl_t upl; 758 unsigned int page_list_count = 0; 759 760 vs->vs_next_seqno--; 761 VS_UNLOCK(vs); 762 763 /* the call below will not be done by caller when we have */ 764 /* a synchronous interface */ 765 /* return KERN_LOCK_OWNED; */ 766 memory_object_super_upl_request(vs->vs_control, 767 (memory_object_offset_t)offset, 768 size, size, 769 &upl, NULL, &page_list_count, 770 UPL_NOBLOCK | UPL_CLEAN_IN_PLACE 771 | UPL_NO_SYNC | UPL_COPYOUT_FROM); 772 upl_abort(upl,0); 773 upl_deallocate(upl); 774 return KERN_SUCCESS; 775 } 776 777 if ((size % vm_page_size) != 0) 778 Panic("bad alignment"); 779 780 vs_start_write(vs); 781 782 783 vs->vs_async_pending += 1; /* protect from backing store contraction */ 784 vs_unlock(vs); 785 786 /* 787 * Write the data via clustered writes. vs_cluster_write will 788 * loop if the address range specified crosses cluster 789 * boundaries. 790 */ 791 vs_cluster_write(vs, 0, (vm_offset_t)offset, size, FALSE, 0); 792 793 vs_finish_write(vs); 794 795 /* temporary, need a finer lock based on cluster */ 796 797 VS_LOCK(vs); 798 vs->vs_async_pending -= 1; /* release vs_async_wait */ 799 if (vs->vs_async_pending == 0 && vs->vs_waiting_async) { 800 vs->vs_waiting_async = FALSE; 801 VS_UNLOCK(vs); 802 thread_wakeup(&vs->vs_async_pending); 803 } else { 804 VS_UNLOCK(vs); 805 } 806 807 808 return KERN_SUCCESS; 809} 810 811/* 812 * Routine: default_pager_memory_object_create 813 * Purpose: 814 * Handle requests for memory objects from the 815 * kernel. 816 * Notes: 817 * Because we only give out the default memory 818 * manager port to the kernel, we don't have to 819 * be so paranoid about the contents. 820 */ 821kern_return_t 822default_pager_memory_object_create( 823 __unused memory_object_default_t dmm, 824 vm_size_t new_size, 825 memory_object_t *new_mem_obj) 826{ 827 vstruct_t vs; 828 829 assert(dmm == default_pager_object); 830 831 vs = vs_object_create(new_size); 832 if (vs == VSTRUCT_NULL) 833 return KERN_RESOURCE_SHORTAGE; 834 835 vs->vs_next_seqno = 0; 836 837 /* 838 * Set up associations between this memory object 839 * and this default_pager structure 840 */ 841 842 vs->vs_pager_ops = &default_pager_ops; 843 vs->vs_mem_obj_ikot = IKOT_MEMORY_OBJECT; 844 845 /* 846 * After this, other threads might receive requests 847 * for this memory object or find it in the port list. 848 */ 849 850 vstruct_list_insert(vs); 851 *new_mem_obj = vs_to_mem_obj(vs); 852 return KERN_SUCCESS; 853} 854 855/* 856 * Create an external object. 857 */ 858kern_return_t 859default_pager_object_create( 860 default_pager_t default_pager, 861 vm_size_t size, 862 memory_object_t *mem_objp) 863{ 864 vstruct_t vs; 865 866 if (default_pager != default_pager_object) 867 return KERN_INVALID_ARGUMENT; 868 869 vs = vs_object_create(size); 870 if (vs == VSTRUCT_NULL) 871 return KERN_RESOURCE_SHORTAGE; 872 873 /* 874 * Set up associations between the default pager 875 * and this vstruct structure 876 */ 877 vs->vs_pager_ops = &default_pager_ops; 878 vstruct_list_insert(vs); 879 *mem_objp = vs_to_mem_obj(vs); 880 return KERN_SUCCESS; 881} 882 883kern_return_t 884default_pager_objects( 885 default_pager_t default_pager, 886 default_pager_object_array_t *objectsp, 887 mach_msg_type_number_t *ocountp, 888 mach_port_array_t *portsp, 889 mach_msg_type_number_t *pcountp) 890{ 891 vm_offset_t oaddr = 0; /* memory for objects */ 892 vm_size_t osize = 0; /* current size */ 893 default_pager_object_t * objects; 894 unsigned int opotential = 0; 895 896 vm_map_copy_t pcopy = 0; /* copy handle for pagers */ 897 vm_size_t psize = 0; /* current size */ 898 memory_object_t * pagers; 899 unsigned int ppotential = 0; 900 901 unsigned int actual; 902 unsigned int num_objects; 903 kern_return_t kr; 904 vstruct_t entry; 905 906 if (default_pager != default_pager_object) 907 return KERN_INVALID_ARGUMENT; 908 909 /* 910 * We will send no more than this many 911 */ 912 actual = vstruct_list.vsl_count; 913 914 /* 915 * Out out-of-line port arrays are simply kalloc'ed. 916 */ 917 psize = round_page(actual * sizeof * pagers); 918 ppotential = psize / sizeof * pagers; 919 pagers = (memory_object_t *)kalloc(psize); 920 if (0 == pagers) 921 return KERN_RESOURCE_SHORTAGE; 922 923 /* 924 * returned out of line data must be allocated out 925 * the ipc_kernel_map, wired down, filled in, and 926 * then "copied in" as if it had been sent by a 927 * user process. 928 */ 929 osize = round_page(actual * sizeof * objects); 930 opotential = osize / sizeof * objects; 931 kr = kmem_alloc(ipc_kernel_map, &oaddr, osize); 932 if (KERN_SUCCESS != kr) { 933 kfree(pagers, psize); 934 return KERN_RESOURCE_SHORTAGE; 935 } 936 objects = (default_pager_object_t *)oaddr; 937 938 939 /* 940 * Now scan the list. 941 */ 942 943 VSL_LOCK(); 944 945 num_objects = 0; 946 queue_iterate(&vstruct_list.vsl_queue, entry, vstruct_t, vs_links) { 947 948 memory_object_t pager; 949 vm_size_t size; 950 951 if ((num_objects >= opotential) || 952 (num_objects >= ppotential)) { 953 954 /* 955 * This should be rare. In any case, 956 * we will only miss recent objects, 957 * because they are added at the end. 958 */ 959 break; 960 } 961 962 /* 963 * Avoid interfering with normal operations 964 */ 965 if (!VS_MAP_TRY_LOCK(entry)) 966 goto not_this_one; 967 size = ps_vstruct_allocated_size(entry); 968 VS_MAP_UNLOCK(entry); 969 970 VS_LOCK(entry); 971 972 /* 973 * We need a reference for our caller. Adding this 974 * reference through the linked list could race with 975 * destruction of the object. If we find the object 976 * has no references, just give up on it. 977 */ 978 VS_LOCK(entry); 979 if (entry->vs_references == 0) { 980 VS_UNLOCK(entry); 981 goto not_this_one; 982 } 983 pager = vs_to_mem_obj(entry); 984 dp_memory_object_reference(pager); 985 VS_UNLOCK(entry); 986 987 /* the arrays are wired, so no deadlock worries */ 988 989 objects[num_objects].dpo_object = (vm_offset_t) entry; 990 objects[num_objects].dpo_size = size; 991 pagers [num_objects++] = pager; 992 continue; 993 994 not_this_one: 995 /* 996 * Do not return garbage 997 */ 998 objects[num_objects].dpo_object = (vm_offset_t) 0; 999 objects[num_objects].dpo_size = 0; 1000 pagers[num_objects++] = MEMORY_OBJECT_NULL; 1001 1002 } 1003 1004 VSL_UNLOCK(); 1005 1006 /* clear out any excess allocation */ 1007 while (num_objects < opotential) { 1008 objects[--opotential].dpo_object = (vm_offset_t) 0; 1009 objects[opotential].dpo_size = 0; 1010 } 1011 while (num_objects < ppotential) { 1012 pagers[--ppotential] = MEMORY_OBJECT_NULL; 1013 } 1014 1015 kr = vm_map_unwire(ipc_kernel_map, vm_map_trunc_page(oaddr), 1016 vm_map_round_page(oaddr + osize), FALSE); 1017 assert(KERN_SUCCESS == kr); 1018 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)oaddr, 1019 (vm_map_size_t)osize, TRUE, &pcopy); 1020 assert(KERN_SUCCESS == kr); 1021 1022 *objectsp = (default_pager_object_array_t)objects; 1023 *ocountp = num_objects; 1024 *portsp = (mach_port_array_t)pcopy; 1025 *pcountp = num_objects; 1026 1027 return KERN_SUCCESS; 1028} 1029 1030kern_return_t 1031default_pager_object_pages( 1032 default_pager_t default_pager, 1033 mach_port_t memory_object, 1034 default_pager_page_array_t *pagesp, 1035 mach_msg_type_number_t *countp) 1036{ 1037 vm_offset_t addr = 0; /* memory for page offsets */ 1038 vm_size_t size = 0; /* current memory size */ 1039 vm_map_copy_t copy; 1040 default_pager_page_t * pages = 0; 1041 unsigned int potential; 1042 unsigned int actual; 1043 kern_return_t kr; 1044 memory_object_t object; 1045 1046 if (default_pager != default_pager_object) 1047 return KERN_INVALID_ARGUMENT; 1048 1049 object = (memory_object_t) memory_object; 1050 1051 potential = 0; 1052 for (;;) { 1053 vstruct_t entry; 1054 1055 VSL_LOCK(); 1056 queue_iterate(&vstruct_list.vsl_queue, entry, vstruct_t, 1057 vs_links) { 1058 VS_LOCK(entry); 1059 if (vs_to_mem_obj(entry) == object) { 1060 VSL_UNLOCK(); 1061 goto found_object; 1062 } 1063 VS_UNLOCK(entry); 1064 } 1065 VSL_UNLOCK(); 1066 1067 /* did not find the object */ 1068 if (0 != addr) 1069 kmem_free(ipc_kernel_map, addr, size); 1070 1071 return KERN_INVALID_ARGUMENT; 1072 1073 found_object: 1074 1075 if (!VS_MAP_TRY_LOCK(entry)) { 1076 /* oh well bad luck */ 1077 int wresult; 1078 1079 VS_UNLOCK(entry); 1080 1081 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_UNINT, 1, 1000*NSEC_PER_USEC); 1082 wresult = thread_block(THREAD_CONTINUE_NULL); 1083 assert(wresult == THREAD_TIMED_OUT); 1084 continue; 1085 } 1086 1087 actual = ps_vstruct_allocated_pages(entry, pages, potential); 1088 VS_MAP_UNLOCK(entry); 1089 VS_UNLOCK(entry); 1090 1091 if (actual <= potential) 1092 break; 1093 1094 /* allocate more memory */ 1095 if (0 != addr) 1096 kmem_free(ipc_kernel_map, addr, size); 1097 1098 size = round_page(actual * sizeof * pages); 1099 kr = kmem_alloc(ipc_kernel_map, &addr, size); 1100 if (KERN_SUCCESS != kr) 1101 return KERN_RESOURCE_SHORTAGE; 1102 1103 pages = (default_pager_page_t *)addr; 1104 potential = size / sizeof * pages; 1105 } 1106 1107 /* 1108 * Clear unused memory. 1109 */ 1110 while (actual < potential) 1111 pages[--potential].dpp_offset = 0; 1112 1113 kr = vm_map_unwire(ipc_kernel_map, vm_map_trunc_page(addr), 1114 vm_map_round_page(addr + size), FALSE); 1115 assert(KERN_SUCCESS == kr); 1116 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)addr, 1117 (vm_map_size_t)size, TRUE, ©); 1118 assert(KERN_SUCCESS == kr); 1119 1120 1121 *pagesp = (default_pager_page_array_t)copy; 1122 *countp = actual; 1123 return KERN_SUCCESS; 1124} 1125