1/* 2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * Mach Operating System 30 * Copyright (c) 1987 Carnegie-Mellon University 31 * All rights reserved. The CMU software License Agreement specifies 32 * the terms and conditions for use and redistribution. 33 */ 34/* 35 * File: vnode_pager.c 36 * 37 * "Swap" pager that pages to/from vnodes. Also 38 * handles demand paging from files. 39 * 40 */ 41 42#include <mach/boolean.h> 43#include <sys/param.h> 44#include <sys/systm.h> 45#include <sys/user.h> 46#include <sys/proc.h> 47#include <sys/kauth.h> 48#include <sys/buf.h> 49#include <sys/uio.h> 50#include <sys/vnode_internal.h> 51#include <sys/namei.h> 52#include <sys/mount_internal.h> /* needs internal due to fhandle_t */ 53#include <sys/ubc_internal.h> 54#include <sys/lock.h> 55#include <sys/disk.h> /* For DKIOC calls */ 56 57#include <mach/mach_types.h> 58#include <mach/memory_object_types.h> 59#include <mach/memory_object_control.h> 60#include <mach/vm_map.h> 61#include <mach/mach_vm.h> 62#include <mach/upl.h> 63#include <mach/sdt.h> 64 65#include <vm/vm_map.h> 66#include <vm/vm_kern.h> 67#include <kern/zalloc.h> 68#include <kern/kalloc.h> 69#include <libkern/libkern.h> 70 71#include <vm/vnode_pager.h> 72#include <vm/vm_pageout.h> 73 74#include <kern/assert.h> 75#include <sys/kdebug.h> 76#include <machine/spl.h> 77 78#include <nfs/rpcv2.h> 79#include <nfs/nfsproto.h> 80#include <nfs/nfs.h> 81 82#include <vm/vm_protos.h> 83 84 85void 86vnode_pager_throttle() 87{ 88 struct uthread *ut; 89 90 ut = get_bsdthread_info(current_thread()); 91 92 if (ut->uu_lowpri_window) 93 throttle_lowpri_io(1); 94} 95 96 97boolean_t 98vnode_pager_isSSD(vnode_t vp) 99{ 100 if (vp->v_mount->mnt_kern_flag & MNTK_SSD) 101 return (TRUE); 102 return (FALSE); 103} 104 105#if CONFIG_IOSCHED 106void 107vnode_pager_issue_reprioritize_io(struct vnode *devvp, uint64_t blkno, uint32_t len, int priority) 108{ 109 u_int32_t blocksize = 0; 110 dk_extent_t extent; 111 dk_set_tier_t set_tier; 112 int error = 0; 113 114 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel()); 115 if (error) 116 return; 117 118 memset(&extent, 0, sizeof(dk_extent_t)); 119 memset(&set_tier, 0, sizeof(dk_set_tier_t)); 120 121 extent.offset = blkno * (u_int64_t) blocksize; 122 extent.length = len; 123 124 set_tier.extents = &extent; 125 set_tier.extentsCount = 1; 126 set_tier.tier = priority; 127 128 error = VNOP_IOCTL(devvp, DKIOCSETTIER, (caddr_t)&set_tier, 0, vfs_context_kernel()); 129 return; 130} 131#endif 132 133uint32_t 134vnode_pager_isinuse(struct vnode *vp) 135{ 136 if (vp->v_usecount > vp->v_kusecount) 137 return (1); 138 return (0); 139} 140 141uint32_t 142vnode_pager_return_throttle_io_limit(struct vnode *vp, uint32_t *limit) 143{ 144 return(cluster_throttle_io_limit(vp, limit)); 145} 146 147vm_object_offset_t 148vnode_pager_get_filesize(struct vnode *vp) 149{ 150 return (vm_object_offset_t) ubc_getsize(vp); 151} 152 153extern int safe_getpath(struct vnode *dvp, char *leafname, char *path, int _len, int *truncated_path); 154 155kern_return_t 156vnode_pager_get_name( 157 struct vnode *vp, 158 char *pathname, 159 vm_size_t pathname_len, 160 char *filename, 161 vm_size_t filename_len, 162 boolean_t *truncated_path_p) 163{ 164 *truncated_path_p = FALSE; 165 if (pathname != NULL) { 166 /* get the path name */ 167 safe_getpath(vp, NULL, 168 pathname, (int) pathname_len, 169 truncated_path_p); 170 } 171 if ((pathname == NULL || *truncated_path_p) && 172 filename != NULL) { 173 /* get the file name */ 174 const char *name; 175 176 name = vnode_getname_printable(vp); 177 strlcpy(filename, name, (size_t) filename_len); 178 vnode_putname_printable(name); 179 } 180 return KERN_SUCCESS; 181} 182 183kern_return_t 184vnode_pager_get_mtime( 185 struct vnode *vp, 186 struct timespec *current_mtime, 187 struct timespec *cs_mtime) 188{ 189 vnode_mtime(vp, current_mtime, vfs_context_current()); 190 if (cs_mtime != NULL) { 191 ubc_get_cs_mtime(vp, cs_mtime); 192 } 193 return KERN_SUCCESS; 194} 195 196kern_return_t 197vnode_pager_get_cs_blobs( 198 struct vnode *vp, 199 void **blobs) 200{ 201 *blobs = ubc_get_cs_blobs(vp); 202 return KERN_SUCCESS; 203} 204 205/* 206 * vnode_trim: 207 * Used to call the DKIOCUNMAP ioctl on the underlying disk device for the specified vnode. 208 * Trims the region at offset bytes into the file, for length bytes. 209 * 210 * Care must be taken to ensure that the vnode is sufficiently reference counted at the time this 211 * function is called; no iocounts or usecounts are taken on the vnode. 212 * This function is non-idempotent in error cases; We cannot un-discard the blocks if only some of them 213 * are successfully discarded. 214 */ 215u_int32_t vnode_trim ( 216 struct vnode *vp, 217 off_t offset, 218 size_t length) 219{ 220 daddr64_t io_blockno; /* Block number corresponding to the start of the extent */ 221 size_t io_bytecount; /* Number of bytes in current extent for the specified range */ 222 size_t trimmed = 0; 223 off_t current_offset = offset; 224 size_t remaining_length = length; 225 int error = 0; 226 u_int32_t blocksize = 0; 227 struct vnode *devvp; 228 dk_extent_t extent; 229 dk_unmap_t unmap; 230 231 232 /* Get the underlying device vnode */ 233 devvp = vp->v_mount->mnt_devvp; 234 235 /* Figure out the underlying device block size */ 236 error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE, (caddr_t)&blocksize, 0, vfs_context_kernel()); 237 if (error) { 238 goto trim_exit; 239 } 240 241 /* 242 * We may not get the entire range from offset -> offset+length in a single 243 * extent from the blockmap call. Keep looping/going until we are sure we've hit 244 * the whole range or if we encounter an error. 245 */ 246 while (trimmed < length) { 247 /* 248 * VNOP_BLOCKMAP will tell us the logical to physical block number mapping for the 249 * specified offset. It returns blocks in contiguous chunks, so if the logical range is 250 * broken into multiple extents, it must be called multiple times, increasing the offset 251 * in each call to ensure that the entire range is covered. 252 */ 253 error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, 254 &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL); 255 256 if (error) { 257 goto trim_exit; 258 } 259 /* 260 * We have a contiguous run. Prepare & issue the ioctl for the device. 261 * the DKIOCUNMAP ioctl takes offset in bytes from the start of the device. 262 */ 263 memset (&extent, 0, sizeof(dk_extent_t)); 264 memset (&unmap, 0, sizeof(dk_unmap_t)); 265 extent.offset = (uint64_t) io_blockno * (u_int64_t) blocksize; 266 extent.length = io_bytecount; 267 unmap.extents = &extent; 268 unmap.extentsCount = 1; 269 error = VNOP_IOCTL(devvp, DKIOCUNMAP, (caddr_t)&unmap, 0, vfs_context_kernel()); 270 271 if (error) { 272 goto trim_exit; 273 } 274 remaining_length = remaining_length - io_bytecount; 275 trimmed = trimmed + io_bytecount; 276 current_offset = current_offset + io_bytecount; 277 } 278trim_exit: 279 280 return error; 281 282} 283 284pager_return_t 285vnode_pageout(struct vnode *vp, 286 upl_t upl, 287 upl_offset_t upl_offset, 288 vm_object_offset_t f_offset, 289 upl_size_t size, 290 int flags, 291 int *errorp) 292{ 293 int result = PAGER_SUCCESS; 294 int error = 0; 295 int error_ret = 0; 296 daddr64_t blkno; 297 int isize; 298 int pg_index; 299 int base_index; 300 upl_offset_t offset; 301 upl_page_info_t *pl; 302 vfs_context_t ctx = vfs_context_current(); /* pager context */ 303 304 isize = (int)size; 305 306 if (isize <= 0) { 307 result = PAGER_ERROR; 308 error_ret = EINVAL; 309 goto out; 310 } 311 312 if (UBCINFOEXISTS(vp) == 0) { 313 result = PAGER_ERROR; 314 error_ret = EINVAL; 315 316 if (upl && !(flags & UPL_NOCOMMIT)) 317 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); 318 goto out; 319 } 320 if ( !(flags & UPL_VNODE_PAGER)) { 321 /* 322 * This is a pageout from the default pager, 323 * just go ahead and call vnop_pageout since 324 * it has already sorted out the dirty ranges 325 */ 326 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 327 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 328 size, 1, 0, 0, 0); 329 330 if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset, 331 (size_t)size, flags, ctx)) ) 332 result = PAGER_ERROR; 333 334 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 335 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 336 size, 1, 0, 0, 0); 337 338 goto out; 339 } 340 if (upl == NULL) { 341 int request_flags; 342 343 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEOUTV2) { 344 /* 345 * filesystem has requested the new form of VNOP_PAGEOUT for file 346 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEOUT... 347 * it is the fileystem's responsibility to grab the range we're denoting 348 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first 349 * take any locks it needs, before effectively locking the pages into a UPL... 350 */ 351 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 352 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 353 size, (int)f_offset, 0, 0, 0); 354 355 if ( (error_ret = VNOP_PAGEOUT(vp, NULL, upl_offset, (off_t)f_offset, 356 size, flags, ctx)) ) { 357 result = PAGER_ERROR; 358 } 359 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 360 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 361 size, 0, 0, 0, 0); 362 363 goto out; 364 } 365 if (flags & UPL_MSYNC) 366 request_flags = UPL_UBC_MSYNC | UPL_RET_ONLY_DIRTY; 367 else 368 request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY; 369 370 if (ubc_create_upl(vp, f_offset, size, &upl, &pl, request_flags) != KERN_SUCCESS) { 371 result = PAGER_ERROR; 372 error_ret = EINVAL; 373 goto out; 374 } 375 upl_offset = 0; 376 } else 377 pl = ubc_upl_pageinfo(upl); 378 379 /* 380 * Ignore any non-present pages at the end of the 381 * UPL so that we aren't looking at a upl that 382 * may already have been freed by the preceeding 383 * aborts/completions. 384 */ 385 base_index = upl_offset / PAGE_SIZE; 386 387 for (pg_index = (upl_offset + isize) / PAGE_SIZE; pg_index > base_index;) { 388 if (upl_page_present(pl, --pg_index)) 389 break; 390 if (pg_index == base_index) { 391 /* 392 * no pages were returned, so release 393 * our hold on the upl and leave 394 */ 395 if ( !(flags & UPL_NOCOMMIT)) 396 ubc_upl_abort_range(upl, upl_offset, isize, UPL_ABORT_FREE_ON_EMPTY); 397 398 goto out; 399 } 400 } 401 isize = ((pg_index + 1) - base_index) * PAGE_SIZE; 402 403 /* 404 * we come here for pageouts to 'real' files and 405 * for msyncs... the upl may not contain any 406 * dirty pages.. it's our responsibility to sort 407 * through it and find the 'runs' of dirty pages 408 * to call VNOP_PAGEOUT on... 409 */ 410 411 if (ubc_getsize(vp) == 0) { 412 /* 413 * if the file has been effectively deleted, then 414 * we need to go through the UPL and invalidate any 415 * buffer headers we might have that reference any 416 * of it's pages 417 */ 418 for (offset = upl_offset; isize; isize -= PAGE_SIZE, offset += PAGE_SIZE) { 419#if NFSCLIENT 420 if (vp->v_tag == VT_NFS) 421 /* check with nfs if page is OK to drop */ 422 error = nfs_buf_page_inval(vp, (off_t)f_offset); 423 else 424#endif 425 { 426 blkno = ubc_offtoblk(vp, (off_t)f_offset); 427 error = buf_invalblkno(vp, blkno, 0); 428 } 429 if (error) { 430 if ( !(flags & UPL_NOCOMMIT)) 431 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); 432 if (error_ret == 0) 433 error_ret = error; 434 result = PAGER_ERROR; 435 436 } else if ( !(flags & UPL_NOCOMMIT)) { 437 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); 438 } 439 f_offset += PAGE_SIZE; 440 } 441 goto out; 442 } 443 444 offset = upl_offset; 445 pg_index = base_index; 446 447 while (isize) { 448 int xsize; 449 int num_of_pages; 450 451 if ( !upl_page_present(pl, pg_index)) { 452 /* 453 * we asked for RET_ONLY_DIRTY, so it's possible 454 * to get back empty slots in the UPL 455 * just skip over them 456 */ 457 f_offset += PAGE_SIZE; 458 offset += PAGE_SIZE; 459 isize -= PAGE_SIZE; 460 pg_index++; 461 462 continue; 463 } 464 if ( !upl_dirty_page(pl, pg_index)) { 465 /* 466 * if the page is not dirty and reached here it is 467 * marked precious or it is due to invalidation in 468 * memory_object_lock request as part of truncation 469 * We also get here from vm_object_terminate() 470 * So all you need to do in these 471 * cases is to invalidate incore buffer if it is there 472 * Note we must not sleep here if the buffer is busy - that is 473 * a lock inversion which causes deadlock. 474 */ 475#if NFSCLIENT 476 if (vp->v_tag == VT_NFS) 477 /* check with nfs if page is OK to drop */ 478 error = nfs_buf_page_inval(vp, (off_t)f_offset); 479 else 480#endif 481 { 482 blkno = ubc_offtoblk(vp, (off_t)f_offset); 483 error = buf_invalblkno(vp, blkno, 0); 484 } 485 if (error) { 486 if ( !(flags & UPL_NOCOMMIT)) 487 ubc_upl_abort_range(upl, offset, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY); 488 if (error_ret == 0) 489 error_ret = error; 490 result = PAGER_ERROR; 491 492 } else if ( !(flags & UPL_NOCOMMIT)) { 493 ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY); 494 } 495 f_offset += PAGE_SIZE; 496 offset += PAGE_SIZE; 497 isize -= PAGE_SIZE; 498 pg_index++; 499 500 continue; 501 } 502 num_of_pages = 1; 503 xsize = isize - PAGE_SIZE; 504 505 while (xsize) { 506 if ( !upl_dirty_page(pl, pg_index + num_of_pages)) 507 break; 508 num_of_pages++; 509 xsize -= PAGE_SIZE; 510 } 511 xsize = num_of_pages * PAGE_SIZE; 512 513 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 514 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 515 xsize, (int)f_offset, 0, 0, 0); 516 517 if ( (error = VNOP_PAGEOUT(vp, upl, offset, (off_t)f_offset, 518 xsize, flags, ctx)) ) { 519 if (error_ret == 0) 520 error_ret = error; 521 result = PAGER_ERROR; 522 } 523 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 524 (MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 525 xsize, 0, 0, 0, 0); 526 527 f_offset += xsize; 528 offset += xsize; 529 isize -= xsize; 530 pg_index += num_of_pages; 531 } 532out: 533 if (errorp) 534 *errorp = error_ret; 535 536 return (result); 537} 538 539 540pager_return_t 541vnode_pagein( 542 struct vnode *vp, 543 upl_t upl, 544 upl_offset_t upl_offset, 545 vm_object_offset_t f_offset, 546 upl_size_t size, 547 int flags, 548 int *errorp) 549{ 550 upl_page_info_t *pl; 551 int result = PAGER_SUCCESS; 552 int error = 0; 553 int pages_in_upl; 554 int start_pg; 555 int last_pg; 556 int first_pg; 557 int xsize; 558 int must_commit = 1; 559 int ignore_valid_page_check = 0; 560 561 if (flags & UPL_NOCOMMIT) 562 must_commit = 0; 563 564 if (flags & UPL_IGNORE_VALID_PAGE_CHECK) 565 ignore_valid_page_check = 1; 566 567 if (UBCINFOEXISTS(vp) == 0) { 568 result = PAGER_ERROR; 569 error = PAGER_ERROR; 570 571 if (upl && must_commit) 572 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); 573 574 goto out; 575 } 576 if (upl == (upl_t)NULL) { 577 flags &= ~UPL_NOCOMMIT; 578 579 if (size > MAX_UPL_SIZE_BYTES) { 580 result = PAGER_ERROR; 581 error = PAGER_ERROR; 582 goto out; 583 } 584 if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSVNOP_PAGEINV2) { 585 /* 586 * filesystem has requested the new form of VNOP_PAGEIN for file 587 * backed objects... we will not grab the UPL befofe calling VNOP_PAGEIN... 588 * it is the fileystem's responsibility to grab the range we're denoting 589 * via 'f_offset' and 'size' into a UPL... this allows the filesystem to first 590 * take any locks it needs, before effectively locking the pages into a UPL... 591 * so we pass a NULL into the filesystem instead of a UPL pointer... the 'upl_offset' 592 * is used to identify the "must have" page in the extent... the filesystem is free 593 * to clip the extent to better fit the underlying FS blocksize if it desires as 594 * long as it continues to include the "must have" page... 'f_offset' + 'upl_offset' 595 * identifies that page 596 */ 597 if ( (error = VNOP_PAGEIN(vp, NULL, upl_offset, (off_t)f_offset, 598 size, flags, vfs_context_current())) ) { 599 result = PAGER_ERROR; 600 error = PAGER_ERROR; 601 } 602 goto out; 603 } 604 ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT); 605 606 if (upl == (upl_t)NULL) { 607 result = PAGER_ABSENT; 608 error = PAGER_ABSENT; 609 goto out; 610 } 611 ubc_upl_range_needed(upl, upl_offset / PAGE_SIZE, 1); 612 613 upl_offset = 0; 614 first_pg = 0; 615 616 /* 617 * if we get here, we've created the upl and 618 * are responsible for commiting/aborting it 619 * regardless of what the caller has passed in 620 */ 621 must_commit = 1; 622 } else { 623 pl = ubc_upl_pageinfo(upl); 624 first_pg = upl_offset / PAGE_SIZE; 625 } 626 pages_in_upl = size / PAGE_SIZE; 627 DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL); 628 629 /* 630 * before we start marching forward, we must make sure we end on 631 * a present page, otherwise we will be working with a freed 632 * upl 633 */ 634 for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) { 635 if (upl_page_present(pl, last_pg)) 636 break; 637 if (last_pg == first_pg) { 638 /* 639 * empty UPL, no pages are present 640 */ 641 if (must_commit) 642 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY); 643 goto out; 644 } 645 } 646 pages_in_upl = last_pg + 1; 647 last_pg = first_pg; 648 649 while (last_pg < pages_in_upl) { 650 /* 651 * skip over missing pages... 652 */ 653 for ( ; last_pg < pages_in_upl; last_pg++) { 654 if (upl_page_present(pl, last_pg)) 655 break; 656 } 657 658 if (ignore_valid_page_check == 1) { 659 start_pg = last_pg; 660 } else { 661 /* 662 * skip over 'valid' pages... we don't want to issue I/O for these 663 */ 664 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { 665 if (!upl_valid_page(pl, last_pg)) 666 break; 667 } 668 } 669 670 if (last_pg > start_pg) { 671 /* 672 * we've found a range of valid pages 673 * if we've got COMMIT responsibility 674 * commit this range of pages back to the 675 * cache unchanged 676 */ 677 xsize = (last_pg - start_pg) * PAGE_SIZE; 678 679 if (must_commit) 680 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY); 681 } 682 if (last_pg == pages_in_upl) 683 /* 684 * we're done... all pages that were present 685 * have either had I/O issued on them or 686 * were aborted unchanged... 687 */ 688 break; 689 690 if (!upl_page_present(pl, last_pg)) { 691 /* 692 * we found a range of valid pages 693 * terminated by a missing page... 694 * bump index to the next page and continue on 695 */ 696 last_pg++; 697 continue; 698 } 699 /* 700 * scan from the found invalid page looking for a valid 701 * or non-present page before the end of the upl is reached, if we 702 * find one, then it will be the last page of the request to 703 * 'cluster_io' 704 */ 705 for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) { 706 if (( !ignore_valid_page_check && upl_valid_page(pl, last_pg)) || !upl_page_present(pl, last_pg)) 707 break; 708 } 709 if (last_pg > start_pg) { 710 int xoff; 711 xsize = (last_pg - start_pg) * PAGE_SIZE; 712 xoff = start_pg * PAGE_SIZE; 713 714 if ( (error = VNOP_PAGEIN(vp, upl, (upl_offset_t) xoff, 715 (off_t)f_offset + xoff, 716 xsize, flags, vfs_context_current())) ) { 717 /* 718 * Usually this UPL will be aborted/committed by the lower cluster layer. 719 * 720 * a) In the case of decmpfs, however, we may return an error (EAGAIN) to avoid 721 * a deadlock with another thread already inflating the file. 722 * 723 * b) In the case of content protection, EPERM is a valid error and we should respect it. 724 * 725 * In those cases, we must take care of our UPL at this layer itself. 726 */ 727 if (must_commit) { 728 if(error == EAGAIN) { 729 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_RESTART); 730 } 731#if CONFIG_PROTECT 732 if(error == EPERM) { 733 ubc_upl_abort_range(upl, (upl_offset_t) xoff, xsize, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR); 734 } 735#endif 736 } 737 result = PAGER_ERROR; 738 error = PAGER_ERROR; 739 740 } 741 } 742 } 743out: 744 if (errorp) 745 *errorp = result; 746 747 return (error); 748} 749 750void 751vnode_pager_shutdown(void) 752{ 753 int i; 754 vnode_t vp; 755 756 for(i = 0; i < MAX_BACKING_STORE; i++) { 757 vp = (vnode_t)(bs_port_table[i]).vp; 758 if (vp) { 759 (bs_port_table[i]).vp = 0; 760 761 /* get rid of macx_swapon() reference */ 762 vnode_rele(vp); 763 } 764 } 765} 766 767 768void * 769upl_get_internal_page_list(upl_t upl) 770{ 771 return(UPL_GET_INTERNAL_PAGE_LIST(upl)); 772 773} 774