1/* 2 * Copyright (c) 2008 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28#if !HFS_COMPRESSION 29/* we need these symbols even though compression is turned off */ 30char register_decmpfs_decompressor; 31char unregister_decmpfs_decompressor; 32#else /* HFS_COMPRESSION */ 33#include <sys/kernel.h> 34#include <sys/vnode_internal.h> 35#include <sys/file_internal.h> 36#include <sys/stat.h> 37#include <sys/fcntl.h> 38#include <sys/xattr.h> 39#include <sys/namei.h> 40#include <sys/user.h> 41#include <sys/mount_internal.h> 42#include <sys/ubc.h> 43#include <sys/decmpfs.h> 44#include <sys/uio_internal.h> 45#include <libkern/OSByteOrder.h> 46 47#pragma mark --- debugging --- 48 49#define COMPRESSION_DEBUG 0 50#define COMPRESSION_DEBUG_VERBOSE 0 51#define MALLOC_DEBUG 0 52 53static const char * 54baseName(const char *path) 55{ 56 if (!path) 57 return NULL; 58 const char *ret = path; 59 int i; 60 for (i = 0; path[i] != 0; i++) { 61 if (path[i] == '/') 62 ret = &path[i + 1]; 63 } 64 return ret; 65} 66 67#define ErrorLog(x, args...) printf("%s:%d:%s: " x, baseName(__FILE__), __LINE__, __FUNCTION__, ## args) 68 69#if COMPRESSION_DEBUG 70#define DebugLog ErrorLog 71#else 72#define DebugLog(x...) do { } while(0) 73#endif 74 75#if COMPRESSION_DEBUG_VERBOSE 76#define VerboseLog ErrorLog 77#else 78#define VerboseLog(x...) do { } while(0) 79#endif 80 81#if MALLOC_DEBUG 82 83static SInt32 totalAlloc; 84 85typedef struct { 86 uint32_t allocSz; 87 uint32_t magic; 88 const char *file; 89 int line; 90} allocated; 91 92static void * 93_malloc(uint32_t sz, __unused int type, __unused int flags, const char *file, int line) 94{ 95 uint32_t allocSz = sz + 2 * sizeof(allocated); 96 97 allocated *alloc = NULL; 98 MALLOC(alloc, allocated *, allocSz, type, flags); 99 if (!alloc) { 100 ErrorLog("malloc failed\n"); 101 return NULL; 102 } 103 104 char *ret = (char*)&alloc[1]; 105 allocated *alloc2 = (allocated*)(ret + sz); 106 107 alloc->allocSz = allocSz; 108 alloc->magic = 0xdadadada; 109 alloc->file = file; 110 alloc->line = line; 111 112 *alloc2 = *alloc; 113 114 int s = OSAddAtomic(sz, &totalAlloc); 115 ErrorLog("malloc(%d) -> %p, total allocations %d\n", sz, ret, s + sz); 116 117 return ret; 118} 119 120static void 121_free(char *ret, __unused int type, const char *file, int line) 122{ 123 if (!ret) { 124 ErrorLog("freeing null\n"); 125 return; 126 } 127 allocated *alloc = (allocated*)ret; 128 alloc--; 129 uint32_t sz = alloc->allocSz - 2 * sizeof(allocated); 130 allocated *alloc2 = (allocated*)(ret + sz); 131 132 if (alloc->magic != 0xdadadada) { 133 panic("freeing bad pointer"); 134 } 135 136 if (memcmp(alloc, alloc2, sizeof(*alloc)) != 0) { 137 panic("clobbered data"); 138 } 139 140 memset(ret, 0xce, sz); 141 alloc2->file = file; 142 alloc2->line = line; 143 FREE(alloc, type); 144 int s = OSAddAtomic(-sz, &totalAlloc); 145 ErrorLog("free(%p,%d) -> total allocations %d\n", ret, sz, s - sz); 146} 147 148#undef MALLOC 149#undef FREE 150#define MALLOC(space, cast, size, type, flags) (space) = (cast)_malloc(size, type, flags, __FILE__, __LINE__) 151#define FREE(addr, type) _free((void *)addr, type, __FILE__, __LINE__) 152 153#endif /* MALLOC_DEBUG */ 154 155#pragma mark --- globals --- 156 157static lck_grp_t *decmpfs_lockgrp; 158 159static decmpfs_registration * decompressors[CMP_MAX]; /* the registered compressors */ 160static lck_rw_t * decompressorsLock; 161static int decompress_channel; /* channel used by decompress_file to wake up waiters */ 162static lck_mtx_t *decompress_channel_mtx; 163 164vfs_context_t decmpfs_ctx; 165 166#pragma mark --- decmp_get_func --- 167 168#define offsetof_func(func) ((uintptr_t)(&(((decmpfs_registration*)NULL)->func))) 169 170static void * 171_func_from_offset(uint32_t type, uintptr_t offset) 172{ 173 /* get the function at the given offset in the registration for the given type */ 174 decmpfs_registration *reg = decompressors[type]; 175 char *regChar = (char*)reg; 176 char *func = ®Char[offset]; 177 void **funcPtr = (void**)func; 178 179 switch (reg->decmpfs_registration) { 180 case DECMPFS_REGISTRATION_VERSION_V1: 181 if (offset > offsetof_func(free_data)) 182 return NULL; 183 break; 184 case DECMPFS_REGISTRATION_VERSION_V3: 185 if (offset > offsetof_func(get_flags)) 186 return NULL; 187 break; 188 default: 189 return NULL; 190 } 191 192 return funcPtr[0]; 193} 194 195extern void IOServicePublishResource( const char * property, boolean_t value ); 196extern boolean_t IOServiceWaitForMatchingResource( const char * property, uint64_t timeout ); 197extern boolean_t IOCatalogueMatchingDriversPresent( const char * property ); 198 199static void * 200_decmp_get_func(uint32_t type, uintptr_t offset) 201{ 202 /* 203 this function should be called while holding a shared lock to decompressorsLock, 204 and will return with the lock held 205 */ 206 207 if (type >= CMP_MAX) 208 return NULL; 209 210 if (decompressors[type] != NULL) { 211 // the compressor has already registered but the function might be null 212 return _func_from_offset(type, offset); 213 } 214 215 // does IOKit know about a kext that is supposed to provide this type? 216 char providesName[80]; 217 snprintf(providesName, sizeof(providesName), "com.apple.AppleFSCompression.providesType%u", type); 218 if (IOCatalogueMatchingDriversPresent(providesName)) { 219 // there is a kext that says it will register for this type, so let's wait for it 220 char resourceName[80]; 221 uint64_t delay = 10000000ULL; // 10 milliseconds. 222 snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", type); 223 printf("waiting for %s\n", resourceName); 224 while(decompressors[type] == NULL) { 225 lck_rw_unlock_shared(decompressorsLock); // we have to unlock to allow the kext to register 226 if (IOServiceWaitForMatchingResource(resourceName, delay)) { 227 lck_rw_lock_shared(decompressorsLock); 228 break; 229 } 230 if (!IOCatalogueMatchingDriversPresent(providesName)) { 231 // 232 printf("the kext with %s is no longer present\n", providesName); 233 lck_rw_lock_shared(decompressorsLock); 234 break; 235 } 236 printf("still waiting for %s\n", resourceName); 237 delay *= 2; 238 lck_rw_lock_shared(decompressorsLock); 239 } 240 // IOKit says the kext is loaded, so it should be registered too! 241 if (decompressors[type] == NULL) { 242 ErrorLog("we found %s, but the type still isn't registered\n", providesName); 243 return NULL; 244 } 245 // it's now registered, so let's return the function 246 return _func_from_offset(type, offset); 247 } 248 249 // the compressor hasn't registered, so it never will unless someone manually kextloads it 250 ErrorLog("tried to access a compressed file of unregistered type %d\n", type); 251 return NULL; 252} 253 254#define decmp_get_func(type, func) ((typeof(((decmpfs_registration*)NULL)->func))_decmp_get_func(type, offsetof_func(func))) 255 256#pragma mark --- utilities --- 257 258#if COMPRESSION_DEBUG 259static char* 260vnpath(vnode_t vp, char *path, int len) 261{ 262 int origlen = len; 263 path[0] = 0; 264 vn_getpath(vp, path, &len); 265 path[origlen - 1] = 0; 266 return path; 267} 268 269static int 270vnsize(vnode_t vp, uint64_t *size) 271{ 272 struct vnode_attr va; 273 VATTR_INIT(&va); 274 VATTR_WANTED(&va, va_data_size); 275 int error = vnode_getattr(vp, &va, decmpfs_ctx); 276 if (error != 0) { 277 ErrorLog("vnode_getattr err %d\n", error); 278 return error; 279 } 280 *size = va.va_data_size; 281 return 0; 282} 283#endif /* COMPRESSION_DEBUG */ 284 285#pragma mark --- cnode routines --- 286 287void 288decmpfs_cnode_init(decmpfs_cnode *cp) 289{ 290 memset(cp, 0, sizeof(*cp)); 291 lck_rw_init(&cp->compressed_data_lock, decmpfs_lockgrp, NULL); 292} 293 294void 295decmpfs_cnode_destroy(decmpfs_cnode *cp) 296{ 297 lck_rw_destroy(&cp->compressed_data_lock, decmpfs_lockgrp); 298} 299 300boolean_t 301decmpfs_trylock_compressed_data(decmpfs_cnode *cp, int exclusive) 302{ 303 void *thread = current_thread(); 304 boolean_t retval = FALSE; 305 306 if (cp->lockowner == thread) { 307 /* this thread is already holding an exclusive lock, so bump the count */ 308 cp->lockcount++; 309 retval = TRUE; 310 } else if (exclusive) { 311 if ((retval = lck_rw_try_lock_exclusive(&cp->compressed_data_lock))) { 312 cp->lockowner = thread; 313 cp->lockcount = 1; 314 } 315 } else { 316 if ((retval = lck_rw_try_lock_shared(&cp->compressed_data_lock))) { 317 cp->lockowner = (void *)-1; 318 } 319 } 320 return retval; 321} 322 323void 324decmpfs_lock_compressed_data(decmpfs_cnode *cp, int exclusive) 325{ 326 void *thread = current_thread(); 327 328 if (cp->lockowner == thread) { 329 /* this thread is already holding an exclusive lock, so bump the count */ 330 cp->lockcount++; 331 } else if (exclusive) { 332 lck_rw_lock_exclusive(&cp->compressed_data_lock); 333 cp->lockowner = thread; 334 cp->lockcount = 1; 335 } else { 336 lck_rw_lock_shared(&cp->compressed_data_lock); 337 cp->lockowner = (void *)-1; 338 } 339} 340 341void 342decmpfs_unlock_compressed_data(decmpfs_cnode *cp, __unused int exclusive) 343{ 344 void *thread = current_thread(); 345 346 if (cp->lockowner == thread) { 347 /* this thread is holding an exclusive lock, so decrement the count */ 348 if ((--cp->lockcount) > 0) { 349 /* the caller still has outstanding locks, so we're done */ 350 return; 351 } 352 cp->lockowner = NULL; 353 } 354 355 lck_rw_done(&cp->compressed_data_lock); 356} 357 358uint32_t 359decmpfs_cnode_get_vnode_state(decmpfs_cnode *cp) 360{ 361 return cp->cmp_state; 362} 363 364void 365decmpfs_cnode_set_vnode_state(decmpfs_cnode *cp, uint32_t state, int skiplock) 366{ 367 if (!skiplock) decmpfs_lock_compressed_data(cp, 1); 368 cp->cmp_state = state; 369 if (state == FILE_TYPE_UNKNOWN) { 370 /* clear out the compression type too */ 371 cp->cmp_type = 0; 372 } 373 if (!skiplock) decmpfs_unlock_compressed_data(cp, 1); 374} 375 376static void 377decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode *cp, uint32_t cmp_type, int skiplock) 378{ 379 if (!skiplock) decmpfs_lock_compressed_data(cp, 1); 380 cp->cmp_type = cmp_type; 381 if (!skiplock) decmpfs_unlock_compressed_data(cp, 1); 382} 383 384static void 385decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode *cp, int minimal_xattr, int skiplock) 386{ 387 if (!skiplock) decmpfs_lock_compressed_data(cp, 1); 388 cp->cmp_minimal_xattr = minimal_xattr; 389 if (!skiplock) decmpfs_unlock_compressed_data(cp, 1); 390} 391 392uint64_t 393decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode *cp) 394{ 395 return cp->uncompressed_size; 396} 397 398static void 399decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode *cp, uint64_t size) 400{ 401 while(1) { 402 uint64_t old = cp->uncompressed_size; 403 if (OSCompareAndSwap64(old, size, (UInt64*)&cp->uncompressed_size)) { 404 return; 405 } else { 406 /* failed to write our value, so loop */ 407 } 408 } 409} 410 411static uint64_t 412decmpfs_cnode_get_decompression_flags(decmpfs_cnode *cp) 413{ 414 return cp->decompression_flags; 415} 416 417static void 418decmpfs_cnode_set_decompression_flags(decmpfs_cnode *cp, uint64_t flags) 419{ 420 while(1) { 421 uint64_t old = cp->decompression_flags; 422 if (OSCompareAndSwap64(old, flags, (UInt64*)&cp->decompression_flags)) { 423 return; 424 } else { 425 /* failed to write our value, so loop */ 426 } 427 } 428} 429 430#pragma mark --- decmpfs state routines --- 431 432static int 433decmpfs_fetch_compressed_header(vnode_t vp, decmpfs_cnode *cp, decmpfs_header **hdrOut, int returnInvalid) 434{ 435 /* 436 fetches vp's compression xattr, converting it into a decmpfs_header; returns 0 or errno 437 if returnInvalid == 1, returns the header even if the type was invalid (out of range), 438 and return ERANGE in that case 439 */ 440 441 size_t read_size = 0; 442 size_t attr_size = 0; 443 uio_t attr_uio = NULL; 444 int err = 0; 445 char *data = NULL; 446 decmpfs_header *hdr = NULL; 447 char uio_buf[ UIO_SIZEOF(1) ]; 448 449 if ((cp != NULL) && 450 (cp->cmp_type != 0) && 451 (cp->cmp_minimal_xattr != 0)) { 452 /* this file's xattr didn't have any extra data when we fetched it, so we can synthesize a header from the data in the cnode */ 453 454 MALLOC(data, char *, sizeof(decmpfs_header), M_TEMP, M_WAITOK); 455 if (!data) { 456 err = ENOMEM; 457 goto out; 458 } 459 hdr = (decmpfs_header*)data; 460 hdr->attr_size = sizeof(decmpfs_disk_header); 461 hdr->compression_magic = DECMPFS_MAGIC; 462 hdr->compression_type = cp->cmp_type; 463 hdr->uncompressed_size = decmpfs_cnode_get_vnode_cached_size(cp); 464 } else { 465 /* figure out how big the xattr is on disk */ 466 err = vn_getxattr(vp, DECMPFS_XATTR_NAME, NULL, &attr_size, XATTR_NOSECURITY, decmpfs_ctx); 467 if (err != 0) 468 goto out; 469 470 if (attr_size < sizeof(decmpfs_disk_header) || attr_size > MAX_DECMPFS_XATTR_SIZE) { 471 err = EINVAL; 472 goto out; 473 } 474 475 /* allocation includes space for the extra attr_size field of a compressed_header */ 476 MALLOC(data, char *, attr_size + sizeof(hdr->attr_size), M_TEMP, M_WAITOK); 477 if (!data) { 478 err = ENOMEM; 479 goto out; 480 } 481 482 /* read the xattr into our buffer, skipping over the attr_size field at the beginning */ 483 attr_uio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, &uio_buf[0], sizeof(uio_buf)); 484 uio_addiov(attr_uio, CAST_USER_ADDR_T(data + sizeof(hdr->attr_size)), attr_size); 485 486 err = vn_getxattr(vp, DECMPFS_XATTR_NAME, attr_uio, &read_size, XATTR_NOSECURITY, decmpfs_ctx); 487 if (err != 0) 488 goto out; 489 if (read_size != attr_size) { 490 err = EINVAL; 491 goto out; 492 } 493 hdr = (decmpfs_header*)data; 494 hdr->attr_size = attr_size; 495 /* swap the fields to native endian */ 496 hdr->compression_magic = OSSwapLittleToHostInt32(hdr->compression_magic); 497 hdr->compression_type = OSSwapLittleToHostInt32(hdr->compression_type); 498 hdr->uncompressed_size = OSSwapLittleToHostInt64(hdr->uncompressed_size); 499 } 500 501 if (hdr->compression_magic != DECMPFS_MAGIC) { 502 ErrorLog("invalid compression_magic 0x%08x, should be 0x%08x\n", hdr->compression_magic, DECMPFS_MAGIC); 503 err = EINVAL; 504 goto out; 505 } 506 507 if (hdr->compression_type >= CMP_MAX) { 508 if (returnInvalid) { 509 /* return the header even though the type is out of range */ 510 err = ERANGE; 511 } else { 512 ErrorLog("compression_type %d out of range\n", hdr->compression_type); 513 err = EINVAL; 514 } 515 goto out; 516 } 517 518out: 519 if (err && (err != ERANGE)) { 520 DebugLog("err %d\n", err); 521 if (data) FREE(data, M_TEMP); 522 *hdrOut = NULL; 523 } else { 524 *hdrOut = hdr; 525 } 526 return err; 527} 528 529static int 530decmpfs_fast_get_state(decmpfs_cnode *cp) 531{ 532 /* 533 return the cached state 534 this should *only* be called when we know that decmpfs_file_is_compressed has already been called, 535 because this implies that the cached state is valid 536 */ 537 int cmp_state = decmpfs_cnode_get_vnode_state(cp); 538 539 switch(cmp_state) { 540 case FILE_IS_NOT_COMPRESSED: 541 case FILE_IS_COMPRESSED: 542 case FILE_IS_CONVERTING: 543 return cmp_state; 544 case FILE_TYPE_UNKNOWN: 545 /* 546 we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode, 547 which should not be possible 548 */ 549 ErrorLog("decmpfs_fast_get_state called on unknown file\n"); 550 return FILE_IS_NOT_COMPRESSED; 551 default: 552 /* */ 553 ErrorLog("unknown cmp_state %d\n", cmp_state); 554 return FILE_IS_NOT_COMPRESSED; 555 } 556} 557 558static int 559decmpfs_fast_file_is_compressed(decmpfs_cnode *cp) 560{ 561 int cmp_state = decmpfs_cnode_get_vnode_state(cp); 562 563 switch(cmp_state) { 564 case FILE_IS_NOT_COMPRESSED: 565 return 0; 566 case FILE_IS_COMPRESSED: 567 case FILE_IS_CONVERTING: 568 return 1; 569 case FILE_TYPE_UNKNOWN: 570 /* 571 we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode, 572 which should not be possible 573 */ 574 ErrorLog("decmpfs_fast_get_state called on unknown file\n"); 575 return 0; 576 default: 577 /* */ 578 ErrorLog("unknown cmp_state %d\n", cmp_state); 579 return 0; 580 } 581} 582 583errno_t 584decmpfs_validate_compressed_file(vnode_t vp, decmpfs_cnode *cp) 585{ 586 /* give a compressor a chance to indicate that a compressed file is invalid */ 587 588 decmpfs_header *hdr = NULL; 589 errno_t err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); 590 if (err) { 591 /* we couldn't get the header */ 592 if (decmpfs_fast_get_state(cp) == FILE_IS_NOT_COMPRESSED) { 593 /* the file is no longer compressed, so return success */ 594 err = 0; 595 } 596 goto out; 597 } 598 599 lck_rw_lock_shared(decompressorsLock); 600 decmpfs_validate_compressed_file_func validate = decmp_get_func(hdr->compression_type, validate); 601 if (validate) { /* make sure this validation function is valid */ 602 /* is the data okay? */ 603 err = validate(vp, decmpfs_ctx, hdr); 604 } else if (decmp_get_func(hdr->compression_type, fetch) == NULL) { 605 /* the type isn't registered */ 606 err = EIO; 607 } else { 608 /* no validate registered, so nothing to do */ 609 err = 0; 610 } 611 lck_rw_unlock_shared(decompressorsLock); 612out: 613 if (hdr) FREE(hdr, M_TEMP); 614#if COMPRESSION_DEBUG 615 if (err) { 616 DebugLog("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n", err, vp->v_flag); 617 } 618#endif 619 return err; 620} 621 622int 623decmpfs_file_is_compressed(vnode_t vp, decmpfs_cnode *cp) 624{ 625 /* 626 determines whether vp points to a compressed file 627 628 to speed up this operation, we cache the result in the cnode, and do as little as possible 629 in the case where the cnode already has a valid cached state 630 631 */ 632 633 int ret = 0; 634 int error = 0; 635 uint32_t cmp_state; 636 struct vnode_attr va_fetch; 637 decmpfs_header *hdr = NULL; 638 mount_t mp = NULL; 639 int cnode_locked = 0; 640 int saveInvalid = 0; // save the header data even though the type was out of range 641 uint64_t decompression_flags = 0; 642 643 if (vnode_isnamedstream(vp)) { 644 /* 645 named streams can't be compressed 646 since named streams of the same file share the same cnode, 647 we don't want to get/set the state in the cnode, just return 0 648 */ 649 return 0; 650 } 651 652 /* examine the cached a state in this cnode */ 653 cmp_state = decmpfs_cnode_get_vnode_state(cp); 654 switch(cmp_state) { 655 case FILE_IS_NOT_COMPRESSED: 656 return 0; 657 case FILE_IS_COMPRESSED: 658 return 1; 659 case FILE_IS_CONVERTING: 660 /* treat the file as compressed, because this gives us a way to block future reads until decompression is done */ 661 return 1; 662 case FILE_TYPE_UNKNOWN: 663 /* the first time we encountered this vnode, so we need to check it out */ 664 break; 665 default: 666 /* unknown state, assume file is not compressed */ 667 ErrorLog("unknown cmp_state %d\n", cmp_state); 668 return 0; 669 } 670 671 if (!vnode_isreg(vp)) { 672 /* only regular files can be compressed */ 673 ret = FILE_IS_NOT_COMPRESSED; 674 goto done; 675 } 676 677 mp = vnode_mount(vp); 678 if (mp == NULL) { 679 /* 680 this should only be true before we mount the root filesystem 681 we short-cut this return to avoid the call to getattr below, which 682 will fail before root is mounted 683 */ 684 ret = FILE_IS_NOT_COMPRESSED; 685 goto done; 686 } 687 if ((mp->mnt_flag & MNT_LOCAL) == 0) { 688 /* compression only supported on local filesystems */ 689 ret = FILE_IS_NOT_COMPRESSED; 690 goto done; 691 } 692 693 /* lock our cnode data so that another caller doesn't change the state under us */ 694 decmpfs_lock_compressed_data(cp, 1); 695 cnode_locked = 1; 696 697 VATTR_INIT(&va_fetch); 698 VATTR_WANTED(&va_fetch, va_flags); 699 error = vnode_getattr(vp, &va_fetch, decmpfs_ctx); 700 if (error) { 701 /* failed to get the bsd flags so the file is not compressed */ 702 ret = FILE_IS_NOT_COMPRESSED; 703 goto done; 704 } 705 if (va_fetch.va_flags & UF_COMPRESSED) { 706 /* UF_COMPRESSED is on, make sure the file has the DECMPFS_XATTR_NAME xattr */ 707 error = decmpfs_fetch_compressed_header(vp, cp, &hdr, 1); 708 if ((hdr != NULL) && (error == ERANGE)) { 709 saveInvalid = 1; 710 } 711 if (error) { 712 /* failed to get the xattr so the file is not compressed */ 713 ret = FILE_IS_NOT_COMPRESSED; 714 goto done; 715 } 716 /* we got the xattr, so the file is compressed */ 717 ret = FILE_IS_COMPRESSED; 718 goto done; 719 } 720 /* UF_COMPRESSED isn't on, so the file isn't compressed */ 721 ret = FILE_IS_NOT_COMPRESSED; 722 723done: 724 if (((ret == FILE_IS_COMPRESSED) || saveInvalid) && hdr) { 725 /* 726 cache the uncompressed size away in the cnode 727 */ 728 729 if (!cnode_locked) { 730 /* 731 we should never get here since the only place ret is set to FILE_IS_COMPRESSED 732 is after the call to decmpfs_lock_compressed_data above 733 */ 734 decmpfs_lock_compressed_data(cp, 1); 735 cnode_locked = 1; 736 } 737 738 decmpfs_cnode_set_vnode_cached_size(cp, hdr->uncompressed_size); 739 decmpfs_cnode_set_vnode_state(cp, ret, 1); 740 decmpfs_cnode_set_vnode_cmp_type(cp, hdr->compression_type, 1); 741 /* remember if the xattr's size was equal to the minimal xattr */ 742 if (hdr->attr_size == sizeof(decmpfs_disk_header)) { 743 decmpfs_cnode_set_vnode_minimal_xattr(cp, 1, 1); 744 } 745 if (ret == FILE_IS_COMPRESSED) { 746 /* update the ubc's size for this file */ 747 ubc_setsize(vp, hdr->uncompressed_size); 748 749 /* update the decompression flags in the decmpfs cnode */ 750 lck_rw_lock_shared(decompressorsLock); 751 decmpfs_get_decompression_flags_func get_flags = decmp_get_func(hdr->compression_type, get_flags); 752 if (get_flags) { 753 decompression_flags = get_flags(vp, decmpfs_ctx, hdr); 754 } 755 lck_rw_unlock_shared(decompressorsLock); 756 decmpfs_cnode_set_decompression_flags(cp, decompression_flags); 757 } 758 } else { 759 /* we might have already taken the lock above; if so, skip taking it again by passing cnode_locked as the skiplock parameter */ 760 decmpfs_cnode_set_vnode_state(cp, ret, cnode_locked); 761 } 762 763 if (cnode_locked) decmpfs_unlock_compressed_data(cp, 1); 764 765 if (hdr) FREE(hdr, M_TEMP); 766 767 switch(ret) { 768 case FILE_IS_NOT_COMPRESSED: 769 return 0; 770 case FILE_IS_COMPRESSED: 771 case FILE_IS_CONVERTING: 772 return 1; 773 default: 774 /* unknown state, assume file is not compressed */ 775 ErrorLog("unknown ret %d\n", ret); 776 return 0; 777 } 778} 779 780int 781decmpfs_update_attributes(vnode_t vp, struct vnode_attr *vap) 782{ 783 int error = 0; 784 785 if (VATTR_IS_ACTIVE(vap, va_flags)) { 786 /* the BSD flags are being updated */ 787 if (vap->va_flags & UF_COMPRESSED) { 788 /* the compressed bit is being set, did it change? */ 789 struct vnode_attr va_fetch; 790 int old_flags = 0; 791 VATTR_INIT(&va_fetch); 792 VATTR_WANTED(&va_fetch, va_flags); 793 error = vnode_getattr(vp, &va_fetch, decmpfs_ctx); 794 if (error) 795 return error; 796 797 old_flags = va_fetch.va_flags; 798 799 if (!(old_flags & UF_COMPRESSED)) { 800 /* 801 * Compression bit was turned on, make sure the file has the DECMPFS_XATTR_NAME attribute. 802 * This precludes anyone from using the UF_COMPRESSED bit for anything else, and it enforces 803 * an order of operation -- you must first do the setxattr and then the chflags. 804 */ 805 806 if (VATTR_IS_ACTIVE(vap, va_data_size)) { 807 /* 808 * don't allow the caller to set the BSD flag and the size in the same call 809 * since this doesn't really make sense 810 */ 811 vap->va_flags &= ~UF_COMPRESSED; 812 return 0; 813 } 814 815 decmpfs_header *hdr = NULL; 816 error = decmpfs_fetch_compressed_header(vp, NULL, &hdr, 1); 817 if (error == 0) { 818 /* 819 allow the flag to be set since the decmpfs attribute is present 820 in that case, we also want to truncate the data fork of the file 821 */ 822 VATTR_SET_ACTIVE(vap, va_data_size); 823 vap->va_data_size = 0; 824 } else if (error == ERANGE) { 825 /* the file had a decmpfs attribute but the type was out of range, so don't muck with the file's data size */ 826 } else { 827 /* no DECMPFS_XATTR_NAME attribute, so deny the update */ 828 vap->va_flags &= ~UF_COMPRESSED; 829 } 830 if (hdr) FREE(hdr, M_TEMP); 831 } 832 } 833 } 834 835 return 0; 836} 837 838static int 839wait_for_decompress(decmpfs_cnode *cp) 840{ 841 int state; 842 lck_mtx_lock(decompress_channel_mtx); 843 do { 844 state = decmpfs_fast_get_state(cp); 845 if (state != FILE_IS_CONVERTING) { 846 /* file is not decompressing */ 847 lck_mtx_unlock(decompress_channel_mtx); 848 return state; 849 } 850 msleep((caddr_t)&decompress_channel, decompress_channel_mtx, PINOD, "wait_for_decompress", NULL); 851 } while(1); 852} 853 854#pragma mark --- decmpfs hide query routines --- 855 856int 857decmpfs_hides_rsrc(vfs_context_t ctx, decmpfs_cnode *cp) 858{ 859 /* 860 WARNING!!! 861 callers may (and do) pass NULL for ctx, so we should only use it 862 for this equality comparison 863 864 This routine should only be called after a file has already been through decmpfs_file_is_compressed 865 */ 866 867 if (ctx == decmpfs_ctx) 868 return 0; 869 870 if (!decmpfs_fast_file_is_compressed(cp)) 871 return 0; 872 873 /* all compressed files hide their resource fork */ 874 return 1; 875} 876 877int 878decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr) 879{ 880 /* 881 WARNING!!! 882 callers may (and do) pass NULL for ctx, so we should only use it 883 for this equality comparison 884 885 This routine should only be called after a file has already been through decmpfs_file_is_compressed 886 */ 887 888 if (ctx == decmpfs_ctx) 889 return 0; 890 if (strncmp(xattr, XATTR_RESOURCEFORK_NAME, 22) == 0) 891 return decmpfs_hides_rsrc(ctx, cp); 892 if (!decmpfs_fast_file_is_compressed(cp)) 893 /* file is not compressed, so don't hide this xattr */ 894 return 0; 895 if (strncmp(xattr, DECMPFS_XATTR_NAME, 11) == 0) 896 /* it's our xattr, so hide it */ 897 return 1; 898 /* don't hide this xattr */ 899 return 0; 900} 901 902#pragma mark --- registration/validation routines --- 903 904static inline int registration_valid(decmpfs_registration *registration) 905{ 906 return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3)); 907} 908 909errno_t 910register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) 911{ 912 /* called by kexts to register decompressors */ 913 914 errno_t ret = 0; 915 int locked = 0; 916 char resourceName[80]; 917 918 if ((compression_type >= CMP_MAX) || !registration_valid(registration)) { 919 ret = EINVAL; 920 goto out; 921 } 922 923 lck_rw_lock_exclusive(decompressorsLock); locked = 1; 924 925 /* make sure the registration for this type is zero */ 926 if (decompressors[compression_type] != NULL) { 927 ret = EEXIST; 928 goto out; 929 } 930 decompressors[compression_type] = registration; 931 snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type); 932 IOServicePublishResource(resourceName, TRUE); 933 934out: 935 if (locked) lck_rw_unlock_exclusive(decompressorsLock); 936 return ret; 937} 938 939errno_t 940unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) 941{ 942 /* called by kexts to unregister decompressors */ 943 944 errno_t ret = 0; 945 int locked = 0; 946 char resourceName[80]; 947 948 if ((compression_type >= CMP_MAX) || !registration_valid(registration)) { 949 ret = EINVAL; 950 goto out; 951 } 952 953 lck_rw_lock_exclusive(decompressorsLock); locked = 1; 954 if (decompressors[compression_type] != registration) { 955 ret = EEXIST; 956 goto out; 957 } 958 decompressors[compression_type] = NULL; 959 snprintf(resourceName, sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type); 960 IOServicePublishResource(resourceName, FALSE); 961 962out: 963 if (locked) lck_rw_unlock_exclusive(decompressorsLock); 964 return ret; 965} 966 967static int 968compression_type_valid(decmpfs_header *hdr) 969{ 970 /* fast pre-check to determine if the given compressor has checked in */ 971 int ret = 0; 972 973 /* every compressor must have at least a fetch function */ 974 lck_rw_lock_shared(decompressorsLock); 975 if (decmp_get_func(hdr->compression_type, fetch) != NULL) { 976 ret = 1; 977 } 978 lck_rw_unlock_shared(decompressorsLock); 979 980 return ret; 981} 982 983#pragma mark --- compression/decompression routines --- 984 985static int 986decmpfs_fetch_uncompressed_data(vnode_t vp, decmpfs_cnode *cp, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read) 987{ 988 /* get the uncompressed bytes for the specified region of vp by calling out to the registered compressor */ 989 990 int err = 0; 991 992 *bytes_read = 0; 993 994 if ((uint64_t)offset >= hdr->uncompressed_size) { 995 /* reading past end of file; nothing to do */ 996 err = 0; 997 goto out; 998 } 999 if (offset < 0) { 1000 /* tried to read from before start of file */ 1001 err = EINVAL; 1002 goto out; 1003 } 1004 if ((uint64_t)(offset + size) > hdr->uncompressed_size) { 1005 /* adjust size so we don't read past the end of the file */ 1006 size = hdr->uncompressed_size - offset; 1007 } 1008 if (size == 0) { 1009 /* nothing to read */ 1010 err = 0; 1011 goto out; 1012 } 1013 1014 lck_rw_lock_shared(decompressorsLock); 1015 decmpfs_fetch_uncompressed_data_func fetch = decmp_get_func(hdr->compression_type, fetch); 1016 if (fetch) { 1017 err = fetch(vp, decmpfs_ctx, hdr, offset, size, nvec, vec, bytes_read); 1018 lck_rw_unlock_shared(decompressorsLock); 1019 if (err == 0) { 1020 uint64_t decompression_flags = decmpfs_cnode_get_decompression_flags(cp); 1021 if (decompression_flags & DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS) { 1022#if !defined(__i386__) && !defined(__x86_64__) 1023 int i; 1024 for (i = 0; i < nvec; i++) { 1025 flush_dcache64((addr64_t)(uintptr_t)vec[i].buf, vec[i].size, FALSE); 1026 } 1027#endif 1028 } 1029 } 1030 } else { 1031 err = ENOTSUP; 1032 lck_rw_unlock_shared(decompressorsLock); 1033 } 1034 1035out: 1036 return err; 1037} 1038 1039static kern_return_t 1040commit_upl(upl_t upl, upl_offset_t pl_offset, size_t uplSize, int flags, int abort) 1041{ 1042 kern_return_t kr = 0; 1043 1044#if CONFIG_IOSCHED 1045 upl_unmark_decmp(upl); 1046#endif /* CONFIG_IOSCHED */ 1047 1048 /* commit the upl pages */ 1049 if (abort) { 1050 VerboseLog("aborting upl, flags 0x%08x\n", flags); 1051 kr = ubc_upl_abort_range(upl, pl_offset, uplSize, flags); 1052 if (kr != KERN_SUCCESS) 1053 ErrorLog("ubc_upl_commit_range error %d\n", (int)kr); 1054 } else { 1055 VerboseLog("committing upl, flags 0x%08x\n", flags | UPL_COMMIT_CLEAR_DIRTY); 1056 kr = ubc_upl_commit_range(upl, pl_offset, uplSize, flags | UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_WRITTEN_BY_KERNEL); 1057 if (kr != KERN_SUCCESS) 1058 ErrorLog("ubc_upl_commit_range error %d\n", (int)kr); 1059 } 1060 return kr; 1061} 1062 1063 1064errno_t 1065decmpfs_pagein_compressed(struct vnop_pagein_args *ap, int *is_compressed, decmpfs_cnode *cp) 1066{ 1067 /* handles a page-in request from vfs for a compressed file */ 1068 1069 int err = 0; 1070 struct vnode *vp = ap->a_vp; 1071 upl_t pl = ap->a_pl; 1072 upl_offset_t pl_offset = ap->a_pl_offset; 1073 off_t f_offset = ap->a_f_offset; 1074 size_t size = ap->a_size; 1075 int flags = ap->a_flags; 1076 off_t uplPos = 0; 1077 user_ssize_t uplSize = 0; 1078 void *data = NULL; 1079 decmpfs_header *hdr = NULL; 1080 int abort_pagein = 0; 1081 uint64_t cachedSize = 0; 1082 int cmpdata_locked = 0; 1083 1084 if(!decmpfs_trylock_compressed_data(cp, 0)) { 1085 return EAGAIN; 1086 } 1087 cmpdata_locked = 1; 1088 1089 1090 if (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)) { 1091 DebugLog("pagein: unknown flags 0x%08x\n", (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD))); 1092 } 1093 1094 err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); 1095 if (err != 0) { 1096 goto out; 1097 } 1098 1099 cachedSize = hdr->uncompressed_size; 1100 1101 if (!compression_type_valid(hdr)) { 1102 /* compressor not registered */ 1103 err = ENOTSUP; 1104 goto out; 1105 } 1106 1107#if CONFIG_IOSCHED 1108 /* Mark the UPL as the requesting UPL for decompression */ 1109 upl_mark_decmp(pl); 1110#endif /* CONFIG_IOSCHED */ 1111 1112 /* map the upl so we can fetch into it */ 1113 kern_return_t kr = ubc_upl_map(pl, (vm_offset_t*)&data); 1114 if ((kr != KERN_SUCCESS) || (data == NULL)) { 1115 err = ENOSPC; 1116#if CONFIG_IOSCHED 1117 upl_unmark_decmp(pl); 1118#endif /* CONFIG_IOSCHED */ 1119 goto out; 1120 } 1121 1122 uplPos = f_offset; 1123 uplSize = size; 1124 1125 /* clip the size to the size of the file */ 1126 if ((uint64_t)uplPos + uplSize > cachedSize) { 1127 /* truncate the read to the size of the file */ 1128 uplSize = cachedSize - uplPos; 1129 } 1130 1131 /* do the fetch */ 1132 decmpfs_vector vec; 1133 1134decompress: 1135 /* the mapped data pointer points to the first page of the page list, so we want to start filling in at an offset of pl_offset */ 1136 vec.buf = (char*)data + pl_offset; 1137 vec.size = size; 1138 1139 uint64_t did_read = 0; 1140 if (decmpfs_fast_get_state(cp) == FILE_IS_CONVERTING) { 1141 ErrorLog("unexpected pagein during decompress\n"); 1142 /* 1143 if the file is converting, this must be a recursive call to pagein from underneath a call to decmpfs_decompress_file; 1144 pretend that it succeeded but don't do anything since we're just going to write over the pages anyway 1145 */ 1146 err = 0; 1147 did_read = 0; 1148 } else { 1149 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, uplPos, uplSize, 1, &vec, &did_read); 1150 } 1151 if (err) { 1152 DebugLog("decmpfs_fetch_uncompressed_data err %d\n", err); 1153 int cmp_state = decmpfs_fast_get_state(cp); 1154 if (cmp_state == FILE_IS_CONVERTING) { 1155 DebugLog("cmp_state == FILE_IS_CONVERTING\n"); 1156 cmp_state = wait_for_decompress(cp); 1157 if (cmp_state == FILE_IS_COMPRESSED) { 1158 DebugLog("cmp_state == FILE_IS_COMPRESSED\n"); 1159 /* a decompress was attempted but it failed, let's try calling fetch again */ 1160 goto decompress; 1161 } 1162 } 1163 if (cmp_state == FILE_IS_NOT_COMPRESSED) { 1164 DebugLog("cmp_state == FILE_IS_NOT_COMPRESSED\n"); 1165 /* the file was decompressed after we started reading it */ 1166 abort_pagein = 1; /* we're not going to commit our data */ 1167 *is_compressed = 0; /* instruct caller to fall back to its normal path */ 1168 } 1169 } 1170 1171 /* zero out whatever we didn't read, and zero out the end of the last page(s) */ 1172 uint64_t total_size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 1173 if (did_read < total_size) { 1174 memset((char*)vec.buf + did_read, 0, total_size - did_read); 1175 } 1176 1177#if CONFIG_IOSCHED 1178 upl_unmark_decmp(pl); 1179#endif /* CONFIG_IOSCHED */ 1180 1181 kr = ubc_upl_unmap(pl); data = NULL; /* make sure to set data to NULL so we don't try to unmap again below */ 1182 if (kr != KERN_SUCCESS) 1183 ErrorLog("ubc_upl_unmap error %d\n", (int)kr); 1184 else { 1185 if (!abort_pagein) { 1186 /* commit our pages */ 1187 kr = commit_upl(pl, pl_offset, total_size, UPL_COMMIT_FREE_ON_EMPTY, 0); 1188 } 1189 } 1190 1191out: 1192 if (data) ubc_upl_unmap(pl); 1193 if (hdr) FREE(hdr, M_TEMP); 1194 if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0); 1195 if (err) 1196 ErrorLog("err %d\n", err); 1197 1198 return err; 1199} 1200 1201errno_t 1202decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_cnode *cp) 1203{ 1204 /* handles a read request from vfs for a compressed file */ 1205 1206 uio_t uio = ap->a_uio; 1207 vnode_t vp = ap->a_vp; 1208 int err = 0; 1209 int countInt = 0; 1210 off_t uplPos = 0; 1211 user_ssize_t uplSize = 0; 1212 user_ssize_t uplRemaining = 0; 1213 off_t curUplPos = 0; 1214 user_ssize_t curUplSize = 0; 1215 kern_return_t kr = KERN_SUCCESS; 1216 int abort_read = 0; 1217 void *data = NULL; 1218 uint64_t did_read = 0; 1219 upl_t upl = NULL; 1220 upl_page_info_t *pli = NULL; 1221 decmpfs_header *hdr = NULL; 1222 uint64_t cachedSize = 0; 1223 off_t uioPos = 0; 1224 user_ssize_t uioRemaining = 0; 1225 int cmpdata_locked = 0; 1226 1227 decmpfs_lock_compressed_data(cp, 0); cmpdata_locked = 1; 1228 1229 uplPos = uio_offset(uio); 1230 uplSize = uio_resid(uio); 1231 VerboseLog("uplPos %lld uplSize %lld\n", uplPos, uplSize); 1232 1233 cachedSize = decmpfs_cnode_get_vnode_cached_size(cp); 1234 1235 if ((uint64_t)uplPos + uplSize > cachedSize) { 1236 /* truncate the read to the size of the file */ 1237 uplSize = cachedSize - uplPos; 1238 } 1239 1240 /* give the cluster layer a chance to fill in whatever it already has */ 1241 countInt = (uplSize > INT_MAX) ? INT_MAX : uplSize; 1242 err = cluster_copy_ubc_data(vp, uio, &countInt, 0); 1243 if (err != 0) 1244 goto out; 1245 1246 /* figure out what's left */ 1247 uioPos = uio_offset(uio); 1248 uioRemaining = uio_resid(uio); 1249 if ((uint64_t)uioPos + uioRemaining > cachedSize) { 1250 /* truncate the read to the size of the file */ 1251 uioRemaining = cachedSize - uioPos; 1252 } 1253 1254 if (uioRemaining <= 0) { 1255 /* nothing left */ 1256 goto out; 1257 } 1258 1259 err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); 1260 if (err != 0) { 1261 goto out; 1262 } 1263 if (!compression_type_valid(hdr)) { 1264 err = ENOTSUP; 1265 goto out; 1266 } 1267 1268 uplPos = uioPos; 1269 uplSize = uioRemaining; 1270#if COMPRESSION_DEBUG 1271 char path[PATH_MAX]; 1272 DebugLog("%s: uplPos %lld uplSize %lld\n", vnpath(vp, path, sizeof(path)), (uint64_t)uplPos, (uint64_t)uplSize); 1273#endif 1274 1275 lck_rw_lock_shared(decompressorsLock); 1276 decmpfs_adjust_fetch_region_func adjust_fetch = decmp_get_func(hdr->compression_type, adjust_fetch); 1277 if (adjust_fetch) { 1278 /* give the compressor a chance to adjust the portion of the file that we read */ 1279 adjust_fetch(vp, decmpfs_ctx, hdr, &uplPos, &uplSize); 1280 VerboseLog("adjusted uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize); 1281 } 1282 lck_rw_unlock_shared(decompressorsLock); 1283 1284 /* clip the adjusted size to the size of the file */ 1285 if ((uint64_t)uplPos + uplSize > cachedSize) { 1286 /* truncate the read to the size of the file */ 1287 uplSize = cachedSize - uplPos; 1288 } 1289 1290 if (uplSize <= 0) { 1291 /* nothing left */ 1292 goto out; 1293 } 1294 1295 /* 1296 since we're going to create a upl for the given region of the file, 1297 make sure we're on page boundaries 1298 */ 1299 1300 if (uplPos & (PAGE_SIZE - 1)) { 1301 /* round position down to page boundary */ 1302 uplSize += (uplPos & (PAGE_SIZE - 1)); 1303 uplPos &= ~(PAGE_SIZE - 1); 1304 } 1305 /* round size up to page multiple */ 1306 uplSize = (uplSize + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 1307 1308 VerboseLog("new uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize); 1309 1310 uplRemaining = uplSize; 1311 curUplPos = uplPos; 1312 curUplSize = 0; 1313 1314 while(uplRemaining > 0) { 1315 /* start after the last upl */ 1316 curUplPos += curUplSize; 1317 1318 /* clip to max upl size */ 1319 curUplSize = uplRemaining; 1320 if (curUplSize > MAX_UPL_SIZE_BYTES) { 1321 curUplSize = MAX_UPL_SIZE_BYTES; 1322 } 1323 1324 /* create the upl */ 1325 kr = ubc_create_upl(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE); 1326 if (kr != KERN_SUCCESS) { 1327 ErrorLog("ubc_create_upl error %d\n", (int)kr); 1328 err = EINVAL; 1329 goto out; 1330 } 1331 VerboseLog("curUplPos %lld curUplSize %lld\n", (uint64_t)curUplPos, (uint64_t)curUplSize); 1332 1333#if CONFIG_IOSCHED 1334 /* Mark the UPL as the requesting UPL for decompression */ 1335 upl_mark_decmp(upl); 1336#endif /* CONFIG_IOSCHED */ 1337 1338 /* map the upl */ 1339 kr = ubc_upl_map(upl, (vm_offset_t*)&data); 1340 if (kr != KERN_SUCCESS) { 1341 1342 commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1); 1343 1344 ErrorLog("ubc_upl_map error %d\n", (int)kr); 1345 err = EINVAL; 1346 goto out; 1347 } 1348 1349 /* make sure the map succeeded */ 1350 if (!data) { 1351 1352 commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1); 1353 1354 ErrorLog("ubc_upl_map mapped null\n"); 1355 err = EINVAL; 1356 goto out; 1357 } 1358 1359 /* fetch uncompressed data into the mapped upl */ 1360 decmpfs_vector vec; 1361 decompress: 1362 vec = (decmpfs_vector){ .buf = data, .size = curUplSize }; 1363 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, curUplPos, curUplSize, 1, &vec, &did_read); 1364 if (err) { 1365 ErrorLog("decmpfs_fetch_uncompressed_data err %d\n", err); 1366 1367 /* maybe the file is converting to decompressed */ 1368 int cmp_state = decmpfs_fast_get_state(cp); 1369 if (cmp_state == FILE_IS_CONVERTING) { 1370 ErrorLog("cmp_state == FILE_IS_CONVERTING\n"); 1371 cmp_state = wait_for_decompress(cp); 1372 if (cmp_state == FILE_IS_COMPRESSED) { 1373 ErrorLog("cmp_state == FILE_IS_COMPRESSED\n"); 1374 /* a decompress was attempted but it failed, let's try fetching again */ 1375 goto decompress; 1376 } 1377 } 1378 if (cmp_state == FILE_IS_NOT_COMPRESSED) { 1379 ErrorLog("cmp_state == FILE_IS_NOT_COMPRESSED\n"); 1380 /* the file was decompressed after we started reading it */ 1381 abort_read = 1; /* we're not going to commit our data */ 1382 *is_compressed = 0; /* instruct caller to fall back to its normal path */ 1383 } 1384 kr = KERN_FAILURE; 1385 did_read = 0; 1386 } 1387 /* zero out the remainder of the last page */ 1388 memset((char*)data + did_read, 0, curUplSize - did_read); 1389 kr = ubc_upl_unmap(upl); 1390 if (kr == KERN_SUCCESS) { 1391 if (abort_read) { 1392 kr = commit_upl(upl, 0, curUplSize, UPL_ABORT_FREE_ON_EMPTY, 1); 1393 } else { 1394 VerboseLog("uioPos %lld uioRemaining %lld\n", (uint64_t)uioPos, (uint64_t)uioRemaining); 1395 if (uioRemaining) { 1396 off_t uplOff = uioPos - curUplPos; 1397 if (uplOff < 0) { 1398 ErrorLog("uplOff %lld should never be negative\n", (int64_t)uplOff); 1399 err = EINVAL; 1400 } else { 1401 off_t count = curUplPos + curUplSize - uioPos; 1402 if (count < 0) { 1403 /* this upl is entirely before the uio */ 1404 } else { 1405 if (count > uioRemaining) 1406 count = uioRemaining; 1407 int io_resid = count; 1408 err = cluster_copy_upl_data(uio, upl, uplOff, &io_resid); 1409 int copied = count - io_resid; 1410 VerboseLog("uplOff %lld count %lld copied %lld\n", (uint64_t)uplOff, (uint64_t)count, (uint64_t)copied); 1411 if (err) { 1412 ErrorLog("cluster_copy_upl_data err %d\n", err); 1413 } 1414 uioPos += copied; 1415 uioRemaining -= copied; 1416 } 1417 } 1418 } 1419 kr = commit_upl(upl, 0, curUplSize, UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE, 0); 1420 if (err) { 1421 goto out; 1422 } 1423 } 1424 } else { 1425 ErrorLog("ubc_upl_unmap error %d\n", (int)kr); 1426 } 1427 1428 uplRemaining -= curUplSize; 1429 } 1430 1431out: 1432 1433 if (hdr) FREE(hdr, M_TEMP); 1434 if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 0); 1435 if (err) {/* something went wrong */ 1436 ErrorLog("err %d\n", err); 1437 return err; 1438 } 1439 1440#if COMPRESSION_DEBUG 1441 uplSize = uio_resid(uio); 1442 if (uplSize) 1443 VerboseLog("still %lld bytes to copy\n", uplSize); 1444#endif 1445 return 0; 1446} 1447 1448int 1449decmpfs_free_compressed_data(vnode_t vp, decmpfs_cnode *cp) 1450{ 1451 /* 1452 call out to the decompressor to free remove any data associated with this compressed file 1453 then delete the file's compression xattr 1454 */ 1455 1456 decmpfs_header *hdr = NULL; 1457 int err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); 1458 if (err) { 1459 ErrorLog("decmpfs_fetch_compressed_header err %d\n", err); 1460 } else { 1461 lck_rw_lock_shared(decompressorsLock); 1462 decmpfs_free_compressed_data_func free_data = decmp_get_func(hdr->compression_type, free_data); 1463 if (free_data) { 1464 err = free_data(vp, decmpfs_ctx, hdr); 1465 } else { 1466 /* nothing to do, so no error */ 1467 err = 0; 1468 } 1469 lck_rw_unlock_shared(decompressorsLock); 1470 1471 if (err != 0) { 1472 ErrorLog("decompressor err %d\n", err); 1473 } 1474 } 1475 1476 /* delete the xattr */ 1477 err = vn_removexattr(vp, DECMPFS_XATTR_NAME, 0, decmpfs_ctx); 1478 if (err != 0) { 1479 goto out; 1480 } 1481 1482out: 1483 if (hdr) FREE(hdr, M_TEMP); 1484 return err; 1485} 1486 1487#pragma mark --- file conversion routines --- 1488 1489static int 1490unset_compressed_flag(vnode_t vp) 1491{ 1492 int err = 0; 1493 struct vnode_attr va; 1494 int new_bsdflags = 0; 1495 1496 VATTR_INIT(&va); 1497 VATTR_WANTED(&va, va_flags); 1498 err = vnode_getattr(vp, &va, decmpfs_ctx); 1499 1500 if (err != 0) { 1501 ErrorLog("vnode_getattr err %d\n", err); 1502 } else { 1503 new_bsdflags = va.va_flags & ~UF_COMPRESSED; 1504 1505 VATTR_INIT(&va); 1506 VATTR_SET(&va, va_flags, new_bsdflags); 1507 err = vnode_setattr(vp, &va, decmpfs_ctx); 1508 if (err != 0) { 1509 ErrorLog("vnode_setattr err %d\n", err); 1510 } 1511 } 1512 return err; 1513} 1514 1515int 1516decmpfs_decompress_file(vnode_t vp, decmpfs_cnode *cp, off_t toSize, int truncate_okay, int skiplock) 1517{ 1518 /* convert a compressed file to an uncompressed file */ 1519 1520 int err = 0; 1521 char *data = NULL; 1522 uio_t uio_w = 0; 1523 off_t offset = 0; 1524 uint32_t old_state = 0; 1525 uint32_t new_state = 0; 1526 int update_file_state = 0; 1527 int allocSize = 0; 1528 decmpfs_header *hdr = NULL; 1529 int cmpdata_locked = 0; 1530 off_t remaining = 0; 1531 uint64_t uncompressed_size = 0; 1532 1533 if (!skiplock) { 1534 decmpfs_lock_compressed_data(cp, 1); cmpdata_locked = 1; 1535 } 1536 1537decompress: 1538 old_state = decmpfs_fast_get_state(cp); 1539 1540 switch(old_state) { 1541 case FILE_IS_NOT_COMPRESSED: 1542 { 1543 /* someone else decompressed the file */ 1544 err = 0; 1545 goto out; 1546 } 1547 1548 case FILE_TYPE_UNKNOWN: 1549 { 1550 /* the file is in an unknown state, so update the state and retry */ 1551 (void)decmpfs_file_is_compressed(vp, cp); 1552 1553 /* try again */ 1554 goto decompress; 1555 } 1556 1557 case FILE_IS_COMPRESSED: 1558 { 1559 /* the file is compressed, so decompress it */ 1560 break; 1561 } 1562 1563 default: 1564 { 1565 /* 1566 this shouldn't happen since multiple calls to decmpfs_decompress_file lock each other out, 1567 and when decmpfs_decompress_file returns, the state should be always be set back to 1568 FILE_IS_NOT_COMPRESSED or FILE_IS_UNKNOWN 1569 */ 1570 err = EINVAL; 1571 goto out; 1572 } 1573 } 1574 1575 err = decmpfs_fetch_compressed_header(vp, cp, &hdr, 0); 1576 if (err != 0) { 1577 goto out; 1578 } 1579 1580 uncompressed_size = hdr->uncompressed_size; 1581 if (toSize == -1) 1582 toSize = hdr->uncompressed_size; 1583 1584 if (toSize == 0) { 1585 /* special case truncating the file to zero bytes */ 1586 goto nodecmp; 1587 } else if ((uint64_t)toSize > hdr->uncompressed_size) { 1588 /* the caller is trying to grow the file, so we should decompress all the data */ 1589 toSize = hdr->uncompressed_size; 1590 } 1591 1592 allocSize = MIN(64*1024, toSize); 1593 MALLOC(data, char *, allocSize, M_TEMP, M_WAITOK); 1594 if (!data) { 1595 err = ENOMEM; 1596 goto out; 1597 } 1598 1599 uio_w = uio_create(1, 0LL, UIO_SYSSPACE, UIO_WRITE); 1600 if (!uio_w) { 1601 err = ENOMEM; 1602 goto out; 1603 } 1604 uio_w->uio_flags |= UIO_FLAGS_IS_COMPRESSED_FILE; 1605 1606 remaining = toSize; 1607 1608 /* tell the buffer cache that this is an empty file */ 1609 ubc_setsize(vp, 0); 1610 1611 /* if we got here, we need to decompress the file */ 1612 decmpfs_cnode_set_vnode_state(cp, FILE_IS_CONVERTING, 1); 1613 1614 while(remaining > 0) { 1615 /* loop decompressing data from the file and writing it into the data fork */ 1616 1617 uint64_t bytes_read = 0; 1618 decmpfs_vector vec = { .buf = data, .size = MIN(allocSize, remaining) }; 1619 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset, vec.size, 1, &vec, &bytes_read); 1620 if (err != 0) { 1621 ErrorLog("decmpfs_fetch_uncompressed_data err %d\n", err); 1622 goto out; 1623 } 1624 1625 if (bytes_read == 0) { 1626 /* we're done reading data */ 1627 break; 1628 } 1629 1630 uio_reset(uio_w, offset, UIO_SYSSPACE, UIO_WRITE); 1631 err = uio_addiov(uio_w, CAST_USER_ADDR_T(data), bytes_read); 1632 if (err != 0) { 1633 ErrorLog("uio_addiov err %d\n", err); 1634 err = ENOMEM; 1635 goto out; 1636 } 1637 1638 err = VNOP_WRITE(vp, uio_w, 0, decmpfs_ctx); 1639 if (err != 0) { 1640 /* if the write failed, truncate the file to zero bytes */ 1641 ErrorLog("VNOP_WRITE err %d\n", err); 1642 break; 1643 } 1644 offset += bytes_read; 1645 remaining -= bytes_read; 1646 } 1647 1648 if (err == 0) { 1649 if (offset != toSize) { 1650 ErrorLog("file decompressed to %lld instead of %lld\n", offset, toSize); 1651 err = EINVAL; 1652 goto out; 1653 } 1654 } 1655 1656 if (err == 0) { 1657 /* sync the data and metadata */ 1658 err = VNOP_FSYNC(vp, MNT_WAIT, decmpfs_ctx); 1659 if (err != 0) { 1660 ErrorLog("VNOP_FSYNC err %d\n", err); 1661 goto out; 1662 } 1663 } 1664 1665 if (err != 0) { 1666 /* write, setattr, or fsync failed */ 1667 ErrorLog("aborting decompress, err %d\n", err); 1668 if (truncate_okay) { 1669 /* truncate anything we might have written */ 1670 int error = vnode_setsize(vp, 0, 0, decmpfs_ctx); 1671 ErrorLog("vnode_setsize err %d\n", error); 1672 } 1673 goto out; 1674 } 1675 1676nodecmp: 1677 /* if we're truncating the file to zero bytes, we'll skip ahead to here */ 1678 1679 /* unset the compressed flag */ 1680 unset_compressed_flag(vp); 1681 1682 /* free the compressed data associated with this file */ 1683 err = decmpfs_free_compressed_data(vp, cp); 1684 if (err != 0) { 1685 ErrorLog("decmpfs_free_compressed_data err %d\n", err); 1686 } 1687 1688 /* 1689 even if free_compressed_data or vnode_getattr/vnode_setattr failed, return success 1690 since we succeeded in writing all of the file data to the data fork 1691 */ 1692 err = 0; 1693 1694 /* if we got this far, the file was successfully decompressed */ 1695 update_file_state = 1; 1696 new_state = FILE_IS_NOT_COMPRESSED; 1697 1698#if COMPRESSION_DEBUG 1699 { 1700 uint64_t filesize = 0; 1701 vnsize(vp, &filesize); 1702 DebugLog("new file size %lld\n", filesize); 1703 } 1704#endif 1705 1706out: 1707 if (hdr) FREE(hdr, M_TEMP); 1708 if (data) FREE(data, M_TEMP); 1709 if (uio_w) uio_free(uio_w); 1710 1711 if (err != 0) { 1712 /* if there was a failure, reset compression flags to unknown and clear the buffer cache data */ 1713 update_file_state = 1; 1714 new_state = FILE_TYPE_UNKNOWN; 1715 if (uncompressed_size) { 1716 ubc_setsize(vp, 0); 1717 ubc_setsize(vp, uncompressed_size); 1718 } 1719 } 1720 1721 if (update_file_state) { 1722 lck_mtx_lock(decompress_channel_mtx); 1723 decmpfs_cnode_set_vnode_state(cp, new_state, 1); 1724 wakeup((caddr_t)&decompress_channel); /* wake up anyone who might have been waiting for decompression */ 1725 lck_mtx_unlock(decompress_channel_mtx); 1726 } 1727 1728 if (cmpdata_locked) decmpfs_unlock_compressed_data(cp, 1); 1729 1730 return err; 1731} 1732 1733#pragma mark --- Type1 compressor --- 1734 1735/* 1736 The "Type1" compressor stores the data fork directly in the compression xattr 1737 */ 1738 1739static int 1740decmpfs_validate_compressed_file_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr) 1741{ 1742 int err = 0; 1743 1744 if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) { 1745 err = EINVAL; 1746 goto out; 1747 } 1748out: 1749 return err; 1750} 1751 1752static int 1753decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read) 1754{ 1755 int err = 0; 1756 int i; 1757 user_ssize_t remaining; 1758 1759 if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) { 1760 err = EINVAL; 1761 goto out; 1762 } 1763 1764#if COMPRESSION_DEBUG 1765 static int dummy = 0; // prevent syslog from coalescing printfs 1766 char path[PATH_MAX]; 1767 DebugLog("%s: %d memcpy %lld at %lld\n", vnpath(vp, path, sizeof(path)), dummy++, size, (uint64_t)offset); 1768#endif 1769 1770 remaining = size; 1771 for (i = 0; (i < nvec) && (remaining > 0); i++) { 1772 user_ssize_t curCopy = vec[i].size; 1773 if (curCopy > remaining) 1774 curCopy = remaining; 1775 memcpy(vec[i].buf, hdr->attr_bytes + offset, curCopy); 1776 offset += curCopy; 1777 remaining -= curCopy; 1778 } 1779 1780 if ((bytes_read) && (err == 0)) 1781 *bytes_read = (size - remaining); 1782 1783out: 1784 return err; 1785} 1786 1787static decmpfs_registration Type1Reg = 1788{ 1789 .decmpfs_registration = DECMPFS_REGISTRATION_VERSION, 1790 .validate = decmpfs_validate_compressed_file_Type1, 1791 .adjust_fetch = NULL, /* no adjust necessary */ 1792 .fetch = decmpfs_fetch_uncompressed_data_Type1, 1793 .free_data = NULL, /* no free necessary */ 1794 .get_flags = NULL /* no flags */ 1795}; 1796 1797#pragma mark --- decmpfs initialization --- 1798 1799void decmpfs_init() 1800{ 1801 static int done = 0; 1802 if (done) return; 1803 1804 decmpfs_ctx = vfs_context_create(vfs_context_kernel()); 1805 1806 lck_grp_attr_t *attr = lck_grp_attr_alloc_init(); 1807 decmpfs_lockgrp = lck_grp_alloc_init("VFSCOMP", attr); 1808 decompressorsLock = lck_rw_alloc_init(decmpfs_lockgrp, NULL); 1809 decompress_channel_mtx = lck_mtx_alloc_init(decmpfs_lockgrp, NULL); 1810 1811 register_decmpfs_decompressor(CMP_Type1, &Type1Reg); 1812 1813 done = 1; 1814} 1815#endif /* HFS_COMPRESSION */ 1816