1/** 2 * compress.c - NTFS kernel compressed attributes handling. 3 * Part of the Linux-NTFS project. 4 * 5 * Copyright (c) 2001-2004 Anton Altaparmakov 6 * Copyright (c) 2002 Richard Russon 7 * 8 * This program/include file is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License as published 10 * by the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program/include file is distributed in the hope that it will be 14 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program (in the main directory of the Linux-NTFS 20 * distribution in the file COPYING); if not, write to the Free Software 21 * Foundation,Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 22 */ 23 24#include <linux/fs.h> 25#include <linux/buffer_head.h> 26#include <linux/blkdev.h> 27#include <linux/vmalloc.h> 28#include <linux/slab.h> 29 30#include "attrib.h" 31#include "inode.h" 32#include "debug.h" 33#include "ntfs.h" 34 35/** 36 * ntfs_compression_constants - enum of constants used in the compression code 37 */ 38typedef enum { 39 /* Token types and access mask. */ 40 NTFS_SYMBOL_TOKEN = 0, 41 NTFS_PHRASE_TOKEN = 1, 42 NTFS_TOKEN_MASK = 1, 43 44 /* Compression sub-block constants. */ 45 NTFS_SB_SIZE_MASK = 0x0fff, 46 NTFS_SB_SIZE = 0x1000, 47 NTFS_SB_IS_COMPRESSED = 0x8000, 48 49 /* 50 * The maximum compression block size is by definition 16 * the cluster 51 * size, with the maximum supported cluster size being 4kiB. Thus the 52 * maximum compression buffer size is 64kiB, so we use this when 53 * initializing the compression buffer. 54 */ 55 NTFS_MAX_CB_SIZE = 64 * 1024, 56} ntfs_compression_constants; 57 58/** 59 * ntfs_compression_buffer - one buffer for the decompression engine 60 */ 61static u8 *ntfs_compression_buffer = NULL; 62 63/** 64 * ntfs_cb_lock - spinlock which protects ntfs_compression_buffer 65 */ 66static DEFINE_SPINLOCK(ntfs_cb_lock); 67 68/** 69 * allocate_compression_buffers - allocate the decompression buffers 70 * 71 * Caller has to hold the ntfs_lock mutex. 72 * 73 * Return 0 on success or -ENOMEM if the allocations failed. 74 */ 75int allocate_compression_buffers(void) 76{ 77 BUG_ON(ntfs_compression_buffer); 78 79 ntfs_compression_buffer = vmalloc(NTFS_MAX_CB_SIZE); 80 if (!ntfs_compression_buffer) 81 return -ENOMEM; 82 return 0; 83} 84 85/** 86 * free_compression_buffers - free the decompression buffers 87 * 88 * Caller has to hold the ntfs_lock mutex. 89 */ 90void free_compression_buffers(void) 91{ 92 BUG_ON(!ntfs_compression_buffer); 93 vfree(ntfs_compression_buffer); 94 ntfs_compression_buffer = NULL; 95} 96 97/** 98 * zero_partial_compressed_page - zero out of bounds compressed page region 99 */ 100static void zero_partial_compressed_page(struct page *page, 101 const s64 initialized_size) 102{ 103 u8 *kp = page_address(page); 104 unsigned int kp_ofs; 105 106 ntfs_debug("Zeroing page region outside initialized size."); 107 if (((s64)page->index << PAGE_CACHE_SHIFT) >= initialized_size) { 108 clear_page(kp); 109 return; 110 } 111 kp_ofs = initialized_size & ~PAGE_CACHE_MASK; 112 memset(kp + kp_ofs, 0, PAGE_CACHE_SIZE - kp_ofs); 113 return; 114} 115 116/** 117 * handle_bounds_compressed_page - test for&handle out of bounds compressed page 118 */ 119static inline void handle_bounds_compressed_page(struct page *page, 120 const loff_t i_size, const s64 initialized_size) 121{ 122 if ((page->index >= (initialized_size >> PAGE_CACHE_SHIFT)) && 123 (initialized_size < i_size)) 124 zero_partial_compressed_page(page, initialized_size); 125 return; 126} 127 128/** 129 * ntfs_decompress - decompress a compression block into an array of pages 130 * @dest_pages: destination array of pages 131 * @dest_index: current index into @dest_pages (IN/OUT) 132 * @dest_ofs: current offset within @dest_pages[@dest_index] (IN/OUT) 133 * @dest_max_index: maximum index into @dest_pages (IN) 134 * @dest_max_ofs: maximum offset within @dest_pages[@dest_max_index] (IN) 135 * @xpage: the target page (-1 if none) (IN) 136 * @xpage_done: set to 1 if xpage was completed successfully (IN/OUT) 137 * @cb_start: compression block to decompress (IN) 138 * @cb_size: size of compression block @cb_start in bytes (IN) 139 * @i_size: file size when we started the read (IN) 140 * @initialized_size: initialized file size when we started the read (IN) 141 * 142 * The caller must have disabled preemption. ntfs_decompress() reenables it when 143 * the critical section is finished. 144 * 145 * This decompresses the compression block @cb_start into the array of 146 * destination pages @dest_pages starting at index @dest_index into @dest_pages 147 * and at offset @dest_pos into the page @dest_pages[@dest_index]. 148 * 149 * When the page @dest_pages[@xpage] is completed, @xpage_done is set to 1. 150 * If xpage is -1 or @xpage has not been completed, @xpage_done is not modified. 151 * 152 * @cb_start is a pointer to the compression block which needs decompressing 153 * and @cb_size is the size of @cb_start in bytes (8-64kiB). 154 * 155 * Return 0 if success or -EOVERFLOW on error in the compressed stream. 156 * @xpage_done indicates whether the target page (@dest_pages[@xpage]) was 157 * completed during the decompression of the compression block (@cb_start). 158 * 159 * Warning: This function *REQUIRES* PAGE_CACHE_SIZE >= 4096 or it will blow up 160 * unpredicatbly! You have been warned! 161 * 162 * Note to hackers: This function may not sleep until it has finished accessing 163 * the compression block @cb_start as it is a per-CPU buffer. 164 */ 165static int ntfs_decompress(struct page *dest_pages[], int *dest_index, 166 int *dest_ofs, const int dest_max_index, const int dest_max_ofs, 167 const int xpage, char *xpage_done, u8 *const cb_start, 168 const u32 cb_size, const loff_t i_size, 169 const s64 initialized_size) 170{ 171 /* 172 * Pointers into the compressed data, i.e. the compression block (cb), 173 * and the therein contained sub-blocks (sb). 174 */ 175 u8 *cb_end = cb_start + cb_size; /* End of cb. */ 176 u8 *cb = cb_start; /* Current position in cb. */ 177 u8 *cb_sb_start = cb; /* Beginning of the current sb in the cb. */ 178 u8 *cb_sb_end; /* End of current sb / beginning of next sb. */ 179 180 /* Variables for uncompressed data / destination. */ 181 struct page *dp; /* Current destination page being worked on. */ 182 u8 *dp_addr; /* Current pointer into dp. */ 183 u8 *dp_sb_start; /* Start of current sub-block in dp. */ 184 u8 *dp_sb_end; /* End of current sb in dp (dp_sb_start + 185 NTFS_SB_SIZE). */ 186 u16 do_sb_start; /* @dest_ofs when starting this sub-block. */ 187 u16 do_sb_end; /* @dest_ofs of end of this sb (do_sb_start + 188 NTFS_SB_SIZE). */ 189 190 /* Variables for tag and token parsing. */ 191 u8 tag; /* Current tag. */ 192 int token; /* Loop counter for the eight tokens in tag. */ 193 194 /* Need this because we can't sleep, so need two stages. */ 195 int completed_pages[dest_max_index - *dest_index + 1]; 196 int nr_completed_pages = 0; 197 198 /* Default error code. */ 199 int err = -EOVERFLOW; 200 201 ntfs_debug("Entering, cb_size = 0x%x.", cb_size); 202do_next_sb: 203 ntfs_debug("Beginning sub-block at offset = 0x%zx in the cb.", 204 cb - cb_start); 205 /* 206 * Have we reached the end of the compression block or the end of the 207 * decompressed data? The latter can happen for example if the current 208 * position in the compression block is one byte before its end so the 209 * first two checks do not detect it. 210 */ 211 if (cb == cb_end || !le16_to_cpup((le16*)cb) || 212 (*dest_index == dest_max_index && 213 *dest_ofs == dest_max_ofs)) { 214 int i; 215 216 ntfs_debug("Completed. Returning success (0)."); 217 err = 0; 218return_error: 219 /* We can sleep from now on, so we drop lock. */ 220 spin_unlock(&ntfs_cb_lock); 221 /* Second stage: finalize completed pages. */ 222 if (nr_completed_pages > 0) { 223 for (i = 0; i < nr_completed_pages; i++) { 224 int di = completed_pages[i]; 225 226 dp = dest_pages[di]; 227 /* 228 * If we are outside the initialized size, zero 229 * the out of bounds page range. 230 */ 231 handle_bounds_compressed_page(dp, i_size, 232 initialized_size); 233 flush_dcache_page(dp); 234 kunmap(dp); 235 SetPageUptodate(dp); 236 unlock_page(dp); 237 if (di == xpage) 238 *xpage_done = 1; 239 else 240 page_cache_release(dp); 241 dest_pages[di] = NULL; 242 } 243 } 244 return err; 245 } 246 247 /* Setup offsets for the current sub-block destination. */ 248 do_sb_start = *dest_ofs; 249 do_sb_end = do_sb_start + NTFS_SB_SIZE; 250 251 /* Check that we are still within allowed boundaries. */ 252 if (*dest_index == dest_max_index && do_sb_end > dest_max_ofs) 253 goto return_overflow; 254 255 /* Does the minimum size of a compressed sb overflow valid range? */ 256 if (cb + 6 > cb_end) 257 goto return_overflow; 258 259 /* Setup the current sub-block source pointers and validate range. */ 260 cb_sb_start = cb; 261 cb_sb_end = cb_sb_start + (le16_to_cpup((le16*)cb) & NTFS_SB_SIZE_MASK) 262 + 3; 263 if (cb_sb_end > cb_end) 264 goto return_overflow; 265 266 /* Get the current destination page. */ 267 dp = dest_pages[*dest_index]; 268 if (!dp) { 269 /* No page present. Skip decompression of this sub-block. */ 270 cb = cb_sb_end; 271 272 /* Advance destination position to next sub-block. */ 273 *dest_ofs = (*dest_ofs + NTFS_SB_SIZE) & ~PAGE_CACHE_MASK; 274 if (!*dest_ofs && (++*dest_index > dest_max_index)) 275 goto return_overflow; 276 goto do_next_sb; 277 } 278 279 /* We have a valid destination page. Setup the destination pointers. */ 280 dp_addr = (u8*)page_address(dp) + do_sb_start; 281 282 /* Now, we are ready to process the current sub-block (sb). */ 283 if (!(le16_to_cpup((le16*)cb) & NTFS_SB_IS_COMPRESSED)) { 284 ntfs_debug("Found uncompressed sub-block."); 285 /* This sb is not compressed, just copy it into destination. */ 286 287 /* Advance source position to first data byte. */ 288 cb += 2; 289 290 /* An uncompressed sb must be full size. */ 291 if (cb_sb_end - cb != NTFS_SB_SIZE) 292 goto return_overflow; 293 294 /* Copy the block and advance the source position. */ 295 memcpy(dp_addr, cb, NTFS_SB_SIZE); 296 cb += NTFS_SB_SIZE; 297 298 /* Advance destination position to next sub-block. */ 299 *dest_ofs += NTFS_SB_SIZE; 300 if (!(*dest_ofs &= ~PAGE_CACHE_MASK)) { 301finalize_page: 302 /* 303 * First stage: add current page index to array of 304 * completed pages. 305 */ 306 completed_pages[nr_completed_pages++] = *dest_index; 307 if (++*dest_index > dest_max_index) 308 goto return_overflow; 309 } 310 goto do_next_sb; 311 } 312 ntfs_debug("Found compressed sub-block."); 313 /* This sb is compressed, decompress it into destination. */ 314 315 /* Setup destination pointers. */ 316 dp_sb_start = dp_addr; 317 dp_sb_end = dp_sb_start + NTFS_SB_SIZE; 318 319 /* Forward to the first tag in the sub-block. */ 320 cb += 2; 321do_next_tag: 322 if (cb == cb_sb_end) { 323 /* Check if the decompressed sub-block was not full-length. */ 324 if (dp_addr < dp_sb_end) { 325 int nr_bytes = do_sb_end - *dest_ofs; 326 327 ntfs_debug("Filling incomplete sub-block with " 328 "zeroes."); 329 /* Zero remainder and update destination position. */ 330 memset(dp_addr, 0, nr_bytes); 331 *dest_ofs += nr_bytes; 332 } 333 /* We have finished the current sub-block. */ 334 if (!(*dest_ofs &= ~PAGE_CACHE_MASK)) 335 goto finalize_page; 336 goto do_next_sb; 337 } 338 339 /* Check we are still in range. */ 340 if (cb > cb_sb_end || dp_addr > dp_sb_end) 341 goto return_overflow; 342 343 /* Get the next tag and advance to first token. */ 344 tag = *cb++; 345 346 /* Parse the eight tokens described by the tag. */ 347 for (token = 0; token < 8; token++, tag >>= 1) { 348 u16 lg, pt, length, max_non_overlap; 349 register u16 i; 350 u8 *dp_back_addr; 351 352 /* Check if we are done / still in range. */ 353 if (cb >= cb_sb_end || dp_addr > dp_sb_end) 354 break; 355 356 /* Determine token type and parse appropriately.*/ 357 if ((tag & NTFS_TOKEN_MASK) == NTFS_SYMBOL_TOKEN) { 358 /* 359 * We have a symbol token, copy the symbol across, and 360 * advance the source and destination positions. 361 */ 362 *dp_addr++ = *cb++; 363 ++*dest_ofs; 364 365 /* Continue with the next token. */ 366 continue; 367 } 368 369 /* 370 * We have a phrase token. Make sure it is not the first tag in 371 * the sb as this is illegal and would confuse the code below. 372 */ 373 if (dp_addr == dp_sb_start) 374 goto return_overflow; 375 376 /* 377 * Determine the number of bytes to go back (p) and the number 378 * of bytes to copy (l). We use an optimized algorithm in which 379 * we first calculate log2(current destination position in sb), 380 * which allows determination of l and p in O(1) rather than 381 * O(n). We just need an arch-optimized log2() function now. 382 */ 383 lg = 0; 384 for (i = *dest_ofs - do_sb_start - 1; i >= 0x10; i >>= 1) 385 lg++; 386 387 /* Get the phrase token into i. */ 388 pt = le16_to_cpup((le16*)cb); 389 390 /* 391 * Calculate starting position of the byte sequence in 392 * the destination using the fact that p = (pt >> (12 - lg)) + 1 393 * and make sure we don't go too far back. 394 */ 395 dp_back_addr = dp_addr - (pt >> (12 - lg)) - 1; 396 if (dp_back_addr < dp_sb_start) 397 goto return_overflow; 398 399 /* Now calculate the length of the byte sequence. */ 400 length = (pt & (0xfff >> lg)) + 3; 401 402 /* Advance destination position and verify it is in range. */ 403 *dest_ofs += length; 404 if (*dest_ofs > do_sb_end) 405 goto return_overflow; 406 407 /* The number of non-overlapping bytes. */ 408 max_non_overlap = dp_addr - dp_back_addr; 409 410 if (length <= max_non_overlap) { 411 /* The byte sequence doesn't overlap, just copy it. */ 412 memcpy(dp_addr, dp_back_addr, length); 413 414 /* Advance destination pointer. */ 415 dp_addr += length; 416 } else { 417 /* 418 * The byte sequence does overlap, copy non-overlapping 419 * part and then do a slow byte by byte copy for the 420 * overlapping part. Also, advance the destination 421 * pointer. 422 */ 423 memcpy(dp_addr, dp_back_addr, max_non_overlap); 424 dp_addr += max_non_overlap; 425 dp_back_addr += max_non_overlap; 426 length -= max_non_overlap; 427 while (length--) 428 *dp_addr++ = *dp_back_addr++; 429 } 430 431 /* Advance source position and continue with the next token. */ 432 cb += 2; 433 } 434 435 /* No tokens left in the current tag. Continue with the next tag. */ 436 goto do_next_tag; 437 438return_overflow: 439 ntfs_error(NULL, "Failed. Returning -EOVERFLOW."); 440 goto return_error; 441} 442 443int ntfs_read_compressed_block(struct page *page) 444{ 445 loff_t i_size; 446 s64 initialized_size; 447 struct address_space *mapping = page->mapping; 448 ntfs_inode *ni = NTFS_I(mapping->host); 449 ntfs_volume *vol = ni->vol; 450 struct super_block *sb = vol->sb; 451 runlist_element *rl; 452 unsigned long flags, block_size = sb->s_blocksize; 453 unsigned char block_size_bits = sb->s_blocksize_bits; 454 u8 *cb, *cb_pos, *cb_end; 455 struct buffer_head **bhs; 456 unsigned long offset, index = page->index; 457 u32 cb_size = ni->itype.compressed.block_size; 458 u64 cb_size_mask = cb_size - 1UL; 459 VCN vcn; 460 LCN lcn; 461 /* The first wanted vcn (minimum alignment is PAGE_CACHE_SIZE). */ 462 VCN start_vcn = (((s64)index << PAGE_CACHE_SHIFT) & ~cb_size_mask) >> 463 vol->cluster_size_bits; 464 /* 465 * The first vcn after the last wanted vcn (minumum alignment is again 466 * PAGE_CACHE_SIZE. 467 */ 468 VCN end_vcn = ((((s64)(index + 1UL) << PAGE_CACHE_SHIFT) + cb_size - 1) 469 & ~cb_size_mask) >> vol->cluster_size_bits; 470 /* Number of compression blocks (cbs) in the wanted vcn range. */ 471 unsigned int nr_cbs = (end_vcn - start_vcn) << vol->cluster_size_bits 472 >> ni->itype.compressed.block_size_bits; 473 /* 474 * Number of pages required to store the uncompressed data from all 475 * compression blocks (cbs) overlapping @page. Due to alignment 476 * guarantees of start_vcn and end_vcn, no need to round up here. 477 */ 478 unsigned int nr_pages = (end_vcn - start_vcn) << 479 vol->cluster_size_bits >> PAGE_CACHE_SHIFT; 480 unsigned int xpage, max_page, cur_page, cur_ofs, i; 481 unsigned int cb_clusters, cb_max_ofs; 482 int block, max_block, cb_max_page, bhs_size, nr_bhs, err = 0; 483 struct page **pages; 484 unsigned char xpage_done = 0; 485 486 ntfs_debug("Entering, page->index = 0x%lx, cb_size = 0x%x, nr_pages = " 487 "%i.", index, cb_size, nr_pages); 488 /* 489 * Bad things happen if we get here for anything that is not an 490 * unnamed $DATA attribute. 491 */ 492 BUG_ON(ni->type != AT_DATA); 493 BUG_ON(ni->name_len); 494 495 pages = kmalloc(nr_pages * sizeof(struct page *), GFP_NOFS); 496 497 /* Allocate memory to store the buffer heads we need. */ 498 bhs_size = cb_size / block_size * sizeof(struct buffer_head *); 499 bhs = kmalloc(bhs_size, GFP_NOFS); 500 501 if (unlikely(!pages || !bhs)) { 502 kfree(bhs); 503 kfree(pages); 504 unlock_page(page); 505 ntfs_error(vol->sb, "Failed to allocate internal buffers."); 506 return -ENOMEM; 507 } 508 509 /* 510 * We have already been given one page, this is the one we must do. 511 * Once again, the alignment guarantees keep it simple. 512 */ 513 offset = start_vcn << vol->cluster_size_bits >> PAGE_CACHE_SHIFT; 514 xpage = index - offset; 515 pages[xpage] = page; 516 /* 517 * The remaining pages need to be allocated and inserted into the page 518 * cache, alignment guarantees keep all the below much simpler. (-8 519 */ 520 read_lock_irqsave(&ni->size_lock, flags); 521 i_size = i_size_read(VFS_I(ni)); 522 initialized_size = ni->initialized_size; 523 read_unlock_irqrestore(&ni->size_lock, flags); 524 max_page = ((i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) - 525 offset; 526 /* Is the page fully outside i_size? (truncate in progress) */ 527 if (xpage >= max_page) { 528 kfree(bhs); 529 kfree(pages); 530 zero_user(page, 0, PAGE_CACHE_SIZE); 531 ntfs_debug("Compressed read outside i_size - truncated?"); 532 SetPageUptodate(page); 533 unlock_page(page); 534 return 0; 535 } 536 if (nr_pages < max_page) 537 max_page = nr_pages; 538 for (i = 0; i < max_page; i++, offset++) { 539 if (i != xpage) 540 pages[i] = grab_cache_page_nowait(mapping, offset); 541 page = pages[i]; 542 if (page) { 543 /* 544 * We only (re)read the page if it isn't already read 545 * in and/or dirty or we would be losing data or at 546 * least wasting our time. 547 */ 548 if (!PageDirty(page) && (!PageUptodate(page) || 549 PageError(page))) { 550 ClearPageError(page); 551 kmap(page); 552 continue; 553 } 554 unlock_page(page); 555 page_cache_release(page); 556 pages[i] = NULL; 557 } 558 } 559 560 /* 561 * We have the runlist, and all the destination pages we need to fill. 562 * Now read the first compression block. 563 */ 564 cur_page = 0; 565 cur_ofs = 0; 566 cb_clusters = ni->itype.compressed.block_clusters; 567do_next_cb: 568 nr_cbs--; 569 nr_bhs = 0; 570 571 /* Read all cb buffer heads one cluster at a time. */ 572 rl = NULL; 573 for (vcn = start_vcn, start_vcn += cb_clusters; vcn < start_vcn; 574 vcn++) { 575 bool is_retry = false; 576 577 if (!rl) { 578lock_retry_remap: 579 down_read(&ni->runlist.lock); 580 rl = ni->runlist.rl; 581 } 582 if (likely(rl != NULL)) { 583 /* Seek to element containing target vcn. */ 584 while (rl->length && rl[1].vcn <= vcn) 585 rl++; 586 lcn = ntfs_rl_vcn_to_lcn(rl, vcn); 587 } else 588 lcn = LCN_RL_NOT_MAPPED; 589 ntfs_debug("Reading vcn = 0x%llx, lcn = 0x%llx.", 590 (unsigned long long)vcn, 591 (unsigned long long)lcn); 592 if (lcn < 0) { 593 /* 594 * When we reach the first sparse cluster we have 595 * finished with the cb. 596 */ 597 if (lcn == LCN_HOLE) 598 break; 599 if (is_retry || lcn != LCN_RL_NOT_MAPPED) 600 goto rl_err; 601 is_retry = true; 602 /* 603 * Attempt to map runlist, dropping lock for the 604 * duration. 605 */ 606 up_read(&ni->runlist.lock); 607 if (!ntfs_map_runlist(ni, vcn)) 608 goto lock_retry_remap; 609 goto map_rl_err; 610 } 611 block = lcn << vol->cluster_size_bits >> block_size_bits; 612 /* Read the lcn from device in chunks of block_size bytes. */ 613 max_block = block + (vol->cluster_size >> block_size_bits); 614 do { 615 ntfs_debug("block = 0x%x.", block); 616 if (unlikely(!(bhs[nr_bhs] = sb_getblk(sb, block)))) 617 goto getblk_err; 618 nr_bhs++; 619 } while (++block < max_block); 620 } 621 622 /* Release the lock if we took it. */ 623 if (rl) 624 up_read(&ni->runlist.lock); 625 626 /* Setup and initiate io on all buffer heads. */ 627 for (i = 0; i < nr_bhs; i++) { 628 struct buffer_head *tbh = bhs[i]; 629 630 if (!trylock_buffer(tbh)) 631 continue; 632 if (unlikely(buffer_uptodate(tbh))) { 633 unlock_buffer(tbh); 634 continue; 635 } 636 get_bh(tbh); 637 tbh->b_end_io = end_buffer_read_sync; 638 submit_bh(READ, tbh); 639 } 640 641 /* Wait for io completion on all buffer heads. */ 642 for (i = 0; i < nr_bhs; i++) { 643 struct buffer_head *tbh = bhs[i]; 644 645 if (buffer_uptodate(tbh)) 646 continue; 647 wait_on_buffer(tbh); 648 /* 649 * We need an optimization barrier here, otherwise we start 650 * hitting the below fixup code when accessing a loopback 651 * mounted ntfs partition. This indicates either there is a 652 * race condition in the loop driver or, more likely, gcc 653 * overoptimises the code without the barrier and it doesn't 654 * do the Right Thing(TM). 655 */ 656 barrier(); 657 if (unlikely(!buffer_uptodate(tbh))) { 658 ntfs_warning(vol->sb, "Buffer is unlocked but not " 659 "uptodate! Unplugging the disk queue " 660 "and rescheduling."); 661 get_bh(tbh); 662 blk_run_address_space(mapping); 663 schedule(); 664 put_bh(tbh); 665 if (unlikely(!buffer_uptodate(tbh))) 666 goto read_err; 667 ntfs_warning(vol->sb, "Buffer is now uptodate. Good."); 668 } 669 } 670 671 /* 672 * Get the compression buffer. We must not sleep any more 673 * until we are finished with it. 674 */ 675 spin_lock(&ntfs_cb_lock); 676 cb = ntfs_compression_buffer; 677 678 BUG_ON(!cb); 679 680 cb_pos = cb; 681 cb_end = cb + cb_size; 682 683 /* Copy the buffer heads into the contiguous buffer. */ 684 for (i = 0; i < nr_bhs; i++) { 685 memcpy(cb_pos, bhs[i]->b_data, block_size); 686 cb_pos += block_size; 687 } 688 689 /* Just a precaution. */ 690 if (cb_pos + 2 <= cb + cb_size) 691 *(u16*)cb_pos = 0; 692 693 /* Reset cb_pos back to the beginning. */ 694 cb_pos = cb; 695 696 /* We now have both source (if present) and destination. */ 697 ntfs_debug("Successfully read the compression block."); 698 699 /* The last page and maximum offset within it for the current cb. */ 700 cb_max_page = (cur_page << PAGE_CACHE_SHIFT) + cur_ofs + cb_size; 701 cb_max_ofs = cb_max_page & ~PAGE_CACHE_MASK; 702 cb_max_page >>= PAGE_CACHE_SHIFT; 703 704 /* Catch end of file inside a compression block. */ 705 if (cb_max_page > max_page) 706 cb_max_page = max_page; 707 708 if (vcn == start_vcn - cb_clusters) { 709 /* Sparse cb, zero out page range overlapping the cb. */ 710 ntfs_debug("Found sparse compression block."); 711 /* We can sleep from now on, so we drop lock. */ 712 spin_unlock(&ntfs_cb_lock); 713 if (cb_max_ofs) 714 cb_max_page--; 715 for (; cur_page < cb_max_page; cur_page++) { 716 page = pages[cur_page]; 717 if (page) { 718 if (likely(!cur_ofs)) 719 clear_page(page_address(page)); 720 else 721 memset(page_address(page) + cur_ofs, 0, 722 PAGE_CACHE_SIZE - 723 cur_ofs); 724 flush_dcache_page(page); 725 kunmap(page); 726 SetPageUptodate(page); 727 unlock_page(page); 728 if (cur_page == xpage) 729 xpage_done = 1; 730 else 731 page_cache_release(page); 732 pages[cur_page] = NULL; 733 } 734 cb_pos += PAGE_CACHE_SIZE - cur_ofs; 735 cur_ofs = 0; 736 if (cb_pos >= cb_end) 737 break; 738 } 739 /* If we have a partial final page, deal with it now. */ 740 if (cb_max_ofs && cb_pos < cb_end) { 741 page = pages[cur_page]; 742 if (page) 743 memset(page_address(page) + cur_ofs, 0, 744 cb_max_ofs - cur_ofs); 745 /* 746 * No need to update cb_pos at this stage: 747 * cb_pos += cb_max_ofs - cur_ofs; 748 */ 749 cur_ofs = cb_max_ofs; 750 } 751 } else if (vcn == start_vcn) { 752 /* We can't sleep so we need two stages. */ 753 unsigned int cur2_page = cur_page; 754 unsigned int cur_ofs2 = cur_ofs; 755 u8 *cb_pos2 = cb_pos; 756 757 ntfs_debug("Found uncompressed compression block."); 758 /* Uncompressed cb, copy it to the destination pages. */ 759 /* 760 * TODO: As a big optimization, we could detect this case 761 * before we read all the pages and use block_read_full_page() 762 * on all full pages instead (we still have to treat partial 763 * pages especially but at least we are getting rid of the 764 * synchronous io for the majority of pages. 765 * Or if we choose not to do the read-ahead/-behind stuff, we 766 * could just return block_read_full_page(pages[xpage]) as long 767 * as PAGE_CACHE_SIZE <= cb_size. 768 */ 769 if (cb_max_ofs) 770 cb_max_page--; 771 /* First stage: copy data into destination pages. */ 772 for (; cur_page < cb_max_page; cur_page++) { 773 page = pages[cur_page]; 774 if (page) 775 memcpy(page_address(page) + cur_ofs, cb_pos, 776 PAGE_CACHE_SIZE - cur_ofs); 777 cb_pos += PAGE_CACHE_SIZE - cur_ofs; 778 cur_ofs = 0; 779 if (cb_pos >= cb_end) 780 break; 781 } 782 /* If we have a partial final page, deal with it now. */ 783 if (cb_max_ofs && cb_pos < cb_end) { 784 page = pages[cur_page]; 785 if (page) 786 memcpy(page_address(page) + cur_ofs, cb_pos, 787 cb_max_ofs - cur_ofs); 788 cb_pos += cb_max_ofs - cur_ofs; 789 cur_ofs = cb_max_ofs; 790 } 791 /* We can sleep from now on, so drop lock. */ 792 spin_unlock(&ntfs_cb_lock); 793 /* Second stage: finalize pages. */ 794 for (; cur2_page < cb_max_page; cur2_page++) { 795 page = pages[cur2_page]; 796 if (page) { 797 /* 798 * If we are outside the initialized size, zero 799 * the out of bounds page range. 800 */ 801 handle_bounds_compressed_page(page, i_size, 802 initialized_size); 803 flush_dcache_page(page); 804 kunmap(page); 805 SetPageUptodate(page); 806 unlock_page(page); 807 if (cur2_page == xpage) 808 xpage_done = 1; 809 else 810 page_cache_release(page); 811 pages[cur2_page] = NULL; 812 } 813 cb_pos2 += PAGE_CACHE_SIZE - cur_ofs2; 814 cur_ofs2 = 0; 815 if (cb_pos2 >= cb_end) 816 break; 817 } 818 } else { 819 /* Compressed cb, decompress it into the destination page(s). */ 820 unsigned int prev_cur_page = cur_page; 821 822 ntfs_debug("Found compressed compression block."); 823 err = ntfs_decompress(pages, &cur_page, &cur_ofs, 824 cb_max_page, cb_max_ofs, xpage, &xpage_done, 825 cb_pos, cb_size - (cb_pos - cb), i_size, 826 initialized_size); 827 /* 828 * We can sleep from now on, lock already dropped by 829 * ntfs_decompress(). 830 */ 831 if (err) { 832 ntfs_error(vol->sb, "ntfs_decompress() failed in inode " 833 "0x%lx with error code %i. Skipping " 834 "this compression block.", 835 ni->mft_no, -err); 836 /* Release the unfinished pages. */ 837 for (; prev_cur_page < cur_page; prev_cur_page++) { 838 page = pages[prev_cur_page]; 839 if (page) { 840 flush_dcache_page(page); 841 kunmap(page); 842 unlock_page(page); 843 if (prev_cur_page != xpage) 844 page_cache_release(page); 845 pages[prev_cur_page] = NULL; 846 } 847 } 848 } 849 } 850 851 /* Release the buffer heads. */ 852 for (i = 0; i < nr_bhs; i++) 853 brelse(bhs[i]); 854 855 /* Do we have more work to do? */ 856 if (nr_cbs) 857 goto do_next_cb; 858 859 /* We no longer need the list of buffer heads. */ 860 kfree(bhs); 861 862 /* Clean up if we have any pages left. Should never happen. */ 863 for (cur_page = 0; cur_page < max_page; cur_page++) { 864 page = pages[cur_page]; 865 if (page) { 866 ntfs_error(vol->sb, "Still have pages left! " 867 "Terminating them with extreme " 868 "prejudice. Inode 0x%lx, page index " 869 "0x%lx.", ni->mft_no, page->index); 870 flush_dcache_page(page); 871 kunmap(page); 872 unlock_page(page); 873 if (cur_page != xpage) 874 page_cache_release(page); 875 pages[cur_page] = NULL; 876 } 877 } 878 879 /* We no longer need the list of pages. */ 880 kfree(pages); 881 882 /* If we have completed the requested page, we return success. */ 883 if (likely(xpage_done)) 884 return 0; 885 886 ntfs_debug("Failed. Returning error code %s.", err == -EOVERFLOW ? 887 "EOVERFLOW" : (!err ? "EIO" : "unknown error")); 888 return err < 0 ? err : -EIO; 889 890read_err: 891 ntfs_error(vol->sb, "IO error while reading compressed data."); 892 /* Release the buffer heads. */ 893 for (i = 0; i < nr_bhs; i++) 894 brelse(bhs[i]); 895 goto err_out; 896 897map_rl_err: 898 ntfs_error(vol->sb, "ntfs_map_runlist() failed. Cannot read " 899 "compression block."); 900 goto err_out; 901 902rl_err: 903 up_read(&ni->runlist.lock); 904 ntfs_error(vol->sb, "ntfs_rl_vcn_to_lcn() failed. Cannot read " 905 "compression block."); 906 goto err_out; 907 908getblk_err: 909 up_read(&ni->runlist.lock); 910 ntfs_error(vol->sb, "getblk() failed. Cannot read compression block."); 911 912err_out: 913 kfree(bhs); 914 for (i = cur_page; i < max_page; i++) { 915 page = pages[i]; 916 if (page) { 917 flush_dcache_page(page); 918 kunmap(page); 919 unlock_page(page); 920 if (i != xpage) 921 page_cache_release(page); 922 } 923 } 924 kfree(pages); 925 return -EIO; 926} 927