test_sparse_basic.c revision 348607
1/*- 2 * Copyright (c) 2010-2012 Michihiro NAKAJIMA 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25#include "test.h" 26__FBSDID("$FreeBSD$"); 27 28#ifdef HAVE_SYS_IOCTL_H 29#include <sys/ioctl.h> 30#endif 31#ifdef HAVE_SYS_PARAM_H 32#include <sys/param.h> 33#endif 34#ifdef HAVE_FCNTL_H 35#include <fcntl.h> 36#endif 37#ifdef HAVE_LIMITS_H 38#include <limits.h> 39#endif 40#ifdef HAVE_UNISTD_H 41#include <unistd.h> 42#endif 43#ifdef HAVE_LINUX_TYPES_H 44#include <linux/types.h> 45#endif 46#ifdef HAVE_LINUX_FIEMAP_H 47#include <linux/fiemap.h> 48#endif 49#ifdef HAVE_LINUX_FS_H 50#include <linux/fs.h> 51#endif 52 53/* The logic to compare sparse file data read from disk with the 54 * specification is a little involved. Set to 1 to have the progress 55 * dumped. */ 56#define DEBUG 0 57 58/* 59 * NOTE: On FreeBSD and Solaris, this test needs ZFS. 60 * You may perform this test as 61 * 'TMPDIR=<a directory on the ZFS> libarchive_test'. 62 */ 63 64struct sparse { 65 enum { DATA, HOLE, END } type; 66 size_t size; 67}; 68 69static void create_sparse_file(const char *, const struct sparse *); 70 71#if defined(__APPLE__) 72/* On APFS holes need to be at least 4096x4097 bytes */ 73#define MIN_HOLE 16781312 74#else 75/* Elsewhere we work with 4096*10 bytes */ 76#define MIN_HOLE 409600 77#endif 78 79#if defined(_WIN32) && !defined(__CYGWIN__) 80#include <winioctl.h> 81/* 82 * Create a sparse file on Windows. 83 */ 84 85#if !defined(PATH_MAX) 86#define PATH_MAX MAX_PATH 87#endif 88#if !defined(__BORLANDC__) 89#define getcwd _getcwd 90#endif 91 92static int 93is_sparse_supported(const char *path) 94{ 95 char root[MAX_PATH+1]; 96 char vol[MAX_PATH+1]; 97 char sys[MAX_PATH+1]; 98 DWORD flags; 99 BOOL r; 100 101 strncpy(root, path, sizeof(root)-1); 102 if (((root[0] >= 'c' && root[0] <= 'z') || 103 (root[0] >= 'C' && root[0] <= 'Z')) && 104 root[1] == ':' && 105 (root[2] == '\\' || root[2] == '/')) 106 root[3] = '\0'; 107 else 108 return (0); 109 assertEqualInt((r = GetVolumeInformation(root, vol, 110 sizeof(vol), NULL, NULL, &flags, sys, sizeof(sys))), 1); 111 return (r != 0 && (flags & FILE_SUPPORTS_SPARSE_FILES) != 0); 112} 113 114static void 115create_sparse_file(const char *path, const struct sparse *s) 116{ 117 char buff[1024]; 118 HANDLE handle; 119 DWORD dmy; 120 121 memset(buff, ' ', sizeof(buff)); 122 123 handle = CreateFileA(path, GENERIC_WRITE, 0, 124 NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, 125 NULL); 126 assert(handle != INVALID_HANDLE_VALUE); 127 assert(DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, 128 NULL, 0, &dmy, NULL) != 0); 129 130 size_t offsetSoFar = 0; 131 132 while (s->type != END) { 133 if (s->type == HOLE) { 134 LARGE_INTEGER fileOffset, beyondOffset, distanceToMove; 135 fileOffset.QuadPart = offsetSoFar; 136 beyondOffset.QuadPart = offsetSoFar + s->size; 137 distanceToMove.QuadPart = s->size; 138 139 FILE_ZERO_DATA_INFORMATION zeroInformation; 140 zeroInformation.FileOffset = fileOffset; 141 zeroInformation.BeyondFinalZero = beyondOffset; 142 143 DWORD bytesReturned; 144 assert(SetFilePointerEx(handle, distanceToMove, 145 NULL, FILE_CURRENT) != 0); 146 assert(SetEndOfFile(handle) != 0); 147 assert(DeviceIoControl(handle, FSCTL_SET_ZERO_DATA, &zeroInformation, 148 sizeof(FILE_ZERO_DATA_INFORMATION), NULL, 0, &bytesReturned, NULL) != 0); 149 } else { 150 DWORD w, wr; 151 size_t size; 152 153 size = s->size; 154 while (size) { 155 if (size > sizeof(buff)) 156 w = sizeof(buff); 157 else 158 w = (DWORD)size; 159 assert(WriteFile(handle, buff, w, &wr, NULL) != 0); 160 size -= wr; 161 } 162 } 163 offsetSoFar += s->size; 164 s++; 165 } 166 assertEqualInt(CloseHandle(handle), 1); 167} 168 169#else 170 171#if defined(HAVE_LINUX_FIEMAP_H) 172/* 173 * FIEMAP, which can detect 'hole' of a sparse file, has 174 * been supported from 2.6.28 175 */ 176 177static int 178is_sparse_supported_fiemap(const char *path) 179{ 180 const struct sparse sparse_file[] = { 181 /* This hole size is too small to create a sparse 182 * files for almost filesystem. */ 183 { HOLE, 1024 }, { DATA, 10240 }, 184 { END, 0 } 185 }; 186 int fd, r; 187 struct fiemap *fm; 188 char buff[1024]; 189 const char *testfile = "can_sparse"; 190 191 (void)path; /* UNUSED */ 192 memset(buff, 0, sizeof(buff)); 193 create_sparse_file(testfile, sparse_file); 194 fd = open(testfile, O_RDWR); 195 if (fd < 0) 196 return (0); 197 fm = (struct fiemap *)buff; 198 fm->fm_start = 0; 199 fm->fm_length = ~0ULL;; 200 fm->fm_flags = FIEMAP_FLAG_SYNC; 201 fm->fm_extent_count = (sizeof(buff) - sizeof(*fm))/ 202 sizeof(struct fiemap_extent); 203 r = ioctl(fd, FS_IOC_FIEMAP, fm); 204 close(fd); 205 unlink(testfile); 206 return (r >= 0); 207} 208 209#if !defined(SEEK_HOLE) || !defined(SEEK_DATA) 210static int 211is_sparse_supported(const char *path) 212{ 213 return is_sparse_supported_fiemap(path); 214} 215#endif 216#endif 217 218#if defined(_PC_MIN_HOLE_SIZE) 219 220/* 221 * FreeBSD and Solaris can detect 'hole' of a sparse file 222 * through lseek(HOLE) on ZFS. (UFS does not support yet) 223 */ 224 225static int 226is_sparse_supported(const char *path) 227{ 228 return (pathconf(path, _PC_MIN_HOLE_SIZE) > 0); 229} 230 231#elif defined(SEEK_HOLE) && defined(SEEK_DATA) 232 233static int 234is_sparse_supported(const char *path) 235{ 236 const struct sparse sparse_file[] = { 237 /* This hole size is too small to create a sparse 238 * files for almost filesystem. */ 239 { HOLE, 1024 }, { DATA, 10240 }, 240 { END, 0 } 241 }; 242 int fd, r; 243 const char *testfile = "can_sparse"; 244 245 (void)path; /* UNUSED */ 246 create_sparse_file(testfile, sparse_file); 247 fd = open(testfile, O_RDWR); 248 if (fd < 0) 249 return (0); 250 r = lseek(fd, 0, SEEK_HOLE); 251 close(fd); 252 unlink(testfile); 253#if defined(HAVE_LINUX_FIEMAP_H) 254 if (r < 0) 255 return (is_sparse_supported_fiemap(path)); 256#endif 257 return (r >= 0); 258} 259 260#elif !defined(HAVE_LINUX_FIEMAP_H) 261 262/* 263 * Other system may do not have the API such as lseek(HOLE), 264 * which detect 'hole' of a sparse file. 265 */ 266 267static int 268is_sparse_supported(const char *path) 269{ 270 (void)path; /* UNUSED */ 271 return (0); 272} 273 274#endif 275 276/* 277 * Create a sparse file on POSIX like system. 278 */ 279 280static void 281create_sparse_file(const char *path, const struct sparse *s) 282{ 283 char buff[1024]; 284 int fd; 285 size_t total_size = 0; 286 const struct sparse *cur = s; 287 288 memset(buff, ' ', sizeof(buff)); 289 assert((fd = open(path, O_CREAT | O_WRONLY, 0600)) != -1); 290 291 /* Handle holes at the end by extending the file */ 292 while (cur->type != END) { 293 total_size += cur->size; 294 ++cur; 295 } 296 assert(ftruncate(fd, total_size) != -1); 297 298 while (s->type != END) { 299 if (s->type == HOLE) { 300 assert(lseek(fd, s->size, SEEK_CUR) != (off_t)-1); 301 } else { 302 size_t w, size; 303 304 size = s->size; 305 while (size) { 306 if (size > sizeof(buff)) 307 w = sizeof(buff); 308 else 309 w = size; 310 assert(write(fd, buff, w) != (ssize_t)-1); 311 size -= w; 312 } 313 } 314 s++; 315 } 316 close(fd); 317} 318 319#endif 320 321/* 322 * Sparse test with directory traversals. 323 */ 324static void 325verify_sparse_file(struct archive *a, const char *path, 326 const struct sparse *sparse, int expected_holes) 327{ 328 struct archive_entry *ae; 329 const void *buff; 330 size_t bytes_read; 331 int64_t offset, expected_offset, last_offset; 332 int holes_seen = 0; 333 334 create_sparse_file(path, sparse); 335 assert((ae = archive_entry_new()) != NULL); 336 assertEqualIntA(a, ARCHIVE_OK, archive_read_disk_open(a, path)); 337 assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header2(a, ae)); 338 339 expected_offset = 0; 340 last_offset = 0; 341 while (ARCHIVE_OK == archive_read_data_block(a, &buff, &bytes_read, 342 &offset)) { 343 const char *start = buff; 344#if DEBUG 345 fprintf(stderr, "%s: bytes_read=%d offset=%d\n", path, (int)bytes_read, (int)offset); 346#endif 347 if (offset > last_offset) { 348 ++holes_seen; 349 } 350 /* Blocks entirely before the data we just read. */ 351 while (expected_offset + (int64_t)sparse->size < offset) { 352#if DEBUG 353 fprintf(stderr, " skipping expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 354#endif 355 /* Must be holes. */ 356 assert(sparse->type == HOLE); 357 expected_offset += sparse->size; 358 ++sparse; 359 } 360 /* Block that overlaps beginning of data */ 361 if (expected_offset < offset 362 && expected_offset + (int64_t)sparse->size <= offset + (int64_t)bytes_read) { 363 const char *end = (const char *)buff + (expected_offset - offset) + (size_t)sparse->size; 364#if DEBUG 365 fprintf(stderr, " overlapping hole expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 366#endif 367 /* Must be a hole, overlap must be filled with '\0' */ 368 if (assert(sparse->type == HOLE)) { 369 assertMemoryFilledWith(start, end - start, '\0'); 370 } 371 start = end; 372 expected_offset += sparse->size; 373 ++sparse; 374 } 375 /* Blocks completely contained in data we just read. */ 376 while (expected_offset + (int64_t)sparse->size <= offset + (int64_t)bytes_read) { 377 const char *end = (const char *)buff + (expected_offset - offset) + (size_t)sparse->size; 378 if (sparse->type == HOLE) { 379#if DEBUG 380 fprintf(stderr, " contained hole expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 381#endif 382 383 /* verify data corresponding to hole is '\0' */ 384 if (end > (const char *)buff + bytes_read) { 385 end = (const char *)buff + bytes_read; 386 } 387 assertMemoryFilledWith(start, end - start, '\0'); 388 start = end; 389 expected_offset += sparse->size; 390 ++sparse; 391 } else if (sparse->type == DATA) { 392#if DEBUG 393 fprintf(stderr, " contained data expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 394#endif 395 /* verify data corresponding to hole is ' ' */ 396 if (assert(expected_offset + sparse->size <= offset + bytes_read)) { 397 assert(start == (const char *)buff + (size_t)(expected_offset - offset)); 398 assertMemoryFilledWith(start, end - start, ' '); 399 } 400 start = end; 401 expected_offset += sparse->size; 402 ++sparse; 403 } else { 404 break; 405 } 406 } 407 /* Block that overlaps end of data */ 408 if (expected_offset < offset + (int64_t)bytes_read) { 409 const char *end = (const char *)buff + bytes_read; 410#if DEBUG 411 fprintf(stderr, " trailing overlap expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 412#endif 413 /* Must be a hole, overlap must be filled with '\0' */ 414 if (assert(sparse->type == HOLE)) { 415 assertMemoryFilledWith(start, end - start, '\0'); 416 } 417 } 418 last_offset = offset + bytes_read; 419 } 420 /* Count a hole at EOF? */ 421 if (last_offset < archive_entry_size(ae)) { 422 ++holes_seen; 423 } 424 425 /* Verify blocks after last read */ 426 while (sparse->type == HOLE) { 427 expected_offset += sparse->size; 428 ++sparse; 429 } 430 assert(sparse->type == END); 431 assertEqualInt(expected_offset, archive_entry_size(ae)); 432 433 failure(path); 434 assertEqualInt(holes_seen, expected_holes); 435 436 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 437 archive_entry_free(ae); 438} 439 440#if defined(_WIN32) && !defined(__CYGWIN__) 441#define close _close 442#define open _open 443#endif 444 445/* 446 * Sparse test without directory traversals. 447 */ 448static void 449verify_sparse_file2(struct archive *a, const char *path, 450 const struct sparse *sparse, int blocks, int preopen) 451{ 452 struct archive_entry *ae; 453 int fd; 454 455 (void)sparse; /* UNUSED */ 456 assert((ae = archive_entry_new()) != NULL); 457 archive_entry_set_pathname(ae, path); 458 if (preopen) 459 fd = open(path, O_RDONLY | O_BINARY); 460 else 461 fd = -1; 462 assertEqualIntA(a, ARCHIVE_OK, 463 archive_read_disk_entry_from_file(a, ae, fd, NULL)); 464 if (fd >= 0) 465 close(fd); 466 /* Verify the number of holes only, not its offset nor its 467 * length because those alignments are deeply dependence on 468 * its filesystem. */ 469 failure(path); 470 assertEqualInt(blocks, archive_entry_sparse_count(ae)); 471 archive_entry_free(ae); 472} 473 474static void 475test_sparse_whole_file_data() 476{ 477 struct archive_entry *ae; 478 int64_t offset; 479 int i; 480 481 assert((ae = archive_entry_new()) != NULL); 482 archive_entry_set_size(ae, 1024*10); 483 484 /* 485 * Add sparse block data up to the file size. 486 */ 487 offset = 0; 488 for (i = 0; i < 10; i++) { 489 archive_entry_sparse_add_entry(ae, offset, 1024); 490 offset += 1024; 491 } 492 493 failure("There should be no sparse"); 494 assertEqualInt(0, archive_entry_sparse_count(ae)); 495 archive_entry_free(ae); 496} 497 498DEFINE_TEST(test_sparse_basic) 499{ 500 char *cwd; 501 struct archive *a; 502 const char *skip_sparse_tests; 503 /* 504 * The alignment of the hole of sparse files deeply depends 505 * on filesystem. In my experience, sparse_file2 test with 506 * 204800 bytes hole size did not pass on ZFS and the result 507 * of that test seemed the size was too small, thus you should 508 * keep a hole size more than 409600 bytes to pass this test 509 * on all platform. 510 */ 511 const struct sparse sparse_file0[] = { 512 // 0 // 1024 513 { DATA, 1024 }, { HOLE, MIN_HOLE + 1638400 }, 514 // 2049024 // 2051072 515 { DATA, 2048 }, { HOLE, MIN_HOLE + 1638400 }, 516 // 4099072 // 4103168 517 { DATA, 4096 }, { HOLE, MIN_HOLE + 20070400 }, 518 // 24583168 // 24591360 519 { DATA, 8192 }, { HOLE, MIN_HOLE + 204390400 }, 520 // 229391360 // 229391361 521 { DATA, 1 }, { END, 0 } 522 }; 523 const struct sparse sparse_file1[] = { 524 { HOLE, MIN_HOLE }, { DATA, 1 }, 525 { HOLE, MIN_HOLE }, { DATA, 1 }, 526 { HOLE, MIN_HOLE }, { END, 0 } 527 }; 528 const struct sparse sparse_file2[] = { 529 { HOLE, MIN_HOLE }, { DATA, 1024 }, 530 { HOLE, MIN_HOLE + 409600 * 1 }, { DATA, 1024 }, 531 { HOLE, MIN_HOLE + 409600 * 2 }, { DATA, 1024 }, 532 { HOLE, MIN_HOLE + 409600 * 3 }, { DATA, 1024 }, 533 { HOLE, MIN_HOLE + 409600 * 4 }, { DATA, 1024 }, 534 { HOLE, MIN_HOLE + 409600 * 5 }, { DATA, 1024 }, 535 { HOLE, MIN_HOLE + 409600 * 6 }, { DATA, 1024 }, 536 { HOLE, MIN_HOLE + 409600 * 7 }, { DATA, 1024 }, 537 { HOLE, MIN_HOLE + 409600 * 8 }, { DATA, 1024 }, 538 { HOLE, MIN_HOLE + 409600 * 9}, { DATA, 1024 },/* 10 */ 539 { HOLE, MIN_HOLE }, { DATA, 1024 * 1 }, 540 { HOLE, MIN_HOLE + 409600 * 1 }, { DATA, 1024 * 2 }, 541 { HOLE, MIN_HOLE + 409600 * 2 }, { DATA, 1024 * 3 }, 542 { HOLE, MIN_HOLE + 409600 * 3 }, { DATA, 1024 * 4 }, 543 { HOLE, MIN_HOLE + 409600 * 4 }, { DATA, 1024 * 5 }, 544 { HOLE, MIN_HOLE + 409600 * 5 }, { DATA, 1024 * 6 }, 545 { HOLE, MIN_HOLE + 409600 * 6 }, { DATA, 1024 * 7 }, 546 { HOLE, MIN_HOLE + 409600 * 7 }, { DATA, 1024 * 8 }, 547 { HOLE, MIN_HOLE + 409600 * 8 }, { DATA, 1024 * 9 }, 548 { HOLE, MIN_HOLE + 409600 * 9}, { DATA, 1024 * 10},/* 20 */ 549 { END, 0 } 550 }; 551 const struct sparse sparse_file3[] = { 552 /* This hole size is too small to create a sparse file */ 553 { HOLE, 1 }, { DATA, 10240 }, 554 { HOLE, 1 }, { DATA, 10240 }, 555 { HOLE, 1 }, { DATA, 10240 }, 556 { END, 0 } 557 }; 558 559 /* 560 * Test for the case that sparse data indicates just the whole file 561 * data. 562 */ 563 test_sparse_whole_file_data(); 564 565 skip_sparse_tests = getenv("SKIP_TEST_SPARSE"); 566 if (skip_sparse_tests != NULL) { 567 skipping("Skipping sparse tests due to SKIP_TEST_SPARSE " 568 "environment variable"); 569 return; 570 } 571 572 /* Check if the filesystem where CWD on can 573 * report the number of the holes of a sparse file. */ 574#ifdef PATH_MAX 575 cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */ 576#else 577 cwd = getcwd(NULL, 0); 578#endif 579 if (!assert(cwd != NULL)) 580 return; 581 if (!is_sparse_supported(cwd)) { 582 free(cwd); 583 skipping("This filesystem or platform do not support " 584 "the reporting of the holes of a sparse file through " 585 "API such as lseek(HOLE)"); 586 return; 587 } 588 589 /* 590 * Get sparse data through directory traversals. 591 */ 592 assert((a = archive_read_disk_new()) != NULL); 593 594 verify_sparse_file(a, "file0", sparse_file0, 4); 595 verify_sparse_file(a, "file1", sparse_file1, 3); 596 verify_sparse_file(a, "file2", sparse_file2, 20); 597 /* Encoded non sparse; expect a data block but no sparse entries. */ 598 verify_sparse_file(a, "file3", sparse_file3, 0); 599 600 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 601 602 /* 603 * Get sparse data through archive_read_disk_entry_from_file(). 604 */ 605 assert((a = archive_read_disk_new()) != NULL); 606 607 verify_sparse_file2(a, "file0", sparse_file0, 5, 0); 608 verify_sparse_file2(a, "file0", sparse_file0, 5, 1); 609 610 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 611 free(cwd); 612} 613 614DEFINE_TEST(test_fully_sparse_files) 615{ 616 char *cwd; 617 struct archive *a; 618 const char *skip_sparse_tests; 619 620 const struct sparse sparse_file[] = { 621 { HOLE, MIN_HOLE }, { END, 0 } 622 }; 623 624 skip_sparse_tests = getenv("SKIP_TEST_SPARSE"); 625 if (skip_sparse_tests != NULL) { 626 skipping("Skipping sparse tests due to SKIP_TEST_SPARSE " 627 "environment variable"); 628 return; 629 } 630 631 /* Check if the filesystem where CWD on can 632 * report the number of the holes of a sparse file. */ 633#ifdef PATH_MAX 634 cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */ 635#else 636 cwd = getcwd(NULL, 0); 637#endif 638 if (!assert(cwd != NULL)) 639 return; 640 if (!is_sparse_supported(cwd)) { 641 free(cwd); 642 skipping("This filesystem or platform do not support " 643 "the reporting of the holes of a sparse file through " 644 "API such as lseek(HOLE)"); 645 return; 646 } 647 648 assert((a = archive_read_disk_new()) != NULL); 649 650 /* Fully sparse files are encoded with a zero-length "data" block. */ 651 verify_sparse_file(a, "file0", sparse_file, 1); 652 653 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 654 free(cwd); 655} 656